diff --git a/.asf.yaml b/.asf.yaml index 5c3b8cb98d964..c0492350a3638 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -37,7 +37,7 @@ github: enabled_merge_buttons: squash: true merge: false - rebase: true + rebase: false protected_branches: master: required_status_checks: @@ -58,6 +58,9 @@ github: required_signatures: false + # Requires all conversations on code to be resolved before a pull request can be merged. + required_conversation_resolution: true + # The following branch protections only ensure that force pushes are not allowed branch-1.15: {} branch-1.16: {} @@ -80,6 +83,7 @@ github: branch-2.10: {} branch-2.11: {} branch-3.0: {} + branch-3.1: {} notifications: commits: commits@pulsar.apache.org diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 43d323c96311c..fe6ade43b06a2 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -28,6 +28,8 @@ body: For suggestions or help, please consider: 1. [User Mail List](mailto:users@pulsar.apache.org) ([subscribe](mailto:users-subscribe@pulsar.apache.org)); 2. [Github Discussion](https://github.com/apache/pulsar/discussions). + + If you are reporting a security vulnerability, please instead follow the [security policy](https://pulsar.apache.org/en/security/). - type: checkboxes attributes: label: Search before asking @@ -37,11 +39,20 @@ body: - label: > I searched in the [issues](https://github.com/apache/pulsar/issues) and found nothing similar. required: true + - type: checkboxes + attributes: + label: Read release policy + description: > + Please check the [supported Pulsar versions in the release policy](https://pulsar.apache.org/contribute/release-policy/#supported-versions). + options: + - label: > + I understand that unsupported versions don't get bug fixes. I will attempt to reproduce the issue on a supported version of Pulsar client and Pulsar broker. + required: true - type: textarea attributes: label: Version description: > - Please provide the OS and Pulsar version you are using. If you are using the master branch, please provide the commit id. + Please provide the OS, Java version and Pulsar versions (client + broker) you are using. validations: required: true - type: textarea diff --git a/.github/ISSUE_TEMPLATE/flaky-test.yml b/.github/ISSUE_TEMPLATE/flaky-test.yml index 44ff64197822c..e7b57e1aeda87 100644 --- a/.github/ISSUE_TEMPLATE/flaky-test.yml +++ b/.github/ISSUE_TEMPLATE/flaky-test.yml @@ -18,7 +18,7 @@ name: Flaky test title: "Flaky-test: test_class.test_method" description: Report a flaky test failure -labels: [ "component/test", "flaky-tests" ] +labels: [ "area/test", "type/flaky-tests" ] body: - type: markdown attributes: diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 01ac26570b2d1..c148e8f61da14 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -18,13 +18,13 @@ Fixes #xyz -Master Issue: #xyz +Main Issue: #xyz PIP: #xyz - + ### Motivation diff --git a/.github/actions/clean-disk/action.yml b/.github/actions/clean-disk/action.yml index 8bcc5f1396802..d74c3f25fc64c 100644 --- a/.github/actions/clean-disk/action.yml +++ b/.github/actions/clean-disk/action.yml @@ -31,7 +31,7 @@ runs: directories=(/usr/local/lib/android /opt/ghc) if [[ "${{ inputs.mode }}" == "full" ]]; then # remove these directories only when mode is 'full' - directories+=(/usr/share/dotnet) + directories+=(/usr/share/dotnet /opt/hostedtoolcache/CodeQL) fi emptydir=/tmp/empty$$/ mkdir $emptydir diff --git a/.github/actions/tune-runner-vm/action.yml b/.github/actions/tune-runner-vm/action.yml index 402b9201dc260..ab0f65767a62d 100644 --- a/.github/actions/tune-runner-vm/action.yml +++ b/.github/actions/tune-runner-vm/action.yml @@ -53,8 +53,8 @@ runs: # tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt # commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds) # nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes) - sudo mount -o remount,nodiscard,commit=999999,barrier=0 / - sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt + sudo mount -o remount,nodiscard,commit=999999,barrier=0 / || true + sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt || true # disable discard/trim at device level since remount with nodiscard doesn't seem to be effective # https://www.spinics.net/lists/linux-ide/msg52562.html for i in /sys/block/sd*/queue/discard_max_bytes; do @@ -77,12 +77,6 @@ runs: # stop Azure Linux agent to save RAM sudo systemctl stop walinuxagent.service || true - # enable docker experimental mode which is - # required for using "docker build --squash" / "-Ddocker.squash=true" - daemon_json="$(sudo cat /etc/docker/daemon.json | jq '.experimental = true')" - echo "$daemon_json" | sudo tee /etc/docker/daemon.json - # restart docker daemon - sudo systemctl restart docker echo '::endgroup::' # show memory diff --git a/.github/actions/upload-coverage/action.yml b/.github/actions/upload-coverage/action.yml index a9706e77333cb..da580dd28fda5 100644 --- a/.github/actions/upload-coverage/action.yml +++ b/.github/actions/upload-coverage/action.yml @@ -51,7 +51,7 @@ runs: - name: "Upload to Codecov (attempt #1)" id: codecov-upload-1 if: steps.repo-check.outputs.passed == 'true' - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 continue-on-error: true with: flags: ${{ inputs.flags }} @@ -64,7 +64,7 @@ runs: - name: "Upload to Codecov (attempt #2)" id: codecov-upload-2 if: steps.codecov-upload-1.outcome == 'failure' - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 continue-on-error: true with: flags: ${{ inputs.flags }} @@ -77,7 +77,7 @@ runs: - name: "Upload to Codecov (attempt #3)" id: codecov-upload-3 if: steps.codecov-upload-2.outcome == 'failure' - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 # fail on last attempt continue-on-error: false with: diff --git a/.github/changes-filter.yaml b/.github/changes-filter.yaml index be6faa957887d..84ccdc8d68ce3 100644 --- a/.github/changes-filter.yaml +++ b/.github/changes-filter.yaml @@ -11,6 +11,9 @@ docs: - '.idea/**' - 'deployment/**' - 'wiki/**' + - 'pip/**' +java_non_tests: + - '**/src/main/java/**/*.java' tests: - added|modified: '**/src/test/java/**/*.java' need_owasp: diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 0000000000000..851dd2ed27219 --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +PIP: + - changed-files: + - any-glob-to-any-file: 'pip/**' diff --git a/.github/workflows/ci-documentbot.yml b/.github/workflows/ci-documentbot.yml index e859940080fe6..1006661b60d2a 100644 --- a/.github/workflows/ci-documentbot.yml +++ b/.github/workflows/ci-documentbot.yml @@ -36,7 +36,7 @@ jobs: if: (github.repository == 'apache/pulsar') && (github.event.pull_request.state == 'open') permissions: pull-requests: write - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Labeling uses: apache/pulsar-test-infra/docbot@master diff --git a/.github/workflows/ci-go-functions.yaml b/.github/workflows/ci-go-functions.yaml index f96a6d6586e6e..655503849b1c3 100644 --- a/.github/workflows/ci-go-functions.yaml +++ b/.github/workflows/ci-go-functions.yaml @@ -32,17 +32,17 @@ concurrency: cancel-in-progress: true env: - MAVEN_OPTS: -Xss1500k -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 jobs: preconditions: name: Preconditions - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 outputs: docs_only: ${{ steps.check_changes.outputs.docs_only }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Detect changed files id: changes @@ -72,20 +72,20 @@ jobs: needs: preconditions if: ${{ needs.preconditions.outputs.docs_only != 'true' }} name: Go ${{ matrix.go-version }} Functions style check - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: matrix: - go-version: [1.15, 1.16, 1.17] + go-version: ['1.21'] steps: - name: Check out code into the Go module directory - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} id: go @@ -93,7 +93,7 @@ jobs: - name: InstallTool run: | cd pulsar-function-go - wget -O - -q https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v1.18.0 + wget -O - -q https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v1.55.2 ./bin/golangci-lint --version - name: Build diff --git a/.github/workflows/ci-maven-cache-update.yaml b/.github/workflows/ci-maven-cache-update.yaml index 15fefaf3f1645..a673a30843417 100644 --- a/.github/workflows/ci-maven-cache-update.yaml +++ b/.github/workflows/ci-maven-cache-update.yaml @@ -42,14 +42,15 @@ on: - cron: '30 */12 * * *' env: - MAVEN_OPTS: -Xss1500k -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + JDK_DISTRIBUTION: corretto jobs: update-maven-dependencies-cache: name: Update Maven dependency cache for ${{ matrix.name }} env: JOB_NAME: Update Maven dependency cache for ${{ matrix.name }} - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} runs-on: ${{ matrix.runs-on }} timeout-minutes: 45 @@ -58,22 +59,22 @@ jobs: matrix: include: - name: all modules - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 cache_name: 'm2-dependencies-all' mvn_arguments: '' - name: all modules - macos - runs-on: macos-11 + runs-on: macos-latest cache_name: 'm2-dependencies-all' - name: core-modules - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 cache_name: 'm2-dependencies-core-modules' mvn_arguments: '-Pcore-modules,-main' steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -91,7 +92,7 @@ jobs: - name: Cache local Maven repository if: ${{ github.event_name == 'schedule' || steps.changes.outputs.poms == 'true' }} id: cache - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -103,10 +104,10 @@ jobs: # cache would be used as the starting point for a new cache entry - name: Set up JDK 17 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 if: ${{ (github.event_name == 'schedule' || steps.changes.outputs.poms == 'true') && steps.cache.outputs.cache-hit != 'true' }} with: - distribution: 'temurin' + distribution: ${{ env.JDK_DISTRIBUTION }} java-version: 17 - name: Download dependencies diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 06edbae51adde..a1c6dd594d3a2 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -24,7 +24,9 @@ on: workflow_dispatch: env: - MAVEN_OPTS: -Xss1500k -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + JDK_DISTRIBUTION: corretto + NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }} jobs: run-owasp-dependency-check: @@ -32,67 +34,98 @@ jobs: name: Check ${{ matrix.branch }} env: JOB_NAME: Check ${{ matrix.branch }} - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - runs-on: ubuntu-20.04 - timeout-minutes: 45 + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + runs-on: ubuntu-22.04 + timeout-minutes: 75 strategy: fail-fast: false + max-parallel: 1 matrix: include: - branch: master + - branch: branch-3.3 + - branch: branch-3.2 - branch: branch-3.0 - - branch: branch-2.11 - - branch: branch-2.10 - jdk: 11 - - branch: branch-2.9 - jdk: 11 - - branch: branch-2.8 - jdk: 11 steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ matrix.branch }} - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Cache local Maven repository - uses: actions/cache@v3 + - name: Restore Maven repository cache + uses: actions/cache/restore@v4 timeout-minutes: 5 with: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-owasp-${{ hashFiles('**/pom.xml') }} + key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} restore-keys: | - ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK ${{ matrix.jdk || '17' }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: - distribution: 'temurin' + distribution: ${{ env.JDK_DISTRIBUTION }} java-version: ${{ matrix.jdk || '17' }} - name: run install by skip tests - run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true + run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true -DnarPluginPhase=none -pl '!distribution/io,!distribution/offloaders' + + - name: OWASP cache key weeknum + id: get-weeknum + run: | + echo "weeknum=$(date -u +"%Y-%U")" >> $GITHUB_OUTPUT + shell: bash + + - name: Restore OWASP Dependency Check data + id: restore-owasp-dependency-check-data + uses: actions/cache/restore@v4 + timeout-minutes: 5 + with: + path: ~/.m2/repository/org/owasp/dependency-check-data + key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }} + enableCrossOsArchive: true + restore-keys: | + owasp-dependency-check-data- + + - name: Update OWASP Dependency Check data + id: update-owasp-dependency-check-data + if: ${{ matrix.branch == 'master' && (steps.restore-owasp-dependency-check-data.outputs.cache-hit != 'true' || steps.restore-owasp-dependency-check-data.outputs.cache-matched-key != steps.restore-owasp-dependency-check-data.outputs.cache-primary-key) }} + run: mvn -B -ntp -Powasp-dependency-check initialize -pl . dependency-check:update-only + + - name: Save OWASP Dependency Check data + if: ${{ steps.update-owasp-dependency-check-data.outcome == 'success' }} + uses: actions/cache/save@v4 + timeout-minutes: 5 + with: + path: ~/.m2/repository/org/owasp/dependency-check-data + key: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-primary-key }} + enableCrossOsArchive: true - name: run OWASP Dependency Check for distribution/server (-DfailBuildOnAnyVulnerability=true) run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/server -DfailBuildOnAnyVulnerability=true - - name: run OWASP Dependency Check for distribution/offloaders, distribution/io and pulsar-sql/presto-distribution - run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/offloaders,distribution/io,pulsar-sql/presto-distribution + - name: run OWASP Dependency Check for offloaders/tiered-storage and pulsar-io connectors (-DfailOnError=false) + if: ${{ !cancelled() }} + run: | + mvnprojects=$(mvn -B -ntp -Dscan=false initialize \ + | grep -- "-< .* >-" \ + | sed -E 's/.*-< (.*) >-.*/\1/' \ + | grep -E 'pulsar-io-|tiered-storage-|offloader' \ + | tr '\n' ',' | sed 's/,$/\n/' ) + set -xe + mvn --fail-at-end -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -DfailOnError=false -pl "${mvnprojects}" - name: Upload OWASP Dependency Check reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: owasp-dependency-check-reports-${{ matrix.branch }} path: | - distribution/server/target/dependency-check-report.html - distribution/offloaders/target/dependency-check-report.html - distribution/io/target/dependency-check-report.html - pulsar-sql/presto-distribution/target/dependency-check-report.html + **/target/dependency-check-report.html \ No newline at end of file diff --git a/.github/workflows/ci-pulsarbot.yaml b/.github/workflows/ci-pulsarbot.yaml index 157d668e6cdf8..4ea83404856d2 100644 --- a/.github/workflows/ci-pulsarbot.yaml +++ b/.github/workflows/ci-pulsarbot.yaml @@ -24,7 +24,7 @@ on: jobs: pulsarbot: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 10 if: github.event_name == 'issue_comment' && contains(github.event.comment.body, '/pulsarbot') steps: diff --git a/.github/workflows/ci-semantic-pull-request.yml b/.github/workflows/ci-semantic-pull-request.yml index 28246eb2b819c..15ac85090243c 100644 --- a/.github/workflows/ci-semantic-pull-request.yml +++ b/.github/workflows/ci-semantic-pull-request.yml @@ -34,7 +34,7 @@ jobs: name: Check pull request title runs-on: ubuntu-latest steps: - - uses: amannn/action-semantic-pull-request@v5.0.2 + - uses: amannn/action-semantic-pull-request@v5.5.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -53,7 +53,6 @@ jobs: # io -> Pulsar Connectors # offload -> tiered storage # sec -> security - # sql -> Pulsar Trino Plugin # txn -> transaction # ws -> websocket # ml -> managed ledger @@ -79,12 +78,12 @@ jobs: schema sec site - sql storage test txn ws zk + pip # The pull request's title should be fulfilled the following pattern: # # [][] diff --git a/.github/workflows/ci-stale-issue-pr.yaml b/.github/workflows/ci-stale-issue-pr.yaml deleted file mode 100644 index 48ed5246001c9..0000000000000 --- a/.github/workflows/ci-stale-issue-pr.yaml +++ /dev/null @@ -1,17 +0,0 @@ -name: 'Stale issues and PRs' -on: - schedule: - - cron: '30 1 * * *' - -jobs: - stale: - runs-on: ubuntu-20.04 - steps: - - uses: actions/stale@v4 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-issue-message: 'The issue had no activity for 30 days, mark with Stale label.' - stale-pr-message: 'The pr had no activity for 30 days, mark with Stale label.' - days-before-stale: 30 - days-before-close: -1 - operations-per-run: 700 diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml new file mode 100644 index 0000000000000..16430d19f3de8 --- /dev/null +++ b/.github/workflows/codeql.yaml @@ -0,0 +1,88 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: "CodeQL" + +on: + push: + branches: [ 'master' ] + schedule: + - cron: '27 21 * * 4' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +env: + JDK_DISTRIBUTION: corretto + +jobs: + analyze: + # only run scheduled analysis in apache/pulsar repository + if: ${{ (github.event_name == 'schedule' && github.repository == 'apache/pulsar') || github.event_name != 'schedule' }} + name: Analyze + runs-on: 'ubuntu-latest' + timeout-minutes: 360 + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + language: [ 'java-kotlin' ] + + steps: + - name: Cache local Maven repository + uses: actions/cache@v4 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK + uses: actions/setup-java@v4 + with: + distribution: ${{ env.JDK_DISTRIBUTION }} + java-version: 17 + + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + + - name: Build Java code + run: | + mvn -B -ntp -Pcore-modules,-main install -DskipTests -Dlicense.skip=true -Drat.skip=true -Dcheckstyle.skip=true + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml new file mode 100644 index 0000000000000..f10e61c8fd20e --- /dev/null +++ b/.github/workflows/labeler.yml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: "Pull Request Labeler" +on: + - pull_request_target + +jobs: + triage: + permissions: + contents: read + pull-requests: write + runs-on: ubuntu-latest + steps: + - uses: actions/labeler@v5 diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml index 555ebdb17292f..bfc5140943172 100644 --- a/.github/workflows/pulsar-ci-flaky.yaml +++ b/.github/workflows/pulsar-ci-flaky.yaml @@ -22,40 +22,93 @@ on: pull_request: branches: - master + - branch-* + - pulsar-* schedule: + # scheduled job with JDK 17 - cron: '0 12 * * *' + # scheduled job with JDK 21 + # if cron expression is changed, make sure to update the expression in jdk_major_version step in preconditions job + - cron: '0 6 * * *' workflow_dispatch: inputs: collect_coverage: description: 'Collect test coverage and upload to Codecov' required: true - default: 'true' + type: boolean + default: true + jdk_major_version: + description: 'JDK major version to use for the build' + required: true + type: choice + options: + - '17' + - '21' + default: '17' + trace_test_resource_cleanup: + description: 'Collect thread & heap information before exiting a test JVM. When set to "on", thread dump and heap histogram will be collected. When set to "full", a heap dump will also be collected.' + required: true + type: choice + options: + - 'off' + - 'on' + - 'full' + default: 'off' + thread_leak_detector_wait_millis: + description: 'Duration in ms to wait for threads to exit in thread leak detection between test classes. It is necessary to wait for threads to complete before they are determined to be leaked threads.' + required: true + type: number + default: 10000 concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '' }} cancel-in-progress: true env: - MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job # it's possible to rerun individual failed jobs when the build artifacts are available # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR ARTIFACT_RETENTION_DAYS: 3 + JDK_DISTRIBUTION: corretto jobs: preconditions: name: Preconditions - runs-on: ubuntu-20.04 - if: (github.event_name != 'schedule') || (github.repository == 'apache/pulsar') + runs-on: ubuntu-22.04 outputs: docs_only: ${{ steps.check_changes.outputs.docs_only }} changed_tests: ${{ steps.changes.outputs.tests_files }} + need_owasp: ${{ steps.changes.outputs.need_owasp }} collect_coverage: ${{ steps.check_coverage.outputs.collect_coverage }} + jdk_major_version: ${{ steps.jdk_major_version.outputs.jdk_major_version }} + steps: + - name: Cancel scheduled jobs in forks by default + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'schedule' }} + uses: actions/github-script@v6 + with: + script: | + await github.rest.actions.cancelWorkflowRun({owner: context.repo.owner, repo: context.repo.repo, run_id: context.runId}); + process.exit(1); + + - name: Select JDK major version + id: jdk_major_version + run: | + # use JDK 21 for the scheduled build with cron expression '0 6 * * *' + if [[ "${{ github.event_name == 'schedule' && github.event.schedule == '0 6 * * *' && 'true' || 'false' }}" == "true" ]]; then + echo "jdk_major_version=21" >> $GITHUB_OUTPUT + exit 0 + fi + # use JDK 17 for build unless overridden with workflow_dispatch input + echo "jdk_major_version=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '17'}}" >> $GITHUB_OUTPUT + - name: checkout - uses: actions/checkout@v3 + if: ${{ github.event_name == 'pull_request' }} + uses: actions/checkout@v4 - name: Detect changed files + if: ${{ github.event_name == 'pull_request' }} id: changes uses: apache/pulsar-test-infra/paths-filter@master with: @@ -63,6 +116,7 @@ jobs: list-files: csv - name: Check changed files + if: ${{ github.event_name == 'pull_request' }} id: check_changes run: | if [[ "${GITHUB_EVENT_NAME}" != "schedule" && "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]]; then @@ -94,17 +148,26 @@ jobs: env: JOB_NAME: Flaky tests suite COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}" - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - runs-on: ubuntu-20.04 + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} + TRACE_TEST_RESOURCE_CLEANUP: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.trace_test_resource_cleanup || 'off' }} + TRACE_TEST_RESOURCE_CLEANUP_DIR: ${{ github.workspace }}/target/trace-test-resource-cleanup + THREAD_LEAK_DETECTOR_WAIT_MILLIS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.thread_leak_detector_wait_millis || 10000 }} + THREAD_LEAK_DETECTOR_DIR: ${{ github.workspace }}/target/thread-leak-dumps + runs-on: ubuntu-22.04 timeout-minutes: 100 if: ${{ needs.preconditions.outputs.docs_only != 'true' }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm + - name: Clean Disk when tracing test resource cleanup + if: ${{ env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }} + uses: ./.github/actions/clean-disk + - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} @@ -114,7 +177,7 @@ jobs: limit-access-to-actor: true - name: Cache local Maven repository - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -124,11 +187,11 @@ jobs: restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules- - - name: Set up JDK 17 - uses: actions/setup-java@v3 + - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} + uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: 17 + distribution: ${{ env.JDK_DISTRIBUTION }} + java-version: ${{ env.CI_JDK_MAJOR_VERSION }} - name: Build core-modules run: | @@ -146,8 +209,24 @@ jobs: if: ${{ always() }} uses: ./.github/actions/copy-test-reports + - name: Publish Test Report + uses: apache/pulsar-test-infra/action-junit-report@master + if: ${{ always() }} + with: + report_paths: 'test-reports/TEST-*.xml' + annotate_only: 'true' + + - name: Report detected thread leaks + if: ${{ always() }} + run: | + if [ -d "$THREAD_LEAK_DETECTOR_DIR" ]; then + cd "$THREAD_LEAK_DETECTOR_DIR" + cat threadleak*.txt | awk '/^Summary:/ {print "::warning::" $0 "\n"; next} {print}' + fi + - name: Create Jacoco reports if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} + continue-on-error: true run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report cd $GITHUB_WORKSPACE/target @@ -155,38 +234,39 @@ jobs: - name: Upload Jacoco report files to build artifacts if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: Jacoco-coverage-report-flaky path: target/jacoco_test_coverage_report_flaky.zip retention-days: 3 + if-no-files-found: ignore - name: Upload to Codecov if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} uses: ./.github/actions/upload-coverage + continue-on-error: true with: flags: unittests - - name: Publish Test Report - uses: apache/pulsar-test-infra/action-junit-report@master - if: ${{ always() }} - with: - report_paths: 'test-reports/TEST-*.xml' - annotate_only: 'true' - - name: Upload Surefire reports - uses: actions/upload-artifact@v3 - if: ${{ !success() }} + uses: actions/upload-artifact@v4 + if: ${{ !success() || env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }} with: name: Unit-BROKER_FLAKY-surefire-reports path: surefire-reports retention-days: 7 + if-no-files-found: ignore - - name: Upload possible heap dump - uses: actions/upload-artifact@v3 + - name: Upload possible heap dump, core dump or crash files + uses: actions/upload-artifact@v4 if: ${{ always() }} with: - name: Unit-BROKER_FLAKY-heapdump - path: /tmp/*.hprof + name: Unit-BROKER_FLAKY-dumps + path: | + /tmp/*.hprof + **/hs_err_*.log + **/core.* + ${{ env.TRACE_TEST_RESOURCE_CLEANUP_DIR }}/* + ${{ env.THREAD_LEAK_DETECTOR_DIR }}/* retention-days: 7 - if-no-files-found: ignore + if-no-files-found: ignore \ No newline at end of file diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index b92599581cd83..091dab25ec696 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -22,42 +22,93 @@ on: pull_request: branches: - master + - branch-* + - pulsar-* schedule: + # scheduled job with JDK 21 - cron: '0 12 * * *' + # scheduled job with JDK 17 + # if cron expression is changed, make sure to update the expression in jdk_major_version step in preconditions job + - cron: '0 6 * * *' workflow_dispatch: inputs: collect_coverage: description: 'Collect test coverage and upload to Codecov' required: true - default: 'true' + type: boolean + default: true + jdk_major_version: + description: 'JDK major version to use for the build' + required: true + type: choice + options: + - '17' + - '21' + default: '21' + trace_test_resource_cleanup: + description: 'Collect thread & heap information before exiting a test JVM. When set to "on", thread dump and heap histogram will be collected. When set to "full", a heap dump will also be collected.' + required: true + type: choice + options: + - 'off' + - 'on' + - 'full' + default: 'off' + thread_leak_detector_wait_millis: + description: 'Duration in ms to wait for threads to exit in thread leak detection between test classes. It is necessary to wait for threads to complete before they are determined to be leaked threads.' + required: true + type: number + default: 10000 concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '' }} cancel-in-progress: true env: - MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job # it's possible to rerun individual failed jobs when the build artifacts are available # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR ARTIFACT_RETENTION_DAYS: 3 + JDK_DISTRIBUTION: corretto jobs: preconditions: name: Preconditions - runs-on: ubuntu-20.04 - if: (github.event_name != 'schedule') || (github.repository == 'apache/pulsar') + runs-on: ubuntu-22.04 outputs: docs_only: ${{ steps.check_changes.outputs.docs_only }} changed_tests: ${{ steps.changes.outputs.tests_files }} need_owasp: ${{ steps.changes.outputs.need_owasp }} collect_coverage: ${{ steps.check_coverage.outputs.collect_coverage }} - + jdk_major_version: ${{ steps.jdk_major_version.outputs.jdk_major_version }} + java_non_tests: ${{ steps.changes.outputs.java_non_tests }} steps: + - name: Cancel scheduled jobs in forks by default + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'schedule' }} + uses: actions/github-script@v6 + with: + script: | + await github.rest.actions.cancelWorkflowRun({owner: context.repo.owner, repo: context.repo.repo, run_id: context.runId}); + process.exit(1); + + - name: Select JDK major version + id: jdk_major_version + run: | + # use JDK 17 for the scheduled build with cron expression '0 6 * * *' + if [[ "${{ github.event_name == 'schedule' && github.event.schedule == '0 6 * * *' && 'true' || 'false' }}" == "true" ]]; then + echo "jdk_major_version=17" >> $GITHUB_OUTPUT + exit 0 + fi + # use JDK 21 for build unless overridden with workflow_dispatch input + echo "jdk_major_version=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '21'}}" >> $GITHUB_OUTPUT + - name: checkout - uses: actions/checkout@v3 + if: ${{ github.event_name == 'pull_request' }} + uses: actions/checkout@v4 - name: Detect changed files + if: ${{ github.event_name == 'pull_request' }} id: changes uses: apache/pulsar-test-infra/paths-filter@master with: @@ -65,6 +116,7 @@ jobs: list-files: csv - name: Check changed files + if: ${{ github.event_name == 'pull_request' }} id: check_changes run: | if [[ "${GITHUB_EVENT_NAME}" != "schedule" && "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]]; then @@ -95,13 +147,14 @@ jobs: name: Build and License check env: JOB_NAME: Build and License check - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - runs-on: ubuntu-20.04 + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} + runs-on: ubuntu-22.04 timeout-minutes: 60 if: ${{ needs.preconditions.outputs.docs_only != 'true' }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -115,7 +168,7 @@ jobs: limit-access-to-actor: true - name: Cache local Maven repository - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -125,11 +178,11 @@ jobs: restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules- - - name: Set up JDK 17 - uses: actions/setup-java@v3 + - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} + uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: 17 + distribution: ${{ env.JDK_DISTRIBUTION }} + java-version: ${{ env.CI_JDK_MAJOR_VERSION }} - name: Check source code license headers run: mvn -B -T 8 -ntp initialize apache-rat:check license:check @@ -171,8 +224,13 @@ jobs: env: JOB_NAME: CI - Unit - ${{ matrix.name }} COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}" - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - runs-on: ubuntu-20.04 + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} + TRACE_TEST_RESOURCE_CLEANUP: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.trace_test_resource_cleanup || 'off' }} + TRACE_TEST_RESOURCE_CLEANUP_DIR: ${{ github.workspace }}/target/trace-test-resource-cleanup + THREAD_LEAK_DETECTOR_WAIT_MILLIS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.thread_leak_detector_wait_millis || 10000 }} + THREAD_LEAK_DETECTOR_DIR: ${{ github.workspace }}/target/thread-leak-dumps + runs-on: ubuntu-22.04 timeout-minutes: ${{ matrix.timeout || 60 }} needs: ['preconditions', 'build-and-license-check'] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} @@ -189,6 +247,8 @@ jobs: group: BROKER_GROUP_2 - name: Brokers - Broker Group 3 group: BROKER_GROUP_3 + - name: Brokers - Broker Group 4 + group: BROKER_GROUP_4 - name: Brokers - Client Api group: BROKER_CLIENT_API - name: Brokers - Client Impl @@ -198,16 +258,26 @@ jobs: - name: Pulsar IO group: PULSAR_IO timeout: 75 + - name: Pulsar IO - Elastic Search + group: PULSAR_IO_ELASTIC + - name: Pulsar IO - Kafka Connect Adaptor + group: PULSAR_IO_KAFKA_CONNECT - name: Pulsar Client group: CLIENT + - name: Pulsar Metadata + group: METADATA steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm + - name: Clean Disk when tracing test resource cleanup + if: ${{ env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }} + uses: ./.github/actions/clean-disk + - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} @@ -217,7 +287,7 @@ jobs: limit-access-to-actor: true - name: Cache Maven dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -227,11 +297,11 @@ jobs: restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules- - - name: Set up JDK ${{ matrix.jdk || '17' }} - uses: actions/setup-java@v3 + - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} + uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: ${{ matrix.jdk || '17' }} + distribution: ${{ env.JDK_DISTRIBUTION }} + java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master @@ -269,16 +339,24 @@ jobs: report_paths: 'test-reports/TEST-*.xml' annotate_only: 'true' + - name: Report detected thread leaks + if: ${{ always() }} + run: | + if [ -d "$THREAD_LEAK_DETECTOR_DIR" ]; then + cd "$THREAD_LEAK_DETECTOR_DIR" + cat threadleak*.txt | awk '/^Summary:/ {print "::warning::" $0 "\n"; next} {print}' + fi + - name: Upload Surefire reports - uses: actions/upload-artifact@v3 - if: ${{ !success() }} + uses: actions/upload-artifact@v4 + if: ${{ !success() || env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }} with: name: Unit-${{ matrix.group }}-surefire-reports path: surefire-reports retention-days: 7 - name: Upload possible heap dump, core dump or crash files - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ always() }} with: name: Unit-${{ matrix.group }}-dumps @@ -286,6 +364,8 @@ jobs: /tmp/*.hprof **/hs_err_*.log **/core.* + ${{ env.TRACE_TEST_RESOURCE_CLEANUP_DIR }}/* + ${{ env.THREAD_LEAK_DETECTOR_DIR }}/* retention-days: 7 if-no-files-found: ignore @@ -300,14 +380,16 @@ jobs: unit-tests-upload-coverage: name: CI - Unit - Upload Coverage - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 30 needs: ['preconditions', 'unit-tests'] + env: + CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -321,7 +403,7 @@ jobs: limit-access-to-actor: true - name: Cache Maven dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -331,11 +413,11 @@ jobs: restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules- - - name: Set up JDK ${{ matrix.jdk || '17' }} - uses: actions/setup-java@v3 + - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} + uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: ${{ matrix.jdk || '17' }} + distribution: ${{ env.JDK_DISTRIBUTION }} + java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master @@ -354,7 +436,7 @@ jobs: zip -qr jacoco_test_coverage_report_unittests.zip jacoco_test_coverage_report || true - name: Upload Jacoco report files to build artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: Jacoco-coverage-report-unittests path: target/jacoco_test_coverage_report_unittests.zip @@ -379,15 +461,23 @@ jobs: pulsar-java-test-image: name: Build Pulsar java-test-image docker image - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 60 needs: ['preconditions', 'build-and-license-check'] if: ${{ needs.preconditions.outputs.docs_only != 'true'}} + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + - linux/arm64 env: - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} + IMAGE_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -401,7 +491,7 @@ jobs: limit-access-to-actor: true - name: Cache Maven dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -411,11 +501,11 @@ jobs: restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules- - - name: Set up JDK 17 - uses: actions/setup-java@v3 + - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} + uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: 17 + distribution: ${{ env.JDK_DISTRIBUTION }} + java-version: ${{ env.CI_JDK_MAJOR_VERSION }} - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master @@ -425,19 +515,21 @@ jobs: cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - - name: Pick ubuntu mirror for the docker image build - run: | - # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh pick_ubuntu_mirror + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 - - name: Build java-test-image docker image + - name: Build java-test-image docker image - ${{ matrix.platform }} run: | # build docker image - mvn -B -am -pl tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \ - -Dmaven.test.skip=true -Ddocker.squash=true -DskipSourceReleaseAssembly=true \ + DOCKER_CLI_EXPERIMENTAL=enabled mvn -B -am -pl docker/pulsar,tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \ + -Ddocker.platforms=${{ matrix.platform }} \ + -Dmaven.test.skip=true -DskipSourceReleaseAssembly=true \ -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true - name: save docker image apachepulsar/java-test-image:latest to Github artifact cache + if: ${{ matrix.platform == 'linux/amd64' }} run: | $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/java-test-image:latest pulsar-java-test-image @@ -451,14 +543,15 @@ jobs: integration-tests: name: CI - Integration - ${{ matrix.name }} - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: ${{ matrix.timeout || 60 }} needs: ['preconditions', 'pulsar-java-test-image'] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} env: JOB_NAME: CI - Integration - ${{ matrix.name }} PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} strategy: fail-fast: false matrix: @@ -473,6 +566,10 @@ jobs: - name: Messaging group: MESSAGING + - name: LoadBalance + group: LOADBALANCE + no_coverage: true + - name: Shade on Java 8 group: SHADE_RUN runtime_jdk: 8 @@ -487,6 +584,13 @@ jobs: - name: Shade on Java 17 group: SHADE_RUN + runtime_jdk: 17 + setup: ./build/run_integration_group.sh SHADE_BUILD + no_coverage: true + + - name: Shade on Java 21 + group: SHADE_RUN + runtime_jdk: 21 setup: ./build/run_integration_group.sh SHADE_BUILD no_coverage: true @@ -496,9 +600,12 @@ jobs: - name: Transaction group: TRANSACTION + - name: Metrics + group: METRICS + steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -512,7 +619,7 @@ jobs: limit-access-to-actor: true - name: Cache Maven dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -522,11 +629,11 @@ jobs: restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules- - - name: Set up JDK 17 - uses: actions/setup-java@v3 + - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} + uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: 17 + distribution: ${{ env.JDK_DISTRIBUTION }} + java-version: ${{ env.CI_JDK_MAJOR_VERSION }} - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master @@ -547,10 +654,10 @@ jobs: ${{ matrix.setup }} - name: Set up runtime JDK ${{ matrix.runtime_jdk }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 if: ${{ matrix.runtime_jdk }} with: - distribution: 'temurin' + distribution: ${{ env.JDK_DISTRIBUTION }} java-version: ${{ matrix.runtime_jdk }} - name: Run integration test group '${{ matrix.group }}' @@ -580,7 +687,7 @@ jobs: annotate_only: 'true' - name: Upload Surefire reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ !success() }} with: name: Integration-${{ matrix.group }}-surefire-reports @@ -588,7 +695,7 @@ jobs: retention-days: 7 - name: Upload container logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ !success() }} continue-on-error: true with: @@ -606,15 +713,16 @@ jobs: integration-tests-upload-coverage: name: CI - Integration - Upload Coverage - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 30 needs: ['preconditions', 'integration-tests'] if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} env: PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest + CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -628,7 +736,7 @@ jobs: limit-access-to-actor: true - name: Cache Maven dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -638,11 +746,11 @@ jobs: restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules- - - name: Set up JDK 17 - uses: actions/setup-java@v3 + - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} + uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: 17 + distribution: ${{ env.JDK_DISTRIBUTION }} + java-version: ${{ env.CI_JDK_MAJOR_VERSION }} - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master @@ -666,7 +774,7 @@ jobs: zip -qr jacoco_test_coverage_report_inttests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true - name: Upload Jacoco report files to build artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: Jacoco-coverage-report-inttests path: target/jacoco_test_coverage_report_inttests.zip @@ -691,7 +799,7 @@ jobs: delete-integration-test-docker-image-artifact: name: "Delete integration test docker image artifact" - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 10 needs: [ 'preconditions', @@ -701,7 +809,7 @@ jobs: if: ${{ needs.preconditions.outputs.docs_only != 'true' }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -715,15 +823,17 @@ jobs: pulsar-test-latest-version-image: name: Build Pulsar docker image - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 60 needs: ['preconditions', 'build-and-license-check'] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} env: - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} + IMAGE_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -738,9 +848,11 @@ jobs: - name: Clean Disk uses: ./.github/actions/clean-disk + with: + mode: full - name: Cache local Maven repository - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -751,11 +863,11 @@ jobs: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} ${{ runner.os }}-m2-dependencies-core-modules- - - name: Set up JDK 17 - uses: actions/setup-java@v3 + - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} + uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: 17 + distribution: ${{ env.JDK_DISTRIBUTION }} + java-version: ${{ env.CI_JDK_MAJOR_VERSION }} - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master @@ -765,24 +877,38 @@ jobs: cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - - name: Pick ubuntu mirror for the docker image build - run: | - # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh pick_ubuntu_mirror - - name: Build latest-version-image docker image run: | # build docker image - # include building of Pulsar SQL, Connectors, Offloaders and server distros - mvn -B -am -pl pulsar-sql/presto-distribution,distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \ - -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \ - -Pmain,docker -Dmaven.test.skip=true -Ddocker.squash=true \ + # include building of Connectors, Offloaders and server distros + DOCKER_CLI_EXPERIMENTAL=enabled mvn -B -am -pl distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \ + -Pmain,docker -Dmaven.test.skip=true \ -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true # check full build artifacts licenses - name: Check binary licenses run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz && src/check-binary-license.sh ./distribution/shell/target/apache-pulsar-shell-*-bin.tar.gz + - name: Run Trivy container scan + id: trivy_scan + uses: aquasecurity/trivy-action@master + if: ${{ github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }} + continue-on-error: true + with: + image-ref: "apachepulsar/pulsar:latest" + scanners: vuln + severity: CRITICAL,HIGH,MEDIUM,LOW + limit-severities-for-sarif: true + format: 'sarif' + output: 'trivy-results.sarif' + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v3 + if: ${{ steps.trivy_scan.outcome == 'success' && github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }} + continue-on-error: true + with: + sarif_file: 'trivy-results.sarif' + - name: Clean up disk space run: | # release disk space since saving docker image consumes local disk space @@ -821,14 +947,15 @@ jobs: system-tests: name: CI - System - ${{ matrix.name }} - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 60 needs: ['preconditions', 'pulsar-test-latest-version-image'] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} env: JOB_NAME: CI - System - ${{ matrix.name }} PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} strategy: fail-fast: false matrix: @@ -853,17 +980,19 @@ jobs: - name: Pulsar IO group: PULSAR_IO - - - name: Sql - group: SQL + clean_disk: true steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm + - name: Clean Disk when needed + if: ${{ matrix.clean_disk }} + uses: ./.github/actions/clean-disk + - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} @@ -873,7 +1002,7 @@ jobs: limit-access-to-actor: true - name: Cache local Maven repository - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -884,11 +1013,11 @@ jobs: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} ${{ runner.os }}-m2-dependencies-core-modules- - - name: Set up JDK 17 - uses: actions/setup-java@v3 + - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} + uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: 17 + distribution: ${{ env.JDK_DISTRIBUTION }} + java-version: ${{ env.CI_JDK_MAJOR_VERSION }} - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master @@ -935,7 +1064,7 @@ jobs: annotate_only: 'true' - name: Upload container logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ !success() }} continue-on-error: true with: @@ -944,7 +1073,7 @@ jobs: retention-days: 7 - name: Upload Surefire reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ !success() }} with: name: System-${{ matrix.name }}-surefire-reports @@ -961,16 +1090,17 @@ jobs: system-tests-upload-coverage: name: CI - System - Upload Coverage - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 30 needs: ['preconditions', 'system-tests'] if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} env: PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest + CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -984,7 +1114,7 @@ jobs: limit-access-to-actor: true - name: Cache local Maven repository - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -995,11 +1125,11 @@ jobs: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} ${{ runner.os }}-m2-dependencies-core-modules- - - name: Set up JDK 17 - uses: actions/setup-java@v3 + - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} + uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: 17 + distribution: ${{ env.JDK_DISTRIBUTION }} + java-version: ${{ env.CI_JDK_MAJOR_VERSION }} - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master @@ -1022,7 +1152,7 @@ jobs: zip -qr jacoco_test_coverage_report_systests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true - name: Upload Jacoco report files to build artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: Jacoco-coverage-report-systests path: target/jacoco_test_coverage_report_systests.zip @@ -1047,14 +1177,15 @@ jobs: flaky-system-tests: name: CI Flaky - System - ${{ matrix.name }} - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 60 needs: [ 'preconditions', 'pulsar-test-latest-version-image' ] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} env: JOB_NAME: CI Flaky - System - ${{ matrix.name }} PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} strategy: fail-fast: false matrix: @@ -1064,14 +1195,19 @@ jobs: - name: Pulsar IO - Oracle group: PULSAR_IO_ORA + clean_disk: true steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm + - name: Clean Disk when needed + if: ${{ matrix.clean_disk }} + uses: ./.github/actions/clean-disk + - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} @@ -1081,7 +1217,7 @@ jobs: limit-access-to-actor: true - name: Cache local Maven repository - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -1092,11 +1228,11 @@ jobs: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} ${{ runner.os }}-m2-dependencies-core-modules- - - name: Set up JDK 17 - uses: actions/setup-java@v3 + - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} + uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: 17 + distribution: ${{ env.JDK_DISTRIBUTION }} + java-version: ${{ env.CI_JDK_MAJOR_VERSION }} - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master @@ -1135,7 +1271,7 @@ jobs: annotate_only: 'true' - name: Upload container logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ !success() }} continue-on-error: true with: @@ -1144,7 +1280,7 @@ jobs: retention-days: 7 - name: Upload Surefire reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ !success() }} with: name: System-${{ matrix.name }}-surefire-reports @@ -1161,7 +1297,7 @@ jobs: delete-system-test-docker-image-artifact: name: "Delete system test docker image artifact" - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 10 needs: [ 'preconditions', @@ -1172,7 +1308,7 @@ jobs: if: ${{ needs.preconditions.outputs.docs_only != 'true' }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -1186,21 +1322,22 @@ jobs: macos-build: name: Build Pulsar on MacOS - runs-on: macos-11 + runs-on: macos-latest timeout-minutes: 120 needs: ['preconditions', 'integration-tests'] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} env: - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Cache Maven dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -1210,26 +1347,93 @@ jobs: restore-keys: | ${{ runner.os }}-m2-dependencies-all- - - name: Set up JDK 17 - uses: actions/setup-java@v3 + - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} + uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: 17 + distribution: ${{ env.JDK_DISTRIBUTION }} + java-version: ${{ env.CI_JDK_MAJOR_VERSION }} - name: build package run: mvn -B clean package -DskipTests -T 1C -ntp + codeql: + name: Run CodeQL Analysis + runs-on: ubuntu-22.04 + timeout-minutes: 60 + needs: ['preconditions', 'unit-tests'] + if: ${{ (needs.preconditions.outputs.java_non_tests == 'true' || github.event_name != 'pull_request') && ((github.event_name == 'pull_request' && github.base_ref == 'master') || (github.event_name != 'pull_request' && github.ref_name == 'master')) }} + permissions: + actions: read + contents: read + security-events: write + env: + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} + CODEQL_LANGUAGE: java-kotlin + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Clean Disk when needed + if: ${{ matrix.clean_disk }} + uses: ./.github/actions/clean-disk + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache local Maven repository + uses: actions/cache@v4 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} + uses: actions/setup-java@v4 + with: + distribution: ${{ env.JDK_DISTRIBUTION }} + java-version: ${{ env.CI_JDK_MAJOR_VERSION }} + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ env.CODEQL_LANGUAGE }} + + - name: Build Java code + run: | + mvn -B -ntp -Pcore-modules,-main install -DskipTests -Dlicense.skip=true -Drat.skip=true -Dcheckstyle.skip=true + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{ env.CODEQL_LANGUAGE }}" + owasp-dep-check: name: OWASP dependency check - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 120 needs: [ 'preconditions', 'integration-tests' ] if: ${{ needs.preconditions.outputs.need_owasp == 'true' }} env: - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} + NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -1242,8 +1446,8 @@ jobs: with: limit-access-to-actor: true - - name: Cache Maven dependencies - uses: actions/cache@v3 + - name: Restore Maven repository cache + uses: actions/cache/restore@v4 timeout-minutes: 5 with: path: | @@ -1252,11 +1456,12 @@ jobs: key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules- - - name: Set up JDK ${{ matrix.jdk || '17' }} - uses: actions/setup-java@v3 + + - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} + uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: ${{ matrix.jdk || '17' }} + distribution: ${{ env.JDK_DISTRIBUTION }} + java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} - name: Clean Disk uses: ./.github/actions/clean-disk @@ -1268,12 +1473,32 @@ jobs: run: | cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - # Projects dependent on flume, hdfs, hbase, and presto currently excluded from the scan. - - name: run "clean verify" to trigger dependency check - run: mvn -q -B -ntp verify -PskipDocker,owasp-dependency-check -DskipTests -pl '!pulsar-sql,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb' + + - name: OWASP cache key weeknum + id: get-weeknum + run: | + echo "weeknum=$(date -u +"%Y-%U")" >> $GITHUB_OUTPUT + shell: bash + + - name: Restore OWASP Dependency Check data + id: restore-owasp-dependency-check-data + uses: actions/cache/restore@v4 + timeout-minutes: 5 + with: + path: ~/.m2/repository/org/owasp/dependency-check-data + key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }} + enableCrossOsArchive: true + restore-keys: | + owasp-dependency-check-data- + + # Projects dependent on flume, hdfs, and hbase currently excluded from the scan. + - name: trigger dependency check + run: | + mvn -B -ntp verify -PskipDocker,skip-all,owasp-dependency-check -Dcheckstyle.skip=true -DskipTests \ + -pl '!distribution/server,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb' - name: Upload report - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ cancelled() || failure() }} continue-on-error: true with: @@ -1288,13 +1513,17 @@ jobs: with: action: wait - # This job is required for pulls to be merged. + # This job is required for pulls to be merged. This job is referenced by name in .asf.yaml file in the + # protected_branches section for master branch required_status_checks. # It depends on all other jobs in this workflow. - # It cleans up the binaries in the same job in order to not spin up another runner for basically doing nothing. + # This job also cleans up the binaries at the end of the workflow. pulsar-ci-checks-completed: name: "Pulsar CI checks completed" - if: ${{ always() && ((github.event_name != 'schedule') || (github.repository == 'apache/pulsar')) }} - runs-on: ubuntu-20.04 + # run always, but skip for other repositories than apache/pulsar when a scheduled workflow is cancelled + # this is to allow the workflow scheduled jobs to show as cancelled instead of failed since scheduled + # jobs are not enabled for other than apache/pulsar repository. + if: ${{ always() && !(cancelled() && github.repository != 'apache/pulsar' && github.event_name == 'schedule') }} + runs-on: ubuntu-22.04 timeout-minutes: 10 needs: [ 'preconditions', @@ -1306,17 +1535,20 @@ jobs: 'unit-tests-upload-coverage', 'integration-tests-upload-coverage', 'system-tests-upload-coverage', - 'owasp-dep-check' + 'owasp-dep-check', + 'codeql' ] steps: - name: Check that all required jobs were completed successfully - if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + if: ${{ needs.preconditions.result != 'success' || needs.preconditions.outputs.docs_only != 'true' }} run: | if [[ ! ( \ - "${{ needs.unit-tests.result }}" == "success" \ + "${{ needs.preconditions.result }}" == "success" \ + && "${{ needs.unit-tests.result }}" == "success" \ && "${{ needs.integration-tests.result }}" == "success" \ && "${{ needs.system-tests.result }}" == "success" \ && "${{ needs.macos-build.result }}" == "success" \ + && ( "${{ needs.codeql.result }}" == "success" || "${{ needs.codeql.result }}" == "skipped" ) \ ) ]]; then echo "Required jobs haven't been completed successfully." exit 1 @@ -1324,7 +1556,7 @@ jobs: - name: checkout if: ${{ needs.preconditions.outputs.docs_only != 'true' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM if: ${{ needs.preconditions.outputs.docs_only != 'true' }} @@ -1338,4 +1570,4 @@ jobs: if: ${{ needs.preconditions.outputs.docs_only != 'true' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }} run: | gh-actions-artifact-client.js delete pulsar-maven-repository-binaries.tar.zst || true - gh-actions-artifact-client.js delete pulsar-server-distribution.tar.zst || true + gh-actions-artifact-client.js delete pulsar-server-distribution.tar.zst || true \ No newline at end of file diff --git a/.gitignore b/.gitignore index cd00c44200059..80d760cd29df7 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,5 @@ test-reports/ # Gradle Enterprise .mvn/.gradle-enterprise/ +# Gradle Develocity +.mvn/.develocity/ diff --git a/.idea/vcs.xml b/.idea/vcs.xml index 99fce8b2b9812..29e5254a1d41b 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -1,36 +1,16 @@ - - - - - - - - + + + + + + + \ No newline at end of file diff --git a/.mvn/gradle-enterprise-custom-user-data.groovy b/.mvn/develocity-custom-user-data.groovy similarity index 100% rename from .mvn/gradle-enterprise-custom-user-data.groovy rename to .mvn/develocity-custom-user-data.groovy diff --git a/.mvn/gradle-enterprise.xml b/.mvn/develocity.xml similarity index 66% rename from .mvn/gradle-enterprise.xml rename to .mvn/develocity.xml index 2667402c23cdb..5c0fbb47c7217 100644 --- a/.mvn/gradle-enterprise.xml +++ b/.mvn/develocity.xml @@ -19,22 +19,21 @@ under the License. --> - + + + #{(env['GRADLE_ENTERPRISE_ACCESS_KEY']?.trim() > '' or env['DEVELOCITY_ACCESS_KEY']?.trim() > '') and !(env['GITHUB_HEAD_REF']?.matches('(?i).*(experiment|wip|private).*') or env['GITHUB_REPOSITORY']?.matches('(?i).*(experiment|wip|private).*'))} https://ge.apache.org false - true true true #{isFalse(env['GITHUB_ACTIONS'])} - ALWAYS - true #{{'0.0.0.0'}} @@ -47,4 +46,4 @@ false - + \ No newline at end of file diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index 872764f899827..eb998dc3471b8 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -23,12 +23,12 @@ xsi:schemaLocation="http://maven.apache.org/EXTENSIONS/1.0.0 http://maven.apache.org/xsd/core-extensions-1.0.0.xsd"> com.gradle - gradle-enterprise-maven-extension - 1.17.1 + develocity-maven-extension + 1.21.6 com.gradle common-custom-user-data-maven-extension - 1.11.1 + 2.0 diff --git a/NOTICE b/NOTICE index bbbe4fab89b56..a88a696cfa59f 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache Pulsar -Copyright 2017-2022 The Apache Software Foundation +Copyright 2017-2024 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/README.md b/README.md index fdbf7c7339b1e..1d53af9f08149 100644 --- a/README.md +++ b/README.md @@ -141,8 +141,6 @@ components in the Pulsar ecosystem, including connectors, adapters, and other la > > This project includes a [Maven Wrapper](https://maven.apache.org/wrapper/) that can be used instead of a system-installed Maven. > Use it by replacing `mvn` by `./mvnw` on Linux and `mvnw.cmd` on Windows in the commands below. -> -> It's better to use CMD rather than Powershell on Windows. Because maven will activate the `windows` profile which runs `rename-netty-native-libs.cmd`. ### Build @@ -192,6 +190,10 @@ Check https://pulsar.apache.org for documentation and examples. ## Build custom docker images +The commands used in the Apache Pulsar release process can be found in the [release process documentation](https://pulsar.apache.org/contribute/release-process/#stage-docker-images). + +Here are some general instructions for building custom docker images: + * Docker images must be built with Java 8 for `branch-2.7` or previous branches because of [ISSUE-8445](https://github.com/apache/pulsar/issues/8445). * Java 11 is the recommended JDK version in `branch-2.8`, `branch-2.9` and `branch-2.10`. * Java 17 is the recommended JDK version in `master`. @@ -200,6 +202,8 @@ The following command builds the docker images `apachepulsar/pulsar-all:latest` ```bash mvn clean install -DskipTests +# setting DOCKER_CLI_EXPERIMENTAL=enabled is required in some environments with older docker versions +export DOCKER_CLI_EXPERIMENTAL=enabled mvn package -Pdocker,-main -am -pl docker/pulsar-all -DskipTests ``` @@ -242,7 +246,11 @@ Pulsar slack channel at https://apache-pulsar.slack.com/ You can self-register at https://communityinviter.com/apps/apache-pulsar/apache-pulsar -##### Report a security vulnerability +## Security Policy + +If you find a security issue with Pulsar then please [read the security policy](https://pulsar.apache.org/security/#security-policy). It is critical to avoid public disclosure. + +### Reporting a security vulnerability To report a vulnerability for Pulsar, contact the [Apache Security Team](https://www.apache.org/security/). When reporting a vulnerability to [security@apache.org](mailto:security@apache.org), you can copy your email to [private@pulsar.apache.org](mailto:private@pulsar.apache.org) to send your report to the Apache Pulsar Project Management Committee. This is a private mailing list. diff --git a/bin/bookkeeper b/bin/bookkeeper index fb516a98acdc2..668c5d4db70a8 100755 --- a/bin/bookkeeper +++ b/bin/bookkeeper @@ -69,6 +69,29 @@ else JAVA=$JAVA_HOME/bin/java fi +# JAVA_MAJOR_VERSION should get set by conf/bkenv.sh, just in case it's not +if [[ -z $JAVA_MAJOR_VERSION ]]; then + for token in $("$JAVA" -version 2>&1 | grep 'version "'); do + if [[ $token =~ \"([[:digit:]]+)\.([[:digit:]]+)(.*)\" ]]; then + if [[ ${BASH_REMATCH[1]} == "1" ]]; then + JAVA_MAJOR_VERSION=${BASH_REMATCH[2]} + else + JAVA_MAJOR_VERSION=${BASH_REMATCH[1]} + fi + break + elif [[ $token =~ \"([[:digit:]]+)(.*)\" ]]; then + # Process the java versions without dots, such as `17-internal`. + JAVA_MAJOR_VERSION=${BASH_REMATCH[1]} + break + fi + done +fi + +if [[ $JAVA_MAJOR_VERSION -lt 17 ]]; then + echo "Error: Bookkeeper included in Pulsar requires Java 17 or later." 1>&2 + exit 1 +fi + # exclude tests jar RELEASE_JAR=`ls $BK_HOME/bookkeeper-server-*.jar 2> /dev/null | grep -v tests | tail -1` if [ $? == 0 ]; then @@ -168,17 +191,12 @@ OPTS="$OPTS -Dlog4j.configurationFile=`basename $BOOKIE_LOG_CONF`" # Allow Netty to use reflection access OPTS="$OPTS -Dio.netty.tryReflectionSetAccessible=true" -IS_JAVA_8=`$JAVA -version 2>&1 |grep version|grep '"1\.8'` -# Start --add-opens options -# '--add-opens' option is not supported in jdk8 -if [[ -z "$IS_JAVA_8" ]]; then - # BookKeeper: enable posix_fadvise usage and DirectMemoryCRC32Digest (https://github.com/apache/bookkeeper/pull/3234) - OPTS="$OPTS --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.util.zip=ALL-UNNAMED" - # Netty: enable java.nio.DirectByteBuffer - # https://github.com/netty/netty/blob/4.1/common/src/main/java/io/netty/util/internal/PlatformDependent0.java - # https://github.com/netty/netty/issues/12265 - OPTS="$OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED" -fi +# BookKeeper: enable posix_fadvise usage and DirectMemoryCRC32Digest (https://github.com/apache/bookkeeper/pull/3234) +OPTS="$OPTS --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.util.zip=ALL-UNNAMED" +# Netty: enable java.nio.DirectByteBuffer +# https://github.com/netty/netty/blob/4.1/common/src/main/java/io/netty/util/internal/PlatformDependent0.java +# https://github.com/netty/netty/issues/12265 +OPTS="$OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED" OPTS="-cp $BOOKIE_CLASSPATH $OPTS" diff --git a/bin/function-localrunner b/bin/function-localrunner index 45a37cb306794..a47f3efa48609 100755 --- a/bin/function-localrunner +++ b/bin/function-localrunner @@ -34,19 +34,46 @@ else JAVA=$JAVA_HOME/bin/java fi +for token in $("$JAVA" -version 2>&1 | grep 'version "'); do + if [[ $token =~ \"([[:digit:]]+)\.([[:digit:]]+)(.*)\" ]]; then + if [[ ${BASH_REMATCH[1]} == "1" ]]; then + JAVA_MAJOR_VERSION=${BASH_REMATCH[2]} + else + JAVA_MAJOR_VERSION=${BASH_REMATCH[1]} + fi + break + elif [[ $token =~ \"([[:digit:]]+)(.*)\" ]]; then + # Process the java versions without dots, such as `17-internal`. + JAVA_MAJOR_VERSION=${BASH_REMATCH[1]} + break + fi +done + PULSAR_MEM=${PULSAR_MEM:-"-Xmx128m -XX:MaxDirectMemorySize=128m"} # Garbage collection options -PULSAR_GC=${PULSAR_GC:-"-XX:+UseZGC -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch"} +if [ -z "$PULSAR_GC" ]; then + PULSAR_GC="-XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch" + if [[ $JAVA_MAJOR_VERSION -ge 21 ]]; then + PULSAR_GC="-XX:+UseZGC -XX:+ZGenerational ${PULSAR_GC}" + else + PULSAR_GC="-XX:+UseZGC ${PULSAR_GC}" + fi +fi # Garbage collection log. -IS_JAVA_8=`$JAVA -version 2>&1 |grep version|grep '"1\.8'` -# java version has space, use [[ -n $PARAM ]] to judge if variable exists -if [[ -n "$IS_JAVA_8" ]]; then - PULSAR_GC_LOG=${PULSAR_GC_LOG:-"-Xloggc:logs/pulsar_gc_%p.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=20M"} -else -# After jdk 9, gc log param should config like this. Ignoring version less than jdk 8 - PULSAR_GC_LOG=${PULSAR_GC_LOG:-"-Xlog:gc:logs/pulsar_gc_%p.log:time,uptime:filecount=10,filesize=20M"} +PULSAR_GC_LOG_DIR=${PULSAR_GC_LOG_DIR:-logs} +if [[ -z "$PULSAR_GC_LOG" ]]; then + if [[ $JAVA_MAJOR_VERSION -gt 8 ]]; then + PULSAR_GC_LOG="-Xlog:gc*,safepoint:${PULSAR_GC_LOG_DIR}/pulsar_gc_%p.log:time,uptime,tags:filecount=10,filesize=20M" + if [[ $JAVA_MAJOR_VERSION -ge 17 ]]; then + # Use async logging on Java 17+ https://bugs.openjdk.java.net/browse/JDK-8264323 + PULSAR_GC_LOG="-Xlog:async ${PULSAR_GC_LOG}" + fi + else + # Java 8 gc log options + PULSAR_GC_LOG="-Xloggc:${PULSAR_GC_LOG_DIR}/pulsar_gc_%p.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=20M" + fi fi # Extra options to be passed to the jvm @@ -88,13 +115,16 @@ OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF`" # Allow Netty to use reflection access OPTS="$OPTS -Dio.netty.tryReflectionSetAccessible=true" +OPTS="$OPTS -Dorg.apache.pulsar.shade.io.netty.tryReflectionSetAccessible=true" + +if [[ $JAVA_MAJOR_VERSION -gt 8 ]]; then + # Required by Pulsar client optimized checksum calculation on other than Linux x86_64 platforms + # reflection access to java.util.zip.CRC32C + OPTS="$OPTS --add-opens java.base/java.util.zip=ALL-UNNAMED" +fi -# Start --add-opens options -# '--add-opens' option is not supported in jdk8 -if [[ -z "$IS_JAVA_8" ]]; then - # Netty: enable java.nio.DirectByteBuffer - # https://github.com/netty/netty/blob/4.1/common/src/main/java/io/netty/util/internal/PlatformDependent0.java - # https://github.com/netty/netty/issues/12265 +if [[ $JAVA_MAJOR_VERSION -ge 11 ]]; then + # Required by Netty for optimized direct byte buffer access OPTS="$OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED" fi diff --git a/bin/pulsar b/bin/pulsar index e3b22caced52e..09be2ac50e279 100755 --- a/bin/pulsar +++ b/bin/pulsar @@ -44,9 +44,6 @@ DEFAULT_PY_INSTANCE_FILE=$PULSAR_HOME/instances/python-instance/python_instance_ PY_INSTANCE_FILE=${PULSAR_PY_INSTANCE_FILE:-"${DEFAULT_PY_INSTANCE_FILE}"} DEFAULT_FUNCTIONS_EXTRA_DEPS_DIR=$PULSAR_HOME/instances/deps FUNCTIONS_EXTRA_DEPS_DIR=${PULSAR_FUNCTIONS_EXTRA_DEPS_DIR:-"${DEFAULT_FUNCTIONS_EXTRA_DEPS_DIR}"} -SQL_HOME=$PULSAR_HOME/pulsar-sql -TRINO_HOME=${PULSAR_HOME}/trino -DEFAULT_PULSAR_TRINO_CONF=${TRINO_HOME}/conf pulsar_help() { cat <&1 | grep 'version "'); do + if [[ $token =~ \"([[:digit:]]+)\.([[:digit:]]+)(.*)\" ]]; then + if [[ ${BASH_REMATCH[1]} == "1" ]]; then + JAVA_MAJOR_VERSION=${BASH_REMATCH[2]} + else + JAVA_MAJOR_VERSION=${BASH_REMATCH[1]} + fi + break + elif [[ $token =~ \"([[:digit:]]+)(.*)\" ]]; then + # Process the java versions without dots, such as `17-internal`. + JAVA_MAJOR_VERSION=${BASH_REMATCH[1]} + break + fi + done +fi + +if [[ $JAVA_MAJOR_VERSION -lt 17 ]]; then + echo "Error: Pulsar requires Java 17 or later." 1>&2 + exit 1 +fi + # exclude tests jar RELEASE_JAR=`ls $PULSAR_HOME/pulsar-*.jar 2> /dev/null | grep -v tests | tail -1` if [ $? == 0 ]; then @@ -179,20 +196,6 @@ if [ ! -f "${PY_INSTANCE_FILE}" ]; then PY_INSTANCE_FILE=${BUILT_PY_INSTANCE_FILE} fi -# find pulsar sql trino distribution location -check_trino_libraries() { - if [ ! -d "${TRINO_HOME}" ]; then - BUILT_TRINO_HOME="${SQL_HOME}/presto-distribution/target/pulsar-presto-distribution" - if [ ! -d "${BUILT_TRINO_HOME}" ]; then - echo "\nCouldn't find trino distribution."; - echo "Make sure you've run 'mvn package'\n"; - exit 1; - fi - TRINO_HOME=${BUILT_TRINO_HOME} - PULSAR_TRINO_CONF=${BUILT_TRINO_HOME}/conf - fi -} - add_maven_deps_to_classpath() { MVN="mvn" if [ "$MAVEN_HOME" != "" ]; then @@ -259,21 +262,6 @@ if [ -z "$PULSAR_LOG_CONF" ]; then PULSAR_LOG_CONF=$DEFAULT_LOG_CONF fi -if [ -z "$PULSAR_TRINO_CONF" ]; then - # TODO: As PIP-200 accepted, this compatibility is not promised. Refactor when we drop this b/w compatibility. - if [ -z "$PULSAR_PRESTO_CONF" ]; then - PULSAR_TRINO_CONF=$DEFAULT_PULSAR_TRINO_CONF - else - PULSAR_TRINO_CONF=$PULSAR_PRESTO_CONF - fi - if [ ! -d "${PULSAR_TRINO_CONF}" ]; then - FALLBACK_PULSAR_PRESTO_CONF=${PULSAR_HOME}/conf/presto - if [ -d "${FALLBACK_PULSAR_PRESTO_CONF}" ]; then - PULSAR_TRINO_CONF=$FALLBACK_PULSAR_PRESTO_CONF - fi - fi -fi - if [ -z "$FUNCTIONS_LOG_CONF" ]; then FUNCTIONS_LOG_CONF=$DEFAULT_FUNCTIONS_LOG_CONF fi @@ -289,27 +277,21 @@ OPTS="$OPTS -Djute.maxbuffer=10485760 -Djava.net.preferIPv4Stack=true" # Enable TCP keepalive for all Zookeeper client connections OPTS="$OPTS -Dzookeeper.clientTcpKeepAlive=true" +# BookKeeper: enable posix_fadvise usage and DirectMemoryCRC32Digest (https://github.com/apache/bookkeeper/pull/3234) +OPTS="$OPTS --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.util.zip=ALL-UNNAMED" +# Required by JvmDefaultGCMetricsLogger & MBeanStatsGenerator +OPTS="$OPTS --add-opens java.management/sun.management=ALL-UNNAMED" +# Required by MBeanStatsGenerator +OPTS="$OPTS --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED" # Allow Netty to use reflection access OPTS="$OPTS -Dio.netty.tryReflectionSetAccessible=true" -IS_JAVA_8=`$JAVA -version 2>&1 |grep version|grep '"1\.8'` -# Start --add-opens options -# '--add-opens' option is not supported in jdk8 -if [[ -z "$IS_JAVA_8" ]]; then - # BookKeeper: enable posix_fadvise usage and DirectMemoryCRC32Digest (https://github.com/apache/bookkeeper/pull/3234) - OPTS="$OPTS --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.util.zip=ALL-UNNAMED" - # Netty: enable java.nio.DirectByteBuffer - # https://github.com/netty/netty/blob/4.1/common/src/main/java/io/netty/util/internal/PlatformDependent0.java - # https://github.com/netty/netty/issues/12265 - OPTS="$OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED" - # netty.DnsResolverUtil - OPTS="$OPTS --add-opens java.base/sun.net=ALL-UNNAMED" - # JvmDefaultGCMetricsLogger & MBeanStatsGenerator - OPTS="$OPTS --add-opens java.management/sun.management=ALL-UNNAMED" - # MBeanStatsGenerator - OPTS="$OPTS --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED" - # LinuxInfoUtils - OPTS="$OPTS --add-opens java.base/jdk.internal.platform=ALL-UNNAMED" -fi +OPTS="$OPTS -Dorg.apache.pulsar.shade.io.netty.tryReflectionSetAccessible=true" +# Netty: enable java.nio.DirectByteBuffer +# https://github.com/netty/netty/blob/4.1/common/src/main/java/io/netty/util/internal/PlatformDependent0.java +# https://github.com/netty/netty/issues/12265 +OPTS="$OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED" +# Required by LinuxInfoUtils +OPTS="$OPTS --add-opens java.base/jdk.internal.platform=ALL-UNNAMED" OPTS="-cp $PULSAR_CLASSPATH $OPTS" @@ -410,12 +392,6 @@ elif [ $COMMAND == "broker-tool" ]; then exec $JAVA $OPTS org.apache.pulsar.broker.tools.BrokerTool $@ elif [ $COMMAND == "compact-topic" ]; then exec $JAVA $OPTS org.apache.pulsar.compaction.CompactorTool --broker-conf $PULSAR_BROKER_CONF $@ -elif [ $COMMAND == "sql" ]; then - check_trino_libraries - exec $JAVA -cp "${TRINO_HOME}/lib/*" io.trino.cli.Trino --server localhost:8081 "${@}" -elif [ $COMMAND == "sql-worker" ]; then - check_trino_libraries - exec python3 ${TRINO_HOME}/bin/launcher.py --etc-dir ${PULSAR_TRINO_CONF} "${@}" elif [ $COMMAND == "tokens" ]; then exec $JAVA $OPTS org.apache.pulsar.utils.auth.tokens.TokensCliUtils $@ elif [ $COMMAND == "version" ]; then diff --git a/bin/pulsar-admin-common.cmd b/bin/pulsar-admin-common.cmd index c52bc1389f68a..c59f0e9b424d3 100644 --- a/bin/pulsar-admin-common.cmd +++ b/bin/pulsar-admin-common.cmd @@ -19,7 +19,7 @@ @echo off -if "%JAVA_HOME%" == "" ( +if not defined JAVA_HOME ( for %%i in (java.exe) do set "JAVACMD=%%~$PATH:i" ) else ( set "JAVACMD=%JAVA_HOME%\bin\java.exe" @@ -28,16 +28,28 @@ if "%JAVA_HOME%" == "" ( if not exist "%JAVACMD%" ( echo The JAVA_HOME environment variable is not defined correctly, so Pulsar CLI cannot be started. >&2 echo JAVA_HOME is set to "%JAVA_HOME%", but "%JAVACMD%" does not exist. >&2 - goto error + exit /b 1 ) +set JAVA_MAJOR_VERSION=0 +REM Requires "setlocal enabledelayedexpansion" to work +for /f tokens^=3 %%g in ('"!JAVACMD!" -version 2^>^&1 ^| findstr /i version') do ( + set JAVA_MAJOR_VERSION=%%g +) +set JAVA_MAJOR_VERSION=%JAVA_MAJOR_VERSION:"=% +for /f "delims=.-_ tokens=1-2" %%v in ("%JAVA_MAJOR_VERSION%") do ( + if /I "%%v" EQU "1" ( + set JAVA_MAJOR_VERSION=%%w + ) else ( + set JAVA_MAJOR_VERSION=%%v + ) +) for %%i in ("%~dp0.") do SET "SCRIPT_PATH=%%~fi" set "PULSAR_HOME_DIR=%SCRIPT_PATH%\..\" for %%i in ("%PULSAR_HOME_DIR%.") do SET "PULSAR_HOME=%%~fi" set "PULSAR_CLASSPATH=%PULSAR_CLASSPATH%;%PULSAR_HOME%\lib\*" - if "%PULSAR_CLIENT_CONF%" == "" set "PULSAR_CLIENT_CONF=%PULSAR_HOME%\conf\client.conf" if "%PULSAR_LOG_CONF%" == "" set "PULSAR_LOG_CONF=%PULSAR_HOME%\conf\log4j2.yaml" @@ -50,18 +62,21 @@ set "PULSAR_CLASSPATH=%PULSAR_CLASSPATH%;%PULSAR_LOG_CONF_DIR%" set "OPTS=%OPTS% -Dlog4j.configurationFile="%PULSAR_LOG_CONF_BASENAME%"" set "OPTS=%OPTS% -Djava.net.preferIPv4Stack=true" -set "isjava8=false" -FOR /F "tokens=*" %%g IN ('"java -version 2>&1"') do ( - echo %%g|find "version" >nul - if errorlevel 0 ( - echo %%g|find "1.8" >nul - if errorlevel 0 ( - set "isjava8=true" - ) - ) +REM Allow Netty to use reflection access +set "OPTS=%OPTS% -Dio.netty.tryReflectionSetAccessible=true" +set "OPTS=%OPTS% -Dorg.apache.pulsar.shade.io.netty.tryReflectionSetAccessible=true" + +if %JAVA_MAJOR_VERSION% GTR 8 ( + set "OPTS=%OPTS% --add-opens java.base/sun.net=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED" + REM Required by Pulsar client optimized checksum calculation on other than Linux x86_64 platforms + REM reflection access to java.util.zip.CRC32C + set "OPTS=%OPTS% --add-opens java.base/java.util.zip=ALL-UNNAMED" ) -if "%isjava8%" == "false" set "OPTS=%OPTS% --add-opens java.base/sun.net=ALL-UNNAMED" +if %JAVA_MAJOR_VERSION% GEQ 11 ( + REM Required by Netty for optimized direct byte buffer access + set "OPTS=%OPTS% --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED" +) set "OPTS=-cp "%PULSAR_CLASSPATH%" %OPTS%" set "OPTS=%OPTS% %PULSAR_EXTRA_OPTS%" @@ -78,7 +93,4 @@ set "OPTS=%OPTS% -Dpulsar.log.dir=%PULSAR_LOG_DIR%" set "OPTS=%OPTS% -Dpulsar.log.level=%PULSAR_LOG_LEVEL%" set "OPTS=%OPTS% -Dpulsar.log.root.level=%PULSAR_LOG_ROOT_LEVEL%" set "OPTS=%OPTS% -Dpulsar.log.immediateFlush=%PULSAR_LOG_IMMEDIATE_FLUSH%" -set "OPTS=%OPTS% -Dpulsar.routing.appender.default=%PULSAR_ROUTING_APPENDER_DEFAULT%" - -:error -exit /b 1 +set "OPTS=%OPTS% -Dpulsar.routing.appender.default=%PULSAR_ROUTING_APPENDER_DEFAULT%" \ No newline at end of file diff --git a/bin/pulsar-admin-common.sh b/bin/pulsar-admin-common.sh index 8223ac5b3bf24..336ff43c1a861 100755 --- a/bin/pulsar-admin-common.sh +++ b/bin/pulsar-admin-common.sh @@ -37,6 +37,21 @@ else JAVA=$JAVA_HOME/bin/java fi +for token in $("$JAVA" -version 2>&1 | grep 'version "'); do + if [[ $token =~ \"([[:digit:]]+)\.([[:digit:]]+)(.*)\" ]]; then + if [[ ${BASH_REMATCH[1]} == "1" ]]; then + JAVA_MAJOR_VERSION=${BASH_REMATCH[2]} + else + JAVA_MAJOR_VERSION=${BASH_REMATCH[1]} + fi + break + elif [[ $token =~ \"([[:digit:]]+)(.*)\" ]]; then + # Process the java versions without dots, such as `17-internal`. + JAVA_MAJOR_VERSION=${BASH_REMATCH[1]} + break + fi +done + # exclude tests jar RELEASE_JAR=`ls $PULSAR_HOME/pulsar-*.jar 2> /dev/null | grep -v tests | tail -1` if [ $? == 0 ]; then @@ -91,11 +106,20 @@ PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH" OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF`" OPTS="$OPTS -Djava.net.preferIPv4Stack=true" -IS_JAVA_8=`$JAVA -version 2>&1 |grep version|grep '"1\.8'` -# Start --add-opens options -# '--add-opens' option is not supported in jdk8 -if [[ -z "$IS_JAVA_8" ]]; then +# Allow Netty to use reflection access +OPTS="$OPTS -Dio.netty.tryReflectionSetAccessible=true" +OPTS="$OPTS -Dorg.apache.pulsar.shade.io.netty.tryReflectionSetAccessible=true" + +if [[ $JAVA_MAJOR_VERSION -gt 8 ]]; then OPTS="$OPTS --add-opens java.base/sun.net=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED" + # Required by Pulsar client optimized checksum calculation on other than Linux x86_64 platforms + # reflection access to java.util.zip.CRC32C + OPTS="$OPTS --add-opens java.base/java.util.zip=ALL-UNNAMED" +fi + +if [[ $JAVA_MAJOR_VERSION -ge 11 ]]; then + # Required by Netty for optimized direct byte buffer access + OPTS="$OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED" fi OPTS="-cp $PULSAR_CLASSPATH $OPTS" diff --git a/bin/pulsar-admin.cmd b/bin/pulsar-admin.cmd index 45bd8d4541fed..e29d804d70c45 100644 --- a/bin/pulsar-admin.cmd +++ b/bin/pulsar-admin.cmd @@ -18,7 +18,7 @@ @REM @echo off - +setlocal enabledelayedexpansion for %%i in ("%~dp0.") do SET "SCRIPT_PATH=%%~fi" set "PULSAR_HOME_DIR=%SCRIPT_PATH%\..\" for %%i in ("%PULSAR_HOME_DIR%.") do SET "PULSAR_HOME=%%~fi" @@ -27,4 +27,5 @@ if ERRORLEVEL 1 ( exit /b 1 ) cd "%PULSAR_HOME%" -"%JAVACMD%" %OPTS% org.apache.pulsar.admin.cli.PulsarAdminTool %PULSAR_CLIENT_CONF% %* \ No newline at end of file +"%JAVACMD%" %OPTS% org.apache.pulsar.admin.cli.PulsarAdminTool %PULSAR_CLIENT_CONF% %* +endlocal \ No newline at end of file diff --git a/bin/pulsar-client.cmd b/bin/pulsar-client.cmd index 9e3cef45a25a7..9cacf71cc3a79 100644 --- a/bin/pulsar-client.cmd +++ b/bin/pulsar-client.cmd @@ -18,7 +18,7 @@ @REM @echo off - +setlocal enabledelayedexpansion for %%i in ("%~dp0.") do SET "SCRIPT_PATH=%%~fi" set "PULSAR_HOME_DIR=%SCRIPT_PATH%\..\" for %%i in ("%PULSAR_HOME_DIR%.") do SET "PULSAR_HOME=%%~fi" @@ -27,4 +27,5 @@ if ERRORLEVEL 1 ( exit /b 1 ) cd "%PULSAR_HOME%" -"%JAVACMD%" %OPTS% org.apache.pulsar.client.cli.PulsarClientTool %PULSAR_CLIENT_CONF% %* \ No newline at end of file +"%JAVACMD%" %OPTS% org.apache.pulsar.client.cli.PulsarClientTool %PULSAR_CLIENT_CONF% %* +endlocal \ No newline at end of file diff --git a/bin/pulsar-daemon b/bin/pulsar-daemon index 210162b6a2190..2c05cb5c49dab 100755 --- a/bin/pulsar-daemon +++ b/bin/pulsar-daemon @@ -157,7 +157,7 @@ start () echo starting $command, logging to $logfile echo Note: Set immediateFlush to true in conf/log4j2.yaml will guarantee the logging event is flushing to disk immediately. The default behavior is switched off due to performance considerations. pulsar=$PULSAR_HOME/bin/pulsar - nohup $pulsar $command "$1" > "$out" 2>&1 < /dev/null & + nohup $pulsar $command "$@" > "$out" 2>&1 < /dev/null & echo $! > $pid sleep 1; head $out sleep 2; @@ -216,7 +216,7 @@ stop () case $startStop in (start) - start "$*" + start "$@" ;; (stop) @@ -224,21 +224,20 @@ case $startStop in ;; (restart) - forceStopFlag=$(echo "$*"|grep "\-force") - if [[ "$forceStopFlag" != "" ]] + if [[ "$1" == "-force" ]] then - stop "-force" + stop -force + # remove "-force" from the arguments + shift else stop fi if [ "$?" == 0 ] then - sleep 3 - paramaters="$*" - startParamaters=${paramaters//-force/} - start "$startParamaters" + sleep 3 + start "$@" else - echo "WARNNING : $command failed restart, for $command is not stopped completely." + echo "WARNNING : $command failed restart, for $command is not stopped completely." fi ;; diff --git a/bin/pulsar-perf b/bin/pulsar-perf index 47c02bc3d67d5..9108a42ef994f 100755 --- a/bin/pulsar-perf +++ b/bin/pulsar-perf @@ -84,37 +84,6 @@ add_maven_deps_to_classpath() { fi PULSAR_CLASSPATH=${CLASSPATH}:`cat "${f}"` } -pulsar_help() { - cat < -where command is one of: - produce Run a producer - consume Run a consumer - transaction Run a transaction repeatedly - read Run a topic reader - - websocket-producer Run a websocket producer - - managed-ledger Write directly on managed-ledgers - monitor-brokers Continuously receive broker data and/or load reports - simulation-client Run a simulation server acting as a Pulsar client - simulation-controller Run a simulation controller to give commands to servers - - gen-doc Generate documentation automatically. - - help This help message - -or command is the full name of a class with a defined main() method. - -Environment variables: - PULSAR_LOG_CONF Log4j configuration file (default $DEFAULT_LOG_CONF) - PULSAR_CLIENT_CONF Configuration file for client (default: $DEFAULT_CLIENT_CONF) - PULSAR_EXTRA_OPTS Extra options to be passed to the jvm - PULSAR_EXTRA_CLASSPATH Add extra paths to the pulsar classpath - -These variable can also be set in conf/pulsar_env.sh -EOF -} if [ -d "$PULSAR_HOME/lib" ]; then PULSAR_CLASSPATH="$PULSAR_CLASSPATH:$PULSAR_HOME/lib/*" @@ -134,11 +103,20 @@ PULSAR_CLASSPATH="$PULSAR_JAR:$PULSAR_CLASSPATH:$PULSAR_EXTRA_CLASSPATH" PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH" OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF` -Djava.net.preferIPv4Stack=true" -IS_JAVA_8=`$JAVA -version 2>&1 |grep version|grep '"1\.8'` -# Start --add-opens options -# '--add-opens' option is not supported in jdk8 -if [[ -z "$IS_JAVA_8" ]]; then +# Allow Netty to use reflection access +OPTS="$OPTS -Dio.netty.tryReflectionSetAccessible=true" +OPTS="$OPTS -Dorg.apache.pulsar.shade.io.netty.tryReflectionSetAccessible=true" + +if [[ $JAVA_MAJOR_VERSION -gt 8 ]]; then OPTS="$OPTS --add-opens java.base/sun.net=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED" + # Required by Pulsar client optimized checksum calculation on other than Linux x86_64 platforms + # reflection access to java.util.zip.CRC32C + OPTS="$OPTS --add-opens java.base/java.util.zip=ALL-UNNAMED" +fi + +if [[ $JAVA_MAJOR_VERSION -ge 11 ]]; then + # Required by Netty for optimized direct byte buffer access + OPTS="$OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED" fi OPTS="-cp $PULSAR_CLASSPATH $OPTS" @@ -162,36 +140,4 @@ OPTS="$OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE" #Change to PULSAR_HOME to support relative paths cd "$PULSAR_HOME" -# if no args specified, show usage -if [ $# = 0 ]; then - pulsar_help; - exit 1; -fi - -# get arguments -COMMAND=$1 -shift - -if [ "$COMMAND" == "produce" ]; then - exec $JAVA $OPTS org.apache.pulsar.testclient.PerformanceProducer --conf-file $PULSAR_PERFTEST_CONF "$@" -elif [ "$COMMAND" == "consume" ]; then - exec $JAVA $OPTS org.apache.pulsar.testclient.PerformanceConsumer --conf-file $PULSAR_PERFTEST_CONF "$@" -elif [ "$COMMAND" == "transaction" ]; then - exec $JAVA $OPTS org.apache.pulsar.testclient.PerformanceTransaction --conf-file $PULSAR_PERFTEST_CONF "$@" -elif [ "$COMMAND" == "read" ]; then - exec $JAVA $OPTS org.apache.pulsar.testclient.PerformanceReader --conf-file $PULSAR_PERFTEST_CONF "$@" -elif [ "$COMMAND" == "monitor-brokers" ]; then - exec $JAVA $OPTS org.apache.pulsar.testclient.BrokerMonitor "$@" -elif [ "$COMMAND" == "simulation-client" ]; then - exec $JAVA $OPTS org.apache.pulsar.testclient.LoadSimulationClient "$@" -elif [ "$COMMAND" == "simulation-controller" ]; then - exec $JAVA $OPTS org.apache.pulsar.testclient.LoadSimulationController "$@" -elif [ "$COMMAND" == "websocket-producer" ]; then - exec $JAVA $OPTS org.apache.pulsar.proxy.socket.client.PerformanceClient "$@" -elif [ "$COMMAND" == "managed-ledger" ]; then - exec $JAVA $OPTS org.apache.pulsar.testclient.ManagedLedgerWriter "$@" -elif [ "$COMMAND" == "gen-doc" ]; then - exec $JAVA $OPTS org.apache.pulsar.testclient.CmdGenerateDocumentation "$@" -else - pulsar_help; -fi +exec $JAVA $OPTS org.apache.pulsar.testclient.PulsarPerfTestTool $PULSAR_PERFTEST_CONF "$@" diff --git a/bin/pulsar-perf.cmd b/bin/pulsar-perf.cmd index cf6c25b77e59d..aaeaa7a21856b 100644 --- a/bin/pulsar-perf.cmd +++ b/bin/pulsar-perf.cmd @@ -18,149 +18,17 @@ @REM @echo off - -if "%JAVA_HOME%" == "" ( - for %%i in (java.exe) do set "JAVACMD=%%~$PATH:i" -) else ( - set "JAVACMD=%JAVA_HOME%\bin\java.exe" -) - -if not exist "%JAVACMD%" ( - echo The JAVA_HOME environment variable is not defined correctly, so Pulsar CLI cannot be started. >&2 - echo JAVA_HOME is set to "%JAVA_HOME%", but "%JAVACMD%" does not exist. >&2 - exit /B 1 -) - +setlocal enabledelayedexpansion for %%i in ("%~dp0.") do SET "SCRIPT_PATH=%%~fi" set "PULSAR_HOME_DIR=%SCRIPT_PATH%\..\" for %%i in ("%PULSAR_HOME_DIR%.") do SET "PULSAR_HOME=%%~fi" -set "PULSAR_CLASSPATH=%PULSAR_CLASSPATH%;%PULSAR_HOME%\lib\*" - - -if "%PULSAR_CLIENT_CONF%" == "" set "PULSAR_CLIENT_CONF=%PULSAR_HOME%\conf\client.conf" -if "%PULSAR_LOG_CONF%" == "" set "PULSAR_LOG_CONF=%PULSAR_HOME%\conf\log4j2.yaml" - -set "PULSAR_LOG_CONF_DIR1=%PULSAR_LOG_CONF%\..\" -for %%i in ("%PULSAR_LOG_CONF_DIR1%.") do SET "PULSAR_LOG_CONF_DIR=%%~fi" -for %%a in ("%PULSAR_LOG_CONF%") do SET "PULSAR_LOG_CONF_BASENAME=%%~nxa" - -set "PULSAR_CLASSPATH=%PULSAR_CLASSPATH%;%PULSAR_LOG_CONF_DIR%" -if not "%PULSAR_EXTRA_CLASSPATH%" == "" set "PULSAR_CLASSPATH=%PULSAR_CLASSPATH%;%PULSAR_EXTRA_CLASSPATH%" - - -if "%PULSAR_PERFTEST_CONF%" == "" set "PULSAR_PERFTEST_CONF=%PULSAR_CLIENT_CONF%" - - -set "OPTS=%OPTS% -Dlog4j.configurationFile="%PULSAR_LOG_CONF_BASENAME%"" -set "OPTS=%OPTS% -Djava.net.preferIPv4Stack=true" - - -set "OPTS=-cp "%PULSAR_CLASSPATH%" %OPTS%" -set "OPTS=%OPTS% %PULSAR_EXTRA_OPTS%" - -if "%PULSAR_LOG_DIR%" == "" set "PULSAR_LOG_DIR=%PULSAR_HOME%\logs" -if "%PULSAR_LOG_FILE%" == "" set "PULSAR_LOG_FILE=pulsar-perftest.log" if "%PULSAR_LOG_APPENDER%" == "" set "PULSAR_LOG_APPENDER=Console" -if "%PULSAR_LOG_LEVEL%" == "" set "PULSAR_LOG_LEVEL=info" -if "%PULSAR_LOG_ROOT_LEVEL%" == "" set "PULSAR_LOG_ROOT_LEVEL=%PULSAR_LOG_LEVEL%" -if "%PULSAR_LOG_IMMEDIATE_FLUSH%" == "" set "PULSAR_LOG_IMMEDIATE_FLUSH=false" - - -set "OPTS=%OPTS% -Dpulsar.log.appender=%PULSAR_LOG_APPENDER%" -set "OPTS=%OPTS% -Dpulsar.log.dir=%PULSAR_LOG_DIR%" -set "OPTS=%OPTS% -Dpulsar.log.level=%PULSAR_LOG_LEVEL%" -set "OPTS=%OPTS% -Dpulsar.log.root.level=%PULSAR_LOG_ROOT_LEVEL%" -set "OPTS=%OPTS% -Dpulsar.log.immediateFlush=%PULSAR_LOG_IMMEDIATE_FLUSH%" - -set "COMMAND=%1" - -for /f "tokens=1,* delims= " %%a in ("%*") do set "_args=%%b" - -if "%COMMAND%" == "produce" ( - call :execCmdWithConfigFile org.apache.pulsar.testclient.PerformanceProducer - exit /B %ERROR_CODE% -) -if "%COMMAND%" == "consume" ( - call :execCmdWithConfigFile org.apache.pulsar.testclient.PerformanceConsumer - exit /B %ERROR_CODE% -) -if "%COMMAND%" == "transaction" ( - call :execCmdWithConfigFile org.apache.pulsar.testclient.PerformanceTransaction - exit /B %ERROR_CODE% -) -if "%COMMAND%" == "read" ( - call :execCmdWithConfigFile org.apache.pulsar.testclient.PerformanceReader - exit /B %ERROR_CODE% -) -if "%COMMAND%" == "monitor-brokers" ( - call :execCmd org.apache.pulsar.testclient.BrokerMonitor - exit /B %ERROR_CODE% -) -if "%COMMAND%" == "simulation-client" ( - call :execCmd org.apache.pulsar.testclient.LoadSimulationClient - exit /B %ERROR_CODE% -) -if "%COMMAND%" == "simulation-controller" ( - call :execCmd org.apache.pulsar.testclient.LoadSimulationController - exit /B %ERROR_CODE% -) -if "%COMMAND%" == "websocket-producer" ( - call :execCmd org.apache.pulsar.proxy.socket.client.PerformanceClient - exit /B %ERROR_CODE% -) -if "%COMMAND%" == "managed-ledger" ( - call :execCmd org.apache.pulsar.testclient.ManagedLedgerWriter - exit /B %ERROR_CODE% -) -if "%COMMAND%" == "gen-doc" ( - call :execCmd org.apache.pulsar.testclient.CmdGenerateDocumentation - exit /B %ERROR_CODE% -) - -call :usage -exit /B %ERROR_CODE% - -:execCmdWithConfigFile -"%JAVACMD%" %OPTS% %1 --conf-file "%PULSAR_PERFTEST_CONF%" %_args% -if ERRORLEVEL 1 ( - call :error -) -goto :eof - -:execCmd -"%JAVACMD%" %OPTS% %1 %_args% +if "%PULSAR_LOG_FILE%" == "" set "PULSAR_LOG_FILE=pulsar-perftest.log" +call "%PULSAR_HOME%\bin\pulsar-admin-common.cmd" if ERRORLEVEL 1 ( - call :error + exit /b 1 ) -goto :eof - - - -:error -set ERROR_CODE=1 -goto :eof - - - +if "%PULSAR_PERFTEST_CONF%" == "" set "PULSAR_PERFTEST_CONF=%PULSAR_CLIENT_CONF%" +"%JAVACMD%" %OPTS% org.apache.pulsar.testclient.PulsarPerfTestTool "%PULSAR_PERFTEST_CONF%" %* +endlocal -:usage -echo Usage: pulsar-perf COMMAND -echo where command is one of: -echo produce Run a producer -echo consume Run a consumer -echo transaction Run a transaction repeatedly -echo read Run a topic reader -echo websocket-producer Run a websocket producer -echo managed-ledger Write directly on managed-ledgers -echo monitor-brokers Continuously receive broker data and/or load reports -echo simulation-client Run a simulation server acting as a Pulsar client -echo simulation-controller Run a simulation controller to give commands to servers -echo gen-doc Generate documentation automatically. -echo help This help message -echo or command is the full name of a class with a defined main() method. -echo Environment variables: -echo PULSAR_LOG_CONF Log4j configuration file (default %PULSAR_HOME%\logs) -echo PULSAR_CLIENT_CONF Configuration file for client (default: %PULSAR_HOME%\conf\client.conf) -echo PULSAR_EXTRA_OPTS Extra options to be passed to the jvm -echo PULSAR_EXTRA_CLASSPATH Add extra paths to the pulsar classpath -goto error diff --git a/bin/pulsar-shell.cmd b/bin/pulsar-shell.cmd index c339d34289572..615408f9c7a6e 100644 --- a/bin/pulsar-shell.cmd +++ b/bin/pulsar-shell.cmd @@ -18,7 +18,7 @@ @REM @echo off - +setlocal enabledelayedexpansion for %%i in ("%~dp0.") do SET "SCRIPT_PATH=%%~fi" set "PULSAR_HOME_DIR=%SCRIPT_PATH%\..\" for %%i in ("%PULSAR_HOME_DIR%.") do SET "PULSAR_HOME=%%~fi" @@ -26,9 +26,9 @@ call "%PULSAR_HOME%\bin\pulsar-admin-common.cmd" if ERRORLEVEL 1 ( exit /b 1 ) - set "OPTS=%OPTS% -Dorg.jline.terminal.jansi=false" set "OPTS=%OPTS% -Dpulsar.shell.config.default=%cd%" set "DEFAULT_CONFIG=-Dpulsar.shell.config.default="%PULSAR_CLIENT_CONF%"" cd "%PULSAR_HOME%" -"%JAVACMD%" %OPTS% %DEFAULT_CONFIG% org.apache.pulsar.shell.PulsarShell %* \ No newline at end of file +"%JAVACMD%" %OPTS% %DEFAULT_CONFIG% org.apache.pulsar.shell.PulsarShell %* +endlocal \ No newline at end of file diff --git a/bouncy-castle/bc/LICENSE b/bouncy-castle/bc/LICENSE index 5921755346e9e..c95d33d3d1ffb 100644 --- a/bouncy-castle/bc/LICENSE +++ b/bouncy-castle/bc/LICENSE @@ -205,6 +205,5 @@ This projects includes binary packages with the following licenses: Bouncy Castle License * Bouncy Castle -- licenses/LICENSE-bouncycastle.txt - - org.bouncycastle-bcpkix-jdk15on-1.60.jar - - org.bouncycastle-bcprov-jdk15on-1.60.jar - - org.bouncycastle-bcprov-ext-jdk15on-1.60.jar + - org.bouncycastle-bcpkix-jdk18on-1.78.1.jar + - org.bouncycastle-bcprov-jdk18on-1.78.1.jar diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index d5882b4659528..e440923af6de5 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT bouncy-castle-bc @@ -42,13 +41,13 @@ org.bouncycastle - bcpkix-jdk15on + bcpkix-jdk18on ${bouncycastle.version} org.bouncycastle - bcprov-ext-jdk15on + bcprov-ext-jdk18on ${bouncycastle.version} diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index e8348be9292cd..f4478174b86dd 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,8 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT bcfips-include-test diff --git a/bouncy-castle/bcfips-include-test/src/test/java/org/apache/pulsar/client/TlsProducerConsumerBase.java b/bouncy-castle/bcfips-include-test/src/test/java/org/apache/pulsar/client/TlsProducerConsumerBase.java index e8e12838defef..7a97a84bc8413 100644 --- a/bouncy-castle/bcfips-include-test/src/test/java/org/apache/pulsar/client/TlsProducerConsumerBase.java +++ b/bouncy-castle/bcfips-include-test/src/test/java/org/apache/pulsar/client/TlsProducerConsumerBase.java @@ -92,9 +92,7 @@ protected void internalSetUpForNamespace() throws Exception { authParams.put("tlsCertFile", getTlsFileForClient("admin.cert")); authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8")); - if (admin != null) { - admin.close(); - } + closeAdmin(); admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString()) .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).allowTlsInsecureConnection(false) diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index a07e5e19907f2..250b3db6b9b08 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT bouncy-castle-bcfips diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index daefeb83b5371..4d85a163104a2 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT diff --git a/build/build_java_test_image.sh b/build/build_java_test_image.sh index 459bf26f98eff..3869b6688051f 100755 --- a/build/build_java_test_image.sh +++ b/build/build_java_test_image.sh @@ -20,13 +20,6 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd "$SCRIPT_DIR/.." -SQUASH_PARAM="" -# check if docker experimental mode is enabled which is required for -# using "docker build --squash" for squashing all intermediate layers of the build to a single layer -if [[ "$(docker version -f '{{.Server.Experimental}}' 2>/dev/null)" == "true" ]]; then - SQUASH_PARAM="-Ddocker.squash=true" -fi mvn -am -pl tests/docker-images/java-test-image -Pcore-modules,-main,integrationTests,docker \ - -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \ - -Dmaven.test.skip=true -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true $SQUASH_PARAM \ + -Dmaven.test.skip=true -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true \ "$@" install \ No newline at end of file diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile deleted file mode 100644 index 7660325567748..0000000000000 --- a/build/docker/Dockerfile +++ /dev/null @@ -1,98 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -FROM ubuntu:20.04 - -# prepare the directory for pulsar related files -RUN mkdir /pulsar - -RUN apt-get update && \ - apt-get install -y software-properties-common && \ - apt-get update && \ - apt-get install -y tig g++ cmake libssl-dev libcurl4-openssl-dev \ - liblog4cxx-dev google-mock libgtest-dev \ - libboost-dev libboost-program-options-dev libboost-system-dev libboost-python-dev \ - libxml2-utils wget apt-transport-https \ - curl doxygen clang-format \ - gnupg2 golang-go zip unzip libzstd-dev libsnappy-dev wireshark-dev - -# Install Eclipse Temurin Package -RUN mkdir -p /etc/apt/keyrings \ - && wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | tee /etc/apt/keyrings/adoptium.asc \ - && echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list \ - && apt-get update \ - && apt-get -y dist-upgrade \ - && apt-get -y install temurin-17-jdk - -# Compile and install gtest & gmock -RUN cd /usr/src/googletest && \ - cmake . && \ - make && \ - make install - -# Include gtest parallel to speed up unit tests -RUN git clone https://github.com/google/gtest-parallel.git - -# Build protobuf 3.x.y from source since the default protobuf from Ubuntu's apt source is 2.x.y -RUN curl -O -L https://github.com/protocolbuffers/protobuf/releases/download/v3.17.3/protobuf-cpp-3.17.3.tar.gz && \ - tar xvfz protobuf-cpp-3.17.3.tar.gz && \ - cd protobuf-3.17.3/ && \ - CXXFLAGS=-fPIC ./configure && \ - make -j8 && make install && \ - cd .. && rm -rf protobuf-3.17.3/ protobuf-cpp-3.17.3.tar.gz -ENV LD_LIBRARY_PATH /usr/local/lib - -## Website build dependencies - -# Install Ruby-2.4.1 -RUN (curl -sSL https://rvm.io/mpapis.asc | gpg --import -) && \ - (curl -sSL https://rvm.io/pkuczynski.asc | gpg --import -) && \ - (curl -sSL https://get.rvm.io | bash -s stable) -ENV PATH "$PATH:/usr/local/rvm/bin" -RUN rvm install 2.4.1 - -# Install nodejs and yarn -RUN curl -sL https://deb.nodesource.com/setup_12.x | bash - -RUN apt-get install -y nodejs -RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - -RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list -RUN apt-get update && apt-get install yarn - -# Install crowdin -RUN wget https://artifacts.crowdin.com/repo/deb/crowdin.deb -O crowdin.deb -RUN dpkg -i crowdin.deb - -# Install PIP -RUN curl https://bootstrap.pypa.io/get-pip.py | python3 - -RUN pip3 install pdoc -# -# Installation -ARG MAVEN_VERSION=3.6.3 -ARG MAVEN_FILENAME="apache-maven-${MAVEN_VERSION}-bin.tar.gz" -ARG MAVEN_HOME=/opt/maven -ARG MAVEN_URL="http://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/${MAVEN_FILENAME}" -ARG MAVEN_TMP="/tmp/${MAVEN_FILENAME}" -RUN wget --no-verbose -O ${MAVEN_TMP} ${MAVEN_URL} - -# Cleanup -RUN tar xzf ${MAVEN_TMP} -C /opt/ \ - && ln -s /opt/apache-maven-${MAVEN_VERSION} ${MAVEN_HOME} \ - && ln -s ${MAVEN_HOME}/bin/mvn /usr/local/bin - -RUN unset MAVEN_VERSION diff --git a/build/docker/publish.sh b/build/docker/publish.sh deleted file mode 100755 index 6bfa56bace6d8..0000000000000 --- a/build/docker/publish.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -ROOT_DIR=$(git rev-parse --show-toplevel) -cd $ROOT_DIR/build/docker - -if [ -z "$DOCKER_USER" ]; then - echo "Docker user in variable \$DOCKER_USER was not set. Skipping image publishing" - exit 1 -fi - -if [ -z "$DOCKER_PASSWORD" ]; then - echo "Docker password in variable \$DOCKER_PASSWORD was not set. Skipping image publishing" - exit 1 -fi - -DOCKER_ORG="${DOCKER_ORG:-apachepulsar}" - -docker login ${DOCKER_REGISTRY} -u="$DOCKER_USER" -p="$DOCKER_PASSWORD" -if [ $? -ne 0 ]; then - echo "Failed to loging to Docker Hub" - exit 1 -fi - -if [[ -z ${DOCKER_REGISTRY} ]]; then - docker_registry_org=${DOCKER_ORG} -else - docker_registry_org=${DOCKER_REGISTRY}/${DOCKER_ORG} - echo "Starting to push images to ${docker_registry_org}..." -fi - -set -x - -# Fail if any of the subsequent commands fail -set -e - -# Push all images and tags -docker push ${docker_registry_org}/pulsar-build:ubuntu-16.04 - -echo "Finished pushing images to ${docker_registry_org}" diff --git a/build/pulsar_ci_tool.sh b/build/pulsar_ci_tool.sh index d946edd395789..3d63f104cd500 100755 --- a/build/pulsar_ci_tool.sh +++ b/build/pulsar_ci_tool.sh @@ -46,8 +46,7 @@ function ci_print_thread_dumps() { # runs maven function _ci_mvn() { - mvn -B -ntp -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \ - "$@" + mvn -B -ntp "$@" } # runs OWASP Dependency Check for all projects @@ -55,40 +54,6 @@ function ci_dependency_check() { _ci_mvn -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl '!pulsar-client-tools-test' "$@" } -function ci_pick_ubuntu_mirror() { - echo "Choosing fastest up-to-date ubuntu mirror based on download speed..." - UBUNTU_MIRROR=$({ - # choose mirrors that are up-to-date by checking the Last-Modified header for - { - # randomly choose up to 10 mirrors using http:// protocol - # (https isn't supported in docker containers that don't have ca-certificates installed) - curl -s http://mirrors.ubuntu.com/mirrors.txt | grep '^http://' | shuf -n 10 - # also consider Azure's Ubuntu mirror - echo http://azure.archive.ubuntu.com/ubuntu/ - } | xargs -I {} sh -c 'echo "$(curl -m 5 -sI {}dists/$(lsb_release -c | cut -f2)-security/Contents-$(dpkg --print-architecture).gz|sed s/\\r\$//|grep Last-Modified|awk -F": " "{ print \$2 }" | LANG=C date -f- -u +%s)" "{}"' | sort -rg | awk '{ if (NR==1) TS=$1; if ($1 == TS) print $2 }' - } | xargs -I {} sh -c 'echo `curl -r 0-102400 -m 5 -s -w %{speed_download} -o /dev/null {}ls-lR.gz` {}' \ - |sort -g -r |head -1| awk '{ print $2 }') - if [ -z "$UBUNTU_MIRROR" ]; then - # fallback to full mirrors list - UBUNTU_MIRROR="mirror://mirrors.ubuntu.com/mirrors.txt" - fi - OLD_MIRROR=$(cat /etc/apt/sources.list | grep '^deb ' | head -1 | awk '{ print $2 }') - echo "Picked '$UBUNTU_MIRROR'. Current mirror is '$OLD_MIRROR'." - if [[ "$OLD_MIRROR" != "$UBUNTU_MIRROR" ]]; then - sudo sed -i "s|$OLD_MIRROR|$UBUNTU_MIRROR|g" /etc/apt/sources.list - sudo apt-get update - fi - # set the chosen mirror also in the UBUNTU_MIRROR and UBUNTU_SECURITY_MIRROR environment variables - # that can be used by docker builds - export UBUNTU_MIRROR - export UBUNTU_SECURITY_MIRROR=$UBUNTU_MIRROR - # make environment variables available for later GitHub Actions steps - if [ -n "$GITHUB_ENV" ]; then - echo "UBUNTU_MIRROR=$UBUNTU_MIRROR" >> $GITHUB_ENV - echo "UBUNTU_SECURITY_MIRROR=$UBUNTU_SECURITY_MIRROR" >> $GITHUB_ENV - fi -} - # installs a tool executable if it's not found on the PATH function ci_install_tool() { local tool_executable=$1 @@ -98,7 +63,6 @@ function ci_install_tool() { echo "::group::Installing ${tool_package}" sudo apt-get -y install ${tool_package} >/dev/null || { echo "Installing the package failed. Switching the ubuntu mirror and retrying..." - ci_pick_ubuntu_mirror # retry after picking the ubuntu mirror sudo apt-get -y install ${tool_package} } @@ -389,6 +353,7 @@ _ci_upload_coverage_files() { --transform="flags=r;s|\\(/jacoco.*\\).exec$|\\1_${testtype}_${testgroup}.exec|" \ --transform="flags=r;s|\\(/tmp/jacocoDir/.*\\).exec$|\\1_${testtype}_${testgroup}.exec|" \ --exclude="*/META-INF/bundled-dependencies/*" \ + --exclude="*/META-INF/versions/*" \ $GITHUB_WORKSPACE/target/classpath_* \ $(find "$GITHUB_WORKSPACE" -path "*/target/jacoco*.exec" -printf "%p\n%h/classes\n" | sort | uniq) \ $([ -d /tmp/jacocoDir ] && echo "/tmp/jacocoDir" ) \ @@ -530,11 +495,11 @@ ci_create_test_coverage_report() { local classfilesArgs="--classfiles $({ { for classpathEntry in $(cat $completeClasspathFile | { grep -v -f $filterArtifactsFile || true; } | sort | uniq | { grep -v -E "$excludeJarsPattern" || true; }); do - if [[ -f $classpathEntry && -n "$(unzip -Z1C $classpathEntry 'META-INF/bundled-dependencies/*' 2>/dev/null)" ]]; then - # file must be processed by removing META-INF/bundled-dependencies + if [[ -f $classpathEntry && -n "$(unzip -Z1C $classpathEntry 'META-INF/bundled-dependencies/*' 'META-INF/versions/*' 2>/dev/null)" ]]; then + # file must be processed by removing META-INF/bundled-dependencies and META-INF/versions local jartempfile=$(mktemp -t jarfile.XXXX --suffix=.jar) cp $classpathEntry $jartempfile - zip -q -d $jartempfile 'META-INF/bundled-dependencies/*' &> /dev/null + zip -q -d $jartempfile 'META-INF/bundled-dependencies/*' 'META-INF/versions/*' &> /dev/null echo $jartempfile else echo $classpathEntry @@ -596,7 +561,7 @@ ci_create_inttest_coverage_report() { # remove jar file that causes duplicate classes issue rm /tmp/jacocoDir/pulsar_lib/org.apache.pulsar-bouncy-castle* || true # remove any bundled dependencies as part of .jar/.nar files - find /tmp/jacocoDir/pulsar_lib '(' -name "*.jar" -or -name "*.nar" ')' -exec echo "Processing {}" \; -exec zip -q -d {} 'META-INF/bundled-dependencies/*' \; |grep -E -v "Nothing to do|^$" || true + find /tmp/jacocoDir/pulsar_lib '(' -name "*.jar" -or -name "*.nar" ')' -exec echo "Processing {}" \; -exec zip -q -d {} 'META-INF/bundled-dependencies/*' 'META-INF/versions/*' \; |grep -E -v "Nothing to do|^$" || true fi # projects that aren't considered as production code and their own src/main/java source code shouldn't be analysed local excludeProjectsPattern="testmocks|testclient|buildtools" diff --git a/build/run_integration_group.sh b/build/run_integration_group.sh index bc1255d8d68aa..2d82fce08878d 100755 --- a/build/run_integration_group.sh +++ b/build/run_integration_group.sh @@ -26,7 +26,8 @@ set -o errexit JAVA_MAJOR_VERSION="$(java -version 2>&1 |grep " version " | awk -F\" '{ print $2 }' | awk -F. '{ if ($1=="1") { print $2 } else { print $1 } }')" # Used to shade run test on Java 8, because the latest TestNG requires Java 11 or higher. -TESTNG_VERSION="7.3.0" +TESTNG_VERSION_JAVA_8="7.3.0" +MOCKITO_VERSION_JAVA_8="4.11.0" # lists all active maven modules with given parameters # parses the modules from the "mvn initialize" output @@ -112,7 +113,11 @@ test_group_shade() { } test_group_shade_build() { - mvn_run_integration_test --build-only "$@" -DShadeTests -DtestForkCount=1 -DtestReuseFork=false + local additional_args + if [[ $JAVA_MAJOR_VERSION -ge 8 && $JAVA_MAJOR_VERSION -lt 11 ]]; then + additional_args="$additional_args -Dtestng.version=$TESTNG_VERSION_JAVA_8 -Dmockito.version=$MOCKITO_VERSION_JAVA_8" + fi + mvn_run_integration_test --build-only "$@" -DShadeTests -DtestForkCount=1 -DtestReuseFork=false $additional_args } test_group_shade_run() { @@ -122,7 +127,7 @@ test_group_shade_run() { fi if [[ $JAVA_MAJOR_VERSION -ge 8 && $JAVA_MAJOR_VERSION -lt 11 ]]; then - additional_args="$additional_args -Dtestng.version=$TESTNG_VERSION" + additional_args="$additional_args -Dtestng.version=$TESTNG_VERSION_JAVA_8 -Dmockito.version=$MOCKITO_VERSION_JAVA_8" fi mvn_run_integration_test --skip-build-deps --clean "$@" -Denforcer.skip=true -DShadeTests -DtestForkCount=1 -DtestReuseFork=false $additional_args @@ -156,6 +161,10 @@ test_group_messaging() { mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-tls.xml -DintegrationTests } +test_group_loadbalance() { + mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-loadbalance.xml -DintegrationTests +} + test_group_plugin() { mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-plugin.xml -DintegrationTests } @@ -172,6 +181,10 @@ test_group_transaction() { mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-transaction.xml -DintegrationTests } +test_group_metrics() { + mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-metrics.xml -DintegrationTests +} + test_group_tiered_filesystem() { mvn_run_integration_test "$@" -DintegrationTestSuiteFile=tiered-filesystem-storage.xml -DintegrationTests } diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh index ba49820ed1d33..cdaf69e351b6d 100755 --- a/build/run_unit_group.sh +++ b/build/run_unit_group.sh @@ -80,11 +80,17 @@ function test_group_broker_group_1() { } function test_group_broker_group_2() { - mvn_test -pl pulsar-broker -Dgroups='schema,utils,functions-worker,broker-io,broker-discovery,broker-compaction,broker-naming,websocket,other' + mvn_test -pl pulsar-broker -Dgroups='schema,utils,functions-worker,broker-io,broker-discovery,broker-compaction,broker-naming,broker-replication,websocket,other' } function test_group_broker_group_3() { mvn_test -pl pulsar-broker -Dgroups='broker-admin' + # run AdminApiTransactionMultiBrokerTest independently with a larger heap size + mvn_test -pl pulsar-broker -DtestMaxHeapSize=1500M -Dtest=org.apache.pulsar.broker.admin.v3.AdminApiTransactionMultiBrokerTest -DtestForkCount=1 -DtestReuseFork=false +} + +function test_group_broker_group_4() { + mvn_test -pl pulsar-broker -Dgroups='cluster-migration' } function test_group_broker_client_api() { @@ -99,6 +105,10 @@ function test_group_client() { mvn_test -pl pulsar-client } +function test_group_metadata() { + mvn_test -pl pulsar-metadata -DtestReuseFork=false +} + # prints summaries of failed tests to console # by using the targer/surefire-reports files # works only when testForkCount > 1 since that is when surefire will create reports for individual test classes @@ -131,13 +141,21 @@ function print_testng_failures() { function test_group_broker_flaky() { echo "::endgroup::" echo "::group::Running quarantined tests" - mvn_test --no-fail-fast -pl pulsar-broker -Dgroups='quarantine' -DexcludedGroups='' -DfailIfNoTests=false \ + mvn_test --no-fail-fast -pl pulsar-broker -Dgroups='quarantine' -DexcludedGroups='flaky' -DfailIfNoTests=false \ -DtestForkCount=2 || print_testng_failures pulsar-broker/target/surefire-reports/testng-failed.xml "Quarantined test failure in" "Quarantined test failures" echo "::endgroup::" echo "::group::Running flaky tests" - mvn_test --no-fail-fast -pl pulsar-broker -Dgroups='flaky' -DtestForkCount=2 + mvn_test --no-fail-fast -pl pulsar-broker -Dgroups='flaky' -DexcludedGroups='quarantine' -DtestForkCount=2 echo "::endgroup::" + local modules_with_flaky_tests=$(git grep -l '@Test.*"flaky"' | grep '/src/test/java/' | \ + awk -F '/src/test/java/' '{ print $1 }' | grep -v -E 'pulsar-broker' | sort | uniq | \ + perl -0777 -p -e 's/\n(\S)/,$1/g') + if [ -n "${modules_with_flaky_tests}" ]; then + echo "::group::Running flaky tests in modules '${modules_with_flaky_tests}'" + mvn_test --no-fail-fast -pl "${modules_with_flaky_tests}" -Dgroups='flaky' -DexcludedGroups='quarantine' -DfailIfNoTests=false + echo "::endgroup::" + fi } function test_group_proxy() { @@ -152,7 +170,7 @@ function test_group_proxy() { function test_group_other() { mvn_test --clean --install \ -pl '!org.apache.pulsar:distribution,!org.apache.pulsar:pulsar-offloader-distribution,!org.apache.pulsar:pulsar-server-distribution,!org.apache.pulsar:pulsar-io-distribution,!org.apache.pulsar:pulsar-all-docker-image' \ - -PskipTestsForUnitGroupOther -DdisableIoMainProfile=true -DdisableSqlMainProfile=true -DskipIntegrationTests \ + -PskipTestsForUnitGroupOther -DdisableIoMainProfile=true -DskipIntegrationTests \ -Dexclude='**/ManagedLedgerTest.java, **/OffloadersCacheTest.java **/PrimitiveSchemaTest.java, @@ -167,11 +185,11 @@ function test_group_other() { echo "::endgroup::" local modules_with_quarantined_tests=$(git grep -l '@Test.*"quarantine"' | grep '/src/test/java/' | \ - awk -F '/src/test/java/' '{ print $1 }' | grep -v -E 'pulsar-broker|pulsar-proxy|pulsar-io|pulsar-sql|pulsar-client' | sort | uniq | \ + awk -F '/src/test/java/' '{ print $1 }' | grep -v -E 'pulsar-broker|pulsar-proxy|pulsar-io|pulsar-client' | sort | uniq | \ perl -0777 -p -e 's/\n(\S)/,$1/g') if [ -n "${modules_with_quarantined_tests}" ]; then echo "::group::Running quarantined tests outside of pulsar-broker & pulsar-proxy (if any)" - mvn_test --no-fail-fast -pl "${modules_with_quarantined_tests}" test -Dgroups='quarantine' -DexcludedGroups='' \ + mvn_test --no-fail-fast -pl "${modules_with_quarantined_tests}" test -Dgroups='quarantine' -DexcludedGroups='flaky' \ -DfailIfNoTests=false || \ echo "::warning::There were test failures in the 'quarantine' test group." echo "::endgroup::" @@ -182,9 +200,17 @@ function test_group_pulsar_io() { echo "::group::Running pulsar-io tests" mvn_test --install -Ppulsar-io-tests,-main echo "::endgroup::" +} + +function test_group_pulsar_io_elastic() { + echo "::group::Running elastic-search tests" + mvn_test --install -Ppulsar-io-elastic-tests,-main + echo "::endgroup::" +} - echo "::group::Running pulsar-sql tests" - mvn_test --install -Ppulsar-sql-tests,-main -DtestForkCount=1 +function test_group_pulsar_io_kafka_connect() { + echo "::group::Running Pulsar IO Kafka connect adaptor tests" + mvn_test --install -Ppulsar-io-kafka-connect-tests,-main echo "::endgroup::" } diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 9e4ac022024e1..b1ae0cd9b73f3 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,28 +31,28 @@ org.apache.pulsar buildtools - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT jar Pulsar Build Tools - 2023-05-03T02:53:27Z + 2024-08-09T08:42:01Z 1.8 1.8 3.1.0 - 2.18.0 - 1.7.32 + 2.23.1 + 2.0.13 7.7.1 3.11 4.1 - 8.37 + 10.14.2 3.1.2 - 4.1.89.Final + 4.1.104.Final 4.2.3 - 31.0.1-jre + 32.1.2-jre 1.10.12 2.0 - 3.12.4 + 5.6.0 --add-opens java.base/jdk.internal.loader=ALL-UNNAMED @@ -63,6 +63,13 @@ + + org.slf4j + slf4j-bom + ${slf4j.version} + pom + import + org.apache.logging.log4j log4j-bom @@ -100,6 +107,12 @@ org.testng testng ${testng.version} + + + org.slf4j + * + + org.apache.logging.log4j @@ -111,12 +124,11 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl org.slf4j jcl-over-slf4j - ${slf4j.version} org.apache.commons @@ -141,11 +153,6 @@ mockito-core ${mockito.version} - - org.mockito - mockito-inline - ${mockito.version} - diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/AnnotationListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/AnnotationListener.java index 38cd2a1747a63..0c464fd97a970 100644 --- a/buildtools/src/main/java/org/apache/pulsar/tests/AnnotationListener.java +++ b/buildtools/src/main/java/org/apache/pulsar/tests/AnnotationListener.java @@ -32,6 +32,10 @@ public class AnnotationListener implements IAnnotationTransformer { private static final long DEFAULT_TEST_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(5); private static final String OTHER_GROUP = "other"; + private static final String FLAKY_GROUP = "flaky"; + + private static final String QUARANTINE_GROUP = "quarantine"; + public AnnotationListener() { System.out.println("Created annotation listener"); } @@ -51,9 +55,27 @@ public void transform(ITestAnnotation annotation, annotation.setTimeOut(DEFAULT_TEST_TIMEOUT_MILLIS); } + replaceGroupsIfFlakyOrQuarantineGroupIsIncluded(annotation); addToOtherGroupIfNoGroupsSpecified(annotation); } + // A test method will inherit the test groups from the class level and this solution ensures that a test method + // added to the flaky or quarantine group will not be executed as part of other groups. + private void replaceGroupsIfFlakyOrQuarantineGroupIsIncluded(ITestAnnotation annotation) { + if (annotation.getGroups() != null && annotation.getGroups().length > 1) { + for (String group : annotation.getGroups()) { + if (group.equals(QUARANTINE_GROUP)) { + annotation.setGroups(new String[]{QUARANTINE_GROUP}); + return; + } + if (group.equals(FLAKY_GROUP)) { + annotation.setGroups(new String[]{FLAKY_GROUP}); + return; + } + } + } + } + private void addToOtherGroupIfNoGroupsSpecified(ITestOrConfiguration annotation) { // Add test to "other" group if there's no specified group if (annotation.getGroups() == null || annotation.getGroups().length == 0) { diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/FailFastNotifier.java b/buildtools/src/main/java/org/apache/pulsar/tests/FailFastNotifier.java index 627a4ec30547b..fe76a79b2c4ce 100644 --- a/buildtools/src/main/java/org/apache/pulsar/tests/FailFastNotifier.java +++ b/buildtools/src/main/java/org/apache/pulsar/tests/FailFastNotifier.java @@ -124,8 +124,7 @@ public void beforeInvocation(IInvokedMethod iInvokedMethod, ITestResult iTestRes || iTestNGMethod.isAfterTestConfiguration())) { throw new FailFastSkipException("Skipped after failure since testFailFast system property is set."); } - } - if (FAIL_FAST_KILLSWITCH_FILE != null && FAIL_FAST_KILLSWITCH_FILE.exists()) { + } else if (FAIL_FAST_KILLSWITCH_FILE != null && FAIL_FAST_KILLSWITCH_FILE.exists()) { throw new FailFastSkipException("Skipped after failure since kill switch file exists."); } } diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/HeapDumpUtil.java b/buildtools/src/main/java/org/apache/pulsar/tests/HeapDumpUtil.java new file mode 100644 index 0000000000000..555e312e1f87c --- /dev/null +++ b/buildtools/src/main/java/org/apache/pulsar/tests/HeapDumpUtil.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests; + +import com.sun.management.HotSpotDiagnosticMXBean; +import java.io.File; +import java.lang.management.ManagementFactory; +import javax.management.MBeanServer; + +public class HeapDumpUtil { + private static final String HOTSPOT_BEAN_NAME = "com.sun.management:type=HotSpotDiagnostic"; + + // Utility method to get the HotSpotDiagnosticMXBean + private static HotSpotDiagnosticMXBean getHotSpotDiagnosticMXBean() { + try { + MBeanServer server = ManagementFactory.getPlatformMBeanServer(); + return ManagementFactory.newPlatformMXBeanProxy(server, HOTSPOT_BEAN_NAME, HotSpotDiagnosticMXBean.class); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Dump the heap of the JVM. + * + * @param file the system-dependent filename + * @param liveObjects if true dump only live objects i.e. objects that are reachable from others + */ + public static void dumpHeap(File file, boolean liveObjects) { + try { + HotSpotDiagnosticMXBean hotspotMBean = getHotSpotDiagnosticMXBean(); + hotspotMBean.dumpHeap(file.getAbsolutePath(), liveObjects); + } catch (Exception e) { + throw new RuntimeException("Error generating heap dump", e); + } + } +} diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/HeapHistogramUtil.java b/buildtools/src/main/java/org/apache/pulsar/tests/HeapHistogramUtil.java new file mode 100644 index 0000000000000..53d66400ae755 --- /dev/null +++ b/buildtools/src/main/java/org/apache/pulsar/tests/HeapHistogramUtil.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.tests; + +import java.lang.management.ManagementFactory; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import javax.management.JMException; +import javax.management.ObjectName; + +public class HeapHistogramUtil { + public static String buildHeapHistogram() { + StringBuilder dump = new StringBuilder(); + dump.append(String.format("Timestamp: %s", DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now()))); + dump.append("\n\n"); + try { + dump.append(callDiagnosticCommand("gcHeapInfo")); + } catch (Exception e) { + e.printStackTrace(); + return null; + } + dump.append("\n"); + try { + dump.append(callDiagnosticCommand("gcClassHistogram")); + } catch (Exception e) { + e.printStackTrace(); + return null; + } + return dump.toString(); + } + + /** + * Calls a diagnostic commands. + * The available operations are similar to what the jcmd commandline tool has, + * however the naming of the operations are different. The "help" operation can be used + * to find out the available operations. For example, the jcmd command "Thread.print" maps + * to "threadPrint" operation name. + */ + static String callDiagnosticCommand(String operationName, String... args) + throws JMException { + return (String) ManagementFactory.getPlatformMBeanServer() + .invoke(new ObjectName("com.sun.management:type=DiagnosticCommand"), + operationName, new Object[] {args}, new String[] {String[].class.getName()}); + } +} diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/PulsarTestListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/PulsarTestListener.java index b3d70621843ca..2d1f1273272c5 100644 --- a/buildtools/src/main/java/org/apache/pulsar/tests/PulsarTestListener.java +++ b/buildtools/src/main/java/org/apache/pulsar/tests/PulsarTestListener.java @@ -44,20 +44,29 @@ public void onTestFailure(ITestResult result) { if (!(result.getThrowable() instanceof SkipException)) { System.out.format("!!!!!!!!! FAILURE-- %s.%s(%s)-------\n", result.getTestClass(), result.getMethod().getMethodName(), Arrays.toString(result.getParameters())); - } - if (result.getThrowable() != null) { - result.getThrowable().printStackTrace(); - if (result.getThrowable() instanceof ThreadTimeoutException) { - System.out.println("====== THREAD DUMPS ======"); - System.out.println(ThreadDumpUtil.buildThreadDiagnosticString()); + if (result.getThrowable() != null) { + result.getThrowable().printStackTrace(); + if (result.getThrowable() instanceof ThreadTimeoutException) { + System.out.println("====== THREAD DUMPS ======"); + System.out.println(ThreadDumpUtil.buildThreadDiagnosticString()); + } } } } @Override public void onTestSkipped(ITestResult result) { - System.out.format("~~~~~~~~~ SKIPPED -- %s.%s(%s)-------\n", result.getTestClass(), - result.getMethod().getMethodName(), Arrays.toString(result.getParameters())); + if (!(result.getThrowable() instanceof SkipException)) { + System.out.format("~~~~~~~~~ SKIPPED -- %s.%s(%s)-------\n", result.getTestClass(), + result.getMethod().getMethodName(), Arrays.toString(result.getParameters())); + if (result.getThrowable() != null) { + result.getThrowable().printStackTrace(); + if (result.getThrowable() instanceof ThreadTimeoutException) { + System.out.println("====== THREAD DUMPS ======"); + System.out.println(ThreadDumpUtil.buildThreadDiagnosticString()); + } + } + } } @Override diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/ThreadDumpUtil.java b/buildtools/src/main/java/org/apache/pulsar/tests/ThreadDumpUtil.java index e20a02f95a992..6484600bad4f5 100644 --- a/buildtools/src/main/java/org/apache/pulsar/tests/ThreadDumpUtil.java +++ b/buildtools/src/main/java/org/apache/pulsar/tests/ThreadDumpUtil.java @@ -25,7 +25,7 @@ import java.lang.management.MonitorInfo; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; -import java.time.LocalDateTime; +import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.Map; import javax.management.JMException; @@ -65,7 +65,7 @@ static String buildThreadDump() { // fallback to using JMX for creating the thread dump StringBuilder dump = new StringBuilder(); - dump.append(String.format("Timestamp: %s", DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(LocalDateTime.now()))); + dump.append(String.format("Timestamp: %s", DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now()))); dump.append("\n\n"); Map stackTraces = Thread.getAllStackTraces(); diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/ThreadLeakDetectorListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/ThreadLeakDetectorListener.java index 803d1c4980bc9..0757730423688 100644 --- a/buildtools/src/main/java/org/apache/pulsar/tests/ThreadLeakDetectorListener.java +++ b/buildtools/src/main/java/org/apache/pulsar/tests/ThreadLeakDetectorListener.java @@ -16,55 +16,240 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.pulsar.tests; +import com.google.common.base.Charsets; +import com.google.common.io.Files; +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.lang.reflect.Field; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.util.Collections; import java.util.LinkedHashSet; import java.util.Objects; import java.util.Set; +import java.util.concurrent.ForkJoinWorkerThread; import java.util.stream.Collectors; import org.apache.commons.lang3.ThreadUtils; +import org.apache.commons.lang3.mutable.MutableBoolean; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Detects new threads that have been created during the test execution. + * Detects new threads that have been created during the test execution. This is useful to detect thread leaks. + * Will create files to the configured directory if new threads are detected and THREAD_LEAK_DETECTOR_WAIT_MILLIS + * is set to a positive value. A recommended value is 10000 for THREAD_LEAK_DETECTOR_WAIT_MILLIS. This will ensure + * that any asynchronous operations should have completed before the detector determines that it has found a leak. */ public class ThreadLeakDetectorListener extends BetweenTestClassesListenerAdapter { private static final Logger LOG = LoggerFactory.getLogger(ThreadLeakDetectorListener.class); + private static final long WAIT_FOR_THREAD_TERMINATION_MILLIS = + Long.parseLong(System.getenv().getOrDefault("THREAD_LEAK_DETECTOR_WAIT_MILLIS", "0")); + private static final File DUMP_DIR = + new File(System.getenv().getOrDefault("THREAD_LEAK_DETECTOR_DIR", "target/thread-leak-dumps")); + private static final long THREAD_TERMINATION_POLL_INTERVAL = + Long.parseLong(System.getenv().getOrDefault("THREAD_LEAK_DETECTOR_POLL_INTERVAL", "250")); + private static final boolean COLLECT_THREADDUMP = + Boolean.parseBoolean(System.getenv().getOrDefault("THREAD_LEAK_DETECTOR_COLLECT_THREADDUMP", "true")); private Set capturedThreadKeys; + private static final Field THREAD_TARGET_FIELD; + static { + Field targetField = null; + try { + targetField = Thread.class.getDeclaredField("target"); + targetField.setAccessible(true); + } catch (NoSuchFieldException e) { + // ignore this error. on Java 21, the field is not present + // TODO: add support for extracting the Runnable target on Java 21 + } + THREAD_TARGET_FIELD = targetField; + } + @Override protected void onBetweenTestClasses(Class endedTestClass, Class startedTestClass) { LOG.info("Capturing identifiers of running threads."); - capturedThreadKeys = compareThreads(capturedThreadKeys, endedTestClass); + MutableBoolean differenceDetected = new MutableBoolean(); + Set currentThreadKeys = + compareThreads(capturedThreadKeys, endedTestClass, WAIT_FOR_THREAD_TERMINATION_MILLIS <= 0, + differenceDetected, null); + if (WAIT_FOR_THREAD_TERMINATION_MILLIS > 0 && endedTestClass != null && differenceDetected.booleanValue()) { + LOG.info("Difference detected in active threads. Waiting up to {} ms for threads to terminate.", + WAIT_FOR_THREAD_TERMINATION_MILLIS); + long endTime = System.currentTimeMillis() + WAIT_FOR_THREAD_TERMINATION_MILLIS; + while (System.currentTimeMillis() < endTime) { + try { + Thread.sleep(THREAD_TERMINATION_POLL_INTERVAL); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + differenceDetected.setFalse(); + currentThreadKeys = compareThreads(capturedThreadKeys, endedTestClass, false, differenceDetected, null); + if (!differenceDetected.booleanValue()) { + break; + } + } + if (differenceDetected.booleanValue()) { + String datetimePart = + DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss.SSS").format(ZonedDateTime.now()); + PrintWriter out = null; + try { + if (!DUMP_DIR.exists()) { + DUMP_DIR.mkdirs(); + } + File threadleakdumpFile = + new File(DUMP_DIR, "threadleak" + datetimePart + endedTestClass.getName() + ".txt"); + out = new PrintWriter(threadleakdumpFile); + } catch (IOException e) { + LOG.error("Cannot write thread leak dump", e); + } + currentThreadKeys = compareThreads(capturedThreadKeys, endedTestClass, true, null, out); + if (out != null) { + out.close(); + } + if (COLLECT_THREADDUMP) { + File threaddumpFile = + new File(DUMP_DIR, "threaddump" + datetimePart + endedTestClass.getName() + ".txt"); + try { + Files.asCharSink(threaddumpFile, Charsets.UTF_8) + .write(ThreadDumpUtil.buildThreadDiagnosticString()); + } catch (IOException e) { + LOG.error("Cannot write thread dump", e); + } + } + } + } + capturedThreadKeys = currentThreadKeys; } - private static Set compareThreads(Set previousThreadKeys, Class endedTestClass) { + private static Set compareThreads(Set previousThreadKeys, Class endedTestClass, + boolean logDifference, MutableBoolean differenceDetected, + PrintWriter out) { Set threadKeys = Collections.unmodifiableSet(ThreadUtils.getAllThreads().stream() + .filter(thread -> !shouldSkipThread(thread)) .map(ThreadKey::of) .collect(Collectors.>toCollection(LinkedHashSet::new))); if (endedTestClass != null && previousThreadKeys != null) { int newThreadsCounter = 0; - LOG.info("Checking for new threads created by {}.", endedTestClass.getName()); for (ThreadKey threadKey : threadKeys) { if (!previousThreadKeys.contains(threadKey)) { newThreadsCounter++; - LOG.warn("Tests in class {} created thread id {} with name '{}'", endedTestClass.getSimpleName(), - threadKey.getThreadId(), threadKey.getThreadName()); + if (differenceDetected != null) { + differenceDetected.setTrue(); + } + if (logDifference || out != null) { + String message = String.format("Tests in class %s created thread id %d with name '%s'", + endedTestClass.getSimpleName(), + threadKey.getThreadId(), threadKey.getThreadName()); + if (logDifference) { + LOG.warn(message); + } + if (out != null) { + out.println(message); + } + } } } - if (newThreadsCounter > 0) { - LOG.warn("Summary: Tests in class {} created {} new threads", endedTestClass.getName(), - newThreadsCounter); + if (newThreadsCounter > 0 && (logDifference || out != null)) { + String message = String.format( + "Summary: Tests in class %s created %d new threads. There are now %d threads in total.", + endedTestClass.getName(), newThreadsCounter, threadKeys.size()); + if (logDifference) { + LOG.warn(message); + } + if (out != null) { + out.println(message); + } } } return threadKeys; } + private static boolean shouldSkipThread(Thread thread) { + // skip ForkJoinPool threads + if (thread instanceof ForkJoinWorkerThread) { + return true; + } + // skip Testcontainers threads + final ThreadGroup threadGroup = thread.getThreadGroup(); + if (threadGroup != null && "testcontainers".equals(threadGroup.getName())) { + return true; + } + String threadName = thread.getName(); + if (threadName != null) { + // skip ClientTestFixtures.SCHEDULER threads + if (threadName.startsWith("ClientTestFixtures-SCHEDULER-")) { + return true; + } + // skip JVM internal threads related to java.lang.Process + if (threadName.equals("process reaper")) { + return true; + } + // skip JVM internal thread used for CompletableFuture.delayedExecutor + if (threadName.equals("CompletableFutureDelayScheduler")) { + return true; + } + // skip threadpool created in dev.failsafe.internal.util.DelegatingScheduler + if (threadName.equals("FailsafeDelayScheduler")) { + return true; + } + // skip Okio Watchdog thread and interrupt it + if (threadName.equals("Okio Watchdog")) { + return true; + } + // skip OkHttp TaskRunner thread + if (threadName.equals("OkHttp TaskRunner")) { + return true; + } + // skip JNA background thread + if (threadName.equals("JNA Cleaner")) { + return true; + } + // skip org.glassfish.grizzly.http.server.DefaultSessionManager thread pool + if (threadName.equals("Grizzly-HttpSession-Expirer")) { + return true; + } + // Testcontainers AbstractWaitStrategy.EXECUTOR + if (threadName.startsWith("testcontainers-wait-")) { + return true; + } + // org.rnorth.ducttape.timeouts.Timeouts.EXECUTOR_SERVICE thread pool, used by Testcontainers + if (threadName.startsWith("ducttape-")) { + return true; + } + } + Runnable target = extractRunnableTarget(thread); + if (target != null) { + String targetClassName = target.getClass().getName(); + // ignore threads that contain a Runnable class under org.testcontainers package + if (targetClassName.startsWith("org.testcontainers.")) { + return true; + } + } + return false; + } + + // use reflection to extract the Runnable target from a thread so that we can detect threads created by + // Testcontainers based on the Runnable's class name. + private static Runnable extractRunnableTarget(Thread thread) { + if (THREAD_TARGET_FIELD == null) { + return null; + } + Runnable target = null; + try { + target = (Runnable) THREAD_TARGET_FIELD.get(thread); + } catch (IllegalAccessException e) { + LOG.warn("Cannot access target field in Thread.class", e); + } + return target; + } + /** * Unique key for a thread * Based on thread id and it's identity hash code diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/TraceTestResourceCleanupListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/TraceTestResourceCleanupListener.java new file mode 100644 index 0000000000000..5692d94bcf15b --- /dev/null +++ b/buildtools/src/main/java/org/apache/pulsar/tests/TraceTestResourceCleanupListener.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.tests; + +import com.google.common.base.Charsets; +import com.google.common.io.Files; +import java.io.File; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import org.testng.IExecutionListener; + +/** + * A TestNG listener that traces test resource cleanup by creating a thread dump, heap histogram and heap dump + * (when mode is 'full') before the TestNG JVM exits. + * The heap dump could help detecting memory leaks in tests or the sources of resource leaks that cannot be + * detected with the ThreadLeakDetectorListener. + */ +public class TraceTestResourceCleanupListener implements IExecutionListener { + enum TraceTestResourceCleanupMode { + OFF, + ON, + FULL // includes heap dump + } + + private static final TraceTestResourceCleanupMode MODE = + TraceTestResourceCleanupMode.valueOf( + System.getenv().getOrDefault("TRACE_TEST_RESOURCE_CLEANUP", "off").toUpperCase()); + private static final File DUMP_DIR = new File( + System.getenv().getOrDefault("TRACE_TEST_RESOURCE_CLEANUP_DIR", "target/trace-test-resource-cleanup")); + private static final long WAIT_BEFORE_DUMP_MILLIS = + Long.parseLong(System.getenv().getOrDefault("TRACE_TEST_RESOURCE_CLEANUP_DELAY", "5000")); + + static { + if (MODE != TraceTestResourceCleanupMode.OFF) { + Runtime.getRuntime().addShutdownHook(new Thread(TraceTestResourceCleanupListener::createDumps)); + } + } + + static void createDumps() { + if (!DUMP_DIR.isDirectory()) { + DUMP_DIR.mkdirs(); + } + try { + Thread.sleep(WAIT_BEFORE_DUMP_MILLIS); + } catch (InterruptedException e) { + // ignore + } + + String datetimePart = + DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss.SSS").format(ZonedDateTime.now()); + try { + String threadDump = ThreadDumpUtil.buildThreadDiagnosticString(); + File threaddumpFile = new File(DUMP_DIR, "threaddump" + datetimePart + ".txt"); + Files.asCharSink(threaddumpFile, Charsets.UTF_8).write(threadDump); + } catch (Throwable t) { + System.err.println("Error dumping threads"); + t.printStackTrace(System.err); + } + + try { + String heapHistogram = HeapHistogramUtil.buildHeapHistogram(); + File heapHistogramFile = new File(DUMP_DIR, "heaphistogram" + datetimePart + ".txt"); + Files.asCharSink(heapHistogramFile, Charsets.UTF_8).write(heapHistogram); + } catch (Throwable t) { + System.err.println("Error dumping heap histogram"); + t.printStackTrace(System.err); + } + + if (MODE == TraceTestResourceCleanupMode.FULL) { + try { + File heapdumpFile = new File(DUMP_DIR, "heapdump" + datetimePart + ".hprof"); + HeapDumpUtil.dumpHeap(heapdumpFile, true); + } catch (Throwable t) { + System.err.println("Error dumping heap"); + t.printStackTrace(System.err); + } + } + } +} diff --git a/buildtools/src/main/resources/log4j2.xml b/buildtools/src/main/resources/log4j2.xml index 184f58487eaf0..b0d01a734c518 100644 --- a/buildtools/src/main/resources/log4j2.xml +++ b/buildtools/src/main/resources/log4j2.xml @@ -22,7 +22,7 @@ - + diff --git a/buildtools/src/main/resources/pulsar/checkstyle.xml b/buildtools/src/main/resources/pulsar/checkstyle.xml index b3812ca8cccd7..14808cf86638b 100644 --- a/buildtools/src/main/resources/pulsar/checkstyle.xml +++ b/buildtools/src/main/resources/pulsar/checkstyle.xml @@ -137,7 +137,7 @@ page at http://checkstyle.sourceforge.net/config.html --> + value="autovalue.shaded, avro.shaded, bk-shade, com.google.api.client.repackaged, com.google.appengine.repackaged, org.apache.curator.shaded, org.testcontainers.shaded, org.junit" /> @@ -179,7 +179,7 @@ page at http://checkstyle.sourceforge.net/config.html --> - + diff --git a/buildtools/src/main/resources/pulsar/suppressions.xml b/buildtools/src/main/resources/pulsar/suppressions.xml index 7c78988db3e90..57a01c60f6a27 100644 --- a/buildtools/src/main/resources/pulsar/suppressions.xml +++ b/buildtools/src/main/resources/pulsar/suppressions.xml @@ -38,7 +38,7 @@ - + diff --git a/conf/bkenv.sh b/conf/bkenv.sh index b41532d3a0c91..8beea47cee312 100644 --- a/conf/bkenv.sh +++ b/conf/bkenv.sh @@ -37,9 +37,6 @@ BOOKIE_LOG_DIR=${BOOKIE_LOG_DIR:-"${PULSAR_LOG_DIR}"} # Memory size options BOOKIE_MEM=${BOOKIE_MEM:-${PULSAR_MEM:-"-Xms2g -Xmx2g -XX:MaxDirectMemorySize=2g"}} -# Garbage collection options -BOOKIE_GC=${BOOKIE_GC:-${PULSAR_GC:-"-XX:+UseZGC -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch"}} - if [ -z "$JAVA_HOME" ]; then JAVA_BIN=java else @@ -60,6 +57,17 @@ for token in $("$JAVA_BIN" -version 2>&1 | grep 'version "'); do fi done +# Garbage collection options +BOOKIE_GC="${BOOKIE_GC:-${PULSAR_GC}}" +if [ -z "$BOOKIE_GC" ]; then + BOOKIE_GC="-XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch" + if [[ $JAVA_MAJOR_VERSION -ge 21 ]]; then + BOOKIE_GC="-XX:+UseZGC -XX:+ZGenerational ${BOOKIE_GC}" + else + BOOKIE_GC="-XX:+UseZGC ${BOOKIE_GC}" + fi +fi + if [[ -z "$BOOKIE_GC_LOG" ]]; then # fallback to PULSAR_GC_LOG if it is set BOOKIE_GC_LOG="$PULSAR_GC_LOG" diff --git a/conf/bookkeeper.conf b/conf/bookkeeper.conf index 9661108acc041..4058d787e2c00 100644 --- a/conf/bookkeeper.conf +++ b/conf/bookkeeper.conf @@ -186,6 +186,12 @@ enableBusyWait=false # True if the bookie should double check readMetadata prior to gc # verifyMetadataOnGC=false +# Allow force compaction when disabling the entry log compaction or not. +# It will enable you to manually force compact the entry log even if +# the entry log compaction is disabled. The 'minorCompactionThreshold' or +# 'majorCompactionThreshold' still needs to be specified. +forceAllowCompaction=true + ############################################################################# ## TLS settings ############################################################################# @@ -243,9 +249,15 @@ enableBusyWait=false # The interval is specified in seconds. auditorPeriodicBookieCheckInterval=86400 -# The number of entries that a replication will rereplicate in parallel. +# The number of entries that a replication will re-replicate in parallel. rereplicationEntryBatchSize=100 +# Enable/disable having read operations for a ledger to be sticky to a single bookie. +stickyReadSEnabled=true + +# Enable/disable reordering read sequence on reading entries. +reorderReadSequenceEnabled=true + # Auto-replication # The grace period, in milliseconds, that the replication worker waits before fencing and # replicating a ledger fragment that's still being written to upon bookie failure. @@ -504,7 +516,7 @@ entryLogFilePreallocationEnabled=true # happens on log rotation. # Flushing in smaller chunks but more frequently reduces spikes in disk # I/O. Flushing too frequently may also affect performance negatively. -flushEntrylogBytes=268435456 +flushEntrylogBytes=33554432 # The number of bytes we should use as capacity for BufferedReadChannel. Default is 512 bytes. readBufferSizeBytes=4096 @@ -608,19 +620,19 @@ readOnlyModeEnabled=true # For each ledger dir, maximum disk space which can be used. # Default is 0.95f. i.e. 95% of disk can be used at most after which nothing will -# be written to that partition. If all ledger dir partitions are full, then bookie +# be written to that partition. If all ledger dir partions are full, then bookie # will turn to readonly mode if 'readOnlyModeEnabled=true' is set, else it will -# shutdown. -# Valid values should be in between 0 and 1 (exclusive). -diskUsageThreshold=0.95 - -# The disk free space low water mark threshold. -# Disk is considered full when usage threshold is exceeded. -# Disk returns back to non-full state when usage is below low water mark threshold. -# This prevents it from going back and forth between these states frequently -# when concurrent writes and compaction are happening. This also prevent bookie from -# switching frequently between read-only and read-writes states in the same cases. -# diskUsageWarnThreshold=0.95 +# shutdown. Bookie will also suspend the minor and major compaction when usage threshold is exceed +# if `isForceGCAllowWhenNoSpace` is disabled. When the usage becomes lower than the threshold, the major and minor +# compaction will be resumed. +# Valid values should be in between 0 and 1 (exclusive). The default value is 0.95. +# diskUsageThreshold=0.95 + +# The disk free space warn threshold. +# Disk is considered almost full when usage threshold is exceeded. Bookie will suspend the major +# compaction when usage threshold is exceed if `isForceGCAllowWhenNoSpace` is disabled. When the usage becomes lower +# than the threshold, the major compaction will be resumed. The default value is 0.90. +# diskUsageWarnThreshold=0.90 # Set the disk free space low water mark threshold. Disk is considered full when # usage threshold is exceeded. Disk returns back to non-full state when usage is @@ -628,7 +640,10 @@ diskUsageThreshold=0.95 # between these states frequently when concurrent writes and compaction are # happening. This also prevent bookie from switching frequently between # read-only and read-writes states in the same cases. -# diskUsageLwmThreshold=0.90 +# If the bookie already runs into read-only mode and the disk usage becomes lower than this threshold, the bookie +# will change from read-only to read-write mode. At the same time, the major and minor compaction will be resumed +# if `isForceGCAllowWhenNoSpace` is disabled. The default value is the same with `diskUsageThreshold`. +# diskUsageLwmThreshold=0.95 # Disk check interval in milli seconds, interval to check the ledger dirs usage. # Default is 10000 @@ -643,9 +658,13 @@ diskCheckInterval=10000 ############################################################################# # Metadata service uri that bookkeeper uses for loading the corresponding metadata driver and resolving its metadata service location -# Examples: -# - metadataServiceUri=zk://my-zk-1:2181/ledgers -# - metadataServiceUri=etcd:http://my-etcd:2379 +# Examples: +# - metadataServiceUri=zk+hierarchical://my-zk-1:2181/ledgers +# - metadataServiceUri=etcd+hierarchical:http://my-etcd:2379 +# - metadataServiceUri=metadata-store:zk:my-zk-1:2281 +# If you use metadata-store configuration, you need to configure following items in JVM option: +# -Dbookkeeper.metadata.client.drivers=org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver +# -Dbookkeeper.metadata.bookie.drivers=org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver metadataServiceUri= # @Deprecated - `ledgerManagerFactoryClass` is deprecated in favor of using `metadataServiceUri` @@ -742,17 +761,24 @@ dbStorage_readAheadCacheBatchSize=1000 ## DbLedgerStorage uses RocksDB to store the indexes from ## (ledgerId, entryId) -> (entryLog, offset) +# These settings are ignored since Pulsar 2.11 / Bookkeeper 4.15 +# NOTICE: The settings in conf/default_rocksdb.conf, conf/entry_location_rocksdb.conf and +# conf/ledger_metadata_rocksdb.conf files are primarily used to configure RocksDB +# settings. dbStorage_rocksDB_* config keys are ignored. + # Size of RocksDB block-cache. For best performance, this cache # should be big enough to hold a significant portion of the index # database which can reach ~2GB in some cases # Default is to use 10% of the direct memory size +# These settings are ignored since Pulsar 2.11 / Bookkeeper 4.15 dbStorage_rocksDB_blockCacheSize= # Other RocksDB specific tunables +# These settings are ignored since Pulsar 2.11 / Bookkeeper 4.15 dbStorage_rocksDB_writeBufferSizeMB=64 dbStorage_rocksDB_sstSizeInMB=64 dbStorage_rocksDB_blockSize=65536 dbStorage_rocksDB_bloomFilterBitsPerKey=10 dbStorage_rocksDB_numLevels=-1 dbStorage_rocksDB_numFilesInLevel0=4 -dbStorage_rocksDB_maxSizeInLevel1MB=256 +dbStorage_rocksDB_maxSizeInLevel1MB=256 \ No newline at end of file diff --git a/conf/broker.conf b/conf/broker.conf index 1183049bf8531..e745fcb2b0a8f 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -88,6 +88,16 @@ advertisedAddress= # If true, the real IP addresses of consumers and producers can be obtained when getting topic statistics data. haProxyProtocolEnabled=false +# Enable or disable the use of HA proxy protocol for resolving the client IP for http/https requests. +webServiceHaProxyProtocolEnabled=false + +# Trust X-Forwarded-For header for resolving the client IP for http/https requests. Default is false. +webServiceTrustXForwardedFor=false + +# Add detailed client/remote and server/local addresses and ports to http/https request logging. +# Defaults to true when either webServiceHaProxyProtocolEnabled or webServiceTrustXForwardedFor is enabled. +webServiceLogDetailedAddresses= + # Number of threads to config Netty Acceptor. Default is 1 numAcceptorThreads= @@ -149,6 +159,14 @@ skipBrokerShutdownOnOOM=false # Factory class-name to create topic with custom workflow topicFactoryClassName= +# Max capacity of the topic name cache. -1 means unlimited cache; 0 means broker will clear all cache +# per "maxSecondsToClearTopicNameCache", it does not mean broker will not cache TopicName. +topicNameCacheMaxCapacity=100000 + +# A Specifies the minimum number of seconds that the topic name stays in memory, to avoid clear cache frequently when +# there are too many topics are in use. +maxSecondsToClearTopicNameCache=7200 + # Enable backlog quota check. Enforces action on topic when the quota is reached backlogQuotaCheckEnabled=true @@ -170,6 +188,10 @@ backlogQuotaDefaultRetentionPolicy=producer_request_hold # Default ttl for namespaces if ttl is not already configured at namespace policies. (disable default-ttl with value 0) ttlDurationDefaultInSeconds=0 +# Additional system subscriptions that will be ignored by ttl check. The cursor names are comma separated. +# Default is empty. +# additionalSystemCursorNames= + # Enable topic auto creation if new producer or consumer connected (disable auto creation with value false) allowAutoTopicCreation=true @@ -333,6 +355,25 @@ maxUnackedMessagesPerBroker=0 # limit/2 messages maxUnackedMessagesPerSubscriptionOnBrokerBlocked=0.16 +# For Key_Shared subscriptions, if messages cannot be dispatched to consumers due to a slow consumer +# or a blocked key hash (because of ordering constraints), the broker will continue reading more +# messages from the backlog and attempt to dispatch them to consumers until the number of replay +# messages reaches the calculated threshold. +# Formula: threshold = min(keySharedLookAheadMsgInReplayThresholdPerConsumer * +# connected consumer count, keySharedLookAheadMsgInReplayThresholdPerSubscription). +# Setting this value to 0 will disable the limit calculated per consumer. +keySharedLookAheadMsgInReplayThresholdPerConsumer=2000 + +# For Key_Shared subscriptions, if messages cannot be dispatched to consumers due to a slow consumer +# or a blocked key hash (because of ordering constraints), the broker will continue reading more +# messages from the backlog and attempt to dispatch them to consumers until the number of replay +# messages reaches the calculated threshold. +# Formula: threshold = min(keySharedLookAheadMsgInReplayThresholdPerConsumer * +# connected consumer count, keySharedLookAheadMsgInReplayThresholdPerSubscription). +# This value should be set to a value less than 2 * managedLedgerMaxUnackedRangesToPersist. +# Setting this value to 0 will disable the limit calculated per subscription. +keySharedLookAheadMsgInReplayThresholdPerSubscription=20000 + # Broker periodically checks if subscription is stuck and unblock if flag is enabled. (Default is disabled) unblockStuckSubscriptionEnabled=false @@ -445,7 +486,17 @@ dispatcherReadFailureBackoffMaxTimeInMs=60000 # The read failure backoff mandatory stop time in milliseconds. By default it is 0s. dispatcherReadFailureBackoffMandatoryStopTimeInMs=0 -# Precise dispathcer flow control according to history message number of each entry +# On Shared and KeyShared subscriptions, if all available messages in the subscription are filtered +# out and not dispatched to any consumer, message dispatching will be rescheduled with a backoff +# delay. This parameter sets the initial backoff delay in milliseconds. +dispatcherRetryBackoffInitialTimeInMs=1 + +# On Shared and KeyShared subscriptions, if all available messages in the subscription are filtered +# out and not dispatched to any consumer, message dispatching will be rescheduled with a backoff +# delay. This parameter sets the maximum backoff delay in milliseconds. +dispatcherRetryBackoffMaxTimeInMs=10 + +# Precise dispatcher flow control according to history message number of each entry preciseDispatcherFlowControl=false # Class name of Pluggable entry filter that can decide whether the entry needs to be filtered @@ -474,8 +525,8 @@ maxConcurrentTopicLoadRequest=5000 # Max concurrent non-persistent message can be processed per connection maxConcurrentNonPersistentMessagePerConnection=1000 -# Number of worker threads to serve non-persistent topic -numWorkerThreadsForNonPersistentTopic= +# Number of worker threads to serve topic ordered executor +topicOrderedExecutorThreadNum= # Enable broker to load persistent topics enablePersistentTopics=true @@ -534,10 +585,19 @@ brokerServiceCompactionMonitorIntervalInSeconds=60 # Using a value of 0, is disabling compression check. brokerServiceCompactionThresholdInBytes=0 -# Timeout for the compaction phase one loop. -# If the execution time of the compaction phase one loop exceeds this time, the compaction will not proceed. +# Timeout for each read request in the compaction phase one loop. +# If the execution time of one single message read operation exceeds this time, the compaction will not proceed. brokerServiceCompactionPhaseOneLoopTimeInSeconds=30 +# Whether retain null-key message during topic compaction +topicCompactionRetainNullKey=false + +# Class name of the factory that implements the topic compaction service. +# If value is "org.apache.pulsar.compaction.EventTimeCompactionServiceFactory", +# will create topic compaction service based on message eventTime. +# By default compaction service is based on message publishing order. +compactionServiceFactoryClassName=org.apache.pulsar.compaction.PulsarCompactionServiceFactory + # Whether to enable the delayed delivery for messages. # If disabled, messages will be immediately delivered and there will # be no tracking overhead. @@ -550,13 +610,11 @@ delayedDeliveryTrackerFactoryClassName=org.apache.pulsar.broker.delayed.InMemory # Control the tick time for when retrying on delayed delivery, # affecting the accuracy of the delivery time compared to the scheduled time. -# Note that this time is used to configure the HashedWheelTimer's tick time for the -# InMemoryDelayedDeliveryTrackerFactory (the default DelayedDeliverTrackerFactory). +# Note that this time is used to configure the HashedWheelTimer's tick time. # Default is 1 second. delayedDeliveryTickTimeMillis=1000 -# When using the InMemoryDelayedDeliveryTrackerFactory (the default DelayedDeliverTrackerFactory), whether -# the deliverAt time is strictly followed. When false (default), messages may be sent to consumers before the deliverAt +# Whether the deliverAt time is strictly followed. When false (default), messages may be sent to consumers before the deliverAt # time by as much as the tickTimeMillis. This can reduce the overhead on the broker of maintaining the delayed index # for a potentially very short time period. When true, messages will not be sent to consumer until the deliverAt time # has passed, and they may be as late as the deliverAt time plus the tickTimeMillis for the topic plus the @@ -582,11 +640,16 @@ delayedDeliveryMaxIndexesPerBucketSnapshotSegment=5000 delayedDeliveryMaxNumBuckets=-1 # Size of the lookahead window to use when detecting if all the messages in the topic -# have a fixed delay. +# have a fixed delay for InMemoryDelayedDeliveryTracker (the default DelayedDeliverTracker). # Default is 50,000. Setting the lookahead window to 0 will disable the logic to handle # fixed delays in messages in a different way. delayedDeliveryFixedDelayDetectionLookahead=50000 +# The max allowed delay for delayed delivery (in milliseconds). If the broker receives a message which exceeds this +# max delay, then it will return an error to the producer. +# The default value is 0 which means there is no limit on the max delivery delay. +delayedDeliveryMaxDelayInMillis=0 + # Whether to enable acknowledge of batch local index. acknowledgmentAtBatchIndexLevelEnabled=false @@ -902,7 +965,7 @@ saslJaasServerRoleTokenSignerSecretPath= ### --- HTTP Server config --- ### -# If >0, it will reject all HTTP requests with bodies larged than the configured limit +# If >0, it will reject all HTTP requests with bodies larger than the configured limit httpMaxRequestSize=-1 # The maximum size in bytes of the request header. Larger headers will allow for more and/or larger cookies plus larger @@ -946,7 +1009,7 @@ bookkeeperMetadataServiceUri= # Authentication plugin to use when connecting to bookies bookkeeperClientAuthenticationPlugin= -# BookKeeper auth plugin implementatation specifics parameters name and values +# BookKeeper auth plugin implementation specifics parameters name and values bookkeeperClientAuthenticationParametersName= bookkeeperClientAuthenticationParameters= @@ -1014,7 +1077,7 @@ bookkeeperClientMinNumRacksPerWriteQuorum=2 bookkeeperClientEnforceMinNumRacksPerWriteQuorum=false # Enable/disable reordering read sequence on reading entries. -bookkeeperClientReorderReadSequenceEnabled=false +bookkeeperClientReorderReadSequenceEnabled=true # Enable bookie isolation by specifying a list of bookie groups to choose from. Any bookie # outside the specified groups will not be used by the broker @@ -1080,8 +1143,8 @@ bookkeeperExplicitLacIntervalInMills=0 # bookkeeperClientExposeStatsToPrometheus=false # If bookkeeperClientExposeStatsToPrometheus is set to true, we can set bookkeeperClientLimitStatsLogging=true -# to limit per_channel_bookie_client metrics. default is false -# bookkeeperClientLimitStatsLogging=false +# to limit per_channel_bookie_client metrics. default is true +# bookkeeperClientLimitStatsLogging=true ### --- Managed Ledger --- ### @@ -1141,6 +1204,16 @@ managedLedgerCacheEvictionTimeThresholdMillis=1000 # and thus should be set as inactive. managedLedgerCursorBackloggedThreshold=1000 +# Minimum cursors that must be in backlog state to cache and reuse the read entries. +# (Default =0 to disable backlog reach cache) +managedLedgerMinimumBacklogCursorsForCaching=0 + +# Minimum backlog entries for any cursor before start caching reads. +managedLedgerMinimumBacklogEntriesForCaching=1000 + +# Maximum backlog entry difference to prevent caching entries that can't be reused. +managedLedgerMaxBacklogBetweenCursorsForCaching=1000 + # Rate limit the amount of writes per second generated by consumer acking the messages managedLedgerDefaultMarkDeleteRateLimit=1.0 @@ -1181,6 +1254,9 @@ managedLedgerDataReadPriority=tiered-storage-first # (default is -1, which is disabled) managedLedgerOffloadThresholdInSeconds=-1 +# Trigger offload on topic load or not. Default is false. +# triggerOffloadOnTopicLoad=false + # Max number of entries to append to a cursor ledger managedLedgerCursorMaxEntriesPerLedger=50000 @@ -1310,6 +1386,9 @@ loadBalancerOverrideBrokerNicSpeedGbps= # Name of load manager to use loadManagerClassName=org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl +# Name of topic bundle assignment strategy to use +topicBundleAssignmentStrategy=org.apache.pulsar.common.naming.ConsistentHashingTopicBundleAssigner + # Supported algorithms name for namespace bundle split. # "range_equally_divide" divides the bundle into two parts with the same hash range size. # "topic_count_equally_divide" divides the bundle into two parts with the same topics count. @@ -1357,21 +1436,19 @@ loadBalancerMsgThroughputMultiplierDifferenceShedderThreshold=4 # It only takes effect in the ThresholdShedder strategy. loadBalancerHistoryResourcePercentage=0.9 -# The BandWithIn usage weight when calculating new resource usage. -# It only takes effect in the ThresholdShedder strategy. -loadBalancerBandwithInResourceWeight=1.0 +# The BandWidthIn usage weight when calculating new resource usage. +loadBalancerBandwidthInResourceWeight=1.0 -# The BandWithOut usage weight when calculating new resource usage. -# It only takes effect in the ThresholdShedder strategy. -loadBalancerBandwithOutResourceWeight=1.0 +# The BandWidthOut usage weight when calculating new resource usage. +loadBalancerBandwidthOutResourceWeight=1.0 # The CPU usage weight when calculating new resource usage. -# It only takes effect in the ThresholdShedder strategy. loadBalancerCPUResourceWeight=1.0 # The direct memory usage weight when calculating new resource usage. -# It only takes effect in the ThresholdShedder strategy. -loadBalancerDirectMemoryResourceWeight=1.0 +# Direct memory usage cannot accurately reflect the machine's load, +# and it is not recommended to use it to score the machine's load. +loadBalancerDirectMemoryResourceWeight=0 # Bundle unload minimum throughput threshold (MB), avoiding bundle unload frequently. # It only takes effect in the ThresholdShedder strategy. @@ -1380,6 +1457,25 @@ loadBalancerBundleUnloadMinThroughputThreshold=10 # Time to wait for the unloading of a namespace bundle namespaceBundleUnloadingTimeoutMs=60000 +# configuration for AvgShedder, a new shedding and placement strategy +# The low threshold for the difference between the highest and lowest loaded brokers. +loadBalancerAvgShedderLowThreshold = 15 + +# The high threshold for the difference between the highest and lowest loaded brokers. +loadBalancerAvgShedderHighThreshold = 40 + +# The number of times the low threshold is triggered before the bundle is unloaded. +loadBalancerAvgShedderHitCountLowThreshold = 8 + +# The number of times the high threshold is triggered before the bundle is unloaded. +loadBalancerAvgShedderHitCountHighThreshold = 2 + +# In the UniformLoadShedder and AvgShedder strategy, the maximum unload ratio. +# For AvgShedder, recommend to set to 0.5, so that it will distribute the load evenly +# between the highest and lowest brokers. +maxUnloadPercentage = 0.2 + + ### --- Load balancer extension --- ### # Option to enable the debug mode for the load balancer logics. @@ -1391,14 +1487,14 @@ loadBalancerDebugModeEnabled=false # (100% resource usage is 1.0 load). # The shedder logic tries to distribute bundle load across brokers to meet this target std. # The smaller value will incur load balancing more frequently. -# (only used in load balancer extension TransferSheddeer) +# (only used in load balancer extension TransferShedder) loadBalancerBrokerLoadTargetStd=0.25 # Threshold to the consecutive count of fulfilled shedding(unload) conditions. # If the unload scheduler consecutively finds bundles that meet unload conditions # many times bigger than this threshold, the scheduler will shed the bundles. # The bigger value will incur less bundle unloading/transfers. -# (only used in load balancer extension TransferSheddeer) +# (only used in load balancer extension TransferShedder) loadBalancerSheddingConditionHitCountThreshold=3 # Option to enable the bundle transfer mode when distributing bundle loads. @@ -1406,18 +1502,18 @@ loadBalancerSheddingConditionHitCountThreshold=3 # -- pre-assigns the destination broker upon unloading). # Off: unload bundles from overloaded brokers # -- post-assigns the destination broker upon lookups). -# (only used in load balancer extension TransferSheddeer) +# (only used in load balancer extension TransferShedder) loadBalancerTransferEnabled=true # Maximum number of brokers to unload bundle load for each unloading cycle. # The bigger value will incur more unloading/transfers for each unloading cycle. -# (only used in load balancer extension TransferSheddeer) +# (only used in load balancer extension TransferShedder) loadBalancerMaxNumberOfBrokerSheddingPerCycle=3 # Delay (in seconds) to the next unloading cycle after unloading. # The logic tries to give enough time for brokers to recompute load after unloading. # The bigger value will delay the next unloading cycle longer. -# (only used in load balancer extension TransferSheddeer) +# (only used in load balancer extension TransferShedder) loadBalanceSheddingDelayInSeconds=180 # Broker load data time to live (TTL in seconds). @@ -1425,14 +1521,17 @@ loadBalanceSheddingDelayInSeconds=180 # and those brokers will be ignored in the load computation. # When tuning this value, please consider loadBalancerReportUpdateMaxIntervalMinutes. #The current default is loadBalancerReportUpdateMaxIntervalMinutes * 2. -# (only used in load balancer extension TransferSheddeer) +# (only used in load balancer extension TransferShedder) loadBalancerBrokerLoadDataTTLInSeconds=1800 # Max number of bundles in bundle load report from each broker. # The load balancer distributes bundles across brokers, # based on topK bundle load data and other broker load data. # The bigger value will increase the overhead of reporting many bundles in load data. -# (only used in load balancer extension logics) +# Used for ExtensibleLoadManagerImpl and ModularLoadManagerImpl, default value is 10. +# User can disable the bundle filtering feature of ModularLoadManagerImpl by setting this value to -1. +# Enabling this feature can reduce the pressure on the zookeeper when doing load report. +# WARNING: too small value could result in a long load balance time. loadBalancerMaxNumberOfBundlesInBundleLoadReport=10 # Service units'(bundles) split interval. Broker periodically checks whether @@ -1464,6 +1563,15 @@ loadBalancerNamespaceBundleSplitConditionHitCountThreshold=3 # (only used in load balancer extension logics) loadBalancerServiceUnitStateTombstoneDelayTimeInSeconds=3600 +# Name of ServiceUnitStateTableView implementation class to use +loadManagerServiceUnitStateTableViewClassName=org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateTableViewImpl + +# Specify ServiceUnitTableViewSyncer to sync service unit(bundle) states between metadata store and +# system topic table views during migration from one to the other. One could enable this +# syncer before migration and disable it after the migration finishes. +# It accepts `MetadataStoreToSystemTopicSyncer` or `SystemTopicToMetadataStoreSyncer` to +# enable it. It accepts `None` to disable it." +loadBalancerServiceUnitTableViewSyncer=None ### --- Replication --- ### @@ -1485,6 +1593,16 @@ replicatorPrefix=pulsar.repl # due to missing ZooKeeper watch (disable with value 0) replicationPolicyCheckDurationSeconds=600 +# Whether the internal replication of the local cluster will trigger topic auto-creation on the remote cluster. +# 1. After enabling namespace-level Geo-Replication: whether the local broker will create topics on the remote +# cluster automatically when calling `pulsar-admin topics create-partitioned-topic`. +# 2. When enabling topic-level Geo-Replication on a partitioned topic: whether the local broker will create topics on +# the remote cluster. +# 3. Whether the internal Geo-Replicator in the local cluster will trigger non-persistent topic auto-creation for +# remote clusters. +# It is not a dynamic config, the default value is "true" to preserve backward-compatible behavior. +createTopicToRemoteClusterForReplication=true + # Default message retention time. # 0 means retention is disabled. -1 means data is not removed by time quota. defaultRetentionTimeInMinutes=0 @@ -1513,6 +1631,9 @@ webSocketNumServiceThreads= # Number of connections per Broker in Pulsar Client used in WebSocket proxy webSocketConnectionsPerBroker= +# Memory limit in MBs for direct memory in Pulsar Client used in WebSocket proxy +webSocketPulsarClientMemoryLimitInMB=0 + # Time in milliseconds that idle WebSocket session times out webSocketSessionIdleTimeoutMillis=300000 @@ -1568,6 +1689,8 @@ exposePublisherStats=true statsUpdateFrequencyInSecs=60 statsUpdateInitialDelayInSecs=60 +healthCheckMetricsUpdateTimeInSeconds=-1 + # Enable expose the precise backlog stats. # Set false to use published counter and consumed counter to calculate, this would be more efficient but may be inaccurate. # Default is false. @@ -1590,6 +1713,10 @@ aggregatePublisherStatsByProducerName=false # if cluster is marked migrated. Disable with value 0. (Default disabled). clusterMigrationCheckDurationSeconds=0 +# Flag to start cluster migration for topic only after creating all topic's resources +# such as tenant, namespaces, subscriptions at new green cluster. (Default disabled). +clusterMigrationAutoResourceCreation=false + ### --- Schema storage --- ### # The schema storage implementation used by this broker schemaRegistryStorageClassName=org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorageFactory @@ -1607,7 +1734,7 @@ schemaCompatibilityStrategy=FULL # The directory for all the offloader implementations offloadersDirectory=./offloaders -# Driver to use to offload old data to long term storage (Possible values: aws-s3, google-cloud-storage, azureblob, aliyun-oss, filesystem) +# Driver to use to offload old data to long term storage (Possible values: aws-s3, google-cloud-storage, azureblob, aliyun-oss, filesystem, S3) # When using google-cloud-storage, Make sure both Google Cloud Storage and Google Cloud Storage JSON API are enabled for # the project (check from Developers Console -> Api&auth -> APIs). managedLedgerOffloadDriver= @@ -1635,10 +1762,10 @@ s3ManagedLedgerOffloadBucket= # For Amazon S3 ledger offload, Alternative endpoint to connect to (useful for testing) s3ManagedLedgerOffloadServiceEndpoint= -# For Amazon S3 ledger offload, Max block size in bytes. (64MB by default, 5MB minimum) +# For Amazon S3 ledger offload, Max block size in bytes. (64MiB by default, 5MiB minimum) s3ManagedLedgerOffloadMaxBlockSizeInBytes=67108864 -# For Amazon S3 ledger offload, Read buffer size in bytes (1MB by default) +# For Amazon S3 ledger offload, Read buffer size in bytes (1MiB by default) s3ManagedLedgerOffloadReadBufferSizeInBytes=1048576 # For Google Cloud Storage ledger offload, region where offload bucket is located. @@ -1648,10 +1775,11 @@ gcsManagedLedgerOffloadRegion= # For Google Cloud Storage ledger offload, Bucket to place offloaded ledger into gcsManagedLedgerOffloadBucket= -# For Google Cloud Storage ledger offload, Max block size in bytes. (64MB by default, 5MB minimum) -gcsManagedLedgerOffloadMaxBlockSizeInBytes=67108864 +# For Google Cloud Storage ledger offload, Max block size in bytes. (128MiB by default, 5MiB minimum) +# Since JClouds limits the maximum number of blocks to 32, the maximum size of a ledger is 32 times the block size. +gcsManagedLedgerOffloadMaxBlockSizeInBytes=134217728 -# For Google Cloud Storage ledger offload, Read buffer size in bytes (1MB by default) +# For Google Cloud Storage ledger offload, Read buffer size in bytes (1MiB by default) gcsManagedLedgerOffloadReadBufferSizeInBytes=1048576 # For Google Cloud Storage, path to json file containing service account credentials. @@ -1765,9 +1893,8 @@ strictBookieAffinityEnabled=false # These settings are left here for compatibility # The heap memory usage weight when calculating new resource usage. -# It only takes effect in the ThresholdShedder strategy. # Deprecated: Memory is no longer used as a load balancing item -loadBalancerMemoryResourceWeight=1.0 +loadBalancerMemoryResourceWeight=0 # Zookeeper quorum connection string # Deprecated: use metadataStoreUrl instead @@ -1812,9 +1939,26 @@ subscriptionKeySharedEnable=true # Deprecated: use managedLedgerMaxUnackedRangesToPersistInMetadataStore managedLedgerMaxUnackedRangesToPersistInZooKeeper=-1 +# After enabling this feature, Pulsar will stop delivery messages to clients if the cursor metadata is too large to +# persist, it will help to reduce the duplicates caused by the ack state that can not be fully persistent. +dispatcherPauseOnAckStatePersistentEnabled=false + # If enabled, the maximum "acknowledgment holes" will not be limited and "acknowledgment holes" are stored in # multiple entries. persistentUnackedRangesWithMultipleEntriesEnabled=false # Deprecated - Use managedLedgerCacheEvictionIntervalMs instead managedLedgerCacheEvictionFrequency=0 + +# Number of worker threads to serve non-persistent topic +# Deprecated - use topicOrderedExecutorThreadNum instead. +numWorkerThreadsForNonPersistentTopic= + +# The directory to locate broker interceptors +brokerInterceptorsDirectory=./interceptors + +# List of broker interceptor to load, which is a list of broker interceptor names +brokerInterceptors= + +# Enable or disable the broker interceptor, which is only used for testing for now +disableBrokerInterceptors=true diff --git a/conf/client.conf b/conf/client.conf index 8a485e5676c7b..25d65c3947e39 100644 --- a/conf/client.conf +++ b/conf/client.conf @@ -41,7 +41,7 @@ authPlugin= # authParams=tlsCertFile:/path/to/client-cert.pem,tlsKeyFile:/path/to/client-key.pem authParams= -# Allow TLS connections to servers whose certificate cannot be +# Allow TLS connections to servers whose certificate cannot # be verified to have been signed by a trusted certificate # authority. tlsAllowInsecureConnection=false diff --git a/conf/default_rocksdb.conf b/conf/default_rocksdb.conf index e1a21bb845222..74e3005ba6687 100644 --- a/conf/default_rocksdb.conf +++ b/conf/default_rocksdb.conf @@ -24,7 +24,14 @@ info_log_level=INFO_LEVEL # set by jni: options.setKeepLogFileNum keep_log_file_num=30 + # set by jni: options.setLogFileTimeToRoll + log_file_time_to_roll=86400 [CFOptions "default"] - # set by jni: options.setLogFileTimeToRoll - log_file_time_to_roll=86400 \ No newline at end of file + #no default setting in CFOptions + +[TableOptions/BlockBasedTable "default"] + # set by jni: tableOptions.setFormatVersion + format_version=5 + # set by jni: tableOptions.setChecksumType + checksum=kxxHash \ No newline at end of file diff --git a/conf/entry_location_rocksdb.conf b/conf/entry_location_rocksdb.conf index 31bd58506ef75..9c675554b24ae 100644 --- a/conf/entry_location_rocksdb.conf +++ b/conf/entry_location_rocksdb.conf @@ -27,7 +27,7 @@ # set by jni: options.setLogFileTimeToRoll log_file_time_to_roll=86400 # set by jni: options.setMaxBackgroundJobs or options.setIncreaseParallelism - max_background_jobs=2 + max_background_jobs=32 # set by jni: options.setMaxSubcompactions max_subcompactions=1 # set by jni: options.setMaxTotalWalSize @@ -52,6 +52,8 @@ max_bytes_for_level_base=268435456 # set by jni: options.setTargetFileSizeBase target_file_size_base=67108864 + # set by jni: options.setLevelCompactionDynamicLevelBytes + level_compaction_dynamic_level_bytes=true [TableOptions/BlockBasedTable "default"] # set by jni: tableOptions.setBlockSize @@ -59,12 +61,10 @@ # set by jni: tableOptions.setBlockCache block_cache=206150041 # set by jni: tableOptions.setFormatVersion - format_version=2 + format_version=5 # set by jni: tableOptions.setChecksumType checksum=kxxHash # set by jni: tableOptions.setFilterPolicy, bloomfilter:[bits_per_key]:[use_block_based_builder] filter_policy=rocksdb.BloomFilter:10:false # set by jni: tableOptions.setCacheIndexAndFilterBlocks - cache_index_and_filter_blocks=true - # set by jni: options.setLevelCompactionDynamicLevelBytes - level_compaction_dynamic_level_bytes=true \ No newline at end of file + cache_index_and_filter_blocks=true \ No newline at end of file diff --git a/conf/functions_log4j2.xml b/conf/functions_log4j2.xml index 6902a3acd8736..fd4042e82e82f 100644 --- a/conf/functions_log4j2.xml +++ b/conf/functions_log4j2.xml @@ -68,7 +68,7 @@ ${sys:pulsar.function.log.dir} 2 - */${sys:pulsar.function.log.file}*log.gz + ${sys:pulsar.function.log.file}*log.gz 30d @@ -101,7 +101,7 @@ ${sys:pulsar.function.log.dir} 2 - */${sys:pulsar.function.log.file}.bk*log.gz + ${sys:pulsar.function.log.file}.bk*log.gz 30d diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml index 4c5b6aab1b7f4..6f995576ebd64 100644 --- a/conf/functions_worker.yml +++ b/conf/functions_worker.yml @@ -27,6 +27,16 @@ workerHostname: localhost workerPort: 6750 workerPortTls: 6751 +# Enable or disable the use of HA proxy protocol for resolving the client IP for http/https requests. +webServiceHaProxyProtocolEnabled: false + +# Trust X-Forwarded-For header for resolving the client IP for http/https requests. Default is false. +webServiceTrustXForwardedFor: false + +# Add detailed client/remote and server/local addresses and ports to http/https request logging. +# Defaults to true when either webServiceHaProxyProtocolEnabled or webServiceTrustXForwardedFor is enabled. +webServiceLogDetailedAddresses: null + # The Configuration metadata store url # Examples: # * zk:my-zk-1:2181,my-zk-2:2181,my-zk-3:2181 @@ -43,6 +53,16 @@ metadataStoreOperationTimeoutSeconds: 30 # Metadata store cache expiry time in seconds metadataStoreCacheExpirySeconds: 300 +# Specifies if the function worker should use classloading for validating submissions for built-in +# connectors and functions. This is required for validateConnectorConfig to take effect. +# Default is false. +enableClassloadingOfBuiltinFiles: false + +# Specifies if the function worker should use classloading for validating submissions for external +# connectors and functions. This is required for validateConnectorConfig to take effect. +# Default is false. +enableClassloadingOfExternalFiles: false + ################################ # Function package management ################################ @@ -398,9 +418,20 @@ saslJaasServerRoleTokenSignerSecretPath: ######################## connectorsDirectory: ./connectors +# Whether to enable referencing connectors directory files by file url in connector (sink/source) creation +enableReferencingConnectorDirectoryFiles: true +# Regex patterns for enabling creation of connectors by referencing packages in matching http/https urls +additionalEnabledConnectorUrlPatterns: [] functionsDirectory: ./functions - -# Should connector config be validated during submission +# Whether to enable referencing functions directory files by file url in functions creation +enableReferencingFunctionsDirectoryFiles: true +# Regex patterns for enabling creation of functions by referencing packages in matching http/https urls +additionalEnabledFunctionsUrlPatterns: [] + +# Enables extended validation for connector config with fine-grain annotation based validation +# during submission. Classloading with either enableClassloadingOfExternalFiles or +# enableClassloadingOfBuiltinFiles must be enabled on the worker for this to take effect. +# Default is false. validateConnectorConfig: false # Whether to initialize distributed log metadata by runtime. diff --git a/conf/ledger_metadata_rocksdb.conf b/conf/ledger_metadata_rocksdb.conf index e1a21bb845222..74e3005ba6687 100644 --- a/conf/ledger_metadata_rocksdb.conf +++ b/conf/ledger_metadata_rocksdb.conf @@ -24,7 +24,14 @@ info_log_level=INFO_LEVEL # set by jni: options.setKeepLogFileNum keep_log_file_num=30 + # set by jni: options.setLogFileTimeToRoll + log_file_time_to_roll=86400 [CFOptions "default"] - # set by jni: options.setLogFileTimeToRoll - log_file_time_to_roll=86400 \ No newline at end of file + #no default setting in CFOptions + +[TableOptions/BlockBasedTable "default"] + # set by jni: tableOptions.setFormatVersion + format_version=5 + # set by jni: tableOptions.setChecksumType + checksum=kxxHash \ No newline at end of file diff --git a/conf/log4j2.yaml b/conf/log4j2.yaml index 9c261a6b89a50..0e49503581c48 100644 --- a/conf/log4j2.yaml +++ b/conf/log4j2.yaml @@ -19,7 +19,7 @@ Configuration: - status: INFO + status: ERROR monitorInterval: 30 name: pulsar packages: io.prometheus.client.log4j2 diff --git a/conf/proxy.conf b/conf/proxy.conf index cfc1e47b7c445..6e6c960e8009e 100644 --- a/conf/proxy.conf +++ b/conf/proxy.conf @@ -28,17 +28,19 @@ metadataStoreUrl= # The metadata store URL for the configuration data. If empty, we fall back to use metadataStoreUrl configurationMetadataStoreUrl= -# If Service Discovery is Disabled this url should point to the discovery service provider. +# If does not set metadataStoreUrl or configurationMetadataStoreUrl, this url should point to the discovery service +# provider, and does not support multi urls yet. # The URL must begin with pulsar:// for plaintext or with pulsar+ssl:// for TLS. brokerServiceURL= brokerServiceURLTLS= -# These settings are unnecessary if `zookeeperServers` is specified +# If does not set metadataStoreUrl or configurationMetadataStoreUrl, this url should point to the discovery service +# provider, and does not support multi urls yet. brokerWebServiceURL= brokerWebServiceURLTLS= -# If function workers are setup in a separate cluster, configure the following 2 settings -# to point to the function workers cluster +# If function workers are setup in a separate cluster, configure the following 2 settings. This url should point to +# the discovery service provider of the function workers cluster, and does not support multi urls yet. functionWorkerWebServiceURL= functionWorkerWebServiceURLTLS= @@ -61,6 +63,16 @@ advertisedAddress= # If true, the real IP addresses of consumers and producers can be obtained when getting topic statistics data. haProxyProtocolEnabled=false +# Enable or disable the use of HA proxy protocol for resolving the client IP for http/https requests. +webServiceHaProxyProtocolEnabled=false + +# Trust X-Forwarded-For header for resolving the client IP for http/https requests. Default is false. +webServiceTrustXForwardedFor=false + +# Add detailed client/remote and server/local addresses and ports to http/https request logging. +# Defaults to true when either webServiceHaProxyProtocolEnabled or webServiceTrustXForwardedFor is enabled. +webServiceLogDetailedAddresses= + # Enables zero-copy transport of data across network interfaces using the splice system call. # Zero copy mode cannot be used when TLS is enabled or when proxyLogLevel is > 0. proxyZeroCopyModeEnabled=true @@ -152,7 +164,7 @@ authorizationEnabled=false # Authorization provider as a fully qualified class name authorizationProvider=org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider -# Whether client authorization credentials are forwared to the broker for re-authorization. +# Whether client authorization credentials are forwarded to the broker for re-authorization. # Authentication must be enabled via authenticationEnabled=true for this to take effect. forwardAuthorizationCredentials=false @@ -370,7 +382,11 @@ zooKeeperCacheExpirySeconds=-1 ### --- Metrics --- ### +# Whether to enable the proxy's /metrics and /proxy-stats http endpoints +enableProxyStatsEndpoints=true # Whether the '/metrics' endpoint requires authentication. Defaults to true authenticateMetricsEndpoint=true -# Enable cache metrics data, default value is false -metricsBufferResponse=false +# Time in milliseconds that metrics endpoint would time out. Default is 30s. +# Set it to 0 to disable timeout. +metricsServletTimeoutMs=30000 + diff --git a/conf/pulsar_env.sh b/conf/pulsar_env.sh index c7bba23c234d9..f95d0ac83c13a 100755 --- a/conf/pulsar_env.sh +++ b/conf/pulsar_env.sh @@ -44,9 +44,6 @@ # Extra options to be passed to the jvm PULSAR_MEM=${PULSAR_MEM:-"-Xms2g -Xmx2g -XX:MaxDirectMemorySize=4g"} -# Garbage collection options -PULSAR_GC=${PULSAR_GC:-"-XX:+UseZGC -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch"} - if [ -z "$JAVA_HOME" ]; then JAVA_BIN=java else @@ -67,6 +64,16 @@ for token in $("$JAVA_BIN" -version 2>&1 | grep 'version "'); do fi done +# Garbage collection options +if [ -z "$PULSAR_GC" ]; then + PULSAR_GC="-XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch" + if [[ $JAVA_MAJOR_VERSION -ge 21 ]]; then + PULSAR_GC="-XX:+UseZGC -XX:+ZGenerational ${PULSAR_GC}" + else + PULSAR_GC="-XX:+UseZGC ${PULSAR_GC}" + fi +fi + PULSAR_GC_LOG_DIR=${PULSAR_GC_LOG_DIR:-"${PULSAR_LOG_DIR}"} if [[ -z "$PULSAR_GC_LOG" ]]; then @@ -94,3 +101,7 @@ PULSAR_EXTRA_OPTS="${PULSAR_EXTRA_OPTS:-" -Dpulsar.allocator.exit_on_oom=true -D #Wait time before forcefully kill the pulsar server instance, if the stop is not successful #PULSAR_STOP_TIMEOUT= +# Enable semantically stable telemetry for JVM metrics, unless otherwise overridden by the user. +if [ -z "$OTEL_SEMCONV_STABILITY_OPT_IN" ]; then + export OTEL_SEMCONV_STABILITY_OPT_IN=jvm +fi diff --git a/conf/pulsar_tools_env.sh b/conf/pulsar_tools_env.sh index 9d22b73905df3..96ee304bf0b3a 100755 --- a/conf/pulsar_tools_env.sh +++ b/conf/pulsar_tools_env.sh @@ -57,9 +57,6 @@ if [ -n "$PULSAR_MEM" ]; then fi PULSAR_MEM=${PULSAR_MEM:-"-Xmx128m -XX:MaxDirectMemorySize=128m"} -# Garbage collection options -PULSAR_GC=${PULSAR_GC:-" -client "} - # Extra options to be passed to the jvm PULSAR_EXTRA_OPTS="${PULSAR_MEM} ${PULSAR_GC} ${PULSAR_GC_LOG} -Dio.netty.leakDetectionLevel=disabled ${PULSAR_EXTRA_OPTS}" diff --git a/conf/schema_example.conf b/conf/schema_example.json similarity index 100% rename from conf/schema_example.conf rename to conf/schema_example.json diff --git a/conf/standalone.conf b/conf/standalone.conf index 19521bb74696b..535800a43f3e0 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -51,6 +51,16 @@ advertisedAddress= # If true, the real IP addresses of consumers and producers can be obtained when getting topic statistics data. haProxyProtocolEnabled=false +# Enable or disable the use of HA proxy protocol for resolving the client IP for http/https requests. +webServiceHaProxyProtocolEnabled=false + +# Trust X-Forwarded-For header for resolving the client IP for http/https requests. Default is false. +webServiceTrustXForwardedFor=false + +# Add detailed client/remote and server/local addresses and ports to http/https request logging. +# Defaults to true when either webServiceHaProxyProtocolEnabled or webServiceTrustXForwardedFor is enabled. +webServiceLogDetailedAddresses= + # Number of threads to use for Netty IO. Default is set to 2 * Runtime.getRuntime().availableProcessors() numIOThreads= @@ -111,6 +121,10 @@ backlogQuotaDefaultLimitSecond=-1 # Default ttl for namespaces if ttl is not already configured at namespace policies. (disable default-ttl with value 0) ttlDurationDefaultInSeconds=0 +# Additional system subscriptions that will be ignored by ttl check. The cursor names are comma separated. +# Default is empty. +# additionalSystemCursorNames= + # Enable the deletion of inactive topics. This parameter need to cooperate with the allowAutoTopicCreation parameter. # If brokerDeleteInactiveTopicsEnabled is set to true, we should ensure that allowAutoTopicCreation is also set to true. brokerDeleteInactiveTopicsEnabled=true @@ -218,6 +232,25 @@ maxUnackedMessagesPerBroker=0 # limit/2 messages maxUnackedMessagesPerSubscriptionOnBrokerBlocked=0.16 +# For Key_Shared subscriptions, if messages cannot be dispatched to consumers due to a slow consumer +# or a blocked key hash (because of ordering constraints), the broker will continue reading more +# messages from the backlog and attempt to dispatch them to consumers until the number of replay +# messages reaches the calculated threshold. +# Formula: threshold = min(keySharedLookAheadMsgInReplayThresholdPerConsumer * +# connected consumer count, keySharedLookAheadMsgInReplayThresholdPerSubscription). +# Setting this value to 0 will disable the limit calculated per consumer. +keySharedLookAheadMsgInReplayThresholdPerConsumer=2000 + +# For Key_Shared subscriptions, if messages cannot be dispatched to consumers due to a slow consumer +# or a blocked key hash (because of ordering constraints), the broker will continue reading more +# messages from the backlog and attempt to dispatch them to consumers until the number of replay +# messages reaches the calculated threshold. +# Formula: threshold = min(keySharedLookAheadMsgInReplayThresholdPerConsumer * +# connected consumer count, keySharedLookAheadMsgInReplayThresholdPerSubscription). +# This value should be set to a value less than 2 * managedLedgerMaxUnackedRangesToPersist. +# Setting this value to 0 will disable the limit calculated per subscription. +keySharedLookAheadMsgInReplayThresholdPerSubscription=20000 + # Tick time to schedule task that checks topic publish rate limiting across all topics # Reducing to lower value can give more accuracy while throttling publish but # it uses more CPU to perform frequent check. (Disable publish throttling with value 0) @@ -269,7 +302,17 @@ dispatcherReadFailureBackoffMaxTimeInMs=60000 # The read failure backoff mandatory stop time in milliseconds. By default it is 0s. dispatcherReadFailureBackoffMandatoryStopTimeInMs=0 -# Precise dispathcer flow control according to history message number of each entry +# On Shared and KeyShared subscriptions, if all available messages in the subscription are filtered +# out and not dispatched to any consumer, message dispatching will be rescheduled with a backoff +# delay. This parameter sets the initial backoff delay in milliseconds. +dispatcherRetryBackoffInitialTimeInMs=1 + +# On Shared and KeyShared subscriptions, if all available messages in the subscription are filtered +# out and not dispatched to any consumer, message dispatching will be rescheduled with a backoff +# delay. This parameter sets the maximum backoff delay in milliseconds. +dispatcherRetryBackoffMaxTimeInMs=10 + +# Precise dispatcher flow control according to history message number of each entry preciseDispatcherFlowControl=false # Max number of concurrent lookup request broker allows to throttle heavy incoming lookup traffic @@ -281,8 +324,8 @@ maxConcurrentTopicLoadRequest=5000 # Max concurrent non-persistent message can be processed per connection maxConcurrentNonPersistentMessagePerConnection=1000 -# Number of worker threads to serve non-persistent topic -numWorkerThreadsForNonPersistentTopic=8 +# Number of worker threads to serve topic ordered executor +topicOrderedExecutorThreadNum=8 # Enable broker to load persistent topics enablePersistentTopics=true @@ -581,7 +624,7 @@ tokenAudience= # Authentication plugin to use when connecting to bookies bookkeeperClientAuthenticationPlugin= -# BookKeeper auth plugin implementatation specifics parameters name and values +# BookKeeper auth plugin implementation specifics parameters name and values bookkeeperClientAuthenticationParametersName= bookkeeperClientAuthenticationParameters= @@ -697,8 +740,8 @@ bookkeeperUseV2WireProtocol=true # bookkeeperClientExposeStatsToPrometheus=false # If bookkeeperClientExposeStatsToPrometheus is set to true, we can set bookkeeperClientLimitStatsLogging=true -# to limit per_channel_bookie_client metrics. default is false -# bookkeeperClientLimitStatsLogging=false +# to limit per_channel_bookie_client metrics. default is true +# bookkeeperClientLimitStatsLogging=true ### --- Managed Ledger --- ### @@ -807,10 +850,10 @@ managedLedgerNewEntriesCheckDelayInMillis=10 managedLedgerMinimumBacklogCursorsForCaching=0 # Minimum backlog entries for any cursor before start caching reads. -managedLedgerMinimumBacklogEntriesForCaching=100 +managedLedgerMinimumBacklogEntriesForCaching=1000 # Maximum backlog entry difference to prevent caching entries that can't be reused. -managedLedgerMaxBacklogBetweenCursorsForCaching=10000 +managedLedgerMaxBacklogBetweenCursorsForCaching=1000 # Use Open Range-Set to cache unacked messages managedLedgerUnackedRangesOpenCacheSetEnabled=true @@ -821,6 +864,9 @@ managedLedgerPrometheusStatsLatencyRolloverSeconds=60 # Whether trace managed ledger task execution time managedLedgerTraceTaskExecution=true +# Trigger offload on topic load or not. Default is false. +# triggerOffloadOnTopicLoad=false + # If you want to custom bookie ID or use a dynamic network address for the bookie, # you can set this option. # Bookie advertises itself using bookieId rather than @@ -893,25 +939,20 @@ loadBalancerBrokerThresholdShedderPercentage=10 # It only takes effect in the ThresholdShedder strategy. loadBalancerHistoryResourcePercentage=0.9 -# The BandWithIn usage weight when calculating new resource usage. -# It only takes effect in the ThresholdShedder strategy. -loadBalancerBandwithInResourceWeight=1.0 +# The BandWidthIn usage weight when calculating new resource usage. +loadBalancerBandwidthInResourceWeight=1.0 -# The BandWithOut usage weight when calculating new resource usage. -# It only takes effect in the ThresholdShedder strategy. -loadBalancerBandwithOutResourceWeight=1.0 +# The BandWidthOut usage weight when calculating new resource usage. +loadBalancerBandwidthOutResourceWeight=1.0 # The CPU usage weight when calculating new resource usage. -# It only takes effect in the ThresholdShedder strategy. loadBalancerCPUResourceWeight=1.0 # The heap memory usage weight when calculating new resource usage. -# It only takes effect in the ThresholdShedder strategy. -loadBalancerMemoryResourceWeight=1.0 +loadBalancerMemoryResourceWeight=0 # The direct memory usage weight when calculating new resource usage. -# It only takes effect in the ThresholdShedder strategy. -loadBalancerDirectMemoryResourceWeight=1.0 +loadBalancerDirectMemoryResourceWeight=0 # Bundle unload minimum throughput threshold (MB), avoiding bundle unload frequently. # It only takes effect in the ThresholdShedder strategy. @@ -937,6 +978,16 @@ replicationProducerQueueSize=1000 # due to missing ZooKeeper watch (disable with value 0) replicationPolicyCheckDurationSeconds=600 +# Whether the internal replication of the local cluster will trigger topic auto-creation on the remote cluster. +# 1. After enabling namespace-level Geo-Replication: whether the local broker will create topics on the remote +# cluster automatically when calling `pulsar-admin topics create-partitioned-topic`. +# 2. When enabling topic-level Geo-Replication on a partitioned topic: whether the local broker will create topics on +# the remote cluster. +# 3. Whether the internal Geo-Replicator in the local cluster will trigger non-persistent topic auto-creation for +# remote clusters. +# It is not a dynamic config, the default value is "true" to preserve backward-compatible behavior. +createTopicToRemoteClusterForReplication=true + # Default message retention time. 0 means retention is disabled. -1 means data is not removed by time quota defaultRetentionTimeInMinutes=0 @@ -957,6 +1008,9 @@ webSocketNumIoThreads=8 # Number of connections per Broker in Pulsar Client used in WebSocket proxy webSocketConnectionsPerBroker=8 +# Memory limit in MBs for direct memory in Pulsar Client used in WebSocket proxy +webSocketPulsarClientMemoryLimitInMB=0 + # Time in milliseconds that idle WebSocket session times out webSocketSessionIdleTimeoutMillis=300000 @@ -999,6 +1053,14 @@ splitTopicAndPartitionLabelInPrometheus=false # Otherwise, aggregate it by list index. aggregatePublisherStatsByProducerName=false +# Interval between checks to see if cluster is migrated and marks topic migrated +# if cluster is marked migrated. Disable with value 0. (Default disabled). +clusterMigrationCheckDurationSeconds=0 + +# Flag to start cluster migration for topic only after creating all topic's resources +# such as tenant, namespaces, subscriptions at new green cluster. (Default disabled). +clusterMigrationAutoResourceCreation=false + ### --- Schema storage --- ### # The schema storage implementation used by this broker. schemaRegistryStorageClassName=org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorageFactory @@ -1076,7 +1138,7 @@ journalSyncData=false # For each ledger dir, maximum disk space which can be used. # Default is 0.95f. i.e. 95% of disk can be used at most after which nothing will -# be written to that partition. If all ledger dir partions are full, then bookie +# be written to that partition. If all ledger dir partitions are full, then bookie # will turn to readonly mode if 'readOnlyModeEnabled=true' is set, else it will # shutdown. # Valid values should be in between 0 and 1 (exclusive). @@ -1201,6 +1263,10 @@ functionsWorkerEnablePackageManagement=false # These settings are left here for compatibility +# Number of worker threads to serve non-persistent topic +# Deprecated - use topicOrderedExecutorThreadNum instead. +numWorkerThreadsForNonPersistentTopic=8 + # Zookeeper session timeout in milliseconds # Deprecated: use metadataStoreSessionTimeoutMillis zooKeeperSessionTimeoutMillis=-1 @@ -1225,6 +1291,10 @@ configurationStoreServers= # Deprecated: use managedLedgerMaxUnackedRangesToPersistInMetadataStore managedLedgerMaxUnackedRangesToPersistInZooKeeper=-1 +# After enabling this feature, Pulsar will stop delivery messages to clients if the cursor metadata is too large to +# persist, it will help to reduce the duplicates caused by the ack state that can not be fully persistent. +dispatcherPauseOnAckStatePersistentEnabled=false + # Whether to enable the delayed delivery for messages. # If disabled, messages will be immediately delivered and there will # be no tracking overhead. @@ -1237,13 +1307,11 @@ delayedDeliveryTrackerFactoryClassName=org.apache.pulsar.broker.delayed.InMemory # Control the tick time for when retrying on delayed delivery, # affecting the accuracy of the delivery time compared to the scheduled time. -# Note that this time is used to configure the HashedWheelTimer's tick time for the -# InMemoryDelayedDeliveryTrackerFactory (the default DelayedDeliverTrackerFactory). +# Note that this time is used to configure the HashedWheelTimer's tick time. # Default is 1 second. delayedDeliveryTickTimeMillis=1000 -# When using the InMemoryDelayedDeliveryTrackerFactory (the default DelayedDeliverTrackerFactory), whether -# the deliverAt time is strictly followed. When false (default), messages may be sent to consumers before the deliverAt +# Whether the deliverAt time is strictly followed. When false (default), messages may be sent to consumers before the deliverAt # time by as much as the tickTimeMillis. This can reduce the overhead on the broker of maintaining the delayed index # for a potentially very short time period. When true, messages will not be sent to consumer until the deliverAt time # has passed, and they may be as late as the deliverAt time plus the tickTimeMillis for the topic plus the @@ -1267,3 +1335,21 @@ delayedDeliveryMaxIndexesPerBucketSnapshotSegment=5000 # after reaching the max buckets limitation, the adjacent buckets will be merged. # (disable with value -1) delayedDeliveryMaxNumBuckets=-1 + +# The directory to locate broker interceptors +brokerInterceptorsDirectory=./interceptors + +# List of broker interceptor to load, which is a list of broker interceptor names +brokerInterceptors= + +# Enable or disable the broker interceptor, which is only used for testing for now +disableBrokerInterceptors=true + +# Whether retain null-key message during topic compaction +topicCompactionRetainNullKey=false + +# Class name of the factory that implements the topic compaction service. +# If value is "org.apache.pulsar.compaction.EventTimeCompactionServiceFactory", +# will create topic compaction service based on message eventTime. +# By default compaction service is based on message publishing order. +compactionServiceFactoryClassName=org.apache.pulsar.compaction.PulsarCompactionServiceFactory \ No newline at end of file diff --git a/conf/websocket.conf b/conf/websocket.conf index 2e2824a838c6f..91f7f7d4c23bb 100644 --- a/conf/websocket.conf +++ b/conf/websocket.conf @@ -46,6 +46,16 @@ statusFilePath= # Hostname or IP address the service binds on, default is 0.0.0.0. bindAddress=0.0.0.0 +# Enable or disable the use of HA proxy protocol for resolving the client IP for http/https requests. +webServiceHaProxyProtocolEnabled=false + +# Trust X-Forwarded-For header for resolving the client IP for http/https requests. Default is false. +webServiceTrustXForwardedFor=false + +# Add detailed client/remote and server/local addresses and ports to http/https request logging. +# Defaults to true when either webServiceHaProxyProtocolEnabled or webServiceTrustXForwardedFor is enabled. +webServiceLogDetailedAddresses= + # Name of the pulsar cluster to connect to clusterName= @@ -61,6 +71,9 @@ numHttpServerThreads= # Number of connections per Broker in Pulsar Client used in WebSocket proxy webSocketConnectionsPerBroker= +# Memory limit in MBs for direct memory in Pulsar Client used in WebSocket proxy +webSocketPulsarClientMemoryLimitInMB=0 + # Time in milliseconds that idle WebSocket session times out webSocketSessionIdleTimeoutMillis=300000 @@ -194,3 +207,6 @@ zooKeeperSessionTimeoutMillis=-1 # ZooKeeper cache expiry time in seconds # Deprecated: use metadataStoreCacheExpirySeconds zooKeeperCacheExpirySeconds=-1 + +# CryptoKeyReader factory classname to support encryption at websocket. +cryptoKeyReaderFactoryClassName= diff --git a/conf/zookeeper.conf b/conf/zookeeper.conf index 73240556d484c..db85e688e5c60 100644 --- a/conf/zookeeper.conf +++ b/conf/zookeeper.conf @@ -63,7 +63,7 @@ forceSync=yes # Default: false sslQuorum=false -# Enable TLS Certificate reloading for Quorum and Server connentions +# Enable TLS Certificate reloading for Quorum and Server connections # Follows Pulsar's general default to reload these files. sslQuorumReloadCertFiles=true client.certReload=true diff --git a/deployment/kubernetes/README.md b/deployment/kubernetes/README.md index 9f4f0e6773349..b42a840c19781 100644 --- a/deployment/kubernetes/README.md +++ b/deployment/kubernetes/README.md @@ -22,5 +22,5 @@ This directory contains the Kubernetes services definitions for all the components required to do a complete Pulsar deployment. -Refer to [Kubernetes.md](../../site2/docs/deploy-kubernetes.md) document for instructions on +Refer to [Kubernetes.md](https://pulsar.apache.org/docs/deploy-kubernetes/) document for instructions on how to deploy Pulsar on a Kubernetes cluster. diff --git a/deployment/terraform-ansible/templates/broker.conf b/deployment/terraform-ansible/templates/broker.conf index f42d4c807d5d9..ff3677174024c 100644 --- a/deployment/terraform-ansible/templates/broker.conf +++ b/deployment/terraform-ansible/templates/broker.conf @@ -320,7 +320,7 @@ dispatcherMinReadBatchSize=1 # Max number of entries to dispatch for a shared subscription. By default it is 20 entries. dispatcherMaxRoundRobinBatchSize=20 -# Precise dispathcer flow control according to history message number of each entry +# Precise dispatcher flow control according to history message number of each entry preciseDispatcherFlowControl=false # Max number of concurrent lookup request broker allows to throttle heavy incoming lookup traffic @@ -332,8 +332,8 @@ maxConcurrentTopicLoadRequest=5000 # Max concurrent non-persistent message can be processed per connection maxConcurrentNonPersistentMessagePerConnection=1000 -# Number of worker threads to serve non-persistent topic -numWorkerThreadsForNonPersistentTopic=8 +# Number of worker threads to serve topic ordered executor +topicOrderedExecutorThreadNum=8 # Enable broker to load persistent topics enablePersistentTopics=true @@ -638,7 +638,7 @@ bookkeeperMetadataServiceUri= # Authentication plugin to use when connecting to bookies bookkeeperClientAuthenticationPlugin= -# BookKeeper auth plugin implementatation specifics parameters name and values +# BookKeeper auth plugin implementation specifics parameters name and values bookkeeperClientAuthenticationParametersName= bookkeeperClientAuthenticationParameters= @@ -745,8 +745,8 @@ bookkeeperExplicitLacIntervalInMills=0 # bookkeeperClientExposeStatsToPrometheus=false # If bookkeeperClientExposeStatsToPrometheus is set to true, we can set bookkeeperClientLimitStatsLogging=true -# to limit per_channel_bookie_client metrics. default is false -# bookkeeperClientLimitStatsLogging=false +# to limit per_channel_bookie_client metrics. default is true +# bookkeeperClientLimitStatsLogging=true ### --- Managed Ledger --- ### @@ -944,7 +944,7 @@ defaultNamespaceBundleSplitAlgorithm=range_equally_divide loadBalancerLoadSheddingStrategy=org.apache.pulsar.broker.loadbalance.impl.ThresholdShedder # The broker resource usage threshold. -# When the broker resource usage is gratter than the pulsar cluster average resource usge, +# When the broker resource usage is greater than the pulsar cluster average resource usge, # the threshold shedder will be triggered to offload bundles from the broker. # It only take effect in ThresholdShedder strategy. loadBalancerBrokerThresholdShedderPercentage=10 @@ -953,27 +953,27 @@ loadBalancerBrokerThresholdShedderPercentage=10 # It only take effect in ThresholdShedder strategy. loadBalancerHistoryResourcePercentage=0.9 -# The BandWithIn usage weight when calculating new resourde usage. +# The BandWidthIn usage weight when calculating new resourde usage. # It only take effect in ThresholdShedder strategy. -loadBalancerBandwithInResourceWeight=1.0 +loadBalancerBandwidthInResourceWeight=1.0 -# The BandWithOut usage weight when calculating new resourde usage. +# The BandWidthOut usage weight when calculating new resourde usage. # It only take effect in ThresholdShedder strategy. -loadBalancerBandwithOutResourceWeight=1.0 +loadBalancerBandwidthOutResourceWeight=1.0 -# The CPU usage weight when calculating new resourde usage. +# The CPU usage weight when calculating new resource usage. # It only take effect in ThresholdShedder strategy. loadBalancerCPUResourceWeight=1.0 -# The heap memory usage weight when calculating new resourde usage. +# The heap memory usage weight when calculating new resource usage. # It only take effect in ThresholdShedder strategy. -loadBalancerMemoryResourceWeight=1.0 +loadBalancerMemoryResourceWeight=0 -# The direct memory usage weight when calculating new resourde usage. +# The direct memory usage weight when calculating new resource usage. # It only take effect in ThresholdShedder strategy. -loadBalancerDirectMemoryResourceWeight=1.0 +loadBalancerDirectMemoryResourceWeight=0 -# Bundle unload minimum throughput threshold (MB), avoding bundle unload frequently. +# Bundle unload minimum throughput threshold (MB), avoiding bundle unload frequently. # It only take effect in ThresholdShedder strategy. loadBalancerBundleUnloadMinThroughputThreshold=10 @@ -995,7 +995,7 @@ replicatorPrefix=pulsar.repl # Duration to check replication policy to avoid replicator inconsistency # due to missing ZooKeeper watch (disable with value 0) -replicatioPolicyCheckDurationSeconds=600 +replicationPolicyCheckDurationSeconds=600 # Default message retention time. 0 means retention is disabled. -1 means data is not removed by time quota defaultRetentionTimeInMinutes=0 @@ -1127,6 +1127,10 @@ fileSystemURI= ### --- Deprecated config variables --- ### +# Number of worker threads to serve non-persistent topic +# Deprecated - use topicOrderedExecutorThreadNum instead. +numWorkerThreadsForNonPersistentTopic=8 + # Deprecated. Use configurationStoreServers globalZookeeperServers={{ zookeeper_servers }} diff --git a/deployment/terraform-ansible/templates/client.conf b/deployment/terraform-ansible/templates/client.conf index ba1d396bf8423..755577cf38e03 100644 --- a/deployment/terraform-ansible/templates/client.conf +++ b/deployment/terraform-ansible/templates/client.conf @@ -41,7 +41,7 @@ authPlugin= # authParams=tlsCertFile:/path/to/client-cert.pem,tlsKeyFile:/path/to/client-key.pem authParams= -# Allow TLS connections to servers whose certificate cannot be +# Allow TLS connections to servers whose certificate cannot # be verified to have been signed by a trusted certificate # authority. tlsAllowInsecureConnection=false diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index 568d76922bf4e..813c4d26d9391 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar distribution - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-io-distribution @@ -137,7 +136,6 @@ org.owasp dependency-check-maven - ${dependency-check-maven.version} diff --git a/distribution/io/src/assemble/io.xml b/distribution/io/src/assemble/io.xml index 5b652170fdbb5..f98ee14bb20c9 100644 --- a/distribution/io/src/assemble/io.xml +++ b/distribution/io/src/assemble/io.xml @@ -81,5 +81,6 @@ ${basedir}/../../pulsar-io/solr/target/pulsar-io-solr-${project.version}.nar ${basedir}/../../pulsar-io/dynamodb/target/pulsar-io-dynamodb-${project.version}.nar ${basedir}/../../pulsar-io/alluxio/target/pulsar-io-alluxio-${project.version}.nar + ${basedir}/../../pulsar-io/azure-data-explorer/target/pulsar-io-azuredataexplorer-${project.version}.nar diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index d23ebec2ef26d..38beeacde8ba4 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar distribution - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-offloader-distribution diff --git a/distribution/pom.xml b/distribution/pom.xml index 36a3fa1c5835a..67604e145dd73 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT distribution diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index f804c9c54b9cd..36641dea20f0c 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar distribution - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-server-distribution @@ -40,6 +39,25 @@ ${project.version} + + ${project.groupId} + pulsar-metadata + ${project.version} + + + + ${project.groupId} + jetcd-core-shaded + ${project.version} + shaded + + + + ${project.groupId} + pulsar-docs-tools + ${project.version} + + ${project.groupId} pulsar-proxy @@ -149,6 +167,12 @@ io.dropwizard.metrics metrics-graphite + + + amqp-client + com.rabbitmq + + @@ -168,7 +192,7 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl @@ -253,6 +277,10 @@ io.grpc grpc-all + + org.bouncycastle + bcpkix-jdk18on + io.perfmark @@ -362,22 +390,6 @@ true - - - - ${project.groupId} - pulsar-presto-distribution - ${project.version} - tar.gz - provided - - - * - * - - - - diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 220f0ac0758b8..8c6e2cfa7159a 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -244,28 +244,31 @@ This projects includes binary packages with the following licenses: The Apache Software License, Version 2.0 * JCommander -- com.beust-jcommander-1.82.jar + * Picocli + - info.picocli-picocli-4.7.5.jar + - info.picocli-picocli-shell-jline3-4.7.5.jar * High Performance Primitive Collections for Java -- com.carrotsearch-hppc-0.9.1.jar * Jackson - - com.fasterxml.jackson.core-jackson-annotations-2.14.2.jar - - com.fasterxml.jackson.core-jackson-core-2.14.2.jar - - com.fasterxml.jackson.core-jackson-databind-2.14.2.jar - - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.14.2.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.14.2.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.14.2.jar - - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.14.2.jar - - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.14.2.jar - - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.14.2.jar - - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.14.2.jar - - com.fasterxml.jackson.module-jackson-module-parameter-names-2.14.2.jar + - com.fasterxml.jackson.core-jackson-annotations-2.17.2.jar + - com.fasterxml.jackson.core-jackson-core-2.17.2.jar + - com.fasterxml.jackson.core-jackson-databind-2.17.2.jar + - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.17.2.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.17.2.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.17.2.jar + - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.17.2.jar + - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.17.2.jar + - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.17.2.jar + - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.17.2.jar + - com.fasterxml.jackson.module-jackson-module-parameter-names-2.17.2.jar * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar - * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.0.1.jar - * Bitbucket -- org.bitbucket.b_c-jose4j-0.9.3.jar + * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.17.0.jar + * Bitbucket -- org.bitbucket.b_c-jose4j-0.9.4.jar * Gson - com.google.code.gson-gson-2.8.9.jar - io.gsonfire-gson-fire-1.8.5.jar * Guava - - com.google.guava-guava-31.0.1-jre.jar + - com.google.guava-guava-32.1.2-jre.jar - com.google.guava-failureaccess-1.0.1.jar - com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar * J2ObjC Annotations -- com.google.j2objc-j2objc-annotations-1.3.jar @@ -281,45 +284,44 @@ The Apache Software License, Version 2.0 - commons-cli-commons-cli-1.5.0.jar - commons-codec-commons-codec-1.15.jar - commons-configuration-commons-configuration-1.10.jar - - commons-io-commons-io-2.8.0.jar + - commons-io-commons-io-2.14.0.jar - commons-lang-commons-lang-2.6.jar - commons-logging-commons-logging-1.1.1.jar - org.apache.commons-commons-collections4-4.4.jar - - org.apache.commons-commons-compress-1.21.jar + - org.apache.commons-commons-compress-1.26.0.jar - org.apache.commons-commons-lang3-3.11.jar - org.apache.commons-commons-text-1.10.0.jar * Netty - - io.netty-netty-buffer-4.1.89.Final.jar - - io.netty-netty-codec-4.1.89.Final.jar - - io.netty-netty-codec-dns-4.1.89.Final.jar - - io.netty-netty-codec-http-4.1.89.Final.jar - - io.netty-netty-codec-http2-4.1.89.Final.jar - - io.netty-netty-codec-socks-4.1.89.Final.jar - - io.netty-netty-codec-haproxy-4.1.89.Final.jar - - io.netty-netty-common-4.1.89.Final.jar - - io.netty-netty-handler-4.1.89.Final.jar - - io.netty-netty-handler-proxy-4.1.89.Final.jar - - io.netty-netty-resolver-4.1.89.Final.jar - - io.netty-netty-resolver-dns-4.1.89.Final.jar - - io.netty-netty-resolver-dns-classes-macos-4.1.89.Final.jar - - io.netty-netty-resolver-dns-native-macos-4.1.89.Final-osx-aarch_64.jar - - io.netty-netty-resolver-dns-native-macos-4.1.89.Final-osx-x86_64.jar - - io.netty-netty-transport-4.1.89.Final.jar - - io.netty-netty-transport-classes-epoll-4.1.89.Final.jar - - io.netty-netty-transport-native-epoll-4.1.89.Final-linux-x86_64.jar - - io.netty-netty-transport-native-epoll-4.1.89.Final.jar - - io.netty-netty-transport-native-unix-common-4.1.89.Final.jar - - io.netty-netty-transport-native-unix-common-4.1.89.Final-linux-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.56.Final.jar - - io.netty-netty-tcnative-boringssl-static-2.0.56.Final-linux-aarch_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.56.Final-linux-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.56.Final-osx-aarch_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.56.Final-osx-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.56.Final-windows-x86_64.jar - - io.netty-netty-tcnative-classes-2.0.56.Final.jar - - io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.18.Final.jar - - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.18.Final-linux-x86_64.jar - - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.18.Final-linux-aarch_64.jar + - io.netty-netty-buffer-4.1.113.Final.jar + - io.netty-netty-codec-4.1.113.Final.jar + - io.netty-netty-codec-dns-4.1.113.Final.jar + - io.netty-netty-codec-http-4.1.113.Final.jar + - io.netty-netty-codec-http2-4.1.113.Final.jar + - io.netty-netty-codec-socks-4.1.113.Final.jar + - io.netty-netty-codec-haproxy-4.1.113.Final.jar + - io.netty-netty-common-4.1.113.Final.jar + - io.netty-netty-handler-4.1.113.Final.jar + - io.netty-netty-handler-proxy-4.1.113.Final.jar + - io.netty-netty-resolver-4.1.113.Final.jar + - io.netty-netty-resolver-dns-4.1.113.Final.jar + - io.netty-netty-resolver-dns-classes-macos-4.1.113.Final.jar + - io.netty-netty-resolver-dns-native-macos-4.1.113.Final-osx-aarch_64.jar + - io.netty-netty-resolver-dns-native-macos-4.1.113.Final-osx-x86_64.jar + - io.netty-netty-transport-4.1.113.Final.jar + - io.netty-netty-transport-classes-epoll-4.1.113.Final.jar + - io.netty-netty-transport-native-epoll-4.1.113.Final-linux-aarch_64.jar + - io.netty-netty-transport-native-epoll-4.1.113.Final-linux-x86_64.jar + - io.netty-netty-transport-native-unix-common-4.1.113.Final.jar + - io.netty-netty-tcnative-boringssl-static-2.0.66.Final.jar + - io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-aarch_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-aarch_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.66.Final-windows-x86_64.jar + - io.netty-netty-tcnative-classes-2.0.66.Final.jar + - io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.24.Final.jar + - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.24.Final-linux-x86_64.jar + - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.24.Final-linux-aarch_64.jar * Prometheus client - io.prometheus.jmx-collector-0.16.1.jar - io.prometheus-simpleclient-0.16.0.jar @@ -334,82 +336,91 @@ The Apache Software License, Version 2.0 - io.prometheus-simpleclient_tracer_common-0.16.0.jar - io.prometheus-simpleclient_tracer_otel-0.16.0.jar - io.prometheus-simpleclient_tracer_otel_agent-0.16.0.jar + * Prometheus exporter + - io.prometheus-prometheus-metrics-config-1.2.1.jar + - io.prometheus-prometheus-metrics-exporter-common-1.2.1.jar + - io.prometheus-prometheus-metrics-exporter-httpserver-1.2.1.jar + - io.prometheus-prometheus-metrics-exposition-formats-1.2.1.jar + - io.prometheus-prometheus-metrics-model-1.2.1.jar + - io.prometheus-prometheus-metrics-shaded-protobuf-1.2.1.jar * Jakarta Bean Validation API - jakarta.validation-jakarta.validation-api-2.0.2.jar - javax.validation-validation-api-1.1.0.Final.jar * Log4J - - org.apache.logging.log4j-log4j-api-2.18.0.jar - - org.apache.logging.log4j-log4j-core-2.18.0.jar - - org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar - - org.apache.logging.log4j-log4j-web-2.18.0.jar + - org.apache.logging.log4j-log4j-api-2.23.1.jar + - org.apache.logging.log4j-log4j-core-2.23.1.jar + - org.apache.logging.log4j-log4j-slf4j2-impl-2.23.1.jar + - org.apache.logging.log4j-log4j-web-2.23.1.jar * Java Native Access JNA - net.java.dev.jna-jna-jpms-5.12.1.jar - net.java.dev.jna-jna-platform-jpms-5.12.1.jar * BookKeeper - - org.apache.bookkeeper-bookkeeper-common-4.16.1.jar - - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.1.jar - - org.apache.bookkeeper-bookkeeper-proto-4.16.1.jar - - org.apache.bookkeeper-bookkeeper-server-4.16.1.jar - - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.1.jar - - org.apache.bookkeeper-circe-checksum-4.16.1.jar - - org.apache.bookkeeper-cpu-affinity-4.16.1.jar - - org.apache.bookkeeper-statelib-4.16.1.jar - - org.apache.bookkeeper-stream-storage-api-4.16.1.jar - - org.apache.bookkeeper-stream-storage-common-4.16.1.jar - - org.apache.bookkeeper-stream-storage-java-client-4.16.1.jar - - org.apache.bookkeeper-stream-storage-java-client-base-4.16.1.jar - - org.apache.bookkeeper-stream-storage-proto-4.16.1.jar - - org.apache.bookkeeper-stream-storage-server-4.16.1.jar - - org.apache.bookkeeper-stream-storage-service-api-4.16.1.jar - - org.apache.bookkeeper-stream-storage-service-impl-4.16.1.jar - - org.apache.bookkeeper.http-http-server-4.16.1.jar - - org.apache.bookkeeper.http-vertx-http-server-4.16.1.jar - - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.1.jar - - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.1.jar - - org.apache.distributedlog-distributedlog-common-4.16.1.jar - - org.apache.distributedlog-distributedlog-core-4.16.1-tests.jar - - org.apache.distributedlog-distributedlog-core-4.16.1.jar - - org.apache.distributedlog-distributedlog-protocol-4.16.1.jar - - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.1.jar - - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.1.jar - - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.1.jar - - org.apache.bookkeeper-native-io-4.16.1.jar + - org.apache.bookkeeper-bookkeeper-common-4.17.1.jar + - org.apache.bookkeeper-bookkeeper-common-allocator-4.17.1.jar + - org.apache.bookkeeper-bookkeeper-proto-4.17.1.jar + - org.apache.bookkeeper-bookkeeper-server-4.17.1.jar + - org.apache.bookkeeper-bookkeeper-tools-framework-4.17.1.jar + - org.apache.bookkeeper-circe-checksum-4.17.1.jar + - org.apache.bookkeeper-cpu-affinity-4.17.1.jar + - org.apache.bookkeeper-statelib-4.17.1.jar + - org.apache.bookkeeper-stream-storage-api-4.17.1.jar + - org.apache.bookkeeper-stream-storage-common-4.17.1.jar + - org.apache.bookkeeper-stream-storage-java-client-4.17.1.jar + - org.apache.bookkeeper-stream-storage-java-client-base-4.17.1.jar + - org.apache.bookkeeper-stream-storage-proto-4.17.1.jar + - org.apache.bookkeeper-stream-storage-server-4.17.1.jar + - org.apache.bookkeeper-stream-storage-service-api-4.17.1.jar + - org.apache.bookkeeper-stream-storage-service-impl-4.17.1.jar + - org.apache.bookkeeper.http-http-server-4.17.1.jar + - org.apache.bookkeeper.http-vertx-http-server-4.17.1.jar + - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.17.1.jar + - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.17.1.jar + - org.apache.distributedlog-distributedlog-common-4.17.1.jar + - org.apache.distributedlog-distributedlog-core-4.17.1-tests.jar + - org.apache.distributedlog-distributedlog-core-4.17.1.jar + - org.apache.distributedlog-distributedlog-protocol-4.17.1.jar + - org.apache.bookkeeper.stats-codahale-metrics-provider-4.17.1.jar + - org.apache.bookkeeper-bookkeeper-slogger-api-4.17.1.jar + - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.17.1.jar + - org.apache.bookkeeper-native-io-4.17.1.jar * Apache HTTP Client - org.apache.httpcomponents-httpclient-4.5.13.jar - org.apache.httpcomponents-httpcore-4.4.15.jar * AirCompressor - - io.airlift-aircompressor-0.20.jar + - io.airlift-aircompressor-0.27.jar * AsyncHttpClient - org.asynchttpclient-async-http-client-2.12.1.jar - org.asynchttpclient-async-http-client-netty-utils-2.12.1.jar * Jetty - - org.eclipse.jetty-jetty-client-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-continuation-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-http-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-io-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-proxy-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-security-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-server-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-servlet-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-servlets-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-util-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-util-ajax-9.4.48.v20220622.jar - - org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.48.v20220622.jar - - org.eclipse.jetty.websocket-websocket-api-9.4.48.v20220622.jar - - org.eclipse.jetty.websocket-websocket-client-9.4.48.v20220622.jar - - org.eclipse.jetty.websocket-websocket-common-9.4.48.v20220622.jar - - org.eclipse.jetty.websocket-websocket-server-9.4.48.v20220622.jar - - org.eclipse.jetty.websocket-websocket-servlet-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-alpn-server-9.4.48.v20220622.jar + - org.eclipse.jetty-jetty-client-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-continuation-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-http-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-io-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-proxy-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-security-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-server-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-servlet-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-servlets-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-util-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-util-ajax-9.4.54.v20240208.jar + - org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.54.v20240208.jar + - org.eclipse.jetty.websocket-websocket-api-9.4.54.v20240208.jar + - org.eclipse.jetty.websocket-websocket-client-9.4.54.v20240208.jar + - org.eclipse.jetty.websocket-websocket-common-9.4.54.v20240208.jar + - org.eclipse.jetty.websocket-websocket-server-9.4.54.v20240208.jar + - org.eclipse.jetty.websocket-websocket-servlet-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.54.v20240208.jar + - org.eclipse.jetty-jetty-alpn-server-9.4.54.v20240208.jar * SnakeYaml -- org.yaml-snakeyaml-2.0.jar * RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar - * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.5.1.jar + * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.24.0.jar * Apache Thrift - org.apache.thrift-libthrift-0.14.2.jar * OkHttp3 - com.squareup.okhttp3-logging-interceptor-4.9.3.jar - com.squareup.okhttp3-okhttp-4.9.3.jar - * Okio - com.squareup.okio-okio-2.8.0.jar + * Okio + - com.squareup.okio-okio-3.4.0.jar + - com.squareup.okio-okio-jvm-3.4.0.jar * Javassist -- org.javassist-javassist-3.25.0-GA.jar * Kotlin Standard Lib - org.jetbrains.kotlin-kotlin-stdlib-1.8.20.jar @@ -418,33 +429,39 @@ The Apache Software License, Version 2.0 - org.jetbrains.kotlin-kotlin-stdlib-jdk8-1.8.20.jar - org.jetbrains-annotations-13.0.jar * gRPC - - io.grpc-grpc-all-1.45.1.jar - - io.grpc-grpc-auth-1.45.1.jar - - io.grpc-grpc-context-1.45.1.jar - - io.grpc-grpc-core-1.45.1.jar - - io.grpc-grpc-netty-1.45.1.jar - - io.grpc-grpc-protobuf-1.45.1.jar - - io.grpc-grpc-protobuf-lite-1.45.1.jar - - io.grpc-grpc-stub-1.45.1.jar - - io.grpc-grpc-alts-1.45.1.jar - - io.grpc-grpc-api-1.45.1.jar - - io.grpc-grpc-grpclb-1.45.1.jar - - io.grpc-grpc-netty-shaded-1.45.1.jar - - io.grpc-grpc-services-1.45.1.jar - - io.grpc-grpc-xds-1.45.1.jar - - io.grpc-grpc-rls-1.45.1.jar + - io.grpc-grpc-all-1.56.1.jar + - io.grpc-grpc-auth-1.56.1.jar + - io.grpc-grpc-context-1.56.1.jar + - io.grpc-grpc-core-1.56.1.jar + - io.grpc-grpc-protobuf-1.56.1.jar + - io.grpc-grpc-protobuf-lite-1.56.1.jar + - io.grpc-grpc-stub-1.56.1.jar + - io.grpc-grpc-alts-1.56.1.jar + - io.grpc-grpc-api-1.56.1.jar + - io.grpc-grpc-grpclb-1.56.1.jar + - io.grpc-grpc-netty-shaded-1.56.1.jar + - io.grpc-grpc-services-1.56.1.jar + - io.grpc-grpc-xds-1.56.1.jar + - io.grpc-grpc-rls-1.56.1.jar + - io.grpc-grpc-servlet-1.56.1.jar + - io.grpc-grpc-servlet-jakarta-1.56.1.jar + - io.grpc-grpc-util-1.60.0.jar * Perfmark - - io.perfmark-perfmark-api-0.19.0.jar + - io.perfmark-perfmark-api-0.26.0.jar * OpenCensus - io.opencensus-opencensus-api-0.28.0.jar - io.opencensus-opencensus-contrib-http-util-0.28.0.jar - io.opencensus-opencensus-proto-0.2.0.jar * Jodah - net.jodah-typetools-0.5.0.jar - - net.jodah-failsafe-2.4.4.jar + - dev.failsafe-failsafe-3.3.2.jar + * Byte Buddy + - net.bytebuddy-byte-buddy-1.14.12.jar + * zt-zip + - org.zeroturnaround-zt-zip-1.17.jar * Apache Avro - - org.apache.avro-avro-1.10.2.jar - - org.apache.avro-avro-protobuf-1.10.2.jar + - org.apache.avro-avro-1.11.4.jar + - org.apache.avro-avro-protobuf-1.11.4.jar * Apache Curator - org.apache.curator-curator-client-5.1.0.jar - org.apache.curator-curator-framework-5.1.0.jar @@ -462,6 +479,11 @@ The Apache Software License, Version 2.0 - io.dropwizard.metrics-metrics-jmx-4.1.12.1.jar * Prometheus - io.prometheus-simpleclient_httpserver-0.16.0.jar + * Oxia + - io.streamnative.oxia-oxia-client-api-0.4.5.jar + - io.streamnative.oxia-oxia-client-0.4.5.jar + * OpenHFT + - net.openhft-zero-allocation-hashing-0.16.jar * Java JSON WebTokens - io.jsonwebtoken-jjwt-api-0.11.1.jar - io.jsonwebtoken-jjwt-impl-0.11.1.jar @@ -469,36 +491,53 @@ The Apache Software License, Version 2.0 * JCTools - Java Concurrency Tools for the JVM - org.jctools-jctools-core-2.1.2.jar * Vertx - - io.vertx-vertx-auth-common-4.3.8.jar - - io.vertx-vertx-bridge-common-4.3.8.jar - - io.vertx-vertx-core-4.3.8.jar - - io.vertx-vertx-web-4.3.8.jar - - io.vertx-vertx-web-common-4.3.8.jar - - io.vertx-vertx-grpc-4.3.5.jar + - io.vertx-vertx-auth-common-4.5.10.jar + - io.vertx-vertx-bridge-common-4.5.10.jar + - io.vertx-vertx-core-4.5.10.jar + - io.vertx-vertx-web-4.5.10.jar + - io.vertx-vertx-web-common-4.5.10.jar * Apache ZooKeeper - - org.apache.zookeeper-zookeeper-3.8.1.jar - - org.apache.zookeeper-zookeeper-jute-3.8.1.jar - - org.apache.zookeeper-zookeeper-prometheus-metrics-3.8.1.jar + - org.apache.zookeeper-zookeeper-3.9.2.jar + - org.apache.zookeeper-zookeeper-jute-3.9.2.jar + - org.apache.zookeeper-zookeeper-prometheus-metrics-3.9.2.jar * Snappy Java - - org.xerial.snappy-snappy-java-1.1.8.4.jar + - org.xerial.snappy-snappy-java-1.1.10.5.jar * Google HTTP Client - com.google.http-client-google-http-client-gson-1.41.0.jar - com.google.http-client-google-http-client-1.41.0.jar - - com.google.auto.value-auto-value-annotations-1.9.jar - - com.google.re2j-re2j-1.5.jar - * Jetcd - - io.etcd-jetcd-api-0.7.5.jar - - io.etcd-jetcd-common-0.7.5.jar - - io.etcd-jetcd-core-0.7.5.jar - - io.etcd-jetcd-grpc-0.7.5.jar + - com.google.auto.value-auto-value-annotations-1.10.1.jar + - com.google.re2j-re2j-1.7.jar + * Jetcd - shaded * IPAddress - - com.github.seancfoley-ipaddress-5.3.3.jar + - com.github.seancfoley-ipaddress-5.5.0.jar * RxJava - io.reactivex.rxjava3-rxjava-3.0.1.jar - * RabbitMQ Java Client - - com.rabbitmq-amqp-client-5.5.3.jar * RoaringBitmap - - org.roaringbitmap-RoaringBitmap-0.9.44.jar + - org.roaringbitmap-RoaringBitmap-1.2.0.jar + * OpenTelemetry + - io.opentelemetry-opentelemetry-api-1.38.0.jar + - io.opentelemetry-opentelemetry-api-incubator-1.38.0-alpha.jar + - io.opentelemetry-opentelemetry-context-1.38.0.jar + - io.opentelemetry-opentelemetry-exporter-common-1.38.0.jar + - io.opentelemetry-opentelemetry-exporter-otlp-1.38.0.jar + - io.opentelemetry-opentelemetry-exporter-otlp-common-1.38.0.jar + - io.opentelemetry-opentelemetry-exporter-prometheus-1.38.0-alpha.jar + - io.opentelemetry-opentelemetry-exporter-sender-okhttp-1.38.0.jar + - io.opentelemetry-opentelemetry-sdk-1.38.0.jar + - io.opentelemetry-opentelemetry-sdk-common-1.38.0.jar + - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-1.38.0.jar + - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-spi-1.38.0.jar + - io.opentelemetry-opentelemetry-sdk-logs-1.38.0.jar + - io.opentelemetry-opentelemetry-sdk-metrics-1.38.0.jar + - io.opentelemetry-opentelemetry-sdk-trace-1.38.0.jar + - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-1.33.3.jar + - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-semconv-1.33.3-alpha.jar + - io.opentelemetry.instrumentation-opentelemetry-resources-1.33.3-alpha.jar + - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java17-1.33.3-alpha.jar + - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java8-1.33.3-alpha.jar + - io.opentelemetry.semconv-opentelemetry-semconv-1.25.0-alpha.jar + * Spotify completable-futures + - com.spotify-completable-futures-0.3.6.jar BSD 3-clause "New" or "Revised" License * Google auth library @@ -515,10 +554,10 @@ BSD 2-Clause License MIT License * Java SemVer -- com.github.zafarkhaja-java-semver-0.9.0.jar -- ../licenses/LICENSE-SemVer.txt * SLF4J -- ../licenses/LICENSE-SLF4J.txt - - org.slf4j-slf4j-api-1.7.32.jar - - org.slf4j-jcl-over-slf4j-1.7.32.jar + - org.slf4j-slf4j-api-2.0.13.jar + - org.slf4j-jcl-over-slf4j-2.0.13.jar * The Checker Framework - - org.checkerframework-checker-qual-3.12.0.jar + - org.checkerframework-checker-qual-3.33.0.jar * oshi - com.github.oshi-oshi-core-java11-6.4.0.jar * Auth0, Inc. @@ -526,12 +565,11 @@ MIT License - com.auth0-jwks-rsa-0.22.0.jar Protocol Buffers License * Protocol Buffers - - com.google.protobuf-protobuf-java-3.19.6.jar -- ../licenses/LICENSE-protobuf.txt - - com.google.protobuf-protobuf-java-util-3.19.6.jar -- ../licenses/LICENSE-protobuf.txt + - com.google.protobuf-protobuf-java-3.25.5.jar -- ../licenses/LICENSE-protobuf.txt + - com.google.protobuf-protobuf-java-util-3.25.5.jar -- ../licenses/LICENSE-protobuf.txt CDDL-1.1 -- ../licenses/LICENSE-CDDL-1.1.txt * Java Annotations API - - javax.annotation-javax.annotation-api-1.3.2.jar - com.sun.activation-javax.activation-1.2.0.jar - javax.xml.bind-jaxb-api-2.3.1.jar * Java Servlet API -- javax.servlet-javax.servlet-api-3.1.0.jar @@ -544,16 +582,16 @@ CDDL-1.1 -- ../licenses/LICENSE-CDDL-1.1.txt - org.glassfish.hk2-osgi-resource-locator-1.0.3.jar - org.glassfish.hk2.external-aopalliance-repackaged-2.6.1.jar * Jersey - - org.glassfish.jersey.containers-jersey-container-servlet-2.34.jar - - org.glassfish.jersey.containers-jersey-container-servlet-core-2.34.jar - - org.glassfish.jersey.core-jersey-client-2.34.jar - - org.glassfish.jersey.core-jersey-common-2.34.jar - - org.glassfish.jersey.core-jersey-server-2.34.jar - - org.glassfish.jersey.ext-jersey-entity-filtering-2.34.jar - - org.glassfish.jersey.media-jersey-media-json-jackson-2.34.jar - - org.glassfish.jersey.media-jersey-media-multipart-2.34.jar - - org.glassfish.jersey.inject-jersey-hk2-2.34.jar - * Mimepull -- org.jvnet.mimepull-mimepull-1.9.13.jar + - org.glassfish.jersey.containers-jersey-container-servlet-2.42.jar + - org.glassfish.jersey.containers-jersey-container-servlet-core-2.42.jar + - org.glassfish.jersey.core-jersey-client-2.42.jar + - org.glassfish.jersey.core-jersey-common-2.42.jar + - org.glassfish.jersey.core-jersey-server-2.42.jar + - org.glassfish.jersey.ext-jersey-entity-filtering-2.42.jar + - org.glassfish.jersey.media-jersey-media-json-jackson-2.42.jar + - org.glassfish.jersey.media-jersey-media-multipart-2.42.jar + - org.glassfish.jersey.inject-jersey-hk2-2.42.jar + * Mimepull -- org.jvnet.mimepull-mimepull-1.9.15.jar Eclipse Distribution License 1.0 -- ../licenses/LICENSE-EDL-1.0.txt * Jakarta Activation @@ -574,10 +612,9 @@ Creative Commons Attribution License Bouncy Castle License * Bouncy Castle -- ../licenses/LICENSE-bouncycastle.txt - - org.bouncycastle-bcpkix-jdk15on-1.69.jar - - org.bouncycastle-bcprov-ext-jdk15on-1.69.jar - - org.bouncycastle-bcprov-jdk15on-1.69.jar - - org.bouncycastle-bcutil-jdk15on-1.69.jar + - org.bouncycastle-bcpkix-jdk18on-1.78.1.jar + - org.bouncycastle-bcprov-jdk18on-1.78.1.jar + - org.bouncycastle-bcutil-jdk18on-1.78.1.jar ------------------------ diff --git a/distribution/server/src/assemble/NOTICE.bin.txt b/distribution/server/src/assemble/NOTICE.bin.txt index bc5e2e6d63b0e..7705416042a17 100644 --- a/distribution/server/src/assemble/NOTICE.bin.txt +++ b/distribution/server/src/assemble/NOTICE.bin.txt @@ -1,6 +1,6 @@ Apache Pulsar -Copyright 2017-2022 The Apache Software Foundation +Copyright 2017-2024 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/distribution/server/src/assemble/bin.xml b/distribution/server/src/assemble/bin.xml index 41ac24d0582da..4dfec015c0e6f 100644 --- a/distribution/server/src/assemble/bin.xml +++ b/distribution/server/src/assemble/bin.xml @@ -60,22 +60,6 @@ ${basedir}/../../pulsar-functions/python-examples examples/python-examples - - ${basedir}/../../pulsar-sql/presto-distribution/target/pulsar-presto-distribution - trino - - bin - bin/** - - - - ${basedir}/../../pulsar-sql/presto-distribution/target/pulsar-presto-distribution - trino - - bin/** - - 755 - @@ -126,7 +110,7 @@ lib false - compile + runtime false @@ -135,12 +119,15 @@ org.apache.pulsar:pulsar-functions-runtime-all - org.projectlombok:lombok - org.apache.pulsar:pulsar-functions-api-examples *:tar.gz + + org.codehaus.mojo:animal-sniffer-annotations + com.google.android:annotations + + net.java.dev.jna:jna diff --git a/distribution/shell/pom.xml b/distribution/shell/pom.xml index 9e3134a75e5bf..45108aba68f87 100644 --- a/distribution/shell/pom.xml +++ b/distribution/shell/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar distribution - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-shell-distribution @@ -51,7 +50,7 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 711890809f1bf..6e0bacb2e8845 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -309,74 +309,73 @@ pulsar-client-cpp/lib/checksum/crc32c_sw.cc This projects includes binary packages with the following licenses: The Apache Software License, Version 2.0 - * JCommander -- jcommander-1.82.jar + * Picocli + - picocli-4.7.5.jar + - picocli-shell-jline3-4.7.5.jar * Jackson - - jackson-annotations-2.14.2.jar - - jackson-core-2.14.2.jar - - jackson-databind-2.14.2.jar - - jackson-dataformat-yaml-2.14.2.jar - - jackson-jaxrs-base-2.14.2.jar - - jackson-jaxrs-json-provider-2.14.2.jar - - jackson-module-jaxb-annotations-2.14.2.jar - - jackson-module-jsonSchema-2.14.2.jar - - jackson-datatype-jdk8-2.14.2.jar - - jackson-datatype-jsr310-2.14.2.jar - - jackson-module-parameter-names-2.14.2.jar + - jackson-annotations-2.17.2.jar + - jackson-core-2.17.2.jar + - jackson-databind-2.17.2.jar + - jackson-dataformat-yaml-2.17.2.jar + - jackson-jaxrs-base-2.17.2.jar + - jackson-jaxrs-json-provider-2.17.2.jar + - jackson-module-jaxb-annotations-2.17.2.jar + - jackson-module-jsonSchema-2.17.2.jar + - jackson-datatype-jdk8-2.17.2.jar + - jackson-datatype-jsr310-2.17.2.jar + - jackson-module-parameter-names-2.17.2.jar * Conscrypt -- conscrypt-openjdk-uber-2.5.2.jar * Gson - gson-2.8.9.jar * Guava - - guava-31.0.1-jre.jar + - guava-32.1.2-jre.jar - failureaccess-1.0.1.jar - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar * J2ObjC Annotations -- j2objc-annotations-1.3.jar * Netty Reactive Streams -- netty-reactive-streams-2.0.6.jar - * Swagger - - swagger-annotations-1.6.2.jar - - swagger-core-1.6.2.jar - - swagger-models-1.6.2.jar + * Swagger -- swagger-annotations-1.6.2.jar * DataSketches - memory-0.8.3.jar - sketches-core-0.8.3.jar * Apache Commons - commons-codec-1.15.jar - commons-configuration-1.10.jar - - commons-io-2.8.0.jar + - commons-io-2.14.0.jar - commons-lang-2.6.jar - commons-logging-1.2.jar - commons-lang3-3.11.jar - commons-text-1.10.0.jar - - commons-compress-1.21.jar + - commons-compress-1.26.0.jar * Netty - - netty-buffer-4.1.89.Final.jar - - netty-codec-4.1.89.Final.jar - - netty-codec-dns-4.1.89.Final.jar - - netty-codec-http-4.1.89.Final.jar - - netty-codec-socks-4.1.89.Final.jar - - netty-codec-haproxy-4.1.89.Final.jar - - netty-common-4.1.89.Final.jar - - netty-handler-4.1.89.Final.jar - - netty-handler-proxy-4.1.89.Final.jar - - netty-resolver-4.1.89.Final.jar - - netty-resolver-dns-4.1.89.Final.jar - - netty-transport-4.1.89.Final.jar - - netty-transport-classes-epoll-4.1.89.Final.jar - - netty-transport-native-epoll-4.1.89.Final-linux-x86_64.jar - - netty-transport-native-unix-common-4.1.89.Final.jar - - netty-transport-native-unix-common-4.1.89.Final-linux-x86_64.jar - - netty-tcnative-boringssl-static-2.0.56.Final.jar - - netty-tcnative-boringssl-static-2.0.56.Final-linux-aarch_64.jar - - netty-tcnative-boringssl-static-2.0.56.Final-linux-x86_64.jar - - netty-tcnative-boringssl-static-2.0.56.Final-osx-aarch_64.jar - - netty-tcnative-boringssl-static-2.0.56.Final-osx-x86_64.jar - - netty-tcnative-boringssl-static-2.0.56.Final-windows-x86_64.jar - - netty-tcnative-classes-2.0.56.Final.jar - - netty-incubator-transport-classes-io_uring-0.0.18.Final.jar - - netty-incubator-transport-native-io_uring-0.0.18.Final-linux-aarch_64.jar - - netty-incubator-transport-native-io_uring-0.0.18.Final-linux-x86_64.jar - - netty-resolver-dns-classes-macos-4.1.89.Final.jar - - netty-resolver-dns-native-macos-4.1.89.Final-osx-aarch_64.jar - - netty-resolver-dns-native-macos-4.1.89.Final-osx-x86_64.jar + - netty-buffer-4.1.113.Final.jar + - netty-codec-4.1.113.Final.jar + - netty-codec-dns-4.1.113.Final.jar + - netty-codec-http-4.1.113.Final.jar + - netty-codec-socks-4.1.113.Final.jar + - netty-codec-haproxy-4.1.113.Final.jar + - netty-common-4.1.113.Final.jar + - netty-handler-4.1.113.Final.jar + - netty-handler-proxy-4.1.113.Final.jar + - netty-resolver-4.1.113.Final.jar + - netty-resolver-dns-4.1.113.Final.jar + - netty-transport-4.1.113.Final.jar + - netty-transport-classes-epoll-4.1.113.Final.jar + - netty-transport-native-epoll-4.1.113.Final-linux-aarch_64.jar + - netty-transport-native-epoll-4.1.113.Final-linux-x86_64.jar + - netty-transport-native-unix-common-4.1.113.Final.jar + - netty-tcnative-boringssl-static-2.0.66.Final.jar + - netty-tcnative-boringssl-static-2.0.66.Final-linux-aarch_64.jar + - netty-tcnative-boringssl-static-2.0.66.Final-linux-x86_64.jar + - netty-tcnative-boringssl-static-2.0.66.Final-osx-aarch_64.jar + - netty-tcnative-boringssl-static-2.0.66.Final-osx-x86_64.jar + - netty-tcnative-boringssl-static-2.0.66.Final-windows-x86_64.jar + - netty-tcnative-classes-2.0.66.Final.jar + - netty-incubator-transport-classes-io_uring-0.0.24.Final.jar + - netty-incubator-transport-native-io_uring-0.0.24.Final-linux-aarch_64.jar + - netty-incubator-transport-native-io_uring-0.0.24.Final-linux-x86_64.jar + - netty-resolver-dns-classes-macos-4.1.113.Final.jar + - netty-resolver-dns-native-macos-4.1.113.Final-osx-aarch_64.jar + - netty-resolver-dns-native-macos-4.1.113.Final-osx-x86_64.jar * Prometheus client - simpleclient-0.16.0.jar - simpleclient_log4j2-0.16.0.jar @@ -384,35 +383,41 @@ The Apache Software License, Version 2.0 - simpleclient_tracer_otel-0.16.0.jar - simpleclient_tracer_otel_agent-0.16.0.jar * Log4J - - log4j-api-2.18.0.jar - - log4j-core-2.18.0.jar - - log4j-slf4j-impl-2.18.0.jar - - log4j-web-2.18.0.jar + - log4j-api-2.23.1.jar + - log4j-core-2.23.1.jar + - log4j-slf4j2-impl-2.23.1.jar + - log4j-web-2.23.1.jar + * OpenTelemetry + - opentelemetry-api-1.38.0.jar + - opentelemetry-api-incubator-1.38.0-alpha.jar + - opentelemetry-context-1.38.0.jar * BookKeeper - - bookkeeper-common-allocator-4.16.1.jar - - cpu-affinity-4.16.1.jar - - circe-checksum-4.16.1.jar + - bookkeeper-common-allocator-4.17.1.jar + - cpu-affinity-4.17.1.jar + - circe-checksum-4.17.1.jar * AirCompressor - - aircompressor-0.20.jar + - aircompressor-0.27.jar * AsyncHttpClient - async-http-client-2.12.1.jar - async-http-client-netty-utils-2.12.1.jar * Jetty - - jetty-client-9.4.48.v20220622.jar - - jetty-http-9.4.48.v20220622.jar - - jetty-io-9.4.48.v20220622.jar - - jetty-util-9.4.48.v20220622.jar - - javax-websocket-client-impl-9.4.48.v20220622.jar - - websocket-api-9.4.48.v20220622.jar - - websocket-client-9.4.48.v20220622.jar - - websocket-common-9.4.48.v20220622.jar + - jetty-client-9.4.54.v20240208.jar + - jetty-http-9.4.54.v20240208.jar + - jetty-io-9.4.54.v20240208.jar + - jetty-util-9.4.54.v20240208.jar + - javax-websocket-client-impl-9.4.54.v20240208.jar + - websocket-api-9.4.54.v20240208.jar + - websocket-client-9.4.54.v20240208.jar + - websocket-common-9.4.54.v20240208.jar * SnakeYaml -- snakeyaml-2.0.jar - * Google Error Prone Annotations - error_prone_annotations-2.5.1.jar + * Google Error Prone Annotations - error_prone_annotations-2.24.0.jar * Javassist -- javassist-3.25.0-GA.jar * Apache Avro - - avro-1.10.2.jar - - avro-protobuf-1.10.2.jar + - avro-1.11.4.jar + - avro-protobuf-1.11.4.jar + * RE2j -- re2j-1.7.jar + * Spotify completable-futures -- completable-futures-0.3.6.jar BSD 3-clause "New" or "Revised" License * JSR305 -- jsr305-3.0.2.jar -- ../licenses/LICENSE-JSR305.txt @@ -420,13 +425,13 @@ BSD 3-clause "New" or "Revised" License MIT License * SLF4J -- ../licenses/LICENSE-SLF4J.txt - - slf4j-api-1.7.32.jar + - slf4j-api-2.0.13.jar * The Checker Framework - - checker-qual-3.12.0.jar + - checker-qual-3.33.0.jar Protocol Buffers License * Protocol Buffers - - protobuf-java-3.19.6.jar -- ../licenses/LICENSE-protobuf.txt + - protobuf-java-3.25.5.jar -- ../licenses/LICENSE-protobuf.txt CDDL-1.1 -- ../licenses/LICENSE-CDDL-1.1.txt * Java Annotations API @@ -442,13 +447,13 @@ CDDL-1.1 -- ../licenses/LICENSE-CDDL-1.1.txt - aopalliance-repackaged-2.6.1.jar - osgi-resource-locator-1.0.3.jar * Jersey - - jersey-client-2.34.jar - - jersey-common-2.34.jar - - jersey-entity-filtering-2.34.jar - - jersey-media-json-jackson-2.34.jar - - jersey-media-multipart-2.34.jar - - jersey-hk2-2.34.jar - * Mimepull -- mimepull-1.9.13.jar + - jersey-client-2.42.jar + - jersey-common-2.42.jar + - jersey-entity-filtering-2.42.jar + - jersey-media-json-jackson-2.42.jar + - jersey-media-multipart-2.42.jar + - jersey-hk2-2.42.jar + * Mimepull -- mimepull-1.9.15.jar Eclipse Distribution License 1.0 -- ../licenses/LICENSE-EDL-1.0.txt * Jakarta Activation @@ -470,10 +475,9 @@ Creative Commons Attribution License Bouncy Castle License * Bouncy Castle -- ../licenses/LICENSE-bouncycastle.txt - - bcpkix-jdk15on-1.69.jar - - bcprov-ext-jdk15on-1.69.jar - - bcprov-jdk15on-1.69.jar - - bcutil-jdk15on-1.69.jar + - bcpkix-jdk18on-1.78.1.jar + - bcprov-jdk18on-1.78.1.jar + - bcutil-jdk18on-1.78.1.jar ------------------------ diff --git a/distribution/shell/src/assemble/NOTICE.bin.txt b/distribution/shell/src/assemble/NOTICE.bin.txt index bc5e2e6d63b0e..41c9bd7d217da 100644 --- a/distribution/shell/src/assemble/NOTICE.bin.txt +++ b/distribution/shell/src/assemble/NOTICE.bin.txt @@ -1,6 +1,6 @@ Apache Pulsar -Copyright 2017-2022 The Apache Software Foundation +Copyright 2017-2024 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). @@ -17,6 +17,9 @@ Copyright (c) 2005 Brian Goetz and Tim Peierls JCommander Copyright 2010 Cedric Beust cedric@beust.com +picocli (http://picocli.info) +Copyright 2017 Remko Popma + EA Agent Loader Copyright (C) 2015 Electronic Arts Inc. All rights reserved. diff --git a/doap_Pulsar.rdf b/doap_Pulsar.rdf new file mode 100644 index 0000000000000..6c81a5c3648e3 --- /dev/null +++ b/doap_Pulsar.rdf @@ -0,0 +1,47 @@ + + + + + + 2018-09-19 + + Apache Pulsar + + + Apache Pulsar is a distributed messaging and streaming platform built for the cloud. + Apache Pulsar is an all-in-one messaging and streaming platform. Messages can be consumed and acknowledged individually or consumed as streams with less than 10ms of latency. Its layered architecture allows rapid scaling across hundreds of nodes, without data reshuffling. + +Its features include multi-tenancy with resource separation and access control, geo-replication across regions, tiered storage and support for six official client languages. It supports up to one million unique topics and is designed to simplify your application architecture. + + + + Java + + + + + + + + + + diff --git a/docker-compose/kitchen-sink/docker-compose.yml b/docker-compose/kitchen-sink/docker-compose.yml index 7323c4253468d..2a51b382a2341 100644 --- a/docker-compose/kitchen-sink/docker-compose.yml +++ b/docker-compose/kitchen-sink/docker-compose.yml @@ -342,7 +342,6 @@ services: # Requires PF_ prefix for some reason in the code PF_pulsarFunctionsCluster: test PF_workerId: fnc1 - PF_pulsarFunctionsCluster: test # This setting does not appear to accept more than one host PF_configurationStoreServers: zk1:2181 PF_pulsarServiceUrl: pulsar://proxy1:6650 @@ -368,8 +367,7 @@ services: image: apachepulsar/pulsar-all:latest restart: on-failure command: > - bash -c "bin/apply-config-from-env-with-prefix.py SQL_PREFIX_ trino/conf/catalog/pulsar.properties && \ - bin/apply-config-from-env.py conf/pulsar_env.sh && \ + bash -c "bin/apply-config-from-env.py conf/pulsar_env.sh && \ bin/watch-znode.py -z $$zookeeperServers -p /initialized-$$clusterName -w && \ exec bin/pulsar sql-worker run" environment: diff --git a/docker/build.sh b/docker/build.sh index d8ab4bea882c4..88be44f23e73f 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -18,7 +18,7 @@ # under the License. # -ROOT_DIR=$(git rev-parse --show-toplevel) +ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. >/dev/null 2>&1 && pwd )" cd $ROOT_DIR/docker mvn package -Pdocker,-main diff --git a/docker/get-version.sh b/docker/get-version.sh index 07145e7cf0c18..0b736baf3b270 100755 --- a/docker/get-version.sh +++ b/docker/get-version.sh @@ -18,7 +18,7 @@ # under the License. # -ROOT_DIR=$(git rev-parse --show-toplevel) +ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. >/dev/null 2>&1 && pwd )" pushd $ROOT_DIR > /dev/null diff --git a/docker/glibc-package/Dockerfile b/docker/glibc-package/Dockerfile new file mode 100644 index 0000000000000..016e5c622365f --- /dev/null +++ b/docker/glibc-package/Dockerfile @@ -0,0 +1,80 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +ARG GLIBC_VERSION=2.38 +ARG ALPINE_VERSION=3.20 + +FROM ubuntu:22.04 as build +ARG GLIBC_VERSION + +RUN apt-get -q update \ + && apt-get -qy install \ + bison \ + build-essential \ + gawk \ + gettext \ + openssl \ + python3 \ + texinfo \ + wget + +# Build GLibc +RUN wget -qO- https://ftpmirror.gnu.org/libc/glibc-${GLIBC_VERSION}.tar.gz | tar zxf - +RUN mkdir /glibc-build +WORKDIR /glibc-build +RUN /glibc-${GLIBC_VERSION}/configure \ + --prefix=/usr/glibc-compat \ + --libdir=/usr/glibc-compat/lib \ + --libexecdir=/usr/glibc-compat/lib \ + --enable-multi-arch \ + --enable-stack-protector=strong +RUN make -j$(nproc) +RUN make install +RUN tar --dereference --hard-dereference -zcf /glibc-bin.tar.gz /usr/glibc-compat + + +################################################ +## Build the APK package +FROM alpine:$ALPINE_VERSION as apk +ARG GLIBC_VERSION + +RUN apk add abuild sudo build-base + +RUN mkdir /build +WORKDIR build + +COPY --from=build /glibc-bin.tar.gz /build + +COPY ./scripts /build + +RUN echo "pkgver=\"${GLIBC_VERSION}\"" >> /build/APKBUILD +RUN echo "sha512sums=\"$(sha512sum glibc-bin.tar.gz ld.so.conf)\"" >> /build/APKBUILD + +RUN abuild-keygen -a -i -n +RUN abuild -F -c -r + +################################################ +## Last stage - Only leaves the packages +FROM busybox +ARG GLIBC_VERSION + +RUN mkdir -p /root/packages +COPY --from=apk /root/packages/*/glibc-${GLIBC_VERSION}-r0.apk /root/packages +COPY --from=apk /root/packages/*/glibc-bin-${GLIBC_VERSION}-r0.apk /root/packages diff --git a/docker/glibc-package/README.md b/docker/glibc-package/README.md new file mode 100644 index 0000000000000..ee1f643705ad2 --- /dev/null +++ b/docker/glibc-package/README.md @@ -0,0 +1,39 @@ + + +# GLibc compatibility package + +This directory includes the Docker scripts to build an image with GLibc compiled for Alpine Linux. + +This is used to ensure plugins that are going to be used in the Pulsar image and that are depeding on GLibc, will +still be working correctly in the Alpine Image. (eg: Netty Tc-Native and Kinesis Producer Library). + +This image only needs to be re-created when we want to upgrade to a newer version of GLibc. + +# Steps + +1. Change the version in the Dockerfile for this directory. +2. Rebuild the image and push it to Docker Hub: +``` +docker buildx build --platform=linux/amd64,linux/arm64 -t apachepulsar/glibc-base:2.38 . --push +``` + +The image tag is then used in `docker/pulsar/Dockerfile`. diff --git a/docker/glibc-package/scripts/APKBUILD b/docker/glibc-package/scripts/APKBUILD new file mode 100644 index 0000000000000..0545508f0a7d4 --- /dev/null +++ b/docker/glibc-package/scripts/APKBUILD @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +pkgname="glibc" +pkgrel="0" +pkgdesc="GNU C Library compatibility layer" +arch="all" +url="https:/pulsar.apache.org" +license="LGPL" +options="!check" +source="glibc-bin.tar.gz +ld.so.conf" +subpackages="${pkgname}-bin ${pkgname}-dev" +triggers="glibc-bin.trigger=/lib:/usr/lib:/usr/glibc-compat/lib" +depends="libuuid libgcc" + +package() { + mkdir -p $pkgdir/lib $pkgdir/usr/glibc-compat/lib/locale $pkgdir/usr/glibc-compat/lib64 $pkgdir/etc $pkgdir/usr/glibc-compat/etc/ + cp -a $srcdir/usr $pkgdir + cp $srcdir/ld.so.conf $pkgdir/usr/glibc-compat/etc/ld.so.conf + cd $pkgdir/usr/glibc-compat + rm -rf etc/rpc bin sbin lib/gconv lib/getconf lib/audit share var include + + FILENAME=$(ls $pkgdir/usr/glibc-compat/lib/ld-linux-*.so.*) + LIBNAME=$(basename $FILENAME) + ln -s /usr/glibc-compat/lib/$LIBNAME $pkgdir/lib/$LIBNAME + ln -s /usr/glibc-compat/lib/$LIBNAME $pkgdir/usr/glibc-compat/lib64/$LIBNAME + ln -s /usr/glibc-compat/etc/ld.so.cache $pkgdir/etc/ld.so.cache +} + +bin() { + depends="$pkgname libc6-compat" + mkdir -p $subpkgdir/usr/glibc-compat + cp -a $srcdir/usr/glibc-compat/bin $subpkgdir/usr/glibc-compat + cp -a $srcdir/usr/glibc-compat/sbin $subpkgdir/usr/glibc-compat +} + diff --git a/pulsar-sql/presto-distribution/src/main/resources/launcher.properties b/docker/glibc-package/scripts/glibc-bin.trigger old mode 100644 new mode 100755 similarity index 91% rename from pulsar-sql/presto-distribution/src/main/resources/launcher.properties rename to docker/glibc-package/scripts/glibc-bin.trigger index a8649925414fd..5bae5d7ca2bda --- a/pulsar-sql/presto-distribution/src/main/resources/launcher.properties +++ b/docker/glibc-package/scripts/glibc-bin.trigger @@ -1,3 +1,4 @@ +#!/bin/sh # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -17,5 +18,4 @@ # under the License. # -main-class=io.trino.server.TrinoServer -process-name=pulsar-presto-distribution \ No newline at end of file +/usr/glibc-compat/sbin/ldconfig \ No newline at end of file diff --git a/pulsar-sql/presto-distribution/src/main/resources/conf/log.properties b/docker/glibc-package/scripts/ld.so.conf similarity index 84% rename from pulsar-sql/presto-distribution/src/main/resources/conf/log.properties rename to docker/glibc-package/scripts/ld.so.conf index 4a796b0e19099..6548b9300bb9c 100644 --- a/pulsar-sql/presto-distribution/src/main/resources/conf/log.properties +++ b/docker/glibc-package/scripts/ld.so.conf @@ -17,7 +17,7 @@ # under the License. # -io.trino=INFO -com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory=WARN -com.ning.http.client=WARN -io.trino.server.PluginManager=DEBUG +/usr/local/lib +/usr/glibc-compat/lib +/usr/lib +/lib diff --git a/docker/pom.xml b/docker/pom.xml index 882240925ef24..ffcaec3ffdc30 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT docker-images Apache Pulsar :: Docker Images @@ -60,6 +60,19 @@ pulsar pulsar-all + + + + pl.project13.maven + git-commit-id-plugin + + false + true + false + + + + diff --git a/docker/publish.sh b/docker/publish.sh index 45b338d85f8ef..651fefc1498e9 100755 --- a/docker/publish.sh +++ b/docker/publish.sh @@ -18,7 +18,7 @@ # under the License. # -ROOT_DIR=$(git rev-parse --show-toplevel) +ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. >/dev/null 2>&1 && pwd )" cd $ROOT_DIR/docker # We should only publish images that are made from official and approved releases @@ -49,6 +49,9 @@ fi MVN_VERSION=`./get-version.sh` echo "Pulsar version: ${MVN_VERSION}" +GIT_COMMIT_ID_ABBREV=$(git rev-parse --short=7 HEAD 2>/dev/null || echo no-git) +GIT_BRANCH=$(git branch --show-current 2>/dev/null || echo no-git) +IMAGE_TAG="${MVN_VERSION}-${GIT_COMMIT_ID_ABBREV}" if [[ -z ${DOCKER_REGISTRY} ]]; then docker_registry_org=${DOCKER_ORG} @@ -62,16 +65,21 @@ set -x # Fail if any of the subsequent commands fail set -e -docker tag apachepulsar/pulsar:latest ${docker_registry_org}/pulsar:latest -docker tag apachepulsar/pulsar-all:latest ${docker_registry_org}/pulsar-all:latest +if [[ "$GIT_BRANCH" == "master" ]]; then + docker tag apachepulsar/pulsar:${IMAGE_TAG} ${docker_registry_org}/pulsar:latest + docker tag apachepulsar/pulsar-all:${IMAGE_TAG} ${docker_registry_org}/pulsar-all:latest +fi -docker tag apachepulsar/pulsar:latest ${docker_registry_org}/pulsar:$MVN_VERSION -docker tag apachepulsar/pulsar-all:latest ${docker_registry_org}/pulsar-all:$MVN_VERSION +docker tag apachepulsar/pulsar:${IMAGE_TAG} ${docker_registry_org}/pulsar:$MVN_VERSION +docker tag apachepulsar/pulsar-all:${IMAGE_TAG} ${docker_registry_org}/pulsar-all:$MVN_VERSION # Push all images and tags -docker push ${docker_registry_org}/pulsar:latest -docker push ${docker_registry_org}/pulsar-all:latest +if [[ "$GIT_BRANCH" == "master" ]]; then + docker push ${docker_registry_org}/pulsar:latest + docker push ${docker_registry_org}/pulsar-all:latest +fi + docker push ${docker_registry_org}/pulsar:$MVN_VERSION docker push ${docker_registry_org}/pulsar-all:$MVN_VERSION -echo "Finished pushing images to ${docker_registry_org}" +echo "Finished pushing images to ${docker_registry_org}" \ No newline at end of file diff --git a/docker/pulsar-all/Dockerfile b/docker/pulsar-all/Dockerfile index 42431fc94a067..81ad74b65000f 100644 --- a/docker/pulsar-all/Dockerfile +++ b/docker/pulsar-all/Dockerfile @@ -17,6 +17,7 @@ # under the License. # +ARG PULSAR_IMAGE FROM busybox as pulsar-all ARG PULSAR_IO_DIR @@ -26,6 +27,6 @@ ADD ${PULSAR_IO_DIR} /connectors ADD ${PULSAR_OFFLOADER_TARBALL} / RUN mv /apache-pulsar-offloaders-*/offloaders /offloaders -FROM apachepulsar/pulsar:latest +FROM $PULSAR_IMAGE COPY --from=pulsar-all /connectors /pulsar/connectors COPY --from=pulsar-all /offloaders /pulsar/offloaders diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index 7a2f492632135..b43121dd0f613 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT 4.0.0 pulsar-all-docker-image @@ -66,6 +66,17 @@ + + git-commit-id-no-git + + + ${basedir}/../../.git + + + + no-git + + docker @@ -134,21 +145,22 @@ package build - tag + push - ${docker.organization}/pulsar-all + ${docker.organization}/${docker.image}-all ${project.basedir} - latest - ${project.version} + ${docker.tag} + ${project.version}-${git.commit.id.abbrev} target/apache-pulsar-io-connectors-${project.version}-bin target/pulsar-offloader-distribution-${project.version}-bin.tar.gz + ${docker.organization}/${docker.image}:${project.version}-${git.commit.id.abbrev} @@ -165,29 +177,12 @@ - docker-push - - - - io.fabric8 - docker-maven-plugin - - - default - package - - build - tag - push - - - - - - + + false + linux/amd64,linux/arm64 + - diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 01e53e0152ac6..f8c22dc14a821 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -17,8 +17,13 @@ # under the License. # +ARG ALPINE_VERSION=3.20 +ARG IMAGE_JDK_MAJOR_VERSION=21 + # First create a stage with just the Pulsar tarball and scripts -FROM busybox as pulsar +FROM alpine:$ALPINE_VERSION as pulsar + +RUN apk add zip ARG PULSAR_TARBALL @@ -26,82 +31,120 @@ ADD ${PULSAR_TARBALL} / RUN mv /apache-pulsar-* /pulsar RUN rm -rf /pulsar/bin/*.cmd -COPY scripts/apply-config-from-env.py /pulsar/bin -COPY scripts/apply-config-from-env-with-prefix.py /pulsar/bin -COPY scripts/gen-yml-from-env.py /pulsar/bin -COPY scripts/generate-zookeeper-config.sh /pulsar/bin -COPY scripts/pulsar-zookeeper-ruok.sh /pulsar/bin -COPY scripts/watch-znode.py /pulsar/bin -COPY scripts/install-pulsar-client.sh /pulsar/bin +COPY build-scripts /build-scripts/ +RUN /build-scripts/remove-unnecessary-native-binaries.sh + +COPY scripts/* /pulsar/bin/ # The final image needs to give the root group sufficient permission for Pulsar components # to write to specific directories within /pulsar +# The ownership is changed to uid 10000 to allow using a different root group. This is necessary when running the +# container when gid=0 is prohibited. In that case, the container must be run with uid 10000 with +# any group id != 0 (for example 10001). # The file permissions are preserved when copying files from this builder image to the target image. -RUN for SUBDIRECTORY in conf data download logs; do \ - [ -d /pulsar/$SUBDIRECTORY ] || mkdir /pulsar/$SUBDIRECTORY; \ - chmod -R g+w /pulsar/$SUBDIRECTORY; \ +RUN for SUBDIRECTORY in conf data download logs instances/deps packages-storage; do \ + mkdir -p /pulsar/$SUBDIRECTORY; \ + chmod -R ug+rwx /pulsar/$SUBDIRECTORY; \ + chown -R 10000:0 /pulsar/$SUBDIRECTORY; \ done -# Trino writes logs to this directory (at least during tests), so we need to give the process permission -# to create those log directories. This should be removed when Trino is removed. -RUN chmod g+w /pulsar/trino - -### Create 2nd stage from Ubuntu image -### and add OpenJDK and Python dependencies (for Pulsar functions) - -FROM ubuntu:20.04 - -ARG DEBIAN_FRONTEND=noninteractive -ARG UBUNTU_MIRROR=mirror://mirrors.ubuntu.com/mirrors.txt -ARG UBUNTU_SECURITY_MIRROR=http://security.ubuntu.com/ubuntu/ - -# Install some utilities -RUN sed -i -e "s|http://archive\.ubuntu\.com/ubuntu/|${UBUNTU_MIRROR:-mirror://mirrors.ubuntu.com/mirrors.txt}|g" \ - -e "s|http://security\.ubuntu\.com/ubuntu/|${UBUNTU_SECURITY_MIRROR:-http://security.ubuntu.com/ubuntu/}|g" /etc/apt/sources.list \ - && echo 'Acquire::http::Timeout "30";\nAcquire::ftp::Timeout "30";\nAcquire::Retries "3";' > /etc/apt/apt.conf.d/99timeout_and_retries \ - && apt-get update \ - && apt-get -y dist-upgrade \ - && apt-get -y install --no-install-recommends netcat dnsutils less procps iputils-ping \ - python3 python3-kazoo python3-pip \ - curl ca-certificates wget apt-transport-https - -# Install Eclipse Temurin Package -RUN mkdir -p /etc/apt/keyrings \ - && wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | tee /etc/apt/keyrings/adoptium.asc \ - && echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list \ - && apt-get update \ - && apt-get -y dist-upgrade \ - && apt-get -y install temurin-17-jdk \ - && export ARCH=$(uname -m | sed -r 's/aarch64/arm64/g' | awk '!/arm64/{$0="amd64"}1') \ - && echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/temurin-17-jdk-$ARCH/conf/security/java.security \ - -# Cleanup apt -RUN apt-get -y --purge autoremove \ - && apt-get autoclean \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -RUN pip3 install pyyaml==5.4.1 - -# Pulsar currently writes to the below directories, assuming the default configuration. -# Note that number 4 is the reason that pulsar components need write access to the /pulsar directory. -# 1. /pulsar/data - both bookkeepers and zookeepers use this directory -# 2. /pulsar/logs - function workers write to this directory and pulsar-admin initializes this directory -# 3. /pulsar/download - functions write to this directory -# 4. /pulsar - hadoop writes to this directory -RUN mkdir /pulsar && chmod g+w /pulsar +RUN chmod -R g+rx /pulsar/bin +RUN chmod -R o+rx /pulsar + +# Enable snappy-java to use system lib +RUN echo 'OPTS="$OPTS -Dorg.xerial.snappy.use.systemlib=true"' >> /pulsar/conf/bkenv.sh + +### Create one stage to include JVM distribution +FROM amazoncorretto:${IMAGE_JDK_MAJOR_VERSION}-alpine AS jvm + +RUN apk add --no-cache binutils + +# Use JLink to create a slimmer JDK distribution (see: https://adoptium.net/blog/2021/10/jlink-to-produce-own-runtime/) +# This still includes all JDK modules, though in the future we could compile a list of required modules +RUN /usr/lib/jvm/default-jvm/bin/jlink --add-modules ALL-MODULE-PATH --compress zip-9 --no-man-pages --no-header-files --strip-debug --output /opt/jvm +RUN echo networkaddress.cache.ttl=1 >> /opt/jvm/conf/security/java.security +RUN echo networkaddress.cache.negative.ttl=1 >> /opt/jvm/conf/security/java.security + +## Create one stage to include snappy-java native lib +# Fix the issue when using snappy-java in x86 arch alpine +# See https://github.com/xerial/snappy-java/issues/181 https://github.com/xerial/snappy-java/issues/579 +# We need to ensure that the version of the native library matches the version of snappy-java imported via Maven +FROM alpine:$ALPINE_VERSION AS snappy-java + +ARG SNAPPY_VERSION +RUN apk add git alpine-sdk util-linux cmake autoconf automake libtool openjdk17 maven curl bash tar +ENV JAVA_HOME=/usr +RUN curl -Ls https://github.com/xerial/snappy-java/archive/refs/tags/v$SNAPPY_VERSION.tar.gz | tar zxf - && cd snappy-java-$SNAPPY_VERSION && make clean-native native +FROM apachepulsar/glibc-base:2.38 as glibc + +## Create final stage from Alpine image +## and add OpenJDK and Python dependencies (for Pulsar functions) +FROM alpine:$ALPINE_VERSION +ENV LANG C.UTF-8 + +# Install some utilities, some are required by Pulsar scripts +RUN apk add --no-cache \ + bash \ + python3 \ + py3-pip \ + py3-grpcio \ + py3-yaml \ + gcompat \ + ca-certificates \ + procps \ + curl \ + bind-tools \ + openssl + +# Upgrade all packages to get latest versions with security fixes +RUN apk upgrade --no-cache + +# Python dependencies + +# The grpcio@1.59.3 is installed by apk, and Pulsar-client@3.4.0 requires grpcio>=1.60.0, which causes the grocio to be reinstalled by pip. +# If pip cannot find the grpcio wheel that the doesn't match the OS, the grpcio will be compiled locally. +# Once https://github.com/apache/pulsar-client-python/pull/211 is released, keep only the pulsar-client[all] and kazoo dependencies, and remove comments. +ARG PULSAR_CLIENT_PYTHON_VERSION +RUN echo -e "\ +#pulsar-client[all]==${PULSAR_CLIENT_PYTHON_VERSION}\n\ +pulsar-client==${PULSAR_CLIENT_PYTHON_VERSION}\n\ +# Zookeeper\n\ +kazoo\n\ +# functions\n\ +protobuf>=3.6.1,<=3.20.3\n\ +grpcio>=1.59.3\n\ +apache-bookkeeper-client>=4.16.1\n\ +prometheus_client\n\ +ratelimit\n\ +# avro\n\ +fastavro>=1.9.2\n\ +" > /requirements.txt + +RUN pip3 install --break-system-packages --no-cache-dir --only-binary grpcio -r /requirements.txt +RUN rm /requirements.txt + +# Install GLibc compatibility library +COPY --from=glibc /root/packages /root/packages +RUN apk add --allow-untrusted --force-overwrite /root/packages/glibc-*.apk + +COPY --from=jvm /opt/jvm /opt/jvm +ENV JAVA_HOME=/opt/jvm + +COPY --from=snappy-java /tmp/libsnappyjava.so /usr/lib/libsnappyjava.so + +# The default is /pulsat/bin and cannot be written. +ENV PULSAR_PID_DIR=/pulsar/logs ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE COPY --from=pulsar /pulsar /pulsar -WORKDIR /pulsar - -ARG PULSAR_CLIENT_PYTHON_VERSION -ENV PULSAR_CLIENT_PYTHON_VERSION ${PULSAR_CLIENT_PYTHON_VERSION} -# This script is intentionally run as the root user to make the dependencies available for all UIDs. -RUN chmod +x /pulsar/bin/install-pulsar-client.sh -RUN /pulsar/bin/install-pulsar-client.sh +WORKDIR /pulsar +ENV PATH=$PATH:$JAVA_HOME/bin:/pulsar/bin +# Use musl libc library for RocksDB +ENV ROCKSDB_MUSL_LIBC=true # The UID must be non-zero. Otherwise, it is arbitrary. No logic should rely on its specific value. +ARG DEFAULT_USERNAME=pulsar +RUN adduser ${DEFAULT_USERNAME} -u 10000 -G root -D -H -h /pulsar/data USER 10000 diff --git a/tests/docker-images/latest-version-image/scripts/init-cluster.sh b/docker/pulsar/build-scripts/remove-unnecessary-native-binaries.sh similarity index 51% rename from tests/docker-images/latest-version-image/scripts/init-cluster.sh rename to docker/pulsar/build-scripts/remove-unnecessary-native-binaries.sh index 926845d5a7747..fe97b71179d43 100755 --- a/tests/docker-images/latest-version-image/scripts/init-cluster.sh +++ b/docker/pulsar/build-scripts/remove-unnecessary-native-binaries.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env sh # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -18,21 +18,34 @@ # under the License. # -set -x +set -e -ZNODE="/initialized-$clusterName" +# Retain only native libraries for the architecture of this +# image +ARCH=$(uname -m) -bin/watch-znode.py -z $zkServers -p / -w +# Remove extra binaries for Netty TCNative +if [ "$ARCH" = "aarch64" ] +then + TC_NATIVE_TO_KEEP=linux-aarch_64 +else + TC_NATIVE_TO_KEEP=linux-$ARCH +fi +ls /pulsar/lib/io.netty-netty-tcnative-boringssl-static*Final-*.jar | grep -v $TC_NATIVE_TO_KEEP | xargs rm -bin/watch-znode.py -z $zkServers -p $ZNODE -e -if [ $? != 0 ]; then - echo Initializing cluster - bin/apply-config-from-env.py conf/bookkeeper.conf && - bin/pulsar initialize-cluster-metadata --cluster $clusterName --zookeeper $zkServers \ - --configuration-store $configurationStore --web-service-url http://$pulsarNode:8080/ \ - --broker-service-url pulsar://$pulsarNode:6650/ && - bin/watch-znode.py -z $zkServers -p $ZNODE -c - echo Initialized +# Prune extra libs from RocksDB JAR +mkdir /tmp/rocksdb +cd /tmp/rocksdb +ROCKSDB_JAR=$(ls /pulsar/lib/org.rocksdb-rocksdbjni-*.jar) +unzip $ROCKSDB_JAR > /dev/null + +if [ "$ARCH" = "x86_64" ] +then + ROCKSDB_TO_KEEP=linux64-musl else - echo Already Initialized + ROCKSDB_TO_KEEP=linux-$ARCH-musl fi + +ls librocksdbjni-* | grep -v librocksdbjni-${ROCKSDB_TO_KEEP}.so | xargs rm +rm $ROCKSDB_JAR +zip -r -9 $ROCKSDB_JAR * > /dev/null diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index e1c1503a3f381..481fc319be732 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT 4.0.0 pulsar-docker-image @@ -47,12 +47,18 @@ - - mirror://mirrors.ubuntu.com/mirrors.txt - http://security.ubuntu.com/ubuntu/ - - + + git-commit-id-no-git + + + ${basedir}/../../.git + + + + no-git + + docker @@ -66,23 +72,23 @@ package build - tag + push - ${docker.organization}/pulsar + ${docker.organization}/${docker.image} target/pulsar-server-distribution-${project.version}-bin.tar.gz ${pulsar.client.python.version} - ${UBUNTU_MIRROR} - ${UBUNTU_SECURITY_MIRROR} + ${snappy.version} + ${IMAGE_JDK_MAJOR_VERSION} ${project.basedir} - latest - ${project.version} + ${docker.tag} + ${project.version}-${git.commit.id.abbrev} @@ -117,29 +123,12 @@ - docker-push - - - - io.fabric8 - docker-maven-plugin - - - default - package - - build - tag - push - - - - - - + + false + linux/amd64,linux/arm64 + - diff --git a/docker/pulsar/scripts/apply-config-from-env-with-prefix.py b/docker/pulsar/scripts/apply-config-from-env-with-prefix.py index 58f6c98975005..9943b283a9f89 100755 --- a/docker/pulsar/scripts/apply-config-from-env-with-prefix.py +++ b/docker/pulsar/scripts/apply-config-from-env-with-prefix.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -32,83 +32,8 @@ # update if they exist and ignored if they don't. ############################################################ -import os -import sys - -if len(sys.argv) < 3: - print('Usage: %s [...]' % (sys.argv[0])) - sys.exit(1) - -# Always apply env config to env scripts as well -prefix = sys.argv[1] -conf_files = sys.argv[2:] - -PF_ENV_DEBUG = (os.environ.get('PF_ENV_DEBUG','0') == '1') - -for conf_filename in conf_files: - lines = [] # List of config file lines - keys = {} # Map a key to its line number in the file - - # Load conf file - for line in open(conf_filename): - lines.append(line) - line = line.strip() - if not line or line.startswith('#'): - continue - - try: - k,v = line.split('=', 1) - keys[k] = len(lines) - 1 - except: - if PF_ENV_DEBUG: - print("[%s] skip Processing %s" % (conf_filename, line)) - - # Update values from Env - for k in sorted(os.environ.keys()): - v = os.environ[k].strip() - - # Hide the value in logs if is password. - if "password" in k.lower(): - displayValue = "********" - else: - displayValue = v - - if k.startswith(prefix): - k = k[len(prefix):] - if k in keys: - print('[%s] Applying config %s = %s' % (conf_filename, k, displayValue)) - idx = keys[k] - lines[idx] = '%s=%s\n' % (k, v) - - - # Ensure we have a new-line at the end of the file, to avoid issue - # when appending more lines to the config - lines.append('\n') - - # Add new keys from Env - for k in sorted(os.environ.keys()): - v = os.environ[k] - if not k.startswith(prefix): - continue - - # Hide the value in logs if is password. - if "password" in k.lower(): - displayValue = "********" - else: - displayValue = v - - k = k[len(prefix):] - if k not in keys: - print('[%s] Adding config %s = %s' % (conf_filename, k, displayValue)) - lines.append('%s=%s\n' % (k, v)) - else: - print('[%s] Updating config %s = %s' % (conf_filename, k, displayValue)) - lines[keys[k]] = '%s=%s\n' % (k, v) - - - # Store back the updated config in the same file - f = open(conf_filename, 'w') - for line in lines: - f.write(line) - f.close() +# DEPRECATED: Use "apply-config-from-env.py --prefix MY_PREFIX_ conf_file" instead +# this is not a python script, but a bash script. Call apply-config-from-env.py with the prefix argument +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +"${SCRIPT_DIR}/apply-config-from-env.py" --prefix "$1" "${@:2}" diff --git a/docker/pulsar/scripts/apply-config-from-env.py b/docker/pulsar/scripts/apply-config-from-env.py index b8b479fc15b85..da51f05f8be66 100755 --- a/docker/pulsar/scripts/apply-config-from-env.py +++ b/docker/pulsar/scripts/apply-config-from-env.py @@ -25,18 +25,29 @@ ## ./apply-config-from-env file.conf ## -import os, sys - -if len(sys.argv) < 2: - print('Usage: %s' % (sys.argv[0])) +import os, sys, argparse + +parser = argparse.ArgumentParser(description='Pulsar configuration file customizer based on environment variables') +parser.add_argument('--prefix', default='PULSAR_PREFIX_', help='Prefix for environment variables, default is PULSAR_PREFIX_') +parser.add_argument('conf_files', nargs='*', help='Configuration files') +args = parser.parse_args() +if not args.conf_files: + parser.print_help() sys.exit(1) -# Always apply env config to env scripts as well -conf_files = sys.argv[1:] +env_prefix = args.prefix +conf_files = args.conf_files -PF_ENV_PREFIX = 'PULSAR_PREFIX_' PF_ENV_DEBUG = (os.environ.get('PF_ENV_DEBUG','0') == '1') +# List of keys where the value should not be displayed in logs +sensitive_keys = ["brokerClientAuthenticationParameters", "bookkeeperClientAuthenticationParameters", "tokenSecretKey"] + +def sanitize_display_value(k, v): + if "password" in k.lower() or k in sensitive_keys or (k == "tokenSecretKey" and v.startswith("data:")): + return "********" + return v + for conf_filename in conf_files: lines = [] # List of config file lines keys = {} # Map a key to its line number in the file @@ -47,7 +58,6 @@ line = line.strip() if not line: continue - try: k,v = line.split('=', 1) if k.startswith('#'): @@ -61,37 +71,26 @@ for k in sorted(os.environ.keys()): v = os.environ[k].strip() - # Hide the value in logs if is password. - if "password" in k.lower(): - displayValue = "********" - else: - displayValue = v - - if k.startswith(PF_ENV_PREFIX): - k = k[len(PF_ENV_PREFIX):] if k in keys: + displayValue = sanitize_display_value(k, v) print('[%s] Applying config %s = %s' % (conf_filename, k, displayValue)) idx = keys[k] lines[idx] = '%s=%s\n' % (k, v) - # Ensure we have a new-line at the end of the file, to avoid issue # when appending more lines to the config lines.append('\n') - - # Add new keys from Env + + # Add new keys from Env for k in sorted(os.environ.keys()): - v = os.environ[k] - if not k.startswith(PF_ENV_PREFIX): + if not k.startswith(env_prefix): continue - # Hide the value in logs if is password. - if "password" in k.lower(): - displayValue = "********" - else: - displayValue = v + v = os.environ[k].strip() + k = k[len(env_prefix):] + + displayValue = sanitize_display_value(k, v) - k = k[len(PF_ENV_PREFIX):] if k not in keys: print('[%s] Adding config %s = %s' % (conf_filename, k, displayValue)) lines.append('%s=%s\n' % (k, v)) @@ -99,10 +98,8 @@ print('[%s] Updating config %s = %s' % (conf_filename, k, displayValue)) lines[keys[k]] = '%s=%s\n' % (k, v) - # Store back the updated config in the same file f = open(conf_filename, 'w') for line in lines: f.write(line) - f.close() - + f.close() \ No newline at end of file diff --git a/docker/pulsar/scripts/gen-yml-from-env.py b/docker/pulsar/scripts/gen-yml-from-env.py index ce19436b7e0dd..916b147f0cbba 100755 --- a/docker/pulsar/scripts/gen-yml-from-env.py +++ b/docker/pulsar/scripts/gen-yml-from-env.py @@ -50,6 +50,9 @@ 'brokerClientTlsProtocols', 'webServiceTlsCiphers', 'webServiceTlsProtocols', + 'additionalJavaRuntimeArguments', + 'additionalEnabledConnectorUrlPatterns', + 'additionalEnabledFunctionsUrlPatterns' ] PF_ENV_PREFIX = 'PF_' @@ -61,7 +64,7 @@ conf_files = sys.argv[1:] for conf_filename in conf_files: - conf = yaml.load(open(conf_filename)) + conf = yaml.load(open(conf_filename), Loader=yaml.FullLoader) # update the config modified = False diff --git a/docker/pulsar/scripts/pulsar-zookeeper-ruok.sh b/docker/pulsar/scripts/pulsar-zookeeper-ruok.sh index 7a0228c2386bd..045258696ff0b 100755 --- a/docker/pulsar/scripts/pulsar-zookeeper-ruok.sh +++ b/docker/pulsar/scripts/pulsar-zookeeper-ruok.sh @@ -20,7 +20,7 @@ # Check ZK server status -status=$(echo ruok | nc -q 1 localhost 2181) +status=$({ echo ruok; sleep 1; } | nc 127.0.0.1 2181) if [ "$status" == "imok" ]; then exit 0 else diff --git a/docker/pulsar/scripts/update-ini-from-env.py b/docker/pulsar/scripts/update-ini-from-env.py new file mode 100755 index 0000000000000..6b0d7a795c3f8 --- /dev/null +++ b/docker/pulsar/scripts/update-ini-from-env.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import os +import sys +import configparser +import re + +def get_first_word(section_name): + # Split the section name by any non-word character and return the first word + return re.split(r'\W+', section_name)[0] + +def update_ini_file(ini_file_path, env_prefix): + # Read the existing INI file + config = configparser.ConfigParser() + config.read(ini_file_path) + + # Flag to track if any updates were made + updated = False + + # Iterate over environment variables + for key, value in os.environ.items(): + if env_prefix and not key.startswith(env_prefix): + continue + + stripped_key = key[len(env_prefix):] if env_prefix else key + + # Iterate through sections + for section in config.sections(): + first_word = get_first_word(section) + prefix = first_word + '_' + if stripped_key.startswith(prefix): + config.set(section, stripped_key[len(prefix):], value) + updated = True + break + elif config.has_option(section, stripped_key): + config.set(section, stripped_key, value) + updated = True + break + + # Write the updated INI file only if there were updates + if updated: + with open(ini_file_path, 'w') as configfile: + config.write(configfile) + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Usage: python3 update-ini-from-env.py ") + sys.exit(1) + + ini_file_path = sys.argv[1] + env_prefix = sys.argv[2] + update_ini_file(ini_file_path, env_prefix) \ No newline at end of file diff --git a/docker/pulsar/scripts/update-rocksdb-conf-from-env.py b/docker/pulsar/scripts/update-rocksdb-conf-from-env.py new file mode 100755 index 0000000000000..2e55b455de3b7 --- /dev/null +++ b/docker/pulsar/scripts/update-rocksdb-conf-from-env.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# allows tuning of RocksDB configuration via environment variables which were effective +# before Pulsar 2.11 / BookKeeper 4.15 / https://github.com/apache/bookkeeper/pull/3056 +# the script should be applied to the `conf/entry_location_rocksdb.conf` file + +import os +import sys +import configparser + +# Constants for section keys +DB_OPTIONS = "DBOptions" +CF_OPTIONS = "CFOptions \"default\"" +TABLE_OPTIONS = "TableOptions/BlockBasedTable \"default\"" + +def update_ini_file(ini_file_path): + config = configparser.ConfigParser() + config.read(ini_file_path) + updated = False + + # Mapping of environment variables to INI sections and keys + env_to_ini_mapping = { + "dbStorage_rocksDB_logPath": (DB_OPTIONS, "log_path"), + "dbStorage_rocksDB_logLevel": (DB_OPTIONS, "info_log_level"), + "dbStorage_rocksDB_lz4CompressionEnabled": (CF_OPTIONS, "compression"), + "dbStorage_rocksDB_writeBufferSizeMB": (CF_OPTIONS, "write_buffer_size"), + "dbStorage_rocksDB_sstSizeInMB": (CF_OPTIONS, "target_file_size_base"), + "dbStorage_rocksDB_blockSize": (TABLE_OPTIONS, "block_size"), + "dbStorage_rocksDB_bloomFilterBitsPerKey": (TABLE_OPTIONS, "filter_policy"), + "dbStorage_rocksDB_blockCacheSize": (TABLE_OPTIONS, "block_cache"), + "dbStorage_rocksDB_numLevels": (CF_OPTIONS, "num_levels"), + "dbStorage_rocksDB_numFilesInLevel0": (CF_OPTIONS, "level0_file_num_compaction_trigger"), + "dbStorage_rocksDB_maxSizeInLevel1MB": (CF_OPTIONS, "max_bytes_for_level_base"), + "dbStorage_rocksDB_format_version": (TABLE_OPTIONS, "format_version") + } + + # Type conversion functions + def mb_to_bytes(mb): + return str(int(mb) * 1024 * 1024) + + def str_to_bool(value): + return True if value.lower() in ["true", "1", "yes"] else False + + # Iterate over environment variables + for key, value in os.environ.items(): + if key.startswith("PULSAR_PREFIX_"): + key = key[len("PULSAR_PREFIX_"):] + + if key in env_to_ini_mapping: + section, option = env_to_ini_mapping[key] + if key in ["dbStorage_rocksDB_writeBufferSizeMB", "dbStorage_rocksDB_sstSizeInMB", "dbStorage_rocksDB_maxSizeInLevel1MB"]: + value = mb_to_bytes(value) + elif key == "dbStorage_rocksDB_lz4CompressionEnabled": + value = "kLZ4Compression" if str_to_bool(value) else "kNoCompression" + elif key == "dbStorage_rocksDB_bloomFilterBitsPerKey": + value = "rocksdb.BloomFilter:{}:false".format(value) + if config.get(section, option, fallback=None) != value: + config.set(section, option, value) + updated = True + + # Write the updated INI file only if there were updates + if updated: + with open(ini_file_path, 'w') as configfile: + config.write(configfile) + +if __name__ == "__main__": + ini_file_path = sys.argv[1] if len(sys.argv) > 1 else "conf/entry_location_rocksdb.conf" + update_ini_file(ini_file_path) \ No newline at end of file diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index dfb155c2d5a7d..dd19faad904bc 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,15 +26,23 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT jclouds-shaded Apache Pulsar :: Jclouds shaded + + + 2.10.1 + 32.0.0-jre + 7.0.0 + 2.0.1 + 3.0.0 + 2.0.0 + - org.apache.jclouds jclouds-allblobstore @@ -61,12 +69,48 @@ jclouds-slf4j ${jclouds.version} - - javax.annotation - javax.annotation-api - + + + + com.google.code.gson + gson + ${gson.version} + + + com.google.guava + guava + ${guava.version} + + + com.google.inject + guice + ${guice.version} + + + com.google.inject.extensions + guice-assistedinject + ${guice.version} + + + jakarta.inject + jakarta.inject-api + ${jakarta.inject.api.version} + + + jakarta.ws.rs + jakarta.ws.rs-api + ${jakarta.ws.rs-api.version} + + + jakarta.annotation + jakarta.annotation-api + ${jakarta.annotation-api.version} + + + + @@ -97,13 +141,13 @@ com.google.inject.extensions:guice-multibindings com.google.code.gson:gson org.apache.httpcomponents:* - javax.ws.rs:* com.jamesmurty.utils:* net.iharder:* aopalliance:* - javax.inject:* - javax.annotation:* com.google.errorprone:* + jakarta.inject:jakarta.inject-api + jakarta.annotation:jakarta.annotation-api + jakarta.ws.rs:jakarta.ws.rs-api @@ -112,10 +156,6 @@ com.google org.apache.pulsar.jcloud.shade.com.google - - javax.ws - org.apache.pulsar.jcloud.shade.javax.ws - com.jamesmurty.utils org.apache.pulsar.jcloud.shade.com.jamesmurty.utils @@ -129,18 +169,17 @@ org.apache.pulsar.jcloud.shade.net.iharder - javax.inject - org.apache.pulsar.jcloud.shade.javax.inject + com.google.errorprone + org.apache.pulsar.jcloud.shade.com.google.errorprone - javax.annotation - org.apache.pulsar.jcloud.shade.javax.annotation + jakarta + org.apache.pulsar.jcloud.shade.jakarta - com.google.errorprone - org.apache.pulsar.jcloud.shade.com.google.errorprone + org.aopalliance + org.apache.pulsar.jcloud.shade.org.aopalliance - diff --git a/jetcd-core-shaded/pom.xml b/jetcd-core-shaded/pom.xml new file mode 100644 index 0000000000000..a0885f8509547 --- /dev/null +++ b/jetcd-core-shaded/pom.xml @@ -0,0 +1,198 @@ + + + + 4.0.0 + + org.apache.pulsar + pulsar + 4.0.0-SNAPSHOT + + + jetcd-core-shaded + Apache Pulsar :: jetcd-core shaded + + + + io.etcd + jetcd-core + + + io.grpc + grpc-netty + + + io.netty + * + + + + + io.grpc + grpc-netty-shaded + + + + dev.failsafe + failsafe + + + io.grpc + grpc-protobuf + + + io.grpc + grpc-stub + + + io.grpc + grpc-grpclb + + + io.grpc + grpc-util + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + true + true + false + + + io.etcd:* + io.vertx:* + + + + + + io.vertx + org.apache.pulsar.jetcd.shaded.io.vertx + + + + META-INF/versions/(\d+)/io/vertx/ + META-INF/versions/$1/org/apache/pulsar/jetcd/shaded/io/vertx/ + true + + + + io.grpc.netty + io.grpc.netty.shaded.io.grpc.netty + + + + io.netty + io.grpc.netty.shaded.io.netty + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + META-INF/maven/${project.groupId}/${project.artifactId}/pom.xml + + + + + + + true + + + + + + META-INF/maven/${project.groupId}/${project.artifactId}/pom.xml + ${project.basedir}/dependency-reduced-pom.xml + + + + true + shaded + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + attach-shaded-jar + package + + attach-artifact + + + + + ${project.build.directory}/${project.artifactId}-${project.version}-shaded.jar + jar + shaded + + + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + ${maven-antrun-plugin.version} + + + unpack-shaded-jar + package + + run + + + + + + + + + + + + diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index a8cb560b7b376..22b093f7aafd7 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT managed-ledger @@ -47,6 +46,12 @@ org.apache.bookkeeper.stats codahale-metrics-provider ${bookkeeper.version} + + + amqp-client + com.rabbitmq + + @@ -66,6 +71,12 @@ ${project.version} + + ${project.groupId} + pulsar-opentelemetry + ${project.version} + + com.google.guava guava @@ -98,6 +109,10 @@ + + org.roaringbitmap + RoaringBitmap + io.dropwizard.metrics metrics-core @@ -115,6 +130,12 @@ test + + io.opentelemetry + opentelemetry-sdk-testing + test + + org.slf4j slf4j-api @@ -141,7 +162,7 @@ - + org.apache.maven.plugins maven-jar-plugin diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/AsyncCallbacks.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/AsyncCallbacks.java index dcf2c225e8b35..70db427afce4f 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/AsyncCallbacks.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/AsyncCallbacks.java @@ -24,7 +24,6 @@ import java.util.Optional; import org.apache.bookkeeper.common.annotation.InterfaceAudience; import org.apache.bookkeeper.common.annotation.InterfaceStability; -import org.apache.bookkeeper.mledger.impl.ReadOnlyManagedLedgerImpl; /** * Definition of all the callbacks used for the ManagedLedger asynchronous API. @@ -48,7 +47,7 @@ interface OpenReadOnlyCursorCallback { } interface OpenReadOnlyManagedLedgerCallback { - void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl managedLedger, Object ctx); + void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedger managedLedger, Object ctx); void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Object ctx); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloader.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloader.java index b60ae41670de7..11148ef1a59f5 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloader.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloader.java @@ -27,7 +27,7 @@ import org.apache.bookkeeper.common.annotation.InterfaceAudience; import org.apache.bookkeeper.common.annotation.InterfaceStability; import org.apache.bookkeeper.mledger.proto.MLDataFormats; -import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; +import org.apache.pulsar.common.policies.data.OffloadPolicies; /** * Interface for offloading ledgers to long-term storage. @@ -212,7 +212,7 @@ default CompletableFuture deleteOffloaded(UUID uid, Map of * * @return offload policies */ - OffloadPoliciesImpl getOffloadPolicies(); + OffloadPolicies getOffloadPolicies(); /** * Close the resources if necessary. @@ -230,5 +230,9 @@ default void scanLedgers(OffloadedLedgerMetadataConsumer consumer, Map offloadDriverMetadata) throws ManagedLedgerException { throw ManagedLedgerException.getManagedLedgerException(new UnsupportedOperationException()); } + + default boolean isAppendable() { + return true; + } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java index 7ecb8f08d573d..9fbf9b73c057e 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java @@ -31,7 +31,7 @@ */ @LimitedPrivate @Evolving -public interface LedgerOffloaderFactory { +public interface LedgerOffloaderFactory extends AutoCloseable { /** * Check whether the provided driver driverName is supported. @@ -111,4 +111,9 @@ default T create(OffloadPoliciesImpl offloadPolicies, throws IOException { return create(offloadPolicies, userMetadata, scheduler, offloaderStats); } + + @Override + default void close() throws Exception { + // no-op + } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java index 7802ed07781ba..042e03998696c 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java @@ -34,7 +34,7 @@ import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.SkipEntriesCallback; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats; /** * A ManagedCursor is a persisted cursor inside a ManagedLedger. @@ -46,6 +46,8 @@ @InterfaceStability.Stable public interface ManagedCursor { + String CURSOR_INTERNAL_PROPERTY_PREFIX = "#pulsar.internal."; + @SuppressWarnings("checkstyle:javadoctype") enum FindPositionConstraint { SearchActiveEntries, SearchAllAvailableEntries @@ -152,7 +154,7 @@ enum IndividualDeletedEntries { * max position can read */ void asyncReadEntries(int numberOfEntriesToRead, ReadEntriesCallback callback, Object ctx, - PositionImpl maxPosition); + Position maxPosition); /** @@ -165,7 +167,7 @@ void asyncReadEntries(int numberOfEntriesToRead, ReadEntriesCallback callback, O * @param maxPosition max position can read */ void asyncReadEntries(int numberOfEntriesToRead, long maxSizeBytes, ReadEntriesCallback callback, - Object ctx, PositionImpl maxPosition); + Object ctx, Position maxPosition); /** * Asynchronously read entries from the ManagedLedger. @@ -178,7 +180,7 @@ void asyncReadEntries(int numberOfEntriesToRead, long maxSizeBytes, ReadEntriesC * @param skipCondition predicate of read filter out */ default void asyncReadEntriesWithSkip(int numberOfEntriesToRead, long maxSizeBytes, ReadEntriesCallback callback, - Object ctx, PositionImpl maxPosition, Predicate skipCondition) { + Object ctx, Position maxPosition, Predicate skipCondition) { asyncReadEntries(numberOfEntriesToRead, maxSizeBytes, callback, ctx, maxPosition); } @@ -256,7 +258,7 @@ List readEntriesOrWait(int maxEntries, long maxSizeBytes) * max position can read */ void asyncReadEntriesOrWait(int numberOfEntriesToRead, ReadEntriesCallback callback, Object ctx, - PositionImpl maxPosition); + Position maxPosition); /** * Asynchronously read entries from the ManagedLedger, up to the specified number and size. @@ -277,7 +279,7 @@ void asyncReadEntriesOrWait(int numberOfEntriesToRead, ReadEntriesCallback callb * max position can read */ void asyncReadEntriesOrWait(int maxEntries, long maxSizeBytes, ReadEntriesCallback callback, Object ctx, - PositionImpl maxPosition); + Position maxPosition); /** * Asynchronously read entries from the ManagedLedger, up to the specified number and size. @@ -298,7 +300,7 @@ void asyncReadEntriesOrWait(int maxEntries, long maxSizeBytes, ReadEntriesCallba * predicate of read filter out */ default void asyncReadEntriesWithSkipOrWait(int maxEntries, ReadEntriesCallback callback, Object ctx, - PositionImpl maxPosition, Predicate skipCondition) { + Position maxPosition, Predicate skipCondition) { asyncReadEntriesOrWait(maxEntries, callback, ctx, maxPosition); } @@ -323,15 +325,15 @@ default void asyncReadEntriesWithSkipOrWait(int maxEntries, ReadEntriesCallback * predicate of read filter out */ default void asyncReadEntriesWithSkipOrWait(int maxEntries, long maxSizeBytes, ReadEntriesCallback callback, - Object ctx, PositionImpl maxPosition, - Predicate skipCondition) { + Object ctx, Position maxPosition, + Predicate skipCondition) { asyncReadEntriesOrWait(maxEntries, maxSizeBytes, callback, ctx, maxPosition); } /** * Cancel a previously scheduled asyncReadEntriesOrWait operation. * - * @see #asyncReadEntriesOrWait(int, ReadEntriesCallback, Object, PositionImpl) + * @see #asyncReadEntriesOrWait(int, ReadEntriesCallback, Object, Position) * @return true if the read operation was canceled or false if there was no pending operation */ boolean cancelPendingReadRequest(); @@ -517,6 +519,10 @@ void markDelete(Position position, Map properties) */ void rewind(); + default void rewind(boolean readCompacted) { + rewind(); + } + /** * Move the cursor to a different read position. * @@ -637,6 +643,23 @@ Position findNewestMatching(FindPositionConstraint constraint, Predicate void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate condition, FindEntryCallback callback, Object ctx); + /** + * Find the newest entry that matches the given predicate. + * + * @param constraint + * search only active entries or all entries + * @param condition + * predicate that reads an entry an applies a condition + * @param callback + * callback object returning the resultant position + * @param ctx + * opaque context + * @param isFindFromLedger + * find the newest entry from ledger + */ + void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate condition, + FindEntryCallback callback, Object ctx, boolean isFindFromLedger); + /** * reset the cursor to specified position to enable replay of messages. * @@ -786,6 +809,12 @@ Set asyncReplayEntries( */ long getEstimatedSizeSinceMarkDeletePosition(); + /** + * If a ledger is lost, this ledger will be skipped after enabled "autoSkipNonRecoverableData", and the method is + * used to delete information about this ledger in the ManagedCursor. + */ + default void skipNonRecoverableLedger(long ledgerId){} + /** * Returns cursor throttle mark-delete rate. * @@ -810,7 +839,7 @@ Set asyncReplayEntries( * Get last individual deleted range. * @return range */ - Range getLastIndividualDeletedRange(); + Range getLastIndividualDeletedRange(); /** * Trim delete entries for the given entries. @@ -820,7 +849,7 @@ Set asyncReplayEntries( /** * Get deleted batch indexes list for a batch message. */ - long[] getDeletedBatchIndexesAsLongArray(PositionImpl position); + long[] getDeletedBatchIndexesAsLongArray(Position position); /** * @return the managed cursor stats MBean @@ -839,4 +868,36 @@ Set asyncReplayEntries( * @return whether this cursor is closed. */ boolean isClosed(); + + default boolean isCursorDataFullyPersistable() { + return true; + } + + /** + * Called by the system to trigger periodic rollover in absence of activity. + */ + default boolean periodicRollover() { + return false; + } + + /** + * Get the attributes associated with the cursor. + * + * @return the attributes associated with the cursor + */ + default ManagedCursorAttributes getManagedCursorAttributes() { + return new ManagedCursorAttributes(this); + } + + ManagedLedgerInternalStats.CursorStats getCursorStats(); + + boolean isMessageDeleted(Position position); + + ManagedCursor duplicateNonDurableCursor(String nonDurableCursorName) throws ManagedLedgerException; + + long[] getBatchPositionAckSet(Position position); + + int applyMaxSizeCap(int maxEntries, long maxSizeBytes); + + void updateReadStats(int readEntriesCount, long readEntriesSize); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursorAttributes.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursorAttributes.java new file mode 100644 index 0000000000000..6c06e68d75e24 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursorAttributes.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger; + +import io.opentelemetry.api.common.Attributes; +import lombok.Getter; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.ManagedCursorOperationStatus; + +@Getter +public class ManagedCursorAttributes { + + private final Attributes attributes; + private final Attributes attributesOperationSucceed; + private final Attributes attributesOperationFailure; + + public ManagedCursorAttributes(ManagedCursor cursor) { + var mlName = cursor.getManagedLedger().getName(); + var topicName = TopicName.get(TopicName.fromPersistenceNamingEncoding(mlName)); + attributes = Attributes.of( + OpenTelemetryAttributes.ML_CURSOR_NAME, cursor.getName(), + OpenTelemetryAttributes.ML_LEDGER_NAME, mlName, + OpenTelemetryAttributes.PULSAR_NAMESPACE, topicName.getNamespace() + ); + attributesOperationSucceed = Attributes.builder() + .putAll(attributes) + .putAll(ManagedCursorOperationStatus.SUCCESS.attributes) + .build(); + attributesOperationFailure = Attributes.builder() + .putAll(attributes) + .putAll(ManagedCursorOperationStatus.FAILURE.attributes) + .build(); + } +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java index 4ca56508891a1..de69d97bb79aa 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java @@ -18,8 +18,10 @@ */ package org.apache.bookkeeper.mledger; +import com.google.common.collect.Range; import io.netty.buffer.ByteBuf; import java.util.Map; +import java.util.NavigableMap; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.function.Predicate; @@ -374,6 +376,8 @@ void asyncOpenCursor(String name, InitialPosition initialPosition, Map range); + /** * Get the total number of active entries for this managed ledger. * @@ -631,6 +635,17 @@ void asyncSetProperties(Map properties, AsyncCallbacks.UpdatePro */ void trimConsumedLedgersInBackground(CompletableFuture promise); + /** + * Rollover cursors in background if needed. + */ + default void rolloverCursorsInBackground() {} + + /** + * If a ledger is lost, this ledger will be skipped after enabled "autoSkipNonRecoverableData", and the method is + * used to delete information about this ledger in the ManagedCursor. + */ + default void skipNonRecoverableLedger(long ledgerId){} + /** * Roll current ledger if it is full. */ @@ -676,11 +691,46 @@ void asyncSetProperties(Map properties, AsyncCallbacks.UpdatePro /** * Check current inactive ledger (based on {@link ManagedLedgerConfig#getInactiveLedgerRollOverTimeMs()} and * roll over that ledger if inactive. + * + * @return true if ledger is considered for rolling over */ - void checkInactiveLedgerAndRollOver(); + boolean checkInactiveLedgerAndRollOver(); /** * Check if managed ledger should cache backlog reads. */ void checkCursorsToCacheEntries(); + + /** + * Get managed ledger attributes. + */ + default ManagedLedgerAttributes getManagedLedgerAttributes() { + return new ManagedLedgerAttributes(this); + } + + void asyncReadEntry(Position position, AsyncCallbacks.ReadEntryCallback callback, Object ctx); + + /** + * Get all the managed ledgers. + */ + NavigableMap getLedgersInfo(); + + Position getNextValidPosition(Position position); + + Position getPreviousPosition(Position position); + + long getEstimatedBacklogSize(Position position); + + Position getPositionAfterN(Position startPosition, long n, PositionBound startRange); + + int getPendingAddEntriesCount(); + + long getCacheSize(); + + default CompletableFuture getLastDispatchablePosition(final Predicate predicate, + final Position startPosition) { + return CompletableFuture.completedFuture(PositionFactory.EARLIEST); + } + + Position getFirstPosition(); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerAttributes.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerAttributes.java new file mode 100644 index 0000000000000..c3759a533a571 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerAttributes.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger; + +import io.opentelemetry.api.common.Attributes; +import lombok.Getter; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.ManagedLedgerOperationStatus; + +@Getter +public class ManagedLedgerAttributes { + + private final Attributes attributes; + private final Attributes attributesOperationSucceed; + private final Attributes attributesOperationFailure; + + public ManagedLedgerAttributes(ManagedLedger ml) { + var mlName = ml.getName(); + attributes = Attributes.of( + OpenTelemetryAttributes.ML_NAME, mlName, + OpenTelemetryAttributes.PULSAR_NAMESPACE, getNamespace(mlName) + ); + attributesOperationSucceed = Attributes.builder() + .putAll(attributes) + .putAll(ManagedLedgerOperationStatus.SUCCESS.attributes) + .build(); + attributesOperationFailure = Attributes.builder() + .putAll(attributes) + .putAll(ManagedLedgerOperationStatus.FAILURE.attributes) + .build(); + } + + private static String getNamespace(String mlName) { + try { + return TopicName.get(TopicName.fromPersistenceNamingEncoding(mlName)).getNamespace(); + } catch (RuntimeException e) { + return null; + } + } +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java index 0c93a5b642cf6..7b28990f35574 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java @@ -33,7 +33,7 @@ import org.apache.bookkeeper.mledger.impl.NullLedgerOffloader; import org.apache.bookkeeper.mledger.intercept.ManagedLedgerInterceptor; import org.apache.commons.collections4.MapUtils; -import org.apache.pulsar.common.util.collections.ConcurrentOpenLongPairRangeSet; +import org.apache.pulsar.common.util.collections.OpenLongPairRangeSet; /** * Configuration class for a ManagedLedger. @@ -64,6 +64,7 @@ public class ManagedLedgerConfig { private long retentionTimeMs = 0; private long retentionSizeInMB = 0; private boolean autoSkipNonRecoverableData; + private boolean ledgerForceRecovery; private boolean lazyCursorRecovery = false; private long metadataOperationsTimeoutSeconds = 60; private long readEntryTimeoutSeconds = 120; @@ -85,6 +86,7 @@ public class ManagedLedgerConfig { private int minimumBacklogCursorsForCaching = 0; private int minimumBacklogEntriesForCaching = 1000; private int maxBacklogBetweenCursorsForCaching = 1000; + private boolean triggerOffloadOnTopicLoad = false; @Getter @Setter @@ -281,7 +283,7 @@ public ManagedLedgerConfig setPassword(String password) { } /** - * should use {@link ConcurrentOpenLongPairRangeSet} to store unacked ranges. + * should use {@link OpenLongPairRangeSet} to store unacked ranges. * @return */ public boolean isUnackedRangesOpenCacheSetEnabled() { @@ -464,6 +466,17 @@ public void setAutoSkipNonRecoverableData(boolean skipNonRecoverableData) { this.autoSkipNonRecoverableData = skipNonRecoverableData; } + /** + * Skip managed ledger failure to recover managed ledger forcefully. + */ + public boolean isLedgerForceRecovery() { + return ledgerForceRecovery; + } + + public void setLedgerForceRecovery(boolean ledgerForceRecovery) { + this.ledgerForceRecovery = ledgerForceRecovery; + } + /** * @return max unacked message ranges that will be persisted and recovered. * @@ -504,8 +517,10 @@ public int getMaxUnackedRangesToPersistInMetadataStore() { return maxUnackedRangesToPersistInMetadataStore; } - public void setMaxUnackedRangesToPersistInMetadataStore(int maxUnackedRangesToPersistInMetadataStore) { + public ManagedLedgerConfig setMaxUnackedRangesToPersistInMetadataStore( + int maxUnackedRangesToPersistInMetadataStore) { this.maxUnackedRangesToPersistInMetadataStore = maxUnackedRangesToPersistInMetadataStore; + return this; } /** @@ -748,6 +763,22 @@ public void setMaxBacklogBetweenCursorsForCaching(int maxBacklogBetweenCursorsFo this.maxBacklogBetweenCursorsForCaching = maxBacklogBetweenCursorsForCaching; } + /** + * Trigger offload on topic load. + * @return + */ + public boolean isTriggerOffloadOnTopicLoad() { + return triggerOffloadOnTopicLoad; + } + + /** + * Set trigger offload on topic load. + * @param triggerOffloadOnTopicLoad + */ + public void setTriggerOffloadOnTopicLoad(boolean triggerOffloadOnTopicLoad) { + this.triggerOffloadOnTopicLoad = triggerOffloadOnTopicLoad; + } + public String getShadowSource() { return MapUtils.getString(properties, PROPERTY_SOURCE_TOPIC_KEY); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java index b1427bab80b22..d9c887fac468e 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java @@ -28,6 +28,8 @@ import org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.OpenReadOnlyCursorCallback; import org.apache.bookkeeper.mledger.impl.cache.EntryCacheManager; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats; /** * A factory to open/create managed ledgers and delete them. @@ -90,7 +92,7 @@ ManagedLedger open(String name, ManagedLedgerConfig config) * opaque context */ void asyncOpen(String name, ManagedLedgerConfig config, OpenLedgerCallback callback, - Supplier mlOwnershipChecker, Object ctx); + Supplier> mlOwnershipChecker, Object ctx); /** * Open a {@link ReadOnlyCursor} positioned to the earliest entry for the specified managed ledger. @@ -233,4 +235,14 @@ void asyncDelete(String name, CompletableFuture mlConfigFut * @return properties of this managedLedger. */ CompletableFuture> getManagedLedgerPropertiesAsync(String name); + + Map getManagedLedgers(); + + ManagedLedgerFactoryMXBean getCacheStats(); + + + void estimateUnloadedTopicBacklog(PersistentOfflineTopicStats offlineTopicStats, TopicName topicName, + boolean accurate, Object ctx) throws Exception; + + ManagedLedgerFactoryConfig getConfig(); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java index 5aa4e8374d73a..386310b3ccbae 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java @@ -39,7 +39,6 @@ public class ManagedLedgerFactoryConfig { */ private double cacheEvictionWatermark = 0.90; - private int numManagedLedgerWorkerThreads = Runtime.getRuntime().availableProcessors(); private int numManagedLedgerSchedulerThreads = Runtime.getRuntime().availableProcessors(); /** @@ -92,8 +91,30 @@ public class ManagedLedgerFactoryConfig { */ private String managedLedgerInfoCompressionType = MLDataFormats.CompressionType.NONE.name(); + /** + * ManagedLedgerInfo compression threshold. If the origin metadata size below configuration. + * compression will not apply. + */ + private long managedLedgerInfoCompressionThresholdInBytes = 0; + /** * ManagedCursorInfo compression type. If the compression type is null or invalid, don't compress data. */ private String managedCursorInfoCompressionType = MLDataFormats.CompressionType.NONE.name(); + + /** + * ManagedCursorInfo compression threshold. If the origin metadata size below configuration. + * compression will not apply. + */ + private long managedCursorInfoCompressionThresholdInBytes = 0; + + public MetadataCompressionConfig getCompressionConfigForManagedLedgerInfo() { + return new MetadataCompressionConfig(managedLedgerInfoCompressionType, + managedLedgerInfoCompressionThresholdInBytes); + } + + public MetadataCompressionConfig getCompressionConfigForManagedCursorInfo() { + return new MetadataCompressionConfig(managedCursorInfoCompressionType, + managedCursorInfoCompressionThresholdInBytes); + } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryMXBean.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryMXBean.java index 35c26c5dfdb89..43e8196daa9ae 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryMXBean.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryMXBean.java @@ -47,26 +47,51 @@ public interface ManagedLedgerFactoryMXBean { */ double getCacheHitsRate(); + /** + * Cumulative number of cache hits. + */ + long getCacheHitsTotal(); + /** * Get the number of cache misses per second. */ double getCacheMissesRate(); + /** + * Cumulative number of cache misses. + */ + long getCacheMissesTotal(); + /** * Get the amount of data is retrieved from the cache in byte/s. */ double getCacheHitsThroughput(); + /** + * Cumulative amount of data retrieved from the cache in bytes. + */ + long getCacheHitsBytesTotal(); + /** * Get the amount of data is retrieved from the bookkeeper in byte/s. */ double getCacheMissesThroughput(); + /** + * Cumulative amount of data retrieved from the bookkeeper in bytes. + */ + long getCacheMissesBytesTotal(); + /** * Get the number of cache evictions during the last minute. */ long getNumberOfCacheEvictions(); + /** + * Cumulative number of cache evictions. + */ + long getNumberOfCacheEvictionsTotal(); + /** * Cumulative number of entries inserted into the cache. */ diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerMXBean.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerMXBean.java index 50a3ffb157961..1d978e2378569 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerMXBean.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerMXBean.java @@ -60,11 +60,21 @@ public interface ManagedLedgerMXBean { */ double getAddEntryBytesRate(); + /** + * @return the total number of bytes written + */ + long getAddEntryBytesTotal(); + /** * @return the bytes/s rate of messages added with replicas */ double getAddEntryWithReplicasBytesRate(); + /** + * @return the total number of bytes written, including replicas + */ + long getAddEntryWithReplicasBytesTotal(); + /** * @return the msg/s rate of messages read */ @@ -75,36 +85,76 @@ public interface ManagedLedgerMXBean { */ double getReadEntriesBytesRate(); + /** + * @return the total number of bytes read + */ + long getReadEntriesBytesTotal(); + /** * @return the rate of mark-delete ops/s */ double getMarkDeleteRate(); + /** + * @return the number of mark-delete ops + */ + long getMarkDeleteTotal(); + /** * @return the number of addEntry requests that succeeded */ long getAddEntrySucceed(); + /** + * @return the total number of addEntry requests that succeeded + */ + long getAddEntrySucceedTotal(); + /** * @return the number of addEntry requests that failed */ long getAddEntryErrors(); + /** + * @return the total number of addEntry requests that failed + */ + long getAddEntryErrorsTotal(); + + /** + * @return the number of entries read from the managed ledger (from cache or BK) + */ + long getEntriesReadTotalCount(); + /** * @return the number of readEntries requests that succeeded */ long getReadEntriesSucceeded(); + /** + * @return the total number of readEntries requests that succeeded + */ + long getReadEntriesSucceededTotal(); + /** * @return the number of readEntries requests that failed */ long getReadEntriesErrors(); + /** + * @return the total number of readEntries requests that failed + */ + long getReadEntriesErrorsTotal(); + /** * @return the number of readEntries requests that cache miss Rate */ double getReadEntriesOpsCacheMissesRate(); + /** + * @return the total number of readEntries requests that cache miss + */ + long getReadEntriesOpsCacheMissesTotal(); + // Entry size statistics double getEntrySizeAverage(); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/MetadataCompressionConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/MetadataCompressionConfig.java new file mode 100644 index 0000000000000..601c270ab7680 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/MetadataCompressionConfig.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.ToString; +import org.apache.bookkeeper.mledger.proto.MLDataFormats; +import org.apache.commons.lang.StringUtils; + +@Data +@AllArgsConstructor +@ToString +public class MetadataCompressionConfig { + MLDataFormats.CompressionType compressionType; + long compressSizeThresholdInBytes; + + public MetadataCompressionConfig(String compressionType) throws IllegalArgumentException { + this(compressionType, 0); + } + + public MetadataCompressionConfig(String compressionType, long compressThreshold) throws IllegalArgumentException { + this.compressionType = parseCompressionType(compressionType); + this.compressSizeThresholdInBytes = compressThreshold; + } + + public static MetadataCompressionConfig noCompression = + new MetadataCompressionConfig(MLDataFormats.CompressionType.NONE, 0); + + private MLDataFormats.CompressionType parseCompressionType(String value) throws IllegalArgumentException { + if (StringUtils.isEmpty(value)) { + return MLDataFormats.CompressionType.NONE; + } + + MLDataFormats.CompressionType compressionType; + compressionType = MLDataFormats.CompressionType.valueOf(value); + + return compressionType; + } +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/OpenTelemetryManagedLedgerCacheStats.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/OpenTelemetryManagedLedgerCacheStats.java new file mode 100644 index 0000000000000..13e7ed6ac6799 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/OpenTelemetryManagedLedgerCacheStats.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.metrics.BatchCallback; +import io.opentelemetry.api.metrics.ObservableLongMeasurement; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; +import org.apache.bookkeeper.mledger.impl.cache.PooledByteBufAllocatorStats; +import org.apache.bookkeeper.mledger.impl.cache.RangeEntryCacheImpl; +import org.apache.pulsar.opentelemetry.Constants; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.CacheEntryStatus; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.CacheOperationStatus; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.PoolArenaType; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.PoolChunkAllocationType; + +public class OpenTelemetryManagedLedgerCacheStats implements AutoCloseable { + + // Replaces pulsar_ml_count + public static final String MANAGED_LEDGER_COUNTER = "pulsar.broker.managed_ledger.count"; + private final ObservableLongMeasurement managedLedgerCounter; + + // Replaces pulsar_ml_cache_evictions + public static final String CACHE_EVICTION_OPERATION_COUNTER = "pulsar.broker.managed_ledger.cache.eviction.count"; + private final ObservableLongMeasurement cacheEvictionOperationCounter; + + // Replaces 'pulsar_ml_cache_entries', + // 'pulsar_ml_cache_inserted_entries_total', + // 'pulsar_ml_cache_evicted_entries_total' + public static final String CACHE_ENTRY_COUNTER = "pulsar.broker.managed_ledger.cache.entry.count"; + private final ObservableLongMeasurement cacheEntryCounter; + + // Replaces pulsar_ml_cache_used_size + public static final String CACHE_SIZE_COUNTER = "pulsar.broker.managed_ledger.cache.entry.size"; + private final ObservableLongMeasurement cacheSizeCounter; + + // Replaces pulsar_ml_cache_hits_rate, pulsar_ml_cache_misses_rate + public static final String CACHE_OPERATION_COUNTER = "pulsar.broker.managed_ledger.cache.operation.count"; + private final ObservableLongMeasurement cacheOperationCounter; + + // Replaces pulsar_ml_cache_hits_throughput, pulsar_ml_cache_misses_throughput + public static final String CACHE_OPERATION_BYTES_COUNTER = "pulsar.broker.managed_ledger.cache.operation.size"; + private final ObservableLongMeasurement cacheOperationBytesCounter; + + // Replaces 'pulsar_ml_cache_pool_active_allocations', + // 'pulsar_ml_cache_pool_active_allocations_huge', + // 'pulsar_ml_cache_pool_active_allocations_normal', + // 'pulsar_ml_cache_pool_active_allocations_small' + public static final String CACHE_POOL_ACTIVE_ALLOCATION_COUNTER = + "pulsar.broker.managed_ledger.cache.pool.allocation.active.count"; + private final ObservableLongMeasurement cachePoolActiveAllocationCounter; + + // Replaces ['pulsar_ml_cache_pool_allocated', 'pulsar_ml_cache_pool_used'] + public static final String CACHE_POOL_ACTIVE_ALLOCATION_SIZE_COUNTER = + "pulsar.broker.managed_ledger.cache.pool.allocation.size"; + private final ObservableLongMeasurement cachePoolActiveAllocationSizeCounter; + + private final BatchCallback batchCallback; + + public OpenTelemetryManagedLedgerCacheStats(OpenTelemetry openTelemetry, ManagedLedgerFactoryImpl factory) { + var meter = openTelemetry.getMeter(Constants.BROKER_INSTRUMENTATION_SCOPE_NAME); + + managedLedgerCounter = meter + .upDownCounterBuilder(MANAGED_LEDGER_COUNTER) + .setUnit("{managed_ledger}") + .setDescription("The total number of managed ledgers.") + .buildObserver(); + + cacheEvictionOperationCounter = meter + .counterBuilder(CACHE_EVICTION_OPERATION_COUNTER) + .setUnit("{eviction}") + .setDescription("The total number of cache eviction operations.") + .buildObserver(); + + cacheEntryCounter = meter + .upDownCounterBuilder(CACHE_ENTRY_COUNTER) + .setUnit("{entry}") + .setDescription("The number of entries in the entry cache.") + .buildObserver(); + + cacheSizeCounter = meter + .upDownCounterBuilder(CACHE_SIZE_COUNTER) + .setUnit("{By}") + .setDescription("The byte amount of entries stored in the entry cache.") + .buildObserver(); + + cacheOperationCounter = meter + .counterBuilder(CACHE_OPERATION_COUNTER) + .setUnit("{entry}") + .setDescription("The number of cache operations.") + .buildObserver(); + + cacheOperationBytesCounter = meter + .counterBuilder(CACHE_OPERATION_BYTES_COUNTER) + .setUnit("{By}") + .setDescription("The byte amount of data retrieved from cache operations.") + .buildObserver(); + + cachePoolActiveAllocationCounter = meter + .upDownCounterBuilder(CACHE_POOL_ACTIVE_ALLOCATION_COUNTER) + .setUnit("{allocation}") + .setDescription("The number of currently active allocations in the direct arena.") + .buildObserver(); + + cachePoolActiveAllocationSizeCounter = meter + .upDownCounterBuilder(CACHE_POOL_ACTIVE_ALLOCATION_SIZE_COUNTER) + .setUnit("{By}") + .setDescription("The memory allocated in the direct arena.") + .buildObserver(); + + + batchCallback = meter.batchCallback(() -> recordMetrics(factory), + managedLedgerCounter, + cacheEvictionOperationCounter, + cacheEntryCounter, + cacheSizeCounter, + cacheOperationCounter, + cacheOperationBytesCounter, + cachePoolActiveAllocationCounter, + cachePoolActiveAllocationSizeCounter); + } + + @Override + public void close() { + batchCallback.close(); + } + + private void recordMetrics(ManagedLedgerFactoryImpl factory) { + var stats = factory.getCacheStats(); + + managedLedgerCounter.record(stats.getNumberOfManagedLedgers()); + cacheEvictionOperationCounter.record(stats.getNumberOfCacheEvictionsTotal()); + + var entriesOut = stats.getCacheEvictedEntriesCount(); + var entriesIn = stats.getCacheInsertedEntriesCount(); + var entriesActive = entriesIn - entriesOut; + cacheEntryCounter.record(entriesActive, CacheEntryStatus.ACTIVE.attributes); + cacheEntryCounter.record(entriesIn, CacheEntryStatus.INSERTED.attributes); + cacheEntryCounter.record(entriesOut, CacheEntryStatus.EVICTED.attributes); + cacheSizeCounter.record(stats.getCacheUsedSize()); + + cacheOperationCounter.record(stats.getCacheHitsTotal(), CacheOperationStatus.HIT.attributes); + cacheOperationBytesCounter.record(stats.getCacheHitsBytesTotal(), CacheOperationStatus.HIT.attributes); + cacheOperationCounter.record(stats.getCacheMissesTotal(), CacheOperationStatus.MISS.attributes); + cacheOperationBytesCounter.record(stats.getCacheMissesBytesTotal(), CacheOperationStatus.MISS.attributes); + + var allocatorStats = new PooledByteBufAllocatorStats(RangeEntryCacheImpl.ALLOCATOR); + cachePoolActiveAllocationCounter.record(allocatorStats.activeAllocationsSmall, PoolArenaType.SMALL.attributes); + cachePoolActiveAllocationCounter.record(allocatorStats.activeAllocationsNormal, + PoolArenaType.NORMAL.attributes); + cachePoolActiveAllocationCounter.record(allocatorStats.activeAllocationsHuge, PoolArenaType.HUGE.attributes); + cachePoolActiveAllocationSizeCounter.record(allocatorStats.totalAllocated, + PoolChunkAllocationType.ALLOCATED.attributes); + cachePoolActiveAllocationSizeCounter.record(allocatorStats.totalUsed, PoolChunkAllocationType.USED.attributes); + } +} \ No newline at end of file diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/Position.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/Position.java index ac5810bbf01e7..d0d6d865c9558 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/Position.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/Position.java @@ -18,6 +18,7 @@ */ package org.apache.bookkeeper.mledger; +import java.util.Optional; import org.apache.bookkeeper.common.annotation.InterfaceAudience; import org.apache.bookkeeper.common.annotation.InterfaceStability; @@ -26,16 +27,108 @@ */ @InterfaceAudience.LimitedPrivate @InterfaceStability.Stable -public interface Position { +public interface Position extends Comparable { + /** + * Get the ledger id of the entry pointed by this position. + * + * @return the ledger id + */ + long getLedgerId(); + + /** + * Get the entry id of the entry pointed by this position. + * + * @return the entry id + */ + long getEntryId(); + + /** + * Compare this position with another position. + * The comparison is first based on the ledger id, and then on the entry id. + * This is implements the Comparable interface. + * @param that the other position to be compared. + * @return -1 if this position is less than the other, 0 if they are equal, 1 if this position is greater than + * the other. + */ + default int compareTo(Position that) { + if (getLedgerId() != that.getLedgerId()) { + return Long.compare(getLedgerId(), that.getLedgerId()); + } + + return Long.compare(getEntryId(), that.getEntryId()); + } + + /** + * Compare this position with another position based on the ledger id and entry id. + * @param ledgerId the ledger id to compare + * @param entryId the entry id to compare + * @return -1 if this position is less than the other, 0 if they are equal, 1 if this position is greater than + * the other. + */ + default int compareTo(long ledgerId, long entryId) { + if (getLedgerId() != ledgerId) { + return Long.compare(getLedgerId(), ledgerId); + } + + return Long.compare(getEntryId(), entryId); + } + + /** + * Calculate the hash code for the position based on ledgerId and entryId. + * This is used in Position implementations to implement the hashCode method. + * @return hash code + */ + default int hashCodeForPosition() { + int result = Long.hashCode(getLedgerId()); + result = 31 * result + Long.hashCode(getEntryId()); + return result; + } + /** * Get the position of the entry next to this one. The returned position might point to a non-existing, or not-yet * existing entry * * @return the position of the next logical entry */ - Position getNext(); + default Position getNext() { + if (getEntryId() < 0) { + return PositionFactory.create(getLedgerId(), 0); + } else { + return PositionFactory.create(getLedgerId(), getEntryId() + 1); + } + } - long getLedgerId(); + /** + * Position after moving entryNum messages, + * if entryNum < 1, then return the current position. + * */ + default Position getPositionAfterEntries(int entryNum) { + if (entryNum < 1) { + return this; + } + if (getEntryId() < 0) { + return PositionFactory.create(getLedgerId(), entryNum - 1); + } else { + return PositionFactory.create(getLedgerId(), getEntryId() + entryNum); + } + } - long getEntryId(); + /** + * Check if the position implementation has an extension of the given class or interface. + * + * @param extensionClass the class of the extension + * @return true if the position has an extension of the given class, false otherwise + */ + default boolean hasExtension(Class extensionClass) { + return getExtension(extensionClass).isPresent(); + } + + /** + * Get the extension instance of the given class or interface that is attached to this position. + * If the position does not have an extension of the given class, an empty optional is returned. + * @param extensionClass the class of the extension + */ + default Optional getExtension(Class extensionClass) { + return Optional.empty(); + } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/RateLimitFunction.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/PositionBound.java similarity index 83% rename from pulsar-common/src/main/java/org/apache/pulsar/common/util/RateLimitFunction.java rename to managed-ledger/src/main/java/org/apache/bookkeeper/mledger/PositionBound.java index 0a4b62e9500f9..a7ab4a48a9b02 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/RateLimitFunction.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/PositionBound.java @@ -16,11 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.common.util; +package org.apache.bookkeeper.mledger; -/** - * Function use when rate limiter renew permit. - * */ -public interface RateLimitFunction { - void apply(); +public enum PositionBound { + // define boundaries for position based seeks and searches + startIncluded, startExcluded } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/PositionFactory.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/PositionFactory.java new file mode 100644 index 0000000000000..0b119844a6268 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/PositionFactory.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger; + +import org.apache.bookkeeper.mledger.impl.ImmutablePositionImpl; + +/** + * Factory for creating {@link Position} instances. + */ +public final class PositionFactory { + /** + * Earliest position. + */ + public static final Position EARLIEST = create(-1, -1); + /** + * Latest position. + */ + public static final Position LATEST = create(Long.MAX_VALUE, Long.MAX_VALUE); + + private PositionFactory() { + } + + /** + * Create a new position. + * + * @param ledgerId ledger id + * @param entryId entry id + * @return new position + */ + public static Position create(long ledgerId, long entryId) { + return new ImmutablePositionImpl(ledgerId, entryId); + } + + /** + * Create a new position. + * + * @param other other position + * @return new position + */ + public static Position create(Position other) { + return new ImmutablePositionImpl(other); + } +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyCursor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyCursor.java index 18d412f893152..016298cb108bb 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyCursor.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyCursor.java @@ -24,7 +24,6 @@ import org.apache.bookkeeper.common.annotation.InterfaceAudience; import org.apache.bookkeeper.common.annotation.InterfaceStability; import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback; -import org.apache.bookkeeper.mledger.impl.PositionImpl; @InterfaceAudience.LimitedPrivate @InterfaceStability.Stable @@ -48,7 +47,7 @@ public interface ReadOnlyCursor { * @see #readEntries(int) */ void asyncReadEntries(int numberOfEntriesToRead, ReadEntriesCallback callback, - Object ctx, PositionImpl maxPosition); + Object ctx, Position maxPosition); /** * Asynchronously read entries from the ManagedLedger. @@ -60,7 +59,7 @@ void asyncReadEntries(int numberOfEntriesToRead, ReadEntriesCallback callback, * @param maxPosition max position can read */ void asyncReadEntries(int numberOfEntriesToRead, long maxSizeBytes, ReadEntriesCallback callback, - Object ctx, PositionImpl maxPosition); + Object ctx, Position maxPosition); /** * Get the read position. This points to the next message to be read from the cursor. @@ -116,7 +115,7 @@ Position findNewestMatching(ManagedCursor.FindPositionConstraint constraint, Pre * @param range the range between two positions * @return the number of entries in range */ - long getNumberOfEntries(Range range); + long getNumberOfEntries(Range range); /** * Close the cursor and releases the associated resources. diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyManagedLedger.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyManagedLedger.java new file mode 100644 index 0000000000000..91b8f92eb637e --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyManagedLedger.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger; + +import java.util.Map; + +public interface ReadOnlyManagedLedger { + + void asyncReadEntry(Position position, AsyncCallbacks.ReadEntryCallback callback, Object ctx); + + long getNumberOfEntries(); + + ReadOnlyCursor createReadOnlyCursor(Position position); + + Map getProperties(); + +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyManagedLedgerImplWrapper.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyManagedLedgerImplWrapper.java new file mode 100644 index 0000000000000..5bc94c04beefd --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyManagedLedgerImplWrapper.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.common.util.OrderedScheduler; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; +import org.apache.bookkeeper.mledger.impl.MetaStore; +import org.apache.bookkeeper.mledger.impl.ReadOnlyManagedLedgerImpl; + +public class ReadOnlyManagedLedgerImplWrapper implements ReadOnlyManagedLedger { + + private final ReadOnlyManagedLedgerImpl readOnlyManagedLedger; + + public ReadOnlyManagedLedgerImplWrapper(ManagedLedgerFactoryImpl factory, BookKeeper bookKeeper, MetaStore store, + ManagedLedgerConfig config, OrderedScheduler scheduledExecutor, + String name) { + this.readOnlyManagedLedger = + new ReadOnlyManagedLedgerImpl(factory, bookKeeper, store, config, scheduledExecutor, name); + } + + public CompletableFuture initialize() { + return readOnlyManagedLedger.initialize(); + } + + @Override + public void asyncReadEntry(Position position, AsyncCallbacks.ReadEntryCallback callback, Object ctx) { + readOnlyManagedLedger.asyncReadEntry(position, callback, ctx); + } + + @Override + public long getNumberOfEntries() { + return readOnlyManagedLedger.getNumberOfEntries(); + } + + @Override + public ReadOnlyCursor createReadOnlyCursor(Position position) { + return readOnlyManagedLedger.createReadOnlyCursor(position); + } + + @Override + public Map getProperties() { + return readOnlyManagedLedger.getProperties(); + } +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AckSetPositionImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AckSetPositionImpl.java new file mode 100644 index 0000000000000..22a99eb3607eb --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AckSetPositionImpl.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import java.util.Optional; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; + +/** + * Position implementation that includes the ack set. + * Use {@link AckSetStateUtil#createPositionWithAckSet(long, long, long[])} to create instances. + */ +public class AckSetPositionImpl implements Position, AckSetState { + private final Optional ackSetStateExtension = Optional.of(this); + protected final long ledgerId; + protected final long entryId; + protected volatile long[] ackSet; + + public AckSetPositionImpl(long ledgerId, long entryId, long[] ackSet) { + this.ledgerId = ledgerId; + this.entryId = entryId; + this.ackSet = ackSet; + } + + public long[] getAckSet() { + return ackSet; + } + + public void setAckSet(long[] ackSet) { + this.ackSet = ackSet; + } + + public long getLedgerId() { + return ledgerId; + } + + public long getEntryId() { + return entryId; + } + + @Override + public Position getNext() { + if (entryId < 0) { + return PositionFactory.create(ledgerId, 0); + } else { + return PositionFactory.create(ledgerId, entryId + 1); + } + } + + @Override + public String toString() { + return ledgerId + ":" + entryId + " (ackSet " + (ackSet == null ? "is null" : + "with long[] size of " + ackSet.length) + ")"; + } + + @Override + public int hashCode() { + return hashCodeForPosition(); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof Position && compareTo((Position) obj) == 0; + } + + @Override + public Optional getExtension(Class extensionClass) { + if (extensionClass == AckSetState.class) { + return (Optional) ackSetStateExtension; + } + return Position.super.getExtension(extensionClass); + } +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AckSetState.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AckSetState.java new file mode 100644 index 0000000000000..363336e83113e --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AckSetState.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +/** + * Interface to manage the ackSet state attached to a position. + * Helpers in {@link AckSetStateUtil} to create positions with + * ackSet state and to extract the state. + */ +public interface AckSetState { + /** + * Get the ackSet bitset information encoded as a long array. + * @return the ackSet + */ + long[] getAckSet(); + + /** + * Set the ackSet bitset information as a long array. + * @param ackSet the ackSet + */ + void setAckSet(long[] ackSet); + + /** + * Check if the ackSet is set. + * @return true if the ackSet is set, false otherwise + */ + default boolean hasAckSet() { + return getAckSet() != null; + } +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AckSetStateUtil.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AckSetStateUtil.java new file mode 100644 index 0000000000000..11ab520b68e92 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AckSetStateUtil.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import java.util.Optional; +import lombok.experimental.UtilityClass; +import org.apache.bookkeeper.mledger.Position; + +/** + * Utility class to manage the ackSet state attached to a position. + */ +@UtilityClass +public class AckSetStateUtil { + /** + * Create a new position with the ackSet state. + * + * @param ledgerId ledger id + * @param entryId entry id + * @param ackSet ack set bitset information encoded as an array of longs + * @return new position + */ + public static Position createPositionWithAckSet(long ledgerId, long entryId, long[] ackSet) { + return new AckSetPositionImpl(ledgerId, entryId, ackSet); + } + + /** + * Get the AckSetState instance from the position if it exists. + * @param position position which possibly contains the AckSetState + */ + public static Optional maybeGetAckSetState(Position position) { + return position.getExtension(AckSetState.class); + } + + /** + * Get the ackSet bitset information encoded as a long array from the position if it exists. + * @param position position which possibly contains the AckSetState + * @return the ackSet or null if the position does not have the AckSetState, or it's not set + */ + public static long[] getAckSetArrayOrNull(Position position) { + return maybeGetAckSetState(position).map(AckSetState::getAckSet).orElse(null); + } + + /** + * Get the AckSetState instance from the position. + * @param position position which contains the AckSetState + * @return AckSetState instance + * @throws IllegalStateException if the position does not have AckSetState + */ + public static AckSetState getAckSetState(Position position) { + return maybeGetAckSetState(position) + .orElseThrow(() -> + new IllegalStateException("Position does not have AckSetState. position=" + position)); + } + + /** + * Check if position contains the ackSet information and it is set. + * @param position position which possibly contains the AckSetState + * @return true if the ackSet is set, false otherwise + */ + public static boolean hasAckSet(Position position) { + return maybeGetAckSetState(position).map(AckSetState::hasAckSet).orElse(false); + } +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryImpl.java index 6512399173f0a..e0e2b859794b5 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryImpl.java @@ -26,10 +26,13 @@ import io.netty.util.ReferenceCounted; import org.apache.bookkeeper.client.api.LedgerEntry; import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.util.AbstractCASReferenceCounted; +import org.apache.bookkeeper.mledger.util.RangeCache; public final class EntryImpl extends AbstractCASReferenceCounted implements Entry, Comparable, - ReferenceCounted { + RangeCache.ValueWithKeyValidation { private static final Recycler RECYCLER = new Recycler() { @Override @@ -42,6 +45,7 @@ protected EntryImpl newObject(Handle handle) { private long timestamp; private long ledgerId; private long entryId; + private Position position; ByteBuf data; private Runnable onDeallocate; @@ -79,7 +83,7 @@ public static EntryImpl create(long ledgerId, long entryId, ByteBuf data) { return entry; } - public static EntryImpl create(PositionImpl position, ByteBuf data) { + public static EntryImpl create(Position position, ByteBuf data) { EntryImpl entry = RECYCLER.get(); entry.timestamp = System.nanoTime(); entry.ledgerId = position.getLedgerId(); @@ -150,8 +154,11 @@ public int getLength() { } @Override - public PositionImpl getPosition() { - return new PositionImpl(ledgerId, entryId); + public Position getPosition() { + if (position == null) { + position = PositionFactory.create(ledgerId, entryId); + } + return position; } @Override @@ -197,7 +204,12 @@ protected void deallocate() { timestamp = -1; ledgerId = -1; entryId = -1; + position = null; recyclerHandle.recycle(this); } + @Override + public boolean matchesKey(Position key) { + return key.compareTo(ledgerId, entryId) == 0; + } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionImplRecyclable.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ImmutablePositionImpl.java similarity index 50% rename from managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionImplRecyclable.java rename to managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ImmutablePositionImpl.java index eb2b33e858d63..06245a6b5f33a 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionImplRecyclable.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ImmutablePositionImpl.java @@ -18,33 +18,45 @@ */ package org.apache.bookkeeper.mledger.impl; -import io.netty.util.Recycler; -import io.netty.util.Recycler.Handle; import org.apache.bookkeeper.mledger.Position; -public class PositionImplRecyclable extends PositionImpl implements Position { +public final class ImmutablePositionImpl implements Position { + private final long ledgerId; + private final long entryId; - private final Handle recyclerHandle; + public ImmutablePositionImpl(long ledgerId, long entryId) { + this.ledgerId = ledgerId; + this.entryId = entryId; + } - private static final Recycler RECYCLER = new Recycler() { - @Override - protected PositionImplRecyclable newObject(Recycler.Handle recyclerHandle) { - return new PositionImplRecyclable(recyclerHandle); - } - }; + public ImmutablePositionImpl(Position other) { + this.ledgerId = other.getLedgerId(); + this.entryId = other.getEntryId(); + } - private PositionImplRecyclable(Handle recyclerHandle) { - super(PositionImpl.EARLIEST); - this.recyclerHandle = recyclerHandle; + public long getLedgerId() { + return ledgerId; } - public static PositionImplRecyclable create() { - return RECYCLER.get(); + public long getEntryId() { + return entryId; } - public void recycle() { - ackSet = null; - recyclerHandle.recycle(this); + /** + * String representation of virtual cursor - LedgerId:EntryId. + */ + @Override + public String toString() { + return ledgerId + ":" + entryId; } -} \ No newline at end of file + @Override + public int hashCode() { + return hashCodeForPosition(); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof Position && compareTo((Position) obj) == 0; + } +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainer.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainer.java index 58c83961d619f..ba901ece51c39 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainer.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainer.java @@ -25,41 +25,118 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.locks.StampedLock; +import lombok.Value; +import lombok.experimental.UtilityClass; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.Position; import org.apache.commons.lang3.tuple.Pair; /** * Contains cursors for a ManagedLedger. - * - *

The goal is to always know the slowest consumer and hence decide which is the oldest ledger we need to keep. - * - *

This data structure maintains a heap and a map of cursors. The map is used to relate a cursor name with + *

+ * The goal is to always know the slowest consumer and hence decide which is the oldest ledger we need to keep. + *

+ * This data structure maintains a heap and a map of cursors. The map is used to relate a cursor name with * an entry index in the heap. The heap data structure sorts cursors in a binary tree which is represented * in a single array. More details about heap implementations: - * https://en.wikipedia.org/wiki/Heap_(data_structure)#Implementation - * - *

The heap is updated and kept sorted when a cursor is updated. + * here + *

+ * The heap is updated and kept sorted when a cursor is updated. * */ public class ManagedCursorContainer implements Iterable { + /** + * This field is incremented everytime the cursor information is updated. + */ + private long version; + + @Value + public static class CursorInfo { + ManagedCursor cursor; + Position position; + + /** + * Cursor info's version. + *

+ * Use {@link DataVersion#compareVersions(long, long)} to compare between two versions, + * since it rolls over to 0 once reaching Long.MAX_VALUE + */ + long version; + } + private static class Item { final ManagedCursor cursor; - PositionImpl position; + Position position; int idx; - Item(ManagedCursor cursor, PositionImpl position, int idx) { + Item(ManagedCursor cursor, Position position, int idx) { this.cursor = cursor; this.position = position; this.idx = idx; } } - public ManagedCursorContainer() { + /** + * Utility class to manage a data version, which rolls over to 0 when reaching Long.MAX_VALUE. + */ + @UtilityClass + public class DataVersion { + + /** + * Compares two data versions, which either rolls overs to 0 when reaching Long.MAX_VALUE. + *

+ * Use {@link DataVersion#getNextVersion(long)} to increment the versions. The assumptions + * are that metric versions are compared with close time proximity one to another, hence, + * they are expected not close to each other in terms of distance, hence we don't + * expect the distance ever to exceed Long.MAX_VALUE / 2, otherwise we wouldn't be able + * to know which one is a later version in case the furthest rolls over to beyond 0. We + * assume the shortest distance between them dictates that. + *

+ * @param v1 First version to compare + * @param v2 Second version to compare + * @return the value {@code 0} if {@code v1 == v2}; + * a value less than {@code 0} if {@code v1 < v2}; and + * a value greater than {@code 0} if {@code v1 > v2} + */ + public static int compareVersions(long v1, long v2) { + if (v1 == v2) { + return 0; + } + + // 0-------v1--------v2--------MAX_LONG + if (v2 > v1) { + long distance = v2 - v1; + long wrapAroundDistance = (Long.MAX_VALUE - v2) + v1; + if (distance < wrapAroundDistance) { + return -1; + } else { + return 1; + } + // 0-------v2--------v1--------MAX_LONG + } else { + long distance = v1 - v2; + long wrapAroundDistance = (Long.MAX_VALUE - v1) + v2; + if (distance < wrapAroundDistance) { + return 1; // v1 is bigger + } else { + return -1; // v2 is bigger + } + } + } + + public static long getNextVersion(long existingVersion) { + if (existingVersion == Long.MAX_VALUE) { + return 0; + } else { + return existingVersion + 1; + } + } } + public ManagedCursorContainer() {} + // Used to keep track of slowest cursor. private final ArrayList heap = new ArrayList<>(); @@ -83,7 +160,7 @@ public ManagedCursorContainer() { public void add(ManagedCursor cursor, Position position) { long stamp = rwLock.writeLock(); try { - Item item = new Item(cursor, (PositionImpl) position, position != null ? heap.size() : -1); + Item item = new Item(cursor, position, position != null ? heap.size() : -1); cursors.put(cursor.getName(), item); if (position != null) { heap.add(item); @@ -94,6 +171,7 @@ public void add(ManagedCursor cursor, Position position) { if (cursor.isDurable()) { durableCursorCount++; } + version = DataVersion.getNextVersion(version); } finally { rwLock.unlockWrite(stamp); } @@ -129,6 +207,7 @@ public boolean removeCursor(String name) { if (item.cursor.isDurable()) { durableCursorCount--; } + version = DataVersion.getNextVersion(version); return true; } else { return false; @@ -150,7 +229,7 @@ public boolean removeCursor(String name) { * @return a pair of positions, representing the previous slowest reader and the new slowest reader (after the * update). */ - public Pair cursorUpdated(ManagedCursor cursor, Position newPosition) { + public Pair cursorUpdated(ManagedCursor cursor, Position newPosition) { requireNonNull(cursor); long stamp = rwLock.writeLock(); @@ -160,8 +239,9 @@ public Pair cursorUpdated(ManagedCursor cursor, Posi return null; } - PositionImpl previousSlowestConsumer = heap.get(0).position; - item.position = (PositionImpl) newPosition; + Position previousSlowestConsumer = heap.get(0).position; + item.position = newPosition; + version = DataVersion.getNextVersion(version); if (heap.size() == 1) { return Pair.of(previousSlowestConsumer, item.position); @@ -174,7 +254,7 @@ public Pair cursorUpdated(ManagedCursor cursor, Posi } else { siftUp(item); } - PositionImpl newSlowestConsumer = heap.get(0).position; + Position newSlowestConsumer = heap.get(0).position; return Pair.of(previousSlowestConsumer, newSlowestConsumer); } finally { rwLock.unlockWrite(stamp); @@ -186,7 +266,7 @@ public Pair cursorUpdated(ManagedCursor cursor, Posi * * @return the slowest reader position */ - public PositionImpl getSlowestReaderPosition() { + public Position getSlowestReaderPosition() { long stamp = rwLock.readLock(); try { return heap.isEmpty() ? null : heap.get(0).position; @@ -204,6 +284,24 @@ public ManagedCursor getSlowestReader() { } } + /** + * @return Returns the CursorInfo for the cursor with the oldest position, + * or null if there aren't any tracked cursors + */ + public CursorInfo getCursorWithOldestPosition() { + long stamp = rwLock.readLock(); + try { + if (heap.isEmpty()) { + return null; + } else { + Item item = heap.get(0); + return new CursorInfo(item.cursor, item.position, version); + } + } finally { + rwLock.unlockRead(stamp); + } + } + /** * Check whether there are any cursors. * @return true is there are no cursors and false if there are diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index ef607fa7ed7cf..f469b88cae8e6 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -59,6 +59,8 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Function; import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.LongStream; import org.apache.bookkeeper.client.AsyncCallback.CloseCallback; import org.apache.bookkeeper.client.AsyncCallback.OpenCallback; import org.apache.bookkeeper.client.BKException; @@ -76,6 +78,7 @@ import org.apache.bookkeeper.mledger.AsyncCallbacks.SkipEntriesCallback; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedCursorAttributes; import org.apache.bookkeeper.mledger.ManagedCursorMXBean; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; @@ -84,17 +87,23 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException; import org.apache.bookkeeper.mledger.ManagedLedgerException.NoMoreEntriesToReadException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionBound; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.ScanOutcome; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.PositionBound; import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback; import org.apache.bookkeeper.mledger.proto.MLDataFormats; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.LongListMap; import org.apache.bookkeeper.mledger.proto.MLDataFormats.LongProperty; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo; import org.apache.bookkeeper.mledger.proto.MLDataFormats.MessageRange; import org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo.Builder; import org.apache.bookkeeper.mledger.proto.MLDataFormats.StringProperty; +import org.apache.commons.lang3.mutable.MutableInt; import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats; +import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.BitSetRecyclable; import org.apache.pulsar.common.util.collections.LongPairRangeSet; @@ -118,30 +127,27 @@ public class ManagedCursorImpl implements ManagedCursor { return 0; }; protected final BookKeeper bookkeeper; - protected final ManagedLedgerConfig config; protected final ManagedLedgerImpl ledger; private final String name; - public static final String CURSOR_INTERNAL_PROPERTY_PREFIX = "#pulsar.internal."; - private volatile Map cursorProperties; private final BookKeeper.DigestType digestType; - protected volatile PositionImpl markDeletePosition; + protected volatile Position markDeletePosition; // this position is have persistent mark delete position - protected volatile PositionImpl persistentMarkDeletePosition; - protected static final AtomicReferenceFieldUpdater + protected volatile Position persistentMarkDeletePosition; + protected static final AtomicReferenceFieldUpdater INPROGRESS_MARKDELETE_PERSIST_POSITION_UPDATER = - AtomicReferenceFieldUpdater.newUpdater(ManagedCursorImpl.class, PositionImpl.class, + AtomicReferenceFieldUpdater.newUpdater(ManagedCursorImpl.class, Position.class, "inProgressMarkDeletePersistPosition"); - protected volatile PositionImpl inProgressMarkDeletePersistPosition; + protected volatile Position inProgressMarkDeletePersistPosition; - protected static final AtomicReferenceFieldUpdater READ_POSITION_UPDATER = - AtomicReferenceFieldUpdater.newUpdater(ManagedCursorImpl.class, PositionImpl.class, "readPosition"); - protected volatile PositionImpl readPosition; + protected static final AtomicReferenceFieldUpdater READ_POSITION_UPDATER = + AtomicReferenceFieldUpdater.newUpdater(ManagedCursorImpl.class, Position.class, "readPosition"); + protected volatile Position readPosition; // keeps sample of last read-position for validation and monitoring if read-position is not moving forward. - protected volatile PositionImpl statsLastReadPosition; + protected volatile Position statsLastReadPosition; protected static final AtomicReferenceFieldUpdater LAST_MARK_DELETE_ENTRY_UPDATER = AtomicReferenceFieldUpdater.newUpdater(ManagedCursorImpl.class, @@ -176,6 +182,7 @@ public class ManagedCursorImpl implements ManagedCursor { // Wether the current cursorLedger is read-only or writable private boolean isCursorLedgerReadOnly = true; + private boolean ledgerForceRecovery; // Stat of the cursor z-node // NOTE: Don't update cursorLedgerStat alone, @@ -183,23 +190,17 @@ public class ManagedCursorImpl implements ManagedCursor { private volatile Stat cursorLedgerStat; private volatile ManagedCursorInfo managedCursorInfo; - private static final LongPairConsumer positionRangeConverter = PositionImpl::new; + private static final LongPairConsumer positionRangeConverter = PositionFactory::create; - private static final RangeBoundConsumer positionRangeReverseConverter = - (position) -> new LongPairRangeSet.LongPair(position.ledgerId, position.entryId); + private static final RangeBoundConsumer positionRangeReverseConverter = + (position) -> new LongPairRangeSet.LongPair(position.getLedgerId(), position.getEntryId()); - private static final LongPairConsumer recyclePositionRangeConverter = (key, value) -> { - PositionImplRecyclable position = PositionImplRecyclable.create(); - position.ledgerId = key; - position.entryId = value; - position.ackSet = null; - return position; - }; - private final RangeSetWrapper individualDeletedMessages; + private static final LongPairConsumer recyclePositionRangeConverter = PositionRecyclable::get; + protected final RangeSetWrapper individualDeletedMessages; // Maintain the deletion status for batch messages // (ledgerId, entryId) -> deletion indexes - private final ConcurrentSkipListMap batchDeletedIndexes; + protected final ConcurrentSkipListMap batchDeletedIndexes; private final ReadWriteLock lock = new ReentrantReadWriteLock(); private RateLimiter markDeleteLimiter; @@ -221,7 +222,7 @@ public class ManagedCursorImpl implements ManagedCursor { private volatile boolean isActive = false; class MarkDeleteEntry { - final PositionImpl newPosition; + final Position newPosition; final MarkDeleteCallback callback; final Object ctx; final Map properties; @@ -231,7 +232,7 @@ class MarkDeleteEntry { // group. List callbackGroup; - public MarkDeleteEntry(PositionImpl newPosition, Map properties, + public MarkDeleteEntry(Position newPosition, Map properties, MarkDeleteCallback callback, Object ctx) { this.newPosition = newPosition; this.properties = properties; @@ -291,6 +292,11 @@ public enum State { protected final ManagedCursorMXBean mbean; + private volatile ManagedCursorAttributes managedCursorAttributes; + private static final AtomicReferenceFieldUpdater ATTRIBUTES_UPDATER = + AtomicReferenceFieldUpdater.newUpdater(ManagedCursorImpl.class, ManagedCursorAttributes.class, + "managedCursorAttributes"); + @SuppressWarnings("checkstyle:javadoctype") public interface VoidCallback { void operationComplete(); @@ -298,36 +304,36 @@ public interface VoidCallback { void operationFailed(ManagedLedgerException exception); } - ManagedCursorImpl(BookKeeper bookkeeper, ManagedLedgerConfig config, ManagedLedgerImpl ledger, String cursorName) { + ManagedCursorImpl(BookKeeper bookkeeper, ManagedLedgerImpl ledger, String cursorName) { this.bookkeeper = bookkeeper; this.cursorProperties = Collections.emptyMap(); - this.config = config; this.ledger = ledger; this.name = cursorName; this.individualDeletedMessages = new RangeSetWrapper<>(positionRangeConverter, positionRangeReverseConverter, this); - if (config.isDeletionAtBatchIndexLevelEnabled()) { + if (getConfig().isDeletionAtBatchIndexLevelEnabled()) { this.batchDeletedIndexes = new ConcurrentSkipListMap<>(); } else { this.batchDeletedIndexes = null; } - this.digestType = BookKeeper.DigestType.fromApiDigestType(config.getDigestType()); + this.digestType = BookKeeper.DigestType.fromApiDigestType(getConfig().getDigestType()); STATE_UPDATER.set(this, State.Uninitialized); PENDING_MARK_DELETED_SUBMITTED_COUNT_UPDATER.set(this, 0); PENDING_READ_OPS_UPDATER.set(this, 0); RESET_CURSOR_IN_PROGRESS_UPDATER.set(this, FALSE); WAITING_READ_OP_UPDATER.set(this, null); - this.clock = config.getClock(); + this.clock = getConfig().getClock(); this.lastActive = this.clock.millis(); this.lastLedgerSwitchTimestamp = this.clock.millis(); - if (config.getThrottleMarkDelete() > 0.0) { - markDeleteLimiter = RateLimiter.create(config.getThrottleMarkDelete()); + if (getConfig().getThrottleMarkDelete() > 0.0) { + markDeleteLimiter = RateLimiter.create(getConfig().getThrottleMarkDelete()); } else { // Disable mark-delete rate limiter markDeleteLimiter = null; } this.mbean = new ManagedCursorMXBeanImpl(this); + this.ledgerForceRecovery = getConfig().isLedgerForceRecovery(); } private void updateCursorLedgerStat(ManagedCursorInfo cursorInfo, Stat stat) { @@ -340,6 +346,16 @@ public Map getProperties() { return lastMarkDeleteEntry != null ? lastMarkDeleteEntry.properties : Collections.emptyMap(); } + @Override + public boolean isCursorDataFullyPersistable() { + lock.readLock().lock(); + try { + return individualDeletedMessages.size() <= getConfig().getMaxUnackedRangesToPersist(); + } finally { + lock.readLock().unlock(); + } + } + @Override public Map getCursorProperties() { return cursorProperties; @@ -349,15 +365,19 @@ private CompletableFuture computeCursorProperties( final Function, Map> updateFunction) { CompletableFuture updateCursorPropertiesResult = new CompletableFuture<>(); - final Stat lastCursorLedgerStat = ManagedCursorImpl.this.cursorLedgerStat; - Map newProperties = updateFunction.apply(ManagedCursorImpl.this.cursorProperties); + if (!isDurable()) { + this.cursorProperties = Collections.unmodifiableMap(newProperties); + updateCursorPropertiesResult.complete(null); + return updateCursorPropertiesResult; + } + ManagedCursorInfo copy = ManagedCursorInfo .newBuilder(ManagedCursorImpl.this.managedCursorInfo) .clearCursorProperties() .addAllCursorProperties(buildStringPropertiesMap(newProperties)) .build(); - + final Stat lastCursorLedgerStat = ManagedCursorImpl.this.cursorLedgerStat; ledger.getStore().asyncUpdateCursorInfo(ledger.getName(), name, copy, lastCursorLedgerStat, new MetaStoreCallback<>() { @Override @@ -467,9 +487,7 @@ void recover(final VoidCallback callback) { ledger.getStore().asyncGetCursorInfo(ledger.getName(), name, new MetaStoreCallback() { @Override public void operationComplete(ManagedCursorInfo info, Stat stat) { - updateCursorLedgerStat(info, stat); - lastActive = info.getLastActive() != 0 ? info.getLastActive() : lastActive; if (log.isDebugEnabled()) { log.debug("[{}] [{}] Recover cursor last active to [{}]", ledger.getName(), name, lastActive); @@ -489,7 +507,7 @@ public void operationComplete(ManagedCursorInfo info, Stat stat) { if (info.getCursorsLedgerId() == -1L) { // There is no cursor ledger to read the last position from. It means the cursor has been properly // closed and the last mark-delete position is stored in the ManagedCursorInfo itself. - PositionImpl recoveredPosition = new PositionImpl(info.getMarkDeleteLedgerId(), + Position recoveredPosition = PositionFactory.create(info.getMarkDeleteLedgerId(), info.getMarkDeleteEntryId()); if (info.getIndividualDeletedMessagesCount() > 0) { recoverIndividualDeletedMessages(info.getIndividualDeletedMessagesList()); @@ -509,7 +527,7 @@ public void operationComplete(ManagedCursorInfo info, Stat stat) { callback.operationComplete(); } else { // Need to proceed and read the last entry in the specified ledger to find out the last position - log.info("[{}] Consumer {} meta-data recover from ledger {}", ledger.getName(), name, + log.info("[{}] Cursor {} meta-data recover from ledger {}", ledger.getName(), name, info.getCursorsLedgerId()); recoverFromLedger(info, callback); } @@ -529,16 +547,16 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac long ledgerId = info.getCursorsLedgerId(); OpenCallback openCallback = (rc, lh, ctx) -> { if (log.isInfoEnabled()) { - log.info("[{}] Opened ledger {} for consumer {}. rc={}", ledger.getName(), ledgerId, name, rc); + log.info("[{}] Opened ledger {} for cursor {}. rc={}", ledger.getName(), ledgerId, name, rc); } - if (isBkErrorNotRecoverable(rc)) { - log.error("[{}] Error opening metadata ledger {} for consumer {}: {}", ledger.getName(), ledgerId, name, + if (isBkErrorNotRecoverable(rc) || ledgerForceRecovery) { + log.error("[{}] Error opening metadata ledger {} for cursor {}: {}", ledger.getName(), ledgerId, name, BKException.getMessage(rc)); // Rewind to oldest entry available - initialize(getRollbackPosition(info), Collections.emptyMap(), Collections.emptyMap(), callback); + initialize(getRollbackPosition(info), Collections.emptyMap(), cursorProperties, callback); return; } else if (rc != BKException.Code.OK) { - log.warn("[{}] Error opening metadata ledger {} for consumer {}: {}", ledger.getName(), ledgerId, name, + log.warn("[{}] Error opening metadata ledger {} for cursor {}: {}", ledger.getName(), ledgerId, name, BKException.getMessage(rc)); callback.operationFailed(new ManagedLedgerException(BKException.getMessage(rc))); return; @@ -548,7 +566,7 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac long lastEntryInLedger = lh.getLastAddConfirmed(); if (lastEntryInLedger < 0) { - log.warn("[{}] Error reading from metadata ledger {} for consumer {}: No entries in ledger", + log.warn("[{}] Error reading from metadata ledger {} for cursor {}: No entries in ledger", ledger.getName(), ledgerId, name); // Rewind to last cursor snapshot available initialize(getRollbackPosition(info), Collections.emptyMap(), cursorProperties, callback); @@ -559,14 +577,14 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac if (log.isDebugEnabled()) { log.debug("[{}} readComplete rc={} entryId={}", ledger.getName(), rc1, lh1.getLastAddConfirmed()); } - if (isBkErrorNotRecoverable(rc1)) { - log.error("[{}] Error reading from metadata ledger {} for consumer {}: {}", ledger.getName(), + if (isBkErrorNotRecoverable(rc1) || ledgerForceRecovery) { + log.error("[{}] Error reading from metadata ledger {} for cursor {}: {}", ledger.getName(), ledgerId, name, BKException.getMessage(rc1)); // Rewind to oldest entry available initialize(getRollbackPosition(info), Collections.emptyMap(), cursorProperties, callback); return; } else if (rc1 != BKException.Code.OK) { - log.warn("[{}] Error reading from metadata ledger {} for consumer {}: {}", ledger.getName(), + log.warn("[{}] Error reading from metadata ledger {} for cursor {}: {}", ledger.getName(), ledgerId, name, BKException.getMessage(rc1)); callback.operationFailed(createManagedLedgerException(rc1)); @@ -593,11 +611,9 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac } } - PositionImpl position = new PositionImpl(positionInfo); - if (positionInfo.getIndividualDeletedMessagesCount() > 0) { - recoverIndividualDeletedMessages(positionInfo.getIndividualDeletedMessagesList()); - } - if (config.isDeletionAtBatchIndexLevelEnabled() + Position position = PositionFactory.create(positionInfo.getLedgerId(), positionInfo.getEntryId()); + recoverIndividualDeletedMessages(positionInfo); + if (getConfig().isDeletionAtBatchIndexLevelEnabled() && positionInfo.getBatchedEntryDeletionIndexInfoCount() > 0) { recoverBatchDeletedIndexes(positionInfo.getBatchedEntryDeletionIndexInfoList()); } @@ -606,7 +622,8 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac }, null); }; try { - bookkeeper.asyncOpenLedger(ledgerId, digestType, config.getPassword(), openCallback, null); + bookkeeper.asyncOpenLedger(ledgerId, digestType, getConfig().getPassword(), openCallback, + null); } catch (Throwable t) { log.error("[{}] Encountered error on opening cursor ledger {} for cursor {}", ledger.getName(), ledgerId, name, t); @@ -614,6 +631,45 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac } } + public void recoverIndividualDeletedMessages(PositionInfo positionInfo) { + if (positionInfo.getIndividualDeletedMessagesCount() > 0) { + recoverIndividualDeletedMessages(positionInfo.getIndividualDeletedMessagesList()); + } else if (positionInfo.getIndividualDeletedMessageRangesCount() > 0) { + List rangeList = positionInfo.getIndividualDeletedMessageRangesList(); + try { + Map rangeMap = rangeList.stream().collect(Collectors.toMap(LongListMap::getKey, + list -> list.getValuesList().stream().mapToLong(i -> i).toArray())); + individualDeletedMessages.build(rangeMap); + } catch (Exception e) { + log.warn("[{}]-{} Failed to recover individualDeletedMessages from serialized data", ledger.getName(), + name, e); + } + } + } + + private List buildLongPropertiesMap(Map properties) { + if (properties.isEmpty()) { + return Collections.emptyList(); + } + List longListMap = new ArrayList<>(); + MutableInt serializedSize = new MutableInt(); + properties.forEach((id, ranges) -> { + if (ranges == null || ranges.length <= 0) { + return; + } + org.apache.bookkeeper.mledger.proto.MLDataFormats.LongListMap.Builder lmBuilder = LongListMap.newBuilder() + .setKey(id); + for (long range : ranges) { + lmBuilder.addValues(range); + } + LongListMap lm = lmBuilder.build(); + longListMap.add(lm); + serializedSize.add(lm.getSerializedSize()); + }); + individualDeletedMessagesSerializedSize = serializedSize.toInteger(); + return longListMap; + } + private void recoverIndividualDeletedMessages(List individualDeletedMessagesList) { lock.writeLock().lock(); try { @@ -662,8 +718,10 @@ private void recoverBatchDeletedIndexes ( for (int i = 0; i < batchDeletedIndexInfo.getDeleteSetList().size(); i++) { array[i] = batchDeletedIndexInfo.getDeleteSetList().get(i); } - this.batchDeletedIndexes.put(PositionImpl.get(batchDeletedIndexInfo.getPosition().getLedgerId(), - batchDeletedIndexInfo.getPosition().getEntryId()), BitSetRecyclable.create().resetWords(array)); + this.batchDeletedIndexes.put( + PositionFactory.create(batchDeletedIndexInfo.getPosition().getLedgerId(), + batchDeletedIndexInfo.getPosition().getEntryId()), + BitSetRecyclable.create().resetWords(array)); } }); } finally { @@ -671,18 +729,18 @@ private void recoverBatchDeletedIndexes ( } } - private void recoveredCursor(PositionImpl position, Map properties, + private void recoveredCursor(Position position, Map properties, Map cursorProperties, LedgerHandle recoveredFromCursorLedger) { // if the position was at a ledger that didn't exist (since it will be deleted if it was previously empty), // we need to move to the next existing ledger - if (!ledger.ledgerExists(position.getLedgerId())) { + if (position.getEntryId() == -1L && !ledger.ledgerExists(position.getLedgerId())) { Long nextExistingLedger = ledger.getNextValidLedger(position.getLedgerId()); if (nextExistingLedger == null) { log.info("[{}] [{}] Couldn't find next next valid ledger for recovery {}", ledger.getName(), name, position); } - position = nextExistingLedger != null ? PositionImpl.get(nextExistingLedger, -1) : position; + position = nextExistingLedger != null ? PositionFactory.create(nextExistingLedger, -1) : position; } if (position.compareTo(ledger.getLastPosition()) > 0) { log.warn("[{}] [{}] Current position {} is ahead of last position {}", ledger.getName(), name, position, @@ -690,7 +748,7 @@ private void recoveredCursor(PositionImpl position, Map properties position = ledger.getLastPosition(); } log.info("[{}] Cursor {} recovered to position {}", ledger.getName(), name, position); - this.cursorProperties = cursorProperties; + this.cursorProperties = cursorProperties == null ? Collections.emptyMap() : cursorProperties; messagesConsumedCounter = -getNumberOfEntries(Range.openClosed(position, ledger.getLastPosition())); markDeletePosition = position; persistentMarkDeletePosition = position; @@ -704,7 +762,7 @@ private void recoveredCursor(PositionImpl position, Map properties STATE_UPDATER.set(this, State.NoLedger); } - void initialize(PositionImpl position, Map properties, Map cursorProperties, + void initialize(Position position, Map properties, Map cursorProperties, final VoidCallback callback) { recoveredCursor(position, properties, cursorProperties, null); if (log.isDebugEnabled()) { @@ -750,7 +808,7 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { counter.countDown(); } - }, null, PositionImpl.LATEST); + }, null, PositionFactory.LATEST); counter.await(); @@ -763,19 +821,19 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { @Override public void asyncReadEntries(final int numberOfEntriesToRead, final ReadEntriesCallback callback, - final Object ctx, PositionImpl maxPosition) { + final Object ctx, Position maxPosition) { asyncReadEntries(numberOfEntriesToRead, NO_MAX_SIZE_LIMIT, callback, ctx, maxPosition); } @Override public void asyncReadEntries(int numberOfEntriesToRead, long maxSizeBytes, ReadEntriesCallback callback, - Object ctx, PositionImpl maxPosition) { + Object ctx, Position maxPosition) { asyncReadEntriesWithSkip(numberOfEntriesToRead, maxSizeBytes, callback, ctx, maxPosition, null); } @Override public void asyncReadEntriesWithSkip(int numberOfEntriesToRead, long maxSizeBytes, ReadEntriesCallback callback, - Object ctx, PositionImpl maxPosition, Predicate skipCondition) { + Object ctx, Position maxPosition, Predicate skipCondition) { checkArgument(numberOfEntriesToRead > 0); if (isClosed()) { callback.readEntriesFailed(new ManagedLedgerException @@ -786,6 +844,8 @@ public void asyncReadEntriesWithSkip(int numberOfEntriesToRead, long maxSizeByte int numOfEntriesToRead = applyMaxSizeCap(numberOfEntriesToRead, maxSizeBytes); PENDING_READ_OPS_UPDATER.incrementAndGet(this); + // Skip deleted entries. + skipCondition = skipCondition == null ? this::isMessageDeleted : skipCondition.or(this::isMessageDeleted); OpReadEntry op = OpReadEntry.create(this, readPosition, numOfEntriesToRead, callback, ctx, maxPosition, skipCondition); ledger.asyncReadEntries(op); @@ -816,6 +876,11 @@ public void readEntryComplete(Entry entry, Object ctx) { result.entry = entry; counter.countDown(); } + + @Override + public String toString() { + return String.format("Cursor [%s] get Nth entry", ManagedCursorImpl.this); + } }, null); counter.await(ledger.getConfig().getMetadataOperationsTimeoutSeconds(), TimeUnit.SECONDS); @@ -837,8 +902,8 @@ public void asyncGetNthEntry(int n, IndividualDeletedEntries deletedEntries, Rea return; } - PositionImpl startPosition = ledger.getNextValidPosition(markDeletePosition); - PositionImpl endPosition = ledger.getLastPosition(); + Position startPosition = ledger.getNextValidPosition(markDeletePosition); + Position endPosition = ledger.getLastPosition(); if (startPosition.compareTo(endPosition) <= 0) { long numOfEntries = getNumberOfEntries(Range.closed(startPosition, endPosition)); if (numOfEntries >= n) { @@ -846,7 +911,7 @@ public void asyncGetNthEntry(int n, IndividualDeletedEntries deletedEntries, Rea if (deletedEntries == IndividualDeletedEntries.Exclude) { deletedMessages = getNumIndividualDeletedEntriesToSkip(n); } - PositionImpl positionAfterN = ledger.getPositionAfterN(markDeletePosition, n + deletedMessages, + Position positionAfterN = ledger.getPositionAfterN(markDeletePosition, n + deletedMessages, PositionBound.startExcluded); ledger.asyncReadEntry(positionAfterN, callback, ctx); } else { @@ -889,7 +954,7 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { counter.countDown(); } - }, null, PositionImpl.LATEST); + }, null, PositionFactory.LATEST); counter.await(); @@ -902,27 +967,27 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { @Override public void asyncReadEntriesOrWait(int numberOfEntriesToRead, ReadEntriesCallback callback, Object ctx, - PositionImpl maxPosition) { + Position maxPosition) { asyncReadEntriesOrWait(numberOfEntriesToRead, NO_MAX_SIZE_LIMIT, callback, ctx, maxPosition); } @Override public void asyncReadEntriesOrWait(int maxEntries, long maxSizeBytes, ReadEntriesCallback callback, Object ctx, - PositionImpl maxPosition) { + Position maxPosition) { asyncReadEntriesWithSkipOrWait(maxEntries, maxSizeBytes, callback, ctx, maxPosition, null); } @Override public void asyncReadEntriesWithSkipOrWait(int maxEntries, ReadEntriesCallback callback, - Object ctx, PositionImpl maxPosition, - Predicate skipCondition) { + Object ctx, Position maxPosition, + Predicate skipCondition) { asyncReadEntriesWithSkipOrWait(maxEntries, NO_MAX_SIZE_LIMIT, callback, ctx, maxPosition, skipCondition); } @Override public void asyncReadEntriesWithSkipOrWait(int maxEntries, long maxSizeBytes, ReadEntriesCallback callback, - Object ctx, PositionImpl maxPosition, - Predicate skipCondition) { + Object ctx, Position maxPosition, + Predicate skipCondition) { checkArgument(maxEntries > 0); if (isClosed()) { callback.readEntriesFailed(new CursorAlreadyClosedException("Cursor was already closed"), ctx); @@ -931,7 +996,7 @@ public void asyncReadEntriesWithSkipOrWait(int maxEntries, long maxSizeBytes, Re int numberOfEntriesToRead = applyMaxSizeCap(maxEntries, maxSizeBytes); - if (hasMoreEntries()) { + if (hasMoreEntries() && maxPosition.compareTo(readPosition) >= 0) { // If we have available entries, we can read them immediately if (log.isDebugEnabled()) { log.debug("[{}] [{}] Read entries immediately", ledger.getName(), name); @@ -939,6 +1004,8 @@ public void asyncReadEntriesWithSkipOrWait(int maxEntries, long maxSizeBytes, Re asyncReadEntriesWithSkip(numberOfEntriesToRead, NO_MAX_SIZE_LIMIT, callback, ctx, maxPosition, skipCondition); } else { + // Skip deleted entries. + skipCondition = skipCondition == null ? this::isMessageDeleted : skipCondition.or(this::isMessageDeleted); OpReadEntry op = OpReadEntry.create(this, readPosition, numberOfEntriesToRead, callback, ctx, maxPosition, skipCondition); @@ -954,10 +1021,10 @@ public void asyncReadEntriesWithSkipOrWait(int maxEntries, long maxSizeBytes, Re // Check again for new entries after the configured time, then if still no entries are available register // to be notified - if (config.getNewEntriesCheckDelayInMillis() > 0) { + if (getConfig().getNewEntriesCheckDelayInMillis() > 0) { ledger.getScheduledExecutor() .schedule(() -> checkForNewEntries(op, callback, ctx), - config.getNewEntriesCheckDelayInMillis(), TimeUnit.MILLISECONDS); + getConfig().getNewEntriesCheckDelayInMillis(), TimeUnit.MILLISECONDS); } else { // If there's no delay, check directly from the same thread checkForNewEntries(op, callback, ctx); @@ -971,13 +1038,18 @@ private void checkForNewEntries(OpReadEntry op, ReadEntriesCallback callback, Ob log.debug("[{}] [{}] Re-trying the read at position {}", ledger.getName(), name, op.readPosition); } + if (isClosed()) { + callback.readEntriesFailed(new CursorAlreadyClosedException("Cursor was already closed"), ctx); + return; + } + if (!hasMoreEntries()) { if (log.isDebugEnabled()) { log.debug("[{}] [{}] Still no entries available. Register for notification", ledger.getName(), name); } // Let the managed ledger know we want to be notified whenever a new entry is published - ledger.waitingCursors.add(this); + ledger.addWaitingCursor(this); } else { if (log.isDebugEnabled()) { log.debug("[{}] [{}] Skip notification registering since we do have entries available", @@ -1045,7 +1117,7 @@ public boolean hasMoreEntries() { // * Writer pointing to "invalid" entry -1 (meaning no entries in that ledger) --> Need to check if the reader // is // at the last entry in the previous ledger - PositionImpl writerPosition = ledger.getLastPosition(); + Position writerPosition = ledger.getLastPosition(); if (writerPosition.getEntryId() != -1) { return readPosition.compareTo(writerPosition) <= 0; } else { @@ -1072,8 +1144,8 @@ public long getNumberOfEntries() { public long getNumberOfEntriesSinceFirstNotAckedMessage() { // sometimes for already caught up consumer: due to race condition markDeletePosition > readPosition. so, // validate it before preparing range - PositionImpl markDeletePosition = this.markDeletePosition; - PositionImpl readPosition = this.readPosition; + Position markDeletePosition = this.markDeletePosition; + Position readPosition = this.readPosition; return (markDeletePosition != null && readPosition != null && markDeletePosition.compareTo(readPosition) < 0) ? ledger.getNumberOfEntries(Range.openClosed(markDeletePosition, readPosition)) : 0; @@ -1081,7 +1153,12 @@ public long getNumberOfEntriesSinceFirstNotAckedMessage() { @Override public int getTotalNonContiguousDeletedMessagesRange() { - return individualDeletedMessages.size(); + lock.readLock().lock(); + try { + return individualDeletedMessages.size(); + } finally { + lock.readLock().unlock(); + } } @Override @@ -1094,6 +1171,13 @@ public long getEstimatedSizeSinceMarkDeletePosition() { return ledger.estimateBacklogFromPosition(markDeletePosition); } + private long getNumberOfEntriesInBacklog() { + if (markDeletePosition.compareTo(ledger.getLastPosition()) >= 0) { + return 0; + } + return getNumberOfEntries(Range.openClosed(markDeletePosition, ledger.getLastPosition())); + } + @Override public long getNumberOfEntriesInBacklog(boolean isPrecise) { if (log.isDebugEnabled()) { @@ -1102,13 +1186,13 @@ public long getNumberOfEntriesInBacklog(boolean isPrecise) { messagesConsumedCounter, markDeletePosition, readPosition); } if (isPrecise) { - return getNumberOfEntries(Range.openClosed(markDeletePosition, ledger.getLastPosition())); + return getNumberOfEntriesInBacklog(); } long backlog = ManagedLedgerImpl.ENTRIES_ADDED_COUNTER_UPDATER.get(ledger) - messagesConsumedCounter; if (backlog < 0) { // In some case the counters get incorrect values, fall back to the precise backlog count - backlog = getNumberOfEntries(Range.openClosed(markDeletePosition, ledger.getLastPosition())); + backlog = getNumberOfEntriesInBacklog(); } return backlog; @@ -1127,7 +1211,7 @@ public Position findNewestMatching(Predicate condition) throws Interrupte public CompletableFuture scan(Optional position, Predicate condition, int batchSize, long maxEntries, long timeOutMs) { - PositionImpl startPosition = (PositionImpl) position.orElseGet( + Position startPosition = position.orElseGet( () -> ledger.getNextValidPosition(markDeletePosition)); CompletableFuture future = new CompletableFuture<>(); OpScan op = new OpScan(this, batchSize, startPosition, condition, new ScanCallback() { @@ -1182,12 +1266,18 @@ public void findEntryFailed(ManagedLedgerException exception, Optional @Override public void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate condition, FindEntryCallback callback, Object ctx) { + asyncFindNewestMatching(constraint, condition, callback, ctx, false); + } + + @Override + public void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate condition, + FindEntryCallback callback, Object ctx, boolean isFindFromLedger) { OpFindNewest op; - PositionImpl startPosition = null; + Position startPosition = null; long max = 0; switch (constraint) { case SearchAllAvailableEntries: - startPosition = (PositionImpl) getFirstPosition(); + startPosition = getFirstPosition(); max = ledger.getNumberOfEntries() - 1; break; case SearchActiveEntries: @@ -1203,7 +1293,11 @@ public void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate Optional.empty(), ctx); return; } - op = new OpFindNewest(this, startPosition, condition, max, callback, ctx); + if (isFindFromLedger) { + op = new OpFindNewest(this.ledger, startPosition, condition, max, callback, ctx); + } else { + op = new OpFindNewest(this, startPosition, condition, max, callback, ctx); + } op.find(); } @@ -1237,21 +1331,22 @@ public void setAlwaysInactive() { @Override public Position getFirstPosition() { Long firstLedgerId = ledger.getLedgersInfo().firstKey(); - return firstLedgerId == null ? null : new PositionImpl(firstLedgerId, 0); + return firstLedgerId == null ? null : PositionFactory.create(firstLedgerId, 0); } - protected void internalResetCursor(PositionImpl proposedReadPosition, + protected void internalResetCursor(Position proposedReadPosition, AsyncCallbacks.ResetCursorCallback resetCursorCallback) { - final PositionImpl newReadPosition; - if (proposedReadPosition.equals(PositionImpl.EARLIEST)) { + final Position newReadPosition; + if (proposedReadPosition.equals(PositionFactory.EARLIEST)) { newReadPosition = ledger.getFirstPosition(); - } else if (proposedReadPosition.equals(PositionImpl.LATEST)) { - newReadPosition = ledger.getLastPosition().getNext(); + } else if (proposedReadPosition.equals(PositionFactory.LATEST)) { + newReadPosition = ledger.getNextValidPosition(ledger.getLastPosition()); } else { newReadPosition = proposedReadPosition; } - log.info("[{}] Initiate reset readPosition to {} on cursor {}", ledger.getName(), newReadPosition, name); + log.info("[{}] Initiate reset readPosition from {} to {} on cursor {}", ledger.getName(), readPosition, + newReadPosition, name); synchronized (pendingMarkDeleteOps) { if (!RESET_CURSOR_IN_PROGRESS_UPDATER.compareAndSet(this, FALSE, TRUE)) { @@ -1266,7 +1361,7 @@ protected void internalResetCursor(PositionImpl proposedReadPosition, final AsyncCallbacks.ResetCursorCallback callback = resetCursorCallback; - final PositionImpl newMarkDeletePosition = ledger.getPreviousPosition(newReadPosition); + final Position newMarkDeletePosition = ledger.getPreviousPosition(newReadPosition); VoidCallback finalCallback = new VoidCallback() { @Override @@ -1286,17 +1381,19 @@ public void operationComplete() { lastMarkDeleteEntry = new MarkDeleteEntry(newMarkDeletePosition, isCompactionCursor() ? getProperties() : Collections.emptyMap(), null, null); individualDeletedMessages.clear(); - if (config.isDeletionAtBatchIndexLevelEnabled()) { + if (getConfig().isDeletionAtBatchIndexLevelEnabled()) { batchDeletedIndexes.values().forEach(BitSetRecyclable::recycle); batchDeletedIndexes.clear(); - long[] resetWords = newReadPosition.ackSet; - if (resetWords != null) { - BitSetRecyclable ackSet = BitSetRecyclable.create().resetWords(resetWords); - batchDeletedIndexes.put(newReadPosition, ackSet); - } + AckSetStateUtil.maybeGetAckSetState(newReadPosition).ifPresent(ackSetState -> { + long[] resetWords = ackSetState.getAckSet(); + if (resetWords != null) { + BitSetRecyclable ackSet = BitSetRecyclable.create().resetWords(resetWords); + batchDeletedIndexes.put(newReadPosition, ackSet); + } + }); } - PositionImpl oldReadPosition = readPosition; + Position oldReadPosition = readPosition; if (oldReadPosition.compareTo(newReadPosition) >= 0) { log.info("[{}] reset readPosition to {} before current read readPosition {} on cursor {}", ledger.getName(), newReadPosition, oldReadPosition, name); @@ -1353,23 +1450,22 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { @Override public void asyncResetCursor(Position newPos, boolean forceReset, AsyncCallbacks.ResetCursorCallback callback) { - checkArgument(newPos instanceof PositionImpl); - final PositionImpl newPosition = (PositionImpl) newPos; + final Position newPosition = newPos; // order trim and reset operations on a ledger ledger.getExecutor().execute(() -> { - PositionImpl actualPosition = newPosition; + Position actualPosition = newPosition; if (!ledger.isValidPosition(actualPosition) - && !actualPosition.equals(PositionImpl.EARLIEST) - && !actualPosition.equals(PositionImpl.LATEST) + && !actualPosition.equals(PositionFactory.EARLIEST) + && !actualPosition.equals(PositionFactory.LATEST) && !forceReset) { actualPosition = ledger.getNextValidPosition(actualPosition); if (actualPosition == null) { // next valid position would only return null when newPos // is larger than all available positions, then it's latest in effect. - actualPosition = PositionImpl.LATEST; + actualPosition = PositionFactory.LATEST; } } @@ -1473,11 +1569,7 @@ public Set asyncReplayEntries(Set positi Set alreadyAcknowledgedPositions = new HashSet<>(); lock.readLock().lock(); try { - positions.stream() - .filter(position -> individualDeletedMessages.contains(((PositionImpl) position).getLedgerId(), - ((PositionImpl) position).getEntryId()) - || ((PositionImpl) position).compareTo(markDeletePosition) <= 0) - .forEach(alreadyAcknowledgedPositions::add); + positions.stream().filter(this::isMessageDeleted).forEach(alreadyAcknowledgedPositions::add); } finally { lock.readLock().unlock(); } @@ -1518,22 +1610,27 @@ public synchronized void readEntryFailed(ManagedLedgerException mle, Object ctx) callback.readEntriesFailed(exception.get(), ctx); } } + + @Override + public String toString() { + return String.format("Cursor [%s] async replay entries", ManagedCursorImpl.this); + } }; positions.stream().filter(position -> !alreadyAcknowledgedPositions.contains(position)) .forEach(p ->{ - if (((PositionImpl) p).compareTo(this.readPosition) == 0) { + if (p.compareTo(this.readPosition) == 0) { this.setReadPosition(this.readPosition.getNext()); log.warn("[{}][{}] replayPosition{} equals readPosition{}," + " need set next readPosition", ledger.getName(), name, p, this.readPosition); } - ledger.asyncReadEntry((PositionImpl) p, cb, ctx); + ledger.asyncReadEntry(p, cb, ctx); }); return alreadyAcknowledgedPositions; } - protected long getNumberOfEntries(Range range) { + protected long getNumberOfEntries(Range range) { long allEntries = ledger.getNumberOfEntries(range); if (log.isDebugEnabled()) { @@ -1544,16 +1641,16 @@ protected long getNumberOfEntries(Range range) { lock.readLock().lock(); try { - if (config.isUnackedRangesOpenCacheSetEnabled()) { + if (getConfig().isUnackedRangesOpenCacheSetEnabled()) { int cardinality = individualDeletedMessages.cardinality( - range.lowerEndpoint().ledgerId, range.lowerEndpoint().entryId, - range.upperEndpoint().ledgerId, range.upperEndpoint().entryId); + range.lowerEndpoint().getLedgerId(), range.lowerEndpoint().getEntryId(), + range.upperEndpoint().getLedgerId(), range.upperEndpoint().getEntryId()); deletedEntries.addAndGet(cardinality); } else { individualDeletedMessages.forEach((r) -> { try { if (r.isConnected(range)) { - Range commonEntries = r.intersection(range); + Range commonEntries = r.intersection(range); long commonCount = ledger.getNumberOfEntries(commonEntries); if (log.isDebugEnabled()) { log.debug("[{}] [{}] Discounting {} entries for already deleted range {}", @@ -1563,9 +1660,9 @@ protected long getNumberOfEntries(Range range) { } return true; } finally { - if (r.lowerEndpoint() instanceof PositionImplRecyclable) { - ((PositionImplRecyclable) r.lowerEndpoint()).recycle(); - ((PositionImplRecyclable) r.upperEndpoint()).recycle(); + if (r.lowerEndpoint() instanceof PositionRecyclable) { + ((PositionRecyclable) r.lowerEndpoint()).recycle(); + ((PositionRecyclable) r.upperEndpoint()).recycle(); } } }, recyclePositionRangeConverter); @@ -1591,7 +1688,6 @@ public void markDelete(Position position) throws InterruptedException, ManagedLe public void markDelete(Position position, Map properties) throws InterruptedException, ManagedLedgerException { requireNonNull(position); - checkArgument(position instanceof PositionImpl); class Result { ManagedLedgerException exception = null; @@ -1744,10 +1840,10 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { private static class InvidualDeletedMessagesHandlingState { long totalEntriesToSkip = 0L; long deletedMessages = 0L; - PositionImpl startPosition; - PositionImpl endPosition; + Position startPosition; + Position endPosition; - InvidualDeletedMessagesHandlingState(PositionImpl startPosition) { + InvidualDeletedMessagesHandlingState(Position startPosition) { this.startPosition = startPosition; } } @@ -1760,7 +1856,7 @@ long getNumIndividualDeletedEntriesToSkip(long numEntries) { try { state.endPosition = r.lowerEndpoint(); if (state.startPosition.compareTo(state.endPosition) <= 0) { - Range range = Range.openClosed(state.startPosition, state.endPosition); + Range range = Range.openClosed(state.startPosition, state.endPosition); long entries = ledger.getNumberOfEntries(range); if (state.totalEntriesToSkip + entries >= numEntries) { // do not process further @@ -1777,9 +1873,8 @@ long getNumIndividualDeletedEntriesToSkip(long numEntries) { } return true; } finally { - if (r.lowerEndpoint() instanceof PositionImplRecyclable) { - ((PositionImplRecyclable) r.lowerEndpoint()).recycle(); - ((PositionImplRecyclable) r.upperEndpoint()).recycle(); + if (r.lowerEndpoint() instanceof PositionRecyclable) { + ((PositionRecyclable) r.lowerEndpoint()).recycle(); } } }, recyclePositionRangeConverter); @@ -1789,15 +1884,15 @@ long getNumIndividualDeletedEntriesToSkip(long numEntries) { } } - boolean hasMoreEntries(PositionImpl position) { - PositionImpl lastPositionInLedger = ledger.getLastPosition(); + boolean hasMoreEntries(Position position) { + Position lastPositionInLedger = ledger.getLastPosition(); if (position.compareTo(lastPositionInLedger) <= 0) { return getNumberOfEntries(Range.closed(position, lastPositionInLedger)) > 0; } return false; } - void initializeCursorPosition(Pair lastPositionCounter) { + void initializeCursorPosition(Pair lastPositionCounter) { readPosition = ledger.getNextValidPosition(lastPositionCounter.getLeft()); ledger.onCursorReadPositionUpdated(this, readPosition); markDeletePosition = lastPositionCounter.getLeft(); @@ -1816,14 +1911,14 @@ void initializeCursorPosition(Pair lastPositionCounter) { * the new acknowledged position * @return the previous acknowledged position */ - PositionImpl setAcknowledgedPosition(PositionImpl newMarkDeletePosition) { + Position setAcknowledgedPosition(Position newMarkDeletePosition) { if (newMarkDeletePosition.compareTo(markDeletePosition) < 0) { throw new MarkDeletingMarkedPosition( "Mark deleting an already mark-deleted position. Current mark-delete: " + markDeletePosition + " -- attempted mark delete: " + newMarkDeletePosition); } - PositionImpl oldMarkDeletePosition = markDeletePosition; + Position oldMarkDeletePosition = markDeletePosition; if (!newMarkDeletePosition.equals(oldMarkDeletePosition)) { long skippedEntries = 0; @@ -1836,14 +1931,14 @@ PositionImpl setAcknowledgedPosition(PositionImpl newMarkDeletePosition) { skippedEntries = getNumberOfEntries(Range.openClosed(oldMarkDeletePosition, newMarkDeletePosition)); } - PositionImpl positionAfterNewMarkDelete = ledger.getNextValidPosition(newMarkDeletePosition); + Position positionAfterNewMarkDelete = ledger.getNextValidPosition(newMarkDeletePosition); // sometime ranges are connected but belongs to different ledgers so, they are placed sequentially // eg: (2:10..3:15] can be returned as (2:10..2:15],[3:0..3:15]. So, try to iterate over connected range and // found the last non-connected range which gives new markDeletePosition while (positionAfterNewMarkDelete.compareTo(ledger.lastConfirmedEntry) <= 0) { if (individualDeletedMessages.contains(positionAfterNewMarkDelete.getLedgerId(), positionAfterNewMarkDelete.getEntryId())) { - Range rangeToBeMarkDeleted = individualDeletedMessages.rangeContaining( + Range rangeToBeMarkDeleted = individualDeletedMessages.rangeContaining( positionAfterNewMarkDelete.getLedgerId(), positionAfterNewMarkDelete.getEntryId()); newMarkDeletePosition = rangeToBeMarkDeleted.upperEndpoint(); positionAfterNewMarkDelete = ledger.getNextValidPosition(newMarkDeletePosition); @@ -1869,7 +1964,7 @@ PositionImpl setAcknowledgedPosition(PositionImpl newMarkDeletePosition) { // If the position that is mark-deleted is past the read position, it // means that the client has skipped some entries. We need to move // read position forward - PositionImpl newReadPosition = ledger.getNextValidPosition(markDeletePosition); + Position newReadPosition = ledger.getNextValidPosition(markDeletePosition); if (log.isDebugEnabled()) { log.debug("[{}] Moved read position from: {} to: {}, and new mark-delete position {}", ledger.getName(), currentReadPosition, newReadPosition, markDeletePosition); @@ -1899,7 +1994,6 @@ public MarkDeletingMarkedPosition(String s) { public void asyncMarkDelete(final Position position, Map properties, final MarkDeleteCallback callback, final Object ctx) { requireNonNull(position); - checkArgument(position instanceof PositionImpl); if (isClosed()) { callback.markDeleteFailed(new ManagedLedgerException @@ -1923,12 +2017,14 @@ public void asyncMarkDelete(final Position position, Map propertie log.debug("[{}] Mark delete cursor {} up to position: {}", ledger.getName(), name, position); } - PositionImpl newPosition = (PositionImpl) position; + Position newPosition = position; - if (config.isDeletionAtBatchIndexLevelEnabled()) { - if (newPosition.ackSet != null) { + Optional ackSetStateOptional = AckSetStateUtil.maybeGetAckSetState(newPosition); + if (getConfig().isDeletionAtBatchIndexLevelEnabled()) { + if (ackSetStateOptional.isPresent()) { AtomicReference bitSetRecyclable = new AtomicReference<>(); - BitSetRecyclable givenBitSet = BitSetRecyclable.create().resetWords(newPosition.ackSet); + BitSetRecyclable givenBitSet = + BitSetRecyclable.create().resetWords(ackSetStateOptional.map(AckSetState::getAckSet).get()); // In order to prevent the batch index recorded in batchDeletedIndexes from rolling back, // only update batchDeletedIndexes when the submitted batch index is greater // than the recorded index. @@ -1950,15 +2046,19 @@ public void asyncMarkDelete(final Position position, Map propertie } newPosition = ledger.getPreviousPosition(newPosition); } - Map subMap = batchDeletedIndexes.subMap(PositionImpl.EARLIEST, newPosition); + Map subMap = batchDeletedIndexes.subMap(PositionFactory.EARLIEST, newPosition); subMap.values().forEach(BitSetRecyclable::recycle); subMap.clear(); - } else if (newPosition.ackSet != null) { - newPosition = ledger.getPreviousPosition(newPosition); - newPosition.ackSet = null; + } else { + if (ackSetStateOptional.isPresent()) { + AckSetState ackSetState = ackSetStateOptional.get(); + if (ackSetState.getAckSet() != null) { + newPosition = ledger.getPreviousPosition(newPosition); + } + } } - if (((PositionImpl) ledger.getLastConfirmedEntry()).compareTo(newPosition) < 0) { + if (ledger.getLastConfirmedEntry().compareTo(newPosition) < 0) { boolean shouldCursorMoveForward = false; try { long ledgerEntries = ledger.getLedgerInfo(markDeletePosition.getLedgerId()).get().getEntries(); @@ -2003,7 +2103,7 @@ public void asyncMarkDelete(final Position position, Map propertie internalAsyncMarkDelete(newPosition, properties, callback, ctx); } - protected void internalAsyncMarkDelete(final PositionImpl newPosition, Map properties, + protected void internalAsyncMarkDelete(final Position newPosition, Map properties, final MarkDeleteCallback callback, final Object ctx) { ledger.mbean.addMarkDeleteOp(); @@ -2058,7 +2158,7 @@ void internalMarkDelete(final MarkDeleteEntry mdEntry) { return; } - PositionImpl inProgressLatest = INPROGRESS_MARKDELETE_PERSIST_POSITION_UPDATER.updateAndGet(this, current -> { + Position inProgressLatest = INPROGRESS_MARKDELETE_PERSIST_POSITION_UPDATER.updateAndGet(this, current -> { if (current != null && current.compareTo(mdEntry.newPosition) > 0) { return current; } else { @@ -2091,7 +2191,7 @@ void internalMarkDelete(final MarkDeleteEntry mdEntry) { } }); - persistPositionToLedger(cursorLedger, mdEntry, new VoidCallback() { + VoidCallback cb = new VoidCallback() { @Override public void operationComplete() { if (log.isDebugEnabled()) { @@ -2108,9 +2208,9 @@ public void operationComplete() { try { individualDeletedMessages.removeAtMost(mdEntry.newPosition.getLedgerId(), mdEntry.newPosition.getEntryId()); - if (config.isDeletionAtBatchIndexLevelEnabled()) { - Map subMap = batchDeletedIndexes.subMap(PositionImpl.EARLIEST, - false, PositionImpl.get(mdEntry.newPosition.getLedgerId(), + if (getConfig().isDeletionAtBatchIndexLevelEnabled()) { + Map subMap = batchDeletedIndexes.subMap(PositionFactory.EARLIEST, + false, PositionFactory.create(mdEntry.newPosition.getLedgerId(), mdEntry.newPosition.getEntryId()), true); subMap.values().forEach(BitSetRecyclable::recycle); subMap.clear(); @@ -2143,7 +2243,17 @@ public void operationFailed(ManagedLedgerException exception) { mdEntry.triggerFailed(exception); } - }); + }; + + if (State.NoLedger.equals(STATE_UPDATER.get(this))) { + if (ledger.isNoMessagesAfterPos(mdEntry.newPosition)) { + persistPositionToMetaStore(mdEntry, cb); + } else { + cb.operationFailed(new ManagedLedgerException("Switch new cursor ledger failed")); + } + } else { + persistPositionToLedger(cursorLedger, mdEntry, cb); + } } @Override @@ -2213,7 +2323,7 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb return; } - PositionImpl newMarkDeletePosition = null; + Position newMarkDeletePosition = null; lock.writeLock().lock(); @@ -2224,8 +2334,8 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb } for (Position pos : positions) { - PositionImpl position = (PositionImpl) requireNonNull(pos); - if (((PositionImpl) ledger.getLastConfirmedEntry()).compareTo(position) < 0) { + Position position = requireNonNull(pos); + if (ledger.getLastConfirmedEntry().compareTo(position) < 0) { if (log.isDebugEnabled()) { log.debug( "[{}] Failed mark delete due to invalid markDelete {} is ahead of last-confirmed-entry {} " @@ -2235,9 +2345,8 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb return; } - if (individualDeletedMessages.contains(position.getLedgerId(), position.getEntryId()) - || position.compareTo(markDeletePosition) <= 0) { - if (config.isDeletionAtBatchIndexLevelEnabled()) { + if (isMessageDeleted(position)) { + if (getConfig().isDeletionAtBatchIndexLevelEnabled()) { BitSetRecyclable bitSetRecyclable = batchDeletedIndexes.remove(position); if (bitSetRecyclable != null) { bitSetRecyclable.recycle(); @@ -2248,8 +2357,9 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb } continue; } - if (position.ackSet == null) { - if (config.isDeletionAtBatchIndexLevelEnabled()) { + long[] ackSet = AckSetStateUtil.getAckSetArrayOrNull(position); + if (ackSet == null) { + if (getConfig().isDeletionAtBatchIndexLevelEnabled()) { BitSetRecyclable bitSetRecyclable = batchDeletedIndexes.remove(position); if (bitSetRecyclable != null) { bitSetRecyclable.recycle(); @@ -2257,7 +2367,7 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb } // Add a range (prev, pos] to the set. Adding the previous entry as an open limit to the range will // make the RangeSet recognize the "continuity" between adjacent Positions. - PositionImpl previousPosition = ledger.getPreviousPosition(position); + Position previousPosition = ledger.getPreviousPosition(position); individualDeletedMessages.addOpenClosed(previousPosition.getLedgerId(), previousPosition.getEntryId(), position.getLedgerId(), position.getEntryId()); MSG_CONSUMED_COUNTER_UPDATER.incrementAndGet(this); @@ -2266,15 +2376,15 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb log.debug("[{}] [{}] Individually deleted messages: {}", ledger.getName(), name, individualDeletedMessages); } - } else if (config.isDeletionAtBatchIndexLevelEnabled()) { - BitSetRecyclable givenBitSet = BitSetRecyclable.create().resetWords(position.ackSet); + } else if (getConfig().isDeletionAtBatchIndexLevelEnabled()) { + BitSetRecyclable givenBitSet = BitSetRecyclable.create().resetWords(ackSet); BitSetRecyclable bitSet = batchDeletedIndexes.computeIfAbsent(position, (v) -> givenBitSet); if (givenBitSet != bitSet) { bitSet.and(givenBitSet); givenBitSet.recycle(); } if (bitSet.isEmpty()) { - PositionImpl previousPosition = ledger.getPreviousPosition(position); + Position previousPosition = ledger.getPreviousPosition(position); individualDeletedMessages.addOpenClosed(previousPosition.getLedgerId(), previousPosition.getEntryId(), position.getLedgerId(), position.getEntryId()); @@ -2294,7 +2404,7 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb // If the lower bound of the range set is the current mark delete position, then we can trigger a new // mark-delete to the upper bound of the first range segment - Range range = individualDeletedMessages.firstRange(); + Range range = individualDeletedMessages.firstRange(); // If the upper bound is before the mark-delete position, we need to move ahead as these // individualDeletedMessages are now irrelevant @@ -2332,8 +2442,9 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb callback.deleteFailed(getManagedLedgerException(e), ctx); return; } finally { + boolean empty = individualDeletedMessages.isEmpty(); lock.writeLock().unlock(); - if (individualDeletedMessages.isEmpty()) { + if (empty) { callback.deleteComplete(ctx); } } @@ -2374,7 +2485,7 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { } // update lastMarkDeleteEntry field if newPosition is later than the current lastMarkDeleteEntry.newPosition - private void updateLastMarkDeleteEntryToLatest(final PositionImpl newPosition, + private void updateLastMarkDeleteEntryToLatest(final Position newPosition, final Map properties) { LAST_MARK_DELETE_ENTRY_UPDATER.updateAndGet(this, last -> { if (last != null && last.newPosition.compareTo(newPosition) > 0) { @@ -2399,13 +2510,13 @@ private void updateLastMarkDeleteEntryToLatest(final PositionImpl newPosition, List filterReadEntries(List entries) { lock.readLock().lock(); try { - Range entriesRange = Range.closed((PositionImpl) entries.get(0).getPosition(), - (PositionImpl) entries.get(entries.size() - 1).getPosition()); + Range entriesRange = Range.closed(entries.get(0).getPosition(), + entries.get(entries.size() - 1).getPosition()); if (log.isDebugEnabled()) { log.debug("[{}] [{}] Filtering entries {} - alreadyDeleted: {}", ledger.getName(), name, entriesRange, individualDeletedMessages); } - Range span = individualDeletedMessages.isEmpty() ? null : individualDeletedMessages.span(); + Range span = individualDeletedMessages.isEmpty() ? null : individualDeletedMessages.span(); if (span == null || !entriesRange.isConnected(span)) { // There are no individually deleted messages in this entry list, no need to perform filtering if (log.isDebugEnabled()) { @@ -2434,8 +2545,12 @@ List filterReadEntries(List entries) { @Override public synchronized String toString() { - return MoreObjects.toStringHelper(this).add("ledger", ledger.getName()).add("name", name) - .add("ackPos", markDeletePosition).add("readPos", readPosition).toString(); + return MoreObjects.toStringHelper(this) + .add("ledger", ledger.getName()) + .add("name", name) + .add("ackPos", markDeletePosition) + .add("readPos", readPosition) + .toString(); } @Override @@ -2475,10 +2590,16 @@ public Position getPersistentMarkDeletedPosition() { @Override public void rewind() { + rewind(false); + } + + @Override + public void rewind(boolean readCompacted) { lock.writeLock().lock(); try { - PositionImpl newReadPosition = ledger.getNextValidPosition(markDeletePosition); - PositionImpl oldReadPosition = readPosition; + Position newReadPosition = + readCompacted ? markDeletePosition.getNext() : ledger.getNextValidPosition(markDeletePosition); + Position oldReadPosition = readPosition; log.info("[{}-{}] Rewind from {} to {}", ledger.getName(), name, oldReadPosition, newReadPosition); @@ -2491,8 +2612,7 @@ public void rewind() { @Override public void seek(Position newReadPositionInt, boolean force) { - checkArgument(newReadPositionInt instanceof PositionImpl); - PositionImpl newReadPosition = (PositionImpl) newReadPositionInt; + Position newReadPosition = newReadPositionInt; lock.writeLock().lock(); try { @@ -2560,7 +2680,7 @@ public void closeFailed(ManagedLedgerException exception, Object ctx) { * @param callback * @param ctx */ - void persistPositionWhenClosing(PositionImpl position, Map properties, + void persistPositionWhenClosing(Position position, Map properties, final AsyncCallbacks.CloseCallback callback, final Object ctx) { if (shouldPersistUnackRangesToLedger()) { @@ -2601,13 +2721,18 @@ public void operationFailed(MetaStoreException e) { } private boolean shouldPersistUnackRangesToLedger() { - return cursorLedger != null - && !isCursorLedgerReadOnly - && config.getMaxUnackedRangesToPersist() > 0 - && individualDeletedMessages.size() > config.getMaxUnackedRangesToPersistInMetadataStore(); + lock.readLock().lock(); + try { + return cursorLedger != null + && !isCursorLedgerReadOnly + && getConfig().getMaxUnackedRangesToPersist() > 0 + && individualDeletedMessages.size() > getConfig().getMaxUnackedRangesToPersistInMetadataStore(); + } finally { + lock.readLock().unlock(); + } } - private void persistPositionMetaStore(long cursorsLedgerId, PositionImpl position, Map properties, + private void persistPositionMetaStore(long cursorsLedgerId, Position position, Map properties, MetaStoreCallback callback, boolean persistIndividualDeletedMessageRanges) { if (state == State.Closed) { ledger.getExecutor().execute(() -> callback.operationFailed(new MetaStoreException( @@ -2629,7 +2754,7 @@ private void persistPositionMetaStore(long cursorsLedgerId, PositionImpl positio info.addAllCursorProperties(buildStringPropertiesMap(cursorProperties)); if (persistIndividualDeletedMessageRanges) { info.addAllIndividualDeletedMessages(buildIndividualDeletedMessageRanges()); - if (config.isDeletionAtBatchIndexLevelEnabled()) { + if (getConfig().isDeletionAtBatchIndexLevelEnabled()) { info.addAllBatchedEntryDeletionIndexInfo(buildBatchEntryDeletionIndexInfoList()); } } @@ -2648,32 +2773,47 @@ public void operationComplete(Void result, Stat stat) { } @Override - public void operationFailed(MetaStoreException e) { - if (e instanceof MetaStoreException.BadVersionException) { + public void operationFailed(MetaStoreException topLevelException) { + if (topLevelException instanceof MetaStoreException.BadVersionException) { log.warn("[{}] Failed to update cursor metadata for {} due to version conflict {}", - ledger.name, name, e.getMessage()); + ledger.name, name, topLevelException.getMessage()); // it means previous owner of the ml might have updated the version incorrectly. So, check // the ownership and refresh the version again. - if (ledger.mlOwnershipChecker != null && ledger.mlOwnershipChecker.get()) { - ledger.getStore().asyncGetCursorInfo(ledger.getName(), name, - new MetaStoreCallback() { - @Override - public void operationComplete(ManagedCursorInfo info, Stat stat) { - updateCursorLedgerStat(info, stat); - } - - @Override - public void operationFailed(MetaStoreException e) { - if (log.isDebugEnabled()) { - log.debug( - "[{}] Failed to refresh cursor metadata-version for {} due " - + "to {}", ledger.name, name, e.getMessage()); - } - } - }); + if (ledger.mlOwnershipChecker != null) { + ledger.mlOwnershipChecker.get().whenComplete((hasOwnership, t) -> { + if (t == null && hasOwnership) { + ledger.getStore().asyncGetCursorInfo(ledger.getName(), name, + new MetaStoreCallback<>() { + @Override + public void operationComplete(ManagedCursorInfo info, Stat stat) { + updateCursorLedgerStat(info, stat); + // fail the top level call so that the caller can retry + callback.operationFailed(topLevelException); + } + + @Override + public void operationFailed(MetaStoreException e) { + if (log.isDebugEnabled()) { + log.debug( + "[{}] Failed to refresh cursor metadata-version " + + "for {} due to {}", ledger.name, name, + e.getMessage()); + } + // fail the top level call so that the caller can retry + callback.operationFailed(topLevelException); + } + }); + } else { + // fail the top level call so that the caller can retry + callback.operationFailed(topLevelException); + } + }); + } else { + callback.operationFailed(topLevelException); } + } else { + callback.operationFailed(topLevelException); } - callback.operationFailed(e); } }); } @@ -2710,14 +2850,46 @@ public void closeFailed(ManagedLedgerException exception, Object ctx) { * @param newReadPositionInt */ void setReadPosition(Position newReadPositionInt) { - checkArgument(newReadPositionInt instanceof PositionImpl); if (this.markDeletePosition == null - || ((PositionImpl) newReadPositionInt).compareTo(this.markDeletePosition) > 0) { - this.readPosition = (PositionImpl) newReadPositionInt; + || newReadPositionInt.compareTo(this.markDeletePosition) > 0) { + this.readPosition = newReadPositionInt; ledger.onCursorReadPositionUpdated(this, newReadPositionInt); } } + /** + * Manually acknowledge all entries in the lost ledger. + * - Since this is an uncommon event, we focus on maintainability. So we do not modify + * {@link #individualDeletedMessages} and {@link #batchDeletedIndexes}, but call + * {@link #asyncDelete(Position, AsyncCallbacks.DeleteCallback, Object)}. + * - This method is valid regardless of the consumer ACK type. + * - If there is a consumer ack request after this event, it will also work. + */ + @Override + public void skipNonRecoverableLedger(final long ledgerId){ + LedgerInfo ledgerInfo = ledger.getLedgersInfo().get(ledgerId); + if (ledgerInfo == null) { + return; + } + log.warn("[{}] [{}] Since the ledger [{}] is lost and the autoSkipNonRecoverableData is true, this ledger will" + + " be auto acknowledge in subscription", ledger.getName(), name, ledgerId); + asyncDelete(() -> LongStream.range(0, ledgerInfo.getEntries()) + .mapToObj(i -> (Position) PositionFactory.create(ledgerId, i)).iterator(), + new AsyncCallbacks.DeleteCallback() { + @Override + public void deleteComplete(Object ctx) { + // ignore. + } + + @Override + public void deleteFailed(ManagedLedgerException ex, Object ctx) { + // The method internalMarkDelete already handled the failure operation. We only need to + // make sure the memory state is updated. + // If the broker crashed, the non-recoverable ledger will be detected again. + } + }, null); + } + // ////////////////////////////////////////////////// void startCreatingNewMetadataLedger() { @@ -2749,16 +2921,25 @@ public void operationComplete() { @Override public void operationFailed(ManagedLedgerException exception) { - log.error("[{}][{}] Metadata ledger creation failed", ledger.getName(), name, exception); + log.error("[{}][{}] Metadata ledger creation failed {}, try to persist the position in the metadata" + + " store.", ledger.getName(), name, exception); synchronized (pendingMarkDeleteOps) { - while (!pendingMarkDeleteOps.isEmpty()) { - MarkDeleteEntry entry = pendingMarkDeleteOps.poll(); - entry.callback.markDeleteFailed(exception, entry.ctx); - } - // At this point we don't have a ledger ready STATE_UPDATER.set(ManagedCursorImpl.this, State.NoLedger); + // There are two case may cause switch ledger fails. + // 1. No enough BKs; BKs are in read-only mode... + // 2. Write ZK fails. + // Regarding the case "No enough BKs", try to persist the position in the metadata store before + // giving up. + if (!(exception instanceof MetaStoreException)) { + flushPendingMarkDeletes(); + } else { + while (!pendingMarkDeleteOps.isEmpty()) { + MarkDeleteEntry entry = pendingMarkDeleteOps.poll(); + entry.callback.markDeleteFailed(exception, entry.ctx); + } + } } } }); @@ -2837,7 +3018,7 @@ public void operationFailed(ManagedLedgerException exception) { private CompletableFuture doCreateNewMetadataLedger() { CompletableFuture future = new CompletableFuture<>(); - ledger.asyncCreateLedger(bookkeeper, config, digestType, (rc, lh, ctx) -> { + ledger.asyncCreateLedger(bookkeeper, getConfig(), digestType, (rc, lh, ctx) -> { if (ledger.checkAndCompleteLedgerOpTask(rc, lh, ctx)) { future.complete(null); @@ -2907,7 +3088,7 @@ private static List buildStringPropertiesMap(Map } private List buildIndividualDeletedMessageRanges() { - lock.readLock().lock(); + lock.writeLock().lock(); try { if (individualDeletedMessages.isEmpty()) { this.individualDeletedMessagesSerializedSize = 0; @@ -2942,21 +3123,21 @@ private List buildIndividualDeletedMessageRanges() { acksSerializedSize.addAndGet(messageRange.getSerializedSize()); rangeList.add(messageRange); - return rangeList.size() <= config.getMaxUnackedRangesToPersist(); + return rangeList.size() <= getConfig().getMaxUnackedRangesToPersist(); }); this.individualDeletedMessagesSerializedSize = acksSerializedSize.get(); individualDeletedMessages.resetDirtyKeys(); return rangeList; } finally { - lock.readLock().unlock(); + lock.writeLock().unlock(); } } private List buildBatchEntryDeletionIndexInfoList() { lock.readLock().lock(); try { - if (!config.isDeletionAtBatchIndexLevelEnabled() || batchDeletedIndexes.isEmpty()) { + if (!getConfig().isDeletionAtBatchIndexLevelEnabled() || batchDeletedIndexes.isEmpty()) { return Collections.emptyList(); } MLDataFormats.NestedPositionInfo.Builder nestedPositionBuilder = MLDataFormats.NestedPositionInfo @@ -2964,9 +3145,9 @@ private List buildBatchEntryDeletio MLDataFormats.BatchedEntryDeletionIndexInfo.Builder batchDeletedIndexInfoBuilder = MLDataFormats .BatchedEntryDeletionIndexInfo.newBuilder(); List result = new ArrayList<>(); - Iterator> iterator = batchDeletedIndexes.entrySet().iterator(); - while (iterator.hasNext() && result.size() < config.getMaxBatchDeletedIndexToPersist()) { - Map.Entry entry = iterator.next(); + Iterator> iterator = batchDeletedIndexes.entrySet().iterator(); + while (iterator.hasNext() && result.size() < getConfig().getMaxBatchDeletedIndexToPersist()) { + Map.Entry entry = iterator.next(); nestedPositionBuilder.setLedgerId(entry.getKey().getLedgerId()); nestedPositionBuilder.setEntryId(entry.getKey().getEntryId()); batchDeletedIndexInfoBuilder.setPosition(nestedPositionBuilder.build()); @@ -2986,13 +3167,24 @@ private List buildBatchEntryDeletio } void persistPositionToLedger(final LedgerHandle lh, MarkDeleteEntry mdEntry, final VoidCallback callback) { - PositionImpl position = mdEntry.newPosition; - PositionInfo pi = PositionInfo.newBuilder().setLedgerId(position.getLedgerId()) + Position position = mdEntry.newPosition; + Builder piBuilder = PositionInfo.newBuilder().setLedgerId(position.getLedgerId()) .setEntryId(position.getEntryId()) - .addAllIndividualDeletedMessages(buildIndividualDeletedMessageRanges()) .addAllBatchedEntryDeletionIndexInfo(buildBatchEntryDeletionIndexInfoList()) - .addAllProperties(buildPropertiesMap(mdEntry.properties)).build(); + .addAllProperties(buildPropertiesMap(mdEntry.properties)); + Map internalRanges = null; + try { + internalRanges = individualDeletedMessages.toRanges(getConfig().getMaxUnackedRangesToPersist()); + } catch (Exception e) { + log.warn("[{}]-{} Failed to serialize individualDeletedMessages", ledger.getName(), name, e); + } + if (internalRanges != null && !internalRanges.isEmpty()) { + piBuilder.addAllIndividualDeletedMessageRanges(buildLongPropertiesMap(internalRanges)); + } else { + piBuilder.addAllIndividualDeletedMessages(buildIndividualDeletedMessageRanges()); + } + PositionInfo pi = piBuilder.build(); if (log.isDebugEnabled()) { log.debug("[{}] Cursor {} Appending to ledger={} position={}", ledger.getName(), name, lh.getId(), @@ -3008,12 +3200,7 @@ void persistPositionToLedger(final LedgerHandle lh, MarkDeleteEntry mdEntry, fin lh1.getId()); } - if (shouldCloseLedger(lh1)) { - if (log.isDebugEnabled()) { - log.debug("[{}] Need to create new metadata ledger for consumer {}", ledger.getName(), name); - } - startCreatingNewMetadataLedger(); - } + rolloverLedgerIfNeeded(lh1); mbean.persistToLedger(true); mbean.addWriteCursorLedgerSize(data.length); @@ -3025,37 +3212,73 @@ void persistPositionToLedger(final LedgerHandle lh, MarkDeleteEntry mdEntry, fin // in the meantime the mark-delete will be queued. STATE_UPDATER.compareAndSet(ManagedCursorImpl.this, State.Open, State.NoLedger); - mbean.persistToLedger(false); - // Before giving up, try to persist the position in the metadata store - persistPositionMetaStore(-1, position, mdEntry.properties, new MetaStoreCallback() { - @Override - public void operationComplete(Void result, Stat stat) { - if (log.isDebugEnabled()) { - log.debug( - "[{}][{}] Updated cursor in meta store after previous failure in ledger at position" - + " {}", ledger.getName(), name, position); - } - mbean.persistToZookeeper(true); - callback.operationComplete(); - } - - @Override - public void operationFailed(MetaStoreException e) { - log.warn("[{}][{}] Failed to update cursor in meta store after previous failure in ledger: {}", - ledger.getName(), name, e.getMessage()); - mbean.persistToZookeeper(false); - callback.operationFailed(createManagedLedgerException(rc)); - } - }, true); + // Before giving up, try to persist the position in the metadata store. + persistPositionToMetaStore(mdEntry, callback); } }, null); } + public boolean periodicRollover() { + LedgerHandle lh = cursorLedger; + if (State.Open.equals(STATE_UPDATER.get(this)) + && lh != null && lh.getLength() > 0) { + boolean triggered = rolloverLedgerIfNeeded(lh); + if (triggered) { + log.info("[{}] Periodic rollover triggered for cursor {} (length={} bytes)", + ledger.getName(), name, lh.getLength()); + } else { + log.debug("[{}] Periodic rollover skipped for cursor {} (length={} bytes)", + ledger.getName(), name, lh.getLength()); + + } + return triggered; + } + return false; + } + + boolean rolloverLedgerIfNeeded(LedgerHandle lh1) { + if (shouldCloseLedger(lh1)) { + if (log.isDebugEnabled()) { + log.debug("[{}] Need to create new metadata ledger for cursor {}", ledger.getName(), name); + } + startCreatingNewMetadataLedger(); + return true; + } + return false; + } + + void persistPositionToMetaStore(MarkDeleteEntry mdEntry, final VoidCallback callback) { + final Position newPosition = mdEntry.newPosition; + STATE_UPDATER.compareAndSet(ManagedCursorImpl.this, State.Open, State.NoLedger); + mbean.persistToLedger(false); + // Before giving up, try to persist the position in the metadata store + persistPositionMetaStore(-1, newPosition, mdEntry.properties, new MetaStoreCallback() { + @Override + public void operationComplete(Void result, Stat stat) { + if (log.isDebugEnabled()) { + log.debug( + "[{}][{}] Updated cursor in meta store after previous failure in ledger at position" + + " {}", ledger.getName(), name, newPosition); + } + mbean.persistToZookeeper(true); + callback.operationComplete(); + } + + @Override + public void operationFailed(MetaStoreException e) { + log.warn("[{}][{}] Failed to update cursor in meta store after previous failure in ledger: {}", + ledger.getName(), name, e.getMessage()); + mbean.persistToZookeeper(false); + callback.operationFailed(createManagedLedgerException(e)); + } + }, true); + } + boolean shouldCloseLedger(LedgerHandle lh) { long now = clock.millis(); if (ledger.getFactory().isMetadataServiceAvailable() - && (lh.getLastAddConfirmed() >= config.getMetadataMaxEntriesPerLedger() - || lastLedgerSwitchTimestamp < (now - config.getLedgerRolloverTimeout() * 1000)) + && (lh.getLastAddConfirmed() >= getConfig().getMetadataMaxEntriesPerLedger() + || lastLedgerSwitchTimestamp < (now - getConfig().getLedgerRolloverTimeout() * 1000)) && (STATE_UPDATER.get(this) != State.Closed && STATE_UPDATER.get(this) != State.Closing)) { // It's safe to modify the timestamp since this method will be only called from a callback, implying that // calls will be serialized on one single thread @@ -3088,7 +3311,7 @@ public void operationComplete(Void result, Stat stat) { @Override public void operationFailed(MetaStoreException e) { - log.warn("[{}] Failed to update consumer {}", ledger.getName(), name, e); + log.warn("[{}] Failed to update cursor metadata {}", ledger.getName(), name, e); // it means it failed to switch the newly created ledger so, it should be // deleted to prevent leak deleteLedgerAsync(lh).thenRun(() -> callback.operationFailed(e)); @@ -3115,7 +3338,7 @@ void notifyEntriesAvailable() { } PENDING_READ_OPS_UPDATER.incrementAndGet(this); - opReadEntry.readPosition = (PositionImpl) getReadPosition(); + opReadEntry.readPosition = getReadPosition(); ledger.asyncReadEntries(opReadEntry); } else { // No one is waiting to be notified. Ignore @@ -3255,9 +3478,10 @@ public static boolean isBkErrorNotRecoverable(int rc) { * * @param info */ - private PositionImpl getRollbackPosition(ManagedCursorInfo info) { - PositionImpl firstPosition = ledger.getFirstPosition(); - PositionImpl snapshottedPosition = new PositionImpl(info.getMarkDeleteLedgerId(), info.getMarkDeleteEntryId()); + private Position getRollbackPosition(ManagedCursorInfo info) { + Position firstPosition = ledger.getFirstPosition(); + Position snapshottedPosition = + PositionFactory.create(info.getMarkDeleteLedgerId(), info.getMarkDeleteEntryId()); if (firstPosition == null) { // There are no ledgers in the ML, any position is good return snapshottedPosition; @@ -3298,23 +3522,37 @@ public String getIndividuallyDeletedMessages() { } @VisibleForTesting - public LongPairRangeSet getIndividuallyDeletedMessagesSet() { + public LongPairRangeSet getIndividuallyDeletedMessagesSet() { return individualDeletedMessages; } + public Position processIndividuallyDeletedMessagesAndGetMarkDeletedPosition( + LongPairRangeSet.RangeProcessor processor) { + final Position mdp; + lock.readLock().lock(); + try { + mdp = markDeletePosition; + individualDeletedMessages.forEach(processor); + } finally { + lock.readLock().unlock(); + } + return mdp; + } + + @Override public boolean isMessageDeleted(Position position) { - checkArgument(position instanceof PositionImpl); - return individualDeletedMessages.contains(((PositionImpl) position).getLedgerId(), - ((PositionImpl) position).getEntryId()) - || ((PositionImpl) position).compareTo(markDeletePosition) <= 0; + lock.readLock().lock(); + try { + return position.compareTo(markDeletePosition) <= 0 + || individualDeletedMessages.contains(position.getLedgerId(), position.getEntryId()); + } finally { + lock.readLock().unlock(); + } } //this method will return a copy of the position's ack set + @Override public long[] getBatchPositionAckSet(Position position) { - if (!(position instanceof PositionImpl)) { - return null; - } - if (batchDeletedIndexes != null) { BitSetRecyclable bitSetRecyclable = batchDeletedIndexes.get(position); if (bitSetRecyclable == null) { @@ -3334,19 +3572,25 @@ public long[] getBatchPositionAckSet(Position position) { * @param position * @return next available position */ - public PositionImpl getNextAvailablePosition(PositionImpl position) { - Range range = individualDeletedMessages.rangeContaining(position.getLedgerId(), - position.getEntryId()); - if (range != null) { - PositionImpl nextPosition = range.upperEndpoint().getNext(); - return (nextPosition != null && nextPosition.compareTo(position) > 0) ? nextPosition : position.getNext(); + public Position getNextAvailablePosition(Position position) { + lock.readLock().lock(); + try { + Range range = individualDeletedMessages.rangeContaining(position.getLedgerId(), + position.getEntryId()); + if (range != null) { + Position nextPosition = range.upperEndpoint().getNext(); + return (nextPosition != null && nextPosition.compareTo(position) > 0) + ? nextPosition : position.getNext(); + } + return position.getNext(); + } finally { + lock.readLock().unlock(); } - return position.getNext(); } public Position getNextLedgerPosition(long currentLedgerId) { Long nextExistingLedger = ledger.getNextValidLedger(currentLedgerId); - return nextExistingLedger != null ? PositionImpl.get(nextExistingLedger, 0) : null; + return nextExistingLedger != null ? PositionFactory.create(nextExistingLedger, 0) : null; } public boolean isIndividuallyDeletedEntriesEmpty() { @@ -3391,15 +3635,19 @@ public ManagedLedger getManagedLedger() { } @Override - public Range getLastIndividualDeletedRange() { - return individualDeletedMessages.lastRange(); + public Range getLastIndividualDeletedRange() { + lock.readLock().lock(); + try { + return individualDeletedMessages.lastRange(); + } finally { + lock.readLock().unlock(); + } } @Override public void trimDeletedEntries(List entries) { entries.removeIf(entry -> { - boolean isDeleted = ((PositionImpl) entry.getPosition()).compareTo(markDeletePosition) <= 0 - || individualDeletedMessages.contains(entry.getLedgerId(), entry.getEntryId()); + boolean isDeleted = isMessageDeleted(entry.getPosition()); if (isDeleted) { entry.release(); } @@ -3412,8 +3660,8 @@ private ManagedCursorImpl cursorImpl() { } @Override - public long[] getDeletedBatchIndexesAsLongArray(PositionImpl position) { - if (config.isDeletionAtBatchIndexLevelEnabled()) { + public long[] getDeletedBatchIndexesAsLongArray(Position position) { + if (getConfig().isDeletionAtBatchIndexLevelEnabled()) { BitSetRecyclable bitSet = batchDeletedIndexes.get(position); return bitSet == null ? null : bitSet.toLongArray(); } else { @@ -3426,7 +3674,8 @@ public ManagedCursorMXBean getStats() { return this.mbean; } - void updateReadStats(int readEntriesCount, long readEntriesSize) { + @Override + public void updateReadStats(int readEntriesCount, long readEntriesSize) { this.entriesReadCount += readEntriesCount; this.entriesReadSize += readEntriesSize; } @@ -3458,7 +3707,8 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { }, null); } - private int applyMaxSizeCap(int maxEntries, long maxSizeBytes) { + @Override + public int applyMaxSizeCap(int maxEntries, long maxSizeBytes) { if (maxSizeBytes == NO_MAX_SIZE_LIMIT) { return maxEntries; } @@ -3486,7 +3736,7 @@ private int applyMaxSizeCap(int maxEntries, long maxSizeBytes) { @Override public boolean checkAndUpdateReadPositionChanged() { - PositionImpl lastEntry = ledger.lastConfirmedEntry; + Position lastEntry = ledger.lastConfirmedEntry; boolean isReadPositionOnTail = lastEntry == null || readPosition == null || (lastEntry.compareTo(readPosition) <= 0); boolean isReadPositionChanged = readPosition != null && !readPosition.equals(statsLastReadPosition); @@ -3514,6 +3764,65 @@ public boolean isCacheReadEntry() { private static final Logger log = LoggerFactory.getLogger(ManagedCursorImpl.class); public ManagedLedgerConfig getConfig() { - return config; + return getManagedLedger().getConfig(); + } + + /*** + * Create a non-durable cursor and copy the ack stats. + */ + @Override + public ManagedCursor duplicateNonDurableCursor(String nonDurableCursorName) throws ManagedLedgerException { + NonDurableCursorImpl newNonDurableCursor = + (NonDurableCursorImpl) ledger.newNonDurableCursor(getMarkDeletedPosition(), nonDurableCursorName); + lock.readLock().lock(); + try { + if (individualDeletedMessages != null) { + this.individualDeletedMessages.forEach(range -> { + newNonDurableCursor.individualDeletedMessages.addOpenClosed( + range.lowerEndpoint().getLedgerId(), + range.lowerEndpoint().getEntryId(), + range.upperEndpoint().getLedgerId(), + range.upperEndpoint().getEntryId()); + return true; + }); + } + } finally { + lock.readLock().unlock(); + } + if (batchDeletedIndexes != null) { + for (Map.Entry entry : this.batchDeletedIndexes.entrySet()) { + BitSetRecyclable copiedBitSet = BitSetRecyclable.valueOf(entry.getValue()); + newNonDurableCursor.batchDeletedIndexes.put(entry.getKey(), copiedBitSet); + } + } + return newNonDurableCursor; + } + + @Override + public ManagedCursorAttributes getManagedCursorAttributes() { + if (managedCursorAttributes != null) { + return managedCursorAttributes; + } + return ATTRIBUTES_UPDATER.updateAndGet(this, old -> old != null ? old : new ManagedCursorAttributes(this)); + } + + @Override + public ManagedLedgerInternalStats.CursorStats getCursorStats() { + ManagedLedgerInternalStats.CursorStats cs = new ManagedLedgerInternalStats.CursorStats(); + cs.markDeletePosition = getMarkDeletedPosition().toString(); + cs.readPosition = getReadPosition().toString(); + cs.waitingReadOp = hasPendingReadRequest(); + cs.pendingReadOps = getPendingReadOpsCount(); + cs.messagesConsumedCounter = getMessagesConsumedCounter(); + cs.cursorLedger = getCursorLedger(); + cs.cursorLedgerLastEntry = getCursorLedgerLastEntry(); + cs.individuallyDeletedMessages = getIndividuallyDeletedMessages(); + cs.lastLedgerSwitchTimestamp = DateFormatter.format(getLastLedgerSwitchTimestamp()); + cs.state = getState(); + cs.active = isActive(); + cs.numberOfEntriesSinceFirstNotAckedMessage = getNumberOfEntriesSinceFirstNotAckedMessage(); + cs.totalNonContiguousDeletedMessagesRange = getTotalNonContiguousDeletedMessagesRange(); + cs.properties = getProperties(); + return cs; } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorMXBeanImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorMXBeanImpl.java index 48465e6294b0e..a183c0d61ce16 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorMXBeanImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorMXBeanImpl.java @@ -90,7 +90,8 @@ public long getPersistZookeeperErrors() { @Override public void addWriteCursorLedgerSize(final long size) { - writeCursorLedgerSize.add(size * ((ManagedCursorImpl) managedCursor).config.getWriteQuorumSize()); + writeCursorLedgerSize.add( + size * managedCursor.getManagedLedger().getConfig().getWriteQuorumSize()); writeCursorLedgerLogicalSize.add(size); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java index 9f3fe9bb0c4a7..f546a487f84be 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java @@ -18,17 +18,23 @@ */ package org.apache.bookkeeper.mledger.impl; -import static com.google.common.base.Preconditions.checkArgument; import static org.apache.bookkeeper.mledger.ManagedLedgerException.getManagedLedgerException; +import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.NULL_OFFLOAD_PROMISE; import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.base.Predicates; +import com.google.common.collect.BoundType; import com.google.common.collect.Maps; +import com.google.common.collect.Range; +import com.google.protobuf.InvalidProtocolBufferException; import io.netty.util.concurrent.DefaultThreadFactory; +import io.opentelemetry.api.OpenTelemetry; import java.io.IOException; import java.util.ArrayList; +import java.util.Enumeration; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.NavigableMap; import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -36,6 +42,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; @@ -43,8 +50,12 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import lombok.Getter; +import org.apache.bookkeeper.client.AsyncCallback; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.BookKeeperAdmin; +import org.apache.bookkeeper.client.LedgerEntry; +import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.mledger.AsyncCallbacks; @@ -65,8 +76,13 @@ import org.apache.bookkeeper.mledger.ManagedLedgerInfo.LedgerInfo; import org.apache.bookkeeper.mledger.ManagedLedgerInfo.MessageRangeInfo; import org.apache.bookkeeper.mledger.ManagedLedgerInfo.PositionInfo; +import org.apache.bookkeeper.mledger.MetadataCompressionConfig; +import org.apache.bookkeeper.mledger.OpenTelemetryManagedLedgerCacheStats; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.ReadOnlyCursor; +import org.apache.bookkeeper.mledger.ReadOnlyManagedLedger; +import org.apache.bookkeeper.mledger.ReadOnlyManagedLedgerImplWrapper; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.ManagedLedgerInitializeLedgerCallback; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.State; import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback; @@ -77,11 +93,14 @@ import org.apache.bookkeeper.mledger.proto.MLDataFormats.LongProperty; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo; import org.apache.bookkeeper.mledger.proto.MLDataFormats.MessageRange; +import org.apache.bookkeeper.mledger.util.Errors; import org.apache.bookkeeper.mledger.util.Futures; import org.apache.bookkeeper.stats.NullStatsLogger; import org.apache.bookkeeper.stats.StatsLogger; import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.EnsemblePlacementPolicyConfig; +import org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.Runnables; @@ -116,6 +135,10 @@ public class ManagedLedgerFactoryImpl implements ManagedLedgerFactory { private volatile long cacheEvictionTimeThresholdNanos; private final MetadataStore metadataStore; + private final OpenTelemetryManagedLedgerCacheStats openTelemetryCacheStats; + private final OpenTelemetryManagedLedgerStats openTelemetryManagedLedgerStats; + private final OpenTelemetryManagedCursorStats openTelemetryManagedCursorStats; + //indicate whether shutdown() is called. private volatile boolean closed; @@ -147,7 +170,7 @@ public ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore, ClientConfi ManagedLedgerFactoryConfig config) throws Exception { this(metadataStore, new DefaultBkFactory(bkClientConfiguration), - true /* isBookkeeperManaged */, config, NullStatsLogger.INSTANCE); + true /* isBookkeeperManaged */, config, NullStatsLogger.INSTANCE, OpenTelemetry.noop()); } public ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore, BookKeeper bookKeeper) @@ -158,7 +181,7 @@ public ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore, BookKeeper public ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore, BookKeeper bookKeeper, ManagedLedgerFactoryConfig config) throws Exception { - this(metadataStore, (policyConfig) -> bookKeeper, config); + this(metadataStore, (policyConfig) -> CompletableFuture.completedFuture(bookKeeper), config); } public ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore, @@ -166,21 +189,28 @@ public ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore, ManagedLedgerFactoryConfig config) throws Exception { this(metadataStore, bookKeeperGroupFactory, false /* isBookkeeperManaged */, - config, NullStatsLogger.INSTANCE); + config, NullStatsLogger.INSTANCE, OpenTelemetry.noop()); } public ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore, BookkeeperFactoryForCustomEnsemblePlacementPolicy bookKeeperGroupFactory, - ManagedLedgerFactoryConfig config, StatsLogger statsLogger) + ManagedLedgerFactoryConfig config, StatsLogger statsLogger, + OpenTelemetry openTelemetry) throws Exception { this(metadataStore, bookKeeperGroupFactory, false /* isBookkeeperManaged */, - config, statsLogger); + config, statsLogger, openTelemetry); } private ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore, BookkeeperFactoryForCustomEnsemblePlacementPolicy bookKeeperGroupFactory, boolean isBookkeeperManaged, - ManagedLedgerFactoryConfig config, StatsLogger statsLogger) throws Exception { + ManagedLedgerFactoryConfig config, + StatsLogger statsLogger, + OpenTelemetry openTelemetry) throws Exception { + MetadataCompressionConfig compressionConfigForManagedLedgerInfo = + config.getCompressionConfigForManagedLedgerInfo(); + MetadataCompressionConfig compressionConfigForManagedCursorInfo = + config.getCompressionConfigForManagedCursorInfo(); scheduledExecutor = OrderedScheduler.newSchedulerBuilder() .numThreads(config.getNumManagedLedgerSchedulerThreads()) .statsLogger(statsLogger) @@ -193,11 +223,12 @@ private ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore, this.bookkeeperFactory = bookKeeperGroupFactory; this.isBookkeeperManaged = isBookkeeperManaged; this.metadataStore = metadataStore; - this.store = new MetaStoreImpl(metadataStore, scheduledExecutor, config.getManagedLedgerInfoCompressionType(), - config.getManagedCursorInfoCompressionType()); + this.store = new MetaStoreImpl(metadataStore, scheduledExecutor, + compressionConfigForManagedLedgerInfo, + compressionConfigForManagedCursorInfo); this.config = config; this.mbean = new ManagedLedgerFactoryMBeanImpl(this); - this.entryCacheManager = new RangeEntryCacheManagerImpl(this); + this.entryCacheManager = new RangeEntryCacheManagerImpl(this, openTelemetry); this.statsTask = scheduledExecutor.scheduleWithFixedDelay(catchingAndLoggingThrowables(this::refreshStats), 0, config.getStatsPeriodSeconds(), TimeUnit.SECONDS); this.flushCursorsTask = scheduledExecutor.scheduleAtFixedRate(catchingAndLoggingThrowables(this::flushCursors), @@ -213,6 +244,10 @@ private ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore, closed = false; metadataStore.registerSessionListener(this::handleMetadataStoreNotification); + + openTelemetryCacheStats = new OpenTelemetryManagedLedgerCacheStats(openTelemetry, this); + openTelemetryManagedLedgerStats = new OpenTelemetryManagedLedgerStats(openTelemetry, this); + openTelemetryManagedCursorStats = new OpenTelemetryManagedCursorStats(openTelemetry, this); } static class DefaultBkFactory implements BookkeeperFactoryForCustomEnsemblePlacementPolicy { @@ -225,8 +260,8 @@ public DefaultBkFactory(ClientConfiguration bkClientConfiguration) } @Override - public BookKeeper get(EnsemblePlacementPolicyConfig policy) { - return bkClient; + public CompletableFuture get(EnsemblePlacementPolicyConfig policy) { + return CompletableFuture.completedFuture(bkClient); } } @@ -281,7 +316,8 @@ private synchronized void doCacheEviction() { * * @return */ - public Map getManagedLedgers() { + @Override + public Map getManagedLedgers() { // Return a view of already created ledger by filtering futures not yet completed return Maps.filterValues(Maps.transformValues(ledgers, future -> future.getNow(null)), Predicates.notNull()); } @@ -329,7 +365,7 @@ public void asyncOpen(String name, OpenLedgerCallback callback, Object ctx) { @Override public void asyncOpen(final String name, final ManagedLedgerConfig config, final OpenLedgerCallback callback, - Supplier mlOwnershipChecker, final Object ctx) { + Supplier> mlOwnershipChecker, final Object ctx) { if (closed) { callback.openLedgerFailed(new ManagedLedgerException.ManagedLedgerFactoryClosedException(), ctx); return; @@ -370,55 +406,68 @@ public void asyncOpen(final String name, final ManagedLedgerConfig config, final ledgers.computeIfAbsent(name, (mlName) -> { // Create the managed ledger CompletableFuture future = new CompletableFuture<>(); - BookKeeper bk = bookkeeperFactory.get( - new EnsemblePlacementPolicyConfig(config.getBookKeeperEnsemblePlacementPolicyClassName(), - config.getBookKeeperEnsemblePlacementPolicyProperties())); - final ManagedLedgerImpl newledger = config.getShadowSource() == null - ? new ManagedLedgerImpl(this, bk, store, config, scheduledExecutor, name, mlOwnershipChecker) - : new ShadowManagedLedgerImpl(this, bk, store, config, scheduledExecutor, name, - mlOwnershipChecker); - PendingInitializeManagedLedger pendingLedger = new PendingInitializeManagedLedger(newledger); - pendingInitializeLedgers.put(name, pendingLedger); - newledger.initialize(new ManagedLedgerInitializeLedgerCallback() { - @Override - public void initializeComplete() { - log.info("[{}] Successfully initialize managed ledger", name); - pendingInitializeLedgers.remove(name, pendingLedger); - future.complete(newledger); + bookkeeperFactory.get( + new EnsemblePlacementPolicyConfig(config.getBookKeeperEnsemblePlacementPolicyClassName(), + config.getBookKeeperEnsemblePlacementPolicyProperties())) + .thenAccept(bk -> { + final ManagedLedgerImpl newledger = config.getShadowSource() == null + ? new ManagedLedgerImpl(this, bk, store, config, scheduledExecutor, name, + mlOwnershipChecker) + : new ShadowManagedLedgerImpl(this, bk, store, config, scheduledExecutor, name, + mlOwnershipChecker); + PendingInitializeManagedLedger pendingLedger = new PendingInitializeManagedLedger(newledger); + pendingInitializeLedgers.put(name, pendingLedger); + newledger.initialize(new ManagedLedgerInitializeLedgerCallback() { + @Override + public void initializeComplete() { + log.info("[{}] Successfully initialize managed ledger", name); + pendingInitializeLedgers.remove(name, pendingLedger); + future.complete(newledger); + + // May need to update the cursor position + newledger.maybeUpdateCursorBeforeTrimmingConsumedLedger(); + // May need to trigger offloading + if (config.isTriggerOffloadOnTopicLoad()) { + newledger.maybeOffloadInBackground(NULL_OFFLOAD_PROMISE); + } + } - // May need to update the cursor position - newledger.maybeUpdateCursorBeforeTrimmingConsumedLedger(); - } + @Override + public void initializeFailed(ManagedLedgerException e) { + if (config.isCreateIfMissing()) { + log.error("[{}] Failed to initialize managed ledger: {}", name, e.getMessage()); + } - @Override - public void initializeFailed(ManagedLedgerException e) { - if (config.isCreateIfMissing()) { - log.error("[{}] Failed to initialize managed ledger: {}", name, e.getMessage()); - } + // Clean the map if initialization fails + ledgers.remove(name, future); + entryCacheManager.removeEntryCache(name); - // Clean the map if initialization fails - ledgers.remove(name, future); + if (pendingInitializeLedgers.remove(name, pendingLedger)) { + pendingLedger.ledger.asyncClose(new CloseCallback() { + @Override + public void closeComplete(Object ctx) { + // no-op + } - if (pendingInitializeLedgers.remove(name, pendingLedger)) { - pendingLedger.ledger.asyncClose(new CloseCallback() { - @Override - public void closeComplete(Object ctx) { - // no-op - } + @Override + public void closeFailed(ManagedLedgerException exception, Object ctx) { + log.warn("[{}] Failed to a pending initialization managed ledger", name, + exception); + } + }, null); + } - @Override - public void closeFailed(ManagedLedgerException exception, Object ctx) { - log.warn("[{}] Failed to a pending initialization managed ledger", name, exception); + future.completeExceptionally(e); } }, null); - } - - future.completeExceptionally(e); - } - }, null); + }).exceptionally(ex -> { + future.completeExceptionally(ex); + return null; + }); return future; }).thenAccept(ml -> callback.openLedgerComplete(ml, ctx)).exceptionally(exception -> { - callback.openLedgerFailed((ManagedLedgerException) exception.getCause(), ctx); + callback.openLedgerFailed(ManagedLedgerException + .getManagedLedgerException(FutureUtil.unwrapCompletionException(exception)), ctx); return null; }); } @@ -431,20 +480,23 @@ public void asyncOpenReadOnlyManagedLedger(String managedLedgerName, callback.openReadOnlyManagedLedgerFailed( new ManagedLedgerException.ManagedLedgerFactoryClosedException(), ctx); } - ReadOnlyManagedLedgerImpl roManagedLedger = new ReadOnlyManagedLedgerImpl(this, - bookkeeperFactory - .get(new EnsemblePlacementPolicyConfig(config.getBookKeeperEnsemblePlacementPolicyClassName(), - config.getBookKeeperEnsemblePlacementPolicyProperties())), - store, config, scheduledExecutor, managedLedgerName); - roManagedLedger.initialize().thenRun(() -> { - log.info("[{}] Successfully initialize Read-only managed ledger", managedLedgerName); - callback.openReadOnlyManagedLedgerComplete(roManagedLedger, ctx); - - }).exceptionally(e -> { - log.error("[{}] Failed to initialize Read-only managed ledger", managedLedgerName, e); - callback.openReadOnlyManagedLedgerFailed((ManagedLedgerException) e.getCause(), ctx); - return null; - }); + + bookkeeperFactory + .get(new EnsemblePlacementPolicyConfig(config.getBookKeeperEnsemblePlacementPolicyClassName(), + config.getBookKeeperEnsemblePlacementPolicyProperties())) + .thenCompose(bk -> { + ReadOnlyManagedLedgerImplWrapper roManagedLedger = new ReadOnlyManagedLedgerImplWrapper(this, bk, + store, config, scheduledExecutor, managedLedgerName); + return roManagedLedger.initialize().thenApply(v -> roManagedLedger); + }).thenAccept(roManagedLedger -> { + log.info("[{}] Successfully initialize Read-only managed ledger", managedLedgerName); + callback.openReadOnlyManagedLedgerComplete(roManagedLedger, ctx); + }).exceptionally(e -> { + log.error("[{}] Failed to initialize Read-only managed ledger", managedLedgerName, e); + callback.openReadOnlyManagedLedgerFailed(ManagedLedgerException + .getManagedLedgerException(FutureUtil.unwrapCompletionException(e)), ctx); + return null; + }); } @Override @@ -486,13 +538,12 @@ public void asyncOpenReadOnlyCursor(String managedLedgerName, Position startPosi callback.openReadOnlyCursorFailed(new ManagedLedgerException.ManagedLedgerFactoryClosedException(), ctx); return; } - checkArgument(startPosition instanceof PositionImpl); AsyncCallbacks.OpenReadOnlyManagedLedgerCallback openReadOnlyManagedLedgerCallback = new AsyncCallbacks.OpenReadOnlyManagedLedgerCallback() { @Override - public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl readOnlyManagedLedger, Object ctx) { + public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedger readOnlyManagedLedger, Object ctx) { callback.openReadOnlyCursorComplete(readOnlyManagedLedger. - createReadOnlyCursor((PositionImpl) startPosition), ctx); + createReadOnlyCursor(startPosition), ctx); } @Override @@ -504,9 +555,19 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob } void close(ManagedLedger ledger) { - // Remove the ledger from the internal factory cache - ledgers.remove(ledger.getName()); - entryCacheManager.removeEntryCache(ledger.getName()); + // If the future in map is not done or has exceptionally complete, it means that @param-ledger is not in the + // map. + CompletableFuture ledgerFuture = ledgers.get(ledger.getName()); + if (ledgerFuture == null || !ledgerFuture.isDone() || ledgerFuture.isCompletedExceptionally()){ + return; + } + if (ledgerFuture.join() != ledger){ + return; + } + // Remove the ledger from the internal factory cache. + if (ledgers.remove(ledger.getName(), ledgerFuture)) { + entryCacheManager.removeEntryCache(ledger.getName()); + } } public CompletableFuture shutdownAsync() throws ManagedLedgerException { @@ -524,13 +585,12 @@ public CompletableFuture shutdownAsync() throws ManagedLedgerException { int numLedgers = ledgerNames.size(); log.info("Closing {} ledgers", numLedgers); for (String ledgerName : ledgerNames) { - CompletableFuture future = new CompletableFuture<>(); - futures.add(future); CompletableFuture ledgerFuture = ledgers.remove(ledgerName); if (ledgerFuture == null) { - future.complete(null); continue; } + CompletableFuture future = new CompletableFuture<>(); + futures.add(future); ledgerFuture.whenCompleteAsync((managedLedger, throwable) -> { if (throwable != null || managedLedger == null) { future.complete(null); @@ -557,106 +617,47 @@ public void closeFailed(ManagedLedgerException exception, Object ctx) { ledgerFuture.completeExceptionally(new ManagedLedgerException.ManagedLedgerFactoryClosedException()); } } - CompletableFuture bookkeeperFuture = new CompletableFuture<>(); - futures.add(bookkeeperFuture); - futures.add(CompletableFuture.runAsync(() -> { - if (isBookkeeperManaged) { - try { - BookKeeper bookkeeper = bookkeeperFactory.get(); - if (bookkeeper != null) { - bookkeeper.close(); - } - bookkeeperFuture.complete(null); - } catch (Throwable throwable) { - bookkeeperFuture.completeExceptionally(throwable); - } - } else { - bookkeeperFuture.complete(null); - } - if (!ledgers.isEmpty()) { - log.info("Force closing {} ledgers.", ledgers.size()); - //make sure all callbacks is called. - ledgers.forEach(((ledgerName, ledgerFuture) -> { - if (!ledgerFuture.isDone()) { - ledgerFuture.completeExceptionally( - new ManagedLedgerException.ManagedLedgerFactoryClosedException()); - } else { - ManagedLedgerImpl managedLedger = ledgerFuture.getNow(null); - if (managedLedger == null) { - return; - } - try { - managedLedger.close(); - } catch (Throwable throwable) { - log.warn("[{}] Got exception when closing managed ledger: {}", managedLedger.getName(), - throwable); + CompletableFuture bookkeeperFuture = isBookkeeperManaged + ? bookkeeperFactory.get() + : CompletableFuture.completedFuture(null); + return bookkeeperFuture + .thenRun(() -> { + log.info("Closing {} ledgers.", ledgers.size()); + //make sure all callbacks is called. + ledgers.forEach(((ledgerName, ledgerFuture) -> { + if (!ledgerFuture.isDone()) { + ledgerFuture.completeExceptionally( + new ManagedLedgerException.ManagedLedgerFactoryClosedException()); + } else { + ManagedLedgerImpl managedLedger = ledgerFuture.getNow(null); + if (managedLedger == null) { + return; + } + try { + managedLedger.close(); + } catch (Throwable throwable) { + log.warn("[{}] Got exception when closing managed ledger: {}", managedLedger.getName(), + throwable); + } } - } - })); - } - })); - entryCacheManager.clear(); - return FutureUtil.waitForAll(futures).thenAccept(__ -> { - //wait for tasks in scheduledExecutor executed. - scheduledExecutor.shutdown(); - }); + })); + }).thenAcceptAsync(__ -> { + //wait for tasks in scheduledExecutor executed. + openTelemetryManagedCursorStats.close(); + openTelemetryManagedLedgerStats.close(); + openTelemetryCacheStats.close(); + scheduledExecutor.shutdownNow(); + entryCacheManager.clear(); + }); } @Override public void shutdown() throws InterruptedException, ManagedLedgerException { - if (closed) { - throw new ManagedLedgerException.ManagedLedgerFactoryClosedException(); + try { + shutdownAsync().get(); + } catch (ExecutionException e) { + throw getManagedLedgerException(e.getCause()); } - closed = true; - - statsTask.cancel(true); - flushCursorsTask.cancel(true); - cacheEvictionExecutor.shutdownNow(); - - // take a snapshot of ledgers currently in the map to prevent race conditions - List> ledgers = new ArrayList<>(this.ledgers.values()); - int numLedgers = ledgers.size(); - final CountDownLatch latch = new CountDownLatch(numLedgers); - log.info("Closing {} ledgers", numLedgers); - - for (CompletableFuture ledgerFuture : ledgers) { - ManagedLedgerImpl ledger = ledgerFuture.getNow(null); - if (ledger == null) { - latch.countDown(); - continue; - } - - ledger.asyncClose(new AsyncCallbacks.CloseCallback() { - @Override - public void closeComplete(Object ctx) { - latch.countDown(); - } - - @Override - public void closeFailed(ManagedLedgerException exception, Object ctx) { - log.warn("[{}] Got exception when closing managed ledger: {}", ledger.getName(), exception); - latch.countDown(); - } - }, null); - } - - latch.await(); - log.info("{} ledgers closed", numLedgers); - - if (isBookkeeperManaged) { - try { - BookKeeper bookkeeper = bookkeeperFactory.get(); - if (bookkeeper != null) { - bookkeeper.close(); - } - } catch (BKException e) { - throw new ManagedLedgerException(e); - } - } - - scheduledExecutor.shutdownNow(); - - entryCacheManager.clear(); } @Override @@ -726,6 +727,7 @@ public void operationComplete(MLDataFormats.ManagedLedgerInfo pbInfo, Stat stat) ledgerInfo.ledgerId = pbLedgerInfo.getLedgerId(); ledgerInfo.entries = pbLedgerInfo.hasEntries() ? pbLedgerInfo.getEntries() : null; ledgerInfo.size = pbLedgerInfo.hasSize() ? pbLedgerInfo.getSize() : null; + ledgerInfo.timestamp = pbLedgerInfo.hasTimestamp() ? pbLedgerInfo.getTimestamp() : null; ledgerInfo.isOffloaded = pbLedgerInfo.hasOffloadContext(); if (pbLedgerInfo.hasOffloadContext()) { MLDataFormats.OffloadContext offloadContext = pbLedgerInfo.getOffloadContext(); @@ -869,7 +871,10 @@ public void asyncDelete(String name, CompletableFuture mlCo // If it's open, delete in the normal way ml.asyncDelete(callback, ctx); }).exceptionally(ex -> { - // If it's failing to get open, just delete from metadata + // If it fails to get open, it will be cleaned by managed ledger opening error handling. + // then retry will go to `future=null` branch. + final Throwable rc = FutureUtil.unwrapCompletionException(ex); + callback.deleteLedgerFailed(getManagedLedgerException(rc), ctx); return null; }); } @@ -884,14 +889,14 @@ void deleteManagedLedger(String managedLedgerName, CompletableFuture> futures = info.cursors.entrySet().stream() - .map(e -> deleteCursor(bkc, managedLedgerName, e.getKey(), e.getValue())) - .collect(Collectors.toList()); - Futures.waitForAll(futures).thenRun(() -> { - deleteManagedLedgerData(bkc, managedLedgerName, info, mlConfigFuture, callback, ctx); + getBookKeeper().thenCompose(bk -> { + // First delete all cursors resources + List> futures = info.cursors.entrySet().stream() + .map(e -> deleteCursor(bk, managedLedgerName, e.getKey(), e.getValue())) + .collect(Collectors.toList()); + return Futures.waitForAll(futures).thenApply(v -> bk); + }).thenAccept(bk -> { + deleteManagedLedgerData(bk, managedLedgerName, info, mlConfigFuture, callback, ctx); }).exceptionally(ex -> { callback.deleteLedgerFailed(new ManagedLedgerException(ex), ctx); return null; @@ -1053,6 +1058,7 @@ public MetaStore getMetaStore() { return store; } + @Override public ManagedLedgerFactoryConfig getConfig() { return config; } @@ -1072,20 +1078,422 @@ public long getCacheEvictionTimeThreshold(){ return cacheEvictionTimeThresholdNanos; } + @Override public ManagedLedgerFactoryMXBean getCacheStats() { return this.mbean; } - public BookKeeper getBookKeeper() { + public CompletableFuture getBookKeeper() { return bookkeeperFactory.get(); } + @Override + public void estimateUnloadedTopicBacklog(PersistentOfflineTopicStats offlineTopicStats, + TopicName topicName, boolean accurate, Object ctx) + throws Exception { + String managedLedgerName = topicName.getPersistenceNamingEncoding(); + long numberOfEntries = 0; + long totalSize = 0; + BookKeeper.DigestType digestType = (BookKeeper.DigestType) ((List) ctx).get(0); + byte[] password = (byte[]) ((List) ctx).get(1); + NavigableMap ledgers = + getManagedLedgersInfo(topicName, accurate, digestType, password); + for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ls : ledgers.values()) { + numberOfEntries += ls.getEntries(); + totalSize += ls.getSize(); + if (accurate) { + offlineTopicStats.addLedgerDetails(ls.getEntries(), ls.getTimestamp(), ls.getSize(), ls.getLedgerId()); + } + } + offlineTopicStats.totalMessages = numberOfEntries; + offlineTopicStats.storageSize = totalSize; + if (log.isDebugEnabled()) { + log.debug("[{}] Total number of entries - {} and size - {}", managedLedgerName, numberOfEntries, totalSize); + } + + // calculate per cursor message backlog + calculateCursorBacklogs(topicName, ledgers, offlineTopicStats, accurate, digestType, password); + offlineTopicStats.statGeneratedAt.setTime(System.currentTimeMillis()); + } + + private NavigableMap getManagedLedgersInfo( + final TopicName topicName, boolean accurate, BookKeeper.DigestType digestType, byte[] password) + throws Exception { + final NavigableMap ledgers = new ConcurrentSkipListMap<>(); + + String managedLedgerName = topicName.getPersistenceNamingEncoding(); + MetaStore store = getMetaStore(); + + final CountDownLatch mlMetaCounter = new CountDownLatch(1); + store.getManagedLedgerInfo(managedLedgerName, false /* createIfMissing */, + new MetaStore.MetaStoreCallback() { + @Override + public void operationComplete(MLDataFormats.ManagedLedgerInfo mlInfo, Stat stat) { + for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ls : mlInfo.getLedgerInfoList()) { + ledgers.put(ls.getLedgerId(), ls); + } + + // find no of entries in last ledger + if (!ledgers.isEmpty()) { + final long id = ledgers.lastKey(); + AsyncCallback.OpenCallback opencb = (rc, lh, ctx1) -> { + if (log.isDebugEnabled()) { + log.debug("[{}] Opened ledger {}: {}", managedLedgerName, id, + BKException.getMessage(rc)); + } + if (rc == BKException.Code.OK) { + MLDataFormats.ManagedLedgerInfo.LedgerInfo info = + MLDataFormats.ManagedLedgerInfo.LedgerInfo + .newBuilder().setLedgerId(id) + .setEntries(lh.getLastAddConfirmed() + 1) + .setSize(lh.getLength()).setTimestamp(System.currentTimeMillis()) + .build(); + ledgers.put(id, info); + mlMetaCounter.countDown(); + } else if (Errors.isNoSuchLedgerExistsException(rc)) { + log.warn("[{}] Ledger not found: {}", managedLedgerName, ledgers.lastKey()); + ledgers.remove(ledgers.lastKey()); + mlMetaCounter.countDown(); + } else { + log.error("[{}] Failed to open ledger {}: {}", managedLedgerName, id, + BKException.getMessage(rc)); + mlMetaCounter.countDown(); + } + }; + + if (log.isDebugEnabled()) { + log.debug("[{}] Opening ledger {}", managedLedgerName, id); + } + getBookKeeper() + .thenAccept(bk -> { + bk.asyncOpenLedgerNoRecovery(id, digestType, password, opencb, null); + }).exceptionally(ex -> { + log.warn("[{}] Failed to open ledger {}: {}", managedLedgerName, id, ex); + opencb.openComplete(-1, null, null); + mlMetaCounter.countDown(); + return null; + }); + } else { + log.warn("[{}] Ledger list empty", managedLedgerName); + mlMetaCounter.countDown(); + } + } + + @Override + public void operationFailed(ManagedLedgerException.MetaStoreException e) { + log.warn("[{}] Unable to obtain managed ledger metadata - {}", managedLedgerName, e); + mlMetaCounter.countDown(); + } + }); + + if (accurate) { + // block until however long it takes for operation to complete + mlMetaCounter.await(); + } else { + mlMetaCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + return ledgers; + } + + public void calculateCursorBacklogs(final TopicName topicName, + final NavigableMap ledgers, + final PersistentOfflineTopicStats offlineTopicStats, boolean accurate, + BookKeeper.DigestType digestType, byte[] password) throws Exception { + if (ledgers.isEmpty()) { + return; + } + String managedLedgerName = topicName.getPersistenceNamingEncoding(); + MetaStore store = getMetaStore(); + BookKeeper bk = getBookKeeper().get(); + final CountDownLatch allCursorsCounter = new CountDownLatch(1); + final long errorInReadingCursor = -1; + final var ledgerRetryMap = new ConcurrentHashMap(); + + final MLDataFormats.ManagedLedgerInfo.LedgerInfo ledgerInfo = ledgers.lastEntry().getValue(); + final Position lastLedgerPosition = + PositionFactory.create(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1); + if (log.isDebugEnabled()) { + log.debug("[{}] Last ledger position {}", managedLedgerName, lastLedgerPosition); + } + + store.getCursors(managedLedgerName, new MetaStore.MetaStoreCallback>() { + @Override + public void operationComplete(List cursors, Stat v) { + // Load existing cursors + if (log.isDebugEnabled()) { + log.debug("[{}] Found {} cursors", managedLedgerName, cursors.size()); + } + + if (cursors.isEmpty()) { + allCursorsCounter.countDown(); + return; + } + + final CountDownLatch cursorCounter = new CountDownLatch(cursors.size()); + + for (final String cursorName : cursors) { + // determine subscription position from cursor ledger + if (log.isDebugEnabled()) { + log.debug("[{}] Loading cursor {}", managedLedgerName, cursorName); + } + + AsyncCallback.OpenCallback cursorLedgerOpenCb = (rc, lh, ctx1) -> { + long ledgerId = lh.getId(); + if (log.isDebugEnabled()) { + log.debug("[{}] Opened cursor ledger {} for cursor {}. rc={}", managedLedgerName, ledgerId, + cursorName, rc); + } + if (rc != BKException.Code.OK) { + log.warn("[{}] Error opening metadata ledger {} for cursor {}: {}", managedLedgerName, + ledgerId, cursorName, BKException.getMessage(rc)); + cursorCounter.countDown(); + return; + } + long lac = lh.getLastAddConfirmed(); + if (log.isDebugEnabled()) { + log.debug("[{}] Cursor {} LAC {} read from ledger {}", managedLedgerName, cursorName, lac, + ledgerId); + } + + if (lac == LedgerHandle.INVALID_ENTRY_ID) { + // save the ledger id and cursor to retry outside of this call back + // since we are trying to read the same cursor ledger, we will block until + // this current callback completes, since an attempt to read the entry + // will block behind this current operation to complete + ledgerRetryMap.put(cursorName, ledgerId); + log.info("[{}] Cursor {} LAC {} read from ledger {}", managedLedgerName, cursorName, lac, + ledgerId); + cursorCounter.countDown(); + return; + } + final long entryId = lac; + // read last acked message position for subscription + lh.asyncReadEntries(entryId, entryId, new AsyncCallback.ReadCallback() { + @Override + public void readComplete(int rc, LedgerHandle lh, Enumeration seq, + Object ctx) { + try { + if (log.isDebugEnabled()) { + log.debug("readComplete rc={} entryId={}", rc, entryId); + } + if (rc != BKException.Code.OK) { + log.warn("[{}] Error reading from metadata ledger {} for cursor {}: {}", + managedLedgerName, ledgerId, cursorName, BKException.getMessage(rc)); + // indicate that this cursor should be excluded + offlineTopicStats.addCursorDetails(cursorName, errorInReadingCursor, + lh.getId()); + } else { + LedgerEntry entry = seq.nextElement(); + MLDataFormats.PositionInfo positionInfo; + try { + positionInfo = MLDataFormats.PositionInfo.parseFrom(entry.getEntry()); + } catch (InvalidProtocolBufferException e) { + log.warn( + "[{}] Error reading position from metadata ledger {} for cursor " + + "{}: {}", managedLedgerName, ledgerId, cursorName, e); + offlineTopicStats.addCursorDetails(cursorName, errorInReadingCursor, + lh.getId()); + return; + } + final Position lastAckedMessagePosition = + PositionFactory.create(positionInfo.getLedgerId(), + positionInfo.getEntryId()); + if (log.isDebugEnabled()) { + log.debug("[{}] Cursor {} MD {} read last ledger position {}", + managedLedgerName, cursorName, lastAckedMessagePosition, + lastLedgerPosition); + } + // calculate cursor backlog + Range range = Range.openClosed(lastAckedMessagePosition, + lastLedgerPosition); + if (log.isDebugEnabled()) { + log.debug("[{}] Calculating backlog for cursor {} using range {}", + managedLedgerName, cursorName, range); + } + long cursorBacklog = getNumberOfEntries(range, ledgers); + offlineTopicStats.messageBacklog += cursorBacklog; + offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, lh.getId()); + } + } finally { + cursorCounter.countDown(); + } + } + }, null); + + }; // end of cursor meta read callback + + store.asyncGetCursorInfo(managedLedgerName, cursorName, + new MetaStore.MetaStoreCallback() { + @Override + public void operationComplete(MLDataFormats.ManagedCursorInfo info, + Stat stat) { + long cursorLedgerId = info.getCursorsLedgerId(); + if (log.isDebugEnabled()) { + log.debug("[{}] Cursor {} meta-data read ledger id {}", managedLedgerName, + cursorName, cursorLedgerId); + } + if (cursorLedgerId != -1) { + bk.asyncOpenLedgerNoRecovery(cursorLedgerId, digestType, password, + cursorLedgerOpenCb, null); + } else { + Position lastAckedMessagePosition = PositionFactory.create( + info.getMarkDeleteLedgerId(), info.getMarkDeleteEntryId()); + Range range = Range.openClosed(lastAckedMessagePosition, + lastLedgerPosition); + if (log.isDebugEnabled()) { + log.debug("[{}] Calculating backlog for cursor {} using range {}", + managedLedgerName, cursorName, range); + } + long cursorBacklog = getNumberOfEntries(range, ledgers); + offlineTopicStats.messageBacklog += cursorBacklog; + offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, cursorLedgerId); + cursorCounter.countDown(); + } + + } + + @Override + public void operationFailed(ManagedLedgerException.MetaStoreException e) { + log.warn("[{}] Unable to obtain cursor ledger for cursor {}: {}", managedLedgerName, + cursorName, e); + cursorCounter.countDown(); + } + }); + } // for every cursor find backlog + try { + if (accurate) { + cursorCounter.await(); + } else { + cursorCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + } catch (Exception e) { + log.warn("[{}] Error reading subscription positions{}", managedLedgerName, e); + } finally { + allCursorsCounter.countDown(); + } + } + + @Override + public void operationFailed(ManagedLedgerException.MetaStoreException e) { + log.warn("[{}] Failed to get the cursors list", managedLedgerName, e); + allCursorsCounter.countDown(); + } + }); + if (accurate) { + allCursorsCounter.await(); + } else { + allCursorsCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + + // go through ledgers where LAC was -1 + if (accurate && ledgerRetryMap.size() > 0) { + ledgerRetryMap.forEach((cursorName, ledgerId) -> { + if (log.isDebugEnabled()) { + log.debug("Cursor {} Ledger {} Trying to obtain MD from BkAdmin", cursorName, ledgerId); + } + Position lastAckedMessagePosition = tryGetMDPosition(bk, ledgerId, cursorName); + if (lastAckedMessagePosition == null) { + log.warn("[{}] Cursor {} read from ledger {}. Unable to determine cursor position", + managedLedgerName, cursorName, ledgerId); + } else { + if (log.isDebugEnabled()) { + log.debug("[{}] Cursor {} read from ledger using bk admin {}. position {}", managedLedgerName, + cursorName, ledgerId, lastAckedMessagePosition); + } + // calculate cursor backlog + Range range = Range.openClosed(lastAckedMessagePosition, lastLedgerPosition); + if (log.isDebugEnabled()) { + log.debug("[{}] Calculating backlog for cursor {} using range {}", managedLedgerName, + cursorName, range); + } + long cursorBacklog = getNumberOfEntries(range, ledgers); + offlineTopicStats.messageBacklog += cursorBacklog; + offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, ledgerId); + } + }); + } + } + + // need a better way than to duplicate the functionality below from ML + private long getNumberOfEntries(Range range, + NavigableMap ledgers) { + Position fromPosition = range.lowerEndpoint(); + boolean fromIncluded = range.lowerBoundType() == BoundType.CLOSED; + Position toPosition = range.upperEndpoint(); + boolean toIncluded = range.upperBoundType() == BoundType.CLOSED; + + if (fromPosition.getLedgerId() == toPosition.getLedgerId()) { + // If the 2 positions are in the same ledger + long count = toPosition.getEntryId() - fromPosition.getEntryId() - 1; + count += fromIncluded ? 1 : 0; + count += toIncluded ? 1 : 0; + return count; + } else { + long count = 0; + // If the from & to are pointing to different ledgers, then we need to : + // 1. Add the entries in the ledger pointed by toPosition + count += toPosition.getEntryId(); + count += toIncluded ? 1 : 0; + + // 2. Add the entries in the ledger pointed by fromPosition + MLDataFormats.ManagedLedgerInfo.LedgerInfo li = ledgers.get(fromPosition.getLedgerId()); + if (li != null) { + count += li.getEntries() - (fromPosition.getEntryId() + 1); + count += fromIncluded ? 1 : 0; + } + + // 3. Add the whole ledgers entries in between + for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ls : ledgers + .subMap(fromPosition.getLedgerId(), false, toPosition.getLedgerId(), false).values()) { + count += ls.getEntries(); + } + + return count; + } + } + + + private Position tryGetMDPosition(BookKeeper bookKeeper, long ledgerId, String cursorName) { + BookKeeperAdmin bookKeeperAdmin = null; + long lastEntry = LedgerHandle.INVALID_ENTRY_ID; + Position lastAckedMessagePosition = null; + try { + bookKeeperAdmin = new BookKeeperAdmin(bookKeeper); + for (LedgerEntry ledgerEntry : bookKeeperAdmin.readEntries(ledgerId, 0, lastEntry)) { + lastEntry = ledgerEntry.getEntryId(); + if (log.isDebugEnabled()) { + log.debug(" Read entry {} from ledger {} for cursor {}", lastEntry, ledgerId, cursorName); + } + MLDataFormats.PositionInfo positionInfo = MLDataFormats.PositionInfo.parseFrom(ledgerEntry.getEntry()); + lastAckedMessagePosition = + PositionFactory.create(positionInfo.getLedgerId(), positionInfo.getEntryId()); + if (log.isDebugEnabled()) { + log.debug("Cursor {} read position {}", cursorName, lastAckedMessagePosition); + } + } + } catch (Exception e) { + log.warn("Unable to determine LAC for ledgerId {} for cursor {}: {}", ledgerId, cursorName, e); + } finally { + if (bookKeeperAdmin != null) { + try { + bookKeeperAdmin.close(); + } catch (Exception e) { + log.warn("Unable to close bk admin for ledgerId {} for cursor {}", ledgerId, cursorName, e); + } + } + + } + return lastAckedMessagePosition; + } + + private static final int META_READ_TIMEOUT_SECONDS = 60; + /** * Factory to create Bookkeeper-client for a given ensemblePlacementPolicy. * */ public interface BookkeeperFactoryForCustomEnsemblePlacementPolicy { - default BookKeeper get() { + default CompletableFuture get() { return get(null); } @@ -1096,7 +1504,7 @@ default BookKeeper get() { * @param ensemblePlacementPolicyMetadata * @return */ - BookKeeper get(EnsemblePlacementPolicyConfig ensemblePlacementPolicyMetadata); + CompletableFuture get(EnsemblePlacementPolicyConfig ensemblePlacementPolicyMetadata); } private static final Logger log = LoggerFactory.getLogger(ManagedLedgerFactoryImpl.class); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryMBeanImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryMBeanImpl.java index cf3d7142d617e..a3038a0e7ff76 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryMBeanImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryMBeanImpl.java @@ -99,26 +99,51 @@ public double getCacheHitsRate() { return cacheHits.getRate(); } + @Override + public long getCacheHitsTotal() { + return cacheHits.getTotalCount(); + } + @Override public double getCacheMissesRate() { return cacheMisses.getRate(); } + @Override + public long getCacheMissesTotal() { + return cacheMisses.getTotalCount(); + } + @Override public double getCacheHitsThroughput() { return cacheHits.getValueRate(); } + @Override + public long getCacheHitsBytesTotal() { + return cacheHits.getTotalValue(); + } + @Override public double getCacheMissesThroughput() { return cacheMisses.getValueRate(); } + @Override + public long getCacheMissesBytesTotal() { + return cacheMisses.getTotalValue(); + } + @Override public long getNumberOfCacheEvictions() { return cacheEvictions.getCount(); } + @Override + public long getNumberOfCacheEvictionsTotal() { + return cacheEvictions.getTotalCount(); + } + public long getCacheInsertedEntriesCount() { return insertedEntryCount.sum(); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 15e9d332fa103..cb19bd94bce01 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.time.Clock; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -77,10 +78,13 @@ import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.BookKeeper.DigestType; import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.client.api.LedgerEntry; +import org.apache.bookkeeper.client.api.LedgerMetadata; import org.apache.bookkeeper.client.api.ReadHandle; import org.apache.bookkeeper.common.util.Backoff; import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.common.util.Retries; +import org.apache.bookkeeper.discover.RegistrationClient; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback; @@ -94,8 +98,10 @@ import org.apache.bookkeeper.mledger.AsyncCallbacks.TerminateCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.UpdatePropertiesCallback; import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.LedgerOffloader; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerAttributes; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerException.BadVersionException; @@ -113,6 +119,8 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.TooManyRequestsException; import org.apache.bookkeeper.mledger.ManagedLedgerMXBean; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionBound; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.WaitingEntryCallBack; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.VoidCallback; import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback; @@ -126,18 +134,19 @@ import org.apache.bookkeeper.mledger.proto.MLDataFormats.OffloadContext; import org.apache.bookkeeper.mledger.util.CallbackMutex; import org.apache.bookkeeper.mledger.util.Futures; +import org.apache.bookkeeper.mledger.util.ManagedLedgerImplUtils; import org.apache.bookkeeper.net.BookieId; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition; import org.apache.pulsar.common.policies.data.EnsemblePlacementPolicyConfig; import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats; import org.apache.pulsar.common.policies.data.OffloadPolicies; -import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; import org.apache.pulsar.common.policies.data.OffloadedReadPriority; import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.LazyLoadableValue; import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap; import org.apache.pulsar.metadata.api.Stat; import org.slf4j.Logger; @@ -213,8 +222,8 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback { private final CallbackMutex trimmerMutex = new CallbackMutex(); private final CallbackMutex offloadMutex = new CallbackMutex(); - private static final CompletableFuture NULL_OFFLOAD_PROMISE = CompletableFuture - .completedFuture(PositionImpl.LATEST); + public static final CompletableFuture NULL_OFFLOAD_PROMISE = CompletableFuture + .completedFuture(PositionFactory.LATEST); protected volatile LedgerHandle currentLedger; protected volatile long currentLedgerEntries = 0; protected volatile long currentLedgerSize = 0; @@ -232,15 +241,18 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback { private static final Random random = new Random(System.currentTimeMillis()); private long maximumRolloverTimeMs; - protected final Supplier mlOwnershipChecker; + protected final Supplier> mlOwnershipChecker; - volatile PositionImpl lastConfirmedEntry; + volatile Position lastConfirmedEntry; protected ManagedLedgerInterceptor managedLedgerInterceptor; protected volatile long lastAddEntryTimeMs = 0; private long inactiveLedgerRollOverTimeMs = 0; + /** A signal that may trigger all the subsequent OpAddEntry of current ledger to be failed due to timeout. **/ + protected volatile AtomicBoolean currentLedgerTimeoutTriggered; + protected static final int DEFAULT_LEDGER_DELETE_RETRIES = 3; protected static final int DEFAULT_LEDGER_DELETE_BACKOFF_TIME_SEC = 60; private static final String MIGRATION_STATE_PROPERTY = "migrated"; @@ -280,10 +292,7 @@ public boolean isFenced() { } } - // define boundaries for position based seeks and searches - public enum PositionBound { - startIncluded, startExcluded - } + protected static final AtomicReferenceFieldUpdater STATE_UPDATER = AtomicReferenceFieldUpdater.newUpdater(ManagedLedgerImpl.class, State.class, "state"); @@ -322,9 +331,12 @@ public enum PositionBound { */ final ConcurrentLinkedQueue pendingAddEntries = new ConcurrentLinkedQueue<>(); + @Getter + private final ManagedLedgerAttributes managedLedgerAttributes; + /** * This variable is used for testing the tests. - * {@link ManagedLedgerTest#testManagedLedgerWithPlacementPolicyInCustomMetadata()} + * ManagedLedgerTest#testManagedLedgerWithPlacementPolicyInCustomMetadata() */ @VisibleForTesting Map createdLedgerCustomMetadata; @@ -334,9 +346,10 @@ public ManagedLedgerImpl(ManagedLedgerFactoryImpl factory, BookKeeper bookKeeper final String name) { this(factory, bookKeeper, store, config, scheduledExecutor, name, null); } + public ManagedLedgerImpl(ManagedLedgerFactoryImpl factory, BookKeeper bookKeeper, MetaStore store, ManagedLedgerConfig config, OrderedScheduler scheduledExecutor, - final String name, final Supplier mlOwnershipChecker) { + final String name, final Supplier> mlOwnershipChecker) { this.factory = factory; this.bookKeeper = bookKeeper; this.config = config; @@ -366,12 +379,10 @@ public ManagedLedgerImpl(ManagedLedgerFactoryImpl factory, BookKeeper bookKeeper this.mlOwnershipChecker = mlOwnershipChecker; this.propertiesMap = new ConcurrentHashMap<>(); this.inactiveLedgerRollOverTimeMs = config.getInactiveLedgerRollOverTimeMs(); - if (config.getManagedLedgerInterceptor() != null) { - this.managedLedgerInterceptor = config.getManagedLedgerInterceptor(); - } this.minBacklogCursorsForCaching = config.getMinimumBacklogCursorsForCaching(); this.minBacklogEntriesForCaching = config.getMinimumBacklogEntriesForCaching(); this.maxBacklogBetweenCursorsForCaching = config.getMaxBacklogBetweenCursorsForCaching(); + this.managedLedgerAttributes = new ManagedLedgerAttributes(this); } synchronized void initialize(final ManagedLedgerInitializeLedgerCallback callback, final Object ctx) { @@ -385,7 +396,9 @@ public void operationComplete(ManagedLedgerInfo mlInfo, Stat stat) { ledgersStat = stat; if (mlInfo.hasTerminatedPosition()) { state = State.Terminated; - lastConfirmedEntry = new PositionImpl(mlInfo.getTerminatedPosition()); + NestedPositionInfo terminatedPosition = mlInfo.getTerminatedPosition(); + lastConfirmedEntry = + PositionFactory.create(terminatedPosition.getLedgerId(), terminatedPosition.getEntryId()); log.info("[{}] Recovering managed ledger terminated at {}", name, lastConfirmedEntry); } for (LedgerInfo ls : mlInfo.getLedgerInfoList()) { @@ -419,13 +432,14 @@ public void operationComplete(ManagedLedgerInfo mlInfo, Stat stat) { .setTimestamp(clock.millis()).build(); ledgers.put(id, info); if (managedLedgerInterceptor != null) { - managedLedgerInterceptor.onManagedLedgerLastLedgerInitialize(name, lh) - .thenRun(() -> initializeBookKeeper(callback)) - .exceptionally(ex -> { - callback.initializeFailed( - new ManagedLedgerInterceptException(ex.getCause())); - return null; - }); + managedLedgerInterceptor + .onManagedLedgerLastLedgerInitialize(name, createLastEntryHandle(lh)) + .thenRun(() -> initializeBookKeeper(callback)) + .exceptionally(ex -> { + callback.initializeFailed( + new ManagedLedgerInterceptException(ex.getCause())); + return null; + }); } else { initializeBookKeeper(callback); } @@ -465,6 +479,42 @@ public void operationFailed(MetaStoreException e) { scheduleTimeoutTask(); } + protected ManagedLedgerInterceptor.LastEntryHandle createLastEntryHandle(LedgerHandle lh) { + return () -> { + CompletableFuture> promise = new CompletableFuture<>(); + if (lh.getLastAddConfirmed() >= 0) { + lh.readAsync(lh.getLastAddConfirmed(), lh.getLastAddConfirmed()) + .whenComplete((entries, ex) -> { + if (ex != null) { + promise.completeExceptionally(ex); + } else { + if (entries != null) { + try { + LedgerEntry ledgerEntry = + entries.getEntry(lh.getLastAddConfirmed()); + if (ledgerEntry != null) { + promise.complete( + Optional.of(EntryImpl.create(ledgerEntry))); + } else { + promise.complete(Optional.empty()); + } + entries.close(); + } catch (Exception e) { + entries.close(); + promise.completeExceptionally(e); + } + } else { + promise.complete(Optional.empty()); + } + } + }); + } else { + promise.complete(Optional.empty()); + } + return promise; + }; + } + protected synchronized void initializeBookKeeper(final ManagedLedgerInitializeLedgerCallback callback) { if (log.isDebugEnabled()) { log.debug("[{}] initializing bookkeeper; ledgers {}", name, ledgers); @@ -528,18 +578,21 @@ public void operationFailed(MetaStoreException e) { return; } - log.info("[{}] Created ledger {}", name, lh.getId()); + log.info("[{}] Created ledger {} after closed {}", name, lh.getId(), + currentLedger == null ? "null" : currentLedger.getId()); STATE_UPDATER.set(this, State.LedgerOpened); updateLastLedgerCreatedTimeAndScheduleRolloverTask(); currentLedger = lh; + currentLedgerTimeoutTriggered = new AtomicBoolean(); - lastConfirmedEntry = new PositionImpl(lh.getId(), -1); + lastConfirmedEntry = PositionFactory.create(lh.getId(), -1); // bypass empty ledgers, find last ledger with Message if possible. while (lastConfirmedEntry.getEntryId() == -1) { Map.Entry formerLedger = ledgers.lowerEntry(lastConfirmedEntry.getLedgerId()); if (formerLedger != null) { LedgerInfo ledgerInfo = formerLedger.getValue(); - lastConfirmedEntry = PositionImpl.get(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1); + lastConfirmedEntry = + PositionFactory.create(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1); } else { break; } @@ -578,7 +631,7 @@ public void operationComplete(List consumers, Stat s) { for (final String cursorName : consumers) { log.info("[{}] Loading cursor {}", name, cursorName); final ManagedCursorImpl cursor; - cursor = new ManagedCursorImpl(bookKeeper, config, ManagedLedgerImpl.this, cursorName); + cursor = new ManagedCursorImpl(bookKeeper, ManagedLedgerImpl.this, cursorName); cursor.recover(new VoidCallback() { @Override @@ -609,7 +662,7 @@ public void operationFailed(ManagedLedgerException exception) { log.debug("[{}] Recovering cursor {} lazily", name, cursorName); } final ManagedCursorImpl cursor; - cursor = new ManagedCursorImpl(bookKeeper, config, ManagedLedgerImpl.this, cursorName); + cursor = new ManagedCursorImpl(bookKeeper, ManagedLedgerImpl.this, cursorName); CompletableFuture cursorRecoveryFuture = new CompletableFuture<>(); uninitializedCursors.put(cursorName, cursorRecoveryFuture); @@ -652,7 +705,7 @@ private void addCursor(ManagedCursorImpl cursor) { if (cursor.isDurable()) { positionForOrdering = cursor.getMarkDeletedPosition(); if (positionForOrdering == null) { - positionForOrdering = PositionImpl.EARLIEST; + positionForOrdering = PositionFactory.EARLIEST; } } cursors.add(cursor, positionForOrdering); @@ -675,37 +728,7 @@ public Position addEntry(byte[] data, int numberOfMessages) throws InterruptedEx @Override public Position addEntry(byte[] data, int offset, int length) throws InterruptedException, ManagedLedgerException { - final CountDownLatch counter = new CountDownLatch(1); - // Result list will contain the status exception and the resulting - // position - class Result { - ManagedLedgerException status = null; - Position position = null; - } - final Result result = new Result(); - - asyncAddEntry(data, offset, length, new AddEntryCallback() { - @Override - public void addComplete(Position position, ByteBuf entryData, Object ctx) { - result.position = position; - counter.countDown(); - } - - @Override - public void addFailed(ManagedLedgerException exception, Object ctx) { - result.status = exception; - counter.countDown(); - } - }, null); - - counter.await(); - - if (result.status != null) { - log.error("[{}] Error adding entry", name, result.status); - throw result.status; - } - - return result.position; + return addEntry(data, 1, offset, length); } @Override @@ -765,18 +788,7 @@ public void asyncAddEntry(final byte[] data, int numberOfMessages, int offset, i @Override public void asyncAddEntry(ByteBuf buffer, AddEntryCallback callback, Object ctx) { - if (log.isDebugEnabled()) { - log.debug("[{}] asyncAddEntry size={} state={}", name, buffer.readableBytes(), state); - } - - // retain buffer in this thread - buffer.retain(); - - // Jump to specific thread to avoid contention from writers writing from different threads - executor.execute(() -> { - OpAddEntry addOperation = OpAddEntry.createNoRetainBuffer(this, buffer, callback, ctx); - internalAsyncAddEntry(addOperation); - }); + asyncAddEntry(buffer, 1, callback, ctx); } @Override @@ -790,7 +802,8 @@ public void asyncAddEntry(ByteBuf buffer, int numberOfMessages, AddEntryCallback // Jump to specific thread to avoid contention from writers writing from different threads executor.execute(() -> { - OpAddEntry addOperation = OpAddEntry.createNoRetainBuffer(this, buffer, numberOfMessages, callback, ctx); + OpAddEntry addOperation = OpAddEntry.createNoRetainBuffer(this, buffer, numberOfMessages, callback, ctx, + currentLedgerTimeoutTriggered); internalAsyncAddEntry(addOperation); }); } @@ -842,6 +855,7 @@ protected synchronized void internalAsyncAddEntry(OpAddEntry addOperation) { // Write into lastLedger addOperation.setLedger(currentLedger); + addOperation.setTimeoutTriggered(currentLedgerTimeoutTriggered); ++currentLedgerEntries; currentLedgerSize += addOperation.data.readableBytes(); @@ -973,7 +987,8 @@ public synchronized void asyncOpenCursor(final String cursorName, final InitialP if (uninitializedCursors.containsKey(cursorName)) { uninitializedCursors.get(cursorName).thenAccept(cursor -> callback.openCursorComplete(cursor, ctx)) .exceptionally(ex -> { - callback.openCursorFailed((ManagedLedgerException) ex, ctx); + callback.openCursorFailed(ManagedLedgerException + .getManagedLedgerException(FutureUtil.unwrapCompletionException(ex)), ctx); return null; }); return; @@ -991,10 +1006,10 @@ public synchronized void asyncOpenCursor(final String cursorName, final InitialP if (log.isDebugEnabled()) { log.debug("[{}] Creating new cursor: {}", name, cursorName); } - final ManagedCursorImpl cursor = new ManagedCursorImpl(bookKeeper, config, this, cursorName); + final ManagedCursorImpl cursor = new ManagedCursorImpl(bookKeeper, this, cursorName); CompletableFuture cursorFuture = new CompletableFuture<>(); uninitializedCursors.put(cursorName, cursorFuture); - PositionImpl position = InitialPosition.Earliest == initialPosition ? getFirstPosition() : getLastPosition(); + Position position = InitialPosition.Earliest == initialPosition ? getFirstPosition() : getLastPosition(); cursor.initialize(position, properties, cursorProperties, new VoidCallback() { @Override public void operationComplete() { @@ -1032,6 +1047,7 @@ public synchronized void asyncDeleteCursor(final String consumerName, final Dele + consumerName), ctx); return; } else if (!cursor.isDurable()) { + cursor.setState(ManagedCursorImpl.State.Closed); cursors.removeCursor(consumerName); deactivateCursorByName(consumerName); callback.deleteCursorComplete(ctx); @@ -1123,8 +1139,8 @@ public ManagedCursor newNonDurableCursor(Position startCursorPosition, String cu return cachedCursor; } - NonDurableCursorImpl cursor = new NonDurableCursorImpl(bookKeeper, config, this, cursorName, - (PositionImpl) startCursorPosition, initialPosition, isReadCompacted); + NonDurableCursorImpl cursor = new NonDurableCursorImpl(bookKeeper, this, cursorName, + startCursorPosition, initialPosition, isReadCompacted); cursor.setActive(); log.info("[{}] Opened new cursor: {}", name, cursor); @@ -1163,7 +1179,7 @@ public long getNumberOfEntries() { @Override public long getNumberOfActiveEntries() { long totalEntries = getNumberOfEntries(); - PositionImpl pos = cursors.getSlowestReaderPosition(); + Position pos = cursors.getSlowestReaderPosition(); if (pos == null) { // If there are no consumers, there are no active entries return 0; @@ -1182,7 +1198,7 @@ public long getTotalSize() { @Override public long getEstimatedBacklogSize() { - PositionImpl pos = getMarkDeletePositionOfSlowestConsumer(); + Position pos = getMarkDeletePositionOfSlowestConsumer(); while (true) { if (pos == null) { @@ -1226,18 +1242,22 @@ public long getEstimatedBacklogSize() { @Override public CompletableFuture getEarliestMessagePublishTimeInBacklog() { - PositionImpl pos = getMarkDeletePositionOfSlowestConsumer(); + Position pos = getMarkDeletePositionOfSlowestConsumer(); return getEarliestMessagePublishTimeOfPos(pos); } - public CompletableFuture getEarliestMessagePublishTimeOfPos(PositionImpl pos) { + private CompletableFuture getEarliestMessagePublishTimeOfPos(Position pos) { CompletableFuture future = new CompletableFuture<>(); if (pos == null) { future.complete(0L); return future; } - PositionImpl nextPos = getNextValidPosition(pos); + Position nextPos = getNextValidPosition(pos); + + if (nextPos.compareTo(lastConfirmedEntry) > 0) { + return CompletableFuture.completedFuture(-1L); + } asyncReadEntry(nextPos, new ReadEntryCallback() { @Override @@ -1258,6 +1278,12 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { log.error("Error read entry for position {}", nextPos, exception); future.completeExceptionally(exception); } + + @Override + public String toString() { + return String.format("ML [%s] get earliest message publish time of pos", + ManagedLedgerImpl.this.name); + } }, null); return future; @@ -1266,14 +1292,14 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { /** * Get estimated backlog size from a specific position. */ - public long getEstimatedBacklogSize(PositionImpl pos) { + public long getEstimatedBacklogSize(Position pos) { if (pos == null) { return 0; } return estimateBacklogFromPosition(pos); } - long estimateBacklogFromPosition(PositionImpl pos) { + long estimateBacklogFromPosition(Position pos) { synchronized (this) { long sizeBeforePosLedger = ledgers.headMap(pos.getLedgerId()).values() .stream().mapToLong(LedgerInfo::getSize).sum(); @@ -1352,7 +1378,7 @@ public synchronized void asyncTerminate(TerminateCallback callback, Object ctx) if (rc != BKException.Code.OK) { callback.terminateFailed(createManagedLedgerException(rc), ctx); } else { - lastConfirmedEntry = new PositionImpl(lh.getId(), lh.getLastAddConfirmed()); + lastConfirmedEntry = PositionFactory.create(lh.getId(), lh.getLastAddConfirmed()); // Store the new state in metadata store.asyncUpdateLedgerIds(name, getManagedLedgerInfo(), ledgersStat, new MetaStoreCallback() { @Override @@ -1528,6 +1554,15 @@ private void closeAllCursors(CloseCallback callback, final Object ctx) { @Override public synchronized void createComplete(int rc, final LedgerHandle lh, Object ctx) { + if (STATE_UPDATER.get(this) == State.Closed) { + if (lh != null) { + log.warn("[{}] ledger create completed after the managed ledger is closed rc={} ledger={}, so just" + + " close this ledger handle.", name, rc, lh != null ? lh.getId() : -1); + lh.closeAsync(); + } + return; + } + if (log.isDebugEnabled()) { log.debug("[{}] createComplete rc={} ledger={}", name, rc, lh != null ? lh.getId() : -1); } @@ -1565,6 +1600,7 @@ public void operationComplete(Void v, Stat stat) { LedgerHandle originalCurrentLedger = currentLedger; ledgers.put(lh.getId(), newLedger); currentLedger = lh; + currentLedgerTimeoutTriggered = new AtomicBoolean(); currentLedgerEntries = 0; currentLedgerSize = 0; updateLedgersIdsComplete(originalCurrentLedger); @@ -1648,9 +1684,12 @@ void createNewOpAddEntryForNewLedger() { if (existsOp != null) { // If op is used by another ledger handle, we need to close it and create a new one if (existsOp.ledger != null) { - existsOp.close(); - existsOp = OpAddEntry.createNoRetainBuffer(existsOp.ml, existsOp.data, - existsOp.getNumberOfMessages(), existsOp.callback, existsOp.ctx); + existsOp = existsOp.duplicateAndClose(currentLedgerTimeoutTriggered); + } else { + // It may happen when the following operations execute at the same time, so it is expected. + // - Adding entry. + // - Switching ledger. + existsOp.setTimeoutTriggered(currentLedgerTimeoutTriggered); } existsOp.setLedger(currentLedger); pendingAddEntries.add(existsOp); @@ -1735,15 +1774,20 @@ synchronized void ledgerClosed(final LedgerHandle lh) { maybeOffloadInBackground(NULL_OFFLOAD_PROMISE); - if (!pendingAddEntries.isEmpty()) { - // Need to create a new ledger to write pending entries - createLedgerAfterClosed(); + createLedgerAfterClosed(); + } + + @Override + public void skipNonRecoverableLedger(long ledgerId){ + for (ManagedCursor managedCursor : cursors) { + managedCursor.skipNonRecoverableLedger(ledgerId); } } synchronized void createLedgerAfterClosed() { if (isNeededCreateNewLedgerAfterCloseLedger()) { - log.info("[{}] Creating a new ledger after closed", name); + log.info("[{}] Creating a new ledger after closed {}", name, + currentLedger == null ? "null" : currentLedger.getId()); STATE_UPDATER.set(this, State.CreatingLedger); this.lastLedgerCreationInitiationTimestamp = System.currentTimeMillis(); mbean.startDataLedgerCreateOp(); @@ -1785,7 +1829,6 @@ public void closeComplete(int rc, LedgerHandle lh, Object o) { } ledgerClosed(lh); - createLedgerAfterClosed(); } }, null); } @@ -1796,7 +1839,7 @@ public CompletableFuture asyncFindPosition(Predicate predicate) CompletableFuture future = new CompletableFuture<>(); Long firstLedgerId = ledgers.firstKey(); - final PositionImpl startPosition = firstLedgerId == null ? null : new PositionImpl(firstLedgerId, 0); + final Position startPosition = firstLedgerId == null ? null : PositionFactory.create(firstLedgerId, 0); if (startPosition == null) { future.complete(null); return future; @@ -1810,7 +1853,7 @@ public void findEntryComplete(Position position, Object ctx) { log.info("[{}] Unable to find position for predicate {}. Use the first position {} instead.", name, predicate, startPosition); } else { - finalPosition = getNextValidPosition((PositionImpl) position); + finalPosition = getNextValidPosition(position); } future.complete(finalPosition); } @@ -1861,7 +1904,7 @@ void asyncReadEntries(OpReadEntry opReadEntry) { if (ledgerInfo == null || ledgerInfo.getEntries() == 0) { // Cursor is pointing to an empty ledger, there's no need to try opening it. Skip this ledger and // move to the next one - opReadEntry.updateReadPosition(new PositionImpl(opReadEntry.readPosition.getLedgerId() + 1, 0)); + opReadEntry.updateReadPosition(PositionFactory.create(opReadEntry.readPosition.getLedgerId() + 1, 0)); opReadEntry.checkReadCompletion(); return; } @@ -1878,12 +1921,12 @@ void asyncReadEntries(OpReadEntry opReadEntry) { } } - public CompletableFuture getLedgerMetadata(long ledgerId) { + public CompletableFuture getLedgerMetadata(long ledgerId) { LedgerHandle currentLedger = this.currentLedger; if (currentLedger != null && ledgerId == currentLedger.getId()) { - return CompletableFuture.completedFuture(currentLedger.getLedgerMetadata().toSafeString()); + return CompletableFuture.completedFuture(currentLedger.getLedgerMetadata()); } else { - return getLedgerHandle(ledgerId).thenApply(rh -> rh.getLedgerMetadata().toSafeString()); + return getLedgerHandle(ledgerId).thenApply(rh -> rh.getLedgerMetadata()); } } @@ -1989,7 +2032,8 @@ public void invalidateLedgerHandle(ReadHandle ledgerHandle) { } } - public void asyncReadEntry(PositionImpl position, ReadEntryCallback callback, Object ctx) { + @Override + public void asyncReadEntry(Position position, ReadEntryCallback callback, Object ctx) { LedgerHandle currentLedger = this.currentLedger; if (log.isDebugEnabled()) { log.debug("[{}] Reading entry ledger {}: {}", name, position.getLedgerId(), position.getEntryId()); @@ -2022,7 +2066,7 @@ private void internalReadFromLedger(ReadHandle ledger, OpReadEntry opReadEntry) long firstEntry = opReadEntry.readPosition.getEntryId(); long lastEntryInLedger; - PositionImpl lastPosition = lastConfirmedEntry; + Position lastPosition = lastConfirmedEntry; if (ledger.getId() == lastPosition.getLedgerId()) { // For the current ledger, we only give read visibility to the last entry we have received a confirmation in @@ -2049,9 +2093,9 @@ private void internalReadFromLedger(ReadHandle ledger, OpReadEntry opReadEntry) // beginning of the next ledger Long nextLedgerId = ledgers.ceilingKey(ledger.getId() + 1); if (nextLedgerId != null) { - opReadEntry.updateReadPosition(new PositionImpl(nextLedgerId, 0)); + opReadEntry.updateReadPosition(PositionFactory.create(nextLedgerId, 0)); } else { - opReadEntry.updateReadPosition(new PositionImpl(ledger.getId() + 1, 0)); + opReadEntry.updateReadPosition(PositionFactory.create(ledger.getId() + 1, 0)); } } else { opReadEntry.updateReadPosition(opReadEntry.readPosition); @@ -2069,7 +2113,7 @@ private void internalReadFromLedger(ReadHandle ledger, OpReadEntry opReadEntry) long lastValidEntry = -1L; long entryId = firstEntry; for (; entryId <= lastEntry; entryId++) { - if (opReadEntry.skipCondition.test(PositionImpl.get(ledger.getId(), entryId))) { + if (opReadEntry.skipCondition.test(PositionFactory.create(ledger.getId(), entryId))) { if (firstValidEntry != -1L) { break; } @@ -2086,7 +2130,7 @@ private void internalReadFromLedger(ReadHandle ledger, OpReadEntry opReadEntry) // then manual call internalReadEntriesComplete to advance read position. if (firstValidEntry == -1L) { opReadEntry.internalReadEntriesComplete(Collections.emptyList(), opReadEntry.ctx, - PositionImpl.get(ledger.getId(), lastEntry)); + PositionFactory.create(ledger.getId(), lastEntry)); return; } @@ -2101,7 +2145,8 @@ private void internalReadFromLedger(ReadHandle ledger, OpReadEntry opReadEntry) asyncReadEntry(ledger, firstEntry, lastEntry, opReadEntry, opReadEntry.ctx); } - protected void asyncReadEntry(ReadHandle ledger, PositionImpl position, ReadEntryCallback callback, Object ctx) { + protected void asyncReadEntry(ReadHandle ledger, Position position, ReadEntryCallback callback, Object ctx) { + mbean.addEntriesRead(1); if (config.getReadEntryTimeoutSeconds() > 0) { // set readOpCount to uniquely validate if ReadEntryCallbackWrapper is already recycled long readOpCount = READ_OP_COUNT_UPDATER.incrementAndGet(this); @@ -2285,8 +2330,8 @@ public ManagedLedgerMXBean getStats() { return mbean; } - public boolean hasMoreEntries(PositionImpl position) { - PositionImpl lastPos = lastConfirmedEntry; + public boolean hasMoreEntries(Position position) { + Position lastPos = lastConfirmedEntry; boolean result = position.compareTo(lastPos) <= 0; if (log.isDebugEnabled()) { log.debug("[{}] hasMoreEntries: pos={} lastPos={} res={}", name, position, lastPos, result); @@ -2307,7 +2352,7 @@ private void invalidateEntriesUpToSlowestReaderPosition() { return; } if (!activeCursors.isEmpty()) { - PositionImpl evictionPos = activeCursors.getSlowestReaderPosition(); + Position evictionPos = activeCursors.getSlowestReaderPosition(); if (evictionPos != null) { entryCache.invalidateEntries(evictionPos); } @@ -2316,7 +2361,7 @@ private void invalidateEntriesUpToSlowestReaderPosition() { } } - void onCursorMarkDeletePositionUpdated(ManagedCursorImpl cursor, PositionImpl newPosition) { + void onCursorMarkDeletePositionUpdated(ManagedCursorImpl cursor, Position newPosition) { if (config.isCacheEvictionByMarkDeletedPosition()) { updateActiveCursor(cursor, newPosition); } @@ -2324,15 +2369,15 @@ void onCursorMarkDeletePositionUpdated(ManagedCursorImpl cursor, PositionImpl ne // non-durable cursors aren't tracked for trimming return; } - Pair pair = cursors.cursorUpdated(cursor, newPosition); + Pair pair = cursors.cursorUpdated(cursor, newPosition); if (pair == null) { // Cursor has been removed in the meantime trimConsumedLedgersInBackground(); return; } - PositionImpl previousSlowestReader = pair.getLeft(); - PositionImpl currentSlowestReader = pair.getRight(); + Position previousSlowestReader = pair.getLeft(); + Position currentSlowestReader = pair.getRight(); if (previousSlowestReader.compareTo(currentSlowestReader) == 0) { // The slowest consumer has not changed position. Nothing to do right now @@ -2346,7 +2391,7 @@ void onCursorMarkDeletePositionUpdated(ManagedCursorImpl cursor, PositionImpl ne } private void updateActiveCursor(ManagedCursorImpl cursor, Position newPosition) { - Pair slowestPositions = activeCursors.cursorUpdated(cursor, newPosition); + Pair slowestPositions = activeCursors.cursorUpdated(cursor, newPosition); if (slowestPositions != null && !slowestPositions.getLeft().equals(slowestPositions.getRight())) { invalidateEntriesUpToSlowestReaderPosition(); @@ -2359,12 +2404,12 @@ public void onCursorReadPositionUpdated(ManagedCursorImpl cursor, Position newRe } } - PositionImpl startReadOperationOnLedger(PositionImpl position) { + Position startReadOperationOnLedger(Position position) { Long ledgerId = ledgers.ceilingKey(position.getLedgerId()); if (ledgerId != null && ledgerId != position.getLedgerId()) { // The ledger pointed by this position does not exist anymore. It was deleted because it was empty. We need // to skip on the next available ledger - position = new PositionImpl(ledgerId, 0); + position = PositionFactory.create(ledgerId, 0); } return position; @@ -2398,7 +2443,7 @@ public void addWaitingEntryCallBack(WaitingEntryCallBack cb) { public void maybeUpdateCursorBeforeTrimmingConsumedLedger() { for (ManagedCursor cursor : cursors) { - PositionImpl lastAckedPosition = (PositionImpl) cursor.getMarkDeletedPosition(); + Position lastAckedPosition = cursor.getMarkDeletedPosition(); LedgerInfo currPointedLedger = ledgers.get(lastAckedPosition.getLedgerId()); LedgerInfo nextPointedLedger = Optional.ofNullable(ledgers.higherEntry(lastAckedPosition.getLedgerId())) .map(Map.Entry::getValue).orElse(null); @@ -2407,7 +2452,7 @@ public void maybeUpdateCursorBeforeTrimmingConsumedLedger() { if (nextPointedLedger != null) { if (lastAckedPosition.getEntryId() != -1 && lastAckedPosition.getEntryId() + 1 >= currPointedLedger.getEntries()) { - lastAckedPosition = new PositionImpl(nextPointedLedger.getLedgerId(), -1); + lastAckedPosition = PositionFactory.create(nextPointedLedger.getLedgerId(), -1); } } else { log.debug("No need to reset cursor: {}, current ledger is the last ledger.", cursor); @@ -2447,13 +2492,12 @@ private void scheduleDeferredTrimming(boolean isTruncate, CompletableFuture p 100, TimeUnit.MILLISECONDS); } - private void maybeOffloadInBackground(CompletableFuture promise) { - if (config.getLedgerOffloader() == null || config.getLedgerOffloader() == NullLedgerOffloader.INSTANCE - || config.getLedgerOffloader().getOffloadPolicies() == null) { + public void maybeOffloadInBackground(CompletableFuture promise) { + if (getOffloadPoliciesIfAppendable().isEmpty()) { return; } - final OffloadPoliciesImpl policies = config.getLedgerOffloader().getOffloadPolicies(); + final OffloadPolicies policies = config.getLedgerOffloader().getOffloadPolicies(); final long offloadThresholdInBytes = Optional.ofNullable(policies.getManagedLedgerOffloadThresholdInBytes()).orElse(-1L); final long offloadThresholdInSeconds = @@ -2464,9 +2508,8 @@ private void maybeOffloadInBackground(CompletableFuture promise) { } private void maybeOffload(long offloadThresholdInBytes, long offloadThresholdInSeconds, - CompletableFuture finalPromise) { - if (config.getLedgerOffloader() == null || config.getLedgerOffloader() == NullLedgerOffloader.INSTANCE - || config.getLedgerOffloader().getOffloadPolicies() == null) { + CompletableFuture finalPromise) { + if (getOffloadPoliciesIfAppendable().isEmpty()) { String msg = String.format("[%s] Nothing to offload due to offloader or offloadPolicies is NULL", name); finalPromise.completeExceptionally(new IllegalArgumentException(msg)); return; @@ -2485,7 +2528,7 @@ private void maybeOffload(long offloadThresholdInBytes, long offloadThresholdInS return; } - CompletableFuture unlockingPromise = new CompletableFuture<>(); + CompletableFuture unlockingPromise = new CompletableFuture<>(); unlockingPromise.whenComplete((res, ex) -> { offloadMutex.unlock(); if (ex != null) { @@ -2531,7 +2574,7 @@ private void maybeOffload(long offloadThresholdInBytes, long offloadThresholdInS + ", total size = {}, already offloaded = {}, to offload = {}", name, toOffload.stream().map(LedgerInfo::getLedgerId).collect(Collectors.toList()), sizeSummed, alreadyOffloadedSize, toOffloadSize); - offloadLoop(unlockingPromise, toOffload, PositionImpl.LATEST, Optional.empty()); + offloadLoop(unlockingPromise, toOffload, PositionFactory.LATEST, Optional.empty()); } else { // offloadLoop will complete immediately with an empty list to offload log.debug("[{}] Nothing to offload, total size = {}, already offloaded = {}, " @@ -2539,25 +2582,24 @@ private void maybeOffload(long offloadThresholdInBytes, long offloadThresholdInS + "managedLedgerOffloadThresholdInSeconds:{}]", name, sizeSummed, alreadyOffloadedSize, offloadThresholdInBytes, TimeUnit.MILLISECONDS.toSeconds(offloadTimeThresholdMillis)); - unlockingPromise.complete(PositionImpl.LATEST); + unlockingPromise.complete(PositionFactory.LATEST); } } - private boolean hasLedgerRetentionExpired(long ledgerTimestamp) { - return config.getRetentionTimeMillis() >= 0 - && clock.millis() - ledgerTimestamp > config.getRetentionTimeMillis(); + private boolean hasLedgerRetentionExpired(long retentionTimeMs, long ledgerTimestamp) { + return retentionTimeMs >= 0 && clock.millis() - ledgerTimestamp > retentionTimeMs; } - private boolean isLedgerRetentionOverSizeQuota(long sizeToDelete) { + private boolean isLedgerRetentionOverSizeQuota(long retentionSizeInMB, long totalSizeOfML, long sizeToDelete) { // Handle the -1 size limit as "infinite" size quota - return config.getRetentionSizeInMB() >= 0 - && TOTAL_SIZE_UPDATER.get(this) - sizeToDelete >= config.getRetentionSizeInMB() * MegaByte; + return retentionSizeInMB >= 0 && totalSizeOfML - sizeToDelete >= retentionSizeInMB * MegaByte; } boolean isOffloadedNeedsDelete(OffloadContext offload, Optional offloadPolicies) { long elapsedMs = clock.millis() - offload.getTimestamp(); return offloadPolicies.filter(policies -> offload.getComplete() && !offload.getBookkeeperDeleted() && policies.getManagedLedgerOffloadDeletionLagInMillis() != null + && policies.getManagedLedgerOffloadDeletionLagInMillis() >= 0 && elapsedMs > policies.getManagedLedgerOffloadDeletionLagInMillis()).isPresent(); } @@ -2570,6 +2612,16 @@ void internalTrimConsumedLedgers(CompletableFuture promise) { internalTrimLedgers(false, promise); } + private Optional getOffloadPoliciesIfAppendable() { + LedgerOffloader ledgerOffloader = config.getLedgerOffloader(); + if (ledgerOffloader == null + || !ledgerOffloader.isAppendable() + || ledgerOffloader.getOffloadPolicies() == null) { + return Optional.empty(); + } + return Optional.ofNullable(ledgerOffloader.getOffloadPolicies()); + } + void internalTrimLedgers(boolean isTruncate, CompletableFuture promise) { if (!factory.isMetadataServiceAvailable()) { // Defer trimming of ledger if we cannot connect to metadata service @@ -2585,10 +2637,7 @@ void internalTrimLedgers(boolean isTruncate, CompletableFuture promise) { List ledgersToDelete = new ArrayList<>(); List offloadedLedgersToDelete = new ArrayList<>(); - Optional optionalOffloadPolicies = Optional.ofNullable(config.getLedgerOffloader() != null - && config.getLedgerOffloader() != NullLedgerOffloader.INSTANCE - ? config.getLedgerOffloader().getOffloadPolicies() - : null); + Optional optionalOffloadPolicies = getOffloadPoliciesIfAppendable(); synchronized (this) { if (log.isDebugEnabled()) { log.debug("[{}] Start TrimConsumedLedgers. ledgers={} totalSize={}", name, ledgers.keySet(), @@ -2610,15 +2659,29 @@ void internalTrimLedgers(boolean isTruncate, CompletableFuture promise) { } long slowestReaderLedgerId = -1; + final LazyLoadableValue slowestNonDurationLedgerId = + new LazyLoadableValue(() -> getTheSlowestNonDurationReadPosition().getLedgerId()); + final long retentionSizeInMB = config.getRetentionSizeInMB(); + final long retentionTimeMs = config.getRetentionTimeMillis(); + final long totalSizeOfML = TOTAL_SIZE_UPDATER.get(this); if (!cursors.hasDurableCursors()) { // At this point the lastLedger will be pointing to the // ledger that has just been closed, therefore the +1 to // include lastLedger in the trimming. slowestReaderLedgerId = currentLedger.getId() + 1; } else { - PositionImpl slowestReaderPosition = cursors.getSlowestReaderPosition(); + Position slowestReaderPosition = cursors.getSlowestReaderPosition(); if (slowestReaderPosition != null) { - slowestReaderLedgerId = slowestReaderPosition.getLedgerId(); + // The slowest reader position is the mark delete position. + // If the slowest reader position point the last entry in the ledger x, + // the slowestReaderLedgerId should be x + 1 and the ledger x could be deleted. + LedgerInfo ledgerInfo = ledgers.get(slowestReaderPosition.getLedgerId()); + if (ledgerInfo != null && ledgerInfo.getLedgerId() != currentLedger.getId() + && ledgerInfo.getEntries() == slowestReaderPosition.getEntryId() + 1) { + slowestReaderLedgerId = slowestReaderPosition.getLedgerId() + 1; + } else { + slowestReaderLedgerId = slowestReaderPosition.getLedgerId(); + } } else { promise.completeExceptionally(new ManagedLedgerException("Couldn't find reader position")); trimmerMutex.unlock(); @@ -2632,7 +2695,10 @@ void internalTrimLedgers(boolean isTruncate, CompletableFuture promise) { long totalSizeToDelete = 0; // skip ledger if retention constraint met - for (LedgerInfo ls : ledgers.headMap(slowestReaderLedgerId, false).values()) { + Iterator ledgerInfoIterator = + ledgers.headMap(slowestReaderLedgerId, false).values().iterator(); + while (ledgerInfoIterator.hasNext()){ + LedgerInfo ls = ledgerInfoIterator.next(); // currentLedger can not be deleted if (ls.getLedgerId() == currentLedger.getId()) { if (log.isDebugEnabled()) { @@ -2652,8 +2718,9 @@ void internalTrimLedgers(boolean isTruncate, CompletableFuture promise) { } totalSizeToDelete += ls.getSize(); - boolean overRetentionQuota = isLedgerRetentionOverSizeQuota(totalSizeToDelete); - boolean expired = hasLedgerRetentionExpired(ls.getTimestamp()); + boolean overRetentionQuota = isLedgerRetentionOverSizeQuota(retentionSizeInMB, totalSizeOfML, + totalSizeToDelete); + boolean expired = hasLedgerRetentionExpired(retentionTimeMs, ls.getTimestamp()); if (log.isDebugEnabled()) { log.debug( "[{}] Checking ledger {} -- time-old: {} sec -- " @@ -2670,14 +2737,19 @@ void internalTrimLedgers(boolean isTruncate, CompletableFuture promise) { } ledgersToDelete.add(ls); } else { - if (ls.getLedgerId() < getTheSlowestNonDurationReadPosition().getLedgerId()) { - // once retention constraint has been met, skip check - if (log.isDebugEnabled()) { - log.debug("[{}] Ledger {} not deleted. Neither expired nor over-quota", name, - ls.getLedgerId()); - } - invalidateReadHandle(ls.getLedgerId()); + // once retention constraint has been met, skip check + if (log.isDebugEnabled()) { + log.debug("[{}] Ledger {} not deleted. Neither expired nor over-quota", name, ls.getLedgerId()); } + releaseReadHandleIfNoLongerRead(ls.getLedgerId(), slowestNonDurationLedgerId.getValue()); + break; + } + } + + while (ledgerInfoIterator.hasNext()) { + LedgerInfo ls = ledgerInfoIterator.next(); + if (!releaseReadHandleIfNoLongerRead(ls.getLedgerId(), slowestNonDurationLedgerId.getValue())) { + break; } } @@ -2763,8 +2835,32 @@ public void operationFailed(MetaStoreException e) { } } + @Override + public void rolloverCursorsInBackground() { + if (cursors.hasDurableCursors()) { + executor.execute(() -> { + cursors.forEach(ManagedCursor::periodicRollover); + }); + } + } + + /** + * @param ledgerId the ledger handle which maybe will be released. + * @return if the ledger handle was released. + */ + private boolean releaseReadHandleIfNoLongerRead(long ledgerId, long slowestNonDurationLedgerId) { + if (ledgerId < slowestNonDurationLedgerId) { + if (log.isDebugEnabled()) { + log.debug("[{}] Ledger {} no longer needs to be read, close the cached readHandle", name, ledgerId); + } + invalidateReadHandle(ledgerId); + return true; + } + return false; + } + protected void doDeleteLedgers(List ledgersToDelete) { - PositionImpl currentLastConfirmedEntry = lastConfirmedEntry; + Position currentLastConfirmedEntry = lastConfirmedEntry; // Update metadata for (LedgerInfo ls : ledgersToDelete) { if (currentLastConfirmedEntry != null && ls.getLedgerId() == currentLastConfirmedEntry.getLedgerId()) { @@ -2794,21 +2890,20 @@ void advanceCursorsIfNecessary(List ledgersToDelete) throws LedgerNo return; } - // need to move mark delete for non-durable cursors to the first ledger NOT marked for deletion - // calling getNumberOfEntries latter for a ledger that is already deleted will be problematic and return - // incorrect results - Long firstNonDeletedLedger = ledgers.higherKey(ledgersToDelete.get(ledgersToDelete.size() - 1).getLedgerId()); - if (firstNonDeletedLedger == null) { - throw new LedgerNotExistException("First non deleted Ledger is not found"); + // Just ack messages like a consumer. Normally, consumers will not confirm a position that does not exist, so + // find the latest existing position to ack. + Position highestPositionToDelete = calculateLastEntryInLedgerList(ledgersToDelete); + if (highestPositionToDelete == null) { + log.warn("[{}] The ledgers to be trim are all empty, skip to advance non-durable cursors: {}", + name, ledgersToDelete); + return; } - PositionImpl highestPositionToDelete = new PositionImpl(firstNonDeletedLedger, -1); - cursors.forEach(cursor -> { // move the mark delete position to the highestPositionToDelete only if it is smaller than the add confirmed // to prevent the edge case where the cursor is caught up to the latest and highestPositionToDelete may be // larger than the last add confirmed - if (highestPositionToDelete.compareTo((PositionImpl) cursor.getMarkDeletedPosition()) > 0 - && highestPositionToDelete.compareTo((PositionImpl) cursor.getManagedLedger() + if (highestPositionToDelete.compareTo(cursor.getMarkDeletedPosition()) > 0 + && highestPositionToDelete.compareTo(cursor.getManagedLedger() .getLastConfirmedEntry()) <= 0 && !(!cursor.isDurable() && cursor instanceof NonDurableCursorImpl && ((NonDurableCursorImpl) cursor).isReadCompacted())) { cursor.asyncMarkDelete(highestPositionToDelete, cursor.getProperties(), new MarkDeleteCallback() { @@ -2826,6 +2921,19 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { }); } + /** + * @return null if all ledgers is empty. + */ + private Position calculateLastEntryInLedgerList(List ledgersToDelete) { + for (int i = ledgersToDelete.size() - 1; i >= 0; i--) { + LedgerInfo ledgerInfo = ledgersToDelete.get(i); + if (ledgerInfo != null && ledgerInfo.hasEntries() && ledgerInfo.getEntries() > 0) { + return PositionFactory.create(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1); + } + } + return null; + } + /** * Delete this ManagedLedger completely from the system. * @@ -2874,9 +2982,8 @@ public void asyncDelete(final DeleteLedgerCallback callback, final Object ctx) { truncateFuture.whenComplete((ignore, exc) -> { if (exc != null) { log.error("[{}] Error truncating ledger for deletion", name, exc); - callback.deleteLedgerFailed(exc instanceof ManagedLedgerException - ? (ManagedLedgerException) exc : new ManagedLedgerException(exc), - ctx); + callback.deleteLedgerFailed(ManagedLedgerException.getManagedLedgerException( + FutureUtil.unwrapCompletionException(exc)), ctx); } else { asyncDeleteInternal(callback, ctx); } @@ -3057,11 +3164,13 @@ public void offloadFailed(ManagedLedgerException e, Object ctx) { @Override public void asyncOffloadPrefix(Position pos, OffloadCallback callback, Object ctx) { - if (config.getLedgerOffloader() != null && config.getLedgerOffloader() == NullLedgerOffloader.INSTANCE) { - callback.offloadFailed(new ManagedLedgerException("NullLedgerOffloader"), ctx); + LedgerOffloader ledgerOffloader = config.getLedgerOffloader(); + if (ledgerOffloader != null && !ledgerOffloader.isAppendable()) { + String msg = String.format("[%s] does not support offload", ledgerOffloader.getClass().getSimpleName()); + callback.offloadFailed(new ManagedLedgerException(msg), ctx); return; } - PositionImpl requestOffloadTo = (PositionImpl) pos; + Position requestOffloadTo = pos; if (!isValidPosition(requestOffloadTo) // Also consider the case where the last ledger is currently // empty. In this the passed position is not technically @@ -3075,7 +3184,7 @@ public void asyncOffloadPrefix(Position pos, OffloadCallback callback, Object ct return; } - PositionImpl firstUnoffloaded; + Position firstUnoffloaded; Queue ledgersToOffload = new ConcurrentLinkedQueue<>(); synchronized (this) { @@ -3114,7 +3223,7 @@ public void asyncOffloadPrefix(Position pos, OffloadCallback callback, Object ct break; } } - firstUnoffloaded = PositionImpl.get(firstLedgerRetained, 0); + firstUnoffloaded = PositionFactory.create(firstLedgerRetained, 0); } if (ledgersToOffload.isEmpty()) { @@ -3127,7 +3236,7 @@ public void asyncOffloadPrefix(Position pos, OffloadCallback callback, Object ct log.info("[{}] Going to offload ledgers {}", name, ledgersToOffload.stream().map(LedgerInfo::getLedgerId).collect(Collectors.toList())); - CompletableFuture promise = new CompletableFuture<>(); + CompletableFuture promise = new CompletableFuture<>(); promise.whenComplete((result, exception) -> { offloadMutex.unlock(); if (exception != null) { @@ -3143,8 +3252,8 @@ public void asyncOffloadPrefix(Position pos, OffloadCallback callback, Object ct } } - private void offloadLoop(CompletableFuture promise, Queue ledgersToOffload, - PositionImpl firstUnoffloaded, Optional firstError) { + void offloadLoop(CompletableFuture promise, Queue ledgersToOffload, + Position firstUnoffloaded, Optional firstError) { State currentState = getState(); if (currentState == State.Closed) { promise.completeExceptionally(new ManagedLedgerAlreadyClosedException( @@ -3176,7 +3285,7 @@ private void offloadLoop(CompletableFuture promise, Queue config.getLedgerOffloader().offload(readHandle, uuid, extraMetadata)) .thenCompose((ignore) -> { return Retries.run(Backoff.exponentialJittered(TimeUnit.SECONDS.toMillis(1), - TimeUnit.SECONDS.toHours(1)).limit(10), + TimeUnit.HOURS.toMillis(1)).limit(10), FAIL_ON_CONFLICT, () -> completeLedgerInfoForOffloaded(ledgerId, uuid), scheduledExecutor, name) @@ -3191,6 +3300,7 @@ private void offloadLoop(CompletableFuture promise, Queue promise, Queue 0) { newFirstUnoffloaded = firstUnoffloaded; } @@ -3395,10 +3505,10 @@ private CompletableFuture completeLedgerInfoForOffloaded(long ledgerId, UU * the position range * @return the count of entries */ - long getNumberOfEntries(Range range) { - PositionImpl fromPosition = range.lowerEndpoint(); + public long getNumberOfEntries(Range range) { + Position fromPosition = range.lowerEndpoint(); boolean fromIncluded = range.lowerBoundType() == BoundType.CLOSED; - PositionImpl toPosition = range.upperEndpoint(); + Position toPosition = range.upperEndpoint(); boolean toIncluded = range.upperBoundType() == BoundType.CLOSED; if (fromPosition.getLedgerId() == toPosition.getLedgerId()) { @@ -3442,7 +3552,8 @@ long getNumberOfEntries(Range range) { * specifies whether to include the start position in calculating the distance * @return the new position that is n entries ahead */ - public PositionImpl getPositionAfterN(final PositionImpl startPosition, long n, PositionBound startRange) { + @Override + public Position getPositionAfterN(final Position startPosition, long n, PositionBound startRange) { long entriesToSkip = n; long currentLedgerId; long currentEntryId; @@ -3450,7 +3561,7 @@ public PositionImpl getPositionAfterN(final PositionImpl startPosition, long n, currentLedgerId = startPosition.getLedgerId(); currentEntryId = startPosition.getEntryId(); } else { - PositionImpl nextValidPosition = getNextValidPosition(startPosition); + Position nextValidPosition = getNextValidPosition(startPosition); currentLedgerId = nextValidPosition.getLedgerId(); currentEntryId = nextValidPosition.getEntryId(); } @@ -3491,7 +3602,7 @@ public PositionImpl getPositionAfterN(final PositionImpl startPosition, long n, } } - PositionImpl positionToReturn = getPreviousPosition(PositionImpl.get(currentLedgerId, currentEntryId)); + Position positionToReturn = getPreviousPosition(PositionFactory.create(currentLedgerId, currentEntryId)); if (positionToReturn.compareTo(lastConfirmedEntry) > 0) { positionToReturn = lastConfirmedEntry; } @@ -3504,6 +3615,34 @@ public PositionImpl getPositionAfterN(final PositionImpl startPosition, long n, return positionToReturn; } + public boolean isNoMessagesAfterPos(Position pos) { + Position lac = getLastConfirmedEntry(); + return isNoMessagesAfterPosForSpecifiedLac(lac, pos); + } + + private boolean isNoMessagesAfterPosForSpecifiedLac(Position specifiedLac, Position pos) { + if (pos.compareTo(specifiedLac) >= 0) { + return true; + } + if (specifiedLac.getEntryId() < 0) { + // Calculate the meaningful LAC. + Position actLac = getPreviousPosition(specifiedLac); + if (actLac.getEntryId() >= 0) { + return pos.compareTo(actLac) >= 0; + } else { + // If the actual LAC is still not meaningful. + if (actLac.equals(specifiedLac)) { + // No entries in maneged ledger. + return true; + } else { + // Continue to find a valid LAC. + return isNoMessagesAfterPosForSpecifiedLac(actLac, pos); + } + } + } + return false; + } + /** * Get the entry position that come before the specified position in the message stream, using information from the * ledger list and each ledger entries count. @@ -3512,9 +3651,10 @@ public PositionImpl getPositionAfterN(final PositionImpl startPosition, long n, * the current position * @return the previous position */ - public PositionImpl getPreviousPosition(PositionImpl position) { + @Override + public Position getPreviousPosition(Position position) { if (position.getEntryId() > 0) { - return PositionImpl.get(position.getLedgerId(), position.getEntryId() - 1); + return PositionFactory.create(position.getLedgerId(), position.getEntryId() - 1); } // The previous position will be the last position of an earlier ledgers @@ -3523,19 +3663,19 @@ public PositionImpl getPreviousPosition(PositionImpl position) { final Map.Entry firstEntry = headMap.firstEntry(); if (firstEntry == null) { // There is no previous ledger, return an invalid position in the current ledger - return PositionImpl.get(position.getLedgerId(), -1); + return PositionFactory.create(position.getLedgerId(), -1); } // We need to find the most recent non-empty ledger for (long ledgerId : headMap.descendingKeySet()) { LedgerInfo li = headMap.get(ledgerId); if (li != null && li.getEntries() > 0) { - return PositionImpl.get(li.getLedgerId(), li.getEntries() - 1); + return PositionFactory.create(li.getLedgerId(), li.getEntries() - 1); } } // in case there are only empty ledgers, we return a position in the first one - return PositionImpl.get(firstEntry.getKey(), -1); + return PositionFactory.create(firstEntry.getKey(), -1); } /** @@ -3545,24 +3685,31 @@ public PositionImpl getPreviousPosition(PositionImpl position) { * the position to validate * @return true if the position is valid, false otherwise */ - public boolean isValidPosition(PositionImpl position) { - PositionImpl last = lastConfirmedEntry; + public boolean isValidPosition(Position position) { + Position lac = lastConfirmedEntry; if (log.isDebugEnabled()) { - log.debug("IsValid position: {} -- last: {}", position, last); + log.debug("IsValid position: {} -- last: {}", position, lac); } - if (position.getEntryId() < 0) { + if (!ledgers.containsKey(position.getLedgerId())){ return false; - } else if (position.getLedgerId() > last.getLedgerId()) { + } else if (position.getEntryId() < 0) { return false; - } else if (position.getLedgerId() == last.getLedgerId()) { - return position.getEntryId() <= (last.getEntryId() + 1); + } else if (currentLedger != null && position.getLedgerId() == currentLedger.getId()) { + // If current ledger is empty, the largest read position can be "{current_ledger: 0}". + // Else, the read position can be set to "{LAC + 1}" when subscribe at LATEST, + return (position.getLedgerId() == lac.getLedgerId() && position.getEntryId() <= lac.getEntryId() + 1) + || position.getEntryId() == 0; + } else if (position.getLedgerId() == lac.getLedgerId()) { + // The ledger witch maintains LAC was closed, and there is an empty current ledger. + // If entry id is larger than LAC, it should be "{current_ledger: 0}". + return position.getEntryId() <= lac.getEntryId(); } else { // Look in the ledgers map LedgerInfo ls = ledgers.get(position.getLedgerId()); if (ls == null) { - if (position.getLedgerId() < last.getLedgerId()) { + if (position.getLedgerId() < lac.getLedgerId()) { // Pointing to a non-existing ledger that is older than the current ledger is invalid return false; } else { @@ -3583,33 +3730,28 @@ public Long getNextValidLedger(long ledgerId) { return ledgers.ceilingKey(ledgerId + 1); } - public PositionImpl getNextValidPosition(final PositionImpl position) { + @Override + public Position getNextValidPosition(final Position position) { return getValidPositionAfterSkippedEntries(position, 1); } - public PositionImpl getValidPositionAfterSkippedEntries(final PositionImpl position, int skippedEntryNum) { - PositionImpl skippedPosition = position.getPositionAfterEntries(skippedEntryNum); + + public Position getValidPositionAfterSkippedEntries(final Position position, int skippedEntryNum) { + Position skippedPosition = position.getPositionAfterEntries(skippedEntryNum); while (!isValidPosition(skippedPosition)) { Long nextLedgerId = ledgers.ceilingKey(skippedPosition.getLedgerId() + 1); + // This means it has jumped to the last position if (nextLedgerId == null) { + if (currentLedgerEntries == 0 && currentLedger != null) { + return PositionFactory.create(currentLedger.getId(), 0); + } return lastConfirmedEntry.getNext(); } - skippedPosition = PositionImpl.get(nextLedgerId, 0); + skippedPosition = PositionFactory.create(nextLedgerId, 0); } return skippedPosition; } - public PositionImpl getNextValidPositionInternal(final PositionImpl position) { - PositionImpl nextPosition = position.getNext(); - while (!isValidPosition(nextPosition)) { - Long nextLedgerId = ledgers.ceilingKey(nextPosition.getLedgerId() + 1); - if (nextLedgerId == null) { - throw new NullPointerException("nextLedgerId is null. No valid next position after " + position); - } - nextPosition = PositionImpl.get(nextLedgerId, 0); - } - return nextPosition; - } - public PositionImpl getFirstPosition() { + public Position getFirstPosition() { Long ledgerId = ledgers.firstKey(); if (ledgerId == null) { return null; @@ -3618,10 +3760,10 @@ public PositionImpl getFirstPosition() { checkState(ledgers.get(ledgerId).getEntries() == 0); ledgerId = lastConfirmedEntry.getLedgerId(); } - return new PositionImpl(ledgerId, -1); + return PositionFactory.create(ledgerId, -1); } - PositionImpl getLastPosition() { + Position getLastPosition() { return lastConfirmedEntry; } @@ -3630,16 +3772,16 @@ public ManagedCursor getSlowestConsumer() { return cursors.getSlowestReader(); } - PositionImpl getMarkDeletePositionOfSlowestConsumer() { + Position getMarkDeletePositionOfSlowestConsumer() { ManagedCursor slowestCursor = getSlowestConsumer(); - return slowestCursor == null ? null : (PositionImpl) slowestCursor.getMarkDeletedPosition(); + return slowestCursor == null ? null : slowestCursor.getMarkDeletedPosition(); } /** * Get the last position written in the managed ledger, alongside with the associated counter. */ - Pair getLastPositionAndCounter() { - PositionImpl pos; + Pair getLastPositionAndCounter() { + Position pos; long count; do { @@ -3655,10 +3797,10 @@ Pair getLastPositionAndCounter() { /** * Get the first position written in the managed ledger, alongside with the associated counter. */ - Pair getFirstPositionAndCounter() { - PositionImpl pos; + Pair getFirstPositionAndCounter() { + Position pos; long count; - Pair lastPositionAndCounter; + Pair lastPositionAndCounter; do { pos = getFirstPosition(); @@ -3677,7 +3819,7 @@ public void activateCursor(ManagedCursor cursor) { ? cursor.getMarkDeletedPosition() : cursor.getReadPosition(); if (positionForOrdering == null) { - positionForOrdering = PositionImpl.EARLIEST; + positionForOrdering = PositionFactory.EARLIEST; } activeCursors.add(cursor, positionForOrdering); } @@ -3701,6 +3843,10 @@ public void removeWaitingCursor(ManagedCursor cursor) { this.waitingCursors.remove(cursor); } + public void addWaitingCursor(ManagedCursorImpl cursor) { + this.waitingCursors.add(cursor); + } + public boolean isCursorActive(ManagedCursor cursor) { return activeCursors.get(cursor.getName()) != null; } @@ -3738,6 +3884,7 @@ public List getLedgersInfoAsList() { return Lists.newArrayList(ledgers.values()); } + @Override public NavigableMap getLedgersInfo() { return ledgers; } @@ -3851,6 +3998,7 @@ public int getWaitingCursorsCount() { return waitingCursors.size(); } + @Override public int getPendingAddEntriesCount() { return pendingAddEntries.size(); } @@ -3864,6 +4012,7 @@ public State getState() { return STATE_UPDATER.get(this); } + @Override public long getCacheSize() { return entryCache.getSize(); } @@ -3916,6 +4065,8 @@ public static ManagedLedgerException createManagedLedgerException(int bkErrorCod public static ManagedLedgerException createManagedLedgerException(Throwable t) { if (t instanceof org.apache.bookkeeper.client.api.BKException) { return createManagedLedgerException(((org.apache.bookkeeper.client.api.BKException) t).getCode()); + } else if (t instanceof ManagedLedgerException) { + return (ManagedLedgerException) t; } else if (t instanceof CompletionException && !(t.getCause() instanceof CompletionException) /* check to avoid stackoverlflow */) { return createManagedLedgerException(t.getCause()); @@ -3937,7 +4088,7 @@ public static ManagedLedgerException createManagedLedgerException(Throwable t) { */ protected void asyncCreateLedger(BookKeeper bookKeeper, ManagedLedgerConfig config, DigestType digestType, CreateCallback cb, Map metadata) { - AtomicBoolean ledgerCreated = new AtomicBoolean(false); + CompletableFuture ledgerFutureHook = new CompletableFuture<>(); Map finalMetadata = new HashMap<>(); finalMetadata.putAll(ledgerMetadata); finalMetadata.putAll(metadata); @@ -3950,33 +4101,39 @@ protected void asyncCreateLedger(BookKeeper bookKeeper, ManagedLedgerConfig conf )); } catch (EnsemblePlacementPolicyConfig.ParseEnsemblePlacementPolicyConfigException e) { log.error("[{}] Serialize the placement configuration failed", name, e); - cb.createComplete(Code.UnexpectedConditionException, null, ledgerCreated); + cb.createComplete(Code.UnexpectedConditionException, null, ledgerFutureHook); return; } } createdLedgerCustomMetadata = finalMetadata; - try { bookKeeper.asyncCreateLedger(config.getEnsembleSize(), config.getWriteQuorumSize(), - config.getAckQuorumSize(), digestType, config.getPassword(), cb, ledgerCreated, finalMetadata); + config.getAckQuorumSize(), digestType, config.getPassword(), cb, ledgerFutureHook, finalMetadata); } catch (Throwable cause) { log.error("[{}] Encountered unexpected error when creating ledger", name, cause); - cb.createComplete(Code.UnexpectedConditionException, null, ledgerCreated); + ledgerFutureHook.completeExceptionally(cause); + cb.createComplete(Code.UnexpectedConditionException, null, ledgerFutureHook); return; } - scheduledExecutor.schedule(() -> { - if (!ledgerCreated.get()) { + + ScheduledFuture timeoutChecker = scheduledExecutor.schedule(() -> { + if (!ledgerFutureHook.isDone() + && ledgerFutureHook.completeExceptionally(new TimeoutException(name + " Create ledger timeout"))) { if (log.isDebugEnabled()) { log.debug("[{}] Timeout creating ledger", name); } - cb.createComplete(BKException.Code.TimeoutException, null, ledgerCreated); + cb.createComplete(BKException.Code.TimeoutException, null, ledgerFutureHook); } else { if (log.isDebugEnabled()) { log.debug("[{}] Ledger already created when timeout task is triggered", name); } } }, config.getMetadataOperationsTimeoutSeconds(), TimeUnit.SECONDS); + + ledgerFutureHook.whenComplete((ignore, ex) -> { + timeoutChecker.cancel(false); + }); } public Clock getClock() { @@ -3985,16 +4142,12 @@ public Clock getClock() { /** * check if ledger-op task is already completed by timeout-task. If completed then delete the created ledger - * - * @param rc - * @param lh - * @param ctx * @return */ protected boolean checkAndCompleteLedgerOpTask(int rc, LedgerHandle lh, Object ctx) { - if (ctx instanceof AtomicBoolean) { + if (ctx instanceof CompletableFuture) { // ledger-creation is already timed out and callback is already completed so, delete this ledger and return. - if (((AtomicBoolean) (ctx)).compareAndSet(false, true)) { + if (((CompletableFuture) ctx).complete(lh)) { return false; } else { if (rc == BKException.Code.OK) { @@ -4036,13 +4189,14 @@ private void checkAddTimeout() { } OpAddEntry opAddEntry = pendingAddEntries.peek(); if (opAddEntry != null) { - final long finalAddOpCount = opAddEntry.addOpCount; boolean isTimedOut = opAddEntry.lastInitTime != -1 && TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - opAddEntry.lastInitTime) >= timeoutSec; if (isTimedOut) { - log.error("Failed to add entry for ledger {} in time-out {} sec", - (opAddEntry.ledger != null ? opAddEntry.ledger.getId() : -1), timeoutSec); - opAddEntry.handleAddTimeoutFailure(opAddEntry.ledger, finalAddOpCount); + log.warn("[{}] Failed to add entry {}:{} in time-out {} sec", this.name, + opAddEntry.ledger != null ? opAddEntry.ledger.getId() : -1, + opAddEntry.entryId, timeoutSec); + currentLedgerTimeoutTriggered.set(true); + opAddEntry.handleAddFailure(opAddEntry.ledger); } } } @@ -4276,7 +4430,7 @@ public CompletableFuture getManagedLedgerInternalSta List ledgersInfos = new ArrayList<>(this.getLedgersInfo().values()); // add asynchronous metadata retrieval operations to a hashmap - Map> ledgerMetadataFutures = new HashMap(); + Map> ledgerMetadataFutures = new HashMap(); if (includeLedgerMetadata) { ledgersInfos.forEach(li -> { long ledgerId = li.getLedgerId(); @@ -4287,24 +4441,55 @@ public CompletableFuture getManagedLedgerInternalSta }); } + + CompletableFuture> bookiesFuture; + if (includeLedgerMetadata) { + RegistrationClient registrationClient = bookKeeper.getMetadataClientDriver().getRegistrationClient(); + bookiesFuture = registrationClient.getReadOnlyBookies() + .thenCombine(registrationClient.getWritableBookies(), (readOnlyBookies, writableBookies) -> { + Set bookies = new HashSet<>(); + bookies.addAll(readOnlyBookies.getValue()); + bookies.addAll(writableBookies.getValue()); + return bookies; + }); + } else { + bookiesFuture = CompletableFuture.completedFuture(null); + } + // wait until metadata has been retrieved - FutureUtil.waitForAll(ledgerMetadataFutures.values()).thenAccept(__ -> { - stats.ledgers = new ArrayList(); - ledgersInfos.forEach(li -> { - ManagedLedgerInternalStats.LedgerInfo info = new ManagedLedgerInternalStats.LedgerInfo(); - info.ledgerId = li.getLedgerId(); - info.entries = li.getEntries(); - info.size = li.getSize(); - info.offloaded = li.hasOffloadContext() && li.getOffloadContext().getComplete(); - if (includeLedgerMetadata) { - // lookup metadata from the hashmap which contains completed async operations - info.metadata = ledgerMetadataFutures.get(li.getLedgerId()).getNow(null); - } - stats.ledgers.add(info); + bookiesFuture.thenCompose(bookies -> + FutureUtil.waitForAll(ledgerMetadataFutures.values()).thenAccept(__ -> { + stats.ledgers = new ArrayList<>(); + ledgersInfos.forEach(li -> { + ManagedLedgerInternalStats.LedgerInfo info = new ManagedLedgerInternalStats.LedgerInfo(); + info.ledgerId = li.getLedgerId(); + info.entries = li.getEntries(); + info.size = li.getSize(); + info.offloaded = li.hasOffloadContext() && li.getOffloadContext().getComplete(); + if (includeLedgerMetadata) { + // lookup metadata from the hashmap which contains completed async operations + LedgerMetadata lm = ledgerMetadataFutures.get(li.getLedgerId()).getNow(null); + if (lm == null) { + info.metadata = null; + info.underReplicated = false; + } else { + info.metadata = lm.toSafeString(); + Set ensemble = lm.getAllEnsembles().values().stream() + .flatMap(Collection::stream) + .collect(Collectors.toSet()); + if (bookies != null) { + info.underReplicated = !bookies.contains(ensemble); + } + } + } + stats.ledgers.add(info); + }); + statFuture.complete(stats); + })) + .exceptionally(e -> { + statFuture.completeExceptionally(e); + return null; }); - statFuture.complete(stats); - }); - return statFuture; } @@ -4353,9 +4538,11 @@ private void cancelScheduledTasks() { } @Override - public void checkInactiveLedgerAndRollOver() { - long currentTimeMs = System.currentTimeMillis(); - if (inactiveLedgerRollOverTimeMs > 0 && currentTimeMs > (lastAddEntryTimeMs + inactiveLedgerRollOverTimeMs)) { + public boolean checkInactiveLedgerAndRollOver() { + if (factory.isMetadataServiceAvailable() + && currentLedgerEntries > 0 + && inactiveLedgerRollOverTimeMs > 0 + && System.currentTimeMillis() > (lastAddEntryTimeMs + inactiveLedgerRollOverTimeMs)) { log.info("[{}] Closing inactive ledger, last-add entry {}", name, lastAddEntryTimeMs); if (STATE_UPDATER.compareAndSet(this, State.LedgerOpened, State.ClosingLedger)) { LedgerHandle currentLedger = this.currentLedger; @@ -4376,8 +4563,10 @@ public void checkInactiveLedgerAndRollOver() { ledgerClosed(lh); // we do not create ledger here, since topic is inactive for a long time. }, null); + return true; } } + return false; } @@ -4419,10 +4608,10 @@ public void checkCursorsToCacheEntries() { } public Position getTheSlowestNonDurationReadPosition() { - PositionImpl theSlowestNonDurableReadPosition = PositionImpl.LATEST; + Position theSlowestNonDurableReadPosition = PositionFactory.LATEST; for (ManagedCursor cursor : cursors) { if (cursor instanceof NonDurableCursorImpl) { - PositionImpl readPosition = (PositionImpl) cursor.getReadPosition(); + Position readPosition = cursor.getReadPosition(); if (readPosition.compareTo(theSlowestNonDurableReadPosition) < 0) { theSlowestNonDurableReadPosition = readPosition; } @@ -4430,4 +4619,11 @@ public Position getTheSlowestNonDurationReadPosition() { } return theSlowestNonDurableReadPosition; } + + @Override + public CompletableFuture getLastDispatchablePosition(final Predicate predicate, + final Position startPosition) { + return ManagedLedgerImplUtils + .asyncGetLastValidPosition(this, predicate, startPosition); + } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java index cb3d72cc5972f..86320f9292468 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java @@ -41,6 +41,7 @@ public class ManagedLedgerMBeanImpl implements ManagedLedgerMXBean { private final Rate readEntriesOpsFailed = new Rate(); private final Rate readEntriesOpsCacheMisses = new Rate(); private final Rate markDeleteOps = new Rate(); + private final Rate entriesRead = new Rate(); private final LongAdder dataLedgerOpenOp = new LongAdder(); private final LongAdder dataLedgerCloseOp = new LongAdder(); @@ -80,6 +81,7 @@ public void refreshStats(long period, TimeUnit unit) { ledgerAddEntryLatencyStatsUsec.refresh(); ledgerSwitchLatencyStatsUsec.refresh(); entryStats.refresh(); + entriesRead.calculateRate(seconds); } public void addAddEntrySample(long size) { @@ -100,8 +102,8 @@ public void recordReadEntriesError() { readEntriesOpsFailed.recordEvent(); } - public void recordReadEntriesOpsCacheMisses() { - readEntriesOpsCacheMisses.recordEvent(); + public void recordReadEntriesOpsCacheMisses(int count, long totalSize) { + readEntriesOpsCacheMisses.recordMultipleEvents(count, totalSize); } public void addAddEntryLatencySample(long latency, TimeUnit unit) { @@ -120,6 +122,10 @@ public void addReadEntriesSample(int count, long totalSize) { readEntriesOps.recordMultipleEvents(count, totalSize); } + public void addEntriesRead(int count) { + entriesRead.recordEvent(count); + } + public void startDataLedgerOpenOp() { dataLedgerOpenOp.increment(); } @@ -189,6 +195,11 @@ public String getName() { return managedLedger.getName(); } + @Override + public long getEntriesReadTotalCount() { + return entriesRead.getTotalCount(); + } + @Override public double getAddEntryMessagesRate() { return addEntryOps.getRate(); @@ -199,11 +210,21 @@ public double getAddEntryBytesRate() { return addEntryOps.getValueRate(); } + @Override + public long getAddEntryBytesTotal() { + return addEntryOps.getTotalValue(); + } + @Override public double getAddEntryWithReplicasBytesRate() { return addEntryWithReplicasOps.getValueRate(); } + @Override + public long getAddEntryWithReplicasBytesTotal() { + return addEntryWithReplicasOps.getTotalValue(); + } + @Override public double getReadEntriesRate() { return readEntriesOps.getRate(); @@ -214,36 +235,71 @@ public double getReadEntriesBytesRate() { return readEntriesOps.getValueRate(); } + @Override + public long getReadEntriesBytesTotal() { + return readEntriesOps.getTotalValue(); + } + @Override public long getAddEntrySucceed() { return addEntryOps.getCount(); } + @Override + public long getAddEntrySucceedTotal() { + return addEntryOps.getTotalCount(); + } + @Override public long getAddEntryErrors() { return addEntryOpsFailed.getCount(); } + @Override + public long getAddEntryErrorsTotal() { + return addEntryOpsFailed.getTotalCount(); + } + @Override public long getReadEntriesSucceeded() { return readEntriesOps.getCount(); } + @Override + public long getReadEntriesSucceededTotal() { + return readEntriesOps.getTotalCount(); + } + @Override public long getReadEntriesErrors() { return readEntriesOpsFailed.getCount(); } + @Override + public long getReadEntriesErrorsTotal() { + return readEntriesOpsFailed.getTotalCount(); + } + @Override public double getReadEntriesOpsCacheMissesRate() { return readEntriesOpsCacheMisses.getRate(); } + @Override + public long getReadEntriesOpsCacheMissesTotal() { + return readEntriesOpsCacheMisses.getTotalCount(); + } + @Override public double getMarkDeleteRate() { return markDeleteOps.getRate(); } + @Override + public long getMarkDeleteTotal() { + return markDeleteOps.getTotalCount(); + } + @Override public double getEntrySizeAverage() { return entryStats.getAvg(); @@ -333,5 +389,4 @@ public PendingBookieOpsStats getPendingBookieOpsStats() { result.cursorLedgerDeleteOp = cursorLedgerDeleteOp.longValue(); return result; } - } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklog.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklog.java index a271d439e0609..9a1753c715eff 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklog.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklog.java @@ -20,27 +20,16 @@ import com.google.common.collect.BoundType; import com.google.common.collect.Range; -import com.google.protobuf.InvalidProtocolBufferException; -import java.util.Enumeration; +import java.util.ArrayList; import java.util.List; import java.util.NavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import org.apache.bookkeeper.client.AsyncCallback; -import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; -import org.apache.bookkeeper.client.BookKeeperAdmin; -import org.apache.bookkeeper.client.LedgerEntry; -import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.client.api.DigestType; -import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.ManagedLedgerFactory; +import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.proto.MLDataFormats; -import org.apache.bookkeeper.mledger.util.Errors; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; -import org.apache.pulsar.metadata.api.Stat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,11 +52,11 @@ public ManagedLedgerOfflineBacklog(DigestType digestType, byte[] password, Strin } // need a better way than to duplicate the functionality below from ML - private long getNumberOfEntries(Range range, + private long getNumberOfEntries(Range range, NavigableMap ledgers) { - PositionImpl fromPosition = range.lowerEndpoint(); + Position fromPosition = range.lowerEndpoint(); boolean fromIncluded = range.lowerBoundType() == BoundType.CLOSED; - PositionImpl toPosition = range.upperEndpoint(); + Position toPosition = range.upperEndpoint(); boolean toIncluded = range.upperBoundType() == BoundType.CLOSED; if (fromPosition.getLedgerId() == toPosition.getLedgerId()) { @@ -100,361 +89,28 @@ private long getNumberOfEntries(Range range, } } - public PersistentOfflineTopicStats getEstimatedUnloadedTopicBacklog(ManagedLedgerFactoryImpl factory, + public PersistentOfflineTopicStats getEstimatedUnloadedTopicBacklog(ManagedLedgerFactory factory, String managedLedgerName) throws Exception { return estimateUnloadedTopicBacklog(factory, TopicName.get("persistent://" + managedLedgerName)); } - public PersistentOfflineTopicStats estimateUnloadedTopicBacklog(ManagedLedgerFactoryImpl factory, - TopicName topicName) throws Exception { + public PersistentOfflineTopicStats estimateUnloadedTopicBacklog(ManagedLedgerFactory factory, + TopicName topicName) throws Exception { String managedLedgerName = topicName.getPersistenceNamingEncoding(); - long numberOfEntries = 0; - long totalSize = 0; - final NavigableMap ledgers = new ConcurrentSkipListMap<>(); final PersistentOfflineTopicStats offlineTopicStats = new PersistentOfflineTopicStats(managedLedgerName, brokerName); - - // calculate total managed ledger size and number of entries without loading the topic - readLedgerMeta(factory, topicName, ledgers); - for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ls : ledgers.values()) { - numberOfEntries += ls.getEntries(); - totalSize += ls.getSize(); - if (accurate) { - offlineTopicStats.addLedgerDetails(ls.getEntries(), ls.getTimestamp(), ls.getSize(), ls.getLedgerId()); - } - } - offlineTopicStats.totalMessages = numberOfEntries; - offlineTopicStats.storageSize = totalSize; - if (log.isDebugEnabled()) { - log.debug("[{}] Total number of entries - {} and size - {}", managedLedgerName, numberOfEntries, totalSize); - } - - // calculate per cursor message backlog - calculateCursorBacklogs(factory, topicName, ledgers, offlineTopicStats); - offlineTopicStats.statGeneratedAt.setTime(System.currentTimeMillis()); - - return offlineTopicStats; - } - - private void readLedgerMeta(final ManagedLedgerFactoryImpl factory, final TopicName topicName, - final NavigableMap ledgers) throws Exception { - String managedLedgerName = topicName.getPersistenceNamingEncoding(); - MetaStore store = factory.getMetaStore(); - BookKeeper bk = factory.getBookKeeper(); - final CountDownLatch mlMetaCounter = new CountDownLatch(1); - - store.getManagedLedgerInfo(managedLedgerName, false /* createIfMissing */, - new MetaStore.MetaStoreCallback() { - @Override - public void operationComplete(MLDataFormats.ManagedLedgerInfo mlInfo, Stat stat) { - for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ls : mlInfo.getLedgerInfoList()) { - ledgers.put(ls.getLedgerId(), ls); - } - - // find no of entries in last ledger - if (!ledgers.isEmpty()) { - final long id = ledgers.lastKey(); - AsyncCallback.OpenCallback opencb = (rc, lh, ctx1) -> { - if (log.isDebugEnabled()) { - log.debug("[{}] Opened ledger {}: {}", managedLedgerName, id, - BKException.getMessage(rc)); - } - if (rc == BKException.Code.OK) { - MLDataFormats.ManagedLedgerInfo.LedgerInfo info = - MLDataFormats.ManagedLedgerInfo.LedgerInfo - .newBuilder().setLedgerId(id).setEntries(lh.getLastAddConfirmed() + 1) - .setSize(lh.getLength()).setTimestamp(System.currentTimeMillis()).build(); - ledgers.put(id, info); - mlMetaCounter.countDown(); - } else if (Errors.isNoSuchLedgerExistsException(rc)) { - log.warn("[{}] Ledger not found: {}", managedLedgerName, ledgers.lastKey()); - ledgers.remove(ledgers.lastKey()); - mlMetaCounter.countDown(); - } else { - log.error("[{}] Failed to open ledger {}: {}", managedLedgerName, id, - BKException.getMessage(rc)); - mlMetaCounter.countDown(); - } - }; - - if (log.isDebugEnabled()) { - log.debug("[{}] Opening ledger {}", managedLedgerName, id); - } - try { - bk.asyncOpenLedgerNoRecovery(id, digestType, password, opencb, null); - } catch (Exception e) { - log.warn("[{}] Failed to open ledger {}: {}", managedLedgerName, id, e); - mlMetaCounter.countDown(); - } - } else { - log.warn("[{}] Ledger list empty", managedLedgerName); - mlMetaCounter.countDown(); - } - } - - @Override - public void operationFailed(ManagedLedgerException.MetaStoreException e) { - log.warn("[{}] Unable to obtain managed ledger metadata - {}", managedLedgerName, e); - mlMetaCounter.countDown(); - } - }); - - if (accurate) { - // block until however long it takes for operation to complete - mlMetaCounter.await(); + if (factory instanceof ManagedLedgerFactoryImpl) { + List ctx = new ArrayList<>(); + ctx.add(digestType); + ctx.add(password); + factory.estimateUnloadedTopicBacklog(offlineTopicStats, topicName, accurate, ctx); } else { - mlMetaCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS); - - } - } - - private void calculateCursorBacklogs(final ManagedLedgerFactoryImpl factory, final TopicName topicName, - final NavigableMap ledgers, - final PersistentOfflineTopicStats offlineTopicStats) throws Exception { - - if (ledgers.isEmpty()) { - return; - } - String managedLedgerName = topicName.getPersistenceNamingEncoding(); - MetaStore store = factory.getMetaStore(); - BookKeeper bk = factory.getBookKeeper(); - final CountDownLatch allCursorsCounter = new CountDownLatch(1); - final long errorInReadingCursor = -1; - ConcurrentOpenHashMap ledgerRetryMap = - ConcurrentOpenHashMap.newBuilder().build(); - - final MLDataFormats.ManagedLedgerInfo.LedgerInfo ledgerInfo = ledgers.lastEntry().getValue(); - final PositionImpl lastLedgerPosition = new PositionImpl(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1); - if (log.isDebugEnabled()) { - log.debug("[{}] Last ledger position {}", managedLedgerName, lastLedgerPosition); + Object ctx = null; + factory.estimateUnloadedTopicBacklog(offlineTopicStats, topicName, accurate, ctx); } - store.getCursors(managedLedgerName, new MetaStore.MetaStoreCallback>() { - @Override - public void operationComplete(List cursors, Stat v) { - // Load existing cursors - if (log.isDebugEnabled()) { - log.debug("[{}] Found {} cursors", managedLedgerName, cursors.size()); - } - if (cursors.isEmpty()) { - allCursorsCounter.countDown(); - return; - } - - final CountDownLatch cursorCounter = new CountDownLatch(cursors.size()); - - for (final String cursorName : cursors) { - // determine subscription position from cursor ledger - if (log.isDebugEnabled()) { - log.debug("[{}] Loading cursor {}", managedLedgerName, cursorName); - } - - AsyncCallback.OpenCallback cursorLedgerOpenCb = (rc, lh, ctx1) -> { - long ledgerId = lh.getId(); - if (log.isDebugEnabled()) { - log.debug("[{}] Opened cursor ledger {} for cursor {}. rc={}", managedLedgerName, ledgerId, - cursorName, rc); - } - if (rc != BKException.Code.OK) { - log.warn("[{}] Error opening metadata ledger {} for cursor {}: {}", managedLedgerName, - ledgerId, cursorName, BKException.getMessage(rc)); - cursorCounter.countDown(); - return; - } - long lac = lh.getLastAddConfirmed(); - if (log.isDebugEnabled()) { - log.debug("[{}] Cursor {} LAC {} read from ledger {}", managedLedgerName, cursorName, lac, - ledgerId); - } - - if (lac == LedgerHandle.INVALID_ENTRY_ID) { - // save the ledger id and cursor to retry outside of this call back - // since we are trying to read the same cursor ledger, we will block until - // this current callback completes, since an attempt to read the entry - // will block behind this current operation to complete - ledgerRetryMap.put(cursorName, ledgerId); - log.info("[{}] Cursor {} LAC {} read from ledger {}", managedLedgerName, cursorName, lac, - ledgerId); - cursorCounter.countDown(); - return; - } - final long entryId = lac; - // read last acked message position for subscription - lh.asyncReadEntries(entryId, entryId, new AsyncCallback.ReadCallback() { - @Override - public void readComplete(int rc, LedgerHandle lh, Enumeration seq, - Object ctx) { - try { - if (log.isDebugEnabled()) { - log.debug("readComplete rc={} entryId={}", rc, entryId); - } - if (rc != BKException.Code.OK) { - log.warn("[{}] Error reading from metadata ledger {} for cursor {}: {}", - managedLedgerName, ledgerId, cursorName, BKException.getMessage(rc)); - // indicate that this cursor should be excluded - offlineTopicStats.addCursorDetails(cursorName, errorInReadingCursor, - lh.getId()); - } else { - LedgerEntry entry = seq.nextElement(); - MLDataFormats.PositionInfo positionInfo; - try { - positionInfo = MLDataFormats.PositionInfo.parseFrom(entry.getEntry()); - } catch (InvalidProtocolBufferException e) { - log.warn( - "[{}] Error reading position from metadata ledger {} for cursor {}: {}", - managedLedgerName, ledgerId, cursorName, e); - offlineTopicStats.addCursorDetails(cursorName, errorInReadingCursor, - lh.getId()); - return; - } - final PositionImpl lastAckedMessagePosition = new PositionImpl(positionInfo); - if (log.isDebugEnabled()) { - log.debug("[{}] Cursor {} MD {} read last ledger position {}", - managedLedgerName, cursorName, lastAckedMessagePosition, - lastLedgerPosition); - } - // calculate cursor backlog - Range range = Range.openClosed(lastAckedMessagePosition, - lastLedgerPosition); - if (log.isDebugEnabled()) { - log.debug("[{}] Calculating backlog for cursor {} using range {}", - managedLedgerName, cursorName, range); - } - long cursorBacklog = getNumberOfEntries(range, ledgers); - offlineTopicStats.messageBacklog += cursorBacklog; - offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, lh.getId()); - } - } finally { - cursorCounter.countDown(); - } - } - }, null); - - }; // end of cursor meta read callback - - store.asyncGetCursorInfo(managedLedgerName, cursorName, - new MetaStore.MetaStoreCallback() { - @Override - public void operationComplete(MLDataFormats.ManagedCursorInfo info, - Stat stat) { - long cursorLedgerId = info.getCursorsLedgerId(); - if (log.isDebugEnabled()) { - log.debug("[{}] Cursor {} meta-data read ledger id {}", managedLedgerName, - cursorName, cursorLedgerId); - } - if (cursorLedgerId != -1) { - bk.asyncOpenLedgerNoRecovery(cursorLedgerId, digestType, password, - cursorLedgerOpenCb, null); - } else { - PositionImpl lastAckedMessagePosition = new PositionImpl( - info.getMarkDeleteLedgerId(), info.getMarkDeleteEntryId()); - Range range = Range.openClosed(lastAckedMessagePosition, - lastLedgerPosition); - if (log.isDebugEnabled()) { - log.debug("[{}] Calculating backlog for cursor {} using range {}", - managedLedgerName, cursorName, range); - } - long cursorBacklog = getNumberOfEntries(range, ledgers); - offlineTopicStats.messageBacklog += cursorBacklog; - offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, cursorLedgerId); - cursorCounter.countDown(); - } - - } - - @Override - public void operationFailed(ManagedLedgerException.MetaStoreException e) { - log.warn("[{}] Unable to obtain cursor ledger for cursor {}: {}", managedLedgerName, - cursorName, e); - cursorCounter.countDown(); - } - }); - } // for every cursor find backlog - try { - if (accurate) { - cursorCounter.await(); - } else { - cursorCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS); - } - } catch (Exception e) { - log.warn("[{}] Error reading subscription positions{}", managedLedgerName, e); - } finally { - allCursorsCounter.countDown(); - } - } - - @Override - public void operationFailed(ManagedLedgerException.MetaStoreException e) { - log.warn("[{}] Failed to get the cursors list", managedLedgerName, e); - allCursorsCounter.countDown(); - } - }); - if (accurate) { - allCursorsCounter.await(); - } else { - allCursorsCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS); - } - - // go through ledgers where LAC was -1 - if (accurate && ledgerRetryMap.size() > 0) { - ledgerRetryMap.forEach((cursorName, ledgerId) -> { - if (log.isDebugEnabled()) { - log.debug("Cursor {} Ledger {} Trying to obtain MD from BkAdmin", cursorName, ledgerId); - } - PositionImpl lastAckedMessagePosition = tryGetMDPosition(bk, ledgerId, cursorName); - if (lastAckedMessagePosition == null) { - log.warn("[{}] Cursor {} read from ledger {}. Unable to determine cursor position", - managedLedgerName, cursorName, ledgerId); - } else { - if (log.isDebugEnabled()) { - log.debug("[{}] Cursor {} read from ledger using bk admin {}. position {}", managedLedgerName, - cursorName, ledgerId, lastAckedMessagePosition); - } - // calculate cursor backlog - Range range = Range.openClosed(lastAckedMessagePosition, lastLedgerPosition); - if (log.isDebugEnabled()) { - log.debug("[{}] Calculating backlog for cursor {} using range {}", managedLedgerName, - cursorName, range); - } - long cursorBacklog = getNumberOfEntries(range, ledgers); - offlineTopicStats.messageBacklog += cursorBacklog; - offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, ledgerId); - } - }); - } - } - - private PositionImpl tryGetMDPosition(BookKeeper bookKeeper, long ledgerId, String cursorName) { - BookKeeperAdmin bookKeeperAdmin = null; - long lastEntry = LedgerHandle.INVALID_ENTRY_ID; - PositionImpl lastAckedMessagePosition = null; - try { - bookKeeperAdmin = new BookKeeperAdmin(bookKeeper); - for (LedgerEntry ledgerEntry : bookKeeperAdmin.readEntries(ledgerId, 0, lastEntry)) { - lastEntry = ledgerEntry.getEntryId(); - if (log.isDebugEnabled()) { - log.debug(" Read entry {} from ledger {} for cursor {}", lastEntry, ledgerId, cursorName); - } - MLDataFormats.PositionInfo positionInfo = MLDataFormats.PositionInfo.parseFrom(ledgerEntry.getEntry()); - lastAckedMessagePosition = new PositionImpl(positionInfo); - if (log.isDebugEnabled()) { - log.debug("Cursor {} read position {}", cursorName, lastAckedMessagePosition); - } - } - } catch (Exception e) { - log.warn("Unable to determine LAC for ledgerId {} for cursor {}: {}", ledgerId, cursorName, e); - } finally { - if (bookKeeperAdmin != null) { - try { - bookKeeperAdmin.close(); - } catch (Exception e) { - log.warn("Unable to close bk admin for ledgerId {} for cursor {}", ledgerId, cursorName, e); - } - } - - } - return lastAckedMessagePosition; + return offlineTopicStats; } private static final Logger log = LoggerFactory.getLogger(ManagedLedgerOfflineBacklog.class); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java index 1bc2d2b04bed0..d9269ec83b179 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java @@ -37,11 +37,11 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException; import org.apache.bookkeeper.mledger.ManagedLedgerException.MetadataNotFoundException; +import org.apache.bookkeeper.mledger.MetadataCompressionConfig; import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.bookkeeper.mledger.proto.MLDataFormats.CompressionType; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo; -import org.apache.commons.lang.StringUtils; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.compression.CompressionCodec; import org.apache.pulsar.common.compression.CompressionCodecProvider; @@ -62,50 +62,35 @@ public class MetaStoreImpl implements MetaStore, Consumer { private final OrderedExecutor executor; private static final int MAGIC_MANAGED_INFO_METADATA = 0x4778; // 0100 0111 0111 1000 - private final CompressionType ledgerInfoCompressionType; - private final CompressionType cursorInfoCompressionType; + private final MetadataCompressionConfig ledgerInfoCompressionConfig; + private final MetadataCompressionConfig cursorInfoCompressionConfig; private final Map> managedLedgerInfoUpdateCallbackMap; public MetaStoreImpl(MetadataStore store, OrderedExecutor executor) { this.store = store; this.executor = executor; - this.ledgerInfoCompressionType = CompressionType.NONE; - this.cursorInfoCompressionType = CompressionType.NONE; + this.ledgerInfoCompressionConfig = MetadataCompressionConfig.noCompression; + this.cursorInfoCompressionConfig = MetadataCompressionConfig.noCompression; managedLedgerInfoUpdateCallbackMap = new ConcurrentHashMap<>(); if (store != null) { store.registerListener(this); } } - public MetaStoreImpl(MetadataStore store, OrderedExecutor executor, String ledgerInfoCompressionType, - String cursorInfoCompressionType) { + public MetaStoreImpl(MetadataStore store, OrderedExecutor executor, + MetadataCompressionConfig ledgerInfoCompressionConfig, + MetadataCompressionConfig cursorInfoCompressionConfig) { this.store = store; this.executor = executor; - this.ledgerInfoCompressionType = parseCompressionType(ledgerInfoCompressionType); - this.cursorInfoCompressionType = parseCompressionType(cursorInfoCompressionType); + this.ledgerInfoCompressionConfig = ledgerInfoCompressionConfig; + this.cursorInfoCompressionConfig = cursorInfoCompressionConfig; managedLedgerInfoUpdateCallbackMap = new ConcurrentHashMap<>(); if (store != null) { store.registerListener(this); } } - private CompressionType parseCompressionType(String value) { - if (StringUtils.isEmpty(value)) { - return CompressionType.NONE; - } - - CompressionType compressionType; - try { - compressionType = CompressionType.valueOf(value); - } catch (Exception e) { - log.error("Failed to get compression type {} error msg: {}.", value, e.getMessage()); - throw e; - } - - return compressionType; - } - @Override public void getManagedLedgerInfo(String ledgerName, boolean createIfMissing, Map properties, MetaStoreCallback callback) { @@ -421,29 +406,43 @@ private static MetaStoreException getException(Throwable t) { } public byte[] compressLedgerInfo(ManagedLedgerInfo managedLedgerInfo) { - if (ledgerInfoCompressionType.equals(CompressionType.NONE)) { + CompressionType compressionType = ledgerInfoCompressionConfig.getCompressionType(); + if (compressionType.equals(CompressionType.NONE)) { return managedLedgerInfo.toByteArray(); } - MLDataFormats.ManagedLedgerInfoMetadata mlInfoMetadata = MLDataFormats.ManagedLedgerInfoMetadata - .newBuilder() - .setCompressionType(ledgerInfoCompressionType) - .setUncompressedSize(managedLedgerInfo.getSerializedSize()) - .build(); - return compressManagedInfo(managedLedgerInfo.toByteArray(), mlInfoMetadata.toByteArray(), - mlInfoMetadata.getSerializedSize(), ledgerInfoCompressionType); + + int uncompressedSize = managedLedgerInfo.getSerializedSize(); + if (uncompressedSize > ledgerInfoCompressionConfig.getCompressSizeThresholdInBytes()) { + MLDataFormats.ManagedLedgerInfoMetadata mlInfoMetadata = MLDataFormats.ManagedLedgerInfoMetadata + .newBuilder() + .setCompressionType(compressionType) + .setUncompressedSize(uncompressedSize) + .build(); + return compressManagedInfo(managedLedgerInfo.toByteArray(), mlInfoMetadata.toByteArray(), + mlInfoMetadata.getSerializedSize(), compressionType); + } + + return managedLedgerInfo.toByteArray(); } public byte[] compressCursorInfo(ManagedCursorInfo managedCursorInfo) { - if (cursorInfoCompressionType.equals(CompressionType.NONE)) { + CompressionType compressionType = cursorInfoCompressionConfig.getCompressionType(); + if (compressionType.equals(CompressionType.NONE)) { return managedCursorInfo.toByteArray(); } - MLDataFormats.ManagedCursorInfoMetadata metadata = MLDataFormats.ManagedCursorInfoMetadata - .newBuilder() - .setCompressionType(cursorInfoCompressionType) - .setUncompressedSize(managedCursorInfo.getSerializedSize()) - .build(); - return compressManagedInfo(managedCursorInfo.toByteArray(), metadata.toByteArray(), - metadata.getSerializedSize(), cursorInfoCompressionType); + + int uncompressedSize = managedCursorInfo.getSerializedSize(); + if (uncompressedSize > cursorInfoCompressionConfig.getCompressSizeThresholdInBytes()) { + MLDataFormats.ManagedCursorInfoMetadata metadata = MLDataFormats.ManagedCursorInfoMetadata + .newBuilder() + .setCompressionType(compressionType) + .setUncompressedSize(uncompressedSize) + .build(); + return compressManagedInfo(managedCursorInfo.toByteArray(), metadata.toByteArray(), + metadata.getSerializedSize(), compressionType); + } + + return managedCursorInfo.toByteArray(); } public ManagedLedgerInfo parseManagedLedgerInfo(byte[] data) throws InvalidProtocolBufferException { diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonAppendableLedgerOffloader.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonAppendableLedgerOffloader.java new file mode 100644 index 0000000000000..f3001ec8050e2 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonAppendableLedgerOffloader.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import org.apache.bookkeeper.client.api.ReadHandle; +import org.apache.bookkeeper.mledger.LedgerOffloader; +import org.apache.pulsar.common.policies.data.OffloadPolicies; +import org.apache.pulsar.common.util.FutureUtil; + +public class NonAppendableLedgerOffloader implements LedgerOffloader { + private LedgerOffloader delegate; + + public NonAppendableLedgerOffloader(LedgerOffloader delegate) { + this.delegate = delegate; + } + + @Override + public String getOffloadDriverName() { + return delegate.getOffloadDriverName(); + } + + @Override + public CompletableFuture offload(ReadHandle ledger, + UUID uid, + Map extraMetadata) { + return FutureUtil.failedFuture(new UnsupportedOperationException()); + } + + @Override + public CompletableFuture readOffloaded(long ledgerId, UUID uid, + Map offloadDriverMetadata) { + return delegate.readOffloaded(ledgerId, uid, offloadDriverMetadata); + } + + @Override + public CompletableFuture deleteOffloaded(long ledgerId, UUID uid, + Map offloadDriverMetadata) { + return delegate.deleteOffloaded(ledgerId, uid, offloadDriverMetadata); + } + + @Override + public OffloadPolicies getOffloadPolicies() { + return delegate.getOffloadPolicies(); + } + + @Override + public void close() { + delegate.close(); + } + + @Override + public boolean isAppendable() { + return false; + } +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java index 9d2829b1707f4..326f8216f1e18 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java @@ -25,7 +25,8 @@ import org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback; -import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.slf4j.Logger; @@ -35,10 +36,10 @@ public class NonDurableCursorImpl extends ManagedCursorImpl { private final boolean readCompacted; - NonDurableCursorImpl(BookKeeper bookkeeper, ManagedLedgerConfig config, ManagedLedgerImpl ledger, String cursorName, - PositionImpl startCursorPosition, CommandSubscribe.InitialPosition initialPosition, + NonDurableCursorImpl(BookKeeper bookkeeper, ManagedLedgerImpl ledger, String cursorName, + Position startCursorPosition, CommandSubscribe.InitialPosition initialPosition, boolean isReadCompacted) { - super(bookkeeper, config, ledger, cursorName); + super(bookkeeper, ledger, cursorName); this.readCompacted = isReadCompacted; // Compare with "latest" position marker by using only the ledger id. Since the C++ client is using 48bits to @@ -54,7 +55,7 @@ public class NonDurableCursorImpl extends ManagedCursorImpl { initializeCursorPosition(ledger.getFirstPositionAndCounter()); break; } - } else if (startCursorPosition.getLedgerId() == PositionImpl.EARLIEST.getLedgerId()) { + } else if (startCursorPosition.getLedgerId() == PositionFactory.EARLIEST.getLedgerId()) { // Start from invalid ledger to read from first available entry recoverCursor(ledger.getPreviousPosition(ledger.getFirstPosition())); } else { @@ -67,10 +68,10 @@ public class NonDurableCursorImpl extends ManagedCursorImpl { readPosition, markDeletePosition); } - private void recoverCursor(PositionImpl mdPosition) { - Pair lastEntryAndCounter = ledger.getLastPositionAndCounter(); + private void recoverCursor(Position mdPosition) { + Pair lastEntryAndCounter = ledger.getLastPositionAndCounter(); this.readPosition = isReadCompacted() ? mdPosition.getNext() : ledger.getNextValidPosition(mdPosition); - markDeletePosition = mdPosition; + markDeletePosition = ledger.getPreviousPosition(this.readPosition); // Initialize the counter such that the difference between the messages written on the ML and the // messagesConsumed is equal to the current backlog (negated). @@ -97,7 +98,7 @@ void recover(final VoidCallback callback) { } @Override - protected void internalAsyncMarkDelete(final PositionImpl newPosition, Map properties, + protected void internalAsyncMarkDelete(final Position newPosition, Map properties, final MarkDeleteCallback callback, final Object ctx) { // Bypass persistence of mark-delete position and individually deleted messages info @@ -138,8 +139,12 @@ public void rewind() { @Override public synchronized String toString() { - return MoreObjects.toStringHelper(this).add("ledger", ledger.getName()).add("ackPos", markDeletePosition) - .add("readPos", readPosition).toString(); + return MoreObjects.toStringHelper(this) + .add("ledger", ledger.getName()) + .add("cursor", getName()) + .add("ackPos", markDeletePosition) + .add("readPos", readPosition) + .toString(); } private static final Logger log = LoggerFactory.getLogger(NonDurableCursorImpl.class); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NullLedgerOffloader.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NullLedgerOffloader.java index 0e5e7cf4b5b55..fe646bc82e55a 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NullLedgerOffloader.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NullLedgerOffloader.java @@ -23,7 +23,7 @@ import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.client.api.ReadHandle; import org.apache.bookkeeper.mledger.LedgerOffloader; -import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; +import org.apache.pulsar.common.policies.data.OffloadPolicies; /** * Null implementation that throws an error on any invokation. @@ -62,7 +62,7 @@ public CompletableFuture deleteOffloaded(long ledgerId, UUID uid, } @Override - public OffloadPoliciesImpl getOffloadPolicies() { + public OffloadPolicies getOffloadPolicies() { return null; } @@ -70,4 +70,9 @@ public OffloadPoliciesImpl getOffloadPolicies() { public void close() { } + + @Override + public boolean isAppendable() { + return false; + } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java index ae2beafb64374..036ce9223e89d 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java @@ -24,8 +24,10 @@ import io.netty.util.Recycler.Handle; import io.netty.util.ReferenceCountUtil; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.AsyncCallback.AddCallback; import org.apache.bookkeeper.client.AsyncCallback.CloseCallback; @@ -34,6 +36,7 @@ import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.intercept.ManagedLedgerInterceptor; @@ -42,10 +45,10 @@ * */ @Slf4j -public class OpAddEntry implements AddCallback, CloseCallback, Runnable { +public class OpAddEntry implements AddCallback, CloseCallback, Runnable, ManagedLedgerInterceptor.AddEntryOperation { protected ManagedLedgerImpl ml; LedgerHandle ledger; - private long entryId; + long entryId; private int numberOfMessages; @SuppressWarnings("unused") @@ -68,6 +71,9 @@ public class OpAddEntry implements AddCallback, CloseCallback, Runnable { AtomicReferenceFieldUpdater.newUpdater(OpAddEntry.class, OpAddEntry.State.class, "state"); volatile State state; + @Setter + private AtomicBoolean timeoutTriggered; + enum State { OPEN, INITIATED, @@ -76,8 +82,8 @@ enum State { } public static OpAddEntry createNoRetainBuffer(ManagedLedgerImpl ml, ByteBuf data, AddEntryCallback callback, - Object ctx) { - OpAddEntry op = createOpAddEntryNoRetainBuffer(ml, data, callback, ctx); + Object ctx, AtomicBoolean timeoutTriggered) { + OpAddEntry op = createOpAddEntryNoRetainBuffer(ml, data, callback, ctx, timeoutTriggered); if (log.isDebugEnabled()) { log.debug("Created new OpAddEntry {}", op); } @@ -85,8 +91,9 @@ public static OpAddEntry createNoRetainBuffer(ManagedLedgerImpl ml, ByteBuf data } public static OpAddEntry createNoRetainBuffer(ManagedLedgerImpl ml, ByteBuf data, int numberOfMessages, - AddEntryCallback callback, Object ctx) { - OpAddEntry op = createOpAddEntryNoRetainBuffer(ml, data, callback, ctx); + AddEntryCallback callback, Object ctx, + AtomicBoolean timeoutTriggered) { + OpAddEntry op = createOpAddEntryNoRetainBuffer(ml, data, callback, ctx, timeoutTriggered); op.numberOfMessages = numberOfMessages; if (log.isDebugEnabled()) { log.debug("Created new OpAddEntry {}", op); @@ -95,7 +102,8 @@ public static OpAddEntry createNoRetainBuffer(ManagedLedgerImpl ml, ByteBuf data } private static OpAddEntry createOpAddEntryNoRetainBuffer(ManagedLedgerImpl ml, ByteBuf data, - AddEntryCallback callback, Object ctx) { + AddEntryCallback callback, Object ctx, + AtomicBoolean timeoutTriggered) { OpAddEntry op = RECYCLER.get(); op.ml = ml; op.ledger = null; @@ -109,6 +117,7 @@ private static OpAddEntry createOpAddEntryNoRetainBuffer(ManagedLedgerImpl ml, B op.startTime = System.nanoTime(); op.state = State.OPEN; op.payloadProcessorHandle = null; + op.timeoutTriggered = timeoutTriggered; ml.mbean.addAddEntrySample(op.dataLength); return op; } @@ -130,8 +139,8 @@ public void initiate() { lastInitTime = System.nanoTime(); if (ml.getManagedLedgerInterceptor() != null) { long originalDataLen = data.readableBytes(); - payloadProcessorHandle = ml.getManagedLedgerInterceptor().processPayloadBeforeLedgerWrite(this, - duplicateBuffer); + payloadProcessorHandle = ml.getManagedLedgerInterceptor() + .processPayloadBeforeLedgerWrite(this.getCtx(), duplicateBuffer); if (payloadProcessorHandle != null) { duplicateBuffer = payloadProcessorHandle.getProcessedPayload(); // If data len of entry changes, correct "dataLength" and "currentLedgerSize". @@ -176,7 +185,9 @@ public void addComplete(int rc, final LedgerHandle lh, long entryId, Object ctx) if (!STATE_UPDATER.compareAndSet(OpAddEntry.this, State.INITIATED, State.COMPLETED)) { log.warn("[{}] The add op is terminal legacy callback for entry {}-{} adding.", ml.getName(), lh.getId(), entryId); - OpAddEntry.this.recycle(); + // Since there is a thread is coping this object, do not recycle this object to avoid other problems. + // For example: we recycled this object, other thread get a null "opAddEntry.{variable_name}". + // Recycling is not mandatory, JVM GC will collect it. return; } @@ -200,7 +211,7 @@ public void addComplete(int rc, final LedgerHandle lh, long entryId, Object ctx) lh == null ? -1 : lh.getId(), entryId, dataLength, rc); } - if (rc != BKException.Code.OK) { + if (rc != BKException.Code.OK || timeoutTriggered.get()) { handleAddFailure(lh); } else { // Trigger addComplete callback in a thread hashed on the managed ledger name @@ -228,7 +239,8 @@ public void run() { ManagedLedgerImpl.TOTAL_SIZE_UPDATER.addAndGet(ml, dataLength); long ledgerId = ledger != null ? ledger.getId() : ((Position) ctx).getLedgerId(); - if (ml.hasActiveCursors()) { + // Don't insert to the entry cache for the ShadowManagedLedger + if (!(ml instanceof ShadowManagedLedgerImpl) && ml.hasActiveCursors()) { // Avoid caching entries if no cursor has been created EntryImpl entry = EntryImpl.create(ledgerId, entryId, data); // EntryCache.insert: duplicates entry by allocating new entry and data. so, recycle entry after calling @@ -237,7 +249,7 @@ public void run() { entry.release(); } - PositionImpl lastEntry = PositionImpl.get(ledgerId, entryId); + Position lastEntry = PositionFactory.create(ledgerId, entryId); ManagedLedgerImpl.ENTRIES_ADDED_COUNTER_UPDATER.incrementAndGet(ml); ml.lastConfirmedEntry = lastEntry; @@ -278,7 +290,7 @@ public void closeComplete(int rc, LedgerHandle lh, Object ctx) { AddEntryCallback cb = callbackUpdater.getAndSet(this, null); if (cb != null) { - cb.addComplete(PositionImpl.get(lh.getId(), entryId), data.asReadOnly(), ctx); + cb.addComplete(PositionFactory.create(lh.getId(), entryId), data.asReadOnly(), ctx); ml.notifyCursors(); ml.notifyWaitingEntryCallBacks(); ReferenceCountUtil.release(data); @@ -307,13 +319,6 @@ private boolean checkAndCompleteOp(Object ctx) { return false; } - void handleAddTimeoutFailure(final LedgerHandle ledger, Object ctx) { - if (checkAndCompleteOp(ctx)) { - this.close(); - this.handleAddFailure(ledger); - } - } - /** * It handles add failure on the given ledger. it can be triggered when add-entry fails or times out. * @@ -333,8 +338,11 @@ void handleAddFailure(final LedgerHandle lh) { }); } - void close() { + OpAddEntry duplicateAndClose(AtomicBoolean timeoutTriggered) { STATE_UPDATER.set(OpAddEntry.this, State.CLOSED); + OpAddEntry duplicate = + OpAddEntry.createNoRetainBuffer(ml, data, getNumberOfMessages(), callback, ctx, timeoutTriggered); + return duplicate; } public State getState() { @@ -389,6 +397,7 @@ public void recycle() { startTime = -1; lastInitTime = -1; payloadProcessorHandle = null; + timeoutTriggered = null; recyclerHandle.recycle(this); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpFindNewest.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpFindNewest.java index 900af9322c791..26d5e8d3f661d 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpFindNewest.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpFindNewest.java @@ -26,13 +26,13 @@ import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.PositionBound; +import org.apache.bookkeeper.mledger.PositionBound; @Slf4j class OpFindNewest implements ReadEntryCallback { private final ManagedCursorImpl cursor; private final ManagedLedgerImpl ledger; - private final PositionImpl startPosition; + private final Position startPosition; private final FindEntryCallback callback; private final Predicate condition; private final Object ctx; @@ -41,13 +41,13 @@ enum State { checkFirst, checkLast, searching } - PositionImpl searchPosition; + Position searchPosition; long min; long max; Position lastMatchedPosition = null; State state; - public OpFindNewest(ManagedCursorImpl cursor, PositionImpl startPosition, Predicate condition, + public OpFindNewest(ManagedCursorImpl cursor, Position startPosition, Predicate condition, long numberOfEntries, FindEntryCallback callback, Object ctx) { this.cursor = cursor; this.ledger = cursor.ledger; @@ -63,7 +63,7 @@ public OpFindNewest(ManagedCursorImpl cursor, PositionImpl startPosition, Predic this.state = State.checkFirst; } - public OpFindNewest(ManagedLedgerImpl ledger, PositionImpl startPosition, Predicate condition, + public OpFindNewest(ManagedLedgerImpl ledger, Position startPosition, Predicate condition, long numberOfEntries, FindEntryCallback callback, Object ctx) { this.cursor = null; this.ledger = ledger; @@ -94,8 +94,10 @@ public void readEntryComplete(Entry entry, Object ctx) { lastMatchedPosition = position; // check last entry state = State.checkLast; - PositionImpl lastPosition = ledger.getLastPosition(); searchPosition = ledger.getPositionAfterN(searchPosition, max, PositionBound.startExcluded); + Position lastPosition = ledger.getLastPosition(); + searchPosition = + ledger.getPositionAfterN(searchPosition, max, PositionBound.startExcluded); if (lastPosition.compareTo(searchPosition) < 0) { if (log.isDebugEnabled()) { log.debug("first position {} matches, last should be {}, but moving to lastPos {}", position, diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java index 19211553a5f74..3fd7e36c433ae 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java @@ -30,26 +30,27 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.NonRecoverableLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerException.TooManyRequestsException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; class OpReadEntry implements ReadEntriesCallback { ManagedCursorImpl cursor; - PositionImpl readPosition; + Position readPosition; private int count; private ReadEntriesCallback callback; Object ctx; // Results private List entries; - private PositionImpl nextReadPosition; - PositionImpl maxPosition; + private Position nextReadPosition; + Position maxPosition; - Predicate skipCondition; + Predicate skipCondition; - public static OpReadEntry create(ManagedCursorImpl cursor, PositionImpl readPositionRef, int count, - ReadEntriesCallback callback, Object ctx, PositionImpl maxPosition, Predicate skipCondition) { + public static OpReadEntry create(ManagedCursorImpl cursor, Position readPositionRef, int count, + ReadEntriesCallback callback, Object ctx, Position maxPosition, Predicate skipCondition) { OpReadEntry op = RECYCLER.get(); op.readPosition = cursor.ledger.startReadOperationOnLedger(readPositionRef); op.cursor = cursor; @@ -57,16 +58,16 @@ public static OpReadEntry create(ManagedCursorImpl cursor, PositionImpl readPosi op.callback = callback; op.entries = new ArrayList<>(); if (maxPosition == null) { - maxPosition = PositionImpl.LATEST; + maxPosition = PositionFactory.LATEST; } op.maxPosition = maxPosition; op.skipCondition = skipCondition; op.ctx = ctx; - op.nextReadPosition = PositionImpl.get(op.readPosition); + op.nextReadPosition = PositionFactory.create(op.readPosition); return op; } - void internalReadEntriesComplete(List returnedEntries, Object ctx, PositionImpl lastPosition) { + void internalReadEntriesComplete(List returnedEntries, Object ctx, Position lastPosition) { // Filter the returned entries for individual deleted messages int entriesCount = returnedEntries.size(); long entriesSize = 0; @@ -76,7 +77,7 @@ void internalReadEntriesComplete(List returnedEntries, Object ctx, Positi cursor.updateReadStats(entriesCount, entriesSize); if (entriesCount != 0) { - lastPosition = (PositionImpl) returnedEntries.get(entriesCount - 1).getPosition(); + lastPosition = returnedEntries.get(entriesCount - 1).getPosition(); } if (log.isDebugEnabled()) { log.debug("[{}][{}] Read entries succeeded batch_size={} cumulative_size={} requested_count={}", @@ -111,14 +112,17 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { callback.readEntriesComplete(entries, ctx); recycle(); }); - } else if (cursor.config.isAutoSkipNonRecoverableData() && exception instanceof NonRecoverableLedgerException) { + } else if (cursor.getConfig().isAutoSkipNonRecoverableData() + && exception instanceof NonRecoverableLedgerException) { log.warn("[{}][{}] read failed from ledger at position:{} : {}", cursor.ledger.getName(), cursor.getName(), readPosition, exception.getMessage()); final ManagedLedgerImpl ledger = (ManagedLedgerImpl) cursor.getManagedLedger(); Position nexReadPosition; + Long lostLedger = null; if (exception instanceof ManagedLedgerException.LedgerNotExistException) { // try to find and move to next valid ledger nexReadPosition = cursor.getNextLedgerPosition(readPosition.getLedgerId()); + lostLedger = readPosition.getLedgerId(); } else { // Skip this read operation nexReadPosition = ledger.getValidPositionAfterSkippedEntries(readPosition, count); @@ -131,6 +135,9 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { return; } updateReadPosition(nexReadPosition); + if (lostLedger != null) { + cursor.getManagedLedger().skipNonRecoverableLedger(lostLedger); + } checkReadCompletion(); } else { if (!(exception instanceof TooManyRequestsException)) { @@ -150,7 +157,7 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { } void updateReadPosition(Position newReadPosition) { - nextReadPosition = (PositionImpl) newReadPosition; + nextReadPosition = newReadPosition; cursor.setReadPosition(nextReadPosition); } @@ -188,7 +195,7 @@ private OpReadEntry(Handle recyclerHandle) { this.recyclerHandle = recyclerHandle; } - private static final Recycler RECYCLER = new Recycler() { + private static final Recycler RECYCLER = new Recycler<>() { @Override protected OpReadEntry newObject(Recycler.Handle recyclerHandle) { return new OpReadEntry(recyclerHandle); @@ -204,8 +211,8 @@ public void recycle() { entries = null; nextReadPosition = null; maxPosition = null; - recyclerHandle.recycle(this); skipCondition = null; + recyclerHandle.recycle(this); } private static final Logger log = LoggerFactory.getLogger(OpReadEntry.class); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpScan.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpScan.java index 6d68b042a7ad6..72d05ede3a0f5 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpScan.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpScan.java @@ -29,8 +29,8 @@ import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionBound; import org.apache.bookkeeper.mledger.ScanOutcome; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.PositionBound; @Slf4j class OpScan implements ReadEntriesCallback { @@ -44,11 +44,11 @@ class OpScan implements ReadEntriesCallback { private final long startTime = System.currentTimeMillis(); private final int batchSize; - PositionImpl searchPosition; + Position searchPosition; Position lastSeenPosition = null; public OpScan(ManagedCursorImpl cursor, int batchSize, - PositionImpl startPosition, Predicate condition, + Position startPosition, Predicate condition, ScanCallback callback, Object ctx, long maxEntries, long timeOutMs) { this.batchSize = batchSize; if (batchSize <= 0) { @@ -88,13 +88,13 @@ public void readEntriesComplete(List entries, Object ctx) { } } } - searchPosition = ledger.getPositionAfterN((PositionImpl) lastPositionForBatch, 1, + searchPosition = ledger.getPositionAfterN(lastPositionForBatch, 1, PositionBound.startExcluded); if (log.isDebugEnabled()) { log.debug("readEntryComplete {} at {} next is {}", lastPositionForBatch, searchPosition); } - if (searchPosition.compareTo((PositionImpl) lastPositionForBatch) == 0) { + if (searchPosition.compareTo(lastPositionForBatch) == 0) { // we have reached the end of the ledger, as we are not doing progress callback.scanComplete(lastSeenPosition, ScanOutcome.COMPLETED, OpScan.this.ctx); return; diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpenTelemetryManagedCursorStats.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpenTelemetryManagedCursorStats.java new file mode 100644 index 0000000000000..ec73c9d5e5eb2 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpenTelemetryManagedCursorStats.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import com.google.common.collect.Streams; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.metrics.BatchCallback; +import io.opentelemetry.api.metrics.ObservableLongMeasurement; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.pulsar.opentelemetry.Constants; + +public class OpenTelemetryManagedCursorStats implements AutoCloseable { + + // Replaces ['pulsar_ml_cursor_persistLedgerSucceed', 'pulsar_ml_cursor_persistLedgerErrors'] + public static final String PERSIST_OPERATION_COUNTER = "pulsar.broker.managed_ledger.persist.operation.count"; + private final ObservableLongMeasurement persistOperationCounter; + + // Replaces ['pulsar_ml_cursor_persistZookeeperSucceed', 'pulsar_ml_cursor_persistZookeeperErrors'] + public static final String PERSIST_OPERATION_METADATA_STORE_COUNTER = + "pulsar.broker.managed_ledger.persist.mds.operation.count"; + private final ObservableLongMeasurement persistOperationMetadataStoreCounter; + + // Replaces pulsar_ml_cursor_nonContiguousDeletedMessagesRange + public static final String NON_CONTIGUOUS_MESSAGE_RANGE_COUNTER = + "pulsar.broker.managed_ledger.message_range.count"; + private final ObservableLongMeasurement nonContiguousMessageRangeCounter; + + // Replaces pulsar_ml_cursor_writeLedgerSize + public static final String OUTGOING_BYTE_COUNTER = "pulsar.broker.managed_ledger.cursor.outgoing.size"; + private final ObservableLongMeasurement outgoingByteCounter; + + // Replaces pulsar_ml_cursor_writeLedgerLogicalSize + public static final String OUTGOING_BYTE_LOGICAL_COUNTER = + "pulsar.broker.managed_ledger.cursor.outgoing.logical.size"; + private final ObservableLongMeasurement outgoingByteLogicalCounter; + + // Replaces pulsar_ml_cursor_readLedgerSize + public static final String INCOMING_BYTE_COUNTER = "pulsar.broker.managed_ledger.cursor.incoming.size"; + private final ObservableLongMeasurement incomingByteCounter; + + private final BatchCallback batchCallback; + + public OpenTelemetryManagedCursorStats(OpenTelemetry openTelemetry, ManagedLedgerFactoryImpl factory) { + var meter = openTelemetry.getMeter(Constants.BROKER_INSTRUMENTATION_SCOPE_NAME); + + persistOperationCounter = meter + .counterBuilder(PERSIST_OPERATION_COUNTER) + .setUnit("{operation}") + .setDescription("The number of acknowledgment operations on the ledger.") + .buildObserver(); + + persistOperationMetadataStoreCounter = meter + .counterBuilder(PERSIST_OPERATION_METADATA_STORE_COUNTER) + .setUnit("{operation}") + .setDescription("The number of acknowledgment operations in the metadata store.") + .buildObserver(); + + nonContiguousMessageRangeCounter = meter + .upDownCounterBuilder(NON_CONTIGUOUS_MESSAGE_RANGE_COUNTER) + .setUnit("{range}") + .setDescription("The number of non-contiguous deleted messages ranges.") + .buildObserver(); + + outgoingByteCounter = meter + .counterBuilder(OUTGOING_BYTE_COUNTER) + .setUnit("{By}") + .setDescription("The total amount of data written to the ledger.") + .buildObserver(); + + outgoingByteLogicalCounter = meter + .counterBuilder(OUTGOING_BYTE_LOGICAL_COUNTER) + .setUnit("{By}") + .setDescription("The total amount of data written to the ledger, not including replicas.") + .buildObserver(); + + incomingByteCounter = meter + .counterBuilder(INCOMING_BYTE_COUNTER) + .setUnit("{By}") + .setDescription("The total amount of data read from the ledger.") + .buildObserver(); + + batchCallback = meter.batchCallback(() -> factory.getManagedLedgers() + .values() + .stream() + .map(ManagedLedger::getCursors) + .flatMap(Streams::stream) + .forEach(this::recordMetrics), + persistOperationCounter, + persistOperationMetadataStoreCounter, + nonContiguousMessageRangeCounter, + outgoingByteCounter, + outgoingByteLogicalCounter, + incomingByteCounter); + } + + @Override + public void close() { + batchCallback.close(); + } + + private void recordMetrics(ManagedCursor cursor) { + var stats = cursor.getStats(); + var cursorAttributesSet = cursor.getManagedCursorAttributes(); + var attributes = cursorAttributesSet.getAttributes(); + var attributesSucceed = cursorAttributesSet.getAttributesOperationSucceed(); + var attributesFailed = cursorAttributesSet.getAttributesOperationFailure(); + + persistOperationCounter.record(stats.getPersistLedgerSucceed(), attributesSucceed); + persistOperationCounter.record(stats.getPersistLedgerErrors(), attributesFailed); + + persistOperationMetadataStoreCounter.record(stats.getPersistZookeeperSucceed(), attributesSucceed); + persistOperationMetadataStoreCounter.record(stats.getPersistZookeeperErrors(), attributesFailed); + + nonContiguousMessageRangeCounter.record(cursor.getTotalNonContiguousDeletedMessagesRange(), attributes); + + outgoingByteCounter.record(stats.getWriteCursorLedgerSize(), attributes); + outgoingByteLogicalCounter.record(stats.getWriteCursorLedgerLogicalSize(), attributes); + incomingByteCounter.record(stats.getReadCursorLedgerSize(), attributes); + } +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpenTelemetryManagedLedgerStats.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpenTelemetryManagedLedgerStats.java new file mode 100644 index 0000000000000..26c4b62cf7694 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpenTelemetryManagedLedgerStats.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.metrics.BatchCallback; +import io.opentelemetry.api.metrics.ObservableLongMeasurement; +import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.pulsar.opentelemetry.Constants; + +public class OpenTelemetryManagedLedgerStats implements AutoCloseable { + + // Replaces pulsar_ml_AddEntryMessagesRate + public static final String ADD_ENTRY_COUNTER = "pulsar.broker.managed_ledger.message.outgoing.count"; + private final ObservableLongMeasurement addEntryCounter; + + // Replaces pulsar_ml_AddEntryBytesRate + public static final String BYTES_OUT_COUNTER = "pulsar.broker.managed_ledger.message.outgoing.logical.size"; + private final ObservableLongMeasurement bytesOutCounter; + + // Replaces pulsar_ml_AddEntryWithReplicasBytesRate + public static final String BYTES_OUT_WITH_REPLICAS_COUNTER = + "pulsar.broker.managed_ledger.message.outgoing.replicated.size"; + private final ObservableLongMeasurement bytesOutWithReplicasCounter; + + // Replaces pulsar_ml_NumberOfMessagesInBacklog + public static final String BACKLOG_COUNTER = "pulsar.broker.managed_ledger.backlog.count"; + private final ObservableLongMeasurement backlogCounter; + + // Replaces pulsar_ml_ReadEntriesRate + public static final String READ_ENTRY_COUNTER = "pulsar.broker.managed_ledger.message.incoming.count"; + private final ObservableLongMeasurement readEntryCounter; + + // Replaces pulsar_ml_ReadEntriesBytesRate + public static final String BYTES_IN_COUNTER = "pulsar.broker.managed_ledger.message.incoming.size"; + private final ObservableLongMeasurement bytesInCounter; + + // Replaces brk_ml_ReadEntriesOpsCacheMissesRate + public static final String READ_ENTRY_CACHE_MISS_COUNTER = + "pulsar.broker.managed_ledger.message.incoming.cache.miss.count"; + private final ObservableLongMeasurement readEntryCacheMissCounter; + + // Replaces pulsar_ml_MarkDeleteRate + public static final String MARK_DELETE_COUNTER = "pulsar.broker.managed_ledger.mark_delete.count"; + private final ObservableLongMeasurement markDeleteCounter; + + private final BatchCallback batchCallback; + + public OpenTelemetryManagedLedgerStats(OpenTelemetry openTelemetry, ManagedLedgerFactoryImpl factory) { + var meter = openTelemetry.getMeter(Constants.BROKER_INSTRUMENTATION_SCOPE_NAME); + + addEntryCounter = meter + .upDownCounterBuilder(ADD_ENTRY_COUNTER) + .setUnit("{operation}") + .setDescription("The number of write operations to this ledger.") + .buildObserver(); + + bytesOutCounter = meter + .counterBuilder(BYTES_OUT_COUNTER) + .setUnit("By") + .setDescription("The total number of messages bytes written to this ledger, excluding replicas.") + .buildObserver(); + + bytesOutWithReplicasCounter = meter + .counterBuilder(BYTES_OUT_WITH_REPLICAS_COUNTER) + .setUnit("By") + .setDescription("The total number of messages bytes written to this ledger, including replicas.") + .buildObserver(); + + backlogCounter = meter + .upDownCounterBuilder(BACKLOG_COUNTER) + .setUnit("{message}") + .setDescription("The number of messages in backlog for all consumers from this ledger.") + .buildObserver(); + + readEntryCounter = meter + .upDownCounterBuilder(READ_ENTRY_COUNTER) + .setUnit("{operation}") + .setDescription("The number of read operations from this ledger.") + .buildObserver(); + + bytesInCounter = meter + .counterBuilder(BYTES_IN_COUNTER) + .setUnit("By") + .setDescription("The total number of messages bytes read from this ledger.") + .buildObserver(); + + readEntryCacheMissCounter = meter + .upDownCounterBuilder(READ_ENTRY_CACHE_MISS_COUNTER) + .setUnit("{operation}") + .setDescription("The number of cache misses during read operations from this ledger.") + .buildObserver(); + + markDeleteCounter = meter + .counterBuilder(MARK_DELETE_COUNTER) + .setUnit("{operation}") + .setDescription("The total number of mark delete operations for this ledger.") + .buildObserver(); + + batchCallback = meter.batchCallback(() -> factory.getManagedLedgers() + .values() + .forEach(this::recordMetrics), + addEntryCounter, + bytesOutCounter, + bytesOutWithReplicasCounter, + backlogCounter, + readEntryCounter, + bytesInCounter, + readEntryCacheMissCounter, + markDeleteCounter); + } + + @Override + public void close() { + batchCallback.close(); + } + + private void recordMetrics(ManagedLedger ml) { + var stats = ml.getStats(); + var ledgerAttributeSet = ml.getManagedLedgerAttributes(); + var attributes = ledgerAttributeSet.getAttributes(); + var attributesSucceed = ledgerAttributeSet.getAttributesOperationSucceed(); + var attributesFailure = ledgerAttributeSet.getAttributesOperationFailure(); + + addEntryCounter.record(stats.getAddEntrySucceedTotal(), attributesSucceed); + addEntryCounter.record(stats.getAddEntryErrorsTotal(), attributesFailure); + bytesOutCounter.record(stats.getAddEntryBytesTotal(), attributes); + bytesOutWithReplicasCounter.record(stats.getAddEntryWithReplicasBytesTotal(), attributes); + + readEntryCounter.record(stats.getReadEntriesSucceededTotal(), attributesSucceed); + readEntryCounter.record(stats.getReadEntriesErrorsTotal(), attributesFailure); + bytesInCounter.record(stats.getReadEntriesBytesTotal(), attributes); + + backlogCounter.record(stats.getNumberOfMessagesInBacklog(), attributes); + markDeleteCounter.record(stats.getMarkDeleteTotal(), attributes); + readEntryCacheMissCounter.record(stats.getReadEntriesOpsCacheMissesTotal(), attributes); + } +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionImpl.java deleted file mode 100644 index ee179b5d059c8..0000000000000 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionImpl.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.bookkeeper.mledger.impl; - -import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.proto.MLDataFormats.NestedPositionInfo; -import org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo; - -public class PositionImpl implements Position, Comparable { - - protected long ledgerId; - protected long entryId; - protected long[] ackSet; - - public static final PositionImpl EARLIEST = new PositionImpl(-1, -1); - public static final PositionImpl LATEST = new PositionImpl(Long.MAX_VALUE, Long.MAX_VALUE); - - public PositionImpl(PositionInfo pi) { - this.ledgerId = pi.getLedgerId(); - this.entryId = pi.getEntryId(); - } - - public PositionImpl(NestedPositionInfo npi) { - this.ledgerId = npi.getLedgerId(); - this.entryId = npi.getEntryId(); - } - - public PositionImpl(long ledgerId, long entryId) { - this.ledgerId = ledgerId; - this.entryId = entryId; - } - - public PositionImpl(long ledgerId, long entryId, long[] ackSet) { - this.ledgerId = ledgerId; - this.entryId = entryId; - this.ackSet = ackSet; - } - - public PositionImpl(PositionImpl other) { - this.ledgerId = other.ledgerId; - this.entryId = other.entryId; - } - - public static PositionImpl get(long ledgerId, long entryId) { - return new PositionImpl(ledgerId, entryId); - } - - public static PositionImpl get(long ledgerId, long entryId, long[] ackSet) { - return new PositionImpl(ledgerId, entryId, ackSet); - } - - public static PositionImpl get(PositionImpl other) { - return new PositionImpl(other); - } - - public long[] getAckSet() { - return ackSet; - } - - public void setAckSet(long[] ackSet) { - this.ackSet = ackSet; - } - - public long getLedgerId() { - return ledgerId; - } - - public long getEntryId() { - return entryId; - } - - @Override - public PositionImpl getNext() { - if (entryId < 0) { - return PositionImpl.get(ledgerId, 0); - } else { - return PositionImpl.get(ledgerId, entryId + 1); - } - } - - /** - * Position after moving entryNum messages, - * if entryNum < 1, then return the current position. - * */ - public PositionImpl getPositionAfterEntries(int entryNum) { - if (entryNum < 1) { - return this; - } - if (entryId < 0) { - return PositionImpl.get(ledgerId, entryNum - 1); - } else { - return PositionImpl.get(ledgerId, entryId + entryNum); - } - } - - /** - * String representation of virtual cursor - LedgerId:EntryId. - */ - @Override - public String toString() { - return ledgerId + ":" + entryId; - } - - @Override - public int compareTo(PositionImpl that) { - if (this.ledgerId != that.ledgerId) { - return (this.ledgerId < that.ledgerId ? -1 : 1); - } - - if (this.entryId != that.entryId) { - return (this.entryId < that.entryId ? -1 : 1); - } - - return 0; - } - - public int compareTo(long ledgerId, long entryId) { - if (this.ledgerId != ledgerId) { - return (this.ledgerId < ledgerId ? -1 : 1); - } - - if (this.entryId != entryId) { - return (this.entryId < entryId ? -1 : 1); - } - - return 0; - } - - @Override - public int hashCode() { - int result = (int) (ledgerId ^ (ledgerId >>> 32)); - result = 31 * result + (int) (entryId ^ (entryId >>> 32)); - return result; - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof PositionImpl) { - PositionImpl other = (PositionImpl) obj; - return ledgerId == other.ledgerId && entryId == other.entryId; - } - return false; - } - - public boolean hasAckSet() { - return ackSet != null; - } - - public PositionInfo getPositionInfo() { - return PositionInfo.newBuilder().setLedgerId(ledgerId).setEntryId(entryId).build(); - } -} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRecyclable.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRecyclable.java new file mode 100644 index 0000000000000..142abf903c2f3 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRecyclable.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import io.netty.util.Recycler; +import io.netty.util.Recycler.Handle; +import org.apache.bookkeeper.mledger.Position; + +/** + * Recyclable implementation of Position that is used to reduce the overhead of creating new Position objects. + */ +public class PositionRecyclable implements Position { + private final Handle recyclerHandle; + + private static final Recycler RECYCLER = new Recycler() { + @Override + protected PositionRecyclable newObject(Recycler.Handle recyclerHandle) { + return new PositionRecyclable(recyclerHandle); + } + }; + + private long ledgerId; + private long entryId; + + private PositionRecyclable(Handle recyclerHandle) { + this.recyclerHandle = recyclerHandle; + } + + @Override + public long getLedgerId() { + return ledgerId; + } + + @Override + public long getEntryId() { + return entryId; + } + + public void recycle() { + ledgerId = -1; + entryId = -1; + recyclerHandle.recycle(this); + } + + @Override + public int hashCode() { + return hashCodeForPosition(); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof Position && compareTo((Position) obj) == 0; + } + + public static PositionRecyclable get(long ledgerId, long entryId) { + PositionRecyclable position = RECYCLER.get(); + position.ledgerId = ledgerId; + position.entryId = entryId; + return position; + } +} \ No newline at end of file diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java index 02e43504482d8..11cce409bec54 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java @@ -24,9 +24,11 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Map; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; -import org.apache.pulsar.common.util.collections.ConcurrentOpenLongPairRangeSet; import org.apache.pulsar.common.util.collections.LongPairRangeSet; +import org.apache.pulsar.common.util.collections.OpenLongPairRangeSet; +import org.roaringbitmap.RoaringBitSet; /** * Wraps other Range classes, and adds LRU, marking dirty data and other features on this basis. @@ -52,10 +54,10 @@ public RangeSetWrapper(LongPairConsumer rangeConverter, RangeBoundConsumer rangeBoundConsumer, ManagedCursorImpl managedCursor) { requireNonNull(managedCursor); - this.config = managedCursor.getConfig(); + this.config = managedCursor.getManagedLedger().getConfig(); this.rangeConverter = rangeConverter; this.rangeSet = config.isUnackedRangesOpenCacheSetEnabled() - ? new ConcurrentOpenLongPairRangeSet<>(4096, rangeConverter) + ? new OpenLongPairRangeSet<>(rangeConverter, RoaringBitSet::new) : new LongPairRangeSet.DefaultRangeSet<>(rangeConverter, rangeBoundConsumer); this.enableMultiEntry = config.isPersistentUnackedRangesWithMultipleEntriesEnabled(); } @@ -141,6 +143,16 @@ public Range lastRange() { return rangeSet.lastRange(); } + @Override + public Map toRanges(int maxRanges) { + return rangeSet.toRanges(maxRanges); + } + + @Override + public void build(Map internalRange) { + rangeSet.build(internalRange); + } + @Override public int cardinality(long lowerKey, long lowerValue, long upperKey, long upperValue) { return rangeSet.cardinality(lowerKey, lowerValue, upperKey, upperValue); @@ -148,16 +160,16 @@ public int cardinality(long lowerKey, long lowerValue, long upperKey, long upper @VisibleForTesting void add(Range range) { - if (!(rangeSet instanceof ConcurrentOpenLongPairRangeSet)) { + if (!(rangeSet instanceof OpenLongPairRangeSet)) { throw new UnsupportedOperationException("Only ConcurrentOpenLongPairRangeSet support this method"); } - ((ConcurrentOpenLongPairRangeSet) rangeSet).add(range); + ((OpenLongPairRangeSet) rangeSet).add(range); } @VisibleForTesting void remove(Range range) { - if (rangeSet instanceof ConcurrentOpenLongPairRangeSet) { - ((ConcurrentOpenLongPairRangeSet) rangeSet).remove((Range) range); + if (rangeSet instanceof OpenLongPairRangeSet) { + ((OpenLongPairRangeSet) rangeSet).remove((Range) range); } else { ((DefaultRangeSet) rangeSet).remove(range); } @@ -175,4 +187,22 @@ public boolean isDirtyLedgers(long ledgerId) { public String toString() { return rangeSet.toString(); } + + @Override + public int hashCode() { + return rangeSet.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof RangeSetWrapper)) { + return false; + } + if (this == obj) { + return true; + } + @SuppressWarnings("rawtypes") + RangeSetWrapper set = (RangeSetWrapper) obj; + return this.rangeSet.equals(set.rangeSet); + } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorImpl.java index 9102339b2904e..00ed5a0c5b9d9 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorImpl.java @@ -22,19 +22,20 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.mledger.AsyncCallbacks; -import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionBound; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.ReadOnlyCursor; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.PositionBound; import org.apache.bookkeeper.mledger.proto.MLDataFormats; @Slf4j public class ReadOnlyCursorImpl extends ManagedCursorImpl implements ReadOnlyCursor { - public ReadOnlyCursorImpl(BookKeeper bookkeeper, ManagedLedgerConfig config, ManagedLedgerImpl ledger, - PositionImpl startPosition, String cursorName) { - super(bookkeeper, config, ledger, cursorName); + public ReadOnlyCursorImpl(BookKeeper bookkeeper, ManagedLedgerImpl ledger, + Position startPosition, String cursorName) { + super(bookkeeper, ledger, cursorName); - if (startPosition.equals(PositionImpl.EARLIEST)) { + if (startPosition.equals(PositionFactory.EARLIEST)) { readPosition = ledger.getFirstPosition().getNext(); } else { readPosition = startPosition; @@ -67,7 +68,12 @@ public MLDataFormats.ManagedLedgerInfo.LedgerInfo getCurrentLedgerInfo() { } @Override - public long getNumberOfEntries(Range range) { + public long getNumberOfEntries(Range range) { return this.ledger.getNumberOfEntries(range); } + + @Override + public boolean isMessageDeleted(Position position) { + return false; + } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyManagedLedgerImpl.java index 944674f6862c2..1fb2aa3629092 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyManagedLedgerImpl.java @@ -30,8 +30,11 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerNotFoundException; import org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException; import org.apache.bookkeeper.mledger.ManagedLedgerException.MetadataNotFoundException; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.ReadOnlyCursor; import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback; +import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo; import org.apache.pulsar.metadata.api.Stat; @@ -40,12 +43,12 @@ public class ReadOnlyManagedLedgerImpl extends ManagedLedgerImpl { public ReadOnlyManagedLedgerImpl(ManagedLedgerFactoryImpl factory, BookKeeper bookKeeper, MetaStore store, - ManagedLedgerConfig config, OrderedScheduler scheduledExecutor, - String name) { + ManagedLedgerConfig config, OrderedScheduler scheduledExecutor, + String name) { super(factory, bookKeeper, store, config, scheduledExecutor, name); } - CompletableFuture initialize() { + public CompletableFuture initialize() { CompletableFuture future = new CompletableFuture<>(); // Fetch the list of existing ledgers in the managed ledger @@ -58,6 +61,13 @@ public void operationComplete(ManagedLedgerInfo mlInfo, Stat stat) { ledgers.put(ls.getLedgerId(), ls); } + if (mlInfo.getPropertiesCount() > 0) { + for (int i = 0; i < mlInfo.getPropertiesCount(); i++) { + MLDataFormats.KeyValue property = mlInfo.getProperties(i); + propertiesMap.put(property.getKey(), property.getValue()); + } + } + // Last ledger stat may be zeroed, we must update it if (ledgers.size() > 0 && ledgers.lastEntry().getValue().getEntries() == 0) { long lastLedgerId = ledgers.lastKey(); @@ -118,33 +128,34 @@ public void operationFailed(MetaStoreException e) { return future; } - ReadOnlyCursor createReadOnlyCursor(PositionImpl startPosition) { + public ReadOnlyCursor createReadOnlyCursor(Position startPosition) { if (ledgers.isEmpty()) { - lastConfirmedEntry = PositionImpl.EARLIEST; + lastConfirmedEntry = PositionFactory.EARLIEST; } else if (ledgers.lastEntry().getValue().getEntries() > 0) { // Last ledger has some of the entries - lastConfirmedEntry = new PositionImpl(ledgers.lastKey(), ledgers.lastEntry().getValue().getEntries() - 1); + lastConfirmedEntry = + PositionFactory.create(ledgers.lastKey(), ledgers.lastEntry().getValue().getEntries() - 1); } else { // Last ledger is empty. If there is a previous ledger, position on the last entry of that ledger if (ledgers.size() > 1) { long lastLedgerId = ledgers.lastKey(); LedgerInfo li = ledgers.headMap(lastLedgerId, false).lastEntry().getValue(); - lastConfirmedEntry = new PositionImpl(li.getLedgerId(), li.getEntries() - 1); + lastConfirmedEntry = PositionFactory.create(li.getLedgerId(), li.getEntries() - 1); } else { - lastConfirmedEntry = PositionImpl.EARLIEST; + lastConfirmedEntry = PositionFactory.EARLIEST; } } - return new ReadOnlyCursorImpl(bookKeeper, config, this, startPosition, "read-only-cursor"); + return new ReadOnlyCursorImpl(bookKeeper, this, startPosition, "read-only-cursor"); } @Override - public void asyncReadEntry(PositionImpl position, AsyncCallbacks.ReadEntryCallback callback, Object ctx) { + public void asyncReadEntry(Position position, AsyncCallbacks.ReadEntryCallback callback, Object ctx) { this.getLedgerHandle(position.getLedgerId()) .thenAccept((ledger) -> asyncReadEntry(ledger, position, callback, ctx)) .exceptionally((ex) -> { - log.error("[{}] Error opening ledger for reading at position {} - {}", this.name, position, - ex.getMessage()); + log.error("[{}] Error opening ledger for reading at position {} - {}. Op: {}", this.name, + position, ex.getMessage(), callback); callback.readEntryFailed(ManagedLedgerException.getManagedLedgerException(ex.getCause()), ctx); return null; }); @@ -152,7 +163,7 @@ public void asyncReadEntry(PositionImpl position, AsyncCallbacks.ReadEntryCallba @Override public long getNumberOfEntries() { - return getNumberOfEntries(Range.openClosed(PositionImpl.EARLIEST, getLastPosition())); + return getNumberOfEntries(Range.openClosed(PositionFactory.EARLIEST, getLastPosition())); } @Override diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java index b33dd87543f77..4b03cad8e0a1d 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java @@ -24,6 +24,8 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.AsyncCallback; @@ -35,6 +37,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo; import org.apache.pulsar.metadata.api.Stat; @@ -50,9 +53,11 @@ public class ShadowManagedLedgerImpl extends ManagedLedgerImpl { public ShadowManagedLedgerImpl(ManagedLedgerFactoryImpl factory, BookKeeper bookKeeper, MetaStore store, ManagedLedgerConfig config, OrderedScheduler scheduledExecutor, - String name, final Supplier mlOwnershipChecker) { + String name, final Supplier> mlOwnershipChecker) { super(factory, bookKeeper, store, config, scheduledExecutor, name, mlOwnershipChecker); this.sourceMLName = config.getShadowSourceName(); + // ShadowManagedLedgerImpl does not implement add entry timeout yet, so this variable will always be false. + this.currentLedgerTimeoutTriggered = new AtomicBoolean(false); } /** @@ -94,7 +99,9 @@ public void operationComplete(MLDataFormats.ManagedLedgerInfo mlInfo, Stat stat) } if (mlInfo.hasTerminatedPosition()) { - lastConfirmedEntry = new PositionImpl(mlInfo.getTerminatedPosition()); + MLDataFormats.NestedPositionInfo terminatedPosition = mlInfo.getTerminatedPosition(); + lastConfirmedEntry = + PositionFactory.create(terminatedPosition.getLedgerId(), terminatedPosition.getEntryId()); log.info("[{}][{}] Recovering managed ledger terminated at {}", name, sourceMLName, lastConfirmedEntry); } @@ -124,7 +131,8 @@ public void operationComplete(MLDataFormats.ManagedLedgerInfo mlInfo, Stat stat) currentLedger = lh; if (managedLedgerInterceptor != null) { - managedLedgerInterceptor.onManagedLedgerLastLedgerInitialize(name, lh) + managedLedgerInterceptor + .onManagedLedgerLastLedgerInitialize(name, createLastEntryHandle(lh)) .thenRun(() -> ShadowManagedLedgerImpl.super.initialize(callback, ctx)) .exceptionally(ex -> { callback.initializeFailed( @@ -201,13 +209,13 @@ private void initLastConfirmedEntry() { if (currentLedger == null) { return; } - lastConfirmedEntry = new PositionImpl(currentLedger.getId(), currentLedger.getLastAddConfirmed()); + lastConfirmedEntry = PositionFactory.create(currentLedger.getId(), currentLedger.getLastAddConfirmed()); // bypass empty ledgers, find last ledger with Message if possible. while (lastConfirmedEntry.getEntryId() == -1) { Map.Entry formerLedger = ledgers.lowerEntry(lastConfirmedEntry.getLedgerId()); if (formerLedger != null) { LedgerInfo ledgerInfo = formerLedger.getValue(); - lastConfirmedEntry = PositionImpl.get(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1); + lastConfirmedEntry = PositionFactory.create(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1); } else { break; } @@ -277,7 +285,9 @@ private synchronized void processSourceManagedLedgerInfo(MLDataFormats.ManagedLe sourceLedgersStat = stat; if (mlInfo.hasTerminatedPosition()) { - lastConfirmedEntry = new PositionImpl(mlInfo.getTerminatedPosition()); + MLDataFormats.NestedPositionInfo terminatedPosition = mlInfo.getTerminatedPosition(); + lastConfirmedEntry = + PositionFactory.create(terminatedPosition.getLedgerId(), terminatedPosition.getEntryId()); log.info("[{}][{}] Process managed ledger terminated at {}", name, sourceMLName, lastConfirmedEntry); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCache.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCache.java index a67756ddeeae9..c2c5cd6bff43e 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCache.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCache.java @@ -21,8 +21,8 @@ import org.apache.bookkeeper.client.api.ReadHandle; import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntryCallback; +import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.EntryImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.tuple.Pair; /** @@ -54,7 +54,7 @@ public interface EntryCache extends Comparable { * @param lastPosition * the position of the last entry to be invalidated (non-inclusive) */ - void invalidateEntries(PositionImpl lastPosition); + void invalidateEntries(Position lastPosition); void invalidateEntriesBeforeTimestamp(long timestamp); @@ -115,7 +115,7 @@ void asyncReadEntry(ReadHandle lh, long firstEntry, long lastEntry, boolean shou * @param ctx * the context object */ - void asyncReadEntry(ReadHandle lh, PositionImpl position, ReadEntryCallback callback, Object ctx); + void asyncReadEntry(ReadHandle lh, Position position, ReadEntryCallback callback, Object ctx); /** * Get the total size in bytes of all the entries stored in this cache. diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java index d2add99b701ac..92541a7a72578 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java @@ -27,9 +27,9 @@ import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.EntryImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.bookkeeper.mledger.intercept.ManagedLedgerInterceptor; import org.apache.commons.lang3.tuple.Pair; @@ -56,7 +56,7 @@ public boolean insert(EntryImpl entry) { } @Override - public void invalidateEntries(PositionImpl lastPosition) { + public void invalidateEntries(Position lastPosition) { } @Override @@ -79,7 +79,7 @@ public void invalidateEntriesBeforeTimestamp(long timestamp) { @Override public void asyncReadEntry(ReadHandle lh, long firstEntry, long lastEntry, boolean isSlowestReader, final AsyncCallbacks.ReadEntriesCallback callback, Object ctx) { - lh.readAsync(firstEntry, lastEntry).thenAcceptAsync( + ReadEntryUtils.readAsync(ml, lh, firstEntry, lastEntry).thenAcceptAsync( ledgerEntries -> { List entries = new ArrayList<>(); long totalSize = 0; @@ -93,7 +93,7 @@ public void asyncReadEntry(ReadHandle lh, long firstEntry, long lastEntry, boole } finally { ledgerEntries.close(); } - ml.getMbean().recordReadEntriesOpsCacheMisses(); + ml.getMbean().recordReadEntriesOpsCacheMisses(entries.size(), totalSize); ml.getFactory().getMbean().recordCacheMiss(entries.size(), totalSize); ml.getMbean().addReadEntriesSample(entries.size(), totalSize); @@ -105,9 +105,9 @@ public void asyncReadEntry(ReadHandle lh, long firstEntry, long lastEntry, boole } @Override - public void asyncReadEntry(ReadHandle lh, PositionImpl position, AsyncCallbacks.ReadEntryCallback callback, + public void asyncReadEntry(ReadHandle lh, Position position, AsyncCallbacks.ReadEntryCallback callback, Object ctx) { - lh.readAsync(position.getEntryId(), position.getEntryId()).whenCompleteAsync( + ReadEntryUtils.readAsync(ml, lh, position.getEntryId(), position.getEntryId()).whenCompleteAsync( (ledgerEntries, exception) -> { if (exception != null) { ml.invalidateLedgerHandle(lh); @@ -121,7 +121,7 @@ public void asyncReadEntry(ReadHandle lh, PositionImpl position, AsyncCallbacks. LedgerEntry ledgerEntry = iterator.next(); EntryImpl returnEntry = RangeEntryCacheManagerImpl.create(ledgerEntry, interceptor); - ml.getMbean().recordReadEntriesOpsCacheMisses(); + ml.getMbean().recordReadEntriesOpsCacheMisses(1, returnEntry.getLength()); ml.getFactory().getMbean().recordCacheMiss(1, returnEntry.getLength()); ml.getMbean().addReadEntriesSample(1, returnEntry.getLength()); callback.readEntryComplete(returnEntry, ctx); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/InflightReadsLimiter.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/InflightReadsLimiter.java index b946dc09a0c71..c87807b86631b 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/InflightReadsLimiter.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/InflightReadsLimiter.java @@ -19,20 +19,37 @@ package org.apache.bookkeeper.mledger.impl.cache; import com.google.common.annotations.VisibleForTesting; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.metrics.ObservableLongCounter; import io.prometheus.client.Gauge; import lombok.AllArgsConstructor; import lombok.ToString; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.opentelemetry.Constants; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.InflightReadLimiterUtilization; +import org.apache.pulsar.opentelemetry.annotations.PulsarDeprecatedMetric; @Slf4j -public class InflightReadsLimiter { +public class InflightReadsLimiter implements AutoCloseable { + public static final String INFLIGHT_READS_LIMITER_LIMIT_METRIC_NAME = + "pulsar.broker.managed_ledger.inflight.read.limit"; + private final ObservableLongCounter inflightReadsLimitCounter; + + @PulsarDeprecatedMetric(newMetricName = INFLIGHT_READS_LIMITER_LIMIT_METRIC_NAME) + @Deprecated private static final Gauge PULSAR_ML_READS_BUFFER_SIZE = Gauge .build() .name("pulsar_ml_reads_inflight_bytes") .help("Estimated number of bytes retained by data read from storage or cache") .register(); + public static final String INFLIGHT_READS_LIMITER_USAGE_METRIC_NAME = + "pulsar.broker.managed_ledger.inflight.read.usage"; + private final ObservableLongCounter inflightReadsUsageCounter; + + @PulsarDeprecatedMetric(newMetricName = INFLIGHT_READS_LIMITER_USAGE_METRIC_NAME) + @Deprecated private static final Gauge PULSAR_ML_READS_AVAILABLE_BUFFER_SIZE = Gauge .build() .name("pulsar_ml_reads_available_inflight_bytes") @@ -42,7 +59,7 @@ public class InflightReadsLimiter { private final long maxReadsInFlightSize; private long remainingBytes; - public InflightReadsLimiter(long maxReadsInFlightSize) { + public InflightReadsLimiter(long maxReadsInFlightSize, OpenTelemetry openTelemetry) { if (maxReadsInFlightSize <= 0) { // set it to -1 in order to show in the metrics that the metric is not available PULSAR_ML_READS_BUFFER_SIZE.set(-1); @@ -50,6 +67,28 @@ public InflightReadsLimiter(long maxReadsInFlightSize) { } this.maxReadsInFlightSize = maxReadsInFlightSize; this.remainingBytes = maxReadsInFlightSize; + + var meter = openTelemetry.getMeter(Constants.BROKER_INSTRUMENTATION_SCOPE_NAME); + inflightReadsLimitCounter = meter.counterBuilder(INFLIGHT_READS_LIMITER_LIMIT_METRIC_NAME) + .setDescription("Maximum number of bytes that can be retained by managed ledger data read from storage " + + "or cache.") + .setUnit("By") + .buildWithCallback(measurement -> { + if (!isDisabled()) { + measurement.record(maxReadsInFlightSize); + } + }); + inflightReadsUsageCounter = meter.counterBuilder(INFLIGHT_READS_LIMITER_USAGE_METRIC_NAME) + .setDescription("Estimated number of bytes retained by managed ledger data read from storage or cache.") + .setUnit("By") + .buildWithCallback(measurement -> { + if (!isDisabled()) { + var freeBytes = getRemainingBytes(); + var usedBytes = maxReadsInFlightSize - freeBytes; + measurement.record(freeBytes, InflightReadLimiterUtilization.FREE.attributes); + measurement.record(usedBytes, InflightReadLimiterUtilization.USED.attributes); + } + }); } @VisibleForTesting @@ -57,6 +96,12 @@ public synchronized long getRemainingBytes() { return remainingBytes; } + @Override + public void close() { + inflightReadsLimitCounter.close(); + inflightReadsUsageCounter.close(); + } + @AllArgsConstructor @ToString static class Handle { diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/PooledByteBufAllocatorStats.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/PooledByteBufAllocatorStats.java new file mode 100644 index 0000000000000..4f6a18cb5d934 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/PooledByteBufAllocatorStats.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl.cache; + +import io.netty.buffer.PooledByteBufAllocator; +import lombok.Value; + +@Value +public class PooledByteBufAllocatorStats { + + public long activeAllocations; + public long activeAllocationsSmall; + public long activeAllocationsNormal; + public long activeAllocationsHuge; + + public long totalAllocated; + public long totalUsed; + + public PooledByteBufAllocatorStats(PooledByteBufAllocator allocator) { + long activeAllocations = 0; + long activeAllocationsSmall = 0; + long activeAllocationsNormal = 0; + long activeAllocationsHuge = 0; + long totalAllocated = 0; + long totalUsed = 0; + + for (var arena : allocator.metric().directArenas()) { + activeAllocations += arena.numActiveAllocations(); + activeAllocationsSmall += arena.numActiveSmallAllocations(); + activeAllocationsNormal += arena.numActiveNormalAllocations(); + activeAllocationsHuge += arena.numActiveHugeAllocations(); + + for (var list : arena.chunkLists()) { + for (var chunk : list) { + int size = chunk.chunkSize(); + int used = size - chunk.freeBytes(); + + totalAllocated += size; + totalUsed += used; + } + } + } + + this.activeAllocations = activeAllocations; + this.activeAllocationsSmall = activeAllocationsSmall; + this.activeAllocationsNormal = activeAllocationsNormal; + this.activeAllocationsHuge = activeAllocationsHuge; + + this.totalAllocated = totalAllocated; + this.totalUsed = totalUsed; + } +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java index 7747f9bcd93b6..cb006a5f0cea9 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java @@ -39,9 +39,10 @@ import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.EntryImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.bookkeeper.mledger.intercept.ManagedLedgerInterceptor; import org.apache.bookkeeper.mledger.util.RangeCache; import org.apache.commons.lang3.tuple.Pair; @@ -61,7 +62,7 @@ public class RangeEntryCacheImpl implements EntryCache { private final RangeEntryCacheManagerImpl manager; final ManagedLedgerImpl ml; private ManagedLedgerInterceptor interceptor; - private final RangeCache entries; + private final RangeCache entries; private final boolean copyEntries; private final PendingReadsManager pendingReadsManager; @@ -130,7 +131,7 @@ public boolean insert(EntryImpl entry) { entry.getLength()); } - PositionImpl position = entry.getPosition(); + Position position = entry.getPosition(); if (entries.exists(position)) { return false; } @@ -182,8 +183,8 @@ private ByteBuf copyEntry(EntryImpl entry) { } @Override - public void invalidateEntries(final PositionImpl lastPosition) { - final PositionImpl firstPosition = PositionImpl.get(-1, 0); + public void invalidateEntries(final Position lastPosition) { + final Position firstPosition = PositionFactory.create(-1, 0); if (firstPosition.compareTo(lastPosition) > 0) { if (log.isDebugEnabled()) { @@ -206,8 +207,8 @@ public void invalidateEntries(final PositionImpl lastPosition) { @Override public void invalidateAllEntries(long ledgerId) { - final PositionImpl firstPosition = PositionImpl.get(ledgerId, 0); - final PositionImpl lastPosition = PositionImpl.get(ledgerId + 1, 0); + final Position firstPosition = PositionFactory.create(ledgerId, 0); + final Position lastPosition = PositionFactory.create(ledgerId + 1, 0); Pair removed = entries.removeRange(firstPosition, lastPosition, false); int entriesRemoved = removed.getLeft(); @@ -222,7 +223,7 @@ public void invalidateAllEntries(long ledgerId) { } @Override - public void asyncReadEntry(ReadHandle lh, PositionImpl position, final ReadEntryCallback callback, + public void asyncReadEntry(ReadHandle lh, Position position, final ReadEntryCallback callback, final Object ctx) { try { asyncReadEntry0(lh, position, callback, ctx); @@ -236,7 +237,7 @@ public void asyncReadEntry(ReadHandle lh, PositionImpl position, final ReadEntry } } - private void asyncReadEntry0(ReadHandle lh, PositionImpl position, final ReadEntryCallback callback, + private void asyncReadEntry0(ReadHandle lh, Position position, final ReadEntryCallback callback, final Object ctx) { if (log.isDebugEnabled()) { log.debug("[{}] Reading entry ledger {}: {}", ml.getName(), lh.getId(), position.getEntryId()); @@ -248,7 +249,7 @@ private void asyncReadEntry0(ReadHandle lh, PositionImpl position, final ReadEnt manager.mlFactoryMBean.recordCacheHit(cachedEntry.getLength()); callback.readEntryComplete(cachedEntry, ctx); } else { - lh.readAsync(position.getEntryId(), position.getEntryId()).thenAcceptAsync( + ReadEntryUtils.readAsync(ml, lh, position.getEntryId(), position.getEntryId()).thenAcceptAsync( ledgerEntries -> { try { Iterator iterator = ledgerEntries.iterator(); @@ -256,7 +257,7 @@ private void asyncReadEntry0(ReadHandle lh, PositionImpl position, final ReadEnt LedgerEntry ledgerEntry = iterator.next(); EntryImpl returnEntry = RangeEntryCacheManagerImpl.create(ledgerEntry, interceptor); - ml.getMbean().recordReadEntriesOpsCacheMisses(); + ml.getMbean().recordReadEntriesOpsCacheMisses(1, returnEntry.getLength()); manager.mlFactoryMBean.recordCacheMiss(1, returnEntry.getLength()); ml.getMbean().addReadEntriesSample(1, returnEntry.getLength()); callback.readEntryComplete(returnEntry, ctx); @@ -310,8 +311,8 @@ void asyncReadEntry0WithLimits(ReadHandle lh, long firstEntry, long lastEntry, b final long ledgerId = lh.getId(); final int entriesToRead = (int) (lastEntry - firstEntry) + 1; - final PositionImpl firstPosition = PositionImpl.get(lh.getId(), firstEntry); - final PositionImpl lastPosition = PositionImpl.get(lh.getId(), lastEntry); + final Position firstPosition = PositionFactory.create(lh.getId(), firstEntry); + final Position lastPosition = PositionFactory.create(lh.getId(), lastEntry); if (log.isDebugEnabled()) { log.debug("[{}] Reading entries range ledger {}: {} to {}", ml.getName(), ledgerId, firstEntry, lastEntry); @@ -428,7 +429,7 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { CompletableFuture> readFromStorage(ReadHandle lh, long firstEntry, long lastEntry, boolean shouldCacheEntry) { final int entriesToRead = (int) (lastEntry - firstEntry) + 1; - CompletableFuture> readResult = lh.readAsync(firstEntry, lastEntry) + CompletableFuture> readResult = ReadEntryUtils.readAsync(ml, lh, firstEntry, lastEntry) .thenApply( ledgerEntries -> { requireNonNull(ml.getName()); @@ -450,7 +451,7 @@ CompletableFuture> readFromStorage(ReadHandle lh, } } - ml.getMbean().recordReadEntriesOpsCacheMisses(); + ml.getMbean().recordReadEntriesOpsCacheMisses(entriesToReturn.size(), totalSize); manager.mlFactoryMBean.recordCacheMiss(entriesToReturn.size(), totalSize); ml.getMbean().addReadEntriesSample(entriesToReturn.size(), totalSize); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java index d5a3019855cb5..34be25df1f476 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java @@ -20,6 +20,7 @@ import com.google.common.collect.Lists; import io.netty.buffer.ByteBuf; +import io.opentelemetry.api.OpenTelemetry; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; @@ -56,10 +57,10 @@ public class RangeEntryCacheManagerImpl implements EntryCacheManager { private static final double evictionTriggerThresholdPercent = 0.98; - public RangeEntryCacheManagerImpl(ManagedLedgerFactoryImpl factory) { + public RangeEntryCacheManagerImpl(ManagedLedgerFactoryImpl factory, OpenTelemetry openTelemetry) { this.maxSize = factory.getConfig().getMaxCacheSize(); this.inflightReadsLimiter = new InflightReadsLimiter( - factory.getConfig().getManagedLedgerMaxReadsInFlightSize()); + factory.getConfig().getManagedLedgerMaxReadsInFlightSize(), openTelemetry); this.evictionTriggerThreshold = (long) (maxSize * evictionTriggerThresholdPercent); this.cacheEvictionWatermark = factory.getConfig().getCacheEvictionWatermark(); this.evictionPolicy = new EntryCacheDefaultEvictionPolicy(); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/ReadEntryUtils.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/ReadEntryUtils.java new file mode 100644 index 0000000000000..5cf5f053f0ce7 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/ReadEntryUtils.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl.cache; + +import java.util.concurrent.CompletableFuture; +import org.apache.bookkeeper.client.api.LedgerEntries; +import org.apache.bookkeeper.client.api.ReadHandle; +import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerException; + +class ReadEntryUtils { + + static CompletableFuture readAsync(ManagedLedger ml, ReadHandle handle, long firstEntry, + long lastEntry) { + if (ml.getOptionalLedgerInfo(handle.getId()).isEmpty()) { + // The read handle comes from another managed ledger, in this case, we can only compare the entry range with + // the LAC of that read handle. Specifically, it happens when this method is called by a + // ReadOnlyManagedLedgerImpl object. + return handle.readAsync(firstEntry, lastEntry); + } + // Compare the entry range with the lastConfirmedEntry maintained by the managed ledger because the entry cache + // of `ShadowManagedLedgerImpl` reads entries via `ReadOnlyLedgerHandle`, which never updates `lastAddConfirmed` + final var lastConfirmedEntry = ml.getLastConfirmedEntry(); + if (lastConfirmedEntry == null) { + return CompletableFuture.failedFuture(new ManagedLedgerException( + "LastConfirmedEntry is null when reading ledger " + handle.getId())); + } + if (handle.getId() > lastConfirmedEntry.getLedgerId()) { + return CompletableFuture.failedFuture(new ManagedLedgerException("LastConfirmedEntry is " + + lastConfirmedEntry + " when reading ledger " + handle.getId())); + } + if (handle.getId() == lastConfirmedEntry.getLedgerId() && lastEntry > lastConfirmedEntry.getEntryId()) { + return CompletableFuture.failedFuture(new ManagedLedgerException("LastConfirmedEntry is " + + lastConfirmedEntry + " when reading entry " + lastEntry)); + } + return handle.readUnconfirmedAsync(firstEntry, lastEntry); + } +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/intercept/ManagedLedgerInterceptor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/intercept/ManagedLedgerInterceptor.java index d26a5e15735aa..0ca6fa9dd866c 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/intercept/ManagedLedgerInterceptor.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/intercept/ManagedLedgerInterceptor.java @@ -20,11 +20,11 @@ import io.netty.buffer.ByteBuf; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; -import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.common.annotation.InterfaceAudience; import org.apache.bookkeeper.common.annotation.InterfaceStability; -import org.apache.bookkeeper.mledger.impl.OpAddEntry; +import org.apache.bookkeeper.mledger.Entry; /** * Interceptor for ManagedLedger. @@ -32,14 +32,34 @@ @InterfaceAudience.LimitedPrivate @InterfaceStability.Stable public interface ManagedLedgerInterceptor { + /** + * An operation to add an entry to a ledger. + */ + interface AddEntryOperation { + /** + * Get the data to be written to the ledger. + * @return data to be written to the ledger + */ + ByteBuf getData(); + /** + * Set the data to be written to the ledger. + * @param data data to be written to the ledger + */ + void setData(ByteBuf data); + /** + * Get the operation context object. + * @return context the context object + */ + Object getCtx(); + } /** - * Intercept an OpAddEntry and return an OpAddEntry. - * @param op an OpAddEntry to be intercepted. + * Intercept adding an entry to a ledger. + * + * @param op an operation to be intercepted. * @param numberOfMessages - * @return an OpAddEntry. */ - OpAddEntry beforeAddEntry(OpAddEntry op, int numberOfMessages); + void beforeAddEntry(AddEntryOperation op, int numberOfMessages); /** * Intercept When add entry failed. @@ -55,12 +75,25 @@ default void afterFailedAddEntry(int numberOfMessages){ */ void onManagedLedgerPropertiesInitialize(Map propertiesMap); + /** + * A handle for reading the last ledger entry. + */ + interface LastEntryHandle { + /** + * Read the last entry from the ledger. + * The caller is responsible for releasing the entry. + * @return the last entry from the ledger, if any + */ + CompletableFuture> readLastEntryAsync(); + } + /** * Intercept when ManagedLedger is initialized. - * @param name name of ManagedLedger - * @param ledgerHandle a LedgerHandle. + * + * @param name name of ManagedLedger + * @param lastEntryHandle a LedgerHandle. */ - CompletableFuture onManagedLedgerLastLedgerInitialize(String name, LedgerHandle ledgerHandle); + CompletableFuture onManagedLedgerLastLedgerInitialize(String name, LastEntryHandle lastEntryHandle); /** * @param propertiesMap map of properties. @@ -93,12 +126,12 @@ default PayloadProcessorHandle processPayloadBeforeEntryCache(ByteBuf dataReadFr /** * Intercept before payload gets written to ledger. - * @param ledgerWriteOp OpAddEntry used to trigger ledger write. + * @param ctx the operation context object * @param dataToBeStoredInLedger data to be stored in ledger * @return handle to the processor */ - default PayloadProcessorHandle processPayloadBeforeLedgerWrite(OpAddEntry ledgerWriteOp, - ByteBuf dataToBeStoredInLedger){ + default PayloadProcessorHandle processPayloadBeforeLedgerWrite(Object ctx, + ByteBuf dataToBeStoredInLedger) { return null; } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloadUtils.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloadUtils.java index 550626f76c000..9c9feb2aa7f7c 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloadUtils.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloadUtils.java @@ -198,7 +198,7 @@ public static CompletableFuture cleanupOffloaded(long ledgerId, UUID uuid, metadataMap.put("ManagedLedgerName", name); return Retries.run(Backoff.exponentialJittered(TimeUnit.SECONDS.toMillis(1), - TimeUnit.SECONDS.toHours(1)).limit(10), + TimeUnit.HOURS.toMillis(1)).limit(10), Retries.NonFatalPredicate, () -> mlConfig.getLedgerOffloader().deleteOffloaded(ledgerId, uuid, metadataMap), executor, name).whenComplete((ignored, exception) -> { diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloaderUtils.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloaderUtils.java index a66af642958b7..7af3680d880b2 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloaderUtils.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloaderUtils.java @@ -53,7 +53,6 @@ static Pair getOffloaderFactory(String n throws IOException { // need to load offloader NAR to the classloader that also loaded LedgerOffloaderFactory in case // LedgerOffloaderFactory is loaded by a classloader that is not the default classloader - // as is the case for the pulsar presto plugin NarClassLoader ncl = NarClassLoaderBuilder.builder() .narFile(new File(narPath)) .parentClassLoader(LedgerOffloaderFactory.class.getClassLoader()) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/Offloaders.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/Offloaders.java index 6910439e09131..cec15599242ae 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/Offloaders.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/Offloaders.java @@ -46,6 +46,12 @@ public LedgerOffloaderFactory getOffloaderFactory(String driverName) throws IOEx @Override public void close() throws Exception { offloaders.forEach(offloader -> { + try { + offloader.getRight().close(); + } catch (Exception e) { + log.warn("Failed to close offloader '{}': {}", + offloader.getRight().getClass(), e.getMessage()); + } try { offloader.getLeft().close(); } catch (IOException e) { diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/ManagedLedgerImplUtils.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/ManagedLedgerImplUtils.java new file mode 100644 index 0000000000000..d13d71d0e5a13 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/ManagedLedgerImplUtils.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.util; + +import java.util.concurrent.CompletableFuture; +import java.util.function.Predicate; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.pulsar.common.classification.InterfaceStability; + +@InterfaceStability.Evolving +public class ManagedLedgerImplUtils { + + /** + * Reverse find last valid position one-entry by one-entry. + */ + public static CompletableFuture asyncGetLastValidPosition(final ManagedLedgerImpl ledger, + final Predicate predicate, + final Position startPosition) { + CompletableFuture future = new CompletableFuture<>(); + internalAsyncReverseFindPositionOneByOne(ledger, predicate, startPosition, future); + return future; + } + + private static void internalAsyncReverseFindPositionOneByOne(final ManagedLedgerImpl ledger, + final Predicate predicate, + final Position position, + final CompletableFuture future) { + if (!ledger.isValidPosition(position)) { + future.complete(position); + return; + } + ledger.asyncReadEntry(position, new AsyncCallbacks.ReadEntryCallback() { + @Override + public void readEntryComplete(Entry entry, Object ctx) { + final Position position = entry.getPosition(); + try { + if (predicate.test(entry)) { + future.complete(position); + return; + } + Position previousPosition = ledger.getPreviousPosition(position); + internalAsyncReverseFindPositionOneByOne(ledger, predicate, previousPosition, future); + } catch (Exception e) { + future.completeExceptionally(e); + } finally { + entry.release(); + } + } + + @Override + public void readEntryFailed(ManagedLedgerException exception, Object ctx) { + future.completeExceptionally(exception); + } + }, null); + } +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/PositionAckSetUtil.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/PositionAckSetUtil.java index 1c607582076a8..a7442215264e4 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/PositionAckSetUtil.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/PositionAckSetUtil.java @@ -18,7 +18,9 @@ */ package org.apache.bookkeeper.mledger.util; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.AckSetState; +import org.apache.bookkeeper.mledger.impl.AckSetStateUtil; import org.apache.pulsar.common.util.collections.BitSetRecyclable; public class PositionAckSetUtil { @@ -41,11 +43,13 @@ public static boolean isAckSetOverlap(long[] currentAckSet, long[] otherAckSet) } //This method is do `and` operation for position's ack set - public static void andAckSet(PositionImpl currentPosition, PositionImpl otherPosition) { + public static void andAckSet(Position currentPosition, Position otherPosition) { if (currentPosition == null || otherPosition == null) { return; } - currentPosition.setAckSet(andAckSet(currentPosition.getAckSet(), otherPosition.getAckSet())); + AckSetState currentAckSetState = AckSetStateUtil.getAckSetState(currentPosition); + AckSetState otherAckSetState = AckSetStateUtil.getAckSetState(otherPosition); + currentAckSetState.setAckSet(andAckSet(currentAckSetState.getAckSet(), otherAckSetState.getAckSet())); } //This method is do `and` operation for ack set @@ -69,7 +73,7 @@ public static boolean isAckSetEmpty(long[] ackSet) { //This method is compare two position which position is bigger than another one. //When the ledgerId and entryId in this position is same to another one and two position all have ack set, it will //compare the ack set next bit index is bigger than another one. - public static int compareToWithAckSet(PositionImpl currentPosition, PositionImpl otherPosition) { + public static int compareToWithAckSet(Position currentPosition, Position otherPosition) { if (currentPosition == null || otherPosition == null) { throw new IllegalArgumentException("Two positions can't be null! " + "current position : [" + currentPosition + "] other position : [" + otherPosition + "]"); @@ -79,16 +83,18 @@ public static int compareToWithAckSet(PositionImpl currentPosition, PositionImpl BitSetRecyclable otherAckSet; BitSetRecyclable currentAckSet; - if (otherPosition.getAckSet() == null) { + long[] otherAckSetArr = AckSetStateUtil.getAckSetArrayOrNull(otherPosition); + if (otherAckSetArr == null) { otherAckSet = BitSetRecyclable.create(); } else { - otherAckSet = BitSetRecyclable.valueOf(otherPosition.getAckSet()); + otherAckSet = BitSetRecyclable.valueOf(otherAckSetArr); } - if (currentPosition.getAckSet() == null) { + long[] currentAckSetArr = AckSetStateUtil.getAckSetArrayOrNull(currentPosition); + if (currentAckSetArr == null) { currentAckSet = BitSetRecyclable.create(); } else { - currentAckSet = BitSetRecyclable.valueOf(currentPosition.getAckSet()); + currentAckSet = BitSetRecyclable.valueOf(currentAckSetArr); } if (currentAckSet.isEmpty() || otherAckSet.isEmpty()) { diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java index 7599e2cc1874f..2f2b161a30684 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java @@ -19,6 +19,10 @@ package org.apache.bookkeeper.mledger.util; import static com.google.common.base.Preconditions.checkArgument; +import com.google.common.base.Predicate; +import io.netty.util.IllegalReferenceCountException; +import io.netty.util.Recycler; +import io.netty.util.Recycler.Handle; import io.netty.util.ReferenceCounted; import java.util.ArrayList; import java.util.Collection; @@ -27,24 +31,153 @@ import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.atomic.AtomicLong; -import org.apache.commons.lang3.mutable.MutableBoolean; +import java.util.concurrent.locks.StampedLock; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.util.RangeCache.ValueWithKeyValidation; import org.apache.commons.lang3.tuple.Pair; /** * Special type of cache where get() and delete() operations can be done over a range of keys. + * The implementation avoids locks and synchronization by relying on ConcurrentSkipListMap for storing the entries. + * Since there are no locks, it's necessary to ensure that a single entry in the cache is removed exactly once. + * Removing an entry multiple times could result in the entries of the cache being released multiple times, + * even while they are still in use. This is prevented by using a custom wrapper around the value to store in the map + * that ensures that the value is removed from the map only if the exact same instance is present in the map. + * There's also a check that ensures that the value matches the key. This is used to detect races without impacting + * consistency. * * @param * Cache key. Needs to be Comparable * @param * Cache value */ -public class RangeCache, Value extends ReferenceCounted> { +@Slf4j +public class RangeCache, Value extends ValueWithKeyValidation> { + public interface ValueWithKeyValidation extends ReferenceCounted { + boolean matchesKey(T key); + } + // Map from key to nodes inside the linked list - private final ConcurrentNavigableMap entries; + private final ConcurrentNavigableMap> entries; private AtomicLong size; // Total size of values stored in cache private final Weighter weighter; // Weighter object used to extract the size from values private final TimestampExtractor timestampExtractor; // Extract the timestamp associated with a value + /** + * Wrapper around the value to store in Map. This is needed to ensure that a specific instance can be removed from + * the map by calling the {@link Map#remove(Object, Object)} method. Certain race conditions could result in the + * wrong value being removed from the map. The instances of this class are recycled to avoid creating new objects. + */ + private static class EntryWrapper { + private final Handle recyclerHandle; + private static final Recycler RECYCLER = new Recycler() { + @Override + protected EntryWrapper newObject(Handle recyclerHandle) { + return new EntryWrapper(recyclerHandle); + } + }; + private final StampedLock lock = new StampedLock(); + private K key; + private V value; + long size; + + private EntryWrapper(Handle recyclerHandle) { + this.recyclerHandle = recyclerHandle; + } + + static EntryWrapper create(K key, V value, long size) { + EntryWrapper entryWrapper = RECYCLER.get(); + long stamp = entryWrapper.lock.writeLock(); + entryWrapper.key = key; + entryWrapper.value = value; + entryWrapper.size = size; + entryWrapper.lock.unlockWrite(stamp); + return entryWrapper; + } + + K getKey() { + long stamp = lock.tryOptimisticRead(); + K localKey = key; + if (!lock.validate(stamp)) { + stamp = lock.readLock(); + localKey = key; + lock.unlockRead(stamp); + } + return localKey; + } + + V getValue(K key) { + long stamp = lock.tryOptimisticRead(); + K localKey = this.key; + V localValue = this.value; + if (!lock.validate(stamp)) { + stamp = lock.readLock(); + localKey = this.key; + localValue = this.value; + lock.unlockRead(stamp); + } + if (localKey != key) { + return null; + } + return localValue; + } + + long getSize() { + long stamp = lock.tryOptimisticRead(); + long localSize = size; + if (!lock.validate(stamp)) { + stamp = lock.readLock(); + localSize = size; + lock.unlockRead(stamp); + } + return localSize; + } + + void recycle() { + key = null; + value = null; + size = 0; + recyclerHandle.recycle(this); + } + } + + /** + * Mutable object to store the number of entries and the total size removed from the cache. The instances + * are recycled to avoid creating new instances. + */ + private static class RemovalCounters { + private final Handle recyclerHandle; + private static final Recycler RECYCLER = new Recycler() { + @Override + protected RemovalCounters newObject(Handle recyclerHandle) { + return new RemovalCounters(recyclerHandle); + } + }; + int removedEntries; + long removedSize; + private RemovalCounters(Handle recyclerHandle) { + this.recyclerHandle = recyclerHandle; + } + + static RemovalCounters create() { + RemovalCounters results = RECYCLER.get(); + results.removedEntries = 0; + results.removedSize = 0; + return results; + } + + void recycle() { + removedEntries = 0; + removedSize = 0; + recyclerHandle.recycle(this); + } + + public void entryRemoved(long size) { + removedSize += size; + removedEntries++; + } + } + /** * Construct a new RangeLruCache with default Weighter. */ @@ -69,34 +202,66 @@ public RangeCache(Weighter weighter, TimestampExtractor timestampE * Insert. * * @param key - * @param value - * ref counted value with at least 1 ref to pass on the cache + * @param value ref counted value with at least 1 ref to pass on the cache * @return whether the entry was inserted in the cache */ public boolean put(Key key, Value value) { - MutableBoolean flag = new MutableBoolean(); - entries.computeIfAbsent(key, (k) -> { - size.addAndGet(weighter.getSize(value)); - flag.setValue(true); - return value; - }); - return flag.booleanValue(); + // retain value so that it's not released before we put it in the cache and calculate the weight + value.retain(); + try { + if (!value.matchesKey(key)) { + throw new IllegalArgumentException("Value '" + value + "' does not match key '" + key + "'"); + } + long entrySize = weighter.getSize(value); + EntryWrapper newWrapper = EntryWrapper.create(key, value, entrySize); + if (entries.putIfAbsent(key, newWrapper) == null) { + this.size.addAndGet(entrySize); + return true; + } else { + // recycle the new wrapper as it was not used + newWrapper.recycle(); + return false; + } + } finally { + value.release(); + } } public boolean exists(Key key) { return key != null ? entries.containsKey(key) : true; } + /** + * Get the value associated with the key and increment the reference count of it. + * The caller is responsible for releasing the reference. + */ public Value get(Key key) { - Value value = entries.get(key); - if (value == null) { + return getValue(key, entries.get(key)); + } + + private Value getValue(Key key, EntryWrapper valueWrapper) { + if (valueWrapper == null) { return null; } else { + Value value = valueWrapper.getValue(key); + if (value == null) { + // the wrapper has been recycled and contains another key + return null; + } try { value.retain(); + } catch (IllegalReferenceCountException e) { + // Value was already deallocated + return null; + } + // check that the value matches the key and that there's at least 2 references to it since + // the cache should be holding one reference and a new reference was just added in this method + if (value.refCnt() > 1 && value.matchesKey(key)) { return value; - } catch (Throwable t) { - // Value was already destroyed between get() and retain() + } else { + // Value or IdentityWrapper was recycled and already contains another value + // release the reference added in this method + value.release(); return null; } } @@ -114,12 +279,10 @@ public Collection getRange(Key first, Key last) { List values = new ArrayList(); // Return the values of the entries found in cache - for (Value value : entries.subMap(first, true, last, true).values()) { - try { - value.retain(); + for (Map.Entry> entry : entries.subMap(first, true, last, true).entrySet()) { + Value value = getValue(entry.getKey(), entry.getValue()); + if (value != null) { values.add(value); - } catch (Throwable t) { - // Value was already destroyed between get() and retain() } } @@ -134,25 +297,105 @@ public Collection getRange(Key first, Key last) { * @return an pair of ints, containing the number of removed entries and the total size */ public Pair removeRange(Key first, Key last, boolean lastInclusive) { - Map subMap = entries.subMap(first, true, last, lastInclusive); + RemovalCounters counters = RemovalCounters.create(); + Map> subMap = entries.subMap(first, true, last, lastInclusive); + for (Map.Entry> entry : subMap.entrySet()) { + removeEntry(entry, counters, true); + } + return handleRemovalResult(counters); + } - int removedEntries = 0; - long removedSize = 0; + enum RemoveEntryResult { + ENTRY_REMOVED, + CONTINUE_LOOP, + BREAK_LOOP; + } - for (Key key : subMap.keySet()) { - Value value = entries.remove(key); - if (value == null) { - continue; - } + private RemoveEntryResult removeEntry(Map.Entry> entry, RemovalCounters counters, + boolean skipInvalid) { + return removeEntry(entry, counters, skipInvalid, x -> true); + } - removedSize += weighter.getSize(value); + private RemoveEntryResult removeEntry(Map.Entry> entry, RemovalCounters counters, + boolean skipInvalid, Predicate removeCondition) { + Key key = entry.getKey(); + EntryWrapper entryWrapper = entry.getValue(); + Value value = entryWrapper.getValue(key); + if (value == null) { + // the wrapper has already been recycled and contains another key + if (!skipInvalid) { + EntryWrapper removed = entries.remove(key); + if (removed != null) { + // log and remove the entry without releasing the value + log.info("Key {} does not match the entry's value wrapper's key {}, removed entry by key without " + + "releasing the value", key, entryWrapper.getKey()); + counters.entryRemoved(removed.getSize()); + return RemoveEntryResult.ENTRY_REMOVED; + } + } + return RemoveEntryResult.CONTINUE_LOOP; + } + try { + // add extra retain to avoid value being released while we are removing it + value.retain(); + } catch (IllegalReferenceCountException e) { + // Value was already released + if (!skipInvalid) { + // remove the specific entry without releasing the value + if (entries.remove(key, entryWrapper)) { + log.info("Value was already released for key {}, removed entry without releasing the value", key); + counters.entryRemoved(entryWrapper.getSize()); + return RemoveEntryResult.ENTRY_REMOVED; + } + } + return RemoveEntryResult.CONTINUE_LOOP; + } + if (!value.matchesKey(key)) { + // this is unexpected since the IdentityWrapper.getValue(key) already checked that the value matches the key + log.warn("Unexpected race condition. Value {} does not match the key {}. Removing entry.", value, key); + } + try { + if (!removeCondition.test(value)) { + return RemoveEntryResult.BREAK_LOOP; + } + if (!skipInvalid) { + // remove the specific entry + boolean entryRemoved = entries.remove(key, entryWrapper); + if (entryRemoved) { + counters.entryRemoved(entryWrapper.getSize()); + // check that the value hasn't been recycled in between + // there should be at least 2 references since this method adds one and the cache should have + // one reference. it is valid that the value contains references even after the key has been + // removed from the cache + if (value.refCnt() > 1) { + entryWrapper.recycle(); + // remove the cache reference + value.release(); + } else { + log.info("Unexpected refCnt {} for key {}, removed entry without releasing the value", + value.refCnt(), key); + } + } + } else if (skipInvalid && value.refCnt() > 1 && entries.remove(key, entryWrapper)) { + // when skipInvalid is true, we don't remove the entry if it doesn't match matches the key + // or the refCnt is invalid + counters.entryRemoved(entryWrapper.getSize()); + entryWrapper.recycle(); + // remove the cache reference + value.release(); + } + } finally { + // remove the extra retain value.release(); - ++removedEntries; } + return RemoveEntryResult.ENTRY_REMOVED; + } - size.addAndGet(-removedSize); - - return Pair.of(removedEntries, removedSize); + private Pair handleRemovalResult(RemovalCounters counters) { + size.addAndGet(-counters.removedSize); + Pair result = Pair.of(counters.removedEntries, counters.removedSize); + counters.recycle(); + return result; } /** @@ -162,24 +405,15 @@ public Pair removeRange(Key first, Key last, boolean lastInclusiv */ public Pair evictLeastAccessedEntries(long minSize) { checkArgument(minSize > 0); - - long removedSize = 0; - int removedEntries = 0; - - while (removedSize < minSize) { - Map.Entry entry = entries.pollFirstEntry(); + RemovalCounters counters = RemovalCounters.create(); + while (counters.removedSize < minSize && !Thread.currentThread().isInterrupted()) { + Map.Entry> entry = entries.firstEntry(); if (entry == null) { break; } - - Value value = entry.getValue(); - ++removedEntries; - removedSize += weighter.getSize(value); - value.release(); + removeEntry(entry, counters, false); } - - size.addAndGet(-removedSize); - return Pair.of(removedEntries, removedSize); + return handleRemovalResult(counters); } /** @@ -188,27 +422,18 @@ public Pair evictLeastAccessedEntries(long minSize) { * @return the tota */ public Pair evictLEntriesBeforeTimestamp(long maxTimestamp) { - long removedSize = 0; - int removedCount = 0; - - while (true) { - Map.Entry entry = entries.firstEntry(); - if (entry == null || timestampExtractor.getTimestamp(entry.getValue()) > maxTimestamp) { + RemovalCounters counters = RemovalCounters.create(); + while (!Thread.currentThread().isInterrupted()) { + Map.Entry> entry = entries.firstEntry(); + if (entry == null) { break; } - Value value = entry.getValue(); - boolean removeHits = entries.remove(entry.getKey(), value); - if (!removeHits) { + if (removeEntry(entry, counters, false, value -> timestampExtractor.getTimestamp(value) <= maxTimestamp) + == RemoveEntryResult.BREAK_LOOP) { break; } - - removedSize += weighter.getSize(value); - removedCount++; - value.release(); } - - size.addAndGet(-removedSize); - return Pair.of(removedCount, removedSize); + return handleRemovalResult(counters); } /** @@ -227,24 +452,16 @@ public long getSize() { * * @return size of removed entries */ - public synchronized Pair clear() { - long removedSize = 0; - int removedCount = 0; - - while (true) { - Map.Entry entry = entries.pollFirstEntry(); + public Pair clear() { + RemovalCounters counters = RemovalCounters.create(); + while (!Thread.currentThread().isInterrupted()) { + Map.Entry> entry = entries.firstEntry(); if (entry == null) { break; } - Value value = entry.getValue(); - removedSize += weighter.getSize(value); - removedCount++; - value.release(); + removeEntry(entry, counters, false); } - - entries.clear(); - size.getAndAdd(-removedSize); - return Pair.of(removedCount, removedSize); + return handleRemovalResult(counters); } /** @@ -276,5 +493,4 @@ public long getSize(Value value) { return 1; } } - } diff --git a/managed-ledger/src/main/proto/MLDataFormats.proto b/managed-ledger/src/main/proto/MLDataFormats.proto index c4e502819fa9e..f196649df0fdf 100644 --- a/managed-ledger/src/main/proto/MLDataFormats.proto +++ b/managed-ledger/src/main/proto/MLDataFormats.proto @@ -82,6 +82,7 @@ message PositionInfo { // Store which index in the batch message has been deleted repeated BatchedEntryDeletionIndexInfo batchedEntryDeletionIndexInfo = 5; + repeated LongListMap individualDeletedMessageRanges = 6; } message NestedPositionInfo { @@ -89,6 +90,11 @@ message NestedPositionInfo { required int64 entryId = 2; } +message LongListMap { + required int64 key = 1; + repeated int64 values = 2; +} + message MessageRange { required NestedPositionInfo lowerEndpoint = 1; required NestedPositionInfo upperEndpoint = 2; @@ -124,7 +130,8 @@ message ManagedCursorInfo { // the current cursor position repeated LongProperty properties = 5; - optional int64 lastActive = 6; + // deprecated, do not persist this field anymore + optional int64 lastActive = 6 [deprecated = true]; // Store which index in the batch message has been deleted repeated BatchedEntryDeletionIndexInfo batchedEntryDeletionIndexInfo = 7; diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/EntryCacheManagerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/EntryCacheManagerTest.java index 1b02cd674c567..f00efb27ca5ab 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/EntryCacheManagerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/EntryCacheManagerTest.java @@ -28,6 +28,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.Random; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -40,9 +41,12 @@ import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.cache.EntryCache; import org.apache.bookkeeper.mledger.impl.cache.EntryCacheDisabled; import org.apache.bookkeeper.mledger.impl.cache.EntryCacheManager; +import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.bookkeeper.test.MockedBookKeeperTestCase; import org.testng.Assert; import org.testng.annotations.Test; @@ -120,7 +124,7 @@ public void simple() throws Exception { assertEquals(cache2.getSize(), 3); // Should remove 1 entry - cache2.invalidateEntries(new PositionImpl(2, 1)); + cache2.invalidateEntries(PositionFactory.create(2, 1)); assertEquals(cacheManager.getSize(), 2); assertEquals(cache2.getSize(), 2); @@ -193,9 +197,9 @@ public void cacheSizeUpdate() throws Exception { } cacheManager.removeEntryCache(ml1.getName()); - assertTrue(cacheManager.getSize() > 0); assertEquals(factory2.getMbean().getCacheInsertedEntriesCount(), 20); assertEquals(factory2.getMbean().getCacheEntriesCount(), 0); + assertEquals(0, cacheManager.getSize()); assertEquals(factory2.getMbean().getCacheEvictedEntriesCount(), 20); } @@ -330,7 +334,7 @@ public void verifyHitsMisses() throws Exception { assertEquals(factory2.getMbean().getCacheHitsThroughput(), 70.0); assertEquals(factory2.getMbean().getNumberOfCacheEvictions(), 0); - PositionImpl pos = (PositionImpl) entries.get(entries.size() - 1).getPosition(); + Position pos = entries.get(entries.size() - 1).getPosition(); c2.setReadPosition(pos); entries.forEach(Entry::release); @@ -390,7 +394,10 @@ void entryCacheDisabledAsyncReadEntry() throws Exception { EntryCache entryCache = cacheManager.getEntryCache(ml1); final CountDownLatch counter = new CountDownLatch(1); - entryCache.asyncReadEntry(lh, new PositionImpl(1L,1L), new AsyncCallbacks.ReadEntryCallback() { + when(ml1.getLastConfirmedEntry()).thenReturn(PositionFactory.create(1L, 1L)); + when(ml1.getOptionalLedgerInfo(lh.getId())).thenReturn(Optional.of(mock( + MLDataFormats.ManagedLedgerInfo.LedgerInfo.class))); + entryCache.asyncReadEntry(lh, PositionFactory.create(1L,1L), new AsyncCallbacks.ReadEntryCallback() { public void readEntryComplete(Entry entry, Object ctx) { Assert.assertNotEquals(entry, null); entry.release(); @@ -404,7 +411,7 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { }, null); counter.await(); - verify(lh).readAsync(anyLong(), anyLong()); + verify(lh).readUnconfirmedAsync(anyLong(), anyLong()); } } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/EntryCacheTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/EntryCacheTest.java index c8338798f271b..551aa80bc07dc 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/EntryCacheTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/EntryCacheTest.java @@ -25,14 +25,16 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; import io.netty.buffer.Unpooled; - import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CountDownLatch; - +import java.util.concurrent.ExecutionException; +import java.util.function.Consumer; import lombok.Cleanup; import org.apache.bookkeeper.client.BKException.BKNoSuchLedgerExistsException; import org.apache.bookkeeper.client.api.LedgerEntries; @@ -43,10 +45,11 @@ import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.cache.EntryCache; import org.apache.bookkeeper.mledger.impl.cache.EntryCacheManager; +import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.bookkeeper.test.MockedBookKeeperTestCase; -import org.testng.Assert; import org.testng.annotations.Test; public class EntryCacheTest extends MockedBookKeeperTestCase { @@ -60,6 +63,8 @@ protected void setUpTestCase() throws Exception { when(ml.getExecutor()).thenReturn(executor); when(ml.getMbean()).thenReturn(new ManagedLedgerMBeanImpl(ml)); when(ml.getConfig()).thenReturn(new ManagedLedgerConfig()); + when(ml.getOptionalLedgerInfo(0L)).thenReturn(Optional.of(mock( + MLDataFormats.ManagedLedgerInfo.LedgerInfo.class))); } @Test(timeOut = 5000) @@ -76,22 +81,13 @@ public void testRead() throws Exception { entryCache.insert(EntryImpl.create(0, i, data)); } - final CountDownLatch counter = new CountDownLatch(1); - - entryCache.asyncReadEntry(lh, 0, 9, false, new ReadEntriesCallback() { - public void readEntriesComplete(List entries, Object ctx) { - assertEquals(entries.size(), 10); - entries.forEach(Entry::release); - counter.countDown(); - } - - public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { - Assert.fail("should not have failed"); - } - }, null); - counter.await(); + when(ml.getLastConfirmedEntry()).thenReturn(PositionFactory.create(0, 9)); + final var entries = readEntry(entryCache, lh, 0, 9, false, null); + assertEquals(entries.size(), 10); + entries.forEach(Entry::release); // Verify no entries were read from bookkeeper + verify(lh, never()).readUnconfirmedAsync(anyLong(), anyLong()); verify(lh, never()).readAsync(anyLong(), anyLong()); } @@ -109,19 +105,9 @@ public void testReadMissingBefore() throws Exception { entryCache.insert(EntryImpl.create(0, i, data)); } - final CountDownLatch counter = new CountDownLatch(1); - - entryCache.asyncReadEntry(lh, 0, 9, false, new ReadEntriesCallback() { - public void readEntriesComplete(List entries, Object ctx) { - assertEquals(entries.size(), 10); - counter.countDown(); - } - - public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { - Assert.fail("should not have failed"); - } - }, null); - counter.await(); + when(ml.getLastConfirmedEntry()).thenReturn(PositionFactory.create(0, 9)); + final var entries = readEntry(entryCache, lh, 0, 9, false, null); + assertEquals(entries.size(), 10); } @Test(timeOut = 5000) @@ -138,19 +124,9 @@ public void testReadMissingAfter() throws Exception { entryCache.insert(EntryImpl.create(0, i, data)); } - final CountDownLatch counter = new CountDownLatch(1); - - entryCache.asyncReadEntry(lh, 0, 9, false, new ReadEntriesCallback() { - public void readEntriesComplete(List entries, Object ctx) { - assertEquals(entries.size(), 10); - counter.countDown(); - } - - public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { - Assert.fail("should not have failed"); - } - }, null); - counter.await(); + when(ml.getLastConfirmedEntry()).thenReturn(PositionFactory.create(0, 9)); + final var entries = readEntry(entryCache, lh, 0, 9, false, null); + assertEquals(entries.size(), 10); } @Test(timeOut = 5000) @@ -168,19 +144,9 @@ public void testReadMissingMiddle() throws Exception { entryCache.insert(EntryImpl.create(0, 8, data)); entryCache.insert(EntryImpl.create(0, 9, data)); - final CountDownLatch counter = new CountDownLatch(1); - - entryCache.asyncReadEntry(lh, 0, 9, false, new ReadEntriesCallback() { - public void readEntriesComplete(List entries, Object ctx) { - assertEquals(entries.size(), 10); - counter.countDown(); - } - - public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { - Assert.fail("should not have failed"); - } - }, null); - counter.await(); + when(ml.getLastConfirmedEntry()).thenReturn(PositionFactory.create(0, 9)); + final var entries = readEntry(entryCache, lh, 0, 9, false, null); + assertEquals(entries.size(), 10); } @Test(timeOut = 5000) @@ -198,19 +164,9 @@ public void testReadMissingMultiple() throws Exception { entryCache.insert(EntryImpl.create(0, 5, data)); entryCache.insert(EntryImpl.create(0, 8, data)); - final CountDownLatch counter = new CountDownLatch(1); - - entryCache.asyncReadEntry(lh, 0, 9, false, new ReadEntriesCallback() { - public void readEntriesComplete(List entries, Object ctx) { - assertEquals(entries.size(), 10); - counter.countDown(); - } - - public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { - Assert.fail("should not have failed"); - } - }, null); - counter.await(); + when(ml.getLastConfirmedEntry()).thenReturn(PositionFactory.create(0, 9)); + final var entries = readEntry(entryCache, lh, 0, 9, false, null); + assertEquals(entries.size(), 10); } @Test @@ -222,19 +178,25 @@ public void testCachedReadReturnsDifferentByteBuffer() throws Exception { @Cleanup(value = "clear") EntryCache entryCache = cacheManager.getEntryCache(ml); - CompletableFuture> cacheMissFutureEntries = new CompletableFuture<>(); - - entryCache.asyncReadEntry(lh, 0, 1, true, new ReadEntriesCallback() { - public void readEntriesComplete(List entries, Object ctx) { - cacheMissFutureEntries.complete(entries); - } - - public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { - cacheMissFutureEntries.completeExceptionally(exception); - } - }, null); - - List cacheMissEntries = cacheMissFutureEntries.get(); + readEntry(entryCache, lh, 0, 1, true, e -> { + assertTrue(e instanceof ManagedLedgerException); + assertTrue(e.getMessage().contains("LastConfirmedEntry is null when reading ledger 0")); + }); + + when(ml.getLastConfirmedEntry()).thenReturn(PositionFactory.create(-1, -1)); + readEntry(entryCache, lh, 0, 1, true, e -> { + assertTrue(e instanceof ManagedLedgerException); + assertTrue(e.getMessage().contains("LastConfirmedEntry is -1:-1 when reading ledger 0")); + }); + + when(ml.getLastConfirmedEntry()).thenReturn(PositionFactory.create(0, 0)); + readEntry(entryCache, lh, 0, 1, true, e -> { + assertTrue(e instanceof ManagedLedgerException); + assertTrue(e.getMessage().contains("LastConfirmedEntry is 0:0 when reading entry 1")); + }); + + when(ml.getLastConfirmedEntry()).thenReturn(PositionFactory.create(0, 1)); + List cacheMissEntries = readEntry(entryCache, lh, 0, 1, true, null); // Ensure first entry is 0 and assertEquals(cacheMissEntries.size(), 2); assertEquals(cacheMissEntries.get(0).getEntryId(), 0); @@ -243,19 +205,7 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { // Move the reader index to simulate consumption cacheMissEntries.get(0).getDataBuffer().readerIndex(10); - CompletableFuture> cacheHitFutureEntries = new CompletableFuture<>(); - - entryCache.asyncReadEntry(lh, 0, 1, true, new ReadEntriesCallback() { - public void readEntriesComplete(List entries, Object ctx) { - cacheHitFutureEntries.complete(entries); - } - - public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { - cacheHitFutureEntries.completeExceptionally(exception); - } - }, null); - - List cacheHitEntries = cacheHitFutureEntries.get(); + List cacheHitEntries = readEntry(entryCache, lh, 0, 1, true, null); assertEquals(cacheHitEntries.get(0).getEntryId(), 0); assertEquals(cacheHitEntries.get(0).getDataBuffer().readerIndex(), 0); } @@ -269,7 +219,7 @@ public void testReadWithError() throws Exception { CompletableFuture future = new CompletableFuture<>(); future.completeExceptionally(new BKNoSuchLedgerExistsException()); return future; - }).when(lh).readAsync(anyLong(), anyLong()); + }).when(lh).readUnconfirmedAsync(anyLong(), anyLong()); EntryCacheManager cacheManager = factory.getEntryCacheManager(); @Cleanup(value = "clear") @@ -278,18 +228,9 @@ public void testReadWithError() throws Exception { byte[] data = new byte[10]; entryCache.insert(EntryImpl.create(0, 2, data)); - final CountDownLatch counter = new CountDownLatch(1); - - entryCache.asyncReadEntry(lh, 0, 9, false, new ReadEntriesCallback() { - public void readEntriesComplete(List entries, Object ctx) { - Assert.fail("should not complete"); - } - - public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { - counter.countDown(); - } - }, null); - counter.await(); + when(ml.getLastConfirmedEntry()).thenReturn(PositionFactory.create(0, 9)); + readEntry(entryCache, lh, 0, 9, false, e -> + assertTrue(e instanceof ManagedLedgerException.LedgerNotExistException)); } static ReadHandle getLedgerHandle() { @@ -306,9 +247,35 @@ static ReadHandle getLedgerHandle() { LedgerEntries ledgerEntries = mock(LedgerEntries.class); doAnswer((invocation2) -> entries.iterator()).when(ledgerEntries).iterator(); return CompletableFuture.completedFuture(ledgerEntries); - }).when(lh).readAsync(anyLong(), anyLong()); + }).when(lh).readUnconfirmedAsync(anyLong(), anyLong()); return lh; } + private List readEntry(EntryCache entryCache, ReadHandle lh, long firstEntry, long lastEntry, + boolean shouldCacheEntry, Consumer assertion) + throws InterruptedException { + final var future = new CompletableFuture>(); + entryCache.asyncReadEntry(lh, firstEntry, lastEntry, shouldCacheEntry, new ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + future.complete(entries); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + future.completeExceptionally(exception); + } + }, null); + try { + final var entries = future.get(); + assertNull(assertion); + return entries; + } catch (ExecutionException e) { + if (assertion != null) { + assertion.accept(e.getCause()); + } + return List.of(); + } + } } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java index 7558f07db76ca..ebcbe31d5e784 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java @@ -51,7 +51,7 @@ public class ManagedCursorConcurrencyTest extends MockedBookKeeperTestCase { private static final Logger log = LoggerFactory.getLogger(ManagedCursorConcurrencyTest.class); - + @DataProvider(name = "useOpenRangeSet") public static Object[][] useOpenRangeSet() { return new Object[][] { { Boolean.TRUE }, { Boolean.FALSE } }; @@ -325,7 +325,7 @@ public void testConcurrentReadOfSameEntry() throws Exception { for (int i = 0; i < N; i++) { ledger.addEntry(("entry" + i).getBytes()); } - long currentLedger = ((PositionImpl) cursors.get(0).getMarkDeletedPosition()).getLedgerId(); + long currentLedger = cursors.get(0).getMarkDeletedPosition().getLedgerId(); // empty the cache ((ManagedLedgerImpl) ledger).entryCache.invalidateAllEntries(currentLedger); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java index 2c01b778caf6b..2afbcef0926e7 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java @@ -18,6 +18,7 @@ */ package org.apache.bookkeeper.mledger.impl; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; @@ -48,6 +49,8 @@ import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats; import org.testng.annotations.Test; public class ManagedCursorContainerTest { @@ -105,19 +108,19 @@ public boolean isDurable() { } @Override - public List readEntries(int numberOfEntriesToRead) throws ManagedLedgerException { + public List readEntries(int numberOfEntriesToRead) { return new ArrayList(); } @Override public void asyncReadEntries(int numberOfEntriesToRead, ReadEntriesCallback callback, Object ctx, - PositionImpl maxPosition) { + Position maxPosition) { callback.readEntriesComplete(null, ctx); } @Override public void asyncReadEntries(int numberOfEntriesToRead, long maxSizeBytes, ReadEntriesCallback callback, - Object ctx, PositionImpl maxPosition) { + Object ctx, Position maxPosition) { callback.readEntriesComplete(null, ctx); } @@ -137,14 +140,14 @@ public long getNumberOfEntriesInBacklog(boolean isPrecise) { } @Override - public void markDelete(Position position) throws ManagedLedgerException { + public void markDelete(Position position) { markDelete(position, Collections.emptyMap()); } @Override - public void markDelete(Position position, Map properties) throws ManagedLedgerException { + public void markDelete(Position position, Map properties) { this.position = position; - container.cursorUpdated(this, (PositionImpl) position); + container.cursorUpdated(this, position); } @Override @@ -209,7 +212,7 @@ public void asyncClose(AsyncCallbacks.CloseCallback callback, Object ctx) { } @Override - public void delete(Position position) throws InterruptedException, ManagedLedgerException { + public void delete(Position position) { } @Override @@ -217,7 +220,7 @@ public void asyncDelete(Position position, DeleteCallback callback, Object ctx) } @Override - public void delete(Iterable positions) throws InterruptedException, ManagedLedgerException { + public void delete(Iterable positions) { } @Override @@ -225,7 +228,7 @@ public void asyncDelete(Iterable position, DeleteCallback callback, Ob } @Override - public void clearBacklog() throws InterruptedException, ManagedLedgerException { + public void clearBacklog() { } @Override @@ -233,8 +236,7 @@ public void asyncClearBacklog(ClearBacklogCallback callback, Object ctx) { } @Override - public void skipEntries(int numEntriesToSkip, IndividualDeletedEntries deletedEntries) - throws InterruptedException, ManagedLedgerException { + public void skipEntries(int numEntriesToSkip, IndividualDeletedEntries deletedEntries) { } @Override @@ -243,13 +245,12 @@ public void asyncSkipEntries(int numEntriesToSkip, IndividualDeletedEntries dele } @Override - public Position findNewestMatching(Predicate condition) - throws InterruptedException, ManagedLedgerException { + public Position findNewestMatching(Predicate condition) { return null; } @Override - public Position findNewestMatching(FindPositionConstraint constraint, Predicate condition) throws InterruptedException, ManagedLedgerException { + public Position findNewestMatching(FindPositionConstraint constraint, Predicate condition) { return null; } @@ -258,6 +259,11 @@ public void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate AsyncCallbacks.FindEntryCallback callback, Object ctx) { } + @Override + public void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate condition, + AsyncCallbacks.FindEntryCallback callback, Object ctx, boolean isFindFromLedger) { + } + @Override public void asyncResetCursor(final Position position, boolean forceReset, AsyncCallbacks.ResetCursorCallback callback) { @@ -265,7 +271,7 @@ public void asyncResetCursor(final Position position, boolean forceReset, } @Override - public void resetCursor(final Position position) throws ManagedLedgerException, InterruptedException { + public void resetCursor(final Position position) { } @@ -279,8 +285,7 @@ public void setAlwaysInactive() { } @Override - public List replayEntries(Set positions) - throws InterruptedException, ManagedLedgerException { + public List replayEntries(Set positions) { return null; } @@ -295,19 +300,18 @@ public Set asyncReplayEntries(Set positi } @Override - public List readEntriesOrWait(int numberOfEntriesToRead) - throws InterruptedException, ManagedLedgerException { + public List readEntriesOrWait(int numberOfEntriesToRead) { return null; } @Override public void asyncReadEntriesOrWait(int numberOfEntriesToRead, ReadEntriesCallback callback, Object ctx, - PositionImpl maxPosition) { + Position maxPosition) { } @Override public void asyncReadEntriesOrWait(int maxEntries, long maxSizeBytes, ReadEntriesCallback callback, - Object ctx, PositionImpl maxPosition) { + Object ctx, Position maxPosition) { } @@ -317,8 +321,7 @@ public boolean cancelPendingReadRequest() { } @Override - public Entry getNthEntry(int N, IndividualDeletedEntries deletedEntries) - throws InterruptedException, ManagedLedgerException { + public Entry getNthEntry(int N, IndividualDeletedEntries deletedEntries) { return null; } @@ -375,7 +378,7 @@ public ManagedLedger getManagedLedger() { } @Override - public Range getLastIndividualDeletedRange() { + public Range getLastIndividualDeletedRange() { return null; } @@ -385,7 +388,7 @@ public void trimDeletedEntries(List entries) { } @Override - public long[] getDeletedBatchIndexesAsLongArray(PositionImpl position) { + public long[] getDeletedBatchIndexesAsLongArray(Position position) { return new long[0]; } @@ -394,13 +397,8 @@ public ManagedCursorMXBean getStats() { return null; } - public void asyncReadEntriesOrWait(int maxEntries, long maxSizeBytes, ReadEntriesCallback callback, - Object ctx) { - } - @Override - public List readEntriesOrWait(int maxEntries, long maxSizeBytes) - throws InterruptedException, ManagedLedgerException { + public List readEntriesOrWait(int maxEntries, long maxSizeBytes) { return null; } @@ -413,44 +411,74 @@ public boolean checkAndUpdateReadPositionChanged() { public boolean isClosed() { return false; } + + @Override + public ManagedLedgerInternalStats.CursorStats getCursorStats() { + return null; + } + + @Override + public boolean isMessageDeleted(Position position) { + return false; + } + + @Override + public ManagedCursor duplicateNonDurableCursor(String nonDurableCursorName) throws ManagedLedgerException { + return null; + } + + @Override + public long[] getBatchPositionAckSet(Position position) { + return new long[0]; + } + + @Override + public int applyMaxSizeCap(int maxEntries, long maxSizeBytes) { + return 0; + } + + @Override + public void updateReadStats(int readEntriesCount, long readEntriesSize) { + + } } @Test - public void testSlowestReadPositionForActiveCursors() throws Exception { + public void testSlowestReadPositionForActiveCursors() { ManagedCursorContainer container = new ManagedCursorContainer(); assertNull(container.getSlowestReaderPosition()); // Add no durable cursor - PositionImpl position = PositionImpl.get(5,5); + Position position = PositionFactory.create(5,5); ManagedCursor cursor1 = spy(new MockManagedCursor(container, "test1", position)); doReturn(false).when(cursor1).isDurable(); doReturn(position).when(cursor1).getReadPosition(); container.add(cursor1, position); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 5)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(5, 5)); // Add no durable cursor - position = PositionImpl.get(1,1); + position = PositionFactory.create(1,1); ManagedCursor cursor2 = spy(new MockManagedCursor(container, "test2", position)); doReturn(false).when(cursor2).isDurable(); doReturn(position).when(cursor2).getReadPosition(); container.add(cursor2, position); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(1, 1)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(1, 1)); // Move forward cursor, cursor1 = 5:5, cursor2 = 5:6, slowest is 5:5 - position = PositionImpl.get(5,6); + position = PositionFactory.create(5,6); container.cursorUpdated(cursor2, position); doReturn(position).when(cursor2).getReadPosition(); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 5)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(5, 5)); // Move forward cursor, cursor1 = 5:8, cursor2 = 5:6, slowest is 5:6 - position = PositionImpl.get(5,8); + position = PositionFactory.create(5,8); doReturn(position).when(cursor1).getReadPosition(); container.cursorUpdated(cursor1, position); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 6)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(5, 6)); // Remove cursor, only cursor1 left, cursor1 = 5:8 container.removeCursor(cursor2.getName()); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 8)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(5, 8)); } @Test @@ -458,41 +486,51 @@ public void simple() throws Exception { ManagedCursorContainer container = new ManagedCursorContainer(); assertNull(container.getSlowestReaderPosition()); - ManagedCursor cursor1 = new MockManagedCursor(container, "test1", new PositionImpl(5, 5)); + ManagedCursor cursor1 = new MockManagedCursor(container, "test1", PositionFactory.create(5, 5)); container.add(cursor1, cursor1.getMarkDeletedPosition()); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 5)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(5, 5)); + assertEqualsCursorAndPosition(container.getCursorWithOldestPosition(), + cursor1, PositionFactory.create(5, 5)); - ManagedCursor cursor2 = new MockManagedCursor(container, "test2", new PositionImpl(2, 2)); + ManagedCursor cursor2 = new MockManagedCursor(container, "test2", PositionFactory.create(2, 2)); container.add(cursor2, cursor2.getMarkDeletedPosition()); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(2, 2)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(2, 2)); + assertEqualsCursorAndPosition(container.getCursorWithOldestPosition(), + cursor2, PositionFactory.create(2, 2)); - ManagedCursor cursor3 = new MockManagedCursor(container, "test3", new PositionImpl(2, 0)); + ManagedCursor cursor3 = new MockManagedCursor(container, "test3", PositionFactory.create(2, 0)); container.add(cursor3, cursor3.getMarkDeletedPosition()); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(2, 0)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(2, 0)); + assertEqualsCursorAndPosition(container.getCursorWithOldestPosition(), + cursor3, PositionFactory.create(2, 0)); assertEquals(container.toString(), "[test1=5:5, test2=2:2, test3=2:0]"); - ManagedCursor cursor4 = new MockManagedCursor(container, "test4", new PositionImpl(4, 0)); + ManagedCursor cursor4 = new MockManagedCursor(container, "test4", PositionFactory.create(4, 0)); container.add(cursor4, cursor4.getMarkDeletedPosition()); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(2, 0)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(2, 0)); - ManagedCursor cursor5 = new MockManagedCursor(container, "test5", new PositionImpl(3, 5)); + ManagedCursor cursor5 = new MockManagedCursor(container, "test5", PositionFactory.create(3, 5)); container.add(cursor5, cursor5.getMarkDeletedPosition()); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(2, 0)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(2, 0)); - cursor3.markDelete(new PositionImpl(3, 0)); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(2, 2)); + cursor3.markDelete(PositionFactory.create(3, 0)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(2, 2)); + assertEqualsCursorAndPosition(container.getCursorWithOldestPosition(), + cursor2, PositionFactory.create(2, 2)); - cursor2.markDelete(new PositionImpl(10, 5)); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(3, 0)); + cursor2.markDelete(PositionFactory.create(10, 5)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(3, 0)); container.removeCursor(cursor3.getName()); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(3, 5)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(3, 5)); container.removeCursor(cursor2.getName()); container.removeCursor(cursor5.getName()); container.removeCursor(cursor1.getName()); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(4, 0)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(4, 0)); + assertEqualsCursorAndPosition(container.getCursorWithOldestPosition(), + cursor4, PositionFactory.create(4, 0)); assertTrue(container.hasDurableCursors()); @@ -501,52 +539,61 @@ public void simple() throws Exception { assertFalse(container.hasDurableCursors()); - ManagedCursor cursor6 = new MockManagedCursor(container, "test6", new PositionImpl(6, 5)); + ManagedCursor cursor6 = new MockManagedCursor(container, "test6", PositionFactory.create(6, 5)); container.add(cursor6, cursor6.getMarkDeletedPosition()); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(6, 5)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(6, 5)); assertEquals(container.toString(), "[test6=6:5]"); } @Test - public void updatingCursorOutsideContainer() throws Exception { + public void updatingCursorOutsideContainer() { ManagedCursorContainer container = new ManagedCursorContainer(); - ManagedCursor cursor1 = new MockManagedCursor(container, "test1", new PositionImpl(5, 5)); + ManagedCursor cursor1 = new MockManagedCursor(container, "test1", PositionFactory.create(5, 5)); container.add(cursor1, cursor1.getMarkDeletedPosition()); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 5)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(5, 5)); - MockManagedCursor cursor2 = new MockManagedCursor(container, "test2", new PositionImpl(2, 2)); + MockManagedCursor cursor2 = new MockManagedCursor(container, "test2", PositionFactory.create(2, 2)); container.add(cursor2, cursor2.getMarkDeletedPosition()); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(2, 2)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(2, 2)); - cursor2.position = new PositionImpl(8, 8); + cursor2.position = PositionFactory.create(8, 8); // Until we don't update the container, the ordering will not change - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(2, 2)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(2, 2)); container.cursorUpdated(cursor2, cursor2.position); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 5)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(5, 5)); + assertEqualsCursorAndPosition(container.getCursorWithOldestPosition(), + cursor1, PositionFactory.create(5, 5)); + } + + private void assertEqualsCursorAndPosition(ManagedCursorContainer.CursorInfo cursorInfo, + ManagedCursor expectedCursor, + Position expectedPosition) { + assertThat(cursorInfo.getCursor().getName()).isEqualTo(expectedCursor.getName()); + assertThat(cursorInfo.getPosition()).isEqualTo(expectedPosition); } @Test - public void removingCursor() throws Exception { + public void removingCursor() { ManagedCursorContainer container = new ManagedCursorContainer(); - ManagedCursor cursor1 = new MockManagedCursor(container, "test1", new PositionImpl(5, 5)); + ManagedCursor cursor1 = new MockManagedCursor(container, "test1", PositionFactory.create(5, 5)); container.add(cursor1, cursor1.getMarkDeletedPosition()); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 5)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(5, 5)); assertEquals(container.get("test1"), cursor1); - MockManagedCursor cursor2 = new MockManagedCursor(container, "test2", new PositionImpl(2, 2)); + MockManagedCursor cursor2 = new MockManagedCursor(container, "test2", PositionFactory.create(2, 2)); container.add(cursor2, cursor2.getMarkDeletedPosition()); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(2, 2)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(2, 2)); assertEquals(container.get("test2"), cursor2); - MockManagedCursor cursor3 = new MockManagedCursor(container, "test3", new PositionImpl(1, 1)); + MockManagedCursor cursor3 = new MockManagedCursor(container, "test3", PositionFactory.create(1, 1)); container.add(cursor3, cursor3.getMarkDeletedPosition()); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(1, 1)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(1, 1)); assertEquals(container.get("test3"), cursor3); assertEquals(container, Lists.newArrayList(cursor1, cursor2, cursor3)); @@ -558,24 +605,24 @@ public void removingCursor() throws Exception { assertNull(container.get("test2")); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(1, 1)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(1, 1)); container.removeCursor("test3"); assertEquals(container, Lists.newArrayList(cursor1)); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 5)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(5, 5)); } @Test public void ordering() throws Exception { ManagedCursorContainer container = new ManagedCursorContainer(); - ManagedCursor cursor1 = new MockManagedCursor(container, "test1", new PositionImpl(5, 5)); - ManagedCursor cursor2 = new MockManagedCursor(container, "test2", new PositionImpl(5, 1)); - ManagedCursor cursor3 = new MockManagedCursor(container, "test3", new PositionImpl(7, 1)); - ManagedCursor cursor4 = new MockManagedCursor(container, "test4", new PositionImpl(6, 4)); - ManagedCursor cursor5 = new MockManagedCursor(container, "test5", new PositionImpl(7, 0)); + ManagedCursor cursor1 = new MockManagedCursor(container, "test1", PositionFactory.create(5, 5)); + ManagedCursor cursor2 = new MockManagedCursor(container, "test2", PositionFactory.create(5, 1)); + ManagedCursor cursor3 = new MockManagedCursor(container, "test3", PositionFactory.create(7, 1)); + ManagedCursor cursor4 = new MockManagedCursor(container, "test4", PositionFactory.create(6, 4)); + ManagedCursor cursor5 = new MockManagedCursor(container, "test5", PositionFactory.create(7, 0)); container.add(cursor1, cursor1.getMarkDeletedPosition()); container.add(cursor2, cursor2.getMarkDeletedPosition()); @@ -583,33 +630,33 @@ public void ordering() throws Exception { container.add(cursor4, cursor4.getMarkDeletedPosition()); container.add(cursor5, cursor5.getMarkDeletedPosition()); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 1)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(5, 1)); container.removeCursor("test2"); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 5)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(5, 5)); container.removeCursor("test1"); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(6, 4)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(6, 4)); container.removeCursor("test4"); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(7, 0)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(7, 0)); container.removeCursor("test5"); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(7, 1)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(7, 1)); container.removeCursor("test3"); assertFalse(container.hasDurableCursors()); } @Test - public void orderingWithUpdates() throws Exception { + public void orderingWithUpdates() { ManagedCursorContainer container = new ManagedCursorContainer(); - MockManagedCursor c1 = new MockManagedCursor(container, "test1", new PositionImpl(5, 5)); - MockManagedCursor c2 = new MockManagedCursor(container, "test2", new PositionImpl(5, 1)); - MockManagedCursor c3 = new MockManagedCursor(container, "test3", new PositionImpl(7, 1)); - MockManagedCursor c4 = new MockManagedCursor(container, "test4", new PositionImpl(6, 4)); - MockManagedCursor c5 = new MockManagedCursor(container, "test5", new PositionImpl(7, 0)); + MockManagedCursor c1 = new MockManagedCursor(container, "test1", PositionFactory.create(5, 5)); + MockManagedCursor c2 = new MockManagedCursor(container, "test2", PositionFactory.create(5, 1)); + MockManagedCursor c3 = new MockManagedCursor(container, "test3", PositionFactory.create(7, 1)); + MockManagedCursor c4 = new MockManagedCursor(container, "test4", PositionFactory.create(6, 4)); + MockManagedCursor c5 = new MockManagedCursor(container, "test5", PositionFactory.create(7, 0)); container.add(c1, c1.getMarkDeletedPosition()); container.add(c2, c2.getMarkDeletedPosition()); @@ -617,64 +664,64 @@ public void orderingWithUpdates() throws Exception { container.add(c4, c4.getMarkDeletedPosition()); container.add(c5, c5.getMarkDeletedPosition()); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 1)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(5, 1)); - c1.position = new PositionImpl(5, 8); + c1.position = PositionFactory.create(5, 8); container.cursorUpdated(c1, c1.position); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 1)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(5, 1)); - c2.position = new PositionImpl(5, 6); + c2.position = PositionFactory.create(5, 6); container.cursorUpdated(c2, c2.position); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 6)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(5, 6)); - c1.position = new PositionImpl(6, 8); + c1.position = PositionFactory.create(6, 8); container.cursorUpdated(c1, c1.position); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 6)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(5, 6)); - c3.position = new PositionImpl(8, 5); + c3.position = PositionFactory.create(8, 5); container.cursorUpdated(c3, c3.position); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 6)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(5, 6)); - c1.position = new PositionImpl(8, 4); + c1.position = PositionFactory.create(8, 4); container.cursorUpdated(c1, c1.position); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 6)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(5, 6)); - c2.position = new PositionImpl(8, 4); + c2.position = PositionFactory.create(8, 4); container.cursorUpdated(c2, c2.position); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(6, 4)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(6, 4)); - c4.position = new PositionImpl(7, 1); + c4.position = PositionFactory.create(7, 1); container.cursorUpdated(c4, c4.position); // //// - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(7, 0)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(7, 0)); container.removeCursor("test5"); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(7, 1)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(7, 1)); container.removeCursor("test4"); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(8, 4)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(8, 4)); container.removeCursor("test1"); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(8, 4)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(8, 4)); container.removeCursor("test2"); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(8, 5)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(8, 5)); container.removeCursor("test3"); assertFalse(container.hasDurableCursors()); } @Test - public void orderingWithUpdatesAndReset() throws Exception { + public void orderingWithUpdatesAndReset() { ManagedCursorContainer container = new ManagedCursorContainer(); - MockManagedCursor c1 = new MockManagedCursor(container, "test1", new PositionImpl(5, 5)); - MockManagedCursor c2 = new MockManagedCursor(container, "test2", new PositionImpl(5, 1)); - MockManagedCursor c3 = new MockManagedCursor(container, "test3", new PositionImpl(7, 1)); - MockManagedCursor c4 = new MockManagedCursor(container, "test4", new PositionImpl(6, 4)); - MockManagedCursor c5 = new MockManagedCursor(container, "test5", new PositionImpl(7, 0)); + MockManagedCursor c1 = new MockManagedCursor(container, "test1", PositionFactory.create(5, 5)); + MockManagedCursor c2 = new MockManagedCursor(container, "test2", PositionFactory.create(5, 1)); + MockManagedCursor c3 = new MockManagedCursor(container, "test3", PositionFactory.create(7, 1)); + MockManagedCursor c4 = new MockManagedCursor(container, "test4", PositionFactory.create(6, 4)); + MockManagedCursor c5 = new MockManagedCursor(container, "test5", PositionFactory.create(7, 0)); container.add(c1, c1.getMarkDeletedPosition()); container.add(c2, c2.getMarkDeletedPosition()); @@ -682,52 +729,104 @@ public void orderingWithUpdatesAndReset() throws Exception { container.add(c4, c4.getMarkDeletedPosition()); container.add(c5, c5.getMarkDeletedPosition()); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 1)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(5, 1)); - c1.position = new PositionImpl(5, 8); + c1.position = PositionFactory.create(5, 8); container.cursorUpdated(c1, c1.position); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 1)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(5, 1)); - c1.position = new PositionImpl(5, 6); + c1.position = PositionFactory.create(5, 6); container.cursorUpdated(c1, c1.position); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 1)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(5, 1)); - c2.position = new PositionImpl(6, 8); + c2.position = PositionFactory.create(6, 8); container.cursorUpdated(c2, c2.position); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 6)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(5, 6)); - c3.position = new PositionImpl(8, 5); + c3.position = PositionFactory.create(8, 5); container.cursorUpdated(c3, c3.position); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 6)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(5, 6)); - c1.position = new PositionImpl(8, 4); + c1.position = PositionFactory.create(8, 4); container.cursorUpdated(c1, c1.position); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(6, 4)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(6, 4)); - c2.position = new PositionImpl(4, 4); + c2.position = PositionFactory.create(4, 4); container.cursorUpdated(c2, c2.position); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(4, 4)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(4, 4)); - c4.position = new PositionImpl(7, 1); + c4.position = PositionFactory.create(7, 1); container.cursorUpdated(c4, c4.position); // //// - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(4, 4)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(4, 4)); container.removeCursor("test2"); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(7, 0)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(7, 0)); container.removeCursor("test5"); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(7, 1)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(7, 1)); container.removeCursor("test1"); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(7, 1)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(7, 1)); container.removeCursor("test4"); - assertEquals(container.getSlowestReaderPosition(), new PositionImpl(8, 5)); + assertEquals(container.getSlowestReaderPosition(), PositionFactory.create(8, 5)); container.removeCursor("test3"); assertFalse(container.hasDurableCursors()); } + + @Test + public void testDataVersion() { + assertThat(ManagedCursorContainer.DataVersion.compareVersions(1L, 3L)).isNegative(); + assertThat(ManagedCursorContainer.DataVersion.compareVersions(3L, 1L)).isPositive(); + assertThat(ManagedCursorContainer.DataVersion.compareVersions(3L, 3L)).isZero(); + + long v1 = Long.MAX_VALUE - 1; + long v2 = ManagedCursorContainer.DataVersion.getNextVersion(v1); + + assertThat(ManagedCursorContainer.DataVersion.compareVersions(v1, v2)).isNegative(); + + v2 = ManagedCursorContainer.DataVersion.getNextVersion(v2); + assertThat(ManagedCursorContainer.DataVersion.compareVersions(v1, v2)).isNegative(); + + v1 = ManagedCursorContainer.DataVersion.getNextVersion(v1); + assertThat(ManagedCursorContainer.DataVersion.compareVersions(v1, v2)).isNegative(); + + v1 = ManagedCursorContainer.DataVersion.getNextVersion(v1); + assertThat(ManagedCursorContainer.DataVersion.compareVersions(v1, v2)).isZero(); + + v1 = ManagedCursorContainer.DataVersion.getNextVersion(v1); + assertThat(ManagedCursorContainer.DataVersion.compareVersions(v1, v2)).isPositive(); + } + + @Test + public void testVersions() { + ManagedCursorContainer container = new ManagedCursorContainer(); + + MockManagedCursor c1 = new MockManagedCursor(container, "test1", PositionFactory.create(5, 5)); + MockManagedCursor c2 = new MockManagedCursor(container, "test2", PositionFactory.create(5, 1)); + + container.add(c1, c1.getMarkDeletedPosition()); + long version = container.getCursorWithOldestPosition().getVersion(); + + container.add(c2, c2.getMarkDeletedPosition()); + long newVersion = container.getCursorWithOldestPosition().getVersion(); + // newVersion > version + assertThat(ManagedCursorContainer.DataVersion.compareVersions(newVersion, version)).isPositive(); + version = newVersion; + + container.cursorUpdated(c2, PositionFactory.create(5, 8)); + newVersion = container.getCursorWithOldestPosition().getVersion(); + // newVersion > version + assertThat(ManagedCursorContainer.DataVersion.compareVersions(newVersion, version)).isPositive(); + version = newVersion; + + container.removeCursor("test2"); + newVersion = container.getCursorWithOldestPosition().getVersion(); + // newVersion > version + assertThat(ManagedCursorContainer.DataVersion.compareVersions(newVersion, version)).isPositive(); + } } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorIndividualDeletedMessagesTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorIndividualDeletedMessagesTest.java index aa0d04783d991..3d4de5b1f4975 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorIndividualDeletedMessagesTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorIndividualDeletedMessagesTest.java @@ -24,15 +24,15 @@ import static org.testng.Assert.assertEquals; import com.google.common.collect.Range; - import java.lang.reflect.Method; import java.util.ArrayList; import java.util.List; import java.util.NavigableMap; import java.util.concurrent.ConcurrentSkipListMap; - import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo; import org.apache.bookkeeper.mledger.proto.MLDataFormats.MessageRange; import org.apache.bookkeeper.mledger.proto.MLDataFormats.NestedPositionInfo; @@ -56,9 +56,10 @@ void testRecoverIndividualDeletedMessages() throws Exception { ManagedLedgerImpl ledger = mock(ManagedLedgerImpl.class); doReturn(ledgersInfo).when(ledger).getLedgersInfo(); + doReturn(config).when(ledger).getConfig(); - ManagedCursorImpl cursor = spy(new ManagedCursorImpl(bookkeeper, config, ledger, "test-cursor")); - LongPairRangeSet deletedMessages = cursor.getIndividuallyDeletedMessagesSet(); + ManagedCursorImpl cursor = spy(new ManagedCursorImpl(bookkeeper, ledger, "test-cursor")); + LongPairRangeSet deletedMessages = cursor.getIndividuallyDeletedMessagesSet(); Method recoverMethod = ManagedCursorImpl.class.getDeclaredMethod("recoverIndividualDeletedMessages", List.class); @@ -67,7 +68,7 @@ void testRecoverIndividualDeletedMessages() throws Exception { // (1) [(1:5..1:10]] List messageRangeList = new ArrayList(); messageRangeList.add(createMessageRange(1, 5, 1, 10)); - List> expectedRangeList = new ArrayList(); + List> expectedRangeList = new ArrayList(); expectedRangeList.add(createPositionRange(1, 5, 1, 10)); recoverMethod.invoke(cursor, messageRangeList); assertEquals(deletedMessages.size(), 1); @@ -119,9 +120,9 @@ private static MessageRange createMessageRange(long lowerLedgerId, long lowerEnt return messageRangeBuilder.build(); } - private static Range createPositionRange(long lowerLedgerId, long lowerEntryId, long upperLedgerId, + private static Range createPositionRange(long lowerLedgerId, long lowerEntryId, long upperLedgerId, long upperEntryId) { - return Range.openClosed(new PositionImpl(lowerLedgerId, lowerEntryId), - new PositionImpl(upperLedgerId, upperEntryId)); + return Range.openClosed(PositionFactory.create(lowerLedgerId, lowerEntryId), + PositionFactory.create(upperLedgerId, upperEntryId)); } } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorInfoMetadataTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorInfoMetadataTest.java index 08d8fd939a01d..70ba4b543ec09 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorInfoMetadataTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorInfoMetadataTest.java @@ -19,11 +19,13 @@ package org.apache.bookkeeper.mledger.impl; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; import static org.testng.Assert.expectThrows; import java.io.IOException; import java.util.ArrayList; import java.util.List; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.MetadataCompressionConfig; import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.pulsar.common.api.proto.CompressionType; import org.testng.annotations.DataProvider; @@ -49,16 +51,14 @@ private Object[][] compressionTypeProvider() { }; } - @Test(dataProvider = "compressionTypeProvider") - public void testEncodeAndDecode(String compressionType) throws IOException { - long ledgerId = 10000; + private MLDataFormats.ManagedCursorInfo.Builder generateManagedCursorInfo(long ledgerId, int positionNumber) { MLDataFormats.ManagedCursorInfo.Builder builder = MLDataFormats.ManagedCursorInfo.newBuilder(); builder.setCursorsLedgerId(ledgerId); builder.setMarkDeleteLedgerId(ledgerId); List batchedEntryDeletionIndexInfos = new ArrayList<>(); - for (int i = 0; i < 1000; i++) { + for (int i = 0; i < positionNumber; i++) { MLDataFormats.NestedPositionInfo nestedPositionInfo = MLDataFormats.NestedPositionInfo.newBuilder() .setEntryId(i).setLedgerId(i).build(); MLDataFormats.BatchedEntryDeletionIndexInfo batchedEntryDeletionIndexInfo = MLDataFormats @@ -67,17 +67,24 @@ public void testEncodeAndDecode(String compressionType) throws IOException { } builder.addAllBatchedEntryDeletionIndexInfo(batchedEntryDeletionIndexInfos); + return builder; + } + + @Test(dataProvider = "compressionTypeProvider") + public void testEncodeAndDecode(String compressionType) throws IOException { + long ledgerId = 10000; + MLDataFormats.ManagedCursorInfo.Builder builder = generateManagedCursorInfo(ledgerId, 1000); MetaStoreImpl metaStore; if (INVALID_TYPE.equals(compressionType)) { IllegalArgumentException compressionTypeEx = expectThrows(IllegalArgumentException.class, () -> { - new MetaStoreImpl(null, null, null, compressionType); + new MetaStoreImpl(null, null, null, new MetadataCompressionConfig(compressionType)); }); assertEquals(compressionTypeEx.getMessage(), "No enum constant org.apache.bookkeeper.mledger.proto.MLDataFormats.CompressionType." + compressionType); return; } else { - metaStore = new MetaStoreImpl(null, null, null, compressionType); + metaStore = new MetaStoreImpl(null, null, null, new MetadataCompressionConfig(compressionType)); } MLDataFormats.ManagedCursorInfo managedCursorInfo = builder.build(); @@ -93,4 +100,42 @@ public void testEncodeAndDecode(String compressionType) throws IOException { MLDataFormats.ManagedCursorInfo info2 = metaStore.parseManagedCursorInfo(managedCursorInfo.toByteArray()); assertEquals(info1, info2); } + + @Test(dataProvider = "compressionTypeProvider") + public void testCompressionThreshold(String compressionType) throws IOException { + int compressThreshold = 512; + + long ledgerId = 10000; + // should not compress + MLDataFormats.ManagedCursorInfo smallInfo = generateManagedCursorInfo(ledgerId, 1).build(); + assertTrue(smallInfo.getSerializedSize() < compressThreshold); + + // should compress + MLDataFormats.ManagedCursorInfo bigInfo = generateManagedCursorInfo(ledgerId, 1000).build(); + assertTrue(bigInfo.getSerializedSize() > compressThreshold); + + MetaStoreImpl metaStore; + if (INVALID_TYPE.equals(compressionType)) { + IllegalArgumentException compressionTypeEx = expectThrows(IllegalArgumentException.class, () -> { + new MetaStoreImpl(null, null, null, + new MetadataCompressionConfig(compressionType, compressThreshold)); + }); + assertEquals(compressionTypeEx.getMessage(), + "No enum constant org.apache.bookkeeper.mledger.proto.MLDataFormats.CompressionType." + + compressionType); + return; + } else { + metaStore = new MetaStoreImpl(null, null, null, + new MetadataCompressionConfig(compressionType, compressThreshold)); + } + + byte[] compressionBytes = metaStore.compressCursorInfo(smallInfo); + // not compressed + assertEquals(compressionBytes.length, smallInfo.getSerializedSize()); + + + byte[] compressionBigBytes = metaStore.compressCursorInfo(bigInfo); + // compressed + assertTrue(compressionBigBytes.length != smallInfo.getSerializedSize()); + } } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorPropertiesTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorPropertiesTest.java index 500de5dd13879..990c298604e59 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorPropertiesTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorPropertiesTest.java @@ -18,7 +18,7 @@ */ package org.apache.bookkeeper.mledger.impl; -import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX; +import static org.apache.bookkeeper.mledger.ManagedCursor.CURSOR_INTERNAL_PROPERTY_PREFIX; import static org.apache.bookkeeper.mledger.util.Futures.executeWithRetry; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index 1b1b5534256f9..8ae5a04a507b1 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -43,6 +43,7 @@ import java.util.Arrays; import java.util.BitSet; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -65,13 +66,18 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; import lombok.Cleanup; +import org.apache.bookkeeper.client.AsyncCallback.OpenCallback; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.BookKeeper.DigestType; import org.apache.bookkeeper.client.LedgerEntry; +import org.apache.bookkeeper.client.api.ReadHandle; +import org.apache.bookkeeper.client.PulsarMockBookKeeper; +import org.apache.bookkeeper.common.util.OrderedExecutor; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCallback; @@ -87,6 +93,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerFactory; import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.ScanOutcome; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.VoidCallback; import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback; @@ -94,6 +101,7 @@ import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo; import org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo; import org.apache.bookkeeper.test.MockedBookKeeperTestCase; +import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.common.api.proto.IntRange; import org.apache.pulsar.common.util.FutureUtil; @@ -138,11 +146,11 @@ public void testCloseCursor() throws Exception { ledger.addEntry(new byte[]{4}); ledger.addEntry(new byte[]{5}); // Persistent cursor info to ledger. - c1.delete(PositionImpl.get(c1.getReadPosition().getLedgerId(), c1.getReadPosition().getEntryId())); + c1.delete(PositionFactory.create(c1.getReadPosition().getLedgerId(), c1.getReadPosition().getEntryId())); Awaitility.await().until(() ->c1.getStats().getPersistLedgerSucceed() > 0); // Make cursor ledger can not work. closeCursorLedger(c1); - c1.delete(PositionImpl.get(c1.getReadPosition().getLedgerId(), c1.getReadPosition().getEntryId() + 2)); + c1.delete(PositionFactory.create(c1.getReadPosition().getLedgerId(), c1.getReadPosition().getEntryId() + 2)); ledger.close(); } @@ -229,6 +237,140 @@ void readTwice() throws Exception { entries.forEach(Entry::release); } + @Test + void testPersistentMarkDeleteIfCreateCursorLedgerFailed() throws Exception { + final int entryCount = 9; + final String cursorName = "c1"; + final String mlName = "ml_test"; + // Avoid creating new empty ledger after the last ledger is full and remove fail future. + final ManagedLedgerConfig mlConfig = new ManagedLedgerConfig().setMaxEntriesPerLedger(2); + ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open(mlName, mlConfig); + + ManagedCursor cursor = ml.openCursor("c1"); + Position lastEntry = null; + for (int i = 0; i < entryCount; i++) { + lastEntry = ml.addEntry(("entry-" + i).getBytes(Encoding)); + } + + // Mock cursor ledger create failed. + bkc.failNow(BKException.Code.NoBookieAvailableException); + + cursor.markDelete(lastEntry); + + // Assert persist mark deleted position to ZK was successful. + Position slowestReadPosition = ml.getCursors().getSlowestReaderPosition(); + assertTrue(slowestReadPosition.getLedgerId() >= lastEntry.getLedgerId()); + assertTrue(slowestReadPosition.getEntryId() >= lastEntry.getEntryId()); + assertEquals(cursor.getStats().getPersistLedgerSucceed(), 0); + assertTrue(cursor.getStats().getPersistZookeeperSucceed() > 0); + assertEquals(cursor.getPersistentMarkDeletedPosition(), lastEntry); + + // Verify the mark delete position can be recovered properly. + ml.close(); + ml = (ManagedLedgerImpl) factory.open(mlName, mlConfig); + ManagedCursorImpl cursorRecovered = (ManagedCursorImpl) ml.openCursor(cursorName); + assertEquals(cursorRecovered.getPersistentMarkDeletedPosition(), lastEntry); + + // cleanup. + ml.delete(); + } + + @Test + void testSwitchLedgerFailed() throws Exception { + final String cursorName = "c1"; + final String mlName = UUID.randomUUID().toString().replaceAll("-", ""); + final ManagedLedgerConfig mlConfig = new ManagedLedgerConfig(); + mlConfig.setMaxEntriesPerLedger(1); + mlConfig.setMetadataMaxEntriesPerLedger(1); + mlConfig.setThrottleMarkDelete(Double.MAX_VALUE); + ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open(mlName, mlConfig); + ManagedCursor cursor = ml.openCursor(cursorName); + + List positionList = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + positionList.add(ml.addEntry(("entry-" + i).getBytes(Encoding))); + } + + // Inject an error when persistent at the third time. + AtomicInteger persistentCounter = new AtomicInteger(); + metadataStore.failConditional(new MetadataStoreException.BadVersionException("mock error"), (op, path) -> { + if (path.equals(String.format("/managed-ledgers/%s/%s", mlName, cursorName)) + && persistentCounter.incrementAndGet() == 3) { + log.info("Trigger an error"); + return true; + } + return false; + }); + + // Verify: the cursor can be recovered after it fails once. + int failedCount = 0; + for (Position position : positionList) { + try { + cursor.markDelete(position); + } catch (Exception ex) { + failedCount++; + } + } + assertEquals(failedCount, 1); + + // cleanup. + ml.delete(); + } + + @Test + void testPersistentMarkDeleteIfSwitchCursorLedgerFailed() throws Exception { + final int entryCount = 10; + final String cursorName = "c1"; + final String mlName = "ml_test"; + final ManagedLedgerConfig mlConfig = new ManagedLedgerConfig().setMaxEntriesPerLedger(1); + ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open(mlName, mlConfig); + + final ManagedCursorImpl cursor = (ManagedCursorImpl) ml.openCursor(cursorName); + ArrayList positions = new ArrayList<>(); + for (int i = 0; i < entryCount; i++) { + positions.add(ml.addEntry(("entry-" + i).getBytes(Encoding))); + } + // Trigger the cursor ledger creating. + cursor.markDelete(positions.get(0)); + assertTrue(cursor.getStats().getPersistLedgerSucceed() > 0); + + // Mock cursor ledger write failed. + bkc.addEntryFailAfter(0, BKException.Code.NoBookieAvailableException); + // Trigger a failed writing of the cursor ledger, then wait the stat of cursor to be "NoLedger". + // This time ZK will be written due to a failure to write BK. + cursor.markDelete(positions.get(1)); + Awaitility.await().untilAsserted(() -> { + assertEquals(cursor.getState(), "NoLedger"); + }); + assertTrue(cursor.getStats().getPersistLedgerErrors() > 0); + long persistZookeeperSucceed1 = cursor.getStats().getPersistZookeeperSucceed(); + assertTrue(persistZookeeperSucceed1 > 0); + + // Mock cursor ledger create failed. + bkc.failNow(BKException.Code.NoBookieAvailableException); + // Verify the cursor status will be persistent to ZK even if the cursor ledger creation always fails. + // This time ZK will be written due to catch up. + Position lastEntry = positions.get(entryCount -1); + cursor.markDelete(lastEntry); + long persistZookeeperSucceed2 = cursor.getStats().getPersistZookeeperSucceed(); + assertTrue(persistZookeeperSucceed2 > persistZookeeperSucceed1); + + // Assert persist mark deleted position to ZK was successful. + Position slowestReadPosition = ml.getCursors().getSlowestReaderPosition(); + assertTrue(slowestReadPosition.getLedgerId() >= lastEntry.getLedgerId()); + assertTrue(slowestReadPosition.getEntryId() >= lastEntry.getEntryId()); + assertEquals(cursor.getPersistentMarkDeletedPosition(), lastEntry); + + // Verify the mark delete position can be recovered properly. + ml.close(); + ml = (ManagedLedgerImpl) factory.open(mlName, mlConfig); + ManagedCursorImpl cursorRecovered = (ManagedCursorImpl) ml.openCursor(cursorName); + assertEquals(cursorRecovered.getPersistentMarkDeletedPosition(), lastEntry); + + // cleanup. + ml.delete(); + } + @Test(timeOut = 20000) void readWithCacheDisabled() throws Exception { ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig(); @@ -456,7 +598,7 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { fail(exception.getMessage()); } - }, null, PositionImpl.LATEST); + }, null, PositionFactory.LATEST); counter.await(); } @@ -484,7 +626,7 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { fail("async-call should not have failed"); } - }, null, PositionImpl.LATEST); + }, null, PositionFactory.LATEST); counter.await(); @@ -506,7 +648,7 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { counter2.countDown(); } - }, null, PositionImpl.LATEST); + }, null, PositionFactory.LATEST); counter2.await(); } @@ -533,7 +675,7 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { counter.countDown(); } - }, null, PositionImpl.LATEST); + }, null, PositionFactory.LATEST); counter.await(); } @@ -656,9 +798,9 @@ void testResetCursor() throws Exception { ledger.addEntry("dummy-entry-1".getBytes(Encoding)); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); ledger.addEntry("dummy-entry-3".getBytes(Encoding)); - PositionImpl lastPosition = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding)); + Position lastPosition = ledger.addEntry("dummy-entry-4".getBytes(Encoding)); final AtomicBoolean moveStatus = new AtomicBoolean(false); - PositionImpl resetPosition = new PositionImpl(lastPosition.getLedgerId(), lastPosition.getEntryId() - 2); + Position resetPosition = PositionFactory.create(lastPosition.getLedgerId(), lastPosition.getEntryId() - 2); try { cursor.resetCursor(resetPosition); moveStatus.set(true); @@ -675,23 +817,23 @@ void testResetCursor() throws Exception { @Test(timeOut = 20000) void testResetCursor1() throws Exception { ManagedLedger ledger = factory.open("my_test_move_cursor_ledger", - new ManagedLedgerConfig().setMaxEntriesPerLedger(2)); + new ManagedLedgerConfig().setMaxEntriesPerLedger(2)); ManagedCursor cursor = ledger.openCursor("trc1"); - PositionImpl actualEarliest = (PositionImpl) ledger.addEntry("dummy-entry-1".getBytes(Encoding)); + Position actualEarliest = ledger.addEntry("dummy-entry-1".getBytes(Encoding)); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); ledger.addEntry("dummy-entry-3".getBytes(Encoding)); - PositionImpl lastInPrev = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding)); - PositionImpl firstInNext = (PositionImpl) ledger.addEntry("dummy-entry-5".getBytes(Encoding)); + Position lastInPrev = ledger.addEntry("dummy-entry-4".getBytes(Encoding)); + Position firstInNext = ledger.addEntry("dummy-entry-5".getBytes(Encoding)); ledger.addEntry("dummy-entry-6".getBytes(Encoding)); ledger.addEntry("dummy-entry-7".getBytes(Encoding)); ledger.addEntry("dummy-entry-8".getBytes(Encoding)); ledger.addEntry("dummy-entry-9".getBytes(Encoding)); - PositionImpl last = (PositionImpl) ledger.addEntry("dummy-entry-10".getBytes(Encoding)); + Position last = ledger.addEntry("dummy-entry-10".getBytes(Encoding)); final AtomicBoolean moveStatus = new AtomicBoolean(false); // reset to earliest - PositionImpl earliest = PositionImpl.EARLIEST; + Position earliest = PositionFactory.EARLIEST; try { cursor.resetCursor(earliest); moveStatus.set(true); @@ -699,12 +841,12 @@ void testResetCursor1() throws Exception { log.warn("error in reset cursor", e.getCause()); } assertTrue(moveStatus.get()); - PositionImpl earliestPos = new PositionImpl(actualEarliest.getLedgerId(), -1); + Position earliestPos = PositionFactory.create(actualEarliest.getLedgerId(), -1); assertEquals(cursor.getReadPosition(), earliestPos); moveStatus.set(false); // reset to one after last entry in a ledger should point to the first entry in the next ledger - PositionImpl resetPosition = new PositionImpl(lastInPrev.getLedgerId(), lastInPrev.getEntryId() + 1); + Position resetPosition = PositionFactory.create(lastInPrev.getLedgerId(), lastInPrev.getEntryId() + 1); try { cursor.resetCursor(resetPosition); moveStatus.set(true); @@ -715,8 +857,8 @@ void testResetCursor1() throws Exception { assertEquals(firstInNext, cursor.getReadPosition()); moveStatus.set(false); - // reset to a non exist larger ledger should point to the first non-exist entry in the last ledger - PositionImpl latest = new PositionImpl(last.getLedgerId() + 2, 0); + // reset to a non exist larger ledger should point to the first non-exist entry in the next ledger + Position latest = PositionFactory.create(last.getLedgerId() + 2, 0); try { cursor.resetCursor(latest); moveStatus.set(true); @@ -724,12 +866,14 @@ void testResetCursor1() throws Exception { log.warn("error in reset cursor", e.getCause()); } assertTrue(moveStatus.get()); - PositionImpl lastPos = new PositionImpl(last.getLedgerId(), last.getEntryId() + 1); - assertEquals(lastPos, cursor.getReadPosition()); + Position lastPos = PositionFactory.create(last.getLedgerId() + 1, 0); + Awaitility.await().untilAsserted(() -> { + assertEquals(lastPos, cursor.getReadPosition()); + }); moveStatus.set(false); - // reset to latest should point to the first non-exist entry in the last ledger - PositionImpl anotherLast = PositionImpl.LATEST; + // reset to latest should point to the first non-exist entry in the next ledger + Position anotherLast = PositionFactory.LATEST; try { cursor.resetCursor(anotherLast); moveStatus.set(true); @@ -751,10 +895,10 @@ void testasyncResetCursor() throws Exception { ledger.addEntry("dummy-entry-1".getBytes(Encoding)); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); ledger.addEntry("dummy-entry-3".getBytes(Encoding)); - PositionImpl lastPosition = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding)); + Position lastPosition = ledger.addEntry("dummy-entry-4".getBytes(Encoding)); final AtomicBoolean moveStatus = new AtomicBoolean(false); CountDownLatch countDownLatch = new CountDownLatch(1); - PositionImpl resetPosition = new PositionImpl(lastPosition.getLedgerId(), lastPosition.getEntryId() - 2); + Position resetPosition = PositionFactory.create(lastPosition.getLedgerId(), lastPosition.getEntryId() - 2); cursor.asyncResetCursor(resetPosition, false, new AsyncCallbacks.ResetCursorCallback() { @Override @@ -791,7 +935,7 @@ void testConcurrentResetCursor() throws Exception { for (int i = 0; i < Messages; i++) { ledger.addEntry("test".getBytes()); } - final PositionImpl lastPosition = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding)); + final Position lastPosition = ledger.addEntry("dummy-entry-4".getBytes(Encoding)); for (int i = 0; i < Consumers; i++) { final ManagedCursor cursor = ledger.openCursor("tcrc" + i); @@ -804,14 +948,14 @@ public AtomicBoolean call() throws Exception { final AtomicBoolean moveStatus = new AtomicBoolean(false); CountDownLatch countDownLatch = new CountDownLatch(1); - final PositionImpl resetPosition = new PositionImpl(lastPosition.getLedgerId(), + final Position resetPosition = PositionFactory.create(lastPosition.getLedgerId(), lastPosition.getEntryId() - (5 * idx)); cursor.asyncResetCursor(resetPosition, false, new AsyncCallbacks.ResetCursorCallback() { @Override public void resetComplete(Object ctx) { moveStatus.set(true); - PositionImpl pos = (PositionImpl) ctx; + Position pos = (Position) ctx; log.info("move to [{}] completed for consumer [{}]", pos.toString(), idx); countDownLatch.countDown(); } @@ -819,7 +963,7 @@ public void resetComplete(Object ctx) { @Override public void resetFailed(ManagedLedgerException exception, Object ctx) { moveStatus.set(false); - PositionImpl pos = (PositionImpl) ctx; + Position pos = (Position) ctx; log.warn("move to [{}] failed for consumer [{}]", pos.toString(), idx); countDownLatch.countDown(); } @@ -846,9 +990,9 @@ void testLastActiveAfterResetCursor() throws Exception { ManagedLedger ledger = factory.open("test_cursor_ledger"); ManagedCursor cursor = ledger.openCursor("tla"); - PositionImpl lastPosition = null; + Position lastPosition = null; for (int i = 0; i < 3; i++) { - lastPosition = (PositionImpl) ledger.addEntry("dummy-entry".getBytes(Encoding)); + lastPosition = ledger.addEntry("dummy-entry".getBytes(Encoding)); } final AtomicBoolean moveStatus = new AtomicBoolean(false); @@ -889,9 +1033,9 @@ void seekPosition() throws Exception { ledger.addEntry("dummy-entry-1".getBytes(Encoding)); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); ledger.addEntry("dummy-entry-3".getBytes(Encoding)); - PositionImpl lastPosition = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding)); + Position lastPosition = ledger.addEntry("dummy-entry-4".getBytes(Encoding)); - cursor.seek(new PositionImpl(lastPosition.getLedgerId(), lastPosition.getEntryId() - 1)); + cursor.seek(PositionFactory.create(lastPosition.getLedgerId(), lastPosition.getEntryId() - 1)); } @Test(timeOut = 20000) @@ -900,12 +1044,12 @@ void seekPosition2() throws Exception { ManagedCursor cursor = ledger.openCursor("c1"); ledger.addEntry("dummy-entry-1".getBytes(Encoding)); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); - PositionImpl seekPosition = (PositionImpl) ledger.addEntry("dummy-entry-3".getBytes(Encoding)); + Position seekPosition = ledger.addEntry("dummy-entry-3".getBytes(Encoding)); ledger.addEntry("dummy-entry-4".getBytes(Encoding)); ledger.addEntry("dummy-entry-5".getBytes(Encoding)); ledger.addEntry("dummy-entry-6".getBytes(Encoding)); - cursor.seek(new PositionImpl(seekPosition.getLedgerId(), seekPosition.getEntryId())); + cursor.seek(PositionFactory.create(seekPosition.getLedgerId(), seekPosition.getEntryId())); } @Test(timeOut = 20000) @@ -915,11 +1059,11 @@ void seekPosition3() throws Exception { ledger.addEntry("dummy-entry-1".getBytes(Encoding)); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); ledger.addEntry("dummy-entry-3".getBytes(Encoding)); - PositionImpl seekPosition = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding)); + Position seekPosition = ledger.addEntry("dummy-entry-4".getBytes(Encoding)); Position entry5 = ledger.addEntry("dummy-entry-5".getBytes(Encoding)); Position entry6 = ledger.addEntry("dummy-entry-6".getBytes(Encoding)); - cursor.seek(new PositionImpl(seekPosition.getLedgerId(), seekPosition.getEntryId())); + cursor.seek(PositionFactory.create(seekPosition.getLedgerId(), seekPosition.getEntryId())); assertEquals(cursor.getReadPosition(), seekPosition); List entries = cursor.readEntries(1); @@ -1016,7 +1160,7 @@ void markDeleteSkippingMessage() throws Exception { Position p1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding)); Position p2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding)); ledger.addEntry("dummy-entry-3".getBytes(Encoding)); - PositionImpl p4 = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding)); + Position p4 = ledger.addEntry("dummy-entry-4".getBytes(Encoding)); assertEquals(cursor.getNumberOfEntries(), 4); @@ -1035,7 +1179,7 @@ void markDeleteSkippingMessage() throws Exception { assertFalse(cursor.hasMoreEntries()); assertEquals(cursor.getNumberOfEntries(), 0); - assertEquals(cursor.getReadPosition(), new PositionImpl(p4.getLedgerId(), p4.getEntryId() + 1)); + assertEquals(cursor.getReadPosition(), PositionFactory.create(p4.getLedgerId(), p4.getEntryId() + 1)); } @Test(timeOut = 20000) @@ -1421,6 +1565,17 @@ void errorRecoveringCursor2() throws Exception { ledger = factory2.open("my_test_ledger"); ManagedCursor cursor = ledger.openCursor("c1"); Position position = ledger.addEntry("test".getBytes()); + // Make persist zk fail once. + AtomicInteger persistZKTimes = new AtomicInteger(); + metadataStore.failConditional(new MetadataStoreException.BadVersionException("mock ex"), (type, path) -> { + if (FaultInjectionMetadataStore.OperationType.PUT.equals(type) + && path.equals("/managed-ledgers/my_test_ledger/c1")) { + if (persistZKTimes.incrementAndGet() == 1) { + return true; + } + } + return false; + }); try { cursor.markDelete(position); fail("should have failed"); @@ -1596,7 +1751,7 @@ void testMarkDeleteTwice(boolean useOpenRangeSet) throws Exception { @Test(timeOut = 20000, dataProvider = "useOpenRangeSet") void testSkipEntries(boolean useOpenRangeSet) throws Exception { - ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig() + ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", new ManagedLedgerConfig() .setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet).setMaxEntriesPerLedger(2)); Position pos; @@ -1610,6 +1765,11 @@ void testSkipEntries(boolean useOpenRangeSet) throws Exception { pos = ledger.addEntry("dummy-entry-1".getBytes(Encoding)); pos = ledger.addEntry("dummy-entry-2".getBytes(Encoding)); + // Wait new empty ledger created completely. + Awaitility.await().untilAsserted(() -> { + assertEquals(ledger.ledgers.size(), 2); + }); + // skip entries in same ledger c1.skipEntries(1, IndividualDeletedEntries.Exclude); assertEquals(c1.getNumberOfEntries(), 1); @@ -1617,7 +1777,7 @@ void testSkipEntries(boolean useOpenRangeSet) throws Exception { // skip entries until end of ledger c1.skipEntries(1, IndividualDeletedEntries.Exclude); assertEquals(c1.getNumberOfEntries(), 0); - assertEquals(c1.getReadPosition(), pos.getNext()); + assertEquals(c1.getReadPosition(), PositionFactory.create(ledger.currentLedger.getId(), 0)); assertEquals(c1.getMarkDeletedPosition(), pos); // skip entries across ledgers @@ -1632,7 +1792,10 @@ void testSkipEntries(boolean useOpenRangeSet) throws Exception { c1.skipEntries(10, IndividualDeletedEntries.Exclude); assertEquals(c1.getNumberOfEntries(), 0); assertFalse(c1.hasMoreEntries()); - assertEquals(c1.getReadPosition(), pos.getNext()); + // We can not check the ledger id because a cursor leger can be created. + Awaitility.await().untilAsserted(() -> { + assertEquals(c1.getReadPosition().getEntryId(), 0); + }); assertEquals(c1.getMarkDeletedPosition(), pos); } @@ -1654,7 +1817,7 @@ void testSkipEntriesWithIndividualDeletedMessages(boolean useOpenRangeSet) throw c1.skipEntries(3, IndividualDeletedEntries.Exclude); assertEquals(c1.getNumberOfEntries(), 0); - assertEquals(c1.getReadPosition(), pos5.getNext()); + assertEquals(c1.getReadPosition(), PositionFactory.create(pos5.getLedgerId() + 1, 0)); assertEquals(c1.getMarkDeletedPosition(), pos5); pos1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding)); @@ -1844,7 +2007,7 @@ public void readEntriesComplete(List entries, Object ctx) { public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { log.error("Error reading", exception); } - }, null, PositionImpl.LATEST); + }, null, PositionFactory.LATEST); } ledger.addEntry("test".getBytes()); @@ -2184,7 +2347,7 @@ void testFindNewestMatchingEdgeCase1() throws Exception { ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1"); assertNull(c1.findNewestMatching( - entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding)))); + entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding)))); } @Test(timeOut = 20000) @@ -2493,7 +2656,7 @@ public void findEntryComplete(Position position, Object ctx) { @Override public void findEntryFailed(ManagedLedgerException exception, Optional failedReadPosition, - Object ctx) { + Object ctx) { result.exception = exception; counter.countDown(); } @@ -2519,7 +2682,7 @@ public void findEntryFailed(ManagedLedgerException exception, Optional } void internalTestFindNewestMatchingAllEntries(final String name, final int entriesPerLedger, - final int expectedEntryId) throws Exception { + final int expectedEntryId) throws Exception { final String ledgerAndCursorName = name; ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setRetentionSizeInMB(10); @@ -2537,7 +2700,7 @@ void internalTestFindNewestMatchingAllEntries(final String name, final int entri Thread.sleep(100); Position newPosition = ledger.addEntry(getEntryPublishTime("expectedresetposition")); long timestamp = System.currentTimeMillis(); - long ledgerId = ((PositionImpl) newPosition).getLedgerId(); + long ledgerId = newPosition.getLedgerId(); Thread.sleep(2); ledger.addEntry(getEntryPublishTime("not-read")); @@ -2552,11 +2715,11 @@ void internalTestFindNewestMatchingAllEntries(final String name, final int entri ledger = factory.open(ledgerAndCursorName, config); c1 = (ManagedCursorImpl) ledger.openCursor(ledgerAndCursorName); - PositionImpl found = (PositionImpl) findPositionFromAllEntries(c1, timestamp); + Position found = findPositionFromAllEntries(c1, timestamp); assertEquals(found.getLedgerId(), ledgerId); assertEquals(found.getEntryId(), expectedEntryId); - found = (PositionImpl) findPositionFromAllEntries(c1, 0); + found = findPositionFromAllEntries(c1, 0); assertNull(found); } @@ -2595,13 +2758,13 @@ void testReplayEntries() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger"); ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1"); - PositionImpl p1 = (PositionImpl) ledger.addEntry("entry1".getBytes(Encoding)); - PositionImpl p2 = (PositionImpl) ledger.addEntry("entry2".getBytes(Encoding)); - PositionImpl p3 = (PositionImpl) ledger.addEntry("entry3".getBytes(Encoding)); + Position p1 = ledger.addEntry("entry1".getBytes(Encoding)); + Position p2 = ledger.addEntry("entry2".getBytes(Encoding)); + Position p3 = ledger.addEntry("entry3".getBytes(Encoding)); ledger.addEntry("entry4".getBytes(Encoding)); // 1. Replay empty position set should return empty entry set - Set positions = new HashSet(); + Set positions = new HashSet(); assertTrue(c1.replayEntries(positions).isEmpty()); positions.add(p1); @@ -2613,11 +2776,11 @@ void testReplayEntries() throws Exception { assertTrue((Arrays.equals(entries.get(0).getData(), "entry1".getBytes(Encoding)) && Arrays.equals(entries.get(1).getData(), "entry3".getBytes(Encoding))) || (Arrays.equals(entries.get(0).getData(), "entry3".getBytes(Encoding)) - && Arrays.equals(entries.get(1).getData(), "entry1".getBytes(Encoding)))); + && Arrays.equals(entries.get(1).getData(), "entry1".getBytes(Encoding)))); entries.forEach(Entry::release); // 3. Fail on reading non-existing position - PositionImpl invalidPosition = new PositionImpl(100, 100); + Position invalidPosition = PositionFactory.create(100, 100); positions.add(invalidPosition); try { @@ -2644,24 +2807,24 @@ void testGetLastIndividualDeletedRange() throws Exception { ManagedLedger ledger = factory.open("test_last_individual_deleted"); ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1"); - PositionImpl markDeletedPosition = (PositionImpl) c1.getMarkDeletedPosition(); + Position markDeletedPosition = c1.getMarkDeletedPosition(); for(int i = 0; i < 10; i++) { ledger.addEntry(("entry" + i).getBytes(Encoding)); } - PositionImpl p1 = PositionImpl.get(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId() + 1); - PositionImpl p2 = PositionImpl.get(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId() + 2); - PositionImpl p3 = PositionImpl.get(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId() + 5); - PositionImpl p4 = PositionImpl.get(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId() + 6); + Position p1 = PositionFactory.create(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId() + 1); + Position p2 = PositionFactory.create(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId() + 2); + Position p3 = PositionFactory.create(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId() + 5); + Position p4 = PositionFactory.create(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId() + 6); c1.delete(Lists.newArrayList(p1, p2, p3, p4)); - assertEquals(c1.getLastIndividualDeletedRange(), Range.openClosed(PositionImpl.get(p3.getLedgerId(), + assertEquals(c1.getLastIndividualDeletedRange(), Range.openClosed(PositionFactory.create(p3.getLedgerId(), p3.getEntryId() - 1), p4)); - PositionImpl p5 = PositionImpl.get(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId() + 8); + Position p5 = PositionFactory.create(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId() + 8); c1.delete(p5); - assertEquals(c1.getLastIndividualDeletedRange(), Range.openClosed(PositionImpl.get(p5.getLedgerId(), + assertEquals(c1.getLastIndividualDeletedRange(), Range.openClosed(PositionFactory.create(p5.getLedgerId(), p5.getEntryId() - 1), p5)); } @@ -2671,14 +2834,14 @@ void testTrimDeletedEntries() throws ManagedLedgerException, InterruptedExceptio ManagedLedger ledger = factory.open("my_test_ledger"); ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1"); - PositionImpl markDeletedPosition = (PositionImpl) c1.getMarkDeletedPosition(); + Position markDeletedPosition = c1.getMarkDeletedPosition(); for(int i = 0; i < 10; i++) { ledger.addEntry(("entry" + i).getBytes(Encoding)); } - PositionImpl p1 = PositionImpl.get(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId() + 1); - PositionImpl p2 = PositionImpl.get(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId() + 2); - PositionImpl p3 = PositionImpl.get(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId() + 5); - PositionImpl p4 = PositionImpl.get(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId() + 6); + Position p1 = PositionFactory.create(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId() + 1); + Position p2 = PositionFactory.create(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId() + 2); + Position p3 = PositionFactory.create(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId() + 5); + Position p4 = PositionFactory.create(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId() + 6); c1.delete(Lists.newArrayList(p1, p2, p3, p4)); @@ -2691,7 +2854,7 @@ void testTrimDeletedEntries() throws ManagedLedgerException, InterruptedExceptio List entries = Lists.newArrayList(entry1, entry2, entry3, entry4, entry5); c1.trimDeletedEntries(entries); assertEquals(entries.size(), 1); - assertEquals(entries.get(0).getPosition(), PositionImpl.get(markDeletedPosition.getLedgerId(), + assertEquals(entries.get(0).getPosition(), PositionFactory.create(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId() + 7)); assertEquals(entry1.refCnt(), 0); @@ -2767,7 +2930,7 @@ void testGetEntryAfterN() throws Exception { List entries = c1.readEntries(4); entries.forEach(Entry::release); - long currentLedger = ((PositionImpl) c1.getMarkDeletedPosition()).getLedgerId(); + long currentLedger = (c1.getMarkDeletedPosition()).getLedgerId(); // check if the first message is returned for '0' Entry e = c1.getNthEntry(1, IndividualDeletedEntries.Exclude); @@ -2790,8 +2953,8 @@ void testGetEntryAfterN() throws Exception { assertNull(e); // check that the mark delete and read positions have not been updated after all the previous operations - assertEquals(c1.getMarkDeletedPosition(), new PositionImpl(currentLedger, -1)); - assertEquals(c1.getReadPosition(), new PositionImpl(currentLedger, 4)); + assertEquals(c1.getMarkDeletedPosition(), PositionFactory.create(currentLedger, -1)); + assertEquals(c1.getReadPosition(), PositionFactory.create(currentLedger, 4)); c1.markDelete(pos4); assertEquals(c1.getMarkDeletedPosition(), pos4); @@ -2848,7 +3011,7 @@ public void readEntriesComplete(List entries, Object ctx) { public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { counter.countDown(); } - }, null, PositionImpl.LATEST); + }, null, PositionFactory.LATEST); assertTrue(c1.cancelPendingReadRequest()); @@ -2864,7 +3027,7 @@ public void readEntriesComplete(List entries, Object ctx) { public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { counter2.countDown(); } - }, null, PositionImpl.LATEST); + }, null, PositionFactory.LATEST); ledger.addEntry("entry-1".getBytes(Encoding)); @@ -3040,7 +3203,7 @@ public void operationFailed(ManagedLedgerException exception) { try { bkc.openLedgerNoRecovery(ledgerId, DigestType.fromApiDigestType(mlConfig.getDigestType()), - mlConfig.getPassword()); + mlConfig.getPassword()); fail("ledger should have deleted due to update-cursor failure"); } catch (BKException e) { // ok @@ -3064,7 +3227,7 @@ public void testOutOfOrderDeletePersistenceIntoLedgerWithClose() throws Exceptio managedLedgerConfig.setMaxUnackedRangesToPersistInMetadataStore(10); ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open(ledgerName, managedLedgerConfig); - ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor(cursorName); + final ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor(cursorName); List addedPositions = new ArrayList<>(); for (int i = 0; i < totalAddEntries; i++) { @@ -3110,7 +3273,8 @@ public void operationFailed(MetaStoreException e) { LedgerEntry entry = seq.nextElement(); PositionInfo positionInfo; positionInfo = PositionInfo.parseFrom(entry.getEntry()); - individualDeletedMessagesCount.set(positionInfo.getIndividualDeletedMessagesCount()); + c1.recoverIndividualDeletedMessages(positionInfo); + individualDeletedMessagesCount.set(c1.getIndividuallyDeletedMessagesSet().asRanges().size()); } catch (Exception e) { } latch.countDown(); @@ -3127,12 +3291,12 @@ public void operationFailed(MetaStoreException e) { @Cleanup("shutdown") ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(metadataStore, bkc); ledger = (ManagedLedgerImpl) factory2.open(ledgerName, managedLedgerConfig); - c1 = (ManagedCursorImpl) ledger.openCursor("c1"); + ManagedCursorImpl reopenCursor = (ManagedCursorImpl) ledger.openCursor("c1"); // verify cursor has been recovered - assertEquals(c1.getNumberOfEntriesInBacklog(false), totalAddEntries / 2); + assertEquals(reopenCursor.getNumberOfEntriesInBacklog(false), totalAddEntries / 2); // try to read entries which should only read non-deleted positions - List entries = c1.readEntries(totalAddEntries); + List entries = reopenCursor.readEntries(totalAddEntries); assertEquals(entries.size(), totalAddEntries / 2); } @@ -3213,7 +3377,7 @@ public void testInvalidMarkDelete() throws Exception { // validate: cursor.asyncMarkDelete(..) CountDownLatch markDeleteCallbackLatch = new CountDownLatch(1); - Position position = PositionImpl.get(100, 100); + Position position = PositionFactory.create(100, 100); AtomicBoolean markDeleteCallFailed = new AtomicBoolean(false); cursor.asyncMarkDelete(position, new MarkDeleteCallback() { @Override @@ -3278,8 +3442,8 @@ public void testEstimatedUnackedSize() throws Exception { @Test(timeOut = 20000) public void testRecoverCursorAheadOfLastPosition() throws Exception { final String mlName = "my_test_ledger"; - final PositionImpl lastPosition = new PositionImpl(1L, 10L); - final PositionImpl nextPosition = new PositionImpl(3L, -1L); + final Position lastPosition = PositionFactory.create(1L, 10L); + final Position nextPosition = PositionFactory.create(3L, -1L); final String cursorName = "my_test_cursor"; final long cursorsLedgerId = -1L; @@ -3307,10 +3471,10 @@ public Object answer(InvocationOnMock invocation) { when(ml.getNextValidLedger(markDeleteLedgerId)).thenReturn(3L); when(ml.getNextValidPosition(lastPosition)).thenReturn(nextPosition); when(ml.ledgerExists(markDeleteLedgerId)).thenReturn(false); + when(ml.getConfig()).thenReturn(new ManagedLedgerConfig()); BookKeeper mockBookKeeper = mock(BookKeeper.class); - final ManagedCursorImpl cursor = new ManagedCursorImpl(mockBookKeeper, new ManagedLedgerConfig(), ml, - cursorName); + final ManagedCursorImpl cursor = new ManagedCursorImpl(mockBookKeeper, ml, cursorName); cursor.recover(new VoidCallback() { @Override @@ -3337,7 +3501,7 @@ public void testRecoverCursorAfterResetToLatestForNewEntry() throws Exception { assertEquals(c.getReadPosition().getEntryId(), 0); assertEquals(ml.getLastConfirmedEntry().getEntryId(), -1); - c.resetCursor(PositionImpl.LATEST); + c.resetCursor(PositionFactory.LATEST); // A reset cursor starts out with these values. The rest of the test assumes this, so we assert it here. assertEquals(c.getMarkDeletedPosition().getEntryId(), -1); @@ -3391,7 +3555,7 @@ public void testRecoverCursorAfterResetToLatestForMultipleEntries() throws Excep assertEquals(c.getReadPosition().getEntryId(), 0); assertEquals(ml.getLastConfirmedEntry().getEntryId(), -1); - c.resetCursor(PositionImpl.LATEST); + c.resetCursor(PositionFactory.LATEST); // A reset cursor starts out with these values. The rest of the test assumes this, so we assert it here. assertEquals(c.getMarkDeletedPosition().getEntryId(), -1); @@ -3404,7 +3568,7 @@ public void testRecoverCursorAfterResetToLatestForMultipleEntries() throws Excep ml.addEntry(new byte[1]); ml.addEntry(new byte[1]); - c.resetCursor(PositionImpl.LATEST); + c.resetCursor(PositionFactory.LATEST); assertEquals(c.getMarkDeletedPosition().getEntryId(), 3); assertEquals(c.getReadPosition().getEntryId(), 4); @@ -3465,7 +3629,7 @@ void testAlwaysInactive() throws Exception { @Test void testNonDurableCursorActive() throws Exception { ManagedLedger ml = factory.open("testInactive"); - ManagedCursor cursor = ml.newNonDurableCursor(PositionImpl.LATEST, "c1"); + ManagedCursor cursor = ml.newNonDurableCursor(PositionFactory.LATEST, "c1"); assertTrue(cursor.isActive()); @@ -3525,19 +3689,19 @@ public void testBatchIndexMarkdelete() throws ManagedLedgerException, Interrupte } assertEquals(cursor.getNumberOfEntries(), totalEntries); markDeleteBatchIndex(cursor, positions[0], 10, 3); - List deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[0]), 10); + List deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray(positions[0]), 10); Assert.assertEquals(1, deletedIndexes.size()); Assert.assertEquals(0, deletedIndexes.get(0).getStart()); Assert.assertEquals(3, deletedIndexes.get(0).getEnd()); markDeleteBatchIndex(cursor, positions[0], 10, 4); - deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[0]), 10); + deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray(positions[0]), 10); Assert.assertEquals(1, deletedIndexes.size()); Assert.assertEquals(0, deletedIndexes.get(0).getStart()); Assert.assertEquals(4, deletedIndexes.get(0).getEnd()); markDeleteBatchIndex(cursor, positions[0], 10, 2); - deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[0]), 10); + deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray(positions[0]), 10); Assert.assertEquals(1, deletedIndexes.size()); Assert.assertEquals(0, deletedIndexes.get(0).getStart()); Assert.assertEquals(4, deletedIndexes.get(0).getEnd()); @@ -3556,19 +3720,19 @@ public void testBatchIndexDelete() throws ManagedLedgerException, InterruptedExc } assertEquals(cursor.getNumberOfEntries(), totalEntries); deleteBatchIndex(cursor, positions[0], 10, Lists.newArrayList(new IntRange().setStart(2).setEnd(4))); - List deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[0]), 10); + List deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray(positions[0]), 10); Assert.assertEquals(1, deletedIndexes.size()); Assert.assertEquals(2, deletedIndexes.get(0).getStart()); Assert.assertEquals(4, deletedIndexes.get(0).getEnd()); deleteBatchIndex(cursor, positions[0], 10, Lists.newArrayList(new IntRange().setStart(3).setEnd(8))); - deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[0]), 10); + deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray(positions[0]), 10); Assert.assertEquals(1, deletedIndexes.size()); Assert.assertEquals(2, deletedIndexes.get(0).getStart()); Assert.assertEquals(8, deletedIndexes.get(0).getEnd()); deleteBatchIndex(cursor, positions[0], 10, Lists.newArrayList(new IntRange().setStart(0).setEnd(0))); - deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[0]), 10); + deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray(positions[0]), 10); Assert.assertEquals(2, deletedIndexes.size()); Assert.assertEquals(0, deletedIndexes.get(0).getStart()); Assert.assertEquals(0, deletedIndexes.get(0).getEnd()); @@ -3577,24 +3741,24 @@ public void testBatchIndexDelete() throws ManagedLedgerException, InterruptedExc deleteBatchIndex(cursor, positions[0], 10, Lists.newArrayList(new IntRange().setStart(1).setEnd(1))); deleteBatchIndex(cursor, positions[0], 10, Lists.newArrayList(new IntRange().setStart(9).setEnd(9))); - deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[0]), 10); + deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray(positions[0]), 10); Assert.assertNull(deletedIndexes); Assert.assertEquals(positions[0], cursor.getMarkDeletedPosition()); deleteBatchIndex(cursor, positions[1], 10, Lists.newArrayList(new IntRange().setStart(0).setEnd(5))); cursor.delete(positions[1]); deleteBatchIndex(cursor, positions[1], 10, Lists.newArrayList(new IntRange().setStart(6).setEnd(8))); - deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[1]), 10); + deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray(positions[1]), 10); Assert.assertNull(deletedIndexes); deleteBatchIndex(cursor, positions[2], 10, Lists.newArrayList(new IntRange().setStart(0).setEnd(5))); cursor.markDelete(positions[3]); - deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[2]), 10); + deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray(positions[2]), 10); Assert.assertNull(deletedIndexes); deleteBatchIndex(cursor, positions[3], 10, Lists.newArrayList(new IntRange().setStart(0).setEnd(5))); cursor.resetCursor(positions[0]); - deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[3]), 10); + deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray(positions[3]), 10); Assert.assertNull(deletedIndexes); } @@ -3631,17 +3795,17 @@ public void testBatchIndexesDeletionPersistAndRecover() throws ManagedLedgerExce ledger = factory.open("test_batch_indexes_deletion_persistent", managedLedgerConfig); cursor = ledger.openCursor("c1"); - List deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[5]), 10); + List deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray(positions[5]), 10); Assert.assertEquals(deletedIndexes.size(), 1); Assert.assertEquals(deletedIndexes.get(0).getStart(), 3); Assert.assertEquals(deletedIndexes.get(0).getEnd(), 6); Assert.assertEquals(cursor.getMarkDeletedPosition(), positions[4]); deleteBatchIndex(cursor, positions[5], 10, Lists.newArrayList(new IntRange().setStart(0).setEnd(9))); - deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[5]), 10); + deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray(positions[5]), 10); Assert.assertNull(deletedIndexes); Assert.assertEquals(cursor.getMarkDeletedPosition(), positions[5]); - deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray((PositionImpl) positions[6]), 10); + deletedIndexes = getAckedIndexRange(cursor.getDeletedBatchIndexesAsLongArray(positions[6]), 10); Assert.assertEquals(deletedIndexes.size(), 1); Assert.assertEquals(deletedIndexes.get(0).getStart(), 1); Assert.assertEquals(deletedIndexes.get(0).getEnd(), 3); @@ -3650,39 +3814,36 @@ public void testBatchIndexesDeletionPersistAndRecover() throws ManagedLedgerExce private void deleteBatchIndex(ManagedCursor cursor, Position position, int batchSize, List deleteIndexes) throws InterruptedException { CountDownLatch latch = new CountDownLatch(1); - PositionImpl pos = (PositionImpl) position; BitSet bitSet = new BitSet(batchSize); bitSet.set(0, batchSize); deleteIndexes.forEach(intRange -> { bitSet.clear(intRange.getStart(), intRange.getEnd() + 1); }); - pos.ackSet = bitSet.toLongArray(); + Position pos = AckSetStateUtil.createPositionWithAckSet(position.getLedgerId(), position.getEntryId(), bitSet.toLongArray()); cursor.asyncDelete(pos, - new DeleteCallback() { - @Override - public void deleteComplete(Object ctx) { - latch.countDown(); - } + new DeleteCallback() { + @Override + public void deleteComplete(Object ctx) { + latch.countDown(); + } - @Override - public void deleteFailed(ManagedLedgerException exception, Object ctx) { - latch.countDown(); - } - }, null); + @Override + public void deleteFailed(ManagedLedgerException exception, Object ctx) { + latch.countDown(); + } + }, null); latch.await(); - pos.ackSet = null; } private void markDeleteBatchIndex(ManagedCursor cursor, Position position, int batchSize, int batchIndex ) throws InterruptedException { CountDownLatch latch = new CountDownLatch(1); - PositionImpl pos = (PositionImpl) position; BitSetRecyclable bitSet = new BitSetRecyclable(); bitSet.set(0, batchSize); bitSet.clear(0, batchIndex + 1); - pos.ackSet = bitSet.toLongArray(); + Position pos = AckSetStateUtil.createPositionWithAckSet(position.getLedgerId(), position.getEntryId(), bitSet.toLongArray()); cursor.asyncMarkDelete(pos, new MarkDeleteCallback() { @Override @@ -3696,7 +3857,6 @@ public void markDeleteComplete(Object ctx) { } }, null); latch.await(); - pos.ackSet = null; } private List getAckedIndexRange(long[] bitSetLongArray, int batchSize) { @@ -3748,8 +3908,8 @@ public void testReadEntriesOrWaitWithMaxPosition() throws Exception { int sendNumber = 20; ManagedLedger ledger = factory.open("testReadEntriesOrWaitWithMaxPosition"); ManagedCursor c = ledger.openCursor("c"); - Position position = PositionImpl.EARLIEST; - Position maxCanReadPosition = PositionImpl.EARLIEST; + Position position = PositionFactory.EARLIEST; + Position maxCanReadPosition = PositionFactory.EARLIEST; for (int i = 0; i < sendNumber; i++) { if (i == readMaxNumber - 1) { position = ledger.addEntry(new byte[1024]); @@ -3771,7 +3931,7 @@ public void readEntriesComplete(List entries, Object ctx) { public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { completableFuture.completeExceptionally(exception); } - }, null, (PositionImpl) position); + }, null, position); int number = completableFuture.get(); assertEquals(number, readMaxNumber); @@ -3786,7 +3946,7 @@ public void readEntriesComplete(List entries, Object ctx) { public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { completableFuture.completeExceptionally(exception); } - }, null, (PositionImpl) maxCanReadPosition); + }, null, maxCanReadPosition); assertEquals(number, sendNumber - readMaxNumber); @@ -3959,11 +4119,11 @@ public void testConsistencyOfIndividualMessages() throws Exception { ManagedLedger ledger1 = factory.open("testConsistencyOfIndividualMessages"); ManagedCursorImpl c1 = (ManagedCursorImpl) ledger1.openCursor("c"); - PositionImpl p1 = (PositionImpl) ledger1.addEntry(new byte[1024]); + Position p1 = ledger1.addEntry(new byte[1024]); c1.markDelete(p1); // Artificially add a position that is before the current mark-delete position - LongPairRangeSet idm = c1.getIndividuallyDeletedMessagesSet(); + LongPairRangeSet idm = c1.getIndividuallyDeletedMessagesSet(); idm.addOpenClosed(p1.getLedgerId() - 1, 0, p1.getLedgerId() - 1, 10); List positions = new ArrayList<>(); @@ -4060,7 +4220,7 @@ public void testCursorGetBacklog() throws Exception { ((ConcurrentSkipListMap) field.get(ledger)).remove(position.getLedgerId()); field = ManagedCursorImpl.class.getDeclaredField("markDeletePosition"); field.setAccessible(true); - field.set(managedCursor, PositionImpl.get(position1.getLedgerId(), -1)); + field.set(managedCursor, PositionFactory.create(position1.getLedgerId(), -1)); Assert.assertEquals(managedCursor.getNumberOfEntriesInBacklog(true), 2); @@ -4123,7 +4283,7 @@ public void testReadEmptyEntryList() throws Exception { .open("testReadEmptyEntryList", managedLedgerConfig); ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("test"); - PositionImpl lastPosition = (PositionImpl) ledger.addEntry("test".getBytes(Encoding)); + Position lastPosition = ledger.addEntry("test".getBytes(Encoding)); ledger.rollCurrentLedgerIfFull(); AtomicBoolean flag = new AtomicBoolean(); @@ -4144,10 +4304,10 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { // op readPosition is bigger than maxReadPosition OpReadEntry opReadEntry = OpReadEntry.create(cursor, ledger.lastConfirmedEntry, 10, callback, - null, PositionImpl.get(lastPosition.getLedgerId(), -1), null); + null, PositionFactory.create(lastPosition.getLedgerId(), -1), null); Field field = ManagedCursorImpl.class.getDeclaredField("readPosition"); field.setAccessible(true); - field.set(cursor, PositionImpl.EARLIEST); + field.set(cursor, PositionFactory.EARLIEST); ledger.asyncReadEntries(opReadEntry); // when readPosition is bigger than maxReadPosition, should complete the opReadEntry @@ -4189,7 +4349,7 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { final int numReadRequests = 3; for (int i = 0; i < numReadRequests; i++) { - cursor.asyncReadEntriesOrWait(1, callback, null, new PositionImpl(0, 0)); + cursor.asyncReadEntriesOrWait(1, callback, null, PositionFactory.create(0, 0)); } Awaitility.await().atMost(Duration.ofSeconds(1)) .untilAsserted(() -> assertEquals(ledger.waitingCursors.size(), 1)); @@ -4276,8 +4436,8 @@ public void testReadEntriesWithSkip() throws ManagedLedgerException, Interrupted int sendNumber = 20; ManagedLedger ledger = factory.open("testReadEntriesWithSkip"); ManagedCursor cursor = ledger.openCursor("c"); - Position position = PositionImpl.EARLIEST; - Position maxCanReadPosition = PositionImpl.EARLIEST; + Position position = PositionFactory.EARLIEST; + Position maxCanReadPosition = PositionFactory.EARLIEST; for (int i = 0; i < sendNumber; i++) { if (i == readMaxNumber - 1) { position = ledger.addEntry(new byte[1024]); @@ -4306,7 +4466,7 @@ public void readEntriesComplete(List entries, Object ctx) { public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { completableFuture.completeExceptionally(exception); } - }, null, (PositionImpl) position, pos -> { + }, null, position, pos -> { return pos.getEntryId() % 2 != 0; }); @@ -4333,7 +4493,7 @@ public void readEntriesComplete(List entries, Object ctx) { public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { completableFuture2.completeExceptionally(exception); } - }, null, (PositionImpl) maxCanReadPosition, pos -> { + }, null, maxCanReadPosition, pos -> { return pos.getEntryId() % 2 != 0; }); @@ -4342,7 +4502,7 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { assertEquals(cursor.getReadPosition().getEntryId(), 20L); - cursor.seek(PositionImpl.EARLIEST); + cursor.seek(PositionFactory.EARLIEST); CompletableFuture completableFuture3 = new CompletableFuture<>(); cursor.asyncReadEntriesWithSkipOrWait(sendNumber, new ReadEntriesCallback() { @Override @@ -4354,13 +4514,13 @@ public void readEntriesComplete(List entries, Object ctx) { public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { completableFuture3.completeExceptionally(exception); } - }, null, (PositionImpl) maxCanReadPosition, pos -> false); + }, null, maxCanReadPosition, pos -> false); int number3 = completableFuture3.get(); assertEquals(number3, sendNumber); assertEquals(cursor.getReadPosition().getEntryId(), 20L); - cursor.seek(PositionImpl.EARLIEST); + cursor.seek(PositionFactory.EARLIEST); CompletableFuture completableFuture4 = new CompletableFuture<>(); cursor.asyncReadEntriesWithSkipOrWait(sendNumber, new ReadEntriesCallback() { @Override @@ -4372,7 +4532,7 @@ public void readEntriesComplete(List entries, Object ctx) { public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { completableFuture4.completeExceptionally(exception); } - }, null, (PositionImpl) maxCanReadPosition, pos -> true); + }, null, maxCanReadPosition, pos -> true); int number4 = completableFuture4.get(); assertEquals(number4, 0); @@ -4382,5 +4542,314 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { ledger.close(); } + @Test + public void testReadEntriesWithSkipDeletedEntries() throws Exception { + @Cleanup + ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testReadEntriesWithSkipDeletedEntries"); + ledger = Mockito.spy(ledger); + List actualReadEntryIds = new ArrayList<>(); + Mockito.doAnswer(inv -> { + long start = inv.getArgument(1); + long end = inv.getArgument(2); + for (long i = start; i <= end; i++) { + actualReadEntryIds.add(i); + } + return inv.callRealMethod(); + }) + .when(ledger) + .asyncReadEntry(Mockito.any(ReadHandle.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + @Cleanup + ManagedCursor cursor = ledger.openCursor("c"); + + int entries = 20; + Position maxReadPosition = null; + Map map = new HashMap<>(); + for (int i = 0; i < entries; i++) { + maxReadPosition = ledger.addEntry(new byte[1024]); + map.put(i, maxReadPosition); + } + + + Set deletedPositions = new HashSet<>(); + deletedPositions.add(map.get(1)); + deletedPositions.add(map.get(4)); + deletedPositions.add(map.get(5)); + deletedPositions.add(map.get(8)); + deletedPositions.add(map.get(9)); + deletedPositions.add(map.get(10)); + deletedPositions.add(map.get(15)); + deletedPositions.add(map.get(17)); + deletedPositions.add(map.get(19)); + cursor.delete(deletedPositions); + + CompletableFuture f0 = new CompletableFuture<>(); + List readEntries = new ArrayList<>(); + cursor.asyncReadEntries(5, -1L, new ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + readEntries.addAll(entries); + f0.complete(null); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + f0.completeExceptionally(exception); + } + }, null, PositionFactory.create(maxReadPosition.getLedgerId(), maxReadPosition.getEntryId()).getNext()); + + f0.get(); + + CompletableFuture f1 = new CompletableFuture<>(); + cursor.asyncReadEntries(5, -1L, new ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + readEntries.addAll(entries); + f1.complete(null); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + f1.completeExceptionally(exception); + } + }, null, PositionFactory.create(maxReadPosition.getLedgerId(), maxReadPosition.getEntryId()).getNext()); + + + f1.get(); + CompletableFuture f2 = new CompletableFuture<>(); + cursor.asyncReadEntries(100, -1L, new ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + readEntries.addAll(entries); + f2.complete(null); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + f2.completeExceptionally(exception); + } + }, null, PositionFactory.create(maxReadPosition.getLedgerId(), maxReadPosition.getEntryId()).getNext()); + + f2.get(); + + Position cursorReadPosition = cursor.getReadPosition(); + Position expectReadPosition = maxReadPosition.getNext(); + assertTrue(cursorReadPosition.getLedgerId() == expectReadPosition.getLedgerId() + && cursorReadPosition.getEntryId() == expectReadPosition.getEntryId()); + + assertEquals(readEntries.size(), actualReadEntryIds.size()); + assertEquals(entries - deletedPositions.size(), actualReadEntryIds.size()); + for (Entry entry : readEntries) { + long entryId = entry.getEntryId(); + assertTrue(actualReadEntryIds.contains(entryId)); + } + } + + + @Test + public void testReadEntriesWithSkipDeletedEntriesAndWithSkipConditions() throws Exception { + @Cleanup + ManagedLedgerImpl ledger = (ManagedLedgerImpl) + factory.open("testReadEntriesWithSkipDeletedEntriesAndWithSkipConditions"); + ledger = Mockito.spy(ledger); + + List actualReadEntryIds = new ArrayList<>(); + Mockito.doAnswer(inv -> { + long start = inv.getArgument(1); + long end = inv.getArgument(2); + for (long i = start; i <= end; i++) { + actualReadEntryIds.add(i); + } + return inv.callRealMethod(); + }) + .when(ledger) + .asyncReadEntry(Mockito.any(ReadHandle.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + @Cleanup + ManagedCursor cursor = ledger.openCursor("c"); + + int entries = 20; + Position maxReadPosition0 = null; + Map map = new HashMap<>(); + for (int i = 0; i < entries; i++) { + maxReadPosition0 = ledger.addEntry(new byte[1024]); + map.put(i, maxReadPosition0); + } + + Position maxReadPosition = + PositionFactory.create(maxReadPosition0.getLedgerId(), maxReadPosition0.getEntryId()).getNext(); + + Set deletedPositions = new HashSet<>(); + deletedPositions.add(map.get(1)); + deletedPositions.add(map.get(3)); + deletedPositions.add(map.get(5)); + cursor.delete(deletedPositions); + + Set skippedPositions = new HashSet<>(); + skippedPositions.add(map.get(6).getEntryId()); + skippedPositions.add(map.get(7).getEntryId()); + skippedPositions.add(map.get(8).getEntryId()); + skippedPositions.add(map.get(11).getEntryId()); + skippedPositions.add(map.get(15).getEntryId()); + skippedPositions.add(map.get(16).getEntryId()); + + Predicate skipCondition = position -> skippedPositions.contains(position.getEntryId()); + List readEntries = new ArrayList<>(); + + CompletableFuture f0 = new CompletableFuture<>(); + cursor.asyncReadEntriesWithSkip(10, -1L, new ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + readEntries.addAll(entries); + f0.complete(null); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + f0.completeExceptionally(exception); + } + }, null, maxReadPosition, skipCondition); + + f0.get(); + CompletableFuture f1 = new CompletableFuture<>(); + cursor.asyncReadEntriesWithSkip(100, -1L, new ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + readEntries.addAll(entries); + f1.complete(null); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + f1.completeExceptionally(exception); + } + }, null, maxReadPosition, skipCondition); + f1.get(); + + + assertEquals(actualReadEntryIds.size(), readEntries.size()); + assertEquals(entries - deletedPositions.size() - skippedPositions.size(), actualReadEntryIds.size()); + for (Entry entry : readEntries) { + long entryId = entry.getEntryId(); + assertTrue(actualReadEntryIds.contains(entryId)); + } + + Position cursorReadPosition = cursor.getReadPosition(); + Position expectReadPosition = maxReadPosition; + assertTrue(cursorReadPosition.getLedgerId() == expectReadPosition.getLedgerId() + && cursorReadPosition.getEntryId() == expectReadPosition.getEntryId()); + } + + @Test + public void testRecoverCursorWithTerminateManagedLedger() throws Exception { + String mlName = "my_test_ledger"; + String cursorName = "c1"; + + ManagedLedgerConfig config = new ManagedLedgerConfig(); + ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open(mlName, config); + ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor(cursorName); + + // Write some data. + Position p0 = ledger.addEntry("entry-0".getBytes()); + Position p1 = ledger.addEntry("entry-1".getBytes()); + + // Read message. + List entries = c1.readEntries(2); + assertEquals(entries.size(), 2); + assertEquals(entries.get(0).getPosition(), p0); + assertEquals(entries.get(1).getPosition(), p1); + entries.forEach(Entry::release); + + // Mark delete the last message. + c1.markDelete(p1); + Position markDeletedPosition = c1.getMarkDeletedPosition(); + Assert.assertEquals(markDeletedPosition, p1); + + // Terminate the managed ledger. + Position lastPosition = ledger.terminate(); + assertEquals(lastPosition, p1); + + // Close the ledger. + ledger.close(); + + // Reopen the ledger. + ledger = (ManagedLedgerImpl) factory.open(mlName, config); + BookKeeper mockBookKeeper = mock(BookKeeper.class); + final ManagedCursorImpl cursor = new ManagedCursorImpl(mockBookKeeper, ledger, cursorName); + + CompletableFuture recoverFuture = new CompletableFuture<>(); + // Recover the cursor. + cursor.recover(new VoidCallback() { + @Override + public void operationComplete() { + recoverFuture.complete(null); + } + + @Override + public void operationFailed(ManagedLedgerException exception) { + recoverFuture.completeExceptionally(exception); + } + }); + + recoverFuture.join(); + assertTrue(recoverFuture.isDone()); + assertFalse(recoverFuture.isCompletedExceptionally()); + + // Verify the cursor state. + assertEquals(cursor.getMarkDeletedPosition(), markDeletedPosition); + assertEquals(cursor.getReadPosition(), markDeletedPosition.getNext()); + } + + @Test + void testForceCursorRecovery() throws Exception { + ManagedLedgerFactoryConfig managedLedgerFactoryConfig = new ManagedLedgerFactoryConfig(); + TestPulsarMockBookKeeper bk = new TestPulsarMockBookKeeper(executor); + factory = new ManagedLedgerFactoryImpl(metadataStore, bk); + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setLedgerForceRecovery(true); + ManagedLedger ledger = factory.open("my_test_ledger", config); + ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1"); + ledger.addEntry("entry-1".getBytes(Encoding)); + long invalidLedger = -1L; + bk.setErrorCodeMap(invalidLedger, BKException.Code.BookieHandleNotAvailableException); + ManagedCursorInfo info = ManagedCursorInfo.newBuilder().setCursorsLedgerId(invalidLedger).build(); + CountDownLatch latch = new CountDownLatch(1); + MutableBoolean recovered = new MutableBoolean(false); + VoidCallback callback = new VoidCallback() { + @Override + public void operationComplete() { + recovered.setValue(true); + latch.countDown(); + } + + @Override + public void operationFailed(ManagedLedgerException exception) { + recovered.setValue(false); + latch.countDown(); + } + }; + c1.recoverFromLedger(info, callback); + latch.await(); + assertTrue(recovered.booleanValue()); + } + + class TestPulsarMockBookKeeper extends PulsarMockBookKeeper { + Map ledgerErrors = new HashMap<>(); + + public TestPulsarMockBookKeeper(OrderedExecutor orderedExecutor) throws Exception { + super(orderedExecutor); + } + + public void setErrorCodeMap(long ledgerId, int rc) { + ledgerErrors.put(ledgerId, rc); + } + + public void asyncOpenLedger(final long lId, final DigestType digestType, final byte[] passwd, + final OpenCallback cb, final Object ctx) { + if (ledgerErrors.containsKey(lId)) { + cb.openComplete(ledgerErrors.get(lId), null, ctx); + } + super.asyncOpenLedger(lId, digestType, passwd, cb, ctx); + } + } + private static final Logger log = LoggerFactory.getLogger(ManagedCursorTest.class); } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java index 0281c8cdd88e3..9635376a782d3 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java @@ -23,7 +23,6 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -import io.netty.buffer.ByteBuf; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -34,7 +33,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import lombok.Cleanup; + import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.BookKeeperTestClient; import org.apache.bookkeeper.client.api.DigestType; @@ -53,8 +52,13 @@ import org.apache.bookkeeper.mledger.util.ThrowableToStringUtil; import org.apache.bookkeeper.test.BookKeeperClusterTestCase; import org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats; +import org.apache.pulsar.common.util.collections.LongPairRangeSet; +import org.awaitility.Awaitility; import org.testng.annotations.Test; +import io.netty.buffer.ByteBuf; +import lombok.Cleanup; + public class ManagedLedgerBkTest extends BookKeeperClusterTestCase { public ManagedLedgerBkTest() { @@ -357,7 +361,7 @@ public void ledgerFencedByAutoReplication() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger" + testName, config); ManagedCursor c1 = ledger.openCursor("c1"); - PositionImpl p1 = (PositionImpl) ledger.addEntry("entry-1".getBytes()); + Position p1 = ledger.addEntry("entry-1".getBytes()); // Trigger the closure of the data ledger bkc.openLedger(p1.getLedgerId(), BookKeeper.DigestType.CRC32C, new byte[] {}); @@ -367,7 +371,7 @@ public void ledgerFencedByAutoReplication() throws Exception { assertEquals(2, c1.getNumberOfEntries()); assertEquals(2, c1.getNumberOfEntriesInBacklog(false)); - PositionImpl p3 = (PositionImpl) ledger.addEntry("entry-3".getBytes()); + Position p3 = ledger.addEntry("entry-3".getBytes()); // Now entry-2 should have been written before entry-3 assertEquals(3, c1.getNumberOfEntries()); @@ -548,6 +552,82 @@ public void testChangeCrcType() throws Exception { } } + @Test + public void testPeriodicRollover() throws Exception { + ManagedLedgerFactoryConfig factoryConf = new ManagedLedgerFactoryConfig(); + factoryConf.setMaxCacheSize(0); + int rolloverTimeForCursorInSeconds = 5; + @Cleanup("shutdown") + ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, factoryConf); + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setEnsembleSize(1).setWriteQuorumSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1) + .setMetadataAckQuorumSize(1) + .setLedgerRolloverTimeout(rolloverTimeForCursorInSeconds); + ManagedLedger ledger = factory.open("my-ledger" + testName, config); + ManagedCursor cursor = ledger.openCursor("c1"); + + Position pos = ledger.addEntry("entry-0".getBytes()); + ledger.addEntry("entry-1".getBytes()); + + List entries = cursor.readEntries(2); + assertEquals(2, entries.size()); + entries.forEach(Entry::release); + + ManagedCursorImpl cursorImpl = (ManagedCursorImpl) cursor; + assertEquals(ManagedCursorImpl.State.NoLedger, cursorImpl.state); + + // this creates the ledger + cursor.delete(pos); + + Awaitility.await().until(() -> cursorImpl.state == ManagedCursorImpl.State.Open); + + Thread.sleep(rolloverTimeForCursorInSeconds * 1000 + 1000); + + long currentLedgerId = cursorImpl.getCursorLedger(); + assertTrue(cursor.periodicRollover()); + Awaitility.await().until(() -> cursorImpl.getCursorLedger() != currentLedgerId); + } + + /** + * This test validates that cursor serializes and deserializes individual-ack list from the bk-ledger. + * + * @throws Exception + */ + @Test + public void testUnackmessagesAndRecovery() throws Exception { + ManagedLedgerFactoryConfig factoryConf = new ManagedLedgerFactoryConfig(); + factoryConf.setMaxCacheSize(0); + + ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, factoryConf); + + ManagedLedgerConfig config = new ManagedLedgerConfig().setEnsembleSize(1).setWriteQuorumSize(1) + .setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataWriteQuorumSize(1) + .setMaxUnackedRangesToPersistInMetadataStore(1).setMaxEntriesPerLedger(5).setMetadataAckQuorumSize(1); + ManagedLedger ledger = factory.open("my_test_unack_messages", config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + + int totalEntries = 100; + for (int i = 0; i < totalEntries; i++) { + Position p = ledger.addEntry("entry".getBytes()); + if (i % 2 == 0) { + cursor.delete(p); + } + } + + LongPairRangeSet unackMessagesBefore = cursor.getIndividuallyDeletedMessagesSet(); + + ledger.close(); + + // open and recover cursor + ledger = factory.open("my_test_unack_messages", config); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + + LongPairRangeSet unackMessagesAfter = cursor.getIndividuallyDeletedMessagesSet(); + assertTrue(unackMessagesBefore.equals(unackMessagesAfter)); + + ledger.close(); + factory.shutdown(); + } } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerErrorsTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerErrorsTest.java index 512e90d17f5e8..d72bffa27d30a 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerErrorsTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerErrorsTest.java @@ -31,12 +31,14 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import lombok.Cleanup; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.client.api.DigestType; +import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback; import org.apache.bookkeeper.mledger.Entry; @@ -381,7 +383,6 @@ public void recoverAfterZnodeVersionError() throws Exception { ledger.addEntry("entry".getBytes()); fail("should fail"); } catch (ManagedLedgerFencedException e) { - assertEquals(e.getCause().getClass(), ManagedLedgerException.BadVersionException.class); // ok } @@ -510,6 +511,35 @@ public void recoverAfterWriteError() throws Exception { entries.forEach(Entry::release); } + @Test + public void recoverAfterOpenManagedLedgerFail() throws Exception { + ManagedLedger ledger = factory.open("recoverAfterOpenManagedLedgerFail"); + Position position = ledger.addEntry("entry".getBytes()); + ledger.close(); + bkc.failAfter(0, BKException.Code.BookieHandleNotAvailableException); + try { + factory.open("recoverAfterOpenManagedLedgerFail"); + } catch (Exception e) { + // ok + } + + ledger = factory.open("recoverAfterOpenManagedLedgerFail"); + CompletableFuture future = new CompletableFuture<>(); + ledger.asyncReadEntry(position, new AsyncCallbacks.ReadEntryCallback() { + @Override + public void readEntryComplete(Entry entry, Object ctx) { + future.complete(entry.getData()); + } + + @Override + public void readEntryFailed(ManagedLedgerException exception, Object ctx) { + future.completeExceptionally(exception); + } + }, null); + byte[] bytes = future.get(30, TimeUnit.SECONDS); + assertEquals(new String(bytes), "entry"); + } + @Test public void recoverLongTimeAfterMultipleWriteErrors() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("recoverLongTimeAfterMultipleWriteErrors"); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryShutdownTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryShutdownTest.java index c223490f1c798..00fc151c6d792 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryShutdownTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryShutdownTest.java @@ -34,6 +34,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.AsyncCallback; import org.apache.bookkeeper.client.BookKeeper; @@ -43,6 +44,7 @@ import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.ReadOnlyCursor; import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.pulsar.metadata.api.GetResult; @@ -130,6 +132,7 @@ private void setup() { LedgerHandle ledgerHandle = mock(LedgerHandle.class); LedgerHandle newLedgerHandle = mock(LedgerHandle.class); + @Cleanup("shutdownNow") OrderedExecutor executor = OrderedExecutor.newBuilder().name("Test").build(); given(bookKeeper.getMainWorkerPool()).willReturn(executor); doAnswer(inv -> { @@ -165,7 +168,7 @@ public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { } }, null); - factory.asyncOpenReadOnlyCursor(ledgerName, PositionImpl.EARLIEST, new ManagedLedgerConfig(), + factory.asyncOpenReadOnlyCursor(ledgerName, PositionFactory.EARLIEST, new ManagedLedgerConfig(), new AsyncCallbacks.OpenReadOnlyCursorCallback() { @Override public void openReadOnlyCursorComplete(ReadOnlyCursor cursor, Object ctx) { @@ -192,6 +195,6 @@ public void openReadOnlyCursorFailed(ManagedLedgerException exception, Object ct Assert.assertThrows(ManagedLedgerException.ManagedLedgerFactoryClosedException.class, () -> factory.open(ledgerName)); Assert.assertThrows(ManagedLedgerException.ManagedLedgerFactoryClosedException.class, - () -> factory.openReadOnlyCursor(ledgerName, PositionImpl.EARLIEST, new ManagedLedgerConfig())); + () -> factory.openReadOnlyCursor(ledgerName, PositionFactory.EARLIEST, new ManagedLedgerConfig())); } } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryTest.java index d49d9ab3e2b6b..dfff9ecb49a3a 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryTest.java @@ -19,13 +19,19 @@ package org.apache.bookkeeper.mledger.impl; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import java.util.UUID; +import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.ManagedLedgerInfo; import org.apache.bookkeeper.mledger.ManagedLedgerInfo.CursorInfo; import org.apache.bookkeeper.mledger.ManagedLedgerInfo.MessageRangeInfo; +import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.test.MockedBookKeeperTestCase; +import org.awaitility.Awaitility; +import org.testng.Assert; import org.testng.annotations.Test; public class ManagedLedgerFactoryTest extends MockedBookKeeperTestCase { @@ -37,9 +43,9 @@ public void testGetManagedLedgerInfoWithClose() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testGetManagedLedgerInfo", conf); ManagedCursor c1 = ledger.openCursor("c1"); - PositionImpl p1 = (PositionImpl) ledger.addEntry("entry1".getBytes()); - PositionImpl p2 = (PositionImpl) ledger.addEntry("entry2".getBytes()); - PositionImpl p3 = (PositionImpl) ledger.addEntry("entry3".getBytes()); + Position p1 = ledger.addEntry("entry1".getBytes()); + Position p2 = ledger.addEntry("entry2".getBytes()); + Position p3 = ledger.addEntry("entry3".getBytes()); ledger.addEntry("entry4".getBytes()); c1.delete(p2); @@ -49,13 +55,17 @@ public void testGetManagedLedgerInfoWithClose() throws Exception { ManagedLedgerInfo info = factory.getManagedLedgerInfo("testGetManagedLedgerInfo"); - assertEquals(info.ledgers.size(), 4); + assertEquals(info.ledgers.size(), 5); assertEquals(info.ledgers.get(0).ledgerId, 3); assertEquals(info.ledgers.get(1).ledgerId, 4); assertEquals(info.ledgers.get(2).ledgerId, 5); assertEquals(info.ledgers.get(3).ledgerId, 6); + for (ManagedLedgerInfo.LedgerInfo linfo : info.ledgers) { + assertNotNull(linfo.timestamp); + } + assertEquals(info.cursors.size(), 1); CursorInfo cursorInfo = info.cursors.get("c1"); @@ -71,4 +81,43 @@ public void testGetManagedLedgerInfoWithClose() throws Exception { assertEquals(mri.to.entryId, 0); } + /** + * see: https://github.com/apache/pulsar/pull/18688 + */ + @Test + public void testConcurrentCloseLedgerAndSwitchLedgerForReproduceIssue() throws Exception { + String managedLedgerName = "lg_" + UUID.randomUUID().toString().replaceAll("-", "_"); + + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setThrottleMarkDelete(1); + config.setMaximumRolloverTime(Integer.MAX_VALUE, TimeUnit.SECONDS); + config.setMaxEntriesPerLedger(5); + + // create managedLedger once and close it. + ManagedLedgerImpl managedLedger1 = (ManagedLedgerImpl) factory.open(managedLedgerName, config); + waitManagedLedgerStateEquals(managedLedger1, ManagedLedgerImpl.State.LedgerOpened); + managedLedger1.close(); + + // create managedLedger the second time. + ManagedLedgerImpl managedLedger2 = (ManagedLedgerImpl) factory.open(managedLedgerName, config); + waitManagedLedgerStateEquals(managedLedger2, ManagedLedgerImpl.State.LedgerOpened); + + // Mock the task create ledger complete now, it will change the state to another value which not is Closed. + // Close managedLedger1 the second time. + managedLedger1.createComplete(1, null, null); + managedLedger1.close(); + + // Verify managedLedger2 is still there. + Assert.assertFalse(factory.ledgers.isEmpty()); + Assert.assertEquals(factory.ledgers.get(managedLedger2.getName()).join(), managedLedger2); + + // cleanup. + managedLedger2.close(); + } + + private void waitManagedLedgerStateEquals(ManagedLedgerImpl managedLedger, ManagedLedgerImpl.State expectedStat){ + Awaitility.await().untilAsserted(() -> + Assert.assertTrue(managedLedger.getState() == expectedStat)); + } + } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerInfoMetadataTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerInfoMetadataTest.java index 7ddf6541c9a39..6e1f447225e53 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerInfoMetadataTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerInfoMetadataTest.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.UUID; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.MetadataCompressionConfig; import org.apache.bookkeeper.mledger.offload.OffloadUtils; import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.commons.lang3.RandomUtils; @@ -33,6 +34,8 @@ import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; /** * ManagedLedgerInfo metadata test. @@ -53,11 +56,9 @@ private Object[][] compressionTypeProvider() { }; } - @Test(dataProvider = "compressionTypeProvider") - public void testEncodeAndDecode(String compressionType) throws IOException { - long ledgerId = 10000; + private MLDataFormats.ManagedLedgerInfo.Builder generateManagedLedgerInfo(long ledgerId, int ledgerInfoNumber) { List ledgerInfoList = new ArrayList<>(); - for (int i = 0; i < 100; i++) { + for (int i = 0; i < ledgerInfoNumber; i++) { MLDataFormats.ManagedLedgerInfo.LedgerInfo.Builder builder = MLDataFormats.ManagedLedgerInfo.LedgerInfo.newBuilder(); builder.setLedgerId(ledgerId); builder.setEntries(RandomUtils.nextInt()); @@ -84,13 +85,18 @@ public void testEncodeAndDecode(String compressionType) throws IOException { ledgerId ++; } - MLDataFormats.ManagedLedgerInfo managedLedgerInfo = MLDataFormats.ManagedLedgerInfo.newBuilder() - .addAllLedgerInfo(ledgerInfoList) - .build(); + return MLDataFormats.ManagedLedgerInfo.newBuilder() + .addAllLedgerInfo(ledgerInfoList); + } + + @Test(dataProvider = "compressionTypeProvider") + public void testEncodeAndDecode(String compressionType) throws IOException { + long ledgerId = 10000; + MLDataFormats.ManagedLedgerInfo managedLedgerInfo = generateManagedLedgerInfo(ledgerId,100).build(); MetaStoreImpl metaStore; try { - metaStore = new MetaStoreImpl(null, null, compressionType, null); + metaStore = new MetaStoreImpl(null, null, new MetadataCompressionConfig(compressionType), null); if ("INVALID_TYPE".equals(compressionType)) { Assert.fail("The managedLedgerInfo compression type is invalid, should fail."); } @@ -126,4 +132,45 @@ public void testParseEmptyData() throws InvalidProtocolBufferException { Assert.assertEquals(managedLedgerInfo.toString(), ""); } + @Test(dataProvider = "compressionTypeProvider") + public void testCompressionThreshold(String compressionType) { + long ledgerId = 10000; + int compressThreshold = 512; + + // should not compress + MLDataFormats.ManagedLedgerInfo smallInfo = generateManagedLedgerInfo(ledgerId, 0).build(); + assertTrue(smallInfo.getSerializedSize() < compressThreshold); + + // should compress + MLDataFormats.ManagedLedgerInfo bigInfo = generateManagedLedgerInfo(ledgerId, 1000).build(); + assertTrue(bigInfo.getSerializedSize() > compressThreshold); + + MLDataFormats.ManagedLedgerInfo managedLedgerInfo = generateManagedLedgerInfo(ledgerId,100).build(); + + MetaStoreImpl metaStore; + try { + MetadataCompressionConfig metadataCompressionConfig = + new MetadataCompressionConfig(compressionType, compressThreshold); + metaStore = new MetaStoreImpl(null, null, metadataCompressionConfig, null); + if ("INVALID_TYPE".equals(compressionType)) { + Assert.fail("The managedLedgerInfo compression type is invalid, should fail."); + } + } catch (Exception e) { + if ("INVALID_TYPE".equals(compressionType)) { + Assert.assertEquals(e.getClass(), IllegalArgumentException.class); + Assert.assertEquals( + "No enum constant org.apache.bookkeeper.mledger.proto.MLDataFormats.CompressionType." + + compressionType, e.getMessage()); + return; + } else { + throw e; + } + } + + byte[] compressionBytes = metaStore.compressLedgerInfo(smallInfo); + assertEquals(compressionBytes.length, smallInfo.getSerializedSize()); + + byte[] compressionBytesBig = metaStore.compressLedgerInfo(bigInfo); + assertTrue(compressionBytesBig.length !=smallInfo.getSerializedSize()); + } } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanTest.java index 2505db6ec55d7..5f6bd0b7ae64d 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanTest.java @@ -77,10 +77,12 @@ public void simple() throws Exception { assertEquals(mbean.getAddEntryWithReplicasBytesRate(), 0.0); assertEquals(mbean.getAddEntryMessagesRate(), 0.0); assertEquals(mbean.getAddEntrySucceed(), 0); + assertEquals(mbean.getAddEntrySucceedTotal(), 0); assertEquals(mbean.getAddEntryErrors(), 0); assertEquals(mbean.getReadEntriesBytesRate(), 0.0); assertEquals(mbean.getReadEntriesRate(), 0.0); assertEquals(mbean.getReadEntriesSucceeded(), 0); + assertEquals(mbean.getReadEntriesSucceededTotal(), 0); assertEquals(mbean.getReadEntriesErrors(), 0); assertEquals(mbean.getMarkDeleteRate(), 0.0); @@ -105,10 +107,12 @@ public void simple() throws Exception { assertEquals(mbean.getAddEntryWithReplicasBytesRate(), 1600.0); assertEquals(mbean.getAddEntryMessagesRate(), 2.0); assertEquals(mbean.getAddEntrySucceed(), 2); + assertEquals(mbean.getAddEntrySucceedTotal(), 2); assertEquals(mbean.getAddEntryErrors(), 0); assertEquals(mbean.getReadEntriesBytesRate(), 0.0); assertEquals(mbean.getReadEntriesRate(), 0.0); assertEquals(mbean.getReadEntriesSucceeded(), 0); + assertEquals(mbean.getReadEntriesSucceededTotal(), 0); assertEquals(mbean.getReadEntriesErrors(), 0); assertTrue(mbean.getMarkDeleteRate() > 0.0); @@ -134,10 +138,14 @@ public void simple() throws Exception { assertEquals(mbean.getReadEntriesBytesRate(), 600.0); assertEquals(mbean.getReadEntriesRate(), 1.0); assertEquals(mbean.getReadEntriesSucceeded(), 1); + assertEquals(mbean.getReadEntriesSucceededTotal(), 1); assertEquals(mbean.getReadEntriesErrors(), 0); assertEquals(mbean.getNumberOfMessagesInBacklog(), 1); assertEquals(mbean.getMarkDeleteRate(), 0.0); + assertEquals(mbean.getAddEntrySucceed(), 0); + assertEquals(mbean.getAddEntrySucceedTotal(), 2); + factory.shutdown(); } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTerminationTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTerminationTest.java index 2150e80b29593..11feb5a41cf08 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTerminationTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTerminationTest.java @@ -31,6 +31,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerTerminatedException; import org.apache.bookkeeper.mledger.ManagedLedgerException.NoMoreEntriesToReadException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.test.MockedBookKeeperTestCase; import org.testng.annotations.Test; @@ -141,7 +142,7 @@ public void terminateWithNonDurableCursor() throws Exception { assertTrue(ledger.isTerminated()); assertEquals(lastPosition, p1); - ManagedCursor c1 = ledger.newNonDurableCursor(PositionImpl.EARLIEST); + ManagedCursor c1 = ledger.newNonDurableCursor(PositionFactory.EARLIEST); List entries = c1.readEntries(10); assertEquals(entries.size(), 2); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index 70ddbb9998fd8..83a6c771513a9 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -21,7 +21,9 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; @@ -34,6 +36,7 @@ import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertSame; @@ -55,17 +58,21 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.NavigableMap; import java.util.Optional; +import java.util.Queue; import java.util.Set; import java.util.UUID; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.FutureTask; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; @@ -90,6 +97,8 @@ import org.apache.bookkeeper.client.api.LedgerEntries; import org.apache.bookkeeper.client.api.LedgerMetadata; import org.apache.bookkeeper.client.api.ReadHandle; +import org.apache.bookkeeper.common.util.BoundedScheduledExecutorService; +import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback; @@ -114,6 +123,8 @@ import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig; import org.apache.bookkeeper.mledger.ManagedLedgerFactoryMXBean; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionBound; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.VoidCallback; import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback; import org.apache.bookkeeper.mledger.impl.cache.EntryCache; @@ -124,6 +135,7 @@ import org.apache.bookkeeper.mledger.util.Futures; import org.apache.bookkeeper.test.MockedBookKeeperTestCase; import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.lang3.mutable.MutableObject; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition; @@ -135,6 +147,8 @@ import org.apache.pulsar.metadata.api.extended.SessionEvent; import org.apache.pulsar.metadata.impl.FaultInjectionMetadataStore; import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; +import org.eclipse.jetty.util.BlockingArrayQueue; import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.DataProvider; @@ -188,10 +202,10 @@ private DeleteLedgerInfo makeDelayIfDoLedgerDelete(LedgerHandle ledger, final At bkc.asyncDeleteLedger(ledgerId, originalCb, ctx); } else { deleteLedgerInfo.hasCalled = true; - new Thread(() -> { + cachedExecutor.submit(() -> { Awaitility.await().atMost(Duration.ofSeconds(60)).until(signal::get); bkc.asyncDeleteLedger(ledgerId, cb, ctx); - }).start(); + }); } return null; }).when(spyBookKeeper).asyncDeleteLedger(any(long.class), any(AsyncCallback.DeleteCallback.class), any()); @@ -208,6 +222,7 @@ private DeleteLedgerInfo makeDelayIfDoLedgerDelete(LedgerHandle ledger, final At public void testLedgerInfoMetaCorrectIfAddEntryTimeOut() throws Exception { String mlName = "testLedgerInfoMetaCorrectIfAddEntryTimeOut"; BookKeeper spyBookKeeper = spy(bkc); + @Cleanup("shutdown") ManagedLedgerFactoryImpl factory = new ManagedLedgerFactoryImpl(metadataStore, spyBookKeeper); ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open(mlName); @@ -586,7 +601,7 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { fail(exception.getMessage()); } - }, cursor, PositionImpl.LATEST); + }, cursor, PositionFactory.LATEST); } @Override @@ -632,8 +647,8 @@ public void spanningMultipleLedgers() throws Exception { assertEquals(entries.size(), 11); assertFalse(cursor.hasMoreEntries()); - PositionImpl first = (PositionImpl) entries.get(0).getPosition(); - PositionImpl last = (PositionImpl) entries.get(entries.size() - 1).getPosition(); + Position first = entries.get(0).getPosition(); + Position last = entries.get(entries.size() - 1).getPosition(); entries.forEach(Entry::release); log.info("First={} Last={}", first, last); @@ -657,8 +672,8 @@ public void testStartReadOperationOnLedgerWithEmptyLedgers() throws ManagedLedge LedgerInfo ledgerInfo = ledgers.firstEntry().getValue(); ledgers.clear(); ManagedCursor c1 = ledger.openCursor("c1"); - PositionImpl position = new PositionImpl(ledgerInfo.getLedgerId(), 0); - PositionImpl maxPosition = new PositionImpl(ledgerInfo.getLedgerId(), 99); + Position position = PositionFactory.create(ledgerInfo.getLedgerId(), 0); + Position maxPosition = PositionFactory.create(ledgerInfo.getLedgerId(), 99); OpReadEntry opReadEntry = OpReadEntry.create((ManagedCursorImpl) c1, position, 20, new ReadEntriesCallback() { @@ -699,8 +714,8 @@ public void spanningMultipleLedgersWithSize() throws Exception { assertEquals(entries.size(), 3); assertFalse(cursor.hasMoreEntries()); - PositionImpl first = (PositionImpl) entries.get(0).getPosition(); - PositionImpl last = (PositionImpl) entries.get(entries.size() - 1).getPosition(); + Position first = entries.get(0).getPosition(); + Position last = entries.get(entries.size() - 1).getPosition(); entries.forEach(Entry::release); // Read again, from next ledger id @@ -1110,9 +1125,13 @@ public void testTrimmer() throws Exception { cursor.markDelete(lastPosition); - while (ledger.getNumberOfEntries() != 2) { - Thread.sleep(10); - } + Awaitility.await().untilAsserted(() -> { + // The number of entries in the ledger should not contain the entry in the mark delete position. + // last position is the position of entry-3. + // cursor.markDelete(lastPosition); + // only entry-4 is left in the ledger. + assertEquals(ledger.getNumberOfEntries(), 1); + }); } @Test(timeOut = 20000) @@ -1315,7 +1334,7 @@ public void closeLedgerWithError() throws Exception { public void deleteWithErrors1() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger"); - PositionImpl position = (PositionImpl) ledger.addEntry("dummy-entry-1".getBytes(Encoding)); + Position position = ledger.addEntry("dummy-entry-1".getBytes(Encoding)); assertEquals(ledger.getNumberOfEntries(), 1); // Force delete a ledger and test that deleting the ML still happens @@ -1685,7 +1704,7 @@ public void previousPosition() throws Exception { Position p0 = cursor.getMarkDeletedPosition(); // This is expected because p0 is already an "invalid" position (since no entry has been mark-deleted yet) - assertEquals(ledger.getPreviousPosition((PositionImpl) p0), p0); + assertEquals(ledger.getPreviousPosition(p0), p0); // Force to close an empty ledger ledger.close(); @@ -1697,8 +1716,8 @@ public void previousPosition() throws Exception { ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2)); - PositionImpl pBeforeWriting = ledger.getLastPosition(); - PositionImpl p1 = (PositionImpl) ledger.addEntry("entry".getBytes()); + Position pBeforeWriting = ledger.getLastPosition(); + Position p1 = ledger.addEntry("entry".getBytes()); ledger.close(); ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", @@ -1708,9 +1727,9 @@ public void previousPosition() throws Exception { Position p4 = ledger.addEntry("entry".getBytes()); assertEquals(ledger.getPreviousPosition(p1), pBeforeWriting); - assertEquals(ledger.getPreviousPosition((PositionImpl) p2), p1); - assertEquals(ledger.getPreviousPosition((PositionImpl) p3), p2); - assertEquals(ledger.getPreviousPosition((PositionImpl) p4), p3); + assertEquals(ledger.getPreviousPosition(p2), p1); + assertEquals(ledger.getPreviousPosition(p3), p2); + assertEquals(ledger.getPreviousPosition(p4), p3); } /** @@ -1768,10 +1787,10 @@ public void invalidateConsumedEntriesFromCache() throws Exception { ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1"); ManagedCursorImpl c2 = (ManagedCursorImpl) ledger.openCursor("c2"); - PositionImpl p1 = (PositionImpl) ledger.addEntry("entry-1".getBytes()); - PositionImpl p2 = (PositionImpl) ledger.addEntry("entry-2".getBytes()); - PositionImpl p3 = (PositionImpl) ledger.addEntry("entry-3".getBytes()); - PositionImpl p4 = (PositionImpl) ledger.addEntry("entry-4".getBytes()); + Position p1 = ledger.addEntry("entry-1".getBytes()); + Position p2 = ledger.addEntry("entry-2".getBytes()); + Position p3 = ledger.addEntry("entry-3".getBytes()); + Position p4 = ledger.addEntry("entry-4".getBytes()); assertEquals(entryCache.getSize(), 7 * 4); assertEquals(cacheManager.getSize(), entryCache.getSize()); @@ -1818,10 +1837,10 @@ public void invalidateEntriesFromCacheByMarkDeletePosition() throws Exception { ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1"); ManagedCursorImpl c2 = (ManagedCursorImpl) ledger.openCursor("c2"); - PositionImpl p1 = (PositionImpl) ledger.addEntry("entry-1".getBytes()); - PositionImpl p2 = (PositionImpl) ledger.addEntry("entry-2".getBytes()); - PositionImpl p3 = (PositionImpl) ledger.addEntry("entry-3".getBytes()); - PositionImpl p4 = (PositionImpl) ledger.addEntry("entry-4".getBytes()); + Position p1 = ledger.addEntry("entry-1".getBytes()); + Position p2 = ledger.addEntry("entry-2".getBytes()); + Position p3 = ledger.addEntry("entry-3".getBytes()); + Position p4 = ledger.addEntry("entry-4".getBytes()); assertEquals(entryCache.getSize(), 7 * 4); assertEquals(cacheManager.getSize(), entryCache.getSize()); @@ -2084,10 +2103,10 @@ public void totalSizeTest() throws Exception { assertEquals(ledger.getTotalSize(), 8); - PositionImpl p2 = (PositionImpl) ledger.addEntry(new byte[12], 2, 5); + Position p2 = ledger.addEntry(new byte[12], 2, 5); assertEquals(ledger.getTotalSize(), 13); - c1.markDelete(new PositionImpl(p2.getLedgerId(), -1)); + c1.markDelete(PositionFactory.create(p2.getLedgerId(), -1)); // Wait for background trimming Thread.sleep(400); @@ -2160,7 +2179,9 @@ public void testNoRolloverIfNoMetadataSession() throws Exception { ledger.addEntry("data".getBytes()); // After the re-establishment, we'll be creating new ledgers - assertEquals(ledger.getLedgersInfoAsList().size(), 3); + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + assertEquals(ledger.getLedgersInfoAsList().size(), 4); + }); } @Test @@ -2330,7 +2351,7 @@ public void testRetention0WithEmptyLedger() throws Exception { ml.deleteCursor(c1.getName()); ml.internalTrimConsumedLedgers(CompletableFuture.completedFuture(null)); - assertTrue(ml.getFirstPosition().ledgerId <= ml.lastConfirmedEntry.ledgerId); + assertTrue(ml.getFirstPosition().getLedgerId() <= ml.lastConfirmedEntry.getLedgerId()); ml.close(); } @@ -2356,8 +2377,8 @@ public void testRetention0WithEmptyLedgerWithoutCursors() throws Exception { ml = (ManagedLedgerImpl) factory.open("deletion_after_retention_test_ledger", config); ml.internalTrimConsumedLedgers(CompletableFuture.completedFuture(null)); - assertTrue(ml.getFirstPosition().ledgerId <= ml.lastConfirmedEntry.ledgerId); - assertFalse(ml.getLedgersInfo().containsKey(ml.lastConfirmedEntry.ledgerId), + assertTrue(ml.getFirstPosition().getLedgerId() <= ml.lastConfirmedEntry.getLedgerId()); + assertFalse(ml.getLedgersInfo().containsKey(ml.lastConfirmedEntry.getLedgerId()), "the ledger at lastConfirmedEntry has not been trimmed!"); ml.close(); } @@ -2426,7 +2447,7 @@ public void testRetentionSize() throws Exception { Awaitility.await().untilAsserted(() -> { assertTrue(ml.getTotalSize() <= retentionSizeInMB * 1024 * 1024); - assertEquals(ml.getLedgersInfoAsList().size(), 5); + assertEquals(ml.getLedgersInfoAsList().size(), 6); }); } @@ -2457,19 +2478,18 @@ public void testTimestampOnWorkingLedger() throws Exception { ml.addEntry("msg02".getBytes()); + // reopen a new ml2 ml.close(); - // Thread.sleep(1000); - iter = ml.getLedgersInfoAsList().iterator(); - ts = -1; - while (iter.hasNext()) { - LedgerInfo i = iter.next(); - if (iter.hasNext()) { - assertTrue(ts <= i.getTimestamp(), i.toString()); - ts = i.getTimestamp(); - } else { - assertTrue(i.getTimestamp() > 0, "well closed LedgerInfo should set a timestamp > 0"); - } - } + ManagedLedgerImpl ml2 = (ManagedLedgerImpl) factory.open("my_test_ledger", conf); + + List ledgers = ml2.getLedgersInfoAsList(); + // after reopen ledgers will be 2 + 1(new open, not contain any entries) + assertEquals(ledgers.size(), 3); + + // the last closed ledger should be the penultimate one. + LedgerInfo lastClosedLeger = ledgers.get(ledgers.size() - 2); + assertTrue(lastClosedLeger.getTimestamp() > 0, "well closed LedgerInfo should set a timestamp > 0"); + ml2.close(); } @Test @@ -2576,39 +2596,39 @@ public void testGetPositionAfterN() throws Exception { long firstLedger = managedLedger.getLedgersInfo().firstKey(); long secondLedger = managedLedger.getLedgersInfoAsList().get(1).getLedgerId(); - PositionImpl startPosition = new PositionImpl(firstLedger, 0); + Position startPosition = PositionFactory.create(firstLedger, 0); - PositionImpl targetPosition = managedLedger.getPositionAfterN(startPosition, 1, ManagedLedgerImpl.PositionBound.startExcluded); + Position targetPosition = managedLedger.getPositionAfterN(startPosition, 1, PositionBound.startExcluded); assertEquals(targetPosition.getLedgerId(), firstLedger); assertEquals(targetPosition.getEntryId(), 1); - targetPosition = managedLedger.getPositionAfterN(startPosition, 4, ManagedLedgerImpl.PositionBound.startExcluded); + targetPosition = managedLedger.getPositionAfterN(startPosition, 4, PositionBound.startExcluded); assertEquals(targetPosition.getLedgerId(), firstLedger); assertEquals(targetPosition.getEntryId(), 4); // test for expiry situation - PositionImpl searchPosition = managedLedger.getNextValidPosition((PositionImpl) managedCursor.getMarkDeletedPosition()); + Position searchPosition = managedLedger.getNextValidPosition(managedCursor.getMarkDeletedPosition()); long length = managedCursor.getNumberOfEntriesInStorage(); // return the last confirm entry position if searchPosition is exceed the last confirm entry - targetPosition = managedLedger.getPositionAfterN(searchPosition, length, ManagedLedgerImpl.PositionBound.startExcluded); + targetPosition = managedLedger.getPositionAfterN(searchPosition, length, PositionBound.startExcluded); log.info("Target position is {}", targetPosition); assertEquals(targetPosition.getLedgerId(), secondLedger); assertEquals(targetPosition.getEntryId(), 4); // test for n > NumberOfEntriesInStorage - searchPosition = new PositionImpl(secondLedger, 0); - targetPosition = managedLedger.getPositionAfterN(searchPosition, 100, ManagedLedgerImpl.PositionBound.startIncluded); + searchPosition = PositionFactory.create(secondLedger, 0); + targetPosition = managedLedger.getPositionAfterN(searchPosition, 100, PositionBound.startIncluded); assertEquals(targetPosition.getLedgerId(), secondLedger); assertEquals(targetPosition.getEntryId(), 4); // test for startPosition > current ledger - searchPosition = new PositionImpl(999, 0); - targetPosition = managedLedger.getPositionAfterN(searchPosition, 0, ManagedLedgerImpl.PositionBound.startIncluded); + searchPosition = PositionFactory.create(999, 0); + targetPosition = managedLedger.getPositionAfterN(searchPosition, 0, PositionBound.startIncluded); assertEquals(targetPosition.getLedgerId(), secondLedger); assertEquals(targetPosition.getEntryId(), 4); - searchPosition = new PositionImpl(999, 0); - targetPosition = managedLedger.getPositionAfterN(searchPosition, 10, ManagedLedgerImpl.PositionBound.startExcluded); + searchPosition = PositionFactory.create(999, 0); + targetPosition = managedLedger.getPositionAfterN(searchPosition, 10, PositionBound.startExcluded); assertEquals(targetPosition.getLedgerId(), secondLedger); assertEquals(targetPosition.getEntryId(), 4); } @@ -2626,10 +2646,10 @@ public void testGetNumberOfEntriesInStorage() throws Exception { managedLedger.addEntry(("entry-" + i).getBytes(Encoding)); } - //trigger ledger rollover and wait for the new ledger created - Field stateUpdater = ManagedLedgerImpl.class.getDeclaredField("state"); - stateUpdater.setAccessible(true); - stateUpdater.set(managedLedger, ManagedLedgerImpl.State.LedgerOpened); + // trigger ledger rollover and wait for the new ledger created + Awaitility.await().untilAsserted(() -> { + assertEquals("LedgerOpened", WhiteboxImpl.getInternalState(managedLedger, "state").toString()); + }); managedLedger.rollCurrentLedgerIfFull(); Awaitility.await().untilAsserted(() -> { assertEquals(managedLedger.getLedgersInfo().size(), 3); @@ -2678,15 +2698,23 @@ public void testGetNextValidPosition() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testGetNextValidPosition", conf); ManagedCursor c1 = ledger.openCursor("c1"); - PositionImpl p1 = (PositionImpl) ledger.addEntry("entry1".getBytes()); - PositionImpl p2 = (PositionImpl) ledger.addEntry("entry2".getBytes()); - PositionImpl p3 = (PositionImpl) ledger.addEntry("entry3".getBytes()); + Position p1 = ledger.addEntry("entry1".getBytes()); + Position p2 = ledger.addEntry("entry2".getBytes()); + Position p3 = ledger.addEntry("entry3".getBytes()); - assertEquals(ledger.getNextValidPosition((PositionImpl) c1.getMarkDeletedPosition()), p1); + assertEquals(ledger.getNextValidPosition(c1.getMarkDeletedPosition()), p1); assertEquals(ledger.getNextValidPosition(p1), p2); - assertEquals(ledger.getNextValidPosition(p3), PositionImpl.get(p3.getLedgerId(), p3.getEntryId() + 1)); - assertEquals(ledger.getNextValidPosition(PositionImpl.get(p3.getLedgerId(), p3.getEntryId() + 1)), PositionImpl.get(p3.getLedgerId(), p3.getEntryId() + 1)); - assertEquals(ledger.getNextValidPosition(PositionImpl.get(p3.getLedgerId() + 1, p3.getEntryId() + 1)), PositionImpl.get(p3.getLedgerId(), p3.getEntryId() + 1)); + Awaitility.await().untilAsserted(() -> { + assertEquals(ledger.getNextValidPosition(p3), PositionFactory.create(p3.getLedgerId() + 1, 0)); + }); + Awaitility.await().untilAsserted(() -> { + assertEquals(ledger.getNextValidPosition(PositionFactory.create(p3.getLedgerId(), p3.getEntryId() + 1)), + PositionFactory.create(p3.getLedgerId() + 1, 0)); + }); + Awaitility.await().untilAsserted(() -> { + assertEquals(ledger.getNextValidPosition(PositionFactory.create(p3.getLedgerId() + 1, p3.getEntryId() + 1)), + PositionFactory.create(p3.getLedgerId() + 1, 0)); + }); } /** @@ -3025,19 +3053,22 @@ public void testConsumerSubscriptionInitializePosition() throws Exception{ String content = "entry" + i; // 5 bytes ledger.addEntry(content.getBytes()); } + Awaitility.await().untilAsserted(() -> { + assertEquals(ledger.currentLedgerSize, 0); + assertEquals(ledger.ledgers.size(), 1); + }); // Open Cursor also adds cursor into activeCursor-container ManagedCursor latestCursor = ledger.openCursor("c1", InitialPosition.Latest); ManagedCursor earliestCursor = ledger.openCursor("c2", InitialPosition.Earliest); // Since getReadPosition returns the next position, we decrease the entryId by 1 - PositionImpl p1 = (PositionImpl) latestCursor.getReadPosition(); - PositionImpl p2 = (PositionImpl) earliestCursor.getReadPosition(); + Position p2 = earliestCursor.getReadPosition(); - Pair latestPositionAndCounter = ledger.getLastPositionAndCounter(); - Pair earliestPositionAndCounter = ledger.getFirstPositionAndCounter(); - - assertEquals(latestPositionAndCounter.getLeft().getNext(), p1); - assertEquals(earliestPositionAndCounter.getLeft().getNext(), p2); + Pair latestPositionAndCounter = ledger.getLastPositionAndCounter(); + Pair earliestPositionAndCounter = ledger.getFirstPositionAndCounter(); + // The read position is the valid next position of the last position instead of the next position. + assertEquals(ledger.getNextValidPosition(latestPositionAndCounter.getLeft()), latestCursor.getReadPosition()); + assertEquals(ledger.getNextValidPosition(earliestPositionAndCounter.getLeft()), p2); assertEquals(latestPositionAndCounter.getRight().longValue(), totalInsertedEntries); assertEquals(earliestPositionAndCounter.getRight().longValue(), totalInsertedEntries - earliestCursor.getNumberOfEntriesInBacklog(false)); @@ -3085,9 +3116,9 @@ public void testManagedLedgerWithCreateLedgerTimeOut() throws Exception { latch.await(config.getMetadataOperationsTimeoutSeconds() + 2, TimeUnit.SECONDS); assertEquals(response.get(), BKException.Code.TimeoutException); - assertTrue(ctxHolder.get() instanceof AtomicBoolean); - AtomicBoolean ledgerCreated = (AtomicBoolean) ctxHolder.get(); - assertFalse(ledgerCreated.get()); + assertTrue(ctxHolder.get() instanceof CompletableFuture); + CompletableFuture ledgerCreateHook = (CompletableFuture) ctxHolder.get(); + assertTrue(ledgerCreateHook.isCompletedExceptionally()); ledger.close(); } @@ -3108,11 +3139,11 @@ public void testManagedLedgerWithReadEntryTimeOut() throws Exception { String ctxStr = "timeoutCtx"; CompletableFuture entriesFuture = new CompletableFuture<>(); ReadHandle ledgerHandle = mock(ReadHandle.class); - doReturn(entriesFuture).when(ledgerHandle).readAsync(PositionImpl.EARLIEST.getLedgerId(), - PositionImpl.EARLIEST.getEntryId()); + doReturn(entriesFuture).when(ledgerHandle).readAsync(PositionFactory.EARLIEST.getLedgerId(), + PositionFactory.EARLIEST.getEntryId()); // (1) test read-timeout for: ManagedLedger.asyncReadEntry(..) - ledger.asyncReadEntry(ledgerHandle, PositionImpl.EARLIEST, new ReadEntryCallback() { + ledger.asyncReadEntry(ledgerHandle, PositionFactory.EARLIEST, new ReadEntryCallback() { @Override public void readEntryComplete(Entry entry, Object ctx) { responseException1.set(null); @@ -3132,8 +3163,8 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { // (2) test read-timeout for: ManagedLedger.asyncReadEntry(..) AtomicReference responseException2 = new AtomicReference<>(); - PositionImpl readPositionRef = PositionImpl.EARLIEST; - ManagedCursorImpl cursor = new ManagedCursorImpl(bk, config, ledger, "cursor1"); + Position readPositionRef = PositionFactory.EARLIEST; + ManagedCursorImpl cursor = new ManagedCursorImpl(bk, ledger, "cursor1"); OpReadEntry opReadEntry = OpReadEntry.create(cursor, readPositionRef, 1, new ReadEntriesCallback() { @Override @@ -3146,8 +3177,8 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { responseException2.set(exception); } - }, null, PositionImpl.LATEST, null); - ledger.asyncReadEntry(ledgerHandle, PositionImpl.EARLIEST.getEntryId(), PositionImpl.EARLIEST.getEntryId(), + }, null, PositionFactory.LATEST, null); + ledger.asyncReadEntry(ledgerHandle, PositionFactory.EARLIEST.getEntryId(), PositionFactory.EARLIEST.getEntryId(), opReadEntry, ctxStr); retryStrategically((test) -> { return responseException2.get() != null; @@ -3159,6 +3190,55 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { ledger.close(); } + @Test + public void testAddEntryResponseTimeout() throws Exception { + // Create ML with feature Add Entry Timeout Check. + final ManagedLedgerConfig config = new ManagedLedgerConfig().setAddEntryTimeoutSeconds(2); + final ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("ml1", config); + final ManagedCursor cursor = ledger.openCursor("c1"); + final CollectCtxAddEntryCallback collectCtxAddEntryCallback = new CollectCtxAddEntryCallback(); + + // Insert a response delay. + bkc.addEntryResponseDelay(8, TimeUnit.SECONDS); + + // Add two entries. + final byte[] msg1 = new byte[]{1}; + final byte[] msg2 = new byte[]{2}; + int ctx1 = 1; + int ctx2 = 2; + ledger.asyncAddEntry(msg1, collectCtxAddEntryCallback, ctx1); + ledger.asyncAddEntry(msg2, collectCtxAddEntryCallback, ctx2); + // Verify all write requests are completed. + Awaitility.await().untilAsserted(() -> { + assertEquals(collectCtxAddEntryCallback.addCompleteCtxList, Arrays.asList(1, 2)); + }); + Entry entry1 = cursor.readEntries(1).get(0); + assertEquals(entry1.getData(), msg1); + entry1.release(); + Entry entry2 = cursor.readEntries(1).get(0); + assertEquals(entry2.getData(), msg2); + entry2.release(); + + // cleanup. + factory.delete(ledger.name); + } + + private static class CollectCtxAddEntryCallback implements AddEntryCallback { + + public List addCompleteCtxList = new BlockingArrayQueue<>(); + public List addFailedCtxList = new BlockingArrayQueue<>(); + + @Override + public void addComplete(Position position, ByteBuf entryData, Object ctx) { + addCompleteCtxList.add(ctx); + } + + @Override + public void addFailed(ManagedLedgerException exception, Object ctx) { + addFailedCtxList.add(ctx); + } + } + /** * It verifies that if bk-client doesn't complete the add-entry in given time out then broker is resilient enough * to create new ledger and add entry successfully. @@ -3234,7 +3314,8 @@ public void avoidUseSameOpAddEntryBetweenDifferentLedger() throws Exception { List oldOps = new ArrayList<>(); for (int i = 0; i < 10; i++) { - OpAddEntry op = OpAddEntry.createNoRetainBuffer(ledger, ByteBufAllocator.DEFAULT.buffer(128).retain(), null, null); + OpAddEntry op = OpAddEntry.createNoRetainBuffer(ledger, + ByteBufAllocator.DEFAULT.buffer(128).retain(), null, null, new AtomicBoolean()); if (i > 4) { op.setLedger(mock(LedgerHandle.class)); } @@ -3389,7 +3470,7 @@ public void openCursorFailed(ManagedLedgerException exception, Object ctx) { @Override public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { } - }, checkOwnershipFlag ? () -> true : null, null); + }, checkOwnershipFlag ? () -> CompletableFuture.completedFuture(true) : null, null); latch.await(); } @@ -3449,7 +3530,6 @@ public void testManagedLedgerRollOverIfFull() throws Exception { ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setRetentionTime(1, TimeUnit.SECONDS); config.setMaxEntriesPerLedger(2); - config.setMinimumRolloverTime(1, TimeUnit.MILLISECONDS); config.setMaximumRolloverTime(500, TimeUnit.MILLISECONDS); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("test_managedLedger_rollOver", config); @@ -3461,7 +3541,8 @@ public void testManagedLedgerRollOverIfFull() throws Exception { ledger.addEntry(new byte[1024 * 1024]); } - Awaitility.await().untilAsserted(() -> Assert.assertEquals(ledger.getLedgersInfoAsList().size(), msgNum / 2)); + Awaitility.await().untilAsserted(() -> Assert.assertEquals(ledger.getLedgersInfoAsList().size(), + msgNum / 2 + 1)); List entries = cursor.readEntries(msgNum); Assert.assertEquals(msgNum, entries.size()); @@ -3476,6 +3557,9 @@ public void testManagedLedgerRollOverIfFull() throws Exception { stateUpdater.setAccessible(true); stateUpdater.set(ledger, ManagedLedgerImpl.State.LedgerOpened); ledger.rollCurrentLedgerIfFull(); + CompletableFuture completableFuture = new CompletableFuture<>(); + ledger.trimConsumedLedgersInBackground(completableFuture); + completableFuture.get(); Awaitility.await().untilAsserted(() -> Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 1)); Awaitility.await().untilAsserted(() -> Assert.assertEquals(ledger.getTotalSize(), 0)); } @@ -3495,7 +3579,7 @@ public void testLedgerReachMaximumRolloverTime() throws Exception { .until(() -> firstLedgerId != ml.addEntry("test".getBytes()).getLedgerId()); } - @Test + @Test(groups = "flaky") public void testLedgerNotRolloverWithoutOpenState() throws Exception { ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(2); @@ -3600,7 +3684,7 @@ public void testAsyncTruncateLedgerSlowestCursor() throws Exception { ManagedLedgerImpl ledger2 = (ManagedLedgerImpl)factory.open("truncate_ledger", config); ledger2.addEntry("test-entry-2".getBytes(Encoding)); ManagedCursor cursor3 = ledger2.openCursor("test-cursor"); - cursor3.resetCursor(new PositionImpl(ledger2.getLastPosition())); + cursor3.resetCursor(PositionFactory.create(ledger2.getLastPosition())); CompletableFuture future = ledger2.asyncTruncate(); future.get(); @@ -3641,8 +3725,12 @@ public void testInvalidateReadHandleWhenDeleteLedger() throws Exception { } List entryList = cursor.readEntries(3); assertEquals(entryList.size(), 3); - assertEquals(ledger.ledgers.size(), 3); - assertEquals(ledger.ledgerCache.size(), 2); + Awaitility.await().untilAsserted(() -> { + log.error("ledger.ledgerCache.size() : " + ledger.ledgerCache.size()); + assertEquals(ledger.ledgerCache.size(), 3); + assertEquals(ledger.ledgers.size(), 4); + }); + cursor.clearBacklog(); cursor2.clearBacklog(); ledger.trimConsumedLedgersInBackground(Futures.NULL_PROMISE); @@ -3671,15 +3759,15 @@ public void testLockReleaseWhenTrimLedger() throws Exception { } List entryList = cursor.readEntries(entries); assertEquals(entryList.size(), entries); - assertEquals(ledger.ledgers.size(), entries); - assertEquals(ledger.ledgerCache.size(), entries - 1); + assertEquals(ledger.ledgers.size() - 1, entries); + assertEquals(ledger.ledgerCache.size() - 1, entries - 1); cursor.clearBacklog(); ledger.trimConsumedLedgersInBackground(Futures.NULL_PROMISE); ledger.trimConsumedLedgersInBackground(Futures.NULL_PROMISE); // Cleanup fails because ManagedLedgerNotFoundException is thrown Awaitility.await().untilAsserted(() -> { - assertEquals(ledger.ledgers.size(), entries); - assertEquals(ledger.ledgerCache.size(), entries - 1); + assertEquals(ledger.ledgers.size() - 1, entries); + assertEquals(ledger.ledgerCache.size() - 1, entries - 1); }); // The lock is released even if an ManagedLedgerNotFoundException occurs, so it can be called repeatedly Awaitility.await().untilAsserted(() -> @@ -3705,13 +3793,13 @@ public void testInvalidateReadHandleWhenConsumed() throws Exception { } List entryList = cursor.readEntries(3); assertEquals(entryList.size(), 3); - assertEquals(ledger.ledgers.size(), 3); - assertEquals(ledger.ledgerCache.size(), 2); + assertEquals(ledger.ledgers.size(), 4); + assertEquals(ledger.ledgerCache.size(), 3); cursor.clearBacklog(); cursor2.clearBacklog(); ledger.trimConsumedLedgersInBackground(Futures.NULL_PROMISE); Awaitility.await().untilAsserted(() -> { - assertEquals(ledger.ledgers.size(), 3); + assertEquals(ledger.ledgers.size(), 4); assertEquals(ledger.ledgerCache.size(), 0); }); @@ -3719,11 +3807,11 @@ public void testInvalidateReadHandleWhenConsumed() throws Exception { ManagedCursor cursor3 = ledger.openCursor("test-cursor3", InitialPosition.Earliest); entryList = cursor3.readEntries(3); assertEquals(entryList.size(), 3); - assertEquals(ledger.ledgerCache.size(), 2); + assertEquals(ledger.ledgerCache.size(), 3); cursor3.clearBacklog(); ledger.trimConsumedLedgersInBackground(Futures.NULL_PROMISE); Awaitility.await().untilAsserted(() -> { - assertEquals(ledger.ledgers.size(), 3); + assertEquals(ledger.ledgers.size(), 4); assertEquals(ledger.ledgerCache.size(), 0); }); @@ -3764,7 +3852,7 @@ public void testDoNotGetOffloadPoliciesMultipleTimesWhenTrimLedgers() throws Exc config.setLedgerOffloader(ledgerOffloader); ledger.internalTrimConsumedLedgers(Futures.NULL_PROMISE); - verify(ledgerOffloader, times(1)).getOffloadPolicies(); + verify(ledgerOffloader, times(1)).isAppendable(); } @Test(timeOut = 30000) @@ -3772,8 +3860,8 @@ public void testReadOtherManagedLedgersEntry() throws Exception { ManagedLedgerImpl managedLedgerA = (ManagedLedgerImpl) factory.open("my_test_ledger_a"); ManagedLedgerImpl managedLedgerB = (ManagedLedgerImpl) factory.open("my_test_ledger_b"); - PositionImpl pa = (PositionImpl) managedLedgerA.addEntry("dummy-entry-a".getBytes(Encoding)); - PositionImpl pb = (PositionImpl) managedLedgerB.addEntry("dummy-entry-b".getBytes(Encoding)); + Position pa = managedLedgerA.addEntry("dummy-entry-a".getBytes(Encoding)); + Position pb = managedLedgerB.addEntry("dummy-entry-b".getBytes(Encoding)); // read managedLegerA's entry using managedLedgerA CompletableFuture completableFutureA = new CompletableFuture<>(); @@ -3854,6 +3942,7 @@ public void testCancellationOfScheduledTasks() throws Exception { public void testInactiveLedgerRollOver() throws Exception { int inactiveLedgerRollOverTimeMs = 5; ManagedLedgerFactoryConfig factoryConf = new ManagedLedgerFactoryConfig(); + @Cleanup("shutdown") ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setInactiveLedgerRollOverTime(inactiveLedgerRollOverTimeMs, TimeUnit.MILLISECONDS); @@ -3885,11 +3974,59 @@ public void testInactiveLedgerRollOver() throws Exception { List ledgers = ledger.getLedgersInfoAsList(); assertEquals(ledgers.size(), totalAddEntries); ledger.close(); - factory.shutdown(); + } + + @Test + public void testDontRollOverEmptyInactiveLedgers() throws Exception { + int inactiveLedgerRollOverTimeMs = 5; + ManagedLedgerFactoryConfig factoryConf = new ManagedLedgerFactoryConfig(); + @Cleanup("shutdown") + ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc); + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setInactiveLedgerRollOverTime(inactiveLedgerRollOverTimeMs, TimeUnit.MILLISECONDS); + ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("rollover_inactive", config); + ManagedCursor cursor = ledger.openCursor("c1"); + + long ledgerId = ledger.currentLedger.getId(); + + Thread.sleep(inactiveLedgerRollOverTimeMs * 5); + ledger.checkInactiveLedgerAndRollOver(); + + Thread.sleep(inactiveLedgerRollOverTimeMs * 5); + ledger.checkInactiveLedgerAndRollOver(); + + assertEquals(ledger.currentLedger.getId(), ledgerId); + + ledger.close(); + } + + @Test + public void testDontRollOverInactiveLedgersWhenMetadataServiceInvalid() throws Exception { + int inactiveLedgerRollOverTimeMs = 5; + @Cleanup("shutdown") + ManagedLedgerFactoryImpl factory = spy(new ManagedLedgerFactoryImpl(metadataStore, bkc)); + // mock metadata service invalid + when(factory.isMetadataServiceAvailable()).thenReturn(false); + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setInactiveLedgerRollOverTime(inactiveLedgerRollOverTimeMs, TimeUnit.MILLISECONDS); + ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("rollover_inactive", config); + + long ledgerId = ledger.currentLedger.getId(); + + Thread.sleep(inactiveLedgerRollOverTimeMs * 5); + ledger.checkInactiveLedgerAndRollOver(); + + Thread.sleep(inactiveLedgerRollOverTimeMs * 5); + ledger.checkInactiveLedgerAndRollOver(); + + assertEquals(ledger.currentLedger.getId(), ledgerId); + + ledger.close(); } @Test public void testOffloadTaskCancelled() throws Exception { + @Cleanup("shutdown") ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(2); @@ -3938,15 +4075,15 @@ public void testGetTheSlowestNonDurationReadPosition() throws Exception { positions.add(ledger.addEntry(("entry-" + i).getBytes(UTF_8))); } - Assert.assertEquals(ledger.getTheSlowestNonDurationReadPosition(), PositionImpl.LATEST); + Assert.assertEquals(ledger.getTheSlowestNonDurationReadPosition(), PositionFactory.LATEST); - ManagedCursor nonDurableCursor = ledger.newNonDurableCursor(PositionImpl.EARLIEST); + ManagedCursor nonDurableCursor = ledger.newNonDurableCursor(PositionFactory.EARLIEST); Assert.assertEquals(ledger.getTheSlowestNonDurationReadPosition(), positions.get(0)); ledger.deleteCursor(nonDurableCursor.getName()); - Assert.assertEquals(ledger.getTheSlowestNonDurationReadPosition(), PositionImpl.LATEST); + Assert.assertEquals(ledger.getTheSlowestNonDurationReadPosition(), PositionFactory.LATEST); ledger.close(); } @@ -3968,6 +4105,70 @@ public void testGetEnsemblesAsync() throws Exception { Assert.assertFalse(managedLedger.ledgerCache.containsKey(lastLedger)); } + @Test + public void testIsNoMessagesAfterPos() throws Exception { + final byte[] data = new byte[]{1,2,3}; + final String cursorName = "c1"; + final String mlName = UUID.randomUUID().toString().replaceAll("-", ""); + final ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open(mlName); + final ManagedCursor managedCursor = ml.openCursor(cursorName); + + // One ledger. + Position p1 = ml.addEntry(data); + Position p2 = ml.addEntry(data); + Position p3 = ml.addEntry(data); + assertFalse(ml.isNoMessagesAfterPos(p1)); + assertFalse(ml.isNoMessagesAfterPos(p2)); + assertTrue(ml.isNoMessagesAfterPos(p3)); + assertTrue(ml.isNoMessagesAfterPos(PositionFactory.create(p3.getLedgerId(), p3.getEntryId() + 1))); + assertTrue(ml.isNoMessagesAfterPos(PositionFactory.create(p3.getLedgerId() + 1, -1))); + + // More than one ledger. + ml.ledgerClosed(ml.currentLedger); + Position p4 = ml.addEntry(data); + Position p5 = ml.addEntry(data); + Position p6 = ml.addEntry(data); + assertFalse(ml.isNoMessagesAfterPos(p1)); + assertFalse(ml.isNoMessagesAfterPos(p2)); + assertFalse(ml.isNoMessagesAfterPos(p3)); + assertFalse(ml.isNoMessagesAfterPos(p4)); + assertFalse(ml.isNoMessagesAfterPos(p5)); + assertTrue(ml.isNoMessagesAfterPos(p6)); + assertTrue(ml.isNoMessagesAfterPos(PositionFactory.create(p6.getLedgerId(), p6.getEntryId() + 1))); + assertTrue(ml.isNoMessagesAfterPos(PositionFactory.create(p6.getLedgerId() + 1, -1))); + + // Switch ledger and make the entry id of Last confirmed entry is -1; + ml.ledgerClosed(ml.currentLedger); + ml.createLedgerAfterClosed(); + Awaitility.await().untilAsserted(() -> { + assertEquals(ml.currentLedgerEntries, 0); + }); + ml.lastConfirmedEntry = PositionFactory.create(ml.currentLedger.getId(), -1); + assertFalse(ml.isNoMessagesAfterPos(p5)); + assertTrue(ml.isNoMessagesAfterPos(p6)); + assertTrue(ml.isNoMessagesAfterPos(PositionFactory.create(p6.getLedgerId(), p6.getEntryId() + 1))); + assertTrue(ml.isNoMessagesAfterPos(PositionFactory.create(p6.getLedgerId() + 1, -1))); + + // Trim ledgers to make there is no entries in ML. + ml.deleteCursor(cursorName); + CompletableFuture future = new CompletableFuture<>(); + ml.trimConsumedLedgersInBackground(true, future); + future.get(); + assertEquals(ml.ledgers.size(), 1); + assertEquals(ml.lastConfirmedEntry.getEntryId(), -1); + assertTrue(ml.isNoMessagesAfterPos(p1)); + assertTrue(ml.isNoMessagesAfterPos(p2)); + assertTrue(ml.isNoMessagesAfterPos(p3)); + assertTrue(ml.isNoMessagesAfterPos(p4)); + assertTrue(ml.isNoMessagesAfterPos(p5)); + assertTrue(ml.isNoMessagesAfterPos(p6)); + assertTrue(ml.isNoMessagesAfterPos(PositionFactory.create(p6.getLedgerId(), p6.getEntryId() + 1))); + assertTrue(ml.isNoMessagesAfterPos(PositionFactory.create(p6.getLedgerId() + 1, -1))); + + // cleanup. + ml.close(); + } + @Test public void testGetEstimatedBacklogSize() throws Exception { ManagedLedgerConfig config = new ManagedLedgerConfig(); @@ -3980,9 +4181,9 @@ public void testGetEstimatedBacklogSize() throws Exception { positions.add(ledger.addEntry(new byte[1])); } - Assert.assertEquals(ledger.getEstimatedBacklogSize(new PositionImpl(-1, -1)), 10); - Assert.assertEquals(ledger.getEstimatedBacklogSize(((PositionImpl) positions.get(1))), 8); - Assert.assertEquals(ledger.getEstimatedBacklogSize(((PositionImpl) positions.get(9)).getNext()), 0); + Assert.assertEquals(ledger.getEstimatedBacklogSize(PositionFactory.create(-1, -1)), 10); + Assert.assertEquals(ledger.getEstimatedBacklogSize((positions.get(1))), 8); + Assert.assertEquals(ledger.getEstimatedBacklogSize((positions.get(9)).getNext()), 0); ledger.close(); } @@ -4010,4 +4211,168 @@ public void operationFailed(MetaStoreException e) { }); future.join(); } + + @Test + public void testNonDurableCursorCreateForInactiveLedger() throws Exception { + String mlName = "testLedgerInfoMetaCorrectIfAddEntryTimeOut"; + BookKeeper spyBookKeeper = spy(bkc); + ManagedLedgerFactoryImpl factory = new ManagedLedgerFactoryImpl(metadataStore, spyBookKeeper); + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setInactiveLedgerRollOverTime(10, TimeUnit.MILLISECONDS); + ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open(mlName, config); + ml.addEntry("entry".getBytes(UTF_8)); + + MutableBoolean isRolledOver = new MutableBoolean(false); + retryStrategically((test) -> { + if (isRolledOver.booleanValue()) { + return true; + } + isRolledOver.setValue(ml.checkInactiveLedgerAndRollOver()); + return isRolledOver.booleanValue(); + }, 5, 1000); + assertTrue(isRolledOver.booleanValue()); + + Position Position = PositionFactory.create(-1L, -1L); + assertNotNull(ml.newNonDurableCursor(Position)); + } + + /*** + * When a ML tries to create a ledger, it will create a delay task to check if the ledger create request is timeout. + * But we should guarantee that the delay task should be canceled after the ledger create request responded. + */ + @Test + public void testNoOrphanScheduledTasksAfterCloseML() throws Exception { + String mlName = UUID.randomUUID().toString(); + ManagedLedgerFactoryImpl factory = new ManagedLedgerFactoryImpl(metadataStore, bkc); + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setMetadataOperationsTimeoutSeconds(3600); + + // Calculate pending task count. + long pendingTaskCountBefore = calculatePendingTaskCount(factory.getScheduledExecutor()); + // Trigger create & close ML 1000 times. + for (int i = 0; i < 1000; i++) { + ManagedLedger ml = factory.open(mlName, config); + ml.close(); + } + // Verify there is no orphan scheduled task. + long pendingTaskCountAfter = calculatePendingTaskCount(factory.getScheduledExecutor()); + // Maybe there are other components also appended scheduled tasks, so leave 100 tasks to avoid flaky. + assertTrue(pendingTaskCountAfter - pendingTaskCountBefore < 100); + } + + /** + * Calculate how many pending tasks in {@link OrderedScheduler} + */ + private long calculatePendingTaskCount(OrderedScheduler orderedScheduler) { + ExecutorService[] threads = WhiteboxImpl.getInternalState(orderedScheduler, "threads"); + long taskCounter = 0; + for (ExecutorService thread : threads) { + BoundedScheduledExecutorService boundedScheduledExecutorService = + WhiteboxImpl.getInternalState(thread, "delegate"); + BlockingQueue queue = WhiteboxImpl.getInternalState(boundedScheduledExecutorService, "queue"); + for (Runnable r : queue) { + if (r instanceof FutureTask) { + FutureTask futureTask = (FutureTask) r; + if (!futureTask.isCancelled() && !futureTask.isDone()) { + taskCounter++; + } + } else { + taskCounter++; + } + } + } + return taskCounter; + } + + @Test + public void testNoCleanupOffloadLedgerWhenMetadataExceptionHappens() throws Exception { + ManagedLedgerConfig config = spy(new ManagedLedgerConfig()); + ManagedLedgerImpl ml = spy((ManagedLedgerImpl) factory.open("testNoCleanupOffloadLedger", config)); + + // mock the ledger offloader + LedgerOffloader ledgerOffloader = mock(NullLedgerOffloader.class); + when(config.getLedgerOffloader()).thenReturn(ledgerOffloader); + when(ledgerOffloader.getOffloadDriverName()).thenReturn("mock"); + + // There will have two put call to the metadata store, the first time is prepare the offload. + // And the second is the complete the offload. This case is testing when completing the offload, + // the metadata store meets an exception. + AtomicInteger metadataPutCallCount = new AtomicInteger(0); + metadataStore.failConditional(new MetadataStoreException("mock completion error"), + (key, value) -> key.equals(FaultInjectionMetadataStore.OperationType.PUT) && + metadataPutCallCount.incrementAndGet() == 2); + + // prepare the arguments for the offloadLoop method + CompletableFuture future = new CompletableFuture<>(); + Queue ledgersToOffload = new LinkedList<>(); + LedgerInfo ledgerInfo = LedgerInfo.getDefaultInstance().toBuilder().setLedgerId(1).setEntries(10).build(); + ledgersToOffload.add(ledgerInfo); + Position firstUnoffloaded = PositionFactory.create(1, 0); + Optional firstError = Optional.empty(); + + // mock the read handle to make the offload successful + CompletableFuture readHandle = new CompletableFuture<>(); + readHandle.complete(mock(ReadHandle.class)); + when(ml.getLedgerHandle(eq(ledgerInfo.getLedgerId()))).thenReturn(readHandle); + when(ledgerOffloader.offload(any(), any(), anyMap())).thenReturn(CompletableFuture.completedFuture(null)); + + ml.ledgers.put(ledgerInfo.getLedgerId(), ledgerInfo); + + // do the offload + ml.offloadLoop(future, ledgersToOffload, firstUnoffloaded, firstError); + + // waiting for the offload complete + try { + future.join(); + fail("The offload should fail"); + } catch (Exception e) { + // the offload should fail + assertTrue(e.getCause().getMessage().contains("mock completion error")); + } + + // the ledger deletion shouldn't happen + verify(ledgerOffloader, times(0)) + .deleteOffloaded(eq(ledgerInfo.getLedgerId()), any(), anyMap()); + } + + + @DataProvider(name = "closeLedgerByAddEntry") + public Object[][] closeLedgerByAddEntry() { + return new Object[][] {{Boolean.TRUE}, {Boolean.FALSE}}; + } + + @Test(dataProvider = "closeLedgerByAddEntry") + public void testDeleteCurrentLedgerWhenItIsClosed(boolean closeLedgerByAddEntry) throws Exception { + // Setup: Open a manageLedger with one initial entry. + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setMaxEntriesPerLedger(10); + ManagedLedgerImpl ml = spy((ManagedLedgerImpl) factory.open("testDeleteCurrentLedgerWhenItIsClosed", + config)); + assertEquals(ml.ledgers.size(), 1); + ml.addEntry(new byte[4]); + // Act: Trigger the rollover of the current ledger. + long currentLedgerID = ml.currentLedger.getId(); + ml.config.setMaximumRolloverTime(10, TimeUnit.MILLISECONDS); + Thread.sleep(10); + if (closeLedgerByAddEntry) { + // Detect the current ledger is full before written entry and close the ledger after writing completely. + ml.addEntry(new byte[4]); + } else { + // Detect the current ledger is full by the timed task. (Imitate: the timed task `checkLedgerRollTask` call + // `rollCurrentLedgerIfFull` periodically). + ml.rollCurrentLedgerIfFull(); + // the ledger closing in the `rollCurrentLedgerIfFull` is async, so the wait is needed. + Awaitility.await().untilAsserted(() -> assertEquals(ml.ledgers.size(), 2)); + } + // Act: Trigger trimming to delete the previous current ledger. + ml.internalTrimLedgers(false, Futures.NULL_PROMISE); + // Verify: A new ledger will be opened after the current ledger is closed and the previous current ledger can be + // deleted. + Awaitility.await().untilAsserted(() -> { + assertEquals(ml.state, ManagedLedgerImpl.State.LedgerOpened); + assertEquals(ml.ledgers.size(), 1); + assertNotEquals(currentLedgerID, ml.currentLedger.getId()); + assertEquals(ml.currentLedgerEntries, 0); + }); + } } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorTest.java index 1ad3f5f8de631..3e1bae7ea7b44 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorTest.java @@ -51,6 +51,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerFactory; import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.test.MockedBookKeeperTestCase; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,7 +66,7 @@ public class NonDurableCursorTest extends MockedBookKeeperTestCase { void readFromEmptyLedger() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger"); - ManagedCursor c1 = ledger.newNonDurableCursor(PositionImpl.EARLIEST); + ManagedCursor c1 = ledger.newNonDurableCursor(PositionFactory.EARLIEST); List entries = c1.readEntries(10); assertEquals(entries.size(), 0); entries.forEach(Entry::release); @@ -80,7 +81,8 @@ void readFromEmptyLedger() throws Exception { entries.forEach(Entry::release); // Test string representation - assertEquals(c1.toString(), "NonDurableCursorImpl{ledger=my_test_ledger, ackPos=3:-1, readPos=3:1}"); + assertEquals(c1.toString(), "NonDurableCursorImpl{ledger=my_test_ledger, cursor=" + + c1.getName() + ", ackPos=3:-1, readPos=3:1}"); } @Test(timeOut = 20000) @@ -88,14 +90,14 @@ void testOpenNonDurableCursorAtNonExistentMessageId() throws Exception { ManagedLedger ledger = factory.open("non_durable_cursor_at_non_existent_msgid"); ManagedLedgerImpl mlImpl = (ManagedLedgerImpl) ledger; - PositionImpl position = mlImpl.getLastPosition(); + Position position = mlImpl.getLastPosition(); - ManagedCursor c1 = ledger.newNonDurableCursor(new PositionImpl( + ManagedCursor c1 = ledger.newNonDurableCursor(PositionFactory.create( position.getLedgerId(), position.getEntryId() - 1 )); - assertEquals(c1.getReadPosition(), new PositionImpl( + assertEquals(c1.getReadPosition(), PositionFactory.create( position.getLedgerId(), 0 )); @@ -108,7 +110,7 @@ void testOpenNonDurableCursorAtNonExistentMessageId() throws Exception { void testZNodeBypassed() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger"); - ManagedCursor c1 = ledger.newNonDurableCursor(PositionImpl.EARLIEST); + ManagedCursor c1 = ledger.newNonDurableCursor(PositionFactory.EARLIEST); assertTrue(ledger.getCursors().iterator().hasNext()); c1.close(); @@ -124,8 +126,8 @@ void readTwice() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setRetentionTime(1, TimeUnit.HOURS).setRetentionSizeInMB(1)); - ManagedCursor c1 = ledger.newNonDurableCursor(PositionImpl.LATEST); - ManagedCursor c2 = ledger.newNonDurableCursor(PositionImpl.LATEST); + ManagedCursor c1 = ledger.newNonDurableCursor(PositionFactory.LATEST); + ManagedCursor c2 = ledger.newNonDurableCursor(PositionFactory.LATEST); ledger.addEntry("entry-1".getBytes(Encoding)); ledger.addEntry("entry-2".getBytes(Encoding)); @@ -157,8 +159,8 @@ void readWithCacheDisabled() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(1) .setRetentionTime(1, TimeUnit.HOURS).setRetentionSizeInMB(1)); - ManagedCursor c1 = ledger.newNonDurableCursor(PositionImpl.LATEST); - ManagedCursor c2 = ledger.newNonDurableCursor(PositionImpl.LATEST); + ManagedCursor c1 = ledger.newNonDurableCursor(PositionFactory.LATEST); + ManagedCursor c2 = ledger.newNonDurableCursor(PositionFactory.LATEST); ledger.addEntry("entry-1".getBytes(Encoding)); ledger.addEntry("entry-2".getBytes(Encoding)); @@ -187,7 +189,7 @@ void readFromClosedLedger() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(1) .setRetentionTime(1, TimeUnit.HOURS).setRetentionSizeInMB(1)); - ManagedCursor c1 = ledger.newNonDurableCursor(PositionImpl.LATEST); + ManagedCursor c1 = ledger.newNonDurableCursor(PositionFactory.LATEST); ledger.close(); @@ -204,15 +206,15 @@ void testNumberOfEntries() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2) .setRetentionTime(1, TimeUnit.HOURS).setRetentionSizeInMB(1)); - ManagedCursor c1 = ledger.newNonDurableCursor(PositionImpl.LATEST); + ManagedCursor c1 = ledger.newNonDurableCursor(PositionFactory.LATEST); ledger.addEntry("dummy-entry-1".getBytes(Encoding)); - ManagedCursor c2 = ledger.newNonDurableCursor(PositionImpl.LATEST); + ManagedCursor c2 = ledger.newNonDurableCursor(PositionFactory.LATEST); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); - ManagedCursor c3 = ledger.newNonDurableCursor(PositionImpl.LATEST); + ManagedCursor c3 = ledger.newNonDurableCursor(PositionFactory.LATEST); ledger.addEntry("dummy-entry-3".getBytes(Encoding)); - ManagedCursor c4 = ledger.newNonDurableCursor(PositionImpl.LATEST); + ManagedCursor c4 = ledger.newNonDurableCursor(PositionFactory.LATEST); ledger.addEntry("dummy-entry-4".getBytes(Encoding)); - ManagedCursor c5 = ledger.newNonDurableCursor(PositionImpl.LATEST); + ManagedCursor c5 = ledger.newNonDurableCursor(PositionFactory.LATEST); assertEquals(c1.getNumberOfEntries(), 4); assertTrue(c1.hasMoreEntries()); @@ -241,15 +243,15 @@ void testNumberOfEntriesInBacklog() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2) .setRetentionTime(1, TimeUnit.HOURS).setRetentionSizeInMB(1)); - ManagedCursor c1 = ledger.newNonDurableCursor(PositionImpl.LATEST); + ManagedCursor c1 = ledger.newNonDurableCursor(PositionFactory.LATEST); Position p1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding)); - ManagedCursor c2 = ledger.newNonDurableCursor(PositionImpl.LATEST); + ManagedCursor c2 = ledger.newNonDurableCursor(PositionFactory.LATEST); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); - ManagedCursor c3 = ledger.newNonDurableCursor(PositionImpl.LATEST); + ManagedCursor c3 = ledger.newNonDurableCursor(PositionFactory.LATEST); Position p3 = ledger.addEntry("dummy-entry-3".getBytes(Encoding)); - ManagedCursor c4 = ledger.newNonDurableCursor(PositionImpl.LATEST); + ManagedCursor c4 = ledger.newNonDurableCursor(PositionFactory.LATEST); Position p4 = ledger.addEntry("dummy-entry-4".getBytes(Encoding)); - ManagedCursor c5 = ledger.newNonDurableCursor(PositionImpl.LATEST); + ManagedCursor c5 = ledger.newNonDurableCursor(PositionFactory.LATEST); assertEquals(c1.getNumberOfEntriesInBacklog(false), 4); assertEquals(c2.getNumberOfEntriesInBacklog(false), 3); @@ -333,7 +335,7 @@ void markDeleteAcrossLedgers() throws Exception { @Test(timeOut = 20000) void markDeleteGreaterThanLastConfirmedEntry() throws Exception { ManagedLedger ml1 = factory.open("my_test_ledger"); - ManagedCursor mc1 = ml1.newNonDurableCursor(PositionImpl.get(Long.MAX_VALUE - 1, Long.MAX_VALUE - 1)); + ManagedCursor mc1 = ml1.newNonDurableCursor(PositionFactory.create(Long.MAX_VALUE - 1, Long.MAX_VALUE - 1)); assertEquals(mc1.getMarkDeletedPosition(), ml1.getLastConfirmedEntry()); } @@ -341,13 +343,13 @@ void markDeleteGreaterThanLastConfirmedEntry() throws Exception { void testResetCursor() throws Exception { ManagedLedger ledger = factory.open("my_test_move_cursor_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(10)); - ManagedCursor cursor = ledger.newNonDurableCursor(PositionImpl.LATEST); + ManagedCursor cursor = ledger.newNonDurableCursor(PositionFactory.LATEST); ledger.addEntry("dummy-entry-1".getBytes(Encoding)); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); ledger.addEntry("dummy-entry-3".getBytes(Encoding)); - PositionImpl lastPosition = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding)); + Position lastPosition = ledger.addEntry("dummy-entry-4".getBytes(Encoding)); final AtomicBoolean moveStatus = new AtomicBoolean(false); - PositionImpl resetPosition = new PositionImpl(lastPosition.getLedgerId(), lastPosition.getEntryId() - 2); + Position resetPosition = PositionFactory.create(lastPosition.getLedgerId(), lastPosition.getEntryId() - 2); try { cursor.resetCursor(resetPosition); moveStatus.set(true); @@ -365,14 +367,14 @@ void testResetCursor() throws Exception { void testasyncResetCursor() throws Exception { ManagedLedger ledger = factory.open("my_test_move_cursor_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(10)); - ManagedCursor cursor = ledger.newNonDurableCursor(PositionImpl.LATEST); + ManagedCursor cursor = ledger.newNonDurableCursor(PositionFactory.LATEST); ledger.addEntry("dummy-entry-1".getBytes(Encoding)); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); ledger.addEntry("dummy-entry-3".getBytes(Encoding)); - PositionImpl lastPosition = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding)); + Position lastPosition = ledger.addEntry("dummy-entry-4".getBytes(Encoding)); final AtomicBoolean moveStatus = new AtomicBoolean(false); CountDownLatch countDownLatch = new CountDownLatch(1); - PositionImpl resetPosition = new PositionImpl(lastPosition.getLedgerId(), lastPosition.getEntryId() - 2); + Position resetPosition = PositionFactory.create(lastPosition.getLedgerId(), lastPosition.getEntryId() - 2); cursor.asyncResetCursor(resetPosition, false, new AsyncCallbacks.ResetCursorCallback() { @Override @@ -398,7 +400,7 @@ public void resetFailed(ManagedLedgerException exception, Object ctx) { void rewind() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2) .setRetentionTime(1, TimeUnit.HOURS).setRetentionSizeInMB(1)); - ManagedCursor c1 = ledger.newNonDurableCursor(PositionImpl.EARLIEST); + ManagedCursor c1 = ledger.newNonDurableCursor(PositionFactory.EARLIEST); Position p1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding)); Position p2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding)); Position p3 = ledger.addEntry("dummy-entry-3".getBytes(Encoding)); @@ -449,11 +451,11 @@ void rewind() throws Exception { @Test(timeOut = 20000) void markDeleteSkippingMessage() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(10)); - ManagedCursor cursor = ledger.newNonDurableCursor(PositionImpl.EARLIEST); + ManagedCursor cursor = ledger.newNonDurableCursor(PositionFactory.EARLIEST); Position p1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding)); Position p2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding)); ledger.addEntry("dummy-entry-3".getBytes(Encoding)); - PositionImpl p4 = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding)); + Position p4 = ledger.addEntry("dummy-entry-4".getBytes(Encoding)); assertEquals(cursor.getNumberOfEntries(), 4); @@ -472,7 +474,7 @@ void markDeleteSkippingMessage() throws Exception { assertFalse(cursor.hasMoreEntries()); assertEquals(cursor.getNumberOfEntries(), 0); - assertEquals(cursor.getReadPosition(), new PositionImpl(p4.getLedgerId(), p4.getEntryId() + 1)); + assertEquals(cursor.getReadPosition(), PositionFactory.create(p4.getLedgerId(), p4.getEntryId() + 1)); } @Test(timeOut = 20000) @@ -545,7 +547,7 @@ void unorderedMarkDelete() throws Exception { void testSingleDelete() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(3) .setRetentionTime(1, TimeUnit.HOURS).setRetentionSizeInMB(1)); - ManagedCursor cursor = ledger.newNonDurableCursor(PositionImpl.LATEST); + ManagedCursor cursor = ledger.newNonDurableCursor(PositionFactory.LATEST); Position p1 = ledger.addEntry("entry1".getBytes()); Position p2 = ledger.addEntry("entry2".getBytes()); @@ -588,12 +590,12 @@ void subscribeToEarliestPositionWithImmediateDeletion() throws Exception { /* Position p1 = */ ledger.addEntry("entry-1".getBytes()); /* Position p2 = */ ledger.addEntry("entry-2".getBytes()); - Position p3 = ledger.addEntry("entry-3".getBytes()); + /* Position p3 = */ ledger.addEntry("entry-3".getBytes()); Thread.sleep(300); - ManagedCursor c1 = ledger.newNonDurableCursor(PositionImpl.EARLIEST); - assertEquals(c1.getReadPosition(), p3); - assertEquals(c1.getMarkDeletedPosition(), new PositionImpl(5, -1)); + ManagedCursor c1 = ledger.newNonDurableCursor(PositionFactory.EARLIEST); + assertEquals(c1.getReadPosition(), PositionFactory.create(6, 0)); + assertEquals(c1.getMarkDeletedPosition(), PositionFactory.create(6, -1)); } @Test // (timeOut = 20000) @@ -608,9 +610,9 @@ void subscribeToEarliestPositionWithDeferredDeletion() throws Exception { /* Position p5 = */ ledger.addEntry("entry-5".getBytes()); /* Position p6 = */ ledger.addEntry("entry-6".getBytes()); - ManagedCursor c1 = ledger.newNonDurableCursor(PositionImpl.EARLIEST); + ManagedCursor c1 = ledger.newNonDurableCursor(PositionFactory.EARLIEST); assertEquals(c1.getReadPosition(), p1); - assertEquals(c1.getMarkDeletedPosition(), new PositionImpl(3, -1)); + assertEquals(c1.getMarkDeletedPosition(), PositionFactory.create(3, -1)); assertEquals(c1.getNumberOfEntries(), 6); assertEquals(c1.getNumberOfEntriesInBacklog(false), 6); @@ -673,7 +675,7 @@ public void testGetSlowestConsumer() throws Exception { // The slowest reader should still be the durable cursor since non-durable readers are not taken into account assertEquals(p3, ledger.getCursors().getSlowestReaderPosition()); - PositionImpl earliestPos = new PositionImpl(-1, -2); + Position earliestPos = PositionFactory.create(-1, -2); ManagedCursor nonCursorEarliest = ledger.newNonDurableCursor(earliestPos, ncEarliest); @@ -701,7 +703,7 @@ public void testBacklogStatsWhenDroppingData() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testBacklogStatsWhenDroppingData", new ManagedLedgerConfig().setMaxEntriesPerLedger(1)); ManagedCursor c1 = ledger.openCursor("c1"); - ManagedCursor nonDurableCursor = ledger.newNonDurableCursor(PositionImpl.EARLIEST); + ManagedCursor nonDurableCursor = ledger.newNonDurableCursor(PositionFactory.EARLIEST); assertEquals(nonDurableCursor.getNumberOfEntries(), 0); assertEquals(nonDurableCursor.getNumberOfEntriesInBacklog(true), 0); @@ -722,9 +724,10 @@ public void testBacklogStatsWhenDroppingData() throws Exception { CompletableFuture promise = new CompletableFuture<>(); ledger.internalTrimConsumedLedgers(promise); promise.join(); - - assertEquals(nonDurableCursor.getNumberOfEntries(), 6); - assertEquals(nonDurableCursor.getNumberOfEntriesInBacklog(true), 6); + // The mark delete position has moved to position 4:1, and the ledger 4 only has one entry, + // so the ledger 4 can be deleted. nonDurableCursor should has the same backlog with durable cursor. + assertEquals(nonDurableCursor.getNumberOfEntries(), 5); + assertEquals(nonDurableCursor.getNumberOfEntriesInBacklog(true), 5); c1.close(); ledger.deleteCursor(c1.getName()); @@ -732,8 +735,8 @@ public void testBacklogStatsWhenDroppingData() throws Exception { ledger.internalTrimConsumedLedgers(promise); promise.join(); - assertEquals(nonDurableCursor.getNumberOfEntries(), 1); - assertEquals(nonDurableCursor.getNumberOfEntriesInBacklog(true), 1); + assertEquals(nonDurableCursor.getNumberOfEntries(), 0); + assertEquals(nonDurableCursor.getNumberOfEntriesInBacklog(true), 0); ledger.close(); } @@ -744,7 +747,7 @@ public void testInvalidateReadHandleWithSlowNonDurableCursor() throws Exception new ManagedLedgerConfig().setMaxEntriesPerLedger(1).setRetentionTime(-1, TimeUnit.SECONDS) .setRetentionSizeInMB(-1)); ManagedCursor c1 = ledger.openCursor("c1"); - ManagedCursor nonDurableCursor = ledger.newNonDurableCursor(PositionImpl.EARLIEST); + ManagedCursor nonDurableCursor = ledger.newNonDurableCursor(PositionFactory.EARLIEST); List positions = new ArrayList<>(); for (int i = 0; i < 10; i++) { @@ -753,7 +756,7 @@ public void testInvalidateReadHandleWithSlowNonDurableCursor() throws Exception CountDownLatch latch = new CountDownLatch(10); for (int i = 0; i < 10; i++) { - ledger.asyncReadEntry((PositionImpl) positions.get(i), new AsyncCallbacks.ReadEntryCallback() { + ledger.asyncReadEntry(positions.get(i), new AsyncCallbacks.ReadEntryCallback() { @Override public void readEntryComplete(Entry entry, Object ctx) { latch.countDown(); @@ -817,7 +820,7 @@ void testCursorWithNameIsNotNull() throws Exception { void deleteNonDurableCursorWithName() throws Exception { ManagedLedger ledger = factory.open("deleteManagedLedgerWithNonDurableCursor"); - ManagedCursor c = ledger.newNonDurableCursor(PositionImpl.EARLIEST, "custom-name"); + ManagedCursor c = ledger.newNonDurableCursor(PositionFactory.EARLIEST, "custom-name"); assertEquals(Iterables.size(ledger.getCursors()), 1); ledger.deleteCursor(c.getName()); @@ -829,7 +832,7 @@ public void testMessagesConsumedCounterInitializedCorrect() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testMessagesConsumedCounterInitializedCorrect", new ManagedLedgerConfig().setRetentionTime(1, TimeUnit.HOURS).setRetentionSizeInMB(1)); Position position = ledger.addEntry("1".getBytes(Encoding)); - NonDurableCursorImpl cursor = (NonDurableCursorImpl) ledger.newNonDurableCursor(PositionImpl.EARLIEST); + NonDurableCursorImpl cursor = (NonDurableCursorImpl) ledger.newNonDurableCursor(PositionFactory.EARLIEST); cursor.delete(position); assertEquals(cursor.getMessagesConsumedCounter(), 1); assertTrue(cursor.getMessagesConsumedCounter() <= ledger.getEntriesAddedCounter()); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadLedgerDeleteTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadLedgerDeleteTest.java index 56da315553ea4..b46f06106cf4c 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadLedgerDeleteTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadLedgerDeleteTest.java @@ -383,6 +383,10 @@ public void isOffloadedNeedsDeleteTest() throws Exception { needsDelete = managedLedger.isOffloadedNeedsDelete(offloadContext, Optional.of(offloadPolicies)); Assert.assertTrue(needsDelete); + offloadPolicies.setManagedLedgerOffloadDeletionLagInMillis(-1L); + needsDelete = managedLedger.isOffloadedNeedsDelete(offloadContext, Optional.of(offloadPolicies)); + Assert.assertFalse(needsDelete); + offloadPolicies.setManagedLedgerOffloadDeletionLagInMillis(1000L * 2); needsDelete = managedLedger.isOffloadedNeedsDelete(offloadContext, Optional.of(offloadPolicies)); Assert.assertFalse(needsDelete); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixReadTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixReadTest.java index cd224e33e2734..48751417e1714 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixReadTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixReadTest.java @@ -29,6 +29,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; import io.netty.buffer.ByteBuf; import java.util.ArrayList; @@ -54,18 +55,43 @@ import org.apache.bookkeeper.mledger.LedgerOffloader; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo; import org.apache.bookkeeper.mledger.util.MockClock; import org.apache.bookkeeper.net.BookieId; import org.apache.bookkeeper.test.MockedBookKeeperTestCase; import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; import org.apache.pulsar.common.policies.data.OffloadedReadPriority; +import org.awaitility.Awaitility; import org.testng.Assert; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; public class OffloadPrefixReadTest extends MockedBookKeeperTestCase { - @Test - public void testOffloadRead() throws Exception { + + private final String offloadTypeAppendable = "NonAppendable"; + + @Override + protected void initManagedLedgerFactoryConfig(ManagedLedgerFactoryConfig config) { + super.initManagedLedgerFactoryConfig(config); + // disable cache. + config.setMaxCacheSize(0); + } + + @DataProvider(name = "offloadAndDeleteTypes") + public Object[][] offloadAndDeleteTypes() { + return new Object[][]{ + {"normal", true}, + {"normal", false}, + {offloadTypeAppendable, true}, + {offloadTypeAppendable, false}, + }; + } + + @Test(dataProvider = "offloadAndDeleteTypes") + public void testOffloadRead(String offloadType, boolean deleteMl) throws Exception { MockLedgerOffloader offloader = spy(MockLedgerOffloader.class); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); @@ -88,12 +114,16 @@ public void testOffloadRead() throws Exception { Assert.assertTrue(ledger.getLedgersInfoAsList().get(1).getOffloadContext().getComplete()); Assert.assertFalse(ledger.getLedgersInfoAsList().get(2).getOffloadContext().getComplete()); + if (offloadTypeAppendable.equals(offloadType)) { + config.setLedgerOffloader(new NonAppendableLedgerOffloader(offloader)); + } + UUID firstLedgerUUID = new UUID(ledger.getLedgersInfoAsList().get(0).getOffloadContext().getUidMsb(), ledger.getLedgersInfoAsList().get(0).getOffloadContext().getUidLsb()); UUID secondLedgerUUID = new UUID(ledger.getLedgersInfoAsList().get(1).getOffloadContext().getUidMsb(), ledger.getLedgersInfoAsList().get(1).getOffloadContext().getUidLsb()); - ManagedCursor cursor = ledger.newNonDurableCursor(PositionImpl.EARLIEST); + ManagedCursor cursor = ledger.newNonDurableCursor(PositionFactory.EARLIEST); int i = 0; for (Entry e : cursor.readEntries(10)) { assertEquals(new String(e.getData()), "entry-" + i++); @@ -115,13 +145,30 @@ public void testOffloadRead() throws Exception { verify(offloader, times(2)) .readOffloaded(anyLong(), (UUID) any(), anyMap()); - ledger.close(); - // Ensure that all the read handles had been closed - assertEquals(offloader.openedReadHandles.get(), 0); + if (!deleteMl) { + ledger.close(); + // Ensure that all the read handles had been closed + assertEquals(offloader.openedReadHandles.get(), 0); + } else { + // Verify: the ledger offloaded will be deleted after managed ledger is deleted. + ledger.delete(); + Awaitility.await().untilAsserted(() -> { + assertTrue(offloader.offloads.size() <= 1); + assertTrue(ledger.ledgers.size() <= 1); + }); + } } - @Test - public void testBookkeeperFirstOffloadRead() throws Exception { + @DataProvider(name = "offloadTypes") + public Object[][] offloadTypes() { + return new Object[][]{ + {"normal"}, + {offloadTypeAppendable}, + }; + } + + @Test(dataProvider = "offloadTypes") + public void testBookkeeperFirstOffloadRead(String offloadType) throws Exception { MockLedgerOffloader offloader = spy(MockLedgerOffloader.class); MockClock clock = new MockClock(); offloader.getOffloadPolicies() @@ -163,7 +210,7 @@ public void testBookkeeperFirstOffloadRead() throws Exception { UUID secondLedgerUUID = new UUID(secondLedger.getOffloadContext().getUidMsb(), secondLedger.getOffloadContext().getUidLsb()); - ManagedCursor cursor = ledger.newNonDurableCursor(PositionImpl.EARLIEST); + ManagedCursor cursor = ledger.newNonDurableCursor(PositionFactory.EARLIEST); int i = 0; for (Entry e : cursor.readEntries(10)) { Assert.assertEquals(new String(e.getData()), "entry-" + i++); @@ -186,6 +233,10 @@ public void testBookkeeperFirstOffloadRead() throws Exception { Assert.assertTrue(ledger.getLedgersInfoAsList().get(0).getOffloadContext().getBookkeeperDeleted()); Assert.assertTrue(ledger.getLedgersInfoAsList().get(1).getOffloadContext().getBookkeeperDeleted()); + if (offloadTypeAppendable.equals(offloadType)) { + config.setLedgerOffloader(new NonAppendableLedgerOffloader(offloader)); + } + for (Entry e : cursor.readEntries(10)) { Assert.assertEquals(new String(e.getData()), "entry-" + i++); } @@ -195,6 +246,56 @@ public void testBookkeeperFirstOffloadRead() throws Exception { .readOffloaded(anyLong(), (UUID) any(), anyMap()); verify(offloader).readOffloaded(anyLong(), eq(secondLedgerUUID), anyMap()); + // Verify: the ledger offloaded will be trimmed after if no backlog. + while (cursor.hasMoreEntries()) { + cursor.readEntries(1); + } + config.setRetentionTime(0, TimeUnit.MILLISECONDS); + config.setRetentionSizeInMB(0); + CompletableFuture trimFuture = new CompletableFuture(); + ledger.trimConsumedLedgersInBackground(trimFuture); + trimFuture.join(); + Awaitility.await().untilAsserted(() -> { + assertTrue(offloader.offloads.size() <= 1); + assertTrue(ledger.ledgers.size() <= 1); + }); + + // cleanup. + ledger.delete(); + } + + + + @Test + public void testSkipOffloadIfReadOnly() throws Exception { + LedgerOffloader ol = new NonAppendableLedgerOffloader(spy(MockLedgerOffloader.class)); + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setMaxEntriesPerLedger(10); + config.setMinimumRolloverTime(0, TimeUnit.SECONDS); + config.setRetentionTime(10, TimeUnit.MINUTES); + config.setRetentionSizeInMB(10); + config.setLedgerOffloader(ol); + ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", config); + + for (int i = 0; i < 25; i++) { + String content = "entry-" + i; + ledger.addEntry(content.getBytes()); + } + assertEquals(ledger.getLedgersInfoAsList().size(), 3); + + try { + ledger.offloadPrefix(ledger.getLastConfirmedEntry()); + } catch (ManagedLedgerException mle) { + assertTrue(mle.getMessage().contains("does not support offload")); + } + + assertEquals(ledger.getLedgersInfoAsList().size(), 3); + Assert.assertFalse(ledger.getLedgersInfoAsList().get(0).getOffloadContext().getComplete()); + Assert.assertFalse(ledger.getLedgersInfoAsList().get(1).getOffloadContext().getComplete()); + Assert.assertFalse(ledger.getLedgersInfoAsList().get(2).getOffloadContext().getComplete()); + + // cleanup. + ledger.delete(); } @@ -313,7 +414,7 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr @Override public CompletableFuture readUnconfirmedAsync(long firstEntry, long lastEntry) { - return unsupported(); + return readAsync(firstEntry, lastEntry); } @Override diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java index 2cdb14fb71e41..3f9f4f8da12f2 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java @@ -49,6 +49,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo; import org.apache.bookkeeper.test.MockedBookKeeperTestCase; @@ -94,7 +95,7 @@ public void testNullOffloader() throws Exception { ledger.offloadPrefix(p); fail("Should have thrown an exception"); } catch (ManagedLedgerException e) { - assertEquals(e.getMessage(), "NullLedgerOffloader"); + assertTrue(e.getMessage().contains("does not support offload")); } assertEquals(ledger.getLedgersInfoAsList().size(), 5); assertEquals(ledger.getLedgersInfoAsList().stream() @@ -201,13 +202,13 @@ public void testPositionOutOfRange() throws Exception { assertEquals(ledger.getLedgersInfoAsList().size(), 3); try { - ledger.offloadPrefix(PositionImpl.EARLIEST); + ledger.offloadPrefix(PositionFactory.EARLIEST); fail("Should have thrown an exception"); } catch (ManagedLedgerException.InvalidCursorPositionException e) { // expected } try { - ledger.offloadPrefix(PositionImpl.LATEST); + ledger.offloadPrefix(PositionFactory.LATEST); fail("Should have thrown an exception"); } catch (ManagedLedgerException.InvalidCursorPositionException e) { // expected @@ -241,7 +242,7 @@ public void testPositionOnEdgeOfLedger() throws Exception { ledger.addEntry("entry-blah".getBytes()); assertEquals(ledger.getLedgersInfoAsList().size(), 3); - PositionImpl firstUnoffloaded = (PositionImpl)ledger.offloadPrefix(p); + Position firstUnoffloaded = ledger.offloadPrefix(p); // only the first ledger should have been offloaded assertEquals(ledger.getLedgersInfoAsList().size(), 3); @@ -254,7 +255,7 @@ public void testPositionOnEdgeOfLedger() throws Exception { assertEquals(firstUnoffloaded.getEntryId(), 0); // offload again, with the position in the third ledger - PositionImpl firstUnoffloaded2 = (PositionImpl)ledger.offloadPrefix(ledger.getLastConfirmedEntry()); + Position firstUnoffloaded2 = ledger.offloadPrefix(ledger.getLastConfirmedEntry()); assertEquals(ledger.getLedgersInfoAsList().size(), 3); assertEquals(offloader.offloadedLedgers().size(), 2); assertTrue(offloader.offloadedLedgers().contains(ledger.getLedgersInfoAsList().get(0).getLedgerId())); @@ -291,9 +292,9 @@ public void testPositionOnLastEmptyLedger() throws Exception { assertEquals(ledger.getLedgersInfoAsList().get(1).getSize(), 0); // position past the end of first ledger - Position p = new PositionImpl(ledger.getLedgersInfoAsList().get(1).getLedgerId(), 0); + Position p = PositionFactory.create(ledger.getLedgersInfoAsList().get(1).getLedgerId(), 0); - PositionImpl firstUnoffloaded = (PositionImpl)ledger.offloadPrefix(p); + Position firstUnoffloaded = ledger.offloadPrefix(p); // only the first ledger should have been offloaded assertEquals(ledger.getLedgersInfoAsList().size(), 2); @@ -335,8 +336,8 @@ public CompletableFuture offload(ReadHandle ledger, } assertEquals(ledger.getLedgersInfoAsList().size(), 3); - PositionImpl startOfSecondLedger = PositionImpl.get(ledger.getLedgersInfoAsList().get(1).getLedgerId(), 0); - PositionImpl startOfThirdLedger = PositionImpl.get(ledger.getLedgersInfoAsList().get(2).getLedgerId(), 0); + Position startOfSecondLedger = PositionFactory.create(ledger.getLedgersInfoAsList().get(1).getLedgerId(), 0); + Position startOfThirdLedger = PositionFactory.create(ledger.getLedgersInfoAsList().get(2).getLedgerId(), 0); // trigger an offload which should offload the first two ledgers OffloadCallbackPromise cbPromise = new OffloadCallbackPromise(); @@ -398,8 +399,8 @@ public CompletableFuture offload(ReadHandle ledger, } assertEquals(ledger.getLedgersInfoAsList().size(), 3); - PositionImpl startOfSecondLedger = PositionImpl.get(ledger.getLedgersInfoAsList().get(1).getLedgerId(), 0); - PositionImpl startOfThirdLedger = PositionImpl.get(ledger.getLedgersInfoAsList().get(2).getLedgerId(), 0); + Position startOfSecondLedger = PositionFactory.create(ledger.getLedgersInfoAsList().get(1).getLedgerId(), 0); + Position startOfThirdLedger = PositionFactory.create(ledger.getLedgersInfoAsList().get(2).getLedgerId(), 0); // trigger an offload which should offload the first two ledgers OffloadCallbackPromise cbPromise = new OffloadCallbackPromise(); @@ -829,7 +830,7 @@ public void testDontOffloadEmpty() throws Exception { ledgers.put(secondLedgerId, ledgers.get(secondLedgerId).toBuilder().setEntries(0).setSize(0).build()); - PositionImpl firstUnoffloaded = (PositionImpl)ledger.offloadPrefix(ledger.getLastConfirmedEntry()); + Position firstUnoffloaded = ledger.offloadPrefix(ledger.getLastConfirmedEntry()); assertEquals(firstUnoffloaded.getLedgerId(), fourthLedgerId); assertEquals(firstUnoffloaded.getEntryId(), 0); @@ -1073,7 +1074,7 @@ public CompletableFuture offload(ReadHandle ledger, } else if (sizeThreshold != null && sizeThreshold.equals(100L) && timeThreshold == null) { // the last 2 ledgers won't be offloaded. assertEquals(cbPromise.join(), - PositionImpl.get(ledger.getLedgersInfoAsList().get(1).getLedgerId(), 0)); + PositionFactory.create(ledger.getLedgersInfoAsList().get(1).getLedgerId(), 0)); assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 2); assertEquals(offloader.offloadedLedgers(), Set.of(ledger.getLedgersInfoAsList().get(0).getLedgerId(), diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionTest.java index f2b1a7062b5e3..763146b6c3fbb 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionTest.java @@ -21,33 +21,35 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotEquals; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo; import org.testng.annotations.Test; public class PositionTest { @Test(expectedExceptions = NullPointerException.class) public void nullParam() { - new PositionImpl((PositionInfo) null); + PositionFactory.create(null); } @Test public void simpleTest() { - PositionImpl pos = new PositionImpl(1, 2); + Position pos = PositionFactory.create(1, 2); assertEquals(pos.getLedgerId(), 1); assertEquals(pos.getEntryId(), 2); - assertEquals(pos, new PositionImpl(1, 2)); + assertEquals(pos, PositionFactory.create(1, 2)); - assertNotEquals(new PositionImpl(1, 3), pos); - assertNotEquals(new PositionImpl(3, 2), pos); + assertNotEquals(PositionFactory.create(1, 3), pos); + assertNotEquals(PositionFactory.create(3, 2), pos); assertNotEquals(pos, "1:2"); } @Test public void comparisons() { - PositionImpl pos1_1 = new PositionImpl(1, 1); - PositionImpl pos2_5 = new PositionImpl(2, 5); - PositionImpl pos10_0 = new PositionImpl(10, 0); - PositionImpl pos10_1 = new PositionImpl(10, 1); + Position pos1_1 = PositionFactory.create(1, 1); + Position pos2_5 = PositionFactory.create(2, 5); + Position pos10_0 = PositionFactory.create(10, 0); + Position pos10_1 = PositionFactory.create(10, 1); assertEquals(0, pos1_1.compareTo(pos1_1)); assertEquals(-1, pos1_1.compareTo(pos2_5)); @@ -72,10 +74,13 @@ public void comparisons() { @Test public void hashes() throws Exception { - PositionImpl p1 = new PositionImpl(5, 15); - PositionImpl p2 = new PositionImpl(PositionInfo.parseFrom(p1.getPositionInfo().toByteArray())); + Position p1 = PositionFactory.create(5, 15); + PositionInfo positionInfo = + PositionInfo.newBuilder().setLedgerId(p1.getLedgerId()).setEntryId(p1.getEntryId()).build(); + PositionInfo parsed = PositionInfo.parseFrom(positionInfo.toByteArray()); + Position p2 = PositionFactory.create(parsed.getLedgerId(), parsed.getEntryId()); assertEquals(p2.getLedgerId(), 5); assertEquals(p2.getEntryId(), 15); - assertEquals(new PositionImpl(5, 15).hashCode(), p2.hashCode()); + assertEquals(PositionFactory.create(5, 15).hashCode(), p2.hashCode()); } } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorTest.java index 0386966ad2284..66a33560b67b4 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorTest.java @@ -31,6 +31,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerNotFoundException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.ReadOnlyCursor; import org.apache.bookkeeper.test.MockedBookKeeperTestCase; import org.testng.annotations.Test; @@ -40,7 +41,7 @@ public class ReadOnlyCursorTest extends MockedBookKeeperTestCase { @Test void notFound() throws Exception { try { - factory.openReadOnlyCursor("notFound", PositionImpl.EARLIEST, new ManagedLedgerConfig()); + factory.openReadOnlyCursor("notFound", PositionFactory.EARLIEST, new ManagedLedgerConfig()); fail("Should have failed"); } catch (ManagedLedgerNotFoundException e) { // Expected @@ -59,7 +60,7 @@ void simple() throws Exception { ledger.addEntry(("entry-" + i).getBytes()); } - ReadOnlyCursor cursor = factory.openReadOnlyCursor("simple", PositionImpl.EARLIEST, new ManagedLedgerConfig()); + ReadOnlyCursor cursor = factory.openReadOnlyCursor("simple", PositionFactory.EARLIEST, new ManagedLedgerConfig()); assertEquals(cursor.getNumberOfEntries(), N); assertTrue(cursor.hasMoreEntries()); @@ -78,7 +79,7 @@ void simple() throws Exception { } // Open a new cursor - cursor = factory.openReadOnlyCursor("simple", PositionImpl.EARLIEST, new ManagedLedgerConfig()); + cursor = factory.openReadOnlyCursor("simple", PositionFactory.EARLIEST, new ManagedLedgerConfig()); assertEquals(cursor.getNumberOfEntries(), 2 * N); assertTrue(cursor.hasMoreEntries()); @@ -114,7 +115,7 @@ void skip() throws Exception { ledger.addEntry(("entry-" + i).getBytes()); } - ReadOnlyCursor cursor = factory.openReadOnlyCursor("skip", PositionImpl.EARLIEST, new ManagedLedgerConfig()); + ReadOnlyCursor cursor = factory.openReadOnlyCursor("skip", PositionFactory.EARLIEST, new ManagedLedgerConfig()); assertEquals(cursor.getNumberOfEntries(), N); assertTrue(cursor.hasMoreEntries()); @@ -138,7 +139,7 @@ void skipAll() throws Exception { ledger.addEntry(("entry-" + i).getBytes()); } - ReadOnlyCursor cursor = factory.openReadOnlyCursor("skip-all", PositionImpl.EARLIEST, + ReadOnlyCursor cursor = factory.openReadOnlyCursor("skip-all", PositionFactory.EARLIEST, new ManagedLedgerConfig()); assertEquals(cursor.getNumberOfEntries(), N); @@ -166,7 +167,7 @@ void skipMultiple() throws Exception { ledger.addEntry(("entry-" + i).getBytes()); } - ReadOnlyCursor cursor = factory.openReadOnlyCursor("skip", PositionImpl.EARLIEST, new ManagedLedgerConfig()); + ReadOnlyCursor cursor = factory.openReadOnlyCursor("skip", PositionFactory.EARLIEST, new ManagedLedgerConfig()); assertEquals(cursor.getNumberOfEntries(), N); assertTrue(cursor.hasMoreEntries()); @@ -188,7 +189,7 @@ void skipMultiple() throws Exception { void empty() throws Exception { factory.open("empty", new ManagedLedgerConfig().setRetentionTime(1, TimeUnit.HOURS)); - ReadOnlyCursor cursor = factory.openReadOnlyCursor("empty", PositionImpl.EARLIEST, new ManagedLedgerConfig()); + ReadOnlyCursor cursor = factory.openReadOnlyCursor("empty", PositionFactory.EARLIEST, new ManagedLedgerConfig()); assertEquals(cursor.getNumberOfEntries(), 0); assertFalse(cursor.hasMoreEntries()); @@ -206,7 +207,7 @@ void specifyStartPosition() throws Exception { ledger.addEntry(("entry-" + i).getBytes()); } - ReadOnlyCursor cursor = factory.openReadOnlyCursor("simple", PositionImpl.EARLIEST, new ManagedLedgerConfig()); + ReadOnlyCursor cursor = factory.openReadOnlyCursor("simple", PositionFactory.EARLIEST, new ManagedLedgerConfig()); assertEquals(cursor.getNumberOfEntries(), N); assertTrue(cursor.hasMoreEntries()); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ReadOnlyManagedLedgerImplTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ReadOnlyManagedLedgerImplTest.java new file mode 100644 index 0000000000000..61056d0b4b602 --- /dev/null +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ReadOnlyManagedLedgerImplTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + + +import static org.testng.Assert.assertEquals; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.ReadOnlyManagedLedger; +import org.apache.bookkeeper.test.MockedBookKeeperTestCase; +import org.testng.annotations.Test; + +public class ReadOnlyManagedLedgerImplTest extends MockedBookKeeperTestCase { + private static final String MANAGED_LEDGER_NAME_NON_PROPERTIES = "ml-non-properties"; + private static final String MANAGED_LEDGER_NAME_ATTACHED_PROPERTIES = "ml-attached-properties"; + + + @Test + public void testReadOnlyManagedLedgerImplAttachProperties() + throws ManagedLedgerException, InterruptedException, ExecutionException, TimeoutException { + final ManagedLedger ledger = factory.open(MANAGED_LEDGER_NAME_ATTACHED_PROPERTIES, + new ManagedLedgerConfig().setRetentionTime(1, TimeUnit.HOURS)); + final String propertiesKey = "test-key"; + final String propertiesValue = "test-value"; + + ledger.setConfig(new ManagedLedgerConfig()); + ledger.addEntry("entry-0".getBytes()); + Map properties = new HashMap<>(); + properties.put(propertiesKey, propertiesValue); + ledger.setProperties(Collections.unmodifiableMap(properties)); + CompletableFuture future = new CompletableFuture<>(); + factory.asyncOpenReadOnlyManagedLedger(MANAGED_LEDGER_NAME_ATTACHED_PROPERTIES, + new AsyncCallbacks.OpenReadOnlyManagedLedgerCallback() { + @Override + public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedger managedLedger, + Object ctx) { + managedLedger.getProperties().forEach((key, value) -> { + assertEquals(key, propertiesKey); + assertEquals(value, propertiesValue); + }); + future.complete(null); + } + + @Override + public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Object ctx) { + future.completeExceptionally(exception); + } + }, new ManagedLedgerConfig(), null); + + future.get(60, TimeUnit.SECONDS); + } + + @Test + public void testReadOnlyManagedLedgerImplNoProperties() + throws ManagedLedgerException, InterruptedException, ExecutionException, TimeoutException { + final ManagedLedger ledger = factory.open(MANAGED_LEDGER_NAME_NON_PROPERTIES, + new ManagedLedgerConfig().setRetentionTime(1, TimeUnit.HOURS)); + ledger.setConfig(new ManagedLedgerConfig()); + ledger.addEntry("entry-0".getBytes()); + CompletableFuture future = new CompletableFuture<>(); + factory.asyncOpenReadOnlyManagedLedger(MANAGED_LEDGER_NAME_NON_PROPERTIES, + new AsyncCallbacks.OpenReadOnlyManagedLedgerCallback() { + @Override + public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedger managedLedger, + Object ctx) { + assertEquals(managedLedger.getProperties().size(), 0); + future.complete(null); + } + + @Override + public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Object ctx) { + future.completeExceptionally(exception); + } + }, new ManagedLedgerConfig(), null); + + future.get(60, TimeUnit.SECONDS); + } + +} \ No newline at end of file diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImplTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImplTest.java index 4482e9944c0ce..fc5450f2c4cfc 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImplTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImplTest.java @@ -32,6 +32,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.test.MockedBookKeeperTestCase; import org.awaitility.Awaitility; import org.testng.annotations.Test; @@ -76,16 +77,13 @@ public void testShadowWrites() throws Exception { //Add new data to source ML Position newPos = sourceML.addEntry(data); - // The state should not be the same. - log.info("Source.LCE={},Shadow.LCE={}", sourceML.lastConfirmedEntry, shadowML.lastConfirmedEntry); - assertNotEquals(sourceML.lastConfirmedEntry, shadowML.lastConfirmedEntry); - //Add new data to source ML, and a new ledger rolled - newPos = sourceML.addEntry(data); - assertEquals(sourceML.ledgers.size(), 4); - Awaitility.await().untilAsserted(()->assertEquals(shadowML.ledgers.size(), 4)); + Awaitility.await().untilAsserted(() -> { + assertEquals(sourceML.ledgers.size(), 4); + assertEquals(shadowML.ledgers.size(), 4); + assertEquals(sourceML.lastConfirmedEntry, shadowML.lastConfirmedEntry); + }); log.info("Source.LCE={},Shadow.LCE={}", sourceML.lastConfirmedEntry, shadowML.lastConfirmedEntry); - Awaitility.await().untilAsserted(()->assertEquals(sourceML.lastConfirmedEntry, shadowML.lastConfirmedEntry)); {// test write entry with ledgerId < currentLedger CompletableFuture future = new CompletableFuture<>(); @@ -130,7 +128,7 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { } {// test write entry with ledgerId > currentLedger - PositionImpl fakePos = PositionImpl.get(newPos.getLedgerId() + 1, newPos.getEntryId()); + Position fakePos = PositionFactory.create(newPos.getLedgerId() + 1, newPos.getEntryId()); CompletableFuture future = new CompletableFuture<>(); shadowML.asyncAddEntry(data, new AsyncCallbacks.AddEntryCallback() { @@ -146,9 +144,12 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { }, fakePos); //This write will be queued unit new ledger is rolled in source. - newPos = sourceML.addEntry(data); // new ledger rolled. - newPos = sourceML.addEntry(data); - Awaitility.await().untilAsserted(() -> assertEquals(shadowML.ledgers.size(), 5)); + sourceML.addEntry(data); // new ledger rolled. + sourceML.addEntry(data); + Awaitility.await().untilAsserted(() -> { + assertEquals(shadowML.ledgers.size(), 5); + assertEquals(shadowML.currentLedgerEntries, 0); + }); assertEquals(future.get(), fakePos); // LCE should be updated. log.info("3.Source.LCE={},Shadow.LCE={}", sourceML.lastConfirmedEntry, shadowML.lastConfirmedEntry); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/cache/InflightReadsLimiterTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/cache/InflightReadsLimiterTest.java index 2b69581ca2c73..89bdda15afb4b 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/cache/InflightReadsLimiterTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/cache/InflightReadsLimiterTest.java @@ -18,45 +18,79 @@ */ package org.apache.bookkeeper.mledger.impl.cache; +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat; +import static org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.InflightReadLimiterUtilization.FREE; +import static org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.InflightReadLimiterUtilization.USED; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; - +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @Slf4j public class InflightReadsLimiterTest { - @Test - public void testDisabled() throws Exception { - - InflightReadsLimiter limiter = new InflightReadsLimiter(0); - assertTrue(limiter.isDisabled()); - - limiter = new InflightReadsLimiter(-1); - assertTrue(limiter.isDisabled()); + @DataProvider + private static Object[][] isDisabled() { + return new Object[][] { + {0, true}, + {-1, true}, + {1, false}, + }; + } - limiter = new InflightReadsLimiter(1); - assertFalse(limiter.isDisabled()); + @Test(dataProvider = "isDisabled") + public void testDisabled(long maxReadsInFlightSize, boolean shouldBeDisabled) throws Exception { + var otel = buildOpenTelemetryAndReader(); + @Cleanup var openTelemetry = otel.getLeft(); + @Cleanup var metricReader = otel.getRight(); + + var limiter = new InflightReadsLimiter(maxReadsInFlightSize, openTelemetry); + assertEquals(limiter.isDisabled(), shouldBeDisabled); + + if (shouldBeDisabled) { + // Verify metrics are not present + var metrics = metricReader.collectAllMetrics(); + assertThat(metrics).noneSatisfy(metricData -> assertThat(metricData) + .hasName(InflightReadsLimiter.INFLIGHT_READS_LIMITER_LIMIT_METRIC_NAME)); + assertThat(metrics).noneSatisfy(metricData -> assertThat(metricData) + .hasName(InflightReadsLimiter.INFLIGHT_READS_LIMITER_USAGE_METRIC_NAME)); + } } @Test public void testBasicAcquireRelease() throws Exception { - InflightReadsLimiter limiter = new InflightReadsLimiter(100); + var otel = buildOpenTelemetryAndReader(); + @Cleanup var openTelemetry = otel.getLeft(); + @Cleanup var metricReader = otel.getRight(); + + InflightReadsLimiter limiter = new InflightReadsLimiter(100, openTelemetry); assertEquals(100, limiter.getRemainingBytes()); + assertLimiterMetrics(metricReader, 100, 0, 100); + InflightReadsLimiter.Handle handle = limiter.acquire(100, null); assertEquals(0, limiter.getRemainingBytes()); assertTrue(handle.success); assertEquals(handle.acquiredPermits, 100); assertEquals(1, handle.trials); + assertLimiterMetrics(metricReader, 100, 100, 0); + limiter.release(handle); assertEquals(100, limiter.getRemainingBytes()); + assertLimiterMetrics(metricReader, 100, 0, 100); } + @Test public void testNotEnoughPermits() throws Exception { - InflightReadsLimiter limiter = new InflightReadsLimiter(100); + InflightReadsLimiter limiter = new InflightReadsLimiter(100, OpenTelemetry.noop()); assertEquals(100, limiter.getRemainingBytes()); InflightReadsLimiter.Handle handle = limiter.acquire(100, null); assertEquals(0, limiter.getRemainingBytes()); @@ -86,7 +120,7 @@ public void testNotEnoughPermits() throws Exception { @Test public void testPartialAcquire() throws Exception { - InflightReadsLimiter limiter = new InflightReadsLimiter(100); + InflightReadsLimiter limiter = new InflightReadsLimiter(100, OpenTelemetry.noop()); assertEquals(100, limiter.getRemainingBytes()); InflightReadsLimiter.Handle handle = limiter.acquire(30, null); @@ -116,7 +150,7 @@ public void testPartialAcquire() throws Exception { @Test public void testTooManyTrials() throws Exception { - InflightReadsLimiter limiter = new InflightReadsLimiter(100); + InflightReadsLimiter limiter = new InflightReadsLimiter(100, OpenTelemetry.noop()); assertEquals(100, limiter.getRemainingBytes()); InflightReadsLimiter.Handle handle = limiter.acquire(30, null); @@ -169,4 +203,25 @@ public void testTooManyTrials() throws Exception { } + private Pair buildOpenTelemetryAndReader() { + var metricReader = InMemoryMetricReader.create(); + var openTelemetry = AutoConfiguredOpenTelemetrySdk.builder() + .addMeterProviderCustomizer((builder, __) -> builder.registerMetricReader(metricReader)) + .build() + .getOpenTelemetrySdk(); + return Pair.of(openTelemetry, metricReader); + } + + private void assertLimiterMetrics(InMemoryMetricReader metricReader, + long expectedLimit, long expectedUsed, long expectedFree) { + var metrics = metricReader.collectAllMetrics(); + assertThat(metrics).anySatisfy(metricData -> assertThat(metricData) + .hasName(InflightReadsLimiter.INFLIGHT_READS_LIMITER_LIMIT_METRIC_NAME) + .hasLongSumSatisfying(longSum -> longSum.hasPointsSatisfying(point -> point.hasValue(expectedLimit)))); + assertThat(metrics).anySatisfy(metricData -> assertThat(metricData) + .hasName(InflightReadsLimiter.INFLIGHT_READS_LIMITER_USAGE_METRIC_NAME) + .hasLongSumSatisfying(longSum -> longSum.hasPointsSatisfying( + point -> point.hasValue(expectedFree).hasAttributes(FREE.attributes), + point -> point.hasValue(expectedUsed).hasAttributes(USED.attributes)))); + } } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/cache/PendingReadsManagerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/cache/PendingReadsManagerTest.java index 6f573ff8d75c8..01976f648aba4 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/cache/PendingReadsManagerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/cache/PendingReadsManagerTest.java @@ -18,8 +18,24 @@ */ package org.apache.bookkeeper.mledger.impl.cache; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.AssertJUnit.assertNotSame; +import static org.testng.AssertJUnit.assertSame; +import io.opentelemetry.api.OpenTelemetry; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.stream.Collectors; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.api.ReadHandle; @@ -30,7 +46,6 @@ import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.EntryImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; - import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.testng.annotations.AfterClass; @@ -38,23 +53,6 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; -import static org.testng.AssertJUnit.assertNotSame; -import static org.testng.AssertJUnit.assertSame; - @Slf4j public class PendingReadsManagerTest { @@ -93,7 +91,7 @@ void setupMocks() { config.setReadEntryTimeoutSeconds(10000); when(rangeEntryCache.getName()).thenReturn("my-topic"); when(rangeEntryCache.getManagedLedgerConfig()).thenReturn(config); - inflighReadsLimiter = new InflightReadsLimiter(0); + inflighReadsLimiter = new InflightReadsLimiter(0, OpenTelemetry.noop()); when(rangeEntryCache.getPendingReadsLimiter()).thenReturn(inflighReadsLimiter); pendingReadsManager = new PendingReadsManager(rangeEntryCache); doAnswer(new Answer() { diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/ManagedLedgerImplUtilsTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/ManagedLedgerImplUtilsTest.java new file mode 100644 index 0000000000000..84842c74cd22a --- /dev/null +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/ManagedLedgerImplUtilsTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.util; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.testng.Assert.assertEquals; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.bookkeeper.test.MockedBookKeeperTestCase; +import org.testng.annotations.Test; + +@Slf4j +public class ManagedLedgerImplUtilsTest extends MockedBookKeeperTestCase { + + @Test + public void testGetLastValidPosition() throws Exception { + final int maxEntriesPerLedger = 5; + + ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); + managedLedgerConfig.setMaxEntriesPerLedger(maxEntriesPerLedger); + managedLedgerConfig.setRetentionSizeInMB(10); + managedLedgerConfig.setRetentionTime(5, TimeUnit.MINUTES); + ManagedLedger ledger = factory.open("testReverseFindPositionOneByOne", managedLedgerConfig); + + String matchEntry = "match-entry"; + String noMatchEntry = "nomatch-entry"; + Predicate predicate = entry -> { + String entryValue = entry.getDataBuffer().toString(UTF_8); + return matchEntry.equals(entryValue); + }; + + // New ledger will return the last position, regardless of whether the conditions are met or not. + Position position = ManagedLedgerImplUtils.asyncGetLastValidPosition((ManagedLedgerImpl) ledger, + predicate, ledger.getLastConfirmedEntry()).get(); + assertEquals(ledger.getLastConfirmedEntry(), position); + + for (int i = 0; i < maxEntriesPerLedger - 1; i++) { + ledger.addEntry(matchEntry.getBytes(StandardCharsets.UTF_8)); + } + Position lastMatchPosition = ledger.addEntry(matchEntry.getBytes(StandardCharsets.UTF_8)); + for (int i = 0; i < maxEntriesPerLedger; i++) { + ledger.addEntry(noMatchEntry.getBytes(StandardCharsets.UTF_8)); + } + + // Returns last position of entry is "match-entry" + position = ManagedLedgerImplUtils.asyncGetLastValidPosition((ManagedLedgerImpl) ledger, + predicate, ledger.getLastConfirmedEntry()).get(); + assertEquals(position, lastMatchPosition); + + ledger.close(); + } + +} diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/PositionAckSetUtilTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/PositionAckSetUtilTest.java index 088220cd35e5f..d9c0c5a11eaee 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/PositionAckSetUtilTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/PositionAckSetUtilTest.java @@ -18,18 +18,19 @@ */ package org.apache.bookkeeper.mledger.util; -import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.pulsar.common.util.collections.BitSetRecyclable; -import org.testng.annotations.Test; - -import java.util.BitSet; - import static org.apache.bookkeeper.mledger.util.PositionAckSetUtil.andAckSet; import static org.apache.bookkeeper.mledger.util.PositionAckSetUtil.compareToWithAckSet; import static org.apache.bookkeeper.mledger.util.PositionAckSetUtil.isAckSetOverlap; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import java.util.BitSet; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.mledger.impl.AckSetState; +import org.apache.bookkeeper.mledger.impl.AckSetStateUtil; +import org.apache.pulsar.common.util.collections.BitSetRecyclable; +import org.testng.annotations.Test; public class PositionAckSetUtilTest { @@ -50,16 +51,16 @@ public void isAckSetRepeatedTest() { @Test public void compareToWithAckSetForCumulativeAckTest() { - PositionImpl positionOne = PositionImpl.get(1, 1); - PositionImpl positionTwo = PositionImpl.get(1, 2); + Position positionOne = PositionFactory.create(1, 1); + Position positionTwo = PositionFactory.create(1, 2); assertEquals(compareToWithAckSet(positionOne, positionTwo), -1); - positionTwo = PositionImpl.get(2, 1); + positionTwo = PositionFactory.create(2, 1); assertEquals(compareToWithAckSet(positionOne, positionTwo), -1); - positionTwo = PositionImpl.get(0, 1); + positionTwo = PositionFactory.create(0, 1); assertEquals(compareToWithAckSet(positionOne, positionTwo), 1); - positionTwo = PositionImpl.get(1, 0); + positionTwo = PositionFactory.create(1, 0); assertEquals(compareToWithAckSet(positionOne, positionTwo), 1); - positionTwo = PositionImpl.get(1, 1); + positionTwo = PositionFactory.create(1, 1); assertEquals(compareToWithAckSet(positionOne, positionTwo), 0); BitSet bitSetOne = new BitSet(); @@ -68,23 +69,24 @@ public void compareToWithAckSetForCumulativeAckTest() { bitSetTwo.set(0, 63); bitSetOne.clear(0, 10); bitSetTwo.clear(0, 10); - positionOne.setAckSet(bitSetOne.toLongArray()); - positionTwo.setAckSet(bitSetTwo.toLongArray()); + positionOne = AckSetStateUtil.createPositionWithAckSet(1, 1, bitSetOne.toLongArray()); + positionTwo = AckSetStateUtil.createPositionWithAckSet(1, 1, bitSetTwo.toLongArray()); assertEquals(compareToWithAckSet(positionOne, positionTwo), 0); bitSetOne.clear(10, 12); - positionOne.setAckSet(bitSetOne.toLongArray()); + AckSetState positionOneAckSetState = AckSetStateUtil.getAckSetState(positionOne); + positionOneAckSetState.setAckSet(bitSetOne.toLongArray()); assertEquals(compareToWithAckSet(positionOne, positionTwo), 2); bitSetOne.set(8, 12); - positionOne.setAckSet(bitSetOne.toLongArray()); + positionOneAckSetState.setAckSet(bitSetOne.toLongArray()); assertEquals(compareToWithAckSet(positionOne, positionTwo), -2); } @Test public void andAckSetTest() { - PositionImpl positionOne = PositionImpl.get(1, 1); - PositionImpl positionTwo = PositionImpl.get(1, 2); + Position positionOne = AckSetStateUtil.createPositionWithAckSet(1, 1, new long[0]); + Position positionTwo = AckSetStateUtil.createPositionWithAckSet(1, 2, new long[0]); BitSet bitSetOne = new BitSet(); BitSet bitSetTwo = new BitSet(); bitSetOne.set(0); @@ -92,20 +94,22 @@ public void andAckSetTest() { bitSetOne.set(4); bitSetOne.set(6); bitSetOne.set(8); - positionOne.setAckSet(bitSetOne.toLongArray()); - positionTwo.setAckSet(bitSetTwo.toLongArray()); + AckSetState positionOneAckSetState = AckSetStateUtil.getAckSetState(positionOne); + positionOneAckSetState.setAckSet(bitSetOne.toLongArray()); + AckSetState positionTwoAckSetState = AckSetStateUtil.getAckSetState(positionTwo); + positionTwoAckSetState.setAckSet(bitSetTwo.toLongArray()); andAckSet(positionOne, positionTwo); - BitSetRecyclable bitSetRecyclable = BitSetRecyclable.valueOf(positionOne.getAckSet()); + BitSetRecyclable bitSetRecyclable = BitSetRecyclable.valueOf(positionOneAckSetState.getAckSet()); assertTrue(bitSetRecyclable.isEmpty()); bitSetTwo.set(2); bitSetTwo.set(4); - positionOne.setAckSet(bitSetOne.toLongArray()); - positionTwo.setAckSet(bitSetTwo.toLongArray()); + positionOneAckSetState.setAckSet(bitSetOne.toLongArray()); + positionTwoAckSetState.setAckSet(bitSetTwo.toLongArray()); andAckSet(positionOne, positionTwo); - bitSetRecyclable = BitSetRecyclable.valueOf(positionOne.getAckSet()); + bitSetRecyclable = BitSetRecyclable.valueOf(positionOneAckSetState.getAckSet()); BitSetRecyclable bitSetRecyclableTwo = BitSetRecyclable.valueOf(bitSetTwo.toLongArray()); assertEquals(bitSetRecyclable, bitSetRecyclableTwo); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/RangeCacheTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/RangeCacheTest.java index 8ce0db4ac4caa..4bcf2cc6c4e35 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/RangeCacheTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/RangeCacheTest.java @@ -23,25 +23,33 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - import com.google.common.collect.Lists; import io.netty.util.AbstractReferenceCounted; import io.netty.util.ReferenceCounted; -import org.apache.commons.lang3.tuple.Pair; -import org.testng.annotations.Test; -import java.util.UUID; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import lombok.Data; +import org.apache.commons.lang3.tuple.Pair; +import org.awaitility.Awaitility; +import org.testng.annotations.Test; public class RangeCacheTest { - class RefString extends AbstractReferenceCounted implements ReferenceCounted { + @Data + class RefString extends AbstractReferenceCounted implements RangeCache.ValueWithKeyValidation { String s; + Integer matchingKey; RefString(String s) { + this(s, null); + } + + RefString(String s, Integer matchingKey) { super(); this.s = s; + this.matchingKey = matchingKey != null ? matchingKey : Integer.parseInt(s); setRefCnt(1); } @@ -65,6 +73,11 @@ public boolean equals(Object obj) { return false; } + + @Override + public boolean matchesKey(Integer key) { + return matchingKey.equals(key); + } } @Test @@ -119,8 +132,8 @@ public void simple() { public void customWeighter() { RangeCache cache = new RangeCache<>(value -> value.s.length(), x -> 0); - cache.put(0, new RefString("zero")); - cache.put(1, new RefString("one")); + cache.put(0, new RefString("zero", 0)); + cache.put(1, new RefString("one", 1)); assertEquals(cache.getSize(), 7); assertEquals(cache.getNumberOfEntries(), 2); @@ -132,9 +145,9 @@ public void customTimeExtraction() { RangeCache cache = new RangeCache<>(value -> value.s.length(), x -> x.s.length()); cache.put(1, new RefString("1")); - cache.put(2, new RefString("22")); - cache.put(3, new RefString("333")); - cache.put(4, new RefString("4444")); + cache.put(22, new RefString("22")); + cache.put(333, new RefString("333")); + cache.put(4444, new RefString("4444")); assertEquals(cache.getSize(), 10); assertEquals(cache.getNumberOfEntries(), 4); @@ -151,12 +164,12 @@ public void customTimeExtraction() { public void doubleInsert() { RangeCache cache = new RangeCache<>(); - RefString s0 = new RefString("zero"); + RefString s0 = new RefString("zero", 0); assertEquals(s0.refCnt(), 1); assertTrue(cache.put(0, s0)); assertEquals(s0.refCnt(), 1); - cache.put(1, new RefString("one")); + cache.put(1, new RefString("one", 1)); assertEquals(cache.getSize(), 2); assertEquals(cache.getNumberOfEntries(), 2); @@ -164,7 +177,7 @@ public void doubleInsert() { assertEquals(s.s, "one"); assertEquals(s.refCnt(), 2); - RefString s1 = new RefString("uno"); + RefString s1 = new RefString("uno", 1); assertEquals(s1.refCnt(), 1); assertFalse(cache.put(1, s1)); assertEquals(s1.refCnt(), 1); @@ -201,10 +214,10 @@ public void getRange() { public void eviction() { RangeCache cache = new RangeCache<>(value -> value.s.length(), x -> 0); - cache.put(0, new RefString("zero")); - cache.put(1, new RefString("one")); - cache.put(2, new RefString("two")); - cache.put(3, new RefString("three")); + cache.put(0, new RefString("zero", 0)); + cache.put(1, new RefString("one", 1)); + cache.put(2, new RefString("two", 2)); + cache.put(3, new RefString("three", 3)); // This should remove the LRU entries: 0, 1 whose combined size is 7 assertEquals(cache.evictLeastAccessedEntries(5), Pair.of(2, (long) 7)); @@ -276,22 +289,53 @@ public void evictions() { } @Test - public void testInParallel() { - RangeCache cache = new RangeCache<>(value -> value.s.length(), x -> 0); - ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); - executor.scheduleWithFixedDelay(cache::clear, 10, 10, TimeUnit.MILLISECONDS); - for (int i = 0; i < 1000; i++) { - cache.put(UUID.randomUUID().toString(), new RefString("zero")); + public void testPutWhileClearIsCalledConcurrently() { + RangeCache cache = new RangeCache<>(value -> value.s.length(), x -> 0); + int numberOfThreads = 8; + @Cleanup("shutdownNow") + ScheduledExecutorService executor = Executors.newScheduledThreadPool(numberOfThreads); + for (int i = 0; i < numberOfThreads; i++) { + executor.scheduleWithFixedDelay(cache::clear, 0, 1, TimeUnit.MILLISECONDS); + } + for (int i = 0; i < 200000; i++) { + cache.put(i, new RefString(String.valueOf(i))); } executor.shutdown(); + // ensure that no clear operation got into endless loop + Awaitility.await().untilAsserted(() -> assertTrue(executor.isTerminated())); + // ensure that clear can be called and all entries are removed + cache.clear(); + assertEquals(cache.getNumberOfEntries(), 0); } @Test public void testPutSameObj() { RangeCache cache = new RangeCache<>(value -> value.s.length(), x -> 0); - RefString s0 = new RefString("zero"); + RefString s0 = new RefString("zero", 0); assertEquals(s0.refCnt(), 1); assertTrue(cache.put(0, s0)); assertFalse(cache.put(0, s0)); } + + @Test + public void testRemoveEntryWithInvalidRefCount() { + RangeCache cache = new RangeCache<>(value -> value.s.length(), x -> 0); + RefString value = new RefString("1"); + cache.put(1, value); + // release the value to make the reference count invalid + value.release(); + cache.clear(); + assertEquals(cache.getNumberOfEntries(), 0); + } + + @Test + public void testRemoveEntryWithInvalidMatchingKey() { + RangeCache cache = new RangeCache<>(value -> value.s.length(), x -> 0); + RefString value = new RefString("1"); + cache.put(1, value); + // change the matching key to make it invalid + value.setMatchingKey(123); + cache.clear(); + assertEquals(cache.getNumberOfEntries(), 0); + } } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java b/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java index 80bb6256591bc..a323ecfeb8ea6 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java @@ -86,7 +86,7 @@ public abstract class BookKeeperClusterTestCase { protected String testName; - @BeforeMethod + @BeforeMethod(alwaysRun = true) public void handleTestMethodName(Method method) { testName = method.getName(); } @@ -148,7 +148,7 @@ public BookKeeperClusterTestCase(int numBookies, int numOfZKNodes, int testTimeo } } - @BeforeTest + @BeforeTest(alwaysRun = true) public void setUp() throws Exception { setUp(getLedgersRootPath()); } @@ -222,7 +222,9 @@ public void tearDown() throws Exception { tearDownException = e; } - executor.shutdownNow(); + if (executor != null) { + executor.shutdownNow(); + } LOG.info("Tearing down test {} in {} ms.", testName, sw.elapsed(TimeUnit.MILLISECONDS)); if (tearDownException != null) { @@ -240,7 +242,9 @@ protected void startZKCluster() throws Exception { zkc = zkUtil.getZooKeeperClient(); metadataStore = new FaultInjectionMetadataStore( MetadataStoreExtended.create(zkUtil.getZooKeeperConnectString(), - MetadataStoreConfig.builder().build())); + MetadataStoreConfig.builder() + .metadataStoreName("metastore-" + getClass().getSimpleName()) + .build())); } /** diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/test/MockedBookKeeperTestCase.java b/managed-ledger/src/test/java/org/apache/bookkeeper/test/MockedBookKeeperTestCase.java index e2101268b09e6..c7685cfaa6594 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/test/MockedBookKeeperTestCase.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/test/MockedBookKeeperTestCase.java @@ -26,6 +26,7 @@ import lombok.SneakyThrows; import org.apache.bookkeeper.client.PulsarMockBookKeeper; import org.apache.bookkeeper.common.util.OrderedScheduler; +import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig; import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; import org.apache.pulsar.metadata.api.MetadataStoreConfig; @@ -70,7 +71,8 @@ public MockedBookKeeperTestCase(int numBookies) { public final void setUp(Method method) throws Exception { LOG.info(">>>>>> starting {}", method); metadataStore = new FaultInjectionMetadataStore( - MetadataStoreExtended.create("memory:local", MetadataStoreConfig.builder().build())); + MetadataStoreExtended.create("memory:local", + MetadataStoreConfig.builder().metadataStoreName("metastore-" + method.getName()).build())); try { // start bookkeeper service @@ -81,13 +83,17 @@ public final void setUp(Method method) throws Exception { } ManagedLedgerFactoryConfig managedLedgerFactoryConfig = new ManagedLedgerFactoryConfig(); - // increase default cache eviction interval so that caching could be tested with less flakyness - managedLedgerFactoryConfig.setCacheEvictionIntervalMs(200); + initManagedLedgerFactoryConfig(managedLedgerFactoryConfig); factory = new ManagedLedgerFactoryImpl(metadataStore, bkc); setUpTestCase(); } + protected void initManagedLedgerFactoryConfig(ManagedLedgerFactoryConfig config) { + // increase default cache eviction interval so that caching could be tested with less flakyness + config.setCacheEvictionIntervalMs(200); + } + protected void setUpTestCase() throws Exception { } @@ -102,7 +108,11 @@ public final void tearDown(Method method) { } try { LOG.info("@@@@@@@@@ stopping " + method); - factory.shutdownAsync().get(10, TimeUnit.SECONDS); + try { + factory.shutdownAsync().get(10, TimeUnit.SECONDS); + } catch (ManagedLedgerException.ManagedLedgerFactoryClosedException e) { + // ignore + } factory = null; stopBookKeeper(); metadataStore.close(); diff --git a/microbench/README.md b/microbench/README.md new file mode 100644 index 0000000000000..780e3a5a1d3e8 --- /dev/null +++ b/microbench/README.md @@ -0,0 +1,43 @@ + + +# Microbenchmarks for Apache Pulsar + +This module contains microbenchmarks for Apache Pulsar. + +## Running the benchmarks + +The benchmarks are written using [JMH](http://openjdk.java.net/projects/code-tools/jmh/). To compile & run the benchmarks, use the following command: + +```bash +# Compile everything for creating the shaded microbenchmarks.jar file +mvn -Pcore-modules,microbench,-main -T 1C clean package + +# run the benchmarks using the standalone shaded jar in any environment +java -jar microbench/target/microbenchmarks.jar +``` + +For fast recompiling of the benchmarks (without compiling Pulsar modules) and creating the shaded jar, you can use the following command: + +```bash +mvn -Pmicrobench -pl microbench clean package +``` + diff --git a/microbench/pom.xml b/microbench/pom.xml new file mode 100644 index 0000000000000..bef02794adbd6 --- /dev/null +++ b/microbench/pom.xml @@ -0,0 +1,132 @@ + + + 4.0.0 + + + org.apache.pulsar + pulsar + 4.0.0-SNAPSHOT + + + microbench + jar + Pulsar Microbenchmarks + + + 1.37 + + + + + microbench + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + microbenchmarks + + + + org.openjdk.jmh.Main + true + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + + + + org.openjdk.jmh + jmh-core + ${jmh.version} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + provided + + + ${project.groupId} + pulsar-broker + ${project.version} + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + org.apache.maven.plugins + maven-surefire-plugin + + true + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + + + + + + + \ No newline at end of file diff --git a/microbench/src/main/java/org/apache/pulsar/broker/qos/AsyncTokenBucketBenchmark.java b/microbench/src/main/java/org/apache/pulsar/broker/qos/AsyncTokenBucketBenchmark.java new file mode 100644 index 0000000000000..4c069e72ea3ba --- /dev/null +++ b/microbench/src/main/java/org/apache/pulsar/broker/qos/AsyncTokenBucketBenchmark.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.qos; + +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; + +@Fork(3) +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@State(Scope.Thread) +public class AsyncTokenBucketBenchmark { + private AsyncTokenBucket asyncTokenBucket; + private DefaultMonotonicSnapshotClock monotonicSnapshotClock = + new DefaultMonotonicSnapshotClock(TimeUnit.MILLISECONDS.toNanos(8), System::nanoTime); + + @Setup(Level.Iteration) + public void setup() { + long ratePerSecond = 100_000_000; + asyncTokenBucket = AsyncTokenBucket.builder().rate(ratePerSecond).clock(monotonicSnapshotClock) + .initialTokens(2 * ratePerSecond).capacity(2 * ratePerSecond).build(); + } + + @TearDown(Level.Iteration) + public void teardown() { + monotonicSnapshotClock.close(); + } + + @Threads(1) + @Benchmark + @Measurement(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1) + @Warmup(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1) + public void consumeTokensBenchmark001Threads() { + asyncTokenBucket.consumeTokens(1); + } + + @Threads(10) + @Benchmark + @Measurement(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1) + @Warmup(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1) + public void consumeTokensBenchmark010Threads() { + asyncTokenBucket.consumeTokens(1); + } + + @Threads(100) + @Benchmark + @Measurement(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1) + @Warmup(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1) + public void consumeTokensBenchmark100Threads() { + asyncTokenBucket.consumeTokens(1); + } +} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/package-info.java b/microbench/src/main/java/org/apache/pulsar/broker/qos/package-info.java similarity index 88% rename from pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/package-info.java rename to microbench/src/main/java/org/apache/pulsar/broker/qos/package-info.java index a5b5e9cfc0d6e..ccea21a210f86 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/package-info.java +++ b/microbench/src/main/java/org/apache/pulsar/broker/qos/package-info.java @@ -17,6 +17,6 @@ * under the License. */ /** - * This package contains decoder for SchemaType.JSON. + * Benchmarks for Pulsar broker Quality of Service (QoS) related classes. */ -package org.apache.pulsar.sql.presto.decoder.json; \ No newline at end of file +package org.apache.pulsar.broker.qos; \ No newline at end of file diff --git a/build/docker/README.md b/microbench/src/main/resources/log4j2.xml similarity index 57% rename from build/docker/README.md rename to microbench/src/main/resources/log4j2.xml index bc2fe532f74e0..7ec5ed8169a66 100644 --- a/build/docker/README.md +++ b/microbench/src/main/resources/log4j2.xml @@ -1,3 +1,4 @@ + - -This folder contains a Docker image that can used to compile the Pulsar C++ client library -and website in a reproducible environment. - -```shell -docker build -t pulsar-build . -``` - -The image is already available at https://hub.docker.com/r/apachepulsar/pulsar-build - -Example: `apachepulsar/pulsar-build:ubuntu-16.04` - -## Build and Publish pulsar-build image - -> Only committers have permissions on publishing pulsar images to `apachepulsar` docker hub. - -### Build pulsar-build image - - -```shell -docker build -t apachepulsar/pulsar-build:ubuntu-16.04 . -``` - -### Publish pulsar-build image - -```shell -publish.sh -``` + + + + + + + + + + + + \ No newline at end of file diff --git a/pip/README.md b/pip/README.md new file mode 100644 index 0000000000000..216cdd56298c6 --- /dev/null +++ b/pip/README.md @@ -0,0 +1,123 @@ +# Pulsar Improvement Proposal (PIP) + +## What is a PIP? + +The PIP is a "Pulsar Improvement Proposal" and it's the mechanism used to propose changes to the Apache Pulsar codebases. + +The changes might be in terms of new features, large code refactoring, changes to APIs. + +In practical terms, the PIP defines a process in which developers can submit a design doc, receive feedback and get the "go ahead" to execute. + +### What is the goal of a PIP? + +There are several goals for the PIP process: + +1. Ensure community technical discussion of major changes to the Apache Pulsar codebase. + +2. Provide clear and thorough design documentation of the proposed changes. Make sure every Pulsar developer will have enough context to effectively perform a code review of the Pull Requests. + +3. Use the PIP document to serve as the baseline on which to create the documentation for the new feature. + +4. Have greater scrutiny to changes are affecting the public APIs (as defined below) to reduce chances of introducing breaking changes or APIs that are not expressing an ideal semantic. + +It is not a goal for PIP to add undue process or slow-down the development. + +### When is a PIP required? + +* Any new feature for Pulsar brokers or client +* Any change to the public APIs (Client APIs, REST APIs, Plugin APIs) +* Any change to the wire protocol APIs +* Any change to the API of Pulsar CLI tools (eg: new options) +* Any change to the semantic of existing functionality, even when current behavior is incorrect. +* Any large code change that will touch multiple components +* Any changes to the metrics (metrics endpoint, topic stats, topics internal stats, broker stats, etc.) +* Any change to the configuration + +### When is a PIP *not* required? + +* Bug-fixes +* Simple enhancements that won't affect the APIs or the semantic +* Small documentation changes +* Small website changes +* Build scripts changes (except: a complete rewrite) + +### Who can create a PIP? + +Any person willing to contribute to the Apache Pulsar project is welcome to create a PIP. + +## How does the PIP process work? + +A PIP proposal can be in these states: +1. **DRAFT**: (Optional) This might be used for contributors to collaborate and to seek feedback on an incomplete version of the proposal. + +2. **DISCUSSION**: The proposal has been submitted to the community for feedback and approval. + +3. **ACCEPTED**: The proposal has been accepted by the Pulsar project. + +4. **REJECTED**: The proposal has not been accepted by the Pulsar project. + +5. **IMPLEMENTED**: The implementation of the proposed changes have been completed and everything has been merged. + +6. **RELEASED**: The proposed changes have been included in an official + Apache Pulsar release. + + +The process works in the following way: + +1. Fork https://github.com/apache/pulsar repository (Using the fork button on GitHub). +2. Clone the repository, and on it, copy the file `pip/TEMPLATE.md` and name it `pip-xxx.md`. The number `xxx` should be the next sequential number after the last contributed PIP. You view the list of contributed PIPs (at any status) as a list of Pull Requests having a "PIP" label. Use the link [here](https://github.com/apache/pulsar/pulls?q=is%3Apr+label%3APIP+) as shortcut. +3. Write the proposal following the section outlined by the template and the explanation for each section in the comment it contains (you can delete the comment once done). + * If you need diagrams, avoid attaching large files. You can use [MermaidJS](https://mermaid.js.org/) as simple language to describe many types of diagrams. +4. Create GitHub Pull request (PR). The PR title should be `[improve][pip] PIP-xxx: {title}`, where the `xxx` match the number given in previous step (file-name). Replace `{title}` with a short title to your proposal. + *Validate* again that your number does not collide, by step (2) numbering check. +5. The author(s) will email the dev@pulsar.apache.org mailing list to kick off a discussion, using subject prefix `[DISCUSS] PIP-xxx: {PIP TITLE}`. The discussion will happen in broader context either on the mailing list or as general comments on the PR. Many of the discussion items will be on particular aspect of the proposal, hence they should be as comments in the PR to specific lines in the proposal file. +6. Update file with a link to the discussion on the mailing. You can obtain it from [Apache Pony Mail](https://lists.apache.org/list.html?dev@pulsar.apache.org). +7. Based on the discussion and feedback, some changes might be applied by authors to the text of the proposal. They will be applied as extra commits, making it easier to track the changes. +8. Once some consensus is reached, there will be a vote to formally approve the proposal. The vote will be held on the dev@pulsar.apache.org mailing list, by + sending a message using subject `[VOTE] PIP-xxx: {PIP TITLE}`. Make sure to include a link to the PIP PR in the body of the message. + Make sure to update the PIP with a link to the vote. You can obtain it from [Apache Pony Mail](https://lists.apache.org/list.html?dev@pulsar.apache.org). + Everyone is welcome to vote on the proposal, though only the vote of the PMC members will be considered binding. + The requirement is to have at least one binding +1 vote from a lazy majority if no binding -1 votes have been cast on the PIP. + The vote should stay open for at least 48 hours. +9. When the vote is closed, if the outcome is positive, ask a PMC member (using voting thread on mailing list) to merge the PR. +10. If the outcome is negative, please close the PR (with a small comment that the close is a result of a vote). + +All the future implementation Pull Requests that will be created, should always reference the PIP-XXX in the commit log message and the PR title. +It is advised to create a master GitHub issue to formulate the execution plan and track its progress. + +### Example +* Eve ran into some issues with the client metrics - she needed a metric which was missing. +* She read the code a bit, and has an idea what metrics she wishes to add. +* She summarized her idea and direction in an email to the DEV mailing list (she located it on +[Discussions]([url](https://pulsar.apache.org/community/#section-discussions)) section on the website. +* She didn't get any response from the community, so she joined the next +[community meeting]([url](https://github.com/apache/pulsar/wiki/Community-Meetings)). There Matteo Merli and Asaf helped +setup a channel in Slack to brainstorm the idea and meet on Zoom with a few Pulsar contributors (e.g. Lari and Tison). +* Once Eve had a good enough context, and good design outline, she opened a new branch in her Pulsar repository, duplicated +TEMPLATE.md and created pip-xxx.MD (the number she will take later). +* She followed the template and submitted the pip as a new PR to pulsar repository. +* Once the PR was created, she modified the version to match the rules described at step 2, both for PR title and file name. +* She sent an email to the DEV mailing list, titled "[DISCUSS] PIP-123: Adding metrics for ..." , described shortly in the +email what the PIP was about and gave a link. +* She got no response for anyone for 2 weeks, so she nudged the people that helped + her brainstorm (e.g. Lary and Tison) and pinged in #dev that she needs more reviewers. +* Once she got 3 reviews from PMC members and the community had at least a few days from the moment + the PR was announceed on DEV, she sent a vote email to the DEV mailing list titled + "[VOTE] PIP-123: Adding metrics for ...". +* She nudged the reviewers to reply with a binding vote, waited for 2-3 days, and then + concluded the vote by sending a reply tallying up the binding and non-binding votes. +* She updated the PIP with links to discuss and vote emails, and then asked a PMC member + who voted +1, to merge (using GitHub mentionon the PR). + + +## List of PIPs + +### Historical PIPs +You can the view list of PIPs previously managed by GitHub wiki or GitHub issues [here](https://github.com/apache/pulsar/wiki#pulsar-improvement-proposals) + +### List of PIPs +1. You can view all PIPs (besides the historical ones) as the list of Pull Requests having title starting with `[improve][pip] PIP-`. Here is the [link](https://github.com/apache/pulsar/pulls?q=is%3Apr+title%3A%22%5Bpip%5D%5Bdesign%5D+PIP-%22) for it. + - Merged PR means the PIP was accepted. + - Closed PR means the PIP was rejected. + - Open PR means the PIP was submitted and is in the process of discussion. +2. You can also take a look at the file in the `pip` folder. Each one is an approved PIP. diff --git a/.github/ISSUE_TEMPLATE/pip.md b/pip/TEMPLATE.md similarity index 84% rename from .github/ISSUE_TEMPLATE/pip.md rename to pip/TEMPLATE.md index 520826f8b1ceb..0eaed8f2b6810 100644 --- a/.github/ISSUE_TEMPLATE/pip.md +++ b/pip/TEMPLATE.md @@ -1,19 +1,20 @@ ---- -name: PIP -about: Submit a Pulsar Improvement Proposal (PIP) -title: 'PIP-XYZ: ' -labels: PIP ---- - +# PIP-XXX: Proposal title + # Background knowledge # Motivation @@ -46,7 +47,7 @@ Describe the problem this proposal is trying to solve. ## Out of Scope @@ -76,9 +77,9 @@ DON'T @@ -90,15 +91,15 @@ Remove the sections you are not changing. Clearly mark any changes which are BREAKING backward compatability. --> -### Public API +### Public API @@ -123,7 +124,7 @@ For each metric provide: @@ -136,18 +137,24 @@ An important aspect to consider is also multi-tenancy: Does the feature I'm addi If there is uncertainty for this section, please submit the PIP and request for feedback on the mailing list. --> -# Backward & Forward Compatability +# Backward & Forward Compatibility + +## Upgrade -## Revert + + +## Downgrade / Rollback -## Upgrade +## Pulsar Geo-Replication Upgrade & Downgrade/Rollback Considerations # Alternatives diff --git a/pip/pip-264.md b/pip/pip-264.md new file mode 100644 index 0000000000000..aa1420aa75698 --- /dev/null +++ b/pip/pip-264.md @@ -0,0 +1,1136 @@ +# TOC + +* [TOC](#toc) +* [Preface](#preface) +* [TL;DR](#tldr) +* [Background](#background) + * [What are metrics?](#what-are-metrics) + * [Messaging Metrics](#messaging-metrics) + * [OpenTelemetry Basic Concepts](#opentelemetry-basic-concepts) +* [Motivation](#motivation) + * [Lack of a single way to define and record metrics](#lack-of-a-single-way-to-define-and-record-metrics) + * [High topic count (cardinality) is not supported](#high-topic-count-cardinality-is-not-supported) + * [Summary is widely used, but not aggregatable across brokers / labels](#summary-is-widely-used-but-not-aggregatable-across-brokers--labels) + * [Existing Prometheus Export is hard to use](#existing-prometheus-export-is-hard-to-use) + * [Integrating metrics as Plugin author is hard, labor-intensive and prevents common functionality](#integrating-metrics-as-plugin-author-is-hard-labor-intensive-and-prevents-common-functionality) + * [Lack of ability to introduce another metrics format](#lack-of-ability-to-introduce-another-metrics-format) + * [Adding Rates is error-prone](#adding-rates-is-error-prone) + * [Inline metrics documentation is lacking](#inline-metrics-documentation-is-lacking) + * [Histograms can’t be visualized or used since not Prometheus conformant](#histograms-cant-be-visualized-or-used-since-not-prometheus-conformant) + * [Some metrics are delta-reset, making it easy to lose data on occasions](#some-metrics-are-delta-reset-making-it-easy-to-lose-data-on-occasions) + * [Prometheus client metrics use static registry, making them susceptible to flaky tests](#prometheus-client-metrics-use-static-registry-making-them-susceptible-to-flaky-tests) + * [Function custom metrics are both delta reset and limited in types to summary](#function-custom-metrics-are-both-delta-reset-and-limited-in-types-to-summary) + * [Inconsistent reporting of partitioned topics](#inconsistent-reporting-of-partitioned-topics) + * [System metrics manually scraped from function processes override each other](#system-metrics-manually-scraped-from-function-processes-override-each-other) + * [No consistent naming convention used throughout pulsar metrics](#no-consistent-naming-convention-used-throughout-pulsar-metrics) +* [Goals](#goals) + * [In Scope](#in-scope) + * [Out of Scope](#out-of-scope) +* [High Level Design](#high-level-design) + * [Consolidating to OpenTelemetry](#consolidating-to-opentelemetry) + * [Aggregate and Filtering to solve cardinality issues](#aggregate-and-filtering-to-solve-cardinality-issues) + * [Aggregation](#aggregation) + * [Filtering](#filtering) + * [Removing existing metric toggles](#removing-existing-metric-toggles) + * [Summary](#summary) + * [Changing the way we measure metrics](#changing-the-way-we-measure-metrics) + * [Moving topic-level Histograms to namespace and broker level only](#moving-topic-level-histograms-to-namespace-and-broker-level-only) + * [Integrating Messaging metrics into Open Telemetry](#integrating-messaging-metrics-into-open-telemetry) + * [Switching Summary to Histograms, in namespace/broker level only](#switching-summary-to-histograms-in-namespacebroker-level-only) + * [Specifying units for Histograms](#specifying-units-for-histograms) + * [Removing Delta Reset](#removing-delta-reset) + * [Reporting topic and partition all the time](#reporting-topic-and-partition-all-the-time) + * [Metrics Exporting](#metrics-exporting) + * [Avoiding static metric registry](#avoiding-static-metric-registry) + * [Metrics documentation](#metrics-documentation) + * [Integration with BK](#integration-with-bk) + * [Integrating with Pulsar Plugins](#integrating-with-pulsar-plugins) + * [Supporting Admin Rest API Statistics endpoints](#supporting-admin-rest-api-statistics-endpoints) + * [Fixing Rate](#fixing-rate) + * [Function metrics](#function-metrics) + * [Background](#background-1) + * [Solving it in Open Telemetry](#solving-it-in-open-telemetry) + * [Collecting metrics](#collecting-metrics) + * [Removing 1min metrics](#removing-1min-metrics) + * [Supporting `getMetrics` RPC](#supporting-getmetrics-rpc) + * [Removing `resetMetrics` RPC](#removing-resetmetrics-rpc) + * [Supporting Python and Go Functions](#supporting-python-and-go-functions) + * [Summary](#summary-1) +* [Detailed Design](#detailed-design) + * [Topic Metric Group configuration](#topic-metric-group-configuration) + * [Integration with Pulsar Plugins](#integration-with-pulsar-plugins) + * [Why OpenTelemetry?](#why-opentelemetry) + * [What’s good about OTel?](#whats-good-about-otel) + * [What we need to fix in OpenTelemetry](#what-we-need-to-fix-in-opentelemetry) + * [Specifying units for histograms](#specifying-units-for-histograms-1) + * [Filtering Configuration](#filtering-configuration) + * [Which summaries / histograms are effected?](#which-summaries--histograms-are-effected) + * [Summaries](#summaries) + * [Histograms](#histograms) + * [Fixing Grafana Dashboards in repository](#fixing-grafana-dashboards-in-repository) +* [Backward Compatibility](#backward-compatibility) + * [Breaking changes](#breaking-changes) +* [API Changes](#api-changes) +* [Security](#security) +* [Links](#links) + + +# Preface + +Roughly 11 months ago, I started working on solving the biggest issue with Pulsar metrics: the lack of ability to monitor a pulsar broker with a large topic count: 10k, 100k, and future support of 1M. This started by mapping the existing functionality and then enumerating all the problems I saw (all documented in this [doc](https://docs.google.com/document/d/1vke4w1nt7EEgOvEerPEUS-Al3aqLTm9cl2wTBkKNXUA/edit?usp=sharing)). + +This PIP is a parent PIP. It aims to gradually solve (using sub-PIPs) all the current metric system's problems and provide the ability to monitor a broker with a large topic count, which is currently lacking. As a parent PIP, it will describe each problem and its solution at a high level, leaving fine-grained details to the sub-PIPs. The parent PIP ensures all solutions align and does not contradict each other. + +The basic building block to solve the monitoring ability of large topic count is aggregating internally (to topic groups) and adding fine-grained filtering. We could have shoe-horned it into the existing metric system, but we thought adding that to a system already ingrained with many problems would be wrong and hard to do gradually, as so many things will break. This is why the second-biggest design decision presented here is consolidating all existing metric libraries into a single one - [OpenTelemetry](https://opentelemetry.io/). The parent PIP will explain why OpenTelemetry was chosen out of existing solutions and why it far exceeds all other options. I’ve been working closely with the OpenTelemetry community in the past eight months: brainstorming this integration, and raising issues, in an afford to remove serious blockers to making this migration successful. + +I made every effort to summarize this document so that it can be concise yet clear. I understand it is an effort to read it and, more so, provide meaningful feedback on such a large document; hence I’m very grateful for each individual who does so. + +I think this design will help improve the user experience immensely, so it is worth the time spent reading it. + +# TL;DR +Working with Metrics today as a user or a developer is hard and has many severe issues. + +From the user perspective: + +* One of Pulsar strongest features is "cheap" topics, so you can easily have 10k - 100k topics per broker. Once you do that, you quickly learn that the amount of metrics you export via "/metrics" (Prometheus style endpoint) becomes massive. The cost to store them becomes too high, queries time-out or even "/metrics" endpoint itself times out, due to heavy performance cost in terms of CPU and memory to process so many metrics. +* The only option Pulsar gives you today is all-or-nothing filtering and very crude aggregation. You switch metrics from topic aggregation level to namespace aggregation level. Also, you can turn off producer and consumer level metrics. You end up doing it all leaving you "blind", looking at the metrics from a namespace level which is too high level. You end up conjuring all kinds of scripts on top of topic stats endpoint to glue some aggregated metrics view for the topics you need. +* Summaries (metric type giving you quantiles like p95) which are used in Pulsar, can't be aggregated across topics / brokers due to its inherent design. +* Plugin authors spend too much time on defining and exposing metrics to Pulsar, since the only interface Pulsar offers is writing your metrics by your self as UTF-8 bytes in Prometheus Text Format to byte stream interface given to you. +* Pulsar histograms are exported in a way that is not conformant with Prometheus, which means you can't get the p95 quantile on such histograms, making them very hard to use in day to day life. +* Too many metrics are rates which also delta reset every interval you configure in Pulsar and restart, instead of relying on cumulative (ever-growing) counters and letting Prometheus use its rate function. +* And many more issues + +From the developer perspective: + +* There are 4 different ways to define and record metrics in Pulsar: Pulsar own metrics library, Prometheus Java Client, Bookkeeper metrics library and plain native Java SDK objects (AtomicLong, ...). It's very confusing for the developer and creates inconsistencies for the end user (e.g. Summary, for example, is different in each). +* Patching your metrics into "/metrics" Prometheus endpoint is confusing, cumbersome and error-prone. +* Many more + +This proposal offers several key changes to solve that: + +* Cardinality (supporting 10k-100k topics per broker) is solved by introducing a new aggregation level for metrics called Topic Metric Group. Using configuration, you specify for each topic its group (using wildcard/regex). This allows you to "zoom" out to a more detailed granularity level, like groups instead of namespaces, which you control how many groups you'll have, hence solving the cardinality issue, without sacrificing level of detail too much. +* Fine-grained filtering mechanism, dynamic. You'll have rule-based dynamic configuration, allowing you to specify per namespace/topic/group which metrics you'd like to keep/drop. Rules allow you to set the default to have a small amount of metrics in group and namespace level only and drop the rest. When needed, you can add an override rule to "open" up a certain group to have more metrics at higher granularity (topic or even consumer/producer level). Since it's dynamic, you "open" such a group when you see it's misbehaving, see it in topic level, and when all resolved, you can "close" it. A bit of similar experience to logging levels in Log4j or Logback, that you default and override per class/package. + +Aggregation and Filtering combined solves the cardinality without sacrificing the level of detail when needed and most importantly, you determine which topic/group/namespace it happens on. + +Since this change is so invasive, it requires a single metrics library to implement all of it on top of; Hence the third big change point is consolidating all four ways to define and record metrics to a single one, a new one: OpenTelemetry Metrics (Java SDK, and also Python and Go for the Pulsar Function runners). +Introducing OpenTelemetry (OTel) solves also the biggest pain point from the developer perspective, since it's a superb metrics library offering everything you need, and there is going to be a single way - only it. Also, it solves the robustness for Plugin authors which will use OpenTelemetry. It so happens that it also solves all the numerous problems described in the doc itself. + +The solution will be introduced as another layer with feature toggles, so you can work with existing system, and/or OTel, until gradually deprecating existing system. Pulsar OTel Metrics will support exporting as Prometheus HTTP endpoint (`/metrics` but different port) for backward compatability and also OTLP, so you can push the metrics to OTel Collector and from there ship it to any destination. + +It's a big breaking change for Pulsar users on many fronts: names, semantics, configuration. Read at the end of this doc to learn exactly what will change for the user (in high level). + +In my opinion, it will make Pulsar user experience so much better, they will want to migrate to it, despite the breaking change. + +This was a very short summary. You are most welcomed to read the full design document below and express feedback, so we can make it better. + +# Background +* [What are metrics?](#what-are-metrics) +* [Messaging Metrics](#messaging-metrics) +* [OpenTelemetry Basic Concepts](#opentelemetry-basic-concepts) + + +## What are metrics? + +Any software we know in the world today, exposes metrics to show what transpires in it in an aggregated way. A metric is defined as: + +- A name - e.g. `pulsar_rate_in` +- Attributes/labels - the context in which the following number applies for. For example `cluster=europe-cluster, topic=orders` +- Timestamp - the time at which the following number was measured. Usually presented in epoch seconds: `1679403600`, which means `Tuesday, March 21, 2023 1:00:00 PM` +- Value - a numerical value. For example, 15,345. For `pulsar_rate_in` it means 15,345 bytes received in the interval (e.g., 1min) + +Composing it all together looks like this: + +`pulsar_rate_in {cluster="europe-cluster", topic="orders"} 1679403600 15345` + +Metrics usually come in all kinds of types: + +- Counter - a number that keeps increasing. Most of the time used to measure a rate. +- Gauge - a number that can go up and down. Example: How many HTTP requests are in-flight right now to the Admin API for a given broker? +- Histogram - most of the time, it’s an explicit bucket histogram: you define several buckets, and each bucket has a range of values. Each time you report a value to the histogram, it finds the bucket in which the value falls inside that range and increases its counter by 1. +- Summary - You record values to it, for example, the latency of API call to write to Bookkeeper. Once requested, it will give you the percentiles of those values, over the last X minutes or until it will be reset. For example, The 95 percentile of the last two minutes, is a number for which 95% of all values recorded to the summary over the last two minutes are below it. + +As opposed to logs which tell you a step-by-step story, metrics give you an aggregated view of Pulsar behavior over time. + +## Messaging Metrics + +Pulsar main feature is messaging: receiving messages from producers, and dispatching messages to consumers via subscriptions. + +The metrics related to messaging are divided into a couple of aggregation levels: broker, namespace, topic, subscription, consumer and producer. + +Some levels can have high cardinality, hence Pulsar offers several configuration to minimize the amount of unique time series exported, by the following toggles: + +- `exposeTopicLevelMetricsInPrometheus` - Settings this value to false will cause metrics to be reported in namespace level granularity only, while if true, the metrics will be reported in topic level granularity. +- `exposeConsumerMetricsInPrometheus` - Setting this value to false will filter out any consumer level metric, i.e. filtering out `pulsar_consumer_*` metrics. +- `exposeProducerLevelMetricsInPrometheus` - Setting this value to false will filter out any producer level metric, i.e. filtering out `pulsar_producer_*` metrics. + +## OpenTelemetry Basic Concepts + +- Measurement - the number you record. It can be for example “5” ms in the case of HTTP request latency histogram, or +2 in the case of an increment to a counter +- Instrument - the object through which you record measurements. +- Instrument Types + - Counter + - A number that only increases + - UpDown Counter + - A number that can increase or decrease + - Gauge + - A number that can increase or decrease, but can’t be aggregated across attributes. For example: temperature. If room 1 has 35c and room 2 has 40c, you can’t add them to get a meaningful number as opposed to number of requests. + - Histogram + - Records numbers and when asked shows a statistical analysis on it. Example: explicit bucket histogram, which shows count per buckets, where each bucket represents a value range. +- Attributes + - List of (name, value) pairs. Example: `cluster=eu-cluster, topic=orders` + - Usually when recording a value to an instrument, e.g. counter, you do it in the context of an attribute set. +- Meter + - A factory object through which you create instruments. All created through it belong to it. + - A Meter has a name and a version. Pulsar can have “pulsar” meter with it’s corresponding version. Plugins can have their own meter, with matching version. + - The name and version will be available via attributes when exported to Prometheus, or any other time-series database. + +# Motivation + +* [Lack of a single way to define and record metrics](#lack-of-a-single-way-to-define-and-record-metrics) +* [High topic count (cardinality) is not supported](#high-topic-count-cardinality-is-not-supported) +* [Summary is widely used, but not aggregatable across brokers / labels](#summary-is-widely-used-but-not-aggregatable-across-brokers--labels) +* [Existing Prometheus Export is hard to use](#existing-prometheus-export-is-hard-to-use) +* [Integrating metrics as Plugin author is hard, labor-intensive and prevents common functionality](#integrating-metrics-as-plugin-author-is-hard-labor-intensive-and-prevents-common-functionality) +* [Lack of ability to introduce another metrics format](#lack-of-ability-to-introduce-another-metrics-format) +* [Adding Rates is error-prone](#adding-rates-is-error-prone) +* [Inline metrics documentation is lacking](#inline-metrics-documentation-is-lacking) +* [Histograms can’t be visualized or used since not Prometheus conformant](#histograms-cant-be-visualized-or-used-since-not-prometheus-conformant) +* [Some metrics are delta-reset, making it easy to lose data on occasions](#some-metrics-are-delta-reset-making-it-easy-to-lose-data-on-occasions) +* [Prometheus client metrics use static registry, making them susceptible to flaky tests](#prometheus-client-metrics-use-static-registry-making-them-susceptible-to-flaky-tests) +* [Function custom metrics are both delta reset and limited in types to summary](#function-custom-metrics-are-both-delta-reset-and-limited-in-types-to-summary) +* [Inconsistent reporting of partitioned topics](#inconsistent-reporting-of-partitioned-topics) +* [System metrics manually scraped from function processes override each other](#system-metrics-manually-scraped-from-function-processes-override-each-other) +* [No consistent naming convention used throughout pulsar metrics](#no-consistent-naming-convention-used-throughout-pulsar-metrics) + +The current metric system has several problems which act as the motivation for this PIP. Each subsection below explains the background to the problem and the actual problem. No prior knowledge is required. + +## Lack of a single way to define and record metrics + +In Pulsar there are multiple ways to define and record metrics - i.e. several metric libraries: + +- Prometheus Client + - Prometheus has a client library in Java, providing objects to define and record several types of metrics: Gauge, Histogram, Counter (a.k.a. Collectors) + - Majority of time, the static Collector registry is used. In some occasions, collector registries are created. +- Pulsar Metrics library + - Pulsar’s own metric library, providing objects to define and record several types of metrics: + - Histogram: `StatsBuckets` + - Rates: `Rate`. + - Summary: `Summary` - An extension for Prometheus Client library providing a more performant version of Summary. +- Bookkeeper Metrics API Implementation + - Apache Bookkeeper (BK) has its own metrics library, divided into an API and SDK (implementation). + - Pulsar has implemented the API, for several purposes + - Integrate BK client metrics into Pulsar metrics exporter (**described below**). + - Use BK objects which uses BK Metrics API and integrate their metrics into Pulsar metrics exporter. Examples: `OrderedExecutors`. + - The BK code used in Pulsar is BK Client and OrderedExecutors. + - Support Pulsar code which directly uses this API in Pulsar. `PulsarZooKeeperClient` and several Pulsar plugins are the most prominent examples +- Native Java SDK + - Plain java objects: `LongAdder` to act as Counters, `AtomicLong` or primitive long with atomic updater to act as Gauge. + +Having multiple metric libraries is a problem for several reasons: + +- Confusing + - Developers don’t really know which one to use +- Completely different + - Each one of them is different from the other. Prometheus client uses labels to record values, while Pulsar Metrics library and the Native Java SDK stores the labels separately and stitches them together only upon exporting +- Different implementations for same exported type + - `Summary` by Pulsar Metrics library uses a fixed time window (1 min) to reset and start accumulating metrics, while Pulsar Client summary uses a moving time window of 10 minutes. + - `StatsBucket` by Pulsar Metrics library resets its bucket counters every interval (1 min) while Prometheus Client `Histogram` does not. + - This creates confusion both for developers and users +- Different usage + - With Pulsar Metrics library and Java SDK, you must remember to follow certain conventions to reset the metrics explicitly and register them for export explicitly. With BK Metrics implementation and Prometheus Client, exporting is implicitly done for you. + +I would summarize it with one word: confusion. + +## High topic count (cardinality) is not supported + +Pulsar, as users know, is unique by allowing you to use a very high number of topics in a cluster - up to 1M. It’s not uncommon to find a broker with 10k up to 100k topics hosted on it. + +For each topic, Pulsar exposes roughly 80 - 100 unique time series (metrics). A single broker with 100k topics will emit 10M unique time series (UTS). This usually results in the following for the user: + +- A single Prometheus, even for a single broker, will not suffice. This forces the user to switch to complicated distributed time series systems like Cortex, M3, VictoriaMetrics as they can horizontally scale. +- If the user works with an observability vendor like DataDog or [Logz.io](http://Logz.io), the cost of 10M UTS per broker, make it too expensive to monitor. +- If the user is fortunate enough to have its own team dedicated to deploying a time series database, the query will most probably timeout due to the huge amount of time-series required to read. For vendors, it will either time out or make the query cost too expensive to use. +- Heavy performance cost on Pulsar in terms of CPU and memory allocation to handle the huge amount of topics which translate to many attribute sets. + +Hence, the common user behavior is: + +- Toggle-off topic level metrics and below (consumer/producer/subscription), leaving them with only namespace level monitoring +- Develop their own scripts to call topics `stats` Admin API, to filter only the metrics they need, and aggregate to a level with reasonable cardinality. +- Ship it to a vendor and bear the high cost + +The filtering supported today is toggle-based (all or nothing) for certain levels, hence very coarse. You can toggle between namespace to topic level. If you chose topic level, you can toggle consumers and producers level metrics separately. + +The aggregations provided today are: + +- Broker level (just merged this March 2023) +- Namespace level or Topic level (normal topics and the partitions of a topic, not partitioned topic level) + +## Summary is widely used, but not aggregatable across brokers / labels + +Summary, as explain in the Background section, is used to provide quantile values like p95, p75, for certain measurements - mostly latencies, but sometimes sizes. It is widely used in Pulsar: Ledger Offloader, Resource Groups, Replicated Subscriptions, Schema Registry, Broker Operability, Transaction Managements, Pulsar Functions and Topics, just to name a few. + +The biggest problem with Summaries is that quantiles are not aggregatable mathematically. You can’t know what is the p95 of schema registry “get” operation latency across the cluster by knowing each p95 per broker. Any math you’ll do on it will result in large numerical error. This means that beyond the scope of a broker or label set (i.e. topic/partition) it’s unusable. + +For example, the user is mostly interested in the topic publish latency, and not the topic-partition publish latency. Without aggregating, it’s impossible to know the publish latency for a topic with partitions, because we need to aggregate the publish latency of each topic-partition, but we can’t aggregate summaries as explained above. + +This is not the case for Explicit Bucket Histograms or the new type called Exponential Bucket histogram. They are aggregatable and produce quantiles (extrapolating) with a small margin of error. + +## Existing Prometheus Export is hard to use + +Most metrics framework offers you objects you create and then register to a registry. The exporting is taken care of for you. In Prometheus for example, you create a Counter, and register it to static collector registry. The exporter simply exports all collectors (the counter being one) registered to the static registry. + +In Pulsar, since it has 4 different libraries you can use to define a metric, the exporter had to be written by Pulsar, to patch all of them together into a single response to the GET`/metrics` request. + +If you have used Prometheus Client, you’re all set, as that integration was written for you. The problem is most usages are not that library, since it has serious performance issues, especially on high cardinality metrics (like topics). + +Using all other libraries, you’re basically required to write a function that has the following signature: `void writeMetrics(SimpleTextOutputStream stream)` for each class containing metrics. Then you add a call to that function in `PrometheusMetricsGenerator`. The argument `stream` is basically a byte-array you’re writing bytes into, which represents the response body for `/metrics` that is about to be delivered. You need to be aware of that, and write the current state of each metric you have in your class, in Prometheus Exposition format. + +This presents multiple problems: + +- The logic of printing metrics to a stream in Prometheus format is copied and pasted in many classes, as there is no types in this stream - it’s just a byte array. +- Prometheus Format dictates that all metric data points with the same metric name be written one after another. The current “API” which just writes text to a Stream (in Prometheus text format) collides with that since it does not force that. It forced Pulsar to find an interim solution which is complicated (See `PrometheusMetricStreams` class), which holds a Stream per metric name. +- Sometimes even the logic of flushing the stream was implemented more than once (e.g. `FunctionMetricsResource` writing their own `Writer` using Heap-based `ByteBuf` instead of direct memory) +- There’s no single place to define shared labels. For example the `cluster` label must be added manually by any function. +- It’s error-prone - you can forget to follow all those steps to export +- It’s confusing for developers +- It’s a lot of work for developers, when adding metrics to their features + +## Integrating metrics as Plugin author is hard, labor-intensive and prevents common functionality + +Plugins have their own metrics. Most plugins were written to run inside Pulsar (you supply JARs or NARs loaded on Broker initialization). Pulsar doesn't provide a single interface through which you create metric objects and register them and integrate with Pulsar metrics reporting. Due to that, the following happens: + +- Plugin authors choose all sorts of metric libraries: BK Metrics API and SDK, Prometheus, and more. +- If they chose Prometheus, and use the static collector, they need to do nothing this gets emitted with Pulsar metrics. This is not well known nor a typed way to define interfaces between Pulsar and Plugins. +- If they chose other libraries, Pulsar provides plugin authors a way to interface their metric library with Pulsar’s, with the usage of the following interface: `PrometheusRawMetricsProvider` which contains a single method:`void generate(SimpleTextOutputStream stream)`. This basically means they need to implement this function, so it will read the metrics from the framework they chose, and write it in Prometheus exposition format, in bytes. + +Due to that, most plugin developers are forced to write their metric exporting logic on their own, causing more work to be done for them. + +Since the interface is very low level, it creates several difficulties going forward: + +- Making sure Prometheus metrics are printed according to its format (for each name, all attributes are printed one after the other) is very difficult. You can’t do that easily with the current interface. +- If you want to introduce any common mechanism for filtering or any other work on those metrics, you can’t since it forces you to decode them from the text format, which will consume too much CPU and memory. + +## Lack of ability to introduce another metrics format + +Due to multiple libraries and lack of high level metrics interface for plugins, it’s basically impossible to add another export format in a performant manner suitable for latency sensitive system such as Pulsar. The metrics system today is coupled to Prometheus format, thus prevents any addition of a new, better format. + +Take for example, OTLP, a new protocol for Traces/Logs/Metrics. It’s more optimized than Prometheus, since for example, for histograms, it mentions the attributes once for all buckets, rather than repeating it for each bucket like Prometheus format. + +OTLP can’t be added to Pulsar as another exporting mechanism, in current metric system. + +## Adding Rates is error-prone + +The way `Rate` is built forces the developer to both: + +1. Manually add a print of the value of the new instance created, to the function which writes the metrics in Prometheus format to `SimpleTextOutputStream` (each class has such a method) +2. Add a call to `reset()` of the `Rate`instance, to a function which runs periodically. + +Both are not something a developer can understand on its own, therefor easy to forget to do, or even call twice by mistake. Also wastes time to learn how to do it. + +## Inline metrics documentation is lacking + +Each metric, in Prometheus format should contain a line starting with `#HELP` which allows Prometheus to parse that line and add a description to this metric, which is later used by UIs like Grafana to be better explain the available metrics. + +Since there are 4 metric libraries, only Prometheus Client offers the typed option of including a description. Most metrics are not using it, hence lack a descent help line. + +## Histograms can’t be visualized or used since not Prometheus conformant + +The main histogram used in Pulsar is `StatsBucket`. It has two major problems: + +1. Bucket counters in it are reset to 0 every 1 min (configurable). This goes against Prometheus assumption that bucket counters only increase. As such, it prevents using Prometheus functions on it like calculating quantiles (`histogram_qunatile`), which is the main reason to use histograms. +2. When exported, the bucket label is encoded in the metric name, and not as `le` label as Prometheus expects, hence makes it impossible to use `histogram_quantile` and calculate quantiles on it. + + For example: `pulsar_storage_write_latency_le_10` Should have been `pulsar_storage_write_latency{le=”10”}` + + +## Some metrics are delta-reset, making it easy to lose data on occasions + +`Rate`, `StatsBucket`, `Summary`, some exported JVM metrics and Pulsar Function metrics are reset to 0 every 1 min (configurable). This means that if from some reason, Prometheus or any other agent, fails to scrape the metrics for several minutes, you lost the visibility to Pulsar during those minutes. When using counters / histograms which are only incremented, the rate is calculated as delta on the counter values hence if two measurements 5 minutes apart, will still give you a descent average on that period. Same goes for histogram quantile calculation. + +## Prometheus client metrics use static registry, making them susceptible to flaky tests + +Most usage of Prometheus Client library is done using the static Collector registry. This exposes it to flaky behavior across tests, as static variables are shared across tests, and not cleaned between them. When using non-static registry, it inherently resets itself every new test, but this is not the case here. + +## Function custom metrics are both delta reset and limited in types to summary + +A Pulsar Function author has a single way to add metrics to their function, using method ``void recordMetric(String metricName, double value)` on `BaseContext` interface. + +What it does, is record this `value` in a Prometheus Summary named `pulsar_function_user_metric_`, under the label `metric={metricName}`. + +This Summary metric is also being reset every 1min by the wrapper code running the user function. + +It has the following problems: + +- The values are reset every 1min, hence subject to data-loss as presented above, +- The user is forced to use a Summary only, and is not offered the ability use types like Counter, Histogram or Gauge. The user find all sort of hacks around it to represent counters using summary’s count and sum. + +## Inconsistent reporting of partitioned topics + +There is a configuration specifying if a Partitioned Topic, composed of several partitions each is a Topic, will be printed using `topic` label only (i.e. `topic=incoming-logs-partition-2`) or split into `topic` and `partition` (i.e. `topic=incoming-logs, partition=2`). + +The problem is that this configuration is only applied to messaging related metrics (namespace, topic, producer, consumer, subscription). It is not applied to any other metric which contains the topic label, such as Transactions, Ledger Offloader, etc. This creates inconsistency in reported metrics. + +## System metrics manually scraped from function processes override each other + +Pulsar Functions are launched by instances (processes) of Pulsar Function Worker. It supports 3 types of runtimes (function launchers): + +1. Thread - run the function in a new thread inside the Function Worker process +2. Process - launch a process, which will run wrapper code executing the function (in Java, Python and Go). +3. Kubernetes - launching a Pod, running the same wrapper code as Process runtime. + +The metrics of the wrapper code, which also includes the function custom metrics (metrics the function authors adds), are exposed on `/metrics` endpoint by the wrapper code. In the case of Kubernetes, the pod is annotated such that Prometheus operators will scrape those metrics directly. In the case of Thread runtime, the metrics are integrated into the Function Worker metrics. In the case or Process runtime, the Function Worker is the one scraping the `/metrics` from each function process, concatenating them and add it to Function Worker `/metrics`. + +Process runtime also includes many JVM and system level metrics registered using Prometheus Client built-in exporters. Due to this (not Pulsar code) it doesn't contain any special label identifying this function. + +When the Function Worker scrapes each `/metric` endpoint, it simply concat the response, and since no unique label exists, the metrics override each other. + +For example, if a Function Worker launched 3 processes, one for each function, then each will contain `jvm_memory_bytes_used{area="heap"} 2000000`, with different numeric value. In it there is no unique label. When concatenating the response from the three functions processes, without any process/function we will not know from each process this arrived from, and they will override each other. + +## No consistent naming convention used throughout pulsar metrics + +- Some domains have a metric prefix, like `pulsar_txn` for transactions related metrics or `pulsar_schema` for Pulsar Schema metrics. Some don’t, like metrics related to messaging (topic metrics) - for example `pulsar_bytes_in` or `pulsar_entry_size_le_*`. +- Some metrics start with `brk_` while others start with `pulsar_`. Some are even replaced from `brk_` to `pulsar_` during metric export. + +This makes it very hard: + +- Defining filters. If you want to exclude messaging related metrics, you can’t as `pulsar_*` will catch all other pulsar’s metrics. +- Compose dashboard: It’s easier to type a domain prefix to zoom in on its metrics, like `pulsar_ledgeroffloader_`but it’s impossible for metrics such as messaging metrics which doesn't really have a prefix. + +# Goals + +## In Scope + +- Allow monitoring Pulsar broker with very high topic count (10k - 1M), without paying the price of high cardinality, by providing a mechanism which aggregates topic-level metrics to an aggregation level called Topic Metric Group which the operator controls dynamically. +- Allow dictating (filtering) which metrics will be exported, per any granularity level metrics: namespace, topic metric group, topic, consumer, producer, subscription. +- Replace Summary with Explicit Bucket Histogram +- Consolidate metrics usage (define, export) to a single library (i.e. OpenTelemetry) +- Provide a rich typed interface to hook into Pulsar metrics system for Plugin authors +- Make adding a Rate robust and error-free +- Make histogram reporting conformant with Prometheus when exported to Prometheus format +- Stop using static metric registries +- Provide ability in the future to correlated metrics with logs and traces, by sharing context +- Provide a pluggable metrics exporting, supporting a more efficient protocol (i.e. OTLP) +- Support the most efficient observability protocol, OTLP. +- Stop using delta reset, everywhere, including function metrics +- Provide rich typed interface to define metrics for Pulsar Functions authors +- All Pulsar metrics are properly named following a well-defined convention, adhering to [OTel Semantic Conventions](https://opentelemetry.io/docs/reference/specification/metrics/semantic_conventions/) for instrument and attribute naming where possible +- All changed listed above will at least as good, performance wise (CPU, latency) as current system +- New system should support the maximum supported number of topics in current system (i.e. 4k topics) without filtering + +## Out of Scope + +- Pulsar client metrics + +# High Level Design + +* [Consolidating to OpenTelemetry](#consolidating-to-opentelemetry) +* [Aggregate and Filtering to solve cardinality issues](#aggregate-and-filtering-to-solve-cardinality-issues) +* [Changing the way we measure metrics](#changing-the-way-we-measure-metrics) +* [Integrating Messaging metrics into Open Telemetry](#integrating-messaging-metrics-into-open-telemetry) +* [Switching Summary to Histograms, in namespace/broker level only](#switching-summary-to-histograms-in-namespacebroker-level-only) +* [Reporting topic and partition all the time](#reporting-topic-and-partition-all-the-time) +* [Metrics Exporting](#metrics-exporting) +* [Avoiding static metric registry](#avoiding-static-metric-registry) +* [Metrics documentation](#metrics-documentation) +* [Integration with BK](#integration-with-bk) +* [Integrating with Pulsar Plugins](#integrating-with-pulsar-plugins) +* [Supporting Admin Rest API Statistics endpoints](#supporting-admin-rest-api-statistics-endpoints) +* [Fixing Rate](#fixing-rate) +* [Function metrics](#function-metrics) + + +## Consolidating to OpenTelemetry + +We will introduce a new metrics library that will be the only metrics library used once this PIP implementation reaches its final phase. The chosen library is OpenTelemetry Java SDK. Full details on what is OpenTelemetry and why it was chosen over other metric libraries is located at the [Detailed Design section](#why-opentelemetry) below. This section focuses on describing it in high level. + +OpenTelemetry is a project that provides several components: API, SDK, Collector and protocol. It’s main purpose is defining a standard way and reference implementation to define, export and manipulate observability signals: metrics, logs and traces. In this explanation we’ll focus on the metrics signal. The API is a set of interfaces used to define and report metrics. The SDK is the implementation of that API and also added functionality such as export of metrics and ability to change how certain metrics are collected and measured. The protocol is called OTLP, and it's the most efficient protocol to transmit metrics, logs and traces to its final destination (be it a database or vendor). The Collector is a light-weight process that allows receiving/pulling metrics/logs/traces in various formats, transform them (i.e. filter, aggregate, maintain state, etc.), and export them to many destinations (many vendors and databases). + +The project’s API also has a part for reporting logs and traces. One its core features is the ability to share common context that can be used across metrics, logs and traces reporting (think Thread Local contains an attribute which is also used when reporting metric’s baggage, logs and traces, hence creates a link between them). + +In this PIP scope we will only use the Metrics API and SDK. Specifically we will use the Java SDK in Pulsar Broker, Proxy and Function Worker, but also the Go and Python SDK for the wrapper code which executes functions written in Go and Python. + +We will keep the current metric system as is, and add a new layer of metrics using OpenTelemetry Java SDK: All of Pulsar’s metrics will be created *also* using OpenTelemetry. A feature flag will allow enabling OpenTelemetry metrics (init, recording and exporting). All the features and changes described here will be done only in the OpenTelemetry layer, allowing to keep the old version working until you’re ready to switch using the OTel (OpenTelemetry) implementation. In the far future, once OTel usage has stabilized and became widely adopted we’ll deprecate current metric system and eventually remove it. We will also make sure there is feature flag to turn off current Prometheus based metric system. +There's no need to have an abstraction layer on both the current metric system and the new one (OTel) as OpenTelemetry API *is* the abstraction layer, and it's the industry standard. + +One very big breaking change (there are several described in the High Level Design, and also summarized in the Backward Compatibility section) is the naming. We are changing all metric names due to several reasons: + +1. Attributes names (a.k.a. Labels) will utilize the Semantic Conventions [defined](https://opentelemetry.io/docs/concepts/semantic-conventions/) in OTel. + 1. OpenTelemetry defined an agreed upon attribute names for many attributes in the industry. +2. Histograms bucket names will be properly encoded using `le` attribute (when exported to Prometheus format) and not inside the metric name. +3. Each domain in Pulsar will be properly prefixed (messaging, transactions, offloader, etc.) + +The move to OpenTelemetry is not without cost, as we both need to improve the library and adjust the way we use metrics in Pulsar with it. The Detailed Design contains a section detailing exactly what we need to improve in OpenTelemetry to make sure it fits Apache Pulsar. It mostly consists of making it performant (almost allocation free), and small additions like removing attribute sets limit per instrument. + +The changes we need to make in Pulsar to use it are detailed in this high level design section below. It’s composed of two big changes: changing all histograms to be only at namespace level instead of topic level, and switching from Summary to Histogram (Explicit Bucket). + +There will be a sub-PIP detailed exactly how OpenTelemetry will be integrated into Pulsar, detailing how users will be able to configure it: their own views, exporters, etc. We need to see how to support that, given we want to introduce our own filtering predicate. + +Here's a very short *idea* of how the code will look like using OpenTelemetry. In reality, we will use batch callback and different design. The sub-PIP will specify the exact details. + +```java +class TopicInstruments { + // ... + meter.counterBuilder("pulsar.messaging.topic.messages.received.size") + .setUnit("bytes") + .buildWithCallback(observableLongMeasurement -> { + for (topic : getTopics()) { + val size = topic.getMessagesReceivedSize(); + observableLongMeasurement.record(size, topic.getAttributes()); + } + }); + // ... +} + + +class PersistentTopic { + // ... + Attributes topicAttributes + LongAdder messagesReceivedSize + // ... + + init(String topicName) { + Attributes topicAttributes = Attributes.builder() + .put("topic", topicName) + .put("namespace", namespaceName) + .build(); + } + // ... + + + messageReceived() { + // ... + messagesReceivedSize.add(msgSize); + } + + getMessagesReceivedSize() { + return messagesReceivedSize.value(); + } +} +``` + +## Aggregate and Filtering to solve cardinality issues + +### Aggregation + +When you have cardinality issues, specifically topics being the root cause since Pulsar support up to 1M topics cluster wide, roughly translated to 100k topics per single broker. The best way to solve it is to reduce the cardinality. + +We will introduce a new term called Topic Metric Group. Each topic will be mapped to a single group. The groups are the tool we’ll use to reduce cardinality to a descent level, which roughly means 10 - 10k. A cardinality scale your time series database can handle properly. + +For each metric Pulsar currently have in topic-level (topic is one of its attributes), we will create another metric, bearing a different name which will have its attributes in the group level (`topicMetricGroup` will be the attribute, without `topic` attribute). Each time we’ll record a value in a topic metric, we’ll also record it in the equivalent Topic Metric Group metric. For consistency, we will do the same also for namespace level. + +Example: + +If today you chose topic level metric, which means you don’t have namespace level metrics, you’ll have the following metric: + +```jsx +pulsar_in_messages_total{topic="orders_company_foobar", namespace="finance", ...} +``` + +After the change (including naming changes discussed later), you’ll have: + +```jsx +pulsar_messaging_**namespace**_in_messages_total{namespace="finance"} +pulsar_messaging_**group**_in_messages_total{topicMetricGroup="orders", namespace="finance"} +pulsar_messaging_**topic**_in_messages_total{topic="orders_company_foobar" topicMetricGroup="orders", + namespace="finance"} +``` + +There will be a dynamic configuration allowing to specify the mapping from topic to topic group, in a convenient way. Since there can be many ways to configure that, we’ll define a plugin interface which will provide that mapping given a topic, and we’ll provide a default implementation for it. This will allow advanced users to customize to their needs. + +The detailed design section contains a more detailed description of this mapping configuration. There will be a sub-PIP detailing how the mapping will be configured, which tools we need to add to allow knowing which topics are in each group, and more. + +### Filtering + +The aggregation above using both namespace and Topic Metric Group, reduces the cardinality. The only thing left for us to do is add a great user experience tool to control which metrics you’d like to see exported, in fine granularity. A bit like logging levels, but with more fine-grained controls. + +We would like to have a dynamic configuration allowing us to toggle which metric we wish to keep or drop. For example: + +- Keep namespace and topic metric group level metrics, for any namespace, any group. +- For any topic, keep only backlog size metric, and drop anything else under “messaging” scope metrics. +- For topic “incoming-logs”, keep all of its metrics. +- For topic group “orders”, keep all metrics in topic level, but drop producer/consumer/producer level metrics. +- For topic “billing”, keep all metrics (up to producer/consumer level). + +We will introduce a new configuration, containing Filtering Rules, evaluated in order. Each rule will define a selector allowing you to select (metric, attributes) pairs based on all sorts of criteria, and then defining actions such as drop all, keep all, keep only, drop only, determining for each (metric, attribute) pair if it should be filtered out or not. + +This configuration will be dynamic, hence allowing you to build namespace / group level dashboards to monitor Pulsar. Once you see a group misbehaving, you can dynamically “open” the filter to allow this group’s topic metric, and you can decide to only allow the certain metric you suspect is problematic. Once you find out the topic, you allow (stop dropping) all of its metrics, to enable you to debug it. Once you’re done, you can roll back to group level metrics. + +We want to allow users to customize and build a filter matching their needs, hence we’ll create a plugin interface, which can decide for a given metric (name, attributes, unit, etc.) if they want to filter it or not. + +The detailed design section will include a more detailed description of the plugin interface and default implementation configuration file. It will also include explanation how we plan to implement that in stages: 1st stage on our own and 2nd stage by introducing push down predicate into OpenTelemetry MetricReader. + +### Removing existing metric toggles + +The fine-grained filtering mechanism described makes using the configuration toggles we have today deprecated hence we will remove them. This includes `exposeConsumerLevelMetricsInPrometheus`, `exposeTopicLevelMetricsInPrometheus` and `exposeProducerLevelMetricsInPrometheus`. + +### Summary + +The aggregation to groups and namespaces which are of reasonable cardinality, coupled with the ability to decide on which metrics and specific attribute sets in them, you wish to export using the filters, solves the cardinality issue. Visibility to monitoring data is not sacrificed since dynamic configuration allows you to zoom in to get the finer details and later shut it off. + +## Changing the way we measure metrics + +### Moving topic-level Histograms to namespace and broker level only + +Histograms are primarily used to record latencies: `pulsar_storage_write_latency_le_*`, `pulsar_storage_ledger_write_latency_le_*`, and `pulsar_compaction_latency_*` + +We have several issues with them: + +- They cost a lot. Each histogram today translates to 12 unique time series, which means it costs like x12 more than a counter. +- A broker is mostly a multi-tenant process by design. It’s almost never a single topic’s fault for latency, and even if it is, other topics will be affected as well. There is contention mostly on CPU of broker, and Bookkeeper, which both are shared across topics. Having it in topic level won’t help to diagnose topic root cause, and sometime mislead you. + +The other problem we have is the topic move. Topics can move between brokers, due to automatic load balancing, or an operator decision to unload a topic. Today, when a topic is unloaded, the broker stops reporting the metrics for it. + +In OTel, the API doesn't support `remove()` for an attribute set on an instrument (counter, histogram, etc.). Normally, if it was supported, we could have called `remove` for each instrument which contains an attribute-set for that topic. + +OTel has two categories of instruments: synchronous and asynchronous. Synchronous instruments, keeps the value of the object in memory and updates it upon recording a new value (behind the scenes: it adds for example +2 to an `AtomicLong` it maintains for a Counter and the attribute set). Asynchronous instruments on the other hand are defined using a callback. When a call is made to collect the values for each instrument, a callback is invoked to retrieve the (attributes, value) pairs. when you record a value to an attribute set in a synchronous instrument, it will remain in memory forever until restart. Async instruments on the other hand, retrieve the (attributes, value) pairs upon collect, and forget them afterward. + +Since `remove()` doesn't exist for instruments, async instruments are the closest thing we have in OTel. Thus Counter and UpDownCounters for topic instruments can be asynchronous instruments, hence when a topic is unloaded, the next callback will simply not record their values. + +Histograms are problematic in that sense, since they yet to have an asynchronous version - alas they are only synchronous. Hence, if we use them for topic instruments, used attributes can never be cleared from memory for them thus we’ll have an “attributes” (topic) leak, as over time it will only grow. This is why currently we can’t use them for topics or topic groups. + +We have opened an [issue](https://github.com/open-telemetry/opentelemetry-specification/issues/3062) and making progress, but this is a long process. + +Coupling the two together - cost, confusion, lack of clear use, and lack of support from OTel - we will change those topic level histograms to be namespace / broker level. + +## Integrating Messaging metrics into Open Telemetry + +As explained before, instruments don’t support `remove()`. Topic, its subscriptions, producers and consumers - each has its own set of metrics. Also, each is ephemeral. Topics can move or be deleted, producers and consumers can stop and start many times, and subscriptions move with topics. + +Hence, for topic, producers, consumer and subscription (and topic group) we will use asynchronous instruments. This means we’ll keep the state using our own `LongAdder`, `AtomicLong` or primitive long using atomic updater. When creating the asynchronous instrument, the callback will retrieve the value from the variable. For example, when a producer is removed, in the next collection cycle, the callback will not report metrics for it since it doesn't exist, and thus it will disappear from memory of OTel as well. + +OTel has a special batch callback mechanism, allowing you to supply a single callback for multiple instruments, making it more efficient, and we plan to use it. + +As explained before, we’ll have instruments per aggregation level: broker, namespace, topic group, topic, subscription, producer and consumer. + +Broker and namespace level will use synchronous instruments, since the amount of namespaces is not expected to have high cardinality, hence not removing them is not a big attribute leak issue. Asynchronous instruments are also needed es explain in “Supporting Admin Rest API Statistics endpoints” section below. + +## Switching Summary to Histograms, in namespace/broker level only + +Summary by design can’t be aggregated across topics or hosts (See Background and Motivation). That was the reason OTel doesn't support them. I opened an [issue](https://github.com/open-telemetry/opentelemetry-specification/issues/2704) for that, but it doesn't seem like something that can be added to OTel. + +Another consideration is CPU. In benchmarks done, it seems that updating the summaries that are based on Apache Data Sketches cost 5% of CPU time, which is a lot compared with a simple +1 to a counter in a bucket in an explicit bucket histogram. + +Due to those reasons, it makes sense to switch all summaries to histograms (explicit bucket type). + +From the same reasons as histograms, they will be broker/namespace level only (not topic). + +Most summaries are used for latency reporting, from same reasons of multi tenancy and careful inspection of existing summaries, we’ve concluded we can convert them to be namespace level. The domains affected by it are: + +- LedgerOffloader stats +- Replication of subscription snapshot +- Transactions + - Transaction Buffer client + - Pending Acks + +The complete list of which summaries are affected is at the Detailed Design section + +Pulsar Functions metrics uses summary for user defined metrics, but we assume the quantiles are actually meaningless since some use it to record the value “1” just to obtain count, and some record value to obtain a sum. We will convert them to Histogram without buckets, just providing sum and count. + +- Each custom metric is actually an attribute set bearing the attribute `metric={user-defined-name}` +- We will define that in the init based on instrument name using views. + +### Specifying units for Histograms + +We’ve inspected our summaries and histograms, and it seems the bucket ranges are common per unit: ms, seconds and bytes. + +OTel has the notion of views. They are created upon init of OTel and can be used to specify the buckets for a set of instruments based on instrument selector (name wildcard, …). We have opened an [issue](https://github.com/open-telemetry/opentelemetry-specification/issues/3101) for OTel SDK Specifications, which was merged to allow specifying a unit in an instrument selector. We only need to implement it in OTel Java SDK (See [issue](https://github.com/open-telemetry/opentelemetry-java/issues/5334) I’ve opened). + +### Removing Delta Reset + +As described in the motivation and background section, Pulsar reset certain type of metrics every configurable interval (i.e. 1min) : rates, explicit bucket histograms and summaries. + +As also explained, it makes hard to use for histograms, and redundant and less accurate for rates. Most time-series databases can easily calculate rates based on ever-increasing counters. + +Hence, in OpenTelemetry we won’t report rates, we’ll switch to counters. That means of course name changing, but as described in this document, all names will be changed anyhow. + +For histograms, we will simply never reset them. + +Summaries are converted to histograms so also never reset anymore. + +We will keep Rates around primarily for the statistics returned through Admin API (i.e. Topic Stats, …), but they will never be exposed through OpenTelemetry. + +It is worth noting that OpenTelemetry supports the notion of Aggregation Temporality. In short, it allows you to define for a given instruments if you want it to be exported as delta or cumulative. Some exporters like Prometheus only support Cumulative, and will override it all to be such. OTLP supports delta. Currently, we’re not explicitly supporting configuring views / readers to allow that, but it’s something very easily added in the future, by the community. It will be perfect for people using OpenSearch or Elasticsearch. + +## Reporting topic and partition all the time + +Today a topic is reported using the attribute `topic={topicName}`. If the topic is actually a partition for a partitioned topic, it will look like `topic={partitionedTopicName}-partition-{partitionNum}`. There is a configuration name `splitTopicAndPartitionLabelInPrometheus` which makes that be reported instead as `topic={partitionedTopicName}, partition={partitionNum}`. + +This is not consistent, in such that not all metrics using `topic` attribute did the split accordingly. + +In OTel metrics we will ignore that flag and always report it as `topic={partitionedTopicName}, partition={partitionNum}`. We will make it consistent across any `topic` attribute usage. Eventually this flag will be removed (probably in the next major version of Pulsar). + +## Metrics Exporting + +OTel has a built-in Prometheus `MetricReader` (and exporter) which exposes `/metrics` endpoint, which we will use. + +Pulsar current metric system has a caching mechanism for `/metrics` responses. This was developed since creating the response for high topic count was a CPU hog and in some cases memory hog. In our case we plan to use Filtering and Aggregation (Metric Topic Group) to drive the response to a reasonable size, hence won’t need to implement that in OTel metrics. + +OTel also has built-in OTLP exporter. OTLP is the efficient protocol OTel has, which OTel Collector supports, and some vendors as well. We wish to use it, yet it seems that it is very heavy on memory allocation. Hence, we will need to improve it to make it allocation free as much as possible. + +## Avoiding static metric registry + +OTel supports a static (Global) OTel instance, but we will refrain from it to make sure test data doesn't leak between tests. + +In OTel we will create an instance of `MeterProvider` during Pulsar init. This object is the factory for `Meter` which by itself is a factory for instruments (Counter, Histogram, etc.). We will pass along a Pulsar `Meter` instance to each class that needs to create instruments. The exact details will be detailed in a sub-PIP of adding OpenTelemetry to Pulsar. + +## Metrics documentation + +OTel doesn't force you to supply documentation. + +We will create a static code analysis rule failing the build if it finds instrument creation without description. + +We will optionally try to somehow create an automated way to export all metrics metadata - instrument name, documentation, in an easy-to-read format to be used for documentation purposes on Pulsar website. + +## Integration with BK + +BookKeeper (client and server) has their own custom metrics library. It’s built upon a set of interfaces. Pulsar metric system has an implementation for it. + +We will create another implementation to patch it to OTel. + +## Integrating with Pulsar Plugins + +We will modify all popular plugin interfaces Pulsar has, such that they will accept an OpenTelemetry instance to be used to grab the `MeterProvider` instance and use it to create their own `Meter` with the plugin name and version which they will use to create their own instruments. + +A user which has decided to turn on OTel metrics will have to verify all Pulsar plugins it uses have been upgraded to use the modified interface, otherwise their metrics will not be exported. + +Plugin authors will need to release a new version of their plugin which implements the new interface and registers the metrics using `OpenTelemetry`. + +One big advantage is that using OTel supports plugins running in stand-alone mode. Some plugins have the option to run some of their code outside Pulsar. By using OTel API, they can integrate either via their own SDK or Pulsar SDK (via `OpenTelemetry` instance). + +A detailed list is at the Detailed Design. + +A sub-PIP will be dedicated to integrating with plugins. + +## Supporting Admin Rest API Statistics endpoints + +Pulsar has several REST API endpoints for retrieving detailed metrics. They are exposing rates, up-down counters and counters. + +OTel instruments doesn't have methods to retrieve the current value. The only facility exposing that is the `MetricReader` that reads the entire metric set. Thus, any metric that is exposed also through Admin REST API will have to have its state maintained by Pulsar, either using `LongAdder`, `AtomicLong` or primitive long with atomic updater. The matching OTel instrument will be defined as Asynchronous instrument, meaning it is defined by supplying a callback function will be executed to retrieve the instrument current value. The callback will simply retrieve the state from the matching `LongAdder`, `AtomicLong` or primitive long (or double to that end) and return it as the current value. + +## Fixing Rate + +We will change `Rate`, in such a way that won’t require the user creating it, calling `reset()` periodically and manually. All rates will be created via a manager class of some sort, and it will be the one responsible for scheduling resets. Rates will always expose a sum counter and a counter to save up on multiple variables. A sub-PIP will explain in detail how this will be achieved. + +## Function metrics + +### Background + +Pulsar supports the notion of Pulsar Functions. It is user-supplied functions, either in Go, Java or Python, which can read messages, and write messages. You can use it to read all messages from a topic and write them to external system like S3 (Sink), or the other way around: read from an external system and write it to a topic (e.g. DB Change log to Pulsar topic - source). The other option is simply transforming the message received and writing it to another topic. + +A user can submit a function, and can also configure the amount of instances it will have. + +The code responsible for coordinating the execution of those functions is located in a component called Function Worker, which can be run as stand-alone process or as part of Pulsar process. You can run many Function Workers, yet only one function as the leader. + +The leader takes care of splitting the work of executing the function instances between the different Function Workers (for load balancing purposes). + +The Function worker has three runtimes, as in, three options to execute the functions it is in charge of: + +1. Thread: Creating a new thread and running the function in it. +2. Process: Creating a new process and running the function in it. +3. Kubernetes: Creating a Deployment for each function. + +Each Function Worker has its own metrics it exposes. + +Each Function has its own metrics it exposes. + +In the Thread runtime, all metrics are funneled into Pulsar metrics (exposing a method which writes them into the `SimpleTextOutputFormat`). + +In the Process runtime, the function is executed in its own process, hence there is a wrapper main() function executing the user supplied function in the process. This main() function has general function execution metrics (e.g. how many messages received, etc.), and also the function metrics (user custom metrics). All the metrics are defined using a single library: Prometheus client. The metrics are exposed using the client’s HTTP server exposing `/metrics` endpoint exposing the two categories of metrics. + +In the Kubernetes runtime, the pod is annotated with Prometheus Operator annotation, and it is expected it will be installed, hence Prometheus will scrape those metrics directly from the process running in the pod. + +In the Process runtime, the Function Worker is iterating over all processes it launched, and for each it issues a GET request to `/metrics`. The responses are concatenated together and printed to `SimpleTextOutputStream`. + +The general function execution metrics comes in two forms: cumulative and 1 min ones. The latter names ends with `_1min`, and they get reset every 1min. (e.g. `*_received_1min`). + +Each process launched also launches a gRPC server supporting commands. Two of those are related to metrics: `resetMetrics` and `getAndResetMetrics`. They reset all metrics - both the general framework ones and the customer user ones. + +Prometheus client was also configured to emit several metrics using built in exporters: memory pool, JMX metrics, etc. + +### Solving it in Open Telemetry + +In phase 1, we’ll keep the reporting of user defined metrics for function authors as is, and mainly focus on the other issues which are: metrics scraping for each runtime, and the 1min metrics. In phase 2, we’ll also add the option to define metrics for pulsar function authors via OTel. + +### Collecting metrics + +**Thread Runtime** + +The framework will use Pulsar OTel SDK, or it’s standalone function worker SDK, thus what ever export method it uses, it will use as well (prometheus, OTLP, …) + +**Kubernetes Runtime** + +OTel supports exporting metrics via `/metrics` endpoint using Prometheus format. We’ll support the same as it is today done with Prometheus client. + +We’ll also support configuring the pod, so it can send metrics via OTLP to defined destination. + +**Process Runtime** + +The existing scraping of `/metrics` solution was not good: + +- It violated Prometheus format, since same name, different attributes lines must be one after another, and in reality it was concat as is. +- The prometheus exporters metrics like memory pools, didn't have any unique attribute for each process, hence when concat, they would have same name same attributes different values, from different processes hence be lost. + +OTel supports both exporting metrics as Prometheus and pushing them using OTLP. Making the Function worker pull the metrics, in effect - to be the hub - is super complicated. It’s much easier to simply let the processes be scraped or push OTLP metrics - configured the same as Pulsar is. + +At phase 1 we won’t support Process Runtime. + +At phase 2, we can use [Prometheus HTTP Service Discovery](https://prometheus.io/docs/prometheus/latest/http_sd/), and expose such an endpoint in the Function Worker leader. Via health pings it gets from each worker, they can also report each process metrics port, thus allowing prometheus to scrape the metrics directly from each process. We’ll garner feedback from the community to see how important is Process runtime, as we have K8s runtime which is much more robust. + +### Removing 1min metrics + +We’ll not define any 1min metric. Any TSDB can calculate that from the cumulative counter. + +### Supporting `getMetrics` RPC + +We can define our own `MetricReader` which we can then filter to return the same metrics as we return today: general function metrics and user-defined metrics. + +### Removing `resetMetrics` RPC + +OTel doesn't support metric reset, and it also violates Prometheus since it expects metrics to be cumulative. Thus, we will remove that method. + +### Supporting Python and Go Functions + +OTel has an SDK for Python and Go, thus we’ll use it to export metrics. + +### Summary + +A sub-pip will be created for Function Metrics which will include detailed design for it. + +# Detailed Design + +* [Topic Metric Group configuration](#topic-metric-group-configuration) +* [Integration with Pulsar Plugins](#integration-with-pulsar-plugins) +* [Why OpenTelemetry?](#why-opentelemetry) +* [What we need to fix in OpenTelemetry](#what-we-need-to-fix-in-opentelemetry) +* [Specifying units for histograms](#specifying-units-for-histograms-1) +* [Filtering Configuration](#filtering-configuration) +* [Which summaries / histograms are effected?](#which-summaries--histograms-are-effected) +* [Fixing Grafana Dashboards in repository](#fixing-grafana-dashboards-in-repository) + +## Topic Metric Group configuration + +As mentioned in the high level design section, we’ll have a plugin interface, allowing to have multiple implementations and to customize the way a topic is mapped to a group. + +The default implementation this PIP will provide will be rule based, and described below. + +We’ll have a configuration, that can look something like this: + +```hocon +bi-data // group name + namespace = bi-ns // condition of the form: attribute name = expression + topic = bi-* // condition of the form: attribute name = expression + +incoming-logs + namespace = * + topic = incoming-logs-* +``` + +The configuration will contain a list of rules. Each rule begins with a group name, and list of matchers: one for namespace and one for topic. The rules will be evaluated in order, and once a topic was matched, we’ll stop iterating the rules. + +There will be a sub-PIP detailing the plugin and default implementation in fine-grained detail: where it will be stored, how it will support changing it dynamically, performance, etc. + +## Integration with Pulsar Plugins + +These are the list of plugins Pulsar currently uses + +- `AdditionalServlet` +- `AdditionalServletWithPulsarService` - we can use `PulsarService` to integrate +- `EntryFilter` - need to add `init` method to supply OTel +- `DelayedDeliveryTrackerFactory` - accepts PulsarService +- `TopicFactory` - accepts BrokerService +- `ResourceUsageTransportManager` - need to add `init` method +- `AuthenticationProvider` - need to add parameter to `initialize` method +- `ModularLoadManager` - accepts `PulsarService` +- `SchemaStorageFactory` - accepts `PulsarService` +- `JvmGCMetricsLogger` - need to add init(). +- `TransactionMetadataStoreProvider` - need to add init +- `TransactionBufferProvider` - need to add init +- `TransactionPendingAckStoreProvider` - need to add init +- `LedgerOffloader` - need to add init() +- `AuthorizationProvider` - need to add parameter to `initialize` method +- `WorkerService` - need to modify init methods +- `PackageStorageProvider` +- `ProtocolHandler` - need to modify init method +- `BrokerInterceptor` - accepts PulsarService +- `BrokerEntryMetadataInterceptor` + +In a sub-PIP we will consider how to update those interfaces effectively allowing to pass `OpenTelemetry` instance, so they can create their own Meter or have access to an auxiliary class and supply only a Pulsar meter. Note that each a Meter has its own name and version and those are emitted as two additional attributes - e.g. `{..., otel_scope_name="s3_offloader_plugin" otel_scope_version="1.2", ...}`, to avoid any metric name collision. + +## Why OpenTelemetry? + +### What’s good about OTel? + +- It’s the new emerging industry-wide standard for observability and specific metrics, as opposed to just a library or a standard adopted and promoted by a single entity/company. +- It’s much more sophisticated than the other libraries + - OTel has the ability to change instruments by overriding their initial definition. For example, a Pulsar operator can change buckets of a histogram for a given bucket, reduce attributes if needed, or even copy-paste an instrument, changing its bucket while maintaining the original one if needed. This feature is called a View. + - Its API is very clear. For example, a gauge can not be aggregated (i.e., CPU Usage), while UpDownCounter can (number of jobs in a queue). + - Using OpenTelemetry Logs and Traces will allow sharing of context between them, making using Pulsar telemetry more powerful and helpful (Out of scope for this PIP, but possible) + - Using an industry-standard API means when in the future libraries will accept `OpenTelemetry` interface for reporting traces/metrics/logs, the integration of it will not require any special development efforts. + - Industry-standard also means when new developers onboard, they don’t need to learn something new + - The SDK is still in the adoption/building phase, so they are more receptive to accepting changes from the community relative to other libraries (This was quite evident from issues I’ve opened that got fixed, community meetings attended, and brainstorming sessions held with maintainers) + - Its design is the most elegant and correct compared to all other libraries (IMO). The idea of each instrument having an interchangeable aggregation, which is also how they implemented it is smart. The same goes for Reader and Exporter separation and Views. + - It has support to decide if one metric or a family of them will be delta or cumulative. For Elasticsearch/OpenSearch users, it’s super powerful, as it allows them to create the same metrics with different names containing delta values and then feed only them to Elastic using the OTel Collector + - Its protocol is much more efficient than other protocols (i.e., Prometheus text exposition format) + - The library allows exporting the metrics as Prometheus and OTLP (push to OTel Collector) and it’s extendable by design + - It has same API and implementation design for Python and Go, which we also need to support for the wrapper code running Pulsar Functions. + + ### Why not other libraries? + + Below I will list the libraries I found and why I don’t think they are suitable. + + - Micrometer + - Micrometer had the vision of becoming the industry standard API like SLF4J is for logging in the Java ecosystem. In reality, it didn't catch on, as can be seen in the Maven Central statistics: It’s used by ~1000 artifacts, compared to `sl4fj-api` which is used by 60k artifacts—as such, picking it as the standard for today, seems like “betting” on the wrong project. + - Micrometer architecture relies heavily on the library to implement all target systems like Datadog, Prometheus, Graphite, OTLP, and more. OTel relies on the collector to implement that as it has more power and can contain the state if one of those systems goes down for some time. I think it’s a smarter choice, and more vendors will likely appear and maintain their exporter in OTel collector as we advance. This makes it easier for operators to have one exporter code base (say to Cortex) across different languages of microservices, so it makes sense people will lean towards this framework and request it soon. + - OTel was built with instrumentation scope in mind, which gives a sort of namespace per library or section of the code (Called Meter in the API). For Pulsar, it can be used to have one per plugin. Micrometer doesn't have that notion. It’s great especially if Pulsar and another plugin are using same library (e.g. Caffeine for caching), thus in Prometheus or other libraries the metrics will override each other, but in OTel the meter provides an attribute for name and version, thus provide a namespace. + - OTel by design has an instrument that you report measurements for a given attribute set, meaning it has that design of `instrument = map(attributes→values)`. In Micrometer, it’s designed in a way that each `(instrument, attributes)` is a metric on its own. Less elegant and more confusing. + - Most innovations are likely to happen in the “new kids on the block,” which is OTel. + - Dropwizard Metrics (previously Codahale) + - Doesn't support different attributes per instrument (no tag support). It was slated for Dropwizard 5.x, but there is no available maintainer available to work on it, which is a problem on its own. + - Prometheus Client + - Currently, prometheus allocates all needed memory per collection. For a large amount of topics, this is a substantial performance issue. We tried conversing with them and pitched an observer pattern. They objected to the idea and wanted benchmark proof. The maintainer thinks it has added complexity. See [here](https://github.com/prometheus/client_java/pull/788#issuecomment-1179611397). In OTel they were happy to brainstorm the problem via GitHub issue, their weekly calls and private zoom sessions. They showed actual care to their open source users, and together we found a solution that is actively being worked on. + - Only the Prometheus format is supported for export. OTLP is a more compact protocol since it packs all the buckets as a map of bucket numbers to their value instead of carrying all the labels for each bucket as the Prometheus client. + - The library doesn't have the notion of different exporters as it was geared to export only to Prometheus. + - No integration with Logs or Traces which will be needed in the future. + +## What we need to fix in OpenTelemetry + +- Performance + - Once we have 1M topics per broker, each topic producing ~70 metric data points (that’s in a super relaxed assumption: we have one producer and one consumer), we’re talking about 70M metric data points. + - The `MetricReader` interface and the `MetricsExporter` interfaces were designed to receive the metrics collected from memory by the SDK using a list, and for each collection cycle, an allocation of at least 70M objects (Metric data points). + - The OTLP exporter specifically serializes the data points to protobuf by creating a Marshaller object per each piece of data in the data point, so 10-30 times 70M metric data points, which are objects to be garbage collected. + - I have opened an issue starting discussion on trying to solve that: https://github.com/open-telemetry/opentelemetry-java/issues/5105 + - After discussion, the maintainer suggested object re-use per attribute set. He already implemented over 60% of the code needed to support it. We need to help the project finish it as detailed in issue description (mainly collection path should be allocation free, including exporters). +- There is a non-configurable hard limit of 2000 attributes per instrument + - There is a [PR](https://github.com/open-telemetry/opentelemetry-specification/pull/2960) to the specifications to allow configuring that limit + - Once the spec is approved, OTel Java SDK must also be amended. +- Supporting push-down predicate to filter (instrument, attribute) pair in `MetricsProducer` + - https://github.com/open-telemetry/opentelemetry-specification/issues/3324 + - This is needed for us to have performant filtering. +- Fix bug: https://github.com/open-telemetry/opentelemetry-java/issues/4901 + +**Nice to have** + +- Ability to remove attributes from an instrument + - Issue we opened: https://github.com/open-telemetry/opentelemetry-specification/issues/3062 + - This will allow us to use histograms if needed on dimensions such as topic and topic group. + +**Issues completed while writing the design** + +- Add ability to specify histogram buckets while creating an instrument + - [https://github.com/open-telemetry/opentelemetry-specification/issues/2229](https://github.com/open-telemetry/opentelemetry-specification/issues/2229) +- Add ability to specify units as instrument selector in a view + - Issue we have added to add it in spec: https://github.com/open-telemetry/opentelemetry-specification/issues/3101 + - This was implemented by the maintainers in the Java SDK + +## Specifying units for histograms + +We have two ways to do that: + +1. Since all latency with same units (milliseconds) share same buckets of histograms, we can use a view, and select all instruments in Pulsar Meter which has units of milliseconds, and there specify the buckets. +2. Specify buckets using newly added hints, at instrument creation. This requires creating a constant and re-using it across all histograms. + +## Filtering Configuration + +As mentioned in the high level design, we will define an interface allowing to have multiple built-in and also custom implementations of filtering. The interface will determine per each metric data point if it will be filtered or not. The data point is composed of: instrument name, attributes, unit, type. + +Our default implementation will be ruled based, and will have the following configuration. I used here [HOCON](https://github.com/lightbend/config/blob/main/HOCON.md) to make it less verbose. Exact syntax will be determined in sub-PIP. The configuration will be dynamic, allowing operators to change it in runtime. The exact mechanisms will be detailed in the sub-PIP. + +```hocon +rules { + // All instruments starting with pulsar_, with a topic attribute + // will be dropped by default. This will keep only topicMetricGroup and namespace level. + default { + instrumentSelect = "pulsar_*" + attrSelect { + topic = "*" + } + filterInstruments { + dropAll = true + } + } + + // single topic, highest granularity + bi-data { + instrumentSelect = "pulsar_*" + attrSelect { + topicGroup = "bi-data" + } + filterInstruments { + keepAll = true + } + } + + // multiple topics, highest granularity, only metrics I need + receipts { + instrumentSelect = "pulsar_*" + attrSelect { + topic = "receipts-us-*" + } + filterInstruments { + keepOnly = ["pulsar_rate_*"] + } + } + + // single topic, don't want subscriptions and consumer level + logs { + instrumentSelect = "pulsar_*" + attrSelect { + topic = "logs" + } + filterInstruments { + dropOnly = ["pulsar_subscription_*", "pulsar_consumer_*"] + } + } +} +``` + +The configuration is made up of a list of “filtering rules”. Each rule stipulates the following: + +- A name, for documentation purposes. +- `instrumentSelect` - ability to select one a more instruments apply this filtering rule for. + - Example: `pulsar_*` +- `attrSelect` - ability to select a group of attributes to apply this rule on, within the selected instruments. + - Example `topic=receipt-us-*` +- `filterInstruments` - For each instrument matched, we allow to set whether we wish to drop the attributes selected for certain instruments or keep them. + - Either: + - `dropOnly` - list the instruments we wish to drop selected attributes for. Example: if `instrumentSelect` is `pulsar_*` , `attrSelect` is `topic="incoming-logs"` and drop is `pulsar_subscription_*` and `pulsar_consumer_*` this means all messaging instruments in the topic level will remain and subscription and consumer level metrics will be dropped, **only for “incoming-logs”** topic. + - `keepOnly` - The opposite of drop. + - `dropAll` + - `keepAll` + +Order matter for the list of filtering rules. This allows us to set a default which applies to a wide range of instruments and then override it for certain instruments. + +We will supply a default filtering rules configuration which should make sense. + +In certain cases the number of rules can reach to 10k or 20k. Since each rule is essentially a regular expression, we will need to cache the resolve of (instrument, attribute) to true/false. For 10k rules, meaning 10k regular expression this might be too much, so we can use [aho-corasik](https://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_algorithm) algorithm to match only on select few regular expressions. + +We have proposed an issue in OpenTelemetry Java SDK to have a push-down predicate, allowing us to do the filtering while iterating over instruments and their attributes. It means OTel SDK will not allocate a datapoints objects if a certain (instrument, attributes) pair is filtered out. See [section above](#what-we-need-to-fix-in-opentelemetry) on what we wish to fix in OTel to see issue link. + +There will be a sub-PIP detailing the exact configuration syntax, storage, how it will be made dynamic, plugin details and more. We will to take into account user experience defining and updating that configuration, especially when it will reach a large size. Also, we want the experience of adding a rule to be smooth, so the CLI should offer auto-complete of metric names if possible and optionally to have a UI dedicated for it. We will also try to protect and validate the filtering rules such that we can reject updating it if the data points amount will exceed a certain threshold - this is to protect against accidental mistake in the configuration (we will accept it if a force parameter will be supplied). + + +## Which summaries / histograms are effected? + +As mentioned in the design, summaries will be converted into histograms, and those and existing histograms will be modified to be at the granularity level of namespace. + +### Summaries + +Prometheus Client + +- `LedgerOffloaderStatsImpl` + - `brk_ledgeroffloader_read_offload_index_latency` + - `brk_ledgeroffloader_read_offload_data_latency` + - `brk_ledgeroffloader_read_ledger_latency` + - We need to change to be reported per namespace, not per topic +- `ResourceGroupService` + - `pulsar_resource_group_aggregate_usage_secs` + - `Time required to aggregate usage of all resource groups, in seconds.` + - `pulsar_resource_group_calculate_quota_secs` + - `Time required to calculate quota of all resource groups, in seconds` + - Not high cardinality, no need to modify aggregation level. +- `ReplicatedSubscriptionsSnapshotBuilder` + - `pulsar_replicated_subscriptions_snapshot_ms` + - `Time taken to create a consistent snapshot across clusters` + - Can be NS level +- `SchemaRegistryStats` + - `pulsar_schema_del_ops_latency` + - `pulsar_schema_get_ops_latency` + - `pulsar_schema_put_ops_latency` +- `BrokerOperabilityMetrics` + - `topic_load_times` + - in milliseconds +- `TransactionBufferClientStatsImpl` + - `pulsar_txn_tb_client_abort_latency` + - `pulsar_txn_tb_client_commit_latency` + - Change it to be NS level, and not topic level. +- `PendingAckHandleStatsImpl` + - `pulsar_txn_tp_commit_latency` +- `ContextImpl` + - `pulsar_function_user_metric_`* + - not high cardinality, and each process runs only one function , so we can use histograms freely. +- `FunctionStatsManager` + - `pulsar_function_``process_latency_ms` + - `pulsar_function_``process_latency_ms_1min` + - we’ll remove the 1min ones + - not high cardinality, and each process runs only one function , so we can use histograms freely +- `WorkerStatsManager` + - `pulsar_function_worker_start_up_time_ms` + - Shouldn’t be a summary in the first place as it is init once + - `schedule_execution_time_total_ms` + - 6 more of those + +Our Summary + +- `ModularLoadManagerImpl` + - `pulsar_broker_load_manager_bundle_assigment` + - `pulsar_broker_lookup` +- `AbstractTopic` + - `pulsar_broker_publish_latency` + - broker level + +OpsStatLogger (uses DataSketches) + +- `PulsarZooKeeperClient` + - one instance per action running against ZK +- Bookkeeper client metrics + - About 10 operation latencies + +### Histograms + +StatsBucket (Our version of Explicit Bucket Histogram) + +- `ManagedLedgerMBeanImpl` + - `pulsar_storage_write_latency_le_*` + - `pulsar_storage_ledger_write_latency_le_*` + - `pulsar_entry_size_le_*` + - all above are topic level +- `CompactionRecord` + - `pulsar_compaction_latency_*` + - topic level +- `TransactionMetadataStoreStats` + - `pulsar_txn_execution_latency_le_` + - labels: cluster, coordinatorId + +## Fixing Grafana Dashboards in repository + +Pulsar repository contains multiple dashboards. We will create the same dashboards using new names alongside existing ones. We’ll add dashboards for different granularity levels: namespace, group and zoom in on specific group/topic. +The dashboards will look the same as existing, since the main changes are done to the query of each panel since the metric name has changed, not the semantics. + +This will be specified in a sub-PIP. + +# Backward Compatibility + +## Breaking changes + +All changes reported here are applicable to the newly added OTel metrics layer. At first, as mentioned in the document, we’ll be able to use both existing metrics system and OTel metric system - you can toggle each. Once everything will be stabilized, we’ll deprecate the current metric system. + +- Names + - Attribute names will use OTel semantic conventions as much as possible and also [Attribute Naming guide](https://opentelemetry.io/docs/reference/specification/common/attribute-naming/). + - Instrument names will follow guidelines mentioned in [OTel Metrics Semantic Conventions](https://opentelemetry.io/docs/reference/specification/metrics/semantic_conventions/) + - Histograms will not encode the bucket range in the instrument name + - Each domain will have a proper prefix in the instrument name. Biggest example is messaging related metrics which today are prefixed with `pulsar_` but should be `pulsar_messaging_`. +- Summary metrics are changed to be histogram metrics +- Histograms are changed from topic level to namespace level. +- The following configuration flags will be deprecated and eventually removed: `exposeConsumerLevelMetricsInPrometheus`, `exposeTopicLevelMetricsInPrometheus` and `exposeProducerLevelMetricsInPrometheus`. +- Counters / Histograms / Gauges will no longer be reset, but will be cumulative (the user will have the option to modify it to delta temporality per their needs for backends which supports it via OpenTelemetry views). +- `topic` will not contain partition number, but it will be specified via `partitionNum` attribute. +- Configuration `splitTopicAndPartitionLabelInPrometheus` will be deprecated and eventually removed. +- Most Pulsar plugins will be modified to allow reporting metrics to OTel via special object Pulsar will supply. Any plugin not using it, will not have its metrics reported to OTel. +- At phase 1 we won’t support Process Runtime in OTel metrics, but only Thread and Kubernetes. If the community will ask for it and discussion yield it as must, we’ll add it. +- All `_1min` metrics are removed from Pulsar Function metrics +- `resetMetrics` RPC operation in processes running Pulsar Function will be removed +- User defined metrics in Pulsar Functions will be reported as 0-bucket histogram (offering count and sum), instead of Summary. + +# API Changes + +The sub-PIPs will specify the exact API changes relevant to their constrained scope, since it will be much easier to review as such. + +# Security + +The sub-PIPs will specify the exact security concerns relevant to their constrained scope, since it will be much easier to review as such. + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/83g3l8doy3hj4ytm36k63z9xv8nj039x +* Mailing List voting thread: https://lists.apache.org/thread/m5k8hj874nkjx1vh0s6lwvhs7q7rgj6x diff --git a/pip/pip-275.md b/pip/pip-275.md new file mode 100644 index 0000000000000..412638056027f --- /dev/null +++ b/pip/pip-275.md @@ -0,0 +1,48 @@ +# Background knowledge +As we can see from the [doc](https://github.com/apache/pulsar/blob/ac46e2e4fc48dff74233623afa3635ef5285e34d/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java#LL1194C16-L1194C72) that `numWorkerThreadsForNonPersistentTopic` is a configuration to specify the number of worker threads to serve non-persistent topic. +Actually, `numWorkerThreadsForNonPersistentTopic` will specify the thread number of `BrokerService#topicOrderedExecutor`. Initially it was meant only for non-persistent topics, +but now it is used for anything that needs to be done under strict order for a topic, like processing Subscriptions even for a persistent topic: +* There is only one place invoke `topicOrderedExecutor` for non-persistent topics.[[1]](https://github.com/apache/pulsar/blob/50b9a93e42e412d9f17b1637287d1a4c7c7ab148/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java#L1706) +* Other places will invoke `topicOrderedExecutor` for persistent-topic or persistent-dispatcher. [[2]](https://github.com/apache/pulsar/blob/50b9a93e42e412d9f17b1637287d1a4c7c7ab148/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java#L141) [[3]](https://github.com/apache/pulsar/blob/50b9a93e42e412d9f17b1637287d1a4c7c7ab148/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java#L279) [[4]](https://github.com/apache/pulsar/blob/50b9a93e42e412d9f17b1637287d1a4c7c7ab148/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java#L82) [[5]](https://github.com/apache/pulsar/blob/50b9a93e42e412d9f17b1637287d1a4c7c7ab148/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java#L395) + +# Motivation + +Making this config has a better name and increase the ability of users to understand what they are configuring. + +# High Level Design + +Introduce `topicOrderedExecutorThreadNum` to deprecate `numWorkerThreadsForNonPersistentTopic`. + +# Detailed Design + +## Design & Implementation Details + +### Configuration + +* Introduce `topicOrderedExecutorThreadNum` with default value `Runtime.getRuntime().availableProcessors()`: +``` +private int topicOrderedExecutorThreadNum = Runtime.getRuntime().availableProcessors(); +``` +* deprecate `numWorkerThreadsForNonPersistentTopic` and change it's default value from `Runtime.getRuntime().availableProcessors()` to `-1`: +``` +private int numWorkerThreadsForNonPersistentTopic = -1; +``` +* Overwrite method `ServiceConfiguration#getTopicOrderedExecutorThreadNum()` from lombok. +``` +public int getTopicOrderedExecutorThreadNum() { + return numWorkerThreadsForNonPersistentTopic > 0 + ? numWorkerThreadsForNonPersistentTopic : topicOrderedExecutorThreadNum; + } +``` + +* And all places calling `ServiceConfiguration#getNumWorkerThreadsForNonPersistentTopic()` will call `ServiceConfiguration#getTopicOrderedExecutorThreadNum()` instead. + +# Backward & Forward Compatibility +Because we have overwritten method `getTopicOrderedExecutorThreadNum()` from lombok, so: +* if user doesn't set the `numWorkerThreadsForNonPersistentTopic`, the value of worker threads will keep `Runtime.getRuntime().availableProcessors()` +* If user has set the `numWorkerThreadsForNonPersistentTopic`, the value will keep what user set before. + + +# Links +* Mailing List discussion thread: https://lists.apache.org/thread/hx8v824v5wdoz3kn44s4t9pzgfnqkt1o +* Mailing List voting thread: https://lists.apache.org/thread/ywk6z440qt0vs32210799m508gbxfshm \ No newline at end of file diff --git a/pip/pip-276.md b/pip/pip-276.md new file mode 100644 index 0000000000000..16fe4ba1d3ab0 --- /dev/null +++ b/pip/pip-276.md @@ -0,0 +1,32 @@ +# Motivation + +The metrics are all started with `pulsar_`, so that both users and operators can quickly find the metrics of the entire system through this prefix. However, due to some other reasons, +it was found that `topic_load_times` was missing the prefix, so want to get it right. + +# High Level Design + +In master branch, keep the old metric `topic_load_times` and add below new metrics: + +* `pulsar_topic_load_times` + +After release-3.1.0, remove ``topic_load_times`. + + +### Metrics + +Add new metrics: + +* `pulsar_topic_load_times` : The topic load latency calculated in milliseconds + +# Monitoring + +After this PIP, users can use `topic_load_times` and `pulsar_topic_load_times` to monitor topic load times. + + +# Links + + +* Mailing List discussion thread: https://lists.apache.org/thread/fcg3f5mm2640fxq4cj8pz6n3lso293f8 +* Mailing List voting thread: https://lists.apache.org/thread/vky6jcn0llx56599fgo73dh6cxfpmxsm diff --git a/pip/pip-277.md b/pip/pip-277.md new file mode 100644 index 0000000000000..295a8c8753688 --- /dev/null +++ b/pip/pip-277.md @@ -0,0 +1,48 @@ +# Motivation + +After configuring the geo-replication on Pulsar clusters, the `clusters list` API will return multiple clusters, including the local Pulsar cluster and remote clusters like + +``` +bin/pulsar-admin clusters list +us-west +us-east +us-cent +``` +But in this return, you can't distinguish the local and the remote cluster. When you need to remove the geo-replication configuration, it will be hard to decide which cluster should be removed on replicated tenants and namespaces unless you record the cluster information. + +# High Level Design + +Add `--current` option to the cluster list cmd and mark the current cluster with `(*)` +``` +bin/pulsar-admin clusters list --current +us-west(*) +us-east +us-cent +``` + +# Detailed Design + +## Implementation Details + +Add `--current` option to the cluster list cmd +``` +@Parameter(names = { "-c", "--current" }, description = "Print the current cluster with (*)", required = false) +private boolean current = false; +``` + +``` +void run() throws PulsarAdminException { + java.util.List clusters = getAdmin().clusters().getClusters(); + String clusterName = getAdmin().brokers().getRuntimeConfigurations().get("clusterName"); + List result = clusters.stream().map(c ->c.equals(clusterName) ? (current ? c + "(*)" : c) : c).collect(Collectors.toList()); + print(result); +} +``` + +# Links + + +* Mailing List discussion thread: https://lists.apache.org/thread/800r6ld5wg7bttbywmk38m1qx12hs6nl +* Mailing List voting thread: https://lists.apache.org/thread/rqn3rd3c4hj11o3b12ygopmztj2yy7pv diff --git a/pip/pip-278.md b/pip/pip-278.md new file mode 100644 index 0000000000000..0b620300de376 --- /dev/null +++ b/pip/pip-278.md @@ -0,0 +1,224 @@ +# Background knowledge + +Apache Pulsar is a distributed messaging system that supports multiple messaging protocols and storage methods. +Among them, Pulsar Topic Compaction provides a key-based data retention mechanism that allows you only to keep the most recent message associated with that key to reduce storage space and improve system efficiency. + +Another Pulsar's internal use case, the Topic Compaction of the new load balancer, changed the strategy of compaction. It only keeps the first value of the key. For more detail, see [PIP-215](https://github.com/apache/pulsar/issues/18099). + +More topic compaction details can be found in [Pulsar Topic Compaction](https://pulsar.apache.org/docs/en/concepts-topic-compaction/). + +# Motivation + +Currently, the implementation of Pulsar Topic Compaction is fixed and does not support customized strategy, which limits users from using more Compactor policies in their applications. + +For example, current topic compaction can work with pulsar format data in KoP, but it can't work with Kafka format data since the data written to the entry is in Kafka format. +The Pulsar compactor doesn't aware of Kafka format data. And it doesn't make sense to support Kafka format data handling in Pulsar. We need to implement a pluggable compactor to support Kafka format data handling in KoP. + +Another long-term consideration is that we may need to support writing the compacted data anywhere, S3, in columnar format, or even partitioning. + +So we need to make the whole topic compaction service (including Write API & Read API) pluggable to support more customize compaction service implementation. + +# Goals + +## In Scope + +* Abstract topic compaction service interface and support topic compaction service pluggable. + +* Migrate the current implementation to a new interface implementation. + +* Makes existing tests compatible with new implementations. + +## Out of Scope + +* For CompactorMetrics, keep the current implementation and don't define related methods in the topic compaction service interface. In the future, it will use the `Otel` interface or other metrics API instead. + +* For `StrategicTwoPhaseCompactor`, it's out of the scope for regular compaction. It's only used for the load balancer. So it won't change. + + +# High Level Design + +To make the whole topic compaction service pluggable, we need to abstract `TopicCompactionService` interface, it can provide the capability that the compactor has and provide the read API to read entries from compacted data. + +We should combine `CompactedTopicImpl` and `TwoPhaseCompactor` to the Pulsar implementation of the topic compaction service and make behavior with the current implementation consistent. + +Class Diagram of core class: +```mermaid +classDiagram + direction BT + class CompactedTopic { + <> + + deleteCompactedLedger(long) CompletableFuture~Void~ + + getCompactionHorizon() Optional~Position~ + + newCompactedLedger(Position, long) CompletableFuture~CompactedTopicContext~ + + asyncReadEntriesOrWait(ManagedCursor, int, boolean, ReadEntriesCallback, Consumer) void + + readLastEntryOfCompactedLedger() CompletableFuture~Entry~ + } + class CompactedTopicImpl { + + newCompactedLedger(Position, long) CompletableFuture~CompactedTopicContext~ + + getCompactedTopicContext() Optional~CompactedTopicContext~ + + asyncReadEntriesOrWait(ManagedCursor, int, boolean, ReadEntriesCallback, Consumer) void + + getCompactionHorizon() Optional~Position~ + + deleteCompactedLedger(long) CompletableFuture~Void~ + + getCompactedTopicContextFuture() CompletableFuture~CompactedTopicContext~ + + readLastEntryOfCompactedLedger() CompletableFuture~Entry~ + } + class CompactionServiceFactory { + <> + + newTopicCompactionService(String) CompletableFuture~TopicCompactionService~ + + initialize(PulsarService) CompletableFuture~Void~ + } + class Compactor { + + getStats() CompactorMXBean + + compact(String) CompletableFuture~Long~ + } + class PulsarCompactionServiceFactory { + + getNullableCompactor() Compactor? + + getCompactor() Compactor + + newTopicCompactionService(String) CompletableFuture~TopicCompactionService~ + + initialize(PulsarService) CompletableFuture~Void~ + + close() void + } + class PulsarCompactorSubscription { + + acknowledgeMessage(List~Position~, AckType, Map<String, Long>) void + } + class PulsarTopicCompactionService { + + compact() CompletableFuture~Void~ + + readCompactedEntries(Position, int) CompletableFuture~List~Entry~~ + + getCompactedLastPosition() CompletableFuture~Position~ + + readCompactedLastEntry() CompletableFuture~Entry~ + + getCompactedTopic() CompactedTopicImpl + + close() void + } + class TopicCompactionService { + <> + + compact() CompletableFuture~Void~ + + readCompactedEntries(Position, int) CompletableFuture~List~Entry~~ + + getCompactedLastPosition() CompletableFuture~Position~ + + readCompactedLastEntry() CompletableFuture~Entry~ + } + class TwoPhaseCompactor + + CompactedTopicImpl ..> CompactedTopic + PulsarCompactionServiceFactory ..> CompactionServiceFactory + PulsarCompactionServiceFactory "1" *--> "compactor 1" Compactor + PulsarCompactionServiceFactory ..> PulsarTopicCompactionService : «create» + PulsarCompactionServiceFactory ..> TwoPhaseCompactor : «create» + PulsarCompactorSubscription "1" *--> "compactedTopic 1" CompactedTopic + PulsarTopicCompactionService ..> CompactedTopicImpl : «create» + PulsarTopicCompactionService "1" *--> "compactedTopic 1" CompactedTopicImpl + PulsarTopicCompactionService ..> TopicCompactionService + TwoPhaseCompactor --> Compactor +``` + +# Detailed Design + +## Design & Implementation Details + +* Define a standard TopicCompactionService interface. + + ```java + import javax.annotation.Nonnull; + + public interface TopicCompactionService extends AutoCloseable { + /** + * Compact the topic. + * Topic Compaction is a key-based retention mechanism. It keeps the most recent value for a given key and + * user reads compacted data from TopicCompactionService. + * + * @return a future that will be completed when the compaction is done. + */ + CompletableFuture compact(); + + /** + * Read the compacted entries from the TopicCompactionService. + * + * @param startPosition the position to start reading from. + * @param numberOfEntriesToRead the maximum number of entries to read. + * @return a future that will be completed with the list of entries, this list can be null. + */ + CompletableFuture> readCompactedEntries(@Nonnull Position startPosition, int numberOfEntriesToRead); + + /** + * Read the last compacted entry from the TopicCompactionService. + * + * @return a future that will be completed with the compacted last entry, this entry can be null. + */ + CompletableFuture readLastCompactedEntry(); + + /** + * Get the last compacted position from the TopicCompactionService. + * + * @return a future that will be completed with the last compacted position, this position can be null. + */ + CompletableFuture getLastCompactedPosition(); + } + ``` + +* Define a standard CompactionServiceFactory interface to manage `TopicCompactionService`. + + ```java + public interface CompactionServiceFactory extends AutoCloseable { + + /** + * Initialize the compaction service factory. + * + * @param pulsarService + * the pulsar service instance + * @return a future represents the initialization result + */ + CompletableFuture initialize(PulsarService pulsarService); + + /** + * Create a new topic compaction service for topic. + * + * @param topic + * the topic name + * @return a future represents the topic compaction service + */ + CompletableFuture newTopicCompactionService(String topic); + } + ``` + +* Implement `PulsarCompactionServiceFactory` and `PulsarCompactionService` + +* Combining `CompactedTopicImpl` and `TwoPhaseCompactor` to `PulsarTopicCompactionService` + +* Rename `CompactorSubscription` to `PulsarCompactorSubscription`, since it is only applicable to the implementation of Pulsar. + +* For `CompactorMetrics`: keep the current implementation. Currently, it only supports `PulsarTopicCompactionService`. In the future, it will use the `Otel` API or other metrics API instead, and customized `TopicCompactedService` should implement the `Otel` API or other metrics API. + +* Fix tests and makes them compatible with new implementations. + +## Public-facing Changes + + +### Configuration + +broker.conf +``` +compactionServiceFactoryClassName=org.apache.pulsar.compaction.PulsarCompactionServiceFactory +``` + +# Backward & Forward Compatability + +## Revert + + +## Upgrade + + +# Alternatives + +* Only make the compactor pluggable +* Make the compaction data serializer and deserializer pluggable in the current Pulsar implementation. + +But they will introduce some short-term configurations and interfaces, so they are not good for the long-term view of Pulsar. +For a discussion of alternatives see: [PIP-274](https://github.com/apache/pulsar/pull/20493) + + +# General Notes + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/ox2bot3p9j9fydqkw3v5gt5twc8jslvd +* Mailing List voting thread: https://lists.apache.org/thread/1pcsmn1osdkz04dtgy3fchgmzoko5jnf \ No newline at end of file diff --git a/pip/pip-279.md b/pip/pip-279.md new file mode 100644 index 0000000000000..1a7f867118955 --- /dev/null +++ b/pip/pip-279.md @@ -0,0 +1,52 @@ +# Motivation + +reformat property,for a http header name cannot contain the following prohibited characters: =,;: \t\r\n\v\f + +for example: +{"city=shanghai":"tag"} +when we run `bin/pulsar-admin topics get-message-by-id `, it will throw exception, the exception is: +`Reason: java.util.concurrent.CompletionException: org.apache.pulsar.client.admin.internal.http.AsyncHttpConnector$RetryException: Could not complete the operation. Number of retries has been exhausted. Failed reason: a header name cannot contain the following prohibited characters: =,;: \t\r\n\v\f: =` + +# High Level Design + +In master branch, +in an http request:getMessageById("/{tenant}/{namespace}/{topic}/ledger/{ledgerId}/entry/{entryId}"), +replace `"X-Pulsar-PROPERTY-" + msgProperties.getKey()` with `"X-Pulsar-PROPERTY"` + +After release-3.1.0, this feature begins to take effect. + +# Concrete Example + +for example, the current message's properties likes this: +``` +"name": "James" +"gender": "man" +"details=man": "good at playing basketball" +``` + +## BEFORE +old response header format: +``` +headers: { + "X-Pulsar-PROPERTY-name": "James", + "X-Pulsar-PROPERTY-gender": "man", + "X-Pulsar-PROPERTY-details=man": "good at playing basketball" +} +``` +but it will throw exception in the end check + +## AFTER +new response header format: +``` +headers: { +"X-Pulsar-PROPERTY": '{"name": "James", "gender": "man", "details=man": "good at playing basketball"}' +} +``` + +# Links + + +* Mailing List discussion thread: https://lists.apache.org/thread/vfc99mbj2z2xgwfs1hq1zxrow13qm2n7 +* Mailing List voting thread: https://lists.apache.org/thread/g354684m9h495o3p0kmzb7fh7vfxhddx diff --git a/pip/pip-280.md b/pip/pip-280.md new file mode 100644 index 0000000000000..bbc7fab32f5b8 --- /dev/null +++ b/pip/pip-280.md @@ -0,0 +1,119 @@ +# Title: Refactor CLI Argument Parsing Logic for Measurement Units using JCommander's custom converter + +## Motivation + +In the current Pulsar codebase, the logic to parse CLI arguments for measurement units like time and bytes is +scattered across various CLI classes. Each value read has its distinct parsing implementation, leading to a lack of code +reuse. + +## Goals + +This PIP is to refactor the argument parsing logic to leverage the `@Parameter.converter` +functionality provided by JCommander [link 3]. This will isolate the measurement-specific parsing logic and increase +code +reusability. + +### In Scope + +- Refactor all `Cmd` classes to utilize the converter functionality of JCommander. This will streamline the parsing + logic and simplify the codebase. +- Refer to bottom section "Concrete Example", before "Links" +- Or on-going PR with small use case in https://github.com/apache/pulsar/pull/20663 + +### Out of Scope + +- Creation of a "util" module is out of the scope of this PIP. + +## Design & Implementation Details + +- The refactoring will be carried out on a class-by-class basis or per inner-class basis. +- Target command classes for this refactoring include + - `CmdNamespaces.java` + - `CmdTopics.java`, + - `CmdTopicPolicies.java`. + +## Note + +- Additional classes may be included as the refactoring progresses. +- Respective PRs will be added here also. +- The refactoring should not introduce any breaking change +- New parameters should be covered by unit test (at least by existing and preferably new) + +## Concrete Example + +Consider the code snippet +from [CmdNamespaces.java](https://github.com/apache/pulsar/blob/200fb562dd4437857ccaba3850bd64b0a9a50b3c/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java#L2352-L2359) +for example. The existing code uses a local variable `maxBlockSizeStr` to temporarily store the value +of `--maxBlockSize` or `-mbs`. This is then parsed and validated in a separate section of the code. + +### BEFORE + +```java + @Parameter( + names = {"--maxBlockSize", "-mbs"}, + description = "Max block size (eg: 32M, 64M), default is 64MB s3 and google-cloud-storage requires this parameter", + required = false) +private String maxBlockSizeStr; +``` + +parsing like below .... + +```java + // parsing like.... + int maxBlockSizeInBytes=OffloadPoliciesImpl.DEFAULT_MAX_BLOCK_SIZE_IN_BYTES; + if(StringUtils.isNotEmpty(maxBlockSizeStr)){ + long maxBlockSize=validateSizeString(maxBlockSizeStr); + if(positiveCheck("MaxBlockSize",maxBlockSize) + &&maxValueCheck("MaxBlockSize",maxBlockSize,Integer.MAX_VALUE)) { + maxBlockSizeInBytes=Long.valueOf(maxBlockSize).intValue(); + } + } +``` + +### AFTER + +```java + @Parameter( + names = {"--maxBlockSize", "-mbs"}, + description = "Max block size (eg: 32M, 64M), default is 64MB s3 and google-cloud-storage requires this parameter", + required = false, converter = MemoryUnitToByteConverter.class) // <--- parsing logic "inline" easy to follow +private long maxBlockSizeStr=DEFAULT_MAX_BLOCK_SIZE_IN_BYTES; // <---- default value in line +``` + +... and actual parsing in isolation, ready for reuse like... + +```java +class MemoryUnitToByteConverter implements IStringConverter { + + private static Set sizeUnit = Sets.newHashSet('k', 'K', 'm', 'M', 'g', 'G', 't', 'T'); + private final long defaultValue; + + public MemoryUnitToByteConverter(long defaultValue) { + this.defaultValue = defaultValue; + } + + @Override + public Long convert(String memoryLimitArgument) { + parseBytes(memoryLimitArgument); + } + + long parseBytes(String memoryLimitArgument) { + if (StringUtils.isNotEmpty(memoryLimitArgument)) { + long memoryLimitArg = validateSizeString(memoryLimitArgument); + if (positiveCheckStatic("memory-limit", memoryLimitArg)) { + return memoryLimitArg; + } + } + return defaultValue; + } + ... + more internal + helper methods +} +``` + +## Links + +- Mailing List discussion thread: https://lists.apache.org/thread/b77bfnjlt62w7zywcs8tqklvyokpykok +- Mailing List voting thread: https://lists.apache.org/thread/0r3bh0h7f86g2x9odvrd1fp2gwddq904 +- [3] https://jcommander.org/#_custom_types_converters_and_splitters diff --git a/pip/pip-281.md b/pip/pip-281.md new file mode 100644 index 0000000000000..7bcf420c3d64a --- /dev/null +++ b/pip/pip-281.md @@ -0,0 +1,103 @@ +# Title: [io] Add notifyError method on PushSource + +## Motivation + +In function framework, when [source.read()](https://github.com/apache/pulsar/blob/f7c0b3c49c9ad8c28d0b00aa30d727850eb8bc04/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java#L496-L506) method throw an exception, it will trigger close function instance. If it is in the k8s environment, it will be restarted, +you can use the [PushSource](https://github.com/apache/pulsar/blob/branch-3.0/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/PushSource.java) class and extend it to quickly implement the push message model. +It overrides the `read` method and provides the `consume` method for the user to call. + +However, if the source connector extends from the class, +it cannot notify the function framework if it encounters an exception while consuming data internally, +in other words, the function call `source.read()` never triggers an exception and never exits the process. + + +## Goals + +Add `notifyError` method on PushSource, This method can receive an exception and put the exception in the queue. The next time an exception is `read`, will throws exception. +```java + + public Record read() throws Exception { + Record record = queue.take(); + if (record instanceof ErrorNotifierRecord) { + throw ((ErrorNotifierRecord) record).getException(); + } + return record; + } + + + /** + * Allows the source to notify errors asynchronously. + * @param ex + */ + public void notifyError(Exception ex) { + consume(new ErrorNotifierRecord(ex)); + } +} +``` + +Just like the implementation of the current [BatchPushSource](https://github.com/apache/pulsar/blob/branch-3.0/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/BatchPushSource.java) + + +### Compatibility + +This PIP is to provide a method for users rather than introducing a new interface. + +- So it is forward compatible +- However, connectors using this method are not backward compatible. +For example, If a Kafka source connector built upon pulsar-io v3.1 (including features introduced in this PIP) and uses the `notifyError` method, +when it switches back to pulsar-io v3.0 (excluding features introduced in this PIP), it will encounter errors during compilation. + +### In Scope + +After this PIP, the source connectors can extends the `PushSource`, and use `notifyError` method to throw exception. Such as: +- [KafkaSourceConnector](https://github.com/apache/pulsar/blob/branch-3.0/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java) +- [CanalSourceConnector](https://github.com/apache/pulsar/blob/82237d3684fe506bcb6426b3b23f413422e6e4fb/pulsar-io/canal/src/main/java/org/apache/pulsar/io/canal/CanalAbstractSource.java#L43) +- [MongoSourceConnector](https://github.com/apache/pulsar/blob/82237d3684fe506bcb6426b3b23f413422e6e4fb/pulsar-io/mongo/src/main/java/org/apache/pulsar/io/mongodb/MongoSource.java#L59) +- etc. + +### Out of Scope +None + +## Design & Implementation Details + +- Abstract BatchPushSource logic to AbstractPushSource. +- Let PushSource to extends AbstractPushSource to extend a new method(notifyError). + +Please refer this PR: https://github.com/apache/pulsar/pull/20791 + +## Note +None + + +## Concrete Example + +### BEFORE +- Not possible + +### AFTER + +```java +public class PushSourceTest { + + PushSource testBatchSource = new PushSource() { + @Override + public void open(Map config, SourceContext context) throws Exception { + + } + + @Override + public void close() throws Exception { + + } + }; + + @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "test exception") + public void testNotifyErrors() throws Exception { + testBatchSource.notifyError(new RuntimeException("test exception")); + testBatchSource.readNext(); + } +} +``` + +## Links +None diff --git a/pip/pip-282.md b/pip/pip-282.md new file mode 100644 index 0000000000000..d07bd15964fb7 --- /dev/null +++ b/pip/pip-282.md @@ -0,0 +1,293 @@ +# Background knowledge + +Key_Shared is one of the subscription types which allows multiple consumer connections. +Messages are distributed across consumers, and messages with the same key or same ordering key are delivered to only one consumer. +No matter how many times the message is re-delivered, it is delivered to the same consumer. + +When disabling `allowOutOfOrderDelivery`, Key_Shared subscription guarantees a key will be processed in order by a single consumer, even if a new consumer is connected. + +# Motivation + +Key_Shared has a mechanism called the "recently joined consumers" to keep message ordering. +However, currently, it doesn't care about some corner cases. +More specifically, we found two out-of-order issues cased by: + +1. [issue-1] The race condition in the "recently joined consumers", where consumers can be added before finishing reading and dispatching messages from ledgers. +2. [issue-2] Messages could be added to messagesToRedeliver without consumer-side operations such as unacknowledgement. + +We should care about these cases in Key_Shared subscription. + +## [issue-1] + +Key_Shared subscription has out-of-order cases because of the race condition of [the "recently joined consumers"](https://github.com/apache/pulsar/blob/e220a5d04ae16d1b8dfd7e35cdddf43f3a43fe86/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java#L378-L386). +Consider the following flow. + +1. Assume that the current read position is `1:6` and the recently joined consumers is empty. +2. Called [OpReadEntry#internalReadEntriesComplete](https://github.com/apache/pulsar/blob/e220a5d04ae16d1b8dfd7e35cdddf43f3a43fe86/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java#L92-L95) from thread-1. + Then, the current read position is updated to `1:12` (Messages from `1:6` to `1:11` have yet to be dispatched to consumers). +3. Called [PersistentStickyKeyDispatcherMultipleConsumers#addConsumer](https://github.com/apache/pulsar/blob/35e9897742b7db4bd29349940075a819b2ad6999/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java#L130-L139) from thread-2. + Then, the new consumer is stored to the recently joined consumers with read position `1:12`. +4. Called [PersistentDispatcherMultipleConsumers#trySendMessagesToConsumers](https://github.com/apache/pulsar/blob/e220a5d04ae16d1b8dfd7e35cdddf43f3a43fe86/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java#L169) from thread-5. + Then, messages from `1:6` to `1:11` can be dispatched to the new consumer since the "recently joined consumers" allow brokers to send messages before the joined position (i.e., `1:12` here). **However, it is not expected.** + For example, if existing consumers have some unacked messages, disconnecting, and redelivering them can cause out-of-order. + +An example scenario is shown below. + +1. Assume that the [entries](https://github.com/apache/pulsar/blob/e220a5d04ae16d1b8dfd7e35cdddf43f3a43fe86/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java#L169) has the following messages, and the dispatcher has two consumers (`c1` `messagesForC` is 1, `c2` `messageForC` is 1000), and the selector will return `c1` if `key-a` and `c2` if `key-b`. + - `1:6` key: `key-a` + - `1:7` key: `key-a` + - `1:8` key: `key-a` + - `1:9` key: `key-b` + - `1:10` key: `key-b` + - `1:11` key: `key-b` +2. Send `1:6` to `c1` and `1:9` - `1:11` to `c2`. + - So, the current read position is `1:12`. + - `c1` never acknowledge `1:6`. +3. Add new consumer `c3`, the selector will return `c3` if `key-a`, and the `recentlyJoinedConsumers` is `{c3=1:12}` +4. Send `1:7` - `1:8` to `c3` because `1:7`, and `1:8` are less than the recently joined consumers position, `1:12`. +5. Disconnect `c1`. +6. Send `1:6` to `c3`. + As a result `c3` receives messages with the following order: `1:7`, `1:8`, `1:6` // out-of-order + +## [issue-2] +Key_Shared subscription has out-of-order cases because messages could be added to messagesToRedeliver without consumer-side operations such as unacknowledgement. +Consider the following flow. + +1. Assume that, + readPosition: `2:1` + messagesToRedeliver: [] + recentlyJoinedConsumers: [] + c1: messagesForC: 1, pending: [] + c2: messagesForC: 1000, pending: [] // Necessary to ensure that the dispatcher reads entries even if c1 has no more permits. + selector: key-a: c1 +2. Dispatch `2:1` (key: `key-a`, type: Normal) + readPosition: `2:2` + messagesToRedeliver: [] + recentlyJoinedConsumers: [] + c1: messagesForC: 0, pending: [`2:1`] + c2: messagesForC: 1000, pending: [] + selector: key-a: c1 +3. Try to dispatch `2:2` (key: `key-a`, type: Normal), but it can't be sent to c1 because c1 has no more permits. Then, it is added to messagesToRedeliver. + readPosition: `2:3` + messagesToRedeliver: [`2:2`] + recentlyJoinedConsumers: [] + c1: messagesForC: 0, pending: [`2:1`] + c2: messagesForC: 1000, pending: [] + selector: key-a: c1 +4. Add consumer c3 + readPosition: `2:3` + messagesToRedeliver: [`2:2`] + recentlyJoinedConsumers: [c3: `2:3`] + c1: messagesForC: 0, pending: [`2:1`] + c2: messagesForC: 1000, pending: [] + c3: messagesForC: 1000, pending: [] + selector: key-a: c3 // modified +5. Dispatch `2:2` (key: `key-a`, type: Replay) from messagesToRedeliver. + readPosition: `2:3` + messagesToRedeliver: [] + recentlyJoinedConsumers: [c3: `2:3`] + c1: messagesForC: 0, pending: [`2:1`] + c2: messagesForC: 1000, pending: [] + c3: messagesForC: 999, pending: [`2:2`] + selector: key-a: c3 +6. Disconnect c1 and redelivery `2:1` + readPosition: `2:3` + messagesToRedeliver: [] + recentlyJoinedConsumers: [c3: `2:3`] + c2: messagesForC: 1000, pending: [] + c3: messagesForC: 998, pending: [`2:2`, `2:1`] // out-of-order + selector: key-a: c3 + +# Goals + +## In Scope + +Fix out-of-order issues above. + +## Out of Scope + +Simplify or improve the specification of Key_Shared. + +# High Level Design + +The root cause of the issues described above is that `recentlyJoinedConsumers` uses "read position" as joined positions for consumers, because this does not guarantee that messages less than or equal to it have already been scheduled to be sent. +Instead, we propose to use "last sent position" as joined positions for consumers. + +Also, change (or add) some stats to know Key_Shared subscription status easily. + +# Detailed Design + +## Design & Implementation Details + +First, introduce the new position, like the mark delete position and the individually deleted messages. In other words, + +- All positions less than or equal to it are already scheduled to be sent. +- Manage individually sent positions to update the position as expected. + +An example of updating the individually sent messages and the last sent position will be as follows. + +Initially, the last sent position is `3:0`, and the individually sent positions is `[]`. +1. Read `3:1` - `3:10` positions +2. Send `3:1` - `3:3`, `3:5`, and `3:8` - `3:10` positions + - last sent position: `3:3` + - individually sent positions: `[(3:4, 3:5], (3:7, 3:10]]` +3. Send `3:7` position + - last sent position: `3:3` + - individually sent positions: `[(3:4, 3:5], (3:6, 3:10]]` +4. Send `3:6` position + - last sent position: `3:3` + - individually sent positions: `[(3:4, 3:10]]` +5. Send `3:4` position + - last sent position: `3:10` + - individually sent positions: `[]` + +More specifically, the recently joined consumers related fields will be as follows. +```diff +diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java +index 8f05530f58b..2b17c580832 100644 +--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java ++++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java +@@ -69,8 +69,12 @@ public class PersistentStickyKeyDispatcherMultipleConsumers extends PersistentDi + * This means that, in order to preserve ordering, new consumers can only receive old + * messages, until the mark-delete position will move past this point. + */ ++ // Map(key: recently joined consumer, value: last sent position when joining) + private final LinkedHashMap recentlyJoinedConsumers; + ++ private PositionImpl lastSentPosition; ++ private final RangeSetWrapper individuallySentPositions; ++ + PersistentStickyKeyDispatcherMultipleConsumers(PersistentTopic topic, ManagedCursor cursor, + Subscription subscription, ServiceConfiguration conf, KeySharedMeta ksm) { + super(topic, cursor, subscription, ksm.isAllowOutOfOrderDelivery()); +``` + +Next, rename the consumer stats as follows. +```diff +--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ConsumerStatsImpl.java ++++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ConsumerStatsImpl.java +@@ -74,8 +74,8 @@ public class ConsumerStatsImpl implements ConsumerStats { + /** Flag to verify if consumer is blocked due to reaching threshold of unacked messages. */ + public boolean blockedConsumerOnUnackedMsgs; + +- /** The read position of the cursor when the consumer joining. */ +- public String readPositionWhenJoining; ++ /** The last sent position of the cursor when the consumer joining. */ ++ public String lastSentPositionWhenJoining; + + /** Address of this consumer. */ + private String address; +``` + +Note that I just renamed the stats from `readPositionWhenJoining` to `lastSentPositionWhenJoining` without keeping the backward-compatibility because readPositionWhenJoining is no longer meaningful and redundant. + +And finally, modify the subscription stats of the definition as follows. +```diff +diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +index dc666f3a18e..7591369277f 100644 +--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java ++++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +@@ -1177,7 +1177,14 @@ public class PersistentSubscription extends AbstractSubscription implements Subs + .getRecentlyJoinedConsumers(); + if (recentlyJoinedConsumers != null && recentlyJoinedConsumers.size() > 0) { + recentlyJoinedConsumers.forEach((k, v) -> { +- subStats.consumersAfterMarkDeletePosition.put(k.consumerName(), v.toString()); ++ // The dispatcher allows same name consumers ++ final StringBuilder stringBuilder = new StringBuilder(); ++ stringBuilder.append("consumerName=").append(k.consumerName()) ++ .append(", consumerId=").append(k.consumerId()); ++ if (k.cnx() != null) { ++ stringBuilder.append(", address=").append(k.cnx().clientAddress()); ++ } ++ subStats.consumersAfterMarkDeletePosition.put(stringBuilder.toString(), v.toString()); + }); + } + } +``` + +## How The Proposal Resolves The Issue + +**[issue-1]** +Consider the following flow. + +1. Assume that the [entries](https://github.com/apache/pulsar/blob/e220a5d04ae16d1b8dfd7e35cdddf43f3a43fe86/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java#L169) has the following messages, and the dispatcher has two consumers (`c1` `messagesForC` is 1, `c2` `messageForC` is 1000), and the selector will return `c1` if `key-a` and `c2` if `key-b`. + - `1:6` key: `key-a` + - `1:7` key: `key-a` + - `1:8` key: `key-a` + - `1:9` key: `key-b` + - `1:10` key: `key-b` + - `1:11` key: `key-b` +2. Send `1:6` to `c1` and `1:9` - `1:11` to `c2`. + - So, the current last sent position is `1:6` and the individually sent positions is `[(1:8, 1:11]]`. + - `c1` never acknowledge `1:6`. +3. Add new consumer `c3`, the selector will return `c3` if `key-a`, and the `recentlyJoinedConsumers` is `{c3=1:6}`. +4. Can't send `1:7` - `1:8` to `c3` because `1:7`, and `1:8` are greater than the recently joined consumers position, `1:6`. +5. Disconnect `c1`. +6. Send `1:6` - `1:8` to `c3`. + Now, `c3` receives messages with expected order regarding `key-a`. + +**[issue-2]** +This mechanism guarantees all messages less than or equal to the last sent position are already scheduled to be sent. Therefore, skipped messages (e.g. `2:2`) are greater than the last sent position. + +1. The last sent position is `2:1`. +2. When add new consumer `c3`, `recentlyJoinedConsumers` is `[{c3: 2:1}]`. + The dispatcher can't send `2:2` to `c3` because `2:2` is greater than the joined position `2:1`. +3. When `c3` receives `2:1` and acknowledges it, then the mark delete position is advanced to `2:1`. + When all messages up to the joined position (i.e., `2:1` ) have been acknowledged, then the consumer (i.e., `c3` ) is removed from `recentlyJoinedConsumers`. + Therefore, `c3` will be able to receive `2:2`. + +**[stats]** +`readPositionWhenJoining` is replaced with `lastSentPositionWhenJoining` in each consumer stats instead. + +## Public-facing Changes + +### Public API + +### Binary protocol + +### Configuration + +### CLI + +### Metrics +* The consumer stats `readPositionWhenJoining` is renamed to `lastSentPositionWhenJoining`. +* The subscription stats `consumersAfterMarkDeletePosition` of the definition is modified as described. + +# Monitoring + +# Security Considerations + +# Backward & Forward Compatability + +## Revert + +## Upgrade + +# Alternatives + +### Alternative-1 +See https://github.com/apache/pulsar/pull/20179 in detail. It isn't merged when publishing this proposal. +The only difference is the message key, i.e., this approach leverages per-key information in addition to the proposal described in this PIP. +For example, the `recentlyJoinedConsumers` will be: + +``` +// Map(key: recently joined consumer, value: Map(key: message key, value: last sent position in the key when joining)) +private final LinkedHashMap> recentlyJoinedConsumers; +``` + +With this change, message delivery stuck on one key will no longer prevent other keys from being dispatched. +However, the codes will be vulnerable to an increase in keys, causing OOM in the worst case. + +### Alternative-2 +Make updating the read position, dispatching messages, and adding new consumers exclusive to ensure that messages less than the read position have already been sent. +However, introducing such an exclusion mechanism disrupts the throughput of the dispatcher. + +# General Notes + +# Links + + +* Mailing List discussion thread: https://lists.apache.org/thread/69fpb0d30y7pc02k3zvg2lpb2lj0smdg +* Mailing List voting thread: https://lists.apache.org/thread/45x056t8njjnzflbkhkofh00gcy4z5g6 diff --git a/pip/pip-284.md b/pip/pip-284.md new file mode 100644 index 0000000000000..5d5b0adbfe338 --- /dev/null +++ b/pip/pip-284.md @@ -0,0 +1,114 @@ +# Background knowledge + +Apache Pulsar introduced the topic-level policies by [PIP-39](https://github.com/apache/pulsar/wiki/PIP-39%3A-Namespace-Change-Events). +It uses the reader API to read messages and relies on compaction to implement table-like logic in the memory. + +[PIP-104](https://github.com/apache/pulsar/issues/12356) Introduced new consumer type `TableView` to support the same logic by +Public Stable API. + +# Motivation + +Due to a lot of problems caused by the complex topic policy logic is as follows. we can use a new stable `TableView` API instead of +previous one. + +- https://github.com/apache/pulsar/pull/20763 +- https://github.com/apache/pulsar/pull/20613 +- https://github.com/apache/pulsar/pull/19746 +- etc... + +# Goals + +## In Scope + +- Reduce complex logic. +- Reuse the public stable API to improve the project quality. + +## Out of Scope + +none. + +# High Level Design + +```mermaid +sequenceDiagram + participant client + participant topic_policy_service + participant system_event_topic + client ->> topic_policy_service: Hi, what's the topic policy of `persistent://public/default/test` + loop policy update event fetching + topic_policy_service ->> system_event_topic: Do you have new events?; + alt new policy event + system_event_topic ->> topic_policy_service: Yes, Sure. here you are. + else no policy event + system_event_topic ->> topic_policy_service: No, Please wait for a moment. + end; + end + Note right of topic_policy_service: Update the local cache table by keeping the latest one; + topic_policy_service -->> client: Here you are. +``` + +# Detailed Design + +## Design & Implementation Details + +The new implementation will continue reuse the previous system topic `namespace/__change_event` to store the topic policy data. +and it will change the reading logic from the raw reader to TableView to enhance the robustness. + +## Public-facing Changes + +none. + +### Public API + +none. + +### Binary protocol + +none. + +### Configuration + +none. + +### CLI + +none. + +### Metrics + +none. + +# Monitoring + +none. + +# Security Considerations + +none. + +# Backward & Forward Compatibility + +Without backward compatibility since we will reuse the same system topic `namesapce/__change_event`. + +## Revert + +none. + +## Upgrade + +none. + +# Alternatives + +none. + +# General Notes + +none. + +# Links + +none. + +* Mailing List discussion thread: https://lists.apache.org/thread/9v00sfpfxjpm775vgltkjoxwnllsgskg +* Mailing List voting thread: https://lists.apache.org/thread/gx43mzh88xp5ttz2gqghfqpz1yq51k60 diff --git a/pip/pip-286.md b/pip/pip-286.md new file mode 100644 index 0000000000000..8449f424fba6d --- /dev/null +++ b/pip/pip-286.md @@ -0,0 +1,59 @@ +# Background knowledge + +In https://github.com/apache/pulsar/pull/11139 we support get position based on timestamp, but it doesn't work well with topic compaction enabled because the data may have been move to the compacted ledger. + +In [PIP-278](https://github.com/apache/pulsar/pull/20624) we introduced the pluggable topic compaction service to extend the compaction. + +# Motivation + +In order for `get-message-id` to work well with topic compaction enabled we need to find the position according to publish time from topic compaction service, +but `TopicCompactionService` missing a method that find positions according to publish time or other metadata, so we should add it. + +In addition, this method can also be used to find the position/offset according to offset/timestamp in the KoP. + +# Goals + +# High Level Design + +We need to add a method to `Topic Compaction Service` that can find the matching position and other metadata information according to publishTime/index, +since the `TopicCompactionService` interface already has `@InterfaceStability.Evolving` annotation, so that we are able to add new methods directly. + +# Detailed Design + +Add `findEntryByPublishTime` in the `TopicCompactionService` API. + +Add `findEntryByEntryIndex` in the `TopicCompactionService` API. + +Implement them in the `PulsarTopicCompactionService` using binary search. + +When get messageId by timestamp, find position from topicCompactionService if we can't find position in the manageLedger. + +## Public-facing Changes + + ```java + @InterfaceAudience.Public + @InterfaceStability.Evolving + public interface TopicCompactionService extends AutoCloseable { + + /** + * Find the first entry that greater or equal to target publishTime. + * + * @param publishTime the publish time of entry. + * @return the first entry that greater or equal to target publishTime, this entry can be null. + */ + CompletableFuture findEntryByPublishTime(long publishTime); + + /** + * Find the first entry that greater or equal to target entryIndex. + * + * @param entryIndex the index of entry. + * @return the first entry that greater or equal to target entryIndex, this entry can be null. + */ + CompletableFuture findEntryByEntryIndex(long entryIndex); + } + ``` + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/85o3sx6rhohvc370j4r7yd2nb1tx736c +* Mailing List voting thread: https://lists.apache.org/thread/q27zg49mpr8otwh29s3sncdcx8ly7ws6 \ No newline at end of file diff --git a/pip/pip-289.md b/pip/pip-289.md new file mode 100644 index 0000000000000..ceee04c2ddfa8 --- /dev/null +++ b/pip/pip-289.md @@ -0,0 +1,242 @@ +# PIP 289: Secure Pulsar Connector Configuration +# Background knowledge + +Pulsar Sinks and Sources (a.k.a. Connectors) allow you to move data from a remote system into and out of a Pulsar cluster. These remote systems often require authentication, which requires secret management. + +The current state of Pulsar Connector secret management is fragmented, is not documented in the "Pulsar IO" docs, and is not possible in certain cases. This PIP aims to address these issues through several changes. + +The easiest way to show the current short comings is by way of example. + +## Elasticsearch Example +Here is the current way to deploy an Elasticsearch Sink without the use of plaintext secrets: + +```shell +$ bin/pulsar-admin sinks create \ + --tenant public \ + --namespace default \ + --sink-type elastic_search \ + --name elasticsearch-test-sink \ + --sink-config '{"elasticSearchUrl":"http://localhost:9200","indexName": "my_index"}' \ + --secrets '{"username": {"MY-K8S-SECRET-USERNAME": "secret-name"},"password": {"MY-K8S-SECRET-PASSWORD": "password123"}}' + --inputs elasticsearch_test +``` + +When run targetting Kubernetes, the above works by mounting secrets `MY-K8S-SECRET-USERNAME` and `MY-K8S-SECRET-PASSWORD` into the sink pod container as [environment variables](https://github.com/apache/pulsar/blob/82237d3684fe506bcb6426b3b23f413422e6e4fb/pulsar-functions/secrets/src/main/java/org/apache/pulsar/functions/secretsproviderconfigurator/KubernetesSecretsProviderConfigurator.java#L85-L99): + +```shell +username=secret-name +password=password123 +``` + +Those environment variables are then [injected](https://github.com/apache/pulsar/blob/674655347da95305cf671f0696f113dcca88b44d/pulsar-io/common/src/main/java/org/apache/pulsar/io/common/IOConfigUtils.java#L67-L78) into the config when it is loaded at runtime based on [annotations](https://github.com/apache/pulsar/blob/b7eab9469177eda2c56e36bb9871aab48a17d4ec/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java#L99-L113) on the `ElasticSearchConfig`. + +### Problem + +The annotation approach, which is the only way to inject secrets into connectors, requires that all secret fields are annotated with `sensitive = true` and that all secret fields are at the top level of their configuration class. However, the Elasticsearch config contains an `ssl` field that has nested secrets. See: + +```json +{ + "elasticSearchUrl": "http://localhost:9200", + "indexName": "my_index", + "username": "username", + "password": "password", + "ssl": { + "enabled": true, + "truststorePath": "/pulsar/security/truststore.jks", + "truststorePassword": "truststorepass", + "keystorePath": "/pulsar/security/keystore.jks", + "keystorePassword": "keystorepass" + } +} +``` + +Because `truststorePassword` and `keystorePassword` are not at the top level, we do not currently have a secure way (i.e. non-plaintext) to configure those settings. + +## RabbitMQ Example + +Another relevant example shows how the Pulsar code base has not consistently implemented secret management for connectors. For the RabbitMQ Sink, the sensitive fields are [annotated correctly](https://github.com/apache/pulsar/blob/82237d3684fe506bcb6426b3b23f413422e6e4fb/pulsar-io/rabbitmq/src/main/java/org/apache/pulsar/io/rabbitmq/RabbitMQAbstractConfig.java#L61-L73), but the configuration is not loaded via the `IOConfigUtils#loadWithSecrets` method, which means the only way to load rabbit secrets is as plaintext values in the config. + +## Kafka Connect Adapter Example + +The final relevant example is the Kafka Connect Adapter. This adapter allows you to run Kafka Connectors in Pulsar Connectors. Because of the recursive nature of these connectors, the configuration for the wrapped connector is stored in a map named [kafkaConnectorConfigProperties](https://github.com/apache/pulsar/blob/55523ac8f31fd6d54aacba326edef1f53028877e/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java#L59-L62). Because this field is an arbitrary map, we cannot rely on the Pulsar `sensitive` annotation flag to determine whether to load the secret when building the config class. + +# Motivation + +Increase Pulsar Function security by giving users a way to configure Pulsar Connectors with non-plaintext secrets. + +The recent [CVE-2023-37579](https://github.com/apache/pulsar/wiki/CVE%E2%80%902023%E2%80%9037579) resulted in the potential to leak connector configurations. Because we do not always provide a way to configure connector configuration in the connector's secrets map, leaking the configuration meant leaking secrets. + +# Goals + +## In Scope + +* Provide users with a secure way to configure official Pulsar Connectors as well as third party connectors. +* Improve documentation to reflect the current state of secrets management in Pulsar Connectors. +* Only sinks and sources will benefit from this change. +* Only the `JavaInstanceRunnable` class will benefit from this change. + +## Out of Scope + +* This PIP will not prevent users from configuring secrets via insecure methods, such as plaintext configuration. +* Functions are out of scope because they do not need arbitrary secret injection. Functions can already access secrets through the `Context#getSecret` method. +* Python and Go Function Runtimes--sinks and sources are not typically written in these languages. + +# High Level Design + +* Add a new secrets injection mechanism which allows for arbitrary secret injection into the connector configuration at runtime. +* Update existing, official connectors to properly use the already available secret injection mechanism. +* Fix the documentation for the existing secrets management methods. + +# Detailed Design + +## Design & Implementation Details + +In order to add a new way to inject, or interpolate, secrets, we need to add a new method to the `SecretsProvider` interface, which can be implemented by users, but is not exposed to function/connector runtimes. This new method will be used to first determine if a secret should be interpolated for a given value, and if so, return the interpolated value. If the value is not a secret, or the secret does not exist, the method will return `null` and no interpolation will occur. The notable difference for this method is that it does not have a "path" to the secret. Therefore, the existing `secrets` map might not apply for certain use cases. In the environment variable scenario, this is a natural fit because the `value` can be interpreted as the name of the environment variable. For usage of the new configuration mechanism, see the [cli](#cli) section. + +In the event of a value collision between the old way and this new way to inject secrets, the old way will take precedence. + +In order to add support for the existing `sensitive` annotation, I propose fixing all the connectors that have explicit secrets in their configurations. + +Fixing the documentation will be a matter of updating the existing documentation to reflect the current state of the code. + +## Public-facing Changes + +### Public API + +#### Add new method to SecretsProvider Interface + +Add the following method to the `SecretsProvider` interface: + +```java +interface SecretsProvider { + /** + * If the passed value is formatted as a reference to a secret, as defined by the implementation, return the + * referenced secret. If the value is not formatted as a secret reference or the referenced secret does not exist, + * return null. + * + * @param value a config value that may be formatted as a reference to a secret + * @return the materialized secret. Otherwise, null. + */ + default String interpolateSecretForValue(String value) { + return null; + } +} +``` + +There are only two official implementations of the `SecretProvider` interface. The `ClearTextSecretsProvider` and the `EnvironmentBasedSecretsProvider`. Given that the `ClearTextSecretsProvider` is only plaintext, it will not override the new method. Here is the proposed implementation for the `EnvironmentBasedSecretsProvider`: + +```java +public class EnvironmentBasedSecretsProvider implements SecretsProvider { + /** + * Pattern to match ${secretName} in the value. + */ + private static final Pattern interpolationPattern = Pattern.compile("\\$\\{(.+?)}"); + + @Override + public String interpolateSecretForValue(String value) { + Matcher m = interpolationPattern.matcher(value); + if (m.matches()) { + String secretName = m.group(1); + // If the secret doesn't exist, we return null and don't override the current value. + return provideSecret(secretName, null); + } + return null; + } +} +``` + +### Binary protocol + +No change. + +### Configuration + +There is no new configuration for this change. It is always enabled. + +### CLI + +* Here is the new way that users will map secrets into nested configs: + + ```bash + $ bin/pulsar-admin sinks create \ + --tenant public \ + --namespace default \ + --sink-type elastic_search \ + --name elasticsearch-test-sink \ + --sink-config '{ + "elasticSearchUrl": "http://localhost:9200", + "indexName": "my_index", + "username": "${username}", + "password": "${password}", + "ssl": { + "enabled": true, + "truststorePath": "/pulsar/security/truststore.jks", + "truststorePassword": "${truststorepass}", + "keystorePath": "/pulsar/security/keystore.jks", + "keystorePassword": "${keystorePassword}" + }' \ + --secrets '{"username": {"MY-K8S-SECRET-USERNAME": "secret-name"},"password": {"MY-K8S-SECRET-PASSWORD": "password123"},"keystorePassword": {"MY-K8S-KEYSTORE-PASS": "xyz"},"truststorepass": {"MY-K8S-TRUSTSTORE-PASS": "abc"}}' + --inputs elasticsearch_test + ``` + +### Metrics + +No new metrics are added by this change. + +# Monitoring + +Not applicable. + +# Security Considerations + +The primary security consideration is whether there is any risk in giving users a way to interpolate environment variables into their connector. This change only affects the `EnvironmentBasedSecretsProvider`, which is only used by the Kubernetes Function runtime. As such, there are no environment variables to leak. Further, all connectors have access to their environment variables, so no additional risk is present. + +# Backward & Forward Compatibility + +## Revert + +Reverting this change is as simple as downgrading the function worker and stopping then starting the function. + +## Upgrade + +Upgrade by upgrading the function worker and stopping then starting the function. Also, the user will need to update their connector configuration to use the new syntax. + +# Alternatives + +While exploring this PIP, I considered several alternatives. + +### Merge Secret Map into Config Map + +Attempt to merge all secrets configured for the connector into the connector's configuration. See https://github.com/apache/pulsar/pull/20863 for an example of this approach. + +The primary issue with this design is the fact that the secrets map configured for a connector is of type `Map` where the keys are meant to be top level fields in the connector configuration and the values are paths to the secrets. As such, we cannot use the secrets map to recursively inject secrets into the config, which is a requirement for some connectors. + +### Directly Inject Secrets into Config Map Based on Value Prefix + +We could consider interpreting configuration values that start with a well known prefix, like `env:`, as values that need to be read from the environment. The primary drawback to this solution is that there is not an easy way to configure the function at this point in the code, which means that it is always on. + +This solution would look something like adding this code block + +```java + // Replace environment variable pointers with their environment variable values + for (Map.Entry entry : config.entrySet()) { + if (entry.getValue() instanceof String && ((String) entry.getValue()).toLowerCase().startsWith("env:")) { + String envVariableName = ((String) entry.getValue()).substring("env:".length()); + String envVariableValue = System.getenv(envVariableName); + entry.setValue(envVariableValue); + } + } +``` + +to this method: https://github.com/apache/pulsar/blob/f7c0b3c49c9ad8c28d0b00aa30d727850eb8bc04/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java#L884-L929. + +# General Notes + +# Links + +* Initial Issue exploring this feature: https://github.com/apache/pulsar/issues/20862 +* PR for new interpolation feature: https://github.com/apache/pulsar/pull/20901 +* PR for correcting `sensitive` annotation flag handling: https://github.com/apache/pulsar/pull/20902 +* Rejected PR for merging secrets map into config map: https://github.com/apache/pulsar/pull/20863 +* Mailing List discussion thread: https://lists.apache.org/thread/xdmhp6zpwto2dyrf1xwk7fhd2cr69xtn +* Mailing List voting thread: https://lists.apache.org/thread/ww88z811bpnzpcdf8popvg4njn6d07jt diff --git a/pip/pip-290.md b/pip/pip-290.md new file mode 100644 index 0000000000000..4e480db88b439 --- /dev/null +++ b/pip/pip-290.md @@ -0,0 +1,255 @@ +# Background knowledge + +### 1. Web Socket Proxy Server +[Web Socket Proxy Server](https://pulsar.apache.org/docs/3.0.x/client-libraries-websocket/#run-the-websocket-service) provides a simple way to interact with Pulsar under `WSS` protocol. +- When a [wss-producer](https://pulsar.apache.org/docs/3.0.x/client-libraries-websocket/#nodejs-producer) was registered, Web Socket Proxy Server will create a one-to-one producer to actually send messages to the Broker. +- When a [wss-consumer](https://pulsar.apache.org/docs/3.0.x/client-libraries-websocket/#nodejs-consumer) was registered, Web Socket Proxy Server will create a one-to-one consumer to actually receive messages from the Broker and send them to WSS Consumer. + +### 2. When a user wants to encrypt the message payload, there are two solutions: +- **Solution 1**: encrypt message payload before WSS Producer sends messages, and decrypt after WSS Consumer receives messages. If the user wants to use different encryption keys for different messages, they can set a [property](https://github.com/apache/pulsar/blob/master/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/ProducerMessage.java#L38) into messages to indicate the message was encrypted by which key. But this solution has a shortcoming: if the user also has consumers with Java clients, then these consumers cannot auto-decrypt the messages(Normally, java clients can [decrypt messages automatically](https://pulsar.apache.org/docs/3.0.x/security-encryption/#how-it-works-in-pulsar)). And the benefit of this solution is that the user does not need to expose the private key to Web Socket Proxy Server. +- **Solution 2**: In the release `2.11`, there is a [feature](https://github.com/apache/pulsar/pull/16234) that provides a way to set encrypt keys for the internal producers and consumers of Web Socket Proxy Server, but needs the user to upload both public key and private key into the Web Socket Proxy Server(in other words: user should expose the keys to Web Socket Proxy Server), there is a un-recommended workaround for this shortcoming[1]. The benefit is that the WSS producer and WSS consumer should not care about encryption and decryption. + +### 3. The message payload process during message sending +- The Producer will composite several message payloads into a batched message payload if the producer is enabled batch; +- The Producer will compress the batched message payload to a compressed payload if enabled compression; +- After the previous two steps, the Producer encrypts the compressed payload to an encrypted payload. + + +### 4. Encrypt context + +The Construction of the Encrypt Context: +```json +{ + "batchSize": 2, // How many single messages are in the batch. If null, it means it is not a batched message. + "compressionType": "NONE", // the compression type. + "uncompressedMessageSize": 0, // the size of the uncompressed payload. + "keys": { + "client-rsa.pem": { // key name. + "keyValue": "asdvfdw==", // key value. + "metadata": {} // extra props of the key. + } + }, + "param": "Tfu1PxVm6S9D3+Hk" // the IV of current encryption for this message. +} +``` +All the fields of Encrypt Context are used to parse the encrypted message payload. +- `keys` and `param` are used to decrypt the encrypted message payload. +- `compressionType` and `uncompressedMessageSize` are used to uncompress the compressed message payload. +- `batchSize` is used to extract the batched message payload. + +There is another attribute named `encryptionAlgo` used to identify what encrypt algo is using, it is an optional attribute, so there is no such property in Encrypt Context. + +When the internal consumer of the Web Socket Proxy Server receives a message, if the message metadata indicates that the message is encrypted, the consumer will add Encrypt Context into the response for the WSS consumer. + +### 5. Quick explanation of the used components in the section Design: +- `CryptoKeyReader`: an interface that requires users to implement to read public key and private key. +- `MessageCrypto`: a tool interface to encrypt and decrypt the message payload and add and extract encryption information for message metadata. + +# Motivation + +Therefore, there is no way to enable encryption under the WSS protocol and meet the following conditions: +- WSS producer and WSS consumer did encrypt and decrypt themselves and did not share private keys to Web Socket Proxy Server. +- Other clients(such as Java and CPP) can automatically decrypt the messages which WSS producer sent. + +# Goals +Provide a way to make Web Socket Proxy Server just passes encrypt information to the client, the WSS producer and WSS consumer did encrypt and decrypt themselves. + +Since the order of producer operation for message payloads is `compression --> encryption,` users need to handle Compression themselves if needed. + +If other clients(such as Java, CPP) are sending messages to the topic that the WSS consumer was subscribed to, it is possible that there are some batched messages in the topic, then the WSS consumer will inevitably receive the batched messages. Since the order of consumer operation for message payload is `deencryption --> un-compression --> extract the batched messages`, users need to handle Un-compression and Extract Batch Messages themselves. + +## Out of Scope +This proposal does not intend to support the three features: +- Support publishing "Null value messages" for WSS producers. +- Support publishing "Chunked messages" for WSS producers. +- Support publishing "Batched messages" for WSS producers. + + +# High-Level Design +**For WSS producers**: +Modify the definition of parameter `encryptionKeys` to make it can set in two ways: +- The original mode: If the producer registered with a string parameter `encryptionKeys`, then Web Socket Proxy Server will still work in the original way, which is defined in the PIP [Support encryption in Web Socket Proxy Server](https://github.com/apache/pulsar/pull/16234) +- The new mode: If a producer registered with a JSON parameter `encryptionKeys`, and the `encryptionKeys[{key_name}].keyValue` is not empty, Web Socket Proxy Server will mark this Producer as Client-Side Encryption Producer, then discard server-side batch messages, server-side compression, and server-side encryption. The constructor of `encryptionKeys` is like below: +```json +{ + "client-ecdsa.pem": { + "keyValue": "BDJfN+Iw==", + "metadata": { + "k1": "v1" + } + } +} +``` + +**For WSS consumers**: Users can set the parameter `cryptoFailureAction` to `CONSUME` to directly receive the undecrypted message payload (it was supported before). + +# Detailed Design +**For the producers marked as Client-Side Encryption Producers**: + +- forcefully set the component `CryptoKeyReader` to `DummyCryptoKeyReaderImpl`. + - `DummyCryptoKeyReaderImpl`: doesn't provide any public key or private key, and just returns `null`. +- forcefully set the component `MessageCrypto` to `WSSDummyMessageCryptoImpl` to skip the message Server-Side encryption. + - `WSSDummyMessageCryptoImpl`: only set the encryption info into the message metadata and discard payload encryption. +- forcefully set `enableBatching` to `false` to skip Server-Side batch messages building, and print a log if the discarded parameters `enableBatching`, `batchingMaxMessages`, `maxPendingMessages`, `batchingMaxPublishDelay` were set. +- forcefully set the `CompressionType` to `None` to skip the Server-Side compression, and print a log if the discarded parameter `compressionType` was set. +- forcefully set the param `enableChunking` to `false`(the default value is `false`) to prevent unexpected problems if the default setting is changed in the future. + +**For the client-side encryption consumers**: + +- To avoid too many warning logs: after setting the config `cryptoFailureAction` of the consumer is `CONSUME`, just print an `DEBUG` level log when receiving an encrypted message if the consumer could not decrypt it(the original log level is `WARN`). + + +### Public API + +#### [Endpoint: producer connect](https://pulsar.apache.org/docs/3.1.x/client-libraries-websocket/#producer-endpoint) +Define a new mode for the parameter `encryptionKeys`: +| param name | description| constructor (before encode) | +| --- | --- | --- | +| `encryptionKeys` | Base64 encoded and URL encoded and JSON formatted encryption keys | `Map` | + +#### [Endpoint: publish messages](https://pulsar.apache.org/docs/3.1.x/client-libraries-websocket/#publish-a-message) +Add JSON attributes below: +| param name | description | constructor (before encode) | +| --- | --- | --- | +| `compressionType` | Compression type. Do not set it if compression is not performed | `CompressionType` | +| `uncompressedMessageSize` | The size of the payload before compression. Do not set it if compression is not performed | `int` | +| `encryptionParam` | Base64 encoded serialized initialization vector used when the client encrypts | `byte[]` | + +### A demo for client-side encryption producer +```java +public void connect() { + String protocolAndHostPort = "ws://localhost:55217"; + String topicName = "perssitent://public/default/tp1"; + String keys = ``` + { + "client-ecdsa.pem": { + "keyValue": "BDJf/72DhLRs0C0/U+vkykeIBfXaaJiwpqPVgWJvV7B7GwqIMvY6OFXdFvi0gx7Co/0xO7vKTHLQP8GZAt8DWrsCb8W1jhxmOjpThHBaksXG0kN+Iw==", + "metadata": { + "k1": "v1" + } + } + } + ``` + StringBuilder producerUrL = new StringBuilder(protocolAndHostPort) + .append("/ws/v2/producer/persistent/") + .append(topicName) + .append("?") + .append("encryptionKeys=").append(base64AndURLEncode(keys)); + WebSocketClient wssClient = new WebSocketClient(); + wssClient.start(); + Session session = wssClient.connect(this, producerUrL, new ClientUpgradeRequest()).get(); +} + +public void sendMessage() { + byte[] payload = "msg-123".getBytes(UTF-8); // [109, 115, 103, 45, 49, 50, 51] + String msgKey = "client-ecdsa.pem"; + // Compression if needed(optional). + CompressionType compressionType = CompressionType.LZ4; + msg.uncompressedMessageSize = 5; + byte[] compressedPayload = compress(payload); // [109, 115, 103, 45, 49, 50, 51] + // Encrypt if needed. + bytes[] encryptionParam = getEncryptionParam(); // [-10, -5, -124, 23, 14, -122, 30, 127, 64, 63, 85, -79] + String base64EncodedEncryptionParam = base64Encode(encryptionParam); // 9vuEFw6GHn9AP1Wx + bytes[] encryptedPayload = encrypt(compressedPayload, encryptionParam); // H2RbToHyfXrAUJq3kCC81wlmpGRU5l4= + // Do send. + ProducerMessage msg = new ProducerMessage(); + msg.key = msgKey; + msg.payload = encryptedPayload; + msg.encryptionParam = base64EncodedEncryptionParam; + msg.compressionType = compressionType; + msg.uncompressedMessageSize = uncompressedMessageSize; + this.session.getRemote().sendString(toJSON(msg)); +} +``` + +### A demo for client-side encryption consumer + +```java +public void connect() { + String protocolAndHostPort = "ws://localhost:55217"; + String topicName = "perssitent://public/default/tp1"; + StringBuilder consumerUri = new StringBuilder(protocolAndHostPort) + .append("/ws/v2/consumer/persistent/") + .append(topicName) + .append("/") + .append(subscriptionName) + .append("?") + .append("subscriptionType=").append(subscriptionType.toString()) + // Set "cryptoFailureAction" to "CONSUME". + .append("&").append("cryptoFailureAction=CONSUME"); + WebSocketClient wssClient = new WebSocketClient(); + wssClient.start(); + Session session = wssClient.connect(this, buildConnectURL(), new ClientUpgradeRequest()).get(); +} + +public byte[] messageReceived(String text) { + /** + * A demo of the parameter "text": + * { + * "messageId": "CAcQADAA", + * "payload": "ApU16CsV0iHO2zbX7T22jhGMzdjE5drm", + * "properties": {}, + * "publishTime": "2023-08-22T02:40:32.856+08:00", + * "redeliveryCount": 0, + * "encryptionContext": { + * "keys": { + * "client-ecdsa.pem": { + * "keyValue": "BMQKA==", + * "metadata": { + * "k1": "v1" + * }, + * "param": "SnqNyjPetp1dGBa6", + * "compressionType": "LZ4", + * "uncompressedMessageSize": 7, + * "batchSize": null + * } + * } + */ + ConsumerMessage msg = parseJsonToObject(text); + /** + * The constructor of encryptionContext: + * { + * "client-ecdsa.pem": { + * "keyValue": "BMQKA==", + * "metadata": { + * "k1": "v1" + * } + * } + * } + */ + EncryptionContext encryptionContext = msg.encryptionContext; + // base64Decode and decrypt message payload. + byte[] decryptedPayload = decrypt(base64Decode(msg.payload), encryptionContext); + //Un-compress is needed. + byte[] unCompressedPayload = unCompressIfNeeded(decryptedPayload); + return unCompressedPayload; +} +``` + +### Test cases +- Pub & Sub with WSS producer and consumer. + - compression & decryption. +- Pub with Java client library and Sub with WSS consumers. + - non-compression & decryption. + - compression & decryption. + - compression & decryption & batch send. +- Pub with WSS protocol and Sub with Java client library(verify it can auto decompression, decryption). + - non-compression & decryption. + - compression & decryption. + +# Footnotes +**[1]**: A workaround to avoid exposing the private key to Web Socket Proxy Server(should expose the public key to Web Socket Proxy Server). +A quick background: there are three policies when a consumer cannot describe the message payload: +- CONSUME: it responds to the user's original message payload and prints a warning log. +- DISCARD: discard this message. +- FAIL: add this message into `unackMessagesTracker.` How this message is ultimately handled depends on the policy of unacknowledged messages. + +**Workaround** +- Set `cryptoFailureAction` to `CONSUME` for the WSS consumer +- Make the return value `EncryptionKeyInfo` to `null` for the `CryptoKeyReader`. This will make the internal consumer of Web Socket Proxy Server decrypt message payload fail. + +Then the flow of Pub & Sub will be executed like the following: +- Users do not encrypt message payload before the WSS producer sends messages. +- The internal producer of WebSocket does message payload encryption by the [feature: Support encryption in Web Socket Proxy Server](https://github.com/apache/pulsar/pull/16234) +- The decryption of the internal consumer of Web Socket Proxy Server message payload will be failed, and just send original message payload to the users. +- Users decrypt the message payload themself. diff --git a/pip/pip-293.md b/pip/pip-293.md new file mode 100644 index 0000000000000..a03ad189ba76b --- /dev/null +++ b/pip/pip-293.md @@ -0,0 +1,20 @@ +# Motivation + +There is a config in ServiceConfiguration called `disableBrokerInterceptors` introduced by [#8157](https://github.com/apache/pulsar/pull/8157), which seems to disable the broker interceptor, but is commented for testing only. +Actually, whether to enable the interceptor depends on whether the broker is loaded into the interceptors. [#10489](https://github.com/apache/pulsar/pull/10489) kept the same implementation. +But [#20422](https://github.com/apache/pulsar/pull/20422) has changed the behavior, it uses `disableBrokerInterceptors` to judge whether to enable the interceptor, which caused an NPE issue mentioned in [#20710](https://github.com/apache/pulsar/pull/20710). +This `disableBrokerInterceptors` config is very confusing, so we decide to delete it. + +# Goals + +Delete config `disableBrokerInterceptors`. + + +# Backward & Forward Compatibility + +No backward & forward compatibility issue + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/vqf5qcwv3rh2y1r62gw0dnpn0xznq9p0 +* Mailing List voting thread: https://lists.apache.org/thread/o11otjlywgd0s8dsv7dg9s8msswdfspp diff --git a/pip/pip-296.md b/pip/pip-296.md new file mode 100644 index 0000000000000..99a57e66624b5 --- /dev/null +++ b/pip/pip-296.md @@ -0,0 +1,93 @@ +# Background knowledge +A consumer is a process that attaches to a topic via a subscription and then receives messages. The reader interface for Pulsar enables applications to manually manage cursors. More knowledge of the Reader and Consumer interface can be found in the [Pulsar Client doc](https://pulsar.apache.org/docs/next/concepts-clients/#reader). + +# Motivation + +Add the `getLastMessageIds` API for Reader. This will help to increase the flexibility of reader usage. + +# Goals + +## In Scope + +Add the `getLastMessageIds` API for Reader. + +## Out of Scope + +None. + + +# High Level Design + +Implement the `getLastMessageIds` method for Reader by internally invoking the Consumer interface. + +# Detailed Design + +## Design & Implementation Details +```java + @Override + public List getLastMessageIds() throws PulsarClientException { + return consumer.getLastMessageIds(); + } + + @Override + public CompletableFuture> getLastMessageIdsAsync() { + return consumer.getLastMessageIdsAsync(); + } +``` + +## Public-facing Changes + + + +### Public API +```java + /** + * Get all the last message id of the topics the reader subscribed. + * + * @return the list of TopicMessageId instances of all the topics that the reader subscribed + * @throws PulsarClientException if failed to get last message id. + * @apiNote It's guaranteed that the owner topic of each TopicMessageId in the returned list is different from owner + * topics of other TopicMessageId instances + */ + List getLastMessageIds() throws PulsarClientException; + + /** + * The asynchronous version of {@link Reader#getLastMessageIds()}. + */ + CompletableFuture> getLastMessageIdsAsync(); +``` +### Binary protocol + +### Configuration + +### CLI + +### Metrics + +# Monitoring + + +# Security Considerations + + +# Backward & Forward Compatibility + +## Revert + + +## Upgrade + + + +# Alternatives + + +# General Notes + +# Links + + +* Mailing List discussion thread: +* Mailing List voting thread: diff --git a/pip/pip-297.md b/pip/pip-297.md new file mode 100644 index 0000000000000..2985864beed43 --- /dev/null +++ b/pip/pip-297.md @@ -0,0 +1,213 @@ +# Title: Support terminating Function & Connector with the fatal exception + +# Background knowledge + +The **Pulsar Function** is a serverless computing framework that runs on top of Pulsar and processes messages. + +The **Pulsar IO Connector** is a framework that allows users to easily integrate Pulsar with external systems, such as +databases, messaging systems, and data pipelines. With Pulsar IO Connector, you can create, deploy, and manage +connectors that read data from or write data to Pulsar topics. There are two types of Pulsar IO Connectors: source and +sink. A **source connector** imports data from another system to Pulsar, while a **sink connector** exports data from +Pulsar to another system. The Pulsar IO Connector is implemented based on the Pulsar Function framework. So in +the following, we treat the connector as a special kind of function. The `function` refers to both function and +connector. + +**Function Instance** is a running instance of a Pulsar IO Connector that interacts with a specific external system or a +Pulsar Function that processes messages from the topic. + +**Function Framework** is a framework for running the Function instance. + +**Function Context** is an interface that provides access to various information and resources for the connector or the +function. The function context is passed to the connector or the function when it is initialized, and then can be used +to interact with the Pulsar system. + +## The current implementation of the exception handler + +**Function instance thread**: The function framework initializes a thread for each function instance to handle the +core logic of the function/connector, including consuming messages from the Pulsar topic for the sink connector, +executing the logic of the function, producing messages to the Pulsar topic for the source connector, handling the +exception, etc. And let's define the **Connector thread/Function thread** as a thread that is created by the connector +or function itself. + +**Exception handling logic**: The function itself can throw exceptions, and this thread will catch the exception and +then close the function. This means that the function will stop working until it is restarted manually or +automatically by the function framework. + +Even though it is not explicitly defined, there are two types of exceptions that should be handled by the function or +the framework: + +- **Fatal exception**: This is an exception that the function cannot recover from by itself and needs to notify the + framework to terminate it. These are fatal exceptions that indicate a configuration issue, a logic error, or an + incompatible system. The function framework will catch these exceptions, report them to users, and terminate the + function. +- **Non-fatal exception** is an exception that the function instance don't need to be terminated for. It could be + handled by the connector or function itself. Or be thrown by the function. This exception won't cause the function + instance to be terminated. + +### How to handle exceptions thrown from connectors + +All the exceptions thrown form the connector are treated as fatal exceptions. + +If the exception is thrown from the function instance thread, the function framework will catch the exception and +terminate the function instance. + +If the exception is thrown from the connector thread that is created by the connector itself, the function framework +will not be able to catch the exception and terminate the function instance. The connector will hang forever. +The `Motivation` part will talk more about this case. + +If the exception is thrown from the external system, the connector implementation could treat it as a retryable +exception and retry to process the message later, or throw it to indicate it as a fatal exception. + +### How to handle exceptions thrown from functions + +All the exceptions thrown from the pulsar function are treated as non-fatal exceptions. The function framework will +catch the exception and log it. But it will not terminate the function instance. + +There is no way for the function developer to throw a fatal exception to the function framework to terminate the +function instance. + +# Motivation + +Currently, the connector and function cannot terminate the function instance if there are fatal exceptions thrown +outside the function instance thread. The current implementation of the connector and Pulsar Function exception handler +cannot handle the fatal exceptions that are thrown outside the function instance thread. + +For example, suppose we have a sink connector that uses its own threads to batch-sink the data to an external system. If +any fatal exceptions occur in those threads, the function instance thread will not be aware of them and will +not be able to terminate the connector. This will cause the connector to hang indefinitely. There is a related issue +here: https://github.com/apache/pulsar/issues/9464 + +The same problem exists for the source connector. The source connector may also use a separate thread to fetch data from +an external system. If any fatal exceptions happen in that thread, the connector will also hang forever. This issue has +been observed for the Kafka source connector: https://github.com/apache/pulsar/issues/9464. We have fixed it by adding +the notifyError method to the `PushSource` class in PIP-281: https://github.com/apache/pulsar/pull/20807. However, this +does not solve the same problem that all source connectors face because not all connectors are implemented based on +the `PushSource` class. + +The problem is same for the Pulsar Function. Currently, the function can't throw fatal exceptions to the function +framework. We need to provide a way for the function developer to implement it. + +We need a way for the connector and function developers to throw fatal exceptions outside the function instance +thread. The function framework should catch these exceptions and terminate the function accordingly. + +# Goals + +## In Scope + +- Support terminating the function instance with fatal exceptions +- This proposal will apply both to the Pulsar Function and the Pulsar Connector. + +## Out of Scope + +- The fixes of the exception-raising issue mentioned in the Motivation part for all the connectors are not included in + this PIP. This PIP only provides the feature for the connector developer to terminate the function instance. The fixes + should be in several different PRs. + +# High Level Design + +Introduce a new method `fatal` to the context. All the connector implementation code and the function code +can use this context and call the `fatal` method to terminate the instance while raising a fatal exception. + +After the connector or function raises the fatal exception, the function instance thread will be interrupted. +The function framework then could catch the exception, log it, and then terminate the function instance. + +# Detailed Design + +## Design & Implementation Details + +This PIP proposes to add a new method`fatal`to the context `BaseContext`. This method allows the connector or the +function code to report a fatal exception to the function framework and terminate the instance. The `SinkContext` +and `SourceContext` are all inherited from `BaseContext`. Therefore, all the sink connectors and source connectors can +invoke this new method. The pulsar function context class `Context` is also inherited from `BaseContext`. Therefore, the +function code can also invoke this new method. + +In the `fatal` method, the function instance thread will be interrupted. The function instance thread can then +catch the interrupt exception and get the fatal exception. The function framework then logs this exception, +reports to the metrics, and finally terminates the function instance. + +Tbe behavior when invoking the `fatal` method: + +- For the connector thread or function thread: + - Invoke the `fatal` method + - Send the exception to the function framework. There is a field `deathException` in the + class `JavaInstanceRunnable` that is used to store the fatal exception. + - Interrupt the function instance thread +- For the function instance thread: + - Catch the interrupt exception + - Get the exception from the function framework + - Report the log and metrics + - Close the function instance + +## Public-facing Changes + +### Public API + +Introduce `fatal` method to the `BaseContext`: + +```java +public interface BaseContext { + /** + * Terminate the function instance with a fatal exception. + * + * @param t the fatal exception to be raised + */ + void fatal(Throwable t); +} +``` + +### Binary protocol + +No changes for this part. + +### Configuration + +No changes for this part. + +### CLI + +No changes for this part. + +### Metrics + +No changes for this part. + +# Monitoring + +No changes for this part. + +# Security Considerations + +No security-related changes. +The new method `fatal` will only take effect on the current function instance. It won't affect other function instances +even they are in the same function worker. + +# Backward & Forward Compatibility + +## Revert + +No operation required. + +## Upgrade + +No operation required. + +# Alternatives + +## Using futures to handle results or exceptions returned the connector + +The benefit of this solution is that it makes the use of exception throwing more intuitive to the connector developer. + +But it requires changes to existing interfaces, including `Source` and `Sink`, which can complicate connector +development. And we still need the `fatal` method to handle some cases such as terminating the instance in code outside +of the message processing logic. This alternative solution can't handle this case. + +Meanwhile, the implementation of this solution will also be more complex, involving changes to the core message +processing logic of the function framework. We need to turn the entire message processing logic into an asynchronous +pattern. + +# General Notes + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/j59gzzwjp8c48lwv5poddm9qzlp2hol0 +* Mailing List voting thread: https://lists.apache.org/thread/ggok3c2601mnbdomr65v3pjth3lk6fr8 diff --git a/pip/pip-298.md b/pip/pip-298.md new file mode 100644 index 0000000000000..a0953ad03a3cd --- /dev/null +++ b/pip/pip-298.md @@ -0,0 +1,197 @@ +# Background + +In the implementation of the Pulsar Transaction, each topic is configured with a `Transaction Buffer` to prevent +consumers from reading uncommitted messages, which are invisible until the transaction is committed. Transaction Buffer +works with Position (maxReadPosition) and `TxnID` Set (aborts). The broker only dispatches messages, before the +maxReadPosition, to the consumers. When the broker dispatches the messages before maxReadPosition to the consumer, the +messages sent by aborted transactions will get filtered by the Transaction Buffer. + +# Motivation + +Currently, Pulsar transactions do not have configurable isolation levels. By introducing isolation level configuration +for consumers, we can enhance the flexibility of Pulsar transactions. + +Let's consider an example: + +**System**: Financial Transaction System + +**Operations**: Large volume of deposit and withdrawal operations, a +small number of transfer operations. + +**Roles**: + +- **Client A1** +- **Client A2** +- **User Account B1** +- **User Account B2** +- **Request Topic C** +- **Real-time Monitoring System D** +- **Business Processing System E** + +**Client Operations**: + +- **Withdrawal**: Client A1 decreases the deposit amount from User + Account B1 or B2. +- **Deposit**: Client A1 increases the deposit amount in User Account B1 or B2. +- **Transfer**: Client A2 decreases the deposit amount from User + Account B1 and increases it in User Account B2. Or vice versa. + +**Real-time Monitoring System D**: Obtains the latest data from +Request Topic C as quickly as possible to monitor transaction data and +changes in bank reserves in real-time. This is necessary for the +timely detection of anomalies and real-time decision-making. + +**Business Processing System E**: Reads data from Request Topic C, +then actually operates User Accounts B1, B2. + +**User Scenario**: Client A1 sends a large number of deposit and +withdrawal requests to Request Topic C. Client A2 writes a small +number of transfer requests to Request Topic C. + +In this case, Business Processing System E needs a read-committed +isolation level to ensure operation consistency and Exactly Once +semantics. The real-time monitoring system does not care if a small +number of transfer requests are incomplete (dirty data). What it +cannot tolerate is a situation where a large number of deposit and +withdrawal requests cannot be presented in real time due to a small +number of transfer requests (the current situation is that uncommitted +transaction messages can block the reading of committed transaction +messages). + +In this case, it is necessary to set different isolation levels for +different consumers/subscriptions. +The uncommitted transactions do not impact actual users' bank accounts. +Business Processing System E only reads committed transactional +messages and operates users' accounts. It needs Exactly-once semantic. +Real-time Monitoring System D reads uncommitted transactional +messages. It does not need Exactly-once semantic. + +They use different subscriptions and choose different isolation +levels. One needs transaction, one does not. +In general, multiple subscriptions of the same topic do not all +require transaction guarantees. +Some want low latency without the exact-once semantic guarantee, and +some must require the exactly-once guarantee. +We just provide a new option for different subscriptions. + +# Goal + +## In Scope + +Implement Read Committed and Read Uncommitted isolation levels for Pulsar transactions. Allow consumers to configure +isolation levels during the building process. + +## Out of Scope + +None. + +# High Level Design + +Add a configuration 'subscriptionIsolationLevel' in the consumer builder to allow users to choose different transaction +isolation levels. + +# Detailed Design + +## Public-facing Changes + +Update the PulsarConsumer builder process to include isolation level configurations for Read Committed and Read +Uncommitted. + +### Before the Change + +The PulsarConsumer builder process currently does not include isolation level configurations. The consumer creation +process might look like this: + +``` +PulsarClient client = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build(); + +Consumer consumer = client.newConsumer(Schema.STRING) + .topic("persistent://my-tenant/my-namespace/my-topic") + .subscriptionName("my-subscription") + .subscriptionType(SubscriptionType.Shared) + .subscribe(); +``` + +### After the Change + +Update the PulsarConsumer builder process to include isolation level configurations for Read Committed and Read +Uncommitted. Introduce a new method subscriptionIsolationLevel() in the consumer builder, which accepts an enumeration +value representing the isolation level: + +``` +public enum SubscriptionIsolationLevel { + // Consumer can only consume all transactional messages which have been committed. + READ_COMMITTED, + + // Consumer can consume all messages, even transactional messages which have been aborted. + READ_UNCOMMITTED; +} +``` + +Then, modify the consumer creation process to include the new isolation level configuration: + +``` +PulsarClient client = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build(); + +Consumer consumer = client.newConsumer(Schema.STRING) + .topic("persistent://my-tenant/my-namespace/my-topic") + .subscriptionName("my-subscription") + .subscriptionType(SubscriptionType.Shared) + .subscriptionIsolationLevel(SubscriptionIsolationLevel.READ_COMMITTED) // Adding the isolation level configuration + .subscribe(); +``` + +With this change, users can now choose between Read Committed and Read Uncommitted isolation levels when creating a new +consumer. If the isolationLevel() method is not called during the builder process, the default isolation level will be +Read Committed. +Note that this is a subscription dimension configuration, and all consumers under the same subscription need to be +configured with the same IsolationLevel. + +## Design & Implementation Details + +### Client Changes + +Update the PulsarConsumer builder to accept isolation level configurations for Read Committed and Read Uncommitted levels. + +In order to achieve the above goals, the following modifications need to be made: + +- Added `IsolationLevel` related fields and methods in `ConsumerConfigurationData` and `ConsumerBuilderImpl` and `ConsumerImpl` + +- Modify PulsarApi.CommandSubscribe, add field -- IsolationLevel + +``` +message CommandSubscribe { + + enum IsolationLevel { + READ_COMMITTED = 0; + READ_UNCOMMITTED = 1; + } + optional IsolationLevel isolation_level = 20 [default = READ_COMMITTED]; +} +``` + +### Broker changes + +Modify the transaction buffer and dispatching mechanisms to handle messages based on the chosen isolation level. + +In order to achieve the above goals, the following modifications need to be made: + +- Determine in the `readMoreEntries` method of Dispatchers such as `PersistentDispatcherSingleActiveConsumer` + and `PersistentDispatcherMultipleConsumers`: + + - If Subscription.isolationLevel == ReadCommitted, then MaxReadPosition = topic.getMaxReadPosition(), that is, + transactionBuffer.getMaxReadPosition() + + - If Subscription.isolationLevel == ReadUnCommitted, then MaxReadPosition = PositionImpl.LATEST + +- Add a new metrics `subscriptionIsolationLevel` in `SubscriptionStatsImpl`. + +# Monitoring + +After this PIP, Users can query the subscription stats of a topic through the admin tool, and observe the `subscriptionIsolationLevel` in the subscription stats to determine the isolation level of the subscription. + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/8ny0qtp7m9qcdbvnfjdvpnkc4c5ssyld +* Mailing List voting thread: https://lists.apache.org/thread/4q1hrv466h8w9ccpf4moxt6jv1jxp1mr +* Document link: https://github.com/apache/pulsar-site/pull/712 diff --git a/pip/pip-299.md b/pip/pip-299.md new file mode 100644 index 0000000000000..342d663cee984 --- /dev/null +++ b/pip/pip-299.md @@ -0,0 +1,81 @@ +# Background knowledge + +The config `managedLedgerMaxUnackedRangesToPersist` + +Indicates the number of `acknowledgment holes` that are going to be persistently stored. When acknowledging out of order, a consumer will leave holes that are supposed to be quickly filled by acking all the messages. The information of which messages are acknowledged is persisted by compressing in `ranges` of messages that were acknowledged. After the maximum number of ranges is reached, the information will only be tracked in memory, and messages will be redelivered in case of crashes. + +The cursor metadata contains the following three data: +- Subscription properties(Usually, this is small); this data part only persists to the ZK node. +- The last sequence ID of each producer. It only exists for the cursor `pulsar.dedup`. If a topic has many, many producers, this part of the data will be large. See [PIP-6:-Guaranteed-Message-Deduplication](https://github.com/apache/pulsar/wiki/PIP-6:-Guaranteed-Message-Deduplication) for more details. +- Individual Deleted Messages(including the acknowledgment of batched messages). This part of the data occupies most of the cursor metadata's space, which is the focus of this proposal. + +Differ with Kafka: Pulsar supports [individual acknowledgment](https://pulsar.apache.org/docs/2.11.x/concepts-messaging/#acknowledgment) (just like ack `{pos-1, pos-3, pos-5}`), so instead of a pointer(acknowledged on the left and un-acknowledged on the right), Pulsar needs to persist the acknowledgment state of each message, we call these records `Individual Deleted Messages.` + +The current persistence mechanism of the cursor metadata(including `Individual Deleted Messages`) works like this: +1. Write the data of cursor metadata(including `Individual Deleted Messages`) to BK in one Entry; by default, the maximum size of the Entry is 5MB. +2. Write the data of cursor metadata(optional to include `Individual Deleted Messages`) to the Metadata Store(such as ZK) if BK-Write fails; data of a Metadata Store Node that is less than 10MB is recommended. Since writing large chunks of data to the Metadata Store frequently makes the Metadata Store work unstable, this is only a backstop measure. + +Is 5MB enough? `Individual Deleted Messages` consists of Position_Rang(each Position_Rang occupies 32 bytes; the implementation will not be explained in this proposal). This means that the Broker can persist `5m / 32bytes` number of Position_Rang for each Subscription, and there is an additional compression mechanism at work, so it is sufficient for almost all scenarios except the following three scenarios: +- Client Miss Acknowledges: Clients receive many messages, and ack some of them, the rest still need to be acknowledged due to errors or other reasons. As time goes on, more and more records will be staying there. +- Delay Messages: Long-delayed and short-delayed messages are mixed, with only the short-delayed message successfully consumed and the long-delayed message not delivered. As time goes on, more and more records will be staying there. +- Large Number of Consumers: If the number of consumers is large and each has some discrete ack records, all add up to a large number. +- Large Number of Producers: If the number of producers is large, there might be a large data of Last Sequence ID to persist. This scenario only exists on the `pulsar.dedup` cursor. + +The config `managedLedgerMaxUnackedRangesToPersist` +If the cursor metadata is too large to persist, the Broker will persist only part of the data according to the following priorities. +- Subscription Properties. This part can't be split up; persist will fail if this part is too large to persist. +- Last sequence ID of producers. This part can't be split up; persist will fail if this part is too large to persist. +- Individual Deleted Message. If it is too large, only one part persists, and then the other part is maintained only in memory + +# Motivation + +Since the frequent persistence of `Individual Deleted Messages` will magnify the amount of BK Written and increase the latency of ack-response, the Broker does not immediately persist it when receiving a consumer's acknowledgment but persists it regularly. + +The data of cursor metadata is recommended to be less than 5MB; if a subscription's `Individual Deleted Messages` data is too large to persist, as the program grows for a long time, there will be more and more non-persistent data. Eventually, there will be an unacceptable amount of repeated consumption of messages when the Broker restarts. + +# Goal + +## In Scope + +To avoid repeated consumption due to the cursor metadata being too large to persist. + +## Out of Scope + +This proposal will not care about this scenario: if so many producers make the metadata of cursor `pulsar.dedup` cannot persist, the task `Take Deduplication Snapshot` will be in vain due to the inability to persist. + +# High-Level Design + +Provide a new config named `dispatcherPauseOnAckStatePersistentEnabled`(default value is `false`) for a new feature: stop dispatch messages to clients when reaching the limitation `managedLedgerMaxUnackedRangesToPersist`. +- If the user does not care about that Individual Deleted Messages can not be fully persistent, resulting in a large number of repeated message consumption, then it can be set to `false`. +- If the user cares about repeated consumption, at can accept a decline in consumption speed when cursor metadata is too large to persist, it can be set to `true`. + + +# Detailed Design +### Public API + +**broker.conf** +``` +/** + * After enabling this feature, Pulsar will stop delivery messages to clients if the cursor metadata is too large to persist, it will help to reduce the duplicates caused by the ack state that can not be fully persistent. Default "false". + */ +boolean dispatcherPauseOnAckStatePersistentEnabled; +``` + +**SubscriptionStats** +```java +/** + * After enabling the feature "dispatcherPauseOnAckStatePersistentEnabled", return "true" if the cursor metadata is too large to persist, else return "false". + * Always return "false" if disabled the feature "dispatcherPauseOnAckStatePersistentEnabled". + */ +boolean isBlockedOnAckStatePersistent(); +``` + +## Design & Implementation Details + +Cache the range count of the Individual Deleted Messages in the memory when doing persist cursor metadata to BK. Stuck delivery messages to clients if reaching the limitation `managedLedgerMaxUnackedRangesToPersist`. + +Since the cache will not be updated in time, the actual count will decrease when clients acknowledge messages(but it does not persist immediately, so the cached value does not update immediately), the cached value is an estimated value. + +# Metrics & Alert + +Nothing. diff --git a/pip/pip-300.md b/pip/pip-300.md new file mode 100644 index 0000000000000..0be8cf191fb06 --- /dev/null +++ b/pip/pip-300.md @@ -0,0 +1,70 @@ +# Motivation + +Dynamic configuration is an important feature in the production environment, the Pulsar supports this feature, +see `ServiceConfiguration.java`, when the field has `@FieldContext(dynamic = true)` means it is a dynamic configuration, +and then we can use the `pulsar-admin` to update this field. + +The Pulsar has multiple pluggable plugins like authentication provider, authorization provider, and so on. In the +plugin, we can get the Pulsar configurations and customized configurations, in some scenarios, we need to use +the `pulsar-admin` to change the values of customized configuration, but the Pulsar doesn't support this operation. + +# Goals + +## In Scope + +The goal of this PIP is to allow the `pulsar-admin` to update the values of customized configuration, and also use the +listener to listen the customized configuration changes +by `org.apache.pulsar.broker.service.BrokerService.registerConfigurationListener`. + +# Detailed Design + +## Design & Implementation Details + +The `org.apache.pulsar.broker.service.BrokerService.dynamicConfigurationMap` holds the dynamic configurations, so we +just to add the configurations that need to be dynamically configured to this map, and then we can use +the `pulsar-admin` to update the values of these configurations. + +Users can only register custom configurations and only once. If multiple registrations are performed, an exception will be thrown. + +Add `void registerCustomDynamicConfiguration(String key, Predicate validator)` to the `BrokerService.java`, + +```java +public void registerCustomDynamicConfiguration(String key, Predicate validator) { + if (dynamicConfigurationMap.containsKey(key)) { + throw new IllegalArgumentException(key + " already exists in the dynamicConfigurationMap"); + } + + ConfigField configField = ConfigField.newCustomConfigField(null); + configField.validator = validator; + dynamicConfigurationMap.put(key, configField); +} +``` + +Example of using this feature in the plugin: + +```java +@Override +public void initialize(PulsarService pulsarService) throws Exception { + String myAuthIdKey = "my-auth-id"; + myAuthIdValue = pulsarService.getConfiguration().getProperties().getProperty(myAuthIdKey); + + pulsarService.getBrokerService().registerCustomDynamicConfiguration(myAuthIdKey, null); + pulsarService.getBrokerService().registerConfigurationListener(myAuthIdKey, (newValue) -> { + // The `myAuthIdKey` value has changed + myAuthIdValue = String.valueOf(newValue); + }); +} +``` + + +# Backward & Forward Compatibility + +## Revert + +# General Notes + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/ysnsnollgy1b6w1dsvmx1t1y2rz1tyd6 +* Mailing List voting thread: https://lists.apache.org/thread/cp4zdlz60md775ptzxbbj8bs2n2osfjb +* PR: https://github.com/apache/pulsar/pull/20884 diff --git a/pip/pip-301.md b/pip/pip-301.md new file mode 100644 index 0000000000000..3450446b20970 --- /dev/null +++ b/pip/pip-301.md @@ -0,0 +1,86 @@ +# Background knowledge + +The following z-nodes store the load and quota data about loadbalance. And the CRUD about them are handled by `localMetadataStore`, not `configurationMetadataStore`. +* `/loadbalance/bundle-data` +* `/loadbalance/broker-time-average` +* `/loadbalance/resource-quota` + +Currently, the access about the above z-nodes are distributed everywhere. It's very easy to call the the wrong `configurationMetadataStore` to handle them, e.g.: +* [[fix] [broker] remove bundle-data in local metadata store.](https://github.com/apache/pulsar/pull/21078) + +# Motivation + +Refactor the access code about balance/load data + +# Goals + +## In Scope + +Introduce `LoadBalanceResources` to unify the CRUD about balance/load data. + +## Out of Scope + +None + +# High Level Design + +Introduce `LoadBalanceResources` which has three inner class: +* `BundleDataResources` +* `BrokerTimeAverageResources` +* `QuotaResources` + +# Detailed Design + +## Design & Implementation Details + +```java +public class LoadBalanceResources { + public static final String BUNDLE_DATA_BASE_PATH = "/loadbalance/bundle-data"; + public static final String BROKER_TIME_AVERAGE_BASE_PATH = "/loadbalance/broker-time-average"; + public static final String RESOURCE_QUOTA_BASE_PATH = "/loadbalance/resource-quota"; + + private final BundleDataResources bundleDataResources; + + public LoadBalanceResources(MetadataStore store, int operationTimeoutSec) { + bundleDataResources = new BundleDataResources(store, operationTimeoutSec); + } + + public static class BundleDataResources extends BaseResources { + public BundleDataResources(MetadataStore store, int operationTimeoutSec) { + super(store, BundleData.class, operationTimeoutSec); + } + // ... + } + + public static class BrokerTimeAverageResources extends BaseResources { + public BrokerTimeAverageResources(MetadataStore store, int operationTimeoutSec) { + super(store, TimeAverageBrokerData.class, operationTimeoutSec); + } + // ... + } + + public static class QuotaResources extends BaseResources { + public QuotaResources(MetadataStore store, int operationTimeoutSec) { + super(store, ResourceQuota.class, operationTimeoutSec); + } + // ... + } +} +``` + +## Public-facing Changes + +None + +### Public API + +None + +# Backward & Forward Compatibility + +None + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/7ngw9dc62tj2c4c5484dgsnlwgtstpbj +* Mailing List voting thread: https://lists.apache.org/thread/26dc8r6hnp7owdsq1hpzb48g8vlfrtxt diff --git a/pip/pip-302.md b/pip/pip-302.md new file mode 100644 index 0000000000000..cf721ee38559b --- /dev/null +++ b/pip/pip-302.md @@ -0,0 +1,140 @@ +# Background Knowledge + +The TableView interface provides a convenient way to access the streaming updatable dataset in a topic by offering a continuously updated key-value map view. The TableView retains the last value of the key which provides you with an almost up-to-date dataset but cannot guarantee you always get the latest data (with the latest written message). + +The TableView can be used to establish a local cache of data. Additionally, clients can register consumers with TableView and specify a listener to scan the map and receive notifications whenever new messages are received. This functionality enables event-driven applications and message monitoring. + +For more detailed information about the TableView, please refer to the [Pulsar documentation](https://pulsar.apache.org/docs/next/concepts-clients/#tableview). + +# Motivation + +When a TableView is created, it retrieves the position of the latest written message and reads all messages from the beginning up to that fetched position. This ensures that the TableView will include any messages written prior to its creation. However, it does not guarantee that the TableView will include any newly added messages during its creation. +Therefore, the value you read from a TableView instance may not be the most recent value, but you will not read an older value once a new value becomes available. It's important to note that this guarantee is not maintained across multiple TableView instances on the same topic. This means that you may receive a newer value from one instance first, and then receive an older value from another instance later. +In addition, we have several other components, such as the transaction buffer snapshot and the topic policies service, that employ a similar mechanism to the TableView. This is because the TableView is not available at that time. However, we cannot replace these implementations with a TableView because they involve multiple TableView instances across brokers within the same system topic, and the data read from these TableViews is not guaranteed to be up-to-date. As a result, subsequent writes may occur based on outdated versions of the data. +For example, in the transaction buffer snapshot, when a broker owns topics within a namespace, it maintains a TableView containing all the transaction buffer snapshots for those topics. It is crucial to ensure that the owner can read the most recently written transaction buffer snapshot when loading a topic (where the topic name serves as the key for the transaction buffer snapshot message). However, the current capabilities provided by TableView do not guarantee this, especially when ownership of the topic is transferred and the TableView of transaction buffer snapshots in the new owner broker is not up-to-date. + +Regarding both the transaction buffer snapshot and topic policies service, updates to a key are only performed by a single writer at a given time until the topic's owner is changed. As a result, it is crucial to ensure that the last written value of this key is read prior to any subsequent writing. By guaranteeing this, all subsequent writes will consistently be based on the most up-to-date value. + +The proposal will introduce a new API to refresh the table view with the latest written data on the topic, ensuring that all subsequent reads are based on the refreshed data. + +```java +tableView.refresh(); +tableView.get(“key”); +``` + +After the refresh, it is ensured that all messages written prior to the refresh will be available to be read. However, it should be noted that the inclusion of newly added messages during or after the refresh is not guaranteed. + +# Goals + +## In Scope + +Providing the capability to refresh the TableView to the last written message of the topic and all the subsequent reads to be conducted using either the refreshed dataset or a dataset that is even more up-to-date than the refreshed one. + +## Out of Scope + + +A static perspective of a TableView at a given moment in time +Read consistency across multiple TableViews on the same topic + +# High-Level Design + +Provide a new API for TableView to support refreshing the dataset of the TableView to the last written message. + +## Design & Implementation Details + +# Public-Facing Changes + +## Public API + +The following changes will be added to the public API of TableView: + +### `refreshAsync()` + +This new API retrieves the position of the latest written message and reads all messages from the beginning up to that fetched position. This ensures that the TableView will include any messages written prior to its refresh. + +```java +/** +* +* Refresh the table view with the latest data in the topic, ensuring that all subsequent reads are based on the refreshed data. +* +* Example usage: +* +* table.refreshAsync().thenApply(__ -> table.get(key)); +* +* This function retrieves the last written message in the topic and refreshes the table view accordingly. +* Once the refresh is complete, all subsequent reads will be performed on the refreshed data or a combination of the refreshed +* data and newly published data. The table view remains synchronized with any newly published data after the refresh. +* +* |x:0|->|y:0|->|z:0|->|x:1|->|z:1|->|x:2|->|y:1|->|y:2| +* +* If a read occurs after the refresh (at the last published message |y:2|), it ensures that outdated data like x=1 is not obtained. +* However, it does not guarantee that the values will always be x=2, y=2, z=1, as the table view may receive updates with newly +* published data. +* +* |x:0|->|y:0|->|z:0|->|x:1|->|z:1|->|x:2|->|y:1|->|y:2| -> |y:3| +* +* Both y=2 or y=3 are possible. Therefore, different readers may receive different values, but all values will be equal to or newer +* than the data refreshed from the last call to the refresh method. +*/ +CompletableFuture refreshAsync(); + +/** +* Refresh the table view with the latest data in the topic, ensuring that all subsequent reads are based on the refreshed data. +* +* @throws PulsarClientException if there is any error refreshing the table view. +*/ +void refresh() throws PulsarClientException; + + +``` + +# Monitoring + +The proposed changes do not introduce any specific monitoring considerations at this time. + +# Security Considerations + +No specific security considerations have been identified for this proposal. + +# Backward & Forward Compatibility + +## Revert + +No specific revert instructions are required for this proposal. + +## Upgrade + +No specific upgrade instructions are required for this proposal. + +# Alternatives + +## Add consistency model policy to TableView +Add new option configuration `STRONG_CONSISTENCY_MODEL` and `EVENTUAL_CONSISTENCY_MODEL` in TableViewConfigurationData. +• `STRONG_CONSISTENCY_MODEL`: any method will be blocked until the latest value is retrieved. +• `EVENTUAL_CONSISTENCY_MODEL`: all methods are non-blocking, but the value retrieved might not be the latest at the time point. + +However, there might be some drawbacks to this approach: +1. As read and write operations might happen simultaneously, we cannot guarantee consistency. If we provide a configuration about consistency, it might confuse users. +2. This operation will block each get operation. We need to add more asynchronous methods. +3. Less flexibility if users don’t want to refresh the TableView for any reads. + +## New method for combining the refresh and get + +Another option is to add new methods for the existing methods to combine the refresh and reads. For example + +CompletableFuture refreshGet(String key); + +It will refresh the dataset of the TableView and perform the get operation based on the refreshed dataset. But we need to add 11 new methods to the public APIs of the TableView. + + +# General Notes + +No additional general notes have been provided. + +# Links + + +* Mailing List discussion thread: +* Mailing List voting thread: diff --git a/pip/pip-303.md b/pip/pip-303.md new file mode 100644 index 0000000000000..53861631cf9d0 --- /dev/null +++ b/pip/pip-303.md @@ -0,0 +1,224 @@ + +# Motivation + +When a topic has a large number of producers or consumers (over 1k), querying the `pulsarAdmin.topics().getPartitionedStats()` interface is slow and the response size is also large. +As a result, it's essential to give users the option of querying producer and consumer information. + + + +# Goals + +## In Scope + +Add the API for `org.apache.pulsar.client.admin.Topics` +```java +CompletableFuture getPartitionedStatsAsync( + String topic, boolean perPartition, GetStatsOptions getStatsOptions); + +CompletableFuture getStatsAsync(String topic, GetStatsOptions getStatsOptions); +``` + + + +## Out of Scope + +None. + + +# High Level Design + +Implement the `getPartitionedStatsAsync` method, and add the `excludePublishers` and `excludeConsumers` parameters to `{tenant}/{namespace}/{topic}/partitioned-stats` API in `PersistentTopics` and `NonPersistentTopics`. + +# Detailed Design + +## Design & Implementation Details + + +Add two fields for `org.apache.pulsar.client.admin.GetStatsOptions` +```java +@Data +@Builder +public class GetStatsOptions { + /** + * Whether to exclude publishers. + */ + private final boolean excludePublishers; + + /** + * Whether to exclude consumers. + */ + private final boolean excludeConsumers; + +} +``` + +Implement the `getPartitionedStatsAsync` and `getStatsAsync` interface for `org.apache.pulsar.client.admin.internal.TopicsImpl` +```java +@Override +public CompletableFuture getPartitionedStatsAsync(String topic, boolean perPartition, GetStatsOptions getStatsOptions){ + TopicName tn = validateTopic(topic); + WebTarget path = topicPath(tn, "partitioned-stats"); + path = path.queryParam("perPartition", perPartition) + .queryParam("getPreciseBacklog", getStatsOptions.isGetPreciseBacklog()) + .queryParam("subscriptionBacklogSize", getStatsOptions.isSubscriptionBacklogSize()) + .queryParam("getEarliestTimeInBacklog", getStatsOptions.isGetEarliestTimeInBacklog()); + .queryParam("excludePublishers", getStatsOptions.isExcludePublishers()) + .queryParam("excludeConsumers", getStatsOptions.isExcludeConsumers()); +} + +@Override +public CompletableFuture getStatsAsync(String topic, GetStatsOptions getStatsOptions){ + TopicName tn = validateTopic(topic); + WebTarget path = topicPath(tn, "stats") + .queryParam("getPreciseBacklog", getStatsOptions.isGetPreciseBacklog()) + .queryParam("subscriptionBacklogSize", getStatsOptions.isSubscriptionBacklogSize()) + .queryParam("getEarliestTimeInBacklog", getStatsOptions.isGetEarliestTimeInBacklog()); + .queryParam("excludePublishers",getStatsOptions.isExcludePublishers()) + .queryParam("excludeConsumers",getStatsOptions.isExcludeConsumers()); +} +``` + +Add the `excludePublishers` and `excludeConsumers` parameters to `{tenant}/{namespace}/{topic}/partitioned-stats` API +```java +@GET +@Path("{tenant}/{namespace}/{topic}/partitioned-stats") +public void getPartitionedStats( + @Suspended final AsyncResponse asyncResponse, + @ApiParam(value = "Specify the tenant", required = true) + @PathParam("tenant") String tenant, + @ApiParam(value = "Specify the namespace", required = true) + @PathParam("namespace") String namespace, + @ApiParam(value = "Specify topic name", required = true) + @PathParam("topic") @Encoded String encodedTopic, + @ApiParam(value = "Get per partition stats") + @QueryParam("perPartition") @DefaultValue("true") boolean perPartition, + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, + @ApiParam(value = "If return precise backlog or imprecise backlog") + @QueryParam("getPreciseBacklog") @DefaultValue("false") boolean getPreciseBacklog, + @ApiParam(value = "If return backlog size for each subscription, require locking on ledger so be careful " + + "not to use when there's heavy traffic.") + @QueryParam("subscriptionBacklogSize") @DefaultValue("true") boolean subscriptionBacklogSize, + @ApiParam(value = "If return the earliest time in backlog") + @QueryParam("getEarliestTimeInBacklog") @DefaultValue("false") boolean getEarliestTimeInBacklog, + @ApiParam(value = "If exclude the publishers") + @QueryParam("excludePublishers") @DefaultValue("false") boolean excludePublishers, + @ApiParam(value = "If exclude the consumers") + @QueryParam("excludeConsumers") @DefaultValue("false") boolean excludeConsumers) + +``` + +Add the `excludePublishers` and `excludeConsumers` parameters to `{tenant}/{namespace}/{topic}/stats` API +```java +@GET +@Path("{tenant}/{namespace}/{topic}/stats") +public void getStats( + @Suspended final AsyncResponse asyncResponse, + @ApiParam(value = "Specify the tenant", required = true) + @PathParam("tenant") String tenant, + @ApiParam(value = "Specify the namespace", required = true) + @PathParam("namespace") String namespace, + @ApiParam(value = "Specify topic name", required = true) + @PathParam("topic") @Encoded String encodedTopic, + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, + @ApiParam(value = "If return precise backlog or imprecise backlog") + @QueryParam("getPreciseBacklog") @DefaultValue("false") boolean getPreciseBacklog, + @ApiParam(value = "If return backlog size for each subscription, require locking on ledger so be careful " + + "not to use when there's heavy traffic.") + @QueryParam("subscriptionBacklogSize") @DefaultValue("true") boolean subscriptionBacklogSize, + @ApiParam(value = "If return time of the earliest message in backlog") + @QueryParam("getEarliestTimeInBacklog") @DefaultValue("false") boolean getEarliestTimeInBacklog, + @ApiParam(value = "If exclude the publishers"), + @QueryParam("excludePublishers") @DefaultValue("false") boolean excludePublishers, + @ApiParam(value = "If exclude the consumers") + @QueryParam("excludeConsumers") @DefaultValue("false") boolean excludeConsumers) +``` + +Add a new method for `org.apache.pulsar.broker.service.Topic` +```java +CompletableFuture asyncGetStats(GetStatsOptions getStatsOptions); + +@Data +@Builder +public class GetStatsOptions { + /** + * Set to true to get precise backlog, Otherwise get imprecise backlog. + */ + private final boolean getPreciseBacklog; + + /** + * Whether to get backlog size for each subscription. + */ + private final boolean subscriptionBacklogSize; + + /** + * Whether to get the earliest time in backlog. + */ + private final boolean getEarliestTimeInBacklog; + + /** + * Whether to exclude publishers. + */ + private final boolean excludePublishers; + + /** + * Whether to exclude consumers. + */ + private final boolean excludeConsumers; + +} +``` + +Add the following logic in `org.apache.pulsar.broker.service.persistent.PersistentTopic.asyncGetStats` and `org.apache.pulsar.broker.service.persistent.PersistentSubscription.getStats`: + +```java + if (!excludePublishers){ + stats.addPublisher(publisherStats); + } + + if (!excludeConsumers){ + subStats.consumers.add(consumerStats); + } +``` + +## Public-facing Changes + + +### Public API + +### Binary protocol + +### Configuration + +### CLI + +### Metrics + + + +# Monitoring + + + +# Security Considerations + + +# Backward & Forward Compatibility + +## Revert + + +## Upgrade + +# Alternatives + +# General Notes + +# Links + + +* Mailing List discussion thread: https://lists.apache.org/thread/c92043zq6lyrsd5z1hnln48mx858n7vj +* Mailing List voting thread: https://lists.apache.org/thread/hjw3y7h5vd0x7st6zslj3btjcd6yf1lx diff --git a/pip/pip-305.md b/pip/pip-305.md new file mode 100644 index 0000000000000..b57196bc46bc2 --- /dev/null +++ b/pip/pip-305.md @@ -0,0 +1,87 @@ +# Background knowledge + +Pulsar client use Netty DNS to resolve hostnames. + +# Motivation + +Currently Pulsar client levereage on JVM detected DNS servers or on Google DNS servers if nothing was found (as per Netty default). You cannot change which DNS use to resolve hostnames but you are forced to use local server one (like DNS servers configured through resolv.conf or similar ways) or leverage on some Netty "black magic" system properties. + +The ability to directly configure which DNS use is strictly necessary in environment with "specialized" DNS servers. + +# Goals + +## In Scope + +Add a new configuration on Pulsar client to explicitly set which DNS use. + +## Out of Scope + +Fully configure DNS layer, properties, timeouts etcetera. + + +# High Level Design + +A new client configuration will be added to list wich DNS server use. Such configuration will be checked when creating Pulsar clients to instantiate the DNS resolver. +If no configuration is provided the client must use current defaults. + + +# Detailed Design + +## Design & Implementation Details + +The new configuration will be read from org.apache.pulsar.client.impl.ConnectionPool to configure a DnsNameResolverBuilder + +## Public-facing Changes +Add new dnsServerAddresses method on org.apache.pulsar.client.api.ClientBuilder. + +There are no breaking changes, if dnsServerAddresses is not configuret Pulsar will continue to behave like now. + + +### Public API + +NA + +### Binary protocol + +NA + +### Configuration + +Add new dnsServerAddresses property on org.apache.pulsar.client.impl.conf.ClientConfigurationData. + +### CLI + +NA + +### Metrics + +NA + +# Monitoring + +NA + +# Security Considerations + +The client will have the ability to use a different set of DNS servers. It is possible to alter hostnames resolutions however it is expected that this does not pose any security risks. + +# Backward & Forward Compatibility + +## Revert + +Just remove dnsServerAddresses configuration + +## Upgrade + +Configure a dnsServerAddresses server list. The configuration is not mandatory, Pulsar can run without it just like before. + +# Alternatives + +Expose an interface builder to fully configure the DNS layer. It has much more impact and conflict with existing configuration properties dnsLookupBindAddress and dnsLookupBindPort. + +# General Notes + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/p0870y7o6brv5y1ghn5tz9hvs24bl1k4 +* Mailing List voting thread: https://lists.apache.org/thread/7dd0htk0qqkrjxztj445lj3qskxr2dky diff --git a/pip/pip-307.md b/pip/pip-307.md new file mode 100644 index 0000000000000..a919991d08991 --- /dev/null +++ b/pip/pip-307.md @@ -0,0 +1,268 @@ + + +# Background knowledge + +- Pulsar broker load balancer periodically unloads bundles from overloaded brokers. During this unload process, previous owner brokers close topic sessions(e.g. producers, subscriptions(consumers), managed ledgers). When re-assigned, new owner brokers recreate the topic sessions. + +- Pulsar clients request `CommandLookupTopic` to lookup or assign owner brokers for topics and connect to them. + +- PIP-192, the extensible load balancer introduced the bundle state channel that event-sources this unloading process in a state machine manner, from `releasing,` `assigned`, to `owned` state order. At `releasing,` the owner broker "releases" the bundle ownership(close topic sessions). + +- PIP-192, the extensible load balancer introduced TransferShedder, a new shedding strategy, which pre-assigns new owner brokers beforehand. + + +# Motivation + +- When unloading closes many topic sessions, then many clients need to request CommandLookupTopic at the same time, which could cause many lookup requests on brokers. This unloading process can be further optimized if we can let the client directly connect to the new owner broker without following `CommandLookupTopic` requests. +- In the new load balancer(pip-192), since the owner broker is already known, we can modify the close command protocol to pass the new destination broker URL and skip the lookup requests. +- Also, when unloading, we can gracefully shutdown ledgers -- we always close old managed ledgers first and then recreate it on the new owner without conflicts. + +# Goals +- Remove clients' lookup requests in the unload protocol to reduce the publish latency spike and e2e latency spike during +unloading and also to resolve bottlenecks (of thundering lookups) when there are a large number of topics in a cluster. +- Gracefully shutdown managed ledgers before new owners create them to reduce possible race-conditions between ledger close and ledger creations during unloading. + +## In Scope + + + +- This change will be added in the extensible load balancer. + +## Out of Scope + + + +- This won't change the existing load balancer behavior(modular load manager). + + + +# High Level Design + + + +To achieve the goals above, we could modify the bundle transfer protocol by the following. +The proposed protocol change is based on the bundle states from PIP-192. + +Basically, we could close the ledgers only in the releasing state and finally disconnect clients in the owned state with destination broker urls. The clients will directly connect to the pre-assigned destination broker url without lookups. Meanwhile, during this transfer, any produced messages will be ignored by the source broker. + +Current Unload and Lookup Sequence in Extensible Load Balancer +```mermaid +sequenceDiagram + participant Clients + participant Owner Broker + participant New Owner Broker + participant Leader Broker + Leader Broker ->> Owner Broker: "state:Releasing:" close topic + Owner Broker ->> Owner Broker: close broker topic sessions + Owner Broker ->> Clients: close producers and consumers + Clients ->> Clients: reconnecting (inital delay 100ms) + Owner Broker ->> New Owner Broker: "state:Assign:" assign new ownership + New Owner Broker ->> Owner Broker: "state:Owned:" ack new ownership + Clients ->> Owner Broker: lookup + Owner Broker ->> Clients: redirect + Clients ->> New Owner Broker: lookup + New Owner Broker ->> Clients: return(connected) +``` + +Proposed Unload Sequence in Extensible Load Balancer without Lookup +```mermaid +sequenceDiagram + participant Clients + participant Owner Broker + participant New Owner Broker + participant Leader Broker + Leader Broker ->> Owner Broker: "state:Releasing:" close topic + Owner Broker ->> Owner Broker: close broker topic sessions(e.g ledgers) without disconnecting producers/consumers(fenced) + Clients -->> Owner Broker: message pubs are ignored + Owner Broker ->> New Owner Broker: "state:Assign:" assign new ownership + New Owner Broker ->> Owner Broker: "state:Owned:" ack new ownership + Owner Broker ->> Owner Broker: close the fenced broker topic sessions + Owner Broker ->> Clients: close producers and consumers (with newOwnerBrokerUrl) + Clients ->> New Owner Broker: immediately connect +``` + + +# Detailed Design + +## Design & Implementation Details + + + +- Modify CommandCloseProducer, CommandCloseConsumer to pass optional brokerServiceUrls +``` +message CommandCloseProducer { +required uint64 producer_id = 1; +required uint64 request_id = 2; ++ optional string assignedBrokerServiceUrl = 3; ++ optional string assignedBrokerServiceUrlTls = 4; +} + +message CommandCloseConsumer { +required uint64 consumer_id = 1; +required uint64 request_id = 2; ++ optional string assignedBrokerServiceUrl = 3; ++ optional string assignedBrokerServiceUrlTls = 4; +} +``` + +- Add new disconnect apis on producer and consumer to pass dstBrokerLookupData +``` +public CompletableFuture disconnect(Optional dstBrokerLookupData) { +``` + +- Modify the Topic.close() behavior to optionally skip producers.disconnect() and consumers.disconnect(). +``` +public CompletableFuture close(boolean closeWithoutWaitingClientDisconnect, + boolean closeWithoutDisconnectingClients) { + +``` + +- please refer to this poc code for more details: https://github.com/apache/pulsar/compare/master...heesung-sn:pulsar:close-command-dst-url + + +### Eventual Consistency of Ownership States + +This protocol and ownership state checks follow the eventual consistency of the bundle state channel introduced in PIP-192. + +After the client connects to the destination broker, the next command(e.g. ProducerCommand) requires +the destination broker to check the ownership again against its local table view of the bundle state channel. + +Upon this local ownership check, there could be the following scenarios: + +Happy case: +- If the ownership state is `owned ` and the current broker is indeed the owner, the command completes. + +Unhappy cases: +- If the ownership state is `owned ` and the current broker is not the owner, the command fails +(the broker returns an error to the client), and the client tries to find the true new owner by lookups. +The global bundle state channel is eventually consistent, and the lookups should eventually converge. +- if the ownership change is still in progress(`releasing`, `assigning`), this check will be deferred +until the state becomes `owned` with a timeout. + +### Failure Recovery of Ownership States + +The failure recovery logic relies on the bundle state channel cleanup logic introduced in PIP-192. + +When the destination or source broker crashes in the middle of unloading, +the leader will find the orphan state and clean the ownership by selecting a new owner, and the client will reconnect to it. +During this transfer process, if alive, the source broker will serve the topic according to the protocol described in the PIP. + + +## Public-facing Changes + + + +### Public API + + +### Binary protocol + +- Modify CommandCloseProducer, CommandCloseConsumer to pass optional assignedBrokerServiceUrls like the above. + +### Configuration + +### CLI + +### Metrics + + + + +# Monitoring + + + +# Security Considerations + + +# Backward & Forward Compatability +- We are adding new parameters in the close producer and consumer command protocol, the old client versions should not see the optional destination urls in the close commands. Hence, they will request lookups. + +## Revert + + + +## Upgrade + + + +# Alternatives + + + +# General Notes + +# Links + + +* Mailing List discussion thread: +* Mailing List voting thread: + diff --git a/pip/pip-312.md b/pip/pip-312.md new file mode 100644 index 0000000000000..b4b12e6cc5c79 --- /dev/null +++ b/pip/pip-312.md @@ -0,0 +1,150 @@ +# PIP-312: Use StateStoreProvider to manage state in Pulsar Functions endpoints + +# Background knowledge + +States are key-value pairs, where a key is a string and its value is arbitrary binary data - counters are stored as 64-bit big-endian binary values. +Keys are scoped to an individual function and shared between instances of that function. + +Pulsar Functions use `StateStoreProvider` to initialize a `StateStore` to manage state, so it can support multiple state storage backend, such as: +- `BKStateStoreProviderImpl`: use Apache BookKeeper as the backend +- `PulsarMetadataStateStoreProviderImpl`: use Pulsar Metadata as the backend + +Users can also implement their own `StateStoreProvider` to support other state storage backend. + +The Broker also exposes two endpoints to put and query a state key of a function: +- GET /{tenant}/{namespace}/{functionName}/state/{key} +- POST /{tenant}/{namespace}/{functionName}/state/{key} + +Although Pulsar Function supports multiple state storage backend, these two endpoints are still using BookKeeper's `StorageAdminClient` directly to put and query state, +this makes the Pulsar Functions' state store highly coupled with Apache BookKeeper. + +See: [code](https://github.com/apache/pulsar/blob/1a66b640c3cd86bfca75dc9ab37bfdb37427a13f/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java#L1152-L1297) + +# Motivation + +This proposal aims to decouple Pulsar Functions' state store from Apache BookKeeper, so it can support other state storage backend. + +# Goals + +## In Scope + +- Pulsar Functions can use other state storage backend other than Apache BookKeeper. + +## Out of Scope + +None + +# High Level Design + +- Replace the `StorageAdminClient` in `ComponentImpl` with `StateStoreProvider` to manage state. +- Add a `cleanup` method to the `StateStoreProvider` interface + +# Detailed Design + +## Design & Implementation Details + +1. In the `ComponentImpl#getFunctionState` and `ComponentImpl#queryState` methods, replace the `StorageAdminClient` with `StateStoreProvider`: + + ```java + String tableNs = getStateNamespace(tenant, namespace); + String tableName = functionName; + + String stateStorageServiceUrl = worker().getWorkerConfig().getStateStorageServiceUrl(); + + if (storageClient.get() == null) { + storageClient.compareAndSet(null, StorageClientBuilder.newBuilder() + .withSettings(StorageClientSettings.newBuilder() + .serviceUri(stateStorageServiceUrl) + .clientName("functions-admin") + .build()) + .withNamespace(tableNs) + .build()); + } + ... + ``` + + Replaced to: + + ```java + DefaultStateStore store = worker().getStateStoreProvider().getStateStore(tenant, namespace, name); + ``` + +2. Add a `cleanup` method to the `StateStoreProvider` interface: + + ```java + default void cleanUp(String tenant, String namespace, String name) throws Exception; + ``` + + Because when delete a function, the related state store should also be deleted. + Currently, it's also using BookKeeper's `StorageAdminClient` to delete the state store table: + + ```java + deleteStatestoreTableAsync(getStateNamespace(tenant, namespace), componentName); + + + private void deleteStatestoreTableAsync(String namespace, String table) { + StorageAdminClient adminClient = worker().getStateStoreAdminClient(); + if (adminClient != null) { + adminClient.deleteStream(namespace, table).whenComplete((res, throwable) -> { + if ((throwable == null && res) + || ((throwable instanceof NamespaceNotFoundException + || throwable instanceof StreamNotFoundException))) { + log.info("{}/{} table deleted successfully", namespace, table); + } else { + if (throwable != null) { + log.error("{}/{} table deletion failed {} but moving on", namespace, table, throwable); + } else { + log.error("{}/{} table deletion failed but moving on", namespace, table); + } + } + }); + } + } + ``` + + So this proposal will add a `cleanup` method to the `StateStoreProvider` and call it after a function is deleted: + + ```java + worker().getStateStoreProvider().cleanUp(tenant, namespace, hashName); + ``` + +3. Add a new `init` method to `StateStoreProvider` interface: + + The current `init` method requires a `FunctionDetails` parameter, but we cannot get the `FunctionDetails` in the `ComponentImpl` class, + and this parameter is not used either in `BKStateStoreProviderImpl` or in `PulsarMetadataStateStoreProviderImpl`, + but for backward compatibility, instead of updating the `init` method, this proposal will add a new `init` method without `FunctionDetails` parameter: + + ```java + default void init(Map config) throws Exception {} + ``` + +## Public-facing Changes + +None + +# Monitoring + +# Security Considerations + +# Backward & Forward Compatibility + +## Revert + +- Nothing needs to be done if users use the Apache BookKeeper as the state storage backend. +- If users use another state storage backend, they need to change it back to BookKeeper. + +## Upgrade + +Nothing needs to be done. + +# Alternatives + +# General Notes + +# Links + + +* Mailing List discussion thread: https://lists.apache.org/thread/0rz29wotonmdck76pdscwbqo19t3rbds +* Mailing List voting thread: https://lists.apache.org/thread/t8vmyxovrrb5xl8jvrp1om50l6nprdjt diff --git a/pip/pip-313.md b/pip/pip-313.md new file mode 100644 index 0000000000000..e43e965a11771 --- /dev/null +++ b/pip/pip-313.md @@ -0,0 +1,76 @@ +# PIP-313: Support force unsubscribe using consumer api + +# Motivation + +As discussed in Issue: https://github.com/apache/pulsar/issues/21451 + +Apache Pulsar provides a messaging queue using a Shared subscription to process unordered messages in parallel using multiple connected consumers. Shared subscription is also commonly used in data processing pipelines where they need to forcefully unsubscribe from the subscription after processing messages on the topic. One example is Pulsar-Storm adapter where [Pulsar spout](https://github.com/apache/pulsar/blob/branch-2.4/pulsar-storm/src/main/java/org/apache/pulsar/storm/PulsarSpout.java#L126) creates Pulsar consumers on a shared subscription for distributed processing and then unsubscribe on the topic. + +However, PulsarSpout always fails to unsubscribe shared subscriptions and it also doesn't close the pulsar consumers if there is more than one consumer connected to the subscription which causes a leaked subscription and consumer for that application. It also causes a backlog on a topic due to failed unsubscribe and application team has to build external service to just address such failures. + +In this usecases, client application can not successfully unsubscribe on a shared subscription when multiple consumers are connected because Pulsar client library first tries to unsubscribe which will not be successful as multiple consumers are still connected on the subscription and eventually Pulsar client lib fails to unsubscribe and close the consumer on the subscription. Because of that none of the consumers can disconnect or unsubscribe from the subscription. This will make it impossible for applications to unsubscribe on a shared subscription and they need an API to forcefully unsubscribe on a shared subscription using consumer API. +We already have the admin-api to unsubscribe forcefully but adding such support in consumer API will allow applications like Pulsar-storm to unsubscribe successfully and also allow consumers to close gracefully. + +# Goals + +Support unsubscribe API with force option in consumer API along with admin API which can help applications to unsubscribe on various subscriptions such as Failover, Shared, Key-Shared. + +# High Level Design + +Consumer API will have additional unsubscribe api with additional flag to enable forceful unsubscribe on a subscription. Pulsar client library will pass the flag to broker while unsubscribing and broker will use it with existing broker side implementation of ubsubscribing forcefully. + + +## Design & Implementation Details + +### (1) Pulsar client library changes + +Add support of unsubscribe api with force option in Consumer API + +``` +Consumer.java + +void unsubscribe(boolean force) throws PulsarClientException; +CompletableFuture unsubscribeAsync(boolean force); +``` + +Calling unsubscribe with force flag will make broker to fence the subscription and disconnect all the consumers forcefully to eventually unsubscribe and delete the subscription. However, reconnection of the consumer can recreate the subscription so, client application should make sure to call force-unsubscribe from all the consumers to eventually delete subscription or disable auto subscription creation based on application usecases. + +### (2) Protobuf changes + +Pulsar client library will pass an additional force flag (with default value =false) to the broker with wire protocol change + +``` +PulsarApi.proto + +message CommandUnsubscribe { + required uint64 consumer_id = 1; + required uint64 request_id = 2; + optional bool force = 3 [default = false]; +} +``` + +### (3) Broker changes + +Broker already supports force delete subscription using admin-api so, broker already has implementation to unsubscribe forcefully but it doesn’t have option to trigger using binary api. Therefore, once client sends additional force flag to broker while unsubscribing , broker reads the flag and passes to the subscription API to forcefully unsubscribe the subscription. + + +# Security Considerations + + + +# General Notes + +# Links + +Issue: https://github.com/apache/pulsar/issues/21451 +Sample PR: https://github.com/apache/pulsar/compare/master...rdhabalia:shared_unsub?expand=1 +Discuss thread: https://lists.apache.org/thread/hptx8z9mktn94gvqtt4547wzcfcgdsrv +Vote thread: https://lists.apache.org/thread/3kp9hfs5opw17fgmkn251sc6cd408yty + + diff --git a/pip/pip-315.md b/pip/pip-315.md new file mode 100644 index 0000000000000..fd6cb3ef25727 --- /dev/null +++ b/pip/pip-315.md @@ -0,0 +1,137 @@ +# PIP-315: Configurable max delay limit for delayed delivery + +# Background knowledge +Delayed message delivery is an important feature which allows a producer to specify that a message should be delivered/consumed at a later time. Currently the broker will save a delayed message without any check. The message's `deliverAt` time is checked when the broker dispatches messages to the Consumer. If a message has a `deliverAt` time, then it is added to the `DelayedDeliveryTracker` and will be delivered later when eligible. + +Delayed message delivery is only available for persistent topics, and shared/key-shared subscription types. + +# Motivation +Currently there is no max delay limit so a producer can specify any delay when publishing a message. + +This poses a few challenges: +1. Producer may miscalculate/misconfigure a very large delay (ex. 1,000 day instead of 100 day delay) +2. Pulsar administrators may want to limit the max allowed delay since unacked messages (ex. messages with a large delay) will be stored forever (unless TTL is configured) +3. The configured delay may be greater than the configured TTL which means the delayed message may be deleted before the `deliverAt` time (before the consumer can process it) + +# Goals +The purpose of this PIP is to introduce an optional configuration to limit the max allowed delay for delayed delivery. + +## In Scope +- Add broker configuration to limit the max allowed delay for delayed delivery +- Configurable at broker/topic/namespace-level + +# High Level Design +We will add a configuration `maxDeliveryDelayInMillis` and if configured, the broker will check incoming delayed messages to see if the message's `deliverAt` time exceeds the configured limit. If it exceeds the limit, the broker will send an error back to the Producer. + +# Detailed Design + +## Design & Implementation Details + +### Broker Changes +A new `maxDeliveryDelayInMillis` config will be added to the broker which is initially defaulted to 0 (disabled). The default (disabled) behavior will match the current delayed delivery behavior (no limit on delivery delay). +``` +# broker.conf +delayedDeliveryMaxDeliveryDelayInMillis=0 +``` + +This field will also be added to the existing `DelayedDeliveryPolicies` interface to support topic & namespace-level configuration: +```java +public interface DelayedDeliveryPolicies { + long getMaxDeliveryDelayInMillis(); +} +``` + +The max delivery delay check will occur in the broker's `Producer` class inside of `checkAndStartPublish` (same place as other checks such as `isEncryptionEnabled`). + +We will give a `ServerError.NotAllowedError` error if all of the following are true: +1. Sending to a persistent topic +2. Topic has `delayedDeliveryEnabled=true` +3. `MessageMetadata` `deliver_at_time` has been specified +4. Topic has `>0` value for `maxDeliveryDelayInMillis` +5. `deliver_at_time - publish_time` > `maxDeliveryDelayInMillis` + +```java +// In org.apache.pulsar.broker.service.Producer#checkAndStartPublish +if (topic.isPersistent()) { + PersistentTopic pTopic = (PersistentTopic) topic; + if (pTopic.isDelayedDeliveryEnabled()) { + headersAndPayload.markReaderIndex(); + MessageMetadata msgMetadata = Commands.parseMessageMetadata(headersAndPayload); + headersAndPayload.resetReaderIndex(); + if (msgMetadata.hasDeliverAtTime()) { + long maxDeliveryDelayInMillis = pTopic.getMaxDeliveryDelayInMillis(); + if (maxDeliveryDelayInMillis > 0 + && msgMetadata.getDeliverAtTime() - msgMetadata.getPublishTime() > maxDeliveryDelayInMillis) { + cnx.execute(() -> { + cnx.getCommandSender().sendSendError(producerId, sequenceId, ServerError.NotAllowedError, + String.format("Exceeds max allowed delivery delay of %s milliseconds", maxDeliveryDelayInMillis)); + cnx.completedSendOperation(false, headersAndPayload.readableBytes()); + }); + return false; + } + } + } +} +``` + +### Consumer Impact +The proposal does not involve any client changes, however it is important to note that setting a max delivery delay may impact the `Consumer` since the `Consumer` uses delayed delivery for retrying to the retry/dlq topic (ex. `reconsumeLater` API). So the max `Consumer` retry delay will be the same as the configured `maxDeliveryDelayInMillis` (if enabled). + +A problem will occur if max delivery delay is configured but a `Consumer` uses a larger custom retry delay. In this scenario, the `Consumer` will actually get stuck redelivering the message as the publish to the retry topic will fail. For this scenario, a larger retry delay should be configured specifically for the Consumer's retry topic (or no delay limit should be used for retry topics). + +A more elegant solution would require a protocol change (see `Alternatives` section below). + +## Public-facing Changes + +### Public API +The optional `maxDeliveryDelayInMillis` field will be added to the admin REST APIs for configuring topic/namespace policies: +- `POST /admin/v2/namespaces/{tenant}/{namespace}/delayedDelivery` +- `POST /admin/v2/persistent/{tenant}/{namespace}/{topic}/delayedDelivery` + +And the corresponding `GET` APIs will show `maxDeliveryDelayInMillis` in the response: +- `GET /admin/v2/namespaces/{tenant}/{namespace}/delayedDelivery` +- `GET /admin/v2/persistent/{tenant}/{namespace}/{topic}/delayedDelivery` + +### Configuration +Broker will have a new config in `broker.conf`: +``` +# The max allowed delay for delayed delivery (in milliseconds). If the broker receives a message which exceeds this max delay, then +# it will return an error to the producer. +# The default value is 0 which means there is no limit on the max delivery delay. +delayedDeliveryMaxDeliveryDelayInMillis=0 +``` + +### CLI +Both `CmdTopics` and `CmdNamespaces` will be updated to include this additional optional configuration. + +# Backward & Forward Compatibility + +## Revert +Reverting to a previous version will simply get rid of this config/limitation which is the previous behavior. + +## Upgrade +We will default the value to 0/disabled (no limitation), so this is a backwards compatible change and will not cause any functional change when upgrading to this feature/version. This feature will only be applied once the config is changed. + +If configured, the `maxDeliveryDelayInMillis` limitation will affect: +1. Producers who configure a longer max delivery delay (PIP-26: 2.4.0+) +2. Consumers who configure a longer retry delay when using retry topic (PIP-58: 2.6.0+) + +# Alternatives +## Add delayed delivery limit check at client-side +An alternative is to add the limit check to the client-side which requires a protocol change so that client `Producer`/`Consumer` will receive the delayed delivery configurations from the broker. The client `Producer` can then throw an exception if the caller provides a delay greater than the configured limit. The client `Consumer` can more elegantly handle when the retry publish delay is greater than the configured limit as it can default to using the limit instead of being stuck waiting for the limit to be increased. + +This would still require the broker-side check as someone may be using a custom client. The main benefit is being able to elegantly handle the `Consumer` retry topic scenario. + +If we were to make this protocol change, then it might make sense to also have the `Producer` check the `delayedDeliveryEnabled` config. If delayed delivery is disabled and the `Producer` tries to send a delayed message, then an exception is thrown to the caller (current behavior is the broker will just deliver the message instantly and no error is provided to the `Producer` so it can be misleading). + +We would also need to add the client-side checks to other supported client libraries. + +Since the scope of this alternative would be quite expansive, we may want to pursue this in a follow-up PIP instead of trying to address it all at once. + +# Links + + +* Mailing List discussion thread: https://lists.apache.org/thread/285nm08842or324rxc2zy83wxgqxtcjp +* Mailing List voting thread: https://lists.apache.org/thread/gkqrfrxx74j0dmrogg3now29v1of9zm9 diff --git a/pip/pip-318.md b/pip/pip-318.md new file mode 100644 index 0000000000000..988eea0bb8b36 --- /dev/null +++ b/pip/pip-318.md @@ -0,0 +1,49 @@ +# PIP-318: Don't retain null-key messages during topic compaction + +# Background knowledge + +Apache Pulsar is supported [Topic Compaction](https://pulsar.apache.org/docs/en/concepts-topic-compaction/) which is a key-based data retention mechanism. + +# Motivation + +Currently, we retain all null-key messages during topic compaction, which I don't think is necessary because when you use topic compaction, it means that you want to retain the value according to the key, so retaining null-key messages is meaningless. + +Additionally, retaining all null-key messages will double the storage cost, and we'll never be able to clean them up since the compacted topic has not supported the retention policy yet. + +In summary, I don't think we should retain null-key messages during topic compaction. + +# Goals + +# High Level Design + +In order to avoid introducing break changes to release version, we need to add a configuration to control whether to retain null-key messages during topic compaction. +If the configuration is true, we will retain null-key messages during topic compaction, otherwise, we will not retain null-key messages during topic compaction. + +## Public-facing Changes + +### Configuration + +Add config to broker.conf/standalone.conf +```properties +topicCompactionRetainNullKey=false +``` + +# Backward & Forward Compatibility + +- Make `topicCompactionRetainNullKey=false` default in the 3.2.0. +- Cherry-pick it to a branch less than 3.2.0 make `topicCompactionRetainNullKey=true` default. +- Delete the configuration `topicCompactionRetainNullKey` in 3.3.0 and don't supply an option to retain null-keys. + +## Revert + +Make `topicCompactionRetainNullKey=true` in broker.conf/standalone.conf. + +## Upgrade + +Make `topicCompactionRetainNullKey=false` in broker.conf/standalone.conf. + + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/68k6vrghfp3np601lrfx5mbfmghbbrjh +* Mailing List voting thread: https://lists.apache.org/thread/36rfmvz5rchgnvqb2wcq4wb64k6st90p \ No newline at end of file diff --git a/pip/pip-320.md b/pip/pip-320.md new file mode 100644 index 0000000000000..3c169e4340bd1 --- /dev/null +++ b/pip/pip-320.md @@ -0,0 +1,256 @@ +# PIP-320 OpenTelemetry Scaffolding + +# Background knowledge + +## PIP-264 - parent PIP titled "Enhanced OTel-based metric system" +[PIP-264](https://github.com/apache/pulsar/pull/21080), which can also be viewed [here](pip-264.md), describes in high +level a plan to greatly enhance Pulsar metric system by replacing it with [OpenTelemetry](https://opentelemetry.io/). +You can read in the PIP the numerous existing problems PIP-264 solves. Among them are: +- Control which metrics to export per topic/group/namespace via the introduction of a metric filter configuration. + This configuration is planned to be dynamic as outline in the [PIP-264](pip-264.md). +- Reduce the immense metrics cardinality due to high topic count (One of Pulsar great features), by introducing +the concept of Metric Group - a group of topics for metric purposes. Metric reporting will also be done to a +group granularity. 100k topics can be downsized to 1k groups. The dynamic metric filter configuration would allow +the user to control which metric group to un-filter. +- Proper histogram exporting +- Clean-up codebase clutter, by relying on a single industry standard API, SDK and metrics protocol (OTLP) instead of +existing mix of home-brew libraries and hard coded Prometheus exporter. +- any many more + +You can [here](pip-264.md#why-opentelemetry) why OpenTelemetry was chosen. + +## OpenTelemetry +Since OpenTelemetry (a.k.a. OTel) is an emerging industry standard, there are plenty of good articles, videos and +documentation about it. In this very short paragraph I'll describe what you need to know about OTel from this PIP +perspective. + +OpenTelemetry is a project aimed to standardize the way we instrument, collect and ship metrics from applications +to telemetry backends, be it databases (e.g. Prometheus, Cortex, Thanos) or vendors (e.g. Datadog, Logz.io). +It is divided into API, SDK and Collector: +- API: interfaces to use to instrument: define a counter, record values to a histogram, etc. +- SDK: a library, available in many languages, implementing the API, and other important features such as +reading the metrics and exporting it out to a telemetry backend or OTel Collector. +- Collector: a lightweight process (application) which can receive or retrieve telemetry, transform it (e.g. +filter, drop, aggregate) and export it (e.g. send it to various backends). The SDK supports out-of-the-box +exporting metrics as Prometheus HTTP endpoint or sending them out using OTLP protocol. Many times companies choose to +ship to the Collector and there ship to their preferred vendors, since each vendor already published their exporter +plugin to OTel Collector. This makes the SDK exporters very light-weight as they don't need to support any +vendor. It's also easier for the DevOps team as they can make OTel Collector their responsibility, and have +application developers only focus on shipping metrics to that collector. + +Just to have some context: Pulsar codebase will use the OTel API to create counters / histograms and records values to +them. So will the Pulsar plugins and Pulsar Function authors. Pulsar itself will be the one creating the SDK +and using that to hand over an implementation of the API where ever needed in Pulsar. Collector is up to the choice +of the user, as OTel provides a way to expose the metrics as `/metrics` endpoint on a configured port, so Prometheus +compatible scrapers can grab it from it directly. They can also send it via OTLP to OTel collector. + +## Telemetry layers +PIP-264 clearly outlined there will be two layers of metrics, collected and exported, side by side: OpenTelemetry +and the existing metric system - currently exporting in Prometheus. This PIP will explain in detail how it will work. +The basic premise is that you will be able to enable or disable OTel metrics, alongside the existing Prometheus +metric exporting. + +## Why OTel in Pulsar will be marked experimental and not GA +As specified in [PIP-264](pip-264.md), OpenTelemetry Java SDK has several fixes the Pulsar community must +complete before it can be used in production. They are [documented](pip-264.md#what-we-need-to-fix-in-opentelemetry) +in PIP-264. The most important one is reducing memory allocations to be negligible. OTel SDK is built upon immutability, +hence allocated memory in O(`#topics`) which is a performance killer for low latency application like Pulsar. + +You can track the proposal and progress the Pulsar and OTel communities are making in +[this issue](https://github.com/open-telemetry/opentelemetry-java/issues/5105). + + +## Metrics endpoint authentication +Today Pulsar metrics endpoint `/metrics` has an option to be protected by the configured `AuthenticationProvider`. +The configuration option is named `authenticateMetricsEndpoint` in the broker and +`authenticateMetricsEndpoint` in the proxy. + + +# Motivation + +Implementing PIP-264 consists of a long list of steps, which are detailed in +[this issue](https://github.com/apache/pulsar/issues/21121). The first step is add all the bare-bones infrastructure +to use OpenTelemetry in Pulsar, such that next PRs can use it to start translating existing metrics to their +OTel form. It means the same metrics will co-exist in the codebase and also in runtime, if OTel was enabled. + +# Goals + +## In Scope +- Ability to add metrics using OpenTelemetry to Pulsar components: Broker, Function Worker and Proxy. +- User can disable or enable OpenTelemetry metrics, which by default will be disabled +- OpenTelemetry metrics will be configured via its native OTel Java SDK configuration options +- All the necessary information to use OTel with Pulsar will be documented in Pulsar documentation site +- OpenTelemetry metrics layer defined as experimental, and *not* GA + + +## Out of Scope +- Ability to add metrics using OpenTelemetry as Pulsar Function author. +- Only authenticated sessions can access OTel Prometheus endpoint, using Pulsar authentication +- Metrics in Pulsar clients (as defined in [PIP-264](pip-264.md#out-of-scope))) + +# High Level Design + +## Configuration +OpenTelemetry, as any good telemetry library (e.g. log4j, logback), has its own configuration mechanisms: +- System properties +- Environment variables +- Experimental file-based configuration + +Pulsar doesn't need to introduce any additional configuration. The user can decide, using OTel configuration +things like: +* How do I want to export the metrics? Prometheus? Which port prometheus will be exposed at +* Change histogram buckets using Views +* and more + +Pulsar will use `AutoConfiguredOpenTelemetrySdk` which uses all the above configuration mechanisms +(documented [here](https://github.com/open-telemetry/opentelemetry-java/tree/main/sdk-extensions/autoconfigure)). +This class builds an `OpenTelemetrySdk` based on configurations. This is the entry point to OpenTelemetry API, as it +implements `OpenTelemetry` API class. + +### Setting sensible defaults for Pulsar +There are some configuration options we wish to change their default, but still allow the users to override it +if they wish. We think those default values will make a much easier user experience. + +* `otel.experimental.metrics.cardinality.limit` - value: 10,000 +This property sets an upper bound on the amount of unique `Attributes` an instrument can have. Take Pulsar for example, +an instrument like `pulsar.broker.messaging.topic.received.size`, the unique `Attributes` would be in the amount of +active topics in the broker. Since Pulsar can handle up to 1M topics, it makes more sense to put the default value +to 10k, which translates to 10k topics. + +`AutoConfiguredOpenTelemetrySdkBuilder` allows to add properties using the method `addPropertiesSupplier`. The +System properties and environment variables override it. The file-based configuration still doesn't take +those properties supplied into account, but it will. + + +## Opting in +We would like to have the ability to toggle OpenTelemetry-based metrics, as they are still new. +We won't need any special Pulsar configuration, as OpenTelemetry SDK comes with a configuration key to do that. +Since OTel is still experimental, it will have to be opt-in, hence we will add the following property to be the default +using the mechanism described [above](#setting-sensible-defaults-for-pulsar): + +* `otel.sdk.disabled` - value: true + This property value disables OpenTelemetry. + +With OTel disabled, the user remains with the existing metrics system. OTel in a disabled state operates in a +no-op mode. This means, instruments do get built, but the instrument builders return the same instance of a +no-op instrument, which does nothing on record-values method (e.g. `add(number)`, `record(number)`). The no-op +`MeterProvider` has no registered `MetricReader` hence no metric collection will be made. The memory impact +is almost 0 and the same goes for CPU impact. + +The current metric system doesn't have a toggle which causes all existing data structures to stop collecting +data. Inserting will need changing in so many places since we don't have a single place which through +all metric instrument are created (one of the motivations for PIP-264). +The current system do have a toggle: `exposeTopicLevelMetricsInPrometheus`. It enables toggling off +topic-level metrics, which means the highest cardinality metrics will be namespace level. +Once that toggle is `false`, the amount of data structures accounting memory would in the range of +a few thousands which shouldn't post a burden memory wise. If the user refrain from calling +`/metrics` it will also reduce the CPU and memory cost associated with collecting metrics. + +When the user enables OTel it means there will be a memory increase, but if the user disabled topic-level +metrics in existing system, as specified above, the majority of the memory increase will be due to topic level +metrics in OTel, at the expense of not having them in the existing metric system. + + + +## Cluster attribute name +A broker is part of a cluster. It is configured in the Pulsar configuration key `clusterName`. When the broker is part +of a cluster, it means it shares the topics defined in that cluster (persisted in Metadata service: e.g. ZK) +among the brokers of that cluster. + +Today, each unique time series emitted in Prometheus metrics contains the `cluster` label (almost all of them, as it +is done manually). We wish the same with OTel - to have that attribute in each exported unique time series. + +OTel has the perfect location to place attributes which are shared across all time series: Resource. An application +can have multiple Resource, with each having 1 or more attributes. You define it once, in OTel initialization or +configuration. It can contain attributes like the hostname, AWS region, etc. The default contains the service name +and some info on the SDK version. + +Attributes can be added dynamically, through `addResourceCustomizer()` in `AutoConfiguredOpenTelemetrySdkBuilder`. +We will use that to inject the `cluster` attribute, taken from the configuration. + +In Prometheus, we submitted a [proposal](https://github.com/open-telemetry/opentelemetry-specification/pull/3761) +to opentelemetry specifications, which was merged, to allow copying resource attributes into each exported +unique time series in Prometheus exporter. +We plan to contribute its implementation to OTel Java SDK. + +Resources in Prometheus exporter, are exported as `target_info{} 1` and the attributes are added to this +time series. This will require making joins to get it, making it extremely difficult to use. +The other alternative was to introduce our own `PulsarAttributesBuilder` class, on top of +`AttributesBuilder` of OTel. Getting every contributor to know this class, use it, is hard. Getting this +across Pulsar Functions or Plugins authors, will be immensely hard. Also, when exporting as +OTLP, it is very inefficient to repeat the attribute across all unique time series, instead of once using +Resource. Hence, this needed to be solved in the Prometheus exporter as we did in the proposal. + +The attribute will be named `pulsar.cluster`, as both the proxy and the broker are part of this cluster. + +## Naming and using OpenTelemetry + +### Attributes +* We shall prefix each attribute with `pulsar.`. Example: `pulsar.topic`, `pulsar.cluster`. + +### Instruments +We should have a clear hierarchy, hence use the following prefix +* `pulsar.broker` +* `pulsar.proxy` +* `pulsar.function_worker` + +### Meter +It's customary to use reverse domain name for meter names. Hence, we'll use: +* `org.apache.pulsar.broker` +* `org.apache.pulsar.proxy` +* `org.apache.pulsar.function_worker` + +OTel meter name is converted to the attribute name `otel_scope_name` and added to each unique time series +attributes by Prometheus exporter. + +We won't specify a meter version, as it is used solely to signify the version of the instrumentation, and +currently we are the first version, hence not use it. + + +# Detailed Design + +## Design & Implementation Details + +* `OpenTelemetryService` class + * Parameters: + * Cluster name + * What it will do: + - Override default max cardinality to 10k + - Register a resource with cluster name + - Place defaults setting to instruct Prometheus Exporter to copy resource attributes + - In the future: place defaults for Memory Mode to be REUSABLE_DATA + +* `PulsarBrokerOpenTelemetry` class + * Initialization + * Construct an `OpenTelemetryService` using the cluster name taken from the broker configuration + * Constructs a Meter for the broker metrics + * Methods + * `getMeter()` returns the `Meter` for the broker + * Notes + * This is the class that will be passed along to other Pulsar service classes that needs to define + telemetry such as metrics (in the future: traces). + +* `PulsarProxyOpenTelemetry` class + * Same as `PulsarBrokerOpenTelemetry` but for Pulsar Proxy +* `PulsarWorkerOpenTelemetry` class + * Same as `PulsarBrokerOpenTelemetry` but for Pulsar function worker + + +## Public-facing Changes + +### Public API +* OTel Prometheus Exporter adds `/metrics` endpoint on a user defined port, if user chose to use it + +### Configuration +* OTel configurations are used + +# Security Considerations +* OTel currently does not support setting a custom Authenticator for Prometheus exporter. +An issue has been raised [here](https://github.com/open-telemetry/opentelemetry-java/issues/6013). + * Once it do we can secure the Prometheus exporter metrics endpoint using `AuthenticationProvider` +* Any user can access metrics, and they are not protected per tenant. Like today's implementation + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/xcn9rm551tyf4vxrpb0th0wj0kktnrr2 +* Mailing List voting thread: https://lists.apache.org/thread/zp6vl9z9dhwbvwbplm60no13t8fvlqs2 diff --git a/pip/pip-321.md b/pip/pip-321.md new file mode 100644 index 0000000000000..41b05e692edae --- /dev/null +++ b/pip/pip-321.md @@ -0,0 +1,311 @@ +# PIP-321: Split the responsibilities of namespace replication-clusters + +# Background knowledge + +Pulsar's geo-replication mechanism is typically used for disaster recovery, enabling the replication of persistently stored message data across multiple data centers. For instance, your application publishes data in one region, and you would like to process it for consumption in other regions. With Pulsar's geo-replication mechanism, messages can be produced and consumed in different geo-replicated regions. See the introduction of geo-replication to get more information.[1] + +A client can set allowed clusters for a tenant. The allowed-cluster for the tenant is a cluster that the tenant can access. + +A client can set replication clusters for a namespace, and the pulsar broker internally manages replication to all the replication clusters. And the replication clusters for a namespace must be a subgroup of the tenant's allowed clusters. + +A client cannot set allowed clusters at the namespace level, but the functionality of replication-clusters essentially accomplishes something similar. A topic cannot be created or loaded at clusters that are not specified in the replication-clusters of the namespace policy. +Subsequently, PIP-8 introduced the concept of peer-clusters for global namespace redirection and fails the PartitionedMetadata-Lookup request if the global namespace's replication-clusters do not include the current/peer-clusters. More information about peer-clusters can be found in PIP-8. [2] + +A namespace has multiple topics. Once a namespace is configured with replication clusters, all the topics under this namespace will enable replication in these clusters. + +Namespace Policy is a configuration in the namespace level, that is stored in the configuration store, e.g. zookeeper, and this configuration can not be accessed across multiple clusters with different configuration stores. + +Replication clusters can be configured at the message level. Pulsar support setting replication clusters when send messages. +```java +producer.newMessage().replicationClusters(List.of("cluster1", "cluster2")).send(); +``` + +[1] https://pulsar.apache.org/docs/3.1.x/concepts-replication +[2] https://github.com/apache/pulsar/pull/903 +[3] https://github.com/apache/pulsar/wiki/PIP-92%3A-Topic-policy-across-multiple-clusters + +# Motivation + +Geo-replication at the topic level and message level can't work as expected when geo-replication is disabled at the namespace level and the clusters use a shared configuration store. +Let's see an example: + +**Example For Topic Level:** + +- Environment: + - cluster1 and cluster2 in different regions sharing the same configuration store. + +- Replication clusters configuration: + - Set namespace `ns` replication clusters : cluster1 (local cluster) + - Set topic `ns/topic1` replication clusters : cluster1, cluster2. + +- Expected: + - Topic `ns/topic1` can replicate between cluster1 and cluster2. + +- Actual: + - Topic cannot be created at cluster2. +``` +PRECONDITION_FAILED: Namespace missing local cluster name in clusters list: local_cluster=cluster2 ns=ns clusters=[cluster1] +``` + +**Example For Message Level** + +- Environment: + - cluster1 and cluster2 in different regions sharing the same Zookeeper cluster. + +- Replication clusters configuration: + - Set namespace `ns` replication clusters : cluster1 (local cluster) + - Set replication clusters when send message1: cluster1, cluster2. + +- Expected: + - Message1 can replicate between cluster1 and cluster2. + +- Actual: + - Topic cannot be created at cluster2, and so the message1 can not be replicated to cluster2. +``` +PRECONDITION_FAILED: Namespace missing local cluster name in clusters list: local_cluster=cluster2 ns=ns clusters=[cluster1] +``` + +The root cause of these issues is that topics cannot access clusters that are not included in the replication-clusters of the namespace policy. +If you set both clusters to the namespace's replication clusters. All the topics under this namespace will start to replicate data between clusters unless they set replication clusters to one cluster to the topic level for all topics. It's super hard for Pulsar maintainers and impossible to control the newly created topics (create a topic first and then set topic policies). The replication clusters and allowed clusters are different. Employing one configuration for two purposes is insoluble. +But in the current implementation, the replication-clusters and allowed-Clusters are all configured by specifying the replication clusters. +This will make the topic unable to have its replication policies. + +To support geo-replication policies at the topic level and the message level, we must make the cluster configuration at the namespace level more clearly. +Introduce `allowed-clusters` at the namespace level and make `replication-clusters` only the default replication clusters for the topics under the namespace. + +# Goals + +## In Scope + +The namespace will have a clearer configuration for clusters. Users can use `replication-clusters` and `allowed-clusters` to specify the clusters that the data of the namespace will replicate and the clusters that can load the topics under the namespace; it's similar to the tenant's `allowed-clusters.` + +## Out of Scope + +This proposal can be used to solve the problem of topic-level and message level geo-replication can not work as expected. It is the initial motivation for this proposal, but this proposal does not involve modifications to geo-replication. + +Out of this proposal, there are others actions needed to perform. +1. Limit the replication configuration at the namespace level, topic level and message level. + - The `replication_clusters` at the namespace level, topic level and message level should be the subgroup of `allowed_clusters` at the namespace level. + - Otherwise, `400 Bad Request` will be returned when specify the `replication_clusters` at the namespace level or topic level. + - Fail send request with a `NotAllowedException` exception when the `replication_clusters` of the message is not the subgroup of the `allowed_clusters` at the namespace level. +2. Implement `allowed_clusters` at the topic level, this should need another proposal. + - If `allowed_clusters` is implemented in the topic policy, the `replication_clusters` at the topic level and message level should be the subgroup of the `allowed_clusters` of the topic level. + - Otherwise, `400 Bad Request` will be returned when specify the `replication_clusters`at the namespace level. + - Fail send request with a `NotAllowedException` exception when the `replication_clusters` of the message is not the subgroup of the `allowed_clusters` at the topic level. + +Fail request of sending message when the message set the configuration of the `replication_clusters` at the message level. + +# High Level Design +A new namespace policy option `allowed_clusters` will be added. The `allowed_clusters` policy will specify the clusters where topics under this namespace can be created or loaded. The `replication_clusters` indicates the clusters that are used to create a full mesh replication for all topics under this namespace. + +When a namespace has the policy with `allowed_clusters` and `replication_clusters`, the topics under this namespace will replicate data to all `replication_clusters` by default. Additionally, the topic can have a flexible replication clusters configuration, which should be a subset of the `allowed_clusters` of the namespace. + +If `allowed_clusters` is not set, `replication_clusters` will be used as the default value for `allowed_clusters`. + +If neither `allowed_clusters` nor `replication_clusters` are set, topics under this namespace will only be able to publish/subscribe at the local cluster. The local cluster will be added in the `allowed_clusters` automatically when creating namespace. + +Message-level replication is similar to topic-level replication. The replication clusters of a message should be the subset of the `allowed_clusters`, and are the `replication_clusters` configured at the topic level or namespace level by default. + +# Detailed Design + +## Public-facing Changes + +### Public API + +#### `setNamespaceAllowedClusters` Endpoint + +This new endpoint allows setting the list of allowed clusters for a specific namespace. + +**Method:** +``` +POST +``` + +**Path:** +``` +/namespaces/{tenant}/{namespace}/allowedClusters +``` + +**HTTP Body Parameters:** + +- `clusterIds`: A list of cluster IDs. + +**Response Codes:** + +- `400 Bad Request`: The list of allowed clusters should include all replication clusters. +- `403 Forbidden`: The requester does not have admin permissions. +- `404 Not Found`: The specified tenant, cluster, or namespace does not exist. +- `409 Conflict`: A peer-cluster cannot be part of an allowed-cluster. +- `412 Precondition Failed`: The namespace is not global or the provided cluster IDs are invalid. + +**Explanation for 409 Conflict:** This follows the behavior of namespace replication clusters. As per PIP-8, a peer-cluster cannot be part of a replication-cluster. Similarly, for allowed-clusters, users could enable replication at the topic level, hence a peer-cluster cannot be part of allowed-clusters as well. + +#### `getNamespaceAllowedClusters` Endpoint + +This new endpoint allows retrieving the list of allowed clusters for a specific namespace. + +**Method:** +``` +GET +``` + +**Path:** +``` +/namespaces/{tenant}/{namespace}/allowedClusters +``` + +**Response Codes:** + +- `403 Forbidden`: The requester does not have admin permissions. +- `404 Not Found`: The specified tenant, cluster, or namespace does not exist. +- `412 Precondition Failed`: The namespace is not global. + +**Example Response:** +``` +[ + "cluster1", + "cluster2", + "cluster3" +] +``` + +### Binary protocol + +### Configuration + +### CLI + +#### `setNamespaceAllowedClusters` Command + +This new command allows you to set the list of allowed clusters for a specific namespace. + +**Usage:** + +``` +$ pulsar admin namespaces set-allowed-clusters --clusters / +``` + +**Options:** + * --clusters, -c + - A comma-separated list of cluster IDs. + +**Response Codes:** + +- `400 Bad Request`: The allowed clusters should contain all replication clusters. +- `403 Forbidden`: You do not have admin permission. +- `404 Not Found`: The tenant, cluster, or namespace does not exist. +- `409 Conflict`: A peer-cluster cannot be part of an allowed-cluster. +- `412 Precondition Failed`: The namespace is not global or the cluster IDs are invalid. + +**Explanation for 409 Conflict:** This follows the behavior of namespace replication clusters. In PIP-8, it introduced the concept of a peer-cluster, which cannot be part of a replication-cluster. For the allowed-clusters, users could enable replication at the topic level, so the peer-cluster cannot be part of the allowed-clusters too. + +#### `getNamespaceAllowedClusters` Command + +This new command allows you to retrieve the list of allowed clusters for a specific namespace. + +**Usage:** + +``` +$ pulsar admin namespaces get-allowed-clusters / +``` + +**Response Codes:** + +- `403 Forbidden`: You do not have admin permission. +- `404 Not Found`: The tenant, cluster, or namespace does not exist. +- `412 Precondition Failed`: The namespace is not global. + +**Example Response:** + +``` +"cluster1" +"cluster2" +"cluster3" +``` + +#### `CreateNamespace` Command + +Add a new option to this command to set allowed clusters when create a new namespace. + +**Usage:** + +``` +$ pulsar admin namespaces create [options] tenant/namespace +``` + +**Options:** + * --bundles, -b + - number of bundles to activate. Default: 0 + * --clusters, -c + - List of replication clusters this namespace will be assigned. (Modified*) + * --allowed-clusters, -a (New*) + - List of allowed clusters this namespace will be assigned. When the `--allowed-clusters` option is not specified, the `--clusters` option will be used as `--allowed-clusters`. + +**Response Codes:** + +- `400 Bad Request`: The specified policies is invalid. The allowed clusters should contain all replication clusters and a peer-cluster cannot be part of an allowed-clusters or replication clusters. (New*) +- `403 Forbidden`: Don't have admin permission. +- `404 Not Found`: Tenant or cluster doesn't exist +- `409 Conflict`: Namespace already exists. +- `412 Precondition Failed`: Namespace name is not valid. + +`Modified*` - This option is modified in this proposal.\ +`New*` - This option is new added in this proposal. + +### Metrics +None. + +# Monitoring +None. +# Security Considerations +If the broker enables authentication, then this configuration can only be set by the client who was authenticated. If the user does not implement their `AuthorizationProvider`, only the superuser and tenant admin is allowed to access the newly added API. + +# Backward & Forward Compatibility +The new namespace policy will not impact the behavior of existing systems. +If users do not utilize the new feature, no operation should be executed during an upgrade or revert + +## Revert +To revert, simply switch back to the old version of Pulsar. However, note that topics will be removed from those clusters that are not included in the replication clusters configured at the namespace level. +For example, replication clusters at the topic level, for topic1, is `cluster1, cluster2, cluster3`. Replication clusters at the namespace level is `cluster1, cluster2`. Allowed clusters at the namespace level is `cluster1, cluster2, cluster3`. After revert pulsar version to the old one, the topic1 will be deleted at the cluster3. + +## Upgrade +No additional operations need to be performed. The replication-clusters will be the default value of allowed-clusters. + +# Alternatives +## Approach 1 +### Changes +- Remove the limit for the system topic and then for the `change_event` topic, which is used to store the topic policy of all the topics under a namespace. +- Check the `replication_clusters` in the topic policy when performing operations such as lookup, fetchPartitionMetadata, and loadingTopic. + +### Work Flow +1. Specify the `replication_clusters` of `topic1` for `cluster1` and `cluster2`. +2. The broker receives this policy message and sets the `replication_clusters` in the metadata of the message. +3. The policy message will be replicated to `cluster2`, assuming `cluster1` is the local cluster. +4. Retrieve the topic policy from the `TopicPoliciesService` when performing operations such as lookup, fetchPartitionMetadata, and loadingTopic. + +Notes: Steps 3 and 4 cannot be replaced with manual operations by the users, as topic policies cannot be specified when the topic has not been created yet. + +### Deprecation Rationale +The `replication_clusters` specified at the message level may not be a subset of the `replication_clusters` at the topic or namespace level. For instance, `replication_clusters` specified in `topic1` could be `cluster1` and `cluster2`, and some messages could set `replication_clusters` for `cluster1` and `cluster3`. This means the topic should be loaded in `cluster3`, which is not specified in the `replication_clusters` of the topic policies or namespace policies. Moreover, messages are sent to the broker side after the topic is created, so any topic could be loaded at any clusters in the `allowed_clusters` of the tenant policy. This renders approach 1 meaningless. + +## Approach 2 +### Changes +- Following the discussion of Approach 1, we understand that checking for the `replication_clusters` of the topic policies when a topic is created or loaded is meaningless. Any topic could be loaded at any clusters in the `allowed_clusters` of the tenant policy. So, could we remove the check for the `replication_clusters` when performing operations such as lookup, fetchPartitionMetadata, or creating a topic? + +### Work Flow +1. Specify the `replication_clusters` of `topic1` for `cluster1` and `cluster2`. +2. `Topic1` could be created at `cluster1` and `cluster2`. Replication at the topic level works as expected. +3. Specify the `replication_clusters` of `message1` sent to `topic1` for `cluster1` and `cluster3`. +4. `Topic1` could be created at `cluster1` and `cluster3`. Replication at the message level works as expected. + +### Deprecation Rationale +In fact, Approach 2 has the same issue as the approach adopted in this proposal. Whether we remove the limit of the `replication_clusters` or add the `allowed_clusters` in the namespace, both are providing a feature for the topics. They allow a topic to be loaded in different clusters and the data of the topic is not replicated from these clusters. To minimize the impact, it is reasonable to introduce a more granular control of `allowed_clusters` at the namespace level. +# General Notes + +# Links + + +* Mailing List discussion thread:https://lists.apache.org/thread/87qfp8ht5s0fvw2y4t3j9yzgfmdzmcnz +* Mailing List voting thread:https://lists.apache.org/thread/grcn2mvpdhjrdtfmqd5py62pfkgcmr9m \ No newline at end of file diff --git a/pip/pip-322.md b/pip/pip-322.md new file mode 100644 index 0000000000000..6a18567ea9012 --- /dev/null +++ b/pip/pip-322.md @@ -0,0 +1,406 @@ +# PIP-322: Pulsar Rate Limiting Refactoring + +# Motivation + +The current rate limiting implementation in Apache Pulsar has several +known issues that impact performance and accuracy when operating Pulsar +clusters under load (detailed in [Problems to +Address](#problems-to-address)). This proposal outlines a refactor to +consolidate multiple existing options into a single improved solution. + +The refactor aims to resolve numerous user complaints regarding default +Pulsar rate limiting being unusable. In addition, inconsistencies and +thread contention with existing rate limiters cause unpredictable +throttling and added latency. + +Refactoring the built-in implementation will improve multi-tenancy, +allow Pulsar to scale to demanding workloads, and address longstanding +issues. + +Rate limiters act as a conduit to more extensive capacity management and +Quality of Service (QoS) controls in Pulsar. They are integral to +Pulsar's core multi-tenancy features. This refactoring will pave the way +for future enhancements. + +# Goals + +## In Scope + +- Preserve current functionality without breaking changes +- Consolidate the multiple existing rate limiting options into a single, + configurable rate limiting solution +- Remove the separate “precise” rate limiter + +### Problems to Address + +- High CPU load with default rate limiter +- High lock contention that impacts shared Netty IO threads and adds + latency to unrelated Pulsar topic producers ([[Bug] RateLimiter lock + contention when use precise publish rate limiter + #21442](https://github.com/apache/pulsar/issues/21442).) +- Multiple limiting implementations (default, precise) which + unnecessarily expose implementation details to users of Pulsar and + make the code harder to maintain and improve +- Inability to limit throughput consistently when using default rate + limiter +- Inconsistent behavior across multiple levels of throttling (broker, + namespace, connection) +- Code maintainability + - Improve understandability of code + +## Out of Scope + +- Custom/pluggable rate limiters +- Additional rate limiting features +- Cluster-wide capacity management +- Addressing the shared connection multiplexing problem where throttling + multiple independent streams multiplexed on the same connection cannot + be consistently throttled by pausing reads on the server side. + +# Current solution + +## Rate limiters in Pulsar + +In Pulsar, rate limiters are used for a few cases: + - publisher rate limiting + - topic level (configured at namespace level or topic level policy + with admin api) + - broker level (configured in broker.conf) + - resource group level (configured with resource groups admin api) + - dispatcher rate limiting + - subscribe rate limiting + - namespace bundle unloading rate limiting + +For producers ("publishers"), there are addition conditions to throttle, +besides the rate limiters: + - limiting pending publish requests per connection, configured with + `maxPendingPublishRequestsPerConnection` in broker configuration + - limiting memory for publishing messages, configured with + `maxMessagePublishBufferSizeInMB` in broker configuration + +### Current publisher rate limiters in Pulsar + +Pulsar contains two implementations for publisher rate limiters: +"default" and "precise". "precise" is the rate limiter implementation +which is used when the broker is configured with +`preciseTopicPublishRateLimiterEnable=true` in broker.conf. + +#### Default publisher rate limiter + +In this approach, a sub-second scheduler runs (configured with +`brokerPublisherThrottlingTickTimeMillis`, defaults to 50ms), iterating +every topic in the broker and checking if the topic has exceeded its +threshold. If so, it will toggle the autoread state of the connection +for the client's producer. Concurrently, a separate one-second scheduler +resets the counters and re-enables throttled connections. This method +results in inaccurate rate limiting. Additionally, this approach can +result in increased CPU usage due to the operation of two schedulers +which are constantly iterating all topics and toggling autoread states. + +#### Precise publisher rate limiter + +In this approach, the rate limit check is done on every send messages +request and thus the rate limiting is enforced more accurately. This +fixes the main issues of the default rate limiters. However, it +introduces a lock contention problem since the rate limiter +implementation extensively uses synchronous methods. Since this lock +content happens on Netty IO threads, it impacts also unrelated topics on +the same broker and causes unnecessary slowdowns as reported by bug +[#21442](https://github.com/apache/pulsar/issues/21442). + +### Publisher Throttling Approach + +In the Pulsar binary protocol, the broker's only method of applying +backpressure to the client is to pause reads and allow the buffers to +fill up. There is no explicit protocol-level, permit-based flow control +as there is for consumers. + +When the broker throttles a producer, it needs to pause reading on the +connection that the client's producer is using. This is achieved by +setting the Netty channel's autoread state to false. + +The broker cannot reject a message that is already in progress. Pausing +reading on the connection prevents the broker from receiving new +messages and this throttles publishing. When reading is paused, Netty +channel and OS-level TCP/IP buffers will fill up and eventually signal +backpressure on the TCP/IP level to the client side. + +In the current solution, when the rate limit is exceeded, the autoread +state is set to false for all producers. Similarly, when the rate falls +below the limit, the autoread state is set to true. When the +broker-level limit is exceeded or falls below the limit, all producers +in the entire broker are iterated. At the topic level, it's for all +producers for the topic. In the resource group, it's all producers part +of the resource group. + +This current solution therefore spends CPU cycles to iterate through the +producers and toggle the autoread flags. It's not necessary to eagerly +iterate all producers in a publisher rate limiter. Rate limiting can +also be achieved when producers are lazily throttled only after they +have sent a publishing send request +([CommandSend](https://pulsar.apache.org/docs/next/developing-binary-protocol/#command-send)) +to the broker. + +It's perfectly acceptable to throttle only active producers one by one +after new messages have arrived when the rate limit has been exceeded. +The end result is the same: the target rate can be kept under the limit. +The calculated rate is always an average rate over a longer period of +time. This is true in both the existing solution and the proposed +solution. Over a very short time span, the observed rate can be +extremely high. This all smoothens out when the rate calculation spans +over hundreds of milliseconds or longer periods. This is why it's +perfectly fine to throttle producers as they produce messages. The +proposed solution accounts all traffic in the rate limiter since the +state is preserved over the rate limiting period (1 second) and isn't +resetted as it is in the current solution which will miss accounting for +traffic around the boundary when the limit has exceeded, but the +connections haven't yet been paused. That's yet another reason why the +lazy approach is suitable for the new proposed solution. + +The externally observable behavior of the rate limiting is actually +better than before since it is possible to achieve fairness in a +producer throttling solution that is implemented in this approach. +Fairness is a general property that is expected in resource sharing +approaches such that each resource consumer is given a similar share of +the resource. In the current publisher rate limiting solution, there's +no way to achieve fairness when the rate limit is being exceeded. In the +proposed solution, fairness is achieved by using a queue to track which +producer is given a turn to produce while the rate limit has been +exceeded and producers are throttled. If the rate limit is again +exceeded, the producer will be put back into the queue and wait for its +turn until it can produce again. In the current solution, the producers +are iterated in the order that they appear in the broker's registry. The +producers at the beginning get more chances to produce than the ones +that are further down the list. The impact of this is the lack of +fairness. + +# High-Level Design + +The proposed refactor will refactor rate limiting internals while +preserving existing user-facing public APIs and user-facing behavior. A +token bucket algorithm will provide efficient and accurate calculations +to throttle throughput. + +Multiple built-in options such as "precise" rate limiter will be +consolidated under a single solution. Performance issues caused by +contention and CPU overhead will be addressed. + +# Detailed Design + +## Proposed Solution + +### Using an asynchronous token bucket algorithm + +Token bucket algorithms are a common industry practice for handling +traffic shaping. It is well understood and it's conceptually simple. A +token bucket is simply a counter which is limited to a maximum value, +the token bucket's capacity. New tokens are added to the bucket with the +configured rate. The usage consumes tokens from the token bucket. When +the token bucket is empty, the usage should be backpressured. In use +cases where the already accepted work cannot be rejected, the token +value needs to also go to negative values. + +Since token bucket algorithm is essentially a counter where new tokens +are added based on the time that has elapsed since the last token update, +it is possible to implement this algorithm in Java in a lockless, +non-blocking way using compare-and-swap (CAS) operations on volatile +fields. There is no need for a scheduler to add new tokens since the +amount of new tokens to add can be calculated from the elapsed time. +This assumption has already been validated in the +https://github.com/lhotari/async-tokenbucket repository. + +There's no current intention to use async-tokenbucket as a separate +library. The AsyncTokenBucket class will be placed directly in the +Pulsar code base. The reason to have async-tokenbucket repository +separately is to have more detailed performance benchmarks there and a +PoC of the high performance. + +The purpose of the proof-of-concept async-tokenbucket was to ensure that +it has an extremely low overhead which makes it feasible to calculate +the amount of tokens in an eventually consistent manner with a +configurable resolution, without a scheduler. The token bucket +operations won't become a bottleneck since on a Dell XPS 2019 i9 laptop +the benchmark showed about 900M token bucket ops/s and on MBP 2023 M3 +Max it was around 2500M token bucket ops/s. + +Internally AsyncTokenBucket uses an eventual consistent approach to +achieve high performance and low overhead. What this means is that the +token balance is updated once in every interval of the configured +resolutionNanos (16 ms default) or when an explicit update of the +balance is requested. + +There is no separate scheduled task to add new tokens to the bucket. New +tokens are calculated based on the elapsed time since the last update +and added to the current tokens balance as part of the token balance +update that happens when tokens are consumed, the throttling period is +calculated or the token balance is queried. + +For example, when tokens are consumed and the balance hasn't been +updated in the current interval, new tokens will be calculated and added +and limited by the token bucket capacity. The consumed tokens and +pending consumed tokens will be flushed and substracted from the balance +during the update. + +If there was already an update for the tokens balance in the current +internal, the consumed tokens are added to the pending consumed tokens +LongAdder counter which will get flushed in the token balance update. + +This makes the tokens balance eventually consistent. The reason for this +design choice is to optimize performance by preventing CAS loop +contention which could cause excessive CPU consumption. + +Key methods in AsyncTokenBucket: +- `consumeTokens()`: Consumes given number of tokens +- `consumeTokensAndCheckIfContainsTokens()`: Consumes given number of + tokens and checks if any tokens remain +- `containsTokens()`: Checks if any tokens remain +- `calculateThrottlingDuration()`: Computes how long throttling should + last until the token bucket contains at least 16 milliseconds worth of + tokens filled with the configured rate. + +The token balance in AsyncTokenBucket is eventually consistent and +differ from the actual token count by up to 16 milliseconds (default +resolutionNanos) worth of consumption. This is not a problem since when +the throttling finally happens, the strongly consistent value is used +for throttling period calculations and no consumed tokens are missed in +the calculations since the token value can go to negative values too. +The average rate will smoothen out to meet the target rate of the rate +limiter with this eventual consistent solution and it doesn't impact the +externally observable behavior. + +For unit tests, eventual consistent behavior can be a challenge. For +that purpose, its possible to switch the AsyncTokenBucket class to a +strongly consistent mode for unit tests by calling static +`switchToConsistentTokensView` and +`resetToDefaultEventualConsistentTokensView` methods on the class. + +One notable improvement of AsyncTokenBucket is that it is completely +non-blocking and lockless. Using AsyncTokenBucket as the basis for +publishing rate limiters in Pulsar will address a severe performance +bottleneck in Pulsar with the "precise" rate limiter. This is reported +as[[Bug] RateLimiter lock contention when use precise publish rate +limiter #21442](https://github.com/apache/pulsar/issues/21442). + +### Unifying rate limiting implementations and improving code maintainability + +In the proposed solution there's no need for separate "default" and +"precise" rate limiting options. A single implementation will be used. +This improves understandability, code quality and maintainability. + +### Fixing the inconsistency of multiple levels of throttling + +In Pulsar throttling can happen for producers in 5 different ways +simultaneously: publisher rate limiting happens at 3 levels: broker, +topic, resource group and in addition there's backpressure for limiting +number of pending publish requests and limiting memory used for +publishing. (detailed in [Rate limiters in +Pulsar](#rate-limiters-in-pulsar)). + +When there are 5 different simultaneous conditions for throttling a +connection, the connection should be throttled as long as any of these +conditions is present. In the current code base, this handling is prone +to errors and overly complex. There are also cases where one rate +limiter sets the autoread to false and another immediately sets it to +true although the connection should remain throttled as long as one of +the conditions exists. + +The fix for this issue is in the proposal by introducing a new concept +ServerCnxThrottleTracker, which will track the "throttle count". When a +throttling condition is present, the throttle count is increased and +when it's no more present, the count is decreased. The autoread should +be switched to false when the counter value goes from 0 to 1 and only +when it goes back from 1 to 0 should it set to true again. The autoread +flag is no more controlled directly from the rate limiters. Rate +limiters are only responsible for their part and it's +ServerCnxThrottleTracker that decides when autoread flag is toggled. + +### Integrating AsyncTokenBucket with the refactored PublishRateLimiterImpl + +In the refactored PublishRateLimiterImpl, there's a AsyncTokenBucket +instance for the message rate limit and for the bytes rate limit. When +the publish operation starts in the broker, it will call the topic's +incrementPublishCount method and pass the reference to the producer that +is starting the operation, in addition to the number of messages and the +total bytes size of the send request (CommandSend message). + +This delegates to a call for all possibly active rate limiters in the topic, +at the broker level, at resource group level and at topic level. + +For each rate limiter, the PublishRateLimiterImpl's handlePublishThrottling +method will be called which also gets the producer reference and the number of message +and total bytes size as input. + +The rate limiter instance could contain both a message limit and a bytes limit. +It will call AsyncTokenBucket's consumeTokensAndCheckIfContainsTokens method +for each instance. If either call returns false, it means that the +producer that produced the message should be throttled. + +Throttling is handled by calling producer's incrementThrottleCount method +which will be delegated producer's connection's ServerCnxThrottleTracker's +incrementThrottleCount method which was described in the previous section. + +The contract of the incrementThrottleCount method is that decrementThrottleCount +method should be called when the throttling is no longer needed from an +individual PublishRateLimiterImpl instance's perspective. + +This is handled by first adding the throttled producer to a queue. +A task will be scheduled to handle unthrottling from the queue after the +throttling duration which is calculated by calling AsyncTokenBucket's +calculateThrottlingDuration method. This task will only be scheduled +unless there's an already scheduled task in progress. + +When the unthrottling task runs, it will process the unthrottling queue +and keep on unthrottling producers while there are available tokens in the +token buckets. If the queue isn't empty, it will repeat the cycle by +scheduling a new task after the throttling duration calculated with +the calculateThrottlingDuration method. This happens until the queue is +empty and will start again if more producers are throttled. + +The producer's connection will get throttled by setting autoread to +false ServerCnxThrottleTracker. The PublishRateLimiterImpl instances +don't have to know whether the connection was already throttled due to +another effective rate limit being over the limit. +ServerCnxThrottleTracker will also handle setting autoread to true once +all rate limiters operating on the same connection have unthrottled the +producer by calling decrementThrottleCount. + +### Preserve Public Contracts + +- Avoid breaking existing configs, APIs or client functionality. +- Handle through internal refactoring. + +## Public-facing Changes + +There are no changes to existing configs, CLI options, monitoring etc. +This PIP is about a large change which includes a major refactoring and +multiple improvements and bug fixes to rate limiting. + +## More Detailed Level Design + +Please refer directly to [the pull request with the proposed +changes](https://github.com/apache/pulsar/pull/21681) for the more +detailed level changes. + +The implementation level detail questions can be handled in the pull +request review. The goal is to document low level details directly in +the Javadoc and comments so that it serves the code maintainers also in +the future. + +# Links + + +* Mailing List discussion thread: + https://lists.apache.org/thread/xzrp2ypggp1oql437tvmkqgfw2b4ft33 +* Mailing List voting thread: + https://lists.apache.org/thread/bbfncm0hdpx42hrj0b2xnzb5oqm1pwyl +* Proposed changes for Pulsar Rate limiting refactoring: + https://github.com/apache/pulsar/pull/21681 + +* [Pulsar Community Meeting minutes + 2023/11/23](https://lists.apache.org/thread/y1sqpyv37fo0k4bm1ox28wggvkb7pbtw) +* [Blog post: Apache Pulsar service level objectives and rate + limiting](https://codingthestreams.com/pulsar/2023/11/22/pulsar-slos-and-rate-limiting.html) +* Proof-of-concept asynchronous token bucket implementation: + https://github.com/lhotari/async-tokenbucket \ No newline at end of file diff --git a/pip/pip-323.md b/pip/pip-323.md new file mode 100644 index 0000000000000..dc607fff3d58c --- /dev/null +++ b/pip/pip-323.md @@ -0,0 +1,171 @@ +# PIP-323: Complete Backlog Quota Telemetry + +# Background knowledge + +## Backlog + +A topic in Pulsar is the place where messages are written to. They are consumed by subscriptions. A topic can have many +subscriptions, and it is those that maintains the state of message acknowledgment, per subscription - which messages +were acknowledged and which were not. + +A subscription backlog is the set of unacknowledged messages in that subscription. +A subscription backlog size is the sum of the size of the unacknowledged messages (in bytes).. + +Since a topic can have many subscriptions, and each has its own backlog, how does one define a backlog for a topic? +A topic backlog is defined as the backlog of the subscription which has the **oldest** unacknowledged message. +Since acknowledged messages can be interleaved with unacknowledged messages, calculating the exact size of that +subscription backlog can be expensive as it requires I/O operations to read the messages from the ledgers. +For that reason, the topic backlog size is actually defined to be the *estimated* backlog size of that subscription. +It does so by summarizing the size of all the ledgers, starting from the current active one (the one being written to), +up to the ledger which contains the oldest unacknowledged message for that subscription (There is actually a faster +way to calculate it, but this was the definition chosen for this estimation in Pulsar). + +A topic backlog age is the age of the oldest unacknowledged message (same subscription as defined for topic backlog size). +If that message was written 30 minutes ago, its age is 30 minutes, and so is the topic backlog age. + +## Backlog Quota + +Pulsar has a feature called [backlog quota](https://pulsar.apache.org/docs/3.1.x/cookbooks-retention-expiry/#backlog-quotas). +It allows a user to define a quota - in effect, a limit - which limits the topic backlog. +There are two types of quotas: + +1. Size based: The limit is for the topic backlog size (as we defined above). +2. Time based: The limit is for the topic backlog age (as we defined above). + +Once a topic backlog exceeds either one of those limits, an action is taken to hold the backlog to that limit: + +* The producer write is placed on hold for a certain amount of time before failing. +* The producer write is failed +* The subscriptions oldest unacknowledged messages will be acknowledged in-order until both the topic backlog size or + age will fall inside the limit (quota). The process is called backlog eviction (happens every interval). + +The quotas can be defined as a default value for any topic, by using the following broker configuration keys: +`backlogQuotaDefaultLimitBytes` and `backlogQuotaDefaultLimitSecond`. + +The quota can also be specified directly for all topics in a given namespace using the namespace policy, +or a specific topic using a topic policy. + +## Monitoring Backlog Quota + +The user today can calculate quota used for size based limit, since there are two metrics exposed today on +a topic level: `pulsar_storage_backlog_quota_limit` and `pulsar_storage_backlog_size`. +You can just divide the two to get a percentage and know how close the topic backlog to its size limit. + +For the time-based limit, the only metric exposed today is the quota itself - `pulsar_storage_backlog_quota_limit_time` + +## Backlog Quota Eviction in the Broker + +The broker has a method called `BrokerService.monitorBacklogQuota()`. It is scheduled to run every x seconds, +as defined by the configuration `backlogQuotaCheckIntervalInSeconds`. +This method loops over all persistent topics, and for each topic is checks whether the topic backlog exceeded +either one of those topics. + +As mentioned before, checking backlog size is a memory-only calculation, since +each topic has the list of ledgers stored in-memory, including the size of each ledger. Same goes for the subscriptions, +they are all stored in memory, and the `ManagedCursor` keeps track of the subscription with the oldest unacknowledged +message, thus retrieveing it is O(1). Checking backlog based on time is costly if configuration key +`preciseTimeBasedBacklogQuotaCheck` was set to true. In that case, it needs to read the oldest message to obtain +its public timestamp, which is expensive in terms of I/O. If it was set to false, it's in-memory access only, since +it uses the age of the ledger instead of the message, and the ledgers metadata is kept in memory. + +For each topic which has exceeded its quota, if the policy chosen is eviction, then the process it performed +synchronously. This process consumes I/O, as it needs read messages (using skip) to know where to stop acknowledging +messages. + + +# Motivation + +Users which have defined backlog quota based on time, have no means today to monitor the backlog quota usage, +time-wise, to know whether the topic backlog is close to its time limit or even passed it. + +If it has passed it, the user has no means to know if it happened, when and how many times. + + +# Goals + +## In Scope +- Allow the user to know the backlog quota usage for time-based quota, per topic +- Allow the user to know how many times backlog eviction happened, and for which backlog quota type + +## Out of Scope + +None + + +# High Level Design + +We'll use the existing backlog monitoring process running in intervals. For each topic, the subscription with +the oldest unacknowledged message is retrieved, to calculate the topic backlog age. At that point, we will +cache the following for the oldest unacknowledged message: +* Subscription name +* Message position +* Message publish timestamp + +That cache will allow us to add a metric exposing the topic backlog age - `pulsar_storage_backlog_age_seconds`, +which will be both consistent (same ones used for deciding on backlog eviction) and cheap to retrieve +(no additional I/O involved). +Coupled with the existing `pulsar_storage_backlog_quota_limit_time` metric, the user can use both to divide and +get the usage of the quota (both are in seconds units). + +We will add the subscription name containing the oldest unacknowledged message to the Admin API +topic stats endpoints (`{tenant}/{namespace}/{topic}/stats` and `{tenant}/{namespace}/{topic}/partitioned-stats`), +allowing the user a complete workflow: alert using metrics when topic backlog is about to be exceeded, then +query topic stats for that topic to retrieve the subscription name which contains the oldest message. +For completeness, we will also add the backlog quota limits, both age and size, and the age of oldest +unacknowledged message. + +We will add a metric allowing the user to know how many times the usage exceeded the quota, both for time or size - +`pulsar_storage_backlog_quota_exceeded_evictions_total`, where the `quota_type` label will be either `time` or +`size`. Monitoring that counter over time will allow the user to know when a topic backlog exceeded its quota, +and if backlog eviction was chosen as action, then it happened, and how many times. + +Some users may want the backlog quota check to happen more frequently, and as a consequence, the backlog age +metric more frequently updated. They can modify `backlogQuotaCheckIntervalInSeconds` configuration key, but without +knowing how long this check takes, it will be hard for them. Hence, we will add the metric +`pulsar_storage_backlog_quota_check_duration_seconds` which will be of histogram type. + +# Detailed Design + +## Public-facing Changes + +### Public API +Adding the following to the response of topic stats, of both `{tenant}/{namespace}/{topic}/stats` +and `{tenant}/{namespace}/{topic}/partitioned-stats`: + +* `backlogQuotaLimitSize` - the size in bytes of the topic backlog quota +* `backlogQuotaLimitTime` - the topic backlog age quota, in seconds. +* `oldestBacklogMessageAgeSeconds` - the age of the oldest unacknowledged (i.e. backlog) message, measured by + the time elapsed from its published time, in seconds. This value is recorded every backlog quota check + interval, hence it represents the value seen in the last check. +* `oldestBacklogMessageSubscriptionName` - the name of the subscription containing the oldest unacknowledged message. + This value is recorded every backlog quota check interval, hence it represents the value seen in the last check. + + +### Metrics + +| Name | Description | Attributes | Units | +|----------------------------------------------------------------|-----------------------------------------------------------------------------------------------------|--------------------------------------------------------|---------| +| `pulsar_storage_backlog_age_seconds` | Gauge. The age of the oldest unacknowledged message (backlog) | cluster, namespace, topic | seconds | +| `pulsar_storage_backlog_quota_exceeded_evictions_total` | Counter. The number of times a backlog was evicted since it has exceeded its quota | cluster, namespace, topic, quota_type = (time \| size) | | +| `pulsar_storage_backlog_quota_check_duration_seconds` | Histogram. The duration of the backlog quota check process. | cluster | seconds | +| `pulsar_broker_storage_backlog_quota_exceeded_evictions_total` | Counter. The number of times a backlog was evicted since it has exceeded its quota, in broker level | cluster, quota_type = (time \| size) | | + +* Since `pulsar_storage_backlog_age_seconds` can not be aggregated, with proper meaning, to a namespace-level, it will + not be included as a metric when configuration key `exposeTopicLevelMetricsInPrometheus` is set to false. +* `pulsar_storage_backlog_quota_exceeded_evictions_total` will be included as a metric also in namespace aggregation. + +# Alternatives + +One alternative is to separate the backlog quota check into 2 separate processes, running in their own frequency: +1. Check backlog quota exceeded for all persistent topics. The result will be marked in memory. + If precise time backlog quota was configured then this will the I/O cost as described before. +2. Evict messages for those topics marked. + +This *may* enable more frequent updates to the backlog age metric making it more fresh, but the cost associated with it +might be high, since it might result in more frequent I/O calls, especially with many topics. +Another disadvantage is that it makes the backlog check and eviction more complex. + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/xv33xjjzc3t2n06ynz2gmcd4s06ckrqh +* Mailing List voting thread: https://lists.apache.org/thread/x2ypnft3x5jdyyxbwgvzxgcw20o44vps diff --git a/pip/pip-324-Alpine Docker images.md b/pip/pip-324-Alpine Docker images.md new file mode 100644 index 0000000000000..c7fcc1903a93d --- /dev/null +++ b/pip/pip-324-Alpine Docker images.md @@ -0,0 +1,145 @@ +# PIP-324: Switch to Alpine Linux base Docker images + + +# Motivation + +Pulsar Docker images are currently based on Ubuntu base images. While these images has served us well in the past years, +there are few shortcomings. + +Alpine Linux is a Linux distribution designed explicitely to work well in container environments and has strong +focus on security and a minimalistic set of included depedendencies. + +### Size of the image + +Ubuntu images come with a much larger set of pre-installed tools. In many cases these are not actually needed by Pulsar, +and it's better not include anything in the container images unless it's strictly required. + +Example of minimal image size: +``` +$ docker images | egrep 'ubuntu|alpine' +alpine 3.19 1dc785547989 4 days ago 7.73MB +ubuntu 22.04 031631b93326 11 days ago 69.3MB +``` + + +Similarly, also the packaged that can be installed in Alpine are generally much smaller than the corresponding Ubuntu +packages. In a complex image like the Pulsar one, this quickly adds up to hundreds of MBs. + +Comparison between the 2 base images with only the Java runtime added (JRE): + +``` +alpine-jre latest eb0e093ee71c 29 seconds ago 211MB +ubuntu-jre latest 4147e1b2c6d1 7 seconds ago 377MB +``` + +Size of Docker images is very important, because these images end up being stored in many registries and downloaded +a million of times, bringing a concern in costs for network transfer as well as for storage. Additionally, in many cases +how fast is an image to download will determine the time it takes to spin up a new container in a new virtual machines +(eg: when scaling a cluster up in response to a traffic increase). + +### Security posture + +By starting with a minimal set of pre-installed tools, Alpine reduces the surface for security issues in the base image. + +At this moment there are 12 Medium/Low CVEs opened in Ubuntu for which there is no resolution available. Some of these +CVEs have been opened for many months. +Even though these CVEs don't look particularly dangerous and might not apply in 100% of cases to the Pulsar deployment, +they will still be flagged in every security review, and they will trigger an in-depth investigation and require ad-hoc +approvals. + +At the same time, there are 0 CVEs in the Alpine image. + +``` +~ docker scout quickview ubuntu:22.04 + ! New version 1.2.2 available (installed version is 1.0.9) at https://github.com/docker/scout-cli + ✓ SBOM of image already cached, 143 packages indexed + + Target │ ubuntu:22.04 │ 0C 0H 2M 10L + digest │ 031631b93326 │ +``` + +``` +~ docker scout quickview alpine:3.19.0 + ! New version 1.2.2 available (installed version is 1.0.9) at https://github.com/docker/scout-cli + ✓ SBOM of image already cached, 19 packages indexed + + Target │ alpine:3.19.0 │ 0C 0H 0M 0L + digest │ 1dc785547989 │ +``` + +# Goals + +## In Scope + +Convert the tooling that produces the Pulsar Docker image to use Alpine as the + +## Out of Scope + +As part of this PIP there will be no explicit work to reduce the size of the Docker image, other than the conversion +of the base image. This could be done as part of further initiatives. + +# High Level Design + +The base of `apachepulsar/pulsar` will be converted to use Alpine Linux base image. All the other images that are part +of the Pulsar projects will be updated to make sure they can work correctly (eg: use `apk add` instead of `apt install`). + +Release notes for Pulsar 3.X.0 release will include note to notify downstream users, who might be doing some advanced +customizations to the official Apache Pulsar images. This should be a tiny minority of users though. In most cases, +users will see no visible change, and will not have to perform any extra step of configuration change during the upgrade +from an Ubuntu based image to an Alpine based image. + +# Detailed Design + +## Public-facing Changes + +### Public API + +No changes + +### Binary protocol + +No changes + +### Configuration + +No changes + +### CLI + +No changes + +### Metrics + +No changes + +# Monitoring + +No changes + +# Security Considerations + + +# Backward & Forward Compatibility + +## Revert + +No compatibility problems. + +## Upgrade + +No difference from a regular upgrade. + + +# Links + + +* Mailing List discussion thread: +* Mailing List voting thread: diff --git a/pip/pip-325.md b/pip/pip-325.md new file mode 100644 index 0000000000000..44d1aebc6a4eb --- /dev/null +++ b/pip/pip-325.md @@ -0,0 +1,94 @@ +# Background knowledge + +In the current implementation of Pulsar Transaction, Topics ensure that consumers do not read messages belonging +to uncommitted transactions through the Transaction Buffer. Within the Transaction Buffer, a Position (`maxReadPosition`) +is maintained, as well as a set of aborted transactions (`aborts`). The `maxReadPosition` controls the maximum message +position that the broker can read, and it is adjusted to the position just before the first message of the first ongoing +transaction when a transaction is committed or aborted. Before distributing messages to consumers, the broker filters out +messages that belong to already aborted transactions using the `aborts` set. + +# Motivation +If we have a stuck transaction, then the transactions after this one cannot be consumed by the consumer +even if they have been committed. The consumer will be stuck until the stuck transaction is aborted due to timeout, +and then it will continue to consume messages. Therefore, we need to add a command to allow cluster administrators +to proactively abort transaction. + +# Goals + +## In Scope + +Introduce a new API for aborting transactions, allowing administrators to proactively abort transaction. + +## Out of Scope + +None. + + +# High Level Design + +Introduce a new API for aborting transactions, allowing administrators to proactively abort transaction. + +# Detailed Design + +## Design & Implementation Details + +Introduce a new API for aborting transactions, allowing administrators to proactively abort transaction. + +## Public-facing Changes + +### Public API +Add a new API to abort transaction: +``` + /** + * Abort a transaction. + * + * @param txnID the txnId + */ + void abortTransaction(TxnID txnID) throws PulsarAdminException; + + /** + * Asynchronously Abort a transaction. + * + * @param txnID the txnId + */ + CompletableFuture abortTransactionAsync(TxnID txnID); +``` +``` +admin.transactions().abortTransaction(txnID); +``` + +### Binary protocol + +### Configuration + +### CLI +Add a command to abort transaction: +``` +pulsar-admin transactions abort-transaction --most-sig-bits 1 --least-sig-bits 2 +``` +### Metrics +None. + +# Monitoring +None. + +# Security Considerations +The transaction owner and super user can access the admin API to abort the transaction. + +# Backward & Forward Compatibility + +## Revert + +## Upgrade + +# Alternatives + +# General Notes + +# Links + + +* Mailing List discussion thread: https://lists.apache.org/thread/ssgngyrlgx36zvygvsd5b2dm5q6krn0f +* Mailing List voting thread: https://lists.apache.org/thread/kp9w4d8drngomx1mdof0203ybgfmvtty diff --git a/pip/pip-326.md b/pip/pip-326.md new file mode 100644 index 0000000000000..26456c4d8dd1f --- /dev/null +++ b/pip/pip-326.md @@ -0,0 +1,209 @@ + + +# PIP-326: Create a BOM to ease dependency management + +# Background knowledge + +A `Bill of Materials` (BOM) is a special kind of POM that is used to control the versions of a project’s dependencies and provide a central place to define and update those versions. +A BOM dependency ensure that all dependencies (both direct and transitive) are at the same version specified in the BOM. + +To illustrate, consider the [Spring Data BOM](https://github.com/spring-projects/spring-data-bom/blob/main/bom/pom.xml) which declares the version for each of the published Spring Data modules. +Without a BOM, consuming applications must specify the version on each of the imported Spring Data module dependencies. +However, when using a BOM the version numbers can be omitted. + +# Motivation + +The BOM provides the following benefits for consuming applications: +1. Reduce burden by not having to specify the version in multiple locations +2. Reduce chance of version mismatch (and therefore errors) + +The **burden** and **chance** of version mismatch is **directly** proportional to the number of modules published by a project. +Pulsar publishes **many (29)** modules and therefore consuming applications are likely to run into the above issues. + +A concrete example of the above symptoms can be found in the [Spring Boot BOM](https://github.com/spring-projects/spring-boot/blob/92a4a1194d7a599cb57b5e5169ee5bbbfce637d8/spring-boot-project/spring-boot-dependencies/build.gradle#L1140-L1215) which provides a section for the list of Pulsar module dependencies as follows: + +```groovy +library("Pulsar", "3.1.1") { + group("org.apache.pulsar") { + modules = [ + "bouncy-castle-bc", + "bouncy-castle-bcfips", + "pulsar-client-1x-base", + "pulsar-client-1x", + "pulsar-client-2x-shaded", + "pulsar-client-admin-api", + "pulsar-client-admin-original", + "pulsar-client-admin", + "pulsar-client-all", + "pulsar-client-api", + "pulsar-client-auth-athenz", + "pulsar-client-auth-sasl", + "pulsar-client-messagecrypto-bc", + "pulsar-client-original", + "pulsar-client-tools-api", + "pulsar-client-tools", + "pulsar-client", + "pulsar-common", + "pulsar-config-validation", + "pulsar-functions-api", + "pulsar-functions-proto", + "pulsar-functions-utils", + "pulsar-io-aerospike", + "pulsar-io-alluxio", + "pulsar-io-aws", + "pulsar-io-batch-data-generator", + "pulsar-io-batch-discovery-triggerers", + "pulsar-io-canal", + "pulsar-io-cassandra", + "pulsar-io-common", + "pulsar-io-core", + "pulsar-io-data-generator", + "pulsar-io-debezium-core", + "pulsar-io-debezium-mongodb", + "pulsar-io-debezium-mssql", + "pulsar-io-debezium-mysql", + "pulsar-io-debezium-oracle", + "pulsar-io-debezium-postgres", + "pulsar-io-debezium", + "pulsar-io-dynamodb", + "pulsar-io-elastic-search", + "pulsar-io-file", + "pulsar-io-flume", + "pulsar-io-hbase", + "pulsar-io-hdfs2", + "pulsar-io-hdfs3", + "pulsar-io-http", + "pulsar-io-influxdb", + "pulsar-io-jdbc-clickhouse", + "pulsar-io-jdbc-core", + "pulsar-io-jdbc-mariadb", + "pulsar-io-jdbc-openmldb", + "pulsar-io-jdbc-postgres", + "pulsar-io-jdbc-sqlite", + "pulsar-io-jdbc", + "pulsar-io-kafka-connect-adaptor-nar", + "pulsar-io-kafka-connect-adaptor", + "pulsar-io-kafka", + "pulsar-io-kinesis", + "pulsar-io-mongo", + "pulsar-io-netty", + "pulsar-io-nsq", + "pulsar-io-rabbitmq", + "pulsar-io-redis", + "pulsar-io-solr", + "pulsar-io-twitter", + "pulsar-io", + "pulsar-metadata", + "pulsar-presto-connector-original", + "pulsar-presto-connector", + "pulsar-sql", + "pulsar-transaction-common", + "pulsar-websocket" + ] + } +} +``` +The problem with this hardcoded approach is that the Spring Boot team is not the expert of Pulsar and this list of modules could become stale and/or invalid rather easily. +A better suitor for this specification is the Pulsar team, the subject-matter-experts who know exactly what is going on with Pulsar (which modules are available and what those version(s) should be). + +If there were a Pulsar BOM, the above Spring Boot dependency section would shrink down to the following: +```groovy +library("Pulsar", "3.1.1") { + group("org.apache.pulsar") { + imports = [ + "pulsar-bom" + ] + } +} +``` + +It is worth noting that This is an industry best practice and more often than not, a library provides a BOM. A handful of examples can be found in the "Links" section at the bottom of this document. + +# Goals +Provide a Pulsar BOM in order to solve the issues listed in the motivation section. + +## In Scope +The intention is to create a single BOM for all published Pulsar modules. +The benefit goes to consumers of the project (our users) as described in the motivation. + +## Out of Scope +This proposal is not attempting to create various BOMs that are tailored to specific usecases. + + +# High Level Design +1. From a build target, generate a list of published Pulsar modules +2. From the list of modules, generate a BOM Maven POM file +3. Publish the BOM artifact as any other Pulsar module is published + +# Detailed Design + +Leaving the detailed design out of the PIP for now. +There is a working prototype and more details will be revealed when and if the PIP is approved. + +## Public-facing Changes +NA (new addition) + + +### Public API +The only public "API" is the newly published POM artifact. + +### Binary protocol +NA + +### Configuration +NA + +### CLI +NA + +### Metrics +NA + +# Monitoring +NA + +# Security Considerations +NA + +# Backward & Forward Compatibility +NA + +## Revert +1. Deprecate the POM module in version `m.n.p`. +2. Stop producing subsequent POM modules in version `m.n+1.0` + +## Upgrade +NA + +# Alternatives +Continue on as-is, not publishing a BOM. + +# General Notes + +# Links + +### Example OSS projects with BOMs +* [Spring Boot](https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-dependencies) +* [Quarkus](https://mvnrepository.com/artifact/io.quarkus/quarkus-bom) +* [MongoDB](https://mvnrepository.com/artifact/io.mongock/mongock-driver-mongodb-bom) +* [AWS SDK](https://aws.amazon.com/blogs/developer/managing-dependencies-with-aws-sdk-for-java-bill-of-materials-module-bom) +* [Junit](https://mvnrepository.com/artifact/org.junit/junit-bom) +* [Mockito](https://mvnrepository.com/artifact/org.mockito/mockito-bom) +* [Jackson](https://github.com/FasterXML/jackson-bom) + +### Threads +* Mailing List discussion thread: https://lists.apache.org/thread/h385452o69b54m7j2zkjxrnwwx771jhr +* Mailing List voting thread: https://lists.apache.org/thread/9xchhq88cn1n1vmxvk0zlvq8037cmt87 diff --git a/pip/pip-327.md b/pip/pip-327.md new file mode 100644 index 0000000000000..00b3de424f1dc --- /dev/null +++ b/pip/pip-327.md @@ -0,0 +1,42 @@ +# PIP-327: Support force topic loading for unrecoverable errors + +# Motivation + +As discussed in Issue: https://github.com/apache/pulsar/issues/21751 + +We have introduced a configuration called `autoSkipNonRecoverableData` before open-sourcing Pulsar as we have come across with various situations when it was not possible to recover ledgers belonging to managed-ledger or managed-cursors and the broker was not able to load the topics. In such situations,`autoSkipNonRecoverableData` flag helps to skip non-recoverable leger-recovery errors such as ledger_not_found and allows the broker to load topics by skipping such ledgers in disaster recovery. + +Brokers can recognize such non-recoverable errors using bookkeeper error codes but in some cases, it’s very tricky and not possible to conclude non-recoverable errors. For example, the broker can not differentiate between all the ensemble bookies of the ledgers that are temporarily unavailable or are permanently removed from the cluster without graceful recovery, and because of that broker doesn’t consider all the bookies deleted as a non-recoverable error though we can not recover ledgers in such situations where all the bookies are removed due to various reasons such as Dev cluster clean up or system faced data disaster with multiple bookie loss. In such situations, the system admin has to manually identify such non-recoverable topics and update those topics’ managed-ledger and managed-cursor’s metadata and reload topics again which requires a lot of manual effort and sometimes it might not be feasible to handle such situations with a large number of topics that require this manual procedure to fix those topics. + +Therefore, the system admin should have a dynamic configuration called `managedLedgerForceRecovery` to use in such situations to allow brokers to forcefully load topics by skipping ledger failures to avoid topic unavailability and perform auto repairs of the topics. This will allow the admin to handle disaster recovery situations in a controlled and automated manner and maintain the topic availability by mitigating such failures. + + + +# Goals + +Support force topic loading and recovery for unrecoverable situation where broker can skip unrecoverable with uncertain bookkeeper error codes. + + +## Design & Implementation Details + +### (1) Broker Changes + +Broker will have new configuration `managedLedgerForceRecovery` and if this flag is enabled then managed ledger will ignore any kind of failure if broker see's while recovering managed-ledger or managed-cursor. + +# Security Considerations + + + +# General Notes + +# Links + +Issue: https://github.com/apache/pulsar/issues/21751 +Discuss thread: https://lists.apache.org/thread/w7w91xztdyy07otw0dh71nl2rn3yy45p +Vote thread: https://lists.apache.org/thread/hh9t6nz0pqjo7tbfn12nbwtylrvq4f43 diff --git a/pip/pip-329.md b/pip/pip-329.md new file mode 100644 index 0000000000000..46727701d8b0f --- /dev/null +++ b/pip/pip-329.md @@ -0,0 +1,70 @@ + + +# PIP-329: Strategy for maintaining the latest tag to Pulsar docker images + +# Motivation + +There is a gap in our current release process concerning the +pushing of the latest tag to our Docker images. Specifically, we need +to decide which version of Pulsar the latest tag should point to. + +We've had initial agreement from previous discussions, found +here: https://lists.apache.org/thread/h4m90ff7dgx0110onctf5ntq0ktydzv1. + +Now, we need to formally propose a PIP to address this. + +# Goals + +## In Scope + +- Define the strategy for maintaining the latest tag to Pulsar docker images in the release process + +## Out of Scope + +- None + +# High Level Design + +Refine the release process to clearly demonstrate the strategy for managing the 'latest' tag for Pulsar Docker images: + +The 'latest' tag should be pointed to the most recent feature release or any subsequent patch of that feature +release. For instance, if the most recent feature release is version 3.1, and it has been updated with patches, the ' +latest' tag could point to version 3.1.2, assuming this is the latest patch for the 3.1 feature. Alternatively, if a new +feature release, say 3.2.0, is introduced, the 'latest' tag would then point to this new version. + +In simpler terms, the `latest` tag will always point to the newest version of a feature. + +# Alternatives + +An alternative strategy is + +> The latest tag could point to the most recent feature release or +> the subsequent patch of that feature release. For instance, it could +> currently point to 3.1.1, and in the future, it could point to 3.1.2 +> or 3.2.0. + +Feedback from the community indicates a preference for the solution proposed by this PIP. + +# General Notes + +- Discussion + of `Strategy for pushing the latest tag to Pulsar docker images`: https://lists.apache.org/thread/h4m90ff7dgx0110onctf5ntq0ktydzv1 +- The implementation PR for this PIP: https://github.com/apache/pulsar-site/pull/745 + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/x7r1f2vgmowykwdcb3mmrv0d8lj4y1t9 +* Mailing List voting thread: https://lists.apache.org/thread/f9j0xjjlyz54880zyzon3xm5y0zn37xb diff --git a/pip/pip-330.md b/pip/pip-330.md new file mode 100644 index 0000000000000..8032940264348 --- /dev/null +++ b/pip/pip-330.md @@ -0,0 +1,54 @@ +# PIP-330: getMessagesById gets all messages + +# Motivation + +The `org.apache.pulsar.client.admin.Topics` provides `getMessageById(java.lang.String, long, long)` method to get the +message, which returns one message. If the message id refers to a batch message, we can only get the first message, not +all messages. + +This behavior affects our analysis of messages by the message id. + +# Goals + +## In Scope + +Add a method that returns all messages by message id to the `org.apache.pulsar.client.admin.Topics` interface. + +# Detailed Design + +## Design & Implementation Details + +1. Add a set of methods to the `org.apache.pulsar.client.admin.Topics` interface: + +```java +public interface Topics { + List> getMessagesById(String topic, long ledgerId, long entryId) throws PulsarAdminException; + CompletableFuture>> getMessagesByIdAsync(String topic, long ledgerId, long entryId); +} +``` + +2. Deprecate the following methods in the `org.apache.pulsar.client.admin.Topics` interface: +```java +public interface Topics { + /** + * @deprecated Use {@link #getMessagesById(String, long, long)} instead. + */ + @Deprecated + Message getMessageById(String topic, long ledgerId, long entryId) throws PulsarAdminException; + + /** + * @deprecated Use {@link #getMessagesByIdAsync(String, long, long)} instead. + */ + @Deprecated + CompletableFuture> getMessageByIdAsync(String topic, long ledgerId, long entryId); +} +``` + +# General Notes + +This PIP doesn't change the output of `bin/pulsar-admin topics get-message-by-id`, which still outputs one message. + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/vqyh3mvtvovd383sd8zxnlzsspdr863z +* Mailing List voting thread: https://lists.apache.org/thread/n1f91v46tct6o5o72pd53hcyvr9xx9qr diff --git a/pip/pip-335 Oxia metadata plugin.md b/pip/pip-335 Oxia metadata plugin.md new file mode 100644 index 0000000000000..fdff8a85593a2 --- /dev/null +++ b/pip/pip-335 Oxia metadata plugin.md @@ -0,0 +1,51 @@ +# PIP-335: Support Oxia metadata store plugin + +# Motivation + +Oxia is a scalable metadata store and coordination system that can be used as the core infrastructure +to build large scale distributed systems. + +Oxia was created with the primary goal of providing an alternative Pulsar to replace ZooKeeper as the +long term preferred metadata store, overcoming all the current limitations in terms of metadata +access throughput and data set size. + +# Goals + +Add a Pulsar MetadataStore plugin that uses Oxia client SDK. + +Users will be able to start a Pulsar cluster using just Oxia, without any ZooKeeper involved. + +## Not in Scope + +It's not in the scope of this proposal to change any default behavior or configuration of Pulsar. + +# Detailed Design + +## Design & Implementation Details + +Oxia semantics and client SDK were already designed with Pulsar and MetadataStore plugin API in mind, so +there is not much integration work that needs to be done here. + +Just few notes: + 1. Oxia client already provides support for transparent batching of read and write operations, + so there will be no use of the batching logic in `AbstractBatchedMetadataStore` + 2. Oxia does not treat keys as a walkable file-system like interface, with directories and files. Instead + all the keys are independent. Though Oxia sorting of keys is aware of '/' and provides efficient key + range scanning operations to identify the first level children of a given key + 3. Oxia, unlike ZooKeeper, doesn't require the parent path of a key to exist. eg: we can create `/a/b/c` key + without `/a/b` and `/a` existing. + In the Pulsar integration for Oxia we're forcing to create all parent keys when they are not there. This + is due to several places in BookKeeper access where it does not create the parent keys, though it will + later make `getChildren()` operations on the parents. + +## Other notes + +Unlike in the ZooKeeper implementation, the notification of events is guaranteed in Oxia, because the Oxia +client SDK will use the transaction offset after server reconnections and session restarted events. This +will ensure that brokers cache will always be properly invalidated. We will then be able to remove the +current 5minutes automatic cache refresh which is in place to prevent the ZooKeeper missed watch issue. + +# Links + +Oxia: https://github.com/streamnative/oxia +Oxia Java Client SDK: https://github.com/streamnative/oxia-java diff --git a/pip/pip-337.md b/pip/pip-337.md new file mode 100644 index 0000000000000..283bb9710de84 --- /dev/null +++ b/pip/pip-337.md @@ -0,0 +1,382 @@ +# PIP-337: SSL Factory Plugin to customize SSLContext/SSLEngine generation + +# Background knowledge +Apache Pulsar supports TLS encrypted communication between the clients and servers. The TLS encryption setup requires +loading the TLS certificates and its respective passwords to generate the SSL Context. Pulsar supports loading these +certificates and passwords via the filesystem. It supports both Java based Keystores/Truststores and TLS information in +".crt", ".pem" & ".key" formats. This information is refreshed based on a configurable interval. + +Apache Pulsar internally uses 3 different frameworks for connection management: + +- Netty: Connection management for Pulsar server and client that understands Pulsar binary protocol. +- Jetty: HTTP Server creation for Pulsar Admin and websocket. Jetty Client is used by proxy for admin client calls. +- AsyncHttpClient: HTTP Client creation for Admin client and HTTP Lookup + +Each of the above frameworks supports customizing the generation of the SSL Context and SSL Engine. Currently, Pulsar +uses these features to feed the SSL Context via its internal security tools after loading the file based certificates. +One of the issues of using these features is that pulsar tries to bootstrap the SSL Context in multiple ways to suit +each framework and file type. + +```mermaid +flowchart TB + Proxy.DirectProxyHandler --> NettyClientSslContextRefresher + Proxy.DirectProxyHandler --> NettySSLContextAutoRefreshBuilder + Proxy.AdminProxyHandler --> KeyStoreSSLContext + Proxy.AdminProxyHandler --> SecurityUtility + Proxy.ServiceChannelInitializer --> NettySSLContextAutoRefreshBuilder + Proxy.ServiceChannelInitializer --> NettyServerSslContextBuilder + Broker.PulsarChannelInitializer --> NettyServerSslContextBuilder + Broker.PulsarChannelInitializer --> NettySSLContextAutoRefreshBuilder + Client.PulsarChannelInitializer --> NettySSLContextAutoRefreshBuilder + Client.PulsarChannelInitializer --> SecurityUtility + Broker.WebService --> JettySSlContextFactory + Proxy.WebServer --> JettySSlContextFactory + PulsarAdmin --> AsyncHttpConnector + AsyncHttpConnector --> KeyStoreSSLContext + AsyncHttpConnector --> SecurityUtility + JettySSlContextFactory --> NetSslContextBuilder + JettySSlContextFactory --> DefaultSslContextBuilder + NettyClientSslContextRefresher -.-> SslContextAutoRefreshBuilder + NettySSLContextAutoRefreshBuilder -.-> SslContextAutoRefreshBuilder + NettyServerSslContextBuilder -.-> SslContextAutoRefreshBuilder + NetSslContextBuilder -.-> SslContextAutoRefreshBuilder + DefaultSslContextBuilder -.-> SslContextAutoRefreshBuilder + Client.HttpLookup.HttpClient --> KeyStoreSSLContext + Client.HttpLookup.HttpClient --> SecurityUtility + SecurityUtility -.-> KeyManagerProxy + SecurityUtility -.-> TrustManagerProxy +``` +The above diagram is an example of the complexity of the TLS encryption setup within Pulsar. The above diagram only +contains the basic components of Pulsar excluding Websockets, Functions, etc. + +Pulsar uses 2 base classes to load the TLS information. + +- `SecurityUtility`: It loads files of type ".crt", ".pem" and ".key" and converts it into SSL Context. This SSL Context +can be of type `io.netty.handler.ssl.SslContext` or `javax.net.ssl.SSLContext` based on the caller. Security Utility +can be used to create SSL Context that internally has KeyManager and Trustmanager proxies that load cert changes +dynamically. +- `KeyStoreSSLContext`: It loads files of type Java Keystore/Truststore and converts it into SSL Context. This SSL +Context will be of type `javax.net.ssl.SSLContext`. This is always used to create the SSL Engine. + +Each of the above classes are either directly used by Pulsar Clients or used via implementations of the abstract class +`SslContextAutoRefreshBuilder`. + +- `SslContextAutoRefreshBuilder` - This abstract class is used to refresh certificates at a configurable interval. It +internally provides a public API to return the SSL Context. + +There are several implementations of the above abstract class to suit the needs of each of the framework and the +respective TLS certificate files: + +- `NettyClientSslContextRefresher` - It internally creates the `io.netty.handler.ssl.SslContext` using the ".crt", +".pem" and ".key" files for the proxy client. +- `NettySSLContextAutoRefreshBuilder` - It internally creates the `KeyStoreSSLContext` using the Java Keystores. +- `NettyServerSslContextBuilder` - It internally creates the `io.netty.handler.ssl.SslContext` using the ".crt", + ".pem" and ".key" files for the server. +- `NetSslContextBuilder` - It internally creates the `javax.net.ssl.SSLContext` using the Java Keystores for the web +server. +- `DefaultSslContextBuilder` - It internally creates the `javax.net.ssl.SSLContext` using the ".crt", ".pem" and ".key" +files for the web server. + +# Motivation +Apache Pulsar's TLS encryption configuration is not pluggable. It only supports file-based certificates. This makes +Pulsar difficult to adopt for organizations that require loading TLS certificates by other mechanisms. + +# Goals +The purpose of this PIP is to introduce the following: + +- Provide a mechanism to plugin a custom SSL Factory that can generate SSL Context and SSL Engine. +- Simplify the Pulsar code base to universally use `javax.net.ssl.SSLContext` and reduce the amount of code required to +build and configure the SSL context taking into consideration backwards compatibility. + +## In Scope + +- Creation of a new interface `PulsarSslFactory` that can generate a SSL Context, Client SSL Engine and Server SSL +Engine. +- Creation of a default implementation of `PulsarSslFactory` that supports loading the SSL Context and SSL Engine via +file-based certificates. Internally it will use the SecurityUtility and KeyStoreSSLContext. +- Creation of a new class called "PulsarSslConfiguration" to store the ssl configuration parameters which will be passed +to the SSL Factory. +- Modify the Pulsar Components to support the `PulsarSslFactory` instead of the SslContextAutoRefreshBuilder, SecurityUtility +and KeyStoreSSLContext. +- Remove the SslContextAutoRefreshBuilder and all its implementations. +- SSL Context refresh will be moved out of the factory. The responsibility of refreshing the ssl context will lie with +the components using the factory. +- The factory will not be thread safe. We are isolating responsibilities by having a single thread perform all writes, +while all channel initializer threads will perform only reads. SSL Context reads can be eventually consistent. +- Each component calling the factory will internally initialize it as part of the constructor as well as create the +ssl context at startup as a blocking call. If this creation/initialization fails then it will cause the Pulsar +Component to shutdown. This is true for all components except the Pulsar client due to past contracts where +authentication provider may not have started before the client. +- Each component will re-use its scheduled executor provider to schedule the refresh of the ssl context based on its +component's certificate refresh configurations. + +# High Level Design +```mermaid +flowchart TB + Proxy.DirectProxyHandler --> PulsarSslFactory + Proxy.AdminProxyHandler --> PulsarSslFactory + Proxy.ServiceChannelInitializer --> PulsarSslFactory + Broker.PulsarChannelInitializer --> PulsarSslFactory + Client.PulsarChannelInitializer --> PulsarSslFactory + Broker.WebService --> JettySSlContextFactory + Proxy.WebServer --> JettySSlContextFactory + PulsarAdmin --> AsyncHttpConnector + AsyncHttpConnector --> PulsarSslFactory + JettySSlContextFactory --> PulsarSslFactory + Client.HttpLookup.HttpClient --> PulsarSslFactory + PulsarSslFactory -.-> DefaultPulsarSslFactory + PulsarSslFactory -.-> CustomPulsarSslFactory +``` + +# Detailed Design + +## Design and Implementation Details + +### Pulsar Common Changes + +A new interface called `PulsarSslFactory` that provides public methods to create a SSL Context, Client SSL Engine and +Server SSL Engine. The SSL Context class returned will be of type `javax.net.ssl.SSLContext`. + +```java +public interface PulsarSslFactory extends AutoCloseable { + /* + * Utilizes the configuration to perform initialization operations and may store information in instance variables. + * @param config PulsarSslConfiguration required by the factory for SSL parameters + */ + void initialize(PulsarSslConfiguration config); + + /* + * Creates a client ssl engine based on the ssl context stored in the instance variable and the respective parameters. + * @param peerHost Name of the peer host + * @param peerPort Port number of the peer + * @return A SSlEngine created using the instance variable stored Ssl Context + */ + SSLEngine createClientSslEngine(String peerHost, int peerPort); + + /* + * Creates a server ssl engine based on the ssl context stored in the instance variable and the respective parameters. + * @return A SSLEngine created using the instance variable stored ssl context + */ + SSLEngine createServerSslEngine(); + + /* + * Returns A boolean stating if the ssl context needs to be updated + * @return Boolean value representing if ssl context needs to be updated + */ + boolean needsUpdate(); + + /* + * Checks if the SSL Context needs to be updated. If true, then a new SSL Context should be internally create and + * should atomically replace the old ssl context stored in the instance variable. + * @throws Exception It can throw an exception if the createInternalSslContext method fails + */ + default void update() throws Exception { + if (this.needsUpdate()) { + this.createInternalSslContext(); + } + } + + /* + * Creates a new SSL Context and internally stores it atomically into an instance variable + * @throws It can throw an exception if the internal ssl context creation fails. + */ + void createInternalSslContext() throws Exception; + + /* + * Returns the internally stored ssl context + * @throws IllegalStateException If the SSL Context has not be created before this call, then it wil throw this + * exception. + */ + SSLContext getInternalSslContext(); + + /* + * Shutdown the factory and close any internal dependencies + * @throws Exception It can throw an exception if there are any issues shutting down the factory. + */ + void close() throws Exception; + +} +``` + +A default implementation of the above SSLFactory class called `DefaultPulsarSslFactory` that will generate the SSL +Context and SSL Engines using File-based Certificates. It will be able to support both Java keystores and "pem/crt/key" +files. + +```java +public class DefaultPulsarSslFactory implements PulsarSslFactory { + public void initialize(PulsarSslConfiguration config); + public SSLEngine createClientSslEngine(String peerHost, int peerPort); + public SSLEngine createServerSslEngine(); + public boolean needsUpdate(); + public void createInternalSslContext() throws Exception; + public SSLContext getInternalSslContext(); + public void close() throws Exception; +} +``` + +### Pulsar Commmon Changes + +4 new configurations will need to be added into the Configurations like `ServiceConfiguration`, +`ClientConfigurationData`, `ProxyConfiguration`, etc. All of the below will be optional. It will use the default values +to match the current behavior of Pulsar. + +- `sslFactoryPlugin`: SSL Factory Plugin class to provide SSLEngine and SSLContext objects. +The default class used is `DefaultPulsarSslFactory`. +- `sslFactoryPluginParams`: SSL Factory plugin configuration parameters. It will be of type string. It can be parsed by +the plugin at its discretion. + +The below configs will be applicable only to the Pulsar Server components like Broker and Proxy: +- `brokerClientSslFactoryPlugin`: SSL Factory Plugin class used by internal client to provide SSLEngine and SSLContext +objects. The default class used is `DefaultPulsarSslFactory`. +- `brokerClientSslFactoryPluginParams`: SSL Factory plugin configuration parameters used by internal client. It can be +parsed by the plugin at its discretion. + +`JettySslContextFactory` class will need to be changed to internally use the `PulsarSslFactory` class to generate the +SslContext. + +### SslFactory Usage across Pulsar Netty based server components + +Example Changes in broker's `PulsarChannelInitializer` to initialize the PulsarSslFactory: +```java +PulsarSslConfiguration pulsarSslConfig = buildSslConfiguration(serviceConfig); +this.sslFactory = (PulsarSslFactory) Class.forName(serviceConfig.getSslFactoryPlugin()) + .getConstructor().newInstance(); +this.sslFactory.initialize(pulsarSslConfig); +this.sslFactory.createInternalSslContext(); +this.pulsar.getExecutor().scheduleWithFixedDelay(this::refreshSslContext, + serviceConfig.getTlsCertRefreshCheckDurationSec(), + serviceConfig.getTlsCertRefreshCheckDurationSec(), + TimeUnit.SECONDS); +``` + +Example changes in `PulsarChannelInitializer` to `initChannel(SocketChannel ch)`: +```java +ch.pipeline().addLast(TLS_HANDLER, new SslHandler(this.sslFactory.createServerSslEngine())); +``` + +The above changes is similar in all the Pulsar Server components that internally utilize Netty. + +### SslFactory Usage across Pulsar Netty based Client components + +Example Changes in Client's `PulsarChannelInitializer` to initialize the SslFactory: +```java +this.pulsarSslFactory = (PulsarSslFactory) Class.forName(conf.getSslFactoryPlugin()) + .getConstructor().newInstance(); +PulsarSslConfiguration sslConfiguration = buildSslConfiguration(conf); +this.pulsarSslFactory.initialize(sslConfiguration); +this.pulsarSslFactory.createInternalSslContext(); +scheduledExecutorProvider.getExecutor()) + .scheduleWithFixedDelay(() -> { + this.refreshSslContext(conf); + }, conf.getAutoCertRefreshSeconds(), + conf.getAutoCertRefreshSeconds(), TimeUnit.SECONDS); +``` + +Example changes in `PulsarChannelInitializer` to `initChannel(SocketChannel ch)`: +```java +SslHandler handler = new SslHandler(sslFactory + .createClientSslEngine(sniHost.getHostName(), sniHost.getPort())); +ch.pipeline().addFirst(TLS_HANDLER, handler); +``` + +The above changes is similar in all the Pulsar client components that internally utilize Netty. + +### SslFactory Usage across Pulsar Jetty Based Server Components + +The initialization of the PulsarSslFactory is similar to the [Netty Server initialization.](#sslfactory-usage-across-pulsar-jetty-based-server-components) + +The usage of the PulsarSslFactory requires changes in the `JettySslContextFactory`. It will internally accept +`PulsarSslFactory` as an input and utilize it to create the SSL Context. +```java +public class JettySslContextFactory { + private static class Server extends SslContextFactory.Server { + private final PulsarSslFactory sslFactory; + + // New + public Server(String sslProviderString, PulsarSslFactory sslFactory, + boolean requireTrustedClientCertOnConnect, Set ciphers, Set protocols) { + this.sslFactory = sslFactory; + // Current implementation + } + + @Override + public SSLContext getSslContext() { + return this.sslFactory.getInternalSslContext(); + } + } +} +``` + +The above `JettySslContextFactory` will be used to create the SSL Context within the Jetty Server. This pattern will be +common across all Web Server created using Jetty within Pulsar. + +### SslFactory Usage across Pulsar AsyncHttpClient based Client Components + +The initialization of the PulsarSslFactory is similar to the [Netty Server initialization.](#sslfactory-usage-across-pulsar-jetty-based-server-components) + +The usage of the PulsarSslFactory requires changes in the `AsyncHttpConnector`. It will internally initialize the +`PulsarSslFactory` and pass it to a new custom `PulsarHttpAsyncSslEngineFactory` that implements `org.asynchttpclient.SSLEngineFactory`. +This new custom class will incorporate the features of the existing `WithSNISslEngineFactory` and `JsseSslEngineFactory` +and replace it. + +```java +public class PulsarHttpAsyncSslEngineFactory extends DefaultSslEngineFactory { + + private final PulsarSslFactory sslFactory; + private final String host; + + public PulsarHttpAsyncSslEngineFactory(PulsarSslFactory sslFactory, String host) { + this.sslFactory = sslFactory; + this.host = host; + } + + @Override + protected void configureSslEngine(SSLEngine sslEngine, AsyncHttpClientConfig config) { + super.configureSslEngine(sslEngine, config); + if (StringUtils.isNotBlank(host)) { + SSLParameters parameters = sslEngine.getSSLParameters(); + parameters.setServerNames(Collections.singletonList(new SNIHostName(host))); + sslEngine.setSSLParameters(parameters); + } + } + + @Override + public SSLEngine newSslEngine(AsyncHttpClientConfig config, String peerHost, int peerPort) { + SSLContext sslContext = this.sslFactory.getInternalSslContext(); + SSLEngine sslEngine = config.isDisableHttpsEndpointIdentificationAlgorithm() + ? sslContext.createSSLEngine() : + sslContext.createSSLEngine(domain(peerHost), peerPort); + configureSslEngine(sslEngine, config); + return sslEngine; + } + +} +``` + +The above `PulsarHttpAsyncSslEngineFactory` will be passed to the DefaultAsyncHttpClientConfig.Builder while creating +the DefaultAsyncHttpClient. This pattern will be common across all HTTP Clients using AsyncHttpClient within Pulsar. + +## Public-facing Changes + +### Configuration + +Same as [Broker Common Changes](#pulsar-commmon-changes) + +### CLI +CLI tools like `PulsarClientTool` and `PulsarAdminTool` will need to be modified to support the new configurations. + +# Backward & Forward Compatibility + +## Revert +Rolling back to the previous version of Pulsar will revert to the previous behavior. + +## Upgrade +Upgrading to the version containing the `PulsarSslFactory` will not cause any behavior change. The `PulsarSslFactory` +for the server, client and brokerclient will default to using the `DefaultPulsarSslFactory` which will +read the TLS certificates via the file system. + +The Pulsar system will use the custom plugin behavior only if the `sslFactoryPlugin` configuration is set. + +# Links + +POC Changes: https://github.com/Apurva007/pulsar/pull/4 \ No newline at end of file diff --git a/pip/pip-339.md b/pip/pip-339.md new file mode 100644 index 0000000000000..a710ad453ea28 --- /dev/null +++ b/pip/pip-339.md @@ -0,0 +1,62 @@ +# PIP-339: Introducing the --log-topic Option for Pulsar Sinks and Sources + +# Motivation + +The `--log-topic` option already exists in Pulsar Functions, enabling users to direct function logs to a specified +"log topic". This feature is useful for debugging and analysis. However, Pulsar Sinks and Sources currently lack this +option, resulting in inconsistent log management across Pulsar Functions and Connectors. + +# Goals + +## In Scope + +The primary objective of this proposal is to integrate the `--log-topic` option into the **create**, **update**, and +**localrun** sub-commands for Pulsar Sinks and Sources. + +# Detailed Design + +## Design & Implementation Details + +1. Integrate the `--log-topic` option into `SinkDetailsCommand` and `SourceDetailsCommand`: + + ```java + @Parameter(names = "--log-topic", description = "The topic to which the logs of a Pulsar Sink/Source are produced") + protected String logTopic; + ``` + +2. Pass this option to `functionDetailsBuilder` when creating, updating, or locally running Pulsar Sinks and Sources: + ```java + if (sinkConfig.getLogTopic() != null) { + functionDetailsBuilder.setLogTopic(sinkConfig.getLogTopic()); + } + ``` + + ```java + if (sourceConfig.getLogTopic() != null) { + functionDetailsBuilder.setLogTopic(sourceConfig.getLogTopic()); + } + ``` + +3. Return the "log topic" when getting Pulsar Sinks and Sources + + ```java + if (!isEmpty(functionDetails.getLogTopic())) { + sinkConfig.setLogTopic(functionDetails.getLogTopic()); + } + ``` + + ```java + if (!isEmpty(functionDetails.getLogTopic())) { + sourceConfig.setLogTopic(functionDetails.getLogTopic()); + } + ``` + +# General Notes + +Upon successful implementation of this proposal, the **create**, **update**, and **localrun** sub-commands for Pulsar +Sinks and Sources will include the --log-topic option. + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/8h6f8jcgs0cvvj96318zvcr18zs9513t +* Mailing List voting thread: https://lists.apache.org/thread/00682h05r4mh1plk10s6qq90p2s2xo74 diff --git a/pip/pip-342 OTel client metrics support.md b/pip/pip-342 OTel client metrics support.md new file mode 100644 index 0000000000000..ebbe1e24660ba --- /dev/null +++ b/pip/pip-342 OTel client metrics support.md @@ -0,0 +1,168 @@ +# PIP 342: Support OpenTelemetry metrics in Pulsar client + +## Motivation + +Current support for metric instrumentation in Pulsar client is very limited and poses a lot of +issues for integrating the metrics into any telemetry system. + +We have 2 ways that metrics are exposed today: + +1. Printing logs every 1 minute: While this is ok as it comes out of the box, it's very hard for + any application to get the data or use it in any meaningful way. +2. `producer.getStats()` or `consumer.getStats()`: Calling these methods will get access to + the rate of events in the last 1-minute interval. This is problematic because out of the + box the metrics are not collected anywhere. One would have to start its own thread to + periodically check these values and export them to some other system. + +Neither of these mechanism that we have today are sufficient to enable application to easily +export the telemetry data of Pulsar client SDK. + +## Goal + +Provide a good way for applications to retrieve and analyze the usage of Pulsar client operation, +in particular with respect to: + +1. Maximizing compatibility with existing telemetry systems +2. Minimizing the effort required to export these metrics + +## Why OpenTelemetry? + +[OpenTelemetry](https://opentelemetry.io/) is quickly becoming the de-facto standard API for metric and +tracing instrumentation. In fact, as part of [PIP-264](https://github.com/apache/pulsar/blob/master/pip/pip-264.md), +we are already migrating the Pulsar server side metrics to use OpenTelemetry. + +For Pulsar client SDK, we need to provide a similar way for application builder to quickly integrate and +export Pulsar metrics. + +### Why exposing OpenTelemetry directly in Pulsar API + +When deciding how to expose the metrics exporter configuration there are multiple options: + +1. Accept an `OpenTelemetry` object directly in Pulsar API +2. Build a pluggable interface that describe all the Pulsar client SDK events and allow application to + provide an implementation, perhaps providing an OpenTelemetry included option. + +For this proposal, we are following the (1) option. Here are the reasons: + +1. In a way, OpenTelemetry can be compared to [SLF4J](https://www.slf4j.org/), in the sense that it provides an API + on top of which different vendor can build multiple implementations. Therefore, there is no need to create a new + Pulsar-specific interface +2. OpenTelemetry has 2 main artifacts: API and SDK. For the context of Pulsar client, we will only depend on its + API. Applications that are going to use OpenTelemetry, will include the OTel SDK +3. Providing a custom interface has several drawbacks: + 1. Applications need to update their implementations every time a new metric is added in Pulsar SDK + 2. The surface of this plugin API can become quite big when there are several metrics + 3. If we imagine an application that uses multiple libraries, like Pulsar SDK, and each of these has its own + custom way to expose metrics, we can see the level of integration burden that is pushed to application + developers +4. It will always be easy to use OpenTelemetry to collect the metrics and export them using a custom metrics API. There + are several examples of this in OpenTelemetry documentation. + +## Public API changes + +### Enabling OpenTelemetry + +When building a `PulsarClient` instance, it will be possible to pass an `OpenTelemetry` object: + +```java +interface ClientBuilder { + // ... + ClientBuilder openTelemetry(io.opentelemetry.api.OpenTelemetry openTelemetry); +} +``` + +The common usage for an application would be something like: + +```java +// Creates a OpenTelemetry instance using environment variables to configure it +OpenTelemetry otel = AutoConfiguredOpenTelemetrySdk.builder().build() + .getOpenTelemetrySdk(); + +PulsarClient client = PulsarClient.builder() + .serviceUrl("pulsar://localhost:6650") + .openTelemetry(otel) + .build(); + +// .... +``` + +Even without passing the `OpenTelemetry` instance to Pulsar client SDK, an application using the OpenTelemetry +agent, will be able to instrument the Pulsar client automatically, because we default to use `GlobalOpenTelemetry.get()`. + +### Deprecating the old stats methods + +The old way of collecting stats will be deprecated in phases: + 1. Pulsar 3.3 - Old metrics deprecated, still enabled by default + 2. Pulsar 3.4 - Old metrics disabled by default + 3. Pulsar 4.0 - Old metrics removed + +Methods to deprecate: + +```java +interface ClientBuilder { + // ... + @Deprecated + ClientBuilder statsInterval(long statsInterval, TimeUnit unit); +} + +interface Producer { + @Deprecated + ProducerStats getStats(); +} + +interface Consumer { + @Deprecated + ConsumerStats getStats(); +} +``` + +## Initial set of metrics to include + +Based on the experience of Pulsar Go client SDK metrics ( +see: https://github.com/apache/pulsar-client-go/blob/master/pulsar/internal/metrics.go), +this is the proposed initial set of metrics to export. + +Additional metrics could be added later on, though it's better to start with the set of most important metrics +and then evaluate any missing information. + +These metrics names and attributes will be considered "Experimental" for 3.3 release and might be subject to changes. +The plan is to finalize all the namings in 4.0 LTS release. + +Attributes with `[name]` brackets will not be included by default, to avoid high cardinality metrics. + +| OTel metric name | Type | Unit | Attributes | Description | +|-------------------------------------------------|---------------|-------------|--------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------| +| `pulsar.client.connection.opened` | Counter | connections | | The number of connections opened | +| `pulsar.client.connection.closed` | Counter | connections | | The number of connections closed | +| `pulsar.client.connection.failed` | Counter | connections | | The number of failed connection attempts | +| `pulsar.client.producer.opened` | Counter | sessions | `pulsar.tenant`, `pulsar.namespace`, [`pulsar.topic`], [`pulsar.partition`] | The number of producer sessions opened | +| `pulsar.client.producer.closed` | Counter | sessions | `pulsar.tenant`, `pulsar.namespace`, [`pulsar.topic`], [`pulsar.partition`] | The number of producer sessions closed | +| `pulsar.client.consumer.opened` | Counter | sessions | `pulsar.tenant`, `pulsar.namespace`, [`pulsar.topic`], [`pulsar.partition`], `pulsar.subscription` | The number of consumer sessions opened | +| `pulsar.client.consumer.closed` | Counter | sessions | `pulsar.tenant`, `pulsar.namespace`, [`pulsar.topic`], [`pulsar.partition`], `pulsar.subscription` | The number of consumer sessions closed | +| `pulsar.client.consumer.message.received.count` | Counter | messages | `pulsar.tenant`, `pulsar.namespace`, [`pulsar.topic`], [`pulsar.partition`], `pulsar.subscription` | The number of messages explicitly received by the consumer application | +| `pulsar.client.consumer.message.received.size` | Counter | bytes | `pulsar.tenant`, `pulsar.namespace`, [`pulsar.topic`], [`pulsar.partition`], `pulsar.subscription` | The number of bytes explicitly received by the consumer application | +| `pulsar.client.consumer.receive_queue.count` | UpDownCounter | messages | `pulsar.tenant`, `pulsar.namespace`, [`pulsar.topic`], [`pulsar.partition`], `pulsar.subscription` | The number of messages currently sitting in the consumer receive queue | +| `pulsar.client.consumer.receive_queue.size` | UpDownCounter | bytes | `pulsar.tenant`, `pulsar.namespace`, [`pulsar.topic`], [`pulsar.partition`], `pulsar.subscription` | The total size in bytes of messages currently sitting in the consumer receive queue | +| `pulsar.client.consumer.message.ack` | Counter | messages | `pulsar.tenant`, `pulsar.namespace`, [`pulsar.topic`], [`pulsar.partition`], `pulsar.subscription` | The number of acknowledged messages | +| `pulsar.client.consumer.message.nack` | Counter | messages | `pulsar.tenant`, `pulsar.namespace`, [`pulsar.topic`], [`pulsar.partition`], `pulsar.subscription` | The number of negatively acknowledged messages | +| `pulsar.client.consumer.message.dlq` | Counter | messages | `pulsar.tenant`, `pulsar.namespace`, [`pulsar.topic`], [`pulsar.partition`], `pulsar.subscription` | The number of messages sent to DLQ | +| `pulsar.client.consumer.message.ack.timeout` | Counter | messages | `pulsar.tenant`, `pulsar.namespace`, [`pulsar.topic`], [`pulsar.partition`], `pulsar.subscription` | The number of messages that were not acknowledged in the configured timeout period, hence, were requested by the client to be redelivered | +| `pulsar.client.producer.message.send.duration` | Histogram | seconds | `pulsar.tenant`, `pulsar.namespace`, [`pulsar.topic`], [`pulsar.partition`] | Publish latency experienced by the application, includes client batching time | +| `pulsar.client.producer.rpc.send.duration` | Histogram | seconds | `pulsar.tenant`, `pulsar.namespace`, [`pulsar.topic`], [`pulsar.partition`], `pulsar.response.status="success\|failed"` | Publish RPC latency experienced internally by the client when sending data to receiving an ack | +| `pulsar.client.producer.message.send.size` | Counter | bytes | `pulsar.tenant`, `pulsar.namespace`, [`pulsar.topic`], [`pulsar.partition`], `pulsar.response.status="success\|failed"` | The number of bytes published | +| `pulsar.client.producer.message.pending.count"` | UpDownCounter | messages | `pulsar.tenant`, `pulsar.namespace`, [`pulsar.topic`], [`pulsar.partition`] | The number of messages in the producer internal send queue, waiting to be sent | +| `pulsar.client.producer.message.pending.size` | UpDownCounter | bytes | `pulsar.tenant`, `pulsar.namespace`, [`pulsar.topic`], [`pulsar.partition`] | The size of the messages in the producer internal queue, waiting to sent | +| `pulsar.client.lookup.duration` | Histogram | seconds | `pulsar.lookup.transport-type="binary\|http"`, `pulsar.lookup.type="topic\|metadata\|schema\|list-topics"`, `pulsar.response.status="success\|failed"` | Duration of different types of client lookup operations | + +## Metrics cardinality + +The metrics data point will be tagged with these attributes: + + * `pulsar.tenant` + * `pulsar.namespace` + * `pulsar.topic` + * `pulsar.partition` + +By default the metrics will be exported with tenant and namespace attributes set. If an application wants to enable +a finer level, with higher cardinality, it can do so by using OpenTelemetry configuration. + diff --git a/pip/pip-343.md b/pip/pip-343.md new file mode 100644 index 0000000000000..85fc323cba6c6 --- /dev/null +++ b/pip/pip-343.md @@ -0,0 +1,143 @@ +# PIP-343: Use picocli instead of jcommander + +# Motivation + +We use the [jcommander](https://github.com/cbeust/jcommander) to build the CLI tool, which is a good library, and is +stable, but it misses modern CLI features likes autocompletion, flag/command suggestion, native image, etc. + +These features are very important because there are many commands in the CLI, but the jcommander doesn't give friendly +hints when we use incorrect flags/commands, which makes the user experience not very friendly. + +In modern times, the [picocli](https://github.com/remkop/picocli) supports these features, which is a popular library. + +The following is some comparison between jcommander and picocli: + +- Error prompt: + ``` + bin/pulsar-admin clusters update cluster-a -b + + # jcommander + Need to provide just 1 parameter + + # picocli + Unknown option: '-b' + ``` + +- Command suggestion: + ``` + bin/pulsar-admin cluste + + # jcommander + Expected a command, got cluste + + # picocli + Unmatched argument at index 0: 'cluste' + Did you mean: pulsar-admin clusters? + ``` + +# Goals + +## In Scope + +Use the picocli instead of the jcommander in our CLI tool: + +- bin/pulsar +- bin/pulsar-admin +- bin/pulsar-client +- bin/pulsar-shell +- bin/pulsar-perf + +I'm sure this will greatly improve the user experience, and in the future we can also consider using native images to +reduce runtime, and improve the CLI document based on picocli. + +## Out Scope + +This PR simply replaces jcommander and does not introduce any enhancements. + +In the CLI, [autocomplete](https://picocli.info/autocomplete.html) is an important feature, and after this PIP is +complete I will make a new PIP to support this feature. + +# Detailed Design + +## Design & Implementation Details + +The jcommander and picocli have similar APIs, this will make the migration task very simple. + +This is [utility argument syntax](https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap12.html): + +``` +utility_name[-a][-b][-c option_argument] + [-d|-e][-f[option_argument]][operand...] +``` + +1. Use `@Command` instead of `@Parameters` to define the class as a command: + + ```java + @Command(name = "my-command", description = "Operations on persistent topics") + public class MyCommand { + + } + ``` + +2. Use `@Option` instead of `@Parameter` to defined the option of command: + + ```java + @Option(names = {"-r", "--role"}) + private String role; + ``` + +3. Use `@Parameters` to get the operand of command: + + ```java + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; + ``` + +4. Migrate jcommander converter to picocli converter: + + ```java + public class TimeUnitToMillisConverter implements ITypeConverter { + @Override + public Long convert(String value) throws Exception { + return TimeUnit.SECONDS.toMillis(RelativeTimeUtil.parseRelativeTimeInSeconds(value)); + } + } + ``` + +5. Add the picocli entrypoint: + + ```java + @Command + public class MyCommand implements Callable { + // Picocli entrypoint. + @Override + public Integer call() throws Exception { + // TODO + // run(); + return 0; + } + } + ``` + +The above is a common migration approach, and then we need to consider pulsar-shell and custom command separately. + +- pulsar-shell + + This is an interactive shell based on jline3 and jcommander, which includes pulsar-admin and pulsar-client commands. + The jcommander does not provide autocompletion because we have implemented it ourselves. In picocli, they + have [picocli-shell-jline3](https://github.com/remkop/picocli/blob/main/picocli-shell-jline3) to help us quickly build + the interactive shell. + +- custom command: + + This is an extension of pulsar-admin, and the plugin's implementation does not depend on jcommander. Since the bridge + is used, we only need to change the generator code based on picocli. + +# Backward & Forward Compatibility + +Fully compatible. + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/ydg1q064cd11pxwz693frtk4by74q32f +* Mailing List voting thread: https://lists.apache.org/thread/1bpsr6tkgm00bb66dt2s74r15o4b37s3 diff --git a/pip/pip-344.md b/pip/pip-344.md new file mode 100644 index 0000000000000..5eafc6fd5c279 --- /dev/null +++ b/pip/pip-344.md @@ -0,0 +1,127 @@ +# PIP-344: Correct the behavior of the public API pulsarClient.getPartitionsForTopic(topicName) + +# Background knowledge + +**Topic auto-creation** +- The partitioned topic auto-creation is dependent on `pulsarClient.getPartitionsForTopic` + - It triggers partitioned metadata creation by `pulsarClient.getPartitionsForTopic` + - And triggers the topic partition creation by producers' registration and consumers' registration. +- When calling `pulsarClient.getPartitionsForTopic(topicName)`, Pulsar will automatically create the partitioned topic metadata if it does not exist, either using `HttpLookupService` or `BinaryProtoLookupService`. + +**Now `pulsarClient.getPartitionsForTopic`'s behavior** +| case | broker allow `auto-create` | param allow
`create if not exists` | non-partitioned topic | partitioned topic | current behavior | +| --- | --- | --- | --- | --- | --- | +| 1 | `true/false` | `true/false` | `exists: true` | | REST API: `partitions: 0`
Binary API: `partitions: 0` | +| 2 | `true/false` | `true/false` | | `exists: true`
`partitions: 3` | REST API: `partitions: 3`
Binary API: `partitions: 3` | +| 3 | `true` | `true` | | | REST API:
  - `create new: true`
  - `partitions: 3`
Binary API:
  - `create new: true`
  - `partitions: 3`
| +| 4 | `true` | `false` | | | REST API:
  - `create new: false`
  - `partitions: 0`
Binary API:
  not support
| +| 5 | `false` | `true` | | | REST API:
  - `create new: false`
  - `partitions: 0`
Binary API:
  - `create new: false`
  - `partitions: 0`
| + +- Broker allows `auto-create`: see also the config `allowAutoTopicCreation` in `broker.conf`. +- Param allow
`create if not exists` + - Regarding the HTTP API `PersistentTopics.getPartitionedMetadata`, it is an optional param which named `checkAllowAutoCreation,` and the default value is `false`. + - Regarding the `pulsar-admin` API, it depends on the HTTP API `PersistentTopics.getPartitionedMetadata`, and it always sets the param `checkAllowAutoCreation` to `false` and can not be set manually. + - Regarding the client API `HttpLookupService.getPartitionedTopicMetadata`, it depends on the HTTP API `PersistentTopics.getPartitionedMetadata`, and it always sets the param `checkAllowAutoCreation` to `true` and can not be set manually. + - Regarding the client API `BinaryProtoLookupService.getPartitionedTopicMetadata`, it always tries to create partitioned metadata. +- `REST API & HTTP API`: Since there are only two implementations of the 4 ways to get partitioned metadata, we call HTTP API `PersistentTopics.getPartitionedMetadata`, `pulsar-admin`, and `HttpLookupService.getPartitionedTopicMetadata` HTTP API, and call `BinaryProtoLookupService.getPartitionedTopicMetadata` Binary API. + +# Motivation + +The param `create if not exists` of the Binary API is always `true.` + +- For case 4 of `pulsarClient.getPartitionsForTopic`'s behavior, it always tries to create the partitioned metadata, but the API name is `getxxx`. +- For case 5 of `pulsarClient.getPartitionsForTopic`'s behavior, it returns a `0` partitioned metadata, but the topic does not exist. For the correct behavior of this case, we had discussed [here](https://github.com/apache/pulsar/issues/8813) before. +- BTW, [flink-connector-pulsar](https://github.com/apache/flink-connector-pulsar/blob/main/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/topic/ProducerRegister.java#L221-L227) is using this API to create partitioned topic metadata. + +# Goals + +- Regarding the case 4: Add a new API `PulsarClient.getPartitionsForTopic(String, boolean)` to support the feature that just get partitioned topic metadata and do not try to create one. See detail below. +- Regarding the case 5: Instead of returning a `0` partitioned metadata, respond to a not found error when calling `pulsarClient.getPartitionsForTopic(String)` if the topic does not exist. + +# Detailed Design + +## Public-facing Changes + +When you call the public API `pulsarClient.getPartitionsForTopic`, pulsar will not create the partitioned metadata anymore. + +### Public API +**LookupService.java** +```java + +- CompletableFuture getPartitionedTopicMetadata(TopicName topicName); + +/** + * 1. Get the partitions if the topic exists. Return "{partition: n}" if a partitioned topic exists; return "{partition: 0}" if a non-partitioned topic exists. + * 2. When {@param createIfAutoCreationEnabled} is "false," neither partitioned topic nor non-partitioned topic does not exist. You will get an {@link PulsarClientException.NotFoundException}. + * 2-1. You will get a {@link PulsarClientException.NotSupportedException} if the broker's version is an older one that does not support this feature and the Pulsar client is using a binary protocol "serviceUrl". + * 3. When {@param createIfAutoCreationEnabled} is "true," it will trigger an auto-creation for this topic(using the default topic auto-creation strategy you set for the broker), and the corresponding result is returned. For the result, see case 1. + * @version 3.3.0 + */ ++ CompletableFuture getPartitionedTopicMetadata(TopicName topicName, boolean createIfAutoCreationEnabled); +``` + +The behavior of the new API `LookupService.getPartitionedTopicMetadata(TopicName, boolean)`. + +| case | client-side param: `createIfAutoCreationEnabled` | non-partitioned topic | partitioned topic | broker-side: topic auto-creation strategy | current behavior | +|------|--------------------------------------------------|-----------------------|-------------------------------------|--------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------| +| 1 | `true/false` | `exists: true` | | | REST/Binary API: `{partitions: 0}` | +| 2 | `true/false` | | `exists: true`
`partitions: 2` | | REST/Binary API: `{partitions: 2}` | +| 3 | `true` | | | `allowAutoTopicCreation`: `true` `allowAutoTopicCreationType`: `non-partitioned` | REST/Binary API:
  - `create new: true`
  - `{partitions: 0}` | +| 4 | `true` | | | `allowAutoTopicCreation`: `true` `allowAutoTopicCreationType`: `partitioned`
`defaultNumPartitions`: `2` | REST/Binary API:
  - `create new: true`
  - `{partitions: 2}` | +| 5 | `false` | | | `allowAutoTopicCreation`: `true` | REST/Binary API:
  - Not found error | +| 6 | `true` | | | `allowAutoTopicCreation`: `false` | REST/Binary API:
  - Not found error | + + +**PulsarClient.java** +```java +// This API existed before. Not change it, thus ensuring compatibility. ++ @Deprecated it is not suggested to use now; please use {@link #getPartitionsForTopic(TopicName, boolean)}. +- CompletableFuture> getPartitionsForTopic(String topic); ++ default CompletableFuture> getPartitionsForTopic(String topic) { ++ getPartitionsForTopic(topic, true); ++ } + +/** + * 1. Get the partitions if the topic exists. Return "[{partition-0}, {partition-1}....{partition-n}}]" if a partitioned topic exists; return "[{topic}]" if a non-partitioned topic exists. + * 2. When {@param createIfAutoCreationEnabled} is "false", neither the partitioned topic nor non-partitioned topic does not exist. You will get an {@link PulsarClientException.NotFoundException}. + * 2-1. You will get a {@link PulsarClientException.NotSupportedException} if the broker's version is an older one that does not support this feature and the Pulsar client is using a binary protocol "serviceUrl". + * 3. When {@param createIfAutoCreationEnabled} is "true," it will trigger an auto-creation for this topic(using the default topic auto-creation strategy you set for the broker), and the corresponding result is returned. For the result, see case 1. + * @version 3.3.0 + */ +CompletableFuture> getPartitionsForTopic(String topic, boolean createIfAutoCreationEnabled); +``` + +The behavior of the new API `PulsarClient.getPartitionsForTopic(String, boolean)`. + +| case | client-side param: `createIfAutoCreationEnabled` | non-partitioned topic | partitioned topic | broker-side: topic autp-creation strategy | current behavior | +|------|--------------------------------------------------|----------------------|-------------------------------------|--------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------| +| 1 | `true/false` | `exists: true` | | | REST/Binary API: `["{tenat}/{ns}/topic"]` | +| 2 | `true/false` | | `exists: true`
`partitions: 2` | | REST/Binary `API`: `["{tenat}/{ns}/topic-partition-0", "{tenat}/{ns}/topic-partition-1"]` | +| 3 | `true` | | | `allowAutoTopicCreation`: `true` `allowAutoTopicCreationType`: `non-partitioned` | REST/Binary API:
  - `create new: true`
  - `["{tenat}/{ns}/topic"]` | +| 4 | `true` | | | `allowAutoTopicCreation`: `true` `allowAutoTopicCreationType`: `partitioned`
`defaultNumPartitions`: `2` | REST/Binary API:
  - `create new: true`
  - `["{tenat}/{ns}/topic-partition-0", "{tenat}/{ns}/topic-partition-1"]` | +| 5 | `false` | | | `allowAutoTopicCreation`: `true` | REST/Binary API:
  - Not found error | +| 5 | `true` | | | `allowAutoTopicCreation`: `false` | REST/Binary API:
  - Not found error | + + + +### Binary protocol + +**CommandPartitionedTopicMetadata** +``` +message CommandPartitionedTopicMetadata { + + optional bool metadata_auto_creation_enabled = 6 [default = true]; +} +``` + +**FeatureFlags** +``` +message FeatureFlags { + + optional bool supports_binary_api_get_partitioned_meta_with_param_created_false = 5 [default = false]; +} +``` + +# Backward & Forward Compatibility + +- Old version client and New version Broker: The client will call the old API. + +- New version client and Old version Broker: The feature flag `supports_binary_api_get_partitioned_meta_with_param_created_false` will be `false`. The client will get a not-support error if the param `createIfAutoCreationEnabled` is false. diff --git a/pip/pip-347.md b/pip/pip-347.md new file mode 100644 index 0000000000000..a5d5d76ae1700 --- /dev/null +++ b/pip/pip-347.md @@ -0,0 +1,37 @@ + +# PIP-347: add role field in consumer's stat + +# Background knowledge + +During the operation and maintenance process, there are many users asking administrator for help to find out the consumers of a topic and notify them about the business change. +Administrators can call `bin/pulsar-admin topics partitioned-stats` to find out the `ip:port` of the consumers, but no role info. So administrators need to take a lot of time to +communicate with users to find out the owner based on the `ip:port`. It's a troublesome work and low efficiency, or even can't find out the owner. + +# Motivation + +This pip can help to solve such kind of problem. By adding a field `appId` in the consumer's stat. +For cluster with JWT-based authentication, the administrator can find out the owner of the consumer directly. +It can save a lot of time and improve the efficiency of the operation and maintenance process. + +# Goals + +- help administrator to find out the owner of the consumer for cluster with JWT-based authentication. + +# Detailed Design + +## Design & Implementation Details +- Add a field `appId` in the consumer's stat, which can show the owner of this consumer for JWT-based authentication users. + +# Backward & Forward Compatibility + +Fully compatible. + +# General Notes + +# Links + + +* Mailing List discussion thread: https://lists.apache.org/thread/p9y9r8pb7ygk8f0jd121c1121phvzd09 +* Mailing List voting thread: https://lists.apache.org/thread/sfv0vq498dnjx6k6zdrnn0cw8f22tz05 diff --git a/pip/pip-348.md b/pip/pip-348.md new file mode 100644 index 0000000000000..7661ef3685867 --- /dev/null +++ b/pip/pip-348.md @@ -0,0 +1,40 @@ +# PIP-348: Trigger offload on topic load stage + +# Background knowledge + +Pulsar tiered storage is introduced by [PIP-17](https://github.com/apache/pulsar/wiki/PIP-17:-Tiered-storage-for-Pulsar-topics) to offload cold data from BookKeeper to external storage. Ledger is the basic offload unit, and one ledger will trigger offload only when the ledger rollover. Pulsar topic offload can be triggered by the following ways: +- Manually trigger offload by using the `bin/pulsar-admin` command. +- Automatically trigger offload by the offload policy. + + +# Motivation +For triggering offload, the offload policy is the most common way. The offload policy can be defined in cluster level, namespace level and topic level, and the offload policy is triggered by the following ways: +- One ledger is closed or rollover +- Check the offload policy +- Trigger offload if the offload policy is satisfied + +If one topic has multiple ledgers and the latest ledgers rollover triggered offload, all the previous ledgers will be added into pending offload queue and trigger offload one by one. However, if the topic is unloaded and loaded again, the offload process will be interrupted and needs to waiting for the next ledger rollover to trigger offload. This will cause the offload process is not efficient and the offload process is not triggered in time. + + +# Goals + +## In Scope + +Trigger offload on topic load stage to improve the offload process efficiency and make sure the offload process is triggered in time. + + +# Detailed Design + +## Design & Implementation Details + +When the topic is loaded, we can check the offload policy to see if the offload policy is satisfied. If the offload policy is satisfied, we can trigger offload immediately. This will improve the offload process efficiency and make sure the offload process is triggered in time. + +In order to reduce the impact on topic load when Pulsar is upgraded from the old versions, I introduce a flag named `triggerOffloadOnTopicLoad` to control whether enable this feature or not. + +# Backward & Forward Compatibility + +Fully compatible. + +# Links +* Mailing List discussion thread: https://lists.apache.org/thread/2ndomp8v4wkcykzthhlyjqfmswor88kv +* Mailing List voting thread: https://lists.apache.org/thread/q4mfn8x69hbgv19nmqx4dmknl3vsn9y8 diff --git a/pip/pip-349.md b/pip/pip-349.md new file mode 100644 index 0000000000000..b676b09aa2ee4 --- /dev/null +++ b/pip/pip-349.md @@ -0,0 +1,33 @@ +# PIP-349: Add additionalSystemCursorNames ignore list for ttl check + +# Background knowledge + +In Pulsar topic, we have [retention policy](https://pulsar.apache.org/docs/3.2.x/cookbooks-retention-expiry/#retention-policies) to control the acknowledged message lifetime. For the unacknowledged messages, we have a separate mechanism to control the message lifetime, which is called [`TTL`](https://pulsar.apache.org/docs/3.2.x/cookbooks-retention-expiry/#time-to-live-ttl). The `TTL` is a time-to-live value for the message, which is controlled by `ttlDurationDefaultInSeconds`. The message will be automatically acknowledged if it is not consumed within the `TTL` value. + +# Motivation + +In Pulsar, we have two kinds of topics, system topic and normal topic. The system topics are used for internal purposes, such as transaction internal topics. The system topics are not supposed to be consumed by the users. However, the system topics are still subject to the `TTL` check. If the system topics are not consumed within the `TTL` value, the messages in the system topics will be automatically acknowledged. This is not the expected behavior for the system topics and may lead to data loss. +For normal topics, we also has two kinds of subscriptions, system subscription and normal subscription. The system subscription is used for internal purposes, such as compaction service or third-party plugins. The system subscription is not supposed to be used by the users. However, the system subscription is still subject to the `TTL` check. If the system subscription is not consumed within the `TTL` value, the messages in the system subscription will be automatically acknowledged. This is not the expected behavior for the system subscription. + +We had one PR [#21865](https://github.com/apache/pulsar/pull/21865) to filter the compaction service cursors for TTL check, but it doesn't cover other system cursors. To provide a general solution and support third-party plugin cursors not impacted by TTL, I proposed to add an additionalSystemCursorNames ignore list to filter the TTL check. + +# Goals + +## In Scope + +Add an additionalSystemCursorNames ignore list to filter the TTL check for additional system subscriptions except for compaction service subscription. The additionalSystemCursorNames ignore list is an optional configuration, and the default value is empty. Pulsar broker will filter the TTL check for the additionalSystemCursorNames subscriptions. +The compaction service subscription is a system subscription and should not be impacted by TTL. To reduce the risk of data loss after enabled compaction service, we will add the compaction service subscription to the TTL ignore list by default and can't be removed. + +# Detailed Design + +## Design & Implementation Details + +Add a additionalSystemCursorNames ignore list to filter the TTL check for system subscriptions. The additionalSystemCursorNames ignore list is an optional configuration, and the default value is empty. Pulsar broker will filter the TTL check for the additionalSystemCursorNames subscriptions. + +# Backward & Forward Compatibility + +This change is fully compatible. + +# Links +* Mailing List discussion thread: https://lists.apache.org/thread/xgcworz4j8rjlqwr476s7sqn9do43f1t +* Mailing List voting thread: https://lists.apache.org/thread/xs3g2y6fgjpfjr8fhf1qghcxkrt3yby7 diff --git a/pip/pip-350.md b/pip/pip-350.md new file mode 100644 index 0000000000000..f48771e7ee17d --- /dev/null +++ b/pip/pip-350.md @@ -0,0 +1,36 @@ +# PIP-350: Allow to disable the managedLedgerOffloadDeletionLagInMillis + +# Background knowledge + +https://pulsar.apache.org/docs/3.2.x/tiered-storage-overview/ +Pulsar provides the ability to offload the data from bookkeeper to the cloud storage with the tiered storage. +Once the data is offloaded to the cloud storage, the data in the bookkeeper can be deleted after a certain period of time. +We use the managedLedgerOffloadDeletionLagInMillis to control the deletion lag time for the offloaded data. +The default value of managedLedgerOffloadDeletionLagInMillis is 4 hours. It means the offloaded data will be deleted after 4 hours by default. + +# Motivation + +In some test scenarios, we want to disable the deletionLag and never delete the data from the bookkeeper. +Then when the tiered storage data is broken, we can still read the data from the bookkeeper. + +# Goals + +## In Scope + +Never deletes the bookkeeper data when the managedLedgerOffloadDeletionLagInMillis is set to -1. + +# Detailed Design + +## Design & Implementation Details + +Only need to check the value of managedLedgerOffloadDeletionLagInMillis in the ManagedLedgerImpl when it is going to delete the bookkeeper data. +https://github.com/apache/pulsar/blob/774a5d42e8342ee50395cf3626b9e7af27da849e/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java#L2579 + +# Backward & Forward Compatibility + +Fully compatible. + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/7tlpkcm2933ddg95kgrb42943r4gq3v9 +* Mailing List voting thread: https://lists.apache.org/thread/c3rh530dlwo6nhrdflpw0mjck85hhfbx diff --git a/pip/pip-351.md b/pip/pip-351.md new file mode 100644 index 0000000000000..17f88b4895533 --- /dev/null +++ b/pip/pip-351.md @@ -0,0 +1,166 @@ + + +# PIP-351: Additional options for Pulsar-Test client to support KeyStore based TLS + +# Background knowledge + + + +In both Pulsar Client and Pulsar Admin, we support the use of KeyStores. This feature is provided by means of the boolean +"useKeyStoreTls". The boolean is also the only way authentication mechanisms such as AuthenticationKeyStoreTls can be utilised +properly, as the logic to use keystores for SSL Connections, from either ClientConfigurationData stored in Pulsar Admin/Client +or AuthData hinges on the "useKeyStoreTls" boolean as can be seen below: + +AsyncHttpConnector.java +```java +if (conf.isUseKeyStoreTls()) { + KeyStoreParams params = authData.hasDataForTls() ? authData.getTlsKeyStoreParams() : + new KeyStoreParams(conf.getTlsKeyStoreType(), conf.getTlsKeyStorePath(), + conf.getTlsKeyStorePassword()); + + final SSLContext sslCtx = KeyStoreSSLContext.createClientSslContext( + conf.getSslProvider(), + params.getKeyStoreType(), + params.getKeyStorePath(), + params.getKeyStorePassword(), + conf.isTlsAllowInsecureConnection(), + conf.getTlsTrustStoreType(), + conf.getTlsTrustStorePath(), + conf.getTlsTrustStorePassword(), + conf.getTlsCiphers(), + conf.getTlsProtocols()); + + JsseSslEngineFactory sslEngineFactory = new JsseSslEngineFactory(sslCtx); + confBuilder.setSslEngineFactory(sslEngineFactory); +} +``` + +None of these options can be currently configured when using Pulsar Test client. + +# Motivation + + + +As we already let users both extend authentication and use just the keystore and truststore properties to set up mTLS +connections, without using any authentication plugin class, a lot of them might want to use this method of authentication +during Performance Testing as well. + +I understand that currently mTLS (for testing purposes) can be achieved by using trust and client certificates. +However, the issue of users extending authentication plugin classes and utilizing keystores is still not covered +with the current options. Therefore, I propose we make these already existing options be configured in test clients, +increasing its usability. + +# Goals + +## In Scope + +Create new Arguments for the following properties, in PerformanceBaseArguments.java : +1. useKeyStoreTls +2. trustStoreType +3. trustStorePath +4. trustStorePass +5. keyStoreType +6. keyStorePath +7. keyStorePass + +Update the code to change between TrustCerts and TrustStore based on useKeyStoreTls. + + + +[//]: # (## Out of Scope) + + + + +[//]: # (# High Level Design) + + + +# Detailed Design + +## Design & Implementation Details + + + +Add the options for utilizing keystores as part of performance base arguments, along with forwarding their values +to the client/admin builders. + +## Public-facing Changes + + + +### CLI + +All places we utilize Pulsar Test client, for example Pulsar-Perf will have the following new options: + +1. --use-keystore-tls → Default value = false +2. --truststore-type → Default value = JKS, Possible values = JKS, PKCS12 +3. --truststore-path → Default value = "" +4. --truststore-pass → Default value = "" +5. --keystore-type → Default value = JKS, Possible values = JKS, PKCS12 +6. --keystore-path → Default value = "" +7. --keystore-pass → Default value = "" + + + +# Backward & Forward Compatibility + +The change will not affect any previous releases. The options can also be brought to previous versions, however, I have +noticed that Pulsar has moved away from JCommander in Version 3.2.x to Picocli (currently in master) +Therefore, to add these options to previous versions, the code has to be replicated to those versions. diff --git a/pip/pip-352.md b/pip/pip-352.md new file mode 100644 index 0000000000000..31641e7e1e1b5 --- /dev/null +++ b/pip/pip-352.md @@ -0,0 +1,68 @@ +# PIP-352: Event time based topic compactor + +# Background knowledge + +Pulsar Topic Compaction provides a key-based data retention mechanism that allows you only to keep the most recent message associated with that key to reduce storage space and improve system efficiency. + +Another Pulsar's internal use case, the Topic Compaction of the new load balancer, changed the strategy of compaction. It only keeps the first value of the key. For more detail, see [PIP-215](https://github.com/apache/pulsar/issues/18099). + +There is also plugable topic compaction service present. For more details, see [PIP-278](https://github.com/apache/pulsar/pull/20624) + +More topic compaction details can be found in [Pulsar Topic Compaction](https://pulsar.apache.org/docs/en/concepts-topic-compaction/). + +# Motivation + +Currently, there are two types of compactors +available: `TwoPhaseCompactor` and `StrategicTwoPhaseCompactor`. The latter +is specifically utilized for internal load balancing purposes and is not +employed for regular compaction of Pulsar topics. On the other hand, the +former can be configured via `CompactionServiceFactory` in the +`broker.conf`. + +I believe it could be advantageous to introduce another type of topic +compactor that operates based on event time. Such a compactor would have +the capability to maintain desired messages within the topic while +preserving the order expected by external applications. Although +applications may send messages with the current event time, variations in +network conditions or redeliveries could result in messages being stored in +the Pulsar topic in a different order than intended. Implementing event +time-based checks could mitigate this inconvenience. + +# Goals +* No impact on current topic compation behavior +* Preserve the order of messages during compaction regardless of network latencies + +## In Scope +* Abstract TwoPhaseCompactor + +* Migrate the current implementation to a new abstraction + +* Introduce new compactor based on event time + +* Makes existing tests compatible with new implementations. + + +# High Level Design + +In order to change the way topic is compacted we need to create `EventTimeCompactionServiceFactory`. This service provides a new +compactor `EventTimeOrderCompactor` which has a logic similar to existing `TwoPhaseCompactor` with a slightly change in algorithm responsible for +deciding which message is outdated. + +New compaction service factory can be enabled via `compactionServiceFactoryClassName` + +# Detailed Design + +## Design & Implementation Details + +* Abstract `TwoPhaseCompactor` and move current logic to new `PublishingOrderCompactor` + +* Implement `EventTimeCompactionServiceFactory` and `EventTimeOrderCompactor` + +* Create `MessageCompactionData` as a holder for compaction related data + +Example implementation can be found [here](https://github.com/apache/pulsar/pull/22517/files) + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/nc8r3tm9xv03vl30zrmfhd19q2k308y2 +* Mailing List voting thread: https://lists.apache.org/thread/pp6c0qqw51yjw9szsnl2jbgjsqrx7wkn diff --git a/pip/pip-353.md b/pip/pip-353.md new file mode 100644 index 0000000000000..5944aaea1abf4 --- /dev/null +++ b/pip/pip-353.md @@ -0,0 +1,137 @@ +# PIP-353: Improve transaction message visibility for peek-messages cli + +## Background knowledge + +This PIP addresses enhancements message visibility for the peek-message CLI, specifically related to transaction messages and markers. +Currently, when peeking messages, users may encounter `server internal marker` and `uncommitted` or `aborted` for transaction message, +which should typically be hidden to ensure data integrity and consistency. + +### Transaction Markers +Transaction markers are internal messages used by Pulsar to denote the start, end, and status (commit/abort) of a transaction. They are not meant to be consumed by clients. + +Similarly, other [internal markers](https://github.com/apache/pulsar/blob/ed5d94ccfdf4eba77678454945a2c3719dce2268/pulsar-common/src/main/proto/PulsarMarkers.proto#L25-L38) +used by Pulsar for various system operations should also not be visible to clients as they could lead to confusion and misinterpretation of the data. + +### Transaction Messages +- Uncommitted Messages: These should not be visible to consumers until the transaction is committed. +- Aborted Messages: These should be filtered out and never made visible to consumers. + +## Motivation + +The current implementation exposes all messages, including transaction markers and messages from uncommitted or aborted transactions, when peeking. +This behavior can confuse users and lead to incorrect data handling. The proposal aims to provide more control over what types of messages are visible during message peeking operations. + +## Goals + +### In Scope + +- Implement flags to selectively display `server markers`, `uncommitted messages(include aborted messages) for transaction` in peek operations. +- Set the default behavior to only show messages from committed transactions to ensure data integrity. + +### Out of Scope +- Any modifications to the core transaction handling mechanism. +- Any changes to consumer logic. + +## High Level Design + +The proposal introduces three new flags to the `peek-messages` command: + +1. `--show-server-marker`: Controls the visibility of server markers (default: `false`). +2. `---transaction-isolation-level`: Controls the visibility of messages for transactions. (default: `READ_COMMITTED`). Options: + - READ_COMMITTED: Can only consume all transactional messages which have been committed. + - READ_UNCOMMITTED: Can consume all messages, even transactional messages which have been aborted. + +These flags will allow administrators and developers to tailor the peek functionality to their needs, improving the usability and security of message handling in transactional contexts. + +## Detailed Design + +### Design & Implementation Details + +To support the `--show-server-marker` and `---transaction-isolation-level` flags, needs to introduce specific tag into the `headers` of messages returned by the +[peekNthMessage REST API](https://github.com/apache/pulsar/blob/8ca01cd42edfd4efd986f752f6f8538ea5bf4f94/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java#L1892-L1905). + +- `X-Pulsar-marker-type`: Already exists. +- `X-Pulsar-txn-uncommitted`: This entry is determined to be an uncommitted transaction by comparing its `position` and `maxReadPosition`. +- `X-Pulsar-txn-aborted`: It is determined to be aborted by calling the `persistentTopic.isAbort()` method. + +Then, In the CLI, these markers can be used to determine whether to filter out these messages and proceed to the next one. For an implementation example, +see the following code: [https://github.com/shibd/pulsar/pull/34](https://github.com/shibd/pulsar/pull/34) + +### Public-facing Changes + +#### CLI Command Flags + +New command line flags added for the `bin/pulsar-admin topics peek-messages` command: + +| Flag | Abbreviation | Type | Default | Description | +|----------------------------------|--------------|---------|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `--show-server-marker` | `-ssm` | Boolean | `false` | Enables the display of internal server write markers. | +| `---transaction-isolation-level` | `-til` | Enum | `false` | Enables theSets the isolation level for consuming messages within transactions.
- 'READ_COMMITTED' allows consuming only committed transactional messages.
- 'READ_UNCOMMITTED' allows consuming all messages, even transactional messages which have been aborted. | + + +## Public-facing Changes + +Add two methods to the admin.Topics() interface. + +```java + /** + * Peek messages from a topic subscription. + * + * @param topic + * topic name + * @param subName + * Subscription name + * @param numMessages + * Number of messages + * @param showServerMarker + * Enables the display of internal server write markers + * @param transactionIsolationLevel + * Sets the isolation level for consuming messages within transactions. + * - 'READ_COMMITTED' allows consuming only committed transactional messages. + * - 'READ_UNCOMMITTED' allows consuming all messages, + * even transactional messages which have been aborted. + * @return + * @throws NotAuthorizedException + * Don't have admin permission + * @throws NotFoundException + * Topic or subscription does not exist + * @throws PulsarAdminException + * Unexpected error + */ + List> peekMessages(String topic, String subName, int numMessages, + boolean showServerMarker, TransactionIsolationLevel transactionIsolationLevel) + throws PulsarAdminException; + + + /** + * Peek messages from a topic subscription asynchronously. + * + * @param topic + * topic name + * @param subName + * Subscription name + * @param numMessages + * Number of messages + * @param showServerMarker + * Enables the display of internal server write markers + @param transactionIsolationLevel + * Sets the isolation level for consuming messages within transactions. + * - 'READ_COMMITTED' allows consuming only committed transactional messages. + * - 'READ_UNCOMMITTED' allows consuming all messages, + * even transactional messages which have been aborted. + * @return a future that can be used to track when the messages are returned + */ + CompletableFuture>> peekMessagesAsync( + String topic, String subName, int numMessages, + boolean showServerMarker, TransactionIsolationLevel transactionIsolationLevel); +``` + +## Backward & Forward Compatibility + +### Revert +Reverting to a previous version of Pulsar without this feature will remove the additional flags from the CLI. Users who prefer the new behavior will need to manually adjust their usage patterns when reverting. + +### Upgrade +While upgrading to the new version of Pulsar that includes these changes, the default behavior of the `peek-messages` command will change. +Existing scripts or commands that rely on the old behavior (where transaction markers and messages from uncommitted or aborted transactions are visible) will need to explicitly set the new flags (`--show-server-marker true` and `--transaction-isolation-level READ_UNCOMMITTED` to maintain the old behavior. +This change is necessary as the previous default behavior did not align with typical expectations around data visibility and integrity in transactional systems. \ No newline at end of file diff --git a/pip/pip-354.md b/pip/pip-354.md new file mode 100644 index 0000000000000..3d85c6696ea73 --- /dev/null +++ b/pip/pip-354.md @@ -0,0 +1,50 @@ +# PIP-354: apply topK mechanism to ModularLoadManagerImpl + +# Background knowledge + +There are mainly two `LoadManager` implementation in Pulsar broker: `ExtensibleLoadManager` and `ModularLoadManagerImpl`. `ModularLoadManagerImpl` is the default load manager, and `ExtensibleLoadManager` is a new load manager which is proposed after 3.0.0 version. + +## ModularLoadManagerImpl +`ModularLoadManagerImpl` rely on zk to store and synchronize metadata about load, which pose greate pressure on zk, threatening the stability of system. Every broker will upload its `LocalBrokerData` to zk, and leader broker will retrieve all `LocalBrokerData` from zk, generate all `BundleData` from each `LocalBrokerData`, and update all `BundleData` to zk. + +## ExtensibleLoadManager +`ExtensibleLoadManager` depends on system topics and table views for load balance metadata store and replication. Though not using zk to store and synchronize metadata about load, it is still necessary to control the number of bundles that need to be updated, for which there is a `loadBalancerMaxNumberOfBundlesInBundleLoadReport` configuration in `ExtensibleLoadManager` that select the top k bundles. + + +# Motivation + +As every bundle in the cluster corresponds to a zk node, it is common that there are thousands of zk nodes in a cluster, which results into thousands of read/update operations to zk. This will cause a lot of pressure on zk. + +**As All Load Shedding Algorithm pick bundles from top to bottom based on throughput/msgRate, bundles with low throughput/msgRate are rarely be selected for shedding. So that we don't need to contain these bundles in the bundle load report.** + + + +# Goals + +Reuse the configuration `loadBalancerMaxNumberOfBundlesInBundleLoadReport` in `ExtensibleLoadManager`, apply the topK mechanism to `ModularLoadManagerImpl`. + +# Detailed Design + +If `loadBalancerMaxNumberOfBundlesInBundleLoadReport` is set to a positive number, `ModularLoadManagerImpl` will only select the `topK * brokerCount` bundles based on throughput/msgRate to update to zk. + +If `loadBalancerMaxNumberOfBundlesInBundleLoadReport` <= 0, `ModularLoadManagerImpl` will update all bundles to zk. + +As the default value of `loadBalancerMaxNumberOfBundlesInBundleLoadReport` is 10, `ModularLoadManagerImpl` will only update the top 10 * brokerCount bundles to zk by default. + +WARNING: too small `loadBalancerMaxNumberOfBundlesInBundleLoadReport` could result in a long load balance time. + +For users who don't want to use this feature, they can set `loadBalancerMaxNumberOfBundlesInBundleLoadReport` to 0. + +# Backward & Forward Compatibility + +This is a modular load balancer behavior change(not forward-compatible). + +User can set `loadBalancerMaxNumberOfBundlesInBundleLoadReport` to 0 to disable this feature. + +# Links + + +* Mailing List discussion thread: https://lists.apache.org/thread/wwybg8og80yz9gvj6bfdbv1znx2dfp4w +* Mailing List voting thread: https://lists.apache.org/thread/67r3nv33gfoxhvo74ql41dydh2rmyvjw diff --git a/pip/pip-355.md b/pip/pip-355.md new file mode 100644 index 0000000000000..cb0e41faefd6d --- /dev/null +++ b/pip/pip-355.md @@ -0,0 +1,36 @@ +# PIP-355: Enhancing Broker-Level Metrics for Pulsar + +# Background Knowledge +Pulsar provides broker-level, namespace-level, and topic-level metrics to monitor and analyze the behavior of the Pulsar service. These metrics are accessible through the Prometheus metrics endpoint. Detailed explanations of all metrics can be found on the Pulsar website: [Pulsar Metrics Reference](https://pulsar.apache.org/docs/3.2.x/reference-metrics/) + +# Motivation +Within Pulsar's current metrics framework, the `pulsar_out_bytes_total` metric is utilized to expose the total bytes dispatched by the broker to consumers. However, there are notable limitations and challenges associated with this metric: +- Inclusion of system subscriptions in the total bytes out, alongside user subscriptions, complicates accurate calculation of user-specific data. +- The granularity of the metric (namespace-level vs. topic-subscription level) impacts the scalability and resource consumption when calculating cluster-level total out bytes. + +# Goals +This proposal aims to address the following objectives: +- Simplify the process of calculating cluster-level total out bytes. +- Enable the calculation of total out bytes dispatched to system subscriptions. + +# High-Level Design +To achieve the outlined goals, the proposal introduces two new broker-level metrics: +- `pulsar_broker_out_bytes_total{system_subscription="true|false"}`: Represents the total out bytes dispatched by the broker to consumers. The label `system_subscription="false"` represents total traffic dispatched to user subscriptions, while `system_subscription="true"` represents total traffic dispatched to system cursors and cursor names added by `additionalSystemCursorNames` introduced in [PIP-349](https://github.com/apache/pulsar/pull/22651). +- `pulsar_broker_in_bytes_total{system_topic="true|false"}`: Tracks the total in bytes sent by producers to the broker. The label `system_topic="false"` represents total traffic from user topics, while `system_topic="true"` represents total traffic from system topics. + +# Detailed Design +The implementation involves the introduction of the following broker-level metrics: +- `pulsar_broker_out_bytes_total{system_subscription="true|false"}`: Aggregates the total out bytes from all topics, presented as a broker-level metric. +- `pulsar_broker_in_bytes_total{system_topic="true|false"}`: Calculation of total in bytes across all topics. + +# Metrics +The proposal includes the addition of two new broker-level metrics: +- `pulsar_broker_out_bytes_total{system_subscription="true|false"}` +- `pulsar_broker_in_bytes_total{system_topic="true|false"}` + +# Backward & Forward Compatibility +The proposed changes ensure full compatibility with existing systems and pave the way for seamless integration with future enhancements. + +# Links +- Mailing List discussion thread: https://lists.apache.org/thread/n3vvh6pso9ml7sg3qpww870om5vcfnpv +- Mailing List voting thread: https://lists.apache.org/thread/h4rjcv77wppz96gc31cpr3hw17v9jc4o diff --git a/pip/pip-356.md b/pip/pip-356.md new file mode 100644 index 0000000000000..c8cf96da58802 --- /dev/null +++ b/pip/pip-356.md @@ -0,0 +1,113 @@ +# PIP-356: Support Geo-Replication starts at earliest position + +# Background knowledge + +Replication reads messages from the source cluster, and copies them to the remote cluster. +- Registers a cursor named `pulsar.repl.{remote-cluster}` on the source cluster. Replicator reads messages relies on this cursor. +- Registers a producer on the remote cluster. Replicator writes messages relies on this producer. + +# Motivation + +If you have some older messages to migrate, the steps recommended are below, which was described at [pulsar doc](https://pulsar.apache.org/docs/3.2.x/administration-geo/#migrate-data-between-clusters-using-geo-replication). +1. Create the cursor that the replicator will use manually: `pulsar-admin topics create-subscription -s pulsar.repl.{remote-cluster} -m earliest `. +2. Enable namespace-level/topic-level Geo-Replication. + +The steps recommended are difficultly to use, for example: +- Create cursor `pulsar.repl.{remote-cluster}` manually. +- The namespace/topic was unloaded due to a re-balance. + - The broker will remove the `pulsar.repl.{remote-cluster}` automatically because the Geo-Replication feature is disabled at this moment. +- Enable namespace-level/topic-level Geo-Replication, but the cursor that was created manually has been deleted, the broker will create a new one with latest position, which is not expected. + + +# Goals +Add an optional config(broker level, namespace level, and topic level) to support Geo-Replication starting at the earliest position. + +### Configuration + +**broker.conf** +```properties +# The position that replication task start at, it can be set to "earliest" or "latest (default)". +replicationStartAt=latest +``` + +**ServiceConfiguration** +```java +@FieldContext( + category = CATEGORY_REPLICATION, + dynamic = true, + doc = "The position that replication task start at, it can be set to earliest or latest (default)." +) +String replicationStartAt = "latest"; +``` + +### Public API + +**V2/Namespaces.java** +```java +@POST +@Path("/{tenant}/{namespace}/replicationStartAt") +public void setNamespaceLevelReplicationStartAt( + @PathParam("tenant") String tenant, + @PathParam("namespace") String namespace, + @QueryParam("replicationStartAt") String replicationStartAt) { + ... + ... +} +``` + +**V2/PersistentTopics.java** +```java +@POST +@Path("/{tenant}/{namespace}/{topic}/replicationStartAt") +public void setNamespaceLevelReplicationStartAt( + @PathParam("tenant") String tenant, + @PathParam("namespace") String namespace, + @PathParam("topic") @Encoded String encodedTopic, + @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal, + @QueryParam("replicationStartAt") String replicationStartAt) { + ... + ... +} +``` + +### CLI + +**Namespaces command** +```shell +pulsar-admin namespaces set-replication-start-at {earliest|latest} +``` + +**Topics command** +```shell +pulsar-admin topics set-replication-start-at {earliest|latest} +``` + +### Binary protocol + +Nothing. + +### Metrics + +Nothing. + +# Monitoring + +Nothing. + +# Security Considerations + +Nothing. + +# Backward & Forward Compatibility + +You can do upgrading or reverting normally, no specified steps are needed to do. + +# Alternatives + +Nothing. + +# General Notes + +# Links +* Mailing List discussion thread: https://lists.apache.org/thread/8tp0rl05mjmqrxbp8m8nxx77d9x42chz +* Mailing List voting thread: https://lists.apache.org/thread/36jwdtdqspl4cq3m1cgz7xjk3gdpj45j diff --git a/pip/pip-357.md b/pip/pip-357.md new file mode 100644 index 0000000000000..716a7d5f5043c --- /dev/null +++ b/pip/pip-357.md @@ -0,0 +1,35 @@ +# PIP-357: Correct the conf name in load balance module. + +# Background knowledge + +We use `loadBalancerBandwithInResourceWeight` and `loadBalancerBandwithOutResourceWeight` to calculate the broker's load in the load balance module. However, the correct conf name should be `loadBalancerBandwidthInResourceWeight` and `loadBalancerBandwidthOutResourceWeight`. This PIP is to correct the conf name in the load balance module. + +# Motivation + +The current conf name is incorrect. + + +# Detailed Design + +- deprecated `loadBalancerBandwithInResourceWeight` and `loadBalancerBandwithOutResourceWeight` in the load balance module. +- add `loadBalancerBandwidthInResourceWeight` and `loadBalancerBandwidthOutResourceWeight` in the load balance module. + +In case of users upgrading to this version don't notice the change, we will still support the old conf name in following way: +- If a configuration is not the default configuration, use that configuration. +- If both the new and the old are configured different from the default value, use the new one. + +# Backward & Forward Compatibility + +Backward compatible, users can upgrade to this version without doing any changes and the old conf name will still work. +If user want to use the new conf name, they can change the conf name in the configuration file. +Just remember that if both the new and the old are configured different from the default value, the new one will be used. + +# General Notes + +# Links + + +* Mailing List discussion thread: https://lists.apache.org/thread/31wfq2hhprn4zknp4jv21lzf5809q6lf +* Mailing List voting thread: https://lists.apache.org/thread/0pggcploqw43mo134cwmk7b3p7t13848 diff --git a/pip/pip-358.md b/pip/pip-358.md new file mode 100644 index 0000000000000..cd5397309851a --- /dev/null +++ b/pip/pip-358.md @@ -0,0 +1,38 @@ + +# PIP-358: let resource weight work for OverloadShedder, LeastLongTermMessageRate, ModularLoadManagerImpl. + +# Background knowledge + +Initially, we introduce `loadBalancerCPUResourceWeight`, `loadBalancerBandwidthInResourceWeight`, `loadBalancerBandwidthOutResourceWeight`, +`loadBalancerMemoryResourceWeight`, `loadBalancerDirectMemoryResourceWeight` in `ThresholdShedder` to control the resource weight for +different resources when calculating the load of the broker. +Then we let it work for `LeastResourceUsageWithWeight` for better bundle placement policy. + +But https://github.com/apache/pulsar/pull/19559 and https://github.com/apache/pulsar/pull/21168 have pointed out that the actual load +of the broker is not related to the memory usage and direct memory usage, thus we have changed the default value of +`loadBalancerMemoryResourceWeight`, `loadBalancerDirectMemoryResourceWeight` to 0.0. + +There are still some places where memory usage and direct memory usage are used to calculate the load of the broker, such as +`OverloadShedder`, `LeastLongTermMessageRate`, `ModularLoadManagerImpl`. We should let the resource weight work for these places +so that we can set the resource weight to 0.0 to avoid the impact of memory usage and direct memory usage on the load of the broker. + +# Motivation + +The actual load of the broker is not related to the memory usage and direct memory usage, thus we should let the resource weight work for +`OverloadShedder`, `LeastLongTermMessageRate`, `ModularLoadManagerImpl` so that we can set the resource weight to 0.0 to avoid the impact of +memory usage and direct memory usage on the load of the broker. + + +# Detailed Design + +Let resource weight work for `OverloadShedder`, `LeastLongTermMessageRate`, `ModularLoadManagerImpl`. +- For `OverloadShedder`, `LeastLongTermMessageRate`, we replace `getMaxResourceUsage()` with `getMaxResourceUsageWithWeight()` in the calculation of the load of the broker. +- For `ModularLoadManagerImpl`, we replace `getMaxResourceUsage()` with `getMaxResourceUsageWithWeight()` when checking if the broker is overloaded and decide whether to update the broker data to metadata store. + +# Backward & Forward Compatibility + + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/lj34s3vmjbzlwmy8d66d0bsb25vnq9ky +* Mailing List voting thread: https://lists.apache.org/thread/b7dzm0yz6l40pkxmxhto5mro7brmz57r diff --git a/pip/pip-359.md b/pip/pip-359.md new file mode 100644 index 0000000000000..52a76193d6cf2 --- /dev/null +++ b/pip/pip-359.md @@ -0,0 +1,216 @@ +# PIP-359: Support custom message listener executor for specific subscription +Implementation PR: [#22861](https://github.com/apache/pulsar/pull/22861) + +# Background knowledge +In the current Pulsar client versions, from the user's perspective, when using a Pulsar Consumer, +we have two main options to consume messages: +1. Pull mode, by calling `consumer.recieve()`(or `consumer.recieveAsync()`) +```java +public class ConsumerExample { + public static void main(String[] args) throws PulsarClientException { + PulsarClient pulsarClient = PulsarClient.builder() + .serviceUrl("pulsar://localhost:6650") + .build(); + Consumer consumer = pulsarClient.newConsumer(Schema.INT64) + .topic("persistent://public/default/my-topic") + .subscriptionName("my-subscription") + .subscribe(); + do { + Message message = consumer.receive(); + consumer.acknowledge(message); + } while (true); + + } +} + +``` +2. Push mode, by registering a `MessageListener` interface, when building the Consumer. +When this method is used, we can't also use `consumer.receive()`(or `consumer.recieveAsync()`). +In the push mode, the MessageListener instance is called by the consumer, hence it is +doing that with a thread taken from its own internal `ExecutorService` (i.e. thread pool). +The problem comes when we build and use multiple Consumers from the same PulsarClient. It +so happens that those consumers will share the same thread pool to call the Message Listeners. +One can be slower from the other. + +```java +public class ConsumerExample { + public static void main(String[] args) throws PulsarClientException { + PulsarClient pulsarClient = PulsarClient.builder() + .serviceUrl("pulsar://localhost:6650") + .build(); + Consumer consumer = pulsarClient.newConsumer(Schema.INT64) + .topic("persistent://public/default/my-topic") + .subscriptionName("my-subscription2") + .messageListener((consumer, message) -> { + // process message + consumer.acknowledgeAsync(message); + }) + .subscribe(); + } +} +``` + +# Motivation + +As [Background knowledge](#background-knowledge) mentioned, when using asynchronous consumer, +by registering a MessageListener interface, there is a problem of different consumer groups +affecting each other, leading to unnecessary consumption delays. +Therefore, for this scenario, this PIP prepare to support specific a message +listener executor of consumer latitudes to solve such problem. +# Goals +1. Improve consumer message listener isolation, solve the consumption delay problem caused by +mutual influence of different consumers from the same PulsarClient instance. + +## In Scope +If this PIP is accepted, it will help Pulsar solve the problem of different consumers +from same `PulsarClient` affecting each other in the asynchronous consumption mode(`MessageListener`). + +## Out of Scope +This PIP will not build the plugin library mentioned in [PR](https://github.com/apache/pulsar/pull/22902#issuecomment-2169962642), +we will open a new PIP in the future to do this + + +# Detailed Design + +## Design & Implementation Details + +1. Add an interface `MessageListenerExecutor`, responsible for executing message listener callback tasks. +Users can customize the implementation to determine in which thread the message listener task is executed. +For example, in the situation described in [Motivation](#motivation) part, users can implement the +interface with an independent underlying thread pool to ensure that the message listener task of each +consumer is executed in a separate thread. The caller would be responsible for the life cycle of the +Executor, and it would be used only for this specific consumer. + ```java + public interface MessageListenerExecutor { + + /** + * select a thread by message(if necessary, for example, + * Key_Shared SubscriptionType, maybe need select thread + * by message order key to ensure order) to execute the runnable! + * + * @param message the message + * @param runnable the runnable to execute + */ + void execute(Message message, Runnable runnable); + } + ``` +2. Add an optional config `messageListenerExecutor` in `ConsumerBuilder`, then +users can pass their implementations. + ```java + ConsumerBuilder messageListenerExecutor(MessageListenerExecutor messageListenerExecutor); + ``` + +### Why need an interface like `MessageListenerExecutor` +Some people may wonder why not just use `java.util.concurrent.ExecutorService`, +but define an interface like `MessageListenerExecutor`. + +The reason is that: + +For sequential consumption scenarios, we need to ensure that messages with the same +key or the same partition are processed by the same thread to ensure order. If we +use `java.util.concurrent.ExecutorService`, refer to the following figure, we will not be able to make such guarantees, +because for ExecutorService, which thread to execute the task is not controlled by the user. +![](https://github.com/AuroraTwinkle/pulsar/assets/25919180/232854d6-01f2-4821-b2df-34d01dda1992) +![](https://github.com/AuroraTwinkle/pulsar/assets/25919180/204f5622-1e5a-4e73-b86b-15220bfb06d6) +### Interface implementation suggestions +When implementing the `MessageListenerExecutor` interface, you should consider the following points. +1. if you need to ensure the order of message processing, +you can select the thread by the message order key or `msg.getTopicName()`(partition topic name), +to ensure that the messages of the same order key (or partition) are processed in same thread. + +### Usage Example +```java + private void startConsumerWithMessageListener(String topic, String subscriptionName) throws PulsarClientException { + // for example: key_shared + MessageListenerExecutor keySharedExecutor = getKeySharedMessageListenerExecutor(subscriptionName); + Consumer keySharedconsumer = + pulsarClient.newConsumer(Schema.INT64) + .topic(topic) + .subscriptionName(subscriptionName) + // set and then message lister will be executed in the executor + .messageListener((c1, msg) -> { + log.info("Received message [{}] in the listener", msg.getValue()); + c1.acknowledgeAsync(msg); + }) + .messageListenerExecutor(keySharedExecutor) + .subscribe(); + + + // for example: partition_ordered + MessageListenerExecutor partitionOrderedExecutor = getPartitionOrderdMessageListenerExecutor(subscriptionName); + Consumer partitionOrderedConsumer = + pulsarClient.newConsumer(Schema.INT64) + .topic(topic) + .subscriptionName(subscriptionName) + // set and then message lister will be executed in the executor + .messageListener((c1, msg) -> { + log.info("Received message [{}] in the listener", msg.getValue()); + c1.acknowledgeAsync(msg); + }) + .messageListenerExecutor(partitionOrderedExecutor) + .subscribe(); + + // for example: out-of-order + ExecutorService executorService = Executors.newFixedThreadPool(10); + Consumer outOfOrderConsumer = + pulsarClient.newConsumer(Schema.INT64) + .topic(topic) + .subscriptionName(subscriptionName) + // not set and then message lister will be executed in the default executor + .messageListener((c1, msg) -> { + log.info("Received message [{}] in the listener", msg.getValue()); + c1.acknowledgeAsync(msg); + }) + .messageListenerExecutor((message, runnable) -> executorService.execute(runnable)) + .subscribe(); +} + +private static MessageListenerExecutor getKeySharedMessageListenerExecutor(String subscriptionName) { + ExecutorProvider executorProvider = new ExecutorProvider(10, subscriptionName + "listener-executor-"); + + return (message, runnable) -> { + byte[] key = "".getBytes(StandardCharsets.UTF_8); + if (message.hasKey()) { + key = message.getKeyBytes(); + } else if (message.hasOrderingKey()) { + key = message.getOrderingKey(); + } + // select a thread by message key to execute the runnable! + // that say, the message listener task with same order key + // will be executed by the same thread + ExecutorService executorService = executorProvider.getExecutor(key); + // executorService is a SingleThreadExecutor + executorService.execute(runnable); + }; +} + +private static MessageListenerExecutor getPartitionOrderdMessageListenerExecutor(String subscriptionName) { + ExecutorProvider executorProvider = new ExecutorProvider(10, subscriptionName + "listener-executor-"); + + return (message, runnable) -> { + // select a thread by partition topic name to execute the runnable! + // that say, the message listener task from the same partition topic + // will be executed by the same thread + ExecutorService executorService = executorProvider.getExecutor(message.getTopicName().getBytes()); + // executorService is a SingleThreadExecutor + executorService.execute(runnable); + }; +} + +``` +## Public-facing Changes + +### Public API + +1. Add an optional config `messageListenerExecutor` in `ConsumerBuilder` +```java +ConsumerBuilder messageListenerExecutor(MessageListenerExecutor messageListenerExecutor); +``` + +# Backward & Forward Compatibility +You can do upgrading or reverting normally, no specified steps are needed to do. + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/8nhqfdhkglsg5bgx6z7c1nho7z7l596l +* Mailing List voting thread: https://lists.apache.org/thread/oo3jdvq3b6bv6p4n7x7sdvypw4gp6hpk diff --git a/pip/pip-360.md b/pip/pip-360.md new file mode 100644 index 0000000000000..21e8e18dc0531 --- /dev/null +++ b/pip/pip-360.md @@ -0,0 +1,83 @@ +# PIP-360: Admin API to display Schema metadata + +# Background knowledge + +Broker loads and initializes Schema of the topic during the topic loading. However, we have seen large number of instances and issues when broker fails to load the topic when topic schema is broken due to missing or corrupt schema ledger, index ledger or even schema data. Therefore, if broker is not able to load the topic for any reason then it is not possible to fetch schema metadata and identify which schema ledger is causing the issue because broker is storing schema metadata into binary format and there is no such API exists which shows schema metadata into readable format. So, it is very important to have an API to read schema metadata with complete information to help system admin to understand topic unavailability issues. It is also very useful to get schema metadata to build various schema related external tools which can be used by system administrator. We already have APIs for managed-ledger and bookkeeper-ledgers which are used by external tools and CLI to read binary data from metadata store and display in readable format. + + +# Motivation + +Schema is one of the important part of the topic because it also plays important part in topic availability and required to successfully load the topic, and if schema initialization failure is causing issue in topic loading then it is very important to get schema metadata information to understand schema related issues and perform appropriate actions to mitigate that issue to successfully load the topic and make it available for users. Therefore, similar to ledger metadata and managed-ledger metadata, Pulsar should have API to show schema metadata and related ledger info which can be used by tools or users to perform appropriate actions during topic availability issues or any other troubleshooting. + +# Goals +Add an .admin API under schema resource which returns schema metadata into readable format + + +# High Level Design + +This PIP will introduce REST api which will accept the topic name and return schema metadata along with ledger information of schema-ledgers and index entries. It will also add CLI support to print schema metadata for users to see it in human readable format. + + +### Public API + + +This PIP will add a new REST endpoint under Schema resource path. +``` +Path: schema/{tenant}/{namespace}/{topic}/metadata +Response code: +307, message = Current broker doesn't serve the namespace of this topic +401, message = Client is not authorized or Don't have admin permission +403, message = Client is not authenticated +404, message = Tenant or Namespace or Topic doesn't exist; or Schema is not found for +412, message = Failed to find the ownership for the topic +``` +This admin API will return below schema metadata response. + +``` +@Data +public class SchemaMetadata { + public Entry info; + public List index; + @Data + @AllArgsConstructor + @NoArgsConstructor + static class Entry { + private long ledgerId; + private long entryId; + private long version; + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("ledgerId", ledgerId) + .add("entryId", entryId) + .add("version", version) + .toString(); + } + } +} +``` + +### CLI + +This PIP will also add appropriate CLI command under Schema command to get schema metadata. +``` +bin/pulsar-admin schemas get-metadata +``` + +# Links + +Sample PR: https://github.com/apache/pulsar/pull/22938 + +* Mailing List discussion thread: +* Mailing List voting thread: diff --git a/pip/pip-363.md b/pip/pip-363.md new file mode 100644 index 0000000000000..2b250e69871e1 --- /dev/null +++ b/pip/pip-363.md @@ -0,0 +1,111 @@ +# PIP-363: Add callback parameters to the method: `org.apache.pulsar.client.impl.SendCallback.sendComplete`. + +# Background knowledge + + +As introduced in [PIP-264](https://github.com/apache/pulsar/blob/master/pip/pip-264.md), Pulsar has been fully integrated into the `OpenTelemetry` system, which defines some metric specifications for [messaging systems](https://opentelemetry.io/docs/specs/semconv/messaging/messaging-metrics/#metric-messagingpublishduration). + +In the current Pulsar client code, it is not possible to obtain the number of messages sent in batches(as well as some other sending data), making it impossible to implement `messaging.publish.messages` metric. + +In the `opentelemetry-java-instrumentation` code, the `org.apache.pulsar.client.impl.SendCallback` interface is used to instrument data points. For specific implementation details, we can refer to [this](https://github.com/open-telemetry/opentelemetry-java-instrumentation/blob/main/instrumentation/pulsar/pulsar-2.8/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/pulsar/v2_8/ProducerImplInstrumentation.java#L89-L135). + +# Motivation + + +In the current situation, `org.apache.pulsar.client.impl.ProducerImpl` does not provide a public method to obtain the `numMessagesInBatch`. + +So, we can add some of `org.apache.pulsar.client.impl.ProducerImpl.OpSendMsg`'s key data into the `org.apache.pulsar.client.impl.SendCallback.sendComplete` method. + +# Detailed Design + +Add callback parameters to the method: `org.apache.pulsar.client.impl.SendCallback.sendComplete`: + +```java +public interface SendCallback { + + /** + * invoked when send operation completes. + * + * @param e + */ + void sendComplete(Throwable e, OpSendMsgStats stats); +} + +public interface OpSendMsgStats { + long getUncompressedSize(); + + long getSequenceId(); + + int getRetryCount(); + + long getBatchSizeByte(); + + int getNumMessagesInBatch(); + + long getHighestSequenceId(); + + int getTotalChunks(); + + int getChunkId(); +} + +@Builder +public class OpSendMsgStatsImpl implements OpSendMsgStats { + private long uncompressedSize; + private long sequenceId; + private int retryCount; + private long batchSizeByte; + private int numMessagesInBatch; + private long highestSequenceId; + private int totalChunks; + private int chunkId; + + @Override + public long getUncompressedSize() { + return uncompressedSize; + } + + @Override + public long getSequenceId() { + return sequenceId; + } + + @Override + public int getRetryCount() { + return retryCount; + } + + @Override + public long getBatchSizeByte() { + return batchSizeByte; + } + + @Override + public int getNumMessagesInBatch() { + return numMessagesInBatch; + } + + @Override + public long getHighestSequenceId() { + return highestSequenceId; + } + + @Override + public int getTotalChunks() { + return totalChunks; + } + + @Override + public int getChunkId() { + return chunkId; + } +} +``` + +# Links + + +* Mailing List discussion thread: https://lists.apache.org/thread/8pgmsvx1bxz4z1w8prpvpnfpt1kb57c9 +* Mailing List voting thread: https://lists.apache.org/thread/t0olt3722j17gjtdxqqsl3cpy104ogpr diff --git a/pip/pip-364.md b/pip/pip-364.md new file mode 100644 index 0000000000000..c589b3b47fc47 --- /dev/null +++ b/pip/pip-364.md @@ -0,0 +1,476 @@ + +# PIP-364: Introduce a new load balance algorithm AvgShedder + +# Background knowledge + +Pulsar has two load balance interfaces: +- `LoadSheddingStrategy` is an unloading strategy that identifies high load brokers and unloads some of the bundles they carry to reduce the load. +- `ModularLoadManagerStrategy` is a placement strategy responsible for assigning bundles to brokers. + +## LoadSheddingStrategy +There are three available algorithms: `ThresholdShedder`, `OverloadShedder`, `UniformLoadShedder`. + +### ThresholdShedder +`ThresholdShedder` uses the following method to calculate the maximum resource utilization rate for each broker, +which includes CPU, direct memory, bandwidth in, and bandwidth out. +``` + public double getMaxResourceUsageWithWeight(final double cpuWeight, + final double directMemoryWeight, final double bandwidthInWeight, + final double bandwidthOutWeight) { + return max(cpu.percentUsage() * cpuWeight, + directMemory.percentUsage() * directMemoryWeight, bandwidthIn.percentUsage() * bandwidthInWeight, + bandwidthOut.percentUsage() * bandwidthOutWeight) / 100; + } +``` + +After calculating the maximum resource utilization rate for each broker, a historical weight algorithm will +also be executed to obtain the final score. +``` +historyUsage = historyUsage == null ? resourceUsage : historyUsage * historyPercentage + (1 - historyPercentage) * resourceUsage; +``` +The historyPercentage is determined by configuring the `loadBalancerHistoryResourcePercentage`. +The default value is 0.9, which means that the last calculated score accounts for 90%, +while the current calculated score only accounts for 10%. + +The introduction of this historical weight algorithm is to avoid bundle switching caused by +short-term abnormal load increase or decrease, but in fact, this algorithm will introduce some +serious problems, which will be explained in detail later. + +Next, calculate the average score of all brokers in the entire cluster: `avgUsage=totalUsage/totalBrokers`. +When the score of any broker exceeds a certain threshold of avgUsage, it is determined that the broker is overloaded. +The threshold is determined by the configuration `loadBalancerBrokerThresholdShedderPercentage`, with a default value of 10. + + +### OverloadShedder +`OverloadShedder` use the same method `getMaxResourceUsageWithWeight` to calculate the maximum resource utilization rate for each broker. +The difference is that `OverloadShedder` will not use the historical weight algorithm to calculate the final score, +the final score is the current maximum resource utilization rate of the broker. + +After obtaining the load score for each broker, compare it with the `loadBalancerBrokerOverloadedThresholdPercentage`. +If the threshold is exceeded, it is considered overloaded, with a default value of 85%. + +This algorithm is relatively simple, but there are many serious corner cases, so it is not recommended to use `OverloadShedder`. +Here are two cases: +- When the load on each broker in the cluster reaches the threshold, the bundle unload will continue to be executed, + but it will only switch from one overloaded broker to another, which is meaningless. +- If there are no broker whose load reaches the threshold, adding new brokers will not balance the traffic to the new added brokers. +The impact of these two points is quite serious, so we won't talk about it next. + + +### UniformLoadShedder +`UniformLoadShedder` will first calculate the maximum and minimum message rates, as well as the maximum and minimum +traffic throughput and corresponding broker. Then calculate the maximum and minimum difference, with two thresholds +corresponding to message rate and throughput size, respectively. + +- loadBalancerMsgRateDifferenceShedderThreshold + +The message rate percentage threshold between the highest and lowest loaded brokers, with a default value of 50, +can trigger bundle unload when the maximum message rate is 1.5 times the minimum message rate. +For example, broker 1 with 50K msgRate and broker 2 with 30K msgRate will have a (50-30)/30=66%>50% difference in msgRate, +and the load balancer can unload the bundle from broker 1 to broker 2. + +- loadBalancerMsgThroughputMultiplierDifferenceShedderThreshold + +The threshold for the message throughput multiplier between the highest and lowest loaded brokers, +with a default value of 4, can trigger bundle unload when the maximum throughput is 4 times the minimum throughput. +For example, if the msgRate of broker 1 is 450MB, broker 2 is 100MB, and the difference in msgThrough +is 450/100=4.5>4 times, then the load balancer can unload the bundle from broker 1 to broker 2. + + +After introducing the algorithm of `UniformLoadShedder`, we can clearly obtain the following information: +#### load jitter +`UniformLoadShedder` does not have the logic to handle load jitter. For example, +when the traffic suddenly increases or decreases. This load data point is adopted, triggering a bundle unload. +However, the traffic of this topic will soon return to normal, so it is very likely to trigger a bundle unload again. +This type of bundle unload should be avoided. This kind of scenario is very common, actually. + +#### heterogeneous environment +`UniformLoadShedder` does not rely on indicators such as CPU usage and network card usage to determine high load +and low load brokers, but rather determines them based on message rate and traffic throughput size, +while `ThresholdShedder` and `OverloadShedder` rely on machine resource indicators such as CPU usage to determine. +If the cluster is heterogeneous, such as different machines with different hardware configurations, +or if there are other processes sharing resources on the machine where the broker is located, +`UniformLoadShedder` is likely to misjudge high and low load brokers, thereby migrating the load from high-performance +but low load brokers to low-performance but high load brokers. +Therefore, it is not recommended for users to use `UniformLoadShedder` in heterogeneous environments. + +#### slow load balancing +`UniformLoadShedder` will only unload the bundle from one of the highest loaded brokers at a time, +which may take a considerable amount of time for a large cluster to complete all load balancing tasks. +For example, if there are 100 high load brokers in the current cluster and 100 new machines to be added, +it is roughly estimated that it will take 100 shedding to complete the balancing. +However, since the execution time interval of the `LoadSheddingStrategy` policy is determined by the +configuration of `loadBalancerSheddingIntervalMinutes`, which defaults to once every 1 minute, +so it will take 100 minutes to complete all tasks. For users using large partition topics, their tasks +are likely to be disconnected multiple times within this 100 minutes, which greatly affects the user experience. + + +## ModularLoadManagerStrategy +The `LoadSheddingStrategy` strategy is used to unload bundles of high load brokers. However, in order to +achieve a good load balancing effect, it is necessary not only to "unload" correctly, but also to "load" correctly. +The `ModularLoadManagerStrategy` strategy is responsible for assigning bundles to brokers. +The coordination between `LoadSheddingStrategy` and `ModularLoadManagerStrategy` is also a key point worth paying attention to. + +### LeastLongTermMessageRate +The `LeastLongTermMessageRate` algorithm directly used the maximum resource usage of CPU and so on as the broker's score, +and reused the `OverloadShedder` configuration, `loadBalancerBrokerOverloadedThresholdPercentage`. +If the score is greater than it (default 85%), set `score=INF`; Otherwise, update the broker's score to the sum of the +message in and out rates obtained from the broker's long-term aggregation. +``` +score = longTerm MsgIn rate+longTerm MsgOut rate, +``` +Finally, randomly select a broker from the broker with the lowest score to return. If the score of each broker is INF, +randomly select broker from all brokers. + +The scoring algorithm in `LeastLongTermMessageRate` is essentially based on message rate. Although it initially examines +the maximum resource utilization, it is to exclude overloaded brokers only. +Therefore, in most cases, brokers are sorted based on the size of the message rate as a score, which results in the same +issues with heterogeneous environments, similar to `UniformLoadShedder`. + + +#### Effect of the combination of `LoadSheddingStrategy` and `LeastLongTermMessageRate` +Next, we will attempt to analyze the effect together with the `LoadSheddingStrategy`. +- **LeastLongTermMessageRate + OverloadShedder** +This is the initial combination, but due to some inherent flaws in `OverloadShedder`, **it is not recommended**. + +- **LeastLongTermMessageRate + ThresholdShedder** +This combination is even worse than `LeastLongTermMessageRate + OverloadShedder` and **is not recommended**. +Because `OverloadShedder` uses the maximum weighted resource usage and historical score to score brokers, +while LeastLongTermMessage Rate is scored based on message rate. Inconsistent unloading and placement criteria +can lead to incorrect load balancing execution. +This is also why a new placement strategy `LeastResourceUsageWithWeight` will be introduced later. + +- **LeastLongTermMessageRate + UniformLoadShedder** +This is **recommended**. Both uninstallation and placement policy are based on message rate, +but using message rate as a standard naturally leads to issues with heterogeneous environments. + + +### LeastResourceUsageWithWeight +`LeastResourceUsageWithWeight` uses the same scoring algorithm as `ThresholdShedder` to score brokers, which uses +weighted maximum resource usage and historical scores to calculate the current score. + +Next, select candidate brokers based on the configuration of `loadBalancerAverageResourceUsageDifferenceThresholdPercentage`. +If a broker's score plus this threshold is still not greater than the average score, the broker will be added to the +candidate broker list. After obtaining the candidate broker list, a broker will be randomly selected from it; +If there are no candidate brokers, randomly select from all brokers. + +For example, if the resource utilization rate of broker 1 is 10%, broker 2 is 30%, and broker 3 is 80%, +the average resource utilization rate is 40%. The placement strategy can choose Broker1 and Broker2 +as the best candidates, as the thresholds are 10, 10+10<=40, 30+10<=40. In this way, the bundles uninstalled +from broker 3 will be evenly distributed among broker 1 and broker 2, rather than being completely placed on broker 1. + +#### over placement problem +Over placement problem is that the bundle is placed on high load brokers and make them overloaded. + +In practice, it will be found that it is difficult to determine a suitable value for `loadBalancerAverageResourceUsageDifferenceThresholdPercentage`, +which often triggers a fallback global random selection logic. For example, if there are 6 brokers in the current +cluster, with scores of 40, 40, 40, 40, 69, and 70 respectively, the average score is 49.83. +Using the default configuration, there are no candidate brokers because 40+10>49.83. +Triggering a bottom-up global random selection logic and the bundle may be offloaded from the overloaded broker5 +to the overloaded broker6, or vice versa, **causing the over placement problem.** + +Attempting to reduce the configuration value to expand the random pool, such as setting it to 0, may also include some +overloaded brokers in the candidate broker list. For example, if there are 5 brokers in the current cluster with scores +of 10, 60, 70, 80, and 80 respectively, the average score is 60. As the configuration value is 0, then broker 1 and +broker 2 are both candidate brokers. If broker 2 shares half of the offloaded traffic, **it is highly likely to overload.** + +Therefore, it is difficult to configure the `LeastResourceUsageWithWeight` algorithm well to avoid incorrect load balancing. +Of course, if you want to use the `ThresholdShedder` algorithm, the combination of `ThresholdShedder+LeastResourceUsageWithWeight` +will still be superior to the combination of `ThresholdShedder+LeastLongTermMessageRate`, because at least the scoring algorithm +of `LeastResourceUsageWithWeight` is consistent with that of `ThresholdShedder`. + +#### why doesn't LeastLongTermMessage Rate have over placement problem? +The root of over placement problem is that the frequency of updating the load data is limited due to the performance +of zookeeper. If we assign a bundle to a broker, the broker's load will increase after a while, and it's load data +also need some time to be updated to leader broker. If there are many bundles unloaded in a shedding, +how can we assign these bundles to brokers? + +The most simple way is to assign them to the broker with the lowest load, but it may cause the over placement problem +as it is most likely that there is only one single broker with the lowest load. With all bundles assigned to this broker, +it will be overloaded. This is the reason why `LeastResourceUsageWithWeight` try to determine a candidate broker list +to avoid the over placement problem. But we also find that candidate broker list can be empty or include some overloaded +brokers, which will also cause the over placement problem. + +So why doesn't `LeastLongTermMessageRate` have over placement problem? The reason is that each time a bundle is assigned, +the bundle will be added into `PreallocatedBundleData`. When scoring a broker, not only will the long-term message rate +aggregated by the broker itself be used, but also the message rate of bundles in `PreallocatedBundleData` that have been +assigned to the broker but have not yet been reflected in the broker's load data will be calculated. + +For example, if there are two bundles with 20KB/s message rate to be assigned, and broker1 and broker2 at 100KB/s +and 110KB/s respectively. The first bundle is assigned to broker1, However, broker1's load data will not be updated +in the short term. Before the load data is updated, `LeastLongTermMessageRate` try to assign the second bundle. +At this time, the score of broker1 is 100+20=120KB/s, where 20KB/s is the message rate of the first bundle +from `PreallocatedBundleData`. As broker1's score is greater than broker2, the second bundle will be assigned to broker2. + +**`LeastLongTermMessageRate` predict the load of the broker after the bundle is assigned to avoid the over placement problem.** + +**Why doesn't `LeastResourceUsageWithWeight` have this feature? Because it is not possible to predict how much resource +utilization a broker will increase when loading a bundle. All algorithms scoring brokers based on resource utilization +can't fix the over placement problem with this feature.** +So `LeastResourceUsageWithWeight` try to determine a candidate broker list to avoid the over placement problem, which is +proved to be not a good solution. + + +#### over unloading problem +Over unloading problem is that the load offloaded from high load brokers is too much and make them underloaded. + +Finally, let's talk about the issue of historical weighted scoring algorithms. The historical weighted scoring algorithm +is used by the `ThresholdShedder` and `LeastResourceUsageWithWeight` algorithms, as follows: +``` +HistoryUsage=historyUsage=null? ResourceUsage: historyUsage * historyPercentage+(1- historyPercentage) * resourceUsage; +``` +The default value of historyPercentage is 0.9, indicating that the score calculated last time has a significant impact on the current score. +The current maximum resource utilization only accounts for 10%, which is to solves the problem of load jitter. +However, introducing this algorithm has its side effects, such as over unloading problem. + +For example, there is currently one broker1 in the cluster with a load of 90%, and broker2 is added with a current load of 10%. +- At the first execution of shedding: broker1 scores 90, broker2 scores 10. For simplicity, assuming that the algorithm will +move some bundles to make their load the same, thus the true load of broker 1 and broker 2 become 50 after load shedding is completed. +- At the second execution of shedding: broker1 scores 90*0.9+50*0.1=86, broker2 scores 10*0.9+50*0.1=14. +**Note that the actual load of broker1 here is 50, but it is overestimated as 86!** +**The true load of broker2 is also 50, but it is underestimated at 14!** +Due to the significant difference in ratings between the two, although their actual loads are already the same, +broker1 will continue to unload traffic corresponding to 36 points from broker1 to broker2, +resulting in broker1's actual load score becoming 14, broker2's actual load score becoming 86. + +- At the third execution of shedding: broker1 scored 86*0.9+14*0.1=78.8, broker2 scored 14*0.9+86*0.1=21.2. +It is ridiculous that broker1 is still considered overloaded, and broker2 is still considered underloaded. +All loads in broker1 are moved to broker2, which is the over unloading problem. + +Although this example is an idealized theoretical analysis, we can still see that using historical scoring algorithms +can seriously overestimate or underestimate the true load of the broker. Although it can avoid the problem of load jitter, +it will introduce a more serious and broader problem: **overestimating or underestimating the true load of the broker, +leading to incorrect load balancing execution**. + + +## Summary +Based on the previous analysis, although we have three shedding strategies and two placement strategies +that can generate 6 combinations of 3 * 2, we actually only have two recommended options: +- ThresholdShedder + LeastResourceUsageWithWeight +- UniformLoadShedder + LeastLongTermMessageRate + +These two options each have their own advantages and disadvantages, and users can choose one according to +their requirements. The following table summarizes the advantages and disadvantages of the two options: + +| Combination | heterogeneous environment | load jitter | over placement problem | over unloading problem | slow load balancing | +|---------------------------------------------|---------------------------|------------|-----------------------|-----------------------|---------------------| +| ThresholdShedder + LeastResourceUsageWithWeight | normal(1) | good | bad | bad | normal(1) | +| UniformLoadShedder + LeastLongTermMessageRate | bad(2) | bad | good | good | normal(1) | + +1. In terms of adapting to heterogeneous environments, `ThresholdShedder+LeastResourceUsageWithWeight` can +only be rated as `normal`. This is because `ThresholdShedder` is not fully adaptable to heterogeneous environments. +Although it does not misjudge overloaded brokers as underloaded, heterogeneous environments can still have a +significant impact on the load balancing effect of `ThresholdShedder`. +For example, there are three brokers in the current cluster with resource utilization rates of 10, 50, and 70, respectively. +Broker1 and Broker2 are isomorphic. Though Broker3 don't bear any load, its resource utilization rate has +reached to 70 due to the deployment of other processes at the same machine. +At this point, we would like broker 1 to share some of the pressure from broker2, but since the average load is +43.33, 43.33+10>50, broker2 will not be judged as overloaded, and overloaded broker 3 also has no traffic to +unload, causing the load balancing algorithm to be in an inoperable state. + +2. In the same scenario, if `UniformLoadShedder+LeastLongTermMessageRate` is used, the problem will be more +severe, as some of the load will be offloaded from broker2 to broker3. As a result, the performance of those +topics in broker3 services will experience significant performance degradation. +Therefore, it is not recommended to run Pulsar in heterogeneous environments as current load balancing algorithms +cannot adapt too well. If it is unavoidable, it is recommended to choose `ThresholdShedder+LeastResourceUsageWithWeight`. + +3. In terms of load balancing speed, although `ThresholdShedder+LeastResourceUsageWithWeight` can unload the load +of all overloaded brokers at once, historical scoring algorithms can seriously affect the accuracy of load +balancing decisions. Therefore, in reality, it also requires multiple load balancing executions to finally +stabilize. This is why the load balancing speed of `ThresholdShedder+LeastResourceUsageWithWeight` is rated as `normal`. + +4. In terms of load balancing speed, `UniformLoadShedder+LeastLongTermMessageRate` can only unload the load of one +overloaded broker at a time, so it takes a long time to complete load balancing when there are many brokers, +so it is also rated as `normal`. + + +# Motivation + +The current load balance algorithm has some serious problems, such as load jitter, heterogeneous environment, slow load balancing, etc. +This PIP aims to introduce a new load balance algorithm `AvgShedder` to solve these problems. + +# Goals + +Introduce a new load balance algorithm `AvgShedder` that can solve the problems of load jitter, heterogeneous environment, slow load balancing, etc. + + +# High Level Design + +## scoring criterion +First of all, to determine high load brokers, it is necessary to rate and sort them. +Currently, there are two scoring criteria: +- Resource utilization rate of broker +- The message rate and throughput of the broker +Based on the previous analysis, it can be seen that scoring based on message rate and throughput will face +the same problem as `UniformLoadShedder` in heterogeneous environments, while scoring based on resource utilization +rate will face the over placement problem like `LeastResourceUsageWithWeight`. + +**To solve the problem of heterogeneous environments, we use the resource utilization rate of the broker as the scoring criterion.** + + +## binding shedding and placement strategies +So how can we avoid the over placement problem? **The key is to bind the shedding and placement strategies together.** +If every bundle unloaded from the high load broker is assigned to the right low load broker in shedding strategy, +the over placement problem will be solved. + +For example, if the broker rating of the current cluster is 20,30,52,80,80, and the shedding and placement strategies are decoupled, +the bundles will be unloaded from the two brokers with score of 80, and then all these bundles will be placed on the broker with a +score of 20, causing the over placement problem. + +If the shedding and placement strategies are coupled, one broker with 80 score can unload some bundles to a broker with 20 score, +and another broker with 80 score can unload the bundle to the broker with 30 score. In this way, we can avoid the over placement problem. + + +## evenly distributed traffic between the highest and lowest loaded brokers +We will first pick out the highest and lowest loaded brokers, and then evenly distribute the traffic between them. + +For example, if the broker rating of the current cluster is 20,30,52,70,80, and the message rate of the highest loaded broker is 1000, +the message rate of the lowest loaded broker is 500. We introduce a threshold to whether trigger the bundle unload, for example, +the threshold is 40. As the difference between the score of the highest and lowest loaded brokers is 100-50=50>40, +the shedding strategy will be triggered. + +To achieve the goal of evenly distributing the traffic between the highest and lowest loaded brokers, the shedding strategy will +try to make the message rate of two brokers the same, which is (1000+500)/2=750. The shedding strategy will unload 250 message rate from the +highest loaded broker to the lowest loaded broker. After the shedding strategy is completed, the message rate of two brokers will be +same, which is 750. + + +## improve the load balancing speed +As we mentioned earlier in `UniformLoadShedder`, if strategy only handles one high load broker at a time, it will take a long time to +complete all load balancing tasks. Therefore, we further optimize it by matching multiple pairs of high and low load brokers in +a single shedding. After sorting the broker scores, the first and last place are paired, the second and and the second to last are paired, +and so on. When the score difference between the two paired brokers is greater than the threshold, the load will be evenly distributed +between the two, which can solve the problem of slow speed. + +For example, if the broker rating of the current cluster is 20,30,52,70,80, we will pair 20 and 80, 30 and 70. As the difference between +the two paired brokers is 80-20=60, 70-30=40, which are both greater than the threshold 40, the shedding strategy will be triggered. + + +## handle load jitter with multiple hits threshold +What about the historical weighting algorithm used in `ThresholdShedder`? It is used to solve the problem of load jitter, but previous +analysis and experiments have shown that it can bring serious negative effects, so we can no longer use this method to solve the +problem of load jitter. + +We mimic the way alarms are triggered: the threshold is triggered multiple times before the bundle unload is finally triggered. +For example, when the difference between a pair of brokers exceeds the threshold three times, load balancing is triggered. + +## high and low threshold +In situations of cluster rolling restart or expansion, there is often a significant load difference between +different brokers, and we hope to complete load balancing more quickly. + +Therefore, we introduce two thresholds: +- loadBalancerAvgShedderLowThreshold, default value is 15 +- loadBalancerAvgShedderHighThreshold, default value is 40 + +Two thresholds correspond to two continuous hit count requirements: +- loadBalancerAvgShedderHitCountLowThreshold, default value is 8 +- loadBalancerAvgShedderHitCountHighThreshold, default value of 2 + +When the difference in scores between two paired brokers exceeds the `loadBalancerAvgShedderLowThreshold` by +`loadBalancerAvgShedderHitCountLowThreshold` times, or exceeds the `loadBalancerAvgShedderHighThreshold` by +`loadBalancerAvgShedderHitCountHighThreshold` times, a bundle unload is triggered. +For example, with the default value, if the score difference exceeds 15, it needs to be triggered 8 times continuously, +and if the score difference exceeds 40, it needs to be triggered 2 times continuously. + +The larger the load difference between brokers, the smaller the number of times it takes to trigger bundle unloads, +which can adapt to scenarios such as cluster rolling restart or expansion. + +## placement strategy +As mentioned earlier, `AvgShedder` bundles the shedding and placement strategies, and a bundle has already determined +its next owner broker based on the shedding strategy during shedding. But we not only use placement strategies after +executing shedding, but also need to use placement strategies to assign bundles during cluster initialization, rolling +restart, and broker shutdown. So how should we assign these bundles without shedding strategies? + +We use a hash allocation method: hash mapping a random number to broker. Hash mapping roughly conforms to +a uniform distribution, so bundles will be roughly evenly distributed across all brokers. However, due to the different +throughput between different bundles, the cluster will exhibit a certain degree of imbalance. However, this problem is +not significant, and the subsequent balancing can be achieved through shedding strategies. Moreover, the frequency of +cluster initialization, rolling restart, and broker shutdown scenarios is not high, so the impact is slight. + +## summary +In summary, `AvgShedder` can solve the problems of load jitter, heterogeneous environment, slow load balancing, etc. +Following table summarizes the advantages and disadvantages of the three options: + +| Combination | heterogeneous environment | load jitter | over placement problem | over unloading problem | slow load balancing | +|---------------------------------------------|------------------------|------------|-----------------------|-----------------------|--------------| +| ThresholdShedder + LeastResourceUsageWithWeight | normal | good | bad | bad | normal | +| UniformLoadShedder + LeastLongTermMessageRate | bad | bad | good | good | normal | +| AvgShedder | normal | good | good | good | good | + + +# Detailed Design + +### Configuration + +To avoid introducing too many configurations when calculating how much traffic needs to be unloaded, `AvgShedder` reuses the +following three `UniformLoadShedder` configurations: +``` + @FieldContext( + dynamic = true, + category = CATEGORY_LOAD_BALANCER, + doc = "In the UniformLoadShedder strategy, the minimum message that triggers unload." + ) + private int minUnloadMessage = 1000; + + @FieldContext( + dynamic = true, + category = CATEGORY_LOAD_BALANCER, + doc = "In the UniformLoadShedder strategy, the minimum throughput that triggers unload." + ) + private int minUnloadMessageThroughput = 1 * 1024 * 1024; + + @FieldContext( + dynamic = true, + category = CATEGORY_LOAD_BALANCER, + doc = "In the UniformLoadShedder strategy, the maximum unload ratio." + ) + private double maxUnloadPercentage = 0.2; +``` + +The `maxUnloadPercentage` controls the allocation ratio. Although the default value is 0.2, our goal is to evenly distribute the +pressure between two brokers. Therefore, we set the value to 0.5, so that after load balancing is completed, the message rate/throughput +of the two brokers will be almost equal. + +The following configurations are introduced to control the shedding strategy: +``` + @FieldContext( + dynamic = true, + category = CATEGORY_LOAD_BALANCER, + doc = "The low threshold for the difference between the highest and lowest loaded brokers." + ) + private int loadBalancerAvgShedderLowThreshold = 15; + + @FieldContext( + dynamic = true, + category = CATEGORY_LOAD_BALANCER, + doc = "The high threshold for the difference between the highest and lowest loaded brokers." + ) + private int loadBalancerAvgShedderHighThreshold = 40; + + @FieldContext( + dynamic = true, + category = CATEGORY_LOAD_BALANCER, + doc = "The number of times the low threshold is triggered before the bundle is unloaded." + ) + private int loadBalancerAvgShedderHitCountLowThreshold = 8; + + @FieldContext( + dynamic = true, + category = CATEGORY_LOAD_BALANCER, + doc = "The number of times the high threshold is triggered before the bundle is unloaded." + ) + private int loadBalancerAvgShedderHitCountHighThreshold = 2; +``` + + + +# Backward & Forward Compatibility + +Fully compatible. + +# General Notes + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/cy39b6jp38n38zyzd3bbw8b9vm5fwf3f +* Mailing List voting thread: https://lists.apache.org/thread/2v9fw5t5m5hlmjkrvjz6ywxjcqpmd02q diff --git a/pip/pip-366.md b/pip/pip-366.md new file mode 100644 index 0000000000000..78e7ad60de9b0 --- /dev/null +++ b/pip/pip-366.md @@ -0,0 +1,70 @@ +# PIP-366: Support to specify different config for Configuration and Local Metadata Store + +# Background knowledge + +Pulsar metadata store maintains all the metadata, configuration, and coordination of a Pulsar cluster, such as topic metadata, schema, broker load data, and so on. + +The metadata store of each Pulsar instance should contain the following two components: + +- A local metadata store ensemble (`metadataStoreUrl`) that stores cluster-specific configuration and coordination, such as which brokers are responsible for which topics as well as ownership metadata, broker load reports, and BookKeeper ledger metadata. +- A configuration store quorum (`configurationMetadataStoreUrl`) stores configuration for clusters, tenants, namespaces, topics, and other entities that need to be globally consistent. + +# Motivation + +When using Geo-Replication and global configuration store for configuration global consistency, the configuration store's config may be different from the local metadata store's config. For example, the configuration store may have a different set of ZooKeeper servers than the local metadata store. + +The global configuration store may deploy in a different data center, and the local metadata store may be deployed in the same data center as the Pulsar broker. In this case, the global configuration store may need to use TLS and authentication to protect the connection to metadata store server, while the local metadata store may not need to use TLS and authentication. + +However, the current implementation of Pulsar only supports configuring different metadata store url for the local metadata store and the configuration store. This limitation makes it impossible to support the above scenario. + +# Goals + +## In Scope + +- Support specifying different configurations for the local metadata store and the configuration store. + +# Detailed Design + +## Design & Implementation Details + +Pulsar support `metadataStoreConfigPath` configuration, but it only supports for `RocksdbMetadataStore`, and it is not able to specify different configuration for Configuration Metadata Store. + +```java + @FieldContext( + category = CATEGORY_SERVER, + doc = "Configuration file path for local metadata store. It's supported by RocksdbMetadataStore for now." + ) + private String metadataStoreConfigPath = null; +``` + +Therefore, we need to add a new configuration `configurationStoreConfigPath` for `ConfigurationMetadataStore`, and the `metadataStoreConfigPath` will be still use for `LocalMetadataStore`. + +```java + @FieldContext( + category = CATEGORY_SERVER, + doc = "Configuration file path for configuration metadata store." + ) + private String configurationStoreConfigPath = null; +``` + +When the `configurationStoreConfigPath` are not set, the `metadataStoreConfigPath` will be used as the configuration file path for the configuration store. + +For each metadata store implementation, we need pass the corresponding configuration file path to the metadata store. For example, for ZKMetadataStore, we can specify config when create the Zookeeper client. + +```java + protected ZooKeeper createZooKeeper() throws IOException { + return new ZooKeeper(connectString, sessionTimeoutMs, watcherManager, allowReadOnlyMode, /** Add the config here **/ new ZKClientConfig(configPath)); + } +``` + +# Backward & Forward Compatibility + +Fully compatible. + +# Links + + +* Mailing List discussion thread: https://lists.apache.org/thread/98ggo1zg1k7dbyx8wr9bc8onm10p16c6 +* Mailing List voting thread: https://lists.apache.org/thread/wm30dy9bkhxxmmcb0v9ftb56ckpknrfr diff --git a/pip/pip-368.md b/pip/pip-368.md new file mode 100644 index 0000000000000..06bba2c12761c --- /dev/null +++ b/pip/pip-368.md @@ -0,0 +1,185 @@ +# PIP-368: Support lookup based on the lookup properties + +# Background knowledge + +## How Pulsar Lookup Works + +Before producing or consuming messages, a Pulsar client must first find the broker responsible for the topic. This +happens through the lookup service. The client sends a `CommandLookupTopic` request with the topic name to the broker +lookup service. + +On the broker side, the broker will register itself to the metadata store using a distributed lock with the value +of [`BrokerLookupData`](https://github.com/apache/pulsar/blob/7fe92ac43cfd2f2de5576a023498aac8b46c7ac8/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java#L34-L44) +when starting. The lookup service will first choose the owner broker. And then retrieve the `BrokerLookupData` of the +owner broker and finally return to the client. The client then interacts with this broker to produce or consume +messages. + +Users can customize the lookup process by setting a custom load manager in the `loadManagerClassName` configuration. + +# Motivation + +Currently, the lookup process uses only the topic name as its parameter. However, to enhance this process, it's +beneficial for clients to provide additional information. This could be done by introducing the `lookupProperties` field +in the client configuration. Clients can then share these properties with the broker during lookup. + +On the broker side, the broker could also contain some properties that are used for the lookup. We can also support the +lookupProperties for the broker. The broker can use these properties to make a better decision on which broker to +return. + +Here is the rack-aware lookup scenario for using the client properties for the lookup: +Assuming there are two brokers that broker-0 configures the lookup property "rack" with "A" and broker-1 configures the +lookup property "rack" with "B". By using the lookup properties, clients can supply rack information during the lookup, +enabling the broker to identify and connect them to the nearest broker within the same rack. If a client that configures +the "rack" property with "A" connects to a lookup broker, the customized load manager can determine broker-0 as the +owner broker since the broker and the client have the same rack property. + +# Goals + +## In Scope + +- Enable setting up lookup properties in both client and broker configurations. +- Allow clients to provide extra lookup information to brokers during the lookup process. + +## Out of Scope + +- The implementation of the rack-aware lookup scenario. + +# High Level Design + +Add new configuration `lookupProperties` to the client. While looking up the broker, the client will send the properties +to the broker through `CommandLookupTopic` request. + +The `lookupProperties` will then be added to the `LookupOptions`. The Load Manager implementation can access +the `properties` through `LookupOptions` to make a better decision on which broker to return. + +The properties are used only when the protocol is the binary protocol, starting with `pulsar://` or `pulsar+ssl://`, or +if the `loadManagerClassName` in the broker is a class that implements the `ExtensibleLoadManager` interface. + +To support configuring the `lookupProperties` on the broker side, introduce a new broker +configuration `lookupPropertyPrefix`. Any broker configuration properties that start with the `lookupPropertyPrefix` +will be included into the `BrokerLookupData` and be persisted in the metadata store. The broker can use these properties +during the lookup. + +In this way, to support the rack-aware lookup scenario mentioned in the "Motivation" part, the client can set the rack +information in the client `lookupProperties`. Similarly, the broker can also set the rack information in the broker +configuration like `lookup.rack`. The `lookup.rack` will be stored in the `BrokerLookupData`. A customized load manager +can then be implemented. For each lookup request, it will go through the `BrokerLookupData` for all brokers and select +the broker in the same rack to return. + +# Detailed Design + +## Design & Implementation Details + +## Public-facing Changes + +### Configuration + +Add new configuration `lookupProperties` to the `ClientBuilder`. + +```java +/** + * Set the properties used for topic lookup. + *

+ * When the broker performs topic lookup, these lookup properties will be taken into consideration in a customized load + * manager. + *

+ * Note: The lookup properties are only used in topic lookup when: + * - The protocol is binary protocol, i.e. the service URL starts with "pulsar://" or "pulsar+ssl://" + * - The `loadManagerClassName` config in broker is a class that implements the `ExtensibleLoadManager` interface + */ +ClientBuilder lookupProperties(Map properties); +``` + +Add new broker configuration `lookupPropertyPrefix` to the `ServiceConfiguration`: + +```java + +@FieldContext( + category = CATEGORY_SERVER, + doc = "The properties whose name starts with this prefix will be uploaded to the metadata store for " + + " the topic lookup" +) +private String lookupPropertyPrefix = "lookup."; +``` + +### Binary protocol + +Add `properties` field to the `CommandLookupTopic`. Now the `CommandLookupTopic` will look like: + +```protobuf +message KeyValue { + required string key = 1; + required string value = 2; +} + +message CommandLookupTopic { + required string topic = 1; + required uint64 request_id = 2; + optional bool authoritative = 3 [default = false]; + optional string original_principal = 4; + optional string original_auth_data = 5; + optional string original_auth_method = 6; + optional string advertised_listener_name = 7; + // The properties used for topic lookup + repeated KeyValue properties = 8; +} +``` + +When the client lookups a topic, it will set the client `lookupPorperties` to the `CommandLookupTopic.properties`. + +### Public API + +Currently, there is a public method `assign` in the `ExtensibleLoadManager` interface that will accept +the `LookupOptions` to lookup the topic. + +```java +public interface ExtensibleLoadManager { + CompletableFuture> assign(Optional topic, + ServiceUnitId serviceUnit, + LookupOptions options); +} +``` + +In this proposal, the `properties` will be added to the `LookupOptions`: + +```java +public class LookupOptions { + // Other fields are omitted ... + + // The properties used for topic lookup + private final Map properties; +} +``` + +The `LookupOptions.properties` will be set to the value of `CommandLookupTopic.properties`. +This way, the custom `ExtensibleLoadManager` implementation can retrieve the `properties` from the `LookupOptions` to +make a better decision on which broker to return. + +# Monitoring + +No new metrics are added in this proposal. + +# Security Considerations + +No new security considerations are added in this proposal. + +# Backward & Forward Compatibility + +## Revert + +No changes are needed to revert to the previous version. + +## Upgrade + +No other changes are needed to upgrade to the new version. + +# Alternatives + +None + +# General Notes + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/7n2gncxk3c5q8dxj8fw9y5gcwg6jjg6z +* Mailing List voting thread: https://lists.apache.org/thread/z0t3dyqj27ldm8rs6nl5jon152ohghvw diff --git a/pip/pip-369.md b/pip/pip-369.md new file mode 100644 index 0000000000000..9aeb598110d72 --- /dev/null +++ b/pip/pip-369.md @@ -0,0 +1,124 @@ +# PIP-369: Flag based selective unload on changing ns-isolation-policy + +# Background knowledge + +In Apache Pulsar, namespace isolation policies are used to limit the ownership of certain subsets of namespaces to specific broker groups. +These policies are defined using regular expressions to match namespaces and specify primary and secondary broker groups along with failover policy configurations. +This ensures that the ownership of the specified namespaces is restricted to the designated broker groups. + +For more information, refer to the [Pulsar documentation on namespace isolation](https://pulsar.apache.org/docs/next/administration-isolation/#isolation-levels). + +# History/Context +In Apache Pulsar 2.7.1+, there was a flag introduced (`enableNamespaceIsolationUpdateOnTime`) that controlled whether to unload namespaces or not when a namespace isolation policy is applied. https://github.com/apache/pulsar/pull/8976 + +Later on, in 2.11, rework was done as part of [PIP-149](https://github.com/apache/pulsar/issues/14365) to make get/set isolationData calls async, +which resulted in namespaces to always get unloaded irrespective of `enableNamespaceIsolationUpdateOnTime` config, not adhering to this config at all. + +And now in 3.3, `enableNamespaceIsolationUpdateOnTime` broker config was deprecated as it no longer serves any purpose. https://github.com/apache/pulsar/pull/22449 + +# Motivation + +In Apache Pulsar 3.x, changing a namespace isolation policy results in unloading all namespace bundles that match the namespace's regular expression provided in the isolation policy. +This can be problematic for cases where the regex matches a large subset of namespaces, such as `tenant-x/.*`. +One of such case is mentioned on this issue [#23092](https://github.com/apache/pulsar/issues/23092) where policy change resulted in 100+ namespace bundles to get unloaded. +And broker exhausted all the available connections due to too many unload calls happening at once resulting in 5xx response. +Other issues that happens with this approach are huge latency spikes as topics are unavailable until bundles are loaded back, increasing the pending produce calls. +The only benefit this approach serves is ensuring that all the namespaces matching the policy regex will come to correct broker group. +But when namespace bundles are already on the correct broker group (according to the policy), unloading those namespaces doesn't serve any purpose. + +This PIP aims to address the need to either prevent unnecessary unloading or provide a more granular approach to determine what should be unloaded. + +Some of the cases covered by this PIP are discussed in [#23094](https://github.com/apache/pulsar/issues/23094) by @grssam. +> - unload nothing as part of the set policy call +> - unload every matching namespace as part of the policy set call +> - unload only the changed namespaces (newly added + removed) + +# Goals + +## In Scope +This PIP proposes a flag-based approach to control what should be unloaded when an isolation policy is applied. +The possible values for this flag are: +- **all_matching**: Unload all the namespaces that matches either old or new policy change. +- **changed**: Only unload namespaces that are either added or removed due to the policy change. +- **none**: Do not unload anything. Unloading can occur naturally due to load balancing or can be done manually using the unload admin call. + +This flag will be a part of isolation policy data with defaults. Objective is to keep the default behavior unchanged on applying the new policy. + +## Out of Scope + +Applying concurrency reducer to limit how many async calls will happen in parallel is out of the scope for this PIP. +This should be addressed in a separate PIP, as solving the issue of infinite asynchronous calls probably requires changes to broker configurations and is a problem present in multiple areas. + +# Detailed Design + +## Design & Implementation Details + +A new flag will be introduced in `NamespaceIsolationData`. + +```java +enum UnloadScope { + all_matching, // unloads everything, OLD ⋃ NEW + changed, // unload namespaces delta, (new ⋃ old) - (new ∩ old) + none, // skip unloading anything, ϕ +}; +``` +Filters will be added based on the above when namespaces are selected for unload in set policy call. +`UnloadScope.all_matching` will be the default in current version. + +> **_NOTE:_** +> For 3.x unchanged behaviour, the `all_matching` UnloadScope option should only unload namespaces matching new policy (NEW). This matches the current behavior and maintains backward compatibility from implementation POV. +> +> For 4.x, +> 1. The behaviour for the `all_matching` flag should change to unload everything matching either the old or new policy (union of both). +> 2. The default flag value should be `changed`, so accidentally missing this flag while applying the policy shouldn't impact workloads already on the correct broker group. + +### Public API + +A new flag will be added in the NamespaceIsolationData. This changes the request body when set policy API is called. +To keep things backwards compatible, `unload_scope` will be optional. API params will remain unchanged. + +Path: `/{cluster}/namespaceIsolationPolicies/{policyName}` +```json +{ + "policy-name": { + "namespaces": [...], + "primary": [...], + "secondary": [...], + "auto_failover_policy": { + ... + }, + "unload_scope": "all_matching|changed|none" + } +} +``` + +### CLI + +```shell +# set call will have an optional flag. Sample command as shown below: +# +pulsar-admin ns-isolation-policy set cluster-name policy-name --unload-scope none +# Possible values for unload-scope: [all_matching, changed, none] +``` + +# Backward & Forward Compatibility + +Added flag is optional, that doesn't require any changes to pre-existing policy data. If the flag is not present then default value shall be considered. + +# Alternatives + +Boolean flag passed during set policy call to either unload the delta namespaces (removed and added) without affecting unchanged namespaces or unload nothing. PR: https://github.com/apache/pulsar/pull/23094 + +Limitation: This approach does not consider cases where unloading is needed for every matching namespace as part of the policy set call. +Manual unloading would be required for unchanged namespaces not on the correct broker group. + +# Links + + +* Mailing List discussion thread: https://lists.apache.org/thread/6f8k1typ48817w65pjh6orhks1smpbqg +* Mailing List voting thread: https://lists.apache.org/thread/0pj3llwpcy73mrs5s3l5t8kctn2mzyf7 + + +PS: This PIP should get cherry-picked to 3.0.x as it provides a way to resolve the bug mentioned at [#23092](https://github.com/apache/pulsar/issues/23092) which exist in all the production system today. \ No newline at end of file diff --git a/pip/pip-370.md b/pip/pip-370.md new file mode 100644 index 0000000000000..a29d556143200 --- /dev/null +++ b/pip/pip-370.md @@ -0,0 +1,108 @@ +# PIP-370: configurable remote topic creation in geo-replication + +# Background knowledge + +**The current topic creation behavior when enabling Geo-Replication** +Users using Geo-Replication backup data across multiple clusters, as well as Admin APIs related to Geo-Replication and internal replicators of brokers, will trigger topics of auto-creation between clusters. +- For partitioned topics. + - After enabling namespace-level Geo-Replication: the broker will create topics on the remote cluster automatically when calling `pulsar-admin topics create-partitioned-topic`. It does not depend on enabling `allowAutoTopicCreation`. + - When enabling topic-level Geo-Replication on a partitioned topic: the broker will create topics on the remote cluster automatically. It does not depend on enabling `allowAutoTopicCreation`. + - When calling `pulsar-admin topics update-partitioned-topic -p {partitions}`, the broker will also update partitions on the remote cluster automatically. +- For non-partitioned topics and partitions of partitioned topics. + - The internal Geo-Replicator will trigger topics auto-creation for remote clusters. **(Highlight)** It depends on enabling `allowAutoTopicCreation`. In fact, this behavior is not related to Geo-Replication, it is the behavior of the internal producer of Geo-Replicator, + +# Motivation + +In the following scenarios, automatic topic creation across clusters is problematic due to race conditions during deployments, and there is no choice that prevents pulsar resource creation affects each other between clusters. + +- Users want to maintain pulsar resources manually. +- Users pulsar resources using `GitOps CD` automated deployment, for which + - Clusters are deployed simultaneously without user intervention. + - Each cluster is precisely configured from git repo config variables - including the list of all tenants/namespaces/topics to be created in each cluster. + - Clusters are configured to be exact clones of each other in terms of pulsar resources. + +**Passed solution**: disable `allowAutoTopicCreation`, the APIs `pulsar-admin topics create-partitioned-topic` still create topics on the remote cluster when enabled namespace level replication, the API `enable topic-level replication` still create topics, And the internal replicator will keep printing error logs due to a not found error. + +# Goals + +- **Phase 1**: Introduce a flag to disable the replicators to automatically trigger topic creation. +- **Phase 2**: Move all topic creation/expand-partitions behaviors related to Replication to the internal Replicator, pulsar admin API that relates to pulsar topics management does not care about replication anymore. + - Move the topic creation operations from `pulsar-admin topics create-partitioned-topic` and `pulsar-admin topics set-replication-clusters` to the component Replicator in the broker internal. + - (The same as before)When calling `pulsar-admin topics update-partitioned-topic -p {partitions}`, the broker will also update partitions on the remote cluster automatically. + +Note: the proposal will only focus on phase one, and the detailed design for phase two with come up with another proposal. + +# Detailed Design + +## Configuration + +**broker.conf** +```properties +# Whether the internal replication of the local cluster will trigger topic auto-creation on the remote cluster. +# 1. After enabling namespace-level Geo-Replication: whether the local broker will create topics on the remote cluster automatically when calling `pulsar-admin topics create-partitioned-topic`. +# 2. When enabling topic-level Geo-Replication on a partitioned topic: whether the local broker will create topics on the remote cluster. +# 3. Whether the internal Geo-Replicator in the local cluster will trigger non-persistent topic auto-creation for remote clusters. +# It is not a dynamic config, the default value is "true" to preserve backward-compatible behavior. +createTopicToRemoteClusterForReplication=true +``` + +## Design & Implementation Details + +### Phase 1: Introduce a flag to disable the replicators to automatically trigger topic creation. +- If `createTopicToRemoteClusterForReplication` is set to `false`. + 1. After enabling namespace-level Geo-Replication: the broker will not create topics on the remote cluster automatically when calling `pulsar-admin topics create-partitioned-topic`. + 2. When enabling topic-level Geo-Replication on a partitioned topic: broker will not create topics on the remote cluster automatically. + 3. The internal Geo-Replicator will not trigger topic auto-creation for remote clusters, it just keeps retrying to check if the topic exists on the remote cluster, once the topic is created, the replicator starts. + 4. It does not change the behavior of creating subscriptions after enabling `enableReplicatedSubscriptions`, the subscription will also be created on the remote cluster after users enable. `enableReplicatedSubscriptions`. + 5. The config `allowAutoTopicCreation` still works for the local cluster as before, it will not be affected by the new config `createTopicToRemoteClusterForReplication`. +- If `createTopicToRemoteClusterForReplication` is set to `true`. + a. All components work as before, see details: `Motivation -> The current topic creation behavior when enabling Geo-Replication` + +### Phase 2: The replicator will check remote topics' partitioned metadata and update partitions in the remote cluster to the same as the current cluster if needed. +- If `createTopicToRemoteClusterForReplication` is set to `false`. + - The behavior is the same as Phase 1. +- If `createTopicToRemoteClusterForReplication` is set to `true`. + - Pulsar admin API that relates to pulsar topics management does not care about replication anymore. + - When a replicator for a topic partition starts, it checks the partitioned metadata in the remote cluster first and updates partitions in the remote cluster to the same as the current cluster if needed. Seem the example as follows: + +| `partitions` of local cluster | `partitions` of remote cluster | After `PIP-370 Phase 2` | Before `PIP-370 Phase 2` | +| --- | --- | --- | --- | +| `2` | no topic exists | create a partitioned topic with `2` partitions in the remote cluster | the replicator will only trigger partition creation (`{topic}-partition-0` and `{topic}-partition-1`), and will not care about partitioned metadata. | +| `2` | `1`| **In dispute:** The replicator copies messages from `partition-0` to the remote cluster, does not copy any data for `partition-1` and just prints error logs in the background. | the replicator will only trigger partition creation (`{topic}-partition-0` and `{topic}-partition-1`), and the partitioned metadata in the remote cluster is still `1` | +| `2` | `2` | modifies nothing. | The same as "After `PIP-370 Phase 2`" | +| `2` | `>2` | **In dispute:** modifies nothing, the messages will be copied to the same partition in the remote cluster, and no message will be copied to the partition who is larger than `2` in the remote cluster | The same as "After `PIP-370 Phase 2`" | +| `2` | `0`(non-partitioned topic) | **In dispute:** The replicator does not copy any data and just prints error logs in the background. | the replicator will only trigger partition creation (`{topic}-partition-0` and `{topic}-partition-1`), then users will get `3` non-partitioned topics: `[{tp}, {topic}-partition-0, {topic}-partition-1`. | +| `0`(non-partitioned topic) | `0`(non-partitioned topic) | Copy data normally | It is the same as before `PIP-370`. | +| `0`(non-partitioned topic) | no topic exists | create a non-partitioned topic in the remote cluster. | It is the same as before `PIP-370`. | +| `0`(non-partitioned topic) | `>=1` | **In dispute:** The replicator does not copy any data and just prints error logs in the background. | The replicator will only trigger a non-partitioned topic's creation, then users will get `1` non-partitioned topic and `1` partitioned topic. | + +## Metrics + + +| Name | Description | Attributes | Units| +| --- |---------------------------------------------------------------------------------------------|---------------------------| --- | +| `pulsar_replication_disconnected_count` | Counter. The number of replicators. | cluster, namespace, topic | - | + + +# Monitoring +- If `pulsar_broker_replication_disconnected_count` keeps larger than `0` for a period of time, it means some replicators do not work, we should push an alert out. + +# Backward & Forward Compatibility + +## Regarding to Phase-1 +This PIP guarantees full compatibility with default settings(the default value of `createTopicToRemoteClusterForReplication` is `true`). If you want to cherry-pick PIP-370 for another branch in the future, you need to cherry-pick PIP-344 as well. Because the behavior of disables `createTopicToRemoteClusterForReplication` depends on the API `PulsarClient.getPartitionsForTopic(String topic, boolean metadataAutoCreationEnabled)`, which was introduced by [PIP-344](https://github.com/apache/pulsar/blob/master/pip/pip-344.md). + +## Regarding to Phase-2 +The two scenarios are as follows, the replication will not work as before, which will lead backlog increase, please take care of checking your clusters before upgrading. +- `local_cluster.topic.partitions = 2` and `remote_cluster.topic.partitions = 0(non-partitioned topic)`: see detail in the section `Design & Implementation Details -> Phase-2`. +- `local_cluster.topic.partitions = 0(non-partitioned topic)` and and `remote_cluster.topic.partitions >= 1`: see detail in the section `Design & Implementation Details -> Phase-2`. + +# Links +* Mailing List discussion thread: https://lists.apache.org/thread/9fx354cqcy3412w1nx8kwdf9h141omdg +* Mailing List voting thread: https://lists.apache.org/thread/vph22st5td1rdh1gd68gkrnp9doo6ct2 diff --git a/pip/pip-374.md b/pip/pip-374.md new file mode 100644 index 0000000000000..49fe337159628 --- /dev/null +++ b/pip/pip-374.md @@ -0,0 +1,71 @@ +# PIP-374: Visibility of messages in receiverQueue for the consumers + +# Background knowledge + +When a consumer connects to the Broker, the broker starts dispatching the messages based on receiverQueueSize configured. +There is no observability for the messages arrived on the consumer side if the user didn't call the receive method. It leads to ambiguities at times as +the consumer application does not know whether the message was actually sent by the broker or is it lost in the network or is it lost in the receiver queue. + +ConsumerInterceptors is a plugin interface that intercept and possibly mutate messages received by the consumer. + + +# Motivation + +* We need to receive queue filling of the event as the particular message is already on particular consumer's receiver queue and waiting for the consumer to pickup and process. It may wait in the recieverQueue longer if the consumer processing takes more time. It's very important to provide the visibility of the messages that are waiting in receiverQueue for processing. + +* Availability of a consumer application w.r.t any messaging system depends on the number of messages dispatched from the server/broker against the number of messages acknowledged from the consumer app. This metric defines the processing rate of a consumer. +Currently, the number of acknowledged messages can be counted by having a counter in onAcknowledge() method of ConsumerInterceptor. But, there is no way to capture the number of messages arrived in Consumer. + + +What does this solve? +* Visibility about the message in receiverQueue for the consumer. +* Stuck consumer state visibility +* Scale the consumers to process the spikes in producer traffic +* Reduce the overhead of processing the redeliveries + + +# Goals + +## In Scope + +The proposal will add a method to the interceptor to allow users to knowthe message has been received by the consumer. + +Add a default abstract method in ConsumerInterceptor called onArrival() and hook this method call in the internal consumer of MultiTopicConsumerImpl and ConsumerImpl. By this way, there will be an observability of message received for the consumer. + + +# High Level Design + +* Add onArrival() abstract method in ConsumerInterceptor interface. +* Hook this method call where the consumer receives the batch messages at once(based on configured receiverQueueSize). + + +# Detailed Design + +## Design & Implementation Details + +* ConsumerInterceptor.java +``` +default Message onArrival()(Consumer consumer, Message message){ + return message; +} + +``` + +* Add hook in ConsumerImpl.messageReceived which calls onArrival method which calculates the the number of message received. +``` +Message interceptMsg = onArrival(consumer,msg); +``` + +# Backward & Forward Compatibility + +## Upgrade + +Since we added a default method onArrival() in interface, one who has provided the implementations for ConsumerInterceptor will not get any compile time error as it has default implementation. If user wants to give implementation from his side, he can override and provide implementation. + +# Links + + +* Mailing List discussion thread: https://lists.apache.org/thread/hcfpm4j6hpwxb2olfrro8g4dls35q8rx +* Mailing List voting thread: https://lists.apache.org/thread/wrr02s4cdzqmo1vonp92w6229qo0rv0z diff --git a/pip/pip-376-Topic-Policies-Service-Pluggable.md b/pip/pip-376-Topic-Policies-Service-Pluggable.md new file mode 100644 index 0000000000000..0659de812af3d --- /dev/null +++ b/pip/pip-376-Topic-Policies-Service-Pluggable.md @@ -0,0 +1,222 @@ +# PIP-376: Make Topic Policies Service Pluggable + +## Background + +### Topic Policies Service and System Topics + +[PIP-39](https://github.com/apache/pulsar/wiki/PIP-39%3A-Namespace-Change-Events) introduces system topics and topic-level policies. Currently, the topic policies service (`TopicPoliciesService`) has only one implementation (`SystemTopicBasedTopicPoliciesService`) that depends on system topics. Therefore, the following configurations are required (though they are enabled by default): + +```properties +systemTopicEnabled=true +topicLevelPoliciesEnabled=true +``` + +However, using system topics to manage topic policies may not always be the best choice. Users might need an alternative approach to manage topic policies. + +### Issues with the Current `TopicPoliciesService` Interface + +The `TopicPoliciesService` interface is poorly designed for third-party implementations due to the following reasons: + +1. **Methods that Should Not Be Exposed**: + - `addOwnedNamespaceBundleAsync` and `removeOwnedNamespaceBundleAsync` are used internally in `SystemTopicBasedTopicPoliciesService`. + - `getTopicPoliciesBypassCacheAsync` is used only in tests to replay the `__change_events` topic and construct the topic policies map. + +2. **Confusing and Inconsistent `getTopicPolicies` Methods**: + - There are two overrides of `getTopicPolicies`: + ```java + TopicPolicies getTopicPolicies(TopicName topicName, boolean isGlobal) throws TopicPoliciesCacheNotInitException; + TopicPolicies getTopicPolicies(TopicName topicName) throws TopicPoliciesCacheNotInitException; + ``` + - The second method is equivalent to `getTopicPolicies(topicName, false)`. + - These methods are asynchronous and start an asynchronous policies initialization, then try to get the policies from the cache. If the initialization hasn't started, they throw `TopicPoliciesCacheNotInitException`. + +These methods are hard to use and are primarily used in tests. The `getTopicPoliciesAsyncWithRetry` method uses a user-provided executor and backoff policy to call `getTopicPolicies` until `TopicPoliciesCacheNotInitException` is not thrown: + +```java +default CompletableFuture> getTopicPoliciesAsyncWithRetry(TopicName topicName, + final Backoff backoff, ScheduledExecutorService scheduledExecutorService, boolean isGlobal) { +``` + +The `getTopicPolicies` methods are confusing for users who want to implement their own topic policies service. They need to look deeply into Pulsar's source code to understand these details. + +[PR #21231](https://github.com/apache/pulsar/pull/21231) adds two asynchronous overrides that are more user-friendly: + +```java +CompletableFuture> getTopicPoliciesAsync(@Nonnull TopicName topicName, boolean isGlobal); +CompletableFuture> getTopicPoliciesAsync(@Nonnull TopicName topicName); +``` + +Now there are five asynchronous `get` methods. Unlike `getTopicPolicies`, `getTopicPoliciesAsync(topic)` is not equivalent to `getTopicPoliciesAsync(topic, false)`. Instead: +- `getTopicPoliciesAsync(topic)` tries getting local policies first, then global policies if absent. +- `getTopicPoliciesAsync(topic, true)` tries getting global policies. +- `getTopicPoliciesAsync(topic, false)` tries getting local policies. + +Since [PR #12517](https://github.com/apache/pulsar/pull/12517), topic policies support global policies across clusters. Therefore, there are local and global policies. + +Currently: +- `getTopicPoliciesAsync(TopicName)` is used in `BrokerService#getTopicPoliciesBypassSystemTopic` for initializing topic policies of `PersistentTopic` objects. +- `getTopicPoliciesAsyncWithRetry` is used in `AdminResource#getTopicPoliciesAsyncWithRetry` for all topic policies admin APIs. +- Other methods are used only in tests. + +There is also a sixth method, `getTopicPoliciesIfExists`, which tries to get local topic policies from the cache: + +```java +TopicPolicies getTopicPoliciesIfExists(TopicName topicName); +``` + +However, this method is called just because there was no `getTopicPoliciesAsync` methods before and `getTopicPolicies` is hard to use. For example, here is an example code snippet in `PersistentTopicsBase#internalUpdatePartitionedTopicAsync`: + +```java +TopicPolicies topicPolicies = + pulsarService.getTopicPoliciesService().getTopicPoliciesIfExists(topicName); +if (topicPolicies != null && topicPolicies.getReplicationClusters() != null) { + replicationClusters = topicPolicies.getReplicationClustersSet(); +} +``` + +With the new `getTopicPoliciesAsync` methods, this code can be replaced with: + +```java +pulsarService.getTopicPoliciesService().getTopicPoliciesAsync(topicName, GetType.LOCAL_ONLY) + .thenAccept(topicPolicies -> { + if (topicPolicies.isPresent() && topicPolicies.get().getReplicationClusters() != null) { + replicationClusters = topicPolicies.get().getReplicationClustersSet(); + } + }); +``` + +## Motivation + +Make `TopicPoliciesService` pluggable so users can customize the topic policies service via another backend metadata store. + +## Goals + +### In Scope + +Redesign a clear and simple `TopicPoliciesService` interface for users to customize. + +## High-Level Design + +Add a `topicPoliciesServiceClassName` configuration to specify the topic policies service class name. If the class name is not the default `SystemTopicBasedTopicPoliciesService`, `systemTopicEnabled` will not be required unless the implementation requires it. + +## Detailed Design + +### Design & Implementation Details + +1. Add a unified method to get topic policies: + ```java + enum GetType { + DEFAULT, // try getting the local topic policies, if not present, then get the global policies + GLOBAL_ONLY, // only get the global policies + LOCAL_ONLY, // only get the local policies + } + CompletableFuture> getTopicPoliciesAsync(TopicName topicName, GetType type); + ``` + + `getTopicPoliciesAsyncWithRetry` will be replaced by `getTopicPoliciesAsync(topicName, LOCAL_ONLY)` or `getTopicPoliciesAsync(topicName, GLOBAL_ONLY)`. The other two original `getTopicPoliciesAsync` methods and `getTopicPoliciesIfExists` will be replaced by `getTopicPoliciesAsync(topicName, DEFAULT)`. + +2. Move `addOwnedNamespaceBundleAsync` and `removeOwnedNamespaceBundleAsync` to private methods of `SystemTopicBasedTopicPoliciesService`. + +3. Add a `TestUtils` class in tests to include `getTopicPolicies` and `getTopicPoliciesBypassCacheAsync` methods. + +4. Remove the generic parameter from `TopicPolicyListener` as the value type should always be `TopicPolicies`. Mark this listener interface as `Stable`. + +5. Add a `PulsarService` parameter to the `start` method so that the implementation can have a constructor with an empty parameter list and get the `PulsarService` instance from the `start` method. + +6. Add a `boolean` return value to `registerListener` since `PersistentTopic#initTopicPolicy` checks if the topic policies are enabled. The return value will indicate if the `TopicPoliciesService` instance is `topicPoliciesServiceClassName.DISABLED`. + +Since the topic policies service is now decoupled from system topics, remove all `isSystemTopicAndTopicLevelPoliciesEnabled()` calls. + +Here is the refactored `TopicPoliciesService` interface: + +```java + /** + * Delete policies for a topic asynchronously. + * + * @param topicName topic name + */ + CompletableFuture deleteTopicPoliciesAsync(TopicName topicName); + + /** + * Update policies for a topic asynchronously. + * + * @param topicName topic name + * @param policies policies for the topic name + */ + CompletableFuture updateTopicPoliciesAsync(TopicName topicName, TopicPolicies policies); + + /** + * It controls the behavior of {@link TopicPoliciesService#getTopicPoliciesAsync}. + */ + enum GetType { + DEFAULT, // try getting the local topic policies, if not present, then get the global policies + GLOBAL_ONLY, // only get the global policies + LOCAL_ONLY, // only get the local policies + } + + /** + * Retrieve the topic policies. + */ + CompletableFuture> getTopicPoliciesAsync(TopicName topicName, GetType type); + + /** + * Start the topic policy service. + */ + default void start(PulsarService pulsar) { + } + + /** + * Close the resources if necessary. + */ + default void close() throws Exception { + } + + /** + * Registers a listener for topic policies updates. + * + *

+ * The listener will receive the latest topic policies when they are updated. If the policies are removed, the + * listener will receive a null value. Note that not every update is guaranteed to trigger the listener. For + * instance, if the policies change from A -> B -> null -> C in quick succession, only the final state (C) is + * guaranteed to be received by the listener. + * In summary, the listener is guaranteed to receive only the latest value. + *

+ * + * @return true if the listener is registered successfully + */ + boolean registerListener(TopicName topicName, TopicPolicyListener listener); + + /** + * Unregister the topic policies listener. + */ + void unregisterListener(TopicName topicName, TopicPolicyListener listener); +``` + +```java +@InterfaceStability.Stable +public interface TopicPolicyListener { + + void onUpdate(TopicPolicies data); +} +``` + +### Configuration + +Add a new configuration `topicPoliciesServiceClassName`. + +## Backward & Forward Compatibility + +If downstream applications need to call APIs from `TopicPoliciesService`, they should modify the code to use the new API. + +## Alternatives + +### Keep the `TopicPoliciesService` Interface Compatible + +The current interface is poorly designed because it has only one implementation. Keeping these methods will burden developers who want to develop a customized interface. They need to understand where these confusing methods are called and handle them carefully. + +## General Notes + +## Links + +* Mailing List discussion thread: https://lists.apache.org/thread/gf6h4n5n1z4n8v6bxdthct1n07onfdxt +* Mailing List voting thread: https://lists.apache.org/thread/potjbkb4w8brcwscgdwzlxnowgdf11gd diff --git a/pip/pip-378.md b/pip/pip-378.md new file mode 100644 index 0000000000000..e44ce7339cf53 --- /dev/null +++ b/pip/pip-378.md @@ -0,0 +1,280 @@ +# PIP-378: Add ServiceUnitStateTableView abstraction (ExtensibleLoadMangerImpl only) + +## Background + +### ExtensibleLoadMangerImpl uses system topics to event-source bundle ownerships + +PIP-192 introduces a new broker load balancer using a persistent system topic to event-source bundle ownerships among brokers. + +PIP-307 introduces graceful ownership change protocol over the system topic (from PIP-192). + +However, using system topics to manage bundle ownerships may not always be the best choice. Users might need an alternative approach to event-source bundle ownerships. + + +## Motivation + +Add `ServiceUnitStateTableView` abstraction and make it pluggable, so users can customize `ServiceUnitStateTableView` implementations and event-source bundles ownerships using other stores. + +## Goals + +### In Scope + +- Add `ServiceUnitStateTableView` interface +- Add `ServiceUnitStateTableViewImpl` implementation that uses Pulsar System topic (compatible with existing behavior) +- Add `ServiceUnitStateMetadataStoreTableViewImpl` implementation that uses Pulsar Metadata Store (new behavior) +- Refactor related code and test code + +## High-Level Design + +- Refactor `ServiceUnitStateChannelImpl` to accept `ServiceUnitStateTableView` interface and `ServiceUnitStateTableViewImpl` system topic implementation. +- Introduce `MetadataStoreTableView` interface to support `ServiceUnitStateMetadataStoreTableViewImpl` implementation. +- `MetadataStoreTableViewImpl` will use shadow hashmap to maintain the metadata tableview. It will initially fill the local tableview by scanning all existing items in the metadata store path. Also, new items will be updated to the tableview via metadata watch notifications. +- Add `BiConsumer>> asyncReloadConsumer` in MetadataCacheConfig to listen the automatic cache async reload. This can be useful to re-sync the the shadow hashmap in MetadataStoreTableViewImpl in case it is out-dated in the worst case(e.g. network or metadata issues). +- Introduce `ServiceUnitStateTableViewSyncer` to sync system topic and metadata store table views to migrate to one from the other. This syncer can be enabled by a dynamic config, `loadBalancerServiceUnitTableViewSyncer`. + +## Detailed Design + +### Design & Implementation Details +```java +/** + * Given that the ServiceUnitStateChannel event-sources service unit (bundle) ownership states via a persistent store + * and reacts to ownership changes, the ServiceUnitStateTableView provides an interface to the + * ServiceUnitStateChannel's persistent store and its locally replicated ownership view (tableview) with listener + * registration. It initially populates its local table view by scanning existing items in the remote store. The + * ServiceUnitStateTableView receives notifications whenever ownership states are updated in the remote store, and + * upon notification, it applies the updates to its local tableview with the listener logic. + */ +public interface ServiceUnitStateTableView extends Closeable { + + /** + * Starts the tableview. + * It initially populates its local table view by scanning existing items in the remote store, and it starts + * listening to service unit ownership changes from the remote store. + * @param pulsar pulsar service reference + * @param tailItemListener listener to listen tail(newly updated) items + * @param existingItemListener listener to listen existing items + * @throws IOException if it fails to init the tableview. + */ + void start(PulsarService pulsar, + BiConsumer tailItemListener, + BiConsumer existingItemListener) throws IOException; + + + /** + * Closes the tableview. + * @throws IOException if it fails to close the tableview. + */ + void close() throws IOException; + + /** + * Gets one item from the local tableview. + * @param key the key to get + * @return value if exists. Otherwise, null. + */ + ServiceUnitStateData get(String key); + + /** + * Tries to put the item in the persistent store. + * If it completes, all peer tableviews (including the local one) will be notified and be eventually consistent + * with this put value. + * + * It ignores put operation if the input value conflicts with the existing one in the persistent store. + * + * @param key the key to put + * @param value the value to put + * @return a future to track the completion of the operation + */ + CompletableFuture put(String key, ServiceUnitStateData value); + + /** + * Tries to delete the item from the persistent store. + * All peer tableviews (including the local one) will be notified and be eventually consistent with this deletion. + * + * It ignores delete operation if the key is not present in the persistent store. + * + * @param key the key to delete + * @return a future to track the completion of the operation + */ + CompletableFuture delete(String key); + + /** + * Returns the entry set of the items in the local tableview. + * @return entry set + */ + Set> entrySet(); + + /** + * Returns service units (namespace bundles) owned by this broker. + * @return a set of owned service units (namespace bundles) + */ + Set ownedServiceUnits(); + + /** + * Tries to flush any batched or buffered updates. + * @param waitDurationInMillis time to wait until complete. + * @throws ExecutionException + * @throws InterruptedException + * @throws TimeoutException + */ + void flush(long waitDurationInMillis) throws ExecutionException, InterruptedException, TimeoutException; +} +``` + +```java +/** + * Defines metadata store tableview. + * MetadataStoreTableView initially fills existing items to its local tableview and eventually + * synchronize remote updates to its local tableview from the remote metadata store. + * This abstraction can help replicate metadata in memory from metadata store. + */ +public interface MetadataStoreTableView { + + class ConflictException extends RuntimeException { + public ConflictException(String msg) { + super(msg); + } + } + + /** + * Starts the tableview by filling existing items to its local tableview from the remote metadata store. + */ + void start() throws MetadataStoreException; + + /** + * Reads whether a specific key exists in the local tableview. + * + * @param key the key to check + * @return true if exists. Otherwise, false. + */ + boolean exists(String key); + + /** + * Gets one item from the local tableview. + *

+ * If the key is not found, return null. + * + * @param key the key to check + * @return value if exists. Otherwise, null. + */ + T get(String key); + + /** + * Tries to put the item in the persistent store. + * All peer tableviews (including the local one) will be notified and be eventually consistent with this put value. + *

+ * This operation can fail if the input value conflicts with the existing one. + * + * @param key the key to check on the tableview + * @return a future to track the completion of the operation + * @throws MetadataStoreTableView.ConflictException + * if the input value conflicts with the existing one. + */ + CompletableFuture put(String key, T value); + + /** + * Tries to delete the item from the persistent store. + * All peer tableviews (including the local one) will be notified and be eventually consistent with this deletion. + *

+ * This can fail if the item is not present in the metadata store. + * + * @param key the key to check on the tableview + * @return a future to track the completion of the operation + * @throws MetadataStoreException.NotFoundException + * if the key is not present in the metadata store. + */ + CompletableFuture delete(String key); + + /** + * Returns the size of the items in the local tableview. + * @return size + */ + int size(); + + /** + * Reads whether the local tableview is empty or not. + * @return true if empty. Otherwise, false + */ + boolean isEmpty(); + + /** + * Returns the entry set of the items in the local tableview. + * @return entry set + */ + Set> entrySet(); + + /** + * Returns the key set of the items in the local tableview. + * @return key set + */ + Set keySet(); + + /** + * Returns the values of the items in the local tableview. + * @return values + */ + Collection values(); + + /** + * Runs the action for each item in the local tableview. + */ + void forEach(BiConsumer action); +} +``` + +```java +public class MetadataCacheConfig { + private static final long DEFAULT_CACHE_REFRESH_TIME_MILLIS = TimeUnit.MINUTES.toMillis(5); + + ... + + /** + * Specifies cache reload consumer behavior when the cache is refreshed automatically at refreshAfterWriteMillis + * frequency. + */ + @Builder.Default + private final BiConsumer>> asyncReloadConsumer = null; +``` + +```java + +/** + * ServiceUnitStateTableViewSyncer can be used to sync system topic and metadata store table views to migrate to one + * from the other. + */ +@Slf4j +public class ServiceUnitStateTableViewSyncer implements Cloneable { + ... + + public void start(PulsarService pulsar) throws IOException { + ... // sync SystemTopicTableView and MetadataStoreTableView + } + + + public void close() throws IOException { + ... // stop syncer + } +... +} + + +``` + +### Configuration + +- Add a `loadManagerServiceUnitStateTableViewClassName` static configuration to specify `ServiceUnitStateTableView` implementation class name. +- Add a `loadBalancerServiceUnitTableViewSyncer` dynamic configuration to enable ServiceUnitTableViewSyncer to sync metadata store and system topic ServiceUnitStateTableView during migration. + +## Backward & Forward Compatibility + +It will ba Backward & Forward compatible as `loadManagerServiceUnitStateTableViewClassName` will be `ServiceUnitStateTableViewImpl`(system topic implementation) by default. + +We will introduce `ServiceUnitStateTableViewSyncer` dynamic config to sync system topic and metadata store table views when migrating to ServiceUnitStateMetadataStoreTableViewImpl from ServiceUnitStateTableViewImpl and vice versa. The admin could enable this syncer before migration and disable it after it is finished. + +## Alternatives + +## General Notes + +## Links + +* Mailing List discussion thread: https://lists.apache.org/thread/v7sod21r56hkt2cjxl9pp348r4jxo6o8 +* Mailing List voting thread: https://lists.apache.org/thread/j453xp0vty8zy2y0ljssjgyvwb47royc diff --git a/pip/pip-379.md b/pip/pip-379.md new file mode 100644 index 0000000000000..3215bb541f11e --- /dev/null +++ b/pip/pip-379.md @@ -0,0 +1,407 @@ +# PIP-379: Key_Shared Draining Hashes for Improved Message Ordering + +## Background Knowledge + +Apache Pulsar's Key_Shared subscription mode is designed to provide ordered message delivery on a per-key basis while allowing multiple consumers to process messages concurrently. This mode is particularly useful in scenarios where maintaining message order for specific keys is crucial, but overall throughput can be improved by parallelizing message consumption across multiple consumers. + +Key concepts: + +- **Key_Shared subscription**: A subscription mode that maintains message ordering per key while allowing multiple consumers. +- **Hash ranges**: In AUTO_SPLIT mode, the hash space is divided among active consumers to distribute message processing. +- **Pending messages**: Messages that have been sent to a consumer but not yet acknowledged (also called "pending acks" or "unacknowledged messages"). + +### Current contract of preserving ordering + +The Key_Shared subscription is described in the [Pulsar documentation](https://pulsar.apache.org/docs/concepts-messaging/#key_shared). + +For this PIP, the most important detail is the "Preserving order of processing" section. +There are recent changes in this section that apply to the master branch of Pulsar and, therefore, to the upcoming Pulsar 4.0. The changes were made as part of ["PIP-282: Change definition of the recently joined consumers position"](https://github.com/apache/pulsar/blob/master/pip/pip-282.md). + +[PIP-282 (master branch / Pulsar 4.0) version of the "Preserving order of processing" section](https://pulsar.apache.org/docs/next/concepts-messaging/#preserving-order-of-processing): + +> Key_Shared Subscription type guarantees a key will be processed by a *single* consumer at any given time. When a new consumer is connected, some keys will change their mapping from existing consumers to the new consumer. Once the connection has been established, the broker will record the current `lastSentPosition` and associate it with the new consumer. The `lastSentPosition` is a marker indicating that messages have been dispatched to the consumers up to this point. The broker will start delivering messages to the new consumer *only* when all messages up to the `lastSentPosition` have been acknowledged. This will guarantee that a certain key is processed by a single consumer at any given time. The trade-off is that if one of the existing consumers is stuck and no time-out was defined (acknowledging for you), the new consumer won't receive any messages until the stuck consumer resumes or gets disconnected. + +[Previous version (applies to Pulsar 3.x) of the "Preserving order of processing" section](https://pulsar.apache.org/docs/3.3.x/concepts-messaging/#preserving-order-of-processing): + +> Key Shared Subscription type guarantees a key will be processed by a *single* consumer at any given time. When a new consumer is connected, some keys will change their mapping from existing consumers to the new consumer. Once the connection has been established, the broker will record the current read position and associate it with the new consumer. The read position is a marker indicating that messages have been dispatched to the consumers up to this point, and after it, no messages have been dispatched yet. The broker will start delivering messages to the new consumer *only* when all messages up to the read position have been acknowledged. This will guarantee that a certain key is processed by a single consumer at any given time. The trade-off is that if one of the existing consumers is stuck and no time-out was defined (acknowledging for you), the new consumer won't receive any messages until the stuck consumer resumes or gets disconnected. + +## Motivation + +The current implementation of Key_Shared subscriptions faces several challenges: + +1. **Complex Contract of Preserving Ordering**: The current contract of preserving ordering is hard to understand and contains a fundamental problem. It explains a solution and then ties the guarantee to the provided solution. It could be interpreted that there's a guarantee as long as this solution is able to handle the case. +2. **Incomplete Ordering Contract Fulfillment**: The current contract seems to make a conditional guarantee that a certain key is processed by a single consumer at any given time. Outside of the described solution in the contract, the current implementation struggles to consistently prevent messages from being sent to another consumer while pending on the original consumer. While Key_Shared subscriptions aim to preserve message ordering per key, the current implementation may not always achieve this, especially during consumer changes. There's a potential corner case reported in [issue #23307](https://github.com/apache/pulsar/issues/23307). +3. **Usability Issues**: Understanding the current system and detecting the reason why messages get blocked is time-consuming and difficult. +4. **Unnecessary Message Blocking**: The current implementation blocks delivery for all messages when any hash range is blocked, even if other keys could be processed independently. This leads to suboptimal utilization of consumers and increased latency for messages that could otherwise be processed. +5. **Observability Challenges**: The current implementation lacks clear visibility into the consuming state when processing gets stuck, making it harder to build automation for detecting and mitigating issues. +6. **Complexity**: The existing solution for managing "recently joined consumers" is overly complex, making the system harder to maintain and debug. + +## Goals + +### In Scope + +- Clarify and fulfill the key-ordered message delivery contract for Key_Shared AUTO_SPLIT mode. +- Fix current issues where messages are sent out-of-order or when a single key is outstanding in multiple consumers at a time. +- Improve the handling of unacknowledged messages to prevent indefinite blocking and consumers getting stuck. +- Minimize memory usage for pending message tracking, eliminating PIP-282's "sent positions" tracking. +- Implement a new "draining hashes" concept to efficiently manage message ordering in Key_Shared subscriptions. +- Enhance the reliability, usability, and scalability of Key_Shared subscriptions. +- Improve observability of Key_Shared subscriptions to aid in troubleshooting and automation. +- Ensure strict ordering guarantees for messages with the same key, even during consumer changes. + +### Out of Scope + +- Changes to other subscription types (Exclusive, Failover, Shared). +- Adding support key based ordering guarantees when negative acknowledgements are used + +## High-Level Design + +### Updated contract of preserving ordering + +The "Preserving order of processing" section of the Key_Shared documentation would be updated to contain this contract: + +_In Key_Shared subscriptions, messages with the same key are delivered and allowed to be in unacknowledged state to only one consumer at a time._ + +When new consumers join or leave, the consumer handling a message key can change when the default AUTO_SPLIT mode is used, but only after all pending messages for a particular key are acknowledged or the original consumer disconnects. + +The Key_Shared subscription doesn't prevent using any methods in the consumer API. For example, the application might call `negativeAcknowledge` or the `redeliverUnacknowledgedMessages` method. When messages are scheduled for delivery due to these methods, they will get redelivered as soon as possible. There's no ordering guarantee in these cases, however the guarantee of delivering a message key to a single consumer at a time will continue to be preserved. + +### Computer Science Perspective: Invariants + +Wikipedia tells us about [invariants](https://en.wikipedia.org/wiki/Invariant_(mathematics)#Invariants_in_computer_science): "In computer science, an invariant is a logical assertion that is always held to be true during a certain phase of execution of a computer program." + +The contract _"In Key_Shared subscriptions, messages with the same key are delivered and allowed to be in an unacknowledged state to only one consumer at a time."_ can be seen as an invariant for Key_Shared subscriptions. It is something that must always be held true for Key_Shared subscriptions. The design and implementation in PIP-379 focuses on ensuring this. + +### Future work in needed for supporting key-based ordering with negative acknowledgements + +The updated contract explicitly states that it is not possible to retain key-based ordering of messages when negative acknowledgements are used. Changing this is out of scope for PIP-379. A potential future solution for handling this would be to modify the client so that when a message is negatively acknowledged, it would also reject all further messages with the same key until the original message gets redelivered. It's already possible to attempt to implement this in client-side code. However, a proper solution would require support on the broker side to block further delivery of the specific key when there are pending negatively acknowledged messages until all negatively acknowledged messages for that particular key have been acknowledged by the consumer. This solution is out of scope for PIP-379. A future implementation to address these problems could build upon PIP-379 concepts such as "draining hashes" and extend that to cover the negative acknowledgement scenarios. + +### High-Level implementation plan + +The proposed solution introduces a "draining hashes" concept to efficiently manage message ordering in Key_Shared subscriptions: + +**1. When consumer hash ranges change (e.g., a consumer joins or leaves), affected hashes of pending messages are added to a "draining hashes" set.** + +Pending messages of the consumer are iterated, and if the hash of a pending message belongs to one of the impacted ranges, the hash gets added to the "draining hashes" tracker. + +Code example to illustrate the implementation: + +```java + private synchronized void registerDrainingHashes(Consumer skipConsumer, + Map> impactedRangesByConsumer) { + for (Map.Entry> entry : impactedRangesByConsumer.entrySet()) { + Consumer c = entry.getKey(); + if (c != skipConsumer) { + // perf optimization: convert the set to an array to avoid iterator allocation in the pending acks loop + Range[] ranges = entry.getValue().toArray(new Range[0]); + // add all pending acks in the impacted hash ranges to the draining hashes tracker + c.getPendingAcks().forEach((ledgerId, entryId, batchSize, stickyKeyHash) -> { + for (Range range : ranges) { + if (range.contains(stickyKeyHash)) { + // add the pending ack to the draining hashes tracker if the hash is in the range + drainingHashesTracker.addEntry(c, stickyKeyHash); + break; + } + // Since ranges are sorted, stop checking further ranges if the start of the current range is + // greater than the stickyKeyHash. + if (range.getStart() > stickyKeyHash) { + break; + } + } + }); + } + } + } +``` + +**2. Following messages with hashes in the "draining hashes" set are blocked from further delivery until pending messages are processed.** + +Code example to illustrate the implementation: + +```java + // If the hash is draining, do not send the message + if (drainingHashesTracker.shouldBlockStickyKeyHash(consumer, stickyKeyHash)) { + return false; + } +``` + +**3. A reference counter tracks pending messages for each hash in the "draining hashes" set.** + +Code example to illustrate the implementation: + +```java + // optimize the memory consumption of the map by using primitive int keys + private final Int2ObjectOpenHashMap drainingHashes = new Int2ObjectOpenHashMap<>(); + + public static class DrainingHashEntry { + private final Consumer consumer; + private int refCount; + private int blockedCount; + + DrainingHashEntry(Consumer consumer) { + this.consumer = consumer; + } + + public Consumer getConsumer() { + return consumer; + } + + void incrementRefCount() { + refCount++; + } + + boolean decrementRefCount() { + return --refCount == 0; + } + + void incrementBlockedCount() { + blockedCount++; + } + + boolean isBlocking() { + return blockedCount > 0; + } + } +``` + +The memory consumption estimate for tracking a hash is 52 bytes: +key: 16 bytes (object header) + 4 bytes (int) = 20 bytes +entry: 16 bytes (object header) + 8 bytes (long) + 4 bytes (int) + 4 bytes (int) = 32 bytes + +Although the estimate is 52 bytes per entry, calculations have been made with 80 bytes per entry to account for possible additional overheads such as memory alignment and the overhead of the Int2ObjectOpenHashMap. + +Memory usage estimate for each subscription after there have been consumer changes: + +- Worst case (all 64k hashes draining for a subscription): about 5MB +- Practical case (less than 1000 hashes draining): less than 80 kilobytes +- For 10,000 draining hashes: about 800 kB + +The memory usage of draining hashes tracking will go down to 0 after all hashes have "drained" and are no longer blocked. This memory usage isn't an overhead that applies at all times. + +The hash range size is reduced to 65535 (2^16-1) from the current 2^31-1 (Integer.MAX_VALUE) in ConsistentHashingStickyKeyConsumerSelector to reduce the worst-case memory consumption. Reducing the hash range size won't significantly impact the accuracy of distributing messages across connected consumers. The proof-of-concept implementation of PIP-379 includes the changes to reduce the hash range size. + +**4. As messages are acknowledged or consumers disconnect and therefore get removed from pending messages, the reference counter is decremented.** + +Individual acks are removed in Consumer's `removePendingAcks` method: + +```java + private boolean removePendingAcks(Consumer ackOwnedConsumer, Position position) { + PendingAcksMap ownedConsumerPendingAcks = ackOwnedConsumer.getPendingAcks(); + if (!ownedConsumerPendingAcks.remove(position.getLedgerId(), position.getEntryId())) { + // Message was already removed by the other consumer + return false; + } +``` + +When the `remove` method in `PendingAcksMap` is called, it will use the `PendingAcksMap.PendingAcksRemoveHandler` callback method `handleRemoving` provided by the dispatcher to trigger the removal also from the `DrainingHashesTracker`: + +```java + consumer.setPendingAcksRemoveHandler(new PendingAcksMap.PendingAcksRemoveHandler() { + @Override + public void handleRemoving(Consumer consumer, long ledgerId, long entryId, int stickyKeyHash, + boolean closing) { + drainingHashesTracker.reduceRefCount(consumer, stickyKeyHash, closing); + } + +``` + +Also when a consumer disconnects, hashes of pending acks are removed. This happens in the `PersistentDispatcherMultipleConsumers`'s `removeConsumer` consumer method: + +```java + consumer.getPendingAcks().forEachAndClose((ledgerId, entryId, batchSize, stickyKeyHash) -> { + addMessageToReplay(ledgerId, entryId, stickyKeyHash); + }); +``` + +`PendingAcksMap`'s `forEachAndClose` method will trigger removals from `DrainingHashesTracker` using the `PendingAcksMap.PendingAcksRemoveHandler` callback method `handleRemoving` after processing each entry. This is how the `DrainingHashesTracker` stays in sync with the `PendingAcksMap` state without having the need to add all logic to `PendingAcksMap`. This is about following the "separation of concerns" design principle where each class handles a specific concern. + +**5. When the reference counter reaches zero, the hash is removed from the set, allowing new message delivery. The dispatcher is notified about this so that the delivery of the blocked messages can occur. Unblocked hashes are batched together to prevent a new notification for each call. This is handled with the `keySharedUnblockingIntervalMs` configuration setting.** + +In the implementation, this is handled in the DrainingHashesTracker's reduceRefCount method: + +```java + // code example is simplified for focus on the essential details + + public synchronized void reduceRefCount(Consumer consumer, int stickyHash) { + DrainingHashEntry entry = drainingHashes.get(stickyHash); + if (entry == null) { + return; + } + if (entry.decrementRefCount()) { + DrainingHashEntry removed = drainingHashes.remove(stickyHash); + if (removed.isBlocking()) { + unblockingHandler.stickyKeyHashUnblocked(stickyHash); + } + } + } +``` + +The `isBlocking()` method of `DrainingHashEntry` returns true when delivery was attempted for that hash, indicating a need to unblock it when it's removed. +The dispatcher is notified via the `unblockingHandler.stickyKeyHashUnblocked(stickyHash)` callback. The implementation simply schedules a read, batching all calls together, and then calls `readMoreEntries` in the dispatcher. + +```java + // code example is simplified for focus on the essential details + + private void stickyKeyHashUnblocked(int stickyKeyHash) { + reScheduleReadInMs(keySharedUnblockingIntervalMsSupplier.getAsLong()); + } + + protected void reScheduleReadInMs(long readAfterMs) { + if (isRescheduleReadInProgress.compareAndSet(false, true)) { + Runnable runnable = () -> { + isRescheduleReadInProgress.set(false); + readMoreEntries(); + }; + topic.getBrokerService().executor().schedule(runnable, readAfterMs, TimeUnit.MILLISECONDS); + } + } +``` + +**6. Consumer hash assignments may change multiple times, and a draining hash might be reassigned to the original consumer.** + +The draining hash data structure contains information about the draining consumer. When a message is attempted for delivery, the system can check if the target consumer is the same as the draining consumer. If they match, there's no need to block the hash. The implementation should also remove such hashes from the draining hashes set. This "lazy" approach reduces the need for actively scanning all draining hashes whenever hash assignments change. + +This is handled in the `DrainingHashesTracker` + +```java + public synchronized boolean shouldBlockStickyKeyHash(Consumer consumer, int stickyKeyHash) { + DrainingHashEntry entry = drainingHashes.get(stickyKeyHash); + // if the entry is not found, the hash is not draining. Don't block the hash. + if (entry == null) { + return false; + } + // hash has been reassigned to the original consumer, remove the entry + // and don't block the hash + if (entry.getConsumer() == consumer) { + drainingHashes.remove(stickyKeyHash, entry); + return false; + } + // increment the blocked count which is used to determine if the hash is blocking + // dispatching to other consumers + entry.incrementBlockedCount(); + // block the hash + return true; + } +``` + +**7. When sending out messages, there are potential race conditions that could allow the delivery of a message that should be blocked.** + +This could happen when a consumer is added while reading and sending messages are already in progress. In PIP-379, the sending process has been modified to perform a check when adding the message to the pending acknowledgments map. There are also additional locks in the pending acks handling which prevent race conditions. + +`addPendingAckIfAllowed` method in `PendingAcksMap` class: + +```java + public boolean addPendingAckIfAllowed(long ledgerId, long entryId, int batchSize, int stickyKeyHash) { + try { + writeLock.lock(); + // prevent adding sticky hash to pending acks if the PendingAcksMap has already been closed + // and there's a race condition between closing the consumer and sending new messages + if (closed) { + return false; + } + // prevent adding sticky hash to pending acks if it's already in draining hashes + // to avoid any race conditions that would break consistency + PendingAcksAddHandler pendingAcksAddHandler = pendingAcksAddHandlerSupplier.get(); + if (pendingAcksAddHandler != null + && !pendingAcksAddHandler.handleAdding(consumer, ledgerId, entryId, stickyKeyHash)) { + return false; + } + Long2ObjectSortedMap ledgerPendingAcks = + pendingAcks.computeIfAbsent(ledgerId, k -> new Long2ObjectRBTreeMap<>()); + ledgerPendingAcks.put(entryId, IntIntPair.of(batchSize, stickyKeyHash)); + return true; + } finally { + writeLock.unlock(); + } + } +``` + +This `addPendingAckIfAllowed` method is called from Consumer's `sendMessages` method: + +```java + boolean sendingAllowed = + pendingAcks.addPendingAckIfAllowed(entry.getLedgerId(), entry.getEntryId(), batchSize, stickyKeyHash); + if (!sendingAllowed) { + // sending isn't allowed when pending acks doesn't accept adding the entry + // this happens when Key_Shared draining hashes contains the stickyKeyHash + // because of race conditions, it might be resolved at the time of sending + totalEntries--; + entries.set(i, null); + entry.release(); + if (log.isDebugEnabled()) { + log.debug("[{}-{}] Skipping sending of {}:{} ledger entry with batchSize of {} since adding" + + " to pending acks failed in broker.service.Consumer for consumerId: {}", + topicName, subscription, entry.getLedgerId(), entry.getEntryId(), batchSize, + consumerId); + } +``` + +If sending isn't allowed, the entry will be skipped from delivery. The `PendingAcksAddHandler` callback will add the message to redelivery if this is the case. +The callback maps to `handleAddingPendingAck` in the dispatcher (`PersistentStickyKeyDispatcherMultipleConsumers`). + +```java + private boolean handleAddingPendingAck(Consumer consumer, long ledgerId, long entryId, int stickyKeyHash) { + DrainingHashesTracker.DrainingHashEntry drainingHashEntry = drainingHashesTracker.getEntry(stickyKeyHash); + if (drainingHashEntry != null && drainingHashEntry.getConsumer() != consumer) { + log.warn("[{}] Another consumer id {} is already draining hash {}. Skipping adding {}:{} to pending acks " + + "for consumer {}. Adding the message to replay.", + getName(), drainingHashEntry.getConsumer(), stickyKeyHash, ledgerId, entryId, consumer); + addMessageToReplay(ledgerId, entryId, stickyKeyHash); + // block message from sending + return false; + } + if (recentReadTypeInSending == ReadType.Normal && redeliveryMessages.containsStickyKeyHash(stickyKeyHash)) { + log.warn("[{}] Sticky hash {} is already in the replay queue. " + + "Skipping adding {}:{} to pending acks. Adding the message to replay.", + getName(), stickyKeyHash, ledgerId, entryId); + addMessageToReplay(ledgerId, entryId, stickyKeyHash); + // block message from sending + return false; + } + // allow adding the message to pending acks and sending the message to the consumer + return true; + } +``` + +This logic will prevent any inconsistency when consumers get added or removed and hash ranges change while the sending of messages is already in progress. It will ensure that the view on pending acknowledgments is consistent so that the tracking of draining hashes will also be consistent in all cases. In addition, this logic will block hashes of messages that have recently been added to the redelivery queue and therefore, for message ordering reasons, should get delivered before any further message delivery happens. + +**Summary** + +This high-level design approach will meet the updated contract of preserving ordering: _"In Key_Shared subscriptions, messages with the same key are delivered and allowed to be in an unacknowledged state to only one consumer at a time."_ + +It also minimizes the impact on performance and memory usage. **The tracking only comes into play during transition states.** When consumers have been connected for a longer duration and all draining hashes have been removed, there won't be a need to check any special rules or maintain any extra state. **When the draining hashes are empty, lookups will essentially be no-ops and won't consume CPU or memory resources.** + +## Public-facing Changes + +### Topic Stats Changes & Observability + +Topic stats for the removed PIP-282 "recently joined consumers"/"last sent position" solution are removed: +- `lastSentPositionWhenJoining` field for each consumer +- `consumersAfterMarkDeletePosition` field for each Key_Shared subscription +- `individuallySentPositions` field for each Key_Shared subscription + +New topic stats will be added to monitor the "draining hashes" feature at the subscription level and consumer level: +1. `draining_hashes_count`: The current number of hashes in the draining state. +2. `draining_hashes_pending_messages`: The total number of pending messages for all draining hashes. +3. `draining_hashes_cleared_total`: The total number of hashes cleared from the draining state. +4. `draining_hashes`: Details at the hash level (available at the consumer level to reduce redundancy of information) + - hash + - number of pending messages + +For improved observability, a separate REST API for listing all pending messages ("pending acks") for a consumer will be considered. This API would allow querying which messages are currently part of a draining hash, providing a way to identify specific message IDs of messages that are holding onto a specific hash and blocking delivery to another consumer. + +## Backward & Forward Compatibility + +The "draining hashes" feature doesn't introduce backward or forward compatibility issues. The state is handled at runtime, and the changes are on the broker side without changes to the client protocol. + +Slightly unrelated to PIP-379 changes, there's a need to ensure that users upgrading from Pulsar 3.x can revert to the "recently joined consumers" logic (before PIP-282) in case of possible regressions caused by PIP-379. Since PIP-282 is also new in Pulsar 4.0.0, there needs to be a feature flag that toggles between the PIP-379 implementation for Key_Shared and the "recently joined consumers" logic before PIP-282. Implemention details for this feature toggle can be handled in the pull request for implementing this. + +## Links + +- Mailing List discussion thread: https://lists.apache.org/thread/l5zjq0fb2dscys3rsn6kfl7505tbndlx +- Mailing List voting thread: https://lists.apache.org/thread/z1kgo34qfkkvdnn3l007bdvjr3qqf4rw +- PIP-379 implementation PR: https://github.com/apache/pulsar/pull/23352 + +- [PIP-282: Change definition of the recently joined consumers position](https://github.com/apache/pulsar/blob/master/pip/pip-282.md) +- [Pulsar issue #23307: Message ordering isn't retained in Key_Shared AUTO_SPLIT mode in a rolling restart type of test scenario](https://github.com/apache/pulsar/issues/23307) +- [Pulsar issue #21199: Key_Shared subscription gets stuck after consumer reconnects](https://github.com/apache/pulsar/issues/21199) \ No newline at end of file diff --git a/pip/pip-381-large-positioninfo.md b/pip/pip-381-large-positioninfo.md new file mode 100644 index 0000000000000..9dbe1cc7935e3 --- /dev/null +++ b/pip/pip-381-large-positioninfo.md @@ -0,0 +1,153 @@ +# PIP-381: Handle large PositionInfo state + +# Background knowledge + +In case of KEY_SHARED subscription and out-of-order acknowledgments, +the PositionInfo state can be persisted to preserve the state, +with configurable maximum number of ranges to persist: + +``` +# Max number of "acknowledgment holes" that are going to be persistently stored. +# When acknowledging out of order, a consumer will leave holes that are supposed +# to be quickly filled by acking all the messages. The information of which +# messages are acknowledged is persisted by compressing in "ranges" of messages +# that were acknowledged. After the max number of ranges is reached, the information +# will only be tracked in memory and messages will be redelivered in case of +# crashes. +managedLedgerMaxUnackedRangesToPersist=10000 +``` + +The PositionInfo state is stored to the BookKeeper as a single entry, and it can grow large if the number of ranges is large. +Currently, this means that BookKeeper can fail persisting too large PositionInfo state, e.g. over 1MB +by default and the ManagedCursor recovery on topic reload might not succeed. + +There is an abandoned PIP-81 for similar problem, this PIP takes over. + +# Motivation + +While keeping the number of ranges low to prevent such problems is a common sense solution, there are cases +where the higher number of ranges is required. For example, in case of the JMS protocol handler, +JMS consumers with filters may end up processing data out of order and/or at different speed, +and the number of ranges can grow large. + +# Goals + +Store the PositionInfo state in a BookKeeper ledger as multiple entries if the state grows too large to be stored as a single entry. + +## In Scope + +Transparent backwards compatibility if the PositionInfo state is small enough. + +## Out of Scope + +Backwards compatibility in case of the PositionInfo state is too large to be stored as a single entry. + +# High Level Design + +Cursor state writes and reads are happening at the same cases as currently, without changes. + +Write path: + +1. serialize the PositionInfo state to a byte array. +2. if the byte array is smaller than the threshold, store it as a single entry, as now. Done. +3. if the byte array is larger than the threshold, split it to smaller chunks and store the chunks in a BookKeeper ledger. +4. write the "footer" into the metadata store as a last entry. + +See `persistPositionToLedger()` in `ManagedCursorImpl` for the implementation. + +The footer is a JSON representation of + +```java + public static final class ChunkSequenceFooter { + private int numParts; + private int length; + } +``` + +Read path: + +1. read the last entry from the metadata store. +2. if the entry does not appear to be a JSON, treat it as serialized PositionInfo state and use it as is. Done. +3. if the footer is a JSON, parse number of chunks and length from the json. +4. read the chunks from the BookKeeper ledger (entries from `startPos = footerPosition - chunkSequenceFooter.numParts` to `footerPosition - 1`) and merge them. +5. parse the merged byte array as a PositionInfo state. + +See `recoverFromLedgerByEntryId()` in `ManagedCursorImpl` for the implementation. + +## Design & Implementation Details + +Proposed implementation: https://github.com/apache/pulsar/pull/22799 + +## Public-facing Changes + +Nothing + +### Public API + +None + +### Binary protocol + +No public-facing changes + +### Configuration + +* **managedLedgerMaxUnackedRangesToPersist**: int, default 10000 (existing parameter). Controls number of unacked ranges to store. +* **persistentUnackedRangesWithMultipleEntriesEnabled**: boolean, default false. If true, the PositionInfo state is stored as multiple entries in BookKeeper if it grows too large. +* **persistentUnackedRangesMaxEntrySize**: int, default 1MB. Maximum size of a single entry in BookKeeper, in bytes. +* **cursorInfoCompressionType**: string, default "NONE". Compression type to use for the PositionInfo state. + +### CLI + +None + +### Metrics + + + + +# Monitoring + +Existing monitoring should be sufficient. + +# Security Considerations + +N/A + +# Backward & Forward Compatibility + +## Upgrade + +Not affected, just upgrade. + +## Downgrade / Rollback + +Not affected, just downgrade **as long as the managedLedgerMaxUnackedRangesToPersist was in the range to fit it into a single entry in BK**. + +## Pulsar Geo-Replication Upgrade & Downgrade/Rollback Considerations + +Not affected AFAIK. + +# Alternatives + +1. Do nothing. Keep the number of ranges low. This does not fit some use cases. +2. Come up with an extremely efficient storage format for the unacked ranges to fit them into a single entry all the time for e.g. 10mil ranges. This breaks backwards compatibility and the feasibility is unclear. + +# General Notes + +# Links + +* Proposed implementation: https://github.com/apache/pulsar/pull/22799 +* PIP-81: https://github.com/apache/pulsar/wiki/PIP-81:-Split-the-individual-acknowledgments-into-multiple-entries +* PR that implements better storage format for the unacked ranges (alternative 2): https://github.com/apache/pulsar/pull/9292 + +ML discussion and voting threads: + +* Mailing List discussion thread: https://lists.apache.org/thread/8sm0h804v5914zowghrqxr92fp7c255d +* Mailing List voting thread: https://lists.apache.org/thread/q31fx0rox9tdt34xsmo1ol1l76q8vk99 diff --git a/pip/pip_307.md b/pip/pip_307.md new file mode 100644 index 0000000000000..6371983592b2f --- /dev/null +++ b/pip/pip_307.md @@ -0,0 +1,30 @@ +# Background knowledge + +WebSocket currently only supports the consumption of a single topic, which cannot satisfy users' consumption scenarios of multiple topics. + +# Motivation + +Supports consumption of multiple topics or pattern topics. + + +# Detailed Design + +Currently, the topic name is specified through path for consumption, like: +``` +/ws/v2/consumer/persistent/my-property/my-ns/my-topic/my-subscription +``` +If we want to support subscribing multi-topics, adding parameters will be confusing. Therefore, add a new v3 request path as follows: + +For consumption of pattern-topics: +``` +/ws/v3/consumer/subscription?topicsPattern="a.*" +``` +For consumption of multi-topics: +``` +/ws/v3/consumer/subscription?topics="a,b,c" +``` + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/co8396ywny161x91dffzvxlt993mo1ht +* Mailing List voting thread: https://lists.apache.org/thread/lk28o483y351s7m44p018320gq3g4507 diff --git a/pom.xml b/pom.xml index 75c25e5478893..c50357b840616 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT Pulsar Pulsar is a distributed pub-sub messaging platform with a very @@ -82,23 +82,26 @@ flexible messaging model and an intuitive client API. ${maven.compiler.target} 8 - 2.10.1 + 3.5.0 + + 21 **/Test*.java,**/*Test.java,**/*Tests.java,**/*TestCase.java - quarantine + quarantine,flaky UTF-8 UTF-8 - 2023-05-03T02:53:27Z + 2024-08-09T08:42:01Z true + true @@ -111,6 +114,7 @@ flexible messaging model and an intuitive client API. --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED --add-opens java.base/jdk.internal.platform=ALL-UNNAMED + 1300M true 4 false @@ -125,75 +129,78 @@ flexible messaging model and an intuitive client API. /tmp kill apachepulsar + pulsar + latest false false package package - 1.21 + 1.26.0 - 4.16.1 - 3.8.1 + 4.17.1 + 3.9.2 1.5.0 1.10.0 - 1.1.8.4 + 1.1.10.5 4.1.12.1 5.1.0 - 4.1.89.Final - 0.0.18.Final - 9.4.48.v20220622 + 4.1.113.Final + 0.0.24.Final + 9.4.54.v20240208 2.5.2 - 2.34 + 2.42 1.10.50 0.16.0 - 4.3.8 + 4.5.10 7.9.2 - 1.7.32 + 2.0.13 4.4 - 2.18.0 - 1.69 - 1.0.6 - 1.0.2.3 - 2.14.2 + 2.23.1 + 1.78.1 + 1.0.7 + 1.0.2.5 + 2.17.2 0.10.2 1.6.2 - 8.37 - 0.42.1 + 10.14.2 + 0.45.0 true 0.5.0 - 3.19.6 + 1.14.12 + 1.17 + 3.25.5 ${protobuf3.version} - 1.45.1 + 1.56.1 1.41.0 - 0.19.0 + 0.26.0 ${grpc.version} 2.8.9 1.2.1 0.8.3 2.2.0 3.11.2 - 4.4.20 + 4.5.0 3.4.0 - 5.5.3 - 1.12.262 - 1.10.2 + 5.18.0 + 1.12.638 + 1.11.4 2.10.10 - 2.5.0 + 2.6.0 5.1.0 - 3.36.0.3 + 3.42.0.0 8.0.11 - 42.5.1 + 42.5.5 0.4.6 2.7.5 0.4.4-hotfix1 3.3.5 2.4.10 - 1.2.4 - 8.5.2 - 368 + 2.16.0 + 8.12.1 1.9.7.Final - 42.5.0 + 42.5.5 8.0.30 1.15.16.Final @@ -202,16 +209,15 @@ flexible messaging model and an intuitive client API. 2.10.2 3.3.5 2.4.16 - 31.0.1-jre + 32.1.2-jre 1.0 0.16.1 6.2.8 - 0.20 + 0.27 2.12.1 - 1.82 3.11 1.10 - 2.8.0 + 2.14.0 1.15 2.1 2.1.9 @@ -222,8 +228,8 @@ flexible messaging model and an intuitive client API. 3.21.0 0.9.1 2.1.0 - 3.18.1 - 1.18.26 + 3.24.2 + 1.18.32 1.3.2 2.3.1 1.2.0 @@ -232,9 +238,10 @@ flexible messaging model and an intuitive client API. 2.0.2 5.12.1 18.0.0 + 0.9.4 4.9.3 - 2.8.0 + 3.4.0 1.8.20 1.0 @@ -242,31 +249,42 @@ flexible messaging model and an intuitive client API. 5.3.27 4.5.13 4.4.15 - 0.7.5 + 0.7.7 + 0.4.5 2.0 1.10.12 - 5.3.3 + 5.5.0 3.4.3 1.5.2-3 2.0.6 + 1.38.0 + ${opentelemetry.version}-alpha + 1.33.3 + ${opentelemetry.instrumentation.version}-alpha + 1.25.0-alpha + 4.7.5 + 1.7 + 0.3.6 + 3.3.2 - 1.17.6 + 1.18.3 2.2 + 5.4.0 - 3.2.13 + 3.3.0 1.1.1 7.7.1 - 3.12.4 + 5.6.0 3.25.0-GA 1.5.0 - 3.1 + 3.3 4.2.0 - 1.2.22 1.5.4 5.4.0 2.33.2 + 1.0.3 0.6.1 @@ -281,29 +299,27 @@ flexible messaging model and an intuitive client API. 3.11.0 3.5.0 2.3.0 - 3.4.1 + 3.6.0 3.1.0 + 3.6.0 1.1.0 - 1.3.4 + 1.5.0 3.1.2 - 4.0.2 + 4.9.10 3.5.3 1.7.0 - 0.8.8 - 4.7.3.0 + 0.8.12 + 4.7.3.6 4.7.3 - 2.5.1 - 9+181-r4173-1 - 0.1.4 + 2.24.0 + 0.1.21 1.3 0.4 - 8.1.2 - 0.9.44 + 10.0.2 + 1.2.0 1.6.1 6.4.0 - - - rename-netty-native-libs.sh + 3.33.0 @@ -340,6 +356,10 @@ flexible messaging model and an intuitive client API. org.yaml * + + org.slf4j + * + @@ -364,9 +384,9 @@ flexible messaging model and an intuitive client API. - org.mockito - mockito-inline - ${mockito.version} + dev.failsafe + failsafe + ${failsafe.version} @@ -407,6 +427,12 @@ flexible messaging model and an intuitive client API. io.dropwizard.metrics metrics-graphite ${dropwizardmetrics.version} + + + com.rabbitmq + amqp-client + + io.dropwizard.metrics @@ -448,6 +474,10 @@ flexible messaging model and an intuitive client API. org.bouncycastle * + + log4j-slf4j-impl + org.apache.logging.log4j + slf4j-log4j12 org.slf4j @@ -484,6 +514,11 @@ flexible messaging model and an intuitive client API. vertx-web ${vertx.version} + + io.vertx + vertx-grpc + ${vertx.version} + org.apache.curator @@ -542,6 +577,14 @@ flexible messaging model and an intuitive client API. com.squareup.okio okio + + jose4j + org.bitbucket.b_c + + + io.grpc + grpc-netty + @@ -578,6 +621,13 @@ flexible messaging model and an intuitive client API. + + io.grpc + grpc-util + + 1.60.0 + + org.apache.bookkeeper bookkeeper-common @@ -608,12 +658,30 @@ flexible messaging model and an intuitive client API. ${bookkeeper.version} + + com.google.re2j + re2j + ${re2j.version} + + + + com.spotify + completable-futures + ${completable-futures.version} + + org.rocksdb rocksdbjni ${rocksdb.version} + + org.bitbucket.b_c + jose4j + ${jose4j.version} + + org.eclipse.jetty jetty-server @@ -672,9 +740,15 @@ flexible messaging model and an intuitive client API. - com.beust - jcommander - ${jcommander.version} + info.picocli + picocli + ${picocli.version} + + + + info.picocli + picocli-shell-jline3 + ${picocli.version} @@ -727,20 +801,10 @@ flexible messaging model and an intuitive client API. org.slf4j - slf4j-api - ${slf4j.version} - - - - org.slf4j - slf4j-simple - ${slf4j.version} - - - - org.slf4j - jcl-over-slf4j + slf4j-bom ${slf4j.version} + pom + import @@ -818,9 +882,15 @@ flexible messaging model and an intuitive client API. - com.github.docker-java - docker-java-core - ${docker-java.version} + com.github.docker-java + docker-java-core + ${docker-java.version} + + + org.bouncycastle + * + + com.github.docker-java @@ -886,7 +956,7 @@ flexible messaging model and an intuitive client API. org.bouncycastle - bcpkix-jdk15on + bcpkix-jdk18on ${bouncycastle.version} @@ -918,6 +988,24 @@ flexible messaging model and an intuitive client API. com.yahoo.athenz athenz-cert-refresher ${athenz.version} + + + org.bouncycastle + * + + + + + + com.yahoo.athenz + athenz-auth-core + ${athenz.version} + + + org.bouncycastle + * + + @@ -972,12 +1060,51 @@ flexible messaging model and an intuitive client API. io.etcd jetcd-core ${jetcd.version} + + + io.grpc + grpc-netty + + - io.etcd jetcd-test ${jetcd.version} + + + io.grpc + grpc-netty + + + io.etcd + jetcd-core + + + io.etcd + jetcd-api + + + io.vertx + * + + + + + ${project.groupId} + jetcd-core-shaded + ${project.version} + shaded + + + io.etcd + * + + + io.vertx + * + + @@ -1030,6 +1157,18 @@ flexible messaging model and an intuitive client API. ${typetools.version} + + net.bytebuddy + byte-buddy + ${byte-buddy.version} + + + + org.zeroturnaround + zt-zip + ${zt-zip.version} + + io.grpc grpc-bom @@ -1059,6 +1198,22 @@ flexible messaging model and an intuitive client API. com.squareup.okio okio + + io.grpc + grpc-netty + + + + + + io.grpc + grpc-xds + ${grpc.version} + + + org.bouncycastle + * + @@ -1113,6 +1268,17 @@ flexible messaging model and an intuitive client API. ${sketches.version} + + io.streamnative.oxia + oxia-client + ${oxia.version} + + + io.streamnative.oxia + oxia-testcontainers + ${oxia.version} + + com.amazonaws aws-java-sdk-bom @@ -1372,12 +1538,6 @@ flexible messaging model and an intuitive client API. ${netty-reactive-streams.version} - - ch.qos.reload4j - reload4j - ${reload4j.version} - - org.roaringbitmap RoaringBitmap @@ -1388,6 +1548,52 @@ flexible messaging model and an intuitive client API. oshi-core-java11 ${oshi.version} + + org.checkerframework + checker-qual + ${checkerframework.version} + + + + io.rest-assured + rest-assured + ${restassured.version} + test + + + + io.opentelemetry + opentelemetry-bom + ${opentelemetry.version} + pom + import + + + io.opentelemetry + opentelemetry-bom-alpha + ${opentelemetry.alpha.version} + pom + import + + + io.opentelemetry.instrumentation + opentelemetry-instrumentation-bom + ${opentelemetry.instrumentation.version} + pom + import + + + io.opentelemetry.instrumentation + opentelemetry-instrumentation-bom-alpha + ${opentelemetry.instrumentation.alpha.version} + pom + import + + + io.opentelemetry.semconv + opentelemetry-semconv + ${opentelemetry.semconv.version} + @@ -1412,12 +1618,6 @@ flexible messaging model and an intuitive client API. test - - org.mockito - mockito-inline - test - - com.github.stefanbirkner system-lambda @@ -1431,6 +1631,12 @@ flexible messaging model and an intuitive client API. test + + io.opentelemetry + opentelemetry-sdk-testing + test + + org.projectlombok lombok @@ -1470,6 +1676,24 @@ flexible messaging model and an intuitive client API. org.apache.zookeeper * + + log4j-slf4j-impl + org.apache.logging.log4j + + + + + + org.apache.bookkeeper + bookkeeper-common + ${bookkeeper.version} + test + tests + + + com.fasterxml.jackson.core + * + @@ -1513,7 +1737,7 @@ flexible messaging model and an intuitive client API. org.apache.maven.plugins maven-surefire-plugin - ${testJacocoAgentArgument} -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${testHeapDumpPath} -XX:+ExitOnOutOfMemoryError -Xmx1G -XX:+UseZGC + ${testJacocoAgentArgument} -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${testHeapDumpPath} -XX:+ExitOnOutOfMemoryError -Xmx${testMaxHeapSize} -XX:+UseZGC -Dpulsar.allocator.pooled=true -Dpulsar.allocator.leak_detection=Advanced -Dpulsar.allocator.exit_on_oom=false @@ -1536,7 +1760,7 @@ flexible messaging model and an intuitive client API. listener - org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.JacocoDumpListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier,org.apache.pulsar.tests.MockitoCleanupListener,org.apache.pulsar.tests.FastThreadLocalCleanupListener,org.apache.pulsar.tests.ThreadLeakDetectorListener,org.apache.pulsar.tests.SingletonCleanerListener + org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.JacocoDumpListener,org.apache.pulsar.tests.TraceTestResourceCleanupListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier,org.apache.pulsar.tests.MockitoCleanupListener,org.apache.pulsar.tests.FastThreadLocalCleanupListener,org.apache.pulsar.tests.ThreadLeakDetectorListener,org.apache.pulsar.tests.SingletonCleanerListener @@ -1571,6 +1795,31 @@ flexible messaging model and an intuitive client API. + + pl.project13.maven + git-commit-id-plugin + ${git-commit-id-plugin.version} + + + git-info + + revision + + + + + true + true + git + false + false + false + + true + + + + com.mycila license-maven-plugin @@ -1609,7 +1858,7 @@ flexible messaging model and an intuitive client API. **/ByteBufCodedOutputStream.java **/ahc.properties bin/proto/* - conf/schema_example.conf + conf/schema_example.json data/** logs/** **/circe/** @@ -1708,7 +1957,6 @@ flexible messaging model and an intuitive client API. **/META-INF/services/com.scurrilous.circe.HashProvider - **/META-INF/services/io.trino.spi.Plugin **/django/stats/migrations/*.py @@ -1718,6 +1966,8 @@ flexible messaging model and an intuitive client API. **/*.crt **/*.key **/*.csr + **/*.srl + **/*.txt **/*.pem **/*.json **/*.htpasswd @@ -1734,7 +1984,7 @@ flexible messaging model and an intuitive client API. **/requirements.txt - conf/schema_example.conf + conf/schema_example.json **/templates/*.tpl @@ -1785,8 +2035,8 @@ flexible messaging model and an intuitive client API. - 17 - Java 17+ is required to build Pulsar. + [17,18),[21,22) + Java 17 or Java 21 is required to build Pulsar. 3.6.1 @@ -1797,32 +2047,29 @@ flexible messaging model and an intuitive client API. - org.apache.maven.plugins - maven-assembly-plugin - ${maven-assembly-plugin.version} - false - - - source-release-assembly-tar-gz - generate-sources - - single - - - ${skipSourceReleaseAssembly} - true - - src/assembly-source-package.xml - - apache-pulsar-${project.version}-src - false - - tar.gz - - posix - - - + org.codehaus.mojo + properties-maven-plugin + + + initialize + + set-system-properties + + + + + + proto_path + ${pulsar.basedir} + + + proto_search_strategy + 2 + + + + + @@ -1951,6 +2198,21 @@ flexible messaging model and an intuitive client API. docker-maven-plugin ${docker-maven.version} + + org.codehaus.mojo + build-helper-maven-plugin + ${build-helper-maven-plugin.version} + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + NIST_NVD_API_KEY + + + + @@ -2128,10 +2390,6 @@ flexible messaging model and an intuitive client API. Windows - - rename-netty-native-libs.cmd - - @@ -2151,8 +2409,10 @@ flexible messaging model and an intuitive client API. managed-ledger tiered-storage pulsar-common + pulsar-bom pulsar-broker-common pulsar-broker + pulsar-cli-utils pulsar-client-api pulsar-client pulsar-client-shaded @@ -2165,16 +2425,17 @@ flexible messaging model and an intuitive client API. pulsar-client-tools-customcommand-example pulsar-client-tools-test pulsar-client-all + pulsar-docs-tools pulsar-websocket pulsar-proxy pulsar-testclient pulsar-broker-auth-athenz pulsar-client-auth-athenz - pulsar-sql pulsar-broker-auth-oidc pulsar-broker-auth-sasl pulsar-client-auth-sasl pulsar-config-validation + pulsar-opentelemetry structured-event-log @@ -2193,6 +2454,7 @@ flexible messaging model and an intuitive client API. pulsar-client-messagecrypto-bc pulsar-metadata + jetcd-core-shaded jclouds-shaded @@ -2203,6 +2465,8 @@ flexible messaging model and an intuitive client API. distribution docker tests + + microbench @@ -2217,8 +2481,10 @@ flexible messaging model and an intuitive client API. testmocks managed-ledger pulsar-common + pulsar-bom pulsar-broker-common pulsar-broker + pulsar-cli-utils pulsar-client-api pulsar-client pulsar-client-admin-api @@ -2227,6 +2493,7 @@ flexible messaging model and an intuitive client API. pulsar-client-tools pulsar-client-tools-customcommand-example pulsar-client-tools-test + pulsar-docs-tools pulsar-websocket pulsar-proxy pulsar-testclient @@ -2234,6 +2501,7 @@ flexible messaging model and an intuitive client API. pulsar-broker-auth-sasl pulsar-client-auth-sasl pulsar-config-validation + pulsar-opentelemetry pulsar-transaction @@ -2252,7 +2520,7 @@ flexible messaging model and an intuitive client API. distribution pulsar-metadata - + jetcd-core-shaded pulsar-package-management @@ -2292,13 +2560,22 @@ flexible messaging model and an intuitive client API. true 128m 1024m - false -XDcompilePolicy=simple -Xlint:-options -Xplugin:ErrorProne -XepExcludedPaths:.*/target/generated-sources/.* -XepDisableWarningsInGeneratedCode -Xep:UnusedVariable:OFF -Xep:FallThrough:OFF -Xep:OverrideThrowableToString:OFF -Xep:UnusedMethod:OFF -Xep:StringSplitter:OFF -Xep:CanonicalDuration:OFF -Xep:Slf4jDoNotLogMessageOfExceptionExplicitly:WARN -Xep:Slf4jSignOnlyFormat:WARN -Xep:Slf4jFormatShouldBeConst:WARN -Xep:Slf4jLoggerShouldBePrivate:WARN -Xep:Slf4jLoggerShouldBeNonStatic:OFF + -J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED @@ -2383,7 +2660,6 @@ flexible messaging model and an intuitive client API. org.owasp dependency-check-maven - ${dependency-check-maven.version} ${pulsar.basedir}/src/owasp-dependency-check-false-positives.xml @@ -2418,7 +2694,6 @@ flexible messaging model and an intuitive client API. org.owasp dependency-check-maven - ${dependency-check-maven.version} @@ -2439,11 +2714,51 @@ flexible messaging model and an intuitive client API. - pulsar-sql-tests + pulsar-io-elastic-tests - pulsar-sql + pulsar-io + + + pulsar-io-kafka-connect-tests + + pulsar-io + + + + + microbench + + microbench + + + true + true + true + true + true + true + none + true + true + true + + + + + jdk-major-version-set + + + env.IMAGE_JDK_MAJOR_VERSION + + + + + ${env.IMAGE_JDK_MAJOR_VERSION} + + + diff --git a/pulsar-bom/pom.xml b/pulsar-bom/pom.xml new file mode 100644 index 0000000000000..d195411fa6479 --- /dev/null +++ b/pulsar-bom/pom.xml @@ -0,0 +1,715 @@ + + + + 4.0.0 + + pom + + org.apache + apache + 29 + + + + org.apache.pulsar + pulsar-bom + 4.0.0-SNAPSHOT + Pulsar BOM + Pulsar (Bill of Materials) + + https://github.com/apache/pulsar + + + Apache Software Foundation + https://www.apache.org/ + + 2017 + + + + Apache Pulsar developers + https://pulsar.apache.org/ + + + + + + Apache License, Version 2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + https://github.com/apache/pulsar + scm:git:https://github.com/apache/pulsar.git + scm:git:ssh://git@github.com:apache/pulsar.git + + + + GitHub Actions + https://github.com/apache/pulsar/actions + + + + Github + https://github.com/apache/pulsar/issues + + + + 17 + 17 + UTF-8 + UTF-8 + 2024-08-09T08:42:01Z + 4.1 + 3.1.2 + 3.5.3 + + + + + + com.mycila + license-maven-plugin + ${license-maven-plugin.version} + + + +

../src/license-header.txt
+ + + + + + org.apache.rat + apache-rat-plugin + + + + dependency-reduced-pom.xml + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + ${maven-checkstyle-plugin.version} + + true + + + + + + org.apache.maven.wagon + wagon-ssh-external + ${wagon-ssh-external.version} + + + + + + + + + + + org.apache.pulsar + bouncy-castle-bc + ${project.version} + + + org.apache.pulsar + bouncy-castle-bcfips + ${project.version} + + + org.apache.pulsar + bouncy-castle-parent + ${project.version} + + + org.apache.pulsar + buildtools + ${project.version} + + + org.apache.pulsar + distribution + ${project.version} + + + org.apache.pulsar + docker-images + ${project.version} + + + org.apache.pulsar + jclouds-shaded + ${project.version} + + + org.apache.pulsar + managed-ledger + ${project.version} + + + org.apache.pulsar + pulsar-all-docker-image + ${project.version} + + + org.apache.pulsar + pulsar-broker-auth-athenz + ${project.version} + + + org.apache.pulsar + pulsar-broker-auth-oidc + ${project.version} + + + org.apache.pulsar + pulsar-broker-auth-sasl + ${project.version} + + + org.apache.pulsar + pulsar-broker-common + ${project.version} + + + org.apache.pulsar + pulsar-broker + ${project.version} + + + org.apache.pulsar + pulsar-cli-utils + ${project.version} + + + org.apache.pulsar + pulsar-client-1x-base + ${project.version} + + + org.apache.pulsar + pulsar-client-1x + ${project.version} + + + org.apache.pulsar + pulsar-client-2x-shaded + ${project.version} + + + org.apache.pulsar + pulsar-client-admin-api + ${project.version} + + + org.apache.pulsar + pulsar-client-admin-original + ${project.version} + + + org.apache.pulsar + pulsar-client-admin + ${project.version} + + + org.apache.pulsar + pulsar-client-all + ${project.version} + + + org.apache.pulsar + pulsar-client-api + ${project.version} + + + org.apache.pulsar + pulsar-client-auth-athenz + ${project.version} + + + org.apache.pulsar + pulsar-client-auth-sasl + ${project.version} + + + org.apache.pulsar + pulsar-client-messagecrypto-bc + ${project.version} + + + org.apache.pulsar + pulsar-client-original + ${project.version} + + + org.apache.pulsar + pulsar-client-tools-api + ${project.version} + + + org.apache.pulsar + pulsar-client-tools + ${project.version} + + + org.apache.pulsar + pulsar-client + ${project.version} + + + org.apache.pulsar + pulsar-common + ${project.version} + + + org.apache.pulsar + pulsar-config-validation + ${project.version} + + + org.apache.pulsar + pulsar-docker-image + ${project.version} + + + org.apache.pulsar + pulsar-docs-tools + ${project.version} + + + org.apache.pulsar + pulsar-functions-api-examples-builtin + ${project.version} + + + org.apache.pulsar + pulsar-functions-api-examples + ${project.version} + + + org.apache.pulsar + pulsar-functions-api + ${project.version} + + + org.apache.pulsar + pulsar-functions-instance + ${project.version} + + + org.apache.pulsar + pulsar-functions-local-runner-original + ${project.version} + + + org.apache.pulsar + pulsar-functions-local-runner + ${project.version} + + + org.apache.pulsar + pulsar-functions-proto + ${project.version} + + + org.apache.pulsar + pulsar-functions-runtime-all + ${project.version} + + + org.apache.pulsar + pulsar-functions-runtime + ${project.version} + + + org.apache.pulsar + pulsar-functions-secrets + ${project.version} + + + org.apache.pulsar + pulsar-functions-utils + ${project.version} + + + org.apache.pulsar + pulsar-functions-worker + ${project.version} + + + org.apache.pulsar + pulsar-functions + ${project.version} + + + org.apache.pulsar + pulsar-io-aerospike + ${project.version} + + + org.apache.pulsar + pulsar-io-alluxio + ${project.version} + + + org.apache.pulsar + pulsar-io-aws + ${project.version} + + + org.apache.pulsar + pulsar-io-batch-data-generator + ${project.version} + + + org.apache.pulsar + pulsar-io-batch-discovery-triggerers + ${project.version} + + + org.apache.pulsar + pulsar-io-canal + ${project.version} + + + org.apache.pulsar + pulsar-io-cassandra + ${project.version} + + + org.apache.pulsar + pulsar-io-common + ${project.version} + + + org.apache.pulsar + pulsar-io-core + ${project.version} + + + org.apache.pulsar + pulsar-io-data-generator + ${project.version} + + + org.apache.pulsar + pulsar-io-debezium-core + ${project.version} + + + org.apache.pulsar + pulsar-io-debezium-mongodb + ${project.version} + + + org.apache.pulsar + pulsar-io-debezium-mssql + ${project.version} + + + org.apache.pulsar + pulsar-io-debezium-mysql + ${project.version} + + + org.apache.pulsar + pulsar-io-debezium-oracle + ${project.version} + + + org.apache.pulsar + pulsar-io-debezium-postgres + ${project.version} + + + org.apache.pulsar + pulsar-io-debezium + ${project.version} + + + org.apache.pulsar + pulsar-io-distribution + ${project.version} + + + org.apache.pulsar + pulsar-io-docs + ${project.version} + + + org.apache.pulsar + pulsar-io-dynamodb + ${project.version} + + + org.apache.pulsar + pulsar-io-elastic-search + ${project.version} + + + org.apache.pulsar + pulsar-io-file + ${project.version} + + + org.apache.pulsar + pulsar-io-flume + ${project.version} + + + org.apache.pulsar + pulsar-io-hbase + ${project.version} + + + org.apache.pulsar + pulsar-io-hdfs2 + ${project.version} + + + org.apache.pulsar + pulsar-io-hdfs3 + ${project.version} + + + org.apache.pulsar + pulsar-io-http + ${project.version} + + + org.apache.pulsar + pulsar-io-influxdb + ${project.version} + + + org.apache.pulsar + pulsar-io-jdbc-clickhouse + ${project.version} + + + org.apache.pulsar + pulsar-io-jdbc-core + ${project.version} + + + org.apache.pulsar + pulsar-io-jdbc-mariadb + ${project.version} + + + org.apache.pulsar + pulsar-io-jdbc-openmldb + ${project.version} + + + org.apache.pulsar + pulsar-io-jdbc-postgres + ${project.version} + + + org.apache.pulsar + pulsar-io-jdbc-sqlite + ${project.version} + + + org.apache.pulsar + pulsar-io-jdbc + ${project.version} + + + org.apache.pulsar + pulsar-io-kafka-connect-adaptor-nar + ${project.version} + + + org.apache.pulsar + pulsar-io-kafka-connect-adaptor + ${project.version} + + + org.apache.pulsar + pulsar-io-kafka + ${project.version} + + + org.apache.pulsar + pulsar-io-kinesis + ${project.version} + + + org.apache.pulsar + pulsar-io-mongo + ${project.version} + + + org.apache.pulsar + pulsar-io-netty + ${project.version} + + + org.apache.pulsar + pulsar-io-nsq + ${project.version} + + + org.apache.pulsar + pulsar-io-rabbitmq + ${project.version} + + + org.apache.pulsar + pulsar-io-redis + ${project.version} + + + org.apache.pulsar + pulsar-io-solr + ${project.version} + + + org.apache.pulsar + pulsar-io-twitter + ${project.version} + + + org.apache.pulsar + pulsar-io + ${project.version} + + + org.apache.pulsar + pulsar-metadata + ${project.version} + + + org.apache.pulsar + pulsar-offloader-distribution + ${project.version} + + + org.apache.pulsar + pulsar-package-bookkeeper-storage + ${project.version} + + + org.apache.pulsar + pulsar-package-core + ${project.version} + + + org.apache.pulsar + pulsar-package-filesystem-storage + ${project.version} + + + org.apache.pulsar + pulsar-package-management + ${project.version} + + + org.apache.pulsar + pulsar-proxy + ${project.version} + + + org.apache.pulsar + pulsar-server-distribution + ${project.version} + + + org.apache.pulsar + pulsar-shell-distribution + ${project.version} + + + org.apache.pulsar + pulsar-testclient + ${project.version} + + + org.apache.pulsar + pulsar-transaction-common + ${project.version} + + + org.apache.pulsar + pulsar-transaction-coordinator + ${project.version} + + + org.apache.pulsar + pulsar-transaction-parent + ${project.version} + + + org.apache.pulsar + pulsar-websocket + ${project.version} + + + org.apache.pulsar + pulsar + ${project.version} + + + org.apache.pulsar + structured-event-log + ${project.version} + + + org.apache.pulsar + testmocks + ${project.version} + + + org.apache.pulsar + tiered-storage-file-system + ${project.version} + + + org.apache.pulsar + tiered-storage-jcloud + ${project.version} + + + org.apache.pulsar + tiered-storage-parent + ${project.version} + + + + diff --git a/pulsar-broker-auth-athenz/pom.xml b/pulsar-broker-auth-athenz/pom.xml index 9c39e07b620ce..991437847abcb 100644 --- a/pulsar-broker-auth-athenz/pom.xml +++ b/pulsar-broker-auth-athenz/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-broker-auth-athenz @@ -53,6 +53,11 @@ athenz-zpe-java-client + + org.bouncycastle + bcpkix-jdk18on + + diff --git a/pulsar-broker-auth-athenz/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenz.java b/pulsar-broker-auth-athenz/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenz.java index 652a922b9a5ad..499ebefc8a081 100644 --- a/pulsar-broker-auth-athenz/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenz.java +++ b/pulsar-broker-auth-athenz/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenz.java @@ -43,6 +43,8 @@ public class AuthenticationProviderAthenz implements AuthenticationProvider { private List domainNameList = null; private int allowedOffset = 30; + private AuthenticationMetrics authenticationMetrics; + public enum ErrorCode { UNKNOWN, NO_CLIENT, @@ -54,6 +56,14 @@ public enum ErrorCode { @Override public void initialize(ServiceConfiguration config) throws IOException { + initialize(Context.builder().config(config).build()); + } + + @Override + public void initialize(Context context) throws IOException { + authenticationMetrics = new AuthenticationMetrics(context.getOpenTelemetry(), + getClass().getSimpleName(), getAuthMethodName()); + var config = context.getConfig(); String domainNames; if (config.getProperty(DOMAIN_NAME_LIST) != null) { domainNames = (String) config.getProperty(DOMAIN_NAME_LIST); @@ -86,6 +96,11 @@ public String getAuthMethodName() { return "athenz"; } + @Override + public void incrementFailureMetric(Enum errorCode) { + authenticationMetrics.recordFailure(errorCode); + } + @Override public String authenticate(AuthenticationDataSource authData) throws AuthenticationException { SocketAddress clientAddress; @@ -141,7 +156,7 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat if (token.validate(ztsPublicKey, allowedOffset, false, null)) { log.debug("Athenz Role Token : {}, Authenticated for Client: {}", roleToken, clientAddress); - AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName()); + authenticationMetrics.recordSuccess(); return token.getPrincipal(); } else { errorCode = ErrorCode.INVALID_TOKEN; diff --git a/pulsar-broker-auth-athenz/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenzTest.java b/pulsar-broker-auth-athenz/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenzTest.java index 89ee5ca083079..63dcd09397886 100644 --- a/pulsar-broker-auth-athenz/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenzTest.java +++ b/pulsar-broker-auth-athenz/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenzTest.java @@ -20,10 +20,8 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.fail; - import com.yahoo.athenz.auth.token.RoleToken; import com.yahoo.athenz.zpe.ZpeConsts; - import java.io.IOException; import java.net.InetSocketAddress; import java.nio.file.Files; @@ -31,9 +29,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Properties; - import javax.naming.AuthenticationException; - import org.apache.pulsar.broker.ServiceConfiguration; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -55,7 +51,7 @@ public void setup() throws Exception { // Initialize authentication provider provider = new AuthenticationProviderAthenz(); - provider.initialize(config); + provider.initialize(AuthenticationProvider.Context.builder().config(config).build()); // Specify Athenz configuration file for AuthZpeClient which is used in AuthenticationProviderAthenz System.setProperty(ZpeConsts.ZPE_PROP_ATHENZ_CONF, "./src/test/resources/athenz.conf.test"); @@ -69,7 +65,7 @@ public void testInitilizeFromSystemPropeties() { emptyConf.setProperties(emptyProp); AuthenticationProviderAthenz sysPropProvider1 = new AuthenticationProviderAthenz(); try { - sysPropProvider1.initialize(emptyConf); + sysPropProvider1.initialize(AuthenticationProvider.Context.builder().config(emptyConf).build()); assertEquals(sysPropProvider1.getAllowedOffset(), 30); // default allowed offset is 30 sec } catch (Exception e) { fail("Fail to Read pulsar.athenz.domain.names from System Properties"); @@ -78,16 +74,16 @@ public void testInitilizeFromSystemPropeties() { System.setProperty("pulsar.athenz.role.token_allowed_offset", "0"); AuthenticationProviderAthenz sysPropProvider2 = new AuthenticationProviderAthenz(); try { - sysPropProvider2.initialize(config); + sysPropProvider2.initialize(AuthenticationProvider.Context.builder().config(config).build()); assertEquals(sysPropProvider2.getAllowedOffset(), 0); } catch (Exception e) { - fail("Failed to get allowd offset from system property"); + fail("Failed to get allowed offset from system property"); } System.setProperty("pulsar.athenz.role.token_allowed_offset", "invalid"); AuthenticationProviderAthenz sysPropProvider3 = new AuthenticationProviderAthenz(); try { - sysPropProvider3.initialize(config); + sysPropProvider3.initialize(AuthenticationProvider.Context.builder().config(config).build()); fail("Invalid allowed offset should not be specified"); } catch (IOException e) { } @@ -95,7 +91,7 @@ public void testInitilizeFromSystemPropeties() { System.setProperty("pulsar.athenz.role.token_allowed_offset", "-1"); AuthenticationProviderAthenz sysPropProvider4 = new AuthenticationProviderAthenz(); try { - sysPropProvider4.initialize(config); + sysPropProvider4.initialize(AuthenticationProvider.Context.builder().config(config).build()); fail("Negative allowed offset should not be specified"); } catch (IOException e) { } diff --git a/pulsar-broker-auth-oidc/pom.xml b/pulsar-broker-auth-oidc/pom.xml index ca2b623d96eae..a398b752b5fad 100644 --- a/pulsar-broker-auth-oidc/pom.xml +++ b/pulsar-broker-auth-oidc/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-broker-auth-oidc @@ -186,6 +186,28 @@ + + maven-resources-plugin + + + copy-resources + test-compile + + copy-resources + + + ${project.build.testOutputDirectory}/certificate-authority + true + + + ${project.parent.basedir}/tests/certificate-authority + false + + + + + + diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java index 2078666a08dd9..38f618091333a 100644 --- a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java @@ -52,6 +52,8 @@ import java.util.concurrent.CompletableFuture; import javax.naming.AuthenticationException; import javax.net.ssl.SSLSession; +import okhttp3.OkHttpClient; +import org.apache.commons.lang.StringUtils; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.authentication.AuthenticationProvider; @@ -86,8 +88,6 @@ public class AuthenticationProviderOpenID implements AuthenticationProvider { private static final Logger log = LoggerFactory.getLogger(AuthenticationProviderOpenID.class); - private static final String SIMPLE_NAME = AuthenticationProviderOpenID.class.getSimpleName(); - // Must match the value used by the OAuth2 Client Plugin. private static final String AUTH_METHOD_NAME = "token"; @@ -144,9 +144,20 @@ public class AuthenticationProviderOpenID implements AuthenticationProvider { // The list of audiences that are allowed to connect to this broker. A valid JWT must contain one of the audiences. private String[] allowedAudiences; + private ApiClient k8sApiClient; + + private AuthenticationMetrics authenticationMetrics; @Override public void initialize(ServiceConfiguration config) throws IOException { + initialize(Context.builder().config(config).build()); + } + + @Override + public void initialize(Context context) throws IOException { + authenticationMetrics = new AuthenticationMetrics(context.getOpenTelemetry(), + getClass().getSimpleName(), getAuthMethodName()); + var config = context.getConfig(); this.allowedAudiences = validateAllowedAudiences(getConfigValueAsSet(config, ALLOWED_AUDIENCES)); this.roleClaim = getConfigValueAsString(config, ROLE_CLAIM, ROLE_CLAIM_DEFAULT); this.isRoleClaimNotSubject = !ROLE_CLAIM_DEFAULT.equals(roleClaim); @@ -163,7 +174,9 @@ public void initialize(ServiceConfiguration config) throws IOException { int readTimeout = getConfigValueAsInt(config, HTTP_READ_TIMEOUT_MILLIS, HTTP_READ_TIMEOUT_MILLIS_DEFAULT); String trustCertsFilePath = getConfigValueAsString(config, ISSUER_TRUST_CERTS_FILE_PATH, null); SslContext sslContext = null; - if (trustCertsFilePath != null) { + // When config is in the conf file but is empty, it defaults to the empty string, which is not meaningful and + // should be ignored. + if (StringUtils.isNotBlank(trustCertsFilePath)) { // Use default settings for everything but the trust store. sslContext = SslContextBuilder.forClient() .trustManager(new File(trustCertsFilePath)) @@ -175,10 +188,9 @@ public void initialize(ServiceConfiguration config) throws IOException { .setSslContext(sslContext) .build(); httpClient = new DefaultAsyncHttpClient(clientConfig); - ApiClient k8sApiClient = - fallbackDiscoveryMode != FallbackDiscoveryMode.DISABLED ? Config.defaultClient() : null; - this.openIDProviderMetadataCache = new OpenIDProviderMetadataCache(config, httpClient, k8sApiClient); - this.jwksCache = new JwksCache(config, httpClient, k8sApiClient); + k8sApiClient = fallbackDiscoveryMode != FallbackDiscoveryMode.DISABLED ? Config.defaultClient() : null; + this.openIDProviderMetadataCache = new OpenIDProviderMetadataCache(this, config, httpClient, k8sApiClient); + this.jwksCache = new JwksCache(this, config, httpClient, k8sApiClient); } @Override @@ -186,6 +198,11 @@ public String getAuthMethodName() { return AUTH_METHOD_NAME; } + @Override + public void incrementFailureMetric(Enum errorCode) { + authenticationMetrics.recordFailure(errorCode); + } + /** * Authenticate the parameterized {@link AuthenticationDataSource} by verifying the issuer is an allowed issuer, * then retrieving the JWKS URI from the issuer, then retrieving the Public key from the JWKS URI, and finally @@ -215,7 +232,7 @@ CompletableFuture authenticateTokenAsync(AuthenticationDataSource au return authenticateToken(token) .whenComplete((jwt, e) -> { if (jwt != null) { - AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName()); + authenticationMetrics.recordSuccess(); } // Failure metrics are incremented within methods above }); @@ -300,7 +317,8 @@ private CompletableFuture authenticateToken(String token) { return verifyIssuerAndGetJwk(jwt) .thenCompose(jwk -> { try { - if (!jwt.getAlgorithm().equals(jwk.getAlgorithm())) { + // verify the algorithm, if it is set ("alg" is optional in the JWK spec) + if (jwk.getAlgorithm() != null && !jwt.getAlgorithm().equals(jwk.getAlgorithm())) { incrementFailureMetric(AuthenticationExceptionCode.ALGORITHM_MISMATCH); return CompletableFuture.failedFuture( new AuthenticationException("JWK's alg [" + jwk.getAlgorithm() @@ -358,7 +376,17 @@ public AuthenticationState newAuthState(AuthData authData, SocketAddress remoteA @Override public void close() throws IOException { - httpClient.close(); + if (httpClient != null) { + httpClient.close(); + } + if (k8sApiClient != null) { + OkHttpClient okHttpClient = k8sApiClient.getHttpClient(); + okHttpClient.dispatcher().executorService().shutdown(); + okHttpClient.connectionPool().evictAll(); + if (okHttpClient.cache() != null) { + okHttpClient.cache().close(); + } + } } /** @@ -448,10 +476,6 @@ DecodedJWT verifyJWT(PublicKey publicKey, } } - static void incrementFailureMetric(AuthenticationExceptionCode code) { - AuthenticationMetrics.authenticateFailure(SIMPLE_NAME, AUTH_METHOD_NAME, code); - } - /** * Validate the configured allow list of allowedIssuers. The allowedIssuers set must be nonempty in order for * the plugin to authenticate any token. Thus, it fails initialization if the configuration is diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java index 73934e9c1e05e..c88661c39c6c2 100644 --- a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.authentication.oidc; +import static org.apache.pulsar.broker.authentication.oidc.AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY; import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_EXPIRATION_SECONDS; import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_EXPIRATION_SECONDS_DEFAULT; import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_REFRESH_AFTER_WRITE_SECONDS; @@ -26,7 +27,6 @@ import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_SIZE_DEFAULT; import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.KEY_ID_CACHE_MISS_REFRESH_SECONDS; import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.KEY_ID_CACHE_MISS_REFRESH_SECONDS_DEFAULT; -import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.incrementFailureMetric; import static org.apache.pulsar.broker.authentication.oidc.ConfigUtils.getConfigValueAsInt; import com.auth0.jwk.Jwk; import com.fasterxml.jackson.databind.ObjectMapper; @@ -49,6 +49,7 @@ import java.util.concurrent.TimeUnit; import javax.naming.AuthenticationException; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.authentication.AuthenticationProvider; import org.asynchttpclient.AsyncHttpClient; public class JwksCache { @@ -60,8 +61,11 @@ public class JwksCache { private final ObjectReader reader = new ObjectMapper().readerFor(HashMap.class); private final AsyncHttpClient httpClient; private final OpenidApi openidApi; + private final AuthenticationProvider authenticationProvider; - JwksCache(ServiceConfiguration config, AsyncHttpClient httpClient, ApiClient apiClient) throws IOException { + JwksCache(AuthenticationProvider authenticationProvider, ServiceConfiguration config, + AsyncHttpClient httpClient, ApiClient apiClient) throws IOException { + this.authenticationProvider = authenticationProvider; // Store the clients this.httpClient = httpClient; this.openidApi = apiClient != null ? new OpenidApi(apiClient) : null; @@ -91,7 +95,7 @@ public class JwksCache { CompletableFuture getJwk(String jwksUri, String keyId) { if (jwksUri == null) { - incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY); + authenticationProvider.incrementFailureMetric(ERROR_RETRIEVING_PUBLIC_KEY); return CompletableFuture.failedFuture(new IllegalArgumentException("jwksUri must not be null.")); } return getJwkAndMaybeReload(Optional.of(jwksUri), keyId, false); @@ -139,10 +143,10 @@ private CompletableFuture> getJwksFromJwksUri(String jwksUri) { reader.readValue(result.getResponseBodyAsBytes()); future.complete(convertToJwks(jwksUri, jwks)); } catch (AuthenticationException e) { - incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY); + authenticationProvider.incrementFailureMetric(ERROR_RETRIEVING_PUBLIC_KEY); future.completeExceptionally(e); } catch (Exception e) { - incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY); + authenticationProvider.incrementFailureMetric(ERROR_RETRIEVING_PUBLIC_KEY); future.completeExceptionally(new AuthenticationException( "Error retrieving public key at " + jwksUri + ": " + e.getMessage())); } @@ -152,7 +156,7 @@ private CompletableFuture> getJwksFromJwksUri(String jwksUri) { CompletableFuture getJwkFromKubernetesApiServer(String keyId) { if (openidApi == null) { - incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY); + authenticationProvider.incrementFailureMetric(ERROR_RETRIEVING_PUBLIC_KEY); return CompletableFuture.failedFuture(new AuthenticationException( "Failed to retrieve public key from Kubernetes API server: Kubernetes fallback is not enabled.")); } @@ -165,7 +169,7 @@ private CompletableFuture> getJwksFromKubernetesApiServer() { openidApi.getServiceAccountIssuerOpenIDKeysetAsync(new ApiCallback() { @Override public void onFailure(ApiException e, int statusCode, Map> responseHeaders) { - incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY); + authenticationProvider.incrementFailureMetric(ERROR_RETRIEVING_PUBLIC_KEY); // We want the message and responseBody here: https://github.com/kubernetes-client/java/issues/2066. future.completeExceptionally( new AuthenticationException("Failed to retrieve public key from Kubernetes API server. " @@ -178,10 +182,10 @@ public void onSuccess(String result, int statusCode, Map> r HashMap jwks = reader.readValue(result); future.complete(convertToJwks("Kubernetes API server", jwks)); } catch (AuthenticationException e) { - incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY); + authenticationProvider.incrementFailureMetric(ERROR_RETRIEVING_PUBLIC_KEY); future.completeExceptionally(e); } catch (Exception e) { - incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY); + authenticationProvider.incrementFailureMetric(ERROR_RETRIEVING_PUBLIC_KEY); future.completeExceptionally(new AuthenticationException( "Error retrieving public key at Kubernetes API server: " + e.getMessage())); } @@ -198,7 +202,7 @@ public void onDownloadProgress(long bytesRead, long contentLength, boolean done) } }); } catch (ApiException e) { - incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY); + authenticationProvider.incrementFailureMetric(ERROR_RETRIEVING_PUBLIC_KEY); future.completeExceptionally( new AuthenticationException("Failed to retrieve public key from Kubernetes API server: " + e.getMessage())); @@ -212,7 +216,7 @@ private Jwk getJwkForKID(Optional maybeJwksUri, List jwks, String k return jwk; } } - incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY); + authenticationProvider.incrementFailureMetric(ERROR_RETRIEVING_PUBLIC_KEY); throw new IllegalArgumentException("No JWK found for Key ID " + keyId); } diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java index 111399adbd72b..cffa52b00aab9 100644 --- a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java @@ -18,13 +18,13 @@ */ package org.apache.pulsar.broker.authentication.oidc; +import static org.apache.pulsar.broker.authentication.oidc.AuthenticationExceptionCode.ERROR_RETRIEVING_PROVIDER_METADATA; import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_EXPIRATION_SECONDS; import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_EXPIRATION_SECONDS_DEFAULT; import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_REFRESH_AFTER_WRITE_SECONDS; import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_REFRESH_AFTER_WRITE_SECONDS_DEFAULT; import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_SIZE; import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_SIZE_DEFAULT; -import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.incrementFailureMetric; import static org.apache.pulsar.broker.authentication.oidc.ConfigUtils.getConfigValueAsInt; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; @@ -43,6 +43,7 @@ import javax.annotation.Nonnull; import javax.naming.AuthenticationException; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.authentication.AuthenticationProvider; import org.asynchttpclient.AsyncHttpClient; /** @@ -51,13 +52,16 @@ class OpenIDProviderMetadataCache { private final ObjectReader reader = new ObjectMapper().readerFor(OpenIDProviderMetadata.class); + private final AuthenticationProvider authenticationProvider; private final AsyncHttpClient httpClient; private final WellKnownApi wellKnownApi; private final AsyncLoadingCache, OpenIDProviderMetadata> cache; private static final String WELL_KNOWN_OPENID_CONFIG = ".well-known/openid-configuration"; private static final String SLASH_WELL_KNOWN_OPENID_CONFIG = "/" + WELL_KNOWN_OPENID_CONFIG; - OpenIDProviderMetadataCache(ServiceConfiguration config, AsyncHttpClient httpClient, ApiClient apiClient) { + OpenIDProviderMetadataCache(AuthenticationProvider authenticationProvider, ServiceConfiguration config, + AsyncHttpClient httpClient, ApiClient apiClient) { + this.authenticationProvider = authenticationProvider; int maxSize = getConfigValueAsInt(config, CACHE_SIZE, CACHE_SIZE_DEFAULT); int refreshAfterWriteSeconds = getConfigValueAsInt(config, CACHE_REFRESH_AFTER_WRITE_SECONDS, CACHE_REFRESH_AFTER_WRITE_SECONDS_DEFAULT); @@ -124,10 +128,10 @@ private CompletableFuture loadOpenIDProviderMetadataForI verifyIssuer(issuer, openIDProviderMetadata, false); future.complete(openIDProviderMetadata); } catch (AuthenticationException e) { - incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PROVIDER_METADATA); + authenticationProvider.incrementFailureMetric(ERROR_RETRIEVING_PROVIDER_METADATA); future.completeExceptionally(e); } catch (Exception e) { - incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PROVIDER_METADATA); + authenticationProvider.incrementFailureMetric(ERROR_RETRIEVING_PROVIDER_METADATA); future.completeExceptionally(new AuthenticationException( "Error retrieving OpenID Provider Metadata at " + issuer + ": " + e.getMessage())); } @@ -151,7 +155,7 @@ CompletableFuture getOpenIDProviderMetadataForKubernetes verifyIssuer(issClaim, openIDProviderMetadata, true); future.complete(openIDProviderMetadata); } catch (AuthenticationException e) { - incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PROVIDER_METADATA); + authenticationProvider.incrementFailureMetric(ERROR_RETRIEVING_PROVIDER_METADATA); future.completeExceptionally(e); } return future; @@ -164,7 +168,7 @@ private CompletableFuture loadOpenIDProviderMetadataForK wellKnownApi.getServiceAccountIssuerOpenIDConfigurationAsync(new ApiCallback<>() { @Override public void onFailure(ApiException e, int statusCode, Map> responseHeaders) { - incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PROVIDER_METADATA); + authenticationProvider.incrementFailureMetric(ERROR_RETRIEVING_PROVIDER_METADATA); // We want the message and responseBody here: https://github.com/kubernetes-client/java/issues/2066. future.completeExceptionally(new AuthenticationException( "Error retrieving OpenID Provider Metadata from Kubernetes API server. Message: " @@ -179,7 +183,7 @@ public void onSuccess(String result, int statusCode, Map> r OpenIDProviderMetadata openIDProviderMetadata = reader.readValue(result); future.complete(openIDProviderMetadata); } catch (Exception e) { - incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PROVIDER_METADATA); + authenticationProvider.incrementFailureMetric(ERROR_RETRIEVING_PROVIDER_METADATA); future.completeExceptionally(new AuthenticationException( "Error retrieving OpenID Provider Metadata from Kubernetes API Server: " + e.getMessage())); @@ -197,7 +201,7 @@ public void onDownloadProgress(long bytesRead, long contentLength, boolean done) } }); } catch (ApiException e) { - incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PROVIDER_METADATA); + authenticationProvider.incrementFailureMetric(ERROR_RETRIEVING_PROVIDER_METADATA); future.completeExceptionally(new AuthenticationException( "Error retrieving OpenID Provider Metadata from Kubernetes API server: " + e.getMessage())); } @@ -221,10 +225,10 @@ private void verifyIssuer(@Nonnull String issuer, OpenIDProviderMetadata metadat boolean isK8s) throws AuthenticationException { if (!issuer.equals(metadata.getIssuer())) { if (isK8s) { - incrementFailureMetric(AuthenticationExceptionCode.UNSUPPORTED_ISSUER); + authenticationProvider.incrementFailureMetric(AuthenticationExceptionCode.UNSUPPORTED_ISSUER); throw new AuthenticationException("Issuer not allowed: " + issuer); } else { - incrementFailureMetric(AuthenticationExceptionCode.ISSUER_MISMATCH); + authenticationProvider.incrementFailureMetric(AuthenticationExceptionCode.ISSUER_MISMATCH); throw new AuthenticationException(String.format("Issuer URL mismatch: [%s] should match [%s]", issuer, metadata.getIssuer())); } diff --git a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java index d2d2de1a1149d..f4663a9ee3ce6 100644 --- a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java +++ b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java @@ -30,6 +30,7 @@ import static org.testng.Assert.fail; import com.github.tomakehurst.wiremock.WireMockServer; import com.github.tomakehurst.wiremock.stubbing.Scenario; +import com.google.common.io.Resources; import io.jsonwebtoken.SignatureAlgorithm; import io.jsonwebtoken.impl.DefaultJwtBuilder; import io.jsonwebtoken.io.Decoders; @@ -69,10 +70,12 @@ public class AuthenticationProviderOpenIDIntegrationTest { AuthenticationProviderOpenID provider; PrivateKey privateKey; + String caCert = Resources.getResource("certificate-authority/jks/broker.truststore.pem").getPath(); // These are the kid values for JWKs in the /keys endpoint String validJwk = "valid"; String invalidJwk = "invalid"; + String validJwkWithoutAlg = "valid_without_alg"; // The valid issuer String issuer; @@ -89,7 +92,11 @@ void beforeClass() throws IOException { // Port matches the port supplied in the fakeKubeConfig.yaml resource, which makes the k8s integration // tests work correctly. - server = new WireMockServer(wireMockConfig().port(0)); + server = new WireMockServer(wireMockConfig().dynamicHttpsPort() + .keystorePath(Resources.getResource("certificate-authority/jks/broker.keystore.jks").getPath()) + .keystoreType("JKS") + .keyManagerPassword("111111") + .keystorePassword("111111")); server.start(); issuer = server.baseUrl(); issuerWithTrailingSlash = issuer + "/trailing-slash/"; @@ -182,10 +189,16 @@ void beforeClass() throws IOException { "kty":"RSA", "n":"invalid-key", "e":"AQAB" + }, + { + "kid":"%s", + "kty":"RSA", + "n":"%s", + "e":"%s" } ] } - """.formatted(validJwk, n, e, invalidJwk)))); + """.formatted(validJwk, n, e, invalidJwk, validJwkWithoutAlg, n, e)))); server.stubFor( get(urlEqualTo("/missing-kid/.well-known/openid-configuration")) @@ -235,7 +248,7 @@ void beforeClass() throws IOException { conf.setAuthenticationEnabled(true); conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName())); Properties props = conf.getProperties(); - props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + props.setProperty(AuthenticationProviderOpenID.ISSUER_TRUST_CERTS_FILE_PATH, caCert); props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuer + "," + issuerWithTrailingSlash + "," + issuerThatFails); @@ -247,11 +260,12 @@ void beforeClass() throws IOException { Files.write(Path.of(System.getenv("KUBECONFIG")), kubeConfig.getBytes()); provider = new AuthenticationProviderOpenID(); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); } @AfterClass - void afterClass() { + void afterClass() throws IOException { + provider.close(); server.stop(); } @@ -268,6 +282,14 @@ public void testTokenWithValidJWK() throws Exception { assertEquals(role, provider.authenticateAsync(new AuthenticationDataCommand(token)).get()); } + @Test + public void testTokenWithValidJWKWithoutAlg() throws Exception { + String role = "superuser"; + // test with a key in JWK that does not have an "alg" field. "alg" is optional in the JWK spec + String token = generateToken(validJwkWithoutAlg, issuer, role, "allowed-audience", 0L, 0L, 10000L); + assertEquals(role, provider.authenticateAsync(new AuthenticationDataCommand(token)).get()); + } + @Test public void testTokenWithTrailingSlashAndValidJWK() throws Exception { String role = "superuser"; @@ -328,14 +350,15 @@ public void testKidCacheMissWhenRefreshConfigZero() throws Exception { conf.setAuthenticationEnabled(true); conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName())); Properties props = conf.getProperties(); - props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + props.setProperty(AuthenticationProviderOpenID.ISSUER_TRUST_CERTS_FILE_PATH, caCert); // Allows us to retrieve the JWK immediately after the cache miss of the KID props.setProperty(AuthenticationProviderOpenID.KEY_ID_CACHE_MISS_REFRESH_SECONDS, "0"); props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuerWithMissingKid); + @Cleanup AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); String role = "superuser"; String token = generateToken(validJwk, issuerWithMissingKid, role, "allowed-audience", 0L, 0L, 10000L); @@ -348,14 +371,15 @@ public void testKidCacheMissWhenRefreshConfigLongerThanDelta() throws Exception conf.setAuthenticationEnabled(true); conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName())); Properties props = conf.getProperties(); - props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + props.setProperty(AuthenticationProviderOpenID.ISSUER_TRUST_CERTS_FILE_PATH, caCert); // This value is high enough that the provider will not refresh the JWK props.setProperty(AuthenticationProviderOpenID.KEY_ID_CACHE_MISS_REFRESH_SECONDS, "100"); props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuerWithMissingKid); + @Cleanup AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); String role = "superuser"; String token = generateToken(validJwk, issuerWithMissingKid, role, "allowed-audience", 0L, 0L, 10000L); @@ -375,14 +399,15 @@ public void testKubernetesApiServerAsDiscoverTrustedIssuerSuccess() throws Excep conf.setAuthenticationEnabled(true); conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName())); Properties props = conf.getProperties(); - props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + props.setProperty(AuthenticationProviderOpenID.ISSUER_TRUST_CERTS_FILE_PATH, caCert); props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); props.setProperty(AuthenticationProviderOpenID.FALLBACK_DISCOVERY_MODE, "KUBERNETES_DISCOVER_TRUSTED_ISSUER"); // Test requires that k8sIssuer is not in the allowed token issuers props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, ""); + @Cleanup AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); String role = "superuser"; // We use the normal issuer on the token because the /k8s endpoint is configured via the kube config file @@ -409,13 +434,14 @@ public void testKubernetesApiServerAsDiscoverTrustedIssuerFailsDueToMismatchedIs conf.setAuthenticationEnabled(true); conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName())); Properties props = conf.getProperties(); - props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + props.setProperty(AuthenticationProviderOpenID.ISSUER_TRUST_CERTS_FILE_PATH, caCert); props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); props.setProperty(AuthenticationProviderOpenID.FALLBACK_DISCOVERY_MODE, "KUBERNETES_DISCOVER_TRUSTED_ISSUER"); props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, ""); + @Cleanup AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); String role = "superuser"; String token = generateToken(validJwk, "http://not-the-k8s-issuer", role, "allowed-audience", 0L, 0L, 10000L); @@ -434,14 +460,15 @@ public void testKubernetesApiServerAsDiscoverPublicKeySuccess() throws Exception conf.setAuthenticationEnabled(true); conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName())); Properties props = conf.getProperties(); - props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + props.setProperty(AuthenticationProviderOpenID.ISSUER_TRUST_CERTS_FILE_PATH, caCert); props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); props.setProperty(AuthenticationProviderOpenID.FALLBACK_DISCOVERY_MODE, "KUBERNETES_DISCOVER_PUBLIC_KEYS"); // Test requires that k8sIssuer is not in the allowed token issuers props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, ""); + @Cleanup AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); String role = "superuser"; String token = generateToken(validJwk, issuer, role, "allowed-audience", 0L, 0L, 10000L); @@ -465,13 +492,14 @@ public void testKubernetesApiServerAsDiscoverPublicKeyFailsDueToMismatchedIssuer conf.setAuthenticationEnabled(true); conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName())); Properties props = conf.getProperties(); - props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + props.setProperty(AuthenticationProviderOpenID.ISSUER_TRUST_CERTS_FILE_PATH, caCert); props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); props.setProperty(AuthenticationProviderOpenID.FALLBACK_DISCOVERY_MODE, "KUBERNETES_DISCOVER_PUBLIC_KEYS"); props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, ""); + @Cleanup AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); String role = "superuser"; String token = generateToken(validJwk, "http://not-the-k8s-issuer", role, "allowed-audience", 0L, 0L, 10000L); @@ -527,13 +555,14 @@ public void testAuthenticationStateOpenIDForTokenExpiration() throws Exception { conf.setAuthenticationEnabled(true); conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName())); Properties props = conf.getProperties(); - props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + props.setProperty(AuthenticationProviderOpenID.ISSUER_TRUST_CERTS_FILE_PATH, caCert); props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuer); // Use the leeway to allow the token to pass validation and then fail expiration props.setProperty(AuthenticationProviderOpenID.ACCEPTED_TIME_LEEWAY_SECONDS, "10"); + @Cleanup AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); String role = "superuser"; String token = generateToken(validJwk, issuer, role, "allowed-audience", 0L, 0L, 0L); @@ -557,7 +586,7 @@ public void testAuthenticationProviderListStateSuccess() throws Exception { conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName(), AuthenticationProviderToken.class.getName())); Properties props = conf.getProperties(); - props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + props.setProperty(AuthenticationProviderOpenID.ISSUER_TRUST_CERTS_FILE_PATH, caCert); props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuer); @@ -597,15 +626,16 @@ public void testAuthenticationProviderListStateSuccess() throws Exception { @Test void ensureRoleClaimForNonSubClaimReturnsRole() throws Exception { + @Cleanup AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); Properties props = new Properties(); props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuer); props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); props.setProperty(AuthenticationProviderOpenID.ROLE_CLAIM, "test"); - props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + props.setProperty(AuthenticationProviderOpenID.ISSUER_TRUST_CERTS_FILE_PATH, caCert); ServiceConfiguration config = new ServiceConfiguration(); config.setProperties(props); - provider.initialize(config); + provider.initialize(AuthenticationProvider.Context.builder().config(config).build()); // Build a JWT with a custom claim HashMap claims = new HashMap(); @@ -617,15 +647,16 @@ void ensureRoleClaimForNonSubClaimReturnsRole() throws Exception { @Test void ensureRoleClaimForNonSubClaimFailsWhenClaimIsMissing() throws Exception { + @Cleanup AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); Properties props = new Properties(); props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuer); props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); props.setProperty(AuthenticationProviderOpenID.ROLE_CLAIM, "test"); - props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + props.setProperty(AuthenticationProviderOpenID.ISSUER_TRUST_CERTS_FILE_PATH, caCert); ServiceConfiguration config = new ServiceConfiguration(); config.setProperties(props); - provider.initialize(config); + provider.initialize(AuthenticationProvider.Context.builder().config(config).build()); // Build a JWT without the "test" claim, which should cause the authentication to fail String token = generateToken(validJwk, issuer, "not-my-role", "allowed-audience", 0L, diff --git a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDTest.java b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDTest.java index 74abffe9c38e8..4a12f61528aca 100644 --- a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDTest.java +++ b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDTest.java @@ -18,26 +18,31 @@ */ package org.apache.pulsar.broker.authentication.oidc; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.testng.Assert.assertNull; import com.auth0.jwt.JWT; import com.auth0.jwt.interfaces.DecodedJWT; import io.jsonwebtoken.SignatureAlgorithm; import io.jsonwebtoken.impl.DefaultJwtBuilder; import io.jsonwebtoken.security.Keys; +import java.io.IOException; import java.security.KeyPair; import java.sql.Date; import java.time.Instant; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Properties; import java.util.Set; import javax.naming.AuthenticationException; +import lombok.Cleanup; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataCommand; +import org.apache.pulsar.broker.authentication.AuthenticationProvider; import org.testng.Assert; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -56,16 +61,31 @@ public class AuthenticationProviderOpenIDTest { // https://www.rfc-editor.org/rfc/rfc7518#section-3.1 + private static final Set SUPPORTED_ALGORITHMS = Set.of( + SignatureAlgorithm.RS256, + SignatureAlgorithm.RS384, + SignatureAlgorithm.RS512, + SignatureAlgorithm.ES256, + SignatureAlgorithm.ES384, + SignatureAlgorithm.ES512 + ); + @DataProvider(name = "supportedAlgorithms") public static Object[][] supportedAlgorithms() { - return new Object[][] { - { SignatureAlgorithm.RS256 }, - { SignatureAlgorithm.RS384 }, - { SignatureAlgorithm.RS512 }, - { SignatureAlgorithm.ES256 }, - { SignatureAlgorithm.ES384 }, - { SignatureAlgorithm.ES512 } - }; + return buildDataProvider(SUPPORTED_ALGORITHMS); + } + + @DataProvider(name = "unsupportedAlgorithms") + public static Object[][] unsupportedAlgorithms() { + var unsupportedAlgorithms = Set.of(SignatureAlgorithm.values()) + .stream() + .filter(alg -> !SUPPORTED_ALGORITHMS.contains(alg)) + .toList(); + return buildDataProvider(unsupportedAlgorithms); + } + + private static Object[][] buildDataProvider(Collection collection) { + return collection.stream().map(o -> new Object[] { o }).toArray(Object[][]::new); } // Provider to use in common tests that are not verifying the configuration of the provider itself. @@ -80,11 +100,17 @@ public void setup() throws Exception { ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); basicProvider = new AuthenticationProviderOpenID(); - basicProvider.initialize(conf); + basicProvider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); + } + + @AfterClass + public void cleanup() throws IOException { + basicProvider.close(); } @Test - public void testNullToken() { + public void testNullToken() throws IOException { + @Cleanup AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); Assert.assertThrows(AuthenticationException.class, () -> provider.authenticate(new AuthenticationDataCommand(null))); @@ -92,22 +118,18 @@ public void testNullToken() { @Test public void testThatNullAlgFails() { - AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); - Assert.assertThrows(AuthenticationException.class, - () -> provider.verifyJWT(null, null, null)); + assertThatThrownBy(() -> basicProvider.verifyJWT(null, null, null)) + .isInstanceOf(AuthenticationException.class) + .hasMessage("PublicKey algorithm cannot be null"); } - @Test - public void testThatUnsupportedAlgsThrowExceptions() { - Set unsupportedAlgs = new HashSet<>(Set.of(SignatureAlgorithm.values())); - Arrays.stream(supportedAlgorithms()).map(o -> (SignatureAlgorithm) o[0]).toList() - .forEach(unsupportedAlgs::remove); - unsupportedAlgs.forEach(unsupportedAlg -> { - AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); - // We don't create a public key because it's irrelevant - Assert.assertThrows(AuthenticationException.class, - () -> provider.verifyJWT(null, unsupportedAlg.getValue(), null)); - }); + @Test(dataProvider = "unsupportedAlgorithms") + public void testThatUnsupportedAlgsThrowExceptions(SignatureAlgorithm unsupportedAlg) { + var algorithm = unsupportedAlg.getValue(); + // We don't create a public key because it's irrelevant + assertThatThrownBy(() -> basicProvider.verifyJWT(null, algorithm, null)) + .isInstanceOf(AuthenticationException.class) + .hasMessage("Unsupported algorithm: " + algorithm); } @Test(dataProvider = "supportedAlgorithms") @@ -124,29 +146,29 @@ public void testThatSupportedAlgsWork(SignatureAlgorithm alg) throws Authenticat } @Test - public void testThatSupportedAlgWithMismatchedPublicKeyFromDifferentAlgFamilyFails() { + public void testThatSupportedAlgWithMismatchedPublicKeyFromDifferentAlgFamilyFails() throws IOException { KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256); - AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); addValidMandatoryClaims(defaultJwtBuilder, basicProviderAudience); defaultJwtBuilder.signWith(keyPair.getPrivate()); DecodedJWT jwt = JWT.decode(defaultJwtBuilder.compact()); // Choose a different algorithm from a different alg family - Assert.assertThrows(AuthenticationException.class, - () -> provider.verifyJWT(keyPair.getPublic(), SignatureAlgorithm.ES512.getValue(), jwt)); + assertThatThrownBy(() -> basicProvider.verifyJWT(keyPair.getPublic(), SignatureAlgorithm.ES512.getValue(), jwt)) + .isInstanceOf(AuthenticationException.class) + .hasMessage("Expected PublicKey alg [ES512] does match actual alg."); } @Test public void testThatSupportedAlgWithMismatchedPublicKeyFromSameAlgFamilyFails() { KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256); - AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); addValidMandatoryClaims(defaultJwtBuilder, basicProviderAudience); defaultJwtBuilder.signWith(keyPair.getPrivate()); DecodedJWT jwt = JWT.decode(defaultJwtBuilder.compact()); // Choose a different algorithm but within the same alg family as above - Assert.assertThrows(AuthenticationException.class, - () -> provider.verifyJWT(keyPair.getPublic(), SignatureAlgorithm.RS512.getValue(), jwt)); + assertThatThrownBy(() -> basicProvider.verifyJWT(keyPair.getPublic(), SignatureAlgorithm.RS512.getValue(), jwt)) + .isInstanceOf(AuthenticationException.class) + .hasMessageStartingWith("JWT algorithm does not match Public Key algorithm"); } @Test @@ -192,6 +214,7 @@ public void ensureRecentlyExpiredTokenWithinConfiguredLeewaySucceeds() throws Ex KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256); // Set up the provider + @Cleanup AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); Properties props = new Properties(); props.setProperty(AuthenticationProviderOpenID.ACCEPTED_TIME_LEEWAY_SECONDS, "10"); @@ -199,7 +222,7 @@ public void ensureRecentlyExpiredTokenWithinConfiguredLeewaySucceeds() throws Ex props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "https://localhost:8080"); ServiceConfiguration config = new ServiceConfiguration(); config.setProperties(props); - provider.initialize(config); + provider.initialize(AuthenticationProvider.Context.builder().config(config).build()); // Build the JWT with an only recently expired token DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); @@ -219,28 +242,33 @@ public void ensureRecentlyExpiredTokenWithinConfiguredLeewaySucceeds() throws Ex } @Test - public void ensureEmptyIssuersFailsInitialization() { + public void ensureEmptyIssuersFailsInitialization() throws IOException { + @Cleanup AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); Properties props = new Properties(); props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, ""); ServiceConfiguration config = new ServiceConfiguration(); config.setProperties(props); - Assert.assertThrows(IllegalArgumentException.class, () -> provider.initialize(config)); + Assert.assertThrows(IllegalArgumentException.class, + () -> provider.initialize(AuthenticationProvider.Context.builder().config(config).build())); } @Test - public void ensureEmptyIssuersFailsInitializationWithDisabledDiscoveryMode() { + public void ensureEmptyIssuersFailsInitializationWithDisabledDiscoveryMode() throws IOException { + @Cleanup AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); Properties props = new Properties(); props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, ""); props.setProperty(AuthenticationProviderOpenID.FALLBACK_DISCOVERY_MODE, "DISABLED"); ServiceConfiguration config = new ServiceConfiguration(); config.setProperties(props); - Assert.assertThrows(IllegalArgumentException.class, () -> provider.initialize(config)); + Assert.assertThrows(IllegalArgumentException.class, + () -> provider.initialize(AuthenticationProvider.Context.builder().config(config).build())); } @Test public void ensureEmptyIssuersWithK8sTrustedIssuerEnabledPassesInitialization() throws Exception { + @Cleanup AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); Properties props = new Properties(); props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "my-audience"); @@ -248,11 +276,12 @@ public void ensureEmptyIssuersWithK8sTrustedIssuerEnabledPassesInitialization() props.setProperty(AuthenticationProviderOpenID.FALLBACK_DISCOVERY_MODE, "KUBERNETES_DISCOVER_TRUSTED_ISSUER"); ServiceConfiguration config = new ServiceConfiguration(); config.setProperties(props); - provider.initialize(config); + provider.initialize(AuthenticationProvider.Context.builder().config(config).build()); } @Test public void ensureEmptyIssuersWithK8sPublicKeyEnabledPassesInitialization() throws Exception { + @Cleanup AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); Properties props = new Properties(); props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "my-audience"); @@ -260,26 +289,30 @@ public void ensureEmptyIssuersWithK8sPublicKeyEnabledPassesInitialization() thro props.setProperty(AuthenticationProviderOpenID.FALLBACK_DISCOVERY_MODE, "KUBERNETES_DISCOVER_PUBLIC_KEYS"); ServiceConfiguration config = new ServiceConfiguration(); config.setProperties(props); - provider.initialize(config); + provider.initialize(AuthenticationProvider.Context.builder().config(config).build()); } @Test - public void ensureNullIssuersFailsInitialization() { + public void ensureNullIssuersFailsInitialization() throws IOException { + @Cleanup AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); ServiceConfiguration config = new ServiceConfiguration(); // Make sure this still defaults to null. assertNull(config.getProperties().get(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS)); - Assert.assertThrows(IllegalArgumentException.class, () -> provider.initialize(config)); + Assert.assertThrows(IllegalArgumentException.class, + () -> provider.initialize(AuthenticationProvider.Context.builder().config(config).build())); } @Test - public void ensureInsecureIssuerFailsInitialization() { + public void ensureInsecureIssuerFailsInitialization() throws IOException { + @Cleanup AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); Properties props = new Properties(); props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "https://myissuer.com,http://myissuer.com"); ServiceConfiguration config = new ServiceConfiguration(); config.setProperties(props); - Assert.assertThrows(IllegalArgumentException.class, () -> provider.initialize(config)); + Assert.assertThrows(IllegalArgumentException.class, + () -> provider.initialize(AuthenticationProvider.Context.builder().config(config).build())); } @Test void ensureMissingRoleClaimReturnsNull() throws Exception { @@ -293,6 +326,7 @@ public void ensureInsecureIssuerFailsInitialization() { } @Test void ensureRoleClaimForStringReturnsRole() throws Exception { + @Cleanup AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); Properties props = new Properties(); props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "https://myissuer.com"); @@ -300,7 +334,7 @@ public void ensureInsecureIssuerFailsInitialization() { props.setProperty(AuthenticationProviderOpenID.ROLE_CLAIM, "sub"); ServiceConfiguration config = new ServiceConfiguration(); config.setProperties(props); - provider.initialize(config); + provider.initialize(AuthenticationProvider.Context.builder().config(config).build()); // Build an empty JWT DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); @@ -312,6 +346,7 @@ public void ensureInsecureIssuerFailsInitialization() { } @Test void ensureRoleClaimForSingletonListReturnsRole() throws Exception { + @Cleanup AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); Properties props = new Properties(); props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "https://myissuer.com"); @@ -319,7 +354,7 @@ public void ensureInsecureIssuerFailsInitialization() { props.setProperty(AuthenticationProviderOpenID.ROLE_CLAIM, "roles"); ServiceConfiguration config = new ServiceConfiguration(); config.setProperties(props); - provider.initialize(config); + provider.initialize(AuthenticationProvider.Context.builder().config(config).build()); // Build an empty JWT DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); @@ -333,6 +368,7 @@ public void ensureInsecureIssuerFailsInitialization() { } @Test void ensureRoleClaimForMultiEntryListReturnsFirstRole() throws Exception { + @Cleanup AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); Properties props = new Properties(); props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "https://myissuer.com"); @@ -340,7 +376,7 @@ public void ensureInsecureIssuerFailsInitialization() { props.setProperty(AuthenticationProviderOpenID.ROLE_CLAIM, "roles"); ServiceConfiguration config = new ServiceConfiguration(); config.setProperties(props); - provider.initialize(config); + provider.initialize(AuthenticationProvider.Context.builder().config(config).build()); // Build an empty JWT DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); @@ -354,6 +390,7 @@ public void ensureInsecureIssuerFailsInitialization() { } @Test void ensureRoleClaimForEmptyListReturnsNull() throws Exception { + @Cleanup AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); Properties props = new Properties(); props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, "https://myissuer.com"); @@ -361,7 +398,7 @@ public void ensureInsecureIssuerFailsInitialization() { props.setProperty(AuthenticationProviderOpenID.ROLE_CLAIM, "roles"); ServiceConfiguration config = new ServiceConfiguration(); config.setProperties(props); - provider.initialize(config); + provider.initialize(AuthenticationProvider.Context.builder().config(config).build()); // Build an empty JWT DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); diff --git a/pulsar-broker-auth-sasl/pom.xml b/pulsar-broker-auth-sasl/pom.xml index 2957159ca93e8..5c1938998a816 100644 --- a/pulsar-broker-auth-sasl/pom.xml +++ b/pulsar-broker-auth-sasl/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-broker-auth-sasl @@ -41,6 +41,12 @@ ${project.version} + + io.opentelemetry + opentelemetry-sdk-testing + test + + org.apache.kerby kerby-config diff --git a/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderSasl.java b/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderSasl.java index 93f7d3f420699..f8841193ba2d2 100644 --- a/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderSasl.java +++ b/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderSasl.java @@ -33,6 +33,8 @@ import static org.apache.pulsar.common.sasl.SaslConstants.SASL_STATE_NEGOTIATE; import static org.apache.pulsar.common.sasl.SaslConstants.SASL_STATE_SERVER; import static org.apache.pulsar.common.sasl.SaslConstants.SASL_STATE_SERVER_CHECK_TOKEN; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; import java.io.IOException; import java.net.SocketAddress; import java.net.URI; @@ -41,7 +43,7 @@ import java.util.Base64; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; import javax.naming.AuthenticationException; @@ -72,9 +74,16 @@ public class AuthenticationProviderSasl implements AuthenticationProvider { private JAASCredentialsContainer jaasCredentialsContainer; private String loginContextName; + private Cache authStates; @Override public void initialize(ServiceConfiguration config) throws IOException { + initialize(Context.builder().config(config).build()); + } + + @Override + public void initialize(Context context) throws IOException { + var config = context.getConfig(); this.configuration = new HashMap<>(); final String allowedIdsPatternRegExp = config.getSaslJaasClientAllowedIds(); configuration.put(JAAS_CLIENT_ALLOWED_IDS, allowedIdsPatternRegExp); @@ -110,6 +119,9 @@ public void initialize(ServiceConfiguration config) throws IOException { throw new IllegalArgumentException(msg); } this.signer = new SaslRoleTokenSigner(secret); + this.authStates = Caffeine.newBuilder() + .maximumSize(config.getMaxInflightSaslContext()) + .expireAfterWrite(config.getInflightSaslContextExpiryMs(), TimeUnit.MILLISECONDS).build(); } @Override @@ -119,6 +131,10 @@ public String getAuthMethodName() { @Override public void close() throws IOException { + if (jaasCredentialsContainer != null) { + jaasCredentialsContainer.close(); + jaasCredentialsContainer = null; + } } @Override @@ -198,8 +214,6 @@ private byte[] readSecretFromUrl(String secretConfUrl) throws IOException { } } - private ConcurrentHashMap authStates = new ConcurrentHashMap<>(); - // return authState if it is in cache. private AuthenticationState getAuthState(HttpServletRequest request) { String id = request.getHeader(SASL_STATE_SERVER); @@ -208,7 +222,7 @@ private AuthenticationState getAuthState(HttpServletRequest request) { } try { - return authStates.get(Long.parseLong(id)); + return authStates.getIfPresent(Long.parseLong(id)); } catch (NumberFormatException e) { log.error("[{}] Wrong Id String in Token {}. e:", request.getRequestURI(), id, e); @@ -295,7 +309,7 @@ public boolean authenticateHttpRequest(HttpServletRequest request, HttpServletRe response.setStatus(HttpServletResponse.SC_OK); // auth completed, no need to keep authState - authStates.remove(state.getStateId()); + authStates.invalidate(state.getStateId()); return false; } else { // auth not complete diff --git a/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/SaslRoleTokenSigner.java b/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/SaslRoleTokenSigner.java index c979566e485ce..82f760e14b7d6 100644 --- a/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/SaslRoleTokenSigner.java +++ b/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/SaslRoleTokenSigner.java @@ -76,7 +76,7 @@ public String verifyAndExtract(String signedStr) throws AuthenticationException String originalSignature = signedStr.substring(index + SIGNATURE.length()); String rawValue = signedStr.substring(0, index); String currentSignature = computeSignature(rawValue); - if (!originalSignature.equals(currentSignature)) { + if (!MessageDigest.isEqual(originalSignature.getBytes(), currentSignature.getBytes())){ throw new AuthenticationException("Invalid signature"); } return rawValue; diff --git a/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/ProxySaslAuthenticationTest.java b/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/ProxySaslAuthenticationTest.java index 261efe680f862..ca28befabc145 100644 --- a/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/ProxySaslAuthenticationTest.java +++ b/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/ProxySaslAuthenticationTest.java @@ -49,6 +49,7 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.auth.AuthenticationSasl; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; +import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.proxy.server.ProxyConfiguration; import org.apache.pulsar.proxy.server.ProxyService; import org.slf4j.Logger; @@ -193,16 +194,19 @@ protected void setup() throws Exception { conf.setAuthenticationProviders(providers); conf.setClusterName("test"); conf.setSuperUserRoles(ImmutableSet.of("client/" + localHostname + "@" + kdc.getRealm())); - - super.init(); - - lookupUrl = new URI(pulsar.getBrokerServiceUrl()); - // set admin auth, to verify admin web resources Map clientSaslConfig = new HashMap<>(); clientSaslConfig.put("saslJaasClientSectionName", "PulsarClient"); clientSaslConfig.put("serverType", "broker"); + conf.setBrokerClientAuthenticationPlugin(AuthenticationSasl.class.getName()); + conf.setBrokerClientAuthenticationParameters(ObjectMapperFactory + .getMapper().getObjectMapper().writeValueAsString(clientSaslConfig)); + + super.init(); + + lookupUrl = new URI(pulsar.getBrokerServiceUrl()); log.info("set client jaas section name: PulsarClient"); + closeAdmin(); admin = PulsarAdmin.builder() .serviceHttpUrl(brokerUrl.toString()) .authentication(AuthenticationFactory.create(AuthenticationSasl.class.getName(), clientSaslConfig)) @@ -238,6 +242,7 @@ void testAuthentication() throws Exception { proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl()); proxyConfig.setSaslJaasClientAllowedIds(".*" + localHostname + ".*"); proxyConfig.setSaslJaasServerSectionName("PulsarProxy"); + proxyConfig.setClusterName(configClusterName); // proxy connect to broker proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationSasl.class.getName()); @@ -255,7 +260,11 @@ void testAuthentication() throws Exception { proxyConfig.setForwardAuthorizationCredentials(true); AuthenticationService authenticationService = new AuthenticationService( PulsarConfigurationLoader.convertFrom(proxyConfig)); - ProxyService proxyService = new ProxyService(proxyConfig, authenticationService); + @Cleanup + final Authentication proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); + ProxyService proxyService = new ProxyService(proxyConfig, authenticationService, proxyClientAuthentication); proxyService.start(); final String proxyServiceUrl = "pulsar://localhost:" + proxyService.getListenPort().get(); diff --git a/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java b/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java index 8a0d0392d1333..226ec15d33afe 100644 --- a/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java +++ b/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java @@ -18,26 +18,33 @@ */ package org.apache.pulsar.broker.authentication; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - +import com.github.benmanes.caffeine.cache.Cache; +import com.google.common.collect.ImmutableSet; import java.io.File; import java.io.FileWriter; +import java.io.IOException; +import java.lang.reflect.Field; import java.net.URI; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.Base64; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; - import javax.security.auth.login.Configuration; - -import com.google.common.collect.ImmutableSet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -53,12 +60,14 @@ import org.apache.pulsar.client.impl.auth.AuthenticationSasl; import org.apache.pulsar.common.api.AuthData; import org.apache.pulsar.common.sasl.SaslConstants; +import org.apache.pulsar.common.util.ObjectMapperFactory; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import org.testng.collections.CollectionUtils; @Slf4j public class SaslAuthenticateTest extends ProducerConsumerBase { @@ -70,7 +79,7 @@ public class SaslAuthenticateTest extends ProducerConsumerBase { private static Properties properties; private static String localHostname = "localhost"; - private static Authentication authSasl; + private Authentication authSasl; @BeforeClass public static void startMiniKdc() throws Exception { @@ -139,17 +148,11 @@ public static void startMiniKdc() throws Exception { System.setProperty("java.security.krb5.conf", krb5file.getAbsolutePath()); Configuration.getConfiguration().refresh(); - // Client config - Map clientSaslConfig = new HashMap<>(); - clientSaslConfig.put("saslJaasClientSectionName", "PulsarClient"); - clientSaslConfig.put("serverType", "broker"); - log.info("set client jaas section name: PulsarClient"); - authSasl = AuthenticationFactory.create(AuthenticationSasl.class.getName(), clientSaslConfig); - log.info("created AuthenticationSasl"); + } @AfterClass(alwaysRun = true) - public static void stopMiniKdc() { + public static void stopMiniKdc() throws IOException { System.clearProperty("java.security.auth.login.config"); System.clearProperty("java.security.krb5.conf"); if (kdc != null) { @@ -168,6 +171,14 @@ protected void setup() throws Exception { // use http lookup to verify HttpClient works well. isTcpLookup = false; + // Client config + Map clientSaslConfig = new HashMap<>(); + clientSaslConfig.put("saslJaasClientSectionName", "PulsarClient"); + clientSaslConfig.put("serverType", "broker"); + log.info("set client jaas section name: PulsarClient"); + authSasl = AuthenticationFactory.create(AuthenticationSasl.class.getName(), clientSaslConfig); + log.info("created AuthenticationSasl"); + conf.setAdvertisedAddress(localHostname); conf.setAuthenticationEnabled(true); conf.setSaslJaasClientAllowedIds(".*" + "client" + ".*"); @@ -180,7 +191,9 @@ protected void setup() throws Exception { conf.setAuthenticationProviders(providers); conf.setClusterName("test"); conf.setSuperUserRoles(ImmutableSet.of("client" + "@" + kdc.getRealm())); - + conf.setBrokerClientAuthenticationPlugin(AuthenticationSasl.class.getName()); + conf.setBrokerClientAuthenticationParameters(ObjectMapperFactory + .getMapper().getObjectMapper().writeValueAsString(clientSaslConfig)); super.init(); lookupUrl = new URI(pulsar.getWebServiceAddress()); @@ -191,10 +204,8 @@ protected void setup() throws Exception { .authentication(authSasl)); // set admin auth, to verify admin web resources - Map clientSaslConfig = new HashMap<>(); - clientSaslConfig.put("saslJaasClientSectionName", "PulsarClient"); - clientSaslConfig.put("serverType", "broker"); log.info("set client jaas section name: PulsarClient"); + closeAdmin(); admin = PulsarAdmin.builder() .serviceHttpUrl(brokerUrl.toString()) .authentication(AuthenticationFactory.create(AuthenticationSasl.class.getName(), clientSaslConfig)) @@ -295,4 +306,74 @@ public void testSaslServerAndClientAuth() throws Exception { log.info("-- {} -- end", methodName); } + @Test + public void testSaslOnlyAuthFirstStage() throws Exception { + @Cleanup + AuthenticationProviderSasl saslServer = new AuthenticationProviderSasl(); + // The cache expiration time is set to 50ms. Residual auth info should be cleaned up + conf.setInflightSaslContextExpiryMs(50); + saslServer.initialize(AuthenticationProvider.Context.builder().config(conf).build()); + + HttpServletRequest servletRequest = mock(HttpServletRequest.class); + doReturn("Init").when(servletRequest).getHeader("State"); + // 10 clients only do one-stage verification, resulting in 10 auth info remaining in memory + for (int i = 0; i < 10; i++) { + AuthenticationDataProvider dataProvider = authSasl.getAuthData("localhost"); + AuthData initData1 = dataProvider.authenticate(AuthData.INIT_AUTH_DATA); + doReturn(Base64.getEncoder().encodeToString(initData1.getBytes())).when( + servletRequest).getHeader("SASL-Token"); + doReturn(String.valueOf(i)).when(servletRequest).getHeader("SASL-Server-ID"); + saslServer.authenticateHttpRequest(servletRequest, mock(HttpServletResponse.class)); + } + Field field = AuthenticationProviderSasl.class.getDeclaredField("authStates"); + field.setAccessible(true); + Cache cache = (Cache) field.get(saslServer); + assertEquals(cache.asMap().size(), 10); + // Add more auth info into memory + for (int i = 0; i < 10; i++) { + AuthenticationDataProvider dataProvider = authSasl.getAuthData("localhost"); + AuthData initData1 = dataProvider.authenticate(AuthData.INIT_AUTH_DATA); + doReturn(Base64.getEncoder().encodeToString(initData1.getBytes())).when( + servletRequest).getHeader("SASL-Token"); + doReturn(String.valueOf(10 + i)).when(servletRequest).getHeader("SASL-Server-ID"); + saslServer.authenticateHttpRequest(servletRequest, mock(HttpServletResponse.class)); + } + long start = System.currentTimeMillis(); + while (true) { + if (System.currentTimeMillis() - start > 1000) { + fail(); + } + cache = (Cache) field.get(saslServer); + // Residual auth info should be cleaned up + if (CollectionUtils.hasElements(cache.asMap())) { + break; + } + Thread.sleep(5); + } + } + + @Test + public void testMaxInflightContext() throws Exception { + @Cleanup + AuthenticationProviderSasl saslServer = new AuthenticationProviderSasl(); + HttpServletRequest servletRequest = mock(HttpServletRequest.class); + doReturn("Init").when(servletRequest).getHeader("State"); + conf.setInflightSaslContextExpiryMs(Integer.MAX_VALUE); + conf.setMaxInflightSaslContext(1); + saslServer.initialize(AuthenticationProvider.Context.builder().config(conf).build()); + // add 10 inflight sasl context + for (int i = 0; i < 10; i++) { + AuthenticationDataProvider dataProvider = authSasl.getAuthData("localhost"); + AuthData initData1 = dataProvider.authenticate(AuthData.INIT_AUTH_DATA); + doReturn(Base64.getEncoder().encodeToString(initData1.getBytes())).when( + servletRequest).getHeader("SASL-Token"); + doReturn(String.valueOf(i)).when(servletRequest).getHeader("SASL-Server-ID"); + saslServer.authenticateHttpRequest(servletRequest, mock(HttpServletResponse.class)); + } + Field field = AuthenticationProviderSasl.class.getDeclaredField("authStates"); + field.setAccessible(true); + Cache cache = (Cache) field.get(saslServer); + //only 1 context was left in the memory + assertEquals(cache.asMap().size(), 1); + } } diff --git a/pulsar-broker-common/pom.xml b/pulsar-broker-common/pom.xml index b81f2b7621ba7..b04d08c6c8f19 100644 --- a/pulsar-broker-common/pom.xml +++ b/pulsar-broker-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-broker-common @@ -49,6 +49,11 @@ simpleclient_jetty + + io.opentelemetry + opentelemetry-api + + javax.servlet javax.servlet-api @@ -82,10 +87,28 @@ awaitility test + + + io.rest-assured + rest-assured + test + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + org.gaul modernizer-maven-plugin diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java index e9e350800b44e..4a5ff746f4039 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java @@ -70,7 +70,7 @@ public class BookieRackAffinityMapping extends AbstractDNSToSwitchMapping private BookiesRackConfiguration racksWithHost = new BookiesRackConfiguration(); private Map bookieInfoMap = new HashMap<>(); - public static MetadataStore createMetadataStore(Configuration conf) throws MetadataException { + static MetadataStore getMetadataStore(Configuration conf) throws MetadataException { MetadataStore store; Object storeProperty = conf.getProperty(METADATA_STORE_INSTANCE); if (storeProperty != null) { @@ -116,14 +116,20 @@ public synchronized void setConf(Configuration conf) { super.setConf(conf); MetadataStore store; try { - store = createMetadataStore(conf); - bookieMappingCache = store.getMetadataCache(BookiesRackConfiguration.class); - store.registerListener(this::handleUpdates); - racksWithHost = bookieMappingCache.get(BOOKIE_INFO_ROOT_PATH).get() - .orElseGet(BookiesRackConfiguration::new); - updateRacksWithHost(racksWithHost); - watchAvailableBookies(); - for (Map bookieMapping : racksWithHost.values()) { + store = getMetadataStore(conf); + } catch (MetadataException e) { + throw new RuntimeException(METADATA_STORE_INSTANCE + " failed to init BookieId list"); + } + + bookieMappingCache = store.getMetadataCache(BookiesRackConfiguration.class); + store.registerListener(this::handleUpdates); + + try { + var racksWithHost = bookieMappingCache.get(BOOKIE_INFO_ROOT_PATH) + .thenApply(optRes -> optRes.orElseGet(BookiesRackConfiguration::new)) + .get(); + + for (var bookieMapping : racksWithHost.values()) { for (String address : bookieMapping.keySet()) { bookieAddressListLastTime.add(BookieId.parse(address)); } @@ -132,9 +138,13 @@ public synchronized void setConf(Configuration conf) { bookieAddressListLastTime); } } - } catch (InterruptedException | ExecutionException | MetadataException e) { - throw new RuntimeException(METADATA_STORE_INSTANCE + " failed to init BookieId list"); + updateRacksWithHost(racksWithHost); + } catch (ExecutionException | InterruptedException e) { + LOG.error("Failed to update rack info. ", e); + throw new RuntimeException(e); } + + watchAvailableBookies(); } private void watchAvailableBookies() { @@ -145,13 +155,13 @@ private void watchAvailableBookies() { field.setAccessible(true); RegistrationClient registrationClient = (RegistrationClient) field.get(bookieAddressResolver); registrationClient.watchWritableBookies(versioned -> { - try { - racksWithHost = bookieMappingCache.get(BOOKIE_INFO_ROOT_PATH).get() - .orElseGet(BookiesRackConfiguration::new); - updateRacksWithHost(racksWithHost); - } catch (InterruptedException | ExecutionException e) { - LOG.error("Failed to update rack info. ", e); - } + bookieMappingCache.get(BOOKIE_INFO_ROOT_PATH) + .thenApply(optRes -> optRes.orElseGet(BookiesRackConfiguration::new)) + .thenAccept(this::updateRacksWithHost) + .exceptionally(ex -> { + LOG.error("Failed to update rack info. ", ex); + return null; + }); }); } catch (NoSuchFieldException | IllegalAccessException e) { LOG.error("Failed watch available bookies.", e); @@ -245,6 +255,7 @@ private void handleUpdates(Notification n) { bookieMappingCache.get(BOOKIE_INFO_ROOT_PATH) .thenAccept(optVal -> { + Set bookieIdSet = new HashSet<>(); synchronized (this) { LOG.info("Bookie rack info updated to {}. Notifying rackaware policy.", optVal); this.updateRacksWithHost(optVal.orElseGet(BookiesRackConfiguration::new)); @@ -259,12 +270,12 @@ private void handleUpdates(Notification n) { LOG.debug("Bookies with rack update from {} to {}", bookieAddressListLastTime, bookieAddressList); } - Set bookieIdSet = new HashSet<>(bookieAddressList); + bookieIdSet.addAll(bookieAddressList); bookieIdSet.addAll(bookieAddressListLastTime); bookieAddressListLastTime = bookieAddressList; - if (rackawarePolicy != null) { - rackawarePolicy.onBookieRackChange(new ArrayList<>(bookieIdSet)); - } + } + if (rackawarePolicy != null) { + rackawarePolicy.onBookieRackChange(new ArrayList<>(bookieIdSet)); } }); } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicy.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicy.java index 2594798485a20..62b7ffa1e29da 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicy.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicy.java @@ -19,15 +19,15 @@ package org.apache.pulsar.bookie.rackawareness; import static org.apache.pulsar.bookie.rackawareness.BookieRackAffinityMapping.METADATA_STORE_INSTANCE; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Sets; import io.netty.util.HashedWheelTimer; -import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.BKException.BKNotEnoughBookiesException; import org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy; @@ -61,6 +61,7 @@ public class IsolatedBookieEnsemblePlacementPolicy extends RackawareEnsemblePlac private static final String PULSAR_SYSTEM_TOPIC_ISOLATION_GROUP = "*"; + private volatile BookiesRackConfiguration cachedRackConfiguration = null; public IsolatedBookieEnsemblePlacementPolicy() { super(); @@ -72,7 +73,7 @@ public RackawareEnsemblePlacementPolicyImpl initialize(ClientConfiguration conf, StatsLogger statsLogger, BookieAddressResolver bookieAddressResolver) { MetadataStore store; try { - store = BookieRackAffinityMapping.createMetadataStore(conf); + store = BookieRackAffinityMapping.getMetadataStore(conf); } catch (MetadataException e) { throw new RuntimeException(METADATA_STORE_INSTANCE + " failed initialized"); } @@ -86,7 +87,12 @@ public RackawareEnsemblePlacementPolicyImpl initialize(ClientConfiguration conf, } // Only add the bookieMappingCache if we have defined an isolation group bookieMappingCache = store.getMetadataCache(BookiesRackConfiguration.class); - bookieMappingCache.get(BookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH).join(); + bookieMappingCache.get(BookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH).thenAccept(opt -> opt.ifPresent( + bookiesRackConfiguration -> cachedRackConfiguration = bookiesRackConfiguration)) + .exceptionally(e -> { + log.warn("Failed to load bookies rack configuration while initialize the PlacementPolicy."); + return null; + }); } if (conf.getProperty(SECONDARY_ISOLATION_BOOKIE_GROUPS) != null) { String secondaryIsolationGroupsString = ConfigurationStringUtil @@ -166,12 +172,12 @@ private static Pair, Set> getIsolationGroup( String secondaryIsolationGroupString = ConfigurationStringUtil .castToString(properties.getOrDefault(SECONDARY_ISOLATION_BOOKIE_GROUPS, "")); if (!primaryIsolationGroupString.isEmpty()) { - pair.setLeft(new HashSet<>(Arrays.asList(primaryIsolationGroupString.split(",")))); + pair.setLeft(Sets.newHashSet(primaryIsolationGroupString.split(","))); } else { pair.setLeft(Collections.emptySet()); } if (!secondaryIsolationGroupString.isEmpty()) { - pair.setRight(new HashSet<>(Arrays.asList(secondaryIsolationGroupString.split(",")))); + pair.setRight(Sets.newHashSet(secondaryIsolationGroupString.split(","))); } else { pair.setRight(Collections.emptySet()); } @@ -179,26 +185,27 @@ private static Pair, Set> getIsolationGroup( return pair; } - private Set getExcludedBookiesWithIsolationGroups(int ensembleSize, + @VisibleForTesting + Set getExcludedBookiesWithIsolationGroups(int ensembleSize, Pair, Set> isolationGroups) { Set excludedBookies = new HashSet<>(); - if (isolationGroups != null && isolationGroups.getLeft().contains(PULSAR_SYSTEM_TOPIC_ISOLATION_GROUP)) { + if (isolationGroups != null && isolationGroups.getLeft().contains(PULSAR_SYSTEM_TOPIC_ISOLATION_GROUP)) { return excludedBookies; } try { if (bookieMappingCache != null) { - CompletableFuture> future = - bookieMappingCache.get(BookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH); + bookieMappingCache.get(BookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH) + .thenAccept(opt -> cachedRackConfiguration = opt.orElse(null)).exceptionally(e -> { + log.warn("Failed to update the newest bookies rack config."); + return null; + }); - Optional optRes = (future.isDone() && !future.isCompletedExceptionally()) - ? future.join() : Optional.empty(); - - if (optRes.isEmpty()) { + BookiesRackConfiguration allGroupsBookieMapping = cachedRackConfiguration; + if (allGroupsBookieMapping == null) { + log.debug("The bookies rack config is not available at now."); return excludedBookies; } - - BookiesRackConfiguration allGroupsBookieMapping = optRes.get(); - Set allBookies = allGroupsBookieMapping.keySet(); + Set allGroups = allGroupsBookieMapping.keySet(); int totalAvailableBookiesInPrimaryGroup = 0; Set primaryIsolationGroup = Collections.emptySet(); Set secondaryIsolationGroup = Collections.emptySet(); @@ -207,7 +214,7 @@ private Set getExcludedBookiesWithIsolationGroups(int ensembleSize, primaryIsolationGroup = isolationGroups.getLeft(); secondaryIsolationGroup = isolationGroups.getRight(); } - for (String group : allBookies) { + for (String group : allGroups) { Set bookiesInGroup = allGroupsBookieMapping.get(group).keySet(); if (!primaryIsolationGroup.contains(group)) { for (String bookieAddress : bookiesInGroup) { diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/PulsarServerException.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/PulsarServerException.java index 2235b9a7128b8..d7c0d0adb3afc 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/PulsarServerException.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/PulsarServerException.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker; import java.io.IOException; +import java.util.concurrent.CompletionException; public class PulsarServerException extends IOException { private static final long serialVersionUID = 1; @@ -44,4 +45,20 @@ public NotFoundException(Throwable t) { super(t); } } + + public static PulsarServerException from(Throwable throwable) { + if (throwable instanceof CompletionException) { + return from(throwable.getCause()); + } + if (throwable instanceof PulsarServerException pulsarServerException) { + return pulsarServerException; + } else { + return new PulsarServerException(throwable); + } + } + + // Wrap this checked exception into a specific unchecked exception + public static CompletionException toUncheckedException(PulsarServerException e) { + return new CompletionException(e); + } } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 9966912bc8eae..81073b1731b24 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -21,9 +21,11 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Properties; @@ -49,6 +51,7 @@ import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.sasl.SaslConstants; +import org.apache.pulsar.common.util.DefaultPulsarSslFactory; import org.apache.pulsar.common.util.DirectMemoryUtils; import org.apache.pulsar.metadata.api.MetadataStoreFactory; import org.apache.pulsar.metadata.impl.ZKMetadataStore; @@ -250,6 +253,22 @@ public class ServiceConfiguration implements PulsarConfiguration { + " when getting topic statistics data.") private boolean haProxyProtocolEnabled; + @FieldContext(category = CATEGORY_SERVER, + doc = "Enable or disable the use of HA proxy protocol for resolving the client IP for http/https " + + "requests. Default is false.") + private boolean webServiceHaProxyProtocolEnabled = false; + + @FieldContext(category = CATEGORY_SERVER, doc = + "Trust X-Forwarded-For header for resolving the client IP for http/https requests.\n" + + "Default is false.") + private boolean webServiceTrustXForwardedFor = false; + + @FieldContext(category = CATEGORY_SERVER, doc = + "Add detailed client/remote and server/local addresses and ports to http/https request logging.\n" + + "Defaults to true when either webServiceHaProxyProtocolEnabled or webServiceTrustXForwardedFor " + + "is enabled.") + private Boolean webServiceLogDetailedAddresses; + @FieldContext( category = CATEGORY_SERVER, doc = "Number of threads to use for Netty Acceptor." @@ -297,6 +316,7 @@ public class ServiceConfiguration implements PulsarConfiguration { + "The cache executor thread pool is used for restarting global zookeeper session. " + "Default is 10" ) + @Deprecated private int numCacheExecutorThreadPoolSize = 10; @FieldContext( @@ -330,6 +350,19 @@ public class ServiceConfiguration implements PulsarConfiguration { + "(0 to disable limiting)") private int maxHttpServerConnections = 2048; + @FieldContext(category = CATEGORY_SERVER, doc = + "Gzip compression is enabled by default. Specific paths can be excluded from compression.\n" + + "There are 2 syntaxes supported, Servlet url-pattern based, and Regex based.\n" + + "If the spec starts with '^' the spec is assumed to be a regex based path spec and will match " + + "with normal Java regex rules.\n" + + "If the spec starts with '/' then spec is assumed to be a Servlet url-pattern rules path spec " + + "for either an exact match or prefix based match.\n" + + "If the spec starts with '*.' then spec is assumed to be a Servlet url-pattern rules path spec " + + "for a suffix based match.\n" + + "All other syntaxes are unsupported.\n" + + "Disable all compression with ^.* or ^.*$") + private List httpServerGzipCompressionExcludedPaths = new ArrayList<>(); + @FieldContext(category = CATEGORY_SERVER, doc = "Whether to enable the delayed delivery for messages.") private boolean delayedDeliveryEnabled = true; @@ -343,17 +376,15 @@ public class ServiceConfiguration implements PulsarConfiguration { @FieldContext(category = CATEGORY_SERVER, doc = "Control the tick time for when retrying on delayed delivery, " + "affecting the accuracy of the delivery time compared to the scheduled time. Default is 1 second. " - + "Note that this time is used to configure the HashedWheelTimer's tick time for the " - + "InMemoryDelayedDeliveryTrackerFactory.") + + "Note that this time is used to configure the HashedWheelTimer's tick time.") private long delayedDeliveryTickTimeMillis = 1000; - @FieldContext(category = CATEGORY_SERVER, doc = "When using the InMemoryDelayedDeliveryTrackerFactory (the default " - + "DelayedDeliverTrackerFactory), whether the deliverAt time is strictly followed. When false (default), " - + "messages may be sent to consumers before the deliverAt time by as much as the tickTimeMillis. This can " - + "reduce the overhead on the broker of maintaining the delayed index for a potentially very short time " - + "period. When true, messages will not be sent to consumer until the deliverAt time has passed, and they " - + "may be as late as the deliverAt time plus the tickTimeMillis for the topic plus the " - + "delayedDeliveryTickTimeMillis.") + @FieldContext(category = CATEGORY_SERVER, doc = "Whether the deliverAt time is strictly followed. " + + "When false (default), messages may be sent to consumers before the deliverAt time by as much " + + "as the tickTimeMillis. This can reduce the overhead on the broker of maintaining the delayed index " + + "for a potentially very short time period. When true, messages will not be sent to consumer until the " + + "deliverAt time has passed, and they may be as late as the deliverAt time plus the tickTimeMillis for " + + "the topic plus the delayedDeliveryTickTimeMillis.") private boolean isDelayedDeliveryDeliverAtTimeStrict = false; @FieldContext(category = CATEGORY_SERVER, doc = """ @@ -379,11 +410,18 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, private int delayedDeliveryMaxNumBuckets = -1; @FieldContext(category = CATEGORY_SERVER, doc = "Size of the lookahead window to use " - + "when detecting if all the messages in the topic have a fixed delay. " + + "when detecting if all the messages in the topic have a fixed delay for " + + "InMemoryDelayedDeliveryTracker (the default DelayedDeliverTracker). " + "Default is 50,000. Setting the lookahead window to 0 will disable the " + "logic to handle fixed delays in messages in a different way.") private long delayedDeliveryFixedDelayDetectionLookahead = 50_000; + @FieldContext(category = CATEGORY_SERVER, doc = """ + The max allowed delay for delayed delivery (in milliseconds). If the broker receives a message which \ + exceeds this max delay, then it will return an error to the producer. \ + The default value is 0 which means there is no limit on the max delivery delay.""") + private long delayedDeliveryMaxDelayInMillis = 0; + @FieldContext(category = CATEGORY_SERVER, doc = "Whether to enable the acknowledge of batch local index") private boolean acknowledgmentAtBatchIndexLevelEnabled = false; @@ -527,10 +565,16 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, @FieldContext( category = CATEGORY_SERVER, - doc = "Configuration file path for local metadata store. It's supported by RocksdbMetadataStore for now." + doc = "Configuration file path for local metadata store." ) private String metadataStoreConfigPath = null; + @FieldContext( + category = CATEGORY_SERVER, + doc = "Configuration file path for configuration metadata store." + ) + private String configurationStoreConfigPath = null; + @FieldContext( dynamic = true, category = CATEGORY_SERVER, @@ -559,6 +603,21 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, ) private boolean backlogQuotaCheckEnabled = true; + @FieldContext( + dynamic = true, + category = CATEGORY_POLICIES, + doc = "Max capacity of the topic name cache. -1 means unlimited cache; 0 means broker will clear all cache" + + " per maxSecondsToClearTopicNameCache, it does not mean broker will not cache TopicName." + ) + private int topicNameCacheMaxCapacity = 100_000; + + @FieldContext( + category = CATEGORY_POLICIES, + doc = "A Specifies the minimum number of seconds that the topic name stays in memory, to avoid clear cache" + + " frequently when there are too many topics are in use." + ) + private int maxSecondsToClearTopicNameCache = 3600 * 2; + @FieldContext( category = CATEGORY_POLICIES, doc = "Whether to enable precise time based backlog quota check. " @@ -617,6 +676,13 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, ) private int ttlDurationDefaultInSeconds = 0; + @FieldContext( + category = CATEGORY_POLICIES, + doc = "Additional system subscriptions that will be ignored by ttl check. " + + "The cursor names are comma separated. Default is empty." + ) + private Set additionalSystemCursorNames = new TreeSet<>(); + @FieldContext( category = CATEGORY_POLICIES, dynamic = true, @@ -883,6 +949,36 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, + " back and unack count reaches to `limit/2`. Using a value of 0, is disabling unackedMessage-limit" + " check and broker doesn't block dispatchers") private int maxUnackedMessagesPerBroker = 0; + + @FieldContext( + category = CATEGORY_POLICIES, + doc = "For Key_Shared subscriptions, if messages cannot be dispatched to consumers due to a slow consumer" + + " or a blocked key hash (because of ordering constraints), the broker will continue reading more" + + " messages from the backlog and attempt to dispatch them to consumers until the number of replay" + + " messages reaches the calculated threshold.\n" + + "Formula: threshold = min(keySharedLookAheadMsgInReplayThresholdPerConsumer *" + + " connected consumer count, keySharedLookAheadMsgInReplayThresholdPerSubscription)" + + ".\n" + + "Setting this value to 0 will disable the limit calculated per consumer.", + dynamic = true + ) + private int keySharedLookAheadMsgInReplayThresholdPerConsumer = 2000; + + @FieldContext( + category = CATEGORY_POLICIES, + doc = "For Key_Shared subscriptions, if messages cannot be dispatched to consumers due to a slow consumer" + + " or a blocked key hash (because of ordering constraints), the broker will continue reading more" + + " messages from the backlog and attempt to dispatch them to consumers until the number of replay" + + " messages reaches the calculated threshold.\n" + + "Formula: threshold = min(keySharedLookAheadMsgInReplayThresholdPerConsumer *" + + " connected consumer count, keySharedLookAheadMsgInReplayThresholdPerSubscription)" + + ".\n" + + "This value should be set to a value less than 2 * managedLedgerMaxUnackedRangesToPersist.\n" + + "Setting this value to 0 will disable the limit calculated per subscription.\n", + dynamic = true + ) + private int keySharedLookAheadMsgInReplayThresholdPerSubscription = 20000; + @FieldContext( category = CATEGORY_POLICIES, doc = "Once broker reaches maxUnackedMessagesPerBroker limit, it blocks subscriptions which has higher " @@ -1130,6 +1226,20 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, ) private int dispatcherReadFailureBackoffMandatoryStopTimeInMs = 0; + @FieldContext( + category = CATEGORY_POLICIES, + doc = "On Shared and KeyShared subscriptions, if all available messages in the subscription are filtered " + + "out and not dispatched to any consumer, message dispatching will be rescheduled with a backoff " + + "delay. This parameter sets the initial backoff delay in milliseconds.") + private int dispatcherRetryBackoffInitialTimeInMs = 1; + + @FieldContext( + category = CATEGORY_POLICIES, + doc = "On Shared and KeyShared subscriptions, if all available messages in the subscription are filtered " + + "out and not dispatched to any consumer, message dispatching will be rescheduled with a backoff " + + "delay. This parameter sets the maximum backoff delay in milliseconds.") + private int dispatcherRetryBackoffMaxTimeInMs = 10; + @FieldContext( dynamic = true, category = CATEGORY_SERVER, @@ -1190,10 +1300,18 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, category = CATEGORY_SERVER, doc = "Max concurrent non-persistent message can be processed per connection") private int maxConcurrentNonPersistentMessagePerConnection = 1000; + + @Deprecated @FieldContext( category = CATEGORY_SERVER, - doc = "Number of worker threads to serve non-persistent topic") - private int numWorkerThreadsForNonPersistentTopic = Runtime.getRuntime().availableProcessors(); + deprecated = true, + doc = "Number of worker threads to serve non-persistent topic.\n" + + "@deprecated - use topicOrderedExecutorThreadNum instead.") + private int numWorkerThreadsForNonPersistentTopic = -1; + @FieldContext( + category = CATEGORY_SERVER, + doc = "Number of worker threads to serve topic ordered executor") + private int topicOrderedExecutorThreadNum = Runtime.getRuntime().availableProcessors(); @FieldContext( category = CATEGORY_SERVER, @@ -1295,6 +1413,12 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, doc = "Max number of snapshot to be cached per subscription.") private int replicatedSubscriptionsSnapshotMaxCachedPerSubscription = 10; + @FieldContext( + category = CATEGORY_SERVER, + dynamic = true, + doc = "The position that replication task start at, it can be set to earliest or latest (default).") + private String replicationStartAt = "latest"; + @FieldContext( category = CATEGORY_SERVER, dynamic = true, @@ -1330,7 +1454,8 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, category = CATEGORY_SERVER, dynamic = true, doc = "The number of partitions per partitioned topic.\n" - + "If try to create or update partitioned topics by exceeded number of partitions, then fail." + + "If try to create or update partitioned topics by exceeded number of partitions, then fail.\n" + + "Use 0 or negative number to disable the check." ) private int maxNumPartitionsPerPartitionedTopic = 0; @@ -1346,12 +1471,6 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, ) private Set brokerInterceptors = new TreeSet<>(); - @FieldContext( - category = CATEGORY_SERVER, - doc = "Enable or disable the broker interceptor, which is only used for testing for now" - ) - private boolean disableBrokerInterceptors = true; - @FieldContext( category = CATEGORY_SERVER, doc = "List of interceptors for payload processing.") @@ -1435,6 +1554,14 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, + "please enable the system topic first.") private boolean topicLevelPoliciesEnabled = true; + @FieldContext( + category = CATEGORY_SERVER, + doc = "The class name of the topic policies service. The default config only takes affect when the " + + "systemTopicEnable config is true" + ) + private String topicPoliciesServiceClassName = + "org.apache.pulsar.broker.service.SystemTopicBasedTopicPoliciesService"; + @FieldContext( category = CATEGORY_SERVER, doc = "List of interceptors for entry metadata.") @@ -1445,11 +1572,10 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, doc = "Enable or disable exposing broker entry metadata to client.") private boolean exposingBrokerEntryMetadataToClientEnabled = false; + @Deprecated @FieldContext( category = CATEGORY_SERVER, - doc = "Enable namespaceIsolation policy update take effect ontime or not," - + " if set to ture, then the related namespaces will be unloaded after reset policy to make it " - + "take effect." + doc = "This config never takes effect and will be removed in the next release" ) private boolean enableNamespaceIsolationUpdateOnTime = false; @@ -1510,6 +1636,15 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, doc = "Specify whether Client certificates are required for TLS Reject.\n" + "the Connection if the Client Certificate is not trusted") private boolean tlsRequireTrustedClientCertOnConnect = false; + @FieldContext( + category = CATEGORY_TLS, + doc = "SSL Factory Plugin class to provide SSLEngine and SSLContext objects. The default " + + " class used is DefaultSslFactory.") + private String sslFactoryPlugin = DefaultPulsarSslFactory.class.getName(); + @FieldContext( + category = CATEGORY_TLS, + doc = "SSL Factory plugin configuration parameters.") + private String sslFactoryPluginParams = ""; /***** --- Authentication. --- ****/ @FieldContext( @@ -1646,6 +1781,18 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, ) private String kinitCommand = "/usr/bin/kinit"; + @FieldContext( + category = CATEGORY_SASL_AUTH, + doc = "how often the broker expires the inflight SASL context." + ) + private long inflightSaslContextExpiryMs = 30_000L; + + @FieldContext( + category = CATEGORY_SASL_AUTH, + doc = "Maximum number of inflight sasl context." + ) + private long maxInflightSaslContext = 50_000L; + /**** --- BookKeeper Client. --- ****/ @FieldContext( category = CATEGORY_STORAGE_BK, @@ -1669,6 +1816,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, category = CATEGORY_STORAGE_BK, doc = "Parameters for bookkeeper auth plugin" ) + @ToString.Exclude private String bookkeeperClientAuthenticationParameters; @FieldContext( @@ -1743,7 +1891,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, @FieldContext( category = CATEGORY_STORAGE_BK, doc = "Enable/disable reordering read sequence on reading entries") - private boolean bookkeeperClientReorderReadSequenceEnabled = false; + private boolean bookkeeperClientReorderReadSequenceEnabled = true; @FieldContext( category = CATEGORY_STORAGE_BK, required = false, @@ -1821,7 +1969,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, category = CATEGORY_STORAGE_BK, doc = "whether limit per_channel_bookie_client metrics of bookkeeper client stats" ) - private boolean bookkeeperClientLimitStatsLogging = false; + private boolean bookkeeperClientLimitStatsLogging = true; @FieldContext( category = CATEGORY_STORAGE_BK, @@ -2033,10 +2181,15 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, ) private long managedLedgerOffloadAutoTriggerSizeThresholdBytes = -1L; @FieldContext( - category = CATEGORY_STORAGE_OFFLOADING, - doc = "The threshold to triggering automatic offload to long term storage" + category = CATEGORY_STORAGE_OFFLOADING, + doc = "The threshold to triggering automatic offload to long term storage" ) private long managedLedgerOffloadThresholdInSeconds = -1L; + @FieldContext( + category = CATEGORY_STORAGE_OFFLOADING, + doc = "Trigger offload on topic load or not. Default is false" + ) + private boolean triggerOffloadOnTopicLoad = false; @FieldContext( category = CATEGORY_STORAGE_ML, doc = "Max number of entries to append to a cursor ledger" @@ -2082,6 +2235,13 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, doc = "Use Open Range-Set to cache unacked messages (it is memory efficient but it can take more cpu)" ) private boolean managedLedgerUnackedRangesOpenCacheSetEnabled = true; + @FieldContext( + dynamic = true, + category = CATEGORY_STORAGE_ML, + doc = "After enabling this feature, Pulsar will stop delivery messages to clients if the cursor metadata is" + + " too large to persist, it will help to reduce the duplicates caused by the ack state that can not be" + + " fully persistent. Default false.") + private boolean dispatcherPauseOnAckStatePersistentEnabled = false; @FieldContext( dynamic = true, category = CATEGORY_STORAGE_ML, @@ -2089,6 +2249,18 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, + " It helps when data-ledgers gets corrupted at bookkeeper and managed-cursor is stuck at that ledger." ) private boolean autoSkipNonRecoverableData = false; + @FieldContext( + dynamic = true, + category = CATEGORY_STORAGE_ML, + doc = "Skip managed ledger failure to forcefully recover managed ledger." + ) + private boolean managedLedgerForceRecovery = false; + @FieldContext( + dynamic = true, + category = CATEGORY_STORAGE_ML, + doc = "Skip schema ledger failure to forcefully recover topic successfully." + ) + private boolean schemaLedgerForceRecovery = false; @FieldContext( category = CATEGORY_STORAGE_ML, doc = "operation timeout while updating managed-ledger metadata." @@ -2136,12 +2308,25 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, + "If value is invalid or NONE, then save the ManagedLedgerInfo bytes data directly.") private String managedLedgerInfoCompressionType = "NONE"; + @FieldContext(category = CATEGORY_STORAGE_ML, + doc = "ManagedLedgerInfo compression size threshold (bytes), " + + "only compress metadata when origin size more then this value.\n" + + "0 means compression will always apply.\n") + private long managedLedgerInfoCompressionThresholdInBytes = 16 * 1024; + @FieldContext(category = CATEGORY_STORAGE_ML, doc = "ManagedCursorInfo compression type, option values (NONE, LZ4, ZLIB, ZSTD, SNAPPY). \n" + "If value is NONE, then save the ManagedCursorInfo bytes data directly.") private String managedCursorInfoCompressionType = "NONE"; + + @FieldContext(category = CATEGORY_STORAGE_ML, + doc = "ManagedCursorInfo compression size threshold (bytes), " + + "only compress metadata when origin size more then this value.\n" + + "0 means compression will always apply.\n") + private long managedCursorInfoCompressionThresholdInBytes = 16 * 1024; + @FieldContext( dynamic = true, category = CATEGORY_STORAGE_ML, @@ -2216,7 +2401,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, @FieldContext( category = CATEGORY_LOAD_BALANCER, dynamic = true, - doc = "Min delay of load report to collect, in milli-seconds" + doc = "Min delay of load report to collect, in minutes" ) private int loadBalancerReportUpdateMaxIntervalMinutes = 15; @FieldContext( @@ -2307,21 +2492,51 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, @FieldContext( dynamic = true, category = CATEGORY_LOAD_BALANCER, - doc = "In the UniformLoadShedder strategy, the minimum message that triggers unload." + doc = "The low threshold for the difference between the highest and lowest loaded brokers." + ) + private int loadBalancerAvgShedderLowThreshold = 15; + + @FieldContext( + dynamic = true, + category = CATEGORY_LOAD_BALANCER, + doc = "The high threshold for the difference between the highest and lowest loaded brokers." + ) + private int loadBalancerAvgShedderHighThreshold = 40; + + @FieldContext( + dynamic = true, + category = CATEGORY_LOAD_BALANCER, + doc = "The number of times the low threshold is triggered before the bundle is unloaded." + ) + private int loadBalancerAvgShedderHitCountLowThreshold = 8; + + @FieldContext( + dynamic = true, + category = CATEGORY_LOAD_BALANCER, + doc = "The number of times the high threshold is triggered before the bundle is unloaded." + ) + private int loadBalancerAvgShedderHitCountHighThreshold = 2; + + @FieldContext( + dynamic = true, + category = CATEGORY_LOAD_BALANCER, + doc = "In the UniformLoadShedder and AvgShedder strategy, the minimum message that triggers unload." ) private int minUnloadMessage = 1000; @FieldContext( dynamic = true, category = CATEGORY_LOAD_BALANCER, - doc = "In the UniformLoadShedder strategy, the minimum throughput that triggers unload." + doc = "In the UniformLoadShedder and AvgShedder strategy, the minimum throughput that triggers unload." ) private int minUnloadMessageThroughput = 1 * 1024 * 1024; @FieldContext( dynamic = true, category = CATEGORY_LOAD_BALANCER, - doc = "In the UniformLoadShedder strategy, the maximum unload ratio." + doc = "In the UniformLoadShedder and AvgShedder strategy, the maximum unload ratio." + + "For AvgShedder, recommend to set to 0.5, so that it will distribute the load " + + "evenly between the highest and lowest brokers." ) private double maxUnloadPercentage = 0.2; @@ -2362,17 +2577,60 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, @FieldContext( dynamic = true, category = CATEGORY_LOAD_BALANCER, - doc = "BandwithIn Resource Usage Weight" + doc = "BandwidthIn Resource Usage Weight" + ) + private double loadBalancerBandwidthInResourceWeight = 1.0; + + @FieldContext( + dynamic = true, + category = CATEGORY_LOAD_BALANCER, + doc = "BandwidthOut Resource Usage Weight" + ) + private double loadBalancerBandwidthOutResourceWeight = 1.0; + + @Deprecated + @FieldContext( + dynamic = true, + category = CATEGORY_LOAD_BALANCER, + doc = "BandwidthIn Resource Usage Weight, Deprecated: Use loadBalancerBandwidthInResourceWeight" ) private double loadBalancerBandwithInResourceWeight = 1.0; + @Deprecated @FieldContext( dynamic = true, category = CATEGORY_LOAD_BALANCER, - doc = "BandwithOut Resource Usage Weight" + doc = "BandwidthOut Resource Usage Weight, Deprecated: Use loadBalancerBandwidthOutResourceWeight" ) private double loadBalancerBandwithOutResourceWeight = 1.0; + /** + * Get the load balancer bandwidth in resource weight. + * To be compatible with the old configuration, we still support the old configuration. + * If a configuration is not the default configuration, use that configuration. + * If both the new and the old are configured different from the default value, use the new one. + * @return + */ + public double getLoadBalancerBandwidthInResourceWeight() { + if (loadBalancerBandwidthInResourceWeight != 1.0) { + return loadBalancerBandwidthInResourceWeight; + } + if (loadBalancerBandwithInResourceWeight != 1.0) { + return loadBalancerBandwithInResourceWeight; + } + return 1.0; + } + + public double getLoadBalancerBandwidthOutResourceWeight() { + if (loadBalancerBandwidthOutResourceWeight != 1.0) { + return loadBalancerBandwidthOutResourceWeight; + } + if (loadBalancerBandwithOutResourceWeight != 1.0) { + return loadBalancerBandwithOutResourceWeight; + } + return 1.0; + } + @FieldContext( dynamic = true, category = CATEGORY_LOAD_BALANCER, @@ -2387,14 +2645,15 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, doc = "Memory Resource Usage Weight. Deprecated: Memory is no longer used as a load balancing item.", deprecated = true ) - private double loadBalancerMemoryResourceWeight = 1.0; + private double loadBalancerMemoryResourceWeight = 0; @FieldContext( dynamic = true, category = CATEGORY_LOAD_BALANCER, - doc = "Direct Memory Resource Usage Weight" + doc = "Direct Memory Resource Usage Weight. Direct memory usage cannot accurately reflect the " + + "machine's load, and it is not recommended to use it to score the machine's load." ) - private double loadBalancerDirectMemoryResourceWeight = 1.0; + private double loadBalancerDirectMemoryResourceWeight = 0; @FieldContext( dynamic = true, @@ -2465,6 +2724,10 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, doc = "Name of load manager to use" ) private String loadManagerClassName = "org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl"; + + @FieldContext(category = CATEGORY_LOAD_BALANCER, doc = "Name of topic bundle assignment strategy to use") + private String topicBundleAssignmentStrategy = + "org.apache.pulsar.common.naming.ConsistentHashingTopicBundleAssigner"; @FieldContext( dynamic = true, category = CATEGORY_LOAD_BALANCER, @@ -2509,7 +2772,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, + "(100% resource usage is 1.0 load). " + "The shedder logic tries to distribute bundle load across brokers to meet this target std. " + "The smaller value will incur load balancing more frequently. " - + "(only used in load balancer extension TransferSheddeer)" + + "(only used in load balancer extension TransferShedder)" ) private double loadBalancerBrokerLoadTargetStd = 0.25; @@ -2520,7 +2783,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, + "If the unload scheduler consecutively finds bundles that meet unload conditions " + "many times bigger than this threshold, the scheduler will shed the bundles. " + "The bigger value will incur less bundle unloading/transfers. " - + "(only used in load balancer extension TransferSheddeer)" + + "(only used in load balancer extension TransferShedder)" ) private int loadBalancerSheddingConditionHitCountThreshold = 3; @@ -2532,7 +2795,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, + "-- pre-assigns the destination broker upon unloading). " + "Off: unload bundles from overloaded brokers " + "-- post-assigns the destination broker upon lookups). " - + "(only used in load balancer extension TransferSheddeer)" + + "(only used in load balancer extension TransferShedder)" ) private boolean loadBalancerTransferEnabled = true; @@ -2541,7 +2804,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, dynamic = true, doc = "Maximum number of brokers to unload bundle load for each unloading cycle. " + "The bigger value will incur more unloading/transfers for each unloading cycle. " - + "(only used in load balancer extension TransferSheddeer)" + + "(only used in load balancer extension TransferShedder)" ) private int loadBalancerMaxNumberOfBrokerSheddingPerCycle = 3; @@ -2551,7 +2814,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, doc = "Delay (in seconds) to the next unloading cycle after unloading. " + "The logic tries to give enough time for brokers to recompute load after unloading. " + "The bigger value will delay the next unloading cycle longer. " - + "(only used in load balancer extension TransferSheddeer)" + + "(only used in load balancer extension TransferShedder)" ) private long loadBalanceSheddingDelayInSeconds = 180; @@ -2562,8 +2825,9 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, + "The logic tries to avoid (possibly unavailable) brokers with out-dated load data, " + "and those brokers will be ignored in the load computation. " + "When tuning this value, please consider loadBalancerReportUpdateMaxIntervalMinutes. " - + "The current default is loadBalancerReportUpdateMaxIntervalMinutes * 2. " - + "(only used in load balancer extension TransferSheddeer)" + + "The current default value is loadBalancerReportUpdateMaxIntervalMinutes * 120, reflecting " + + "twice the duration in seconds. " + + "(only used in load balancer extension TransferShedder)" ) private long loadBalancerBrokerLoadDataTTLInSeconds = 1800; @@ -2574,7 +2838,10 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, + "The load balancer distributes bundles across brokers, " + "based on topK bundle load data and other broker load data." + "The bigger value will increase the overhead of reporting many bundles in load data. " - + "(only used in load balancer extension logics)" + + "Used for ExtensibleLoadManagerImpl and ModularLoadManagerImpl, default value is 10. " + + "User can disable the bundle filtering feature of ModularLoadManagerImpl by setting to -1." + + "Enabling this feature can reduce the pressure on the zookeeper when doing load report." + + "WARNING: too small value could result in a long load balance time." ) private int loadBalancerMaxNumberOfBundlesInBundleLoadReport = 10; @FieldContext( @@ -2628,6 +2895,54 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, ) private boolean loadBalancerSheddingBundlesWithPoliciesEnabled = false; + @FieldContext( + category = CATEGORY_LOAD_BALANCER, + doc = "Time to wait before fixing any stuck in-flight service unit states. " + + "The leader monitor fixes any in-flight service unit(bundle) states " + + "by reassigning the ownerships if stuck too long, longer than this period." + + "(only used in load balancer extension logics)" + ) + private long loadBalancerInFlightServiceUnitStateWaitingTimeInMillis = 30 * 1000; + + @FieldContext( + category = CATEGORY_LOAD_BALANCER, + doc = "Interval between service unit state monitor checks. " + + "The service unit(bundle) state channel is periodically monitored" + + " by the leader broker at this interval" + + " to fix any orphan bundle ownerships, stuck in-flight states, and other cleanup jobs." + + "`loadBalancerServiceUnitStateTombstoneDelayTimeInSeconds` * 1000 must be bigger than " + + "`loadBalancerInFlightServiceUnitStateWaitingTimeInMillis`." + + "(only used in load balancer extension logics)" + ) + private long loadBalancerServiceUnitStateMonitorIntervalInSeconds = 60; + + @FieldContext( + category = CATEGORY_LOAD_BALANCER, + doc = "Enables the multi-phase unloading of bundles. Set to true, forwards destination broker information " + + "to consumers and producers during bundle unload, allowing them to quickly reconnect to the " + + "broker without performing an additional topic lookup." + ) + private boolean loadBalancerMultiPhaseBundleUnload = true; + + @FieldContext( + dynamic = false, + category = CATEGORY_LOAD_BALANCER, + doc = "Name of ServiceUnitStateTableView implementation class to use" + ) + private String loadManagerServiceUnitStateTableViewClassName = + "org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateTableViewImpl"; + + @FieldContext( + dynamic = true, + category = CATEGORY_LOAD_BALANCER, + doc = "Specify ServiceUnitTableViewSyncer to sync service unit(bundle) states between metadata store and " + + "system topic table views during migration from one to the other. One could enable this" + + " syncer before migration and disable it after the migration finishes. " + + "It accepts `MetadataStoreToSystemTopicSyncer` or `SystemTopicToMetadataStoreSyncer` to " + + "enable it. It accepts `None` to disable it." + ) + private ServiceUnitTableViewSyncerType loadBalancerServiceUnitTableViewSyncer = ServiceUnitTableViewSyncerType.None; + /**** --- Replication. --- ****/ @FieldContext( category = CATEGORY_REPLICATION, @@ -2659,6 +2974,11 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, + "inconsistency due to missing ZooKeeper watch (disable with value 0)" ) private int replicationPolicyCheckDurationSeconds = 600; + @FieldContext( + category = CATEGORY_REPLICATION, + doc = "Whether the internal replicator will trigger topic auto-creation on the remote cluster." + ) + private boolean createTopicToRemoteClusterForReplication = true; @Deprecated @FieldContext( category = CATEGORY_REPLICATION, @@ -2711,6 +3031,13 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, @com.fasterxml.jackson.annotation.JsonIgnore private Properties properties = new Properties(); + @FieldContext( + category = CATEGORY_SERVER, + doc = "The properties whose name starts with this prefix will be uploaded to the metadata store for " + + " the topic lookup" + ) + private String lookupPropertyPrefix = "lookup."; + @FieldContext( dynamic = true, category = CATEGORY_SERVER, @@ -2734,11 +3061,17 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, @FieldContext( category = CATEGORY_SERVER, - doc = "Timeout for the compaction phase one loop, If the execution time of the compaction " - + "phase one loop exceeds this time, the compaction will not proceed." + doc = "Timeout for each read request in the compaction phase one loop, If the execution time of one " + + "single message read operation exceeds this time, the compaction will not proceed." ) private long brokerServiceCompactionPhaseOneLoopTimeInSeconds = 30; + @FieldContext( + category = CATEGORY_SERVER, + doc = "Whether retain null-key message during topic compaction." + ) + private boolean topicCompactionRetainNullKey = false; + @FieldContext( category = CATEGORY_SERVER, doc = "Interval between checks to see if cluster is migrated and marks topic migrated " @@ -2746,6 +3079,13 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, ) private int clusterMigrationCheckDurationSeconds = 0; + @FieldContext( + category = CATEGORY_SERVER, + doc = "Flag to start cluster migration for topic only after creating all topic's resources" + + " such as tenant, namespaces, subscriptions at new green cluster. (Default disabled)." + ) + private boolean clusterMigrationAutoResourceCreation = false; + @FieldContext( category = CATEGORY_SCHEMA, doc = "Enforce schema validation on following cases:\n\n" @@ -2794,6 +3134,13 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, doc = "Number of connections per Broker in Pulsar Client used in WebSocket proxy" ) private int webSocketConnectionsPerBroker = Runtime.getRuntime().availableProcessors(); + + @FieldContext( + category = CATEGORY_WEBSOCKET, + doc = "Memory limit in MBs for direct memory in Pulsar Client used in WebSocket proxy" + ) + private int webSocketPulsarClientMemoryLimitInMB = 0; + @FieldContext( category = CATEGORY_WEBSOCKET, doc = "Time in milliseconds that idle WebSocket session times out" @@ -2826,8 +3173,10 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, private boolean exposeTopicLevelMetricsInPrometheus = true; @FieldContext( category = CATEGORY_METRICS, - doc = "If true, export buffered metrics" - ) + doc = "Set to true to enable the broker to cache the metrics response; the default is false. " + + "The caching period is defined by `managedLedgerStatsPeriodSeconds`. " + + "The broker returns the same response for subsequent requests within the same period. " + + "Ensure that the scrape interval of your monitoring system matches the caching period.") private boolean metricsBufferResponse = false; @FieldContext( category = CATEGORY_METRICS, @@ -2931,6 +3280,12 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, doc = "Stats update initial delay in seconds" ) private int statsUpdateInitialDelayInSecs = 60; + @FieldContext( + category = CATEGORY_METRICS, + minValue = -1, + doc = "HealthCheck update frequency in seconds. Disable health check with value -1 (Default value -1)" + ) + private int healthCheckMetricsUpdateTimeInSeconds = -1; @FieldContext( category = CATEGORY_METRICS, doc = "If true, aggregate publisher stats of PartitionedTopicStats by producerName" @@ -3141,6 +3496,12 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, ) private int transactionPendingAckBatchedWriteMaxDelayInMillis = 1; + @FieldContext( + category = CATEGORY_SERVER, + doc = "The class name of the factory that implements the topic compaction service." + ) + private String compactionServiceFactoryClassName = "org.apache.pulsar.compaction.PulsarCompactionServiceFactory"; + /**** --- KeyStore TLS config variables. --- ****/ @FieldContext( category = CATEGORY_KEYSTORE_TLS, @@ -3209,6 +3570,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, doc = "Authentication parameters of the authentication plugin the broker is using to connect " + "to other brokers" ) + @ToString.Exclude private String brokerClientAuthenticationParameters = ""; @FieldContext( category = CATEGORY_REPLICATION, @@ -3297,6 +3659,15 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, + " used by the internal client to authenticate with Pulsar brokers" ) private Set brokerClientTlsProtocols = new TreeSet<>(); + @FieldContext( + category = CATEGORY_TLS, + doc = "SSL Factory Plugin class used by internal client to provide SSLEngine and SSLContext objects. " + + "The default class used is DefaultSslFactory.") + private String brokerClientSslFactoryPlugin = DefaultPulsarSslFactory.class.getName(); + @FieldContext( + category = CATEGORY_TLS, + doc = "SSL Factory plugin configuration parameters used by internal client.") + private String brokerClientSslFactoryPluginParams = ""; /* packages management service configurations (begin) */ @@ -3461,4 +3832,29 @@ public long getManagedLedgerCacheEvictionIntervalMs() { MIN_ML_CACHE_EVICTION_FREQUENCY)) : Math.min(MAX_ML_CACHE_EVICTION_INTERVAL_MS, managedLedgerCacheEvictionIntervalMs); } + + public int getTopicOrderedExecutorThreadNum() { + return numWorkerThreadsForNonPersistentTopic > 0 + ? numWorkerThreadsForNonPersistentTopic : topicOrderedExecutorThreadNum; + } + + public Map lookupProperties() { + final var map = new HashMap(); + properties.forEach((key, value) -> { + if (key instanceof String && value instanceof String && ((String) key).startsWith(lookupPropertyPrefix)) { + map.put((String) key, (String) value); + } + }); + return map; + } + + public boolean isLoadBalancerServiceUnitTableViewSyncerEnabled() { + return loadBalancerServiceUnitTableViewSyncer != ServiceUnitTableViewSyncerType.None; + } + + public enum ServiceUnitTableViewSyncerType { + None, + MetadataStoreToSystemTopicSyncer, + SystemTopicToMetadataStoreSyncer; + } } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataSubscription.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataSubscription.java index 69ef526012daa..9a7324a6d077a 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataSubscription.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationDataSubscription.java @@ -71,4 +71,8 @@ public boolean hasSubscription() { public String getSubscription() { return subscription; } + + public AuthenticationDataSource getAuthData() { + return authData; + } } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java index 7862a35b5e871..d0a3a487b3478 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java @@ -20,6 +20,7 @@ import static org.apache.pulsar.broker.web.AuthenticationFilter.AuthenticatedDataAttributeName; import static org.apache.pulsar.broker.web.AuthenticationFilter.AuthenticatedRoleAttributeName; +import io.opentelemetry.api.OpenTelemetry; import java.io.Closeable; import java.io.IOException; import java.net.SocketAddress; @@ -29,6 +30,8 @@ import javax.net.ssl.SSLSession; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import lombok.Builder; +import lombok.Value; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.metrics.AuthenticationMetrics; import org.apache.pulsar.common.api.AuthData; @@ -47,8 +50,30 @@ public interface AuthenticationProvider extends Closeable { * @throws IOException * if the initialization fails */ + @Deprecated(since = "3.4.0") void initialize(ServiceConfiguration config) throws IOException; + @Builder + @Value + class Context { + ServiceConfiguration config; + + @Builder.Default + OpenTelemetry openTelemetry = OpenTelemetry.noop(); + } + + /** + * Perform initialization for the authentication provider. + * + * @param context + * the authentication provider context + * @throws IOException + * if the initialization fails + */ + default void initialize(Context context) throws IOException { + initialize(context.getConfig()); + } + /** * @return the authentication method name supported by this provider */ diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java index ca5150c9bdb60..91bf56a071c42 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java @@ -46,6 +46,8 @@ public class AuthenticationProviderBasic implements AuthenticationProvider { private static final String CONF_PULSAR_PROPERTY_KEY = "basicAuthConf"; private Map users; + private AuthenticationMetrics authenticationMetrics; + private enum ErrorCode { UNKNOWN, EMPTY_AUTH_DATA, @@ -75,6 +77,14 @@ public static byte[] readData(String data) @Override public void initialize(ServiceConfiguration config) throws IOException { + initialize(Context.builder().config(config).build()); + } + + @Override + public void initialize(Context context) throws IOException { + authenticationMetrics = new AuthenticationMetrics(context.getOpenTelemetry(), + getClass().getSimpleName(), getAuthMethodName()); + var config = context.getConfig(); String data = config.getProperties().getProperty(CONF_PULSAR_PROPERTY_KEY); if (StringUtils.isEmpty(data)) { data = System.getProperty(CONF_SYSTEM_PROPERTY_KEY); @@ -106,6 +116,11 @@ public String getAuthMethodName() { return "basic"; } + @Override + public void incrementFailureMetric(Enum errorCode) { + authenticationMetrics.recordFailure(errorCode); + } + @Override public String authenticate(AuthenticationDataSource authData) throws AuthenticationException { AuthParams authParams = new AuthParams(authData); @@ -138,7 +153,7 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat incrementFailureMetric(errorCode); throw exception; } - AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName()); + authenticationMetrics.recordSuccess(); return userId; } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java index 663a6253f4460..0e5559b3c3aab 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java @@ -38,6 +38,8 @@ @Slf4j public class AuthenticationProviderList implements AuthenticationProvider { + private AuthenticationMetrics authenticationMetrics; + private interface AuthProcessor { T apply(W process) throws AuthenticationException; @@ -49,7 +51,8 @@ private enum ErrorCode { AUTH_REQUIRED, } - static T applyAuthProcessor(List processors, AuthProcessor authFunc) + private static T applyAuthProcessor(List processors, AuthenticationMetrics metrics, + AuthProcessor authFunc) throws AuthenticationException { AuthenticationException authenticationException = null; String errorCode = ErrorCode.UNKNOWN.name(); @@ -67,30 +70,29 @@ static T applyAuthProcessor(List processors, AuthProcessor authF } if (null == authenticationException) { - AuthenticationMetrics.authenticateFailure( - AuthenticationProviderList.class.getSimpleName(), + metrics.recordFailure(AuthenticationProviderList.class.getSimpleName(), "authentication-provider-list", ErrorCode.AUTH_REQUIRED); throw new AuthenticationException("Authentication required"); } else { - AuthenticationMetrics.authenticateFailure( - AuthenticationProviderList.class.getSimpleName(), + metrics.recordFailure(AuthenticationProviderList.class.getSimpleName(), "authentication-provider-list", errorCode); throw authenticationException; } - } private static class AuthenticationListState implements AuthenticationState { private final List states; private volatile AuthenticationState authState; + private final AuthenticationMetrics metrics; - AuthenticationListState(List states) { + AuthenticationListState(List states, AuthenticationMetrics metrics) { if (states == null || states.isEmpty()) { throw new IllegalArgumentException("Authentication state requires at least one state"); } this.states = states; this.authState = states.get(0); + this.metrics = metrics; } private AuthenticationState getAuthState() throws AuthenticationException { @@ -120,7 +122,8 @@ public CompletableFuture authenticateAsync(AuthData authData) { if (log.isDebugEnabled()) { log.debug("Authentication failed for auth provider " + authState.getClass() + ": ", ex); } - authenticateRemainingAuthStates(authChallengeFuture, authData, ex, states.size() - 1); + authenticateRemainingAuthStates(authChallengeFuture, authData, ex, + states.isEmpty() ? -1 : 0); } }); return authChallengeFuture; @@ -130,19 +133,20 @@ private void authenticateRemainingAuthStates(CompletableFuture authCha AuthData clientAuthData, Throwable previousException, int index) { - if (index < 0) { + if (index < 0 || index >= states.size()) { if (previousException == null) { previousException = new AuthenticationException("Authentication required"); } - AuthenticationMetrics.authenticateFailure(AuthenticationProviderList.class.getSimpleName(), - "authentication-provider-list", ErrorCode.AUTH_REQUIRED); + metrics.recordFailure(AuthenticationProviderList.class.getSimpleName(), + "authentication-provider-list", + ErrorCode.AUTH_REQUIRED); authChallengeFuture.completeExceptionally(previousException); return; } AuthenticationState state = states.get(index); if (state == authState) { // Skip the current auth state - authenticateRemainingAuthStates(authChallengeFuture, clientAuthData, null, index - 1); + authenticateRemainingAuthStates(authChallengeFuture, clientAuthData, null, index + 1); } else { state.authenticateAsync(clientAuthData) .whenComplete((authChallenge, ex) -> { @@ -155,7 +159,7 @@ private void authenticateRemainingAuthStates(CompletableFuture authCha log.debug("Authentication failed for auth provider " + authState.getClass() + ": ", ex); } - authenticateRemainingAuthStates(authChallengeFuture, clientAuthData, ex, index - 1); + authenticateRemainingAuthStates(authChallengeFuture, clientAuthData, ex, index + 1); } }); } @@ -165,6 +169,7 @@ private void authenticateRemainingAuthStates(CompletableFuture authCha public AuthData authenticate(AuthData authData) throws AuthenticationException { return applyAuthProcessor( states, + metrics, as -> { AuthData ad = as.authenticate(authData); AuthenticationListState.this.authState = as; @@ -215,8 +220,15 @@ public List getProviders() { @Override public void initialize(ServiceConfiguration config) throws IOException { + initialize(Context.builder().config(config).build()); + } + + @Override + public void initialize(Context context) throws IOException { + authenticationMetrics = new AuthenticationMetrics(context.getOpenTelemetry(), + getClass().getSimpleName(), getAuthMethodName()); for (AuthenticationProvider ap : providers) { - ap.initialize(config); + ap.initialize(context); } } @@ -225,10 +237,15 @@ public String getAuthMethodName() { return providers.get(0).getAuthMethodName(); } + @Override + public void incrementFailureMetric(Enum errorCode) { + authenticationMetrics.recordFailure(errorCode); + } + @Override public CompletableFuture authenticateAsync(AuthenticationDataSource authData) { CompletableFuture roleFuture = new CompletableFuture<>(); - authenticateRemainingAuthProviders(roleFuture, authData, null, providers.size() - 1); + authenticateRemainingAuthProviders(roleFuture, authData, null, providers.isEmpty() ? -1 : 0); return roleFuture; } @@ -236,11 +253,11 @@ private void authenticateRemainingAuthProviders(CompletableFuture roleFu AuthenticationDataSource authData, Throwable previousException, int index) { - if (index < 0) { + if (index < 0 || index >= providers.size()) { if (previousException == null) { previousException = new AuthenticationException("Authentication required"); } - AuthenticationMetrics.authenticateFailure(AuthenticationProviderList.class.getSimpleName(), + authenticationMetrics.recordFailure(AuthenticationProvider.class.getSimpleName(), "authentication-provider-list", ErrorCode.AUTH_REQUIRED); roleFuture.completeExceptionally(previousException); return; @@ -254,7 +271,7 @@ private void authenticateRemainingAuthProviders(CompletableFuture roleFu if (log.isDebugEnabled()) { log.debug("Authentication failed for auth provider " + provider.getClass() + ": ", ex); } - authenticateRemainingAuthProviders(roleFuture, authData, ex, index - 1); + authenticateRemainingAuthProviders(roleFuture, authData, ex, index + 1); } }); } @@ -263,6 +280,7 @@ private void authenticateRemainingAuthProviders(CompletableFuture roleFu public String authenticate(AuthenticationDataSource authData) throws AuthenticationException { return applyAuthProcessor( providers, + authenticationMetrics, provider -> provider.authenticate(authData) ); } @@ -293,7 +311,7 @@ public AuthenticationState newAuthState(AuthData authData, SocketAddress remoteA throw new AuthenticationException("Failed to initialize a new auth state from " + remoteAddress); } } else { - return new AuthenticationListState(states); + return new AuthenticationListState(states, authenticationMetrics); } } @@ -324,7 +342,7 @@ public AuthenticationState newHttpAuthState(HttpServletRequest request) throws A "Failed to initialize a new http auth state from " + request.getRemoteHost()); } } else { - return new AuthenticationListState(states); + return new AuthenticationListState(states, authenticationMetrics); } } @@ -332,6 +350,7 @@ public AuthenticationState newHttpAuthState(HttpServletRequest request) throws A public boolean authenticateHttpRequest(HttpServletRequest request, HttpServletResponse response) throws Exception { Boolean authenticated = applyAuthProcessor( providers, + authenticationMetrics, provider -> { try { return provider.authenticateHttpRequest(request, response); diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTls.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTls.java index a4c44121b4b96..f7ff47fe8e61e 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTls.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTls.java @@ -27,6 +27,8 @@ public class AuthenticationProviderTls implements AuthenticationProvider { + private AuthenticationMetrics authenticationMetrics; + private enum ErrorCode { UNKNOWN, INVALID_CERTS, @@ -40,7 +42,13 @@ public void close() throws IOException { @Override public void initialize(ServiceConfiguration config) throws IOException { - // noop + initialize(Context.builder().config(config).build()); + } + + @Override + public void initialize(Context context) throws IOException { + authenticationMetrics = new AuthenticationMetrics(context.getOpenTelemetry(), + getClass().getSimpleName(), getAuthMethodName()); } @Override @@ -48,6 +56,11 @@ public String getAuthMethodName() { return "tls"; } + @Override + public void incrementFailureMetric(Enum errorCode) { + authenticationMetrics.recordFailure(errorCode); + } + @Override public String authenticate(AuthenticationDataSource authData) throws AuthenticationException { String commonName = null; @@ -96,7 +109,7 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat errorCode = ErrorCode.INVALID_CN; throw new AuthenticationException("Client unable to authenticate with TLS certificate"); } - AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName()); + authenticationMetrics.recordSuccess(); } catch (AuthenticationException exception) { incrementFailureMetric(errorCode); throw exception; diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java index f8992b21ff49f..74bc85ad3ffc3 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java @@ -21,7 +21,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.pulsar.broker.web.AuthenticationFilter.AuthenticatedDataAttributeName; import static org.apache.pulsar.broker.web.AuthenticationFilter.AuthenticatedRoleAttributeName; -import com.google.common.annotations.VisibleForTesting; import io.jsonwebtoken.Claims; import io.jsonwebtoken.ExpiredJwtException; import io.jsonwebtoken.Jwt; @@ -31,8 +30,6 @@ import io.jsonwebtoken.RequiredTypeException; import io.jsonwebtoken.SignatureAlgorithm; import io.jsonwebtoken.security.SignatureException; -import io.prometheus.client.Counter; -import io.prometheus.client.Histogram; import java.io.IOException; import java.net.SocketAddress; import java.security.Key; @@ -44,7 +41,7 @@ import javax.servlet.http.HttpServletResponse; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.authentication.metrics.AuthenticationMetrics; +import org.apache.pulsar.broker.authentication.metrics.AuthenticationMetricsToken; import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; import org.apache.pulsar.common.api.AuthData; @@ -79,17 +76,6 @@ public class AuthenticationProviderToken implements AuthenticationProvider { static final String TOKEN = "token"; - private static final Counter expiredTokenMetrics = Counter.build() - .name("pulsar_expired_token_total") - .help("Pulsar expired token") - .register(); - - private static final Histogram expiringTokenMinutesMetrics = Histogram.build() - .name("pulsar_expiring_token_minutes") - .help("The remaining time of expiring token in minutes") - .buckets(5, 10, 60, 240) - .register(); - private Key validationKey; private String roleClaim; private SignatureAlgorithm publicKeyAlg; @@ -106,6 +92,8 @@ public class AuthenticationProviderToken implements AuthenticationProvider { private String confTokenAudienceSettingName; private String confTokenAllowedClockSkewSecondsSettingName; + private AuthenticationMetricsToken authenticationMetricsToken; + public enum ErrorCode { INVALID_AUTH_DATA, INVALID_TOKEN, @@ -117,14 +105,17 @@ public void close() throws IOException { // noop } - @VisibleForTesting - public static void resetMetrics() { - expiredTokenMetrics.clear(); - expiringTokenMinutesMetrics.clear(); + @Override + public void initialize(ServiceConfiguration config) throws IOException { + initialize(Context.builder().config(config).build()); } @Override - public void initialize(ServiceConfiguration config) throws IOException, IllegalArgumentException { + public void initialize(Context context) throws IOException { + authenticationMetricsToken = new AuthenticationMetricsToken(context.getOpenTelemetry(), + getClass().getSimpleName(), getAuthMethodName()); + + var config = context.getConfig(); String prefix = (String) config.getProperty(CONF_TOKEN_SETTING_PREFIX); if (null == prefix) { prefix = ""; @@ -162,6 +153,11 @@ public String getAuthMethodName() { return TOKEN; } + @Override + public void incrementFailureMetric(Enum errorCode) { + authenticationMetricsToken.recordFailure(errorCode); + } + @Override public String authenticate(AuthenticationDataSource authData) throws AuthenticationException { String token; @@ -174,7 +170,7 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat } // Parse Token by validating String role = getPrincipal(authenticateToken(token)); - AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName()); + authenticationMetricsToken.recordSuccess(); return role; } @@ -263,14 +259,13 @@ private Jwt authenticateToken(final String token) throws Authenticati } } - if (jwt.getBody().getExpiration() != null) { - expiringTokenMinutesMetrics.observe( - (double) (jwt.getBody().getExpiration().getTime() - new Date().getTime()) / (60 * 1000)); - } + var expiration = jwt.getBody().getExpiration(); + var tokenRemainingDurationMs = expiration != null ? expiration.getTime() - new Date().getTime() : null; + authenticationMetricsToken.recordTokenDuration(tokenRemainingDurationMs); return jwt; } catch (JwtException e) { if (e instanceof ExpiredJwtException) { - expiredTokenMetrics.inc(); + authenticationMetricsToken.recordTokenExpired(); } incrementFailureMetric(ErrorCode.INVALID_TOKEN); throw new AuthenticationException("Failed to authentication token: " + e.getMessage()); diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationService.java index 22296b86b4e0c..f6eb785d2e479 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationService.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationService.java @@ -20,6 +20,7 @@ import static org.apache.pulsar.broker.web.AuthenticationFilter.AuthenticatedDataAttributeName; import static org.apache.pulsar.broker.web.AuthenticationFilter.AuthenticatedRoleAttributeName; +import io.opentelemetry.api.OpenTelemetry; import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; @@ -51,6 +52,11 @@ public class AuthenticationService implements Closeable { private final Map providers = new HashMap<>(); public AuthenticationService(ServiceConfiguration conf) throws PulsarServerException { + this(conf, OpenTelemetry.noop()); + } + + public AuthenticationService(ServiceConfiguration conf, OpenTelemetry openTelemetry) + throws PulsarServerException { anonymousUserRole = conf.getAnonymousUserRole(); if (conf.isAuthenticationEnabled()) { try { @@ -70,6 +76,10 @@ public AuthenticationService(ServiceConfiguration conf) throws PulsarServerExcep providerList.add(provider); } + var authenticationProviderContext = AuthenticationProvider.Context.builder() + .config(conf) + .openTelemetry(openTelemetry) + .build(); for (Map.Entry> entry : providerMap.entrySet()) { AuthenticationProvider provider; if (entry.getValue().size() == 1) { @@ -77,7 +87,7 @@ public AuthenticationService(ServiceConfiguration conf) throws PulsarServerExcep } else { provider = new AuthenticationProviderList(entry.getValue()); } - provider.initialize(conf); + provider.initialize(authenticationProviderContext); providers.put(provider.getAuthMethodName(), provider); LOG.info("[{}] has been loaded.", entry.getValue().stream().map( diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetrics.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetrics.java index 5faaccbe15716..931ad50e11728 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetrics.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetrics.java @@ -18,28 +18,27 @@ */ package org.apache.pulsar.broker.authentication.metrics; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.LongCounter; import io.prometheus.client.Counter; public class AuthenticationMetrics { + @Deprecated private static final Counter authSuccessMetrics = Counter.build() .name("pulsar_authentication_success_total") .help("Pulsar authentication success") .labelNames("provider_name", "auth_method") .register(); + @Deprecated private static final Counter authFailuresMetrics = Counter.build() .name("pulsar_authentication_failures_total") .help("Pulsar authentication failures") .labelNames("provider_name", "auth_method", "reason") .register(); - /** - * Log authenticate success event to the authentication metrics. - * @param providerName The short class name of the provider - * @param authMethod Authentication method name - */ - public static void authenticateSuccess(String providerName, String authMethod) { - authSuccessMetrics.labels(providerName, authMethod).inc(); - } + public static final String INSTRUMENTATION_SCOPE_NAME = "org.apache.pulsar.authentication"; /** * Log authenticate failure event to the authentication metrics. @@ -62,8 +61,58 @@ public static void authenticateFailure(String providerName, String authMethod, S * @param authMethod Authentication method name. * @param errorCode Error code. */ + @Deprecated public static void authenticateFailure(String providerName, String authMethod, Enum errorCode) { authFailuresMetrics.labels(providerName, authMethod, errorCode.name()).inc(); } + public static final String AUTHENTICATION_COUNTER_METRIC_NAME = "pulsar.authentication.operation.count"; + private final LongCounter authenticationCounter; + + public static final AttributeKey PROVIDER_KEY = AttributeKey.stringKey("pulsar.authentication.provider"); + public static final AttributeKey AUTH_METHOD_KEY = AttributeKey.stringKey("pulsar.authentication.method"); + public static final AttributeKey ERROR_CODE_KEY = AttributeKey.stringKey("pulsar.authentication.error"); + public static final AttributeKey AUTH_RESULT_KEY = AttributeKey.stringKey("pulsar.authentication.result"); + public enum AuthenticationResult { + SUCCESS, + FAILURE; + } + + private final String providerName; + private final String authMethod; + + public AuthenticationMetrics(OpenTelemetry openTelemetry, String providerName, String authMethod) { + this.providerName = providerName; + this.authMethod = authMethod; + var meter = openTelemetry.getMeter(INSTRUMENTATION_SCOPE_NAME); + authenticationCounter = meter.counterBuilder(AUTHENTICATION_COUNTER_METRIC_NAME) + .setDescription("The number of authentication operations") + .setUnit("{operation}") + .build(); + } + + public void recordSuccess() { + authSuccessMetrics.labels(providerName, authMethod).inc(); + var attributes = Attributes.of(PROVIDER_KEY, providerName, + AUTH_METHOD_KEY, authMethod, + AUTH_RESULT_KEY, AuthenticationResult.SUCCESS.name().toLowerCase()); + authenticationCounter.add(1, attributes); + } + + public void recordFailure(Enum errorCode) { + recordFailure(providerName, authMethod, errorCode.name()); + } + + public void recordFailure(String providerName, String authMethod, Enum errorCode) { + recordFailure(providerName, authMethod, errorCode.name()); + } + + public void recordFailure(String providerName, String authMethod, String errorCode) { + authenticateFailure(providerName, authMethod, errorCode); + var attributes = Attributes.of(PROVIDER_KEY, providerName, + AUTH_METHOD_KEY, authMethod, + AUTH_RESULT_KEY, AuthenticationResult.FAILURE.name().toLowerCase(), + ERROR_CODE_KEY, errorCode); + authenticationCounter.add(1, attributes); + } } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetricsToken.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetricsToken.java new file mode 100644 index 0000000000000..4e9d1d6b16a92 --- /dev/null +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetricsToken.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authentication.metrics; + +import com.google.common.annotations.VisibleForTesting; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.LongCounter; +import io.prometheus.client.Counter; +import io.prometheus.client.Histogram; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.apache.pulsar.common.stats.MetricsUtil; + +public class AuthenticationMetricsToken extends AuthenticationMetrics { + + @Deprecated + private static final Counter expiredTokenMetrics = Counter.build() + .name("pulsar_expired_token_total") + .help("Pulsar expired token") + .register(); + public static final String EXPIRED_TOKEN_COUNTER_METRIC_NAME = "pulsar.authentication.token.expired.count"; + private LongCounter expiredTokensCounter; + + private static final List TOKEN_DURATION_BUCKET_BOUNDARIES_SECONDS = List.of( + TimeUnit.MINUTES.toSeconds(5), + TimeUnit.MINUTES.toSeconds(10), + TimeUnit.HOURS.toSeconds(1), + TimeUnit.HOURS.toSeconds(4), + TimeUnit.DAYS.toSeconds(1), + TimeUnit.DAYS.toSeconds(7), + TimeUnit.DAYS.toSeconds(14), + TimeUnit.DAYS.toSeconds(30), + TimeUnit.DAYS.toSeconds(90), + TimeUnit.DAYS.toSeconds(180), + TimeUnit.DAYS.toSeconds(270), + TimeUnit.DAYS.toSeconds(365)); + + @Deprecated + private static final Histogram expiringTokenMinutesMetrics = Histogram.build() + .name("pulsar_expiring_token_minutes") + .help("The remaining time of expiring token in minutes") + .buckets(TOKEN_DURATION_BUCKET_BOUNDARIES_SECONDS.stream() + .map(TimeUnit.SECONDS::toMinutes) + .mapToDouble(Double::valueOf) + .toArray()) + .register(); + public static final String EXPIRING_TOKEN_HISTOGRAM_METRIC_NAME = "pulsar.authentication.token.expiry.duration"; + private DoubleHistogram expiringTokenSeconds; + + public AuthenticationMetricsToken(OpenTelemetry openTelemetry, String providerName, + String authMethod) { + super(openTelemetry, providerName, authMethod); + + var meter = openTelemetry.getMeter(AuthenticationMetrics.INSTRUMENTATION_SCOPE_NAME); + expiredTokensCounter = meter.counterBuilder(EXPIRED_TOKEN_COUNTER_METRIC_NAME) + .setDescription("The total number of expired tokens") + .setUnit("{token}") + .build(); + expiringTokenSeconds = meter.histogramBuilder(EXPIRING_TOKEN_HISTOGRAM_METRIC_NAME) + .setExplicitBucketBoundariesAdvice( + TOKEN_DURATION_BUCKET_BOUNDARIES_SECONDS.stream().map(Double::valueOf).toList()) + .setDescription("The remaining time of expiring token in seconds") + .setUnit("s") + .build(); + } + + public void recordTokenDuration(Long durationMs) { + if (durationMs == null) { + // Special case signals a token without expiry. OpenTelemetry supports reporting infinite values. + expiringTokenSeconds.record(Double.POSITIVE_INFINITY); + } else if (durationMs > 0) { + expiringTokenMinutesMetrics.observe(durationMs / 60_000.0d); + expiringTokenSeconds.record(MetricsUtil.convertToSeconds(durationMs, TimeUnit.MILLISECONDS)); + } else { + // Duration can be negative if token expires at processing time. OpenTelemetry does not support negative + // values, so record token expiry instead. + recordTokenExpired(); + } + } + + public void recordTokenExpired() { + expiredTokenMetrics.inc(); + expiredTokensCounter.add(1); + } + + @VisibleForTesting + @Deprecated + public static void reset() { + expiredTokenMetrics.clear(); + expiringTokenMinutesMetrics.clear(); + } +} diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/utils/AuthTokenUtils.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/utils/AuthTokenUtils.java index cb917a9e0bf00..e29b106f4f17e 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/utils/AuthTokenUtils.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/utils/AuthTokenUtils.java @@ -35,6 +35,7 @@ import java.security.spec.PKCS8EncodedKeySpec; import java.security.spec.X509EncodedKeySpec; import java.util.Date; +import java.util.Map; import java.util.Optional; import javax.crypto.SecretKey; import lombok.experimental.UtilityClass; @@ -89,16 +90,22 @@ public static String encodeKeyBase64(Key key) { return Encoders.BASE64.encode(key.getEncoded()); } - public static String createToken(Key signingKey, String subject, Optional expiryTime) { + public static String createToken(Key signingKey, String subject, Optional expiryTime, + Optional> headers) { JwtBuilder builder = Jwts.builder() .setSubject(subject) .signWith(signingKey); expiryTime.ifPresent(builder::setExpiration); + headers.ifPresent(builder::setHeaderParams); return builder.compact(); } + public static String createToken(Key signingKey, String subject, Optional expiryTime) { + return createToken(signingKey, subject, expiryTime, Optional.empty()); + } + public static byte[] readKeyFromUrl(String keyConfUrl) throws IOException { if (keyConfUrl.startsWith("data:") || keyConfUrl.startsWith("file:")) { try { diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java index 67e096bee63e4..7d25580ff92bb 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationProvider.java @@ -20,9 +20,9 @@ import java.io.Closeable; import java.io.IOException; +import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.resources.PulsarResources; @@ -36,7 +36,6 @@ import org.apache.pulsar.common.policies.data.TenantOperation; import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.RestException; import org.apache.pulsar.metadata.api.MetadataStoreException; /** @@ -58,20 +57,6 @@ default CompletableFuture isSuperUser(String role, return CompletableFuture.completedFuture(role != null && superUserRoles.contains(role)); } - /** - * @deprecated - Use method {@link #isSuperUser(String, AuthenticationDataSource, ServiceConfiguration)}. - * Will be removed after 2.12. - * Check if specified role is a super user - * @param role the role to check - * @return a CompletableFuture containing a boolean in which true means the role is a super user - * and false if it is not - */ - @Deprecated - default CompletableFuture isSuperUser(String role, ServiceConfiguration serviceConfiguration) { - Set superUserRoles = serviceConfiguration.getSuperUserRoles(); - return CompletableFuture.completedFuture(role != null && superUserRoles.contains(role)); - } - /** * Check if specified role is an admin of the tenant. * @param tenant the tenant to check @@ -185,6 +170,18 @@ CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, String CompletableFuture grantPermissionAsync(NamespaceName namespace, Set actions, String role, String authDataJson); + /** + * Revoke authorization-action permission on a namespace to the given client. + * @param namespace + * @param role + * @return CompletableFuture + */ + default CompletableFuture revokePermissionAsync(NamespaceName namespace, String role) { + return FutureUtil.failedFuture(new IllegalStateException( + String.format("revokePermissionAsync on namespace %s is not supported by the Authorization", + namespace))); + } + /** * Grant permission to roles that can access subscription-admin api. * @@ -193,7 +190,7 @@ CompletableFuture grantPermissionAsync(NamespaceName namespace, Set */ CompletableFuture grantSubscriptionPermissionAsync(NamespaceName namespace, String subscriptionName, Set roles, String authDataJson); @@ -203,7 +200,7 @@ CompletableFuture grantSubscriptionPermissionAsync(NamespaceName namespace * @param namespace * @param subscriptionName * @param role - * @return + * @return CompletableFuture */ CompletableFuture revokeSubscriptionPermissionAsync(NamespaceName namespace, String subscriptionName, String role, String authDataJson); @@ -226,6 +223,19 @@ CompletableFuture revokeSubscriptionPermissionAsync(NamespaceName namespac CompletableFuture grantPermissionAsync(TopicName topicName, Set actions, String role, String authDataJson); + + /** + * Revoke authorization-action permission on a topic to the given client. + * @param topicName + * @param role + * @return CompletableFuture + */ + default CompletableFuture revokePermissionAsync(TopicName topicName, String role) { + return FutureUtil.failedFuture(new IllegalStateException( + String.format("revokePermissionAsync on topicName %s is not supported by the Authorization", + topicName))); + } + /** * Check if a given role is allowed to execute a given operation on the tenant. * @@ -244,21 +254,6 @@ default CompletableFuture allowTenantOperationAsync(String tenantName, operation.toString(), tenantName))); } - /** - * @deprecated - will be removed after 2.12. Use async variant. - */ - @Deprecated - default Boolean allowTenantOperation(String tenantName, String role, TenantOperation operation, - AuthenticationDataSource authData) { - try { - return allowTenantOperationAsync(tenantName, role, operation, authData).get(); - } catch (InterruptedException e) { - throw new RestException(e); - } catch (ExecutionException e) { - throw new RestException(e.getCause()); - } - } - /** * Check if a given role is allowed to execute a given operation on the namespace. * @@ -277,23 +272,6 @@ default CompletableFuture allowNamespaceOperationAsync(NamespaceName na + "the Authorization provider you are using.")); } - /** - * @deprecated - will be removed after 2.12. Use async variant. - */ - @Deprecated - default Boolean allowNamespaceOperation(NamespaceName namespaceName, - String role, - NamespaceOperation operation, - AuthenticationDataSource authData) { - try { - return allowNamespaceOperationAsync(namespaceName, role, operation, authData).get(); - } catch (InterruptedException e) { - throw new RestException(e); - } catch (ExecutionException e) { - throw new RestException(e.getCause()); - } - } - /** * Check if a given role is allowed to execute a given policy operation on the namespace. * @@ -314,24 +292,6 @@ default CompletableFuture allowNamespacePolicyOperationAsync(NamespaceN + "is not supported by is not supported by the Authorization provider you are using.")); } - /** - * @deprecated - will be removed after 2.12. Use async variant. - */ - @Deprecated - default Boolean allowNamespacePolicyOperation(NamespaceName namespaceName, - PolicyName policy, - PolicyOperation operation, - String role, - AuthenticationDataSource authData) { - try { - return allowNamespacePolicyOperationAsync(namespaceName, policy, operation, role, authData).get(); - } catch (InterruptedException e) { - throw new RestException(e); - } catch (ExecutionException e) { - throw new RestException(e.getCause()); - } - } - /** * Check if a given role is allowed to execute a given topic operation on the topic. * @@ -350,23 +310,6 @@ default CompletableFuture allowTopicOperationAsync(TopicName topic, + "provider you are using.")); } - /** - * @deprecated - will be removed after 2.12. Use async variant. - */ - @Deprecated - default Boolean allowTopicOperation(TopicName topicName, - String role, - TopicOperation operation, - AuthenticationDataSource authData) { - try { - return allowTopicOperationAsync(topicName, role, operation, authData).get(); - } catch (InterruptedException e) { - throw new RestException(e); - } catch (ExecutionException e) { - throw new RestException(e.getCause()); - } - } - /** * Check if a given role is allowed to execute a given topic operation on topic's policy. * @@ -387,20 +330,44 @@ default CompletableFuture allowTopicPolicyOperationAsync(TopicName topi } /** - * @deprecated - will be removed after 2.12. Use async variant. + * Remove authorization-action permissions on a topic. + * @param topicName + * @return CompletableFuture */ - @Deprecated - default Boolean allowTopicPolicyOperation(TopicName topicName, - String role, - PolicyName policy, - PolicyOperation operation, - AuthenticationDataSource authData) { - try { - return allowTopicPolicyOperationAsync(topicName, role, policy, operation, authData).get(); - } catch (InterruptedException e) { - throw new RestException(e); - } catch (ExecutionException e) { - throw new RestException(e.getCause()); - } + default CompletableFuture removePermissionsAsync(TopicName topicName) { + return CompletableFuture.completedFuture(null); + } + + /** + * Get authorization-action permissions on a topic. + * @param topicName + * @return CompletableFuture>> + */ + default CompletableFuture>> getPermissionsAsync(TopicName topicName) { + return FutureUtil.failedFuture(new IllegalStateException( + String.format("getPermissionsAsync on topicName %s is not supported by the Authorization", + topicName))); + } + + /** + * Get authorization-action permissions on a topic. + * @param namespaceName + * @return CompletableFuture>> + */ + default CompletableFuture>> getSubscriptionPermissionsAsync(NamespaceName namespaceName) { + return FutureUtil.failedFuture(new IllegalStateException( + String.format("getSubscriptionPermissionsAsync on namespace %s is not supported by the Authorization", + namespaceName))); + } + + /** + * Get authorization-action permissions on a namespace. + * @param namespaceName + * @return CompletableFuture>> + */ + default CompletableFuture>> getPermissionsAsync(NamespaceName namespaceName) { + return FutureUtil.failedFuture(new IllegalStateException( + String.format("getPermissionsAsync on namespaceName %s is not supported by the Authorization", + namespaceName))); } } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java index 6f303e2117fe0..c121d93b9b750 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java @@ -20,6 +20,7 @@ import static java.util.concurrent.TimeUnit.SECONDS; import java.net.SocketAddress; +import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -123,6 +124,17 @@ public CompletableFuture grantPermissionAsync(NamespaceName namespace, Set return provider.grantPermissionAsync(namespace, actions, role, authDataJson); } + /** + * + * Revoke authorization-action permission on a namespace to the given client. + * + * @param namespace + * @param role + */ + public CompletableFuture revokePermissionAsync(NamespaceName namespace, String role) { + return provider.revokePermissionAsync(namespace, role); + } + /** * Grant permission to roles that can access subscription-admin api. * @@ -157,16 +169,26 @@ public CompletableFuture revokeSubscriptionPermissionAsync(NamespaceName n * NOTE: used to complete with {@link IllegalArgumentException} when namespace not found or with * {@link IllegalStateException} when failed to grant permission. * - * @param topicname + * @param topicName * @param role * @param authDataJson * additional authdata in json for targeted authorization provider * @completesWith null when the permissions are updated successfully. * @completesWith {@link MetadataStoreException} when the MetadataStore is not updated. */ - public CompletableFuture grantPermissionAsync(TopicName topicname, Set actions, String role, + public CompletableFuture grantPermissionAsync(TopicName topicName, Set actions, String role, String authDataJson) { - return provider.grantPermissionAsync(topicname, actions, role, authDataJson); + return provider.grantPermissionAsync(topicName, actions, role, authDataJson); + } + + /** + * Revoke authorization-action permission on a topic to the given client. + * + * @param topicName + * @param role + */ + public CompletableFuture revokePermissionAsync(TopicName topicName, String role) { + return provider.revokePermissionAsync(topicName, role); } /** @@ -418,7 +440,7 @@ private boolean isValidOriginalPrincipal(AuthenticationParameters authParams) { /** * Whether the authenticatedPrincipal and the originalPrincipal form a valid pair. This method assumes that * authenticatedPrincipal and originalPrincipal can be equal, as long as they are not a proxy role. This use - * case is relvant for the admin server because of the way the proxy handles authentication. The binary protocol + * case is relevant for the admin server because of the way the proxy handles authentication. The binary protocol * should not use this method. * @return true when roles are a valid combination and false when roles are an invalid combination */ @@ -794,4 +816,20 @@ public Boolean allowTopicOperation(TopicName topicName, throw new RestException(e.getCause()); } } + + public CompletableFuture removePermissionsAsync(TopicName topicName) { + return provider.removePermissionsAsync(topicName); + } + + public CompletableFuture>> getPermissionsAsync(TopicName topicName) { + return provider.getPermissionsAsync(topicName); + } + + public CompletableFuture>> getPermissionsAsync(NamespaceName namespaceName) { + return provider.getPermissionsAsync(namespaceName); + } + + public CompletableFuture>> getSubscriptionPermissionsAsync(NamespaceName namespaceName) { + return provider.getSubscriptionPermissionsAsync(namespaceName); + } } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java index fa613245cfa27..fdab233a51098 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java @@ -35,6 +35,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; @@ -89,15 +90,18 @@ public void initialize(ServiceConfiguration conf, PulsarResources pulsarResource @Override public CompletableFuture isSuperUser(String role, AuthenticationDataSource authenticationData, ServiceConfiguration serviceConfiguration) { - Set roles = getRoles(authenticationData); - if (roles.isEmpty()) { - return CompletableFuture.completedFuture(false); - } + // if superUser role contains in config, return true. Set superUserRoles = serviceConfiguration.getSuperUserRoles(); if (superUserRoles.isEmpty()) { return CompletableFuture.completedFuture(false); } - + if (role != null && superUserRoles.contains(role)) { + return CompletableFuture.completedFuture(true); + } + Set roles = getRoles(role, authenticationData); + if (roles.isEmpty()) { + return CompletableFuture.completedFuture(false); + } return CompletableFuture.completedFuture(roles.stream().anyMatch(superUserRoles::contains)); } @@ -109,7 +113,7 @@ public CompletableFuture validateTenantAdminAccess(String tenantName, S if (isSuperUser) { return CompletableFuture.completedFuture(true); } - Set roles = getRoles(authData); + Set roles = getRoles(role, authData); if (roles.isEmpty()) { return CompletableFuture.completedFuture(false); } @@ -140,7 +144,12 @@ public CompletableFuture validateTenantAdminAccess(String tenantName, S }); } - private Set getRoles(AuthenticationDataSource authData) { + private Set getRoles(String role, AuthenticationDataSource authData) { + if (authData == null || (authData instanceof AuthenticationDataSubscription + && ((AuthenticationDataSubscription) authData).getAuthData() == null)) { + return Collections.singleton(role); + } + String token = null; if (authData.hasDataFromCommand()) { @@ -174,7 +183,14 @@ private Set getRoles(AuthenticationDataSource authData) { Jwt jwt = parser.parseClaimsJwt(unsignedToken); try { - return new HashSet<>(Collections.singletonList(jwt.getBody().get(roleClaim, String.class))); + final String jwtRole = jwt.getBody().get(roleClaim, String.class); + if (jwtRole == null) { + if (log.isDebugEnabled()) { + log.debug("Do not have corresponding claim in jwt token. claim={}", roleClaim); + } + return Collections.emptySet(); + } + return new HashSet<>(Collections.singletonList(jwtRole)); } catch (RequiredTypeException requiredTypeException) { try { List list = jwt.getBody().get(roleClaim, List.class); @@ -189,15 +205,21 @@ private Set getRoles(AuthenticationDataSource authData) { return Collections.emptySet(); } - public CompletableFuture authorize(AuthenticationDataSource authenticationData, Function> authorizeFunc) { - Set roles = getRoles(authenticationData); - if (roles.isEmpty()) { - return CompletableFuture.completedFuture(false); - } - List> futures = new ArrayList<>(roles.size()); - roles.forEach(r -> futures.add(authorizeFunc.apply(r))); - return FutureUtil.waitForAny(futures, ret -> (boolean) ret).thenApply(v -> v.isPresent()); + public CompletableFuture authorize(String role, AuthenticationDataSource authenticationData, + Function> authorizeFunc) { + return isSuperUser(role, authenticationData, conf) + .thenCompose(superUser -> { + if (superUser) { + return CompletableFuture.completedFuture(true); + } + Set roles = getRoles(role, authenticationData); + if (roles.isEmpty()) { + return CompletableFuture.completedFuture(false); + } + List> futures = new ArrayList<>(roles.size()); + roles.forEach(r -> futures.add(authorizeFunc.apply(r))); + return FutureUtil.waitForAny(futures, ret -> (boolean) ret).thenApply(v -> v.isPresent()); + }); } /** @@ -209,7 +231,7 @@ public CompletableFuture authorize(AuthenticationDataSource authenticat @Override public CompletableFuture canProduceAsync(TopicName topicName, String role, AuthenticationDataSource authenticationData) { - return authorize(authenticationData, r -> super.canProduceAsync(topicName, r, authenticationData)); + return authorize(role, authenticationData, r -> super.canProduceAsync(topicName, r, authenticationData)); } /** @@ -224,7 +246,7 @@ public CompletableFuture canProduceAsync(TopicName topicName, String ro public CompletableFuture canConsumeAsync(TopicName topicName, String role, AuthenticationDataSource authenticationData, String subscription) { - return authorize(authenticationData, r -> super.canConsumeAsync(topicName, r, authenticationData, + return authorize(role, authenticationData, r -> super.canConsumeAsync(topicName, r, authenticationData, subscription)); } @@ -241,25 +263,27 @@ public CompletableFuture canConsumeAsync(TopicName topicName, String ro @Override public CompletableFuture canLookupAsync(TopicName topicName, String role, AuthenticationDataSource authenticationData) { - return authorize(authenticationData, r -> super.canLookupAsync(topicName, r, authenticationData)); + return authorize(role, authenticationData, r -> super.canLookupAsync(topicName, r, authenticationData)); } @Override public CompletableFuture allowFunctionOpsAsync(NamespaceName namespaceName, String role, AuthenticationDataSource authenticationData) { - return authorize(authenticationData, r -> super.allowFunctionOpsAsync(namespaceName, r, authenticationData)); + return authorize(role, authenticationData, + r -> super.allowFunctionOpsAsync(namespaceName, r, authenticationData)); } @Override public CompletableFuture allowSourceOpsAsync(NamespaceName namespaceName, String role, AuthenticationDataSource authenticationData) { - return authorize(authenticationData, r -> super.allowSourceOpsAsync(namespaceName, r, authenticationData)); + return authorize(role, authenticationData, + r -> super.allowSourceOpsAsync(namespaceName, r, authenticationData)); } @Override public CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, String role, AuthenticationDataSource authenticationData) { - return authorize(authenticationData, r -> super.allowSinkOpsAsync(namespaceName, r, authenticationData)); + return authorize(role, authenticationData, r -> super.allowSinkOpsAsync(namespaceName, r, authenticationData)); } @Override @@ -267,7 +291,7 @@ public CompletableFuture allowTenantOperationAsync(String tenantName, String role, TenantOperation operation, AuthenticationDataSource authData) { - return authorize(authData, r -> super.allowTenantOperationAsync(tenantName, r, operation, authData)); + return authorize(role, authData, r -> super.allowTenantOperationAsync(tenantName, r, operation, authData)); } @Override @@ -275,7 +299,8 @@ public CompletableFuture allowNamespaceOperationAsync(NamespaceName nam String role, NamespaceOperation operation, AuthenticationDataSource authData) { - return authorize(authData, r -> super.allowNamespaceOperationAsync(namespaceName, r, operation, authData)); + return authorize(role, authData, + r -> super.allowNamespaceOperationAsync(namespaceName, r, operation, authData)); } @Override @@ -284,8 +309,8 @@ public CompletableFuture allowNamespacePolicyOperationAsync(NamespaceNa PolicyOperation operation, String role, AuthenticationDataSource authData) { - return authorize(authData, r -> super.allowNamespacePolicyOperationAsync(namespaceName, policy, operation, r, - authData)); + return authorize(role, authData, + r -> super.allowNamespacePolicyOperationAsync(namespaceName, policy, operation, r, authData)); } @Override @@ -293,7 +318,7 @@ public CompletableFuture allowTopicOperationAsync(TopicName topicName, String role, TopicOperation operation, AuthenticationDataSource authData) { - return authorize(authData, r -> super.allowTopicOperationAsync(topicName, r, operation, authData)); + return authorize(role, authData, r -> super.allowTopicOperationAsync(topicName, r, operation, authData)); } @Override @@ -302,7 +327,7 @@ public CompletableFuture allowTopicPolicyOperationAsync(TopicName topic PolicyName policyName, PolicyOperation policyOperation, AuthenticationDataSource authData) { - return authorize(authData, r -> super.allowTopicPolicyOperationAsync(topicName, r, policyName, policyOperation, - authData)); + return authorize(role, authData, + r -> super.allowTopicPolicyOperationAsync(topicName, r, policyName, policyOperation, authData)); } } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java index 3f6d38194713e..a39c3d0560760 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java @@ -20,6 +20,7 @@ import static java.util.Objects.requireNonNull; import static org.apache.commons.lang3.StringUtils.isNotBlank; +import com.google.common.collect.Sets; import java.io.IOException; import java.util.Collections; import java.util.HashMap; @@ -34,6 +35,7 @@ import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.AuthAction; +import org.apache.pulsar.common.policies.data.AuthPolicies; import org.apache.pulsar.common.policies.data.NamespaceOperation; import org.apache.pulsar.common.policies.data.PolicyName; import org.apache.pulsar.common.policies.data.PolicyOperation; @@ -249,6 +251,36 @@ public CompletableFuture grantPermissionAsync(TopicName topicName, Set revokePermissionAsync(TopicName topicName, String role) { + return getPoliciesReadOnlyAsync().thenCompose(readonly -> { + if (readonly) { + if (log.isDebugEnabled()) { + log.debug("Policies are read-only. Broker cannot do read-write operations"); + } + throw new IllegalStateException("policies are in readonly mode"); + } + return pulsarResources.getNamespaceResources() + .setPoliciesAsync(topicName.getNamespaceObject(), policies -> { + policies.auth_policies.getTopicAuthentication() + .computeIfPresent(topicName.toString(), (topicNameUri, roles) -> { + roles.remove(role); + if (roles.isEmpty()) { + return null; + } + return roles; + }); + return policies; + }).whenComplete((__, ex) -> { + if (ex != null) { + log.error("Failed to revoke permissions for role {} on topic {}", role, topicName, ex); + } else { + log.info("Successfully revoke permissions for role {} on topic {}", role, topicName); + } + }); + }); + } + @Override public CompletableFuture grantPermissionAsync(NamespaceName namespaceName, Set actions, String role, String authDataJson) { @@ -274,6 +306,29 @@ public CompletableFuture grantPermissionAsync(NamespaceName namespaceName, }); } + @Override + public CompletableFuture revokePermissionAsync(NamespaceName namespaceName, String role) { + return getPoliciesReadOnlyAsync().thenCompose(readonly -> { + if (readonly) { + if (log.isDebugEnabled()) { + log.debug("Policies are read-only. Broker cannot do read-write operations"); + } + throw new IllegalStateException("policies are in readonly mode"); + } + return pulsarResources.getNamespaceResources() + .setPoliciesAsync(namespaceName, policies -> { + policies.auth_policies.getNamespaceAuthentication().remove(role); + return policies; + }).whenComplete((__, ex) -> { + if (ex != null) { + log.error("Failed to revoke permissions for role {} namespace {}", role, namespaceName, ex); + } else { + log.info("Successfully revoke permissions for role {} namespace {}", role, namespaceName); + } + }); + }); + } + @Override public CompletableFuture grantSubscriptionPermissionAsync(NamespaceName namespace, String subscriptionName, Set roles, String authDataJson) { @@ -302,6 +357,9 @@ private CompletableFuture updateSubscriptionPermissionAsync(NamespaceName policies.auth_policies.getSubscriptionAuthentication().get(subscriptionName); if (subscriptionAuth != null) { subscriptionAuth.removeAll(roles); + if (subscriptionAuth.isEmpty()) { + policies.auth_policies.getSubscriptionAuthentication().remove(subscriptionName); + } } else { log.info("[{}] Couldn't find role {} while revoking for sub = {}", namespace, roles, subscriptionName); @@ -539,6 +597,7 @@ public CompletableFuture allowTopicOperationAsync(TopicName topicName, case COMPACT: case OFFLOAD: case UNLOAD: + case TRIM_TOPIC: case DELETE_METADATA: case UPDATE_METADATA: case ADD_BUNDLE_RANGE: @@ -588,4 +647,131 @@ public CompletableFuture validateTenantAdminAccess(String tenantName, S }); } + @Override + public CompletableFuture removePermissionsAsync(TopicName topicName) { + return getPoliciesReadOnlyAsync().thenCompose(readonly -> { + if (readonly) { + if (log.isDebugEnabled()) { + log.debug("Policies are read-only. Broker cannot do read-write operations"); + } + throw new IllegalStateException("policies are in readonly mode"); + } + return pulsarResources.getNamespaceResources().getPoliciesAsync(topicName.getNamespaceObject()) + .thenCompose(policies -> { + if (!policies.isPresent() + || !policies.get().auth_policies.getTopicAuthentication() + .containsKey(topicName.toString())) { + return CompletableFuture.completedFuture(null); + } + return pulsarResources.getNamespaceResources(). + setPoliciesAsync(topicName.getNamespaceObject(), policies2 -> { + policies2.auth_policies.getTopicAuthentication().remove(topicName.toString()); + return policies2; + }).whenComplete((__, ex) -> { + if (ex != null) { + log.error("Failed to remove permissions on topic {}", topicName, ex); + } else { + log.info("Successfully remove permissions on topic {}", topicName); + } + }); + }); + }); + } + + @Override + public CompletableFuture>> getPermissionsAsync(TopicName topicName) { + return getPoliciesReadOnlyAsync().thenCompose(readonly -> { + if (readonly) { + if (log.isDebugEnabled()) { + log.debug("Policies are read-only. Broker cannot do read-write operations"); + } + throw new IllegalStateException("policies are in readonly mode"); + } + return pulsarResources.getNamespaceResources().getPoliciesAsync(topicName.getNamespaceObject()) + .thenApply(policies -> { + if (!policies.isPresent()) { + throw new RestException(Response.Status.NOT_FOUND, "Namespace does not exist"); + } + Map> permissions = new HashMap<>(); + String topicUri = topicName.toString(); + AuthPolicies auth = policies.get().auth_policies; + // First add namespace level permissions + permissions.putAll(auth.getNamespaceAuthentication()); + // Then add topic level permissions + if (auth.getTopicAuthentication().containsKey(topicUri)) { + for (Map.Entry> entry : + auth.getTopicAuthentication().get(topicUri).entrySet()) { + String role = entry.getKey(); + Set topicPermissions = entry.getValue(); + + if (!permissions.containsKey(role)) { + permissions.put(role, topicPermissions); + } else { + // Do the union between namespace and topic level + Set union = Sets.union(permissions.get(role), topicPermissions); + permissions.put(role, union); + } + } + } + return permissions; + }).whenComplete((__, ex) -> { + if (ex != null) { + log.error("Failed to get permissions on topic {}", topicName, ex); + } else { + log.info("Successfully get permissions on topic {}", topicName); + } + }); + }); + } + + @Override + public CompletableFuture>> getSubscriptionPermissionsAsync(NamespaceName namespaceName) { + return getPoliciesReadOnlyAsync().thenCompose(readonly -> { + if (readonly) { + if (log.isDebugEnabled()) { + log.debug("Policies are read-only. Broker cannot do read-write operations"); + } + throw new IllegalStateException("policies are in readonly mode"); + } + return pulsarResources.getNamespaceResources().getPoliciesAsync(namespaceName) + .thenApply(policies -> { + if (!policies.isPresent()) { + throw new RestException(Response.Status.NOT_FOUND, "Namespace does not exist"); + } + + return policies.get().auth_policies.getSubscriptionAuthentication(); + }).whenComplete((__, ex) -> { + if (ex != null) { + log.error("Failed to get subscription permissions on namespace {}", namespaceName, ex); + } else { + log.info("Successfully get subscription permissions on namespaceName {}", namespaceName); + } + }); + }); + } + + @Override + public CompletableFuture>> getPermissionsAsync(NamespaceName namespaceName) { + return getPoliciesReadOnlyAsync().thenCompose(readonly -> { + if (readonly) { + if (log.isDebugEnabled()) { + log.debug("Policies are read-only. Broker cannot do read-write operations"); + } + throw new IllegalStateException("policies are in readonly mode"); + } + return pulsarResources.getNamespaceResources().getPoliciesAsync(namespaceName) + .thenApply(policies -> { + if (!policies.isPresent()) { + throw new RestException(Response.Status.NOT_FOUND, "Namespace does not exist"); + } + return policies.get().auth_policies.getNamespaceAuthentication(); + }).whenComplete((__, ex) -> { + if (ex != null) { + log.error("Failed to get permissions on namespaceName {}", namespaceName, ex); + } else { + log.info("Successfully get permissions on namespaceName {}", namespaceName); + } + }); + }); + } } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java index 42add4271f684..00e381e07292f 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java @@ -20,11 +20,16 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Joiner; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -78,6 +83,37 @@ protected CompletableFuture> getChildrenAsync(String path) { return cache.getChildren(path); } + protected CompletableFuture> getChildrenRecursiveAsync(String path) { + Set children = ConcurrentHashMap.newKeySet(); + CompletableFuture> result = new CompletableFuture<>(); + getChildrenRecursiveAsync(path, children, result, new AtomicInteger(1), path); + return result; + } + + private void getChildrenRecursiveAsync(String path, Set children, CompletableFuture> result, + AtomicInteger totalResults, String parent) { + cache.getChildren(path).thenAccept(childList -> { + childList = childList != null ? childList : Collections.emptyList(); + if (totalResults.decrementAndGet() == 0 && childList.isEmpty()) { + result.complete(new ArrayList<>(children)); + return; + } + if (childList.isEmpty()) { + return; + } + // remove current node from children if current node is not leaf + children.remove(parent); + // childPrefix creates a path hierarchy if children has multi level path + String childPrefix = path.equals(parent) ? "" : parent + "/"; + totalResults.addAndGet(childList.size()); + for (String child : childList) { + children.add(childPrefix + child); + String childPath = path + "/" + child; + getChildrenRecursiveAsync(childPath, children, result, totalResults, child); + } + }); + } + protected Optional get(String path) throws MetadataStoreException { try { return getAsync(path).get(operationTimeoutSec, TimeUnit.SECONDS); @@ -161,22 +197,21 @@ protected CompletableFuture deleteAsync(String path) { } protected CompletableFuture deleteIfExistsAsync(String path) { - return cache.exists(path).thenCompose(exists -> { - if (!exists) { - return CompletableFuture.completedFuture(null); + log.info("Deleting path: {}", path); + CompletableFuture future = new CompletableFuture<>(); + cache.delete(path).whenComplete((ignore, ex) -> { + if (ex != null && ex.getCause() instanceof MetadataStoreException.NotFoundException) { + log.info("Path {} did not exist in metadata store", path); + future.complete(null); + } else if (ex != null) { + log.info("Failed to delete path from metadata store: {}", path, ex); + future.completeExceptionally(ex); + } else { + log.info("Deleted path from metadata store: {}", path); + future.complete(null); } - CompletableFuture future = new CompletableFuture<>(); - cache.delete(path).whenComplete((ignore, ex) -> { - if (ex != null && ex.getCause() instanceof MetadataStoreException.NotFoundException) { - future.complete(null); - } else if (ex != null) { - future.completeExceptionally(ex); - } else { - future.complete(null); - } - }); - return future; }); + return future; } protected boolean exists(String path) throws MetadataStoreException { diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ClusterResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ClusterResources.java index 843cec7b20594..b0cc50edf1f1d 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ClusterResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ClusterResources.java @@ -29,6 +29,7 @@ import lombok.Getter; import org.apache.commons.collections4.CollectionUtils; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.ClusterPoliciesImpl; import org.apache.pulsar.common.policies.data.FailureDomainImpl; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.MetadataStore; @@ -39,10 +40,19 @@ public class ClusterResources extends BaseResources { @Getter private FailureDomainResources failureDomainResources; - - public ClusterResources(MetadataStore store, int operationTimeoutSec) { - super(store, ClusterData.class, operationTimeoutSec); - this.failureDomainResources = new FailureDomainResources(store, FailureDomainImpl.class, operationTimeoutSec); + @Getter + private ClusterPoliciesResources clusterPoliciesResources; + + public ClusterResources(MetadataStore localStore, MetadataStore configurationStore, int operationTimeoutSec) { + super(configurationStore, ClusterData.class, operationTimeoutSec); + this.failureDomainResources = new FailureDomainResources(configurationStore, FailureDomainImpl.class, + operationTimeoutSec); + if (localStore != null) { + this.clusterPoliciesResources = new ClusterPoliciesResources(localStore, ClusterPoliciesImpl.class, + operationTimeoutSec); + } else { + this.clusterPoliciesResources = null; + } } public CompletableFuture> listAsync() { @@ -216,4 +226,26 @@ public void registerListener(Consumer listener) { }); } } + + public static class ClusterPoliciesResources extends BaseResources { + public static final String LOCAL_POLICIES_PATH = "policies"; + + public ClusterPoliciesResources(MetadataStore store, Class clazz, + int operationTimeoutSec) { + super(store, clazz, operationTimeoutSec); + } + + public Optional getClusterPolicies(String clusterName) throws MetadataStoreException { + return get(joinPath(BASE_CLUSTERS_PATH, clusterName, LOCAL_POLICIES_PATH)); + } + + public CompletableFuture> getClusterPoliciesAsync(String clusterName) { + return getAsync(joinPath(BASE_CLUSTERS_PATH, clusterName, LOCAL_POLICIES_PATH)); + } + + public CompletableFuture setPoliciesWithCreateAsync(String clusterName, + Function, ClusterPoliciesImpl> createFunction) { + return setWithCreateAsync(joinPath(BASE_CLUSTERS_PATH, clusterName, LOCAL_POLICIES_PATH), createFunction); + } + } } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LoadBalanceResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LoadBalanceResources.java new file mode 100644 index 0000000000000..57a2d16e4e89c --- /dev/null +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LoadBalanceResources.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.resources; + +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import lombok.Getter; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.policies.data.ResourceQuota; +import org.apache.pulsar.metadata.api.MetadataStore; +import org.apache.pulsar.policies.data.loadbalancer.BundleData; +import org.apache.pulsar.policies.data.loadbalancer.TimeAverageBrokerData; + +@Getter +public class LoadBalanceResources { + public static final String BUNDLE_DATA_BASE_PATH = "/loadbalance/bundle-data"; + public static final String BROKER_TIME_AVERAGE_BASE_PATH = "/loadbalance/broker-time-average"; + public static final String RESOURCE_QUOTA_BASE_PATH = "/loadbalance/resource-quota"; + + private final BundleDataResources bundleDataResources; + private final BrokerTimeAverageDataResources brokerTimeAverageDataResources; + private final QuotaResources quotaResources; + + public LoadBalanceResources(MetadataStore store, int operationTimeoutSec) { + bundleDataResources = new BundleDataResources(store, operationTimeoutSec); + brokerTimeAverageDataResources = new BrokerTimeAverageDataResources(store, operationTimeoutSec); + quotaResources = new QuotaResources(store, operationTimeoutSec); + } + + public static class BundleDataResources extends BaseResources { + public BundleDataResources(MetadataStore store, int operationTimeoutSec) { + super(store, BundleData.class, operationTimeoutSec); + } + + public CompletableFuture> getBundleData(String bundle) { + return getAsync(getBundleDataPath(bundle)); + } + + public CompletableFuture updateBundleData(String bundle, BundleData data) { + return setWithCreateAsync(getBundleDataPath(bundle), __ -> data); + } + + public CompletableFuture deleteBundleData(String bundle) { + return deleteAsync(getBundleDataPath(bundle)); + } + + // clear resource of `/loadbalance/bundle-data/{tenant}/{namespace}/` in metadata-store + public CompletableFuture deleteBundleDataAsync(NamespaceName ns) { + final String namespaceBundlePath = joinPath(BUNDLE_DATA_BASE_PATH, ns.toString()); + return getStore().deleteRecursive(namespaceBundlePath); + } + + // clear resource of `/loadbalance/bundle-data/{tenant}/` in metadata-store + public CompletableFuture deleteBundleDataTenantAsync(String tenant) { + final String tenantBundlePath = joinPath(BUNDLE_DATA_BASE_PATH, tenant); + return getStore().deleteRecursive(tenantBundlePath); + } + + // Get the metadata store path for the given bundle full name. + private String getBundleDataPath(final String bundle) { + return BUNDLE_DATA_BASE_PATH + "/" + bundle; + } + } + + public static class BrokerTimeAverageDataResources extends BaseResources { + public BrokerTimeAverageDataResources(MetadataStore store, int operationTimeoutSec) { + super(store, TimeAverageBrokerData.class, operationTimeoutSec); + } + + public CompletableFuture updateTimeAverageBrokerData(String brokerLookupAddress, + TimeAverageBrokerData data) { + return setWithCreateAsync(getTimeAverageBrokerDataPath(brokerLookupAddress), __ -> data); + } + + public CompletableFuture deleteTimeAverageBrokerData(String brokerLookupAddress) { + return deleteAsync(getTimeAverageBrokerDataPath(brokerLookupAddress)); + } + + private String getTimeAverageBrokerDataPath(final String brokerLookupAddress) { + return BROKER_TIME_AVERAGE_BASE_PATH + "/" + brokerLookupAddress; + } + } + + public static class QuotaResources extends BaseResources { + public QuotaResources(MetadataStore store, int operationTimeoutSec) { + super(store, ResourceQuota.class, operationTimeoutSec); + } + + public CompletableFuture> getQuota(String bundle) { + return getAsync(getBundleQuotaPath(bundle)); + } + + public CompletableFuture> getDefaultQuota() { + return getAsync(getDefaultBundleQuotaPath()); + } + + public CompletableFuture setWithCreateQuotaAsync(String bundle, ResourceQuota quota) { + return setWithCreateAsync(getBundleQuotaPath(bundle), __ -> quota); + } + + public CompletableFuture setWithCreateDefaultQuotaAsync(ResourceQuota quota) { + return setWithCreateAsync(getDefaultBundleQuotaPath(), __ -> quota); + } + + public CompletableFuture deleteQuota(String bundle) { + return deleteAsync(getBundleQuotaPath(bundle)); + } + + private String getBundleQuotaPath(String bundle) { + return String.format("%s/%s", RESOURCE_QUOTA_BASE_PATH, bundle); + } + + private String getDefaultBundleQuotaPath() { + return getBundleQuotaPath("default"); + } + } +} diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LocalPoliciesResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LocalPoliciesResources.java index c6b658c3bd025..ae3479fde59b8 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LocalPoliciesResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LocalPoliciesResources.java @@ -79,7 +79,7 @@ public void deleteLocalPolicies(NamespaceName ns) throws MetadataStoreException } public CompletableFuture deleteLocalPoliciesAsync(NamespaceName ns) { - return deleteAsync(joinPath(LOCAL_POLICIES_ROOT, ns.toString())); + return deleteIfExistsAsync(joinPath(LOCAL_POLICIES_ROOT, ns.toString())); } public CompletableFuture deleteLocalPoliciesTenantAsync(String tenant) { diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java index 48f8259656729..9d7c60cd34453 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java @@ -24,6 +24,8 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.ForkJoinPool; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.function.Supplier; @@ -52,17 +54,20 @@ public class NamespaceResources extends BaseResources { public static final String POLICIES_READONLY_FLAG_PATH = "/admin/flags/policies-readonly"; private static final String NAMESPACE_BASE_PATH = "/namespace"; - private static final String BUNDLE_DATA_BASE_PATH = "/loadbalance/bundle-data"; public NamespaceResources(MetadataStore configurationStore, int operationTimeoutSec) { + this(configurationStore, operationTimeoutSec, ForkJoinPool.commonPool()); + } + + public NamespaceResources(MetadataStore configurationStore, int operationTimeoutSec, Executor executor) { super(configurationStore, Policies.class, operationTimeoutSec); this.configurationStore = configurationStore; isolationPolicies = new IsolationPolicyResources(configurationStore, operationTimeoutSec); - partitionedTopicResources = new PartitionedTopicResources(configurationStore, operationTimeoutSec); + partitionedTopicResources = new PartitionedTopicResources(configurationStore, operationTimeoutSec, executor); } public CompletableFuture> listNamespacesAsync(String tenant) { - return getChildrenAsync(joinPath(BASE_POLICIES_PATH, tenant)); + return getChildrenRecursiveAsync(joinPath(BASE_POLICIES_PATH, tenant)); } public CompletableFuture getPoliciesReadOnlyAsync() { @@ -110,13 +115,20 @@ public void deletePolicies(NamespaceName ns) throws MetadataStoreException{ } public CompletableFuture deletePoliciesAsync(NamespaceName ns){ - return deleteAsync(joinPath(BASE_POLICIES_PATH, ns.toString())); + return deleteIfExistsAsync(joinPath(BASE_POLICIES_PATH, ns.toString())); } public Optional getPolicies(NamespaceName ns) throws MetadataStoreException{ return get(joinPath(BASE_POLICIES_PATH, ns.toString())); } + /** + * Get the namespace policy from the metadata cache. This method will not trigger the load of metadata cache. + * + * @deprecated Since this method may introduce inconsistent namespace policies. we should use + * #{@link NamespaceResources#getPoliciesAsync} + */ + @Deprecated public Optional getPoliciesIfCached(NamespaceName ns) { return getCache().getIfCached(joinPath(BASE_POLICIES_PATH, ns.toString())); } @@ -143,10 +155,18 @@ public static boolean pathIsNamespaceLocalPolicies(String path) { && path.substring(LOCAL_POLICIES_ROOT.length() + 1).contains("/"); } - // clear resource of `/namespace/{namespaceName}` for zk-node + /** + * Clear resource of `/namespace/{namespaceName}` for zk-node. + * @param ns the namespace name + * @return a handle to the results of the operation + * */ + // public CompletableFuture deleteNamespaceAsync(NamespaceName ns) { final String namespacePath = joinPath(NAMESPACE_BASE_PATH, ns.toString()); - return deleteIfExistsAsync(namespacePath); + // please beware that this will delete all the children of the namespace + // including the ownership nodes (ephemeral nodes) + // see ServiceUnitUtils.path(ns) for the ownership node path + return getStore().deleteRecursive(namespacePath); } // clear resource of `/namespace/{tenant}` for zk-node @@ -228,9 +248,11 @@ public void setIsolationDataWithCreate(String cluster, public static class PartitionedTopicResources extends BaseResources { private static final String PARTITIONED_TOPIC_PATH = "/admin/partitioned-topics"; + private final Executor executor; - public PartitionedTopicResources(MetadataStore configurationStore, int operationTimeoutSec) { + public PartitionedTopicResources(MetadataStore configurationStore, int operationTimeoutSec, Executor executor) { super(configurationStore, PartitionedTopicMetadata.class, operationTimeoutSec); + this.executor = executor; } public CompletableFuture updatePartitionedTopicAsync(TopicName tn, Function deletePartitionedTopicAsync(TopicName tn) { public CompletableFuture clearPartitionedTopicMetadataAsync(NamespaceName namespaceName) { final String globalPartitionedPath = joinPath(PARTITIONED_TOPIC_PATH, namespaceName.toString()); + log.info("Clearing partitioned topic metadata for namespace {}, path is {}", + namespaceName, globalPartitionedPath); return getStore().deleteRecursive(globalPartitionedPath); } public CompletableFuture clearPartitionedTopicTenantAsync(String tenant) { final String partitionedTopicPath = joinPath(PARTITIONED_TOPIC_PATH, tenant); + log.info("Clearing partitioned topic metadata for tenant {}, path is {}", tenant, partitionedTopicPath); return deleteIfExistsAsync(partitionedTopicPath); } @@ -365,22 +390,9 @@ public CompletableFuture runWithMarkDeleteAsync(TopicName topic, future.complete(deleteResult); } }); - }); + }, executor); return future; } } - - // clear resource of `/loadbalance/bundle-data/{tenant}/{namespace}/` in metadata-store - public CompletableFuture deleteBundleDataAsync(NamespaceName ns) { - final String namespaceBundlePath = joinPath(BUNDLE_DATA_BASE_PATH, ns.toString()); - return getStore().deleteRecursive(namespaceBundlePath); - } - - // clear resource of `/loadbalance/bundle-data/{tenant}/` in metadata-store - public CompletableFuture deleteBundleDataTenantAsync(String tenant) { - final String tenantBundlePath = joinPath(BUNDLE_DATA_BASE_PATH, tenant); - return getStore().deleteRecursive(tenantBundlePath); - } - } \ No newline at end of file diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/PulsarResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/PulsarResources.java index dfcd0a4194ff5..cc64eeb52f6eb 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/PulsarResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/PulsarResources.java @@ -19,6 +19,8 @@ package org.apache.pulsar.broker.resources; import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.concurrent.ForkJoinPool; import lombok.Getter; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.MetadataStoreConfig; @@ -48,6 +50,8 @@ public class PulsarResources { @Getter private final TopicResources topicResources; @Getter + private final LoadBalanceResources loadBalanceResources; + @Getter private final Optional localMetadataStore; @Getter private final Optional configurationMetadataStore; @@ -55,12 +59,19 @@ public class PulsarResources { public PulsarResources(MetadataStore localMetadataStore, MetadataStore configurationMetadataStore) { this(localMetadataStore, configurationMetadataStore, DEFAULT_OPERATION_TIMEOUT_SEC); } + + public PulsarResources(MetadataStore localMetadataStore, MetadataStore configurationMetadataStore, + int operationTimeoutSec) { + this(localMetadataStore, configurationMetadataStore, operationTimeoutSec, ForkJoinPool.commonPool()); + } + public PulsarResources(MetadataStore localMetadataStore, MetadataStore configurationMetadataStore, - int operationTimeoutSec) { + int operationTimeoutSec, Executor executor) { if (configurationMetadataStore != null) { tenantResources = new TenantResources(configurationMetadataStore, operationTimeoutSec); - clusterResources = new ClusterResources(configurationMetadataStore, operationTimeoutSec); - namespaceResources = new NamespaceResources(configurationMetadataStore, operationTimeoutSec); + clusterResources = new ClusterResources(localMetadataStore, configurationMetadataStore, + operationTimeoutSec); + namespaceResources = new NamespaceResources(configurationMetadataStore, operationTimeoutSec, executor); resourcegroupResources = new ResourceGroupResources(configurationMetadataStore, operationTimeoutSec); } else { tenantResources = null; @@ -75,12 +86,14 @@ public PulsarResources(MetadataStore localMetadataStore, MetadataStore configura loadReportResources = new LoadManagerReportResources(localMetadataStore, operationTimeoutSec); bookieResources = new BookieResources(localMetadataStore, operationTimeoutSec); topicResources = new TopicResources(localMetadataStore); + loadBalanceResources = new LoadBalanceResources(localMetadataStore, operationTimeoutSec); } else { dynamicConfigResources = null; localPolicies = null; loadReportResources = null; bookieResources = null; topicResources = null; + loadBalanceResources = null; } this.localMetadataStore = Optional.ofNullable(localMetadataStore); diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TopicResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TopicResources.java index 840ced0a1c1c4..f607da76b3c11 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TopicResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TopicResources.java @@ -50,6 +50,9 @@ public TopicResources(MetadataStore store) { store.registerListener(this::handleNotification); } + /*** + * List persistent topics names under a namespace, the topic name contains the partition suffix. + */ public CompletableFuture> listPersistentTopicsAsync(NamespaceName ns) { String path = MANAGED_LEDGER_PATH + "/" + ns + "/persistent"; @@ -72,11 +75,6 @@ public CompletableFuture> getExistingPartitions(NamespaceName ns, T ); } - public CompletableFuture deletePersistentTopicAsync(TopicName topic) { - String path = MANAGED_LEDGER_PATH + "/" + topic.getPersistenceNamingEncoding(); - return store.delete(path, Optional.of(-1L)); - } - public CompletableFuture createPersistentTopicAsync(TopicName topic) { String path = MANAGED_LEDGER_PATH + "/" + topic.getPersistenceNamingEncoding(); return store.put(path, new byte[0], Optional.of(-1L)) @@ -90,38 +88,20 @@ public CompletableFuture persistentTopicExists(TopicName topic) { public CompletableFuture clearNamespacePersistence(NamespaceName ns) { String path = MANAGED_LEDGER_PATH + "/" + ns; - return store.exists(path) - .thenCompose(exists -> { - if (exists) { - return store.delete(path, Optional.empty()); - } else { - return CompletableFuture.completedFuture(null); - } - }); + log.info("Clearing namespace persistence for namespace: {}, path {}", ns, path); + return store.deleteIfExists(path, Optional.empty()); } public CompletableFuture clearDomainPersistence(NamespaceName ns) { String path = MANAGED_LEDGER_PATH + "/" + ns + "/persistent"; - return store.exists(path) - .thenCompose(exists -> { - if (exists) { - return store.delete(path, Optional.empty()); - } else { - return CompletableFuture.completedFuture(null); - } - }); + log.info("Clearing domain persistence for namespace: {}, path {}", ns, path); + return store.deleteIfExists(path, Optional.empty()); } public CompletableFuture clearTenantPersistence(String tenant) { String path = MANAGED_LEDGER_PATH + "/" + tenant; - return store.exists(path) - .thenCompose(exists -> { - if (exists) { - return store.delete(path, Optional.empty()); - } else { - return CompletableFuture.completedFuture(null); - } - }); + log.info("Clearing tenant persistence for tenant: {}, path {}", tenant, path); + return store.deleteRecursive(path); } void handleNotification(Notification notification) { diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtils.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtils.java index 828d9871bb3de..077d5280b5102 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtils.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtils.java @@ -76,7 +76,7 @@ public static void generateSystemMetrics(SimpleTextOutputStream stream, String c } for (int j = 0; j < sample.labelNames.size(); j++) { String labelValue = sample.labelValues.get(j); - if (labelValue != null) { + if (labelValue != null && labelValue.indexOf('"') > -1) { labelValue = labelValue.replace("\"", "\\\""); } if (j > 0) { diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java index 64d1fcdab6f14..8685348174cd6 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java @@ -25,9 +25,13 @@ import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import javax.servlet.AsyncContext; +import javax.servlet.AsyncEvent; +import javax.servlet.AsyncListener; import javax.servlet.ServletException; -import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -35,67 +39,133 @@ import org.slf4j.LoggerFactory; public class PrometheusMetricsServlet extends HttpServlet { - + public static final String DEFAULT_METRICS_PATH = "/metrics"; private static final long serialVersionUID = 1L; - private static final int HTTP_STATUS_OK_200 = 200; - private static final int HTTP_STATUS_INTERNAL_SERVER_ERROR_500 = 500; - - private final long metricsServletTimeoutMs; - private final String cluster; + static final int HTTP_STATUS_OK_200 = 200; + static final int HTTP_STATUS_INTERNAL_SERVER_ERROR_500 = 500; + protected final long metricsServletTimeoutMs; + protected final String cluster; protected List metricsProviders; - private ExecutorService executor = null; + protected ExecutorService executor = null; + protected final int executorMaxThreads; public PrometheusMetricsServlet(long metricsServletTimeoutMs, String cluster) { + this(metricsServletTimeoutMs, cluster, 1); + } + + public PrometheusMetricsServlet(long metricsServletTimeoutMs, String cluster, int executorMaxThreads) { this.metricsServletTimeoutMs = metricsServletTimeoutMs; this.cluster = cluster; + this.executorMaxThreads = executorMaxThreads; } @Override public void init() throws ServletException { - executor = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("prometheus-stats")); + if (executorMaxThreads > 0) { + executor = + Executors.newScheduledThreadPool(executorMaxThreads, new DefaultThreadFactory("prometheus-stats")); + } } @Override protected void doGet(HttpServletRequest request, HttpServletResponse response) { AsyncContext context = request.startAsync(); - context.setTimeout(metricsServletTimeoutMs); - executor.execute(() -> { - long start = System.currentTimeMillis(); - HttpServletResponse res = (HttpServletResponse) context.getResponse(); - try { - res.setStatus(HTTP_STATUS_OK_200); - res.setContentType("text/plain;charset=utf-8"); - generateMetrics(cluster, res.getOutputStream()); - } catch (Exception e) { - long end = System.currentTimeMillis(); - long time = end - start; - if (e instanceof EOFException) { - // NO STACKTRACE - log.error("Failed to send metrics, " - + "likely the client or this server closed " - + "the connection due to a timeout ({} ms elapsed): {}", time, e + ""); - } else { - log.error("Failed to generate prometheus stats, {} ms elapsed", time, e); + // set hard timeout to 2 * timeout + if (metricsServletTimeoutMs > 0) { + context.setTimeout(metricsServletTimeoutMs * 2); + } + long startNanos = System.nanoTime(); + AtomicBoolean taskStarted = new AtomicBoolean(false); + Future future = executor.submit(() -> { + taskStarted.set(true); + long elapsedNanos = System.nanoTime() - startNanos; + // check if the request has been timed out, implement a soft timeout + // so that response writing can continue to up to 2 * timeout + if (metricsServletTimeoutMs > 0 && elapsedNanos > TimeUnit.MILLISECONDS.toNanos(metricsServletTimeoutMs)) { + log.warn("Prometheus metrics request was too long in queue ({}ms). Skipping sending metrics.", + TimeUnit.NANOSECONDS.toMillis(elapsedNanos)); + if (!response.isCommitted()) { + response.setStatus(HTTP_STATUS_INTERNAL_SERVER_ERROR_500); } - res.setStatus(HTTP_STATUS_INTERNAL_SERVER_ERROR_500); - } finally { - long end = System.currentTimeMillis(); - long time = end - start; - try { - context.complete(); - } catch (IllegalStateException e) { - // this happens when metricsServletTimeoutMs expires - // java.lang.IllegalStateException: AsyncContext completed and/or Request lifecycle recycled - log.error("Failed to generate prometheus stats, " - + "this is likely due to metricsServletTimeoutMs: {} ms elapsed: {}", time, e + ""); + context.complete(); + return; + } + handleAsyncMetricsRequest(context); + }); + context.addListener(new AsyncListener() { + @Override + public void onComplete(AsyncEvent asyncEvent) throws IOException { + if (!taskStarted.get()) { + future.cancel(false); } } + + @Override + public void onTimeout(AsyncEvent asyncEvent) throws IOException { + if (!taskStarted.get()) { + future.cancel(false); + } + log.warn("Prometheus metrics request timed out"); + HttpServletResponse res = (HttpServletResponse) context.getResponse(); + if (!res.isCommitted()) { + res.setStatus(HTTP_STATUS_INTERNAL_SERVER_ERROR_500); + } + context.complete(); + } + + @Override + public void onError(AsyncEvent asyncEvent) throws IOException { + if (!taskStarted.get()) { + future.cancel(false); + } + } + + @Override + public void onStartAsync(AsyncEvent asyncEvent) throws IOException { + + } }); + + } + + private void handleAsyncMetricsRequest(AsyncContext context) { + long start = System.currentTimeMillis(); + HttpServletResponse res = (HttpServletResponse) context.getResponse(); + try { + generateMetricsSynchronously(res); + } catch (Exception e) { + long end = System.currentTimeMillis(); + long time = end - start; + if (e instanceof EOFException) { + // NO STACKTRACE + log.error("Failed to send metrics, " + + "likely the client or this server closed " + + "the connection due to a timeout ({} ms elapsed): {}", time, e + ""); + } else { + log.error("Failed to generate prometheus stats, {} ms elapsed", time, e); + } + if (!res.isCommitted()) { + res.setStatus(HTTP_STATUS_INTERNAL_SERVER_ERROR_500); + } + } finally { + long end = System.currentTimeMillis(); + long time = end - start; + try { + context.complete(); + } catch (IllegalStateException e) { + // this happens when metricsServletTimeoutMs expires + // java.lang.IllegalStateException: AsyncContext completed and/or Request lifecycle recycled + log.error("Failed to generate prometheus stats, " + + "this is likely due to metricsServletTimeoutMs: {} ms elapsed: {}", time, e + ""); + } + } } - protected void generateMetrics(String cluster, ServletOutputStream outputStream) throws IOException { - PrometheusMetricsGeneratorUtils.generate(cluster, outputStream, metricsProviders); + private void generateMetricsSynchronously(HttpServletResponse res) throws IOException { + res.setStatus(HTTP_STATUS_OK_200); + res.setContentType("text/plain;charset=utf-8"); + PrometheusMetricsGeneratorUtils.generate(cluster, res.getOutputStream(), metricsProviders); } @Override diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/AuthenticationFilter.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/AuthenticationFilter.java index 6f13185ca7540..3b85d9b03e4e6 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/AuthenticationFilter.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/AuthenticationFilter.java @@ -50,22 +50,27 @@ public AuthenticationFilter(AuthenticationService authenticationService) { } @Override - public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) - throws IOException, ServletException { + public void doFilter( + ServletRequest request, ServletResponse response, FilterChain chain + ) throws IOException, ServletException { + final HttpServletRequest httpRequest = (HttpServletRequest) request; + final HttpServletResponse httpResponse = (HttpServletResponse) response; + + final boolean doFilter; try { - boolean doFilter = authenticationService - .authenticateHttpRequest((HttpServletRequest) request, (HttpServletResponse) response); - if (doFilter) { - chain.doFilter(request, response); - } + doFilter = authenticationService.authenticateHttpRequest(httpRequest, httpResponse); } catch (Exception e) { - HttpServletResponse httpResponse = (HttpServletResponse) response; httpResponse.sendError(HttpServletResponse.SC_UNAUTHORIZED, "Authentication required"); if (e instanceof AuthenticationException) { LOG.warn("[{}] Failed to authenticate HTTP request: {}", request.getRemoteAddr(), e.getMessage()); } else { LOG.error("[{}] Error performing authentication for HTTP", request.getRemoteAddr(), e); } + return; + } + + if (doFilter) { + chain.doFilter(request, response); } } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/GzipHandlerUtil.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/GzipHandlerUtil.java new file mode 100644 index 0000000000000..9e980cecb791f --- /dev/null +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/GzipHandlerUtil.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.web; + +import java.util.List; +import org.eclipse.jetty.http.pathmap.PathSpecSet; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.handler.gzip.GzipHandler; +import org.eclipse.jetty.util.IncludeExclude; + +public class GzipHandlerUtil { + public static Handler wrapWithGzipHandler(Handler innerHandler, List gzipCompressionExcludedPaths) { + Handler wrappedHandler; + if (isGzipCompressionCompletelyDisabled(gzipCompressionExcludedPaths)) { + // no need to add GZIP handler if it's disabled by setting the excluded path to "^.*" or "^.*$" + wrappedHandler = innerHandler; + } else { + // add GZIP handler which is active when the request contains "Accept-Encoding: gzip" header + GzipHandler gzipHandler = new GzipHandler(); + gzipHandler.setHandler(innerHandler); + if (gzipCompressionExcludedPaths != null && gzipCompressionExcludedPaths.size() > 0) { + gzipHandler.setExcludedPaths(gzipCompressionExcludedPaths.toArray(new String[0])); + } + wrappedHandler = gzipHandler; + } + return wrappedHandler; + } + + public static boolean isGzipCompressionCompletelyDisabled(List gzipCompressionExcludedPaths) { + return gzipCompressionExcludedPaths != null && gzipCompressionExcludedPaths.size() == 1 + && (gzipCompressionExcludedPaths.get(0).equals("^.*") + || gzipCompressionExcludedPaths.get(0).equals("^.*$")); + } + + /** + * Check if GZIP compression is enabled for the given endpoint. + * @param gzipCompressionExcludedPaths list of paths that should not be compressed + * @param endpoint the endpoint to check + * @return true if GZIP compression is enabled for the endpoint, false otherwise + */ + public static boolean isGzipCompressionEnabledForEndpoint(List gzipCompressionExcludedPaths, + String endpoint) { + if (gzipCompressionExcludedPaths == null || gzipCompressionExcludedPaths.isEmpty()) { + return true; + } + if (isGzipCompressionCompletelyDisabled(gzipCompressionExcludedPaths)) { + return false; + } + IncludeExclude paths = new IncludeExclude<>(PathSpecSet.class); + paths.exclude(gzipCompressionExcludedPaths.toArray(new String[0])); + return paths.test(endpoint); + } +} diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/JettyRequestLogFactory.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/JettyRequestLogFactory.java index e5daa5852b51f..fc88647eb49ea 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/JettyRequestLogFactory.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/JettyRequestLogFactory.java @@ -18,9 +18,23 @@ */ package org.apache.pulsar.broker.web; +import java.net.InetSocketAddress; import java.util.TimeZone; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.extern.slf4j.Slf4j; +import org.eclipse.jetty.io.Connection; +import org.eclipse.jetty.io.EndPoint; +import org.eclipse.jetty.server.Connector; import org.eclipse.jetty.server.CustomRequestLog; +import org.eclipse.jetty.server.ProxyConnectionFactory; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.RequestLog; +import org.eclipse.jetty.server.Response; +import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.Slf4jRequestLogWriter; +import org.eclipse.jetty.util.HostPort; +import org.eclipse.jetty.util.component.ContainerLifeCycle; /** * Class to standardize initialization of a Jetty request logger for all pulsar components. @@ -58,7 +72,184 @@ public class JettyRequestLogFactory { * Build a new Jetty request logger using the format defined in this class. * @return a request logger */ - public static CustomRequestLog createRequestLogger() { - return new CustomRequestLog(new Slf4jRequestLogWriter(), LOG_FORMAT); + public static RequestLog createRequestLogger() { + return createRequestLogger(false, null); + } + + /** + * Build a new Jetty request logger using the format defined in this class. + * @param showDetailedAddresses whether to show detailed addresses and ports in logs + * @return a request logger + */ + public static RequestLog createRequestLogger(boolean showDetailedAddresses, Server server) { + if (!showDetailedAddresses) { + return new CustomRequestLog(new Slf4jRequestLogWriter(), LOG_FORMAT); + } else { + return new OriginalClientIPRequestLog(server); + } + } + + /** + * Logs the original and real remote (client) and local (server) IP addresses + * when detailed addresses are enabled. + * Tracks the real addresses of remote and local using a registered Connection.Listener + * when detailed addresses are enabled. + * This is necessary when Proxy Protocol is used to pass the original client IP. + */ + @Slf4j + private static class OriginalClientIPRequestLog extends ContainerLifeCycle implements RequestLog { + private final ThreadLocal requestLogStringBuilder = ThreadLocal.withInitial(StringBuilder::new); + private final CustomRequestLog delegate; + private final Slf4jRequestLogWriter delegateLogWriter; + + OriginalClientIPRequestLog(Server server) { + delegate = new CustomRequestLog(this::write, LOG_FORMAT); + addBean(delegate); + delegateLogWriter = new Slf4jRequestLogWriter(); + addBean(delegateLogWriter); + if (server != null) { + for (Connector connector : server.getConnectors()) { + // adding the listener is only necessary for connectors that use ProxyConnectionFactory + if (connector.getDefaultConnectionFactory() instanceof ProxyConnectionFactory) { + connector.addBean(proxyProtocolOriginalEndpointListener); + } + } + } + } + + void write(String requestEntry) { + StringBuilder sb = requestLogStringBuilder.get(); + sb.setLength(0); + sb.append(requestEntry); + } + + @Override + public void log(Request request, Response response) { + delegate.log(request, response); + StringBuilder sb = requestLogStringBuilder.get(); + sb.append(" [R:"); + sb.append(request.getRemoteHost()); + sb.append(':'); + sb.append(request.getRemotePort()); + InetSocketAddress realRemoteAddress = lookupRealAddress(request.getHttpChannel().getRemoteAddress()); + if (realRemoteAddress != null) { + String realRemoteHost = HostPort.normalizeHost(realRemoteAddress.getHostString()); + int realRemotePort = realRemoteAddress.getPort(); + if (!realRemoteHost.equals(request.getRemoteHost()) || realRemotePort != request.getRemotePort()) { + sb.append(" via "); + sb.append(realRemoteHost); + sb.append(':'); + sb.append(realRemotePort); + } + } + sb.append("]->[L:"); + InetSocketAddress realLocalAddress = lookupRealAddress(request.getHttpChannel().getLocalAddress()); + if (realLocalAddress != null) { + String realLocalHost = HostPort.normalizeHost(realLocalAddress.getHostString()); + int realLocalPort = realLocalAddress.getPort(); + sb.append(realLocalHost); + sb.append(':'); + sb.append(realLocalPort); + if (!realLocalHost.equals(request.getLocalAddr()) || realLocalPort != request.getLocalPort()) { + sb.append(" dst "); + sb.append(request.getLocalAddr()); + sb.append(':'); + sb.append(request.getLocalPort()); + } + } else { + sb.append(request.getLocalAddr()); + sb.append(':'); + sb.append(request.getLocalPort()); + } + sb.append(']'); + try { + delegateLogWriter.write(sb.toString()); + } catch (Exception e) { + log.warn("Failed to write request log", e); + } + } + + private InetSocketAddress lookupRealAddress(InetSocketAddress socketAddress) { + if (socketAddress == null) { + return null; + } + if (proxyProtocolRealAddressMapping.isEmpty()) { + return socketAddress; + } + AddressEntry entry = proxyProtocolRealAddressMapping.get(new AddressKey(socketAddress.getHostString(), + socketAddress.getPort())); + if (entry != null) { + return entry.realAddress; + } else { + return socketAddress; + } + } + + private final Connection.Listener proxyProtocolOriginalEndpointListener = + new ProxyProtocolOriginalEndpointListener(); + + private final ConcurrentHashMap proxyProtocolRealAddressMapping = + new ConcurrentHashMap<>(); + + // Use a record as key since InetSocketAddress hash code changes if the address gets resolved + record AddressKey(String hostString, int port) { + + } + + record AddressEntry(InetSocketAddress realAddress, AtomicInteger referenceCount) { + + } + + // Tracks the real addresses of remote and local when detailed addresses are enabled. + // This is necessary when Proxy Protocol is used to pass the original client IP. + // The Proxy Protocol implementation in Jetty wraps the original endpoint with a ProxyEndPoint + // and the real endpoint information isn't available in the request object. + // This listener is added to all connectors to track the real addresses of the client and server. + class ProxyProtocolOriginalEndpointListener implements Connection.Listener { + @Override + public void onOpened(Connection connection) { + handleConnection(connection, true); + } + + @Override + public void onClosed(Connection connection) { + handleConnection(connection, false); + } + + private void handleConnection(Connection connection, boolean increment) { + if (connection.getEndPoint() instanceof ProxyConnectionFactory.ProxyEndPoint) { + ProxyConnectionFactory.ProxyEndPoint proxyEndPoint = + (ProxyConnectionFactory.ProxyEndPoint) connection.getEndPoint(); + EndPoint originalEndpoint = proxyEndPoint.unwrap(); + mapAddress(proxyEndPoint.getLocalAddress(), originalEndpoint.getLocalAddress(), increment); + mapAddress(proxyEndPoint.getRemoteAddress(), originalEndpoint.getRemoteAddress(), increment); + } + } + + private void mapAddress(InetSocketAddress current, InetSocketAddress real, boolean increment) { + // don't add the mapping if the current address is the same as the real address + if (real != null && current != null && current.equals(real)) { + return; + } + AddressKey key = new AddressKey(current.getHostString(), current.getPort()); + proxyProtocolRealAddressMapping.compute(key, (__, entry) -> { + if (entry == null) { + if (increment) { + entry = new AddressEntry(real, new AtomicInteger(1)); + } + } else { + if (increment) { + entry.referenceCount.incrementAndGet(); + } else { + if (entry.referenceCount.decrementAndGet() == 0) { + // remove the entry if the reference count drops to 0 + entry = null; + } + } + } + return entry; + }); + } + } } } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/RateLimitingFilter.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/RateLimitingFilter.java index 502b691fa34b0..0618df6609c49 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/RateLimitingFilter.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/RateLimitingFilter.java @@ -19,6 +19,10 @@ package org.apache.pulsar.broker.web; import com.google.common.util.concurrent.RateLimiter; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.Meter; import io.prometheus.client.Counter; import java.io.IOException; import javax.servlet.Filter; @@ -33,15 +37,32 @@ public class RateLimitingFilter implements Filter { private final RateLimiter limiter; - public RateLimitingFilter(double rateLimit) { - limiter = RateLimiter.create(rateLimit); + public static final String RATE_LIMIT_REQUEST_COUNT_METRIC_NAME = + "pulsar.web.filter.rate_limit.request.count"; + private final LongCounter rateLimitRequestCounter; + + public static final AttributeKey RATE_LIMIT_RESULT = + AttributeKey.stringKey("pulsar.web.filter.rate_limit.result"); + public enum Result { + ACCEPTED, + REJECTED; + public final Attributes attributes = Attributes.of(RATE_LIMIT_RESULT, name().toLowerCase()); } + @Deprecated private static final Counter httpRejectedRequests = Counter.build() .name("pulsar_broker_http_rejected_requests") .help("Counter of HTTP requests rejected by rate limiting") .register(); + public RateLimitingFilter(double rateLimit, Meter meter) { + limiter = RateLimiter.create(rateLimit); + rateLimitRequestCounter = meter.counterBuilder(RATE_LIMIT_REQUEST_COUNT_METRIC_NAME) + .setDescription("Counter of HTTP requests processed by the rate limiting filter.") + .setUnit("{request}") + .build(); + } + @Override public void init(FilterConfig filterConfig) throws ServletException { } @@ -50,9 +71,11 @@ public void init(FilterConfig filterConfig) throws ServletException { public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { if (limiter.tryAcquire()) { + rateLimitRequestCounter.add(1, Result.ACCEPTED.attributes); chain.doFilter(request, response); } else { httpRejectedRequests.inc(); + rateLimitRequestCounter.add(1, Result.REJECTED.attributes); HttpServletResponse httpResponse = (HttpServletResponse) response; httpResponse.sendError(429, "Too Many Requests"); } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/WebExecutorThreadPoolStats.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/WebExecutorThreadPoolStats.java new file mode 100644 index 0000000000000..6bfe4e33b8e5b --- /dev/null +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/WebExecutorThreadPoolStats.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.web; + +import com.google.common.annotations.VisibleForTesting; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.metrics.ObservableLongUpDownCounter; + +public class WebExecutorThreadPoolStats implements AutoCloseable { + // Replaces ['pulsar_web_executor_max_threads', 'pulsar_web_executor_min_threads'] + public static final String LIMIT_COUNTER = "pulsar.web.executor.thread.limit"; + private final ObservableLongUpDownCounter limitCounter; + + // Replaces + // ['pulsar_web_executor_active_threads', 'pulsar_web_executor_current_threads', 'pulsar_web_executor_idle_threads'] + public static final String USAGE_COUNTER = "pulsar.web.executor.thread.usage"; + private final ObservableLongUpDownCounter usageCounter; + + public static final AttributeKey LIMIT_TYPE_KEY = + AttributeKey.stringKey("pulsar.web.executor.thread.limit.type"); + @VisibleForTesting + enum LimitType { + MAX, + MIN; + public final Attributes attributes = Attributes.of(LIMIT_TYPE_KEY, name().toLowerCase()); + } + + public static final AttributeKey USAGE_TYPE_KEY = + AttributeKey.stringKey("pulsar.web.executor.thread.usage.type"); + @VisibleForTesting + enum UsageType { + ACTIVE, + CURRENT, + IDLE; + public final Attributes attributes = Attributes.of(USAGE_TYPE_KEY, name().toLowerCase()); + } + + public WebExecutorThreadPoolStats(Meter meter, WebExecutorThreadPool executor) { + limitCounter = meter + .upDownCounterBuilder(LIMIT_COUNTER) + .setUnit("{thread}") + .setDescription("The thread limits for the pulsar-web executor pool.") + .buildWithCallback(measurement -> { + measurement.record(executor.getMaxThreads(), LimitType.MAX.attributes); + measurement.record(executor.getMinThreads(), LimitType.MIN.attributes); + }); + usageCounter = meter + .upDownCounterBuilder(USAGE_COUNTER) + .setUnit("{thread}") + .setDescription("The current usage of threads in the pulsar-web executor pool.") + .buildWithCallback(measurement -> { + var idleThreads = executor.getIdleThreads(); + var currentThreads = executor.getThreads(); + measurement.record(idleThreads, UsageType.IDLE.attributes); + measurement.record(currentThreads, UsageType.CURRENT.attributes); + measurement.record(currentThreads - idleThreads, UsageType.ACTIVE.attributes); + }); + } + + @Override + public synchronized void close() { + limitCounter.close(); + usageCounter.close(); + } +} diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletWithClassLoader.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletWithClassLoader.java index c2b4b90073391..bc1f25c5af933 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletWithClassLoader.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletWithClassLoader.java @@ -22,7 +22,6 @@ import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.broker.ClassLoaderSwitcher; import org.apache.pulsar.common.configuration.PulsarConfiguration; import org.apache.pulsar.common.nar.NarClassLoader; import org.eclipse.jetty.servlet.ServletHolder; @@ -40,29 +39,45 @@ public class AdditionalServletWithClassLoader implements AdditionalServlet { @Override public void loadConfig(PulsarConfiguration pulsarConfiguration) { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + ClassLoader prevClassLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(classLoader); servlet.loadConfig(pulsarConfiguration); + } finally { + Thread.currentThread().setContextClassLoader(prevClassLoader); } } @Override public String getBasePath() { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + ClassLoader prevClassLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(classLoader); return servlet.getBasePath(); + } finally { + Thread.currentThread().setContextClassLoader(prevClassLoader); } } @Override public ServletHolder getServletHolder() { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + ClassLoader prevClassLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(classLoader); return servlet.getServletHolder(); + } finally { + Thread.currentThread().setContextClassLoader(prevClassLoader); } } @Override public void close() { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + ClassLoader prevClassLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(classLoader); servlet.close(); + } finally { + Thread.currentThread().setContextClassLoader(prevClassLoader); } try { classLoader.close(); diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServlets.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServlets.java index 0046d28afa444..f6fc42ff0fccf 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServlets.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServlets.java @@ -79,12 +79,16 @@ public static AdditionalServlets load(PulsarConfiguration conf) throws IOExcepti return null; } + String[] additionalServletsList = additionalServlets.split(","); + if (additionalServletsList.length == 0) { + return null; + } + AdditionalServletDefinitions definitions = AdditionalServletUtils.searchForServlets(additionalServletDirectory , narExtractionDirectory); ImmutableMap.Builder builder = ImmutableMap.builder(); - String[] additionalServletsList = additionalServlets.split(","); for (String servletName : additionalServletsList) { AdditionalServletMetadata definition = definitions.servlets().get(servletName); if (null == definition) { @@ -106,7 +110,7 @@ public static AdditionalServlets load(PulsarConfiguration conf) throws IOExcepti } Map servlets = builder.build(); - if (servlets != null && !servlets.isEmpty()) { + if (!servlets.isEmpty()) { return new AdditionalServlets(servlets); } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/tls/JettySslContextFactory.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/tls/JettySslContextFactory.java index 46a86045995f9..0ac1b78ca993f 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/tls/JettySslContextFactory.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/tls/JettySslContextFactory.java @@ -21,10 +21,8 @@ import java.util.Set; import javax.net.ssl.SSLContext; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.common.util.DefaultSslContextBuilder; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.apache.pulsar.common.util.SecurityUtility; -import org.apache.pulsar.common.util.SslContextAutoRefreshBuilder; -import org.apache.pulsar.common.util.keystoretls.NetSslContextBuilder; import org.eclipse.jetty.util.ssl.SslContextFactory; @Slf4j @@ -35,57 +33,21 @@ public class JettySslContextFactory { } } - public static SslContextFactory.Server createServerSslContextWithKeystore(String sslProviderString, - String keyStoreTypeString, - String keyStore, - String keyStorePassword, - boolean allowInsecureConnection, - String trustStoreTypeString, - String trustStore, - String trustStorePassword, - boolean requireTrustedClientCertOnConnect, - Set ciphers, - Set protocols, - long certRefreshInSec) { - NetSslContextBuilder sslCtxRefresher = new NetSslContextBuilder( - sslProviderString, - keyStoreTypeString, - keyStore, - keyStorePassword, - allowInsecureConnection, - trustStoreTypeString, - trustStore, - trustStorePassword, - requireTrustedClientCertOnConnect, - certRefreshInSec); - - return new JettySslContextFactory.Server(sslProviderString, sslCtxRefresher, + public static SslContextFactory.Server createSslContextFactory(String sslProviderString, + PulsarSslFactory pulsarSslFactory, + boolean requireTrustedClientCertOnConnect, + Set ciphers, Set protocols) { + return new JettySslContextFactory.Server(sslProviderString, pulsarSslFactory, requireTrustedClientCertOnConnect, ciphers, protocols); } - public static SslContextFactory createServerSslContext(String sslProviderString, boolean tlsAllowInsecureConnection, - String tlsTrustCertsFilePath, - String tlsCertificateFilePath, - String tlsKeyFilePath, - boolean tlsRequireTrustedClientCertOnConnect, - Set ciphers, - Set protocols, - long certRefreshInSec) { - DefaultSslContextBuilder sslCtxRefresher = - new DefaultSslContextBuilder(tlsAllowInsecureConnection, tlsTrustCertsFilePath, tlsCertificateFilePath, - tlsKeyFilePath, tlsRequireTrustedClientCertOnConnect, certRefreshInSec, sslProviderString); - - return new JettySslContextFactory.Server(sslProviderString, sslCtxRefresher, - tlsRequireTrustedClientCertOnConnect, ciphers, protocols); - } - private static class Server extends SslContextFactory.Server { - private final SslContextAutoRefreshBuilder sslCtxRefresher; + private final PulsarSslFactory pulsarSslFactory; - public Server(String sslProviderString, SslContextAutoRefreshBuilder sslCtxRefresher, + public Server(String sslProviderString, PulsarSslFactory pulsarSslFactory, boolean requireTrustedClientCertOnConnect, Set ciphers, Set protocols) { super(); - this.sslCtxRefresher = sslCtxRefresher; + this.pulsarSslFactory = pulsarSslFactory; if (ciphers != null && ciphers.size() > 0) { this.setIncludeCipherSuites(ciphers.toArray(new String[0])); @@ -110,7 +72,7 @@ public Server(String sslProviderString, SslContextAutoRefreshBuilder @Override public SSLContext getSslContext() { - return sslCtxRefresher.get(); + return this.pulsarSslFactory.getInternalSslContext(); } } } diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMappingTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMappingTest.java index d7be7dabd0db1..9cd8160444249 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMappingTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMappingTest.java @@ -21,6 +21,7 @@ import static org.apache.bookkeeper.feature.SettableFeatureProvider.DISABLE_ALL; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -28,6 +29,7 @@ import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -35,7 +37,11 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import lombok.Cleanup; import org.apache.bookkeeper.client.DefaultBookieAddressResolver; import org.apache.bookkeeper.client.EnsemblePlacementPolicy; import org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy; @@ -46,6 +52,7 @@ import org.apache.bookkeeper.net.BookieId; import org.apache.bookkeeper.net.BookieNode; import org.apache.bookkeeper.net.BookieSocketAddress; +import org.apache.bookkeeper.net.NetworkTopology; import org.apache.bookkeeper.proto.BookieAddressResolver; import org.apache.bookkeeper.stats.NullStatsLogger; import org.apache.bookkeeper.stats.StatsLogger; @@ -55,6 +62,8 @@ import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.MetadataStoreFactory; +import org.apache.pulsar.metadata.api.Notification; +import org.apache.pulsar.metadata.api.NotificationType; import org.apache.pulsar.metadata.bookkeeper.BookieServiceInfoSerde; import org.apache.pulsar.metadata.bookkeeper.PulsarRegistrationClient; import org.awaitility.Awaitility; @@ -254,6 +263,7 @@ public void testWithPulsarRegistrationClient() throws Exception { bkClientConf.getTimeoutTimerNumTicks()); RackawareEnsemblePlacementPolicy repp = new RackawareEnsemblePlacementPolicy(); + mapping.registerRackChangeListener(repp); Class clazz1 = Class.forName("org.apache.bookkeeper.client.TopologyAwareEnsemblePlacementPolicy"); Field field1 = clazz1.getDeclaredField("knownBookies"); field1.setAccessible(true); @@ -323,6 +333,81 @@ public void testWithPulsarRegistrationClient() throws Exception { assertEquals(knownBookies.get(BOOKIE2.toBookieId()).getNetworkLocation(), "/rack1"); assertEquals(knownBookies.get(BOOKIE3.toBookieId()).getNetworkLocation(), "/default-rack"); + //remove bookie2 rack, the bookie2 rack should be /default-rack + data = "{\"group1\": {\"" + BOOKIE1 + + "\": {\"rack\": \"/rack0\", \"hostname\": \"bookie1.example.com\"}}}"; + store.put(BookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH, data.getBytes(), Optional.empty()).join(); + Awaitility.await().atMost(30, TimeUnit.SECONDS).until(() -> ((BookiesRackConfiguration)field.get(mapping)).get("group1").size() == 1); + + racks = mapping + .resolve(Lists.newArrayList(BOOKIE1.getHostName(), BOOKIE2.getHostName(), BOOKIE3.getHostName())) + .stream().filter(Objects::nonNull).toList(); + assertEquals(racks.size(), 1); + assertEquals(racks.get(0), "/rack0"); + assertEquals(knownBookies.size(), 3); + assertEquals(knownBookies.get(BOOKIE1.toBookieId()).getNetworkLocation(), "/rack0"); + assertEquals(knownBookies.get(BOOKIE2.toBookieId()).getNetworkLocation(), "/default-rack"); + assertEquals(knownBookies.get(BOOKIE3.toBookieId()).getNetworkLocation(), "/default-rack"); + timer.stop(); } + + @Test + public void testNoDeadlockWithRackawarePolicy() throws Exception { + ClientConfiguration bkClientConf = new ClientConfiguration(); + bkClientConf.setProperty(BookieRackAffinityMapping.METADATA_STORE_INSTANCE, store); + + BookieRackAffinityMapping mapping = new BookieRackAffinityMapping(); + mapping.setBookieAddressResolver(BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); + mapping.setConf(bkClientConf); + + @Cleanup("stop") + HashedWheelTimer timer = new HashedWheelTimer(new ThreadFactoryBuilder().setNameFormat("TestTimer-%d").build(), + bkClientConf.getTimeoutTimerTickDurationMs(), TimeUnit.MILLISECONDS, + bkClientConf.getTimeoutTimerNumTicks()); + + RackawareEnsemblePlacementPolicy repp = new RackawareEnsemblePlacementPolicy(); + repp.initialize(bkClientConf, Optional.of(mapping), timer, + DISABLE_ALL, NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); + repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK); + + mapping.registerRackChangeListener(repp); + + @Cleanup("shutdownNow") + ExecutorService executor1 = Executors.newSingleThreadExecutor(); + @Cleanup("shutdownNow") + ExecutorService executor2 = Executors.newSingleThreadExecutor(); + + CountDownLatch count = new CountDownLatch(2); + + executor1.submit(() -> { + try { + Method handleUpdates = + BookieRackAffinityMapping.class.getDeclaredMethod("handleUpdates", Notification.class); + handleUpdates.setAccessible(true); + Notification n = + new Notification(NotificationType.Modified, BookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH); + long start = System.currentTimeMillis(); + while (System.currentTimeMillis() - start < 2_000) { + handleUpdates.invoke(mapping, n); + } + count.countDown(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + executor2.submit(() -> { + Set writableBookies = new HashSet<>(); + writableBookies.add(BOOKIE1.toBookieId()); + long start = System.currentTimeMillis(); + while (System.currentTimeMillis() - start < 2_000) { + repp.onClusterChanged(writableBookies, Collections.emptySet()); + repp.onClusterChanged(Collections.emptySet(), Collections.emptySet()); + } + count.countDown(); + }); + + assertTrue(count.await(3, TimeUnit.SECONDS)); + } } diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java index f535ced08f731..beb00197e4e9a 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java @@ -18,11 +18,14 @@ */ package org.apache.pulsar.bookie.rackawareness; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Sets; import io.netty.util.HashedWheelTimer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -34,18 +37,23 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.client.BKException.BKNotEnoughBookiesException; import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.feature.SettableFeatureProvider; import org.apache.bookkeeper.net.BookieId; import org.apache.bookkeeper.net.BookieSocketAddress; import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.common.policies.data.BookieInfo; +import org.apache.pulsar.common.policies.data.BookiesRackConfiguration; import org.apache.pulsar.common.policies.data.EnsemblePlacementPolicyConfig; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.MetadataStoreFactory; +import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.apache.pulsar.metadata.cache.impl.MetadataCacheImpl; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -114,6 +122,113 @@ public void testNonRegionBookie() throws Exception { assertFalse(ensemble.contains(new BookieSocketAddress(BOOKIE4).toBookieId())); } + @Test + public void testMetadataStoreCases() throws Exception { + Map mainBookieGroup = new HashMap<>(); + mainBookieGroup.put(BOOKIE1, BookieInfo.builder().rack("rack0").build()); + mainBookieGroup.put(BOOKIE2, BookieInfo.builder().rack("rack1").build()); + mainBookieGroup.put(BOOKIE3, BookieInfo.builder().rack("rack1").build()); + mainBookieGroup.put(BOOKIE4, BookieInfo.builder().rack("rack0").build()); + + Map secondaryBookieGroup = new HashMap<>(); + + store = mock(MetadataStoreExtended.class); + MetadataCacheImpl cache = mock(MetadataCacheImpl.class); + when(store.getMetadataCache(BookiesRackConfiguration.class)).thenReturn(cache); + CompletableFuture> initialFuture = new CompletableFuture<>(); + //The initialFuture only has group1. + BookiesRackConfiguration rackConfiguration1 = new BookiesRackConfiguration(); + rackConfiguration1.put("group1", mainBookieGroup); + rackConfiguration1.put("group2", secondaryBookieGroup); + initialFuture.complete(Optional.of(rackConfiguration1)); + + long waitTime = 2000; + CompletableFuture> waitingCompleteFuture = new CompletableFuture<>(); + new Thread(() -> { + try { + Thread.sleep(waitTime); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + //The waitingCompleteFuture has group1 and group2. + BookiesRackConfiguration rackConfiguration2 = new BookiesRackConfiguration(); + Map mainBookieGroup2 = new HashMap<>(); + mainBookieGroup2.put(BOOKIE1, BookieInfo.builder().rack("rack0").build()); + mainBookieGroup2.put(BOOKIE2, BookieInfo.builder().rack("rack1").build()); + mainBookieGroup2.put(BOOKIE4, BookieInfo.builder().rack("rack0").build()); + + Map secondaryBookieGroup2 = new HashMap<>(); + secondaryBookieGroup2.put(BOOKIE3, BookieInfo.builder().rack("rack0").build()); + rackConfiguration2.put("group1", mainBookieGroup2); + rackConfiguration2.put("group2", secondaryBookieGroup2); + waitingCompleteFuture.complete(Optional.of(rackConfiguration2)); + }).start(); + + long longWaitTime = 4000; + CompletableFuture> emptyFuture = new CompletableFuture<>(); + new Thread(() -> { + try { + Thread.sleep(longWaitTime); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + //The emptyFuture means that the zk node /bookies already be removed. + emptyFuture.complete(Optional.empty()); + }).start(); + + //Return different future means that cache expire. + when(cache.get(BookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH)) + .thenReturn(initialFuture).thenReturn(initialFuture) + .thenReturn(waitingCompleteFuture).thenReturn(waitingCompleteFuture) + .thenReturn(emptyFuture).thenReturn(emptyFuture); + + IsolatedBookieEnsemblePlacementPolicy isolationPolicy = new IsolatedBookieEnsemblePlacementPolicy(); + ClientConfiguration bkClientConf = new ClientConfiguration(); + bkClientConf.setProperty(BookieRackAffinityMapping.METADATA_STORE_INSTANCE, store); + bkClientConf.setProperty(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, isolationGroups); + isolationPolicy.initialize(bkClientConf, Optional.empty(), timer, SettableFeatureProvider.DISABLE_ALL, NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); + isolationPolicy.onClusterChanged(writableBookies, readOnlyBookies); + + MutablePair, Set> groups = new MutablePair<>(); + groups.setLeft(Sets.newHashSet("group1")); + groups.setRight(new HashSet<>()); + + //initialFuture, the future is waiting done. + Set blacklist = + isolationPolicy.getExcludedBookiesWithIsolationGroups(2, groups); + assertTrue(blacklist.isEmpty()); + + //waitingCompleteFuture, the future is waiting done. + blacklist = + isolationPolicy.getExcludedBookiesWithIsolationGroups(2, groups); + assertTrue(blacklist.isEmpty()); + + Thread.sleep(waitTime); + + //waitingCompleteFuture, the future is already done. + blacklist = + isolationPolicy.getExcludedBookiesWithIsolationGroups(2, groups); + assertFalse(blacklist.isEmpty()); + assertEquals(blacklist.size(), 1); + BookieId excludeBookie = blacklist.iterator().next(); + assertEquals(excludeBookie.toString(), BOOKIE3); + + //emptyFuture, the future is waiting done. + blacklist = + isolationPolicy.getExcludedBookiesWithIsolationGroups(2, groups); + assertFalse(blacklist.isEmpty()); + assertEquals(blacklist.size(), 1); + excludeBookie = blacklist.iterator().next(); + assertEquals(excludeBookie.toString(), BOOKIE3); + + Thread.sleep(longWaitTime - waitTime); + + //emptyFuture, the future is already done. + blacklist = + isolationPolicy.getExcludedBookiesWithIsolationGroups(2, groups); + assertTrue(blacklist.isEmpty()); + } + @Test public void testBasic() throws Exception { Map> bookieMapping = new HashMap<>(); diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasicTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasicTest.java index 723fde7083d38..f6e4b8e969ac1 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasicTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasicTest.java @@ -52,7 +52,7 @@ public void testLoadFileFromPulsarProperties() throws Exception { Properties properties = new Properties(); properties.setProperty("basicAuthConf", basicAuthConf); serviceConfiguration.setProperties(properties); - provider.initialize(serviceConfiguration); + provider.initialize(AuthenticationProvider.Context.builder().config(serviceConfiguration).build()); testAuthenticate(provider); } @@ -64,7 +64,7 @@ public void testLoadBase64FromPulsarProperties() throws Exception { Properties properties = new Properties(); properties.setProperty("basicAuthConf", basicAuthConfBase64); serviceConfiguration.setProperties(properties); - provider.initialize(serviceConfiguration); + provider.initialize(AuthenticationProvider.Context.builder().config(serviceConfiguration).build()); testAuthenticate(provider); } @@ -74,7 +74,7 @@ public void testLoadFileFromSystemProperties() throws Exception { AuthenticationProviderBasic provider = new AuthenticationProviderBasic(); ServiceConfiguration serviceConfiguration = new ServiceConfiguration(); System.setProperty("pulsar.auth.basic.conf", basicAuthConf); - provider.initialize(serviceConfiguration); + provider.initialize(AuthenticationProvider.Context.builder().config(serviceConfiguration).build()); testAuthenticate(provider); } @@ -84,7 +84,7 @@ public void testLoadBase64FromSystemProperties() throws Exception { AuthenticationProviderBasic provider = new AuthenticationProviderBasic(); ServiceConfiguration serviceConfiguration = new ServiceConfiguration(); System.setProperty("pulsar.auth.basic.conf", basicAuthConfBase64); - provider.initialize(serviceConfiguration); + provider.initialize(AuthenticationProvider.Context.builder().config(serviceConfiguration).build()); testAuthenticate(provider); } diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderListTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderListTest.java index 7793a5c029f2a..e81198217b5b6 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderListTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderListTest.java @@ -29,7 +29,6 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; - import io.jsonwebtoken.SignatureAlgorithm; import io.jsonwebtoken.io.Decoders; import io.jsonwebtoken.security.Keys; @@ -90,7 +89,7 @@ public void setUp() throws Exception { ); ServiceConfiguration confA = new ServiceConfiguration(); confA.setProperties(propertiesA); - providerA.initialize(confA); + providerA.initialize(AuthenticationProvider.Context.builder().config(confA).build()); Properties propertiesB = new Properties(); propertiesB.setProperty(AuthenticationProviderToken.CONF_TOKEN_SETTING_PREFIX, "b"); @@ -103,7 +102,7 @@ public void setUp() throws Exception { ); ServiceConfiguration confB = new ServiceConfiguration(); confB.setProperties(propertiesB); - providerB.initialize(confB); + providerB.initialize(AuthenticationProvider.Context.builder().config(confB).build()); this.authProvider = new AuthenticationProviderList(Lists.newArrayList( providerA, providerB diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTokenTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTokenTest.java index f50731c7654af..3e1a3e180349e 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTokenTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTokenTest.java @@ -55,7 +55,9 @@ import javax.naming.AuthenticationException; import javax.servlet.http.HttpServletRequest; import lombok.Cleanup; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.authentication.metrics.AuthenticationMetricsToken; import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; import org.apache.pulsar.common.api.AuthData; import org.mockito.Mockito; @@ -70,7 +72,7 @@ public void testInvalidInitialize() throws Exception { AuthenticationProviderToken provider = new AuthenticationProviderToken(); try { - provider.initialize(new ServiceConfiguration()); + provider.initialize(AuthenticationProvider.Context.builder().config(new ServiceConfiguration()).build()); fail("should have failed"); } catch (IOException e) { // Expected, secret key was not defined @@ -135,7 +137,7 @@ public void testAuthSecretKey() throws Exception { ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); try { provider.authenticate(new AuthenticationDataSource() { @@ -249,7 +251,7 @@ public void testTrimAuthSecretKeyFilePath() throws Exception { ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); } @Test @@ -268,7 +270,7 @@ public void testAuthSecretKeyFromFile() throws Exception { ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); String token = AuthTokenUtils.createToken(secretKey, SUBJECT, Optional.empty()); @@ -303,7 +305,7 @@ public void testAuthSecretKeyFromValidFile() throws Exception { ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); String token = AuthTokenUtils.createToken(secretKey, SUBJECT, Optional.empty()); @@ -335,7 +337,7 @@ public void testAuthSecretKeyFromDataBase64() throws Exception { ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); String token = AuthTokenUtils.createToken(secretKey, SUBJECT, Optional.empty()); @@ -370,7 +372,7 @@ public void testAuthSecretKeyPair() throws Exception { ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); // Use private key to generate token PrivateKey privateKey = AuthTokenUtils.decodePrivateKey(Decoders.BASE64.decode(privateKeyStr), SignatureAlgorithm.RS256); @@ -413,8 +415,7 @@ public void testAuthSecretKeyPairWithCustomClaim() throws Exception { ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - provider.initialize(conf); - + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); // Use private key to generate token PrivateKey privateKey = AuthTokenUtils.decodePrivateKey(Decoders.BASE64.decode(privateKeyStr), SignatureAlgorithm.RS256); @@ -460,7 +461,7 @@ public void testAuthSecretKeyPairWithECDSA() throws Exception { ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); // Use private key to generate token PrivateKey privateKey = AuthTokenUtils.decodePrivateKey(Decoders.BASE64.decode(privateKeyStr), SignatureAlgorithm.ES256); @@ -484,8 +485,11 @@ public String getCommandData() { } @Test(expectedExceptions = AuthenticationException.class) - public void testAuthenticateWhenNoJwtPassed() throws AuthenticationException { + public void testAuthenticateWhenNoJwtPassed() throws Exception { + @Cleanup AuthenticationProviderToken provider = new AuthenticationProviderToken(); + FieldUtils.writeDeclaredField( + provider, "authenticationMetricsToken", mock(AuthenticationMetricsToken.class), true); provider.authenticate(new AuthenticationDataSource() { @Override public boolean hasDataFromCommand() { @@ -500,8 +504,11 @@ public boolean hasDataFromHttp() { } @Test(expectedExceptions = AuthenticationException.class) - public void testAuthenticateWhenAuthorizationHeaderNotExist() throws AuthenticationException { + public void testAuthenticateWhenAuthorizationHeaderNotExist() throws Exception { + @Cleanup AuthenticationProviderToken provider = new AuthenticationProviderToken(); + FieldUtils.writeDeclaredField( + provider, "authenticationMetricsToken", mock(AuthenticationMetricsToken.class), true); provider.authenticate(new AuthenticationDataSource() { @Override public String getHttpHeader(String name) { @@ -516,8 +523,11 @@ public boolean hasDataFromHttp() { } @Test(expectedExceptions = AuthenticationException.class) - public void testAuthenticateWhenAuthHeaderValuePrefixIsInvalid() throws AuthenticationException { + public void testAuthenticateWhenAuthHeaderValuePrefixIsInvalid() throws Exception { + @Cleanup AuthenticationProviderToken provider = new AuthenticationProviderToken(); + FieldUtils.writeDeclaredField( + provider, "authenticationMetricsToken", mock(AuthenticationMetricsToken.class), true); provider.authenticate(new AuthenticationDataSource() { @Override public String getHttpHeader(String name) { @@ -532,8 +542,11 @@ public boolean hasDataFromHttp() { } @Test(expectedExceptions = AuthenticationException.class) - public void testAuthenticateWhenJwtIsBlank() throws AuthenticationException { + public void testAuthenticateWhenJwtIsBlank() throws Exception { + @Cleanup AuthenticationProviderToken provider = new AuthenticationProviderToken(); + FieldUtils.writeDeclaredField( + provider, "authenticationMetricsToken", mock(AuthenticationMetricsToken.class), true); provider.authenticate(new AuthenticationDataSource() { @Override public String getHttpHeader(String name) { @@ -559,7 +572,7 @@ public void testAuthenticateWhenInvalidTokenIsPassed() throws AuthenticationExce conf.setProperties(properties); AuthenticationProviderToken provider = new AuthenticationProviderToken(); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); provider.authenticate(new AuthenticationDataSource() { @Override public String getHttpHeader(String name) { @@ -582,7 +595,7 @@ public void testValidationKeyWhenBlankSecretKeyIsPassed() throws IOException { conf.setProperties(properties); AuthenticationProviderToken provider = new AuthenticationProviderToken(); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); } @Test(expectedExceptions = IOException.class) @@ -594,7 +607,7 @@ public void testValidationKeyWhenBlankPublicKeyIsPassed() throws IOException { conf.setProperties(properties); AuthenticationProviderToken provider = new AuthenticationProviderToken(); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); } @Test(expectedExceptions = IOException.class) @@ -606,7 +619,7 @@ public void testInitializeWhenSecretKeyFilePathIsInvalid() throws IOException { ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - new AuthenticationProviderToken().initialize(conf); + new AuthenticationProviderToken().initialize(AuthenticationProvider.Context.builder().config(conf).build()); } @Test(expectedExceptions = IOException.class) @@ -618,7 +631,7 @@ public void testInitializeWhenSecretKeyIsValidPathOrBase64() throws IOException ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - new AuthenticationProviderToken().initialize(conf); + new AuthenticationProviderToken().initialize(AuthenticationProvider.Context.builder().config(conf).build()); } @Test(expectedExceptions = IllegalArgumentException.class) @@ -633,7 +646,7 @@ public void testInitializeWhenSecretKeyFilePathIfNotExist() throws IOException { ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - new AuthenticationProviderToken().initialize(conf); + new AuthenticationProviderToken().initialize(AuthenticationProvider.Context.builder().config(conf).build()); } @Test(expectedExceptions = IOException.class) @@ -645,7 +658,7 @@ public void testInitializeWhenPublicKeyFilePathIsInvalid() throws IOException { ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - new AuthenticationProviderToken().initialize(conf); + new AuthenticationProviderToken().initialize(AuthenticationProvider.Context.builder().config(conf).build()); } @Test(expectedExceptions = IllegalArgumentException.class) @@ -657,7 +670,7 @@ public void testValidationWhenPublicKeyAlgIsInvalid() throws IOException { ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - new AuthenticationProviderToken().initialize(conf); + new AuthenticationProviderToken().initialize(AuthenticationProvider.Context.builder().config(conf).build()); } @@ -676,7 +689,7 @@ public void testExpiringToken() throws Exception { ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); // Create a token that will expire in 3 seconds String expiringToken = AuthTokenUtils.createToken(secretKey, SUBJECT, @@ -709,7 +722,7 @@ public void testExpiredTokenFailsOnAuthenticate() throws Exception { ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); // Create a token that is already expired String expiringToken = AuthTokenUtils.createToken(secretKey, SUBJECT, @@ -828,7 +841,7 @@ public void testArrayTypeRoleClaim() throws Exception { ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); // Use private key to generate token PrivateKey privateKey = AuthTokenUtils.decodePrivateKey(Decoders.BASE64.decode(privateKeyStr), SignatureAlgorithm.RS256); @@ -877,7 +890,7 @@ public void testTokenSettingPrefix() throws Exception { ); Mockito.when(mockConf.getProperty(AuthenticationProviderToken.CONF_TOKEN_SETTING_PREFIX)).thenReturn(prefix); - provider.initialize(mockConf); + provider.initialize(AuthenticationProvider.Context.builder().config(mockConf).build()); // Each property is fetched only once. Prevent multiple fetches. Mockito.verify(mockConf, Mockito.times(1)).getProperty(AuthenticationProviderToken.CONF_TOKEN_SETTING_PREFIX); @@ -908,7 +921,7 @@ public void testTokenFromHttpParams() throws Exception { ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); String token = AuthTokenUtils.createToken(secretKey, SUBJECT, Optional.empty()); HttpServletRequest servletRequest = mock(HttpServletRequest.class); @@ -934,7 +947,7 @@ public void testTokenFromHttpHeaders() throws Exception { ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); String token = AuthTokenUtils.createToken(secretKey, SUBJECT, Optional.empty()); HttpServletRequest servletRequest = mock(HttpServletRequest.class); @@ -960,7 +973,7 @@ public void testTokenStateUpdatesAuthenticationDataSource() throws Exception { ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); AuthenticationState authState = provider.newAuthState(null,null, null); @@ -1016,7 +1029,7 @@ private static void testTokenAudienceWithDifferentConfig(Properties properties, properties.setProperty(AuthenticationProviderToken.CONF_TOKEN_SECRET_KEY, secretKeyFile.toString()); ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); String token = createTokenWithAudience(secretKey, audienceClaim, audiences); diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java index 7e329d14307c7..ed9626dffe23f 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java @@ -24,13 +24,16 @@ import io.jsonwebtoken.Jwts; import io.jsonwebtoken.SignatureAlgorithm; import java.util.Properties; +import java.util.function.Function; +import lombok.Cleanup; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription; import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; import org.apache.pulsar.broker.resources.PulsarResources; import org.testng.annotations.Test; - import javax.crypto.SecretKey; +import java.util.Set; import java.util.concurrent.CompletableFuture; public class MultiRolesTokenAuthorizationProviderTest { @@ -43,6 +46,8 @@ public void testMultiRolesAuthz() throws Exception { String token = Jwts.builder().claim("sub", new String[]{userA, userB}).signWith(secretKey).compact(); MultiRolesTokenAuthorizationProvider provider = new MultiRolesTokenAuthorizationProvider(); + ServiceConfiguration conf = new ServiceConfiguration(); + provider.initialize(conf, mock(PulsarResources.class)); AuthenticationDataSource ads = new AuthenticationDataSource() { @Override @@ -60,18 +65,18 @@ public String getHttpHeader(String name) { } }; - assertTrue(provider.authorize(ads, role -> { + assertTrue(provider.authorize("test", ads, role -> { if (role.equals(userB)) { return CompletableFuture.completedFuture(true); // only userB has permission } return CompletableFuture.completedFuture(false); }).get()); - assertTrue(provider.authorize(ads, role -> { + assertTrue(provider.authorize("test", ads, role -> { return CompletableFuture.completedFuture(true); // all users has permission }).get()); - assertFalse(provider.authorize(ads, role -> { + assertFalse(provider.authorize("test", ads, role -> { return CompletableFuture.completedFuture(false); // all users has no permission }).get()); } @@ -82,6 +87,8 @@ public void testMultiRolesAuthzWithEmptyRoles() throws Exception { String token = Jwts.builder().claim("sub", new String[]{}).signWith(secretKey).compact(); MultiRolesTokenAuthorizationProvider provider = new MultiRolesTokenAuthorizationProvider(); + ServiceConfiguration conf = new ServiceConfiguration(); + provider.initialize(conf, mock(PulsarResources.class)); AuthenticationDataSource ads = new AuthenticationDataSource() { @Override @@ -99,7 +106,7 @@ public String getHttpHeader(String name) { } }; - assertFalse(provider.authorize(ads, role -> CompletableFuture.completedFuture(false)).get()); + assertFalse(provider.authorize("test", ads, role -> CompletableFuture.completedFuture(false)).get()); } @Test @@ -109,6 +116,8 @@ public void testMultiRolesAuthzWithSingleRole() throws Exception { String token = Jwts.builder().claim("sub", testRole).signWith(secretKey).compact(); MultiRolesTokenAuthorizationProvider provider = new MultiRolesTokenAuthorizationProvider(); + ServiceConfiguration conf = new ServiceConfiguration(); + provider.initialize(conf, mock(PulsarResources.class)); AuthenticationDataSource ads = new AuthenticationDataSource() { @Override @@ -126,7 +135,7 @@ public String getHttpHeader(String name) { } }; - assertTrue(provider.authorize(ads, role -> { + assertTrue(provider.authorize("test", ads, role -> { if (role.equals(testRole)) { return CompletableFuture.completedFuture(true); } @@ -134,11 +143,66 @@ public String getHttpHeader(String name) { }).get()); } + @Test + public void testMultiRolesAuthzWithoutClaim() throws Exception { + final SecretKey secretKey = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); + final String testRole = "test-role"; + // broker will use "sub" as the claim by default. + final String token = Jwts.builder() + .claim("whatever", testRole).signWith(secretKey).compact(); + ServiceConfiguration conf = new ServiceConfiguration(); + final MultiRolesTokenAuthorizationProvider provider = new MultiRolesTokenAuthorizationProvider(); + provider.initialize(conf, mock(PulsarResources.class)); + final AuthenticationDataSource ads = new AuthenticationDataSource() { + @Override + public boolean hasDataFromHttp() { + return true; + } + + @Override + public String getHttpHeader(String name) { + if (name.equals("Authorization")) { + return "Bearer " + token; + } else { + throw new IllegalArgumentException("Wrong HTTP header"); + } + } + }; + + assertFalse(provider.authorize("test", ads, role -> { + if (role == null) { + throw new IllegalStateException("We should avoid pass null to sub providers"); + } + return CompletableFuture.completedFuture(role.equals(testRole)); + }).get()); + } + + @Test + public void testMultiRolesAuthzWithAnonymousUser() throws Exception { + @Cleanup + MultiRolesTokenAuthorizationProvider provider = new MultiRolesTokenAuthorizationProvider(); + ServiceConfiguration conf = new ServiceConfiguration(); + + provider.initialize(conf, mock(PulsarResources.class)); + + Function> authorizeFunc = (String role) -> { + if (role.equals("test-role")) { + return CompletableFuture.completedFuture(true); + } + return CompletableFuture.completedFuture(false); + }; + assertTrue(provider.authorize("test-role", null, authorizeFunc).get()); + assertFalse(provider.authorize("test-role-x", null, authorizeFunc).get()); + assertTrue(provider.authorize("test-role", new AuthenticationDataSubscription(null, "test-sub"), authorizeFunc).get()); + } + @Test public void testMultiRolesNotFailNonJWT() throws Exception { String token = "a-non-jwt-token"; MultiRolesTokenAuthorizationProvider provider = new MultiRolesTokenAuthorizationProvider(); + ServiceConfiguration conf = new ServiceConfiguration(); + provider.initialize(conf, mock(PulsarResources.class)); AuthenticationDataSource ads = new AuthenticationDataSource() { @Override @@ -156,7 +220,7 @@ public String getHttpHeader(String name) { } }; - assertFalse(provider.authorize(ads, role -> CompletableFuture.completedFuture(false)).get()); + assertFalse(provider.authorize("test", ads, role -> CompletableFuture.completedFuture(false)).get()); } @Test @@ -191,11 +255,51 @@ public String getHttpHeader(String name) { } }; - assertTrue(provider.authorize(ads, role -> { + assertTrue(provider.authorize("test", ads, role -> { if (role.equals(testRole)) { return CompletableFuture.completedFuture(true); } return CompletableFuture.completedFuture(false); }).get()); } + + @Test + public void testMultiRolesAuthzWithSuperUser() throws Exception { + SecretKey secretKey = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); + String testAdminRole = "admin"; + String token = Jwts.builder().claim("sub", testAdminRole).signWith(secretKey).compact(); + + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setSuperUserRoles(Set.of(testAdminRole)); + + MultiRolesTokenAuthorizationProvider provider = new MultiRolesTokenAuthorizationProvider(); + provider.initialize(conf, mock(PulsarResources.class)); + + AuthenticationDataSource ads = new AuthenticationDataSource() { + @Override + public boolean hasDataFromHttp() { + return true; + } + + @Override + public String getHttpHeader(String name) { + if (name.equals("Authorization")) { + return "Bearer " + token; + } else { + throw new IllegalArgumentException("Wrong HTTP header"); + } + } + }; + + assertTrue(provider.isSuperUser(testAdminRole, ads, conf).get()); + Function> authorizeFunc = (String role) -> { + if (role.equals("admin1")) { + return CompletableFuture.completedFuture(true); + } + return CompletableFuture.completedFuture(false); + }; + assertTrue(provider.authorize(testAdminRole, ads, (String role) -> CompletableFuture.completedFuture(false)).get()); + assertTrue(provider.authorize("admin1", null, authorizeFunc).get()); + assertFalse(provider.authorize("admin2", null, authorizeFunc).get()); + } } diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/resources/LoadBalanceResourcesTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/resources/LoadBalanceResourcesTest.java new file mode 100644 index 0000000000000..cd7dd01b66576 --- /dev/null +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/resources/LoadBalanceResourcesTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.resources; + +import static org.apache.pulsar.broker.resources.BaseResources.joinPath; +import static org.apache.pulsar.broker.resources.LoadBalanceResources.BUNDLE_DATA_BASE_PATH; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.testng.Assert.assertThrows; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.metadata.api.MetadataStore; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class LoadBalanceResourcesTest { + private MetadataStore configurationStore; + private MetadataStore localStore; + private LoadBalanceResources loadBalanceResources; + + @BeforeMethod + public void setup() { + localStore = mock(MetadataStore.class); + configurationStore = mock(MetadataStore.class); + loadBalanceResources = new LoadBalanceResources(localStore, 30); + } + + /** + * Test that the bundle-data node is deleted from the local stores. + */ + @Test + public void testDeleteBundleDataAsync() { + NamespaceName ns = NamespaceName.get("my-tenant/my-ns"); + String namespaceBundlePath = joinPath(BUNDLE_DATA_BASE_PATH, ns.toString()); + loadBalanceResources.getBundleDataResources().deleteBundleDataAsync(ns); + + String tenant="my-tenant"; + String tenantBundlePath = joinPath(BUNDLE_DATA_BASE_PATH, tenant); + loadBalanceResources.getBundleDataResources().deleteBundleDataTenantAsync(tenant); + + verify(localStore).deleteRecursive(namespaceBundlePath); + verify(localStore).deleteRecursive(tenantBundlePath); + + assertThrows(()-> verify(configurationStore).deleteRecursive(namespaceBundlePath)); + assertThrows(()-> verify(configurationStore).deleteRecursive(tenantBundlePath)); + } +} diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/resources/NamespaceResourcesTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/resources/NamespaceResourcesTest.java index 85f54a76dc3c4..7fb9e2c476d08 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/resources/NamespaceResourcesTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/resources/NamespaceResourcesTest.java @@ -22,8 +22,8 @@ import static org.testng.Assert.assertTrue; import org.testng.annotations.Test; - public class NamespaceResourcesTest { + @Test public void test_pathIsFromNamespace() { assertFalse(NamespaceResources.pathIsFromNamespace("/admin/clusters")); @@ -31,4 +31,6 @@ public void test_pathIsFromNamespace() { assertFalse(NamespaceResources.pathIsFromNamespace("/admin/policies/my-tenant")); assertTrue(NamespaceResources.pathIsFromNamespace("/admin/policies/my-tenant/my-ns")); } + + } \ No newline at end of file diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsClient.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsClient.java new file mode 100644 index 0000000000000..6d724c289b52c --- /dev/null +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsClient.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats.prometheus; + +import static org.assertj.core.api.Fail.fail; +import static org.testng.Assert.assertTrue; +import com.google.common.base.MoreObjects; +import com.google.common.base.Splitter; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; +import io.restassured.RestAssured; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.commons.lang3.tuple.Pair; + +public class PrometheusMetricsClient { + private final String host; + private final int port; + + public PrometheusMetricsClient(String host, int port) { + this.host = host; + this.port = port; + } + + @SuppressWarnings("HttpUrlsUsage") + public Metrics getMetrics() { + String metrics = RestAssured.given().baseUri("http://" + host).port(port).get("/metrics").asString(); + return new Metrics(parseMetrics(metrics)); + } + + /** + * Hacky parsing of Prometheus text format. Should be good enough for unit tests + */ + public static Multimap parseMetrics(String metrics) { + Multimap parsed = ArrayListMultimap.create(); + + // Example of lines are + // jvm_threads_current{cluster="standalone",} 203.0 + // or + // pulsar_subscriptions_count{cluster="standalone", namespace="public/default", + // topic="persistent://public/default/test-2"} 0.0 + Pattern pattern = Pattern.compile("^(\\w+)\\{([^}]+)}\\s([+-]?[\\d\\w.+-]+)$"); + Pattern tagsPattern = Pattern.compile("(\\w+)=\"([^\"]+)\"(,\\s?)?"); + + Splitter.on("\n").split(metrics).forEach(line -> { + if (line.isEmpty() || line.startsWith("#")) { + return; + } + + Matcher matcher = pattern.matcher(line); + assertTrue(matcher.matches(), "line " + line + " does not match pattern " + pattern); + String name = matcher.group(1); + + Metric m = new Metric(); + String numericValue = matcher.group(3); + if (numericValue.equalsIgnoreCase("-Inf")) { + m.value = Double.NEGATIVE_INFINITY; + } else if (numericValue.equalsIgnoreCase("+Inf")) { + m.value = Double.POSITIVE_INFINITY; + } else { + m.value = Double.parseDouble(numericValue); + } + String tags = matcher.group(2); + Matcher tagsMatcher = tagsPattern.matcher(tags); + while (tagsMatcher.find()) { + String tag = tagsMatcher.group(1); + String value = tagsMatcher.group(2); + m.tags.put(tag, value); + } + + parsed.put(name, m); + }); + + return parsed; + } + + public static class Metric { + public Map tags = new TreeMap<>(); + public double value; + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("tags", tags).add("value", value).toString(); + } + + public boolean contains(String labelName, String labelValue) { + String value = tags.get(labelName); + return value != null && value.equals(labelValue); + } + } + + public static class Metrics { + final Multimap nameToDataPoints; + + public Metrics(Multimap nameToDataPoints) { + this.nameToDataPoints = nameToDataPoints; + } + + public List findByNameAndLabels(String metricName, String labelName, String labelValue) { + return nameToDataPoints.get(metricName) + .stream() + .filter(metric -> metric.contains(labelName, labelValue)) + .toList(); + } + + @SafeVarargs + public final List findByNameAndLabels(String metricName, Pair... nameValuePairs) { + return nameToDataPoints.get(metricName) + .stream() + .filter(metric -> { + for (Pair nameValuePair : nameValuePairs) { + String labelName = nameValuePair.getLeft(); + String labelValue = nameValuePair.getRight(); + if (!metric.contains(labelName, labelValue)) { + return false; + } + } + return true; + }) + .toList(); + } + + @SafeVarargs + public final Metric findSingleMetricByNameAndLabels(String metricName, Pair... nameValuePairs) { + List metricByNameAndLabels = findByNameAndLabels(metricName, nameValuePairs); + if (metricByNameAndLabels.size() != 1) { + fail("Expected to find 1 metric, but found the following: "+metricByNameAndLabels + + ". Metrics are = "+nameToDataPoints.get(metricName)+". Labels requested = "+ Arrays.toString( + nameValuePairs)); + } + return metricByNameAndLabels.get(0); + } + } +} diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/web/GzipHandlerUtilTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/web/GzipHandlerUtilTest.java new file mode 100644 index 0000000000000..d6958695dec9f --- /dev/null +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/web/GzipHandlerUtilTest.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.web; + +import static org.testng.Assert.*; +import java.util.Arrays; +import org.testng.annotations.Test; + +public class GzipHandlerUtilTest { + + @Test + public void testIsGzipCompressionEnabledForEndpoint() { + assertTrue(GzipHandlerUtil.isGzipCompressionEnabledForEndpoint(null, "/metrics")); + assertFalse(GzipHandlerUtil.isGzipCompressionEnabledForEndpoint(Arrays.asList("^.*"), "/metrics")); + assertFalse(GzipHandlerUtil.isGzipCompressionEnabledForEndpoint(Arrays.asList("^.*$"), "/metrics")); + assertFalse(GzipHandlerUtil.isGzipCompressionEnabledForEndpoint(Arrays.asList("/metrics"), "/metrics")); + assertTrue(GzipHandlerUtil.isGzipCompressionEnabledForEndpoint(Arrays.asList("/metrics"), "/metrics2")); + assertTrue(GzipHandlerUtil.isGzipCompressionEnabledForEndpoint(Arrays.asList("/admin", "/custom"), "/metrics")); + } +} \ No newline at end of file diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletUtilsTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletUtilsTest.java index 819d0b2f711b5..6cfab9d560cfa 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletUtilsTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/web/plugin/servlet/AdditionalServletUtilsTest.java @@ -68,7 +68,7 @@ public void testLoadEventListener() throws Exception { } @Test(expectedExceptions = IOException.class) - public void testLoadEventListenerWithBlankListerClass() throws Exception { + public void testLoadEventListenerWithBlankListenerClass() throws Exception { AdditionalServletDefinition def = new AdditionalServletDefinition(); def.setDescription("test-proxy-listener"); @@ -95,7 +95,7 @@ public void testLoadEventListenerWithBlankListerClass() throws Exception { } @Test(expectedExceptions = IOException.class) - public void testLoadEventListenerWithWrongListerClass() throws Exception { + public void testLoadEventListenerWithWrongListenerClass() throws Exception { AdditionalServletDefinition def = new AdditionalServletDefinition(); def.setAdditionalServletClass(Runnable.class.getName()); def.setDescription("test-proxy-listener"); diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryTest.java index bf91dab14fe1f..019627f52cbcf 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryTest.java @@ -26,6 +26,7 @@ import java.util.List; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLHandshakeException; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.http.client.methods.HttpGet; import org.apache.http.config.RegistryBuilder; @@ -41,24 +42,32 @@ import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.testng.annotations.Test; +import org.apache.pulsar.common.util.DefaultPulsarSslFactory; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; @Slf4j public class JettySslContextFactoryTest { @Test public void testJettyTlsServerTls() throws Exception { + @Cleanup("stop") Server server = new Server(); List connectors = new ArrayList<>(); - SslContextFactory factory = JettySslContextFactory.createServerSslContext( - null, - false, - Resources.getResource("ssl/my-ca/ca.pem").getPath(), - Resources.getResource("ssl/my-ca/server-ca.pem").getPath(), - Resources.getResource("ssl/my-ca/server-key.pem").getPath(), - true, - null, - null, - 600); + PulsarSslConfiguration sslConfiguration = PulsarSslConfiguration.builder() + .tlsTrustCertsFilePath(Resources.getResource("ssl/my-ca/ca.pem").getPath()) + .tlsCertificateFilePath(Resources.getResource("ssl/my-ca/server-ca.pem").getPath()) + .tlsKeyFilePath(Resources.getResource("ssl/my-ca/server-key.pem").getPath()) + .allowInsecureConnection(false) + .requireTrustedClientCertOnConnect(true) + .tlsEnabledWithKeystore(false) + .isHttps(true) + .build(); + PulsarSslFactory sslFactory = new DefaultPulsarSslFactory(); + sslFactory.initialize(sslConfiguration); + sslFactory.createInternalSslContext(); + SslContextFactory factory = JettySslContextFactory.createSslContextFactory(null, + sslFactory, true, null, null); ServerConnector connector = new ServerConnector(server, factory); connector.setPort(0); @@ -72,31 +81,41 @@ public void testJettyTlsServerTls() throws Exception { new SSLConnectionSocketFactory(getClientSslContext(), new NoopHostnameVerifier())); PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registryBuilder.build()); httpClientBuilder.setConnectionManager(cm); + @Cleanup CloseableHttpClient httpClient = httpClientBuilder.build(); HttpGet httpGet = new HttpGet("https://localhost:" + connector.getLocalPort()); httpClient.execute(httpGet); - httpClient.close(); - server.stop(); } @Test(expectedExceptions = SSLHandshakeException.class) public void testJettyTlsServerInvalidTlsProtocol() throws Exception { + @Cleanup("stop") Server server = new Server(); List connectors = new ArrayList<>(); - SslContextFactory factory = JettySslContextFactory.createServerSslContext( - null, - false, - Resources.getResource("ssl/my-ca/ca.pem").getPath(), - Resources.getResource("ssl/my-ca/server-ca.pem").getPath(), - Resources.getResource("ssl/my-ca/server-key.pem").getPath(), - true, - null, + PulsarSslConfiguration sslConfiguration = PulsarSslConfiguration.builder() + .tlsProtocols(new HashSet() { + { + this.add("TLSv1.3"); + } + }) + .tlsTrustCertsFilePath(Resources.getResource("ssl/my-ca/ca.pem").getPath()) + .tlsCertificateFilePath(Resources.getResource("ssl/my-ca/server-ca.pem").getPath()) + .tlsKeyFilePath(Resources.getResource("ssl/my-ca/server-key.pem").getPath()) + .allowInsecureConnection(false) + .requireTrustedClientCertOnConnect(true) + .tlsEnabledWithKeystore(false) + .isHttps(true) + .build(); + PulsarSslFactory sslFactory = new DefaultPulsarSslFactory(); + sslFactory.initialize(sslConfiguration); + sslFactory.createInternalSslContext(); + SslContextFactory factory = JettySslContextFactory.createSslContextFactory(null, + sslFactory, true, null, new HashSet() { { this.add("TLSv1.3"); } - }, - 600); + }); factory.setHostnameVerifier((s, sslSession) -> true); ServerConnector connector = new ServerConnector(server, factory); connector.setPort(0); @@ -110,24 +129,41 @@ public void testJettyTlsServerInvalidTlsProtocol() throws Exception { new String[]{"TLSv1.2"}, null, new NoopHostnameVerifier())); PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registryBuilder.build()); httpClientBuilder.setConnectionManager(cm); + @Cleanup CloseableHttpClient httpClient = httpClientBuilder.build(); HttpGet httpGet = new HttpGet("https://localhost:" + connector.getLocalPort()); httpClient.execute(httpGet); - httpClient.close(); - server.stop(); } @Test(expectedExceptions = SSLHandshakeException.class) public void testJettyTlsServerInvalidCipher() throws Exception { + @Cleanup("stop") Server server = new Server(); List connectors = new ArrayList<>(); - SslContextFactory factory = JettySslContextFactory.createServerSslContext( - null, - false, - Resources.getResource("ssl/my-ca/ca.pem").getPath(), - Resources.getResource("ssl/my-ca/server-ca.pem").getPath(), - Resources.getResource("ssl/my-ca/server-key.pem").getPath(), - true, + PulsarSslConfiguration sslConfiguration = PulsarSslConfiguration.builder() + .tlsCiphers(new HashSet() { + { + this.add("TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"); + } + }) + .tlsProtocols(new HashSet() { + { + this.add("TLSv1.3"); + } + }) + .tlsTrustCertsFilePath(Resources.getResource("ssl/my-ca/ca.pem").getPath()) + .tlsCertificateFilePath(Resources.getResource("ssl/my-ca/server-ca.pem").getPath()) + .tlsKeyFilePath(Resources.getResource("ssl/my-ca/server-key.pem").getPath()) + .allowInsecureConnection(false) + .requireTrustedClientCertOnConnect(true) + .isHttps(true) + .tlsEnabledWithKeystore(false) + .build(); + PulsarSslFactory sslFactory = new DefaultPulsarSslFactory(); + sslFactory.initialize(sslConfiguration); + sslFactory.createInternalSslContext(); + SslContextFactory factory = JettySslContextFactory.createSslContextFactory(null, + sslFactory, true, new HashSet() { { this.add("TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"); @@ -135,11 +171,9 @@ public void testJettyTlsServerInvalidCipher() throws Exception { }, new HashSet() { { - this.add("TLSv1.2"); + this.add("TLSv1.3"); } - }, - 600); - + }); factory.setHostnameVerifier((s, sslSession) -> true); ServerConnector connector = new ServerConnector(server, factory); connector.setPort(0); @@ -154,11 +188,10 @@ public void testJettyTlsServerInvalidCipher() throws Exception { new NoopHostnameVerifier())); PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registryBuilder.build()); httpClientBuilder.setConnectionManager(cm); + @Cleanup CloseableHttpClient httpClient = httpClientBuilder.build(); HttpGet httpGet = new HttpGet("https://localhost:" + connector.getLocalPort()); httpClient.execute(httpGet); - httpClient.close(); - server.stop(); } private static SSLContext getClientSslContext() throws GeneralSecurityException, IOException { diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryWithKeyStoreTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryWithKeyStoreTest.java index 1d41cd3684124..30fbc50257d4c 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryWithKeyStoreTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryWithKeyStoreTest.java @@ -30,6 +30,7 @@ import javax.net.ssl.SSLContext; import javax.net.ssl.SSLHandshakeException; import javax.net.ssl.TrustManagerFactory; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.http.client.methods.HttpGet; import org.apache.http.config.RegistryBuilder; @@ -42,6 +43,9 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; +import org.apache.pulsar.common.util.DefaultPulsarSslFactory; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.util.ssl.SslContextFactory; @@ -62,12 +66,25 @@ public class JettySslContextFactoryWithKeyStoreTest { @Test public void testJettyTlsServerTls() throws Exception { + @Cleanup("stop") Server server = new Server(); List connectors = new ArrayList<>(); - SslContextFactory.Server factory = JettySslContextFactory.createServerSslContextWithKeystore(null, - keyStoreType, brokerKeyStorePath, keyStorePassword, false, keyStoreType, - clientTrustStorePath, keyStorePassword, true, null, - null, 600); + PulsarSslConfiguration sslConfiguration = PulsarSslConfiguration.builder() + .tlsKeyStoreType(keyStoreType) + .tlsKeyStorePath(brokerKeyStorePath) + .tlsKeyStorePassword(keyStorePassword) + .tlsTrustStoreType(keyStoreType) + .tlsTrustStorePath(clientTrustStorePath) + .tlsTrustStorePassword(keyStorePassword) + .requireTrustedClientCertOnConnect(true) + .tlsEnabledWithKeystore(true) + .isHttps(true) + .build(); + PulsarSslFactory sslFactory = new DefaultPulsarSslFactory(); + sslFactory.initialize(sslConfiguration); + sslFactory.createInternalSslContext(); + SslContextFactory.Server factory = JettySslContextFactory.createSslContextFactory(null, + sslFactory, true, null, null); factory.setHostnameVerifier((s, sslSession) -> true); ServerConnector connector = new ServerConnector(server, factory); connector.setPort(0); @@ -81,26 +98,44 @@ public void testJettyTlsServerTls() throws Exception { new SSLConnectionSocketFactory(getClientSslContext(), new NoopHostnameVerifier())); PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registryBuilder.build()); httpClientBuilder.setConnectionManager(cm); + @Cleanup CloseableHttpClient httpClient = httpClientBuilder.build(); HttpGet httpGet = new HttpGet("https://localhost:" + connector.getLocalPort()); httpClient.execute(httpGet); - httpClient.close(); - server.stop(); } @Test(expectedExceptions = SSLHandshakeException.class) public void testJettyTlsServerInvalidTlsProtocol() throws Exception { Configurator.setRootLevel(Level.INFO); + @Cleanup("stop") Server server = new Server(); List connectors = new ArrayList<>(); - SslContextFactory.Server factory = JettySslContextFactory.createServerSslContextWithKeystore(null, - keyStoreType, brokerKeyStorePath, keyStorePassword, false, keyStoreType, clientTrustStorePath, - keyStorePassword, true, null, + PulsarSslConfiguration sslConfiguration = PulsarSslConfiguration.builder() + .tlsKeyStoreType(keyStoreType) + .tlsKeyStorePath(brokerKeyStorePath) + .tlsKeyStorePassword(keyStorePassword) + .tlsTrustStoreType(keyStoreType) + .tlsTrustStorePath(clientTrustStorePath) + .tlsTrustStorePassword(keyStorePassword) + .tlsProtocols(new HashSet() { + { + this.add("TLSv1.3"); + } + }) + .requireTrustedClientCertOnConnect(true) + .tlsEnabledWithKeystore(true) + .isHttps(true) + .build(); + PulsarSslFactory sslFactory = new DefaultPulsarSslFactory(); + sslFactory.initialize(sslConfiguration); + sslFactory.createInternalSslContext(); + SslContextFactory.Server factory = JettySslContextFactory.createSslContextFactory(null, + sslFactory, true, null, new HashSet() { { this.add("TLSv1.3"); } - }, 600); + }); factory.setHostnameVerifier((s, sslSession) -> true); ServerConnector connector = new ServerConnector(server, factory); connector.setPort(0); @@ -114,20 +149,44 @@ public void testJettyTlsServerInvalidTlsProtocol() throws Exception { new String[]{"TLSv1.2"}, null, new NoopHostnameVerifier())); PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registryBuilder.build()); httpClientBuilder.setConnectionManager(cm); + @Cleanup CloseableHttpClient httpClient = httpClientBuilder.build(); HttpGet httpGet = new HttpGet("https://localhost:" + connector.getLocalPort()); httpClient.execute(httpGet); - httpClient.close(); - server.stop(); } @Test(expectedExceptions = SSLHandshakeException.class) public void testJettyTlsServerInvalidCipher() throws Exception { + @Cleanup("stop") Server server = new Server(); List connectors = new ArrayList<>(); - SslContextFactory.Server factory = JettySslContextFactory.createServerSslContextWithKeystore(null, - keyStoreType, brokerKeyStorePath, keyStorePassword, false, keyStoreType, clientTrustStorePath, - keyStorePassword, true, new HashSet() { + PulsarSslConfiguration sslConfiguration = PulsarSslConfiguration.builder() + .tlsKeyStoreType(keyStoreType) + .tlsKeyStorePath(brokerKeyStorePath) + .tlsKeyStorePassword(keyStorePassword) + .tlsTrustStoreType(keyStoreType) + .tlsTrustStorePath(clientTrustStorePath) + .tlsTrustStorePassword(keyStorePassword) + .tlsCiphers(new HashSet() { + { + this.add("TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"); + } + }) + .tlsProtocols(new HashSet() { + { + this.add("TLSv1.3"); + } + }) + .requireTrustedClientCertOnConnect(true) + .tlsEnabledWithKeystore(true) + .isHttps(true) + .build(); + PulsarSslFactory sslFactory = new DefaultPulsarSslFactory(); + sslFactory.initialize(sslConfiguration); + sslFactory.createInternalSslContext(); + SslContextFactory.Server factory = JettySslContextFactory.createSslContextFactory(null, + sslFactory, true, + new HashSet() { { this.add("TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"); } @@ -135,8 +194,9 @@ public void testJettyTlsServerInvalidCipher() throws Exception { new HashSet() { { this.add("TLSv1.2"); + this.add("TLSv1.3"); } - }, 600); + }); factory.setHostnameVerifier((s, sslSession) -> true); ServerConnector connector = new ServerConnector(server, factory); connector.setPort(0); @@ -151,11 +211,10 @@ public void testJettyTlsServerInvalidCipher() throws Exception { new NoopHostnameVerifier())); PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registryBuilder.build()); httpClientBuilder.setConnectionManager(cm); + @Cleanup CloseableHttpClient httpClient = httpClientBuilder.build(); HttpGet httpGet = new HttpGet("https://localhost:" + connector.getLocalPort()); httpClient.execute(httpGet); - httpClient.close(); - server.stop(); } private static SSLContext getClientSslContext() { diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index b327375613c59..ee22762719175 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-broker @@ -82,6 +81,12 @@ ${project.version} + + ${project.groupId} + pulsar-cli-utils + ${project.version} + + ${project.groupId} managed-ledger @@ -137,6 +142,12 @@ ${project.version} + + ${project.groupId} + pulsar-opentelemetry + ${project.version} + + ${project.groupId} pulsar-io-batch-discovery-triggerers @@ -151,6 +162,26 @@ test + + com.github.tomakehurst + wiremock-jre8 + ${wiremock.version} + test + + + + io.github.hakky54 + consolecaptor + ${consolecaptor.version} + test + + + + io.streamnative.oxia + oxia-testcontainers + test + + io.dropwizard.metrics @@ -190,6 +221,14 @@ test + + ${project.groupId} + pulsar-broker-common + ${project.version} + test-jar + test + + @@ -277,13 +316,26 @@ - com.beust - jcommander + ${project.groupId} + pulsar-docs-tools + ${project.version} + + + io.swagger + * + + + + + + info.picocli + picocli io.swagger swagger-annotations + provided @@ -309,6 +361,7 @@ io.swagger swagger-core + provided @@ -403,6 +456,12 @@ javax.activation + + io.rest-assured + rest-assured + test + + @@ -419,6 +478,18 @@ ${project.version} + + ${project.groupId} + jetcd-core-shaded + ${project.version} + shaded + test + + + io.grpc + grpc-netty-shaded + test + io.etcd jetcd-test @@ -430,7 +501,6 @@ pulsar-package-filesystem-storage ${project.version} - @@ -561,6 +631,7 @@ protobuf-maven-plugin ${protobuf-maven-plugin.version} + com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} true @@ -579,30 +650,6 @@ - - org.codehaus.mojo - properties-maven-plugin - - - initialize - - set-system-properties - - - - - proto_path - ${project.parent.basedir} - - - proto_search_strategy - 2 - - - - - - com.github.splunk.lightproto lightproto-maven-plugin diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java index 92fc8c5c9acfa..2d031cc8a74f6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java @@ -23,9 +23,6 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.pulsar.common.configuration.PulsarConfigurationLoader.create; import static org.apache.pulsar.common.configuration.PulsarConfigurationLoader.isComplete; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; import com.google.common.annotations.VisibleForTesting; import java.io.File; import java.io.FileInputStream; @@ -34,10 +31,10 @@ import java.nio.file.Path; import java.text.DateFormat; import java.text.SimpleDateFormat; -import java.util.Arrays; import java.util.Date; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.common.component.ComponentStarter; import org.apache.bookkeeper.common.component.LifecycleComponent; @@ -55,15 +52,21 @@ import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm; import org.apache.pulsar.common.protocol.Commands; -import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.DirectMemoryUtils; import org.apache.pulsar.common.util.ShutdownUtil; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerService; import org.apache.pulsar.functions.worker.service.WorkerServiceLoader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import picocli.CommandLine; +import picocli.CommandLine.ArgGroup; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.ScopeType; +@Command(description = "broker", showDefaultValues = true, scope = ScopeType.INHERIT) public class PulsarBrokerStarter { private static ServiceConfiguration loadConfig(String configFile) throws Exception { @@ -76,31 +79,30 @@ private static ServiceConfiguration loadConfig(String configFile) throws Excepti } @VisibleForTesting - @Parameters(commandDescription = "Options") private static class StarterArguments { - @Parameter(names = {"-c", "--broker-conf"}, description = "Configuration file for Broker") + @Option(names = {"-c", "--broker-conf"}, description = "Configuration file for Broker") private String brokerConfigFile = "conf/broker.conf"; - @Parameter(names = {"-rb", "--run-bookie"}, description = "Run Bookie together with Broker") + @Option(names = {"-rb", "--run-bookie"}, description = "Run Bookie together with Broker") private boolean runBookie = false; - @Parameter(names = {"-ra", "--run-bookie-autorecovery"}, + @Option(names = {"-ra", "--run-bookie-autorecovery"}, description = "Run Bookie Autorecovery together with broker") private boolean runBookieAutoRecovery = false; - @Parameter(names = {"-bc", "--bookie-conf"}, description = "Configuration file for Bookie") + @Option(names = {"-bc", "--bookie-conf"}, description = "Configuration file for Bookie") private String bookieConfigFile = "conf/bookkeeper.conf"; - @Parameter(names = {"-rfw", "--run-functions-worker"}, description = "Run functions worker with Broker") + @Option(names = {"-rfw", "--run-functions-worker"}, description = "Run functions worker with Broker") private boolean runFunctionsWorker = false; - @Parameter(names = {"-fwc", "--functions-worker-conf"}, description = "Configuration file for Functions Worker") + @Option(names = {"-fwc", "--functions-worker-conf"}, description = "Configuration file for Functions Worker") private String fnWorkerConfigFile = "conf/functions_worker.yml"; - @Parameter(names = {"-h", "--help"}, description = "Show this help message") + @Option(names = {"-h", "--help"}, usageHelp = true, description = "Show this help message") private boolean help = false; - @Parameter(names = {"-g", "--generate-docs"}, description = "Generate docs") + @Option(names = {"-g", "--generate-docs"}, description = "Generate docs") private boolean generateDocs = false; } @@ -125,43 +127,46 @@ private static ServerConfiguration readBookieConfFile(String bookieConfigFile) t return bookieConf; } - private static boolean argsContains(String[] args, String arg) { - return Arrays.asList(args).contains(arg); - } - - private static class BrokerStarter { - private final ServiceConfiguration brokerConfig; - private final PulsarService pulsarService; - private final LifecycleComponent bookieServer; + protected static class BrokerStarter implements Callable { + private ServiceConfiguration brokerConfig; + private PulsarService pulsarService; + private LifecycleComponent bookieServer; private volatile CompletableFuture bookieStartFuture; - private final AutoRecoveryMain autoRecoveryMain; - private final StatsProvider bookieStatsProvider; - private final ServerConfiguration bookieConfig; - private final WorkerService functionsWorkerService; - private final WorkerConfig workerConfig; - - BrokerStarter(String[] args) throws Exception { - StarterArguments starterArguments = new StarterArguments(); - JCommander jcommander = new JCommander(starterArguments); - jcommander.setProgramName("PulsarBrokerStarter"); - - // parse args by JCommander - jcommander.parse(args); + private AutoRecoveryMain autoRecoveryMain; + private StatsProvider bookieStatsProvider; + private ServerConfiguration bookieConfig; + private WorkerService functionsWorkerService; + private WorkerConfig workerConfig; + + private CommandLine commander; + + @ArgGroup(exclusive = false) + private final StarterArguments starterArguments = new StarterArguments(); + + BrokerStarter() { + commander = new CommandLine(this); + } + + public int start(String[] args) { + return commander.execute(args); + } + + public Integer call() throws Exception { if (starterArguments.help) { - jcommander.usage(); - System.exit(0); + commander.usage(commander.getOut()); + return 0; } if (starterArguments.generateDocs) { CmdGenerateDocs cmd = new CmdGenerateDocs("pulsar"); - cmd.addCommand("broker", starterArguments); + cmd.addCommand("broker", commander); cmd.run(null); - System.exit(0); + return 0; } // init broker config if (isBlank(starterArguments.brokerConfigFile)) { - jcommander.usage(); + commander.usage(commander.getOut()); throw new IllegalArgumentException("Need to specify a configuration file for broker"); } else { final String filepath = Path.of(starterArguments.brokerConfigFile) @@ -209,20 +214,16 @@ private static class BrokerStarter { }); // if no argument to run bookie in cmd line, read from pulsar config - if (!argsContains(args, "-rb") && !argsContains(args, "--run-bookie")) { - checkState(!starterArguments.runBookie, - "runBookie should be false if has no argument specified"); + if (!starterArguments.runBookie) { starterArguments.runBookie = brokerConfig.isEnableRunBookieTogether(); } - if (!argsContains(args, "-ra") && !argsContains(args, "--run-bookie-autorecovery")) { - checkState(!starterArguments.runBookieAutoRecovery, - "runBookieAutoRecovery should be false if has no argument specified"); + if (!starterArguments.runBookieAutoRecovery) { starterArguments.runBookieAutoRecovery = brokerConfig.isEnableRunBookieAutoRecoveryTogether(); } if ((starterArguments.runBookie || starterArguments.runBookieAutoRecovery) - && isBlank(starterArguments.bookieConfigFile)) { - jcommander.usage(); + && isBlank(starterArguments.bookieConfigFile)) { + commander.usage(commander.getOut()); throw new IllegalArgumentException("No configuration file for Bookie"); } @@ -257,9 +258,7 @@ && isBlank(starterArguments.bookieConfigFile)) { } else { autoRecoveryMain = null; } - } - public void start() throws Exception { if (bookieStatsProvider != null) { bookieStatsProvider.start(bookieConfig); log.info("started bookieStatsProvider."); @@ -275,15 +274,17 @@ public void start() throws Exception { pulsarService.start(); log.info("PulsarService started."); + return 0; } public void join() throws InterruptedException { - pulsarService.waitUntilClosed(); - - try { - pulsarService.close(); - } catch (PulsarServerException e) { - throw new RuntimeException(); + if (pulsarService != null) { + pulsarService.waitUntilClosed(); + try { + pulsarService.close(); + } catch (PulsarServerException e) { + throw new RuntimeException(); + } } if (bookieStartFuture != null) { @@ -301,8 +302,10 @@ public void shutdown() throws Exception { log.info("Shut down functions worker service successfully."); } - pulsarService.close(); - log.info("Shut down broker service successfully."); + if (pulsarService != null) { + pulsarService.close(); + log.info("Shut down broker service successfully."); + } if (bookieStatsProvider != null) { bookieStatsProvider.stop(); @@ -317,6 +320,11 @@ public void shutdown() throws Exception { log.info("Shut down autoRecoveryMain successfully."); } } + + @VisibleForTesting + CommandLine getCommander() { + return commander; + } } @@ -330,7 +338,7 @@ public static void main(String[] args) throws Exception { exception.printStackTrace(System.out); }); - BrokerStarter starter = new BrokerStarter(args); + BrokerStarter starter = new BrokerStarter(); Runtime.getRuntime().addShutdownHook( new Thread(() -> { try { @@ -344,16 +352,21 @@ public static void main(String[] args) throws Exception { ); PulsarByteBufAllocator.registerOOMListener(oomException -> { - if (starter.brokerConfig.isSkipBrokerShutdownOnOOM()) { + if (starter.brokerConfig != null && starter.brokerConfig.isSkipBrokerShutdownOnOOM()) { log.error("-- Received OOM exception: {}", oomException.getMessage(), oomException); } else { log.error("-- Shutting down - Received OOM exception: {}", oomException.getMessage(), oomException); - starter.pulsarService.shutdownNow(); + if (starter.pulsarService != null) { + starter.pulsarService.shutdownNow(); + } } }); try { - starter.start(); + int start = starter.start(args); + if (start != 0) { + System.exit(start); + } } catch (Throwable t) { log.error("Failed to start pulsar service.", t); ShutdownUtil.triggerImmediateForcefulShutdown(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java index 9b757c55ccd1d..96ea8877c5b61 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java @@ -19,8 +19,6 @@ package org.apache.pulsar; import static org.apache.pulsar.common.policies.data.PoliciesUtil.getBundles; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; import java.io.IOException; import java.util.Collections; import java.util.Optional; @@ -44,8 +42,8 @@ import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.TenantInfoImpl; -import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.ShutdownUtil; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.apache.pulsar.functions.worker.WorkerUtils; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.MetadataStoreConfig; @@ -58,6 +56,10 @@ import org.apache.pulsar.metadata.impl.ZKMetadataStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.ScopeType; /** * Setup the metadata for a new Pulsar cluster. @@ -66,75 +68,98 @@ public class PulsarClusterMetadataSetup { private static final int DEFAULT_BUNDLE_NUMBER = 16; + @Command(name = "initialize-cluster-metadata", showDefaultValues = true, scope = ScopeType.INHERIT) private static class Arguments { - @Parameter(names = { "-c", "--cluster" }, description = "Cluster name", required = true) + @Option(names = {"-c", "--cluster"}, description = "Cluster name", required = true) private String cluster; - @Parameter(names = {"-bn", + @Option(names = {"-bn", "--default-namespace-bundle-number"}, description = "The bundle numbers for the default namespaces(public/default), default is 16", required = false) private int numberOfDefaultNamespaceBundles; - @Parameter(names = { "-uw", - "--web-service-url" }, description = "Web-service URL for new cluster", required = true) + @Option(names = {"-uw", + "--web-service-url"}, description = "Web-service URL for new cluster", required = true) private String clusterWebServiceUrl; - @Parameter(names = {"-tw", + @Option(names = {"-tw", "--web-service-url-tls"}, description = "Web-service URL for new cluster with TLS encryption", required = false) private String clusterWebServiceUrlTls; - @Parameter(names = { "-ub", - "--broker-service-url" }, description = "Broker-service URL for new cluster", required = false) + @Option(names = {"-ub", + "--broker-service-url"}, description = "Broker-service URL for new cluster", required = false) private String clusterBrokerServiceUrl; - @Parameter(names = {"-tb", + @Option(names = {"-tb", "--broker-service-url-tls"}, description = "Broker-service URL for new cluster with TLS encryption", required = false) private String clusterBrokerServiceUrlTls; - @Parameter(names = { "-zk", - "--zookeeper" }, description = "Local ZooKeeper quorum connection string", + @Option(names = {"-te", + "--tls-enable"}, + description = "Enable TLS connection for new cluster") + private Boolean clusterBrokerClientTlsEnabled; + + @Option(names = "--auth-plugin", + description = "The authentication plugin for new cluster") + protected String clusterAuthenticationPlugin; + + @Option(names = "--auth-parameters", + description = "The authentication parameters for new cluster") + protected String clusterAuthenticationParameters; + + @Option(names = {"-zk", + "--zookeeper"}, description = "Local ZooKeeper quorum connection string", required = false, hidden = true - ) + ) private String zookeeper; - @Parameter(names = { "-md", - "--metadata-store" }, description = "Metadata Store service url. eg: zk:my-zk:2181", required = false) + @Option(names = {"-md", + "--metadata-store"}, description = "Metadata Store service url. eg: zk:my-zk:2181", required = false) private String metadataStoreUrl; - @Parameter(names = { - "--zookeeper-session-timeout-ms" + @Option(names = { + "--zookeeper-session-timeout-ms" }, description = "Local zookeeper session timeout ms") private int zkSessionTimeoutMillis = 30000; - @Parameter(names = {"-gzk", + @Option(names = {"-gzk", "--global-zookeeper"}, description = "Global ZooKeeper quorum connection string", required = false, hidden = true) private String globalZookeeper; - @Parameter(names = {"-cs", + @Option(names = {"-cs", "--configuration-store"}, description = "Configuration Store connection string", hidden = true) private String configurationStore; - @Parameter(names = {"-cms", + @Option(names = {"-cms", "--configuration-metadata-store"}, description = "Configuration Metadata Store connection string", hidden = false) private String configurationMetadataStore; - @Parameter(names = { - "--initial-num-stream-storage-containers" + @Option(names = {"-mscp", + "--metadata-store-config-path"}, description = "Metadata Store config path", hidden = false) + private String metadataStoreConfigPath; + + @Option(names = {"-cmscp", + "--configuration-metadata-store-config-path"}, description = "Configuration Metadata Store config path", + hidden = false) + private String configurationStoreConfigPath; + + @Option(names = { + "--initial-num-stream-storage-containers" }, description = "Num storage containers of BookKeeper stream storage") private int numStreamStorageContainers = 16; - @Parameter(names = { + @Option(names = { "--initial-num-transaction-coordinators" }, description = "Num transaction coordinators will assigned in cluster") private int numTransactionCoordinators = 16; - @Parameter(names = { + @Option(names = { "--existing-bk-metadata-service-uri"}, description = "The metadata service URI of the existing BookKeeper cluster that you want to use") private String existingBkMetadataServiceUri; @@ -142,26 +167,26 @@ private static class Arguments { // Hide and marked as deprecated this flag because we use the new name '--existing-bk-metadata-service-uri' to // pass the service url. For compatibility of the command, we should keep both to avoid the exceptions. @Deprecated - @Parameter(names = { - "--bookkeeper-metadata-service-uri"}, - description = "The metadata service URI of the existing BookKeeper cluster that you want to use", - hidden = true) + @Option(names = { + "--bookkeeper-metadata-service-uri"}, + description = "The metadata service URI of the existing BookKeeper cluster that you want to use", + hidden = true) private String bookieMetadataServiceUri; - @Parameter(names = { "-pp", - "--proxy-protocol" }, + @Option(names = {"-pp", + "--proxy-protocol"}, description = "Proxy protocol to select type of routing at proxy. Possible Values: [SNI]", required = false) private ProxyProtocol clusterProxyProtocol; - @Parameter(names = { "-pu", - "--proxy-url" }, description = "Proxy-server URL to which to connect.", required = false) + @Option(names = {"-pu", + "--proxy-url"}, description = "Proxy-server URL to which to connect.", required = false) private String clusterProxyUrl; - @Parameter(names = { "-h", "--help" }, description = "Show this help message") + @Option(names = {"-h", "--help"}, usageHelp = true, description = "Show this help message") private boolean help = false; - @Parameter(names = {"-g", "--generate-docs"}, description = "Generate docs") + @Option(names = {"-g", "--generate-docs"}, description = "Generate docs") private boolean generateDocs = false; } @@ -197,28 +222,27 @@ public static void main(String[] args) throws Exception { System.setProperty("bookkeeper.metadata.client.drivers", PulsarMetadataClientDriver.class.getName()); Arguments arguments = new Arguments(); - JCommander jcommander = new JCommander(); + CommandLine commander = new CommandLine(arguments); try { - jcommander.addObject(arguments); - jcommander.parse(args); + commander.parseArgs(args); if (arguments.help) { - jcommander.usage(); + commander.usage(commander.getOut()); return; } if (arguments.generateDocs) { CmdGenerateDocs cmd = new CmdGenerateDocs("pulsar"); - cmd.addCommand("initialize-cluster-metadata", arguments); + cmd.addCommand("initialize-cluster-metadata", commander); cmd.run(null); return; } } catch (Exception e) { - jcommander.usage(); + commander.getErr().println(e); throw e; } if (arguments.metadataStoreUrl == null && arguments.zookeeper == null) { System.err.println("Metadata store address argument is required (--metadata-store)"); - jcommander.usage(); + commander.usage(commander.getOut()); System.exit(1); } @@ -226,7 +250,7 @@ public static void main(String[] args) throws Exception { && arguments.globalZookeeper == null) { System.err.println( "Configuration metadata store address argument is required (--configuration-metadata-store)"); - jcommander.usage(); + commander.usage(commander.getOut()); System.exit(1); } @@ -234,7 +258,7 @@ public static void main(String[] args) throws Exception { || arguments.globalZookeeper != null)) { System.err.println("Configuration metadata store argument (--configuration-metadata-store) " + "supersedes the deprecated (--global-zookeeper and --configuration-store) argument"); - jcommander.usage(); + commander.usage(commander.getOut()); System.exit(1); } @@ -268,9 +292,11 @@ private static void initializeCluster(Arguments arguments, int bundleNumberForDe log.info("Setting up cluster {} with metadata-store={} configuration-metadata-store={}", arguments.cluster, arguments.metadataStoreUrl, arguments.configurationMetadataStore); - MetadataStoreExtended localStore = - initLocalMetadataStore(arguments.metadataStoreUrl, arguments.zkSessionTimeoutMillis); + MetadataStoreExtended localStore = initLocalMetadataStore(arguments.metadataStoreUrl, + arguments.metadataStoreConfigPath, + arguments.zkSessionTimeoutMillis); MetadataStoreExtended configStore = initConfigMetadataStore(arguments.configurationMetadataStore, + arguments.configurationStoreConfigPath, arguments.zkSessionTimeoutMillis); final String metadataStoreUrlNoIdentifer = MetadataStoreFactoryImpl @@ -315,14 +341,36 @@ private static void initializeCluster(Arguments arguments, int bundleNumberForDe PulsarResources resources = new PulsarResources(localStore, configStore); - ClusterData clusterData = ClusterData.builder() - .serviceUrl(arguments.clusterWebServiceUrl) - .serviceUrlTls(arguments.clusterWebServiceUrlTls) - .brokerServiceUrl(arguments.clusterBrokerServiceUrl) - .brokerServiceUrlTls(arguments.clusterBrokerServiceUrlTls) - .proxyServiceUrl(arguments.clusterProxyUrl) - .proxyProtocol(arguments.clusterProxyProtocol) - .build(); + ClusterData.Builder clusterDataBuilder = ClusterData.builder(); + if (arguments.clusterWebServiceUrl != null) { + clusterDataBuilder.serviceUrl(arguments.clusterWebServiceUrl); + } + if (arguments.clusterWebServiceUrlTls != null) { + clusterDataBuilder.serviceUrlTls(arguments.clusterWebServiceUrlTls); + } + if (arguments.clusterBrokerServiceUrl != null) { + clusterDataBuilder.brokerServiceUrl(arguments.clusterBrokerServiceUrl); + } + if (arguments.clusterBrokerServiceUrlTls != null) { + clusterDataBuilder.brokerServiceUrlTls(arguments.clusterBrokerServiceUrlTls); + } + if (arguments.clusterBrokerClientTlsEnabled != null) { + clusterDataBuilder.brokerClientTlsEnabled(arguments.clusterBrokerClientTlsEnabled); + } + if (arguments.clusterAuthenticationPlugin != null) { + clusterDataBuilder.authenticationPlugin(arguments.clusterAuthenticationPlugin); + } + if (arguments.clusterAuthenticationParameters != null) { + clusterDataBuilder.authenticationParameters(arguments.clusterAuthenticationParameters); + } + if (arguments.clusterProxyUrl != null) { + clusterDataBuilder.proxyServiceUrl(arguments.clusterProxyUrl); + } + if (arguments.clusterProxyProtocol != null) { + clusterDataBuilder.proxyProtocol(arguments.clusterProxyProtocol); + } + + ClusterData clusterData = clusterDataBuilder.build(); if (!resources.getClusterResources().clusterExists(arguments.cluster)) { resources.getClusterResources().createCluster(arguments.cluster, clusterData); } @@ -356,8 +404,8 @@ private static void initializeCluster(Arguments arguments, int bundleNumberForDe log.info("Cluster metadata for '{}' setup correctly", arguments.cluster); } - static void createTenantIfAbsent(PulsarResources resources, String tenant, String cluster) throws IOException, - InterruptedException, ExecutionException { + public static void createTenantIfAbsent(PulsarResources resources, String tenant, String cluster) + throws IOException, InterruptedException, ExecutionException { TenantResources tenantResources = resources.getTenantResources(); @@ -373,8 +421,8 @@ static void createTenantIfAbsent(PulsarResources resources, String tenant, Strin } } - static void createNamespaceIfAbsent(PulsarResources resources, NamespaceName namespaceName, - String cluster, int bundleNumber) throws IOException { + public static void createNamespaceIfAbsent(PulsarResources resources, NamespaceName namespaceName, + String cluster, int bundleNumber) throws IOException { NamespaceResources namespaceResources = resources.getNamespaceResources(); if (!namespaceResources.namespaceExists(namespaceName)) { @@ -427,9 +475,17 @@ static void createPartitionedTopic(MetadataStore configStore, TopicName topicNam } } - public static MetadataStoreExtended initLocalMetadataStore(String connection, int sessionTimeout) throws Exception { + public static MetadataStoreExtended initLocalMetadataStore(String connection, + int sessionTimeout) throws Exception { + return initLocalMetadataStore(connection, null, sessionTimeout); + } + + public static MetadataStoreExtended initLocalMetadataStore(String connection, + String configPath, + int sessionTimeout) throws Exception { MetadataStoreExtended store = MetadataStoreExtended.create(connection, MetadataStoreConfig.builder() .sessionTimeoutMillis(sessionTimeout) + .configFilePath(configPath) .metadataStoreName(MetadataStoreConfig.METADATA_STORE) .build()); if (store instanceof MetadataStoreLifecycle) { @@ -438,10 +494,19 @@ public static MetadataStoreExtended initLocalMetadataStore(String connection, in return store; } - public static MetadataStoreExtended initConfigMetadataStore(String connection, int sessionTimeout) + public static MetadataStoreExtended initConfigMetadataStore(String connection, + int sessionTimeout) + throws Exception { + return initConfigMetadataStore(connection, null, sessionTimeout); + } + + public static MetadataStoreExtended initConfigMetadataStore(String connection, + String configPath, + int sessionTimeout) throws Exception { MetadataStoreExtended store = MetadataStoreExtended.create(connection, MetadataStoreConfig.builder() .sessionTimeoutMillis(sessionTimeout) + .configFilePath(configPath) .metadataStoreName(MetadataStoreConfig.CONFIGURATION_METADATA_STORE) .build()); if (store instanceof MetadataStoreLifecycle) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataTeardown.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataTeardown.java index cbbf909cdf7fa..a2984a352b9f8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataTeardown.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataTeardown.java @@ -18,8 +18,6 @@ */ package org.apache.pulsar; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; import com.google.protobuf.InvalidProtocolBufferException; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -33,43 +31,48 @@ import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; import org.apache.pulsar.broker.service.schema.SchemaStorageFormat.SchemaLocator; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.MetadataStoreFactory; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.ScopeType; /** * Teardown the metadata for a existed Pulsar cluster. */ public class PulsarClusterMetadataTeardown { + @Command(name = "delete-cluster-metadata", showDefaultValues = true, scope = ScopeType.INHERIT) private static class Arguments { - @Parameter(names = { "-zk", + @Option(names = { "-zk", "--zookeeper"}, description = "Local ZooKeeper quorum connection string", required = true) private String zookeeper; - @Parameter(names = { + @Option(names = { "--zookeeper-session-timeout-ms" }, description = "Local zookeeper session timeout ms") private int zkSessionTimeoutMillis = 30000; - @Parameter(names = { "-c", "-cluster", "--cluster" }, description = "Cluster name") + @Option(names = { "-c", "-cluster", "--cluster" }, description = "Cluster name") private String cluster; - @Parameter(names = { "-cs", "--configuration-store" }, description = "Configuration Store connection string") + @Option(names = { "-cs", "--configuration-store" }, description = "Configuration Store connection string") private String configurationStore; - @Parameter(names = { "--bookkeeper-metadata-service-uri" }, description = "Metadata service uri of BookKeeper") + @Option(names = { "--bookkeeper-metadata-service-uri" }, description = "Metadata service uri of BookKeeper") private String bkMetadataServiceUri; - @Parameter(names = { "-h", "--help" }, description = "Show this help message") + @Option(names = { "-h", "--help" }, description = "Show this help message") private boolean help = false; - @Parameter(names = {"-g", "--generate-docs"}, description = "Generate docs") + @Option(names = {"-g", "--generate-docs"}, description = "Generate docs") private boolean generateDocs = false; } @@ -78,22 +81,21 @@ private static class Arguments { public static void main(String[] args) throws Exception { Arguments arguments = new Arguments(); - JCommander jcommander = new JCommander(); + CommandLine commander = new CommandLine(arguments); try { - jcommander.addObject(arguments); - jcommander.parse(args); + commander.parseArgs(args); if (arguments.help) { - jcommander.usage(); + commander.usage(commander.getOut()); return; } if (arguments.generateDocs) { CmdGenerateDocs cmd = new CmdGenerateDocs("pulsar"); - cmd.addCommand("delete-cluster-metadata", arguments); + cmd.addCommand("delete-cluster-metadata", commander); cmd.run(null); return; } } catch (Exception e) { - jcommander.usage(); + commander.getErr().println(e); throw e; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarInitialNamespaceSetup.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarInitialNamespaceSetup.java index c4020546d1c6c..912f43958f469 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarInitialNamespaceSetup.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarInitialNamespaceSetup.java @@ -18,72 +18,82 @@ */ package org.apache.pulsar; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; import java.util.List; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.common.naming.NamespaceName; -import org.apache.pulsar.common.util.CmdGenerateDocs; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.apache.pulsar.metadata.api.MetadataStore; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; +import picocli.CommandLine.ScopeType; /** * Setup the initial namespace of the cluster without startup the Pulsar broker. */ public class PulsarInitialNamespaceSetup { + @Command(name = "initialize-namespace", showDefaultValues = true, scope = ScopeType.INHERIT) private static class Arguments { - @Parameter(names = { "-c", "--cluster" }, description = "Cluster name", required = true) + @Option(names = { "-c", "--cluster" }, description = "Cluster name", required = true) private String cluster; - @Parameter(names = { "-cs", + @Option(names = { "-cs", "--configuration-store" }, description = "Configuration Store connection string", required = true) private String configurationStore; - @Parameter(names = { + @Option(names = {"-cmscp", + "--configuration-metadata-store-config-path"}, description = "Configuration Metadata Store config path", + hidden = false) + private String configurationStoreConfigPath; + + @Option(names = { "--zookeeper-session-timeout-ms" }, description = "Local zookeeper session timeout ms") private int zkSessionTimeoutMillis = 30000; - @Parameter(description = "tenant/namespace", required = true) + @Parameters(description = "tenant/namespace", arity = "1") private List namespaces; - @Parameter(names = { "-h", "--help" }, description = "Show this help message") + @Option(names = { "-h", "--help" }, description = "Show this help message") private boolean help = false; - @Parameter(names = {"-g", "--generate-docs"}, description = "Generate docs") + @Option(names = {"-g", "--generate-docs"}, description = "Generate docs") private boolean generateDocs = false; } public static int doMain(String[] args) throws Exception { Arguments arguments = new Arguments(); - JCommander jcommander = new JCommander(); + CommandLine commander = new CommandLine(arguments); try { - jcommander.addObject(arguments); - jcommander.parse(args); + commander.parseArgs(args); if (arguments.help) { - jcommander.usage(); + commander.usage(commander.getOut()); return 0; } if (arguments.generateDocs) { CmdGenerateDocs cmd = new CmdGenerateDocs("pulsar"); - cmd.addCommand("initialize-namespace", arguments); + cmd.addCommand("initialize-namespace", commander); cmd.run(null); return 0; } } catch (Exception e) { - jcommander.usage(); + commander.getErr().println(e); return 1; } if (arguments.configurationStore == null) { System.err.println("Configuration store address argument is required (--configuration-store)"); - jcommander.usage(); + commander.usage(commander.getOut()); return 1; } - try (MetadataStore configStore = PulsarClusterMetadataSetup - .initConfigMetadataStore(arguments.configurationStore, arguments.zkSessionTimeoutMillis)) { + try (MetadataStore configStore = PulsarClusterMetadataSetup.initConfigMetadataStore( + arguments.configurationStore, + arguments.configurationStoreConfigPath, + arguments.zkSessionTimeoutMillis)) { PulsarResources pulsarResources = new PulsarResources(null, configStore); for (String namespace : arguments.namespaces) { NamespaceName namespaceName = null; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java index ba136e7c91058..d0118b06e7c05 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java @@ -18,13 +18,14 @@ */ package org.apache.pulsar; +import static org.apache.commons.io.FileUtils.cleanDirectory; import static org.apache.pulsar.common.naming.NamespaceName.SYSTEM_NAMESPACE; import static org.apache.pulsar.common.naming.SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN; -import com.beust.jcommander.Parameter; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; import io.netty.util.internal.PlatformDependent; import java.io.File; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; @@ -52,8 +53,12 @@ import org.apache.pulsar.metadata.impl.ZKMetadataStore; import org.apache.pulsar.packages.management.storage.filesystem.FileSystemPackagesStorageProvider; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.ScopeType; @Slf4j +@Command(name = "standalone", showDefaultValues = true, scope = ScopeType.INHERIT) public class PulsarStandalone implements AutoCloseable { private static final String PULSAR_STANDALONE_USE_ZOOKEEPER = "PULSAR_STANDALONE_USE_ZOOKEEPER"; @@ -211,60 +216,60 @@ public boolean isHelp() { return help; } - @Parameter(names = { "-c", "--config" }, description = "Configuration file path") + @Option(names = { "-c", "--config" }, description = "Configuration file path") private String configFile; - @Parameter(names = { "--wipe-data" }, description = "Clean up previous ZK/BK data") + @Option(names = { "--wipe-data" }, description = "Clean up previous ZK/BK data") private boolean wipeData = false; - @Parameter(names = { "--num-bookies" }, description = "Number of local Bookies") + @Option(names = { "--num-bookies" }, description = "Number of local Bookies") private int numOfBk = 1; - @Parameter(names = { "--metadata-dir" }, + @Option(names = { "--metadata-dir" }, description = "Directory for storing metadata") private String metadataDir = "data/metadata"; - @Parameter(names = { "--metadata-url" }, + @Option(names = { "--metadata-url" }, description = "Metadata store url") private String metadataStoreUrl = ""; - @Parameter(names = {"--zookeeper-port"}, description = "Local zookeeper's port", + @Option(names = {"--zookeeper-port"}, description = "Local zookeeper's port", hidden = true) private int zkPort = 2181; - @Parameter(names = { "--bookkeeper-port" }, description = "Local bookies base port") + @Option(names = { "--bookkeeper-port" }, description = "Local bookies base port") private int bkPort = 3181; - @Parameter(names = { "--zookeeper-dir" }, + @Option(names = { "--zookeeper-dir" }, description = "Local zooKeeper's data directory", hidden = true) private String zkDir = "data/standalone/zookeeper"; - @Parameter(names = { "--bookkeeper-dir" }, description = "Local bookies base data directory") + @Option(names = { "--bookkeeper-dir" }, description = "Local bookies base data directory") private String bkDir = "data/standalone/bookkeeper"; - @Parameter(names = { "--no-broker" }, description = "Only start ZK and BK services, no broker") + @Option(names = { "--no-broker" }, description = "Only start ZK and BK services, no broker") private boolean noBroker = false; - @Parameter(names = { "--only-broker" }, description = "Only start Pulsar broker service (no ZK, BK)") + @Option(names = { "--only-broker" }, description = "Only start Pulsar broker service (no ZK, BK)") private boolean onlyBroker = false; - @Parameter(names = {"-nfw", "--no-functions-worker"}, description = "Run functions worker with Broker") + @Option(names = {"-nfw", "--no-functions-worker"}, description = "Run functions worker with Broker") private boolean noFunctionsWorker = false; - @Parameter(names = {"-fwc", "--functions-worker-conf"}, description = "Configuration file for Functions Worker") + @Option(names = {"-fwc", "--functions-worker-conf"}, description = "Configuration file for Functions Worker") private String fnWorkerConfigFile = "conf/functions_worker.yml"; - @Parameter(names = {"-nss", "--no-stream-storage"}, description = "Disable stream storage") + @Option(names = {"-nss", "--no-stream-storage"}, description = "Disable stream storage") private boolean noStreamStorage = false; - @Parameter(names = { "--stream-storage-port" }, description = "Local bookies stream storage port") + @Option(names = { "--stream-storage-port" }, description = "Local bookies stream storage port") private int streamStoragePort = 4181; - @Parameter(names = { "-a", "--advertised-address" }, description = "Standalone broker advertised address") + @Option(names = { "-a", "--advertised-address" }, description = "Standalone broker advertised address") private String advertisedAddress = null; - @Parameter(names = { "-h", "--help" }, description = "Show this help message") + @Option(names = { "-h", "--help" }, description = "Show this help message") private boolean help = false; private boolean usingNewDefaultsPIP117; @@ -417,18 +422,22 @@ public void close() { try { if (fnWorkerService != null) { fnWorkerService.stop(); + fnWorkerService = null; } if (broker != null) { broker.close(); + broker = null; } if (bkCluster != null) { bkCluster.close(); + bkCluster = null; } if (bkEnsemble != null) { bkEnsemble.stop(); + bkEnsemble = null; } } catch (Exception e) { log.error("Shutdown failed: {}", e.getMessage(), e); @@ -439,7 +448,12 @@ public void close() { void startBookieWithMetadataStore() throws Exception { if (StringUtils.isBlank(metadataStoreUrl)){ log.info("Starting BK with RocksDb metadata store"); - metadataStoreUrl = "rocksdb://" + Paths.get(metadataDir).toAbsolutePath(); + Path metadataDirPath = Paths.get(metadataDir); + metadataStoreUrl = "rocksdb://" + metadataDirPath.toAbsolutePath(); + if (wipeData && Files.exists(metadataDirPath)) { + log.info("Wiping RocksDb metadata store at {}", metadataStoreUrl); + cleanDirectory(metadataDirPath.toFile()); + } } else { log.info("Starting BK with metadata store: {}", metadataStoreUrl); } @@ -493,5 +507,11 @@ private static void processTerminator(int exitCode) { ShutdownUtil.triggerImmediateForcefulShutdown(exitCode); } + public String getBrokerServiceUrl() { + return broker.getBrokerServiceUrl(); + } + public String getWebServiceUrl() { + return broker.getWebServiceAddress(); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java index 33dd58deac095..29feac8cb46eb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java @@ -19,35 +19,41 @@ package org.apache.pulsar; import static org.apache.commons.lang3.StringUtils.isBlank; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import java.io.FileInputStream; import java.util.Arrays; +import lombok.AccessLevel; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; -import org.apache.pulsar.common.util.CmdGenerateDocs; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; +import picocli.CommandLine; +import picocli.CommandLine.Option; @Slf4j public class PulsarStandaloneStarter extends PulsarStandalone { private static final String PULSAR_CONFIG_FILE = "pulsar.config.file"; - @Parameter(names = {"-g", "--generate-docs"}, description = "Generate docs") + @Option(names = {"-g", "--generate-docs"}, description = "Generate docs") private boolean generateDocs = false; + private Thread shutdownThread; + @Setter(AccessLevel.PACKAGE) + private boolean testMode; public PulsarStandaloneStarter(String[] args) throws Exception { - JCommander jcommander = new JCommander(); + CommandLine commander = new CommandLine(this); + try { - jcommander.addObject(this); - jcommander.parse(args); + commander.parseArgs(args); if (this.isHelp()) { - jcommander.usage(); - System.exit(0); + commander.usage(commander.getOut()); + exit(0); } if (Strings.isNullOrEmpty(this.getConfigFile())) { String configFile = System.getProperty(PULSAR_CONFIG_FILE); @@ -62,18 +68,18 @@ public PulsarStandaloneStarter(String[] args) throws Exception { CmdGenerateDocs cmd = new CmdGenerateDocs("pulsar"); cmd.addCommand("standalone", this); cmd.run(null); - System.exit(0); + exit(0); } if (this.isNoBroker() && this.isOnlyBroker()) { log.error("Only one option is allowed between '--no-broker' and '--only-broker'"); - jcommander.usage(); + commander.usage(commander.getOut()); return; } } catch (Exception e) { - jcommander.usage(); + commander.usage(commander.getOut()); log.error(e.getMessage()); - System.exit(1); + exit(1); } try (FileInputStream inputStream = new FileInputStream(this.getConfigFile())) { @@ -108,26 +114,58 @@ public PulsarStandaloneStarter(String[] args) throws Exception { } } } + } - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { - if (fnWorkerService != null) { - fnWorkerService.stop(); - } - - if (broker != null) { - broker.close(); - } + @Override + public synchronized void start() throws Exception { + registerShutdownHook(); + super.start(); + } - if (bkEnsemble != null) { - bkEnsemble.stop(); - } + protected void registerShutdownHook() { + if (shutdownThread != null) { + throw new IllegalStateException("Shutdown hook already registered"); + } + shutdownThread = new Thread(() -> { + try { + doClose(false); } catch (Exception e) { log.error("Shutdown failed: {}", e.getMessage(), e); } finally { - LogManager.shutdown(); + if (!testMode) { + LogManager.shutdown(); + } } - })); + }); + Runtime.getRuntime().addShutdownHook(shutdownThread); + } + + // simulate running the shutdown hook, for testing + @VisibleForTesting + void runShutdownHook() { + if (!testMode) { + throw new IllegalStateException("Not in test mode"); + } + Runtime.getRuntime().removeShutdownHook(shutdownThread); + shutdownThread.run(); + shutdownThread = null; + } + + @Override + public void close() { + doClose(true); + } + + private synchronized void doClose(boolean removeShutdownHook) { + super.close(); + if (shutdownThread != null && removeShutdownHook) { + Runtime.getRuntime().removeShutdownHook(shutdownThread); + shutdownThread = null; + } + } + + protected void exit(int status) { + System.exit(status); } private static boolean argsContains(String[] args, String arg) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarTransactionCoordinatorMetadataSetup.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarTransactionCoordinatorMetadataSetup.java index 78ce55f1b2ce2..06b68decf36f0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarTransactionCoordinatorMetadataSetup.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarTransactionCoordinatorMetadataSetup.java @@ -18,70 +18,77 @@ */ package org.apache.pulsar; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.SystemTopicNames; -import org.apache.pulsar.common.util.CmdGenerateDocs; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.ScopeType; /** - * Setup the transaction coordinator metadata for a cluster, the setup will create pulsar/system namespace and create + * Set up the transaction coordinator metadata for a cluster, the setup will create pulsar/system namespace and create * partitioned topic for transaction coordinator assign. */ public class PulsarTransactionCoordinatorMetadataSetup { + @Command(name = "initialize-transaction-coordinator-metadata", showDefaultValues = true, scope = ScopeType.INHERIT) private static class Arguments { - @Parameter(names = { "-c", "--cluster" }, description = "Cluster name", required = true) + @Option(names = { "-c", "--cluster" }, description = "Cluster name", required = true) private String cluster; - @Parameter(names = { "-cs", + @Option(names = { "-cs", "--configuration-store" }, description = "Configuration Store connection string", required = true) private String configurationStore; - @Parameter(names = { + @Option(names = {"-cmscp", + "--configuration-metadata-store-config-path"}, description = "Configuration Metadata Store config path", + hidden = false) + private String configurationStoreConfigPath; + + @Option(names = { "--zookeeper-session-timeout-ms" }, description = "Local zookeeper session timeout ms") private int zkSessionTimeoutMillis = 30000; - @Parameter(names = { + @Option(names = { "--initial-num-transaction-coordinators" }, description = "Num transaction coordinators will assigned in cluster") private int numTransactionCoordinators = 16; - @Parameter(names = { "-h", "--help" }, description = "Show this help message") + @Option(names = { "-h", "--help" }, description = "Show this help message") private boolean help = false; - @Parameter(names = {"-g", "--generate-docs"}, description = "Generate docs") + @Option(names = {"-g", "--generate-docs"}, description = "Generate docs") private boolean generateDocs = false; } public static void main(String[] args) throws Exception { Arguments arguments = new Arguments(); - JCommander jcommander = new JCommander(); + CommandLine commander = new CommandLine(arguments); try { - jcommander.addObject(arguments); - jcommander.parse(args); + commander.parseArgs(args); if (arguments.help) { - jcommander.usage(); + commander.usage(commander.getOut()); return; } if (arguments.generateDocs) { CmdGenerateDocs cmd = new CmdGenerateDocs("pulsar"); - cmd.addCommand("initialize-transaction-coordinator-metadata", arguments); + cmd.addCommand("initialize-transaction-coordinator-metadata", commander); cmd.run(null); return; } } catch (Exception e) { - jcommander.usage(); + commander.usage(commander.getOut()); throw e; } if (arguments.configurationStore == null) { System.err.println("Configuration store address argument is required (--configuration-store)"); - jcommander.usage(); + commander.usage(commander.getOut()); System.exit(1); } @@ -90,8 +97,10 @@ public static void main(String[] args) throws Exception { System.exit(1); } - try (MetadataStoreExtended configStore = PulsarClusterMetadataSetup - .initConfigMetadataStore(arguments.configurationStore, arguments.zkSessionTimeoutMillis)) { + try (MetadataStoreExtended configStore = PulsarClusterMetadataSetup.initConfigMetadataStore( + arguments.configurationStore, + arguments.configurationStoreConfigPath, + arguments.zkSessionTimeoutMillis)) { PulsarResources pulsarResources = new PulsarResources(null, configStore); // Create system tenant PulsarClusterMetadataSetup diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarVersionStarter.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarVersionStarter.java index 7eee9d083fc29..32876b6481c8a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarVersionStarter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarVersionStarter.java @@ -18,41 +18,43 @@ */ package org.apache.pulsar; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import org.apache.pulsar.common.util.CmdGenerateDocs; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.ScopeType; /** * Pulsar version entry point. */ public class PulsarVersionStarter { + @Command(name = "version", showDefaultValues = true, scope = ScopeType.INHERIT) private static class Arguments { - @Parameter(names = {"-h", "--help"}, description = "Show this help message") + @Option(names = {"-h", "--help"}, usageHelp = true, description = "Show this help message") private boolean help = false; - @Parameter(names = {"-g", "--generate-docs"}, description = "Generate docs") + @Option(names = {"-g", "--generate-docs"}, description = "Generate docs") private boolean generateDocs = false; } public static void main(String[] args) { Arguments arguments = new Arguments(); - JCommander jcommander = new JCommander(); + CommandLine commander = new CommandLine(arguments); try { - jcommander.addObject(arguments); - jcommander.parse(args); + commander.parseArgs(args); if (arguments.help) { - jcommander.usage(); + commander.usage(commander.getOut()); return; } if (arguments.generateDocs) { CmdGenerateDocs cmd = new CmdGenerateDocs("pulsar"); - cmd.addCommand("version", arguments); + cmd.addCommand("version", commander); cmd.run(null); return; } } catch (Exception e) { - jcommander.usage(); + commander.getErr().println(e); return; } System.out.println("Current version of pulsar is: " + PulsarVersion.getVersion()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactory.java index 95923baac0294..5ab1a01838df7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactory.java @@ -19,9 +19,9 @@ package org.apache.pulsar.broker; import io.netty.channel.EventLoopGroup; -import java.io.IOException; import java.util.Map; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.EnsemblePlacementPolicy; import org.apache.bookkeeper.stats.StatsLogger; @@ -31,13 +31,16 @@ * Provider of a new BookKeeper client instance. */ public interface BookKeeperClientFactory { - BookKeeper create(ServiceConfiguration conf, MetadataStoreExtended store, EventLoopGroup eventLoopGroup, - Optional> ensemblePlacementPolicyClass, - Map ensemblePlacementPolicyProperties) throws IOException; + CompletableFuture create(ServiceConfiguration conf, MetadataStoreExtended store, + EventLoopGroup eventLoopGroup, + Optional> policyClass, + Map ensemblePlacementPolicyProperties); + + CompletableFuture create(ServiceConfiguration conf, MetadataStoreExtended store, + EventLoopGroup eventLoopGroup, + Optional> policyClass, + Map ensemblePlacementPolicyProperties, + StatsLogger statsLogger); - BookKeeper create(ServiceConfiguration conf, MetadataStoreExtended store, EventLoopGroup eventLoopGroup, - Optional> ensemblePlacementPolicyClass, - Map ensemblePlacementPolicyProperties, - StatsLogger statsLogger) throws IOException; void close(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java index 0259dfc7a58cf..45299d9ed05d5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.util.Map; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.BKException; @@ -49,40 +50,42 @@ import org.apache.pulsar.metadata.bookkeeper.AbstractMetadataDriver; import org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver; -@SuppressWarnings("deprecation") @Slf4j public class BookKeeperClientFactoryImpl implements BookKeeperClientFactory { @Override - public BookKeeper create(ServiceConfiguration conf, MetadataStoreExtended store, - EventLoopGroup eventLoopGroup, - Optional> ensemblePlacementPolicyClass, - Map properties) throws IOException { - return create(conf, store, eventLoopGroup, ensemblePlacementPolicyClass, properties, + public CompletableFuture create(ServiceConfiguration conf, MetadataStoreExtended store, + EventLoopGroup eventLoopGroup, + Optional> policyClass, + Map properties) { + return create(conf, store, eventLoopGroup, policyClass, properties, NullStatsLogger.INSTANCE); } @Override - public BookKeeper create(ServiceConfiguration conf, MetadataStoreExtended store, + public CompletableFuture create(ServiceConfiguration conf, MetadataStoreExtended store, EventLoopGroup eventLoopGroup, Optional> ensemblePlacementPolicyClass, - Map properties, StatsLogger statsLogger) throws IOException { + Map properties, StatsLogger statsLogger) { PulsarMetadataClientDriver.init(); ClientConfiguration bkConf = createBkClientConfiguration(store, conf); if (properties != null) { - properties.forEach((key, value) -> bkConf.setProperty(key, value)); + properties.forEach(bkConf::setProperty); } if (ensemblePlacementPolicyClass.isPresent()) { setEnsemblePlacementPolicy(bkConf, conf, store, ensemblePlacementPolicyClass.get()); } else { setDefaultEnsemblePlacementPolicy(bkConf, conf, store); } - try { - return getBookKeeperBuilder(conf, eventLoopGroup, statsLogger, bkConf).build(); - } catch (InterruptedException | BKException e) { - throw new IOException(e); - } + + return CompletableFuture.supplyAsync(() -> { + try { + return getBookKeeperBuilder(conf, eventLoopGroup, statsLogger, bkConf).build(); + } catch (InterruptedException | BKException | IOException e) { + throw new RuntimeException(e); + } + }); } @VisibleForTesting @@ -220,7 +223,7 @@ static void setDefaultEnsemblePlacementPolicy( } } - private void setEnsemblePlacementPolicy(ClientConfiguration bkConf, ServiceConfiguration conf, MetadataStore store, + static void setEnsemblePlacementPolicy(ClientConfiguration bkConf, ServiceConfiguration conf, MetadataStore store, Class policyClass) { bkConf.setEnsemblePlacementPolicy(policyClass); bkConf.setProperty(BookieRackAffinityMapping.METADATA_STORE_INSTANCE, store); @@ -228,6 +231,9 @@ private void setEnsemblePlacementPolicy(ClientConfiguration bkConf, ServiceConfi bkConf.setProperty(REPP_DNS_RESOLVER_CLASS, conf.getProperties().getProperty(REPP_DNS_RESOLVER_CLASS, BookieRackAffinityMapping.class.getName())); + bkConf.setMinNumRacksPerWriteQuorum(conf.getBookkeeperClientMinNumRacksPerWriteQuorum()); + bkConf.setEnforceMinNumRacksPerWriteQuorum(conf.isBookkeeperClientEnforceMinNumRacksPerWriteQuorum()); + bkConf.setProperty(NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY, conf.getProperties().getProperty( NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java index b16b9a7dd4833..9bbc2857863ff 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java @@ -18,12 +18,15 @@ */ package org.apache.pulsar.broker; +import com.github.benmanes.caffeine.cache.AsyncCache; +import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.annotations.VisibleForTesting; import io.netty.channel.EventLoopGroup; +import io.opentelemetry.api.OpenTelemetry; import java.io.IOException; import java.util.Map; import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.RejectedExecutionException; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.conf.ClientConfiguration; @@ -48,13 +51,15 @@ public class ManagedLedgerClientFactory implements ManagedLedgerStorage { private ManagedLedgerFactory managedLedgerFactory; private BookKeeper defaultBkClient; - private final Map - bkEnsemblePolicyToBkClientMap = new ConcurrentHashMap<>(); + private final AsyncCache + bkEnsemblePolicyToBkClientMap = Caffeine.newBuilder().buildAsync(); private StatsProvider statsProvider = new NullStatsProvider(); + @Override public void initialize(ServiceConfiguration conf, MetadataStoreExtended metadataStore, BookKeeperClientFactory bookkeeperProvider, - EventLoopGroup eventLoopGroup) throws Exception { + EventLoopGroup eventLoopGroup, + OpenTelemetry openTelemetry) throws Exception { ManagedLedgerFactoryConfig managedLedgerFactoryConfig = new ManagedLedgerFactoryConfig(); managedLedgerFactoryConfig.setMaxCacheSize(conf.getManagedLedgerCacheSizeMB() * 1024L * 1024L); managedLedgerFactoryConfig.setCacheEvictionWatermark(conf.getManagedLedgerCacheEvictionWatermark()); @@ -70,8 +75,12 @@ public void initialize(ServiceConfiguration conf, MetadataStoreExtended metadata managedLedgerFactoryConfig.setTraceTaskExecution(conf.isManagedLedgerTraceTaskExecution()); managedLedgerFactoryConfig.setCursorPositionFlushSeconds(conf.getManagedLedgerCursorPositionFlushSeconds()); managedLedgerFactoryConfig.setManagedLedgerInfoCompressionType(conf.getManagedLedgerInfoCompressionType()); + managedLedgerFactoryConfig.setManagedLedgerInfoCompressionThresholdInBytes( + conf.getManagedLedgerInfoCompressionThresholdInBytes()); managedLedgerFactoryConfig.setStatsPeriodSeconds(conf.getManagedLedgerStatsPeriodSeconds()); managedLedgerFactoryConfig.setManagedCursorInfoCompressionType(conf.getManagedCursorInfoCompressionType()); + managedLedgerFactoryConfig.setManagedCursorInfoCompressionThresholdInBytes( + conf.getManagedCursorInfoCompressionThresholdInBytes()); Configuration configuration = new ClientConfiguration(); if (conf.isBookkeeperClientExposeStatsToPrometheus()) { @@ -85,31 +94,31 @@ public void initialize(ServiceConfiguration conf, MetadataStoreExtended metadata StatsLogger statsLogger = statsProvider.getStatsLogger("pulsar_managedLedger_client"); this.defaultBkClient = - bookkeeperProvider.create(conf, metadataStore, eventLoopGroup, Optional.empty(), null, statsLogger); + bookkeeperProvider.create(conf, metadataStore, eventLoopGroup, Optional.empty(), null, statsLogger) + .get(); BookkeeperFactoryForCustomEnsemblePlacementPolicy bkFactory = ( EnsemblePlacementPolicyConfig ensemblePlacementPolicyConfig) -> { - BookKeeper bkClient = null; - // find or create bk-client in cache for a specific ensemblePlacementPolicy - if (ensemblePlacementPolicyConfig != null && ensemblePlacementPolicyConfig.getPolicyClass() != null) { - bkClient = bkEnsemblePolicyToBkClientMap.computeIfAbsent(ensemblePlacementPolicyConfig, (key) -> { - try { - return bookkeeperProvider.create(conf, metadataStore, eventLoopGroup, - Optional.ofNullable(ensemblePlacementPolicyConfig.getPolicyClass()), - ensemblePlacementPolicyConfig.getProperties(), statsLogger); - } catch (Exception e) { - log.error("Failed to initialize bk-client for policy {}, properties {}", - ensemblePlacementPolicyConfig.getPolicyClass(), - ensemblePlacementPolicyConfig.getProperties(), e); - } - return this.defaultBkClient; - }); + if (ensemblePlacementPolicyConfig == null || ensemblePlacementPolicyConfig.getPolicyClass() == null) { + return CompletableFuture.completedFuture(defaultBkClient); } - return bkClient != null ? bkClient : defaultBkClient; + + // find or create bk-client in cache for a specific ensemblePlacementPolicy + return bkEnsemblePolicyToBkClientMap.get(ensemblePlacementPolicyConfig, + (config, executor) -> bookkeeperProvider.create(conf, metadataStore, eventLoopGroup, + Optional.ofNullable(ensemblePlacementPolicyConfig.getPolicyClass()), + ensemblePlacementPolicyConfig.getProperties(), statsLogger)); }; - this.managedLedgerFactory = - new ManagedLedgerFactoryImpl(metadataStore, bkFactory, managedLedgerFactoryConfig, statsLogger); + try { + this.managedLedgerFactory = + new ManagedLedgerFactoryImpl(metadataStore, bkFactory, managedLedgerFactoryConfig, statsLogger, + openTelemetry); + } catch (Exception e) { + statsProvider.stop(); + defaultBkClient.close(); + throw e; + } } public ManagedLedgerFactory getManagedLedgerFactory() { @@ -126,7 +135,7 @@ public StatsProvider getStatsProvider() { @VisibleForTesting public Map getBkEnsemblePolicyToBookKeeperMap() { - return bkEnsemblePolicyToBkClientMap; + return bkEnsemblePolicyToBkClientMap.synchronous().asMap(); } @Override @@ -154,17 +163,15 @@ public void close() throws IOException { // factory, however that might be introducing more unknowns. log.warn("Encountered exceptions on closing bookkeeper client", ree); } - if (bkEnsemblePolicyToBkClientMap != null) { - bkEnsemblePolicyToBkClientMap.forEach((policy, bk) -> { - try { - if (bk != null) { - bk.close(); - } - } catch (Exception e) { - log.warn("Failed to close bookkeeper-client for policy {}", policy, e); + bkEnsemblePolicyToBkClientMap.synchronous().asMap().forEach((policy, bk) -> { + try { + if (bk != null) { + bk.close(); } - }); - } + } catch (Exception e) { + log.warn("Failed to close bookkeeper-client for policy {}", policy, e); + } + }); log.info("Closed BookKeeper client"); } catch (Exception e) { log.warn(e.getMessage(), e); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 62d4634fa2d62..6c768a078974f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -21,6 +21,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.pulsar.broker.admin.impl.BrokersBase.getHeartbeatTopicName; import static org.apache.pulsar.broker.resourcegroup.ResourceUsageTransportManager.DISABLE_RESOURCE_USAGE_TRANSPORT_MANAGER; import static org.apache.pulsar.common.naming.SystemTopicNames.isTransactionInternalName; import com.google.common.annotations.VisibleForTesting; @@ -30,10 +31,13 @@ import io.netty.util.HashedWheelTimer; import io.netty.util.Timer; import io.netty.util.concurrent.DefaultThreadFactory; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdkBuilder; import java.io.IOException; import java.lang.reflect.Constructor; import java.net.InetSocketAddress; import java.net.MalformedURLException; +import java.time.Clock; import java.time.Duration; import java.util.ArrayList; import java.util.Collection; @@ -71,6 +75,7 @@ import org.apache.bookkeeper.mledger.LedgerOffloader; import org.apache.bookkeeper.mledger.LedgerOffloaderFactory; import org.apache.bookkeeper.mledger.LedgerOffloaderStats; +import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerFactory; import org.apache.bookkeeper.mledger.impl.NullLedgerOffloader; import org.apache.bookkeeper.mledger.offload.Offloaders; @@ -83,7 +88,6 @@ import org.apache.pulsar.broker.intercept.BrokerInterceptors; import org.apache.pulsar.broker.loadbalance.LeaderBroker; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; -import org.apache.pulsar.broker.loadbalance.LinuxInfoUtils; import org.apache.pulsar.broker.loadbalance.LoadManager; import org.apache.pulsar.broker.loadbalance.LoadReportUpdaterTask; import org.apache.pulsar.broker.loadbalance.LoadResourceQuotaUpdaterTask; @@ -92,6 +96,8 @@ import org.apache.pulsar.broker.lookup.v1.TopicLookup; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.protocol.ProtocolHandlers; +import org.apache.pulsar.broker.qos.DefaultMonotonicSnapshotClock; +import org.apache.pulsar.broker.qos.MonotonicSnapshotClock; import org.apache.pulsar.broker.resourcegroup.ResourceGroupService; import org.apache.pulsar.broker.resourcegroup.ResourceUsageTopicTransportManager; import org.apache.pulsar.broker.resourcegroup.ResourceUsageTransportManager; @@ -107,6 +113,15 @@ import org.apache.pulsar.broker.service.schema.SchemaRegistryService; import org.apache.pulsar.broker.service.schema.SchemaStorageFactory; import org.apache.pulsar.broker.stats.MetricsGenerator; +import org.apache.pulsar.broker.stats.OpenTelemetryConsumerStats; +import org.apache.pulsar.broker.stats.OpenTelemetryProducerStats; +import org.apache.pulsar.broker.stats.OpenTelemetryReplicatedSubscriptionStats; +import org.apache.pulsar.broker.stats.OpenTelemetryReplicatorStats; +import org.apache.pulsar.broker.stats.OpenTelemetryTopicStats; +import org.apache.pulsar.broker.stats.OpenTelemetryTransactionCoordinatorStats; +import org.apache.pulsar.broker.stats.OpenTelemetryTransactionPendingAckStoreStats; +import org.apache.pulsar.broker.stats.PulsarBrokerOpenTelemetry; +import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsServlet; import org.apache.pulsar.broker.stats.prometheus.PrometheusRawMetricsProvider; import org.apache.pulsar.broker.stats.prometheus.PulsarPrometheusMetricsServlet; import org.apache.pulsar.broker.storage.ManagedLedgerStorage; @@ -147,9 +162,11 @@ import org.apache.pulsar.common.util.Reflections; import org.apache.pulsar.common.util.ThreadDumpUtil; import org.apache.pulsar.common.util.netty.EventLoopUtil; +import org.apache.pulsar.compaction.CompactionServiceFactory; import org.apache.pulsar.compaction.Compactor; +import org.apache.pulsar.compaction.PulsarCompactionServiceFactory; import org.apache.pulsar.compaction.StrategicTwoPhaseCompactor; -import org.apache.pulsar.compaction.TwoPhaseCompactor; +import org.apache.pulsar.compaction.TopicCompactionService; import org.apache.pulsar.functions.worker.ErrorNotifier; import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerService; @@ -173,7 +190,7 @@ import org.apache.pulsar.transaction.coordinator.TransactionMetadataStoreProvider; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStoreProvider; import org.apache.pulsar.websocket.WebSocketConsumerServlet; -import org.apache.pulsar.websocket.WebSocketPingPongServlet; +import org.apache.pulsar.websocket.WebSocketMultiTopicConsumerServlet; import org.apache.pulsar.websocket.WebSocketProducerServlet; import org.apache.pulsar.websocket.WebSocketReaderServlet; import org.apache.pulsar.websocket.WebSocketService; @@ -185,13 +202,13 @@ /** * Main class for Pulsar broker service. */ - @Getter(AccessLevel.PUBLIC) @Setter(AccessLevel.PROTECTED) public class PulsarService implements AutoCloseable, ShutdownService { private static final Logger LOG = LoggerFactory.getLogger(PulsarService.class); private static final double GRACEFUL_SHUTDOWN_TIMEOUT_RATIO_OF_TOTAL_TIMEOUT = 0.5d; - private ServiceConfiguration config = null; + private static final int DEFAULT_MONOTONIC_CLOCK_GRANULARITY_MILLIS = 8; + private final ServiceConfiguration config; private NamespaceService nsService = null; private ManagedLedgerStorage managedLedgerClientFactory = null; private LeaderElectionService leaderElectionService = null; @@ -200,13 +217,13 @@ public class PulsarService implements AutoCloseable, ShutdownService { private WebSocketService webSocketService = null; private TopicPoliciesService topicPoliciesService = TopicPoliciesService.DISABLED; private BookKeeperClientFactory bkClientFactory; - private Compactor compactor; + protected CompactionServiceFactory compactionServiceFactory; private StrategicTwoPhaseCompactor strategicCompactor; private ResourceUsageTransportManager resourceUsageTransportManager; private ResourceGroupService resourceGroupServiceManager; + private final Clock clock; private final ScheduledExecutorService executor; - private final ScheduledExecutorService cacheExecutor; private OrderedExecutor orderedExecutor; private final ScheduledExecutorService loadManagerExecutor; @@ -246,6 +263,14 @@ public class PulsarService implements AutoCloseable, ShutdownService { private final Timer brokerClientSharedTimer; private MetricsGenerator metricsGenerator; + private final PulsarBrokerOpenTelemetry openTelemetry; + private OpenTelemetryTopicStats openTelemetryTopicStats; + private OpenTelemetryConsumerStats openTelemetryConsumerStats; + private OpenTelemetryProducerStats openTelemetryProducerStats; + private OpenTelemetryReplicatorStats openTelemetryReplicatorStats; + private OpenTelemetryReplicatedSubscriptionStats openTelemetryReplicatedSubscriptionStats; + private OpenTelemetryTransactionCoordinatorStats openTelemetryTransactionCoordinatorStats; + private OpenTelemetryTransactionPendingAckStoreStats openTelemetryTransactionPendingAckStoreStats; private TransactionMetadataStoreService transactionMetadataStoreService; private TransactionBufferProvider transactionBufferProvider; @@ -256,7 +281,7 @@ public class PulsarService implements AutoCloseable, ShutdownService { private AdditionalServlets brokerAdditionalServlets; // packages management service - private Optional packagesManagement = Optional.empty(); + private PackagesManagement packagesManagement = null; private PulsarPrometheusMetricsServlet metricsServlet; private List pendingMetricsProviders; @@ -272,6 +297,10 @@ public class PulsarService implements AutoCloseable, ShutdownService { private TransactionPendingAckStoreProvider transactionPendingAckStoreProvider; private final ExecutorProvider transactionExecutorProvider; + private final DefaultMonotonicSnapshotClock monotonicSnapshotClock; + private String brokerId; + private final CompletableFuture readyForIncomingRequestsFuture = new CompletableFuture<>(); + private final List pendingTasksBeforeReadyForIncomingRequests = new ArrayList<>(); public enum State { Init, Started, Closing, Closed @@ -286,10 +315,8 @@ public enum State { private Map advertisedListeners; public PulsarService(ServiceConfiguration config) { - this(config, Optional.empty(), (exitCode) -> { - LOG.info("Process termination requested with code {}. " - + "Ignoring, as this constructor is intended for tests. ", exitCode); - }); + this(config, Optional.empty(), (exitCode) -> LOG.info("Process termination requested with code {}. " + + "Ignoring, as this constructor is intended for tests. ", exitCode)); } public PulsarService(ServiceConfiguration config, Optional functionWorkerService, @@ -301,11 +328,23 @@ public PulsarService(ServiceConfiguration config, WorkerConfig workerConfig, Optional functionWorkerService, Consumer processTerminator) { + this(config, workerConfig, functionWorkerService, processTerminator, null); + } + + public PulsarService(ServiceConfiguration config, + WorkerConfig workerConfig, + Optional functionWorkerService, + Consumer processTerminator, + Consumer openTelemetrySdkBuilderCustomizer) { state = State.Init; // Validate correctness of configuration PulsarConfigurationLoader.isComplete(config); TransactionBatchedWriteValidator.validate(config); + this.config = config; + this.clock = Clock.systemUTC(); + + this.openTelemetry = new PulsarBrokerOpenTelemetry(config, openTelemetrySdkBuilderCustomizer); // validate `advertisedAddress`, `advertisedListeners`, `internalListenerName` this.advertisedListeners = MultipleListenerValidator.validateAndAnalysisAdvertisedListener(config); @@ -316,7 +355,6 @@ public PulsarService(ServiceConfiguration config, // use `internalListenerName` listener as `advertisedAddress` this.bindAddress = ServiceConfigurationUtils.getDefaultOrConfiguredAddress(config.getBindAddress()); this.brokerVersion = PulsarVersion.getVersion(); - this.config = config; this.processTerminator = processTerminator; this.loadManagerExecutor = Executors .newSingleThreadScheduledExecutor(new ExecutorProvider.ExtendedThreadFactory("pulsar-load-manager")); @@ -324,8 +362,6 @@ public PulsarService(ServiceConfiguration config, this.functionWorkerService = functionWorkerService; this.executor = Executors.newScheduledThreadPool(config.getNumExecutorThreadPoolSize(), new ExecutorProvider.ExtendedThreadFactory("pulsar")); - this.cacheExecutor = Executors.newScheduledThreadPool(config.getNumCacheExecutorThreadPoolSize(), - new ExecutorProvider.ExtendedThreadFactory("zk-cache-callback")); if (config.isTransactionCoordinatorEnabled()) { this.transactionExecutorProvider = new ExecutorProvider(this.getConfiguration() @@ -353,27 +389,36 @@ public PulsarService(ServiceConfiguration config, // here in the constructor we don't have the offloader scheduler yet this.offloaderStats = LedgerOffloaderStats.create(false, false, null, 0); + + this.monotonicSnapshotClock = new DefaultMonotonicSnapshotClock(TimeUnit.MILLISECONDS.toNanos( + DEFAULT_MONOTONIC_CLOCK_GRANULARITY_MILLIS), System::nanoTime); } - public MetadataStore createConfigurationMetadataStore(PulsarMetadataEventSynchronizer synchronizer) + public MetadataStore createConfigurationMetadataStore(PulsarMetadataEventSynchronizer synchronizer, + OpenTelemetry openTelemetry) throws MetadataStoreException { + String configFilePath = config.getMetadataStoreConfigPath(); + if (StringUtils.isNotBlank(config.getConfigurationStoreConfigPath())) { + configFilePath = config.getConfigurationStoreConfigPath(); + } return MetadataStoreFactory.create(config.getConfigurationMetadataStoreUrl(), MetadataStoreConfig.builder() .sessionTimeoutMillis((int) config.getMetadataStoreSessionTimeoutMillis()) .allowReadOnlyOperations(config.isMetadataStoreAllowReadOnlyOperations()) - .configFilePath(config.getMetadataStoreConfigPath()) + .configFilePath(configFilePath) .batchingEnabled(config.isMetadataStoreBatchingEnabled()) .batchingMaxDelayMillis(config.getMetadataStoreBatchingMaxDelayMillis()) .batchingMaxOperations(config.getMetadataStoreBatchingMaxOperations()) .batchingMaxSizeKb(config.getMetadataStoreBatchingMaxSizeKb()) .metadataStoreName(MetadataStoreConfig.CONFIGURATION_METADATA_STORE) .synchronizer(synchronizer) + .openTelemetry(openTelemetry) .build()); } /** * Close the session to the metadata service. - * + *

* This will immediately release all the resource locks held by this broker on the coordination service. * * @throws Exception if the close operation fails @@ -383,7 +428,7 @@ public void closeMetadataServiceSession() throws Exception { } private void closeLeaderElectionService() throws Exception { - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(this)) { ExtensibleLoadManagerImpl.get(loadManager.get()).getLeaderElectionService().close(); } else { if (this.leaderElectionService != null) { @@ -393,6 +438,41 @@ private void closeLeaderElectionService() throws Exception { } } + private boolean isManagedLedgerNotFoundException(Throwable e) { + Throwable realCause = e.getCause(); + return realCause instanceof ManagedLedgerException.MetadataNotFoundException + || realCause instanceof MetadataStoreException.NotFoundException; + } + + private void deleteHeartbeatResource() { + if (this.brokerService != null) { + LOG.info("forcefully delete heartbeat topic when close broker"); + + String heartbeatTopicNameV1 = getHeartbeatTopicName(getBrokerId(), getConfiguration(), false); + String heartbeatTopicNameV2 = getHeartbeatTopicName(getBrokerId(), getConfiguration(), true); + + try { + this.brokerService.deleteTopic(heartbeatTopicNameV1, true).get(); + } catch (Exception e) { + if (!isManagedLedgerNotFoundException(e)) { + LOG.error("Closed with errors in delete heartbeat topic [{}]", + heartbeatTopicNameV1, e); + } + } + + try { + this.brokerService.deleteTopic(heartbeatTopicNameV2, true).get(); + } catch (Exception e) { + if (!isManagedLedgerNotFoundException(e)) { + LOG.error("Closed with errors in delete heartbeat topic [{}]", + heartbeatTopicNameV2, e); + } + } + + LOG.info("finish forcefully delete heartbeat topic when close broker"); + } + } + @Override public void close() throws PulsarServerException { try { @@ -403,8 +483,12 @@ public void close() throws PulsarServerException { throw (PulsarServerException) cause; } else if (getConfiguration().getBrokerShutdownTimeoutMs() == 0 && (cause instanceof TimeoutException || cause instanceof CancellationException)) { - // ignore shutdown timeout when timeout is 0, which is primarily used in tests - // to forcefully shutdown the broker + if (LOG.isDebugEnabled()) { + LOG.debug( + "Shutdown timeout ignored when timeout is 0, " + + "which is primarily used in tests to forcefully shutdown the broker", + cause); + } } else { throw new PulsarServerException(cause); } @@ -419,12 +503,30 @@ public void close() throws PulsarServerException { public CompletableFuture closeAsync() { mutex.lock(); try { + // Close protocol handler before unloading namespace bundles because protocol handlers might maintain + // Pulsar clients that could send lookup requests that affect unloading. + if (protocolHandlers != null) { + protocolHandlers.close(); + protocolHandlers = null; + } if (closeFuture != null) { return closeFuture; } LOG.info("Closing PulsarService"); + if (topicPoliciesService != null) { + topicPoliciesService.close(); + } + if (brokerService != null) { + brokerService.unloadNamespaceBundlesGracefully(); + } + // It only tells the Pulsar clients that this service is not ready to serve for the lookup requests state = State.Closing; + if (brokerId != null) { + // forcefully delete heartbeat topic when close broker + deleteHeartbeatResource(); + } + // close the service in reverse order v.s. in which they are started if (this.resourceUsageTransportManager != null) { try { @@ -454,6 +556,18 @@ public CompletableFuture closeAsync() { } resetMetricsServlet(); + if (openTelemetry != null) { + openTelemetry.close(); + } + + if (this.compactionServiceFactory != null) { + try { + this.compactionServiceFactory.close(); + } catch (Exception e) { + LOG.warn("CompactionServiceFactory closing failed {}", e.getMessage()); + } + this.compactionServiceFactory = null; + } if (this.webSocketService != null) { this.webSocketService.close(); @@ -472,16 +586,9 @@ public CompletableFuture closeAsync() { (long) (GRACEFUL_SHUTDOWN_TIMEOUT_RATIO_OF_TOTAL_TIMEOUT * getConfiguration() .getBrokerShutdownTimeoutMs()))); - // close protocol handler before closing broker service - if (protocolHandlers != null) { - protocolHandlers.close(); - protocolHandlers = null; - } // cancel loadShedding task and shutdown the loadManager executor before shutting down the broker - if (this.loadSheddingTask != null) { - this.loadSheddingTask.cancel(); - } + cancelLoadBalancerTasks(); executorServicesShutdown.shutdown(loadManagerExecutor); List> asyncCloseFutures = new ArrayList<>(); @@ -529,6 +636,7 @@ public CompletableFuture closeAsync() { transactionBufferClient.close(); } + if (client != null) { client.close(); client = null; @@ -543,7 +651,6 @@ public CompletableFuture closeAsync() { executorServicesShutdown.shutdown(offloaderScheduler); executorServicesShutdown.shutdown(executor); executorServicesShutdown.shutdown(orderedExecutor); - executorServicesShutdown.shutdown(cacheExecutor); LoadManager loadManager = this.loadManager.get(); if (loadManager != null) { @@ -567,13 +674,12 @@ public CompletableFuture closeAsync() { } } - closeLocalMetadataStore(); + asyncCloseFutures.add(closeLocalMetadataStore()); + if (configMetadataSynchronizer != null) { + asyncCloseFutures.add(configMetadataSynchronizer.closeAsync()); + } if (configurationMetadataStore != null && shouldShutdownConfigurationMetadataStore) { configurationMetadataStore.close(); - if (configMetadataSynchronizer != null) { - configMetadataSynchronizer.close(); - configMetadataSynchronizer = null; - } } if (transactionExecutorProvider != null) { @@ -589,6 +695,32 @@ public CompletableFuture closeAsync() { brokerClientSharedInternalExecutorProvider.shutdownNow(); brokerClientSharedScheduledExecutorProvider.shutdownNow(); brokerClientSharedTimer.stop(); + monotonicSnapshotClock.close(); + + if (openTelemetryTransactionPendingAckStoreStats != null) { + openTelemetryTransactionPendingAckStoreStats.close(); + openTelemetryTransactionPendingAckStoreStats = null; + } + if (openTelemetryTransactionCoordinatorStats != null) { + openTelemetryTransactionCoordinatorStats.close(); + openTelemetryTransactionCoordinatorStats = null; + } + if (openTelemetryReplicatorStats != null) { + openTelemetryReplicatorStats.close(); + openTelemetryReplicatorStats = null; + } + if (openTelemetryProducerStats != null) { + openTelemetryProducerStats.close(); + openTelemetryProducerStats = null; + } + if (openTelemetryConsumerStats != null) { + openTelemetryConsumerStats.close(); + openTelemetryConsumerStats = null; + } + if (openTelemetryTopicStats != null) { + openTelemetryTopicStats.close(); + openTelemetryTopicStats = null; + } asyncCloseFutures.add(EventLoopUtil.shutdownGracefully(ioEventLoopGroup)); @@ -633,14 +765,18 @@ private synchronized void resetMetricsServlet() { } private CompletableFuture addTimeoutHandling(CompletableFuture future) { + long brokerShutdownTimeoutMs = getConfiguration().getBrokerShutdownTimeoutMs(); + if (brokerShutdownTimeoutMs <= 0) { + return future; + } ScheduledExecutorService shutdownExecutor = Executors.newSingleThreadScheduledExecutor( new ExecutorProvider.ExtendedThreadFactory(getClass().getSimpleName() + "-shutdown")); FutureUtil.addTimeoutHandling(future, - Duration.ofMillis(Math.max(1L, getConfiguration().getBrokerShutdownTimeoutMs())), + Duration.ofMillis(brokerShutdownTimeoutMs), shutdownExecutor, () -> FutureUtil.createTimeoutException("Timeout in close", getClass(), "close")); future.handle((v, t) -> { - if (t != null && getConfiguration().getBrokerShutdownTimeoutMs() > 0) { - LOG.info("Shutdown timed out after {} ms", getConfiguration().getBrokerShutdownTimeoutMs()); + if (t instanceof TimeoutException) { + LOG.info("Shutdown timed out after {} ms", brokerShutdownTimeoutMs); LOG.info(ThreadDumpUtil.buildThreadDiagnosticString()); } // shutdown the shutdown executor @@ -697,7 +833,7 @@ public void start() throws PulsarServerException { throw new PulsarServerException("Cannot start the service once it was stopped"); } - if (!config.getWebServicePort().isPresent() && !config.getWebServicePortTls().isPresent()) { + if (config.getWebServicePort().isEmpty() && config.getWebServicePortTls().isEmpty()) { throw new IllegalArgumentException("webServicePort/webServicePortTls must be present"); } @@ -726,18 +862,17 @@ public void start() throws PulsarServerException { config.getDefaultRetentionTimeInMinutes() * 60)); } - if (!config.getLoadBalancerOverrideBrokerNicSpeedGbps().isPresent() - && config.isLoadBalancerEnabled() - && LinuxInfoUtils.isLinux() - && !LinuxInfoUtils.checkHasNicSpeeds()) { - throw new IllegalStateException("Unable to read VM NIC speed. You must set " - + "[loadBalancerOverrideBrokerNicSpeedGbps] to override it when load balancer is enabled."); - } + openTelemetryTopicStats = new OpenTelemetryTopicStats(this); + openTelemetryConsumerStats = new OpenTelemetryConsumerStats(this); + openTelemetryProducerStats = new OpenTelemetryProducerStats(this); + openTelemetryReplicatorStats = new OpenTelemetryReplicatorStats(this); + openTelemetryReplicatedSubscriptionStats = new OpenTelemetryReplicatedSubscriptionStats(this); localMetadataSynchronizer = StringUtils.isNotBlank(config.getMetadataSyncEventTopic()) ? new PulsarMetadataEventSynchronizer(this, config.getMetadataSyncEventTopic()) : null; - localMetadataStore = createLocalMetadataStore(localMetadataSynchronizer); + localMetadataStore = createLocalMetadataStore(localMetadataSynchronizer, + openTelemetry.getOpenTelemetryService().getOpenTelemetry()); localMetadataStore.registerSessionListener(this::handleMetadataSessionEvent); coordinationService = new CoordinationServiceImpl(localMetadataStore); @@ -746,7 +881,8 @@ public void start() throws PulsarServerException { configMetadataSynchronizer = StringUtils.isNotBlank(config.getConfigurationMetadataSyncEventTopic()) ? new PulsarMetadataEventSynchronizer(this, config.getConfigurationMetadataSyncEventTopic()) : null; - configurationMetadataStore = createConfigurationMetadataStore(configMetadataSynchronizer); + configurationMetadataStore = createConfigurationMetadataStore(configMetadataSynchronizer, + openTelemetry.getOpenTelemetryService().getOpenTelemetry()); shouldShutdownConfigurationMetadataStore = true; } else { configurationMetadataStore = localMetadataStore; @@ -775,7 +911,7 @@ public void start() throws PulsarServerException { schemaStorage = createAndStartSchemaStorage(); schemaRegistryService = SchemaRegistryService.create( - schemaStorage, config.getSchemaRegistryCompatibilityCheckers(), this.executor); + schemaStorage, config.getSchemaRegistryCompatibilityCheckers(), this); OffloadPoliciesImpl defaultOffloadPolicies = OffloadPoliciesImpl.create(this.getConfiguration().getProperties()); @@ -788,7 +924,7 @@ public void start() throws PulsarServerException { exposeTopicMetrics, offloaderScheduler, interval); this.defaultOffloader = createManagedLedgerOffloader(defaultOffloadPolicies); - this.brokerInterceptor = BrokerInterceptors.load(config); + setBrokerInterceptor(newBrokerInterceptor()); // use getter to support mocking getBrokerInterceptor method in tests BrokerInterceptor interceptor = getBrokerInterceptor(); if (interceptor != null) { @@ -817,6 +953,15 @@ public void start() throws PulsarServerException { this.brokerServiceUrl = brokerUrl(config); this.brokerServiceUrlTls = brokerUrlTls(config); + // the broker id is used in the load manager to identify the broker + // it should not be used for making connections to the broker + this.brokerId = + String.format("%s:%s", advertisedAddress, config.getWebServicePort() + .or(config::getWebServicePortTls).orElseThrow()); + + if (this.compactionServiceFactory == null) { + this.compactionServiceFactory = loadCompactionServiceFactory(); + } if (null != this.webSocketService) { ClusterDataImpl clusterData = ClusterDataImpl.builder() @@ -843,11 +988,8 @@ public void start() throws PulsarServerException { this.nsService.initialize(); // Start topic level policies service - if (config.isTopicLevelPoliciesEnabled() && config.isSystemTopicEnabled()) { - this.topicPoliciesService = new SystemTopicBasedTopicPoliciesService(this); - } - - this.topicPoliciesService.start(); + this.topicPoliciesService = initTopicPoliciesService(); + this.topicPoliciesService.start(this); // Register heartbeat and bootstrap namespaces. this.nsService.registerBootstrapNamespaces(); @@ -857,7 +999,7 @@ public void start() throws PulsarServerException { MLTransactionMetadataStoreProvider.initBufferedWriterMetrics(getAdvertisedAddress()); MLPendingAckStoreProvider.initBufferedWriterMetrics(getAdvertisedAddress()); - this.transactionBufferSnapshotServiceFactory = new TransactionBufferSnapshotServiceFactory(getClient()); + this.transactionBufferSnapshotServiceFactory = new TransactionBufferSnapshotServiceFactory(this); this.transactionTimer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-transaction-timer")); @@ -873,10 +1015,22 @@ public void start() throws PulsarServerException { .newProvider(config.getTransactionBufferProviderClassName()); transactionPendingAckStoreProvider = TransactionPendingAckStoreProvider .newProvider(config.getTransactionPendingAckStoreProviderClassName()); + + openTelemetryTransactionCoordinatorStats = new OpenTelemetryTransactionCoordinatorStats(this); + openTelemetryTransactionPendingAckStoreStats = new OpenTelemetryTransactionPendingAckStoreStats(this); } this.metricsGenerator = new MetricsGenerator(this); + // the broker is ready to accept incoming requests by Pulsar binary protocol and http/https + final List runnables; + synchronized (pendingTasksBeforeReadyForIncomingRequests) { + runnables = new ArrayList<>(pendingTasksBeforeReadyForIncomingRequests); + pendingTasksBeforeReadyForIncomingRequests.clear(); + readyForIncomingRequestsFuture.complete(null); + } + runnables.forEach(Runnable::run); + // Initialize the message protocol handlers. // start the protocol handlers only after the broker is ready, // so that the protocol handlers can access broker service properly. @@ -900,7 +1054,7 @@ public void start() throws PulsarServerException { if (isNotBlank(config.getResourceUsageTransportClassName())) { Class clazz = Class.forName(config.getResourceUsageTransportClassName()); Constructor ctor = clazz.getConstructor(PulsarService.class); - Object object = ctor.newInstance(new Object[]{this}); + Object object = ctor.newInstance(this); this.resourceUsageTransportManager = (ResourceUsageTopicTransportManager) object; } this.resourceGroupServiceManager = new ResourceGroupService(this); @@ -925,12 +1079,40 @@ public void start() throws PulsarServerException { state = State.Started; } catch (Exception e) { LOG.error("Failed to start Pulsar service: {}", e.getMessage(), e); - throw new PulsarServerException(e); + PulsarServerException startException = PulsarServerException.from(e); + readyForIncomingRequestsFuture.completeExceptionally(startException); + throw startException; } finally { mutex.unlock(); } } + public void runWhenReadyForIncomingRequests(Runnable runnable) { + // Here we don't call the thenRun() methods because CompletableFuture maintains a stack for pending callbacks, + // not a queue. Once the future is complete, the pending callbacks will be executed in reverse order of + // when they were added. + final boolean addedToPendingTasks; + synchronized (pendingTasksBeforeReadyForIncomingRequests) { + if (readyForIncomingRequestsFuture.isDone()) { + addedToPendingTasks = false; + } else { + pendingTasksBeforeReadyForIncomingRequests.add(runnable); + addedToPendingTasks = true; + } + } + if (!addedToPendingTasks) { + runnable.run(); + } + } + + public void waitUntilReadyForIncomingRequests() throws ExecutionException, InterruptedException { + readyForIncomingRequestsFuture.get(); + } + + protected BrokerInterceptor newBrokerInterceptor() throws IOException { + return BrokerInterceptors.load(config); + } + @VisibleForTesting protected OrderedExecutor newOrderedExecutor() { return OrderedExecutor.newBuilder() @@ -943,14 +1125,14 @@ protected OrderedExecutor newOrderedExecutor() { protected ManagedLedgerStorage newManagedLedgerClientFactory() throws Exception { return ManagedLedgerStorage.create( config, localMetadataStore, - bkClientFactory, ioEventLoopGroup + bkClientFactory, ioEventLoopGroup, openTelemetry.getOpenTelemetryService().getOpenTelemetry() ); } @VisibleForTesting protected PulsarResources newPulsarResources() { PulsarResources pulsarResources = new PulsarResources(localMetadataStore, configurationMetadataStore, - config.getMetadataStoreOperationTimeoutSeconds()); + config.getMetadataStoreOperationTimeoutSeconds(), getExecutor()); pulsarResources.getClusterResources().getStore().registerListener(this::handleDeleteCluster); return pulsarResources; @@ -999,7 +1181,7 @@ private void addWebServerHandlers(WebService webService, true, attributeMap, true, Topics.class); // Add metrics servlet - webService.addServlet("/metrics", + webService.addServlet(PrometheusMetricsServlet.DEFAULT_METRICS_PATH, new ServletHolder(metricsServlet), config.isAuthenticateMetricsEndpoint(), attributeMap); @@ -1073,23 +1255,23 @@ private void addWebSocketServiceHandler(WebService webService, webService.addServlet(WebSocketReaderServlet.SERVLET_PATH_V2, new ServletHolder(readerWebSocketServlet), true, attributeMap); - final WebSocketServlet pingPongWebSocketServlet = new WebSocketPingPongServlet(webSocketService); - webService.addServlet(WebSocketPingPongServlet.SERVLET_PATH, - new ServletHolder(pingPongWebSocketServlet), true, attributeMap); - webService.addServlet(WebSocketPingPongServlet.SERVLET_PATH_V2, - new ServletHolder(pingPongWebSocketServlet), true, attributeMap); + final WebSocketMultiTopicConsumerServlet multiTopicConsumerWebSocketServlet = + new WebSocketMultiTopicConsumerServlet(webSocketService); + webService.addServlet(WebSocketMultiTopicConsumerServlet.SERVLET_PATH, + new ServletHolder(multiTopicConsumerWebSocketServlet), true, attributeMap); } } private void handleDeleteCluster(Notification notification) { - if (ClusterResources.pathRepresentsClusterName(notification.getPath()) + if (isRunning() && ClusterResources.pathRepresentsClusterName(notification.getPath()) && notification.getType() == NotificationType.Deleted) { final String clusterName = ClusterResources.clusterNameFromPath(notification.getPath()); getBrokerService().closeAndRemoveReplicationClient(clusterName); } } - public MetadataStoreExtended createLocalMetadataStore(PulsarMetadataEventSynchronizer synchronizer) + public MetadataStoreExtended createLocalMetadataStore(PulsarMetadataEventSynchronizer synchronizer, + OpenTelemetry openTelemetry) throws MetadataStoreException, PulsarServerException { return MetadataStoreExtended.create(config.getMetadataStoreUrl(), MetadataStoreConfig.builder() @@ -1102,73 +1284,82 @@ public MetadataStoreExtended createLocalMetadataStore(PulsarMetadataEventSynchro .batchingMaxSizeKb(config.getMetadataStoreBatchingMaxSizeKb()) .synchronizer(synchronizer) .metadataStoreName(MetadataStoreConfig.METADATA_STORE) + .openTelemetry(openTelemetry) .build()); } - protected void closeLocalMetadataStore() throws Exception { + protected CompletableFuture closeLocalMetadataStore() throws Exception { if (localMetadataStore != null) { localMetadataStore.close(); } if (localMetadataSynchronizer != null) { - localMetadataSynchronizer.close(); + CompletableFuture closeSynchronizer = localMetadataSynchronizer.closeAsync(); localMetadataSynchronizer = null; + return closeSynchronizer; } + return CompletableFuture.completedFuture(null); } protected void startLeaderElectionService() { - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(this)) { LOG.info("The load manager extension is enabled. Skipping PulsarService LeaderElectionService."); return; } - this.leaderElectionService = new LeaderElectionService(coordinationService, getSafeWebServiceAddress(), + this.leaderElectionService = + new LeaderElectionService(coordinationService, getBrokerId(), getSafeWebServiceAddress(), state -> { if (state == LeaderElectionState.Leading) { - LOG.info("This broker was elected leader"); + LOG.info("This broker {} was elected leader", getBrokerId()); if (getConfiguration().isLoadBalancerEnabled()) { - long resourceQuotaUpdateInterval = TimeUnit.MINUTES - .toMillis(getConfiguration().getLoadBalancerResourceQuotaUpdateIntervalMinutes()); - - if (loadSheddingTask != null) { - loadSheddingTask.cancel(); - } - if (loadResourceQuotaTask != null) { - loadResourceQuotaTask.cancel(false); - } - loadSheddingTask = new LoadSheddingTask(loadManager, loadManagerExecutor, config); - loadSheddingTask.start(); - loadResourceQuotaTask = loadManagerExecutor.scheduleAtFixedRate( - new LoadResourceQuotaUpdaterTask(loadManager), resourceQuotaUpdateInterval, - resourceQuotaUpdateInterval, TimeUnit.MILLISECONDS); + startLoadBalancerTasks(); } } else { if (leaderElectionService != null) { final Optional currentLeader = leaderElectionService.getCurrentLeader(); if (currentLeader.isPresent()) { - LOG.info("This broker is a follower. Current leader is {}", + LOG.info("This broker {} is a follower. Current leader is {}", getBrokerId(), currentLeader); } else { - LOG.info("This broker is a follower. No leader has been elected yet"); + LOG.info("This broker {} is a follower. No leader has been elected yet", getBrokerId()); } } - if (loadSheddingTask != null) { - loadSheddingTask.cancel(); - loadSheddingTask = null; - } - if (loadResourceQuotaTask != null) { - loadResourceQuotaTask.cancel(false); - loadResourceQuotaTask = null; - } + cancelLoadBalancerTasks(); } }); leaderElectionService.start(); } + private synchronized void cancelLoadBalancerTasks() { + if (loadSheddingTask != null) { + loadSheddingTask.cancel(); + loadSheddingTask = null; + } + if (loadResourceQuotaTask != null) { + loadResourceQuotaTask.cancel(false); + loadResourceQuotaTask = null; + } + } + + private synchronized void startLoadBalancerTasks() { + cancelLoadBalancerTasks(); + if (isRunning()) { + long resourceQuotaUpdateInterval = TimeUnit.MINUTES + .toMillis(getConfiguration().getLoadBalancerResourceQuotaUpdateIntervalMinutes()); + loadSheddingTask = new LoadSheddingTask(loadManager, loadManagerExecutor, + config, getManagedLedgerFactory()); + loadSheddingTask.start(); + loadResourceQuotaTask = loadManagerExecutor.scheduleAtFixedRate( + new LoadResourceQuotaUpdaterTask(loadManager), resourceQuotaUpdateInterval, + resourceQuotaUpdateInterval, TimeUnit.MILLISECONDS); + } + } + protected void acquireSLANamespace() { try { // Namespace not created hence no need to unload it - NamespaceName nsName = NamespaceService.getSLAMonitorNamespace(getAdvertisedAddress(), config); + NamespaceName nsName = NamespaceService.getSLAMonitorNamespace(getBrokerId(), config); if (!this.pulsarResources.getNamespaceResources().namespaceExists(nsName)) { LOG.info("SLA Namespace = {} doesn't exist.", nsName); return; @@ -1230,7 +1421,7 @@ protected void startLoadManagementService() throws PulsarServerException { LOG.info("Starting load management service ..."); this.loadManager.get().start(); - if (config.isLoadBalancerEnabled() && !ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + if (config.isLoadBalancerEnabled() && !ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(this)) { LOG.info("Starting load balancer"); if (this.loadReportTask == null) { long loadReportMinInterval = config.getLoadBalancerReportUpdateMinIntervalMillis(); @@ -1245,7 +1436,6 @@ protected void startLoadManagementService() throws PulsarServerException { * Load all the topics contained in a namespace. * * @param bundle NamespaceBundle to identify the service unit - * @throws Exception */ public void loadNamespaceTopics(NamespaceBundle bundle) { executor.submit(() -> { @@ -1300,15 +1490,9 @@ public InternalConfigurationData getInternalConfigurationData() { config.getConfigurationMetadataStoreUrl(), new ClientConfiguration().getZkLedgersRootPath(), config.isBookkeeperMetadataStoreSeparated() ? config.getBookkeeperMetadataStoreUrl() : null, - this.getWorkerConfig().map(wc -> wc.getStateStorageServiceUrl()).orElse(null)); + this.getWorkerConfig().map(WorkerConfig::getStateStorageServiceUrl).orElse(null)); } - /** - * Get the current pulsar state. - */ - public State getState() { - return this.state; - } /** * check the current pulsar service is running, including Started and Init state. @@ -1324,7 +1508,7 @@ public boolean isRunning() { * @return a reference of the current LeaderElectionService instance. */ public LeaderElectionService getLeaderElectionService() { - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(this)) { return ExtensibleLoadManagerImpl.get(loadManager.get()).getLeaderElectionService(); } else { return this.leaderElectionService; @@ -1350,16 +1534,6 @@ public WorkerService getWorkerService() throws UnsupportedOperationException { + "is not enabled, probably functionsWorkerEnabled is set to false")); } - /** - * Get a reference of the current BrokerService instance associated with the current - * PulsarService instance. - * - * @return a reference of the current BrokerService instance. - */ - public BrokerService getBrokerService() { - return this.brokerService; - } - public BookKeeper getBookKeeperClient() { return getManagedLedgerClientFactory().getBookKeeperClient(); } @@ -1368,10 +1542,6 @@ public ManagedLedgerFactory getManagedLedgerFactory() { return getManagedLedgerClientFactory().getManagedLedgerFactory(); } - public ManagedLedgerStorage getManagedLedgerClientFactory() { - return managedLedgerClientFactory; - } - /** * First, get LedgerOffloader from local map cache, * create new LedgerOffloader if not in cache or @@ -1415,7 +1585,7 @@ public LedgerOffloader createManagedLedgerOffloader(OffloadPoliciesImpl offloadP Offloaders offloaders = offloadersCache.getOrLoadOffloaders( offloadPolicies.getOffloadersDirectory(), config.getNarExtractionDirectory()); - LedgerOffloaderFactory offloaderFactory = offloaders.getOffloaderFactory( + LedgerOffloaderFactory offloaderFactory = offloaders.getOffloaderFactory( offloadPolicies.getManagedLedgerOffloadDriver()); try { return offloaderFactory.create( @@ -1450,26 +1620,6 @@ private SchemaStorage createAndStartSchemaStorage() throws Exception { return schemaStorage; } - public ScheduledExecutorService getExecutor() { - return executor; - } - - public ScheduledExecutorService getCacheExecutor() { - return cacheExecutor; - } - - public ExecutorProvider getTransactionExecutorProvider() { - return transactionExecutorProvider; - } - - public ScheduledExecutorService getLoadManagerExecutor() { - return loadManagerExecutor; - } - - public OrderedExecutor getOrderedExecutor() { - return orderedExecutor; - } - public BookKeeperClientFactory newBookKeeperClientFactory() { return new BookKeeperClientFactoryImpl(); } @@ -1478,7 +1628,7 @@ public BookKeeperClientFactory getBookKeeperClientFactory() { return bkClientFactory; } - protected synchronized ScheduledExecutorService getCompactorExecutor() { + public synchronized ScheduledExecutorService getCompactorExecutor() { if (this.compactorExecutor == null) { compactorExecutor = Executors.newSingleThreadScheduledExecutor( new ExecutorProvider.ExtendedThreadFactory("compaction")); @@ -1486,25 +1636,16 @@ protected synchronized ScheduledExecutorService getCompactorExecutor() { return this.compactorExecutor; } - // only public so mockito can mock it - public Compactor newCompactor() throws PulsarServerException { - return new TwoPhaseCompactor(this.getConfiguration(), - getClient(), getBookKeeperClient(), - getCompactorExecutor()); - } - - public synchronized Compactor getCompactor() throws PulsarServerException { - if (this.compactor == null) { - this.compactor = newCompactor(); - } - return this.compactor; - } - // This method is used for metrics, which is allowed to as null // Because it's no operation on the compactor, so let's remove the synchronized on this method // to avoid unnecessary lock competition. + // Only the pulsar's compaction service provides the compaction stats. The compaction service plugin, + // it should be done by the plugin itself to expose the compaction metrics. public Compactor getNullableCompactor() { - return this.compactor; + if (this.compactionServiceFactory instanceof PulsarCompactionServiceFactory pulsarCompactedServiceFactory) { + return pulsarCompactedServiceFactory.getNullableCompactor(); + } + return null; } public StrategicTwoPhaseCompactor newStrategicCompactor() throws PulsarServerException { @@ -1571,6 +1712,8 @@ public synchronized PulsarClient getClient() throws PulsarServerException { conf.setTlsProtocols(this.getConfiguration().getBrokerClientTlsProtocols()); conf.setTlsAllowInsecureConnection(this.getConfiguration().isTlsAllowInsecureConnection()); conf.setTlsHostnameVerificationEnable(this.getConfiguration().isTlsHostnameVerificationEnabled()); + conf.setSslFactoryPlugin(this.getConfiguration().getBrokerClientSslFactoryPlugin()); + conf.setSslFactoryPluginParams(this.getConfiguration().getBrokerClientSslFactoryPluginParams()); if (this.getConfiguration().isBrokerClientTlsEnabledWithKeyStore()) { conf.setUseKeyStoreTls(true); conf.setTlsTrustStoreType(this.getConfiguration().getBrokerClientTlsTrustStoreType()); @@ -1621,15 +1764,17 @@ public synchronized PulsarAdmin getAdminClient() throws PulsarServerException { // Apply all arbitrary configuration. This must be called before setting any fields annotated as // @Secret on the ClientConfigurationData object because of the way they are serialized. // See https://github.com/apache/pulsar/issues/8509 for more information. - builder.loadConf(PropertiesUtils.filterAndMapProperties(config.getProperties(), "brokerClient_")); + builder.loadConf(PropertiesUtils.filterAndMapProperties(conf.getProperties(), "brokerClient_")); builder.authentication( conf.getBrokerClientAuthenticationPlugin(), conf.getBrokerClientAuthenticationParameters()); if (conf.isBrokerClientTlsEnabled()) { - builder.tlsCiphers(config.getBrokerClientTlsCiphers()) - .tlsProtocols(config.getBrokerClientTlsProtocols()); + builder.tlsCiphers(conf.getBrokerClientTlsCiphers()) + .tlsProtocols(conf.getBrokerClientTlsProtocols()) + .sslFactoryPlugin(conf.getBrokerClientSslFactoryPlugin()) + .sslFactoryPluginParams(conf.getBrokerClientSslFactoryPluginParams()); if (conf.isBrokerClientTlsEnabledWithKeyStore()) { builder.useKeyStoreTls(true).tlsTrustStoreType(conf.getBrokerClientTlsTrustStoreType()) .tlsTrustStorePath(conf.getBrokerClientTlsTrustStore()) @@ -1660,22 +1805,6 @@ public synchronized PulsarAdmin getAdminClient() throws PulsarServerException { return this.adminClient; } - public MetricsGenerator getMetricsGenerator() { - return metricsGenerator; - } - - public TransactionMetadataStoreService getTransactionMetadataStoreService() { - return transactionMetadataStoreService; - } - - public TransactionBufferProvider getTransactionBufferProvider() { - return transactionBufferProvider; - } - - public TransactionBufferClient getTransactionBufferClient() { - return transactionBufferClient; - } - /** * Gets the broker service URL (non-TLS) associated with the internal listener. */ @@ -1707,7 +1836,8 @@ public String webAddress(ServiceConfiguration config) { AdvertisedListener internalListener = ServiceConfigurationUtils.getInternalListener(config, "http"); return internalListener.getBrokerHttpUrl() != null ? internalListener.getBrokerHttpUrl().toString() - : webAddress(ServiceConfigurationUtils.getWebServiceAddress(config), getListenPortHTTP().get()); + : webAddress(ServiceConfigurationUtils.getWebServiceAddress(config), + getListenPortHTTP().orElseThrow()); } else { return null; } @@ -1722,7 +1852,8 @@ public String webAddressTls(ServiceConfiguration config) { AdvertisedListener internalListener = ServiceConfigurationUtils.getInternalListener(config, "https"); return internalListener.getBrokerHttpsUrl() != null ? internalListener.getBrokerHttpsUrl().toString() - : webAddressTls(ServiceConfigurationUtils.getWebServiceAddress(config), getListenPortHTTPS().get()); + : webAddressTls(ServiceConfigurationUtils.getWebServiceAddress(config), + getListenPortHTTPS().orElseThrow()); } else { return null; } @@ -1733,26 +1864,24 @@ public static String webAddressTls(String host, int port) { } public String getSafeWebServiceAddress() { - return webServiceAddress != null ? webServiceAddress : webServiceAddressTls; + return webServiceAddressTls != null ? webServiceAddressTls : webServiceAddress; } @Deprecated public String getSafeBrokerServiceUrl() { - return brokerServiceUrl != null ? brokerServiceUrl : brokerServiceUrlTls; - } - - public String getLookupServiceAddress() { - return String.format("%s:%s", advertisedAddress, config.getWebServicePort().isPresent() - ? config.getWebServicePort().get() - : config.getWebServicePortTls().get()); + return brokerServiceUrlTls != null ? brokerServiceUrlTls : brokerServiceUrl; } - public TopicPoliciesService getTopicPoliciesService() { - return topicPoliciesService; - } - - public ResourceUsageTransportManager getResourceUsageTransportManager() { - return resourceUsageTransportManager; + /** + * Return the broker id. The broker id is used in the load manager to uniquely identify the broker at runtime. + * It should not be used for making connections to the broker. The broker id is available after {@link #start()} + * has been called. + * + * @return broker id + */ + public String getBrokerId() { + return Objects.requireNonNull(brokerId, + "brokerId is not initialized before start has been called"); } public synchronized void addPrometheusRawMetricsProvider(PrometheusRawMetricsProvider metricsProvider) { @@ -1806,21 +1935,22 @@ private void startWorkerService(AuthenticationService authenticationService, } public PackagesManagement getPackagesManagement() throws UnsupportedOperationException { - return packagesManagement.orElseThrow(() -> new UnsupportedOperationException("Package Management Service " - + "is not enabled in the broker.")); + if (packagesManagement == null) { + throw new UnsupportedOperationException("Package Management Service is not enabled in the broker."); + } + return packagesManagement; } private void startPackagesManagementService() throws IOException { // TODO: using provider to initialize the packages management service. - PackagesManagement packagesManagementService = new PackagesManagementImpl(); - this.packagesManagement = Optional.of(packagesManagementService); + this.packagesManagement = new PackagesManagementImpl(); PackagesStorageProvider storageProvider = PackagesStorageProvider .newProvider(config.getPackagesManagementStorageProvider()); DefaultPackagesStorageConfiguration storageConfiguration = new DefaultPackagesStorageConfiguration(); storageConfiguration.setProperty(config.getProperties()); PackagesStorage storage = storageProvider.getStorage(storageConfiguration); storage.initialize(); - packagesManagementService.initialize(storage); + this.packagesManagement.initialize(storage); } public Optional getListenPortHTTP() { @@ -1839,12 +1969,8 @@ public Optional getBrokerListenPortTls() { return brokerService.getListenPortTls(); } - public MetadataStoreExtended getLocalMetadataStore() { - return localMetadataStore; - } - - public CoordinationService getCoordinationService() { - return coordinationService; + public MonotonicSnapshotClock getMonotonicSnapshotClock() { + return monotonicSnapshotClock; } public static WorkerConfig initializeWorkerConfigFromBrokerConfig(ServiceConfiguration brokerConfig, @@ -1919,4 +2045,110 @@ public void shutdownNow() { protected BrokerService newBrokerService(PulsarService pulsar) throws Exception { return new BrokerService(pulsar, ioEventLoopGroup); } + + @VisibleForTesting + public void setTransactionBufferProvider(TransactionBufferProvider transactionBufferProvider) { + this.transactionBufferProvider = transactionBufferProvider; + } + + private CompactionServiceFactory loadCompactionServiceFactory() { + String compactionServiceFactoryClassName = config.getCompactionServiceFactoryClassName(); + var compactionServiceFactory = + Reflections.createInstance(compactionServiceFactoryClassName, CompactionServiceFactory.class, + Thread.currentThread().getContextClassLoader()); + compactionServiceFactory.initialize(this).join(); + return compactionServiceFactory; + } + + public CompletableFuture newTopicCompactionService(String topic) { + try { + CompactionServiceFactory compactionServiceFactory = this.getCompactionServiceFactory(); + return compactionServiceFactory.newTopicCompactionService(topic); + } catch (Throwable e) { + return CompletableFuture.failedFuture(e); + } + } + + public void initConfigMetadataSynchronizerIfNeeded() { + mutex.lock(); + try { + final String newTopic = config.getConfigurationMetadataSyncEventTopic(); + final PulsarMetadataEventSynchronizer oldSynchronizer = configMetadataSynchronizer; + // Skip if not support. + if (!(configurationMetadataStore instanceof MetadataStoreExtended)) { + LOG.info( + "Skip to update Metadata Synchronizer because of the Configuration Metadata Store using[{}]" + + " does not support.", configurationMetadataStore.getClass().getName()); + return; + } + // Skip if no changes. + // case-1: both null. + // case-2: both topics are the same. + if ((oldSynchronizer == null && StringUtils.isBlank(newTopic))) { + LOG.info("Skip to update Metadata Synchronizer because the topic[null] does not changed."); + } + if (StringUtils.isNotBlank(newTopic) && oldSynchronizer != null) { + TopicName newTopicName = TopicName.get(newTopic); + TopicName oldTopicName = TopicName.get(oldSynchronizer.getTopicName()); + if (newTopicName.equals(oldTopicName)) { + LOG.info("Skip to update Metadata Synchronizer because the topic[{}] does not changed.", + oldTopicName); + } + } + // Update(null or not null). + // 1.set the new one. + // 2.close the old one. + // 3.async start the new one. + if (StringUtils.isBlank(newTopic)) { + configMetadataSynchronizer = null; + } else { + configMetadataSynchronizer = new PulsarMetadataEventSynchronizer(this, newTopic); + } + // close the old one and start the new one. + PulsarMetadataEventSynchronizer newSynchronizer = configMetadataSynchronizer; + MetadataStoreExtended metadataStoreExtended = (MetadataStoreExtended) configurationMetadataStore; + metadataStoreExtended.updateMetadataEventSynchronizer(newSynchronizer); + Runnable startNewSynchronizer = () -> { + if (newSynchronizer == null) { + return; + } + try { + newSynchronizer.start(); + } catch (Exception e) { + // It only occurs when get internal client fails. + LOG.error("Start Metadata Synchronizer with topic {} failed.", + newTopic, e); + } + }; + executor.submit(() -> { + if (oldSynchronizer != null) { + oldSynchronizer.closeAsync().whenComplete((ignore, ex) -> { + startNewSynchronizer.run(); + }); + } else { + startNewSynchronizer.run(); + } + }); + } finally { + mutex.unlock(); + } + } + + private TopicPoliciesService initTopicPoliciesService() throws Exception { + if (!config.isTopicLevelPoliciesEnabled()) { + return TopicPoliciesService.DISABLED; + } + final var className = Optional.ofNullable(config.getTopicPoliciesServiceClassName()) + .orElse(SystemTopicBasedTopicPoliciesService.class.getName()); + if (className.equals(SystemTopicBasedTopicPoliciesService.class.getName())) { + if (config.isSystemTopicEnabled()) { + return new SystemTopicBasedTopicPoliciesService(this); + } else { + LOG.warn("System topic is disabled while the topic policies service is {}, disable it", className); + return TopicPoliciesService.DISABLED; + } + } + return (TopicPoliciesService) Reflections.createInstance(className, + Thread.currentThread().getContextClassLoader()); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java index 3e3b044ec51b8..c80580b02f19a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java @@ -85,16 +85,13 @@ public class TransactionMetadataStoreService { private final Timer transactionOpRetryTimer; // this semaphore for loading one transaction coordinator with the same tc id on the same time private final ConcurrentLongHashMap tcLoadSemaphores; - // one connect request open the transactionMetaStore the other request will add to the queue, when the open op - // finished the request will be poll and complete the future + // one connect request opens the transactionMetaStore the other request will add to the queue, when the open op + // finishes the request will be polled and will complete the future private final ConcurrentLongHashMap>> pendingConnectRequests; private final ExecutorService internalPinnedExecutor; private static final long HANDLE_PENDING_CONNECT_TIME_OUT = 30000L; - private final ThreadFactory threadFactory = - new ExecutorProvider.ExtendedThreadFactory("transaction-coordinator-thread-factory"); - public TransactionMetadataStoreService(TransactionMetadataStoreProvider transactionMetadataStoreProvider, PulsarService pulsarService, TransactionBufferClient tbClient, @@ -108,6 +105,8 @@ public TransactionMetadataStoreService(TransactionMetadataStoreProvider transact this.tcLoadSemaphores = ConcurrentLongHashMap.newBuilder().build(); this.pendingConnectRequests = ConcurrentLongHashMap.>>newBuilder().build(); + ThreadFactory threadFactory = + new ExecutorProvider.ExtendedThreadFactory("transaction-coordinator-thread-factory"); this.internalPinnedExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); } @@ -169,7 +168,8 @@ public CompletableFuture handleTcClientConnect(TransactionCoordinatorID tc tcLoadSemaphore.release(); })).exceptionally(e -> { internalPinnedExecutor.execute(() -> { - completableFuture.completeExceptionally(e.getCause()); + Throwable realCause = FutureUtil.unwrapCompletionException(e); + completableFuture.completeExceptionally(realCause); // release before handle request queue, //in order to client reconnect infinite loop tcLoadSemaphore.release(); @@ -180,7 +180,7 @@ public CompletableFuture handleTcClientConnect(TransactionCoordinatorID tc CompletableFuture future = deque.poll(); if (future != null) { // this means that this tc client connection connect fail - future.completeExceptionally(e); + future.completeExceptionally(realCause); } else { break; } @@ -199,7 +199,7 @@ public CompletableFuture handleTcClientConnect(TransactionCoordinatorID tc // then handle the requests witch in the queue deque.add(completableFuture); if (LOG.isDebugEnabled()) { - LOG.debug("Handle tc client connect added into pending queue! tcId : {}", tcId.toString()); + LOG.debug("Handle tc client connect added into pending queue! tcId : {}", tcId); } } })).exceptionally(ex -> { @@ -366,17 +366,11 @@ public void endTransaction(TxnID txnID, int txnAction, boolean isTimeout, private CompletionStage fakeAsyncCheckTxnStatus(TxnStatus txnStatus, int txnAction, TxnID txnID, TxnStatus expectStatus) { - boolean isLegal; - switch (txnStatus) { - case COMMITTING: - isLegal = (txnAction == TxnAction.COMMIT.getValue()); - break; - case ABORTING: - isLegal = (txnAction == TxnAction.ABORT.getValue()); - break; - default: - isLegal = false; - } + boolean isLegal = switch (txnStatus) { + case COMMITTING -> (txnAction == TxnAction.COMMIT.getValue()); + case ABORTING -> (txnAction == TxnAction.ABORT.getValue()); + default -> false; + }; if (!isLegal) { if (LOG.isDebugEnabled()) { LOG.debug("EndTxnInTransactionBuffer op retry! TxnId : {}, TxnAction : {}", txnID, txnAction); @@ -501,15 +495,14 @@ public CompletableFuture verifyTxnOwnership(TxnID txnID, String checkOw public void close () { this.internalPinnedExecutor.shutdown(); - stores.forEach((tcId, metadataStore) -> { + stores.forEach((tcId, metadataStore) -> metadataStore.closeAsync().whenComplete((v, ex) -> { if (ex != null) { LOG.error("Close transaction metadata store with id " + tcId, ex); } else { LOG.info("Removed and closed transaction meta store {}", tcId); } - }); - }); + })); stores.clear(); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java index 4190b4c486a4c..3268f07b13d88 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java @@ -23,6 +23,7 @@ import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -42,9 +43,14 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.authorization.AuthorizationService; +import org.apache.pulsar.broker.resources.ClusterResources; +import org.apache.pulsar.broker.service.TopicPoliciesService; import org.apache.pulsar.broker.service.plugin.InvalidEntryFilterException; import org.apache.pulsar.broker.web.PulsarWebResource; import org.apache.pulsar.broker.web.RestException; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.admin.internal.TopicsImpl; import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace; import org.apache.pulsar.common.naming.Constants; @@ -56,11 +62,11 @@ import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.BundlesData; import org.apache.pulsar.common.policies.data.EntryFilters; +import org.apache.pulsar.common.policies.data.LocalPolicies; import org.apache.pulsar.common.policies.data.NamespaceOperation; +import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; import org.apache.pulsar.common.policies.data.PersistencePolicies; import org.apache.pulsar.common.policies.data.Policies; -import org.apache.pulsar.common.policies.data.PolicyName; -import org.apache.pulsar.common.policies.data.PolicyOperation; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.policies.data.SubscribeRate; @@ -302,7 +308,9 @@ protected Policies getNamespacePolicies(NamespaceName namespaceName) { // fetch bundles from LocalZK-policies BundlesData bundleData = pulsar().getNamespaceService().getNamespaceBundleFactory() .getBundles(namespaceName).getBundlesData(); + Optional localPolicies = getLocalPolicies().getLocalPolicies(namespaceName); policies.bundles = bundleData != null ? bundleData : policies.bundles; + policies.migrated = localPolicies.isPresent() ? localPolicies.get().migrated : false; if (policies.is_allow_auto_update_schema == null) { // the type changed from boolean to Boolean. return broker value here for keeping compatibility. policies.is_allow_auto_update_schema = pulsar().getConfig().isAllowAutoUpdateSchemaEnabled(); @@ -319,32 +327,31 @@ protected Policies getNamespacePolicies(NamespaceName namespaceName) { } protected CompletableFuture getNamespacePoliciesAsync(NamespaceName namespaceName) { - return namespaceResources().getPoliciesAsync(namespaceName).thenCompose(policies -> { - if (policies.isPresent()) { - return pulsar() - .getNamespaceService() - .getNamespaceBundleFactory() - .getBundlesAsync(namespaceName) - .thenCompose(bundles -> { - BundlesData bundleData = null; - try { - bundleData = bundles.getBundlesData(); - } catch (Exception e) { - log.error("[{}] Failed to get namespace policies {}", clientAppId(), namespaceName, e); - return FutureUtil.failedFuture(new RestException(e)); - } - policies.get().bundles = bundleData != null ? bundleData : policies.get().bundles; - if (policies.get().is_allow_auto_update_schema == null) { - // the type changed from boolean to Boolean. return broker value here for keeping compatibility. - policies.get().is_allow_auto_update_schema = pulsar().getConfig() - .isAllowAutoUpdateSchemaEnabled(); + CompletableFuture result = new CompletableFuture<>(); + namespaceResources().getPoliciesAsync(namespaceName) + .thenCombine(getLocalPolicies().getLocalPoliciesAsync(namespaceName), (pl, localPolicies) -> { + if (pl.isPresent()) { + Policies policies = pl.get(); + if (localPolicies.isPresent()) { + policies.bundles = localPolicies.get().bundles; + policies.migrated = localPolicies.get().migrated; + } + if (policies.is_allow_auto_update_schema == null) { + // the type changed from boolean to Boolean. return + // broker value here for keeping compatibility. + policies.is_allow_auto_update_schema = pulsar().getConfig() + .isAllowAutoUpdateSchemaEnabled(); + } + result.complete(policies); + } else { + result.completeExceptionally(new RestException(Status.NOT_FOUND, "Namespace does not exist")); } - return CompletableFuture.completedFuture(policies.get()); + return null; + }).exceptionally(ex -> { + result.completeExceptionally(ex.getCause()); + return null; }); - } else { - return FutureUtil.failedFuture(new RestException(Status.NOT_FOUND, "Namespace does not exist")); - } - }); + return result; } protected BacklogQuota namespaceBacklogQuota(NamespaceName namespace, @@ -359,14 +366,8 @@ protected CompletableFuture> getTopicPoliciesAsyncWithRe protected CompletableFuture> getTopicPoliciesAsyncWithRetry(TopicName topicName, boolean isGlobal) { - try { - checkTopicLevelPolicyEnable(); - return pulsar().getTopicPoliciesService() - .getTopicPoliciesAsyncWithRetry(topicName, null, pulsar().getExecutor(), isGlobal); - } catch (Exception e) { - log.error("[{}] Failed to get topic policies {}", clientAppId(), topicName, e); - return FutureUtil.failedFuture(e); - } + final var type = isGlobal ? TopicPoliciesService.GetType.GLOBAL_ONLY : TopicPoliciesService.GetType.LOCAL_ONLY; + return pulsar().getTopicPoliciesService().getTopicPoliciesAsync(topicName, type); } protected boolean checkBacklogQuota(BacklogQuota quota, RetentionPolicies retention) { @@ -390,13 +391,6 @@ protected boolean checkBacklogQuota(BacklogQuota quota, RetentionPolicies retent return true; } - protected void checkTopicLevelPolicyEnable() { - if (!config().isTopicLevelPoliciesEnabled()) { - throw new RestException(Status.METHOD_NOT_ALLOWED, - "Topic level policies is disabled, to enable the topic level policy and retry."); - } - } - protected DispatchRateImpl dispatchRate() { return DispatchRateImpl.builder() .dispatchThrottlingRateInMsg(config().getDispatchThrottlingRatePerTopicInMsg()) @@ -478,9 +472,9 @@ protected CompletableFuture getPartitionedTopicMetadat // validates global-namespace contains local/peer cluster: if peer/local cluster present then lookup can // serve/redirect request else fail partitioned-metadata-request so, client fails while creating // producer/consumer - return validateClusterOwnershipAsync(topicName.getCluster()) + return validateTopicOperationAsync(topicName, TopicOperation.LOOKUP) + .thenCompose(__ -> validateClusterOwnershipAsync(topicName.getCluster())) .thenCompose(__ -> validateGlobalNamespaceOwnershipAsync(topicName.getNamespaceObject())) - .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.LOOKUP)) .thenCompose(__ -> { if (checkAllowAutoCreation) { return pulsar().getBrokerService() @@ -603,11 +597,15 @@ protected void internalCreatePartitionedTopic(AsyncResponse asyncResponse, int n .thenCompose(__ -> provisionPartitionedTopicPath(numPartitions, createLocalTopicOnly, properties)) .thenCompose(__ -> tryCreatePartitionsAsync(numPartitions)) .thenRun(() -> { - if (!createLocalTopicOnly && topicName.isGlobal()) { + if (!createLocalTopicOnly && topicName.isGlobal() + && pulsar().getConfig().isCreateTopicToRemoteClusterForReplication()) { internalCreatePartitionedTopicToReplicatedClustersInBackground(numPartitions); + log.info("[{}] Successfully created partitioned for topic {} for the remote clusters", + clientAppId(), topicName); + } else { + log.info("[{}] Skip creating partitioned for topic {} for the remote clusters", + clientAppId(), topicName); } - log.info("[{}] Successfully created partitions for topic {} in cluster {}", - clientAppId(), topicName, pulsar().getConfiguration().getClusterName()); asyncResponse.resume(Response.noContent().build()); }) .exceptionally(ex -> { @@ -619,35 +617,90 @@ protected void internalCreatePartitionedTopic(AsyncResponse asyncResponse, int n private void internalCreatePartitionedTopicToReplicatedClustersInBackground(int numPartitions) { getNamespaceReplicatedClustersAsync(namespaceName) - .thenAccept(clusters -> { - for (String cluster : clusters) { - if (!cluster.equals(pulsar().getConfiguration().getClusterName())) { - // this call happens in the background without async composition. completion is logged. - pulsar().getPulsarResources().getClusterResources() - .getClusterAsync(cluster) - .thenCompose(clusterDataOp -> - ((TopicsImpl) pulsar().getBrokerService() - .getClusterPulsarAdmin(cluster, - clusterDataOp).topics()) - .createPartitionedTopicAsync( - topicName.getPartitionedTopicName(), - numPartitions, - true, null)) - .whenComplete((__, ex) -> { - if (ex != null) { - log.error( - "[{}] Failed to create partitioned topic {} in cluster {}.", - clientAppId(), topicName, cluster, ex); - } else { - log.info( - "[{}] Successfully created partitioned topic {} in " - + "cluster {}", - clientAppId(), topicName, cluster); - } - }); - } + .thenAccept(clusters -> { + // this call happens in the background without async composition. completion is logged. + internalCreatePartitionedTopicToReplicatedClustersInBackground(clusters, numPartitions); + }); + } + + protected Map> internalCreatePartitionedTopicToReplicatedClustersInBackground ( + Set clusters, int numPartitions) { + final String shortTopicName = topicName.getPartitionedTopicName(); + Map> tasksForAllClusters = new HashMap<>(); + for (String cluster : clusters) { + if (cluster.equals(pulsar().getConfiguration().getClusterName())) { + continue; + } + ClusterResources clusterResources = pulsar().getPulsarResources().getClusterResources(); + CompletableFuture createRemoteTopicFuture = new CompletableFuture<>(); + tasksForAllClusters.put(cluster, createRemoteTopicFuture); + clusterResources.getClusterAsync(cluster).whenComplete((clusterData, ex1) -> { + if (ex1 != null) { + // Unexpected error, such as NPE. Catch all error to avoid the "createRemoteTopicFuture" stuck. + log.error("[{}] An un-expected error occurs when trying to create partitioned topic {} in cluster" + + " {}.", clientAppId(), topicName, cluster, ex1); + createRemoteTopicFuture.completeExceptionally(new RestException(ex1)); + return; + } + PulsarAdmin remotePulsarAdmin; + try { + remotePulsarAdmin = pulsar().getBrokerService().getClusterPulsarAdmin(cluster, clusterData); + } catch (Exception ex) { + log.error("[{}] [{}] An un-expected error occurs when trying to create remote pulsar admin for" + + " cluster {}", clientAppId(), topicName, cluster, ex); + createRemoteTopicFuture.completeExceptionally(new RestException(ex)); + return; + } + // Get cluster data success. + TopicsImpl topics = (TopicsImpl) remotePulsarAdmin.topics(); + topics.createPartitionedTopicAsync(shortTopicName, numPartitions, true, null) + .whenComplete((ignore, ex2) -> { + if (ex2 == null) { + // Create success. + log.info("[{}] Successfully created partitioned topic {} in cluster {}", + clientAppId(), topicName, cluster); + createRemoteTopicFuture.complete(null); + return; + } + // Create topic on the remote cluster error. + Throwable unwrapEx2 = FutureUtil.unwrapCompletionException(ex2); + // The topic has been created before, check the partitions count is expected. + if (unwrapEx2 instanceof PulsarAdminException.ConflictException) { + topics.getPartitionedTopicMetadataAsync(shortTopicName).whenComplete((topicMeta, ex3) -> { + if (ex3 != null) { + // Unexpected error, such as NPE. Catch all error to avoid the + // "createRemoteTopicFuture" stuck. + log.error("[{}] Failed to check remote-cluster's topic metadata when creating" + + " partitioned topic {} in cluster {}.", + clientAppId(), topicName, cluster, ex3); + createRemoteTopicFuture.completeExceptionally(new RestException(ex3)); + } + // Call get partitioned metadata of remote cluster success. + if (topicMeta.partitions == numPartitions) { + log.info("[{}] Skip created partitioned topic {} in cluster {}, because that {}", + clientAppId(), topicName, cluster, unwrapEx2.getMessage()); + createRemoteTopicFuture.complete(null); + } else { + String errorMsg = String.format("[%s] There is an exists topic %s with different" + + " partitions %s on the remote cluster %s, you want to create it" + + " with partitions %s", + clientAppId(), shortTopicName, topicMeta.partitions, cluster, + numPartitions); + log.error(errorMsg); + createRemoteTopicFuture.completeExceptionally( + new RestException(Status.PRECONDITION_FAILED, errorMsg)); + } + }); + } else { + // An HTTP error was responded from the remote cluster. + log.error("[{}] Failed to create partitioned topic {} in cluster {}.", + clientAppId(), topicName, cluster, ex2); + createRemoteTopicFuture.completeExceptionally(new RestException(unwrapEx2)); } }); + }); + } + return tasksForAllClusters; } /** @@ -710,10 +763,7 @@ private CompletableFuture provisionPartitionedTopicPath(int numPartitions, } protected CompletableFuture getSchemaCompatibilityStrategyAsync() { - return validateTopicPolicyOperationAsync(topicName, - PolicyName.SCHEMA_COMPATIBILITY_STRATEGY, - PolicyOperation.READ) - .thenCompose((__) -> getSchemaCompatibilityStrategyAsyncWithoutAuth()).whenComplete((__, ex) -> { + return getSchemaCompatibilityStrategyAsyncWithoutAuth().whenComplete((__, ex) -> { if (ex != null) { log.error("[{}] Failed to get schema compatibility strategy of topic {} {}", clientAppId(), topicName, ex); @@ -722,11 +772,8 @@ protected CompletableFuture getSchemaCompatibilityS } protected CompletableFuture getSchemaCompatibilityStrategyAsyncWithoutAuth() { - CompletableFuture future = CompletableFuture.completedFuture(null); - if (config().isTopicLevelPoliciesEnabled()) { - future = getTopicPoliciesAsyncWithRetry(topicName) - .thenApply(op -> op.map(TopicPolicies::getSchemaCompatibilityStrategy).orElse(null)); - } + CompletableFuture future = getTopicPoliciesAsyncWithRetry(topicName) + .thenApply(op -> op.map(TopicPolicies::getSchemaCompatibilityStrategy).orElse(null)); return future.thenCompose((topicSchemaCompatibilityStrategy) -> { if (!SchemaCompatibilityStrategy.isUndefined(topicSchemaCompatibilityStrategy)) { @@ -830,6 +877,10 @@ protected static boolean isNotFoundException(Throwable ex) { == Status.NOT_FOUND.getStatusCode(); } + protected static boolean isNot307And404Exception(Throwable ex) { + return !isRedirectException(ex) && !isNotFoundException(ex); + } + protected static String getTopicNotFoundErrorMessage(String topic) { return String.format("Topic %s not found", topic); } @@ -847,4 +898,30 @@ protected List filterSystemTopic(List topics, boolean includeSys .filter(topic -> includeSystemTopic ? true : !pulsar().getBrokerService().isSystemTopic(topic)) .collect(Collectors.toList()); } + + protected AuthorizationService getAuthorizationService() { + return pulsar().getBrokerService().getAuthorizationService(); + } + + protected void validateOffloadPolicies(OffloadPoliciesImpl offloadPolicies) { + if (offloadPolicies == null) { + log.warn("[{}] Failed to update offload configuration for namespace {}: offloadPolicies is null", + clientAppId(), namespaceName); + throw new RestException(Status.PRECONDITION_FAILED, + "The offloadPolicies must be specified for namespace offload."); + } + if (!offloadPolicies.driverSupported()) { + log.warn("[{}] Failed to update offload configuration for namespace {}: " + + "driver is not supported, support value: {}", + clientAppId(), namespaceName, OffloadPoliciesImpl.getSupportedDriverNames()); + throw new RestException(Status.PRECONDITION_FAILED, + "The driver is not supported, support value: " + OffloadPoliciesImpl.getSupportedDriverNames()); + } + if (!offloadPolicies.bucketValid()) { + log.warn("[{}] Failed to update offload configuration for namespace {}: bucket must be specified", + clientAppId(), namespaceName); + throw new RestException(Status.PRECONDITION_FAILED, + "The bucket must be specified for namespace offload."); + } + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokerStatsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokerStatsBase.java index 6d49dd81da13d..48577fc701486 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokerStatsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokerStatsBase.java @@ -125,6 +125,9 @@ public AllocatorStats getAllocatorStats(@PathParam("allocator") String allocator @GET @Path("/bookieops") @ApiOperation(value = "Get pending bookie client op stats by namespace", + notes = "Returns a nested map structure which Swagger does not fully support for display. " + + "Structure: Map>." + + " Please refer to this structure for details.", response = PendingBookieOpsStats.class, // https://github.com/swagger-api/swagger-core/issues/449 // nested containers are not supported diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java index b367ce7aad955..da4cee7b4651c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java @@ -26,6 +26,7 @@ import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -34,6 +35,7 @@ import java.util.Objects; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; @@ -46,8 +48,10 @@ import javax.ws.rs.container.Suspended; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; +import org.apache.commons.lang.StringUtils; import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.PulsarService.State; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.admin.AdminResource; @@ -80,12 +84,18 @@ public class BrokersBase extends AdminResource { // log a full thread dump when a deadlock is detected in healthcheck once every 10 minutes // to prevent excessive logging private static final long LOG_THREADDUMP_INTERVAL_WHEN_DEADLOCK_DETECTED = 600000L; - private volatile long threadDumpLoggedTimestamp; + // there is a timeout of 60 seconds default in the client(readTimeoutMs), so we need to set the timeout + // a bit shorter than 60 seconds to avoid the client timeout exception thrown before the server timeout exception. + // or we can't propagate the server timeout exception to the client. + private static final Duration HEALTH_CHECK_READ_TIMEOUT = Duration.ofSeconds(58); + private static final TimeoutException HEALTH_CHECK_TIMEOUT_EXCEPTION = + FutureUtil.createTimeoutException("Timeout", BrokersBase.class, "healthCheckRecursiveReadNext(...)"); + private static volatile long threadDumpLoggedTimestamp; @GET @Path("/{cluster}") @ApiOperation( - value = "Get the list of active brokers (web service addresses) in the cluster." + value = "Get the list of active brokers (broker ids) in the cluster." + "If authorization is not enabled, any cluster name is valid.", response = String.class, responseContainer = "Set") @@ -115,7 +125,7 @@ public void getActiveBrokers(@Suspended final AsyncResponse asyncResponse, @GET @ApiOperation( - value = "Get the list of active brokers (web service addresses) in the local cluster." + value = "Get the list of active brokers (broker ids) in the local cluster." + "If authorization is not enabled", response = String.class, responseContainer = "Set") @@ -141,7 +151,9 @@ public void getLeaderBroker(@Suspended final AsyncResponse asyncResponse) { validateSuperUserAccessAsync().thenAccept(__ -> { LeaderBroker leaderBroker = pulsar().getLeaderElectionService().getCurrentLeader() .orElseThrow(() -> new RestException(Status.NOT_FOUND, "Couldn't find leader broker")); - BrokerInfo brokerInfo = BrokerInfo.builder().serviceUrl(leaderBroker.getServiceUrl()).build(); + BrokerInfo brokerInfo = BrokerInfo.builder() + .serviceUrl(leaderBroker.getServiceUrl()) + .brokerId(leaderBroker.getBrokerId()).build(); LOG.info("[{}] Successfully to get the information of the leader broker.", clientAppId()); asyncResponse.resume(brokerInfo); }) @@ -153,8 +165,8 @@ public void getLeaderBroker(@Suspended final AsyncResponse asyncResponse) { } @GET - @Path("/{clusterName}/{broker-webserviceurl}/ownedNamespaces") - @ApiOperation(value = "Get the list of namespaces served by the specific broker", + @Path("/{clusterName}/{brokerId}/ownedNamespaces") + @ApiOperation(value = "Get the list of namespaces served by the specific broker id", response = NamespaceOwnershipStatus.class, responseContainer = "Map") @ApiResponses(value = { @ApiResponse(code = 307, message = "Current broker doesn't serve the cluster"), @@ -162,9 +174,9 @@ public void getLeaderBroker(@Suspended final AsyncResponse asyncResponse) { @ApiResponse(code = 404, message = "Cluster doesn't exist") }) public void getOwnedNamespaces(@Suspended final AsyncResponse asyncResponse, @PathParam("clusterName") String cluster, - @PathParam("broker-webserviceurl") String broker) { + @PathParam("brokerId") String brokerId) { validateSuperUserAccessAsync() - .thenAccept(__ -> validateBrokerName(broker)) + .thenCompose(__ -> maybeRedirectToBroker(brokerId)) .thenCompose(__ -> validateClusterOwnershipAsync(cluster)) .thenCompose(__ -> pulsar().getNamespaceService().getOwnedNameSpacesStatusAsync()) .thenAccept(asyncResponse::resume) @@ -172,7 +184,7 @@ public void getOwnedNamespaces(@Suspended final AsyncResponse asyncResponse, // If the exception is not redirect exception we need to log it. if (!isRedirectException(ex)) { LOG.error("[{}] Failed to get the namespace ownership status. cluster={}, broker={}", - clientAppId(), cluster, broker); + clientAppId(), cluster, brokerId); } resumeAsyncResponseExceptionally(asyncResponse, ex); return null; @@ -209,7 +221,7 @@ public void updateDynamicConfiguration(@Suspended AsyncResponse asyncResponse, @ApiOperation(value = "Delete dynamic ServiceConfiguration into metadata only." + " This operation requires Pulsar super-user privileges.") - @ApiResponses(value = { @ApiResponse(code = 204, message = "Service configuration updated successfully"), + @ApiResponses(value = { @ApiResponse(code = 204, message = "Service configuration delete successfully"), @ApiResponse(code = 403, message = "You don't have admin permission to update service-configuration"), @ApiResponse(code = 412, message = "Invalid dynamic-config value"), @ApiResponse(code = 500, message = "Internal server error") }) @@ -230,7 +242,8 @@ public void deleteDynamicConfiguration( @GET @Path("/configuration/values") - @ApiOperation(value = "Get value of all dynamic configurations' value overridden on local config") + @ApiOperation(value = "Get value of all dynamic configurations' value overridden on local config", + response = String.class, responseContainer = "Map") @ApiResponses(value = { @ApiResponse(code = 403, message = "You don't have admin permission to view configuration"), @ApiResponse(code = 404, message = "Configuration not found"), @@ -248,7 +261,8 @@ public void getAllDynamicConfigurations(@Suspended AsyncResponse asyncResponse) @GET @Path("/configuration") - @ApiOperation(value = "Get all updatable dynamic configurations's name") + @ApiOperation(value = "Get all updatable dynamic configurations's name", + response = String.class, responseContainer = "List") @ApiResponses(value = { @ApiResponse(code = 403, message = "You don't have admin permission to get configuration")}) public void getDynamicConfigurationName(@Suspended AsyncResponse asyncResponse) { @@ -263,7 +277,8 @@ public void getDynamicConfigurationName(@Suspended AsyncResponse asyncResponse) @GET @Path("/configuration/runtime") - @ApiOperation(value = "Get all runtime configurations. This operation requires Pulsar super-user privileges.") + @ApiOperation(value = "Get all runtime configurations. This operation requires Pulsar super-user privileges.", + response = String.class, responseContainer = "Map") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission") }) public void getRuntimeConfiguration(@Suspended AsyncResponse asyncResponse) { validateSuperUserAccessAsync() @@ -320,7 +335,7 @@ public void getInternalConfigurationData(@Suspended AsyncResponse asyncResponse) @Path("/backlog-quota-check") @ApiOperation(value = "An REST endpoint to trigger backlogQuotaCheck") @ApiResponses(value = { - @ApiResponse(code = 200, message = "Everything is OK"), + @ApiResponse(code = 204, message = "Everything is OK"), @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 500, message = "Internal server error")}) public void backlogQuotaCheck(@Suspended AsyncResponse asyncResponse) { @@ -355,20 +370,26 @@ public void isReady(@Suspended AsyncResponse asyncResponse) { @ApiOperation(value = "Run a healthCheck against the broker") @ApiResponses(value = { @ApiResponse(code = 200, message = "Everything is OK"), + @ApiResponse(code = 307, message = "Current broker is not the target broker"), @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Cluster doesn't exist"), @ApiResponse(code = 500, message = "Internal server error")}) - @ApiParam(value = "Topic Version") public void healthCheck(@Suspended AsyncResponse asyncResponse, - @QueryParam("topicVersion") TopicVersion topicVersion) { + @ApiParam(value = "Topic Version") + @QueryParam("topicVersion") TopicVersion topicVersion, + @QueryParam("brokerId") String brokerId) { validateSuperUserAccessAsync() .thenAccept(__ -> checkDeadlockedThreads()) + .thenCompose(__ -> maybeRedirectToBroker( + StringUtils.isBlank(brokerId) ? pulsar().getBrokerId() : brokerId)) .thenCompose(__ -> internalRunHealthCheck(topicVersion)) .thenAccept(__ -> { LOG.info("[{}] Successfully run health check.", clientAppId()); - asyncResponse.resume("ok"); + asyncResponse.resume(Response.ok("ok").build()); }).exceptionally(ex -> { - LOG.error("[{}] Fail to run health check.", clientAppId(), ex); + if (!isRedirectException(ex)) { + LOG.error("[{}] Fail to run health check.", clientAppId(), ex); + } resumeAsyncResponseExceptionally(asyncResponse, ex); return null; }); @@ -394,29 +415,43 @@ private void checkDeadlockedThreads() { } } + public static String getHeartbeatTopicName(String brokerId, ServiceConfiguration configuration, boolean isV2) { + NamespaceName namespaceName = isV2 + ? NamespaceService.getHeartbeatNamespaceV2(brokerId, configuration) + : NamespaceService.getHeartbeatNamespace(brokerId, configuration); + return String.format("persistent://%s/%s", namespaceName, HEALTH_CHECK_TOPIC_SUFFIX); + } private CompletableFuture internalRunHealthCheck(TopicVersion topicVersion) { + return internalRunHealthCheck(topicVersion, pulsar(), clientAppId()); + } + + + public static CompletableFuture internalRunHealthCheck(TopicVersion topicVersion, PulsarService pulsar, + String clientAppId) { NamespaceName namespaceName = (topicVersion == TopicVersion.V2) - ? NamespaceService.getHeartbeatNamespaceV2(pulsar().getAdvertisedAddress(), pulsar().getConfiguration()) - : NamespaceService.getHeartbeatNamespace(pulsar().getAdvertisedAddress(), pulsar().getConfiguration()); - final String topicName = String.format("persistent://%s/%s", namespaceName, HEALTH_CHECK_TOPIC_SUFFIX); - LOG.info("[{}] Running healthCheck with topic={}", clientAppId(), topicName); + ? NamespaceService.getHeartbeatNamespaceV2(pulsar.getAdvertisedAddress(), pulsar.getConfiguration()) + : NamespaceService.getHeartbeatNamespace(pulsar.getAdvertisedAddress(), pulsar.getConfiguration()); + String brokerId = pulsar.getBrokerId(); + final String topicName = + getHeartbeatTopicName(brokerId, pulsar.getConfiguration(), (topicVersion == TopicVersion.V2)); + LOG.info("[{}] Running healthCheck with topic={}", clientAppId, topicName); final String messageStr = UUID.randomUUID().toString(); final String subscriptionName = "healthCheck-" + messageStr; // create non-partitioned topic manually and close the previous reader if present. - return pulsar().getBrokerService().getTopic(topicName, true) + return pulsar.getBrokerService().getTopic(topicName, true) .thenCompose(topicOptional -> { if (!topicOptional.isPresent()) { LOG.error("[{}] Fail to run health check while get topic {}. because get null value.", - clientAppId(), topicName); + clientAppId, topicName); throw new RestException(Status.NOT_FOUND, String.format("Topic [%s] not found after create.", topicName)); } PulsarClient client; try { - client = pulsar().getClient(); + client = pulsar.getClient(); } catch (PulsarServerException e) { - LOG.error("[{}] Fail to run health check while get client.", clientAppId()); + LOG.error("[{}] Fail to run health check while get client.", clientAppId); throw new RestException(e); } CompletableFuture resultFuture = new CompletableFuture<>(); @@ -426,14 +461,18 @@ private CompletableFuture internalRunHealthCheck(TopicVersion topicVersion .startMessageId(MessageId.latest) .createAsync().exceptionally(createException -> { producer.closeAsync().exceptionally(ex -> { - LOG.error("[{}] Close producer fail while heath check.", clientAppId()); + LOG.error("[{}] Close producer fail while heath check.", clientAppId); return null; }); throw FutureUtil.wrapToCompletionException(createException); }).thenCompose(reader -> producer.sendAsync(messageStr) - .thenCompose(__ -> healthCheckRecursiveReadNext(reader, messageStr)) + .thenCompose(__ -> FutureUtil.addTimeoutHandling( + healthCheckRecursiveReadNext(reader, messageStr), + HEALTH_CHECK_READ_TIMEOUT, pulsar.getBrokerService().executor(), + () -> HEALTH_CHECK_TIMEOUT_EXCEPTION)) .whenComplete((__, ex) -> { - closeAndReCheck(producer, reader, topicOptional.get(), subscriptionName) + closeAndReCheck(producer, reader, topicOptional.get(), subscriptionName, + clientAppId) .whenComplete((unused, innerEx) -> { if (ex != null) { resultFuture.completeExceptionally(ex); @@ -451,6 +490,11 @@ private CompletableFuture internalRunHealthCheck(TopicVersion topicVersion }); } + private CompletableFuture closeAndReCheck(Producer producer, Reader reader, + Topic topic, String subscriptionName) { + return closeAndReCheck(producer, reader, topic, subscriptionName, clientAppId()); + } + /** * Close producer and reader and then to re-check if this operation is success. * @@ -463,8 +507,8 @@ private CompletableFuture internalRunHealthCheck(TopicVersion topicVersion * @param topic Topic * @param subscriptionName Subscription name */ - private CompletableFuture closeAndReCheck(Producer producer, Reader reader, - Topic topic, String subscriptionName) { + private static CompletableFuture closeAndReCheck(Producer producer, Reader reader, + Topic topic, String subscriptionName, String clientAppId) { // no matter exception or success, we still need to // close producer/reader CompletableFuture producerFuture = producer.closeAsync(); @@ -475,7 +519,7 @@ private CompletableFuture closeAndReCheck(Producer producer, Reade return FutureUtil.waitForAll(Collections.unmodifiableList(futures)) .exceptionally(closeException -> { if (readerFuture.isCompletedExceptionally()) { - LOG.error("[{}] Close reader fail while heath check.", clientAppId()); + LOG.error("[{}] Close reader fail while heath check.", clientAppId); Subscription subscription = topic.getSubscription(subscriptionName); // re-check subscription after reader close @@ -483,24 +527,24 @@ private CompletableFuture closeAndReCheck(Producer producer, Reade LOG.warn("[{}] Force delete subscription {} " + "when it still exists after the" + " reader is closed.", - clientAppId(), subscription); + clientAppId, subscription); subscription.deleteForcefully() .exceptionally(ex -> { LOG.error("[{}] Force delete subscription fail" + " while health check", - clientAppId(), ex); + clientAppId, ex); return null; }); } } else { // producer future fail. - LOG.error("[{}] Close producer fail while heath check.", clientAppId()); + LOG.error("[{}] Close producer fail while heath check.", clientAppId); } return null; }); } - private CompletableFuture healthCheckRecursiveReadNext(Reader reader, String content) { + private static CompletableFuture healthCheckRecursiveReadNext(Reader reader, String content) { return reader.readNextAsync() .thenCompose(msg -> { if (!Objects.equals(content, msg.getValue())) { @@ -512,7 +556,7 @@ private CompletableFuture healthCheckRecursiveReadNext(Reader read private CompletableFuture internalDeleteDynamicConfigurationOnMetadataAsync(String configName) { if (!pulsar().getBrokerService().isDynamicConfiguration(configName)) { - throw new RestException(Status.PRECONDITION_FAILED, " Can't update non-dynamic configuration"); + throw new RestException(Status.PRECONDITION_FAILED, "Can't delete non-dynamic configuration"); } else { return dynamicConfigurationResources().setDynamicConfigurationAsync(old -> { if (old != null) { @@ -527,7 +571,7 @@ private CompletableFuture internalDeleteDynamicConfigurationOnMetadataAsyn @Path("/version") @ApiOperation(value = "Get version of current broker") @ApiResponses(value = { - @ApiResponse(code = 200, message = "Everything is OK"), + @ApiResponse(code = 200, message = "The Pulsar version", response = String.class), @ApiResponse(code = 500, message = "Internal server error")}) public String version() throws Exception { return PulsarVersion.getVersion(); @@ -545,16 +589,26 @@ public void shutDownBrokerGracefully( @ApiParam(name = "maxConcurrentUnloadPerSec", value = "if the value absent(value=0) means no concurrent limitation.") @QueryParam("maxConcurrentUnloadPerSec") int maxConcurrentUnloadPerSec, - @QueryParam("forcedTerminateTopic") @DefaultValue("true") boolean forcedTerminateTopic + @QueryParam("forcedTerminateTopic") @DefaultValue("true") boolean forcedTerminateTopic, + @Suspended final AsyncResponse asyncResponse ) { validateSuperUserAccess(); - doShutDownBrokerGracefully(maxConcurrentUnloadPerSec, forcedTerminateTopic); + doShutDownBrokerGracefullyAsync(maxConcurrentUnloadPerSec, forcedTerminateTopic) + .thenAccept(__ -> { + LOG.info("[{}] Successfully shutdown broker gracefully", clientAppId()); + asyncResponse.resume(Response.noContent().build()); + }) + .exceptionally(ex -> { + LOG.error("[{}] Failed to shutdown broker gracefully", clientAppId(), ex); + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); } - private void doShutDownBrokerGracefully(int maxConcurrentUnloadPerSec, - boolean forcedTerminateTopic) { + private CompletableFuture doShutDownBrokerGracefullyAsync(int maxConcurrentUnloadPerSec, + boolean forcedTerminateTopic) { pulsar().getBrokerService().unloadNamespaceBundlesGracefully(maxConcurrentUnloadPerSec, forcedTerminateTopic); - pulsar().closeAsync(); + return pulsar().closeAsync(); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ClustersBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ClustersBase.java index 5d4ed54c33466..b261033ca52c9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ClustersBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ClustersBase.java @@ -27,14 +27,16 @@ import io.swagger.annotations.ExampleProperty; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.stream.Stream; import javax.ws.rs.DELETE; import javax.ws.rs.GET; import javax.ws.rs.POST; @@ -60,10 +62,12 @@ import org.apache.pulsar.common.policies.data.BrokerNamespaceIsolationData; import org.apache.pulsar.common.policies.data.BrokerNamespaceIsolationDataImpl; import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.ClusterData.ClusterUrl; import org.apache.pulsar.common.policies.data.ClusterDataImpl; +import org.apache.pulsar.common.policies.data.ClusterPolicies.ClusterUrl; +import org.apache.pulsar.common.policies.data.ClusterPoliciesImpl; import org.apache.pulsar.common.policies.data.FailureDomainImpl; import org.apache.pulsar.common.policies.data.NamespaceIsolationDataImpl; +import org.apache.pulsar.common.policies.data.NamespaceIsolationPolicyUnloadScope; import org.apache.pulsar.common.policies.impl.NamespaceIsolationPolicies; import org.apache.pulsar.common.policies.impl.NamespaceIsolationPolicyImpl; import org.apache.pulsar.common.util.FutureUtil; @@ -132,7 +136,7 @@ public void getCluster(@Suspended AsyncResponse asyncResponse, notes = "This operation requires Pulsar superuser privileges, and the name cannot contain the '/' characters." ) @ApiResponses(value = { - @ApiResponse(code = 204, message = "Cluster has been created."), + @ApiResponse(code = 200, message = "Cluster has been created."), @ApiResponse(code = 400, message = "Bad request parameter."), @ApiResponse(code = 403, message = "You don't have admin permission to create the cluster."), @ApiResponse(code = 409, message = "Cluster already exists."), @@ -198,7 +202,7 @@ public void createCluster( value = "Update the configuration for a cluster.", notes = "This operation requires Pulsar superuser privileges.") @ApiResponses(value = { - @ApiResponse(code = 204, message = "Cluster has been updated."), + @ApiResponse(code = 200, message = "Cluster has been updated."), @ApiResponse(code = 400, message = "Bad request parameter."), @ApiResponse(code = 403, message = "Don't have admin permission or policies are read-only."), @ApiResponse(code = 404, message = "Cluster doesn't exist."), @@ -247,13 +251,51 @@ public void updateCluster( }); } + @GET + @Path("/{cluster}/migrate") + @ApiOperation( + value = "Get the cluster migration configuration for the specified cluster.", + response = ClusterDataImpl.class, + notes = "This operation requires Pulsar superuser privileges." + ) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Return the cluster data.", response = ClusterDataImpl.class), + @ApiResponse(code = 403, message = "Don't have admin permission."), + @ApiResponse(code = 404, message = "Cluster doesn't exist."), + @ApiResponse(code = 500, message = "Internal server error.") + }) + public void getClusterMigration( + @Suspended AsyncResponse asyncResponse, + @ApiParam( + value = "The cluster name", + required = true + ) + @PathParam("cluster") String cluster) { + validateSuperUserAccessAsync() + .thenCompose(__ -> clusterResources().getClusterPoliciesResources().getClusterPoliciesAsync(cluster)) + .thenAccept(policies -> { + asyncResponse.resume( + policies.orElseThrow(() -> new RestException(Status.NOT_FOUND, "Cluster does not exist"))); + }) + .exceptionally(ex -> { + log.error("[{}] Failed to get cluster {} migration", clientAppId(), cluster, ex); + Throwable realCause = FutureUtil.unwrapCompletionException(ex); + if (realCause instanceof MetadataStoreException.NotFoundException) { + asyncResponse.resume(new RestException(Status.NOT_FOUND, "Cluster does not exist")); + return null; + } + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); + } + @POST @Path("/{cluster}/migrate") @ApiOperation( value = "Update the configuration for a cluster migration.", notes = "This operation requires Pulsar superuser privileges.") @ApiResponses(value = { - @ApiResponse(code = 204, message = "Cluster has been updated."), + @ApiResponse(code = 200, message = "Cluster has been updated."), @ApiResponse(code = 400, message = "Cluster url must not be empty."), @ApiResponse(code = 403, message = "Don't have admin permission or policies are read-only."), @ApiResponse(code = 404, message = "Cluster doesn't exist."), @@ -286,8 +328,9 @@ public void updateClusterMigration( } validateSuperUserAccessAsync() .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) - .thenCompose(__ -> clusterResources().updateClusterAsync(cluster, old -> { - ClusterDataImpl data = (ClusterDataImpl) old; + .thenCompose(__ -> clusterResources().getClusterPoliciesResources().setPoliciesWithCreateAsync(cluster, + old -> { + ClusterPoliciesImpl data = old.orElse(new ClusterPoliciesImpl()); data.setMigrated(isMigrated); data.setMigratedClusterUrl(clusterUrl); return data; @@ -652,6 +695,7 @@ public void getBrokerWithNamespaceIsolationPolicy( notes = "This operation requires Pulsar superuser privileges." ) @ApiResponses(value = { + @ApiResponse(code = 204, message = "Set namespace isolation policy successfully."), @ApiResponse(code = 400, message = "Namespace isolation policy data is invalid."), @ApiResponse(code = 403, message = "Don't have admin permission or policies are read-only."), @ApiResponse(code = 404, message = "Namespace isolation policy doesn't exist."), @@ -680,10 +724,13 @@ public void setNamespaceIsolationPolicy( .setIsolationDataWithCreateAsync(cluster, (p) -> Collections.emptyMap()) .thenApply(__ -> new NamespaceIsolationPolicies())) ).thenCompose(nsIsolationPolicies -> { + NamespaceIsolationDataImpl oldPolicy = nsIsolationPolicies + .getPolicies().getOrDefault(policyName, null); nsIsolationPolicies.setPolicy(policyName, policyData); return namespaceIsolationPolicies() - .setIsolationDataAsync(cluster, old -> nsIsolationPolicies.getPolicies()); - }).thenCompose(__ -> filterAndUnloadMatchedNamespaceAsync(policyData)) + .setIsolationDataAsync(cluster, old -> nsIsolationPolicies.getPolicies()) + .thenApply(__ -> oldPolicy); + }).thenCompose(oldPolicy -> filterAndUnloadMatchedNamespaceAsync(cluster, policyData, oldPolicy)) .thenAccept(__ -> { log.info("[{}] Successful to update clusters/{}/namespaceIsolationPolicies/{}.", clientAppId(), cluster, policyName); @@ -717,42 +764,94 @@ public void setNamespaceIsolationPolicy( /** * Get matched namespaces; call unload for each namespaces. */ - private CompletableFuture filterAndUnloadMatchedNamespaceAsync(NamespaceIsolationDataImpl policyData) { + private CompletableFuture filterAndUnloadMatchedNamespaceAsync(String cluster, + NamespaceIsolationDataImpl policyData, + NamespaceIsolationDataImpl oldPolicy) { + // exit early if none of the namespaces need to be unloaded + if (NamespaceIsolationPolicyUnloadScope.none.equals(policyData.getUnloadScope())) { + return CompletableFuture.completedFuture(null); + } + PulsarAdmin adminClient; try { adminClient = pulsar().getAdminClient(); } catch (PulsarServerException e) { return FutureUtil.failedFuture(e); } - return adminClient.tenants().getTenantsAsync() - .thenCompose(tenants -> { - Stream>> completableFutureStream = tenants.stream() - .map(tenant -> adminClient.namespaces().getNamespacesAsync(tenant)); - return FutureUtil.waitForAll(completableFutureStream) - .thenApply(namespaces -> { - // if namespace match any policy regex, add it to ns list to be unload. - return namespaces.stream() - .filter(namespaceName -> - policyData.getNamespaces().stream().anyMatch(namespaceName::matches)) - .collect(Collectors.toList()); - }); - }).thenCompose(shouldUnloadNamespaces -> { - if (CollectionUtils.isEmpty(shouldUnloadNamespaces)) { - return CompletableFuture.completedFuture(null); - } - List> futures = shouldUnloadNamespaces.stream() - .map(namespaceName -> adminClient.namespaces().unloadAsync(namespaceName)) - .collect(Collectors.toList()); - return FutureUtil.waitForAll(futures) - .thenAccept(__ -> { - try { - // write load info to load manager to make the load happens fast - pulsar().getLoadManager().get().writeLoadReportOnZookeeper(true); - } catch (Exception e) { - log.warn("[{}] Failed to writeLoadReportOnZookeeper.", clientAppId(), e); - } - }); - }); + Set combinedNamespaces = new HashSet<>(policyData.getNamespaces()); + final List oldNamespaces = new ArrayList<>(); + if (oldPolicy != null) { + oldNamespaces.addAll(oldPolicy.getNamespaces()); + combinedNamespaces.addAll(oldNamespaces); + } + return adminClient.tenants().getTenantsAsync().thenCompose(tenants -> { + List>> filteredNamespacesForEachTenant = tenants.stream() + .map(tenant -> adminClient.namespaces().getNamespacesAsync(tenant).thenCompose(namespaces -> { + List> namespaceNamesInCluster = namespaces.stream() + .map(namespaceName -> adminClient.namespaces().getPoliciesAsync(namespaceName) + .thenApply(policies -> policies.replication_clusters.contains(cluster) + ? namespaceName : null)) + .collect(Collectors.toList()); + return FutureUtil.waitForAll(namespaceNamesInCluster).thenApply( + __ -> namespaceNamesInCluster.stream() + .map(CompletableFuture::join) + .filter(Objects::nonNull) + .collect(Collectors.toList())); + })).toList(); + return FutureUtil.waitForAll(filteredNamespacesForEachTenant) + .thenApply(__ -> filteredNamespacesForEachTenant.stream() + .map(CompletableFuture::join) + .flatMap(List::stream) + .collect(Collectors.toList())); + }).thenCompose(clusterLocalNamespaces -> { + if (CollectionUtils.isEmpty(clusterLocalNamespaces)) { + return CompletableFuture.completedFuture(null); + } + // If unload type is 'changed', we need to figure out a further subset of namespaces whose placement might + // actually have been changed. + + log.debug("Old policy: {} ; new policy: {}", oldPolicy, policyData); + + boolean unloadAllNamespaces = false; + // We also compare that the previous primary broker list is same as current, in case all namespaces need + // to be placed again anyway. + if (NamespaceIsolationPolicyUnloadScope.all_matching.equals(policyData.getUnloadScope()) + || (oldPolicy != null + && !CollectionUtils.isEqualCollection(oldPolicy.getPrimary(), policyData.getPrimary()))) { + unloadAllNamespaces = true; + } + // list is same, so we continue finding the changed namespaces. + + // We create a intersection of the old and new regexes. These won't need to be unloaded. + Set commonNamespaces = new HashSet<>(policyData.getNamespaces()); + commonNamespaces.retainAll(oldNamespaces); + + log.debug("combined regexes: {}; common regexes:{}", combinedNamespaces, commonNamespaces); + + if (!unloadAllNamespaces) { + // Find the changed regexes ((new U old) - (new ∩ old)). + combinedNamespaces.removeAll(commonNamespaces); + log.debug("changed regexes: {}", commonNamespaces); + } + + // Now we further filter the filtered namespaces based on this combinedNamespaces set + List namespacePatterns = combinedNamespaces.stream().map(Pattern::compile).toList(); + clusterLocalNamespaces = clusterLocalNamespaces.stream() + .filter(name -> namespacePatterns.stream().anyMatch(pattern -> pattern.matcher(name).matches())) + .toList(); + + List> futures = clusterLocalNamespaces.stream() + .map(namespaceName -> adminClient.namespaces().unloadAsync(namespaceName)) + .collect(Collectors.toList()); + return FutureUtil.waitForAll(futures).thenAccept(__ -> { + try { + // write load info to load manager to make the load happens fast + pulsar().getLoadManager().get().writeLoadReportOnZookeeper(true); + } catch (Exception e) { + log.warn("[{}] Failed to writeLoadReportOnZookeeper.", clientAppId(), e); + } + }); + }); } @DELETE @@ -762,6 +861,7 @@ private CompletableFuture filterAndUnloadMatchedNamespaceAsync(NamespaceIs notes = "This operation requires Pulsar superuser privileges." ) @ApiResponses(value = { + @ApiResponse(code = 204, message = "Delete namespace isolation policy successfully."), @ApiResponse(code = 403, message = "Don't have admin permission or policies are read only."), @ApiResponse(code = 404, message = "Namespace isolation policy doesn't exist."), @ApiResponse(code = 412, message = "Cluster doesn't exist."), @@ -809,6 +909,7 @@ public void deleteNamespaceIsolationPolicy( notes = "This operation requires Pulsar superuser privileges." ) @ApiResponses(value = { + @ApiResponse(code = 204, message = "Set the failure domain of the cluster successfully."), @ApiResponse(code = 403, message = "Don't have admin permission."), @ApiResponse(code = 404, message = "Failure domain doesn't exist."), @ApiResponse(code = 409, message = "Broker already exists in another domain."), @@ -944,6 +1045,7 @@ public void getDomain( notes = "This operation requires Pulsar superuser privileges." ) @ApiResponses(value = { + @ApiResponse(code = 200, message = "Delete the failure domain of the cluster successfully"), @ApiResponse(code = 403, message = "Don't have admin permission or policy is read only"), @ApiResponse(code = 404, message = "FailureDomain doesn't exist"), @ApiResponse(code = 412, message = "Cluster doesn't exist"), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java index 4350316e2f011..42971ae231c05 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java @@ -39,7 +39,6 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.StreamingOutput; import org.apache.pulsar.broker.admin.AdminResource; -import org.apache.pulsar.client.api.Message; import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.functions.FunctionDefinition; import org.apache.pulsar.common.functions.FunctionState; @@ -486,7 +485,7 @@ public List listFunctions( @POST @ApiOperation( value = "Triggers a Pulsar Function with a user-specified value or file data", - response = Message.class + response = String.class ) @ApiResponses(value = { @ApiResponse(code = 400, message = "Invalid request"), @@ -541,6 +540,7 @@ public FunctionState getFunctionState( value = "Put the state associated with a Pulsar Function" ) @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 400, message = "Invalid request"), @ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), @ApiResponse(code = 404, message = "The Pulsar Function does not exist"), @@ -557,8 +557,9 @@ public void putFunctionState(final @PathParam("tenant") String tenant, } @POST - @ApiOperation(value = "Restart an instance of a Pulsar Function", response = Void.class) + @ApiOperation(value = "Restart an instance of a Pulsar Function") @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this function"), @ApiResponse(code = 400, message = "Invalid request"), @ApiResponse(code = 404, message = "The Pulsar Function does not exist"), @@ -578,8 +579,9 @@ public void restartFunction( } @POST - @ApiOperation(value = "Restart all instances of a Pulsar Function", response = Void.class) + @ApiOperation(value = "Restart all instances of a Pulsar Function") @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 400, message = "Invalid request"), @ApiResponse(code = 404, message = "The Pulsar Function does not exist"), @ApiResponse(code = 500, message = "Internal server error") @@ -597,8 +599,9 @@ public void restartFunction( } @POST - @ApiOperation(value = "Stop an instance of a Pulsar Function", response = Void.class) + @ApiOperation(value = "Stop an instance of a Pulsar Function") @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 400, message = "Invalid request"), @ApiResponse(code = 404, message = "The Pulsar Function does not exist"), @ApiResponse(code = 500, message = "Internal server error") @@ -617,8 +620,9 @@ public void stopFunction( } @POST - @ApiOperation(value = "Stop all instances of a Pulsar Function", response = Void.class) + @ApiOperation(value = "Stop all instances of a Pulsar Function") @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 400, message = "Invalid request"), @ApiResponse(code = 404, message = "The Pulsar Function does not exist"), @ApiResponse(code = 500, message = "Internal server error") @@ -636,8 +640,9 @@ public void stopFunction( } @POST - @ApiOperation(value = "Start an instance of a Pulsar Function", response = Void.class) + @ApiOperation(value = "Start an instance of a Pulsar Function") @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 400, message = "Invalid request"), @ApiResponse(code = 404, message = "The Pulsar Function does not exist"), @ApiResponse(code = 500, message = "Internal server error") @@ -656,8 +661,9 @@ public void startFunction( } @POST - @ApiOperation(value = "Start all instances of a Pulsar Function", response = Void.class) + @ApiOperation(value = "Start all instances of a Pulsar Function") @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 400, message = "Invalid request"), @ApiResponse(code = 404, message = "The Pulsar Function does not exist"), @ApiResponse(code = 500, message = "Internal server error") @@ -718,7 +724,8 @@ public StreamingOutput downloadFunction( @GET @ApiOperation( value = "Fetches a list of supported Pulsar IO connectors currently running in cluster mode", - response = List.class + response = ConnectorDefinition.class, + responseContainer = "List" ) @ApiResponses(value = { @ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), @@ -739,6 +746,7 @@ public List getConnectorsList() throws IOException { value = "Reload the built-in Functions" ) @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 401, message = "This operation requires super-user access"), @ApiResponse(code = 503, message = "Function worker service is now initializing. Please try again later."), @ApiResponse(code = 500, message = "Internal server error") @@ -768,6 +776,7 @@ public List getBuiltinFunction() { @PUT @ApiOperation(value = "Updates a Pulsar Function on the worker leader", hidden = true) @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 403, message = "The requester doesn't have super-user permissions"), @ApiResponse(code = 404, message = "The function does not exist"), @ApiResponse(code = 400, message = "Invalid request"), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 97029eb5ce128..4d26fe2a4c35b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -56,7 +56,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.admin.AdminResource; -import org.apache.pulsar.broker.authorization.AuthorizationService; import org.apache.pulsar.broker.loadbalance.LeaderBroker; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.service.BrokerServiceException; @@ -234,14 +233,14 @@ private void internalRetryableDeleteNamespaceAsync0(boolean force, int retryTime })) .thenCompose(topics -> { List allTopics = topics.get(0); - ArrayList allUserCreatedTopics = new ArrayList<>(); + Set allUserCreatedTopics = new HashSet<>(); List allPartitionedTopics = topics.get(1); - ArrayList allUserCreatedPartitionTopics = new ArrayList<>(); + Set allUserCreatedPartitionTopics = new HashSet<>(); boolean hasNonSystemTopic = false; - List allSystemTopics = new ArrayList<>(); - List allPartitionedSystemTopics = new ArrayList<>(); - List topicPolicy = new ArrayList<>(); - List partitionedTopicPolicy = new ArrayList<>(); + Set allSystemTopics = new HashSet<>(); + Set allPartitionedSystemTopics = new HashSet<>(); + Set topicPolicy = new HashSet<>(); + Set partitionedTopicPolicy = new HashSet<>(); for (String topic : allTopics) { if (!pulsar().getBrokerService().isSystemTopic(TopicName.get(topic))) { hasNonSystemTopic = true; @@ -310,8 +309,14 @@ private void internalRetryableDeleteNamespaceAsync0(boolean force, int retryTime clientAppId(), ex); return FutureUtil.failedFuture(ex); } + log.info("[{}] Deleting namespace bundle {}/{}", clientAppId(), + namespaceName, bundle.getBundleRange()); return admin.namespaces().deleteNamespaceBundleAsync(namespaceName.toString(), bundle.getBundleRange(), force); + } else { + log.warn("[{}] Skipping deleting namespace bundle {}/{} " + + "as it's not owned by any broker", + clientAppId(), namespaceName, bundle.getBundleRange()); } return CompletableFuture.completedFuture(null); }) @@ -322,8 +327,11 @@ private void internalRetryableDeleteNamespaceAsync0(boolean force, int retryTime final Throwable rc = FutureUtil.unwrapCompletionException(error); if (rc instanceof MetadataStoreException) { if (rc.getCause() != null && rc.getCause() instanceof KeeperException.NotEmptyException) { + KeeperException.NotEmptyException ne = + (KeeperException.NotEmptyException) rc.getCause(); log.info("[{}] There are in-flight topics created during the namespace deletion, " - + "retry to delete the namespace again.", namespaceName); + + "retry to delete the namespace again. (path {} is not empty on metadata)", + namespaceName, ne.getPath()); final int next = retryTimes - 1; if (next > 0) { // async recursive @@ -331,7 +339,8 @@ private void internalRetryableDeleteNamespaceAsync0(boolean force, int retryTime } else { callback.completeExceptionally( new RestException(Status.CONFLICT, "The broker still have in-flight topics" - + " created during namespace deletion, please try again.")); + + " created during namespace deletion (path " + ne.getPath() + ") " + + "is not empty on metadata store, please try again.")); // drop out recursive } return; @@ -349,7 +358,7 @@ private boolean isDeletedAlongWithUserCreatedTopic(String topic) { return topic.endsWith(SystemTopicNames.PENDING_ACK_STORE_SUFFIX); } - private CompletableFuture internalDeletePartitionedTopicsAsync(List topicNames) { + private CompletableFuture internalDeletePartitionedTopicsAsync(Set topicNames) { if (CollectionUtils.isEmpty(topicNames)) { return CompletableFuture.completedFuture(null); } @@ -363,7 +372,7 @@ private CompletableFuture internalDeletePartitionedTopicsAsync(List internalDeleteTopicsAsync(List topicNames) { + private CompletableFuture internalDeleteTopicsAsync(Set topicNames) { if (CollectionUtils.isEmpty(topicNames)) { return CompletableFuture.completedFuture(null); } @@ -469,13 +478,16 @@ protected CompletableFuture internalClearZkSources() { // clear z-node of local policies .thenCompose(ignore -> getLocalPolicies().deleteLocalPoliciesAsync(namespaceName)) // clear /loadbalance/bundle-data - .thenCompose(ignore -> namespaceResources().deleteBundleDataAsync(namespaceName)); + .thenCompose(ignore -> + loadBalanceResources().getBundleDataResources().deleteBundleDataAsync(namespaceName)); } @SuppressWarnings("deprecation") protected CompletableFuture internalDeleteNamespaceBundleAsync(String bundleRange, boolean authoritative, boolean force) { + log.info("[{}] Deleting namespace bundle {}/{} authoritative:{} force:{}", + clientAppId(), namespaceName, bundleRange, authoritative, force); return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.DELETE_BUNDLE) .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) .thenCompose(__ -> { @@ -570,109 +582,86 @@ protected CompletableFuture internalDeleteNamespaceBundleAsync(String bund } protected CompletableFuture internalGrantPermissionOnNamespaceAsync(String role, Set actions) { - AuthorizationService authService = pulsar().getBrokerService().getAuthorizationService(); - if (null != authService) { - return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.GRANT_PERMISSION) - .thenAccept(__ -> { - checkNotNull(role, "Role should not be null"); - checkNotNull(actions, "Actions should not be null"); - }).thenCompose(__ -> - authService.grantPermissionAsync(namespaceName, actions, role, null)) - .thenAccept(unused -> - log.info("[{}] Successfully granted access for role {}: {} - namespaceName {}", - clientAppId(), role, actions, namespaceName)) - .exceptionally(ex -> { - Throwable realCause = FutureUtil.unwrapCompletionException(ex); - //The IllegalArgumentException and the IllegalStateException were historically thrown by the - // grantPermissionAsync method, so we catch them here to ensure backwards compatibility. - if (realCause instanceof MetadataStoreException.NotFoundException - || realCause instanceof IllegalArgumentException) { - log.warn("[{}] Failed to set permissions for namespace {}: does not exist", clientAppId(), - namespaceName, ex); - throw new RestException(Status.NOT_FOUND, "Topic's namespace does not exist"); - } else if (realCause instanceof MetadataStoreException.BadVersionException - || realCause instanceof IllegalStateException) { - log.warn("[{}] Failed to set permissions for namespace {}: {}", - clientAppId(), namespaceName, ex.getCause().getMessage(), ex); - throw new RestException(Status.CONFLICT, "Concurrent modification"); - } else { - log.error("[{}] Failed to get permissions for namespace {}", - clientAppId(), namespaceName, ex); - throw new RestException(realCause); - } - }); - } else { - String msg = "Authorization is not enabled"; - return FutureUtil.failedFuture(new RestException(Status.NOT_IMPLEMENTED, msg)); - } + return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.GRANT_PERMISSION) + .thenAccept(__ -> { + checkNotNull(role, "Role should not be null"); + checkNotNull(actions, "Actions should not be null"); + }).thenCompose(__ -> + getAuthorizationService().grantPermissionAsync(namespaceName, actions, role, null)) + .thenAccept(unused -> + log.info("[{}] Successfully granted access for role {}: {} - namespaceName {}", + clientAppId(), role, actions, namespaceName)) + .exceptionally(ex -> { + Throwable realCause = FutureUtil.unwrapCompletionException(ex); + //The IllegalArgumentException and the IllegalStateException were historically thrown by the + // grantPermissionAsync method, so we catch them here to ensure backwards compatibility. + if (realCause instanceof MetadataStoreException.NotFoundException + || realCause instanceof IllegalArgumentException) { + log.warn("[{}] Failed to set permissions for namespace {}: does not exist", clientAppId(), + namespaceName, ex); + throw new RestException(Status.NOT_FOUND, "Topic's namespace does not exist"); + } else if (realCause instanceof MetadataStoreException.BadVersionException + || realCause instanceof IllegalStateException) { + log.warn("[{}] Failed to set permissions for namespace {}: {}", + clientAppId(), namespaceName, ex.getCause().getMessage(), ex); + throw new RestException(Status.CONFLICT, "Concurrent modification"); + } else { + log.error("[{}] Failed to get permissions for namespace {}", + clientAppId(), namespaceName, ex); + throw new RestException(realCause); + } + }); } protected CompletableFuture internalGrantPermissionOnSubscriptionAsync(String subscription, Set roles) { - AuthorizationService authService = pulsar().getBrokerService().getAuthorizationService(); - if (null != authService) { - return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.GRANT_PERMISSION) - .thenAccept(__ -> { - checkNotNull(subscription, "Subscription should not be null"); - checkNotNull(roles, "Roles should not be null"); - }) - .thenCompose(__ -> authService.grantSubscriptionPermissionAsync(namespaceName, subscription, - roles, null)) - .thenAccept(unused -> { - log.info("[{}] Successfully granted permission on subscription for role {}:{} - " - + "namespaceName {}", clientAppId(), roles, subscription, namespaceName); - }) - .exceptionally(ex -> { - Throwable realCause = FutureUtil.unwrapCompletionException(ex); - //The IllegalArgumentException and the IllegalStateException were historically thrown by the - // grantPermissionAsync method, so we catch them here to ensure backwards compatibility. - if (realCause.getCause() instanceof IllegalArgumentException) { - log.warn("[{}] Failed to set permissions for namespace {}: does not exist", clientAppId(), - namespaceName); - throw new RestException(Status.NOT_FOUND, "Namespace does not exist"); - } else if (realCause.getCause() instanceof IllegalStateException) { - log.warn("[{}] Failed to set permissions for namespace {}: concurrent modification", - clientAppId(), namespaceName); - throw new RestException(Status.CONFLICT, "Concurrent modification"); - } else { - log.error("[{}] Failed to get permissions for namespace {}", - clientAppId(), namespaceName, realCause); - throw new RestException(realCause); - } - }); - } else { - String msg = "Authorization is not enabled"; - return FutureUtil.failedFuture(new RestException(Status.NOT_IMPLEMENTED, msg)); - } + return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.GRANT_PERMISSION) + .thenAccept(__ -> { + checkNotNull(subscription, "Subscription should not be null"); + checkNotNull(roles, "Roles should not be null"); + }) + .thenCompose(__ -> getAuthorizationService() + .grantSubscriptionPermissionAsync(namespaceName, subscription, roles, null)) + .thenAccept(unused -> { + log.info("[{}] Successfully granted permission on subscription for role {}:{} - " + + "namespaceName {}", clientAppId(), roles, subscription, namespaceName); + }) + .exceptionally(ex -> { + Throwable realCause = FutureUtil.unwrapCompletionException(ex); + //The IllegalArgumentException and the IllegalStateException were historically thrown by the + // grantPermissionAsync method, so we catch them here to ensure backwards compatibility. + if (realCause.getCause() instanceof IllegalArgumentException) { + log.warn("[{}] Failed to set permissions for namespace {}: does not exist", clientAppId(), + namespaceName); + throw new RestException(Status.NOT_FOUND, "Namespace does not exist"); + } else if (realCause.getCause() instanceof IllegalStateException) { + log.warn("[{}] Failed to set permissions for namespace {}: concurrent modification", + clientAppId(), namespaceName); + throw new RestException(Status.CONFLICT, "Concurrent modification"); + } else { + log.error("[{}] Failed to get permissions for namespace {}", + clientAppId(), namespaceName, realCause); + throw new RestException(realCause); + } + }); } protected CompletableFuture internalRevokePermissionsOnNamespaceAsync(String role) { return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.REVOKE_PERMISSION) .thenAccept(__ -> checkNotNull(role, "Role should not be null")) - .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) - .thenCompose(__ -> updatePoliciesAsync(namespaceName, policies -> { - policies.auth_policies.getNamespaceAuthentication().remove(role); - return policies; - })); + .thenCompose(__ -> getAuthorizationService().revokePermissionAsync(namespaceName, role)); } protected CompletableFuture internalRevokePermissionsOnSubscriptionAsync(String subscriptionName, String role) { - AuthorizationService authService = pulsar().getBrokerService().getAuthorizationService(); - if (null != authService) { - return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.REVOKE_PERMISSION) - .thenAccept(__ -> { - checkNotNull(subscriptionName, "SubscriptionName should not be null"); - checkNotNull(role, "Role should not be null"); - }) - .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) - .thenCompose(__ -> authService.revokeSubscriptionPermissionAsync(namespaceName, - subscriptionName, role, null/* additional auth-data json */)); - } else { - String msg = "Authorization is not enabled"; - return FutureUtil.failedFuture(new RestException(Status.NOT_IMPLEMENTED, msg)); - } + return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.REVOKE_PERMISSION) + .thenAccept(__ -> { + checkNotNull(subscriptionName, "SubscriptionName should not be null"); + checkNotNull(role, "Role should not be null"); + }) + .thenCompose(__ -> getAuthorizationService().revokeSubscriptionPermissionAsync(namespaceName, + subscriptionName, role, null/* additional auth-data json */)); } protected CompletableFuture> internalGetNamespaceReplicationClustersAsync() { @@ -691,7 +680,9 @@ protected CompletableFuture internalSetNamespaceReplicationClusters(List validatePoliciesReadOnlyAccessAsync()) .thenApply(__ -> { - checkNotNull(clusterIds, "ClusterIds should not be null"); + if (CollectionUtils.isEmpty(clusterIds)) { + throw new RestException(Status.PRECONDITION_FAILED, "ClusterIds should not be null or empty"); + } if (!namespaceName.isGlobal() && !(clusterIds.size() == 1 && clusterIds.get(0).equals(pulsar().getConfiguration().getClusterName()))) { throw new RestException(Status.PRECONDITION_FAILED, @@ -712,9 +703,21 @@ protected CompletableFuture internalSetNamespaceReplicationClusters(List - validateClusterForTenantAsync( - namespaceName.getTenant(), clusterId)); + .thenCompose(__ -> getNamespacePoliciesAsync(this.namespaceName) + .thenCompose(nsPolicies -> { + if (nsPolicies.allowed_clusters.isEmpty()) { + return validateClusterForTenantAsync( + namespaceName.getTenant(), clusterId); + } + if (!nsPolicies.allowed_clusters.contains(clusterId)) { + String msg = String.format("Cluster [%s] is not in the " + + "list of allowed clusters list for namespace " + + "[%s]", clusterId, namespaceName.toString()); + log.info(msg); + throw new RestException(Status.FORBIDDEN, msg); + } + return CompletableFuture.completedFuture(null); + })); }).collect(Collectors.toList()); return FutureUtil.waitForAll(futures).thenApply(__ -> replicationClusterSet); })) @@ -944,9 +947,9 @@ private CompletableFuture validateLeaderBrokerAsync() { return FutureUtil.failedFuture(new RestException(Response.Status.PRECONDITION_FAILED, errorStr)); } LeaderBroker leaderBroker = pulsar().getLeaderElectionService().getCurrentLeader().get(); - String leaderBrokerUrl = leaderBroker.getServiceUrl(); + String leaderBrokerId = leaderBroker.getBrokerId(); return pulsar().getNamespaceService() - .createLookupResult(leaderBrokerUrl, false, null) + .createLookupResult(leaderBrokerId, false, null) .thenCompose(lookupResult -> { String redirectUrl = isRequestHttps() ? lookupResult.getLookupData().getHttpUrlTls() : lookupResult.getLookupData().getHttpUrl(); @@ -969,7 +972,7 @@ private CompletableFuture validateLeaderBrokerAsync() { return FutureUtil.failedFuture(( new WebApplicationException(Response.temporaryRedirect(redirect).build()))); } catch (MalformedURLException exception) { - log.error("The leader broker url is malformed - {}", leaderBrokerUrl); + log.error("The redirect url is malformed - {}", redirectUrl); return FutureUtil.failedFuture(new RestException(exception)); } }); @@ -990,13 +993,13 @@ public CompletableFuture setNamespaceBundleAffinityAsync(String bundleRang return CompletableFuture.completedFuture(null); }) .thenCompose(__ -> { - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config())) { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar())) { return CompletableFuture.completedFuture(null); } return validateLeaderBrokerAsync(); }) .thenAccept(__ -> { - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config())) { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar())) { return; } // For ExtensibleLoadManager, this operation will be ignored. @@ -1005,8 +1008,11 @@ public CompletableFuture setNamespaceBundleAffinityAsync(String bundleRang } public CompletableFuture internalUnloadNamespaceBundleAsync(String bundleRange, - String destinationBroker, + String destinationBrokerParam, boolean authoritative) { + String destinationBroker = StringUtils.isBlank(destinationBrokerParam) ? null : + // ensure backward compatibility: strip the possible http:// or https:// prefix + destinationBrokerParam.replaceFirst("http[s]?://", ""); return validateSuperUserAccessAsync() .thenCompose(__ -> setNamespaceBundleAffinityAsync(bundleRange, destinationBroker)) .thenAccept(__ -> { @@ -1199,7 +1205,8 @@ protected void internalSetPublishRate(PublishRate maxPublishMessageRate) { protected CompletableFuture internalSetPublishRateAsync(PublishRate maxPublishMessageRate) { log.info("[{}] Set namespace publish-rate {}/{}", clientAppId(), namespaceName, maxPublishMessageRate); - return validateSuperUserAccessAsync().thenCompose(__ -> updatePoliciesAsync(namespaceName, policies -> { + return validateNamespacePolicyOperationAsync(namespaceName, PolicyName.RATE, PolicyOperation.WRITE) + .thenCompose(__ -> updatePoliciesAsync(namespaceName, policies -> { policies.publishMaxMessageRate.put(pulsar().getConfiguration().getClusterName(), maxPublishMessageRate); log.info("[{}] Successfully updated the publish_max_message_rate for cluster on namespace {}", clientAppId(), namespaceName); @@ -1228,7 +1235,8 @@ protected void internalRemovePublishRate() { protected CompletableFuture internalRemovePublishRateAsync() { log.info("[{}] Remove namespace publish-rate {}/{}", clientAppId(), namespaceName, topicName); - return validateSuperUserAccessAsync().thenCompose(__ -> updatePoliciesAsync(namespaceName, policies -> { + return validateNamespacePolicyOperationAsync(namespaceName, PolicyName.RATE, PolicyOperation.WRITE) + .thenCompose(__ -> updatePoliciesAsync(namespaceName, policies -> { if (policies.publishMaxMessageRate != null) { policies.publishMaxMessageRate.remove(pulsar().getConfiguration().getClusterName()); } @@ -1248,7 +1256,8 @@ protected CompletableFuture internalGetPublishRateAsync() { @SuppressWarnings("deprecation") protected CompletableFuture internalSetTopicDispatchRateAsync(DispatchRateImpl dispatchRate) { log.info("[{}] Set namespace dispatch-rate {}/{}", clientAppId(), namespaceName, dispatchRate); - return validateSuperUserAccessAsync().thenCompose(__ -> updatePoliciesAsync(namespaceName, policies -> { + return validateNamespacePolicyOperationAsync(namespaceName, PolicyName.RATE, PolicyOperation.WRITE) + .thenCompose(__ -> updatePoliciesAsync(namespaceName, policies -> { policies.topicDispatchRate.put(pulsar().getConfiguration().getClusterName(), dispatchRate); policies.clusterDispatchRate.put(pulsar().getConfiguration().getClusterName(), dispatchRate); log.info("[{}] Successfully updated the dispatchRate for cluster on namespace {}", clientAppId(), @@ -1258,7 +1267,8 @@ protected CompletableFuture internalSetTopicDispatchRateAsync(DispatchRate } protected CompletableFuture internalDeleteTopicDispatchRateAsync() { - return validateSuperUserAccessAsync().thenCompose(__ -> updatePoliciesAsync(namespaceName, policies -> { + return validateNamespacePolicyOperationAsync(namespaceName, PolicyName.RATE, PolicyOperation.WRITE) + .thenCompose(__ -> updatePoliciesAsync(namespaceName, policies -> { policies.topicDispatchRate.remove(pulsar().getConfiguration().getClusterName()); policies.clusterDispatchRate.remove(pulsar().getConfiguration().getClusterName()); log.info("[{}] Successfully delete the dispatchRate for cluster on namespace {}", clientAppId(), @@ -1275,7 +1285,7 @@ protected CompletableFuture internalGetTopicDispatchRateAsync() { } protected CompletableFuture internalSetSubscriptionDispatchRateAsync(DispatchRateImpl dispatchRate) { - return validateSuperUserAccessAsync() + return validateNamespacePolicyOperationAsync(namespaceName, PolicyName.RATE, PolicyOperation.WRITE) .thenCompose(__ -> updatePoliciesAsync(namespaceName, policies -> { policies.subscriptionDispatchRate.put(pulsar().getConfiguration().getClusterName(), dispatchRate); log.info("[{}] Successfully updated the subscriptionDispatchRate for cluster on namespace {}", @@ -1285,7 +1295,7 @@ protected CompletableFuture internalSetSubscriptionDispatchRateAsync(Dispa } protected CompletableFuture internalDeleteSubscriptionDispatchRateAsync() { - return validateSuperUserAccessAsync() + return validateNamespacePolicyOperationAsync(namespaceName, PolicyName.RATE, PolicyOperation.WRITE) .thenCompose(__ -> updatePoliciesAsync(namespaceName, policies -> { policies.subscriptionDispatchRate.remove(pulsar().getConfiguration().getClusterName()); log.info("[{}] Successfully delete the subscriptionDispatchRate for cluster on namespace {}", @@ -1303,7 +1313,8 @@ protected CompletableFuture internalGetSubscriptionDispatchRateAsy protected CompletableFuture internalSetSubscribeRateAsync(SubscribeRate subscribeRate) { log.info("[{}] Set namespace subscribe-rate {}/{}", clientAppId(), namespaceName, subscribeRate); - return validateSuperUserAccessAsync().thenCompose(__ -> updatePoliciesAsync(namespaceName, policies -> { + return validateNamespacePolicyOperationAsync(namespaceName, PolicyName.RATE, PolicyOperation.WRITE) + .thenCompose(__ -> updatePoliciesAsync(namespaceName, policies -> { policies.clusterSubscribeRate.put(pulsar().getConfiguration().getClusterName(), subscribeRate); log.info("[{}] Successfully updated the subscribeRate for cluster on namespace {}", clientAppId(), namespaceName); @@ -1312,7 +1323,8 @@ protected CompletableFuture internalSetSubscribeRateAsync(SubscribeRate su } protected CompletableFuture internalDeleteSubscribeRateAsync() { - return validateSuperUserAccessAsync().thenCompose(__ -> updatePoliciesAsync(namespaceName, policies -> { + return validateNamespacePolicyOperationAsync(namespaceName, PolicyName.RATE, PolicyOperation.WRITE) + .thenCompose(__ -> updatePoliciesAsync(namespaceName, policies -> { policies.clusterSubscribeRate.remove(pulsar().getConfiguration().getClusterName()); log.info("[{}] Successfully delete the subscribeRate for cluster on namespace {}", clientAppId(), namespaceName); @@ -1347,6 +1359,7 @@ protected CompletableFuture setBacklogQuotaAsync(BacklogQuotaType backlogQ "Backlog Quota exceeds configured retention quota for namespace." + " Please increase retention quota and retry"); } + policies.backlog_quota_map.put(quotaType, quota); return policies; }); } @@ -1644,7 +1657,7 @@ protected Boolean internalGetEncryptionRequired() { } protected void internalSetInactiveTopic(InactiveTopicPolicies inactiveTopicPolicies) { - validateSuperUserAccess(); + validateNamespacePolicyOperation(namespaceName, PolicyName.INACTIVE_TOPIC, PolicyOperation.WRITE); validatePoliciesReadOnlyAccess(); internalSetPolicies("inactive_topic_policies", inactiveTopicPolicies); } @@ -2030,7 +2043,7 @@ protected void internalSetMaxUnackedMessagesPerConsumer(Integer maxUnackedMessag } protected void internalSetMaxSubscriptionsPerTopic(Integer maxSubscriptionsPerTopic){ - validateSuperUserAccess(); + validateNamespacePolicyOperation(namespaceName, PolicyName.MAX_SUBSCRIPTIONS, PolicyOperation.WRITE); validatePoliciesReadOnlyAccess(); if (maxSubscriptionsPerTopic != null && maxSubscriptionsPerTopic < 0) { throw new RestException(Status.PRECONDITION_FAILED, @@ -2136,9 +2149,10 @@ protected CompletableFuture internalSetOffloadThresholdInSecondsAsync(long f.complete(null); }) .exceptionally(t -> { + Throwable cause = FutureUtil.unwrapCompletionException(t); log.error("[{}] Failed to update offloadThresholdInSeconds configuration for namespace {}", clientAppId(), namespaceName, t); - f.completeExceptionally(new RestException(t)); + f.completeExceptionally(new RestException(cause)); return null; }); @@ -2333,28 +2347,6 @@ protected void internalRemoveOffloadPolicies(AsyncResponse asyncResponse) { } } - private void validateOffloadPolicies(OffloadPoliciesImpl offloadPolicies) { - if (offloadPolicies == null) { - log.warn("[{}] Failed to update offload configuration for namespace {}: offloadPolicies is null", - clientAppId(), namespaceName); - throw new RestException(Status.PRECONDITION_FAILED, - "The offloadPolicies must be specified for namespace offload."); - } - if (!offloadPolicies.driverSupported()) { - log.warn("[{}] Failed to update offload configuration for namespace {}: " - + "driver is not supported, support value: {}", - clientAppId(), namespaceName, OffloadPoliciesImpl.getSupportedDriverNames()); - throw new RestException(Status.PRECONDITION_FAILED, - "The driver is not supported, support value: " + OffloadPoliciesImpl.getSupportedDriverNames()); - } - if (!offloadPolicies.bucketValid()) { - log.warn("[{}] Failed to update offload configuration for namespace {}: bucket must be specified", - clientAppId(), namespaceName); - throw new RestException(Status.PRECONDITION_FAILED, - "The bucket must be specified for namespace offload."); - } - } - protected void internalRemoveMaxTopicsPerNamespace() { validateNamespacePolicyOperation(namespaceName, PolicyName.MAX_TOPICS, PolicyOperation.WRITE); internalSetMaxTopicsPerNamespace(null); @@ -2372,102 +2364,110 @@ protected void internalSetMaxTopicsPerNamespace(Integer maxTopicsPerNamespace) { } protected void internalSetProperty(String key, String value, AsyncResponse asyncResponse) { - validatePoliciesReadOnlyAccess(); - updatePoliciesAsync(namespaceName, policies -> { - policies.properties.put(key, value); - return policies; - }).thenAccept(v -> { - log.info("[{}] Successfully set property for key {} on namespace {}", clientAppId(), key, - namespaceName); - asyncResponse.resume(Response.noContent().build()); - }).exceptionally(ex -> { - Throwable cause = ex.getCause(); - log.error("[{}] Failed to set property for key {} on namespace {}", clientAppId(), key, - namespaceName, cause); - asyncResponse.resume(cause); - return null; - }); + validateAdminAccessForTenantAsync(namespaceName.getTenant()) + .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) + .thenCompose(__ -> updatePoliciesAsync(namespaceName, policies -> { + policies.properties.put(key, value); + return policies; + })) + .thenAccept(v -> { + log.info("[{}] Successfully set property for key {} on namespace {}", clientAppId(), key, + namespaceName); + asyncResponse.resume(Response.noContent().build()); + }).exceptionally(ex -> { + Throwable cause = ex.getCause(); + log.error("[{}] Failed to set property for key {} on namespace {}", clientAppId(), key, + namespaceName, cause); + asyncResponse.resume(cause); + return null; + }); } protected void internalSetProperties(Map properties, AsyncResponse asyncResponse) { - validatePoliciesReadOnlyAccess(); - updatePoliciesAsync(namespaceName, policies -> { - policies.properties.putAll(properties); - return policies; - }).thenAccept(v -> { - log.info("[{}] Successfully set {} properties on namespace {}", clientAppId(), properties.size(), - namespaceName); - asyncResponse.resume(Response.noContent().build()); - }).exceptionally(ex -> { - Throwable cause = ex.getCause(); - log.error("[{}] Failed to set {} properties on namespace {}", clientAppId(), properties.size(), - namespaceName, cause); - asyncResponse.resume(cause); - return null; - }); + validateAdminAccessForTenantAsync(namespaceName.getTenant()) + .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) + .thenCompose(__ -> updatePoliciesAsync(namespaceName, policies -> { + policies.properties.putAll(properties); + return policies; + })) + .thenAccept(v -> { + log.info("[{}] Successfully set {} properties on namespace {}", clientAppId(), properties.size(), + namespaceName); + asyncResponse.resume(Response.noContent().build()); + }).exceptionally(ex -> { + Throwable cause = ex.getCause(); + log.error("[{}] Failed to set {} properties on namespace {}", clientAppId(), properties.size(), + namespaceName, cause); + asyncResponse.resume(cause); + return null; + }); } protected void internalGetProperty(String key, AsyncResponse asyncResponse) { - getNamespacePoliciesAsync(namespaceName).thenAccept(policies -> { - asyncResponse.resume(policies.properties.get(key)); - }).exceptionally(ex -> { - Throwable cause = ex.getCause(); - log.error("[{}] Failed to get property for key {} of namespace {}", clientAppId(), key, - namespaceName, cause); - asyncResponse.resume(cause); - return null; - }); + validateAdminAccessForTenantAsync(namespaceName.getTenant()) + .thenCompose(__ -> getNamespacePoliciesAsync(namespaceName)) + .thenAccept(policies -> asyncResponse.resume(policies.properties.get(key))) + .exceptionally(ex -> { + Throwable cause = ex.getCause(); + log.error("[{}] Failed to get property for key {} of namespace {}", clientAppId(), key, + namespaceName, cause); + asyncResponse.resume(cause); + return null; + }); } protected void internalGetProperties(AsyncResponse asyncResponse) { - getNamespacePoliciesAsync(namespaceName).thenAccept(policies -> { - asyncResponse.resume(policies.properties); - }).exceptionally(ex -> { - Throwable cause = ex.getCause(); - log.error("[{}] Failed to get properties of namespace {}", clientAppId(), namespaceName, cause); - asyncResponse.resume(cause); - return null; - }); + validateAdminAccessForTenantAsync(namespaceName.getTenant()) + .thenCompose(__ -> getNamespacePoliciesAsync(namespaceName)) + .thenAccept(policies -> asyncResponse.resume(policies.properties)) + .exceptionally(ex -> { + Throwable cause = ex.getCause(); + log.error("[{}] Failed to get properties of namespace {}", clientAppId(), namespaceName, cause); + asyncResponse.resume(cause); + return null; + }); } protected void internalRemoveProperty(String key, AsyncResponse asyncResponse) { - validatePoliciesReadOnlyAccess(); - AtomicReference oldVal = new AtomicReference<>(null); - updatePoliciesAsync(namespaceName, policies -> { - oldVal.set(policies.properties.remove(key)); - return policies; - }).thenAccept(v -> { - asyncResponse.resume(oldVal.get()); - log.info("[{}] Successfully remove property for key {} on namespace {}", clientAppId(), key, - namespaceName); - }).exceptionally(ex -> { - Throwable cause = ex.getCause(); - log.error("[{}] Failed to remove property for key {} on namespace {}", clientAppId(), key, - namespaceName, cause); - asyncResponse.resume(cause); - return null; - }); + validateAdminAccessForTenantAsync(namespaceName.getTenant()) + .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) + .thenCompose(__ -> updatePoliciesAsync(namespaceName, policies -> { + oldVal.set(policies.properties.remove(key)); + return policies; + })).thenAccept(v -> { + asyncResponse.resume(oldVal.get()); + log.info("[{}] Successfully remove property for key {} on namespace {}", clientAppId(), key, + namespaceName); + }).exceptionally(ex -> { + Throwable cause = ex.getCause(); + log.error("[{}] Failed to remove property for key {} on namespace {}", clientAppId(), key, + namespaceName, cause); + asyncResponse.resume(cause); + return null; + }); } protected void internalClearProperties(AsyncResponse asyncResponse) { - validatePoliciesReadOnlyAccess(); AtomicReference clearedCount = new AtomicReference<>(0); - updatePoliciesAsync(namespaceName, policies -> { - clearedCount.set(policies.properties.size()); - policies.properties.clear(); - return policies; - }).thenAccept(v -> { - asyncResponse.resume(Response.noContent().build()); - log.info("[{}] Successfully clear {} properties on namespace {}", clientAppId(), clearedCount.get(), - namespaceName); - }).exceptionally(ex -> { - Throwable cause = ex.getCause(); - log.error("[{}] Failed to clear {} properties on namespace {}", clientAppId(), clearedCount.get(), - namespaceName, cause); - asyncResponse.resume(cause); - return null; - }); + validateAdminAccessForTenantAsync(namespaceName.getTenant()) + .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) + .thenCompose(__ -> updatePoliciesAsync(namespaceName, policies -> { + clearedCount.set(policies.properties.size()); + policies.properties.clear(); + return policies; + })) + .thenAccept(v -> { + asyncResponse.resume(Response.noContent().build()); + log.info("[{}] Successfully clear {} properties on namespace {}", clientAppId(), clearedCount.get(), + namespaceName); + }).exceptionally(ex -> { + Throwable cause = ex.getCause(); + log.error("[{}] Failed to clear {} properties on namespace {}", clientAppId(), clearedCount.get(), + namespaceName, cause); + asyncResponse.resume(cause); + return null; + }); } private CompletableFuture updatePoliciesAsync(NamespaceName ns, Function updateFunction) { @@ -2560,7 +2560,7 @@ protected CompletableFuture internalSetEntryFiltersPerTopicAsync(EntryFilt * Notion: don't re-use this logic. */ protected void internalSetReplicatorDispatchRate(AsyncResponse asyncResponse, DispatchRateImpl dispatchRate) { - validateSuperUserAccessAsync() + validateNamespacePolicyOperationAsync(namespaceName, PolicyName.REPLICATION_RATE, PolicyOperation.WRITE) .thenAccept(__ -> { log.info("[{}] Set namespace replicator dispatch-rate {}/{}", clientAppId(), namespaceName, dispatchRate); @@ -2605,7 +2605,7 @@ protected void internalGetReplicatorDispatchRate(AsyncResponse asyncResponse) { * Notion: don't re-use this logic. */ protected void internalRemoveReplicatorDispatchRate(AsyncResponse asyncResponse) { - validateSuperUserAccessAsync() + validateNamespacePolicyOperationAsync(namespaceName, PolicyName.REPLICATION_RATE, PolicyOperation.WRITE) .thenCompose(__ -> namespaceResources().setPoliciesAsync(namespaceName, policies -> { String clusterName = pulsar().getConfiguration().getClusterName(); policies.replicatorDispatchRate.remove(clusterName); @@ -2686,4 +2686,113 @@ protected void internalRemoveBacklogQuota(AsyncResponse asyncResponse, BacklogQu return null; }); } + + protected void internalEnableMigration(boolean migrated) { + validateSuperUserAccess(); + try { + getLocalPolicies().setLocalPolicies(namespaceName, (policies) -> { + policies.migrated = migrated; + return policies; + }); + log.info("Successfully updated migration on namespace {}", namespaceName); + } catch (Exception e) { + log.error("Failed to update migration on namespace {}", namespaceName, e); + throw new RestException(e); + } + } + + protected Policies getDefaultPolicesIfNull(Policies policies) { + if (policies == null) { + policies = new Policies(); + } + int defaultNumberOfBundles = config().getDefaultNumberOfNamespaceBundles(); + if (policies.bundles == null) { + policies.bundles = getBundles(defaultNumberOfBundles); + } + return policies; + } + + protected CompletableFuture internalSetDispatcherPauseOnAckStatePersistentAsync( + boolean dispatcherPauseOnAckStatePersistentEnabled) { + return validateNamespacePolicyOperationAsync(namespaceName, + PolicyName.DISPATCHER_PAUSE_ON_ACK_STATE_PERSISTENT, PolicyOperation.WRITE) + .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) + .thenCompose(__ -> updatePoliciesAsync(namespaceName, policies -> { + policies.dispatcherPauseOnAckStatePersistentEnabled = dispatcherPauseOnAckStatePersistentEnabled; + return policies; + })); + } + + protected CompletableFuture internalGetDispatcherPauseOnAckStatePersistentAsync() { + return validateNamespacePolicyOperationAsync(namespaceName, + PolicyName.DISPATCHER_PAUSE_ON_ACK_STATE_PERSISTENT, PolicyOperation.READ) + .thenCompose(__ -> namespaceResources().getPoliciesAsync(namespaceName)) + .thenApply(policiesOpt -> { + if (!policiesOpt.isPresent()) { + throw new RestException(Response.Status.NOT_FOUND, "Namespace policies does not exist"); + } + return policiesOpt.map(p -> p.dispatcherPauseOnAckStatePersistentEnabled).orElse(false); + }); + } + + protected CompletableFuture internalSetNamespaceAllowedClusters(List clusterIds) { + return validateNamespacePolicyOperationAsync(namespaceName, PolicyName.ALLOW_CLUSTERS, PolicyOperation.WRITE) + .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) + // Allowed clusters in the namespace policy should be included in the allowed clusters in the tenant + // policy. + .thenCompose(__ -> FutureUtil.waitForAll(clusterIds.stream().map(clusterId -> + validateClusterForTenantAsync(namespaceName.getTenant(), clusterId)) + .collect(Collectors.toList()))) + // Allowed clusters should include all the existed replication clusters and could not contain global + // cluster. + .thenCompose(__ -> { + checkNotNull(clusterIds, "ClusterIds should not be null"); + if (clusterIds.contains("global")) { + throw new RestException(Status.PRECONDITION_FAILED, + "Cannot specify global in the list of allowed clusters"); + } + return getNamespacePoliciesAsync(this.namespaceName).thenApply(namespacePolicies -> { + namespacePolicies.replication_clusters.forEach(replicationCluster -> { + if (!clusterIds.contains(replicationCluster)) { + throw new RestException(Status.BAD_REQUEST, + String.format("Allowed clusters do not contain the replication cluster %s. " + + "Please remove the replication cluster if the cluster is not allowed " + + "for this namespace", replicationCluster)); + } + }); + return Sets.newHashSet(clusterIds); + }); + }) + // Verify the allowed clusters are valid and they do not contain the peer clusters. + .thenCompose(allowedClusters -> clustersAsync() + .thenCompose(clusters -> { + List> futures = + allowedClusters.stream().map(clusterId -> { + if (!clusters.contains(clusterId)) { + throw new RestException(Status.FORBIDDEN, + "Invalid cluster id: " + clusterId); + } + return validatePeerClusterConflictAsync(clusterId, allowedClusters); + }).collect(Collectors.toList()); + return FutureUtil.waitForAll(futures).thenApply(__ -> allowedClusters); + })) + // Update allowed clusters into policies. + .thenCompose(allowedClusterSet -> updatePoliciesAsync(namespaceName, policies -> { + policies.allowed_clusters = allowedClusterSet; + return policies; + })); + } + + protected CompletableFuture> internalGetNamespaceAllowedClustersAsync() { + return validateNamespacePolicyOperationAsync(namespaceName, PolicyName.ALLOW_CLUSTERS, PolicyOperation.READ) + .thenAccept(__ -> { + if (!namespaceName.isGlobal()) { + throw new RestException(Status.PRECONDITION_FAILED, + "Cannot get the allowed clusters for a non-global namespace"); + } + }).thenCompose(__ -> getNamespacePoliciesAsync(namespaceName)) + .thenApply(policies -> policies.allowed_clusters); + } + + } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PackagesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PackagesBase.java index c5b0f60e91c5e..615cc6cb7a6ac 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PackagesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PackagesBase.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.admin.impl; import java.io.InputStream; +import java.nio.file.FileAlreadyExistsException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import javax.ws.rs.WebApplicationException; @@ -27,7 +28,6 @@ import javax.ws.rs.core.StreamingOutput; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.admin.AdminResource; -import org.apache.pulsar.broker.authorization.AuthorizationService; import org.apache.pulsar.broker.web.RestException; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.policies.data.NamespaceOperation; @@ -40,8 +40,6 @@ @Slf4j public class PackagesBase extends AdminResource { - private AuthorizationService authorizationService; - private PackagesManagement getPackagesManagement() { return pulsar().getPackagesManagement(); } @@ -67,6 +65,8 @@ private Void handleError(Throwable throwable, AsyncResponse asyncResponse) { asyncResponse.resume(throwable); } else if (throwable instanceof UnsupportedOperationException) { asyncResponse.resume(new RestException(Response.Status.SERVICE_UNAVAILABLE, throwable.getMessage())); + } else if (throwable instanceof FileAlreadyExistsException) { + asyncResponse.resume(new RestException(Response.Status.CONFLICT, throwable.getMessage())); } else { log.error("Encountered unexpected error", throwable); asyncResponse.resume(new RestException(Response.Status.INTERNAL_SERVER_ERROR, throwable.getMessage())); @@ -197,12 +197,4 @@ private CompletableFuture checkPermissions(String tenant, String namespace } return future; } - - private AuthorizationService getAuthorizationService() { - if (authorizationService == null) { - authorizationService = pulsar().getBrokerService().getAuthorizationService(); - return authorizationService; - } - return authorizationService; - } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index e0f168eb8d842..8860c9bb06d4d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.admin.impl; +import static org.apache.pulsar.common.api.proto.CompressionType.NONE; import static org.apache.pulsar.common.naming.SystemTopicNames.isSystemTopic; import static org.apache.pulsar.common.naming.SystemTopicNames.isTransactionCoordinatorAssign; import static org.apache.pulsar.common.naming.SystemTopicNames.isTransactionInternalName; @@ -27,6 +28,7 @@ import com.github.zafarkhaja.semver.Version; import com.google.common.base.Throwables; import com.google.common.collect.Sets; +import com.google.gson.Gson; import io.netty.buffer.ByteBuf; import java.io.IOException; import java.io.UncheckedIOException; @@ -61,11 +63,10 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerInfo; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.ScanOutcome; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.bookkeeper.mledger.impl.AckSetStateUtil; import org.apache.bookkeeper.mledger.impl.ManagedLedgerOfflineBacklog; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; @@ -79,8 +80,11 @@ import org.apache.pulsar.broker.service.BrokerServiceException.AlreadyRunningException; import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException; import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionInvalidCursorPosition; +import org.apache.pulsar.broker.service.GetStatsOptions; +import org.apache.pulsar.broker.service.MessageExpirer; import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.TopicPoliciesService; import org.apache.pulsar.broker.service.persistent.PersistentReplicator; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -94,6 +98,7 @@ import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; @@ -112,7 +117,6 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.AuthAction; -import org.apache.pulsar.common.policies.data.AuthPolicies; import org.apache.pulsar.common.policies.data.AutoSubscriptionCreationOverride; import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies; @@ -126,8 +130,6 @@ import org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats; import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; import org.apache.pulsar.common.policies.data.Policies; -import org.apache.pulsar.common.policies.data.PolicyName; -import org.apache.pulsar.common.policies.data.PolicyOperation; import org.apache.pulsar.common.policies.data.PublishRate; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; @@ -146,6 +148,7 @@ import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.BitSetRecyclable; +import org.apache.pulsar.compaction.Compactor; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; import org.slf4j.Logger; @@ -186,19 +189,6 @@ protected CompletableFuture> internalGetListAsync(Optional ); } - protected CompletableFuture> internalGetListAsync() { - return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.GET_TOPICS) - .thenCompose(__ -> namespaceResources().namespaceExistsAsync(namespaceName)) - .thenAccept(exists -> { - if (!exists) { - throw new RestException(Status.NOT_FOUND, "Namespace does not exist"); - } - }) - .thenCompose(__ -> topicResources().listPersistentTopicsAsync(namespaceName)) - .thenApply(topics -> topics.stream().filter(topic -> - !isTransactionInternalName(TopicName.get(topic))).collect(Collectors.toList())); - } - protected CompletableFuture> internalGetPartitionedTopicListAsync() { return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.GET_TOPICS) .thenCompose(__ -> namespaceResources().namespaceExistsAsync(namespaceName)) @@ -217,36 +207,8 @@ protected CompletableFuture> internalGetPartitionedTopicListAsync() protected CompletableFuture>> internalGetPermissionsOnTopic() { // This operation should be reading from zookeeper and it should be allowed without having admin privileges return validateAdminAccessForTenantAsync(namespaceName.getTenant()) - .thenCompose(__ -> namespaceResources().getPoliciesAsync(namespaceName) - .thenApply(policies -> { - if (!policies.isPresent()) { - throw new RestException(Status.NOT_FOUND, "Namespace does not exist"); - } - - Map> permissions = new HashMap<>(); - String topicUri = topicName.toString(); - AuthPolicies auth = policies.get().auth_policies; - // First add namespace level permissions - permissions.putAll(auth.getNamespaceAuthentication()); - - // Then add topic level permissions - if (auth.getTopicAuthentication().containsKey(topicUri)) { - for (Map.Entry> entry : - auth.getTopicAuthentication().get(topicUri).entrySet()) { - String role = entry.getKey(); - Set topicPermissions = entry.getValue(); - - if (!permissions.containsKey(role)) { - permissions.put(role, topicPermissions); - } else { - // Do the union between namespace and topic level - Set union = Sets.union(permissions.get(role), topicPermissions); - permissions.put(role, union); - } - } - } - return permissions; - })); + .thenCompose(__ -> internalCheckTopicExists(topicName)) + .thenCompose(__ -> getAuthorizationService().getPermissionsAsync(topicName)); } protected void validateCreateTopic(TopicName topicName) { @@ -297,21 +259,11 @@ protected void internalGrantPermissionsOnTopic(final AsyncResponse asyncResponse Set actions) { // This operation should be reading from zookeeper and it should be allowed without having admin privileges validateAdminAccessForTenantAsync(namespaceName.getTenant()) - .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync().thenCompose(unused1 -> - getPartitionedTopicMetadataAsync(topicName, true, false) - .thenCompose(metadata -> { - int numPartitions = metadata.partitions; - CompletableFuture future = CompletableFuture.completedFuture(null); - if (numPartitions > 0) { - for (int i = 0; i < numPartitions; i++) { - TopicName topicNamePartition = topicName.getPartition(i); - future = future.thenCompose(unused -> grantPermissionsAsync(topicNamePartition, role, - actions)); - } - } - return future.thenCompose(unused -> grantPermissionsAsync(topicName, role, actions)) - .thenAccept(unused -> asyncResponse.resume(Response.noContent().build())); - }))).exceptionally(ex -> { + .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) + .thenCompose(__ -> internalCheckTopicExists(topicName)) + .thenCompose(unused1 -> grantPermissionsAsync(topicName, role, actions)) + .thenAccept(unused -> asyncResponse.resume(Response.noContent().build())) + .exceptionally(ex -> { Throwable realCause = FutureUtil.unwrapCompletionException(ex); log.error("[{}] Failed to get permissions for topic {}", clientAppId(), topicName, realCause); resumeAsyncResponseExceptionally(asyncResponse, realCause); @@ -319,45 +271,11 @@ protected void internalGrantPermissionsOnTopic(final AsyncResponse asyncResponse }); } - private CompletableFuture revokePermissionsAsync(String topicUri, String role, boolean force) { - return namespaceResources().getPoliciesAsync(namespaceName).thenCompose( - policiesOptional -> { - Policies policies = policiesOptional.orElseThrow(() -> - new RestException(Status.NOT_FOUND, "Namespace does not exist")); - if (!policies.auth_policies.getTopicAuthentication().containsKey(topicUri) - || !policies.auth_policies.getTopicAuthentication().get(topicUri).containsKey(role)) { - log.warn("[{}] Failed to revoke permission from role {} on topic: Not set at topic level {}", - clientAppId(), role, topicUri); - if (force) { - return CompletableFuture.completedFuture(null); - } else { - return FutureUtil.failedFuture(new RestException(Status.PRECONDITION_FAILED, - "Permissions are not set at the topic level")); - } - } - // Write the new policies to metadata store - return namespaceResources().setPoliciesAsync(namespaceName, p -> { - p.auth_policies.getTopicAuthentication().computeIfPresent(topicUri, (k, roles) -> { - roles.remove(role); - if (roles.isEmpty()) { - return null; - } - return roles; - }); - return p; - }).thenAccept(__ -> - log.info("[{}] Successfully revoke access for role {} - topic {}", clientAppId(), role, - topicUri) - ); - } - ); - } - protected void internalRevokePermissionsOnTopic(AsyncResponse asyncResponse, String role) { // This operation should be reading from zookeeper and it should be allowed without having admin privileges validateAdminAccessForTenantAsync(namespaceName.getTenant()) - .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync().thenCompose(unused1 -> - getPartitionedTopicMetadataAsync(topicName, true, false) + .thenCompose(__ -> internalCheckTopicExists(topicName)) + .thenCompose(__ -> getPartitionedTopicMetadataAsync(topicName, true, false) .thenCompose(metadata -> { int numPartitions = metadata.partitions; CompletableFuture future = CompletableFuture.completedFuture(null); @@ -365,13 +283,14 @@ protected void internalRevokePermissionsOnTopic(AsyncResponse asyncResponse, Str for (int i = 0; i < numPartitions; i++) { TopicName topicNamePartition = topicName.getPartition(i); future = future.thenComposeAsync(unused -> - revokePermissionsAsync(topicNamePartition.toString(), role, true)); + getAuthorizationService().revokePermissionAsync(topicNamePartition, role)); } } - return future.thenComposeAsync(unused -> revokePermissionsAsync(topicName.toString(), role, false)) - .thenAccept(unused -> asyncResponse.resume(Response.noContent().build())); - })) - ).exceptionally(ex -> { + return future.thenComposeAsync(unused -> + getAuthorizationService().revokePermissionAsync(topicName, role)) + .thenAccept(unused -> asyncResponse.resume(Response.noContent().build())); + }) + ).exceptionally(ex -> { Throwable realCause = FutureUtil.unwrapCompletionException(ex); log.error("[{}] Failed to revoke permissions for topic {}", clientAppId(), topicName, realCause); resumeAsyncResponseExceptionally(asyncResponse, realCause); @@ -427,14 +346,15 @@ protected CompletableFuture internalCreateNonPartitionedTopicAsync(boolean } if (expectPartitions < currentMetadataPartitions) { throw new RestException(422 /* Unprocessable entity*/, - String.format("Expect partitions %s can't less than current partitions %s.", + String.format("Desired partitions %s can't be less than the current partitions %s.", expectPartitions, currentMetadataPartitions)); } int brokerMaximumPartitionsPerTopic = pulsarService.getConfiguration() .getMaxNumPartitionsPerPartitionedTopic(); - if (brokerMaximumPartitionsPerTopic != 0 && expectPartitions > brokerMaximumPartitionsPerTopic) { + if (brokerMaximumPartitionsPerTopic > 0 && expectPartitions > brokerMaximumPartitionsPerTopic) { throw new RestException(422 /* Unprocessable entity*/, - String.format("Expect partitions %s grater than maximum partitions per topic %s", + String.format("Desired partitions %s can't be greater than the maximum partitions per" + + " topic %s.", expectPartitions, brokerMaximumPartitionsPerTopic)); } final PulsarAdmin admin; @@ -528,13 +448,9 @@ protected CompletableFuture internalCreateNonPartitionedTopicAsync(boolean return CompletableFuture.completedFuture(null); } // update remote cluster - return namespaceResources().getPoliciesAsync(namespaceName) - .thenCompose(policies -> { - if (!policies.isPresent()) { - return CompletableFuture.completedFuture(null); - } - final Set replicationClusters = policies.get().replication_clusters; - if (replicationClusters.size() == 0) { + return getReplicationClusters() + .thenCompose(replicationClusters -> { + if (replicationClusters == null || replicationClusters.isEmpty()) { return CompletableFuture.completedFuture(null); } boolean containsCurrentCluster = @@ -549,6 +465,7 @@ protected CompletableFuture internalCreateNonPartitionedTopicAsync(boolean // The replication clusters just has the current cluster itself. return CompletableFuture.completedFuture(null); } + // Do sync operation to other clusters. List> futures = replicationClusters.stream() .map(replicationCluster -> admin.clusters().getClusterAsync(replicationCluster) .thenCompose(clusterData -> pulsarService.getBrokerService() @@ -568,16 +485,34 @@ protected CompletableFuture internalCreateNonPartitionedTopicAsync(boolean }); } + private CompletableFuture> getReplicationClusters() { + return namespaceResources().getPoliciesAsync(namespaceName).thenCompose(optionalPolicies -> { + if (optionalPolicies.isEmpty()) { + return CompletableFuture.completedFuture(null); + } + // Query the topic-level policies only if the namespace-level policies exist + final var namespacePolicies = optionalPolicies.get(); + return pulsar().getTopicPoliciesService().getTopicPoliciesAsync(topicName, + TopicPoliciesService.GetType.DEFAULT + ).thenApply(optionalTopicPolicies -> optionalTopicPolicies.map(TopicPolicies::getReplicationClustersSet) + .orElse(namespacePolicies.replication_clusters)); + }); + } + protected void internalCreateMissedPartitions(AsyncResponse asyncResponse) { getPartitionedTopicMetadataAsync(topicName, false, false).thenAccept(metadata -> { - if (metadata != null) { - tryCreatePartitionsAsync(metadata.partitions).thenAccept(v -> { + if (metadata != null && metadata.partitions > 0) { + CompletableFuture future = validateNamespaceOperationAsync(topicName.getNamespaceObject(), + NamespaceOperation.CREATE_TOPIC); + future.thenCompose(__ -> tryCreatePartitionsAsync(metadata.partitions)).thenAccept(v -> { asyncResponse.resume(Response.noContent().build()); }).exceptionally(e -> { log.error("[{}] Failed to create partitions for topic {}", clientAppId(), topicName); resumeAsyncResponseExceptionally(asyncResponse, e); return null; }); + } else { + throw new RestException(Status.NOT_FOUND, String.format("Topic %s does not exist", topicName)); } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. @@ -599,6 +534,8 @@ protected CompletableFuture internalSetDelayedDeliveryPolicies(DelayedDeli topicPolicies.setDelayedDeliveryEnabled(deliveryPolicies == null ? null : deliveryPolicies.isActive()); topicPolicies.setDelayedDeliveryTickTimeMillis( deliveryPolicies == null ? null : deliveryPolicies.getTickTime()); + topicPolicies.setDelayedDeliveryMaxDelayInMillis( + deliveryPolicies == null ? null : deliveryPolicies.getMaxDeliveryDelayInMillis()); return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, topicPolicies); }); } @@ -613,25 +550,35 @@ protected CompletableFuture internalGetPartitionedMeta boolean checkAllowAutoCreation) { return getPartitionedTopicMetadataAsync(topicName, authoritative, checkAllowAutoCreation) .thenCompose(metadata -> { - CompletableFuture ret; - if (metadata.partitions == 0 && !checkAllowAutoCreation) { + if (metadata.partitions > 1) { + // Some clients does not support partitioned topic. + return internalValidateClientVersionAsync().thenApply(__ -> metadata); + } else if (metadata.partitions == 1) { + return CompletableFuture.completedFuture(metadata); + } else { + // metadata.partitions == 0 // The topic may be a non-partitioned topic, so check if it exists here. // However, when checkAllowAutoCreation is true, the client will create the topic if // it doesn't exist. In this case, `partitions == 0` means the automatically created topic // is a non-partitioned topic so we shouldn't check if the topic exists. - ret = internalCheckTopicExists(topicName); - } else if (metadata.partitions > 1) { - ret = internalValidateClientVersionAsync(); - } else { - ret = CompletableFuture.completedFuture(null); + return pulsar().getBrokerService().isAllowAutoTopicCreationAsync(topicName) + .thenCompose(brokerAllowAutoTopicCreation -> { + if (checkAllowAutoCreation && brokerAllowAutoTopicCreation) { + // Whether it exists or not, auto create a non-partitioned topic by client. + return CompletableFuture.completedFuture(metadata); + } else { + // If it does not exist, response a Not Found error. + // Otherwise, response a non-partitioned metadata. + return internalCheckNonPartitionedTopicExists(topicName).thenApply(__ -> metadata); + } + }); } - return ret.thenApply(__ -> metadata); }); } protected CompletableFuture> internalGetPropertiesAsync(boolean authoritative) { - return validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.GET_METADATA)) + return validateTopicOperationAsync(topicName, TopicOperation.GET_METADATA) + .thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) .thenCompose(__ -> { if (topicName.isPartitioned()) { return getPropertiesAsync(); @@ -663,27 +610,27 @@ protected CompletableFuture internalUpdatePropertiesAsync(boolean authorit log.warn("[{}] [{}] properties is empty, ignore update", clientAppId(), topicName); return CompletableFuture.completedFuture(null); } - return validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.UPDATE_METADATA)) - .thenCompose(__ -> { - if (topicName.isPartitioned()) { - return internalUpdateNonPartitionedTopicProperties(properties); - } else { - return pulsar().getBrokerService().fetchPartitionedTopicMetadataAsync(topicName) - .thenCompose(metadata -> { - if (metadata.partitions == 0) { - return internalUpdateNonPartitionedTopicProperties(properties); - } - return namespaceResources() - .getPartitionedTopicResources().updatePartitionedTopicAsync(topicName, - p -> new PartitionedTopicMetadata(p.partitions, - p.properties == null ? properties - : MapUtils.putAll(p.properties, properties.entrySet().toArray()))); - }); - } - }).thenAccept(__ -> - log.info("[{}] [{}] update properties success with properties {}", - clientAppId(), topicName, properties)); + return validateTopicOperationAsync(topicName, TopicOperation.UPDATE_METADATA) + .thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) + .thenCompose(__ -> { + if (topicName.isPartitioned()) { + return internalUpdateNonPartitionedTopicProperties(properties); + } else { + return pulsar().getBrokerService().fetchPartitionedTopicMetadataAsync(topicName) + .thenCompose(metadata -> { + if (metadata.partitions == 0) { + return internalUpdateNonPartitionedTopicProperties(properties); + } + return namespaceResources() + .getPartitionedTopicResources().updatePartitionedTopicAsync(topicName, + p -> new PartitionedTopicMetadata(p.partitions, + p.properties == null ? properties + : MapUtils.putAll(p.properties, properties.entrySet().toArray()))); + }); + } + }).thenAccept(__ -> + log.info("[{}] [{}] update properties success with properties {}", clientAppId(), + topicName, properties)); } private CompletableFuture internalUpdateNonPartitionedTopicProperties(Map properties) { @@ -717,8 +664,8 @@ public void updatePropertiesFailed(ManagedLedgerException exception, Object ctx) } protected CompletableFuture internalRemovePropertiesAsync(boolean authoritative, String key) { - return validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.DELETE_METADATA)) + return validateTopicOperationAsync(topicName, TopicOperation.DELETE_METADATA) + .thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) .thenCompose(__ -> { if (topicName.isPartitioned()) { return internalRemoveNonPartitionedTopicProperties(key); @@ -770,6 +717,17 @@ public void updatePropertiesFailed(ManagedLedgerException exception, Object ctx) protected CompletableFuture internalCheckTopicExists(TopicName topicName) { return pulsar().getNamespaceService().checkTopicExists(topicName) + .thenAccept(info -> { + boolean exists = info.isExists(); + info.recycle(); + if (!exists) { + throw new RestException(Status.NOT_FOUND, getTopicNotFoundErrorMessage(topicName.toString())); + } + }); + } + + protected CompletableFuture internalCheckNonPartitionedTopicExists(TopicName topicName) { + return pulsar().getNamespaceService().checkNonPartitionedTopicExists(topicName) .thenAccept(exist -> { if (!exist) { throw new RestException(Status.NOT_FOUND, getTopicNotFoundErrorMessage(topicName.toString())); @@ -780,21 +738,32 @@ protected CompletableFuture internalCheckTopicExists(TopicName topicName) protected void internalDeletePartitionedTopic(AsyncResponse asyncResponse, boolean authoritative, boolean force) { - validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(__ -> validateNamespaceOperationAsync(topicName.getNamespaceObject(), - NamespaceOperation.DELETE_TOPIC)) + validateNamespaceOperationAsync(topicName.getNamespaceObject(), NamespaceOperation.DELETE_TOPIC) + .thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) .thenCompose(__ -> pulsar().getBrokerService() .fetchPartitionedTopicMetadataAsync(topicName) .thenCompose(partitionedMeta -> { final int numPartitions = partitionedMeta.partitions; if (numPartitions < 1) { - return CompletableFuture.completedFuture(null); + return pulsar().getNamespaceService().checkNonPartitionedTopicExists(topicName) + .thenApply(exists -> { + if (exists) { + throw new RestException(Response.Status.CONFLICT, + String.format("%s is a non-partitioned topic. Instead of calling" + + " delete-partitioned-topic please call delete.", topicName)); + } else { + throw new RestException(Status.NOT_FOUND, + String.format("Topic %s not found.", topicName)); + } + }); } - return internalRemovePartitionsAuthenticationPoliciesAsync(numPartitions) + return internalRemovePartitionsAuthenticationPoliciesAsync() .thenCompose(unused -> internalRemovePartitionsTopicAsync(numPartitions, force)); }) // Only tries to delete the znode for partitioned topic when all its partitions are successfully deleted - ).thenCompose(__ -> getPulsarResources().getNamespaceResources().getPartitionedTopicResources() + ).thenCompose(ignore -> + pulsar().getBrokerService().deleteSchema(topicName).exceptionally(ex -> null)) + .thenCompose(__ -> getPulsarResources().getNamespaceResources().getPartitionedTopicResources() .runWithMarkDeleteAsync(topicName, () -> namespaceResources() .getPartitionedTopicResources().deletePartitionedTopicAsync(topicName))) .thenAccept(__ -> { @@ -832,10 +801,10 @@ protected void internalDeletePartitionedTopic(AsyncResponse asyncResponse, private CompletableFuture internalRemovePartitionsTopicAsync(int numPartitions, boolean force) { return pulsar().getPulsarResources().getNamespaceResources().getPartitionedTopicResources() .runWithMarkDeleteAsync(topicName, - () -> internalRemovePartitionsTopicNoAutocreationDisableAsync(numPartitions, force)); + () -> internalRemovePartitionsTopicNoAutoCreationDisableAsync(numPartitions, force)); } - private CompletableFuture internalRemovePartitionsTopicNoAutocreationDisableAsync(int numPartitions, + private CompletableFuture internalRemovePartitionsTopicNoAutoCreationDisableAsync(int numPartitions, boolean force) { return FutureUtil.waitForAll(IntStream.range(0, numPartitions) .mapToObj(i -> { @@ -877,16 +846,9 @@ private CompletableFuture internalRemovePartitionsTopicNoAutocreationDisab }).collect(Collectors.toList())); } - private CompletableFuture internalRemovePartitionsAuthenticationPoliciesAsync(int numPartitions) { + private CompletableFuture internalRemovePartitionsAuthenticationPoliciesAsync() { CompletableFuture future = new CompletableFuture<>(); - pulsar().getPulsarResources().getNamespaceResources() - .setPoliciesAsync(topicName.getNamespaceObject(), p -> { - IntStream.range(0, numPartitions) - .forEach(i -> p.auth_policies.getTopicAuthentication() - .remove(topicName.getPartition(i).toString())); - p.auth_policies.getTopicAuthentication().remove(topicName.toString()); - return p; - }) + getAuthorizationService().removePermissionsAsync(topicName) .whenComplete((r, ex) -> { if (ex != null){ Throwable realCause = FutureUtil.unwrapCompletionException(ex); @@ -908,13 +870,13 @@ private CompletableFuture internalRemovePartitionsAuthenticationPoliciesAs protected void internalUnloadTopic(AsyncResponse asyncResponse, boolean authoritative) { log.info("[{}] Unloading topic {}", clientAppId(), topicName); - CompletableFuture future; - if (topicName.isGlobal()) { - future = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - future = CompletableFuture.completedFuture(null); - } - future.thenAccept(__ -> { + CompletableFuture future = validateTopicOperationAsync(topicName, TopicOperation.UNLOAD); + future.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } + return CompletableFuture.completedFuture(null); + }).thenAccept(__ -> { // If the topic name is a partition name, no need to get partition topic metadata again if (topicName.isPartitioned()) { if (isTransactionCoordinatorAssign(topicName)) { @@ -962,7 +924,7 @@ protected void internalUnloadTopic(AsyncResponse asyncResponse, boolean authorit } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get partitioned metadata while unloading topic {}", clientAppId(), topicName, ex); } @@ -972,7 +934,7 @@ protected void internalUnloadTopic(AsyncResponse asyncResponse, boolean authorit } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to validate the global namespace ownership while unloading topic {}", clientAppId(), topicName, ex); } @@ -991,6 +953,7 @@ protected CompletableFuture internalGetDelayedDeliveryP delayedDeliveryPolicies = DelayedDeliveryPolicies.builder() .tickTime(policies.getDelayedDeliveryTickTimeMillis()) .active(policies.getDelayedDeliveryEnabled()) + .maxDeliveryDelayInMillis(policies.getDelayedDeliveryMaxDelayInMillis()) .build(); } if (delayedDeliveryPolicies == null && applied) { @@ -999,6 +962,8 @@ protected CompletableFuture internalGetDelayedDeliveryP delayedDeliveryPolicies = DelayedDeliveryPolicies.builder() .tickTime(pulsar().getConfiguration().getDelayedDeliveryTickTimeMillis()) .active(pulsar().getConfiguration().isDelayedDeliveryEnabled()) + .maxDeliveryDelayInMillis( + pulsar().getConfiguration().getDelayedDeliveryMaxDelayInMillis()) .build(); } } @@ -1020,8 +985,8 @@ protected CompletableFuture internalGetOffloadPolicies(bool }); } - protected CompletableFuture internalSetOffloadPolicies - (OffloadPoliciesImpl offloadPolicies, boolean isGlobal) { + protected CompletableFuture internalSetOffloadPolicies(OffloadPoliciesImpl offloadPolicies, + boolean isGlobal) { return getTopicPoliciesAsyncWithRetry(topicName, isGlobal) .thenCompose(op -> { TopicPolicies topicPolicies = op.orElseGet(TopicPolicies::new); @@ -1131,16 +1096,15 @@ protected CompletableFuture internalSetDeduplicationSnapshotInterval(Integ private void internalUnloadNonPartitionedTopicAsync(AsyncResponse asyncResponse, boolean authoritative) { validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(unused -> validateTopicOperationAsync(topicName, TopicOperation.UNLOAD) .thenCompose(__ -> getTopicReferenceAsync(topicName)) .thenCompose(topic -> topic.close(false)) .thenRun(() -> { log.info("[{}] Successfully unloaded topic {}", clientAppId(), topicName); asyncResponse.resume(Response.noContent().build()); - })) + }) .exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to unload topic {}, {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -1150,19 +1114,17 @@ private void internalUnloadNonPartitionedTopicAsync(AsyncResponse asyncResponse, private void internalUnloadTransactionCoordinatorAsync(AsyncResponse asyncResponse, boolean authoritative) { validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.UNLOAD) - .thenCompose(v -> pulsar() - .getTransactionMetadataStoreService() - .removeTransactionMetadataStore( - TransactionCoordinatorID.get(topicName.getPartitionIndex()))) - .thenRun(() -> { - log.info("[{}] Successfully unloaded tc {}", clientAppId(), - topicName.getPartitionIndex()); - asyncResponse.resume(Response.noContent().build()); - })) + .thenCompose(v -> pulsar() + .getTransactionMetadataStoreService() + .removeTransactionMetadataStore( + TransactionCoordinatorID.get(topicName.getPartitionIndex()))) + .thenRun(() -> { + log.info("[{}] Successfully unloaded tc {}", clientAppId(), topicName.getPartitionIndex()); + asyncResponse.resume(Response.noContent().build()); + }) .exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to unload tc {},{}", clientAppId(), topicName.getPartitionIndex(), ex); } @@ -1193,98 +1155,89 @@ private boolean isUnexpectedTopicName(PartitionedTopicMetadata topicMetadata) { } protected void internalGetSubscriptions(AsyncResponse asyncResponse, boolean authoritative) { - CompletableFuture future; - if (topicName.isGlobal()) { - future = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - future = CompletableFuture.completedFuture(null); - } - future.thenCompose(__ -> - validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(unused -> validateTopicOperationAsync(topicName, TopicOperation.GET_SUBSCRIPTIONS)) - .thenAccept(unused1 -> { - // If the topic name is a partition name, no need to get partition topic metadata again - if (topicName.isPartitioned()) { - internalGetSubscriptionsForNonPartitionedTopic(asyncResponse); - } else { - getPartitionedTopicMetadataAsync(topicName, authoritative, false) - .thenAccept(partitionMetadata -> { - if (partitionMetadata.partitions > 0 && !isUnexpectedTopicName(partitionMetadata)) { - try { - final Set subscriptions = - Collections.newSetFromMap( - new ConcurrentHashMap<>(partitionMetadata.partitions)); - final List> subscriptionFutures = new ArrayList<>(); - if (topicName.getDomain() == TopicDomain.persistent) { - final Map> existsFutures = - new ConcurrentHashMap<>(partitionMetadata.partitions); - for (int i = 0; i < partitionMetadata.partitions; i++) { - existsFutures.put(i, - topicResources().persistentTopicExists(topicName.getPartition(i))); - } - FutureUtil.waitForAll(new ArrayList<>(existsFutures.values())) - .thenApply(unused2 -> - existsFutures.entrySet().stream().filter(e -> e.getValue().join()) - .map(item -> topicName.getPartition(item.getKey()).toString()) - .collect(Collectors.toList()) - ).thenAccept(topics -> { - if (log.isDebugEnabled()) { - log.debug("activeTopics : {}", topics); - } - topics.forEach(topic -> { - try { - CompletableFuture> subscriptionsAsync = pulsar() - .getAdminClient() - .topics().getSubscriptionsAsync(topic); - subscriptionFutures.add(subscriptionsAsync - .thenApply(subscriptions::addAll)); - } catch (PulsarServerException e) { - throw new RestException(e); - } - }); - }).thenAccept(unused3 -> resumeAsyncResponse(asyncResponse, - subscriptions, subscriptionFutures)); - } else { - for (int i = 0; i < partitionMetadata.partitions; i++) { + CompletableFuture future = validateTopicOperationAsync(topicName, TopicOperation.GET_SUBSCRIPTIONS); + future.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) + .thenAccept(__ -> { + // If the topic name is a partition name, no need to get partition topic metadata again + if (topicName.isPartitioned()) { + internalGetSubscriptionsForNonPartitionedTopic(asyncResponse); + } else { + getPartitionedTopicMetadataAsync(topicName, authoritative, false) + .thenAccept(partitionMetadata -> { + if (partitionMetadata.partitions > 0 && !isUnexpectedTopicName(partitionMetadata)) { + try { + final Set subscriptions = + Collections.newSetFromMap( + new ConcurrentHashMap<>(partitionMetadata.partitions)); + final List> subscriptionFutures = new ArrayList<>(); + if (topicName.getDomain() == TopicDomain.persistent) { + final Map> existsFutures = + new ConcurrentHashMap<>(partitionMetadata.partitions); + for (int i = 0; i < partitionMetadata.partitions; i++) { + existsFutures.put(i, + topicResources().persistentTopicExists(topicName.getPartition(i))); + } + FutureUtil.waitForAll(new ArrayList<>(existsFutures.values())) + .thenApply(unused2 -> + existsFutures.entrySet().stream().filter(e -> e.getValue().join()) + .map(item -> topicName.getPartition(item.getKey()).toString()) + .collect(Collectors.toList()) + ).thenAccept(topics -> { + if (log.isDebugEnabled()) { + log.debug("activeTopics : {}", topics); + } + topics.forEach(topic -> { + try { CompletableFuture> subscriptionsAsync = pulsar() - .getAdminClient().topics() - .getSubscriptionsAsync(topicName.getPartition(i).toString()); + .getAdminClient() + .topics().getSubscriptionsAsync(topic); subscriptionFutures.add(subscriptionsAsync .thenApply(subscriptions::addAll)); + } catch (PulsarServerException e) { + throw new RestException(e); } - resumeAsyncResponse(asyncResponse, subscriptions, subscriptionFutures); - } - } catch (Exception e) { - log.error("[{}] Failed to get list of subscriptions for {}", - clientAppId(), topicName, e); - asyncResponse.resume(e); - } + }); + }).thenAccept(unused3 -> resumeAsyncResponse(asyncResponse, + subscriptions, subscriptionFutures)); } else { - internalGetSubscriptionsForNonPartitionedTopic(asyncResponse); - } - }).exceptionally(ex -> { - // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { - log.error("[{}] Failed to get partitioned topic metadata while get" - + " subscriptions for topic {}", clientAppId(), topicName, ex); + for (int i = 0; i < partitionMetadata.partitions; i++) { + CompletableFuture> subscriptionsAsync = pulsar() + .getAdminClient().topics() + .getSubscriptionsAsync(topicName.getPartition(i).toString()); + subscriptionFutures.add(subscriptionsAsync + .thenApply(subscriptions::addAll)); + } + resumeAsyncResponse(asyncResponse, subscriptions, subscriptionFutures); } - resumeAsyncResponseExceptionally(asyncResponse, ex); - return null; - }); + } catch (Exception e) { + log.error("[{}] Failed to get list of subscriptions for {}", + clientAppId(), topicName, e); + asyncResponse.resume(e); + } + } else { + internalGetSubscriptionsForNonPartitionedTopic(asyncResponse); } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { - log.error("[{}] Failed to validate the global namespace/topic ownership while get subscriptions" - + " for topic {}", clientAppId(), topicName, ex); + if (isNot307And404Exception(ex)) { + log.error("[{}] Failed to get partitioned topic metadata while get" + + " subscriptions for topic {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); return null; - }) - ).exceptionally(ex -> { + }); + } + }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { - log.error("[{}] Failed to get subscriptions for {}", clientAppId(), topicName, ex); + if (isNot307And404Exception(ex)) { + log.error("[{}] Failed to validate the global namespace/topic ownership while get subscriptions" + + " for topic {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); return null; @@ -1312,17 +1265,17 @@ private void resumeAsyncResponse(AsyncResponse asyncResponse, Set subscr return; } } else { - asyncResponse.resume(new ArrayList<>(subscriptions)); + asyncResponse.resume(subscriptions); } }); } private void internalGetSubscriptionsForNonPartitionedTopic(AsyncResponse asyncResponse) { getTopicReferenceAsync(topicName) - .thenAccept(topic -> asyncResponse.resume(new ArrayList<>(topic.getSubscriptions().keys()))) + .thenAccept(topic -> asyncResponse.resume(topic.getSubscriptions().keySet())) .exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get list of subscriptions for {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -1331,52 +1284,47 @@ private void internalGetSubscriptionsForNonPartitionedTopic(AsyncResponse asyncR } protected CompletableFuture internalGetStatsAsync(boolean authoritative, - boolean getPreciseBacklog, - boolean subscriptionBacklogSize, - boolean getEarliestTimeInBacklog) { - CompletableFuture future; - - if (topicName.isGlobal()) { - future = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - future = CompletableFuture.completedFuture(null); - } - - return future.thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) - .thenComposeAsync(__ -> validateTopicOperationAsync(topicName, TopicOperation.GET_STATS)) - .thenCompose(__ -> getTopicReferenceAsync(topicName)) - .thenCompose(topic -> topic.asyncGetStats(getPreciseBacklog, subscriptionBacklogSize, - getEarliestTimeInBacklog)); + GetStatsOptions getStatsOptions) { + CompletableFuture future = validateTopicOperationAsync(topicName, TopicOperation.GET_STATS); + return future.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) + .thenCompose(__ -> getTopicReferenceAsync(topicName)) + .thenCompose(topic -> topic.asyncGetStats(getStatsOptions)); } protected CompletableFuture internalGetInternalStatsAsync(boolean authoritative, boolean metadata) { - CompletableFuture ret; - if (topicName.isGlobal()) { - ret = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - ret = CompletableFuture.completedFuture(null); - } - return ret.thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) - .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.GET_STATS)) - .thenCompose(__ -> { - if (metadata) { - return validateTopicOperationAsync(topicName, TopicOperation.GET_METADATA); - } - return CompletableFuture.completedFuture(null); - }) - .thenCompose(__ -> getTopicReferenceAsync(topicName)) - .thenCompose(topic -> topic.getInternalStats(metadata)); + CompletableFuture future = validateTopicOperationAsync(topicName, TopicOperation.GET_STATS); + return future.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) + .thenCompose(__ -> { + if (metadata) { + return validateTopicOperationAsync(topicName, TopicOperation.GET_METADATA); + } + return CompletableFuture.completedFuture(null); + }) + .thenCompose(__ -> getTopicReferenceAsync(topicName)) + .thenCompose(topic -> topic.getInternalStats(metadata)); } protected void internalGetManagedLedgerInfo(AsyncResponse asyncResponse, boolean authoritative) { - CompletableFuture future; - if (topicName.isGlobal()) { - future = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - future = CompletableFuture.completedFuture(null); - } - future.thenAccept(__ -> { + CompletableFuture future = validateTopicOperationAsync(topicName, TopicOperation.GET_STATS); + future.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } + return CompletableFuture.completedFuture(null); + }).thenAccept(__ -> { // If the topic name is a partition name, no need to get partition topic metadata again if (topicName.isPartitioned()) { internalGetManagedLedgerInfoForNonPartitionedTopic(asyncResponse); @@ -1434,7 +1382,7 @@ protected void internalGetManagedLedgerInfo(AsyncResponse asyncResponse, boolean } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get partitioned metadata while get managed info for {}", clientAppId(), topicName, ex); } @@ -1444,7 +1392,7 @@ protected void internalGetManagedLedgerInfo(AsyncResponse asyncResponse, boolean } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to validate the global namespace ownership while get managed info for {}", clientAppId(), topicName, ex); } @@ -1479,15 +1427,14 @@ public void getInfoFailed(ManagedLedgerException exception, Object ctx) { } protected void internalGetPartitionedStats(AsyncResponse asyncResponse, boolean authoritative, boolean perPartition, - boolean getPreciseBacklog, boolean subscriptionBacklogSize, - boolean getEarliestTimeInBacklog) { - CompletableFuture future; - if (topicName.isGlobal()) { - future = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - future = CompletableFuture.completedFuture(null); - } - future.thenCompose(__ -> getPartitionedTopicMetadataAsync(topicName, + GetStatsOptions getStatsOptions) { + CompletableFuture future = validateTopicOperationAsync(topicName, TopicOperation.GET_STATS); + future.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } + return CompletableFuture.completedFuture(null); + }).thenCompose(__ -> getPartitionedTopicMetadataAsync(topicName, authoritative, false)).thenAccept(partitionMetadata -> { if (partitionMetadata.partitions == 0) { asyncResponse.resume(new RestException(Status.NOT_FOUND, @@ -1496,6 +1443,14 @@ protected void internalGetPartitionedStats(AsyncResponse asyncResponse, boolean } PartitionedTopicStatsImpl stats = new PartitionedTopicStatsImpl(partitionMetadata); List> topicStatsFutureList = new ArrayList<>(partitionMetadata.partitions); + org.apache.pulsar.client.admin.GetStatsOptions statsOptions = + new org.apache.pulsar.client.admin.GetStatsOptions( + getStatsOptions.isGetPreciseBacklog(), + getStatsOptions.isSubscriptionBacklogSize(), + getStatsOptions.isGetEarliestTimeInBacklog(), + getStatsOptions.isExcludePublishers(), + getStatsOptions.isExcludeConsumers() + ); for (int i = 0; i < partitionMetadata.partitions; i++) { TopicName partition = topicName.getPartition(i); topicStatsFutureList.add( @@ -1505,13 +1460,11 @@ protected void internalGetPartitionedStats(AsyncResponse asyncResponse, boolean if (owned) { return getTopicReferenceAsync(partition) .thenApply(ref -> - ref.getStats(getPreciseBacklog, subscriptionBacklogSize, - getEarliestTimeInBacklog)); + ref.getStats(getStatsOptions)); } else { try { return pulsar().getAdminClient().topics().getStatsAsync( - partition.toString(), getPreciseBacklog, subscriptionBacklogSize, - getEarliestTimeInBacklog); + partition.toString(), statsOptions); } catch (PulsarServerException e) { return FutureUtil.failedFuture(e); } @@ -1558,7 +1511,7 @@ protected void internalGetPartitionedStats(AsyncResponse asyncResponse, boolean }); }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get partitioned internal stats for {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -1567,14 +1520,15 @@ protected void internalGetPartitionedStats(AsyncResponse asyncResponse, boolean } protected void internalGetPartitionedStatsInternal(AsyncResponse asyncResponse, boolean authoritative) { - CompletableFuture future; - if (topicName.isGlobal()) { - future = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - future = CompletableFuture.completedFuture(null); - } - future.thenCompose(__ -> getPartitionedTopicMetadataAsync(topicName, authoritative, false)) - .thenAccept(partitionMetadata -> { + CompletableFuture future = validateTopicOperationAsync(topicName, TopicOperation.GET_STATS); + future.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenCompose(__ -> getPartitionedTopicMetadataAsync(topicName, authoritative, false)) + .thenAccept(partitionMetadata -> { if (partitionMetadata.partitions == 0) { asyncResponse.resume(new RestException(Status.NOT_FOUND, getPartitionedTopicNotFoundErrorMessage(topicName.toString()))); @@ -1613,7 +1567,7 @@ protected void internalGetPartitionedStatsInternal(AsyncResponse asyncResponse, }); }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get partitioned internal stats for {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -1624,14 +1578,14 @@ protected void internalGetPartitionedStatsInternal(AsyncResponse asyncResponse, protected CompletableFuture internalDeleteSubscriptionAsync(String subName, boolean authoritative, boolean force) { - CompletableFuture future; - if (topicName.isGlobal()) { - future = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - future = CompletableFuture.completedFuture(null); - } - + CompletableFuture future = validateTopicOperationAsync(topicName, TopicOperation.UNSUBSCRIBE, subName); return future.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenCompose(__ -> { if (topicName.isPartitioned()) { return internalDeleteSubscriptionForNonPartitionedTopicAsync(subName, authoritative, force); } else { @@ -1648,7 +1602,7 @@ protected CompletableFuture internalDeleteSubscriptionAsync(String subName for (int i = 0; i < partitionMetadata.partitions; i++) { TopicName topicNamePartition = topicName.getPartition(i); futures.add(adminClient.topics() - .deleteSubscriptionAsync(topicNamePartition.toString(), subName, false)); + .deleteSubscriptionAsync(topicNamePartition.toString(), subName, force)); } return FutureUtil.waitForAll(futures).handle((result, exception) -> { @@ -1667,18 +1621,17 @@ protected CompletableFuture internalDeleteSubscriptionAsync(String subName return null; }); } - return internalDeleteSubscriptionForNonPartitionedTopicAsync(subName, authoritative, - force); + return internalDeleteSubscriptionForNonPartitionedTopicAsync(subName, authoritative, force); }); } }); } + // Note: this method expects the caller to check authorization private CompletableFuture internalDeleteSubscriptionForNonPartitionedTopicAsync(String subName, boolean authoritative, boolean force) { return validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose((__) -> validateTopicOperationAsync(topicName, TopicOperation.UNSUBSCRIBE, subName)) .thenCompose(__ -> getTopicReferenceAsync(topicName)) .thenCompose((topic) -> { Subscription sub = topic.getSubscription(subName); @@ -1739,7 +1692,7 @@ private void internalAnalyzeSubscriptionBacklogForNonPartitionedTopic(AsyncRespo }).exceptionally(ex -> { Throwable cause = ex.getCause(); // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to analyze subscription backlog {} {}", clientAppId(), topicName, subName, cause); } @@ -1766,7 +1719,7 @@ private void internalUpdateSubscriptionPropertiesForNonPartitionedTopic(AsyncRes }).exceptionally(ex -> { Throwable cause = ex.getCause(); // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to update subscription {} {}", clientAppId(), topicName, subName, cause); } asyncResponse.resume(new RestException(cause)); @@ -1778,7 +1731,6 @@ private void internalGetSubscriptionPropertiesForNonPartitionedTopic(AsyncRespon String subName, boolean authoritative) { validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.CONSUME, subName)) .thenCompose(__ -> getTopicReferenceAsync(topicName)) .thenApply((Topic topic) -> { Subscription sub = topic.getSubscription(subName); @@ -1795,7 +1747,7 @@ private void internalGetSubscriptionPropertiesForNonPartitionedTopic(AsyncRespon }).exceptionally(ex -> { Throwable cause = ex.getCause(); // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to update subscription {} {}", clientAppId(), topicName, subName, cause); } asyncResponse.resume(new RestException(cause)); @@ -1803,74 +1755,67 @@ private void internalGetSubscriptionPropertiesForNonPartitionedTopic(AsyncRespon }); } - protected void internalDeleteSubscriptionForcefully(AsyncResponse asyncResponse, - String subName, boolean authoritative) { - CompletableFuture future; - if (topicName.isGlobal()) { - future = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - future = CompletableFuture.completedFuture(null); - } - - future.thenAccept(__ -> { + protected void internalSkipAllMessages(AsyncResponse asyncResponse, String subName, boolean authoritative) { + CompletableFuture future = validateTopicOperationAsync(topicName, TopicOperation.SKIP, subName); + future.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) + .thenCompose(__ -> { + // If the topic name is a partition name, no need to get partition topic metadata again if (topicName.isPartitioned()) { - internalDeleteSubscriptionForNonPartitionedTopicForcefully(asyncResponse, subName, authoritative); + return internalSkipAllMessagesForNonPartitionedTopicAsync(asyncResponse, subName); } else { - getPartitionedTopicMetadataAsync(topicName, - authoritative, false).thenAccept(partitionMetadata -> { + return getPartitionedTopicMetadataAsync(topicName, + authoritative, false).thenCompose(partitionMetadata -> { if (partitionMetadata.partitions > 0) { final List> futures = new ArrayList<>(); for (int i = 0; i < partitionMetadata.partitions; i++) { TopicName topicNamePartition = topicName.getPartition(i); try { - futures.add(pulsar().getAdminClient().topics() - .deleteSubscriptionAsync(topicNamePartition.toString(), subName, true)); + futures.add(pulsar() + .getAdminClient() + .topics() + .skipAllMessagesAsync(topicNamePartition.toString(), + subName)); } catch (Exception e) { - log.error("[{}] Failed to delete subscription forcefully {} {}", - clientAppId(), topicNamePartition, subName, - e); + log.error("[{}] Failed to skip all messages {} {}", + clientAppId(), topicNamePartition, subName, e); asyncResponse.resume(new RestException(e)); - return; + return CompletableFuture.completedFuture(null); } } - FutureUtil.waitForAll(futures).handle((result, exception) -> { + return FutureUtil.waitForAll(futures).handle((result, exception) -> { if (exception != null) { Throwable t = exception.getCause(); if (t instanceof NotFoundException) { - asyncResponse.resume(new RestException(Status.NOT_FOUND, - getSubNotFoundErrorMessage(topicName.toString(), subName))); - return null; + asyncResponse.resume( + new RestException(Status.NOT_FOUND, + getSubNotFoundErrorMessage(topicName.toString(), subName))); } else { - log.error("[{}] Failed to delete subscription forcefully {} {}", - clientAppId(), topicName, subName, t); + log.error("[{}] Failed to skip all messages {} {}", + clientAppId(), topicName, subName, t); asyncResponse.resume(new RestException(t)); - return null; } + return null; } - asyncResponse.resume(Response.noContent().build()); return null; }); } else { - internalDeleteSubscriptionForNonPartitionedTopicForcefully(asyncResponse, subName, - authoritative); - } - }).exceptionally(ex -> { - // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { - log.error("[{}] Failed to delete subscription forcefully {} from topic {}", - clientAppId(), subName, topicName, ex); + return internalSkipAllMessagesForNonPartitionedTopicAsync(asyncResponse, subName); } - resumeAsyncResponseExceptionally(asyncResponse, ex); - return null; }); } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { - log.error("[{}] Failed to delete subscription {} from topic {}", + if (isNot307And404Exception(ex)) { + log.error("[{}] Failed to skip all messages for subscription {} on topic {}", clientAppId(), subName, topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -1878,109 +1823,14 @@ protected void internalDeleteSubscriptionForcefully(AsyncResponse asyncResponse, }); } - private void internalDeleteSubscriptionForNonPartitionedTopicForcefully(AsyncResponse asyncResponse, - String subName, boolean authoritative) { - validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.UNSUBSCRIBE, subName)) - .thenCompose(__ -> getTopicReferenceAsync(topicName)) - .thenCompose(topic -> { - Subscription sub = topic.getSubscription(subName); - if (sub == null) { - throw new RestException(Status.NOT_FOUND, - getSubNotFoundErrorMessage(topicName.toString(), subName)); - } - return sub.deleteForcefully(); - }).thenRun(() -> { - log.info("[{}][{}] Deleted subscription forcefully {}", clientAppId(), topicName, subName); - asyncResponse.resume(Response.noContent().build()); - }).exceptionally(ex -> { - // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { - log.error("[{}] Failed to delete subscription forcefully {} {}", - clientAppId(), topicName, subName, ex); - } - resumeAsyncResponseExceptionally(asyncResponse, ex); - return null; - }); - } - - protected void internalSkipAllMessages(AsyncResponse asyncResponse, String subName, boolean authoritative) { - CompletableFuture future; - if (topicName.isGlobal()) { - future = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - future = CompletableFuture.completedFuture(null); - } - - future.thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) - .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.SKIP, subName)) - .thenCompose(__ -> { - // If the topic name is a partition name, no need to get partition topic metadata again - if (topicName.isPartitioned()) { - return internalSkipAllMessagesForNonPartitionedTopicAsync(asyncResponse, subName); - } else { - return getPartitionedTopicMetadataAsync(topicName, - authoritative, false).thenCompose(partitionMetadata -> { - if (partitionMetadata.partitions > 0) { - final List> futures = new ArrayList<>(); - - for (int i = 0; i < partitionMetadata.partitions; i++) { - TopicName topicNamePartition = topicName.getPartition(i); - try { - futures.add(pulsar() - .getAdminClient() - .topics() - .skipAllMessagesAsync(topicNamePartition.toString(), - subName)); - } catch (Exception e) { - log.error("[{}] Failed to skip all messages {} {}", - clientAppId(), topicNamePartition, subName, e); - asyncResponse.resume(new RestException(e)); - return CompletableFuture.completedFuture(null); - } - } - - return FutureUtil.waitForAll(futures).handle((result, exception) -> { - if (exception != null) { - Throwable t = exception.getCause(); - if (t instanceof NotFoundException) { - asyncResponse.resume( - new RestException(Status.NOT_FOUND, - getSubNotFoundErrorMessage(topicName.toString(), subName))); - } else { - log.error("[{}] Failed to skip all messages {} {}", - clientAppId(), topicName, subName, t); - asyncResponse.resume(new RestException(t)); - } - return null; - } - asyncResponse.resume(Response.noContent().build()); - return null; - }); - } else { - return internalSkipAllMessagesForNonPartitionedTopicAsync(asyncResponse, subName); - } - }); - } - }).exceptionally(ex -> { - // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { - log.error("[{}] Failed to skip all messages for subscription {} on topic {}", - clientAppId(), subName, topicName, ex); - } - resumeAsyncResponseExceptionally(asyncResponse, ex); - return null; - }); - } - - private CompletableFuture internalSkipAllMessagesForNonPartitionedTopicAsync(AsyncResponse asyncResponse, - String subName) { - return getTopicReferenceAsync(topicName).thenCompose(t -> { - PersistentTopic topic = (PersistentTopic) t; - BiConsumer biConsumer = (v, ex) -> { - if (ex != null) { - asyncResponse.resume(new RestException(ex)); - log.error("[{}] Failed to skip all messages {} {}", + private CompletableFuture internalSkipAllMessagesForNonPartitionedTopicAsync(AsyncResponse asyncResponse, + String subName) { + return getTopicReferenceAsync(topicName).thenCompose(t -> { + PersistentTopic topic = (PersistentTopic) t; + BiConsumer biConsumer = (v, ex) -> { + if (ex != null) { + asyncResponse.resume(new RestException(ex)); + log.error("[{}] Failed to skip all messages {} {}", clientAppId(), topicName, subName, ex); } else { asyncResponse.resume(Response.noContent().build()); @@ -2008,7 +1858,7 @@ private CompletableFuture internalSkipAllMessagesForNonPartitionedTopicAsy } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to skip all messages for subscription {} on topic {}", clientAppId(), subName, topicName, ex); } @@ -2019,130 +1869,129 @@ private CompletableFuture internalSkipAllMessagesForNonPartitionedTopicAsy protected void internalSkipMessages(AsyncResponse asyncResponse, String subName, int numMessages, boolean authoritative) { - CompletableFuture future; - if (topicName.isGlobal()) { - future = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - future = CompletableFuture.completedFuture(null); - } - future.thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) - .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.SKIP, subName)) - .thenCompose(__ -> getPartitionedTopicMetadataAsync(topicName, authoritative, false) - .thenCompose(partitionMetadata -> { - if (partitionMetadata.partitions > 0) { - String msg = "Skip messages on a partitioned topic is not allowed"; - log.warn("[{}] {} {} {}", clientAppId(), msg, topicName, subName); - throw new RestException(Status.METHOD_NOT_ALLOWED, msg); + CompletableFuture future = validateTopicOperationAsync(topicName, TopicOperation.SKIP, subName); + future.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) + .thenCompose(__ -> getPartitionedTopicMetadataAsync(topicName, authoritative, false)) + .thenCompose(partitionMetadata -> { + if (partitionMetadata.partitions > 0) { + String msg = "Skip messages on a partitioned topic is not allowed"; + log.warn("[{}] {} {} {}", clientAppId(), msg, topicName, subName); + throw new RestException(Status.METHOD_NOT_ALLOWED, msg); + } + return getTopicReferenceAsync(topicName).thenCompose(t -> { + PersistentTopic topic = (PersistentTopic) t; + if (topic == null) { + throw new RestException(new RestException(Status.NOT_FOUND, + getTopicNotFoundErrorMessage(topicName.toString()))); + } + if (subName.startsWith(topic.getReplicatorPrefix())) { + String remoteCluster = PersistentReplicator.getRemoteCluster(subName); + PersistentReplicator repl = + (PersistentReplicator) topic.getPersistentReplicator(remoteCluster); + if (repl == null) { + return FutureUtil.failedFuture( + new RestException(Status.NOT_FOUND, "Replicator not found")); + } + return repl.skipMessages(numMessages).thenAccept(unused -> { + log.info("[{}] Skipped {} messages on {} {}", clientAppId(), numMessages, + topicName, subName); + asyncResponse.resume(Response.noContent().build()); } - return getTopicReferenceAsync(topicName).thenCompose(t -> { - PersistentTopic topic = (PersistentTopic) t; - if (topic == null) { - throw new RestException(new RestException(Status.NOT_FOUND, - getTopicNotFoundErrorMessage(topicName.toString()))); - } - if (subName.startsWith(topic.getReplicatorPrefix())) { - String remoteCluster = PersistentReplicator.getRemoteCluster(subName); - PersistentReplicator repl = - (PersistentReplicator) topic.getPersistentReplicator(remoteCluster); - if (repl == null) { - return FutureUtil.failedFuture( - new RestException(Status.NOT_FOUND, "Replicator not found")); - } - return repl.skipMessages(numMessages).thenAccept(unused -> { - log.info("[{}] Skipped {} messages on {} {}", clientAppId(), numMessages, - topicName, subName); - asyncResponse.resume(Response.noContent().build()); - } - ); - } else { - PersistentSubscription sub = topic.getSubscription(subName); - if (sub == null) { - return FutureUtil.failedFuture( - new RestException(Status.NOT_FOUND, - getSubNotFoundErrorMessage(topicName.toString(), subName))); - } - return sub.skipMessages(numMessages).thenAccept(unused -> { - log.info("[{}] Skipped {} messages on {} {}", clientAppId(), numMessages, - topicName, subName); - asyncResponse.resume(Response.noContent().build()); - } - ); - } - }); - }) - ).exceptionally(ex -> { - // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { - log.error("[{}] Failed to skip {} messages {} {}", clientAppId(), numMessages, topicName, - subName, ex); - } - resumeAsyncResponseExceptionally(asyncResponse, ex); - return null; - }); + ); + } else { + PersistentSubscription sub = topic.getSubscription(subName); + if (sub == null) { + return FutureUtil.failedFuture( + new RestException(Status.NOT_FOUND, + getSubNotFoundErrorMessage(topicName.toString(), subName))); + } + return sub.skipMessages(numMessages).thenAccept(unused -> { + log.info("[{}] Skipped {} messages on {} {}", clientAppId(), numMessages, + topicName, subName); + asyncResponse.resume(Response.noContent().build()); + } + ); + } + }); + } + ).exceptionally(ex -> { + // If the exception is not redirect exception we need to log it. + if (isNot307And404Exception(ex)) { + log.error("[{}] Failed to skip {} messages {} {}", clientAppId(), numMessages, topicName, + subName, ex); + } + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); } protected void internalExpireMessagesForAllSubscriptions(AsyncResponse asyncResponse, int expireTimeInSeconds, boolean authoritative) { - CompletableFuture future; - if (topicName.isGlobal()) { - future = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - future = CompletableFuture.completedFuture(null); - } - future.thenCompose(__ -> - getPartitionedTopicMetadataAsync(topicName, authoritative, false) - .thenAccept(partitionMetadata -> { - if (topicName.isPartitioned()) { - internalExpireMessagesForAllSubscriptionsForNonPartitionedTopic(asyncResponse, - partitionMetadata, expireTimeInSeconds, authoritative); - } else { - if (partitionMetadata.partitions > 0) { - final List> futures = new ArrayList<>(partitionMetadata.partitions); - - // expire messages for each partition topic - for (int i = 0; i < partitionMetadata.partitions; i++) { - TopicName topicNamePartition = topicName.getPartition(i); - try { - futures.add(pulsar() - .getAdminClient() - .topics() - .expireMessagesForAllSubscriptionsAsync( - topicNamePartition.toString(), expireTimeInSeconds)); - } catch (Exception e) { - log.error("[{}] Failed to expire messages up to {} on {}", - clientAppId(), expireTimeInSeconds, - topicNamePartition, e); - asyncResponse.resume(new RestException(e)); - return; - } - } + CompletableFuture future = validateTopicOperationAsync(topicName, TopicOperation.EXPIRE_MESSAGES); + future.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenCompose(__ -> getPartitionedTopicMetadataAsync(topicName, authoritative, false)) + .thenAccept(partitionMetadata -> { + if (topicName.isPartitioned()) { + internalExpireMessagesForAllSubscriptionsForNonPartitionedTopic(asyncResponse, + partitionMetadata, expireTimeInSeconds, authoritative); + } else { + if (partitionMetadata.partitions > 0) { + final List> futures = new ArrayList<>(partitionMetadata.partitions); - FutureUtil.waitForAll(futures).handle((result, exception) -> { - if (exception != null) { - Throwable t = FutureUtil.unwrapCompletionException(exception); - if (t instanceof PulsarAdminException) { - log.warn("[{}] Failed to expire messages up to {} on {}: {}", clientAppId(), - expireTimeInSeconds, topicName, t.toString()); - } else { - log.error("[{}] Failed to expire messages up to {} on {}", clientAppId(), - expireTimeInSeconds, topicName, t); - } - resumeAsyncResponseExceptionally(asyncResponse, t); - return null; - } - asyncResponse.resume(Response.noContent().build()); - return null; - }); - } else { - internalExpireMessagesForAllSubscriptionsForNonPartitionedTopic(asyncResponse, - partitionMetadata, expireTimeInSeconds, authoritative); + // expire messages for each partition topic + for (int i = 0; i < partitionMetadata.partitions; i++) { + TopicName topicNamePartition = topicName.getPartition(i); + try { + futures.add(pulsar() + .getAdminClient() + .topics() + .expireMessagesForAllSubscriptionsAsync( + topicNamePartition.toString(), expireTimeInSeconds)); + } catch (Exception e) { + log.error("[{}] Failed to expire messages up to {} on {}", + clientAppId(), expireTimeInSeconds, + topicNamePartition, e); + asyncResponse.resume(new RestException(e)); + return; } } + + FutureUtil.waitForAll(futures).handle((result, exception) -> { + if (exception != null) { + Throwable t = FutureUtil.unwrapCompletionException(exception); + if (t instanceof PulsarAdminException) { + log.warn("[{}] Failed to expire messages up to {} on {}: {}", clientAppId(), + expireTimeInSeconds, topicName, t.toString()); + } else { + log.error("[{}] Failed to expire messages up to {} on {}", clientAppId(), + expireTimeInSeconds, topicName, t); + } + resumeAsyncResponseExceptionally(asyncResponse, t); + return null; + } + asyncResponse.resume(Response.noContent().build()); + return null; + }); + } else { + internalExpireMessagesForAllSubscriptionsForNonPartitionedTopic(asyncResponse, + partitionMetadata, expireTimeInSeconds, authoritative); } - ) + } + } ).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to expire messages for all subscription on topic {}", clientAppId(), topicName, ex); } @@ -2159,7 +2008,6 @@ private void internalExpireMessagesForAllSubscriptionsForNonPartitionedTopic(Asy boolean authoritative) { // validate ownership and redirect if current broker is not owner validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.EXPIRE_MESSAGES)) .thenCompose(__ -> getTopicReferenceAsync(topicName).thenAccept(t -> { if (t == null) { resumeAsyncResponseExceptionally(asyncResponse, new RestException(Status.NOT_FOUND, @@ -2175,10 +2023,9 @@ private void internalExpireMessagesForAllSubscriptionsForNonPartitionedTopic(Asy final List> futures = new ArrayList<>((int) topic.getReplicators().size()); List subNames = - new ArrayList<>((int) topic.getReplicators().size() - + (int) topic.getSubscriptions().size()); - subNames.addAll(topic.getReplicators().keys()); - subNames.addAll(topic.getSubscriptions().keys()); + new ArrayList<>((int) topic.getSubscriptions().size()); + subNames.addAll(topic.getSubscriptions().keySet().stream().filter( + subName -> !subName.equals(Compactor.COMPACTION_SUBSCRIPTION)).toList()); for (int i = 0; i < subNames.size(); i++) { try { futures.add(internalExpireMessagesByTimestampForSinglePartitionAsync(partitionMetadata, @@ -2210,7 +2057,7 @@ private void internalExpireMessagesForAllSubscriptionsForNonPartitionedTopic(Asy }) ).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to expire messages for all subscription up to {} on {}", clientAppId(), expireTimeInSeconds, topicName, ex); } @@ -2221,23 +2068,22 @@ private void internalExpireMessagesForAllSubscriptionsForNonPartitionedTopic(Asy protected CompletableFuture internalResetCursorAsync(String subName, long timestamp, boolean authoritative) { - CompletableFuture future; - if (topicName.isGlobal()) { - future = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - future = CompletableFuture.completedFuture(null); - } - return future - .thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) - .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.RESET_CURSOR, subName)) - .thenCompose(__ -> { - // If the topic name is a partition name, no need to get partition topic metadata again - if (topicName.isPartitioned()) { - return internalResetCursorForNonPartitionedTopic(subName, timestamp, authoritative); - } else { - return internalResetCursorForPartitionedTopic(subName, timestamp, authoritative); - } - }); + CompletableFuture future = validateTopicOperationAsync(topicName, TopicOperation.RESET_CURSOR, subName); + return future.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) + .thenCompose(__ -> { + // If the topic name is a partition name, no need to get partition topic metadata again + if (topicName.isPartitioned()) { + return internalResetCursorForNonPartitionedTopic(subName, timestamp, authoritative); + } else { + return internalResetCursorForPartitionedTopic(subName, timestamp, authoritative); + } + }); } private CompletableFuture internalResetCursorForPartitionedTopic(String subName, long timestamp, @@ -2326,13 +2172,14 @@ private CompletableFuture internalResetCursorForNonPartitionedTopic(String protected void internalCreateSubscription(AsyncResponse asyncResponse, String subscriptionName, MessageIdImpl messageId, boolean authoritative, boolean replicated, Map properties) { - CompletableFuture ret; - if (topicName.isGlobal()) { - ret = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - ret = CompletableFuture.completedFuture(null); - } - ret.thenAccept(__ -> { + CompletableFuture ret = validateTopicOperationAsync(topicName, TopicOperation.SUBSCRIBE, + subscriptionName); + ret.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } + return CompletableFuture.completedFuture(null); + }).thenAccept(__ -> { final MessageIdImpl targetMessageId = messageId == null ? (MessageIdImpl) MessageId.latest : messageId; log.info("[{}][{}] Creating subscription {} at message id {} with properties {}", clientAppId(), topicName, subscriptionName, targetMessageId, properties); @@ -2417,7 +2264,7 @@ protected void internalCreateSubscription(AsyncResponse asyncResponse, String su })).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to create subscription {} on topic {}", clientAppId(), subscriptionName, topicName, ex); } @@ -2427,7 +2274,7 @@ protected void internalCreateSubscription(AsyncResponse asyncResponse, String su } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to create subscription {} on topic {}", clientAppId(), subscriptionName, topicName, ex); } @@ -2463,7 +2310,7 @@ private void internalCreateSubscriptionForNonPartitionedTopic( // Mark the cursor as "inactive" as it was created without a real consumer connected ((PersistentSubscription) subscription).deactivateCursor(); return subscription.resetCursor( - PositionImpl.get(targetMessageId.getLedgerId(), targetMessageId.getEntryId())); + PositionFactory.create(targetMessageId.getLedgerId(), targetMessageId.getEntryId())); }).thenRun(() -> { log.info("[{}][{}] Successfully created subscription {} at message id {}", clientAppId(), topicName, subscriptionName, targetMessageId); @@ -2491,14 +2338,13 @@ private void internalCreateSubscriptionForNonPartitionedTopic( protected void internalUpdateSubscriptionProperties(AsyncResponse asyncResponse, String subName, Map subscriptionProperties, boolean authoritative) { - CompletableFuture future; - if (topicName.isGlobal()) { - future = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - future = CompletableFuture.completedFuture(null); - } - - future.thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)).thenAccept(__ -> { + CompletableFuture future = validateTopicOperationAsync(topicName, TopicOperation.SUBSCRIBE, subName); + future.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } + return CompletableFuture.completedFuture(null); + }).thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)).thenAccept(__ -> { if (topicName.isPartitioned()) { internalUpdateSubscriptionPropertiesForNonPartitionedTopic(asyncResponse, subName, subscriptionProperties, authoritative); @@ -2558,7 +2404,7 @@ protected void internalUpdateSubscriptionProperties(AsyncResponse asyncResponse, } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to update subscription {} from topic {}", clientAppId(), subName, topicName, ex); } @@ -2570,14 +2416,13 @@ protected void internalUpdateSubscriptionProperties(AsyncResponse asyncResponse, protected void internalAnalyzeSubscriptionBacklog(AsyncResponse asyncResponse, String subName, Optional position, boolean authoritative) { - CompletableFuture future; - if (topicName.isGlobal()) { - future = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - future = CompletableFuture.completedFuture(null); - } - - future.thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) + CompletableFuture future = validateTopicOperationAsync(topicName, TopicOperation.CONSUME, subName); + future.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } + return CompletableFuture.completedFuture(null); + }).thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) .thenCompose(__ -> { if (topicName.isPartitioned()) { return CompletableFuture.completedFuture(null); @@ -2598,7 +2443,7 @@ protected void internalAnalyzeSubscriptionBacklog(AsyncResponse asyncResponse, S }) .exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to analyze back log of subscription {} from topic {}", clientAppId(), subName, topicName, ex); } @@ -2609,14 +2454,13 @@ protected void internalAnalyzeSubscriptionBacklog(AsyncResponse asyncResponse, S protected void internalGetSubscriptionProperties(AsyncResponse asyncResponse, String subName, boolean authoritative) { - CompletableFuture future; - if (topicName.isGlobal()) { - future = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - future = CompletableFuture.completedFuture(null); - } - - future.thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)).thenAccept(__ -> { + CompletableFuture future = validateTopicOperationAsync(topicName, TopicOperation.CONSUME, subName); + future.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } + return CompletableFuture.completedFuture(null); + }).thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)).thenAccept(__ -> { if (topicName.isPartitioned()) { internalGetSubscriptionPropertiesForNonPartitionedTopic(asyncResponse, subName, authoritative); @@ -2683,7 +2527,7 @@ protected void internalGetSubscriptionProperties(AsyncResponse asyncResponse, St } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to update subscription {} from topic {}", clientAppId(), subName, topicName, ex); } @@ -2694,91 +2538,80 @@ protected void internalGetSubscriptionProperties(AsyncResponse asyncResponse, St protected void internalResetCursorOnPosition(AsyncResponse asyncResponse, String subName, boolean authoritative, MessageIdImpl messageId, boolean isExcluded, int batchIndex) { - CompletableFuture ret; - // If the topic name is a partition name, no need to get partition topic metadata again - if (!topicName.isPartitioned()) { - ret = getPartitionedTopicMetadataAsync(topicName, authoritative, false) - .thenCompose(topicMetadata -> { - if (topicMetadata.partitions > 0) { - log.warn("[{}] Not supported operation on partitioned-topic {} {}", - clientAppId(), topicName, subName); - throw new CompletionException(new RestException(Status.METHOD_NOT_ALLOWED, - "Reset-cursor at position is not allowed for partitioned-topic")); - } - return CompletableFuture.completedFuture(null); - }); - } else { - ret = CompletableFuture.completedFuture(null); - } - - CompletableFuture future; - if (topicName.isGlobal()) { - future = ret.thenCompose(__ -> validateGlobalNamespaceOwnershipAsync(namespaceName)); - } else { - future = CompletableFuture.completedFuture(null); - } - future.thenAccept(__ -> { + CompletableFuture ret = validateTopicOperationAsync(topicName, TopicOperation.RESET_CURSOR, subName); + ret.thenCompose(__ -> { + // If the topic name is a partition name, no need to get partition topic metadata again + if (!topicName.isPartitioned()) { + return getPartitionedTopicMetadataAsync(topicName, authoritative, false) + .thenCompose(topicMetadata -> { + if (topicMetadata.partitions > 0) { + log.warn("[{}] Not supported operation on partitioned-topic {} {}", + clientAppId(), topicName, subName); + throw new CompletionException(new RestException(Status.METHOD_NOT_ALLOWED, + "Reset-cursor at position is not allowed for partitioned-topic")); + } + return CompletableFuture.completedFuture(null); + }); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenCompose(__ -> { log.info("[{}][{}] received reset cursor on subscription {} to position {}", clientAppId(), topicName, subName, messageId); - validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(ignore -> - validateTopicOperationAsync(topicName, TopicOperation.RESET_CURSOR, subName)) - .thenCompose(ignore -> getTopicReferenceAsync(topicName)) - .thenAccept(topic -> { - if (topic == null) { - asyncResponse.resume(new RestException(Status.NOT_FOUND, - getTopicNotFoundErrorMessage(topicName.toString()))); - return; - } - PersistentSubscription sub = ((PersistentTopic) topic).getSubscription(subName); - if (sub == null) { - asyncResponse.resume(new RestException(Status.NOT_FOUND, - getSubNotFoundErrorMessage(topicName.toString(), subName))); - return; - } - CompletableFuture batchSizeFuture = new CompletableFuture<>(); - getEntryBatchSize(batchSizeFuture, (PersistentTopic) topic, messageId, batchIndex); - batchSizeFuture.thenAccept(bi -> { - PositionImpl seekPosition = calculatePositionAckSet(isExcluded, bi, batchIndex, - messageId); - sub.resetCursor(seekPosition).thenRun(() -> { - log.info("[{}][{}] successfully reset cursor on subscription {}" - + " to position {}", clientAppId(), - topicName, subName, messageId); - asyncResponse.resume(Response.noContent().build()); - }).exceptionally(ex -> { - Throwable t = (ex instanceof CompletionException ? ex.getCause() : ex); - log.warn("[{}][{}] Failed to reset cursor on subscription {}" - + " to position {}", clientAppId(), - topicName, subName, messageId, t); - if (t instanceof SubscriptionInvalidCursorPosition) { - asyncResponse.resume(new RestException(Status.PRECONDITION_FAILED, - "Unable to find position for position specified: " - + t.getMessage())); - } else if (t instanceof SubscriptionBusyException) { - asyncResponse.resume(new RestException(Status.PRECONDITION_FAILED, - "Failed for Subscription Busy: " + t.getMessage())); - } else { - resumeAsyncResponseExceptionally(asyncResponse, t); - } - return null; - }); - }).exceptionally(e -> { - asyncResponse.resume(e); - return null; - }); - }).exceptionally(ex -> { - // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { - log.warn("[{}][{}] Failed to reset cursor on subscription {} to position {}", - clientAppId(), topicName, subName, messageId, ex.getCause()); - } - resumeAsyncResponseExceptionally(asyncResponse, ex.getCause()); - return null; - }); + return validateTopicOwnershipAsync(topicName, authoritative); + }).thenCompose(ignore -> getTopicReferenceAsync(topicName)) + .thenAccept(topic -> { + if (topic == null) { + asyncResponse.resume(new RestException(Status.NOT_FOUND, + getTopicNotFoundErrorMessage(topicName.toString()))); + return; + } + PersistentSubscription sub = ((PersistentTopic) topic).getSubscription(subName); + if (sub == null) { + asyncResponse.resume(new RestException(Status.NOT_FOUND, + getSubNotFoundErrorMessage(topicName.toString(), subName))); + return; + } + CompletableFuture batchSizeFuture = new CompletableFuture<>(); + getEntryBatchSize(batchSizeFuture, (PersistentTopic) topic, messageId, batchIndex); + batchSizeFuture.thenAccept(bi -> { + Position seekPosition = calculatePositionAckSet(isExcluded, bi, batchIndex, + messageId); + sub.resetCursor(seekPosition).thenRun(() -> { + log.info("[{}][{}] successfully reset cursor on subscription {}" + + " to position {}", clientAppId(), + topicName, subName, messageId); + asyncResponse.resume(Response.noContent().build()); + }).exceptionally(ex -> { + Throwable t = (ex instanceof CompletionException ? ex.getCause() : ex); + log.warn("[{}][{}] Failed to reset cursor on subscription {}" + + " to position {}", clientAppId(), + topicName, subName, messageId, t); + if (t instanceof SubscriptionInvalidCursorPosition) { + asyncResponse.resume(new RestException(Status.PRECONDITION_FAILED, + "Unable to find position for position specified: " + + t.getMessage())); + } else if (t instanceof SubscriptionBusyException) { + asyncResponse.resume(new RestException(Status.PRECONDITION_FAILED, + "Failed for Subscription Busy: " + t.getMessage())); + } else { + resumeAsyncResponseExceptionally(asyncResponse, t); + } + return null; + }); + }).exceptionally(e -> { + asyncResponse.resume(e); + return null; + }); }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.warn("[{}][{}] Failed to reset cursor on subscription {} to position {}", clientAppId(), topicName, subName, messageId, ex.getCause()); } @@ -2791,8 +2624,8 @@ private void getEntryBatchSize(CompletableFuture batchSizeFuture, Persi MessageIdImpl messageId, int batchIndex) { if (batchIndex >= 0) { try { - ManagedLedgerImpl ledger = (ManagedLedgerImpl) topic.getManagedLedger(); - ledger.asyncReadEntry(new PositionImpl(messageId.getLedgerId(), + ManagedLedger ledger = topic.getManagedLedger(); + ledger.asyncReadEntry(PositionFactory.create(messageId.getLedgerId(), messageId.getEntryId()), new AsyncCallbacks.ReadEntryCallback() { @Override public void readEntryFailed(ManagedLedgerException exception, Object ctx) { @@ -2822,6 +2655,12 @@ public void readEntryComplete(Entry entry, Object ctx) { } } } + + @Override + public String toString() { + return String.format("Topic [%s] get entry batch size", + PersistentTopicsBase.this.topicName); + } }, null); } catch (NullPointerException npe) { batchSizeFuture.completeExceptionally(new RestException(Status.NOT_FOUND, "Message not found")); @@ -2835,9 +2674,9 @@ public void readEntryComplete(Entry entry, Object ctx) { } } - private PositionImpl calculatePositionAckSet(boolean isExcluded, int batchSize, + private Position calculatePositionAckSet(boolean isExcluded, int batchSize, int batchIndex, MessageIdImpl messageId) { - PositionImpl seekPosition; + Position seekPosition; if (batchSize > 0) { long[] ackSet; BitSetRecyclable bitSet = BitSetRecyclable.create(); @@ -2846,38 +2685,39 @@ private PositionImpl calculatePositionAckSet(boolean isExcluded, int batchSize, bitSet.clear(0, Math.max(batchIndex + 1, 0)); if (bitSet.length() > 0) { ackSet = bitSet.toLongArray(); - seekPosition = PositionImpl.get(messageId.getLedgerId(), + seekPosition = AckSetStateUtil.createPositionWithAckSet(messageId.getLedgerId(), messageId.getEntryId(), ackSet); } else { - seekPosition = PositionImpl.get(messageId.getLedgerId(), messageId.getEntryId()); + seekPosition = PositionFactory.create(messageId.getLedgerId(), messageId.getEntryId()); seekPosition = seekPosition.getNext(); } } else { if (batchIndex - 1 >= 0) { bitSet.clear(0, batchIndex); ackSet = bitSet.toLongArray(); - seekPosition = PositionImpl.get(messageId.getLedgerId(), + seekPosition = AckSetStateUtil.createPositionWithAckSet(messageId.getLedgerId(), messageId.getEntryId(), ackSet); } else { - seekPosition = PositionImpl.get(messageId.getLedgerId(), messageId.getEntryId()); + seekPosition = PositionFactory.create(messageId.getLedgerId(), messageId.getEntryId()); } } bitSet.recycle(); } else { - seekPosition = PositionImpl.get(messageId.getLedgerId(), messageId.getEntryId()); + seekPosition = PositionFactory.create(messageId.getLedgerId(), messageId.getEntryId()); seekPosition = isExcluded ? seekPosition.getNext() : seekPosition; } return seekPosition; } protected CompletableFuture internalGetMessageById(long ledgerId, long entryId, boolean authoritative) { - CompletableFuture future; - if (topicName.isGlobal()) { - future = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - future = CompletableFuture.completedFuture(null); - } + CompletableFuture future = validateTopicOperationAsync(topicName, TopicOperation.PEEK_MESSAGES); return future.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenCompose(__ -> { if (topicName.isPartitioned()) { return CompletableFuture.completedFuture(null); } else { @@ -2890,28 +2730,27 @@ protected CompletableFuture internalGetMessageById(long ledgerId, long "GetMessageById is not allowed on partitioned-topic"); } }); - } - }) - .thenCompose(ignore -> validateTopicOwnershipAsync(topicName, authoritative)) - .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.PEEK_MESSAGES)) + }).thenCompose(ignore -> validateTopicOwnershipAsync(topicName, authoritative)) .thenCompose(__ -> getTopicReferenceAsync(topicName)) .thenCompose(topic -> { CompletableFuture results = new CompletableFuture<>(); - ManagedLedgerImpl ledger = - (ManagedLedgerImpl) ((PersistentTopic) topic).getManagedLedger(); - ledger.asyncReadEntry(new PositionImpl(ledgerId, entryId), + ManagedLedger ledger = ((PersistentTopic) topic).getManagedLedger(); + ledger.asyncReadEntry(PositionFactory.create(ledgerId, entryId), new AsyncCallbacks.ReadEntryCallback() { @Override public void readEntryFailed(ManagedLedgerException exception, Object ctx) { + if (exception instanceof ManagedLedgerException.LedgerNotExistException) { + throw new RestException(Status.NOT_FOUND, "Message id not found"); + } throw new RestException(exception); } @Override public void readEntryComplete(Entry entry, Object ctx) { try { - results.complete(generateResponseWithEntry(entry)); + results.complete(generateResponseWithEntry(entry, (PersistentTopic) topic)); } catch (IOException exception) { throw new RestException(exception); } finally { @@ -2920,20 +2759,26 @@ public void readEntryComplete(Entry entry, Object ctx) { } } } + + @Override + public String toString() { + return String.format("Topic [%s] internal get message by id", + PersistentTopicsBase.this.topicName); + } }, null); return results; }); } protected CompletableFuture internalGetMessageIdByTimestampAsync(long timestamp, boolean authoritative) { - CompletableFuture future; - if (topicName.isGlobal()) { - future = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - future = CompletableFuture.completedFuture(null); - } - + CompletableFuture future = validateTopicOperationAsync(topicName, TopicOperation.PEEK_MESSAGES); return future.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenCompose(__ -> { if (topicName.isPartitioned()) { return CompletableFuture.completedFuture(null); } else { @@ -2947,7 +2792,6 @@ protected CompletableFuture internalGetMessageIdByTimestampAsync(long }); } }).thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) - .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.PEEK_MESSAGES)) .thenCompose(__ -> getTopicReferenceAsync(topicName)) .thenCompose(topic -> { if (!(topic instanceof PersistentTopic)) { @@ -2955,163 +2799,209 @@ protected CompletableFuture internalGetMessageIdByTimestampAsync(long throw new RestException(Status.METHOD_NOT_ALLOWED, "Get message ID by timestamp on a non-persistent topic is not allowed"); } - ManagedLedger ledger = ((PersistentTopic) topic).getManagedLedger(); - return ledger.asyncFindPosition(entry -> { + final PersistentTopic persistentTopic = (PersistentTopic) topic; + + return persistentTopic.getTopicCompactionService().readLastCompactedEntry().thenCompose(lastEntry -> { + if (lastEntry == null) { + return findMessageIdByPublishTime(timestamp, persistentTopic.getManagedLedger()); + } + MessageMetadata metadata; + Position position = lastEntry.getPosition(); try { - long entryTimestamp = Commands.getEntryTimestamp(entry.getDataBuffer()); - return MessageImpl.isEntryPublishedEarlierThan(entryTimestamp, timestamp); - } catch (Exception e) { - log.error("[{}] Error deserializing message for message position find", topicName, e); + metadata = Commands.parseMessageMetadata(lastEntry.getDataBuffer()); } finally { - entry.release(); + lastEntry.release(); } - return false; - }).thenApply(position -> { - if (position == null) { - return null; + if (timestamp == metadata.getPublishTime()) { + return CompletableFuture.completedFuture(new MessageIdImpl(position.getLedgerId(), + position.getEntryId(), topicName.getPartitionIndex())); + } else if (timestamp < metadata.getPublishTime()) { + return persistentTopic.getTopicCompactionService().findEntryByPublishTime(timestamp) + .thenApply(compactedEntry -> { + try { + return new MessageIdImpl(compactedEntry.getLedgerId(), + compactedEntry.getEntryId(), topicName.getPartitionIndex()); + } finally { + compactedEntry.release(); + } + }); } else { - return new MessageIdImpl(position.getLedgerId(), position.getEntryId(), - topicName.getPartitionIndex()); + return findMessageIdByPublishTime(timestamp, persistentTopic.getManagedLedger()); } }); }); } + private CompletableFuture findMessageIdByPublishTime(long timestamp, ManagedLedger managedLedger) { + return managedLedger.asyncFindPosition(entry -> { + try { + long entryTimestamp = Commands.getEntryTimestamp(entry.getDataBuffer()); + return MessageImpl.isEntryPublishedEarlierThan(entryTimestamp, timestamp); + } catch (Exception e) { + log.error("[{}] Error deserializing message for message position find", + topicName, + e); + } finally { + entry.release(); + } + return false; + }).thenApply(position -> { + if (position == null) { + return null; + } else { + return new MessageIdImpl(position.getLedgerId(), position.getEntryId(), + topicName.getPartitionIndex()); + } + }); + } + protected CompletableFuture internalPeekNthMessageAsync(String subName, int messagePosition, boolean authoritative) { - CompletableFuture ret; - // If the topic name is a partition name, no need to get partition topic metadata again - if (!topicName.isPartitioned()) { - ret = getPartitionedTopicMetadataAsync(topicName, authoritative, false) - .thenCompose(topicMetadata -> { - if (topicMetadata.partitions > 0) { - throw new RestException(Status.METHOD_NOT_ALLOWED, - "Peek messages on a partitioned topic is not allowed"); - } - return CompletableFuture.completedFuture(null); - }); - } else { - ret = CompletableFuture.completedFuture(null); - } - return ret.thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) - .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.PEEK_MESSAGES, subName)) - .thenCompose(__ -> getTopicReferenceAsync(topicName)) - .thenCompose(topic -> { - CompletableFuture entry; - if (!(topic instanceof PersistentTopic)) { - log.error("[{}] Not supported operation of non-persistent topic {} {}", clientAppId(), - topicName, subName); - throw new RestException(Status.METHOD_NOT_ALLOWED, - "Peek messages on a non-persistent topic is not allowed"); - } else { - if (subName.startsWith(((PersistentTopic) topic).getReplicatorPrefix())) { - PersistentReplicator repl = getReplicatorReference(subName, (PersistentTopic) topic); - entry = repl.peekNthMessage(messagePosition); - } else { - PersistentSubscription sub = - (PersistentSubscription) getSubscriptionReference(subName, (PersistentTopic) topic); - entry = sub.peekNthMessage(messagePosition); - } - } - return entry; - }).thenCompose(entry -> { - try { - Response response = generateResponseWithEntry(entry); - return CompletableFuture.completedFuture(response); - } catch (NullPointerException npe) { - throw new RestException(Status.NOT_FOUND, "Message not found"); - } catch (Exception exception) { - log.error("[{}] Failed to peek message at position {} from {} {}", clientAppId(), - messagePosition, topicName, subName, exception); - throw new RestException(exception); - } finally { - if (entry != null) { - entry.release(); - } - } - }); + CompletableFuture ret = validateTopicOperationAsync(topicName, TopicOperation.PEEK_MESSAGES, subName); + return ret.thenCompose(__ -> { + // If the topic name is a partition name, no need to get partition topic metadata again + if (!topicName.isPartitioned()) { + return getPartitionedTopicMetadataAsync(topicName, authoritative, false) + .thenCompose(topicMetadata -> { + if (topicMetadata.partitions > 0) { + throw new RestException(Status.METHOD_NOT_ALLOWED, + "Peek messages on a partitioned topic is not allowed"); + } + return CompletableFuture.completedFuture(null); + }); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) + .thenCompose(__ -> getTopicReferenceAsync(topicName)) + .thenCompose(topic -> { + CompletableFuture entry; + if (!(topic instanceof PersistentTopic)) { + log.error("[{}] Not supported operation of non-persistent topic {} {}", clientAppId(), + topicName, subName); + throw new RestException(Status.METHOD_NOT_ALLOWED, + "Peek messages on a non-persistent topic is not allowed"); + } else { + if (subName.startsWith(((PersistentTopic) topic).getReplicatorPrefix())) { + PersistentReplicator repl = getReplicatorReference(subName, (PersistentTopic) topic); + entry = repl.peekNthMessage(messagePosition); + } else { + PersistentSubscription sub = + (PersistentSubscription) getSubscriptionReference(subName, (PersistentTopic) topic); + entry = sub.peekNthMessage(messagePosition); + } + } + return entry.thenApply(e -> Pair.of(e, (PersistentTopic) topic)); + }).thenCompose(entryTopicPair -> { + Entry entry = entryTopicPair.getLeft(); + PersistentTopic persistentTopic = entryTopicPair.getRight(); + try { + Response response = generateResponseWithEntry(entry, persistentTopic); + return CompletableFuture.completedFuture(response); + } catch (NullPointerException npe) { + throw new RestException(Status.NOT_FOUND, "Message not found"); + } catch (Exception exception) { + log.error("[{}] Failed to peek message at position {} from {} {}", clientAppId(), + messagePosition, topicName, subName, exception); + throw new RestException(exception); + } finally { + if (entry != null) { + entry.release(); + } + } + }); } protected CompletableFuture internalExamineMessageAsync(String initialPosition, long messagePosition, boolean authoritative) { - CompletableFuture ret; - if (topicName.isGlobal()) { - ret = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - ret = CompletableFuture.completedFuture(null); - } - - ret = ret.thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)); long messagePositionLocal = messagePosition < 1 ? 1 : messagePosition; String initialPositionLocal = initialPosition == null ? "latest" : initialPosition; - if (!topicName.isPartitioned()) { - ret = ret.thenCompose(__ -> getPartitionedTopicMetadataAsync(topicName, authoritative, false)) - .thenCompose(partitionedTopicMetadata -> { - if (partitionedTopicMetadata.partitions > 0) { - throw new RestException(Status.METHOD_NOT_ALLOWED, - "Examine messages on a partitioned topic is not allowed, " - + "please try examine message on specific topic partition"); - } else { - return CompletableFuture.completedFuture(null); - } - }); - } - return ret.thenCompose(__ -> getTopicReferenceAsync(topicName)) - .thenCompose(topic -> { - if (!(topic instanceof PersistentTopic)) { - log.error("[{}] Not supported operation of non-persistent topic {} ", clientAppId(), topicName); - throw new RestException(Status.METHOD_NOT_ALLOWED, - "Examine messages on a non-persistent topic is not allowed"); - } - try { - PersistentTopic persistentTopic = (PersistentTopic) topic; - long totalMessage = persistentTopic.getNumberOfEntries(); - if (totalMessage <= 0) { - throw new RestException(Status.PRECONDITION_FAILED, - "Could not examine messages due to the total message is zero"); - } - PositionImpl startPosition = persistentTopic.getFirstPosition(); - - long messageToSkip = initialPositionLocal.equals("earliest") ? messagePositionLocal : - totalMessage - messagePositionLocal + 1; - CompletableFuture future = new CompletableFuture<>(); - PositionImpl readPosition = persistentTopic.getPositionAfterN(startPosition, messageToSkip); - persistentTopic.asyncReadEntry(readPosition, new AsyncCallbacks.ReadEntryCallback() { - @Override - public void readEntryComplete(Entry entry, Object ctx) { - future.complete(entry); + CompletableFuture ret = validateTopicOperationAsync(topicName, TopicOperation.PEEK_MESSAGES); + return ret.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) + .thenCompose(__ -> { + if (!topicName.isPartitioned()) { + return getPartitionedTopicMetadataAsync(topicName, authoritative, false) + .thenCompose(partitionedTopicMetadata -> { + if (partitionedTopicMetadata.partitions > 0) { + throw new RestException(Status.METHOD_NOT_ALLOWED, + "Examine messages on a partitioned topic is not allowed, " + + "please try examine message on specific topic partition"); + } else { + return CompletableFuture.completedFuture(null); } + }); + } + return CompletableFuture.completedFuture(null); + }).thenCompose(__ -> getTopicReferenceAsync(topicName)) + .thenCompose(topic -> { + if (!(topic instanceof PersistentTopic)) { + log.error("[{}] Not supported operation of non-persistent topic {} ", clientAppId(), topicName); + throw new RestException(Status.METHOD_NOT_ALLOWED, + "Examine messages on a non-persistent topic is not allowed"); + } + try { + PersistentTopic persistentTopic = (PersistentTopic) topic; + long totalMessage = persistentTopic.getNumberOfEntries(); + if (totalMessage <= 0) { + throw new RestException(Status.PRECONDITION_FAILED, + "Could not examine messages due to the total message is zero"); + } + Position startPosition = persistentTopic.getFirstPosition(); - @Override - public void readEntryFailed(ManagedLedgerException exception, Object ctx) { - future.completeExceptionally(exception); - } - }, null); - return future; - } catch (ManagedLedgerException exception) { - log.error("[{}] Failed to examine message at position {} from {} due to {}", clientAppId(), - messagePosition, - topicName, exception); - throw new RestException(exception); + long messageToSkip = initialPositionLocal.equals("earliest") ? messagePositionLocal : + totalMessage - messagePositionLocal + 1; + CompletableFuture future = new CompletableFuture<>(); + Position readPosition = persistentTopic.getPositionAfterN(startPosition, messageToSkip); + persistentTopic.asyncReadEntry(readPosition, new AsyncCallbacks.ReadEntryCallback() { + @Override + public void readEntryComplete(Entry entry, Object ctx) { + future.complete(entry); } - }).thenApply(entry -> { - try { - return generateResponseWithEntry(entry); - } catch (IOException exception) { - throw new RestException(exception); - } finally { - if (entry != null) { - entry.release(); - } + @Override + public void readEntryFailed(ManagedLedgerException exception, Object ctx) { + future.completeExceptionally(exception); } - }); + + @Override + public String toString() { + return String.format("Topic [%s] internal examine message async", + PersistentTopicsBase.this.topicName); + } + }, null); + return future.thenApply(entry -> Pair.of(entry, (PersistentTopic) topic)); + } catch (ManagedLedgerException exception) { + log.error("[{}] Failed to examine message at position {} from {} due to {}", clientAppId(), + messagePosition, + topicName, exception); + throw new RestException(exception); + } + }).thenApply(entryTopicPair -> { + Entry entry = entryTopicPair.getLeft(); + PersistentTopic persistentTopic = entryTopicPair.getRight(); + try { + return generateResponseWithEntry(entry, persistentTopic); + } catch (IOException exception) { + throw new RestException(exception); + } finally { + if (entry != null) { + entry.release(); + } + } + }); } - private Response generateResponseWithEntry(Entry entry) throws IOException { + private Response generateResponseWithEntry(Entry entry, PersistentTopic persistentTopic) throws IOException { checkNotNull(entry); - PositionImpl pos = (PositionImpl) entry.getPosition(); + Position pos = entry.getPosition(); ByteBuf metadataAndPayload = entry.getDataBuffer(); + boolean isEncrypted = false; long totalSize = metadataAndPayload.readableBytes(); BrokerEntryMetadata brokerEntryMetadata = Commands.peekBrokerEntryMetadataIfExist(metadataAndPayload); @@ -3119,9 +3009,11 @@ private Response generateResponseWithEntry(Entry entry) throws IOException { ResponseBuilder responseBuilder = Response.ok(); responseBuilder.header("X-Pulsar-Message-ID", pos.toString()); - for (KeyValue keyValue : metadata.getPropertiesList()) { - responseBuilder.header("X-Pulsar-PROPERTY-" + keyValue.getKey(), keyValue.getValue()); - } + + Map properties = metadata.getPropertiesList().stream() + .collect(Collectors.toMap(KeyValue::getKey, KeyValue::getValue, (v1, v2) -> v2)); + responseBuilder.header("X-Pulsar-PROPERTY", new Gson().toJson(properties)); + if (brokerEntryMetadata != null) { if (brokerEntryMetadata.hasBrokerTimestamp()) { responseBuilder.header("X-Pulsar-Broker-Entry-METADATA-timestamp", @@ -3181,6 +3073,7 @@ private Response generateResponseWithEntry(Entry entry) throws IOException { for (EncryptionKeys encryptionKeys : metadata.getEncryptionKeysList()) { responseBuilder.header("X-Pulsar-Base64-encryption-keys", Base64.getEncoder().encodeToString(encryptionKeys.toByteArray())); + isEncrypted = true; } if (metadata.hasEncryptionParam()) { responseBuilder.header("X-Pulsar-Base64-encryption-param", @@ -3224,9 +3117,18 @@ private Response generateResponseWithEntry(Entry entry) throws IOException { if (metadata.hasNullPartitionKey()) { responseBuilder.header("X-Pulsar-null-partition-key", metadata.isNullPartitionKey()); } + if (metadata.hasTxnidMostBits() && metadata.hasTxnidLeastBits()) { + TxnID txnID = new TxnID(metadata.getTxnidMostBits(), metadata.getTxnidLeastBits()); + boolean isTxnAborted = persistentTopic.isTxnAborted(txnID, entry.getPosition()); + responseBuilder.header("X-Pulsar-txn-aborted", isTxnAborted); + } + boolean isTxnUncommitted = (entry.getPosition()) + .compareTo(persistentTopic.getMaxReadPosition()) > 0; + responseBuilder.header("X-Pulsar-txn-uncommitted", isTxnUncommitted); // Decode if needed - CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(metadata.getCompression()); + CompressionCodec codec = CompressionCodecProvider + .getCompressionCodec(isEncrypted ? NONE : metadata.getCompression()); ByteBuf uncompressedPayload = codec.decode(metadataAndPayload, metadata.getUncompressedSize()); // Copy into a heap buffer for output stream compatibility @@ -3272,7 +3174,7 @@ protected CompletableFuture internalGetBacklogAsync try { PersistentOfflineTopicStats estimateOfflineTopicStats = offlineTopicBacklog.estimateUnloadedTopicBacklog( - (ManagedLedgerFactoryImpl) pulsar().getManagedLedgerFactory(), + pulsar().getManagedLedgerFactory(), topicName); pulsar().getBrokerService() .cacheOfflineTopicStats(topicName, estimateOfflineTopicStats); @@ -3313,65 +3215,55 @@ protected CompletableFuture> in protected void internalGetBacklogSizeByMessageId(AsyncResponse asyncResponse, MessageIdImpl messageId, boolean authoritative) { - CompletableFuture ret; - // If the topic name is a partition name, no need to get partition topic metadata again - if (!topicName.isPartitioned()) { - ret = getPartitionedTopicMetadataAsync(topicName, authoritative, false) - .thenCompose(topicMetadata -> { - if (topicMetadata.partitions > 0) { - log.warn("[{}] Not supported calculate backlog size operation on partitioned-topic {}", - clientAppId(), topicName); - asyncResponse.resume(new RestException(Status.METHOD_NOT_ALLOWED, - "calculate backlog size is not allowed for partitioned-topic")); - } - return CompletableFuture.completedFuture(null); - }); - } else { - ret = CompletableFuture.completedFuture(null); - } - CompletableFuture future; - if (topicName.isGlobal()) { - future = ret.thenCompose(__ -> validateGlobalNamespaceOwnershipAsync(namespaceName)); - } else { - future = ret; - } - future.thenAccept(__ -> validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(unused -> validateTopicOperationAsync(topicName, - TopicOperation.GET_BACKLOG_SIZE)) - .thenCompose(unused -> getTopicReferenceAsync(topicName)) - .thenAccept(t -> { - PersistentTopic topic = (PersistentTopic) t; - PositionImpl pos = new PositionImpl(messageId.getLedgerId(), - messageId.getEntryId()); - if (topic == null) { - asyncResponse.resume(new RestException(Status.NOT_FOUND, - getTopicNotFoundErrorMessage(topicName.toString()))); - return; - } - ManagedLedgerImpl managedLedger = - (ManagedLedgerImpl) topic.getManagedLedger(); - if (messageId.getLedgerId() == -1) { - asyncResponse.resume(managedLedger.getTotalSize()); - } else { - asyncResponse.resume(managedLedger.getEstimatedBacklogSize(pos)); - } - }).exceptionally(ex -> { - // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { - log.error("[{}] Failed to get backlog size for topic {}", clientAppId(), - topicName, ex); - } - resumeAsyncResponseExceptionally(asyncResponse, ex); - return null; - })).exceptionally(ex -> { - // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { - log.error("[{}] Failed to validate global namespace ownership " - + "to get backlog size for topic {}", clientAppId(), topicName, ex); - } - resumeAsyncResponseExceptionally(asyncResponse, ex); - return null; - }); + CompletableFuture ret = validateTopicOperationAsync(topicName, TopicOperation.GET_BACKLOG_SIZE); + ret.thenCompose(__ -> { + // If the topic name is a partition name, no need to get partition topic metadata again + if (!topicName.isPartitioned()) { + return getPartitionedTopicMetadataAsync(topicName, authoritative, false) + .thenCompose(topicMetadata -> { + if (topicMetadata.partitions > 0) { + log.warn("[{}] Not supported calculate backlog size operation on partitioned-topic {}", + clientAppId(), topicName); + asyncResponse.resume(new RestException(Status.METHOD_NOT_ALLOWED, + "calculate backlog size is not allowed for partitioned-topic")); + } + return CompletableFuture.completedFuture(null); + }); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) + .thenCompose(unused -> getTopicReferenceAsync(topicName)) + .thenAccept(t -> { + PersistentTopic topic = (PersistentTopic) t; + Position pos = PositionFactory.create(messageId.getLedgerId(), + messageId.getEntryId()); + if (topic == null) { + asyncResponse.resume(new RestException(Status.NOT_FOUND, + getTopicNotFoundErrorMessage(topicName.toString()))); + return; + } + ManagedLedger managedLedger = topic.getManagedLedger(); + if (messageId.getLedgerId() == -1) { + asyncResponse.resume(managedLedger.getTotalSize()); + } else { + asyncResponse.resume(managedLedger.getEstimatedBacklogSize(pos)); + } + }).exceptionally(ex -> { + // If the exception is not redirect exception we need to log it. + if (!isNot307And404Exception(ex)) { + log.error("[{}] Failed to validate global namespace ownership " + + "to get backlog size for topic {}", clientAppId(), topicName, ex); + } + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); } protected CompletableFuture internalSetBacklogQuota(BacklogQuota.BacklogQuotaType backlogQuotaType, @@ -3379,8 +3271,7 @@ protected CompletableFuture internalSetBacklogQuota(BacklogQuota.BacklogQu BacklogQuota.BacklogQuotaType finalBacklogQuotaType = backlogQuotaType == null ? BacklogQuota.BacklogQuotaType.destination_storage : backlogQuotaType; - return validateTopicPolicyOperationAsync(topicName, PolicyName.BACKLOG, PolicyOperation.WRITE) - .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) + return validatePoliciesReadOnlyAccessAsync() .thenCompose(__ -> getTopicPoliciesAsyncWithRetry(topicName, isGlobal)) .thenCompose(op -> { TopicPolicies topicPolicies = op.orElseGet(TopicPolicies::new); @@ -3421,16 +3312,20 @@ protected CompletableFuture internalSetBacklogQuota(BacklogQuota.BacklogQu } protected CompletableFuture internalSetReplicationClusters(List clusterIds) { - - return validateTopicPolicyOperationAsync(topicName, PolicyName.REPLICATION, PolicyOperation.WRITE) - .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) - .thenCompose(__ -> { - Set replicationClusters = Sets.newHashSet(clusterIds); + if (CollectionUtils.isEmpty(clusterIds)) { + return CompletableFuture.failedFuture(new RestException(Status.PRECONDITION_FAILED, + "ClusterIds should not be null or empty")); + } + Set replicationClusters = Sets.newHashSet(clusterIds); + return validatePoliciesReadOnlyAccessAsync() + .thenAccept(__ -> { if (replicationClusters.contains("global")) { throw new RestException(Status.PRECONDITION_FAILED, "Cannot specify global in the list of replication clusters"); } - Set clusters = clusters(); + }) + .thenCompose(__ -> clustersAsync()) + .thenCompose(clusters -> { List> futures = new ArrayList<>(replicationClusters.size()); for (String clusterId : replicationClusters) { if (!clusters.contains(clusterId)) { @@ -3440,29 +3335,30 @@ protected CompletableFuture internalSetReplicationClusters(List cl futures.add(validateClusterForTenantAsync(namespaceName.getTenant(), clusterId)); } return FutureUtil.waitForAll(futures); - }).thenCompose(__ -> - getTopicPoliciesAsyncWithRetry(topicName).thenCompose(op -> { - TopicPolicies topicPolicies = op.orElseGet(TopicPolicies::new); - topicPolicies.setReplicationClusters(clusterIds); - return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, topicPolicies) - .thenRun(() -> { - log.info("[{}] Successfully set replication clusters for namespace={}, " - + "topic={}, clusters={}", - clientAppId(), - namespaceName, - topicName.getLocalName(), - topicPolicies.getReplicationClusters()); - }); - } - )); - } - - protected CompletableFuture internalRemoveReplicationClusters() { - return validateTopicPolicyOperationAsync(topicName, PolicyName.REPLICATION, PolicyOperation.WRITE) - .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) - .thenCompose(__ -> getTopicPoliciesAsyncWithRetry(topicName).thenCompose(op -> { + }).thenCompose(__ -> { + if (!pulsar().getConfig().isCreateTopicToRemoteClusterForReplication()) { + log.info("[{}] Skip creating partitioned for topic {} for the remote clusters {}", + clientAppId(), topicName, replicationClusters.stream().filter(v -> + !pulsar().getConfig().getClusterName().equals(v)).collect(Collectors.toList())); + return CompletableFuture.completedFuture(null); + } + // Sync to create partitioned topic on the remote cluster if needed. + TopicName topicNameWithoutPartition = TopicName.get(topicName.getPartitionedTopicName()); + return pulsar().getPulsarResources().getNamespaceResources().getPartitionedTopicResources() + .getPartitionedTopicMetadataAsync(topicNameWithoutPartition).thenCompose(topicMetaOp -> { + // Skip to create topic if the topic is non-partitioned, because the replicator will create + // it automatically. + if (topicMetaOp.isEmpty()) { + return CompletableFuture.completedFuture(null); + } + return FutureUtil.waitForAll( + internalCreatePartitionedTopicToReplicatedClustersInBackground(replicationClusters, + topicMetaOp.get().partitions).values()); + }); + }).thenCompose(__ -> + getTopicPoliciesAsyncWithRetry(topicName).thenCompose(op -> { TopicPolicies topicPolicies = op.orElseGet(TopicPolicies::new); - topicPolicies.setReplicationClusters(null); + topicPolicies.setReplicationClusters(clusterIds); return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, topicPolicies) .thenRun(() -> { log.info("[{}] Successfully set replication clusters for namespace={}, " @@ -3472,8 +3368,26 @@ protected CompletableFuture internalRemoveReplicationClusters() { topicName.getLocalName(), topicPolicies.getReplicationClusters()); }); - }) - ); + } + )); + } + + protected CompletableFuture internalRemoveReplicationClusters() { + return validatePoliciesReadOnlyAccessAsync() + .thenCompose(__ -> getTopicPoliciesAsyncWithRetry(topicName)) + .thenCompose(op -> { + TopicPolicies topicPolicies = op.orElseGet(TopicPolicies::new); + topicPolicies.setReplicationClusters(null); + return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, topicPolicies) + .thenRun(() -> { + log.info("[{}] Successfully set replication clusters for namespace={}, " + + "topic={}, clusters={}", + clientAppId(), + namespaceName, + topicName.getLocalName(), + topicPolicies.getReplicationClusters()); + }); + }); } protected CompletableFuture internalGetDeduplication(boolean applied, boolean isGlobal) { @@ -3580,6 +3494,34 @@ protected CompletableFuture internalRemoveRetention(boolean isGlobal) { }); } + protected CompletableFuture internalSetDispatcherPauseOnAckStatePersistent(boolean isGlobal) { + return getTopicPoliciesAsyncWithRetry(topicName, isGlobal) + .thenCompose(op -> { + TopicPolicies topicPolicies = op.orElseGet(TopicPolicies::new); + topicPolicies.setDispatcherPauseOnAckStatePersistentEnabled(true); + topicPolicies.setIsGlobal(isGlobal); + return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, topicPolicies); + }); + } + + protected CompletableFuture internalRemoveDispatcherPauseOnAckStatePersistent(boolean isGlobal) { + return getTopicPoliciesAsyncWithRetry(topicName, isGlobal) + .thenCompose(op -> { + if (!op.isPresent()) { + return CompletableFuture.completedFuture(null); + } + op.get().setDispatcherPauseOnAckStatePersistentEnabled(false); + return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, op.get()); + }); + } + + protected CompletableFuture internalGetDispatcherPauseOnAckStatePersistent(boolean applied, + boolean isGlobal) { + return getTopicPoliciesAsyncWithRetry(topicName, isGlobal) + .thenApply(op -> op.map(TopicPolicies::getDispatcherPauseOnAckStatePersistentEnabled) + .orElse(false)); +} + protected CompletableFuture internalGetPersistence(boolean applied, boolean isGlobal) { return getTopicPoliciesAsyncWithRetry(topicName, isGlobal) .thenApply(op -> op.map(TopicPolicies::getPersistence) @@ -3807,29 +3749,29 @@ protected CompletableFuture internalTerminateAsync(boolean authoritat "Termination of a system topic is not allowed")); } - CompletableFuture ret; - if (topicName.isGlobal()) { - ret = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - ret = CompletableFuture.completedFuture(null); - } - return ret.thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) - .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.TERMINATE)) - .thenCompose(__ -> getPartitionedTopicMetadataAsync(topicName, authoritative, false)) - .thenAccept(partitionMetadata -> { - if (partitionMetadata.partitions > 0) { - throw new RestException(Status.METHOD_NOT_ALLOWED, - "Termination of a partitioned topic is not allowed"); - } - }) - .thenCompose(__ -> getTopicReferenceAsync(topicName)) - .thenCompose(topic -> { - if (!(topic instanceof PersistentTopic)) { - throw new RestException(Status.METHOD_NOT_ALLOWED, - "Termination of a non-persistent topic is not allowed"); - } - return ((PersistentTopic) topic).terminate(); - }); + CompletableFuture ret = validateTopicOperationAsync(topicName, TopicOperation.TERMINATE); + return ret.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) + .thenCompose(__ -> getPartitionedTopicMetadataAsync(topicName, authoritative, false)) + .thenAccept(partitionMetadata -> { + if (partitionMetadata.partitions > 0) { + throw new RestException(Status.METHOD_NOT_ALLOWED, + "Termination of a partitioned topic is not allowed"); + } + }) + .thenCompose(__ -> getTopicReferenceAsync(topicName)) + .thenCompose(topic -> { + if (!(topic instanceof PersistentTopic)) { + throw new RestException(Status.METHOD_NOT_ALLOWED, + "Termination of a non-persistent topic is not allowed"); + } + return ((PersistentTopic) topic).terminate(); + }); } protected void internalTerminatePartitionedTopic(AsyncResponse asyncResponse, boolean authoritative) { @@ -3840,75 +3782,65 @@ protected void internalTerminatePartitionedTopic(AsyncResponse asyncResponse, bo asyncResponse.resume(new RestException(Status.METHOD_NOT_ALLOWED, msg)); return; } + CompletableFuture future = validateTopicOperationAsync(topicName, TopicOperation.TERMINATE); + future.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenCompose(unused -> getPartitionedTopicMetadataAsync(topicName, authoritative, false)) + .thenAccept(partitionMetadata -> { + if (partitionMetadata.partitions == 0) { + String msg = "Termination of a non-partitioned topic is not allowed using partitioned-terminate" + + ", please use terminate commands"; + log.error("[{}] [{}] {}", clientAppId(), topicName, msg); + asyncResponse.resume(new RestException(Status.METHOD_NOT_ALLOWED, msg)); + return; + } + if (partitionMetadata.partitions > 0) { + Map messageIds = new ConcurrentHashMap<>(partitionMetadata.partitions); + final List> futures = + new ArrayList<>(partitionMetadata.partitions); - CompletableFuture future; - if (topicName.isGlobal()) { - future = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - future = CompletableFuture.completedFuture(null); - } - - future.thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.TERMINATE) - .thenCompose(unused -> getPartitionedTopicMetadataAsync(topicName, authoritative, false)) - .thenAccept(partitionMetadata -> { - if (partitionMetadata.partitions == 0) { - String msg = "Termination of a non-partitioned topic is not allowed using partitioned-terminate" - + ", please use terminate commands"; - log.error("[{}] [{}] {}", clientAppId(), topicName, msg); - asyncResponse.resume(new RestException(Status.METHOD_NOT_ALLOWED, msg)); - return; + for (int i = 0; i < partitionMetadata.partitions; i++) { + TopicName topicNamePartition = topicName.getPartition(i); + try { + int finalI = i; + futures.add(pulsar().getAdminClient().topics() + .terminateTopicAsync(topicNamePartition.toString()) + .whenComplete((messageId, throwable) -> { + if (throwable != null) { + log.error("[{}] Failed to terminate topic {}", clientAppId(), + topicNamePartition, throwable); + asyncResponse.resume(new RestException(throwable)); + } + messageIds.put(finalI, messageId); + })); + } catch (Exception e) { + log.error("[{}] Failed to terminate topic {}", clientAppId(), topicNamePartition, + e); + throw new RestException(e); } - if (partitionMetadata.partitions > 0) { - Map messageIds = new ConcurrentHashMap<>(partitionMetadata.partitions); - final List> futures = - new ArrayList<>(partitionMetadata.partitions); - - for (int i = 0; i < partitionMetadata.partitions; i++) { - TopicName topicNamePartition = topicName.getPartition(i); - try { - int finalI = i; - futures.add(pulsar().getAdminClient().topics() - .terminateTopicAsync(topicNamePartition.toString()) - .whenComplete((messageId, throwable) -> { - if (throwable != null) { - log.error("[{}] Failed to terminate topic {}", clientAppId(), - topicNamePartition, throwable); - asyncResponse.resume(new RestException(throwable)); - } - messageIds.put(finalI, messageId); - })); - } catch (Exception e) { - log.error("[{}] Failed to terminate topic {}", clientAppId(), topicNamePartition, - e); - throw new RestException(e); - } + } + FutureUtil.waitForAll(futures).handle((result, exception) -> { + if (exception != null) { + Throwable t = exception.getCause(); + if (t instanceof NotFoundException) { + asyncResponse.resume(new RestException(Status.NOT_FOUND, + getTopicNotFoundErrorMessage(topicName.toString()))); + } else { + log.error("[{}] Failed to terminate topic {}", clientAppId(), topicName, t); + asyncResponse.resume(new RestException(t)); } - FutureUtil.waitForAll(futures).handle((result, exception) -> { - if (exception != null) { - Throwable t = exception.getCause(); - if (t instanceof NotFoundException) { - asyncResponse.resume(new RestException(Status.NOT_FOUND, - getTopicNotFoundErrorMessage(topicName.toString()))); - } else { - log.error("[{}] Failed to terminate topic {}", clientAppId(), topicName, t); - asyncResponse.resume(new RestException(t)); - } - } - asyncResponse.resume(messageIds); - return null; - }); - } - }).exceptionally(ex -> { - // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { - log.error("[{}] Failed to terminate topic {}", clientAppId(), topicName, ex); } - resumeAsyncResponseExceptionally(asyncResponse, ex); + asyncResponse.resume(messageIds); return null; - }) - ).exceptionally(ex -> { + }); + } + }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to terminate topic {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -3918,87 +3850,86 @@ protected void internalTerminatePartitionedTopic(AsyncResponse asyncResponse, bo protected void internalExpireMessagesByTimestamp(AsyncResponse asyncResponse, String subName, int expireTimeInSeconds, boolean authoritative) { - CompletableFuture future; - if (topicName.isGlobal()) { - future = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - future = CompletableFuture.completedFuture(null); - } - future.thenCompose(__ -> - validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(unused -> validateTopicOperationAsync(topicName, TopicOperation.EXPIRE_MESSAGES, subName)) - .thenCompose(unused2 -> - // If the topic name is a partition name, no need to get partition topic metadata again - getPartitionedTopicMetadataAsync(topicName, authoritative, false) - .thenCompose(partitionMetadata -> { - if (topicName.isPartitioned()) { - return internalExpireMessagesByTimestampForSinglePartitionAsync - (partitionMetadata, subName, expireTimeInSeconds) - .thenAccept(unused3 -> - asyncResponse.resume(Response.noContent().build())); - } else { - if (partitionMetadata.partitions > 0) { - return CompletableFuture.completedFuture(null).thenAccept(unused -> { - final List> futures = new ArrayList<>(); - - // expire messages for each partition topic - for (int i = 0; i < partitionMetadata.partitions; i++) { - TopicName topicNamePartition = topicName.getPartition(i); - try { - futures.add(pulsar() - .getAdminClient() - .topics() - .expireMessagesAsync(topicNamePartition.toString(), - subName, expireTimeInSeconds)); - } catch (Exception e) { - log.error("[{}] Failed to expire messages up to {} on {}", - clientAppId(), - expireTimeInSeconds, topicNamePartition, e); - asyncResponse.resume(new RestException(e)); - return; - } - } - - FutureUtil.waitForAll(futures).handle((result, exception) -> { - if (exception != null) { - Throwable t = FutureUtil.unwrapCompletionException(exception); - if (t instanceof NotFoundException) { - asyncResponse.resume(new RestException(Status.NOT_FOUND, - getSubNotFoundErrorMessage(topicName.toString(), - subName))); - return null; - } else { - if (t instanceof PulsarAdminException) { - log.warn("[{}] Failed to expire messages up " - + "to {} on {}: {}", clientAppId(), - expireTimeInSeconds, topicName, - t.toString()); - } else { - log.error("[{}] Failed to expire messages up " - + "to {} on {}", clientAppId(), - expireTimeInSeconds, topicName, t); - } - resumeAsyncResponseExceptionally(asyncResponse, t); - return null; - } - } - asyncResponse.resume(Response.noContent().build()); - return null; - }); - }); - } else { - return internalExpireMessagesByTimestampForSinglePartitionAsync - (partitionMetadata, subName, expireTimeInSeconds) - .thenAccept(unused -> - asyncResponse.resume(Response.noContent().build())); + CompletableFuture future = validateTopicOperationAsync(topicName, TopicOperation.EXPIRE_MESSAGES, + subName); + future.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) + .thenCompose(__ -> + // If the topic name is a partition name, no need to get partition topic metadata again + getPartitionedTopicMetadataAsync(topicName, authoritative, false) + .thenCompose(partitionMetadata -> { + if (topicName.isPartitioned()) { + return internalExpireMessagesByTimestampForSinglePartitionAsync + (partitionMetadata, subName, expireTimeInSeconds) + .thenAccept(unused3 -> + asyncResponse.resume(Response.noContent().build())); + } else { + if (partitionMetadata.partitions > 0) { + return CompletableFuture.completedFuture(null).thenAccept(unused -> { + final List> futures = new ArrayList<>(); + + // expire messages for each partition topic + for (int i = 0; i < partitionMetadata.partitions; i++) { + TopicName topicNamePartition = topicName.getPartition(i); + try { + futures.add(pulsar() + .getAdminClient() + .topics() + .expireMessagesAsync(topicNamePartition.toString(), + subName, expireTimeInSeconds)); + } catch (Exception e) { + log.error("[{}] Failed to expire messages up to {} on {}", + clientAppId(), + expireTimeInSeconds, topicNamePartition, e); + asyncResponse.resume(new RestException(e)); + return; } } - })) + FutureUtil.waitForAll(futures).handle((result, exception) -> { + if (exception != null) { + Throwable t = FutureUtil.unwrapCompletionException(exception); + if (t instanceof NotFoundException) { + asyncResponse.resume(new RestException(Status.NOT_FOUND, + getSubNotFoundErrorMessage(topicName.toString(), + subName))); + return null; + } else { + if (t instanceof PulsarAdminException) { + log.warn("[{}] Failed to expire messages up " + + "to {} on {}: {}", clientAppId(), + expireTimeInSeconds, topicName, + t.toString()); + } else { + log.error("[{}] Failed to expire messages up " + + "to {} on {}", clientAppId(), + expireTimeInSeconds, topicName, t); + } + resumeAsyncResponseExceptionally(asyncResponse, t); + return null; + } + } + asyncResponse.resume(Response.noContent().build()); + return null; + }); + }); + } else { + return internalExpireMessagesByTimestampForSinglePartitionAsync + (partitionMetadata, subName, expireTimeInSeconds) + .thenAccept(unused -> + asyncResponse.resume(Response.noContent().build())); + } + } + }) ).exceptionally(ex -> { Throwable cause = FutureUtil.unwrapCompletionException(ex); // If the exception is not redirect exception we need to log it. - if (!isRedirectException(cause)) { + if (isNot307And404Exception(cause)) { if (cause instanceof RestException) { log.warn("[{}] Failed to expire messages up to {} on {}: {}", clientAppId(), expireTimeInSeconds, topicName, cause.toString()); @@ -4032,28 +3963,20 @@ private CompletableFuture internalExpireMessagesByTimestampForSinglePartit } PersistentTopic topic = (PersistentTopic) t; - boolean issued; + final MessageExpirer messageExpirer; if (subName.startsWith(topic.getReplicatorPrefix())) { String remoteCluster = PersistentReplicator.getRemoteCluster(subName); - PersistentReplicator repl = (PersistentReplicator) topic - .getPersistentReplicator(remoteCluster); - if (repl == null) { - resultFuture.completeExceptionally( - new RestException(Status.NOT_FOUND, "Replicator not found")); - return; - } - issued = repl.expireMessages(expireTimeInSeconds); + messageExpirer = (PersistentReplicator) topic.getPersistentReplicator(remoteCluster); } else { - PersistentSubscription sub = topic.getSubscription(subName); - if (sub == null) { - resultFuture.completeExceptionally( - new RestException(Status.NOT_FOUND, - getSubNotFoundErrorMessage(topicName.toString(), subName))); - return; - } - issued = sub.expireMessages(expireTimeInSeconds); + messageExpirer = topic.getSubscription(subName); + } + if (messageExpirer == null) { + final String message = subName.startsWith(topic.getReplicatorPrefix()) + ? "Replicator not found" : getSubNotFoundErrorMessage(topicName.toString(), subName); + resultFuture.completeExceptionally(new RestException(Status.NOT_FOUND, message)); + return; } - if (issued) { + if (messageExpirer.expireMessages(expireTimeInSeconds)) { log.info("[{}] Message expire started up to {} on {} {}", clientAppId(), expireTimeInSeconds, topicName, subName); resultFuture.complete(null); @@ -4090,44 +4013,43 @@ protected void internalExpireMessagesByPosition(AsyncResponse asyncResponse, Str asyncResponse.resume(new RestException(Status.PRECONDITION_FAILED, msg)); return; } - CompletableFuture ret; - // If the topic name is a partition name, no need to get partition topic metadata again - if (!topicName.isPartitioned()) { - ret = getPartitionedTopicMetadataAsync(topicName, authoritative, false) - .thenCompose(topicMetadata -> { - if (topicMetadata.partitions > 0) { - String msg = "Expire message at position is not supported for partitioned-topic"; - log.warn("[{}] {} {}({}) {}", clientAppId(), msg, topicName, messageId, subName); - asyncResponse.resume(new RestException(Status.METHOD_NOT_ALLOWED, msg)); - } - return CompletableFuture.completedFuture(null); - }); - } else { - ret = CompletableFuture.completedFuture(null); - } - CompletableFuture future; - if (topicName.isGlobal()) { - future = ret.thenCompose(__ -> validateGlobalNamespaceOwnershipAsync(namespaceName)); - } else { - future = ret; - } - - future.thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) - .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.EXPIRE_MESSAGES, subName)) - .thenCompose(__ -> { - log.info("[{}][{}] Received expire messages on subscription {} to position {}", clientAppId(), - topicName, subName, messageId); - return internalExpireMessagesNonPartitionedTopicByPosition(asyncResponse, subName, - messageId, isExcluded, batchIndex); - }).exceptionally(ex -> { - // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { - log.error("[{}] Failed to expire messages up to {} on subscription {} to position {}", - clientAppId(), topicName, subName, messageId, ex); - } - resumeAsyncResponseExceptionally(asyncResponse, ex); - return null; - }); + CompletableFuture ret = validateTopicOperationAsync(topicName, TopicOperation.EXPIRE_MESSAGES, subName); + ret.thenCompose(__ -> { + // If the topic name is a partition name, no need to get partition topic metadata again + if (!topicName.isPartitioned()) { + return getPartitionedTopicMetadataAsync(topicName, authoritative, false) + .thenCompose(topicMetadata -> { + if (topicMetadata.partitions > 0) { + String msg = "Expire message at position is not supported for partitioned-topic"; + log.warn("[{}] {} {}({}) {}", clientAppId(), msg, topicName, messageId, subName); + asyncResponse.resume(new RestException(Status.METHOD_NOT_ALLOWED, msg)); + } + return CompletableFuture.completedFuture(null); + }); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) + .thenCompose(__ -> { + log.info("[{}][{}] Received expire messages on subscription {} to position {}", clientAppId(), + topicName, subName, messageId); + return internalExpireMessagesNonPartitionedTopicByPosition(asyncResponse, subName, + messageId, isExcluded, batchIndex); + }).exceptionally(ex -> { + // If the exception is not redirect exception we need to log it. + if (isNot307And404Exception(ex)) { + log.error("[{}] Failed to expire messages up to {} on subscription {} to position {}", + clientAppId(), topicName, subName, messageId, ex); + } + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); } private CompletableFuture internalExpireMessagesNonPartitionedTopicByPosition(AsyncResponse asyncResponse, @@ -4143,32 +4065,27 @@ private CompletableFuture internalExpireMessagesNonPartitionedTopicByPosit return; } try { - PersistentSubscription sub = topic.getSubscription(subName); - if (sub == null) { - asyncResponse.resume(new RestException(Status.NOT_FOUND, - getSubNotFoundErrorMessage(topicName.toString(), subName))); + final MessageExpirer messageExpirer; + if (subName.startsWith(topic.getReplicatorPrefix())) { + String remoteCluster = PersistentReplicator.getRemoteCluster(subName); + messageExpirer = (PersistentReplicator) topic.getPersistentReplicator(remoteCluster); + } else { + messageExpirer = topic.getSubscription(subName); + } + if (messageExpirer == null) { + final String message = (subName.startsWith(topic.getReplicatorPrefix())) + ? "Replicator not found" : getSubNotFoundErrorMessage(topicName.toString(), subName); + asyncResponse.resume(new RestException(Status.NOT_FOUND, message)); return; } + CompletableFuture batchSizeFuture = new CompletableFuture<>(); getEntryBatchSize(batchSizeFuture, topic, messageId, batchIndex); + batchSizeFuture.thenAccept(bi -> { - PositionImpl position = calculatePositionAckSet(isExcluded, bi, batchIndex, messageId); - boolean issued; + Position position = calculatePositionAckSet(isExcluded, bi, batchIndex, messageId); try { - if (subName.startsWith(topic.getReplicatorPrefix())) { - String remoteCluster = PersistentReplicator.getRemoteCluster(subName); - PersistentReplicator repl = (PersistentReplicator) - topic.getPersistentReplicator(remoteCluster); - if (repl == null) { - asyncResponse.resume(new RestException(Status.NOT_FOUND, - "Replicator not found")); - return; - } - issued = repl.expireMessages(position); - } else { - issued = sub.expireMessages(position); - } - if (issued) { + if (messageExpirer.expireMessages(position)) { log.info("[{}] Message expire started up to {} on {} {}", clientAppId(), position, topicName, subName); } else { @@ -4220,13 +4137,14 @@ private CompletableFuture internalExpireMessagesNonPartitionedTopicByPosit protected void internalTriggerCompaction(AsyncResponse asyncResponse, boolean authoritative) { log.info("[{}] Trigger compaction on topic {}", clientAppId(), topicName); - CompletableFuture future; - if (topicName.isGlobal()) { - future = validateGlobalNamespaceOwnershipAsync(namespaceName); - } else { - future = CompletableFuture.completedFuture(null); - } - future.thenAccept(__ -> { + CompletableFuture future = validateTopicOperationAsync(topicName, TopicOperation.COMPACT); + future.thenCompose(__ -> { + if (topicName.isGlobal()) { + return validateGlobalNamespaceOwnershipAsync(namespaceName); + } else { + return CompletableFuture.completedFuture(null); + } + }).thenAccept(__ -> { // If the topic name is a partition name, no need to get partition topic metadata again if (topicName.isPartitioned()) { internalTriggerCompactionNonPartitionedTopic(asyncResponse, authoritative); @@ -4276,7 +4194,7 @@ protected void internalTriggerCompaction(AsyncResponse asyncResponse, boolean au } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to trigger compaction on topic {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -4285,7 +4203,7 @@ protected void internalTriggerCompaction(AsyncResponse asyncResponse, boolean au } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to validate global namespace ownership to trigger compaction on topic {}", clientAppId(), topicName, ex); } @@ -4314,7 +4232,7 @@ protected void internalTriggerCompactionNonPartitionedTopic(AsyncResponse asyncR } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to trigger compaction for {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -4324,16 +4242,16 @@ protected void internalTriggerCompactionNonPartitionedTopic(AsyncResponse asyncR } protected CompletableFuture internalCompactionStatusAsync(boolean authoritative) { - return validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.COMPACT)) + return validateTopicOperationAsync(topicName, TopicOperation.COMPACT) + .thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) .thenCompose(__ -> getTopicReferenceAsync(topicName)) .thenApply(topic -> ((PersistentTopic) topic).compactionStatus()); } protected void internalTriggerOffload(AsyncResponse asyncResponse, boolean authoritative, MessageIdImpl messageId) { - validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.OFFLOAD)) + validateTopicOperationAsync(topicName, TopicOperation.OFFLOAD) + .thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) .thenCompose(__ -> getTopicReferenceAsync(topicName)) .thenAccept(topic -> { try { @@ -4350,7 +4268,7 @@ protected void internalTriggerOffload(AsyncResponse asyncResponse, } }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to trigger offload for {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -4359,15 +4277,15 @@ protected void internalTriggerOffload(AsyncResponse asyncResponse, } protected void internalOffloadStatus(AsyncResponse asyncResponse, boolean authoritative) { - validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.OFFLOAD)) + validateTopicOperationAsync(topicName, TopicOperation.OFFLOAD) + .thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) .thenCompose(__ -> getTopicReferenceAsync(topicName)) .thenAccept(topic -> { OffloadProcessStatus offloadProcessStatus = ((PersistentTopic) topic).offloadStatus(); asyncResponse.resume(offloadProcessStatus); }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to offload status on topic {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -4566,54 +4484,6 @@ protected CompletableFuture internalValidateClientVersionAsync() { return CompletableFuture.completedFuture(null); } - /** - * Validate update of number of partition for partitioned topic. - * If there's already non partition topic with same name and contains partition suffix "-partition-" - * followed by numeric value X then the new number of partition of that partitioned topic can not be greater - * than that X else that non partition topic will essentially be overwritten and cause unexpected consequence. - * - * @param topicName - */ - private CompletableFuture validatePartitionTopicUpdateAsync(String topicName, int numberOfPartition) { - return internalGetListAsync().thenCompose(existingTopicList -> { - TopicName partitionTopicName = TopicName.get(domain(), namespaceName, topicName); - String prefix = partitionTopicName.getPartitionedTopicName() + PARTITIONED_TOPIC_SUFFIX; - return getPartitionedTopicMetadataAsync(partitionTopicName, false, false) - .thenAccept(metadata -> { - int oldPartition = metadata.partitions; - for (String existingTopicName : existingTopicList) { - if (existingTopicName.startsWith(prefix)) { - try { - long suffix = Long.parseLong(existingTopicName.substring( - existingTopicName.indexOf(PARTITIONED_TOPIC_SUFFIX) - + PARTITIONED_TOPIC_SUFFIX.length())); - // Skip partition of partitioned topic by making sure - // the numeric suffix greater than old partition number. - if (suffix >= oldPartition && suffix <= (long) numberOfPartition) { - log.warn( - "[{}] Already have non partition topic {} which contains partition" - + " suffix '-partition-' and end with numeric value smaller" - + " than the new number of partition. Update of partitioned" - + " topic {} could cause conflict.", - clientAppId(), - existingTopicName, topicName); - throw new RestException(Status.PRECONDITION_FAILED, - "Already have non partition topic " + existingTopicName - + " which contains partition suffix '-partition-' " - + "and end with numeric value and end with numeric value" - + " smaller than the new number of partition. Update of" - + " partitioned topic " + topicName + " could cause conflict."); - } - } catch (NumberFormatException e) { - // Do nothing, if value after partition suffix is not pure numeric value, - // as it can't conflict with internal created partitioned topic's name. - } - } - } - }); - }); - } - /** * Validate non partition topic name, * Validation will fail and throw RestException if @@ -4640,11 +4510,11 @@ private CompletableFuture validateNonPartitionTopicNameAsync(String topicN // Partition topic index is 0 to (number of partition - 1) if (metadata.partitions > 0 && suffix >= (long) metadata.partitions) { log.warn("[{}] Can't create topic {} with \"-partition-\" followed by" - + " a number smaller then number of partition of partitioned topic {}.", + + " a number smaller than number of partition of partitioned topic {}.", clientAppId(), topicName, partitionTopicName.getLocalName()); throw new RestException(Status.PRECONDITION_FAILED, "Can't create topic " + topicName + " with \"-partition-\" followed by" - + " a number smaller then number of partition of partitioned topic " + + " a number smaller than number of partition of partitioned topic " + partitionTopicName.getLocalName()); } else if (metadata.partitions == 0) { log.warn("[{}] Can't create topic {} with \"-partition-\" followed by" @@ -4668,8 +4538,8 @@ private CompletableFuture validateNonPartitionTopicNameAsync(String topicN } protected void internalGetLastMessageId(AsyncResponse asyncResponse, boolean authoritative) { - validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.PEEK_MESSAGES)) + validateTopicOperationAsync(topicName, TopicOperation.PEEK_MESSAGES) + .thenCompose(__ -> validateTopicOwnershipAsync(topicName, authoritative)) .thenCompose(__ -> getTopicReferenceAsync(topicName)) .thenAccept(topic -> { if (topic == null) { @@ -4692,7 +4562,7 @@ protected void internalGetLastMessageId(AsyncResponse asyncResponse, boolean aut }); }).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get last messageId {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -4706,11 +4576,12 @@ protected CompletableFuture internalTrimTopic(AsyncResponse asyncResponse, "Trim on a non-persistent topic is not allowed")); return null; } + CompletableFuture future = validateTopicOperationAsync(topicName, TopicOperation.TRIM_TOPIC); if (topicName.isPartitioned()) { - return validateTopicOperationAsync(topicName, TopicOperation.TRIM_TOPIC).thenCompose((x) + return future.thenCompose((x) -> trimNonPartitionedTopic(asyncResponse, topicName, authoritative)); } - return validateTopicOperationAsync(topicName, TopicOperation.TRIM_TOPIC) + return future .thenCompose(__ -> pulsar().getBrokerService().fetchPartitionedTopicMetadataAsync(topicName)) .thenCompose(metadata -> { if (metadata.partitions > 0) { @@ -5070,9 +4941,7 @@ protected CompletableFuture internalRemoveSubscribeRate(boolean isGlobal) protected void handleTopicPolicyException(String methodName, Throwable thr, AsyncResponse asyncResponse) { Throwable cause = thr.getCause(); - if (!(cause instanceof WebApplicationException) || !( - ((WebApplicationException) cause).getResponse().getStatus() == 307 - || ((WebApplicationException) cause).getResponse().getStatus() == 404)) { + if (isNot307And404Exception(cause)) { log.error("[{}] Failed to perform {} on topic {}", clientAppId(), methodName, topicName, cause); } @@ -5198,7 +5067,7 @@ protected void internalSetReplicatedSubscriptionStatus(AsyncResponse asyncRespon resultFuture.exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.warn("[{}] Failed to change replicated subscription status to {} - {} {}", clientAppId(), enabled, topicName, subName, ex); } @@ -5245,7 +5114,7 @@ private void internalSetReplicatedSubscriptionStatusForNonPartitionedTopic( } ).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to set replicated subscription status on {} {}", clientAppId(), topicName, subName, ex); } @@ -5346,7 +5215,7 @@ protected void internalGetReplicatedSubscriptionStatus(AsyncResponse asyncRespon } resultFuture.exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get replicated subscription status on {} {}", clientAppId(), topicName, subName, ex); } @@ -5397,30 +5266,24 @@ protected CompletableFuture internalGetSchemaCompat if (applied) { return getSchemaCompatibilityStrategyAsync(); } - return validateTopicPolicyOperationAsync(topicName, - PolicyName.SCHEMA_COMPATIBILITY_STRATEGY, - PolicyOperation.READ) - .thenCompose(n -> getTopicPoliciesAsyncWithRetry(topicName).thenApply(op -> { + return getTopicPoliciesAsyncWithRetry(topicName).thenApply(op -> { if (!op.isPresent()) { return null; } SchemaCompatibilityStrategy strategy = op.get().getSchemaCompatibilityStrategy(); return SchemaCompatibilityStrategy.isUndefined(strategy) ? null : strategy; - })); + }); } protected CompletableFuture internalSetSchemaCompatibilityStrategy(SchemaCompatibilityStrategy strategy) { - return validateTopicPolicyOperationAsync(topicName, - PolicyName.SCHEMA_COMPATIBILITY_STRATEGY, - PolicyOperation.WRITE) - .thenCompose((__) -> getTopicPoliciesAsyncWithRetry(topicName) + return getTopicPoliciesAsyncWithRetry(topicName) .thenCompose(op -> { TopicPolicies topicPolicies = op.orElseGet(TopicPolicies::new); topicPolicies.setSchemaCompatibilityStrategy( strategy == SchemaCompatibilityStrategy.UNDEFINED ? null : strategy); return pulsar().getTopicPoliciesService() .updateTopicPoliciesAsync(topicName, topicPolicies); - })); + }); } protected CompletableFuture internalGetSchemaValidationEnforced(boolean applied) { @@ -5444,54 +5307,47 @@ protected CompletableFuture internalSetSchemaValidationEnforced(boolean sc } protected CompletableFuture internalGetEntryFilters(boolean applied, boolean isGlobal) { - return validateTopicPolicyOperationAsync(topicName, PolicyName.ENTRY_FILTERS, PolicyOperation.READ) - .thenCompose(__ -> { - if (!applied) { - return getTopicPoliciesAsyncWithRetry(topicName, isGlobal) - .thenApply(op -> op.map(TopicPolicies::getEntryFilters).orElse(null)); - } - if (!pulsar().getConfiguration().isAllowOverrideEntryFilters()) { - return CompletableFuture.completedFuture(new EntryFilters(String.join(",", - pulsar().getConfiguration().getEntryFilterNames()))); + if (!applied) { + return getTopicPoliciesAsyncWithRetry(topicName, isGlobal) + .thenApply(op -> op.map(TopicPolicies::getEntryFilters).orElse(null)); + } + if (!pulsar().getConfiguration().isAllowOverrideEntryFilters()) { + return CompletableFuture.completedFuture(new EntryFilters(String.join(",", + pulsar().getConfiguration().getEntryFilterNames()))); + } + return getTopicPoliciesAsyncWithRetry(topicName, isGlobal) + .thenApply(op -> op.map(TopicPolicies::getEntryFilters)) + .thenCompose(policyEntryFilters -> { + if (policyEntryFilters.isPresent()) { + return CompletableFuture.completedFuture(policyEntryFilters.get()); } - return getTopicPoliciesAsyncWithRetry(topicName, isGlobal) - .thenApply(op -> op.map(TopicPolicies::getEntryFilters)) - .thenCompose(policyEntryFilters -> { - if (policyEntryFilters.isPresent()) { - return CompletableFuture.completedFuture(policyEntryFilters.get()); + return getNamespacePoliciesAsync(namespaceName) + .thenApply(policies -> policies.entryFilters) + .thenCompose(nsEntryFilters -> { + if (nsEntryFilters != null) { + return CompletableFuture.completedFuture(nsEntryFilters); } - return getNamespacePoliciesAsync(namespaceName) - .thenApply(policies -> policies.entryFilters) - .thenCompose(nsEntryFilters -> { - if (nsEntryFilters != null) { - return CompletableFuture.completedFuture(nsEntryFilters); - } - return CompletableFuture.completedFuture(new EntryFilters(String.join(",", - pulsar().getConfiguration().getEntryFilterNames()))); - }); + return CompletableFuture.completedFuture(new EntryFilters(String.join(",", + pulsar().getConfiguration().getEntryFilterNames()))); }); }); } protected CompletableFuture internalSetEntryFilters(EntryFilters entryFilters, boolean isGlobal) { - - return validateTopicPolicyOperationAsync(topicName, PolicyName.ENTRY_FILTERS, PolicyOperation.WRITE) - .thenAccept(__ -> validateEntryFilters(entryFilters)) - .thenCompose(__ -> getTopicPoliciesAsyncWithRetry(topicName, isGlobal) + validateEntryFilters(entryFilters); + return getTopicPoliciesAsyncWithRetry(topicName, isGlobal) .thenCompose(op -> { TopicPolicies topicPolicies = op.orElseGet(TopicPolicies::new); topicPolicies.setEntryFilters(entryFilters); topicPolicies.setIsGlobal(isGlobal); return pulsar().getTopicPoliciesService() .updateTopicPoliciesAsync(topicName, topicPolicies); - })); + }); } protected CompletableFuture internalRemoveEntryFilters(boolean isGlobal) { - return validateTopicPolicyOperationAsync(topicName, PolicyName.ENTRY_FILTERS, PolicyOperation.WRITE) - .thenCompose(__ -> - getTopicPoliciesAsyncWithRetry(topicName, isGlobal) + return getTopicPoliciesAsyncWithRetry(topicName, isGlobal) .thenCompose(op -> { if (!op.isPresent()) { return CompletableFuture.completedFuture(null); @@ -5499,7 +5355,7 @@ protected CompletableFuture internalRemoveEntryFilters(boolean isGlobal) { op.get().setEntryFilters(null); op.get().setIsGlobal(isGlobal); return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, op.get()); - })); + }); } protected CompletableFuture validateShadowTopics(List shadowTopics) { @@ -5512,8 +5368,10 @@ protected CompletableFuture validateShadowTopics(List shadowTopics "Only persistent topic can be set as shadow topic")); } futures.add(pulsar().getNamespaceService().checkTopicExists(shadowTopicName) - .thenAccept(isExists -> { - if (!isExists) { + .thenAccept(info -> { + boolean exists = info.isExists(); + info.recycle(); + if (!exists) { throw new RestException(Status.PRECONDITION_FAILED, "Shadow topic [" + shadowTopic + "] not exists."); } @@ -5535,8 +5393,7 @@ protected CompletableFuture internalSetShadowTopic(List shadowTopi return FutureUtil.failedFuture(new RestException(Status.PRECONDITION_FAILED, "Cannot specify empty shadow topics, please use remove command instead.")); } - return validateTopicPolicyOperationAsync(topicName, PolicyName.SHADOW_TOPIC, PolicyOperation.WRITE) - .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) + return validatePoliciesReadOnlyAccessAsync() .thenCompose(__ -> validateShadowTopics(shadowTopics)) .thenCompose(__ -> getTopicPoliciesAsyncWithRetry(topicName)) .thenCompose(op -> { @@ -5548,8 +5405,7 @@ protected CompletableFuture internalSetShadowTopic(List shadowTopi } protected CompletableFuture internalDeleteShadowTopics() { - return validateTopicPolicyOperationAsync(topicName, PolicyName.SHADOW_TOPIC, PolicyOperation.WRITE) - .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) + return validatePoliciesReadOnlyAccessAsync() .thenCompose(shadowTopicName -> getTopicPoliciesAsyncWithRetry(topicName)) .thenCompose(op -> { TopicPolicies topicPolicies = op.orElseGet(TopicPolicies::new); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java index 0bab772044a6d..886db9c7abb37 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java @@ -31,12 +31,15 @@ import javax.ws.rs.core.Response; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.admin.AdminResource; +import org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage; import org.apache.pulsar.broker.service.schema.SchemaRegistry.SchemaAndMetadata; import org.apache.pulsar.broker.service.schema.SchemaRegistryService; import org.apache.pulsar.broker.web.RestException; +import org.apache.pulsar.client.impl.schema.SchemaUtils; import org.apache.pulsar.client.internal.DefaultImplementation; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; +import org.apache.pulsar.common.policies.data.SchemaMetadata; import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.protocol.schema.GetAllVersionsSchemaResponse; import org.apache.pulsar.common.protocol.schema.GetSchemaResponse; @@ -104,6 +107,13 @@ public CompletableFuture> getAllSchemasAsync(boolean aut }); } + public CompletableFuture getSchemaMetadataAsync(boolean authoritative) { + String schemaId = getSchemaId(); + BookkeeperSchemaStorage storage = (BookkeeperSchemaStorage) pulsar().getSchemaStorage(); + return validateOwnershipAndOperationAsync(authoritative, TopicOperation.GET_METADATA) + .thenCompose(__ -> storage.getSchemaMetadata(schemaId)); + } + public CompletableFuture deleteSchemaAsync(boolean authoritative, boolean force) { return validateDestinationAndAdminOperationAsync(authoritative) .thenCompose(__ -> { @@ -146,8 +156,13 @@ public CompletableFuture> testCompati .thenCompose(__ -> getSchemaCompatibilityStrategyAsync()) .thenCompose(strategy -> { String schemaId = getSchemaId(); + final SchemaType schemaType = SchemaType.valueOf(payload.getType()); + byte[] data = payload.getSchema().getBytes(StandardCharsets.UTF_8); + if (schemaType.getValue() == SchemaType.KEY_VALUE.getValue()) { + data = SchemaUtils.convertKeyValueDataStringToSchemaInfoSchema(data); + } return pulsar().getSchemaRegistryService().isCompatible(schemaId, - SchemaData.builder().data(payload.getSchema().getBytes(StandardCharsets.UTF_8)) + SchemaData.builder().data(data) .isDeleted(false) .timestamp(clock.millis()).type(SchemaType.valueOf(payload.getType())) .user(defaultIfEmpty(clientAppId(), "")) @@ -161,10 +176,14 @@ public CompletableFuture getVersionBySchemaAsync(PostSchemaPayload payload return validateOwnershipAndOperationAsync(authoritative, TopicOperation.GET_METADATA) .thenCompose(__ -> { String schemaId = getSchemaId(); + final SchemaType schemaType = SchemaType.valueOf(payload.getType()); + byte[] data = payload.getSchema().getBytes(StandardCharsets.UTF_8); + if (schemaType.getValue() == SchemaType.KEY_VALUE.getValue()) { + data = SchemaUtils.convertKeyValueDataStringToSchemaInfoSchema(data); + } return pulsar().getSchemaRegistryService() .findSchemaVersion(schemaId, - SchemaData.builder().data(payload.getSchema().getBytes(StandardCharsets.UTF_8)) - .isDeleted(false).timestamp(clock.millis()) + SchemaData.builder().data(data).isDeleted(false).timestamp(clock.millis()) .type(SchemaType.valueOf(payload.getType())) .user(defaultIfEmpty(clientAppId(), "")) .props(payload.getProperties()).build()); @@ -228,7 +247,7 @@ private CompletableFuture validateOwnershipAndOperationAsync(boolean autho protected boolean shouldPrintErrorLog(Throwable ex) { - return !isRedirectException(ex) && !isNotFoundException(ex); + return isNot307And404Exception(ex); } private static final Logger log = LoggerFactory.getLogger(SchemasResourceBase.class); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SinksBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SinksBase.java index 80ad72d6f9aa9..0a76fe27e0a35 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SinksBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SinksBase.java @@ -389,8 +389,9 @@ public List listSinks(@ApiParam(value = "The tenant of a Pulsar Sink") } @POST - @ApiOperation(value = "Restart an instance of a Pulsar Sink", response = Void.class) + @ApiOperation(value = "Restart an instance of a Pulsar Sink") @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this sink"), @ApiResponse(code = 400, message = "Invalid restart request"), @ApiResponse(code = 401, message = "The client is not authorized to perform this operation"), @@ -415,8 +416,9 @@ public void restartSink(@ApiParam(value = "The tenant of a Pulsar Sink") } @POST - @ApiOperation(value = "Restart all instances of a Pulsar Sink", response = Void.class) + @ApiOperation(value = "Restart all instances of a Pulsar Sink") @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 400, message = "Invalid restart request"), @ApiResponse(code = 401, message = "The client is not authorized to perform this operation"), @ApiResponse(code = 404, message = "The Pulsar Sink does not exist"), @@ -436,8 +438,9 @@ public void restartSink(@ApiParam(value = "The tenant of a Pulsar Sink") } @POST - @ApiOperation(value = "Stop an instance of a Pulsar Sink", response = Void.class) + @ApiOperation(value = "Stop an instance of a Pulsar Sink") @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 400, message = "Invalid stop request"), @ApiResponse(code = 404, message = "The Pulsar Sink instance does not exist"), @ApiResponse(code = 500, message = @@ -460,8 +463,9 @@ public void stopSink(@ApiParam(value = "The tenant of a Pulsar Sink") } @POST - @ApiOperation(value = "Stop all instances of a Pulsar Sink", response = Void.class) + @ApiOperation(value = "Stop all instances of a Pulsar Sink") @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 400, message = "Invalid stop request"), @ApiResponse(code = 404, message = "The Pulsar Sink does not exist"), @ApiResponse(code = 500, message = @@ -481,8 +485,9 @@ public void stopSink(@ApiParam(value = "The tenant of a Pulsar Sink") } @POST - @ApiOperation(value = "Start an instance of a Pulsar Sink", response = Void.class) + @ApiOperation(value = "Start an instance of a Pulsar Sink") @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 400, message = "Invalid start request"), @ApiResponse(code = 404, message = "The Pulsar Sink does not exist"), @ApiResponse(code = 500, message = @@ -505,8 +510,9 @@ public void startSink(@ApiParam(value = "The tenant of a Pulsar Sink") } @POST - @ApiOperation(value = "Start all instances of a Pulsar Sink", response = Void.class) + @ApiOperation(value = "Start all instances of a Pulsar Sink") @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 400, message = "Invalid start request"), @ApiResponse(code = 404, message = "The Pulsar Sink does not exist"), @ApiResponse(code = 500, message = diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SourcesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SourcesBase.java index 4af0afc0d6ec5..0d037dd42362f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SourcesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SourcesBase.java @@ -323,7 +323,7 @@ public SourceStatus getSourceStatus( @ApiOperation( value = "Lists all Pulsar Sources currently deployed in a given namespace", response = String.class, - responseContainer = "Collection" + responseContainer = "List" ) @ApiResponses(value = { @ApiResponse(code = 400, message = "Invalid request"), @@ -342,8 +342,9 @@ public List listSources( } @POST - @ApiOperation(value = "Restart an instance of a Pulsar Source", response = Void.class) + @ApiOperation(value = "Restart an instance of a Pulsar Source") @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this source"), @ApiResponse(code = 400, message = "Invalid request"), @ApiResponse(code = 401, message = "Client is not authorized to perform operation"), @@ -365,8 +366,9 @@ public void restartSource( } @POST - @ApiOperation(value = "Restart all instances of a Pulsar Source", response = Void.class) + @ApiOperation(value = "Restart all instances of a Pulsar Source") @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 400, message = "Invalid request"), @ApiResponse(code = 401, message = "Client is not authorized to perform operation"), @ApiResponse(code = 404, message = "Not Found(The Pulsar Source doesn't exist)"), @@ -386,8 +388,9 @@ public void restartSource( } @POST - @ApiOperation(value = "Stop instance of a Pulsar Source", response = Void.class) + @ApiOperation(value = "Stop instance of a Pulsar Source") @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 400, message = "Invalid request"), @ApiResponse(code = 401, message = "Client is not authorized to perform operation"), @ApiResponse(code = 404, message = "Not Found(The Pulsar Source doesn't exist)"), @@ -407,8 +410,9 @@ public void stopSource( } @POST - @ApiOperation(value = "Stop all instances of a Pulsar Source", response = Void.class) + @ApiOperation(value = "Stop all instances of a Pulsar Source") @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 400, message = "Invalid request"), @ApiResponse(code = 401, message = "Client is not authorized to perform operation"), @ApiResponse(code = 404, message = "Not Found(The Pulsar Source doesn't exist)"), @@ -428,8 +432,9 @@ public void stopSource( } @POST - @ApiOperation(value = "Start an instance of a Pulsar Source", response = Void.class) + @ApiOperation(value = "Start an instance of a Pulsar Source") @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 400, message = "Invalid request"), @ApiResponse(code = 401, message = "Client is not authorized to perform operation"), @ApiResponse(code = 404, message = "Not Found(The Pulsar Source doesn't exist)"), @@ -449,8 +454,9 @@ public void startSource( } @POST - @ApiOperation(value = "Start all instances of a Pulsar Source", response = Void.class) + @ApiOperation(value = "Start all instances of a Pulsar Source") @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 400, message = "Invalid request"), @ApiResponse(code = 401, message = "Client is not authorized to perform operation"), @ApiResponse(code = 404, message = "Not Found(The Pulsar Source doesn't exist)"), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java index b93f3e3c6ebcc..0d1f79a09dc14 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TenantsBase.java @@ -103,7 +103,9 @@ public void getTenantAdmin(@Suspended final AsyncResponse asyncResponse, @PUT @Path("/{tenant}") @ApiOperation(value = "Create a new tenant.", notes = "This operation requires Pulsar super-user privileges.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), @ApiResponse(code = 409, message = "Tenant already exists"), @ApiResponse(code = 412, message = "Tenant name is not valid"), @ApiResponse(code = 412, message = "Clusters can not be empty"), @@ -122,6 +124,7 @@ public void createTenant(@Suspended final AsyncResponse asyncResponse, validateSuperUserAccessAsync() .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) .thenCompose(__ -> validateClustersAsync(tenantInfo)) + .thenCompose(__ -> validateAdminRoleAsync(tenantInfo)) .thenCompose(__ -> tenantResources().tenantExistsAsync(tenant)) .thenAccept(exist -> { if (exist) { @@ -155,7 +158,9 @@ public void createTenant(@Suspended final AsyncResponse asyncResponse, @Path("/{tenant}") @ApiOperation(value = "Update the admins for a tenant.", notes = "This operation requires Pulsar super-user privileges.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), @ApiResponse(code = 404, message = "Tenant does not exist"), @ApiResponse(code = 409, message = "Tenant already exists"), @ApiResponse(code = 412, message = "Clusters can not be empty"), @@ -167,6 +172,7 @@ public void updateTenant(@Suspended final AsyncResponse asyncResponse, validateSuperUserAccessAsync() .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) .thenCompose(__ -> validateClustersAsync(newTenantAdmin)) + .thenCompose(__ -> validateAdminRoleAsync(newTenantAdmin)) .thenCompose(__ -> tenantResources().getTenantAsync(tenant)) .thenCompose(tenantAdmin -> { if (!tenantAdmin.isPresent()) { @@ -190,7 +196,9 @@ public void updateTenant(@Suspended final AsyncResponse asyncResponse, @DELETE @Path("/{tenant}") @ApiOperation(value = "Delete a tenant and all namespaces and topics under it.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), @ApiResponse(code = 404, message = "Tenant does not exist"), @ApiResponse(code = 405, message = "Broker doesn't allow forced deletion of tenants"), @ApiResponse(code = 409, message = "The tenant still has active namespaces")}) @@ -236,7 +244,7 @@ protected CompletableFuture internalDeleteTenantAsync(String tenant) { .getPartitionedTopicResources().clearPartitionedTopicTenantAsync(tenant)) .thenCompose(__ -> pulsar().getPulsarResources().getLocalPolicies() .deleteLocalPoliciesTenantAsync(tenant)) - .thenCompose(__ -> pulsar().getPulsarResources().getNamespaceResources() + .thenCompose(__ -> pulsar().getPulsarResources().getLoadBalanceResources().getBundleDataResources() .deleteBundleDataTenantAsync(tenant)); } @@ -282,4 +290,18 @@ private CompletableFuture validateClustersAsync(TenantInfo info) { } }); } + + private CompletableFuture validateAdminRoleAsync(TenantInfoImpl info) { + if (info.getAdminRoles() != null && !info.getAdminRoles().isEmpty()) { + for (String adminRole : info.getAdminRoles()) { + if (!StringUtils.trim(adminRole).equals(adminRole)) { + log.warn("[{}] Failed to validate due to adminRole {} contains whitespace in the beginning or end.", + clientAppId(), adminRole); + return FutureUtil.failedFuture(new RestException(Status.PRECONDITION_FAILED, + "AdminRoles contains whitespace in the beginning or end.")); + } + } + } + return CompletableFuture.completedFuture(null); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java index d596cbdd39db9..55767136f8151 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.admin.impl; +import static javax.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR; import static javax.ws.rs.core.Response.Status.METHOD_NOT_ALLOWED; import static javax.ws.rs.core.Response.Status.NOT_FOUND; import static javax.ws.rs.core.Response.Status.SERVICE_UNAVAILABLE; @@ -33,20 +34,24 @@ import javax.ws.rs.core.Response; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedLedger; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.admin.AdminResource; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; import org.apache.pulsar.broker.web.RestException; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.Transactions; import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.common.api.proto.TxnAction; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; +import org.apache.pulsar.common.policies.data.SnapshotSystemTopicInternalStats; +import org.apache.pulsar.common.policies.data.TransactionBufferInternalStats; import org.apache.pulsar.common.policies.data.TransactionBufferStats; import org.apache.pulsar.common.policies.data.TransactionCoordinatorInfo; import org.apache.pulsar.common.policies.data.TransactionCoordinatorInternalStats; @@ -170,9 +175,10 @@ protected CompletableFuture internalGetTransactionInBu } protected CompletableFuture internalGetTransactionBufferStats(boolean authoritative, - boolean lowWaterMarks) { + boolean lowWaterMarks, + boolean segmentStats) { return getExistingPersistentTopicAsync(authoritative) - .thenApply(topic -> topic.getTransactionBufferStats(lowWaterMarks)); + .thenApply(topic -> topic.getTransactionBufferStats(lowWaterMarks, segmentStats)); } protected CompletableFuture internalGetPendingAckStats( @@ -431,16 +437,78 @@ protected CompletableFuture internalGetPendi ); } + protected CompletableFuture internalGetTransactionBufferInternalStats( + boolean authoritative, boolean metadata) { + TransactionBufferInternalStats transactionBufferInternalStats = new TransactionBufferInternalStats(); + return getExistingPersistentTopicAsync(authoritative) + .thenCompose(topic -> { + AbortedTxnProcessor.SnapshotType snapshotType = topic.getTransactionBuffer().getSnapshotType(); + if (snapshotType == null) { + return FutureUtil.failedFuture(new RestException(NOT_FOUND, + "Transaction buffer Snapshot for the topic does not exist")); + } else if (snapshotType == AbortedTxnProcessor.SnapshotType.Segment) { + transactionBufferInternalStats.snapshotType = snapshotType.toString(); + TopicName segmentTopic = TopicName.get(TopicDomain.persistent.toString(), namespaceName, + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS); + CompletableFuture segmentInternalStatsFuture = + getTxnSnapshotInternalStats(segmentTopic, metadata); + TopicName indexTopic = TopicName.get(TopicDomain.persistent.toString(), + namespaceName, + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_INDEXES); + CompletableFuture segmentIndexInternalStatsFuture = + getTxnSnapshotInternalStats(indexTopic, metadata); + return segmentIndexInternalStatsFuture + .thenCombine(segmentInternalStatsFuture, (indexStats, segmentStats) -> { + transactionBufferInternalStats.segmentIndexInternalStats = indexStats; + transactionBufferInternalStats.segmentInternalStats = segmentStats; + return transactionBufferInternalStats; + }); + } else if (snapshotType == AbortedTxnProcessor.SnapshotType.Single) { + transactionBufferInternalStats.snapshotType = snapshotType.toString(); + TopicName singleSnapshotTopic = TopicName.get(TopicDomain.persistent.toString(), namespaceName, + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT); + return getTxnSnapshotInternalStats(singleSnapshotTopic, metadata) + .thenApply(snapshotSystemTopicInternalStats -> { + transactionBufferInternalStats.singleSnapshotSystemTopicInternalStats = + snapshotSystemTopicInternalStats; + return transactionBufferInternalStats; + }); + } + return FutureUtil.failedFuture(new RestException(INTERNAL_SERVER_ERROR, "Unknown SnapshotType " + + snapshotType)); + }); + } + + private CompletableFuture getTxnSnapshotInternalStats(TopicName topicName, + boolean metadata) { + final PulsarAdmin admin; + try { + admin = pulsar().getAdminClient(); + } catch (PulsarServerException e) { + return FutureUtil.failedFuture(new RestException(e)); + } + return admin.topics().getInternalStatsAsync(topicName.toString(), metadata) + .thenApply(persistentTopicInternalStats -> { + SnapshotSystemTopicInternalStats + snapshotSystemTopicInternalStats = new SnapshotSystemTopicInternalStats(); + snapshotSystemTopicInternalStats.managedLedgerInternalStats = persistentTopicInternalStats; + snapshotSystemTopicInternalStats.managedLedgerName = topicName.getEncodedLocalName(); + return snapshotSystemTopicInternalStats; + }); + } + protected CompletableFuture getExistingPersistentTopicAsync(boolean authoritative) { return validateTopicOwnershipAsync(topicName, authoritative).thenCompose(__ -> { CompletableFuture> topicFuture = pulsar().getBrokerService() .getTopics().get(topicName.toString()); if (topicFuture == null) { - return FutureUtil.failedFuture(new RestException(NOT_FOUND, "Topic not found")); + return FutureUtil.failedFuture(new RestException(NOT_FOUND, + String.format("Topic not found %s", topicName.toString()))); } return topicFuture.thenCompose(optionalTopic -> { if (!optionalTopic.isPresent()) { - return FutureUtil.failedFuture(new RestException(NOT_FOUND, "Topic not found")); + return FutureUtil.failedFuture(new RestException(NOT_FOUND, + String.format("Topic not found %s", topicName.toString()))); } return CompletableFuture.completedFuture((PersistentTopic) optionalTopic.get()); }); @@ -480,7 +548,7 @@ protected CompletableFuture internalScaleTransactionCoordinators(int repli } protected CompletableFuture internalGetPositionStatsPendingAckStats( - boolean authoritative, String subName, PositionImpl position, Integer batchIndex) { + boolean authoritative, String subName, Position position, Integer batchIndex) { CompletableFuture completableFuture = new CompletableFuture<>(); getExistingPersistentTopicAsync(authoritative) .thenAccept(topic -> { @@ -493,4 +561,20 @@ protected CompletableFuture internalGetPositionStatsP }); return completableFuture; } + + protected CompletableFuture internalAbortTransaction(boolean authoritative, long mostSigBits, + long leastSigBits) { + + if (mostSigBits < 0 || mostSigBits > Integer.MAX_VALUE) { + return CompletableFuture.failedFuture(new IllegalArgumentException("mostSigBits out of bounds")); + } + + int partitionIdx = (int) mostSigBits; + + return validateTopicOwnershipAsync( + SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN.getPartition(partitionIdx), authoritative) + .thenCompose(__ -> validateSuperUserAccessAsync()) + .thenCompose(__ -> pulsar().getTransactionMetadataStoreService() + .endTransaction(new TxnID(mostSigBits, leastSigBits), TxnAction.ABORT_VALUE, false)); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java index 153e29506c3d0..9cf394e77f4f2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java @@ -293,8 +293,8 @@ public void getPermissions(@Suspended AsyncResponse response, @PathParam("namespace") String namespace) { validateNamespaceName(property, cluster, namespace); validateNamespaceOperationAsync(NamespaceName.get(property, namespace), NamespaceOperation.GET_PERMISSION) - .thenCompose(__ -> getNamespacePoliciesAsync(namespaceName)) - .thenAccept(policies -> response.resume(policies.auth_policies.getNamespaceAuthentication())) + .thenCompose(__ -> getAuthorizationService().getPermissionsAsync(namespaceName)) + .thenAccept(permissions -> response.resume(permissions)) .exceptionally(ex -> { log.error("Failed to get permissions for namespace {}", namespaceName, ex); resumeAsyncResponseExceptionally(response, ex); @@ -314,8 +314,8 @@ public void getPermissionOnSubscription(@Suspended AsyncResponse response, @PathParam("namespace") String namespace) { validateNamespaceName(property, cluster, namespace); validateNamespaceOperationAsync(NamespaceName.get(property, namespace), NamespaceOperation.GET_PERMISSION) - .thenCompose(__ -> getNamespacePoliciesAsync(namespaceName)) - .thenAccept(policies -> response.resume(policies.auth_policies.getSubscriptionAuthentication())) + .thenCompose(__ -> getAuthorizationService().getSubscriptionPermissionsAsync(namespaceName)) + .thenAccept(permissions -> response.resume(permissions)) .exceptionally(ex -> { log.error("[{}] Failed to get permissions on subscription for namespace {}: {} ", clientAppId(), namespaceName, @@ -1709,5 +1709,54 @@ public void setSchemaAutoUpdateCompatibilityStrategy(@PathParam("tenant") String internalSetSchemaAutoUpdateCompatibilityStrategy(strategy); } + @POST + @Path("/{property}/{cluster}/{namespace}/migration") + @ApiOperation(hidden = true, value = "Update migration for all topics in a namespace") + @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Property or cluster or namespace doesn't exist") }) + public void enableMigration(@PathParam("property") String property, + @PathParam("cluster") String cluster, + @PathParam("namespace") String namespace, + boolean migrated) { + validateNamespaceName(property, cluster, namespace); + internalEnableMigration(migrated); + } + + @PUT + @Path("/{property}/{cluster}/{namespace}/policy") + @ApiOperation(value = "Creates a new namespace with the specified policies") + @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Property or cluster or namespace doesn't exist"), + @ApiResponse(code = 409, message = "Namespace already exists"), + @ApiResponse(code = 412, message = "Namespace name is not valid") }) + public void createNamespace(@Suspended AsyncResponse response, + @PathParam("property") String property, + @PathParam("cluster") String cluster, + @PathParam("namespace") String namespace, + @ApiParam(value = "Policies for the namespace") Policies policies) { + validateNamespaceName(property, cluster, namespace); + CompletableFuture ret; + if (!namespaceName.isGlobal()) { + // If the namespace is non global, make sure property has the access on the cluster. For global namespace, + // same check is made at the time of setting replication. + ret = validateClusterForTenantAsync(namespaceName.getTenant(), namespaceName.getCluster()); + } else { + ret = CompletableFuture.completedFuture(null); + } + + ret.thenApply(__ -> getDefaultPolicesIfNull(policies)).thenCompose(this::internalCreateNamespace) + .thenAccept(__ -> response.resume(Response.noContent().build())) + .exceptionally(ex -> { + Throwable root = FutureUtil.unwrapCompletionException(ex); + if (root instanceof MetadataStoreException.AlreadyExistsException) { + response.resume(new RestException(Status.CONFLICT, "Namespace already exists")); + } else { + log.error("[{}] Failed to create namespace {}", clientAppId(), namespaceName, ex); + resumeAsyncResponseExceptionally(response, ex); + } + return null; + }); + } + private static final Logger log = LoggerFactory.getLogger(Namespaces.class); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/NonPersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/NonPersistentTopics.java index 0d857f2211f41..1c1dd74719641 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/NonPersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/NonPersistentTopics.java @@ -284,11 +284,12 @@ public List getListFromBundle(@PathParam("property") String property, @P } } - private Topic getTopicReference(TopicName topicName) { + private Topic getTopicReference(final TopicName topicName) { try { return pulsar().getBrokerService().getTopicIfExists(topicName.toString()) .get(config().getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS) - .orElseThrow(() -> new RestException(Status.NOT_FOUND, "Topic not found")); + .orElseThrow(() -> new RestException(Status.NOT_FOUND, + String.format("Topic not found %s", topicName.toString()))); } catch (ExecutionException e) { throw new RuntimeException(e.getCause()); } catch (InterruptedException | TimeoutException e) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java index e9bb1a4054764..43224248fdca0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java @@ -41,8 +41,10 @@ import javax.ws.rs.container.Suspended; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import org.apache.pulsar.broker.admin.AdminResource; import org.apache.pulsar.broker.admin.impl.PersistentTopicsBase; import org.apache.pulsar.broker.service.BrokerServiceException; +import org.apache.pulsar.broker.service.GetStatsOptions; import org.apache.pulsar.broker.web.RestException; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.impl.ResetCursorData; @@ -320,8 +322,15 @@ public void getPartitionedMetadata( internalGetPartitionedMetadataAsync(authoritative, checkAllowAutoCreation) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { - log.error("[{}] Failed to get partitioned metadata topic {}", clientAppId(), topicName, ex); + Throwable t = FutureUtil.unwrapCompletionException(ex); + if (!isRedirectException(t)) { + if (AdminResource.isNotFoundException(t)) { + log.error("[{}] Failed to get partitioned metadata topic {}: {}", + clientAppId(), topicName, ex.getMessage()); + } else { + log.error("[{}] Failed to get partitioned metadata topic {}", + clientAppId(), topicName, t); + } } resumeAsyncResponseExceptionally(asyncResponse, ex); return null; @@ -436,7 +445,9 @@ public void getStats( @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @QueryParam("getPreciseBacklog") @DefaultValue("false") boolean getPreciseBacklog) { validateTopicName(property, cluster, namespace, encodedTopic); - internalGetStatsAsync(authoritative, getPreciseBacklog, false, false) + GetStatsOptions getStatsOptions = + new GetStatsOptions(getPreciseBacklog, false, false, false, false); + internalGetStatsAsync(authoritative, getStatsOptions) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { // If the exception is not redirect exception we need to log it. @@ -503,7 +514,8 @@ public void getPartitionedStats(@Suspended final AsyncResponse asyncResponse, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { try { validateTopicName(property, cluster, namespace, encodedTopic); - internalGetPartitionedStats(asyncResponse, authoritative, perPartition, false, false, false); + GetStatsOptions getStatsOptions = new GetStatsOptions(false, false, false, false, false); + internalGetPartitionedStats(asyncResponse, authoritative, perPartition, getStatsOptions); } catch (WebApplicationException wae) { asyncResponse.resume(wae); } catch (Exception e) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/SchemasResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/SchemasResource.java index edc600707a120..0d6c3814bf863 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/SchemasResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/SchemasResource.java @@ -43,6 +43,7 @@ import org.apache.pulsar.broker.admin.impl.SchemasResourceBase; import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; import org.apache.pulsar.broker.service.schema.exceptions.InvalidSchemaDataException; +import org.apache.pulsar.common.policies.data.SchemaMetadata; import org.apache.pulsar.common.protocol.schema.DeleteSchemaResponse; import org.apache.pulsar.common.protocol.schema.GetAllVersionsSchemaResponse; import org.apache.pulsar.common.protocol.schema.GetSchemaResponse; @@ -170,6 +171,37 @@ public void getAllSchemas( }); } + @GET + @Path("/{tenant}/{cluster}/{namespace}/{topic}/metadata") + @Produces(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Get the schema metadata of a topic", response = SchemaMetadata.class) + @ApiResponses(value = { + @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), + @ApiResponse(code = 401, message = "Client is not authorized or Don't have admin permission"), + @ApiResponse(code = 403, message = "Client is not authenticated"), + @ApiResponse(code = 404, + message = "Tenant or Namespace or Topic doesn't exist; or Schema is not found for this topic"), + @ApiResponse(code = 412, message = "Failed to find the ownership for the topic"), + @ApiResponse(code = 500, message = "Internal Server Error"), + }) + public void getSchemaMetadata( + @PathParam("tenant") String tenant, + @PathParam("cluster") String cluster, + @PathParam("namespace") String namespace, + @PathParam("topic") String topic, + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, + @Suspended final AsyncResponse response + ) { + validateTopicName(tenant, cluster, namespace, topic); + getSchemaMetadataAsync(authoritative) + .thenAccept(response::resume) + .exceptionally(ex -> { + log.error("[{}] Failed to get schema metadata for topic {}", clientAppId(), topicName, ex); + resumeAsyncResponseExceptionally(response, ex); + return null; + }); + } + @DELETE @Path("/{tenant}/{cluster}/{namespace}/{topic}/schema") @Produces(MediaType.APPLICATION_JSON) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Bookies.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Bookies.java index a50bc7515ff6f..c7b09ca9b0aa1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Bookies.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Bookies.java @@ -20,6 +20,7 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; import io.swagger.annotations.ApiResponse; import io.swagger.annotations.ApiResponses; import java.util.ArrayList; @@ -113,7 +114,7 @@ public void getBookieRackInfo(@Suspended final AsyncResponse asyncResponse, asyncResponse.resume(bi.get()); } else { asyncResponse.resume(new RestException(Status.NOT_FOUND, - "Bookie address not found: " + bookieAddress)); + "Bookie rack placement configuration not found: " + bookieAddress)); } }).exceptionally(ex -> { asyncResponse.resume(ex); @@ -124,7 +125,10 @@ public void getBookieRackInfo(@Suspended final AsyncResponse asyncResponse, @DELETE @Path("/racks-info/{bookie}") @ApiOperation(value = "Removed the rack placement information for a specific bookie in the cluster") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission")}) + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission") + }) public void deleteBookieRackInfo(@Suspended final AsyncResponse asyncResponse, @PathParam("bookie") String bookieAddress) throws Exception { validateSuperUserAccess(); @@ -136,7 +140,7 @@ public void deleteBookieRackInfo(@Suspended final AsyncResponse asyncResponse, if (!brc.removeBookie(bookieAddress)) { asyncResponse.resume(new RestException(Status.NOT_FOUND, - "Bookie address not found: " + bookieAddress)); + "Bookie rack placement configuration not found: " + bookieAddress)); } return brc; @@ -153,11 +157,17 @@ public void deleteBookieRackInfo(@Suspended final AsyncResponse asyncResponse, @Path("/racks-info/{bookie}") @ApiOperation(value = "Updates the rack placement information for a specific bookie in the cluster (note." + " bookie address format:`address:port`)") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission")}) + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission")} + ) public void updateBookieRackInfo(@Suspended final AsyncResponse asyncResponse, + @ApiParam(value = "The bookie address", required = true) @PathParam("bookie") String bookieAddress, + @ApiParam(value = "The group", required = true) @QueryParam("group") String group, - BookieInfo bookieInfo) throws Exception { + @ApiParam(value = "The bookie info", required = true) + BookieInfo bookieInfo) throws Exception { validateSuperUserAccess(); if (group == null) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/BrokerStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/BrokerStats.java index aba6cb1a0aba4..6f280e8d197f8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/BrokerStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/BrokerStats.java @@ -61,7 +61,12 @@ public StreamingOutput getTopics2() throws Exception { + "sum of all of the resource usage percent is called broker-resource-availability" + "

THIS API IS ONLY FOR USE BY TESTING FOR CONFIRMING NAMESPACE ALLOCATION ALGORITHM", response = ResourceUnit.class, responseContainer = "Map") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Returns broker resource availability as Map>." + + "Since `ResourceUnit` is an interface, its specific content is not determinable via class " + + "reflection. Refer to the source code or interface tests for detailed type definitions.", + response = Map.class), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 409, message = "Load-manager doesn't support operation") }) public Map> getBrokerResourceAvailability(@PathParam("tenant") String tenant, @PathParam("namespace") String namespace) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index b4f1194f92f0a..54cceaf09e9fe 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -18,12 +18,13 @@ */ package org.apache.pulsar.broker.admin.v2; -import static org.apache.pulsar.common.policies.data.PoliciesUtil.getBundles; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import io.swagger.annotations.ApiResponse; import io.swagger.annotations.ApiResponses; +import io.swagger.annotations.Example; +import io.swagger.annotations.ExampleProperty; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import java.util.HashSet; @@ -59,9 +60,11 @@ import org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType; import org.apache.pulsar.common.policies.data.BookieAffinityGroupData; import org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies; +import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.EntryFilters; import org.apache.pulsar.common.policies.data.InactiveTopicPolicies; import org.apache.pulsar.common.policies.data.NamespaceOperation; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; import org.apache.pulsar.common.policies.data.PersistencePolicies; import org.apache.pulsar.common.policies.data.Policies; @@ -73,6 +76,12 @@ import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.policies.data.SubscribeRate; import org.apache.pulsar.common.policies.data.SubscriptionAuthMode; +import org.apache.pulsar.common.policies.data.TopicHashPositions; +import org.apache.pulsar.common.policies.data.impl.AutoSubscriptionCreationOverrideImpl; +import org.apache.pulsar.common.policies.data.impl.AutoTopicCreationOverrideImpl; +import org.apache.pulsar.common.policies.data.impl.BacklogQuotaImpl; +import org.apache.pulsar.common.policies.data.impl.BookieAffinityGroupDataImpl; +import org.apache.pulsar.common.policies.data.impl.BundlesDataImpl; import org.apache.pulsar.common.policies.data.impl.DispatchRateImpl; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.MetadataStoreException; @@ -152,7 +161,9 @@ public void getPolicies(@Suspended AsyncResponse response, @PUT @Path("/{tenant}/{namespace}") @ApiOperation(value = "Creates a new namespace with the specified policies") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster doesn't exist"), @ApiResponse(code = 409, message = "Namespace already exists"), @ApiResponse(code = 412, message = "Namespace name is not valid") }) @@ -180,6 +191,7 @@ public void createNamespace(@Suspended AsyncResponse response, @Path("/{tenant}/{namespace}") @ApiOperation(value = "Delete a namespace and all the topics under it.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace"), @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), @@ -208,6 +220,7 @@ public void deleteNamespace(@Suspended final AsyncResponse asyncResponse, @PathP @Path("/{tenant}/{namespace}/{bundle}") @ApiOperation(value = "Delete a namespace bundle and all the topics under it.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace"), @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), @@ -231,7 +244,10 @@ public void deleteNamespaceBundle(@Suspended AsyncResponse response, @PathParam( @GET @Path("/{tenant}/{namespace}/permissions") - @ApiOperation(value = "Retrieve the permissions for a namespace.") + @ApiOperation(value = "Retrieve the permissions for a namespace.", + notes = "Returns a nested map structure which Swagger does not fully support for display. " + + "Structure: Map>. Please refer to this structure for details.", + response = AuthAction.class, responseContainer = "Map") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), @ApiResponse(code = 409, message = "Namespace is not empty") }) @@ -240,8 +256,8 @@ public void getPermissions(@Suspended AsyncResponse response, @PathParam("namespace") String namespace) { validateNamespaceName(tenant, namespace); validateNamespaceOperationAsync(NamespaceName.get(tenant, namespace), NamespaceOperation.GET_PERMISSION) - .thenCompose(__ -> getNamespacePoliciesAsync(namespaceName)) - .thenAccept(policies -> response.resume(policies.auth_policies.getNamespaceAuthentication())) + .thenCompose(__ -> getAuthorizationService().getPermissionsAsync(namespaceName)) + .thenAccept(permissions -> response.resume(permissions)) .exceptionally(ex -> { log.error("Failed to get permissions for namespace {}", namespaceName, ex); resumeAsyncResponseExceptionally(response, ex); @@ -251,7 +267,10 @@ public void getPermissions(@Suspended AsyncResponse response, @GET @Path("/{tenant}/{namespace}/permissions/subscription") - @ApiOperation(value = "Retrieve the permissions for a subscription.") + @ApiOperation(value = "Retrieve the permissions for a subscription.", + notes = "Returns a nested map structure which Swagger does not fully support for display. " + + "Structure: Map>. Please refer to this structure for details.", + response = String.class, responseContainer = "Map") @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), @ApiResponse(code = 409, message = "Namespace is not empty")}) @@ -260,8 +279,8 @@ public void getPermissionOnSubscription(@Suspended AsyncResponse response, @PathParam("namespace") String namespace) { validateNamespaceName(tenant, namespace); validateNamespaceOperationAsync(NamespaceName.get(tenant, namespace), NamespaceOperation.GET_PERMISSION) - .thenCompose(__ -> getNamespacePoliciesAsync(namespaceName)) - .thenAccept(policies -> response.resume(policies.auth_policies.getSubscriptionAuthentication())) + .thenCompose(__ -> getAuthorizationService().getSubscriptionPermissionsAsync(namespaceName)) + .thenAccept(permissions -> response.resume(permissions)) .exceptionally(ex -> { log.error("[{}] Failed to get permissions on subscription for namespace {}: {} ", clientAppId(), namespaceName, ex.getCause().getMessage(), ex); @@ -273,7 +292,9 @@ public void getPermissionOnSubscription(@Suspended AsyncResponse response, @POST @Path("/{tenant}/{namespace}/permissions/{role}") @ApiOperation(value = "Grant a new permission to a role on a namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @ApiResponse(code = 501, message = "Authorization is not enabled")}) @@ -297,7 +318,9 @@ public void grantPermissionOnNamespace(@Suspended AsyncResponse asyncResponse, @Path("/{property}/{namespace}/permissions/subscription/{subscription}") @ApiOperation(hidden = true, value = "Grant a new permission to roles for a subscription." + "[Tenant admin is allowed to perform this operation]") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Property or cluster or namespace doesn't exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @ApiResponse(code = 501, message = "Authorization is not enabled") }) @@ -321,7 +344,9 @@ public void grantPermissionOnSubscription(@Suspended AsyncResponse asyncResponse @DELETE @Path("/{tenant}/{namespace}/permissions/{role}") @ApiOperation(value = "Revoke all permissions to a role on a namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist") }) public void revokePermissionsOnNamespace(@Suspended AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @@ -340,7 +365,9 @@ public void revokePermissionsOnNamespace(@Suspended AsyncResponse asyncResponse, @DELETE @Path("/{property}/{namespace}/permissions/{subscription}/{role}") @ApiOperation(hidden = true, value = "Revoke subscription admin-api access permission for a role.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Property or cluster or namespace doesn't exist") }) public void revokePermissionOnSubscription(@Suspended AsyncResponse asyncResponse, @PathParam("property") String property, @@ -360,7 +387,7 @@ public void revokePermissionOnSubscription(@Suspended AsyncResponse asyncRespons @GET @Path("/{tenant}/{namespace}/replication") @ApiOperation(value = "Get the replication clusters for a namespace.", - response = String.class, responseContainer = "List") + response = String.class, responseContainer = "Set") @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), @ApiResponse(code = 412, message = "Namespace is not global")}) @@ -381,7 +408,9 @@ public void getNamespaceReplicationClusters(@Suspended AsyncResponse asyncRespon @POST @Path("/{tenant}/{namespace}/replication") @ApiOperation(value = "Set the replication clusters for a namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), @ApiResponse(code = 409, message = "Peer-cluster can't be part of replication-cluster"), @ApiResponse(code = 412, message = "Namespace is not global or invalid cluster ids") }) @@ -422,7 +451,9 @@ public void getNamespaceMessageTTL(@Suspended AsyncResponse asyncResponse, @Path @POST @Path("/{tenant}/{namespace}/messageTTL") @ApiOperation(value = "Set message TTL in seconds for namespace") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), @ApiResponse(code = 412, message = "Invalid TTL") }) public void setNamespaceMessageTTL(@Suspended AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @@ -442,7 +473,9 @@ public void setNamespaceMessageTTL(@Suspended AsyncResponse asyncResponse, @Path @DELETE @Path("/{tenant}/{namespace}/messageTTL") @ApiOperation(value = "Remove message TTL in seconds for namespace") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), @ApiResponse(code = 412, message = "Invalid TTL")}) public void removeNamespaceMessageTTL(@Suspended AsyncResponse asyncResponse, @@ -460,14 +493,15 @@ public void removeNamespaceMessageTTL(@Suspended AsyncResponse asyncResponse, @GET @Path("/{tenant}/{namespace}/subscriptionExpirationTime") - @ApiOperation(value = "Get the subscription expiration time for the namespace") + @ApiOperation(value = "Get the subscription expiration time for the namespace", response = Integer.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist") }) public void getSubscriptionExpirationTime(@Suspended AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace) { validateNamespaceName(tenant, namespace); - validateAdminAccessForTenantAsync(tenant) + validateNamespacePolicyOperationAsync(namespaceName, PolicyName.SUBSCRIPTION_EXPIRATION_TIME, + PolicyOperation.READ) .thenCompose(__ -> getNamespacePoliciesAsync(namespaceName)) .thenAccept(policies -> asyncResponse.resume(policies.subscription_expiration_time_minutes)) .exceptionally(ex -> { @@ -481,7 +515,9 @@ public void getSubscriptionExpirationTime(@Suspended AsyncResponse asyncResponse @POST @Path("/{tenant}/{namespace}/subscriptionExpirationTime") @ApiOperation(value = "Set subscription expiration time in minutes for namespace") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), @ApiResponse(code = 412, message = "Invalid expiration time")}) public void setSubscriptionExpirationTime(@Suspended AsyncResponse asyncResponse, @@ -504,7 +540,9 @@ public void setSubscriptionExpirationTime(@Suspended AsyncResponse asyncResponse @DELETE @Path("/{tenant}/{namespace}/subscriptionExpirationTime") @ApiOperation(value = "Remove subscription expiration time for namespace") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist")}) public void removeSubscriptionExpirationTime(@Suspended AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @@ -522,7 +560,7 @@ public void removeSubscriptionExpirationTime(@Suspended AsyncResponse asyncRespo @GET @Path("/{tenant}/{namespace}/deduplication") - @ApiOperation(value = "Get broker side deduplication for all topics in a namespace") + @ApiOperation(value = "Get broker side deduplication for all topics in a namespace", response = Boolean.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist") }) public void getDeduplication(@Suspended AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @@ -540,7 +578,9 @@ public void getDeduplication(@Suspended AsyncResponse asyncResponse, @PathParam( @POST @Path("/{tenant}/{namespace}/deduplication") @ApiOperation(value = "Enable or disable broker side deduplication for all topics in a namespace") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist") }) public void modifyDeduplication(@Suspended AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @@ -560,7 +600,9 @@ public void modifyDeduplication(@Suspended AsyncResponse asyncResponse, @PathPar @DELETE @Path("/{tenant}/{namespace}/deduplication") @ApiOperation(value = "Remove broker side deduplication for all topics in a namespace") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist") }) public void removeDeduplication(@Suspended AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace) { @@ -577,7 +619,7 @@ public void removeDeduplication(@Suspended AsyncResponse asyncResponse, @PathPar @GET @Path("/{tenant}/{namespace}/autoTopicCreation") - @ApiOperation(value = "Get autoTopicCreation info in a namespace") + @ApiOperation(value = "Get autoTopicCreation info in a namespace", response = AutoTopicCreationOverrideImpl.class) @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist")}) public void getAutoTopicCreation(@Suspended AsyncResponse asyncResponse, @@ -596,7 +638,9 @@ public void getAutoTopicCreation(@Suspended AsyncResponse asyncResponse, @POST @Path("/{tenant}/{namespace}/autoTopicCreation") @ApiOperation(value = "Override broker's allowAutoTopicCreation setting for a namespace") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), @ApiResponse(code = 406, message = "The number of partitions should be less than or" + " equal to maxNumPartitionsPerPartitionedTopic"), @@ -632,7 +676,9 @@ public void setAutoTopicCreation( @DELETE @Path("/{tenant}/{namespace}/autoTopicCreation") @ApiOperation(value = "Remove override of broker's allowAutoTopicCreation in a namespace") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist") }) public void removeAutoTopicCreation(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace) { @@ -660,7 +706,9 @@ public void removeAutoTopicCreation(@Suspended final AsyncResponse asyncResponse @POST @Path("/{tenant}/{namespace}/autoSubscriptionCreation") @ApiOperation(value = "Override broker's allowAutoSubscriptionCreation setting for a namespace") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), @ApiResponse(code = 400, message = "Invalid autoSubscriptionCreation override")}) public void setAutoSubscriptionCreation( @@ -690,7 +738,8 @@ public void setAutoSubscriptionCreation( @GET @Path("/{tenant}/{namespace}/autoSubscriptionCreation") - @ApiOperation(value = "Get autoSubscriptionCreation info in a namespace") + @ApiOperation(value = "Get autoSubscriptionCreation info in a namespace", + response = AutoSubscriptionCreationOverrideImpl.class) @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist")}) public void getAutoSubscriptionCreation(@Suspended final AsyncResponse asyncResponse, @@ -709,7 +758,9 @@ public void getAutoSubscriptionCreation(@Suspended final AsyncResponse asyncResp @DELETE @Path("/{tenant}/{namespace}/autoSubscriptionCreation") @ApiOperation(value = "Remove override of broker's allowAutoSubscriptionCreation in a namespace") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist") }) public void removeAutoSubscriptionCreation(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace) { @@ -735,7 +786,7 @@ public void removeAutoSubscriptionCreation(@Suspended final AsyncResponse asyncR @GET @Path("/{tenant}/{namespace}/bundles") - @ApiOperation(value = "Get the bundles split data.") + @ApiOperation(value = "Get the bundles split data.", response = BundlesDataImpl.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), @ApiResponse(code = 412, message = "Namespace is not setup to split in bundles") }) @@ -767,6 +818,7 @@ public void getBundlesData(@Suspended final AsyncResponse asyncResponse, + " since it wouldresult in non-persistent message loss and" + " unexpected connection closure to the clients.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace"), @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist"), @@ -799,6 +851,7 @@ public void unloadNamespace(@Suspended final AsyncResponse asyncResponse, @Path("/{tenant}/{namespace}/{bundle}/unload") @ApiOperation(value = "Unload a namespace bundle") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace"), @ApiResponse(code = 404, message = "Namespace doesn't exist"), @ApiResponse(code = 403, message = "Don't have admin permission") }) @@ -828,6 +881,7 @@ public void unloadNamespaceBundle(@Suspended final AsyncResponse asyncResponse, @Path("/{tenant}/{namespace}/{bundle}/split") @ApiOperation(value = "Split a namespace bundle") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace"), @ApiResponse(code = 404, message = "Namespace doesn't exist"), @ApiResponse(code = 403, message = "Don't have admin permission") }) @@ -864,7 +918,7 @@ public void splitNamespaceBundle( @GET @Path("/{tenant}/{namespace}/{bundle}/topicHashPositions") - @ApiOperation(value = "Get hash positions for topics") + @ApiOperation(value = "Get hash positions for topics", response = TopicHashPositions.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist")}) @@ -890,7 +944,9 @@ public void getTopicHashPositions( @POST @Path("/{property}/{namespace}/publishRate") @ApiOperation(hidden = true, value = "Set publish-rate throttling for all topics of the namespace") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission") }) + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission") }) public void setPublishRate(@Suspended AsyncResponse asyncResponse, @PathParam("property") String property, @PathParam("namespace") String namespace, @ApiParam(value = "Publish rate for all topics of the specified namespace") PublishRate publishRate) { @@ -906,7 +962,9 @@ public void setPublishRate(@Suspended AsyncResponse asyncResponse, @PathParam("p @DELETE @Path("/{property}/{namespace}/publishRate") @ApiOperation(hidden = true, value = "Set publish-rate throttling for all topics of the namespace") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission") }) + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission") }) public void removePublishRate(@Suspended AsyncResponse asyncResponse, @PathParam("property") String property, @PathParam("namespace") String namespace) { validateNamespaceName(property, namespace); @@ -924,7 +982,8 @@ public void removePublishRate(@Suspended AsyncResponse asyncResponse, @PathParam @Path("/{property}/{namespace}/publishRate") @ApiOperation(hidden = true, value = "Get publish-rate configured for the namespace, null means publish-rate not configured, " - + "-1 means msg-publish-rate or byte-publish-rate not configured in publish-rate yet") + + "-1 means msg-publish-rate or byte-publish-rate not configured in publish-rate yet", + response = PublishRate.class) @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist")}) public void getPublishRate(@Suspended AsyncResponse asyncResponse, @@ -943,7 +1002,9 @@ public void getPublishRate(@Suspended AsyncResponse asyncResponse, @POST @Path("/{tenant}/{namespace}/dispatchRate") @ApiOperation(value = "Set dispatch-rate throttling for all topics of the namespace") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission") }) + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission") }) public void setDispatchRate(@Suspended AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @ApiParam(value = "Dispatch rate for all topics of the specified namespace") @@ -962,7 +1023,9 @@ public void setDispatchRate(@Suspended AsyncResponse asyncResponse, @PathParam(" @DELETE @Path("/{tenant}/{namespace}/dispatchRate") @ApiOperation(value = "Delete dispatch-rate throttling for all topics of the namespace") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission") }) + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission") }) public void deleteDispatchRate(@Suspended AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace) { validateNamespaceName(tenant, namespace); @@ -979,7 +1042,8 @@ public void deleteDispatchRate(@Suspended AsyncResponse asyncResponse, @PathPara @GET @Path("/{tenant}/{namespace}/dispatchRate") @ApiOperation(value = "Get dispatch-rate configured for the namespace, null means dispatch-rate not configured, " - + "-1 means msg-dispatch-rate or byte-dispatch-rate not configured in dispatch-rate yet") + + "-1 means msg-dispatch-rate or byte-dispatch-rate not configured in dispatch-rate yet", + response = DispatchRate.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public void getDispatchRate(@Suspended AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @@ -996,7 +1060,9 @@ public void getDispatchRate(@Suspended AsyncResponse asyncResponse, @PathParam(" @POST @Path("/{tenant}/{namespace}/subscriptionDispatchRate") @ApiOperation(value = "Set Subscription dispatch-rate throttling for all topics of the namespace") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission")}) + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission")}) public void setSubscriptionDispatchRate(@Suspended AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @@ -1018,7 +1084,7 @@ public void setSubscriptionDispatchRate(@Suspended AsyncResponse asyncResponse, @Path("/{tenant}/{namespace}/subscriptionDispatchRate") @ApiOperation(value = "Get subscription dispatch-rate configured for the namespace, null means subscription " + "dispatch-rate not configured, -1 means msg-dispatch-rate or byte-dispatch-rate not configured " - + "in dispatch-rate yet") + + "in dispatch-rate yet", response = DispatchRate.class) @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist")}) public void getSubscriptionDispatchRate(@Suspended AsyncResponse asyncResponse, @@ -1038,7 +1104,9 @@ public void getSubscriptionDispatchRate(@Suspended AsyncResponse asyncResponse, @DELETE @Path("/{tenant}/{namespace}/subscriptionDispatchRate") @ApiOperation(value = "Delete Subscription dispatch-rate throttling for all topics of the namespace") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission") }) + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission") }) public void deleteSubscriptionDispatchRate(@Suspended AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace) { @@ -1056,7 +1124,9 @@ public void deleteSubscriptionDispatchRate(@Suspended AsyncResponse asyncRespons @DELETE @Path("/{tenant}/{namespace}/subscribeRate") @ApiOperation(value = "Delete subscribe-rate throttling for all topics of the namespace") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission")}) + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission")}) public void deleteSubscribeRate(@Suspended AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace) { validateNamespaceName(tenant, namespace); @@ -1073,7 +1143,9 @@ public void deleteSubscribeRate(@Suspended AsyncResponse asyncResponse, @PathPar @POST @Path("/{tenant}/{namespace}/subscribeRate") @ApiOperation(value = "Set subscribe-rate throttling for all topics of the namespace") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission")}) + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission")}) public void setSubscribeRate(@Suspended AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @ApiParam(value = "Subscribe rate for all topics of the specified namespace") @@ -1091,7 +1163,7 @@ public void setSubscribeRate(@Suspended AsyncResponse asyncResponse, @PathParam( @GET @Path("/{tenant}/{namespace}/subscribeRate") - @ApiOperation(value = "Get subscribe-rate configured for the namespace") + @ApiOperation(value = "Get subscribe-rate configured for the namespace", response = SubscribeRate.class) @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist")}) public void getSubscribeRate(@Suspended AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @@ -1109,7 +1181,9 @@ public void getSubscribeRate(@Suspended AsyncResponse asyncResponse, @PathParam( @DELETE @Path("/{tenant}/{namespace}/replicatorDispatchRate") @ApiOperation(value = "Remove replicator dispatch-rate throttling for all topics of the namespace") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission")}) + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission")}) public void removeReplicatorDispatchRate(@Suspended AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace) { @@ -1120,7 +1194,9 @@ public void removeReplicatorDispatchRate(@Suspended AsyncResponse asyncResponse, @POST @Path("/{tenant}/{namespace}/replicatorDispatchRate") @ApiOperation(value = "Set replicator dispatch-rate throttling for all topics of the namespace") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission")}) + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission")}) public void setReplicatorDispatchRate(@Suspended AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @@ -1134,7 +1210,7 @@ public void setReplicatorDispatchRate(@Suspended AsyncResponse asyncResponse, @Path("/{tenant}/{namespace}/replicatorDispatchRate") @ApiOperation(value = "Get replicator dispatch-rate configured for the namespace, null means replicator " + "dispatch-rate not configured, -1 means msg-dispatch-rate or byte-dispatch-rate not configured " - + "in dispatch-rate yet") + + "in dispatch-rate yet", response = DispatchRateImpl.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public void getReplicatorDispatchRate(@Suspended final AsyncResponse asyncResponse, @@ -1146,7 +1222,8 @@ public void getReplicatorDispatchRate(@Suspended final AsyncResponse asyncRespon @GET @Path("/{tenant}/{namespace}/backlogQuotaMap") - @ApiOperation(value = "Get backlog quota map on a namespace.") + @ApiOperation(value = "Get backlog quota map on a namespace.", + response = BacklogQuotaImpl.class, responseContainer = "Map") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public void getBacklogQuotaMap( @@ -1160,7 +1237,9 @@ public void getBacklogQuotaMap( @POST @Path("/{tenant}/{namespace}/backlogQuota") @ApiOperation(value = " Set a backlog quota for all the topics on a namespace.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @ApiResponse(code = 412, @@ -1178,7 +1257,9 @@ public void setBacklogQuota( @DELETE @Path("/{tenant}/{namespace}/backlogQuota") @ApiOperation(value = "Remove a backlog quota policy from a namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification") }) public void removeBacklogQuota( @@ -1191,7 +1272,7 @@ public void removeBacklogQuota( @GET @Path("/{tenant}/{namespace}/retention") - @ApiOperation(value = "Get retention config on a namespace.") + @ApiOperation(value = "Get retention config on a namespace.", response = RetentionPolicies.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public void getRetention(@Suspended final AsyncResponse asyncResponse, @@ -1212,7 +1293,9 @@ public void getRetention(@Suspended final AsyncResponse asyncResponse, @POST @Path("/{tenant}/{namespace}/retention") @ApiOperation(value = " Set retention configuration on a namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @ApiResponse(code = 412, message = "Retention Quota must exceed backlog quota") }) @@ -1225,7 +1308,9 @@ public void setRetention(@PathParam("tenant") String tenant, @PathParam("namespa @DELETE @Path("/{tenant}/{namespace}/retention") @ApiOperation(value = " Remove retention configuration on a namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @ApiResponse(code = 412, message = "Retention Quota must exceed backlog quota") }) @@ -1238,7 +1323,9 @@ public void removeRetention(@PathParam("tenant") String tenant, @PathParam("name @POST @Path("/{tenant}/{namespace}/persistence") @ApiOperation(value = "Set the persistence configuration for all the topics on a namespace.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @ApiResponse(code = 400, message = "Invalid persistence policies")}) @@ -1260,7 +1347,9 @@ public void setPersistence(@Suspended final AsyncResponse asyncResponse, @PathPa @DELETE @Path("/{tenant}/{namespace}/persistence") @ApiOperation(value = "Delete the persistence configuration for all topics on a namespace") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission") }) + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission") }) public void deletePersistence(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace) { validateNamespaceName(tenant, namespace); @@ -1278,6 +1367,7 @@ public void deletePersistence(@Suspended final AsyncResponse asyncResponse, @Pat @Path("/{tenant}/{namespace}/persistence/bookieAffinity") @ApiOperation(value = "Set the bookie-affinity-group to namespace-persistent policy.") @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace"), @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @@ -1291,7 +1381,8 @@ public void setBookieAffinityGroup(@PathParam("tenant") String tenant, @PathPara @GET @Path("/{property}/{namespace}/persistence/bookieAffinity") - @ApiOperation(value = "Get the bookie-affinity-group from namespace-local policy.") + @ApiOperation(value = "Get the bookie-affinity-group from namespace-local policy.", + response = BookieAffinityGroupDataImpl.class) @ApiResponses(value = { @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace"), @ApiResponse(code = 403, message = "Don't have admin permission"), @@ -1306,7 +1397,9 @@ public BookieAffinityGroupData getBookieAffinityGroup(@PathParam("property") Str @DELETE @Path("/{property}/{namespace}/persistence/bookieAffinity") @ApiOperation(value = "Delete the bookie-affinity-group from namespace-local policy.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification") }) public void deleteBookieAffinityGroup(@PathParam("property") String property, @@ -1317,7 +1410,7 @@ public void deleteBookieAffinityGroup(@PathParam("property") String property, @GET @Path("/{tenant}/{namespace}/persistence") - @ApiOperation(value = "Get the persistence configuration for a namespace.") + @ApiOperation(value = "Get the persistence configuration for a namespace.", response = PersistencePolicies.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification") }) @@ -1341,6 +1434,7 @@ public void getPersistence( @Path("/{tenant}/{namespace}/clearBacklog") @ApiOperation(value = "Clear backlog for all topics on a namespace.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 403, message = "Don't have admin or operate permission on the namespace"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public void clearNamespaceBacklog(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @@ -1360,6 +1454,7 @@ public void clearNamespaceBacklog(@Suspended final AsyncResponse asyncResponse, @Path("/{tenant}/{namespace}/{bundle}/clearBacklog") @ApiOperation(value = "Clear backlog for all topics on a namespace bundle.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace"), @ApiResponse(code = 403, message = "Don't have admin or operate permission on the namespace"), @ApiResponse(code = 404, message = "Namespace does not exist") }) @@ -1374,6 +1469,7 @@ public void clearNamespaceBundleBacklog(@PathParam("tenant") String tenant, @Path("/{tenant}/{namespace}/clearBacklog/{subscription}") @ApiOperation(value = "Clear backlog for a given subscription on all topics on a namespace.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 403, message = "Don't have admin or operate permission on the namespace"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public void clearNamespaceBacklogForSubscription(@Suspended final AsyncResponse asyncResponse, @@ -1394,6 +1490,7 @@ public void clearNamespaceBacklogForSubscription(@Suspended final AsyncResponse @Path("/{tenant}/{namespace}/{bundle}/clearBacklog/{subscription}") @ApiOperation(value = "Clear backlog for a given subscription on all topics on a namespace bundle.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace"), @ApiResponse(code = 403, message = "Don't have admin or operate permission on the namespace"), @ApiResponse(code = 404, message = "Namespace does not exist") }) @@ -1409,6 +1506,7 @@ public void clearNamespaceBundleBacklogForSubscription(@PathParam("tenant") Stri @Path("/{tenant}/{namespace}/unsubscribe/{subscription}") @ApiOperation(value = "Unsubscribes the given subscription on all topics on a namespace.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 403, message = "Don't have admin or operate permission on the namespacen"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public void unsubscribeNamespace(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @@ -1429,6 +1527,7 @@ public void unsubscribeNamespace(@Suspended final AsyncResponse asyncResponse, @ @Path("/{tenant}/{namespace}/{bundle}/unsubscribe/{subscription}") @ApiOperation(value = "Unsubscribes the given subscription on all topics on a namespace bundle.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 403, message = "Don't have admin or operate permission on the namespace"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public void unsubscribeNamespaceBundle(@PathParam("tenant") String tenant, @@ -1442,7 +1541,9 @@ public void unsubscribeNamespaceBundle(@PathParam("tenant") String tenant, @POST @Path("/{tenant}/{namespace}/subscriptionAuthMode") @ApiOperation(value = " Set a subscription auth mode for all the topics on a namespace.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification")}) public void setSubscriptionAuthMode(@PathParam("tenant") String tenant, @@ -1455,7 +1556,7 @@ public void setSubscriptionAuthMode(@PathParam("tenant") String tenant, @GET @Path("/{tenant}/{namespace}/subscriptionAuthMode") - @ApiOperation(value = "Get subscription auth mode in a namespace") + @ApiOperation(value = "Get subscription auth mode in a namespace", response = SubscriptionAuthMode.class) @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist")}) public void getSubscriptionAuthMode( @@ -1477,7 +1578,9 @@ public void getSubscriptionAuthMode( @POST @Path("/{tenant}/{namespace}/encryptionRequired") @ApiOperation(value = "Message encryption is required or not for all topics in a namespace") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), @ApiResponse(code = 409, message = "Concurrent modification"), }) public void modifyEncryptionRequired( @@ -1491,7 +1594,7 @@ public void modifyEncryptionRequired( @GET @Path("/{tenant}/{namespace}/encryptionRequired") - @ApiOperation(value = "Get message encryption required status in a namespace") + @ApiOperation(value = "Get message encryption required status in a namespace", response = Boolean.class) @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist")}) public void getEncryptionRequired(@Suspended AsyncResponse asyncResponse, @@ -1511,7 +1614,8 @@ public void getEncryptionRequired(@Suspended AsyncResponse asyncResponse, @GET @Path("/{tenant}/{namespace}/delayedDelivery") - @ApiOperation(value = "Get delayed delivery messages config on a namespace.") + @ApiOperation(value = "Get delayed delivery messages config on a namespace.", + response = DelayedDeliveryPolicies.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), @ApiResponse(code = 409, message = "Concurrent modification"), }) @@ -1533,7 +1637,9 @@ public void getDelayedDeliveryPolicies(@Suspended final AsyncResponse asyncRespo @POST @Path("/{tenant}/{namespace}/delayedDelivery") @ApiOperation(value = "Set delayed delivery messages config on a namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), }) public void setDelayedDeliveryPolicies(@PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @@ -1546,7 +1652,9 @@ public void setDelayedDeliveryPolicies(@PathParam("tenant") String tenant, @DELETE @Path("/{tenant}/{namespace}/delayedDelivery") @ApiOperation(value = "Delete delayed delivery messages config on a namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), }) public void removeDelayedDeliveryPolicies(@PathParam("tenant") String tenant, @PathParam("namespace") String namespace) { @@ -1556,7 +1664,7 @@ public void removeDelayedDeliveryPolicies(@PathParam("tenant") String tenant, @GET @Path("/{tenant}/{namespace}/inactiveTopicPolicies") - @ApiOperation(value = "Get inactive topic policies config on a namespace.") + @ApiOperation(value = "Get inactive topic policies config on a namespace.", response = InactiveTopicPolicies.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), @ApiResponse(code = 409, message = "Concurrent modification"), }) @@ -1578,7 +1686,9 @@ public void getInactiveTopicPolicies(@Suspended final AsyncResponse asyncRespons @DELETE @Path("/{tenant}/{namespace}/inactiveTopicPolicies") @ApiOperation(value = "Remove inactive topic policies from a namespace.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification")}) public void removeInactiveTopicPolicies(@PathParam("tenant") String tenant, @@ -1590,7 +1700,9 @@ public void removeInactiveTopicPolicies(@PathParam("tenant") String tenant, @POST @Path("/{tenant}/{namespace}/inactiveTopicPolicies") @ApiOperation(value = "Set inactive topic policies config on a namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), }) public void setInactiveTopicPolicies(@PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @@ -1602,7 +1714,7 @@ public void setInactiveTopicPolicies(@PathParam("tenant") String tenant, @GET @Path("/{tenant}/{namespace}/maxProducersPerTopic") - @ApiOperation(value = "Get maxProducersPerTopic config on a namespace.") + @ApiOperation(value = "Get maxProducersPerTopic config on a namespace.", response = Integer.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public void getMaxProducersPerTopic( @@ -1624,7 +1736,9 @@ public void getMaxProducersPerTopic( @POST @Path("/{tenant}/{namespace}/maxProducersPerTopic") @ApiOperation(value = " Set maxProducersPerTopic configuration on a namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @ApiResponse(code = 412, message = "maxProducersPerTopic value is not valid") }) @@ -1637,7 +1751,9 @@ public void setMaxProducersPerTopic(@PathParam("tenant") String tenant, @PathPar @DELETE @Path("/{tenant}/{namespace}/maxProducersPerTopic") @ApiOperation(value = "Remove maxProducersPerTopic configuration on a namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification") }) public void removeMaxProducersPerTopic(@PathParam("tenant") String tenant, @@ -1648,7 +1764,7 @@ public void removeMaxProducersPerTopic(@PathParam("tenant") String tenant, @GET @Path("/{tenant}/{namespace}/deduplicationSnapshotInterval") - @ApiOperation(value = "Get deduplicationSnapshotInterval config on a namespace.") + @ApiOperation(value = "Get deduplicationSnapshotInterval config on a namespace.", response = Integer.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public void getDeduplicationSnapshotInterval( @@ -1670,7 +1786,9 @@ public void getDeduplicationSnapshotInterval( @POST @Path("/{tenant}/{namespace}/deduplicationSnapshotInterval") @ApiOperation(value = "Set deduplicationSnapshotInterval config on a namespace.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist")}) public void setDeduplicationSnapshotInterval(@PathParam("tenant") String tenant , @PathParam("namespace") String namespace @@ -1682,7 +1800,7 @@ public void setDeduplicationSnapshotInterval(@PathParam("tenant") String tenant @GET @Path("/{tenant}/{namespace}/maxConsumersPerTopic") - @ApiOperation(value = "Get maxConsumersPerTopic config on a namespace.") + @ApiOperation(value = "Get maxConsumersPerTopic config on a namespace.", response = Integer.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public void getMaxConsumersPerTopic( @@ -1704,7 +1822,9 @@ public void getMaxConsumersPerTopic( @POST @Path("/{tenant}/{namespace}/maxConsumersPerTopic") @ApiOperation(value = " Set maxConsumersPerTopic configuration on a namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @ApiResponse(code = 412, message = "maxConsumersPerTopic value is not valid") }) @@ -1717,7 +1837,9 @@ public void setMaxConsumersPerTopic(@PathParam("tenant") String tenant, @PathPar @DELETE @Path("/{tenant}/{namespace}/maxConsumersPerTopic") @ApiOperation(value = "Remove maxConsumersPerTopic configuration on a namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification") }) public void removeMaxConsumersPerTopic(@PathParam("tenant") String tenant, @@ -1728,7 +1850,7 @@ public void removeMaxConsumersPerTopic(@PathParam("tenant") String tenant, @GET @Path("/{tenant}/{namespace}/maxConsumersPerSubscription") - @ApiOperation(value = "Get maxConsumersPerSubscription config on a namespace.") + @ApiOperation(value = "Get maxConsumersPerSubscription config on a namespace.", response = Integer.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public void getMaxConsumersPerSubscription( @@ -1750,7 +1872,9 @@ public void getMaxConsumersPerSubscription( @POST @Path("/{tenant}/{namespace}/maxConsumersPerSubscription") @ApiOperation(value = " Set maxConsumersPerSubscription configuration on a namespace.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @ApiResponse(code = 412, message = "maxConsumersPerSubscription value is not valid")}) @@ -1766,7 +1890,9 @@ public void setMaxConsumersPerSubscription(@PathParam("tenant") String tenant, @DELETE @Path("/{tenant}/{namespace}/maxConsumersPerSubscription") @ApiOperation(value = " Set maxConsumersPerSubscription configuration on a namespace.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @ApiResponse(code = 412, message = "maxConsumersPerSubscription value is not valid")}) @@ -1778,7 +1904,7 @@ public void removeMaxConsumersPerSubscription(@PathParam("tenant") String tenant @GET @Path("/{tenant}/{namespace}/maxUnackedMessagesPerConsumer") - @ApiOperation(value = "Get maxUnackedMessagesPerConsumer config on a namespace.") + @ApiOperation(value = "Get maxUnackedMessagesPerConsumer config on a namespace.", response = Integer.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public void getMaxUnackedMessagesPerConsumer(@Suspended final AsyncResponse asyncResponse, @@ -1799,7 +1925,9 @@ public void getMaxUnackedMessagesPerConsumer(@Suspended final AsyncResponse asyn @POST @Path("/{tenant}/{namespace}/maxUnackedMessagesPerConsumer") @ApiOperation(value = " Set maxConsumersPerTopic configuration on a namespace.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @ApiResponse(code = 412, message = "maxUnackedMessagesPerConsumer value is not valid")}) @@ -1815,7 +1943,9 @@ public void setMaxUnackedMessagesPerConsumer(@PathParam("tenant") String tenant, @DELETE @Path("/{tenant}/{namespace}/maxUnackedMessagesPerConsumer") @ApiOperation(value = "Remove maxUnackedMessagesPerConsumer config on a namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public void removeMaxUnackedmessagesPerConsumer(@PathParam("tenant") String tenant, @PathParam("namespace") String namespace) { @@ -1825,7 +1955,7 @@ public void removeMaxUnackedmessagesPerConsumer(@PathParam("tenant") String tena @GET @Path("/{tenant}/{namespace}/maxUnackedMessagesPerSubscription") - @ApiOperation(value = "Get maxUnackedMessagesPerSubscription config on a namespace.") + @ApiOperation(value = "Get maxUnackedMessagesPerSubscription config on a namespace.", response = Integer.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public void getMaxUnackedmessagesPerSubscription( @@ -1847,7 +1977,9 @@ public void getMaxUnackedmessagesPerSubscription( @POST @Path("/{tenant}/{namespace}/maxUnackedMessagesPerSubscription") @ApiOperation(value = " Set maxUnackedMessagesPerSubscription configuration on a namespace.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @ApiResponse(code = 412, message = "maxUnackedMessagesPerSubscription value is not valid")}) @@ -1862,7 +1994,9 @@ public void setMaxUnackedMessagesPerSubscription( @DELETE @Path("/{tenant}/{namespace}/maxUnackedMessagesPerSubscription") @ApiOperation(value = "Remove maxUnackedMessagesPerSubscription config on a namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public void removeMaxUnackedmessagesPerSubscription(@PathParam("tenant") String tenant, @PathParam("namespace") String namespace) { @@ -1872,7 +2006,7 @@ public void removeMaxUnackedmessagesPerSubscription(@PathParam("tenant") String @GET @Path("/{tenant}/{namespace}/maxSubscriptionsPerTopic") - @ApiOperation(value = "Get maxSubscriptionsPerTopic config on a namespace.") + @ApiOperation(value = "Get maxSubscriptionsPerTopic config on a namespace.", response = Integer.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public void getMaxSubscriptionsPerTopic(@Suspended final AsyncResponse asyncResponse, @@ -1893,7 +2027,9 @@ public void getMaxSubscriptionsPerTopic(@Suspended final AsyncResponse asyncResp @POST @Path("/{tenant}/{namespace}/maxSubscriptionsPerTopic") @ApiOperation(value = " Set maxSubscriptionsPerTopic configuration on a namespace.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @ApiResponse(code = 412, message = "maxUnackedMessagesPerSubscription value is not valid")}) @@ -1908,7 +2044,9 @@ public void setMaxSubscriptionsPerTopic( @DELETE @Path("/{tenant}/{namespace}/maxSubscriptionsPerTopic") @ApiOperation(value = "Remove maxSubscriptionsPerTopic configuration on a namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification") }) public void removeMaxSubscriptionsPerTopic(@PathParam("tenant") String tenant, @@ -1920,7 +2058,9 @@ public void removeMaxSubscriptionsPerTopic(@PathParam("tenant") String tenant, @POST @Path("/{tenant}/{namespace}/antiAffinity") @ApiOperation(value = "Set anti-affinity group for a namespace") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), @ApiResponse(code = 412, message = "Invalid antiAffinityGroup")}) public void setNamespaceAntiAffinityGroup(@PathParam("tenant") String tenant, @@ -1934,7 +2074,7 @@ public void setNamespaceAntiAffinityGroup(@PathParam("tenant") String tenant, @GET @Path("/{tenant}/{namespace}/antiAffinity") - @ApiOperation(value = "Get anti-affinity group of a namespace.") + @ApiOperation(value = "Get anti-affinity group of a namespace.", response = String.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist") }) public String getNamespaceAntiAffinityGroup(@PathParam("tenant") String tenant, @@ -1946,7 +2086,9 @@ public String getNamespaceAntiAffinityGroup(@PathParam("tenant") String tenant, @DELETE @Path("/{tenant}/{namespace}/antiAffinity") @ApiOperation(value = "Remove anti-affinity group of a namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification") }) public void removeNamespaceAntiAffinityGroup(@PathParam("tenant") String tenant, @@ -1958,7 +2100,8 @@ public void removeNamespaceAntiAffinityGroup(@PathParam("tenant") String tenant, @GET @Path("{cluster}/antiAffinity/{group}") @ApiOperation(value = "Get all namespaces that are grouped by given anti-affinity group in a given cluster." - + " api can be only accessed by admin of any of the existing tenant") + + " api can be only accessed by admin of any of the existing tenant", + response = String.class, responseContainer = "List") @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 412, message = "Cluster not exist/Anti-affinity group can't be empty.")}) public List getAntiAffinityNamespaces(@PathParam("cluster") String cluster, @@ -1966,25 +2109,11 @@ public List getAntiAffinityNamespaces(@PathParam("cluster") String clust return internalGetAntiAffinityNamespaces(cluster, antiAffinityGroup, tenant); } - private Policies getDefaultPolicesIfNull(Policies policies) { - if (policies == null) { - policies = new Policies(); - } - - int defaultNumberOfBundles = config().getDefaultNumberOfNamespaceBundles(); - - if (policies.bundles == null) { - policies.bundles = getBundles(defaultNumberOfBundles); - } - - return policies; - } - @GET @Path("/{tenant}/{namespace}/compactionThreshold") @ApiOperation(value = "Maximum number of uncompacted bytes in topics before compaction is triggered.", notes = "The backlog size is compared to the threshold periodically. " - + "A threshold of 0 disabled automatic compaction") + + "A threshold of 0 disabled automatic compaction", response = Long.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace doesn't exist") }) public void getCompactionThreshold( @@ -2008,7 +2137,9 @@ public void getCompactionThreshold( @ApiOperation(value = "Set maximum number of uncompacted bytes in a topic before compaction is triggered.", notes = "The backlog size is compared to the threshold periodically. " + "A threshold of 0 disabled automatic compaction") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace doesn't exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @ApiResponse(code = 412, message = "compactionThreshold value is not valid")}) @@ -2026,7 +2157,9 @@ public void setCompactionThreshold(@PathParam("tenant") String tenant, @ApiOperation(value = "Delete maximum number of uncompacted bytes in a topic before compaction is triggered.", notes = "The backlog size is compared to the threshold periodically. " + "A threshold of 0 disabled automatic compaction") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace doesn't exist"), @ApiResponse(code = 409, message = "Concurrent modification")}) public void deleteCompactionThreshold(@PathParam("tenant") String tenant, @@ -2039,7 +2172,7 @@ public void deleteCompactionThreshold(@PathParam("tenant") String tenant, @Path("/{tenant}/{namespace}/offloadThreshold") @ApiOperation(value = "Maximum number of bytes stored on the pulsar cluster for a topic," + " before the broker will start offloading to longterm storage", - notes = "A negative value disables automatic offloading") + notes = "A negative value disables automatic offloading", response = Long.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace doesn't exist") }) public void getOffloadThreshold( @@ -2070,7 +2203,9 @@ public void getOffloadThreshold( + " before the broker will start offloading to longterm storage", notes = "-1 will revert to using the cluster default." + " A negative value disables automatic offloading. ") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace doesn't exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @ApiResponse(code = 412, message = "offloadThreshold value is not valid")}) @@ -2088,7 +2223,7 @@ public void setOffloadThreshold(@PathParam("tenant") String tenant, @Path("/{tenant}/{namespace}/offloadThresholdInSeconds") @ApiOperation(value = "Maximum number of bytes stored on the pulsar cluster for a topic," + " before the broker will start offloading to longterm storage", - notes = "A negative value disables automatic offloading") + notes = "A negative value disables automatic offloading", response = Long.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace doesn't exist") }) public void getOffloadThresholdInSeconds( @@ -2118,7 +2253,9 @@ public void getOffloadThresholdInSeconds( @ApiOperation(value = "Set maximum number of seconds stored on the pulsar cluster for a topic," + " before the broker will start offloading to longterm storage", notes = "A negative value disables automatic offloading") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace doesn't exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @ApiResponse(code = 412, message = "offloadThresholdInSeconds value is not valid") }) @@ -2142,7 +2279,7 @@ public void setOffloadThresholdInSeconds( + " from the Pulsar cluster's local storage (i.e. BookKeeper)", notes = "A negative value denotes that deletion has been completely disabled." + " 'null' denotes that the topics in the namespace will fall back to the" - + " broker default for deletion lag.") + + " broker default for deletion lag.", response = Long.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace doesn't exist") }) public void getOffloadDeletionLag( @@ -2172,7 +2309,9 @@ public void getOffloadDeletionLag( @ApiOperation(value = "Set number of milliseconds to wait before deleting a ledger segment which has been offloaded" + " from the Pulsar cluster's local storage (i.e. BookKeeper)", notes = "A negative value disables the deletion completely.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace doesn't exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @ApiResponse(code = 412, message = "offloadDeletionLagMs value is not valid")}) @@ -2191,6 +2330,7 @@ public void setOffloadDeletionLag(@PathParam("tenant") String tenant, @ApiOperation(value = "Clear the namespace configured offload deletion lag. The topics in the namespace" + " will fallback to using the default configured deletion lag for the broker") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 200, message = "Operation successful"), @ApiResponse(code = 404, message = "Namespace doesn't exist"), @ApiResponse(code = 409, message = "Concurrent modification") }) public void clearOffloadDeletionLag(@PathParam("tenant") String tenant, @@ -2204,7 +2344,8 @@ public void clearOffloadDeletionLag(@PathParam("tenant") String tenant, @ApiOperation(value = "The strategy used to check the compatibility of new schemas," + " provided by producers, before automatically updating the schema", notes = "The value AutoUpdateDisabled prevents producers from updating the schema. " - + " If set to AutoUpdateDisabled, schemas must be updated through the REST api") + + " If set to AutoUpdateDisabled, schemas must be updated through the REST api", + response = SchemaAutoUpdateCompatibilityStrategy.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace doesn't exist"), @ApiResponse(code = 409, message = "Concurrent modification") }) @@ -2221,7 +2362,9 @@ public SchemaAutoUpdateCompatibilityStrategy getSchemaAutoUpdateCompatibilityStr + " provided by producers, before automatically updating the schema", notes = "The value AutoUpdateDisabled prevents producers from updating the schema. " + " If set to AutoUpdateDisabled, schemas must be updated through the REST api") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace doesn't exist"), @ApiResponse(code = 409, message = "Concurrent modification")}) public void setSchemaAutoUpdateCompatibilityStrategy( @@ -2235,7 +2378,8 @@ public void setSchemaAutoUpdateCompatibilityStrategy( @GET @Path("/{tenant}/{namespace}/schemaCompatibilityStrategy") - @ApiOperation(value = "The strategy of the namespace schema compatibility ") + @ApiOperation(value = "The strategy of the namespace schema compatibility ", + response = SchemaCompatibilityStrategy.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace doesn't exist"), @ApiResponse(code = 409, message = "Concurrent modification") }) @@ -2259,7 +2403,9 @@ public void getSchemaCompatibilityStrategy( @PUT @Path("/{tenant}/{namespace}/schemaCompatibilityStrategy") @ApiOperation(value = "Update the strategy used to check the compatibility of new schema") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace doesn't exist"), @ApiResponse(code = 409, message = "Concurrent modification")}) public void setSchemaCompatibilityStrategy( @@ -2273,7 +2419,7 @@ public void setSchemaCompatibilityStrategy( @GET @Path("/{tenant}/{namespace}/isAllowAutoUpdateSchema") - @ApiOperation(value = "The flag of whether allow auto update schema") + @ApiOperation(value = "The flag of whether allow auto update schema", response = Boolean.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace doesn't exist"), @ApiResponse(code = 409, message = "Concurrent modification") }) @@ -2303,7 +2449,9 @@ public void getIsAllowAutoUpdateSchema( @POST @Path("/{tenant}/{namespace}/isAllowAutoUpdateSchema") @ApiOperation(value = "Update flag of whether allow auto update schema") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace doesn't exist"), @ApiResponse(code = 409, message = "Concurrent modification")}) public void setIsAllowAutoUpdateSchema( @@ -2317,7 +2465,8 @@ public void setIsAllowAutoUpdateSchema( @GET @Path("/{tenant}/{namespace}/subscriptionTypesEnabled") - @ApiOperation(value = "The set of whether allow subscription types") + @ApiOperation(value = "The set of whether allow subscription types", + response = SubscriptionType.class, responseContainer = "Set") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace doesn't exist"), @ApiResponse(code = 409, message = "Concurrent modification") }) @@ -2345,7 +2494,9 @@ public void getSubscriptionTypesEnabled( @POST @Path("/{tenant}/{namespace}/subscriptionTypesEnabled") @ApiOperation(value = "Update set of whether allow share sub type") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace doesn't exist"), @ApiResponse(code = 409, message = "Concurrent modification")}) public void setSubscriptionTypesEnabled( @@ -2376,7 +2527,8 @@ public void removeSubscriptionTypesEnabled(@PathParam("tenant") String tenant, notes = "If the flag is set to true, when a producer without a schema attempts to produce to a topic" + " with schema in this namespace, the producer will be failed to connect. PLEASE be" + " carefully on using this, since non-java clients don't support schema.if you enable" - + " this setting, it will cause non-java clients failed to produce.") + + " this setting, it will cause non-java clients failed to produce.", + response = Boolean.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenants or Namespace doesn't exist") }) public void getSchemaValidtionEnforced( @@ -2411,7 +2563,9 @@ public void getSchemaValidtionEnforced( + " with schema in this namespace, the producer will be failed to connect. PLEASE be" + " carefully on using this, since non-java clients don't support schema.if you enable" + " this setting, it will cause non-java clients failed to produce.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or Namespace doesn't exist"), @ApiResponse(code = 412, message = "schemaValidationEnforced value is not valid")}) public void setSchemaValidationEnforced(@PathParam("tenant") String tenant, @@ -2426,8 +2580,9 @@ public void setSchemaValidationEnforced(@PathParam("tenant") String tenant, @POST @Path("/{tenant}/{namespace}/offloadPolicies") - @ApiOperation(value = " Set offload configuration on a namespace.") + @ApiOperation(value = "Set offload configuration on a namespace.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @@ -2451,6 +2606,7 @@ public void setOffloadPolicies(@PathParam("tenant") String tenant, @PathParam("n @Path("/{tenant}/{namespace}/removeOffloadPolicies") @ApiOperation(value = " Set offload configuration on a namespace.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @@ -2470,7 +2626,7 @@ public void removeOffloadPolicies(@PathParam("tenant") String tenant, @PathParam @GET @Path("/{tenant}/{namespace}/offloadPolicies") - @ApiOperation(value = "Get offload configuration on a namespace.") + @ApiOperation(value = "Get offload configuration on a namespace.", response = OffloadPolicies.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist")}) @@ -2492,7 +2648,7 @@ public void getOffloadPolicies( @GET @Path("/{tenant}/{namespace}/maxTopicsPerNamespace") - @ApiOperation(value = "Get maxTopicsPerNamespace config on a namespace.") + @ApiOperation(value = "Get maxTopicsPerNamespace config on a namespace.", response = Integer.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or namespace does not exist") }) public void getMaxTopicsPerNamespace(@Suspended final AsyncResponse asyncResponse, @@ -2517,7 +2673,9 @@ public void getMaxTopicsPerNamespace(@Suspended final AsyncResponse asyncRespons @POST @Path("/{tenant}/{namespace}/maxTopicsPerNamespace") @ApiOperation(value = "Set maxTopicsPerNamespace config on a namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist"), }) public void setMaxTopicsPerNamespace(@PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @@ -2530,7 +2688,9 @@ public void setMaxTopicsPerNamespace(@PathParam("tenant") String tenant, @DELETE @Path("/{tenant}/{namespace}/maxTopicsPerNamespace") @ApiOperation(value = "Remove maxTopicsPerNamespace config on a namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist"), }) public void removeMaxTopicsPerNamespace(@PathParam("tenant") String tenant, @PathParam("namespace") String namespace) { @@ -2541,7 +2701,9 @@ public void removeMaxTopicsPerNamespace(@PathParam("tenant") String tenant, @PUT @Path("/{tenant}/{namespace}/property/{key}/{value}") @ApiOperation(value = "Put a key value pair property on a namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist"), }) public void setProperty( @Suspended final AsyncResponse asyncResponse, @@ -2555,7 +2717,7 @@ public void setProperty( @GET @Path("/{tenant}/{namespace}/property/{key}") - @ApiOperation(value = "Get property value for a given key on a namespace.") + @ApiOperation(value = "Get property value for a given key on a namespace.", response = String.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist"), }) public void getProperty( @@ -2570,7 +2732,9 @@ public void getProperty( @DELETE @Path("/{tenant}/{namespace}/property/{key}") @ApiOperation(value = "Remove property value for a given key on a namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist"), }) public void removeProperty( @Suspended final AsyncResponse asyncResponse, @@ -2584,7 +2748,9 @@ public void removeProperty( @PUT @Path("/{tenant}/{namespace}/properties") @ApiOperation(value = "Put key value pairs property on a namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist"), }) public void setProperties( @Suspended final AsyncResponse asyncResponse, @@ -2598,7 +2764,8 @@ public void setProperties( @GET @Path("/{tenant}/{namespace}/properties") - @ApiOperation(value = "Get key value pair properties for a given namespace.") + @ApiOperation(value = "Get key value pair properties for a given namespace.", + response = String.class, responseContainer = "Map") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist"), }) public void getProperties( @@ -2612,7 +2779,9 @@ public void getProperties( @DELETE @Path("/{tenant}/{namespace}/properties") @ApiOperation(value = "Clear properties on a given namespace.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist"), }) public void clearProperties( @Suspended final AsyncResponse asyncResponse, @@ -2624,7 +2793,7 @@ public void clearProperties( @GET @Path("/{tenant}/{namespace}/resourcegroup") - @ApiOperation(value = "Get the resource group attached to the namespace") + @ApiOperation(value = "Get the resource group attached to the namespace", response = String.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist") }) public void getNamespaceResourceGroup( @@ -2646,7 +2815,9 @@ public void getNamespaceResourceGroup( @POST @Path("/{tenant}/{namespace}/resourcegroup/{resourcegroup}") @ApiOperation(value = "Set resourcegroup for a namespace") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), @ApiResponse(code = 412, message = "Invalid resourcegroup") }) public void setNamespaceResourceGroup(@PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @@ -2658,7 +2829,9 @@ public void setNamespaceResourceGroup(@PathParam("tenant") String tenant, @PathP @DELETE @Path("/{tenant}/{namespace}/resourcegroup") @ApiOperation(value = "Delete resourcegroup for a namespace") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), @ApiResponse(code = 412, message = "Invalid resourcegroup")}) public void removeNamespaceResourceGroup(@PathParam("tenant") String tenant, @@ -2670,7 +2843,13 @@ public void removeNamespaceResourceGroup(@PathParam("tenant") String tenant, @GET @Path("/{tenant}/{namespace}/scanOffloadedLedgers") @ApiOperation(value = "Trigger the scan of offloaded Ledgers on the LedgerOffloader for the given namespace") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Successful get of offloaded ledger data", response = String.class, + examples = @Example(value = { @ExampleProperty(mediaType = "application/json", + value = "{\"objects\":[{\"key1\":\"value1\",\"key2\":\"value2\"}]," + + "\"total\":100,\"errors\":5,\"unknown\":3}") + })), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace doesn't exist") }) public Response scanOffloadedLedgers(@PathParam("tenant") String tenant, @PathParam("namespace") String namespace) { @@ -2719,7 +2898,7 @@ public void finished(int total, int errors, int unknown) throws Exception { @GET @Path("/{tenant}/{namespace}/entryFilters") - @ApiOperation(value = "Get maxConsumersPerSubscription config on a namespace.") + @ApiOperation(value = "Get maxConsumersPerSubscription config on a namespace.", response = EntryFilters.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public void getEntryFiltersPerTopic( @@ -2742,6 +2921,7 @@ public void getEntryFiltersPerTopic( @Path("/{tenant}/{namespace}/entryFilters") @ApiOperation(value = "Set entry filters for namespace") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 400, message = "Specified entry filters are not valid"), @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist") @@ -2763,7 +2943,9 @@ public void setEntryFiltersPerTopic(@Suspended AsyncResponse asyncResponse, @Pat @DELETE @Path("/{tenant}/{namespace}/entryFilters") @ApiOperation(value = "Remove entry filters for namespace") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), @ApiResponse(code = 412, message = "Invalid TTL")}) public void removeNamespaceEntryFilters(@Suspended AsyncResponse asyncResponse, @@ -2779,7 +2961,132 @@ public void removeNamespaceEntryFilters(@Suspended AsyncResponse asyncResponse, }); } + @POST + @Path("/{tenant}/{namespace}/migration") + @ApiOperation(hidden = true, value = "Update migration for all topics in a namespace") + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Property or cluster or namespace doesn't exist") }) + public void enableMigration(@PathParam("tenant") String tenant, + @PathParam("namespace") String namespace, + boolean migrated) { + validateNamespaceName(tenant, namespace); + internalEnableMigration(migrated); + } + @POST + @Path("/{tenant}/{namespace}/dispatcherPauseOnAckStatePersistent") + @ApiOperation(value = "Set dispatcher pause on ack state persistent configuration for specified namespace.") + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), + @ApiResponse(code = 409, message = "Concurrent modification")}) + public void setDispatcherPauseOnAckStatePersistent(@Suspended final AsyncResponse asyncResponse, + @PathParam("tenant") String tenant, + @PathParam("namespace") String namespace) { + validateNamespaceName(tenant, namespace); + internalSetDispatcherPauseOnAckStatePersistentAsync(true) + .thenRun(() -> { + log.info("[{}] Successfully enabled dispatcherPauseOnAckStatePersistent: namespace={}", + clientAppId(), namespaceName); + asyncResponse.resume(Response.noContent().build()); + }) + .exceptionally(ex -> { + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); + } + + @DELETE + @Path("/{tenant}/{namespace}/dispatcherPauseOnAckStatePersistent") + @ApiOperation(value = "Remove dispatcher pause on ack state persistent configuration for specified namespace.") + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), + @ApiResponse(code = 409, message = "Concurrent modification")}) + public void removeDispatcherPauseOnAckStatePersistent(@Suspended final AsyncResponse asyncResponse, + @PathParam("tenant") String tenant, + @PathParam("namespace") String namespace) { + validateNamespaceName(tenant, namespace); + internalSetDispatcherPauseOnAckStatePersistentAsync(false) + .thenRun(() -> { + log.info("[{}] Successfully remove dispatcherPauseOnAckStatePersistent: namespace={}", + clientAppId(), namespaceName); + asyncResponse.resume(Response.noContent().build()); + }) + .exceptionally(ex -> { + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); + } + + @GET + @Path("/{tenant}/{namespace}/dispatcherPauseOnAckStatePersistent") + @ApiOperation(value = "Get dispatcher pause on ack state persistent config on a namespace.", + response = Boolean.class) + @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist") }) + public void getDispatcherPauseOnAckStatePersistent(@Suspended final AsyncResponse asyncResponse, + @PathParam("tenant") String tenant, + @PathParam("namespace") String namespace) { + validateNamespaceName(tenant, namespace); + internalGetDispatcherPauseOnAckStatePersistentAsync() + .thenApply(asyncResponse::resume) + .exceptionally(ex -> { + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); + } + + + @POST + @Path("/{tenant}/{namespace}/allowedClusters") + @ApiOperation(value = "Set the allowed clusters for a namespace.") + @ApiResponses(value = { + @ApiResponse(code = 400, message = "The list of allowed clusters should include all replication clusters."), + @ApiResponse(code = 403, message = "The requester does not have admin permissions."), + @ApiResponse(code = 404, message = "The specified tenant, cluster, or namespace does not exist."), + @ApiResponse(code = 409, message = "A peer-cluster cannot be part of an allowed-cluster."), + @ApiResponse(code = 412, message = "The namespace is not global or the provided cluster IDs are invalid.")}) + public void setNamespaceAllowedClusters(@Suspended AsyncResponse asyncResponse, + @PathParam("tenant") String tenant, + @PathParam("namespace") String namespace, + @ApiParam(value = "List of allowed clusters", required = true) + List clusterIds) { + validateNamespaceName(tenant, namespace); + internalSetNamespaceAllowedClusters(clusterIds) + .thenAccept(asyncResponse::resume) + .exceptionally(e -> { + log.error("[{}] Failed to set namespace allowed clusters on namespace {}", + clientAppId(), namespace, e); + resumeAsyncResponseExceptionally(asyncResponse, e); + return null; + }); + } + + @GET + @Path("/{tenant}/{namespace}/allowedClusters") + @ApiOperation(value = "Get the allowed clusters for a namespace.", + response = String.class, responseContainer = "List") + @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Tenant or cluster or namespace doesn't exist"), + @ApiResponse(code = 412, message = "Namespace is not global")}) + public void getNamespaceAllowedClusters(@Suspended AsyncResponse asyncResponse, + @PathParam("tenant") String tenant, + @PathParam("namespace") String namespace) { + validateNamespaceName(tenant, namespace); + internalGetNamespaceAllowedClustersAsync() + .thenAccept(asyncResponse::resume) + .exceptionally(e -> { + log.error("[{}] Failed to get namespace allowed clusters on namespace {}", clientAppId(), + namespace, e); + resumeAsyncResponseExceptionally(asyncResponse, e); + return null; + }); + } private static final Logger log = LoggerFactory.getLogger(Namespaces.class); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java index cc269d02831d8..edf4303e1adef 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java @@ -51,17 +51,17 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.web.RestException; -import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.EntryFilters; import org.apache.pulsar.common.policies.data.NamespaceOperation; +import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.policies.data.stats.NonPersistentPartitionedTopicStatsImpl; import org.apache.pulsar.common.policies.data.stats.NonPersistentTopicStatsImpl; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,7 +75,7 @@ public class NonPersistentTopics extends PersistentTopics { @GET @Path("/{tenant}/{namespace}/{topic}/partitions") - @ApiOperation(value = "Get partitioned topic metadata.") + @ApiOperation(value = "Get partitioned topic metadata.", response = PartitionedTopicMetadata.class) @ApiResponses(value = { @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to manage resources on this tenant"), @@ -97,13 +97,25 @@ public void getPartitionedMetadata( @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Is check configuration required to automatically create topic") @QueryParam("checkAllowAutoCreation") @DefaultValue("false") boolean checkAllowAutoCreation) { - super.getPartitionedMetadata(asyncResponse, tenant, namespace, encodedTopic, authoritative, - checkAllowAutoCreation); + validateTopicName(tenant, namespace, encodedTopic); + validateTopicOwnershipAsync(topicName, authoritative).whenComplete((__, ex) -> { + if (ex != null) { + Throwable actEx = FutureUtil.unwrapCompletionException(ex); + if (isNot307And404Exception(actEx)) { + log.error("[{}] Failed to get internal stats for topic {}", clientAppId(), topicName, ex); + } + resumeAsyncResponseExceptionally(asyncResponse, actEx); + } else { + // "super.getPartitionedMetadata" will handle error itself. + super.getPartitionedMetadata(asyncResponse, tenant, namespace, encodedTopic, authoritative, + checkAllowAutoCreation); + } + }); } @GET @Path("{tenant}/{namespace}/{topic}/internalStats") - @ApiOperation(value = "Get the internal stats for the topic.") + @ApiOperation(value = "Get the internal stats for the topic.", response = PersistentTopicInternalStats.class) @ApiResponses(value = { @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to manage resources on this tenant"), @@ -133,7 +145,7 @@ public void getInternalStats( }) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get internal stats for topic {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -146,6 +158,7 @@ public void getInternalStats( @ApiOperation(value = "Create a partitioned topic.", notes = "It needs to be called before creating a producer on a partitioned topic.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to manage resources on this tenant"), @ApiResponse(code = 403, message = "Don't have admin permission"), @@ -215,9 +228,17 @@ public void getPartitionedStats( + "not to use when there's heavy traffic.") @QueryParam("subscriptionBacklogSize") @DefaultValue("false") boolean subscriptionBacklogSize, @ApiParam(value = "If return the earliest time in backlog") - @QueryParam("getEarliestTimeInBacklog") @DefaultValue("false") boolean getEarliestTimeInBacklog) { + @QueryParam("getEarliestTimeInBacklog") @DefaultValue("false") boolean getEarliestTimeInBacklog, + @ApiParam(value = "If exclude the publishers") + @QueryParam("excludePublishers") @DefaultValue("false") boolean excludePublishers, + @ApiParam(value = "If exclude the consumers") + @QueryParam("excludeConsumers") @DefaultValue("false") boolean excludeConsumers) { try { - validatePartitionedTopicName(tenant, namespace, encodedTopic); + validateTopicName(tenant, namespace, encodedTopic); + if (topicName.isPartitioned()) { + throw new RestException(Response.Status.PRECONDITION_FAILED, + "Partitioned Topic Name should not contain '-partition-'"); + } if (topicName.isGlobal()) { try { validateGlobalNamespaceOwnership(namespaceName); @@ -230,18 +251,26 @@ public void getPartitionedStats( getPartitionedTopicMetadataAsync(topicName, authoritative, false).thenAccept(partitionMetadata -> { if (partitionMetadata.partitions == 0) { - asyncResponse.resume(new RestException(Status.NOT_FOUND, "Partitioned Topic not found")); + asyncResponse.resume(new RestException(Status.NOT_FOUND, + String.format("Partitioned topic not found %s", topicName.toString()))); return; } NonPersistentPartitionedTopicStatsImpl stats = new NonPersistentPartitionedTopicStatsImpl(partitionMetadata); List> topicStatsFutureList = new ArrayList<>(); + org.apache.pulsar.client.admin.GetStatsOptions statsOptions = + new org.apache.pulsar.client.admin.GetStatsOptions( + getPreciseBacklog, + subscriptionBacklogSize, + getEarliestTimeInBacklog, + excludePublishers, + excludeConsumers + ); for (int i = 0; i < partitionMetadata.partitions; i++) { try { topicStatsFutureList .add(pulsar().getAdminClient().topics().getStatsAsync( - (topicName.getPartition(i).toString()), getPreciseBacklog, - subscriptionBacklogSize, getEarliestTimeInBacklog)); + (topicName.getPartition(i).toString()), statsOptions)); } catch (PulsarServerException e) { asyncResponse.resume(new RestException(e)); return; @@ -302,6 +331,7 @@ public void getPartitionedStats( @Path("/{tenant}/{namespace}/{topic}/unload") @ApiOperation(value = "Unload a topic") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "This operation requires super-user access"), @ApiResponse(code = 403, message = "Don't have admin permission"), @@ -445,44 +475,38 @@ public void getListFromBundle( bundleRange); asyncResponse.resume(Response.noContent().build()); } else { - NamespaceBundle nsBundle; - try { - nsBundle = validateNamespaceBundleOwnership(namespaceName, policies.bundles, - bundleRange, true, true); - } catch (WebApplicationException wae) { - asyncResponse.resume(wae); - return; - } - try { - ConcurrentOpenHashMap> bundleTopics = - pulsar().getBrokerService().getMultiLayerTopicsMap().get(namespaceName.toString()); - if (bundleTopics == null || bundleTopics.isEmpty()) { - asyncResponse.resume(Collections.emptyList()); - return; - } - final List topicList = new ArrayList<>(); - String bundleKey = namespaceName.toString() + "/" + nsBundle.getBundleRange(); - ConcurrentOpenHashMap topicMap = bundleTopics.get(bundleKey); - if (topicMap != null) { - topicList.addAll(topicMap.keys().stream() - .filter(name -> !TopicName.get(name).isPersistent()) - .collect(Collectors.toList())); - } - asyncResponse.resume(topicList); - } catch (Exception e) { - log.error("[{}] Failed to list topics on namespace bundle {}/{}", clientAppId(), - namespaceName, bundleRange, e); - asyncResponse.resume(new RestException(e)); - } + validateNamespaceBundleOwnershipAsync(namespaceName, policies.bundles, bundleRange, true, true) + .thenAccept(nsBundle -> { + final var bundleTopics = pulsar().getBrokerService().getMultiLayerTopicsMap() + .get(namespaceName.toString()); + if (bundleTopics == null || bundleTopics.isEmpty()) { + asyncResponse.resume(Collections.emptyList()); + return; + } + final List topicList = new ArrayList<>(); + String bundleKey = namespaceName.toString() + "/" + nsBundle.getBundleRange(); + final var topicMap = bundleTopics.get(bundleKey); + if (topicMap != null) { + topicList.addAll(topicMap.keySet().stream() + .filter(name -> !TopicName.get(name).isPersistent()) + .collect(Collectors.toList())); + } + asyncResponse.resume(topicList); + }).exceptionally(ex -> { + if (isNot307And404Exception(ex)) { + log.error("[{}] Failed to list topics on namespace bundle {}/{}", clientAppId(), + namespaceName, bundleRange, ex); + } + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); } }).exceptionally(ex -> { - log.error("[{}] Failed to list topics on namespace bundle {}/{}", clientAppId(), - namespaceName, bundleRange, ex); - if (ex.getCause() instanceof WebApplicationException) { - asyncResponse.resume(ex.getCause()); - } else { - asyncResponse.resume(new RestException(ex.getCause())); + if (isNot307And404Exception(ex)) { + log.error("[{}] Failed to list topics on namespace bundle {}/{}", clientAppId(), + namespaceName, bundleRange, ex); } + resumeAsyncResponseExceptionally(asyncResponse, ex); return null; }); } @@ -492,6 +516,7 @@ public void getListFromBundle( @ApiOperation(value = "Truncate a topic.", notes = "NonPersistentTopic does not support truncate.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 412, message = "NonPersistentTopic does not support truncate.") }) public void truncateTopic( @@ -515,7 +540,7 @@ protected void validateAdminOperationOnTopic(TopicName topicName, boolean author @GET @Path("/{tenant}/{namespace}/{topic}/entryFilters") - @ApiOperation(value = "Get entry filters for a topic.") + @ApiOperation(value = "Get entry filters for a topic.", response = EntryFilters.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenants or Namespace doesn't exist") }) public void getEntryFilters(@Suspended AsyncResponse asyncResponse, @@ -543,7 +568,9 @@ public void getEntryFilters(@Suspended AsyncResponse asyncResponse, @POST @Path("/{tenant}/{namespace}/{topic}/entryFilters") @ApiOperation(value = "Set entry filters for specified topic") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, please enable the topic level policy and retry"), @@ -571,7 +598,9 @@ public void setEntryFilters(@Suspended final AsyncResponse asyncResponse, @DELETE @Path("/{tenant}/{namespace}/{topic}/entryFilters") @ApiOperation(value = "Remove entry filters for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, please enable the topic level policy and retry"), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java index eee363aeed86b..a8e5e7a3ce77b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java @@ -45,11 +45,16 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.pulsar.broker.admin.AdminResource; import org.apache.pulsar.broker.admin.impl.PersistentTopicsBase; import org.apache.pulsar.broker.service.BrokerServiceException; +import org.apache.pulsar.broker.service.GetStatsOptions; import org.apache.pulsar.broker.web.RestException; +import org.apache.pulsar.client.admin.LongRunningProcessStatus; +import org.apache.pulsar.client.admin.OffloadProcessStatus; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.impl.ResetCursorData; @@ -75,6 +80,7 @@ import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.policies.data.SubscribeRate; +import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.policies.data.impl.AutoSubscriptionCreationOverrideImpl; import org.apache.pulsar.common.policies.data.impl.BacklogQuotaImpl; @@ -117,7 +123,7 @@ public void getList( internalGetListAsync(Optional.ofNullable(bundle)) .thenAccept(topicList -> asyncResponse.resume(filterSystemTopic(topicList, includeSystemTopic))) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get topic list {}", clientAppId(), namespaceName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -148,7 +154,7 @@ public void getPartitionedTopicList( .thenAccept(partitionedTopicList -> asyncResponse.resume( filterSystemTopic(partitionedTopicList, includeSystemTopic))) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get partitioned topic list {}", clientAppId(), namespaceName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -161,7 +167,10 @@ public void getPartitionedTopicList( @ApiOperation(value = "Get permissions on a topic.", notes = "Retrieve the effective permissions for a topic." + " These permissions are defined by the permissions set at the" - + "namespace level combined (union) with any eventual specific permission set on the topic.") + + "namespace level combined (union) with any eventual specific permission set on the topic." + + "Returns a nested map structure which Swagger does not fully support for display. " + + "Structure: Map>. Please refer to this structure for details.", + response = AuthAction.class, responseContainer = "Map") @ApiResponses(value = { @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"), @ApiResponse(code = 403, message = "Don't have admin permission"), @@ -194,6 +203,7 @@ public void getPermissionsOnTopic( @Path("/{tenant}/{namespace}/{topic}/permissions/{role}") @ApiOperation(value = "Grant a new permission to a role on a single topic.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"), @ApiResponse(code = 403, message = "Don't have admin permission"), @@ -231,6 +241,7 @@ public void grantPermissionsOnTopic( + "level, but rather at the namespace level," + " this operation will return an error (HTTP status code 412).") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"), @ApiResponse(code = 403, message = "Don't have admin permission"), @@ -262,6 +273,7 @@ public void revokePermissionsOnTopic( @ApiOperation(value = "Create a partitioned topic.", notes = "It needs to be called before creating a producer on a partitioned topic.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"), @ApiResponse(code = 403, message = "Don't have admin permission"), @@ -304,6 +316,7 @@ public void createPartitionedTopic( @ApiOperation(value = "Create a non-partitioned topic.", notes = "This is the only REST endpoint from which non-partitioned topics could be created.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"), @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist"), @@ -333,7 +346,7 @@ public void createNonPartitionedTopic( internalCreateNonPartitionedTopicAsync(authoritative, properties) .thenAccept(__ -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to create non-partitioned topic {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -343,7 +356,7 @@ public void createNonPartitionedTopic( @GET @Path("/{tenant}/{namespace}/{topic}/offloadPolicies") - @ApiOperation(value = "Get offload policies on a topic.") + @ApiOperation(value = "Get offload policies on a topic.", response = OffloadPoliciesImpl.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic doesn't exist"), @ApiResponse(code = 500, message = "Internal server error"), }) @@ -356,7 +369,8 @@ public void getOffloadPolicies(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.OFFLOAD, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetOffloadPolicies(applied, isGlobal)) .thenApply(asyncResponse::resume) .exceptionally(ex -> { @@ -368,7 +382,9 @@ public void getOffloadPolicies(@Suspended final AsyncResponse asyncResponse, @POST @Path("/{tenant}/{namespace}/{topic}/offloadPolicies") @ApiOperation(value = "Set offload policies on a topic.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic doesn't exist"), }) public void setOffloadPolicies(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @@ -379,7 +395,9 @@ public void setOffloadPolicies(@Suspended final AsyncResponse asyncResponse, @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal, @ApiParam(value = "Offload policies for the specified topic") OffloadPoliciesImpl offloadPolicies) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.OFFLOAD, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) + .thenAccept(__ -> validateOffloadPolicies(offloadPolicies)) .thenCompose(__ -> internalSetOffloadPolicies(offloadPolicies, isGlobal)) .thenRun(() -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { @@ -391,7 +409,9 @@ public void setOffloadPolicies(@Suspended final AsyncResponse asyncResponse, @DELETE @Path("/{tenant}/{namespace}/{topic}/offloadPolicies") @ApiOperation(value = "Delete offload policies on a topic.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic doesn't exist"), }) public void removeOffloadPolicies(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @@ -401,7 +421,8 @@ public void removeOffloadPolicies(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.OFFLOAD, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetOffloadPolicies(null, isGlobal)) .thenRun(() -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { @@ -425,7 +446,8 @@ public void getMaxUnackedMessagesOnConsumer(@Suspended final AsyncResponse async @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.MAX_UNACKED, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetMaxUnackedMessagesOnConsumer(applied, isGlobal)) .thenApply(asyncResponse::resume).exceptionally(ex -> { handleTopicPolicyException("getMaxUnackedMessagesOnConsumer", ex, asyncResponse); @@ -436,7 +458,9 @@ public void getMaxUnackedMessagesOnConsumer(@Suspended final AsyncResponse async @POST @Path("/{tenant}/{namespace}/{topic}/maxUnackedMessagesOnConsumer") @ApiOperation(value = "Set max unacked messages per consumer config on a topic.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic doesn't exist"), }) public void setMaxUnackedMessagesOnConsumer( @Suspended final AsyncResponse asyncResponse, @@ -449,7 +473,8 @@ public void setMaxUnackedMessagesOnConsumer( @ApiParam(value = "Max unacked messages on consumer policies for the specified topic") Integer maxUnackedNum) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.MAX_UNACKED, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetMaxUnackedMessagesOnConsumer(maxUnackedNum, isGlobal)) .thenRun(() -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { @@ -461,7 +486,9 @@ public void setMaxUnackedMessagesOnConsumer( @DELETE @Path("/{tenant}/{namespace}/{topic}/maxUnackedMessagesOnConsumer") @ApiOperation(value = "Delete max unacked messages per consumer config on a topic.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic doesn't exist"), }) public void deleteMaxUnackedMessagesOnConsumer(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @@ -471,7 +498,8 @@ public void deleteMaxUnackedMessagesOnConsumer(@Suspended final AsyncResponse as @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.MAX_UNACKED, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetMaxUnackedMessagesOnConsumer(null, isGlobal)) .thenRun(() -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { @@ -494,7 +522,8 @@ public void getDeduplicationSnapshotInterval(@Suspended final AsyncResponse asyn @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.DEDUPLICATION_SNAPSHOT, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> getTopicPoliciesAsyncWithRetry(topicName, isGlobal)) .thenAccept(op -> { TopicPolicies topicPolicies = op.orElseGet(TopicPolicies::new); @@ -509,7 +538,9 @@ public void getDeduplicationSnapshotInterval(@Suspended final AsyncResponse asyn @POST @Path("/{tenant}/{namespace}/{topic}/deduplicationSnapshotInterval") @ApiOperation(value = "Set deduplicationSnapshotInterval config on a topic.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic doesn't exist"), }) public void setDeduplicationSnapshotInterval( @Suspended final AsyncResponse asyncResponse, @@ -522,7 +553,8 @@ public void setDeduplicationSnapshotInterval( @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.DEDUPLICATION_SNAPSHOT, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetDeduplicationSnapshotInterval(interval, isGlobal)) .thenRun(() -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { @@ -534,7 +566,9 @@ public void setDeduplicationSnapshotInterval( @DELETE @Path("/{tenant}/{namespace}/{topic}/deduplicationSnapshotInterval") @ApiOperation(value = "Delete deduplicationSnapshotInterval config on a topic.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic doesn't exist"), }) public void deleteDeduplicationSnapshotInterval(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @@ -544,7 +578,8 @@ public void deleteDeduplicationSnapshotInterval(@Suspended final AsyncResponse a @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.DEDUPLICATION_SNAPSHOT, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetDeduplicationSnapshotInterval(null, isGlobal)) .thenRun(() -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { @@ -568,7 +603,8 @@ public void getInactiveTopicPolicies(@Suspended final AsyncResponse asyncRespons @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.INACTIVE_TOPIC, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetInactiveTopicPolicies(applied, isGlobal)) .thenApply(asyncResponse::resume).exceptionally(ex -> { handleTopicPolicyException("getInactiveTopicPolicies", ex, asyncResponse); @@ -579,7 +615,9 @@ public void getInactiveTopicPolicies(@Suspended final AsyncResponse asyncRespons @POST @Path("/{tenant}/{namespace}/{topic}/inactiveTopicPolicies") @ApiOperation(value = "Set inactive topic policies on a topic.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic doesn't exist"), }) public void setInactiveTopicPolicies(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @@ -591,7 +629,8 @@ public void setInactiveTopicPolicies(@Suspended final AsyncResponse asyncRespons @ApiParam(value = "inactive topic policies for the specified topic") InactiveTopicPolicies inactiveTopicPolicies) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.INACTIVE_TOPIC, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetInactiveTopicPolicies(inactiveTopicPolicies, isGlobal)) .thenRun(() -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { @@ -603,7 +642,9 @@ public void setInactiveTopicPolicies(@Suspended final AsyncResponse asyncRespons @DELETE @Path("/{tenant}/{namespace}/{topic}/inactiveTopicPolicies") @ApiOperation(value = "Delete inactive topic policies on a topic.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic doesn't exist"), }) public void deleteInactiveTopicPolicies(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @@ -613,7 +654,8 @@ public void deleteInactiveTopicPolicies(@Suspended final AsyncResponse asyncResp @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.INACTIVE_TOPIC, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetInactiveTopicPolicies(null, isGlobal)) .thenRun(() -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { @@ -637,7 +679,8 @@ public void getMaxUnackedMessagesOnSubscription(@Suspended final AsyncResponse a @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.MAX_UNACKED, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetMaxUnackedMessagesOnSubscription(applied, isGlobal)) .thenApply(asyncResponse::resume) .exceptionally(ex -> { @@ -649,7 +692,9 @@ public void getMaxUnackedMessagesOnSubscription(@Suspended final AsyncResponse a @POST @Path("/{tenant}/{namespace}/{topic}/maxUnackedMessagesOnSubscription") @ApiOperation(value = "Set max unacked messages per subscription config on a topic.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic doesn't exist"), }) public void setMaxUnackedMessagesOnSubscription( @Suspended final AsyncResponse asyncResponse, @@ -677,7 +722,9 @@ public void setMaxUnackedMessagesOnSubscription( @DELETE @Path("/{tenant}/{namespace}/{topic}/maxUnackedMessagesOnSubscription") @ApiOperation(value = "Delete max unacked messages per subscription config on a topic.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic doesn't exist"), }) public void deleteMaxUnackedMessagesOnSubscription(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @@ -712,7 +759,8 @@ public void getDelayedDeliveryPolicies(@Suspended final AsyncResponse asyncRespo @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.DELAYED_DELIVERY, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetDelayedDeliveryPolicies(applied, isGlobal)) .thenApply(asyncResponse::resume) .exceptionally(ex -> { @@ -724,7 +772,9 @@ public void getDelayedDeliveryPolicies(@Suspended final AsyncResponse asyncRespo @POST @Path("/{tenant}/{namespace}/{topic}/delayedDelivery") @ApiOperation(value = "Set delayed delivery messages config on a topic.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic doesn't exist"), }) public void setDelayedDeliveryPolicies( @Suspended final AsyncResponse asyncResponse, @@ -753,7 +803,9 @@ public void setDelayedDeliveryPolicies( @DELETE @Path("/{tenant}/{namespace}/{topic}/delayedDelivery") @ApiOperation(value = "Set delayed delivery messages config on a topic.") - @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic doesn't exist"), }) public void deleteDelayedDeliveryPolicies(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @@ -810,7 +862,11 @@ public void updatePartitionedTopic( @ApiParam(value = "The number of partitions for the topic", required = true, type = "int", defaultValue = "0") int numPartitions) { - validatePartitionedTopicName(tenant, namespace, encodedTopic); + validateTopicName(tenant, namespace, encodedTopic); + if (topicName.isPartitioned()) { + throw new RestException(Response.Status.PRECONDITION_FAILED, + "Partitioned Topic Name should not contain '-partition-'"); + } validateTopicPolicyOperationAsync(topicName, PolicyName.PARTITION, PolicyOperation.WRITE) .thenCompose(__ -> internalUpdatePartitionedTopicAsync(numPartitions, updateLocalTopic, force)) .thenAccept(__ -> { @@ -818,7 +874,7 @@ public void updatePartitionedTopic( asyncResponse.resume(Response.noContent().build()); }) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}][{}] Failed to update partition to {}", clientAppId(), topicName, numPartitions, ex); } @@ -832,6 +888,7 @@ public void updatePartitionedTopic( @Path("/{tenant}/{namespace}/{topic}/createMissedPartitions") @ApiOperation(value = "Create missed partitions of an existing partitioned topic.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"), @@ -886,8 +943,15 @@ public void getPartitionedMetadata( internalGetPartitionedMetadataAsync(authoritative, checkAllowAutoCreation) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { - log.error("[{}] Failed to get partitioned metadata topic {}", clientAppId(), topicName, ex); + Throwable t = FutureUtil.unwrapCompletionException(ex); + if (!isRedirectException(t)) { + if (AdminResource.isNotFoundException(t)) { + log.error("[{}] Failed to get partitioned metadata topic {}: {}", + clientAppId(), topicName, ex.getMessage()); + } else { + log.error("[{}] Failed to get partitioned metadata topic {}", + clientAppId(), topicName, t); + } } resumeAsyncResponseExceptionally(asyncResponse, ex); return null; @@ -920,7 +984,7 @@ public void getProperties( internalGetPropertiesAsync(authoritative) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get topic {} properties", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -932,6 +996,7 @@ public void getProperties( @Path("/{tenant}/{namespace}/{topic}/properties") @ApiOperation(value = "Update the properties on the given topic.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" + "subscriber is not authorized to access this operation"), @@ -956,7 +1021,7 @@ public void updateProperties( internalUpdatePropertiesAsync(authoritative, properties) .thenAccept(__ -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to update topic {} properties", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -968,6 +1033,7 @@ public void updateProperties( @Path("/{tenant}/{namespace}/{topic}/properties") @ApiOperation(value = "Remove the key in properties on the given topic.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"), @ApiResponse(code = 403, message = "Don't have admin permission"), @@ -990,7 +1056,7 @@ public void removeProperties( internalRemovePropertiesAsync(authoritative, key) .thenAccept(__ -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to remove key {} in properties on topic {}", clientAppId(), key, topicName, ex); } @@ -1004,6 +1070,7 @@ public void removeProperties( @ApiOperation(value = "Delete a partitioned topic.", notes = "It will also delete all the partitions of the topic if it exists.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"), @ApiResponse(code = 403, message = "Don't have admin permission"), @@ -1044,6 +1111,7 @@ public void deletePartitionedTopic( @Path("/{tenant}/{namespace}/{topic}/unload") @ApiOperation(value = "Unload a topic") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"), @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic does not exist"), @@ -1078,6 +1146,7 @@ public void unloadTopic( + "subscription or producer connected to the it. " + "Force delete ignores connected clients and deletes topic by explicitly closing them.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"), @ApiResponse(code = 403, message = "Don't have admin permission"), @@ -1098,7 +1167,17 @@ public void deleteTopic( @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - internalDeleteTopicAsync(authoritative, force) + + getPulsarResources().getNamespaceResources().getPartitionedTopicResources() + .partitionedTopicExistsAsync(topicName).thenAccept(exists -> { + if (exists) { + RestException restException = new RestException(Response.Status.CONFLICT, + String.format("%s is a partitioned topic, instead of calling delete topic, please call" + + " delete-partitioned-topic.", topicName)); + resumeAsyncResponseExceptionally(asyncResponse, restException); + return; + } + internalDeleteTopicAsync(authoritative, force) .thenAccept(__ -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { Throwable t = FutureUtil.unwrapCompletionException(ex); @@ -1111,12 +1190,14 @@ public void deleteTopic( } else if (isManagedLedgerNotFoundException(t)) { ex = new RestException(Response.Status.NOT_FOUND, getTopicNotFoundErrorMessage(topicName.toString())); - } else if (!isRedirectException(ex)) { + } else if (isNot307And404Exception(ex)) { log.error("[{}] Failed to delete topic {}", clientAppId(), topicName, t); } resumeAsyncResponseExceptionally(asyncResponse, ex); return null; }); + }); + } @GET @@ -1182,13 +1263,20 @@ public void getStats( + "not to use when there's heavy traffic.") @QueryParam("subscriptionBacklogSize") @DefaultValue("true") boolean subscriptionBacklogSize, @ApiParam(value = "If return time of the earliest message in backlog") - @QueryParam("getEarliestTimeInBacklog") @DefaultValue("false") boolean getEarliestTimeInBacklog) { + @QueryParam("getEarliestTimeInBacklog") @DefaultValue("false") boolean getEarliestTimeInBacklog, + @ApiParam(value = "If exclude the publishers") + @QueryParam("excludePublishers") @DefaultValue("false") boolean excludePublishers, + @ApiParam(value = "If exclude the consumers") + @QueryParam("excludeConsumers") @DefaultValue("false") boolean excludeConsumers) { validateTopicName(tenant, namespace, encodedTopic); - internalGetStatsAsync(authoritative, getPreciseBacklog, subscriptionBacklogSize, getEarliestTimeInBacklog) + GetStatsOptions getStatsOptions = + new GetStatsOptions(getPreciseBacklog, subscriptionBacklogSize, getEarliestTimeInBacklog, + excludePublishers, excludeConsumers); + internalGetStatsAsync(authoritative, getStatsOptions) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get stats for {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -1222,7 +1310,7 @@ public void getInternalStats( internalGetInternalStatsAsync(authoritative, metadata) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get internal stats for topic {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -1284,11 +1372,20 @@ public void getPartitionedStats( + "not to use when there's heavy traffic.") @QueryParam("subscriptionBacklogSize") @DefaultValue("true") boolean subscriptionBacklogSize, @ApiParam(value = "If return the earliest time in backlog") - @QueryParam("getEarliestTimeInBacklog") @DefaultValue("false") boolean getEarliestTimeInBacklog) { + @QueryParam("getEarliestTimeInBacklog") @DefaultValue("false") boolean getEarliestTimeInBacklog, + @ApiParam(value = "If exclude the publishers") + @QueryParam("excludePublishers") @DefaultValue("false") boolean excludePublishers, + @ApiParam(value = "If exclude the consumers") + @QueryParam("excludeConsumers") @DefaultValue("false") boolean excludeConsumers) { try { - validatePartitionedTopicName(tenant, namespace, encodedTopic); - internalGetPartitionedStats(asyncResponse, authoritative, perPartition, getPreciseBacklog, - subscriptionBacklogSize, getEarliestTimeInBacklog); + validateTopicName(tenant, namespace, encodedTopic); + if (topicName.isPartitioned()) { + throw new RestException(Response.Status.PRECONDITION_FAILED, + "Partitioned Topic Name should not contain '-partition-'"); + } + GetStatsOptions getStatsOptions = new GetStatsOptions(getPreciseBacklog, subscriptionBacklogSize, + getEarliestTimeInBacklog, excludePublishers, excludeConsumers); + internalGetPartitionedStats(asyncResponse, authoritative, perPartition, getStatsOptions); } catch (WebApplicationException wae) { asyncResponse.resume(wae); } catch (Exception e) { @@ -1337,6 +1434,7 @@ public void getPartitionedStatsInternal( + " there are any active consumers attached to it. " + "Force delete ignores connected consumers and deletes subscription by explicitly closing them.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"), @ApiResponse(code = 403, message = "Don't have admin permission"), @@ -1389,6 +1487,7 @@ public void deleteSubscription( @ApiOperation(value = "Skip all messages on a topic subscription.", notes = "Completely clears the backlog on the subscription.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" + "subscriber is not authorized to access this operation"), @@ -1424,6 +1523,7 @@ public void skipAllMessages( @Path("/{tenant}/{namespace}/{topic}/subscription/{subName}/skip/{numMessages}") @ApiOperation(value = "Skipping messages on a topic subscription.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"), @ApiResponse(code = 403, message = "Don't have admin permission"), @@ -1460,6 +1560,7 @@ public void skipMessages( @Path("/{tenant}/{namespace}/{topic}/subscription/{subName}/expireMessages/{expireTimeInSeconds}") @ApiOperation(value = "Expiry messages on a topic subscription.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" + "subscriber is not authorized to access this operation"), @@ -1497,6 +1598,7 @@ public void expireTopicMessages( @Path("/{tenant}/{namespace}/{topic}/subscription/{subName}/expireMessages") @ApiOperation(value = "Expiry messages on a topic subscription.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" + "subscriber is not authorized to access this operation"), @@ -1536,6 +1638,7 @@ public void expireTopicMessages( @Path("/{tenant}/{namespace}/{topic}/all_subscription/expireMessages/{expireTimeInSeconds}") @ApiOperation(value = "Expiry messages on all subscriptions of topic.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" + "subscriber is not authorized to access this operation"), @@ -1572,6 +1675,7 @@ public void expireMessagesForAllSubscriptions( @ApiOperation(value = "Create a subscription on the topic.", notes = "Creates a subscription on the topic at the specified message id") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 400, message = "Create subscription on non persistent topic is not supported"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" @@ -1589,7 +1693,7 @@ public void createSubscription( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String topic, - @ApiParam(value = "Subscription to create position on", required = true) + @ApiParam(value = "Name of subscription to be created", required = true) @PathParam("subscriptionName") String encodedSubName, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @@ -1627,6 +1731,7 @@ public void createSubscription( @ApiOperation(value = "Reset subscription to message position closest to absolute timestamp (in ms).", notes = "It fence cursor and disconnects all active consumers before resetting cursor.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" + "subscriber is not authorized to access this operation"), @@ -1677,6 +1782,7 @@ public void resetCursor( @Path("/{tenant}/{namespace}/{topic}/subscription/{subName}/properties") @ApiOperation(value = "Replace all the properties on the given subscription") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" + "subscriber is not authorized to access this operation"), @@ -1712,7 +1818,8 @@ public void updateSubscriptionProperties( @GET @Path("/{tenant}/{namespace}/{topic}/subscription/{subName}/properties") - @ApiOperation(value = "Return all the properties on the given subscription") + @ApiOperation(value = "Return all the properties on the given subscription", + response = String.class, responseContainer = "Map") @ApiResponses(value = { @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" @@ -1750,6 +1857,7 @@ public void getSubscriptionProperties( @Path("/{tenant}/{namespace}/{topic}/subscription/{subName}/analyzeBacklog") @ApiOperation(value = "Analyse a subscription, by scanning all the unprocessed messages") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" + "subscriber is not authorized to access this operation"), @@ -1776,7 +1884,7 @@ public void analyzeSubscriptionBacklog( try { Optional positionImpl; if (position != null) { - positionImpl = Optional.of(new PositionImpl(position.getLedgerId(), + positionImpl = Optional.of(PositionFactory.create(position.getLedgerId(), position.getEntryId())); } else { positionImpl = Optional.empty(); @@ -1796,6 +1904,7 @@ public void analyzeSubscriptionBacklog( @ApiOperation(value = "Reset subscription to message position closest to given position.", notes = "It fence cursor and disconnects all active consumers before resetting cursor.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" + "subscriber is not authorized to access this operation"), @@ -1834,6 +1943,13 @@ public void resetCursorOnPosition( @Path("/{tenant}/{namespace}/{topic}/subscription/{subName}/position/{messagePosition}") @ApiOperation(value = "Peek nth message on a topic subscription.") @ApiResponses(value = { + @ApiResponse( + code = 200, + message = "Successfully retrieved the message. The response is a binary byte stream " + + "containing the message data. Clients need to parse this binary stream based" + + " on the message metadata provided in the response headers.", + response = byte[].class + ), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" + "subscriber is not authorized to access this operation"), @@ -1862,7 +1978,7 @@ public void peekNthMessage( internalPeekNthMessageAsync(decode(encodedSubName), messagePosition, authoritative) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get peek nth message for topic {} subscription {}", clientAppId(), topicName, decode(encodedSubName), ex); } @@ -1876,6 +1992,13 @@ public void peekNthMessage( @ApiOperation(value = "Examine a specific message on a topic by position relative to the earliest or the latest message.") @ApiResponses(value = { + @ApiResponse( + code = 200, + message = "Successfully retrieved the message. The response is a binary byte stream " + + "containing the message data. Clients need to parse this binary stream based" + + " on the message metadata provided in the response headers.", + response = byte[].class + ), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic, the message position does not exist"), @@ -1902,21 +2025,28 @@ public void examineMessage( @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); internalExamineMessageAsync(initialPosition, messagePosition, authoritative) - .thenAccept(asyncResponse::resume) - .exceptionally(ex -> { - if (!isRedirectException(ex)) { - log.error("[{}] Failed to examine a specific message on the topic {}", clientAppId(), topicName, - ex); - } - resumeAsyncResponseExceptionally(asyncResponse, ex); - return null; - }); + .thenAccept(asyncResponse::resume) + .exceptionally(ex -> { + if (isNot307And404Exception(ex)) { + log.error("[{}] Failed to examine a specific message on the topic {}", clientAppId(), topicName, + ex); + } + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); } @GET @Path("/{tenant}/{namespace}/{topic}/ledger/{ledgerId}/entry/{entryId}") @ApiOperation(value = "Get message by its messageId.") @ApiResponses(value = { + @ApiResponse( + code = 200, + message = "Successfully retrieved the message. The response is a binary byte stream " + + "containing the message data. Clients need to parse this binary stream based" + + " on the message metadata provided in the response headers.", + response = byte[].class + ), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" + "subscriber is not authorized to access this operation"), @@ -1946,7 +2076,7 @@ public void getMessageById( .thenAccept(asyncResponse::resume) .exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get message with ledgerId {} entryId {} from {}", clientAppId(), ledgerId, entryId, topicName, ex); } @@ -1957,7 +2087,8 @@ public void getMessageById( @GET @Path("/{tenant}/{namespace}/{topic}/messageid/{timestamp}") - @ApiOperation(value = "Get message ID published at or just after this absolute timestamp (in ms).") + @ApiOperation(value = "Get message ID published at or just after this absolute timestamp (in ms).", + response = MessageIdAdv.class) @ApiResponses(value = { @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" @@ -1990,7 +2121,7 @@ public void getMessageIdByTimestamp( } }) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get message ID by timestamp {} from {}", clientAppId(), timestamp, topicName, ex); } @@ -2017,7 +2148,8 @@ public void getBacklog( @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - internalGetBacklogAsync(authoritative) + validateTopicOperationAsync(topicName, TopicOperation.GET_BACKLOG_SIZE) + .thenCompose(__ -> internalGetBacklogAsync(authoritative)) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { Throwable t = FutureUtil.unwrapCompletionException(ex); @@ -2025,7 +2157,7 @@ public void getBacklog( log.warn("[{}] Failed to get topic backlog {}: Namespace does not exist", clientAppId(), namespaceName); ex = new RestException(Response.Status.NOT_FOUND, "Namespace does not exist"); - } else if (!isRedirectException(ex)) { + } else if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get estimated backlog for topic {}", clientAppId(), encodedTopic, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -2073,7 +2205,8 @@ public void getBacklogQuotaMap( @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.BACKLOG, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetBacklogQuota(applied, isGlobal)) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { @@ -2085,7 +2218,9 @@ public void getBacklogQuotaMap( @POST @Path("/{tenant}/{namespace}/{topic}/backlogQuota") @ApiOperation(value = "Set a backlog quota for a topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @ApiResponse(code = 405, @@ -2102,7 +2237,8 @@ public void setBacklogQuota( @QueryParam("backlogQuotaType") BacklogQuotaType backlogQuotaType, @ApiParam(value = "backlog quota policies for the specified topic") BacklogQuotaImpl backlogQuota) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.BACKLOG, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetBacklogQuota(backlogQuotaType, backlogQuota, isGlobal)) .thenRun(() -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { @@ -2114,7 +2250,9 @@ public void setBacklogQuota( @DELETE @Path("/{tenant}/{namespace}/{topic}/backlogQuota") @ApiOperation(value = "Remove a backlog quota policy from a topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, to enable the topic level policy and retry"), @@ -2127,7 +2265,8 @@ public void removeBacklogQuota(@Suspended final AsyncResponse asyncResponse, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.BACKLOG, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetBacklogQuota(backlogQuotaType, null, isGlobal)) .thenRun(() -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { @@ -2156,7 +2295,8 @@ public void getReplicationClusters(@Suspended final AsyncResponse asyncResponse, + "For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.REPLICATION, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> getTopicPoliciesAsyncWithRetry(topicName)) .thenAccept(op -> { asyncResponse.resume(op.map(TopicPolicies::getReplicationClustersSet).orElseGet(() -> { @@ -2175,7 +2315,9 @@ public void getReplicationClusters(@Suspended final AsyncResponse asyncResponse, @POST @Path("/{tenant}/{namespace}/{topic}/replication") @ApiOperation(value = "Set the replication clusters for a topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @ApiResponse(code = 405, @@ -2189,7 +2331,8 @@ public void setReplicationClusters( @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "List of replication clusters", required = true) List clusterIds) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.REPLICATION, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetReplicationClusters(clusterIds)) .thenRun(() -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { @@ -2201,7 +2344,9 @@ public void setReplicationClusters( @DELETE @Path("/{tenant}/{namespace}/{topic}/replication") @ApiOperation(value = "Remove the replication clusters from a topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, to enable the topic level policy and retry"), @@ -2209,11 +2354,11 @@ public void setReplicationClusters( public void removeReplicationClusters(@Suspended final AsyncResponse asyncResponse, @PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @QueryParam("backlogQuotaType") BacklogQuotaType backlogQuotaType, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.REPLICATION, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalRemoveReplicationClusters()) .thenRun(() -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { @@ -2238,7 +2383,8 @@ public void getMessageTTL(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.TTL, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> getTopicPoliciesAsyncWithRetry(topicName, isGlobal)) .thenAccept(op -> asyncResponse.resume(op .map(TopicPolicies::getMessageTTLInSeconds) @@ -2259,7 +2405,9 @@ public void getMessageTTL(@Suspended final AsyncResponse asyncResponse, @POST @Path("/{tenant}/{namespace}/{topic}/messageTTL") @ApiOperation(value = "Set message TTL in seconds for a topic") - @ApiResponses(value = {@ApiResponse(code = 403, message = + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Not authenticate to perform the request or policy is read only"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = @@ -2275,7 +2423,8 @@ public void setMessageTTL(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.TTL, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetMessageTTL(messageTTL, isGlobal)) .thenRun(() -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { @@ -2288,6 +2437,7 @@ public void setMessageTTL(@Suspended final AsyncResponse asyncResponse, @Path("/{tenant}/{namespace}/{topic}/messageTTL") @ApiOperation(value = "Remove message TTL in seconds for a topic") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 403, message = "Not authenticate to perform the request or policy is read only"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @@ -2302,7 +2452,8 @@ public void removeMessageTTL(@Suspended final AsyncResponse asyncResponse, @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.TTL, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetMessageTTL(null, isGlobal)) .thenRun(() -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { @@ -2328,7 +2479,8 @@ public void getDeduplication(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.DEDUPLICATION, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetDeduplication(applied, isGlobal)) .thenApply(asyncResponse::resume) .exceptionally(ex -> { @@ -2340,7 +2492,9 @@ public void getDeduplication(@Suspended final AsyncResponse asyncResponse, @POST @Path("/{tenant}/{namespace}/{topic}/deduplicationEnabled") @ApiOperation(value = "Set deduplication enabled on a topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, to enable the topic level policy and retry")}) @@ -2355,7 +2509,8 @@ public void setDeduplication( @ApiParam(value = "DeduplicationEnabled policies for the specified topic") Boolean enabled) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.DEDUPLICATION, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetDeduplication(enabled, isGlobal)) .thenRun(() -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { @@ -2367,7 +2522,9 @@ public void setDeduplication( @DELETE @Path("/{tenant}/{namespace}/{topic}/deduplicationEnabled") @ApiOperation(value = "Remove deduplication configuration for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, to enable the topic level policy and retry"), @@ -2380,7 +2537,8 @@ public void removeDeduplication(@Suspended final AsyncResponse asyncResponse, @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.DEDUPLICATION, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetDeduplication(null, isGlobal)) .thenRun(() -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { @@ -2406,7 +2564,8 @@ public void getRetention(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RETENTION, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetRetention(applied, isGlobal)) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { @@ -2418,7 +2577,9 @@ public void getRetention(@Suspended final AsyncResponse asyncResponse, @POST @Path("/{tenant}/{namespace}/{topic}/retention") @ApiOperation(value = "Set retention configuration for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, to enable the topic level policy and retry"), @@ -2433,7 +2594,8 @@ public void setRetention(@Suspended final AsyncResponse asyncResponse, @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal, @ApiParam(value = "Retention policies for the specified topic") RetentionPolicies retention) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RETENTION, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetRetention(retention, isGlobal)) .thenRun(() -> { try { @@ -2455,7 +2617,9 @@ public void setRetention(@Suspended final AsyncResponse asyncResponse, @DELETE @Path("/{tenant}/{namespace}/{topic}/retention") @ApiOperation(value = "Remove retention configuration for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, to enable the topic level policy and retry"), @@ -2469,7 +2633,8 @@ public void removeRetention(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RETENTION, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalRemoveRetention(isGlobal)) .thenRun(() -> { log.info("[{}] Successfully remove retention: namespace={}, topic={}", @@ -2484,6 +2649,97 @@ public void removeRetention(@Suspended final AsyncResponse asyncResponse, }); } + @POST + @Path("/{tenant}/{namespace}/{topic}/dispatcherPauseOnAckStatePersistent") + @ApiOperation(value = "Set dispatcher pause on ack state persistent configuration for specified topic.") + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), + @ApiResponse(code = 405, message = + "Topic level policy is disabled, to enable the topic level policy and retry"), + @ApiResponse(code = 409, message = "Concurrent modification")}) + public void setDispatcherPauseOnAckStatePersistent(@Suspended final AsyncResponse asyncResponse, + @PathParam("tenant") String tenant, + @PathParam("namespace") String namespace, + @PathParam("topic") @Encoded String encodedTopic, + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, + @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal) { + validateTopicName(tenant, namespace, encodedTopic); + validateTopicPolicyOperationAsync(topicName, + PolicyName.DISPATCHER_PAUSE_ON_ACK_STATE_PERSISTENT, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) + .thenCompose(__ -> internalSetDispatcherPauseOnAckStatePersistent(isGlobal)) + .thenRun(() -> { + log.info("[{}] Successfully enabled dispatcherPauseOnAckStatePersistent: namespace={}, topic={}", + clientAppId(), namespaceName, topicName.getLocalName()); + asyncResponse.resume(Response.noContent().build()); + }) + .exceptionally(ex -> { + handleTopicPolicyException("setDispatcherPauseOnAckStatePersistent", ex, asyncResponse); + return null; + }); + } + + @DELETE + @Path("/{tenant}/{namespace}/{topic}/dispatcherPauseOnAckStatePersistent") + @ApiOperation(value = "Remove dispatcher pause on ack state persistent configuration for specified topic.") + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), + @ApiResponse(code = 405, + message = "Topic level policy is disabled, to enable the topic level policy and retry"), + @ApiResponse(code = 409, message = "Concurrent modification")}) + public void removeDispatcherPauseOnAckStatePersistent(@Suspended final AsyncResponse asyncResponse, + @PathParam("tenant") String tenant, + @PathParam("namespace") String namespace, + @PathParam("topic") @Encoded String encodedTopic, + @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal, + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + validateTopicName(tenant, namespace, encodedTopic); + validateTopicPolicyOperationAsync(topicName, + PolicyName.DISPATCHER_PAUSE_ON_ACK_STATE_PERSISTENT, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) + .thenCompose(__ -> internalRemoveDispatcherPauseOnAckStatePersistent(isGlobal)) + .thenRun(() -> { + log.info("[{}] Successfully remove dispatcherPauseOnAckStatePersistent: namespace={}, topic={}", + clientAppId(), namespaceName, topicName.getLocalName()); + asyncResponse.resume(Response.noContent().build()); + }) + .exceptionally(ex -> { + handleTopicPolicyException("removeDispatcherPauseOnAckStatePersistent", ex, asyncResponse); + return null; + }); + } + + @GET + @Path("/{tenant}/{namespace}/{topic}/dispatcherPauseOnAckStatePersistent") + @ApiOperation(value = "Get dispatcher pause on ack state persistent config on a topic.", response = Integer.class) + @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic doesn't exist"), + @ApiResponse(code = 500, message = "Internal server error"), }) + public void getDispatcherPauseOnAckStatePersistent(@Suspended final AsyncResponse asyncResponse, + @PathParam("tenant") String tenant, + @PathParam("namespace") String namespace, + @PathParam("topic") @Encoded String encodedTopic, + @QueryParam("applied") @DefaultValue("false") boolean applied, + @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal, + @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + validateTopicName(tenant, namespace, encodedTopic); + validateTopicPolicyOperationAsync(topicName, + PolicyName.DISPATCHER_PAUSE_ON_ACK_STATE_PERSISTENT, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) + .thenCompose(__ -> internalGetDispatcherPauseOnAckStatePersistent(applied, isGlobal)) + .thenApply(asyncResponse::resume).exceptionally(ex -> { + handleTopicPolicyException("getDispatcherPauseOnAckStatePersistent", ex, asyncResponse); + return null; + }); + } + @GET @Path("/{tenant}/{namespace}/{topic}/persistence") @ApiOperation( @@ -2504,7 +2760,8 @@ public void getPersistence(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.PERSISTENCE, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetPersistence(applied, isGlobal)) .thenApply(asyncResponse::resume) .exceptionally(ex -> { @@ -2516,7 +2773,9 @@ public void getPersistence(@Suspended final AsyncResponse asyncResponse, @POST @Path("/{tenant}/{namespace}/{topic}/persistence") @ApiOperation(value = "Set configuration of persistence policies for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, to enable the topic level policy and retry"), @@ -2532,7 +2791,8 @@ public void setPersistence(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Bookkeeper persistence policies for specified topic") PersistencePolicies persistencePolicies) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.PERSISTENCE, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetPersistence(persistencePolicies, isGlobal)) .thenRun(() -> { try { @@ -2555,7 +2815,9 @@ public void setPersistence(@Suspended final AsyncResponse asyncResponse, @DELETE @Path("/{tenant}/{namespace}/{topic}/persistence") @ApiOperation(value = "Remove configuration of persistence policies for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, to enable the topic level policy and retry"), @@ -2568,7 +2830,8 @@ public void removePersistence(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.PERSISTENCE, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalRemovePersistence(isGlobal)) .thenRun(() -> { log.info("[{}] Successfully remove persistence policies: namespace={}, topic={}", @@ -2599,7 +2862,8 @@ public void getMaxSubscriptionsPerTopic(@Suspended final AsyncResponse asyncResp @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.MAX_SUBSCRIPTIONS, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetMaxSubscriptionsPerTopic(isGlobal)) .thenAccept(op -> asyncResponse.resume(op.isPresent() ? op.get() : Response.noContent().build())) @@ -2612,7 +2876,9 @@ public void getMaxSubscriptionsPerTopic(@Suspended final AsyncResponse asyncResp @POST @Path("/{tenant}/{namespace}/{topic}/maxSubscriptionsPerTopic") @ApiOperation(value = "Set maxSubscriptionsPerTopic config for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, to enable the topic level policy and retry"), @@ -2627,7 +2893,8 @@ public void setMaxSubscriptionsPerTopic(@Suspended final AsyncResponse asyncResp @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "The max subscriptions of the topic") int maxSubscriptionsPerTopic) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.MAX_SUBSCRIPTIONS, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetMaxSubscriptionsPerTopic(maxSubscriptionsPerTopic, isGlobal)) .thenRun(() -> { log.info("[{}] Successfully updated maxSubscriptionsPerTopic: namespace={}, topic={}" @@ -2644,7 +2911,9 @@ public void setMaxSubscriptionsPerTopic(@Suspended final AsyncResponse asyncResp @DELETE @Path("/{tenant}/{namespace}/{topic}/maxSubscriptionsPerTopic") @ApiOperation(value = "Remove maxSubscriptionsPerTopic config for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, to enable the topic level policy and retry"), @@ -2657,7 +2926,8 @@ public void removeMaxSubscriptionsPerTopic(@Suspended final AsyncResponse asyncR @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.MAX_SUBSCRIPTIONS, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetMaxSubscriptionsPerTopic(null, isGlobal)) .thenRun(() -> { log.info("[{}] Successfully remove maxSubscriptionsPerTopic: namespace={}, topic={}", @@ -2687,7 +2957,8 @@ public void getReplicatorDispatchRate(@Suspended final AsyncResponse asyncRespon @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.REPLICATION_RATE, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetReplicatorDispatchRate(applied, isGlobal)) .thenApply(asyncResponse::resume) .exceptionally(ex -> { @@ -2699,7 +2970,9 @@ public void getReplicatorDispatchRate(@Suspended final AsyncResponse asyncRespon @POST @Path("/{tenant}/{namespace}/{topic}/replicatorDispatchRate") @ApiOperation(value = "Set replicatorDispatchRate config for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, to enable the topic level policy and retry"), @@ -2714,7 +2987,8 @@ public void setReplicatorDispatchRate(@Suspended final AsyncResponse asyncRespon @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Replicator dispatch rate of the topic") DispatchRateImpl dispatchRate) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.REPLICATION_RATE, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetReplicatorDispatchRate(dispatchRate, isGlobal)) .thenRun(() -> { log.info("[{}] Successfully updated replicatorDispatchRate: namespace={}, topic={}" @@ -2731,7 +3005,9 @@ public void setReplicatorDispatchRate(@Suspended final AsyncResponse asyncRespon @DELETE @Path("/{tenant}/{namespace}/{topic}/replicatorDispatchRate") @ApiOperation(value = "Remove replicatorDispatchRate config for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, to enable the topic level policy and retry"), @@ -2744,7 +3020,8 @@ public void removeReplicatorDispatchRate(@Suspended final AsyncResponse asyncRes @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.REPLICATION_RATE, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetReplicatorDispatchRate(null, isGlobal)) .thenRun(() -> { log.info("[{}] Successfully remove replicatorDispatchRate limit: namespace={}, topic={}", @@ -2774,7 +3051,8 @@ public void getMaxProducers(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.MAX_PRODUCERS, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetMaxProducers(applied, isGlobal)) .thenApply(asyncResponse::resume) .exceptionally(ex -> { @@ -2786,7 +3064,9 @@ public void getMaxProducers(@Suspended final AsyncResponse asyncResponse, @POST @Path("/{tenant}/{namespace}/{topic}/maxProducers") @ApiOperation(value = "Set maxProducers config for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, to enable the topic level policy and retry"), @@ -2801,7 +3081,8 @@ public void setMaxProducers(@Suspended final AsyncResponse asyncResponse, @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal, @ApiParam(value = "The max producers of the topic") int maxProducers) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.MAX_PRODUCERS, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetMaxProducers(maxProducers, isGlobal)) .thenRun(() -> { log.info("[{}] Successfully updated max producers: namespace={}, topic={}, maxProducers={}", @@ -2820,7 +3101,9 @@ public void setMaxProducers(@Suspended final AsyncResponse asyncResponse, @DELETE @Path("/{tenant}/{namespace}/{topic}/maxProducers") @ApiOperation(value = "Remove maxProducers config for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, to enable the topic level policy and retry"), @@ -2833,7 +3116,8 @@ public void removeMaxProducers(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.MAX_PRODUCERS, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalRemoveMaxProducers(isGlobal)) .thenRun(() -> { log.info("[{}] Successfully remove max producers: namespace={}, topic={}", @@ -2865,7 +3149,8 @@ public void getMaxConsumers(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.MAX_CONSUMERS, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetMaxConsumers(applied, isGlobal)) .thenApply(asyncResponse::resume) .exceptionally(ex -> { @@ -2877,7 +3162,9 @@ public void getMaxConsumers(@Suspended final AsyncResponse asyncResponse, @POST @Path("/{tenant}/{namespace}/{topic}/maxConsumers") @ApiOperation(value = "Set maxConsumers config for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, to enable the topic level policy and retry"), @@ -2892,7 +3179,8 @@ public void setMaxConsumers(@Suspended final AsyncResponse asyncResponse, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "The max consumers of the topic") int maxConsumers) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.MAX_CONSUMERS, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetMaxConsumers(maxConsumers, isGlobal)) .thenRun(() -> { log.info("[{}] Successfully updated max consumers: namespace={}, topic={}, maxConsumers={}", @@ -2911,7 +3199,9 @@ public void setMaxConsumers(@Suspended final AsyncResponse asyncResponse, @DELETE @Path("/{tenant}/{namespace}/{topic}/maxConsumers") @ApiOperation(value = "Remove maxConsumers config for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, to enable the topic level policy and retry"), @@ -2924,7 +3214,8 @@ public void removeMaxConsumers(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.MAX_CONSUMERS, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalRemoveMaxConsumers(isGlobal)) .thenRun(() -> { log.info("[{}] Successfully remove max consumers: namespace={}, topic={}", @@ -2955,7 +3246,8 @@ public void getMaxMessageSize(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateAdminAccessForTenantAsync(topicName.getTenant()) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetMaxMessageSize(isGlobal)) .thenAccept(policies -> { asyncResponse.resume(policies.isPresent() ? policies.get() : Response.noContent().build()); @@ -2969,7 +3261,9 @@ public void getMaxMessageSize(@Suspended final AsyncResponse asyncResponse, @POST @Path("/{tenant}/{namespace}/{topic}/maxMessageSize") @ApiOperation(value = "Set maxMessageSize config for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, to enable the topic level policy and retry"), @@ -2984,7 +3278,8 @@ public void setMaxMessageSize(@Suspended final AsyncResponse asyncResponse, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "The max message size of the topic") int maxMessageSize) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateAdminAccessForTenantAsync(topicName.getTenant()) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetMaxMessageSize(maxMessageSize, isGlobal)) .thenRun(() -> { log.info( @@ -3005,7 +3300,9 @@ public void setMaxMessageSize(@Suspended final AsyncResponse asyncResponse, @DELETE @Path("/{tenant}/{namespace}/{topic}/maxMessageSize") @ApiOperation(value = "Remove maxMessageSize config for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, to enable the topic level policy and retry"), @@ -3018,7 +3315,8 @@ public void removeMaxMessageSize(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateAdminAccessForTenantAsync(topicName.getTenant()) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetMaxMessageSize(null, isGlobal)) .thenRun(() -> { log.info("[{}] Successfully remove max message size: namespace={}, topic={}", @@ -3039,6 +3337,12 @@ public void removeMaxMessageSize(@Suspended final AsyncResponse asyncResponse, @ApiOperation(value = "Terminate a topic. A topic that is terminated will not accept any more " + "messages to be published and will let consumer to drain existing messages in backlog") @ApiResponses(value = { + @ApiResponse( + code = 200, + message = "Operation terminated successfully. The response includes the 'lastMessageId'," + + " which is the identifier of the last message processed.", + response = MessageIdAdv.class + ), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" + "subscriber is not authorized to access this operation"), @@ -3063,7 +3367,7 @@ public void terminate( internalTerminateAsync(authoritative) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to terminated topic {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -3076,6 +3380,7 @@ public void terminate( @ApiOperation(value = "Terminate all partitioned topic. A topic that is terminated will not accept any more " + "messages to be published and will let consumer to drain existing messages in backlog") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" + "subscriber is not authorized to access this operation"), @ApiResponse(code = 403, message = "Don't have admin permission"), @@ -3102,6 +3407,7 @@ public void terminatePartitionedTopic(@Suspended final AsyncResponse asyncRespon @Path("/{tenant}/{namespace}/{topic}/compaction") @ApiOperation(value = "Trigger a compaction operation on a topic.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" + "subscriber is not authorized to access this operation"), @@ -3134,7 +3440,8 @@ public void compact( @GET @Path("/{tenant}/{namespace}/{topic}/compaction") - @ApiOperation(value = "Get the status of a compaction operation for a topic.") + @ApiOperation(value = "Get the status of a compaction operation for a topic.", + response = LongRunningProcessStatus.class) @ApiResponses(value = { @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" @@ -3159,7 +3466,7 @@ public void compactionStatus( internalCompactionStatusAsync(authoritative) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get the status of a compaction operation for the topic {}", clientAppId(), topicName, ex); } @@ -3172,6 +3479,7 @@ public void compactionStatus( @Path("/{tenant}/{namespace}/{topic}/offload") @ApiOperation(value = "Offload a prefix of a topic to long term storage") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 400, message = "Message ID is null"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" @@ -3209,7 +3517,7 @@ public void triggerOffload( @GET @Path("/{tenant}/{namespace}/{topic}/offload") - @ApiOperation(value = "Offload a prefix of a topic to long term storage") + @ApiOperation(value = "Offload a prefix of a topic to long term storage", response = OffloadProcessStatus.class) @ApiResponses(value = { @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" @@ -3242,7 +3550,7 @@ public void offloadStatus( @GET @Path("/{tenant}/{namespace}/{topic}/lastMessageId") - @ApiOperation(value = "Return the last commit message id of topic") + @ApiOperation(value = "Return the last commit message id of topic", response = MessageIdAdv.class) @ApiResponses(value = { @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" @@ -3275,6 +3583,7 @@ public void getLastMessageId( @Path("/{tenant}/{namespace}/{topic}/trim") @ApiOperation(value = " Trim a topic") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or" + "subscriber is not authorized to access this operation"), @@ -3298,7 +3607,7 @@ public void trimTopic( validateTopicName(tenant, namespace, encodedTopic); internalTrimTopic(asyncResponse, authoritative).exceptionally(ex -> { // If the exception is not redirect exception we need to log it. - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to trim topic {}", clientAppId(), topicName, ex); } resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -3311,7 +3620,7 @@ public void trimTopic( @GET @Path("/{tenant}/{namespace}/{topic}/dispatchRate") - @ApiOperation(value = "Get dispatch rate configuration for specified topic.") + @ApiOperation(value = "Get dispatch rate configuration for specified topic.", response = DispatchRateImpl.class) @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, @@ -3326,7 +3635,8 @@ public void getDispatchRate(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RATE, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetDispatchRate(applied, isGlobal)) .thenApply(asyncResponse::resume) .exceptionally(ex -> { @@ -3338,7 +3648,9 @@ public void getDispatchRate(@Suspended final AsyncResponse asyncResponse, @POST @Path("/{tenant}/{namespace}/{topic}/dispatchRate") @ApiOperation(value = "Set message dispatch rate configuration for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, please enable the topic level policy and retry"), @@ -3352,7 +3664,8 @@ public void setDispatchRate(@Suspended final AsyncResponse asyncResponse, @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal, @ApiParam(value = "Dispatch rate for the specified topic") DispatchRateImpl dispatchRate) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RATE, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetDispatchRate(dispatchRate, isGlobal)) .thenRun(() -> { try { @@ -3375,7 +3688,9 @@ public void setDispatchRate(@Suspended final AsyncResponse asyncResponse, @DELETE @Path("/{tenant}/{namespace}/{topic}/dispatchRate") @ApiOperation(value = "Remove message dispatch rate configuration for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, please enable the topic level policy and retry"), @@ -3388,7 +3703,8 @@ public void removeDispatchRate(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RATE, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalRemoveDispatchRate(isGlobal)) .thenRun(() -> { log.info("[{}] Successfully remove topic dispatch rate: tenant={}, namespace={}, topic={}", @@ -3424,7 +3740,8 @@ public void getSubscriptionDispatchRate(@Suspended final AsyncResponse asyncResp @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RATE, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetSubscriptionDispatchRate(applied, isGlobal)) .thenApply(asyncResponse::resume) .exceptionally(ex -> { @@ -3436,7 +3753,9 @@ public void getSubscriptionDispatchRate(@Suspended final AsyncResponse asyncResp @POST @Path("/{tenant}/{namespace}/{topic}/subscriptionDispatchRate") @ApiOperation(value = "Set subscription message dispatch rate configuration for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, please enable the topic level policy and retry"), @@ -3452,7 +3771,8 @@ public void setSubscriptionDispatchRate( @ApiParam(value = "Subscription message dispatch rate for the specified topic") DispatchRateImpl dispatchRate) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RATE, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetSubscriptionDispatchRate(dispatchRate, isGlobal)) .thenRun(() -> { try { @@ -3475,7 +3795,9 @@ public void setSubscriptionDispatchRate( @DELETE @Path("/{tenant}/{namespace}/{topic}/subscriptionDispatchRate") @ApiOperation(value = "Remove subscription message dispatch rate configuration for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, please enable the topic level policy and retry"), @@ -3488,7 +3810,8 @@ public void removeSubscriptionDispatchRate(@Suspended final AsyncResponse asyncR @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RATE, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalRemoveSubscriptionDispatchRate(isGlobal)) .thenRun(() -> { log.info("[{}] Successfully remove topic subscription dispatch rate: tenant={}, namespace={}, topic={}", @@ -3506,7 +3829,8 @@ public void removeSubscriptionDispatchRate(@Suspended final AsyncResponse asyncR @GET @Path("/{tenant}/{namespace}/{topic}/{subName}/dispatchRate") - @ApiOperation(value = "Get message dispatch rate configuration for specified subscription.") + @ApiOperation(value = "Get message dispatch rate configuration for specified subscription.", + response = DispatchRate.class) @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, @@ -3522,7 +3846,8 @@ public void getSubscriptionLevelDispatchRate(@Suspended final AsyncResponse asyn @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RATE, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetSubscriptionLevelDispatchRate( Codec.decode(encodedSubscriptionName), applied, isGlobal)) .thenApply(asyncResponse::resume) @@ -3535,7 +3860,9 @@ public void getSubscriptionLevelDispatchRate(@Suspended final AsyncResponse asyn @POST @Path("/{tenant}/{namespace}/{topic}/{subName}/dispatchRate") @ApiOperation(value = "Set message dispatch rate configuration for specified subscription.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, please enable the topic level policy and retry"), @@ -3552,7 +3879,8 @@ public void setSubscriptionLevelDispatchRate( @ApiParam(value = "Subscription message dispatch rate for the specified topic") DispatchRateImpl dispatchRate) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RATE, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetSubscriptionLevelDispatchRate( Codec.decode(encodedSubscriptionName), dispatchRate, isGlobal)) .thenRun(() -> { @@ -3575,7 +3903,9 @@ public void setSubscriptionLevelDispatchRate( @DELETE @Path("/{tenant}/{namespace}/{topic}/{subName}/dispatchRate") @ApiOperation(value = "Remove message dispatch rate configuration for specified subscription.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, please enable the topic level policy and retry"), @@ -3590,7 +3920,8 @@ public void removeSubscriptionLevelDispatchRate( @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RATE, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalRemoveSubscriptionLevelDispatchRate( Codec.decode(encodedSubscriptionName), isGlobal)) .thenRun(() -> { @@ -3622,7 +3953,8 @@ public void getCompactionThreshold(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.COMPACTION, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetCompactionThreshold(applied, isGlobal)) .thenApply(asyncResponse::resume) .exceptionally(ex -> { @@ -3634,7 +3966,9 @@ public void getCompactionThreshold(@Suspended final AsyncResponse asyncResponse, @POST @Path("/{tenant}/{namespace}/{topic}/compactionThreshold") @ApiOperation(value = "Set compaction threshold configuration for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, please enable the topic level policy and retry"), @@ -3648,7 +3982,8 @@ public void setCompactionThreshold(@Suspended final AsyncResponse asyncResponse, @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal, @ApiParam(value = "Dispatch rate for the specified topic") long compactionThreshold) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.COMPACTION, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetCompactionThreshold(compactionThreshold, isGlobal)) .thenRun(() -> { try { @@ -3671,7 +4006,9 @@ public void setCompactionThreshold(@Suspended final AsyncResponse asyncResponse, @DELETE @Path("/{tenant}/{namespace}/{topic}/compactionThreshold") @ApiOperation(value = "Remove compaction threshold configuration for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, please enable the topic level policy and retry"), @@ -3684,7 +4021,8 @@ public void removeCompactionThreshold(@Suspended final AsyncResponse asyncRespon @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.COMPACTION, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalRemoveCompactionThreshold(isGlobal)) .thenRun(() -> { log.info("[{}] Successfully remove topic compaction threshold: tenant={}, namespace={}, topic={}", @@ -3719,7 +4057,8 @@ public void getMaxConsumersPerSubscription(@Suspended final AsyncResponse asyncR @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.MAX_CONSUMERS, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetMaxConsumersPerSubscription(isGlobal)) .thenAccept(op -> asyncResponse.resume(op.isPresent() ? op.get() : Response.noContent().build())) @@ -3732,7 +4071,9 @@ public void getMaxConsumersPerSubscription(@Suspended final AsyncResponse asyncR @POST @Path("/{tenant}/{namespace}/{topic}/maxConsumersPerSubscription") @ApiOperation(value = "Set max consumers per subscription configuration for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, please enable the topic level policy and retry"), @@ -3747,7 +4088,8 @@ public void setMaxConsumersPerSubscription( @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Dispatch rate for the specified topic") int maxConsumersPerSubscription) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.MAX_CONSUMERS, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetMaxConsumersPerSubscription(maxConsumersPerSubscription, isGlobal)) .thenRun(() -> { try { @@ -3770,7 +4112,9 @@ public void setMaxConsumersPerSubscription( @DELETE @Path("/{tenant}/{namespace}/{topic}/maxConsumersPerSubscription") @ApiOperation(value = "Remove max consumers per subscription configuration for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, please enable the topic level policy and retry"), @@ -3783,7 +4127,8 @@ public void removeMaxConsumersPerSubscription(@Suspended final AsyncResponse asy @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.MAX_CONSUMERS, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalRemoveMaxConsumersPerSubscription(isGlobal)) .thenRun(() -> { log.info("[{}] Successfully remove topic max consumers per subscription:" @@ -3816,7 +4161,8 @@ public void getPublishRate(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RATE, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetPublishRate(isGlobal)) .thenAccept(op -> asyncResponse.resume(op.isPresent() ? op.get() : Response.noContent().build())) @@ -3829,7 +4175,9 @@ public void getPublishRate(@Suspended final AsyncResponse asyncResponse, @POST @Path("/{tenant}/{namespace}/{topic}/publishRate") @ApiOperation(value = "Set message publish rate configuration for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, please enable the topic level policy and retry"), @@ -3843,7 +4191,8 @@ public void setPublishRate(@Suspended final AsyncResponse asyncResponse, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Dispatch rate for the specified topic") PublishRate publishRate) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RATE, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetPublishRate(publishRate, isGlobal)) .thenRun(() -> { try { @@ -3867,7 +4216,9 @@ public void setPublishRate(@Suspended final AsyncResponse asyncResponse, @DELETE @Path("/{tenant}/{namespace}/{topic}/publishRate") @ApiOperation(value = "Remove message publish rate configuration for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, please enable the topic level policy and retry"), @@ -3880,7 +4231,8 @@ public void removePublishRate(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RATE, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalRemovePublishRate(isGlobal)) .thenRun(() -> { log.info("[{}] Successfully remove topic publish rate: tenant={}, namespace={}, topic={}, isGlobal={}", @@ -3917,7 +4269,8 @@ public void getSubscriptionTypesEnabled(@Suspended final AsyncResponse asyncResp @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.SUBSCRIPTION_AUTH_MODE, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetSubscriptionTypesEnabled(isGlobal)) .thenAccept(op -> { asyncResponse.resume(op.isPresent() ? op.get() @@ -3932,7 +4285,9 @@ public void getSubscriptionTypesEnabled(@Suspended final AsyncResponse asyncResp @POST @Path("/{tenant}/{namespace}/{topic}/subscriptionTypesEnabled") @ApiOperation(value = "Set is enable sub types for specified topic") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, please enable the topic level policy and retry"), @@ -3947,7 +4302,8 @@ public void setSubscriptionTypesEnabled(@Suspended final AsyncResponse asyncResp @ApiParam(value = "Enable sub types for the specified topic") Set subscriptionTypesEnabled) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.SUBSCRIPTION_AUTH_MODE, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetSubscriptionTypesEnabled(subscriptionTypesEnabled, isGlobal)) .thenRun(() -> { try { @@ -3970,7 +4326,9 @@ public void setSubscriptionTypesEnabled(@Suspended final AsyncResponse asyncResp @DELETE @Path("/{tenant}/{namespace}/{topic}/subscriptionTypesEnabled") @ApiOperation(value = "Remove subscription types enabled for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, to enable the topic level policy and retry"), @@ -3983,7 +4341,8 @@ public void removeSubscriptionTypesEnabled(@Suspended final AsyncResponse asyncR @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.SUBSCRIPTION_AUTH_MODE, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalRemoveSubscriptionTypesEnabled(isGlobal)) .thenRun(() -> { log.info("[{}] Successfully remove subscription types enabled: namespace={}, topic={}", @@ -4015,7 +4374,8 @@ public void getSubscribeRate(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RATE, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetSubscribeRate(applied, isGlobal)) .thenApply(asyncResponse::resume).exceptionally(ex -> { handleTopicPolicyException("getSubscribeRate", ex, asyncResponse); @@ -4026,7 +4386,9 @@ public void getSubscribeRate(@Suspended final AsyncResponse asyncResponse, @POST @Path("/{tenant}/{namespace}/{topic}/subscribeRate") @ApiOperation(value = "Set subscribe rate configuration for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, please enable the topic level policy and retry"), @@ -4041,7 +4403,8 @@ public void setSubscribeRate( @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Subscribe rate for the specified topic") SubscribeRate subscribeRate) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RATE, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetSubscribeRate(subscribeRate, isGlobal)) .thenRun(() -> { try { @@ -4065,7 +4428,9 @@ public void setSubscribeRate( @DELETE @Path("/{tenant}/{namespace}/{topic}/subscribeRate") @ApiOperation(value = "Remove subscribe rate configuration for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, please enable the topic level policy and retry"), @@ -4079,7 +4444,8 @@ public void removeSubscribeRate(@Suspended final AsyncResponse asyncResponse, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Subscribe rate for the specified topic") SubscribeRate subscribeRate) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RATE, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalRemoveSubscribeRate(isGlobal)) .thenRun(() -> { log.info( @@ -4103,6 +4469,7 @@ public void removeSubscribeRate(@Suspended final AsyncResponse asyncResponse, notes = "The truncate operation will move all cursors to the end of the topic " + "and delete all inactive ledgers.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"), @ApiResponse(code = 403, message = "Don't have admin permission"), @@ -4138,6 +4505,7 @@ public void truncateTopic( @Path("/{tenant}/{namespace}/{topic}/subscription/{subName}/replicatedSubscriptionStatus") @ApiOperation(value = "Enable or disable a replicated subscription on a topic.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant or " + "subscriber is not authorized to access this operation"), @@ -4220,8 +4588,8 @@ public void getSchemaCompatibilityStrategy( @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.SCHEMA_COMPATIBILITY_STRATEGY, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__-> internalGetSchemaCompatibilityStrategy(applied)) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { @@ -4234,6 +4602,7 @@ public void getSchemaCompatibilityStrategy( @Path("/{tenant}/{namespace}/{topic}/schemaCompatibilityStrategy") @ApiOperation(value = "Set schema compatibility strategy on a topic") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 405, message = "Operation not allowed on persistent topic"), @@ -4251,8 +4620,8 @@ public void setSchemaCompatibilityStrategy( @ApiParam(value = "Strategy used to check the compatibility of new schema") SchemaCompatibilityStrategy strategy) { validateTopicName(tenant, namespace, encodedTopic); - - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.SCHEMA_COMPATIBILITY_STRATEGY, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetSchemaCompatibilityStrategy(strategy)) .thenRun(() -> { log.info( @@ -4274,6 +4643,7 @@ public void setSchemaCompatibilityStrategy( @Path("/{tenant}/{namespace}/{topic}/schemaCompatibilityStrategy") @ApiOperation(value = "Remove schema compatibility strategy on a topic") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 405, message = "Operation not allowed on persistent topic"), @@ -4291,8 +4661,8 @@ public void removeSchemaCompatibilityStrategy( @ApiParam(value = "Strategy used to check the compatibility of new schema") SchemaCompatibilityStrategy strategy) { validateTopicName(tenant, namespace, encodedTopic); - - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.SCHEMA_COMPATIBILITY_STRATEGY, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetSchemaCompatibilityStrategy(null)) .thenRun(() -> { log.info( @@ -4312,7 +4682,7 @@ public void removeSchemaCompatibilityStrategy( @GET @Path("/{tenant}/{namespace}/{topic}/schemaValidationEnforced") - @ApiOperation(value = "Get schema validation enforced flag for topic.") + @ApiOperation(value = "Get schema validation enforced flag for topic.", response = Boolean.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenants or Namespace doesn't exist") }) public void getSchemaValidationEnforced(@Suspended AsyncResponse asyncResponse, @@ -4327,7 +4697,8 @@ public void getSchemaValidationEnforced(@Suspended AsyncResponse asyncResponse, + "broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.SCHEMA_COMPATIBILITY_STRATEGY, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetSchemaValidationEnforced(applied)) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { @@ -4339,7 +4710,9 @@ public void getSchemaValidationEnforced(@Suspended AsyncResponse asyncResponse, @POST @Path("/{tenant}/{namespace}/{topic}/schemaValidationEnforced") @ApiOperation(value = "Set schema validation enforced flag on topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or Namespace doesn't exist"), @ApiResponse(code = 412, message = "schemaValidationEnforced value is not valid")}) public void setSchemaValidationEnforced(@Suspended AsyncResponse asyncResponse, @@ -4354,7 +4727,8 @@ public void setSchemaValidationEnforced(@Suspended AsyncResponse asyncResponse, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(required = true) boolean schemaValidationEnforced) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.SCHEMA_COMPATIBILITY_STRATEGY, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetSchemaValidationEnforced(schemaValidationEnforced)) .thenAccept(__ -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { @@ -4365,7 +4739,7 @@ public void setSchemaValidationEnforced(@Suspended AsyncResponse asyncResponse, @GET @Path("/{tenant}/{namespace}/{topic}/entryFilters") - @ApiOperation(value = "Get entry filters for a topic.") + @ApiOperation(value = "Get entry filters for a topic.", response = EntryFilters.class) @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenants or Namespace doesn't exist") }) public void getEntryFilters(@Suspended AsyncResponse asyncResponse, @@ -4381,7 +4755,8 @@ public void getEntryFilters(@Suspended AsyncResponse asyncResponse, + "broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.ENTRY_FILTERS, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetEntryFilters(applied, isGlobal)) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { @@ -4393,7 +4768,9 @@ public void getEntryFilters(@Suspended AsyncResponse asyncResponse, @POST @Path("/{tenant}/{namespace}/{topic}/entryFilters") @ApiOperation(value = "Set entry filters for specified topic") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, please enable the topic level policy and retry"), @@ -4409,7 +4786,8 @@ public void setEntryFilters(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Entry filters for the specified topic") EntryFilters entryFilters) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.ENTRY_FILTERS, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetEntryFilters(entryFilters, isGlobal)) .thenAccept(__ -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { @@ -4421,7 +4799,9 @@ public void setEntryFilters(@Suspended final AsyncResponse asyncResponse, @DELETE @Path("/{tenant}/{namespace}/{topic}/entryFilters") @ApiOperation(value = "Remove entry filters for specified topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, please enable the topic level policy and retry"), @@ -4435,7 +4815,8 @@ public void removeEntryFilters(@Suspended final AsyncResponse asyncResponse, + "call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.ENTRY_FILTERS, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalRemoveEntryFilters(isGlobal)) .thenRun(() -> { log.info( @@ -4455,7 +4836,8 @@ public void removeEntryFilters(@Suspended final AsyncResponse asyncResponse, @GET @Path("/{tenant}/{namespace}/{topic}/shadowTopics") - @ApiOperation(value = "Get the shadow topic list for a topic") + @ApiOperation(value = "Get the shadow topic list for a topic", + response = String.class, responseContainer = "List") @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = @@ -4468,9 +4850,8 @@ public void getShadowTopics( @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) - .thenCompose(__ -> validateTopicPolicyOperationAsync(topicName, PolicyName.SHADOW_TOPIC, - PolicyOperation.READ)) + validateTopicPolicyOperationAsync(topicName, PolicyName.SHADOW_TOPIC, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> getTopicPoliciesAsyncWithRetry(topicName)) .thenAccept(op -> asyncResponse.resume(op.map(TopicPolicies::getShadowTopics).orElse(null))) .exceptionally(ex -> { @@ -4482,7 +4863,9 @@ public void getShadowTopics( @PUT @Path("/{tenant}/{namespace}/{topic}/shadowTopics") @ApiOperation(value = "Set shadow topic list for a topic") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, enable the topic level policy and retry"), @@ -4497,7 +4880,8 @@ public void setShadowTopics( @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "List of shadow topics", required = true) List shadowTopics) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.SHADOW_TOPIC, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetShadowTopic(shadowTopics)) .thenRun(() -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { @@ -4509,7 +4893,9 @@ public void setShadowTopics( @DELETE @Path("/{tenant}/{namespace}/{topic}/shadowTopics") @ApiOperation(value = "Delete shadow topics for a topic") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace or topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, enable the topic level policy and retry"), @@ -4523,7 +4909,8 @@ public void deleteShadowTopics( @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.SHADOW_TOPIC, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalDeleteShadowTopics()) .thenRun(() -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { @@ -4535,7 +4922,9 @@ public void deleteShadowTopics( @POST @Path("/{tenant}/{namespace}/{topic}/autoSubscriptionCreation") @ApiOperation(value = "Override namespace's allowAutoSubscriptionCreation setting for a topic") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Topic doesn't exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, enable the topic level policy and retry"), @@ -4550,7 +4939,8 @@ public void setAutoSubscriptionCreation( @ApiParam(value = "Settings for automatic subscription creation") AutoSubscriptionCreationOverrideImpl autoSubscriptionCreationOverride) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.AUTO_SUBSCRIPTION_CREATION, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetAutoSubscriptionCreation(autoSubscriptionCreationOverride, isGlobal)) .thenAccept(__ -> asyncResponse.resume(Response.noContent().build())) .exceptionally(ex -> { @@ -4561,7 +4951,8 @@ public void setAutoSubscriptionCreation( @GET @Path("/{tenant}/{namespace}/{topic}/autoSubscriptionCreation") - @ApiOperation(value = "Get autoSubscriptionCreation info in a topic") + @ApiOperation(value = "Get autoSubscriptionCreation info in a topic", + response = AutoSubscriptionCreationOverrideImpl.class) @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Topic does not exist"), @ApiResponse(code = 405, @@ -4576,7 +4967,8 @@ public void getAutoSubscriptionCreation( @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.AUTO_SUBSCRIPTION_CREATION, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetAutoSubscriptionCreation(applied, isGlobal)) .thenApply(asyncResponse::resume).exceptionally(ex -> { handleTopicPolicyException("getAutoSubscriptionCreation", ex, asyncResponse); @@ -4587,7 +4979,9 @@ public void getAutoSubscriptionCreation( @DELETE @Path("/{tenant}/{namespace}/{topic}/autoSubscriptionCreation") @ApiOperation(value = "Remove autoSubscriptionCreation ina a topic.") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Topic does not exist"), @ApiResponse(code = 405, message = "Topic level policy is disabled, please enable the topic level policy and retry"), @@ -4601,7 +4995,8 @@ public void removeAutoSubscriptionCreation( @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.AUTO_SUBSCRIPTION_CREATION, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetAutoSubscriptionCreation(null, isGlobal)) .thenRun(() -> { log.info( diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ResourceGroups.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ResourceGroups.java index 52fd03b18ed0b..58f593e20ce3b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ResourceGroups.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ResourceGroups.java @@ -60,7 +60,9 @@ public ResourceGroup getResourceGroup(@PathParam("resourcegroup") String resourc @PUT @Path("/{resourcegroup}") @ApiOperation(value = "Creates a new resourcegroup with the specified rate limiters") - @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "cluster doesn't exist")}) public void createOrUpdateResourceGroup(@PathParam("resourcegroup") String name, @ApiParam(value = "Rate limiters for the resourcegroup") @@ -72,6 +74,7 @@ public void createOrUpdateResourceGroup(@PathParam("resourcegroup") String name, @Path("/{resourcegroup}") @ApiOperation(value = "Delete a resourcegroup.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "ResourceGroup doesn't exist"), @ApiResponse(code = 409, message = "ResourceGroup is in use")}) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ResourceQuotas.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ResourceQuotas.java index 58ccc1c10288c..d2884e8ea6f7e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ResourceQuotas.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/ResourceQuotas.java @@ -75,7 +75,7 @@ public void setDefaultResourceQuota( @GET @Path("/{tenant}/{namespace}/{bundle}") - @ApiOperation(value = "Get resource quota of a namespace bundle.") + @ApiOperation(value = "Get resource quota of a namespace bundle.", response = ResourceQuota.class) @ApiResponses(value = { @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace"), @ApiResponse(code = 403, message = "Don't have admin permission"), @@ -103,6 +103,7 @@ public void getNamespaceBundleResourceQuota( @Path("/{tenant}/{namespace}/{bundle}") @ApiOperation(value = "Set resource quota on a namespace.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace"), @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 409, message = "Concurrent modification") }) @@ -133,6 +134,7 @@ public void setNamespaceBundleResourceQuota( @Path("/{tenant}/{namespace}/{bundle}") @ApiOperation(value = "Remove resource quota for a namespace.") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace"), @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 409, message = "Concurrent modification") }) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/SchemasResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/SchemasResource.java index dd8ed58c853fa..07758436f6ca7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/SchemasResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/SchemasResource.java @@ -166,6 +166,36 @@ public void getAllSchemas( }); } + @GET + @Path("/{tenant}/{namespace}/{topic}/metadata") + @Produces(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Get the schema metadata of a topic", response = GetAllVersionsSchemaResponse.class) + @ApiResponses(value = { + @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), + @ApiResponse(code = 401, message = "Client is not authorized or Don't have admin permission"), + @ApiResponse(code = 403, message = "Client is not authenticated"), + @ApiResponse(code = 404, + message = "Tenant or Namespace or Topic doesn't exist; or Schema is not found for this topic"), + @ApiResponse(code = 412, message = "Failed to find the ownership for the topic"), + @ApiResponse(code = 500, message = "Internal Server Error"), + }) + public void getSchemaMetadata( + @PathParam("tenant") String tenant, + @PathParam("namespace") String namespace, + @PathParam("topic") String topic, + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, + @Suspended final AsyncResponse response + ) { + validateTopicName(tenant, namespace, topic); + getSchemaMetadataAsync(authoritative) + .thenAccept(response::resume) + .exceptionally(ex -> { + log.error("[{}] Failed to get schema metadata for topic {}", clientAppId(), topicName, ex); + resumeAsyncResponseExceptionally(response, ex); + return null; + }); + } + @DELETE @Path("/{tenant}/{namespace}/{topic}/schema") @Produces(MediaType.APPLICATION_JSON) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Worker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Worker.java index 3813790e4f428..7178b565719ca 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Worker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Worker.java @@ -87,7 +87,9 @@ public WorkerInfo getClusterLeader() { @GET @ApiOperation( value = "Fetches information about which Pulsar Functions are assigned to which Pulsar clusters", - response = Map.class + response = Map.class, + notes = "Returns a nested map structure which Swagger does not fully support for display." + + "Structure: Map>. Please refer to this structure for details." ) @ApiResponses(value = { @ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), @@ -102,7 +104,8 @@ public Map> getAssignments() { @GET @ApiOperation( value = "Fetches a list of supported Pulsar IO connectors currently running in cluster mode", - response = List.class + response = ConnectorDefinition.class, + responseContainer = "List" ) @ApiResponses(value = { @ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), @@ -120,6 +123,7 @@ public List getConnectorsList() throws IOException { value = "Triggers a rebalance of functions to workers" ) @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), @ApiResponse(code = 400, message = "Invalid request"), @ApiResponse(code = 408, message = "Request timeout") @@ -134,6 +138,7 @@ public void rebalance() { value = "Drains the specified worker, i.e., moves its work-assignments to other workers" ) @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 400, message = "Invalid request"), @ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), @ApiResponse(code = 408, message = "Request timeout"), @@ -150,6 +155,7 @@ public void drainAtLeader(@QueryParam("workerId") String workerId) { value = "Drains this worker, i.e., moves its work-assignments to other workers" ) @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 400, message = "Invalid request"), @ApiResponse(code = 403, message = "The requester doesn't have admin permissions"), @ApiResponse(code = 408, message = "Request timeout"), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Packages.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Packages.java index 15e7b69554dc7..4ca7e3948ff5a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Packages.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Packages.java @@ -80,7 +80,7 @@ public void getMeta( ) @ApiResponses( value = { - @ApiResponse(code = 200, message = "Update the metadata of the specified package successfully."), + @ApiResponse(code = 204, message = "Update the metadata of the specified package successfully."), @ApiResponse(code = 404, message = "The specified package is not existent."), @ApiResponse(code = 412, message = "The package name is illegal."), @ApiResponse(code = 500, message = "Internal server error."), @@ -113,7 +113,7 @@ public void updateMeta( ) @ApiResponses( value = { - @ApiResponse(code = 200, message = "Upload the specified package successfully."), + @ApiResponse(code = 204, message = "Upload the specified package successfully."), @ApiResponse(code = 412, message = "The package name is illegal."), @ApiResponse(code = 500, message = "Internal server error."), @ApiResponse(code = 503, message = "Package Management Service is not enabled in the broker.") @@ -169,7 +169,7 @@ public StreamingOutput download( @Path("/{type}/{tenant}/{namespace}/{packageName}/{version}") @ApiResponses( value = { - @ApiResponse(code = 200, message = "Delete the specified package successfully."), + @ApiResponse(code = 204, message = "Delete the specified package successfully."), @ApiResponse(code = 404, message = "The specified package is not existent."), @ApiResponse(code = 412, message = "The package name is illegal."), @ApiResponse(code = 500, message = "Internal server error."), @@ -218,7 +218,8 @@ public void listPackageVersion( @Path("/{type}/{tenant}/{namespace}") @ApiOperation( value = "Get all the specified type packages in a namespace.", - response = PackageMetadata.class + response = PackageMetadata.class, + responseContainer = "List" ) @ApiResponses( value = { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Transactions.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Transactions.java index aa24dbdcc3ae9..089ec53069287 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Transactions.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/Transactions.java @@ -39,11 +39,24 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.pulsar.broker.admin.impl.TransactionsBase; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.web.RestException; +import org.apache.pulsar.common.policies.data.TransactionBufferInternalStats; +import org.apache.pulsar.common.policies.data.TransactionBufferStats; +import org.apache.pulsar.common.policies.data.TransactionCoordinatorInfo; +import org.apache.pulsar.common.policies.data.TransactionCoordinatorInternalStats; +import org.apache.pulsar.common.policies.data.TransactionCoordinatorStats; +import org.apache.pulsar.common.policies.data.TransactionInBufferStats; +import org.apache.pulsar.common.policies.data.TransactionInPendingAckStats; +import org.apache.pulsar.common.policies.data.TransactionMetadata; +import org.apache.pulsar.common.policies.data.TransactionPendingAckInternalStats; +import org.apache.pulsar.common.policies.data.TransactionPendingAckStats; +import org.apache.pulsar.common.stats.PositionInPendingAckStats; import org.apache.pulsar.common.util.FutureUtil; +import org.jetbrains.annotations.Nullable; @Path("/transactions") @Produces(MediaType.APPLICATION_JSON) @@ -54,7 +67,8 @@ public class Transactions extends TransactionsBase { @GET @Path("/coordinators") - @ApiOperation(value = "List transaction coordinators.") + @ApiOperation(value = "List transaction coordinators.", + response = TransactionCoordinatorInfo.class, responseContainer = "List") @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 503, message = "This Broker is not " + "configured with transactionCoordinatorEnabled=true.")}) @@ -65,7 +79,7 @@ public void listCoordinators(@Suspended final AsyncResponse asyncResponse) { @GET @Path("/coordinatorStats") - @ApiOperation(value = "Get transaction coordinator stats.") + @ApiOperation(value = "Get transaction coordinator stats.", response = TransactionCoordinatorStats.class) @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 503, message = "This Broker is not " + "configured with transactionCoordinatorEnabled=true."), @@ -81,7 +95,7 @@ public void getCoordinatorStats(@Suspended final AsyncResponse asyncResponse, @GET @Path("/transactionInBufferStats/{tenant}/{namespace}/{topic}/{mostSigBits}/{leastSigBits}") - @ApiOperation(value = "Get transaction state in transaction buffer.") + @ApiOperation(value = "Get transaction state in transaction buffer.", response = TransactionInBufferStats.class) @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic doesn't exist"), @ApiResponse(code = 503, message = "This Broker is not configured " @@ -104,7 +118,7 @@ public void getTransactionInBufferStats(@Suspended final AsyncResponse asyncResp Long.parseLong(leastSigBits)) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get transaction state in transaction buffer {}", clientAppId(), topicName, ex); } @@ -118,7 +132,7 @@ public void getTransactionInBufferStats(@Suspended final AsyncResponse asyncResp @GET @Path("/transactionInPendingAckStats/{tenant}/{namespace}/{topic}/{subName}/{mostSigBits}/{leastSigBits}") - @ApiOperation(value = "Get transaction state in pending ack.") + @ApiOperation(value = "Get transaction state in pending ack.", response = TransactionInPendingAckStats.class) @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic doesn't exist"), @ApiResponse(code = 503, message = "This Broker is not configured " @@ -142,7 +156,7 @@ public void getTransactionInPendingAckStats(@Suspended final AsyncResponse async Long.parseLong(leastSigBits), subName) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get transaction state in pending ack {}", clientAppId(), topicName, ex); } @@ -156,7 +170,7 @@ public void getTransactionInPendingAckStats(@Suspended final AsyncResponse async @GET @Path("/transactionBufferStats/{tenant}/{namespace}/{topic}") - @ApiOperation(value = "Get transaction buffer stats in topic.") + @ApiOperation(value = "Get transaction buffer stats in topic.", response = TransactionBufferStats.class) @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic doesn't exist"), @ApiResponse(code = 503, message = "This Broker is not configured " @@ -171,14 +185,16 @@ public void getTransactionBufferStats(@Suspended final AsyncResponse asyncRespon @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("lowWaterMarks") @DefaultValue("false") - boolean lowWaterMarks) { + boolean lowWaterMarks, + @QueryParam("segmentStats") @DefaultValue("false") + boolean segmentStats) { try { checkTransactionCoordinatorEnabled(); validateTopicName(tenant, namespace, encodedTopic); - internalGetTransactionBufferStats(authoritative, lowWaterMarks) + internalGetTransactionBufferStats(authoritative, lowWaterMarks, segmentStats) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get transaction buffer stats in topic {}", clientAppId(), topicName, ex); } @@ -192,7 +208,7 @@ public void getTransactionBufferStats(@Suspended final AsyncResponse asyncRespon @GET @Path("/pendingAckStats/{tenant}/{namespace}/{topic}/{subName}") - @ApiOperation(value = "Get transaction pending ack stats in topic.") + @ApiOperation(value = "Get transaction pending ack stats in topic.", response = TransactionPendingAckStats.class) @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic or subName doesn't exist"), @ApiResponse(code = 503, message = "This Broker is not configured " @@ -214,7 +230,7 @@ public void getPendingAckStats(@Suspended final AsyncResponse asyncResponse, internalGetPendingAckStats(authoritative, subName, lowWaterMarks) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get transaction pending ack stats in topic {}", clientAppId(), topicName, ex); } @@ -228,7 +244,7 @@ public void getPendingAckStats(@Suspended final AsyncResponse asyncResponse, @GET @Path("/transactionMetadata/{mostSigBits}/{leastSigBits}") - @ApiOperation(value = "Get transaction metadata") + @ApiOperation(value = "Get transaction metadata", response = TransactionMetadata.class) @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic " + "or coordinator or transaction doesn't exist"), @@ -249,7 +265,7 @@ public void getTransactionMetadata(@Suspended final AsyncResponse asyncResponse, @GET @Path("/slowTransactions/{timeout}") - @ApiOperation(value = "Get slow transactions.") + @ApiOperation(value = "Get slow transactions.", response = TransactionMetadata.class, responseContainer = "Map") @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic " + "or coordinator or transaction doesn't exist"), @@ -269,7 +285,7 @@ public void getSlowTransactions(@Suspended final AsyncResponse asyncResponse, @GET @Path("/coordinatorInternalStats/{coordinatorId}") - @ApiOperation(value = "Get coordinator internal stats.") + @ApiOperation(value = "Get coordinator internal stats.", response = TransactionCoordinatorInternalStats.class) @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 503, message = "This Broker is not " + "configured with transactionCoordinatorEnabled=true."), @@ -287,7 +303,8 @@ public void getCoordinatorInternalStats(@Suspended final AsyncResponse asyncResp @GET @Path("/pendingAckInternalStats/{tenant}/{namespace}/{topic}/{subName}") - @ApiOperation(value = "Get transaction pending ack internal stats.") + @ApiOperation(value = "Get transaction pending ack internal stats.", + response = TransactionPendingAckInternalStats.class) @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic " + "or subscription name doesn't exist"), @@ -311,21 +328,62 @@ public void getPendingAckInternalStats(@Suspended final AsyncResponse asyncRespo internalGetPendingAckInternalStats(authoritative, subName, metadata) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { - if (!isRedirectException(ex)) { + if (isNot307And404Exception(ex)) { log.error("[{}] Failed to get pending ack internal stats {}", clientAppId(), topicName, ex); } - Throwable cause = FutureUtil.unwrapCompletionException(ex); - if (cause instanceof BrokerServiceException.ServiceUnitNotReadyException) { - asyncResponse.resume(new RestException(SERVICE_UNAVAILABLE, cause)); - } else if (cause instanceof BrokerServiceException.NotAllowedException) { - asyncResponse.resume(new RestException(METHOD_NOT_ALLOWED, cause)); - } else if (cause instanceof BrokerServiceException.SubscriptionNotFoundException) { - asyncResponse.resume(new RestException(NOT_FOUND, cause)); - } else { - asyncResponse.resume(new RestException(cause)); + return resumeAsyncResponseWithBrokerException(asyncResponse, ex); + }); + } catch (Exception ex) { + resumeAsyncResponseExceptionally(asyncResponse, ex); + } + } + + @Nullable + private Void resumeAsyncResponseWithBrokerException(@Suspended AsyncResponse asyncResponse, + Throwable ex) { + Throwable cause = FutureUtil.unwrapCompletionException(ex); + if (cause instanceof BrokerServiceException.ServiceUnitNotReadyException) { + asyncResponse.resume(new RestException(SERVICE_UNAVAILABLE, cause)); + } else if (cause instanceof BrokerServiceException.NotAllowedException) { + asyncResponse.resume(new RestException(METHOD_NOT_ALLOWED, cause)); + } else if (cause instanceof BrokerServiceException.SubscriptionNotFoundException) { + asyncResponse.resume(new RestException(NOT_FOUND, cause)); + } else { + asyncResponse.resume(new RestException(cause)); + } + return null; + } + + @GET + @Path("/transactionBufferInternalStats/{tenant}/{namespace}/{topic}") + @ApiOperation(value = "Get transaction buffer internal stats.", response = TransactionBufferInternalStats.class) + @ApiResponses(value = { + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic doesn't exist"), + @ApiResponse(code = 503, message = "This Broker is not enable transaction"), + @ApiResponse(code = 307, message = "Topic is not owned by this broker!"), + @ApiResponse(code = 405, message = "Transaction buffer don't use managedLedger!"), + @ApiResponse(code = 400, message = "Topic is not a persistent topic!"), + @ApiResponse(code = 409, message = "Concurrent modification") + }) + public void getTransactionBufferInternalStats(@Suspended final AsyncResponse asyncResponse, + @QueryParam("authoritative") + @DefaultValue("false") boolean authoritative, + @PathParam("tenant") String tenant, + @PathParam("namespace") String namespace, + @PathParam("topic") @Encoded String encodedTopic, + @QueryParam("metadata") @DefaultValue("false") boolean metadata) { + try { + validateTopicName(tenant, namespace, encodedTopic); + internalGetTransactionBufferInternalStats(authoritative, metadata) + .thenAccept(asyncResponse::resume) + .exceptionally(ex -> { + if (isNot307And404Exception(ex)) { + log.error("[{}] Failed to get transaction buffer internal stats {}", + clientAppId(), topicName, ex); } - return null; + return resumeAsyncResponseWithBrokerException(asyncResponse, ex); }); } catch (Exception ex) { resumeAsyncResponseExceptionally(asyncResponse, ex); @@ -335,6 +393,7 @@ public void getPendingAckInternalStats(@Suspended final AsyncResponse asyncRespo @POST @Path("/transactionCoordinator/replicas") @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), @ApiResponse(code = 503, message = "This Broker is not configured " + "with transactionCoordinatorEnabled=true."), @ApiResponse(code = 406, message = "The number of replicas should be more than " @@ -357,7 +416,7 @@ public void scaleTransactionCoordinators(@Suspended final AsyncResponse asyncRes @GET @Path("/positionStatsInPendingAck/{tenant}/{namespace}/{topic}/{subName}/{ledgerId}/{entryId}") - @ApiOperation(value = "Get position stats in pending ack.") + @ApiOperation(value = "Get position stats in pending ack.", response = PositionInPendingAckStats.class) @ApiResponses(value = {@ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic " + "or subscription name doesn't exist"), @@ -380,7 +439,7 @@ public void getPositionStatsInPendingAck(@Suspended final AsyncResponse asyncRes try { checkTransactionCoordinatorEnabled(); validateTopicName(tenant, namespace, encodedTopic); - PositionImpl position = new PositionImpl(ledgerId, entryId); + Position position = PositionFactory.create(ledgerId, entryId); internalGetPositionStatsPendingAckStats(authoritative, subName, position, batchIndex) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { @@ -395,4 +454,34 @@ public void getPositionStatsInPendingAck(@Suspended final AsyncResponse asyncRes } } + @POST + @Path("/abortTransaction/{mostSigBits}/{leastSigBits}") + @ApiOperation(value = "Abort transaction") + @ApiResponses(value = { + @ApiResponse(code = 204, message = "Operation successful"), + @ApiResponse(code = 404, message = "Tenant or cluster or namespace or topic " + + "or coordinator or transaction doesn't exist"), + @ApiResponse(code = 503, message = "This Broker is not configured " + + "with transactionCoordinatorEnabled=true."), + @ApiResponse(code = 307, message = "Topic is not owned by this broker!"), + @ApiResponse(code = 400, message = "Topic is not a persistent topic!"), + @ApiResponse(code = 409, message = "Concurrent modification"), + @ApiResponse(code = 401, message = "This operation requires super-user access")}) + public void abortTransaction(@Suspended final AsyncResponse asyncResponse, + @QueryParam("authoritative") + @DefaultValue("false") boolean authoritative, + @PathParam("mostSigBits") String mostSigBits, + @PathParam("leastSigBits") String leastSigBits) { + try { + checkTransactionCoordinatorEnabled(); + internalAbortTransaction(authoritative, Long.parseLong(mostSigBits), Long.parseLong(leastSigBits)) + .thenAccept(__ -> asyncResponse.resume(Response.noContent().build())) + .exceptionally(ex -> { + resumeAsyncResponseExceptionally(asyncResponse, ex); + return null; + }); + } catch (Exception e) { + resumeAsyncResponseExceptionally(asyncResponse, e); + } + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/cache/BundlesQuotas.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/cache/BundlesQuotas.java index d70520a09f3ce..d61fa0b0c81d5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/cache/BundlesQuotas.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/cache/BundlesQuotas.java @@ -19,18 +19,13 @@ package org.apache.pulsar.broker.cache; import java.util.concurrent.CompletableFuture; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.resources.LoadBalanceResources; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.policies.data.ResourceQuota; -import org.apache.pulsar.metadata.api.MetadataCache; -import org.apache.pulsar.metadata.api.MetadataStore; public class BundlesQuotas { - - // Root path for resource-quota - private static final String RESOURCE_QUOTA_ROOT = "/loadbalance/resource-quota"; - private static final String DEFAULT_RESOURCE_QUOTA_PATH = RESOURCE_QUOTA_ROOT + "/default"; - - private final MetadataCache resourceQuotaCache; + LoadBalanceResources loadBalanceResources; // Default initial quota static final ResourceQuota INITIAL_QUOTA = new ResourceQuota(); @@ -44,24 +39,21 @@ public class BundlesQuotas { INITIAL_QUOTA.setDynamic(true); // allow dynamically re-calculating } - public BundlesQuotas(MetadataStore localStore) { - this.resourceQuotaCache = localStore.getMetadataCache(ResourceQuota.class); + public BundlesQuotas(PulsarService pulsar) { + loadBalanceResources = pulsar.getPulsarResources().getLoadBalanceResources(); } public CompletableFuture setDefaultResourceQuota(ResourceQuota quota) { - return resourceQuotaCache.readModifyUpdateOrCreate(DEFAULT_RESOURCE_QUOTA_PATH, __ -> quota) - .thenApply(__ -> null); + return loadBalanceResources.getQuotaResources().setWithCreateDefaultQuotaAsync(quota); } public CompletableFuture getDefaultResourceQuota() { - return resourceQuotaCache.get(DEFAULT_RESOURCE_QUOTA_PATH) + return loadBalanceResources.getQuotaResources().getDefaultQuota() .thenApply(optResourceQuota -> optResourceQuota.orElse(INITIAL_QUOTA)); } public CompletableFuture setResourceQuota(String bundle, ResourceQuota quota) { - return resourceQuotaCache.readModifyUpdateOrCreate(RESOURCE_QUOTA_ROOT + "/" + bundle, - __ -> quota) - .thenApply(__ -> null); + return loadBalanceResources.getQuotaResources().setWithCreateQuotaAsync(bundle, quota); } public CompletableFuture setResourceQuota(NamespaceBundle bundle, ResourceQuota quota) { @@ -73,7 +65,7 @@ public CompletableFuture getResourceQuota(NamespaceBundle bundle) } public CompletableFuture getResourceQuota(String bundle) { - return resourceQuotaCache.get(RESOURCE_QUOTA_ROOT + "/" + bundle) + return loadBalanceResources.getQuotaResources().getQuota(bundle) .thenCompose(optResourceQuota -> { if (optResourceQuota.isPresent()) { return CompletableFuture.completedFuture(optResourceQuota.get()); @@ -84,7 +76,6 @@ public CompletableFuture getResourceQuota(String bundle) { } public CompletableFuture resetResourceQuota(NamespaceBundle bundle) { - return resourceQuotaCache.delete(RESOURCE_QUOTA_ROOT + "/" + bundle.toString()); + return loadBalanceResources.getQuotaResources().deleteQuota(bundle.toString()); } - } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java index 6a00bfd199584..11ad243e0c9d1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.delayed; +import com.google.common.annotations.VisibleForTesting; import io.netty.util.HashedWheelTimer; import io.netty.util.Timer; import io.netty.util.concurrent.DefaultThreadFactory; @@ -27,15 +28,21 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.commons.collections4.MapUtils; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.delayed.bucket.BookkeeperBucketSnapshotStorage; import org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker; import org.apache.pulsar.broker.delayed.bucket.BucketSnapshotStorage; +import org.apache.pulsar.broker.delayed.bucket.RecoverDelayedDeliveryTrackerException; +import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.apache.pulsar.common.util.FutureUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class BucketDelayedDeliveryTrackerFactory implements DelayedDeliveryTrackerFactory { + private static final Logger log = LoggerFactory.getLogger(BucketDelayedDeliveryTrackerFactory.class); BucketSnapshotStorage bucketSnapshotStorage; @@ -72,8 +79,28 @@ public void initialize(PulsarService pulsarService) throws Exception { @Override public DelayedDeliveryTracker newTracker(PersistentDispatcherMultipleConsumers dispatcher) { - return new BucketDelayedDeliveryTracker(dispatcher, timer, tickTimeMillis, isDelayedDeliveryDeliverAtTimeStrict, - bucketSnapshotStorage, delayedDeliveryMinIndexCountPerBucket, + String topicName = dispatcher.getTopic().getName(); + String subscriptionName = dispatcher.getSubscription().getName(); + BrokerService brokerService = dispatcher.getTopic().getBrokerService(); + DelayedDeliveryTracker tracker; + + try { + tracker = newTracker0(dispatcher); + } catch (RecoverDelayedDeliveryTrackerException ex) { + log.warn("Failed to recover BucketDelayedDeliveryTracker, fallback to InMemoryDelayedDeliveryTracker." + + " topic {}, subscription {}", topicName, subscriptionName, ex); + // If failed to create BucketDelayedDeliveryTracker, fallback to InMemoryDelayedDeliveryTracker + brokerService.initializeFallbackDelayedDeliveryTrackerFactory(); + tracker = brokerService.getFallbackDelayedDeliveryTrackerFactory().newTracker(dispatcher); + } + return tracker; + } + + @VisibleForTesting + BucketDelayedDeliveryTracker newTracker0(PersistentDispatcherMultipleConsumers dispatcher) + throws RecoverDelayedDeliveryTrackerException { + return new BucketDelayedDeliveryTracker(dispatcher, timer, tickTimeMillis, + isDelayedDeliveryDeliverAtTimeStrict, bucketSnapshotStorage, delayedDeliveryMinIndexCountPerBucket, TimeUnit.SECONDS.toMillis(delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds), delayedDeliveryMaxIndexesPerBucketSnapshotSegment, delayedDeliveryMaxNumBuckets); } @@ -85,6 +112,9 @@ public DelayedDeliveryTracker newTracker(PersistentDispatcherMultipleConsumers d */ public CompletableFuture cleanResidualSnapshots(ManagedCursor cursor) { Map cursorProperties = cursor.getCursorProperties(); + if (MapUtils.isEmpty(cursorProperties)) { + return CompletableFuture.completedFuture(null); + } List> futures = new ArrayList<>(); FutureUtil.Sequencer sequencer = FutureUtil.Sequencer.create(); cursorProperties.forEach((k, v) -> { @@ -105,5 +135,8 @@ public void close() throws Exception { if (bucketSnapshotStorage != null) { bucketSnapshotStorage.close(); } + if (timer != null) { + timer.stop(); + } } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java index 78229fef25a5a..7c954879fe845 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java @@ -21,7 +21,7 @@ import com.google.common.annotations.Beta; import java.util.NavigableSet; import java.util.concurrent.CompletableFuture; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; /** * Represent the tracker for the delayed delivery of messages for a particular subscription. @@ -59,7 +59,7 @@ public interface DelayedDeliveryTracker extends AutoCloseable { /** * Get a set of position of messages that have already reached the delivery time. */ - NavigableSet getScheduledMessages(int maxMessages); + NavigableSet getScheduledMessages(int maxMessages); /** * Tells whether the dispatcher should pause any message deliveries, until the DelayedDeliveryTracker has @@ -85,4 +85,51 @@ public interface DelayedDeliveryTracker extends AutoCloseable { * Close the subscription tracker and release all resources. */ void close(); + + DelayedDeliveryTracker DISABLE = new DelayedDeliveryTracker() { + @Override + public boolean addMessage(long ledgerId, long entryId, long deliveryAt) { + return false; + } + + @Override + public boolean hasMessageAvailable() { + return false; + } + + @Override + public long getNumberOfDelayedMessages() { + return 0; + } + + @Override + public long getBufferMemoryUsage() { + return 0; + } + + @Override + public NavigableSet getScheduledMessages(int maxMessages) { + return null; + } + + @Override + public boolean shouldPauseAllDeliveries() { + return false; + } + + @Override + public void resetTickTime(long tickTime) { + + } + + @Override + public CompletableFuture clear() { + return null; + } + + @Override + public void close() { + + } + }; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java index 58358b06a46bb..8bd9fafa13715 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java @@ -26,7 +26,8 @@ import java.util.concurrent.CompletableFuture; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; @@ -114,9 +115,9 @@ public boolean hasMessageAvailable() { * Get a set of position of messages that have already reached. */ @Override - public NavigableSet getScheduledMessages(int maxMessages) { + public NavigableSet getScheduledMessages(int maxMessages) { int n = maxMessages; - NavigableSet positions = new TreeSet<>(); + NavigableSet positions = new TreeSet<>(); long cutoffTime = getCutoffTime(); while (n > 0 && !priorityQueue.isEmpty()) { @@ -127,7 +128,7 @@ public NavigableSet getScheduledMessages(int maxMessages) { long ledgerId = priorityQueue.peekN2(); long entryId = priorityQueue.peekN3(); - positions.add(new PositionImpl(ledgerId, entryId)); + positions.add(PositionFactory.create(ledgerId, entryId)); priorityQueue.pop(); --n; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTrackerFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTrackerFactory.java index e7dc3f18f4630..179cf74db4179 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTrackerFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTrackerFactory.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.delayed; +import com.google.common.annotations.VisibleForTesting; import io.netty.util.HashedWheelTimer; import io.netty.util.Timer; import io.netty.util.concurrent.DefaultThreadFactory; @@ -25,8 +26,11 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class InMemoryDelayedDeliveryTrackerFactory implements DelayedDeliveryTrackerFactory { + private static final Logger log = LoggerFactory.getLogger(InMemoryDelayedDeliveryTrackerFactory.class); private Timer timer; @@ -48,6 +52,21 @@ public void initialize(PulsarService pulsarService) { @Override public DelayedDeliveryTracker newTracker(PersistentDispatcherMultipleConsumers dispatcher) { + String topicName = dispatcher.getTopic().getName(); + String subscriptionName = dispatcher.getSubscription().getName(); + DelayedDeliveryTracker tracker = DelayedDeliveryTracker.DISABLE; + try { + tracker = newTracker0(dispatcher); + } catch (Exception e) { + // it should never go here + log.warn("Failed to create InMemoryDelayedDeliveryTracker, topic {}, subscription {}", + topicName, subscriptionName, e); + } + return tracker; + } + + @VisibleForTesting + InMemoryDelayedDeliveryTracker newTracker0(PersistentDispatcherMultipleConsumers dispatcher) { return new InMemoryDelayedDeliveryTracker(dispatcher, timer, tickTimeMillis, isDelayedDeliveryDeliverAtTimeStrict, fixedDelayDetectionLookahead); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java index e99f39b382f56..8dcfe8d39a8b4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java @@ -107,7 +107,7 @@ public void start() throws Exception { pulsar.getIoEventLoopGroup(), Optional.empty(), null - ); + ).get(); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 67a7de1f01339..47c78fa9ee2ec 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -19,7 +19,7 @@ package org.apache.pulsar.broker.delayed.bucket; import static com.google.common.base.Preconditions.checkArgument; -import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX; +import static org.apache.bookkeeper.mledger.ManagedCursor.CURSOR_INTERNAL_PROPERTY_PREFIX; import static org.apache.pulsar.broker.delayed.bucket.Bucket.DELIMITER; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.HashBasedTable; @@ -48,8 +48,10 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.mutable.MutableLong; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.delayed.AbstractDelayedDeliveryTracker; @@ -104,22 +106,24 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker private CompletableFuture pendingLoad = null; public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, - Timer timer, long tickTimeMillis, - boolean isDelayedDeliveryDeliverAtTimeStrict, - BucketSnapshotStorage bucketSnapshotStorage, - long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegmentInMillis, - int maxIndexesPerBucketSnapshotSegment, int maxNumBuckets) { + Timer timer, long tickTimeMillis, + boolean isDelayedDeliveryDeliverAtTimeStrict, + BucketSnapshotStorage bucketSnapshotStorage, + long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegmentInMillis, + int maxIndexesPerBucketSnapshotSegment, int maxNumBuckets) + throws RecoverDelayedDeliveryTrackerException { this(dispatcher, timer, tickTimeMillis, Clock.systemUTC(), isDelayedDeliveryDeliverAtTimeStrict, bucketSnapshotStorage, minIndexCountPerBucket, timeStepPerBucketSnapshotSegmentInMillis, maxIndexesPerBucketSnapshotSegment, maxNumBuckets); } public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, - Timer timer, long tickTimeMillis, Clock clock, - boolean isDelayedDeliveryDeliverAtTimeStrict, - BucketSnapshotStorage bucketSnapshotStorage, - long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegmentInMillis, - int maxIndexesPerBucketSnapshotSegment, int maxNumBuckets) { + Timer timer, long tickTimeMillis, Clock clock, + boolean isDelayedDeliveryDeliverAtTimeStrict, + BucketSnapshotStorage bucketSnapshotStorage, + long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegmentInMillis, + int maxIndexesPerBucketSnapshotSegment, int maxNumBuckets) + throws RecoverDelayedDeliveryTrackerException { super(dispatcher, timer, tickTimeMillis, clock, isDelayedDeliveryDeliverAtTimeStrict); this.minIndexCountPerBucket = minIndexCountPerBucket; this.timeStepPerBucketSnapshotSegmentInMillis = timeStepPerBucketSnapshotSegmentInMillis; @@ -132,14 +136,27 @@ public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispat new MutableBucket(dispatcher.getName(), dispatcher.getCursor(), FutureUtil.Sequencer.create(), bucketSnapshotStorage); this.stats = new BucketDelayedMessageIndexStats(); - this.numberDelayedMessages = recoverBucketSnapshot(); + + // Close the tracker if failed to recover. + try { + this.numberDelayedMessages = recoverBucketSnapshot(); + } catch (RecoverDelayedDeliveryTrackerException e) { + close(); + throw e; + } } - private synchronized long recoverBucketSnapshot() throws RuntimeException { + private synchronized long recoverBucketSnapshot() throws RecoverDelayedDeliveryTrackerException { ManagedCursor cursor = this.lastMutableBucket.getCursor(); + Map cursorProperties = cursor.getCursorProperties(); + if (MapUtils.isEmpty(cursorProperties)) { + log.info("[{}] Recover delayed message index bucket snapshot finish, don't find bucket snapshot", + dispatcher.getName()); + return 0; + } FutureUtil.Sequencer sequencer = this.lastMutableBucket.getSequencer(); Map, ImmutableBucket> toBeDeletedBucketMap = new HashMap<>(); - cursor.getCursorProperties().keySet().forEach(key -> { + cursorProperties.keySet().forEach(key -> { if (key.startsWith(DELAYED_BUCKET_KEY_PREFIX)) { String[] keys = key.split(DELIMITER); checkArgument(keys.length == 3); @@ -174,7 +191,7 @@ private synchronized long recoverBucketSnapshot() throws RuntimeException { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } - throw new RuntimeException(e); + throw new RecoverDelayedDeliveryTrackerException(e); } for (Map.Entry, CompletableFuture>> entry : futures.entrySet()) { @@ -543,7 +560,7 @@ public long getBufferMemoryUsage() { } @Override - public synchronized NavigableSet getScheduledMessages(int maxMessages) { + public synchronized NavigableSet getScheduledMessages(int maxMessages) { if (!checkPendingLoadDone()) { if (log.isDebugEnabled()) { log.debug("[{}] Skip getScheduledMessages to wait for bucket snapshot load finish.", @@ -556,7 +573,7 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa lastMutableBucket.moveScheduledMessageToSharedQueue(cutoffTime, sharedBucketPriorityQueue); - NavigableSet positions = new TreeSet<>(); + NavigableSet positions = new TreeSet<>(); int n = maxMessages; while (n > 0 && !sharedBucketPriorityQueue.isEmpty()) { @@ -640,7 +657,7 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa } } - positions.add(new PositionImpl(ledgerId, entryId)); + positions.add(PositionFactory.create(ledgerId, entryId)); sharedBucketPriorityQueue.pop(); removeIndexBit(ledgerId, entryId); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/RecoverDelayedDeliveryTrackerException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/RecoverDelayedDeliveryTrackerException.java new file mode 100644 index 0000000000000..71a851100fe4e --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/RecoverDelayedDeliveryTrackerException.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed.bucket; + +public class RecoverDelayedDeliveryTrackerException extends Exception { + public RecoverDelayedDeliveryTrackerException(Throwable cause) { + super(cause); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoader.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoader.java index faee5799289d0..3997e214f4316 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoader.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoader.java @@ -29,7 +29,6 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Entry; -import org.apache.pulsar.broker.ClassLoaderSwitcher; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.broker.service.Producer; @@ -51,16 +50,20 @@ public class BrokerInterceptorWithClassLoader implements BrokerInterceptor { private final BrokerInterceptor interceptor; - private final NarClassLoader classLoader; + private final NarClassLoader narClassLoader; @Override public void beforeSendMessage(Subscription subscription, Entry entry, long[] ackSet, MessageMetadata msgMetadata) { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + final ClassLoader previousContext = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(narClassLoader); this.interceptor.beforeSendMessage( subscription, entry, ackSet, msgMetadata); + } finally { + Thread.currentThread().setContextClassLoader(previousContext); } } @@ -70,25 +73,37 @@ public void beforeSendMessage(Subscription subscription, long[] ackSet, MessageMetadata msgMetadata, Consumer consumer) { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + final ClassLoader previousContext = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(narClassLoader); this.interceptor.beforeSendMessage( subscription, entry, ackSet, msgMetadata, consumer); + } finally { + Thread.currentThread().setContextClassLoader(previousContext); } } @Override public void onMessagePublish(Producer producer, ByteBuf headersAndPayload, Topic.PublishContext publishContext) { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + final ClassLoader previousContext = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(narClassLoader); this.interceptor.onMessagePublish(producer, headersAndPayload, publishContext); + } finally { + Thread.currentThread().setContextClassLoader(previousContext); } } @Override public void producerCreated(ServerCnx cnx, Producer producer, Map metadata){ - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + final ClassLoader previousContext = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(narClassLoader); this.interceptor.producerCreated(cnx, producer, metadata); + } finally { + Thread.currentThread().setContextClassLoader(previousContext); } } @@ -96,8 +111,12 @@ public void producerCreated(ServerCnx cnx, Producer producer, public void producerClosed(ServerCnx cnx, Producer producer, Map metadata) { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + final ClassLoader previousContext = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(narClassLoader); this.interceptor.producerClosed(cnx, producer, metadata); + } finally { + Thread.currentThread().setContextClassLoader(previousContext); } } @@ -105,9 +124,12 @@ public void producerClosed(ServerCnx cnx, public void consumerCreated(ServerCnx cnx, Consumer consumer, Map metadata) { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { - this.interceptor.consumerCreated( - cnx, consumer, metadata); + final ClassLoader previousContext = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(narClassLoader); + this.interceptor.consumerCreated(cnx, consumer, metadata); + } finally { + Thread.currentThread().setContextClassLoader(previousContext); } } @@ -115,8 +137,12 @@ public void consumerCreated(ServerCnx cnx, public void consumerClosed(ServerCnx cnx, Consumer consumer, Map metadata) { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + final ClassLoader previousContext = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(narClassLoader); this.interceptor.consumerClosed(cnx, consumer, metadata); + } finally { + Thread.currentThread().setContextClassLoader(previousContext); } } @@ -124,87 +150,140 @@ public void consumerClosed(ServerCnx cnx, @Override public void messageProduced(ServerCnx cnx, Producer producer, long startTimeNs, long ledgerId, long entryId, Topic.PublishContext publishContext) { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + final ClassLoader previousContext = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(narClassLoader); this.interceptor.messageProduced(cnx, producer, startTimeNs, ledgerId, entryId, publishContext); + } finally { + Thread.currentThread().setContextClassLoader(previousContext); } } @Override public void messageDispatched(ServerCnx cnx, Consumer consumer, long ledgerId, long entryId, ByteBuf headersAndPayload) { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + final ClassLoader previousContext = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(narClassLoader); this.interceptor.messageDispatched(cnx, consumer, ledgerId, entryId, headersAndPayload); + } finally { + Thread.currentThread().setContextClassLoader(previousContext); } } @Override public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + final ClassLoader previousContext = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(narClassLoader); this.interceptor.messageAcked(cnx, consumer, ackCmd); + } finally { + Thread.currentThread().setContextClassLoader(previousContext); } } @Override public void txnOpened(long tcId, String txnID) { - this.interceptor.txnOpened(tcId, txnID); + final ClassLoader previousContext = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(narClassLoader); + this.interceptor.txnOpened(tcId, txnID); + } finally { + Thread.currentThread().setContextClassLoader(previousContext); + } } @Override public void txnEnded(String txnID, long txnAction) { - this.interceptor.txnEnded(txnID, txnAction); + final ClassLoader previousContext = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(narClassLoader); + this.interceptor.txnEnded(txnID, txnAction); + } finally { + Thread.currentThread().setContextClassLoader(previousContext); + } } @Override public void onConnectionCreated(ServerCnx cnx) { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + final ClassLoader previousContext = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(narClassLoader); this.interceptor.onConnectionCreated(cnx); + } finally { + Thread.currentThread().setContextClassLoader(previousContext); } } @Override public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + final ClassLoader previousContext = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(narClassLoader); this.interceptor.onPulsarCommand(command, cnx); + } finally { + Thread.currentThread().setContextClassLoader(previousContext); } } @Override public void onConnectionClosed(ServerCnx cnx) { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + final ClassLoader previousContext = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(narClassLoader); this.interceptor.onConnectionClosed(cnx); + } finally { + Thread.currentThread().setContextClassLoader(previousContext); } } @Override public void onWebserviceRequest(ServletRequest request) throws IOException, ServletException, InterceptException { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + final ClassLoader previousContext = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(narClassLoader); this.interceptor.onWebserviceRequest(request); + } finally { + Thread.currentThread().setContextClassLoader(previousContext); } } @Override public void onWebserviceResponse(ServletRequest request, ServletResponse response) throws IOException, ServletException { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + final ClassLoader previousContext = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(narClassLoader); this.interceptor.onWebserviceResponse(request, response); + } finally { + Thread.currentThread().setContextClassLoader(previousContext); } } @Override public void initialize(PulsarService pulsarService) throws Exception { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + final ClassLoader previousContext = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(narClassLoader); this.interceptor.initialize(pulsarService); + } finally { + Thread.currentThread().setContextClassLoader(previousContext); } } @Override public void close() { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + final ClassLoader previousContext = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(narClassLoader); interceptor.close(); + } finally { + Thread.currentThread().setContextClassLoader(previousContext); } + try { - classLoader.close(); + narClassLoader.close(); } catch (IOException e) { log.warn("Failed to close the broker interceptor class loader", e); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java index cef3f0eb609a1..30d1874a97299 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java @@ -59,6 +59,9 @@ public BrokerInterceptors(Map intercep * @return the collection of broker event interceptor */ public static BrokerInterceptor load(ServiceConfiguration conf) throws IOException { + if (conf.getBrokerInterceptors().isEmpty()) { + return null; + } BrokerInterceptorDefinitions definitions = BrokerInterceptorUtils.searchForInterceptors(conf.getBrokerInterceptorsDirectory(), conf.getNarExtractionDirectory()); @@ -87,7 +90,7 @@ public static BrokerInterceptor load(ServiceConfiguration conf) throws IOExcepti }); Map interceptors = builder.build(); - if (interceptors != null && !interceptors.isEmpty()) { + if (!interceptors.isEmpty()) { return new BrokerInterceptors(interceptors); } else { return null; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java index 02c6c575fd919..db138989a8eee 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java @@ -23,9 +23,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; -import org.apache.bookkeeper.client.LedgerHandle; -import org.apache.bookkeeper.client.api.LedgerEntry; -import org.apache.bookkeeper.mledger.impl.OpAddEntry; +import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.intercept.ManagedLedgerInterceptor; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.pulsar.common.api.proto.BrokerEntryMetadata; @@ -85,12 +83,11 @@ public long getIndex() { } @Override - public OpAddEntry beforeAddEntry(OpAddEntry op, int numberOfMessages) { + public void beforeAddEntry(AddEntryOperation op, int numberOfMessages) { if (op == null || numberOfMessages <= 0) { - return op; + return; } op.setData(Commands.addBrokerEntryMetadata(op.getData(), brokerEntryMetadataInterceptors, numberOfMessages)); - return op; } @Override @@ -115,43 +112,22 @@ public void onManagedLedgerPropertiesInitialize(Map propertiesMa } @Override - public CompletableFuture onManagedLedgerLastLedgerInitialize(String name, LedgerHandle lh) { - CompletableFuture promise = new CompletableFuture<>(); - boolean hasAppendIndexMetadataInterceptor = appendIndexMetadataInterceptor != null; - if (hasAppendIndexMetadataInterceptor && lh.getLastAddConfirmed() >= 0) { - lh.readAsync(lh.getLastAddConfirmed(), lh.getLastAddConfirmed()).whenComplete((entries, ex) -> { - if (ex != null) { - log.error("[{}] Read last entry error.", name, ex); - promise.completeExceptionally(ex); - } else { - if (entries != null) { - try { - LedgerEntry ledgerEntry = entries.getEntry(lh.getLastAddConfirmed()); - if (ledgerEntry != null) { - BrokerEntryMetadata brokerEntryMetadata = - Commands.parseBrokerEntryMetadataIfExist(ledgerEntry.getEntryBuffer()); - if (brokerEntryMetadata != null && brokerEntryMetadata.hasIndex()) { - appendIndexMetadataInterceptor.recoveryIndexGenerator( - brokerEntryMetadata.getIndex()); - } - } - entries.close(); - promise.complete(null); - } catch (Exception e) { - entries.close(); - log.error("[{}] Failed to recover the index generator from the last add confirmed entry.", - name, e); - promise.completeExceptionally(e); - } - } else { - promise.complete(null); + public CompletableFuture onManagedLedgerLastLedgerInitialize(String name, LastEntryHandle lh) { + return lh.readLastEntryAsync().thenAccept(lastEntryOptional -> { + if (lastEntryOptional.isPresent()) { + Entry lastEntry = lastEntryOptional.get(); + try { + BrokerEntryMetadata brokerEntryMetadata = + Commands.parseBrokerEntryMetadataIfExist(lastEntry.getDataBuffer()); + if (brokerEntryMetadata != null && brokerEntryMetadata.hasIndex()) { + appendIndexMetadataInterceptor.recoveryIndexGenerator( + brokerEntryMetadata.getIndex()); } + } finally { + lastEntry.release(); } - }); - } else { - promise.complete(null); - } - return promise; + } + }); } @Override @@ -189,11 +165,11 @@ public void release() { }; } @Override - public PayloadProcessorHandle processPayloadBeforeLedgerWrite(OpAddEntry op, ByteBuf ledgerData) { + public PayloadProcessorHandle processPayloadBeforeLedgerWrite(Object ctx, ByteBuf ledgerData) { if (this.inputProcessors == null || this.inputProcessors.size() == 0) { return null; } - return processPayload(this.inputProcessors, op.getCtx(), ledgerData); + return processPayload(this.inputProcessors, ctx, ledgerData); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderBroker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderBroker.java index acd34e151ed2a..d7c21de5ea1fa 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderBroker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderBroker.java @@ -30,5 +30,23 @@ @AllArgsConstructor @NoArgsConstructor public class LeaderBroker { + private String brokerId; private String serviceUrl; + + public String getBrokerId() { + if (brokerId != null) { + return brokerId; + } else { + // for backward compatibility at runtime with older versions of Pulsar + return parseHostAndPort(serviceUrl); + } + } + + private static String parseHostAndPort(String serviceUrl) { + int uriSeparatorPos = serviceUrl.indexOf("://"); + if (uriSeparatorPos == -1) { + throw new IllegalArgumentException("'" + serviceUrl + "' isn't an URI."); + } + return serviceUrl.substring(uriSeparatorPos + 3); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java index 05fe4353f3e76..2e53b54e98f61 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java @@ -35,17 +35,17 @@ public class LeaderElectionService implements AutoCloseable { private final LeaderElection leaderElection; private final LeaderBroker localValue; - public LeaderElectionService(CoordinationService cs, String localWebServiceAddress, - Consumer listener) { - this(cs, localWebServiceAddress, ELECTION_ROOT, listener); + public LeaderElectionService(CoordinationService cs, String brokerId, + String serviceUrl, Consumer listener) { + this(cs, brokerId, serviceUrl, ELECTION_ROOT, listener); } public LeaderElectionService(CoordinationService cs, - String localWebServiceAddress, - String electionRoot, + String brokerId, + String serviceUrl, String electionRoot, Consumer listener) { this.leaderElection = cs.getLeaderElection(LeaderBroker.class, electionRoot, listener); - this.localValue = new LeaderBroker(localWebServiceAddress); + this.localValue = new LeaderBroker(brokerId, serviceUrl); } public void start() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java index 17aa7170fc63c..b63f0fe85b20c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.AllArgsConstructor; @@ -54,7 +55,8 @@ public class LinuxInfoUtils { // NIC type private static final int ARPHRD_ETHER = 1; private static final String NIC_SPEED_TEMPLATE = "/sys/class/net/%s/speed"; - + private static final long errLogPrintedFrequencyInReadingNicLimits = 1000; + private static final AtomicLong failedCounterInReadingNicLimits = new AtomicLong(0); private static Object /*jdk.internal.platform.Metrics*/ metrics; private static Method getMetricsProviderMethod; private static Method getCpuQuotaMethod; @@ -161,7 +163,8 @@ public static long getCpuUsageForCGroup() { * *

* Line is split in "words", filtering the first. The sum of all numbers give the amount of cpu cycles used this - * far. Real CPU usage should equal the sum subtracting the idle cycles, this would include iowait, irq and steal. + * far. Real CPU usage should equal the sum substracting the idle cycles(that is idle+iowait), this would include + * cpu, user, nice, system, irq, softirq, steal, guest and guest_nice. */ public static ResourceUsage getCpuUsageForEntireHost() { try (Stream stream = Files.lines(Paths.get(PROC_STAT_PATH))) { @@ -175,7 +178,7 @@ public static ResourceUsage getCpuUsageForEntireHost() { .filter(s -> !s.contains("cpu")) .mapToLong(Long::parseLong) .sum(); - long idle = Long.parseLong(words[4]); + long idle = Long.parseLong(words[4]) + Long.parseLong(words[5]); return ResourceUsage.builder() .usage(total - idle) .idle(idle) @@ -197,12 +200,20 @@ private static boolean isPhysicalNic(Path nicPath) { return false; } // Check the type to make sure it's ethernet (type "1") - String type = readTrimStringFromFile(nicPath.resolve("type")); + final Path nicTypePath = nicPath.resolve("type"); + if (!Files.exists(nicTypePath)) { + if (log.isDebugEnabled()) { + log.debug("Failed to read NIC type, the expected linux type file does not exist." + + " nic_type_path={}", nicTypePath); + } + return false; + } // wireless NICs don't report speed, ignore them. - return Integer.parseInt(type) == ARPHRD_ETHER; - } catch (Exception e) { - log.warn("[LinuxInfo] Failed to read {} NIC type, the detail is: {}", nicPath, e.getMessage()); - // Read type got error. + return Integer.parseInt(readTrimStringFromFile(nicTypePath)) == ARPHRD_ETHER; + } catch (Exception ex) { + if (log.isDebugEnabled()) { + log.debug("Failed to read NIC type. nic_path={}", nicPath, ex); + } return false; } } @@ -242,7 +253,15 @@ public static double getTotalNicLimit(List nics, BitRateUnit bitRateUnit try { return readDoubleFromFile(getReplacedNICPath(NIC_SPEED_TEMPLATE, nicPath)); } catch (IOException e) { - log.error("[LinuxInfo] Failed to get total nic limit.", e); + // ERROR-level logs about NIC rate limiting reading failures are periodically printed but not + // continuously printed + if (failedCounterInReadingNicLimits.getAndIncrement() % errLogPrintedFrequencyInReadingNicLimits == 0) { + log.error("[LinuxInfo] Failed to get the nic limit of {}.", nicPath, e); + } else { + if (log.isDebugEnabled()) { + log.debug("[LinuxInfo] Failed to get the nic limit of {}.", nicPath, e); + } + } return 0d; } }).sum(), BitRateUnit.Megabit); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java index 87f630f1a09fb..c1fe2a4930c34 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java @@ -64,7 +64,7 @@ public Map getBundleData() { public Map getBundleDataForLoadShedding() { return bundleData.entrySet().stream() - .filter(e -> !NamespaceService.isSystemServiceNamespace( + .filter(e -> !NamespaceService.isSLAOrHeartbeatNamespace( NamespaceBundle.getBundleNamespace(e.getKey()))) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadManager.java index 17bff57b85c42..db2fb2ffd0fa6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadManager.java @@ -31,6 +31,7 @@ import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerWrapper; import org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl; import org.apache.pulsar.broker.lookup.LookupResult; +import org.apache.pulsar.broker.namespace.LookupOptions; import org.apache.pulsar.common.naming.ServiceUnitId; import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.common.util.Reflections; @@ -42,7 +43,7 @@ * LoadManager runs through set of load reports collected from different brokers and generates a recommendation of * namespace/ServiceUnit placement on machines/ResourceUnit. Each Concrete Load Manager will use different algorithms to * generate this mapping. - * + *

* Concrete Load Manager is also return the least loaded broker that should own the new namespace. */ public interface LoadManager { @@ -52,6 +53,10 @@ public interface LoadManager { void start() throws PulsarServerException; + default boolean started() { + return true; + } + /** * Is centralized decision making to assign a new bundle. */ @@ -63,7 +68,7 @@ public interface LoadManager { Optional getLeastLoaded(ServiceUnitId su) throws Exception; default CompletableFuture> findBrokerServiceUrl( - Optional topic, ServiceUnitId bundle) { + Optional topic, ServiceUnitId bundle, LookupOptions options) { throw new UnsupportedOperationException(); } @@ -88,7 +93,7 @@ default CompletableFuture checkOwnershipAsync(Optional t /** * Publish the current load report on ZK, forced or not. - * By default rely on method writeLoadReportOnZookeeper(). + * By default, rely on method writeLoadReportOnZookeeper(). */ default void writeLoadReportOnZookeeper(boolean force) throws Exception { writeLoadReportOnZookeeper(); @@ -118,15 +123,15 @@ default void writeLoadReportOnZookeeper(boolean force) throws Exception { * Removes visibility of current broker from loadbalancer list so, other brokers can't redirect any request to this * broker and this broker won't accept new connection requests. * - * @throws Exception + * @throws Exception if there is any error while disabling broker */ void disableBroker() throws Exception; /** * Get list of available brokers in cluster. * - * @return - * @throws Exception + * @return the list of available brokers + * @throws Exception if there is any error while getting available brokers */ Set getAvailableBrokers() throws Exception; @@ -150,12 +155,11 @@ static LoadManager create(final PulsarService pulsar) { // Assume there is a constructor with one argument of PulsarService. final Object loadManagerInstance = Reflections.createInstance(conf.getLoadManagerClassName(), Thread.currentThread().getContextClassLoader()); - if (loadManagerInstance instanceof LoadManager) { - final LoadManager casted = (LoadManager) loadManagerInstance; + if (loadManagerInstance instanceof LoadManager casted) { casted.initialize(pulsar); return casted; - } else if (loadManagerInstance instanceof ModularLoadManager) { - final LoadManager casted = new ModularLoadManagerWrapper((ModularLoadManager) loadManagerInstance); + } else if (loadManagerInstance instanceof ModularLoadManager modularLoadManager) { + final LoadManager casted = new ModularLoadManagerWrapper(modularLoadManager); casted.initialize(pulsar); return casted; } else if (loadManagerInstance instanceof ExtensibleLoadManager) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadSheddingTask.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadSheddingTask.java index eb7eacec60815..25a0a2752d11b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadSheddingTask.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadSheddingTask.java @@ -22,6 +22,8 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import org.apache.bookkeeper.mledger.ManagedLedgerFactory; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; import org.apache.pulsar.broker.ServiceConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,12 +42,16 @@ public class LoadSheddingTask implements Runnable { private volatile ScheduledFuture future; + private final ManagedLedgerFactory factory; + public LoadSheddingTask(AtomicReference loadManager, ScheduledExecutorService loadManagerExecutor, - ServiceConfiguration config) { + ServiceConfiguration config, + ManagedLedgerFactory factory) { this.loadManager = loadManager; this.loadManagerExecutor = loadManagerExecutor; this.config = config; + this.factory = factory; } @Override @@ -53,6 +59,10 @@ public void run() { if (isCancel) { return; } + if (factory instanceof ManagedLedgerFactoryImpl + && !((ManagedLedgerFactoryImpl) factory).isMetadataServiceAvailable()) { + return; + } try { loadManager.get().doLoadShedding(); } catch (Exception e) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java index 0de2ae92db61a..f9f36b705d4c4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java @@ -43,7 +43,7 @@ public class NoopLoadManager implements LoadManager { private PulsarService pulsar; - private String lookupServiceAddress; + private String brokerId; private ResourceUnit localResourceUnit; private LockManager lockManager; private Map bundleBrokerAffinityMap; @@ -57,16 +57,15 @@ public void initialize(PulsarService pulsar) { @Override public void start() throws PulsarServerException { - lookupServiceAddress = pulsar.getLookupServiceAddress(); - localResourceUnit = new SimpleResourceUnit(String.format("http://%s", lookupServiceAddress), - new PulsarResourceDescription()); + brokerId = pulsar.getBrokerId(); + localResourceUnit = new SimpleResourceUnit(brokerId, new PulsarResourceDescription()); - LocalBrokerData localData = new LocalBrokerData(pulsar.getSafeWebServiceAddress(), + LocalBrokerData localData = new LocalBrokerData(pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls(), pulsar.getAdvertisedListeners()); localData.setProtocols(pulsar.getProtocolDataToAdvertise()); localData.setLoadManagerClassName(this.pulsar.getConfig().getLoadManagerClassName()); - String brokerReportPath = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + lookupServiceAddress; + String brokerReportPath = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + brokerId; try { log.info("Acquiring broker resource lock on {}", brokerReportPath); @@ -129,12 +128,12 @@ public void disableBroker() throws Exception { @Override public Set getAvailableBrokers() throws Exception { - return Collections.singleton(lookupServiceAddress); + return Collections.singleton(brokerId); } @Override public CompletableFuture> getAvailableBrokersAsync() { - return CompletableFuture.completedFuture(Collections.singleton(lookupServiceAddress)); + return CompletableFuture.completedFuture(Collections.singleton(brokerId)); } @Override @@ -153,7 +152,6 @@ public String setNamespaceBundleAffinity(String bundle, String broker) { if (StringUtils.isBlank(broker)) { return this.bundleBrokerAffinityMap.remove(bundle); } - broker = broker.replaceFirst("http[s]?://", ""); return this.bundleBrokerAffinityMap.put(bundle, broker); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ResourceUnit.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ResourceUnit.java index ef4dd2a97b280..c28a8be4c0d3a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ResourceUnit.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ResourceUnit.java @@ -23,8 +23,6 @@ */ public interface ResourceUnit extends Comparable { - String PROPERTY_KEY_BROKER_ZNODE_NAME = "__advertised_addr"; - String getResourceId(); ResourceDescription getAvailableResource(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistry.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistry.java index 8133d4c482752..d154edfbb320e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistry.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistry.java @@ -25,6 +25,8 @@ import java.util.function.BiConsumer; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.common.classification.InterfaceAudience; +import org.apache.pulsar.common.classification.InterfaceStability; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.NotificationType; @@ -32,6 +34,8 @@ * Responsible for registering the current Broker lookup info to * the distributed store (e.g. Zookeeper) for broker discovery. */ +@InterfaceAudience.LimitedPrivate +@InterfaceStability.Unstable public interface BrokerRegistry extends AutoCloseable { /** @@ -44,10 +48,15 @@ public interface BrokerRegistry extends AutoCloseable { */ boolean isStarted(); + /** + * Return the broker has been registered. + */ + boolean isRegistered(); + /** * Register local broker to metadata store. */ - void register() throws MetadataStoreException; + CompletableFuture registerAsync(); /** * Unregister the broker. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java index 921ce35b5c65e..5a8307df27a63 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java @@ -21,17 +21,18 @@ import static org.apache.pulsar.broker.loadbalance.LoadManager.LOADBALANCE_BROKERS_ROOT; import com.google.common.annotations.VisibleForTesting; import java.util.ArrayList; +import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.PulsarServerException; @@ -39,11 +40,11 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.Notification; import org.apache.pulsar.metadata.api.NotificationType; -import org.apache.pulsar.metadata.api.coordination.LockManager; -import org.apache.pulsar.metadata.api.coordination.ResourceLock; +import org.apache.pulsar.metadata.api.extended.CreateOption; /** * The broker registry impl, base on the LockManager. @@ -51,40 +52,43 @@ @Slf4j public class BrokerRegistryImpl implements BrokerRegistry { + private static final int MAX_REGISTER_RETRY_DELAY_IN_MILLIS = 1000; + private final PulsarService pulsar; private final ServiceConfiguration conf; private final BrokerLookupData brokerLookupData; - private final LockManager brokerLookupDataLockManager; + private final MetadataCache brokerLookupDataMetadataCache; - private final String brokerId; + private final String brokerIdKeyPath; private final ScheduledExecutorService scheduler; private final List> listeners; - private volatile ResourceLock brokerLookupDataLock; - protected enum State { Init, Started, Registered, + Unregistering, Closed } - private State state; + @VisibleForTesting + final AtomicReference state = new AtomicReference<>(State.Init); - public BrokerRegistryImpl(PulsarService pulsar) { + @VisibleForTesting + BrokerRegistryImpl(PulsarService pulsar, MetadataCache brokerLookupDataMetadataCache) { this.pulsar = pulsar; this.conf = pulsar.getConfiguration(); - this.brokerLookupDataLockManager = pulsar.getCoordinationService().getLockManager(BrokerLookupData.class); + this.brokerLookupDataMetadataCache = brokerLookupDataMetadataCache; this.scheduler = pulsar.getLoadManagerExecutor(); this.listeners = new ArrayList<>(); - this.brokerId = pulsar.getLookupServiceAddress(); + this.brokerIdKeyPath = keyPath(pulsar.getBrokerId()); this.brokerLookupData = new BrokerLookupData( - pulsar.getSafeWebServiceAddress(), + pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls(), @@ -94,70 +98,112 @@ public BrokerRegistryImpl(PulsarService pulsar) { pulsar.getConfiguration().isEnableNonPersistentTopics(), conf.getLoadManagerClassName(), System.currentTimeMillis(), - pulsar.getBrokerVersion()); - this.state = State.Init; + pulsar.getBrokerVersion(), + pulsar.getConfig().lookupProperties()); + } + + public BrokerRegistryImpl(PulsarService pulsar) { + this(pulsar, pulsar.getLocalMetadataStore().getMetadataCache(BrokerLookupData.class)); } @Override public synchronized void start() throws PulsarServerException { - if (this.state != State.Init) { - return; + if (!this.state.compareAndSet(State.Init, State.Started)) { + throw new PulsarServerException("Cannot start the broker registry in state " + state.get()); } pulsar.getLocalMetadataStore().registerListener(this::handleMetadataStoreNotification); try { - this.state = State.Started; - this.register(); - } catch (MetadataStoreException e) { - throw new PulsarServerException(e); + this.registerAsync().get(conf.getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS); + } catch (ExecutionException | InterruptedException | TimeoutException e) { + throw PulsarServerException.from(e); } } @Override public boolean isStarted() { - return this.state == State.Started || this.state == State.Registered; + final var state = this.state.get(); + return state == State.Started || state == State.Registered; } @Override - public synchronized void register() throws MetadataStoreException { - if (this.state == State.Started) { - try { - this.brokerLookupDataLock = brokerLookupDataLockManager.acquireLock(keyPath(brokerId), brokerLookupData) - .get(conf.getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS); - this.state = State.Registered; - } catch (InterruptedException | ExecutionException | TimeoutException e) { - throw MetadataStoreException.unwrap(e); - } + public boolean isRegistered() { + final var state = this.state.get(); + return state == State.Registered; + } + + @Override + public CompletableFuture registerAsync() { + final var state = this.state.get(); + if (state != State.Started && state != State.Registered) { + log.info("[{}] Skip registering self because the state is {}", getBrokerId(), state); + return CompletableFuture.completedFuture(null); } + log.info("[{}] Started registering self to {} (state: {})", getBrokerId(), brokerIdKeyPath, state); + return brokerLookupDataMetadataCache.put(brokerIdKeyPath, brokerLookupData, EnumSet.of(CreateOption.Ephemeral)) + .orTimeout(pulsar.getConfiguration().getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS) + .whenComplete((__, ex) -> { + if (ex == null) { + this.state.set(State.Registered); + log.info("[{}] Finished registering self", getBrokerId()); + } else { + log.error("[{}] Failed registering self", getBrokerId(), ex); + } + }); + } + + private void doRegisterAsyncWithRetries(int retry, CompletableFuture future) { + pulsar.getExecutor().schedule(() -> { + registerAsync().whenComplete((__, e) -> { + if (e != null) { + doRegisterAsyncWithRetries(retry + 1, future); + } else { + future.complete(null); + } + }); + }, Math.min(MAX_REGISTER_RETRY_DELAY_IN_MILLIS, retry * retry * 50), TimeUnit.MILLISECONDS); + } + + private CompletableFuture registerAsyncWithRetries() { + var retryFuture = new CompletableFuture(); + doRegisterAsyncWithRetries(0, retryFuture); + return retryFuture; } @Override public synchronized void unregister() throws MetadataStoreException { - if (this.state == State.Registered) { + if (state.compareAndSet(State.Registered, State.Unregistering)) { try { - this.brokerLookupDataLock.release() + brokerLookupDataMetadataCache.delete(brokerIdKeyPath) .get(conf.getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS); - this.state = State.Started; - } catch (CompletionException | InterruptedException | ExecutionException | TimeoutException e) { + } catch (ExecutionException e) { + if (e.getCause() instanceof MetadataStoreException.NotFoundException) { + log.warn("{} has already been unregistered", brokerIdKeyPath); + } else { + throw MetadataStoreException.unwrap(e); + } + } catch (InterruptedException | TimeoutException e) { throw MetadataStoreException.unwrap(e); + } finally { + state.set(State.Started); } } } @Override public String getBrokerId() { - return this.brokerId; + return pulsar.getBrokerId(); } @Override public CompletableFuture> getAvailableBrokersAsync() { this.checkState(); - return brokerLookupDataLockManager.listLocks(LOADBALANCE_BROKERS_ROOT).thenApply(ArrayList::new); + return brokerLookupDataMetadataCache.getChildren(LOADBALANCE_BROKERS_ROOT); } @Override public CompletableFuture> lookupAsync(String broker) { this.checkState(); - return brokerLookupDataLockManager.readLock(keyPath(broker)); + return brokerLookupDataMetadataCache.get(keyPath(broker)); } public CompletableFuture> getAvailableBrokerLookupDataAsync() { @@ -185,21 +231,16 @@ public synchronized void addListener(BiConsumer listen @Override public synchronized void close() throws PulsarServerException { - if (this.state == State.Closed) { + if (this.state.get() == State.Closed) { return; } try { this.listeners.clear(); this.unregister(); - this.brokerLookupDataLockManager.close(); } catch (Exception ex) { - if (ex.getCause() instanceof MetadataStoreException.NotFoundException) { - throw new PulsarServerException.NotFoundException(MetadataStoreException.unwrap(ex)); - } else { - throw new PulsarServerException(MetadataStoreException.unwrap(ex)); - } + log.error("Unexpected error when unregistering the broker registry", ex); } finally { - this.state = State.Closed; + this.state.set(State.Closed); } } @@ -211,15 +252,29 @@ private void handleMetadataStoreNotification(Notification t) { if (log.isDebugEnabled()) { log.debug("Handle notification: [{}]", t); } - if (listeners.isEmpty()) { - return; + // The registered node is an ephemeral node that could be deleted when the metadata store client's session + // is expired. In this case, we should register again. + final var brokerId = t.getPath().substring(LOADBALANCE_BROKERS_ROOT.length() + 1); + + CompletableFuture register; + if (t.getType() == NotificationType.Deleted && getBrokerId().equals(brokerId)) { + this.state.set(State.Started); + register = registerAsyncWithRetries(); + } else { + register = CompletableFuture.completedFuture(null); } - this.scheduler.submit(() -> { - String brokerId = t.getPath().substring(LOADBALANCE_BROKERS_ROOT.length() + 1); - for (BiConsumer listener : listeners) { - listener.accept(brokerId, t.getType()); + // Make sure to run the listeners after re-registered. + register.thenAccept(__ -> { + if (listeners.isEmpty()) { + return; } + this.scheduler.submit(() -> { + for (BiConsumer listener : listeners) { + listener.accept(brokerId, t.getType()); + } + }); }); + } catch (RejectedExecutionException e) { // Executor is shutting down } @@ -237,7 +292,7 @@ protected static String keyPath(String brokerId) { } private void checkState() throws IllegalStateException { - if (this.state == State.Closed) { + if (this.state.get() == State.Closed) { throw new IllegalStateException("The registry already closed."); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManager.java index b7da70d1cf1de..eabf6005b439b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManager.java @@ -60,9 +60,12 @@ public interface ExtensibleLoadManager extends Closeable { * (e.g. {@link NamespaceService#internalGetWebServiceUrl(NamespaceBundle, LookupOptions)}), * So the topic is optional. * @param serviceUnit service unit (e.g. bundle). + * @param options The lookup options. * @return The broker lookup data. */ - CompletableFuture> assign(Optional topic, ServiceUnitId serviceUnit); + CompletableFuture> assign(Optional topic, + ServiceUnitId serviceUnit, + LookupOptions options); /** * Check the incoming service unit is owned by the current broker. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index cbaed8ee8f94f..abca2bb398232 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -21,6 +21,7 @@ import static java.lang.String.format; import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Follower; import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Leader; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateTableViewImpl.TOPIC; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Success; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Admin; import com.google.common.annotations.VisibleForTesting; @@ -33,20 +34,26 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.mutable.MutableObject; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; import org.apache.pulsar.broker.loadbalance.LoadManager; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateTableViewImpl; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateTableViewSyncer; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; @@ -73,12 +80,16 @@ import org.apache.pulsar.broker.loadbalance.extensions.scheduler.SplitScheduler; import org.apache.pulsar.broker.loadbalance.extensions.scheduler.UnloadScheduler; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; -import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStoreException; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStoreFactory; import org.apache.pulsar.broker.loadbalance.extensions.strategy.BrokerSelectionStrategy; +import org.apache.pulsar.broker.loadbalance.extensions.strategy.BrokerSelectionStrategyFactory; import org.apache.pulsar.broker.loadbalance.extensions.strategy.LeastResourceUsageWithWeight; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; +import org.apache.pulsar.broker.namespace.LookupOptions; +import org.apache.pulsar.broker.namespace.NamespaceEphemeralData; +import org.apache.pulsar.broker.namespace.NamespaceService; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm; import org.apache.pulsar.common.naming.NamespaceName; @@ -87,12 +98,11 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.metadata.api.coordination.LeaderElectionState; import org.slf4j.Logger; @Slf4j -public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { +public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager, BrokerSelectionStrategyFactory { public static final String BROKER_LOAD_DATA_STORE_TOPIC = TopicName.get( TopicDomain.non_persistent.value(), @@ -108,9 +118,15 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { private static final long MONITOR_INTERVAL_IN_MILLIS = 120_000; + public static final long COMPACTION_THRESHOLD = 5 * 1024 * 1024; + private static final String ELECTION_ROOT = "/loadbalance/extension/leader"; - private PulsarService pulsar; + public static final Set INTERNAL_TOPICS = + Set.of(BROKER_LOAD_DATA_STORE_TOPIC, TOP_BUNDLES_LOAD_DATA_STORE_TOPIC, TOPIC); + + @VisibleForTesting + protected PulsarService pulsar; private ServiceConfiguration conf; @@ -128,7 +144,10 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { @Getter private IsolationPoliciesHelper isolationPoliciesHelper; + @Getter private LoadDataStore brokerLoadDataStore; + + @Getter private LoadDataStore topBundlesLoadDataStore; private LoadManagerScheduler unloadScheduler; @@ -144,6 +163,7 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { @Getter private final List brokerFilterPipeline; + /** * The load data reporter. */ @@ -151,39 +171,78 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { private TopBundleLoadDataReporter topBundleLoadDataReporter; - private ScheduledFuture brokerLoadDataReportTask; - private ScheduledFuture topBundlesLoadDataReportTask; + @Getter + protected ServiceUnitStateTableViewSyncer serviceUnitStateTableViewSyncer; + + private volatile ScheduledFuture brokerLoadDataReportTask; + private volatile ScheduledFuture topBundlesLoadDataReportTask; - private ScheduledFuture monitorTask; + private volatile ScheduledFuture monitorTask; private SplitScheduler splitScheduler; private UnloadManager unloadManager; private SplitManager splitManager; - private boolean started = false; + enum State { + INIT, + RUNNING, + // It's removing visibility of the current broker from other brokers. In this state, it cannot play as a leader + // or follower. + DISABLED, + } + private final AtomicReference state = new AtomicReference<>(State.INIT); + + private boolean configuredSystemTopics = false; private final AssignCounter assignCounter = new AssignCounter(); @Getter private final UnloadCounter unloadCounter = new UnloadCounter(); private final SplitCounter splitCounter = new SplitCounter(); + // Record the ignored send msg count during unloading + @Getter + private final AtomicLong ignoredSendMsgCount = new AtomicLong(); + @Getter + private final AtomicLong ignoredAckCount = new AtomicLong(); + // record unload metrics - private final AtomicReference> unloadMetrics = new AtomicReference(); + private final AtomicReference> unloadMetrics = new AtomicReference<>(); // record split metrics private final AtomicReference> splitMetrics = new AtomicReference<>(); - private final ConcurrentOpenHashMap>> - lookupRequests = ConcurrentOpenHashMap.>>newBuilder() - .build(); - private final CountDownLatch initWaiter = new CountDownLatch(1); + private final ConcurrentHashMap>> + lookupRequests = new ConcurrentHashMap<>(); + private final CompletableFuture initWaiter = new CompletableFuture<>(); + + /** + * Get all the bundles that are owned by this broker. + */ + @Deprecated + public CompletableFuture> getOwnedServiceUnitsAsync() { + return CompletableFuture.completedFuture(getOwnedServiceUnits()); + } + + public Set getOwnedServiceUnits() { + if (state.get() == State.INIT) { + log.warn("Failed to get owned service units, load manager is not started."); + return Collections.emptySet(); + } + + return serviceUnitStateChannel.getOwnedServiceUnits(); + } + + @Override + public BrokerSelectionStrategy createBrokerSelectionStrategy() { + return new LeastResourceUsageWithWeight(); + } public enum Role { Leader, Follower } + @Getter private volatile Role role; /** @@ -194,12 +253,11 @@ public ExtensibleLoadManagerImpl() { this.brokerFilterPipeline.add(new BrokerLoadManagerClassFilter()); this.brokerFilterPipeline.add(new BrokerMaxTopicCountFilter()); this.brokerFilterPipeline.add(new BrokerVersionFilter()); - // TODO: Make brokerSelectionStrategy configurable. - this.brokerSelectionStrategy = new LeastResourceUsageWithWeight(); + this.brokerSelectionStrategy = createBrokerSelectionStrategy(); } - public static boolean isLoadManagerExtensionEnabled(ServiceConfiguration conf) { - return ExtensibleLoadManagerImpl.class.getName().equals(conf.getLoadManagerClassName()); + public static boolean isLoadManagerExtensionEnabled(PulsarService pulsar) { + return pulsar.getLoadManager().get() instanceof ExtensibleLoadManagerWrapper; } public static ExtensibleLoadManagerImpl get(LoadManager loadManager) { @@ -209,110 +267,220 @@ public static ExtensibleLoadManagerImpl get(LoadManager loadManager) { return loadManagerWrapper.get(); } + /** + * A static util func to get the ExtensibleLoadManagerImpl instance. + * @param pulsar PulsarService + * @return the ExtensibleLoadManagerImpl instance + */ + public static ExtensibleLoadManagerImpl get(PulsarService pulsar) { + return get(pulsar.getLoadManager().get()); + } + public static boolean debug(ServiceConfiguration config, Logger log) { return config.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled(); } + public static void createSystemTopic(PulsarService pulsar, String topic) throws PulsarServerException { + try { + pulsar.getAdminClient().topics().createNonPartitionedTopic(topic); + log.info("Created topic {}.", topic); + } catch (PulsarAdminException.ConflictException ex) { + if (debug(pulsar.getConfiguration(), log)) { + log.info("Topic {} already exists.", topic); + } + } catch (PulsarAdminException e) { + throw new PulsarServerException(e); + } + } + + private static void createSystemTopics(PulsarService pulsar) throws PulsarServerException { + createSystemTopic(pulsar, BROKER_LOAD_DATA_STORE_TOPIC); + createSystemTopic(pulsar, TOP_BUNDLES_LOAD_DATA_STORE_TOPIC); + } + + public static boolean configureSystemTopics(PulsarService pulsar, long target) { + try { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar) + && pulsar.getConfiguration().isTopicLevelPoliciesEnabled()) { + Long threshold = pulsar.getAdminClient().topicPolicies().getCompactionThreshold(TOPIC); + if (threshold == null || target != threshold.longValue()) { + pulsar.getAdminClient().topicPolicies().setCompactionThreshold(TOPIC, target); + log.info("Set compaction threshold: {} bytes for system topic {}.", target, TOPIC); + } + } else { + log.warn("System topic or topic level policies is disabled. " + + "{} compaction threshold follows the broker or namespace policies.", TOPIC); + } + return true; + } catch (Exception e) { + log.error("Failed to set compaction threshold for system topic:{}", TOPIC, e); + } + return false; + } + + /** + * Gets the assigned broker for the given topic. + * @param pulsar PulsarService instance + * @param topic Topic Name + * @return the assigned broker's BrokerLookupData instance. Empty, if not assigned by Extensible LoadManager or the + * optimized bundle unload process is disabled. + */ + public static CompletableFuture> getAssignedBrokerLookupData(PulsarService pulsar, + String topic) { + var config = pulsar.getConfig(); + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar) + && config.isLoadBalancerMultiPhaseBundleUnload()) { + var topicName = TopicName.get(topic); + try { + return pulsar.getNamespaceService().getBundleAsync(topicName) + .thenCompose(bundle -> { + var loadManager = ExtensibleLoadManagerImpl.get(pulsar); + var assigned = loadManager.getServiceUnitStateChannel() + .getAssigned(bundle.toString()); + if (assigned.isPresent()) { + return loadManager.getBrokerRegistry().lookupAsync(assigned.get()); + } else { + return CompletableFuture.completedFuture(Optional.empty()); + } + } + ); + } catch (Throwable e) { + log.error("Failed to lookup destination broker for topic:{}", topic, e); + return CompletableFuture.completedFuture(Optional.empty()); + } + } + return CompletableFuture.completedFuture(Optional.empty()); + } + @Override public void start() throws PulsarServerException { - if (this.started) { + if (state.get() != State.INIT) { return; } - this.brokerRegistry = new BrokerRegistryImpl(pulsar); - this.leaderElectionService = new LeaderElectionService( - pulsar.getCoordinationService(), pulsar.getSafeWebServiceAddress(), ELECTION_ROOT, - state -> { - pulsar.getLoadManagerExecutor().execute(() -> { - if (state == LeaderElectionState.Leading) { - playLeader(); - } else { - playFollower(); - } - }); - }); - this.serviceUnitStateChannel = new ServiceUnitStateChannelImpl(pulsar); - this.brokerRegistry.start(); - this.splitManager = new SplitManager(splitCounter); - this.unloadManager = new UnloadManager(unloadCounter); - this.serviceUnitStateChannel.listen(unloadManager); - this.serviceUnitStateChannel.listen(splitManager); - this.leaderElectionService.start(); - this.serviceUnitStateChannel.start(); - this.antiAffinityGroupPolicyHelper = - new AntiAffinityGroupPolicyHelper(pulsar, serviceUnitStateChannel); - antiAffinityGroupPolicyHelper.listenFailureDomainUpdate(); - this.antiAffinityGroupPolicyFilter = new AntiAffinityGroupPolicyFilter(antiAffinityGroupPolicyHelper); - this.brokerFilterPipeline.add(antiAffinityGroupPolicyFilter); - SimpleResourceAllocationPolicies policies = new SimpleResourceAllocationPolicies(pulsar); - this.isolationPoliciesHelper = new IsolationPoliciesHelper(policies); - this.brokerFilterPipeline.add(new BrokerIsolationPoliciesFilter(isolationPoliciesHelper)); - try { + this.brokerRegistry = createBrokerRegistry(pulsar); + this.leaderElectionService = new LeaderElectionService( + pulsar.getCoordinationService(), pulsar.getBrokerId(), + pulsar.getSafeWebServiceAddress(), ELECTION_ROOT, + state -> { + pulsar.runWhenReadyForIncomingRequests(() -> { + pulsar.getLoadManagerExecutor().execute(() -> { + if (state == LeaderElectionState.Leading) { + playLeader(); + } else { + playFollower(); + } + }); + }); + }); + this.serviceUnitStateChannel = createServiceUnitStateChannel(pulsar); + this.brokerRegistry.start(); + this.splitManager = new SplitManager(splitCounter); + this.unloadManager = new UnloadManager(unloadCounter, pulsar.getBrokerId()); + this.serviceUnitStateChannel.listen(unloadManager); + this.serviceUnitStateChannel.listen(splitManager); + this.leaderElectionService.start(); + + this.antiAffinityGroupPolicyHelper = + new AntiAffinityGroupPolicyHelper(pulsar, serviceUnitStateChannel); + antiAffinityGroupPolicyHelper.listenFailureDomainUpdate(); + this.antiAffinityGroupPolicyFilter = new AntiAffinityGroupPolicyFilter(antiAffinityGroupPolicyHelper); + this.brokerFilterPipeline.add(antiAffinityGroupPolicyFilter); + SimpleResourceAllocationPolicies policies = new SimpleResourceAllocationPolicies(pulsar); + this.isolationPoliciesHelper = new IsolationPoliciesHelper(policies); + this.brokerFilterPipeline.add(new BrokerIsolationPoliciesFilter(isolationPoliciesHelper)); this.brokerLoadDataStore = LoadDataStoreFactory - .create(pulsar.getClient(), BROKER_LOAD_DATA_STORE_TOPIC, BrokerLoadData.class); - this.brokerLoadDataStore.startTableView(); + .create(pulsar, BROKER_LOAD_DATA_STORE_TOPIC, BrokerLoadData.class); this.topBundlesLoadDataStore = LoadDataStoreFactory - .create(pulsar.getClient(), TOP_BUNDLES_LOAD_DATA_STORE_TOPIC, TopBundlesLoadData.class); - } catch (LoadDataStoreException e) { - throw new PulsarServerException(e); + .create(pulsar, TOP_BUNDLES_LOAD_DATA_STORE_TOPIC, TopBundlesLoadData.class); + + this.context = LoadManagerContextImpl.builder() + .configuration(conf) + .brokerRegistry(brokerRegistry) + .brokerLoadDataStore(brokerLoadDataStore) + .topBundleLoadDataStore(topBundlesLoadDataStore).build(); + + this.brokerLoadDataReporter = + new BrokerLoadDataReporter(pulsar, brokerRegistry.getBrokerId(), brokerLoadDataStore); + + this.topBundleLoadDataReporter = + new TopBundleLoadDataReporter(pulsar, brokerRegistry.getBrokerId(), topBundlesLoadDataStore); + this.serviceUnitStateChannel.listen(brokerLoadDataReporter); + this.serviceUnitStateChannel.listen(topBundleLoadDataReporter); + + this.unloadScheduler = new UnloadScheduler( + pulsar, pulsar.getLoadManagerExecutor(), unloadManager, context, + serviceUnitStateChannel, unloadCounter, unloadMetrics); + this.splitScheduler = new SplitScheduler( + pulsar, serviceUnitStateChannel, splitManager, splitCounter, splitMetrics, context); + this.serviceUnitStateTableViewSyncer = new ServiceUnitStateTableViewSyncer(); + + pulsar.runWhenReadyForIncomingRequests(() -> { + try { + this.serviceUnitStateChannel.start(); + var interval = conf.getLoadBalancerReportUpdateMinIntervalMillis(); + + this.brokerLoadDataReportTask = this.pulsar.getLoadManagerExecutor() + .scheduleAtFixedRate(() -> { + try { + brokerLoadDataReporter.reportAsync(false); + // TODO: update broker load metrics using getLocalData + } catch (Throwable e) { + log.error("Failed to run the broker load manager executor job.", e); + } + }, + interval, + interval, TimeUnit.MILLISECONDS); + + this.topBundlesLoadDataReportTask = this.pulsar.getLoadManagerExecutor() + .scheduleAtFixedRate(() -> { + try { + // TODO: consider excluding the bundles that are in the process of split. + topBundleLoadDataReporter.reportAsync(false); + } catch (Throwable e) { + log.error("Failed to run the top bundles load manager executor job.", e); + } + }, + interval, + interval, TimeUnit.MILLISECONDS); + + this.monitorTask = this.pulsar.getLoadManagerExecutor() + .scheduleAtFixedRate(() -> { + monitor(); + }, + MONITOR_INTERVAL_IN_MILLIS, + MONITOR_INTERVAL_IN_MILLIS, TimeUnit.MILLISECONDS); + + this.splitScheduler.start(); + this.initWaiter.complete(true); + if (!state.compareAndSet(State.INIT, State.RUNNING)) { + failForUnexpectedState("start"); + } + log.info("Started load manager."); + } catch (Throwable e) { + failStarting(e); + } + }); + } catch (Throwable ex) { + failStarting(ex); } + } - this.context = LoadManagerContextImpl.builder() - .configuration(conf) - .brokerRegistry(brokerRegistry) - .brokerLoadDataStore(brokerLoadDataStore) - .topBundleLoadDataStore(topBundlesLoadDataStore).build(); - - this.brokerLoadDataReporter = - new BrokerLoadDataReporter(pulsar, brokerRegistry.getBrokerId(), brokerLoadDataStore); - - this.topBundleLoadDataReporter = - new TopBundleLoadDataReporter(pulsar, brokerRegistry.getBrokerId(), topBundlesLoadDataStore); - this.serviceUnitStateChannel.listen(brokerLoadDataReporter); - this.serviceUnitStateChannel.listen(topBundleLoadDataReporter); - var interval = conf.getLoadBalancerReportUpdateMinIntervalMillis(); - this.brokerLoadDataReportTask = this.pulsar.getLoadManagerExecutor() - .scheduleAtFixedRate(() -> { - try { - brokerLoadDataReporter.reportAsync(false); - // TODO: update broker load metrics using getLocalData - } catch (Throwable e) { - log.error("Failed to run the broker load manager executor job.", e); - } - }, - interval, - interval, TimeUnit.MILLISECONDS); - - this.topBundlesLoadDataReportTask = this.pulsar.getLoadManagerExecutor() - .scheduleAtFixedRate(() -> { - try { - // TODO: consider excluding the bundles that are in the process of split. - topBundleLoadDataReporter.reportAsync(false); - } catch (Throwable e) { - log.error("Failed to run the top bundles load manager executor job.", e); - } - }, - interval, - interval, TimeUnit.MILLISECONDS); - - this.monitorTask = this.pulsar.getLoadManagerExecutor() - .scheduleAtFixedRate(() -> { - monitor(); - }, - MONITOR_INTERVAL_IN_MILLIS, - MONITOR_INTERVAL_IN_MILLIS, TimeUnit.MILLISECONDS); - - this.unloadScheduler = new UnloadScheduler( - pulsar, pulsar.getLoadManagerExecutor(), unloadManager, context, - serviceUnitStateChannel, unloadCounter, unloadMetrics); - this.unloadScheduler.start(); - this.splitScheduler = new SplitScheduler( - pulsar, serviceUnitStateChannel, splitManager, splitCounter, splitMetrics, context); - this.splitScheduler.start(); - this.initWaiter.countDown(); - this.started = true; + private void failStarting(Throwable throwable) { + if (this.brokerRegistry != null) { + try { + brokerRegistry.close(); + } catch (PulsarServerException e) { + // If close failed, this broker might still exist in the metadata store. Then it could be found by other + // brokers as an available broker. Hence, print a warning log for it. + log.warn("Failed to close the broker registry: {}", e.getMessage()); + } + } + initWaiter.complete(false); // exit the background thread gracefully + throw PulsarServerException.toUncheckedException(PulsarServerException.from(throwable)); } + @Override public void initialize(PulsarService pulsar) { this.pulsar = pulsar; @@ -321,77 +489,129 @@ public void initialize(PulsarService pulsar) { @Override public CompletableFuture> assign(Optional topic, - ServiceUnitId serviceUnit) { + ServiceUnitId serviceUnit, + LookupOptions options) { final String bundle = serviceUnit.toString(); - CompletableFuture> future = lookupRequests.computeIfAbsent(bundle, k -> { + return dedupeLookupRequest(bundle, k -> { final CompletableFuture> owner; // Assign the bundle to channel owner if is internal topic, to avoid circular references. if (topic.isPresent() && isInternalTopic(topic.get().toString())) { owner = serviceUnitStateChannel.getChannelOwnerAsync(); } else { - owner = serviceUnitStateChannel.getOwnerAsync(bundle).thenCompose(broker -> { - // If the bundle not assign yet, select and publish assign event to channel. - if (broker.isEmpty()) { - return this.selectAsync(serviceUnit).thenCompose(brokerOpt -> { - if (brokerOpt.isPresent()) { - assignCounter.incrementSuccess(); - log.info("Selected new owner broker: {} for bundle: {}.", brokerOpt.get(), bundle); - return serviceUnitStateChannel.publishAssignEventAsync(bundle, brokerOpt.get()) - .thenApply(Optional::of); - } else { - throw new IllegalStateException( - "Failed to select the new owner broker for bundle: " + bundle); - } - }); + owner = getHeartbeatOrSLAMonitorBrokerId(serviceUnit).thenCompose(candidateBrokerId -> { + if (candidateBrokerId != null) { + return CompletableFuture.completedFuture(Optional.of(candidateBrokerId)); } - assignCounter.incrementSkip(); - // Already assigned, return it. - return CompletableFuture.completedFuture(broker); + return getOrSelectOwnerAsync(serviceUnit, bundle, options).thenApply(Optional::ofNullable); }); } + return getBrokerLookupData(owner, bundle); + }); + } - return owner.thenCompose(broker -> { - if (broker.isEmpty()) { - String errorMsg = String.format( - "Failed to get or assign the owner for bundle:%s", bundle); - log.error(errorMsg); - throw new IllegalStateException(errorMsg); - } - return CompletableFuture.completedFuture(broker.get()); - }).thenCompose(broker -> this.getBrokerRegistry().lookupAsync(broker).thenCompose(brokerLookupData -> { + private CompletableFuture getHeartbeatOrSLAMonitorBrokerId(ServiceUnitId serviceUnit) { + return pulsar.getNamespaceService().getHeartbeatOrSLAMonitorBrokerId(serviceUnit, + cb -> brokerRegistry.lookupAsync(cb).thenApply(Optional::isPresent)); + } + + private CompletableFuture getOrSelectOwnerAsync(ServiceUnitId serviceUnit, + String bundle, + LookupOptions options) { + return serviceUnitStateChannel.getOwnerAsync(bundle).thenCompose(broker -> { + // If the bundle not assign yet, select and publish assign event to channel. + if (broker.isEmpty()) { + return this.selectAsync(serviceUnit, Collections.emptySet(), options).thenCompose(brokerOpt -> { + if (brokerOpt.isPresent()) { + assignCounter.incrementSuccess(); + log.info("Selected new owner broker: {} for bundle: {}.", brokerOpt.get(), bundle); + return serviceUnitStateChannel.publishAssignEventAsync(bundle, brokerOpt.get()); + } + return CompletableFuture.completedFuture(null); + }); + } + assignCounter.incrementSkip(); + // Already assigned, return it. + return CompletableFuture.completedFuture(broker.get()); + }); + } + + private CompletableFuture> getBrokerLookupData( + CompletableFuture> owner, + String bundle) { + return owner.thenCompose(broker -> { + if (broker.isEmpty()) { + return CompletableFuture.completedFuture(Optional.empty()); + } + return this.getBrokerRegistry().lookupAsync(broker.get()).thenCompose(brokerLookupData -> { if (brokerLookupData.isEmpty()) { String errorMsg = String.format( - "Failed to look up a broker registry:%s for bundle:%s", broker, bundle); + "Failed to lookup broker:%s for bundle:%s, the broker has not been registered.", + broker, bundle); log.error(errorMsg); throw new IllegalStateException(errorMsg); } return CompletableFuture.completedFuture(brokerLookupData); - })); + }); }); - future.whenComplete((r, t) -> { - if (t != null) { - assignCounter.incrementFailure(); + } + + /** + * Method to get the current owner of the NamespaceBundle + * or set the local broker as the owner if absent. + * + * @param namespaceBundle the NamespaceBundle + * @return The ephemeral node data showing the current ownership info in ServiceUnitStateChannel + */ + public CompletableFuture tryAcquiringOwnership(NamespaceBundle namespaceBundle) { + log.info("Try acquiring ownership for bundle: {} - {}.", namespaceBundle, brokerRegistry.getBrokerId()); + final String bundle = namespaceBundle.toString(); + return assign(Optional.empty(), namespaceBundle, LookupOptions.builder().readOnly(false).build()) + .thenApply(brokerLookupData -> { + if (brokerLookupData.isEmpty()) { + String errorMsg = String.format( + "Failed to get the broker lookup data for bundle:%s", bundle); + log.error(errorMsg); + throw new IllegalStateException(errorMsg); } - lookupRequests.remove(bundle); - } - ); - return future; + return brokerLookupData.get().toNamespaceEphemeralData(); + }); } - public CompletableFuture> selectAsync(ServiceUnitId bundle) { - return selectAsync(bundle, Collections.emptySet()); + private CompletableFuture> dedupeLookupRequest( + String key, Function>> provider) { + final MutableObject>> newFutureCreated = new MutableObject<>(); + try { + return lookupRequests.computeIfAbsent(key, k -> { + CompletableFuture> future = provider.apply(k); + newFutureCreated.setValue(future); + return future; + }); + } finally { + if (newFutureCreated.getValue() != null) { + newFutureCreated.getValue().whenComplete((v, ex) -> { + if (ex != null) { + assignCounter.incrementFailure(); + } + lookupRequests.remove(key); + }); + } + } } public CompletableFuture> selectAsync(ServiceUnitId bundle, - Set excludeBrokerSet) { + Set excludeBrokerSet, + LookupOptions options) { + if (options.isReadOnly()) { + return CompletableFuture.completedFuture(Optional.empty()); + } BrokerRegistry brokerRegistry = getBrokerRegistry(); return brokerRegistry.getAvailableBrokerLookupDataAsync() - .thenCompose(availableBrokers -> { + .thenComposeAsync(availableBrokers -> { LoadManagerContext context = this.getContext(); - Map availableBrokerCandidates = new HashMap<>(availableBrokers); + Map availableBrokerCandidates = new ConcurrentHashMap<>(availableBrokers); if (!excludeBrokerSet.isEmpty()) { for (String exclude : excludeBrokerSet) { availableBrokerCandidates.remove(exclude); @@ -400,24 +620,24 @@ public CompletableFuture> selectAsync(ServiceUnitId bundle, // Filter out brokers that do not meet the rules. List filterPipeline = getBrokerFilterPipeline(); + ArrayList>> futures = + new ArrayList<>(filterPipeline.size()); for (final BrokerFilter filter : filterPipeline) { - try { - filter.filter(availableBrokerCandidates, bundle, context); - // Preserve the filter successes result. - availableBrokers.keySet().retainAll(availableBrokerCandidates.keySet()); - } catch (BrokerFilterException e) { - // TODO: We may need to revisit this error case. - log.error("Failed to filter out brokers.", e); - availableBrokerCandidates = new HashMap<>(availableBrokers); - } - } - if (availableBrokerCandidates.isEmpty()) { - return CompletableFuture.completedFuture(Optional.empty()); + CompletableFuture> future = + filter.filterAsync(availableBrokerCandidates, bundle, context); + futures.add(future); } - Set candidateBrokers = availableBrokerCandidates.keySet(); - - return CompletableFuture.completedFuture( - getBrokerSelectionStrategy().select(candidateBrokers, bundle, context)); + return FutureUtil.waitForAll(futures).exceptionally(e -> { + // TODO: We may need to revisit this error case. + log.error("Failed to filter out brokers when select bundle: {}", bundle, e); + return null; + }).thenApply(__ -> { + if (availableBrokerCandidates.isEmpty()) { + return Optional.empty(); + } + Set candidateBrokers = availableBrokerCandidates.keySet(); + return getBrokerSelectionStrategy().select(candidateBrokers, bundle, context); + }); }); } @@ -428,15 +648,17 @@ public CompletableFuture checkOwnershipAsync(Optional to } public CompletableFuture> getOwnershipAsync(Optional topic, - ServiceUnitId bundleUnit) { - final String bundle = bundleUnit.toString(); - CompletableFuture> owner; + ServiceUnitId serviceUnit) { + final String bundle = serviceUnit.toString(); if (topic.isPresent() && isInternalTopic(topic.get().toString())) { - owner = serviceUnitStateChannel.getChannelOwnerAsync(); - } else { - owner = serviceUnitStateChannel.getOwnerAsync(bundle); + return serviceUnitStateChannel.getChannelOwnerAsync(); } - return owner; + return getHeartbeatOrSLAMonitorBrokerId(serviceUnit).thenCompose(candidateBroker -> { + if (candidateBroker != null) { + return CompletableFuture.completedFuture(Optional.of(candidateBroker)); + } + return serviceUnitStateChannel.getOwnerAsync(bundle); + }); } public CompletableFuture> getOwnershipWithLookupDataAsync(ServiceUnitId bundleUnit) { @@ -449,7 +671,17 @@ public CompletableFuture> getOwnershipWithLookupDataA } public CompletableFuture unloadNamespaceBundleAsync(ServiceUnitId bundle, - Optional destinationBroker) { + Optional destinationBroker, + boolean force, + long timeout, + TimeUnit timeoutUnit) { + if (state.get() == State.INIT) { + return CompletableFuture.completedFuture(null); + } + if (NamespaceService.isSLAOrHeartbeatNamespace(bundle.getNamespaceObject().toString())) { + log.info("Skip unloading namespace bundle: {}.", bundle); + return CompletableFuture.completedFuture(null); + } return getOwnershipAsync(Optional.empty(), bundle) .thenCompose(brokerOpt -> { if (brokerOpt.isEmpty()) { @@ -464,11 +696,11 @@ public CompletableFuture unloadNamespaceBundleAsync(ServiceUnitId bundle, log.warn(msg); throw new IllegalArgumentException(msg); } - Unload unload = new Unload(sourceBroker, bundle.toString(), destinationBroker, true); + Unload unload = new Unload(sourceBroker, bundle.toString(), destinationBroker, force); UnloadDecision unloadDecision = new UnloadDecision(unload, UnloadDecision.Label.Success, UnloadDecision.Reason.Admin); return unloadAsync(unloadDecision, - conf.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS); + timeout, timeoutUnit); }); } @@ -484,6 +716,10 @@ private CompletableFuture unloadAsync(UnloadDecision unloadDecision, public CompletableFuture splitNamespaceBundleAsync(ServiceUnitId bundle, NamespaceBundleSplitAlgorithm splitAlgorithm, List boundaries) { + if (NamespaceService.isSLAOrHeartbeatNamespace(bundle.getNamespaceObject().toString())) { + log.info("Skip split namespace bundle: {}.", bundle); + return CompletableFuture.completedFuture(null); + } final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle.toString()); final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle.toString()); NamespaceBundle namespaceBundle = @@ -530,26 +766,14 @@ private CompletableFuture splitAsync(SplitDecision decision, @Override public void close() throws PulsarServerException { - if (!this.started) { + if (state.get() == State.INIT) { return; } try { - if (brokerLoadDataReportTask != null) { - brokerLoadDataReportTask.cancel(true); - } - - if (topBundlesLoadDataReportTask != null) { - topBundlesLoadDataReportTask.cancel(true); - } - - if (monitorTask != null) { - monitorTask.cancel(true); - } - - this.brokerLoadDataStore.close(); - this.topBundlesLoadDataStore.close(); + stopLoadDataReportTasks(); this.unloadScheduler.close(); this.splitScheduler.close(); + this.serviceUnitStateTableViewSyncer.close(); } catch (IOException ex) { throw new PulsarServerException(ex); } finally { @@ -565,7 +789,7 @@ public void close() throws PulsarServerException { } catch (Exception e) { throw new PulsarServerException(e); } finally { - this.started = false; + state.set(State.INIT); } } @@ -573,82 +797,160 @@ public void close() throws PulsarServerException { } } - private boolean isInternalTopic(String topic) { - return topic.startsWith(ServiceUnitStateChannelImpl.TOPIC) + private void stopLoadDataReportTasks() { + if (brokerLoadDataReportTask != null) { + brokerLoadDataReportTask.cancel(true); + } + if (topBundlesLoadDataReportTask != null) { + topBundlesLoadDataReportTask.cancel(true); + } + if (monitorTask != null) { + monitorTask.cancel(true); + } + try { + brokerLoadDataStore.shutdown(); + } catch (IOException e) { + log.warn("Failed to shutdown brokerLoadDataStore", e); + } + try { + topBundlesLoadDataStore.shutdown(); + } catch (IOException e) { + log.warn("Failed to shutdown topBundlesLoadDataStore", e); + } + } + + public static boolean isInternalTopic(String topic) { + return INTERNAL_TOPICS.contains(topic) + || topic.startsWith(TOPIC) || topic.startsWith(BROKER_LOAD_DATA_STORE_TOPIC) || topic.startsWith(TOP_BUNDLES_LOAD_DATA_STORE_TOPIC); } @VisibleForTesting - void playLeader() { - if (role != Leader) { - log.info("This broker:{} is changing the role from {} to {}", - pulsar.getLookupServiceAddress(), role, Leader); - int retry = 0; - while (true) { - try { - initWaiter.await(); - serviceUnitStateChannel.scheduleOwnershipMonitor(); - topBundlesLoadDataStore.startTableView(); - unloadScheduler.start(); + synchronized void playLeader() { + log.info("This broker:{} is setting the role from {} to {}", + pulsar.getBrokerId(), role, Leader); + int retry = 0; + boolean becameFollower = false; + while (!Thread.currentThread().isInterrupted()) { + try { + if (!initWaiter.get() || disabled()) { + return; + } + if (!serviceUnitStateChannel.isChannelOwner()) { + becameFollower = true; break; - } catch (Throwable e) { - log.error("The broker:{} failed to change the role. Retrying {} th ...", - pulsar.getLookupServiceAddress(), ++retry, e); - try { - Thread.sleep(Math.min(retry * 10, MAX_ROLE_CHANGE_RETRY_DELAY_IN_MILLIS)); - } catch (InterruptedException ex) { - log.warn("Interrupted while sleeping."); - } + } + if (disabled()) { + return; + } + // Confirm the system topics have been created or create them if they do not exist. + // If the leader has changed, the new leader need to reset + // the local brokerService.topics (by this topic creations). + // Otherwise, the system topic existence check will fail on the leader broker. + createSystemTopics(pulsar); + brokerLoadDataStore.init(); + topBundlesLoadDataStore.init(); + unloadScheduler.start(); + serviceUnitStateChannel.scheduleOwnershipMonitor(); + if (pulsar.getConfiguration().isLoadBalancerServiceUnitTableViewSyncerEnabled()) { + serviceUnitStateTableViewSyncer.start(pulsar); + } + break; + } catch (Throwable e) { + if (disabled()) { + log.warn("The broker:{} failed to set the role but exit because it's disabled", + pulsar.getBrokerId(), e); + return; + } + log.warn("The broker:{} failed to set the role. Retrying {} th ...", + pulsar.getBrokerId(), ++retry, e); + try { + Thread.sleep(Math.min(retry * 10, MAX_ROLE_CHANGE_RETRY_DELAY_IN_MILLIS)); + } catch (InterruptedException ex) { + log.warn("Interrupted while sleeping."); + // preserve thread's interrupt status + Thread.currentThread().interrupt(); } } - role = Leader; - log.info("This broker:{} plays the leader now.", pulsar.getLookupServiceAddress()); } - - // flush the load data when the leader is elected. - if (brokerLoadDataReporter != null) { - brokerLoadDataReporter.reportAsync(true); + if (disabled()) { + return; } - if (topBundleLoadDataReporter != null) { - topBundleLoadDataReporter.reportAsync(true); + + if (becameFollower) { + log.warn("The broker:{} became follower while initializing leader role.", pulsar.getBrokerId()); + playFollower(); + return; } + + role = Leader; + log.info("This broker:{} plays the leader now.", pulsar.getBrokerId()); + + // flush the load data when the leader is elected. + brokerLoadDataReporter.reportAsync(true); + topBundleLoadDataReporter.reportAsync(true); } @VisibleForTesting - void playFollower() { - if (role != Follower) { - log.info("This broker:{} is changing the role from {} to {}", - pulsar.getLookupServiceAddress(), role, Follower); - int retry = 0; - while (true) { - try { - initWaiter.await(); - serviceUnitStateChannel.cancelOwnershipMonitor(); - topBundlesLoadDataStore.closeTableView(); - unloadScheduler.close(); + synchronized void playFollower() { + log.info("This broker:{} is setting the role from {} to {}", + pulsar.getBrokerId(), role, Follower); + int retry = 0; + boolean becameLeader = false; + while (!Thread.currentThread().isInterrupted()) { + try { + if (!initWaiter.get() || disabled()) { + return; + } + if (serviceUnitStateChannel.isChannelOwner()) { + becameLeader = true; break; - } catch (Throwable e) { - log.error("The broker:{} failed to change the role. Retrying {} th ...", - pulsar.getLookupServiceAddress(), ++retry, e); - try { - Thread.sleep(Math.min(retry * 10, MAX_ROLE_CHANGE_RETRY_DELAY_IN_MILLIS)); - } catch (InterruptedException ex) { - log.warn("Interrupted while sleeping."); - } + } + if (disabled()) { + return; + } + unloadScheduler.close(); + serviceUnitStateChannel.cancelOwnershipMonitor(); + closeInternalTopics(); + brokerLoadDataStore.init(); + topBundlesLoadDataStore.close(); + topBundlesLoadDataStore.startProducer(); + serviceUnitStateTableViewSyncer.close(); + break; + } catch (Throwable e) { + if (disabled()) { + log.warn("The broker:{} failed to set the role but exit because it's disabled", + pulsar.getBrokerId(), e); + return; + } + log.warn("The broker:{} failed to set the role. Retrying {} th ...", + pulsar.getBrokerId(), ++retry, e); + try { + Thread.sleep(Math.min(retry * 10, MAX_ROLE_CHANGE_RETRY_DELAY_IN_MILLIS)); + } catch (InterruptedException ex) { + log.warn("Interrupted while sleeping."); + // preserve thread's interrupt status + Thread.currentThread().interrupt(); } } - role = Follower; - log.info("This broker:{} plays a follower now.", pulsar.getLookupServiceAddress()); } - - // flush the load data when the leader is elected. - if (brokerLoadDataReporter != null) { - brokerLoadDataReporter.reportAsync(true); + if (disabled()) { + return; } - if (topBundleLoadDataReporter != null) { - topBundleLoadDataReporter.reportAsync(true); + + if (becameLeader) { + log.warn("This broker:{} became leader while initializing follower role.", pulsar.getBrokerId()); + playLeader(); + return; } + + role = Follower; + log.info("This broker:{} plays a follower now.", pulsar.getBrokerId()); + + // flush the load data when the leader is elected. + brokerLoadDataReporter.reportAsync(true); + topBundleLoadDataReporter.reportAsync(true); } public List getMetrics() { @@ -666,31 +968,59 @@ public List getMetrics() { } metricsCollection.addAll(this.assignCounter.toMetrics(pulsar.getAdvertisedAddress())); - metricsCollection.addAll(this.serviceUnitStateChannel.getMetrics()); + metricsCollection.addAll(getIgnoredCommandMetrics(pulsar.getAdvertisedAddress())); return metricsCollection; } - private void monitor() { + private List getIgnoredCommandMetrics(String advertisedBrokerAddress) { + var dimensions = Map.of("broker", advertisedBrokerAddress, "metric", "bundleUnloading"); + var metric = Metrics.create(dimensions); + metric.put("brk_lb_ignored_ack_total", ignoredAckCount.get()); + metric.put("brk_lb_ignored_send_total", ignoredSendMsgCount.get()); + return List.of(metric); + } + + @VisibleForTesting + protected void monitor() { try { - initWaiter.await(); + if (!initWaiter.get()) { + return; + } + + // Monitor broker registry + // Periodically check the broker registry in case metadata store fails. + validateBrokerRegistry(); // Monitor role - // Periodically check the role in case ZK watcher fails. + // Periodically check the role in case metadata store fails. var isChannelOwner = serviceUnitStateChannel.isChannelOwner(); if (isChannelOwner) { + // System topic config might fail due to the race condition + // with topic policy init(Topic policies cache have not init). + if (isPersistentSystemTopicUsed() && !configuredSystemTopics) { + configuredSystemTopics = configureSystemTopics(pulsar, COMPACTION_THRESHOLD); + } if (role != Leader) { log.warn("Current role:{} does not match with the channel ownership:{}. " + "Playing the leader role.", role, isChannelOwner); playLeader(); } + + if (pulsar.getConfiguration().isLoadBalancerServiceUnitTableViewSyncerEnabled()) { + serviceUnitStateTableViewSyncer.start(pulsar); + } else { + serviceUnitStateTableViewSyncer.close(); + } + } else { if (role != Follower) { log.warn("Current role:{} does not match with the channel ownership:{}. " + "Playing the follower role.", role, isChannelOwner); playFollower(); } + serviceUnitStateTableViewSyncer.close(); } } catch (Throwable e) { log.error("Failed to get the channel ownership.", e); @@ -698,8 +1028,80 @@ private void monitor() { } public void disableBroker() throws Exception { + // TopicDoesNotExistException might be thrown and it's not recoverable. Enable this flag to exit playFollower() + // or playLeader() quickly. + if (!state.compareAndSet(State.RUNNING, State.DISABLED)) { + failForUnexpectedState("disableBroker"); + } + stopLoadDataReportTasks(); serviceUnitStateChannel.cleanOwnerships(); - leaderElectionService.close(); brokerRegistry.unregister(); + leaderElectionService.close(); + final var availableBrokers = brokerRegistry.getAvailableBrokersAsync() + .get(conf.getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS); + if (availableBrokers.isEmpty()) { + close(); + } + // Close the internal topics (if owned any) after giving up the possible leader role, + // so that the subsequent lookups could hit the next leader. + closeInternalTopics(); + } + + private void closeInternalTopics() { + List> futures = new ArrayList<>(); + for (String name : INTERNAL_TOPICS) { + pulsar.getBrokerService() + .getTopicReference(name) + .ifPresent(topic -> futures.add(topic.close(true) + .exceptionally(__ -> { + log.warn("Failed to close internal topic:{}", name); + return null; + }))); + } + try { + FutureUtil.waitForAll(futures) + .get(pulsar.getConfiguration().getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS); + } catch (Throwable e) { + log.warn("Failed to wait for closing internal topics", e); + } + } + + @VisibleForTesting + protected BrokerRegistry createBrokerRegistry(PulsarService pulsar) { + return new BrokerRegistryImpl(pulsar); + } + + @VisibleForTesting + protected ServiceUnitStateChannel createServiceUnitStateChannel(PulsarService pulsar) { + return new ServiceUnitStateChannelImpl(pulsar); } + + private void failForUnexpectedState(String msg) { + throw new IllegalStateException("Failed to " + msg + ", state: " + state.get()); + } + + boolean running() { + return state.get() == State.RUNNING; + } + + private boolean disabled() { + return state.get() == State.DISABLED; + } + + private boolean isPersistentSystemTopicUsed() { + return ServiceUnitStateTableViewImpl.class.getName() + .equals(pulsar.getConfiguration().getLoadManagerServiceUnitStateTableViewClassName()); + } + + private void validateBrokerRegistry() + throws ExecutionException, InterruptedException, TimeoutException { + var timeout = pulsar.getConfiguration().getMetadataStoreOperationTimeoutSeconds(); + var lookup = brokerRegistry.lookupAsync(brokerRegistry.getBrokerId()).get(timeout, TimeUnit.SECONDS); + if (lookup.isEmpty()) { + log.warn("Found this broker:{} has not registered yet. Trying to register it", + brokerRegistry.getBrokerId()); + brokerRegistry.registerAsync().get(timeout, TimeUnit.SECONDS); + } + } + } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java index 18e949537dedb..35f6cfcbcf549 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java @@ -28,10 +28,11 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.loadbalance.LoadManager; import org.apache.pulsar.broker.loadbalance.ResourceUnit; -import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.lookup.LookupResult; +import org.apache.pulsar.broker.namespace.LookupOptions; import org.apache.pulsar.common.naming.ServiceUnitId; import org.apache.pulsar.common.stats.Metrics; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.policies.data.loadbalancer.LoadManagerReport; public class ExtensibleLoadManagerWrapper implements LoadManager { @@ -49,6 +50,10 @@ public void start() throws PulsarServerException { loadManager.start(); } + public boolean started() { + return loadManager.running() && loadManager.getServiceUnitStateChannel().started(); + } + @Override public void initialize(PulsarService pulsar) { loadManager.initialize(pulsar); @@ -62,9 +67,15 @@ public boolean isCentralized() { @Override public CompletableFuture> findBrokerServiceUrl( - Optional topic, ServiceUnitId bundle) { - return loadManager.assign(topic, bundle) - .thenApply(lookupData -> lookupData.map(BrokerLookupData::toLookupResult)); + Optional topic, ServiceUnitId bundle, LookupOptions options) { + return loadManager.assign(topic, bundle, options) + .thenApply(lookupData -> lookupData.map(data -> { + try { + return data.toLookupResult(options); + } catch (PulsarServerException ex) { + throw FutureUtil.wrapToCompletionException(ex); + } + })); } @Override @@ -118,13 +129,11 @@ public void setLoadReportForceUpdateFlag() { @Override public void writeLoadReportOnZookeeper() throws Exception { // No-op, this operation is not useful, the load data reporter will automatically write. - throw new UnsupportedOperationException(); } @Override public void writeResourceQuotasToZooKeeper() throws Exception { // No-op, this operation is not useful, the load data reporter will automatically write. - throw new UnsupportedOperationException(); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitState.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitState.java index 42ef55593ae1a..b823a8277d376 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitState.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitState.java @@ -42,7 +42,13 @@ public enum ServiceUnitState { Deleted; // deleted in the system (semi-terminal state) - private static final Map> validTransitions = Map.of( + + public enum StorageType { + SystemTopic, + MetadataStore; + } + + private static final Map> validTransitionsOverSystemTopic = Map.of( // (Init -> all states) transitions are required // when the topic is compacted in the middle of assign, transfer or split. Init, Set.of(Free, Owned, Assigning, Releasing, Splitting, Deleted), @@ -54,12 +60,24 @@ public enum ServiceUnitState { Deleted, Set.of(Init) ); + private static final Map> validTransitionsOverMetadataStore = Map.of( + Init, Set.of(Assigning), + Free, Set.of(Assigning), + Owned, Set.of(Splitting, Releasing), + Assigning, Set.of(Owned), + Releasing, Set.of(Assigning, Free), + Splitting, Set.of(Deleted), + Deleted, Set.of(Init) + ); + private static final Set inFlightStates = Set.of( Assigning, Releasing, Splitting ); - public static boolean isValidTransition(ServiceUnitState from, ServiceUnitState to) { - Set transitions = validTransitions.get(from); + public static boolean isValidTransition(ServiceUnitState from, ServiceUnitState to, StorageType storageType) { + Set transitions = + (storageType == StorageType.SystemTopic) ? validTransitionsOverSystemTopic.get(from) + : validTransitionsOverMetadataStore.get(from); return transitions.contains(to); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java index 6e75fe91a914f..ac9897a20e75c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java @@ -24,13 +24,14 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.loadbalance.extensions.manager.StateChangeListener; import org.apache.pulsar.broker.loadbalance.extensions.models.Split; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; +import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.stats.Metrics; -import org.apache.pulsar.metadata.api.NotificationType; -import org.apache.pulsar.metadata.api.extended.SessionEvent; /** * Defines the ServiceUnitStateChannel interface. @@ -43,6 +44,11 @@ public interface ServiceUnitStateChannel extends Closeable { */ void start() throws PulsarServerException; + /** + * Whether the channel started. + */ + boolean started(); + /** * Closes the ServiceUnitStateChannel. * @throws PulsarServerException if it fails to close the channel. @@ -51,130 +57,79 @@ public interface ServiceUnitStateChannel extends Closeable { void close() throws PulsarServerException; /** - * Asynchronously gets the current owner broker of the system topic in this channel. - * @return the service url without the protocol prefix, 'http://'. e.g. broker-xyz:abcd - * - * ServiceUnitStateChannel elects the separate leader as the owner broker of the system topic in this channel. + * Asynchronously gets the current owner broker of this channel. + * @return a future of owner brokerId to track the completion of the operation */ CompletableFuture> getChannelOwnerAsync(); /** - * Asynchronously checks if the current broker is the owner broker of the system topic in this channel. - * @return True if the current broker is the owner. Otherwise, false. + * Asynchronously checks if the current broker is the owner broker of this channel. + * @return a future of check result to track the completion of the operation */ CompletableFuture isChannelOwnerAsync(); /** - * Checks if the current broker is the owner broker of the system topic in this channel. + * Checks if the current broker is the owner broker of this channel. * @return True if the current broker is the owner. Otherwise, false. + * @throws ExecutionException + * @throws InterruptedException + * @throws TimeoutException */ - boolean isChannelOwner(); + boolean isChannelOwner() throws ExecutionException, InterruptedException, TimeoutException; /** - * Handles the metadata session events to track - * if the connection between the broker and metadata store is stable or not. - * This will be registered as a metadata SessionEvent listener. - * - * The stability of the metadata connection is important - * to determine how to handle the broker deletion(unavailable) event notified from the metadata store. - * - * Please refer to handleBrokerRegistrationEvent(String broker, NotificationType type) for more details. + * Asynchronously gets the current owner broker of the service unit. * - * @param event metadata session events + * @param serviceUnit (e.g. bundle) + * @return a future of owner brokerId to track the completion of the operation */ - void handleMetadataSessionEvent(SessionEvent event); + CompletableFuture> getOwnerAsync(String serviceUnit); /** - * Handles the broker registration event from the broker registry. - * This will be registered as a broker registry listener. - * - * Case 1: If NotificationType is Deleted, - * it will schedule a clean-up operation to release the ownerships of the deleted broker. - * - * Sub-case1: If the metadata connection has been stable for long time, - * it will immediately execute the cleanup operation to guarantee high-availability. - * - * Sub-case2: If the metadata connection has been stable only for short time, - * it will defer the clean-up operation for some time and execute it. - * This is to gracefully handle the case when metadata connection is flaky -- - * If the deleted broker comes back very soon, - * we better cancel the clean-up operation for high-availability. - * - * Sub-case3: If the metadata connection is unstable, - * it will not schedule the clean-up operation, as the broker-metadata connection is lost. - * The brokers will continue to serve existing topics connections, - * and we better not to interrupt the existing topic connections for high-availability. - * + * Asynchronously gets the assigned broker of the service unit. * - * Case 2: If NotificationType is Created, - * it will cancel any scheduled clean-up operation if still not executed. - * - * @param broker notified broker - * @param type notification type + * @param serviceUnit (e.g. bundle)) + * @return assigned brokerId */ - void handleBrokerRegistrationEvent(String broker, NotificationType type); + Optional getAssigned(String serviceUnit); - /** - * Asynchronously gets the current owner broker of the service unit. - * - * - * @param serviceUnit (e.g. bundle) - * @return the future object of the owner broker - * - * Case 1: If the service unit is owned, it returns the completed future object with the current owner. - * Case 2: If the service unit's assignment is ongoing, it returns the non-completed future object. - * Sub-case1: If the assigned broker is available and finally takes the ownership, - * the future object will complete and return the owner broker. - * Sub-case2: If the assigned broker does not take the ownership in time, - * the future object will time out. - * Case 3: If none of them, it returns Optional.empty(). - */ - CompletableFuture> getOwnerAsync(String serviceUnit); /** * Checks if the target broker is the owner of the service unit. * - * * @param serviceUnit (e.g. bundle) - * @param targetBroker - * @return true if the target broker is the owner. false if unknown. + * @param targetBrokerId + * @return true if the target brokerId is the owner brokerId. false if unknown. */ - boolean isOwner(String serviceUnit, String targetBroker); + boolean isOwner(String serviceUnit, String targetBrokerId); /** * Checks if the current broker is the owner of the service unit. * - * * @param serviceUnit (e.g. bundle)) * @return true if the current broker is the owner. false if unknown. */ boolean isOwner(String serviceUnit); /** - * Asynchronously publishes the service unit assignment event to the system topic in this channel. - * It de-duplicates assignment events if there is any ongoing assignment event for the same service unit. + * Asynchronously publishes the service unit assignment event to this channel. * @param serviceUnit (e.g bundle) - * @param broker the assigned broker - * @return the completable future object with the owner broker - * case 1: If the assigned broker is available and takes the ownership, - * the future object will complete and return the owner broker. - * The returned owner broker could be different from the input broker (due to assignment race-condition). - * case 2: If the assigned broker does not take the ownership in time, - * the future object will time out. + * @param brokerId the assigned brokerId + * @return a future of owner brokerId to track the completion of the operation */ - CompletableFuture publishAssignEventAsync(String serviceUnit, String broker); + CompletableFuture publishAssignEventAsync(String serviceUnit, String brokerId); /** - * Asynchronously publishes the service unit unload event to the system topic in this channel. + * Asynchronously publishes the service unit unload event to this channel. * @param unload (unload specification object) - * @return the completable future object staged from the event message sendAsync. + * @return a future to track the completion of the operation */ CompletableFuture publishUnloadEventAsync(Unload unload); /** - * Asynchronously publishes the bundle split event to the system topic in this channel. + * Asynchronously publishes the bundle split event to this channel. * @param split (split specification object) - * @return the completable future object staged from the event message sendAsync. + * @return a future to track the completion of the operation */ CompletableFuture publishSplitEventAsync(Split split); @@ -185,18 +140,24 @@ public interface ServiceUnitStateChannel extends Closeable { List getMetrics(); /** - * Add a state change listener. + * Adds a state change listener. * * @param listener State change listener. */ void listen(StateChangeListener listener); /** - * Returns service unit ownership entry set. - * @return a set of service unit ownership entries + * Asynchronously returns service unit ownership entry set. + * @return a set of service unit ownership entries to track the completion of the operation */ Set> getOwnershipEntrySet(); + /** + * Asynchronously returns service units owned by this broker. + * @return a set of owned service units to track the completion of the operation + */ + Set getOwnedServiceUnits(); + /** * Schedules ownership monitor to periodically check and correct invalid ownership states. */ @@ -208,7 +169,7 @@ public interface ServiceUnitStateChannel extends Closeable { void cancelOwnershipMonitor(); /** - * Cleans the service unit ownerships from the current broker's channel. + * Cleans(gives up) any service unit ownerships from this broker. */ void cleanOwnerships(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 6246c26e57bcc..49d038d512e59 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -32,6 +32,7 @@ import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.isInFlightState; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.ChannelState.Closed; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.ChannelState.Constructed; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.ChannelState.Disabled; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.ChannelState.LeaderElectionServiceStarted; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.ChannelState.Started; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.EventType.Assign; @@ -53,6 +54,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.ScheduledFuture; @@ -66,10 +68,12 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.commons.lang3.mutable.MutableObject; import org.apache.pulsar.PulsarClusterMetadataSetup; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.LeaderBroker; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; @@ -79,66 +83,54 @@ import org.apache.pulsar.broker.loadbalance.extensions.models.Split; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; +import org.apache.pulsar.broker.namespace.LookupOptions; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerServiceException; -import org.apache.pulsar.client.api.CompressionType; -import org.apache.pulsar.client.api.MessageId; -import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.api.TableView; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceBundleFactory; import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm; import org.apache.pulsar.common.naming.NamespaceBundles; -import org.apache.pulsar.common.naming.TopicDomain; -import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.naming.TopicVersion; import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.apache.pulsar.common.util.Reflections; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.NotificationType; import org.apache.pulsar.metadata.api.extended.SessionEvent; @Slf4j public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel { - public static final String TOPIC = TopicName.get( - TopicDomain.persistent.value(), - SYSTEM_NAMESPACE, - "loadbalancer-service-unit-state").toString(); - - public static final CompressionType MSG_COMPRESSION_TYPE = CompressionType.ZSTD; - private static final long MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS = 30 * 1000; // 30sec private static final int OWNERSHIP_CLEAN_UP_MAX_WAIT_TIME_IN_MILLIS = 5000; private static final int OWNERSHIP_CLEAN_UP_WAIT_RETRY_DELAY_IN_MILLIS = 100; - private static final int OWNERSHIP_CLEAN_UP_CONVERGENCE_DELAY_IN_MILLIS = 3000; public static final long VERSION_ID_INIT = 1; // initial versionId - private static final long OWNERSHIP_MONITOR_DELAY_TIME_IN_SECS = 60; public static final long MAX_CLEAN_UP_DELAY_TIME_IN_SECS = 3 * 60; // 3 mins private static final long MIN_CLEAN_UP_DELAY_TIME_IN_SECS = 0; // 0 secs to clean immediately private static final long MAX_CHANNEL_OWNER_ELECTION_WAITING_TIME_IN_SECS = 10; - private static final int MAX_OUTSTANDING_PUB_MESSAGES = 500; private static final long MAX_OWNED_BUNDLE_COUNT_DELAY_TIME_IN_MILLIS = 10 * 60 * 1000; + private static final long MAX_BROKER_HEALTH_CHECK_RETRY = 3; + private static final long MAX_BROKER_HEALTH_CHECK_DELAY_IN_MILLIS = 1000; private final PulsarService pulsar; private final ServiceConfiguration config; private final Schema schema; - private final ConcurrentOpenHashMap> getOwnerRequests; - private final String lookupServiceAddress; - private final ConcurrentOpenHashMap> cleanupJobs; + private final Map> getOwnerRequests; + private final String brokerId; + private final Map> cleanupJobs; private final StateChangeListeners stateChangeListeners; - private ExtensibleLoadManagerImpl loadManager; + private BrokerRegistry brokerRegistry; private LeaderElectionService leaderElectionService; - private TableView tableview; - private Producer producer; + + private ServiceUnitStateTableView tableview; private ScheduledFuture monitorTask; private SessionEvent lastMetadataSessionEvent = SessionReestablished; private long lastMetadataSessionEventTimestamp = 0; private long inFlightStateWaitingTimeInMillis; private long ownershipMonitorDelayTimeInSecs; - private long semiTerminalStateWaitingTimeInMillis; + private long stateTombstoneDelayTimeInMillis; private long maxCleanupDelayTimeInSecs; private long minCleanupDelayTimeInSecs; // cleanup metrics @@ -160,7 +152,6 @@ public enum EventType { Split, Unload, Override - } @Getter @@ -168,7 +159,8 @@ public enum EventType { public static class Counters { private final AtomicLong total; private final AtomicLong failure; - public Counters(){ + + public Counters() { total = new AtomicLong(); failure = new AtomicLong(); } @@ -183,11 +175,13 @@ enum ChannelState { Closed(0), Constructed(1), LeaderElectionServiceStarted(2), - Started(3); + Started(3), + Disabled(4); ChannelState(int id) { this.id = id; } + int id; } @@ -197,23 +191,25 @@ enum MetadataState { Unstable } + @VisibleForTesting public ServiceUnitStateChannelImpl(PulsarService pulsar) { this.pulsar = pulsar; this.config = pulsar.getConfig(); - this.lookupServiceAddress = pulsar.getLookupServiceAddress(); + this.brokerId = pulsar.getBrokerId(); this.schema = Schema.JSON(ServiceUnitStateData.class); - this.getOwnerRequests = ConcurrentOpenHashMap.>newBuilder().build(); - this.cleanupJobs = ConcurrentOpenHashMap.>newBuilder().build(); + this.getOwnerRequests = new ConcurrentHashMap<>(); + this.cleanupJobs = new ConcurrentHashMap<>(); this.stateChangeListeners = new StateChangeListeners(); - this.semiTerminalStateWaitingTimeInMillis = config.getLoadBalancerServiceUnitStateTombstoneDelayTimeInSeconds() + this.stateTombstoneDelayTimeInMillis = config.getLoadBalancerServiceUnitStateTombstoneDelayTimeInSeconds() * 1000; - this.inFlightStateWaitingTimeInMillis = MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS; - this.ownershipMonitorDelayTimeInSecs = OWNERSHIP_MONITOR_DELAY_TIME_IN_SECS; - if (semiTerminalStateWaitingTimeInMillis < inFlightStateWaitingTimeInMillis) { + this.inFlightStateWaitingTimeInMillis = config.getLoadBalancerInFlightServiceUnitStateWaitingTimeInMillis(); + this.ownershipMonitorDelayTimeInSecs = config.getLoadBalancerServiceUnitStateMonitorIntervalInSeconds(); + if (stateTombstoneDelayTimeInMillis < inFlightStateWaitingTimeInMillis) { throw new IllegalArgumentException( - "Invalid Config: loadBalancerServiceUnitStateCleanUpDelayTimeInSeconds < " - + (MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS / 1000) + " secs"); + "Invalid Config: loadBalancerServiceUnitStateTombstoneDelayTimeInSeconds" + + stateTombstoneDelayTimeInMillis / 1000 + " secs" + + "< loadBalancerInFlightServiceUnitStateWaitingTimeInMillis" + + inFlightStateWaitingTimeInMillis + " millis"); } this.maxCleanupDelayTimeInSecs = MAX_CLEAN_UP_DELAY_TIME_IN_SECS; this.minCleanupDelayTimeInSecs = MIN_CLEAN_UP_DELAY_TIME_IN_SECS; @@ -234,6 +230,7 @@ public ServiceUnitStateChannelImpl(PulsarService pulsar) { this.channelState = Constructed; } + @Override public void scheduleOwnershipMonitor() { if (monitorTask == null) { this.monitorTask = this.pulsar.getLoadManagerExecutor() @@ -247,24 +244,47 @@ public void scheduleOwnershipMonitor() { }, 0, ownershipMonitorDelayTimeInSecs, SECONDS); log.info("This leader broker:{} started the ownership monitor.", - lookupServiceAddress); + brokerId); } } + @Override public void cancelOwnershipMonitor() { if (monitorTask != null) { monitorTask.cancel(false); monitorTask = null; log.info("This previous leader broker:{} stopped the ownership monitor.", - lookupServiceAddress); + brokerId); } } @Override public void cleanOwnerships() { - doCleanup(lookupServiceAddress); + disable(); + doCleanup(brokerId, true); + } + + @Override + public synchronized boolean started() { + return validateChannelState(Started, true); + } + + private ServiceUnitStateTableView createServiceUnitStateTableView() { + ServiceConfiguration conf = pulsar.getConfiguration(); + try { + ServiceUnitStateTableView tableview = + Reflections.createInstance(conf.getLoadManagerServiceUnitStateTableViewClassName(), + ServiceUnitStateTableView.class, Thread.currentThread().getContextClassLoader()); + log.info("Created service unit state tableview: {}", tableview.getClass().getCanonicalName()); + return tableview; + } catch (Throwable e) { + log.error("Error when trying to create service unit state tableview: {}.", + conf.getLoadManagerServiceUnitStateTableViewClassName(), e); + throw e; + } } + @Override public synchronized void start() throws PulsarServerException { if (!validateChannelState(LeaderElectionServiceStarted, false)) { throw new IllegalStateException("Invalid channel state:" + channelState.name()); @@ -283,42 +303,18 @@ public synchronized void start() throws PulsarServerException { log.warn("Failed to find the channel leader."); } this.channelState = LeaderElectionServiceStarted; - loadManager = getLoadManager(); - if (producer != null) { - producer.close(); - if (debug) { - log.info("Closed the channel producer."); - } - } - PulsarClusterMetadataSetup.createNamespaceIfAbsent - (pulsar.getPulsarResources(), SYSTEM_NAMESPACE, config.getClusterName()); + PulsarClusterMetadataSetup.createTenantIfAbsent + (pulsar.getPulsarResources(), SYSTEM_NAMESPACE.getTenant(), + pulsar.getConfiguration().getClusterName()); - producer = pulsar.getClient().newProducer(schema) - .enableBatching(true) - .compressionType(MSG_COMPRESSION_TYPE) - .maxPendingMessages(MAX_OUTSTANDING_PUB_MESSAGES) - .blockIfQueueFull(true) - .topic(TOPIC) - .create(); + PulsarClusterMetadataSetup.createNamespaceIfAbsent + (pulsar.getPulsarResources(), SYSTEM_NAMESPACE, pulsar.getConfiguration().getClusterName(), + pulsar.getConfiguration().getDefaultNumberOfNamespaceBundles()); - if (debug) { - log.info("Successfully started the channel producer."); - } + tableview = createServiceUnitStateTableView(); + tableview.start(pulsar, this::handleEvent, this::handleExisting); - if (tableview != null) { - tableview.close(); - if (debug) { - log.info("Closed the channel tableview."); - } - } - tableview = pulsar.getClient().newTableViewBuilder(schema) - .topic(TOPIC) - .loadConf(Map.of( - "topicCompactionStrategyClassName", - ServiceUnitStateCompactionStrategy.class.getName())) - .create(); - tableview.listen((key, value) -> handle(key, value)); if (debug) { log.info("Successfully started the channel tableview."); } @@ -359,23 +355,20 @@ protected LeaderElectionService getLeaderElectionService() { .get().getLeaderElectionService(); } + @VisibleForTesting + protected PulsarAdmin getPulsarAdmin() throws PulsarServerException { + return pulsar.getAdminClient(); + } + + @Override public synchronized void close() throws PulsarServerException { channelState = Closed; - boolean debug = debug(); try { leaderElectionService = null; + if (tableview != null) { tableview.close(); tableview = null; - if (debug) { - log.info("Successfully closed the channel tableview."); - } - } - - if (producer != null) { - producer.close(); - producer = null; - log.info("Successfully closed the channel producer."); } if (brokerRegistry != null) { @@ -413,51 +406,35 @@ private boolean debug() { return ExtensibleLoadManagerImpl.debug(config, log); } + @Override public CompletableFuture> getChannelOwnerAsync() { if (!validateChannelState(LeaderElectionServiceStarted, true)) { return CompletableFuture.failedFuture( new IllegalStateException("Invalid channel state:" + channelState.name())); } - return leaderElectionService.readCurrentLeader().thenApply(leader -> { - //expecting http://broker-xyz:port - // TODO: discard this protocol prefix removal - // by a util func that returns lookupServiceAddress(serviceUrl) - if (leader.isPresent()) { - String broker = leader.get().getServiceUrl(); - broker = broker.substring(broker.lastIndexOf('/') + 1); - return Optional.of(broker); - } else { - return Optional.empty(); - } - } - ); + return leaderElectionService.readCurrentLeader() + .thenApply(leader -> leader.map(LeaderBroker::getBrokerId)); } + @Override public CompletableFuture isChannelOwnerAsync() { return getChannelOwnerAsync().thenApply(owner -> { if (owner.isPresent()) { return isTargetBroker(owner.get()); } else { - String msg = "There is no channel owner now."; - log.error(msg); - throw new IllegalStateException(msg); + throw new IllegalStateException("There is no channel owner now."); } }); } - - public boolean isChannelOwner() { - try { - return isChannelOwnerAsync().get( - MAX_CHANNEL_OWNER_ELECTION_WAITING_TIME_IN_SECS, TimeUnit.SECONDS); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - String msg = "Failed to get the channel owner."; - log.error(msg, e); - throw new RuntimeException(msg, e); - } + @Override + public boolean isChannelOwner() throws ExecutionException, InterruptedException, TimeoutException { + return isChannelOwnerAsync().get( + MAX_CHANNEL_OWNER_ELECTION_WAITING_TIME_IN_SECS, TimeUnit.SECONDS); } - public boolean isOwner(String serviceUnit, String targetBroker) { + @Override + public boolean isOwner(String serviceUnit, String targetBrokerId) { if (!validateChannelState(Started, true)) { throw new IllegalStateException("Invalid channel state:" + channelState.name()); } @@ -466,39 +443,89 @@ public boolean isOwner(String serviceUnit, String targetBroker) { return false; } var owner = ownerFuture.join(); - if (owner.isPresent() && StringUtils.equals(targetBroker, owner.get())) { + if (owner.isPresent() && StringUtils.equals(targetBrokerId, owner.get())) { return true; } return false; } + @Override public boolean isOwner(String serviceUnit) { - return isOwner(serviceUnit, lookupServiceAddress); + return isOwner(serviceUnit, brokerId); } + private CompletableFuture> getActiveOwnerAsync( + String serviceUnit, + ServiceUnitState state, + Optional owner) { + + // If this broker's registry does not exist(possibly suffering from connecting to the metadata store), + // we return the owner without its activeness check. + // This broker tries to serve lookups on a best efforts basis when metadata store connection is unstable. + if (!brokerRegistry.isRegistered()) { + return CompletableFuture.completedFuture(owner); + } + + return dedupeGetOwnerRequest(serviceUnit) + .thenCompose(newOwner -> { + if (newOwner == null) { + return CompletableFuture.completedFuture(null); + } + + return brokerRegistry.lookupAsync(newOwner) + .thenApply(lookupData -> { + if (lookupData.isPresent()) { + return newOwner; + } else { + throw new IllegalStateException( + "The new owner " + newOwner + " is inactive."); + } + }); + }).whenComplete((__, e) -> { + if (e != null) { + log.error("{} failed to get active owner broker. serviceUnit:{}, state:{}, owner:{}", + brokerId, serviceUnit, state, owner, e); + ownerLookUpCounters.get(state).getFailure().incrementAndGet(); + } + }).thenApply(Optional::ofNullable); + } + + /** + * Case 1: If the service unit is owned, it returns the completed future object with the current owner. + * Case 2: If the service unit's assignment is ongoing, it returns the non-completed future object. + * Sub-case1: If the assigned broker is available and finally takes the ownership, + * the future object will complete and return the owner broker. + * Sub-case2: If the assigned broker does not take the ownership in time, + * the future object will time out. + * Case 3: If none of them, it returns Optional.empty(). + */ + @Override public CompletableFuture> getOwnerAsync(String serviceUnit) { if (!validateChannelState(Started, true)) { return CompletableFuture.failedFuture( new IllegalStateException("Invalid channel state:" + channelState.name())); } - - ServiceUnitStateData data = tableview.get(serviceUnit); + var data = tableview.get(serviceUnit); ServiceUnitState state = state(data); ownerLookUpCounters.get(state).getTotal().incrementAndGet(); switch (state) { case Owned -> { - return CompletableFuture.completedFuture(Optional.of(data.dstBroker())); + return getActiveOwnerAsync(serviceUnit, state, Optional.of(data.dstBroker())); } case Splitting -> { - return CompletableFuture.completedFuture(Optional.of(data.sourceBroker())); + return getActiveOwnerAsync(serviceUnit, state, Optional.of(data.sourceBroker())); } case Assigning, Releasing -> { - return deferGetOwnerRequest(serviceUnit).whenComplete((__, e) -> { - if (e != null) { - ownerLookUpCounters.get(state).getFailure().incrementAndGet(); - } - }).thenApply( - broker -> broker == null ? Optional.empty() : Optional.of(broker)); + if (isTargetBroker(data.dstBroker())) { + return getActiveOwnerAsync(serviceUnit, state, Optional.of(data.dstBroker())); + } + // If this broker is not the dst broker, return the dst broker as the owner(or empty). + // Clients need to connect(redirect) to the dst broker anyway + // and wait for the dst broker to receive `Owned`. + // This is also required to return getOwnerAsync on the src broker immediately during unloading. + // Otherwise, topic creation(getOwnerAsync) could block unloading bundles, + // if the topic creation(getOwnerAsync) happens during unloading on the src broker. + return CompletableFuture.completedFuture(Optional.ofNullable(data.dstBroker())); } case Init, Free -> { return CompletableFuture.completedFuture(Optional.empty()); @@ -509,32 +536,93 @@ public CompletableFuture> getOwnerAsync(String serviceUnit) { } default -> { ownerLookUpCounters.get(state).getFailure().incrementAndGet(); - String errorMsg = String.format("Failed to process service unit state data: %s when get owner.", data); + String errorMsg = + String.format("Failed to process service unit state data: %s when get owner.", data); log.error(errorMsg); return CompletableFuture.failedFuture(new IllegalStateException(errorMsg)); } } } - private long getNextVersionId(String serviceUnit) { + private Optional getOwnerNow(String serviceUnit) { + if (!validateChannelState(Started, true)) { + throw new IllegalStateException("Invalid channel state:" + channelState.name()); + } + var data = tableview.get(serviceUnit); + ServiceUnitState state = state(data); + switch (state) { + case Owned -> { + return Optional.of(data.dstBroker()); + } + case Splitting -> { + return Optional.of(data.sourceBroker()); + } + case Init, Free -> { + return Optional.empty(); + } + default -> { + return null; + } + } + } + + + @Override + public Optional getAssigned(String serviceUnit) { + if (!validateChannelState(Started, true)) { + return Optional.empty(); + } + var data = tableview.get(serviceUnit); - return getNextVersionId(data); + if (data == null) { + return Optional.empty(); + } + ServiceUnitState state = state(data); + switch (state) { + case Owned, Assigning -> { + return Optional.of(data.dstBroker()); + } + case Releasing -> { + return Optional.ofNullable(data.dstBroker()); + } + case Splitting -> { + return Optional.of(data.sourceBroker()); + } + case Init, Free -> { + return Optional.empty(); + } + case Deleted -> { + log.warn("Trying to get the assigned broker from the deleted serviceUnit:{}", serviceUnit); + return Optional.empty(); + } + default -> { + log.warn("Trying to get the assigned broker from unknown state:{} serviceUnit:{}", state, + serviceUnit); + return Optional.empty(); + } + } + } + + private Long getNextVersionId(String serviceUnit) { + return getNextVersionId(tableview.get(serviceUnit)); } private long getNextVersionId(ServiceUnitStateData data) { return data == null ? VERSION_ID_INIT : data.versionId() + 1; } - public CompletableFuture publishAssignEventAsync(String serviceUnit, String broker) { + @Override + public CompletableFuture publishAssignEventAsync(String serviceUnit, String brokerId) { if (!validateChannelState(Started, true)) { return CompletableFuture.failedFuture( new IllegalStateException("Invalid channel state:" + channelState.name())); } EventType eventType = Assign; eventCounters.get(eventType).getTotal().incrementAndGet(); - CompletableFuture getOwnerRequest = deferGetOwnerRequest(serviceUnit); + CompletableFuture getOwnerRequest = dedupeGetOwnerRequest(serviceUnit); - pubAsync(serviceUnit, new ServiceUnitStateData(Assigning, broker, getNextVersionId(serviceUnit))) + pubAsync(serviceUnit, + new ServiceUnitStateData(Assigning, brokerId, getNextVersionId(serviceUnit))) .whenComplete((__, ex) -> { if (ex != null) { getOwnerRequests.remove(serviceUnit, getOwnerRequest); @@ -544,24 +632,18 @@ public CompletableFuture publishAssignEventAsync(String serviceUnit, Str eventCounters.get(eventType).getFailure().incrementAndGet(); } }); + return getOwnerRequest; } private CompletableFuture publishOverrideEventAsync(String serviceUnit, - ServiceUnitStateData orphanData, - ServiceUnitStateData override) { + ServiceUnitStateData override) { if (!validateChannelState(Started, true)) { throw new IllegalStateException("Invalid channel state:" + channelState.name()); } EventType eventType = EventType.Override; eventCounters.get(eventType).getTotal().incrementAndGet(); - return pubAsync(serviceUnit, override).whenComplete((__, e) -> { - if (e != null) { - eventCounters.get(eventType).getFailure().incrementAndGet(); - log.error("Failed to override serviceUnit:{} from orphanData:{} to overrideData:{}", - serviceUnit, orphanData, override, e); - } - }).thenApply(__ -> null); + return pubAsync(serviceUnit, override).thenApply(__ -> null); } public CompletableFuture publishUnloadEventAsync(Unload unload) { @@ -606,14 +688,23 @@ public CompletableFuture publishSplitEventAsync(Split split) { }).thenApply(__ -> null); } - private void handle(String serviceUnit, ServiceUnitStateData data) { + private void handleEvent(String serviceUnit, ServiceUnitStateData data) { + long totalHandledRequests = getHandlerTotalCounter(data).incrementAndGet(); - if (log.isDebugEnabled()) { + if (debug()) { log.info("{} received a handle request for serviceUnit:{}, data:{}. totalHandledRequests:{}", - lookupServiceAddress, serviceUnit, data, totalHandledRequests); + brokerId, serviceUnit, data, totalHandledRequests); } ServiceUnitState state = state(data); + if (channelState == Disabled && (data == null || !data.force())) { + final var request = getOwnerRequests.remove(serviceUnit); + if (request != null) { + request.completeExceptionally(new BrokerServiceException.ServiceUnitNotReadyException( + "cancel the lookup request for " + serviceUnit + " when receiving " + state)); + } + return; + } try { switch (state) { case Owned -> handleOwnEvent(serviceUnit, data); @@ -625,13 +716,24 @@ private void handle(String serviceUnit, ServiceUnitStateData data) { case Init -> handleInitEvent(serviceUnit); default -> throw new IllegalStateException("Failed to handle channel data:" + data); } - } catch (Throwable e){ + } catch (Throwable e) { log.error("Failed to handle the event. serviceUnit:{}, data:{}, handlerFailureCount:{}", serviceUnit, data, getHandlerFailureCounter(data).incrementAndGet(), e); throw e; } } + private void handleExisting(String serviceUnit, ServiceUnitStateData data) { + if (debug()) { + log.info("Loaded the service unit state data. serviceUnit: {}, data: {}", serviceUnit, data); + } + ServiceUnitState state = state(data); + if (state.equals(Owned) && isTargetBroker(data.dstBroker())) { + pulsar.getNamespaceService() + .onNamespaceBundleOwned(LoadManagerShared.getNamespaceBundle(pulsar, serviceUnit)); + } + } + private static boolean isTransferCommand(ServiceUnitStateData data) { if (data == null) { return false; @@ -669,12 +771,12 @@ private AtomicLong getHandlerCounter(ServiceUnitStateData data, boolean total) { private void log(Throwable e, String serviceUnit, ServiceUnitStateData data, ServiceUnitStateData next) { if (e == null) { - if (log.isDebugEnabled() || isTransferCommand(data)) { + if (debug() || isTransferCommand(data)) { long handlerTotalCount = getHandlerTotalCounter(data).get(); long handlerFailureCount = getHandlerFailureCounter(data).get(); log.info("{} handled {} event for serviceUnit:{}, cur:{}, next:{}, " + "totalHandledRequests:{}, totalFailedRequests:{}", - lookupServiceAddress, getLogEventTag(data), serviceUnit, + brokerId, getLogEventTag(data), serviceUnit, data == null ? "" : data, next == null ? "" : next, handlerTotalCount, handlerFailureCount @@ -685,7 +787,7 @@ lookupServiceAddress, getLogEventTag(data), serviceUnit, long handlerFailureCount = getHandlerFailureCounter(data).incrementAndGet(); log.error("{} failed to handle {} event for serviceUnit:{}, cur:{}, next:{}, " + "totalHandledRequests:{}, totalFailedRequests:{}", - lookupServiceAddress, getLogEventTag(data), serviceUnit, + brokerId, getLogEventTag(data), serviceUnit, data == null ? "" : data, next == null ? "" : next, handlerTotalCount, handlerFailureCount, @@ -693,17 +795,42 @@ lookupServiceAddress, getLogEventTag(data), serviceUnit, } } + private void handleSkippedEvent(String serviceUnit) { + var getOwnerRequest = getOwnerRequests.get(serviceUnit); + if (getOwnerRequest != null) { + var data = tableview.get(serviceUnit); + if (data != null && data.state() == Owned) { + getOwnerRequest.complete(data.dstBroker()); + getOwnerRequests.remove(serviceUnit); + stateChangeListeners.notify(serviceUnit, data, null); + } + } + } + private void handleOwnEvent(String serviceUnit, ServiceUnitStateData data) { var getOwnerRequest = getOwnerRequests.remove(serviceUnit); if (getOwnerRequest != null) { + if (debug()) { + log.info("Returned owner request for serviceUnit:{}", serviceUnit); + } getOwnerRequest.complete(data.dstBroker()); } - stateChangeListeners.notify(serviceUnit, data, null); + if (isTargetBroker(data.dstBroker())) { - log(null, serviceUnit, data, null); + pulsar.getNamespaceService() + .onNamespaceBundleOwned(LoadManagerShared.getNamespaceBundle(pulsar, serviceUnit)); lastOwnEventHandledAt = System.currentTimeMillis(); - } else if (data.force() && isTargetBroker(data.sourceBroker())) { - closeServiceUnit(serviceUnit); + stateChangeListeners.notify(serviceUnit, data, null); + log(null, serviceUnit, data, null); + } else if (isTargetBroker(data.sourceBroker())) { + var isOrphanCleanup = data.force(); + var isTransfer = isTransferCommand(data) && pulsar.getConfig().isLoadBalancerMultiPhaseBundleUnload(); + var future = isOrphanCleanup || isTransfer + ? closeServiceUnit(serviceUnit, true) : CompletableFuture.completedFuture(null); + stateChangeListeners.notifyOnCompletion(future, serviceUnit, data) + .whenComplete((__, e) -> log(e, serviceUnit, data, null)); + } else { + stateChangeListeners.notify(serviceUnit, data, null); } } @@ -719,15 +846,20 @@ private void handleAssignEvent(String serviceUnit, ServiceUnitStateData data) { private void handleReleaseEvent(String serviceUnit, ServiceUnitStateData data) { if (isTargetBroker(data.sourceBroker())) { ServiceUnitStateData next; + CompletableFuture unloadFuture; if (isTransferCommand(data)) { next = new ServiceUnitStateData( Assigning, data.dstBroker(), data.sourceBroker(), getNextVersionId(data)); - // TODO: when close, pass message to clients to connect to the new broker + // If the optimized bundle unload is disabled, disconnect the clients at time of RELEASE. + var disconnectClients = !pulsar.getConfig().isLoadBalancerMultiPhaseBundleUnload(); + unloadFuture = closeServiceUnit(serviceUnit, disconnectClients); } else { next = new ServiceUnitStateData( Free, null, data.sourceBroker(), getNextVersionId(data)); + unloadFuture = closeServiceUnit(serviceUnit, true); } - stateChangeListeners.notifyOnCompletion(closeServiceUnit(serviceUnit) + // If the optimized bundle unload is disabled, disconnect the clients at time of RELEASE. + stateChangeListeners.notifyOnCompletion(unloadFuture .thenCompose(__ -> pubAsync(serviceUnit, next)), serviceUnit, data) .whenComplete((__, e) -> log(e, serviceUnit, data, next)); } @@ -740,14 +872,24 @@ private void handleSplitEvent(String serviceUnit, ServiceUnitStateData data) { } } - private void handleFreeEvent(String serviceUnit, ServiceUnitStateData data) { + private CompletableFuture handleFreeEvent(String serviceUnit, ServiceUnitStateData data) { var getOwnerRequest = getOwnerRequests.remove(serviceUnit); if (getOwnerRequest != null) { getOwnerRequest.complete(null); } - stateChangeListeners.notify(serviceUnit, data, null); + if (isTargetBroker(data.sourceBroker())) { - log(null, serviceUnit, data, null); + // If data.force(), try closeServiceUnit and tombstone the bundle. + CompletableFuture future = + (data.force() ? closeServiceUnit(serviceUnit, true) + .thenCompose(__ -> tombstoneAsync(serviceUnit)) + : CompletableFuture.completedFuture(0)).thenApply(__ -> null); + stateChangeListeners.notifyOnCompletion(future, serviceUnit, data) + .whenComplete((__, e) -> log(e, serviceUnit, data, null)); + return future; + } else { + stateChangeListeners.notify(serviceUnit, data, null); + return CompletableFuture.completedFuture(null); } } @@ -756,9 +898,13 @@ private void handleDeleteEvent(String serviceUnit, ServiceUnitStateData data) { if (getOwnerRequest != null) { getOwnerRequest.completeExceptionally(new IllegalStateException(serviceUnit + "has been deleted.")); } - stateChangeListeners.notify(serviceUnit, data, null); + if (isTargetBroker(data.sourceBroker())) { - log(null, serviceUnit, data, null); + stateChangeListeners.notifyOnCompletion( + tombstoneAsync(serviceUnit), serviceUnit, data) + .whenComplete((__, e) -> log(e, serviceUnit, data, null)); + } else { + stateChangeListeners.notify(serviceUnit, data, null); } } @@ -771,64 +917,102 @@ private void handleInitEvent(String serviceUnit) { log(null, serviceUnit, null, null); } - private CompletableFuture pubAsync(String serviceUnit, ServiceUnitStateData data) { - CompletableFuture future = new CompletableFuture<>(); - producer.newMessage() - .key(serviceUnit) - .value(data) - .sendAsync() - .whenComplete((messageId, e) -> { + private CompletableFuture pubAsync(String serviceUnit, ServiceUnitStateData data) { + return tableview.put(serviceUnit, data) + .whenComplete((__, e) -> { if (e != null) { log.error("Failed to publish the message: serviceUnit:{}, data:{}", serviceUnit, data, e); - future.completeExceptionally(e); - } else { - future.complete(messageId); } }); - return future; } - private CompletableFuture tombstoneAsync(String serviceUnit) { - return pubAsync(serviceUnit, null); + private CompletableFuture tombstoneAsync(String serviceUnit) { + return tableview.delete(serviceUnit) + .whenComplete((__, e) -> { + if (e != null) { + log.error("Failed to tombstone the serviceUnit:{}}", + serviceUnit, e); + } + }); } private boolean isTargetBroker(String broker) { if (broker == null) { return false; } - return broker.equals(lookupServiceAddress); + return broker.equals(brokerId); } - private NamespaceBundle getNamespaceBundle(String bundle) { - final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle); - final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle); - return pulsar.getNamespaceService().getNamespaceBundleFactory().getBundle(namespaceName, bundleRange); + + private CompletableFuture deferGetOwner(String serviceUnit) { + var future = new CompletableFuture().orTimeout(inFlightStateWaitingTimeInMillis, + TimeUnit.MILLISECONDS) + .exceptionally(e -> { + var ownerAfter = getOwnerNow(serviceUnit); + log.warn("{} failed to wait for owner for serviceUnit:{}; Trying to " + + "return the current owner:{}", + brokerId, serviceUnit, ownerAfter, e); + if (ownerAfter == null) { + throw new IllegalStateException(e); + } + return ownerAfter.orElse(null); + }); + if (debug()) { + log.info("{} is waiting for owner for serviceUnit:{}", brokerId, serviceUnit); + } + return future; } - private CompletableFuture deferGetOwnerRequest(String serviceUnit) { - return getOwnerRequests - .computeIfAbsent(serviceUnit, k -> { - CompletableFuture future = new CompletableFuture<>(); - future.orTimeout(inFlightStateWaitingTimeInMillis, TimeUnit.MILLISECONDS) - .whenComplete((v, e) -> { - if (e != null) { - getOwnerRequests.remove(serviceUnit, future); - log.warn("Failed to getOwner for serviceUnit:{}", - serviceUnit, e); - } - } - ); - return future; + private CompletableFuture dedupeGetOwnerRequest(String serviceUnit) { + + var requested = new MutableObject>(); + try { + return getOwnerRequests.computeIfAbsent(serviceUnit, k -> { + var ownerBefore = getOwnerNow(serviceUnit); + if (ownerBefore != null && ownerBefore.isPresent()) { + // Here, we do the broker active check first with the computeIfAbsent lock + requested.setValue(brokerRegistry.lookupAsync(ownerBefore.get()) + .thenCompose(brokerLookupData -> { + if (brokerLookupData.isPresent()) { + // The owner broker is active. + // Immediately return the request. + return CompletableFuture.completedFuture(ownerBefore.get()); + } else { + // The owner broker is inactive. + // The leader broker should be cleaning up the orphan service units. + // Defer this request til the leader notifies the new ownerships. + return deferGetOwner(serviceUnit); + } + })); + } else { + // The owner broker has not been declared yet. + // The ownership should be in the middle of transferring or assigning. + // Defer this request til the inflight ownership change is complete. + requested.setValue(deferGetOwner(serviceUnit)); + } + return requested.getValue(); + }); + } finally { + var future = requested.getValue(); + if (future != null) { + future.whenComplete((__, e) -> { + getOwnerRequests.remove(serviceUnit); + if (e != null) { + log.warn("{} failed to getOwner for serviceUnit:{}", brokerId, serviceUnit, e); + } }); + } + } } - private CompletableFuture closeServiceUnit(String serviceUnit) { + private CompletableFuture closeServiceUnit(String serviceUnit, boolean disconnectClients) { long startTime = System.nanoTime(); MutableInt unloadedTopics = new MutableInt(); - NamespaceBundle bundle = getNamespaceBundle(serviceUnit); + NamespaceBundle bundle = LoadManagerShared.getNamespaceBundle(pulsar, serviceUnit); return pulsar.getBrokerService().unloadServiceUnit( bundle, + disconnectClients, true, pulsar.getConfig().getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS) @@ -837,13 +1021,19 @@ private CompletableFuture closeServiceUnit(String serviceUnit) { return numUnloadedTopics; }) .whenComplete((__, ex) -> { - // clean up topics that failed to unload from the broker ownership cache - pulsar.getBrokerService().cleanUnloadedTopicFromCache(bundle); + if (disconnectClients) { + // clean up topics that failed to unload from the broker ownership cache + pulsar.getBrokerService().cleanUnloadedTopicFromCache(bundle); + } + pulsar.getNamespaceService().onNamespaceBundleUnload(bundle); double unloadBundleTime = TimeUnit.NANOSECONDS .toMillis((System.nanoTime() - startTime)); if (ex != null) { log.error("Failed to close topics under bundle:{} in {} ms", bundle.toString(), unloadBundleTime, ex); + if (!disconnectClients) { + pulsar.getBrokerService().cleanUnloadedTopicFromCache(bundle); + } } else { log.info("Unloading bundle:{} with {} topics completed in {} ms", bundle, unloadedTopics, unloadBundleTime); @@ -856,7 +1046,7 @@ private CompletableFuture splitServiceUnit(String serviceUnit, ServiceUnit long startTime = System.nanoTime(); NamespaceService namespaceService = pulsar.getNamespaceService(); NamespaceBundleFactory bundleFactory = namespaceService.getNamespaceBundleFactory(); - NamespaceBundle bundle = getNamespaceBundle(serviceUnit); + NamespaceBundle bundle = LoadManagerShared.getNamespaceBundle(pulsar, serviceUnit); CompletableFuture completionFuture = new CompletableFuture<>(); Map> bundleToDestBroker = data.splitServiceUnitToDestBroker(); List boundaries = null; @@ -885,7 +1075,6 @@ private CompletableFuture splitServiceUnit(String serviceUnit, ServiceUnit } - @VisibleForTesting protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService, NamespaceBundleFactory bundleFactory, @@ -905,11 +1094,12 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService, .thenAccept(__ -> // Update bundled_topic cache for load-report-generation pulsar.getBrokerService().refreshTopicToStatsMaps(parentBundle)) .thenAccept(__ -> pubAsync(parentBundle.toString(), new ServiceUnitStateData( - Deleted, null, parentData.sourceBroker(), getNextVersionId(parentData)))) + Deleted, null, parentData.sourceBroker(), getNextVersionId(parentData)))) .thenAccept(__ -> { double splitBundleTime = TimeUnit.NANOSECONDS.toMillis((System.nanoTime() - startTime)); log.info("Successfully split {} parent namespace-bundle to {} in {} ms", parentBundle, childBundles, splitBundleTime); + namespaceService.onNamespaceBundleSplit(parentBundle); completionFuture.complete(null); }) .exceptionally(ex -> { @@ -920,7 +1110,7 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService, log.warn("Failed to update bundle range in metadata store. Retrying {} th / {} limit", counter.get(), NamespaceService.BUNDLE_SPLIT_RETRY_LIMIT, ex); pulsar.getExecutor().schedule(() -> splitServiceUnitOnceAndRetry( - namespaceService, bundleFactory, algorithm, parentBundle, childBundles, + namespaceService, bundleFactory, algorithm, parentBundle, childBundles, boundaries, parentData, counter, startTime, completionFuture), 100, MILLISECONDS); } else { @@ -967,45 +1157,43 @@ private CompletableFuture getSplitNamespaceBundles(NamespaceSe NamespaceBundle parentBundle, List childBundles, List boundaries) { - CompletableFuture future = new CompletableFuture(); final var debug = debug(); - var targetNsBundle = bundleFactory.getBundles(parentBundle.getNamespaceObject()); - boolean found = false; - try { - targetNsBundle.validateBundle(parentBundle); - } catch (IllegalArgumentException e) { - if (debug) { - log.info("Namespace bundles do not contain the parent bundle:{}", - parentBundle); - } - for (var childBundle : childBundles) { - try { - targetNsBundle.validateBundle(childBundle); - if (debug) { - log.info("Namespace bundles contain the child bundle:{}", - childBundle); + return bundleFactory.getBundlesAsync(parentBundle.getNamespaceObject()) + .thenCompose(targetNsBundle -> { + boolean found = false; + try { + targetNsBundle.validateBundle(parentBundle); + } catch (IllegalArgumentException e) { + if (debug) { + log.info("Namespace bundles do not contain the parent bundle:{}", + parentBundle); + } + for (var childBundle : childBundles) { + try { + targetNsBundle.validateBundle(childBundle); + if (debug) { + log.info("Namespace bundles contain the child bundle:{}", + childBundle); + } + } catch (Exception ex) { + throw FutureUtil.wrapToCompletionException( + new BrokerServiceException.ServiceUnitNotReadyException( + "Namespace bundles do not contain the child bundle:" + childBundle, e)); + } + } + found = true; + } catch (Exception e) { + throw FutureUtil.wrapToCompletionException( + new BrokerServiceException.ServiceUnitNotReadyException( + "Failed to validate the parent bundle in the namespace bundles.", e)); } - } catch (Exception ex) { - future.completeExceptionally( - new BrokerServiceException.ServiceUnitNotReadyException( - "Namespace bundles do not contain the child bundle:" + childBundle, e)); - return future; - } - } - found = true; - } catch (Exception e) { - future.completeExceptionally( - new BrokerServiceException.ServiceUnitNotReadyException( - "Failed to validate the parent bundle in the namespace bundles.", e)); - return future; - } - if (found) { - future.complete(targetNsBundle); - return future; - } else { - return namespaceService.getSplitBoundary(parentBundle, algorithm, boundaries) - .thenApply(splitBundlesPair -> splitBundlesPair.getLeft()); - } + if (found) { + return CompletableFuture.completedFuture(targetNsBundle); + } else { + return namespaceService.getSplitBoundary(parentBundle, algorithm, boundaries) + .thenApply(splitBundlesPair -> splitBundlesPair.getLeft()); + } + }); } private CompletableFuture updateSplitNamespaceBundlesAsync( @@ -1026,7 +1214,12 @@ private CompletableFuture updateSplitNamespaceBundlesAsync( }); } - public void handleMetadataSessionEvent(SessionEvent e) { + /** + * The stability of the metadata connection is important + * to determine how to handle the broker deletion(unavailable) event notified from the metadata store. + */ + @VisibleForTesting + protected void handleMetadataSessionEvent(SessionEvent e) { if (e == SessionReestablished || e == SessionLost) { lastMetadataSessionEvent = e; lastMetadataSessionEventTimestamp = System.currentTimeMillis(); @@ -1035,7 +1228,30 @@ public void handleMetadataSessionEvent(SessionEvent e) { } } - public void handleBrokerRegistrationEvent(String broker, NotificationType type) { + /** + * Case 1: If NotificationType is Deleted, + * it will schedule a clean-up operation to release the ownerships of the deleted broker. + * + * Sub-case1: If the metadata connection has been stable for long time, + * it will immediately execute the cleanup operation to guarantee high-availability. + * + * Sub-case2: If the metadata connection has been stable only for short time, + * it will defer the clean-up operation for some time and execute it. + * This is to gracefully handle the case when metadata connection is flaky -- + * If the deleted broker comes back very soon, + * we better cancel the clean-up operation for high-availability. + * + * Sub-case3: If the metadata connection is unstable, + * it will not schedule the clean-up operation, as the broker-metadata connection is lost. + * The brokers will continue to serve existing topics connections, + * and we better not to interrupt the existing topic connections for high-availability. + * + * + * Case 2: If NotificationType is Created, + * it will cancel any scheduled clean-up operation if still not executed. + */ + @VisibleForTesting + protected void handleBrokerRegistrationEvent(String broker, NotificationType type) { if (type == NotificationType.Created) { log.info("BrokerRegistry detected the broker:{} registry has been created.", broker); handleBrokerCreationEvent(broker); @@ -1057,25 +1273,40 @@ private MetadataState getMetadataState() { } private void handleBrokerCreationEvent(String broker) { - CompletableFuture future = cleanupJobs.remove(broker); - if (future != null) { - future.cancel(false); - totalInactiveBrokerCleanupCancelledCnt++; - log.info("Successfully cancelled the ownership cleanup for broker:{}." - + " Active cleanup job count:{}", - broker, cleanupJobs.size()); - } else { - if (debug()) { - log.info("No needs to cancel the ownership cleanup for broker:{}." - + " There was no scheduled cleanup job. Active cleanup job count:{}", - broker, cleanupJobs.size()); - } + + if (!cleanupJobs.isEmpty() && cleanupJobs.containsKey(broker)) { + healthCheckBrokerAsync(broker) + .thenAccept(__ -> { + CompletableFuture future = cleanupJobs.remove(broker); + if (future != null) { + future.cancel(false); + totalInactiveBrokerCleanupCancelledCnt++; + log.info("Successfully cancelled the ownership cleanup for broker:{}." + + " Active cleanup job count:{}", + broker, cleanupJobs.size()); + } else { + if (debug()) { + log.info("No needs to cancel the ownership cleanup for broker:{}." + + " There was no scheduled cleanup job. Active cleanup job count:{}", + broker, cleanupJobs.size()); + } + } + }); } } private void handleBrokerDeletionEvent(String broker) { - if (!isChannelOwner()) { - log.warn("This broker is not the leader now. Ignoring BrokerDeletionEvent for broker {}.", broker); + try { + if (!isChannelOwner()) { + log.warn("This broker is not the leader now. Ignoring BrokerDeletionEvent for broker {}.", broker); + return; + } + } catch (Exception e) { + if (e instanceof ExecutionException && e.getCause() instanceof IllegalStateException) { + log.warn("Failed to handle broker deletion event due to {}", e.getMessage()); + } else { + log.error("Failed to handle broker deletion event.", e); + } return; } MetadataState state = getMetadataState(); @@ -1093,62 +1324,96 @@ private void handleBrokerDeletionEvent(String broker) { } private void scheduleCleanup(String broker, long delayInSecs) { - cleanupJobs.computeIfAbsent(broker, k -> { - Executor delayed = CompletableFuture - .delayedExecutor(delayInSecs, TimeUnit.SECONDS, pulsar.getLoadManagerExecutor()); - totalInactiveBrokerCleanupScheduledCnt++; - return CompletableFuture - .runAsync(() -> { - try { - doCleanup(broker); - } catch (Throwable e) { - log.error("Failed to run the cleanup job for the broker {}, " - + "totalCleanupErrorCnt:{}.", - broker, totalCleanupErrorCnt.incrementAndGet(), e); - } finally { - cleanupJobs.remove(broker); + var scheduled = new MutableObject>(); + try { + final var channelState = this.channelState; + if (channelState == Disabled || channelState == Closed) { + log.warn("[{}] Skip scheduleCleanup because the state is {} now", brokerId, channelState); + return; + } + cleanupJobs.computeIfAbsent(broker, k -> { + Executor delayed = CompletableFuture + .delayedExecutor(delayInSecs, TimeUnit.SECONDS, pulsar.getLoadManagerExecutor()); + totalInactiveBrokerCleanupScheduledCnt++; + var future = CompletableFuture + .runAsync(() -> { + try { + doCleanup(broker, false); + } catch (Throwable e) { + log.error("Failed to run the cleanup job for the broker {}, " + + "totalCleanupErrorCnt:{}.", + broker, totalCleanupErrorCnt.incrementAndGet(), e); + } } - } - , delayed); - }); + , delayed); + scheduled.setValue(future); + return future; + }); + } finally { + var future = scheduled.getValue(); + if (future != null) { + future.whenComplete((v, ex) -> { + cleanupJobs.remove(broker); + }); + } + } log.info("Scheduled ownership cleanup for broker:{} with delay:{} secs. Pending clean jobs:{}.", broker, delayInSecs, cleanupJobs.size()); } - private ServiceUnitStateData getOverrideInactiveBrokerStateData(ServiceUnitStateData orphanData, - String selectedBroker, - String inactiveBroker) { - if (orphanData.state() == Splitting) { - return new ServiceUnitStateData(Splitting, orphanData.dstBroker(), selectedBroker, - Map.copyOf(orphanData.splitServiceUnitToDestBroker()), - true, getNextVersionId(orphanData)); - } else { - return new ServiceUnitStateData(Owned, selectedBroker, inactiveBroker, - true, getNextVersionId(orphanData)); - } - } - - private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanData, String inactiveBroker) { - Optional selectedBroker = selectBroker(serviceUnit, inactiveBroker); - if (selectedBroker.isPresent()) { - var override = getOverrideInactiveBrokerStateData( - orphanData, selectedBroker.get(), inactiveBroker); - log.info("Overriding ownership serviceUnit:{} from orphanData:{} to overrideData:{}", - serviceUnit, orphanData, override); - publishOverrideEventAsync(serviceUnit, orphanData, override) - .exceptionally(e -> { - log.error( - "Failed to override the ownership serviceUnit:{} orphanData:{}. " - + "Failed to publish override event. totalCleanupErrorCnt:{}", - serviceUnit, orphanData, totalCleanupErrorCnt.incrementAndGet()); - return null; - }); - } else { - log.error("Failed to override the ownership serviceUnit:{} orphanData:{}. Empty selected broker. " + private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanData, String inactiveBroker, + boolean gracefully) { + + final var version = getNextVersionId(orphanData); + try { + selectBroker(serviceUnit, inactiveBroker) + .thenApply(selectedOpt -> + selectedOpt.map(selectedBroker -> { + if (orphanData.state() == Splitting) { + // if Splitting, set orphan.dstBroker() as dst to indicate where it was from. + // (The src broker runs handleSplitEvent.) + return new ServiceUnitStateData(Splitting, orphanData.dstBroker(), selectedBroker, + Map.copyOf(orphanData.splitServiceUnitToDestBroker()), true, version); + } else if (orphanData.state() == Owned) { + // if Owned, set orphan.dstBroker() as source to clean it up in case it is still + // alive. + var sourceBroker = selectedBroker.equals(orphanData.dstBroker()) ? null : + orphanData.dstBroker(); + // if gracefully, try to release ownership first + var overrideState = gracefully && sourceBroker != null ? Releasing : Owned; + return new ServiceUnitStateData( + overrideState, + selectedBroker, + sourceBroker, + true, version); + } else { + // if Assigning or Releasing, set orphan.sourceBroker() as source + // to clean it up in case it is still alive. + return new ServiceUnitStateData(Owned, selectedBroker, + selectedBroker.equals(orphanData.sourceBroker()) ? null : + orphanData.sourceBroker(), + true, version); + } + // If no broker is selected(available), free the ownership. + // If the previous owner is still active, it will close the bundle(topic) ownership. + }).orElseGet(() -> new ServiceUnitStateData(Free, null, + orphanData.state() == Owned ? orphanData.dstBroker() : orphanData.sourceBroker(), + true, + version))) + .thenCompose(override -> { + log.info( + "Overriding inactiveBroker:{}, ownership serviceUnit:{} from orphanData:{} to " + + "overrideData:{}", + inactiveBroker, serviceUnit, orphanData, override); + return publishOverrideEventAsync(serviceUnit, override); + }).get(config.getMetadataStoreOperationTimeoutSeconds(), SECONDS); + } catch (Throwable e) { + log.error( + "Failed to override inactiveBroker:{} ownership serviceUnit:{} orphanData:{}. " + "totalCleanupErrorCnt:{}", - serviceUnit, orphanData, totalCleanupErrorCnt.incrementAndGet()); + inactiveBroker, serviceUnit, orphanData, totalCleanupErrorCnt.incrementAndGet(), e); } } @@ -1166,63 +1431,119 @@ private void waitForCleanups(String broker, boolean excludeSystemTopics, int max if (data.state() == Owned && broker.equals(data.dstBroker())) { cleaned = false; + log.info("[{}] bundle {} is still owned by this, data: {}", broker, serviceUnit, data); break; } } if (cleaned) { - try { - MILLISECONDS.sleep(OWNERSHIP_CLEAN_UP_CONVERGENCE_DELAY_IN_MILLIS); - } catch (InterruptedException e) { - log.warn("Interrupted while gracefully waiting for the cleanup convergence."); - } break; } else { try { - MILLISECONDS.sleep(OWNERSHIP_CLEAN_UP_WAIT_RETRY_DELAY_IN_MILLIS); + tableview.flush(OWNERSHIP_CLEAN_UP_WAIT_RETRY_DELAY_IN_MILLIS / 2); + Thread.sleep(OWNERSHIP_CLEAN_UP_MAX_WAIT_TIME_IN_MILLIS / 2); } catch (InterruptedException e) { log.warn("Interrupted while delaying the next service unit clean-up. Cleaning broker:{}", - lookupServiceAddress); + brokerId); + } catch (ExecutionException e) { + log.error("Failed to flush table view", e.getCause()); + } catch (TimeoutException e) { + log.warn("Failed to flush the table view in {} ms", OWNERSHIP_CLEAN_UP_WAIT_RETRY_DELAY_IN_MILLIS); } } } + log.info("Finished cleanup waiting for orphan broker:{}. Elapsed {} ms", brokerId, + System.currentTimeMillis() - started); } - private synchronized void doCleanup(String broker) { + private CompletableFuture healthCheckBrokerAsync(String brokerId) { + CompletableFuture future = new CompletableFuture<>(); + doHealthCheckBrokerAsyncWithRetries(brokerId, 0, future); + return future; + } + + private void doHealthCheckBrokerAsyncWithRetries(String brokerId, int retry, CompletableFuture future) { + try { + var admin = getPulsarAdmin(); + admin.brokers().healthcheckAsync(TopicVersion.V2, Optional.of(brokerId)) + .whenComplete((__, e) -> { + if (e == null) { + log.info("Completed health-check broker :{}", brokerId, e); + future.complete(null); + return; + } + if (retry == MAX_BROKER_HEALTH_CHECK_RETRY) { + log.error("Failed health-check broker :{}", brokerId, e); + future.completeExceptionally(FutureUtil.unwrapCompletionException(e)); + } else { + pulsar.getExecutor() + .schedule(() -> doHealthCheckBrokerAsyncWithRetries(brokerId, retry + 1, future), + Math.min(MAX_BROKER_HEALTH_CHECK_DELAY_IN_MILLIS, retry * retry * 50), + MILLISECONDS); + } + }); + } catch (PulsarServerException e) { + future.completeExceptionally(e); + } + } + + private synchronized void doCleanup(String broker, boolean gracefully) { + try { + if (getChannelOwnerAsync().get(MAX_CHANNEL_OWNER_ELECTION_WAITING_TIME_IN_SECS, TimeUnit.SECONDS) + .isEmpty()) { + log.error("Found the channel owner is empty. Skip the inactive broker:{}'s orphan bundle cleanup", + broker); + return; + } + } catch (Exception e) { + log.error("Failed to find the channel owner. Skip the inactive broker:{}'s orphan bundle cleanup", broker); + return; + } + + // if not gracefully, verify the broker is inactive by health-check. + if (!gracefully) { + try { + healthCheckBrokerAsync(broker).get( + pulsar.getConfiguration().getMetadataStoreOperationTimeoutSeconds(), SECONDS); + log.warn("Found that the broker to clean is healthy. Skip the broker:{}'s orphan bundle cleanup", + broker); + return; + } catch (Exception e) { + if (debug()) { + log.info("Failed to check broker:{} health", broker, e); + } + log.info("Checked the broker:{} health. Continue the orphan bundle cleanup", broker); + } + } + + long startTime = System.nanoTime(); log.info("Started ownership cleanup for the inactive broker:{}", broker); int orphanServiceUnitCleanupCnt = 0; long totalCleanupErrorCntStart = totalCleanupErrorCnt.get(); - + try { + tableview.flush(OWNERSHIP_CLEAN_UP_MAX_WAIT_TIME_IN_MILLIS); + } catch (Exception e) { + log.error("Failed to flush", e); + } Map orphanSystemServiceUnits = new HashMap<>(); for (var etr : tableview.entrySet()) { var stateData = etr.getValue(); var serviceUnit = etr.getKey(); var state = state(stateData); - if (StringUtils.equals(broker, stateData.dstBroker())) { - if (isActiveState(state)) { - if (serviceUnit.startsWith(SYSTEM_NAMESPACE.toString())) { - orphanSystemServiceUnits.put(serviceUnit, stateData); - } else { - overrideOwnership(serviceUnit, stateData, broker); - } - orphanServiceUnitCleanupCnt++; - } - - } else if (StringUtils.equals(broker, stateData.sourceBroker())) { - if (isInFlightState(state)) { - if (serviceUnit.startsWith(SYSTEM_NAMESPACE.toString())) { - orphanSystemServiceUnits.put(serviceUnit, stateData); - } else { - overrideOwnership(serviceUnit, stateData, broker); - } - orphanServiceUnitCleanupCnt++; + if (StringUtils.equals(broker, stateData.dstBroker()) && isActiveState(state) + || StringUtils.equals(broker, stateData.sourceBroker()) && isInFlightState(state)) { + if (serviceUnit.startsWith(SYSTEM_NAMESPACE.toString())) { + orphanSystemServiceUnits.put(serviceUnit, stateData); + } else { + overrideOwnership(serviceUnit, stateData, broker, gracefully); } + orphanServiceUnitCleanupCnt++; } } try { - producer.flush(); - } catch (PulsarClientException e) { + tableview.flush(OWNERSHIP_CLEAN_UP_MAX_WAIT_TIME_IN_MILLIS); + } catch (Exception e) { log.error("Failed to flush the in-flight non-system bundle override messages.", e); } @@ -1241,12 +1562,12 @@ private synchronized void doCleanup(String broker) { // clean system bundles in the end for (var orphanSystemServiceUnit : orphanSystemServiceUnits.entrySet()) { log.info("Overriding orphan system service unit:{}", orphanSystemServiceUnit.getKey()); - overrideOwnership(orphanSystemServiceUnit.getKey(), orphanSystemServiceUnit.getValue(), broker); + overrideOwnership(orphanSystemServiceUnit.getKey(), orphanSystemServiceUnit.getValue(), broker, gracefully); } try { - producer.flush(); - } catch (PulsarClientException e) { + tableview.flush(OWNERSHIP_CLEAN_UP_MAX_WAIT_TIME_IN_MILLIS); + } catch (Exception e) { log.error("Failed to flush the in-flight system bundle override messages.", e); } @@ -1263,68 +1584,28 @@ private synchronized void doCleanup(String broker) { broker, cleanupTime, orphanServiceUnitCleanupCnt, - totalCleanupErrorCntStart - totalCleanupErrorCnt.get(), + totalCleanupErrorCnt.get() - totalCleanupErrorCntStart, printCleanupMetrics()); } - private Optional selectBroker(String serviceUnit, String inactiveBroker) { - try { - return loadManager.selectAsync(getNamespaceBundle(serviceUnit), Set.of(inactiveBroker)) - .get(inFlightStateWaitingTimeInMillis, MILLISECONDS); - } catch (Throwable e) { - log.error("Failed to select a broker for serviceUnit:{}", serviceUnit); - } - return Optional.empty(); + private CompletableFuture> selectBroker(String serviceUnit, String inactiveBroker) { + return getLoadManager().selectAsync( + LoadManagerShared.getNamespaceBundle(pulsar, serviceUnit), + inactiveBroker == null ? Set.of() : Set.of(inactiveBroker), + LookupOptions.builder().build()); } - private Optional getRollForwardStateData(String serviceUnit, - String inactiveBroker, - long nextVersionId) { - Optional selectedBroker = selectBroker(serviceUnit, inactiveBroker); - if (selectedBroker.isEmpty()) { - return Optional.empty(); - } - return Optional.of(new ServiceUnitStateData(Owned, selectedBroker.get(), true, nextVersionId)); - } - - - private Optional getOverrideInFlightStateData( - String serviceUnit, ServiceUnitStateData orphanData, - Set availableBrokers) { - long nextVersionId = getNextVersionId(orphanData); - var state = orphanData.state(); - switch (state) { - case Assigning: { - return getRollForwardStateData(serviceUnit, orphanData.dstBroker(), nextVersionId); - } - case Splitting: { - return Optional.of(new ServiceUnitStateData(Splitting, - orphanData.dstBroker(), orphanData.sourceBroker(), - Map.copyOf(orphanData.splitServiceUnitToDestBroker()), - true, nextVersionId)); - } - case Releasing: { - if (availableBrokers.contains(orphanData.sourceBroker())) { - // rollback to the src - return Optional.of(new ServiceUnitStateData(Owned, orphanData.sourceBroker(), true, nextVersionId)); - } else { - return getRollForwardStateData(serviceUnit, orphanData.sourceBroker(), nextVersionId); - } - } - default: { - var msg = String.format("Failed to get the overrideStateData from serviceUnit=%s, orphanData=%s", - serviceUnit, orphanData); - log.error(msg); - throw new IllegalStateException(msg); - } - } - } @VisibleForTesting protected void monitorOwnerships(List brokers) { - if (!isChannelOwner()) { - log.warn("This broker is not the leader now. Skipping ownership monitor."); + try { + if (!isChannelOwner()) { + log.warn("This broker is not the leader now. Skipping ownership monitor."); + return; + } + } catch (Exception e) { + log.error("Failed to monitor ownerships", e); return; } @@ -1347,7 +1628,7 @@ protected void monitorOwnerships(List brokers) { long startTime = System.nanoTime(); Set inactiveBrokers = new HashSet<>(); Set activeBrokers = new HashSet<>(brokers); - Map orphanServiceUnits = new HashMap<>(); + Map timedOutInFlightStateServiceUnits = new HashMap<>(); int serviceUnitTombstoneCleanupCnt = 0; int orphanServiceUnitCleanupCnt = 0; long totalCleanupErrorCntStart = totalCleanupErrorCnt.get(); @@ -1359,16 +1640,28 @@ protected void monitorOwnerships(List brokers) { String srcBroker = stateData.sourceBroker(); var state = stateData.state(); - if (isActiveState(state)) { - if (StringUtils.isNotBlank(srcBroker) && !activeBrokers.contains(srcBroker)) { - inactiveBrokers.add(srcBroker); - } else if (StringUtils.isNotBlank(dstBroker) && !activeBrokers.contains(dstBroker)) { - inactiveBrokers.add(dstBroker); - } else if (isInFlightState(state) - && now - stateData.timestamp() > inFlightStateWaitingTimeInMillis) { - orphanServiceUnits.put(serviceUnit, stateData); - } - } else if (now - stateData.timestamp() > semiTerminalStateWaitingTimeInMillis) { + if (state == Owned && (StringUtils.isBlank(dstBroker) || !activeBrokers.contains(dstBroker))) { + inactiveBrokers.add(dstBroker); + continue; + } + + if (isInFlightState(state) && StringUtils.isNotBlank(srcBroker) && !activeBrokers.contains(srcBroker)) { + inactiveBrokers.add(srcBroker); + continue; + } + if (isInFlightState(state) && StringUtils.isNotBlank(dstBroker) && !activeBrokers.contains(dstBroker)) { + inactiveBrokers.add(dstBroker); + continue; + } + + if (isInFlightState(state) + && now - stateData.timestamp() > inFlightStateWaitingTimeInMillis) { + timedOutInFlightStateServiceUnits.put(serviceUnit, stateData); + continue; + } + + + if (!isActiveState(state) && now - stateData.timestamp() > stateTombstoneDelayTimeInMillis) { log.info("Found semi-terminal states to tombstone" + " serviceUnit:{}, stateData:{}", serviceUnit, stateData); tombstoneAsync(serviceUnit).whenComplete((__, e) -> { @@ -1383,43 +1676,27 @@ protected void monitorOwnerships(List brokers) { } } - // Skip cleaning orphan bundles if inactiveBrokers exist. This is a bigger problem. + if (!inactiveBrokers.isEmpty()) { for (String inactiveBroker : inactiveBrokers) { handleBrokerDeletionEvent(inactiveBroker); } - } else if (!orphanServiceUnits.isEmpty()) { - for (var etr : orphanServiceUnits.entrySet()) { + } + + // timedOutInFlightStateServiceUnits are the in-flight ones although their src and dst brokers are known to + // be active. + if (!timedOutInFlightStateServiceUnits.isEmpty()) { + for (var etr : timedOutInFlightStateServiceUnits.entrySet()) { var orphanServiceUnit = etr.getKey(); var orphanData = etr.getValue(); - var overrideData = getOverrideInFlightStateData( - orphanServiceUnit, orphanData, activeBrokers); - if (overrideData.isPresent()) { - log.info("Overriding in-flight state ownership serviceUnit:{} " - + "from orphanData:{} to overrideData:{}", - orphanServiceUnit, orphanData, overrideData); - publishOverrideEventAsync(orphanServiceUnit, orphanData, overrideData.get()) - .whenComplete((__, e) -> { - if (e != null) { - log.error("Failed cleaning the ownership orphanServiceUnit:{}, orphanData:{}, " - + "cleanupErrorCnt:{}.", - orphanServiceUnit, orphanData, - totalCleanupErrorCnt.incrementAndGet() - totalCleanupErrorCntStart, e); - } - }); - orphanServiceUnitCleanupCnt++; - } else { - log.warn("Failed get the overrideStateData from orphanServiceUnit:{}, orphanData:{}," - + " cleanupErrorCnt:{}. will retry..", - orphanServiceUnit, orphanData, - totalCleanupErrorCnt.incrementAndGet() - totalCleanupErrorCntStart); - } + overrideOwnership(orphanServiceUnit, orphanData, null, false); + orphanServiceUnitCleanupCnt++; } } try { - producer.flush(); - } catch (PulsarClientException e) { + tableview.flush(OWNERSHIP_CLEAN_UP_MAX_WAIT_TIME_IN_MILLIS); + } catch (Exception e) { log.error("Failed to flush the in-flight messages.", e); } @@ -1446,7 +1723,7 @@ protected void monitorOwnerships(List brokers) { inactiveBrokers, inactiveBrokers.size(), orphanServiceUnitCleanupCnt, serviceUnitTombstoneCleanupCnt, - totalCleanupErrorCntStart - totalCleanupErrorCnt.get(), + totalCleanupErrorCnt.get() - totalCleanupErrorCntStart, printCleanupMetrics()); } @@ -1479,10 +1756,8 @@ private int getTotalOwnedServiceUnitCnt() { if (lastOwnEventHandledAt > lastOwnedServiceUnitCountAt || now - lastOwnedServiceUnitCountAt > MAX_OWNED_BUNDLE_COUNT_DELAY_TIME_IN_MILLIS) { int cnt = 0; - for (var data : tableview.values()) { - if (data.state() == Owned && isTargetBroker(data.dstBroker())) { - cnt++; - } + for (var e : tableview.ownedServiceUnits()) { + cnt++; } lastOwnedServiceUnitCountAt = now; totalOwnedServiceUnitCnt = cnt; @@ -1622,10 +1897,31 @@ public void listen(StateChangeListener listener) { @Override public Set> getOwnershipEntrySet() { + if (!validateChannelState(Started, true)) { + throw new IllegalStateException("Invalid channel state:" + channelState.name()); + } return tableview.entrySet(); } + @Override + public Set getOwnedServiceUnits() { + if (!validateChannelState(Started, true)) { + throw new IllegalStateException("Invalid channel state:" + channelState.name()); + } + return tableview.ownedServiceUnits(); + } + public static ServiceUnitStateChannel get(PulsarService pulsar) { return ExtensibleLoadManagerImpl.get(pulsar.getLoadManager().get()).getServiceUnitStateChannel(); } + + @VisibleForTesting + protected void disable() { + channelState = Disabled; + } + + @VisibleForTesting + protected void enable() { + channelState = Started; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java index 307d3a4acb175..4a990ddbc9b21 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java @@ -34,7 +34,7 @@ public record ServiceUnitStateData( public ServiceUnitStateData { Objects.requireNonNull(state); - if (StringUtils.isBlank(dstBroker) && StringUtils.isBlank(sourceBroker)) { + if (state != ServiceUnitState.Free && StringUtils.isBlank(dstBroker) && StringUtils.isBlank(sourceBroker)) { throw new IllegalArgumentException("Empty broker"); } } @@ -75,4 +75,19 @@ public ServiceUnitStateData(ServiceUnitState state, String dstBroker, boolean fo public static ServiceUnitState state(ServiceUnitStateData data) { return data == null ? ServiceUnitState.Init : data.state(); } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + ServiceUnitStateData that = (ServiceUnitStateData) o; + + return versionId == that.versionId; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataConflictResolver.java similarity index 74% rename from pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java rename to pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataConflictResolver.java index 72b05b5cd62c8..3e43237f4c00e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataConflictResolver.java @@ -20,22 +20,41 @@ import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.StorageType.MetadataStore; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.StorageType.SystemTopic; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state; import com.google.common.annotations.VisibleForTesting; +import java.util.function.BiConsumer; +import lombok.Setter; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.topics.TopicCompactionStrategy; -public class ServiceUnitStateCompactionStrategy implements TopicCompactionStrategy { +public class ServiceUnitStateDataConflictResolver implements TopicCompactionStrategy { private final Schema schema; + private BiConsumer skippedMsgHandler; private boolean checkBrokers = true; - public ServiceUnitStateCompactionStrategy() { + @Setter + private ServiceUnitState.StorageType storageType = SystemTopic; + + public ServiceUnitStateDataConflictResolver() { schema = Schema.JSON(ServiceUnitStateData.class); } + public void setSkippedMsgHandler(BiConsumer skippedMsgHandler) { + this.skippedMsgHandler = skippedMsgHandler; + } + + @Override + public void handleSkippedMessage(String key, ServiceUnitStateData cur) { + if (skippedMsgHandler != null) { + skippedMsgHandler.accept(key, cur); + } + } + @Override public Schema getSchema() { return schema; @@ -57,8 +76,16 @@ public boolean shouldKeepLeft(ServiceUnitStateData from, ServiceUnitStateData to } else if (from.versionId() >= to.versionId()) { return true; } else if (from.versionId() < to.versionId() - 1) { // Compacted - return false; + // If the system topic is compacted, to.versionId can be bigger than from.versionId by 2 or more. + // e.g. (Owned, v1) -> (Owned, v3) + return storageType != SystemTopic; } // else from.versionId() == to.versionId() - 1 // continue to check further + } else { + // If `from` is null, to.versionId should start at 1 over metadata store. + // In this case, to.versionId can be bigger than 1 over the system topic, if compacted. + if (storageType == MetadataStore) { + return to.versionId() != 1; + } } if (to.force()) { @@ -67,7 +94,7 @@ public boolean shouldKeepLeft(ServiceUnitStateData from, ServiceUnitStateData to ServiceUnitState prevState = state(from); ServiceUnitState state = state(to); - if (!ServiceUnitState.isValidTransition(prevState, state)) { + if (!ServiceUnitState.isValidTransition(prevState, state, storageType)) { return true; } @@ -118,4 +145,4 @@ private boolean invalidUnload(ServiceUnitStateData from, ServiceUnitStateData to || !from.dstBroker().equals(to.sourceBroker()) || from.dstBroker().equals(to.dstBroker()); } -} \ No newline at end of file +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateMetadataStoreTableViewImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateMetadataStoreTableViewImpl.java new file mode 100644 index 0000000000000..f488b31c77415 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateMetadataStoreTableViewImpl.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.loadbalance.extensions.channel; + +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.StorageType.MetadataStore; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.metadata.api.MetadataStoreTableView; +import org.apache.pulsar.metadata.tableview.impl.MetadataStoreTableViewImpl; + +@Slf4j +public class ServiceUnitStateMetadataStoreTableViewImpl extends ServiceUnitStateTableViewBase { + public static final String PATH_PREFIX = "/service_unit_state"; + private static final String VALID_PATH_REG_EX = "^\\/service_unit_state\\/.*\\/0x[0-9a-fA-F]{8}_0x[0-9a-fA-F]{8}$"; + private static final Pattern VALID_PATH_PATTERN; + + static { + try { + VALID_PATH_PATTERN = Pattern.compile(VALID_PATH_REG_EX); + } catch (PatternSyntaxException error) { + log.error("Invalid regular expression {}", VALID_PATH_REG_EX, error); + throw new IllegalArgumentException(error); + } + } + private ServiceUnitStateDataConflictResolver conflictResolver; + private volatile MetadataStoreTableView tableview; + + public void start(PulsarService pulsar, + BiConsumer tailItemListener, + BiConsumer existingItemListener) + throws MetadataStoreException { + init(pulsar); + conflictResolver = new ServiceUnitStateDataConflictResolver(); + conflictResolver.setStorageType(MetadataStore); + tableview = new MetadataStoreTableViewImpl<>(ServiceUnitStateData.class, + pulsar.getBrokerId(), + pulsar.getLocalMetadataStore(), + PATH_PREFIX, + this::resolveConflict, + this::validateServiceUnitPath, + List.of(this::updateOwnedServiceUnits, tailItemListener), + List.of(this::updateOwnedServiceUnits, existingItemListener), + TimeUnit.SECONDS.toMillis(pulsar.getConfiguration().getMetadataStoreOperationTimeoutSeconds()) + ); + tableview.start(); + + } + + protected boolean resolveConflict(ServiceUnitStateData prev, ServiceUnitStateData cur) { + return !conflictResolver.shouldKeepLeft(prev, cur); + } + + + protected boolean validateServiceUnitPath(String path) { + try { + var matcher = VALID_PATH_PATTERN.matcher(path); + return matcher.matches(); + } catch (Exception e) { + return false; + } + } + + + @Override + public void close() throws IOException { + if (tableview != null) { + tableview = null; + log.info("Successfully closed the channel tableview."); + } + } + + private boolean isValidState() { + if (tableview == null) { + return false; + } + return true; + } + + @Override + public ServiceUnitStateData get(String key) { + if (!isValidState()) { + throw new IllegalStateException(INVALID_STATE_ERROR_MSG); + } + return tableview.get(key); + } + + @Override + public CompletableFuture put(String key, @NonNull ServiceUnitStateData value) { + if (!isValidState()) { + return CompletableFuture.failedFuture(new IllegalStateException(INVALID_STATE_ERROR_MSG)); + } + return tableview.put(key, value).exceptionally(e -> { + if (e.getCause() instanceof MetadataStoreTableView.ConflictException) { + return null; + } + throw FutureUtil.wrapToCompletionException(e); + }); + } + + @Override + public void flush(long waitDurationInMillis) { + // no-op + } + + @Override + public CompletableFuture delete(String key) { + if (!isValidState()) { + return CompletableFuture.failedFuture(new IllegalStateException(INVALID_STATE_ERROR_MSG)); + } + return tableview.delete(key).exceptionally(e -> { + if (e.getCause() instanceof MetadataStoreException.NotFoundException) { + return null; + } + throw FutureUtil.wrapToCompletionException(e); + }); + } + + + @Override + public Set> entrySet() { + if (!isValidState()) { + throw new IllegalStateException(INVALID_STATE_ERROR_MSG); + } + return tableview.entrySet(); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableView.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableView.java new file mode 100644 index 0000000000000..5ac57fe5c19c6 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableView.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.channel; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; +import java.util.function.BiConsumer; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.common.naming.NamespaceBundle; + +/** + * Given that the ServiceUnitStateChannel event-sources service unit (bundle) ownership states via a persistent store + * and reacts to ownership changes, the ServiceUnitStateTableView provides an interface to the + * ServiceUnitStateChannel's persistent store and its locally replicated ownership view (tableview) with listener + * registration. It initially populates its local table view by scanning existing items in the remote store. The + * ServiceUnitStateTableView receives notifications whenever ownership states are updated in the remote store, and + * upon notification, it applies the updates to its local tableview with the listener logic. + */ +public interface ServiceUnitStateTableView extends Closeable { + + /** + * Starts the tableview. + * It initially populates its local table view by scanning existing items in the remote store, and it starts + * listening to service unit ownership changes from the remote store. + * @param pulsar pulsar service reference + * @param tailItemListener listener to listen tail(newly updated) items + * @param existingItemListener listener to listen existing items + * @throws IOException if it fails to init the tableview. + */ + void start(PulsarService pulsar, + BiConsumer tailItemListener, + BiConsumer existingItemListener) throws IOException; + + + /** + * Closes the tableview. + * @throws IOException if it fails to close the tableview. + */ + void close() throws IOException; + + /** + * Gets one item from the local tableview. + * @param key the key to get + * @return value if exists. Otherwise, null. + */ + ServiceUnitStateData get(String key); + + /** + * Tries to put the item in the persistent store. + * If it completes, all peer tableviews (including the local one) will be notified and be eventually consistent + * with this put value. + * + * It ignores put operation if the input value conflicts with the existing one in the persistent store. + * + * @param key the key to put + * @param value the value to put + * @return a future to track the completion of the operation + */ + CompletableFuture put(String key, ServiceUnitStateData value); + + /** + * Tries to delete the item from the persistent store. + * All peer tableviews (including the local one) will be notified and be eventually consistent with this deletion. + * + * It ignores delete operation if the key is not present in the persistent store. + * + * @param key the key to delete + * @return a future to track the completion of the operation + */ + CompletableFuture delete(String key); + + /** + * Returns the entry set of the items in the local tableview. + * @return entry set + */ + Set> entrySet(); + + /** + * Returns service units (namespace bundles) owned by this broker. + * @return a set of owned service units (namespace bundles) + */ + Set ownedServiceUnits(); + + /** + * Tries to flush any batched or buffered updates. + * @param waitDurationInMillis time to wait until complete. + * @throws ExecutionException + * @throws InterruptedException + * @throws TimeoutException + */ + void flush(long waitDurationInMillis) throws ExecutionException, InterruptedException, TimeoutException; +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewBase.java new file mode 100644 index 0000000000000..b690ef101e168 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewBase.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.loadbalance.extensions.channel; + +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; +import org.apache.pulsar.broker.namespace.NamespaceService; +import org.apache.pulsar.common.naming.NamespaceBundle; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.metadata.api.MetadataStoreException; + +/** + * ServiceUnitStateTableView base class. + */ +@Slf4j +abstract class ServiceUnitStateTableViewBase implements ServiceUnitStateTableView { + protected static final String INVALID_STATE_ERROR_MSG = "The tableview has not been started."; + private final Map ownedServiceUnitsMap = new ConcurrentHashMap<>(); + private final Set ownedServiceUnits = Collections.unmodifiableSet(ownedServiceUnitsMap.keySet()); + private String brokerId; + private PulsarService pulsar; + protected void init(PulsarService pulsar) throws MetadataStoreException { + this.pulsar = pulsar; + this.brokerId = pulsar.getBrokerId(); + // Add heartbeat and SLA monitor namespace bundle. + NamespaceName heartbeatNamespace = + NamespaceService.getHeartbeatNamespace(brokerId, pulsar.getConfiguration()); + NamespaceName heartbeatNamespaceV2 = NamespaceService + .getHeartbeatNamespaceV2(brokerId, pulsar.getConfiguration()); + NamespaceName slaMonitorNamespace = NamespaceService + .getSLAMonitorNamespace(brokerId, pulsar.getConfiguration()); + try { + pulsar.getNamespaceService().getNamespaceBundleFactory() + .getFullBundleAsync(heartbeatNamespace) + .thenAccept(fullBundle -> ownedServiceUnitsMap.put(fullBundle, true)) + .thenCompose(__ -> pulsar.getNamespaceService().getNamespaceBundleFactory() + .getFullBundleAsync(heartbeatNamespaceV2)) + .thenAccept(fullBundle -> ownedServiceUnitsMap.put(fullBundle, true)) + .thenCompose(__ -> pulsar.getNamespaceService().getNamespaceBundleFactory() + .getFullBundleAsync(slaMonitorNamespace)) + .thenAccept(fullBundle -> ownedServiceUnitsMap.put(fullBundle, true)) + .thenApply(__ -> null).get(pulsar.getConfiguration().getMetadataStoreOperationTimeoutSeconds(), + TimeUnit.SECONDS); + } catch (Exception e) { + throw new MetadataStoreException(e); + } + } + + @Override + public Set ownedServiceUnits() { + return ownedServiceUnits; + } + + protected void updateOwnedServiceUnits(String key, ServiceUnitStateData val) { + NamespaceBundle namespaceBundle = LoadManagerShared.getNamespaceBundle(pulsar, key); + var state = ServiceUnitStateData.state(val); + ownedServiceUnitsMap.compute(namespaceBundle, (k, v) -> { + if (state == Owned && brokerId.equals(val.dstBroker())) { + return true; + } else if (state == Splitting && brokerId.equals(val.sourceBroker())) { + return true; + } else { + return null; + } + }); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewImpl.java new file mode 100644 index 0000000000000..12cf87445a3dd --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewImpl.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.loadbalance.extensions.channel; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.apache.pulsar.common.naming.NamespaceName.SYSTEM_NAMESPACE; +import java.io.IOException; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; +import java.util.function.BiConsumer; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.client.api.CompressionType; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.TableView; +import org.apache.pulsar.common.naming.TopicDomain; +import org.apache.pulsar.common.naming.TopicName; + +@Slf4j +public class ServiceUnitStateTableViewImpl extends ServiceUnitStateTableViewBase { + + public static final String TOPIC = TopicName.get( + TopicDomain.persistent.value(), + SYSTEM_NAMESPACE, + "loadbalancer-service-unit-state").toString(); + private static final int MAX_OUTSTANDING_PUB_MESSAGES = 500; + public static final CompressionType MSG_COMPRESSION_TYPE = CompressionType.ZSTD; + private volatile Producer producer; + private volatile TableView tableview; + + public void start(PulsarService pulsar, + BiConsumer tailItemListener, + BiConsumer existingItemListener) throws IOException { + boolean debug = ExtensibleLoadManagerImpl.debug(pulsar.getConfiguration(), log); + + init(pulsar); + + var schema = Schema.JSON(ServiceUnitStateData.class); + + ExtensibleLoadManagerImpl.createSystemTopic(pulsar, TOPIC); + + if (producer != null) { + producer.close(); + if (debug) { + log.info("Closed the channel producer."); + } + } + + producer = pulsar.getClient().newProducer(schema) + .enableBatching(true) + .compressionType(MSG_COMPRESSION_TYPE) + .maxPendingMessages(MAX_OUTSTANDING_PUB_MESSAGES) + .blockIfQueueFull(true) + .topic(TOPIC) + .create(); + + if (debug) { + log.info("Successfully started the channel producer."); + } + + if (tableview != null) { + tableview.close(); + if (debug) { + log.info("Closed the channel tableview."); + } + } + + tableview = pulsar.getClient().newTableViewBuilder(schema) + .topic(TOPIC) + .loadConf(Map.of( + "topicCompactionStrategyClassName", + ServiceUnitStateDataConflictResolver.class.getName())) + .create(); + tableview.listen(this::updateOwnedServiceUnits); + tableview.listen(tailItemListener); + tableview.forEach(this::updateOwnedServiceUnits); + tableview.forEach(existingItemListener); + + } + + private boolean isValidState() { + if (tableview == null || producer == null) { + return false; + } + return true; + } + + + @Override + public void close() throws IOException { + + if (tableview != null) { + tableview.close(); + tableview = null; + log.info("Successfully closed the channel tableview."); + } + + if (producer != null) { + producer.close(); + producer = null; + log.info("Successfully closed the channel producer."); + } + } + + @Override + public ServiceUnitStateData get(String key) { + if (!isValidState()) { + throw new IllegalStateException(INVALID_STATE_ERROR_MSG); + } + return tableview.get(key); + } + + @Override + public CompletableFuture put(String key, ServiceUnitStateData value) { + if (!isValidState()) { + return CompletableFuture.failedFuture(new IllegalStateException(INVALID_STATE_ERROR_MSG)); + } + CompletableFuture future = new CompletableFuture<>(); + producer.newMessage() + .key(key) + .value(value) + .sendAsync() + .whenComplete((messageId, e) -> { + if (e != null) { + if (e instanceof PulsarClientException.AlreadyClosedException) { + log.info("Skip publishing the message since the producer is closed, serviceUnit: {}, data: " + + "{}", key, value); + } else { + log.error("Failed to publish the message: serviceUnit:{}, data:{}", + key, value, e); + } + future.completeExceptionally(e); + } else { + future.complete(null); + } + }); + return future; + } + + @Override + public void flush(long waitDurationInMillis) throws InterruptedException, TimeoutException, ExecutionException { + if (!isValidState()) { + throw new IllegalStateException(INVALID_STATE_ERROR_MSG); + } + final var deadline = System.currentTimeMillis() + waitDurationInMillis; + var waitTimeMs = waitDurationInMillis; + producer.flushAsync().get(waitTimeMs, MILLISECONDS); + waitTimeMs = deadline - System.currentTimeMillis(); + if (waitTimeMs < 0) { + waitTimeMs = 0; + } + tableview.refreshAsync().get(waitTimeMs, MILLISECONDS); + } + + @Override + public CompletableFuture delete(String key) { + return put(key, null); + } + + @Override + public Set> entrySet() { + if (!isValidState()) { + throw new IllegalStateException(INVALID_STATE_ERROR_MSG); + } + return tableview.entrySet(); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewSyncer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewSyncer.java new file mode 100644 index 0000000000000..10ab39a66d279 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewSyncer.java @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.loadbalance.extensions.channel; + +import static org.apache.pulsar.broker.ServiceConfiguration.ServiceUnitTableViewSyncerType.SystemTopicToMetadataStoreSyncer; +import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.COMPACTION_THRESHOLD; +import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.configureSystemTopics; +import com.fasterxml.jackson.core.JsonProcessingException; +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.ObjectMapperFactory; + +/** + * Defines ServiceUnitTableViewSyncer. + * It syncs service unit(bundle) states between metadata store and system topic table views. + * One could enable this syncer before migration from one to the other and disable it after the migration finishes. + */ +@Slf4j +public class ServiceUnitStateTableViewSyncer implements Closeable { + private static final int MAX_CONCURRENT_SYNC_COUNT = 100; + private static final int SYNC_WAIT_TIME_IN_SECS = 300; + private PulsarService pulsar; + private volatile ServiceUnitStateTableView systemTopicTableView; + private volatile ServiceUnitStateTableView metadataStoreTableView; + private volatile boolean isActive = false; + + + public void start(PulsarService pulsar) + throws IOException, TimeoutException, InterruptedException, ExecutionException { + if (!pulsar.getConfiguration().isLoadBalancerServiceUnitTableViewSyncerEnabled()) { + return; + } + + if (isActive) { + return; + } + this.pulsar = pulsar; + + try { + + syncExistingItems(); + // disable compaction + if (!configureSystemTopics(pulsar, 0)) { + throw new IllegalStateException("Failed to disable compaction"); + } + syncTailItems(); + + isActive = true; + + } catch (Throwable e) { + log.error("Failed to start ServiceUnitStateTableViewSyncer", e); + throw e; + } + } + + private CompletableFuture syncToSystemTopic(String key, ServiceUnitStateData data) { + return systemTopicTableView.put(key, data); + } + + private CompletableFuture syncToMetadataStore(String key, ServiceUnitStateData data) { + return metadataStoreTableView.put(key, data); + } + + private void dummy(String key, ServiceUnitStateData data) { + } + + private void syncExistingItems() + throws IOException, ExecutionException, InterruptedException, TimeoutException { + long started = System.currentTimeMillis(); + @Cleanup + ServiceUnitStateTableView metadataStoreTableView = new ServiceUnitStateMetadataStoreTableViewImpl(); + metadataStoreTableView.start( + pulsar, + this::dummy, + this::dummy + ); + + @Cleanup + ServiceUnitStateTableView systemTopicTableView = new ServiceUnitStateTableViewImpl(); + systemTopicTableView.start( + pulsar, + this::dummy, + this::dummy + ); + + + var syncer = pulsar.getConfiguration().getLoadBalancerServiceUnitTableViewSyncer(); + if (syncer == SystemTopicToMetadataStoreSyncer) { + clean(metadataStoreTableView); + syncExistingItemsToMetadataStore(systemTopicTableView); + } else { + clean(systemTopicTableView); + syncExistingItemsToSystemTopic(metadataStoreTableView, systemTopicTableView); + } + + if (!waitUntilSynced(metadataStoreTableView, systemTopicTableView, started)) { + throw new TimeoutException( + syncer + " failed to sync existing items in tableviews. MetadataStoreTableView.size: " + + metadataStoreTableView.entrySet().size() + + ", SystemTopicTableView.size: " + systemTopicTableView.entrySet().size() + " in " + + SYNC_WAIT_TIME_IN_SECS + " secs"); + } + + log.info("Synced existing items MetadataStoreTableView.size:{} , " + + "SystemTopicTableView.size: {} in {} secs", + metadataStoreTableView.entrySet().size(), systemTopicTableView.entrySet().size(), + TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - started)); + } + + private void syncTailItems() throws InterruptedException, IOException, TimeoutException { + long started = System.currentTimeMillis(); + + if (metadataStoreTableView != null) { + metadataStoreTableView.close(); + metadataStoreTableView = null; + } + + if (systemTopicTableView != null) { + systemTopicTableView.close(); + systemTopicTableView = null; + } + + this.metadataStoreTableView = new ServiceUnitStateMetadataStoreTableViewImpl(); + this.metadataStoreTableView.start( + pulsar, + this::syncToSystemTopic, + this::dummy + ); + log.info("Started MetadataStoreTableView"); + + this.systemTopicTableView = new ServiceUnitStateTableViewImpl(); + this.systemTopicTableView.start( + pulsar, + this::syncToMetadataStore, + this::dummy + ); + log.info("Started SystemTopicTableView"); + + var syncer = pulsar.getConfiguration().getLoadBalancerServiceUnitTableViewSyncer(); + if (!waitUntilSynced(metadataStoreTableView, systemTopicTableView, started)) { + throw new TimeoutException( + syncer + " failed to sync tableviews. MetadataStoreTableView.size: " + + metadataStoreTableView.entrySet().size() + + ", SystemTopicTableView.size: " + systemTopicTableView.entrySet().size() + " in " + + SYNC_WAIT_TIME_IN_SECS + " secs"); + } + + + log.info("Successfully started ServiceUnitStateTableViewSyncer MetadataStoreTableView.size:{} , " + + "SystemTopicTableView.size: {} in {} secs", + metadataStoreTableView.entrySet().size(), systemTopicTableView.entrySet().size(), + TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - started)); + } + + private void syncExistingItemsToMetadataStore(ServiceUnitStateTableView src) + throws JsonProcessingException, ExecutionException, InterruptedException, TimeoutException { + // Directly use store to sync existing items to metadataStoreTableView(otherwise, they are conflicted out) + var store = pulsar.getLocalMetadataStore(); + var writer = ObjectMapperFactory.getMapper().writer(); + var opTimeout = pulsar.getConfiguration().getMetadataStoreOperationTimeoutSeconds(); + List> futures = new ArrayList<>(); + var srcIter = src.entrySet().iterator(); + while (srcIter.hasNext()) { + var e = srcIter.next(); + futures.add(store.put(ServiceUnitStateMetadataStoreTableViewImpl.PATH_PREFIX + "/" + e.getKey(), + writer.writeValueAsBytes(e.getValue()), Optional.empty()).thenApply(__ -> null)); + if (futures.size() == MAX_CONCURRENT_SYNC_COUNT || !srcIter.hasNext()) { + FutureUtil.waitForAll(futures).get(opTimeout, TimeUnit.SECONDS); + } + } + } + + private void syncExistingItemsToSystemTopic(ServiceUnitStateTableView src, + ServiceUnitStateTableView dst) + throws ExecutionException, InterruptedException, TimeoutException { + var opTimeout = pulsar.getConfiguration().getMetadataStoreOperationTimeoutSeconds(); + List> futures = new ArrayList<>(); + var srcIter = src.entrySet().iterator(); + while (srcIter.hasNext()) { + var e = srcIter.next(); + futures.add(dst.put(e.getKey(), e.getValue())); + if (futures.size() == MAX_CONCURRENT_SYNC_COUNT || !srcIter.hasNext()) { + FutureUtil.waitForAll(futures).get(opTimeout, TimeUnit.SECONDS); + } + } + } + + private void clean(ServiceUnitStateTableView dst) + throws ExecutionException, InterruptedException, TimeoutException { + var opTimeout = pulsar.getConfiguration().getMetadataStoreOperationTimeoutSeconds(); + var dstIter = dst.entrySet().iterator(); + List> futures = new ArrayList<>(); + while (dstIter.hasNext()) { + var e = dstIter.next(); + futures.add(dst.delete(e.getKey())); + if (futures.size() == MAX_CONCURRENT_SYNC_COUNT || !dstIter.hasNext()) { + FutureUtil.waitForAll(futures).get(opTimeout, TimeUnit.SECONDS); + } + } + } + + private boolean waitUntilSynced(ServiceUnitStateTableView srt, ServiceUnitStateTableView dst, long started) + throws InterruptedException { + while (srt.entrySet().size() != dst.entrySet().size()) { + if (TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - started) + > SYNC_WAIT_TIME_IN_SECS) { + return false; + } + Thread.sleep(100); + } + return true; + } + + @Override + public void close() throws IOException { + if (!isActive) { + return; + } + + if (!configureSystemTopics(pulsar, COMPACTION_THRESHOLD)) { + throw new IllegalStateException("Failed to enable compaction"); + } + + try { + if (systemTopicTableView != null) { + systemTopicTableView.close(); + systemTopicTableView = null; + log.info("Closed SystemTopicTableView"); + } + } catch (Exception e) { + log.error("Failed to close SystemTopicTableView", e); + throw e; + } + + try { + if (metadataStoreTableView != null) { + metadataStoreTableView.close(); + metadataStoreTableView = null; + log.info("Closed MetadataStoreTableView"); + } + } catch (Exception e) { + log.error("Failed to close MetadataStoreTableView", e); + throw e; + } + + log.info("Successfully closed ServiceUnitStateTableViewSyncer."); + isActive = false; + } + + public boolean isActive() { + return isActive; + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/StateChangeListeners.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/StateChangeListeners.java index 1d396f500b648..f0d99e931bf4c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/StateChangeListeners.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/StateChangeListeners.java @@ -51,7 +51,21 @@ public void close() { public CompletableFuture notifyOnCompletion(CompletableFuture future, String serviceUnit, ServiceUnitStateData data) { - return future.whenComplete((r, ex) -> notify(serviceUnit, data, ex)); + return notifyOnArrival(serviceUnit, data). + thenCombine(future, (unused, t) -> t). + whenComplete((r, ex) -> notify(serviceUnit, data, ex)); + } + + private CompletableFuture notifyOnArrival(String serviceUnit, ServiceUnitStateData data) { + stateChangeListeners.forEach(listener -> { + try { + listener.beforeEvent(serviceUnit, data); + } catch (Throwable ex) { + log.error("StateChangeListener: {} exception while notifying arrival event {} for service unit {}", + listener, data, serviceUnit, ex); + } + }); + return CompletableFuture.completedFuture(null); } public void notify(String serviceUnit, ServiceUnitStateData data, Throwable t) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java index 48665d39a0d3e..95d89932ed96d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java @@ -56,8 +56,8 @@ public class BrokerLoadData { private double msgThroughputOut; // bytes/sec private double msgRateIn; // messages/sec private double msgRateOut; // messages/sec - private int bundleCount; - private int topics; + private long bundleCount; + private long topics; // Load data features computed from the above resources. private double maxResourceUsage; // max of resource usages @@ -69,8 +69,8 @@ public class BrokerLoadData { * loadBalancerCPUResourceWeight, * loadBalancerMemoryResourceWeight, * loadBalancerDirectMemoryResourceWeight, - * loadBalancerBandwithInResourceWeight, and - * loadBalancerBandwithOutResourceWeight. + * loadBalancerBandwidthInResourceWeight, and + * loadBalancerBandwidthOutResourceWeight. * * The historical resource percentage is configured by loadBalancerHistoryResourcePercentage. */ @@ -115,8 +115,8 @@ public void update(final SystemResourceUsage usage, double msgThroughputOut, double msgRateIn, double msgRateOut, - int bundleCount, - int topics, + long bundleCount, + long topics, ServiceConfiguration conf) { updateSystemResourceUsage(usage.cpu, usage.memory, usage.directMemory, usage.bandwidthIn, usage.bandwidthOut); this.msgThroughputIn = msgThroughputIn; @@ -186,8 +186,8 @@ private void updateWeightedMaxEMA(ServiceConfiguration conf) { var weightedMax = getMaxResourceUsageWithWeight( conf.getLoadBalancerCPUResourceWeight(), conf.getLoadBalancerMemoryResourceWeight(), conf.getLoadBalancerDirectMemoryResourceWeight(), - conf.getLoadBalancerBandwithInResourceWeight(), - conf.getLoadBalancerBandwithOutResourceWeight()); + conf.getLoadBalancerBandwidthInResourceWeight(), + conf.getLoadBalancerBandwidthOutResourceWeight()); weightedMaxEMA = updatedAt == 0 ? weightedMax : weightedMaxEMA * historyPercentage + (1 - historyPercentage) * weightedMax; } @@ -220,9 +220,9 @@ public void clear() { public String toString(ServiceConfiguration conf) { return String.format("cpu= %.2f%%, memory= %.2f%%, directMemory= %.2f%%, " - + "bandwithIn= %.2f%%, bandwithOut= %.2f%%, " + + "bandwidthIn= %.2f%%, bandwidthOut= %.2f%%, " + "cpuWeight= %f, memoryWeight= %f, directMemoryWeight= %f, " - + "bandwithInResourceWeight= %f, bandwithOutResourceWeight= %f, " + + "bandwidthInResourceWeight= %f, bandwidthOutResourceWeight= %f, " + "msgThroughputIn= %.2f, msgThroughputOut= %.2f, msgRateIn= %.2f, msgRateOut= %.2f, " + "bundleCount= %d, " + "maxResourceUsage= %.2f%%, weightedMaxEMA= %.2f%%, msgThroughputEMA= %.2f, " @@ -233,8 +233,8 @@ public String toString(ServiceConfiguration conf) { conf.getLoadBalancerCPUResourceWeight(), conf.getLoadBalancerMemoryResourceWeight(), conf.getLoadBalancerDirectMemoryResourceWeight(), - conf.getLoadBalancerBandwithInResourceWeight(), - conf.getLoadBalancerBandwithOutResourceWeight(), + conf.getLoadBalancerBandwidthInResourceWeight(), + conf.getLoadBalancerBandwidthOutResourceWeight(), msgThroughputIn, msgThroughputOut, msgRateIn, msgRateOut, bundleCount, maxResourceUsage * 100, weightedMaxEMA * 100, msgThroughputEMA, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java index 41f5b18e321e8..5d982076bd609 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java @@ -18,9 +18,12 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.data; +import java.net.URI; import java.util.Map; import java.util.Optional; +import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.lookup.LookupResult; +import org.apache.pulsar.broker.namespace.LookupOptions; import org.apache.pulsar.broker.namespace.NamespaceEphemeralData; import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener; import org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData; @@ -38,7 +41,8 @@ public record BrokerLookupData (String webServiceUrl, boolean nonPersistentTopicsEnabled, String loadManagerClassName, long startTimestamp, - String brokerVersion) implements ServiceLookupData { + String brokerVersion, + Map properties) implements ServiceLookupData { @Override public String getWebServiceUrl() { return this.webServiceUrl(); @@ -79,7 +83,19 @@ public long getStartTimestamp() { return this.startTimestamp; } - public LookupResult toLookupResult() { + public LookupResult toLookupResult(LookupOptions options) throws PulsarServerException { + if (options.hasAdvertisedListenerName()) { + AdvertisedListener listener = advertisedListeners.get(options.getAdvertisedListenerName()); + if (listener == null) { + throw new PulsarServerException("the broker do not have " + + options.getAdvertisedListenerName() + " listener"); + } + URI url = listener.getBrokerServiceUrl(); + URI urlTls = listener.getBrokerServiceUrlTls(); + return new LookupResult(webServiceUrl, webServiceUrlTls, + url == null ? null : url.toString(), + urlTls == null ? null : urlTls.toString(), LookupResult.Type.BrokerUrl, false); + } return new LookupResult(webServiceUrl, webServiceUrlTls, pulsarServiceUrl, pulsarServiceUrlTls, LookupResult.Type.BrokerUrl, false); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java index 462f8f0e3597a..37b35d8661dd8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.loadbalance.extensions.filter; import java.util.Map; +import java.util.concurrent.CompletableFuture; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper; @@ -38,10 +39,9 @@ public AntiAffinityGroupPolicyFilter(AntiAffinityGroupPolicyHelper helper) { } @Override - public Map filter( + public CompletableFuture> filterAsync( Map brokers, ServiceUnitId serviceUnitId, LoadManagerContext context) { - helper.filter(brokers, serviceUnitId.toString()); - return brokers; + return helper.filterAsync(brokers, serviceUnitId.toString()); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java index d9cbfdc391ed4..2950a0133899c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.loadbalance.extensions.filter; import java.util.Map; +import java.util.concurrent.CompletableFuture; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; @@ -42,9 +43,23 @@ public interface BrokerFilter { * @param context The load manager context. * @return Filtered broker list. */ - Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) - throws BrokerFilterException; + @Deprecated + default Map filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) throws BrokerFilterException { + return filterAsync(brokers, serviceUnit, context).join(); + } + + /** + * Filter out async unqualified brokers based on implementation. + * + * @param brokers The full broker and lookup data. + * @param serviceUnit The current serviceUnit. + * @param context The load manager context. + * @return Filtered broker list. + */ + CompletableFuture> filterAsync(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java index eeb0d9d3a3309..306c4c36f4880 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java @@ -19,9 +19,8 @@ package org.apache.pulsar.broker.loadbalance.extensions.filter; import java.util.Map; -import java.util.Set; +import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper; @@ -45,13 +44,13 @@ public String name() { } @Override - public Map filter(Map availableBrokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) - throws BrokerFilterException { - Set brokerCandidateCache = - isolationPoliciesHelper.applyIsolationPolicies(availableBrokers, serviceUnit); - availableBrokers.keySet().retainAll(brokerCandidateCache); - return availableBrokers; + public CompletableFuture> filterAsync(Map availableBrokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { + return isolationPoliciesHelper.applyIsolationPoliciesAsync(availableBrokers, serviceUnit) + .thenApply(brokerCandidateCache -> { + availableBrokers.keySet().retainAll(brokerCandidateCache); + return availableBrokers; + }); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java index 07109b277ae98..54d2a555e6103 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java @@ -20,7 +20,7 @@ import java.util.Map; import java.util.Objects; -import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import java.util.concurrent.CompletableFuture; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.common.naming.ServiceUnitId; @@ -34,13 +34,12 @@ public String name() { } @Override - public Map filter( + public CompletableFuture> filterAsync( Map brokers, ServiceUnitId serviceUnit, - LoadManagerContext context) - throws BrokerFilterException { + LoadManagerContext context) { if (brokers.isEmpty()) { - return brokers; + return CompletableFuture.completedFuture(brokers); } brokers.entrySet().removeIf(entry -> { BrokerLookupData v = entry.getValue(); @@ -48,6 +47,6 @@ public Map filter( return !Objects.equals(v.getLoadManagerClassName(), context.brokerConfiguration().getLoadManagerClassName()); }); - return brokers; + return CompletableFuture.completedFuture(brokers); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java index 0bceae36bb8c2..9863d05ee751e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java @@ -20,7 +20,7 @@ import java.util.Map; import java.util.Optional; -import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import java.util.concurrent.CompletableFuture; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; @@ -36,16 +36,21 @@ public String name() { } @Override - public Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) throws BrokerFilterException { + public CompletableFuture> filterAsync(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { int loadBalancerBrokerMaxTopics = context.brokerConfiguration().getLoadBalancerBrokerMaxTopics(); brokers.keySet().removeIf(broker -> { - Optional brokerLoadDataOpt = context.brokerLoadDataStore().get(broker); - long topics = brokerLoadDataOpt.map(BrokerLoadData::getTopics).orElse(0); + final Optional brokerLoadDataOpt; + try { + brokerLoadDataOpt = context.brokerLoadDataStore().get(broker); + } catch (IllegalStateException ignored) { + return false; + } + long topics = brokerLoadDataOpt.map(BrokerLoadData::getTopics).orElse(0L); // TODO: The broker load data might be delayed, so the max topic check might not accurate. return topics >= loadBalancerBrokerMaxTopics; }); - return brokers; + return CompletableFuture.completedFuture(brokers); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java index 7420fcc211309..1af39a6adb5fe 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java @@ -21,13 +21,14 @@ import com.github.zafarkhaja.semver.Version; import java.util.Iterator; import java.util.Map; +import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.BrokerFilterBadVersionException; -import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.common.naming.ServiceUnitId; +import org.apache.pulsar.common.util.FutureUtil; /** * Filter by broker version. @@ -46,13 +47,12 @@ public class BrokerVersionFilter implements BrokerFilter { * */ @Override - public Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) - throws BrokerFilterException { + public CompletableFuture> filterAsync(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { ServiceConfiguration conf = context.brokerConfiguration(); if (!conf.isPreferLaterVersions() || brokers.isEmpty()) { - return brokers; + return CompletableFuture.completedFuture(brokers); } Version latestVersion; @@ -63,7 +63,8 @@ public Map filter(Map broker } } catch (Exception ex) { log.warn("Disabling PreferLaterVersions feature; reason: " + ex.getMessage()); - throw new BrokerFilterBadVersionException("Cannot determine newest broker version: " + ex.getMessage()); + return FutureUtil.failedFuture( + new BrokerFilterBadVersionException("Cannot determine newest broker version: " + ex.getMessage())); } int numBrokersLatestVersion = 0; @@ -88,7 +89,7 @@ public Map filter(Map broker if (numBrokersOlderVersion == 0) { log.info("All {} brokers are running the latest version [{}]", numBrokersLatestVersion, latestVersion); } - return brokers; + return CompletableFuture.completedFuture(brokers); } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManager.java index 71ebbc92a87db..ac21e4c624163 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManager.java @@ -97,7 +97,7 @@ public void handleEvent(String serviceUnit, ServiceUnitStateData data, Throwable return; } switch (state) { - case Deleted, Owned, Init -> this.complete(serviceUnit, t); + case Init -> this.complete(serviceUnit, t); default -> { if (log.isDebugEnabled()) { log.debug("Handling {} for service unit {}", data, serviceUnit); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/StateChangeListener.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/StateChangeListener.java index 7ba8be8771b91..0d26859f82ef6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/StateChangeListener.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/StateChangeListener.java @@ -23,7 +23,15 @@ public interface StateChangeListener { /** - * Handle the service unit state change. + * Called before the state change is handled. + * + * @param serviceUnit - Service Unit(Namespace bundle). + * @param data - Service unit state data. + */ + default void beforeEvent(String serviceUnit, ServiceUnitStateData data) { } + + /** + * Called after the service unit state change has been handled. * * @param serviceUnit - Service Unit(Namespace bundle). * @param data - Service unit state data. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java index 2dde0c4708e41..42fd2fc8473d7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java @@ -18,13 +18,20 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.manager; +import static com.google.common.base.Preconditions.checkArgument; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigning; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown; +import com.google.common.annotations.VisibleForTesting; +import io.prometheus.client.Histogram; import java.util.Map; +import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter; @@ -38,13 +45,77 @@ public class UnloadManager implements StateChangeListener { private final UnloadCounter counter; private final Map> inFlightUnloadRequest; + private final String brokerId; - public UnloadManager(UnloadCounter counter) { + @VisibleForTesting + public enum LatencyMetric { + UNLOAD(buildHistogram( + "brk_lb_unload_latency", "Total time duration of unload operations on source brokers"), true, false), + ASSIGN(buildHistogram( + "brk_lb_assign_latency", "Time spent in the load balancing ASSIGN state on destination brokers"), + false, true), + RELEASE(buildHistogram( + "brk_lb_release_latency", "Time spent in the load balancing RELEASE state on source brokers"), true, false), + DISCONNECT(buildHistogram( + "brk_lb_disconnect_latency", "Time spent in the load balancing disconnected state on source brokers"), + true, false); + + private static Histogram buildHistogram(String name, String help) { + return Histogram.build(name, help).unit("ms").labelNames("broker", "metric"). + buckets(new double[] {1.0, 10.0, 100.0, 200.0, 1000.0}).register(); + } + private static final long OP_TIMEOUT_NS = TimeUnit.HOURS.toNanos(1); + + private final Histogram histogram; + private final Map> futures = new ConcurrentHashMap<>(); + private final boolean isSourceBrokerMetric; + private final boolean isDestinationBrokerMetric; + + LatencyMetric(Histogram histogram, boolean isSourceBrokerMetric, boolean isDestinationBrokerMetric) { + this.histogram = histogram; + this.isSourceBrokerMetric = isSourceBrokerMetric; + this.isDestinationBrokerMetric = isDestinationBrokerMetric; + } + + public void beginMeasurement(String serviceUnit, String brokerId, ServiceUnitStateData data) { + if ((isSourceBrokerMetric && brokerId.equals(data.sourceBroker())) + || (isDestinationBrokerMetric && brokerId.equals(data.dstBroker()))) { + var startTimeNs = System.nanoTime(); + futures.computeIfAbsent(serviceUnit, ignore -> { + var future = new CompletableFuture(); + future.completeOnTimeout(null, OP_TIMEOUT_NS, TimeUnit.NANOSECONDS). + thenAccept(__ -> { + var durationMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNs); + log.info("Operation {} for service unit {} took {} ms", this, serviceUnit, durationMs); + histogram.labels(brokerId, "bundleUnloading").observe(durationMs); + }).whenComplete((__, throwable) -> futures.remove(serviceUnit, future)); + return future; + }); + } + } + + public void endMeasurement(String serviceUnit) { + var future = futures.get(serviceUnit); + if (future != null) { + future.complete(null); + } + } + } + + public UnloadManager(UnloadCounter counter, String brokerId) { this.counter = counter; - this.inFlightUnloadRequest = new ConcurrentHashMap<>(); + this.brokerId = Objects.requireNonNull(brokerId); + inFlightUnloadRequest = new ConcurrentHashMap<>(); } private void complete(String serviceUnit, Throwable ex) { + LatencyMetric.UNLOAD.endMeasurement(serviceUnit); + LatencyMetric.DISCONNECT.endMeasurement(serviceUnit); + if (ex != null) { + LatencyMetric.RELEASE.endMeasurement(serviceUnit); + LatencyMetric.ASSIGN.endMeasurement(serviceUnit); + } + inFlightUnloadRequest.computeIfPresent(serviceUnit, (__, future) -> { if (!future.isDone()) { if (ex != null) { @@ -62,7 +133,6 @@ public CompletableFuture waitAsync(CompletableFuture eventPubFuture, UnloadDecision decision, long timeout, TimeUnit timeoutUnit) { - return eventPubFuture.thenCompose(__ -> inFlightUnloadRequest.computeIfAbsent(bundle, ignore -> { if (log.isDebugEnabled()) { log.debug("Handle unload bundle: {}, timeout: {} {}", bundle, timeout, timeoutUnit); @@ -86,16 +156,58 @@ public CompletableFuture waitAsync(CompletableFuture eventPubFuture, }); } + @Override + public void beforeEvent(String serviceUnit, ServiceUnitStateData data) { + if (log.isDebugEnabled()) { + log.debug("Handling arrival of {} for service unit {}", data, serviceUnit); + } + ServiceUnitState state = ServiceUnitStateData.state(data); + switch (state) { + case Free, Owned -> LatencyMetric.DISCONNECT.beginMeasurement(serviceUnit, brokerId, data); + case Releasing -> { + LatencyMetric.RELEASE.beginMeasurement(serviceUnit, brokerId, data); + LatencyMetric.UNLOAD.beginMeasurement(serviceUnit, brokerId, data); + } + case Assigning -> LatencyMetric.ASSIGN.beginMeasurement(serviceUnit, brokerId, data); + } + } + @Override public void handleEvent(String serviceUnit, ServiceUnitStateData data, Throwable t) { ServiceUnitState state = ServiceUnitStateData.state(data); + + if ((state == Owned || state == Assigning) && StringUtils.isBlank(data.sourceBroker())) { + if (log.isDebugEnabled()) { + log.debug("Skipping {} for service unit {} from the assignment command.", data, serviceUnit); + } + return; + } + + if (t != null) { + if (log.isDebugEnabled()) { + log.debug("Handling {} for service unit {} with exception.", data, serviceUnit, t); + } + complete(serviceUnit, t); + return; + } + + if (log.isDebugEnabled()) { + log.debug("Handling {} for service unit {}", data, serviceUnit); + } + switch (state) { - case Free, Owned -> this.complete(serviceUnit, t); - default -> { - if (log.isDebugEnabled()) { - log.debug("Handling {} for service unit {}", data, serviceUnit); + case Free -> { + if (!data.force()) { + complete(serviceUnit, t); } } + case Init -> { + checkArgument(data == null, "Init state must be associated with null data"); + complete(serviceUnit, t); + } + case Owned -> complete(serviceUnit, t); + case Releasing -> LatencyMetric.RELEASE.endMeasurement(serviceUnit); + case Assigning -> LatencyMetric.ASSIGN.endMeasurement(serviceUnit); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java index 2f5c32197c1fd..ec26521af41f5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java @@ -30,6 +30,8 @@ import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; +import org.apache.pulsar.broker.namespace.NamespaceService; +import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; @@ -70,7 +72,8 @@ public void update(Map bundleStats, int topk) { pulsar.getConfiguration().isLoadBalancerSheddingBundlesWithPoliciesEnabled(); for (var etr : bundleStats.entrySet()) { String bundle = etr.getKey(); - if (bundle.startsWith(NamespaceName.SYSTEM_NAMESPACE.toString())) { + // TODO: do not filter system topic while shedding + if (NamespaceService.isSystemServiceNamespace(NamespaceBundle.getBundleNamespace(bundle))) { continue; } if (!isLoadBalancerSheddingBundlesWithPoliciesEnabled && hasPolicies(bundle)) { @@ -96,7 +99,7 @@ public void update(Map bundleStats, int topk) { } } - static void partitionSort(List> arr, int k) { + public static void partitionSort(List> arr, int k) { int start = 0; int end = arr.size() - 1; int target = k - 1; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java index 4a5d41f7576ba..72be586e465a1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java @@ -108,7 +108,6 @@ public void updateUnloadBrokerCount(int unloadBrokerCount) { } public List toMetrics(String advertisedBrokerAddress) { - var metrics = new ArrayList(); var dimensions = new HashMap(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java index 44360bc77d83f..3781c9c95f6a7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java @@ -20,6 +20,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; @@ -41,11 +42,11 @@ public AntiAffinityGroupPolicyHelper(PulsarService pulsar, this.channel = channel; } - public void filter( - Map brokers, String bundle) { - LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar, bundle, - brokers.keySet(), - channel.getOwnershipEntrySet(), brokerToFailureDomainMap); + public CompletableFuture> filterAsync(Map brokers, + String bundle) { + return LoadManagerShared.filterAntiAffinityGroupOwnedBrokersAsync(pulsar, bundle, + brokers.keySet(), channel.getOwnershipEntrySet(), brokerToFailureDomainMap) + .thenApply(__ -> brokers); } public boolean hasAntiAffinityGroupPolicy(String bundle) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java index 67dc702cc0c9f..56238d6528e60 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java @@ -18,10 +18,9 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.policies; -import io.netty.util.concurrent.FastThreadLocal; -import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; @@ -38,32 +37,22 @@ public IsolationPoliciesHelper(SimpleResourceAllocationPolicies policies) { this.policies = policies; } - private static final FastThreadLocal> localBrokerCandidateCache = new FastThreadLocal<>() { - @Override - protected Set initialValue() { - return new HashSet<>(); - } - }; - - public Set applyIsolationPolicies(Map availableBrokers, - ServiceUnitId serviceUnit) { - Set brokerCandidateCache = localBrokerCandidateCache.get(); - brokerCandidateCache.clear(); - LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, + public CompletableFuture> applyIsolationPoliciesAsync(Map availableBrokers, + ServiceUnitId serviceUnit) { + return LoadManagerShared.applyNamespacePoliciesAsync(serviceUnit, policies, availableBrokers.keySet(), new LoadManagerShared.BrokerTopicLoadingPredicate() { @Override - public boolean isEnablePersistentTopics(String brokerUrl) { - BrokerLookupData lookupData = availableBrokers.get(brokerUrl.replace("http://", "")); + public boolean isEnablePersistentTopics(String brokerId) { + BrokerLookupData lookupData = availableBrokers.get(brokerId); return lookupData != null && lookupData.persistentTopicsEnabled(); } @Override - public boolean isEnableNonPersistentTopics(String brokerUrl) { - BrokerLookupData lookupData = availableBrokers.get(brokerUrl.replace("http://", "")); + public boolean isEnableNonPersistentTopics(String brokerId) { + BrokerLookupData lookupData = availableBrokers.get(brokerId); return lookupData != null && lookupData.nonPersistentTopicsEnabled(); } }); - return brokerCandidateCache; } public boolean hasIsolationPolicy(NamespaceName namespaceName) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java index b07acfda7f77d..3061969120bb3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java @@ -55,7 +55,7 @@ public class BrokerLoadDataReporter implements LoadDataReporter, private final BrokerHostUsage brokerHostUsage; - private final String lookupServiceAddress; + private final String brokerId; @Getter private final BrokerLoadData localData; @@ -67,10 +67,10 @@ public class BrokerLoadDataReporter implements LoadDataReporter, private long tombstoneDelayInMillis; public BrokerLoadDataReporter(PulsarService pulsar, - String lookupServiceAddress, + String brokerId, LoadDataStore brokerLoadDataStore) { this.brokerLoadDataStore = brokerLoadDataStore; - this.lookupServiceAddress = lookupServiceAddress; + this.brokerId = brokerId; this.pulsar = pulsar; this.conf = this.pulsar.getConfiguration(); if (SystemUtils.IS_OS_LINUX) { @@ -111,7 +111,7 @@ public CompletableFuture reportAsync(boolean force) { log.info("publishing load report:{}", localData.toString(conf)); } CompletableFuture future = - this.brokerLoadDataStore.pushAsync(this.lookupServiceAddress, newLoadData); + this.brokerLoadDataStore.pushAsync(this.brokerId, newLoadData); future.whenComplete((__, ex) -> { if (ex == null) { localData.setReportedAt(System.currentTimeMillis()); @@ -185,7 +185,7 @@ protected void tombstone() { } var lastSuccessfulTombstonedAt = lastTombstonedAt; lastTombstonedAt = now; // dedup first - brokerLoadDataStore.removeAsync(lookupServiceAddress) + brokerLoadDataStore.removeAsync(brokerId) .whenComplete((__, e) -> { if (e != null) { log.error("Failed to clean broker load data.", e); @@ -209,13 +209,13 @@ public void handleEvent(String serviceUnit, ServiceUnitStateData data, Throwable ServiceUnitState state = ServiceUnitStateData.state(data); switch (state) { case Releasing, Splitting -> { - if (StringUtils.equals(data.sourceBroker(), lookupServiceAddress)) { + if (StringUtils.equals(data.sourceBroker(), brokerId)) { localData.clear(); tombstone(); } } case Owned -> { - if (StringUtils.equals(data.dstBroker(), lookupServiceAddress)) { + if (StringUtils.equals(data.dstBroker(), brokerId)) { localData.clear(); tombstone(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java index 0fa37d3687c20..43e05ad1ac972 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java @@ -41,7 +41,7 @@ public class TopBundleLoadDataReporter implements LoadDataReporter bundleLoadDataStore; @@ -53,10 +53,10 @@ public class TopBundleLoadDataReporter implements LoadDataReporter bundleLoadDataStore) { this.pulsar = pulsar; - this.lookupServiceAddress = lookupServiceAddress; + this.brokerId = brokerId; this.bundleLoadDataStore = bundleLoadDataStore; this.lastBundleStatsUpdatedAt = 0; this.topKBundles = new TopKBundles(pulsar); @@ -88,7 +88,7 @@ public CompletableFuture reportAsync(boolean force) { if (ExtensibleLoadManagerImpl.debug(pulsar.getConfiguration(), log)) { log.info("Reporting TopBundlesLoadData:{}", topKBundles.getLoadData()); } - return this.bundleLoadDataStore.pushAsync(lookupServiceAddress, topKBundles.getLoadData()) + return this.bundleLoadDataStore.pushAsync(brokerId, topKBundles.getLoadData()) .exceptionally(e -> { log.error("Failed to report top-bundles load data.", e); return null; @@ -106,7 +106,7 @@ protected void tombstone() { } var lastSuccessfulTombstonedAt = lastTombstonedAt; lastTombstonedAt = now; // dedup first - bundleLoadDataStore.removeAsync(lookupServiceAddress) + bundleLoadDataStore.removeAsync(brokerId) .whenComplete((__, e) -> { if (e != null) { log.error("Failed to clean broker load data.", e); @@ -129,12 +129,12 @@ public void handleEvent(String serviceUnit, ServiceUnitStateData data, Throwable ServiceUnitState state = ServiceUnitStateData.state(data); switch (state) { case Releasing, Splitting -> { - if (StringUtils.equals(data.sourceBroker(), lookupServiceAddress)) { + if (StringUtils.equals(data.sourceBroker(), brokerId)) { tombstone(); } } case Owned -> { - if (StringUtils.equals(data.dstBroker(), lookupServiceAddress)) { + if (StringUtils.equals(data.dstBroker(), brokerId)) { tombstone(); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java index 07d521a28afa7..7126ccb034196 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java @@ -47,7 +47,6 @@ import lombok.experimental.Accessors; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; @@ -69,7 +68,7 @@ /** * Load shedding strategy that unloads bundles from the highest loaded brokers. - * This strategy is only configurable in the broker load balancer extenstions introduced by + * This strategy is only configurable in the broker load balancer extensions introduced by * PIP-192[https://github.com/apache/pulsar/issues/16691]. * * This load shedding strategy has the following goals: @@ -363,7 +362,7 @@ public Set findBundlesForUnloading(LoadManagerContext context, final double targetStd = conf.getLoadBalancerBrokerLoadTargetStd(); boolean transfer = conf.isLoadBalancerTransferEnabled(); if (stats.std() > targetStd - || isUnderLoaded(context, stats.peekMinBroker(), stats.avg) + || isUnderLoaded(context, stats.peekMinBroker(), stats) || isOverLoaded(context, stats.peekMaxBroker(), stats.avg)) { unloadConditionHitCount++; } else { @@ -391,7 +390,7 @@ public Set findBundlesForUnloading(LoadManagerContext context, UnloadDecision.Reason reason; if (stats.std() > targetStd) { reason = Overloaded; - } else if (isUnderLoaded(context, stats.peekMinBroker(), stats.avg)) { + } else if (isUnderLoaded(context, stats.peekMinBroker(), stats)) { reason = Underloaded; if (debugMode) { log.info(String.format("broker:%s is underloaded:%s although " @@ -670,19 +669,27 @@ public Set findBundlesForUnloading(LoadManagerContext context, } - private boolean isUnderLoaded(LoadManagerContext context, String broker, double avgLoad) { + private boolean isUnderLoaded(LoadManagerContext context, String broker, LoadStats stats) { var brokerLoadDataOptional = context.brokerLoadDataStore().get(broker); if (brokerLoadDataOptional.isEmpty()) { return false; } var brokerLoadData = brokerLoadDataOptional.get(); - if (brokerLoadData.getMsgThroughputEMA() < 1) { + + var underLoadedMultiplier = + Math.min(0.5, Math.max(0.0, context.brokerConfiguration().getLoadBalancerBrokerLoadTargetStd() / 2.0)); + + if (brokerLoadData.getWeightedMaxEMA() < stats.avg * underLoadedMultiplier) { return true; } - return brokerLoadData.getWeightedMaxEMA() - < avgLoad * Math.min(0.5, Math.max(0.0, - context.brokerConfiguration().getLoadBalancerBrokerLoadTargetStd() / 2)); + var maxBrokerLoadDataOptional = context.brokerLoadDataStore().get(stats.peekMaxBroker()); + if (maxBrokerLoadDataOptional.isEmpty()) { + return false; + } + + return brokerLoadData.getMsgThroughputEMA() + < maxBrokerLoadDataOptional.get().getMsgThroughputEMA() * underLoadedMultiplier; } private boolean isOverLoaded(LoadManagerContext context, String broker, double avgLoad) { @@ -720,8 +727,10 @@ private boolean isTransferable(LoadManagerContext context, Map candidates = new HashMap<>(availableBrokers); for (var filter : brokerFilterPipeline) { try { - filter.filter(candidates, namespaceBundle, context); - } catch (BrokerFilterException e) { + filter.filterAsync(candidates, namespaceBundle, context) + .get(context.brokerConfiguration().getMetadataStoreOperationTimeoutSeconds(), + TimeUnit.SECONDS); + } catch (InterruptedException | ExecutionException | TimeoutException e) { log.error("Failed to filter brokers with filter: {}", filter.getClass().getName(), e); return false; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java index d6c754c90fcf6..218f57932a56b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java @@ -219,9 +219,8 @@ private static NamespaceUnloadStrategy createNamespaceUnloadStrategy(PulsarServi Thread.currentThread().getContextClassLoader()); log.info("Created namespace unload strategy:{}", unloadStrategy.getClass().getCanonicalName()); } catch (Exception e) { - log.error("Error when trying to create namespace unload strategy: {}", - conf.getLoadBalancerLoadPlacementStrategy(), e); - log.error("create namespace unload strategy failed. using TransferShedder instead."); + log.error("Error when trying to create namespace unload strategy: {}. Using {} instead.", + conf.getLoadBalancerLoadSheddingStrategy(), TransferShedder.class.getCanonicalName(), e); unloadStrategy = new TransferShedder(); } unloadStrategy.initialize(pulsar); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStore.java index 680a36523a214..8096d1908b928 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStore.java @@ -81,9 +81,32 @@ public interface LoadDataStore extends Closeable { */ void closeTableView() throws IOException; + + /** + * Starts the data store (both producer and table view). + */ + void start() throws LoadDataStoreException; + + /** + * Inits the data store (close and start the data store). + */ + void init() throws IOException; + /** * Starts the table view. */ void startTableView() throws LoadDataStoreException; + + /** + * Starts the producer. + */ + void startProducer() throws LoadDataStoreException; + + /** + * Shutdowns the data store. + */ + default void shutdown() throws IOException { + close(); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreFactory.java index 18f39abd76b76..bcb2657c67f05 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreFactory.java @@ -18,15 +18,16 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.store; -import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.broker.PulsarService; /** * The load data store factory, use to create the load data store. */ public class LoadDataStoreFactory { - public static LoadDataStore create(PulsarClient client, String name, Class clazz) + public static LoadDataStore create(PulsarService pulsar, String name, + Class clazz) throws LoadDataStoreException { - return new TableViewLoadDataStoreImpl<>(client, name, clazz); + return new TableViewLoadDataStoreImpl<>(pulsar, name, clazz); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java index a400163ebf122..3ce44a1e65a73 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java @@ -23,10 +23,14 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.TableView; @@ -35,64 +39,111 @@ * * @param Load data type. */ +@Slf4j public class TableViewLoadDataStoreImpl implements LoadDataStore { - private TableView tableView; - - private final Producer producer; - + private static final long LOAD_DATA_REPORT_UPDATE_MAX_INTERVAL_MULTIPLIER_BEFORE_RESTART = 2; + private static final String SHUTDOWN_ERR_MSG = "This load store tableview has been shutdown"; + private static final long INIT_TIMEOUT_IN_SECS = 5; + private volatile TableView tableView; + private volatile long tableViewLastUpdateTimestamp; + private volatile long producerLastPublishTimestamp; + private volatile Producer producer; + private final ServiceConfiguration conf; private final PulsarClient client; - private final String topic; - private final Class clazz; + private volatile boolean isShutdown; - public TableViewLoadDataStoreImpl(PulsarClient client, String topic, Class clazz) throws LoadDataStoreException { + public TableViewLoadDataStoreImpl(PulsarService pulsar, String topic, Class clazz) + throws LoadDataStoreException { try { - this.client = client; - this.producer = client.newProducer(Schema.JSON(clazz)).topic(topic).create(); + this.conf = pulsar.getConfiguration(); + this.client = pulsar.getClient(); this.topic = topic; this.clazz = clazz; + this.isShutdown = false; } catch (Exception e) { throw new LoadDataStoreException(e); } } @Override - public CompletableFuture pushAsync(String key, T loadData) { - return producer.newMessage().key(key).value(loadData).sendAsync().thenAccept(__ -> {}); + public synchronized CompletableFuture pushAsync(String key, T loadData) { + String msg = validateProducer(); + if (StringUtils.isNotBlank(msg)) { + return CompletableFuture.failedFuture(new IllegalStateException(msg)); + } + return producer.newMessage().key(key).value(loadData).sendAsync() + .thenAccept(__ -> producerLastPublishTimestamp = System.currentTimeMillis()); } @Override - public CompletableFuture removeAsync(String key) { - return producer.newMessage().key(key).value(null).sendAsync().thenAccept(__ -> {}); + public synchronized CompletableFuture removeAsync(String key) { + String msg = validateProducer(); + if (StringUtils.isNotBlank(msg)) { + return CompletableFuture.failedFuture(new IllegalStateException(msg)); + } + return producer.newMessage().key(key).value(null).sendAsync() + .thenAccept(__ -> producerLastPublishTimestamp = System.currentTimeMillis()); } @Override - public Optional get(String key) { - validateTableViewStart(); + public synchronized Optional get(String key) { + String msg = validateTableView(); + if (StringUtils.isNotBlank(msg)) { + if (msg.equals(SHUTDOWN_ERR_MSG)) { + return Optional.empty(); + } else { + throw new IllegalStateException(msg); + } + } return Optional.ofNullable(tableView.get(key)); } @Override - public void forEach(BiConsumer action) { - validateTableViewStart(); + public synchronized void forEach(BiConsumer action) { + String msg = validateTableView(); + if (StringUtils.isNotBlank(msg)) { + throw new IllegalStateException(msg); + } tableView.forEach(action); } - public Set> entrySet() { - validateTableViewStart(); + public synchronized Set> entrySet() { + String msg = validateTableView(); + if (StringUtils.isNotBlank(msg)) { + throw new IllegalStateException(msg); + } return tableView.entrySet(); } @Override - public int size() { - validateTableViewStart(); + public synchronized int size() { + String msg = validateTableView(); + if (StringUtils.isNotBlank(msg)) { + throw new IllegalStateException(msg); + } return tableView.size(); } + private void validateState() { + if (isShutdown) { + throw new IllegalStateException(SHUTDOWN_ERR_MSG); + } + } + + @Override - public void closeTableView() throws IOException { + public synchronized void init() throws IOException { + validateState(); + close(); + start(); + } + + @Override + public synchronized void closeTableView() throws IOException { + validateState(); if (tableView != null) { tableView.close(); tableView = null; @@ -100,29 +151,119 @@ public void closeTableView() throws IOException { } @Override - public void startTableView() throws LoadDataStoreException { + public synchronized void start() throws LoadDataStoreException { + validateState(); + startProducer(); + startTableView(); + } + + private synchronized void closeProducer() throws IOException { + validateState(); + if (producer != null) { + producer.close(); + producer = null; + } + } + @Override + public synchronized void startTableView() throws LoadDataStoreException { + validateState(); if (tableView == null) { try { - tableView = client.newTableViewBuilder(Schema.JSON(clazz)).topic(topic).create(); - } catch (PulsarClientException e) { + tableView = client.newTableViewBuilder(Schema.JSON(clazz)).topic(topic).createAsync() + .get(INIT_TIMEOUT_IN_SECS, TimeUnit.SECONDS); + tableViewLastUpdateTimestamp = System.currentTimeMillis(); + tableView.forEachAndListen((k, v) -> + tableViewLastUpdateTimestamp = System.currentTimeMillis()); + } catch (Exception e) { tableView = null; throw new LoadDataStoreException(e); } } } + @Override + public synchronized void startProducer() throws LoadDataStoreException { + validateState(); + if (producer == null) { + try { + producer = client.newProducer(Schema.JSON(clazz)).topic(topic).createAsync() + .get(INIT_TIMEOUT_IN_SECS, TimeUnit.SECONDS); + producerLastPublishTimestamp = System.currentTimeMillis(); + } catch (Exception e) { + producer = null; + throw new LoadDataStoreException(e); + } + } + } @Override - public void close() throws IOException { - if (producer != null) { - producer.close(); + public synchronized void close() throws IOException { + if (isShutdown) { + return; } + closeProducer(); closeTableView(); } - private void validateTableViewStart() { - if (tableView == null) { - throw new IllegalStateException("table view has not been started"); + @Override + public synchronized void shutdown() throws IOException { + close(); + isShutdown = true; + } + + private String validateProducer() { + if (isShutdown) { + return SHUTDOWN_ERR_MSG; } + String restartReason = getRestartReason(producer, producerLastPublishTimestamp); + if (StringUtils.isNotBlank(restartReason)) { + try { + closeProducer(); + startProducer(); + log.info("Restarted producer on {}, {}", topic, restartReason); + } catch (Exception e) { + String msg = "Failed to restart producer on " + topic + ", restart reason: " + restartReason; + log.error(msg, e); + return msg; + } + } + return null; + } + + private String validateTableView() { + if (isShutdown) { + return SHUTDOWN_ERR_MSG; + } + String restartReason = getRestartReason(tableView, tableViewLastUpdateTimestamp); + if (StringUtils.isNotBlank(restartReason)) { + try { + closeTableView(); + startTableView(); + log.info("Restarted tableview on {}, {}", topic, restartReason); + } catch (Exception e) { + String msg = "Failed to tableview on " + topic + ", restart reason: " + restartReason; + log.error(msg, e); + return msg; + } + } + return null; } + private String getRestartReason(Object obj, long lastUpdateTimestamp) { + + String restartReason = null; + + if (obj == null) { + restartReason = "object is null"; + } else { + long inactiveDuration = System.currentTimeMillis() - lastUpdateTimestamp; + long threshold = TimeUnit.MINUTES.toMillis(conf.getLoadBalancerReportUpdateMaxIntervalMinutes()) + * LOAD_DATA_REPORT_UPDATE_MAX_INTERVAL_MULTIPLIER_BEFORE_RESTART; + if (inactiveDuration > threshold) { + restartReason = String.format("inactiveDuration=%d secs > threshold = %d secs", + TimeUnit.MILLISECONDS.toSeconds(inactiveDuration), + TimeUnit.MILLISECONDS.toSeconds(threshold)); + } + } + return restartReason; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/BrokerSelectionStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/BrokerSelectionStrategy.java index e0a9122383c22..b240cb5b5f6a6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/BrokerSelectionStrategy.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/BrokerSelectionStrategy.java @@ -21,11 +21,13 @@ import java.util.Optional; import java.util.Set; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.common.classification.InterfaceStability; import org.apache.pulsar.common.naming.ServiceUnitId; /** * The broker selection strategy is designed to select the broker according to different implementations. */ +@InterfaceStability.Evolving public interface BrokerSelectionStrategy { /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/BrokerSelectionStrategyFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/BrokerSelectionStrategyFactory.java new file mode 100644 index 0000000000000..61b9fbcfcb9e5 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/BrokerSelectionStrategyFactory.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.strategy; + +import org.apache.pulsar.common.classification.InterfaceStability; + +@InterfaceStability.Stable +public interface BrokerSelectionStrategyFactory { + + BrokerSelectionStrategy createBrokerSelectionStrategy(); +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java index 98986d84b9858..9bf16ac179532 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java @@ -96,8 +96,7 @@ public Optional select( // select one of them at the end. double totalUsage = 0.0d; - // TODO: use loadBalancerDebugModeEnabled too. - boolean debugMode = log.isDebugEnabled(); + boolean debugMode = log.isDebugEnabled() || conf.isLoadBalancerDebugModeEnabled(); for (String broker : candidates) { var brokerLoadDataOptional = context.brokerLoadDataStore().get(broker); if (brokerLoadDataOptional.isEmpty()) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/AvgShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/AvgShedder.java new file mode 100644 index 0000000000000..39ff242fc6c17 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/AvgShedder.java @@ -0,0 +1,318 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.impl; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; +import com.google.common.hash.Hashing; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Random; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.mutable.MutableDouble; +import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.LoadData; +import org.apache.pulsar.broker.loadbalance.LoadSheddingStrategy; +import org.apache.pulsar.broker.loadbalance.ModularLoadManagerStrategy; +import org.apache.pulsar.policies.data.loadbalancer.BrokerData; +import org.apache.pulsar.policies.data.loadbalancer.BundleData; +import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; +import org.apache.pulsar.policies.data.loadbalancer.TimeAverageMessageData; + +@Slf4j +public class AvgShedder implements LoadSheddingStrategy, ModularLoadManagerStrategy { + // map bundle to broker. + private final Map bundleBrokerMap = new HashMap<>(); + // map broker to Scores. scores:0-100 + private final Map brokerScoreMap = new HashMap<>(); + // map broker hit count for high threshold/low threshold + private final Map brokerHitCountForHigh = new HashMap<>(); + private final Map brokerHitCountForLow = new HashMap<>(); + private static final double MB = 1024 * 1024; + + @Override + public Multimap findBundlesForUnloading(LoadData loadData, ServiceConfiguration conf) { + // result returned by shedding, map broker to bundles. + Multimap selectedBundlesCache = ArrayListMultimap.create(); + + // configuration for shedding. + final double minThroughputThreshold = conf.getMinUnloadMessageThroughput(); + final double minMsgThreshold = conf.getMinUnloadMessage(); + final double maxUnloadPercentage = conf.getMaxUnloadPercentage(); + final double lowThreshold = conf.getLoadBalancerAvgShedderLowThreshold(); + final double highThreshold = conf.getLoadBalancerAvgShedderHighThreshold(); + final int hitCountHighThreshold = conf.getLoadBalancerAvgShedderHitCountHighThreshold(); + final int hitCountLowThreshold = conf.getLoadBalancerAvgShedderHitCountLowThreshold(); + if (log.isDebugEnabled()) { + log.debug("highThreshold:{}, lowThreshold:{}, hitCountHighThreshold:{}, hitCountLowThreshold:{}, " + + "minMsgThreshold:{}, minThroughputThreshold:{}", + highThreshold, lowThreshold, hitCountHighThreshold, hitCountLowThreshold, + minMsgThreshold, minThroughputThreshold); + } + + List brokers = calculateScoresAndSort(loadData, conf); + log.info("sorted broker list:{}", brokers); + + // find broker pairs for shedding. + List> pairs = findBrokerPairs(brokers, lowThreshold, highThreshold); + log.info("brokerHitCountForHigh:{}, brokerHitCountForLow:{}", brokerHitCountForHigh, brokerHitCountForLow); + if (pairs.isEmpty()) { + if (log.isDebugEnabled()) { + log.debug("there is no any overload broker, no need to shedding bundles."); + } + brokerHitCountForHigh.clear(); + brokerHitCountForLow.clear(); + return selectedBundlesCache; + } + + // choosing bundles to unload. + for (Pair pair : pairs) { + String overloadedBroker = pair.getRight(); + String underloadedBroker = pair.getLeft(); + + // check hit count for high threshold and low threshold. + if (!(brokerHitCountForHigh.computeIfAbsent(underloadedBroker, __ -> new MutableInt(0)) + .intValue() >= hitCountHighThreshold) + && !(brokerHitCountForHigh.computeIfAbsent(overloadedBroker, __ -> new MutableInt(0)) + .intValue() >= hitCountHighThreshold) + && !(brokerHitCountForLow.computeIfAbsent(underloadedBroker, __ -> new MutableInt(0)) + .intValue() >= hitCountLowThreshold) + && !(brokerHitCountForLow.computeIfAbsent(overloadedBroker, __ -> new MutableInt(0)) + .intValue() >= hitCountLowThreshold)) { + continue; + } + + // if hit, remove entry. + brokerHitCountForHigh.remove(underloadedBroker); + brokerHitCountForHigh.remove(overloadedBroker); + brokerHitCountForLow.remove(underloadedBroker); + brokerHitCountForLow.remove(overloadedBroker); + + // select bundle for unloading. + selectBundleForUnloading(loadData, overloadedBroker, underloadedBroker, minThroughputThreshold, + minMsgThreshold, maxUnloadPercentage, selectedBundlesCache); + } + return selectedBundlesCache; + } + + private void selectBundleForUnloading(LoadData loadData, String overloadedBroker, String underloadedBroker, + double minThroughputThreshold, double minMsgThreshold, + double maxUnloadPercentage, Multimap selectedBundlesCache) { + // calculate how much throughput to unload. + LocalBrokerData minLocalBrokerData = loadData.getBrokerData().get(underloadedBroker).getLocalData(); + LocalBrokerData maxLocalBrokerData = loadData.getBrokerData().get(overloadedBroker).getLocalData(); + + double minMsgRate = minLocalBrokerData.getMsgRateIn() + minLocalBrokerData.getMsgRateOut(); + double maxMsgRate = maxLocalBrokerData.getMsgRateIn() + maxLocalBrokerData.getMsgRateOut(); + + double minThroughput = minLocalBrokerData.getMsgThroughputIn() + minLocalBrokerData.getMsgThroughputOut(); + double maxThroughput = maxLocalBrokerData.getMsgThroughputIn() + maxLocalBrokerData.getMsgThroughputOut(); + + double msgRequiredFromUnloadedBundles = (maxMsgRate - minMsgRate) * maxUnloadPercentage; + double throughputRequiredFromUnloadedBundles = (maxThroughput - minThroughput) * maxUnloadPercentage; + + boolean isMsgRateToOffload; + MutableDouble trafficMarkedToOffload = new MutableDouble(0); + + if (msgRequiredFromUnloadedBundles > minMsgThreshold) { + isMsgRateToOffload = true; + trafficMarkedToOffload.setValue(msgRequiredFromUnloadedBundles); + } else if (throughputRequiredFromUnloadedBundles > minThroughputThreshold) { + isMsgRateToOffload = false; + trafficMarkedToOffload.setValue(throughputRequiredFromUnloadedBundles); + } else { + log.info( + "broker:[{}] is planning to shed bundles to broker:[{}],but the throughput {} MByte/s is " + + "less than minimumThroughputThreshold {} MByte/s, and the msgRate {} rate/s" + + " is also less than minimumMsgRateThreshold {} rate/s, skipping bundle unload.", + overloadedBroker, underloadedBroker, throughputRequiredFromUnloadedBundles / MB, + minThroughputThreshold / MB, msgRequiredFromUnloadedBundles, minMsgThreshold); + return; + } + + if (maxLocalBrokerData.getBundles().size() == 1) { + log.warn("HIGH USAGE WARNING : Sole namespace bundle {} is overloading broker {}. " + + "No Load Shedding will be done on this broker", + maxLocalBrokerData.getBundles().iterator().next(), overloadedBroker); + } else if (maxLocalBrokerData.getBundles().isEmpty()) { + log.warn("Broker {} is overloaded despite having no bundles", overloadedBroker); + } + + // do shedding + log.info( + "broker:[{}] is planning to shed bundles to broker:[{}]. " + + "maxBroker stat:scores:{}, throughput:{}, msgRate:{}. " + + "minBroker stat:scores:{}, throughput:{}, msgRate:{}. " + + "isMsgRateToOffload:{}, trafficMarkedToOffload:{}", + overloadedBroker, underloadedBroker, brokerScoreMap.get(overloadedBroker), maxThroughput, + maxMsgRate, brokerScoreMap.get(underloadedBroker), minThroughput, minMsgRate, + isMsgRateToOffload, trafficMarkedToOffload); + + loadData.getBundleDataForLoadShedding().entrySet().stream().filter(e -> + maxLocalBrokerData.getBundles().contains(e.getKey()) + ).filter(e -> + !loadData.getRecentlyUnloadedBundles().containsKey(e.getKey()) + ).map((e) -> { + BundleData bundleData = e.getValue(); + TimeAverageMessageData shortTermData = bundleData.getShortTermData(); + double traffic = isMsgRateToOffload + ? shortTermData.getMsgRateIn() + shortTermData.getMsgRateOut() + : shortTermData.getMsgThroughputIn() + shortTermData.getMsgThroughputOut(); + return Pair.of(e, traffic); + }).sorted((e1, e2) -> + Double.compare(e2.getRight(), e1.getRight()) + ).forEach(e -> { + Map.Entry bundle = e.getLeft(); + double traffic = e.getRight(); + if (traffic > 0 && traffic <= trafficMarkedToOffload.getValue()) { + selectedBundlesCache.put(overloadedBroker, bundle.getKey()); + bundleBrokerMap.put(bundle.getValue(), underloadedBroker); + trafficMarkedToOffload.add(-traffic); + if (log.isDebugEnabled()) { + log.debug("Found bundle to unload:{}, isMsgRateToOffload:{}, traffic:{}", + bundle, isMsgRateToOffload, traffic); + } + } + }); + } + + @Override + public void onActiveBrokersChange(Set activeBrokers) { + LoadSheddingStrategy.super.onActiveBrokersChange(activeBrokers); + } + + private List calculateScoresAndSort(LoadData loadData, ServiceConfiguration conf) { + brokerScoreMap.clear(); + + // calculate scores of brokers. + for (Map.Entry entry : loadData.getBrokerData().entrySet()) { + LocalBrokerData localBrokerData = entry.getValue().getLocalData(); + String broker = entry.getKey(); + Double score = calculateScores(localBrokerData, conf); + brokerScoreMap.put(broker, score); + if (log.isDebugEnabled()) { + log.info("broker:{}, scores:{}, throughput:{}, messageRate:{}", broker, score, + localBrokerData.getMsgThroughputIn() + localBrokerData.getMsgThroughputOut(), + localBrokerData.getMsgRateIn() + localBrokerData.getMsgRateOut()); + } + } + + // sort brokers by scores. + return brokerScoreMap.entrySet().stream().sorted((o1, o2) -> (int) (o1.getValue() - o2.getValue())) + .map(Map.Entry::getKey).toList(); + } + + private Double calculateScores(LocalBrokerData localBrokerData, final ServiceConfiguration conf) { + return localBrokerData.getMaxResourceUsageWithWeight( + conf.getLoadBalancerCPUResourceWeight(), + conf.getLoadBalancerDirectMemoryResourceWeight(), + conf.getLoadBalancerBandwidthInResourceWeight(), + conf.getLoadBalancerBandwidthOutResourceWeight()) * 100; + } + + private List> findBrokerPairs(List brokers, + double lowThreshold, double highThreshold) { + List> pairs = new LinkedList<>(); + int i = 0, j = brokers.size() - 1; + while (i <= j) { + String maxBroker = brokers.get(j); + String minBroker = brokers.get(i); + if (brokerScoreMap.get(maxBroker) - brokerScoreMap.get(minBroker) < lowThreshold) { + brokerHitCountForHigh.remove(maxBroker); + brokerHitCountForHigh.remove(minBroker); + + brokerHitCountForLow.remove(maxBroker); + brokerHitCountForLow.remove(minBroker); + } else { + pairs.add(Pair.of(minBroker, maxBroker)); + if (brokerScoreMap.get(maxBroker) - brokerScoreMap.get(minBroker) < highThreshold) { + brokerHitCountForLow.computeIfAbsent(minBroker, k -> new MutableInt(0)).increment(); + brokerHitCountForLow.computeIfAbsent(maxBroker, k -> new MutableInt(0)).increment(); + + brokerHitCountForHigh.remove(maxBroker); + brokerHitCountForHigh.remove(minBroker); + } else { + brokerHitCountForLow.computeIfAbsent(minBroker, k -> new MutableInt(0)).increment(); + brokerHitCountForLow.computeIfAbsent(maxBroker, k -> new MutableInt(0)).increment(); + + brokerHitCountForHigh.computeIfAbsent(minBroker, k -> new MutableInt(0)).increment(); + brokerHitCountForHigh.computeIfAbsent(maxBroker, k -> new MutableInt(0)).increment(); + } + } + i++; + j--; + } + return pairs; + } + + @Override + public Optional selectBroker(Set candidates, BundleData bundleToAssign, LoadData loadData, + ServiceConfiguration conf) { + final var brokerToUnload = bundleBrokerMap.getOrDefault(bundleToAssign, null); + if (brokerToUnload == null || !candidates.contains(bundleBrokerMap.get(bundleToAssign))) { + // cluster initializing or broker is shutdown + if (log.isDebugEnabled()) { + if (!bundleBrokerMap.containsKey(bundleToAssign)) { + log.debug("cluster is initializing"); + } else { + log.debug("expected broker:{} is shutdown, candidates:{}", bundleBrokerMap.get(bundleToAssign), + candidates); + } + } + String broker = getExpectedBroker(candidates, bundleToAssign); + bundleBrokerMap.put(bundleToAssign, broker); + return Optional.of(broker); + } else { + return Optional.of(brokerToUnload); + } + } + + private static String getExpectedBroker(Collection brokers, BundleData bundle) { + List sortedBrokers = new ArrayList<>(brokers); + Collections.sort(sortedBrokers); + + try { + // use random number as input of hashing function to avoid special case that, + // if there is 4 brokers running in the cluster,and add broker5,and shutdown broker3, + // then all bundles belonging to broker3 will be loaded on the same broker. + final long hashcode = Hashing.crc32().hashString(String.valueOf(new Random().nextInt()), + StandardCharsets.UTF_8).padToLong(); + final int index = (int) (Math.abs(hashcode) % sortedBrokers.size()); + if (log.isDebugEnabled()) { + log.debug("Assignment details: brokers={}, bundle={}, hashcode={}, index={}", + sortedBrokers, bundle, hashcode, index); + } + return sortedBrokers.get(index); + } catch (Throwable e) { + // theoretically this logic branch should not be executed + log.error("Bundle format of {} is invalid", bundle, e); + return sortedBrokers.get(Math.abs(bundle.hashCode()) % sortedBrokers.size()); + } + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BundleRangeCache.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BundleRangeCache.java new file mode 100644 index 0000000000000..5cb92682232a5 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BundleRangeCache.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.impl; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Stream; + +/** + * The cache for the bundle ranges. + * The first key is the broker id and the second key is the namespace name, the value is the set of bundle ranges of + * that namespace. When the broker key is accessed if the associated value is not present, an empty map will be created + * as the initial value that will never be removed. + * Therefore, for each broker, there could only be one internal map during the whole lifetime. Then it will be safe + * to apply the synchronized key word on the value for thread safe operations. + */ +public class BundleRangeCache { + + // Map from brokers to namespaces to the bundle ranges in that namespace assigned to that broker. + // Used to distribute bundles within a namespace evenly across brokers. + private final Map>> data = new ConcurrentHashMap<>(); + + public void reloadFromBundles(String broker, Stream bundles) { + final var namespaceToBundleRange = data.computeIfAbsent(broker, __ -> new HashMap<>()); + synchronized (namespaceToBundleRange) { + namespaceToBundleRange.clear(); + bundles.forEach(bundleName -> { + final String namespace = LoadManagerShared.getNamespaceNameFromBundleName(bundleName); + final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundleName); + namespaceToBundleRange.computeIfAbsent(namespace, __ -> new HashSet<>()).add(bundleRange); + }); + } + } + + public void add(String broker, String namespace, String bundleRange) { + final var namespaceToBundleRange = data.computeIfAbsent(broker, __ -> new HashMap<>()); + synchronized (namespaceToBundleRange) { + namespaceToBundleRange.computeIfAbsent(namespace, __ -> new HashSet<>()).add(bundleRange); + } + } + + public int getBundleRangeCount(String broker, String namespace) { + final var namespaceToBundleRange = data.computeIfAbsent(broker, __ -> new HashMap<>()); + synchronized (namespaceToBundleRange) { + final var bundleRangeSet = namespaceToBundleRange.get(namespace); + return bundleRangeSet != null ? bundleRangeSet.size() : 0; + } + } + + /** + * Get the map whose key is the broker and value is the namespace that has at least 1 cached bundle range. + */ + public Map> getBrokerToNamespacesMap() { + final var brokerToNamespaces = new HashMap>(); + for (var entry : data.entrySet()) { + final var broker = entry.getKey(); + final var namespaceToBundleRange = entry.getValue(); + synchronized (namespaceToBundleRange) { + brokerToNamespaces.put(broker, namespaceToBundleRange.keySet().stream().toList()); + } + } + return brokerToNamespaces; + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/DeviationShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/DeviationShedder.java deleted file mode 100644 index fd90a728478f4..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/DeviationShedder.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.loadbalance.impl; - -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.Multimap; -import java.util.Map; -import java.util.TreeSet; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.loadbalance.LoadData; -import org.apache.pulsar.broker.loadbalance.LoadSheddingStrategy; -import org.apache.pulsar.policies.data.loadbalancer.BrokerData; - -/** - * An abstract class which makes a LoadSheddingStrategy which makes decisions based on standard deviation easier to - * implement. Assuming there exists some real number metric which may estimate the load on a server, this load shedding - * strategy calculates the standard deviation with respect to that metric and sheds load on brokers whose standard - * deviation is above some threshold. - */ -public abstract class DeviationShedder implements LoadSheddingStrategy { - // A Set of pairs is used in favor of a Multimap for simplicity. - protected TreeSet> metricTreeSetCache; - protected TreeSet> bundleTreeSetCache; - - /** - * Initialize this DeviationShedder. - */ - public DeviationShedder() { - bundleTreeSetCache = new TreeSet<>(); - metricTreeSetCache = new TreeSet<>(); - } - - // Measure the load incurred by a bundle. - protected abstract double bundleValue(String bundle, BrokerData brokerData, ServiceConfiguration conf); - - // Measure the load suffered by a broker. - protected abstract double brokerValue(BrokerData brokerData, ServiceConfiguration conf); - - // Get the threshold above which the standard deviation of a broker is large - // enough to warrant unloading bundles. - protected abstract double getDeviationThreshold(ServiceConfiguration conf); - - /** - * Recommend that all of the returned bundles be unloaded based on observing excessive standard deviations according - * to some metric. - * - * @param loadData - * The load data to used to make the unloading decision. - * @param conf - * The service configuration. - * @return A map from all selected bundles to the brokers on which they reside. - */ - @Override - public Multimap findBundlesForUnloading(final LoadData loadData, final ServiceConfiguration conf) { - final Multimap result = ArrayListMultimap.create(); - bundleTreeSetCache.clear(); - metricTreeSetCache.clear(); - double sum = 0; - double squareSum = 0; - final Map brokerDataMap = loadData.getBrokerData(); - - // Treating each broker as a data point, calculate the sum and squared - // sum of the evaluated broker metrics. - // These may be used to calculate the standard deviation. - for (Map.Entry entry : brokerDataMap.entrySet()) { - final double value = brokerValue(entry.getValue(), conf); - sum += value; - squareSum += value * value; - metricTreeSetCache.add(new ImmutablePair<>(value, entry.getKey())); - } - // Mean cannot change by just moving around bundles. - final double mean = sum / brokerDataMap.size(); - double standardDeviation = Math.sqrt(squareSum / brokerDataMap.size() - mean * mean); - final double deviationThreshold = getDeviationThreshold(conf); - String lastMostOverloaded = null; - // While the most loaded broker is above the standard deviation - // threshold, continue to move bundles. - while ((metricTreeSetCache.last().getKey() - mean) / standardDeviation > deviationThreshold) { - final Pair mostLoadedPair = metricTreeSetCache.last(); - final double highestValue = mostLoadedPair.getKey(); - final String mostLoaded = mostLoadedPair.getValue(); - - final Pair leastLoadedPair = metricTreeSetCache.first(); - final double leastValue = leastLoadedPair.getKey(); - final String leastLoaded = metricTreeSetCache.first().getValue(); - - if (!mostLoaded.equals(lastMostOverloaded)) { - // Reset the bundle tree set now that a different broker is - // being considered. - bundleTreeSetCache.clear(); - for (String bundle : brokerDataMap.get(mostLoaded).getLocalData().getBundles()) { - if (!result.containsKey(bundle)) { - // Don't consider bundles that are already going to be - // moved. - bundleTreeSetCache.add( - new ImmutablePair<>(bundleValue(bundle, brokerDataMap.get(mostLoaded), conf), bundle)); - } - } - lastMostOverloaded = mostLoaded; - } - boolean selected = false; - while (!(bundleTreeSetCache.isEmpty() || selected)) { - Pair mostExpensivePair = bundleTreeSetCache.pollLast(); - double loadIncurred = mostExpensivePair.getKey(); - // When the bundle is moved, we want the now least loaded server - // to have lower overall load than the - // most loaded server does not. Thus, we will only consider - // moving the bundle if this condition - // holds, and otherwise we will try the next bundle. - if (loadIncurred + leastValue < highestValue) { - // Update the standard deviation and replace the old load - // values in the broker tree set with the - // load values assuming this move took place. - final String bundleToMove = mostExpensivePair.getValue(); - result.put(bundleToMove, mostLoaded); - metricTreeSetCache.remove(mostLoadedPair); - metricTreeSetCache.remove(leastLoadedPair); - final double newHighLoad = highestValue - loadIncurred; - final double newLowLoad = leastValue - loadIncurred; - squareSum -= highestValue * highestValue + leastValue * leastValue; - squareSum += newHighLoad * newHighLoad + newLowLoad * newLowLoad; - standardDeviation = Math.sqrt(squareSum / brokerDataMap.size() - mean * mean); - metricTreeSetCache.add(new ImmutablePair<>(newLowLoad, leastLoaded)); - metricTreeSetCache.add(new ImmutablePair<>(newHighLoad, mostLoaded)); - selected = true; - } - } - if (!selected) { - // Move on to the next broker if no bundle could be moved. - metricTreeSetCache.pollLast(); - } - } - return result; - } -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastLongTermMessageRate.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastLongTermMessageRate.java index fe161467338ff..f51ca797f0edb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastLongTermMessageRate.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastLongTermMessageRate.java @@ -52,7 +52,11 @@ public LeastLongTermMessageRate() { // Any broker at (or above) the overload threshold will have a score of POSITIVE_INFINITY. private static double getScore(final BrokerData brokerData, final ServiceConfiguration conf) { final double overloadThreshold = conf.getLoadBalancerBrokerOverloadedThresholdPercentage() / 100.0; - final double maxUsage = brokerData.getLocalData().getMaxResourceUsage(); + final double maxUsage = brokerData.getLocalData().getMaxResourceUsageWithWeight( + conf.getLoadBalancerCPUResourceWeight(), + conf.getLoadBalancerDirectMemoryResourceWeight(), + conf.getLoadBalancerBandwidthInResourceWeight(), + conf.getLoadBalancerBandwidthOutResourceWeight()); if (maxUsage > overloadThreshold) { log.warn("Broker {} is overloaded: max usage={}", brokerData.getLocalData().getWebServiceUrl(), maxUsage); return Double.POSITIVE_INFINITY; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastResourceUsageWithWeight.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastResourceUsageWithWeight.java index ab3e63e9d133f..2baf58c9f05b5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastResourceUsageWithWeight.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastResourceUsageWithWeight.java @@ -68,8 +68,8 @@ private double getMaxResourceUsageWithWeight(final String broker, final BrokerDa localData.getDirectMemory().percentUsage(), localData.getBandwidthIn().percentUsage(), localData.getBandwidthOut().percentUsage(), conf.getLoadBalancerCPUResourceWeight(), conf.getLoadBalancerMemoryResourceWeight(), conf.getLoadBalancerDirectMemoryResourceWeight(), - conf.getLoadBalancerBandwithInResourceWeight(), - conf.getLoadBalancerBandwithOutResourceWeight()); + conf.getLoadBalancerBandwidthInResourceWeight(), + conf.getLoadBalancerBandwidthOutResourceWeight()); } if (log.isDebugEnabled()) { @@ -99,8 +99,8 @@ private double updateAndGetMaxResourceUsageWithWeight(String broker, BrokerData double resourceUsage = brokerData.getLocalData().getMaxResourceUsageWithWeight( conf.getLoadBalancerCPUResourceWeight(), conf.getLoadBalancerDirectMemoryResourceWeight(), - conf.getLoadBalancerBandwithInResourceWeight(), - conf.getLoadBalancerBandwithOutResourceWeight()); + conf.getLoadBalancerBandwidthInResourceWeight(), + conf.getLoadBalancerBandwidthOutResourceWeight()); historyUsage = historyUsage == null ? resourceUsage : historyUsage * historyPercentage + (1 - historyPercentage) * resourceUsage; if (log.isDebugEnabled()) { @@ -110,8 +110,8 @@ private double updateAndGetMaxResourceUsageWithWeight(String broker, BrokerData + "OUT weight: {} ", broker, historyUsage, historyPercentage, conf.getLoadBalancerCPUResourceWeight(), conf.getLoadBalancerMemoryResourceWeight(), conf.getLoadBalancerDirectMemoryResourceWeight(), - conf.getLoadBalancerBandwithInResourceWeight(), - conf.getLoadBalancerBandwithOutResourceWeight()); + conf.getLoadBalancerBandwidthInResourceWeight(), + conf.getLoadBalancerBandwidthOutResourceWeight()); } brokerAvgResourceUsageWithWeight.put(broker, historyUsage); return historyUsage; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java index 2f7ca614943b1..6d0e6bb907346 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java @@ -155,7 +155,8 @@ private double getTotalCpuUsageForCGroup(double elapsedTimeSeconds) { * * * Line is split in "words", filtering the first. The sum of all numbers give the amount of cpu cycles used this - * far. Real CPU usage should equal the sum subtracting the idle cycles, this would include iowait, irq and steal. + * far. Real CPU usage should equal the sum substracting the idle cycles(that is idle+iowait), this would include + * cpu, user, nice, system, irq, softirq, steal, guest and guest_nice. */ private double getTotalCpuUsageForEntireHost() { LinuxInfoUtils.ResourceUsage cpuUsageForEntireHost = getCpuUsageForEntireHost(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java index 6818ae03b5280..7ca2b926db7db 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java @@ -21,8 +21,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.apache.pulsar.common.stats.JvmMetrics.getJvmDirectMemoryUsed; import io.netty.util.concurrent.FastThreadLocal; -import java.net.MalformedURLException; -import java.net.URL; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -48,8 +46,6 @@ import org.apache.pulsar.common.policies.data.FailureDomainImpl; import org.apache.pulsar.common.util.DirectMemoryUtils; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.policies.data.loadbalancer.BrokerData; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; @@ -106,59 +102,56 @@ public static void applyNamespacePolicies(final ServiceUnitId serviceUnit, if (isIsolationPoliciesPresent) { LOG.debug("Isolation Policies Present for namespace - [{}]", namespace.toString()); } - for (final String broker : availableBrokers) { - final String brokerUrlString = String.format("http://%s", broker); - URL brokerUrl; + for (final String brokerId : availableBrokers) { + String brokerHost; try { - brokerUrl = new URL(brokerUrlString); - } catch (MalformedURLException e) { - LOG.error("Unable to parse brokerUrl from ResourceUnitId", e); + brokerHost = parseBrokerHost(brokerId); + } catch (IllegalArgumentException e) { + LOG.error("Unable to parse host from {}", brokerId, e); continue; } // todo: in future check if the resource unit has resources to take the namespace if (isIsolationPoliciesPresent) { // note: serviceUnitID is namespace name and ResourceID is brokerName - if (policies.isPrimaryBroker(namespace, brokerUrl.getHost())) { - primariesCache.add(broker); + if (policies.isPrimaryBroker(namespace, brokerHost)) { + primariesCache.add(brokerId); if (LOG.isDebugEnabled()) { LOG.debug("Added Primary Broker - [{}] as possible Candidates for" - + " namespace - [{}] with policies", brokerUrl.getHost(), namespace.toString()); + + " namespace - [{}] with policies", brokerHost, namespace.toString()); } - } else if (policies.isSecondaryBroker(namespace, brokerUrl.getHost())) { - secondaryCache.add(broker); + } else if (policies.isSecondaryBroker(namespace, brokerHost)) { + secondaryCache.add(brokerId); if (LOG.isDebugEnabled()) { LOG.debug( "Added Shared Broker - [{}] as possible " + "Candidates for namespace - [{}] with policies", - brokerUrl.getHost(), namespace.toString()); + brokerHost, namespace.toString()); } } else { if (LOG.isDebugEnabled()) { LOG.debug("Skipping Broker - [{}] not primary broker and not shared" + " for namespace - [{}] ", - brokerUrl.getHost(), namespace.toString()); + brokerHost, namespace.toString()); } } } else { // non-persistent topic can be assigned to only those brokers that enabled for non-persistent topic - if (isNonPersistentTopic - && !brokerTopicLoadingPredicate.isEnableNonPersistentTopics(brokerUrlString)) { + if (isNonPersistentTopic && !brokerTopicLoadingPredicate.isEnableNonPersistentTopics(brokerId)) { if (LOG.isDebugEnabled()) { LOG.debug("Filter broker- [{}] because it doesn't support non-persistent namespace - [{}]", - brokerUrl.getHost(), namespace.toString()); + brokerHost, namespace.toString()); } - } else if (!isNonPersistentTopic - && !brokerTopicLoadingPredicate.isEnablePersistentTopics(brokerUrlString)) { + } else if (!isNonPersistentTopic && !brokerTopicLoadingPredicate.isEnablePersistentTopics(brokerId)) { // persistent topic can be assigned to only brokers that enabled for persistent-topic if (LOG.isDebugEnabled()) { LOG.debug("Filter broker- [{}] because broker only supports non-persistent namespace - [{}]", - brokerUrl.getHost(), namespace.toString()); + brokerHost, namespace.toString()); } - } else if (policies.isSharedBroker(brokerUrl.getHost())) { - secondaryCache.add(broker); + } else if (policies.isSharedBroker(brokerHost)) { + secondaryCache.add(brokerId); if (LOG.isDebugEnabled()) { LOG.debug("Added Shared Broker - [{}] as possible Candidates for namespace - [{}]", - brokerUrl.getHost(), namespace.toString()); + brokerHost, namespace.toString()); } } } @@ -181,22 +174,110 @@ public static void applyNamespacePolicies(final ServiceUnitId serviceUnit, } } - /** - * Using the given bundles, populate the namespace to bundle range map. - * - * @param bundles - * Bundles with which to populate. - * @param target - * Map to fill. - */ - public static void fillNamespaceToBundlesMap(final Set bundles, - final ConcurrentOpenHashMap> target) { - bundles.forEach(bundleName -> { - final String namespaceName = getNamespaceNameFromBundleName(bundleName); - final String bundleRange = getBundleRangeFromBundleName(bundleName); - target.computeIfAbsent(namespaceName, - k -> ConcurrentOpenHashSet.newBuilder().build()) - .add(bundleRange); + private static String parseBrokerHost(String brokerId) { + // use last index to support ipv6 addresses + int lastIdx = brokerId.lastIndexOf(':'); + if (lastIdx > -1) { + return brokerId.substring(0, lastIdx); + } else { + throw new IllegalArgumentException("Invalid brokerId: " + brokerId); + } + } + + public static CompletableFuture> applyNamespacePoliciesAsync( + final ServiceUnitId serviceUnit, final SimpleResourceAllocationPolicies policies, + final Set availableBrokers, final BrokerTopicLoadingPredicate brokerTopicLoadingPredicate) { + NamespaceName namespace = serviceUnit.getNamespaceObject(); + return policies.areIsolationPoliciesPresentAsync(namespace).thenApply(isIsolationPoliciesPresent -> { + final Set brokerCandidateCache = new HashSet<>(); + Set primariesCache = localPrimariesCache.get(); + primariesCache.clear(); + + Set secondaryCache = localSecondaryCache.get(); + secondaryCache.clear(); + boolean isNonPersistentTopic = (serviceUnit instanceof NamespaceBundle) + ? ((NamespaceBundle) serviceUnit).hasNonPersistentTopic() : false; + if (isIsolationPoliciesPresent) { + if (LOG.isDebugEnabled()) { + LOG.debug("Isolation Policies Present for namespace - [{}]", namespace.toString()); + } + } + for (final String brokerId : availableBrokers) { + String brokerHost; + try { + brokerHost = parseBrokerHost(brokerId); + } catch (IllegalArgumentException e) { + LOG.error("Unable to parse host from {}", brokerId, e); + continue; + } + // todo: in future check if the resource unit has resources to take the namespace + if (isIsolationPoliciesPresent) { + // note: serviceUnitID is namespace name and ResourceID is brokerName + if (policies.isPrimaryBroker(namespace, brokerHost)) { + primariesCache.add(brokerId); + if (LOG.isDebugEnabled()) { + LOG.debug("Added Primary Broker - [{}] as possible Candidates for" + + " namespace - [{}] with policies", brokerHost, namespace.toString()); + } + } else if (policies.isSecondaryBroker(namespace, brokerHost)) { + secondaryCache.add(brokerId); + if (LOG.isDebugEnabled()) { + LOG.debug( + "Added Shared Broker - [{}] as possible " + + "Candidates for namespace - [{}] with policies", + brokerHost, namespace.toString()); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Skipping Broker - [{}] not primary broker and not shared" + + " for namespace - [{}] ", brokerHost, namespace.toString()); + } + + } + } else { + // non-persistent topic can be assigned to only those brokers that enabled for non-persistent topic + if (isNonPersistentTopic && !brokerTopicLoadingPredicate.isEnableNonPersistentTopics(brokerId)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Filter broker- [{}] because it doesn't support non-persistent namespace - [{}]", + brokerId, namespace.toString()); + } + } else if (!isNonPersistentTopic && !brokerTopicLoadingPredicate + .isEnablePersistentTopics(brokerId)) { + // persistent topic can be assigned to only brokers that enabled for persistent-topic + if (LOG.isDebugEnabled()) { + LOG.debug("Filter broker- [{}] because broker only supports non-persistent " + + "namespace - [{}]", brokerId, namespace.toString()); + } + } else if (policies.isSharedBroker(brokerHost)) { + secondaryCache.add(brokerId); + if (LOG.isDebugEnabled()) { + LOG.debug("Added Shared Broker - [{}] as possible Candidates for namespace - [{}]", + brokerHost, namespace.toString()); + } + } + } + } + if (isIsolationPoliciesPresent) { + brokerCandidateCache.addAll(primariesCache); + if (policies.shouldFailoverToSecondaries(namespace, primariesCache.size())) { + if (LOG.isDebugEnabled()) { + LOG.debug( + "Not enough of primaries [{}] available for namespace - [{}], " + + "adding shared [{}] as possible candidate owners", + primariesCache.size(), namespace.toString(), secondaryCache.size()); + } + brokerCandidateCache.addAll(secondaryCache); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug( + "Policies not present for namespace - [{}] so only " + + "considering shared [{}] brokers for possible owner", + namespace.toString(), secondaryCache.size()); + } + brokerCandidateCache.addAll(secondaryCache); + } + return brokerCandidateCache; }); } @@ -258,8 +339,7 @@ public static boolean isLoadSheddingEnabled(final PulsarService pulsar) { public static void removeMostServicingBrokersForNamespace( final String assignedBundleName, final Set candidates, - final ConcurrentOpenHashMap>> - brokerToNamespaceToBundleRange) { + final BundleRangeCache brokerToNamespaceToBundleRange) { if (candidates.isEmpty()) { return; } @@ -268,13 +348,7 @@ public static void removeMostServicingBrokersForNamespace( int leastBundles = Integer.MAX_VALUE; for (final String broker : candidates) { - int bundles = (int) brokerToNamespaceToBundleRange - .computeIfAbsent(broker, - k -> ConcurrentOpenHashMap.>newBuilder().build()) - .computeIfAbsent(namespaceName, - k -> ConcurrentOpenHashSet.newBuilder().build()) - .size(); + int bundles = brokerToNamespaceToBundleRange.getBundleRangeCount(broker, namespaceName); leastBundles = Math.min(leastBundles, bundles); if (leastBundles == 0) { break; @@ -285,13 +359,8 @@ public static void removeMostServicingBrokersForNamespace( // `leastBundles` may differ from the actual value. final int finalLeastBundles = leastBundles; - candidates.removeIf( - broker -> brokerToNamespaceToBundleRange.computeIfAbsent(broker, - k -> ConcurrentOpenHashMap.>newBuilder().build()) - .computeIfAbsent(namespaceName, - k -> ConcurrentOpenHashSet.newBuilder().build()) - .size() > finalLeastBundles); + candidates.removeIf(broker -> + brokerToNamespaceToBundleRange.getBundleRangeCount(broker, namespaceName) > finalLeastBundles); } /** @@ -325,8 +394,7 @@ public static void removeMostServicingBrokersForNamespace( public static void filterAntiAffinityGroupOwnedBrokers( final PulsarService pulsar, final String assignedBundleName, final Set candidates, - final ConcurrentOpenHashMap>> - brokerToNamespaceToBundleRange, + final BundleRangeCache brokerToNamespaceToBundleRange, Map brokerToDomainMap) { if (candidates.isEmpty()) { return; @@ -408,6 +476,22 @@ public static void filterAntiAffinityGroupOwnedBrokers( } } + public static CompletableFuture filterAntiAffinityGroupOwnedBrokersAsync( + final PulsarService pulsar, final String assignedBundleName, + final Set candidates, + Set> bundleOwnershipData, + Map brokerToDomainMap + ) { + if (candidates.isEmpty()) { + return CompletableFuture.completedFuture(null); + } + final String namespaceName = getNamespaceNameFromBundleName(assignedBundleName); + return getAntiAffinityNamespaceOwnedBrokers(pulsar, namespaceName, bundleOwnershipData) + .thenAccept(brokerToAntiAffinityNamespaceCount -> + filterAntiAffinityGroupOwnedBrokers(pulsar, candidates, + brokerToDomainMap, brokerToAntiAffinityNamespaceCount)); + } + /** * It computes least number of namespace owned by any of the domain and then it filters out all the domains that own * namespaces more than this count. @@ -455,8 +539,7 @@ private static void filterDomainsNotHavingLeastNumberAntiAffinityNamespaces( */ public static CompletableFuture> getAntiAffinityNamespaceOwnedBrokers( final PulsarService pulsar, final String namespaceName, - final ConcurrentOpenHashMap>> - brokerToNamespaceToBundleRange) { + final BundleRangeCache brokerToNamespaceToBundleRange) { CompletableFuture> antiAffinityNsBrokersResult = new CompletableFuture<>(); getNamespaceAntiAffinityGroupAsync(pulsar, namespaceName) @@ -467,21 +550,16 @@ public static CompletableFuture> getAntiAffinityNamespaceOw } final String antiAffinityGroup = antiAffinityGroupOptional.get(); final Map brokerToAntiAffinityNamespaceCount = new ConcurrentHashMap<>(); - final List> futures = new ArrayList<>(); - brokerToNamespaceToBundleRange.forEach((broker, nsToBundleRange) -> { - nsToBundleRange.forEach((ns, bundleRange) -> { - if (bundleRange.isEmpty()) { - return; - } - - CompletableFuture future = new CompletableFuture<>(); - futures.add(future); - countAntiAffinityNamespaceOwnedBrokers(broker, ns, future, + final var brokerToNamespaces = brokerToNamespaceToBundleRange.getBrokerToNamespacesMap(); + FutureUtil.waitForAll(brokerToNamespaces.entrySet().stream().flatMap(e -> { + final var broker = e.getKey(); + return e.getValue().stream().map(namespace -> { + final var future = new CompletableFuture(); + countAntiAffinityNamespaceOwnedBrokers(broker, namespace, future, pulsar, antiAffinityGroup, brokerToAntiAffinityNamespaceCount); + return future; }); - }); - FutureUtil.waitForAll(futures) - .thenAccept(r -> antiAffinityNsBrokersResult.complete(brokerToAntiAffinityNamespaceCount)); + }).toList()).thenAccept(__ -> antiAffinityNsBrokersResult.complete(brokerToAntiAffinityNamespaceCount)); }).exceptionally(ex -> { // namespace-policies has not been created yet antiAffinityNsBrokersResult.complete(null); @@ -581,7 +659,6 @@ public static Optional getNamespaceAntiAffinityGroup( * by different broker. * * @param namespace - * @param bundle * @param currentBroker * @param pulsar * @param brokerToNamespaceToBundleRange @@ -590,10 +667,9 @@ public static Optional getNamespaceAntiAffinityGroup( * @throws Exception */ public static boolean shouldAntiAffinityNamespaceUnload( - String namespace, String bundle, String currentBroker, + String namespace, String currentBroker, final PulsarService pulsar, - final ConcurrentOpenHashMap>> - brokerToNamespaceToBundleRange, + final BundleRangeCache brokerToNamespaceToBundleRange, Set candidateBrokers) throws Exception { Map brokerNamespaceCount = getAntiAffinityNamespaceOwnedBrokers(pulsar, namespace, @@ -648,9 +724,9 @@ public static boolean shouldAntiAffinityNamespaceUnload( } public interface BrokerTopicLoadingPredicate { - boolean isEnablePersistentTopics(String brokerUrl); + boolean isEnablePersistentTopics(String brokerId); - boolean isEnableNonPersistentTopics(String brokerUrl); + boolean isEnableNonPersistentTopics(String brokerId); } /** @@ -711,4 +787,10 @@ public static void refreshBrokerToFailureDomainMap(PulsarService pulsar, LOG.warn("Failed to get domain-list for cluster {}", e.getMessage()); } } + + public static NamespaceBundle getNamespaceBundle(PulsarService pulsar, String bundle) { + final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle); + final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle); + return pulsar.getNamespaceService().getNamespaceBundleFactory().getBundle(namespaceName, bundleRange); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index 73b4f318f3a36..48a6121b9dd13 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.loadbalance.impl; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import java.util.ArrayList; @@ -37,10 +38,12 @@ import java.util.concurrent.Executors; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; @@ -56,7 +59,9 @@ import org.apache.pulsar.broker.loadbalance.LoadSheddingStrategy; import org.apache.pulsar.broker.loadbalance.ModularLoadManager; import org.apache.pulsar.broker.loadbalance.ModularLoadManagerStrategy; +import org.apache.pulsar.broker.loadbalance.extensions.models.TopKBundles; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared.BrokerTopicLoadingPredicate; +import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.broker.stats.prometheus.metrics.Summary; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.util.ExecutorProvider; @@ -68,9 +73,6 @@ import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.Reflections; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet; -import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.MetadataStoreException.NotFoundException; import org.apache.pulsar.metadata.api.Notification; @@ -90,9 +92,6 @@ public class ModularLoadManagerImpl implements ModularLoadManager { private static final Logger log = LoggerFactory.getLogger(ModularLoadManagerImpl.class); - // Path to ZNode whose children contain BundleData jsons for each bundle (new API version of ResourceQuota). - public static final String BUNDLE_DATA_PATH = "/loadbalance/bundle-data"; - // Default message rate to assume for unseen bundles. public static final double DEFAULT_MESSAGE_RATE = 50; @@ -106,12 +105,6 @@ public class ModularLoadManagerImpl implements ModularLoadManager { // The number of effective samples to keep for observing short term data. public static final int NUM_SHORT_SAMPLES = 10; - // Path to ZNode whose children contain ResourceQuota jsons. - public static final String RESOURCE_QUOTA_ZPATH = "/loadbalance/resource-quota/namespace"; - - // Path to ZNode containing TimeAverageBrokerData jsons for each broker. - public static final String TIME_AVERAGE_BROKER_ZPATH = "/loadbalance/broker-time-average"; - // Set of broker candidates to reuse so that object creation is avoided. private final Set brokerCandidateCache; @@ -119,17 +112,10 @@ public class ModularLoadManagerImpl implements ModularLoadManager { private LockManager brokersData; private ResourceLock brokerDataLock; - private MetadataCache bundlesCache; - private MetadataCache resourceQuotaCache; - private MetadataCache timeAverageBrokerDataCache; - // Broker host usage object used to calculate system resource usage. private BrokerHostUsage brokerHostUsage; - // Map from brokers to namespaces to the bundle ranges in that namespace assigned to that broker. - // Used to distribute bundles within a namespace evenly across brokers. - private final ConcurrentOpenHashMap>> - brokerToNamespaceToBundleRange; + private final BundleRangeCache brokerToNamespaceToBundleRange = new BundleRangeCache(); // Path to the ZNode containing the LocalBrokerData json for this broker. private String brokerZnodePath; @@ -150,8 +136,8 @@ public class ModularLoadManagerImpl implements ModularLoadManager { // LocalBrokerData available before most recent update. private LocalBrokerData lastData; - // Pipeline used to determine what namespaces, if any, should be unloaded. - private final List loadSheddingPipeline; + // Used to determine what namespaces, if any, should be unloaded. + private LoadSheddingStrategy loadSheddingStrategy; // Local data for the broker this is running on. private LocalBrokerData localData; @@ -168,8 +154,11 @@ public class ModularLoadManagerImpl implements ModularLoadManager { // Policies used to determine which brokers are available for particular namespaces. private SimpleResourceAllocationPolicies policies; + @VisibleForTesting // Pulsar service used to initialize this. - private PulsarService pulsar; + protected PulsarService pulsar; + + private PulsarResources pulsarResources; // Executor service used to update broker data. private final ExecutorService executors; @@ -197,20 +186,18 @@ public class ModularLoadManagerImpl implements ModularLoadManager { private final Lock lock = new ReentrantLock(); private final Set knownBrokers = new HashSet<>(); private Map bundleBrokerAffinityMap; + // array used for sorting and select topK bundles + private final List> bundleArr = new ArrayList<>(); + /** * Initializes fields which do not depend on PulsarService. initialize(PulsarService) should subsequently be called. */ public ModularLoadManagerImpl() { brokerCandidateCache = new HashSet<>(); - brokerToNamespaceToBundleRange = - ConcurrentOpenHashMap.>>newBuilder() - .build(); defaultStats = new NamespaceBundleStats(); filterPipeline = new ArrayList<>(); loadData = new LoadData(); - loadSheddingPipeline = new ArrayList<>(); preallocatedBundleToBroker = new ConcurrentHashMap<>(); executors = Executors.newSingleThreadExecutor( new ExecutorProvider.ExtendedThreadFactory("pulsar-modular-load-manager")); @@ -218,15 +205,15 @@ public ModularLoadManagerImpl() { this.bundleBrokerAffinityMap = new ConcurrentHashMap<>(); this.brokerTopicLoadingPredicate = new BrokerTopicLoadingPredicate() { @Override - public boolean isEnablePersistentTopics(String brokerUrl) { - final BrokerData brokerData = loadData.getBrokerData().get(brokerUrl.replace("http://", "")); + public boolean isEnablePersistentTopics(String brokerId) { + final BrokerData brokerData = loadData.getBrokerData().get(brokerId); return brokerData != null && brokerData.getLocalData() != null && brokerData.getLocalData().isPersistentTopicsEnabled(); } @Override - public boolean isEnableNonPersistentTopics(String brokerUrl) { - final BrokerData brokerData = loadData.getBrokerData().get(brokerUrl.replace("http://", "")); + public boolean isEnableNonPersistentTopics(String brokerId) { + final BrokerData brokerData = loadData.getBrokerData().get(brokerId); return brokerData != null && brokerData.getLocalData() != null && brokerData.getLocalData().isNonPersistentTopicsEnabled(); } @@ -242,10 +229,8 @@ public boolean isEnableNonPersistentTopics(String brokerUrl) { @Override public void initialize(final PulsarService pulsar) { this.pulsar = pulsar; + this.pulsarResources = pulsar.getPulsarResources(); brokersData = pulsar.getCoordinationService().getLockManager(LocalBrokerData.class); - bundlesCache = pulsar.getLocalMetadataStore().getMetadataCache(BundleData.class); - resourceQuotaCache = pulsar.getLocalMetadataStore().getMetadataCache(ResourceQuota.class); - timeAverageBrokerDataCache = pulsar.getLocalMetadataStore().getMetadataCache(TimeAverageBrokerData.class); pulsar.getLocalMetadataStore().registerListener(this::handleDataNotification); pulsar.getLocalMetadataStore().registerSessionListener(this::handleMetadataSessionEvent); @@ -272,13 +257,27 @@ public void initialize(final PulsarService pulsar) { LoadManagerShared.refreshBrokerToFailureDomainMap(pulsar, brokerToFailureDomainMap); // register listeners for domain changes - pulsar.getPulsarResources().getClusterResources().getFailureDomainResources() + pulsarResources.getClusterResources().getFailureDomainResources() .registerListener(__ -> { executors.execute( () -> LoadManagerShared.refreshBrokerToFailureDomainMap(pulsar, brokerToFailureDomainMap)); }); - loadSheddingPipeline.add(createLoadSheddingStrategy()); + if (placementStrategy instanceof LoadSheddingStrategy) { + // if the placement strategy is also a load shedding strategy + // we need to check two strategies are the same + if (!conf.getLoadBalancerLoadSheddingStrategy().equals( + conf.getLoadBalancerLoadPlacementStrategy())) { + throw new IllegalArgumentException("The load shedding strategy: " + + conf.getLoadBalancerLoadSheddingStrategy() + + " can't work with the placement strategy: " + + conf.getLoadBalancerLoadPlacementStrategy()); + } + // bind the load shedding strategy and the placement strategy + loadSheddingStrategy = (LoadSheddingStrategy) placementStrategy; + } else { + loadSheddingStrategy = createLoadSheddingStrategy(); + } } public void handleDataNotification(Notification t) { @@ -301,15 +300,8 @@ private void handleMetadataSessionEvent(SessionEvent e) { } private LoadSheddingStrategy createLoadSheddingStrategy() { - try { - return Reflections.createInstance(conf.getLoadBalancerLoadSheddingStrategy(), LoadSheddingStrategy.class, - Thread.currentThread().getContextClassLoader()); - } catch (Exception e) { - log.error("Error when trying to create load shedding strategy: {}", - conf.getLoadBalancerLoadPlacementStrategy(), e); - } - log.error("create load shedding strategy failed. using OverloadShedder instead."); - return new OverloadShedder(); + return Reflections.createInstance(conf.getLoadBalancerLoadSheddingStrategy(), LoadSheddingStrategy.class, + Thread.currentThread().getContextClassLoader()); } /** @@ -350,8 +342,7 @@ private void reapDeadBrokerPreallocations(List aliveBrokers) { @Override public Set getAvailableBrokers() { try { - return new HashSet<>(brokersData.listLocks(LoadManager.LOADBALANCE_BROKERS_ROOT) - .get(conf.getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS)); + return getAvailableBrokersAsync().get(conf.getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS); } catch (Exception e) { log.warn("Error when trying to get active brokers", e); return loadData.getBrokerData().keySet(); @@ -380,13 +371,14 @@ public CompletableFuture> getAvailableBrokersAsync() { public BundleData getBundleDataOrDefault(final String bundle) { BundleData bundleData = null; try { - Optional optBundleData = bundlesCache.get(getBundleDataPath(bundle)).join(); + Optional optBundleData = + pulsarResources.getLoadBalanceResources().getBundleDataResources().getBundleData(bundle).join(); if (optBundleData.isPresent()) { return optBundleData.get(); } - Optional optQuota = resourceQuotaCache - .get(String.format("%s/%s", RESOURCE_QUOTA_ZPATH, bundle)).join(); + Optional optQuota = pulsarResources.getLoadBalanceResources().getQuotaResources() + .getQuota(bundle).join(); if (optQuota.isPresent()) { ResourceQuota quota = optQuota.get(); bundleData = new BundleData(NUM_SHORT_SAMPLES, NUM_LONG_SAMPLES); @@ -417,11 +409,6 @@ public BundleData getBundleDataOrDefault(final String bundle) { return bundleData; } - // Get the metadata store path for the given bundle full name. - public static String getBundleDataPath(final String bundle) { - return BUNDLE_DATA_PATH + "/" + bundle; - } - // Use the Pulsar client to acquire the namespace bundle stats. private Map getBundleStats() { return pulsar.getBrokerService().getBundleStats(); @@ -438,6 +425,14 @@ private double percentChange(final double oldValue, final double newValue) { return 100 * Math.abs((oldValue - newValue) / oldValue); } + private double getMaxResourceUsageWithWeight(LocalBrokerData localBrokerData, ServiceConfiguration conf) { + return localBrokerData.getMaxResourceUsageWithWeight( + conf.getLoadBalancerCPUResourceWeight(), + conf.getLoadBalancerDirectMemoryResourceWeight(), + conf.getLoadBalancerBandwidthInResourceWeight(), + conf.getLoadBalancerBandwidthOutResourceWeight()); + } + // Determine if the broker data requires an update by delegating to the update condition. private boolean needBrokerDataUpdate() { final long updateMaxIntervalMillis = TimeUnit.MINUTES @@ -450,14 +445,15 @@ private boolean needBrokerDataUpdate() { // Always update after surpassing the maximum interval. return true; } - final double maxChange = Math - .max(100.0 * (Math.abs(lastData.getMaxResourceUsage() - localData.getMaxResourceUsage())), - Math.max(percentChange(lastData.getMsgRateIn() + lastData.getMsgRateOut(), - localData.getMsgRateIn() + localData.getMsgRateOut()), - Math.max( - percentChange(lastData.getMsgThroughputIn() + lastData.getMsgThroughputOut(), - localData.getMsgThroughputIn() + localData.getMsgThroughputOut()), - percentChange(lastData.getNumBundles(), localData.getNumBundles())))); + final double maxChange = LocalBrokerData.max( + percentChange(lastData.getMsgRateIn() + lastData.getMsgRateOut(), + localData.getMsgRateIn() + localData.getMsgRateOut()), + percentChange(lastData.getMsgThroughputIn() + lastData.getMsgThroughputOut(), + localData.getMsgThroughputIn() + localData.getMsgThroughputOut()), + percentChange(lastData.getNumBundles(), localData.getNumBundles()), + 100.0 * Math.abs(getMaxResourceUsageWithWeight(lastData, conf) + - getMaxResourceUsageWithWeight(localData, conf)) + ); if (maxChange > conf.getLoadBalancerReportUpdateThresholdPercentage()) { log.info("Writing local data to metadata store because maximum change {}% exceeded threshold {}%; " + "time since last report written is {} seconds", maxChange, @@ -488,9 +484,7 @@ private synchronized void cleanupDeadBrokersData() { if (pulsar.getLeaderElectionService() != null && pulsar.getLeaderElectionService().isLeader()) { deadBrokers.forEach(this::deleteTimeAverageDataFromMetadataStoreAsync); - for (LoadSheddingStrategy loadSheddingStrategy : loadSheddingPipeline) { - loadSheddingStrategy.onActiveBrokersChange(activeBrokers); - } + loadSheddingStrategy.onActiveBrokersChange(activeBrokers); placementStrategy.onActiveBrokersChange(activeBrokers); } } @@ -558,17 +552,6 @@ private void updateBundleData() { bundleData.put(bundle, currentBundleData); } } - - //Remove not active bundle from loadData - for (String bundle : bundleData.keySet()) { - if (!activeBundles.contains(bundle)){ - bundleData.remove(bundle); - if (pulsar.getLeaderElectionService().isLeader()){ - deleteBundleDataFromMetadataStore(bundle); - } - } - } - // Remove all loaded bundles from the preallocated maps. final Map preallocatedBundleData = brokerData.getPreallocatedBundleData(); Set ownedNsBundles = pulsar.getNamespaceService().getOwnedServiceUnits() @@ -591,16 +574,18 @@ private void updateBundleData() { TimeAverageBrokerData timeAverageData = new TimeAverageBrokerData(); timeAverageData.reset(statsMap.keySet(), bundleData, defaultStats); brokerData.setTimeAverageData(timeAverageData); - final ConcurrentOpenHashMap> namespaceToBundleRange = - brokerToNamespaceToBundleRange - .computeIfAbsent(broker, k -> - ConcurrentOpenHashMap.>newBuilder() - .build()); - synchronized (namespaceToBundleRange) { - namespaceToBundleRange.clear(); - LoadManagerShared.fillNamespaceToBundlesMap(statsMap.keySet(), namespaceToBundleRange); - LoadManagerShared.fillNamespaceToBundlesMap(preallocatedBundleData.keySet(), namespaceToBundleRange); + + brokerToNamespaceToBundleRange.reloadFromBundles(broker, + Stream.of(statsMap.keySet(), preallocatedBundleData.keySet()).flatMap(Collection::stream)); + } + + // Remove not active bundle from loadData + for (String bundle : bundleData.keySet()) { + if (!activeBundles.contains(bundle)){ + bundleData.remove(bundle); + if (pulsar.getLeaderElectionService().isLeader()){ + deleteBundleDataFromMetadataStore(bundle); + } } } } @@ -645,45 +630,57 @@ public synchronized void doLoadShedding() { final Map recentlyUnloadedBundles = loadData.getRecentlyUnloadedBundles(); recentlyUnloadedBundles.keySet().removeIf(e -> recentlyUnloadedBundles.get(e) < timeout); - for (LoadSheddingStrategy strategy : loadSheddingPipeline) { - final Multimap bundlesToUnload = strategy.findBundlesForUnloading(loadData, conf); + final Multimap bundlesToUnload = loadSheddingStrategy.findBundlesForUnloading(loadData, conf); - bundlesToUnload.asMap().forEach((broker, bundles) -> { - bundles.forEach(bundle -> { - final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle); - final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle); - if (!shouldNamespacePoliciesUnload(namespaceName, bundleRange, broker)) { - return; - } + bundlesToUnload.asMap().forEach((broker, bundles) -> { + AtomicBoolean unloadBundleForBroker = new AtomicBoolean(false); + bundles.forEach(bundle -> { + final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle); + final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle); + if (!shouldNamespacePoliciesUnload(namespaceName, bundleRange, broker)) { + return; + } - if (!shouldAntiAffinityNamespaceUnload(namespaceName, bundleRange, broker)) { - return; - } + if (!shouldAntiAffinityNamespaceUnload(namespaceName, bundleRange, broker)) { + return; + } + NamespaceBundle bundleToUnload = LoadManagerShared.getNamespaceBundle(pulsar, bundle); + Optional destBroker = this.selectBroker(bundleToUnload); + if (!destBroker.isPresent()) { + log.info("[{}] No broker available to unload bundle {} from broker {}", + loadSheddingStrategy.getClass().getSimpleName(), bundle, broker); + return; + } + if (destBroker.get().equals(broker)) { + log.warn("[{}] The destination broker {} is the same as the current owner broker for Bundle {}", + loadSheddingStrategy.getClass().getSimpleName(), destBroker.get(), bundle); + return; + } - log.info("[{}] Unloading bundle: {} from broker {}", - strategy.getClass().getSimpleName(), bundle, broker); - try { - pulsar.getAdminClient().namespaces().unloadNamespaceBundle(namespaceName, bundleRange); - loadData.getRecentlyUnloadedBundles().put(bundle, System.currentTimeMillis()); - } catch (PulsarServerException | PulsarAdminException e) { - log.warn("Error when trying to perform load shedding on {} for broker {}", bundle, broker, e); - } - }); + log.info("[{}] Unloading bundle: {} from broker {} to dest broker {}", + loadSheddingStrategy.getClass().getSimpleName(), bundle, broker, destBroker.get()); + try { + pulsar.getAdminClient().namespaces() + .unloadNamespaceBundle(namespaceName, bundleRange, destBroker.get()); + loadData.getRecentlyUnloadedBundles().put(bundle, System.currentTimeMillis()); + unloadBundleCount++; + unloadBundleForBroker.set(true); + } catch (PulsarServerException | PulsarAdminException e) { + log.warn("Error when trying to perform load shedding on {} for broker {}", bundle, broker, e); + } }); + if (unloadBundleForBroker.get()) { + unloadBrokerCount++; + } + }); - updateBundleUnloadingMetrics(bundlesToUnload); - } + updateBundleUnloadingMetrics(); } /** * As leader broker, update bundle unloading metrics. - * - * @param bundlesToUnload */ - private void updateBundleUnloadingMetrics(Multimap bundlesToUnload) { - unloadBrokerCount += bundlesToUnload.keySet().size(); - unloadBundleCount += bundlesToUnload.values().size(); - + private void updateBundleUnloadingMetrics() { List metrics = new ArrayList<>(); Map dimensions = new HashMap<>(); @@ -723,7 +720,7 @@ public boolean shouldAntiAffinityNamespaceUnload(String namespace, String bundle .getBundle(namespace, bundle); LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, getAvailableBrokers(), brokerTopicLoadingPredicate); - return LoadManagerShared.shouldAntiAffinityNamespaceUnload(namespace, bundle, currentBroker, pulsar, + return LoadManagerShared.shouldAntiAffinityNamespaceUnload(namespace, currentBroker, pulsar, brokerToNamespaceToBundleRange, brokerCandidateCache); } @@ -837,99 +834,112 @@ public Optional selectBrokerForAssignment(final ServiceUnitId serviceUni // If the given bundle is already in preallocated, return the selected broker. return Optional.of(preallocatedBundleToBroker.get(bundle)); } - final BundleData data = loadData.getBundleData().computeIfAbsent(bundle, - key -> getBundleDataOrDefault(bundle)); - brokerCandidateCache.clear(); - LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, - getAvailableBrokers(), - brokerTopicLoadingPredicate); - - // filter brokers which owns topic higher than threshold - LoadManagerShared.filterBrokersWithLargeTopicCount(brokerCandidateCache, loadData, - conf.getLoadBalancerBrokerMaxTopics()); - // distribute namespaces to domain and brokers according to anti-affinity-group - LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar, serviceUnit.toString(), - brokerCandidateCache, - brokerToNamespaceToBundleRange, brokerToFailureDomainMap); - - // distribute bundles evenly to candidate-brokers if enable - if (conf.isLoadBalancerDistributeBundlesEvenlyEnabled()) { - LoadManagerShared.removeMostServicingBrokersForNamespace(serviceUnit.toString(), - brokerCandidateCache, - brokerToNamespaceToBundleRange); - if (log.isDebugEnabled()) { - log.debug("enable distribute bundles evenly to candidate-brokers, broker candidate count={}", - brokerCandidateCache.size()); - } + Optional broker = selectBroker(serviceUnit); + if (!broker.isPresent()) { + // If no broker is selected, return empty. + return broker; } - log.info("{} brokers being considered for assignment of {}", brokerCandidateCache.size(), bundle); + // Add new bundle to preallocated. + preallocateBundle(bundle, broker.get()); + return broker; + } + } finally { + selectBrokerForAssignment.observe(System.nanoTime() - startTime, TimeUnit.NANOSECONDS); + } + } - // Use the filter pipeline to finalize broker candidates. - try { - for (BrokerFilter filter : filterPipeline) { - filter.filter(brokerCandidateCache, data, loadData, conf); - } - } catch (BrokerFilterException x) { - // restore the list of brokers to the full set - LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, - getAvailableBrokers(), - brokerTopicLoadingPredicate); - } + private void preallocateBundle(String bundle, String broker) { + final BundleData data = loadData.getBundleData().computeIfAbsent(bundle, + key -> getBundleDataOrDefault(bundle)); + loadData.getBrokerData().get(broker).getPreallocatedBundleData().put(bundle, data); + preallocatedBundleToBroker.put(bundle, broker); - if (brokerCandidateCache.isEmpty()) { - // restore the list of brokers to the full set - LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, - getAvailableBrokers(), - brokerTopicLoadingPredicate); - } + final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle); + final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle); + brokerToNamespaceToBundleRange.add(broker, namespaceName, bundleRange); + } - // Choose a broker among the potentially smaller filtered list, when possible - Optional broker = placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf); + @VisibleForTesting + Optional selectBroker(final ServiceUnitId serviceUnit) { + synchronized (brokerCandidateCache) { + final String bundle = serviceUnit.toString(); + final BundleData data = loadData.getBundleData().computeIfAbsent(bundle, + key -> getBundleDataOrDefault(bundle)); + brokerCandidateCache.clear(); + LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, + getAvailableBrokers(), + brokerTopicLoadingPredicate); + + // filter brokers which owns topic higher than threshold + LoadManagerShared.filterBrokersWithLargeTopicCount(brokerCandidateCache, loadData, + conf.getLoadBalancerBrokerMaxTopics()); + + // distribute namespaces to domain and brokers according to anti-affinity-group + LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar, bundle, + brokerCandidateCache, + brokerToNamespaceToBundleRange, brokerToFailureDomainMap); + + // distribute bundles evenly to candidate-brokers if enable + // or system-namespace bundles + if (conf.isLoadBalancerDistributeBundlesEvenlyEnabled() + || serviceUnit.getNamespaceObject().equals(NamespaceName.SYSTEM_NAMESPACE)) { + LoadManagerShared.removeMostServicingBrokersForNamespace(bundle, + brokerCandidateCache, + brokerToNamespaceToBundleRange); if (log.isDebugEnabled()) { - log.debug("Selected broker {} from candidate brokers {}", broker, brokerCandidateCache); + log.debug("enable distribute bundles evenly to candidate-brokers, broker candidate count={}", + brokerCandidateCache.size()); } + } - if (!broker.isPresent()) { - // No brokers available - return broker; - } + log.info("{} brokers being considered for assignment of {}", brokerCandidateCache.size(), bundle); - final double overloadThreshold = conf.getLoadBalancerBrokerOverloadedThresholdPercentage() / 100.0; - final double maxUsage = loadData.getBrokerData().get(broker.get()).getLocalData().getMaxResourceUsage(); - if (maxUsage > overloadThreshold) { - // All brokers that were in the filtered list were overloaded, so check if there is a better broker - LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, - getAvailableBrokers(), - brokerTopicLoadingPredicate); - Optional brokerTmp = - placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf); - if (brokerTmp.isPresent()) { - broker = brokerTmp; - } + // Use the filter pipeline to finalize broker candidates. + try { + for (BrokerFilter filter : filterPipeline) { + filter.filter(brokerCandidateCache, data, loadData, conf); } + } catch (BrokerFilterException x) { + // restore the list of brokers to the full set + LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, + getAvailableBrokers(), + brokerTopicLoadingPredicate); + } - // Add new bundle to preallocated. - loadData.getBrokerData().get(broker.get()).getPreallocatedBundleData().put(bundle, data); - preallocatedBundleToBroker.put(bundle, broker.get()); + if (brokerCandidateCache.isEmpty()) { + // restore the list of brokers to the full set + LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, + getAvailableBrokers(), + brokerTopicLoadingPredicate); + } - final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle); - final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle); - final ConcurrentOpenHashMap> namespaceToBundleRange = - brokerToNamespaceToBundleRange - .computeIfAbsent(broker.get(), - k -> ConcurrentOpenHashMap.>newBuilder() - .build()); - synchronized (namespaceToBundleRange) { - namespaceToBundleRange.computeIfAbsent(namespaceName, - k -> ConcurrentOpenHashSet.newBuilder().build()) - .add(bundleRange); - } + // Choose a broker among the potentially smaller filtered list, when possible + Optional broker = placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf); + if (log.isDebugEnabled()) { + log.debug("Selected broker {} from candidate brokers {}", broker, brokerCandidateCache); + } + + if (!broker.isPresent()) { + // No brokers available return broker; } - } finally { - selectBrokerForAssignment.observe(System.nanoTime() - startTime, TimeUnit.NANOSECONDS); + + final double overloadThreshold = conf.getLoadBalancerBrokerOverloadedThresholdPercentage() / 100.0; + final double maxUsage = getMaxResourceUsageWithWeight( + loadData.getBrokerData().get(broker.get()).getLocalData(), conf); + if (maxUsage > overloadThreshold) { + // All brokers that were in the filtered list were overloaded, so check if there is a better broker + LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, + getAvailableBrokers(), + brokerTopicLoadingPredicate); + Optional brokerTmp = + placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf); + if (brokerTmp.isPresent()) { + broker = brokerTmp; + } + } + return broker; } } @@ -945,14 +955,14 @@ public void start() throws PulsarServerException { // At this point, the ports will be updated with the real port number that the server was assigned Map protocolData = pulsar.getProtocolDataToAdvertise(); - lastData = new LocalBrokerData(pulsar.getSafeWebServiceAddress(), pulsar.getWebServiceAddressTls(), + lastData = new LocalBrokerData(pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls(), pulsar.getAdvertisedListeners()); lastData.setProtocols(protocolData); // configure broker-topic mode lastData.setPersistentTopicsEnabled(pulsar.getConfiguration().isEnablePersistentTopics()); lastData.setNonPersistentTopicsEnabled(pulsar.getConfiguration().isEnableNonPersistentTopics()); - localData = new LocalBrokerData(pulsar.getSafeWebServiceAddress(), pulsar.getWebServiceAddressTls(), + localData = new LocalBrokerData(pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls(), pulsar.getAdvertisedListeners()); localData.setProtocols(protocolData); localData.setBrokerVersionString(pulsar.getBrokerVersion()); @@ -961,15 +971,15 @@ public void start() throws PulsarServerException { localData.setNonPersistentTopicsEnabled(pulsar.getConfiguration().isEnableNonPersistentTopics()); localData.setLoadManagerClassName(conf.getLoadManagerClassName()); - String lookupServiceAddress = pulsar.getLookupServiceAddress(); - brokerZnodePath = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + lookupServiceAddress; - final String timeAverageZPath = TIME_AVERAGE_BROKER_ZPATH + "/" + lookupServiceAddress; + String brokerId = pulsar.getBrokerId(); + brokerZnodePath = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + brokerId; updateLocalBrokerData(); brokerDataLock = brokersData.acquireLock(brokerZnodePath, localData).join(); - - timeAverageBrokerDataCache.readModifyUpdateOrCreate(timeAverageZPath, - __ -> new TimeAverageBrokerData()).join(); + pulsarResources.getLoadBalanceResources() + .getBrokerTimeAverageDataResources() + .updateTimeAverageBrokerData(brokerId, new TimeAverageBrokerData()) + .join(); updateAll(); } catch (Exception e) { log.error("Unable to acquire lock for broker: [{}]", brokerZnodePath, e); @@ -1106,6 +1116,32 @@ public void writeBrokerDataOnZooKeeper(boolean force) { } } + /** + * sort bundles by load and select topK bundles for each broker. + * @return the number of bundles selected + */ + private int selectTopKBundle() { + bundleArr.clear(); + bundleArr.addAll(loadData.getBundleData().entrySet()); + + int maxNumberOfBundlesInBundleLoadReport = pulsar.getConfiguration() + .getLoadBalancerMaxNumberOfBundlesInBundleLoadReport(); + if (maxNumberOfBundlesInBundleLoadReport <= 0) { + // select all bundle + return bundleArr.size(); + } else { + // select topK bundle for each broker, so select topK * brokerCount bundle in total + int brokerCount = Math.max(1, loadData.getBrokerData().size()); + int updateBundleCount = Math.min(maxNumberOfBundlesInBundleLoadReport * brokerCount, bundleArr.size()); + if (updateBundleCount == 0) { + // no bundle to update + return 0; + } + TopKBundles.partitionSort(bundleArr, updateBundleCount); + return updateBundleCount; + } + } + /** * As the leader broker, write bundle data aggregated from all brokers to metadata store. */ @@ -1115,20 +1151,20 @@ public void writeBundleDataOnZooKeeper() { // Write the bundle data to metadata store. List> futures = new ArrayList<>(); - for (Map.Entry entry : loadData.getBundleData().entrySet()) { - final String bundle = entry.getKey(); - final BundleData data = entry.getValue(); - futures.add(bundlesCache.readModifyUpdateOrCreate(getBundleDataPath(bundle), __ -> data) - .thenApply(__ -> null)); + // use synchronized to protect bundleArr. + synchronized (bundleArr) { + int updateBundleCount = selectTopKBundle(); + bundleArr.stream().limit(updateBundleCount).forEach(entry -> futures.add( + pulsarResources.getLoadBalanceResources().getBundleDataResources().updateBundleData( + entry.getKey(), (BundleData) entry.getValue()))); } // Write the time average broker data to metadata store. for (Map.Entry entry : loadData.getBrokerData().entrySet()) { final String broker = entry.getKey(); final TimeAverageBrokerData data = entry.getValue().getTimeAverageData(); - futures.add(timeAverageBrokerDataCache.readModifyUpdateOrCreate( - TIME_AVERAGE_BROKER_ZPATH + "/" + broker, __ -> data) - .thenApply(__ -> null)); + futures.add(pulsarResources.getLoadBalanceResources() + .getBrokerTimeAverageDataResources().updateTimeAverageBrokerData(broker, data)); } try { @@ -1140,7 +1176,7 @@ public void writeBundleDataOnZooKeeper() { private void deleteBundleDataFromMetadataStore(String bundle) { try { - bundlesCache.delete(getBundleDataPath(bundle)).join(); + pulsarResources.getLoadBalanceResources().getBundleDataResources().deleteBundleData(bundle).join(); } catch (Exception e) { if (!(e.getCause() instanceof NotFoundException)) { log.warn("Failed to delete bundle-data {} from metadata store", bundle, e); @@ -1149,13 +1185,13 @@ private void deleteBundleDataFromMetadataStore(String bundle) { } private void deleteTimeAverageDataFromMetadataStoreAsync(String broker) { - final String timeAverageZPath = TIME_AVERAGE_BROKER_ZPATH + "/" + broker; - timeAverageBrokerDataCache.delete(timeAverageZPath).whenComplete((__, ex) -> { - if (ex != null && !(ex.getCause() instanceof MetadataStoreException.NotFoundException)) { - log.warn("Failed to delete dead broker {} time " - + "average data from metadata store", broker, ex); - } - }); + pulsarResources.getLoadBalanceResources() + .getBrokerTimeAverageDataResources().deleteTimeAverageBrokerData(broker).whenComplete((__, ex) -> { + if (ex != null && !(ex.getCause() instanceof MetadataStoreException.NotFoundException)) { + log.warn("Failed to delete dead broker {} time " + + "average data from metadata store", broker, ex); + } + }); } @Override @@ -1197,7 +1233,6 @@ public String setNamespaceBundleAffinity(String bundle, String broker) { if (StringUtils.isBlank(broker)) { return this.bundleBrokerAffinityMap.remove(bundle); } - broker = broker.replaceFirst("http[s]?://", ""); return this.bundleBrokerAffinityMap.put(bundle, broker); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java index c61d39cf3159a..c8d81bda1bc13 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerWrapper.java @@ -19,7 +19,6 @@ package org.apache.pulsar.broker.loadbalance.impl; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -32,7 +31,6 @@ import org.apache.pulsar.common.naming.ServiceUnitId; import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.policies.data.loadbalancer.LoadManagerReport; -import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; /** * Wrapper class allowing classes of instance ModularLoadManager to be compatible with the interface LoadManager. @@ -75,20 +73,6 @@ public Optional getLeastLoaded(final ServiceUnitId serviceUnit) { return leastLoadedBroker.map(this::buildBrokerResourceUnit); } - private String getBrokerWebServiceUrl(String broker) { - LocalBrokerData localData = (loadManager).getBrokerLocalData(broker); - if (localData != null) { - return localData.getWebServiceUrl() != null ? localData.getWebServiceUrl() - : localData.getWebServiceUrlTls(); - } - return String.format("http://%s", broker); - } - - private String getBrokerZnodeName(String broker, String webServiceUrl) { - String scheme = webServiceUrl.substring(0, webServiceUrl.indexOf("://")); - return String.format("%s://%s", scheme, broker); - } - @Override public List getLoadBalancingMetrics() { return loadManager.getLoadBalancingMetrics(); @@ -149,10 +133,7 @@ public CompletableFuture> getAvailableBrokersAsync() { } private SimpleResourceUnit buildBrokerResourceUnit (String broker) { - String webServiceUrl = getBrokerWebServiceUrl(broker); - String brokerZnodeName = getBrokerZnodeName(broker, webServiceUrl); - return new SimpleResourceUnit(webServiceUrl, - new PulsarResourceDescription(), Map.of(ResourceUnit.PROPERTY_KEY_BROKER_ZNODE_NAME, brokerZnodeName)); + return new SimpleResourceUnit(broker, new PulsarResourceDescription()); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedder.java index a4eb5077224ce..fb31548227b31 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedder.java @@ -36,7 +36,8 @@ /** * Load shedding strategy which will attempt to shed exactly one bundle on brokers which are overloaded, that is, whose * maximum system resource usage exceeds loadBalancerBrokerOverloadedThresholdPercentage. To see which resources are - * considered when determining the maximum system resource, see {@link LocalBrokerData#getMaxResourceUsage()}. A bundle + * considered when determining the maximum system resource, see + * {@link LocalBrokerData#getMaxResourceUsageWithWeight(double, double, double, double)}. A bundle * is recommended for unloading off that broker if and only if the following conditions hold: The broker has at * least two bundles assigned and the broker has at least one bundle that has not been unloaded recently according to * LoadBalancerSheddingGracePeriodMinutes. The unloaded bundle will be the most expensive bundle in terms of message @@ -71,7 +72,11 @@ public Multimap findBundlesForUnloading(final LoadData loadData, loadData.getBrokerData().forEach((broker, brokerData) -> { final LocalBrokerData localData = brokerData.getLocalData(); - final double currentUsage = localData.getMaxResourceUsage(); + final double currentUsage = localData.getMaxResourceUsageWithWeight( + conf.getLoadBalancerCPUResourceWeight(), + conf.getLoadBalancerDirectMemoryResourceWeight(), + conf.getLoadBalancerBandwidthInResourceWeight(), + conf.getLoadBalancerBandwidthOutResourceWeight()); if (currentUsage < overloadThreshold) { if (log.isDebugEnabled()) { log.debug("[{}] Broker is not overloaded, ignoring at this point ({})", broker, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/PulsarResourceDescription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/PulsarResourceDescription.java index 1f87dac8ec0b3..f64c559038ac4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/PulsarResourceDescription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/PulsarResourceDescription.java @@ -110,7 +110,7 @@ public long calculateRank() { percentageUsage = (entry.getValue().usage / entry.getValue().limit) * 100; } // give equal weight to each resource - double resourceWeight = weight * percentageUsage; + int resourceWeight = (int) (weight * percentageUsage); // any resource usage over 75% doubles the whole weight per resource if (percentageUsage > throttle) { final int i = resourcesWithHighUsage++; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java index 5e99456971147..30a7359ce0eb8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java @@ -28,6 +28,7 @@ import com.google.common.collect.TreeMultimap; import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -47,6 +48,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; import org.apache.pulsar.broker.PulsarServerException; @@ -62,8 +64,6 @@ import org.apache.pulsar.common.policies.data.ResourceQuota; import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet; import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.Notification; @@ -107,10 +107,7 @@ public class SimpleLoadManagerImpl implements LoadManager, Consumer bundleGainsCache; private final Set bundleLossesCache; - // Map from brokers to namespaces to the bundle ranges in that namespace assigned to that broker. - // Used to distribute bundles within a namespace evenly across brokers. - private final ConcurrentOpenHashMap>> brokerToNamespaceToBundleRange; + private final BundleRangeCache brokerToNamespaceToBundleRange = new BundleRangeCache(); // CPU usage per msg/sec private double realtimeCpuLoadFactor = 0.025; @@ -205,21 +202,17 @@ public SimpleLoadManagerImpl() { bundleLossesCache = new HashSet<>(); brokerCandidateCache = new HashSet<>(); availableBrokersCache = new HashSet<>(); - brokerToNamespaceToBundleRange = - ConcurrentOpenHashMap.>>newBuilder() - .build(); this.brokerTopicLoadingPredicate = new BrokerTopicLoadingPredicate() { @Override - public boolean isEnablePersistentTopics(String brokerUrl) { - ResourceUnit ru = new SimpleResourceUnit(brokerUrl, new PulsarResourceDescription()); + public boolean isEnablePersistentTopics(String brokerId) { + ResourceUnit ru = new SimpleResourceUnit(brokerId, new PulsarResourceDescription()); LoadReport loadReport = currentLoadReports.get(ru); return loadReport != null && loadReport.isPersistentTopicsEnabled(); } @Override - public boolean isEnableNonPersistentTopics(String brokerUrl) { - ResourceUnit ru = new SimpleResourceUnit(brokerUrl, new PulsarResourceDescription()); + public boolean isEnableNonPersistentTopics(String brokerId) { + ResourceUnit ru = new SimpleResourceUnit(brokerId, new PulsarResourceDescription()); LoadReport loadReport = currentLoadReports.get(ru); return loadReport != null && loadReport.isNonPersistentTopicsEnabled(); } @@ -234,7 +227,7 @@ public void initialize(final PulsarService pulsar) { brokerHostUsage = new GenericBrokerHostUsageImpl(pulsar); } this.policies = new SimpleResourceAllocationPolicies(pulsar); - lastLoadReport = new LoadReport(pulsar.getSafeWebServiceAddress(), pulsar.getWebServiceAddressTls(), + lastLoadReport = new LoadReport(pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls()); lastLoadReport.setProtocols(pulsar.getProtocolDataToAdvertise()); lastLoadReport.setPersistentTopicsEnabled(pulsar.getConfiguration().isEnablePersistentTopics()); @@ -266,8 +259,8 @@ public SimpleLoadManagerImpl(PulsarService pulsar) { @Override public void start() throws PulsarServerException { // Register the brokers in metadata store - String lookupServiceAddress = pulsar.getLookupServiceAddress(); - String brokerLockPath = LOADBALANCE_BROKERS_ROOT + "/" + lookupServiceAddress; + String brokerId = pulsar.getBrokerId(); + String brokerLockPath = LOADBALANCE_BROKERS_ROOT + "/" + brokerId; try { LoadReport loadReport = null; @@ -653,7 +646,6 @@ public void writeResourceQuotasToZooKeeper() throws Exception { */ private synchronized void doLoadRanking() { ResourceUnitRanking.setCpuUsageByMsgRate(this.realtimeCpuLoadFactor); - String hostname = pulsar.getAdvertisedAddress(); String strategy = this.getLoadBalancerPlacementStrategy(); log.info("doLoadRanking - load balancing strategy: {}", strategy); if (!currentLoadReports.isEmpty()) { @@ -702,8 +694,8 @@ private synchronized void doLoadRanking() { } // update metrics - if (resourceUnit.getResourceId().contains(hostname)) { - updateLoadBalancingMetrics(hostname, finalRank, ranking); + if (resourceUnit.getResourceId().equals(pulsar.getBrokerId())) { + updateLoadBalancingMetrics(pulsar.getAdvertisedAddress(), finalRank, ranking); } } updateBrokerToNamespaceToBundle(); @@ -711,7 +703,7 @@ private synchronized void doLoadRanking() { this.resourceUnitRankings = newResourceUnitRankings; } else { log.info("Leader broker[{}] No ResourceUnits to rank this run, Using Old Ranking", - pulsar.getSafeWebServiceAddress()); + pulsar.getBrokerId()); } } @@ -854,14 +846,7 @@ private synchronized ResourceUnit findBrokerForPlacement(Multimap ConcurrentOpenHashMap.>newBuilder() - .build()) - .computeIfAbsent(namespaceName, k -> - ConcurrentOpenHashSet.newBuilder().build()) - .add(bundleRange); + brokerToNamespaceToBundleRange.add(selectedRU.getResourceId(), namespaceName, bundleRange); ranking.addPreAllocatedServiceUnit(serviceUnitId, quota); resourceUnitRankings.put(selectedRU, ranking); } @@ -876,7 +861,7 @@ private Multimap getFinalCandidates(ServiceUnitId serviceUni availableBrokersCache.clear(); for (final Set resourceUnits : availableBrokers.values()) { for (final ResourceUnit resourceUnit : resourceUnits) { - availableBrokersCache.add(resourceUnit.getResourceId().replace("http://", "")); + availableBrokersCache.add(resourceUnit.getResourceId()); } } brokerCandidateCache.clear(); @@ -899,7 +884,7 @@ private Multimap getFinalCandidates(ServiceUnitId serviceUni final Long rank = entry.getKey(); final Set resourceUnits = entry.getValue(); for (final ResourceUnit resourceUnit : resourceUnits) { - if (brokerCandidateCache.contains(resourceUnit.getResourceId().replace("http://", ""))) { + if (brokerCandidateCache.contains(resourceUnit.getResourceId())) { result.put(rank, resourceUnit); } } @@ -928,8 +913,7 @@ private Map> getAvailableBrokers(ServiceUnitId serviceUn availableBrokers = new HashMap<>(); for (String broker : activeBrokers) { - ResourceUnit resourceUnit = new SimpleResourceUnit(String.format("http://%s", broker), - new PulsarResourceDescription()); + ResourceUnit resourceUnit = new SimpleResourceUnit(broker, new PulsarResourceDescription()); availableBrokers.computeIfAbsent(0L, key -> new TreeSet<>()).add(resourceUnit); } log.info("Choosing at random from broker list: [{}]", availableBrokers.values()); @@ -956,7 +940,7 @@ private synchronized ResourceUnit getLeastLoadedBroker(ServiceUnitId serviceUnit Iterator> candidateIterator = finalCandidates.entries().iterator(); while (candidateIterator.hasNext()) { Map.Entry candidate = candidateIterator.next(); - String candidateBrokerName = candidate.getValue().getResourceId().replace("http://", ""); + String candidateBrokerName = candidate.getValue().getResourceId(); if (!activeBrokers.contains(candidateBrokerName)) { candidateIterator.remove(); // Current candidate points to an inactive broker, so remove it } @@ -1005,8 +989,7 @@ private void updateRanking() { try { String key = String.format("%s/%s", LOADBALANCE_BROKERS_ROOT, broker); LoadReport lr = loadReports.readLock(key).join().get(); - ResourceUnit ru = new SimpleResourceUnit(String.format("http://%s", lr.getName()), - fromLoadReport(lr)); + ResourceUnit ru = new SimpleResourceUnit(lr.getName(), fromLoadReport(lr)); this.currentLoadReports.put(ru, lr); } catch (Exception e) { log.warn("Error reading load report from Cache for broker - [{}], [{}]", broker, e); @@ -1072,13 +1055,13 @@ public LoadReport generateLoadReport() throws Exception { private LoadReport generateLoadReportForcefully() throws Exception { synchronized (bundleGainsCache) { try { - LoadReport loadReport = new LoadReport(pulsar.getSafeWebServiceAddress(), + LoadReport loadReport = new LoadReport(pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls()); loadReport.setProtocols(pulsar.getProtocolDataToAdvertise()); loadReport.setNonPersistentTopicsEnabled(pulsar.getConfiguration().isEnableNonPersistentTopics()); loadReport.setPersistentTopicsEnabled(pulsar.getConfiguration().isEnablePersistentTopics()); - loadReport.setName(pulsar.getLookupServiceAddress()); + loadReport.setName(pulsar.getBrokerId()); loadReport.setBrokerVersionString(pulsar.getBrokerVersion()); SystemResourceUsage systemResourceUsage = this.getSystemResourceUsage(); @@ -1121,8 +1104,8 @@ private LoadReport generateLoadReportForcefully() throws Exception { loadReport.setAllocatedMsgRateIn(allocatedQuota.getMsgRateIn()); loadReport.setAllocatedMsgRateOut(allocatedQuota.getMsgRateOut()); - final ResourceUnit resourceUnit = new SimpleResourceUnit( - String.format("http://%s", loadReport.getName()), fromLoadReport(loadReport)); + final ResourceUnit resourceUnit = + new SimpleResourceUnit(loadReport.getName(), fromLoadReport(loadReport)); Set preAllocatedBundles; if (resourceUnitRankings.containsKey(resourceUnit)) { preAllocatedBundles = resourceUnitRankings.get(resourceUnit).getPreAllocatedBundles(); @@ -1275,15 +1258,8 @@ private synchronized void updateBrokerToNamespaceToBundle() { final String broker = resourceUnit.getResourceId(); final Set loadedBundles = ranking.getLoadedBundles(); final Set preallocatedBundles = resourceUnitRankings.get(resourceUnit).getPreAllocatedBundles(); - final ConcurrentOpenHashMap> namespaceToBundleRange = - brokerToNamespaceToBundleRange - .computeIfAbsent(broker.replace("http://", ""), - k -> ConcurrentOpenHashMap.>newBuilder() - .build()); - namespaceToBundleRange.clear(); - LoadManagerShared.fillNamespaceToBundlesMap(loadedBundles, namespaceToBundleRange); - LoadManagerShared.fillNamespaceToBundlesMap(preallocatedBundles, namespaceToBundleRange); + brokerToNamespaceToBundleRange.reloadFromBundles(broker, + Stream.of(loadedBundles, preallocatedBundles).flatMap(Collection::stream)); }); } @@ -1455,14 +1431,15 @@ public String setNamespaceBundleAffinity(String bundle, String broker) { if (StringUtils.isBlank(broker)) { return this.bundleBrokerAffinityMap.remove(bundle); } - broker = broker.replaceFirst("http[s]?://", ""); return this.bundleBrokerAffinityMap.put(bundle, broker); } @Override public void stop() throws PulsarServerException { try { - loadReports.close(); + if (loadReports != null) { + loadReports.close(); + } scheduler.shutdownNow(); scheduler.awaitTermination(5, TimeUnit.SECONDS); } catch (Exception e) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleResourceAllocationPolicies.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleResourceAllocationPolicies.java index 4a1577a4e28b6..25fbc152b1163 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleResourceAllocationPolicies.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleResourceAllocationPolicies.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.loadbalance.LoadReport; import org.apache.pulsar.broker.loadbalance.ResourceUnit; @@ -53,6 +54,20 @@ private Optional getIsolationPolicies(String cluster } } + private CompletableFuture> getIsolationPoliciesAsync(String clusterName) { + return this.pulsar.getPulsarResources().getNamespaceResources() + .getIsolationPolicies().getIsolationDataPoliciesAsync(clusterName); + } + + public CompletableFuture areIsolationPoliciesPresentAsync(NamespaceName namespace) { + return getIsolationPoliciesAsync(pulsar.getConfiguration().getClusterName()) + .thenApply(policies -> { + return policies.filter(isolationPolicies -> + isolationPolicies.getPolicyByNamespace(namespace) != null) + .isPresent(); + }); + } + public boolean areIsolationPoliciesPresent(NamespaceName namespace) { try { Optional policies = diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java index 86df49f952674..aa556cd0ca5d3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java @@ -41,7 +41,7 @@ * configured threshold. As a consequence, this strategy tends to distribute load among all brokers. It does this by * first computing the average resource usage per broker for the whole cluster. The resource usage for each broker is * calculated using the following method: - * {@link LocalBrokerData#getMaxResourceUsageWithWeight(double, double, double, double, double)}. The weights + * {@link LocalBrokerData#getMaxResourceUsageWithWeight(double, double, double, double)}. The weights * for each resource are configurable. Historical observations are included in the running average based on the broker's * setting for loadBalancerHistoryResourcePercentage. Once the average resource usage is calculated, a broker's * current/historical usage is compared to the average broker usage. If a broker's usage is greater than the average @@ -79,7 +79,8 @@ public synchronized Multimap findBundlesForUnloading(final LoadD final double currentUsage = brokerAvgResourceUsage.getOrDefault(broker, 0.0); if (currentUsage < avgUsage + threshold) { if (log.isDebugEnabled()) { - log.debug("[{}] broker is not overloaded, ignoring at this point", broker); + log.debug("[{}] broker is not overloaded, ignoring at this point ({})", broker, + localData.printResourceUsage()); } return; } @@ -92,17 +93,19 @@ public synchronized Multimap findBundlesForUnloading(final LoadD if (minimumThroughputToOffload < minThroughputThreshold) { if (log.isDebugEnabled()) { log.debug("[{}] broker is planning to shed throughput {} MByte/s less than " - + "minimumThroughputThreshold {} MByte/s, skipping bundle unload.", - broker, minimumThroughputToOffload / MB, minThroughputThreshold / MB); + + "minimumThroughputThreshold {} MByte/s, skipping bundle unload ({})", + broker, minimumThroughputToOffload / MB, minThroughputThreshold / MB, + localData.printResourceUsage()); } return; } log.info( - "Attempting to shed load on {}, which has max resource usage above avgUsage and threshold {}%" - + " > {}% + {}% -- Offloading at least {} MByte/s of traffic, left throughput {} MByte/s", + "Attempting to shed load on {}, which has max resource usage above avgUsage and threshold {}%" + + " > {}% + {}% -- Offloading at least {} MByte/s of traffic," + + " left throughput {} MByte/s ({})", broker, 100 * currentUsage, 100 * avgUsage, 100 * threshold, minimumThroughputToOffload / MB, - (brokerCurrentThroughput - minimumThroughputToOffload) / MB); + (brokerCurrentThroughput - minimumThroughputToOffload) / MB, localData.printResourceUsage()); if (localData.getBundles().size() > 1) { filterAndSelectBundle(loadData, recentlyUnloadedBundles, broker, localData, minimumThroughputToOffload); @@ -170,8 +173,8 @@ private double updateAvgResourceUsage(String broker, LocalBrokerData localBroker double resourceUsage = localBrokerData.getMaxResourceUsageWithWeight( conf.getLoadBalancerCPUResourceWeight(), conf.getLoadBalancerDirectMemoryResourceWeight(), - conf.getLoadBalancerBandwithInResourceWeight(), - conf.getLoadBalancerBandwithOutResourceWeight()); + conf.getLoadBalancerBandwidthInResourceWeight(), + conf.getLoadBalancerBandwidthOutResourceWeight()); historyUsage = historyUsage == null ? resourceUsage : historyUsage * historyPercentage + (1 - historyPercentage) * resourceUsage; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/UniformLoadShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/UniformLoadShedder.java index b92af5b7c69f3..78bdbc5711201 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/UniformLoadShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/UniformLoadShedder.java @@ -25,7 +25,7 @@ import org.apache.commons.lang3.mutable.MutableDouble; import org.apache.commons.lang3.mutable.MutableInt; import org.apache.commons.lang3.mutable.MutableObject; -import org.apache.commons.lang3.tuple.Triple; +import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.LoadData; import org.apache.pulsar.broker.loadbalance.LoadSheddingStrategy; @@ -36,7 +36,7 @@ /** * This strategy tends to distribute load uniformly across all brokers. This strategy checks load difference between - * broker with highest load and broker with lowest load. If the difference is higher than configured thresholds + * broker with the highest load and broker with the lowest load. If the difference is higher than configured thresholds * {@link ServiceConfiguration#getLoadBalancerMsgRateDifferenceShedderThreshold()} or * {@link ServiceConfiguration#getLoadBalancerMsgThroughputMultiplierDifferenceShedderThreshold()} then it finds out * bundles which can be unloaded to distribute traffic evenly across all brokers. @@ -63,25 +63,37 @@ public Multimap findBundlesForUnloading(final LoadData loadData, Map loadBundleData = loadData.getBundleDataForLoadShedding(); Map recentlyUnloadedBundles = loadData.getRecentlyUnloadedBundles(); - MutableObject overloadedBroker = new MutableObject<>(); - MutableObject underloadedBroker = new MutableObject<>(); + MutableObject msgRateOverloadedBroker = new MutableObject<>(); + MutableObject msgThroughputOverloadedBroker = new MutableObject<>(); + MutableObject msgRateUnderloadedBroker = new MutableObject<>(); + MutableObject msgThroughputUnderloadedBroker = new MutableObject<>(); MutableDouble maxMsgRate = new MutableDouble(-1); - MutableDouble maxThroughputRate = new MutableDouble(-1); + MutableDouble maxThroughput = new MutableDouble(-1); MutableDouble minMsgRate = new MutableDouble(Integer.MAX_VALUE); - MutableDouble minThroughputRate = new MutableDouble(Integer.MAX_VALUE); + MutableDouble minThroughput = new MutableDouble(Integer.MAX_VALUE); + brokersData.forEach((broker, data) -> { double msgRate = data.getLocalData().getMsgRateIn() + data.getLocalData().getMsgRateOut(); double throughputRate = data.getLocalData().getMsgThroughputIn() + data.getLocalData().getMsgThroughputOut(); - if (msgRate > maxMsgRate.getValue() || throughputRate > maxThroughputRate.getValue()) { - overloadedBroker.setValue(broker); + if (msgRate > maxMsgRate.getValue()) { + msgRateOverloadedBroker.setValue(broker); maxMsgRate.setValue(msgRate); - maxThroughputRate.setValue(throughputRate); } - if (msgRate < minMsgRate.getValue() || throughputRate < minThroughputRate.getValue()) { - underloadedBroker.setValue(broker); + + if (throughputRate > maxThroughput.getValue()) { + msgThroughputOverloadedBroker.setValue(broker); + maxThroughput.setValue(throughputRate); + } + + if (msgRate < minMsgRate.getValue()) { + msgRateUnderloadedBroker.setValue(broker); minMsgRate.setValue(msgRate); - minThroughputRate.setValue(throughputRate); + } + + if (throughputRate < minThroughput.getValue()) { + msgThroughputUnderloadedBroker.setValue(broker); + minThroughput.setValue(throughputRate); } }); @@ -91,12 +103,12 @@ public Multimap findBundlesForUnloading(final LoadData loadData, if (minMsgRate.getValue() <= EPS && minMsgRate.getValue() >= -EPS) { minMsgRate.setValue(1.0); } - if (minThroughputRate.getValue() <= EPS && minThroughputRate.getValue() >= -EPS) { - minThroughputRate.setValue(1.0); + if (minThroughput.getValue() <= EPS && minThroughput.getValue() >= -EPS) { + minThroughput.setValue(1.0); } double msgRateDifferencePercentage = ((maxMsgRate.getValue() - minMsgRate.getValue()) * 100) / (minMsgRate.getValue()); - double msgThroughputDifferenceRate = maxThroughputRate.getValue() / minThroughputRate.getValue(); + double msgThroughputDifferenceRate = maxThroughput.getValue() / minThroughput.getValue(); // if the threshold matches then find out how much load needs to be unloaded by considering number of msgRate // and throughput. @@ -105,66 +117,91 @@ public Multimap findBundlesForUnloading(final LoadData loadData, boolean isMsgThroughputThresholdExceeded = conf .getLoadBalancerMsgThroughputMultiplierDifferenceShedderThreshold() > 0 && msgThroughputDifferenceRate > conf - .getLoadBalancerMsgThroughputMultiplierDifferenceShedderThreshold(); + .getLoadBalancerMsgThroughputMultiplierDifferenceShedderThreshold(); if (isMsgRateThresholdExceeded || isMsgThroughputThresholdExceeded) { - if (log.isDebugEnabled()) { - log.debug( - "Found bundles for uniform load balancing. " - + "overloaded broker {} with (msgRate,throughput)= ({},{}) " - + "and underloaded broker {} with (msgRate,throughput)= ({},{})", - overloadedBroker.getValue(), maxMsgRate.getValue(), maxThroughputRate.getValue(), - underloadedBroker.getValue(), minMsgRate.getValue(), minThroughputRate.getValue()); - } MutableInt msgRateRequiredFromUnloadedBundles = new MutableInt( (int) ((maxMsgRate.getValue() - minMsgRate.getValue()) * conf.getMaxUnloadPercentage())); MutableInt msgThroughputRequiredFromUnloadedBundles = new MutableInt( - (int) ((maxThroughputRate.getValue() - minThroughputRate.getValue()) + (int) ((maxThroughput.getValue() - minThroughput.getValue()) * conf.getMaxUnloadPercentage())); - LocalBrokerData overloadedBrokerData = brokersData.get(overloadedBroker.getValue()).getLocalData(); - - if (overloadedBrokerData.getBundles().size() > 1 - && (msgRateRequiredFromUnloadedBundles.getValue() >= conf.getMinUnloadMessage() - || msgThroughputRequiredFromUnloadedBundles.getValue() >= conf.getMinUnloadMessageThroughput())) { - // Sort bundles by throughput, then pick the bundle which can help to reduce load uniformly with - // under-loaded broker - loadBundleData.entrySet().stream() - .filter(e -> overloadedBrokerData.getBundles().contains(e.getKey())) - .map((e) -> { - String bundle = e.getKey(); - BundleData bundleData = e.getValue(); - TimeAverageMessageData shortTermData = bundleData.getShortTermData(); - double throughput = isMsgRateThresholdExceeded - ? shortTermData.getMsgRateIn() + shortTermData.getMsgRateOut() - : shortTermData.getMsgThroughputIn() + shortTermData.getMsgThroughputOut(); - return Triple.of(bundle, bundleData, throughput); - }).filter(e -> !recentlyUnloadedBundles.containsKey(e.getLeft())) - .sorted((e1, e2) -> Double.compare(e2.getRight(), e1.getRight())).forEach((e) -> { - if (conf.getMaxUnloadBundleNumPerShedding() != -1 - && selectedBundlesCache.size() >= conf.getMaxUnloadBundleNumPerShedding()) { - return; - } - String bundle = e.getLeft(); - BundleData bundleData = e.getMiddle(); - TimeAverageMessageData shortTermData = bundleData.getShortTermData(); - double throughput = shortTermData.getMsgThroughputIn() - + shortTermData.getMsgThroughputOut(); - double bundleMsgRate = shortTermData.getMsgRateIn() + shortTermData.getMsgRateOut(); - if (isMsgRateThresholdExceeded) { + if (isMsgRateThresholdExceeded) { + if (log.isDebugEnabled()) { + log.debug("Found bundles for uniform load balancing. " + + "msgRate overloaded broker: {} with msgRate: {}, " + + "msgRate underloaded broker: {} with msgRate: {}", + msgRateOverloadedBroker.getValue(), maxMsgRate.getValue(), + msgRateUnderloadedBroker.getValue(), minMsgRate.getValue()); + } + LocalBrokerData overloadedBrokerData = + brokersData.get(msgRateOverloadedBroker.getValue()).getLocalData(); + if (overloadedBrokerData.getBundles().size() > 1 + && (msgRateRequiredFromUnloadedBundles.getValue() >= conf.getMinUnloadMessage())) { + // Sort bundles by msgRate, then pick the bundle which can help to reduce load uniformly with + // under-loaded broker + loadBundleData.entrySet().stream() + .filter(e -> overloadedBrokerData.getBundles().contains(e.getKey())) + .map((e) -> { + String bundle = e.getKey(); + TimeAverageMessageData shortTermData = e.getValue().getShortTermData(); + double msgRate = shortTermData.getMsgRateIn() + shortTermData.getMsgRateOut(); + return Pair.of(bundle, msgRate); + }).filter(e -> !recentlyUnloadedBundles.containsKey(e.getLeft())) + .sorted((e1, e2) -> Double.compare(e2.getRight(), e1.getRight())).forEach((e) -> { + if (conf.getMaxUnloadBundleNumPerShedding() != -1 + && selectedBundlesCache.size() >= conf.getMaxUnloadBundleNumPerShedding()) { + return; + } + String bundle = e.getLeft(); + double bundleMsgRate = e.getRight(); if (bundleMsgRate <= (msgRateRequiredFromUnloadedBundles.getValue() + 1000/* delta */)) { log.info("Found bundle to unload with msgRate {}", bundleMsgRate); msgRateRequiredFromUnloadedBundles.add(-bundleMsgRate); - selectedBundlesCache.put(overloadedBroker.getValue(), bundle); + selectedBundlesCache.put(msgRateOverloadedBroker.getValue(), bundle); } - } else { - if (throughput <= (msgThroughputRequiredFromUnloadedBundles.getValue())) { - log.info("Found bundle to unload with throughput {}", throughput); - msgThroughputRequiredFromUnloadedBundles.add(-throughput); - selectedBundlesCache.put(overloadedBroker.getValue(), bundle); + }); + } + } else { + if (log.isDebugEnabled()) { + log.debug("Found bundles for uniform load balancing. " + + "msgThroughput overloaded broker: {} with msgThroughput {}, " + + "msgThroughput underloaded broker: {} with msgThroughput: {}", + msgThroughputOverloadedBroker.getValue(), maxThroughput.getValue(), + msgThroughputUnderloadedBroker.getValue(), minThroughput.getValue()); + } + LocalBrokerData overloadedBrokerData = + brokersData.get(msgThroughputOverloadedBroker.getValue()).getLocalData(); + if (overloadedBrokerData.getBundles().size() > 1 + && + msgThroughputRequiredFromUnloadedBundles.getValue() >= conf.getMinUnloadMessageThroughput()) { + // Sort bundles by throughput, then pick the bundle which can help to reduce load uniformly with + // under-loaded broker + loadBundleData.entrySet().stream() + .filter(e -> overloadedBrokerData.getBundles().contains(e.getKey())) + .map((e) -> { + String bundle = e.getKey(); + TimeAverageMessageData shortTermData = e.getValue().getShortTermData(); + double msgThroughput = shortTermData.getMsgThroughputIn() + + shortTermData.getMsgThroughputOut(); + return Pair.of(bundle, msgThroughput); + }).filter(e -> !recentlyUnloadedBundles.containsKey(e.getLeft())) + .sorted((e1, e2) -> Double.compare(e2.getRight(), e1.getRight())).forEach((e) -> { + if (conf.getMaxUnloadBundleNumPerShedding() != -1 + && selectedBundlesCache.size() >= conf.getMaxUnloadBundleNumPerShedding()) { + return; } - } - }); + String bundle = e.getLeft(); + double msgThroughput = e.getRight(); + if (msgThroughput <= (msgThroughputRequiredFromUnloadedBundles.getValue() + + 1000/* delta */)) { + log.info("Found bundle to unload with msgThroughput {}", msgThroughput); + msgThroughputRequiredFromUnloadedBundles.add(-msgThroughput); + selectedBundlesCache.put(msgThroughputOverloadedBroker.getValue(), bundle); + } + }); + } + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/WRRPlacementStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/WRRPlacementStrategy.java index bee9ae6d5f00f..93b21028eb7b3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/WRRPlacementStrategy.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/WRRPlacementStrategy.java @@ -76,7 +76,7 @@ public ResourceUnit findBrokerForPlacement(Multimap finalCan } int weightedSelector = rand.nextInt(totalAvailability); log.debug("Generated Weighted Selector Number - [{}] ", weightedSelector); - int weightRangeSoFar = 0; + long weightRangeSoFar = 0; for (Map.Entry candidateOwner : finalCandidates.entries()) { weightRangeSoFar += candidateOwner.getKey(); if (weightedSelector < weightRangeSoFar) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java index bd70201cba55d..42f145d32aab1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java @@ -24,13 +24,15 @@ import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; +import java.util.Collections; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; import javax.ws.rs.Encoded; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Response; import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; @@ -48,6 +50,7 @@ import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.metadata.api.MetadataStoreException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,7 +59,7 @@ public class TopicLookupBase extends PulsarWebResource { private static final String LOOKUP_PATH_V1 = "/lookup/v2/destination/"; private static final String LOOKUP_PATH_V2 = "/lookup/v2/topic/"; - protected CompletableFuture internalLookupTopicAsync(TopicName topicName, boolean authoritative, + protected CompletableFuture internalLookupTopicAsync(final TopicName topicName, boolean authoritative, String listenerName) { if (!pulsar().getBrokerService().getLookupRequestSemaphore().tryAcquire()) { log.warn("No broker was found available for topic {}", topicName); @@ -66,20 +69,27 @@ protected CompletableFuture internalLookupTopicAsync(TopicName topic .thenCompose(__ -> validateGlobalNamespaceOwnershipAsync(topicName.getNamespaceObject())) .thenCompose(__ -> validateTopicOperationAsync(topicName, TopicOperation.LOOKUP, null)) .thenCompose(__ -> { + // Case-1: Non-persistent topic. // Currently, it's hard to check the non-persistent-non-partitioned topic, because it only exists // in the broker, it doesn't have metadata. If the topic is non-persistent and non-partitioned, - // we'll return the true flag. - CompletableFuture existFuture = (!topicName.isPersistent() && !topicName.isPartitioned()) - ? CompletableFuture.completedFuture(true) - : pulsar().getNamespaceService().checkTopicExists(topicName) - .thenCompose(exists -> exists ? CompletableFuture.completedFuture(true) - : pulsar().getBrokerService().isAllowAutoTopicCreationAsync(topicName)); - - return existFuture; + // we'll return the true flag. So either it is a partitioned topic or not, the result will be true. + if (!topicName.isPersistent()) { + return CompletableFuture.completedFuture(true); + } + // Case-2: Persistent topic. + return pulsar().getNamespaceService().checkTopicExists(topicName).thenCompose(info -> { + boolean exists = info.isExists(); + info.recycle(); + if (exists) { + return CompletableFuture.completedFuture(true); + } + return pulsar().getBrokerService().isAllowAutoTopicCreationAsync(topicName); + }); }) .thenCompose(exist -> { if (!exist) { - throw new RestException(Response.Status.NOT_FOUND, "Topic not found."); + throw new RestException(Response.Status.NOT_FOUND, + String.format("Topic not found %s", topicName.toString())); } CompletableFuture> lookupFuture = pulsar().getNamespaceService() .getBrokerServiceUrlAsync(topicName, @@ -131,10 +141,10 @@ protected CompletableFuture internalLookupTopicAsync(TopicName topic pulsar().getBrokerService().getLookupRequestSemaphore().release(); return result.getLookupData(); } - }).exceptionally(ex->{ - pulsar().getBrokerService().getLookupRequestSemaphore().release(); - throw FutureUtil.wrapToCompletionException(ex); }); + }).exceptionally(ex -> { + pulsar().getBrokerService().getLookupRequestSemaphore().release(); + throw FutureUtil.wrapToCompletionException(ex); }); } @@ -172,7 +182,7 @@ protected String internalGetNamespaceBundle(TopicName topicName) { public static CompletableFuture lookupTopicAsync(PulsarService pulsarService, TopicName topicName, boolean authoritative, String clientAppId, AuthenticationDataSource authenticationData, long requestId) { return lookupTopicAsync(pulsarService, topicName, authoritative, clientAppId, - authenticationData, requestId, null); + authenticationData, requestId, null, Collections.emptyMap()); } /** @@ -200,7 +210,8 @@ public static CompletableFuture lookupTopicAsync(PulsarService pulsarSe public static CompletableFuture lookupTopicAsync(PulsarService pulsarService, TopicName topicName, boolean authoritative, String clientAppId, AuthenticationDataSource authenticationData, - long requestId, final String advertisedListenerName) { + long requestId, final String advertisedListenerName, + Map properties) { final CompletableFuture validationFuture = new CompletableFuture<>(); final CompletableFuture lookupfuture = new CompletableFuture<>(); @@ -291,6 +302,7 @@ public static CompletableFuture lookupTopicAsync(PulsarService pulsarSe .authoritative(authoritative) .advertisedListenerName(advertisedListenerName) .loadTopicsInBundle(true) + .properties(properties) .build(); pulsarService.getNamespaceService().getBrokerServiceUrlAsync(topicName, options) .thenAccept(lookupResult -> { @@ -318,35 +330,40 @@ public static CompletableFuture lookupTopicAsync(PulsarService pulsarSe requestId, shouldRedirectThroughServiceUrl(conf, lookupData))); } }).exceptionally(ex -> { - if (ex instanceof CompletionException && ex.getCause() instanceof IllegalStateException) { - log.info("Failed to lookup {} for topic {} with error {}", clientAppId, - topicName.toString(), ex.getCause().getMessage()); - } else { - log.warn("Failed to lookup {} for topic {} with error {}", clientAppId, - topicName.toString(), ex.getMessage(), ex); - } - lookupfuture.complete( - newLookupErrorResponse(ServerError.ServiceNotReady, ex.getMessage(), requestId)); - return null; - }); + handleLookupError(lookupfuture, topicName.toString(), clientAppId, requestId, ex); + return null; + }); } - }).exceptionally(ex -> { - if (ex instanceof CompletionException && ex.getCause() instanceof IllegalStateException) { - log.info("Failed to lookup {} for topic {} with error {}", clientAppId, topicName.toString(), - ex.getCause().getMessage()); - } else { - log.warn("Failed to lookup {} for topic {} with error {}", clientAppId, topicName.toString(), - ex.getMessage(), ex); - } - - lookupfuture.complete(newLookupErrorResponse(ServerError.ServiceNotReady, ex.getMessage(), requestId)); + handleLookupError(lookupfuture, topicName.toString(), clientAppId, requestId, ex); return null; }); return lookupfuture; } + private static void handleLookupError(CompletableFuture lookupFuture, String topicName, String clientAppId, + long requestId, Throwable ex){ + Throwable unwrapEx = FutureUtil.unwrapCompletionException(ex); + final String errorMsg = unwrapEx.getMessage(); + if (unwrapEx instanceof PulsarServerException) { + unwrapEx = FutureUtil.unwrapCompletionException(unwrapEx.getCause()); + } + if (unwrapEx instanceof IllegalStateException) { + // Current broker still hold the bundle's lock, but the bundle is being unloading. + log.info("Failed to lookup {} for topic {} with error {}", clientAppId, topicName, errorMsg); + lookupFuture.complete(newLookupErrorResponse(ServerError.MetadataError, errorMsg, requestId)); + } else if (unwrapEx instanceof MetadataStoreException) { + // Load bundle ownership or acquire lock failed. + // Differ with "IllegalStateException", print warning log. + log.warn("Failed to lookup {} for topic {} with error {}", clientAppId, topicName, errorMsg); + lookupFuture.complete(newLookupErrorResponse(ServerError.MetadataError, errorMsg, requestId)); + } else { + log.warn("Failed to lookup {} for topic {} with error {}", clientAppId, topicName, errorMsg); + lookupFuture.complete(newLookupErrorResponse(ServerError.ServiceNotReady, errorMsg, requestId)); + } + } + protected TopicName getTopicName(String topicDomain, String tenant, String cluster, String namespace, @Encoded String encodedTopic) { String decodedName = Codec.decode(encodedTopic); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/LookupOptions.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/LookupOptions.java index 431266682c51c..be5450646329d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/LookupOptions.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/LookupOptions.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.namespace; +import java.util.Map; import lombok.Builder; import lombok.Data; import org.apache.commons.lang3.StringUtils; @@ -46,6 +47,7 @@ public class LookupOptions { private final boolean requestHttps; private final String advertisedListenerName; + private final Map properties; public boolean hasAdvertisedListenerName() { return StringUtils.isNotBlank(advertisedListenerName); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceBundleSplitListener.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceBundleSplitListener.java new file mode 100644 index 0000000000000..a3312f5689e38 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceBundleSplitListener.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.namespace; + +import java.util.function.Predicate; +import org.apache.pulsar.common.naming.NamespaceBundle; + +/** + * Listener for NamespaceBundle split. + */ +public interface NamespaceBundleSplitListener extends Predicate { + void onSplit(NamespaceBundle bundle); +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 9d8d9e3890a19..b2ee299bb030e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -23,8 +23,12 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.pulsar.client.api.PulsarClientException.FailedFeatureCheck.SupportsGetPartitionedMetadataWithoutAutoCreation; import static org.apache.pulsar.common.naming.NamespaceName.SYSTEM_NAMESPACE; import com.google.common.hash.Hashing; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.DoubleHistogram; import io.prometheus.client.Counter; import java.net.URI; import java.net.URL; @@ -37,18 +41,23 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; +import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; @@ -67,6 +76,7 @@ import org.apache.pulsar.broker.stats.prometheus.metrics.Summary; import org.apache.pulsar.broker.web.PulsarWebResource; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; @@ -96,10 +106,12 @@ import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; import org.apache.pulsar.common.policies.impl.NamespaceIsolationPolicies; +import org.apache.pulsar.common.stats.MetricsUtil; +import org.apache.pulsar.common.topics.TopicList; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.opentelemetry.annotations.PulsarDeprecatedMetric; import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener; import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; import org.slf4j.Logger; @@ -116,6 +128,7 @@ * * @see org.apache.pulsar.broker.PulsarService */ +@Slf4j public class NamespaceService implements AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(NamespaceService.class); @@ -132,43 +145,74 @@ public class NamespaceService implements AutoCloseable { public static final Pattern HEARTBEAT_NAMESPACE_PATTERN = Pattern.compile("pulsar/[^/]+/([^:]+:\\d+)"); public static final Pattern HEARTBEAT_NAMESPACE_PATTERN_V2 = Pattern.compile("pulsar/([^:]+:\\d+)"); public static final Pattern SLA_NAMESPACE_PATTERN = Pattern.compile(SLA_NAMESPACE_PROPERTY + "/[^/]+/([^:]+:\\d+)"); - public static final String HEARTBEAT_NAMESPACE_FMT = "pulsar/%s/%s:%s"; - public static final String HEARTBEAT_NAMESPACE_FMT_V2 = "pulsar/%s:%s"; - public static final String SLA_NAMESPACE_FMT = SLA_NAMESPACE_PROPERTY + "/%s/%s:%s"; + public static final String HEARTBEAT_NAMESPACE_FMT = "pulsar/%s/%s"; + public static final String HEARTBEAT_NAMESPACE_FMT_V2 = "pulsar/%s"; + public static final String SLA_NAMESPACE_FMT = SLA_NAMESPACE_PROPERTY + "/%s/%s"; - private final ConcurrentOpenHashMap namespaceClients; + private final Map namespaceClients = new ConcurrentHashMap<>(); private final List bundleOwnershipListeners; - private final RedirectManager redirectManager; + + private final List bundleSplitListeners; + private final RedirectManager redirectManager; + + public static final String LOOKUP_REQUEST_DURATION_METRIC_NAME = "pulsar.broker.request.topic.lookup.duration"; + + private static final AttributeKey PULSAR_LOOKUP_RESPONSE_ATTRIBUTE = + AttributeKey.stringKey("pulsar.lookup.response"); + public static final Attributes PULSAR_LOOKUP_RESPONSE_BROKER_ATTRIBUTES = Attributes.builder() + .put(PULSAR_LOOKUP_RESPONSE_ATTRIBUTE, "broker") + .build(); + public static final Attributes PULSAR_LOOKUP_RESPONSE_REDIRECT_ATTRIBUTES = Attributes.builder() + .put(PULSAR_LOOKUP_RESPONSE_ATTRIBUTE, "redirect") + .build(); + public static final Attributes PULSAR_LOOKUP_RESPONSE_FAILURE_ATTRIBUTES = Attributes.builder() + .put(PULSAR_LOOKUP_RESPONSE_ATTRIBUTE, "failure") + .build(); + + @PulsarDeprecatedMetric(newMetricName = LOOKUP_REQUEST_DURATION_METRIC_NAME) private static final Counter lookupRedirects = Counter.build("pulsar_broker_lookup_redirects", "-").register(); + + @PulsarDeprecatedMetric(newMetricName = LOOKUP_REQUEST_DURATION_METRIC_NAME) private static final Counter lookupFailures = Counter.build("pulsar_broker_lookup_failures", "-").register(); + + @PulsarDeprecatedMetric(newMetricName = LOOKUP_REQUEST_DURATION_METRIC_NAME) private static final Counter lookupAnswers = Counter.build("pulsar_broker_lookup_answers", "-").register(); + @PulsarDeprecatedMetric(newMetricName = LOOKUP_REQUEST_DURATION_METRIC_NAME) private static final Summary lookupLatency = Summary.build("pulsar_broker_lookup", "-") .quantile(0.50) .quantile(0.99) .quantile(0.999) .quantile(1.0) .register(); + private final DoubleHistogram lookupLatencyHistogram; + private ConcurrentHashMap>> inProgressQueryUserTopics = + new ConcurrentHashMap<>(); /** * Default constructor. */ public NamespaceService(PulsarService pulsar) { this.pulsar = pulsar; - host = pulsar.getAdvertisedAddress(); + this.host = pulsar.getAdvertisedAddress(); this.config = pulsar.getConfiguration(); this.loadManager = pulsar.getLoadManager(); this.bundleFactory = new NamespaceBundleFactory(pulsar, Hashing.crc32()); - this.ownershipCache = new OwnershipCache(pulsar, bundleFactory, this); - this.namespaceClients = - ConcurrentOpenHashMap.newBuilder().build(); + this.ownershipCache = new OwnershipCache(pulsar, this); this.bundleOwnershipListeners = new CopyOnWriteArrayList<>(); + this.bundleSplitListeners = new CopyOnWriteArrayList<>(); this.localBrokerDataCache = pulsar.getLocalMetadataStore().getMetadataCache(LocalBrokerData.class); this.redirectManager = new RedirectManager(pulsar); + + this.lookupLatencyHistogram = pulsar.getOpenTelemetry().getMeter() + .histogramBuilder(LOOKUP_REQUEST_DURATION_METRIC_NAME) + .setDescription("The duration of topic lookup requests (either binary or HTTP)") + .setUnit("s") + .build(); } public void initialize() { @@ -183,14 +227,14 @@ public CompletableFuture> getBrokerServiceUrlAsync(TopicN CompletableFuture> future = getBundleAsync(topic) .thenCompose(bundle -> { // Do redirection if the cluster is in rollback or deploying. - return redirectManager.findRedirectLookupResultAsync().thenCompose(optResult -> { + return findRedirectLookupResultAsync(bundle).thenCompose(optResult -> { if (optResult.isPresent()) { LOG.info("[{}] Redirect lookup request to {} for topic {}", - pulsar.getSafeWebServiceAddress(), optResult.get(), topic); + pulsar.getBrokerId(), optResult.get(), topic); return CompletableFuture.completedFuture(optResult); } - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { - return loadManager.get().findBrokerServiceUrl(Optional.of(topic), bundle); + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar)) { + return loadManager.get().findBrokerServiceUrl(Optional.of(topic), bundle, options); } else { // TODO: Add unit tests cover it. return findBrokerServiceUrl(bundle, options); @@ -198,23 +242,40 @@ public CompletableFuture> getBrokerServiceUrlAsync(TopicN }); }); - future.thenAccept(optResult -> { - lookupLatency.observe(System.nanoTime() - startTime, TimeUnit.NANOSECONDS); - if (optResult.isPresent()) { - if (optResult.get().isRedirect()) { - lookupRedirects.inc(); + future.whenComplete((lookupResult, throwable) -> { + var latencyNs = System.nanoTime() - startTime; + lookupLatency.observe(latencyNs, TimeUnit.NANOSECONDS); + Attributes attributes; + if (throwable == null) { + if (lookupResult.isPresent()) { + if (lookupResult.get().isRedirect()) { + lookupRedirects.inc(); + attributes = PULSAR_LOOKUP_RESPONSE_REDIRECT_ATTRIBUTES; + } else { + lookupAnswers.inc(); + attributes = PULSAR_LOOKUP_RESPONSE_BROKER_ATTRIBUTES; + } } else { - lookupAnswers.inc(); + // No lookup result, default to reporting as failure. + attributes = PULSAR_LOOKUP_RESPONSE_FAILURE_ATTRIBUTES; } + } else { + lookupFailures.inc(); + attributes = PULSAR_LOOKUP_RESPONSE_FAILURE_ATTRIBUTES; } - }).exceptionally(ex -> { - lookupFailures.inc(); - return null; + lookupLatencyHistogram.record(MetricsUtil.convertToSeconds(latencyNs, TimeUnit.NANOSECONDS), attributes); }); return future; } + private CompletableFuture> findRedirectLookupResultAsync(ServiceUnitId bundle) { + if (isSLAOrHeartbeatNamespace(bundle.getNamespaceObject().toString())) { + return CompletableFuture.completedFuture(Optional.empty()); + } + return redirectManager.findRedirectLookupResultAsync(); + } + public CompletableFuture getBundleAsync(TopicName topic) { return bundleFactory.getBundlesAsync(topic.getNamespaceObject()) .thenApply(bundles -> bundles.findBundle(topic)); @@ -243,28 +304,27 @@ private CompletableFuture getFullBundleAsync(NamespaceName fqnn /** * Return the URL of the broker who's owning a particular service unit in asynchronous way. - * + *

* If the service unit is not owned, return a CompletableFuture with empty optional. */ public CompletableFuture> getWebServiceUrlAsync(ServiceUnitId suName, LookupOptions options) { - if (suName instanceof TopicName) { - TopicName name = (TopicName) suName; + if (suName instanceof TopicName name) { if (LOG.isDebugEnabled()) { LOG.debug("Getting web service URL of topic: {} - options: {}", name, options); } return getBundleAsync(name) .thenCompose(namespaceBundle -> - internalGetWebServiceUrl(Optional.of(name), namespaceBundle, options)); + internalGetWebServiceUrl(name, namespaceBundle, options)); } - if (suName instanceof NamespaceName) { - return getFullBundleAsync((NamespaceName) suName) + if (suName instanceof NamespaceName namespaceName) { + return getFullBundleAsync(namespaceName) .thenCompose(namespaceBundle -> - internalGetWebServiceUrl(Optional.empty(), namespaceBundle, options)); + internalGetWebServiceUrl(null, namespaceBundle, options)); } - if (suName instanceof NamespaceBundle) { - return internalGetWebServiceUrl(Optional.empty(), (NamespaceBundle) suName, options); + if (suName instanceof NamespaceBundle namespaceBundle) { + return internalGetWebServiceUrl(null, namespaceBundle, options); } throw new IllegalArgumentException("Unrecognized class of NamespaceBundle: " + suName.getClass().getName()); @@ -272,7 +332,7 @@ public CompletableFuture> getWebServiceUrlAsync(ServiceUnitId suNa /** * Return the URL of the broker who's owning a particular service unit. - * + *

* If the service unit is not owned, return an empty optional */ public Optional getWebServiceUrl(ServiceUnitId suName, LookupOptions options) throws Exception { @@ -280,14 +340,13 @@ public Optional getWebServiceUrl(ServiceUnitId suName, LookupOptions option .get(pulsar.getConfiguration().getMetadataStoreOperationTimeoutSeconds(), SECONDS); } - private CompletableFuture> internalGetWebServiceUrl(Optional topic, + private CompletableFuture> internalGetWebServiceUrl(@Nullable ServiceUnitId topic, NamespaceBundle bundle, LookupOptions options) { - - return redirectManager.findRedirectLookupResultAsync().thenCompose(optResult -> { + return findRedirectLookupResultAsync(bundle).thenCompose(optResult -> { if (optResult.isPresent()) { LOG.info("[{}] Redirect lookup request to {} for topic {}", - pulsar.getSafeWebServiceAddress(), optResult.get(), topic); + pulsar.getBrokerId(), optResult.get(), topic); try { LookupData lookupData = optResult.get().getLookupData(); final String redirectUrl = options.isRequestHttps() @@ -300,8 +359,8 @@ private CompletableFuture> internalGetWebServiceUrl(Optional> future = - ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config) - ? loadManager.get().findBrokerServiceUrl(topic, bundle) : + ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar) + ? loadManager.get().findBrokerServiceUrl(Optional.ofNullable(topic), bundle, options) : findBrokerServiceUrl(bundle, options); return future.thenApply(lookupResult -> { @@ -324,18 +383,20 @@ private CompletableFuture> internalGetWebServiceUrl(Optional>> - findingBundlesAuthoritative = - ConcurrentOpenHashMap.>>newBuilder() - .build(); - private final ConcurrentOpenHashMap>> - findingBundlesNotAuthoritative = - ConcurrentOpenHashMap.>>newBuilder() - .build(); + private final Map>> + findingBundlesAuthoritative = new ConcurrentHashMap<>(); + private final Map>> + findingBundlesNotAuthoritative = new ConcurrentHashMap<>(); /** * Main internal method to lookup and setup ownership of service unit to a broker. * - * @param bundle - * @param options - * @return - * @throws PulsarServerException + * @param bundle the namespace bundle + * @param options the lookup options + * @return the lookup result */ private CompletableFuture> findBrokerServiceUrl( NamespaceBundle bundle, LookupOptions options) { @@ -416,7 +476,7 @@ private CompletableFuture> findBrokerServiceUrl( LOG.debug("findBrokerServiceUrl: {} - options: {}", bundle, options); } - ConcurrentOpenHashMap>> targetMap; + Map>> targetMap; if (options.isAuthoritative()) { targetMap = findingBundlesAuthoritative; } else { @@ -428,7 +488,7 @@ private CompletableFuture> findBrokerServiceUrl( // First check if we or someone else already owns the bundle ownershipCache.getOwnerAsync(bundle).thenAccept(nsData -> { - if (!nsData.isPresent()) { + if (nsData.isEmpty()) { // No one owns this bundle if (options.isReadOnly()) { @@ -436,9 +496,7 @@ private CompletableFuture> findBrokerServiceUrl( future.complete(Optional.empty()); } else { // Now, no one owns the namespace yet. Hence, we will try to dynamically assign it - pulsar.getExecutor().execute(() -> { - searchForCandidateBroker(bundle, future, options); - }); + pulsar.getExecutor().execute(() -> searchForCandidateBroker(bundle, future, options)); } } else if (nsData.get().isDisabled()) { future.completeExceptionally( @@ -462,7 +520,6 @@ private CompletableFuture> findBrokerServiceUrl( url == null ? null : url.toString(), urlTls == null ? null : urlTls.toString()))); } - return; } else { future.complete(Optional.of(new LookupResult(nsData.get()))); } @@ -481,62 +538,74 @@ private CompletableFuture> findBrokerServiceUrl( }); } + /** + * Check if this is Heartbeat or SLAMonitor namespace and return the broker id. + * + * @param serviceUnit the service unit + * @param isBrokerActive the function to check if the broker is active + * @return the broker id + */ + public CompletableFuture getHeartbeatOrSLAMonitorBrokerId( + ServiceUnitId serviceUnit, Function> isBrokerActive) { + String candidateBroker = NamespaceService.checkHeartbeatNamespace(serviceUnit); + if (candidateBroker != null) { + return CompletableFuture.completedFuture(candidateBroker); + } + candidateBroker = NamespaceService.checkHeartbeatNamespaceV2(serviceUnit); + if (candidateBroker != null) { + return CompletableFuture.completedFuture(candidateBroker); + } + candidateBroker = NamespaceService.getSLAMonitorBrokerName(serviceUnit); + if (candidateBroker != null) { + // Check if the broker is available + final String finalCandidateBroker = candidateBroker; + return isBrokerActive.apply(candidateBroker).thenApply(isActive -> { + if (isActive) { + return finalCandidateBroker; + } else { + return null; + } + }); + } + return CompletableFuture.completedFuture(null); + } + private void searchForCandidateBroker(NamespaceBundle bundle, CompletableFuture> lookupFuture, LookupOptions options) { - if (null == pulsar.getLeaderElectionService()) { + String candidateBroker; + LeaderElectionService les = pulsar.getLeaderElectionService(); + if (les == null) { LOG.warn("The leader election has not yet been completed! NamespaceBundle[{}]", bundle); lookupFuture.completeExceptionally( new IllegalStateException("The leader election has not yet been completed!")); return; } - String candidateBroker = null; - String candidateBrokerAdvertisedAddr = null; - - LeaderElectionService les = pulsar.getLeaderElectionService(); - if (les == null) { - // The leader election service was not initialized yet. This can happen because the broker service is - // initialized first and it might start receiving lookup requests before the leader election service is - // fully initialized. - LOG.warn("Leader election service isn't initialized yet. " - + "Returning empty result to lookup. NamespaceBundle[{}]", - bundle); - lookupFuture.complete(Optional.empty()); - return; - } boolean authoritativeRedirect = les.isLeader(); try { // check if this is Heartbeat or SLAMonitor namespace - candidateBroker = checkHeartbeatNamespace(bundle); - if (candidateBroker == null) { - candidateBroker = checkHeartbeatNamespaceV2(bundle); - } - if (candidateBroker == null) { - String broker = getSLAMonitorBrokerName(bundle); - // checking if the broker is up and running - if (broker != null && isBrokerActive(broker)) { - candidateBroker = broker; - } - } + candidateBroker = getHeartbeatOrSLAMonitorBrokerId(bundle, cb -> + CompletableFuture.completedFuture(isBrokerActive(cb))) + .get(config.getMetadataStoreOperationTimeoutSeconds(), SECONDS); if (candidateBroker == null) { Optional currentLeader = pulsar.getLeaderElectionService().getCurrentLeader(); if (options.isAuthoritative()) { // leader broker already assigned the current broker as owner - candidateBroker = pulsar.getSafeWebServiceAddress(); + candidateBroker = pulsar.getBrokerId(); } else { LoadManager loadManager = this.loadManager.get(); boolean makeLoadManagerDecisionOnThisBroker = !loadManager.isCentralized() || les.isLeader(); if (!makeLoadManagerDecisionOnThisBroker) { // If leader is not active, fallback to pick the least loaded from current broker loadmanager boolean leaderBrokerActive = currentLeader.isPresent() - && isBrokerActive(currentLeader.get().getServiceUrl()); + && isBrokerActive(currentLeader.get().getBrokerId()); if (!leaderBrokerActive) { makeLoadManagerDecisionOnThisBroker = true; - if (!currentLeader.isPresent()) { + if (currentLeader.isEmpty()) { LOG.warn( "The information about the current leader broker wasn't available. " + "Handling load manager decisions in a decentralized way. " @@ -552,20 +621,19 @@ private void searchForCandidateBroker(NamespaceBundle bundle, } } if (makeLoadManagerDecisionOnThisBroker) { - Optional> availableBroker = getLeastLoadedFromLoadManager(bundle); - if (!availableBroker.isPresent()) { + Optional availableBroker = getLeastLoadedFromLoadManager(bundle); + if (availableBroker.isEmpty()) { LOG.warn("Load manager didn't return any available broker. " + "Returning empty result to lookup. NamespaceBundle[{}]", bundle); lookupFuture.complete(Optional.empty()); return; } - candidateBroker = availableBroker.get().getLeft(); - candidateBrokerAdvertisedAddr = availableBroker.get().getRight(); + candidateBroker = availableBroker.get(); authoritativeRedirect = true; } else { // forward to leader broker to make assignment - candidateBroker = currentLeader.get().getServiceUrl(); + candidateBroker = currentLeader.get().getBrokerId(); } } } @@ -578,7 +646,7 @@ private void searchForCandidateBroker(NamespaceBundle bundle, try { Objects.requireNonNull(candidateBroker); - if (candidateBroker.equals(pulsar.getSafeWebServiceAddress())) { + if (candidateBroker.equals(pulsar.getBrokerId())) { // Load manager decided that the local broker should try to become the owner ownershipCache.tryAcquiringOwnership(bundle).thenAccept(ownerInfo -> { if (ownerInfo.isDisabled()) { @@ -591,7 +659,7 @@ private void searchForCandidateBroker(NamespaceBundle bundle, // Found owner for the namespace bundle if (options.isLoadTopicsInBundle()) { - // Schedule the task to pre-load topics + // Schedule the task to preload topics pulsar.loadNamespaceTopics(bundle); } // find the target @@ -602,7 +670,6 @@ private void searchForCandidateBroker(NamespaceBundle bundle, lookupFuture.completeExceptionally( new PulsarServerException("the broker do not have " + options.getAdvertisedListenerName() + " listener")); - return; } else { URI url = listener.getBrokerServiceUrl(); URI urlTls = listener.getBrokerServiceUrlTls(); @@ -610,11 +677,9 @@ private void searchForCandidateBroker(NamespaceBundle bundle, new LookupResult(ownerInfo, url == null ? null : url.toString(), urlTls == null ? null : urlTls.toString()))); - return; } } else { lookupFuture.complete(Optional.of(new LookupResult(ownerInfo))); - return; } } }).exceptionally(exception -> { @@ -632,8 +697,7 @@ private void searchForCandidateBroker(NamespaceBundle bundle, } // Now setting the redirect url - createLookupResult(candidateBrokerAdvertisedAddr == null ? candidateBroker - : candidateBrokerAdvertisedAddr, authoritativeRedirect, options.getAdvertisedListenerName()) + createLookupResult(candidateBroker, authoritativeRedirect, options.getAdvertisedListenerName()) .thenAccept(lookupResult -> lookupFuture.complete(Optional.of(lookupResult))) .exceptionally(ex -> { lookupFuture.completeExceptionally(ex); @@ -653,7 +717,7 @@ public CompletableFuture createLookupResult(String candidateBroker CompletableFuture lookupFuture = new CompletableFuture<>(); try { checkArgument(StringUtils.isNotBlank(candidateBroker), "Lookup broker can't be null %s", candidateBroker); - String path = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + parseHostAndPort(candidateBroker); + String path = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + candidateBroker; localBrokerDataCache.get(path).thenAccept(reportData -> { if (reportData.isPresent()) { @@ -689,31 +753,20 @@ public CompletableFuture createLookupResult(String candidateBroker return lookupFuture; } - private boolean isBrokerActive(String candidateBroker) { - String candidateBrokerHostAndPort = parseHostAndPort(candidateBroker); + public boolean isBrokerActive(String candidateBroker) { Set availableBrokers = getAvailableBrokers(); - if (availableBrokers.contains(candidateBrokerHostAndPort)) { + if (availableBrokers.contains(candidateBroker)) { if (LOG.isDebugEnabled()) { - LOG.debug("Broker {} ({}) is available for.", candidateBroker, candidateBrokerHostAndPort); + LOG.debug("Broker {} is available for.", candidateBroker); } return true; } else { - LOG.warn("Broker {} ({}) couldn't be found in available brokers {}", - candidateBroker, candidateBrokerHostAndPort, - availableBrokers.stream().collect(Collectors.joining(","))); + LOG.warn("Broker {} couldn't be found in available brokers {}", + candidateBroker, String.join(",", availableBrokers)); return false; } } - private static String parseHostAndPort(String candidateBroker) { - int uriSeparatorPos = candidateBroker.indexOf("://"); - if (uriSeparatorPos == -1) { - throw new IllegalArgumentException("'" + candidateBroker + "' isn't an URI."); - } - String candidateBrokerHostAndPort = candidateBroker.substring(uriSeparatorPos + 3); - return candidateBrokerHostAndPort; - } - private Set getAvailableBrokers() { try { return loadManager.get().getAvailableBrokers(); @@ -725,26 +778,25 @@ private Set getAvailableBrokers() { /** * Helper function to encapsulate the logic to invoke between old and new load manager. * - * @return - * @throws Exception + * @param serviceUnit the service unit + * @return the least loaded broker addresses + * @throws Exception if an error occurs */ - private Optional> getLeastLoadedFromLoadManager(ServiceUnitId serviceUnit) throws Exception { + private Optional getLeastLoadedFromLoadManager(ServiceUnitId serviceUnit) throws Exception { Optional leastLoadedBroker = loadManager.get().getLeastLoaded(serviceUnit); - if (!leastLoadedBroker.isPresent()) { + if (leastLoadedBroker.isEmpty()) { LOG.warn("No broker is available for {}", serviceUnit); return Optional.empty(); } String lookupAddress = leastLoadedBroker.get().getResourceId(); - String advertisedAddr = (String) leastLoadedBroker.get() - .getProperty(ResourceUnit.PROPERTY_KEY_BROKER_ZNODE_NAME); if (LOG.isDebugEnabled()) { LOG.debug("{} : redirecting to the least loaded broker, lookup address={}", - pulsar.getSafeWebServiceAddress(), + pulsar.getBrokerId(), lookupAddress); } - return Optional.of(Pair.of(lookupAddress, advertisedAddr)); + return Optional.of(lookupAddress); } public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle) { @@ -752,21 +804,42 @@ public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle) { } public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle, Optional destinationBroker) { - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { - return ExtensibleLoadManagerImpl.get(loadManager.get()) - .unloadNamespaceBundleAsync(bundle, destinationBroker); - } + // unload namespace bundle - return unloadNamespaceBundle(bundle, config.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS); + return unloadNamespaceBundle(bundle, destinationBroker, + config.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS); } - public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle, long timeout, TimeUnit timeoutUnit) { - return unloadNamespaceBundle(bundle, timeout, timeoutUnit, true); + public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle, + Optional destinationBroker, + long timeout, + TimeUnit timeoutUnit) { + return unloadNamespaceBundle(bundle, destinationBroker, timeout, timeoutUnit, true); } - public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle, long timeout, + public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle, + long timeout, + TimeUnit timeoutUnit) { + return unloadNamespaceBundle(bundle, Optional.empty(), timeout, timeoutUnit, true); + } + + public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle, + long timeout, + TimeUnit timeoutUnit, + boolean closeWithoutWaitingClientDisconnect) { + return unloadNamespaceBundle(bundle, Optional.empty(), timeout, + timeoutUnit, closeWithoutWaitingClientDisconnect); + } + + public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle, + Optional destinationBroker, + long timeout, TimeUnit timeoutUnit, boolean closeWithoutWaitingClientDisconnect) { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar)) { + return ExtensibleLoadManagerImpl.get(loadManager.get()) + .unloadNamespaceBundleAsync(bundle, destinationBroker, false, timeout, timeoutUnit); + } // unload namespace bundle OwnedBundle ob = ownershipCache.getOwnedBundle(bundle); if (ob == null) { @@ -777,6 +850,11 @@ public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle, lon } public CompletableFuture isNamespaceBundleOwned(NamespaceBundle bundle) { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar)) { + ExtensibleLoadManagerImpl extensibleLoadManager = ExtensibleLoadManagerImpl.get(loadManager.get()); + return extensibleLoadManager.getOwnershipAsync(Optional.empty(), bundle) + .thenApply(Optional::isPresent); + } return pulsar.getLocalMetadataStore().exists(ServiceUnitUtils.path(bundle)); } @@ -785,13 +863,23 @@ public CompletableFuture> getOwnedNameSpac .getIsolationDataPoliciesAsync(pulsar.getConfiguration().getClusterName()) .thenApply(nsIsolationPoliciesOpt -> nsIsolationPoliciesOpt.orElseGet(NamespaceIsolationPolicies::new)) .thenCompose(namespaceIsolationPolicies -> { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar)) { + ExtensibleLoadManagerImpl extensibleLoadManager = + ExtensibleLoadManagerImpl.get(loadManager.get()); + return extensibleLoadManager.getOwnedServiceUnitsAsync() + .thenApply(OwnedServiceUnits -> OwnedServiceUnits.stream() + .collect(Collectors.toMap(NamespaceBundle::toString, + bundle -> getNamespaceOwnershipStatus(true, + namespaceIsolationPolicies.getPolicyByNamespace( + bundle.getNamespaceObject()))))); + } Collection> futures = ownershipCache.getOwnedBundlesAsync().values(); return FutureUtil.waitForAll(futures) .thenApply(__ -> futures.stream() .map(CompletableFuture::join) .collect(Collectors.toMap(bundle -> bundle.getNamespaceBundle().toString(), - bundle -> getNamespaceOwnershipStatus(bundle, + bundle -> getNamespaceOwnershipStatus(bundle.isActive(), namespaceIsolationPolicies.getPolicyByNamespace( bundle.getNamespaceBundle().getNamespaceObject())) )) @@ -799,10 +887,10 @@ public CompletableFuture> getOwnedNameSpac }); } - private NamespaceOwnershipStatus getNamespaceOwnershipStatus(OwnedBundle nsObj, + private NamespaceOwnershipStatus getNamespaceOwnershipStatus(boolean isActive, NamespaceIsolationPolicy nsIsolationPolicy) { NamespaceOwnershipStatus nsOwnedStatus = new NamespaceOwnershipStatus(BrokerAssignment.shared, false, - nsObj.isActive()); + isActive); if (nsIsolationPolicy == null) { // no matching policy found, this namespace must be an uncontrolled one and using shared broker return nsOwnedStatus; @@ -820,7 +908,7 @@ private NamespaceOwnershipStatus getNamespaceOwnershipStatus(OwnedBundle nsObj, public boolean isNamespaceBundleDisabled(NamespaceBundle bundle) throws Exception { try { - // Does ZooKeeper says that the namespace is disabled? + // Does ZooKeeper say that the namespace is disabled? CompletableFuture> nsDataFuture = ownershipCache.getOwnerAsync(bundle); if (nsDataFuture != null) { Optional nsData = nsDataFuture.getNow(null); @@ -843,17 +931,19 @@ public boolean isNamespaceBundleDisabled(NamespaceBundle bundle) throws Exceptio /** * 1. split the given bundle into two bundles 2. assign ownership of both the bundles to current broker 3. update * policies with newly created bundles into LocalZK 4. disable original bundle and refresh the cache. - * + *

* It will call splitAndOwnBundleOnceAndRetry to do the real retry work, which will retry "retryTimes". * - * @param bundle - * @return - * @throws Exception + * @param bundle the bundle to split + * @param unload whether to unload the new split bundles + * @param splitAlgorithm the algorithm to split the bundle + * @param boundaries the boundaries to split the bundle + * @return a future that will complete when the bundle is split and owned */ public CompletableFuture splitAndOwnBundle(NamespaceBundle bundle, boolean unload, NamespaceBundleSplitAlgorithm splitAlgorithm, List boundaries) { - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar)) { return ExtensibleLoadManagerImpl.get(loadManager.get()) .splitNamespaceBundleAsync(bundle, splitAlgorithm, boundaries); } @@ -883,36 +973,36 @@ void splitAndOwnBundleOnceAndRetry(NamespaceBundle bundle, } try { bundleFactory.splitBundles(bundle, splitBoundaries.size() + 1, splitBoundaries) - .thenAccept(splittedBundles -> { + .thenAccept(splitBundles -> { // Split and updateNamespaceBundles. Update may fail because of concurrent write to // Zookeeper. - if (splittedBundles == null) { + if (splitBundles == null) { String msg = format("bundle %s not found under namespace", bundle.toString()); LOG.warn(msg); updateFuture.completeExceptionally(new ServiceUnitNotReadyException(msg)); return; } - Objects.requireNonNull(splittedBundles.getLeft()); - Objects.requireNonNull(splittedBundles.getRight()); - checkArgument(splittedBundles.getRight().size() == splitBoundaries.size() + 1, + Objects.requireNonNull(splitBundles.getLeft()); + Objects.requireNonNull(splitBundles.getRight()); + checkArgument(splitBundles.getRight().size() == splitBoundaries.size() + 1, "bundle has to be split in " + (splitBoundaries.size() + 1) + " bundles"); NamespaceName nsname = bundle.getNamespaceObject(); if (LOG.isDebugEnabled()) { LOG.debug("[{}] splitAndOwnBundleOnce: {}, counter: {}, bundles: {}", nsname.toString(), bundle.getBundleRange(), counter.get(), - splittedBundles.getRight()); + splitBundles.getRight()); } try { // take ownership of newly split bundles - for (NamespaceBundle sBundle : splittedBundles.getRight()) { + for (NamespaceBundle sBundle : splitBundles.getRight()) { Objects.requireNonNull(ownershipCache.tryAcquiringOwnership(sBundle)); } - updateNamespaceBundles(nsname, splittedBundles.getLeft()).thenCompose(__ -> { - return updateNamespaceBundlesForPolicies(nsname, splittedBundles.getLeft()); - }).thenRun(() -> { - bundleFactory.invalidateBundleCache(bundle.getNamespaceObject()); - updateFuture.complete(splittedBundles.getRight()); + updateNamespaceBundles(nsname, splitBundles.getLeft()).thenCompose(__ -> + updateNamespaceBundlesForPolicies(nsname, splitBundles.getLeft())) + .thenRun(() -> { + bundleFactory.invalidateBundleCache(bundle.getNamespaceObject()); + updateFuture.complete(splitBundles.getRight()); }).exceptionally(ex1 -> { String msg = format("failed to update namespace policies [%s], " + "NamespaceBundle: %s due to %s", @@ -975,11 +1065,12 @@ void splitAndOwnBundleOnceAndRetry(NamespaceBundle bundle, // affect the split operation which is already safely completed r.forEach(this::unloadNamespaceBundle); } + onNamespaceBundleSplit(bundle); }) .exceptionally(e -> { String msg1 = format( "failed to disable bundle %s under namespace [%s] with error %s", - bundle.getNamespaceObject().toString(), bundle.toString(), ex.getMessage()); + bundle.getNamespaceObject().toString(), bundle, ex.getMessage()); LOG.warn(msg1, e); completionFuture.completeExceptionally(new ServiceUnitNotReadyException(msg1)); return null; @@ -1049,9 +1140,8 @@ public NamespaceBundleSplitAlgorithm getNamespaceBundleSplitAlgorithmByName(Stri * Update new bundle-range to admin/policies/namespace. * Update may fail because of concurrent write to Zookeeper. * - * @param nsname - * @param nsBundles - * @throws Exception + * @param nsname the namespace name + * @param nsBundles the new namespace bundles */ public CompletableFuture updateNamespaceBundlesForPolicies(NamespaceName nsname, NamespaceBundles nsBundles) { @@ -1078,9 +1168,8 @@ public CompletableFuture updateNamespaceBundlesForPolicies(NamespaceName n * Update new bundle-range to LocalZk (create a new node if not present). * Update may fail because of concurrent write to Zookeeper. * - * @param nsname - * @param nsBundles - * @throws Exception + * @param nsname the namespace name + * @param nsBundles the new namespace bundles */ public CompletableFuture updateNamespaceBundles(NamespaceName nsname, NamespaceBundles nsBundles) { Objects.requireNonNull(nsname); @@ -1097,24 +1186,21 @@ public OwnershipCache getOwnershipCache() { } public Set getOwnedServiceUnits() { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar)) { + ExtensibleLoadManagerImpl extensibleLoadManager = ExtensibleLoadManagerImpl.get(loadManager.get()); + try { + return extensibleLoadManager.getOwnedServiceUnitsAsync() + .get(config.getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS); + } catch (Exception e) { + throw new RuntimeException(e); + } + } return ownershipCache.getOwnedBundles().values().stream().map(OwnedBundle::getNamespaceBundle) .collect(Collectors.toSet()); } public boolean isServiceUnitOwned(ServiceUnitId suName) throws Exception { - if (suName instanceof TopicName) { - return isTopicOwnedAsync((TopicName) suName).get(); - } - - if (suName instanceof NamespaceName) { - return isNamespaceOwned((NamespaceName) suName); - } - - if (suName instanceof NamespaceBundle) { - return ownershipCache.isNamespaceBundleOwned((NamespaceBundle) suName); - } - - throw new IllegalArgumentException("Invalid class of NamespaceBundle: " + suName.getClass().getName()); + return isServiceUnitOwnedAsync(suName).get(config.getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS); } public CompletableFuture isServiceUnitOwnedAsync(ServiceUnitId suName) { @@ -1127,7 +1213,7 @@ public CompletableFuture isServiceUnitOwnedAsync(ServiceUnitId suName) } if (suName instanceof NamespaceBundle) { - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar)) { return loadManager.get().checkOwnershipAsync(Optional.empty(), suName); } // TODO: Add unit tests cover it. @@ -1140,7 +1226,7 @@ public CompletableFuture isServiceUnitOwnedAsync(ServiceUnitId suName) } /** - * @Deprecated This method is only used in test now. + * @deprecated This method is only used in test now. */ @Deprecated public boolean isServiceUnitActive(TopicName topicName) { @@ -1155,26 +1241,22 @@ public boolean isServiceUnitActive(TopicName topicName) { public CompletableFuture isServiceUnitActiveAsync(TopicName topicName) { // TODO: Add unit tests cover it. - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar)) { return getBundleAsync(topicName) .thenCompose(bundle -> loadManager.get().checkOwnershipAsync(Optional.of(topicName), bundle)); } return getBundleAsync(topicName).thenCompose(bundle -> { Optional> optionalFuture = ownershipCache.getOwnedBundleAsync(bundle); - if (!optionalFuture.isPresent()) { + if (optionalFuture.isEmpty()) { return CompletableFuture.completedFuture(false); } return optionalFuture.get().thenApply(ob -> ob != null && ob.isActive()); }); } - private boolean isNamespaceOwned(NamespaceName fqnn) throws Exception { - return ownershipCache.getOwnedBundle(getFullBundle(fqnn)) != null; - } - private CompletableFuture isNamespaceOwnedAsync(NamespaceName fqnn) { // TODO: Add unit tests cover it. - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar)) { return getFullBundleAsync(fqnn) .thenCompose(bundle -> loadManager.get().checkOwnershipAsync(Optional.empty(), bundle)); } @@ -1184,16 +1266,16 @@ private CompletableFuture isNamespaceOwnedAsync(NamespaceName fqnn) { private CompletableFuture isTopicOwnedAsync(TopicName topic) { // TODO: Add unit tests cover it. - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar)) { return getBundleAsync(topic) .thenCompose(bundle -> loadManager.get().checkOwnershipAsync(Optional.of(topic), bundle)); } - return getBundleAsync(topic).thenApply(bundle -> ownershipCache.isNamespaceBundleOwned(bundle)); + return getBundleAsync(topic).thenApply(ownershipCache::isNamespaceBundleOwned); } public CompletableFuture checkTopicOwnership(TopicName topicName) { // TODO: Add unit tests cover it. - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar)) { return getBundleAsync(topicName) .thenCompose(bundle -> loadManager.get().checkOwnershipAsync(Optional.of(topicName), bundle)); } @@ -1203,29 +1285,43 @@ public CompletableFuture checkTopicOwnership(TopicName topicName) { public CompletableFuture removeOwnedServiceUnitAsync(NamespaceBundle nsBundle) { CompletableFuture future; - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar)) { ExtensibleLoadManagerImpl extensibleLoadManager = ExtensibleLoadManagerImpl.get(loadManager.get()); - future = extensibleLoadManager.unloadNamespaceBundleAsync(nsBundle, Optional.empty()); + future = extensibleLoadManager.unloadNamespaceBundleAsync( + nsBundle, Optional.empty(), true, + pulsar.getConfig().getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS); } else { future = ownershipCache.removeOwnership(nsBundle); } return future.thenRun(() -> bundleFactory.invalidateBundleCache(nsBundle.getNamespaceObject())); } - protected void onNamespaceBundleOwned(NamespaceBundle bundle) { + public void onNamespaceBundleOwned(NamespaceBundle bundle) { for (NamespaceBundleOwnershipListener bundleOwnedListener : bundleOwnershipListeners) { notifyNamespaceBundleOwnershipListener(bundle, bundleOwnedListener); } } - protected void onNamespaceBundleUnload(NamespaceBundle bundle) { + public void onNamespaceBundleUnload(NamespaceBundle bundle) { for (NamespaceBundleOwnershipListener bundleOwnedListener : bundleOwnershipListeners) { try { if (bundleOwnedListener.test(bundle)) { bundleOwnedListener.unLoad(bundle); } } catch (Throwable t) { - LOG.error("Call bundle {} ownership lister error", bundle, t); + LOG.error("Call bundle {} ownership listener error", bundle, t); + } + } + } + + public void onNamespaceBundleSplit(NamespaceBundle bundle) { + for (NamespaceBundleSplitListener bundleSplitListener : bundleSplitListeners) { + try { + if (bundleSplitListener.test(bundle)) { + bundleSplitListener.onSplit(bundle); + } + } catch (Throwable t) { + LOG.error("Call bundle {} split listener {} error", bundle, bundleSplitListener, t); } } } @@ -1237,7 +1333,22 @@ public void addNamespaceBundleOwnershipListener(NamespaceBundleOwnershipListener bundleOwnershipListeners.add(listener); } } - getOwnedServiceUnits().forEach(bundle -> notifyNamespaceBundleOwnershipListener(bundle, listeners)); + pulsar.runWhenReadyForIncomingRequests(() -> { + try { + getOwnedServiceUnits().forEach(bundle -> notifyNamespaceBundleOwnershipListener(bundle, listeners)); + } catch (Exception e) { + LOG.error("Failed to notify namespace bundle ownership listener", e); + } + }); + } + + public void addNamespaceBundleSplitListener(NamespaceBundleSplitListener... listeners) { + Objects.requireNonNull(listeners); + for (NamespaceBundleSplitListener listener : listeners) { + if (listener != null) { + bundleSplitListeners.add(listener); + } + } } private void notifyNamespaceBundleOwnershipListener(NamespaceBundle bundle, @@ -1249,7 +1360,7 @@ private void notifyNamespaceBundleOwnershipListener(NamespaceBundle bundle, listener.onLoad(bundle); } } catch (Throwable t) { - LOG.error("Call bundle {} ownership lister error", bundle, t); + LOG.error("Call bundle {} ownership listener error", bundle, t); } } } @@ -1297,45 +1408,111 @@ public CompletableFuture> getOwnedTopicListForNamespaceBundle(Names }); } - public CompletableFuture checkTopicExists(TopicName topic) { + /*** + * Check topic exists( partitioned or non-partitioned ). + */ + public CompletableFuture checkTopicExists(TopicName topic) { + return pulsar.getBrokerService() + .fetchPartitionedTopicMetadataAsync(TopicName.get(topic.toString())) + .thenCompose(metadata -> { + if (metadata.partitions > 0) { + return CompletableFuture.completedFuture( + TopicExistsInfo.newPartitionedTopicExists(metadata.partitions)); + } + return checkNonPartitionedTopicExists(topic) + .thenApply(b -> b ? TopicExistsInfo.newNonPartitionedTopicExists() + : TopicExistsInfo.newTopicNotExists()); + }); + } + + /*** + * Check non-partitioned topic exists. + */ + public CompletableFuture checkNonPartitionedTopicExists(TopicName topic) { if (topic.isPersistent()) { - if (topic.isPartitioned()) { - return pulsar.getBrokerService() - .fetchPartitionedTopicMetadataAsync(TopicName.get(topic.getPartitionedTopicName())) - .thenCompose(metadata -> { - // Allow creating the non-partitioned persistent topic that name includes `-partition-` - if (metadata.partitions == 0 - || topic.getPartitionIndex() < metadata.partitions) { - return pulsar.getPulsarResources().getTopicResources().persistentTopicExists(topic); - } - return CompletableFuture.completedFuture(false); - }); - } else { - return pulsar.getPulsarResources().getTopicResources().persistentTopicExists(topic); - } + return pulsar.getPulsarResources().getTopicResources().persistentTopicExists(topic); } else { - if (topic.isPartitioned()) { - final TopicName partitionedTopicName = TopicName.get(topic.getPartitionedTopicName()); - return pulsar.getBrokerService() - .fetchPartitionedTopicMetadataAsync(partitionedTopicName) - .thenApply((metadata) -> topic.getPartitionIndex() < metadata.partitions); - } else { - // only checks and don't do any topic creating and loading. - CompletableFuture> topicFuture = - pulsar.getBrokerService().getTopics().get(topic.toString()); - if (topicFuture == null) { - return CompletableFuture.completedFuture(false); - } else { - return topicFuture.thenApply(Optional::isPresent).exceptionally(throwable -> { - LOG.warn("[{}] topicFuture completed with exception when checkTopicExists, {}", - topic, throwable.getMessage()); - return false; - }); - } - } + return checkNonPersistentNonPartitionedTopicExists(topic.toString()); } } + /** + * Regarding non-persistent topic, we do not know whether it exists or not. Redirect the request to the ownership + * broker of this topic. HTTP API has implemented the mechanism that redirect to ownership broker, so just call + * HTTP API here. + */ + public CompletableFuture checkNonPersistentNonPartitionedTopicExists(String topic) { + TopicName topicName = TopicName.get(topic); + // "non-partitioned & non-persistent" topics only exist on the owner broker. + return checkTopicOwnership(TopicName.get(topic)).thenCompose(isOwned -> { + // The current broker is the owner. + if (isOwned) { + CompletableFuture> nonPersistentTopicFuture = pulsar.getBrokerService() + .getTopic(topic, false); + if (nonPersistentTopicFuture != null) { + return nonPersistentTopicFuture.thenApply(Optional::isPresent); + } else { + return CompletableFuture.completedFuture(false); + } + } + + // Forward to the owner broker. + PulsarClientImpl pulsarClient; + try { + pulsarClient = (PulsarClientImpl) pulsar.getClient(); + } catch (Exception ex) { + // This error will never occur. + log.error("{} Failed to get partition metadata due to create internal admin client fails", topic, ex); + return FutureUtil.failedFuture(ex); + } + LookupOptions lookupOptions = LookupOptions.builder().readOnly(false).authoritative(true).build(); + return getBrokerServiceUrlAsync(TopicName.get(topic), lookupOptions) + .thenCompose(lookupResult -> { + if (!lookupResult.isPresent()) { + log.error("{} Failed to get partition metadata due can not find the owner broker", topic); + return FutureUtil.failedFuture(new ServiceUnitNotReadyException( + "No broker was available to own " + topicName)); + } + LookupData lookupData = lookupResult.get().getLookupData(); + String brokerUrl; + if (pulsar.getConfiguration().isBrokerClientTlsEnabled() + && StringUtils.isNotEmpty(lookupData.getBrokerUrlTls())) { + brokerUrl = lookupData.getBrokerUrlTls(); + } else { + brokerUrl = lookupData.getBrokerUrl(); + } + return pulsarClient.getLookup(brokerUrl) + .getPartitionedTopicMetadata(topicName, false) + .thenApply(metadata -> true) + .exceptionallyCompose(ex -> { + Throwable actEx = FutureUtil.unwrapCompletionException(ex); + if (actEx instanceof PulsarClientException.NotFoundException + || actEx instanceof PulsarClientException.TopicDoesNotExistException + || actEx instanceof PulsarAdminException.NotFoundException) { + return CompletableFuture.completedFuture(false); + } else if (actEx instanceof PulsarClientException.FeatureNotSupportedException fe){ + if (fe.getFailedFeatureCheck() == SupportsGetPartitionedMetadataWithoutAutoCreation) { + // Since the feature PIP-344 isn't supported, restore the behavior to previous + // behavior before https://github.com/apache/pulsar/pull/22838 changes. + log.info("{} Checking the existence of a non-persistent non-partitioned topic " + + "was performed using the behavior prior to PIP-344 changes, " + + "because the broker does not support the PIP-344 feature " + + "'supports_get_partitioned_metadata_without_auto_creation'.", + topic); + return CompletableFuture.completedFuture(false); + } else { + log.error("{} Failed to get partition metadata", topic, ex); + return CompletableFuture.failedFuture(ex); + } + } else { + log.error("{} Failed to get partition metadata", topic, ex); + return CompletableFuture.failedFuture(ex); + } + }); + }); + }); + } + public CompletableFuture> getListOfTopics(NamespaceName namespaceName, Mode mode) { switch (mode) { case ALL: @@ -1348,6 +1525,23 @@ public CompletableFuture> getListOfTopics(NamespaceName namespaceNa } } + public CompletableFuture> getListOfUserTopics(NamespaceName namespaceName, Mode mode) { + String key = String.format("%s://%s", mode, namespaceName); + final MutableBoolean initializedByCurrentThread = new MutableBoolean(); + CompletableFuture> queryRes = inProgressQueryUserTopics.computeIfAbsent(key, k -> { + initializedByCurrentThread.setTrue(); + return getListOfTopics(namespaceName, mode).thenApplyAsync(list -> { + return TopicList.filterSystemTopic(list); + }, pulsar.getExecutor()); + }); + if (initializedByCurrentThread.getValue()) { + queryRes.whenComplete((ignore, ex) -> { + inProgressQueryUserTopics.remove(key, queryRes); + }); + } + return queryRes; + } + public CompletableFuture> getAllPartitions(NamespaceName namespaceName) { return getPartitions(namespaceName, TopicDomain.persistent) .thenCombine(getPartitions(namespaceName, TopicDomain.non_persistent), @@ -1396,6 +1590,9 @@ private CompletableFuture> getPartitionsForTopic(TopicName topicNam }); } + /*** + * List persistent topics names under a namespace, the topic name contains the partition suffix. + */ public CompletableFuture> getListOfPersistentTopics(NamespaceName namespaceName) { return pulsar.getPulsarResources().getTopicResources().listPersistentTopicsAsync(namespaceName); } @@ -1409,21 +1606,19 @@ public CompletableFuture> getListOfNonPersistentTopics(NamespaceNam if (peerClusterData != null) { return getNonPersistentTopicsFromPeerCluster(peerClusterData, namespaceName); } else { - // Non-persistent topics don't have managed ledgers so we have to retrieve them from local + // Non-persistent topics don't have managed ledgers. So we have to retrieve them from local // cache. List topics = new ArrayList<>(); - synchronized (pulsar.getBrokerService().getMultiLayerTopicMap()) { - if (pulsar.getBrokerService().getMultiLayerTopicMap() + synchronized (pulsar.getBrokerService().getMultiLayerTopicsMap()) { + if (pulsar.getBrokerService().getMultiLayerTopicsMap() .containsKey(namespaceName.toString())) { - pulsar.getBrokerService().getMultiLayerTopicMap().get(namespaceName.toString()) - .forEach((__, bundle) -> { - bundle.forEach((topicName, topic) -> { - if (topic instanceof NonPersistentTopic - && ((NonPersistentTopic) topic).isActive()) { - topics.add(topicName); - } - }); - }); + pulsar.getBrokerService().getMultiLayerTopicsMap().get(namespaceName.toString()) + .forEach((__, bundle) -> bundle.forEach((topicName, topic) -> { + if (topic instanceof NonPersistentTopic + && ((NonPersistentTopic) topic).isActive()) { + topics.add(topicName); + } + })); } } @@ -1473,7 +1668,9 @@ public PulsarClientImpl getNamespaceClient(ClusterDataImpl cluster) { .enableTls(true) .tlsTrustCertsFilePath(pulsar.getConfiguration().getBrokerClientTrustCertsFilePath()) .allowTlsInsecureConnection(pulsar.getConfiguration().isTlsAllowInsecureConnection()) - .enableTlsHostnameVerification(pulsar.getConfiguration().isTlsHostnameVerificationEnabled()); + .enableTlsHostnameVerification(pulsar.getConfiguration().isTlsHostnameVerificationEnabled()) + .sslFactoryPlugin(pulsar.getConfiguration().getBrokerClientSslFactoryPlugin()) + .sslFactoryPluginParams(pulsar.getConfiguration().getBrokerClientSslFactoryPluginParams()); } else { clientBuilder.serviceUrl(isNotBlank(cluster.getBrokerServiceUrl()) ? cluster.getBrokerServiceUrl() : cluster.getServiceUrl()); @@ -1489,17 +1686,13 @@ public PulsarClientImpl getNamespaceClient(ClusterDataImpl cluster) { } public CompletableFuture> getOwnerAsync(NamespaceBundle bundle) { - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar)) { ExtensibleLoadManagerImpl extensibleLoadManager = ExtensibleLoadManagerImpl.get(loadManager.get()); return extensibleLoadManager.getOwnershipWithLookupDataAsync(bundle) - .thenCompose(lookupData -> { - if (lookupData.isPresent()) { - return CompletableFuture.completedFuture( - Optional.of(lookupData.get().toNamespaceEphemeralData())); - } else { - return CompletableFuture.completedFuture(Optional.empty()); - } - }); + .thenCompose(lookupData -> lookupData + .map(brokerLookupData -> + CompletableFuture.completedFuture(Optional.of(brokerLookupData.toNamespaceEphemeralData()))) + .orElseGet(() -> CompletableFuture.completedFuture(Optional.empty()))); } return ownershipCache.getOwnerAsync(bundle); } @@ -1510,11 +1703,7 @@ public boolean checkOwnershipPresent(NamespaceBundle bundle) throws Exception { } public CompletableFuture checkOwnershipPresentAsync(NamespaceBundle bundle) { - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { - if (bundle.getNamespaceObject().equals(SYSTEM_NAMESPACE)) { - return FutureUtil.failedFuture(new UnsupportedOperationException( - "Ownership check for system namespace is not supported")); - } + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar)) { ExtensibleLoadManagerImpl extensibleLoadManager = ExtensibleLoadManagerImpl.get(loadManager.get()); return extensibleLoadManager.getOwnershipAsync(Optional.empty(), bundle) .thenApply(Optional::isPresent); @@ -1523,8 +1712,7 @@ public CompletableFuture checkOwnershipPresentAsync(NamespaceBundle bun } public void unloadSLANamespace() throws Exception { - PulsarAdmin adminClient = null; - NamespaceName namespaceName = getSLAMonitorNamespace(host, config); + NamespaceName namespaceName = getSLAMonitorNamespace(pulsar.getBrokerId(), config); LOG.info("Checking owner for SLA namespace {}", namespaceName); @@ -1536,46 +1724,28 @@ public void unloadSLANamespace() throws Exception { } LOG.info("Trying to unload SLA namespace {}", namespaceName); - adminClient = pulsar.getAdminClient(); + PulsarAdmin adminClient = pulsar.getAdminClient(); adminClient.namespaces().unload(namespaceName.toString()); LOG.info("Namespace {} unloaded successfully", namespaceName); } - public static NamespaceName getHeartbeatNamespace(String host, ServiceConfiguration config) { - Integer port = null; - if (config.getWebServicePort().isPresent()) { - port = config.getWebServicePort().get(); - } else if (config.getWebServicePortTls().isPresent()) { - port = config.getWebServicePortTls().get(); - } - return NamespaceName.get(String.format(HEARTBEAT_NAMESPACE_FMT, config.getClusterName(), host, port)); + public static NamespaceName getHeartbeatNamespace(String lookupBroker, ServiceConfiguration config) { + return NamespaceName.get(String.format(HEARTBEAT_NAMESPACE_FMT, config.getClusterName(), lookupBroker)); } - public static NamespaceName getHeartbeatNamespaceV2(String host, ServiceConfiguration config) { - Integer port = null; - if (config.getWebServicePort().isPresent()) { - port = config.getWebServicePort().get(); - } else if (config.getWebServicePortTls().isPresent()) { - port = config.getWebServicePortTls().get(); - } - return NamespaceName.get(String.format(HEARTBEAT_NAMESPACE_FMT_V2, host, port)); + public static NamespaceName getHeartbeatNamespaceV2(String lookupBroker, ServiceConfiguration config) { + return NamespaceName.get(String.format(HEARTBEAT_NAMESPACE_FMT_V2, lookupBroker)); } - public static NamespaceName getSLAMonitorNamespace(String host, ServiceConfiguration config) { - Integer port = null; - if (config.getWebServicePort().isPresent()) { - port = config.getWebServicePort().get(); - } else if (config.getWebServicePortTls().isPresent()) { - port = config.getWebServicePortTls().get(); - } - return NamespaceName.get(String.format(SLA_NAMESPACE_FMT, config.getClusterName(), host, port)); + public static NamespaceName getSLAMonitorNamespace(String lookupBroker, ServiceConfiguration config) { + return NamespaceName.get(String.format(SLA_NAMESPACE_FMT, config.getClusterName(), lookupBroker)); } public static String checkHeartbeatNamespace(ServiceUnitId ns) { Matcher m = HEARTBEAT_NAMESPACE_PATTERN.matcher(ns.getNamespaceObject().toString()); if (m.matches()) { LOG.debug("Heartbeat namespace matched the lookup namespace {}", ns.getNamespaceObject().toString()); - return String.format("http://%s", m.group(1)); + return m.group(1); } else { return null; } @@ -1585,7 +1755,7 @@ public static String checkHeartbeatNamespaceV2(ServiceUnitId ns) { Matcher m = HEARTBEAT_NAMESPACE_PATTERN_V2.matcher(ns.getNamespaceObject().toString()); if (m.matches()) { LOG.debug("Heartbeat namespace v2 matched the lookup namespace {}", ns.getNamespaceObject().toString()); - return String.format("http://%s", m.group(1)); + return m.group(1); } else { return null; } @@ -1594,7 +1764,7 @@ public static String checkHeartbeatNamespaceV2(ServiceUnitId ns) { public static String getSLAMonitorBrokerName(ServiceUnitId ns) { Matcher m = SLA_NAMESPACE_PATTERN.matcher(ns.getNamespaceObject().toString()); if (m.matches()) { - return String.format("http://%s", m.group(1)); + return m.group(1); } else { return null; } @@ -1607,6 +1777,17 @@ public static boolean isSystemServiceNamespace(String namespace) { || HEARTBEAT_NAMESPACE_PATTERN_V2.matcher(namespace).matches(); } + /** + * used for filtering bundles in special namespace. + * @param namespace the namespace name + * @return True if namespace is HEARTBEAT_NAMESPACE or SLA_NAMESPACE + */ + public static boolean isSLAOrHeartbeatNamespace(String namespace) { + return SLA_NAMESPACE_PATTERN.matcher(namespace).matches() + || HEARTBEAT_NAMESPACE_PATTERN.matcher(namespace).matches() + || HEARTBEAT_NAMESPACE_PATTERN_V2.matcher(namespace).matches(); + } + public static boolean isHeartbeatNamespace(ServiceUnitId ns) { String namespace = ns.getNamespaceObject().toString(); return HEARTBEAT_NAMESPACE_PATTERN.matcher(namespace).matches() @@ -1614,14 +1795,16 @@ public static boolean isHeartbeatNamespace(ServiceUnitId ns) { } public boolean registerSLANamespace() throws PulsarServerException { - boolean isNameSpaceRegistered = registerNamespace(getSLAMonitorNamespace(host, config), false); + String brokerId = pulsar.getBrokerId(); + boolean isNameSpaceRegistered = registerNamespace(getSLAMonitorNamespace(brokerId, config), false); if (isNameSpaceRegistered) { if (LOG.isDebugEnabled()) { LOG.debug("Added SLA Monitoring namespace name in local cache: ns={}", - getSLAMonitorNamespace(host, config)); + getSLAMonitorNamespace(brokerId, config)); } } else if (LOG.isDebugEnabled()) { - LOG.debug("SLA Monitoring not owned by the broker: ns={}", getSLAMonitorNamespace(host, config)); + LOG.debug("SLA Monitoring not owned by the broker: ns={}", + getSLAMonitorNamespace(brokerId, config)); } return isNameSpaceRegistered; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnedBundle.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnedBundle.java index e7cf23a042750..cdedac1136e4d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnedBundle.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnedBundle.java @@ -136,7 +136,7 @@ public CompletableFuture handleUnloadRequest(PulsarService pulsar, long ti return pulsar.getNamespaceService().getOwnershipCache() .updateBundleState(this.bundle, false) .thenCompose(v -> pulsar.getBrokerService().unloadServiceUnit( - bundle, closeWithoutWaitingClientDisconnect, timeout, timeoutUnit)) + bundle, true, closeWithoutWaitingClientDisconnect, timeout, timeoutUnit)) .handle((numUnloadedTopics, ex) -> { if (ex != null) { // ignore topic-close failure to unload bundle diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java index 86003153714cb..9a4534f538774 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java @@ -36,7 +36,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.common.naming.NamespaceBundle; -import org.apache.pulsar.common.naming.NamespaceBundleFactory; import org.apache.pulsar.common.naming.NamespaceBundles; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.coordination.LockManager; @@ -115,17 +114,16 @@ public CompletableFuture asyncLoad(NamespaceBundle namespaceBundle, * * the local broker URL that will be set as owner for the ServiceUnit */ - public OwnershipCache(PulsarService pulsar, NamespaceBundleFactory bundleFactory, - NamespaceService namespaceService) { + public OwnershipCache(PulsarService pulsar, NamespaceService namespaceService) { this.namespaceService = namespaceService; this.pulsar = pulsar; this.ownerBrokerUrl = pulsar.getBrokerServiceUrl(); this.ownerBrokerUrlTls = pulsar.getBrokerServiceUrlTls(); this.selfOwnerInfo = new NamespaceEphemeralData(ownerBrokerUrl, ownerBrokerUrlTls, - pulsar.getSafeWebServiceAddress(), pulsar.getWebServiceAddressTls(), + pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), false, pulsar.getAdvertisedListeners()); this.selfOwnerInfoDisabled = new NamespaceEphemeralData(ownerBrokerUrl, ownerBrokerUrlTls, - pulsar.getSafeWebServiceAddress(), pulsar.getWebServiceAddressTls(), + pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), true, pulsar.getAdvertisedListeners()); this.lockManager = pulsar.getCoordinationService().getLockManager(NamespaceEphemeralData.class); this.locallyAcquiredLocks = new ConcurrentHashMap<>(); @@ -336,7 +334,7 @@ public Map> getLocallyAcqu public synchronized boolean refreshSelfOwnerInfo() { this.selfOwnerInfo = new NamespaceEphemeralData(pulsar.getBrokerServiceUrl(), - pulsar.getBrokerServiceUrlTls(), pulsar.getSafeWebServiceAddress(), + pulsar.getBrokerServiceUrlTls(), pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), false, pulsar.getAdvertisedListeners()); return selfOwnerInfo.getNativeUrl() != null || selfOwnerInfo.getNativeUrlTls() != null; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/ServiceUnitUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/ServiceUnitUtils.java index c86aac5316fb9..432aa29798ebd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/ServiceUnitUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/ServiceUnitUtils.java @@ -36,7 +36,7 @@ public final class ServiceUnitUtils { */ private static final String OWNER_INFO_ROOT = "/namespace"; - static String path(NamespaceBundle suname) { + public static String path(NamespaceBundle suname) { // The ephemeral node path for new namespaces should always have bundle name appended return OWNER_INFO_ROOT + "/" + suname.toString(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/TopicExistsInfo.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/TopicExistsInfo.java new file mode 100644 index 0000000000000..1c3f117719e8e --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/TopicExistsInfo.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.namespace; + +import io.netty.util.Recycler; +import lombok.Getter; +import org.apache.pulsar.common.policies.data.TopicType; + +public class TopicExistsInfo { + + private static final Recycler RECYCLER = new Recycler<>() { + @Override + protected TopicExistsInfo newObject(Handle handle) { + return new TopicExistsInfo(handle); + } + }; + + private static TopicExistsInfo nonPartitionedExists = new TopicExistsInfo(true, 0); + + private static TopicExistsInfo notExists = new TopicExistsInfo(false, 0); + + public static TopicExistsInfo newPartitionedTopicExists(Integer partitions){ + TopicExistsInfo info = RECYCLER.get(); + info.exists = true; + info.partitions = partitions.intValue(); + return info; + } + + public static TopicExistsInfo newNonPartitionedTopicExists(){ + return nonPartitionedExists; + } + + public static TopicExistsInfo newTopicNotExists(){ + return notExists; + } + + private final Recycler.Handle handle; + + @Getter + private int partitions; + @Getter + private boolean exists; + + private TopicExistsInfo(Recycler.Handle handle) { + this.handle = handle; + } + + private TopicExistsInfo(boolean exists, int partitions) { + this.handle = null; + this.partitions = partitions; + this.exists = exists; + } + + public void recycle() { + if (this == notExists || this == nonPartitionedExists || this.handle == null) { + return; + } + this.exists = false; + this.partitions = 0; + this.handle.recycle(this); + } + + public TopicType getTopicType() { + return this.partitions > 0 ? TopicType.PARTITIONED : TopicType.NON_PARTITIONED; + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/protocol/ProtocolHandlerWithClassLoader.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/protocol/ProtocolHandlerWithClassLoader.java index d648c261403d4..eb4bcb0a9bf4b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/protocol/ProtocolHandlerWithClassLoader.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/protocol/ProtocolHandlerWithClassLoader.java @@ -26,7 +26,6 @@ import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.broker.ClassLoaderSwitcher; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.common.nar.NarClassLoader; @@ -44,52 +43,79 @@ class ProtocolHandlerWithClassLoader implements ProtocolHandler { @Override public String protocolName() { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + ClassLoader prevClassLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(classLoader); return handler.protocolName(); + } finally { + Thread.currentThread().setContextClassLoader(prevClassLoader); } } @Override public boolean accept(String protocol) { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + ClassLoader prevClassLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(classLoader); return handler.accept(protocol); + } finally { + Thread.currentThread().setContextClassLoader(prevClassLoader); } } @Override public void initialize(ServiceConfiguration conf) throws Exception { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + ClassLoader prevClassLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(classLoader); handler.initialize(conf); + } finally { + Thread.currentThread().setContextClassLoader(prevClassLoader); } } @Override public String getProtocolDataToAdvertise() { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + ClassLoader prevClassLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(classLoader); return handler.getProtocolDataToAdvertise(); + } finally { + Thread.currentThread().setContextClassLoader(prevClassLoader); } } @Override public void start(BrokerService service) { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + ClassLoader prevClassLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(classLoader); handler.start(service); + } finally { + Thread.currentThread().setContextClassLoader(prevClassLoader); } } @Override public Map> newChannelInitializers() { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + ClassLoader prevClassLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(classLoader); return handler.newChannelInitializers(); + } finally { + Thread.currentThread().setContextClassLoader(prevClassLoader); } } @Override public void close() { - try (ClassLoaderSwitcher ignored = new ClassLoaderSwitcher(classLoader)) { + ClassLoader prevClassLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(classLoader); handler.close(); + } finally { + Thread.currentThread().setContextClassLoader(prevClassLoader); } - try { classLoader.close(); } catch (IOException e) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/protocol/ProtocolHandlers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/protocol/ProtocolHandlers.java index 42a82b2de762b..4059ccf5f26eb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/protocol/ProtocolHandlers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/protocol/ProtocolHandlers.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketAddress; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -51,6 +52,9 @@ public class ProtocolHandlers implements AutoCloseable { * @return the collection of protocol handlers */ public static ProtocolHandlers load(ServiceConfiguration conf) throws IOException { + if (conf.getMessagingProtocols().isEmpty()) { + return new ProtocolHandlers(Collections.emptyMap()); + } ProtocolHandlerDefinitions definitions = ProtocolHandlerUtils.searchForHandlers( conf.getProtocolHandlerDirectory(), conf.getNarExtractionDirectory()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/AsyncTokenBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/AsyncTokenBucket.java new file mode 100644 index 0000000000000..ac9a1f03e592b --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/AsyncTokenBucket.java @@ -0,0 +1,344 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.qos; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLongFieldUpdater; +import java.util.concurrent.atomic.LongAdder; + +/** + * An asynchronous token bucket algorithm implementation that is optimized for performance with highly concurrent + * use. CAS (compare-and-swap) operations are used and multiple levels of CAS fields are used to minimize contention + * when using CAS fields. The {@link LongAdder} class is used in the hot path to hold the sum of consumed tokens. + * It is eventually consistent, meaning that the tokens are not updated on every call to the "consumeTokens" method. + *

Main usage flow: + * 1. Tokens are consumed by invoking the "consumeTokens" or "consumeTokensAndCheckIfContainsTokens" methods. + * 2. The "consumeTokensAndCheckIfContainsTokens" or "containsTokens" methods return false if there are no + * tokens available, indicating a need for throttling. + * 3. In case of throttling, the application should throttle in a way that is suitable for the use case + * and then call the "calculateThrottlingDuration" method to calculate the duration of the required pause. + * 4. After the pause duration, the application should verify if there are any available tokens by invoking the + * containsTokens method. If tokens are available, the application should cease throttling. However, if tokens are + * not available, the application should maintain the throttling and recompute the throttling duration. In a + * concurrent environment, it is advisable to use a throttling queue to ensure fair distribution of resources across + * throttled connections or clients. Once the throttling duration has elapsed, the application should select the next + * connection or client from the throttling queue to unthrottle. Before unthrottling, the application should check + * for available tokens. If tokens are still not available, the application should continue with throttling and + * repeat the throttling loop. + *

This class does not produce side effects outside its own scope. It functions similarly to a stateful function, + * akin to a counter function. In essence, it is a sophisticated counter. It can serve as a foundational component for + * constructing higher-level asynchronous rate limiter implementations, which require side effects for throttling. + *

To achieve optimal performance, pass a {@link DefaultMonotonicSnapshotClock} instance as the clock . + */ +public abstract class AsyncTokenBucket { + public static final MonotonicSnapshotClock DEFAULT_SNAPSHOT_CLOCK = requestSnapshot -> System.nanoTime(); + static final long ONE_SECOND_NANOS = TimeUnit.SECONDS.toNanos(1); + // 2^24 nanoseconds is 16 milliseconds + private static final long DEFAULT_RESOLUTION_NANOS = TimeUnit.MILLISECONDS.toNanos(16); + + // The default resolution is 16 milliseconds. This means that the consumed tokens are subtracted from the + // current amount of tokens about every 16 milliseconds. This solution helps prevent a CAS loop what could cause + // extra CPU usage when a single CAS field is updated at a high rate from multiple threads. + static long defaultResolutionNanos = DEFAULT_RESOLUTION_NANOS; + + // used in tests to disable the optimization and instead use a consistent view of the tokens + public static void switchToConsistentTokensView() { + defaultResolutionNanos = 0; + } + + public static void resetToDefaultEventualConsistentTokensView() { + defaultResolutionNanos = DEFAULT_RESOLUTION_NANOS; + } + + // atomic field updaters for the volatile fields in this class + + private static final AtomicLongFieldUpdater LAST_NANOS_UPDATER = + AtomicLongFieldUpdater.newUpdater(AsyncTokenBucket.class, "lastNanos"); + + private static final AtomicLongFieldUpdater LAST_INCREMENT_UPDATER = + AtomicLongFieldUpdater.newUpdater(AsyncTokenBucket.class, "lastIncrement"); + + private static final AtomicLongFieldUpdater TOKENS_UPDATER = + AtomicLongFieldUpdater.newUpdater(AsyncTokenBucket.class, "tokens"); + + private static final AtomicLongFieldUpdater REMAINDER_NANOS_UPDATER = + AtomicLongFieldUpdater.newUpdater(AsyncTokenBucket.class, "remainderNanos"); + + /** + * This field represents the number of tokens in the bucket. It is eventually consistent, as the + * pendingConsumedTokens are subtracted from the total number of tokens at most once during each "tick" or + * "increment", when time advances according to the configured resolution. + */ + protected volatile long tokens; + /** + * This field represents the last time the tokens were updated, in nanoseconds. + * The configured clockSource is used to obtain the current nanoseconds. + * By default, a monotonic clock (System.nanoTime()) is used. + */ + private volatile long lastNanos; + /** + * This field represents the last time the tokens were updated, in increments. + */ + private volatile long lastIncrement; + /** + * As time progresses, tokens are added to the bucket. When the rate is low, significant rounding errors could + * accumulate over time if the remainder nanoseconds are not accounted for in the calculations. This field is used + * to carry forward the leftover nanoseconds in the update calculation. + */ + private volatile long remainderNanos; + + /** + * The resolution in nanoseconds. This is the amount of time that must pass before the tokens are updated. + */ + protected final long resolutionNanos; + /** + * This field is used to obtain the current monotonic clock time in nanoseconds. + */ + private final MonotonicSnapshotClock clockSource; + /** + * This field is used to hold the sum of consumed tokens that are pending to be subtracted from the total amount of + * tokens. This solution is to prevent CAS loop contention problem. pendingConsumedTokens used JVM's LongAdder + * which has a complex solution to prevent the CAS loop content problem. + */ + private final LongAdder pendingConsumedTokens = new LongAdder(); + + protected AsyncTokenBucket(MonotonicSnapshotClock clockSource, long resolutionNanos) { + this.clockSource = clockSource; + this.resolutionNanos = resolutionNanos; + } + + public static FinalRateAsyncTokenBucketBuilder builder() { + return new FinalRateAsyncTokenBucketBuilder(); + } + + public static DynamicRateAsyncTokenBucketBuilder builderForDynamicRate() { + return new DynamicRateAsyncTokenBucketBuilder(); + } + + protected abstract long getRatePeriodNanos(); + + protected abstract long getTargetAmountOfTokensAfterThrottling(); + + /** + * Consumes tokens and possibly updates the tokens balance. New tokens are calculated and added to the current + * tokens balance each time the update takes place. The update takes place once in every interval of the configured + * resolutionNanos or when the forceUpdateTokens parameter is true. + * When the tokens balance isn't updated, the consumed tokens are added to the pendingConsumedTokens LongAdder + * counter which gets flushed the next time the tokens are updated. This makes the tokens balance + * eventually consistent. The reason for this design choice is to optimize performance by preventing CAS loop + * contention which could cause excessive CPU consumption. + * + * @param consumeTokens number of tokens to consume, can be 0 to update the tokens balance + * @param forceUpdateTokens if true, the tokens are updated even if the configured resolution hasn't passed + * @return the current number of tokens in the bucket or Long.MIN_VALUE when the number of tokens is unknown due + * to eventual consistency + */ + private long consumeTokensAndMaybeUpdateTokensBalance(long consumeTokens, boolean forceUpdateTokens) { + if (consumeTokens < 0) { + throw new IllegalArgumentException("consumeTokens must be >= 0"); + } + long currentNanos = clockSource.getTickNanos(forceUpdateTokens); + // check if the tokens should be updated immediately + if (shouldUpdateTokensImmediately(currentNanos, forceUpdateTokens)) { + // calculate the number of new tokens since the last update + long newTokens = calculateNewTokensSinceLastUpdate(currentNanos); + // calculate the total amount of tokens to consume in this update + // flush the pendingConsumedTokens by calling "sumThenReset" + long totalConsumedTokens = consumeTokens + pendingConsumedTokens.sumThenReset(); + // update the tokens and return the current token value + return TOKENS_UPDATER.updateAndGet(this, + currentTokens -> + // after adding new tokens, limit the tokens to the capacity + Math.min(currentTokens + newTokens, getCapacity()) + // subtract the consumed tokens + - totalConsumedTokens); + } else { + // eventual consistent fast path, tokens are not updated immediately + + // add the consumed tokens to the pendingConsumedTokens LongAdder counter + if (consumeTokens > 0) { + pendingConsumedTokens.add(consumeTokens); + } + + // return Long.MIN_VALUE if the current value of tokens is unknown due to the eventual consistency + return Long.MIN_VALUE; + } + } + + /** + * Check if the tokens should be updated immediately. + * + * The tokens will be updated once every resolutionNanos nanoseconds. + * This method checks if the configured resolutionNanos has passed since the last update. + * If the forceUpdateTokens is true, the tokens will be updated immediately. + * + * @param currentNanos the current monotonic clock time in nanoseconds + * @param forceUpdateTokens if true, the tokens will be updated immediately + * @return true if the tokens should be updated immediately, false otherwise + */ + private boolean shouldUpdateTokensImmediately(long currentNanos, boolean forceUpdateTokens) { + long currentIncrement = resolutionNanos != 0 ? currentNanos / resolutionNanos : 0; + long currentLastIncrement = lastIncrement; + return currentIncrement == 0 + || (currentIncrement > currentLastIncrement + && LAST_INCREMENT_UPDATER.compareAndSet(this, currentLastIncrement, currentIncrement)) + || forceUpdateTokens; + } + + /** + * Calculate the number of new tokens since the last update. + * This will carry forward the remainder nanos so that a possible rounding error is eliminated. + * + * @param currentNanos the current monotonic clock time in nanoseconds + * @return the number of new tokens to add since the last update + */ + private long calculateNewTokensSinceLastUpdate(long currentNanos) { + long newTokens; + long previousLastNanos = LAST_NANOS_UPDATER.getAndSet(this, currentNanos); + if (previousLastNanos == 0) { + newTokens = 0; + } else { + long durationNanos = currentNanos - previousLastNanos + REMAINDER_NANOS_UPDATER.getAndSet(this, 0); + long currentRate = getRate(); + long currentRatePeriodNanos = getRatePeriodNanos(); + // new tokens is the amount of tokens that are created in the duration since the last update + // with the configured rate + newTokens = (durationNanos * currentRate) / currentRatePeriodNanos; + // carry forward the remainder nanos so that the rounding error is eliminated + long remainderNanos = durationNanos - ((newTokens * currentRatePeriodNanos) / currentRate); + if (remainderNanos > 0) { + REMAINDER_NANOS_UPDATER.addAndGet(this, remainderNanos); + } + } + return newTokens; + } + + /** + * Eventually consume tokens from the bucket. + * The number of tokens is eventually consistent with the configured granularity of resolutionNanos. + * + * @param consumeTokens the number of tokens to consume + */ + public void consumeTokens(long consumeTokens) { + consumeTokensAndMaybeUpdateTokensBalance(consumeTokens, false); + } + + /** + * Eventually consume tokens from the bucket and check if tokens remain available. + * The number of tokens is eventually consistent with the configured granularity of resolutionNanos. + * Therefore, the returned result is not definite. + * + * @param consumeTokens the number of tokens to consume + * @return true if there is tokens remains, false if tokens are all consumed. The answer isn't definite since the + * comparison is made with eventually consistent token value. + */ + public boolean consumeTokensAndCheckIfContainsTokens(long consumeTokens) { + long currentTokens = consumeTokensAndMaybeUpdateTokensBalance(consumeTokens, false); + if (currentTokens > 0) { + // tokens remain in the bucket + return true; + } else if (currentTokens == Long.MIN_VALUE) { + // when currentTokens is Long.MIN_VALUE, the current tokens balance is unknown since consumed tokens + // was added to the pendingConsumedTokens LongAdder counter. In this case, assume that tokens balance + // hasn't been updated yet and calculate a best guess of the current value by substracting the consumed + // tokens from the current tokens balance + return tokens - consumeTokens > 0; + } else { + // no tokens remain in the bucket + return false; + } + } + + /** + * Returns the current token balance. When forceUpdateTokens is true, the tokens balance is updated before + * returning. If forceUpdateTokens is false, the tokens balance could be updated if the last updated happened + * more than resolutionNanos nanoseconds ago. + * + * @param forceUpdateTokens if true, the tokens balance is updated before returning + * @return the current token balance + */ + protected long tokens(boolean forceUpdateTokens) { + long currentTokens = consumeTokensAndMaybeUpdateTokensBalance(0, forceUpdateTokens); + if (currentTokens != Long.MIN_VALUE) { + // when currentTokens isn't Long.MIN_VALUE, the current tokens balance is known + return currentTokens; + } else { + // return the current tokens balance, ignore the possible pendingConsumedTokens LongAdder counter + return tokens; + } + } + + /** + * Calculate the required throttling duration in nanoseconds to fill up the bucket with the minimum amount of + * tokens. + * This method shouldn't be called from the hot path since it calculates a consistent value for the tokens which + * isn't necessary on the hotpath. + */ + public long calculateThrottlingDuration() { + long currentTokens = consumeTokensAndMaybeUpdateTokensBalance(0, true); + if (currentTokens == Long.MIN_VALUE) { + throw new IllegalArgumentException( + "Unexpected result from updateAndConsumeTokens with forceUpdateTokens set to true"); + } + if (currentTokens > 0) { + return 0L; + } + // currentTokens is negative, so subtracting a negative value results in adding the absolute value (-(-x) -> +x) + long needTokens = getTargetAmountOfTokensAfterThrottling() - currentTokens; + return (needTokens * getRatePeriodNanos()) / getRate(); + } + + public abstract long getCapacity(); + + /** + * Returns the current number of tokens in the bucket. + * The token balance is updated if the configured resolutionNanos has passed since the last update. + */ + public final long getTokens() { + return tokens(false); + } + + public abstract long getRate(); + + /** + * Checks if the bucket contains tokens. + * The token balance is updated before the comparison if the configured resolutionNanos has passed since the last + * update. It's possible that the returned result is not definite since the token balance is eventually consistent. + * + * @return true if the bucket contains tokens, false otherwise + */ + public boolean containsTokens() { + return containsTokens(false); + } + + /** + * Checks if the bucket contains tokens. + * The token balance is updated before the comparison if the configured resolutionNanos has passed since the last + * update. The token balance is also updated when forceUpdateTokens is true. + * It's possible that the returned result is not definite since the token balance is eventually consistent. + * + * @param forceUpdateTokens if true, the token balance is updated before the comparison + * @return true if the bucket contains tokens, false otherwise + */ + public boolean containsTokens(boolean forceUpdateTokens) { + return tokens(forceUpdateTokens) > 0; + } + +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/AsyncTokenBucketBuilder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/AsyncTokenBucketBuilder.java new file mode 100644 index 0000000000000..ee256d5a37d64 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/AsyncTokenBucketBuilder.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.qos; + +// CHECKSTYLE.OFF: ClassTypeParameterName +public abstract class AsyncTokenBucketBuilder> { + protected MonotonicSnapshotClock clock = AsyncTokenBucket.DEFAULT_SNAPSHOT_CLOCK; + protected long resolutionNanos = AsyncTokenBucket.defaultResolutionNanos; + + protected AsyncTokenBucketBuilder() { + } + + protected SELF self() { + return (SELF) this; + } + + public SELF clock(MonotonicSnapshotClock clock) { + this.clock = clock; + return self(); + } + + public SELF resolutionNanos(long resolutionNanos) { + this.resolutionNanos = resolutionNanos; + return self(); + } + + public abstract AsyncTokenBucket build(); +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/DefaultMonotonicSnapshotClock.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/DefaultMonotonicSnapshotClock.java new file mode 100644 index 0000000000000..df3843921ed55 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/DefaultMonotonicSnapshotClock.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.qos; + +import java.util.concurrent.TimeUnit; +import java.util.function.LongSupplier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Default implementation of {@link MonotonicSnapshotClock}. + * + * Starts a daemon thread that updates the snapshot value periodically with a configured interval. The close method + * should be called to stop the thread. + */ +public class DefaultMonotonicSnapshotClock implements MonotonicSnapshotClock, AutoCloseable { + private static final Logger LOG = LoggerFactory.getLogger(DefaultMonotonicSnapshotClock.class); + private final long sleepMillis; + private final int sleepNanos; + private final LongSupplier clockSource; + private final Thread thread; + private volatile long snapshotTickNanos; + + public DefaultMonotonicSnapshotClock(long snapshotIntervalNanos, LongSupplier clockSource) { + if (snapshotIntervalNanos < TimeUnit.MILLISECONDS.toNanos(1)) { + throw new IllegalArgumentException("snapshotIntervalNanos must be at least 1 millisecond"); + } + this.sleepMillis = TimeUnit.NANOSECONDS.toMillis(snapshotIntervalNanos); + this.sleepNanos = (int) (snapshotIntervalNanos - TimeUnit.MILLISECONDS.toNanos(sleepMillis)); + this.clockSource = clockSource; + updateSnapshotTickNanos(); + thread = new Thread(this::snapshotLoop, getClass().getSimpleName() + "-update-loop"); + thread.setDaemon(true); + thread.start(); + } + + /** {@inheritDoc} */ + @Override + public long getTickNanos(boolean requestSnapshot) { + if (requestSnapshot) { + updateSnapshotTickNanos(); + } + return snapshotTickNanos; + } + + private void updateSnapshotTickNanos() { + snapshotTickNanos = clockSource.getAsLong(); + } + + private void snapshotLoop() { + try { + while (!Thread.currentThread().isInterrupted()) { + updateSnapshotTickNanos(); + try { + Thread.sleep(sleepMillis, sleepNanos); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + } catch (Throwable t) { + // report unexpected error since this would be a fatal error when the clock doesn't progress anymore + // this is very unlikely to happen, but it's better to log it in any case + LOG.error("Unexpected fatal error that stopped the clock.", t); + } + } + + @Override + public void close() { + thread.interrupt(); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/DynamicRateAsyncTokenBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/DynamicRateAsyncTokenBucket.java new file mode 100644 index 0000000000000..8edc73d1f51e3 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/DynamicRateAsyncTokenBucket.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.qos; + +import java.util.function.LongSupplier; + +/** + * A subclass of {@link AsyncTokenBucket} that represents a token bucket with a dynamic rate. + * The rate and capacity of the token bucket can change over time based on the rate function and capacity factor. + */ +public class DynamicRateAsyncTokenBucket extends AsyncTokenBucket { + private final LongSupplier rateFunction; + private final LongSupplier ratePeriodNanosFunction; + private final double capacityFactor; + + private final double targetFillFactorAfterThrottling; + + protected DynamicRateAsyncTokenBucket(double capacityFactor, LongSupplier rateFunction, + MonotonicSnapshotClock clockSource, LongSupplier ratePeriodNanosFunction, + long resolutionNanos, double initialTokensFactor, + double targetFillFactorAfterThrottling) { + super(clockSource, resolutionNanos); + this.capacityFactor = capacityFactor; + this.rateFunction = rateFunction; + this.ratePeriodNanosFunction = ratePeriodNanosFunction; + this.targetFillFactorAfterThrottling = targetFillFactorAfterThrottling; + this.tokens = (long) (rateFunction.getAsLong() * initialTokensFactor); + tokens(false); + } + + @Override + protected long getRatePeriodNanos() { + return ratePeriodNanosFunction.getAsLong(); + } + + @Override + protected long getTargetAmountOfTokensAfterThrottling() { + return (long) (getRate() * targetFillFactorAfterThrottling); + } + + @Override + public long getCapacity() { + return capacityFactor == 1.0d ? getRate() : (long) (getRate() * capacityFactor); + } + + @Override + public long getRate() { + return rateFunction.getAsLong(); + } + +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/DynamicRateAsyncTokenBucketBuilder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/DynamicRateAsyncTokenBucketBuilder.java new file mode 100644 index 0000000000000..22270484c72f0 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/DynamicRateAsyncTokenBucketBuilder.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.qos; + +import java.util.function.LongSupplier; + +/** + * A builder class for creating instances of {@link DynamicRateAsyncTokenBucket}. + */ +public class DynamicRateAsyncTokenBucketBuilder + extends AsyncTokenBucketBuilder { + protected LongSupplier rateFunction; + protected double capacityFactor = 1.0d; + protected double initialFillFactor = 1.0d; + protected LongSupplier ratePeriodNanosFunction; + protected double targetFillFactorAfterThrottling = 0.01d; + + protected DynamicRateAsyncTokenBucketBuilder() { + } + + public DynamicRateAsyncTokenBucketBuilder rateFunction(LongSupplier rateFunction) { + this.rateFunction = rateFunction; + return this; + } + + public DynamicRateAsyncTokenBucketBuilder ratePeriodNanosFunction(LongSupplier ratePeriodNanosFunction) { + this.ratePeriodNanosFunction = ratePeriodNanosFunction; + return this; + } + + public DynamicRateAsyncTokenBucketBuilder capacityFactor(double capacityFactor) { + this.capacityFactor = capacityFactor; + return this; + } + + public DynamicRateAsyncTokenBucketBuilder initialFillFactor(double initialFillFactor) { + this.initialFillFactor = initialFillFactor; + return this; + } + + public DynamicRateAsyncTokenBucketBuilder targetFillFactorAfterThrottling( + double targetFillFactorAfterThrottling) { + this.targetFillFactorAfterThrottling = targetFillFactorAfterThrottling; + return this; + } + + @Override + public AsyncTokenBucket build() { + return new DynamicRateAsyncTokenBucket(this.capacityFactor, this.rateFunction, + this.clock, + this.ratePeriodNanosFunction, this.resolutionNanos, + this.initialFillFactor, + targetFillFactorAfterThrottling); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/FinalRateAsyncTokenBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/FinalRateAsyncTokenBucket.java new file mode 100644 index 0000000000000..627c5ee1334b2 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/FinalRateAsyncTokenBucket.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.qos; + +/** + * A subclass of {@link AsyncTokenBucket} that represents a token bucket with a rate which is final. + * The rate and capacity of the token bucket are constant and do not change over time. + */ +class FinalRateAsyncTokenBucket extends AsyncTokenBucket { + private final long capacity; + private final long rate; + private final long ratePeriodNanos; + private final long targetAmountOfTokensAfterThrottling; + + protected FinalRateAsyncTokenBucket(long capacity, long rate, MonotonicSnapshotClock clockSource, + long ratePeriodNanos, long resolutionNanos, long initialTokens) { + super(clockSource, resolutionNanos); + this.capacity = capacity; + this.rate = rate; + this.ratePeriodNanos = ratePeriodNanos != -1 ? ratePeriodNanos : ONE_SECOND_NANOS; + // The target amount of tokens is the amount of tokens made available in the resolution duration + this.targetAmountOfTokensAfterThrottling = Math.max(this.resolutionNanos * rate / ratePeriodNanos, 1); + this.tokens = initialTokens; + tokens(false); + } + + @Override + protected final long getRatePeriodNanos() { + return ratePeriodNanos; + } + + @Override + protected final long getTargetAmountOfTokensAfterThrottling() { + return targetAmountOfTokensAfterThrottling; + } + + @Override + public final long getCapacity() { + return capacity; + } + + @Override + public final long getRate() { + return rate; + } + +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/FinalRateAsyncTokenBucketBuilder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/FinalRateAsyncTokenBucketBuilder.java new file mode 100644 index 0000000000000..ff4ed53c6c7fa --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/FinalRateAsyncTokenBucketBuilder.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.qos; + +/** + * A builder class for creating instances of {@link FinalRateAsyncTokenBucket}. + */ +public class FinalRateAsyncTokenBucketBuilder + extends AsyncTokenBucketBuilder { + protected Long capacity; + protected Long initialTokens; + protected Long rate; + protected long ratePeriodNanos = AsyncTokenBucket.ONE_SECOND_NANOS; + + protected FinalRateAsyncTokenBucketBuilder() { + } + + public FinalRateAsyncTokenBucketBuilder rate(long rate) { + this.rate = rate; + return this; + } + + public FinalRateAsyncTokenBucketBuilder ratePeriodNanos(long ratePeriodNanos) { + this.ratePeriodNanos = ratePeriodNanos; + return this; + } + + public FinalRateAsyncTokenBucketBuilder capacity(long capacity) { + this.capacity = capacity; + return this; + } + + public FinalRateAsyncTokenBucketBuilder initialTokens(long initialTokens) { + this.initialTokens = initialTokens; + return this; + } + + public AsyncTokenBucket build() { + return new FinalRateAsyncTokenBucket(this.capacity != null ? this.capacity : this.rate, this.rate, + this.clock, + this.ratePeriodNanos, this.resolutionNanos, + this.initialTokens != null ? this.initialTokens : this.rate + ); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/MonotonicSnapshotClock.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/MonotonicSnapshotClock.java new file mode 100644 index 0000000000000..8f61bd5125b5f --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/MonotonicSnapshotClock.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.qos; + +/** + * An interface representing a clock that provides a monotonic counter in nanoseconds. + * The counter is guaranteed to be monotonic, ensuring it will always increase or remain constant, but never decrease. + * + * Monotonicity ensures the time will always progress forward, making it ideal for measuring elapsed time. + * The monotonic clock is not related to the wall-clock time and is not affected by changes to the system time. + * The tick value is only significant when compared to other values obtained from the same clock source + * and should not be used for other purposes. + * + * This interface assumes that the implementation can be implemented in a granular way. This means that the value is + * advanced in steps of a configurable resolution that snapshots the underlying high precision monotonic clock source + * value. + * This design allows for optimizations that can improve performance on platforms where obtaining the value of a + * platform monotonic clock is relatively expensive. + */ +public interface MonotonicSnapshotClock { + /** + * Retrieves the latest snapshot of the tick value of the monotonic clock in nanoseconds. + * + * When requestSnapshot is set to true, the method will snapshot the underlying high-precision monotonic clock + * source so that the latest snapshot value is as accurate as possible. This may be a relatively expensive + * compared to a non-snapshot request. + * + * When requestSnapshot is set to false, the method will return the latest snapshot value which is updated by + * either a call that requested a snapshot or by an update thread that is configured to update the snapshot value + * periodically. + * + * This method returns a value that is guaranteed to be monotonic, meaning it will always increase or remain the + * same, never decrease. The returned value is only significant when compared to other values obtained from the same + * clock source and should not be used for other purposes. + * + * @param requestSnapshot If set to true, the method will request a new snapshot from the underlying more + * high-precision monotonic clock. + * @return The current tick value of the monotonic clock in nanoseconds. + */ + long getTickNanos(boolean requestSnapshot); +} \ No newline at end of file diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/package-info.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/package-info.java similarity index 89% rename from pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/package-info.java rename to pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/package-info.java index 3f11037fcf649..1078d86894efe 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/package-info.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/qos/package-info.java @@ -17,6 +17,6 @@ * under the License. */ /** - * Implementation of the connector to the Presto engine. + * Pulsar broker Quality of Service (QoS) related classes. */ -package org.apache.pulsar.sql.presto; \ No newline at end of file +package org.apache.pulsar.broker.qos; \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroup.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroup.java index ef40a18ab08ed..541a645f18bf3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroup.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroup.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.resourcegroup; +import com.google.common.annotations.VisibleForTesting; import io.prometheus.client.Counter; import java.util.HashMap; import java.util.Set; @@ -81,7 +82,8 @@ protected ResourceGroup(ResourceGroupService rgs, String name, this.setResourceGroupMonitoringClassFields(); this.setResourceGroupConfigParameters(rgConfig); this.setDefaultResourceUsageTransportHandlers(); - this.resourceGroupPublishLimiter = new ResourceGroupPublishLimiter(rgConfig, rgs.getPulsar().getExecutor()); + this.resourceGroupPublishLimiter = new ResourceGroupPublishLimiter(rgConfig, rgs.getPulsar() + .getMonotonicSnapshotClock()); log.info("attaching publish rate limiter {} to {} get {}", this.resourceGroupPublishLimiter, name, this.getResourceGroupPublishLimiter()); } @@ -96,7 +98,8 @@ protected ResourceGroup(ResourceGroupService rgs, String rgName, this.resourceGroupName = rgName; this.setResourceGroupMonitoringClassFields(); this.setResourceGroupConfigParameters(rgConfig); - this.resourceGroupPublishLimiter = new ResourceGroupPublishLimiter(rgConfig, rgs.getPulsar().getExecutor()); + this.resourceGroupPublishLimiter = new ResourceGroupPublishLimiter(rgConfig, rgs.getPulsar() + .getMonotonicSnapshotClock()); this.ruPublisher = rgPublisher; this.ruConsumer = rgConsumer; } @@ -216,24 +219,28 @@ public void rgFillResourceUsage(ResourceUsage resourceUsage) { resourceUsage.setOwner(this.getID()); p = resourceUsage.setPublish(); - this.setUsageInMonitoredEntity(ResourceGroupMonitoringClass.Publish, p); + if (!this.setUsageInMonitoredEntity(ResourceGroupMonitoringClass.Publish, p)) { + resourceUsage.clearPublish(); + } p = resourceUsage.setDispatch(); - this.setUsageInMonitoredEntity(ResourceGroupMonitoringClass.Dispatch, p); + if (!this.setUsageInMonitoredEntity(ResourceGroupMonitoringClass.Dispatch, p)) { + resourceUsage.clearDispatch(); + } // Punt storage for now. } // Transport manager mandated op. public void rgResourceUsageListener(String broker, ResourceUsage resourceUsage) { - NetworkUsage p; - - p = resourceUsage.getPublish(); - this.getUsageFromMonitoredEntity(ResourceGroupMonitoringClass.Publish, p, broker); - - p = resourceUsage.getDispatch(); - this.getUsageFromMonitoredEntity(ResourceGroupMonitoringClass.Dispatch, p, broker); + if (resourceUsage.hasPublish()) { + this.getUsageFromMonitoredEntity(ResourceGroupMonitoringClass.Publish, resourceUsage.getPublish(), broker); + } + if (resourceUsage.hasDispatch()) { + this.getUsageFromMonitoredEntity(ResourceGroupMonitoringClass.Dispatch, resourceUsage.getDispatch(), + broker); + } // Punt storage for now. } @@ -323,8 +330,10 @@ protected BytesAndMessagesCount getLocalUsageStatsFromBrokerReports(ResourceGrou retval.bytes = pbus.usedValues.bytes; retval.messages = pbus.usedValues.messages; } else { - log.info("getLocalUsageStatsFromBrokerReports: no usage report found for broker={} and monClass={}", - myBrokerId, monClass); + if (log.isDebugEnabled()) { + log.debug("getLocalUsageStatsFromBrokerReports: no usage report found for broker={} and monClass={}", + myBrokerId, monClass); + } } return retval; @@ -404,8 +413,8 @@ protected static double getRgRemoteUsageMessageCount (String rgName, String monC } // Visibility for unit testing - protected static double getRgUsageReportedCount (String rgName, String monClassName) { - return rgLocalUsageReportCount.labels(rgName, monClassName).get(); + protected static long getRgUsageReportedCount (String rgName, String monClassName) { + return (long) rgLocalUsageReportCount.labels(rgName, monClassName).get(); } // Visibility for unit testing @@ -450,18 +459,15 @@ protected boolean setUsageInMonitoredEntity(ResourceGroupMonitoringClass monClas bytesUsed = monEntity.usedLocallySinceLastReport.bytes; messagesUsed = monEntity.usedLocallySinceLastReport.messages; monEntity.usedLocallySinceLastReport.bytes = monEntity.usedLocallySinceLastReport.messages = 0; - - monEntity.totalUsedLocally.bytes += bytesUsed; - monEntity.totalUsedLocally.messages += messagesUsed; - - monEntity.lastResourceUsageFillTimeMSecsSinceEpoch = System.currentTimeMillis(); - if (sendReport) { p.setBytesPerPeriod(bytesUsed); p.setMessagesPerPeriod(messagesUsed); monEntity.lastReportedValues.bytes = bytesUsed; monEntity.lastReportedValues.messages = messagesUsed; monEntity.numSuppressedUsageReports = 0; + monEntity.totalUsedLocally.bytes += bytesUsed; + monEntity.totalUsedLocally.messages += messagesUsed; + monEntity.lastResourceUsageFillTimeMSecsSinceEpoch = System.currentTimeMillis(); } else { numSuppressions = monEntity.numSuppressedUsageReports++; } @@ -594,6 +600,11 @@ public void acceptResourceUsage(String broker, ResourceUsage resourceUsage) { }; } + @VisibleForTesting + PerMonitoringClassFields getMonitoredEntity(ResourceGroupMonitoringClass monClass) { + return this.monitoringClassFields[monClass.ordinal()]; + } + public final String resourceGroupName; public PerMonitoringClassFields[] monitoringClassFields = diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupConfigListener.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupConfigListener.java index c15edd2be4e43..4a5b8a8bcc244 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupConfigListener.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupConfigListener.java @@ -18,15 +18,22 @@ */ package org.apache.pulsar.broker.resourcegroup; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.resources.ResourceGroupResources; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.common.policies.data.ResourceGroup; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.Notification; import org.apache.pulsar.metadata.api.NotificationType; import org.slf4j.Logger; @@ -47,24 +54,32 @@ public class ResourceGroupConfigListener implements Consumer { private final ResourceGroupService rgService; private final PulsarService pulsarService; private final ResourceGroupResources rgResources; - private final ResourceGroupNamespaceConfigListener rgNamespaceConfigListener; + private volatile ResourceGroupNamespaceConfigListener rgNamespaceConfigListener; public ResourceGroupConfigListener(ResourceGroupService rgService, PulsarService pulsarService) { this.rgService = rgService; this.pulsarService = pulsarService; this.rgResources = pulsarService.getPulsarResources().getResourcegroupResources(); - loadAllResourceGroups(); this.rgResources.getStore().registerListener(this); - rgNamespaceConfigListener = new ResourceGroupNamespaceConfigListener( - rgService, pulsarService, this); + execute(() -> loadAllResourceGroupsWithRetryAsync(0)); } - private void loadAllResourceGroups() { - rgResources.listResourceGroupsAsync().whenCompleteAsync((rgList, ex) -> { - if (ex != null) { - LOG.error("Exception when fetching resource groups", ex); - return; + private void loadAllResourceGroupsWithRetryAsync(long retry) { + loadAllResourceGroupsAsync().thenAccept(__ -> { + if (rgNamespaceConfigListener == null) { + rgNamespaceConfigListener = new ResourceGroupNamespaceConfigListener(rgService, pulsarService, this); } + }).exceptionally(e -> { + long nextRetry = retry + 1; + long delay = 500 * nextRetry; + LOG.error("Failed to load all resource groups during initialization, retrying after {}ms: ", delay, e); + schedule(() -> loadAllResourceGroupsWithRetryAsync(nextRetry), delay); + return null; + }); + } + + private CompletableFuture loadAllResourceGroupsAsync() { + return rgResources.listResourceGroupsAsync().thenCompose(rgList -> { final Set existingSet = rgService.resourceGroupGetAll(); HashSet newSet = new HashSet<>(); @@ -72,21 +87,26 @@ private void loadAllResourceGroups() { final Sets.SetView deleteList = Sets.difference(existingSet, newSet); - for (String rgName: deleteList) { + for (String rgName : deleteList) { deleteResourceGroup(rgName); } final Sets.SetView addList = Sets.difference(newSet, existingSet); - for (String rgName: addList) { - pulsarService.getPulsarResources().getResourcegroupResources() - .getResourceGroupAsync(rgName).thenAcceptAsync(optionalRg -> { - ResourceGroup rg = optionalRg.get(); - createResourceGroup(rgName, rg); - }).exceptionally((ex1) -> { - LOG.error("Failed to fetch resourceGroup", ex1); - return null; - }); + List> futures = new ArrayList<>(); + for (String rgName : addList) { + futures.add(pulsarService.getPulsarResources() + .getResourcegroupResources() + .getResourceGroupAsync(rgName) + .thenAccept(optionalRg -> { + if (optionalRg.isPresent()) { + ResourceGroup rg = optionalRg.get(); + createResourceGroup(rgName, rg); + } + }) + ); } + + return FutureUtil.waitForAll(futures); }); } @@ -140,7 +160,10 @@ public void accept(Notification notification) { Optional rgName = ResourceGroupResources.resourceGroupNameFromPath(notifyPath); if ((notification.getType() == NotificationType.ChildrenChanged) || (notification.getType() == NotificationType.Created)) { - loadAllResourceGroups(); + loadAllResourceGroupsAsync().exceptionally((ex) -> { + LOG.error("Exception when fetching resource groups", ex); + return null; + }); } else if (rgName.isPresent()) { switch (notification.getType()) { case Modified: @@ -151,4 +174,17 @@ public void accept(Notification notification) { } } } + + protected void execute(Runnable runnable) { + pulsarService.getExecutor().execute(catchingAndLoggingThrowables(runnable)); + } + + protected void schedule(Runnable runnable, long delayMs) { + pulsarService.getExecutor().schedule(catchingAndLoggingThrowables(runnable), delayMs, TimeUnit.MILLISECONDS); + } + + @VisibleForTesting + ResourceGroupNamespaceConfigListener getRgNamespaceConfigListener() { + return rgNamespaceConfigListener; + } } \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupPublishLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupPublishLimiter.java index 85e00bb2f87dc..a733db555a351 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupPublishLimiter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupPublishLimiter.java @@ -18,53 +18,22 @@ */ package org.apache.pulsar.broker.resourcegroup; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; +import org.apache.pulsar.broker.qos.MonotonicSnapshotClock; import org.apache.pulsar.broker.resourcegroup.ResourceGroup.BytesAndMessagesCount; -import org.apache.pulsar.broker.service.PublishRateLimiter; +import org.apache.pulsar.broker.service.PublishRateLimiterImpl; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.PublishRate; import org.apache.pulsar.common.policies.data.ResourceGroup; -import org.apache.pulsar.common.util.RateLimitFunction; -import org.apache.pulsar.common.util.RateLimiter; -public class ResourceGroupPublishLimiter implements PublishRateLimiter, RateLimitFunction, AutoCloseable { - protected volatile long publishMaxMessageRate = 0; - protected volatile long publishMaxByteRate = 0; - protected volatile boolean publishThrottlingEnabled = false; - private volatile RateLimiter publishRateLimiterOnMessage; - private volatile RateLimiter publishRateLimiterOnByte; - private final ScheduledExecutorService scheduledExecutorService; +public class ResourceGroupPublishLimiter extends PublishRateLimiterImpl { + private volatile long publishMaxMessageRate; + private volatile long publishMaxByteRate; - ConcurrentHashMap rateLimitFunctionMap = new ConcurrentHashMap<>(); - - public ResourceGroupPublishLimiter(ResourceGroup resourceGroup, ScheduledExecutorService scheduledExecutorService) { - this.scheduledExecutorService = scheduledExecutorService; + public ResourceGroupPublishLimiter(ResourceGroup resourceGroup, MonotonicSnapshotClock monotonicSnapshotClock) { + super(monotonicSnapshotClock); update(resourceGroup); } - @Override - public void checkPublishRate() { - // No-op - } - - @Override - public void incrementPublishCount(int numOfMessages, long msgSizeInBytes) { - // No-op - } - - @Override - public boolean resetPublishCount() { - return true; - } - - @Override - public boolean isPublishRateExceeded() { - return false; - } - @Override public void update(Policies policies, String clusterName) { // No-op @@ -94,102 +63,12 @@ public void update(ResourceGroup resourceGroup) { publishRateInMsgs = resourceGroup.getPublishRateInMsgs() == null ? -1 : resourceGroup.getPublishRateInMsgs(); } - update(publishRateInMsgs, publishRateInBytes); } public void update(long publishRateInMsgs, long publishRateInBytes) { - replaceLimiters(() -> { - if (publishRateInMsgs > 0 || publishRateInBytes > 0) { - this.publishThrottlingEnabled = true; - this.publishMaxMessageRate = Math.max(publishRateInMsgs, 0); - this.publishMaxByteRate = Math.max(publishRateInBytes, 0); - if (this.publishMaxMessageRate > 0) { - publishRateLimiterOnMessage = RateLimiter.builder() - .scheduledExecutorService(scheduledExecutorService) - .permits(publishMaxMessageRate) - .rateTime(1L) - .timeUnit(TimeUnit.SECONDS) - .rateLimitFunction(this::apply) - .build(); - } - if (this.publishMaxByteRate > 0) { - publishRateLimiterOnByte = - RateLimiter.builder() - .scheduledExecutorService(scheduledExecutorService) - .permits(publishMaxByteRate) - .rateTime(1L) - .timeUnit(TimeUnit.SECONDS) - .rateLimitFunction(this::apply) - .build(); - } - } else { - this.publishMaxMessageRate = 0; - this.publishMaxByteRate = 0; - this.publishThrottlingEnabled = false; - publishRateLimiterOnMessage = null; - publishRateLimiterOnByte = null; - } - }); - } - - public boolean tryAcquire(int numbers, long bytes) { - return (publishRateLimiterOnMessage == null || publishRateLimiterOnMessage.tryAcquire(numbers)) - && (publishRateLimiterOnByte == null || publishRateLimiterOnByte.tryAcquire(bytes)); - } - - public void registerRateLimitFunction(String name, RateLimitFunction func) { - rateLimitFunctionMap.put(name, func); - } - - public void unregisterRateLimitFunction(String name) { - rateLimitFunctionMap.remove(name); - } - - private void replaceLimiters(Runnable updater) { - RateLimiter previousPublishRateLimiterOnMessage = publishRateLimiterOnMessage; - publishRateLimiterOnMessage = null; - RateLimiter previousPublishRateLimiterOnByte = publishRateLimiterOnByte; - publishRateLimiterOnByte = null; - try { - if (updater != null) { - updater.run(); - } - } finally { - // Close previous limiters to prevent resource leakages. - // Delay closing of previous limiters after new ones are in place so that updating the limiter - // doesn't cause unavailability. - if (previousPublishRateLimiterOnMessage != null) { - previousPublishRateLimiterOnMessage.close(); - } - if (previousPublishRateLimiterOnByte != null) { - previousPublishRateLimiterOnByte.close(); - } - } - } - - @Override - public void close() { - // Unblock any producers, consumers waiting first. - // This needs to be done before replacing the filters to null - this.apply(); - replaceLimiters(null); - } - - @Override - public void apply() { - // Make sure that both the rate limiters are applied before opening the flood gates. - RateLimiter currentTopicPublishRateLimiterOnMessage = publishRateLimiterOnMessage; - RateLimiter currentTopicPublishRateLimiterOnByte = publishRateLimiterOnByte; - if ((currentTopicPublishRateLimiterOnMessage != null - && currentTopicPublishRateLimiterOnMessage.getAvailablePermits() <= 0) - || (currentTopicPublishRateLimiterOnByte != null - && currentTopicPublishRateLimiterOnByte.getAvailablePermits() <= 0)) { - return; - } - - for (Map.Entry entry: rateLimitFunctionMap.entrySet()) { - entry.getValue().apply(); - } + this.publishMaxMessageRate = publishRateInMsgs; + this.publishMaxByteRate = publishRateInBytes; + updateTokenBuckets(publishRateInMsgs, publishRateInBytes); } } \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java index d3f8eb7613a40..29633ab19feff 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java @@ -173,7 +173,6 @@ public void resourceGroupDelete(String name) throws PulsarAdminException { throw new PulsarAdminException(errMesg); } - rg.resourceGroupPublishLimiter.close(); rg.resourceGroupPublishLimiter = null; resourceGroupsMap.remove(name); } @@ -484,48 +483,48 @@ protected BytesAndMessagesCount getPublishRateLimiters (String rgName) throws Pu } // Visibility for testing. - protected static double getRgQuotaByteCount (String rgName, String monClassName) { - return rgCalculatedQuotaBytes.labels(rgName, monClassName).get(); + protected static long getRgQuotaByteCount (String rgName, String monClassName) { + return (long) rgCalculatedQuotaBytes.labels(rgName, monClassName).get(); } // Visibility for testing. - protected static double getRgQuotaMessageCount (String rgName, String monClassName) { - return rgCalculatedQuotaMessages.labels(rgName, monClassName).get(); + protected static long getRgQuotaMessageCount (String rgName, String monClassName) { + return (long) rgCalculatedQuotaMessages.labels(rgName, monClassName).get(); } // Visibility for testing. - protected static double getRgLocalUsageByteCount (String rgName, String monClassName) { - return rgLocalUsageBytes.labels(rgName, monClassName).get(); + protected static long getRgLocalUsageByteCount (String rgName, String monClassName) { + return (long) rgLocalUsageBytes.labels(rgName, monClassName).get(); } // Visibility for testing. - protected static double getRgLocalUsageMessageCount (String rgName, String monClassName) { - return rgLocalUsageMessages.labels(rgName, monClassName).get(); + protected static long getRgLocalUsageMessageCount (String rgName, String monClassName) { + return (long) rgLocalUsageMessages.labels(rgName, monClassName).get(); } // Visibility for testing. - protected static double getRgUpdatesCount (String rgName) { - return rgUpdates.labels(rgName).get(); + protected static long getRgUpdatesCount (String rgName) { + return (long) rgUpdates.labels(rgName).get(); } // Visibility for testing. - protected static double getRgTenantRegistersCount (String rgName) { - return rgTenantRegisters.labels(rgName).get(); + protected static long getRgTenantRegistersCount (String rgName) { + return (long) rgTenantRegisters.labels(rgName).get(); } // Visibility for testing. - protected static double getRgTenantUnRegistersCount (String rgName) { - return rgTenantUnRegisters.labels(rgName).get(); + protected static long getRgTenantUnRegistersCount (String rgName) { + return (long) rgTenantUnRegisters.labels(rgName).get(); } // Visibility for testing. - protected static double getRgNamespaceRegistersCount (String rgName) { - return rgNamespaceRegisters.labels(rgName).get(); + protected static long getRgNamespaceRegistersCount (String rgName) { + return (long) rgNamespaceRegisters.labels(rgName).get(); } // Visibility for testing. - protected static double getRgNamespaceUnRegistersCount (String rgName) { - return rgNamespaceUnRegisters.labels(rgName).get(); + protected static long getRgNamespaceUnRegistersCount (String rgName) { + return (long) rgNamespaceUnRegisters.labels(rgName).get(); } // Visibility for testing. @@ -687,7 +686,7 @@ protected void calculateQuotaForAllResourceGroups() { timeUnitScale); this.resourceUsagePublishPeriodInSeconds = newPeriodInSeconds; maxIntervalForSuppressingReportsMSecs = - this.resourceUsagePublishPeriodInSeconds * MaxUsageReportSuppressRounds; + TimeUnit.SECONDS.toMillis(this.resourceUsagePublishPeriodInSeconds) * MaxUsageReportSuppressRounds; } } @@ -706,7 +705,7 @@ private void initialize() { periodInSecs, this.timeUnitScale); maxIntervalForSuppressingReportsMSecs = - this.resourceUsagePublishPeriodInSeconds * MaxUsageReportSuppressRounds; + TimeUnit.SECONDS.toMillis(this.resourceUsagePublishPeriodInSeconds) * MaxUsageReportSuppressRounds; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImpl.java index 41291100a0ddf..b3000c9d77fe5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImpl.java @@ -84,8 +84,10 @@ public long computeLocalQuota(long confUsage, long myUsage, long[] allUsages) th float calculatedQuota = max(myUsage + residual * myUsageFraction, 1); val longCalculatedQuota = (long) calculatedQuota; - log.info("computeLocalQuota: myUsage={}, totalUsage={}, myFraction={}; newQuota returned={} [long: {}]", - myUsage, totalUsage, myUsageFraction, calculatedQuota, longCalculatedQuota); + if (log.isDebugEnabled()) { + log.debug("computeLocalQuota: myUsage={}, totalUsage={}, myFraction={}; newQuota returned={} [long: {}]", + myUsage, totalUsage, myUsageFraction, calculatedQuota, longCalculatedQuota); + } return longCalculatedQuota; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/RestMessagePublishContext.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/RestMessagePublishContext.java index 3c9adbd3e4fe4..f3b84090056be 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/RestMessagePublishContext.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/RestMessagePublishContext.java @@ -22,7 +22,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.pulsar.broker.service.Topic; /** @@ -33,7 +34,7 @@ public class RestMessagePublishContext implements Topic.PublishContext { private Topic topic; private long startTimeNs; - private CompletableFuture positionFuture; + private CompletableFuture positionFuture; /** * Executed from managed ledger thread when the message is persisted. @@ -54,13 +55,13 @@ public void completed(Exception exception, long ledgerId, long entryId) { topic.getName(), ledgerId, entryId); } topic.recordAddLatency(System.nanoTime() - startTimeNs, TimeUnit.NANOSECONDS); - positionFuture.complete(PositionImpl.get(ledgerId, entryId)); + positionFuture.complete(PositionFactory.create(ledgerId, entryId)); } recycle(); } // recycler - public static RestMessagePublishContext get(CompletableFuture positionFuture, Topic topic, + public static RestMessagePublishContext get(CompletableFuture positionFuture, Topic topic, long startTimeNs) { RestMessagePublishContext callback = RECYCLER.get(); callback.positionFuture = positionFuture; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java index 6f3ac7f8c09ca..bf6e7350186c2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java @@ -38,7 +38,9 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import javax.ws.rs.container.AsyncResponse; @@ -53,7 +55,8 @@ import org.apache.avro.io.Decoder; import org.apache.avro.io.DecoderFactory; import org.apache.bookkeeper.mledger.ManagedLedgerException; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.admin.impl.PersistentTopicsBase; import org.apache.pulsar.broker.authentication.AuthenticationParameters; @@ -94,7 +97,6 @@ import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.ObjectMapperFactory; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet; import org.apache.pulsar.websocket.data.ProducerAck; import org.apache.pulsar.websocket.data.ProducerAcks; import org.apache.pulsar.websocket.data.ProducerMessage; @@ -121,8 +123,9 @@ protected void publishMessages(AsyncResponse asyncResponse, ProducerMessages req .thenAccept(schemaMeta -> { // Both schema version and schema data are necessary. if (schemaMeta.getLeft() != null && schemaMeta.getRight() != null) { - internalPublishMessages(topicName, request, pulsar().getBrokerService() - .getOwningTopics().get(topic).values(), asyncResponse, + final var partitionIndexes = pulsar().getBrokerService().getOwningTopics() + .getOrDefault(topic, Set.of()).stream().toList(); + internalPublishMessages(topicName, request, partitionIndexes, asyncResponse, AutoConsumeSchema.getSchema(schemaMeta.getLeft().toSchemaInfo()), schemaMeta.getRight()); } else { @@ -193,7 +196,7 @@ private void internalPublishMessagesToPartition(TopicName topicName, ProducerMes String producerName = (null == request.getProducerName() || request.getProducerName().isEmpty()) ? defaultProducerName : request.getProducerName(); List messages = buildMessage(request, schema, producerName, topicName); - List> publishResults = new ArrayList<>(); + List> publishResults = new ArrayList<>(); List produceMessageResults = new ArrayList<>(); for (int index = 0; index < messages.size(); index++) { ProducerAck produceMessageResult = new ProducerAck(); @@ -234,7 +237,7 @@ private void internalPublishMessages(TopicName topicName, ProducerMessages reque String producerName = (null == request.getProducerName() || request.getProducerName().isEmpty()) ? defaultProducerName : request.getProducerName(); List messages = buildMessage(request, schema, producerName, topicName); - List> publishResults = new ArrayList<>(); + List> publishResults = new ArrayList<>(); List produceMessageResults = new ArrayList<>(); // Try to publish messages to all partitions this broker owns in round robin mode. for (int index = 0; index < messages.size(); index++) { @@ -265,8 +268,8 @@ private void internalPublishMessages(TopicName topicName, ProducerMessages reque } } - private CompletableFuture publishSingleMessageToPartition(String topic, Message message) { - CompletableFuture publishResult = new CompletableFuture<>(); + private CompletableFuture publishSingleMessageToPartition(String topic, Message message) { + CompletableFuture publishResult = new CompletableFuture<>(); pulsar().getBrokerService().getTopic(topic, false) .thenAccept(t -> { // TODO: Check message backlog and fail if backlog too large. @@ -296,11 +299,11 @@ private CompletableFuture publishSingleMessageToPartition(String t // Process results for all message publishing attempts private void processPublishMessageResults(List produceMessageResults, - List> publishResults) { + List> publishResults) { // process publish message result for (int index = 0; index < publishResults.size(); index++) { try { - PositionImpl position = publishResults.get(index).get(); + Position position = publishResults.get(index).get(); MessageId messageId = new MessageIdImpl(position.getLedgerId(), position.getEntryId(), Integer.parseInt(produceMessageResults.get(index).getMessageId())); produceMessageResults.get(index).setMessageId(messageId.toString()); @@ -433,7 +436,10 @@ private CompletableFuture lookUpBrokerForTopic(TopicName partitionedTopicN } LookupResult result = optionalResult.get(); - if (result.getLookupData().getHttpUrl().equals(pulsar().getWebServiceAddress())) { + String httpUrl = result.getLookupData().getHttpUrl(); + String httpUrlTls = result.getLookupData().getHttpUrlTls(); + if ((StringUtils.isNotBlank(httpUrl) && httpUrl.equals(pulsar().getWebServiceAddress())) + || (StringUtils.isNotBlank(httpUrlTls) && httpUrlTls.equals(pulsar().getWebServiceAddressTls()))) { // Current broker owns the topic, add to owning topic. if (log.isDebugEnabled()) { log.debug("Complete topic look up for rest produce message request for topic {}, " @@ -442,7 +448,7 @@ private CompletableFuture lookUpBrokerForTopic(TopicName partitionedTopicN } pulsar().getBrokerService().getOwningTopics().computeIfAbsent(partitionedTopicName .getPartitionedTopicName(), - (key) -> ConcurrentOpenHashSet.newBuilder().build()) + __ -> ConcurrentHashMap.newKeySet()) .add(partitionedTopicName.getPartitionIndex()); completeLookup(Pair.of(Collections.emptyList(), false), redirectAddresses, future); } else { @@ -454,12 +460,10 @@ private CompletableFuture lookUpBrokerForTopic(TopicName partitionedTopicN } if (result.isRedirect()) { // Redirect lookup. - completeLookup(Pair.of(Arrays.asList(result.getLookupData().getHttpUrl(), - result.getLookupData().getHttpUrlTls()), false), redirectAddresses, future); + completeLookup(Pair.of(Arrays.asList(httpUrl, httpUrlTls), false), redirectAddresses, future); } else { // Found owner for topic. - completeLookup(Pair.of(Arrays.asList(result.getLookupData().getHttpUrl(), - result.getLookupData().getHttpUrlTls()), true), redirectAddresses, future); + completeLookup(Pair.of(Arrays.asList(httpUrl, httpUrlTls), true), redirectAddresses, future); } } }).exceptionally(exception -> { @@ -515,7 +519,7 @@ private CompletableFuture addSchema(SchemaData schemaData) { // Only need to add to first partition the broker owns since the schema id in schema registry are // same for all partitions which is the partitionedTopicName List partitions = pulsar().getBrokerService().getOwningTopics() - .get(topicName.getPartitionedTopicName()).values(); + .get(topicName.getPartitionedTopicName()).stream().toList(); CompletableFuture result = new CompletableFuture<>(); for (int index = 0; index < partitions.size(); index++) { CompletableFuture future = new CompletableFuture<>(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java index 8f6caa7a20801..fb5c457fcc874 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java @@ -32,15 +32,16 @@ import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.mledger.impl.AckSetStateUtil; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.intercept.BrokerInterceptor; -import org.apache.pulsar.broker.service.persistent.CompactorSubscription; import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.service.persistent.PulsarCompactorSubscription; import org.apache.pulsar.broker.service.plugin.EntryFilter; import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; import org.apache.pulsar.client.api.transaction.TxnID; @@ -49,6 +50,7 @@ import org.apache.pulsar.common.api.proto.ReplicatedSubscriptionsSnapshot; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.Markers; +import org.apache.pulsar.compaction.Compactor; import org.checkerframework.checker.nullness.qual.Nullable; @Slf4j @@ -124,7 +126,7 @@ public int filterEntriesForConsumer(@Nullable MessageMetadata[] metadataArray, i int filteredEntryCount = 0; long filteredBytesCount = 0; List entriesToFiltered = hasFilter ? new ArrayList<>() : null; - List entriesToRedeliver = hasFilter ? new ArrayList<>() : null; + List entriesToRedeliver = hasFilter ? new ArrayList<>() : null; for (int i = 0, entriesSize = entries.size(); i < entriesSize; i++) { final Entry entry = entries.get(i); if (entry == null) { @@ -160,7 +162,7 @@ public int filterEntriesForConsumer(@Nullable MessageMetadata[] metadataArray, i entry.release(); continue; } else if (filterResult == EntryFilter.FilterResult.RESCHEDULE) { - entriesToRedeliver.add((PositionImpl) entry.getPosition()); + entriesToRedeliver.add(entry.getPosition()); entries.set(i, null); // FilterResult will be always `ACCEPTED` when there is No Filter // dont need to judge whether `hasFilter` is true or not. @@ -171,37 +173,49 @@ public int filterEntriesForConsumer(@Nullable MessageMetadata[] metadataArray, i entry.release(); continue; } - if (!isReplayRead && msgMetadata != null && msgMetadata.hasTxnidMostBits() + if (msgMetadata != null && msgMetadata.hasTxnidMostBits() && msgMetadata.hasTxnidLeastBits()) { if (Markers.isTxnMarker(msgMetadata)) { - // because consumer can receive message is smaller than maxReadPosition, - // so this marker is useless for this subscription - individualAcknowledgeMessageIfNeeded(entry.getPosition(), Collections.emptyMap()); - entries.set(i, null); - entry.release(); - continue; + if (cursor == null || !cursor.getName().equals(Compactor.COMPACTION_SUBSCRIPTION)) { + // because consumer can receive message is smaller than maxReadPosition, + // so this marker is useless for this subscription + individualAcknowledgeMessageIfNeeded(Collections.singletonList(entry.getPosition()), + Collections.emptyMap()); + entries.set(i, null); + entry.release(); + continue; + } } else if (((PersistentTopic) subscription.getTopic()) .isTxnAborted(new TxnID(msgMetadata.getTxnidMostBits(), msgMetadata.getTxnidLeastBits()), - (PositionImpl) entry.getPosition())) { - individualAcknowledgeMessageIfNeeded(entry.getPosition(), Collections.emptyMap()); + entry.getPosition())) { + individualAcknowledgeMessageIfNeeded(Collections.singletonList(entry.getPosition()), + Collections.emptyMap()); entries.set(i, null); entry.release(); continue; } } - if (msgMetadata == null || Markers.isServerOnlyMarker(msgMetadata)) { - PositionImpl pos = (PositionImpl) entry.getPosition(); + if (msgMetadata == null || (Markers.isServerOnlyMarker(msgMetadata))) { + Position pos = entry.getPosition(); // Message metadata was corrupted or the messages was a server-only marker if (Markers.isReplicatedSubscriptionSnapshotMarker(msgMetadata)) { + final int readerIndex = metadataAndPayload.readerIndex(); processReplicatedSubscriptionSnapshot(pos, metadataAndPayload); + metadataAndPayload.readerIndex(readerIndex); } - entries.set(i, null); - entry.release(); - individualAcknowledgeMessageIfNeeded(pos, Collections.emptyMap()); - continue; + // Deliver marker to __compaction cursor to avoid compaction task stuck, + // and filter out them when doing topic compaction. + if (msgMetadata == null || cursor == null + || !cursor.getName().equals(Compactor.COMPACTION_SUBSCRIPTION)) { + entries.set(i, null); + entry.release(); + individualAcknowledgeMessageIfNeeded(Collections.singletonList(pos), + Collections.emptyMap()); + continue; + } } else if (trackDelayedDelivery(entry.getLedgerId(), entry.getEntryId(), msgMetadata)) { // The message is marked for delayed delivery. Ignore for now. entries.set(i, null); @@ -213,32 +227,28 @@ public int filterEntriesForConsumer(@Nullable MessageMetadata[] metadataArray, i this.filterAcceptedMsgs.add(entryMsgCnt); } - totalEntries++; int batchSize = msgMetadata.getNumMessagesInBatch(); - totalMessages += batchSize; - totalBytes += metadataAndPayload.readableBytes(); - totalChunkedMessages += msgMetadata.hasChunkId() ? 1 : 0; - batchSizes.setBatchSize(i, batchSize); long[] ackSet = null; if (indexesAcks != null && cursor != null) { - PositionImpl position = PositionImpl.get(entry.getLedgerId(), entry.getEntryId()); + Position position = PositionFactory.create(entry.getLedgerId(), entry.getEntryId()); ackSet = cursor .getDeletedBatchIndexesAsLongArray(position); // some batch messages ack bit sit will be in pendingAck state, so don't send all bit sit to consumer if (subscription instanceof PersistentSubscription && ((PersistentSubscription) subscription) .getPendingAckHandle() instanceof PendingAckHandleImpl) { - PositionImpl positionInPendingAck = + Position positionInPendingAck = ((PersistentSubscription) subscription).getPositionInPendingAck(position); // if this position not in pendingAck state, don't need to do any op if (positionInPendingAck != null) { - if (positionInPendingAck.hasAckSet()) { + long[] pendingAckSet = AckSetStateUtil.getAckSetArrayOrNull(positionInPendingAck); + if (pendingAckSet != null) { // need to or ackSet in pendingAck state and cursor ackSet which bit sit has been acked if (ackSet != null) { - ackSet = andAckSet(ackSet, positionInPendingAck.getAckSet()); + ackSet = andAckSet(ackSet, pendingAckSet); } else { // if actSet is null, use pendingAck ackSet - ackSet = positionInPendingAck.getAckSet(); + ackSet = pendingAckSet; } // if the result of pendingAckSet(in pendingAckHandle) AND the ackSet(in cursor) is empty // filter this entry @@ -262,6 +272,12 @@ public int filterEntriesForConsumer(@Nullable MessageMetadata[] metadataArray, i } } + totalEntries++; + totalMessages += batchSize; + totalBytes += metadataAndPayload.readableBytes(); + totalChunkedMessages += msgMetadata.hasChunkId() ? 1 : 0; + batchSizes.setBatchSize(i, batchSize); + BrokerInterceptor interceptor = subscription.interceptor(); if (null != interceptor) { // keep for compatibility if users has implemented the old interface @@ -270,8 +286,7 @@ public int filterEntriesForConsumer(@Nullable MessageMetadata[] metadataArray, i } } if (CollectionUtils.isNotEmpty(entriesToFiltered)) { - subscription.acknowledgeMessage(entriesToFiltered, AckType.Individual, - Collections.emptyMap()); + individualAcknowledgeMessageIfNeeded(entriesToFiltered, Collections.emptyMap()); int filtered = entriesToFiltered.size(); Topic topic = subscription.getTopic(); @@ -300,9 +315,9 @@ public int filterEntriesForConsumer(@Nullable MessageMetadata[] metadataArray, i return totalEntries; } - private void individualAcknowledgeMessageIfNeeded(Position position, Map properties) { - if (!(subscription instanceof CompactorSubscription)) { - subscription.acknowledgeMessage(Collections.singletonList(position), AckType.Individual, properties); + private void individualAcknowledgeMessageIfNeeded(List positions, Map properties) { + if (!(subscription instanceof PulsarCompactorSubscription)) { + subscription.acknowledgeMessage(positions, AckType.Individual, properties); } } @@ -312,10 +327,10 @@ protected void acquirePermitsForDeliveredMessages(Topic topic, ManagedCursor cur || (cursor != null && !cursor.isActive())) { long permits = dispatchThrottlingOnBatchMessageEnabled ? totalEntries : totalMessagesSent; topic.getBrokerDispatchRateLimiter().ifPresent(rateLimiter -> - rateLimiter.tryDispatchPermit(permits, totalBytesSent)); + rateLimiter.consumeDispatchQuota(permits, totalBytesSent)); topic.getDispatchRateLimiter().ifPresent(rateLimter -> - rateLimter.tryDispatchPermit(permits, totalBytesSent)); - getRateLimiter().ifPresent(rateLimiter -> rateLimiter.tryDispatchPermit(permits, totalBytesSent)); + rateLimter.consumeDispatchQuota(permits, totalBytesSent)); + getRateLimiter().ifPresent(rateLimiter -> rateLimiter.consumeDispatchQuota(permits, totalBytesSent)); } } @@ -334,7 +349,7 @@ protected boolean isConsumersExceededOnSubscription(AbstractTopic topic, int con && maxConsumersPerSubscription <= consumerSize; } - private void processReplicatedSubscriptionSnapshot(PositionImpl pos, ByteBuf headersAndPayload) { + private void processReplicatedSubscriptionSnapshot(Position pos, ByteBuf headersAndPayload) { // Remove the protobuf headers Commands.skipMessageMetadata(headersAndPayload); @@ -353,16 +368,6 @@ public void resetCloseFuture() { protected abstract void reScheduleRead(); - protected boolean reachDispatchRateLimit(DispatchRateLimiter dispatchRateLimiter) { - if (dispatchRateLimiter.isDispatchRateLimitingEnabled()) { - if (!dispatchRateLimiter.hasMessageDispatchPermit()) { - reScheduleRead(); - return true; - } - } - return false; - } - protected Pair updateMessagesToRead(DispatchRateLimiter dispatchRateLimiter, int messagesToRead, long bytesToRead) { // update messagesToRead according to available dispatch rate limit. @@ -373,11 +378,11 @@ protected Pair updateMessagesToRead(DispatchRateLimiter dispatchR protected static Pair computeReadLimits(int messagesToRead, int availablePermitsOnMsg, long bytesToRead, long availablePermitsOnByte) { - if (availablePermitsOnMsg > 0) { + if (availablePermitsOnMsg >= 0) { messagesToRead = Math.min(messagesToRead, availablePermitsOnMsg); } - if (availablePermitsOnByte > 0) { + if (availablePermitsOnByte >= 0) { bytesToRead = Math.min(bytesToRead, availablePermitsOnByte); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java index 5098890242b6c..9980b6ae97c6a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java @@ -24,16 +24,18 @@ import java.util.Map; import java.util.NavigableMap; import java.util.Objects; +import java.util.Optional; import java.util.TreeMap; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; -import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException; import org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException; +import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter; import org.apache.pulsar.client.impl.Murmur3Hash32; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.util.FutureUtil; @@ -44,9 +46,6 @@ public abstract class AbstractDispatcherSingleActiveConsumer extends AbstractBaseDispatcher { protected final String topicName; - protected static final AtomicReferenceFieldUpdater - ACTIVE_CONSUMER_UPDATER = AtomicReferenceFieldUpdater.newUpdater( - AbstractDispatcherSingleActiveConsumer.class, Consumer.class, "activeConsumer"); private volatile Consumer activeConsumer = null; protected final CopyOnWriteArrayList consumers; protected StickyKeyConsumerSelector stickyKeyConsumerSelector; @@ -75,13 +74,16 @@ public AbstractDispatcherSingleActiveConsumer(SubType subscriptionType, int part this.partitionIndex = partitionIndex; this.subscriptionType = subscriptionType; this.cursor = cursor; - ACTIVE_CONSUMER_UPDATER.set(this, null); } + /** + * @apiNote this method does not need to be thread safe + */ protected abstract void scheduleReadOnActiveConsumer(); - protected abstract void readMoreEntries(Consumer consumer); - + /** + * @apiNote this method does not need to be thread safe + */ protected abstract void cancelPendingRead(); protected void notifyActiveConsumerChanged(Consumer activeConsumer) { @@ -98,6 +100,7 @@ protected void notifyActiveConsumerChanged(Consumer activeConsumer) { * distributed partitions evenly across consumers with highest priority level. * * @return the true consumer if the consumer is changed, otherwise false. + * @apiNote this method is not thread safe */ protected boolean pickAndScheduleActiveConsumer() { checkArgument(!consumers.isEmpty()); @@ -127,14 +130,14 @@ protected boolean pickAndScheduleActiveConsumer() { ? partitionIndex % consumersSize : peekConsumerIndexFromHashRing(makeHashRing(consumersSize)); - Consumer prevConsumer = ACTIVE_CONSUMER_UPDATER.getAndSet(this, consumers.get(index)); + Consumer selectedConsumer = consumers.get(index); - Consumer activeConsumer = ACTIVE_CONSUMER_UPDATER.get(this); - if (prevConsumer == activeConsumer) { + if (selectedConsumer == activeConsumer) { // Active consumer did not change. Do nothing at this point return false; } else { // If the active consumer is changed, send notification. + activeConsumer = selectedConsumer; scheduleReadOnActiveConsumer(); return true; } @@ -166,7 +169,22 @@ public synchronized CompletableFuture addConsumer(Consumer consumer) { } if (subscriptionType == SubType.Exclusive && !consumers.isEmpty()) { - return FutureUtil.failedFuture(new ConsumerBusyException("Exclusive consumer is already connected")); + Consumer actConsumer = getActiveConsumer(); + if (actConsumer != null) { + return actConsumer.cnx().checkConnectionLiveness().thenCompose(actConsumerStillAlive -> { + if (actConsumerStillAlive.isEmpty() || actConsumerStillAlive.get()) { + return FutureUtil.failedFuture(new ConsumerBusyException("Exclusive consumer is already" + + " connected")); + } else { + return addConsumer(consumer); + } + }); + } else { + // It should never happen. + + return FutureUtil.failedFuture(new ConsumerBusyException("Active consumer is in a strange state." + + " Active consumer is null, but there are " + consumers.size() + " registered.")); + } } if (subscriptionType == SubType.Failover && isConsumersExceededOnSubscription()) { @@ -194,7 +212,7 @@ public synchronized CompletableFuture addConsumer(Consumer consumer) { if (!pickAndScheduleActiveConsumer()) { // the active consumer is not changed - Consumer currentActiveConsumer = ACTIVE_CONSUMER_UPDATER.get(this); + Consumer currentActiveConsumer = getActiveConsumer(); if (null == currentActiveConsumer) { if (log.isDebugEnabled()) { log.debug("Current active consumer disappears while adding consumer {}", consumer); @@ -214,7 +232,7 @@ public synchronized void removeConsumer(Consumer consumer) throws BrokerServiceE } if (consumers.isEmpty()) { - ACTIVE_CONSUMER_UPDATER.set(this, null); + activeConsumer = null; } if (closeFuture == null && !consumers.isEmpty()) { @@ -239,12 +257,16 @@ public synchronized void removeConsumer(Consumer consumer) throws BrokerServiceE * Calling consumer object */ public synchronized boolean canUnsubscribe(Consumer consumer) { - return (consumers.size() == 1) && Objects.equals(consumer, ACTIVE_CONSUMER_UPDATER.get(this)); + return (consumers.size() == 1) && Objects.equals(consumer, activeConsumer); } - public CompletableFuture close() { + @Override + public CompletableFuture close(boolean disconnectConsumers, + Optional assignedBrokerLookupData) { IS_CLOSED_UPDATER.set(this, TRUE); - return disconnectAllConsumers(); + getRateLimiter().ifPresent(DispatchRateLimiter::close); + return disconnectConsumers + ? disconnectAllConsumers(false, assignedBrokerLookupData) : CompletableFuture.completedFuture(null); } public boolean isClosed() { @@ -253,15 +275,23 @@ public boolean isClosed() { /** * Disconnect all consumers on this dispatcher (server side close). This triggers channelInactive on the inbound - * handler which calls dispatcher.removeConsumer(), where the closeFuture is completed + * handler which calls dispatcher.removeConsumer(), where the closeFuture is completed. * - * @return + * @param isResetCursor + * Specifies if the cursor has been reset. + * @param assignedBrokerLookupData + * Optional target broker redirect information. Allows the consumer to quickly reconnect to a broker + * during bundle unloading. + * + * @return CompletableFuture indicating the completion of the operation. */ - public synchronized CompletableFuture disconnectAllConsumers(boolean isResetCursor) { + @Override + public synchronized CompletableFuture disconnectAllConsumers( + boolean isResetCursor, Optional assignedBrokerLookupData) { closeFuture = new CompletableFuture<>(); if (!consumers.isEmpty()) { - consumers.forEach(consumer -> consumer.disconnect(isResetCursor)); + consumers.forEach(consumer -> consumer.disconnect(isResetCursor, assignedBrokerLookupData)); cancelPendingRead(); } else { // no consumer connected, complete disconnect immediately @@ -294,7 +324,7 @@ public SubType getType() { } public Consumer getActiveConsumer() { - return ACTIVE_CONSUMER_UPDATER.get(this); + return activeConsumer; } @Override @@ -303,7 +333,7 @@ public List getConsumers() { } public boolean isConsumerConnected() { - return ACTIVE_CONSUMER_UPDATER.get(this) != null; + return activeConsumer != null; } private static final Logger log = LoggerFactory.getLogger(AbstractDispatcherSingleActiveConsumer.class); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java index deab89cda72dc..34fd9f17f6ea6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java @@ -18,26 +18,37 @@ */ package org.apache.pulsar.broker.service; +import com.google.common.annotations.VisibleForTesting; +import io.opentelemetry.api.common.Attributes; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import lombok.Getter; import org.apache.bookkeeper.mledger.Position; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.service.BrokerServiceException.NamingException; import org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.MessageRoutingMode; +import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.impl.Backoff; +import org.apache.pulsar.client.impl.ProducerBuilderImpl; import org.apache.pulsar.client.impl.ProducerImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.util.Backoff; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.StringInterner; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class AbstractReplicator { +public abstract class AbstractReplicator implements Replicator { protected final BrokerService brokerService; protected final String localTopicName; @@ -47,6 +58,8 @@ public abstract class AbstractReplicator { protected final PulsarClientImpl replicationClient; protected final PulsarClientImpl client; protected String replicatorId; + @Getter + protected final Topic localTopic; protected volatile ProducerImpl producer; public static final String REPL_PRODUCER_NAME_DELIMITER = "-->"; @@ -61,21 +74,47 @@ public abstract class AbstractReplicator { protected static final AtomicReferenceFieldUpdater STATE_UPDATER = AtomicReferenceFieldUpdater.newUpdater(AbstractReplicator.class, State.class, "state"); - private volatile State state = State.Stopped; - - protected enum State { - Stopped, Starting, Started, Stopping + @VisibleForTesting + @Getter + protected volatile State state = State.Disconnected; + + private volatile Attributes attributes = null; + private static final AtomicReferenceFieldUpdater ATTRIBUTES_UPDATER = + AtomicReferenceFieldUpdater.newUpdater(AbstractReplicator.class, Attributes.class, "attributes"); + + public enum State { + /** + * This enum has two mean meanings： + * Init: replicator is just created, has not been started now. + * Disconnected: the producer was closed after {@link PersistentTopic#checkGC} called {@link #disconnect}. + */ + // The internal producer is disconnected. + Disconnected, + // Trying to create a new internal producer. + Starting, + // The internal producer has started, and tries copy data. + Started, + /** + * The producer is closing after {@link PersistentTopic#checkGC} called {@link #disconnect}. + */ + // The internal producer is trying to disconnect. + Disconnecting, + // The replicator is in terminating. + Terminating, + // The replicator is never used again. Pulsar will create a new Replicator when enable replication again. + Terminated; } - public AbstractReplicator(String localCluster, String localTopicName, String remoteCluster, String remoteTopicName, + public AbstractReplicator(String localCluster, Topic localTopic, String remoteCluster, String remoteTopicName, String replicatorPrefix, BrokerService brokerService, PulsarClientImpl replicationClient) throws PulsarServerException { this.brokerService = brokerService; - this.localTopicName = localTopicName; + this.localTopic = localTopic; + this.localTopicName = localTopic.getName(); this.replicatorPrefix = replicatorPrefix; - this.localCluster = localCluster.intern(); + this.localCluster = StringInterner.intern(localCluster); this.remoteTopicName = remoteTopicName; - this.remoteCluster = remoteCluster.intern(); + this.remoteCluster = StringInterner.intern(remoteCluster); this.replicationClient = replicationClient; this.client = (PulsarClientImpl) brokerService.pulsar().getClient(); this.producer = null; @@ -92,102 +131,182 @@ public AbstractReplicator(String localCluster, String localTopicName, String rem .sendTimeout(0, TimeUnit.SECONDS) // .maxPendingMessages(producerQueueSize) // .producerName(getProducerName()); - STATE_UPDATER.set(this, State.Stopped); + STATE_UPDATER.set(this, State.Disconnected); } protected abstract String getProducerName(); - protected abstract void readEntries(org.apache.pulsar.client.api.Producer producer); + protected abstract void setProducerAndTriggerReadEntries(org.apache.pulsar.client.api.Producer producer); protected abstract Position getReplicatorReadPosition(); - protected abstract long getNumberOfEntriesInBacklog(); + public abstract long getNumberOfEntriesInBacklog(); protected abstract void disableReplicatorRead(); + @Override + public boolean isConnected() { + var producer = this.producer; + return producer != null && producer.isConnected(); + } + + public long getReplicationDelayMs() { + var producer = this.producer; + return producer == null ? 0 : producer.getDelayInMillis(); + } + public String getRemoteCluster() { return remoteCluster; } - // This method needs to be synchronized with disconnects else if there is a disconnect followed by startProducer - // the end result can be disconnect. - public synchronized void startProducer() { - if (STATE_UPDATER.get(this) == State.Stopping) { - long waitTimeMs = backOff.next(); - if (log.isDebugEnabled()) { - log.debug( - "[{}] waiting for producer to close before attempting to reconnect, retrying in {} s", - replicatorId, waitTimeMs / 1000.0); - } - // BackOff before retrying - brokerService.executor().schedule(this::startProducer, waitTimeMs, TimeUnit.MILLISECONDS); - return; - } - State state = STATE_UPDATER.get(this); - if (!STATE_UPDATER.compareAndSet(this, State.Stopped, State.Starting)) { - if (state == State.Started) { - // Already running + protected CompletableFuture prepareCreateProducer() { + return CompletableFuture.completedFuture(null); + } + + public void startProducer() { + // Guarantee only one task call "producerBuilder.createAsync()". + Pair setStartingRes = compareSetAndGetState(State.Disconnected, State.Starting); + if (!setStartingRes.getLeft()) { + if (setStartingRes.getRight() == State.Starting) { + log.info("[{}] Skip the producer creation since other thread is doing starting, state : {}", + replicatorId, state); + } else if (setStartingRes.getRight() == State.Started) { + // Since the method "startProducer" will be called even if it is started, only print debug-level log. + if (log.isDebugEnabled()) { + log.debug("[{}] Replicator was already running. state: {}", replicatorId, state); + } + } else if (setStartingRes.getRight() == State.Disconnecting) { if (log.isDebugEnabled()) { - log.debug("[{}] Replicator was already running", replicatorId); + log.debug("[{}] Rep.producer is closing, delay to retry(wait the producer close success)." + + " state: {}", replicatorId, state); } + delayStartProducerAfterDisconnected(); } else { - log.info("[{}] Replicator already being started. Replicator state: {}", replicatorId, state); + /** {@link State.Terminating}, {@link State.Terminated}. **/ + log.info("[{}] Skip the producer creation since the replicator state is : {}", replicatorId, state); } - return; } log.info("[{}] Starting replicator", replicatorId); - producerBuilder.createAsync().thenAccept(producer -> { - readEntries(producer); + + // Force only replicate messages to a non-partitioned topic, to avoid auto-create a partitioned topic on + // the remote cluster. + prepareCreateProducer().thenCompose(ignore -> { + ProducerBuilderImpl builderImpl = (ProducerBuilderImpl) producerBuilder; + builderImpl.getConf().setNonPartitionedTopicExpected(true); + return producerBuilder.createAsync().thenAccept(producer -> { + setProducerAndTriggerReadEntries(producer); + }); }).exceptionally(ex -> { - if (STATE_UPDATER.compareAndSet(this, State.Starting, State.Stopped)) { + Pair setDisconnectedRes = compareSetAndGetState(State.Starting, State.Disconnected); + if (setDisconnectedRes.getLeft()) { long waitTimeMs = backOff.next(); log.warn("[{}] Failed to create remote producer ({}), retrying in {} s", replicatorId, ex.getMessage(), waitTimeMs / 1000.0); - // BackOff before retrying - brokerService.executor().schedule(this::startProducer, waitTimeMs, TimeUnit.MILLISECONDS); + scheduleCheckTopicActiveAndStartProducer(waitTimeMs); } else { - log.warn("[{}] Failed to create remote producer. Replicator state: {}", replicatorId, - STATE_UPDATER.get(this), ex); + if (setDisconnectedRes.getRight() == State.Terminating + || setDisconnectedRes.getRight() == State.Terminated) { + log.info("[{}] Skip to create producer, because it has been terminated, state is : {}", + replicatorId, state); + } else { + /** {@link State.Disconnected}, {@link State.Starting}, {@link State.Started} **/ + // Since only one task can call "producerBuilder.createAsync()", this scenario is not expected. + // So print a warn log. + log.warn("[{}] Other thread will try to create the producer again. so skipped current one task." + + " State is : {}", + replicatorId, state); + } } return null; }); } - protected synchronized CompletableFuture closeProducerAsync() { - if (producer == null) { - STATE_UPDATER.set(this, State.Stopped); - return CompletableFuture.completedFuture(null); + /*** + * The producer is disconnecting, delay to start the producer. + * If we start a producer immediately, we will get a conflict producer(same name producer) registered error. + */ + protected void delayStartProducerAfterDisconnected() { + long waitTimeMs = backOff.next(); + if (log.isDebugEnabled()) { + log.debug( + "[{}] waiting for producer to close before attempting to reconnect, retrying in {} s", + replicatorId, waitTimeMs / 1000.0); } - CompletableFuture future = producer.closeAsync(); - future.thenRun(() -> { - STATE_UPDATER.set(this, State.Stopped); - this.producer = null; - // deactivate further read - disableReplicatorRead(); - }).exceptionally(ex -> { - long waitTimeMs = backOff.next(); - log.warn( - "[{}] Exception: '{}' occurred while trying to close the producer." - + " retrying again in {} s", - replicatorId, ex.getMessage(), waitTimeMs / 1000.0); - // BackOff before retrying - brokerService.executor().schedule(this::closeProducerAsync, waitTimeMs, TimeUnit.MILLISECONDS); - return null; - }); - return future; + scheduleCheckTopicActiveAndStartProducer(waitTimeMs); } + protected void scheduleCheckTopicActiveAndStartProducer(final long waitTimeMs) { + brokerService.executor().schedule(() -> { + if (state == State.Terminating || state == State.Terminated) { + log.info("[{}] Skip scheduled to start the producer since the replicator state is : {}", + replicatorId, state); + return; + } + CompletableFuture> topicFuture = brokerService.getTopics().get(localTopicName); + if (topicFuture == null) { + // Topic closed. + log.info("[{}] Skip scheduled to start the producer since the topic was closed successfully." + + " And trigger a terminate.", replicatorId); + terminate(); + return; + } + topicFuture.thenAccept(optional -> { + if (optional.isEmpty()) { + // Topic closed. + log.info("[{}] Skip scheduled to start the producer since the topic was closed. And trigger a" + + " terminate.", replicatorId); + terminate(); + return; + } + if (optional.get() != localTopic) { + // Topic closed and created a new one, current replicator is outdated. + log.info("[{}] Skip scheduled to start the producer since the topic was closed. And trigger a" + + " terminate.", replicatorId); + terminate(); + return; + } + Replicator replicator = localTopic.getReplicators().get(remoteCluster); + if (replicator != AbstractReplicator.this) { + // Current replicator has been closed, and created a new one. + log.info("[{}] Skip scheduled to start the producer since a new replicator has instead current" + + " one. And trigger a terminate.", replicatorId); + terminate(); + return; + } + startProducer(); + }).exceptionally(ex -> { + log.error("[{}] [{}] Stop retry to create producer due to unknown error(topic create failed), and" + + " trigger a terminate. Replicator state: {}", + localTopicName, replicatorId, STATE_UPDATER.get(this), ex); + terminate(); + return null; + }); + }, waitTimeMs, TimeUnit.MILLISECONDS); + } - public CompletableFuture disconnect() { - return disconnect(false); + protected CompletableFuture isLocalTopicActive() { + CompletableFuture> topicFuture = brokerService.getTopics().get(localTopicName); + if (topicFuture == null){ + return CompletableFuture.completedFuture(false); + } + return topicFuture.thenApplyAsync(optional -> { + if (optional.isEmpty()) { + return false; + } + return optional.get() == localTopic; + }, brokerService.executor()); } - public synchronized CompletableFuture disconnect(boolean failIfHasBacklog) { - if (failIfHasBacklog && getNumberOfEntriesInBacklog() > 0) { + /** + * This method only be used by {@link PersistentTopic#checkGC} now. + */ + public CompletableFuture disconnect(boolean failIfHasBacklog, boolean closeTheStartingProducer) { + long backlog = getNumberOfEntriesInBacklog(); + if (failIfHasBacklog && backlog > 0) { CompletableFuture disconnectFuture = new CompletableFuture<>(); disconnectFuture.completeExceptionally(new TopicBusyException("Cannot close a replicator with backlog")); if (log.isDebugEnabled()) { @@ -195,21 +314,121 @@ public synchronized CompletableFuture disconnect(boolean failIfHasBacklog) } return disconnectFuture; } + log.info("[{}] Disconnect replicator at position {} with backlog {}", replicatorId, + getReplicatorReadPosition(), backlog); + return closeProducerAsync(closeTheStartingProducer); + } - if (STATE_UPDATER.get(this) == State.Stopping) { - // Do nothing since the all "STATE_UPDATER.set(this, Stopping)" instructions are followed by - // closeProducerAsync() - // which will at some point change the state to stopped + /** + * This method only be used by {@link PersistentTopic#checkGC} now. + */ + protected CompletableFuture closeProducerAsync(boolean closeTheStartingProducer) { + Pair setDisconnectingRes = compareSetAndGetState(State.Started, State.Disconnecting); + if (!setDisconnectingRes.getLeft()) { + if (setDisconnectingRes.getRight() == State.Starting) { + if (closeTheStartingProducer) { + /** + * Delay retry(wait for the start producer task is finish). + * Note: If the producer always start fail, the start producer task will always retry until the + * state changed to {@link State.Terminated}. + * Nit: The better solution is creating a {@link CompletableFuture} to trace the in-progress + * creation and call "inProgressCreationFuture.thenApply(closeProducer())". + */ + long waitTimeMs = backOff.next(); + brokerService.executor().schedule(() -> closeProducerAsync(true), + waitTimeMs, TimeUnit.MILLISECONDS); + } else { + log.info("[{}] Skip current producer closing since the previous producer has been closed," + + " and trying start a new one, state : {}", + replicatorId, setDisconnectingRes.getRight()); + } + } else if (setDisconnectingRes.getRight() == State.Disconnected + || setDisconnectingRes.getRight() == State.Disconnecting) { + log.info("[{}] Skip current producer closing since other thread did closing, state : {}", + replicatorId, setDisconnectingRes.getRight()); + } else if (setDisconnectingRes.getRight() == State.Terminating + || setDisconnectingRes.getRight() == State.Terminated) { + log.info("[{}] Skip current producer closing since other thread is doing termination, state : {}", + replicatorId, state); + } + log.info("[{}] Skip current termination since other thread is doing close producer or termination," + + " state : {}", replicatorId, state); return CompletableFuture.completedFuture(null); } - if (STATE_UPDATER.compareAndSet(this, State.Starting, State.Stopping) - || STATE_UPDATER.compareAndSet(this, State.Started, State.Stopping)) { - log.info("[{}] Disconnect replicator at position {} with backlog {}", replicatorId, - getReplicatorReadPosition(), getNumberOfEntriesInBacklog()); + // Close producer and update state. + return doCloseProducerAsync(producer, () -> { + Pair setDisconnectedRes = compareSetAndGetState(State.Disconnecting, State.Disconnected); + if (setDisconnectedRes.getLeft()) { + this.producer = null; + // deactivate further read + disableReplicatorRead(); + return; + } + if (setDisconnectedRes.getRight() == State.Terminating + || setDisconnectingRes.getRight() == State.Terminated) { + log.info("[{}] Skip setting state to terminated because it was terminated, state : {}", + replicatorId, state); + } else { + // Since only one task can call "doCloseProducerAsync(producer, action)", this scenario is not expected. + // So print a warn log. + log.warn("[{}] Other task has change the state to terminated. so skipped current one task." + + " State is : {}", + replicatorId, state); + } + }); + } + + protected CompletableFuture doCloseProducerAsync(Producer producer, Runnable actionAfterClosed) { + CompletableFuture future = + producer == null ? CompletableFuture.completedFuture(null) : producer.closeAsync(); + return future.thenRun(() -> { + actionAfterClosed.run(); + }).exceptionally(ex -> { + long waitTimeMs = backOff.next(); + log.warn( + "[{}] Exception: '{}' occurred while trying to close the producer. Replicator state: {}." + + " Retrying again in {} s.", + replicatorId, ex.getMessage(), state, waitTimeMs / 1000.0); + // BackOff before retrying + brokerService.executor().schedule(() -> doCloseProducerAsync(producer, actionAfterClosed), + waitTimeMs, TimeUnit.MILLISECONDS); + return null; + }); + } + + public CompletableFuture terminate() { + if (!tryChangeStatusToTerminating()) { + log.info("[{}] Skip current termination since other thread is doing termination, state : {}", replicatorId, + state); + return CompletableFuture.completedFuture(null); } + return doCloseProducerAsync(producer, () -> { + STATE_UPDATER.set(this, State.Terminated); + this.producer = null; + // set the cursor as inactive. + disableReplicatorRead(); + // release resources. + doReleaseResources(); + }); + } - return closeProducerAsync(); + protected void doReleaseResources() {} + + protected boolean tryChangeStatusToTerminating() { + if (STATE_UPDATER.compareAndSet(this, State.Starting, State.Terminating)){ + return true; + } + if (STATE_UPDATER.compareAndSet(this, State.Started, State.Terminating)){ + return true; + } + if (STATE_UPDATER.compareAndSet(this, State.Disconnecting, State.Terminating)){ + return true; + } + if (STATE_UPDATER.compareAndSet(this, State.Disconnected, State.Terminating)) { + return true; + } + return false; } public CompletableFuture remove() { @@ -228,7 +447,7 @@ public static String getRemoteCluster(String remoteCursor) { } public static String getReplicatorName(String replicatorPrefix, String cluster) { - return (replicatorPrefix + "." + cluster).intern(); + return StringInterner.intern(replicatorPrefix + "." + cluster); } /** @@ -270,4 +489,44 @@ public static CompletableFuture validatePartitionedTopicAsync(String topic public State getState() { return state; } + + protected ImmutablePair compareSetAndGetState(State expect, State update) { + State original1 = state; + if (STATE_UPDATER.compareAndSet(this, expect, update)) { + return ImmutablePair.of(true, expect); + } + State original2 = state; + // Maybe the value changed more than once even if "original1 == original2", but the probability is very small, + // so let's ignore this case for prevent using a lock. + if (original1 == original2) { + return ImmutablePair.of(false, original1); + } + return compareSetAndGetState(expect, update); + } + + public boolean isTerminated() { + return state == State.Terminating || state == State.Terminated; + } + + public Attributes getAttributes() { + if (attributes != null) { + return attributes; + } + return ATTRIBUTES_UPDATER.updateAndGet(this, old -> { + if (old != null) { + return old; + } + var topicName = TopicName.get(getLocalTopic().getName()); + var builder = Attributes.builder() + .put(OpenTelemetryAttributes.PULSAR_DOMAIN, topicName.getDomain().toString()) + .put(OpenTelemetryAttributes.PULSAR_TENANT, topicName.getTenant()) + .put(OpenTelemetryAttributes.PULSAR_NAMESPACE, topicName.getNamespace()) + .put(OpenTelemetryAttributes.PULSAR_TOPIC, topicName.getPartitionedTopicName()); + if (topicName.isPartitioned()) { + builder.put(OpenTelemetryAttributes.PULSAR_PARTITION_INDEX, topicName.getPartitionIndex()); + } + builder.put(OpenTelemetryAttributes.PULSAR_REPLICATION_REMOTE_CLUSTER_NAME, getRemoteCluster()); + return builder.build(); + }); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 4614b846c8eee..76dd277159cf4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -19,10 +19,14 @@ package org.apache.pulsar.broker.service; import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; import static org.apache.bookkeeper.mledger.impl.ManagedLedgerMBeanImpl.ENTRY_LATENCY_BUCKETS_USEC; +import static org.apache.pulsar.compaction.Compactor.COMPACTION_SUBSCRIPTION; import com.google.common.base.MoreObjects; +import java.time.Clock; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.List; @@ -31,14 +35,18 @@ import java.util.Optional; import java.util.Queue; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.ToLongFunction; +import javax.annotation.Nonnull; import lombok.Getter; import org.apache.bookkeeper.mledger.util.StatsBuckets; import org.apache.commons.collections4.CollectionUtils; @@ -55,14 +63,15 @@ import org.apache.pulsar.broker.service.BrokerServiceException.TopicMigratedException; import org.apache.pulsar.broker.service.BrokerServiceException.TopicTerminatedException; import org.apache.pulsar.broker.service.plugin.EntryFilter; -import org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage; import org.apache.pulsar.broker.service.schema.SchemaRegistryService; import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; +import org.apache.pulsar.broker.service.schema.exceptions.SchemaException; import org.apache.pulsar.broker.stats.prometheus.metrics.Summary; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.BacklogQuota; -import org.apache.pulsar.common.policies.data.ClusterData.ClusterUrl; +import org.apache.pulsar.common.policies.data.ClusterPolicies.ClusterUrl; import org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies; import org.apache.pulsar.common.policies.data.EntryFilters; import org.apache.pulsar.common.policies.data.HierarchyTopicPolicies; @@ -81,7 +90,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class AbstractTopic implements Topic, TopicPolicyListener { +public abstract class AbstractTopic implements Topic, TopicPolicyListener { protected static final long POLICY_UPDATE_FAILURE_RETRY_TIME_SECONDS = 60; @@ -115,9 +124,7 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener RATE_LIMITED_UPDATER = AtomicLongFieldUpdater.newUpdater(AbstractTopic.class, "publishRateLimitedTimes"); - protected volatile long publishRateLimitedTimes = 0; + protected volatile long publishRateLimitedTimes = 0L; + private static final AtomicLongFieldUpdater TOTAL_RATE_LIMITED_UPDATER = + AtomicLongFieldUpdater.newUpdater(AbstractTopic.class, "totalPublishRateLimitedCounter"); + protected volatile long totalPublishRateLimitedCounter = 0L; + + private static final AtomicIntegerFieldUpdater USER_CREATED_PRODUCER_COUNTER_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(AbstractTopic.class, "userCreatedProducerCount"); + protected volatile int userCreatedProducerCount = 0; protected volatile Optional topicEpoch = Optional.empty(); private volatile boolean hasExclusiveProducer; @@ -149,10 +164,17 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener> entryFilters; + protected volatile boolean transferring = false; + private volatile List activeRateLimiters; + protected final Clock clock; + + protected Set additionalSystemCursorNames = new TreeSet<>(); public AbstractTopic(String topic, BrokerService brokerService) { this.topic = topic; + this.clock = brokerService.getClock(); this.brokerService = brokerService; this.producers = new ConcurrentHashMap<>(); this.isFenced = false; @@ -164,6 +186,10 @@ public AbstractTopic(String topic, BrokerService brokerService) { this.lastActive = System.nanoTime(); this.preciseTopicPublishRateLimitingEnable = config.isPreciseTopicPublishRateLimiterEnable(); + topicPublishRateLimiter = new PublishRateLimiterImpl(brokerService.getPulsar().getMonotonicSnapshotClock()); + updateActiveRateLimiters(); + + additionalSystemCursorNames = brokerService.pulsar().getConfiguration().getAdditionalSystemCursorNames(); } public SubscribeRate getSubscribeRate() { @@ -209,13 +235,18 @@ protected void updateTopicPolicy(TopicPolicies data) { .updateTopicValue(formatSchemaCompatibilityStrategy(data.getSchemaCompatibilityStrategy())); } topicPolicies.getRetentionPolicies().updateTopicValue(data.getRetentionPolicies()); - topicPolicies.getMaxSubscriptionsPerTopic().updateTopicValue(data.getMaxSubscriptionsPerTopic()); - topicPolicies.getMaxUnackedMessagesOnConsumer().updateTopicValue(data.getMaxUnackedMessagesOnConsumer()); + topicPolicies.getDispatcherPauseOnAckStatePersistentEnabled() + .updateTopicValue(data.getDispatcherPauseOnAckStatePersistentEnabled()); + topicPolicies.getMaxSubscriptionsPerTopic() + .updateTopicValue(normalizeValue(data.getMaxSubscriptionsPerTopic())); + topicPolicies.getMaxUnackedMessagesOnConsumer() + .updateTopicValue(normalizeValue(data.getMaxUnackedMessagesOnConsumer())); topicPolicies.getMaxUnackedMessagesOnSubscription() - .updateTopicValue(data.getMaxUnackedMessagesOnSubscription()); - topicPolicies.getMaxProducersPerTopic().updateTopicValue(data.getMaxProducerPerTopic()); - topicPolicies.getMaxConsumerPerTopic().updateTopicValue(data.getMaxConsumerPerTopic()); - topicPolicies.getMaxConsumersPerSubscription().updateTopicValue(data.getMaxConsumersPerSubscription()); + .updateTopicValue(normalizeValue(data.getMaxUnackedMessagesOnSubscription())); + topicPolicies.getMaxProducersPerTopic().updateTopicValue(normalizeValue(data.getMaxProducerPerTopic())); + topicPolicies.getMaxConsumerPerTopic().updateTopicValue(normalizeValue(data.getMaxConsumerPerTopic())); + topicPolicies.getMaxConsumersPerSubscription() + .updateTopicValue(normalizeValue(data.getMaxConsumersPerSubscription())); topicPolicies.getInactiveTopicPolicies().updateTopicValue(data.getInactiveTopicPolicies()); topicPolicies.getDeduplicationEnabled().updateTopicValue(data.getDeduplicationEnabled()); topicPolicies.getDeduplicationSnapshotIntervalSeconds().updateTopicValue( @@ -226,13 +257,14 @@ protected void updateTopicPolicy(TopicPolicies data) { Arrays.stream(BacklogQuota.BacklogQuotaType.values()).forEach(type -> this.topicPolicies.getBackLogQuotaMap().get(type).updateTopicValue( data.getBackLogQuotaMap() == null ? null : data.getBackLogQuotaMap().get(type.toString()))); - topicPolicies.getTopicMaxMessageSize().updateTopicValue(data.getMaxMessageSize()); - topicPolicies.getMessageTTLInSeconds().updateTopicValue(data.getMessageTTLInSeconds()); + topicPolicies.getTopicMaxMessageSize().updateTopicValue(normalizeValue(data.getMaxMessageSize())); + topicPolicies.getMessageTTLInSeconds().updateTopicValue(normalizeValue(data.getMessageTTLInSeconds())); topicPolicies.getPublishRate().updateTopicValue(PublishRate.normalize(data.getPublishRate())); topicPolicies.getDelayedDeliveryEnabled().updateTopicValue(data.getDelayedDeliveryEnabled()); topicPolicies.getReplicatorDispatchRate().updateTopicValue( DispatchRateImpl.normalize(data.getReplicatorDispatchRate())); topicPolicies.getDelayedDeliveryTickTimeMillis().updateTopicValue(data.getDelayedDeliveryTickTimeMillis()); + topicPolicies.getDelayedDeliveryMaxDelayInMillis().updateTopicValue(data.getDelayedDeliveryMaxDelayInMillis()); topicPolicies.getSubscribeRate().updateTopicValue(SubscribeRate.normalize(data.getSubscribeRate())); topicPolicies.getSubscriptionDispatchRate().updateTopicValue( DispatchRateImpl.normalize(data.getSubscriptionDispatchRate())); @@ -240,6 +272,8 @@ protected void updateTopicPolicy(TopicPolicies data) { topicPolicies.getDispatchRate().updateTopicValue(DispatchRateImpl.normalize(data.getDispatchRate())); topicPolicies.getSchemaValidationEnforced().updateTopicValue(data.getSchemaValidationEnforced()); topicPolicies.getEntryFilters().updateTopicValue(data.getEntryFilters()); + topicPolicies.getDispatcherPauseOnAckStatePersistentEnabled() + .updateTopicValue(data.getDispatcherPauseOnAckStatePersistentEnabled()); this.subscriptionPolicies = data.getSubscriptionPolicies(); updateEntryFilters(); @@ -254,15 +288,19 @@ protected void updateTopicPolicyByNamespacePolicy(Policies namespacePolicies) { topicPolicies.getReplicationClusters().updateNamespaceValue( new ArrayList<>(CollectionUtils.emptyIfNull(namespacePolicies.replication_clusters))); topicPolicies.getMaxUnackedMessagesOnConsumer() - .updateNamespaceValue(namespacePolicies.max_unacked_messages_per_consumer); + .updateNamespaceValue(normalizeValue(namespacePolicies.max_unacked_messages_per_consumer)); topicPolicies.getMaxUnackedMessagesOnSubscription() - .updateNamespaceValue(namespacePolicies.max_unacked_messages_per_subscription); - topicPolicies.getMessageTTLInSeconds().updateNamespaceValue(namespacePolicies.message_ttl_in_seconds); - topicPolicies.getMaxSubscriptionsPerTopic().updateNamespaceValue(namespacePolicies.max_subscriptions_per_topic); - topicPolicies.getMaxProducersPerTopic().updateNamespaceValue(namespacePolicies.max_producers_per_topic); - topicPolicies.getMaxConsumerPerTopic().updateNamespaceValue(namespacePolicies.max_consumers_per_topic); + .updateNamespaceValue(normalizeValue(namespacePolicies.max_unacked_messages_per_subscription)); + topicPolicies.getMessageTTLInSeconds() + .updateNamespaceValue(normalizeValue(namespacePolicies.message_ttl_in_seconds)); + topicPolicies.getMaxSubscriptionsPerTopic() + .updateNamespaceValue(normalizeValue(namespacePolicies.max_subscriptions_per_topic)); + topicPolicies.getMaxProducersPerTopic() + .updateNamespaceValue(normalizeValue(namespacePolicies.max_producers_per_topic)); + topicPolicies.getMaxConsumerPerTopic() + .updateNamespaceValue(normalizeValue(namespacePolicies.max_consumers_per_topic)); topicPolicies.getMaxConsumersPerSubscription() - .updateNamespaceValue(namespacePolicies.max_consumers_per_subscription); + .updateNamespaceValue(normalizeValue(namespacePolicies.max_consumers_per_subscription)); topicPolicies.getInactiveTopicPolicies().updateNamespaceValue(namespacePolicies.inactive_topic_policies); topicPolicies.getDeduplicationEnabled().updateNamespaceValue(namespacePolicies.deduplicationEnabled); topicPolicies.getDeduplicationSnapshotIntervalSeconds().updateNamespaceValue( @@ -274,6 +312,9 @@ protected void updateTopicPolicyByNamespacePolicy(Policies namespacePolicies) { topicPolicies.getDelayedDeliveryTickTimeMillis().updateNamespaceValue( Optional.ofNullable(namespacePolicies.delayed_delivery_policies) .map(DelayedDeliveryPolicies::getTickTime).orElse(null)); + topicPolicies.getDelayedDeliveryMaxDelayInMillis().updateNamespaceValue( + Optional.ofNullable(namespacePolicies.delayed_delivery_policies) + .map(DelayedDeliveryPolicies::getMaxDeliveryDelayInMillis).orElse(null)); topicPolicies.getSubscriptionTypesEnabled().updateNamespaceValue( subTypeStringsToEnumSet(namespacePolicies.subscription_types_enabled)); updateNamespaceReplicatorDispatchRate(namespacePolicies, @@ -289,9 +330,16 @@ protected void updateTopicPolicyByNamespacePolicy(Policies namespacePolicies) { topicPolicies.getSchemaValidationEnforced().updateNamespaceValue(namespacePolicies.schema_validation_enforced); topicPolicies.getEntryFilters().updateNamespaceValue(namespacePolicies.entryFilters); + topicPolicies.getDispatcherPauseOnAckStatePersistentEnabled().updateNamespaceValue( + namespacePolicies.dispatcherPauseOnAckStatePersistentEnabled); + updateEntryFilters(); } + private Integer normalizeValue(Integer policyValue) { + return policyValue != null && policyValue < 0 ? null : policyValue; + } + private void updateNamespaceDispatchRate(Policies namespacePolicies, String cluster) { DispatchRateImpl dispatchRate = namespacePolicies.topicDispatchRate.get(cluster); if (dispatchRate == null) { @@ -350,12 +398,11 @@ private void updateTopicPolicyByBrokerConfig() { topicPolicies.getMaxConsumerPerTopic().updateBrokerValue(config.getMaxConsumersPerTopic()); topicPolicies.getMaxConsumersPerSubscription().updateBrokerValue(config.getMaxConsumersPerSubscription()); topicPolicies.getDeduplicationEnabled().updateBrokerValue(config.isBrokerDeduplicationEnabled()); - topicPolicies.getRetentionPolicies().updateBrokerValue(new RetentionPolicies( - config.getDefaultRetentionTimeInMinutes(), config.getDefaultRetentionSizeInMB())); - topicPolicies.getDeduplicationSnapshotIntervalSeconds().updateBrokerValue( - config.getBrokerDeduplicationSnapshotIntervalSeconds()); - topicPolicies.getMaxUnackedMessagesOnConsumer() - .updateBrokerValue(config.getMaxUnackedMessagesPerConsumer()); + topicPolicies.getRetentionPolicies().updateBrokerValue( + new RetentionPolicies(config.getDefaultRetentionTimeInMinutes(), config.getDefaultRetentionSizeInMB())); + topicPolicies.getDeduplicationSnapshotIntervalSeconds() + .updateBrokerValue(config.getBrokerDeduplicationSnapshotIntervalSeconds()); + topicPolicies.getMaxUnackedMessagesOnConsumer().updateBrokerValue(config.getMaxUnackedMessagesPerConsumer()); topicPolicies.getMaxUnackedMessagesOnSubscription() .updateBrokerValue(config.getMaxUnackedMessagesPerSubscription()); //init backlogQuota @@ -371,6 +418,8 @@ private void updateTopicPolicyByBrokerConfig() { topicPolicies.getPublishRate().updateBrokerValue(publishRateInBroker(config)); topicPolicies.getDelayedDeliveryEnabled().updateBrokerValue(config.isDelayedDeliveryEnabled()); topicPolicies.getDelayedDeliveryTickTimeMillis().updateBrokerValue(config.getDelayedDeliveryTickTimeMillis()); + topicPolicies.getDelayedDeliveryMaxDelayInMillis() + .updateBrokerValue(config.getDelayedDeliveryMaxDelayInMillis()); topicPolicies.getCompactionThreshold().updateBrokerValue(config.getBrokerServiceCompactionThresholdInBytes()); topicPolicies.getReplicationClusters().updateBrokerValue(Collections.emptyList()); SchemaCompatibilityStrategy schemaCompatibilityStrategy = config.getSchemaCompatibilityStrategy(); @@ -386,7 +435,8 @@ private void updateTopicPolicyByBrokerConfig() { topicPolicies.getSchemaValidationEnforced().updateBrokerValue(config.isSchemaValidationEnforced()); topicPolicies.getEntryFilters().updateBrokerValue(new EntryFilters(String.join(",", config.getEntryFilterNames()))); - + topicPolicies.getDispatcherPauseOnAckStatePersistentEnabled() + .updateBrokerValue(config.isDispatcherPauseOnAckStatePersistentEnabled()); updateEntryFilters(); } @@ -442,35 +492,33 @@ private PublishRate publishRateInBroker(ServiceConfiguration config) { return new PublishRate(config.getMaxPublishRatePerTopicInMessages(), config.getMaxPublishRatePerTopicInBytes()); } + public boolean isProducersExceeded(String producerName) { + String replicatorPrefix = brokerService.getPulsar().getConfig().getReplicatorPrefix() + "."; + boolean isRemote = producerName.startsWith(replicatorPrefix); + return isProducersExceeded(isRemote); + } + protected boolean isProducersExceeded(Producer producer) { - if (isSystemTopic() || producer.isRemote()) { + return isProducersExceeded(producer.isRemote()); + } + + protected boolean isProducersExceeded(boolean isRemote) { + if (isSystemTopic() || isRemote) { return false; } Integer maxProducers = topicPolicies.getMaxProducersPerTopic().get(); - if (maxProducers != null && maxProducers > 0 && maxProducers <= getUserCreatedProducersSize()) { - return true; - } - return false; - } - - private long getUserCreatedProducersSize() { - return producers.values().stream().filter(p -> !p.isRemote()).count(); + return maxProducers != null && maxProducers > 0 + && maxProducers <= USER_CREATED_PRODUCER_COUNTER_UPDATER.get(this); } protected void registerTopicPolicyListener() { - if (brokerService.pulsar().getConfig().isSystemTopicEnabled() - && brokerService.pulsar().getConfig().isTopicLevelPoliciesEnabled()) { - brokerService.getPulsar().getTopicPoliciesService() - .registerListener(TopicName.getPartitionedTopicName(topic), this); - } + brokerService.getPulsar().getTopicPoliciesService() + .registerListener(TopicName.getPartitionedTopicName(topic), this); } protected void unregisterTopicPolicyListener() { - if (brokerService.pulsar().getConfig().isSystemTopicEnabled() - && brokerService.pulsar().getConfig().isTopicLevelPoliciesEnabled()) { - brokerService.getPulsar().getTopicPoliciesService() - .unregisterListener(TopicName.getPartitionedTopicName(topic), this); - } + brokerService.getPulsar().getTopicPoliciesService() + .unregisterListener(TopicName.getPartitionedTopicName(topic), this); } protected boolean isSameAddressProducersExceeded(Producer producer) { @@ -500,7 +548,7 @@ public int getNumberOfSameAddressProducers(final String clientAddress) { return count; } - protected boolean isConsumersExceededOnTopic() { + public boolean isConsumersExceededOnTopic() { if (isSystemTopic()) { return false; } @@ -531,7 +579,7 @@ && getNumberOfSameAddressConsumers(consumer.getClientAddress()) >= maxSameAddres public abstract int getNumberOfSameAddressConsumers(String clientAddress); protected int getNumberOfSameAddressConsumers(final String clientAddress, - final List subscriptions) { + final Collection subscriptions) { int count = 0; if (clientAddress != null) { for (Subscription subscription : subscriptions) { @@ -563,16 +611,6 @@ protected Consumer getActiveConsumer(Subscription subscription) { return null; } - @Override - public void disableCnxAutoRead() { - producers.values().forEach(producer -> producer.getCnx().disableCnxAutoRead()); - } - - @Override - public void enableCnxAutoRead() { - producers.values().forEach(producer -> producer.getCnx().enableCnxAutoRead()); - } - protected boolean hasLocalProducers() { if (producers.isEmpty()) { return false; @@ -630,9 +668,14 @@ protected String getSchemaId() { } @Override public CompletableFuture hasSchema() { - return brokerService.pulsar() - .getSchemaRegistryService() - .getSchema(getSchemaId()).thenApply(Objects::nonNull); + return brokerService.pulsar().getSchemaRegistryService().getSchema(getSchemaId()).thenApply(Objects::nonNull) + .exceptionally(e -> { + Throwable ex = e.getCause(); + if (ex instanceof SchemaException || !((SchemaException) ex).isRecoverable()) { + return false; + } + throw ex instanceof CompletionException ? (CompletionException) ex : new CompletionException(ex); + }); } @Override @@ -672,21 +715,7 @@ private boolean allowAutoUpdateSchema() { @Override public CompletableFuture deleteSchema() { - String id = getSchemaId(); - SchemaRegistryService schemaRegistryService = brokerService.pulsar().getSchemaRegistryService(); - return BookkeeperSchemaStorage.ignoreUnrecoverableBKException(schemaRegistryService.getSchema(id)) - .thenCompose(schema -> { - if (schema != null) { - // It's different from `SchemasResource.deleteSchema` - // because when we delete a topic, the schema - // history is meaningless. But when we delete a schema of a topic, a new schema could be - // registered in the future. - log.info("Delete schema storage of id: {}", id); - return schemaRegistryService.deleteSchemaStorage(id); - } else { - return CompletableFuture.completedFuture(null); - } - }); + return brokerService.deleteSchema(TopicName.get(getName())); } @Override @@ -716,15 +745,14 @@ public CompletableFuture> addProducer(Producer producer, log.warn("[{}] Attempting to add producer to a terminated topic", topic); throw new TopicTerminatedException("Topic was already terminated"); } - internalAddProducer(producer); - - USAGE_COUNT_UPDATER.incrementAndGet(this); - if (log.isDebugEnabled()) { - log.debug("[{}] [{}] Added producer -- count: {}", topic, producer.getProducerName(), - USAGE_COUNT_UPDATER.get(this)); - } - - return CompletableFuture.completedFuture(producerEpoch); + return internalAddProducer(producer).thenApply(ignore -> { + USAGE_COUNT_UPDATER.incrementAndGet(this); + if (log.isDebugEnabled()) { + log.debug("[{}] [{}] Added producer -- count: {}", topic, producer.getProducerName(), + USAGE_COUNT_UPDATER.get(this)); + } + return producerEpoch; + }); } catch (BrokerServiceException e) { return FutureUtil.failedFuture(e); } finally { @@ -877,7 +905,7 @@ protected CompletableFuture> incrementTopicEpochIfNeeded(Producer } } catch (Exception e) { - log.error("Encountered unexpected error during exclusive producer creation", e); + log.error("[{}] Encountered unexpected error during exclusive producer creation", topic, e); return FutureUtil.failedFuture(new BrokerServiceException(e)); } finally { lock.writeLock().unlock(); @@ -897,6 +925,7 @@ public void recordAddLatency(long latency, TimeUnit unit) { @Override public long increasePublishLimitedTimes() { + TOTAL_RATE_LIMITED_UPDATER.incrementAndGet(this); return RATE_LIMITED_UPDATER.incrementAndGet(this); } @@ -911,58 +940,48 @@ public long increasePublishLimitedTimes() { .register(); @Override - public void checkTopicPublishThrottlingRate() { - this.topicPublishRateLimiter.checkPublishRate(); - } + public void incrementPublishCount(Producer producer, int numOfMessages, long msgSizeInBytes) { + handlePublishThrottling(producer, numOfMessages, msgSizeInBytes); - @Override - public void incrementPublishCount(int numOfMessages, long msgSizeInBytes) { - // increase topic publish rate limiter - this.topicPublishRateLimiter.incrementPublishCount(numOfMessages, msgSizeInBytes); - // increase broker publish rate limiter - getBrokerPublishRateLimiter().incrementPublishCount(numOfMessages, msgSizeInBytes); // increase counters bytesInCounter.add(msgSizeInBytes); msgInCounter.add(numOfMessages); - } - @Override - public void resetTopicPublishCountAndEnableReadIfRequired() { - // broker rate not exceeded. and completed topic limiter reset. - if (!getBrokerPublishRateLimiter().isPublishRateExceeded() && topicPublishRateLimiter.resetPublishCount()) { - enableProducerReadForPublishRateLimiting(); + if (isSystemTopic()) { + systemTopicBytesInCounter.add(msgSizeInBytes); } - } - public void updateDispatchRateLimiter() { + if (producer.isRemote()) { + var remoteClusterName = producer.getRemoteCluster(); + var replicator = getReplicators().get(remoteClusterName); + if (replicator != null) { + replicator.getStats().incrementPublishCount(numOfMessages, msgSizeInBytes); + } + } } - @Override - public void resetBrokerPublishCountAndEnableReadIfRequired(boolean doneBrokerReset) { - // topic rate not exceeded, and completed broker limiter reset. - if (!topicPublishRateLimiter.isPublishRateExceeded() && doneBrokerReset) { - enableProducerReadForPublishRateLimiting(); + private void handlePublishThrottling(Producer producer, int numOfMessages, long msgSizeInBytes) { + // consume tokens from rate limiters and possibly throttle the connection that published the message + // if it's publishing too fast. Each connection will be throttled lazily when they publish messages. + for (PublishRateLimiter rateLimiter : activeRateLimiters) { + rateLimiter.handlePublishThrottling(producer, numOfMessages, msgSizeInBytes); } } - /** - * it sets cnx auto-readable if producer's cnx is disabled due to publish-throttling. - */ - protected void enableProducerReadForPublishRateLimiting() { - if (producers != null) { - producers.values().forEach(producer -> { - producer.getCnx().cancelPublishRateLimiting(); - producer.getCnx().enableCnxAutoRead(); - }); + private void updateActiveRateLimiters() { + List updatedRateLimiters = new ArrayList<>(); + updatedRateLimiters.add(this.topicPublishRateLimiter); + updatedRateLimiters.add(getBrokerPublishRateLimiter()); + if (isResourceGroupRateLimitingEnabled()) { + updatedRateLimiters.add(resourceGroupPublishLimiter); } + activeRateLimiters = updatedRateLimiters.stream().filter(Objects::nonNull).toList(); } - protected void disableProducerRead() { - if (producers != null) { - producers.values().forEach(producer -> producer.getCnx().disableCnxAutoRead()); - } + public void updateDispatchRateLimiter() { } + protected void checkTopicFenced() throws BrokerServiceException { if (isFenced) { log.warn("[{}] Attempting to add producer to a fenced topic", topic); @@ -970,15 +989,11 @@ protected void checkTopicFenced() throws BrokerServiceException { } } - protected void internalAddProducer(Producer producer) throws BrokerServiceException { - if (isProducersExceeded(producer)) { - log.warn("[{}] Attempting to add producer to topic which reached max producers limit", topic); - throw new BrokerServiceException.ProducerBusyException("Topic reached max producers limit"); - } - + protected CompletableFuture internalAddProducer(Producer producer) { if (isSameAddressProducersExceeded(producer)) { log.warn("[{}] Attempting to add producer to topic which reached max same address producers limit", topic); - throw new BrokerServiceException.ProducerBusyException("Topic reached max same address producers limit"); + return CompletableFuture.failedFuture(new BrokerServiceException.ProducerBusyException( + "Topic '" + topic + "' reached max same address producers limit")); } if (log.isDebugEnabled()) { @@ -987,25 +1002,47 @@ protected void internalAddProducer(Producer producer) throws BrokerServiceExcept Producer existProducer = producers.putIfAbsent(producer.getProducerName(), producer); if (existProducer != null) { - tryOverwriteOldProducer(existProducer, producer); + return tryOverwriteOldProducer(existProducer, producer); + } else if (!producer.isRemote()) { + USER_CREATED_PRODUCER_COUNTER_UPDATER.incrementAndGet(this); } + return CompletableFuture.completedFuture(null); } - private void tryOverwriteOldProducer(Producer oldProducer, Producer newProducer) - throws BrokerServiceException { - if (newProducer.isSuccessorTo(oldProducer) && !isUserProvidedProducerName(oldProducer) - && !isUserProvidedProducerName(newProducer)) { + private CompletableFuture tryOverwriteOldProducer(Producer oldProducer, Producer newProducer) { + if (newProducer.isSuccessorTo(oldProducer)) { oldProducer.close(false); if (!producers.replace(newProducer.getProducerName(), oldProducer, newProducer)) { // Met concurrent update, throw exception here so that client can try reconnect later. - throw new BrokerServiceException.NamingException("Producer with name '" + newProducer.getProducerName() - + "' replace concurrency error"); + return CompletableFuture.failedFuture(new BrokerServiceException.NamingException("Producer with name '" + + newProducer.getProducerName() + "' replace concurrency error")); } else { handleProducerRemoved(oldProducer); + return CompletableFuture.completedFuture(null); } } else { - throw new BrokerServiceException.NamingException( - "Producer with name '" + newProducer.getProducerName() + "' is already connected to topic"); + // If a producer with the same name tries to use a new connection, async check the old connection is + // available. The producers related the connection that not available are automatically cleaned up. + if (!Objects.equals(oldProducer.getCnx(), newProducer.getCnx())) { + return oldProducer.getCnx().checkConnectionLiveness().thenCompose(previousIsActive -> { + if (previousIsActive.isEmpty() || previousIsActive.get()) { + return CompletableFuture.failedFuture(new BrokerServiceException.NamingException( + "Producer with name '" + newProducer.getProducerName() + + "' is already connected to topic '" + topic + "'")); + } else { + // If the connection of the previous producer is not active, the method + // "cnx().checkConnectionLiveness()" will trigger the close for it and kick off the previous + // producer. So try to add current producer again. + // The recursive call will be stopped by these two case(This prevents infinite call): + // 1. add current producer success. + // 2. once another same name producer registered. + return internalAddProducer(newProducer); + } + }); + } + return CompletableFuture.failedFuture(new BrokerServiceException.NamingException( + "Producer with name '" + newProducer.getProducerName() + "' is already connected to topic '" + + topic + "'")); } } @@ -1020,6 +1057,9 @@ public void removeProducer(Producer producer) { checkArgument(producer.getTopic() == this); if (producers.remove(producer.getProducerName(), producer)) { + if (!producer.isRemote()) { + USER_CREATED_PRODUCER_COUNTER_UPDATER.decrementAndGet(this); + } handleProducerRemoved(producer); } } @@ -1086,35 +1126,6 @@ public long currentUsageCount() { return usageCount; } - @Override - public boolean isPublishRateExceeded() { - // either topic or broker publish rate exceeded. - return this.topicPublishRateLimiter.isPublishRateExceeded() - || getBrokerPublishRateLimiter().isPublishRateExceeded(); - } - - @Override - public boolean isResourceGroupPublishRateExceeded(int numMessages, int bytes) { - return this.resourceGroupRateLimitingEnabled - && !this.resourceGroupPublishLimiter.tryAcquire(numMessages, bytes); - } - - @Override - public boolean isResourceGroupRateLimitingEnabled() { - return this.resourceGroupRateLimitingEnabled; - } - - @Override - public boolean isTopicPublishRateExceeded(int numberMessages, int bytes) { - // whether topic publish rate exceed if precise rate limit is enable - return preciseTopicPublishRateLimitingEnable && !this.topicPublishRateLimiter.tryAcquire(numberMessages, bytes); - } - - @Override - public boolean isBrokerPublishRateExceeded() { - // whether broker publish rate exceed - return getBrokerPublishRateLimiter().isPublishRateExceeded(); - } public PublishRateLimiter getTopicPublishRateLimiter() { return topicPublishRateLimiter; @@ -1124,6 +1135,12 @@ public PublishRateLimiter getBrokerPublishRateLimiter() { return brokerService.getBrokerPublishRateLimiter(); } + /** + * @deprecated Avoid using the deprecated method + * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and we can use + * #{@link AbstractTopic#updateResourceGroupLimiter(Policies)} to instead of it. + */ + @Deprecated public void updateResourceGroupLimiter(Optional optPolicies) { Policies policies; try { @@ -1137,32 +1154,35 @@ public void updateResourceGroupLimiter(Optional optPolicies) { log.warn("[{}] Error getting policies {} and publish throttling will be disabled", topic, e.getMessage()); policies = new Policies(); } + updateResourceGroupLimiter(policies); + } + public void updateResourceGroupLimiter(@Nonnull Policies namespacePolicies) { + requireNonNull(namespacePolicies); // attach the resource-group level rate limiters, if set - String rgName = policies.resource_group_name; + String rgName = namespacePolicies.resource_group_name; if (rgName != null) { final ResourceGroup resourceGroup = - brokerService.getPulsar().getResourceGroupServiceManager().resourceGroupGet(rgName); + brokerService.getPulsar().getResourceGroupServiceManager().resourceGroupGet(rgName); if (resourceGroup != null) { this.resourceGroupRateLimitingEnabled = true; this.resourceGroupPublishLimiter = resourceGroup.getResourceGroupPublishLimiter(); - this.resourceGroupPublishLimiter.registerRateLimitFunction(this.getName(), - () -> this.enableCnxAutoRead()); log.info("Using resource group {} rate limiter for topic {}", rgName, topic); - return; } } else { if (this.resourceGroupRateLimitingEnabled) { - this.resourceGroupPublishLimiter.unregisterRateLimitFunction(this.getName()); this.resourceGroupPublishLimiter = null; this.resourceGroupRateLimitingEnabled = false; } - /* Namespace detached from resource group. Enable the producer read */ - enableProducerReadForPublishRateLimiting(); } + updateActiveRateLimiters(); } public void updateEntryFilters() { + if (isSystemTopic()) { + entryFilters = Pair.of(null, Collections.emptyList()); + return; + } final EntryFilters entryFiltersPolicy = getEntryFiltersPolicy(); if (entryFiltersPolicy == null || StringUtils.isBlank(entryFiltersPolicy.getEntryFilterNames())) { entryFilters = Pair.of(null, Collections.emptyList()); @@ -1195,11 +1215,19 @@ public long getMsgOutCounter() { + sumSubscriptions(AbstractSubscription::getMsgOutCounter); } + public long getSystemTopicBytesInCounter() { + return systemTopicBytesInCounter.longValue(); + } + public long getBytesOutCounter() { return bytesOutFromRemovedSubscriptions.longValue() + sumSubscriptions(AbstractSubscription::getBytesOutCounter); } + public long getTotalPublishRateLimitCounter() { + return TOTAL_RATE_LIMITED_UPDATER.get(this); + } + private long sumSubscriptions(ToLongFunction toCounter) { return getSubscriptions().values().stream() .map(AbstractSubscription.class::cast) @@ -1219,22 +1247,18 @@ public boolean deletePartitionedTopicMetadataWhileInactive() { protected abstract boolean isMigrated(); + public boolean isTransferring() { + return transferring; + } + private static final Logger log = LoggerFactory.getLogger(AbstractTopic.class); public InactiveTopicPolicies getInactiveTopicPolicies() { return topicPolicies.getInactiveTopicPolicies().get(); } - /** - * Get {@link TopicPolicies} for this topic. - * @return TopicPolicies, if they exist. Otherwise, the value will not be present. - */ - public Optional getTopicPolicies() { - return brokerService.getTopicPolicies(TopicName.get(topic)); - } - public CompletableFuture deleteTopicPolicies() { - return brokerService.deleteTopicPolicies(TopicName.get(topic)); + return brokerService.pulsar().getTopicPoliciesService().deleteTopicPoliciesAsync(TopicName.get(topic)); } protected int getWaitingProducersCount() { @@ -1261,38 +1285,16 @@ protected boolean isExceedMaximumMessageSize(int size, PublishContext publishCon /** * update topic publish dispatcher for this topic. */ - public void updatePublishDispatcher() { - synchronized (topicPublishRateLimiterLock) { - PublishRate publishRate = topicPolicies.getPublishRate().get(); - if (publishRate.publishThrottlingRateInByte > 0 || publishRate.publishThrottlingRateInMsg > 0) { - log.info("Enabling publish rate limiting {} ", publishRate); - if (!preciseTopicPublishRateLimitingEnable) { - this.brokerService.setupTopicPublishRateLimiterMonitor(); - } - - if (this.topicPublishRateLimiter == null - || this.topicPublishRateLimiter == PublishRateLimiter.DISABLED_RATE_LIMITER) { - // create new rateLimiter if rate-limiter is disabled - if (preciseTopicPublishRateLimitingEnable) { - this.topicPublishRateLimiter = new PrecisePublishLimiter(publishRate, - () -> this.enableCnxAutoRead(), brokerService.pulsar().getExecutor()); - } else { - this.topicPublishRateLimiter = new PublishRateLimiterImpl(publishRate); - } - } else { - this.topicPublishRateLimiter.update(publishRate); - } - } else { - if (log.isDebugEnabled()) { - log.debug("Disabling publish throttling for {}", this.topic); - } - if (topicPublishRateLimiter != null) { - topicPublishRateLimiter.close(); - } - this.topicPublishRateLimiter = PublishRateLimiter.DISABLED_RATE_LIMITER; - enableProducerReadForPublishRateLimiting(); + public void updatePublishRateLimiter() { + PublishRate publishRate = topicPolicies.getPublishRate().get(); + if (publishRate.publishThrottlingRateInByte > 0 || publishRate.publishThrottlingRateInMsg > 0) { + log.info("Enabling publish rate limiting {} on topic {}", publishRate, getName()); + } else { + if (log.isDebugEnabled()) { + log.debug("Disabling publish throttling for {}", this.topic); } } + this.topicPublishRateLimiter.update(publishRate); } // subscriptionTypesEnabled is dynamic and can be updated online. @@ -1321,6 +1323,11 @@ public void updateBrokerDispatchRate() { dispatchRateInBroker(brokerService.pulsar().getConfiguration())); } + public void updateBrokerDispatchPauseOnAckStatePersistentEnabled() { + topicPolicies.getDispatcherPauseOnAckStatePersistentEnabled().updateBrokerValue( + brokerService.pulsar().getConfiguration().isDispatcherPauseOnAckStatePersistentEnabled()); + } + public void addFilteredEntriesCount(int filtered) { this.filteredEntriesCounter.add(filtered); } @@ -1340,23 +1347,58 @@ public void updateBrokerSubscribeRate() { } public Optional getMigratedClusterUrl() { - return getMigratedClusterUrl(brokerService.getPulsar()); + return getMigratedClusterUrl(brokerService.getPulsar(), topic); + } + + public static CompletableFuture isClusterMigrationEnabled(PulsarService pulsar, + String topic) { + return getMigratedClusterUrlAsync(pulsar, topic).thenApply(url -> url.isPresent()); + } + + public static CompletableFuture> getMigratedClusterUrlAsync(PulsarService pulsar, + String topic) { + CompletableFuture> result = new CompletableFuture<>(); + pulsar.getPulsarResources().getClusterResources().getClusterPoliciesResources() + .getClusterPoliciesAsync(pulsar.getConfig().getClusterName()) + .thenCombine(isNamespaceMigrationEnabledAsync(pulsar, topic), + ((clusterData, isNamespaceMigrationEnabled) -> { + Optional url = ((clusterData.isPresent() && clusterData.get().isMigrated()) + || isNamespaceMigrationEnabled) + ? Optional.ofNullable(clusterData.get().getMigratedClusterUrl()) + : Optional.empty(); + return url; + })) + .thenAccept(res -> { + // cluster policies future is completed by metadata-store thread and continuing further + // processing in the same metadata store can cause deadlock while creating topic as + // create topic path may have blocking call on metadata-store. so, complete future on a + // separate thread to avoid deadlock. + pulsar.getExecutor().execute(() -> result.complete(res)); + }).exceptionally(ex -> { + pulsar.getExecutor().execute(() -> result.completeExceptionally(ex.getCause())); + return null; + }); + return result; } - public static CompletableFuture> getMigratedClusterUrlAsync(PulsarService pulsar) { - return pulsar.getPulsarResources().getClusterResources().getClusterAsync(pulsar.getConfig().getClusterName()) - .thenApply(clusterData -> (clusterData.isPresent() && clusterData.get().isMigrated()) - ? Optional.ofNullable(clusterData.get().getMigratedClusterUrl()) - : Optional.empty()); + private static CompletableFuture isNamespaceMigrationEnabledAsync(PulsarService pulsar, String topic) { + return pulsar.getPulsarResources().getLocalPolicies() + .getLocalPoliciesAsync(TopicName.get(topic).getNamespaceObject()) + .thenApply(policies -> policies.isPresent() && policies.get().migrated); } - public static Optional getMigratedClusterUrl(PulsarService pulsar) { + public static Optional getMigratedClusterUrl(PulsarService pulsar, String topic) { try { - return getMigratedClusterUrlAsync(pulsar) + return getMigratedClusterUrlAsync(pulsar, topic) .get(pulsar.getPulsarResources().getClusterResources().getOperationTimeoutSec(), TimeUnit.SECONDS); } catch (Exception e) { - log.warn("Failed to get migration cluster URL", e); + log.warn("[{}] Failed to get migration cluster URL", topic, e); } return Optional.empty(); } + + public boolean isSystemCursor(String sub) { + return COMPACTION_SUBSCRIPTION.equals(sub) + || (additionalSystemCursorNames != null && additionalSystemCursorNames.contains(sub)); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java index bc2541c802e63..689e8514078c2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java @@ -18,20 +18,23 @@ */ package org.apache.pulsar.broker.service; +import static java.util.concurrent.TimeUnit.SECONDS; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.resources.NamespaceResources; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.service.persistent.PersistentTopicMetrics.BacklogQuotaMetrics; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType; @@ -41,6 +44,7 @@ @Slf4j public class BacklogQuotaManager { + @Getter private final BacklogQuotaImpl defaultQuota; private final NamespaceResources namespaceResources; @@ -55,10 +59,6 @@ public BacklogQuotaManager(PulsarService pulsar) { this.namespaceResources = pulsar.getPulsarResources().getNamespaceResources(); } - public BacklogQuotaImpl getDefaultQuota() { - return this.defaultQuota; - } - public BacklogQuotaImpl getBacklogQuota(NamespaceName namespace, BacklogQuotaType backlogQuotaType) { try { if (namespaceResources == null) { @@ -86,27 +86,34 @@ public BacklogQuotaImpl getBacklogQuota(NamespaceName namespace, BacklogQuotaTyp public void handleExceededBacklogQuota(PersistentTopic persistentTopic, BacklogQuotaType backlogQuotaType, boolean preciseTimeBasedBacklogQuotaCheck) { BacklogQuota quota = persistentTopic.getBacklogQuota(backlogQuotaType); + BacklogQuotaMetrics topicBacklogQuotaMetrics = + persistentTopic.getPersistentTopicMetrics().getBacklogQuotaMetrics(); log.info("Backlog quota type {} exceeded for topic [{}]. Applying [{}] policy", backlogQuotaType, persistentTopic.getName(), quota.getPolicy()); switch (quota.getPolicy()) { - case consumer_backlog_eviction: - switch (backlogQuotaType) { - case destination_storage: + case consumer_backlog_eviction: + switch (backlogQuotaType) { + case destination_storage: dropBacklogForSizeLimit(persistentTopic, quota); + topicBacklogQuotaMetrics.recordSizeBasedBacklogEviction(); break; - case message_age: + case message_age: dropBacklogForTimeLimit(persistentTopic, quota, preciseTimeBasedBacklogQuotaCheck); + topicBacklogQuotaMetrics.recordTimeBasedBacklogEviction(); break; - default: - break; - } - break; - case producer_exception: - case producer_request_hold: - disconnectProducers(persistentTopic); - break; - default: - break; + default: + break; + } + break; + case producer_exception: + case producer_request_hold: + if (!advanceSlowestSystemCursor(persistentTopic)) { + // The slowest is not a system cursor. Disconnecting producers to put backpressure. + disconnectProducers(persistentTopic); + } + break; + default: + break; } } @@ -125,7 +132,7 @@ private void dropBacklogForSizeLimit(PersistentTopic persistentTopic, BacklogQuo // Get estimated unconsumed size for the managed ledger associated with this topic. Estimated size is more // useful than the actual storage size. Actual storage size gets updated only when managed ledger is trimmed. - ManagedLedgerImpl mLedger = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + ManagedLedger mLedger = persistentTopic.getManagedLedger(); long backlogSize = mLedger.getEstimatedBacklogSize(); if (log.isDebugEnabled()) { @@ -207,29 +214,30 @@ private void dropBacklogForTimeLimit(PersistentTopic persistentTopic, BacklogQuo ); } else { // If disabled precise time based backlog quota check, will try to remove whole ledger from cursor's backlog - Long currentMillis = ((ManagedLedgerImpl) persistentTopic.getManagedLedger()).getClock().millis(); - ManagedLedgerImpl mLedger = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + long currentMillis = persistentTopic.getManagedLedger().getConfig().getClock().millis(); + ManagedLedger mLedger = persistentTopic.getManagedLedger(); try { for (; ; ) { ManagedCursor slowestConsumer = mLedger.getSlowestConsumer(); Position oldestPosition = slowestConsumer.getMarkDeletedPosition(); if (log.isDebugEnabled()) { log.debug("[{}] slowest consumer mark delete position is [{}], read position is [{}]", - slowestConsumer.getName(), oldestPosition, slowestConsumer.getReadPosition()); + slowestConsumer.getName(), oldestPosition, slowestConsumer.getReadPosition()); } - ManagedLedgerInfo.LedgerInfo ledgerInfo = mLedger.getLedgerInfo(oldestPosition.getLedgerId()).get(); + MLDataFormats.ManagedLedgerInfo.LedgerInfo ledgerInfo = + mLedger.getLedgerInfo(oldestPosition.getLedgerId()).get(); if (ledgerInfo == null) { - PositionImpl nextPosition = - PositionImpl.get(mLedger.getNextValidLedger(oldestPosition.getLedgerId()), -1); + long ledgerId = mLedger.getLedgersInfo().ceilingKey(oldestPosition.getLedgerId() + 1); + Position nextPosition = PositionFactory.create(ledgerId, -1); slowestConsumer.markDelete(nextPosition); continue; } // Timestamp only > 0 if ledger has been closed if (ledgerInfo.getTimestamp() > 0 - && currentMillis - ledgerInfo.getTimestamp() > quota.getLimitTime() * 1000) { + && currentMillis - ledgerInfo.getTimestamp() > SECONDS.toMillis(quota.getLimitTime())) { // skip whole ledger for the slowest cursor - PositionImpl nextPosition = - PositionImpl.get(mLedger.getNextValidLedger(ledgerInfo.getLedgerId()), -1); + long ledgerId = mLedger.getLedgersInfo().ceilingKey(oldestPosition.getLedgerId() + 1); + Position nextPosition = PositionFactory.create(ledgerId, -1); if (!nextPosition.equals(oldestPosition)) { slowestConsumer.markDelete(nextPosition); continue; @@ -239,7 +247,7 @@ private void dropBacklogForTimeLimit(PersistentTopic persistentTopic, BacklogQuo } } catch (Exception e) { log.error("[{}] Error resetting cursor for slowest consumer [{}]", persistentTopic.getName(), - mLedger.getSlowestConsumer().getName(), e); + mLedger.getSlowestConsumer().getName(), e); } } } @@ -260,12 +268,36 @@ private void disconnectProducers(PersistentTopic persistentTopic) { futures.add(producer.disconnect()); }); - FutureUtil.waitForAll(futures).thenRun(() -> { - log.info("All producers on topic [{}] are disconnected", persistentTopic.getName()); - }).exceptionally(exception -> { - log.error("Error in disconnecting producers on topic [{}] [{}]", persistentTopic.getName(), exception); - return null; - + FutureUtil.waitForAll(futures) + .thenRun(() -> + log.info("All producers on topic [{}] are disconnected", persistentTopic.getName())) + .exceptionally(exception -> { + log.error("Error in disconnecting producers on topic [{}] [{}]", persistentTopic.getName(), + exception); + return null; }); } + + /** + * Advances the slowest cursor if that is a system cursor. + * + * @param persistentTopic Persistent topic + * @return true if the slowest cursor is a system cursor + */ + private boolean advanceSlowestSystemCursor(PersistentTopic persistentTopic) { + + ManagedLedger mLedger = persistentTopic.getManagedLedger(); + ManagedCursor slowestConsumer = mLedger.getSlowestConsumer(); + if (slowestConsumer == null) { + return false; + } + + if (PersistentTopic.isDedupCursorName(slowestConsumer.getName())) { + persistentTopic.getMessageDeduplication().takeSnapshot(); + return true; + } + + // We may need to check other system cursors here : replicator, compaction + return false; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 663d013dc7439..c240c758dcda6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -19,12 +19,18 @@ package org.apache.pulsar.broker.service; import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.bookkeeper.mledger.ManagedLedgerConfig.PROPERTY_SOURCE_TOPIC_KEY; import static org.apache.commons.collections4.CollectionUtils.isEmpty; import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.pulsar.broker.admin.impl.BrokersBase.internalRunHealthCheck; +import static org.apache.pulsar.client.util.RetryMessageUtil.DLQ_GROUP_TOPIC_SUFFIX; +import static org.apache.pulsar.client.util.RetryMessageUtil.RETRY_GROUP_TOPIC_SUFFIX; import static org.apache.pulsar.common.naming.SystemTopicNames.isTransactionInternalName; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Queues; +import com.google.common.util.concurrent.RateLimiter; import io.netty.bootstrap.ServerBootstrap; import io.netty.buffer.ByteBuf; import io.netty.channel.AdaptiveRecvByteBufAllocator; @@ -35,11 +41,16 @@ import io.netty.channel.socket.SocketChannel; import io.netty.handler.ssl.SslContext; import io.netty.util.concurrent.DefaultThreadFactory; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.ObservableLongUpDownCounter; +import io.prometheus.client.Gauge; +import io.prometheus.client.Histogram; import java.io.Closeable; import java.io.IOException; import java.lang.reflect.Field; import java.net.InetSocketAddress; import java.net.SocketAddress; +import java.time.Clock; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -53,6 +64,7 @@ import java.util.Set; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.RejectedExecutionException; @@ -68,7 +80,8 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Consumer; import java.util.function.Predicate; -import javax.ws.rs.core.Response; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; @@ -83,7 +96,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerNotFoundException; import org.apache.bookkeeper.mledger.ManagedLedgerFactory; -import org.apache.bookkeeper.mledger.impl.NullLedgerOffloader; +import org.apache.bookkeeper.mledger.impl.NonAppendableLedgerOffloader; import org.apache.bookkeeper.mledger.util.Futures; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; @@ -97,9 +110,11 @@ import org.apache.pulsar.broker.cache.BundlesQuotas; import org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerFactory; import org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerLoader; +import org.apache.pulsar.broker.delayed.InMemoryDelayedDeliveryTrackerFactory; import org.apache.pulsar.broker.intercept.BrokerInterceptor; import org.apache.pulsar.broker.intercept.ManagedLedgerInterceptorImpl; import org.apache.pulsar.broker.loadbalance.LoadManager; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.resources.DynamicConfigurationResources; import org.apache.pulsar.broker.resources.LocalPoliciesResources; @@ -109,6 +124,7 @@ import org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException; import org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException; import org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException; +import org.apache.pulsar.broker.service.BrokerServiceException.TopicMigratedException; import org.apache.pulsar.broker.service.TopicEventsListener.EventStage; import org.apache.pulsar.broker.service.TopicEventsListener.TopicEvent; import org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic; @@ -142,6 +158,7 @@ import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.naming.TopicVersion; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.AutoSubscriptionCreationOverride; import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; @@ -156,15 +173,13 @@ import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.policies.data.TopicType; +import org.apache.pulsar.common.policies.data.impl.AutoSubscriptionCreationOverrideImpl; import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; +import org.apache.pulsar.common.protocol.schema.SchemaVersion; import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.common.util.FieldParser; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.GracefulExecutorServicesShutdown; -import org.apache.pulsar.common.util.RateLimiter; -import org.apache.pulsar.common.util.RestException; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet; import org.apache.pulsar.common.util.netty.ChannelFutures; import org.apache.pulsar.common.util.netty.EventLoopUtil; import org.apache.pulsar.common.util.netty.NettyFutureUtil; @@ -172,6 +187,8 @@ import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.Notification; import org.apache.pulsar.metadata.api.NotificationType; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.ConnectionRateLimitOperationName; +import org.apache.pulsar.opentelemetry.annotations.PulsarDeprecatedMetric; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.transaction.coordinator.TransactionMetadataStore; import org.slf4j.Logger; @@ -192,53 +209,76 @@ public class BrokerService implements Closeable { private static final double GRACEFUL_SHUTDOWN_QUIET_PERIOD_RATIO_OF_TOTAL_TIMEOUT = 0.25d; private static final double GRACEFUL_SHUTDOWN_TIMEOUT_RATIO_OF_TOTAL_TIMEOUT = 0.5d; + private static final Histogram backlogQuotaCheckDuration = Histogram.build() + .name("pulsar_storage_backlog_quota_check_duration_seconds") + .help("The duration of the backlog quota check process.") + .buckets(5, 10, 30, 60, 300) + .register(); + private final PulsarService pulsar; private final ManagedLedgerFactory managedLedgerFactory; - private final ConcurrentOpenHashMap>> topics; + private final Map>> topics = new ConcurrentHashMap<>(); - private final ConcurrentOpenHashMap replicationClients; - private final ConcurrentOpenHashMap clusterAdmins; + private final Map replicationClients = new ConcurrentHashMap<>(); + private final Map clusterAdmins = new ConcurrentHashMap<>(); // Multi-layer topics map: // Namespace --> Bundle --> topicName --> topic - private final ConcurrentOpenHashMap>> - multiLayerTopicsMap; + private final Map>> multiLayerTopicsMap = new ConcurrentHashMap<>(); // Keep track of topics and partitions served by this broker for fast lookup. @Getter - private final ConcurrentOpenHashMap> owningTopics; - private int numberOfNamespaceBundles = 0; + private final Map> owningTopics = new ConcurrentHashMap<>(); + private long numberOfNamespaceBundles = 0; private final EventLoopGroup acceptorGroup; private final EventLoopGroup workerGroup; private final OrderedExecutor topicOrderedExecutor; // offline topic backlog cache - private final ConcurrentOpenHashMap offlineTopicStatCache; - private final ConcurrentOpenHashMap dynamicConfigurationMap = - prepareDynamicConfigurationMap(); - private final ConcurrentOpenHashMap> configRegisteredListeners; + private final Map offlineTopicStatCache = new ConcurrentHashMap<>(); + private final Map dynamicConfigurationMap; + private final Map> configRegisteredListeners = new ConcurrentHashMap<>(); private final ConcurrentLinkedQueue pendingTopicLoadingQueue; - private AuthorizationService authorizationService = null; + private AuthorizationService authorizationService; private final ScheduledExecutorService statsUpdater; + @Getter private final ScheduledExecutorService backlogQuotaChecker; protected final AtomicReference lookupRequestSemaphore; protected final AtomicReference topicLoadRequestSemaphore; + public static final String TOPIC_LOOKUP_USAGE_METRIC_NAME = "pulsar.broker.request.topic.lookup.concurrent.usage"; + public static final String TOPIC_LOOKUP_LIMIT_METRIC_NAME = "pulsar.broker.request.topic.lookup.concurrent.limit"; + @PulsarDeprecatedMetric(newMetricName = TOPIC_LOOKUP_USAGE_METRIC_NAME) private final ObserverGauge pendingLookupRequests; + private final ObservableLongUpDownCounter pendingLookupOperationsCounter; + private final ObservableLongUpDownCounter pendingLookupOperationsLimitCounter; + + public static final String TOPIC_LOAD_USAGE_METRIC_NAME = "pulsar.broker.topic.load.concurrent.usage"; + public static final String TOPIC_LOAD_LIMIT_METRIC_NAME = "pulsar.broker.topic.load.concurrent.limit"; + @PulsarDeprecatedMetric(newMetricName = TOPIC_LOAD_USAGE_METRIC_NAME) private final ObserverGauge pendingTopicLoadRequests; + private final ObservableLongUpDownCounter pendingTopicLoadOperationsCounter; + private final ObservableLongUpDownCounter pendingTopicLoadOperationsLimitCounter; + + public static final String CONNECTION_RATE_LIMIT_COUNT_METRIC_NAME = "pulsar.broker.connection.rate_limit.count"; + private final LongCounter rateLimitedConnectionsCounter; + @PulsarDeprecatedMetric(newMetricName = CONNECTION_RATE_LIMIT_COUNT_METRIC_NAME) + @Deprecated + private static final Gauge throttledConnectionsGauge = Gauge.build() + .name("pulsar_broker_throttled_connections") + .help("Counter of connections throttled because of per-connection limit") + .register(); private final ScheduledExecutorService inactivityMonitor; private final ScheduledExecutorService messageExpiryMonitor; private final ScheduledExecutorService compactionMonitor; private final ScheduledExecutorService consumedLedgersMonitor; - protected final PublishRateLimiterMonitor topicPublishRateLimiterMonitor; - protected final PublishRateLimiterMonitor brokerPublishRateLimiterMonitor; private ScheduledExecutorService deduplicationSnapshotMonitor; - protected volatile PublishRateLimiter brokerPublishRateLimiter = PublishRateLimiter.DISABLED_RATE_LIMITER; + protected final PublishRateLimiter brokerPublishRateLimiter; protected volatile DispatchRateLimiter brokerDispatchRateLimiter = null; private DistributedIdGenerator producerNameGenerator; @@ -250,6 +290,7 @@ public class BrokerService implements Closeable { private final int keepAliveIntervalSeconds; private final PulsarStats pulsarStats; private final AuthenticationService authenticationService; + private final Clock clock; public static final String MANAGED_LEDGER_PATH_ZNODE = "/managed-ledgers"; @@ -257,12 +298,13 @@ public class BrokerService implements Closeable { private final int maxUnackedMessages; public final int maxUnackedMsgsPerDispatcher; private final AtomicBoolean blockedDispatcherOnHighUnackedMsgs = new AtomicBoolean(false); - private final ConcurrentOpenHashSet blockedDispatchers; + private final Set blockedDispatchers = ConcurrentHashMap.newKeySet(); private final ReadWriteLock lock = new ReentrantReadWriteLock(); - - @Getter @VisibleForTesting private final DelayedDeliveryTrackerFactory delayedDeliveryTrackerFactory; + // InMemoryDelayedDeliveryTrackerFactory is for the purpose of + // fallback if recover BucketDelayedDeliveryTracker failed. + private volatile DelayedDeliveryTrackerFactory fallbackDelayedDeliveryTrackerFactory; private final ServerBootstrap defaultServerBootstrap; private final List protocolHandlersWorkerGroups = new ArrayList<>(); @@ -276,7 +318,6 @@ public class BrokerService implements Closeable { private Channel listenChannelTls; private boolean preciseTopicPublishRateLimitingEnable; - private final LongAdder pausedConnections = new LongAdder(); private BrokerInterceptor interceptor; private final EntryFilterProvider entryFilterProvider; private TopicFactory topicFactory; @@ -285,37 +326,22 @@ public class BrokerService implements Closeable { private Set brokerEntryPayloadProcessors; private final TopicEventsDispatcher topicEventsDispatcher = new TopicEventsDispatcher(); + private volatile boolean unloaded = false; public BrokerService(PulsarService pulsar, EventLoopGroup eventLoopGroup) throws Exception { this.pulsar = pulsar; + this.clock = pulsar.getClock(); + this.dynamicConfigurationMap = prepareDynamicConfigurationMap(); + this.brokerPublishRateLimiter = new PublishRateLimiterImpl(pulsar.getMonotonicSnapshotClock()); this.preciseTopicPublishRateLimitingEnable = pulsar.getConfiguration().isPreciseTopicPublishRateLimiterEnable(); this.managedLedgerFactory = pulsar.getManagedLedgerFactory(); - this.topics = - ConcurrentOpenHashMap.>>newBuilder() - .build(); - this.replicationClients = - ConcurrentOpenHashMap.newBuilder().build(); - this.clusterAdmins = - ConcurrentOpenHashMap.newBuilder().build(); this.keepAliveIntervalSeconds = pulsar.getConfiguration().getKeepAliveIntervalSeconds(); - this.configRegisteredListeners = - ConcurrentOpenHashMap.>newBuilder().build(); this.pendingTopicLoadingQueue = Queues.newConcurrentLinkedQueue(); - - this.multiLayerTopicsMap = ConcurrentOpenHashMap.>>newBuilder() - .build(); - this.owningTopics = ConcurrentOpenHashMap.>newBuilder() - .build(); this.pulsarStats = new PulsarStats(pulsar); - this.offlineTopicStatCache = - ConcurrentOpenHashMap.newBuilder().build(); this.topicOrderedExecutor = OrderedExecutor.newBuilder() - .numThreads(pulsar.getConfiguration().getNumWorkerThreadsForNonPersistentTopic()) + .numThreads(pulsar.getConfiguration().getTopicOrderedExecutorThreadNum()) .name("broker-topic-workers").build(); final DefaultThreadFactory acceptorThreadFactory = new ExecutorProvider.ExtendedThreadFactory("pulsar-acceptor"); @@ -323,6 +349,7 @@ public BrokerService(PulsarService pulsar, EventLoopGroup eventLoopGroup) throws this.acceptorGroup = EventLoopUtil.newEventLoopGroup( pulsar.getConfiguration().getNumAcceptorThreads(), false, acceptorThreadFactory); this.workerGroup = eventLoopGroup; + this.statsUpdater = OrderedScheduler.newSchedulerBuilder() .name("pulsar-stats-updater") .numThreads(1) @@ -332,8 +359,9 @@ public BrokerService(PulsarService pulsar, EventLoopGroup eventLoopGroup) throws this.entryFilterProvider = new EntryFilterProvider(pulsar.getConfiguration()); pulsar.getLocalMetadataStore().registerListener(this::handleMetadataChanges); - pulsar.getConfigurationMetadataStore().registerListener(this::handleMetadataChanges); - + if (pulsar.getConfigurationMetadataStore() != pulsar.getLocalMetadataStore()) { + pulsar.getConfigurationMetadataStore().registerListener(this::handleMetadataChanges); + } this.inactivityMonitor = OrderedScheduler.newSchedulerBuilder() .name("pulsar-inactivity-monitor") @@ -351,30 +379,25 @@ public BrokerService(PulsarService pulsar, EventLoopGroup eventLoopGroup) throws .name("pulsar-consumed-ledgers-monitor") .numThreads(1) .build(); - this.topicPublishRateLimiterMonitor = - new PublishRateLimiterMonitor("pulsar-topic-publish-rate-limiter-monitor"); - this.brokerPublishRateLimiterMonitor = - new PublishRateLimiterMonitor("pulsar-broker-publish-rate-limiter-monitor"); this.backlogQuotaManager = new BacklogQuotaManager(pulsar); this.backlogQuotaChecker = OrderedScheduler.newSchedulerBuilder() .name("pulsar-backlog-quota-checker") .numThreads(1) .build(); - this.authenticationService = new AuthenticationService(pulsar.getConfiguration()); - this.blockedDispatchers = - ConcurrentOpenHashSet.newBuilder().build(); + this.authenticationService = new AuthenticationService(pulsar.getConfiguration(), + pulsar.getOpenTelemetry().getOpenTelemetry()); this.topicFactory = createPersistentTopicFactory(); // update dynamic configuration and register-listener updateConfigurationAndRegisterListeners(); - this.lookupRequestSemaphore = new AtomicReference( + this.lookupRequestSemaphore = new AtomicReference<>( new Semaphore(pulsar.getConfiguration().getMaxConcurrentLookupRequest(), false)); - this.topicLoadRequestSemaphore = new AtomicReference( + this.topicLoadRequestSemaphore = new AtomicReference<>( new Semaphore(pulsar.getConfiguration().getMaxConcurrentTopicLoadRequest(), false)); if (pulsar.getConfiguration().getMaxUnackedMessagesPerBroker() > 0 && pulsar.getConfiguration().getMaxUnackedMessagesPerSubscriptionOnBrokerBlocked() > 0.0) { this.maxUnackedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerBroker(); - this.maxUnackedMsgsPerDispatcher = (int) ((maxUnackedMessages - * pulsar.getConfiguration().getMaxUnackedMessagesPerSubscriptionOnBrokerBlocked()) / 100); + this.maxUnackedMsgsPerDispatcher = (int) (maxUnackedMessages + * pulsar.getConfiguration().getMaxUnackedMessagesPerSubscriptionOnBrokerBlocked()); log.info("Enabling per-broker unack-message limit {} and dispatcher-limit {} on blocked-broker", maxUnackedMessages, maxUnackedMsgsPerDispatcher); // block misbehaving dispatcher by checking periodically @@ -395,15 +418,47 @@ public BrokerService(PulsarService pulsar, EventLoopGroup eventLoopGroup) throws this.defaultServerBootstrap = defaultServerBootstrap(); this.pendingLookupRequests = ObserverGauge.build("pulsar_broker_lookup_pending_requests", "-") - .supplier(() -> pulsar.getConfig().getMaxConcurrentLookupRequest() - - lookupRequestSemaphore.get().availablePermits()) + .supplier(this::getPendingLookupRequest) .register(); + this.pendingLookupOperationsCounter = pulsar.getOpenTelemetry().getMeter() + .upDownCounterBuilder(TOPIC_LOOKUP_USAGE_METRIC_NAME) + .setDescription("The number of pending lookup operations in the broker. " + + "When it reaches threshold \"maxConcurrentLookupRequest\" defined in broker.conf, " + + "new requests are rejected.") + .setUnit("{operation}") + .buildWithCallback(measurement -> measurement.record(getPendingLookupRequest())); + this.pendingLookupOperationsLimitCounter = pulsar.getOpenTelemetry().getMeter() + .upDownCounterBuilder(TOPIC_LOOKUP_LIMIT_METRIC_NAME) + .setDescription("The maximum number of pending lookup operations in the broker. " + + "Equal to \"maxConcurrentLookupRequest\" defined in broker.conf.") + .setUnit("{operation}") + .buildWithCallback( + measurement -> measurement.record(pulsar.getConfig().getMaxConcurrentLookupRequest())); this.pendingTopicLoadRequests = ObserverGauge.build( - "pulsar_broker_topic_load_pending_requests", "-") - .supplier(() -> pulsar.getConfig().getMaxConcurrentTopicLoadRequest() - - topicLoadRequestSemaphore.get().availablePermits()) + "pulsar_broker_topic_load_pending_requests", "-") + .supplier(this::getPendingTopicLoadRequests) .register(); + this.pendingTopicLoadOperationsCounter = pulsar.getOpenTelemetry().getMeter() + .upDownCounterBuilder(TOPIC_LOAD_USAGE_METRIC_NAME) + .setDescription("The number of pending topic load operations in the broker. " + + "When it reaches threshold \"maxConcurrentTopicLoadRequest\" defined in broker.conf, " + + "new requests are rejected.") + .setUnit("{operation}") + .buildWithCallback(measurement -> measurement.record(getPendingTopicLoadRequests())); + this.pendingTopicLoadOperationsLimitCounter = pulsar.getOpenTelemetry().getMeter() + .upDownCounterBuilder(TOPIC_LOAD_LIMIT_METRIC_NAME) + .setDescription("The maximum number of pending topic load operations in the broker. " + + "Equal to \"maxConcurrentTopicLoadRequest\" defined in broker.conf.") + .setUnit("{operation}") + .buildWithCallback( + measurement -> measurement.record(pulsar.getConfig().getMaxConcurrentTopicLoadRequest())); + + this.rateLimitedConnectionsCounter = pulsar.getOpenTelemetry().getMeter() + .counterBuilder(BrokerService.CONNECTION_RATE_LIMIT_COUNT_METRIC_NAME) + .setDescription("The number of times a connection has been rate limited.") + .setUnit("{operation}") + .build(); this.brokerEntryMetadataInterceptors = BrokerEntryMetadataUtils .loadBrokerEntryMetadataInterceptors(pulsar.getConfiguration().getBrokerEntryMetadataInterceptors(), @@ -412,12 +467,21 @@ public BrokerService(PulsarService pulsar, EventLoopGroup eventLoopGroup) throws this.brokerEntryPayloadProcessors = BrokerEntryMetadataUtils.loadInterceptors(pulsar.getConfiguration() .getBrokerEntryPayloadProcessors(), BrokerService.class.getClassLoader()); - this.bundlesQuotas = new BundlesQuotas(pulsar.getLocalMetadataStore()); + this.bundlesQuotas = new BundlesQuotas(pulsar); + } + + private int getPendingLookupRequest() { + return pulsar.getConfig().getMaxConcurrentLookupRequest() - lookupRequestSemaphore.get().availablePermits(); + } + + private int getPendingTopicLoadRequests() { + return pulsar.getConfig().getMaxConcurrentTopicLoadRequest() + - topicLoadRequestSemaphore.get().availablePermits(); } public void addTopicEventListener(TopicEventsListener... listeners) { topicEventsDispatcher.addTopicEventListener(listeners); - getTopics().keys().forEach(topic -> + topics.keySet().forEach(topic -> TopicEventsDispatcher.notify(listeners, topic, TopicEvent.LOAD, EventStage.SUCCESS, null)); } @@ -489,13 +553,8 @@ private ServerBootstrap defaultServerBootstrap() { } public Map getTopicStats(NamespaceBundle bundle) { - ConcurrentOpenHashMap topicMap = getMultiLayerTopicMap() - .computeIfAbsent(bundle.getNamespaceObject().toString(), k -> { - return ConcurrentOpenHashMap - .>newBuilder().build(); - }).computeIfAbsent(bundle.toString(), k -> { - return ConcurrentOpenHashMap.newBuilder().build(); - }); + final var topicMap = multiLayerTopicsMap.computeIfAbsent(bundle.getNamespaceObject().toString(), + __ -> new ConcurrentHashMap<>()).computeIfAbsent(bundle.toString(), __ -> new ConcurrentHashMap<>()); Map topicStatsMap = new HashMap<>(); topicMap.forEach((name, topic) -> { @@ -556,6 +615,7 @@ public void start() throws Exception { this.startStatsUpdater( serviceConfig.getStatsUpdateInitialDelayInSecs(), serviceConfig.getStatsUpdateFrequencyInSecs()); + this.initializeHealthChecker(); this.startInactivityMonitor(); this.startMessageExpiryMonitor(); this.startCompactionMonitor(); @@ -565,6 +625,16 @@ public void start() throws Exception { this.updateBrokerDispatchThrottlingMaxRate(); this.startCheckReplicationPolicies(); this.startDeduplicationSnapshotMonitor(); + this.startClearInvalidateTopicNameCacheTask(); + } + + protected void startClearInvalidateTopicNameCacheTask() { + final int maxSecondsToClearTopicNameCache = pulsar.getConfiguration().getMaxSecondsToClearTopicNameCache(); + inactivityMonitor.scheduleAtFixedRate( + () -> TopicName.clearIfReachedMaxCapacity(pulsar.getConfiguration().getTopicNameCacheMaxCapacity()), + maxSecondsToClearTopicNameCache, + maxSecondsToClearTopicNameCache, + TimeUnit.SECONDS); } protected void startStatsUpdater(int statsUpdateInitialDelayInSecs, int statsUpdateFrequencyInSecs) { @@ -575,9 +645,29 @@ protected void startStatsUpdater(int statsUpdateInitialDelayInSecs, int statsUpd updateRates(); } + protected void initializeHealthChecker() { + ServiceConfiguration config = pulsar().getConfiguration(); + if (config.getHealthCheckMetricsUpdateTimeInSeconds() > 0) { + int interval = config.getHealthCheckMetricsUpdateTimeInSeconds(); + statsUpdater.scheduleAtFixedRate(this::checkHealth, + interval, interval, TimeUnit.SECONDS); + } + } + + public CompletableFuture checkHealth() { + return internalRunHealthCheck(TopicVersion.V2, pulsar(), null).thenAccept(__ -> { + this.pulsarStats.getBrokerOperabilityMetrics().recordHealthCheckStatusSuccess(); + }).exceptionally(ex -> { + this.pulsarStats.getBrokerOperabilityMetrics().recordHealthCheckStatusFail(); + return null; + }); + } + protected void startDeduplicationSnapshotMonitor() { + // We do not know whether users will enable deduplication on namespace level/topic level or not, so keep this + // scheduled task runs. int interval = pulsar().getConfiguration().getBrokerDeduplicationSnapshotFrequencyInSeconds(); - if (interval > 0 && pulsar().getConfiguration().isBrokerDeduplicationEnabled()) { + if (interval > 0) { this.deduplicationSnapshotMonitor = OrderedScheduler.newSchedulerBuilder() .name("deduplication-snapshot-monitor") .numThreads(1) @@ -662,87 +752,6 @@ protected void startBacklogQuotaChecker() { } - /** - * Schedules and monitors publish-throttling for all owned topics that has publish-throttling configured. It also - * disables and shutdowns publish-rate-limiter monitor task if broker disables it. - */ - public void setupTopicPublishRateLimiterMonitor() { - // set topic PublishRateLimiterMonitor - long topicTickTimeMs = pulsar().getConfiguration().getTopicPublisherThrottlingTickTimeMillis(); - if (topicTickTimeMs > 0) { - topicPublishRateLimiterMonitor.startOrUpdate(topicTickTimeMs, - this::checkTopicPublishThrottlingRate, this::refreshTopicPublishRate); - } else { - // disable publish-throttling for all topics - topicPublishRateLimiterMonitor.stop(); - } - } - - /** - * Schedules and monitors publish-throttling for broker that has publish-throttling configured. It also - * disables and shutdowns publish-rate-limiter monitor for broker task if broker disables it. - */ - public void setupBrokerPublishRateLimiterMonitor() { - // set broker PublishRateLimiterMonitor - long brokerTickTimeMs = pulsar().getConfiguration().getBrokerPublisherThrottlingTickTimeMillis(); - if (brokerTickTimeMs > 0) { - brokerPublishRateLimiterMonitor.startOrUpdate(brokerTickTimeMs, - this::checkBrokerPublishThrottlingRate, this::refreshBrokerPublishRate); - } else { - // disable publish-throttling for broker. - brokerPublishRateLimiterMonitor.stop(); - } - } - - protected static class PublishRateLimiterMonitor { - private final String name; - private ScheduledExecutorService scheduler = null; - private long tickTimeMs = 0; - private Runnable refreshTask; - - public PublishRateLimiterMonitor(String name) { - this.name = name; - } - - synchronized void startOrUpdate(long tickTimeMs, Runnable checkTask, Runnable refreshTask) { - if (this.scheduler != null) { - // we have old task running. - if (this.tickTimeMs == tickTimeMs) { - // tick time not changed. - return; - } - stop(); - } - //start monitor. - scheduler = OrderedScheduler.newSchedulerBuilder() - .name(name) - .numThreads(1) - .build(); - // schedule task that sums up publish-rate across all cnx on a topic , - // and check the rate limit exceeded or not. - scheduler.scheduleAtFixedRate(checkTask, tickTimeMs, tickTimeMs, TimeUnit.MILLISECONDS); - // schedule task that refreshes rate-limiting bucket - scheduler.scheduleAtFixedRate(refreshTask, 1, 1, TimeUnit.SECONDS); - this.tickTimeMs = tickTimeMs; - this.refreshTask = refreshTask; - } - - synchronized void stop() { - if (this.scheduler != null) { - this.scheduler.shutdownNow(); - // make sure topics are not being throttled - refreshTask.run(); - this.scheduler = null; - this.tickTimeMs = 0; - } - } - - @VisibleForTesting - protected synchronized long getTickTimeMs() { - return tickTimeMs; - } - } - public void close() throws IOException { try { closeAsync().get(); @@ -766,7 +775,7 @@ public CompletableFuture closeAndRemoveReplicationClient(String clusterNam if (ot.isPresent()) { Replicator r = ot.get().getReplicators().get(clusterName); if (r != null && r.isConnected()) { - r.disconnect(false).whenComplete((v, e) -> f.complete(null)); + r.terminate().whenComplete((v, e) -> f.complete(null)); return; } } @@ -821,6 +830,7 @@ public CompletableFuture closeAsync() { for (EventLoopGroup group : protocolHandlersWorkerGroups) { shutdownEventLoops.add(shutdownEventLoopGracefully(group)); } + CompletableFuture shutdownFuture = CompletableFuture.allOf(shutdownEventLoops.toArray(new CompletableFuture[0])) .handle((v, t) -> { @@ -831,7 +841,7 @@ public CompletableFuture closeAsync() { } return null; }) - .thenCompose(__ -> { + .thenComposeAsync(__ -> { log.info("Continuing to second phase in shutdown."); List> asyncCloseFutures = new ArrayList<>(); @@ -852,8 +862,13 @@ public CompletableFuture closeAsync() { log.warn("Error in closing authenticationService", e); } pulsarStats.close(); + pendingTopicLoadOperationsCounter.close(); + pendingLookupOperationsCounter.close(); try { delayedDeliveryTrackerFactory.close(); + if (fallbackDelayedDeliveryTrackerFactory != null) { + fallbackDelayedDeliveryTrackerFactory.close(); + } } catch (Exception e) { log.warn("Error in closing delayedDeliveryTrackerFactory", e); } @@ -873,8 +888,6 @@ public CompletableFuture closeAsync() { consumedLedgersMonitor, backlogQuotaChecker, topicOrderedExecutor, - topicPublishRateLimiterMonitor.scheduler, - brokerPublishRateLimiterMonitor.scheduler, deduplicationSnapshotMonitor) .handle()); @@ -895,6 +908,12 @@ public CompletableFuture closeAsync() { return null; }); return combined; + }, runnable -> { + // run the 2nd phase of the shutdown in a separate thread + Thread thread = new Thread(runnable); + thread.setName("BrokerService-shutdown-phase2"); + thread.setDaemon(false); + thread.start(); }); FutureUtil.whenCancelledOrTimedOut(shutdownFuture, () -> cancellableDownstreamFutureReference .thenAccept(future -> future.cancel(false))); @@ -912,7 +931,7 @@ CompletableFuture shutdownEventLoopGracefully(EventLoopGroup eventLoopGrou long timeout = (long) (GRACEFUL_SHUTDOWN_TIMEOUT_RATIO_OF_TOTAL_TIMEOUT * brokerShutdownTimeoutMs); return NettyFutureUtil.toCompletableFutureVoid( eventLoopGroup.shutdownGracefully(quietPeriod, - timeout, TimeUnit.MILLISECONDS)); + timeout, MILLISECONDS)); } private CompletableFuture closeChannel(Channel channel) { @@ -941,9 +960,13 @@ public void unloadNamespaceBundlesGracefully() { } public void unloadNamespaceBundlesGracefully(int maxConcurrentUnload, boolean closeWithoutWaitingClientDisconnect) { + if (unloaded) { + return; + } try { log.info("Unloading namespace-bundles..."); // make broker-node unavailable from the cluster + long disableBrokerStartTime = System.nanoTime(); if (pulsar.getLoadManager() != null && pulsar.getLoadManager().get() != null) { try { pulsar.getLoadManager().get().disableBroker(); @@ -952,32 +975,36 @@ public void unloadNamespaceBundlesGracefully(int maxConcurrentUnload, boolean cl // still continue and release bundle ownership as broker's registration node doesn't exist. } } + double disableBrokerTimeSeconds = + TimeUnit.NANOSECONDS.toMillis((System.nanoTime() - disableBrokerStartTime)) + / 1000.0; + log.info("Disable broker in load manager completed in {} seconds", disableBrokerTimeSeconds); // unload all namespace-bundles gracefully long closeTopicsStartTime = System.nanoTime(); Set serviceUnits = pulsar.getNamespaceService() != null ? pulsar.getNamespaceService().getOwnedServiceUnits() : null; if (serviceUnits != null) { - try (RateLimiter rateLimiter = maxConcurrentUnload > 0 ? RateLimiter.builder() - .scheduledExecutorService(pulsar.getExecutor()) - .rateTime(1).timeUnit(TimeUnit.SECONDS) - .permits(maxConcurrentUnload).build() : null) { - serviceUnits.forEach(su -> { - if (su != null) { - try { - if (rateLimiter != null) { - rateLimiter.acquire(1); - } - long timeout = pulsar.getConfiguration().getNamespaceBundleUnloadingTimeoutMs(); - pulsar.getNamespaceService().unloadNamespaceBundle(su, timeout, TimeUnit.MILLISECONDS, - closeWithoutWaitingClientDisconnect).get(timeout, TimeUnit.MILLISECONDS); - } catch (Exception e) { + RateLimiter rateLimiter = maxConcurrentUnload > 0 ? RateLimiter.create(maxConcurrentUnload) : null; + serviceUnits.forEach(su -> { + if (su != null) { + try { + if (rateLimiter != null) { + rateLimiter.acquire(1); + } + long timeout = pulsar.getConfiguration().getNamespaceBundleUnloadingTimeoutMs(); + pulsar.getNamespaceService().unloadNamespaceBundle(su, timeout, MILLISECONDS, + closeWithoutWaitingClientDisconnect).get(timeout, MILLISECONDS); + } catch (Exception e) { + if (e instanceof ExecutionException + && e.getCause() instanceof ServiceUnitNotReadyException) { + log.warn("Failed to unload namespace bundle {}: {}", su, e.getMessage()); + } else { log.warn("Failed to unload namespace bundle {}", su, e); } } - }); - } - + } + }); double closeTopicsTimeSeconds = TimeUnit.NANOSECONDS.toMillis((System.nanoTime() - closeTopicsStartTime)) / 1000.0; @@ -986,6 +1013,8 @@ public void unloadNamespaceBundlesGracefully(int maxConcurrentUnload, boolean cl } } catch (Exception e) { log.error("Failed to disable broker from loadbalancer list {}", e.getMessage(), e); + } finally { + unloaded = true; } } @@ -1008,59 +1037,94 @@ public CompletableFuture> getTopic(final String topic, boolean c return getTopic(TopicName.get(topic), createIfMissing, properties); } + /** + * Retrieves or creates a topic based on the specified parameters. + * 0. If disable PersistentTopics or NonPersistentTopics, it will return a failed future with NotAllowedException. + * 1. If topic future exists in the cache returned directly regardless of whether it fails or timeout. + * 2. If the topic metadata exists, the topic is created regardless of {@code createIfMissing}. + * 3. If the topic metadata not exists, and {@code createIfMissing} is false, + * returns an empty Optional in a CompletableFuture. And this empty future not be added to the map. + * 4. Otherwise, use computeIfAbsent. It returns the existing topic or creates and adds a new topicFuture. + * Any exceptions will remove the topicFuture from the map. + * + * @param topicName The name of the topic, potentially including partition information. + * @param createIfMissing If true, creates the topic if it does not exist. + * @param properties Topic configuration properties used during creation. + * @return CompletableFuture with an Optional of the topic if found or created, otherwise empty. + */ public CompletableFuture> getTopic(final TopicName topicName, boolean createIfMissing, Map properties) { try { - CompletableFuture> topicFuture = topics.get(topicName.toString()); - if (topicFuture != null) { - if (topicFuture.isCompletedExceptionally() - || (topicFuture.isDone() && !topicFuture.getNow(Optional.empty()).isPresent())) { - // Exceptional topics should be recreated. - topics.remove(topicName.toString(), topicFuture); - } else { - // a non-existing topic in the cache shouldn't prevent creating a topic - if (createIfMissing) { - if (topicFuture.isDone() && topicFuture.getNow(Optional.empty()).isPresent()) { - return topicFuture; - } else { - return topicFuture.thenCompose(value -> { - if (!value.isPresent()) { - // retry and create topic - return getTopic(topicName, createIfMissing, properties); - } else { - // in-progress future completed successfully - return CompletableFuture.completedFuture(value); - } - }); - } - } else { - return topicFuture; - } - } + // If topic future exists in the cache returned directly regardless of whether it fails or timeout. + CompletableFuture> tp = topics.get(topicName.toString()); + if (tp != null) { + return tp; } final boolean isPersistentTopic = topicName.getDomain().equals(TopicDomain.persistent); if (isPersistentTopic) { - return topics.computeIfAbsent(topicName.toString(), (tpName) -> { - if (topicName.isPartitioned()) { - return fetchPartitionedTopicMetadataAsync(TopicName.get(topicName.getPartitionedTopicName())) - .thenCompose((metadata) -> { - // Allow crate non-partitioned persistent topic that name includes `partition` - if (metadata.partitions == 0 - || topicName.getPartitionIndex() < metadata.partitions) { - return loadOrCreatePersistentTopic(tpName, createIfMissing, properties); - } - return CompletableFuture.completedFuture(Optional.empty()); - }); + if (!pulsar.getConfiguration().isEnablePersistentTopics()) { + if (log.isDebugEnabled()) { + log.debug("Broker is unable to load persistent topic {}", topicName); + } + return FutureUtil.failedFuture(new NotAllowedException( + "Broker is unable to load persistent topic")); + } + return pulsar.getPulsarResources().getTopicResources().persistentTopicExists(topicName) + .thenCompose(exists -> { + if (!exists && !createIfMissing) { + return CompletableFuture.completedFuture(Optional.empty()); } - return loadOrCreatePersistentTopic(tpName, createIfMissing, properties); + return getTopicPoliciesBypassSystemTopic(topicName).exceptionally(ex -> { + final Throwable rc = FutureUtil.unwrapCompletionException(ex); + final String errorInfo = String.format("Topic creation encountered an exception by initialize" + + " topic policies service. topic_name=%s error_message=%s", topicName, + rc.getMessage()); + log.error(errorInfo, rc); + throw FutureUtil.wrapToCompletionException(new ServiceUnitNotReadyException(errorInfo)); + }).thenCompose(optionalTopicPolicies -> { + final TopicPolicies topicPolicies = optionalTopicPolicies.orElse(null); + if (topicName.isPartitioned()) { + final TopicName topicNameEntity = TopicName.get(topicName.getPartitionedTopicName()); + return fetchPartitionedTopicMetadataAsync(topicNameEntity) + .thenCompose((metadata) -> { + // Allow crate non-partitioned persistent topic that name includes + // `partition` + if (metadata.partitions == 0 + || topicName.getPartitionIndex() < metadata.partitions) { + return topics.computeIfAbsent(topicName.toString(), (tpName) -> + loadOrCreatePersistentTopic(tpName, + createIfMissing, properties, topicPolicies)); + } else { + final String errorMsg = + String.format("Illegal topic partition name %s with max allowed " + + "%d partitions", topicName, metadata.partitions); + log.warn(errorMsg); + return FutureUtil.failedFuture( + new BrokerServiceException.NotAllowedException(errorMsg)); + } + }); + } else { + return topics.computeIfAbsent(topicName.toString(), (tpName) -> + loadOrCreatePersistentTopic(tpName, createIfMissing, properties, topicPolicies)); + } + }); }); } else { - return topics.computeIfAbsent(topicName.toString(), (name) -> { + if (!pulsar.getConfiguration().isEnableNonPersistentTopics()) { + if (log.isDebugEnabled()) { + log.debug("Broker is unable to load non-persistent topic {}", topicName); + } + return FutureUtil.failedFuture(new NotAllowedException( + "Broker is unable to load persistent topic")); + } + if (!topics.containsKey(topicName.toString())) { topicEventsDispatcher.notify(topicName.toString(), TopicEvent.LOAD, EventStage.BEFORE); - if (topicName.isPartitioned()) { - final TopicName partitionedTopicName = TopicName.get(topicName.getPartitionedTopicName()); - return this.fetchPartitionedTopicMetadataAsync(partitionedTopicName).thenCompose((metadata) -> { - if (topicName.getPartitionIndex() < metadata.partitions) { + } + if (topicName.isPartitioned()) { + final TopicName partitionedTopicName = TopicName.get(topicName.getPartitionedTopicName()); + return this.fetchPartitionedTopicMetadataAsync(partitionedTopicName).thenCompose((metadata) -> { + if (topicName.getPartitionIndex() < metadata.partitions) { + return topics.computeIfAbsent(topicName.toString(), (name) -> { topicEventsDispatcher .notify(topicName.toString(), TopicEvent.CREATE, EventStage.BEFORE); @@ -1071,11 +1135,13 @@ public CompletableFuture> getTopic(final TopicName topicName, bo topicEventsDispatcher .notifyOnCompletion(eventFuture, topicName.toString(), TopicEvent.LOAD); return res; - } - topicEventsDispatcher.notify(topicName.toString(), TopicEvent.LOAD, EventStage.FAILURE); - return CompletableFuture.completedFuture(Optional.empty()); - }); - } else if (createIfMissing) { + }); + } + topicEventsDispatcher.notify(topicName.toString(), TopicEvent.LOAD, EventStage.FAILURE); + return CompletableFuture.completedFuture(Optional.empty()); + }); + } else if (createIfMissing) { + return topics.computeIfAbsent(topicName.toString(), (name) -> { topicEventsDispatcher.notify(topicName.toString(), TopicEvent.CREATE, EventStage.BEFORE); CompletableFuture> res = createNonPersistentTopic(name); @@ -1085,11 +1151,15 @@ public CompletableFuture> getTopic(final TopicName topicName, bo topicEventsDispatcher .notifyOnCompletion(eventFuture, topicName.toString(), TopicEvent.LOAD); return res; - } else { + }); + } else { + CompletableFuture> topicFuture = topics.get(topicName.toString()); + if (topicFuture == null) { topicEventsDispatcher.notify(topicName.toString(), TopicEvent.LOAD, EventStage.FAILURE); - return CompletableFuture.completedFuture(Optional.empty()); + topicFuture = CompletableFuture.completedFuture(Optional.empty()); } - }); + return topicFuture; + } } } catch (IllegalArgumentException e) { log.warn("[{}] Illegalargument exception when loading topic", topicName, e); @@ -1106,6 +1176,14 @@ public CompletableFuture> getTopic(final TopicName topicName, bo } } + private CompletableFuture> getTopicPoliciesBypassSystemTopic(@Nonnull TopicName topicName) { + if (ExtensibleLoadManagerImpl.isInternalTopic(topicName.toString())) { + return CompletableFuture.completedFuture(Optional.empty()); + } + return pulsar.getTopicPoliciesService().getTopicPoliciesAsync(topicName, + TopicPoliciesService.GetType.DEFAULT); + } + public CompletableFuture deleteTopic(String topic, boolean forceDelete) { topicEventsDispatcher.notify(topic, TopicEvent.DELETE, EventStage.BEFORE); CompletableFuture result = deleteTopicInternal(topic, forceDelete); @@ -1126,7 +1204,7 @@ private CompletableFuture deleteTopicInternal(String topic, boolean forceD // v2 topics have a global name so check if the topic is replicated. if (t.isReplicated()) { // Delete is disallowed on global topic - final List clusters = t.getReplicators().keys(); + final var clusters = t.getReplicators().keySet(); log.error("Delete forbidden topic {} is replicated on clusters {}", topic, clusters); return FutureUtil.failedFuture( new IllegalStateException("Delete forbidden topic is replicated on clusters " + clusters)); @@ -1134,7 +1212,7 @@ private CompletableFuture deleteTopicInternal(String topic, boolean forceD // shadow topic should be deleted first. if (t.isShadowReplicated()) { - final List shadowTopics = t.getShadowReplicators().keys(); + final var shadowTopics = t.getShadowReplicators().keySet(); log.error("Delete forbidden. Topic {} is replicated to shadow topics: {}", topic, shadowTopics); return FutureUtil.failedFuture(new IllegalStateException( "Delete forbidden. Topic " + topic + " is replicated to shadow topics.")); @@ -1155,26 +1233,27 @@ private CompletableFuture deleteTopicInternal(String topic, boolean forceD CompletableFuture future = new CompletableFuture<>(); CompletableFuture deleteTopicAuthenticationFuture = new CompletableFuture<>(); deleteTopicAuthenticationWithRetry(topic, deleteTopicAuthenticationFuture, 5); - - deleteTopicAuthenticationFuture.whenComplete((v, ex) -> { + deleteTopicAuthenticationFuture + .thenCompose(__ -> deleteSchema(tn)) + .thenCompose(__ -> pulsar.getTopicPoliciesService().deleteTopicPoliciesAsync(tn)).whenComplete((v, ex) -> { if (ex != null) { future.completeExceptionally(ex); return; } CompletableFuture mlConfigFuture = getManagedLedgerConfig(topicName); managedLedgerFactory.asyncDelete(tn.getPersistenceNamingEncoding(), - mlConfigFuture, new DeleteLedgerCallback() { - @Override - public void deleteLedgerComplete(Object ctx) { - future.complete(null); - } + mlConfigFuture, new DeleteLedgerCallback() { + @Override + public void deleteLedgerComplete(Object ctx) { + future.complete(null); + } - @Override - public void deleteLedgerFailed(ManagedLedgerException exception, Object ctx) { - future.completeExceptionally(exception); - } - }, null); - }); + @Override + public void deleteLedgerFailed(ManagedLedgerException exception, Object ctx) { + future.completeExceptionally(exception); + } + }, null); + }); return future; } @@ -1185,23 +1264,9 @@ public void deleteTopicAuthenticationWithRetry(String topic, CompletableFuture { - if (!optPolicies.isPresent() || !optPolicies.get().auth_policies.getTopicAuthentication() - .containsKey(topic)) { - // if there is no auth policy for the topic, just complete and return - if (log.isDebugEnabled()) { - log.debug("Authentication policies not found for topic {}", topic); - } - future.complete(null); - return; - } - pulsar.getPulsarResources().getNamespaceResources() - .setPoliciesAsync(TopicName.get(topic).getNamespaceObject(), p -> { - p.auth_policies.getTopicAuthentication().remove(topic); - return p; - }).thenAccept(v -> { + authorizationService.removePermissionsAsync(TopicName.get(topic)) + .thenAccept(v -> { log.info("Successfully delete authentication policies for topic {}", topic); future.complete(null); }).exceptionally(ex1 -> { @@ -1217,29 +1282,23 @@ public void deleteTopicAuthenticationWithRetry(String topic, CompletableFuture { - log.error("Failed to get policies for topic {}", topic, ex); - future.completeExceptionally(ex); - return null; - }); } private CompletableFuture> createNonPersistentTopic(String topic) { CompletableFuture> topicFuture = new CompletableFuture<>(); - if (!pulsar.getConfiguration().isEnableNonPersistentTopics()) { - if (log.isDebugEnabled()) { - log.debug("Broker is unable to load non-persistent topic {}", topic); - } - return FutureUtil.failedFuture( - new NotAllowedException("Broker is not unable to load non-persistent topic")); - } + topicFuture.exceptionally(t -> { + pulsarStats.recordTopicLoadFailed(); + pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); + return null; + }); final long topicCreateTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()); NonPersistentTopic nonPersistentTopic; try { nonPersistentTopic = newTopic(topic, null, this, NonPersistentTopic.class); } catch (Throwable e) { log.warn("Failed to create topic {}", topic, e); - return FutureUtil.failedFuture(e); + topicFuture.completeExceptionally(e); + return topicFuture; } CompletableFuture isOwner = checkTopicNsOwnership(topic); isOwner.thenRun(() -> { @@ -1254,7 +1313,6 @@ private CompletableFuture> createNonPersistentTopic(String topic }).exceptionally(ex -> { log.warn("Replication check failed. Removing topic from topics list {}, {}", topic, ex.getCause()); nonPersistentTopic.stopReplProducers().whenComplete((v, exception) -> { - pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); topicFuture.completeExceptionally(ex); }); return null; @@ -1325,7 +1383,9 @@ public PulsarClient getReplicationClient(String cluster, Optional c data.getBrokerClientTrustCertsFilePath(), data.getBrokerClientKeyFilePath(), data.getBrokerClientCertificateFilePath(), - pulsar.getConfiguration().isTlsHostnameVerificationEnabled() + pulsar.getConfiguration().isTlsHostnameVerificationEnabled(), + data.getBrokerClientSslFactoryPlugin(), + data.getBrokerClientSslFactoryPluginParams() ); } else if (pulsar.getConfiguration().isBrokerClientTlsEnabled()) { configTlsSettings(clientBuilder, serviceUrlTls, @@ -1340,7 +1400,9 @@ public PulsarClient getReplicationClient(String cluster, Optional c pulsar.getConfiguration().getBrokerClientTrustCertsFilePath(), pulsar.getConfiguration().getBrokerClientKeyFilePath(), pulsar.getConfiguration().getBrokerClientCertificateFilePath(), - pulsar.getConfiguration().isTlsHostnameVerificationEnabled() + pulsar.getConfiguration().isTlsHostnameVerificationEnabled(), + pulsar.getConfiguration().getBrokerClientSslFactoryPlugin(), + pulsar.getConfiguration().getBrokerClientSslFactoryPluginParams() ); } else { clientBuilder.serviceUrl( @@ -1371,11 +1433,16 @@ private void configTlsSettings(ClientBuilder clientBuilder, String serviceUrl, String brokerClientTlsKeyStore, String brokerClientTlsKeyStorePassword, String brokerClientTrustCertsFilePath, String brokerClientKeyFilePath, String brokerClientCertificateFilePath, - boolean isTlsHostnameVerificationEnabled) { + boolean isTlsHostnameVerificationEnabled, String brokerClientSslFactoryPlugin, + String brokerClientSslFactoryPluginParams) { clientBuilder .serviceUrl(serviceUrl) .allowTlsInsecureConnection(isTlsAllowInsecureConnection) .enableTlsHostnameVerification(isTlsHostnameVerificationEnabled); + if (StringUtils.isNotBlank(brokerClientSslFactoryPlugin)) { + clientBuilder.sslFactoryPlugin(brokerClientSslFactoryPlugin) + .sslFactoryPluginParams(brokerClientSslFactoryPluginParams); + } if (brokerClientTlsEnabledWithKeyStore) { clientBuilder.useKeyStoreTls(true) .tlsTrustStoreType(brokerClientTlsTrustStoreType) @@ -1398,7 +1465,8 @@ private void configAdminTlsSettings(PulsarAdminBuilder adminBuilder, boolean bro String brokerClientTlsKeyStore, String brokerClientTlsKeyStorePassword, String brokerClientTrustCertsFilePath, String brokerClientKeyFilePath, String brokerClientCertificateFilePath, - boolean isTlsHostnameVerificationEnabled) { + boolean isTlsHostnameVerificationEnabled, String brokerClientSslFactoryPlugin, + String brokerClientSslFactoryPluginParams) { if (brokerClientTlsEnabledWithKeyStore) { adminBuilder.useKeyStoreTls(true) .tlsTrustStoreType(brokerClientTlsTrustStoreType) @@ -1413,7 +1481,9 @@ private void configAdminTlsSettings(PulsarAdminBuilder adminBuilder, boolean bro .tlsCertificateFilePath(brokerClientCertificateFilePath); } adminBuilder.allowTlsInsecureConnection(isTlsAllowInsecureConnection) - .enableTlsHostnameVerification(isTlsHostnameVerificationEnabled); + .enableTlsHostnameVerification(isTlsHostnameVerificationEnabled) + .sslFactoryPlugin(brokerClientSslFactoryPlugin) + .sslFactoryPluginParams(brokerClientSslFactoryPluginParams); } public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional clusterDataOp) { @@ -1441,13 +1511,11 @@ public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional c } boolean isTlsEnabled = data.isBrokerClientTlsEnabled() || conf.isBrokerClientTlsEnabled(); - if (isTlsEnabled && StringUtils.isEmpty(data.getServiceUrlTls())) { - throw new IllegalArgumentException("serviceUrlTls is empty, brokerClientTlsEnabled: " + final String adminApiUrl = isTlsEnabled ? data.getServiceUrlTls() : data.getServiceUrl(); + if (StringUtils.isEmpty(adminApiUrl)) { + throw new IllegalArgumentException("The adminApiUrl is empty, brokerClientTlsEnabled: " + isTlsEnabled); - } else if (StringUtils.isEmpty(data.getServiceUrl())) { - throw new IllegalArgumentException("serviceUrl is empty, brokerClientTlsEnabled: " + isTlsEnabled); } - String adminApiUrl = isTlsEnabled ? data.getServiceUrlTls() : data.getServiceUrl(); builder.serviceHttpUrl(adminApiUrl); if (data.isBrokerClientTlsEnabled()) { configAdminTlsSettings(builder, @@ -1462,7 +1530,9 @@ public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional c data.getBrokerClientTrustCertsFilePath(), data.getBrokerClientKeyFilePath(), data.getBrokerClientCertificateFilePath(), - pulsar.getConfiguration().isTlsHostnameVerificationEnabled() + pulsar.getConfiguration().isTlsHostnameVerificationEnabled(), + data.getBrokerClientSslFactoryPlugin(), + data.getBrokerClientSslFactoryPluginParams() ); } else if (conf.isBrokerClientTlsEnabled()) { configAdminTlsSettings(builder, @@ -1477,7 +1547,9 @@ public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional c conf.getBrokerClientTrustCertsFilePath(), conf.getBrokerClientKeyFilePath(), conf.getBrokerClientCertificateFilePath(), - pulsar.getConfiguration().isTlsHostnameVerificationEnabled() + pulsar.getConfiguration().isTlsHostnameVerificationEnabled(), + conf.getBrokerClientSslFactoryPlugin(), + conf.getBrokerClientSslFactoryPluginParams() ); } @@ -1503,38 +1575,45 @@ public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional c * @throws RuntimeException */ protected CompletableFuture> loadOrCreatePersistentTopic(final String topic, - boolean createIfMissing, Map properties) throws RuntimeException { + boolean createIfMissing, Map properties, @Nullable TopicPolicies topicPolicies) { final CompletableFuture> topicFuture = FutureUtil.createFutureWithTimeout( Duration.ofSeconds(pulsar.getConfiguration().getTopicLoadTimeoutSeconds()), executor(), () -> FAILED_TO_LOAD_TOPIC_TIMEOUT_EXCEPTION); - if (!pulsar.getConfiguration().isEnablePersistentTopics()) { - if (log.isDebugEnabled()) { - log.debug("Broker is unable to load persistent topic {}", topic); - } - topicFuture.completeExceptionally(new NotAllowedException( - "Broker is unable to load persistent topic")); - return topicFuture; - } + + topicFuture.exceptionally(t -> { + pulsarStats.recordTopicLoadFailed(); + return null; + }); checkTopicNsOwnership(topic) .thenRun(() -> { final Semaphore topicLoadSemaphore = topicLoadRequestSemaphore.get(); if (topicLoadSemaphore.tryAcquire()) { - checkOwnershipAndCreatePersistentTopic(topic, createIfMissing, topicFuture, properties); + checkOwnershipAndCreatePersistentTopic(topic, createIfMissing, topicFuture, + properties, topicPolicies); topicFuture.handle((persistentTopic, ex) -> { // release permit and process pending topic topicLoadSemaphore.release(); + // do not recreate topic if topic is already migrated and deleted by broker + // so, avoid creating a new topic if migration is already started + if (ex != null && (ex.getCause() instanceof TopicMigratedException)) { + pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); + topicFuture.completeExceptionally(ex.getCause()); + return null; + } createPendingLoadTopic(); return null; }); } else { - pendingTopicLoadingQueue.add(new TopicLoadingContext(topic, topicFuture, properties)); + pendingTopicLoadingQueue.add(new TopicLoadingContext(topic, + createIfMissing, topicFuture, properties, topicPolicies)); if (log.isDebugEnabled()) { log.debug("topic-loading for {} added into pending queue", topic); } } }).exceptionally(ex -> { + pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); topicFuture.completeExceptionally(ex.getCause()); return null; }); @@ -1570,7 +1649,7 @@ protected CompletableFuture> fetchTopicPropertiesAsync(Topic private void checkOwnershipAndCreatePersistentTopic(final String topic, boolean createIfMissing, CompletableFuture> topicFuture, - Map properties) { + Map properties, @Nullable TopicPolicies topicPolicies) { TopicName topicName = TopicName.get(topic); pulsar.getNamespaceService().isServiceUnitActiveAsync(topicName) .thenAccept(isActive -> { @@ -1584,7 +1663,8 @@ private void checkOwnershipAndCreatePersistentTopic(final String topic, boolean } propertiesFuture.thenAccept(finalProperties -> //TODO add topicName in properties? - createPersistentTopic(topic, createIfMissing, topicFuture, finalProperties) + createPersistentTopic(topic, createIfMissing, topicFuture, + finalProperties, topicPolicies) ).exceptionally(throwable -> { log.warn("[{}] Read topic property failed", topic, throwable); pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); @@ -1599,6 +1679,7 @@ private void checkOwnershipAndCreatePersistentTopic(final String topic, boolean topicFuture.completeExceptionally(new ServiceUnitNotReadyException(msg)); } }).exceptionally(ex -> { + pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); topicFuture.completeExceptionally(ex); return null; }); @@ -1609,12 +1690,12 @@ private void checkOwnershipAndCreatePersistentTopic(final String topic, boolean public void createPersistentTopic0(final String topic, boolean createIfMissing, CompletableFuture> topicFuture, Map properties) { - createPersistentTopic(topic, createIfMissing, topicFuture, properties); + createPersistentTopic(topic, createIfMissing, topicFuture, properties, null); } private void createPersistentTopic(final String topic, boolean createIfMissing, CompletableFuture> topicFuture, - Map properties) { + Map properties, @Nullable TopicPolicies topicPolicies) { TopicName topicName = TopicName.get(topic); final long topicCreateTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()); @@ -1630,7 +1711,11 @@ private void createPersistentTopic(final String topic, boolean createIfMissing, ? checkMaxTopicsPerNamespace(topicName, 1) : CompletableFuture.completedFuture(null); - maxTopicsCheck.thenCompose(__ -> getManagedLedgerConfig(topicName)).thenAccept(managedLedgerConfig -> { + CompletableFuture isTopicAlreadyMigrated = checkTopicAlreadyMigrated(topicName); + + maxTopicsCheck.thenCompose(__ -> isTopicAlreadyMigrated) + .thenCompose(__ -> getManagedLedgerConfig(topicName, topicPolicies)) + .thenAccept(managedLedgerConfig -> { if (isBrokerEntryMetadataEnabled() || isBrokerPayloadProcessorEnabled()) { // init managedLedger interceptor Set interceptors = new HashSet<>(); @@ -1693,41 +1778,45 @@ public void openLedgerComplete(ManagedLedger ledger, Object ctx) { long topicLoadLatencyMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()) - topicCreateTimeMs; pulsarStats.recordTopicLoadTimeValue(topic, topicLoadLatencyMs); - if (topicFuture.isCompletedExceptionally()) { + if (!topicFuture.complete(Optional.of(persistentTopic))) { // Check create persistent topic timeout. - log.warn("{} future is already completed with failure {}, closing the" - + " topic", topic, FutureUtil.getException(topicFuture)); - persistentTopic.getTransactionBuffer() - .closeAsync() - .exceptionally(t -> { - log.error("[{}] Close transactionBuffer failed", topic, t); - return null; - }); - persistentTopic.stopReplProducers() - .whenCompleteAsync((v, exception) -> { - topics.remove(topic, topicFuture); - }, executor()); + if (topicFuture.isCompletedExceptionally()) { + log.warn("{} future is already completed with failure {}, closing" + + " the topic", topic, FutureUtil.getException(topicFuture)); + } else { + // It should not happen. + log.error("{} future is already completed by another thread, " + + "which is not expected. Closing the current one", topic); + } + executor().submit(() -> { + persistentTopic.close().whenComplete((ignore, ex) -> { + topics.remove(topic, topicFuture); + if (ex != null) { + log.warn("[{}] Get an error when closing topic.", + topic, ex); + } + }); + }); } else { addTopicToStatsMaps(topicName, persistentTopic); - topicFuture.complete(Optional.of(persistentTopic)); } }) .exceptionally((ex) -> { log.warn("Replication or dedup check failed." + " Removing topic from topics list {}, {}", topic, ex); - persistentTopic.getTransactionBuffer() - .closeAsync() - .exceptionally(t -> { - log.error("[{}] Close transactionBuffer failed", topic, t); - return null; - }); - persistentTopic.stopReplProducers().whenCompleteAsync((v, exception) -> { - topics.remove(topic, topicFuture); - topicFuture.completeExceptionally(ex); - }, executor()); + executor().submit(() -> { + persistentTopic.close().whenComplete((ignore, closeEx) -> { + topics.remove(topic, topicFuture); + if (closeEx != null) { + log.warn("[{}] Get an error when closing topic.", + topic, closeEx); + } + topicFuture.completeExceptionally(ex); + }); + }); return null; }); - } catch (PulsarServerException e) { + } catch (Exception e) { log.warn("Failed to create topic {}: {}", topic, e.getMessage()); pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); topicFuture.completeExceptionally(e); @@ -1738,6 +1827,7 @@ public void openLedgerComplete(ManagedLedger ledger, Object ctx) { public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { if (!createIfMissing && exception instanceof ManagedLedgerNotFoundException) { // We were just trying to load a topic and the topic doesn't exist + pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); loadFuture.completeExceptionally(exception); topicFuture.complete(Optional.empty()); } else { @@ -1746,7 +1836,7 @@ public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { topicFuture.completeExceptionally(new PersistenceException(exception)); } } - }, () -> isTopicNsOwnedByBroker(topicName), null); + }, () -> isTopicNsOwnedByBrokerAsync(topicName), null); }).exceptionally((exception) -> { log.warn("[{}] Failed to get topic configuration: {}", topic, exception.getMessage(), exception); @@ -1758,165 +1848,198 @@ public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { }); } - public CompletableFuture getManagedLedgerConfig(TopicName topicName) { + private CompletableFuture checkTopicAlreadyMigrated(TopicName topicName) { + if (ExtensibleLoadManagerImpl.isInternalTopic(topicName.toString())) { + return CompletableFuture.completedFuture(null); + } + CompletableFuture result = new CompletableFuture<>(); + AbstractTopic.isClusterMigrationEnabled(pulsar, topicName.toString()).handle((isMigrated, ex) -> { + if (isMigrated) { + result.completeExceptionally( + new BrokerServiceException.TopicMigratedException(topicName + " already migrated")); + } else { + result.complete(null); + } + return null; + }); + return result; + } + + public CompletableFuture getManagedLedgerConfig(@Nonnull TopicName topicName) { + final CompletableFuture> topicPoliciesFuture = + getTopicPoliciesBypassSystemTopic(topicName); + return topicPoliciesFuture.thenCompose(optionalTopicPolicies -> + getManagedLedgerConfig(topicName, optionalTopicPolicies.orElse(null))); + } + + private CompletableFuture getManagedLedgerConfig(@Nonnull TopicName topicName, + @Nullable TopicPolicies topicPolicies) { + requireNonNull(topicName); NamespaceName namespace = topicName.getNamespaceObject(); ServiceConfiguration serviceConfig = pulsar.getConfiguration(); NamespaceResources nsr = pulsar.getPulsarResources().getNamespaceResources(); LocalPoliciesResources lpr = pulsar.getPulsarResources().getLocalPolicies(); - return nsr.getPoliciesAsync(namespace) - .thenCombine(lpr.getLocalPoliciesAsync(namespace), (policies, localPolicies) -> { - PersistencePolicies persistencePolicies = null; - RetentionPolicies retentionPolicies = null; - OffloadPoliciesImpl topicLevelOffloadPolicies = null; - - if (pulsar.getConfig().isTopicLevelPoliciesEnabled() - && !NamespaceService.isSystemServiceNamespace(namespace.toString())) { - final TopicPolicies topicPolicies = pulsar.getTopicPoliciesService() - .getTopicPoliciesIfExists(topicName); - if (topicPolicies != null) { - persistencePolicies = topicPolicies.getPersistence(); - retentionPolicies = topicPolicies.getRetentionPolicies(); - topicLevelOffloadPolicies = topicPolicies.getOffloadPolicies(); - } - } + final CompletableFuture> nsPolicies = nsr.getPoliciesAsync(namespace); + final CompletableFuture> lcPolicies = lpr.getLocalPoliciesAsync(namespace); + return nsPolicies.thenCombine(lcPolicies, (policies, localPolicies) -> { + PersistencePolicies persistencePolicies = null; + RetentionPolicies retentionPolicies = null; + OffloadPoliciesImpl topicLevelOffloadPolicies = null; + if (topicPolicies != null) { + persistencePolicies = topicPolicies.getPersistence(); + retentionPolicies = topicPolicies.getRetentionPolicies(); + topicLevelOffloadPolicies = topicPolicies.getOffloadPolicies(); + } - if (persistencePolicies == null) { - persistencePolicies = policies.map(p -> p.persistence).orElseGet( - () -> new PersistencePolicies(serviceConfig.getManagedLedgerDefaultEnsembleSize(), - serviceConfig.getManagedLedgerDefaultWriteQuorum(), - serviceConfig.getManagedLedgerDefaultAckQuorum(), - serviceConfig.getManagedLedgerDefaultMarkDeleteRateLimit())); - } + if (persistencePolicies == null) { + persistencePolicies = policies.map(p -> p.persistence).orElseGet( + () -> new PersistencePolicies(serviceConfig.getManagedLedgerDefaultEnsembleSize(), + serviceConfig.getManagedLedgerDefaultWriteQuorum(), + serviceConfig.getManagedLedgerDefaultAckQuorum(), + serviceConfig.getManagedLedgerDefaultMarkDeleteRateLimit())); + } - if (retentionPolicies == null) { - retentionPolicies = policies.map(p -> p.retention_policies).orElseGet( - () -> new RetentionPolicies(serviceConfig.getDefaultRetentionTimeInMinutes(), - serviceConfig.getDefaultRetentionSizeInMB()) - ); + if (retentionPolicies == null) { + if (SystemTopicNames.isSystemTopic(topicName)) { + if (log.isDebugEnabled()) { + log.debug("{} Disable data retention policy for system topic.", topicName); } + retentionPolicies = new RetentionPolicies(0, 0); + } else { + retentionPolicies = policies.map(p -> p.retention_policies).orElseGet( + () -> new RetentionPolicies(serviceConfig.getDefaultRetentionTimeInMinutes(), + serviceConfig.getDefaultRetentionSizeInMB()) + ); + } + } - ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); - managedLedgerConfig.setEnsembleSize(persistencePolicies.getBookkeeperEnsemble()); - managedLedgerConfig.setWriteQuorumSize(persistencePolicies.getBookkeeperWriteQuorum()); - managedLedgerConfig.setAckQuorumSize(persistencePolicies.getBookkeeperAckQuorum()); - - if (serviceConfig.isStrictBookieAffinityEnabled()) { - managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyClassName( - IsolatedBookieEnsemblePlacementPolicy.class); - if (localPolicies.isPresent() && localPolicies.get().bookieAffinityGroup != null) { - Map properties = new HashMap<>(); - properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, - localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupPrimary()); - properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, - localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupSecondary()); - managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); - } else if (isSystemTopic(topicName)) { - Map properties = new HashMap<>(); - properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, "*"); - properties.put(IsolatedBookieEnsemblePlacementPolicy - .SECONDARY_ISOLATION_BOOKIE_GROUPS, "*"); - managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); - } else { - Map properties = new HashMap<>(); - properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, ""); - properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, ""); - managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); - } - } else { - if (localPolicies.isPresent() && localPolicies.get().bookieAffinityGroup != null) { - managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyClassName( - IsolatedBookieEnsemblePlacementPolicy.class); - Map properties = new HashMap<>(); - properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, - localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupPrimary()); - properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, - localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupSecondary()); - managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); - } - } + ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); + managedLedgerConfig.setEnsembleSize(persistencePolicies.getBookkeeperEnsemble()); + managedLedgerConfig.setWriteQuorumSize(persistencePolicies.getBookkeeperWriteQuorum()); + managedLedgerConfig.setAckQuorumSize(persistencePolicies.getBookkeeperAckQuorum()); + + if (serviceConfig.isStrictBookieAffinityEnabled()) { + managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyClassName( + IsolatedBookieEnsemblePlacementPolicy.class); + if (localPolicies.isPresent() && localPolicies.get().bookieAffinityGroup != null) { + Map properties = new HashMap<>(); + properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, + localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupPrimary()); + properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, + localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupSecondary()); + managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); + } else if (isSystemTopic(topicName)) { + Map properties = new HashMap<>(); + properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, "*"); + properties.put(IsolatedBookieEnsemblePlacementPolicy + .SECONDARY_ISOLATION_BOOKIE_GROUPS, "*"); + managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); + } else { + Map properties = new HashMap<>(); + properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, ""); + properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, ""); + managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); + } + } else { + if (localPolicies.isPresent() && localPolicies.get().bookieAffinityGroup != null) { + managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyClassName( + IsolatedBookieEnsemblePlacementPolicy.class); + Map properties = new HashMap<>(); + properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, + localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupPrimary()); + properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, + localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupSecondary()); + managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); + } + } - managedLedgerConfig.setThrottleMarkDelete(persistencePolicies.getManagedLedgerMaxMarkDeleteRate()); - managedLedgerConfig.setDigestType(serviceConfig.getManagedLedgerDigestType()); - managedLedgerConfig.setPassword(serviceConfig.getManagedLedgerPassword()); - - managedLedgerConfig - .setMaxUnackedRangesToPersist(serviceConfig.getManagedLedgerMaxUnackedRangesToPersist()); - managedLedgerConfig.setPersistentUnackedRangesWithMultipleEntriesEnabled( - serviceConfig.isPersistentUnackedRangesWithMultipleEntriesEnabled()); - managedLedgerConfig.setMaxUnackedRangesToPersistInMetadataStore( - serviceConfig.getManagedLedgerMaxUnackedRangesToPersistInMetadataStore()); - managedLedgerConfig.setMaxEntriesPerLedger(serviceConfig.getManagedLedgerMaxEntriesPerLedger()); - managedLedgerConfig - .setMinimumRolloverTime(serviceConfig.getManagedLedgerMinLedgerRolloverTimeMinutes(), - TimeUnit.MINUTES); - managedLedgerConfig - .setMaximumRolloverTime(serviceConfig.getManagedLedgerMaxLedgerRolloverTimeMinutes(), - TimeUnit.MINUTES); - managedLedgerConfig.setMaxSizePerLedgerMb(serviceConfig.getManagedLedgerMaxSizePerLedgerMbytes()); - - managedLedgerConfig.setMetadataOperationsTimeoutSeconds( - serviceConfig.getManagedLedgerMetadataOperationsTimeoutSeconds()); - managedLedgerConfig - .setReadEntryTimeoutSeconds(serviceConfig.getManagedLedgerReadEntryTimeoutSeconds()); - managedLedgerConfig - .setAddEntryTimeoutSeconds(serviceConfig.getManagedLedgerAddEntryTimeoutSeconds()); - managedLedgerConfig.setMetadataEnsembleSize(serviceConfig.getManagedLedgerDefaultEnsembleSize()); - managedLedgerConfig.setUnackedRangesOpenCacheSetEnabled( - serviceConfig.isManagedLedgerUnackedRangesOpenCacheSetEnabled()); - managedLedgerConfig.setMetadataWriteQuorumSize(serviceConfig.getManagedLedgerDefaultWriteQuorum()); - managedLedgerConfig.setMetadataAckQuorumSize(serviceConfig.getManagedLedgerDefaultAckQuorum()); - managedLedgerConfig - .setMetadataMaxEntriesPerLedger(serviceConfig.getManagedLedgerCursorMaxEntriesPerLedger()); - - managedLedgerConfig - .setLedgerRolloverTimeout(serviceConfig.getManagedLedgerCursorRolloverTimeInSeconds()); - managedLedgerConfig - .setRetentionTime(retentionPolicies.getRetentionTimeInMinutes(), TimeUnit.MINUTES); - managedLedgerConfig.setRetentionSizeInMB(retentionPolicies.getRetentionSizeInMB()); - managedLedgerConfig.setAutoSkipNonRecoverableData(serviceConfig.isAutoSkipNonRecoverableData()); - managedLedgerConfig.setLazyCursorRecovery(serviceConfig.isLazyCursorRecovery()); - managedLedgerConfig.setInactiveLedgerRollOverTime( - serviceConfig.getManagedLedgerInactiveLedgerRolloverTimeSeconds(), TimeUnit.SECONDS); - managedLedgerConfig.setCacheEvictionByMarkDeletedPosition( - serviceConfig.isCacheEvictionByMarkDeletedPosition()); - managedLedgerConfig.setMinimumBacklogCursorsForCaching( - serviceConfig.getManagedLedgerMinimumBacklogCursorsForCaching()); - managedLedgerConfig.setMinimumBacklogEntriesForCaching( - serviceConfig.getManagedLedgerMinimumBacklogEntriesForCaching()); - managedLedgerConfig.setMaxBacklogBetweenCursorsForCaching( - serviceConfig.getManagedLedgerMaxBacklogBetweenCursorsForCaching()); - - OffloadPoliciesImpl nsLevelOffloadPolicies = - (OffloadPoliciesImpl) policies.map(p -> p.offload_policies).orElse(null); - OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.mergeConfiguration( - topicLevelOffloadPolicies, - OffloadPoliciesImpl.oldPoliciesCompatible(nsLevelOffloadPolicies, policies.orElse(null)), - getPulsar().getConfig().getProperties()); - if (NamespaceService.isSystemServiceNamespace(namespace.toString())) { - managedLedgerConfig.setLedgerOffloader(NullLedgerOffloader.INSTANCE); - } else { - if (topicLevelOffloadPolicies != null) { - try { - LedgerOffloader topicLevelLedgerOffLoader = - pulsar().createManagedLedgerOffloader(offloadPolicies); - managedLedgerConfig.setLedgerOffloader(topicLevelLedgerOffLoader); - } catch (PulsarServerException e) { - throw new RuntimeException(e); - } - } else { - //If the topic level policy is null, use the namespace level - managedLedgerConfig - .setLedgerOffloader(pulsar.getManagedLedgerOffloader(namespace, offloadPolicies)); - } - } + managedLedgerConfig.setThrottleMarkDelete(persistencePolicies.getManagedLedgerMaxMarkDeleteRate()); + managedLedgerConfig.setDigestType(serviceConfig.getManagedLedgerDigestType()); + managedLedgerConfig.setPassword(serviceConfig.getManagedLedgerPassword()); + + managedLedgerConfig + .setMaxUnackedRangesToPersist(serviceConfig.getManagedLedgerMaxUnackedRangesToPersist()); + managedLedgerConfig.setPersistentUnackedRangesWithMultipleEntriesEnabled( + serviceConfig.isPersistentUnackedRangesWithMultipleEntriesEnabled()); + managedLedgerConfig.setMaxUnackedRangesToPersistInMetadataStore( + serviceConfig.getManagedLedgerMaxUnackedRangesToPersistInMetadataStore()); + managedLedgerConfig.setMaxEntriesPerLedger(serviceConfig.getManagedLedgerMaxEntriesPerLedger()); + managedLedgerConfig + .setMinimumRolloverTime(serviceConfig.getManagedLedgerMinLedgerRolloverTimeMinutes(), + TimeUnit.MINUTES); + managedLedgerConfig + .setMaximumRolloverTime(serviceConfig.getManagedLedgerMaxLedgerRolloverTimeMinutes(), + TimeUnit.MINUTES); + managedLedgerConfig.setMaxSizePerLedgerMb(serviceConfig.getManagedLedgerMaxSizePerLedgerMbytes()); + + managedLedgerConfig.setMetadataOperationsTimeoutSeconds( + serviceConfig.getManagedLedgerMetadataOperationsTimeoutSeconds()); + managedLedgerConfig + .setReadEntryTimeoutSeconds(serviceConfig.getManagedLedgerReadEntryTimeoutSeconds()); + managedLedgerConfig + .setAddEntryTimeoutSeconds(serviceConfig.getManagedLedgerAddEntryTimeoutSeconds()); + managedLedgerConfig.setMetadataEnsembleSize(serviceConfig.getManagedLedgerDefaultEnsembleSize()); + managedLedgerConfig.setUnackedRangesOpenCacheSetEnabled( + serviceConfig.isManagedLedgerUnackedRangesOpenCacheSetEnabled()); + managedLedgerConfig.setMetadataWriteQuorumSize(serviceConfig.getManagedLedgerDefaultWriteQuorum()); + managedLedgerConfig.setMetadataAckQuorumSize(serviceConfig.getManagedLedgerDefaultAckQuorum()); + managedLedgerConfig + .setMetadataMaxEntriesPerLedger(serviceConfig.getManagedLedgerCursorMaxEntriesPerLedger()); + + managedLedgerConfig + .setLedgerRolloverTimeout(serviceConfig.getManagedLedgerCursorRolloverTimeInSeconds()); + managedLedgerConfig + .setRetentionTime(retentionPolicies.getRetentionTimeInMinutes(), TimeUnit.MINUTES); + managedLedgerConfig.setRetentionSizeInMB(retentionPolicies.getRetentionSizeInMB()); + managedLedgerConfig.setAutoSkipNonRecoverableData(serviceConfig.isAutoSkipNonRecoverableData()); + managedLedgerConfig.setLedgerForceRecovery(serviceConfig.isManagedLedgerForceRecovery()); + managedLedgerConfig.setLazyCursorRecovery(serviceConfig.isLazyCursorRecovery()); + managedLedgerConfig.setInactiveLedgerRollOverTime( + serviceConfig.getManagedLedgerInactiveLedgerRolloverTimeSeconds(), TimeUnit.SECONDS); + managedLedgerConfig.setCacheEvictionByMarkDeletedPosition( + serviceConfig.isCacheEvictionByMarkDeletedPosition()); + managedLedgerConfig.setMinimumBacklogCursorsForCaching( + serviceConfig.getManagedLedgerMinimumBacklogCursorsForCaching()); + managedLedgerConfig.setMinimumBacklogEntriesForCaching( + serviceConfig.getManagedLedgerMinimumBacklogEntriesForCaching()); + managedLedgerConfig.setMaxBacklogBetweenCursorsForCaching( + serviceConfig.getManagedLedgerMaxBacklogBetweenCursorsForCaching()); + + OffloadPoliciesImpl nsLevelOffloadPolicies = + (OffloadPoliciesImpl) policies.map(p -> p.offload_policies).orElse(null); + OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.mergeConfiguration( + topicLevelOffloadPolicies, + OffloadPoliciesImpl.oldPoliciesCompatible(nsLevelOffloadPolicies, policies.orElse(null)), + getPulsar().getConfig().getProperties()); + if (topicLevelOffloadPolicies != null) { + try { + LedgerOffloader topicLevelLedgerOffLoader = pulsar().createManagedLedgerOffloader(offloadPolicies); + managedLedgerConfig.setLedgerOffloader(topicLevelLedgerOffLoader); + } catch (PulsarServerException e) { + throw new RuntimeException(e); + } + } else { + //If the topic level policy is null, use the namespace level + managedLedgerConfig + .setLedgerOffloader(pulsar.getManagedLedgerOffloader(namespace, offloadPolicies)); + } + if (managedLedgerConfig.getLedgerOffloader() != null + && managedLedgerConfig.getLedgerOffloader().isAppendable() + && (NamespaceService.isSystemServiceNamespace(namespace.toString()) + || SystemTopicNames.isSystemTopic(topicName))) { + managedLedgerConfig.setLedgerOffloader( + new NonAppendableLedgerOffloader(managedLedgerConfig.getLedgerOffloader())); + } - managedLedgerConfig.setDeletionAtBatchIndexLevelEnabled( - serviceConfig.isAcknowledgmentAtBatchIndexLevelEnabled()); - managedLedgerConfig.setNewEntriesCheckDelayInMillis( - serviceConfig.getManagedLedgerNewEntriesCheckDelayInMillis()); - return managedLedgerConfig; - }); + managedLedgerConfig.setTriggerOffloadOnTopicLoad(serviceConfig.isTriggerOffloadOnTopicLoad()); + + managedLedgerConfig.setDeletionAtBatchIndexLevelEnabled( + serviceConfig.isAcknowledgmentAtBatchIndexLevelEnabled()); + managedLedgerConfig.setNewEntriesCheckDelayInMillis( + serviceConfig.getManagedLedgerNewEntriesCheckDelayInMillis()); + return managedLedgerConfig; + }); } private void addTopicToStatsMaps(TopicName topicName, Topic topic) { @@ -1925,14 +2048,10 @@ private void addTopicToStatsMaps(TopicName topicName, Topic topic) { if (namespaceBundle != null) { synchronized (multiLayerTopicsMap) { String serviceUnit = namespaceBundle.toString(); - multiLayerTopicsMap // - .computeIfAbsent(topicName.getNamespace(), - k -> ConcurrentOpenHashMap.>newBuilder() - .build()) // - .computeIfAbsent(serviceUnit, - k -> ConcurrentOpenHashMap.newBuilder().build()) // - .put(topicName.toString(), topic); + multiLayerTopicsMap.computeIfAbsent(topicName.getNamespace(), + __ -> new ConcurrentHashMap<>() + ).computeIfAbsent(serviceUnit, __ -> new ConcurrentHashMap<>() + ).put(topicName.toString(), topic); } } invalidateOfflineTopicStatCache(topicName); @@ -1944,7 +2063,7 @@ private void addTopicToStatsMaps(TopicName topicName, Topic topic) { } public void refreshTopicToStatsMaps(NamespaceBundle oldBundle) { - Objects.requireNonNull(oldBundle); + requireNonNull(oldBundle); try { // retrieve all topics under existing old bundle List topics = getAllTopicsFromNamespaceBundle(oldBundle.getNamespaceObject().toString(), @@ -2048,6 +2167,7 @@ private void checkConsumedLedgers() { Optional.ofNullable(((PersistentTopic) t).getManagedLedger()).ifPresent( managedLedger -> { managedLedger.trimConsumedLedgersInBackground(Futures.NULL_PROMISE); + managedLedger.rolloverCursorsInBackground(); } ); } @@ -2062,26 +2182,6 @@ public void checkInactiveSubscriptions() { forEachTopic(Topic::checkInactiveSubscriptions); } - public void checkTopicPublishThrottlingRate() { - forEachTopic(Topic::checkTopicPublishThrottlingRate); - } - - private void refreshTopicPublishRate() { - forEachTopic(Topic::resetTopicPublishCountAndEnableReadIfRequired); - } - - public void checkBrokerPublishThrottlingRate() { - brokerPublishRateLimiter.checkPublishRate(); - if (brokerPublishRateLimiter.isPublishRateExceeded()) { - forEachTopic(topic -> ((AbstractTopic) topic).disableProducerRead()); - } - } - - private void refreshBrokerPublishRate() { - boolean doneReset = brokerPublishRateLimiter.resetPublishCount(); - forEachTopic(topic -> topic.resetBrokerPublishCountAndEnableReadIfRequired(doneReset)); - } - /** * Iterates over all loaded topics in the broker. */ @@ -2103,6 +2203,7 @@ public BacklogQuotaManager getBacklogQuotaManager() { } public void monitorBacklogQuota() { + long startTimeMillis = System.currentTimeMillis(); forEachPersistentTopic(topic -> { if (topic.isSizeBacklogExceeded()) { getBacklogQuotaManager().handleExceededBacklogQuota(topic, @@ -2122,18 +2223,24 @@ public void monitorBacklogQuota() { log.error("Error when checkTimeBacklogExceeded({}) in monitorBacklogQuota", topic.getName(), throwable); return null; + }).whenComplete((unused, throwable) -> { + backlogQuotaCheckDuration.observe( + MILLISECONDS.toSeconds(System.currentTimeMillis() - startTimeMillis)); }); } }); } - public boolean isTopicNsOwnedByBroker(TopicName topicName) { - try { - return pulsar.getNamespaceService().isServiceUnitOwned(topicName); - } catch (Exception e) { - log.warn("Failed to check the ownership of the topic: {}, {}", topicName, e.getMessage()); - } - return false; + public CompletableFuture isTopicNsOwnedByBrokerAsync(TopicName topicName) { + return pulsar.getNamespaceService().isServiceUnitOwnedAsync(topicName) + .handle((hasOwnership, t) -> { + if (t == null) { + return hasOwnership; + } else { + log.warn("Failed to check the ownership of the topic: {}, {}", topicName, t.getMessage()); + return false; + } + }); } public CompletableFuture checkTopicNsOwnership(final String topic) { @@ -2146,7 +2253,7 @@ public CompletableFuture checkTopicNsOwnership(final String topic) { } else { String msg = String.format("Namespace bundle for topic (%s) not served by this instance:%s. " + "Please redo the lookup. Request is denied: namespace=%s", - topic, pulsar.getLookupServiceAddress(), topicName.getNamespace()); + topic, pulsar.getBrokerId(), topicName.getNamespace()); log.warn(msg); return FutureUtil.failedFuture(new ServiceUnitNotReadyException(msg)); } @@ -2154,8 +2261,10 @@ public CompletableFuture checkTopicNsOwnership(final String topic) { } public CompletableFuture unloadServiceUnit(NamespaceBundle serviceUnit, + boolean disconnectClients, boolean closeWithoutWaitingClientDisconnect, long timeout, TimeUnit unit) { - CompletableFuture future = unloadServiceUnit(serviceUnit, closeWithoutWaitingClientDisconnect); + CompletableFuture future = unloadServiceUnit( + serviceUnit, disconnectClients, closeWithoutWaitingClientDisconnect); ScheduledFuture taskTimeout = executor().schedule(() -> { if (!future.isDone()) { log.warn("Unloading of {} has timed out", serviceUnit); @@ -2172,23 +2281,59 @@ public CompletableFuture unloadServiceUnit(NamespaceBundle serviceUnit, * Unload all the topic served by the broker service under the given service unit. * * @param serviceUnit + * @param disconnectClients disconnect clients * @param closeWithoutWaitingClientDisconnect don't wait for clients to disconnect * and forcefully close managed-ledger * @return */ private CompletableFuture unloadServiceUnit(NamespaceBundle serviceUnit, + boolean disconnectClients, boolean closeWithoutWaitingClientDisconnect) { List> closeFutures = new ArrayList<>(); topics.forEach((name, topicFuture) -> { TopicName topicName = TopicName.get(name); if (serviceUnit.includes(topicName)) { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar) + && ExtensibleLoadManagerImpl.isInternalTopic(topicName.toString())) { + if (ExtensibleLoadManagerImpl.debug(pulsar.getConfiguration(), log)) { + log.info("[{}] Skip unloading ExtensibleLoadManager internal topics. Such internal topic " + + "should be closed when shutting down the broker.", topicName); + } + return; + } + // Topic needs to be unloaded log.info("[{}] Unloading topic", topicName); + if (topicFuture.isCompletedExceptionally()) { + try { + topicFuture.get(); + } catch (InterruptedException | ExecutionException ex) { + if (ex.getCause() instanceof ServiceUnitNotReadyException) { + // Topic was already unloaded + if (log.isDebugEnabled()) { + log.debug("[{}] Topic was already unloaded", topicName); + } + return; + } else { + log.warn("[{}] Got exception when closing topic", topicName, ex); + } + } + } closeFutures.add(topicFuture - .thenCompose(t -> t.isPresent() ? t.get().close(closeWithoutWaitingClientDisconnect) - : CompletableFuture.completedFuture(null))); + .thenCompose(t -> t.isPresent() ? t.get().close( + disconnectClients, closeWithoutWaitingClientDisconnect) + : CompletableFuture.completedFuture(null)) + .exceptionally(e -> { + if (e.getCause() instanceof BrokerServiceException.ServiceUnitNotReadyException + && e.getMessage().contains("Please redo the lookup")) { + log.warn("[{}] Topic ownership check failed. Skipping it", topicName); + return null; + } + throw FutureUtil.wrapToCompletionException(e); + })); } }); + if (getPulsar().getConfig().isTransactionCoordinatorEnabled() && serviceUnit.getNamespaceObject().equals(NamespaceName.SYSTEM_NAMESPACE)) { TransactionMetadataStoreService metadataStoreService = @@ -2205,7 +2350,7 @@ private CompletableFuture unloadServiceUnit(NamespaceBundle serviceUnit } public void cleanUnloadedTopicFromCache(NamespaceBundle serviceUnit) { - for (String topic : topics.keys()) { + for (String topic : topics.keySet()) { TopicName topicName = TopicName.get(topic); if (serviceUnit.includes(topicName) && getTopicReference(topic).isPresent()) { log.info("[{}][{}] Clean unloaded topic from cache.", serviceUnit.toString(), topic); @@ -2218,10 +2363,6 @@ public AuthorizationService getAuthorizationService() { return authorizationService; } - public CompletableFuture removeTopicFromCache(String topicName) { - return removeTopicFutureFromCache(topicName, null); - } - public CompletableFuture removeTopicFromCache(Topic topic) { Optional>> createTopicFuture = findTopicFutureInCache(topic); if (createTopicFuture.isEmpty()){ @@ -2274,10 +2415,9 @@ private void removeTopicFromCache(String topic, NamespaceBundle namespaceBundle, topicEventsDispatcher.notify(topic, TopicEvent.UNLOAD, EventStage.BEFORE); synchronized (multiLayerTopicsMap) { - ConcurrentOpenHashMap> namespaceMap = multiLayerTopicsMap - .get(namespaceName); + final var namespaceMap = multiLayerTopicsMap.get(namespaceName); if (namespaceMap != null) { - ConcurrentOpenHashMap bundleMap = namespaceMap.get(bundleName); + final var bundleMap = namespaceMap.get(bundleName); if (bundleMap != null) { bundleMap.remove(topic); if (bundleMap.isEmpty()) { @@ -2310,7 +2450,7 @@ private void removeTopicFromCache(String topic, NamespaceBundle namespaceBundle, topicEventsDispatcher.notify(topic, TopicEvent.UNLOAD, EventStage.SUCCESS); } - public int getNumberOfNamespaceBundles() { + public long getNumberOfNamespaceBundles() { this.numberOfNamespaceBundles = 0; this.multiLayerTopicsMap.forEach((namespaceName, bundles) -> { this.numberOfNamespaceBundles += bundles.size(); @@ -2318,10 +2458,6 @@ public int getNumberOfNamespaceBundles() { return this.numberOfNamespaceBundles; } - public ConcurrentOpenHashMap>> getTopics() { - return topics; - } - private void handleMetadataChanges(Notification n) { if (n.getType() == NotificationType.Modified && NamespaceResources.pathIsFromNamespace(n.getPath())) { @@ -2407,37 +2543,71 @@ private void handleDynamicConfigurationUpdates() { if (dynamicConfigResources != null) { dynamicConfigResources.getDynamicConfigurationAsync() - .thenAccept(optMap -> { - if (!optMap.isPresent()) { - return; + .thenAccept(optMap -> { + // Case some dynamic configs have been removed. + dynamicConfigurationMap.forEach((configKey, fieldWrapper) -> { + boolean configRemoved = optMap.isEmpty() || !optMap.get().containsKey(configKey); + if (fieldWrapper.lastDynamicValue != null && configRemoved) { + configValueChanged(configKey, null); } - Map data = optMap.get(); - data.forEach((configKey, value) -> { - ConfigField configFieldWrapper = dynamicConfigurationMap.get(configKey); - if (configFieldWrapper == null) { - log.warn("{} does not exist in dynamicConfigurationMap, skip this config.", configKey); - return; - } - Field configField = configFieldWrapper.field; - Object newValue = FieldParser.value(data.get(configKey), configField); - if (configField != null) { - Consumer listener = configRegisteredListeners.get(configKey); - try { - Object existingValue = configField.get(pulsar.getConfiguration()); - configField.set(pulsar.getConfiguration(), newValue); - log.info("Successfully updated configuration {}/{}", configKey, - data.get(configKey)); - if (listener != null && !existingValue.equals(newValue)) { - listener.accept(newValue); - } - } catch (Exception e) { - log.error("Failed to update config {}/{}", configKey, newValue); - } - } else { - log.error("Found non-dynamic field in dynamicConfigMap {}/{}", configKey, newValue); - } - }); }); + // Some configs have been changed. + if (!optMap.isPresent()) { + return; + } + Map data = optMap.get(); + data.forEach((configKey, value) -> { + configValueChanged(configKey, value); + }); + }); + } + } + + private void configValueChanged(String configKey, String newValueStr) { + ConfigField configFieldWrapper = dynamicConfigurationMap.get(configKey); + if (configFieldWrapper == null) { + log.warn("{} does not exist in dynamicConfigurationMap, skip this config.", configKey); + return; + } + Consumer listener = configRegisteredListeners.get(configKey); + try { + // Convert existingValue and newValue. + final Object existingValue; + final Object newValue; + if (configFieldWrapper.field != null) { + if (StringUtils.isBlank(newValueStr)) { + newValue = configFieldWrapper.defaultValue; + } else { + newValue = FieldParser.value(newValueStr, configFieldWrapper.field); + } + existingValue = configFieldWrapper.field.get(pulsar.getConfiguration()); + configFieldWrapper.field.set(pulsar.getConfiguration(), newValue); + } else { + // This case only occurs when it is a customized item. + // See: https://github.com/apache/pulsar/blob/master/pip/pip-300.md. + log.info("Skip update customized dynamic configuration {}/{} in memory, only trigger an event" + + " listeners.", configKey, newValueStr); + existingValue = configFieldWrapper.lastDynamicValue; + newValue = newValueStr == null ? configFieldWrapper.defaultValue : newValueStr; + } + // Record the latest dynamic config. + configFieldWrapper.lastDynamicValue = newValueStr; + + if (newValueStr == null) { + log.info("Successfully remove the dynamic configuration {}, and revert to the default value", + configKey); + } else { + log.info("Successfully updated configuration {}/{}", configKey, newValueStr); + } + + if (listener != null && !Objects.equals(existingValue, newValue)) { + // So far, all config items that related to configuration listeners, their default value is not null. + // And the customized config can be null before. + // So call "listener.accept(null)" is okay. + listener.accept(newValue); + } + } catch (Exception e) { + log.error("Failed to update config {}", configKey, e); } } @@ -2482,10 +2652,6 @@ public EventLoopGroup executor() { return workerGroup; } - public ConcurrentOpenHashMap getReplicationClients() { - return replicationClients; - } - public boolean isAuthenticationEnabled() { return pulsar.getConfiguration().isAuthenticationEnabled(); } @@ -2515,17 +2681,17 @@ public AuthenticationService getAuthenticationService() { } public List getAllTopicsFromNamespaceBundle(String namespace, String bundle) { - ConcurrentOpenHashMap> map1 = multiLayerTopicsMap.get(namespace); + final var map1 = multiLayerTopicsMap.get(namespace); if (map1 == null) { return Collections.emptyList(); } - ConcurrentOpenHashMap map2 = map1.get(bundle); + final var map2 = map1.get(bundle); if (map2 == null) { return Collections.emptyList(); } - return map2.values(); + return map2.values().stream().toList(); } /** @@ -2556,14 +2722,25 @@ private void updateConfigurationAndRegisterListeners() { new Semaphore((int) maxConcurrentTopicLoadRequest, false))); registerConfigurationListener("loadManagerClassName", className -> { pulsar.getExecutor().execute(() -> { + LoadManager newLoadManager = null; try { - final LoadManager newLoadManager = LoadManager.create(pulsar); + newLoadManager = LoadManager.create(pulsar); log.info("Created load manager: {}", className); pulsar.getLoadManager().get().stop(); newLoadManager.start(); - pulsar.getLoadManager().set(newLoadManager); } catch (Exception ex) { log.warn("Failed to change load manager", ex); + try { + if (newLoadManager != null) { + newLoadManager.stop(); + newLoadManager = null; + } + } catch (PulsarServerException e) { + log.warn("Failed to close created load manager", e); + } + } + if (newLoadManager != null) { + pulsar.getLoadManager().set(newLoadManager); } }); }); @@ -2584,7 +2761,7 @@ private void updateConfigurationAndRegisterListeners() { // add listener to notify broker managedLedgerCacheEvictionTimeThresholdMillis dynamic config registerConfigurationListener( "managedLedgerCacheEvictionTimeThresholdMillis", (cacheEvictionTimeThresholdMills) -> { - managedLedgerFactory.updateCacheEvictionTimeThreshold(TimeUnit.MILLISECONDS + managedLedgerFactory.updateCacheEvictionTimeThreshold(MILLISECONDS .toNanos((long) cacheEvictionTimeThresholdMills)); }); @@ -2609,6 +2786,10 @@ private void updateConfigurationAndRegisterListeners() { registerConfigurationListener("dispatchThrottlingRatePerSubscriptionInByte", (dispatchRatePerTopicInByte) -> { updateSubscriptionMessageDispatchRate(); }); + // add listener to update "dispatcherPauseOnAckStatePersistentEnabled" in byte for subscription + registerConfigurationListener("dispatcherPauseOnAckStatePersistentEnabled", (dispatchRatePerTopicInByte) -> { + updateDispatchPauseOnAckStatePersistentEnabled(); + }); // add listener to update message-dispatch-rate in msg for replicator registerConfigurationListener("dispatchThrottlingRatePerReplicatorInMsg", @@ -2621,12 +2802,6 @@ private void updateConfigurationAndRegisterListeners() { updateReplicatorMessageDispatchRate(); }); - // add listener to notify broker publish-rate monitoring - registerConfigurationListener("brokerPublisherThrottlingTickTimeMillis", - (publisherThrottlingTickTimeMillis) -> { - setupBrokerPublishRateLimiterMonitor(); - }); - // add listener to update topic publish-rate dynamic config registerConfigurationListener("maxPublishRatePerTopicInMessages", maxPublishRatePerTopicInMessages -> updateMaxPublishRatePerTopicInMessages() @@ -2655,13 +2830,6 @@ private void updateConfigurationAndRegisterListeners() { registerConfigurationListener("dispatchThrottlingRateInByte", (dispatchThrottlingRateInByte) -> updateBrokerDispatchThrottlingMaxRate()); - // add listener to notify topic publish-rate monitoring - if (!preciseTopicPublishRateLimitingEnable) { - registerConfigurationListener("topicPublisherThrottlingTickTimeMillis", - (publisherThrottlingTickTimeMillis) -> { - setupTopicPublishRateLimiterMonitor(); - }); - } // add listener to notify topic subscriptionTypesEnabled changed. registerConfigurationListener("subscriptionTypesEnabled", this::updateBrokerSubscriptionTypesEnabled); @@ -2681,6 +2849,11 @@ private void updateConfigurationAndRegisterListeners() { pulsar.getWebService().updateHttpRequestsFailOnUnknownPropertiesEnabled((boolean) enabled); }); + // add listener to notify web service httpRequestsFailOnUnknownPropertiesEnabled changed. + registerConfigurationListener("configurationMetadataSyncEventTopic", enabled -> { + pulsar.initConfigMetadataSynchronizerIfNeeded(); + }); + // add more listeners here // (3) create dynamic-config if not exist. @@ -2723,7 +2896,7 @@ private void updateMaxPublishRatePerTopicInMessages() { forEachTopic(topic -> { if (topic instanceof AbstractTopic) { ((AbstractTopic) topic).updateBrokerPublishRate(); - ((AbstractTopic) topic).updatePublishDispatcher(); + ((AbstractTopic) topic).updatePublishRateLimiter(); } })); } @@ -2741,29 +2914,11 @@ private void updateSubscribeRate() { private void updateBrokerPublisherThrottlingMaxRate() { int currentMaxMessageRate = pulsar.getConfiguration().getBrokerPublisherThrottlingMaxMessageRate(); long currentMaxByteRate = pulsar.getConfiguration().getBrokerPublisherThrottlingMaxByteRate(); - int brokerTickMs = pulsar.getConfiguration().getBrokerPublisherThrottlingTickTimeMillis(); - - // not enable - if (brokerTickMs <= 0 || (currentMaxByteRate <= 0 && currentMaxMessageRate <= 0)) { - if (brokerPublishRateLimiter != PublishRateLimiter.DISABLED_RATE_LIMITER) { - refreshBrokerPublishRate(); - brokerPublishRateLimiter = PublishRateLimiter.DISABLED_RATE_LIMITER; - } - return; - } final PublishRate publishRate = new PublishRate(currentMaxMessageRate, currentMaxByteRate); log.info("Update broker publish rate limiting {}", publishRate); - // lazy init broker Publish-rateLimiting monitoring if not initialized yet - this.setupBrokerPublishRateLimiterMonitor(); - if (brokerPublishRateLimiter == null - || brokerPublishRateLimiter == PublishRateLimiter.DISABLED_RATE_LIMITER) { - // create new rateLimiter if rate-limiter is disabled - brokerPublishRateLimiter = new PublishRateLimiterImpl(publishRate); - } else { - brokerPublishRateLimiter.update(publishRate); - } + brokerPublishRateLimiter.update(publishRate); } private void updateTopicMessageDispatchRate() { @@ -2778,6 +2933,18 @@ private void updateTopicMessageDispatchRate() { }); } + private void updateDispatchPauseOnAckStatePersistentEnabled() { + this.pulsar().getExecutor().execute(() -> { + forEachTopic(topic -> { + if (topic instanceof PersistentTopic) { + // Update policies. + PersistentTopic persistentTopic = (PersistentTopic) topic; + persistentTopic.updateBrokerDispatchPauseOnAckStatePersistentEnabled(); + } + }); + }); + } + private void updateBrokerSubscriptionTypesEnabled(Object subscriptionTypesEnabled) { this.pulsar().getExecutor().execute(() -> { // update subscriptionTypesEnabled @@ -2848,6 +3015,9 @@ private void updateManagedLedgerConfig() { * On notification, listener should first check if config value has been changed and after taking appropriate * action, listener should update config value with new value if it has been changed (so, next time listener can * compare values on configMap change). + * + * Note: The new value that the {@param listener} may accept could be a null value. + * * @param * * @param configKey @@ -2862,44 +3032,30 @@ public void registerConfigurationListener(String configKey, Consumer list private void addDynamicConfigValidator(String key, Predicate validator) { validateConfigKey(key); - if (dynamicConfigurationMap.containsKey(key)) { - dynamicConfigurationMap.get(key).validator = validator; - } + dynamicConfigurationMap.get(key).validator = validator; } private void validateConfigKey(String key) { - try { - ServiceConfiguration.class.getDeclaredField(key); - } catch (Exception e) { - log.error("ServiceConfiguration key {} not found {}", key, e.getMessage()); - throw new IllegalArgumentException("Invalid service config " + key, e); - } - } - - private void createDynamicConfigPathIfNotExist() { - try { - Optional> configCache = - pulsar().getPulsarResources().getDynamicConfigResources().getDynamicConfiguration(); - - // create dynamic-config if not exist. - if (!configCache.isPresent()) { - pulsar().getPulsarResources().getDynamicConfigResources() - .setDynamicConfigurationWithCreate(n -> new HashMap<>()); - } - } catch (Exception e) { - log.warn("Failed to read dynamic broker configuration", e); + if (!dynamicConfigurationMap.containsKey(key)) { + throw new IllegalArgumentException(key + " doesn't exits in the dynamicConfigurationMap"); } } /** - * Updates pulsar.ServiceConfiguration's dynamic field with value persistent into zk-dynamic path. It also validates - * dynamic-value before updating it and throws {@code IllegalArgumentException} if validation fails + * Allows the third-party plugin to register a custom dynamic configuration. */ - private void updateDynamicServiceConfiguration() { - Optional> configCache = Optional.empty(); + public void registerCustomDynamicConfiguration(String key, Predicate validator) { + if (dynamicConfigurationMap.containsKey(key)) { + throw new IllegalArgumentException(key + " already exists in the dynamicConfigurationMap"); + } + ConfigField configField = ConfigField.newCustomConfigField(null); + configField.validator = validator; + dynamicConfigurationMap.put(key, configField); + } + private void createDynamicConfigPathIfNotExist() { try { - configCache = + Optional> configCache = pulsar().getPulsarResources().getDynamicConfigResources().getDynamicConfiguration(); // create dynamic-config if not exist. @@ -2910,28 +3066,6 @@ private void updateDynamicServiceConfiguration() { } catch (Exception e) { log.warn("Failed to read dynamic broker configuration", e); } - - configCache.ifPresent(stringStringMap -> stringStringMap.forEach((key, value) -> { - // validate field - if (dynamicConfigurationMap.containsKey(key) && dynamicConfigurationMap.get(key).validator != null) { - if (!dynamicConfigurationMap.get(key).validator.test(value)) { - log.error("Failed to validate dynamic config {} with value {}", key, value); - throw new IllegalArgumentException( - String.format("Failed to validate dynamic-config %s/%s", key, value)); - } - } - // update field value - try { - Field field = ServiceConfiguration.class.getDeclaredField(key); - if (field != null && field.isAnnotationPresent(FieldContext.class)) { - field.setAccessible(true); - field.set(pulsar().getConfiguration(), FieldParser.value(value, field)); - log.info("Successfully updated {}/{}", key, value); - } - } catch (Exception e) { - log.warn("Failed to update service configuration {}/{}, {}", key, value, e.getMessage()); - } - })); } public DelayedDeliveryTrackerFactory getDelayedDeliveryTrackerFactory() { @@ -2939,12 +3073,12 @@ public DelayedDeliveryTrackerFactory getDelayedDeliveryTrackerFactory() { } public List getDynamicConfiguration() { - return dynamicConfigurationMap.keys(); + return dynamicConfigurationMap.keySet().stream().toList(); } public Map getRuntimeConfiguration() { Map configMap = new HashMap<>(); - ConcurrentOpenHashMap runtimeConfigurationMap = getRuntimeConfigurationMap(); + ConcurrentHashMap runtimeConfigurationMap = getRuntimeConfigurationMap(); runtimeConfigurationMap.forEach((key, value) -> { configMap.put(key, String.valueOf(value)); }); @@ -2962,23 +3096,28 @@ public boolean validateDynamicConfiguration(String key, String value) { return true; } - private static ConcurrentOpenHashMap prepareDynamicConfigurationMap() { - ConcurrentOpenHashMap dynamicConfigurationMap = - ConcurrentOpenHashMap.newBuilder().build(); - for (Field field : ServiceConfiguration.class.getDeclaredFields()) { - if (field != null && field.isAnnotationPresent(FieldContext.class)) { - field.setAccessible(true); - if (field.getAnnotation(FieldContext.class).dynamic()) { - dynamicConfigurationMap.put(field.getName(), new ConfigField(field)); + private Map prepareDynamicConfigurationMap() { + final var dynamicConfigurationMap = new ConcurrentHashMap(); + try { + for (Field field : ServiceConfiguration.class.getDeclaredFields()) { + if (field != null && field.isAnnotationPresent(FieldContext.class)) { + field.setAccessible(true); + if (field.getAnnotation(FieldContext.class).dynamic()) { + Object defaultValue = field.get(pulsar.getConfiguration()); + dynamicConfigurationMap.put(field.getName(), new ConfigField(field, defaultValue)); + } } } + } catch (IllegalArgumentException | IllegalAccessException ex) { + // This error never occurs. + log.error("Failed to initialize dynamic configuration map", ex); + throw new RuntimeException(ex); } return dynamicConfigurationMap; } - private ConcurrentOpenHashMap getRuntimeConfigurationMap() { - ConcurrentOpenHashMap runtimeConfigurationMap = - ConcurrentOpenHashMap.newBuilder().build(); + private ConcurrentHashMap getRuntimeConfigurationMap() { + final var runtimeConfigurationMap = new ConcurrentHashMap(); for (Field field : ServiceConfiguration.class.getDeclaredFields()) { if (field != null && field.isAnnotationPresent(FieldContext.class)) { field.setAccessible(true); @@ -3010,7 +3149,10 @@ private void createPendingLoadTopic() { CompletableFuture> pendingFuture = pendingTopic.getTopicFuture(); final Semaphore topicLoadSemaphore = topicLoadRequestSemaphore.get(); final boolean acquiredPermit = topicLoadSemaphore.tryAcquire(); - checkOwnershipAndCreatePersistentTopic(topic, true, pendingFuture, pendingTopic.getProperties()); + checkOwnershipAndCreatePersistentTopic(topic, + pendingTopic.isCreateIfMissing(), + pendingFuture, + pendingTopic.getProperties(), pendingTopic.getTopicPolicies()); pendingFuture.handle((persistentTopic, ex) -> { // release permit and process next pending topic if (acquiredPermit) { @@ -3024,7 +3166,7 @@ private void createPendingLoadTopic() { pendingTopic.getTopicFuture() .completeExceptionally((e instanceof RuntimeException && e.getCause() != null) ? e.getCause() : e); // schedule to process next pending topic - inactivityMonitor.schedule(this::createPendingLoadTopic, 100, TimeUnit.MILLISECONDS); + inactivityMonitor.schedule(this::createPendingLoadTopic, 100, MILLISECONDS); return null; }); } @@ -3034,63 +3176,66 @@ public CompletableFuture fetchPartitionedTopicMetadata if (pulsar.getNamespaceService() == null) { return FutureUtil.failedFuture(new NamingException("namespace service is not ready")); } - Optional policies = - pulsar.getPulsarResources().getNamespaceResources() - .getPoliciesIfCached(topicName.getNamespaceObject()); - return pulsar.getNamespaceService().checkTopicExists(topicName) - .thenCompose(topicExists -> { - return fetchPartitionedTopicMetadataAsync(topicName) - .thenCompose(metadata -> { - CompletableFuture future = new CompletableFuture<>(); - - // There are a couple of potentially blocking calls, which we cannot make from the - // MetadataStore callback thread. - pulsar.getExecutor().execute(() -> { - // If topic is already exist, creating partitioned topic is not allowed. - - if (metadata.partitions == 0 - && !topicExists - && !topicName.isPartitioned() - && pulsar.getBrokerService() - .isDefaultTopicTypePartitioned(topicName, policies)) { - isAllowAutoTopicCreationAsync(topicName, policies).thenAccept(allowed -> { - if (allowed) { - pulsar.getBrokerService() - .createDefaultPartitionedTopicAsync(topicName, policies) - .thenAccept(md -> future.complete(md)) - .exceptionally(ex -> { - if (ex.getCause() - instanceof MetadataStoreException - .AlreadyExistsException) { - // The partitioned topic might be created concurrently - fetchPartitionedTopicMetadataAsync(topicName) - .whenComplete((metadata2, ex2) -> { - if (ex2 == null) { - future.complete(metadata2); - } else { - future.completeExceptionally(ex2); - } - }); - } else { - future.completeExceptionally(ex); - } - return null; - }); - } else { - future.complete(metadata); - } - }).exceptionally(ex -> { - future.completeExceptionally(ex); - return null; - }); - } else { - future.complete(metadata); - } - }); + return pulsar.getNamespaceService().checkTopicExists(topicName).thenComposeAsync(topicExistsInfo -> { + final boolean topicExists = topicExistsInfo.isExists(); + final TopicType topicType = topicExistsInfo.getTopicType(); + final Integer partitions = topicExistsInfo.getPartitions(); + topicExistsInfo.recycle(); + + // Topic exists. + if (topicExists) { + if (topicType.equals(TopicType.PARTITIONED)) { + return CompletableFuture.completedFuture(new PartitionedTopicMetadata(partitions)); + } + return CompletableFuture.completedFuture(new PartitionedTopicMetadata(0)); + } - return future; + // Try created if allowed to create a partitioned topic automatically. + return pulsar.getPulsarResources().getNamespaceResources().getPoliciesAsync(topicName.getNamespaceObject()) + .thenComposeAsync(policies -> { + return isAllowAutoTopicCreationAsync(topicName, policies).thenComposeAsync(allowed -> { + // Not Allow auto-creation. + if (!allowed) { + // Do not change the original behavior, or default return a non-partitioned topic. + return CompletableFuture.completedFuture(new PartitionedTopicMetadata(0)); + } + + // Allow auto create non-partitioned topic. + boolean autoCreatePartitionedTopic = pulsar.getBrokerService() + .isDefaultTopicTypePartitioned(topicName, policies); + if (!autoCreatePartitionedTopic || topicName.isPartitioned()) { + return CompletableFuture.completedFuture(new PartitionedTopicMetadata(0)); + } + + // Create partitioned metadata. + return pulsar.getBrokerService().createDefaultPartitionedTopicAsync(topicName, policies) + .exceptionallyCompose(ex -> { + // The partitioned topic might be created concurrently. + if (ex.getCause() instanceof MetadataStoreException.AlreadyExistsException) { + log.info("[{}] The partitioned topic is already created, try to refresh the cache" + + " and read again.", topicName); + CompletableFuture recheckFuture = + fetchPartitionedTopicMetadataAsync(topicName, true); + recheckFuture.exceptionally(ex2 -> { + // Just for printing a log if error occurs. + log.error("[{}] Fetch partitioned topic metadata failed", topicName, ex); + return null; + }); + return recheckFuture; + } else { + log.error("[{}] operation of creating partitioned topic metadata failed", + topicName, ex); + return CompletableFuture.failedFuture(ex); + } }); - }); + }, pulsar.getExecutor()).exceptionallyCompose(ex -> { + log.error("[{}] operation of get partitioned metadata failed due to calling" + + " isAllowAutoTopicCreationAsync failed", + topicName, ex); + return CompletableFuture.failedFuture(ex); + }); + }, pulsar.getExecutor()); + }, pulsar.getExecutor()); } @SuppressWarnings("deprecation") @@ -3118,9 +3263,14 @@ private CompletableFuture createDefaultPartitionedTopi } public CompletableFuture fetchPartitionedTopicMetadataAsync(TopicName topicName) { + return fetchPartitionedTopicMetadataAsync(topicName, false); + } + + public CompletableFuture fetchPartitionedTopicMetadataAsync(TopicName topicName, + boolean refreshCacheAndGet) { // gets the number of partitions from the configuration cache return pulsar.getPulsarResources().getNamespaceResources().getPartitionedTopicResources() - .getPartitionedTopicMetadataAsync(topicName).thenApply(metadata -> { + .getPartitionedTopicMetadataAsync(topicName, refreshCacheAndGet).thenApply(metadata -> { // if the partitioned topic is not found in metadata, then the topic is not partitioned return metadata.orElseGet(() -> new PartitionedTopicMetadata()); }); @@ -3130,11 +3280,6 @@ public OrderedExecutor getTopicOrderedExecutor() { return topicOrderedExecutor; } - public ConcurrentOpenHashMap>> - getMultiLayerTopicMap() { - return multiLayerTopicsMap; - } - /** * If per-broker unacked message reached to limit then it blocks dispatcher if its unacked message limit has been * reached to {@link #maxUnackedMsgsPerDispatcher}. @@ -3186,7 +3331,7 @@ public void checkUnAckMessageDispatching() { } else if (blockedDispatcherOnHighUnackedMsgs.get() && unAckedMessages < maxUnackedMessages / 2) { // unblock broker-dispatching if received enough acked messages back if (blockedDispatcherOnHighUnackedMsgs.compareAndSet(true, false)) { - unblockDispatchersOnUnAckMessages(blockedDispatchers.values()); + unblockDispatchersOnUnAckMessages(blockedDispatchers.stream().toList()); } } @@ -3239,13 +3384,49 @@ public void unblockDispatchersOnUnAckMessages(List validator; - public ConfigField(Field field) { + public ConfigField(Field field, Object defaultValue) { super(); this.field = field; + this.defaultValue = defaultValue; + } + + public static ConfigField newCustomConfigField(String customValue) { + ConfigField configField = new ConfigField(null, null); + configField.lastDynamicValue = customValue; + return configField; } } @@ -3282,10 +3463,9 @@ public CompletableFuture isAllowAutoTopicCreationAsync(final String top } public CompletableFuture isAllowAutoTopicCreationAsync(final TopicName topicName) { - Optional policies = - pulsar.getPulsarResources().getNamespaceResources() - .getPoliciesIfCached(topicName.getNamespaceObject()); - return isAllowAutoTopicCreationAsync(topicName, policies); + return pulsar.getPulsarResources().getNamespaceResources() + .getPoliciesAsync(topicName.getNamespaceObject()) + .thenCompose(policies -> isAllowAutoTopicCreationAsync(topicName, policies)); } private CompletableFuture isAllowAutoTopicCreationAsync(final TopicName topicName, @@ -3295,10 +3475,19 @@ private CompletableFuture isAllowAutoTopicCreationAsync(final TopicName topicName.getNamespaceObject()); return CompletableFuture.completedFuture(false); } - //System topic can always be created automatically + + // ExtensibleLoadManagerImpl.internal topics expects to be non-partitioned-topics now. + // We don't allow the auto-creation here. + // ExtensibleLoadManagerImpl.start() is responsible to create the internal system topics. + if (ExtensibleLoadManagerImpl.isInternalTopic(topicName.toString())) { + return CompletableFuture.completedFuture(false); + } + + //Other system topics can be created automatically if (pulsar.getConfiguration().isSystemTopicEnabled() && isSystemTopic(topicName)) { return CompletableFuture.completedFuture(true); } + final boolean allowed; AutoTopicCreationOverride autoTopicCreationOverride = getAutoTopicCreationOverride(topicName, policies); if (autoTopicCreationOverride != null) { @@ -3319,6 +3508,10 @@ private CompletableFuture isAllowAutoTopicCreationAsync(final TopicName } public boolean isDefaultTopicTypePartitioned(final TopicName topicName, final Optional policies) { + if (topicName.getPartitionedTopicName().endsWith(DLQ_GROUP_TOPIC_SUFFIX) + || topicName.getPartitionedTopicName().endsWith(RETRY_GROUP_TOPIC_SUFFIX)) { + return false; + } AutoTopicCreationOverride autoTopicCreationOverride = getAutoTopicCreationOverride(topicName, policies); if (autoTopicCreationOverride != null) { return TopicType.PARTITIONED.toString().equals(autoTopicCreationOverride.getTopicType()); @@ -3346,35 +3539,29 @@ private AutoTopicCreationOverride getAutoTopicCreationOverride(final TopicName t return null; } - public boolean isAllowAutoSubscriptionCreation(final String topic) { - TopicName topicName = TopicName.get(topic); - return isAllowAutoSubscriptionCreation(topicName); - } - - public boolean isAllowAutoSubscriptionCreation(final TopicName topicName) { - AutoSubscriptionCreationOverride autoSubscriptionCreationOverride = - getAutoSubscriptionCreationOverride(topicName); - if (autoSubscriptionCreationOverride != null) { - return autoSubscriptionCreationOverride.isAllowAutoSubscriptionCreation(); - } else { - return pulsar.getConfiguration().isAllowAutoSubscriptionCreation(); - } - } - - private AutoSubscriptionCreationOverride getAutoSubscriptionCreationOverride(final TopicName topicName) { - Optional topicPolicies = getTopicPolicies(topicName); - if (topicPolicies.isPresent() && topicPolicies.get().getAutoSubscriptionCreationOverride() != null) { - return topicPolicies.get().getAutoSubscriptionCreationOverride(); - } - - Optional policies = - pulsar.getPulsarResources().getNamespaceResources().getPoliciesIfCached(topicName.getNamespaceObject()); - // If namespace policies have the field set, it will override the broker-level setting - if (policies.isPresent() && policies.get().autoSubscriptionCreationOverride != null) { - return policies.get().autoSubscriptionCreationOverride; + public @Nonnull CompletableFuture isAllowAutoSubscriptionCreationAsync(@Nonnull TopicName tpName) { + requireNonNull(tpName); + // Policies priority: topic level -> namespace level -> broker level + if (ExtensibleLoadManagerImpl.isInternalTopic(tpName.toString())) { + return CompletableFuture.completedFuture(true); } - log.debug("No autoSubscriptionCreateOverride policy found for {}", topicName); - return null; + return pulsar.getTopicPoliciesService() + .getTopicPoliciesAsync(tpName, TopicPoliciesService.GetType.LOCAL_ONLY) + .thenCompose(optionalTopicPolicies -> { + Boolean allowed = optionalTopicPolicies.map(TopicPolicies::getAutoSubscriptionCreationOverride) + .map(AutoSubscriptionCreationOverrideImpl::isAllowAutoSubscriptionCreation) + .orElse(null); + if (allowed != null) { + return CompletableFuture.completedFuture(allowed); + } + // namespace level policies + return pulsar.getPulsarResources().getNamespaceResources().getPoliciesAsync( + tpName.getNamespaceObject() + ).thenApply(optionalPolicies -> optionalPolicies.map(__ -> __.autoSubscriptionCreationOverride) + .map(AutoSubscriptionCreationOverride::isAllowAutoSubscriptionCreation) + // broker level policies + .orElse(pulsar.getConfiguration().isAllowAutoSubscriptionCreation())); + }); } public boolean isSystemTopic(String topic) { @@ -3386,29 +3573,28 @@ public boolean isSystemTopic(TopicName topicName) { || SystemTopicNames.isSystemTopic(topicName); } - /** - * Get {@link TopicPolicies} for the parameterized topic. - * @param topicName - * @return TopicPolicies, if they exist. Otherwise, the value will not be present. - */ - public Optional getTopicPolicies(TopicName topicName) { - if (!pulsar().getConfig().isTopicLevelPoliciesEnabled()) { - return Optional.empty(); - } - return Optional.ofNullable(pulsar.getTopicPoliciesService() - .getTopicPoliciesIfExists(topicName)); - } - - public CompletableFuture deleteTopicPolicies(TopicName topicName) { - final PulsarService pulsarService = pulsar(); - if (!pulsarService.getConfig().isTopicLevelPoliciesEnabled()) { + public CompletableFuture deleteSchema(TopicName topicName) { + // delete schema at the upper level when deleting the partitioned topic. + if (topicName.isPartitioned()) { return CompletableFuture.completedFuture(null); } - return pulsar.getTopicPoliciesService() - .deleteTopicPoliciesAsync(TopicName.get(topicName.getPartitionedTopicName())); + String base = topicName.getPartitionedTopicName(); + String id = TopicName.get(base).getSchemaName(); + return getPulsar().getSchemaRegistryService().deleteSchemaStorage(id).whenComplete((vid, ex) -> { + if (vid != null && ex == null) { + // It's different from `SchemasResource.deleteSchema` + // because when we delete a topic, the schema + // history is meaningless. But when we delete a schema of a topic, a new schema could be + // registered in the future. + log.info("Deleted schema storage of id: {}", id); + } + }); } private CompletableFuture checkMaxTopicsPerNamespace(TopicName topicName, int numPartitions) { + if (isSystemTopic(topicName)) { + return CompletableFuture.completedFuture(null); + } return pulsar.getPulsarResources().getNamespaceResources() .getPoliciesAsync(topicName.getNamespaceObject()) .thenCompose(optPolicies -> { @@ -3427,7 +3613,7 @@ private CompletableFuture checkMaxTopicsPerNamespace(TopicName topicName, log.error("Failed to create persistent topic {}, " + "exceed maximum number of topics in namespace", topicName); return FutureUtil.failedFuture( - new RestException(Response.Status.PRECONDITION_FAILED, + new NotAllowedException( "Exceed maximum number of topics in namespace.")); } else { return CompletableFuture.completedFuture(null); @@ -3455,16 +3641,22 @@ public boolean isBrokerPayloadProcessorEnabled() { return !brokerEntryPayloadProcessors.isEmpty(); } - public void pausedConnections(int numberOfConnections) { - pausedConnections.add(numberOfConnections); + public void recordConnectionPaused() { + rateLimitedConnectionsCounter.add(1, ConnectionRateLimitOperationName.PAUSED.attributes); + } + + public void recordConnectionResumed() { + rateLimitedConnectionsCounter.add(1, ConnectionRateLimitOperationName.RESUMED.attributes); } - public void resumedConnections(int numberOfConnections) { - pausedConnections.add(-numberOfConnections); + public void recordConnectionThrottled() { + rateLimitedConnectionsCounter.add(1, ConnectionRateLimitOperationName.THROTTLED.attributes); + throttledConnectionsGauge.inc(); } - public long getPausedConnections() { - return pausedConnections.longValue(); + public void recordConnectionUnthrottled() { + rateLimitedConnectionsCounter.add(1, ConnectionRateLimitOperationName.UNTHROTTLED.attributes); + throttledConnectionsGauge.dec(); } @SuppressWarnings("unchecked") @@ -3508,7 +3700,9 @@ public void setPulsarChannelInitializerFactory(PulsarChannelInitializer.Factory @Getter private static class TopicLoadingContext { private final String topic; + private final boolean createIfMissing; private final CompletableFuture> topicFuture; private final Map properties; + private final TopicPolicies topicPolicies; } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java index 3e77588b2459f..d30dfc319e098 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java @@ -146,6 +146,10 @@ public static class TopicBusyException extends BrokerServiceException { public TopicBusyException(String msg) { super(msg); } + + public TopicBusyException(String msg, Throwable t) { + super(msg, t); + } } public static class TopicNotFoundException extends BrokerServiceException { @@ -214,12 +218,6 @@ public ConsumerAssignException(String msg) { } } - public static class TopicPoliciesCacheNotInitException extends BrokerServiceException { - public TopicPoliciesCacheNotInitException() { - super("Topic policies cache have not init."); - } - } - public static class TopicBacklogQuotaExceededException extends BrokerServiceException { @Getter private final BacklogQuota.RetentionPolicy retentionPolicy; @@ -258,6 +256,8 @@ private static ServerError getClientErrorCode(Throwable t, boolean checkCauseIfU return ServerError.ServiceNotReady; } else if (t instanceof TopicNotFoundException) { return ServerError.TopicNotFound; + } else if (t instanceof SubscriptionNotFoundException) { + return ServerError.SubscriptionNotFound; } else if (t instanceof IncompatibleSchemaException || t instanceof InvalidSchemaDataException) { // for backward compatible with old clients, invalid schema data diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java index ea491bd40d332..1ae9a6ff96b7d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java @@ -18,10 +18,8 @@ */ package org.apache.pulsar.broker.service; -import com.google.common.collect.Lists; import java.util.ArrayList; -import java.util.Comparator; -import java.util.LinkedHashMap; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.NavigableMap; @@ -39,11 +37,14 @@ * number of keys assigned to each consumer. */ public class ConsistentHashingStickyKeyConsumerSelector implements StickyKeyConsumerSelector { - + // use NUL character as field separator for hash key calculation + private static final String KEY_SEPARATOR = "\0"; private final ReadWriteLock rwLock = new ReentrantReadWriteLock(); // Consistent-Hash ring - private final NavigableMap> hashRing; + private final NavigableMap hashRing; + // Tracks the used consumer name indexes for each consumer name + private final ConsumerNameIndexTracker consumerNameIndexTracker = new ConsumerNameIndexTracker(); private final int numberOfPoints; @@ -56,22 +57,20 @@ public ConsistentHashingStickyKeyConsumerSelector(int numberOfPoints) { public CompletableFuture addConsumer(Consumer consumer) { rwLock.writeLock().lock(); try { + ConsumerIdentityWrapper consumerIdentityWrapper = new ConsumerIdentityWrapper(consumer); // Insert multiple points on the hash ring for every consumer // The points are deterministically added based on the hash of the consumer name for (int i = 0; i < numberOfPoints; i++) { - String key = consumer.consumerName() + i; - int hash = Murmur3_32Hash.getInstance().makeHash(key.getBytes()); - hashRing.compute(hash, (k, v) -> { - if (v == null) { - return Lists.newArrayList(consumer); - } else { - if (!v.contains(consumer)) { - v.add(consumer); - v.sort(Comparator.comparing(Consumer::consumerName, String::compareTo)); - } - return v; - } - }); + int consumerNameIndex = + consumerNameIndexTracker.increaseConsumerRefCountAndReturnIndex(consumerIdentityWrapper); + int hash = calculateHashForConsumerAndIndex(consumer, consumerNameIndex, i); + // When there's a collision, the new consumer will replace the old one. + // This is a rare case, and it is acceptable to replace the old consumer since there + // are multiple points for each consumer. This won't affect the overall distribution significantly. + ConsumerIdentityWrapper removed = hashRing.put(hash, consumerIdentityWrapper); + if (removed != null) { + consumerNameIndexTracker.decreaseConsumerRefCount(removed); + } } return CompletableFuture.completedFuture(null); } finally { @@ -79,25 +78,36 @@ public CompletableFuture addConsumer(Consumer consumer) { } } + /** + * Calculate the hash for a consumer and hash ring point. + * The hash is calculated based on the consumer name, consumer name index, and hash ring point index. + * The resulting hash is used as the key to insert the consumer into the hash ring. + * + * @param consumer the consumer + * @param consumerNameIndex the index of the consumer name + * @param hashRingPointIndex the index of the hash ring point + * @return the hash value + */ + private static int calculateHashForConsumerAndIndex(Consumer consumer, int consumerNameIndex, + int hashRingPointIndex) { + String key = consumer.consumerName() + KEY_SEPARATOR + consumerNameIndex + KEY_SEPARATOR + hashRingPointIndex; + return Murmur3_32Hash.getInstance().makeHash(key.getBytes()); + } + @Override public void removeConsumer(Consumer consumer) { rwLock.writeLock().lock(); try { - // Remove all the points that were added for this consumer - for (int i = 0; i < numberOfPoints; i++) { - String key = consumer.consumerName() + i; - int hash = Murmur3_32Hash.getInstance().makeHash(key.getBytes()); - hashRing.compute(hash, (k, v) -> { - if (v == null) { - return null; - } else { - v.removeIf(c -> c.equals(consumer)); - if (v.isEmpty()) { - v = null; - } - return v; + ConsumerIdentityWrapper consumerIdentityWrapper = new ConsumerIdentityWrapper(consumer); + int consumerNameIndex = consumerNameIndexTracker.getTrackedIndex(consumerIdentityWrapper); + if (consumerNameIndex > -1) { + // Remove all the points that were added for this consumer + for (int i = 0; i < numberOfPoints; i++) { + int hash = calculateHashForConsumerAndIndex(consumer, consumerNameIndex, i); + if (hashRing.remove(hash, consumerIdentityWrapper)) { + consumerNameIndexTracker.decreaseConsumerRefCount(consumerIdentityWrapper); } - }); + } } } finally { rwLock.writeLock().unlock(); @@ -111,16 +121,13 @@ public Consumer select(int hash) { if (hashRing.isEmpty()) { return null; } - - List consumerList; - Map.Entry> ceilingEntry = hashRing.ceilingEntry(hash); + Map.Entry ceilingEntry = hashRing.ceilingEntry(hash); if (ceilingEntry != null) { - consumerList = ceilingEntry.getValue(); + return ceilingEntry.getValue().consumer; } else { - consumerList = hashRing.firstEntry().getValue(); + // Handle wrap-around in the hash ring, return the first consumer + return hashRing.firstEntry().getValue().consumer; } - - return consumerList.get(hash % consumerList.size()); } finally { rwLock.readLock().unlock(); } @@ -128,16 +135,27 @@ public Consumer select(int hash) { @Override public Map> getConsumerKeyHashRanges() { - Map> result = new LinkedHashMap<>(); + Map> result = new IdentityHashMap<>(); rwLock.readLock().lock(); try { + if (hashRing.isEmpty()) { + return result; + } int start = 0; - for (Map.Entry> entry: hashRing.entrySet()) { - for (Consumer consumer: entry.getValue()) { - result.computeIfAbsent(consumer, key -> new ArrayList<>()) - .add(Range.of(start, entry.getKey())); - } - start = entry.getKey() + 1; + int lastKey = 0; + for (Map.Entry entry: hashRing.entrySet()) { + Consumer consumer = entry.getValue().consumer; + result.computeIfAbsent(consumer, key -> new ArrayList<>()) + .add(Range.of(start, entry.getKey())); + lastKey = entry.getKey(); + start = lastKey + 1; + } + // Handle wrap-around in the hash ring, the first consumer will also contain the range from the last key + // to the maximum value of the hash range + Consumer firstConsumer = hashRing.firstEntry().getValue().consumer; + List ranges = result.get(firstConsumer); + if (lastKey != Integer.MAX_VALUE - 1) { + ranges.add(Range.of(lastKey + 1, Integer.MAX_VALUE - 1)); } } finally { rwLock.readLock().unlock(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index a3f9da41e6b35..7f46e8969eb53 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -25,6 +25,8 @@ import com.google.common.util.concurrent.AtomicDouble; import io.netty.util.concurrent.Future; import io.netty.util.concurrent.Promise; +import io.opentelemetry.api.common.Attributes; +import java.time.Instant; import java.util.ArrayList; import java.util.BitSet; import java.util.Collections; @@ -34,16 +36,20 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.concurrent.atomic.LongAdder; import java.util.stream.Collectors; import lombok.Getter; import lombok.Setter; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.mledger.impl.AckSetStateUtil; import org.apache.commons.lang3.mutable.MutableInt; import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.MessageId; @@ -56,7 +62,7 @@ import org.apache.pulsar.common.api.proto.KeySharedMeta; import org.apache.pulsar.common.api.proto.MessageIdData; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.ClusterData.ClusterUrl; +import org.apache.pulsar.common.policies.data.ClusterPolicies.ClusterUrl; import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; import org.apache.pulsar.common.protocol.Commands; @@ -67,6 +73,7 @@ import org.apache.pulsar.common.util.collections.BitSetRecyclable; import org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap; import org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap.LongPair; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; import org.apache.pulsar.transaction.common.exception.TransactionConflictException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -89,7 +96,9 @@ public class Consumer { private final Rate msgOut; private final Rate msgRedeliver; private final LongAdder msgOutCounter; + private final LongAdder msgRedeliverCounter; private final LongAdder bytesOutCounter; + private final LongAdder messageAckCounter; private final Rate messageAckRate; private volatile long lastConsumedTimestamp; @@ -115,6 +124,9 @@ public class Consumer { private final ConsumerStatsImpl stats; private final boolean isDurable; + + private final boolean isPersistentTopic; + private static final AtomicIntegerFieldUpdater UNACKED_MESSAGES_UPDATER = AtomicIntegerFieldUpdater.newUpdater(Consumer.class, "unackedMessages"); private volatile int unackedMessages = 0; @@ -134,7 +146,7 @@ public class Consumer { private static final double avgPercent = 0.9; private boolean preciseDispatcherFlowControl; - private PositionImpl readPositionWhenJoining; + private Position lastSentPositionWhenJoining; private final String clientAddress; // IP address only, no port number included private final MessageId startMessageId; private final boolean isAcknowledgmentAtBatchIndexLevelEnabled; @@ -143,11 +155,18 @@ public class Consumer { @Setter private volatile long consumerEpoch; - private long negtiveUnackedMsgsTimestamp; + private long negativeUnackedMsgsTimestamp; @Getter private final SchemaType schemaType; + @Getter + private final Instant connectedSince = Instant.now(); + + private volatile Attributes openTelemetryAttributes; + private static final AtomicReferenceFieldUpdater OPEN_TELEMETRY_ATTRIBUTES_FIELD_UPDATER = + AtomicReferenceFieldUpdater.newUpdater(Consumer.class, Attributes.class, "openTelemetryAttributes"); + public Consumer(Subscription subscription, SubType subType, String topicName, long consumerId, int priorityLevel, String consumerName, boolean isDurable, TransportCnx cnx, String appId, @@ -172,13 +191,16 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo this.readCompacted = readCompacted; this.consumerName = consumerName; this.isDurable = isDurable; + this.isPersistentTopic = subscription.getTopic() instanceof PersistentTopic; this.keySharedMeta = keySharedMeta; this.cnx = cnx; this.msgOut = new Rate(); this.chunkedMessageRate = new Rate(); this.msgRedeliver = new Rate(); + this.msgRedeliverCounter = new LongAdder(); this.bytesOutCounter = new LongAdder(); this.msgOutCounter = new LongAdder(); + this.messageAckCounter = new LongAdder(); this.messageAckRate = new Rate(); this.appId = appId; @@ -193,13 +215,10 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo this.metadata = metadata != null ? metadata : Collections.emptyMap(); stats = new ConsumerStatsImpl(); - if (cnx.hasHAProxyMessage()) { - stats.setAddress(cnx.getHAProxyMessage().sourceAddress() + ":" + cnx.getHAProxyMessage().sourcePort()); - } else { - stats.setAddress(cnx.clientAddress().toString()); - } + stats.setAddress(cnx.clientSourceAddressAndPort()); stats.consumerName = consumerName; - stats.setConnectedSince(DateFormatter.now()); + stats.appId = appId; + stats.setConnectedSince(DateFormatter.format(connectedSince)); stats.setClientVersion(cnx.getClientVersion()); stats.metadata = this.metadata; @@ -221,6 +240,8 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo .getPulsar().getConfiguration().isAcknowledgmentAtBatchIndexLevelEnabled(); this.schemaType = schemaType; + + OPEN_TELEMETRY_ATTRIBUTES_FIELD_UPDATER.set(this, null); } @VisibleForTesting @@ -237,12 +258,15 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo this.consumerName = consumerName; this.msgOut = null; this.msgRedeliver = null; + this.msgRedeliverCounter = null; this.msgOutCounter = null; this.bytesOutCounter = null; + this.messageAckCounter = null; this.messageAckRate = null; this.pendingAcks = null; this.stats = null; this.isDurable = false; + this.isPersistentTopic = false; this.metadata = null; this.keySharedMeta = null; this.clientAddress = null; @@ -250,6 +274,7 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo this.isAcknowledgmentAtBatchIndexLevelEnabled = false; this.schemaType = null; MESSAGE_PERMITS_UPDATER.set(this, availablePermits); + OPEN_TELEMETRY_ATTRIBUTES_FIELD_UPDATER.set(this, null); } public SubType subType() { @@ -284,16 +309,29 @@ public Future sendMessages(final List entries, EntryBatch totalChunkedMessages, redeliveryTracker, DEFAULT_CONSUMER_EPOCH); } + public Future sendMessages(final List entries, EntryBatchSizes batchSizes, + EntryBatchIndexesAcks batchIndexesAcks, + int totalMessages, long totalBytes, long totalChunkedMessages, + RedeliveryTracker redeliveryTracker, long epoch) { + return sendMessages(entries, null, batchSizes, batchIndexesAcks, totalMessages, totalBytes, + totalChunkedMessages, redeliveryTracker, epoch); + } + /** * Dispatch a list of entries to the consumer.
* It is also responsible to release entries data and recycle entries object. * * @return a SendMessageInfo object that contains the detail of what was sent to consumer */ - public Future sendMessages(final List entries, EntryBatchSizes batchSizes, + public Future sendMessages(final List entries, + final List stickyKeyHashes, + EntryBatchSizes batchSizes, EntryBatchIndexesAcks batchIndexesAcks, - int totalMessages, long totalBytes, long totalChunkedMessages, - RedeliveryTracker redeliveryTracker, long epoch) { + int totalMessages, + long totalBytes, + long totalChunkedMessages, + RedeliveryTracker redeliveryTracker, + long epoch) { this.lastConsumedTimestamp = System.currentTimeMillis(); if (entries.isEmpty() || totalMessages == 0) { @@ -321,8 +359,8 @@ public Future sendMessages(final List entries, EntryBatch // because this consumer is possible to disconnect at this time. if (pendingAcks != null) { int batchSize = batchSizes.getBatchSize(i); - int stickyKeyHash = getStickyKeyHash(entry); - long[] ackSet = getCursorAckSet(PositionImpl.get(entry.getLedgerId(), entry.getEntryId())); + int stickyKeyHash = stickyKeyHashes == null ? getStickyKeyHash(entry) : stickyKeyHashes.get(i); + long[] ackSet = batchIndexesAcks == null ? null : batchIndexesAcks.getAckSet(i); if (ackSet != null) { unackedMessages -= (batchSize - BitSet.valueOf(ackSet).cardinality()); } @@ -365,6 +403,12 @@ public Future sendMessages(final List entries, EntryBatch msgOutCounter.add(totalMessages); bytesOutCounter.add(totalBytes); chunkedMessageRate.recordMultipleEvents(totalChunkedMessages, 0); + } else { + if (log.isDebugEnabled()) { + log.debug("[{}-{}] Sent messages to client fail by IO exception[{}], close the connection" + + " immediately. Consumer: {}", topicName, subscription, + status.cause() == null ? "" : status.cause().getMessage(), this.toString()); + } } }); return writeAndFlushPromise; @@ -400,8 +444,12 @@ public void disconnect() { } public void disconnect(boolean isResetCursor) { + disconnect(isResetCursor, Optional.empty()); + } + + public void disconnect(boolean isResetCursor, Optional assignedBrokerLookupData) { log.info("Disconnecting consumer: {}", this); - cnx.closeConsumer(this); + cnx.closeConsumer(this, assignedBrokerLookupData); try { close(isResetCursor); } catch (BrokerServiceException e) { @@ -409,8 +457,8 @@ public void disconnect(boolean isResetCursor) { } } - public void doUnsubscribe(final long requestId) { - subscription.doUnsubscribe(this).thenAccept(v -> { + public void doUnsubscribe(final long requestId, boolean force) { + subscription.doUnsubscribe(this, force).thenAccept(v -> { log.info("Unsubscribed successfully from {}", subscription); cnx.removedConsumer(this); cnx.getCommandSender().sendSuccessResponse(requestId); @@ -444,20 +492,20 @@ public CompletableFuture messageAcked(CommandAck ack) { return CompletableFuture.completedFuture(null); } - PositionImpl position; + Position position; MessageIdData msgId = ack.getMessageIdAt(0); if (msgId.getAckSetsCount() > 0) { long[] ackSets = new long[msgId.getAckSetsCount()]; for (int j = 0; j < msgId.getAckSetsCount(); j++) { ackSets[j] = msgId.getAckSetAt(j); } - position = PositionImpl.get(msgId.getLedgerId(), msgId.getEntryId(), ackSets); + position = AckSetStateUtil.createPositionWithAckSet(msgId.getLedgerId(), msgId.getEntryId(), ackSets); } else { - position = PositionImpl.get(msgId.getLedgerId(), msgId.getEntryId()); + position = PositionFactory.create(msgId.getLedgerId(), msgId.getEntryId()); } if (ack.hasTxnidMostBits() && ack.hasTxnidLeastBits()) { - List positionsAcked = Collections.singletonList(position); + List positionsAcked = Collections.singletonList(position); future = transactionCumulativeAcknowledge(ack.getTxnidMostBits(), ack.getTxnidLeastBits(), positionsAcked) .thenApply(unused -> 1L); @@ -477,26 +525,29 @@ public CompletableFuture messageAcked(CommandAck ack) { return future .thenApply(v -> { this.messageAckRate.recordEvent(v); + this.messageAckCounter.add(v); return null; }); } //this method is for individual ack not carry the transaction private CompletableFuture individualAckNormal(CommandAck ack, Map properties) { - List positionsAcked = new ArrayList<>(); + List> positionsAcked = new ArrayList<>(); long totalAckCount = 0; for (int i = 0; i < ack.getMessageIdsCount(); i++) { MessageIdData msgId = ack.getMessageIdAt(i); - PositionImpl position; - long ackedCount = 0; - long batchSize = getBatchSize(msgId); - Consumer ackOwnerConsumer = getAckOwnerConsumer(msgId.getLedgerId(), msgId.getEntryId()); + Position position; + Pair ackOwnerConsumerAndBatchSize = + getAckOwnerConsumerAndBatchSize(msgId.getLedgerId(), msgId.getEntryId()); + Consumer ackOwnerConsumer = ackOwnerConsumerAndBatchSize.getLeft(); + long ackedCount; + long batchSize = ackOwnerConsumerAndBatchSize.getRight(); if (msgId.getAckSetsCount() > 0) { long[] ackSets = new long[msgId.getAckSetsCount()]; for (int j = 0; j < msgId.getAckSetsCount(); j++) { ackSets[j] = msgId.getAckSetAt(j); } - position = PositionImpl.get(msgId.getLedgerId(), msgId.getEntryId(), ackSets); + position = AckSetStateUtil.createPositionWithAckSet(msgId.getLedgerId(), msgId.getEntryId(), ackSets); ackedCount = getAckedCountForBatchIndexLevelEnabled(position, batchSize, ackSets, ackOwnerConsumer); if (isTransactionEnabled()) { //sync the batch position bit set point, in order to delete the position in pending acks @@ -505,32 +556,36 @@ private CompletableFuture individualAckNormal(CommandAck ack, Map completableFuture = new CompletableFuture<>(); completableFuture.complete(totalAckCount); if (isTransactionEnabled() && Subscription.isIndividualAckMode(subType)) { - completableFuture.whenComplete((v, e) -> positionsAcked.forEach(position -> { + completableFuture.whenComplete((v, e) -> positionsAcked.forEach(positionPair -> { + Consumer ackOwnerConsumer = positionPair.getLeft(); + Position position = positionPair.getRight(); //check if the position can remove from the consumer pending acks. // the bit set is empty in pending ack handle. - if (((PositionImpl) position).getAckSet() != null) { + if (AckSetStateUtil.hasAckSet(position)) { if (((PersistentSubscription) subscription) - .checkIsCanDeleteConsumerPendingAck((PositionImpl) position)) { - removePendingAcks((PositionImpl) position); + .checkIsCanDeleteConsumerPendingAck(position)) { + removePendingAcks(ackOwnerConsumer, position); } } })); @@ -542,7 +597,7 @@ private CompletableFuture individualAckNormal(CommandAck ack, Map individualAckWithTransaction(CommandAck ack) { // Individual ack - List> positionsAcked = new ArrayList<>(); + List>> positionsAcked = new ArrayList<>(); if (!isTransactionEnabled()) { return FutureUtil.failedFuture( new BrokerServiceException.NotAllowedException("Server don't support transaction ack!")); @@ -551,33 +606,36 @@ private CompletableFuture individualAckWithTransaction(CommandAck ack) { LongAdder totalAckCount = new LongAdder(); for (int i = 0; i < ack.getMessageIdsCount(); i++) { MessageIdData msgId = ack.getMessageIdAt(i); - PositionImpl position = PositionImpl.get(msgId.getLedgerId(), msgId.getEntryId()); + Position position = AckSetStateUtil.createPositionWithAckSet(msgId.getLedgerId(), msgId.getEntryId(), null); + Consumer ackOwnerConsumer = getAckOwnerConsumerAndBatchSize(msgId.getLedgerId(), + msgId.getEntryId()).getLeft(); // acked count at least one - long ackedCount = 0; - long batchSize = 0; + long ackedCount; + long batchSize; if (msgId.hasBatchSize()) { batchSize = msgId.getBatchSize(); // ack batch messages set ackeCount = batchSize ackedCount = msgId.getBatchSize(); - positionsAcked.add(new MutablePair<>(position, msgId.getBatchSize())); + positionsAcked.add(Pair.of(ackOwnerConsumer, new MutablePair<>(position, msgId.getBatchSize()))); } else { // ack no batch message set ackedCount = 1 + batchSize = 0; ackedCount = 1; - positionsAcked.add(new MutablePair<>(position, (int) batchSize)); + positionsAcked.add(Pair.of(ackOwnerConsumer, new MutablePair<>(position, (int) batchSize))); } - Consumer ackOwnerConsumer = getAckOwnerConsumer(msgId.getLedgerId(), msgId.getEntryId()); + if (msgId.getAckSetsCount() > 0) { long[] ackSets = new long[msgId.getAckSetsCount()]; for (int j = 0; j < msgId.getAckSetsCount(); j++) { ackSets[j] = msgId.getAckSetAt(j); } - position.setAckSet(ackSets); + AckSetStateUtil.getAckSetState(position).setAckSet(ackSets); ackedCount = getAckedCountForTransactionAck(batchSize, ackSets); } addAndGetUnAckedMsgs(ackOwnerConsumer, -(int) ackedCount); - checkCanRemovePendingAcksAndHandle(position, msgId); + checkCanRemovePendingAcksAndHandle(ackOwnerConsumer, position, msgId); checkAckValidationError(ack, position); @@ -585,14 +643,16 @@ private CompletableFuture individualAckWithTransaction(CommandAck ack) { } CompletableFuture completableFuture = transactionIndividualAcknowledge(ack.getTxnidMostBits(), - ack.getTxnidLeastBits(), positionsAcked); + ack.getTxnidLeastBits(), positionsAcked.stream().map(Pair::getRight).collect(Collectors.toList())); if (Subscription.isIndividualAckMode(subType)) { completableFuture.whenComplete((v, e) -> - positionsAcked.forEach(positionLongMutablePair -> { - if (positionLongMutablePair.getLeft().getAckSet() != null) { + positionsAcked.forEach(positionPair -> { + Consumer ackOwnerConsumer = positionPair.getLeft(); + MutablePair positionLongMutablePair = positionPair.getRight(); + if (AckSetStateUtil.hasAckSet(positionLongMutablePair.getLeft())) { if (((PersistentSubscription) subscription) .checkIsCanDeleteConsumerPendingAck(positionLongMutablePair.left)) { - removePendingAcks(positionLongMutablePair.left); + removePendingAcks(ackOwnerConsumer, positionLongMutablePair.left); } } })); @@ -600,25 +660,7 @@ private CompletableFuture individualAckWithTransaction(CommandAck ack) { return completableFuture.thenApply(__ -> totalAckCount.sum()); } - private long getBatchSize(MessageIdData msgId) { - long batchSize = 1; - if (Subscription.isIndividualAckMode(subType)) { - LongPair longPair = pendingAcks.get(msgId.getLedgerId(), msgId.getEntryId()); - // Consumer may ack the msg that not belongs to it. - if (longPair == null) { - Consumer ackOwnerConsumer = getAckOwnerConsumer(msgId.getLedgerId(), msgId.getEntryId()); - longPair = ackOwnerConsumer.getPendingAcks().get(msgId.getLedgerId(), msgId.getEntryId()); - if (longPair != null) { - batchSize = longPair.first; - } - } else { - batchSize = longPair.first; - } - } - return batchSize; - } - - private long getAckedCountForMsgIdNoAckSets(long batchSize, PositionImpl position, Consumer consumer) { + private long getAckedCountForMsgIdNoAckSets(long batchSize, Position position, Consumer consumer) { if (isAcknowledgmentAtBatchIndexLevelEnabled && Subscription.isIndividualAckMode(subType)) { long[] cursorAckSet = getCursorAckSet(position); if (cursorAckSet != null) { @@ -628,7 +670,7 @@ private long getAckedCountForMsgIdNoAckSets(long batchSize, PositionImpl positio return batchSize; } - private long getAckedCountForBatchIndexLevelEnabled(PositionImpl position, long batchSize, long[] ackSets, + private long getAckedCountForBatchIndexLevelEnabled(Position position, long batchSize, long[] ackSets, Consumer consumer) { long ackedCount = 0; if (isAcknowledgmentAtBatchIndexLevelEnabled && Subscription.isIndividualAckMode(subType) @@ -657,7 +699,7 @@ private long getAckedCountForTransactionAck(long batchSize, long[] ackSets) { return ackedCount; } - private long getUnAckedCountForBatchIndexLevelEnabled(PositionImpl position, long batchSize) { + private long getUnAckedCountForBatchIndexLevelEnabled(Position position, long batchSize) { long unAckedCount = batchSize; if (isAcknowledgmentAtBatchIndexLevelEnabled) { long[] cursorAckSet = getCursorAckSet(position); @@ -670,35 +712,49 @@ private long getUnAckedCountForBatchIndexLevelEnabled(PositionImpl position, lon return unAckedCount; } - private void checkAckValidationError(CommandAck ack, PositionImpl position) { + private void checkAckValidationError(CommandAck ack, Position position) { if (ack.hasValidationError()) { log.error("[{}] [{}] Received ack for corrupted message at {} - Reason: {}", subscription, consumerId, position, ack.getValidationError()); } } - private void checkCanRemovePendingAcksAndHandle(PositionImpl position, MessageIdData msgId) { + private boolean checkCanRemovePendingAcksAndHandle(Consumer ackOwnedConsumer, + Position position, MessageIdData msgId) { if (Subscription.isIndividualAckMode(subType) && msgId.getAckSetsCount() == 0) { - removePendingAcks(position); + return removePendingAcks(ackOwnedConsumer, position); } + return false; } - private Consumer getAckOwnerConsumer(long ledgerId, long entryId) { - Consumer ackOwnerConsumer = this; + /** + * Retrieves the acknowledgment owner consumer and batch size for the specified ledgerId and entryId. + * + * @param ledgerId The ID of the ledger. + * @param entryId The ID of the entry. + * @return Pair + */ + private Pair getAckOwnerConsumerAndBatchSize(long ledgerId, long entryId) { if (Subscription.isIndividualAckMode(subType)) { - if (!getPendingAcks().containsKey(ledgerId, entryId)) { + LongPair longPair = getPendingAcks().get(ledgerId, entryId); + if (longPair != null) { + return Pair.of(this, longPair.first); + } else { + // If there are more consumers, this step will consume more CPU, and it should be optimized later. for (Consumer consumer : subscription.getConsumers()) { - if (consumer != this && consumer.getPendingAcks().containsKey(ledgerId, entryId)) { - ackOwnerConsumer = consumer; - break; + if (consumer != this) { + longPair = consumer.getPendingAcks().get(ledgerId, entryId); + if (longPair != null) { + return Pair.of(consumer, longPair.first); + } } } } } - return ackOwnerConsumer; + return Pair.of(this, 1L); } - private long[] getCursorAckSet(PositionImpl position) { + private long[] getCursorAckSet(Position position) { if (!(subscription instanceof PersistentSubscription)) { return null; } @@ -714,7 +770,7 @@ private boolean isTransactionEnabled() { private CompletableFuture transactionIndividualAcknowledge( long txnidMostBits, long txnidLeastBits, - List> positionList) { + List> positionList) { if (subscription instanceof PersistentSubscription) { TxnID txnID = new TxnID(txnidMostBits, txnidLeastBits); return ((PersistentSubscription) subscription).transactionIndividualAcknowledge(txnID, positionList); @@ -726,7 +782,7 @@ private CompletableFuture transactionIndividualAcknowledge( } private CompletableFuture transactionCumulativeAcknowledge(long txnidMostBits, long txnidLeastBits, - List positionList) { + List positionList) { if (!isTransactionEnabled()) { return FutureUtil.failedFuture( new BrokerServiceException.NotAllowedException("Server don't support transaction ack!")); @@ -816,6 +872,21 @@ public void topicMigrated(Optional clusterUrl) { } } + public boolean checkAndApplyTopicMigration() { + if (subscription.isSubscriptionMigrated()) { + Optional clusterUrl = AbstractTopic.getMigratedClusterUrl(cnx.getBrokerService().getPulsar(), + topicName); + if (clusterUrl.isPresent()) { + ClusterUrl url = clusterUrl.get(); + cnx.getCommandSender().sendTopicMigrated(ResourceType.Consumer, consumerId, url.getBrokerServiceUrl(), + url.getBrokerServiceUrlTls()); + // disconnect consumer after sending migrated cluster url + disconnect(); + return true; + } + } + return false; + } /** * Checks if consumer-blocking on unAckedMessages is allowed for below conditions:
* a. consumer must have Shared-subscription
@@ -867,8 +938,8 @@ public ConsumerStatsImpl getStats() { stats.unackedMessages = unackedMessages; stats.blockedConsumerOnUnackedMsgs = blockedConsumerOnUnackedMsgs; stats.avgMessagesPerEntry = getAvgMessagesPerEntry(); - if (readPositionWhenJoining != null) { - stats.readPositionWhenJoining = readPositionWhenJoining.toString(); + if (lastSentPositionWhenJoining != null) { + stats.lastSentPositionWhenJoining = lastSentPositionWhenJoining.toString(); } return stats; } @@ -881,6 +952,14 @@ public long getBytesOutCounter() { return bytesOutCounter.longValue(); } + public long getMessageAckCounter() { + return messageAckCounter.sum(); + } + + public long getMessageRedeliverCounter() { + return msgRedeliverCounter.sum(); + } + public int getUnackedMessages() { return unackedMessages; } @@ -893,7 +972,7 @@ public KeySharedMeta getKeySharedMeta() { public String toString() { if (subscription != null && cnx != null) { return MoreObjects.toStringHelper(this).add("subscription", subscription).add("consumerId", consumerId) - .add("consumerName", consumerName).add("address", this.cnx.clientAddress()).toString(); + .add("consumerName", consumerName).add("address", this.cnx.toString()).toString(); } else { return MoreObjects.toStringHelper(this).add("consumerId", consumerId) .add("consumerName", consumerName).toString(); @@ -947,42 +1026,24 @@ public int hashCode() { * * @param position */ - private void removePendingAcks(PositionImpl position) { - Consumer ackOwnedConsumer = null; - if (pendingAcks.get(position.getLedgerId(), position.getEntryId()) == null) { - for (Consumer consumer : subscription.getConsumers()) { - if (!consumer.equals(this) && consumer.getPendingAcks().containsKey(position.getLedgerId(), - position.getEntryId())) { - ackOwnedConsumer = consumer; - break; - } - } - } else { - ackOwnedConsumer = this; + private boolean removePendingAcks(Consumer ackOwnedConsumer, Position position) { + if (!ackOwnedConsumer.getPendingAcks().remove(position.getLedgerId(), position.getEntryId())) { + // Message was already removed by the other consumer + return false; } - - // remove pending message from appropriate consumer and unblock unAckMsg-flow if requires - LongPair ackedPosition = ackOwnedConsumer != null - ? ackOwnedConsumer.getPendingAcks().get(position.getLedgerId(), position.getEntryId()) - : null; - if (ackedPosition != null) { - if (!ackOwnedConsumer.getPendingAcks().remove(position.getLedgerId(), position.getEntryId())) { - // Message was already removed by the other consumer - return; - } - if (log.isDebugEnabled()) { - log.debug("[{}-{}] consumer {} received ack {}", topicName, subscription, consumerId, position); - } - // unblock consumer-throttling when limit check is disabled or receives half of maxUnackedMessages => - // consumer can start again consuming messages - int unAckedMsgs = UNACKED_MESSAGES_UPDATER.get(ackOwnedConsumer); - if ((((unAckedMsgs <= getMaxUnackedMessages() / 2) && ackOwnedConsumer.blockedConsumerOnUnackedMsgs) - && ackOwnedConsumer.shouldBlockConsumerOnUnackMsgs()) - || !shouldBlockConsumerOnUnackMsgs()) { - ackOwnedConsumer.blockedConsumerOnUnackedMsgs = false; - flowConsumerBlockedPermits(ackOwnedConsumer); - } + if (log.isDebugEnabled()) { + log.debug("[{}-{}] consumer {} received ack {}", topicName, subscription, consumerId, position); } + // unblock consumer-throttling when limit check is disabled or receives half of maxUnackedMessages => + // consumer can start again consuming messages + int unAckedMsgs = UNACKED_MESSAGES_UPDATER.get(ackOwnedConsumer); + if ((((unAckedMsgs <= getMaxUnackedMessages() / 2) && ackOwnedConsumer.blockedConsumerOnUnackedMsgs) + && ackOwnedConsumer.shouldBlockConsumerOnUnackMsgs()) + || !shouldBlockConsumerOnUnackMsgs()) { + ackOwnedConsumer.blockedConsumerOnUnackedMsgs = false; + flowConsumerBlockedPermits(ackOwnedConsumer); + } + return true; } public ConcurrentLongLongPairHashMap getPendingAcks() { @@ -1002,20 +1063,23 @@ public void redeliverUnacknowledgedMessages(long consumerEpoch) { } if (pendingAcks != null) { - List pendingPositions = new ArrayList<>((int) pendingAcks.size()); + List pendingPositions = new ArrayList<>((int) pendingAcks.size()); MutableInt totalRedeliveryMessages = new MutableInt(0); pendingAcks.forEach((ledgerId, entryId, batchSize, stickyKeyHash) -> { - int unAckedCount = (int) getUnAckedCountForBatchIndexLevelEnabled(PositionImpl.get(ledgerId, entryId), - batchSize); + int unAckedCount = + (int) getUnAckedCountForBatchIndexLevelEnabled(PositionFactory.create(ledgerId, entryId), + batchSize); totalRedeliveryMessages.add(unAckedCount); - pendingPositions.add(new PositionImpl(ledgerId, entryId)); + pendingPositions.add(PositionFactory.create(ledgerId, entryId)); }); - for (PositionImpl p : pendingPositions) { + for (Position p : pendingPositions) { pendingAcks.remove(p.getLedgerId(), p.getEntryId()); } msgRedeliver.recordMultipleEvents(totalRedeliveryMessages.intValue(), totalRedeliveryMessages.intValue()); + msgRedeliverCounter.add(totalRedeliveryMessages.intValue()); + subscription.redeliverUnacknowledgedMessages(this, pendingPositions); } else { subscription.redeliverUnacknowledgedMessages(this, consumerEpoch); @@ -1026,9 +1090,9 @@ public void redeliverUnacknowledgedMessages(long consumerEpoch) { public void redeliverUnacknowledgedMessages(List messageIds) { int totalRedeliveryMessages = 0; - List pendingPositions = new ArrayList<>(); + List pendingPositions = new ArrayList<>(); for (MessageIdData msg : messageIds) { - PositionImpl position = PositionImpl.get(msg.getLedgerId(), msg.getEntryId()); + Position position = PositionFactory.create(msg.getLedgerId(), msg.getEntryId()); LongPair longPair = pendingAcks.get(position.getLedgerId(), position.getEntryId()); if (longPair != null) { int unAckedCount = (int) getUnAckedCountForBatchIndexLevelEnabled(position, longPair.first); @@ -1048,6 +1112,7 @@ public void redeliverUnacknowledgedMessages(List messageIds) { subscription.redeliverUnacknowledgedMessages(this, pendingPositions); msgRedeliver.recordMultipleEvents(totalRedeliveryMessages, totalRedeliveryMessages); + msgRedeliverCounter.add(totalRedeliveryMessages); int numberOfBlockedPermits = PERMITS_RECEIVED_WHILE_CONSUMER_BLOCKED_UPDATER.getAndSet(this, 0); @@ -1068,12 +1133,12 @@ public Subscription getSubscription() { private int addAndGetUnAckedMsgs(Consumer consumer, int ackedMessages) { int unackedMsgs = 0; - if (Subscription.isIndividualAckMode(subType)) { + if (isPersistentTopic && Subscription.isIndividualAckMode(subType)) { subscription.addUnAckedMessages(ackedMessages); unackedMsgs = UNACKED_MESSAGES_UPDATER.addAndGet(consumer, ackedMessages); } - if (unackedMsgs < 0 && System.currentTimeMillis() - negtiveUnackedMsgsTimestamp >= 10_000) { - negtiveUnackedMsgsTimestamp = System.currentTimeMillis(); + if (unackedMsgs < 0 && System.currentTimeMillis() - negativeUnackedMsgsTimestamp >= 10_000) { + negativeUnackedMsgsTimestamp = System.currentTimeMillis(); log.warn("unackedMsgs is : {}, ackedMessages : {}, consumer : {}", unackedMsgs, ackedMessages, consumer); } return unackedMsgs; @@ -1088,8 +1153,8 @@ public boolean isPreciseDispatcherFlowControl() { return preciseDispatcherFlowControl; } - public void setReadPositionWhenJoining(PositionImpl readPositionWhenJoining) { - this.readPositionWhenJoining = readPositionWhenJoining; + public void setLastSentPositionWhenJoining(Position lastSentPositionWhenJoining) { + this.lastSentPositionWhenJoining = lastSentPositionWhenJoining; } public int getMaxUnackedMessages() { @@ -1110,6 +1175,14 @@ public String getClientAddress() { return clientAddress; } + public String getClientAddressAndPort() { + return cnx.clientSourceAddressAndPort(); + } + + public String getClientVersion() { + return cnx.getClientVersion(); + } + public MessageId getStartMessageId() { return startMessageId; } @@ -1129,4 +1202,30 @@ private int getStickyKeyHash(Entry entry) { } private static final Logger log = LoggerFactory.getLogger(Consumer.class); + + public Attributes getOpenTelemetryAttributes() { + if (openTelemetryAttributes != null) { + return openTelemetryAttributes; + } + return OPEN_TELEMETRY_ATTRIBUTES_FIELD_UPDATER.updateAndGet(this, oldValue -> { + if (oldValue != null) { + return oldValue; + } + var topicName = TopicName.get(subscription.getTopic().getName()); + + var builder = Attributes.builder() + .put(OpenTelemetryAttributes.PULSAR_CONSUMER_NAME, consumerName) + .put(OpenTelemetryAttributes.PULSAR_CONSUMER_ID, consumerId) + .put(OpenTelemetryAttributes.PULSAR_SUBSCRIPTION_NAME, subscription.getName()) + .put(OpenTelemetryAttributes.PULSAR_SUBSCRIPTION_TYPE, subType.toString()) + .put(OpenTelemetryAttributes.PULSAR_DOMAIN, topicName.getDomain().toString()) + .put(OpenTelemetryAttributes.PULSAR_TENANT, topicName.getTenant()) + .put(OpenTelemetryAttributes.PULSAR_NAMESPACE, topicName.getNamespace()) + .put(OpenTelemetryAttributes.PULSAR_TOPIC, topicName.getPartitionedTopicName()); + if (topicName.isPartitioned()) { + builder.put(OpenTelemetryAttributes.PULSAR_PARTITION_INDEX, topicName.getPartitionIndex()); + } + return builder.build(); + }); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsumerIdentityWrapper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsumerIdentityWrapper.java new file mode 100644 index 0000000000000..2aae1d9b0622e --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsumerIdentityWrapper.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +/** + * A wrapper class for a Consumer instance that provides custom implementations + * of equals and hashCode methods. The equals method returns true if and only if + * the compared instance is the same instance. + * + *

The reason for this class is the custom implementation of {@link Consumer#equals(Object)}. + * Using this wrapper class will be useful in use cases where it's necessary to match a key + * in a map by instance or a value in a set by instance.

+ */ +class ConsumerIdentityWrapper { + final Consumer consumer; + + public ConsumerIdentityWrapper(Consumer consumer) { + this.consumer = consumer; + } + + /** + * Compares this wrapper to the specified object. The result is true if and only if + * the argument is not null and is a ConsumerIdentityWrapper object that wraps + * the same Consumer instance. + * + * @param obj the object to compare this ConsumerIdentityWrapper against + * @return true if the given object represents a ConsumerIdentityWrapper + * equivalent to this wrapper, false otherwise + */ + @Override + public boolean equals(Object obj) { + if (obj instanceof ConsumerIdentityWrapper) { + ConsumerIdentityWrapper other = (ConsumerIdentityWrapper) obj; + return consumer == other.consumer; + } + return false; + } + + /** + * Returns a hash code for this wrapper. The hash code is computed based on + * the wrapped Consumer instance. + * + * @return a hash code value for this object + */ + @Override + public int hashCode() { + return consumer.hashCode(); + } + + @Override + public String toString() { + return consumer.toString(); + } +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsumerNameIndexTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsumerNameIndexTracker.java new file mode 100644 index 0000000000000..1f93313ab1b71 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsumerNameIndexTracker.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import java.util.HashMap; +import java.util.Map; +import javax.annotation.concurrent.NotThreadSafe; +import org.apache.commons.lang3.mutable.MutableInt; +import org.roaringbitmap.RoaringBitmap; + +/** + * Tracks the used consumer name indexes for each consumer name. + * This is used by {@link ConsistentHashingStickyKeyConsumerSelector} to get a unique "consumer name index" + * for each consumer name. It is useful when there are multiple consumers with the same name, but they are + * different consumers. The purpose of the index is to prevent collisions in the hash ring. + * + * The consumer name index serves as an additional key for the hash ring assignment. The logic keeps track of + * used "index slots" for each consumer name and assigns the first unused index when a new consumer is added. + * This approach minimizes hash collisions due to using the same consumer name. + * + * An added benefit of this tracking approach is that a consumer that leaves and then rejoins immediately will get the + * same index and therefore the same assignments in the hash ring. This improves stability since the hash assignment + * changes are minimized over time, although a better solution would be to avoid reusing the same consumer name + * in the first place. + * + * When a consumer is removed, the index is deallocated. RoaringBitmap is used to keep track of the used indexes. + * The data structure to track a consumer name is removed when the reference count of the consumer name is zero. + * + * This class is not thread-safe and should be used in a synchronized context in the caller. + */ +@NotThreadSafe +class ConsumerNameIndexTracker { + // tracks the used index slots for each consumer name + private final Map consumerNameIndexSlotsMap = new HashMap<>(); + // tracks the active consumer entries + private final Map consumerEntries = new HashMap<>(); + + // Represents a consumer entry in the tracker, including the consumer name, index, and reference count. + record ConsumerEntry(String consumerName, int nameIndex, MutableInt refCount) { + } + + /* + * Tracks the used indexes for a consumer name using a RoaringBitmap. + * A specific index slot is used when the bit is set. + * When all bits are cleared, the customer name can be removed from tracking. + */ + static class ConsumerNameIndexSlots { + private RoaringBitmap indexSlots = new RoaringBitmap(); + + public int allocateIndexSlot() { + // find the first index that is not set, if there is no such index, add a new one + int index = (int) indexSlots.nextAbsentValue(0); + if (index == -1) { + index = indexSlots.getCardinality(); + } + indexSlots.add(index); + return index; + } + + public boolean deallocateIndexSlot(int index) { + indexSlots.remove(index); + return indexSlots.isEmpty(); + } + } + + /* + * Adds a reference to the consumer and returns the index assigned to this consumer. + */ + public int increaseConsumerRefCountAndReturnIndex(ConsumerIdentityWrapper wrapper) { + ConsumerEntry entry = consumerEntries.computeIfAbsent(wrapper, k -> { + String consumerName = wrapper.consumer.consumerName(); + return new ConsumerEntry(consumerName, allocateConsumerNameIndex(consumerName), new MutableInt(0)); + }); + entry.refCount.increment(); + return entry.nameIndex; + } + + private int allocateConsumerNameIndex(String consumerName) { + return getConsumerNameIndexBitmap(consumerName).allocateIndexSlot(); + } + + private ConsumerNameIndexSlots getConsumerNameIndexBitmap(String consumerName) { + return consumerNameIndexSlotsMap.computeIfAbsent(consumerName, k -> new ConsumerNameIndexSlots()); + } + + /* + * Decreases the reference count of the consumer and removes the consumer name from tracking if the ref count is + * zero. + */ + public void decreaseConsumerRefCount(ConsumerIdentityWrapper removed) { + ConsumerEntry consumerEntry = consumerEntries.get(removed); + int refCount = consumerEntry.refCount.decrementAndGet(); + if (refCount == 0) { + deallocateConsumerNameIndex(consumerEntry.consumerName, consumerEntry.nameIndex); + consumerEntries.remove(removed, consumerEntry); + } + } + + private void deallocateConsumerNameIndex(String consumerName, int index) { + if (getConsumerNameIndexBitmap(consumerName).deallocateIndexSlot(index)) { + consumerNameIndexSlotsMap.remove(consumerName); + } + } + + /* + * Returns the currently tracked index for the consumer. + */ + public int getTrackedIndex(ConsumerIdentityWrapper wrapper) { + ConsumerEntry consumerEntry = consumerEntries.get(wrapper); + return consumerEntry != null ? consumerEntry.nameIndex : -1; + } + + int getTrackedConsumerNamesCount() { + return consumerNameIndexSlotsMap.size(); + } + + int getTrackedConsumersCount() { + return consumerEntries.size(); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java index 3ca06dc83d9aa..d1d44709a9c52 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java @@ -16,12 +16,15 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.pulsar.broker.service; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.Position; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.api.proto.MessageMetadata; @@ -49,7 +52,11 @@ public interface Dispatcher { * * @return */ - CompletableFuture close(); + default CompletableFuture close() { + return close(true, Optional.empty()); + } + + CompletableFuture close(boolean disconnectClients, Optional assignedBrokerLookupData); boolean isClosed(); @@ -63,12 +70,17 @@ public interface Dispatcher { * * @return */ - CompletableFuture disconnectAllConsumers(boolean isResetCursor); + default CompletableFuture disconnectAllConsumers(boolean isResetCursor) { + return disconnectAllConsumers(isResetCursor, Optional.empty()); + } default CompletableFuture disconnectAllConsumers() { return disconnectAllConsumers(false); } + CompletableFuture disconnectAllConsumers(boolean isResetCursor, + Optional assignedBrokerLookupData); + void resetCloseFuture(); /** @@ -80,7 +92,7 @@ default CompletableFuture disconnectAllConsumers() { void redeliverUnacknowledgedMessages(Consumer consumer, long consumerEpoch); - void redeliverUnacknowledgedMessages(Consumer consumer, List positions); + void redeliverUnacknowledgedMessages(Consumer consumer, List positions); void addUnAckedMessages(int unAckMessages); @@ -91,7 +103,8 @@ default Optional getRateLimiter() { } default void updateRateLimiter() { - //No-op + initializeDispatchRateLimiterIfNeeded(); + getRateLimiter().ifPresent(DispatchRateLimiter::updateDispatchRate); } default boolean initializeDispatchRateLimiterIfNeeded() { @@ -129,6 +142,24 @@ default boolean checkAndUnblockIfStuck() { return false; } + /** + * A callback hook after acknowledge messages. + * @param exOfDeletion the ex of {@link org.apache.bookkeeper.mledger.ManagedCursor#asyncDelete}, + * {@link ManagedCursor#asyncClearBacklog} or {@link ManagedCursor#asyncSkipEntries)}. + * @param ctxOfDeletion the param ctx of calling {@link org.apache.bookkeeper.mledger.ManagedCursor#asyncDelete}, + * {@link ManagedCursor#asyncClearBacklog} or {@link ManagedCursor#asyncSkipEntries)}. + */ + default void afterAckMessages(Throwable exOfDeletion, Object ctxOfDeletion){} + + /** + * Trigger a new "readMoreEntries" if the dispatching has been paused before. This method is only implemented in + * {@link org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers} right now, other + * implements are not necessary to implement this method. + * @return did a resume. + */ + default boolean checkAndResumeIfPaused(){ + return false; + } default long getFilterProcessedMsgCount() { return 0; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/EntryAndMetadata.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/EntryAndMetadata.java index 70643d5de2a3f..efa89a8ff16f6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/EntryAndMetadata.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/EntryAndMetadata.java @@ -55,6 +55,9 @@ public byte[] getStickyKey() { return metadata.getOrderingKey(); } else if (metadata.hasPartitionKey()) { return metadata.getPartitionKey().getBytes(StandardCharsets.UTF_8); + } else if (metadata.hasProducerName() && metadata.hasSequenceId()) { + String fallbackKey = metadata.getProducerName() + "-" + metadata.getSequenceId(); + return fallbackKey.getBytes(StandardCharsets.UTF_8); } } return "NONE_KEY".getBytes(StandardCharsets.UTF_8); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/EntryFilterSupport.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/EntryFilterSupport.java index 4a9b33a9afdb1..03d6f0750e02e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/EntryFilterSupport.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/EntryFilterSupport.java @@ -35,7 +35,8 @@ public class EntryFilterSupport { public EntryFilterSupport(Subscription subscription) { this.subscription = subscription; - if (subscription != null && subscription.getTopic() != null) { + if (subscription != null && subscription.getTopic() != null + && !subscription.getTopic().isSystemTopic()) { final BrokerService brokerService = subscription.getTopic().getBrokerService(); final boolean allowOverrideEntryFilters = brokerService .pulsar().getConfiguration().isAllowOverrideEntryFilters(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/GetStatsOptions.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/GetStatsOptions.java new file mode 100644 index 0000000000000..ec239a6c14172 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/GetStatsOptions.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +@AllArgsConstructor +public class GetStatsOptions { + /** + * Set to true to get precise backlog, Otherwise get imprecise backlog. + */ + private final boolean getPreciseBacklog; + + /** + * Whether to get backlog size for each subscription. + */ + private final boolean subscriptionBacklogSize; + + /** + * Whether to get the earliest time in backlog. + */ + private final boolean getEarliestTimeInBacklog; + + /** + * Whether to exclude publishers. + */ + private final boolean excludePublishers; + + /** + * Whether to exclude consumers. + */ + private final boolean excludeConsumers; +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/InMemoryRedeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/InMemoryRedeliveryTracker.java index 8c992d2f7a90b..12e28793557b3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/InMemoryRedeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/InMemoryRedeliveryTracker.java @@ -20,7 +20,6 @@ import java.util.List; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap; import org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap.LongPair; @@ -34,7 +33,7 @@ public class InMemoryRedeliveryTracker implements RedeliveryTracker { @Override public int incrementAndGetRedeliveryCount(Position position) { - PositionImpl positionImpl = (PositionImpl) position; + Position positionImpl = position; LongPair count = trackerCache.get(positionImpl.getLedgerId(), positionImpl.getEntryId()); int newCount = (int) (count != null ? count.first + 1 : 1); trackerCache.put(positionImpl.getLedgerId(), positionImpl.getEntryId(), newCount, 0L); @@ -49,7 +48,7 @@ public int getRedeliveryCount(long ledgerId, long entryId) { @Override public void remove(Position position) { - PositionImpl positionImpl = (PositionImpl) position; + Position positionImpl = position; trackerCache.remove(positionImpl.getLedgerId(), positionImpl.getEntryId()); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/MessageExpirer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/MessageExpirer.java new file mode 100644 index 0000000000000..7cb1d2a904aa4 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/MessageExpirer.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import org.apache.bookkeeper.mledger.Position; +import org.apache.pulsar.common.classification.InterfaceStability; + +@InterfaceStability.Evolving +public interface MessageExpirer { + + boolean expireMessages(Position position); + + boolean expireMessages(int messageTTLInSeconds); +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PersistentTopicAttributes.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PersistentTopicAttributes.java new file mode 100644 index 0000000000000..51f5bdb354dc9 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PersistentTopicAttributes.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import io.opentelemetry.api.common.Attributes; +import lombok.Getter; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; + +@Getter +public class PersistentTopicAttributes extends TopicAttributes { + + private final Attributes timeBasedQuotaAttributes; + private final Attributes sizeBasedQuotaAttributes; + + private final Attributes compactionSuccessAttributes; + private final Attributes compactionFailureAttributes; + + private final Attributes transactionActiveAttributes; + private final Attributes transactionCommittedAttributes; + private final Attributes transactionAbortedAttributes; + + private final Attributes transactionBufferClientCommitSucceededAttributes; + private final Attributes transactionBufferClientCommitFailedAttributes; + private final Attributes transactionBufferClientAbortSucceededAttributes; + private final Attributes transactionBufferClientAbortFailedAttributes; + + public PersistentTopicAttributes(TopicName topicName) { + super(topicName); + + timeBasedQuotaAttributes = Attributes.builder() + .putAll(commonAttributes) + .putAll(OpenTelemetryAttributes.BacklogQuotaType.TIME.attributes) + .build(); + sizeBasedQuotaAttributes = Attributes.builder() + .putAll(commonAttributes) + .putAll(OpenTelemetryAttributes.BacklogQuotaType.SIZE.attributes) + .build(); + + transactionActiveAttributes = Attributes.builder() + .putAll(commonAttributes) + .putAll(OpenTelemetryAttributes.TransactionStatus.ACTIVE.attributes) + .build(); + transactionCommittedAttributes = Attributes.builder() + .putAll(commonAttributes) + .putAll(OpenTelemetryAttributes.TransactionStatus.COMMITTED.attributes) + .build(); + transactionAbortedAttributes = Attributes.builder() + .putAll(commonAttributes) + .putAll(OpenTelemetryAttributes.TransactionStatus.ABORTED.attributes) + .build(); + + transactionBufferClientCommitSucceededAttributes = Attributes.builder() + .putAll(commonAttributes) + .remove(OpenTelemetryAttributes.PULSAR_DOMAIN) + .putAll(OpenTelemetryAttributes.TransactionStatus.COMMITTED.attributes) + .putAll(OpenTelemetryAttributes.TransactionBufferClientOperationStatus.SUCCESS.attributes) + .build(); + transactionBufferClientCommitFailedAttributes = Attributes.builder() + .putAll(commonAttributes) + .remove(OpenTelemetryAttributes.PULSAR_DOMAIN) + .putAll(OpenTelemetryAttributes.TransactionStatus.COMMITTED.attributes) + .putAll(OpenTelemetryAttributes.TransactionBufferClientOperationStatus.FAILURE.attributes) + .build(); + transactionBufferClientAbortSucceededAttributes = Attributes.builder() + .putAll(commonAttributes) + .remove(OpenTelemetryAttributes.PULSAR_DOMAIN) + .putAll(OpenTelemetryAttributes.TransactionStatus.ABORTED.attributes) + .putAll(OpenTelemetryAttributes.TransactionBufferClientOperationStatus.SUCCESS.attributes) + .build(); + transactionBufferClientAbortFailedAttributes = Attributes.builder() + .putAll(commonAttributes) + .remove(OpenTelemetryAttributes.PULSAR_DOMAIN) + .putAll(OpenTelemetryAttributes.TransactionStatus.ABORTED.attributes) + .putAll(OpenTelemetryAttributes.TransactionBufferClientOperationStatus.FAILURE.attributes) + .build(); + + compactionSuccessAttributes = Attributes.builder() + .putAll(commonAttributes) + .putAll(OpenTelemetryAttributes.CompactionStatus.SUCCESS.attributes) + .build(); + compactionFailureAttributes = Attributes.builder() + .putAll(commonAttributes) + .putAll(OpenTelemetryAttributes.CompactionStatus.FAILURE.attributes) + .build(); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisePublishLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisePublishLimiter.java deleted file mode 100644 index ce14f6d7dd71e..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisePublishLimiter.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.service; - -import java.util.concurrent.ScheduledExecutorService; -import org.apache.pulsar.common.policies.data.Policies; -import org.apache.pulsar.common.policies.data.PublishRate; -import org.apache.pulsar.common.util.RateLimitFunction; -import org.apache.pulsar.common.util.RateLimiter; - -public class PrecisePublishLimiter implements PublishRateLimiter { - protected volatile int publishMaxMessageRate = 0; - protected volatile long publishMaxByteRate = 0; - // precise mode for publish rate limiter - private volatile RateLimiter topicPublishRateLimiterOnMessage; - private volatile RateLimiter topicPublishRateLimiterOnByte; - private final RateLimitFunction rateLimitFunction; - private final ScheduledExecutorService scheduledExecutorService; - - public PrecisePublishLimiter(Policies policies, String clusterName, RateLimitFunction rateLimitFunction) { - this.rateLimitFunction = rateLimitFunction; - update(policies, clusterName); - this.scheduledExecutorService = null; - } - - public PrecisePublishLimiter(PublishRate publishRate, RateLimitFunction rateLimitFunction) { - this(publishRate, rateLimitFunction, null); - } - - public PrecisePublishLimiter(PublishRate publishRate, RateLimitFunction rateLimitFunction, - ScheduledExecutorService scheduledExecutorService) { - this.rateLimitFunction = rateLimitFunction; - update(publishRate); - this.scheduledExecutorService = scheduledExecutorService; - } - - @Override - public void checkPublishRate() { - // No-op - } - - @Override - public void incrementPublishCount(int numOfMessages, long msgSizeInBytes) { - // No-op - } - - @Override - public boolean resetPublishCount() { - return true; - } - - @Override - public boolean isPublishRateExceeded() { - return false; - } - - // If all rate limiters are not exceeded, re-enable auto read from socket. - private void tryReleaseConnectionThrottle() { - RateLimiter currentTopicPublishRateLimiterOnMessage = topicPublishRateLimiterOnMessage; - RateLimiter currentTopicPublishRateLimiterOnByte = topicPublishRateLimiterOnByte; - if ((currentTopicPublishRateLimiterOnMessage != null - && currentTopicPublishRateLimiterOnMessage.getAvailablePermits() <= 0) - || (currentTopicPublishRateLimiterOnByte != null - && currentTopicPublishRateLimiterOnByte.getAvailablePermits() <= 0)) { - return; - } - this.rateLimitFunction.apply(); - } - - @Override - public void update(Policies policies, String clusterName) { - final PublishRate maxPublishRate = policies.publishMaxMessageRate != null - ? policies.publishMaxMessageRate.get(clusterName) - : null; - this.update(maxPublishRate); - } - - public void update(PublishRate maxPublishRate) { - replaceLimiters(() -> { - if (maxPublishRate != null - && (maxPublishRate.publishThrottlingRateInMsg > 0 - || maxPublishRate.publishThrottlingRateInByte > 0)) { - this.publishMaxMessageRate = Math.max(maxPublishRate.publishThrottlingRateInMsg, 0); - this.publishMaxByteRate = Math.max(maxPublishRate.publishThrottlingRateInByte, 0); - if (this.publishMaxMessageRate > 0) { - topicPublishRateLimiterOnMessage = - RateLimiter.builder() - .scheduledExecutorService(scheduledExecutorService) - .permits(publishMaxMessageRate) - .rateLimitFunction(this::tryReleaseConnectionThrottle) - .isDispatchOrPrecisePublishRateLimiter(true) - .build(); - } - if (this.publishMaxByteRate > 0) { - topicPublishRateLimiterOnByte = RateLimiter.builder() - .scheduledExecutorService(scheduledExecutorService) - .permits(publishMaxByteRate) - .rateLimitFunction(this::tryReleaseConnectionThrottle) - .isDispatchOrPrecisePublishRateLimiter(true) - .build(); - } - } else { - this.publishMaxMessageRate = 0; - this.publishMaxByteRate = 0; - } - }); - } - - @Override - public boolean tryAcquire(int numbers, long bytes) { - RateLimiter currentTopicPublishRateLimiterOnMessage = topicPublishRateLimiterOnMessage; - RateLimiter currentTopicPublishRateLimiterOnByte = topicPublishRateLimiterOnByte; - return (currentTopicPublishRateLimiterOnMessage == null - || currentTopicPublishRateLimiterOnMessage.tryAcquire(numbers)) - && (currentTopicPublishRateLimiterOnByte == null - || currentTopicPublishRateLimiterOnByte.tryAcquire(bytes)); - } - - @Override - public void close() { - rateLimitFunction.apply(); - replaceLimiters(null); - } - - private void replaceLimiters(Runnable updater) { - RateLimiter previousTopicPublishRateLimiterOnMessage = topicPublishRateLimiterOnMessage; - topicPublishRateLimiterOnMessage = null; - RateLimiter previousTopicPublishRateLimiterOnByte = topicPublishRateLimiterOnByte; - topicPublishRateLimiterOnByte = null; - try { - if (updater != null) { - updater.run(); - } - } finally { - // Close previous limiters to prevent resource leakages. - // Delay closing of previous limiters after new ones are in place so that updating the limiter - // doesn't cause unavailability. - if (previousTopicPublishRateLimiterOnMessage != null) { - previousTopicPublishRateLimiterOnMessage.close(); - } - if (previousTopicPublishRateLimiterOnByte != null) { - previousTopicPublishRateLimiterOnByte.close(); - } - } - } -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java index 53b79f06e8e24..b4578711027ef 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java @@ -23,10 +23,12 @@ import static org.apache.pulsar.common.protocol.Commands.hasChecksum; import static org.apache.pulsar.common.protocol.Commands.readChecksum; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.CaseFormat; import com.google.common.base.MoreObjects; import io.netty.buffer.ByteBuf; import io.netty.util.Recycler; import io.netty.util.Recycler.Handle; +import io.opentelemetry.api.common.Attributes; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -36,9 +38,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLongFieldUpdater; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import org.apache.bookkeeper.mledger.Position; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.intercept.BrokerInterceptor; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.service.BrokerServiceException.TopicClosedException; import org.apache.pulsar.broker.service.BrokerServiceException.TopicTerminatedException; import org.apache.pulsar.broker.service.Topic.PublishContext; @@ -50,14 +54,14 @@ import org.apache.pulsar.common.api.proto.ProducerAccessMode; import org.apache.pulsar.common.api.proto.ServerError; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.ClusterData.ClusterUrl; +import org.apache.pulsar.common.policies.data.ClusterPolicies.ClusterUrl; import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.policies.data.stats.NonPersistentPublisherStatsImpl; import org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.schema.SchemaVersion; -import org.apache.pulsar.common.stats.Rate; import org.apache.pulsar.common.util.DateFormatter; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,10 +77,6 @@ public class Producer { private final long producerId; private final String appId; private final BrokerInterceptor brokerInterceptor; - private Rate msgIn; - private Rate chunkedMessageRate; - // it records msg-drop rate only for non-persistent topic - private final Rate msgDrop; private volatile long pendingPublishAcks = 0; private static final AtomicLongFieldUpdater pendingPublishAcksUpdater = AtomicLongFieldUpdater @@ -86,6 +86,10 @@ public class Producer { private final CompletableFuture closeFuture; private final PublisherStatsImpl stats; + private volatile Attributes attributes = null; + private static final AtomicReferenceFieldUpdater ATTRIBUTES_FIELD_UPDATER = + AtomicReferenceFieldUpdater.newUpdater(Producer.class, Attributes.class, "attributes"); + private final boolean isRemote; private final String remoteCluster; private final boolean isNonPersistentTopic; @@ -117,21 +121,14 @@ public Producer(Topic topic, TransportCnx cnx, long producerId, String producerN this.epoch = epoch; this.closeFuture = new CompletableFuture<>(); this.appId = appId; - this.msgIn = new Rate(); - this.chunkedMessageRate = new Rate(); this.isNonPersistentTopic = topic instanceof NonPersistentTopic; - this.msgDrop = this.isNonPersistentTopic ? new Rate() : null; this.isShadowTopic = topic instanceof PersistentTopic && ((PersistentTopic) topic).getShadowSourceTopic().isPresent(); this.metadata = metadata != null ? metadata : Collections.emptyMap(); this.stats = isNonPersistentTopic ? new NonPersistentPublisherStatsImpl() : new PublisherStatsImpl(); - if (cnx.hasHAProxyMessage()) { - stats.setAddress(cnx.getHAProxyMessage().sourceAddress() + ":" + cnx.getHAProxyMessage().sourcePort()); - } else { - stats.setAddress(cnx.clientAddress().toString()); - } + stats.setAddress(cnx.clientSourceAddressAndPort()); stats.setConnectedSince(DateFormatter.now()); stats.setClientVersion(cnx.getClientVersion()); stats.setProducerName(producerName); @@ -273,7 +270,7 @@ public boolean checkAndStartPublish(long producerId, long sequenceId, ByteBuf he private void publishMessageToTopic(ByteBuf headersAndPayload, long sequenceId, long batchSize, boolean isChunked, boolean isMarker, Position position) { MessagePublishContext messagePublishContext = - MessagePublishContext.get(this, sequenceId, msgIn, headersAndPayload.readableBytes(), + MessagePublishContext.get(this, sequenceId, headersAndPayload.readableBytes(), batchSize, isChunked, System.nanoTime(), isMarker, position); if (brokerInterceptor != null) { brokerInterceptor @@ -285,7 +282,7 @@ private void publishMessageToTopic(ByteBuf headersAndPayload, long sequenceId, l private void publishMessageToTopic(ByteBuf headersAndPayload, long lowestSequenceId, long highestSequenceId, long batchSize, boolean isChunked, boolean isMarker, Position position) { MessagePublishContext messagePublishContext = MessagePublishContext.get(this, lowestSequenceId, - highestSequenceId, msgIn, headersAndPayload.readableBytes(), batchSize, + highestSequenceId, headersAndPayload.readableBytes(), batchSize, isChunked, System.nanoTime(), isMarker, position); if (brokerInterceptor != null) { brokerInterceptor @@ -324,7 +321,7 @@ private void startPublishOperation(int batchSize, long msgSize) { // barrier pendingPublishAcksUpdater.lazySet(this, pendingPublishAcks + 1); // increment publish-count - this.getTopic().incrementPublishCount(batchSize, msgSize); + this.getTopic().incrementPublishCount(this, batchSize, msgSize); } private void publishOperationCompleted() { @@ -342,8 +339,8 @@ private void publishOperationCompleted() { } public void recordMessageDrop(int batchSize) { - if (this.isNonPersistentTopic) { - msgDrop.recordEvent(batchSize); + if (stats instanceof NonPersistentPublisherStatsImpl nonPersistentPublisherStats) { + nonPersistentPublisherStats.recordMsgDrop(batchSize); } } @@ -377,7 +374,6 @@ private static final class MessagePublishContext implements PublishContext, Runn private long sequenceId; private long ledgerId; private long entryId; - private Rate rateIn; private int msgSize; private long batchSize; private boolean chunked; @@ -393,11 +389,6 @@ private static final class MessagePublishContext implements PublishContext, Runn private long entryTimestamp; - @Override - public Position getNext() { - return null; - } - @Override public long getLedgerId() { return ledgerId; @@ -491,9 +482,16 @@ public void completed(Exception exception, long ledgerId, long entryId) { final ServerError serverError = getServerError(exception); producer.cnx.execute(() -> { - if (!(exception instanceof TopicClosedException)) { + // if the topic is transferring, we don't send error code to the clients. + if (producer.getTopic().isTransferring()) { + if (log.isDebugEnabled()) { + log.debug("[{}] Received producer exception: {} while transferring.", + producer.getTopic().getName(), exception.getMessage(), exception); + } + } else if (!(exception instanceof TopicClosedException)) { // For TopicClosed exception there's no need to send explicit error, since the client was // already notified + // For TopicClosingOrDeleting exception, a notification will be sent separately long callBackSequenceId = Math.max(highestSequenceId, sequenceId); producer.cnx.getCommandSender().sendSendError(producer.producerId, callBackSequenceId, serverError, exception.getMessage()); @@ -537,13 +535,13 @@ public void run() { } // stats - rateIn.recordMultipleEvents(batchSize, msgSize); + producer.stats.recordMsgIn(batchSize, msgSize); producer.topic.recordAddLatency(System.nanoTime() - startTimeNs, TimeUnit.NANOSECONDS); producer.cnx.getCommandSender().sendSendReceiptResponse(producer.producerId, sequenceId, highestSequenceId, ledgerId, entryId); producer.cnx.completedSendOperation(producer.isNonPersistentTopic, msgSize); if (this.chunked) { - producer.chunkedMessageRate.recordEvent(); + producer.stats.recordChunkedMsgIn(); } producer.publishOperationCompleted(); if (producer.brokerInterceptor != null) { @@ -553,12 +551,11 @@ public void run() { recycle(); } - static MessagePublishContext get(Producer producer, long sequenceId, Rate rateIn, int msgSize, - long batchSize, boolean chunked, long startTimeNs, boolean isMarker, Position position) { + static MessagePublishContext get(Producer producer, long sequenceId, int msgSize, long batchSize, + boolean chunked, long startTimeNs, boolean isMarker, Position position) { MessagePublishContext callback = RECYCLER.get(); callback.producer = producer; callback.sequenceId = sequenceId; - callback.rateIn = rateIn; callback.msgSize = msgSize; callback.batchSize = batchSize; callback.chunked = chunked; @@ -574,13 +571,12 @@ static MessagePublishContext get(Producer producer, long sequenceId, Rate rateIn return callback; } - static MessagePublishContext get(Producer producer, long lowestSequenceId, long highestSequenceId, Rate rateIn, - int msgSize, long batchSize, boolean chunked, long startTimeNs, boolean isMarker, Position position) { + static MessagePublishContext get(Producer producer, long lowestSequenceId, long highestSequenceId, int msgSize, + long batchSize, boolean chunked, long startTimeNs, boolean isMarker, Position position) { MessagePublishContext callback = RECYCLER.get(); callback.producer = producer; callback.sequenceId = lowestSequenceId; callback.highestSequenceId = highestSequenceId; - callback.rateIn = rateIn; callback.msgSize = msgSize; callback.batchSize = batchSize; callback.originalProducerName = null; @@ -629,7 +625,6 @@ public void recycle() { highestSequenceId = -1L; originalSequenceId = -1L; originalHighestSequenceId = -1L; - rateIn = null; msgSize = 0; ledgerId = -1L; entryId = -1L; @@ -662,7 +657,7 @@ public Map getMetadata() { @Override public String toString() { - return MoreObjects.toStringHelper(this).add("topic", topic).add("client", cnx.clientAddress()) + return MoreObjects.toStringHelper(this).add("topic", topic).add("client", cnx.toString()) .add("producerName", producerName).add("producerId", producerId).toString(); } @@ -703,17 +698,21 @@ public void closeNow(boolean removeFromTopic) { isDisconnecting.set(false); } + public CompletableFuture disconnect() { + return disconnect(Optional.empty()); + } + /** * It closes the producer from server-side and sends command to client to disconnect producer from existing * connection without closing that connection. * * @return Completable future indicating completion of producer close */ - public CompletableFuture disconnect() { + public CompletableFuture disconnect(Optional assignedBrokerLookupData) { if (!closeFuture.isDone() && isDisconnecting.compareAndSet(false, true)) { - log.info("Disconnecting producer: {}", this); + log.info("Disconnecting producer: {}, assignedBrokerLookupData: {}", this, assignedBrokerLookupData); cnx.execute(() -> { - cnx.closeProducer(this); + cnx.closeProducer(this, assignedBrokerLookupData); closeNow(true); }); } @@ -730,25 +729,12 @@ public void topicMigrated(Optional clusterUrl) { } public void updateRates() { - msgIn.calculateRate(); - chunkedMessageRate.calculateRate(); - stats.msgRateIn = msgIn.getRate(); - stats.msgThroughputIn = msgIn.getValueRate(); - stats.averageMsgSize = msgIn.getAverageValue(); - stats.chunkedMessageRate = chunkedMessageRate.getRate(); - if (chunkedMessageRate.getCount() > 0 && this.topic instanceof PersistentTopic) { - ((PersistentTopic) this.topic).msgChunkPublished = true; - } - if (this.isNonPersistentTopic) { - msgDrop.calculateRate(); - ((NonPersistentPublisherStatsImpl) stats).msgDropRate = msgDrop.getRate(); + stats.calculateRates(); + if (stats.getMsgChunkIn().getCount() > 0 && topic instanceof PersistentTopic persistentTopic) { + persistentTopic.msgChunkPublished = true; } } - public void updateRates(int numOfMessages, long msgSizeInBytes) { - msgIn.recordMultipleEvents(numOfMessages, msgSizeInBytes); - } - public boolean isRemote() { return isRemote; } @@ -814,7 +800,7 @@ public void publishTxnMessage(TxnID txnID, long producerId, long sequenceId, lon return; } MessagePublishContext messagePublishContext = - MessagePublishContext.get(this, sequenceId, highSequenceId, msgIn, + MessagePublishContext.get(this, sequenceId, highSequenceId, headersAndPayload.readableBytes(), batchSize, isChunked, System.nanoTime(), isMarker, null); if (brokerInterceptor != null) { brokerInterceptor @@ -845,4 +831,52 @@ public boolean isDisconnecting() { private static final Logger log = LoggerFactory.getLogger(Producer.class); + /** + * This method increments a counter that is used to control the throttling of a connection. + * The connection's read operations are paused when the counter's value is greater than 0, indicating that + * throttling is in effect. + * It's important to note that after calling this method, it is the caller's responsibility to ensure that the + * counter is decremented by calling the {@link #decrementThrottleCount()} method when throttling is no longer + * needed on the connection. + */ + public void incrementThrottleCount() { + cnx.incrementThrottleCount(); + } + + /** + * This method decrements a counter that is used to control the throttling of a connection. + * The connection's read operations are resumed when the counter's value is 0, indicating that + * throttling is no longer in effect. + * It's important to note that before calling this method, the caller should have previously + * incremented the counter by calling the {@link #incrementThrottleCount()} method when throttling + * was needed on the connection. + */ + public void decrementThrottleCount() { + cnx.decrementThrottleCount(); + } + + public Attributes getOpenTelemetryAttributes() { + if (attributes != null) { + return attributes; + } + return ATTRIBUTES_FIELD_UPDATER.updateAndGet(this, old -> { + if (old != null) { + return old; + } + var topicName = TopicName.get(topic.getName()); + var builder = Attributes.builder() + .put(OpenTelemetryAttributes.PULSAR_PRODUCER_NAME, producerName) + .put(OpenTelemetryAttributes.PULSAR_PRODUCER_ID, producerId) + .put(OpenTelemetryAttributes.PULSAR_PRODUCER_ACCESS_MODE, + CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE, accessMode.name())) + .put(OpenTelemetryAttributes.PULSAR_DOMAIN, topicName.getDomain().toString()) + .put(OpenTelemetryAttributes.PULSAR_TENANT, topicName.getTenant()) + .put(OpenTelemetryAttributes.PULSAR_NAMESPACE, topicName.getNamespace()) + .put(OpenTelemetryAttributes.PULSAR_TOPIC, topicName.getPartitionedTopicName()); + if (topicName.isPartitioned()) { + builder.put(OpenTelemetryAttributes.PULSAR_PARTITION_INDEX, topicName.getPartitionIndex()); + } + return builder.build(); + }); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiter.java index fde2e7cb56dce..cdc2f448abd32 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiter.java @@ -21,35 +21,19 @@ import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.PublishRate; -public interface PublishRateLimiter extends AutoCloseable { - - PublishRateLimiter DISABLED_RATE_LIMITER = PublishRateLimiterDisable.DISABLED_RATE_LIMITER; +public interface PublishRateLimiter { /** - * checks and update state of current publish and marks if it has exceeded the rate-limiting threshold. - */ - void checkPublishRate(); - - /** - * increments current publish count. + * Consumes publishing quota and handles throttling. + *

+ * The rate limiter implementation calls {@link Producer#incrementThrottleCount()} to indicate + * that the producer should be throttled. The rate limiter must schedule a call to + * {@link Producer#decrementThrottleCount()} after a throttling period that it calculates. * - * @param numOfMessages - * @param msgSizeInBytes - */ - void incrementPublishCount(int numOfMessages, long msgSizeInBytes); - - /** - * reset current publish count. - * - * @return - */ - boolean resetPublishCount(); - - /** - * returns true if current publish has reached the rate-limiting threshold. - * @return + * @param numOfMessages number of messages to publish + * @param msgSizeInBytes size of messages in bytes to publish */ - boolean isPublishRateExceeded(); + void handlePublishThrottling(Producer producer, int numOfMessages, long msgSizeInBytes); /** * updates rate-limiting threshold based on policies. @@ -63,17 +47,4 @@ public interface PublishRateLimiter extends AutoCloseable { * @param maxPublishRate */ void update(PublishRate maxPublishRate); - - /** - * try to acquire permit. - * - * @param numbers - * @param bytes - */ - boolean tryAcquire(int numbers, long bytes); - - /** - * Close the limiter. - */ - void close(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiterImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiterImpl.java index 1947acd87fc66..8255d9b6931ff 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiterImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiterImpl.java @@ -16,70 +16,135 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.pulsar.broker.service; -import java.util.concurrent.atomic.LongAdder; +import com.google.common.annotations.VisibleForTesting; +import io.netty.channel.EventLoopGroup; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.pulsar.broker.qos.AsyncTokenBucket; +import org.apache.pulsar.broker.qos.MonotonicSnapshotClock; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.PublishRate; +import org.jctools.queues.MessagePassingQueue; +import org.jctools.queues.MpscUnboundedArrayQueue; public class PublishRateLimiterImpl implements PublishRateLimiter { - protected volatile int publishMaxMessageRate = 0; - protected volatile long publishMaxByteRate = 0; - protected volatile boolean publishThrottlingEnabled = false; - protected volatile boolean publishRateExceeded = false; - protected volatile LongAdder currentPublishMsgCount = new LongAdder(); - protected volatile LongAdder currentPublishByteCount = new LongAdder(); - - public PublishRateLimiterImpl(Policies policies, String clusterName) { - update(policies, clusterName); - } + private volatile AsyncTokenBucket tokenBucketOnMessage; + private volatile AsyncTokenBucket tokenBucketOnByte; + private final MonotonicSnapshotClock monotonicSnapshotClock; - public PublishRateLimiterImpl(PublishRate maxPublishRate) { - update(maxPublishRate); + private final MessagePassingQueue unthrottlingQueue = new MpscUnboundedArrayQueue<>(1024); + + private final AtomicInteger throttledProducersCount = new AtomicInteger(0); + private final AtomicBoolean processingQueuedProducers = new AtomicBoolean(false); + + public PublishRateLimiterImpl(MonotonicSnapshotClock monotonicSnapshotClock) { + this.monotonicSnapshotClock = monotonicSnapshotClock; } + /** + * {@inheritDoc} + */ @Override - public void checkPublishRate() { - if (this.publishThrottlingEnabled && !publishRateExceeded) { - if (this.publishMaxByteRate > 0) { - long currentPublishByteRate = this.currentPublishByteCount.sum(); - if (currentPublishByteRate > this.publishMaxByteRate) { - publishRateExceeded = true; - return; - } - } - - if (this.publishMaxMessageRate > 0) { - long currentPublishMsgRate = this.currentPublishMsgCount.sum(); - if (currentPublishMsgRate > this.publishMaxMessageRate) { - publishRateExceeded = true; - } - } + public void handlePublishThrottling(Producer producer, int numOfMessages, + long msgSizeInBytes) { + boolean shouldThrottle = false; + AsyncTokenBucket currentTokenBucketOnMessage = tokenBucketOnMessage; + if (currentTokenBucketOnMessage != null) { + // consume tokens from the token bucket for messages + // we should throttle if it returns false since the token bucket is empty in that case + shouldThrottle = !currentTokenBucketOnMessage.consumeTokensAndCheckIfContainsTokens(numOfMessages); + } + AsyncTokenBucket currentTokenBucketOnByte = tokenBucketOnByte; + if (currentTokenBucketOnByte != null) { + // consume tokens from the token bucket for bytes + // we should throttle if it returns false since the token bucket is empty in that case + shouldThrottle |= !currentTokenBucketOnByte.consumeTokensAndCheckIfContainsTokens(msgSizeInBytes); + } + if (shouldThrottle) { + // throttle the producer by incrementing the throttle count + producer.incrementThrottleCount(); + // schedule decrementing the throttle count to possibly unthrottle the producer after the + // throttling period + scheduleDecrementThrottleCount(producer); } } - @Override - public void incrementPublishCount(int numOfMessages, long msgSizeInBytes) { - if (this.publishThrottlingEnabled) { - this.currentPublishMsgCount.add(numOfMessages); - this.currentPublishByteCount.add(msgSizeInBytes); + private void scheduleDecrementThrottleCount(Producer producer) { + // add the producer to the queue of producers to be unthrottled + unthrottlingQueue.offer(producer); + // schedule unthrottling when the throttling count is incremented to 1 + // this is to avoid scheduling unthrottling multiple times for concurrent producers + if (throttledProducersCount.incrementAndGet() == 1) { + EventLoopGroup executor = producer.getCnx().getBrokerService().executor(); + scheduleUnthrottling(executor, calculateThrottlingDurationNanos()); } } - @Override - public boolean resetPublishCount() { - if (this.publishThrottlingEnabled) { - this.currentPublishMsgCount.reset(); - this.currentPublishByteCount.reset(); - this.publishRateExceeded = false; - return true; + /** + * Schedules the unthrottling operation after a throttling period. + * + * This method will usually be called only once at a time. However, in a multi-threaded environment, + * it's possible for concurrent threads to call this method simultaneously. This is acceptable and does not + * disrupt the functionality, as the method is designed to handle such scenarios gracefully. + * + * The solution avoids using locks and this nonblocking approach requires allowing concurrent calls to this method. + * The implementation intends to prevent skipping of scheduling as a result of a race condition, which could + * result in a producer never being unthrottled. + * + * The solution for skipping of scheduling is to allow 2 threads to schedule unthrottling when the throttling + * count is exactly 1 when unthrottleQueuedProducers checks whether there's a need to reschedule. There might + * be another thread that added it and also scheduled unthrottling. This is acceptable and intended for resolving + * the race condition. + * + * @param executor The executor service used to schedule the unthrottling operation. + * @param delayNanos + */ + private void scheduleUnthrottling(ScheduledExecutorService executor, long delayNanos) { + executor.schedule(() -> this.unthrottleQueuedProducers(executor), delayNanos, + TimeUnit.NANOSECONDS); + } + + private long calculateThrottlingDurationNanos() { + AsyncTokenBucket currentTokenBucketOnMessage = tokenBucketOnMessage; + long throttlingDurationNanos = 0L; + if (currentTokenBucketOnMessage != null) { + throttlingDurationNanos = currentTokenBucketOnMessage.calculateThrottlingDuration(); } - return false; + AsyncTokenBucket currentTokenBucketOnByte = tokenBucketOnByte; + if (currentTokenBucketOnByte != null) { + throttlingDurationNanos = Math.max(throttlingDurationNanos, + currentTokenBucketOnByte.calculateThrottlingDuration()); + } + return throttlingDurationNanos; } - @Override - public boolean isPublishRateExceeded() { - return publishRateExceeded; + private void unthrottleQueuedProducers(ScheduledExecutorService executor) { + if (!processingQueuedProducers.compareAndSet(false, true)) { + // another thread is already processing unthrottling + return; + } + try { + Producer producer; + long throttlingDuration = 0L; + // unthrottle as many producers as possible while there are token available + while ((throttlingDuration = calculateThrottlingDurationNanos()) == 0L + && (producer = unthrottlingQueue.poll()) != null) { + producer.decrementThrottleCount(); + throttledProducersCount.decrementAndGet(); + } + // if there are still producers to be unthrottled, schedule unthrottling again + // after another throttling period + if (throttledProducersCount.get() > 0) { + scheduleUnthrottling(executor, throttlingDuration); + } + } finally { + processingQueuedProducers.set(false); + } } @Override @@ -91,26 +156,36 @@ public void update(Policies policies, String clusterName) { } public void update(PublishRate maxPublishRate) { - if (maxPublishRate != null - && (maxPublishRate.publishThrottlingRateInMsg > 0 || maxPublishRate.publishThrottlingRateInByte > 0)) { - this.publishThrottlingEnabled = true; - this.publishMaxMessageRate = Math.max(maxPublishRate.publishThrottlingRateInMsg, 0); - this.publishMaxByteRate = Math.max(maxPublishRate.publishThrottlingRateInByte, 0); + if (maxPublishRate != null) { + updateTokenBuckets(maxPublishRate.publishThrottlingRateInMsg, maxPublishRate.publishThrottlingRateInByte); } else { - this.publishMaxMessageRate = 0; - this.publishMaxByteRate = 0; - this.publishThrottlingEnabled = false; + tokenBucketOnMessage = null; + tokenBucketOnByte = null; } - resetPublishCount(); } - @Override - public boolean tryAcquire(int numbers, long bytes) { - return false; + protected void updateTokenBuckets(long publishThrottlingRateInMsg, long publishThrottlingRateInByte) { + if (publishThrottlingRateInMsg > 0) { + tokenBucketOnMessage = + AsyncTokenBucket.builder().rate(publishThrottlingRateInMsg).clock(monotonicSnapshotClock).build(); + } else { + tokenBucketOnMessage = null; + } + if (publishThrottlingRateInByte > 0) { + tokenBucketOnByte = + AsyncTokenBucket.builder().rate(publishThrottlingRateInByte).clock(monotonicSnapshotClock).build(); + } else { + tokenBucketOnByte = null; + } } - @Override - public void close() { - // no-op + @VisibleForTesting + public AsyncTokenBucket getTokenBucketOnMessage() { + return tokenBucketOnMessage; + } + + @VisibleForTesting + public AsyncTokenBucket getTokenBucketOnByte() { + return tokenBucketOnByte; } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarChannelInitializer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarChannelInitializer.java index 5308b3c981eb4..3b78d5931599e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarChannelInitializer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarChannelInitializer.java @@ -19,14 +19,14 @@ package org.apache.pulsar.broker.service; import com.google.common.annotations.VisibleForTesting; +import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelInitializer; import io.netty.channel.socket.SocketChannel; import io.netty.handler.codec.LengthFieldBasedFrameDecoder; import io.netty.handler.flow.FlowControlHandler; import io.netty.handler.flush.FlushConsolidationHandler; -import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslHandler; -import io.netty.handler.ssl.SslProvider; +import java.util.concurrent.TimeUnit; import lombok.Builder; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -35,9 +35,8 @@ import org.apache.pulsar.common.protocol.ByteBufPair; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.OptionalProxyProtocolDecoder; -import org.apache.pulsar.common.util.NettyServerSslContextBuilder; -import org.apache.pulsar.common.util.SslContextAutoRefreshBuilder; -import org.apache.pulsar.common.util.keystoretls.NettySSLContextAutoRefreshBuilder; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; @Slf4j public class PulsarChannelInitializer extends ChannelInitializer { @@ -47,10 +46,8 @@ public class PulsarChannelInitializer extends ChannelInitializer private final PulsarService pulsar; private final String listenerName; private final boolean enableTls; - private final boolean tlsEnabledWithKeyStore; - private SslContextAutoRefreshBuilder sslCtxRefresher; private final ServiceConfiguration brokerConf; - private NettySSLContextAutoRefreshBuilder nettySSLContextAutoRefreshBuilder; + private PulsarSslFactory sslFactory; /** * @param pulsar @@ -64,58 +61,32 @@ public PulsarChannelInitializer(PulsarService pulsar, PulsarChannelOptions opts) this.listenerName = opts.getListenerName(); this.enableTls = opts.isEnableTLS(); ServiceConfiguration serviceConfig = pulsar.getConfiguration(); - this.tlsEnabledWithKeyStore = serviceConfig.isTlsEnabledWithKeyStore(); if (this.enableTls) { - if (tlsEnabledWithKeyStore) { - nettySSLContextAutoRefreshBuilder = new NettySSLContextAutoRefreshBuilder( - serviceConfig.getTlsProvider(), - serviceConfig.getTlsKeyStoreType(), - serviceConfig.getTlsKeyStore(), - serviceConfig.getTlsKeyStorePassword(), - serviceConfig.isTlsAllowInsecureConnection(), - serviceConfig.getTlsTrustStoreType(), - serviceConfig.getTlsTrustStore(), - serviceConfig.getTlsTrustStorePassword(), - serviceConfig.isTlsRequireTrustedClientCertOnConnect(), - serviceConfig.getTlsCiphers(), - serviceConfig.getTlsProtocols(), - serviceConfig.getTlsCertRefreshCheckDurationSec()); - } else { - SslProvider sslProvider = null; - if (serviceConfig.getTlsProvider() != null) { - sslProvider = SslProvider.valueOf(serviceConfig.getTlsProvider()); - } - sslCtxRefresher = new NettyServerSslContextBuilder( - sslProvider, - serviceConfig.isTlsAllowInsecureConnection(), - serviceConfig.getTlsTrustCertsFilePath(), - serviceConfig.getTlsCertificateFilePath(), - serviceConfig.getTlsKeyFilePath(), - serviceConfig.getTlsCiphers(), - serviceConfig.getTlsProtocols(), - serviceConfig.isTlsRequireTrustedClientCertOnConnect(), - serviceConfig.getTlsCertRefreshCheckDurationSec()); + PulsarSslConfiguration pulsarSslConfig = buildSslConfiguration(serviceConfig); + this.sslFactory = (PulsarSslFactory) Class.forName(serviceConfig.getSslFactoryPlugin()) + .getConstructor().newInstance(); + this.sslFactory.initialize(pulsarSslConfig); + this.sslFactory.createInternalSslContext(); + if (serviceConfig.getTlsCertRefreshCheckDurationSec() > 0) { + this.pulsar.getExecutor().scheduleWithFixedDelay(this::refreshSslContext, + serviceConfig.getTlsCertRefreshCheckDurationSec(), + serviceConfig.getTlsCertRefreshCheckDurationSec(), + TimeUnit.SECONDS); } - } else { - this.sslCtxRefresher = null; } this.brokerConf = pulsar.getConfiguration(); } @Override protected void initChannel(SocketChannel ch) throws Exception { + // disable auto read explicitly so that requests aren't served until auto read is enabled + // ServerCnx must enable auto read in channelActive after PulsarService is ready to accept incoming requests + ch.config().setAutoRead(false); ch.pipeline().addLast("consolidation", new FlushConsolidationHandler(1024, true)); if (this.enableTls) { - if (this.tlsEnabledWithKeyStore) { - ch.pipeline().addLast(TLS_HANDLER, - new SslHandler(nettySSLContextAutoRefreshBuilder.get().createSSLEngine())); - } else { - ch.pipeline().addLast(TLS_HANDLER, sslCtxRefresher.get().newHandler(ch.alloc())); - } - ch.pipeline().addLast("ByteBufPairEncoder", ByteBufPair.COPYING_ENCODER); - } else { - ch.pipeline().addLast("ByteBufPairEncoder", ByteBufPair.ENCODER); + ch.pipeline().addLast(TLS_HANDLER, new SslHandler(this.sslFactory.createServerSslEngine(ch.alloc()))); } + ch.pipeline().addLast("ByteBufPairEncoder", ByteBufPair.getEncoder(this.enableTls)); if (pulsar.getConfiguration().isHaProxyProtocolEnabled()) { ch.pipeline().addLast(OptionalProxyProtocolDecoder.NAME, new OptionalProxyProtocolDecoder()); @@ -128,7 +99,8 @@ protected void initChannel(SocketChannel ch) throws Exception { // ServerCnx ends up reading higher number of messages and broker can not throttle the messages by disabling // auto-read. ch.pipeline().addLast("flowController", new FlowControlHandler()); - ServerCnx cnx = newServerCnx(pulsar, listenerName); + // using "ChannelHandler" type to workaround an IntelliJ bug that shows a false positive error + ChannelHandler cnx = newServerCnx(pulsar, listenerName); ch.pipeline().addLast("handler", cnx); } @@ -158,4 +130,33 @@ public static class PulsarChannelOptions { */ private String listenerName; } + + protected PulsarSslConfiguration buildSslConfiguration(ServiceConfiguration serviceConfig) { + return PulsarSslConfiguration.builder() + .tlsKeyStoreType(serviceConfig.getTlsKeyStoreType()) + .tlsKeyStorePath(serviceConfig.getTlsKeyStore()) + .tlsKeyStorePassword(serviceConfig.getTlsKeyStorePassword()) + .tlsTrustStoreType(serviceConfig.getTlsTrustStoreType()) + .tlsTrustStorePath(serviceConfig.getTlsTrustStore()) + .tlsTrustStorePassword(serviceConfig.getTlsTrustStorePassword()) + .tlsCiphers(serviceConfig.getTlsCiphers()) + .tlsProtocols(serviceConfig.getTlsProtocols()) + .tlsTrustCertsFilePath(serviceConfig.getTlsTrustCertsFilePath()) + .tlsCertificateFilePath(serviceConfig.getTlsCertificateFilePath()) + .tlsKeyFilePath(serviceConfig.getTlsKeyFilePath()) + .allowInsecureConnection(serviceConfig.isTlsAllowInsecureConnection()) + .requireTrustedClientCertOnConnect(serviceConfig.isTlsRequireTrustedClientCertOnConnect()) + .tlsEnabledWithKeystore(serviceConfig.isTlsEnabledWithKeyStore()) + .tlsCustomParams(serviceConfig.getSslFactoryPluginParams()) + .serverMode(true) + .build(); + } + + protected void refreshSslContext() { + try { + this.sslFactory.update(); + } catch (Exception e) { + log.error("Failed to refresh SSL context", e); + } + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSenderImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSenderImpl.java index dd74fc4e71ed2..105650caaaf13 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSenderImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSenderImpl.java @@ -356,12 +356,18 @@ public void sendEndTxnErrorResponse(long requestId, TxnID txnID, ServerError err writeAndFlush(outBuf); } + /*** + * @param topics topic names which are matching, the topic name contains the partition suffix. + */ @Override public void sendWatchTopicListSuccess(long requestId, long watcherId, String topicsHash, List topics) { BaseCommand command = Commands.newWatchTopicListSuccess(requestId, watcherId, topicsHash, topics); interceptAndWriteCommand(command); } + /*** + * {@inheritDoc} + */ @Override public void sendWatchTopicListUpdate(long watcherId, List newTopics, List deletedTopics, String topicsHash) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java index 80743e44ab7d2..8b2ebf200537e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java @@ -19,11 +19,15 @@ package org.apache.pulsar.broker.service; import static org.apache.pulsar.broker.service.persistent.PersistentTopic.MESSAGE_RATE_BACKOFF_MS; +import java.util.Arrays; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; +import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; @@ -33,8 +37,8 @@ import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; -import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.common.util.Backoff; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.MetadataEvent; import org.apache.pulsar.metadata.api.MetadataEventSynchronizer; @@ -46,6 +50,7 @@ public class PulsarMetadataEventSynchronizer implements MetadataEventSynchronize private static final Logger log = LoggerFactory.getLogger(PulsarMetadataEventSynchronizer.class); protected PulsarService pulsar; protected BrokerService brokerService; + @Getter protected String topicName; protected PulsarClientImpl client; protected volatile Producer producer; @@ -53,19 +58,32 @@ public class PulsarMetadataEventSynchronizer implements MetadataEventSynchronize private final CopyOnWriteArrayList>> listeners = new CopyOnWriteArrayList<>(); - private volatile boolean started = false; + static final AtomicReferenceFieldUpdater STATE_UPDATER = + AtomicReferenceFieldUpdater.newUpdater(PulsarMetadataEventSynchronizer.class, State.class, "state"); + @Getter + private volatile State state; public static final String SUBSCRIPTION_NAME = "metadata-syncer"; private static final int MAX_PRODUCER_PENDING_SIZE = 1000; protected final Backoff backOff = new Backoff(100, TimeUnit.MILLISECONDS, 1, TimeUnit.MINUTES, 0, TimeUnit.MILLISECONDS); + private volatile CompletableFuture closeFuture; - public PulsarMetadataEventSynchronizer(PulsarService pulsar, String topicName) throws PulsarServerException { + public enum State { + Init, + Starting_Producer, + Starting_Consumer, + Started, + Closing, + Closed; + } + + public PulsarMetadataEventSynchronizer(PulsarService pulsar, String topicName) { this.pulsar = pulsar; this.brokerService = pulsar.getBrokerService(); this.topicName = topicName; + this.state = State.Init; if (!StringUtils.isNotBlank(topicName)) { log.info("Metadata synchronizer is disabled"); - return; } } @@ -74,10 +92,11 @@ public void start() throws PulsarServerException { log.info("metadata topic doesn't exist.. skipping metadata synchronizer init.."); return; } + log.info("Metadata event synchronizer is starting on topic {}", topicName); this.client = (PulsarClientImpl) pulsar.getClient(); - startProducer(); - startConsumer(); - log.info("Metadata event synchronizer started on topic {}", topicName); + if (STATE_UPDATER.compareAndSet(this, State.Init, State.Starting_Producer)) { + startProducer(); + } } @Override @@ -98,7 +117,7 @@ public String getClusterName() { } private void publishAsync(MetadataEvent event, CompletableFuture future) { - if (!started) { + if (!isProducerStarted()) { log.info("Producer is not started on {}, failed to publish {}", topicName, event); future.completeExceptionally(new IllegalStateException("producer is not started yet")); } @@ -114,62 +133,100 @@ private void publishAsync(MetadataEvent event, CompletableFuture future) { } private void startProducer() { + if (isClosingOrClosed()) { + log.info("[{}] Skip to start new producer because the synchronizer is closed", topicName); + } + if (producer != null) { + log.error("[{}] Failed to start the producer because the producer has been set, state: {}", + topicName, state); + return; + } log.info("[{}] Starting producer", topicName); client.newProducer(Schema.AVRO(MetadataEvent.class)).topic(topicName) - .messageRoutingMode(MessageRoutingMode.SinglePartition).enableBatching(false).enableBatching(false) - .sendTimeout(0, TimeUnit.SECONDS) // - .maxPendingMessages(MAX_PRODUCER_PENDING_SIZE).createAsync().thenAccept(prod -> { + .messageRoutingMode(MessageRoutingMode.SinglePartition).enableBatching(false).enableBatching(false) + .sendTimeout(0, TimeUnit.SECONDS) // + .maxPendingMessages(MAX_PRODUCER_PENDING_SIZE).createAsync().thenAccept(prod -> { + backOff.reset(); + if (STATE_UPDATER.compareAndSet(this, State.Starting_Producer, State.Starting_Consumer)) { producer = prod; - started = true; log.info("producer is created successfully {}", topicName); - }).exceptionally(ex -> { - long waitTimeMs = backOff.next(); - log.warn("[{}] Failed to create producer ({}), retrying in {} s", topicName, ex.getMessage(), - waitTimeMs / 1000.0); - // BackOff before retrying - brokerService.executor().schedule(this::startProducer, waitTimeMs, TimeUnit.MILLISECONDS); - return null; - }); + PulsarMetadataEventSynchronizer.this.startConsumer(); + } else { + State stateTransient = state; + log.info("[{}] Closing the new producer because the synchronizer state is {}", prod, + stateTransient); + CompletableFuture closeProducer = new CompletableFuture<>(); + closeResource(() -> prod.closeAsync(), closeProducer); + closeProducer.thenRun(() -> { + log.info("[{}] Closed the new producer because the synchronizer state is {}", prod, + stateTransient); + }); + } + }).exceptionally(ex -> { + long waitTimeMs = backOff.next(); + log.warn("[{}] Failed to create producer ({}), retrying in {} s", topicName, ex.getMessage(), + waitTimeMs / 1000.0); + // BackOff before retrying + brokerService.executor().schedule(this::startProducer, waitTimeMs, TimeUnit.MILLISECONDS); + return null; + }); } private void startConsumer() { + if (isClosingOrClosed()) { + log.info("[{}] Skip to start new consumer because the synchronizer is closed", topicName); + } if (consumer != null) { + log.error("[{}] Failed to start the consumer because the consumer has been set, state: {}", + topicName, state); return; } + log.info("[{}] Starting consumer", topicName); ConsumerBuilder consumerBuilder = client.newConsumer(Schema.AVRO(MetadataEvent.class)) - .topic(topicName).subscriptionName(SUBSCRIPTION_NAME).ackTimeout(60, TimeUnit.SECONDS) - .subscriptionType(SubscriptionType.Failover).messageListener((c, msg) -> { - log.info("Processing metadata event for {} with listeners {}", msg.getValue().getPath(), - listeners.size()); - try { - if (listeners.size() == 0) { - c.acknowledgeAsync(msg); - return; - - } - if (listeners.size() == 1) { - listeners.get(0).apply(msg.getValue()).thenApply(__ -> c.acknowledgeAsync(msg)) - .exceptionally(ex -> { - log.warn("Failed to synchronize {} for {}", msg.getMessageId(), topicName, - ex.getCause()); - return null; - }); - } else { - FutureUtil - .waitForAll(listeners.stream().map(listener -> listener.apply(msg.getValue())) - .collect(Collectors.toList())) - .thenApply(__ -> c.acknowledgeAsync(msg)).exceptionally(ex -> { - log.warn("Failed to synchronize {} for {}", msg.getMessageId(), topicName); - return null; - }); - } - } catch (Exception e) { - log.warn("Failed to synchronize {} for {}", msg.getMessageId(), topicName); + .topic(topicName).subscriptionName(SUBSCRIPTION_NAME).ackTimeout(60, TimeUnit.SECONDS) + .subscriptionType(SubscriptionType.Failover).messageListener((c, msg) -> { + log.info("Processing metadata event for {} with listeners {}", msg.getValue().getPath(), + listeners.size()); + try { + if (listeners.size() == 0) { + c.acknowledgeAsync(msg); + return; + } - }); + if (listeners.size() == 1) { + listeners.get(0).apply(msg.getValue()).thenApply(__ -> c.acknowledgeAsync(msg)) + .exceptionally(ex -> { + log.warn("Failed to synchronize {} for {}", msg.getMessageId(), topicName, + ex.getCause()); + return null; + }); + } else { + FutureUtil + .waitForAll(listeners.stream().map(listener -> listener.apply(msg.getValue())) + .collect(Collectors.toList())) + .thenApply(__ -> c.acknowledgeAsync(msg)).exceptionally(ex -> { + log.warn("Failed to synchronize {} for {}", msg.getMessageId(), topicName); + return null; + }); + } + } catch (Exception e) { + log.warn("Failed to synchronize {} for {}", msg.getMessageId(), topicName); + } + }); consumerBuilder.subscribeAsync().thenAccept(consumer -> { - log.info("successfully created consumer {}", topicName); - this.consumer = consumer; + backOff.reset(); + if (STATE_UPDATER.compareAndSet(this, State.Starting_Consumer, State.Started)) { + this.consumer = consumer; + log.info("successfully created consumer {}", topicName); + } else { + State stateTransient = state; + log.info("[{}] Closing the new consumer because the synchronizer state is {}", stateTransient); + CompletableFuture closeConsumer = new CompletableFuture<>(); + closeResource(() -> consumer.closeAsync(), closeConsumer); + closeConsumer.thenRun(() -> { + log.info("[{}] Closed the new consumer because the synchronizer state is {}", stateTransient); + }); + } }).exceptionally(ex -> { long waitTimeMs = backOff.next(); log.warn("[{}] Failed to create consumer ({}), retrying in {} s", topicName, ex.getMessage(), @@ -181,19 +238,81 @@ private void startConsumer() { } public boolean isStarted() { - return started; + return this.state == State.Started; + } + + public boolean isProducerStarted() { + return this.state.ordinal() > State.Starting_Producer.ordinal() + && this.state.ordinal() < State.Closing.ordinal(); + } + + public boolean isClosingOrClosed() { + return this.state == State.Closing || this.state == State.Closed; } @Override - public void close() { - started = false; - if (producer != null) { - producer.closeAsync(); - producer = null; + public synchronized CompletableFuture closeAsync() { + int tryChangeStateCounter = 0; + while (true) { + if (isClosingOrClosed()) { + return closeFuture; + } + if (STATE_UPDATER.compareAndSet(this, State.Init, State.Closing) + || STATE_UPDATER.compareAndSet(this, State.Starting_Producer, State.Closing) + || STATE_UPDATER.compareAndSet(this, State.Starting_Consumer, State.Closing) + || STATE_UPDATER.compareAndSet(this, State.Started, State.Closing)) { + break; + } + // Just for avoid spinning loop which would cause 100% CPU consumption here. + if (++tryChangeStateCounter > 100) { + log.error("Unexpected error: the state can not be changed to closing {}, state: {}", topicName, state); + return CompletableFuture.failedFuture(new RuntimeException("Unexpected error," + + " the state can not be changed to closing")); + } } - if (consumer != null) { - consumer.closeAsync(); - consumer = null; + CompletableFuture closeProducer = new CompletableFuture<>(); + CompletableFuture closeConsumer = new CompletableFuture<>(); + if (producer == null) { + closeProducer.complete(null); + } else { + closeResource(() -> producer.closeAsync(), closeProducer); + } + if (consumer == null) { + closeConsumer.complete(null); + } else { + closeResource(() -> consumer.closeAsync(), closeConsumer); + } + + // Add logs. + closeProducer.thenRun(() -> log.info("Successfully close producer {}", topicName)); + closeConsumer.thenRun(() -> log.info("Successfully close consumer {}", topicName)); + + closeFuture = FutureUtil.waitForAll(Arrays.asList(closeProducer, closeConsumer)); + closeFuture.thenRun(() -> { + this.state = State.Closed; + log.info("Successfully close metadata store synchronizer {}", topicName); + }); + return closeFuture; + } + + private void closeResource(final Supplier> asyncCloseable, + final CompletableFuture future) { + if (asyncCloseable == null) { + future.complete(null); + return; } + asyncCloseable.get().whenComplete((ignore, ex) -> { + if (ex == null) { + backOff.reset(); + future.complete(null); + return; + } + // Retry. + long waitTimeMs = backOff.next(); + log.warn("[{}] Exception: '{}' occurred while trying to close the %s. Retrying again in {} s.", + topicName, ex.getMessage(), asyncCloseable.getClass().getSimpleName(), waitTimeMs / 1000.0, ex); + brokerService.executor().schedule(() -> closeResource(asyncCloseable, future), waitTimeMs, + TimeUnit.MILLISECONDS); + }); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java index e959e9bbda2bb..b96e00a8909d6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java @@ -38,7 +38,6 @@ import org.apache.pulsar.broker.stats.NamespaceStats; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.stats.Metrics; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.utils.StatsOutputStream; import org.slf4j.Logger; @@ -78,8 +77,7 @@ public PulsarStats(PulsarService pulsar) { this.bundleStats = new ConcurrentHashMap<>(); this.tempMetricsCollection = new ArrayList<>(); this.metricsCollection = new ArrayList<>(); - this.brokerOperabilityMetrics = new BrokerOperabilityMetrics(pulsar.getConfiguration().getClusterName(), - pulsar.getAdvertisedAddress()); + this.brokerOperabilityMetrics = new BrokerOperabilityMetrics(pulsar); this.tempNonPersistentTopics = new ArrayList<>(); this.exposePublisherStats = pulsar.getConfiguration().isExposePublisherStats(); @@ -102,9 +100,7 @@ public ClusterReplicationMetrics getClusterReplicationMetrics() { return clusterReplicationMetrics; } - public synchronized void updateStats( - ConcurrentOpenHashMap>> - topicsMap) { + public synchronized void updateStats(Map>> topicsMap) { StatsOutputStream topicStatsStream = new StatsOutputStream(tempTopicStatsBuf); @@ -265,6 +261,10 @@ public void recordTopicLoadTimeValue(String topic, long topicLoadLatencyMs) { } } + public void recordTopicLoadFailed() { + brokerOperabilityMetrics.recordTopicLoadFailed(); + } + public void recordConnectionCreate() { brokerOperabilityMetrics.recordConnectionCreate(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java index 482fa2cbd2300..667063e491085 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java @@ -27,11 +27,13 @@ public interface Replicator { void startProducer(); - ReplicatorStatsImpl getStats(); + Topic getLocalTopic(); + + ReplicatorStatsImpl computeStats(); - CompletableFuture disconnect(); + CompletableFuture terminate(); - CompletableFuture disconnect(boolean b); + CompletableFuture disconnect(boolean failIfHasBacklog, boolean closeTheStartingProducer); void updateRates(); @@ -51,4 +53,8 @@ default Optional getRateLimiter() { boolean isConnected(); long getNumberOfEntriesInBacklog(); + + boolean isTerminated(); + + ReplicatorStatsImpl getStats(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 888668e15b167..aedd68d416fe7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -26,11 +26,13 @@ import static org.apache.pulsar.broker.admin.impl.PersistentTopicsBase.unsafeGetPartitionedTopicMetadataAsync; import static org.apache.pulsar.broker.lookup.TopicLookupBase.lookupTopicAsync; import static org.apache.pulsar.broker.service.persistent.PersistentTopic.getMigratedClusterUrl; +import static org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage.ignoreUnrecoverableBKException; import static org.apache.pulsar.common.api.proto.ProtocolVersion.v5; import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; import static org.apache.pulsar.common.protocol.Commands.newLookupErrorResponse; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; +import com.google.re2j.Pattern; import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; @@ -40,7 +42,6 @@ import io.netty.util.concurrent.FastThreadLocal; import io.netty.util.concurrent.Promise; import io.netty.util.concurrent.ScheduledFuture; -import io.prometheus.client.Gauge; import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketAddress; @@ -55,22 +56,25 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; -import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.naming.AuthenticationException; import javax.net.ssl.SSLSession; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Response; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.mledger.impl.AckSetStateUtil; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.commons.lang3.mutable.MutableInt; -import org.apache.commons.lang3.mutable.MutableLong; +import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.TransactionMetadataStoreService; @@ -80,6 +84,10 @@ import org.apache.pulsar.broker.authentication.AuthenticationState; import org.apache.pulsar.broker.intercept.BrokerInterceptor; import org.apache.pulsar.broker.limiter.ConnectionController; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.broker.namespace.LookupOptions; +import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException; import org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException; import org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException; @@ -145,15 +153,17 @@ import org.apache.pulsar.common.compression.CompressionCodec; import org.apache.pulsar.common.compression.CompressionCodecProvider; import org.apache.pulsar.common.intercept.InterceptException; +import org.apache.pulsar.common.lookup.data.LookupData; import org.apache.pulsar.common.naming.Metadata; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType; -import org.apache.pulsar.common.policies.data.ClusterData.ClusterUrl; +import org.apache.pulsar.common.policies.data.ClusterPolicies.ClusterUrl; import org.apache.pulsar.common.policies.data.NamespaceOperation; import org.apache.pulsar.common.policies.data.TopicOperation; +import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; import org.apache.pulsar.common.protocol.ByteBufPair; import org.apache.pulsar.common.protocol.CommandUtils; @@ -164,10 +174,12 @@ import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.topics.TopicList; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.StringInterner; import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap; import org.apache.pulsar.common.util.netty.NettyChannelUtil; import org.apache.pulsar.common.util.netty.NettyFutureUtil; import org.apache.pulsar.functions.utils.Exceptions; +import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; import org.apache.pulsar.transaction.coordinator.exceptions.CoordinatorException; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStore; @@ -184,7 +196,7 @@ public class ServerCnx extends PulsarHandler implements TransportCnx { private final BrokerService service; private final SchemaRegistryService schemaService; private final String listenerName; - private final HashMap recentlyClosedProducers; + private final Map recentlyClosedProducers; private final ConcurrentLongHashMap> producers; private final ConcurrentLongHashMap> consumers; private final boolean enableSubscriptionPatternEvaluation; @@ -214,6 +226,7 @@ public class ServerCnx extends PulsarHandler implements TransportCnx { private final String replicatorPrefix; private String clientVersion = null; private String proxyVersion = null; + private String clientSourceAddressAndPort; private int nonPersistentPendingMessages = 0; private final int maxNonPersistentPendingMessages; private String originalPrincipal = null; @@ -222,11 +235,8 @@ public class ServerCnx extends PulsarHandler implements TransportCnx { private final int maxMessageSize; private boolean preciseDispatcherFlowControl; - private boolean preciseTopicPublishRateLimitingEnable; private boolean encryptionRequireOnProducer; - // Flag to manage throttling-rate by atomically enable/disable read-channel. - private volatile boolean autoReadDisabledRateLimiting = false; private FeatureFlags features; private PulsarCommandSender commandSender; @@ -235,23 +245,52 @@ public class ServerCnx extends PulsarHandler implements TransportCnx { private static final KeySharedMeta emptyKeySharedMeta = new KeySharedMeta() .setKeySharedMode(KeySharedMode.AUTO_SPLIT); - // Flag to manage throttling-publish-buffer by atomically enable/disable read-channel. - private boolean autoReadDisabledPublishBufferLimiting = false; private final long maxPendingBytesPerThread; private final long resumeThresholdPendingBytesPerThread; private final long connectionLivenessCheckTimeoutMillis; - // Number of bytes pending to be published from a single specific IO thread. - private static final FastThreadLocal pendingBytesPerThread = new FastThreadLocal() { - @Override - protected MutableLong initialValue() throws Exception { - return new MutableLong(); + // Tracks and limits number of bytes pending to be published from a single specific IO thread. + static final class PendingBytesPerThreadTracker { + private static final FastThreadLocal pendingBytesPerThread = + new FastThreadLocal<>() { + @Override + protected PendingBytesPerThreadTracker initialValue() throws Exception { + return new PendingBytesPerThreadTracker(); + } + }; + + private long pendingBytes; + private boolean limitExceeded; + + public static PendingBytesPerThreadTracker getInstance() { + return pendingBytesPerThread.get(); + } + + public void incrementPublishBytes(long bytes, long maxPendingBytesPerThread) { + pendingBytes += bytes; + // when the limit is exceeded we throttle all connections that are sharing the same thread + if (maxPendingBytesPerThread > 0 && pendingBytes > maxPendingBytesPerThread + && !limitExceeded) { + limitExceeded = true; + cnxsPerThread.get().forEach(cnx -> cnx.throttleTracker.setPublishBufferLimiting(true)); + } } - }; + + public void decrementPublishBytes(long bytes, long resumeThresholdPendingBytesPerThread) { + pendingBytes -= bytes; + // when the limit has been exceeded, and we are below the resume threshold + // we resume all connections sharing the same thread + if (limitExceeded && pendingBytes <= resumeThresholdPendingBytesPerThread) { + limitExceeded = false; + cnxsPerThread.get().forEach(cnx -> cnx.throttleTracker.setPublishBufferLimiting(false)); + } + } + } + // A set of connections tied to the current thread - private static final FastThreadLocal> cnxsPerThread = new FastThreadLocal>() { + private static final FastThreadLocal> cnxsPerThread = new FastThreadLocal<>() { @Override protected Set initialValue() throws Exception { return Collections.newSetFromMap(new IdentityHashMap<>()); @@ -262,6 +301,8 @@ enum State { Start, Connected, Failed, Connecting } + private final ServerCnxThrottleTracker throttleTracker; + public ServerCnx(PulsarService pulsar) { this(pulsar, null); } @@ -288,7 +329,7 @@ public ServerCnx(PulsarService pulsar, String listenerName) { .expectedItems(8) .concurrencyLevel(1) .build(); - this.recentlyClosedProducers = new HashMap<>(); + this.recentlyClosedProducers = new ConcurrentHashMap<>(); this.replicatorPrefix = conf.getReplicatorPrefix(); this.maxNonPersistentPendingMessages = conf.getMaxConcurrentNonPersistentMessagePerConnection(); this.schemaValidationEnforced = conf.isSchemaValidationEnforced(); @@ -296,7 +337,6 @@ public ServerCnx(PulsarService pulsar, String listenerName) { this.maxPendingSendRequests = conf.getMaxPendingPublishRequestsPerConnection(); this.resumeReadsThreshold = maxPendingSendRequests / 2; this.preciseDispatcherFlowControl = conf.isPreciseDispatcherFlowControl(); - this.preciseTopicPublishRateLimitingEnable = conf.isPreciseTopicPublishRateLimiterEnable(); this.encryptionRequireOnProducer = conf.isEncryptionRequireOnProducer(); // Assign a portion of max-pending bytes to each IO thread this.maxPendingBytesPerThread = conf.getMaxMessagePublishBufferSizeInMB() * 1024L * 1024L @@ -310,6 +350,7 @@ public ServerCnx(PulsarService pulsar, String listenerName) { this.topicListService = new TopicListService(pulsar, this, enableSubscriptionPatternEvaluation, maxSubscriptionPatternLength); this.brokerInterceptor = this.service != null ? this.service.getInterceptor() : null; + this.throttleTracker = new ServerCnxThrottleTracker(this); } @Override @@ -332,6 +373,10 @@ public void channelActive(ChannelHandlerContext ctx) throws Exception { this.commandSender = new PulsarCommandSenderImpl(brokerInterceptor, this); this.service.getPulsarStats().recordConnectionCreate(); cnxsPerThread.get().add(this); + service.getPulsar().runWhenReadyForIncomingRequests(() -> { + // enable auto read after PulsarService is ready to accept incoming requests + ctx.channel().config().setAutoRead(true); + }); } @Override @@ -388,7 +433,7 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { // complete possible pending connection check future if (connectionCheckInProgress != null && !connectionCheckInProgress.isDone()) { - connectionCheckInProgress.complete(false); + connectionCheckInProgress.complete(Optional.of(false)); } } @@ -502,9 +547,19 @@ protected void handleLookup(CommandLookupTopic lookup) { isTopicOperationAllowed(topicName, TopicOperation.LOOKUP, authenticationData, originalAuthData).thenApply( isAuthorized -> { if (isAuthorized) { + final Map properties; + if (lookup.getPropertiesCount() > 0) { + properties = new HashMap<>(); + for (int i = 0; i < lookup.getPropertiesCount(); i++) { + final var keyValue = lookup.getPropertyAt(i); + properties.put(keyValue.getKey(), keyValue.getValue()); + } + } else { + properties = Collections.emptyMap(); + } lookupTopicAsync(getBrokerService().pulsar(), topicName, authoritative, getPrincipal(), getAuthenticationData(), - requestId, advertisedListenerName).handle((lookupResponse, ex) -> { + requestId, advertisedListenerName, properties).handle((lookupResponse, ex) -> { if (ex == null) { writeAndFlush(lookupResponse); } else { @@ -575,35 +630,76 @@ protected void handlePartitionMetadataRequest(CommandPartitionedTopicMetadata pa isTopicOperationAllowed(topicName, TopicOperation.LOOKUP, authenticationData, originalAuthData).thenApply( isAuthorized -> { if (isAuthorized) { - unsafeGetPartitionedTopicMetadataAsync(getBrokerService().pulsar(), topicName) - .handle((metadata, ex) -> { - if (ex == null) { - int partitions = metadata.partitions; - commandSender.sendPartitionMetadataResponse(partitions, requestId); + // Get if exists, respond not found error if not exists. + getBrokerService().isAllowAutoTopicCreationAsync(topicName).thenAccept(brokerAllowAutoCreate -> { + boolean autoCreateIfNotExist = partitionMetadata.isMetadataAutoCreationEnabled() + && brokerAllowAutoCreate; + if (!autoCreateIfNotExist) { + NamespaceService namespaceService = getBrokerService().getPulsar().getNamespaceService(); + namespaceService.checkTopicExists(topicName).thenAccept(topicExistsInfo -> { + lookupSemaphore.release(); + if (!topicExistsInfo.isExists()) { + writeAndFlush(Commands.newPartitionMetadataResponse( + ServerError.TopicNotFound, "", requestId)); + } else if (topicExistsInfo.getTopicType().equals(TopicType.PARTITIONED)) { + commandSender.sendPartitionMetadataResponse(topicExistsInfo.getPartitions(), + requestId); } else { - if (ex instanceof PulsarClientException) { - log.warn("Failed to authorize {} at [{}] on topic {} : {}", getRole(), - remoteAddress, topicName, ex.getMessage()); - commandSender.sendPartitionMetadataResponse(ServerError.AuthorizationError, - ex.getMessage(), requestId); + commandSender.sendPartitionMetadataResponse(0, requestId); + } + // release resources. + topicExistsInfo.recycle(); + }).exceptionally(ex -> { + lookupSemaphore.release(); + log.error("{} {} Failed to get partition metadata", topicName, + ServerCnx.this.toString(), ex); + writeAndFlush( + Commands.newPartitionMetadataResponse(ServerError.MetadataError, + "Failed to get partition metadata", + requestId)); + return null; + }); + } else { + // Get if exists, create a new one if not exists. + unsafeGetPartitionedTopicMetadataAsync(getBrokerService().pulsar(), topicName) + .whenComplete((metadata, ex) -> { + lookupSemaphore.release(); + if (ex == null) { + int partitions = metadata.partitions; + commandSender.sendPartitionMetadataResponse(partitions, requestId); } else { - log.warn("Failed to get Partitioned Metadata [{}] {}: {}", remoteAddress, - topicName, ex.getMessage(), ex); - ServerError error = ServerError.ServiceNotReady; - if (ex instanceof RestException restException){ - int responseCode = restException.getResponse().getStatus(); - if (responseCode == NOT_FOUND.getStatusCode()){ - error = ServerError.TopicNotFound; - } else if (responseCode < INTERNAL_SERVER_ERROR.getStatusCode()){ + if (ex instanceof PulsarClientException) { + log.warn("Failed to authorize {} at [{}] on topic {} : {}", getRole(), + remoteAddress, topicName, ex.getMessage()); + commandSender.sendPartitionMetadataResponse(ServerError.AuthorizationError, + ex.getMessage(), requestId); + } else { + ServerError error = ServerError.ServiceNotReady; + if (ex instanceof MetadataStoreException) { error = ServerError.MetadataError; + } else if (ex instanceof RestException restException){ + int responseCode = restException.getResponse().getStatus(); + if (responseCode == NOT_FOUND.getStatusCode()){ + error = ServerError.TopicNotFound; + } else if (responseCode < INTERNAL_SERVER_ERROR.getStatusCode()){ + error = ServerError.MetadataError; + } } + if (error == ServerError.TopicNotFound) { + log.info("Trying to get Partitioned Metadata for a resource not exist" + + "[{}] {}: {}", remoteAddress, + topicName, ex.getMessage()); + } else { + log.warn("Failed to get Partitioned Metadata [{}] {}: {}", + remoteAddress, topicName, ex.getMessage(), ex); + } + commandSender.sendPartitionMetadataResponse(error, ex.getMessage(), + requestId); } - commandSender.sendPartitionMetadataResponse(error, ex.getMessage(), requestId); } - } - lookupSemaphore.release(); - return null; - }); + }); + } + }); } else { final String msg = "Client is not authorized to Get Partition Metadata"; log.warn("[{}] {} with role {} on topic {}", remoteAddress, msg, getPrincipal(), topicName); @@ -614,6 +710,16 @@ protected void handlePartitionMetadataRequest(CommandPartitionedTopicMetadata pa return null; }).exceptionally(ex -> { logAuthException(remoteAddress, "partition-metadata", getPrincipal(), Optional.of(topicName), ex); + Throwable actEx = FutureUtil.unwrapCompletionException(ex); + if (actEx instanceof WebApplicationException restException) { + if (restException.getResponse().getStatus() == Response.Status.NOT_FOUND.getStatusCode()) { + writeAndFlush(Commands.newPartitionMetadataResponse(ServerError.MetadataError, + "Tenant or namespace or topic does not exist: " + topicName.getNamespace() , + requestId)); + lookupSemaphore.release(); + return null; + } + } final String msg = "Exception occurred while trying to authorize get Partition Metadata"; writeAndFlush(Commands.newPartitionMetadataResponse(ServerError.AuthorizationError, msg, requestId)); @@ -717,7 +823,7 @@ private void completeConnect(int clientProtoVersion, String clientVersion) { } setRemoteEndpointProtocolVersion(clientProtoVersion); if (isNotBlank(clientVersion)) { - this.clientVersion = clientVersion.intern(); + this.clientVersion = StringInterner.intern(clientVersion); } if (!service.isAuthenticationEnabled()) { log.info("[{}] connected with clientVersion={}, clientProtocolVersion={}, proxyVersion={}", remoteAddress, @@ -987,7 +1093,6 @@ protected void handleConnect(CommandConnect connect) { try { byte[] authData = connect.hasAuthData() ? connect.getAuthData() : emptyArray; AuthData clientData = AuthData.of(authData); - // init authentication if (connect.hasAuthMethodName()) { authMethod = connect.getAuthMethodName(); @@ -1046,10 +1151,22 @@ protected void handleConnect(CommandConnect connect) { .getAuthenticationService() .getAuthenticationProvider(originalAuthMethod); + /** + * When both the broker and the proxy are configured with anonymousUserRole + * if the client does not configure an authentication method + * the proxy side will set the value of anonymousUserRole to clientAuthRole when it creates a connection + * and the value of clientAuthMethod will be none. + * Similarly, should also set the value of authRole to anonymousUserRole on the broker side. + */ if (originalAuthenticationProvider == null) { - throw new AuthenticationException( - String.format("Can't find AuthenticationProvider for original role" - + " using auth method [%s] is not available", originalAuthMethod)); + authRole = getBrokerService().getAuthenticationService().getAnonymousUserRole() + .orElseThrow(() -> + new AuthenticationException("No anonymous role, and can't find " + + "AuthenticationProvider for original role using auth method " + + "[" + originalAuthMethod + "] is not available")); + originalPrincipal = authRole; + completeConnect(clientProtocolVersion, clientVersion); + return; } originalAuthDataCopy = AuthData.of(connect.getOriginalAuthData().getBytes()); @@ -1158,7 +1275,8 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { remoteAddress, getPrincipal()); } - log.info("[{}] Subscribing on topic {} / {}", remoteAddress, topicName, subscriptionName); + log.info("[{}] Subscribing on topic {} / {}. consumerId: {}, role: {}", this.toString(), topicName, + subscriptionName, consumerId, getPrincipal()); try { Metadata.validateMetadata(metadata, service.getPulsar().getConfiguration().getMaxConsumerMetadataSize()); @@ -1181,15 +1299,25 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { commandSender.sendErrorResponse(requestId, ServerError.ServiceNotReady, "Consumer is already present on the connection"); } else if (existingConsumerFuture.isCompletedExceptionally()){ + log.warn("[{}][{}][{}] A failed consumer with id is already present on the connection," + + " consumerId={}", remoteAddress, topicName, subscriptionName, consumerId); ServerError error = getErrorCodeWithErrorLog(existingConsumerFuture, true, - String.format("Consumer subscribe failure. remoteAddress: %s, subscription: %s", - remoteAddress, subscriptionName)); - consumers.remove(consumerId, existingConsumerFuture); + String.format("A failed consumer with id is already present on the connection." + + " consumerId: %s, remoteAddress: %s, subscription: %s", + consumerId, remoteAddress, subscriptionName)); + /** + * This future may was failed due to the client closed a in-progress subscribing. + * See {@link #handleCloseConsumer(CommandCloseConsumer)} + * Do not remove the failed future at current line, it will be removed after the progress of + * the previous subscribing is done. + * Before the previous subscribing is done, the new subscribe request will always fail. + * This mechanism is in order to prevent more complex logic to handle the race conditions. + */ commandSender.sendErrorResponse(requestId, error, "Consumer that failed is already present on the connection"); } else { Consumer consumer = existingConsumerFuture.getNow(null); - log.info("[{}] Consumer with the same id is already created:" + log.warn("[{}] Consumer with the same id is already created:" + " consumerId={}, consumer={}", remoteAddress, consumerId, consumer); commandSender.sendSuccessResponse(requestId); @@ -1207,41 +1335,63 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { .failedFuture(new TopicNotFoundException( "Topic " + topicName + " does not exist")); } - - Topic topic = optTopic.get(); - - boolean rejectSubscriptionIfDoesNotExist = isDurable - && !service.isAllowAutoSubscriptionCreation(topicName.toString()) - && !topic.getSubscriptions().containsKey(subscriptionName) - && topic.isPersistent(); - - if (rejectSubscriptionIfDoesNotExist) { - return FutureUtil - .failedFuture( - new SubscriptionNotFoundException( - "Subscription does not exist")); + final Topic topic = optTopic.get(); + // Check max consumer limitation to avoid unnecessary ops wasting resources. For example: + // the new consumer reached max producer limitation, but pulsar did schema check first, + // it would waste CPU. + if (((AbstractTopic) topic).isConsumersExceededOnTopic()) { + log.warn("[{}] Attempting to add consumer to topic which reached max" + + " consumers limit", topic); + Throwable t = + new ConsumerBusyException("Topic reached max consumers limit"); + return FutureUtil.failedFuture(t); } + return service.isAllowAutoSubscriptionCreationAsync(topicName) + .thenCompose(isAllowedAutoSubscriptionCreation -> { + boolean rejectSubscriptionIfDoesNotExist = isDurable + && !isAllowedAutoSubscriptionCreation + && !topic.getSubscriptions().containsKey(subscriptionName) + && topic.isPersistent(); + + if (rejectSubscriptionIfDoesNotExist) { + return FutureUtil + .failedFuture( + new SubscriptionNotFoundException( + "Subscription does not exist")); + } - SubscriptionOption option = SubscriptionOption.builder().cnx(ServerCnx.this) - .subscriptionName(subscriptionName) - .consumerId(consumerId).subType(subType).priorityLevel(priorityLevel) - .consumerName(consumerName).isDurable(isDurable) - .startMessageId(startMessageId).metadata(metadata).readCompacted(readCompacted) - .initialPosition(initialPosition) - .startMessageRollbackDurationSec(startMessageRollbackDurationSec) - .replicatedSubscriptionStateArg(isReplicated).keySharedMeta(keySharedMeta) - .subscriptionProperties(subscriptionProperties) - .consumerEpoch(consumerEpoch) - .schemaType(schema == null ? null : schema.getType()) - .build(); - if (schema != null && schema.getType() != SchemaType.AUTO_CONSUME) { - return topic.addSchemaIfIdleOrCheckCompatible(schema) - .thenCompose(v -> topic.subscribe(option)); - } else { - return topic.subscribe(option); - } + SubscriptionOption option = SubscriptionOption.builder().cnx(ServerCnx.this) + .subscriptionName(subscriptionName) + .consumerId(consumerId).subType(subType) + .priorityLevel(priorityLevel) + .consumerName(consumerName).isDurable(isDurable) + .startMessageId(startMessageId).metadata(metadata) + .readCompacted(readCompacted) + .initialPosition(initialPosition) + .startMessageRollbackDurationSec(startMessageRollbackDurationSec) + .replicatedSubscriptionStateArg(isReplicated) + .keySharedMeta(keySharedMeta) + .subscriptionProperties(subscriptionProperties) + .consumerEpoch(consumerEpoch) + .schemaType(schema == null ? null : schema.getType()) + .build(); + if (schema != null && schema.getType() != SchemaType.AUTO_CONSUME) { + return ignoreUnrecoverableBKException + (topic.addSchemaIfIdleOrCheckCompatible(schema)) + .thenCompose(v -> topic.subscribe(option)); + } else { + return topic.subscribe(option); + } + }); }) .thenAccept(consumer -> { + if (consumer.checkAndApplyTopicMigration()) { + log.info("[{}] Disconnecting consumer {} on migrated subscription on topic {} / {}", + remoteAddress, consumerId, subscriptionName, topicName); + consumers.remove(consumerId, consumerFuture); + return; + } + if (consumerFuture.complete(consumer)) { log.info("[{}] Created subscription on topic {} / {}", remoteAddress, topicName, subscriptionName); @@ -1278,6 +1428,24 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { remoteAddress, topicName, subscriptionName, exception.getCause().getMessage()); } + } else if (exception.getCause() instanceof BrokerServiceException.TopicMigratedException) { + Optional clusterURL = getMigratedClusterUrl(service.getPulsar(), + topicName.toString()); + if (clusterURL.isPresent()) { + log.info("[{}] redirect migrated consumer to topic {}: " + + "consumerId={}, subName={}, {}", remoteAddress, + topicName, consumerId, subscriptionName, exception.getCause().getMessage()); + boolean msgSent = commandSender.sendTopicMigrated(ResourceType.Consumer, consumerId, + clusterURL.get().getBrokerServiceUrl(), + clusterURL.get().getBrokerServiceUrlTls()); + if (!msgSent) { + log.info("consumer client doesn't support topic migration handling {}-{}-{}", + topicName, remoteAddress, consumerId); + } + consumers.remove(consumerId, consumerFuture); + closeConsumer(consumerId, Optional.empty()); + return null; + } } else if (exception.getCause() instanceof BrokerServiceException) { log.warn("[{}][{}][{}] Failed to create consumer: consumerId={}, {}", remoteAddress, topicName, subscriptionName, @@ -1292,7 +1460,7 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { // Send error back to client, only if not completed already. if (consumerFuture.completeExceptionally(exception)) { commandSender.sendErrorResponse(requestId, - BrokerServiceException.getClientErrorCode(exception), + BrokerServiceException.getClientErrorCode(exception.getCause()), exception.getCause().getMessage()); } consumers.remove(consumerId, consumerFuture); @@ -1352,7 +1520,7 @@ protected void handleProducer(final CommandProducer cmdProducer) { cmdProducer.hasInitialSubscriptionName() ? cmdProducer.getInitialSubscriptionName() : null; final boolean supportsPartialProducer = supportsPartialProducer(); - TopicName topicName = validateTopicName(cmdProducer.getTopic(), requestId, cmdProducer); + final TopicName topicName = validateTopicName(cmdProducer.getTopic(), requestId, cmdProducer); if (topicName == null) { return; } @@ -1383,39 +1551,48 @@ protected void handleProducer(final CommandProducer cmdProducer) { CompletableFuture existingProducerFuture = producers.putIfAbsent(producerId, producerFuture); if (existingProducerFuture != null) { - if (existingProducerFuture.isDone() && !existingProducerFuture.isCompletedExceptionally()) { - Producer producer = existingProducerFuture.getNow(null); - log.info("[{}] Producer with the same id is already created:" - + " producerId={}, producer={}", remoteAddress, producerId, producer); - commandSender.sendProducerSuccessResponse(requestId, producer.getProducerName(), - producer.getSchemaVersion()); - return null; - } else { + if (!existingProducerFuture.isDone()) { // There was an early request to create a producer with same producerId. // This can happen when client timeout is lower than the broker timeouts. // We need to wait until the previous producer creation request // either complete or fails. - ServerError error = null; - if (!existingProducerFuture.isDone()) { - error = ServerError.ServiceNotReady; - } else { - error = getErrorCode(existingProducerFuture); - // remove producer with producerId as it's already completed with exception - producers.remove(producerId, existingProducerFuture); - } log.warn("[{}][{}] Producer with id is already present on the connection, producerId={}", remoteAddress, topicName, producerId); - commandSender.sendErrorResponse(requestId, error, "Producer is already present on the connection"); - return null; + commandSender.sendErrorResponse(requestId, ServerError.ServiceNotReady, + "Producer is already present on the connection"); + } else if (existingProducerFuture.isCompletedExceptionally()) { + // remove producer with producerId as it's already completed with exception + log.warn("[{}][{}] Producer with id is failed to register present on the connection, producerId={}", + remoteAddress, topicName, producerId); + ServerError error = getErrorCode(existingProducerFuture); + producers.remove(producerId, existingProducerFuture); + commandSender.sendErrorResponse(requestId, error, + "Producer is already failed to register present on the connection"); + } else { + Producer producer = existingProducerFuture.getNow(null); + log.info("[{}] [{}] Producer with the same id is already created:" + + " producerId={}, producer={}", remoteAddress, topicName, producerId, producer); + commandSender.sendProducerSuccessResponse(requestId, producer.getProducerName(), + producer.getSchemaVersion()); } + return null; } if (log.isDebugEnabled()) { - log.debug("[{}][{}] Creating producer. producerId={}, schema is {}", remoteAddress, topicName, - producerId, schema == null ? "absent" : "present"); + log.debug("[{}][{}] Creating producer. producerId={}, producerName={}, schema is {}", remoteAddress, + topicName, producerId, producerName, schema == null ? "absent" : "present"); } service.getOrCreateTopic(topicName.toString()).thenCompose((Topic topic) -> { + // Check max producer limitation to avoid unnecessary ops wasting resources. For example: the new + // producer reached max producer limitation, but pulsar did schema check first, it would waste CPU + if (((AbstractTopic) topic).isProducersExceeded(producerName)) { + log.warn("[{}] Attempting to add producer to topic which reached max producers limit", topic); + String errorMsg = "Topic '" + topicName.toString() + "' reached max producers limit"; + Throwable t = new BrokerServiceException.ProducerBusyException(errorMsg); + return CompletableFuture.failedFuture(t); + } + // Before creating producer, check if backlog quota exceeded // on topic for size based limit and time based limit CompletableFuture backlogQuotaCheckFuture = CompletableFuture.allOf( @@ -1457,61 +1634,53 @@ protected void handleProducer(final CommandProducer cmdProducer) { }); schemaVersionFuture.thenAccept(schemaVersion -> { - topic.checkIfTransactionBufferRecoverCompletely(isTxnEnabled).thenAccept(future -> { - CompletableFuture createInitSubFuture; - if (!Strings.isNullOrEmpty(initialSubscriptionName) - && topic.isPersistent() - && !topic.getSubscriptions().containsKey(initialSubscriptionName)) { - if (!this.getBrokerService().isAllowAutoSubscriptionCreation(topicName)) { - String msg = - "Could not create the initial subscription due to the auto subscription " - + "creation is not allowed."; - if (producerFuture.completeExceptionally( - new BrokerServiceException.NotAllowedException(msg))) { - log.warn("[{}] {} initialSubscriptionName: {}, topic: {}", - remoteAddress, msg, initialSubscriptionName, topicName); - commandSender.sendErrorResponse(requestId, - ServerError.NotAllowedError, msg); - } - producers.remove(producerId, producerFuture); - return; - } - createInitSubFuture = - topic.createSubscription(initialSubscriptionName, InitialPosition.Earliest, - false, null); - } else { - createInitSubFuture = CompletableFuture.completedFuture(null); - } + CompletionStage createInitSubFuture; + if (!Strings.isNullOrEmpty(initialSubscriptionName) + && topic.isPersistent() + && !topic.getSubscriptions().containsKey(initialSubscriptionName)) { + createInitSubFuture = service.isAllowAutoSubscriptionCreationAsync(topicName) + .thenCompose(isAllowAutoSubscriptionCreation -> { + if (!isAllowAutoSubscriptionCreation) { + return CompletableFuture.failedFuture( + new BrokerServiceException.NotAllowedException( + "Could not create the initial subscription due to the " + + "auto subscription creation is not allowed.")); + } + return topic.createSubscription(initialSubscriptionName, + InitialPosition.Earliest, false, null); + }); + } else { + createInitSubFuture = CompletableFuture.completedFuture(null); + } - createInitSubFuture.whenComplete((sub, ex) -> { - if (ex != null) { - String msg = - "Failed to create the initial subscription: " + ex.getCause().getMessage(); + createInitSubFuture.whenComplete((sub, ex) -> { + if (ex != null) { + final Throwable rc = FutureUtil.unwrapCompletionException(ex); + if (rc instanceof BrokerServiceException.NotAllowedException) { log.warn("[{}] {} initialSubscriptionName: {}, topic: {}", - remoteAddress, msg, initialSubscriptionName, topicName); - if (producerFuture.completeExceptionally(ex)) { + remoteAddress, rc.getMessage(), initialSubscriptionName, topicName); + if (producerFuture.completeExceptionally(rc)) { commandSender.sendErrorResponse(requestId, - BrokerServiceException.getClientErrorCode(ex), msg); + ServerError.NotAllowedError, rc.getMessage()); } producers.remove(producerId, producerFuture); return; } + String msg = + "Failed to create the initial subscription: " + ex.getCause().getMessage(); + log.warn("[{}] {} initialSubscriptionName: {}, topic: {}", + remoteAddress, msg, initialSubscriptionName, topicName); + if (producerFuture.completeExceptionally(ex)) { + commandSender.sendErrorResponse(requestId, + BrokerServiceException.getClientErrorCode(ex), msg); + } + producers.remove(producerId, producerFuture); + return; + } - buildProducerAndAddTopic(topic, producerId, producerName, requestId, isEncrypted, + buildProducerAndAddTopic(topic, producerId, producerName, requestId, isEncrypted, metadata, schemaVersion, epoch, userProvidedProducerName, topicName, producerAccessMode, topicEpoch, supportsPartialProducer, producerFuture); - }); - }).exceptionally(exception -> { - Throwable cause = exception.getCause(); - log.error("producerId {}, requestId {} : TransactionBuffer recover failed", - producerId, requestId, exception); - if (producerFuture.completeExceptionally(exception)) { - commandSender.sendErrorResponse(requestId, - ServiceUnitNotReadyException.getClientErrorCode(cause), - cause.getMessage()); - } - producers.remove(producerId, producerFuture); - return null; }); }); }); @@ -1536,11 +1705,27 @@ protected void handleProducer(final CommandProducer cmdProducer) { } producers.remove(producerId, producerFuture); return null; + } else if (cause instanceof BrokerServiceException.TopicMigratedException) { + Optional clusterURL = getMigratedClusterUrl(service.getPulsar(), topicName.toString()); + if (clusterURL.isPresent()) { + log.info("[{}] redirect migrated producer to topic {}: " + + "producerId={}, producerName = {}, {}", remoteAddress, + topicName, producerId, producerName, cause.getMessage()); + boolean msgSent = commandSender.sendTopicMigrated(ResourceType.Producer, producerId, + clusterURL.get().getBrokerServiceUrl(), clusterURL.get().getBrokerServiceUrlTls()); + if (!msgSent) { + log.info("client doesn't support topic migration handling {}-{}-{}", topicName, + remoteAddress, producerId); + } + producers.remove(producerId, producerFuture); + closeProducer(producerId, -1L, Optional.empty()); + return null; + } } // Do not print stack traces for expected exceptions if (cause instanceof NoSuchElementException) { - cause = new TopicNotFoundException("Topic Not Found."); + cause = new TopicNotFoundException(String.format("Topic not found %s", topicName.toString())); log.warn("[{}] Failed to load topic {}, producerId={}: Topic not found", remoteAddress, topicName, producerId); } else if (!Exceptions.areExceptionsPresentInChain(cause, @@ -1581,7 +1766,7 @@ private void buildProducerAndAddTopic(Topic topic, long producerId, String produ topic.addProducer(producer, producerQueuedFuture).thenAccept(newTopicEpoch -> { if (isActive()) { if (producerFuture.complete(producer)) { - log.info("[{}] Created new producer: {}", remoteAddress, producer); + log.info("[{}] Created new producer: {}, role: {}", remoteAddress, producer, getPrincipal()); commandSender.sendProducerSuccessResponse(requestId, producerName, producer.getLastSequenceId(), producer.getSchemaVersion(), newTopicEpoch, true /* producer is ready now */); @@ -1611,11 +1796,11 @@ private void buildProducerAndAddTopic(Topic topic, long producerId, String produ } producers.remove(producerId, producerFuture); - }).exceptionally(ex -> { + }).exceptionallyAsync(ex -> { if (ex.getCause() instanceof BrokerServiceException.TopicMigratedException) { - Optional clusterURL = getMigratedClusterUrl(service.getPulsar()); + Optional clusterURL = getMigratedClusterUrl(service.getPulsar(), topic.getName()); if (clusterURL.isPresent()) { - if (topic.isReplicationBacklogExist()) { + if (!topic.shouldProducerMigrate()) { log.info("Topic {} is migrated but replication backlog exist: " + "producerId = {}, producerName = {}, {}", topicName, producerId, producerName, ex.getCause().getMessage()); @@ -1652,7 +1837,7 @@ private void buildProducerAndAddTopic(Topic topic, long producerId, String produ BrokerServiceException.getClientErrorCode(ex), ex.getMessage()); } return null; - }); + }, ctx.executor()); producerQueuedFuture.thenRun(() -> { // If the producer is queued waiting, we will get an immediate notification @@ -1695,6 +1880,22 @@ protected void handleSend(CommandSend send, ByteBuf headersAndPayload) { printSendCommandDebug(send, headersAndPayload); } + // New messages are silently ignored during topic transfer. Note that the transferring flag is only set when the + // Extensible Load Manager is enabled. + if (producer.getTopic().isTransferring()) { + var pulsar = getBrokerService().pulsar(); + var ignoredMsgCount = send.getNumMessages(); + var ignoredSendMsgTotalCount = ExtensibleLoadManagerImpl.get(pulsar).getIgnoredSendMsgCount(). + addAndGet(ignoredMsgCount); + if (log.isDebugEnabled()) { + log.debug("Ignoring {} messages from:{}:{} to fenced topic:{} while transferring." + + " Total ignored message count: {}.", + ignoredMsgCount, remoteAddress, send.getProducerId(), producer.getTopic().getName(), + ignoredSendMsgTotalCount); + } + return; + } + if (producer.isNonPersistentTopic()) { // avoid processing non-persist message if reached max concurrent-message limit if (nonPersistentPendingMessages > maxNonPersistentPendingMessages) { @@ -1711,7 +1912,7 @@ protected void handleSend(CommandSend send, ByteBuf headersAndPayload) { } } - startSendOperation(producer, headersAndPayload.readableBytes(), send.getNumMessages()); + increasePendingSendRequestsAndPublishBytes(headersAndPayload.readableBytes()); if (send.hasTxnidMostBits() && send.hasTxnidLeastBits()) { TxnID txnID = new TxnID(send.getTxnidMostBits(), send.getTxnidLeastBits()); @@ -1723,7 +1924,7 @@ protected void handleSend(CommandSend send, ByteBuf headersAndPayload) { // This position is only used for shadow replicator Position position = send.hasMessageId() - ? PositionImpl.get(send.getMessageId().getLedgerId(), send.getMessageId().getEntryId()) : null; + ? PositionFactory.create(send.getMessageId().getLedgerId(), send.getMessageId().getEntryId()) : null; // Persist the message if (send.hasHighestSequenceId() && send.getSequenceId() <= send.getHighestSequenceId()) { @@ -1741,11 +1942,12 @@ private void printSendCommandDebug(CommandSend send, ByteBuf headersAndPayload) headersAndPayload.resetReaderIndex(); if (log.isDebugEnabled()) { log.debug("[{}] Received send message request. producer: {}:{} {}:{} size: {}," - + " partition key is: {}, ordering key is {}", + + " partition key is: {}, ordering key is {}, uncompressedSize is {}", remoteAddress, send.getProducerId(), send.getSequenceId(), msgMetadata.getProducerName(), msgMetadata.getSequenceId(), headersAndPayload.readableBytes(), msgMetadata.hasPartitionKey() ? msgMetadata.getPartitionKey() : null, - msgMetadata.hasOrderingKey() ? msgMetadata.getOrderingKey() : null); + msgMetadata.hasOrderingKey() ? msgMetadata.getOrderingKey() : null, + msgMetadata.getUncompressedSize()); } } @@ -1761,6 +1963,20 @@ protected void handleAck(CommandAck ack) { if (consumerFuture != null && consumerFuture.isDone() && !consumerFuture.isCompletedExceptionally()) { Consumer consumer = consumerFuture.getNow(null); + Subscription subscription = consumer.getSubscription(); + // Message acks are silently ignored during topic transfer. Note that the transferring flag is only set when + // the Extensible Load Manager is enabled. + if (subscription.getTopic().isTransferring()) { + var pulsar = getBrokerService().getPulsar(); + var ignoredAckCount = ack.getMessageIdsCount(); + var ignoredAckTotalCount = ExtensibleLoadManagerImpl.get(pulsar).getIgnoredAckCount(). + addAndGet(ignoredAckCount); + if (log.isDebugEnabled()) { + log.debug("[{}] [{}] Ignoring {} message acks during topic transfer. Total ignored ack count: {}", + subscription, consumerId, ignoredAckCount, ignoredAckTotalCount); + } + return; + } consumer.messageAcked(ack).thenRun(() -> { if (hasRequestId) { writeAndFlush(Commands.newAckResponse( @@ -1781,6 +1997,12 @@ protected void handleAck(CommandAck ack) { } return null; }); + } else { + if (log.isDebugEnabled()) { + log.debug("Consumer future is not complete(not complete or error), but received command ack. so discard" + + " this command. consumerId: {}, cnx: {}, messageIdCount: {}", ack.getConsumerId(), + this.toString(), ack.getMessageIdsCount()); + } } } @@ -1836,7 +2058,7 @@ protected void handleUnsubscribe(CommandUnsubscribe unsubscribe) { CompletableFuture consumerFuture = consumers.get(unsubscribe.getConsumerId()); if (consumerFuture != null && consumerFuture.isDone() && !consumerFuture.isCompletedExceptionally()) { - consumerFuture.getNow(null).doUnsubscribe(unsubscribe.getRequestId()); + consumerFuture.getNow(null).doUnsubscribe(unsubscribe.getRequestId(), unsubscribe.isForce()); } else { commandSender.sendErrorResponse(unsubscribe.getRequestId(), ServerError.MetadataError, "Consumer not found"); @@ -1871,7 +2093,7 @@ protected void handleSeek(CommandSeek seek) { } } - Position position = new PositionImpl(msgIdData.getLedgerId(), + Position position = AckSetStateUtil.createPositionWithAckSet(msgIdData.getLedgerId(), msgIdData.getEntryId(), ackSet); @@ -2032,23 +2254,31 @@ protected void handleGetLastMessageId(CommandGetLastMessageId getLastMessageId) long requestId = getLastMessageId.getRequestId(); Topic topic = consumer.getSubscription().getTopic(); - Position lastPosition = topic.getLastPosition(); - int partitionIndex = TopicName.getPartitionIndex(topic.getName()); - - Position markDeletePosition = null; - if (consumer.getSubscription() instanceof PersistentSubscription) { - markDeletePosition = ((PersistentSubscription) consumer.getSubscription()).getCursor() - .getMarkDeletedPosition(); - } - - getLargestBatchIndexWhenPossible( - topic, - (PositionImpl) lastPosition, - (PositionImpl) markDeletePosition, - partitionIndex, - requestId, - consumer.getSubscription().getName()); - + topic.checkIfTransactionBufferRecoverCompletely() + .thenCompose(__ -> topic.getLastDispatchablePosition()) + .thenApply(lastPosition -> { + int partitionIndex = TopicName.getPartitionIndex(topic.getName()); + + Position markDeletePosition = null; + if (consumer.getSubscription() instanceof PersistentSubscription) { + markDeletePosition = ((PersistentSubscription) consumer.getSubscription()).getCursor() + .getMarkDeletedPosition(); + } + + getLargestBatchIndexWhenPossible( + topic, + lastPosition, + markDeletePosition, + partitionIndex, + requestId, + consumer.getSubscription().getName(), + consumer.readCompacted()); + return null; + }).exceptionally(e -> { + writeAndFlush(Commands.newError(getLastMessageId.getRequestId(), + ServerError.UnknownError, "Failed to recover Transaction Buffer.")); + return null; + }); } else { writeAndFlush(Commands.newError(getLastMessageId.getRequestId(), ServerError.MetadataError, "Consumer not found")); @@ -2057,75 +2287,109 @@ protected void handleGetLastMessageId(CommandGetLastMessageId getLastMessageId) private void getLargestBatchIndexWhenPossible( Topic topic, - PositionImpl lastPosition, - PositionImpl markDeletePosition, + Position lastPosition, + Position markDeletePosition, int partitionIndex, long requestId, - String subscriptionName) { + String subscriptionName, + boolean readCompacted) { PersistentTopic persistentTopic = (PersistentTopic) topic; - ManagedLedgerImpl ml = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + ManagedLedger ml = persistentTopic.getManagedLedger(); // If it's not pointing to a valid entry, respond messageId of the current position. // If the compaction cursor reach the end of the topic, respond messageId from compacted ledger - Optional compactionHorizon = persistentTopic.getCompactedTopic().getCompactionHorizon(); - if (lastPosition.getEntryId() == -1 || (compactionHorizon.isPresent() - && lastPosition.compareTo((PositionImpl) compactionHorizon.get()) <= 0)) { - handleLastMessageIdFromCompactedLedger(persistentTopic, requestId, partitionIndex, - markDeletePosition); - return; - } + CompletableFuture compactionHorizonFuture = readCompacted + ? persistentTopic.getTopicCompactionService().getLastCompactedPosition() : + CompletableFuture.completedFuture(null); + + compactionHorizonFuture.whenComplete((compactionHorizon, ex) -> { + if (ex != null) { + log.error("Failed to get compactionHorizon.", ex); + writeAndFlush(Commands.newError(requestId, ServerError.MetadataError, ex.getMessage())); + return; + } + - // For a valid position, we read the entry out and parse the batch size from its metadata. - CompletableFuture entryFuture = new CompletableFuture<>(); - ml.asyncReadEntry(lastPosition, new AsyncCallbacks.ReadEntryCallback() { - @Override - public void readEntryComplete(Entry entry, Object ctx) { - entryFuture.complete(entry); + if (lastPosition.getEntryId() == -1 || !ml.getLedgersInfo().containsKey(lastPosition.getLedgerId())) { + // there is no entry in the original topic + if (compactionHorizon != null) { + // if readCompacted is true, we need to read the last entry from compacted topic + handleLastMessageIdFromCompactionService(persistentTopic, requestId, partitionIndex, + markDeletePosition); + } else { + // if readCompacted is false, we need to return MessageId.earliest + writeAndFlush(Commands.newGetLastMessageIdResponse(requestId, -1, -1, partitionIndex, -1, + markDeletePosition != null ? markDeletePosition.getLedgerId() : -1, + markDeletePosition != null ? markDeletePosition.getEntryId() : -1)); + } + return; } - @Override - public void readEntryFailed(ManagedLedgerException exception, Object ctx) { - entryFuture.completeExceptionally(exception); + if (compactionHorizon != null && lastPosition.compareTo(compactionHorizon) <= 0) { + handleLastMessageIdFromCompactionService(persistentTopic, requestId, partitionIndex, + markDeletePosition); + return; } - }, null); - CompletableFuture batchSizeFuture = entryFuture.thenApply(entry -> { - MessageMetadata metadata = Commands.parseMessageMetadata(entry.getDataBuffer()); - int batchSize = metadata.getNumMessagesInBatch(); - entry.release(); - return metadata.hasNumMessagesInBatch() ? batchSize : -1; - }); + // For a valid position, we read the entry out and parse the batch size from its metadata. + CompletableFuture entryFuture = new CompletableFuture<>(); + ml.asyncReadEntry(lastPosition, new AsyncCallbacks.ReadEntryCallback() { + @Override + public void readEntryComplete(Entry entry, Object ctx) { + entryFuture.complete(entry); + } - batchSizeFuture.whenComplete((batchSize, e) -> { - if (e != null) { - if (e.getCause() instanceof ManagedLedgerException.NonRecoverableLedgerException) { - handleLastMessageIdFromCompactedLedger(persistentTopic, requestId, partitionIndex, - markDeletePosition); - } else { - writeAndFlush(Commands.newError( - requestId, ServerError.MetadataError, - "Failed to get batch size for entry " + e.getMessage())); + @Override + public void readEntryFailed(ManagedLedgerException exception, Object ctx) { + entryFuture.completeExceptionally(exception); } - } else { - int largestBatchIndex = batchSize > 0 ? batchSize - 1 : -1; - if (log.isDebugEnabled()) { - log.debug("[{}] [{}][{}] Get LastMessageId {} partitionIndex {}", remoteAddress, - topic.getName(), subscriptionName, lastPosition, partitionIndex); + @Override + public String toString() { + return String.format("ServerCnx [%s] get largest batch index when possible", + ServerCnx.this.toString()); } + }, null); - writeAndFlush(Commands.newGetLastMessageIdResponse(requestId, lastPosition.getLedgerId(), - lastPosition.getEntryId(), partitionIndex, largestBatchIndex, - markDeletePosition != null ? markDeletePosition.getLedgerId() : -1, - markDeletePosition != null ? markDeletePosition.getEntryId() : -1)); - } + CompletableFuture batchSizeFuture = entryFuture.thenApply(entry -> { + MessageMetadata metadata = Commands.parseMessageMetadata(entry.getDataBuffer()); + int batchSize = metadata.getNumMessagesInBatch(); + entry.release(); + return metadata.hasNumMessagesInBatch() ? batchSize : -1; + }); + + batchSizeFuture.whenComplete((batchSize, e) -> { + if (e != null) { + if (e.getCause() instanceof ManagedLedgerException.NonRecoverableLedgerException + && readCompacted) { + handleLastMessageIdFromCompactionService(persistentTopic, requestId, partitionIndex, + markDeletePosition); + } else { + writeAndFlush(Commands.newError( + requestId, ServerError.MetadataError, + "Failed to get batch size for entry " + e.getMessage())); + } + } else { + int largestBatchIndex = batchSize > 0 ? batchSize - 1 : -1; + + if (log.isDebugEnabled()) { + log.debug("[{}] [{}][{}] Get LastMessageId {} partitionIndex {}", remoteAddress, + topic.getName(), subscriptionName, lastPosition, partitionIndex); + } + + writeAndFlush(Commands.newGetLastMessageIdResponse(requestId, lastPosition.getLedgerId(), + lastPosition.getEntryId(), partitionIndex, largestBatchIndex, + markDeletePosition != null ? markDeletePosition.getLedgerId() : -1, + markDeletePosition != null ? markDeletePosition.getEntryId() : -1)); + } + }); }); } - private void handleLastMessageIdFromCompactedLedger(PersistentTopic persistentTopic, long requestId, - int partitionIndex, PositionImpl markDeletePosition) { - persistentTopic.getCompactedTopic().readLastEntryOfCompactedLedger().thenAccept(entry -> { + private void handleLastMessageIdFromCompactionService(PersistentTopic persistentTopic, long requestId, + int partitionIndex, Position markDeletePosition) { + persistentTopic.getTopicCompactionService().readLastCompactedEntry().thenAccept(entry -> { if (entry != null) { try { // in this case, all the data has been compacted, so return the last position @@ -2235,11 +2499,11 @@ protected void handleGetTopicsOfNamespace(CommandGetTopicsOfNamespace commandGet if (lookupSemaphore.tryAcquire()) { isNamespaceOperationAllowed(namespaceName, NamespaceOperation.GET_TOPICS).thenApply(isAuthorized -> { if (isAuthorized) { - getBrokerService().pulsar().getNamespaceService().getListOfTopics(namespaceName, mode) + getBrokerService().pulsar().getNamespaceService().getListOfUserTopics(namespaceName, mode) .thenAccept(topics -> { boolean filterTopics = false; // filter system topic - List filteredTopics = TopicList.filterSystemTopic(topics); + List filteredTopics = topics; if (enableSubscriptionPatternEvaluation && topicsPattern.isPresent()) { if (topicsPattern.get().length() <= maxSubscriptionPatternLength) { @@ -2326,9 +2590,10 @@ remoteAddress, new String(commandGetSchema.getSchemaVersion()), schemaVersion = schemaService.versionFromBytes(commandGetSchema.getSchemaVersion()); } + final String topic = commandGetSchema.getTopic(); String schemaName; try { - schemaName = TopicName.get(commandGetSchema.getTopic()).getSchemaName(); + schemaName = TopicName.get(topic).getSchemaName(); } catch (Throwable t) { commandSender.sendGetSchemaErrorResponse(requestId, ServerError.InvalidTopicName, t.getMessage()); return; @@ -2337,7 +2602,7 @@ remoteAddress, new String(commandGetSchema.getSchemaVersion()), schemaService.getSchema(schemaName, schemaVersion).thenAccept(schemaAndMetadata -> { if (schemaAndMetadata == null) { commandSender.sendGetSchemaErrorResponse(requestId, ServerError.TopicNotFound, - "Topic not found or no-schema"); + String.format("Topic not found or no-schema %s", topic)); } else { commandSender.sendGetSchemaResponse(requestId, SchemaInfoUtil.newSchemaInfo(schemaName, schemaAndMetadata.schema), schemaAndMetadata.version); @@ -2355,7 +2620,7 @@ protected void handleGetOrCreateSchema(CommandGetOrCreateSchema commandGetOrCrea log.debug("Received CommandGetOrCreateSchema call from {}", remoteAddress); } long requestId = commandGetOrCreateSchema.getRequestId(); - String topicName = commandGetOrCreateSchema.getTopic(); + final String topicName = commandGetOrCreateSchema.getTopic(); SchemaData schemaData = getSchema(commandGetOrCreateSchema.getSchema()); SchemaData schema = schemaData.getType() == SchemaType.NONE ? null : schemaData; service.getTopicIfExists(topicName).thenAccept(topicOpt -> { @@ -2375,7 +2640,7 @@ protected void handleGetOrCreateSchema(CommandGetOrCreateSchema commandGetOrCrea }); } else { commandSender.sendGetOrCreateSchemaErrorResponse(requestId, ServerError.TopicNotFound, - "Topic not found"); + String.format("Topic not found %s", topicName)); } }).exceptionally(ex -> { ServerError errorCode = BrokerServiceException.getClientErrorCode(ex); @@ -2731,7 +2996,7 @@ protected void handleEndTxnOnSubscription(CommandEndTxnOnSubscription command) { Commands.newEndTxnOnSubscriptionResponse(requestId, txnidLeastBits, txnidMostBits)); return; } - // we only accept super user becase this endpoint is reserved for tc to broker communication + // we only accept super user because this endpoint is reserved for tc to broker communication isSuperUser() .thenCompose(isOwner -> { if (!isOwner) { @@ -2805,7 +3070,8 @@ private CompletableFuture tryAddSchema(Topic topic, SchemaData sc CompletableFuture result = new CompletableFuture<>(); if (hasSchema && (schemaValidationEnforced || topic.getSchemaValidationEnforced())) { result.completeExceptionally(new IncompatibleSchemaException( - "Producers cannot connect or send message without a schema to topics with a schema")); + "Producers cannot connect or send message without a schema to topics with a schema" + + "when SchemaValidationEnforced is enabled")); } else { result.complete(SchemaVersion.Empty); } @@ -2933,15 +3199,44 @@ protected void interceptCommand(BaseCommand command) throws InterceptException { public void closeProducer(Producer producer) { // removes producer-connection from map and send close command to producer safelyRemoveProducer(producer); + closeProducer(producer.getProducerId(), producer.getEpoch(), Optional.empty()); + } + + @Override + public void closeProducer(Producer producer, Optional assignedBrokerLookupData) { + // removes producer-connection from map and send close command to producer + safelyRemoveProducer(producer); + closeProducer(producer.getProducerId(), producer.getEpoch(), assignedBrokerLookupData); + } + + private LookupData getLookupData(BrokerLookupData lookupData) { + LookupOptions.LookupOptionsBuilder builder = LookupOptions.builder(); + if (StringUtils.isNotBlank((listenerName))) { + builder.advertisedListenerName(listenerName); + } + try { + return lookupData.toLookupResult(builder.build()).getLookupData(); + } catch (PulsarServerException e) { + log.error("Failed to get lookup data", e); + throw new RuntimeException(e); + } + } + + private void closeProducer(long producerId, long epoch, Optional assignedBrokerLookupData) { if (getRemoteEndpointProtocolVersion() >= v5.getValue()) { - writeAndFlush(Commands.newCloseProducer(producer.getProducerId(), -1L)); + assignedBrokerLookupData.ifPresentOrElse(lookup -> { + LookupData lookupData = getLookupData(lookup); + writeAndFlush(Commands.newCloseProducer(producerId, -1L, + lookupData.getBrokerUrl(), + lookupData.getBrokerUrlTls())); + }, + () -> writeAndFlush(Commands.newCloseProducer(producerId, -1L))); + // The client does not necessarily know that the producer is closed, but the connection is still // active, and there could be messages in flight already. We want to ignore these messages for a time // because they are expected. Once the interval has passed, the client should have received the // CloseProducer command and should not send any additional messages until it sends a create Producer // command. - final long epoch = producer.getEpoch(); - final long producerId = producer.getProducerId(); recentlyClosedProducers.put(producerId, epoch); ctx.executor().schedule(() -> { recentlyClosedProducers.remove(producerId, epoch); @@ -2953,11 +3248,21 @@ public void closeProducer(Producer producer) { } @Override - public void closeConsumer(Consumer consumer) { + public void closeConsumer(Consumer consumer, Optional assignedBrokerLookupData) { // removes consumer-connection from map and send close command to consumer safelyRemoveConsumer(consumer); + closeConsumer(consumer.consumerId(), assignedBrokerLookupData); + } + + private void closeConsumer(long consumerId, Optional assignedBrokerLookupData) { if (getRemoteEndpointProtocolVersion() >= v5.getValue()) { - writeAndFlush(Commands.newCloseConsumer(consumer.consumerId(), -1L)); + assignedBrokerLookupData.ifPresentOrElse(lookup -> { + LookupData lookupData = getLookupData(lookup); + writeAndFlush(Commands.newCloseConsumer(consumerId, -1L, + lookupData.getBrokerUrl(), + lookupData.getBrokerUrlTls())); + }, + () -> writeAndFlush(Commands.newCloseConsumer(consumerId, -1L, null, null))); } else { close(); } @@ -3028,64 +3333,20 @@ public boolean isWritable() { return ctx.channel().isWritable(); } - private static final Gauge throttledConnections = Gauge.build() - .name("pulsar_broker_throttled_connections") - .help("Counter of connections throttled because of per-connection limit") - .register(); - - private static final Gauge throttledConnectionsGlobal = Gauge.build() - .name("pulsar_broker_throttled_connections_global_limit") - .help("Counter of connections throttled because of per-connection limit") - .register(); - - public void startSendOperation(Producer producer, int msgSize, int numMessages) { - boolean isPublishRateExceeded = false; - if (preciseTopicPublishRateLimitingEnable) { - boolean isPreciseTopicPublishRateExceeded = - producer.getTopic().isTopicPublishRateExceeded(numMessages, msgSize); - if (isPreciseTopicPublishRateExceeded) { - producer.getTopic().disableCnxAutoRead(); - return; - } - isPublishRateExceeded = producer.getTopic().isBrokerPublishRateExceeded(); - } else { - if (producer.getTopic().isResourceGroupRateLimitingEnabled()) { - final boolean resourceGroupPublishRateExceeded = - producer.getTopic().isResourceGroupPublishRateExceeded(numMessages, msgSize); - if (resourceGroupPublishRateExceeded) { - producer.getTopic().disableCnxAutoRead(); - return; - } - } - isPublishRateExceeded = producer.getTopic().isPublishRateExceeded(); - } - - if (++pendingSendRequest == maxPendingSendRequests || isPublishRateExceeded) { - // When the quota of pending send requests is reached, stop reading from socket to cause backpressure on - // client connection, possibly shared between multiple producers - disableCnxAutoRead(); - autoReadDisabledRateLimiting = isPublishRateExceeded; - throttledConnections.inc(); - } - - if (pendingBytesPerThread.get().addAndGet(msgSize) >= maxPendingBytesPerThread - && !autoReadDisabledPublishBufferLimiting - && maxPendingBytesPerThread > 0) { - // Disable reading from all the connections associated with this thread - MutableInt pausedConnections = new MutableInt(); - cnxsPerThread.get().forEach(cnx -> { - if (cnx.hasProducers() && !cnx.autoReadDisabledPublishBufferLimiting) { - cnx.disableCnxAutoRead(); - cnx.autoReadDisabledPublishBufferLimiting = true; - pausedConnections.increment(); - } - }); - - getBrokerService().pausedConnections(pausedConnections.intValue()); + // handle throttling based on pending send requests in the same connection + // or the pending publish bytes + private void increasePendingSendRequestsAndPublishBytes(int msgSize) { + if (++pendingSendRequest == maxPendingSendRequests) { + throttleTracker.setPendingSendRequestsExceeded(true); } + PendingBytesPerThreadTracker.getInstance().incrementPublishBytes(msgSize, maxPendingBytesPerThread); } - private void recordRateLimitMetrics(ConcurrentLongHashMap> producers) { + + /** + * Increase the throttling metric for the topic when a producer is throttled. + */ + void increasePublishLimitedTimesForTopics() { producers.forEach((key, producerFuture) -> { if (producerFuture != null && producerFuture.isDone()) { Producer p = producerFuture.getNow(null); @@ -3098,65 +3359,17 @@ private void recordRateLimitMetrics(ConcurrentLongHashMap { - if (cnx.autoReadDisabledPublishBufferLimiting) { - cnx.autoReadDisabledPublishBufferLimiting = false; - cnx.enableCnxAutoRead(); - resumedConnections.increment(); - } - }); - - getBrokerService().resumedConnections(resumedConnections.intValue()); - } + PendingBytesPerThreadTracker.getInstance().decrementPublishBytes(msgSize, resumeThresholdPendingBytesPerThread); if (--pendingSendRequest == resumeReadsThreshold) { - enableCnxAutoRead(); + throttleTracker.setPendingSendRequestsExceeded(false); } + if (isNonPersistentTopic) { nonPersistentPendingMessages--; } } - @Override - public void enableCnxAutoRead() { - // we can add check (&& pendingSendRequest < MaxPendingSendRequests) here but then it requires - // pendingSendRequest to be volatile and it can be expensive while writing. also this will be called on if - // throttling is enable on the topic. so, avoid pendingSendRequest check will be fine. - if (ctx != null && !ctx.channel().config().isAutoRead() - && !autoReadDisabledRateLimiting && !autoReadDisabledPublishBufferLimiting) { - // Resume reading from socket if pending-request is not reached to threshold - ctx.channel().config().setAutoRead(true); - throttledConnections.dec(); - } - } - - @Override - public void disableCnxAutoRead() { - if (ctx != null && ctx.channel().config().isAutoRead()) { - ctx.channel().config().setAutoRead(false); - recordRateLimitMetrics(producers); - } - } - - @Override - public void cancelPublishRateLimiting() { - if (autoReadDisabledRateLimiting) { - autoReadDisabledRateLimiting = false; - } - } - - @Override - public void cancelPublishBufferLimiting() { - if (autoReadDisabledPublishBufferLimiting) { - autoReadDisabledPublishBufferLimiting = false; - throttledConnectionsGlobal.dec(); - } - } - private ServerError getErrorCode(CompletableFuture future) { return getErrorCodeWithErrorLog(future, false, null); } @@ -3188,7 +3401,7 @@ private void disableTcpNoDelayIfNeeded(String topic, String producerName) { } } catch (Throwable t) { log.warn("[{}] [{}] Failed to remove TCP no-delay property on client cnx {}", topic, producerName, - ctx.channel()); + this.toString()); } } } @@ -3251,6 +3464,31 @@ public SocketAddress getRemoteAddress() { return remoteAddress; } + /** + * Demo: [id: 0x2561bcd1, L:/10.0.136.103:6650 ! R:/240.240.0.5:58038] [SR:/240.240.0.5:58038]. + * L: local Address. + * R: remote address. + * SR: source remote address. It is the source address when enabled "haProxyProtocolEnabled". + */ + @Override + public String toString() { + ChannelHandlerContext ctx = ctx(); + // ctx.channel(): 96. + // clientSourceAddress: 5 + 46(ipv6). + // state: 19. + // Len = 166. + StringBuilder buf = new StringBuilder(166); + if (ctx == null) { + buf.append("[ctx: null]"); + } else { + buf.append(ctx.channel().toString()); + } + String clientSourceAddr = clientSourceAddress(); + buf.append(" [SR:").append(clientSourceAddr == null ? "-" : clientSourceAddr) + .append(", state:").append(state).append("]"); + return buf.toString(); + } + @Override public BrokerService getBrokerService() { return service; @@ -3307,11 +3545,6 @@ public String getProxyVersion() { return proxyVersion; } - @VisibleForTesting - void setAutoReadDisabledRateLimiting(boolean isLimiting) { - this.autoReadDisabledRateLimiting = isLimiting; - } - @Override public boolean isPreciseDispatcherFlowControl() { return preciseDispatcherFlowControl; @@ -3374,22 +3607,56 @@ public String clientSourceAddress() { } } - CompletableFuture connectionCheckInProgress; + @Override + public String clientSourceAddressAndPort() { + if (clientSourceAddressAndPort == null) { + if (hasHAProxyMessage()) { + clientSourceAddressAndPort = + getHAProxyMessage().sourceAddress() + ":" + getHAProxyMessage().sourcePort(); + } else { + clientSourceAddressAndPort = clientAddress().toString(); + } + } + return clientSourceAddressAndPort; + } + + CompletableFuture> connectionCheckInProgress; @Override - public CompletableFuture checkConnectionLiveness() { + public CompletableFuture> checkConnectionLiveness() { + if (!isActive()) { + return CompletableFuture.completedFuture(Optional.of(false)); + } if (connectionLivenessCheckTimeoutMillis > 0) { return NettyFutureUtil.toCompletableFuture(ctx.executor().submit(() -> { + if (!isActive()) { + return CompletableFuture.completedFuture(Optional.of(false)); + } if (connectionCheckInProgress != null) { return connectionCheckInProgress; } else { - final CompletableFuture finalConnectionCheckInProgress = new CompletableFuture<>(); + final CompletableFuture> finalConnectionCheckInProgress = + new CompletableFuture<>(); connectionCheckInProgress = finalConnectionCheckInProgress; ctx.executor().schedule(() -> { - if (finalConnectionCheckInProgress == connectionCheckInProgress - && !finalConnectionCheckInProgress.isDone()) { - log.warn("[{}] Connection check timed out. Closing connection.", remoteAddress); + if (!isActive()) { + finalConnectionCheckInProgress.complete(Optional.of(false)); + return; + } + if (finalConnectionCheckInProgress.isDone()) { + return; + } + if (finalConnectionCheckInProgress == connectionCheckInProgress) { + /** + * {@link #connectionCheckInProgress} will be completed when + * {@link #channelInactive(ChannelHandlerContext)} event occurs, so skip set it here. + */ + log.warn("[{}] Connection check timed out. Closing connection.", this.toString()); ctx.close(); + } else { + log.error("[{}] Reached unexpected code block. Completing connection check.", + this.toString()); + finalConnectionCheckInProgress.complete(Optional.of(true)); } }, connectionLivenessCheckTimeoutMillis, TimeUnit.MILLISECONDS); sendPing(); @@ -3398,7 +3665,7 @@ public CompletableFuture checkConnectionLiveness() { })).thenCompose(java.util.function.Function.identity()); } else { // check is disabled - return CompletableFuture.completedFuture((Boolean) null); + return CompletableFuture.completedFuture(Optional.empty()); } } @@ -3406,7 +3673,7 @@ public CompletableFuture checkConnectionLiveness() { protected void messageReceived() { super.messageReceived(); if (connectionCheckInProgress != null && !connectionCheckInProgress.isDone()) { - connectionCheckInProgress.complete(true); + connectionCheckInProgress.complete(Optional.of(true)); connectionCheckInProgress = null; } } @@ -3414,13 +3681,22 @@ protected void messageReceived() { private static void logAuthException(SocketAddress remoteAddress, String operation, String principal, Optional topic, Throwable ex) { String topicString = topic.map(t -> ", topic=" + t.toString()).orElse(""); - if (ex instanceof AuthenticationException) { + Throwable actEx = FutureUtil.unwrapCompletionException(ex); + if (actEx instanceof AuthenticationException) { log.info("[{}] Failed to authenticate: operation={}, principal={}{}, reason={}", - remoteAddress, operation, principal, topicString, ex.getMessage()); - } else { - log.error("[{}] Error trying to authenticate: operation={}, principal={}{}", - remoteAddress, operation, principal, topicString, ex); + remoteAddress, operation, principal, topicString, actEx.getMessage()); + return; + } else if (actEx instanceof WebApplicationException restException){ + // Do not print error log if users tries to access a not found resource. + if (restException.getResponse().getStatus() == Response.Status.NOT_FOUND.getStatusCode()) { + log.info("[{}] Trying to authenticate for a topic which under a namespace not exists: operation={}," + + " principal={}{}, reason: {}", + remoteAddress, operation, principal, topicString, actEx.getMessage()); + return; + } } + log.error("[{}] Error trying to authenticate: operation={}, principal={}{}", + remoteAddress, operation, principal, topicString, ex); } private static void logNamespaceNameAuthException(SocketAddress remoteAddress, String operation, @@ -3463,4 +3739,20 @@ protected AuthenticationState getOriginalAuthState() { protected void setAuthRole(String authRole) { this.authRole = authRole; } + + /** + * {@inheritDoc} + */ + @Override + public void incrementThrottleCount() { + throttleTracker.incrementThrottleCount(); + } + + /** + * {@inheritDoc} + */ + @Override + public void decrementThrottleCount() { + throttleTracker.decrementThrottleCount(); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnxThrottleTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnxThrottleTracker.java new file mode 100644 index 0000000000000..78bac024218d8 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnxThrottleTracker.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import lombok.extern.slf4j.Slf4j; + +/** + * Tracks the state of throttling for a connection. The throttling happens by pausing reads by setting + * Netty {@link io.netty.channel.ChannelConfig#setAutoRead(boolean)} to false for the channel (connection). + *

+ * There can be multiple rate limiters that can throttle a connection. Each rate limiter will independently + * call the {@link #incrementThrottleCount()} and {@link #decrementThrottleCount()} methods to signal that the + * connection should be throttled or not. The connection will be throttled if the counter is greater than 0. + *

+ * Besides the rate limiters, the connection can also be throttled if the number of pending publish requests exceeds + * a configured threshold. This throttling is toggled with the {@link #setPendingSendRequestsExceeded} method. + * There's also per-thread memory limits which could throttle the connection. This throttling is toggled with the + * {@link #setPublishBufferLimiting} method. Internally, these two methods will call the + * {@link #incrementThrottleCount()} and {@link #decrementThrottleCount()} methods when the state changes. + */ +@Slf4j +final class ServerCnxThrottleTracker { + + private static final AtomicIntegerFieldUpdater THROTTLE_COUNT_UPDATER = + AtomicIntegerFieldUpdater.newUpdater( + ServerCnxThrottleTracker.class, "throttleCount"); + + private static final AtomicIntegerFieldUpdater + PENDING_SEND_REQUESTS_EXCEEDED_UPDATER = + AtomicIntegerFieldUpdater.newUpdater( + ServerCnxThrottleTracker.class, "pendingSendRequestsExceeded"); + private static final AtomicIntegerFieldUpdater PUBLISH_BUFFER_LIMITING_UPDATER = + AtomicIntegerFieldUpdater.newUpdater( + ServerCnxThrottleTracker.class, "publishBufferLimiting"); + private final ServerCnx serverCnx; + private volatile int throttleCount; + private volatile int pendingSendRequestsExceeded; + private volatile int publishBufferLimiting; + + + public ServerCnxThrottleTracker(ServerCnx serverCnx) { + this.serverCnx = serverCnx; + + } + + /** + * See {@link Producer#incrementThrottleCount()} for documentation. + */ + public void incrementThrottleCount() { + int currentThrottleCount = THROTTLE_COUNT_UPDATER.incrementAndGet(this); + if (currentThrottleCount == 1) { + changeAutoRead(false); + } + } + + /** + * See {@link Producer#decrementThrottleCount()} for documentation. + */ + public void decrementThrottleCount() { + int currentThrottleCount = THROTTLE_COUNT_UPDATER.decrementAndGet(this); + if (currentThrottleCount == 0) { + changeAutoRead(true); + } + } + + private void changeAutoRead(boolean autoRead) { + if (isChannelActive()) { + if (log.isDebugEnabled()) { + log.debug("[{}] Setting auto read to {}", serverCnx.toString(), autoRead); + } + // change the auto read flag on the channel + serverCnx.ctx().channel().config().setAutoRead(autoRead); + } + // update the metrics that track throttling + if (autoRead) { + serverCnx.getBrokerService().recordConnectionResumed(); + } else if (isChannelActive()) { + serverCnx.increasePublishLimitedTimesForTopics(); + serverCnx.getBrokerService().recordConnectionPaused(); + } + } + + private boolean isChannelActive() { + return serverCnx.isActive() && serverCnx.ctx() != null && serverCnx.ctx().channel().isActive(); + } + + public void setPublishBufferLimiting(boolean throttlingEnabled) { + changeThrottlingFlag(PUBLISH_BUFFER_LIMITING_UPDATER, throttlingEnabled); + } + + public void setPendingSendRequestsExceeded(boolean throttlingEnabled) { + boolean changed = changeThrottlingFlag(PENDING_SEND_REQUESTS_EXCEEDED_UPDATER, throttlingEnabled); + if (changed) { + // update the metrics that track throttling due to pending send requests + if (throttlingEnabled) { + serverCnx.getBrokerService().recordConnectionThrottled(); + } else { + serverCnx.getBrokerService().recordConnectionUnthrottled(); + } + } + } + + private boolean changeThrottlingFlag(AtomicIntegerFieldUpdater throttlingFlagFieldUpdater, + boolean throttlingEnabled) { + // don't change a throttling flag if the channel is not active + if (!isChannelActive()) { + return false; + } + if (throttlingFlagFieldUpdater.compareAndSet(this, booleanToInt(!throttlingEnabled), + booleanToInt(throttlingEnabled))) { + if (throttlingEnabled) { + incrementThrottleCount(); + } else { + decrementThrottleCount(); + } + return true; + } else { + return false; + } + } + + private static int booleanToInt(boolean value) { + return value ? 1 : 0; + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Subscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Subscription.java index 1f418a6e9f85f..452c30b45febb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Subscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Subscription.java @@ -24,13 +24,13 @@ import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.intercept.BrokerInterceptor; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.common.api.proto.CommandAck.AckType; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.api.proto.ReplicatedSubscriptionsSnapshot; -public interface Subscription { +public interface Subscription extends MessageExpirer { BrokerInterceptor interceptor(); @@ -64,16 +64,18 @@ default long getNumberOfEntriesDelayed() { List getConsumers(); - CompletableFuture close(); - CompletableFuture delete(); CompletableFuture deleteForcefully(); - CompletableFuture disconnect(); + CompletableFuture disconnect(Optional assignedBrokerLookupData); + + CompletableFuture close(boolean disconnectConsumers, Optional assignedBrokerLookupData); CompletableFuture doUnsubscribe(Consumer consumer); + CompletableFuture doUnsubscribe(Consumer consumer, boolean forcefully); + CompletableFuture clearBacklog(); CompletableFuture skipMessages(int numMessagesToSkip); @@ -84,13 +86,9 @@ default long getNumberOfEntriesDelayed() { CompletableFuture peekNthMessage(int messagePosition); - boolean expireMessages(int messageTTLInSeconds); - - boolean expireMessages(Position position); - void redeliverUnacknowledgedMessages(Consumer consumer, long consumerEpoch); - void redeliverUnacknowledgedMessages(Consumer consumer, List positions); + void redeliverUnacknowledgedMessages(Consumer consumer, List positions); void markTopicWithBatchMessagePublished(); @@ -106,6 +104,8 @@ default long getNumberOfEntriesDelayed() { CompletableFuture updateSubscriptionProperties(Map subscriptionProperties); + boolean isSubscriptionMigrated(); + default void processReplicatedSubscriptionSnapshot(ReplicatedSubscriptionsSnapshot snapshot) { // Default is no-op } @@ -134,4 +134,5 @@ static boolean isCumulativeAckMode(SubType subType) { static boolean isIndividualAckMode(SubType subType) { return SubType.Shared.equals(subType) || SubType.Key_Shared.equals(subType); } + } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 9b10055f36fac..6ff6408916b1c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -18,38 +18,43 @@ */ package org.apache.pulsar.broker.service; +import static java.util.Objects.requireNonNull; +import com.github.benmanes.caffeine.cache.AsyncLoadingCache; +import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.Nonnull; +import org.apache.commons.lang3.concurrent.ConcurrentInitializer; +import org.apache.commons.lang3.concurrent.LazyInitializer; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.namespace.NamespaceBundleOwnershipListener; import org.apache.pulsar.broker.namespace.NamespaceService; -import org.apache.pulsar.broker.service.BrokerServiceException.TopicPoliciesCacheNotInitException; import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.client.impl.TopicMessageImpl; -import org.apache.pulsar.client.util.RetryUtil; import org.apache.pulsar.common.events.ActionType; import org.apache.pulsar.common.events.EventType; import org.apache.pulsar.common.events.PulsarEvent; import org.apache.pulsar.common.events.TopicPoliciesEvent; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.util.FutureUtil; @@ -66,7 +71,20 @@ public class SystemTopicBasedTopicPoliciesService implements TopicPoliciesServic private final PulsarService pulsarService; private final HashSet localCluster; private final String clusterName; - private volatile NamespaceEventsSystemTopicFactory namespaceEventsSystemTopicFactory; + private final AtomicBoolean closed = new AtomicBoolean(false); + + private final ConcurrentInitializer + namespaceEventsSystemTopicFactoryLazyInitializer = new LazyInitializer<>() { + @Override + protected NamespaceEventsSystemTopicFactory initialize() { + try { + return new NamespaceEventsSystemTopicFactory(pulsarService.getClient()); + } catch (PulsarServerException e) { + log.error("Create namespace event system topic factory error.", e); + throw new RuntimeException(e); + } + } + }; @VisibleForTesting final Map policiesCache = new ConcurrentHashMap<>(); @@ -77,34 +95,58 @@ public class SystemTopicBasedTopicPoliciesService implements TopicPoliciesServic private final Map>> readerCaches = new ConcurrentHashMap<>(); - @VisibleForTesting - final Map policyCacheInitMap = new ConcurrentHashMap<>(); + + final Map> policyCacheInitMap = new ConcurrentHashMap<>(); @VisibleForTesting - final Map>> listeners = new ConcurrentHashMap<>(); + final Map> listeners = new ConcurrentHashMap<>(); + + private final AsyncLoadingCache> writerCaches; public SystemTopicBasedTopicPoliciesService(PulsarService pulsarService) { this.pulsarService = pulsarService; this.clusterName = pulsarService.getConfiguration().getClusterName(); this.localCluster = Sets.newHashSet(clusterName); + this.writerCaches = Caffeine.newBuilder() + .expireAfterAccess(5, TimeUnit.MINUTES) + .removalListener((namespaceName, writer, cause) -> { + try { + ((SystemTopicClient.Writer) writer).close(); + } catch (Exception e) { + log.error("[{}] Close writer error.", namespaceName, e); + } + }) + .executor(pulsarService.getExecutor()) + .buildAsync((namespaceName, executor) -> { + if (closed.get()) { + return CompletableFuture.failedFuture( + new BrokerServiceException(getClass().getName() + " is closed.")); + } + SystemTopicClient systemTopicClient = getNamespaceEventsSystemTopicFactory() + .createTopicPoliciesSystemTopicClient(namespaceName); + return systemTopicClient.newWriterAsync(); + }); } @Override public CompletableFuture deleteTopicPoliciesAsync(TopicName topicName) { + if (NamespaceService.isHeartbeatNamespace(topicName.getNamespaceObject()) || isSelf(topicName)) { + return CompletableFuture.completedFuture(null); + } return sendTopicPolicyEvent(topicName, ActionType.DELETE, null); } @Override public CompletableFuture updateTopicPoliciesAsync(TopicName topicName, TopicPolicies policies) { + if (NamespaceService.isHeartbeatNamespace(topicName.getNamespaceObject())) { + return CompletableFuture.failedFuture(new BrokerServiceException.NotAllowedException( + "Not allowed to update topic policy for the heartbeat topic")); + } return sendTopicPolicyEvent(topicName, ActionType.UPDATE, policies); } private CompletableFuture sendTopicPolicyEvent(TopicName topicName, ActionType actionType, TopicPolicies policies) { - if (NamespaceService.isHeartbeatNamespace(topicName.getNamespaceObject())) { - return CompletableFuture.failedFuture( - new BrokerServiceException.NotAllowedException("Not allowed to send event to health check topic")); - } return pulsarService.getPulsarResources().getNamespaceResources() .getPoliciesAsync(topicName.getNamespaceObject()) .thenCompose(namespacePolicies -> { @@ -118,39 +160,32 @@ private CompletableFuture sendTopicPolicyEvent(TopicName topicName, Action } catch (PulsarServerException e) { return CompletableFuture.failedFuture(e); } - - SystemTopicClient systemTopicClient = namespaceEventsSystemTopicFactory - .createTopicPoliciesSystemTopicClient(topicName.getNamespaceObject()); - - return systemTopicClient.newWriterAsync() - .thenCompose(writer -> { - PulsarEvent event = getPulsarEvent(topicName, actionType, policies); - CompletableFuture writeFuture = - ActionType.DELETE.equals(actionType) ? writer.deleteAsync(getEventKey(event), event) - : writer.writeAsync(getEventKey(event), event); - return writeFuture.handle((messageId, e) -> { - if (e != null) { - return CompletableFuture.failedFuture(e); + CompletableFuture result = new CompletableFuture<>(); + writerCaches.get(topicName.getNamespaceObject()) + .whenComplete((writer, cause) -> { + if (cause != null) { + writerCaches.synchronous().invalidate(topicName.getNamespaceObject()); + result.completeExceptionally(cause); } else { - if (messageId != null) { - return CompletableFuture.completedFuture(null); - } else { - return CompletableFuture.failedFuture( - new RuntimeException("Got message id is null.")); - } - } - }).thenRun(() -> - writer.closeAsync().whenComplete((v, cause) -> { - if (cause != null) { - log.error("[{}] Close writer error.", topicName, cause); + PulsarEvent event = getPulsarEvent(topicName, actionType, policies); + CompletableFuture writeFuture = ActionType.DELETE.equals(actionType) + ? writer.deleteAsync(getEventKey(event), event) + : writer.writeAsync(getEventKey(event), event); + writeFuture.whenComplete((messageId, e) -> { + if (e != null) { + result.completeExceptionally(e); + } else { + if (messageId != null) { + result.complete(null); } else { - if (log.isDebugEnabled()) { - log.debug("[{}] Close writer success.", topicName); - } + result.completeExceptionally( + new RuntimeException("Got message id is null.")); } - }) - ); + } + }); + } }); + return result; }); } @@ -179,8 +214,12 @@ private void notifyListener(Message msg) { if (msg.getValue() == null) { TopicName topicName = TopicName.get(TopicName.get(msg.getKey()).getPartitionedTopicName()); if (listeners.get(topicName) != null) { - for (TopicPolicyListener listener : listeners.get(topicName)) { - listener.onUpdate(null); + for (TopicPolicyListener listener : listeners.get(topicName)) { + try { + listener.onUpdate(null); + } catch (Throwable error) { + log.error("[{}] call listener error.", topicName, error); + } } } return; @@ -194,122 +233,156 @@ private void notifyListener(Message msg) { event.getNamespace(), event.getTopic()); if (listeners.get(topicName) != null) { TopicPolicies policies = event.getPolicies(); - for (TopicPolicyListener listener : listeners.get(topicName)) { - listener.onUpdate(policies); + for (TopicPolicyListener listener : listeners.get(topicName)) { + try { + listener.onUpdate(policies); + } catch (Throwable error) { + log.error("[{}] call listener error.", topicName, error); + } } } } @Override - public TopicPolicies getTopicPolicies(TopicName topicName) throws TopicPoliciesCacheNotInitException { - return getTopicPolicies(topicName, false); - } - - @Override - public TopicPolicies getTopicPolicies(TopicName topicName, - boolean isGlobal) throws TopicPoliciesCacheNotInitException { - if (!policyCacheInitMap.containsKey(topicName.getNamespaceObject())) { - NamespaceName namespace = topicName.getNamespaceObject(); - prepareInitPoliciesCache(namespace, new CompletableFuture<>()); + public CompletableFuture> getTopicPoliciesAsync(TopicName topicName, GetType type) { + requireNonNull(topicName); + final var namespace = topicName.getNamespaceObject(); + if (NamespaceService.isHeartbeatNamespace(namespace) || isSelf(topicName)) { + return CompletableFuture.completedFuture(Optional.empty()); } - if (policyCacheInitMap.containsKey(topicName.getNamespaceObject()) - && !policyCacheInitMap.get(topicName.getNamespaceObject())) { - throw new TopicPoliciesCacheNotInitException(); + // When the extensible load manager initializes its channel topic, it will trigger the topic policies + // initialization by calling this method. At the moment, the load manager does not start so the lookup + // for "__change_events" will fail. In this case, just return an empty policies to avoid deadlock. + final var loadManager = pulsarService.getLoadManager().get(); + if (loadManager == null || !loadManager.started() || closed.get()) { + return CompletableFuture.completedFuture(Optional.empty()); } - return isGlobal ? globalPoliciesCache.get(TopicName.get(topicName.getPartitionedTopicName())) - : policiesCache.get(TopicName.get(topicName.getPartitionedTopicName())); - } - - @Override - public TopicPolicies getTopicPoliciesIfExists(TopicName topicName) { - return policiesCache.get(TopicName.get(topicName.getPartitionedTopicName())); - } - - @Override - public CompletableFuture getTopicPoliciesBypassCacheAsync(TopicName topicName) { - CompletableFuture result = new CompletableFuture<>(); - try { - createSystemTopicFactoryIfNeeded(); - } catch (PulsarServerException e) { - result.complete(null); - return result; - } - SystemTopicClient systemTopicClient = namespaceEventsSystemTopicFactory - .createTopicPoliciesSystemTopicClient(topicName.getNamespaceObject()); - systemTopicClient.newReaderAsync().thenAccept(r -> - fetchTopicPoliciesAsyncAndCloseReader(r, topicName, null, result)); - return result; + final CompletableFuture preparedFuture = prepareInitPoliciesCacheAsync(topicName.getNamespaceObject()); + final var resultFuture = new CompletableFuture>(); + preparedFuture.thenAccept(inserted -> policyCacheInitMap.compute(namespace, (___, existingFuture) -> { + if (!inserted || existingFuture != null) { + final var partitionedTopicName = TopicName.get(topicName.getPartitionedTopicName()); + final var policies = Optional.ofNullable(switch (type) { + case DEFAULT -> Optional.ofNullable(policiesCache.get(partitionedTopicName)) + .orElseGet(() -> globalPoliciesCache.get(partitionedTopicName)); + case GLOBAL_ONLY -> globalPoliciesCache.get(partitionedTopicName); + case LOCAL_ONLY -> policiesCache.get(partitionedTopicName); + }); + resultFuture.complete(policies); + } else { + CompletableFuture.runAsync(() -> { + log.info("The future of {} has been removed from cache, retry getTopicPolicies again", namespace); + // Call it in another thread to avoid recursive update because getTopicPoliciesAsync() could call + // policyCacheInitMap.computeIfAbsent() + getTopicPoliciesAsync(topicName, type).whenComplete((result, e) -> { + if (e == null) { + resultFuture.complete(result); + } else { + resultFuture.completeExceptionally(e); + } + }); + }); + } + return existingFuture; + })).exceptionally(e -> { + resultFuture.completeExceptionally(e); + return null; + }); + return resultFuture; } - @Override - public CompletableFuture addOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { - CompletableFuture result = new CompletableFuture<>(); + public void addOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { NamespaceName namespace = namespaceBundle.getNamespaceObject(); if (NamespaceService.isHeartbeatNamespace(namespace)) { - result.complete(null); - return result; + return; } synchronized (this) { if (readerCaches.get(namespace) != null) { ownedBundlesCountPerNamespace.get(namespace).incrementAndGet(); - result.complete(null); } else { - prepareInitPoliciesCache(namespace, result); + prepareInitPoliciesCacheAsync(namespace); } } - return result; - } - - private void prepareInitPoliciesCache(@Nonnull NamespaceName namespace, CompletableFuture result) { - if (policyCacheInitMap.putIfAbsent(namespace, false) == null) { - CompletableFuture> readerCompletableFuture = - createSystemTopicClientWithRetry(namespace); - readerCaches.put(namespace, readerCompletableFuture); - ownedBundlesCountPerNamespace.putIfAbsent(namespace, new AtomicInteger(1)); - readerCompletableFuture.thenAccept(reader -> { - initPolicesCache(reader, result); - result.thenRun(() -> readMorePolicies(reader)); - }).exceptionally(ex -> { - log.error("[{}] Failed to create reader on __change_events topic", namespace, ex); - cleanCacheAndCloseReader(namespace, false); - result.completeExceptionally(ex); - return null; - }); + } + + @VisibleForTesting + @Nonnull CompletableFuture prepareInitPoliciesCacheAsync(@Nonnull NamespaceName namespace) { + requireNonNull(namespace); + if (closed.get()) { + return CompletableFuture.completedFuture(false); } + return pulsarService.getPulsarResources().getNamespaceResources().getPoliciesAsync(namespace) + .thenCompose(namespacePolicies -> { + if (namespacePolicies.isEmpty() || namespacePolicies.get().deleted) { + log.info("[{}] skip prepare init policies cache since the namespace is deleted", + namespace); + return CompletableFuture.completedFuture(false); + } + + return policyCacheInitMap.computeIfAbsent(namespace, (k) -> { + final CompletableFuture> readerCompletableFuture = + createSystemTopicClient(namespace); + readerCaches.put(namespace, readerCompletableFuture); + ownedBundlesCountPerNamespace.putIfAbsent(namespace, new AtomicInteger(1)); + final CompletableFuture initFuture = readerCompletableFuture + .thenCompose(reader -> { + final CompletableFuture stageFuture = new CompletableFuture<>(); + initPolicesCache(reader, stageFuture); + return stageFuture + // Read policies in background + .thenAccept(__ -> readMorePoliciesAsync(reader)); + }); + initFuture.exceptionally(ex -> { + try { + if (closed.get()) { + return null; + } + log.error("[{}] Failed to create reader on __change_events topic", + namespace, ex); + cleanCacheAndCloseReader(namespace, false); + } catch (Throwable cleanupEx) { + // Adding this catch to avoid break callback chain + log.error("[{}] Failed to cleanup reader on __change_events topic", + namespace, cleanupEx); + } + return null; + }); + // let caller know we've got an exception. + return initFuture; + }).thenApply(__ -> true); + }); } - protected CompletableFuture> createSystemTopicClientWithRetry( + protected CompletableFuture> createSystemTopicClient( NamespaceName namespace) { - CompletableFuture> result = new CompletableFuture<>(); + if (closed.get()) { + return CompletableFuture.failedFuture( + new BrokerServiceException(getClass().getName() + " is closed.")); + } try { createSystemTopicFactoryIfNeeded(); - } catch (PulsarServerException e) { - result.completeExceptionally(e); - return result; + } catch (PulsarServerException ex) { + return FutureUtil.failedFuture(ex); } - SystemTopicClient systemTopicClient = namespaceEventsSystemTopicFactory + final SystemTopicClient systemTopicClient = getNamespaceEventsSystemTopicFactory() .createTopicPoliciesSystemTopicClient(namespace); - Backoff backoff = new Backoff(1, TimeUnit.SECONDS, 3, TimeUnit.SECONDS, 10, TimeUnit.SECONDS); - RetryUtil.retryAsynchronously(systemTopicClient::newReaderAsync, backoff, pulsarService.getExecutor(), result); - return result; + return systemTopicClient.newReaderAsync(); } - @Override - public CompletableFuture removeOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { + private void removeOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { NamespaceName namespace = namespaceBundle.getNamespaceObject(); if (NamespaceService.checkHeartbeatNamespace(namespace) != null || NamespaceService.checkHeartbeatNamespaceV2(namespace) != null) { - return CompletableFuture.completedFuture(null); + return; } AtomicInteger bundlesCount = ownedBundlesCountPerNamespace.get(namespace); if (bundlesCount == null || bundlesCount.decrementAndGet() <= 0) { - cleanCacheAndCloseReader(namespace, true); + cleanCacheAndCloseReader(namespace, true, true); } - return CompletableFuture.completedFuture(null); } @Override - public void start() { + public void start(PulsarService pulsarService) { pulsarService.getNamespaceService().addNamespaceBundleOwnershipListener( new NamespaceBundleOwnershipListener() { @@ -332,6 +405,11 @@ public boolean test(NamespaceBundle namespaceBundle) { } private void initPolicesCache(SystemTopicClient.Reader reader, CompletableFuture future) { + if (closed.get()) { + future.completeExceptionally(new BrokerServiceException(getClass().getName() + " is closed.")); + cleanCacheAndCloseReader(reader.getSystemTopic().getTopicName().getNamespaceObject(), false); + return; + } reader.hasMoreEventsAsync().whenComplete((hasMore, ex) -> { if (ex != null) { log.error("[{}] Failed to check the move events for the system topic", @@ -359,24 +437,36 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp if (log.isDebugEnabled()) { log.debug("[{}] Reach the end of the system topic.", reader.getSystemTopic().getTopicName()); } - policyCacheInitMap.computeIfPresent( - reader.getSystemTopic().getTopicName().getNamespaceObject(), (k, v) -> true); + // replay policy message policiesCache.forEach(((topicName, topicPolicies) -> { if (listeners.get(topicName) != null) { - for (TopicPolicyListener listener : listeners.get(topicName)) { - listener.onUpdate(topicPolicies); + for (TopicPolicyListener listener : listeners.get(topicName)) { + try { + listener.onUpdate(topicPolicies); + } catch (Throwable error) { + log.error("[{}] call listener error.", topicName, error); + } } } })); + future.complete(null); } }); } private void cleanCacheAndCloseReader(@Nonnull NamespaceName namespace, boolean cleanOwnedBundlesCount) { + cleanCacheAndCloseReader(namespace, cleanOwnedBundlesCount, false); + } + + private void cleanCacheAndCloseReader(@Nonnull NamespaceName namespace, boolean cleanOwnedBundlesCount, + boolean cleanWriterCache) { + if (cleanWriterCache) { + writerCaches.synchronous().invalidate(namespace); + } CompletableFuture> readerFuture = readerCaches.remove(namespace); - policiesCache.entrySet().removeIf(entry -> Objects.equals(entry.getKey().getNamespaceObject(), namespace)); + if (cleanOwnedBundlesCount) { ownedBundlesCountPerNamespace.remove(namespace); } @@ -387,10 +477,24 @@ private void cleanCacheAndCloseReader(@Nonnull NamespaceName namespace, boolean return null; }); } - policyCacheInitMap.remove(namespace); + + policyCacheInitMap.compute(namespace, (k, v) -> { + policiesCache.entrySet().removeIf(entry -> Objects.equals(entry.getKey().getNamespaceObject(), namespace)); + return null; + }); } - private void readMorePolicies(SystemTopicClient.Reader reader) { + /** + * This is an async method for the background reader to continue syncing new messages. + * + * Note: You should not do any blocking call here. because it will affect + * #{@link SystemTopicBasedTopicPoliciesService#getTopicPoliciesAsync} method to block loading topic. + */ + private void readMorePoliciesAsync(SystemTopicClient.Reader reader) { + if (closed.get()) { + cleanCacheAndCloseReader(reader.getSystemTopic().getTopicName().getNamespaceObject(), false); + return; + } reader.readNextAsync() .thenAccept(msg -> { refreshTopicPoliciesCache(msg); @@ -398,16 +502,17 @@ private void readMorePolicies(SystemTopicClient.Reader reader) { }) .whenComplete((__, ex) -> { if (ex == null) { - readMorePolicies(reader); + readMorePoliciesAsync(reader); } else { Throwable cause = FutureUtil.unwrapCompletionException(ex); if (cause instanceof PulsarClientException.AlreadyClosedException) { - log.warn("Read more topic policies exception, close the read now!", ex); + log.info("Closing the topic policies reader for {}", + reader.getSystemTopic().getTopicName()); cleanCacheAndCloseReader( reader.getSystemTopic().getTopicName().getNamespaceObject(), false); } else { log.warn("Read more topic polices exception, read again.", ex); - readMorePolicies(reader); + readMorePoliciesAsync(reader); } } }); @@ -455,7 +560,7 @@ private void refreshTopicPoliciesCache(Message msg) { log.error("Failed to create system topic factory"); break; } - SystemTopicClient systemTopicClient = namespaceEventsSystemTopicFactory + SystemTopicClient systemTopicClient = getNamespaceEventsSystemTopicFactory() .createTopicPoliciesSystemTopicClient(topicName.getNamespaceObject()); systemTopicClient.newWriterAsync().thenAccept(writer -> writer.deleteAsync(getEventKey(topicName), @@ -489,60 +594,21 @@ private boolean hasReplicateTo(Message message) { } private void createSystemTopicFactoryIfNeeded() throws PulsarServerException { - if (namespaceEventsSystemTopicFactory == null) { - synchronized (this) { - if (namespaceEventsSystemTopicFactory == null) { - try { - namespaceEventsSystemTopicFactory = - new NamespaceEventsSystemTopicFactory(pulsarService.getClient()); - } catch (PulsarServerException e) { - log.error("Create namespace event system topic factory error.", e); - throw e; - } - } - } + try { + getNamespaceEventsSystemTopicFactory(); + } catch (Exception e) { + throw new PulsarServerException(e); } } - private void fetchTopicPoliciesAsyncAndCloseReader(SystemTopicClient.Reader reader, - TopicName topicName, TopicPolicies policies, - CompletableFuture future) { - reader.hasMoreEventsAsync().whenComplete((hasMore, ex) -> { - if (ex != null) { - future.completeExceptionally(ex); - } - if (hasMore) { - reader.readNextAsync().whenComplete((msg, e) -> { - if (e != null) { - future.completeExceptionally(e); - } - if (msg.getValue() != null - && EventType.TOPIC_POLICY.equals(msg.getValue().getEventType())) { - TopicPoliciesEvent topicPoliciesEvent = msg.getValue().getTopicPoliciesEvent(); - if (topicName.equals(TopicName.get( - topicPoliciesEvent.getDomain(), - topicPoliciesEvent.getTenant(), - topicPoliciesEvent.getNamespace(), - topicPoliciesEvent.getTopic())) - ) { - fetchTopicPoliciesAsyncAndCloseReader(reader, topicName, - topicPoliciesEvent.getPolicies(), future); - } else { - fetchTopicPoliciesAsyncAndCloseReader(reader, topicName, policies, future); - } - } else { - future.complete(null); - } - }); - } else { - future.complete(policies); - reader.closeAsync().whenComplete((v, e) -> { - if (e != null) { - log.error("[{}] Close reader error.", topicName, e); - } - }); - } - }); + @VisibleForTesting + NamespaceEventsSystemTopicFactory getNamespaceEventsSystemTopicFactory() { + try { + return namespaceEventsSystemTopicFactoryLazyInitializer.get(); + } catch (Exception e) { + log.error("Create namespace event system topic factory error.", e); + throw new RuntimeException(e); + } } public static String getEventKey(PulsarEvent event) { @@ -564,23 +630,18 @@ long getPoliciesCacheSize() { return policiesCache.size(); } - @VisibleForTesting - long getReaderCacheCount() { - return readerCaches.size(); - } - @VisibleForTesting boolean checkReaderIsCached(NamespaceName namespaceName) { return readerCaches.get(namespaceName) != null; } @VisibleForTesting - public Boolean getPoliciesCacheInit(NamespaceName namespaceName) { + public CompletableFuture getPoliciesCacheInit(NamespaceName namespaceName) { return policyCacheInitMap.get(namespaceName); } @Override - public void registerListener(TopicName topicName, TopicPolicyListener listener) { + public boolean registerListener(TopicName topicName, TopicPolicyListener listener) { listeners.compute(topicName, (k, topicListeners) -> { if (topicListeners == null) { topicListeners = new CopyOnWriteArrayList<>(); @@ -588,10 +649,11 @@ public void registerListener(TopicName topicName, TopicPolicyListener listener) { + public void unregisterListener(TopicName topicName, TopicPolicyListener listener) { listeners.compute(topicName, (k, topicListeners) -> { if (topicListeners != null){ topicListeners.remove(listener); @@ -609,9 +671,50 @@ protected Map getPoliciesCache() { } @VisibleForTesting - protected Map>> getListeners() { + protected Map> getListeners() { return listeners; } + @VisibleForTesting + protected AsyncLoadingCache> getWriterCaches() { + return writerCaches; + } + private static final Logger log = LoggerFactory.getLogger(SystemTopicBasedTopicPoliciesService.class); + + @Override + public void close() throws Exception { + if (closed.compareAndSet(false, true)) { + writerCaches.synchronous().invalidateAll(); + readerCaches.values().forEach(future -> { + try { + final var reader = future.getNow(null); + if (reader != null) { + reader.close(); + log.info("Closed the reader for topic policies"); + } else { + // Avoid blocking the thread that the reader is created + future.thenAccept(SystemTopicClient.Reader::closeAsync).whenComplete((__, e) -> { + if (e == null) { + log.info("Closed the reader for topic policies"); + } else { + log.error("Failed to close the reader for topic policies", e); + } + }); + } + } catch (Throwable ignored) { + } + }); + readerCaches.clear(); + } + } + + private static boolean isSelf(TopicName topicName) { + final var localName = topicName.getLocalName(); + if (!topicName.isPartitioned()) { + return localName.equals(SystemTopicNames.NAMESPACE_EVENTS_LOCAL_NAME); + } + final var index = localName.lastIndexOf(TopicName.PARTITIONED_TOPIC_SUFFIX); + return localName.substring(0, index).equals(SystemTopicNames.NAMESPACE_EVENTS_LOCAL_NAME); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java index 332d754cf97d2..ba6cbee355775 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java @@ -22,12 +22,16 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.systopic.SystemTopicClientBase; -import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.broker.transaction.buffer.impl.TableView; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.events.EventType; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; @@ -42,6 +46,8 @@ public class SystemTopicTxnBufferSnapshotService { protected final EventType systemTopicType; private final ConcurrentHashMap> refCountedWriterMap; + @Getter + private final TableView tableView; // The class ReferenceCountedWriter will maintain the reference count, // when the reference count decrement to 0, it will be removed from writerFutureMap, the writer will be closed. @@ -95,13 +101,16 @@ public synchronized void release() { } - public SystemTopicTxnBufferSnapshotService(PulsarClient client, EventType systemTopicType, - Class schemaType) { + public SystemTopicTxnBufferSnapshotService(PulsarService pulsar, EventType systemTopicType, + Class schemaType) throws PulsarServerException { + final var client = (PulsarClientImpl) pulsar.getClient(); this.namespaceEventsSystemTopicFactory = new NamespaceEventsSystemTopicFactory(client); this.systemTopicType = systemTopicType; this.schemaType = schemaType; this.clients = new ConcurrentHashMap<>(); this.refCountedWriterMap = new ConcurrentHashMap<>(); + this.tableView = new TableView<>(this::createReader, + client.getConfiguration().getOperationTimeoutMs(), pulsar.getExecutor()); } public CompletableFuture> createReader(TopicName topicName) { @@ -142,8 +151,26 @@ private SystemTopicClient getTransactionBufferSystemTopicClient(NamespaceName public void close() throws Exception { for (Map.Entry> entry : clients.entrySet()) { - entry.getValue().close(); + try { + entry.getValue().close(); + } catch (Exception e) { + log.error("Failed to close system topic client for namespace {}", entry.getKey(), e); + } + } + clients.clear(); + for (Map.Entry> entry : refCountedWriterMap.entrySet()) { + CompletableFuture> future = entry.getValue().getFuture(); + if (!future.isCompletedExceptionally()) { + future.thenAccept(writer -> { + try { + writer.close(); + } catch (Exception e) { + log.error("Failed to close writer for namespace {}", entry.getKey(), e); + } + }); + } } + refCountedWriterMap.clear(); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java index 7657d77e1299f..ec7889af6bbbe 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java @@ -44,7 +44,6 @@ import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.protocol.schema.SchemaVersion; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.utils.StatsOutputStream; @@ -68,7 +67,7 @@ default void setOriginalSequenceId(long originalSequenceId) { /** * Return the producer name for the original producer. - * + *

* For messages published locally, this will return the same local producer name, though in case of replicated * messages, the original producer name will differ */ @@ -136,7 +135,7 @@ default void setEntryTimestamp(long entryTimestamp) { /** * Tries to add a producer to the topic. Several validations will be performed. * - * @param producer + * @param producer Producer to add * @param producerQueuedFuture * a future that will be triggered if the producer is being queued up prior of getting established * @return the "topic epoch" if there is one or empty @@ -146,12 +145,11 @@ default void setEntryTimestamp(long entryTimestamp) { void removeProducer(Producer producer); /** - * Wait TransactionBuffer Recovers completely. - * Take snapshot after TB Recovers completely. - * @param isTxnEnabled - * @return a future which has completely if isTxn = false. Or a future return by takeSnapshot. + * Wait TransactionBuffer recovers completely. + * + * @return a future that will be completed after the transaction buffer recover completely. */ - CompletableFuture checkIfTransactionBufferRecoverCompletely(boolean isTxnEnabled); + CompletableFuture checkIfTransactionBufferRecoverCompletely(); /** * record add-latency. @@ -184,7 +182,7 @@ CompletableFuture createSubscription(String subscriptionName, Init CompletableFuture unsubscribe(String subName); - ConcurrentOpenHashMap getSubscriptions(); + Map getSubscriptions(); CompletableFuture delete(); @@ -196,6 +194,9 @@ CompletableFuture createSubscription(String subscriptionName, Init CompletableFuture close(boolean closeWithoutWaitingClientDisconnect); + CompletableFuture close( + boolean disconnectClients, boolean closeWithoutWaitingClientDisconnect); + void checkGC(); CompletableFuture checkClusterMigration(); @@ -210,36 +211,28 @@ CompletableFuture createSubscription(String subscriptionName, Init void checkCursorsToCacheEntries(); + /** + * Indicate if the current topic enabled server side deduplication. + * This is a dynamic configuration, user may update it by namespace/topic policies. + * + * @return whether enabled server side deduplication + */ + default boolean isDeduplicationEnabled() { + return false; + } + void checkDeduplicationSnapshot(); void checkMessageExpiry(); void checkMessageDeduplicationInfo(); - void checkTopicPublishThrottlingRate(); + void incrementPublishCount(Producer producer, int numOfMessages, long msgSizeInBytes); - void incrementPublishCount(int numOfMessages, long msgSizeInBytes); - - void resetTopicPublishCountAndEnableReadIfRequired(); - - void resetBrokerPublishCountAndEnableReadIfRequired(boolean doneReset); - - boolean isPublishRateExceeded(); - - boolean isTopicPublishRateExceeded(int msgSize, int numMessages); - - boolean isResourceGroupRateLimitingEnabled(); - - boolean isResourceGroupPublishRateExceeded(int msgSize, int numMessages); - - boolean isBrokerPublishRateExceeded(); + boolean shouldProducerMigrate(); boolean isReplicationBacklogExist(); - void disableCnxAutoRead(); - - void enableCnxAutoRead(); - CompletableFuture onPoliciesUpdate(Policies data); CompletableFuture checkBacklogQuotaExceeded(String producerName, BacklogQuotaType backlogQuotaType); @@ -258,27 +251,45 @@ CompletableFuture createSubscription(String subscriptionName, Init BacklogQuota getBacklogQuota(BacklogQuotaType backlogQuotaType); + /** + * Uses the best-effort (not necessarily up-to-date) information available to return the age. + * @return The oldest unacknowledged message age in seconds, or -1 if not available + */ + long getBestEffortOldestUnacknowledgedMessageAgeSeconds(); + + void updateRates(NamespaceStats nsStats, NamespaceBundleStats currentBundleStats, StatsOutputStream topicStatsStream, ClusterReplicationMetrics clusterReplicationMetrics, String namespaceName, boolean hydratePublishers); Subscription getSubscription(String subscription); - ConcurrentOpenHashMap getReplicators(); + Map getReplicators(); - ConcurrentOpenHashMap getShadowReplicators(); + Map getShadowReplicators(); TopicStatsImpl getStats(boolean getPreciseBacklog, boolean subscriptionBacklogSize, boolean getEarliestTimeInBacklog); + TopicStatsImpl getStats(GetStatsOptions getStatsOptions); + CompletableFuture asyncGetStats(boolean getPreciseBacklog, boolean subscriptionBacklogSize, boolean getEarliestTimeInBacklog); + CompletableFuture asyncGetStats(GetStatsOptions getStatsOptions); + CompletableFuture getInternalStats(boolean includeLedgerMetadata); Position getLastPosition(); + /** + * Get the last message position that can be dispatch. + */ + default CompletableFuture getLastDispatchablePosition() { + throw new UnsupportedOperationException("getLastDispatchablePosition is not supported by default"); + } + CompletableFuture getLastMessageId(); /** @@ -329,6 +340,8 @@ default boolean isSystemTopic() { boolean isPersistent(); + boolean isTransferring(); + /* ------ Transaction related ------ */ /** @@ -369,4 +382,9 @@ default boolean isSystemTopic() { */ HierarchyTopicPolicies getHierarchyTopicPolicies(); + /** + * Get OpenTelemetry attribute set. + * @return + */ + TopicAttributes getTopicAttributes(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicAttributes.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicAttributes.java new file mode 100644 index 0000000000000..60dc9ae093964 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicAttributes.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import io.opentelemetry.api.common.Attributes; +import java.util.Objects; +import lombok.Getter; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; + +@Getter +public class TopicAttributes { + + protected final Attributes commonAttributes; + + public TopicAttributes(TopicName topicName) { + Objects.requireNonNull(topicName); + var builder = Attributes.builder() + .put(OpenTelemetryAttributes.PULSAR_DOMAIN, topicName.getDomain().toString()) + .put(OpenTelemetryAttributes.PULSAR_TENANT, topicName.getTenant()) + .put(OpenTelemetryAttributes.PULSAR_NAMESPACE, topicName.getNamespace()) + .put(OpenTelemetryAttributes.PULSAR_TOPIC, topicName.getPartitionedTopicName()); + if (topicName.isPartitioned()) { + builder.put(OpenTelemetryAttributes.PULSAR_PARTITION_INDEX, topicName.getPartitionIndex()); + } + commonAttributes = builder.build(); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java index 7aa50057d73c9..e04d07460a2cb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java @@ -18,19 +18,20 @@ */ package org.apache.pulsar.broker.service; +import com.google.re2j.Pattern; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Semaphore; import java.util.function.BiConsumer; -import java.util.regex.Pattern; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.resources.TopicResources; import org.apache.pulsar.common.api.proto.CommandWatchTopicListClose; import org.apache.pulsar.common.api.proto.ServerError; import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.topics.TopicList; import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap; import org.apache.pulsar.metadata.api.NotificationType; @@ -42,11 +43,16 @@ public class TopicListService { public static class TopicListWatcher implements BiConsumer { + /** Topic names which are matching, the topic name contains the partition suffix. **/ private final List matchingTopics; private final TopicListService topicListService; private final long id; + /** The regexp for the topic name(not contains partition suffix). **/ private final Pattern topicsPattern; + /*** + * @param topicsPattern The regexp for the topic name(not contains partition suffix). + */ public TopicListWatcher(TopicListService topicListService, long id, Pattern topicsPattern, List topics) { this.topicListService = topicListService; @@ -59,9 +65,12 @@ public List getMatchingTopics() { return matchingTopics; } + /*** + * @param topicName topic name which contains partition suffix. + */ @Override public void accept(String topicName, NotificationType notificationType) { - if (topicsPattern.matcher(topicName).matches()) { + if (topicsPattern.matcher(TopicName.get(topicName).getPartitionedTopicName()).matches()) { List newTopics; List deletedTopics; if (notificationType == NotificationType.Deleted) { @@ -109,6 +118,9 @@ public void inactivate() { } } + /*** + * @param topicsPattern The regexp for the topic name(not contains partition suffix). + */ public void handleWatchTopicList(NamespaceName namespaceName, long watcherId, long requestId, Pattern topicsPattern, String topicsHash, Semaphore lookupSemaphore) { @@ -119,7 +131,7 @@ public void handleWatchTopicList(NamespaceName namespaceName, long watcherId, lo } else { msg += "Pattern longer than maximum: " + maxSubscriptionPatternLength; } - log.warn("[{}] {} on namespace {}", connection.getRemoteAddress(), msg, namespaceName); + log.warn("[{}] {} on namespace {}", connection.toString(), msg, namespaceName); connection.getCommandSender().sendErrorResponse(requestId, ServerError.NotAllowedError, msg); lookupSemaphore.release(); return; @@ -132,14 +144,14 @@ public void handleWatchTopicList(NamespaceName namespaceName, long watcherId, lo TopicListWatcher watcher = existingWatcherFuture.getNow(null); log.info("[{}] Watcher with the same id is already created:" + " watcherId={}, watcher={}", - connection.getRemoteAddress(), watcherId, watcher); + connection.toString(), watcherId, watcher); watcherFuture = existingWatcherFuture; } else { // There was an early request to create a watcher with the same watcherId. This can happen when // client timeout is lower the broker timeouts. We need to wait until the previous watcher // creation request either completes or fails. log.warn("[{}] Watcher with id is already present on the connection," - + " consumerId={}", connection.getRemoteAddress(), watcherId); + + " consumerId={}", connection.toString(), watcherId); ServerError error; if (!existingWatcherFuture.isDone()) { error = ServerError.ServiceNotReady; @@ -167,14 +179,14 @@ public void handleWatchTopicList(NamespaceName namespaceName, long watcherId, lo if (log.isDebugEnabled()) { log.debug( "[{}] Received WatchTopicList for namespace [//{}] by {}", - connection.getRemoteAddress(), namespaceName, requestId); + connection.toString(), namespaceName, requestId); } connection.getCommandSender().sendWatchTopicListSuccess(requestId, watcherId, hash, topicList); lookupSemaphore.release(); }) .exceptionally(ex -> { log.warn("[{}] Error WatchTopicList for namespace [//{}] by {}", - connection.getRemoteAddress(), namespaceName, requestId); + connection.toString(), namespaceName, requestId); connection.getCommandSender().sendErrorResponse(requestId, BrokerServiceException.getClientErrorCode( new BrokerServiceException.ServerMetadataException(ex)), ex.getMessage()); @@ -184,7 +196,9 @@ public void handleWatchTopicList(NamespaceName namespaceName, long watcherId, lo }); } - + /*** + * @param topicsPattern The regexp for the topic name(not contains partition suffix). + */ public void initializeTopicsListWatcher(CompletableFuture watcherFuture, NamespaceName namespace, long watcherId, Pattern topicsPattern) { namespaceService.getListOfPersistentTopics(namespace). @@ -199,7 +213,7 @@ public void initializeTopicsListWatcher(CompletableFuture watc } else { if (!watcherFuture.complete(watcher)) { log.warn("[{}] Watcher future was already completed. Deregistering watcherId={}.", - connection.getRemoteAddress(), watcherId); + connection.toString(), watcherId); topicResources.deregisterPersistentTopicListener(watcher); } } @@ -218,7 +232,7 @@ public void deleteTopicListWatcher(Long watcherId) { CompletableFuture watcherFuture = watchers.get(watcherId); if (watcherFuture == null) { log.info("[{}] TopicListWatcher was not registered on the connection: {}", - watcherId, connection.getRemoteAddress()); + watcherId, connection.toString()); return; } @@ -228,14 +242,14 @@ public void deleteTopicListWatcher(Long watcherId) { // watcher future as failed and we can tell the client the close operation was successful. When the actual // create operation will complete, the new watcher will be discarded. log.info("[{}] Closed watcher before its creation was completed. watcherId={}", - connection.getRemoteAddress(), watcherId); + connection.toString(), watcherId); watchers.remove(watcherId); return; } if (watcherFuture.isCompletedExceptionally()) { log.info("[{}] Closed watcher that already failed to be created. watcherId={}", - connection.getRemoteAddress(), watcherId); + connection.toString(), watcherId); watchers.remove(watcherId); return; } @@ -243,9 +257,13 @@ public void deleteTopicListWatcher(Long watcherId) { // Proceed with normal watcher close topicResources.deregisterPersistentTopicListener(watcherFuture.getNow(null)); watchers.remove(watcherId); - log.info("[{}] Closed watcher, watcherId={}", connection.getRemoteAddress(), watcherId); + log.info("[{}] Closed watcher, watcherId={}", connection.toString(), watcherId); } + /** + * @param deletedTopics topic names deleted(contains the partition suffix). + * @param newTopics topics names added(contains the partition suffix). + */ public void sendTopicListUpdate(long watcherId, String topicsHash, List deletedTopics, List newTopics) { connection.getCommandSender().sendWatchTopicListUpdate(watcherId, newTopics, deletedTopics, topicsHash); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java index c4bcc0c39353c..9b5d9a28ac216 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java @@ -20,14 +20,9 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import org.apache.pulsar.broker.service.BrokerServiceException.TopicPoliciesCacheNotInitException; -import org.apache.pulsar.client.impl.Backoff; -import org.apache.pulsar.client.impl.BackoffBuilder; -import org.apache.pulsar.client.util.RetryUtil; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.common.classification.InterfaceAudience; import org.apache.pulsar.common.classification.InterfaceStability; -import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.util.FutureUtil; @@ -35,21 +30,21 @@ /** * Topic policies service. */ -@InterfaceStability.Evolving -public interface TopicPoliciesService { +@InterfaceStability.Stable +@InterfaceAudience.LimitedPrivate +public interface TopicPoliciesService extends AutoCloseable { TopicPoliciesService DISABLED = new TopicPoliciesServiceDisabled(); - long DEFAULT_GET_TOPIC_POLICY_TIMEOUT = 30_000; /** - * Delete policies for a topic async. + * Delete policies for a topic asynchronously. * * @param topicName topic name */ CompletableFuture deleteTopicPoliciesAsync(TopicName topicName); /** - * Update policies for a topic async. + * Update policies for a topic asynchronously. * * @param topicName topic name * @param policies policies for the topic name @@ -57,93 +52,56 @@ public interface TopicPoliciesService { CompletableFuture updateTopicPoliciesAsync(TopicName topicName, TopicPolicies policies); /** - * Get policies for a topic async. - * @param topicName topic name - * @return future of the topic policies - */ - TopicPolicies getTopicPolicies(TopicName topicName) throws TopicPoliciesCacheNotInitException; - - /** - * Get policies from current cache. - * @param topicName topic name - * @return the topic policies + * It controls the behavior of {@link TopicPoliciesService#getTopicPoliciesAsync}. */ - TopicPolicies getTopicPoliciesIfExists(TopicName topicName); + enum GetType { + DEFAULT, // try getting the local topic policies, if not present, then get the global policies + GLOBAL_ONLY, // only get the global policies + LOCAL_ONLY, // only get the local policies + } /** - * Get global policies for a topic async. - * @param topicName topic name - * @return future of the topic policies + * Retrieve the topic policies. */ - TopicPolicies getTopicPolicies(TopicName topicName, boolean isGlobal) throws TopicPoliciesCacheNotInitException; + CompletableFuture> getTopicPoliciesAsync(TopicName topicName, GetType type); /** - * When getting TopicPolicies, if the initialization has not been completed, - * we will go back off and try again until time out. - * @param topicName topic name - * @param backoff back off policy - * @param isGlobal is global policies - * @return CompletableFuture<Optional<TopicPolicies>> + * Start the topic policy service. */ - default CompletableFuture> getTopicPoliciesAsyncWithRetry(TopicName topicName, - final Backoff backoff, ScheduledExecutorService scheduledExecutorService, boolean isGlobal) { - CompletableFuture> response = new CompletableFuture<>(); - Backoff usedBackoff = backoff == null ? new BackoffBuilder() - .setInitialTime(500, TimeUnit.MILLISECONDS) - .setMandatoryStop(DEFAULT_GET_TOPIC_POLICY_TIMEOUT, TimeUnit.MILLISECONDS) - .setMax(DEFAULT_GET_TOPIC_POLICY_TIMEOUT, TimeUnit.MILLISECONDS) - .create() : backoff; - try { - RetryUtil.retryAsynchronously(() -> { - CompletableFuture> future = new CompletableFuture<>(); - try { - future.complete(Optional.ofNullable(getTopicPolicies(topicName, isGlobal))); - } catch (BrokerServiceException.TopicPoliciesCacheNotInitException exception) { - future.completeExceptionally(exception); - } - return future; - }, usedBackoff, scheduledExecutorService, response); - } catch (Exception e) { - response.completeExceptionally(e); - } - return response; + default void start(PulsarService pulsar) { } /** - * Get policies for a topic without cache async. - * @param topicName topic name - * @return future of the topic policies + * Close the resources if necessary. */ - CompletableFuture getTopicPoliciesBypassCacheAsync(TopicName topicName); + default void close() throws Exception { + } /** - * Add owned namespace bundle async. + * Registers a listener for topic policies updates. * - * @param namespaceBundle namespace bundle - */ - CompletableFuture addOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle); - - /** - * Remove owned namespace bundle async. + *

+ * The listener will receive the latest topic policies when they are updated. If the policies are removed, the + * listener will receive a null value. Note that not every update is guaranteed to trigger the listener. For + * instance, if the policies change from A -> B -> null -> C in quick succession, only the final state (C) is + * guaranteed to be received by the listener. + * In summary, the listener is guaranteed to receive only the latest value. + *

* - * @param namespaceBundle namespace bundle + * @return true if the listener is registered successfully */ - CompletableFuture removeOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle); + boolean registerListener(TopicName topicName, TopicPolicyListener listener); /** - * Start the topic policy service. + * Unregister the topic policies listener. */ - void start(); - - void registerListener(TopicName topicName, TopicPolicyListener listener); - - void unregisterListener(TopicName topicName, TopicPolicyListener listener); + void unregisterListener(TopicName topicName, TopicPolicyListener listener); class TopicPoliciesServiceDisabled implements TopicPoliciesService { @Override public CompletableFuture deleteTopicPoliciesAsync(TopicName topicName) { - return FutureUtil.failedFuture(new UnsupportedOperationException("Topic policies service is disabled.")); + return CompletableFuture.completedFuture(null); } @Override @@ -152,50 +110,17 @@ public CompletableFuture updateTopicPoliciesAsync(TopicName topicName, Top } @Override - public TopicPolicies getTopicPolicies(TopicName topicName) throws TopicPoliciesCacheNotInitException { - return null; - } - - @Override - public TopicPolicies getTopicPolicies(TopicName topicName, boolean isGlobal) - throws TopicPoliciesCacheNotInitException { - return null; + public CompletableFuture> getTopicPoliciesAsync(TopicName topicName, GetType type) { + return CompletableFuture.completedFuture(Optional.empty()); } @Override - public TopicPolicies getTopicPoliciesIfExists(TopicName topicName) { - return null; - } - - @Override - public CompletableFuture getTopicPoliciesBypassCacheAsync(TopicName topicName) { - return CompletableFuture.completedFuture(null); - } - - @Override - public CompletableFuture addOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { - //No-op - return CompletableFuture.completedFuture(null); - } - - @Override - public CompletableFuture removeOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { - //No-op - return CompletableFuture.completedFuture(null); - } - - @Override - public void start() { - //No-op - } - - @Override - public void registerListener(TopicName topicName, TopicPolicyListener listener) { - //No-op + public boolean registerListener(TopicName topicName, TopicPolicyListener listener) { + return false; } @Override - public void unregisterListener(TopicName topicName, TopicPolicyListener listener) { + public void unregisterListener(TopicName topicName, TopicPolicyListener listener) { //No-op } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPolicyListener.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPolicyListener.java index 7f7fd154ab035..a597e2ef9aedf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPolicyListener.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPolicyListener.java @@ -18,6 +18,13 @@ */ package org.apache.pulsar.broker.service; -public interface TopicPolicyListener { - void onUpdate(T data); +import org.apache.pulsar.common.classification.InterfaceAudience; +import org.apache.pulsar.common.classification.InterfaceStability; +import org.apache.pulsar.common.policies.data.TopicPolicies; + +@InterfaceStability.Stable +@InterfaceAudience.LimitedPrivate +public interface TopicPolicyListener { + + void onUpdate(TopicPolicies data); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransactionBufferSnapshotServiceFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransactionBufferSnapshotServiceFactory.java index 4b8548fae47c7..d54f65572f594 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransactionBufferSnapshotServiceFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransactionBufferSnapshotServiceFactory.java @@ -18,12 +18,15 @@ */ package org.apache.pulsar.broker.service; +import lombok.Getter; +import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.transaction.buffer.metadata.TransactionBufferSnapshot; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexes; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotSegment; -import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.common.events.EventType; +@Getter public class TransactionBufferSnapshotServiceFactory { private SystemTopicTxnBufferSnapshotService txnBufferSnapshotService; @@ -33,29 +36,16 @@ public class TransactionBufferSnapshotServiceFactory { private SystemTopicTxnBufferSnapshotService txnBufferSnapshotIndexService; - public TransactionBufferSnapshotServiceFactory(PulsarClient pulsarClient) { - this.txnBufferSnapshotSegmentService = new SystemTopicTxnBufferSnapshotService<>(pulsarClient, + public TransactionBufferSnapshotServiceFactory(PulsarService pulsar) throws PulsarServerException { + this.txnBufferSnapshotSegmentService = new SystemTopicTxnBufferSnapshotService<>(pulsar, EventType.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS, TransactionBufferSnapshotSegment.class); - this.txnBufferSnapshotIndexService = new SystemTopicTxnBufferSnapshotService<>(pulsarClient, + this.txnBufferSnapshotIndexService = new SystemTopicTxnBufferSnapshotService<>(pulsar, EventType.TRANSACTION_BUFFER_SNAPSHOT_INDEXES, TransactionBufferSnapshotIndexes.class); - this.txnBufferSnapshotService = new SystemTopicTxnBufferSnapshotService<>(pulsarClient, + this.txnBufferSnapshotService = new SystemTopicTxnBufferSnapshotService<>(pulsar, EventType.TRANSACTION_BUFFER_SNAPSHOT, TransactionBufferSnapshot.class); } - public SystemTopicTxnBufferSnapshotService getTxnBufferSnapshotIndexService() { - return this.txnBufferSnapshotIndexService; - } - - public SystemTopicTxnBufferSnapshotService - getTxnBufferSnapshotSegmentService() { - return this.txnBufferSnapshotSegmentService; - } - - public SystemTopicTxnBufferSnapshotService getTxnBufferSnapshotService() { - return this.txnBufferSnapshotService; - } - public void close() throws Exception { if (this.txnBufferSnapshotIndexService != null) { this.txnBufferSnapshotIndexService.close(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java index d267160652ae4..eb2b318b7ead1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TransportCnx.java @@ -21,8 +21,10 @@ import io.netty.handler.codec.haproxy.HAProxyMessage; import io.netty.util.concurrent.Promise; import java.net.SocketAddress; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; public interface TransportCnx { @@ -31,6 +33,8 @@ public interface TransportCnx { SocketAddress clientAddress(); + String clientSourceAddressAndPort(); + BrokerService getBrokerService(); PulsarCommandSender getCommandSender(); @@ -55,20 +59,13 @@ public interface TransportCnx { void removedProducer(Producer producer); void closeProducer(Producer producer); - - void cancelPublishRateLimiting(); - - void cancelPublishBufferLimiting(); - - void disableCnxAutoRead(); - - void enableCnxAutoRead(); + void closeProducer(Producer producer, Optional assignedBrokerLookupData); void execute(Runnable runnable); void removedConsumer(Consumer consumer); - void closeConsumer(Consumer consumer); + void closeConsumer(Consumer consumer, Optional assignedBrokerLookupData); boolean isPreciseDispatcherFlowControl(); @@ -85,7 +82,25 @@ public interface TransportCnx { * by actively sending a Ping message to the client. * * @return a completable future where the result is true if the connection is alive, false otherwise. The result - * is null if the connection liveness check is disabled. + * is empty if the connection liveness check is disabled. + */ + CompletableFuture> checkConnectionLiveness(); + + /** + * Increments the counter that controls the throttling of the connection by pausing reads. + * The connection will be throttled while the counter is greater than 0. + *

+ * The caller is responsible for decrementing the counter by calling {@link #decrementThrottleCount()} when the + * connection should no longer be throttled. + */ + void incrementThrottleCount(); + + /** + * Decrements the counter that controls the throttling of the connection by pausing reads. + * The connection will be throttled while the counter is greater than 0. + *

+ * This method should be called when the connection should no longer be throttled. However, the caller should have + * previously called {@link #incrementThrottleCount()}. */ - CompletableFuture checkConnectionLiveness(); + void decrementThrottleCount(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcher.java index 0a8f254f12189..af14fad0ee24a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcher.java @@ -20,7 +20,7 @@ import java.util.List; import org.apache.bookkeeper.mledger.Entry; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.broker.service.Dispatcher; import org.apache.pulsar.common.stats.Rate; @@ -40,7 +40,7 @@ default void redeliverUnacknowledgedMessages(Consumer consumer, long consumerEpo } @Override - default void redeliverUnacknowledgedMessages(Consumer consumer, List positions) { + default void redeliverUnacknowledgedMessages(Consumer consumer, List positions) { // No-op } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java index c106b1603f6bd..399a524a197e9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherMultipleConsumers.java @@ -19,10 +19,12 @@ package org.apache.pulsar.broker.service.nonpersistent; import java.util.List; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import org.apache.bookkeeper.mledger.Entry; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.service.AbstractDispatcherMultipleConsumers; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException; @@ -32,6 +34,7 @@ import org.apache.pulsar.broker.service.RedeliveryTrackerDisabled; import org.apache.pulsar.broker.service.SendMessageInfo; import org.apache.pulsar.broker.service.Subscription; +import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.stats.Rate; @@ -126,9 +129,12 @@ public synchronized boolean canUnsubscribe(Consumer consumer) { } @Override - public CompletableFuture close() { + public CompletableFuture close(boolean disconnectConsumers, + Optional assignedBrokerLookupData) { IS_CLOSED_UPDATER.set(this, TRUE); - return disconnectAllConsumers(); + getRateLimiter().ifPresent(DispatchRateLimiter::close); + return disconnectConsumers + ? disconnectAllConsumers(false, assignedBrokerLookupData) : CompletableFuture.completedFuture(null); } @Override @@ -147,12 +153,13 @@ public synchronized void consumerFlow(Consumer consumer, int additionalNumberOfM } @Override - public synchronized CompletableFuture disconnectAllConsumers(boolean isResetCursor) { + public synchronized CompletableFuture disconnectAllConsumers( + boolean isResetCursor, Optional assignedBrokerLookupData) { closeFuture = new CompletableFuture<>(); if (consumerList.isEmpty()) { closeFuture.complete(null); } else { - consumerList.forEach(Consumer::disconnect); + consumerList.forEach(consumer -> consumer.disconnect(isResetCursor, assignedBrokerLookupData)); } return closeFuture; } @@ -184,7 +191,7 @@ public RedeliveryTracker getRedeliveryTracker() { } @Override - public void sendMessages(List entries) { + public synchronized void sendMessages(List entries) { Consumer consumer = TOTAL_AVAILABLE_PERMITS_UPDATER.get(this) > 0 ? getNextConsumer() : null; if (consumer != null) { SendMessageInfo sendMessageInfo = SendMessageInfo.getThreadLocal(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherSingleActiveConsumer.java index 25e3e2894daa1..ec9b7ac40ce66 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentDispatcherSingleActiveConsumer.java @@ -53,7 +53,7 @@ public NonPersistentDispatcherSingleActiveConsumer(SubType subscriptionType, int @Override public void sendMessages(List entries) { - Consumer currentConsumer = ACTIVE_CONSUMER_UPDATER.get(this); + Consumer currentConsumer = getActiveConsumer(); if (currentConsumer != null && currentConsumer.getAvailablePermits() > 0 && currentConsumer.isWritable()) { SendMessageInfo sendMessageInfo = SendMessageInfo.getThreadLocal(); EntryBatchSizes batchSizes = EntryBatchSizes.get(entries.size()); @@ -83,7 +83,7 @@ public Rate getMessageDropRate() { @Override public boolean hasPermits() { - return ACTIVE_CONSUMER_UPDATER.get(this) != null && ACTIVE_CONSUMER_UPDATER.get(this).getAvailablePermits() > 0; + return getActiveConsumer() != null && getActiveConsumer().getAvailablePermits() > 0; } @Override @@ -101,11 +101,6 @@ protected void scheduleReadOnActiveConsumer() { // No-op } - @Override - protected void readMoreEntries(Consumer consumer) { - // No-op - } - @Override protected void cancelPendingRead() { // No-op diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java index 514db4219db98..45b4ebf6e17cc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java @@ -33,6 +33,7 @@ import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.impl.MessageImpl; +import org.apache.pulsar.client.impl.OpSendMsgStats; import org.apache.pulsar.client.impl.ProducerImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.SendCallback; @@ -50,7 +51,7 @@ public class NonPersistentReplicator extends AbstractReplicator implements Repli public NonPersistentReplicator(NonPersistentTopic topic, String localCluster, String remoteCluster, BrokerService brokerService, PulsarClientImpl replicationClient) throws PulsarServerException { - super(localCluster, topic.getName(), remoteCluster, topic.getName(), topic.getReplicatorPrefix(), brokerService, + super(localCluster, topic, remoteCluster, topic.getName(), topic.getReplicatorPrefix(), brokerService, replicationClient); producerBuilder.blockIfQueueFull(false); @@ -67,7 +68,7 @@ protected String getProducerName() { } @Override - protected void readEntries(Producer producer) { + protected void setProducerAndTriggerReadEntries(Producer producer) { this.producer = (ProducerImpl) producer; if (STATE_UPDATER.compareAndSet(this, State.Starting, State.Started)) { @@ -78,8 +79,7 @@ protected void readEntries(Producer producer) { "[{}] Replicator was stopped while creating the producer." + " Closing it. Replicator state: {}", replicatorId, STATE_UPDATER.get(this)); - STATE_UPDATER.set(this, State.Stopping); - closeProducerAsync(); + doCloseProducerAsync(producer, () -> {}); return; } } @@ -117,6 +117,8 @@ public void sendMessage(Entry entry) { } msgOut.recordEvent(headersAndPayload.readableBytes()); + stats.incrementMsgOutCounter(); + stats.incrementBytesOutCounter(headersAndPayload.readableBytes()); msg.setReplicatedFrom(localCluster); @@ -130,6 +132,7 @@ public void sendMessage(Entry entry) { replicatorId); } msgDrop.recordEvent(); + stats.incrementMsgDropCount(); entry.release(); } } @@ -144,11 +147,11 @@ public void updateRates() { } @Override - public NonPersistentReplicatorStatsImpl getStats() { - stats.connected = producer != null && producer.isConnected(); - stats.replicationDelayInSeconds = getReplicationDelayInSeconds(); - + public NonPersistentReplicatorStatsImpl computeStats() { ProducerImpl producer = this.producer; + stats.connected = isConnected(); + stats.replicationDelayInSeconds = TimeUnit.MILLISECONDS.toSeconds(getReplicationDelayMs()); + if (producer != null) { stats.outboundConnection = producer.getConnectionId(); stats.outboundConnectedSince = producer.getConnectedSince(); @@ -160,11 +163,9 @@ public NonPersistentReplicatorStatsImpl getStats() { return stats; } - private long getReplicationDelayInSeconds() { - if (producer != null) { - return TimeUnit.MILLISECONDS.toSeconds(producer.getDelayInMillis()); - } - return 0L; + @Override + public NonPersistentReplicatorStatsImpl getStats() { + return stats; } private static final class ProducerSendCallback implements SendCallback { @@ -173,7 +174,7 @@ private static final class ProducerSendCallback implements SendCallback { private MessageImpl msg; @Override - public void sendComplete(Exception exception) { + public void sendComplete(Throwable exception, OpSendMsgStats opSendMsgStats) { if (exception != null) { log.error("[{}] Error producing on remote broker", replicator.replicatorId, exception); } else { @@ -257,10 +258,4 @@ public long getNumberOfEntriesInBacklog() { protected void disableReplicatorRead() { // No-op } - - @Override - public boolean isConnected() { - ProducerImpl producer = this.producer; - return producer != null && producer.isConnected(); - } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumers.java index 2cad253f96ee2..fb7bd22de94a7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumers.java @@ -126,6 +126,14 @@ protected Map> initialValue() throws Exception { } }; + private static final FastThreadLocal>> localGroupedStickyKeyHashes = + new FastThreadLocal>>() { + @Override + protected Map> initialValue() throws Exception { + return new HashMap<>(); + } + }; + @Override public void sendMessages(List entries) { if (entries.isEmpty()) { @@ -139,28 +147,38 @@ public void sendMessages(List entries) { final Map> groupedEntries = localGroupedEntries.get(); groupedEntries.clear(); + final Map> consumerStickyKeyHashesMap = localGroupedStickyKeyHashes.get(); + consumerStickyKeyHashesMap.clear(); for (Entry entry : entries) { - Consumer consumer = selector.select(peekStickyKey(entry.getDataBuffer())); + byte[] stickyKey = peekStickyKey(entry.getDataBuffer()); + int stickyKeyHash = StickyKeyConsumerSelector.makeStickyKeyHash(stickyKey); + + Consumer consumer = selector.select(stickyKeyHash); if (consumer != null) { - groupedEntries.computeIfAbsent(consumer, k -> new ArrayList<>()).add(entry); + int startingSize = Math.max(10, entries.size() / (2 * consumerSet.size())); + groupedEntries.computeIfAbsent(consumer, k -> new ArrayList<>(startingSize)).add(entry); + consumerStickyKeyHashesMap + .computeIfAbsent(consumer, k -> new ArrayList<>(startingSize)).add(stickyKeyHash); } else { entry.release(); } } for (Map.Entry> entriesByConsumer : groupedEntries.entrySet()) { - Consumer consumer = entriesByConsumer.getKey(); - List entriesForConsumer = entriesByConsumer.getValue(); + final Consumer consumer = entriesByConsumer.getKey(); + final List entriesForConsumer = entriesByConsumer.getValue(); + final List stickyKeysForConsumer = consumerStickyKeyHashesMap.get(consumer); SendMessageInfo sendMessageInfo = SendMessageInfo.getThreadLocal(); EntryBatchSizes batchSizes = EntryBatchSizes.get(entriesForConsumer.size()); filterEntriesForConsumer(entriesForConsumer, batchSizes, sendMessageInfo, null, null, false, consumer); if (consumer.getAvailablePermits() > 0 && consumer.isWritable()) { - consumer.sendMessages(entriesForConsumer, batchSizes, null, sendMessageInfo.getTotalMessages(), + consumer.sendMessages(entriesForConsumer, stickyKeysForConsumer, batchSizes, + null, sendMessageInfo.getTotalMessages(), sendMessageInfo.getTotalBytes(), sendMessageInfo.getTotalChunkedMessages(), - getRedeliveryTracker()); + getRedeliveryTracker(), Commands.DEFAULT_CONSUMER_EPOCH); TOTAL_AVAILABLE_PERMITS_UPDATER.addAndGet(this, -sendMessageInfo.getTotalMessages()); } else { entriesForConsumer.forEach(e -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java index 1048864ad64b2..e92eef5cb7bff 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java @@ -27,9 +27,9 @@ import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.collections4.CollectionUtils; import org.apache.pulsar.broker.intercept.BrokerInterceptor; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.service.AbstractSubscription; import org.apache.pulsar.broker.service.AnalyzeBacklogResult; import org.apache.pulsar.broker.service.BrokerServiceException; @@ -38,7 +38,7 @@ import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionFencedException; import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.broker.service.Dispatcher; -import org.apache.pulsar.broker.service.Subscription; +import org.apache.pulsar.broker.service.GetStatsOptions; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.common.api.proto.CommandAck.AckType; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; @@ -51,7 +51,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class NonPersistentSubscription extends AbstractSubscription implements Subscription { +public class NonPersistentSubscription extends AbstractSubscription { private final NonPersistentTopic topic; private volatile NonPersistentDispatcher dispatcher; private final String topicName; @@ -272,39 +272,71 @@ public NonPersistentDispatcher getDispatcher() { } @Override - public CompletableFuture close() { + public boolean isSubscriptionMigrated() { + return topic.isMigrated(); + } + + /** + * Disconnect all consumers from this subscription. + * + * @return CompletableFuture indicating the completion of the operation. + */ + @Override + public synchronized CompletableFuture disconnect(Optional assignedBrokerLookupData) { + CompletableFuture closeFuture = new CompletableFuture<>(); + + (dispatcher != null + ? dispatcher.disconnectAllConsumers(false, assignedBrokerLookupData) + : CompletableFuture.completedFuture(null)) + .thenRun(() -> { + log.info("[{}][{}] Successfully disconnected subscription consumers", topicName, subName); + closeFuture.complete(null); + }).exceptionally(exception -> { + log.error("[{}][{}] Error disconnecting subscription consumers", topicName, subName, exception); + closeFuture.completeExceptionally(exception); + return null; + }); + + return closeFuture; + + } + + private CompletableFuture fence() { IS_FENCED_UPDATER.set(this, TRUE); return CompletableFuture.completedFuture(null); } + /** - * Disconnect all consumers attached to the dispatcher and close this subscription. + * Fence this subscription and optionally disconnect all consumers. * - * @return CompletableFuture indicating the completion of disconnect operation + * @return CompletableFuture indicating the completion of the operation. */ @Override - public synchronized CompletableFuture disconnect() { - CompletableFuture disconnectFuture = new CompletableFuture<>(); + public synchronized CompletableFuture close(boolean disconnectConsumers, + Optional assignedBrokerLookupData) { + CompletableFuture closeFuture = new CompletableFuture<>(); // block any further consumers on this subscription IS_FENCED_UPDATER.set(this, TRUE); - (dispatcher != null ? dispatcher.close() : CompletableFuture.completedFuture(null)).thenCompose(v -> close()) + (dispatcher != null + ? dispatcher.close(disconnectConsumers, assignedBrokerLookupData) + : CompletableFuture.completedFuture(null)) .thenRun(() -> { - log.info("[{}][{}] Successfully disconnected and closed subscription", topicName, subName); - disconnectFuture.complete(null); + log.info("[{}][{}] Successfully closed subscription", topicName, subName); + closeFuture.complete(null); }).exceptionally(exception -> { IS_FENCED_UPDATER.set(this, FALSE); if (dispatcher != null) { dispatcher.reset(); } - log.error("[{}][{}] Error disconnecting consumers from subscription", topicName, subName, - exception); - disconnectFuture.completeExceptionally(exception); + log.error("[{}][{}] Error closing subscription", topicName, subName, exception); + closeFuture.completeExceptionally(exception); return null; }); - return disconnectFuture; + return closeFuture; } /** @@ -343,7 +375,7 @@ private CompletableFuture delete(boolean closeIfConsumersConnected) { CompletableFuture closeSubscriptionFuture = new CompletableFuture<>(); if (closeIfConsumersConnected) { - this.disconnect().thenRun(() -> { + this.close(true, Optional.empty()).thenRun(() -> { closeSubscriptionFuture.complete(null); }).exceptionally(ex -> { log.error("[{}][{}] Error disconnecting and closing subscription", topicName, subName, ex); @@ -351,7 +383,7 @@ private CompletableFuture delete(boolean closeIfConsumersConnected) { return null; }); } else { - this.close().thenRun(() -> { + this.fence().thenRun(() -> { closeSubscriptionFuture.complete(null); }).exceptionally(exception -> { log.error("[{}][{}] Error closing subscription", topicName, subName, exception); @@ -395,11 +427,24 @@ private CompletableFuture delete(boolean closeIfConsumersConnected) { */ @Override public CompletableFuture doUnsubscribe(Consumer consumer) { + return doUnsubscribe(consumer, false); + } + + /** + * Handle unsubscribe command from the client API Check with the dispatcher is this consumer can proceed with + * unsubscribe. + * + * @param consumer consumer object that is initiating the unsubscribe operation + * @param force unsubscribe forcefully by disconnecting consumers and closing subscription + * @return CompletableFuture indicating the completion of ubsubscribe operation + */ + @Override + public CompletableFuture doUnsubscribe(Consumer consumer, boolean force) { CompletableFuture future = new CompletableFuture<>(); try { - if (dispatcher.canUnsubscribe(consumer)) { + if (force || dispatcher.canUnsubscribe(consumer)) { consumer.close(); - return delete(); + return delete(force); } future.completeExceptionally( new ServerMetadataException("Unconnected or shared consumer attempting to unsubscribe")); @@ -432,7 +477,7 @@ public boolean expireMessages(Position position) { + " non-persistent topic."); } - public NonPersistentSubscriptionStatsImpl getStats() { + public NonPersistentSubscriptionStatsImpl getStats(GetStatsOptions getStatsOptions) { NonPersistentSubscriptionStatsImpl subStats = new NonPersistentSubscriptionStatsImpl(); subStats.bytesOutCounter = bytesOutFromRemovedConsumers.longValue(); subStats.msgOutCounter = msgOutFromRemovedConsumer.longValue(); @@ -441,7 +486,9 @@ public NonPersistentSubscriptionStatsImpl getStats() { if (dispatcher != null) { dispatcher.getConsumers().forEach(consumer -> { ConsumerStatsImpl consumerStats = consumer.getStats(); - subStats.consumers.add(consumerStats); + if (!getStatsOptions.isExcludeConsumers()) { + subStats.consumers.add(consumerStats); + } subStats.msgRateOut += consumerStats.msgRateOut; subStats.messageAckRate += consumerStats.messageAckRate; subStats.msgThroughputOut += consumerStats.msgThroughputOut; @@ -473,7 +520,7 @@ public synchronized void redeliverUnacknowledgedMessages(Consumer consumer, long } @Override - public synchronized void redeliverUnacknowledgedMessages(Consumer consumer, List positions) { + public synchronized void redeliverUnacknowledgedMessages(Consumer consumer, List positions) { // No-op } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 9fe0a735c90d9..34c2678f847a5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.service.nonpersistent; -import static com.google.common.base.Preconditions.checkArgument; import static org.apache.bookkeeper.mledger.impl.cache.RangeEntryCacheManagerImpl.create; import static org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType; import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; @@ -34,12 +33,16 @@ import java.util.Set; import java.util.TreeMap; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLongFieldUpdater; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.Position; +import org.apache.commons.lang3.mutable.MutableInt; import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.resources.NamespaceResources; import org.apache.pulsar.broker.service.AbstractReplicator; @@ -55,14 +58,17 @@ import org.apache.pulsar.broker.service.BrokerServiceException.UnsupportedVersionException; import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.broker.service.Dispatcher; -import org.apache.pulsar.broker.service.Producer; +import org.apache.pulsar.broker.service.GetStatsOptions; import org.apache.pulsar.broker.service.Replicator; import org.apache.pulsar.broker.service.StreamingStats; import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.SubscriptionOption; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.TopicAttributes; import org.apache.pulsar.broker.service.TopicPolicyListener; import org.apache.pulsar.broker.service.TransportCnx; +import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; +import org.apache.pulsar.broker.service.schema.exceptions.NotExistSchemaException; import org.apache.pulsar.broker.stats.ClusterReplicationMetrics; import org.apache.pulsar.broker.stats.NamespaceStats; import org.apache.pulsar.client.api.MessageId; @@ -71,9 +77,10 @@ import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.api.proto.KeySharedMeta; +import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.BacklogQuota; -import org.apache.pulsar.common.policies.data.ClusterData.ClusterUrl; +import org.apache.pulsar.common.policies.data.ClusterPolicies.ClusterUrl; import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.CursorStats; import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; import org.apache.pulsar.common.policies.data.Policies; @@ -86,21 +93,21 @@ import org.apache.pulsar.common.policies.data.stats.NonPersistentTopicStatsImpl; import org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl; import org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl; +import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.utils.StatsOutputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class NonPersistentTopic extends AbstractTopic implements Topic, TopicPolicyListener { +public class NonPersistentTopic extends AbstractTopic implements Topic, TopicPolicyListener { // Subscriptions to this topic - private final ConcurrentOpenHashMap subscriptions; + private final Map subscriptions = new ConcurrentHashMap<>(); - private final ConcurrentOpenHashMap replicators; + private final Map replicators = new ConcurrentHashMap<>(); // Ever increasing counter of entries added private static final AtomicLongFieldUpdater ENTRIES_ADDED_COUNTER_UPDATER = @@ -115,6 +122,11 @@ protected TopicStats initialValue() { } }; + private volatile TopicAttributes topicAttributes = null; + private static final AtomicReferenceFieldUpdater + TOPIC_ATTRIBUTES_FIELD_UPDATER = AtomicReferenceFieldUpdater.newUpdater( + NonPersistentTopic.class, TopicAttributes.class, "topicAttributes"); + private static class TopicStats { public double averageMsgSize; public double aggMsgRateIn; @@ -140,44 +152,32 @@ public void reset() { public NonPersistentTopic(String topic, BrokerService brokerService) { super(topic, brokerService); - - this.subscriptions = - ConcurrentOpenHashMap.newBuilder() - .expectedItems(16) - .concurrencyLevel(1) - .build(); - this.replicators = - ConcurrentOpenHashMap.newBuilder() - .expectedItems(16) - .concurrencyLevel(1) - .build(); this.isFenced = false; registerTopicPolicyListener(); } private CompletableFuture updateClusterMigrated() { - return getMigratedClusterUrlAsync(brokerService.getPulsar()).thenAccept(url -> migrated = url.isPresent()); - } - - private Optional getClusterMigrationUrl() { - return getMigratedClusterUrl(brokerService.getPulsar()); + return getMigratedClusterUrlAsync(brokerService.getPulsar(), topic) + .thenAccept(url -> migrated = url.isPresent()); } public CompletableFuture initialize() { return brokerService.pulsar().getPulsarResources().getNamespaceResources() .getPoliciesAsync(TopicName.get(topic).getNamespaceObject()) .thenCompose(optPolicies -> { - if (!optPolicies.isPresent()) { + final Policies policies; + if (optPolicies.isEmpty()) { log.warn("[{}] Policies not present and isEncryptionRequired will be set to false", topic); isEncryptionRequired = false; + policies = new Policies(); } else { - Policies policies = optPolicies.get(); + policies = optPolicies.get(); updateTopicPolicyByNamespacePolicy(policies); isEncryptionRequired = policies.encryption_required; isAllowAutoUpdateSchema = policies.is_allow_auto_update_schema; } - updatePublishDispatcher(); - updateResourceGroupLimiter(optPolicies); + updatePublishRateLimiter(); + updateResourceGroupLimiter(policies); return updateClusterMigrated(); }); } @@ -195,7 +195,7 @@ public void publishMessage(ByteBuf data, PublishContext callback) { subscriptions.forEach((name, subscription) -> { ByteBuf duplicateBuffer = data.retainedDuplicate(); Entry entry = create(0L, 0L, duplicateBuffer); - // entry internally retains data so, duplicateBuffer should be release here + // entry internally retains data so, duplicateBuffer should be released here duplicateBuffer.release(); if (subscription.getDispatcher() != null) { // Dispatcher needs to call the set method to support entry filter feature. @@ -237,20 +237,17 @@ public void checkMessageDeduplicationInfo() { } @Override - public boolean isReplicationBacklogExist() { - return false; + public boolean shouldProducerMigrate() { + return true; } @Override - public void removeProducer(Producer producer) { - checkArgument(producer.getTopic() == this); - if (producers.remove(producer.getProducerName(), producer)) { - handleProducerRemoved(producer); - } + public boolean isReplicationBacklogExist() { + return false; } @Override - public CompletableFuture checkIfTransactionBufferRecoverCompletely(boolean isTxnEnabled) { + public CompletableFuture checkIfTransactionBufferRecoverCompletely() { return CompletableFuture.completedFuture(null); } @@ -330,7 +327,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St false, cnx, cnx.getAuthRole(), metadata, readCompacted, keySharedMeta, MessageId.latest, DEFAULT_CONSUMER_EPOCH, schemaType); if (isMigrated()) { - consumer.topicMigrated(getClusterMigrationUrl()); + consumer.topicMigrated(getMigratedClusterUrl()); } addConsumerToSubscription(subscription, consumer).thenRun(() -> { @@ -413,9 +410,9 @@ private CompletableFuture delete(boolean failIfHasSubscriptions, boolean c CompletableFuture closeClientFuture = new CompletableFuture<>(); if (closeIfClientsConnected) { List> futures = new ArrayList<>(); - replicators.forEach((cluster, replicator) -> futures.add(replicator.disconnect())); + replicators.forEach((cluster, replicator) -> futures.add(replicator.terminate())); producers.values().forEach(producer -> futures.add(producer.disconnect())); - subscriptions.forEach((s, sub) -> futures.add(sub.disconnect())); + subscriptions.forEach((s, sub) -> futures.add(sub.close(true, Optional.empty()))); FutureUtil.waitForAll(futures).thenRun(() -> { closeClientFuture.complete(null); }).exceptionally(ex -> { @@ -438,8 +435,8 @@ private CompletableFuture delete(boolean failIfHasSubscriptions, boolean c if (failIfHasSubscriptions) { if (!subscriptions.isEmpty()) { isFenced = false; - deleteFuture.completeExceptionally( - new TopicBusyException("Topic has subscriptions:" + subscriptions.keys())); + deleteFuture.completeExceptionally(new TopicBusyException("Topic has subscriptions:" + + subscriptions.keySet().stream().toList())); return; } } else { @@ -470,7 +467,8 @@ private CompletableFuture delete(boolean failIfHasSubscriptions, boolean c } }).exceptionally(ex -> { deleteFuture.completeExceptionally( - new TopicBusyException("Failed to close clients before deleting topic.")); + new TopicBusyException("Failed to close clients before deleting topic.", + FutureUtil.unwrapCompletionException(ex))); return null; }); } finally { @@ -480,18 +478,29 @@ private CompletableFuture delete(boolean failIfHasSubscriptions, boolean c return deleteFuture; } + + @Override + public CompletableFuture close(boolean closeWithoutWaitingClientDisconnect) { + return close(true, closeWithoutWaitingClientDisconnect); + } + /** * Close this topic - close all producers and subscriptions associated with this topic. * + * @param disconnectClients disconnect clients * @param closeWithoutWaitingClientDisconnect don't wait for client disconnect and forcefully close managed-ledger * @return Completable future indicating completion of close operation */ @Override - public CompletableFuture close(boolean closeWithoutWaitingClientDisconnect) { + public CompletableFuture close( + boolean disconnectClients, boolean closeWithoutWaitingClientDisconnect) { CompletableFuture closeFuture = new CompletableFuture<>(); lock.writeLock().lock(); try { + if (!disconnectClients) { + transferring = true; + } if (!isFenced || closeWithoutWaitingClientDisconnect) { isFenced = true; } else { @@ -505,14 +514,24 @@ public CompletableFuture close(boolean closeWithoutWaitingClientDisconnect List> futures = new ArrayList<>(); - replicators.forEach((cluster, replicator) -> futures.add(replicator.disconnect())); - producers.values().forEach(producer -> futures.add(producer.disconnect())); - if (topicPublishRateLimiter != null) { - topicPublishRateLimiter.close(); - } - subscriptions.forEach((s, sub) -> futures.add(sub.disconnect())); - if (this.resourceGroupPublishLimiter != null) { - this.resourceGroupPublishLimiter.unregisterRateLimitFunction(this.getName()); + replicators.forEach((cluster, replicator) -> futures.add(replicator.terminate())); + if (disconnectClients) { + futures.add(ExtensibleLoadManagerImpl.getAssignedBrokerLookupData( + brokerService.getPulsar(), topic).thenAccept(lookupData -> { + producers.values().forEach(producer -> futures.add(producer.disconnect(lookupData))); + // Topics unloaded due to the ExtensibleLoadManager undergo closing twice: first with + // disconnectClients = false, second with disconnectClients = true. The check below identifies the + // cases when Topic.close is called outside the scope of the ExtensibleLoadManager. In these + // situations, we must pursue the regular Subscription.close, as Topic.close is invoked just once. + if (isTransferring()) { + subscriptions.forEach((s, sub) -> futures.add(sub.disconnect(lookupData))); + } else { + subscriptions.forEach((s, sub) -> futures.add(sub.close(true, lookupData))); + } + } + )); + } else { + subscriptions.forEach((s, sub) -> futures.add(sub.close(false, Optional.empty()))); } if (entryFilters != null) { @@ -534,9 +553,13 @@ public CompletableFuture close(boolean closeWithoutWaitingClientDisconnect // unload topic iterates over topics map and removing from the map with the same thread creates deadlock. // so, execute it in different thread brokerService.executor().execute(() -> { - brokerService.removeTopicFromCache(NonPersistentTopic.this); - unregisterTopicPolicyListener(); + + if (disconnectClients) { + brokerService.removeTopicFromCache(NonPersistentTopic.this); + unregisterTopicPolicyListener(); + } closeFuture.complete(null); + }); }).exceptionally(exception -> { log.error("[{}] Error closing topic", topic, exception); @@ -550,14 +573,15 @@ public CompletableFuture close(boolean closeWithoutWaitingClientDisconnect public CompletableFuture stopReplProducers() { List> closeFutures = new ArrayList<>(); - replicators.forEach((region, replicator) -> closeFutures.add(replicator.disconnect())); + replicators.forEach((region, replicator) -> closeFutures.add(replicator.terminate())); return FutureUtil.waitForAll(closeFutures); } @Override public CompletableFuture checkReplication() { TopicName name = TopicName.get(topic); - if (!name.isGlobal() || NamespaceService.isHeartbeatNamespace(name)) { + if (!name.isGlobal() || NamespaceService.isHeartbeatNamespace(name) + || ExtensibleLoadManagerImpl.isInternalTopic(topic)) { return CompletableFuture.completedFuture(null); } @@ -630,8 +654,9 @@ CompletableFuture removeReplicator(String remoteCluster) { String name = NonPersistentReplicator.getReplicatorName(replicatorPrefix, remoteCluster); - replicators.get(remoteCluster).disconnect().thenRun(() -> { + replicators.get(remoteCluster).terminate().thenRun(() -> { log.info("[{}] Successfully removed replicator {}", name, remoteCluster); + replicators.remove(remoteCluster); }).exceptionally(e -> { log.error("[{}] Failed to close replication producer {} {}", topic, name, e.getMessage(), e); @@ -678,18 +703,18 @@ public int getNumberOfSameAddressConsumers(final String clientAddress) { } @Override - public ConcurrentOpenHashMap getSubscriptions() { + public Map getSubscriptions() { return subscriptions; } @Override - public ConcurrentOpenHashMap getReplicators() { + public Map getReplicators() { return replicators; } @Override - public ConcurrentOpenHashMap getShadowReplicators() { - return ConcurrentOpenHashMap.emptyMap(); + public Map getShadowReplicators() { + return Map.of(); } @Override @@ -711,8 +736,7 @@ public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats replicators.forEach((region, replicator) -> replicator.updateRates()); - nsStats.producerCount += producers.size(); - bundleStats.producerCount += producers.size(); + final MutableInt producerCount = new MutableInt(); topicStatsStream.startObject(topic); topicStatsStream.startList("publishers"); @@ -725,14 +749,19 @@ public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats if (producer.isRemote()) { topicStats.remotePublishersStats.put(producer.getRemoteCluster(), publisherStats); - } - - if (hydratePublishers) { - StreamingStats.writePublisherStats(topicStatsStream, publisherStats); + } else { + // Exclude producers for replication from "publishers" and "producerCount" + producerCount.increment(); + if (hydratePublishers) { + StreamingStats.writePublisherStats(topicStatsStream, publisherStats); + } } }); topicStatsStream.endList(); + nsStats.producerCount += producerCount.intValue(); + bundleStats.producerCount += producerCount.intValue(); + // Start replicator stats topicStatsStream.startObject("replication"); nsStats.replicatorCount += topicStats.remotePublishersStats.size(); @@ -821,7 +850,7 @@ public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats // Remaining dest stats. topicStats.averageMsgSize = topicStats.aggMsgRateIn == 0.0 ? 0.0 : (topicStats.aggMsgThroughputIn / topicStats.aggMsgRateIn); - topicStatsStream.writePair("producerCount", producers.size()); + topicStatsStream.writePair("producerCount", producerCount.intValue()); topicStatsStream.writePair("averageMsgSize", topicStats.averageMsgSize); topicStatsStream.writePair("msgRateIn", topicStats.aggMsgRateIn); topicStatsStream.writePair("msgRateOut", topicStats.aggMsgRateOut); @@ -861,10 +890,27 @@ public NonPersistentTopicStatsImpl getStats(boolean getPreciseBacklog, boolean s } } + @Override + public TopicStatsImpl getStats(GetStatsOptions getStatsOptions) { + try { + return asyncGetStats(getStatsOptions).get(); + } catch (InterruptedException | ExecutionException e) { + log.error("[{}] Fail to get stats", topic, e); + return null; + } + } + @Override public CompletableFuture asyncGetStats(boolean getPreciseBacklog, boolean subscriptionBacklogSize, boolean getEarliestTimeInBacklog) { + GetStatsOptions getStatsOptions = new GetStatsOptions(getPreciseBacklog, subscriptionBacklogSize, + getEarliestTimeInBacklog, false, false); + return (CompletableFuture) asyncGetStats(getStatsOptions); + } + + @Override + public CompletableFuture asyncGetStats(GetStatsOptions getStatsOptions) { CompletableFuture future = new CompletableFuture<>(); NonPersistentTopicStatsImpl stats = new NonPersistentTopicStatsImpl(); @@ -877,7 +923,8 @@ public CompletableFuture asyncGetStats(boolean getP if (producer.isRemote()) { remotePublishersStats.put(producer.getRemoteCluster(), publisherStats); - } else { + } else if (!getStatsOptions.isExcludePublishers()) { + // Exclude producers for replication from "publishers" stats.addPublisher(publisherStats); } }); @@ -885,22 +932,28 @@ public CompletableFuture asyncGetStats(boolean getP stats.averageMsgSize = stats.msgRateIn == 0.0 ? 0.0 : (stats.msgThroughputIn / stats.msgRateIn); stats.msgInCounter = getMsgInCounter(); stats.bytesInCounter = getBytesInCounter(); + stats.systemTopicBytesInCounter = getSystemTopicBytesInCounter(); stats.waitingPublishers = getWaitingProducersCount(); stats.bytesOutCounter = bytesOutFromRemovedSubscriptions.longValue(); stats.msgOutCounter = msgOutFromRemovedSubscriptions.longValue(); + stats.bytesOutInternalCounter = bytesOutFromRemovedSystemSubscriptions.longValue(); subscriptions.forEach((name, subscription) -> { - NonPersistentSubscriptionStatsImpl subStats = subscription.getStats(); + NonPersistentSubscriptionStatsImpl subStats = subscription.getStats(getStatsOptions); stats.msgRateOut += subStats.msgRateOut; stats.msgThroughputOut += subStats.msgThroughputOut; stats.bytesOutCounter += subStats.bytesOutCounter; stats.msgOutCounter += subStats.msgOutCounter; stats.getSubscriptions().put(name, subStats); + + if (isSystemCursor(name)) { + stats.bytesOutInternalCounter += subStats.bytesOutCounter; + } }); replicators.forEach((cluster, replicator) -> { - NonPersistentReplicatorStatsImpl replicatorStats = replicator.getStats(); + NonPersistentReplicatorStatsImpl replicatorStats = replicator.computeStats(); // Add incoming msg rates PublisherStatsImpl pubStats = remotePublishersStats.get(replicator.getRemoteCluster()); @@ -918,7 +971,7 @@ public CompletableFuture asyncGetStats(boolean getP }); stats.topicEpoch = topicEpoch.orElse(null); - stats.ownerBroker = brokerService.pulsar().getLookupServiceAddress(); + stats.ownerBroker = brokerService.pulsar().getBrokerId(); future.complete(stats); return future; } @@ -946,7 +999,11 @@ public boolean isActive() { @Override public CompletableFuture checkClusterMigration() { - Optional url = getClusterMigrationUrl(); + if (ExtensibleLoadManagerImpl.isInternalTopic(topic)) { + return CompletableFuture.completedFuture(null); + } + + Optional url = getMigratedClusterUrl(); if (url.isPresent()) { this.migrated = true; producers.forEach((__, producer) -> { @@ -957,10 +1014,30 @@ public CompletableFuture checkClusterMigration() { consumer.topicMigrated(url); }); }); + return disconnectReplicators().thenCompose(__ -> checkAndUnsubscribeSubscriptions()); } return CompletableFuture.completedFuture(null); } + private CompletableFuture checkAndUnsubscribeSubscriptions() { + List> futures = new ArrayList<>(); + subscriptions.forEach((s, subscription) -> { + if (subscription.getConsumers().isEmpty()) { + futures.add(subscription.delete()); + } + }); + + return FutureUtil.waitForAll(futures); + } + + private CompletableFuture disconnectReplicators() { + List> futures = new ArrayList<>(); + replicators.forEach((r, replicator) -> { + futures.add(replicator.terminate()); + }); + return FutureUtil.waitForAll(futures); + } + @Override public void checkGC() { if (!isDeleteWhileInactive()) { @@ -1121,9 +1198,15 @@ public CompletableFuture unsubscribe(String subscriptionName) { NonPersistentSubscription sub = subscriptions.remove(subscriptionName); if (sub != null) { // preserve accumulative stats form removed subscription - SubscriptionStatsImpl stats = sub.getStats(); + GetStatsOptions getStatsOptions = new GetStatsOptions(false, false, false, false, false); + SubscriptionStatsImpl stats = sub.getStats(getStatsOptions); bytesOutFromRemovedSubscriptions.add(stats.bytesOutCounter); msgOutFromRemovedSubscriptions.add(stats.msgOutCounter); + + if (isSystemCursor(subscriptionName) + || subscriptionName.startsWith(SystemTopicNames.SYSTEM_READER_PREFIX)) { + bytesOutFromRemovedSystemSubscriptions.add(stats.bytesOutCounter); + } } }, brokerService.executor()); } @@ -1152,7 +1235,16 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem || (!producers.isEmpty()) || (numActiveConsumersWithoutAutoSchema != 0) || ENTRIES_ADDED_COUNTER_UPDATER.get(this) != 0) { - return checkSchemaCompatibleForConsumer(schema); + return checkSchemaCompatibleForConsumer(schema) + .exceptionally(ex -> { + Throwable realCause = FutureUtil.unwrapCompletionException(ex); + if (realCause instanceof NotExistSchemaException) { + throw FutureUtil.wrapToCompletionException( + new IncompatibleSchemaException("Failed to add schema to an active topic" + + " with empty(BYTES) schema: new schema type " + schema.getType())); + } + throw FutureUtil.wrapToCompletionException(realCause); + }); } else { return addSchema(schema).thenCompose(schemaVersion -> CompletableFuture.completedFuture(null)); } @@ -1189,4 +1281,18 @@ protected boolean isMigrated() { public boolean isPersistent() { return false; } + + @Override + public long getBestEffortOldestUnacknowledgedMessageAgeSeconds() { + return -1; + } + + @Override + public TopicAttributes getTopicAttributes() { + if (topicAttributes != null) { + return topicAttributes; + } + return TOPIC_ATTRIBUTES_FIELD_UPDATER.updateAndGet(this, + old -> old != null ? old : new TopicAttributes(TopicName.get(topic))); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java index b1e4803548414..b29cbcd660db1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java @@ -21,19 +21,17 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.qos.AsyncTokenBucket; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.Policies; -import org.apache.pulsar.common.util.RateLimiter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class DispatchRateLimiter { - public enum Type { TOPIC, SUBSCRIPTION, @@ -47,8 +45,8 @@ public enum Type { private final Type type; private final BrokerService brokerService; - private RateLimiter dispatchRateLimiterOnMessage; - private RateLimiter dispatchRateLimiterOnByte; + private volatile AsyncTokenBucket dispatchRateLimiterOnMessage; + private volatile AsyncTokenBucket dispatchRateLimiterOnByte; public DispatchRateLimiter(PersistentTopic topic, Type type) { this(topic, null, type); @@ -78,7 +76,7 @@ public DispatchRateLimiter(BrokerService brokerService) { * @return */ public long getAvailableDispatchRateLimitOnMsg() { - return dispatchRateLimiterOnMessage == null ? -1 : dispatchRateLimiterOnMessage.getAvailablePermits(); + return dispatchRateLimiterOnMessage == null ? -1 : Math.max(dispatchRateLimiterOnMessage.getTokens(), 0); } /** @@ -87,32 +85,22 @@ public long getAvailableDispatchRateLimitOnMsg() { * @return */ public long getAvailableDispatchRateLimitOnByte() { - return dispatchRateLimiterOnByte == null ? -1 : dispatchRateLimiterOnByte.getAvailablePermits(); + return dispatchRateLimiterOnByte == null ? -1 : Math.max(dispatchRateLimiterOnByte.getTokens(), 0); } /** * It acquires msg and bytes permits from rate-limiter and returns if acquired permits succeed. * - * @param msgPermits - * @param bytePermits - * @return + * @param numberOfMessages + * @param byteSize */ - public boolean tryDispatchPermit(long msgPermits, long bytePermits) { - boolean acquiredMsgPermit = msgPermits <= 0 || dispatchRateLimiterOnMessage == null - || dispatchRateLimiterOnMessage.tryAcquire(msgPermits); - boolean acquiredBytePermit = bytePermits <= 0 || dispatchRateLimiterOnByte == null - || dispatchRateLimiterOnByte.tryAcquire(bytePermits); - return acquiredMsgPermit && acquiredBytePermit; - } - - /** - * checks if dispatch-rate limit is configured and if it's configured then check if permits are available or not. - * - * @return - */ - public boolean hasMessageDispatchPermit() { - return (dispatchRateLimiterOnMessage == null || dispatchRateLimiterOnMessage.getAvailablePermits() > 0) - && (dispatchRateLimiterOnByte == null || dispatchRateLimiterOnByte.getAvailablePermits() > 0); + public void consumeDispatchQuota(long numberOfMessages, long byteSize) { + if (numberOfMessages > 0 && dispatchRateLimiterOnMessage != null) { + dispatchRateLimiterOnMessage.consumeTokens(numberOfMessages); + } + if (byteSize > 0 && dispatchRateLimiterOnByte != null) { + dispatchRateLimiterOnByte.consumeTokens(byteSize); + } } /** @@ -192,8 +180,8 @@ public void updateDispatchRate() { if (type == Type.BROKER) { log.info("configured broker message-dispatch rate {}", dispatchRate); } else { - log.info("[{}] configured {} message-dispatch rate at broker {}", - this.topicName, type, dispatchRate); + log.info("[{}] configured {} message-dispatch rate at broker {} subscriptionName [{}]", + this.topicName, type, subscriptionName == null ? "null" : subscriptionName, dispatchRate); } updateDispatchRate(dispatchRate); } @@ -204,6 +192,12 @@ public static CompletableFuture> getPoliciesAsync(BrokerServi return brokerService.pulsar().getPulsarResources().getNamespaceResources().getPoliciesAsync(namespace); } + /** + * @deprecated Avoid using the deprecated method + * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and blocking + * call. we can use #{@link DispatchRateLimiter#getPoliciesAsync(BrokerService, String)} to instead of it. + */ + @Deprecated public static Optional getPolicies(BrokerService brokerService, String topicName) { final NamespaceName namespace = TopicName.get(topicName).getNamespaceObject(); return brokerService.pulsar().getPulsarResources().getNamespaceResources().getPoliciesIfCached(namespace); @@ -221,60 +215,40 @@ public synchronized void updateDispatchRate(DispatchRate dispatchRate) { long msgRate = dispatchRate.getDispatchThrottlingRateInMsg(); long byteRate = dispatchRate.getDispatchThrottlingRateInByte(); - long ratePeriod = dispatchRate.getRatePeriodInSecond(); + long ratePeriodNanos = TimeUnit.SECONDS.toNanos(Math.max(dispatchRate.getRatePeriodInSecond(), 1)); - Supplier permitUpdaterMsg = dispatchRate.isRelativeToPublishRate() - ? () -> getRelativeDispatchRateInMsg(dispatchRate) - : null; // update msg-rateLimiter if (msgRate > 0) { - if (this.dispatchRateLimiterOnMessage == null) { + if (dispatchRate.isRelativeToPublishRate()) { this.dispatchRateLimiterOnMessage = - RateLimiter.builder() - .scheduledExecutorService(brokerService.pulsar().getExecutor()) - .permits(msgRate) - .rateTime(ratePeriod) - .timeUnit(TimeUnit.SECONDS) - .permitUpdater(permitUpdaterMsg) - .isDispatchOrPrecisePublishRateLimiter(true) + AsyncTokenBucket.builderForDynamicRate() + .rateFunction(() -> getRelativeDispatchRateInMsg(dispatchRate)) + .ratePeriodNanosFunction(() -> ratePeriodNanos) .build(); } else { - this.dispatchRateLimiterOnMessage.setRate(msgRate, dispatchRate.getRatePeriodInSecond(), - TimeUnit.SECONDS, permitUpdaterMsg); + this.dispatchRateLimiterOnMessage = + AsyncTokenBucket.builder().rate(msgRate).ratePeriodNanos(ratePeriodNanos) + .build(); } } else { - // message-rate should be disable and close - if (this.dispatchRateLimiterOnMessage != null) { - this.dispatchRateLimiterOnMessage.close(); - this.dispatchRateLimiterOnMessage = null; - } + this.dispatchRateLimiterOnMessage = null; } - Supplier permitUpdaterByte = dispatchRate.isRelativeToPublishRate() - ? () -> getRelativeDispatchRateInByte(dispatchRate) - : null; // update byte-rateLimiter if (byteRate > 0) { - if (this.dispatchRateLimiterOnByte == null) { + if (dispatchRate.isRelativeToPublishRate()) { this.dispatchRateLimiterOnByte = - RateLimiter.builder() - .scheduledExecutorService(brokerService.pulsar().getExecutor()) - .permits(byteRate) - .rateTime(ratePeriod) - .timeUnit(TimeUnit.SECONDS) - .permitUpdater(permitUpdaterByte) - .isDispatchOrPrecisePublishRateLimiter(true) + AsyncTokenBucket.builderForDynamicRate() + .rateFunction(() -> getRelativeDispatchRateInByte(dispatchRate)) + .ratePeriodNanosFunction(() -> ratePeriodNanos) .build(); } else { - this.dispatchRateLimiterOnByte.setRate(byteRate, dispatchRate.getRatePeriodInSecond(), - TimeUnit.SECONDS, permitUpdaterByte); + this.dispatchRateLimiterOnByte = + AsyncTokenBucket.builder().rate(byteRate).ratePeriodNanos(ratePeriodNanos) + .build(); } } else { - // message-rate should be disable and close - if (this.dispatchRateLimiterOnByte != null) { - this.dispatchRateLimiterOnByte.close(); - this.dispatchRateLimiterOnByte = null; - } + this.dispatchRateLimiterOnByte = null; } } @@ -317,11 +291,9 @@ public static boolean isDispatchRateEnabled(DispatchRate dispatchRate) { public void close() { // close rate-limiter if (dispatchRateLimiterOnMessage != null) { - dispatchRateLimiterOnMessage.close(); dispatchRateLimiterOnMessage = null; } if (dispatchRateLimiterOnByte != null) { - dispatchRateLimiterOnByte.close(); dispatchRateLimiterOnByte = null; } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java index 08882982297ab..cd5b2ba721215 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java @@ -24,14 +24,15 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedCursor; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.service.BrokerService; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.protocol.Markers; import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.util.FutureUtil; @Slf4j public class GeoPersistentReplicator extends PersistentReplicator { @@ -51,6 +52,33 @@ protected String getProducerName() { return getReplicatorName(replicatorPrefix, localCluster) + REPL_PRODUCER_NAME_DELIMITER + remoteCluster; } + @Override + protected CompletableFuture prepareCreateProducer() { + if (brokerService.getPulsar().getConfig().isCreateTopicToRemoteClusterForReplication()) { + return CompletableFuture.completedFuture(null); + } else { + CompletableFuture topicCheckFuture = new CompletableFuture<>(); + replicationClient.getPartitionedTopicMetadata(localTopic.getName(), false, false) + .whenComplete((metadata, ex) -> { + if (ex == null) { + if (metadata.partitions == 0) { + topicCheckFuture.complete(null); + } else { + String errorMsg = String.format("{} Can not create the replicator due to the partitions in the" + + " remote cluster is not 0, but is %s", + replicatorId, metadata.partitions); + log.error(errorMsg); + topicCheckFuture.completeExceptionally( + new PulsarClientException.NotAllowedException(errorMsg)); + } + } else { + topicCheckFuture.completeExceptionally(FutureUtil.unwrapCompletionException(ex)); + } + }); + return topicCheckFuture; + } + } + @Override protected boolean replicateEntries(List entries) { boolean atLeastOneMessageSentForReplication = false; @@ -92,7 +120,7 @@ protected boolean replicateEntries(List entries) { if (msg.getMessageBuilder().hasTxnidLeastBits() && msg.getMessageBuilder().hasTxnidMostBits()) { TxnID tx = new TxnID(msg.getMessageBuilder().getTxnidMostBits(), msg.getMessageBuilder().getTxnidLeastBits()); - if (topic.isTxnAborted(tx, (PositionImpl) entry.getPosition())) { + if (topic.isTxnAborted(tx, entry.getPosition())) { cursor.asyncDelete(entry.getPosition(), this, entry.getPosition()); entry.release(); msg.recycle(); @@ -123,18 +151,6 @@ protected boolean replicateEntries(List entries) { continue; } - if (msg.isExpired(messageTTLInSeconds)) { - msgExpired.recordEvent(0 /* no value stat */); - if (log.isDebugEnabled()) { - log.debug("[{}] Discarding expired message at position {}, replicateTo {}", - replicatorId, entry.getPosition(), msg.getReplicateTo()); - } - cursor.asyncDelete(entry.getPosition(), this, entry.getPosition()); - entry.release(); - msg.recycle(); - continue; - } - if (STATE_UPDATER.get(this) != State.Started || isLocalMessageSkippedOnce) { // The producer is not ready yet after having stopped/restarted. Drop the message because it will // recovered when the producer is ready @@ -148,10 +164,7 @@ protected boolean replicateEntries(List entries) { continue; } - dispatchRateLimiter.ifPresent(rateLimiter -> rateLimiter.tryDispatchPermit(1, entry.getLength())); - - msgOut.recordEvent(headersAndPayload.readableBytes()); - + dispatchRateLimiter.ifPresent(rateLimiter -> rateLimiter.consumeDispatchQuota(1, entry.getLength())); msg.setReplicatedFrom(localCluster); headersAndPayload.retain(); @@ -181,6 +194,9 @@ protected boolean replicateEntries(List entries) { msg.setSchemaInfoForReplicator(schemaFuture.get()); msg.getMessageBuilder().clearTxnidMostBits(); msg.getMessageBuilder().clearTxnidLeastBits(); + msgOut.recordEvent(headersAndPayload.readableBytes()); + stats.incrementMsgOutCounter(); + stats.incrementBytesOutCounter(headersAndPayload.readableBytes()); // Increment pending messages for messages produced locally PENDING_MESSAGES_UPDATER.incrementAndGet(this); producer.sendAsync(msg, ProducerSendCallback.create(this, entry, msg)); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java index 030d74a014f29..dfb8b9d2edb12 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java @@ -20,13 +20,14 @@ import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.ByteBuf; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback; @@ -36,12 +37,11 @@ import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.Topic.PublishContext; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.protocol.Commands; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,6 +55,8 @@ public class MessageDeduplication { private final ManagedLedger managedLedger; private ManagedCursor managedCursor; + private static final String IS_LAST_CHUNK = "isLastChunk"; + enum Status { // Deduplication is initialized @@ -98,20 +100,12 @@ public MessageDupUnknownException() { // Map that contains the highest sequenceId that have been sent by each producers. The map will be updated before // the messages are persisted @VisibleForTesting - final ConcurrentOpenHashMap highestSequencedPushed = - ConcurrentOpenHashMap.newBuilder() - .expectedItems(16) - .concurrencyLevel(1) - .build(); + final Map highestSequencedPushed = new ConcurrentHashMap<>(); // Map that contains the highest sequenceId that have been persistent by each producers. The map will be updated // after the messages are persisted @VisibleForTesting - final ConcurrentOpenHashMap highestSequencedPersisted = - ConcurrentOpenHashMap.newBuilder() - .expectedItems(16) - .concurrencyLevel(1) - .build(); + final Map highestSequencedPersisted = new ConcurrentHashMap<>(); // Number of persisted entries after which to store a snapshot of the sequence ids map private final int snapshotInterval; @@ -126,10 +120,13 @@ public MessageDupUnknownException() { private final int maxNumberOfProducers; // Map used to track the inactive producer along with the timestamp of their last activity - private final Map inactiveProducers = new HashMap<>(); + private final Map inactiveProducers = new ConcurrentHashMap<>(); private final String replicatorPrefix; + + private final AtomicBoolean snapshotTaking = new AtomicBoolean(false); + public MessageDeduplication(PulsarService pulsar, PersistentTopic topic, ManagedLedger managedLedger) { this.pulsar = pulsar; this.topic = topic; @@ -151,9 +148,15 @@ private CompletableFuture recoverSequenceIdsMap() { // Replay all the entries and apply all the sequence ids updates log.info("[{}] Replaying {} entries for deduplication", topic.getName(), managedCursor.getNumberOfEntries()); - CompletableFuture future = new CompletableFuture<>(); + CompletableFuture future = new CompletableFuture<>(); replayCursor(future); - return future; + return future.thenCompose(lastPosition -> { + if (lastPosition != null && snapshotCounter >= snapshotInterval) { + snapshotCounter = 0; + return takeSnapshot(lastPosition); + } + return CompletableFuture.completedFuture(null); + }); } /** @@ -162,11 +165,11 @@ private CompletableFuture recoverSequenceIdsMap() { * * @param future future to trigger when the replay is complete */ - private void replayCursor(CompletableFuture future) { + private void replayCursor(CompletableFuture future) { managedCursor.asyncReadEntries(100, new ReadEntriesCallback() { @Override public void readEntriesComplete(List entries, Object ctx) { - + Position lastPosition = null; for (Entry entry : entries) { ByteBuf messageMetadataAndPayload = entry.getDataBuffer(); MessageMetadata md = Commands.parseMessageMetadata(messageMetadataAndPayload); @@ -176,7 +179,8 @@ public void readEntriesComplete(List entries, Object ctx) { highestSequencedPushed.put(producerName, sequenceId); highestSequencedPersisted.put(producerName, sequenceId); producerRemoved(producerName); - + snapshotCounter++; + lastPosition = entry.getPosition(); entry.release(); } @@ -185,7 +189,7 @@ public void readEntriesComplete(List entries, Object ctx) { pulsar.getExecutor().execute(() -> replayCursor(future)); } else { // Done replaying - future.complete(null); + future.complete(lastPosition); } } @@ -193,7 +197,7 @@ public void readEntriesComplete(List entries, Object ctx) { public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { future.completeExceptionally(exception); } - }, null, PositionImpl.LATEST); + }, null, PositionFactory.LATEST); } public Status getStatus() { @@ -205,7 +209,7 @@ public Status getStatus() { * returning a future to track the completion of the task */ public CompletableFuture checkStatus() { - boolean shouldBeEnabled = isDeduplicationEnabled(); + boolean shouldBeEnabled = topic.isDeduplicationEnabled(); synchronized (this) { if (status == Status.Recovering || status == Status.Removing) { // If there's already a transition happening, check later for status @@ -324,11 +328,12 @@ public MessageDupStatus isDuplicate(PublishContext publishContext, ByteBuf heade String producerName = publishContext.getProducerName(); long sequenceId = publishContext.getSequenceId(); long highestSequenceId = Math.max(publishContext.getHighestSequenceId(), sequenceId); + MessageMetadata md = null; if (producerName.startsWith(replicatorPrefix)) { // Message is coming from replication, we need to use the original producer name and sequence id // for the purpose of deduplication and not rely on the "replicator" name. int readerIndex = headersAndPayload.readerIndex(); - MessageMetadata md = Commands.parseMessageMetadata(headersAndPayload); + md = Commands.parseMessageMetadata(headersAndPayload); producerName = md.getProducerName(); sequenceId = md.getSequenceId(); highestSequenceId = Math.max(md.getHighestSequenceId(), sequenceId); @@ -337,7 +342,23 @@ public MessageDupStatus isDuplicate(PublishContext publishContext, ByteBuf heade publishContext.setOriginalHighestSequenceId(highestSequenceId); headersAndPayload.readerIndex(readerIndex); } - + long chunkID = -1; + long totalChunk = -1; + if (publishContext.isChunked()) { + if (md == null) { + int readerIndex = headersAndPayload.readerIndex(); + md = Commands.parseMessageMetadata(headersAndPayload); + headersAndPayload.readerIndex(readerIndex); + } + chunkID = md.getChunkId(); + totalChunk = md.getNumChunksFromMsg(); + } + // All chunks of a message use the same message metadata and sequence ID, + // so we only need to check the sequence ID for the last chunk in a chunk message. + if (chunkID != -1 && chunkID != totalChunk - 1) { + publishContext.setProperty(IS_LAST_CHUNK, Boolean.FALSE); + return MessageDupStatus.NotDup; + } // Synchronize the get() and subsequent put() on the map. This would only be relevant if the producer // disconnects and re-connects very quickly. At that point the call can be coming from a different thread synchronized (highestSequencedPushed) { @@ -363,13 +384,18 @@ public MessageDupStatus isDuplicate(PublishContext publishContext, ByteBuf heade } highestSequencedPushed.put(producerName, highestSequenceId); } + // Only put sequence ID into highestSequencedPushed and + // highestSequencedPersisted until receive and persistent the last chunk. + if (chunkID != -1 && chunkID == totalChunk - 1) { + publishContext.setProperty(IS_LAST_CHUNK, Boolean.TRUE); + } return MessageDupStatus.NotDup; } /** * Call this method whenever a message is persisted to get the chance to trigger a snapshot. */ - public void recordMessagePersisted(PublishContext publishContext, PositionImpl position) { + public void recordMessagePersisted(PublishContext publishContext, Position position) { if (!isEnabled() || publishContext.isMarkerMessage()) { return; } @@ -383,8 +409,10 @@ public void recordMessagePersisted(PublishContext publishContext, PositionImpl p sequenceId = publishContext.getOriginalSequenceId(); highestSequenceId = publishContext.getOriginalHighestSequenceId(); } - - highestSequencedPersisted.put(producerName, Math.max(highestSequenceId, sequenceId)); + Boolean isLastChunk = (Boolean) publishContext.getProperty(IS_LAST_CHUNK); + if (isLastChunk == null || isLastChunk) { + highestSequencedPersisted.put(producerName, Math.max(highestSequenceId, sequenceId)); + } if (++snapshotCounter >= snapshotInterval) { snapshotCounter = 0; takeSnapshot(position); @@ -397,15 +425,22 @@ public void resetHighestSequenceIdPushed() { } highestSequencedPushed.clear(); - for (String producer : highestSequencedPersisted.keys()) { + for (String producer : highestSequencedPersisted.keySet()) { highestSequencedPushed.put(producer, highestSequencedPersisted.get(producer)); } } - private void takeSnapshot(Position position) { + private CompletableFuture takeSnapshot(Position position) { + CompletableFuture future = new CompletableFuture<>(); if (log.isDebugEnabled()) { log.debug("[{}] Taking snapshot of sequence ids map", topic.getName()); } + + if (!snapshotTaking.compareAndSet(false, true)) { + future.complete(null); + return future; + } + Map snapshot = new TreeMap<>(); highestSequencedPersisted.forEach((producerName, sequenceId) -> { if (snapshot.size() < maxNumberOfProducers) { @@ -420,23 +455,29 @@ public void markDeleteComplete(Object ctx) { log.debug("[{}] Stored new deduplication snapshot at {}", topic.getName(), position); } lastSnapshotTimestamp = System.currentTimeMillis(); + snapshotTaking.set(false); + future.complete(null); } @Override public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { - log.warn("[{}] Failed to store new deduplication snapshot at {}", topic.getName(), position); + log.warn("[{}] Failed to store new deduplication snapshot at {}", + topic.getName(), position, exception); + snapshotTaking.set(false); + future.completeExceptionally(exception); } }, null); - } - - private boolean isDeduplicationEnabled() { - return topic.getHierarchyTopicPolicies().getDeduplicationEnabled().get(); + return future; } /** * Topic will call this method whenever a producer connects. */ - public synchronized void producerAdded(String producerName) { + public void producerAdded(String producerName) { + if (!isEnabled()) { + return; + } + // Producer is no-longer inactive inactiveProducers.remove(producerName); } @@ -444,7 +485,11 @@ public synchronized void producerAdded(String producerName) { /** * Topic will call this method whenever a producer disconnects. */ - public synchronized void producerRemoved(String producerName) { + public void producerRemoved(String producerName) { + if (!isEnabled()) { + return; + } + // Producer is no-longer active inactiveProducers.put(producerName, System.currentTimeMillis()); } @@ -456,7 +501,15 @@ public synchronized void purgeInactiveProducers() { long minimumActiveTimestamp = System.currentTimeMillis() - TimeUnit.MINUTES .toMillis(pulsar.getConfiguration().getBrokerDeduplicationProducerInactivityTimeoutMinutes()); - Iterator> mapIterator = inactiveProducers.entrySet().iterator(); + // if not enabled just clear all inactive producer record. + if (!isEnabled()) { + if (!inactiveProducers.isEmpty()) { + inactiveProducers.clear(); + } + return; + } + + Iterator> mapIterator = inactiveProducers.entrySet().iterator(); boolean hasInactive = false; while (mapIterator.hasNext()) { java.util.Map.Entry entry = mapIterator.next(); @@ -482,17 +535,21 @@ public long getLastPublishedSequenceId(String producerName) { } public void takeSnapshot() { + if (!isEnabled()) { + return; + } + Integer interval = topic.getHierarchyTopicPolicies().getDeduplicationSnapshotIntervalSeconds().get(); long currentTimeStamp = System.currentTimeMillis(); if (interval == null || interval <= 0 || currentTimeStamp - lastSnapshotTimestamp < TimeUnit.SECONDS.toMillis(interval)) { return; } - PositionImpl position = (PositionImpl) managedLedger.getLastConfirmedEntry(); + Position position = managedLedger.getLastConfirmedEntry(); if (position == null) { return; } - PositionImpl markDeletedPosition = (PositionImpl) managedCursor.getMarkDeletedPosition(); + Position markDeletedPosition = managedCursor.getMarkDeletedPosition(); if (markDeletedPosition != null && position.compareTo(markDeletedPosition) <= 0) { return; } @@ -504,5 +561,10 @@ ManagedCursor getManagedCursor() { return managedCursor; } + @VisibleForTesting + Map getInactiveProducers() { + return inactiveProducers; + } + private static final Logger log = LoggerFactory.getLogger(MessageDeduplication.class); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryController.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryController.java index 5bf3f5506fa81..fa6e1412151b6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryController.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryController.java @@ -22,9 +22,13 @@ import java.util.ArrayList; import java.util.List; import java.util.NavigableSet; +import java.util.Optional; import java.util.Set; +import java.util.TreeSet; +import java.util.function.Predicate; import javax.annotation.concurrent.NotThreadSafe; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.util.collections.ConcurrentLongLongHashMap; import org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap; import org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap.LongPair; @@ -95,6 +99,14 @@ private void removeFromHashBlocker(long ledgerId, long entryId) { } } + public Long getHash(long ledgerId, long entryId) { + LongPair value = hashesToBeBlocked.get(ledgerId, entryId); + if (value == null) { + return null; + } + return value.first; + } + public void removeAllUpTo(long markDeleteLedgerId, long markDeleteEntryId) { if (!allowOutOfOrderDelivery) { List keysToRemove = new ArrayList<>(); @@ -137,7 +149,40 @@ public boolean containsStickyKeyHashes(Set stickyKeyHashes) { return false; } - public NavigableSet getMessagesToReplayNow(int maxMessagesToRead) { - return messagesToRedeliver.items(maxMessagesToRead, PositionImpl::new); + public boolean containsStickyKeyHash(int stickyKeyHash) { + return !allowOutOfOrderDelivery && hashesRefCount.containsKey(stickyKeyHash); + } + + public Optional getFirstPositionInReplay() { + return messagesToRedeliver.first(PositionFactory::create); + } + + /** + * Get the messages to replay now. + * + * @param maxMessagesToRead + * the max messages to read + * @param filter + * the filter to use to select the messages to replay + * @return the messages to replay now + */ + public NavigableSet getMessagesToReplayNow(int maxMessagesToRead, Predicate filter) { + NavigableSet items = new TreeSet<>(); + messagesToRedeliver.processItems(PositionFactory::create, item -> { + if (filter.test(item)) { + items.add(item); + } + return items.size() < maxMessagesToRead; + }); + return items; + } + + /** + * Get the number of messages registered for replay in the redelivery controller. + * + * @return number of messages + */ + public int size() { + return messagesToRedeliver.size(); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index b3d48252efe58..d479d8f384ee9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.service.persistent; import static org.apache.pulsar.broker.service.persistent.PersistentTopic.MESSAGE_RATE_BACKOFF_MS; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.common.collect.Range; import java.util.ArrayList; @@ -44,13 +45,15 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.NoMoreEntriesToReadException; import org.apache.bookkeeper.mledger.ManagedLedgerException.TooManyRequestsException; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory; import org.apache.pulsar.broker.delayed.DelayedDeliveryTracker; import org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerFactory; import org.apache.pulsar.broker.delayed.InMemoryDelayedDeliveryTracker; import org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.service.AbstractDispatcherMultipleConsumers; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException; @@ -68,11 +71,11 @@ import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter.Type; import org.apache.pulsar.broker.transaction.exception.buffer.TransactionBufferException; -import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.policies.data.stats.TopicMetricBean; import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.util.Backoff; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.FutureUtil; import org.slf4j.Logger; @@ -83,10 +86,9 @@ */ public class PersistentDispatcherMultipleConsumers extends AbstractDispatcherMultipleConsumers implements Dispatcher, ReadEntriesCallback { - protected final PersistentTopic topic; protected final ManagedCursor cursor; - protected volatile Range lastIndividualDeletedRangeFromCursorRecovery; + protected volatile Range lastIndividualDeletedRangeFromCursorRecovery; private CompletableFuture closeFuture = null; protected final MessageRedeliveryController redeliveryMessages; @@ -96,7 +98,7 @@ public class PersistentDispatcherMultipleConsumers extends AbstractDispatcherMul protected volatile boolean havePendingRead = false; protected volatile boolean havePendingReplayRead = false; - protected volatile PositionImpl minReplayedPosition = null; + protected volatile Position minReplayedPosition = null; protected boolean shouldRewindBeforeReadingOrReplaying = false; protected final String name; private boolean sendInProgress = false; @@ -112,6 +114,17 @@ public class PersistentDispatcherMultipleConsumers extends AbstractDispatcherMul AtomicIntegerFieldUpdater.newUpdater(PersistentDispatcherMultipleConsumers.class, "totalUnackedMessages"); protected volatile int totalUnackedMessages = 0; + /** + * A signature that relate to the check of "Dispatching has paused on cursor data can fully persist". + * Note: It is a tool that helps determine whether it should trigger a new reading after acknowledgments to avoid + * too many CPU circles, see {@link #afterAckMessages(Throwable, Object)} for more details. Do not use this + * to confirm whether the delivery should be paused, please call {@link #shouldPauseOnAckStatePersist}. + */ + protected static final AtomicIntegerFieldUpdater + BLOCKED_DISPATCHER_ON_CURSOR_DATA_CAN_NOT_FULLY_PERSIST_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(PersistentDispatcherMultipleConsumers.class, + "blockedDispatcherOnCursorDataCanNotFullyPersist"); + private volatile int blockedDispatcherOnCursorDataCanNotFullyPersist = FALSE; private volatile int blockedDispatcherOnUnackedMsgs = FALSE; protected static final AtomicIntegerFieldUpdater BLOCKED_DISPATCHER_ON_UNACKMSG_UPDATER = @@ -121,7 +134,13 @@ public class PersistentDispatcherMultipleConsumers extends AbstractDispatcherMul private AtomicBoolean isRescheduleReadInProgress = new AtomicBoolean(false); protected final ExecutorService dispatchMessagesThread; private final SharedConsumerAssignor assignor; - + // tracks how many entries were processed by consumers in the last trySendMessagesToConsumers call + // the number includes also delayed messages, marker messages, aborted txn messages and filtered messages + // When no messages were processed, the value is 0. This is also an indication that the dispatcher didn't + // make progress in the last trySendMessagesToConsumers call. + protected int lastNumberOfEntriesProcessed; + protected boolean skipNextBackoff; + private final Backoff retryBackoff; protected enum ReadType { Normal, Replay } @@ -146,10 +165,15 @@ public PersistentDispatcherMultipleConsumers(PersistentTopic topic, ManagedCurso this.readBatchSize = serviceConfig.getDispatcherMaxReadBatchSize(); this.initializeDispatchRateLimiterIfNeeded(); this.assignor = new SharedConsumerAssignor(this::getNextConsumer, this::addMessageToReplay); + ServiceConfiguration serviceConfiguration = topic.getBrokerService().pulsar().getConfiguration(); this.readFailureBackoff = new Backoff( - topic.getBrokerService().pulsar().getConfiguration().getDispatcherReadFailureBackoffInitialTimeInMs(), + serviceConfiguration.getDispatcherReadFailureBackoffInitialTimeInMs(), TimeUnit.MILLISECONDS, 1, TimeUnit.MINUTES, 0, TimeUnit.MILLISECONDS); + retryBackoff = new Backoff( + serviceConfiguration.getDispatcherRetryBackoffInitialTimeInMs(), TimeUnit.MILLISECONDS, + serviceConfiguration.getDispatcherRetryBackoffMaxTimeInMs(), TimeUnit.MILLISECONDS, + 0, TimeUnit.MILLISECONDS); } @Override @@ -177,9 +201,15 @@ public synchronized CompletableFuture addConsumer(Consumer consumer) { } if (isConsumersExceededOnSubscription()) { - log.warn("[{}] Attempting to add consumer to subscription which reached max consumers limit", name); + log.warn("[{}] Attempting to add consumer to subscription which reached max consumers limit {}", + name, consumer); return FutureUtil.failedFuture(new ConsumerBusyException("Subscription reached max consumers limit")); } + // This is not an expected scenario, it will never happen in expected. Just print a warn log if the unexpected + // scenario happens. See more detail: https://github.com/apache/pulsar/pull/22283. + if (consumerSet.contains(consumer)) { + log.warn("[{}] Attempting to add a consumer that already registered {}", name, consumer); + } consumerList.add(consumer); if (consumerList.size() > 1 @@ -204,15 +234,7 @@ public synchronized void removeConsumer(Consumer consumer) throws BrokerServiceE consumerList.remove(consumer); log.info("Removed consumer {} with pending {} acks", consumer, consumer.getPendingAcks().size()); if (consumerList.isEmpty()) { - cancelPendingRead(); - - redeliveryMessages.clear(); - redeliveryTracker.clear(); - if (closeFuture != null) { - log.info("[{}] All consumers removed. Subscription is disconnected", name); - closeFuture.complete(null); - } - totalAvailablePermits = 0; + clearComponentsAfterRemovedAllConsumers(); } else { if (log.isDebugEnabled()) { log.debug("[{}] Consumer are left, reading more entries", name); @@ -229,10 +251,31 @@ public synchronized void removeConsumer(Consumer consumer) throws BrokerServiceE readMoreEntries(); } } else { - log.info("[{}] Trying to remove a non-connected consumer: {}", name, consumer); + /** + * This is not an expected scenario, it will never happen in expected. + * Just add a defensive code to avoid the topic can not be unloaded anymore: remove the consumers which + * are not mismatch with {@link #consumerSet}. See more detail: https://github.com/apache/pulsar/pull/22270. + */ + log.error("[{}] Trying to remove a non-connected consumer: {}", name, consumer); + consumerList.removeIf(c -> consumer.equals(c)); + if (consumerList.isEmpty()) { + clearComponentsAfterRemovedAllConsumers(); + } } } + private synchronized void clearComponentsAfterRemovedAllConsumers() { + cancelPendingRead(); + + redeliveryMessages.clear(); + redeliveryTracker.clear(); + if (closeFuture != null) { + log.info("[{}] All consumers removed. Subscription is disconnected", name); + closeFuture.complete(null); + } + totalAvailablePermits = 0; + } + @Override public void consumerFlow(Consumer consumer, int additionalNumberOfMessages) { topic.getBrokerService().executor().execute(() -> { @@ -267,12 +310,30 @@ public void readMoreEntriesAsync() { } public synchronized void readMoreEntries() { + if (cursor.isClosed()) { + if (log.isDebugEnabled()) { + log.debug("[{}] Cursor is already closed, skipping read more entries.", cursor.getName()); + } + return; + } if (isSendInProgress()) { // we cannot read more entries while sending the previous batch // otherwise we could re-read the same entries and send duplicates + if (log.isDebugEnabled()) { + log.debug("[{}] [{}] Skipping read for the topic, Due to sending in-progress.", + topic.getName(), getSubscriptionName()); + } return; } if (shouldPauseDeliveryForDelayTracker()) { + if (log.isDebugEnabled()) { + log.debug("[{}] [{}] Skipping read for the topic, Due to pause delivery for delay tracker.", + topic.getName(), getSubscriptionName()); + } + return; + } + if (topic.isTransferring()) { + // Do not deliver messages for topics that are undergoing transfer, as the acknowledgments would be ignored. return; } @@ -289,22 +350,21 @@ public synchronized void readMoreEntries() { return; } - NavigableSet messagesToReplayNow = getMessagesToReplayNow(messagesToRead); - + Set messagesToReplayNow = + canReplayMessages() ? getMessagesToReplayNow(messagesToRead) : Collections.emptySet(); if (!messagesToReplayNow.isEmpty()) { if (log.isDebugEnabled()) { - log.debug("[{}] Schedule replay of {} messages for {} consumers", name, messagesToReplayNow.size(), - consumerList.size()); + log.debug("[{}] Schedule replay of {} messages for {} consumers", name, + messagesToReplayNow.size(), consumerList.size()); } - havePendingReplayRead = true; - minReplayedPosition = messagesToReplayNow.first(); + updateMinReplayedPosition(); Set deletedMessages = topic.isDelayedDeliveryEnabled() - ? asyncReplayEntriesInOrder(messagesToReplayNow) : asyncReplayEntries(messagesToReplayNow); + ? asyncReplayEntriesInOrder(messagesToReplayNow) + : asyncReplayEntries(messagesToReplayNow); // clear already acked positions from replay bucket - - deletedMessages.forEach(position -> redeliveryMessages.remove(((PositionImpl) position).getLedgerId(), - ((PositionImpl) position).getEntryId())); + deletedMessages.forEach(position -> redeliveryMessages.remove(position.getLedgerId(), + position.getEntryId())); // if all the entries are acked-entries and cleared up from redeliveryMessages, try to read // next entries as readCompletedEntries-callback was never called if ((messagesToReplayNow.size() - deletedMessages.size()) == 0) { @@ -316,23 +376,30 @@ public synchronized void readMoreEntries() { log.debug("[{}] Dispatcher read is blocked due to unackMessages {} reached to max {}", name, totalUnackedMessages, topic.getMaxUnackedMessagesOnSubscription()); } - } else if (!havePendingRead) { + } else if (doesntHavePendingRead()) { + if (!isNormalReadAllowed()) { + handleNormalReadNotAllowed(); + return; + } + if (shouldPauseOnAckStatePersist(ReadType.Normal)) { + if (log.isDebugEnabled()) { + log.debug("[{}] [{}] Skipping read for the topic, Due to blocked on ack state persistent.", + topic.getName(), getSubscriptionName()); + } + return; + } if (log.isDebugEnabled()) { log.debug("[{}] Schedule read of {} messages for {} consumers", name, messagesToRead, consumerList.size()); } havePendingRead = true; - NavigableSet toReplay = getMessagesToReplayNow(1); - if (!toReplay.isEmpty()) { - minReplayedPosition = toReplay.first(); - redeliveryMessages.add(minReplayedPosition.getLedgerId(), minReplayedPosition.getEntryId()); - } else { - minReplayedPosition = null; - } + updateMinReplayedPosition(); + + messagesToRead = Math.min(messagesToRead, getMaxEntriesReadLimit()); // Filter out and skip read delayed messages exist in DelayedDeliveryTracker if (delayedDeliveryTracker.isPresent()) { - Predicate skipCondition = null; + Predicate skipCondition = null; final DelayedDeliveryTracker deliveryTracker = delayedDeliveryTracker.get(); if (deliveryTracker instanceof BucketDelayedDeliveryTracker) { skipCondition = position -> ((BucketDelayedDeliveryTracker) deliveryTracker) @@ -345,7 +412,9 @@ public synchronized void readMoreEntries() { topic.getMaxReadPosition()); } } else { - log.debug("[{}] Cannot schedule next read until previous one is done", name); + if (log.isDebugEnabled()) { + log.debug("[{}] Cannot schedule next read until previous one is done", name); + } } } else { if (log.isDebugEnabled()) { @@ -354,18 +423,80 @@ public synchronized void readMoreEntries() { } } + /** + * Sets a hard limit on the number of entries to read from the Managed Ledger. + * Subclasses can override this method to set a different limit. + * By default, this method does not impose an additional limit. + * + * @return the maximum number of entries to read from the Managed Ledger + */ + protected int getMaxEntriesReadLimit() { + return Integer.MAX_VALUE; + } + + /** + * Checks if there's a pending read operation that hasn't completed yet. + * This allows to avoid scheduling a new read operation while the previous one is still in progress. + * @return true if there's a pending read operation + */ + protected boolean doesntHavePendingRead() { + return !havePendingRead; + } + + protected void handleNormalReadNotAllowed() { + // do nothing + } + + /** + * Controls whether replaying entries is currently enabled. + * Subclasses can override this method to temporarily disable replaying entries. + * @return true if replaying entries is currently enabled + */ + protected boolean canReplayMessages() { + return true; + } + + private void updateMinReplayedPosition() { + minReplayedPosition = getFirstPositionInReplay().orElse(null); + } + + private boolean shouldPauseOnAckStatePersist(ReadType readType) { + // Allows new consumers to consume redelivered messages caused by the just-closed consumer. + if (readType != ReadType.Normal) { + return false; + } + if (!((PersistentTopic) subscription.getTopic()).isDispatcherPauseOnAckStatePersistentEnabled()) { + return false; + } + if (cursor == null) { + return true; + } + return blockedDispatcherOnCursorDataCanNotFullyPersist == TRUE; + } + @Override protected void reScheduleRead() { + reScheduleReadInMs(MESSAGE_RATE_BACKOFF_MS); + } + + protected synchronized void reScheduleReadWithBackoff() { + reScheduleReadInMs(retryBackoff.next()); + } + + protected void reScheduleReadInMs(long readAfterMs) { if (isRescheduleReadInProgress.compareAndSet(false, true)) { if (log.isDebugEnabled()) { - log.debug("[{}] [{}] Reschedule message read in {} ms", topic.getName(), name, MESSAGE_RATE_BACKOFF_MS); + log.debug("[{}] [{}] Reschedule message read in {} ms", topic.getName(), name, readAfterMs); + } + Runnable runnable = () -> { + isRescheduleReadInProgress.set(false); + readMoreEntries(); + }; + if (readAfterMs > 0) { + topic.getBrokerService().executor().schedule(runnable, readAfterMs, TimeUnit.MILLISECONDS); + } else { + topic.getBrokerService().executor().execute(runnable); } - topic.getBrokerService().executor().schedule( - () -> { - isRescheduleReadInProgress.set(false); - readMoreEntries(); - }, - MESSAGE_RATE_BACKOFF_MS, TimeUnit.MILLISECONDS); } } @@ -397,52 +528,52 @@ protected Pair calculateToRead(int currentTotalAvailablePermits) if (serviceConfig.isDispatchThrottlingOnNonBacklogConsumerEnabled() || !cursor.isActive()) { if (topic.getBrokerDispatchRateLimiter().isPresent()) { DispatchRateLimiter brokerRateLimiter = topic.getBrokerDispatchRateLimiter().get(); - if (reachDispatchRateLimit(brokerRateLimiter)) { + Pair calculateToRead = + updateMessagesToRead(brokerRateLimiter, messagesToRead, bytesToRead); + messagesToRead = calculateToRead.getLeft(); + bytesToRead = calculateToRead.getRight(); + if (messagesToRead == 0 || bytesToRead == 0) { if (log.isDebugEnabled()) { log.debug("[{}] message-read exceeded broker message-rate {}/{}, schedule after a {}", name, brokerRateLimiter.getDispatchRateOnMsg(), brokerRateLimiter.getDispatchRateOnByte(), MESSAGE_RATE_BACKOFF_MS); } + reScheduleRead(); return Pair.of(-1, -1L); - } else { - Pair calculateToRead = - updateMessagesToRead(brokerRateLimiter, messagesToRead, bytesToRead); - messagesToRead = calculateToRead.getLeft(); - bytesToRead = calculateToRead.getRight(); } } if (topic.getDispatchRateLimiter().isPresent()) { DispatchRateLimiter topicRateLimiter = topic.getDispatchRateLimiter().get(); - if (reachDispatchRateLimit(topicRateLimiter)) { + Pair calculateToRead = + updateMessagesToRead(topicRateLimiter, messagesToRead, bytesToRead); + messagesToRead = calculateToRead.getLeft(); + bytesToRead = calculateToRead.getRight(); + if (messagesToRead == 0 || bytesToRead == 0) { if (log.isDebugEnabled()) { log.debug("[{}] message-read exceeded topic message-rate {}/{}, schedule after a {}", name, topicRateLimiter.getDispatchRateOnMsg(), topicRateLimiter.getDispatchRateOnByte(), MESSAGE_RATE_BACKOFF_MS); } + reScheduleRead(); return Pair.of(-1, -1L); - } else { - Pair calculateToRead = - updateMessagesToRead(topicRateLimiter, messagesToRead, bytesToRead); - messagesToRead = calculateToRead.getLeft(); - bytesToRead = calculateToRead.getRight(); } } if (dispatchRateLimiter.isPresent()) { - if (reachDispatchRateLimit(dispatchRateLimiter.get())) { + Pair calculateToRead = + updateMessagesToRead(dispatchRateLimiter.get(), messagesToRead, bytesToRead); + messagesToRead = calculateToRead.getLeft(); + bytesToRead = calculateToRead.getRight(); + if (messagesToRead == 0 || bytesToRead == 0) { if (log.isDebugEnabled()) { log.debug("[{}] message-read exceeded subscription message-rate {}/{}, schedule after a {}", name, dispatchRateLimiter.get().getDispatchRateOnMsg(), dispatchRateLimiter.get().getDispatchRateOnByte(), MESSAGE_RATE_BACKOFF_MS); } + reScheduleRead(); return Pair.of(-1, -1L); - } else { - Pair calculateToRead = - updateMessagesToRead(dispatchRateLimiter.get(), messagesToRead, bytesToRead); - messagesToRead = calculateToRead.getLeft(); - bytesToRead = calculateToRead.getRight(); } } } @@ -484,7 +615,8 @@ public synchronized boolean canUnsubscribe(Consumer consumer) { } @Override - public CompletableFuture close() { + public CompletableFuture close(boolean disconnectConsumers, + Optional assignedBrokerLookupData) { IS_CLOSED_UPDATER.set(this, TRUE); Optional delayedDeliveryTracker; @@ -494,19 +626,23 @@ public CompletableFuture close() { } delayedDeliveryTracker.ifPresent(DelayedDeliveryTracker::close); - dispatchRateLimiter.ifPresent(DispatchRateLimiter::close); - return disconnectAllConsumers(); + return disconnectConsumers + ? disconnectAllConsumers(false, assignedBrokerLookupData) : CompletableFuture.completedFuture(null); } @Override - public synchronized CompletableFuture disconnectAllConsumers(boolean isResetCursor) { + public synchronized CompletableFuture disconnectAllConsumers( + boolean isResetCursor, Optional assignedBrokerLookupData) { closeFuture = new CompletableFuture<>(); if (consumerList.isEmpty()) { closeFuture.complete(null); } else { - consumerList.forEach(consumer -> consumer.disconnect(isResetCursor)); + // Iterator of CopyOnWriteArrayList uses the internal array to do the for-each, and CopyOnWriteArrayList + // will create a new internal array when adding/removing a new item. So remove items in the for-each + // block is safety when the for-each and add/remove are using a same lock. + consumerList.forEach(consumer -> consumer.disconnect(isResetCursor, assignedBrokerLookupData)); cancelPendingRead(); } return closeFuture; @@ -514,8 +650,9 @@ public synchronized CompletableFuture disconnectAllConsumers(boolean isRes @Override protected void cancelPendingRead() { - if (havePendingRead && cursor.cancelPendingReadRequest()) { + if ((havePendingRead || havePendingReplayRead) && cursor.cancelPendingReadRequest()) { havePendingRead = false; + havePendingReplayRead = false; } } @@ -573,8 +710,8 @@ public final synchronized void readEntriesComplete(List entries, Object c log.debug("[{}] Distributing {} messages to {} consumers", name, entries.size(), consumerList.size()); } - long size = entries.stream().mapToLong(Entry::getLength).sum(); - updatePendingBytesToDispatch(size); + long totalBytesSize = entries.stream().mapToLong(Entry::getLength).sum(); + updatePendingBytesToDispatch(totalBytesSize); // dispatch messages to a separate thread, but still in order for this subscription // sendMessagesToConsumers is responsible for running broker-side filters @@ -584,19 +721,34 @@ public final synchronized void readEntriesComplete(List entries, Object c // in a separate thread, and we want to prevent more reads acquireSendInProgress(); dispatchMessagesThread.execute(() -> { - if (sendMessagesToConsumers(readType, entries, false)) { - updatePendingBytesToDispatch(-size); - readMoreEntries(); - } else { - updatePendingBytesToDispatch(-size); - } + handleSendingMessagesAndReadingMore(readType, entries, false, totalBytesSize); }); } else { - if (sendMessagesToConsumers(readType, entries, true)) { - updatePendingBytesToDispatch(-size); - readMoreEntriesAsync(); + handleSendingMessagesAndReadingMore(readType, entries, true, totalBytesSize); + } + } + + private synchronized void handleSendingMessagesAndReadingMore(ReadType readType, List entries, + boolean needAcquireSendInProgress, + long totalBytesSize) { + boolean triggerReadingMore = sendMessagesToConsumers(readType, entries, needAcquireSendInProgress); + int entriesProcessed = lastNumberOfEntriesProcessed; + updatePendingBytesToDispatch(-totalBytesSize); + boolean canReadMoreImmediately = false; + if (entriesProcessed > 0 || skipNextBackoff) { + // Reset the backoff when messages were processed + retryBackoff.reset(); + // Reset the possible flag to skip the backoff delay + skipNextBackoff = false; + canReadMoreImmediately = true; + } + if (triggerReadingMore) { + if (canReadMoreImmediately) { + // Call readMoreEntries in the same thread to trigger the next read + readMoreEntries(); } else { - updatePendingBytesToDispatch(-size); + // reschedule a new read with an increasing backoff delay + reScheduleReadWithBackoff(); } } } @@ -635,6 +787,7 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis if (needTrimAckedMessages()) { cursor.trimDeletedEntries(entries); } + lastNumberOfEntriesProcessed = 0; int entriesToDispatch = entries.size(); // Trigger read more messages @@ -645,8 +798,9 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis int remainingMessages = 0; boolean hasChunk = false; for (int i = 0; i < metadataArray.length; i++) { - final MessageMetadata metadata = Commands.peekAndCopyMessageMetadata( - entries.get(i).getDataBuffer(), subscription.toString(), -1); + Entry entry = entries.get(i); + MessageMetadata metadata = entry instanceof EntryAndMetadata ? ((EntryAndMetadata) entry).getMetadata() + : Commands.peekAndCopyMessageMetadata(entry.getDataBuffer(), subscription.toString(), -1); if (metadata != null) { remainingMessages += metadata.getNumMessagesInBatch(); if (!hasChunk && metadata.hasUuid()) { @@ -663,44 +817,45 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis long totalMessagesSent = 0; long totalBytesSent = 0; long totalEntries = 0; + long totalEntriesProcessed = 0; int avgBatchSizePerMsg = remainingMessages > 0 ? Math.max(remainingMessages / entries.size(), 1) : 1; - int firstAvailableConsumerPermits, currentTotalAvailablePermits; - boolean dispatchMessage; - while (entriesToDispatch > 0) { - firstAvailableConsumerPermits = getFirstAvailableConsumerPermits(); - currentTotalAvailablePermits = Math.max(totalAvailablePermits, firstAvailableConsumerPermits); - dispatchMessage = currentTotalAvailablePermits > 0 && firstAvailableConsumerPermits > 0; - if (!dispatchMessage) { - break; - } + // If the dispatcher is closed, firstAvailableConsumerPermits will be 0, which skips dispatching the + // messages. + while (entriesToDispatch > 0 && isAtleastOneConsumerAvailable()) { Consumer c = getNextConsumer(); if (c == null) { // Do nothing, cursor will be rewind at reconnection log.info("[{}] rewind because no available consumer found from total {}", name, consumerList.size()); entries.subList(start, entries.size()).forEach(Entry::release); cursor.rewind(); + lastNumberOfEntriesProcessed = (int) totalEntriesProcessed; return false; } // round-robin dispatch batch size for this consumer int availablePermits = c.isWritable() ? c.getAvailablePermits() : 1; - if (c.getMaxUnackedMessages() > 0) { - // Avoid negative number - int remainUnAckedMessages = Math.max(c.getMaxUnackedMessages() - c.getUnackedMessages(), 0); - availablePermits = Math.min(availablePermits, remainUnAckedMessages); - } if (log.isDebugEnabled() && !c.isWritable()) { log.debug("[{}-{}] consumer is not writable. dispatching only 1 message to {}; " + "availablePermits are {}", topic.getName(), name, c, c.getAvailablePermits()); } - int messagesForC = Math.min(Math.min(remainingMessages, availablePermits), - serviceConfig.getDispatcherMaxRoundRobinBatchSize()); - messagesForC = Math.max(messagesForC / avgBatchSizePerMsg, 1); - - int end = Math.min(start + messagesForC, entries.size()); + int maxMessagesInThisBatch = + Math.max(remainingMessages, serviceConfig.getDispatcherMaxRoundRobinBatchSize()); + if (c.getMaxUnackedMessages() > 0) { + // Calculate the maximum number of additional unacked messages allowed + int maxAdditionalUnackedMessages = Math.max(c.getMaxUnackedMessages() - c.getUnackedMessages(), 0); + maxMessagesInThisBatch = Math.min(maxMessagesInThisBatch, maxAdditionalUnackedMessages); + } + int maxEntriesInThisBatch = Math.min(availablePermits, + // use the average batch size per message to calculate the number of entries to + // dispatch. round up to the next integer without using floating point arithmetic. + (maxMessagesInThisBatch + avgBatchSizePerMsg - 1) / avgBatchSizePerMsg); + // pick at least one entry to dispatch + maxEntriesInThisBatch = Math.max(maxEntriesInThisBatch, 1); + + int end = Math.min(start + maxEntriesInThisBatch, entries.size()); List entriesForThisConsumer = entries.subList(start, end); // remove positions first from replay list first : sendMessages recycles entries @@ -714,17 +869,19 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis EntryBatchSizes batchSizes = EntryBatchSizes.get(entriesForThisConsumer.size()); EntryBatchIndexesAcks batchIndexesAcks = EntryBatchIndexesAcks.get(entriesForThisConsumer.size()); + totalEntries += filterEntriesForConsumer(metadataArray, start, entriesForThisConsumer, batchSizes, sendMessageInfo, batchIndexesAcks, cursor, readType == ReadType.Replay, c); + totalEntriesProcessed += entriesForThisConsumer.size(); c.sendMessages(entriesForThisConsumer, batchSizes, batchIndexesAcks, sendMessageInfo.getTotalMessages(), sendMessageInfo.getTotalBytes(), sendMessageInfo.getTotalChunkedMessages(), redeliveryTracker); int msgSent = sendMessageInfo.getTotalMessages(); remainingMessages -= msgSent; - start += messagesForC; - entriesToDispatch -= messagesForC; + start += maxEntriesInThisBatch; + entriesToDispatch -= maxEntriesInThisBatch; TOTAL_AVAILABLE_PERMITS_UPDATER.addAndGet(this, -(msgSent - batchIndexesAcks.getTotalAckedIndexCount())); if (log.isDebugEnabled()) { @@ -736,6 +893,7 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis totalBytesSent += sendMessageInfo.getTotalBytes(); } + lastNumberOfEntriesProcessed = (int) totalEntriesProcessed; acquirePermitsForDeliveredMessages(topic, cursor, totalEntries, totalMessagesSent, totalBytesSent); if (entriesToDispatch > 0) { @@ -744,14 +902,19 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis entries.size() - start); } entries.subList(start, entries.size()).forEach(entry -> { - long stickyKeyHash = getStickyKeyHash(entry); - addMessageToReplay(entry.getLedgerId(), entry.getEntryId(), stickyKeyHash); + addEntryToReplay(entry); entry.release(); }); } + return true; } + protected void addEntryToReplay(Entry entry) { + long stickyKeyHash = getStickyKeyHash(entry); + addMessageToReplay(entry.getLedgerId(), entry.getEntryId(), stickyKeyHash); + } + private boolean sendChunkedMessagesToConsumers(ReadType readType, List entries, MessageMetadata[] metadataArray) { @@ -765,6 +928,7 @@ private boolean sendChunkedMessagesToConsumers(ReadType readType, long totalMessagesSent = 0; long totalBytesSent = 0; long totalEntries = 0; + long totalEntriesProcessed = 0; final AtomicInteger numConsumers = new AtomicInteger(assignResult.size()); for (Map.Entry> current : assignResult.entrySet()) { final Consumer consumer = current.getKey(); @@ -795,6 +959,7 @@ private boolean sendChunkedMessagesToConsumers(ReadType readType, totalEntries += filterEntriesForConsumer(entryAndMetadataList, batchSizes, sendMessageInfo, batchIndexesAcks, cursor, readType == ReadType.Replay, consumer); + totalEntriesProcessed += entryAndMetadataList.size(); consumer.sendMessages(entryAndMetadataList, batchSizes, batchIndexesAcks, sendMessageInfo.getTotalMessages(), sendMessageInfo.getTotalBytes(), sendMessageInfo.getTotalChunkedMessages(), getRedeliveryTracker() @@ -810,6 +975,7 @@ private boolean sendChunkedMessagesToConsumers(ReadType readType, totalBytesSent += sendMessageInfo.getTotalBytes(); } + lastNumberOfEntriesProcessed = (int) totalEntriesProcessed; acquirePermitsForDeliveredMessages(topic, cursor, totalEntries, totalMessagesSent, totalBytesSent); return numConsumers.get() == 0; // trigger a new readMoreEntries() call @@ -821,7 +987,14 @@ public synchronized void readEntriesFailed(ManagedLedgerException exception, Obj ReadType readType = (ReadType) ctx; long waitTimeMillis = readFailureBackoff.next(); - if (exception instanceof NoMoreEntriesToReadException) { + // Do not keep reading more entries if the cursor is already closed. + if (exception instanceof ManagedLedgerException.CursorAlreadyClosedException) { + if (log.isDebugEnabled()) { + log.debug("[{}] Cursor is already closed, skipping read more entries", cursor.getName()); + } + // Set the wait time to -1 to avoid rescheduling the read. + waitTimeMillis = -1; + } else if (exception instanceof NoMoreEntriesToReadException) { if (cursor.getNumberOfEntriesInBacklog(false) == 0) { // Topic has been terminated and there are no more entries to read // Notify the consumer only if all the messages were already acknowledged @@ -854,13 +1027,20 @@ public synchronized void readEntriesFailed(ManagedLedgerException exception, Obj } else { havePendingReplayRead = false; if (exception instanceof ManagedLedgerException.InvalidReplayPositionException) { - PositionImpl markDeletePosition = (PositionImpl) cursor.getMarkDeletedPosition(); + Position markDeletePosition = cursor.getMarkDeletedPosition(); redeliveryMessages.removeAllUpTo(markDeletePosition.getLedgerId(), markDeletePosition.getEntryId()); } } readBatchSize = serviceConfig.getDispatcherMinReadBatchSize(); + // Skip read if the waitTimeMillis is a nagetive value. + if (waitTimeMillis >= 0) { + scheduleReadEntriesWithDelay(exception, readType, waitTimeMillis); + } + } + @VisibleForTesting + void scheduleReadEntriesWithDelay(Exception e, ReadType readType, long waitTimeMillis) { topic.getBrokerService().executor().schedule(() -> { synchronized (PersistentDispatcherMultipleConsumers.this) { // If it's a replay read we need to retry even if there's already @@ -870,11 +1050,10 @@ public synchronized void readEntriesFailed(ManagedLedgerException exception, Obj log.info("[{}] Retrying read operation", name); readMoreEntries(); } else { - log.info("[{}] Skipping read retry: havePendingRead {}", name, havePendingRead, exception); + log.info("[{}] Skipping read retry: havePendingRead {}", name, havePendingRead, e); } } }, waitTimeMillis, TimeUnit.MILLISECONDS); - } private boolean needTrimAckedMessages() { @@ -882,7 +1061,7 @@ private boolean needTrimAckedMessages() { return false; } else { return lastIndividualDeletedRangeFromCursorRecovery.upperEndpoint() - .compareTo((PositionImpl) cursor.getReadPosition()) > 0; + .compareTo(cursor.getReadPosition()) > 0; } } @@ -902,7 +1081,7 @@ protected int getFirstAvailableConsumerPermits() { return 0; } for (Consumer consumer : consumerList) { - if (consumer != null && !consumer.isBlocked()) { + if (consumer != null && !consumer.isBlocked() && consumer.cnx().isActive()) { int availablePermits = consumer.getAvailablePermits(); if (availablePermits > 0) { return availablePermits; @@ -926,14 +1105,15 @@ private boolean isConsumerWritable() { @Override public boolean isConsumerAvailable(Consumer consumer) { - return consumer != null && !consumer.isBlocked() && consumer.getAvailablePermits() > 0; + return consumer != null && !consumer.isBlocked() && consumer.cnx().isActive() + && consumer.getAvailablePermits() > 0; } @Override public synchronized void redeliverUnacknowledgedMessages(Consumer consumer, long consumerEpoch) { consumer.getPendingAcks().forEach((ledgerId, entryId, batchSize, stickyKeyHash) -> { if (addMessageToReplay(ledgerId, entryId, stickyKeyHash)) { - redeliveryTracker.incrementAndGetRedeliveryCount((PositionImpl.get(ledgerId, entryId))); + redeliveryTracker.incrementAndGetRedeliveryCount((PositionFactory.create(ledgerId, entryId))); } }); if (log.isDebugEnabled()) { @@ -944,7 +1124,7 @@ public synchronized void redeliverUnacknowledgedMessages(Consumer consumer, long } @Override - public synchronized void redeliverUnacknowledgedMessages(Consumer consumer, List positions) { + public synchronized void redeliverUnacknowledgedMessages(Consumer consumer, List positions) { positions.forEach(position -> { // TODO: We want to pass a sticky key hash as a third argument to guarantee the order of the messages // on Key_Shared subscription, but it's difficult to get the sticky key here @@ -995,6 +1175,53 @@ public void addUnAckedMessages(int numberOfMessages) { topic.getBrokerService().addUnAckedMessages(this, numberOfMessages); } + @Override + public void afterAckMessages(Throwable exOfDeletion, Object ctxOfDeletion) { + boolean unPaused = blockedDispatcherOnCursorDataCanNotFullyPersist == FALSE; + // Trigger a new read if needed. + boolean shouldPauseNow = !checkAndResumeIfPaused(); + // Switch stat to "paused" if needed. + if (unPaused && shouldPauseNow) { + if (!BLOCKED_DISPATCHER_ON_CURSOR_DATA_CAN_NOT_FULLY_PERSIST_UPDATER + .compareAndSet(this, FALSE, TRUE)) { + // Retry due to conflict update. + afterAckMessages(exOfDeletion, ctxOfDeletion); + } + } + } + + @Override + public boolean checkAndResumeIfPaused() { + boolean paused = blockedDispatcherOnCursorDataCanNotFullyPersist == TRUE; + // Calling "cursor.isCursorDataFullyPersistable()" will loop the collection "individualDeletedMessages". It is + // not a light method. + // If never enabled "dispatcherPauseOnAckStatePersistentEnabled", skip the following checks to improve + // performance. + if (!paused && !topic.isDispatcherPauseOnAckStatePersistentEnabled()){ + // "true" means no need to pause. + return true; + } + // Enabled "dispatcherPauseOnAckStatePersistentEnabled" before. + boolean shouldPauseNow = !cursor.isCursorDataFullyPersistable() + && topic.isDispatcherPauseOnAckStatePersistentEnabled(); + // No need to change. + if (paused == shouldPauseNow) { + return !shouldPauseNow; + } + // Should change to "un-pause". + if (paused && !shouldPauseNow) { + // If there was no previous pause due to cursor data is too large to persist, we don't need to manually + // trigger a new read. This can avoid too many CPU circles. + if (BLOCKED_DISPATCHER_ON_CURSOR_DATA_CAN_NOT_FULLY_PERSIST_UPDATER.compareAndSet(this, TRUE, FALSE)) { + readMoreEntriesAsync(); + } else { + // Retry due to conflict update. + checkAndResumeIfPaused(); + } + } + return !shouldPauseNow; + } + public boolean isBlockedDispatcherOnUnackedMsgs() { return blockedDispatcherOnUnackedMsgs == TRUE; } @@ -1025,13 +1252,6 @@ public Optional getRateLimiter() { return dispatchRateLimiter; } - @Override - public void updateRateLimiter() { - if (!initializeDispatchRateLimiterIfNeeded()) { - this.dispatchRateLimiter.ifPresent(DispatchRateLimiter::updateDispatchRate); - } - } - @Override public boolean initializeDispatchRateLimiterIfNeeded() { if (!dispatchRateLimiter.isPresent() && DispatchRateLimiter.isDispatchRateEnabled( @@ -1051,15 +1271,15 @@ public boolean trackDelayedDelivery(long ledgerId, long entryId, MessageMetadata } synchronized (this) { - if (!delayedDeliveryTracker.isPresent()) { + if (delayedDeliveryTracker.isEmpty()) { if (!msgMetadata.hasDeliverAtTime()) { // No need to initialize the tracker here return false; } // Initialize the tracker the first time we need to use it - delayedDeliveryTracker = Optional - .of(topic.getBrokerService().getDelayedDeliveryTrackerFactory().newTracker(this)); + delayedDeliveryTracker = Optional.of( + topic.getBrokerService().getDelayedDeliveryTrackerFactory().newTracker(this)); } delayedDeliveryTracker.get().resetTickTime(topic.getDelayedDeliveryTickTimeMillis()); @@ -1069,21 +1289,48 @@ public boolean trackDelayedDelivery(long ledgerId, long entryId, MessageMetadata } } - protected synchronized NavigableSet getMessagesToReplayNow(int maxMessagesToRead) { + protected synchronized NavigableSet getMessagesToReplayNow(int maxMessagesToRead) { if (delayedDeliveryTracker.isPresent() && delayedDeliveryTracker.get().hasMessageAvailable()) { delayedDeliveryTracker.get().resetTickTime(topic.getDelayedDeliveryTickTimeMillis()); - NavigableSet messagesAvailableNow = + NavigableSet messagesAvailableNow = delayedDeliveryTracker.get().getScheduledMessages(maxMessagesToRead); messagesAvailableNow.forEach(p -> redeliveryMessages.add(p.getLedgerId(), p.getEntryId())); } - if (!redeliveryMessages.isEmpty()) { - return redeliveryMessages.getMessagesToReplayNow(maxMessagesToRead); + return redeliveryMessages.getMessagesToReplayNow(maxMessagesToRead, createFilterForReplay()); } else { return Collections.emptyNavigableSet(); } } + protected Optional getFirstPositionInReplay() { + return redeliveryMessages.getFirstPositionInReplay(); + } + + /** + * Creates a stateful filter for filtering replay positions. + * This is only used for Key_Shared mode to skip replaying certain entries. + * Filter out the entries that will be discarded due to the order guarantee mechanism of Key_Shared mode. + * This method is in order to avoid the scenario below: + * - Get positions from the Replay queue. + * - Read entries from BK. + * - The order guarantee mechanism of Key_Shared mode filtered out all the entries. + * - Delivery non entry to the client, but we did a BK read. + */ + protected Predicate createFilterForReplay() { + // pick all positions from the replay + return position -> true; + } + + /** + * Checks if the dispatcher is allowed to read messages from the cursor. + */ + protected boolean isNormalReadAllowed() { + return true; + } + + + protected synchronized boolean shouldPauseDeliveryForDelayTracker() { return delayedDeliveryTracker.isPresent() && delayedDeliveryTracker.get().shouldPauseAllDeliveries(); } @@ -1192,5 +1439,14 @@ protected int getStickyKeyHash(Entry entry) { return StickyKeyConsumerSelector.makeStickyKeyHash(peekStickyKey(entry.getDataBuffer())); } + + public Subscription getSubscription() { + return subscription; + } + + public long getNumberOfMessagesInReplay() { + return redeliveryMessages.size(); + } + private static final Logger log = LoggerFactory.getLogger(PersistentDispatcherMultipleConsumers.class); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java index 7cbf7bd2c787a..b451a8ad5dc0d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java @@ -20,12 +20,12 @@ import static org.apache.pulsar.broker.service.persistent.PersistentTopic.MESSAGE_RATE_BACKOFF_MS; import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; +import com.google.common.annotations.VisibleForTesting; import io.netty.util.Recycler; import java.util.Iterator; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -37,7 +37,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.ConcurrentWaitCallbackException; import org.apache.bookkeeper.mledger.ManagedLedgerException.NoMoreEntriesToReadException; import org.apache.bookkeeper.mledger.ManagedLedgerException.TooManyRequestsException; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.service.AbstractDispatcherSingleActiveConsumer; import org.apache.pulsar.broker.service.Consumer; @@ -50,9 +50,13 @@ import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter.Type; import org.apache.pulsar.broker.transaction.exception.buffer.TransactionBufferException; -import org.apache.pulsar.client.impl.Backoff; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; +import org.apache.pulsar.common.util.Backoff; import org.apache.pulsar.common.util.Codec; +import org.apache.pulsar.compaction.CompactedTopicUtils; +import org.apache.pulsar.compaction.Compactor; +import org.apache.pulsar.compaction.TopicCompactionService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,7 +65,7 @@ public class PersistentDispatcherSingleActiveConsumer extends AbstractDispatcher private final AtomicBoolean isRescheduleReadInProgress = new AtomicBoolean(false); protected final PersistentTopic topic; - protected final Executor topicExecutor; + protected final Executor executor; protected final String name; private Optional dispatchRateLimiter = Optional.empty(); @@ -70,7 +74,7 @@ public class PersistentDispatcherSingleActiveConsumer extends AbstractDispatcher protected volatile int readBatchSize; protected final Backoff readFailureBackoff; - private volatile ScheduledFuture readOnActiveConsumerTask = null; + private ScheduledFuture readOnActiveConsumerTask = null; private final RedeliveryTracker redeliveryTracker; @@ -79,7 +83,7 @@ public PersistentDispatcherSingleActiveConsumer(ManagedCursor cursor, SubType su super(subscriptionType, partitionIndex, topic.getName(), subscription, topic.getBrokerService().pulsar().getConfiguration(), cursor); this.topic = topic; - this.topicExecutor = topic.getBrokerService().getTopicOrderedExecutor().chooseThread(topicName); + this.executor = topic.getBrokerService().getTopicOrderedExecutor().chooseThread(); this.name = topic.getName() + " / " + (cursor.getName() != null ? Codec.decode(cursor.getName()) : ""/* NonDurableCursor doesn't have name */); this.readBatchSize = serviceConfig.getDispatcherMaxReadBatchSize(); @@ -105,9 +109,9 @@ protected void scheduleReadOnActiveConsumer() { if (log.isDebugEnabled()) { log.debug("[{}] Rewind cursor and read more entries without delay", name); } - cursor.rewind(); + Consumer activeConsumer = getActiveConsumer(); + cursor.rewind(activeConsumer != null && activeConsumer.readCompacted()); - Consumer activeConsumer = ACTIVE_CONSUMER_UPDATER.get(this); notifyActiveConsumerChanged(activeConsumer); readMoreEntries(activeConsumer); return; @@ -125,13 +129,16 @@ protected void scheduleReadOnActiveConsumer() { log.debug("[{}] Rewind cursor and read more entries after {} ms delay", name, serviceConfig.getActiveConsumerFailoverDelayTimeMillis()); } - cursor.rewind(); + Consumer activeConsumer = getActiveConsumer(); + cursor.rewind(activeConsumer != null && activeConsumer.readCompacted()); - Consumer activeConsumer = ACTIVE_CONSUMER_UPDATER.get(this); notifyActiveConsumerChanged(activeConsumer); readMoreEntries(activeConsumer); - readOnActiveConsumerTask = null; + synchronized (this) { + readOnActiveConsumerTask = null; + } }, serviceConfig.getActiveConsumerFailoverDelayTimeMillis(), TimeUnit.MILLISECONDS); + } @Override @@ -148,10 +155,10 @@ protected void cancelPendingRead() { @Override public void readEntriesComplete(final List entries, Object obj) { - topicExecutor.execute(() -> internalReadEntriesComplete(entries, obj)); + executor.execute(() -> internalReadEntriesComplete(entries, obj)); } - public synchronized void internalReadEntriesComplete(final List entries, Object obj) { + private synchronized void internalReadEntriesComplete(final List entries, Object obj) { ReadEntriesCtx readEntriesCtx = (ReadEntriesCtx) obj; Consumer readConsumer = readEntriesCtx.getConsumer(); long epoch = readEntriesCtx.getEpoch(); @@ -175,7 +182,7 @@ public synchronized void internalReadEntriesComplete(final List entries, readFailureBackoff.reduceToHalf(); - Consumer currentConsumer = ACTIVE_CONSUMER_UPDATER.get(this); + Consumer currentConsumer = getActiveConsumer(); if (isKeyHashRangeFiltered) { Iterator iterator = entries.iterator(); @@ -191,14 +198,20 @@ public synchronized void internalReadEntriesComplete(final List entries, } } - if (currentConsumer == null || readConsumer != currentConsumer) { - // Active consumer has changed since the read request has been issued. We need to rewind the cursor and - // re-issue the read request for the new consumer + if (currentConsumer == null || readConsumer != currentConsumer || topic.isTransferring()) { + // Active consumer has changed since the read request has been issued, or the topic is being transferred to + // another broker. We need to rewind the cursor and re-issue the read request for the new consumer. if (log.isDebugEnabled()) { - log.debug("[{}] rewind because no available consumer found", name); + if (currentConsumer == null) { + log.debug("[{}] rewind because no available consumer found", name); + } else if (readConsumer != currentConsumer) { + log.debug("[{}] rewind because active consumer changed", name); + } else { + log.debug("[{}] rewind because topic is transferring", name); + } } entries.forEach(Entry::release); - cursor.rewind(); + cursor.rewind(currentConsumer != null ? currentConsumer.readCompacted() : readConsumer.readCompacted()); if (currentConsumer != null) { notifyActiveConsumerChanged(currentConsumer); readMoreEntries(currentConsumer); @@ -226,19 +239,14 @@ protected void dispatchEntriesToConsumer(Consumer currentConsumer, List e sendMessageInfo.getTotalMessages(), sendMessageInfo.getTotalBytes()); // Schedule a new read batch operation only after the previous batch has been written to the socket. - topicExecutor.execute(() -> { - synchronized (PersistentDispatcherSingleActiveConsumer.this) { - Consumer newConsumer = getActiveConsumer(); - readMoreEntries(newConsumer); - } - }); + executor.execute(() -> readMoreEntries(getActiveConsumer())); } }); } @Override public void consumerFlow(Consumer consumer, int additionalNumberOfMessages) { - topicExecutor.execute(() -> internalConsumerFlow(consumer)); + executor.execute(() -> internalConsumerFlow(consumer)); } private synchronized void internalConsumerFlow(Consumer consumer) { @@ -247,7 +255,7 @@ private synchronized void internalConsumerFlow(Consumer consumer) { log.debug("[{}-{}] Ignoring flow control message since we already have a pending read req", name, consumer); } - } else if (ACTIVE_CONSUMER_UPDATER.get(this) != consumer) { + } else if (getActiveConsumer() != consumer) { if (log.isDebugEnabled()) { log.debug("[{}-{}] Ignoring flow control message since consumer is not active partition consumer", name, consumer); @@ -267,7 +275,7 @@ private synchronized void internalConsumerFlow(Consumer consumer) { @Override public void redeliverUnacknowledgedMessages(Consumer consumer, long consumerEpoch) { - topicExecutor.execute(() -> internalRedeliverUnacknowledgedMessages(consumer, consumerEpoch)); + executor.execute(() -> internalRedeliverUnacknowledgedMessages(consumer, consumerEpoch)); } private synchronized void internalRedeliverUnacknowledgedMessages(Consumer consumer, long consumerEpoch) { @@ -280,7 +288,7 @@ private synchronized void internalRedeliverUnacknowledgedMessages(Consumer consu consumer.setConsumerEpoch(consumerEpoch); } - if (consumer != ACTIVE_CONSUMER_UPDATER.get(this)) { + if (consumer != getActiveConsumer()) { log.info("[{}-{}] Ignoring reDeliverUnAcknowledgedMessages: Only the active consumer can call resend", name, consumer); return; @@ -293,7 +301,7 @@ private synchronized void internalRedeliverUnacknowledgedMessages(Consumer consu } cursor.cancelPendingReadRequest(); havePendingRead = false; - cursor.rewind(); + cursor.rewind(consumer.readCompacted()); if (log.isDebugEnabled()) { log.debug("[{}-{}] Cursor rewinded, redelivering unacknowledged messages. ", name, consumer); } @@ -301,30 +309,50 @@ private synchronized void internalRedeliverUnacknowledgedMessages(Consumer consu } @Override - public void redeliverUnacknowledgedMessages(Consumer consumer, List positions) { + public void redeliverUnacknowledgedMessages(Consumer consumer, List positions) { // We cannot redeliver single messages to single consumers to preserve ordering. redeliverUnacknowledgedMessages(consumer, DEFAULT_CONSUMER_EPOCH); } - @Override - protected void readMoreEntries(Consumer consumer) { + @VisibleForTesting + void readMoreEntries(Consumer consumer) { + if (cursor.isClosed()) { + if (log.isDebugEnabled()) { + log.debug("[{}] Cursor is already closed, skipping read more entries", cursor.getName()); + } + return; + } // consumer can be null when all consumers are disconnected from broker. // so skip reading more entries if currently there is no active consumer. if (null == consumer) { if (log.isDebugEnabled()) { - log.debug("[{}] Skipping read for the topic, Due to the current consumer is null", topic.getName()); + log.debug("[{}] [{}] Skipping read for the topic, Due to the current consumer is null", topic.getName(), + getSubscriptionName()); } return; } if (havePendingRead) { if (log.isDebugEnabled()) { - log.debug("[{}] Skipping read for the topic, Due to we have pending read.", topic.getName()); + log.debug("[{}] [{}] Skipping read for the topic, Due to we have pending read.", topic.getName(), + getSubscriptionName()); + } + return; + } + if (topic.isTransferring()) { + if (log.isDebugEnabled()) { + log.debug("[{}] Skipping read for the topic: topic is transferring", topic.getName()); } return; } if (consumer.getAvailablePermits() > 0) { synchronized (this) { + final Consumer activeConsumer = getActiveConsumer(); + if (consumer != activeConsumer) { + log.info("[{}] cancel the readMoreEntries because consumer {} is no longer the active consumer {}", + topic.getName(), consumer.consumerName(), activeConsumer.consumerName()); + return; + } if (havePendingRead) { if (log.isDebugEnabled()) { log.debug("[{}] Skipping read for the topic, Due to we have pending read.", topic.getName()); @@ -347,8 +375,12 @@ protected void readMoreEntries(Consumer consumer) { } havePendingRead = true; if (consumer.readCompacted()) { - topic.getCompactedTopic().asyncReadEntriesOrWait(cursor, messagesToRead, isFirstRead, - this, consumer); + boolean readFromEarliest = isFirstRead && MessageId.earliest.equals(consumer.getStartMessageId()) + && (!cursor.isDurable() || cursor.getName().equals(Compactor.COMPACTION_SUBSCRIPTION) + || hasValidMarkDeletePosition(cursor)); + TopicCompactionService topicCompactionService = topic.getTopicCompactionService(); + CompactedTopicUtils.asyncReadCompactedEntries(topicCompactionService, cursor, messagesToRead, + bytesToRead, topic.getMaxReadPosition(), readFromEarliest, this, true, consumer); } else { ReadEntriesCtx readEntriesCtx = ReadEntriesCtx.create(consumer, consumer.getConsumerEpoch()); @@ -363,6 +395,13 @@ protected void readMoreEntries(Consumer consumer) { } } + private boolean hasValidMarkDeletePosition(ManagedCursor cursor) { + // If `markDeletedPosition.entryID == -1L` then the md-position is an invalid position, + // since the initial md-position of the consumer will be set to it. + // See ManagedLedgerImpl#asyncOpenCursor and ManagedLedgerImpl#getFirstPosition + return cursor.getMarkDeletedPosition() != null && cursor.getMarkDeletedPosition().getEntryId() == -1L; + } + @Override protected void reScheduleRead() { if (isRescheduleReadInProgress.compareAndSet(false, true)) { @@ -371,7 +410,7 @@ protected void reScheduleRead() { } topic.getBrokerService().executor().schedule(() -> { isRescheduleReadInProgress.set(false); - Consumer currentConsumer = ACTIVE_CONSUMER_UPDATER.get(this); + Consumer currentConsumer = getActiveConsumer(); readMoreEntries(currentConsumer); }, MESSAGE_RATE_BACKOFF_MS, TimeUnit.MILLISECONDS); } @@ -401,52 +440,52 @@ protected Pair calculateToRead(Consumer consumer) { if (serviceConfig.isDispatchThrottlingOnNonBacklogConsumerEnabled() || !cursor.isActive()) { if (topic.getBrokerDispatchRateLimiter().isPresent()) { DispatchRateLimiter brokerRateLimiter = topic.getBrokerDispatchRateLimiter().get(); - if (reachDispatchRateLimit(brokerRateLimiter)) { + Pair calculateToRead = + updateMessagesToRead(brokerRateLimiter, messagesToRead, bytesToRead); + messagesToRead = calculateToRead.getLeft(); + bytesToRead = calculateToRead.getRight(); + if (messagesToRead == 0 || bytesToRead == 0) { if (log.isDebugEnabled()) { log.debug("[{}] message-read exceeded broker message-rate {}/{}, schedule after a {}", name, brokerRateLimiter.getDispatchRateOnMsg(), brokerRateLimiter.getDispatchRateOnByte(), MESSAGE_RATE_BACKOFF_MS); } + reScheduleRead(); return Pair.of(-1, -1L); - } else { - Pair calculateToRead = - updateMessagesToRead(brokerRateLimiter, messagesToRead, bytesToRead); - messagesToRead = calculateToRead.getLeft(); - bytesToRead = calculateToRead.getRight(); } } if (topic.getDispatchRateLimiter().isPresent()) { DispatchRateLimiter topicRateLimiter = topic.getDispatchRateLimiter().get(); - if (reachDispatchRateLimit(topicRateLimiter)) { + Pair calculateToRead = + updateMessagesToRead(topicRateLimiter, messagesToRead, bytesToRead); + messagesToRead = calculateToRead.getLeft(); + bytesToRead = calculateToRead.getRight(); + if (messagesToRead == 0 || bytesToRead == 0) { if (log.isDebugEnabled()) { log.debug("[{}] message-read exceeded topic message-rate {}/{}, schedule after a {}", name, topicRateLimiter.getDispatchRateOnMsg(), topicRateLimiter.getDispatchRateOnByte(), MESSAGE_RATE_BACKOFF_MS); } + reScheduleRead(); return Pair.of(-1, -1L); - } else { - Pair calculateToRead = - updateMessagesToRead(topicRateLimiter, messagesToRead, bytesToRead); - messagesToRead = calculateToRead.getLeft(); - bytesToRead = calculateToRead.getRight(); } } if (dispatchRateLimiter.isPresent()) { - if (reachDispatchRateLimit(dispatchRateLimiter.get())) { + Pair calculateToRead = + updateMessagesToRead(dispatchRateLimiter.get(), messagesToRead, bytesToRead); + messagesToRead = calculateToRead.getLeft(); + bytesToRead = calculateToRead.getRight(); + if (messagesToRead == 0 || bytesToRead == 0) { if (log.isDebugEnabled()) { log.debug("[{}] message-read exceeded subscription message-rate {}/{}, schedule after a {}", name, dispatchRateLimiter.get().getDispatchRateOnMsg(), dispatchRateLimiter.get().getDispatchRateOnByte(), MESSAGE_RATE_BACKOFF_MS); } + reScheduleRead(); return Pair.of(-1, -1L); - } else { - Pair calculateToRead = - updateMessagesToRead(dispatchRateLimiter.get(), messagesToRead, bytesToRead); - messagesToRead = calculateToRead.getLeft(); - bytesToRead = calculateToRead.getRight(); } } } @@ -459,7 +498,7 @@ protected Pair calculateToRead(Consumer consumer) { @Override public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { - topicExecutor.execute(() -> internalReadEntriesFailed(exception, ctx)); + executor.execute(() -> internalReadEntriesFailed(exception, ctx)); } private synchronized void internalReadEntriesFailed(ManagedLedgerException exception, Object ctx) { @@ -468,6 +507,14 @@ private synchronized void internalReadEntriesFailed(ManagedLedgerException excep Consumer c = readEntriesCtx.getConsumer(); readEntriesCtx.recycle(); + // Do not keep reading messages from a closed cursor. + if (exception instanceof ManagedLedgerException.CursorAlreadyClosedException) { + if (log.isDebugEnabled()) { + log.debug("[{}] Cursor was already closed, skipping read more entries", cursor.getName()); + } + return; + } + if (exception instanceof ConcurrentWaitCallbackException) { // At most one pending read request is allowed when there are no more entries, we should not trigger more // read operations in this case and just wait the existing read operation completes. @@ -504,12 +551,17 @@ private synchronized void internalReadEntriesFailed(ManagedLedgerException excep // Reduce read batch size to avoid flooding bookies with retries readBatchSize = serviceConfig.getDispatcherMinReadBatchSize(); + scheduleReadEntriesWithDelay(c, waitTimeMillis); + } + + @VisibleForTesting + void scheduleReadEntriesWithDelay(Consumer c, long delay) { topic.getBrokerService().executor().schedule(() -> { // Jump again into dispatcher dedicated thread - topicExecutor.execute(() -> { + executor.execute(() -> { synchronized (PersistentDispatcherSingleActiveConsumer.this) { - Consumer currentConsumer = ACTIVE_CONSUMER_UPDATER.get(this); + Consumer currentConsumer = getActiveConsumer(); // we should retry the read if we have an active consumer and there is no pending read if (currentConsumer != null && !havePendingRead) { if (log.isDebugEnabled()) { @@ -525,8 +577,7 @@ private synchronized void internalReadEntriesFailed(ManagedLedgerException excep } } }); - }, waitTimeMillis, TimeUnit.MILLISECONDS); - + }, delay, TimeUnit.MILLISECONDS); } @Override @@ -544,13 +595,6 @@ public Optional getRateLimiter() { return dispatchRateLimiter; } - @Override - public void updateRateLimiter() { - if (!initializeDispatchRateLimiterIfNeeded()) { - this.dispatchRateLimiter.ifPresent(DispatchRateLimiter::updateDispatchRate); - } - } - @Override public boolean initializeDispatchRateLimiterIfNeeded() { if (!dispatchRateLimiter.isPresent() && DispatchRateLimiter.isDispatchRateEnabled( @@ -562,16 +606,9 @@ public boolean initializeDispatchRateLimiterIfNeeded() { return false; } - @Override - public CompletableFuture close() { - IS_CLOSED_UPDATER.set(this, TRUE); - dispatchRateLimiter.ifPresent(DispatchRateLimiter::close); - return disconnectAllConsumers(); - } - @Override public boolean checkAndUnblockIfStuck() { - Consumer consumer = ACTIVE_CONSUMER_UPDATER.get(this); + Consumer consumer = getActiveConsumer(); if (consumer == null || cursor.checkAndUpdateReadPositionChanged()) { return false; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java index bddcb1b334df1..3b4bc9d8bceb1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java @@ -18,35 +18,39 @@ */ package org.apache.pulsar.broker.service.persistent; +import com.google.common.annotations.VisibleForTesting; import java.util.Objects; import java.util.Optional; +import java.util.SortedMap; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.LongAdder; +import javax.annotation.Nullable; import org.apache.bookkeeper.mledger.AsyncCallbacks.FindEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback; import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerException.LedgerNotExistException; import org.apache.bookkeeper.mledger.ManagedLedgerException.NonRecoverableLedgerException; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.mledger.proto.MLDataFormats; +import org.apache.pulsar.broker.service.MessageExpirer; import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.stats.Rate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - /** */ -public class PersistentMessageExpiryMonitor implements FindEntryCallback { +public class PersistentMessageExpiryMonitor implements FindEntryCallback, MessageExpirer { private final ManagedCursor cursor; private final String subName; + private final PersistentTopic topic; private final String topicName; private final Rate msgExpired; private final LongAdder totalMsgExpired; - private final boolean autoSkipNonRecoverableData; private final PersistentSubscription subscription; private static final int FALSE = 0; @@ -57,24 +61,32 @@ public class PersistentMessageExpiryMonitor implements FindEntryCallback { expirationCheckInProgressUpdater = AtomicIntegerFieldUpdater .newUpdater(PersistentMessageExpiryMonitor.class, "expirationCheckInProgress"); - public PersistentMessageExpiryMonitor(String topicName, String subscriptionName, ManagedCursor cursor, - PersistentSubscription subscription) { - this.topicName = topicName; + public PersistentMessageExpiryMonitor(PersistentTopic topic, String subscriptionName, ManagedCursor cursor, + @Nullable PersistentSubscription subscription) { + this.topic = topic; + this.topicName = topic.getName(); this.cursor = cursor; this.subName = subscriptionName; this.subscription = subscription; this.msgExpired = new Rate(); this.totalMsgExpired = new LongAdder(); + } + + @VisibleForTesting + public boolean isAutoSkipNonRecoverableData() { // check to avoid test failures - this.autoSkipNonRecoverableData = this.cursor.getManagedLedger() != null + return this.cursor.getManagedLedger() != null && this.cursor.getManagedLedger().getConfig().isAutoSkipNonRecoverableData(); } + @Override public boolean expireMessages(int messageTTLInSeconds) { if (expirationCheckInProgressUpdater.compareAndSet(this, FALSE, TRUE)) { log.info("[{}][{}] Starting message expiry check, ttl= {} seconds", topicName, subName, messageTTLInSeconds); - + // First filter the entire Ledger reached TTL based on the Ledger closing time to avoid client clock skew + checkExpiryByLedgerClosureTime(cursor, messageTTLInSeconds); + // Some part of entries in active Ledger may have reached TTL, so we need to continue searching. cursor.asyncFindNewestMatching(ManagedCursor.FindPositionConstraint.SearchActiveEntries, entry -> { try { long entryTimestamp = Commands.getEntryTimestamp(entry.getDataBuffer()); @@ -96,13 +108,42 @@ public boolean expireMessages(int messageTTLInSeconds) { } } + private void checkExpiryByLedgerClosureTime(ManagedCursor cursor, int messageTTLInSeconds) { + if (messageTTLInSeconds <= 0) { + return; + } + ManagedLedger managedLedger = cursor.getManagedLedger(); + Position deletedPosition = cursor.getMarkDeletedPosition(); + SortedMap ledgerInfoSortedMap = + managedLedger.getLedgersInfo().subMap(deletedPosition.getLedgerId(), true, + managedLedger.getLedgersInfo().lastKey(), true); + MLDataFormats.ManagedLedgerInfo.LedgerInfo info = null; + for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ledgerInfo : ledgerInfoSortedMap.values()) { + if (!ledgerInfo.hasTimestamp() || ledgerInfo.getTimestamp() == 0L + || !MessageImpl.isEntryExpired(messageTTLInSeconds, ledgerInfo.getTimestamp())) { + break; + } + info = ledgerInfo; + } + if (info != null && info.getLedgerId() > -1) { + Position position = PositionFactory.create(info.getLedgerId(), info.getEntries() - 1); + if (managedLedger.getLastConfirmedEntry().compareTo(position) < 0) { + findEntryComplete(managedLedger.getLastConfirmedEntry(), null); + } else { + findEntryComplete(position, null); + } + } + } + + @Override public boolean expireMessages(Position messagePosition) { // If it's beyond last position of this topic, do nothing. - if (((PositionImpl) subscription.getTopic().getLastPosition()).compareTo((PositionImpl) messagePosition) < 0) { + Position topicLastPosition = this.topic.getLastPosition(); + if (topicLastPosition.compareTo(messagePosition) < 0) { if (log.isDebugEnabled()) { log.debug("[{}][{}] Ignore expire-message scheduled task, given position {} is beyond " - + "current topic's last position {}", topicName, subName, messagePosition, - subscription.getTopic().getLastPosition()); + + "current topic's last position {}", topicName, subName, messagePosition, + topicLastPosition); } return false; } @@ -113,7 +154,7 @@ public boolean expireMessages(Position messagePosition) { cursor.asyncFindNewestMatching(ManagedCursor.FindPositionConstraint.SearchActiveEntries, entry -> { try { // If given position larger than entry position. - return ((PositionImpl) entry.getPosition()).compareTo((PositionImpl) messagePosition) <= 0; + return entry.getPosition().compareTo(messagePosition) <= 0; } finally { entry.release(); } @@ -171,7 +212,8 @@ public void findEntryComplete(Position position, Object ctx) { if (position != null) { log.info("[{}][{}] Expiring all messages until position {}", topicName, subName, position); Position prevMarkDeletePos = cursor.getMarkDeletedPosition(); - cursor.asyncMarkDelete(position, markDeleteCallback, cursor.getNumberOfEntriesInBacklog(false)); + cursor.asyncMarkDelete(position, cursor.getProperties(), markDeleteCallback, + cursor.getNumberOfEntriesInBacklog(false)); if (!Objects.equals(cursor.getMarkDeletedPosition(), prevMarkDeletePos) && subscription != null) { subscription.updateLastMarkDeleteAdvancedTimestamp(); } @@ -189,24 +231,25 @@ public void findEntryFailed(ManagedLedgerException exception, Optional if (log.isDebugEnabled()) { log.debug("[{}][{}] Finding expired entry operation failed", topicName, subName, exception); } - if (autoSkipNonRecoverableData && failedReadPosition.isPresent() + if (isAutoSkipNonRecoverableData() && failedReadPosition.isPresent() && (exception instanceof NonRecoverableLedgerException)) { log.warn("[{}][{}] read failed from ledger at position:{} : {}", topicName, subName, failedReadPosition, exception.getMessage()); if (exception instanceof LedgerNotExistException) { long failedLedgerId = failedReadPosition.get().getLedgerId(); - ManagedLedgerImpl ledger = ((ManagedLedgerImpl) cursor.getManagedLedger()); + ManagedLedger ledger = cursor.getManagedLedger(); Position lastPositionInLedger = ledger.getOptionalLedgerInfo(failedLedgerId) - .map(ledgerInfo -> PositionImpl.get(failedLedgerId, ledgerInfo.getEntries() - 1)) + .map(ledgerInfo -> PositionFactory.create(failedLedgerId, ledgerInfo.getEntries() - 1)) .orElseGet(() -> { - Long nextExistingLedger = ledger.getNextValidLedger(failedReadPosition.get().getLedgerId()); + Long nextExistingLedger = + ledger.getLedgersInfo().ceilingKey(failedReadPosition.get().getLedgerId() + 1); if (nextExistingLedger == null) { log.info("[{}] [{}] Couldn't find next next valid ledger for expiry monitor when find " + "entry failed {}", ledger.getName(), ledger.getName(), failedReadPosition); - return (PositionImpl) failedReadPosition.get(); + return failedReadPosition.get(); } else { - return PositionImpl.get(nextExistingLedger, -1); + return PositionFactory.create(nextExistingLedger, -1); } }); log.info("[{}][{}] ledger not existed, will complete the last position of the non-existed" diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageFinder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageFinder.java index d2e6f6f5ff869..08273155e4cfa 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageFinder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageFinder.java @@ -71,7 +71,7 @@ public void findMessages(final long timestamp, AsyncCallbacks.FindEntryCallback entry.release(); } return false; - }, this, callback); + }, this, callback, true); } else { if (log.isDebugEnabled()) { log.debug("[{}][{}] Ignore message position find scheduled task, last find is still running", topicName, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index d882cbf56b2e8..b3d7546beed81 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -18,18 +18,25 @@ */ package org.apache.pulsar.broker.service.persistent; +import static org.apache.pulsar.broker.service.AbstractReplicator.State.Started; +import static org.apache.pulsar.broker.service.AbstractReplicator.State.Starting; +import static org.apache.pulsar.broker.service.AbstractReplicator.State.Terminated; +import static org.apache.pulsar.broker.service.AbstractReplicator.State.Terminating; import static org.apache.pulsar.broker.service.persistent.PersistentTopic.MESSAGE_RATE_BACKOFF_MS; import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.ByteBuf; import io.netty.util.Recycler; import io.netty.util.Recycler.Handle; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.Getter; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.AsyncCallbacks.ClearBacklogCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCallback; @@ -42,18 +49,18 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.CursorAlreadyClosedException; import org.apache.bookkeeper.mledger.ManagedLedgerException.TooManyRequestsException; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.service.AbstractReplicator; import org.apache.pulsar.broker.service.BrokerService; -import org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException; +import org.apache.pulsar.broker.service.MessageExpirer; import org.apache.pulsar.broker.service.Replicator; import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter.Type; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.client.impl.MessageImpl; +import org.apache.pulsar.client.impl.OpSendMsgStats; import org.apache.pulsar.client.impl.ProducerImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.SendCallback; @@ -61,12 +68,13 @@ import org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.stats.Rate; +import org.apache.pulsar.common.util.Backoff; import org.apache.pulsar.common.util.Codec; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public abstract class PersistentReplicator extends AbstractReplicator - implements Replicator, ReadEntriesCallback, DeleteCallback { + implements Replicator, ReadEntriesCallback, DeleteCallback, MessageExpirer { protected final PersistentTopic topic; protected final ManagedCursor cursor; @@ -104,7 +112,8 @@ public abstract class PersistentReplicator extends AbstractReplicator // for connected subscriptions, message expiry will be checked if the backlog is greater than this threshold private static final int MINIMUM_BACKLOG_FOR_EXPIRY_CHECK = 1000; - private final ReplicatorStatsImpl stats = new ReplicatorStatsImpl(); + @Getter + protected final ReplicatorStatsImpl stats = new ReplicatorStatsImpl(); protected volatile boolean fetchSchemaInProgress = false; @@ -112,11 +121,11 @@ public PersistentReplicator(String localCluster, PersistentTopic localTopic, Man String remoteCluster, String remoteTopic, BrokerService brokerService, PulsarClientImpl replicationClient) throws PulsarServerException { - super(localCluster, localTopic.getName(), remoteCluster, remoteTopic, localTopic.getReplicatorPrefix(), + super(localCluster, localTopic, remoteCluster, remoteTopic, localTopic.getReplicatorPrefix(), brokerService, replicationClient); this.topic = localTopic; - this.cursor = cursor; - this.expiryMonitor = new PersistentMessageExpiryMonitor(localTopicName, + this.cursor = Objects.requireNonNull(cursor); + this.expiryMonitor = new PersistentMessageExpiryMonitor(localTopic, Codec.decode(cursor.getName()), cursor, null); HAVE_PENDING_READ_UPDATER.set(this, FALSE); PENDING_MESSAGES_UPDATER.set(this, 0); @@ -133,30 +142,50 @@ public PersistentReplicator(String localCluster, PersistentTopic localTopic, Man } @Override - protected void readEntries(Producer producer) { - // Rewind the cursor to be sure to read again all non-acked messages sent while restarting + protected void setProducerAndTriggerReadEntries(Producer producer) { + // Rewind the cursor to be sure to read again all non-acked messages sent while restarting. cursor.rewind(); - cursor.cancelPendingReadRequest(); - HAVE_PENDING_READ_UPDATER.set(this, FALSE); - this.producer = (ProducerImpl) producer; - if (STATE_UPDATER.compareAndSet(this, State.Starting, State.Started)) { - log.info("[{}] Created replicator producer", replicatorId); + /** + * 1. Try change state to {@link Started}. + * 2. Atoms modify multiple properties if change state success, to avoid another thread get a null value + * producer when the state is {@link Started}. + */ + Pair changeStateRes; + changeStateRes = compareSetAndGetState(Starting, Started); + if (changeStateRes.getLeft()) { + if (!(producer instanceof ProducerImpl)) { + log.error("[{}] The partitions count between two clusters is not the same, the replicator can not be" + + " created successfully: {}", replicatorId, state); + doCloseProducerAsync(producer, () -> {}); + throw new ClassCastException(producer.getClass().getName() + " can not be cast to ProducerImpl"); + } + this.producer = (ProducerImpl) producer; + HAVE_PENDING_READ_UPDATER.set(this, FALSE); + // Trigger a new read. + log.info("[{}] Created replicator producer, Replicator state: {}", replicatorId, state); backOff.reset(); - // activate cursor: so, entries can be cached + // activate cursor: so, entries can be cached. this.cursor.setActive(); // read entries readMoreEntries(); } else { - log.info( - "[{}] Replicator was stopped while creating the producer." - + " Closing it. Replicator state: {}", - replicatorId, STATE_UPDATER.get(this)); - STATE_UPDATER.set(this, State.Stopping); - closeProducerAsync(); + if (changeStateRes.getRight() == Started) { + // Since only one task can call "producerBuilder.createAsync()", this scenario is not expected. + // So print a warn log. + log.warn("[{}] Replicator was already started by another thread while creating the producer." + + " Closing the producer newly created. Replicator state: {}", replicatorId, state); + } else if (changeStateRes.getRight() == Terminating || changeStateRes.getRight() == Terminated) { + log.info("[{}] Replicator was terminated, so close the producer. Replicator state: {}", + replicatorId, state); + } else { + log.error("[{}] Replicator state is not expected, so close the producer. Replicator state: {}", + replicatorId, changeStateRes.getRight()); + } + // Close the producer if change the state fail. + doCloseProducerAsync(producer, () -> {}); } - } @Override @@ -169,23 +198,41 @@ public long getNumberOfEntriesInBacklog() { return cursor.getNumberOfEntriesInBacklog(true); } + public long getMessageExpiredCount() { + return expiryMonitor.getTotalMessageExpired(); + } + @Override protected void disableReplicatorRead() { - if (this.cursor != null) { - // deactivate cursor after successfully close the producer - this.cursor.setInactive(); + // deactivate cursor after successfully close the producer + this.cursor.setInactive(); + } + + @Data + @AllArgsConstructor + private static class AvailablePermits { + private int messages; + private long bytes; + + /** + * messages, bytes + * 0, O: Producer queue is full, no permits. + * -1, -1: Rate Limiter reaches limit. + * >0, >0: available permits for read entries. + */ + public boolean isExceeded() { + return messages == -1 && bytes == -1; + } + + public boolean isReadable() { + return messages > 0 && bytes > 0; } } /** * Calculate available permits for read entries. - * - * @return - * 0: Producer queue is full, no permits. - * -1: Rate Limiter reaches limit. - * >0: available permits for read entries. */ - private int getAvailablePermits() { + private AvailablePermits getAvailablePermits() { int availablePermits = producerQueueSize - PENDING_MESSAGES_UPDATER.get(this); // return 0, if Producer queue is full, it will pause read entries. @@ -194,14 +241,20 @@ private int getAvailablePermits() { log.debug("[{}] Producer queue is full, availablePermits: {}, pause reading", replicatorId, availablePermits); } - return 0; + return new AvailablePermits(0, 0); } + long availablePermitsOnMsg = -1; + long availablePermitsOnByte = -1; + // handle rate limit if (dispatchRateLimiter.isPresent() && dispatchRateLimiter.get().isDispatchRateLimitingEnabled()) { DispatchRateLimiter rateLimiter = dispatchRateLimiter.get(); + // if dispatch-rate is in msg then read only msg according to available permit + availablePermitsOnMsg = rateLimiter.getAvailableDispatchRateLimitOnMsg(); + availablePermitsOnByte = rateLimiter.getAvailableDispatchRateLimitOnByte(); // no permits from rate limit - if (!rateLimiter.hasMessageDispatchPermit()) { + if (availablePermitsOnByte == 0 || availablePermitsOnMsg == 0) { if (log.isDebugEnabled()) { log.debug("[{}] message-read exceeded topic replicator message-rate {}/{}," + " schedule after a {}", @@ -210,17 +263,18 @@ private int getAvailablePermits() { rateLimiter.getDispatchRateOnByte(), MESSAGE_RATE_BACKOFF_MS); } - return -1; - } - - // if dispatch-rate is in msg then read only msg according to available permit - long availablePermitsOnMsg = rateLimiter.getAvailableDispatchRateLimitOnMsg(); - if (availablePermitsOnMsg > 0) { - availablePermits = Math.min(availablePermits, (int) availablePermitsOnMsg); + return new AvailablePermits(-1, -1); } } - return availablePermits; + availablePermitsOnMsg = + availablePermitsOnMsg == -1 ? availablePermits : Math.min(availablePermits, availablePermitsOnMsg); + availablePermitsOnMsg = Math.min(availablePermitsOnMsg, readBatchSize); + + availablePermitsOnByte = + availablePermitsOnByte == -1 ? readMaxSizeBytes : Math.min(readMaxSizeBytes, availablePermitsOnByte); + + return new AvailablePermits((int) availablePermitsOnMsg, availablePermitsOnByte); } protected void readMoreEntries() { @@ -228,10 +282,10 @@ protected void readMoreEntries() { log.info("[{}] Skip the reading due to new detected schema", replicatorId); return; } - int availablePermits = getAvailablePermits(); - - if (availablePermits > 0) { - int messagesToRead = Math.min(availablePermits, readBatchSize); + AvailablePermits availablePermits = getAvailablePermits(); + if (availablePermits.isReadable()) { + int messagesToRead = availablePermits.getMessages(); + long bytesToRead = availablePermits.getBytes(); if (!isWritable()) { if (log.isDebugEnabled()) { log.debug("[{}] Throttling replication traffic because producer is not writable", replicatorId); @@ -240,23 +294,21 @@ protected void readMoreEntries() { messagesToRead = 1; } - // If messagesToRead is 0 or less, correct it to 1 to prevent IllegalArgumentException - messagesToRead = Math.max(messagesToRead, 1); - // Schedule read if (HAVE_PENDING_READ_UPDATER.compareAndSet(this, FALSE, TRUE)) { if (log.isDebugEnabled()) { - log.debug("[{}] Schedule read of {} messages", replicatorId, messagesToRead); + log.debug("[{}] Schedule read of {} messages or {} bytes", replicatorId, messagesToRead, + bytesToRead); } - cursor.asyncReadEntriesOrWait(messagesToRead, readMaxSizeBytes, this, + cursor.asyncReadEntriesOrWait(messagesToRead, bytesToRead, this, null, topic.getMaxReadPosition()); } else { if (log.isDebugEnabled()) { - log.debug("[{}] Not scheduling read due to pending read. Messages To Read {}", - replicatorId, messagesToRead); + log.debug("[{}] Not scheduling read due to pending read. Messages To Read {}, Bytes To Read {}", + replicatorId, messagesToRead, bytesToRead); } } - } else if (availablePermits == -1) { + } else if (availablePermits.isExceeded()) { // no permits from rate limit topic.getBrokerService().executor().schedule( () -> readMoreEntries(), MESSAGE_RATE_BACKOFF_MS, TimeUnit.MILLISECONDS); @@ -313,12 +365,10 @@ protected CompletableFuture getSchemaInfo(MessageImpl msg) throws Ex } public void updateCursorState() { - if (this.cursor != null) { - if (producer != null && producer.isConnected()) { - this.cursor.setActive(); - } else { - this.cursor.setInactive(); - } + if (isConnected()) { + cursor.setActive(); + } else { + cursor.setInactive(); } } @@ -328,7 +378,7 @@ protected static final class ProducerSendCallback implements SendCallback { private MessageImpl msg; @Override - public void sendComplete(Exception exception) { + public void sendComplete(Throwable exception, OpSendMsgStats opSendMsgStats) { if (exception != null && !(exception instanceof PulsarClientException.InvalidMessageException)) { log.error("[{}] Error producing on remote broker", replicator.replicatorId, exception); // cursor should be rewinded since it was incremented when readMoreEntries @@ -419,8 +469,8 @@ public CompletableFuture getFuture() { @Override public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { - if (STATE_UPDATER.get(this) != State.Started) { - log.info("[{}] Replicator was stopped while reading entries." + if (state != Started) { + log.info("[{}] Replicator was disconnected while reading entries." + " Stop reading. Replicator state: {}", replicatorId, STATE_UPDATER.get(this)); return; @@ -432,11 +482,11 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { long waitTimeMillis = readFailureBackoff.next(); if (exception instanceof CursorAlreadyClosedException) { - log.error("[{}] Error reading entries because replicator is" + log.warn("[{}] Error reading entries because replicator is" + " already deleted and cursor is already closed {}, ({})", replicatorId, ctx, exception.getMessage(), exception); - // replicator is already deleted and cursor is already closed so, producer should also be stopped - closeProducerAsync(); + // replicator is already deleted and cursor is already closed so, producer should also be disconnected. + terminate(); return; } else if (!(exception instanceof TooManyRequestsException)) { log.error("[{}] Error reading entries at {}. Retrying to read in {}s. ({})", @@ -529,6 +579,12 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { public void readEntryComplete(Entry entry, Object ctx) { future.complete(entry); } + + @Override + public String toString() { + return String.format("Replication [%s] peek Nth message", + PersistentReplicator.this.producer.getProducerName()); + } }, null); return future; @@ -546,17 +602,17 @@ public void deleteFailed(ManagedLedgerException exception, Object ctx) { log.error("[{}] Failed to delete message at {}: {}", replicatorId, ctx, exception.getMessage(), exception); if (exception instanceof CursorAlreadyClosedException) { - log.error("[{}] Asynchronous ack failure because replicator is already deleted and cursor is already" + log.warn("[{}] Asynchronous ack failure because replicator is already deleted and cursor is already" + " closed {}, ({})", replicatorId, ctx, exception.getMessage(), exception); - // replicator is already deleted and cursor is already closed so, producer should also be stopped - closeProducerAsync(); + // replicator is already deleted and cursor is already closed so, producer should also be disconnected. + terminate(); return; } - if (ctx instanceof PositionImpl) { - PositionImpl deletedEntry = (PositionImpl) ctx; - if (deletedEntry.compareTo((PositionImpl) cursor.getMarkDeletedPosition()) > 0) { + if (ctx instanceof Position) { + Position deletedEntry = (Position) ctx; + if (deletedEntry.compareTo(cursor.getMarkDeletedPosition()) > 0) { brokerService.getPulsar().getExecutor().schedule( - () -> cursor.asyncDelete(deletedEntry, (PersistentReplicator) this, deletedEntry), 10, + () -> cursor.asyncDelete(deletedEntry, this, deletedEntry), 10, TimeUnit.SECONDS); } } @@ -572,10 +628,10 @@ public void updateRates() { stats.msgRateExpired = msgExpired.getRate() + expiryMonitor.getMessageExpiryRate(); } - public ReplicatorStatsImpl getStats() { - stats.replicationBacklog = cursor != null ? cursor.getNumberOfEntriesInBacklog(false) : 0; - stats.connected = producer != null && producer.isConnected(); - stats.replicationDelayInSeconds = getReplicationDelayInSeconds(); + public ReplicatorStatsImpl computeStats() { + stats.replicationBacklog = cursor.getNumberOfEntriesInBacklog(false); + stats.connected = isConnected(); + stats.replicationDelayInSeconds = TimeUnit.MILLISECONDS.toSeconds(getReplicationDelayMs()); ProducerImpl producer = this.producer; if (producer != null) { @@ -593,13 +649,7 @@ public void updateMessageTTL(int messageTTLInSeconds) { this.messageTTLInSeconds = messageTTLInSeconds; } - private long getReplicationDelayInSeconds() { - if (producer != null) { - return TimeUnit.MILLISECONDS.toSeconds(producer.getDelayInMillis()); - } - return 0L; - } - + @Override public boolean expireMessages(int messageTTLInSeconds) { if ((cursor.getNumberOfEntriesInBacklog(false) == 0) || (cursor.getNumberOfEntriesInBacklog(false) < MINIMUM_BACKLOG_FOR_EXPIRY_CHECK @@ -611,6 +661,7 @@ public boolean expireMessages(int messageTTLInSeconds) { return expiryMonitor.expireMessages(messageTTLInSeconds); } + @Override public boolean expireMessages(Position position) { return expiryMonitor.expireMessages(position); } @@ -624,8 +675,9 @@ public Optional getRateLimiter() { public void initializeDispatchRateLimiterIfNeeded() { synchronized (dispatchRateLimiterLock) { if (!dispatchRateLimiter.isPresent() - && DispatchRateLimiter.isDispatchRateEnabled(topic.getReplicatorDispatchRate())) { - this.dispatchRateLimiter = Optional.of(new DispatchRateLimiter(topic, Type.REPLICATOR)); + && DispatchRateLimiter.isDispatchRateEnabled(topic.getReplicatorDispatchRate())) { + this.dispatchRateLimiter = Optional.of( + new DispatchRateLimiter(topic, Codec.decode(cursor.getName()), Type.REPLICATOR)); } } } @@ -666,33 +718,8 @@ protected void checkReplicatedSubscriptionMarker(Position position, MessageImpl< } @Override - public CompletableFuture disconnect() { - return disconnect(false); - } - - @Override - public synchronized CompletableFuture disconnect(boolean failIfHasBacklog) { - final CompletableFuture future = new CompletableFuture<>(); - - super.disconnect(failIfHasBacklog).thenRun(() -> { - dispatchRateLimiter.ifPresent(DispatchRateLimiter::close); - future.complete(null); - }).exceptionally(ex -> { - Throwable t = (ex instanceof CompletionException ? ex.getCause() : ex); - if (!(t instanceof TopicBusyException)) { - log.error("[{}] Failed to close dispatch rate limiter: {}", replicatorId, ex.getMessage()); - } - future.completeExceptionally(t); - return null; - }); - - return future; - } - - @Override - public boolean isConnected() { - ProducerImpl producer = this.producer; - return producer != null && producer.isConnected(); + protected void doReleaseResources() { + dispatchRateLimiter.ifPresent(DispatchRateLimiter::close); } private static final Logger log = LoggerFactory.getLogger(PersistentReplicator.class); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java index 8f05530f58bfa..ecd3f19a14028 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java @@ -19,24 +19,28 @@ package org.apache.pulsar.broker.service.persistent; import com.google.common.annotations.VisibleForTesting; -import io.netty.util.concurrent.FastThreadLocal; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.NavigableSet; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Predicate; +import javax.annotation.Nullable; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.mutable.MutableBoolean; +import org.apache.commons.lang3.mutable.MutableInt; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.ConsistentHashingStickyKeyConsumerSelector; @@ -53,6 +57,8 @@ import org.apache.pulsar.common.api.proto.KeySharedMeta; import org.apache.pulsar.common.api.proto.KeySharedMode; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.collections.ConcurrentOpenLongPairRangeSet; +import org.apache.pulsar.common.util.collections.LongPairRangeSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,8 +66,9 @@ public class PersistentStickyKeyDispatcherMultipleConsumers extends PersistentDi private final boolean allowOutOfOrderDelivery; private final StickyKeyConsumerSelector selector; + private final boolean recentlyJoinedConsumerTrackingRequired; - private boolean isDispatcherStuckOnReplays = false; + private boolean skipNextReplayToTriggerLookAhead = false; private final KeySharedMode keySharedMode; /** @@ -69,15 +76,30 @@ public class PersistentStickyKeyDispatcherMultipleConsumers extends PersistentDi * This means that, in order to preserve ordering, new consumers can only receive old * messages, until the mark-delete position will move past this point. */ - private final LinkedHashMap recentlyJoinedConsumers; + private final LinkedHashMap recentlyJoinedConsumers; + + /** + * The lastSentPosition and the individuallySentPositions are not thread safe. + */ + @Nullable + private Position lastSentPosition; + private final LongPairRangeSet individuallySentPositions; + private static final LongPairRangeSet.LongPairConsumer positionRangeConverter = PositionFactory::create; PersistentStickyKeyDispatcherMultipleConsumers(PersistentTopic topic, ManagedCursor cursor, Subscription subscription, ServiceConfiguration conf, KeySharedMeta ksm) { super(topic, cursor, subscription, ksm.isAllowOutOfOrderDelivery()); this.allowOutOfOrderDelivery = ksm.isAllowOutOfOrderDelivery(); - this.recentlyJoinedConsumers = allowOutOfOrderDelivery ? null : new LinkedHashMap<>(); this.keySharedMode = ksm.getKeySharedMode(); + // recent joined consumer tracking is required only for AUTO_SPLIT mode when out-of-order delivery is disabled + this.recentlyJoinedConsumerTrackingRequired = + keySharedMode == KeySharedMode.AUTO_SPLIT && !allowOutOfOrderDelivery; + this.recentlyJoinedConsumers = recentlyJoinedConsumerTrackingRequired ? new LinkedHashMap<>() : null; + this.individuallySentPositions = + recentlyJoinedConsumerTrackingRequired + ? new ConcurrentOpenLongPairRangeSet<>(4096, positionRangeConverter) + : null; switch (this.keySharedMode) { case AUTO_SPLIT: if (conf.isSubscriptionKeySharedUseConsistentHashing()) { @@ -122,15 +144,18 @@ public synchronized CompletableFuture addConsumer(Consumer consumer) { }) ).thenRun(() -> { synchronized (PersistentStickyKeyDispatcherMultipleConsumers.this) { - PositionImpl readPositionWhenJoining = (PositionImpl) cursor.getReadPosition(); - consumer.setReadPositionWhenJoining(readPositionWhenJoining); - // If this was the 1st consumer, or if all the messages are already acked, then we - // don't need to do anything special - if (!allowOutOfOrderDelivery - && recentlyJoinedConsumers != null - && consumerList.size() > 1 - && cursor.getNumberOfEntriesSinceFirstNotAckedMessage() > 1) { - recentlyJoinedConsumers.put(consumer, readPositionWhenJoining); + if (recentlyJoinedConsumerTrackingRequired) { + final Position lastSentPositionWhenJoining = updateIfNeededAndGetLastSentPosition(); + if (lastSentPositionWhenJoining != null) { + consumer.setLastSentPositionWhenJoining(lastSentPositionWhenJoining); + // If this was the 1st consumer, or if all the messages are already acked, then we + // don't need to do anything special + if (recentlyJoinedConsumers != null + && consumerList.size() > 1 + && cursor.getNumberOfEntriesSinceFirstNotAckedMessage() > 1) { + recentlyJoinedConsumers.put(consumer, lastSentPositionWhenJoining); + } + } } } }); @@ -146,10 +171,16 @@ public synchronized void removeConsumer(Consumer consumer) throws BrokerServiceE // eventually causing all consumers to get stuck. selector.removeConsumer(consumer); super.removeConsumer(consumer); - if (recentlyJoinedConsumers != null) { + if (recentlyJoinedConsumerTrackingRequired) { recentlyJoinedConsumers.remove(consumer); if (consumerList.size() == 1) { recentlyJoinedConsumers.clear(); + } else if (consumerList.isEmpty()) { + // The subscription removes consumers if rewind or reset cursor operations are called. + // The dispatcher must clear lastSentPosition and individuallySentPositions because + // these operations trigger re-sending messages. + lastSentPosition = null; + individuallySentPositions.clear(); } if (removeConsumersFromRecentJoinedConsumers() || !redeliveryMessages.isEmpty()) { readMoreEntries(); @@ -157,19 +188,13 @@ public synchronized void removeConsumer(Consumer consumer) throws BrokerServiceE } } - private static final FastThreadLocal>> localGroupedEntries = - new FastThreadLocal>>() { - @Override - protected Map> initialValue() throws Exception { - return new HashMap<>(); - } - }; - @Override protected synchronized boolean trySendMessagesToConsumers(ReadType readType, List entries) { + lastNumberOfEntriesProcessed = 0; long totalMessagesSent = 0; long totalBytesSent = 0; long totalEntries = 0; + long totalEntriesProcessed = 0; int entriesCount = entries.size(); // Trigger read more messages @@ -183,18 +208,12 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis return false; } - // A corner case that we have to retry a readMoreEntries in order to preserver order delivery. - // This may happen when consumer closed. See issue #12885 for details. if (!allowOutOfOrderDelivery) { - NavigableSet messagesToReplayNow = this.getMessagesToReplayNow(1); - if (messagesToReplayNow != null && !messagesToReplayNow.isEmpty()) { - PositionImpl replayPosition = messagesToReplayNow.first(); - - // We have received a message potentially from the delayed tracker and, since we're not using it - // right now, it needs to be added to the redelivery tracker or we won't attempt anymore to - // resend it (until we disconnect consumer). - redeliveryMessages.add(replayPosition.getLedgerId(), replayPosition.getEntryId()); - + // A corner case that we have to retry a readMoreEntries in order to preserver order delivery. + // This may happen when consumer closed. See issue #12885 for details. + Optional firstReplayPosition = getFirstPositionInReplay(); + if (firstReplayPosition.isPresent()) { + Position replayPosition = firstReplayPosition.get(); if (this.minReplayedPosition != null) { // If relayPosition is a new entry wither smaller position is inserted for redelivery during this // async read, it is possible that this relayPosition should dispatch to consumer first. So in @@ -215,144 +234,400 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis } else if (readType == ReadType.Replay) { entries.forEach(Entry::release); } + skipNextBackoff = true; return true; } } } } - final Map> groupedEntries = localGroupedEntries.get(); - groupedEntries.clear(); - final Map> consumerStickyKeyHashesMap = new HashMap<>(); - - for (Entry entry : entries) { - int stickyKeyHash = getStickyKeyHash(entry); - Consumer c = selector.select(stickyKeyHash); - if (c != null) { - groupedEntries.computeIfAbsent(c, k -> new ArrayList<>()).add(entry); - consumerStickyKeyHashesMap.computeIfAbsent(c, k -> new HashSet<>()).add(stickyKeyHash); - } else { - addMessageToReplay(entry.getLedgerId(), entry.getEntryId(), stickyKeyHash); - entry.release(); + if (recentlyJoinedConsumerTrackingRequired) { + // Update if the markDeletePosition move forward + updateIfNeededAndGetLastSentPosition(); + + // Should not access to individualDeletedMessages from outside managed cursor + // because it doesn't guarantee thread safety. + if (lastSentPosition == null) { + if (cursor.getMarkDeletedPosition() != null) { + lastSentPosition = ((ManagedCursorImpl) cursor) + .processIndividuallyDeletedMessagesAndGetMarkDeletedPosition(range -> { + final Position lower = range.lowerEndpoint(); + final Position upper = range.upperEndpoint(); + individuallySentPositions.addOpenClosed(lower.getLedgerId(), lower.getEntryId(), + upper.getLedgerId(), upper.getEntryId()); + return true; + }); + } } } - AtomicInteger keyNumbers = new AtomicInteger(groupedEntries.size()); + // returns a boolean indicating whether look-ahead could be useful, when there's a consumer + // with available permits, and it's not able to make progress because of blocked hashes. + MutableBoolean triggerLookAhead = new MutableBoolean(); + // filter and group the entries by consumer for dispatching + final Map> entriesByConsumerForDispatching = + filterAndGroupEntriesForDispatching(entries, readType, triggerLookAhead); - int currentThreadKeyNumber = groupedEntries.size(); - if (currentThreadKeyNumber == 0) { - currentThreadKeyNumber = -1; - } - for (Map.Entry> current : groupedEntries.entrySet()) { + AtomicInteger remainingConsumersToFinishSending = new AtomicInteger(entriesByConsumerForDispatching.size()); + for (Map.Entry> current : entriesByConsumerForDispatching.entrySet()) { Consumer consumer = current.getKey(); - assert consumer != null; // checked when added to groupedEntries - List entriesWithSameKey = current.getValue(); - int entriesWithSameKeyCount = entriesWithSameKey.size(); - int availablePermits = Math.max(consumer.getAvailablePermits(), 0); - if (consumer.getMaxUnackedMessages() > 0) { - int remainUnAckedMessages = - // Avoid negative number - Math.max(consumer.getMaxUnackedMessages() - consumer.getUnackedMessages(), 0); - availablePermits = Math.min(availablePermits, remainUnAckedMessages); - } - int maxMessagesForC = Math.min(entriesWithSameKeyCount, availablePermits); - int messagesForC = getRestrictedMaxEntriesForConsumer(consumer, entriesWithSameKey, maxMessagesForC, - readType, consumerStickyKeyHashesMap.get(consumer)); + List entriesForConsumer = current.getValue(); if (log.isDebugEnabled()) { log.debug("[{}] select consumer {} with messages num {}, read type is {}", - name, consumer.consumerName(), messagesForC, readType); + name, consumer.consumerName(), entriesForConsumer.size(), readType); } - - if (messagesForC < entriesWithSameKeyCount) { - // We are not able to push all the messages with given key to its consumer, - // so we discard for now and mark them for later redelivery - for (int i = messagesForC; i < entriesWithSameKeyCount; i++) { - Entry entry = entriesWithSameKey.get(i); - long stickyKeyHash = getStickyKeyHash(entry); - addMessageToReplay(entry.getLedgerId(), entry.getEntryId(), stickyKeyHash); - entry.release(); - entriesWithSameKey.set(i, null); - } - } - - if (messagesForC > 0) { + final ManagedLedger managedLedger = cursor.getManagedLedger(); + for (Entry entry : entriesForConsumer) { // remove positions first from replay list first : sendMessages recycles entries if (readType == ReadType.Replay) { - for (int i = 0; i < messagesForC; i++) { - Entry entry = entriesWithSameKey.get(i); - redeliveryMessages.remove(entry.getLedgerId(), entry.getEntryId()); + redeliveryMessages.remove(entry.getLedgerId(), entry.getEntryId()); + } + // Add positions to individuallySentPositions if necessary + if (recentlyJoinedConsumerTrackingRequired) { + final Position position = entry.getPosition(); + // Store to individuallySentPositions even if lastSentPosition is null + if ((lastSentPosition == null || position.compareTo(lastSentPosition) > 0) + && !individuallySentPositions.contains(position.getLedgerId(), position.getEntryId())) { + final Position previousPosition = managedLedger.getPreviousPosition(position); + individuallySentPositions.addOpenClosed(previousPosition.getLedgerId(), + previousPosition.getEntryId(), position.getLedgerId(), position.getEntryId()); } } + } + + SendMessageInfo sendMessageInfo = SendMessageInfo.getThreadLocal(); + EntryBatchSizes batchSizes = EntryBatchSizes.get(entriesForConsumer.size()); + EntryBatchIndexesAcks batchIndexesAcks = EntryBatchIndexesAcks.get(entriesForConsumer.size()); + totalEntries += filterEntriesForConsumer(entriesForConsumer, batchSizes, sendMessageInfo, + batchIndexesAcks, cursor, readType == ReadType.Replay, consumer); + totalEntriesProcessed += entriesForConsumer.size(); + consumer.sendMessages(entriesForConsumer, batchSizes, batchIndexesAcks, + sendMessageInfo.getTotalMessages(), + sendMessageInfo.getTotalBytes(), sendMessageInfo.getTotalChunkedMessages(), + getRedeliveryTracker()).addListener(future -> { + if (future.isDone() && remainingConsumersToFinishSending.decrementAndGet() == 0) { + readMoreEntries(); + } + }); + + TOTAL_AVAILABLE_PERMITS_UPDATER.getAndAdd(this, + -(sendMessageInfo.getTotalMessages() - batchIndexesAcks.getTotalAckedIndexCount())); + totalMessagesSent += sendMessageInfo.getTotalMessages(); + totalBytesSent += sendMessageInfo.getTotalBytes(); + } + + // Update the last sent position and remove ranges from individuallySentPositions if necessary + if (recentlyJoinedConsumerTrackingRequired && lastSentPosition != null) { + final ManagedLedger managedLedger = cursor.getManagedLedger(); + com.google.common.collect.Range range = individuallySentPositions.firstRange(); + + // If the upper bound is before the last sent position, we need to move ahead as these + // individuallySentPositions are now irrelevant. + if (range != null && range.upperEndpoint().compareTo(lastSentPosition) <= 0) { + individuallySentPositions.removeAtMost(lastSentPosition.getLedgerId(), + lastSentPosition.getEntryId()); + range = individuallySentPositions.firstRange(); + } - SendMessageInfo sendMessageInfo = SendMessageInfo.getThreadLocal(); - EntryBatchSizes batchSizes = EntryBatchSizes.get(messagesForC); - EntryBatchIndexesAcks batchIndexesAcks = EntryBatchIndexesAcks.get(messagesForC); - totalEntries += filterEntriesForConsumer(entriesWithSameKey, batchSizes, sendMessageInfo, - batchIndexesAcks, cursor, readType == ReadType.Replay, consumer); - - consumer.sendMessages(entriesWithSameKey, batchSizes, batchIndexesAcks, - sendMessageInfo.getTotalMessages(), - sendMessageInfo.getTotalBytes(), sendMessageInfo.getTotalChunkedMessages(), - getRedeliveryTracker()).addListener(future -> { - if (future.isDone() && keyNumbers.decrementAndGet() == 0) { - readMoreEntries(); + if (range != null) { + // If the lowerBound is ahead of the last sent position, + // verify if there are any entries in-between. + if (range.lowerEndpoint().compareTo(lastSentPosition) <= 0 || managedLedger + .getNumberOfEntries(com.google.common.collect.Range.openClosed(lastSentPosition, + range.lowerEndpoint())) <= 0) { + if (log.isDebugEnabled()) { + log.debug("[{}] Found a position range to last sent: {}", name, range); + } + Position newLastSentPosition = range.upperEndpoint(); + Position positionAfterNewLastSent = managedLedger + .getNextValidPosition(newLastSentPosition); + // sometime ranges are connected but belongs to different ledgers + // so, they are placed sequentially + // eg: (2:10..3:15] can be returned as (2:10..2:15],[3:0..3:15]. + // So, try to iterate over connected range and found the last non-connected range + // which gives new last sent position. + final Position lastConfirmedEntrySnapshot = managedLedger.getLastConfirmedEntry(); + if (lastConfirmedEntrySnapshot != null) { + while (positionAfterNewLastSent.compareTo(lastConfirmedEntrySnapshot) <= 0) { + if (individuallySentPositions.contains(positionAfterNewLastSent.getLedgerId(), + positionAfterNewLastSent.getEntryId())) { + range = individuallySentPositions.rangeContaining( + positionAfterNewLastSent.getLedgerId(), positionAfterNewLastSent.getEntryId()); + newLastSentPosition = range.upperEndpoint(); + positionAfterNewLastSent = managedLedger.getNextValidPosition(newLastSentPosition); + // check if next valid position is also deleted and part of the deleted-range + continue; + } + break; + } } - }); - TOTAL_AVAILABLE_PERMITS_UPDATER.getAndAdd(this, - -(sendMessageInfo.getTotalMessages() - batchIndexesAcks.getTotalAckedIndexCount())); - totalMessagesSent += sendMessageInfo.getTotalMessages(); - totalBytesSent += sendMessageInfo.getTotalBytes(); - } else { - currentThreadKeyNumber = keyNumbers.decrementAndGet(); + if (lastSentPosition.compareTo(newLastSentPosition) < 0) { + lastSentPosition = newLastSentPosition; + } + individuallySentPositions.removeAtMost(lastSentPosition.getLedgerId(), + lastSentPosition.getEntryId()); + } } } + lastNumberOfEntriesProcessed = (int) totalEntriesProcessed; + // acquire message-dispatch permits for already delivered messages acquirePermitsForDeliveredMessages(topic, cursor, totalEntries, totalMessagesSent, totalBytesSent); - if (totalMessagesSent == 0 && (recentlyJoinedConsumers == null || recentlyJoinedConsumers.isEmpty())) { - // This means, that all the messages we've just read cannot be dispatched right now. - // This condition can only happen when: - // 1. We have consumers ready to accept messages (otherwise the would not haven been triggered) - // 2. All keys in the current set of messages are routing to consumers that are currently busy - // - // The solution here is to move on and read next batch of messages which might hopefully contain - // also keys meant for other consumers. - // - // We do it unless that are "recently joined consumers". In that case, we would be looking - // ahead in the stream while the new consumers are not ready to accept the new messages, - // therefore would be most likely only increase the distance between read-position and mark-delete - // position. - isDispatcherStuckOnReplays = true; + // trigger read more messages if necessary + if (triggerLookAhead.booleanValue()) { + // When all messages get filtered and no messages are sent, we should read more entries, "look ahead" + // so that a possible next batch of messages might contain messages that can be dispatched. + // This is done only when there's a consumer with available permits, and it's not able to make progress + // because of blocked hashes. Without this rule we would be looking ahead in the stream while the + // new consumers are not ready to accept the new messages, + // therefore would be most likely only increase the distance between read-position and mark-delete position. + skipNextReplayToTriggerLookAhead = true; + // skip backoff delay before reading ahead in the "look ahead" mode to prevent any additional latency + skipNextBackoff = true; return true; - } else if (currentThreadKeyNumber == 0) { + } + + // if no messages were sent to consumers, we should retry + if (totalEntries == 0) { return true; } + return false; } - private int getRestrictedMaxEntriesForConsumer(Consumer consumer, List entries, int maxMessages, - ReadType readType, Set stickyKeyHashes) { - if (maxMessages == 0) { - return 0; + private boolean isReplayQueueSizeBelowLimit() { + return redeliveryMessages.size() < getEffectiveLookAheadLimit(); + } + + private int getEffectiveLookAheadLimit() { + return getEffectiveLookAheadLimit(serviceConfig, consumerList.size()); + } + + static int getEffectiveLookAheadLimit(ServiceConfiguration serviceConfig, int consumerCount) { + int perConsumerLimit = serviceConfig.getKeySharedLookAheadMsgInReplayThresholdPerConsumer(); + int perSubscriptionLimit = serviceConfig.getKeySharedLookAheadMsgInReplayThresholdPerSubscription(); + int effectiveLimit; + if (perConsumerLimit <= 0) { + effectiveLimit = perSubscriptionLimit; + } else { + effectiveLimit = perConsumerLimit * consumerCount; + if (perSubscriptionLimit > 0 && perSubscriptionLimit < effectiveLimit) { + effectiveLimit = perSubscriptionLimit; + } } - if (readType == ReadType.Normal && stickyKeyHashes != null - && redeliveryMessages.containsStickyKeyHashes(stickyKeyHashes)) { - // If redeliveryMessages contains messages that correspond to the same hash as the messages - // that the dispatcher is trying to send, do not send those messages for order guarantee - return 0; + if (effectiveLimit <= 0) { + // use max unacked messages limits if key shared look-ahead limits are disabled + int maxUnackedMessagesPerSubscription = serviceConfig.getMaxUnackedMessagesPerSubscription(); + if (maxUnackedMessagesPerSubscription <= 0) { + maxUnackedMessagesPerSubscription = Integer.MAX_VALUE; + } + int maxUnackedMessagesByConsumers = consumerCount * serviceConfig.getMaxUnackedMessagesPerConsumer(); + if (maxUnackedMessagesByConsumers <= 0) { + maxUnackedMessagesByConsumers = Integer.MAX_VALUE; + } + effectiveLimit = Math.min(maxUnackedMessagesPerSubscription, maxUnackedMessagesByConsumers); + } + return effectiveLimit; + } + + // groups the entries by consumer and filters out the entries that should not be dispatched + // the entries are handled in the order they are received instead of first grouping them by consumer and + // then filtering them + private Map> filterAndGroupEntriesForDispatching(List entries, ReadType readType, + MutableBoolean triggerLookAhead) { + // entries grouped by consumer + Map> entriesGroupedByConsumer = new HashMap<>(); + // permits for consumer, permits are for entries/batches + Map permitsForConsumer = new HashMap<>(); + // maxLastSentPosition cache for consumers, used when recently joined consumers exist + boolean hasRecentlyJoinedConsumers = hasRecentlyJoinedConsumers(); + Map maxLastSentPositionCache = hasRecentlyJoinedConsumers ? new HashMap<>() : null; + boolean lookAheadAllowed = isReplayQueueSizeBelowLimit(); + // in normal read mode, keep track of consumers that are blocked by hash, to check if look-ahead could be useful + Set blockedByHashConsumers = lookAheadAllowed && readType == ReadType.Normal ? new HashSet<>() : null; + // in replay read mode, keep track of consumers for entries, used for look-ahead check + Set consumersForEntriesForLookaheadCheck = lookAheadAllowed ? new HashSet<>() : null; + + for (Entry entry : entries) { + int stickyKeyHash = getStickyKeyHash(entry); + Consumer consumer = selector.select(stickyKeyHash); + MutableBoolean blockedByHash = null; + boolean dispatchEntry = false; + if (consumer != null) { + if (lookAheadAllowed) { + consumersForEntriesForLookaheadCheck.add(consumer); + } + Position maxLastSentPosition = hasRecentlyJoinedConsumers ? maxLastSentPositionCache.computeIfAbsent( + consumer, __ -> resolveMaxLastSentPositionForRecentlyJoinedConsumer(consumer, readType)) : null; + blockedByHash = lookAheadAllowed && readType == ReadType.Normal ? new MutableBoolean(false) : null; + MutableInt permits = + permitsForConsumer.computeIfAbsent(consumer, + k -> new MutableInt(getAvailablePermits(consumer))); + // a consumer was found for the sticky key hash and the entry can be dispatched + if (permits.intValue() > 0 && canDispatchEntry(entry, readType, stickyKeyHash, + maxLastSentPosition, blockedByHash)) { + // decrement the permits for the consumer + permits.decrement(); + // allow the entry to be dispatched + dispatchEntry = true; + } + } + if (dispatchEntry) { + // add the entry to consumer's entry list for dispatching + List consumerEntries = + entriesGroupedByConsumer.computeIfAbsent(consumer, k -> new ArrayList<>()); + consumerEntries.add(entry); + } else { + if (blockedByHash != null && blockedByHash.isTrue()) { + // the entry is blocked by hash, add the consumer to the blocked set + blockedByHashConsumers.add(consumer); + } + // add the message to replay + addMessageToReplay(entry.getLedgerId(), entry.getEntryId(), stickyKeyHash); + // release the entry as it will not be dispatched + entry.release(); + } + } + // + // determine whether look-ahead could be useful for making more progress + // + if (lookAheadAllowed && entriesGroupedByConsumer.isEmpty()) { + // check if look-ahead could be useful for the consumers that are blocked by a hash that is in the replay + // queue. This check applies only to the normal read mode. + if (readType == ReadType.Normal) { + for (Consumer consumer : blockedByHashConsumers) { + // if the consumer isn't in the entriesGroupedByConsumer, it means that it won't receive any + // messages + // if it has available permits, then look-ahead could be useful for this particular consumer + // to make further progress + if (!entriesGroupedByConsumer.containsKey(consumer) + && permitsForConsumer.get(consumer).intValue() > 0) { + triggerLookAhead.setTrue(); + break; + } + } + } + // check if look-ahead could be useful for other consumers + if (!triggerLookAhead.booleanValue()) { + for (Consumer consumer : getConsumers()) { + // filter out the consumers that are already checked when the entries were processed for entries + if (!consumersForEntriesForLookaheadCheck.contains(consumer)) { + // if another consumer has available permits, then look-ahead could be useful + if (getAvailablePermits(consumer) > 0) { + triggerLookAhead.setTrue(); + break; + } + } + } + } + } + return entriesGroupedByConsumer; + } + + // checks if the entry can be dispatched to the consumer + private boolean canDispatchEntry(Entry entry, + ReadType readType, int stickyKeyHash, Position maxLastSentPosition, + MutableBoolean blockedByHash) { + // check if the entry can be replayed to a recently joined consumer + if (maxLastSentPosition != null && entry.getPosition().compareTo(maxLastSentPosition) > 0) { + return false; + } + + // If redeliveryMessages contains messages that correspond to the same hash as the entry to be dispatched + // do not send those messages for order guarantee + if (readType == ReadType.Normal && redeliveryMessages.containsStickyKeyHash(stickyKeyHash)) { + if (blockedByHash != null) { + blockedByHash.setTrue(); + } + return false; } + + return true; + } + + /** + * Creates a filter for replaying messages. The filter is stateful and shouldn't be cached or reused. + * @see PersistentDispatcherMultipleConsumers#createFilterForReplay() + */ + @Override + protected Predicate createFilterForReplay() { + return new ReplayPositionFilter(); + } + + /** + * Filter for replaying messages. The filter is stateful for a single invocation and shouldn't be cached, shared + * or reused. This is a short-lived object, and optimizing it for the "no garbage" coding style of Pulsar is + * unnecessary since the JVM can optimize allocations for short-lived objects. + */ + private class ReplayPositionFilter implements Predicate { + // tracks the available permits for each consumer for the duration of the filter usage + // the filter is stateful and shouldn't be shared or reused later + private final Map availablePermitsMap = new HashMap<>(); + private final Map maxLastSentPositionCache = + hasRecentlyJoinedConsumers() ? new HashMap<>() : null; + + @Override + public boolean test(Position position) { + // if out of order delivery is allowed, then any position will be replayed + if (isAllowOutOfOrderDelivery()) { + return true; + } + // lookup the sticky key hash for the entry at the replay position + Long stickyKeyHash = redeliveryMessages.getHash(position.getLedgerId(), position.getEntryId()); + if (stickyKeyHash == null) { + // the sticky key hash is missing for delayed messages, the filtering will happen at the time of + // dispatch after reading the entry from the ledger + if (log.isDebugEnabled()) { + log.debug("[{}] replay of entry at position {} doesn't contain sticky key hash.", name, position); + } + return true; + } + // find the consumer for the sticky key hash + Consumer consumer = selector.select(stickyKeyHash.intValue()); + // skip replaying the message position if there's no assigned consumer + if (consumer == null) { + return false; + } + // lookup the available permits for the consumer + MutableInt availablePermits = + availablePermitsMap.computeIfAbsent(consumer, + k -> new MutableInt(getAvailablePermits(consumer))); + // skip replaying the message position if the consumer has no available permits + if (availablePermits.intValue() <= 0) { + return false; + } + // check if the entry position can be replayed to a recently joined consumer + Position maxLastSentPosition = maxLastSentPositionCache != null + ? maxLastSentPositionCache.computeIfAbsent(consumer, __ -> + resolveMaxLastSentPositionForRecentlyJoinedConsumer(consumer, ReadType.Replay)) + : null; + if (maxLastSentPosition != null && position.compareTo(maxLastSentPosition) > 0) { + return false; + } + availablePermits.decrement(); + return true; + } + } + + /** + * Contains the logic to resolve the max last sent position for a consumer + * when the consumer has recently joined. This is only applicable for key shared mode when + * allowOutOfOrderDelivery=false. + */ + private Position resolveMaxLastSentPositionForRecentlyJoinedConsumer(Consumer consumer, ReadType readType) { if (recentlyJoinedConsumers == null) { - return maxMessages; + return null; } removeConsumersFromRecentJoinedConsumers(); - PositionImpl maxReadPosition = recentlyJoinedConsumers.get(consumer); + Position maxLastSentPosition = recentlyJoinedConsumers.get(consumer); // At this point, all the old messages were already consumed and this consumer // is now ready to receive any message - if (maxReadPosition == null) { + if (maxLastSentPosition == null) { // The consumer has not recently joined, so we can send all messages - return maxMessages; + return null; } // If the read type is Replay, we should avoid send messages that hold by other consumer to the new consumers, @@ -369,32 +644,24 @@ private int getRestrictedMaxEntriesForConsumer(Consumer consumer, List en // But the message [2,3] should not dispatch to consumer2. if (readType == ReadType.Replay) { - PositionImpl minReadPositionForRecentJoinedConsumer = recentlyJoinedConsumers.values().iterator().next(); - if (minReadPositionForRecentJoinedConsumer != null - && minReadPositionForRecentJoinedConsumer.compareTo(maxReadPosition) < 0) { - maxReadPosition = minReadPositionForRecentJoinedConsumer; - } - } - // Here, the consumer is one that has recently joined, so we can only send messages that were - // published before it has joined. - for (int i = 0; i < maxMessages; i++) { - if (((PositionImpl) entries.get(i).getPosition()).compareTo(maxReadPosition) >= 0) { - // We have already crossed the divider line. All messages in the list are now - // newer than what we can currently dispatch to this consumer - return i; + Position minLastSentPositionForRecentJoinedConsumer = recentlyJoinedConsumers.values().iterator().next(); + if (minLastSentPositionForRecentJoinedConsumer != null + && minLastSentPositionForRecentJoinedConsumer.compareTo(maxLastSentPosition) < 0) { + maxLastSentPosition = minLastSentPositionForRecentJoinedConsumer; } } - return maxMessages; + return maxLastSentPosition; } + @Override public void markDeletePositionMoveForward() { // Execute the notification in different thread to avoid a mutex chain here // from the delete operation that was completed topic.getBrokerService().getTopicOrderedExecutor().execute(() -> { synchronized (PersistentStickyKeyDispatcherMultipleConsumers.this) { - if (recentlyJoinedConsumers != null && !recentlyJoinedConsumers.isEmpty() + if (hasRecentlyJoinedConsumers() && removeConsumersFromRecentJoinedConsumers()) { // After we process acks, we need to check whether the mark-delete position was advanced and we // can finally read more messages. It's safe to call readMoreEntries() multiple times. @@ -404,16 +671,21 @@ && removeConsumersFromRecentJoinedConsumers()) { }); } + private boolean hasRecentlyJoinedConsumers() { + return !MapUtils.isEmpty(recentlyJoinedConsumers); + } + private boolean removeConsumersFromRecentJoinedConsumers() { - Iterator> itr = recentlyJoinedConsumers.entrySet().iterator(); + if (MapUtils.isEmpty(recentlyJoinedConsumers)) { + return false; + } + Iterator> itr = recentlyJoinedConsumers.entrySet().iterator(); boolean hasConsumerRemovedFromTheRecentJoinedConsumers = false; - PositionImpl mdp = (PositionImpl) cursor.getMarkDeletedPosition(); + Position mdp = cursor.getMarkDeletedPosition(); if (mdp != null) { - PositionImpl nextPositionOfTheMarkDeletePosition = - ((ManagedLedgerImpl) cursor.getManagedLedger()).getNextValidPosition(mdp); while (itr.hasNext()) { - Map.Entry entry = itr.next(); - if (entry.getValue().compareTo(nextPositionOfTheMarkDeletePosition) <= 0) { + Map.Entry entry = itr.next(); + if (entry.getValue().compareTo(mdp) <= 0) { itr.remove(); hasConsumerRemovedFromTheRecentJoinedConsumers = true; } else { @@ -424,17 +696,114 @@ private boolean removeConsumersFromRecentJoinedConsumers() { return hasConsumerRemovedFromTheRecentJoinedConsumers; } + @Nullable + private synchronized Position updateIfNeededAndGetLastSentPosition() { + if (lastSentPosition == null) { + return null; + } + final Position mdp = cursor.getMarkDeletedPosition(); + if (mdp != null && mdp.compareTo(lastSentPosition) > 0) { + lastSentPosition = mdp; + } + return lastSentPosition; + } + + /** + * The dispatcher will skip replaying messages when all messages in the replay queue are filtered out when + * skipNextReplayToTriggerLookAhead=true. The flag gets resetted after the call. + * + * If we're stuck on replay, we want to move forward reading on the topic (until the configured look ahead + * limits kick in), instead of keep replaying the same old messages, since the consumer that these + * messages are routing to might be busy at the moment. + * + * Please see {@link ServiceConfiguration#getKeySharedLookAheadMsgInReplayThresholdPerConsumer} and + * {@link ServiceConfiguration#getKeySharedLookAheadMsgInReplayThresholdPerSubscription} for configuring the limits. + */ @Override - protected synchronized NavigableSet getMessagesToReplayNow(int maxMessagesToRead) { - if (isDispatcherStuckOnReplays) { - // If we're stuck on replay, we want to move forward reading on the topic (until the overall max-unacked - // messages kicks in), instead of keep replaying the same old messages, since the consumer that these - // messages are routing to might be busy at the moment - this.isDispatcherStuckOnReplays = false; - return Collections.emptyNavigableSet(); + protected synchronized boolean canReplayMessages() { + if (skipNextReplayToTriggerLookAhead) { + skipNextReplayToTriggerLookAhead = false; + return false; + } + return true; + } + + private int getAvailablePermits(Consumer c) { + // skip consumers that are currently closing + if (!c.cnx().isActive()) { + return 0; + } + int availablePermits = Math.max(c.getAvailablePermits(), 0); + if (availablePermits > 0 && c.getMaxUnackedMessages() > 0) { + // Calculate the maximum number of additional unacked messages allowed + int maxAdditionalUnackedMessages = Math.max(c.getMaxUnackedMessages() - c.getUnackedMessages(), 0); + if (maxAdditionalUnackedMessages == 0) { + // if the consumer has reached the max unacked messages, then no more messages can be dispatched + return 0; + } + // Estimate the remaining permits based on the average messages per entry + // add "avgMessagesPerEntry - 1" to round up the division to the next integer without the need to use + // floating point arithmetic + int avgMessagesPerEntry = Math.max(c.getAvgMessagesPerEntry(), 1); + int estimatedRemainingPermits = + (maxAdditionalUnackedMessages + avgMessagesPerEntry - 1) / avgMessagesPerEntry; + // return the minimum of current available permits and estimated remaining permits + return Math.min(availablePermits, estimatedRemainingPermits); } else { - return super.getMessagesToReplayNow(maxMessagesToRead); + return availablePermits; + } + } + + /** + * For Key_Shared subscription, the dispatcher will not read more entries while there are pending reads + * or pending replay reads. + * @return true if there are no pending reads or pending replay reads + */ + @Override + protected boolean doesntHavePendingRead() { + return !havePendingRead && !havePendingReplayRead; + } + + /** + * For Key_Shared subscription, the dispatcher will not attempt to read more entries if the replay queue size + * has reached the limit or if there are no consumers with permits. + */ + @Override + protected boolean isNormalReadAllowed() { + // don't allow reading more if the replay queue size has reached the limit + if (!isReplayQueueSizeBelowLimit()) { + return false; + } + for (Consumer consumer : consumerList) { + // skip blocked consumers + if (consumer == null || consumer.isBlocked()) { + continue; + } + // before reading more, check that there's at least one consumer that has permits + if (getAvailablePermits(consumer) > 0) { + return true; + } + } + return false; + } + + @Override + protected int getMaxEntriesReadLimit() { + // prevent the redelivery queue from growing over the limit by limiting the number of entries to read + // to the maximum number of entries that can be added to the redelivery queue + return Math.max(getEffectiveLookAheadLimit() - redeliveryMessages.size(), 1); + } + + /** + * When a normal read is not allowed, the dispatcher will reschedule a read with a backoff. + */ + @Override + protected void handleNormalReadNotAllowed() { + if (log.isDebugEnabled()) { + log.debug("[{}] [{}] Skipping read for the topic since normal read isn't allowed. " + + "Rescheduling a read with a backoff.", topic.getName(), getSubscriptionName()); } + reScheduleReadWithBackoff(); } @Override @@ -460,14 +829,37 @@ public boolean hasSameKeySharedPolicy(KeySharedMeta ksm) { && ksm.isAllowOutOfOrderDelivery() == this.allowOutOfOrderDelivery); } - public LinkedHashMap getRecentlyJoinedConsumers() { + public LinkedHashMap getRecentlyJoinedConsumers() { return recentlyJoinedConsumers; } + public synchronized String getLastSentPosition() { + if (lastSentPosition == null) { + return null; + } + return lastSentPosition.toString(); + } + + @VisibleForTesting + public Position getLastSentPositionField() { + return lastSentPosition; + } + + public synchronized String getIndividuallySentPositions() { + if (individuallySentPositions == null) { + return null; + } + return individuallySentPositions.toString(); + } + + @VisibleForTesting + public LongPairRangeSet getIndividuallySentPositionsField() { + return individuallySentPositions; + } + public Map> getConsumerKeyHashRanges() { return selector.getConsumerKeyHashRanges(); } private static final Logger log = LoggerFactory.getLogger(PersistentStickyKeyDispatcherMultipleConsumers.class); - } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index 09dabcd4bfc59..9a0545e6f0ab2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -23,19 +23,21 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import io.netty.buffer.ByteBuf; +import java.io.IOException; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.TreeMap; +import java.util.UUID; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; import java.util.stream.Collectors; +import lombok.Getter; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.AsyncCallbacks.ClearBacklogCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCallback; @@ -50,13 +52,12 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.InvalidCursorPositionException; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.ScanOutcome; -import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.intercept.BrokerInterceptor; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.service.AbstractSubscription; import org.apache.pulsar.broker.service.AnalyzeBacklogResult; import org.apache.pulsar.broker.service.BrokerServiceException; @@ -68,6 +69,7 @@ import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.broker.service.Dispatcher; import org.apache.pulsar.broker.service.EntryFilterSupport; +import org.apache.pulsar.broker.service.GetStatsOptions; import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.plugin.EntryFilter; @@ -93,7 +95,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class PersistentSubscription extends AbstractSubscription implements Subscription { +public class PersistentSubscription extends AbstractSubscription { protected final PersistentTopic topic; protected final ManagedCursor cursor; protected volatile Dispatcher dispatcher; @@ -120,17 +122,16 @@ public class PersistentSubscription extends AbstractSubscription implements Subs // Map of properties that is used to mark this subscription as "replicated". // Since this is the only field at this point, we can just keep a static // instance of the map. - private static final Map REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES = new TreeMap<>(); - private static final Map NON_REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES = Collections.emptyMap(); + private static final Map REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES = + Map.of(REPLICATED_SUBSCRIPTION_PROPERTY, 1L); + private static final Map NON_REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES = Map.of(); private volatile ReplicatedSubscriptionSnapshotCache replicatedSubscriptionSnapshotCache; + @Getter private final PendingAckHandle pendingAckHandle; private volatile Map subscriptionProperties; private volatile CompletableFuture fenceFuture; - - static { - REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES.put(REPLICATED_SUBSCRIPTION_PROPERTY, 1L); - } + private volatile CompletableFuture inProgressResetCursorFuture; static Map getBaseCursorProperties(boolean isReplicated) { return isReplicated ? REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES : NON_REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES; @@ -152,12 +153,13 @@ public PersistentSubscription(PersistentTopic topic, String subscriptionName, Ma this.topicName = topic.getName(); this.subName = subscriptionName; this.fullName = MoreObjects.toStringHelper(this).add("topic", topicName).add("name", subName).toString(); - this.expiryMonitor = new PersistentMessageExpiryMonitor(topicName, subscriptionName, cursor, this); + this.expiryMonitor = new PersistentMessageExpiryMonitor(topic, subscriptionName, cursor, this); this.setReplicated(replicated); this.subscriptionProperties = MapUtils.isEmpty(subscriptionProperties) ? Collections.emptyMap() : Collections.unmodifiableMap(subscriptionProperties); if (topic.getBrokerService().getPulsar().getConfig().isTransactionCoordinatorEnabled() - && !isEventSystemTopic(TopicName.get(topicName))) { + && !isEventSystemTopic(TopicName.get(topicName)) + && !ExtensibleLoadManagerImpl.isInternalTopic(topicName)) { this.pendingAckHandle = new PendingAckHandleImpl(this); } else { this.pendingAckHandle = new PendingAckHandleDisabled(); @@ -202,7 +204,12 @@ public boolean setReplicated(boolean replicated) { if (this.cursor != null) { if (replicated) { - return this.cursor.putProperty(REPLICATED_SUBSCRIPTION_PROPERTY, 1L); + if (!config.isEnableReplicatedSubscriptions()) { + log.warn("[{}][{}] Failed set replicated subscription status to {}, please enable the " + + "configuration enableReplicatedSubscriptions", topicName, subName, replicated); + } else { + return this.cursor.putProperty(REPLICATED_SUBSCRIPTION_PROPERTY, 1L); + } } else { return this.cursor.removeProperty(REPLICATED_SUBSCRIPTION_PROPERTY); } @@ -213,6 +220,16 @@ public boolean setReplicated(boolean replicated) { @Override public CompletableFuture addConsumer(Consumer consumer) { + CompletableFuture inProgressResetCursorFuture = this.inProgressResetCursorFuture; + if (inProgressResetCursorFuture != null) { + return inProgressResetCursorFuture.handle((ignore, ignoreEx) -> null) + .thenCompose(ignore -> addConsumerInternal(consumer)); + } else { + return addConsumerInternal(consumer); + } + } + + private CompletableFuture addConsumerInternal(Consumer consumer) { return pendingAckHandle.pendingAckHandleFuture().thenCompose(future -> { synchronized (PersistentSubscription.this) { cursor.updateLastActive(); @@ -301,11 +318,11 @@ public synchronized void removeConsumer(Consumer consumer, boolean isResetCursor if (dispatcher != null && dispatcher.getConsumers().isEmpty()) { deactivateCursor(); - topic.getManagedLedger().removeWaitingCursor(cursor); if (!cursor.isDurable()) { - // If cursor is not durable, we need to clean up the subscription as well - this.close().thenRun(() -> { + // If cursor is not durable, we need to clean up the subscription as well. No need to check for active + // consumers since we already validated that there are no consumers on this dispatcher. + this.closeCursor(false).thenRun(() -> { synchronized (this) { if (dispatcher != null) { dispatcher.close().thenRun(() -> { @@ -324,17 +341,21 @@ public synchronized void removeConsumer(Consumer consumer, boolean isResetCursor // when topic closes: it iterates through concurrent-subscription map to close each subscription. so, // topic.remove again try to access same map which creates deadlock. so, execute it in different thread. topic.getBrokerService().pulsar().getExecutor().execute(() -> { - topic.removeSubscription(subName); - // Also need remove the cursor here, otherwise the data deletion will not work well. - // Because data deletion depends on the mark delete position of all cursors. - if (!isResetCursor) { - try { - topic.getManagedLedger().deleteCursor(cursor.getName()); - } catch (InterruptedException | ManagedLedgerException e) { - log.warn("[{}] [{}] Failed to remove non durable cursor", topic.getName(), subName, e); + topic.removeSubscription(subName).thenRunAsync(() -> { + // Also need remove the cursor here, otherwise the data deletion will not work well. + // Because data deletion depends on the mark delete position of all cursors. + if (!isResetCursor) { + try { + topic.getManagedLedger().deleteCursor(cursor.getName()); + topic.getManagedLedger().removeWaitingCursor(cursor); + } catch (InterruptedException | ManagedLedgerException e) { + log.warn("[{}] [{}] Failed to remove non durable cursor", topic.getName(), subName, e); + } } - } + }, topic.getBrokerService().pulsar().getExecutor()); }); + } else { + topic.getManagedLedger().removeWaitingCursor(cursor); } } @@ -383,7 +404,7 @@ public void acknowledgeMessage(List positions, AckType ackType, Map { - if (((ManagedCursorImpl) cursor).isMessageDeleted(position)) { + if ((cursor.isMessageDeleted(position))) { pendingAckHandle.clearIndividualPosition(position); } }); @@ -401,7 +422,7 @@ public void acknowledgeMessage(List positions, AckType ackType, Map c.localSubscriptionUpdated(subName, snapshot)); @@ -419,24 +440,28 @@ public void acknowledgeMessage(List positions, AckType ackType, Map transactionIndividualAcknowledge( TxnID txnId, - List> positions) { + List> positions) { return pendingAckHandle.individualAcknowledgeMessage(txnId, positions); } - public CompletableFuture transactionCumulativeAcknowledge(TxnID txnId, List positions) { + public CompletableFuture transactionCumulativeAcknowledge(TxnID txnId, List positions) { return pendingAckHandle.cumulativeAcknowledgeMessage(txnId, positions); } private final MarkDeleteCallback markDeleteCallback = new MarkDeleteCallback() { @Override public void markDeleteComplete(Object ctx) { - PositionImpl oldMD = (PositionImpl) ctx; - PositionImpl newMD = (PositionImpl) cursor.getMarkDeletedPosition(); + Position oldMD = (Position) ctx; + Position newMD = cursor.getMarkDeletedPosition(); if (log.isDebugEnabled()) { log.debug("[{}][{}] Mark deleted messages to position {} from position {}", topicName, subName, newMD, oldMD); } // Signal the dispatchers to give chance to take extra actions + if (dispatcher != null) { + dispatcher.afterAckMessages(null, ctx); + } + // Signal the dispatchers to give chance to take extra actions notifyTheMarkDeletePositionMoveForwardIfNeeded(oldMD); } @@ -446,28 +471,40 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { if (log.isDebugEnabled()) { log.debug("[{}][{}] Failed to mark delete for position {}: {}", topicName, subName, ctx, exception); } + // Signal the dispatchers to give chance to take extra actions + if (dispatcher != null) { + dispatcher.afterAckMessages(null, ctx); + } } }; private final DeleteCallback deleteCallback = new DeleteCallback() { @Override - public void deleteComplete(Object position) { + public void deleteComplete(Object context) { if (log.isDebugEnabled()) { - log.debug("[{}][{}] Deleted message at {}", topicName, subName, position); + // The value of the param "context" is a position. + log.debug("[{}][{}] Deleted message at {}", topicName, subName, context); } // Signal the dispatchers to give chance to take extra actions - notifyTheMarkDeletePositionMoveForwardIfNeeded((PositionImpl) position); + if (dispatcher != null) { + dispatcher.afterAckMessages(null, context); + } + notifyTheMarkDeletePositionMoveForwardIfNeeded((Position) context); } @Override public void deleteFailed(ManagedLedgerException exception, Object ctx) { log.warn("[{}][{}] Failed to delete message at {}: {}", topicName, subName, ctx, exception); + // Signal the dispatchers to give chance to take extra actions + if (dispatcher != null) { + dispatcher.afterAckMessages(exception, ctx); + } } }; private void notifyTheMarkDeletePositionMoveForwardIfNeeded(Position oldPosition) { - PositionImpl oldMD = (PositionImpl) oldPosition; - PositionImpl newMD = (PositionImpl) cursor.getMarkDeletedPosition(); + Position oldMD = oldPosition; + Position newMD = cursor.getMarkDeletedPosition(); if (dispatcher != null && newMD.compareTo(oldMD) > 0) { dispatcher.markDeletePositionMoveForward(); } @@ -509,9 +546,15 @@ public String getTypeString() { return "Null"; } - @Override public CompletableFuture analyzeBacklog(Optional position) { - + final ManagedLedger managedLedger = topic.getManagedLedger(); + final String newNonDurableCursorName = "analyze-backlog-" + UUID.randomUUID(); + ManagedCursor newNonDurableCursor; + try { + newNonDurableCursor = cursor.duplicateNonDurableCursor(newNonDurableCursorName); + } catch (ManagedLedgerException e) { + return CompletableFuture.failedFuture(e); + } long start = System.currentTimeMillis(); if (log.isDebugEnabled()) { log.debug("[{}][{}] Starting to analyze backlog", topicName, subName); @@ -526,7 +569,7 @@ public CompletableFuture analyzeBacklog(Optional AtomicLong rejectedMessages = new AtomicLong(); AtomicLong rescheduledMessages = new AtomicLong(); - Position currentPosition = cursor.getMarkDeletedPosition(); + Position currentPosition = newNonDurableCursor.getMarkDeletedPosition(); if (log.isDebugEnabled()) { log.debug("[{}][{}] currentPosition {}", @@ -586,7 +629,7 @@ public CompletableFuture analyzeBacklog(Optional return true; }; - return cursor.scan( + CompletableFuture res = newNonDurableCursor.scan( position, condition, batchSize, @@ -613,7 +656,22 @@ public CompletableFuture analyzeBacklog(Optional topicName, subName, end - start, result); return result; }); + res.whenComplete((__, ex) -> { + managedLedger.asyncDeleteCursor(newNonDurableCursorName, + new AsyncCallbacks.DeleteCursorCallback(){ + @Override + public void deleteCursorComplete(Object ctx) { + // Nothing to do. + } + @Override + public void deleteCursorFailed(ManagedLedgerException exception, Object ctx) { + log.warn("[{}][{}] Delete non-durable cursor[{}] failed when analyze backlog.", + topicName, subName, newNonDurableCursor.getName()); + } + }, null); + }); + return res; } @Override @@ -640,6 +698,7 @@ public void clearBacklogComplete(Object ctx) { future.complete(null); } }); + dispatcher.afterAckMessages(null, ctx); } else { future.complete(null); } @@ -649,6 +708,9 @@ public void clearBacklogComplete(Object ctx) { public void clearBacklogFailed(ManagedLedgerException exception, Object ctx) { log.error("[{}][{}] Failed to clear backlog", topicName, subName, exception); future.completeExceptionally(exception); + if (dispatcher != null) { + dispatcher.afterAckMessages(exception, ctx); + } } }, null); @@ -672,6 +734,9 @@ public void skipEntriesComplete(Object ctx) { numMessagesToSkip, cursor.getNumberOfEntriesInBacklog(false)); } future.complete(null); + if (dispatcher != null) { + dispatcher.afterAckMessages(null, ctx); + } } @Override @@ -679,6 +744,9 @@ public void skipEntriesFailed(ManagedLedgerException exception, Object ctx) { log.error("[{}][{}] Failed to skip {} messages", topicName, subName, numMessagesToSkip, exception); future.completeExceptionally(exception); + if (dispatcher != null) { + dispatcher.afterAckMessages(exception, ctx); + } } }, null); @@ -717,7 +785,8 @@ public void findEntryComplete(Position position, Object ctx) { } else { finalPosition = position.getNext(); } - resetCursor(finalPosition, future); + CompletableFuture resetCursorFuture = resetCursor(finalPosition); + FutureUtil.completeAfter(future, resetCursorFuture); } @Override @@ -736,18 +805,13 @@ public void findEntryFailed(ManagedLedgerException exception, } @Override - public CompletableFuture resetCursor(Position position) { - CompletableFuture future = new CompletableFuture<>(); - resetCursor(position, future); - return future; - } - - private void resetCursor(Position finalPosition, CompletableFuture future) { + public CompletableFuture resetCursor(Position finalPosition) { if (!IS_FENCED_UPDATER.compareAndSet(PersistentSubscription.this, FALSE, TRUE)) { - future.completeExceptionally(new SubscriptionBusyException("Failed to fence subscription")); - return; + return CompletableFuture.failedFuture(new SubscriptionBusyException("Failed to fence subscription")); } + final CompletableFuture future = new CompletableFuture<>(); + inProgressResetCursorFuture = future; final CompletableFuture disconnectFuture; // Lock the Subscription object before locking the Dispatcher object to avoid deadlocks @@ -767,6 +831,7 @@ private void resetCursor(Position finalPosition, CompletableFuture future) if (throwable != null) { log.error("[{}][{}] Failed to disconnect consumer from subscription", topicName, subName, throwable); IS_FENCED_UPDATER.set(PersistentSubscription.this, FALSE); + inProgressResetCursorFuture = null; future.completeExceptionally( new SubscriptionBusyException("Failed to disconnect consumers from subscription")); return; @@ -775,16 +840,26 @@ private void resetCursor(Position finalPosition, CompletableFuture future) log.info("[{}][{}] Successfully disconnected consumers from subscription, proceeding with cursor reset", topicName, subName); - try { - boolean forceReset = false; - if (topic.getCompactedTopic() != null && topic.getCompactedTopic().getCompactionHorizon().isPresent()) { - PositionImpl horizon = (PositionImpl) topic.getCompactedTopic().getCompactionHorizon().get(); - PositionImpl resetTo = (PositionImpl) finalPosition; - if (horizon.compareTo(resetTo) >= 0) { - forceReset = true; + CompletableFuture forceReset = new CompletableFuture<>(); + if (topic.getTopicCompactionService() == null) { + forceReset.complete(false); + } else { + topic.getTopicCompactionService().getLastCompactedPosition().thenAccept(lastCompactedPosition -> { + Position resetTo = finalPosition; + if (lastCompactedPosition != null && resetTo.compareTo(lastCompactedPosition.getLedgerId(), + lastCompactedPosition.getEntryId()) <= 0) { + forceReset.complete(true); + } else { + forceReset.complete(false); } - } - cursor.asyncResetCursor(finalPosition, forceReset, new AsyncCallbacks.ResetCursorCallback() { + }).exceptionally(ex -> { + forceReset.completeExceptionally(ex); + return null; + }); + } + + forceReset.thenAccept(forceResetValue -> { + cursor.asyncResetCursor(finalPosition, forceResetValue, new AsyncCallbacks.ResetCursorCallback() { @Override public void resetComplete(Object ctx) { if (log.isDebugEnabled()) { @@ -793,8 +868,10 @@ public void resetComplete(Object ctx) { } if (dispatcher != null) { dispatcher.cursorIsReset(); + dispatcher.afterAckMessages(null, finalPosition); } IS_FENCED_UPDATER.set(PersistentSubscription.this, FALSE); + inProgressResetCursorFuture = null; future.complete(null); } @@ -803,6 +880,7 @@ public void resetFailed(ManagedLedgerException exception, Object ctx) { log.error("[{}][{}] Failed to reset subscription to position {}", topicName, subName, finalPosition, exception); IS_FENCED_UPDATER.set(PersistentSubscription.this, FALSE); + inProgressResetCursorFuture = null; // todo - retry on InvalidCursorPositionException // or should we just ask user to retry one more time? if (exception instanceof InvalidCursorPositionException) { @@ -814,12 +892,15 @@ public void resetFailed(ManagedLedgerException exception, Object ctx) { } } }); - } catch (Exception e) { + }).exceptionally((e) -> { log.error("[{}][{}] Error while resetting cursor", topicName, subName, e); IS_FENCED_UPDATER.set(PersistentSubscription.this, FALSE); + inProgressResetCursorFuture = null; future.completeExceptionally(new BrokerServiceException(e)); - } + return null; + }); }); + return future; } @Override @@ -841,6 +922,12 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { public void readEntryComplete(Entry entry, Object ctx) { future.complete(entry); } + + @Override + public String toString() { + return String.format("Subscription [%s-%s] async replay entries", PersistentSubscription.this.topicName, + PersistentSubscription.this.subName); + } }, null); return future; @@ -865,49 +952,77 @@ public int getTotalNonContiguousDeletedMessagesRange() { } /** - * Close the cursor ledger for this subscription. Requires that there are no active consumers on the dispatcher + * Close the cursor ledger for this subscription. Optionally verifies that there are no active consumers on the + * dispatcher. * - * @return CompletableFuture indicating the completion of delete operation + * @return CompletableFuture indicating the completion of close operation */ - @Override - public CompletableFuture close() { - synchronized (this) { - if (dispatcher != null && dispatcher.isConsumerConnected()) { - return FutureUtil.failedFuture(new SubscriptionBusyException("Subscription has active consumers")); - } - return this.pendingAckHandle.closeAsync().thenAccept(v -> { - IS_FENCED_UPDATER.set(this, TRUE); - log.info("[{}][{}] Successfully closed subscription [{}]", topicName, subName, cursor); - }); + private synchronized CompletableFuture closeCursor(boolean checkActiveConsumers) { + if (checkActiveConsumers && dispatcher != null && dispatcher.isConsumerConnected()) { + return FutureUtil.failedFuture(new SubscriptionBusyException("Subscription has active consumers")); } + return this.pendingAckHandle.closeAsync().thenAccept(v -> { + IS_FENCED_UPDATER.set(this, TRUE); + log.info("[{}][{}] Successfully closed subscription [{}]", topicName, subName, cursor); + }); + } + + + /** + * Disconnect all consumers from this subscription. + * + * @return CompletableFuture indicating the completion of the operation. + */ + @Override + public synchronized CompletableFuture disconnect(Optional assignedBrokerLookupData) { + CompletableFuture disconnectFuture = new CompletableFuture<>(); + + (dispatcher != null + ? dispatcher.disconnectAllConsumers(false, assignedBrokerLookupData) + : CompletableFuture.completedFuture(null)) + .thenRun(() -> { + log.info("[{}][{}] Successfully disconnected subscription consumers", topicName, subName); + disconnectFuture.complete(null); + }).exceptionally(exception -> { + log.error("[{}][{}] Error disconnecting subscription consumers", topicName, subName, exception); + disconnectFuture.completeExceptionally(exception); + return null; + }); + + return disconnectFuture; } /** - * Disconnect all consumers attached to the dispatcher and close this subscription. + * Fence this subscription and optionally disconnect all consumers. * - * @return CompletableFuture indicating the completion of disconnect operation + * @return CompletableFuture indicating the completion of the operation. */ @Override - public synchronized CompletableFuture disconnect() { - if (fenceFuture != null){ + public synchronized CompletableFuture close(boolean disconnectConsumers, + Optional assignedBrokerLookupData) { + if (fenceFuture != null) { return fenceFuture; } + fenceFuture = new CompletableFuture<>(); // block any further consumers on this subscription IS_FENCED_UPDATER.set(this, TRUE); - (dispatcher != null ? dispatcher.close() : CompletableFuture.completedFuture(null)) - .thenCompose(v -> close()).thenRun(() -> { - log.info("[{}][{}] Successfully disconnected and closed subscription", topicName, subName); + (dispatcher != null + ? dispatcher.close(disconnectConsumers, assignedBrokerLookupData) + : CompletableFuture.completedFuture(null)) + // checkActiveConsumers is false since we just closed all of them if we wanted. + .thenCompose(__ -> closeCursor(false)).thenRun(() -> { + log.info("[{}][{}] Successfully closed the subscription", topicName, subName); fenceFuture.complete(null); }).exceptionally(exception -> { - log.error("[{}][{}] Error disconnecting consumers from subscription", topicName, subName, - exception); + log.error("[{}][{}] Error closing the subscription", topicName, subName, exception); fenceFuture.completeExceptionally(exception); resumeAfterFence(); return null; }); + return fenceFuture; } @@ -915,7 +1030,7 @@ public synchronized CompletableFuture disconnect() { * Resume subscription after topic deletion or close failure. */ public synchronized void resumeAfterFence() { - // If "fenceFuture" is null, it means that "disconnect" has never been called. + // If "fenceFuture" is null, it means that "close" has never been called. if (fenceFuture != null) { fenceFuture.whenComplete((ignore, ignoreEx) -> { synchronized (PersistentSubscription.this) { @@ -972,7 +1087,7 @@ private CompletableFuture delete(boolean closeIfConsumersConnected) { CompletableFuture closeSubscriptionFuture = new CompletableFuture<>(); if (closeIfConsumersConnected) { - this.disconnect().thenRun(() -> { + this.close(true, Optional.empty()).thenRun(() -> { closeSubscriptionFuture.complete(null); }).exceptionally(ex -> { log.error("[{}][{}] Error disconnecting and closing subscription", topicName, subName, ex); @@ -980,7 +1095,7 @@ private CompletableFuture delete(boolean closeIfConsumersConnected) { return null; }); } else { - this.close().thenRun(() -> { + this.closeCursor(true).thenRun(() -> { closeSubscriptionFuture.complete(null); }).exceptionally(exception -> { log.error("[{}][{}] Error closing subscription", topicName, subName, exception); @@ -1024,11 +1139,27 @@ private CompletableFuture delete(boolean closeIfConsumersConnected) { */ @Override public CompletableFuture doUnsubscribe(Consumer consumer) { + return doUnsubscribe(consumer, false); + } + + /** + * Handle unsubscribe command from the client API Check with the dispatcher is this consumer can proceed with + * unsubscribe. + * + * @param consumer consumer object that is initiating the unsubscribe operation + * @param force unsubscribe forcefully by disconnecting consumers and closing subscription + * @return CompletableFuture indicating the completion of unsubscribe operation + */ + @Override + public CompletableFuture doUnsubscribe(Consumer consumer, boolean force) { CompletableFuture future = new CompletableFuture<>(); try { - if (dispatcher.canUnsubscribe(consumer)) { + if (force || dispatcher.canUnsubscribe(consumer)) { + if (log.isDebugEnabled()) { + log.debug("[{}] unsubscribing forcefully {}-{}", topicName, subName, consumer.consumerName()); + } consumer.close(); - return delete(); + return delete(force); } future.completeExceptionally( new ServerMetadataException("Unconnected or shared consumer attempting to unsubscribe")); @@ -1051,8 +1182,9 @@ public List getConsumers() { @Override public boolean expireMessages(int messageTTLInSeconds) { - if ((getNumberOfEntriesInBacklog(false) == 0) || (dispatcher != null && dispatcher.isConsumerConnected() - && getNumberOfEntriesInBacklog(false) < MINIMUM_BACKLOG_FOR_EXPIRY_CHECK + long backlog = getNumberOfEntriesInBacklog(false); + if (backlog == 0 || (dispatcher != null && dispatcher.isConsumerConnected() + && backlog < MINIMUM_BACKLOG_FOR_EXPIRY_CHECK && !topic.isOldestMessageExpired(cursor, messageTTLInSeconds))) { // don't do anything for almost caught-up connected subscriptions return false; @@ -1079,8 +1211,7 @@ public long estimateBacklogSize() { return cursor.getEstimatedSizeSinceMarkDeletePosition(); } - public SubscriptionStatsImpl getStats(Boolean getPreciseBacklog, boolean subscriptionBacklogSize, - boolean getEarliestTimeInBacklog) { + public CompletableFuture getStatsAsync(GetStatsOptions getStatsOptions) { SubscriptionStatsImpl subStats = new SubscriptionStatsImpl(); subStats.lastExpireTimestamp = lastExpireTimestamp; subStats.lastConsumedFlowTimestamp = lastConsumedFlowTimestamp; @@ -1094,7 +1225,9 @@ public SubscriptionStatsImpl getStats(Boolean getPreciseBacklog, boolean subscri ? ((PersistentStickyKeyDispatcherMultipleConsumers) dispatcher).getConsumerKeyHashRanges() : null; dispatcher.getConsumers().forEach(consumer -> { ConsumerStatsImpl consumerStats = consumer.getStats(); - subStats.consumers.add(consumerStats); + if (!getStatsOptions.isExcludeConsumers()) { + subStats.consumers.add(consumerStats); + } subStats.msgRateOut += consumerStats.msgRateOut; subStats.msgThroughputOut += consumerStats.msgThroughputOut; subStats.bytesOutCounter += consumerStats.bytesOutCounter; @@ -1142,26 +1275,16 @@ public SubscriptionStatsImpl getStats(Boolean getPreciseBacklog, boolean subscri subStats.unackedMessages = d.getTotalUnackedMessages(); subStats.blockedSubscriptionOnUnackedMsgs = d.isBlockedDispatcherOnUnackedMsgs(); subStats.msgDelayed = d.getNumberOfDelayedMessages(); + subStats.msgInReplay = d.getNumberOfMessagesInReplay(); } } - subStats.msgBacklog = getNumberOfEntriesInBacklog(getPreciseBacklog); - if (subscriptionBacklogSize) { - subStats.backlogSize = ((ManagedLedgerImpl) topic.getManagedLedger()) - .getEstimatedBacklogSize((PositionImpl) cursor.getMarkDeletedPosition()); + subStats.msgBacklog = getNumberOfEntriesInBacklog(getStatsOptions.isGetPreciseBacklog()); + if (getStatsOptions.isSubscriptionBacklogSize()) { + subStats.backlogSize = topic.getManagedLedger() + .getEstimatedBacklogSize(cursor.getMarkDeletedPosition()); } else { subStats.backlogSize = -1; } - if (getEarliestTimeInBacklog && subStats.msgBacklog > 0) { - ManagedLedgerImpl managedLedger = ((ManagedLedgerImpl) cursor.getManagedLedger()); - PositionImpl markDeletedPosition = (PositionImpl) cursor.getMarkDeletedPosition(); - long result = 0; - try { - result = managedLedger.getEarliestMessagePublishTimeOfPos(markDeletedPosition).get(); - } catch (InterruptedException | ExecutionException e) { - result = -1; - } - subStats.earliestMsgPublishTimeInBacklog = result; - } subStats.msgBacklogNoDelayed = subStats.msgBacklog - subStats.msgDelayed; subStats.msgRateExpired = expiryMonitor.getMessageExpiryRate(); subStats.totalMsgExpired = expiryMonitor.getTotalMessageExpired(); @@ -1175,18 +1298,90 @@ public SubscriptionStatsImpl getStats(Boolean getPreciseBacklog, boolean subscri subStats.allowOutOfOrderDelivery = keySharedDispatcher.isAllowOutOfOrderDelivery(); subStats.keySharedMode = keySharedDispatcher.getKeySharedMode().toString(); - LinkedHashMap recentlyJoinedConsumers = keySharedDispatcher + LinkedHashMap recentlyJoinedConsumers = keySharedDispatcher .getRecentlyJoinedConsumers(); if (recentlyJoinedConsumers != null && recentlyJoinedConsumers.size() > 0) { recentlyJoinedConsumers.forEach((k, v) -> { - subStats.consumersAfterMarkDeletePosition.put(k.consumerName(), v.toString()); + // The dispatcher allows same name consumers + final StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("consumerName=").append(k.consumerName()) + .append(", consumerId=").append(k.consumerId()); + if (k.cnx() != null) { + stringBuilder.append(", address=").append(k.cnx().clientAddress()); + } + subStats.consumersAfterMarkDeletePosition.put(stringBuilder.toString(), v.toString()); }); } + final String lastSentPosition = ((PersistentStickyKeyDispatcherMultipleConsumers) dispatcher) + .getLastSentPosition(); + if (lastSentPosition != null) { + subStats.lastSentPosition = lastSentPosition; + } + final String individuallySentPositions = ((PersistentStickyKeyDispatcherMultipleConsumers) dispatcher) + .getIndividuallySentPositions(); + if (individuallySentPositions != null) { + subStats.individuallySentPositions = individuallySentPositions; + } } subStats.nonContiguousDeletedMessagesRanges = cursor.getTotalNonContiguousDeletedMessagesRange(); subStats.nonContiguousDeletedMessagesRangesSerializedSize = cursor.getNonContiguousDeletedMessagesRangeSerializedSize(); - return subStats; + if (!getStatsOptions.isGetEarliestTimeInBacklog()) { + return CompletableFuture.completedFuture(subStats); + } + if (subStats.msgBacklog > 0) { + ManagedLedger managedLedger = cursor.getManagedLedger(); + Position markDeletedPosition = cursor.getMarkDeletedPosition(); + return getEarliestMessagePublishTimeOfPos(managedLedger, markDeletedPosition).thenApply(v -> { + subStats.earliestMsgPublishTimeInBacklog = v; + return subStats; + }); + } else { + subStats.earliestMsgPublishTimeInBacklog = -1; + return CompletableFuture.completedFuture(subStats); + } + } + + private CompletableFuture getEarliestMessagePublishTimeOfPos(ManagedLedger ml, Position pos) { + CompletableFuture future = new CompletableFuture<>(); + if (pos == null) { + future.complete(0L); + return future; + } + Position nextPos = ml.getNextValidPosition(pos); + + if (nextPos.compareTo(ml.getLastConfirmedEntry()) > 0) { + return CompletableFuture.completedFuture(-1L); + } + + ml.asyncReadEntry(nextPos, new ReadEntryCallback() { + @Override + public void readEntryComplete(Entry entry, Object ctx) { + try { + long entryTimestamp = Commands.getEntryTimestamp(entry.getDataBuffer()); + future.complete(entryTimestamp); + } catch (IOException e) { + log.error("Error deserializing message for message position {}", nextPos, e); + future.completeExceptionally(e); + } finally { + entry.release(); + } + } + + @Override + public void readEntryFailed(ManagedLedgerException exception, Object ctx) { + log.error("Error read entry for position {}", nextPos, exception); + future.completeExceptionally(exception); + } + + @Override + public String toString() { + return String.format("ML [%s] get earliest message publish time of pos", + ml.getName()); + } + }, null); + + return future; } @Override @@ -1198,16 +1393,16 @@ public void redeliverUnacknowledgedMessages(Consumer consumer, long consumerEpoc } @Override - public void redeliverUnacknowledgedMessages(Consumer consumer, List positions) { + public void redeliverUnacknowledgedMessages(Consumer consumer, List positions) { Dispatcher dispatcher = getDispatcher(); if (dispatcher != null) { dispatcher.redeliverUnacknowledgedMessages(consumer, positions); } } - private void trimByMarkDeletePosition(List positions) { + private void trimByMarkDeletePosition(List positions) { positions.removeIf(position -> cursor.getMarkDeletedPosition() != null - && position.compareTo((PositionImpl) cursor.getMarkDeletedPosition()) <= 0); + && position.compareTo(cursor.getMarkDeletedPosition()) <= 0); } @Override @@ -1239,12 +1434,18 @@ void topicTerminated() { } } + @Override + public boolean isSubscriptionMigrated() { + log.info("backlog for {} - {}", topicName, cursor.getNumberOfEntriesInBacklog(true)); + return topic.isMigrated() && cursor.getNumberOfEntriesInBacklog(true) <= 0; + } + @Override public Map getSubscriptionProperties() { return subscriptionProperties; } - public PositionImpl getPositionInPendingAck(PositionImpl position) { + public Position getPositionInPendingAck(Position position) { return pendingAckHandle.getPositionInPendingAck(position); } @Override @@ -1309,16 +1510,11 @@ public ManagedCursor getCursor() { return cursor; } - @VisibleForTesting - public PendingAckHandle getPendingAckHandle() { - return pendingAckHandle; - } - - public void syncBatchPositionBitSetForPendingAck(PositionImpl position) { + public void syncBatchPositionBitSetForPendingAck(Position position) { this.pendingAckHandle.syncBatchPositionAckSetForTransaction(position); } - public boolean checkIsCanDeleteConsumerPendingAck(PositionImpl position) { + public boolean checkIsCanDeleteConsumerPendingAck(Position position) { return this.pendingAckHandle.checkIsCanDeleteConsumerPendingAck(position); } @@ -1346,7 +1542,7 @@ public boolean checkIfPendingAckStoreInit() { return this.pendingAckHandle.checkIfPendingAckStoreInit(); } - public PositionInPendingAckStats checkPositionInPendingAckState(PositionImpl position, Integer batchIndex) { + public PositionInPendingAckStats checkPositionInPendingAckState(Position position, Integer batchIndex) { return pendingAckHandle.checkPositionInPendingAckState(position, batchIndex); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 15854f55c5cd1..f8581cfc79985 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -18,7 +18,10 @@ */ package org.apache.pulsar.broker.service.persistent; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateTableViewImpl.TOPIC; import static org.apache.pulsar.broker.service.persistent.SubscribeRateLimiter.isSubscribeRateEnabled; import static org.apache.pulsar.common.naming.SystemTopicNames.isEventSystemTopic; import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; @@ -41,16 +44,23 @@ import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.function.BiFunction; -import java.util.stream.Collectors; import javax.annotation.Nonnull; import lombok.Getter; +import lombok.Value; +import org.apache.bookkeeper.client.BKException.BKNoSuchLedgerExistsException; +import org.apache.bookkeeper.client.BKException.BKNoSuchLedgerExistsOnMetadataServerException; import org.apache.bookkeeper.client.api.LedgerMetadata; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback; @@ -71,19 +81,19 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.MetadataNotFoundException; import org.apache.bookkeeper.mledger.ManagedLedgerException.NonRecoverableLedgerException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionBound; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedCursorContainer; -import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.bookkeeper.mledger.impl.ShadowManagedLedgerImpl; -import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.mledger.impl.ManagedCursorContainer.CursorInfo; +import org.apache.bookkeeper.mledger.util.Futures; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.mutable.MutableInt; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory; import org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerFactory; -import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; -import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateCompactionStrategy; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateDataConflictResolver; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources; import org.apache.pulsar.broker.service.AbstractReplicator; @@ -102,28 +112,37 @@ import org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException; import org.apache.pulsar.broker.service.BrokerServiceException.TopicClosedException; import org.apache.pulsar.broker.service.BrokerServiceException.TopicFencedException; +import org.apache.pulsar.broker.service.BrokerServiceException.TopicMigratedException; import org.apache.pulsar.broker.service.BrokerServiceException.TopicTerminatedException; import org.apache.pulsar.broker.service.BrokerServiceException.UnsupportedSubscriptionException; import org.apache.pulsar.broker.service.BrokerServiceException.UnsupportedVersionException; import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.broker.service.Dispatcher; +import org.apache.pulsar.broker.service.GetStatsOptions; +import org.apache.pulsar.broker.service.PersistentTopicAttributes; import org.apache.pulsar.broker.service.Producer; import org.apache.pulsar.broker.service.Replicator; import org.apache.pulsar.broker.service.StreamingStats; import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.SubscriptionOption; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.TopicPoliciesService; import org.apache.pulsar.broker.service.TransportCnx; import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter.Type; import org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage; +import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; +import org.apache.pulsar.broker.service.schema.exceptions.NotExistSchemaException; import org.apache.pulsar.broker.stats.ClusterReplicationMetrics; import org.apache.pulsar.broker.stats.NamespaceStats; import org.apache.pulsar.broker.stats.ReplicationMetrics; import org.apache.pulsar.broker.transaction.buffer.TransactionBuffer; +import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferDisable; import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore; import org.apache.pulsar.client.admin.LongRunningProcessStatus; import org.apache.pulsar.client.admin.OffloadProcessStatus; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException.ConflictException; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.BatchMessageIdImpl; @@ -136,11 +155,13 @@ import org.apache.pulsar.common.api.proto.KeySharedMeta; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.api.proto.TxnAction; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType; -import org.apache.pulsar.common.policies.data.ClusterData.ClusterUrl; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.ClusterPolicies.ClusterUrl; import org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode; import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.CursorStats; import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.LedgerInfo; @@ -160,45 +181,54 @@ import org.apache.pulsar.common.policies.data.stats.TopicMetricBean; import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.protocol.Markers; import org.apache.pulsar.common.protocol.schema.SchemaData; +import org.apache.pulsar.common.protocol.schema.SchemaVersion; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.topics.TopicCompactionStrategy; import org.apache.pulsar.common.util.Codec; -import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; -import org.apache.pulsar.compaction.CompactedTopic; import org.apache.pulsar.compaction.CompactedTopicContext; import org.apache.pulsar.compaction.CompactedTopicImpl; import org.apache.pulsar.compaction.Compactor; import org.apache.pulsar.compaction.CompactorMXBean; +import org.apache.pulsar.compaction.PulsarTopicCompactionService; +import org.apache.pulsar.compaction.TopicCompactionService; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.utils.StatsOutputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCallback { // Managed ledger associated with the topic protected final ManagedLedger ledger; // Subscriptions to this topic - private final ConcurrentOpenHashMap subscriptions; + private final Map subscriptions = new ConcurrentHashMap<>(); - private final ConcurrentOpenHashMap replicators; - private final ConcurrentOpenHashMap shadowReplicators; + private final Map replicators = new ConcurrentHashMap<>(); + private final Map shadowReplicators = new ConcurrentHashMap<>(); @Getter private volatile List shadowTopics; private final TopicName shadowSourceTopic; - static final String DEDUPLICATION_CURSOR_NAME = "pulsar.dedup"; + public static final String DEDUPLICATION_CURSOR_NAME = "pulsar.dedup"; + + public static boolean isDedupCursorName(String name) { + return DEDUPLICATION_CURSOR_NAME.equals(name); + } private static final String TOPIC_EPOCH_PROPERTY_NAME = "pulsar.topic.epoch"; private static final double MESSAGE_EXPIRY_THRESHOLD = 1.5; private static final long POLICY_UPDATE_FAILURE_RETRY_TIME_SECONDS = 60; + private static final String MIGRATION_CLUSTER_NAME = "migration-cluster"; + private volatile boolean migrationSubsCreated = false; + // topic has every published chunked message since topic is loaded public boolean msgChunkPublished; @@ -210,14 +240,15 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal protected final MessageDeduplication messageDeduplication; - private static final long COMPACTION_NEVER_RUN = -0xfebecffeL; - private CompletableFuture currentCompaction = CompletableFuture.completedFuture(COMPACTION_NEVER_RUN); - private final CompactedTopic compactedTopic; + private static final Long COMPACTION_NEVER_RUN = -0xfebecffeL; + private volatile CompletableFuture currentCompaction = CompletableFuture.completedFuture( + COMPACTION_NEVER_RUN); + private TopicCompactionService topicCompactionService; // TODO: Create compaction strategy from topic policy when exposing strategic compaction to users. private static Map strategicCompactionMap = Map.of( - ServiceUnitStateChannelImpl.TOPIC, - new ServiceUnitStateCompactionStrategy()); + TOPIC, + new ServiceUnitStateDataConflictResolver()); private CompletableFuture currentOffload = CompletableFuture.completedFuture( (MessageIdImpl) MessageId.earliest); @@ -242,12 +273,93 @@ protected TopicStatsHelper initialValue() { @Getter protected final TransactionBuffer transactionBuffer; + @Getter + private final TopicTransactionBuffer.MaxReadPositionCallBack maxReadPositionCallBack = + (oldPosition, newPosition) -> updateMaxReadPositionMovedForwardTimestamp(); - // Record the last time a data message (ie: not an internal Pulsar marker) is published on the topic - private volatile long lastDataMessagePublishedTimestamp = 0; + // Record the last time max read position is moved forward, unless it's a marker message. + @Getter + private volatile long lastMaxReadPositionMovedForwardTimestamp = 0; @Getter private final ExecutorService orderedExecutor; + private volatile CloseFutures closeFutures; + + @Getter + private final PersistentTopicMetrics persistentTopicMetrics = new PersistentTopicMetrics(); + + private volatile PersistentTopicAttributes persistentTopicAttributes = null; + private static final AtomicReferenceFieldUpdater + PERSISTENT_TOPIC_ATTRIBUTES_FIELD_UPDATER = AtomicReferenceFieldUpdater.newUpdater( + PersistentTopic.class, PersistentTopicAttributes.class, "persistentTopicAttributes"); + + private volatile TimeBasedBacklogQuotaCheckResult timeBasedBacklogQuotaCheckResult; + private static final AtomicReferenceFieldUpdater + TIME_BASED_BACKLOG_QUOTA_CHECK_RESULT_UPDATER = AtomicReferenceFieldUpdater.newUpdater( + PersistentTopic.class, + TimeBasedBacklogQuotaCheckResult.class, + "timeBasedBacklogQuotaCheckResult"); + @Value + private static class TimeBasedBacklogQuotaCheckResult { + Position oldestCursorMarkDeletePosition; + String cursorName; + long positionPublishTimestampInMillis; + long dataVersion; + } + + @Value + private static class EstimateTimeBasedBacklogQuotaCheckResult { + boolean truncateBacklogToMatchQuota; + Long estimatedOldestUnacknowledgedMessageTimestamp; + } + + // The last position that can be dispatched to consumers + private volatile Position lastDispatchablePosition; + + /*** + * We use 3 futures to prevent a new closing if there is an in-progress deletion or closing. We make Pulsar return + * the in-progress one when it is called the second time. + * + * The topic closing will be called the below scenarios: + * 1. Calling "pulsar-admin topics unload". Relate to {@link CloseFutures#waitDisconnectClients}. + * 2. Namespace bundle transfer or unloading. + * a. The unloading topic triggered by unloading namespace bundles will not wait for clients disconnect. Relate + * to {@link CloseFutures#notWaitDisconnectClients}. + * b. The unloading topic triggered by unloading namespace bundles was seperated to two steps when using + * {@link ExtensibleLoadManagerImpl}. + * b-1. step-1: fence the topic on the original Broker, and do not trigger reconnections of clients. Relate + * to {@link CloseFutures#transferring}. This step is a half closing. + * b-2. step-2: send the owner broker information to clients and disconnect clients. Relate + * to {@link CloseFutures#notWaitDisconnectClients}. + * + * The three futures will be setting as the below rule: + * Event: Topic close. + * - If the first one closing is called by "close and not disconnect clients": + * - {@link CloseFutures#transferring} will be initialized as "close and not disconnect clients". + * - {@link CloseFutures#waitDisconnectClients} ang {@link CloseFutures#notWaitDisconnectClients} will be empty, + * the second closing will do a new close after {@link CloseFutures#transferring} is completed. + * - If the first one closing is called by "close and not wait for clients disconnect": + * - {@link CloseFutures#waitDisconnectClients} will be initialized as "waiting for clients disconnect". + * - {@link CloseFutures#notWaitDisconnectClients} ang {@link CloseFutures#transferring} will be + * initialized as "not waiting for clients disconnect" . + * - If the first one closing is called by "close and wait for clients disconnect", the three futures will be + * initialized as "waiting for clients disconnect". + * Event: Topic delete. + * the three futures will be initialized as "waiting for clients disconnect". + */ + private class CloseFutures { + private final CompletableFuture transferring; + private final CompletableFuture notWaitDisconnectClients; + private final CompletableFuture waitDisconnectClients; + + public CloseFutures(CompletableFuture transferring, CompletableFuture waitDisconnectClients, + CompletableFuture notWaitDisconnectClients) { + this.transferring = transferring; + this.waitDisconnectClients = waitDisconnectClients; + this.notWaitDisconnectClients = notWaitDisconnectClients; + } + } + private static class TopicStatsHelper { public double averageMsgSize; public double aggMsgRateIn; @@ -280,79 +392,69 @@ public PersistentTopic(String topic, ManagedLedger ledger, BrokerService brokerS ? brokerService.getTopicOrderedExecutor().chooseThread(topic) : null; this.ledger = ledger; - this.subscriptions = ConcurrentOpenHashMap.newBuilder() - .expectedItems(16) - .concurrencyLevel(1) - .build(); - this.replicators = ConcurrentOpenHashMap.newBuilder() - .expectedItems(16) - .concurrencyLevel(1) - .build(); - this.shadowReplicators = ConcurrentOpenHashMap.newBuilder() - .expectedItems(16) - .concurrencyLevel(1) - .build(); this.backloggedCursorThresholdEntries = brokerService.pulsar().getConfiguration().getManagedLedgerCursorBackloggedThreshold(); - registerTopicPolicyListener(); - - this.compactedTopic = new CompactedTopicImpl(brokerService.pulsar().getBookKeeperClient()); - - for (ManagedCursor cursor : ledger.getCursors()) { - if (cursor.getName().equals(DEDUPLICATION_CURSOR_NAME) - || cursor.getName().startsWith(replicatorPrefix)) { - // This is not a regular subscription, we are going to - // ignore it for now and let the message dedup logic to take care of it - } else { - final String subscriptionName = Codec.decode(cursor.getName()); - subscriptions.put(subscriptionName, createPersistentSubscription(subscriptionName, cursor, - PersistentSubscription.isCursorFromReplicatedSubscription(cursor), - cursor.getCursorProperties())); - // subscription-cursor gets activated by default: deactivate as there is no active subscription right - // now - subscriptions.get(subscriptionName).deactivateCursor(); - } - } this.messageDeduplication = new MessageDeduplication(brokerService.pulsar(), this, ledger); if (ledger.getProperties().containsKey(TOPIC_EPOCH_PROPERTY_NAME)) { topicEpoch = Optional.of(Long.parseLong(ledger.getProperties().get(TOPIC_EPOCH_PROPERTY_NAME))); } - checkReplicatedSubscriptionControllerState(); TopicName topicName = TopicName.get(topic); if (brokerService.getPulsar().getConfiguration().isTransactionCoordinatorEnabled() - && !isEventSystemTopic(topicName)) { + && !isEventSystemTopic(topicName) + && !NamespaceService.isHeartbeatNamespace(topicName.getNamespaceObject()) + && !ExtensibleLoadManagerImpl.isInternalTopic(topic)) { this.transactionBuffer = brokerService.getPulsar() .getTransactionBufferProvider().newTransactionBuffer(this); } else { - this.transactionBuffer = new TransactionBufferDisable(); + this.transactionBuffer = new TransactionBufferDisable(this); } - transactionBuffer.syncMaxReadPositionForNormalPublish((PositionImpl) ledger.getLastConfirmedEntry()); - if (ledger instanceof ShadowManagedLedgerImpl) { + transactionBuffer.syncMaxReadPositionForNormalPublish(ledger.getLastConfirmedEntry(), true); + if (ledger.getConfig().getShadowSource() != null) { shadowSourceTopic = TopicName.get(ledger.getConfig().getShadowSource()); } else { shadowSourceTopic = null; } } + @VisibleForTesting + PersistentTopic(String topic, BrokerService brokerService, ManagedLedger ledger, + MessageDeduplication messageDeduplication) { + super(topic, brokerService); + // null check for backwards compatibility with tests which mock the broker service + this.orderedExecutor = brokerService.getTopicOrderedExecutor() != null + ? brokerService.getTopicOrderedExecutor().chooseThread(topic) + : null; + this.ledger = ledger; + this.messageDeduplication = messageDeduplication; + this.backloggedCursorThresholdEntries = + brokerService.pulsar().getConfiguration().getManagedLedgerCursorBackloggedThreshold(); + + if (brokerService.pulsar().getConfiguration().isTransactionCoordinatorEnabled()) { + this.transactionBuffer = brokerService.getPulsar() + .getTransactionBufferProvider().newTransactionBuffer(this); + } else { + this.transactionBuffer = new TransactionBufferDisable(this); + } + shadowSourceTopic = null; + } + @Override public CompletableFuture initialize() { List> futures = new ArrayList<>(); - for (ManagedCursor cursor : ledger.getCursors()) { - if (cursor.getName().startsWith(replicatorPrefix)) { - String localCluster = brokerService.pulsar().getConfiguration().getClusterName(); - String remoteCluster = PersistentReplicator.getRemoteCluster(cursor.getName()); - futures.add(addReplicationCluster(remoteCluster, cursor, localCluster)); - } - } + futures.add(brokerService.getPulsar().newTopicCompactionService(topic).thenAccept(service -> { + PersistentTopic.this.topicCompactionService = service; + this.createPersistentSubscriptions(); + })); + return FutureUtil.waitForAll(futures).thenCompose(__ -> brokerService.pulsar().getPulsarResources().getNamespaceResources() .getPoliciesAsync(TopicName.get(topic).getNamespaceObject()) .thenAcceptAsync(optPolicies -> { if (!optPolicies.isPresent()) { isEncryptionRequired = false; - updatePublishDispatcher(); - updateResourceGroupLimiter(optPolicies); + updatePublishRateLimiter(); + updateResourceGroupLimiter(new Policies()); initializeDispatchRateLimiterIfNeeded(); updateSubscribeRateLimiter(); return; @@ -366,15 +468,16 @@ public CompletableFuture initialize() { updateSubscribeRateLimiter(); - updatePublishDispatcher(); + updatePublishRateLimiter(); - updateResourceGroupLimiter(optPolicies); + updateResourceGroupLimiter(policies); this.isEncryptionRequired = policies.encryption_required; isAllowAutoUpdateSchema = policies.is_allow_auto_update_schema; }, getOrderedExecutor()) .thenCompose(ignore -> initTopicPolicy()) + .thenCompose(ignore -> removeOrphanReplicationCursors()) .exceptionally(ex -> { log.warn("[{}] Error getting policies {} and isEncryptionRequired will be set to false", topic, ex.getMessage()); @@ -383,42 +486,6 @@ public CompletableFuture initialize() { })); } - // for testing purposes - @VisibleForTesting - PersistentTopic(String topic, BrokerService brokerService, ManagedLedger ledger, - MessageDeduplication messageDeduplication) { - super(topic, brokerService); - // null check for backwards compatibility with tests which mock the broker service - this.orderedExecutor = brokerService.getTopicOrderedExecutor() != null - ? brokerService.getTopicOrderedExecutor().chooseThread(topic) - : null; - this.ledger = ledger; - this.messageDeduplication = messageDeduplication; - this.subscriptions = ConcurrentOpenHashMap.newBuilder() - .expectedItems(16) - .concurrencyLevel(1) - .build(); - this.replicators = ConcurrentOpenHashMap.newBuilder() - .expectedItems(16) - .concurrencyLevel(1) - .build(); - this.shadowReplicators = ConcurrentOpenHashMap.newBuilder() - .expectedItems(16) - .concurrencyLevel(1) - .build(); - this.compactedTopic = new CompactedTopicImpl(brokerService.pulsar().getBookKeeperClient()); - this.backloggedCursorThresholdEntries = - brokerService.pulsar().getConfiguration().getManagedLedgerCursorBackloggedThreshold(); - - if (brokerService.pulsar().getConfiguration().isTransactionCoordinatorEnabled()) { - this.transactionBuffer = brokerService.getPulsar() - .getTransactionBufferProvider().newTransactionBuffer(this); - } else { - this.transactionBuffer = new TransactionBufferDisable(); - } - shadowSourceTopic = null; - } - private void initializeDispatchRateLimiterIfNeeded() { synchronized (dispatchRateLimiterLock) { // dispatch rate limiter for topic @@ -434,6 +501,40 @@ public AtomicLong getPendingWriteOps() { return pendingWriteOps; } + private void createPersistentSubscriptions() { + for (ManagedCursor cursor : ledger.getCursors()) { + if (cursor.getName().equals(DEDUPLICATION_CURSOR_NAME) + || cursor.getName().startsWith(replicatorPrefix)) { + // This is not a regular subscription, we are going to + // ignore it for now and let the message dedup logic to take care of it + } else { + final String subscriptionName = Codec.decode(cursor.getName()); + subscriptions.put(subscriptionName, createPersistentSubscription(subscriptionName, cursor, + PersistentSubscription.isCursorFromReplicatedSubscription(cursor), + cursor.getCursorProperties())); + // subscription-cursor gets activated by default: deactivate as there is no active subscription + // right now + subscriptions.get(subscriptionName).deactivateCursor(); + } + } + checkReplicatedSubscriptionControllerState(); + } + + private CompletableFuture removeOrphanReplicationCursors() { + List> futures = new ArrayList<>(); + List replicationClusters = topicPolicies.getReplicationClusters().get(); + for (ManagedCursor cursor : ledger.getCursors()) { + if (cursor.getName().startsWith(replicatorPrefix)) { + String remoteCluster = PersistentReplicator.getRemoteCluster(cursor.getName()); + if (!replicationClusters.contains(remoteCluster)) { + log.warn("Remove the orphan replicator because the cluster '{}' does not exist", remoteCluster); + futures.add(removeReplicator(remoteCluster)); + } + } + } + return FutureUtil.waitForAll(futures); + } + /** * Unload a subscriber. * @throws SubscriptionNotFoundException If subscription not founded. @@ -452,11 +553,11 @@ public CompletableFuture unloadSubscription(@Nonnull String subName) { new UnsupportedSubscriptionException(String.format("Unsupported subscription: %s", subName))); } // Fence old subscription -> Rewind cursor -> Replace with a new subscription. - return sub.disconnect().thenCompose(ignore -> { + return sub.close(true, Optional.empty()).thenCompose(ignore -> { if (!lock.writeLock().tryLock()) { return CompletableFuture.failedFuture(new SubscriptionConflictUnloadException(String.format("Conflict" + " topic-close, topic-delete, another-subscribe-unload, cannot unload subscription %s now", - topic, subName))); + subName))); } try { if (isFenced) { @@ -481,15 +582,17 @@ public CompletableFuture unloadSubscription(@Nonnull String subName) { private PersistentSubscription createPersistentSubscription(String subscriptionName, ManagedCursor cursor, boolean replicated, Map subscriptionProperties) { - Objects.requireNonNull(compactedTopic); - if (isCompactionSubscription(subscriptionName)) { - return new CompactorSubscription(this, compactedTopic, subscriptionName, cursor); + requireNonNull(topicCompactionService); + if (isCompactionSubscription(subscriptionName) + && topicCompactionService instanceof PulsarTopicCompactionService pulsarTopicCompactionService) { + CompactedTopicImpl compactedTopic = pulsarTopicCompactionService.getCompactedTopic(); + return new PulsarCompactorSubscription(this, compactedTopic, subscriptionName, cursor); } else { return new PersistentSubscription(this, subscriptionName, cursor, replicated, subscriptionProperties); } } - private static boolean isCompactionSubscription(String subscriptionName) { + public static boolean isCompactionSubscription(String subscriptionName) { return COMPACTION_SUBSCRIPTION.equals(subscriptionName); } @@ -502,8 +605,15 @@ public void publishMessage(ByteBuf headersAndPayload, PublishContext publishCont return; } if (isExceedMaximumMessageSize(headersAndPayload.readableBytes(), publishContext)) { - publishContext.completed(new NotAllowedException("Exceed maximum message size") - , -1, -1); + publishContext.completed(new NotAllowedException("Exceed maximum message size"), -1, -1); + decrementPendingWriteOpsAndCheck(); + return; + } + if (isExceedMaximumDeliveryDelay(headersAndPayload)) { + publishContext.completed( + new NotAllowedException( + String.format("Exceeds max allowed delivery delay of %s milliseconds", + getDelayedDeliveryMaxDelayInMillis())), -1, -1); decrementPendingWriteOpsAndCheck(); return; } @@ -545,41 +655,20 @@ public void updateSubscribeRateLimiter() { } private void asyncAddEntry(ByteBuf headersAndPayload, PublishContext publishContext) { - if (brokerService.isBrokerEntryMetadataEnabled()) { - ledger.asyncAddEntry(headersAndPayload, - (int) publishContext.getNumberOfMessages(), this, publishContext); - } else { - ledger.asyncAddEntry(headersAndPayload, this, publishContext); - } + ledger.asyncAddEntry(headersAndPayload, + (int) publishContext.getNumberOfMessages(), this, publishContext); } - public void asyncReadEntry(PositionImpl position, AsyncCallbacks.ReadEntryCallback callback, Object ctx) { - if (ledger instanceof ManagedLedgerImpl) { - ((ManagedLedgerImpl) ledger).asyncReadEntry(position, callback, ctx); - } else { - callback.readEntryFailed(new ManagedLedgerException( - "Unexpected managedledger implementation, doesn't support " - + "direct read entry operation."), ctx); - } + public void asyncReadEntry(Position position, AsyncCallbacks.ReadEntryCallback callback, Object ctx) { + ledger.asyncReadEntry(position, callback, ctx); } - public PositionImpl getPositionAfterN(PositionImpl startPosition, long n) throws ManagedLedgerException { - if (ledger instanceof ManagedLedgerImpl) { - return ((ManagedLedgerImpl) ledger).getPositionAfterN(startPosition, n, - ManagedLedgerImpl.PositionBound.startExcluded); - } else { - throw new ManagedLedgerException("Unexpected managedledger implementation, doesn't support " - + "getPositionAfterN operation."); - } + public Position getPositionAfterN(Position startPosition, long n) throws ManagedLedgerException { + return ledger.getPositionAfterN(startPosition, n, PositionBound.startExcluded); } - public PositionImpl getFirstPosition() throws ManagedLedgerException { - if (ledger instanceof ManagedLedgerImpl) { - return ((ManagedLedgerImpl) ledger).getFirstPosition(); - } else { - throw new ManagedLedgerException("Unexpected managedledger implementation, doesn't support " - + "getFirstPosition operation."); - } + public Position getFirstPosition() throws ManagedLedgerException { + return ledger.getFirstPosition(); } public long getNumberOfEntries() { @@ -603,20 +692,21 @@ private void decrementPendingWriteOpsAndCheck() { } } + private void updateMaxReadPositionMovedForwardTimestamp() { + lastMaxReadPositionMovedForwardTimestamp = Clock.systemUTC().millis(); + } + @Override public void addComplete(Position pos, ByteBuf entryData, Object ctx) { PublishContext publishContext = (PublishContext) ctx; - PositionImpl position = (PositionImpl) pos; + Position position = pos; // Message has been successfully persisted messageDeduplication.recordMessagePersisted(publishContext, position); - if (!publishContext.isMarkerMessage()) { - lastDataMessagePublishedTimestamp = Clock.systemUTC().millis(); - } - // in order to sync the max position when cursor read entries - transactionBuffer.syncMaxReadPositionForNormalPublish((PositionImpl) ledger.getLastConfirmedEntry()); + transactionBuffer.syncMaxReadPositionForNormalPublish(ledger.getLastConfirmedEntry(), + publishContext.isMarkerMessage()); publishContext.setMetadataFromEntryData(entryData); publishContext.completed(null, position.getLedgerId(), position.getEntryId()); decrementPendingWriteOpsAndCheck(); @@ -624,6 +714,19 @@ public void addComplete(Position pos, ByteBuf entryData, Object ctx) { @Override public synchronized void addFailed(ManagedLedgerException exception, Object ctx) { + /* If the topic is being transferred(in the Releasing bundle state), + we don't want to forcefully close topic here. + Instead, we will rely on the service unit state channel's bundle(topic) transfer protocol. + At the end of the transfer protocol, at Owned state, the source broker should close the topic properly. + */ + if (transferring) { + if (log.isDebugEnabled()) { + log.debug("[{}] Failed to persist msg in store: {} while transferring.", + topic, exception.getMessage(), exception); + } + return; + } + PublishContext callback = (PublishContext) ctx; if (exception instanceof ManagedLedgerFencedException) { // If the managed ledger has been fenced, we cannot continue using it. We need to close and reopen @@ -637,8 +740,9 @@ public synchronized void addFailed(ManagedLedgerException exception, Object ctx) List> futures = new ArrayList<>(); // send migration url metadata to producers before disconnecting them if (isMigrated()) { - if (isReplicationBacklogExist()) { - log.info("Topic {} is migrated but replication backlog exists. Closing producers.", topic); + if (!shouldProducerMigrate()) { + log.info("Topic {} is migrated but replication-backlog exists or " + + "subs not created. Closing producers.", topic); } else { producers.forEach((__, producer) -> producer.topicMigrated(getMigratedClusterUrl())); } @@ -687,8 +791,8 @@ public CompletableFuture> addProducer(Producer producer, } @Override - public CompletableFuture checkIfTransactionBufferRecoverCompletely(boolean isTxnEnabled) { - return getTransactionBuffer().checkIfTBRecoverCompletely(isTxnEnabled); + public CompletableFuture checkIfTransactionBufferRecoverCompletely() { + return getTransactionBuffer().checkIfTBRecoverCompletely(); } @Override @@ -757,15 +861,15 @@ public CompletableFuture startReplProducers() { public CompletableFuture stopReplProducers() { List> closeFutures = new ArrayList<>(); - replicators.forEach((region, replicator) -> closeFutures.add(replicator.disconnect())); - shadowReplicators.forEach((__, replicator) -> closeFutures.add(replicator.disconnect())); + replicators.forEach((region, replicator) -> closeFutures.add(replicator.terminate())); + shadowReplicators.forEach((__, replicator) -> closeFutures.add(replicator.terminate())); return FutureUtil.waitForAll(closeFutures); } private synchronized CompletableFuture closeReplProducersIfNoBacklog() { List> closeFutures = new ArrayList<>(); - replicators.forEach((region, replicator) -> closeFutures.add(replicator.disconnect(true))); - shadowReplicators.forEach((__, replicator) -> closeFutures.add(replicator.disconnect(true))); + replicators.forEach((region, replicator) -> closeFutures.add(replicator.disconnect(true, true))); + shadowReplicators.forEach((__, replicator) -> closeFutures.add(replicator.disconnect(true, true))); return FutureUtil.waitForAll(closeFutures); } @@ -819,7 +923,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St } try { - if (!topic.endsWith(SystemTopicNames.NAMESPACE_EVENTS_LOCAL_NAME) + if (!SystemTopicNames.isTopicPoliciesSystemTopic(topic) && !checkSubscriptionTypesEnable(subType)) { return FutureUtil.failedFuture( new NotAllowedException("Topic[{" + topic + "}] doesn't support " @@ -876,8 +980,8 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St lock.readLock().unlock(); } - CompletableFuture subscriptionFuture = isDurable ? // - getDurableSubscription(subscriptionName, initialPosition, startMessageRollbackDurationSec, + CompletableFuture subscriptionFuture = isDurable + ? getDurableSubscription(subscriptionName, initialPosition, startMessageRollbackDurationSec, replicatedSubscriptionState, subscriptionProperties) : getNonDurableSubscription(subscriptionName, startMessageId, initialPosition, startMessageRollbackDurationSec, readCompacted, subscriptionProperties); @@ -896,8 +1000,8 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St consumer.close(); } catch (BrokerServiceException e) { if (e instanceof ConsumerBusyException) { - log.warn("[{}][{}] Consumer {} {} already connected", - topic, subscriptionName, consumerId, consumerName); + log.warn("[{}][{}] Consumer {} {} already connected: {}", + topic, subscriptionName, consumerId, consumerName, e.getMessage()); } else if (e instanceof SubscriptionBusyException) { log.warn("[{}][{}] {}", topic, subscriptionName, e.getMessage()); } @@ -927,8 +1031,8 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St decrementUsageCount(); if (ex.getCause() instanceof ConsumerBusyException) { - log.warn("[{}][{}] Consumer {} {} already connected", topic, subscriptionName, consumerId, - consumerName); + log.warn("[{}][{}] Consumer {} {} already connected: {}", topic, subscriptionName, consumerId, + consumerName, ex.getCause().getMessage()); Consumer consumer = null; try { consumer = subscriptionFuture.isDone() ? getActiveConsumer(subscriptionFuture.get()) : null; @@ -968,7 +1072,9 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs } private CompletableFuture getDurableSubscription(String subscriptionName, - InitialPosition initialPosition, long startMessageRollbackDurationSec, boolean replicated, + InitialPosition initialPosition, + long startMessageRollbackDurationSec, + boolean replicated, Map subscriptionProperties) { CompletableFuture subscriptionFuture = new CompletableFuture<>(); if (checkMaxSubscriptionsPerTopicExceed(subscriptionName)) { @@ -978,7 +1084,6 @@ private CompletableFuture getDurableSubscription(String subscripti } Map properties = PersistentSubscription.getBaseCursorProperties(replicated); - ledger.asyncOpenCursor(Codec.encode(subscriptionName), initialPosition, properties, subscriptionProperties, new OpenCursorCallback() { @Override @@ -1060,7 +1165,7 @@ private CompletableFuture getNonDurableSubscription(Stri entryId = msgId.getEntryId() - 1; } - Position startPosition = new PositionImpl(ledgerId, entryId); + Position startPosition = PositionFactory.create(ledgerId, entryId); ManagedCursor cursor = null; try { cursor = ledger.newNonDurableCursor(startPosition, subscriptionName, initialPosition, @@ -1092,7 +1197,7 @@ private CompletableFuture getNonDurableSubscription(Stri private void resetSubscriptionCursor(Subscription subscription, CompletableFuture subscriptionFuture, long startMessageRollbackDurationSec) { long timestamp = System.currentTimeMillis() - - TimeUnit.SECONDS.toMillis(startMessageRollbackDurationSec); + - SECONDS.toMillis(startMessageRollbackDurationSec); final Subscription finalSubscription = subscription; subscription.resetCursor(timestamp).handle((s, ex) -> { if (ex != null) { @@ -1156,16 +1261,16 @@ public void deleteLedgerFailed(ManagedLedgerException exception, Object ctx) { private void asyncDeleteCursorWithClearDelayedMessage(String subscriptionName, CompletableFuture unsubscribeFuture) { - if (!isDelayedDeliveryEnabled() - || !(brokerService.getDelayedDeliveryTrackerFactory() instanceof BucketDelayedDeliveryTrackerFactory)) { - asyncDeleteCursor(subscriptionName, unsubscribeFuture); + PersistentSubscription persistentSubscription = subscriptions.get(subscriptionName); + if (persistentSubscription == null) { + log.warn("[{}][{}] Can't find subscription, skip delete cursor", topic, subscriptionName); + unsubscribeFuture.complete(null); return; } - PersistentSubscription persistentSubscription = subscriptions.get(subscriptionName); - if (persistentSubscription == null) { - log.warn("[{}][{}] Can't find subscription, skip clear delayed message", topic, subscriptionName); - asyncDeleteCursor(subscriptionName, unsubscribeFuture); + if (!isDelayedDeliveryEnabled() + || !(brokerService.getDelayedDeliveryTrackerFactory() instanceof BucketDelayedDeliveryTrackerFactory)) { + asyncDeleteCursorWithCleanCompactionLedger(persistentSubscription, unsubscribeFuture); return; } @@ -1180,7 +1285,7 @@ private void asyncDeleteCursorWithClearDelayedMessage(String subscriptionName, if (ex != null) { unsubscribeFuture.completeExceptionally(ex); } else { - asyncDeleteCursor(subscriptionName, unsubscribeFuture); + asyncDeleteCursorWithCleanCompactionLedger(persistentSubscription, unsubscribeFuture); } }); } @@ -1190,6 +1295,29 @@ private void asyncDeleteCursorWithClearDelayedMessage(String subscriptionName, dispatcher.clearDelayedMessages().whenComplete((__, ex) -> { if (ex != null) { unsubscribeFuture.completeExceptionally(ex); + } else { + asyncDeleteCursorWithCleanCompactionLedger(persistentSubscription, unsubscribeFuture); + } + }); + } + + private void asyncDeleteCursorWithCleanCompactionLedger(PersistentSubscription subscription, + CompletableFuture unsubscribeFuture) { + final String subscriptionName = subscription.getName(); + if ((!isCompactionSubscription(subscriptionName)) || !(subscription instanceof PulsarCompactorSubscription)) { + asyncDeleteCursor(subscriptionName, unsubscribeFuture); + return; + } + + currentCompaction.handle((__, e) -> { + if (e != null) { + log.warn("[{}][{}] Last compaction task failed", topic, subscriptionName); + } + return ((PulsarCompactorSubscription) subscription).cleanCompactedLedger(); + }).whenComplete((__, ex) -> { + if (ex != null) { + log.error("[{}][{}] Error cleaning compacted ledger", topic, subscriptionName, ex); + unsubscribeFuture.completeExceptionally(ex); } else { asyncDeleteCursor(subscriptionName, unsubscribeFuture); } @@ -1224,14 +1352,23 @@ public void deleteCursorFailed(ManagedLedgerException exception, Object ctx) { }, null); } - void removeSubscription(String subscriptionName) { + CompletableFuture removeSubscription(String subscriptionName) { PersistentSubscription sub = subscriptions.remove(subscriptionName); if (sub != null) { // preserve accumulative stats form removed subscription - SubscriptionStatsImpl stats = sub.getStats(false, false, false); - bytesOutFromRemovedSubscriptions.add(stats.bytesOutCounter); - msgOutFromRemovedSubscriptions.add(stats.msgOutCounter); + return sub + .getStatsAsync(new GetStatsOptions(false, false, false, false, false)) + .thenAccept(stats -> { + bytesOutFromRemovedSubscriptions.add(stats.bytesOutCounter); + msgOutFromRemovedSubscriptions.add(stats.msgOutCounter); + + if (isSystemCursor(subscriptionName) + || subscriptionName.startsWith(SystemTopicNames.SYSTEM_READER_PREFIX)) { + bytesOutFromRemovedSystemSubscriptions.add(stats.bytesOutCounter); + } + }); } + return CompletableFuture.completedFuture(null); } /** @@ -1293,10 +1430,10 @@ private CompletableFuture delete(boolean failIfHasSubscriptions, // In this case, we shouldn't care if the usageCount is 0 or not, just proceed if (!closeIfClientsConnected) { if (failIfHasSubscriptions && !subscriptions.isEmpty()) { - return FutureUtil.failedFuture( - new TopicBusyException("Topic has subscriptions: " + subscriptions.keys())); + return FutureUtil.failedFuture(new TopicBusyException("Topic has subscriptions: " + + subscriptions.keySet().stream().toList())); } else if (failIfHasBacklogs) { - if (hasBacklogs()) { + if (hasBacklogs(false)) { List backlogSubs = subscriptions.values().stream() .filter(sub -> sub.getNumberOfEntriesInBacklog(false) > 0) @@ -1314,22 +1451,32 @@ private CompletableFuture delete(boolean failIfHasSubscriptions, } fenceTopicToCloseOrDelete(); // Avoid clients reconnections while deleting + // Mark the progress of close to prevent close calling concurrently. + this.closeFutures = + new CloseFutures(new CompletableFuture(), new CompletableFuture(), new CompletableFuture()); - return getBrokerService().getPulsar().getPulsarResources().getNamespaceResources() + CompletableFuture res = getBrokerService().getPulsar().getPulsarResources().getNamespaceResources() .getPartitionedTopicResources().runWithMarkDeleteAsync(TopicName.get(topic), () -> { CompletableFuture deleteFuture = new CompletableFuture<>(); CompletableFuture closeClientFuture = new CompletableFuture<>(); List> futures = new ArrayList<>(); - subscriptions.forEach((s, sub) -> futures.add(sub.disconnect())); + subscriptions.forEach((s, sub) -> futures.add(sub.close(true, Optional.empty()))); if (closeIfClientsConnected) { - replicators.forEach((cluster, replicator) -> futures.add(replicator.disconnect())); - shadowReplicators.forEach((__, replicator) -> futures.add(replicator.disconnect())); + replicators.forEach((cluster, replicator) -> futures.add(replicator.terminate())); + shadowReplicators.forEach((__, replicator) -> futures.add(replicator.terminate())); producers.values().forEach(producer -> futures.add(producer.disconnect())); } FutureUtil.waitForAll(futures).thenRunAsync(() -> { closeClientFuture.complete(null); - }, getOrderedExecutor()).exceptionally(ex -> { + }, command -> { + try { + getOrderedExecutor().execute(command); + } catch (RejectedExecutionException e) { + // executor has been shut down, execute in current thread + command.run(); + } + }).exceptionally(ex -> { log.error("[{}] Error closing clients", topic, ex); unfenceTopicToResume(); closeClientFuture.completeExceptionally(ex); @@ -1341,14 +1488,7 @@ private CompletableFuture delete(boolean failIfHasSubscriptions, brokerService.deleteTopicAuthenticationWithRetry(topic, deleteTopicAuthenticationFuture, 5); deleteTopicAuthenticationFuture.thenCompose(ignore -> deleteSchema()) - .thenCompose(ignore -> { - if (!SystemTopicNames.isTopicPoliciesSystemTopic(topic) - && brokerService.getPulsar().getConfiguration().isSystemTopicEnabled()) { - return deleteTopicPolicies(); - } else { - return CompletableFuture.completedFuture(null); - } - }) + .thenCompose(ignore -> deleteTopicPolicies()) .thenCompose(ignore -> transactionBufferCleanupAndClose()) .whenComplete((v, ex) -> { if (ex != null) { @@ -1406,7 +1546,8 @@ public void deleteLedgerComplete(Object ctx) { }).exceptionally(ex->{ unfenceTopicToResume(); deleteFuture.completeExceptionally( - new TopicBusyException("Failed to close clients before deleting topic.")); + new TopicBusyException("Failed to close clients before deleting topic.", + FutureUtil.unwrapCompletionException(ex))); return null; }); @@ -1417,6 +1558,11 @@ public void deleteLedgerComplete(Object ctx) { unfenceTopicToResume(); } }); + + FutureUtil.completeAfter(closeFutures.transferring, res); + FutureUtil.completeAfter(closeFutures.notWaitDisconnectClients, res); + FutureUtil.completeAfter(closeFutures.waitDisconnectClients, res); + return res; } finally { lock.writeLock().unlock(); } @@ -1424,46 +1570,97 @@ public void deleteLedgerComplete(Object ctx) { } public CompletableFuture close() { - return close(false); + return close(true, false); + } + + @Override + public CompletableFuture close(boolean closeWithoutWaitingClientDisconnect) { + return close(true, closeWithoutWaitingClientDisconnect); + } + + private enum CloseTypes { + transferring, + notWaitDisconnectClients, + waitDisconnectClients; } /** * Close this topic - close all producers and subscriptions associated with this topic. * + * @param disconnectClients disconnect clients * @param closeWithoutWaitingClientDisconnect don't wait for client disconnect and forcefully close managed-ledger * @return Completable future indicating completion of close operation */ @Override - public CompletableFuture close(boolean closeWithoutWaitingClientDisconnect) { - CompletableFuture closeFuture = new CompletableFuture<>(); - + public CompletableFuture close( + boolean disconnectClients, boolean closeWithoutWaitingClientDisconnect) { lock.writeLock().lock(); - try { + // Choose the close type. + CloseTypes closeType; + if (!disconnectClients) { + closeType = CloseTypes.transferring; + } else if (closeWithoutWaitingClientDisconnect) { + closeType = CloseTypes.notWaitDisconnectClients; + } else { // closing managed-ledger waits until all producers/consumers/replicators get closed. Sometimes, broker // forcefully wants to close managed-ledger without waiting all resources to be closed. - if (!isClosingOrDeleting || closeWithoutWaitingClientDisconnect) { - fenceTopicToCloseOrDelete(); + closeType = CloseTypes.waitDisconnectClients; + } + /** Maybe there is a in-progress half closing task. see the section 2-b-1 of {@link CloseFutures}. **/ + CompletableFuture inProgressTransferCloseTask = null; + try { + // Return in-progress future if exists. + if (isClosingOrDeleting) { + if (closeType == CloseTypes.transferring) { + return closeFutures.transferring; + } + if (closeType == CloseTypes.notWaitDisconnectClients && closeFutures.notWaitDisconnectClients != null) { + return closeFutures.notWaitDisconnectClients; + } + if (closeType == CloseTypes.waitDisconnectClients && closeFutures.waitDisconnectClients != null) { + return closeFutures.waitDisconnectClients; + } + if (transferring) { + inProgressTransferCloseTask = closeFutures.transferring; + } + } + fenceTopicToCloseOrDelete(); + if (closeType == CloseTypes.transferring) { + transferring = true; + this.closeFutures = new CloseFutures(new CompletableFuture(), null, null); } else { - log.warn("[{}] Topic is already being closed or deleted", topic); - closeFuture.completeExceptionally(new TopicFencedException("Topic is already fenced")); - return closeFuture; + this.closeFutures = + new CloseFutures(new CompletableFuture(), new CompletableFuture(), new CompletableFuture()); } } finally { lock.writeLock().unlock(); } List> futures = new ArrayList<>(); + if (inProgressTransferCloseTask != null) { + futures.add(inProgressTransferCloseTask); + } futures.add(transactionBuffer.closeAsync()); - replicators.forEach((cluster, replicator) -> futures.add(replicator.disconnect())); - shadowReplicators.forEach((__, replicator) -> futures.add(replicator.disconnect())); - producers.values().forEach(producer -> futures.add(producer.disconnect())); - if (topicPublishRateLimiter != null) { - topicPublishRateLimiter.close(); - } - subscriptions.forEach((s, sub) -> futures.add(sub.disconnect())); - if (this.resourceGroupPublishLimiter != null) { - this.resourceGroupPublishLimiter.unregisterRateLimitFunction(this.getName()); + replicators.forEach((cluster, replicator) -> futures.add(replicator.terminate())); + shadowReplicators.forEach((__, replicator) -> futures.add(replicator.terminate())); + if (closeType != CloseTypes.transferring) { + futures.add(ExtensibleLoadManagerImpl.getAssignedBrokerLookupData( + brokerService.getPulsar(), topic).thenAccept(lookupData -> { + producers.values().forEach(producer -> futures.add(producer.disconnect(lookupData))); + // Topics unloaded due to the ExtensibleLoadManager undergo closing twice: first with + // disconnectClients = false, second with disconnectClients = true. The check below identifies the + // cases when Topic.close is called outside the scope of the ExtensibleLoadManager. In these + // situations, we must pursue the regular Subscription.close, as Topic.close is invoked just once. + if (isTransferring()) { + subscriptions.forEach((s, sub) -> futures.add(sub.disconnect(lookupData))); + } else { + subscriptions.forEach((s, sub) -> futures.add(sub.close(true, lookupData))); + } + } + )); + } else { + subscriptions.forEach((s, sub) -> futures.add(sub.close(false, Optional.empty()))); } //close entry filters @@ -1477,37 +1674,114 @@ public CompletableFuture close(boolean closeWithoutWaitingClientDisconnect }); } - CompletableFuture clientCloseFuture = closeWithoutWaitingClientDisconnect - ? CompletableFuture.completedFuture(null) - : FutureUtil.waitForAll(futures); + if (topicCompactionService != null) { + try { + topicCompactionService.close(); + } catch (Exception e) { + log.warn("Error close topicCompactionService ", e); + } + } - clientCloseFuture.thenRun(() -> { - // After having disconnected all producers/consumers, close the managed ledger - ledger.asyncClose(new CloseCallback() { - @Override - public void closeComplete(Object ctx) { + CompletableFuture disconnectClientsInCurrentCall = null; + // Note: "disconnectClientsToCache" is a non-able value, it is null when close type is transferring. + AtomicReference> disconnectClientsToCache = new AtomicReference<>(); + switch (closeType) { + case transferring -> { + disconnectClientsInCurrentCall = FutureUtil.waitForAll(futures); + break; + } + case notWaitDisconnectClients -> { + disconnectClientsInCurrentCall = CompletableFuture.completedFuture(null); + disconnectClientsToCache.set(FutureUtil.waitForAll(futures)); + break; + } + case waitDisconnectClients -> { + disconnectClientsInCurrentCall = FutureUtil.waitForAll(futures); + disconnectClientsToCache.set(disconnectClientsInCurrentCall); + } + } + + CompletableFuture closeFuture = new CompletableFuture<>(); + Runnable closeLedgerAfterCloseClients = (() -> ledger.asyncClose(new CloseCallback() { + @Override + public void closeComplete(Object ctx) { + if (closeType != CloseTypes.transferring) { // Everything is now closed, remove the topic from map disposeTopic(closeFuture); + } else { + closeFuture.complete(null); } + } - @Override - public void closeFailed(ManagedLedgerException exception, Object ctx) { - log.error("[{}] Failed to close managed ledger, proceeding anyway.", topic, exception); + @Override + public void closeFailed(ManagedLedgerException exception, Object ctx) { + log.error("[{}] Failed to close managed ledger, proceeding anyway.", topic, exception); + if (closeType != CloseTypes.transferring) { disposeTopic(closeFuture); + } else { + closeFuture.complete(null); } - }, null); - }).exceptionally(exception -> { + } + }, null)); + + disconnectClientsInCurrentCall.thenRun(closeLedgerAfterCloseClients).exceptionally(exception -> { log.error("[{}] Error closing topic", topic, exception); unfenceTopicToResume(); closeFuture.completeExceptionally(exception); return null; }); + switch (closeType) { + case transferring -> { + FutureUtil.completeAfterAll(closeFutures.transferring, closeFuture); + break; + } + case notWaitDisconnectClients -> { + FutureUtil.completeAfterAll(closeFutures.transferring, closeFuture); + FutureUtil.completeAfter(closeFutures.notWaitDisconnectClients, closeFuture); + FutureUtil.completeAfterAll(closeFutures.waitDisconnectClients, + closeFuture.thenCompose(ignore -> disconnectClientsToCache.get().exceptionally(ex -> { + // Since the managed ledger has been closed, eat the error of clients disconnection. + log.error("[{}] Closed managed ledger, but disconnect clients failed," + + " this topic will be marked closed", topic, ex); + return null; + }))); + break; + } + case waitDisconnectClients -> { + FutureUtil.completeAfterAll(closeFutures.transferring, closeFuture); + FutureUtil.completeAfter(closeFutures.notWaitDisconnectClients, closeFuture); + FutureUtil.completeAfterAll(closeFutures.waitDisconnectClients, closeFuture); + } + } + return closeFuture; } + private boolean isClosed() { + if (closeFutures == null) { + return false; + } + if (closeFutures.transferring != null + && closeFutures.transferring.isDone() + && !closeFutures.transferring.isCompletedExceptionally()) { + return true; + } + if (closeFutures.notWaitDisconnectClients != null + && closeFutures.notWaitDisconnectClients.isDone() + && !closeFutures.notWaitDisconnectClients.isCompletedExceptionally()) { + return true; + } + if (closeFutures.waitDisconnectClients != null + && closeFutures.waitDisconnectClients.isDone() + && !closeFutures.waitDisconnectClients.isCompletedExceptionally()) { + return true; + } + return false; + } + private void disposeTopic(CompletableFuture closeFuture) { - brokerService.removeTopicFromCache(topic) + brokerService.removeTopicFromCache(PersistentTopic.this) .thenRun(() -> { replicatedSubscriptionsController.ifPresent(ReplicatedSubscriptionsController::close); @@ -1528,9 +1802,11 @@ private void disposeTopic(CompletableFuture closeFuture) { @VisibleForTesting CompletableFuture checkReplicationAndRetryOnFailure() { + if (isClosed()) { + return CompletableFuture.completedFuture(null); + } CompletableFuture result = new CompletableFuture(); checkReplication().thenAccept(res -> { - log.info("[{}] Policies updated successfully", topic); result.complete(null); }).exceptionally(th -> { log.error("[{}] Policies update failed {}, scheduled retry in {} seconds", topic, th.getMessage(), @@ -1538,7 +1814,7 @@ CompletableFuture checkReplicationAndRetryOnFailure() { if (!(th.getCause() instanceof TopicFencedException)) { // retriable exception brokerService.executor().schedule(this::checkReplicationAndRetryOnFailure, - POLICY_UPDATE_FAILURE_RETRY_TIME_SECONDS, TimeUnit.SECONDS); + POLICY_UPDATE_FAILURE_RETRY_TIME_SECONDS, SECONDS); } result.completeExceptionally(th); return null; @@ -1550,7 +1826,8 @@ public CompletableFuture checkDeduplicationStatus() { return messageDeduplication.checkStatus(); } - private CompletableFuture checkPersistencePolicies() { + @VisibleForTesting + CompletableFuture checkPersistencePolicies() { TopicName topicName = TopicName.get(topic); CompletableFuture future = new CompletableFuture<>(); brokerService.getManagedLedgerConfig(topicName).thenAccept(config -> { @@ -1568,53 +1845,86 @@ private CompletableFuture checkPersistencePolicies() { @Override public CompletableFuture checkReplication() { TopicName name = TopicName.get(topic); - if (!name.isGlobal() || NamespaceService.isHeartbeatNamespace(name)) { + if (!name.isGlobal() || NamespaceService.isHeartbeatNamespace(name) + || ExtensibleLoadManagerImpl.isInternalTopic(topic)) { return CompletableFuture.completedFuture(null); } if (log.isDebugEnabled()) { log.debug("[{}] Checking replication status", name); } - List configuredClusters = topicPolicies.getReplicationClusters().get(); - int newMessageTTLInSeconds = topicPolicies.getMessageTTLInSeconds().get(); - - String localCluster = brokerService.pulsar().getConfiguration().getClusterName(); - - // if local cluster is removed from global namespace cluster-list : then delete topic forcefully - // because pulsar doesn't serve global topic without local repl-cluster configured. - if (TopicName.get(topic).isGlobal() && !configuredClusters.contains(localCluster)) { - log.info("Deleting topic [{}] because local cluster is not part of " - + " global namespace repl list {}", topic, configuredClusters); - return deleteForcefully(); + if (CollectionUtils.isEmpty(configuredClusters)) { + log.warn("[{}] No replication clusters configured", name); + return CompletableFuture.completedFuture(null); } - List> futures = new ArrayList<>(); + String localCluster = brokerService.pulsar().getConfiguration().getClusterName(); - // Check for missing replicators - for (String cluster : configuredClusters) { - if (cluster.equals(localCluster)) { - continue; + return checkAllowedCluster(localCluster).thenCompose(success -> { + if (!success) { + // if local cluster is removed from global namespace cluster-list : then delete topic forcefully + // because pulsar doesn't serve global topic without local repl-cluster configured. + return deleteForcefully(); } - if (!replicators.containsKey(cluster)) { - futures.add(startReplicator(cluster)); - } - } - // Check for replicators to be stopped - replicators.forEach((cluster, replicator) -> { - // Update message TTL - ((PersistentReplicator) replicator).updateMessageTTL(newMessageTTLInSeconds); - if (!cluster.equals(localCluster)) { - if (!configuredClusters.contains(cluster)) { - futures.add(removeReplicator(cluster)); + int newMessageTTLInSeconds = topicPolicies.getMessageTTLInSeconds().get(); + + removeTerminatedReplicators(replicators); + List> futures = new ArrayList<>(); + + // The replication clusters at namespace level will get local cluster when creating a namespace. + // If there are only one cluster in the replication clusters, it means the replication is not enabled. + // If the cluster 1 and cluster 2 use the same configuration store and the namespace is created in cluster1 + // without enabling geo-replication, then the replication clusters always has cluster1. + // + // When a topic under the namespace is load in the cluster2, the `cluster1` may be identified as + // remote cluster and start geo-replication. This check is to avoid the above case. + if (!(configuredClusters.size() == 1 && replicators.isEmpty())) { + // Check for missing replicators + for (String cluster : configuredClusters) { + if (cluster.equals(localCluster)) { + continue; + } + if (!replicators.containsKey(cluster)) { + futures.add(startReplicator(cluster)); + } } + // Check for replicators to be stopped + replicators.forEach((cluster, replicator) -> { + // Update message TTL + ((PersistentReplicator) replicator).updateMessageTTL(newMessageTTLInSeconds); + if (!cluster.equals(localCluster)) { + if (!configuredClusters.contains(cluster)) { + futures.add(removeReplicator(cluster)); + } + } + }); } - }); - futures.add(checkShadowReplication()); + futures.add(checkShadowReplication()); - return FutureUtil.waitForAll(futures); + return FutureUtil.waitForAll(futures); + }); + } + + private CompletableFuture checkAllowedCluster(String localCluster) { + List replicationClusters = topicPolicies.getReplicationClusters().get(); + return brokerService.pulsar().getPulsarResources().getNamespaceResources() + .getPoliciesAsync(TopicName.get(topic).getNamespaceObject()).thenCompose(policiesOptional -> { + Set allowedClusters = Set.of(); + if (policiesOptional.isPresent()) { + allowedClusters = policiesOptional.get().allowed_clusters; + } + if (TopicName.get(topic).isGlobal() && !replicationClusters.contains(localCluster) + && !allowedClusters.contains(localCluster)) { + log.warn("Local cluster {} is not part of global namespace repl list {} and allowed list {}", + localCluster, replicationClusters, allowedClusters); + return CompletableFuture.completedFuture(false); + } else { + return CompletableFuture.completedFuture(true); + } + }); } private CompletableFuture checkShadowReplication() { @@ -1627,6 +1937,8 @@ private CompletableFuture checkShadowReplication() { if (log.isDebugEnabled()) { log.debug("[{}] Checking shadow replication status, shadowTopics={}", topic, configuredShadowTopics); } + + removeTerminatedReplicators(shadowReplicators); List> futures = new ArrayList<>(); // Check for missing replicators @@ -1651,11 +1963,13 @@ private CompletableFuture checkShadowReplication() { public void checkMessageExpiry() { int messageTtlInSeconds = topicPolicies.getMessageTTLInSeconds().get(); if (messageTtlInSeconds != 0) { - subscriptions.forEach((__, sub) -> sub.expireMessages(messageTtlInSeconds)); - replicators.forEach((__, replicator) - -> ((PersistentReplicator) replicator).expireMessages(messageTtlInSeconds)); - shadowReplicators.forEach((__, replicator) - -> ((PersistentReplicator) replicator).expireMessages(messageTtlInSeconds)); + subscriptions.forEach((__, sub) -> { + if (!isCompactionSubscription(sub.getName()) + && (additionalSystemCursorNames.isEmpty() + || !additionalSystemCursorNames.contains(sub.getName()))) { + sub.expireMessages(messageTtlInSeconds); + } + }); } } @@ -1728,7 +2042,14 @@ CompletableFuture startReplicator(String remoteCluster) { final CompletableFuture future = new CompletableFuture<>(); String name = PersistentReplicator.getReplicatorName(replicatorPrefix, remoteCluster); - ledger.asyncOpenCursor(name, new OpenCursorCallback() { + final InitialPosition initialPosition; + if (MessageId.earliest.toString() + .equalsIgnoreCase(getBrokerService().getPulsar().getConfiguration().getReplicationStartAt())) { + initialPosition = InitialPosition.Earliest; + } else { + initialPosition = InitialPosition.Latest; + } + ledger.asyncOpenCursor(name, initialPosition, new OpenCursorCallback() { @Override public void openCursorComplete(ManagedCursor cursor, Object ctx) { String localCluster = brokerService.pulsar().getConfiguration().getClusterName(); @@ -1751,45 +2072,40 @@ public void openCursorFailed(ManagedLedgerException exception, Object ctx) { return future; } - private CompletableFuture checkReplicationCluster(String remoteCluster) { - return brokerService.getPulsar().getPulsarResources().getNamespaceResources() - .getPoliciesAsync(TopicName.get(topic).getNamespaceObject()) - .thenApply(optPolicies -> optPolicies.map(policies -> policies.replication_clusters) - .orElse(Collections.emptySet()).contains(remoteCluster) - || topicPolicies.getReplicationClusters().get().contains(remoteCluster)); - } - protected CompletableFuture addReplicationCluster(String remoteCluster, ManagedCursor cursor, String localCluster) { return AbstractReplicator.validatePartitionedTopicAsync(PersistentTopic.this.getName(), brokerService) - .thenCompose(__ -> checkReplicationCluster(remoteCluster)) - .thenCompose(clusterExists -> { - if (!clusterExists) { - log.warn("Remove the replicator because the cluster '{}' does not exist", remoteCluster); - return removeReplicator(remoteCluster).thenApply(__ -> null); - } - return brokerService.pulsar().getPulsarResources().getClusterResources() - .getClusterAsync(remoteCluster) - .thenApply(clusterData -> - brokerService.getReplicationClient(remoteCluster, clusterData)); - }) + .thenCompose(__ -> brokerService.pulsar().getPulsarResources().getClusterResources() + .getClusterAsync(remoteCluster) + .thenApply(clusterData -> + brokerService.getReplicationClient(remoteCluster, clusterData))) .thenAccept(replicationClient -> { if (replicationClient == null) { + log.error("[{}] Can not create replicator because the remote client can not be created." + + " remote cluster: {}. State of transferring : {}", + topic, remoteCluster, transferring); return; } - Replicator replicator = replicators.computeIfAbsent(remoteCluster, r -> { - try { - return new GeoPersistentReplicator(PersistentTopic.this, cursor, localCluster, - remoteCluster, brokerService, (PulsarClientImpl) replicationClient); - } catch (PulsarServerException e) { - log.error("[{}] Replicator startup failed {}", topic, remoteCluster, e); + lock.readLock().lock(); + try { + if (isClosingOrDeleting) { + // Whether is "transferring" or not, do not create new replicator. + log.info("[{}] Skip to create replicator because this topic is closing." + + " remote cluster: {}. State of transferring : {}", + topic, remoteCluster, transferring); + return; } - return null; - }); - - // clean up replicator if startup is failed - if (replicator == null) { - replicators.removeNullValue(remoteCluster); + Replicator replicator = replicators.computeIfAbsent(remoteCluster, r -> { + try { + return new GeoPersistentReplicator(PersistentTopic.this, cursor, localCluster, + remoteCluster, brokerService, (PulsarClientImpl) replicationClient); + } catch (PulsarServerException e) { + log.error("[{}] Replicator startup failed {}", topic, remoteCluster, e); + } + return null; + }); + } finally { + lock.readLock().unlock(); } }); } @@ -1800,7 +2116,7 @@ CompletableFuture removeReplicator(String remoteCluster) { String name = PersistentReplicator.getReplicatorName(replicatorPrefix, remoteCluster); - Optional.ofNullable(replicators.get(remoteCluster)).map(Replicator::disconnect) + Optional.ofNullable(replicators.get(remoteCluster)).map(Replicator::terminate) .orElse(CompletableFuture.completedFuture(null)).thenRun(() -> { ledger.asyncDeleteCursor(name, new DeleteCursorCallback() { @Override @@ -1831,7 +2147,7 @@ CompletableFuture startShadowReplicator(String shadowTopic) { String name = ShadowReplicator.getShadowReplicatorName(replicatorPrefix, shadowTopic); ManagedCursor cursor; try { - cursor = ledger.newNonDurableCursor(PositionImpl.LATEST, name); + cursor = ledger.newNonDurableCursor(PositionFactory.LATEST, name); } catch (ManagedLedgerException e) { log.error("[{}]Open non-durable cursor for shadow replicator failed, name={}", topic, name, e); return FutureUtil.failedFuture(e); @@ -1853,18 +2169,18 @@ protected CompletableFuture addShadowReplicationCluster(String shadowTopic .thenAccept(replicationClient -> { Replicator replicator = shadowReplicators.computeIfAbsent(shadowTopic, r -> { try { - return new ShadowReplicator(shadowTopic, PersistentTopic.this, cursor, brokerService, - (PulsarClientImpl) replicationClient); + TopicName sourceTopicName = TopicName.get(getName()); + String shadowPartitionTopic = shadowTopic; + if (sourceTopicName.isPartitioned()) { + shadowPartitionTopic += "-partition-" + sourceTopicName.getPartitionIndex(); + } + return new ShadowReplicator(shadowPartitionTopic, PersistentTopic.this, cursor, + brokerService, (PulsarClientImpl) replicationClient); } catch (PulsarServerException e) { log.error("[{}] ShadowReplicator startup failed {}", topic, shadowTopic, e); } return null; }); - - // clean up replicator if startup is failed - if (replicator == null) { - shadowReplicators.removeNullValue(shadowTopic); - } }); } @@ -1872,7 +2188,7 @@ CompletableFuture removeShadowReplicator(String shadowTopic) { log.info("[{}] Removing shadow topic replicator to {}", topic, shadowTopic); final CompletableFuture future = new CompletableFuture<>(); String name = ShadowReplicator.getShadowReplicatorName(replicatorPrefix, shadowTopic); - shadowReplicators.get(shadowTopic).disconnect().thenRun(() -> { + shadowReplicators.get(shadowTopic).terminate().thenRun(() -> { ledger.asyncDeleteCursor(name, new DeleteCursorCallback() { @Override @@ -1898,10 +2214,6 @@ public void deleteCursorFailed(ManagedLedgerException exception, Object ctx) { return future; } - public boolean isDeduplicationEnabled() { - return messageDeduplication.isEnabled(); - } - @Override public int getNumberOfConsumers() { int count = 0; @@ -1928,7 +2240,7 @@ protected String getSchemaId() { } @Override - public ConcurrentOpenHashMap getSubscriptions() { + public Map getSubscriptions() { return subscriptions; } @@ -1938,12 +2250,12 @@ public PersistentSubscription getSubscription(String subscriptionName) { } @Override - public ConcurrentOpenHashMap getReplicators() { + public Map getReplicators() { return replicators; } @Override - public ConcurrentOpenHashMap getShadowReplicators() { + public Map getShadowReplicators() { return shadowReplicators; } @@ -1965,8 +2277,7 @@ public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats replicators.forEach((region, replicator) -> replicator.updateRates()); - nsStats.producerCount += producers.size(); - bundleStats.producerCount += producers.size(); + final MutableInt producerCount = new MutableInt(); topicStatsStream.startObject(topic); // start publisher stats @@ -1980,14 +2291,19 @@ public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats if (producer.isRemote()) { topicStatsHelper.remotePublishersStats.put(producer.getRemoteCluster(), publisherStats); - } - - // Populate consumer specific stats here - if (hydratePublishers) { - StreamingStats.writePublisherStats(topicStatsStream, publisherStats); + } else { + // Exclude producers for replication from "publishers" and "producerCount" + producerCount.increment(); + if (hydratePublishers) { + StreamingStats.writePublisherStats(topicStatsStream, publisherStats); + } } }); topicStatsStream.endList(); + + nsStats.producerCount += producerCount.intValue(); + bundleStats.producerCount += producerCount.intValue(); + // if publish-rate increases (eg: 0 to 1K) then pick max publish-rate and if publish-rate decreases then keep // average rate. lastUpdatedAvgPublishRateInMsg = topicStatsHelper.aggMsgRateIn > lastUpdatedAvgPublishRateInMsg @@ -2008,7 +2324,7 @@ public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats } // Update replicator stats - ReplicatorStatsImpl rStat = replicator.getStats(); + ReplicatorStatsImpl rStat = replicator.computeStats(); // Add incoming msg rates PublisherStatsImpl pubStats = topicStatsHelper.remotePublishersStats.get(replicator.getRemoteCluster()); @@ -2155,7 +2471,7 @@ public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats // Remaining dest stats. topicStatsHelper.averageMsgSize = topicStatsHelper.aggMsgRateIn == 0.0 ? 0.0 : (topicStatsHelper.aggMsgThroughputIn / topicStatsHelper.aggMsgRateIn); - topicStatsStream.writePair("producerCount", producers.size()); + topicStatsStream.writePair("producerCount", producerCount.intValue()); topicStatsStream.writePair("averageMsgSize", topicStatsHelper.averageMsgSize); topicStatsStream.writePair("msgRateIn", topicStatsHelper.aggMsgRateIn); topicStatsStream.writePair("msgRateOut", topicStatsHelper.aggMsgRateOut); @@ -2167,7 +2483,7 @@ public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats topicStatsStream.writePair("msgThroughputOut", topicStatsHelper.aggMsgThroughputOut); topicStatsStream.writePair("storageSize", ledger.getTotalSize()); topicStatsStream.writePair("backlogSize", ledger.getEstimatedBacklogSize()); - topicStatsStream.writePair("pendingAddEntriesCount", ((ManagedLedgerImpl) ledger).getPendingAddEntriesCount()); + topicStatsStream.writePair("pendingAddEntriesCount", ledger.getPendingAddEntriesCount()); topicStatsStream.writePair("filteredEntriesCount", getFilteredEntriesCount()); nsStats.msgRateIn += topicStatsHelper.aggMsgRateIn; @@ -2180,7 +2496,7 @@ public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats bundleStats.msgRateOut += topicStatsHelper.aggMsgRateOut; bundleStats.msgThroughputIn += topicStatsHelper.aggMsgThroughputIn; bundleStats.msgThroughputOut += topicStatsHelper.aggMsgThroughputOut; - bundleStats.cacheSize += ((ManagedLedgerImpl) ledger).getCacheSize(); + bundleStats.cacheSize += ledger.getCacheSize(); // Close topic object topicStatsStream.endObject(); @@ -2210,11 +2526,27 @@ public TopicStatsImpl getStats(boolean getPreciseBacklog, boolean subscriptionBa } } + @Override + public TopicStatsImpl getStats(GetStatsOptions getStatsOptions) { + try { + return asyncGetStats(getStatsOptions).get(); + } catch (InterruptedException | ExecutionException e) { + log.error("[{}] Fail to get stats", topic, e); + return null; + } + } + @Override public CompletableFuture asyncGetStats(boolean getPreciseBacklog, boolean subscriptionBacklogSize, boolean getEarliestTimeInBacklog) { + GetStatsOptions getStatsOptions = new GetStatsOptions(getPreciseBacklog, subscriptionBacklogSize, + getEarliestTimeInBacklog, false, false); + return (CompletableFuture) asyncGetStats(getStatsOptions); + } + + @Override + public CompletableFuture asyncGetStats(GetStatsOptions getStatsOptions) { - CompletableFuture statsFuture = new CompletableFuture<>(); TopicStatsImpl stats = new TopicStatsImpl(); ObjectObjectHashMap remotePublishersStats = new ObjectObjectHashMap<>(); @@ -2226,48 +2558,29 @@ public CompletableFuture asyncGetStats(boolean getPreciseBacklog if (producer.isRemote()) { remotePublishersStats.put(producer.getRemoteCluster(), publisherStats); + } else if (!getStatsOptions.isExcludePublishers()) { + // Exclude producers for replication from "publishers" + stats.addPublisher(publisherStats); } - stats.addPublisher(publisherStats); }); stats.averageMsgSize = stats.msgRateIn == 0.0 ? 0.0 : (stats.msgThroughputIn / stats.msgRateIn); stats.msgInCounter = getMsgInCounter(); stats.bytesInCounter = getBytesInCounter(); + stats.systemTopicBytesInCounter = getSystemTopicBytesInCounter(); stats.msgChunkPublished = this.msgChunkPublished; stats.waitingPublishers = getWaitingProducersCount(); stats.bytesOutCounter = bytesOutFromRemovedSubscriptions.longValue(); stats.msgOutCounter = msgOutFromRemovedSubscriptions.longValue(); + stats.bytesOutInternalCounter = bytesOutFromRemovedSystemSubscriptions.longValue(); stats.publishRateLimitedTimes = publishRateLimitedTimes; TransactionBuffer txnBuffer = getTransactionBuffer(); stats.ongoingTxnCount = txnBuffer.getOngoingTxnCount(); stats.abortedTxnCount = txnBuffer.getAbortedTxnCount(); stats.committedTxnCount = txnBuffer.getCommittedTxnCount(); - subscriptions.forEach((name, subscription) -> { - SubscriptionStatsImpl subStats = - subscription.getStats(getPreciseBacklog, subscriptionBacklogSize, getEarliestTimeInBacklog); - - stats.msgRateOut += subStats.msgRateOut; - stats.msgThroughputOut += subStats.msgThroughputOut; - stats.bytesOutCounter += subStats.bytesOutCounter; - stats.msgOutCounter += subStats.msgOutCounter; - stats.subscriptions.put(name, subStats); - stats.nonContiguousDeletedMessagesRanges += subStats.nonContiguousDeletedMessagesRanges; - stats.nonContiguousDeletedMessagesRangesSerializedSize += - subStats.nonContiguousDeletedMessagesRangesSerializedSize; - stats.delayedMessageIndexSizeInBytes += subStats.delayedMessageIndexSizeInBytes; - - subStats.bucketDelayedIndexStats.forEach((k, v) -> { - TopicMetricBean topicMetricBean = - stats.bucketDelayedIndexStats.computeIfAbsent(k, __ -> new TopicMetricBean()); - topicMetricBean.name = v.name; - topicMetricBean.labelsAndValues = v.labelsAndValues; - topicMetricBean.value += v.value; - }); - }); - replicators.forEach((cluster, replicator) -> { - ReplicatorStatsImpl replicatorStats = replicator.getStats(); + ReplicatorStatsImpl replicatorStats = replicator.computeStats(); // Add incoming msg rates PublisherStatsImpl pubStats = remotePublishersStats.get(replicator.getRemoteCluster()); @@ -2288,13 +2601,23 @@ public CompletableFuture asyncGetStats(boolean getPreciseBacklog stats.backlogSize = ledger.getEstimatedBacklogSize(); stats.deduplicationStatus = messageDeduplication.getStatus().toString(); stats.topicEpoch = topicEpoch.orElse(null); - stats.ownerBroker = brokerService.pulsar().getLookupServiceAddress(); + stats.ownerBroker = brokerService.pulsar().getBrokerId(); stats.offloadedStorageSize = ledger.getOffloadedSize(); stats.lastOffloadLedgerId = ledger.getLastOffloadedLedgerId(); stats.lastOffloadSuccessTimeStamp = ledger.getLastOffloadedSuccessTimestamp(); stats.lastOffloadFailureTimeStamp = ledger.getLastOffloadedFailureTimestamp(); Optional mxBean = getCompactorMXBean(); + stats.backlogQuotaLimitSize = getBacklogQuota(BacklogQuotaType.destination_storage).getLimitSize(); + stats.backlogQuotaLimitTime = getBacklogQuota(BacklogQuotaType.message_age).getLimitTime(); + + TimeBasedBacklogQuotaCheckResult backlogQuotaCheckResult = timeBasedBacklogQuotaCheckResult; + stats.oldestBacklogMessageAgeSeconds = getBestEffortOldestUnacknowledgedMessageAgeSeconds(); + stats.oldestBacklogMessageSubscriptionName = (backlogQuotaCheckResult == null) + || !hasBacklogs(getStatsOptions.isGetPreciseBacklog()) + ? null + : backlogQuotaCheckResult.getCursorName(); + stats.compaction.reset(); mxBean.flatMap(bean -> bean.getCompactionRecordForTopic(topic)).map(compactionRecord -> { stats.compaction.lastCompactionRemovedEventCount = compactionRecord.getLastCompactionRemovedEventCount(); @@ -2305,21 +2628,52 @@ public CompletableFuture asyncGetStats(boolean getPreciseBacklog return compactionRecord; }); - if (getEarliestTimeInBacklog && stats.backlogSize != 0) { - ledger.getEarliestMessagePublishTimeInBacklog().whenComplete((earliestTime, e) -> { - if (e != null) { - log.error("[{}] Failed to get earliest message publish time in backlog", topic, e); - statsFuture.completeExceptionally(e); - } else { - stats.earliestMsgPublishTimeInBacklogs = earliestTime; - statsFuture.complete(stats); - } - }); - } else { - statsFuture.complete(stats); - } + Map> subscriptionFutures = new HashMap<>(); + subscriptions.forEach((name, subscription) -> { + subscriptionFutures.put(name, subscription.getStatsAsync(getStatsOptions)); + }); + return FutureUtil.waitForAll(subscriptionFutures.values()).thenCompose(ignore -> { + for (Map.Entry> e : subscriptionFutures.entrySet()) { + String name = e.getKey(); + SubscriptionStatsImpl subStats = e.getValue().join(); + stats.msgRateOut += subStats.msgRateOut; + stats.msgThroughputOut += subStats.msgThroughputOut; + stats.bytesOutCounter += subStats.bytesOutCounter; + stats.msgOutCounter += subStats.msgOutCounter; + stats.subscriptions.put(name, subStats); + stats.nonContiguousDeletedMessagesRanges += subStats.nonContiguousDeletedMessagesRanges; + stats.nonContiguousDeletedMessagesRangesSerializedSize += + subStats.nonContiguousDeletedMessagesRangesSerializedSize; + stats.delayedMessageIndexSizeInBytes += subStats.delayedMessageIndexSizeInBytes; + + subStats.bucketDelayedIndexStats.forEach((k, v) -> { + TopicMetricBean topicMetricBean = + stats.bucketDelayedIndexStats.computeIfAbsent(k, ignore2 -> new TopicMetricBean()); + topicMetricBean.name = v.name; + topicMetricBean.labelsAndValues = v.labelsAndValues; + topicMetricBean.value += v.value; + }); - return statsFuture; + if (isSystemCursor(name) || name.startsWith(SystemTopicNames.SYSTEM_READER_PREFIX)) { + stats.bytesOutInternalCounter += subStats.bytesOutCounter; + } + } + if (getStatsOptions.isGetEarliestTimeInBacklog() && stats.backlogSize != 0) { + CompletableFuture finalRes = ledger.getEarliestMessagePublishTimeInBacklog() + .thenApply((earliestTime) -> { + stats.earliestMsgPublishTimeInBacklogs = earliestTime; + return stats; + }); + // print error log. + finalRes.exceptionally(ex -> { + log.error("[{}] Failed to get earliest message publish time in backlog", topic, ex); + return null; + }); + return finalRes; + } else { + return CompletableFuture.completedFuture(stats); + } + }); } private Optional getCompactorMXBean() { @@ -2327,203 +2681,197 @@ private Optional getCompactorMXBean() { return Optional.ofNullable(compactor).map(c -> c.getStats()); } + @Override + public CompletableFuture deleteSchema() { + if (TopicName.get(getName()).isPartitioned()) { + // Only delete schema when partitioned metadata is deleting. + return CompletableFuture.completedFuture(null); + } + return brokerService.deleteSchema(TopicName.get(getName())); + } + @Override public CompletableFuture getInternalStats(boolean includeLedgerMetadata) { CompletableFuture statFuture = new CompletableFuture<>(); - PersistentTopicInternalStats stats = new PersistentTopicInternalStats(); - - ManagedLedgerImpl ml = (ManagedLedgerImpl) ledger; - stats.entriesAddedCounter = ml.getEntriesAddedCounter(); - stats.numberOfEntries = ml.getNumberOfEntries(); - stats.totalSize = ml.getTotalSize(); - stats.currentLedgerEntries = ml.getCurrentLedgerEntries(); - stats.currentLedgerSize = ml.getCurrentLedgerSize(); - stats.lastLedgerCreatedTimestamp = DateFormatter.format(ml.getLastLedgerCreatedTimestamp()); - if (ml.getLastLedgerCreationFailureTimestamp() != 0) { - stats.lastLedgerCreationFailureTimestamp = DateFormatter.format(ml.getLastLedgerCreationFailureTimestamp()); - } - - stats.waitingCursorsCount = ml.getWaitingCursorsCount(); - stats.pendingAddEntriesCount = ml.getPendingAddEntriesCount(); - - stats.lastConfirmedEntry = ml.getLastConfirmedEntry().toString(); - stats.state = ml.getState().toString(); - - stats.ledgers = new ArrayList<>(); - Set> futures = Sets.newConcurrentHashSet(); - CompletableFuture> availableBookiesFuture = - brokerService.pulsar().getPulsarResources().getBookieResources().listAvailableBookiesAsync(); - futures.add( - availableBookiesFuture - .whenComplete((bookies, e) -> { - if (e != null) { - log.error("[{}] Failed to fetch available bookies.", topic, e); - statFuture.completeExceptionally(e); - } else { - ml.getLedgersInfo().forEach((id, li) -> { - LedgerInfo info = new LedgerInfo(); - info.ledgerId = li.getLedgerId(); - info.entries = li.getEntries(); - info.size = li.getSize(); - info.offloaded = li.hasOffloadContext() && li.getOffloadContext().getComplete(); - stats.ledgers.add(info); - if (includeLedgerMetadata) { - futures.add(ml.getLedgerMetadata(li.getLedgerId()).handle((lMetadata, ex) -> { - if (ex == null) { - info.metadata = lMetadata; - } - return null; - })); - futures.add(ml.getEnsemblesAsync(li.getLedgerId()).handle((ensembles, ex) -> { - if (ex == null) { - info.underReplicated = - !bookies.containsAll(ensembles.stream().map(BookieId::toString) - .collect(Collectors.toList())); - } - return null; - })); - } - }); + + ledger.getManagedLedgerInternalStats(includeLedgerMetadata) + .thenCombine(getCompactedTopicContextAsync(), (ledgerInternalStats, compactedTopicContext) -> { + PersistentTopicInternalStats stats = new PersistentTopicInternalStats(); + stats.entriesAddedCounter = ledgerInternalStats.getEntriesAddedCounter(); + stats.numberOfEntries = ledgerInternalStats.getNumberOfEntries(); + stats.totalSize = ledgerInternalStats.getTotalSize(); + stats.currentLedgerEntries = ledgerInternalStats.getCurrentLedgerEntries(); + stats.currentLedgerSize = ledgerInternalStats.getCurrentLedgerSize(); + stats.lastLedgerCreatedTimestamp = ledgerInternalStats.getLastLedgerCreatedTimestamp(); + stats.lastLedgerCreationFailureTimestamp = ledgerInternalStats.getLastLedgerCreationFailureTimestamp(); + stats.waitingCursorsCount = ledgerInternalStats.getWaitingCursorsCount(); + stats.pendingAddEntriesCount = ledgerInternalStats.getPendingAddEntriesCount(); + stats.lastConfirmedEntry = ledgerInternalStats.getLastConfirmedEntry(); + stats.state = ledgerInternalStats.getState(); + stats.ledgers = ledgerInternalStats.ledgers; + + // Add ledger info for compacted topic ledger if exist. + LedgerInfo info = new LedgerInfo(); + info.ledgerId = -1; + info.entries = -1; + info.size = -1; + if (compactedTopicContext != null) { + info.ledgerId = compactedTopicContext.getLedger().getId(); + info.entries = compactedTopicContext.getLedger().getLastAddConfirmed() + 1; + info.size = compactedTopicContext.getLedger().getLength(); + } + + stats.compactedLedger = info; + + stats.cursors = new HashMap<>(); + ledger.getCursors().forEach(c -> { + CursorStats cs = new CursorStats(); + + CursorStats cursorInternalStats = c.getCursorStats(); + cs.markDeletePosition = cursorInternalStats.getMarkDeletePosition(); + cs.readPosition = cursorInternalStats.getReadPosition(); + cs.waitingReadOp = cursorInternalStats.isWaitingReadOp(); + cs.pendingReadOps = cursorInternalStats.getPendingReadOps(); + cs.messagesConsumedCounter = cursorInternalStats.getMessagesConsumedCounter(); + cs.cursorLedger = cursorInternalStats.getCursorLedger(); + cs.cursorLedgerLastEntry = cursorInternalStats.getCursorLedgerLastEntry(); + cs.individuallyDeletedMessages = cursorInternalStats.getIndividuallyDeletedMessages(); + cs.lastLedgerSwitchTimestamp = cursorInternalStats.getLastLedgerSwitchTimestamp(); + cs.state = cursorInternalStats.getState(); + cs.active = cursorInternalStats.isActive(); + cs.numberOfEntriesSinceFirstNotAckedMessage = + cursorInternalStats.getNumberOfEntriesSinceFirstNotAckedMessage(); + cs.totalNonContiguousDeletedMessagesRange = + cursorInternalStats.getTotalNonContiguousDeletedMessagesRange(); + cs.properties = cursorInternalStats.getProperties(); + // subscription metrics + PersistentSubscription sub = subscriptions.get(Codec.decode(c.getName())); + if (sub != null) { + if (sub.getDispatcher() instanceof PersistentDispatcherMultipleConsumers) { + PersistentDispatcherMultipleConsumers dispatcher = + (PersistentDispatcherMultipleConsumers) sub.getDispatcher(); + cs.subscriptionHavePendingRead = dispatcher.havePendingRead; + cs.subscriptionHavePendingReplayRead = dispatcher.havePendingReplayRead; + } else if (sub.getDispatcher() instanceof PersistentDispatcherSingleActiveConsumer) { + PersistentDispatcherSingleActiveConsumer dispatcher = + (PersistentDispatcherSingleActiveConsumer) sub.getDispatcher(); + cs.subscriptionHavePendingRead = dispatcher.havePendingRead; + } } - }) - ); - - // Add ledger info for compacted topic ledger if exist. - LedgerInfo info = new LedgerInfo(); - info.ledgerId = -1; - info.entries = -1; - info.size = -1; - - Optional compactedTopicContext = getCompactedTopicContext(); - if (compactedTopicContext.isPresent()) { - CompactedTopicContext ledgerContext = compactedTopicContext.get(); - info.ledgerId = ledgerContext.getLedger().getId(); - info.entries = ledgerContext.getLedger().getLastAddConfirmed() + 1; - info.size = ledgerContext.getLedger().getLength(); - } - - stats.compactedLedger = info; - - stats.cursors = new HashMap<>(); - ml.getCursors().forEach(c -> { - ManagedCursorImpl cursor = (ManagedCursorImpl) c; - CursorStats cs = new CursorStats(); - cs.markDeletePosition = cursor.getMarkDeletedPosition().toString(); - cs.readPosition = cursor.getReadPosition().toString(); - cs.waitingReadOp = cursor.hasPendingReadRequest(); - cs.pendingReadOps = cursor.getPendingReadOpsCount(); - cs.messagesConsumedCounter = cursor.getMessagesConsumedCounter(); - cs.cursorLedger = cursor.getCursorLedger(); - cs.cursorLedgerLastEntry = cursor.getCursorLedgerLastEntry(); - cs.individuallyDeletedMessages = cursor.getIndividuallyDeletedMessages(); - cs.lastLedgerSwitchTimestamp = DateFormatter.format(cursor.getLastLedgerSwitchTimestamp()); - cs.state = cursor.getState(); - cs.active = cursor.isActive(); - cs.numberOfEntriesSinceFirstNotAckedMessage = cursor.getNumberOfEntriesSinceFirstNotAckedMessage(); - cs.totalNonContiguousDeletedMessagesRange = cursor.getTotalNonContiguousDeletedMessagesRange(); - cs.properties = cursor.getProperties(); - // subscription metrics - PersistentSubscription sub = subscriptions.get(Codec.decode(c.getName())); - if (sub != null) { - if (sub.getDispatcher() instanceof PersistentDispatcherMultipleConsumers) { - PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) sub - .getDispatcher(); - cs.subscriptionHavePendingRead = dispatcher.havePendingRead; - cs.subscriptionHavePendingReplayRead = dispatcher.havePendingReplayRead; - } else if (sub.getDispatcher() instanceof PersistentDispatcherSingleActiveConsumer) { - PersistentDispatcherSingleActiveConsumer dispatcher = (PersistentDispatcherSingleActiveConsumer) sub - .getDispatcher(); - cs.subscriptionHavePendingRead = dispatcher.havePendingRead; - } - } - stats.cursors.put(cursor.getName(), cs); - }); + stats.cursors.put(c.getName(), cs); + }); - //Schema store ledgers - String schemaId; - try { - schemaId = TopicName.get(topic).getSchemaName(); - } catch (Throwable t) { - statFuture.completeExceptionally(t); - return statFuture; - } - - - CompletableFuture schemaStoreLedgersFuture = new CompletableFuture<>(); - stats.schemaLedgers = Collections.synchronizedList(new ArrayList<>()); - if (brokerService.getPulsar().getSchemaStorage() != null - && brokerService.getPulsar().getSchemaStorage() instanceof BookkeeperSchemaStorage) { - ((BookkeeperSchemaStorage) brokerService.getPulsar().getSchemaStorage()) - .getStoreLedgerIdsBySchemaId(schemaId) - .thenAccept(ledgers -> { - List> getLedgerMetadataFutures = new ArrayList<>(); - ledgers.forEach(ledgerId -> { - CompletableFuture completableFuture = new CompletableFuture<>(); - getLedgerMetadataFutures.add(completableFuture); - CompletableFuture metadataFuture = null; - try { - metadataFuture = brokerService.getPulsar().getBookKeeperClient() - .getLedgerMetadata(ledgerId); - } catch (NullPointerException e) { - // related to bookkeeper issue https://github.com/apache/bookkeeper/issues/2741 - if (log.isDebugEnabled()) { - log.debug("{{}} Failed to get ledger metadata for the schema ledger {}", + //Schema store ledgers + String schemaId; + try { + schemaId = TopicName.get(topic).getSchemaName(); + } catch (Throwable t) { + statFuture.completeExceptionally(t); + return null; + } + + + CompletableFuture schemaStoreLedgersFuture = new CompletableFuture<>(); + stats.schemaLedgers = Collections.synchronizedList(new ArrayList<>()); + if (brokerService.getPulsar().getSchemaStorage() != null + && brokerService.getPulsar().getSchemaStorage() instanceof BookkeeperSchemaStorage) { + ((BookkeeperSchemaStorage) brokerService.getPulsar().getSchemaStorage()) + .getStoreLedgerIdsBySchemaId(schemaId) + .thenAccept(ledgers -> { + List> getLedgerMetadataFutures = new ArrayList<>(); + ledgers.forEach(ledgerId -> { + CompletableFuture completableFuture = new CompletableFuture<>(); + getLedgerMetadataFutures.add(completableFuture); + CompletableFuture metadataFuture = null; + try { + metadataFuture = brokerService.getPulsar().getBookKeeperClient() + .getLedgerMetadata(ledgerId); + } catch (NullPointerException e) { + // related to bookkeeper issue https://github.com/apache/bookkeeper/issues/2741 + if (log.isDebugEnabled()) { + log.debug("{{}} Failed to get ledger metadata for the schema ledger {}", topic, ledgerId, e); - } - } - if (metadataFuture != null) { - metadataFuture.thenAccept(metadata -> { - LedgerInfo schemaLedgerInfo = new LedgerInfo(); - schemaLedgerInfo.ledgerId = metadata.getLedgerId(); - schemaLedgerInfo.entries = metadata.getLastEntryId() + 1; - schemaLedgerInfo.size = metadata.getLength(); - if (includeLedgerMetadata) { - info.metadata = metadata.toSafeString(); } - stats.schemaLedgers.add(schemaLedgerInfo); + } + if (metadataFuture != null) { + metadataFuture.thenAccept(metadata -> { + LedgerInfo schemaLedgerInfo = new LedgerInfo(); + schemaLedgerInfo.ledgerId = metadata.getLedgerId(); + schemaLedgerInfo.entries = metadata.getLastEntryId() + 1; + schemaLedgerInfo.size = metadata.getLength(); + if (includeLedgerMetadata) { + info.metadata = metadata.toSafeString(); + } + stats.schemaLedgers.add(schemaLedgerInfo); + completableFuture.complete(null); + }).exceptionally(e -> { + log.error("[{}] Failed to get ledger metadata for the schema ledger {}", + topic, ledgerId, e); + if ((e.getCause() instanceof BKNoSuchLedgerExistsOnMetadataServerException) + || (e.getCause() instanceof BKNoSuchLedgerExistsException)) { + completableFuture.complete(null); + return null; + } + completableFuture.completeExceptionally(e); + return null; + }); + } else { completableFuture.complete(null); - }).exceptionally(e -> { - completableFuture.completeExceptionally(e); - return null; - }); - } else { - completableFuture.complete(null); - } - }); - FutureUtil.waitForAll(getLedgerMetadataFutures).thenRun(() -> { - schemaStoreLedgersFuture.complete(null); + } + }); + FutureUtil.waitForAll(getLedgerMetadataFutures).thenRun(() -> { + schemaStoreLedgersFuture.complete(null); + }).exceptionally(e -> { + schemaStoreLedgersFuture.completeExceptionally(e); + return null; + }); }).exceptionally(e -> { schemaStoreLedgersFuture.completeExceptionally(e); return null; }); - }).exceptionally(e -> { - schemaStoreLedgersFuture.completeExceptionally(e); + } else { + schemaStoreLedgersFuture.complete(null); + } + schemaStoreLedgersFuture.whenComplete((r, ex) -> { + if (ex != null) { + statFuture.completeExceptionally(ex); + } else { + statFuture.complete(stats); + } + }); return null; - }); - } else { - schemaStoreLedgersFuture.complete(null); - } - schemaStoreLedgersFuture.thenRun(() -> - FutureUtil.waitForAll(futures).handle((res, ex) -> { - statFuture.complete(stats); + }) + .exceptionally(ex -> { + statFuture.completeExceptionally(ex); return null; - })).exceptionally(e -> { - statFuture.completeExceptionally(e); - return null; - }); + }); return statFuture; } public Optional getCompactedTopicContext() { try { - return ((CompactedTopicImpl) compactedTopic).getCompactedTopicContext(); - } catch (ExecutionException | InterruptedException e) { + if (topicCompactionService instanceof PulsarTopicCompactionService pulsarCompactedService) { + return pulsarCompactedService.getCompactedTopic().getCompactedTopicContext(); + } + } catch (ExecutionException | InterruptedException | TimeoutException e) { log.warn("[{}]Fail to get ledger information for compacted topic.", topic); } return Optional.empty(); } + public CompletableFuture getCompactedTopicContextAsync() { + if (topicCompactionService instanceof PulsarTopicCompactionService pulsarCompactedService) { + CompletableFuture res = + pulsarCompactedService.getCompactedTopic().getCompactedTopicContextFuture(); + if (res == null) { + return CompletableFuture.completedFuture(null); + } + return res; + } + return CompletableFuture.completedFuture(null); + } + public long getBacklogSize() { return ledger.getEstimatedBacklogSize(); } @@ -2536,7 +2884,7 @@ public boolean isActive(InactiveTopicDeleteMode deleteMode) { } break; case delete_when_subscriptions_caught_up: - if (hasBacklogs()) { + if (hasBacklogs(false)) { return true; } break; @@ -2549,28 +2897,183 @@ public boolean isActive(InactiveTopicDeleteMode deleteMode) { } } - private boolean hasBacklogs() { - return subscriptions.values().stream().anyMatch(sub -> sub.getNumberOfEntriesInBacklog(false) > 0); + private boolean hasBacklogs(boolean getPreciseBacklog) { + return subscriptions.values().stream().anyMatch(sub -> sub.getNumberOfEntriesInBacklog(getPreciseBacklog) > 0); } @Override public CompletableFuture checkClusterMigration() { + if (ExtensibleLoadManagerImpl.isInternalTopic(topic)) { + return CompletableFuture.completedFuture(null); + } + Optional clusterUrl = getMigratedClusterUrl(); - if (!isMigrated() && clusterUrl.isPresent()) { - log.info("{} triggering topic migration", topic); - return ledger.asyncMigrate().thenCompose(r -> null); - } else { + + if (!clusterUrl.isPresent()) { return CompletableFuture.completedFuture(null); } + + if (isReplicated()) { + if (isReplicationBacklogExist()) { + if (!ledger.isMigrated()) { + log.info("{} applying migration with replication backlog", topic); + ledger.asyncMigrate(); + } + if (log.isDebugEnabled()) { + log.debug("{} has replication backlog and applied migration", topic); + } + return CompletableFuture.completedFuture(null); + } + } + + return initMigration().thenCompose(subCreated -> { + migrationSubsCreated = true; + CompletableFuture migrated = !isMigrated() ? ledger.asyncMigrate() + : CompletableFuture.completedFuture(null); + return migrated.thenApply(__ -> { + subscriptions.forEach((name, sub) -> { + if (sub.isSubscriptionMigrated()) { + sub.getConsumers().forEach(Consumer::checkAndApplyTopicMigration); + } + }); + return null; + }).thenCompose(__ -> checkAndDisconnectReplicators()) + .thenCompose(__ -> checkAndUnsubscribeSubscriptions()) + .thenCompose(__ -> checkAndDisconnectProducers()); + }); + } + + /** + * Initialize migration for a topic by creating topic's resources at migration cluster. + */ + private CompletableFuture initMigration() { + if (migrationSubsCreated) { + return CompletableFuture.completedFuture(null); + } + log.info("{} initializing subscription created at migration cluster", topic); + return getMigratedClusterUrlAsync(getBrokerService().getPulsar(), topic).thenCompose(clusterUrl -> { + if (!brokerService.getPulsar().getConfig().isClusterMigrationAutoResourceCreation()) { + return CompletableFuture.completedFuture(null); + } + if (!clusterUrl.isPresent()) { + return FutureUtil + .failedFuture(new TopicMigratedException("cluster migration service-url is not configured")); + } + ClusterUrl url = clusterUrl.get(); + ClusterData clusterData = ClusterData.builder().serviceUrl(url.getServiceUrl()) + .serviceUrlTls(url.getServiceUrlTls()).brokerServiceUrl(url.getBrokerServiceUrl()) + .brokerServiceUrlTls(url.getBrokerServiceUrlTls()).build(); + PulsarAdmin admin = getBrokerService().getClusterPulsarAdmin(MIGRATION_CLUSTER_NAME, + Optional.of(clusterData)); + + // namespace creation + final String tenant = TopicName.get(topic).getTenant(); + final NamespaceName ns = TopicName.get(topic).getNamespaceObject(); + List> subResults = new ArrayList<>(); + + return brokerService.getPulsar().getPulsarResources().getTenantResources().getTenantAsync(tenant) + .thenCompose(tenantInfo -> { + if (!tenantInfo.isPresent()) { + return CompletableFuture.completedFuture(null); + } + CompletableFuture ts = new CompletableFuture<>(); + admin.tenants().createTenantAsync(tenant, tenantInfo.get()).handle((__, ex) -> { + if (ex == null || ex instanceof ConflictException) { + log.info("[{}] successfully created tenant {} for migration", topic, tenant); + ts.complete(null); + return null; + } + log.warn("[{}] Failed to create tenant {} on migration cluster {}", topic, tenant, + ex.getCause().getMessage()); + ts.completeExceptionally(ex.getCause()); + return null; + }); + return ts; + }).thenCompose(t -> { + return brokerService.getPulsar().getPulsarResources().getNamespaceResources() + .getPoliciesAsync(ns).thenCompose(policies -> { + if (!policies.isPresent()) { + return CompletableFuture.completedFuture(null); + } + CompletableFuture nsFuture = new CompletableFuture<>(); + admin.namespaces().createNamespaceAsync(ns.toString(), policies.get()) + .handle((__, ex) -> { + if (ex == null || ex instanceof ConflictException) { + log.info("[{}] successfully created namespace {} for migration", + topic, ns); + nsFuture.complete(null); + return null; + } + log.warn("[{}] Failed to create namespace {} on migration cluster {}", + topic, ns, ex.getCause().getMessage()); + nsFuture.completeExceptionally(ex.getCause()); + return null; + }); + return nsFuture; + }).thenCompose(p -> { + subscriptions.forEach((subName, sub) -> { + CompletableFuture subResult = new CompletableFuture<>(); + subResults.add(subResult); + admin.topics().createSubscriptionAsync(topic, subName, MessageId.earliest) + .handle((__, ex) -> { + if (ex == null || ex instanceof ConflictException) { + log.info("[{}] successfully created sub {} for migration", + topic, subName); + subResult.complete(null); + return null; + } + log.warn("[{}] Failed to create sub {} on migration cluster, {}", + topic, subName, ex.getCause().getMessage()); + subResult.completeExceptionally(ex.getCause()); + return null; + }); + }); + return Futures.waitForAll(subResults); + }); + }); + }); + } + + private CompletableFuture checkAndUnsubscribeSubscriptions() { + List> futures = new ArrayList<>(); + subscriptions.forEach((s, subscription) -> { + if (subscription.getNumberOfEntriesInBacklog(true) == 0 + && subscription.getConsumers().isEmpty()) { + futures.add(subscription.delete()); + } + }); + + return FutureUtil.waitForAll(futures); + } + + private CompletableFuture checkAndDisconnectProducers() { + List> futures = new ArrayList<>(); + producers.forEach((name, producer) -> { + futures.add(producer.disconnect()); + }); + + return FutureUtil.waitForAll(futures); + } + + private CompletableFuture checkAndDisconnectReplicators() { + List> futures = new ArrayList<>(); + replicators.forEach((r, replicator) -> { + if (replicator.getNumberOfEntriesInBacklog() <= 0) { + futures.add(replicator.terminate()); + } + }); + return FutureUtil.waitForAll(futures); + } + + public boolean shouldProducerMigrate() { + return !isReplicationBacklogExist() && migrationSubsCreated; } + @Override public boolean isReplicationBacklogExist() { - ConcurrentOpenHashMap replicators = getReplicators(); - if (replicators != null) { - for (Replicator replicator : replicators.values()) { - if (replicator.getNumberOfEntriesInBacklog() != 0) { - return true; - } + for (Replicator replicator : replicators.values()) { + if (replicator.getNumberOfEntriesInBacklog() > 0) { + return true; } } return false; @@ -2587,7 +3090,7 @@ public void checkGC() { int maxInactiveDurationInSec = topicPolicies.getInactiveTopicPolicies().get().getMaxInactiveDurationSeconds(); if (isActive(deleteMode)) { lastActive = System.nanoTime(); - } else if (System.nanoTime() - lastActive < TimeUnit.SECONDS.toNanos(maxInactiveDurationInSec)) { + } else if (System.nanoTime() - lastActive < SECONDS.toNanos(maxInactiveDurationInSec)) { // Gc interval did not expire yet return; } else if (shouldTopicBeRetained()) { @@ -2604,6 +3107,15 @@ public void checkGC() { log.debug("[{}] Global topic inactive for {} seconds, closing repl producers.", topic, maxInactiveDurationInSec); } + /** + * There is a race condition that may cause a NPE: + * - task 1: a callback of "replicator.cursor.asyncRead" will trigger a replication. + * - task 2: "closeReplProducersIfNoBacklog" called by current thread will make the variable + * "replicator.producer" to a null value. + * Race condition: task 1 will get a NPE when it tries to send messages using the variable + * "replicator.producer", because task 2 will set this variable to "null". + * TODO Create a seperated PR to fix it. + */ closeReplProducersIfNoBacklog().thenRun(() -> { if (hasRemoteProducers()) { if (log.isDebugEnabled()) { @@ -2630,7 +3142,7 @@ public void checkGC() { replCloseFuture.thenCompose(v -> delete(deleteMode == InactiveTopicDeleteMode.delete_when_no_subscriptions, deleteMode == InactiveTopicDeleteMode.delete_when_subscriptions_caught_up, false)) - .thenApply((res) -> tryToDeletePartitionedMetadata()) + .thenCompose((res) -> tryToDeletePartitionedMetadata()) .thenRun(() -> log.info("[{}] Topic deleted successfully due to inactivity", topic)) .exceptionally(e -> { if (e.getCause() instanceof TopicBusyException) { @@ -2638,6 +3150,8 @@ public void checkGC() { if (log.isDebugEnabled()) { log.debug("[{}] Did not delete busy topic: {}", topic, e.getCause().getMessage()); } + } else if (e.getCause() instanceof UnsupportedOperationException) { + log.info("[{}] Skip to delete partitioned topic: {}", topic, e.getCause().getMessage()); } else { log.warn("[{}] Inactive topic deletion failed", topic, e); } @@ -2682,7 +3196,7 @@ private CompletableFuture tryToDeletePartitionedMetadata() { .filter(topicExist -> topicExist) .findAny(); if (anyExistPartition.isPresent()) { - log.error("[{}] Delete topic metadata failed because" + log.info("[{}] Delete topic metadata failed because" + " another partition exist.", topicName); throw new UnsupportedOperationException( String.format("Another partition exists for [%s].", @@ -2710,17 +3224,7 @@ public void checkInactiveSubscriptions() { final Integer nsExpirationTime = policies.subscription_expiration_time_minutes; final long expirationTimeMillis = TimeUnit.MINUTES .toMillis(nsExpirationTime == null ? defaultExpirationTime : nsExpirationTime); - if (expirationTimeMillis > 0) { - subscriptions.forEach((subName, sub) -> { - if (sub.dispatcher != null && sub.dispatcher.isConsumerConnected() || sub.isReplicated()) { - return; - } - if (System.currentTimeMillis() - sub.cursor.getLastActive() > expirationTimeMillis) { - sub.delete().thenAccept(v -> log.info("[{}][{}] The subscription was deleted due to expiration " - + "with last active [{}]", topic, subName, sub.cursor.getLastActive())); - } - }); - } + checkInactiveSubscriptions(expirationTimeMillis); } catch (Exception e) { if (log.isDebugEnabled()) { log.debug("[{}] Error getting policies", topic); @@ -2728,6 +3232,23 @@ public void checkInactiveSubscriptions() { } } + @VisibleForTesting + public void checkInactiveSubscriptions(long expirationTimeMillis) { + if (expirationTimeMillis > 0) { + subscriptions.forEach((subName, sub) -> { + if (sub.dispatcher != null && sub.dispatcher.isConsumerConnected() + || sub.isReplicated() + || isCompactionSubscription(subName)) { + return; + } + if (System.currentTimeMillis() - sub.cursor.getLastActive() > expirationTimeMillis) { + sub.delete().thenAccept(v -> log.info("[{}][{}] The subscription was deleted due to expiration " + + "with last active [{}]", topic, subName, sub.cursor.getLastActive())); + } + }); + } + } + @Override public void checkBackloggedCursors() { subscriptions.forEach((subName, subscription) -> { @@ -2786,7 +3307,8 @@ public void updateDispatchRateLimiter() { } @Override - public CompletableFuture onPoliciesUpdate(Policies data) { + public CompletableFuture onPoliciesUpdate(@Nonnull Policies data) { + requireNonNull(data); if (log.isDebugEnabled()) { log.debug("[{}] isEncryptionRequired changes: {} -> {}", topic, isEncryptionRequired, data.encryption_required); @@ -2796,39 +3318,62 @@ public CompletableFuture onPoliciesUpdate(Policies data) { return CompletableFuture.completedFuture(null); } + // Update props. + // The component "EntryFilters" is update in the method "updateTopicPolicyByNamespacePolicy(data)". + // see more detail: https://github.com/apache/pulsar/pull/19364. updateTopicPolicyByNamespacePolicy(data); - + checkReplicatedSubscriptionControllerState(); isEncryptionRequired = data.encryption_required; - isAllowAutoUpdateSchema = data.is_allow_auto_update_schema; - updateDispatchRateLimiter(); - - updateSubscribeRateLimiter(); + // Apply policies for components. + List> applyPolicyTasks = applyUpdatedTopicPolicies(); + applyPolicyTasks.add(applyUpdatedNamespacePolicies(data)); + return FutureUtil.waitForAll(applyPolicyTasks) + .thenAccept(__ -> log.info("[{}] namespace-level policies updated successfully", topic)) + .exceptionally(ex -> { + log.error("[{}] update namespace polices : {} error", this.getName(), data, ex); + throw FutureUtil.wrapToCompletionException(ex); + }); + } - updatePublishDispatcher(); + private CompletableFuture applyUpdatedNamespacePolicies(Policies namespaceLevelPolicies) { + return FutureUtil.runWithCurrentThread(() -> updateResourceGroupLimiter(namespaceLevelPolicies)); + } - this.updateResourceGroupLimiter(Optional.of(data)); + private List> applyUpdatedTopicPolicies() { + List> applyPoliciesFutureList = new ArrayList<>(); - List> producerCheckFutures = new ArrayList<>(producers.size()); - producers.values().forEach(producer -> producerCheckFutures.add( + // Client permission check. + subscriptions.forEach((subName, sub) -> { + sub.getConsumers().forEach(consumer -> applyPoliciesFutureList.add(consumer.checkPermissionsAsync())); + }); + producers.values().forEach(producer -> applyPoliciesFutureList.add( producer.checkPermissionsAsync().thenRun(producer::checkEncryption))); + // Check message expiry. + applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> checkMessageExpiry())); - return FutureUtil.waitForAll(producerCheckFutures).thenCompose((__) -> { - return updateSubscriptionsDispatcherRateLimiter().thenCompose((___) -> { - replicators.forEach((name, replicator) -> replicator.updateRateLimiter()); - shadowReplicators.forEach((name, replicator) -> replicator.updateRateLimiter()); - checkMessageExpiry(); - CompletableFuture replicationFuture = checkReplicationAndRetryOnFailure(); - CompletableFuture dedupFuture = checkDeduplicationStatus(); - CompletableFuture persistentPoliciesFuture = checkPersistencePolicies(); - return CompletableFuture.allOf(replicationFuture, dedupFuture, persistentPoliciesFuture, - preCreateSubscriptionForCompactionIfNeeded()); - }); - }).exceptionally(ex -> { - log.error("[{}] update namespace polices : {} error", this.getName(), data, ex); - throw FutureUtil.wrapToCompletionException(ex); - }); + // Update rate limiters. + applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> updateDispatchRateLimiter())); + applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> updateSubscribeRateLimiter())); + applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> updatePublishRateLimiter())); + + applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> updateSubscriptionsDispatcherRateLimiter())); + applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread( + () -> replicators.forEach((name, replicator) -> replicator.updateRateLimiter()))); + applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread( + () -> shadowReplicators.forEach((name, replicator) -> replicator.updateRateLimiter()))); + + // Other components. + applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> checkReplicationAndRetryOnFailure())); + applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> checkDeduplicationStatus())); + applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> checkPersistencePolicies())); + applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread( + () -> preCreateSubscriptionForCompactionIfNeeded())); + applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread( + () -> updateBrokerDispatchPauseOnAckStatePersistentEnabled())); + + return applyPoliciesFutureList; } /** @@ -2852,14 +3397,14 @@ public CompletableFuture checkBacklogQuotaExceeded(String producerName, Ba if ((retentionPolicy == BacklogQuota.RetentionPolicy.producer_request_hold || retentionPolicy == BacklogQuota.RetentionPolicy.producer_exception)) { if (backlogQuotaType == BacklogQuotaType.destination_storage && isSizeBacklogExceeded()) { - log.info("[{}] Size backlog quota exceeded. Cannot create producer [{}]", this.getName(), + log.debug("[{}] Size backlog quota exceeded. Cannot create producer [{}]", this.getName(), producerName); return FutureUtil.failedFuture(new TopicBacklogQuotaExceededException(retentionPolicy)); } if (backlogQuotaType == BacklogQuotaType.message_age) { return checkTimeBacklogExceeded().thenCompose(isExceeded -> { if (isExceeded) { - log.info("[{}] Time backlog quota exceeded. Cannot create producer [{}]", this.getName(), + log.debug("[{}] Time backlog quota exceeded. Cannot create producer [{}]", this.getName(), producerName); return FutureUtil.failedFuture(new TopicBacklogQuotaExceededException(retentionPolicy)); } else { @@ -2893,36 +3438,134 @@ public boolean isSizeBacklogExceeded() { return (storageSize >= backlogQuotaLimitInBytes); } + @Override + public long getBestEffortOldestUnacknowledgedMessageAgeSeconds() { + if (!hasBacklogs(false)) { + return 0; + } + TimeBasedBacklogQuotaCheckResult result = timeBasedBacklogQuotaCheckResult; + if (result == null) { + return -1; + } else { + return TimeUnit.MILLISECONDS.toSeconds( + Clock.systemUTC().millis() - result.getPositionPublishTimestampInMillis()); + } + } + + private void updateResultIfNewer(TimeBasedBacklogQuotaCheckResult updatedResult) { + TIME_BASED_BACKLOG_QUOTA_CHECK_RESULT_UPDATER.updateAndGet(this, + existingResult -> { + if (existingResult == null + || ManagedCursorContainer.DataVersion.compareVersions( + updatedResult.getDataVersion(), existingResult.getDataVersion()) > 0) { + return updatedResult; + } else { + return existingResult; + } + }); + + } + /** * @return determine if backlog quota enforcement needs to be done for topic based on time limit */ public CompletableFuture checkTimeBacklogExceeded() { TopicName topicName = TopicName.get(getName()); int backlogQuotaLimitInSecond = getBacklogQuota(BacklogQuotaType.message_age).getLimitTime(); + if (log.isDebugEnabled()) { + log.debug("[{}] Time backlog quota = [{}]. Checking if exceeded.", topicName, backlogQuotaLimitInSecond); + } - // If backlog quota by time is not set and we have no durable cursor. - if (backlogQuotaLimitInSecond <= 0 - || ((ManagedCursorContainer) ledger.getCursors()).getSlowestReaderPosition() == null) { + // If backlog quota by time is not set + if (backlogQuotaLimitInSecond <= 0) { return CompletableFuture.completedFuture(false); } + ManagedCursorContainer managedCursorContainer = (ManagedCursorContainer) ledger.getCursors(); + CursorInfo oldestMarkDeleteCursorInfo = managedCursorContainer.getCursorWithOldestPosition(); + + // If we have no durable cursor since `ledger.getCursors()` only managed durable cursors + if (oldestMarkDeleteCursorInfo == null + || oldestMarkDeleteCursorInfo.getPosition() == null) { + if (log.isDebugEnabled()) { + log.debug("[{}] No durable cursor found. Skipping time based backlog quota check." + + " Oldest mark-delete cursor info: {}", topicName, oldestMarkDeleteCursorInfo); + } + return CompletableFuture.completedFuture(false); + } + + Position oldestMarkDeletePosition = oldestMarkDeleteCursorInfo.getPosition(); + + TimeBasedBacklogQuotaCheckResult lastCheckResult = timeBasedBacklogQuotaCheckResult; + if (lastCheckResult != null + && oldestMarkDeletePosition.compareTo(lastCheckResult.getOldestCursorMarkDeletePosition()) == 0) { + + // Same position, but the cursor causing it has changed? + if (!lastCheckResult.getCursorName().equals(oldestMarkDeleteCursorInfo.getCursor().getName())) { + final TimeBasedBacklogQuotaCheckResult updatedResult = new TimeBasedBacklogQuotaCheckResult( + lastCheckResult.getOldestCursorMarkDeletePosition(), + oldestMarkDeleteCursorInfo.getCursor().getName(), + lastCheckResult.getPositionPublishTimestampInMillis(), + oldestMarkDeleteCursorInfo.getVersion()); + + updateResultIfNewer(updatedResult); + if (log.isDebugEnabled()) { + log.debug("[{}] Time-based backlog quota check. Updating cached result for position {}, " + + "since cursor causing it has changed from {} to {}", + topicName, + oldestMarkDeletePosition, + lastCheckResult.getCursorName(), + oldestMarkDeleteCursorInfo.getCursor().getName()); + } + } + + long entryTimestamp = lastCheckResult.getPositionPublishTimestampInMillis(); + boolean expired = MessageImpl.isEntryExpired(backlogQuotaLimitInSecond, entryTimestamp); + if (log.isDebugEnabled()) { + log.debug("[{}] Time based backlog quota check. Using cache result for position {}. " + + "Entry timestamp: {}, expired: {}", + topicName, oldestMarkDeletePosition, entryTimestamp, expired); + } + return CompletableFuture.completedFuture(expired); + } + if (brokerService.pulsar().getConfiguration().isPreciseTimeBasedBacklogQuotaCheck()) { + if (!hasBacklogs(true)) { + return CompletableFuture.completedFuture(false); + } CompletableFuture future = new CompletableFuture<>(); // Check if first unconsumed message(first message after mark delete position) // for slowest cursor's has expired. - PositionImpl position = ((ManagedLedgerImpl) ledger).getNextValidPosition(((ManagedCursorContainer) - ledger.getCursors()).getSlowestReaderPosition()); - ((ManagedLedgerImpl) ledger).asyncReadEntry(position, + Position position = ledger.getNextValidPosition(oldestMarkDeletePosition); + ledger.asyncReadEntry(position, new AsyncCallbacks.ReadEntryCallback() { @Override public void readEntryComplete(Entry entry, Object ctx) { try { long entryTimestamp = Commands.getEntryTimestamp(entry.getDataBuffer()); + + updateResultIfNewer( + new TimeBasedBacklogQuotaCheckResult( + oldestMarkDeleteCursorInfo.getPosition(), + oldestMarkDeleteCursorInfo.getCursor().getName(), + entryTimestamp, + oldestMarkDeleteCursorInfo.getVersion())); + boolean expired = MessageImpl.isEntryExpired(backlogQuotaLimitInSecond, entryTimestamp); - if (expired && log.isDebugEnabled()) { - log.debug("Time based backlog quota exceeded, oldest entry in cursor {}'s backlog" - + "exceeded quota {}", ((ManagedLedgerImpl) ledger).getSlowestConsumer().getName(), - backlogQuotaLimitInSecond); + if (log.isDebugEnabled()) { + log.debug("[{}] Time based backlog quota check. Oldest unacked entry read from BK. " + + "Oldest entry in cursor {}'s backlog: {}. " + + "Oldest mark-delete position: {}. " + + "Quota {}. Last check result position [{}]. " + + "Expired: {}, entryTimestamp: {}", + topicName, + oldestMarkDeleteCursorInfo.getCursor().getName(), + position, + oldestMarkDeletePosition, + backlogQuotaLimitInSecond, + lastCheckResult.getOldestCursorMarkDeletePosition(), + expired, + entryTimestamp); } future.complete(expired); } catch (Exception e) { @@ -2942,9 +3585,22 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { }, null); return future; } else { - PositionImpl slowestPosition = ((ManagedCursorContainer) ledger.getCursors()).getSlowestReaderPosition(); try { - return slowestReaderTimeBasedBacklogQuotaCheck(slowestPosition); + if (!hasBacklogs(false)) { + return CompletableFuture.completedFuture(false); + } + EstimateTimeBasedBacklogQuotaCheckResult checkResult = + estimatedTimeBasedBacklogQuotaCheck(oldestMarkDeletePosition); + if (checkResult.getEstimatedOldestUnacknowledgedMessageTimestamp() != null) { + updateResultIfNewer( + new TimeBasedBacklogQuotaCheckResult( + oldestMarkDeleteCursorInfo.getPosition(), + oldestMarkDeleteCursorInfo.getCursor().getName(), + checkResult.getEstimatedOldestUnacknowledgedMessageTimestamp(), + oldestMarkDeleteCursorInfo.getVersion())); + } + + return CompletableFuture.completedFuture(checkResult.isTruncateBacklogToMatchQuota()); } catch (Exception e) { log.error("[{}][{}] Error reading entry for precise time based backlog check", topicName, e); return CompletableFuture.completedFuture(false); @@ -2952,33 +3608,46 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { } } - private CompletableFuture slowestReaderTimeBasedBacklogQuotaCheck(PositionImpl slowestPosition) + private EstimateTimeBasedBacklogQuotaCheckResult estimatedTimeBasedBacklogQuotaCheck( + Position markDeletePosition) throws ExecutionException, InterruptedException { int backlogQuotaLimitInSecond = getBacklogQuota(BacklogQuotaType.message_age).getLimitTime(); - Long ledgerId = slowestPosition.getLedgerId(); - if (((ManagedLedgerImpl) ledger).getLedgersInfo().lastKey().equals(ledgerId)) { - return CompletableFuture.completedFuture(false); + + // The ledger timestamp is only known when ledger is closed, hence when the mark-delete + // is at active ledger (open) we can't estimate it. + if (ledger.getLedgersInfo().lastKey().equals(markDeletePosition.getLedgerId())) { + return new EstimateTimeBasedBacklogQuotaCheckResult(false, null); } - int result; + org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo - ledgerInfo = ledger.getLedgerInfo(ledgerId).get(); - if (ledgerInfo != null && ledgerInfo.hasTimestamp() && ledgerInfo.getTimestamp() > 0 - && ((ManagedLedgerImpl) ledger).getClock().millis() - ledgerInfo.getTimestamp() - > backlogQuotaLimitInSecond * 1000 && (result = slowestPosition.compareTo( - new PositionImpl(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1))) <= 0) { - if (result < 0) { - if (log.isDebugEnabled()) { - log.debug("Time based backlog quota exceeded, quota {}, age of ledger " - + "slowest cursor currently on {}", backlogQuotaLimitInSecond * 1000, - ((ManagedLedgerImpl) ledger).getClock().millis() - ledgerInfo.getTimestamp()); - } - return CompletableFuture.completedFuture(true); - } else { - return slowestReaderTimeBasedBacklogQuotaCheck( - ((ManagedLedgerImpl) ledger).getNextValidPosition(slowestPosition)); + markDeletePositionLedgerInfo = ledger.getLedgerInfo(markDeletePosition.getLedgerId()).get(); + + org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo positionToCheckLedgerInfo = + markDeletePositionLedgerInfo; + + // if the mark-delete position is the last entry it means all entries for + // that ledger are acknowledged + if (markDeletePosition.getEntryId() == markDeletePositionLedgerInfo.getEntries() - 1) { + Position positionToCheck = ledger.getNextValidPosition(markDeletePosition); + positionToCheckLedgerInfo = ledger.getLedgerInfo(positionToCheck.getLedgerId()).get(); + } + + if (positionToCheckLedgerInfo != null + && positionToCheckLedgerInfo.hasTimestamp() + && positionToCheckLedgerInfo.getTimestamp() > 0) { + long estimateMsgAgeMs = clock.millis() - positionToCheckLedgerInfo.getTimestamp(); + boolean shouldTruncateBacklog = estimateMsgAgeMs > SECONDS.toMillis(backlogQuotaLimitInSecond); + if (log.isDebugEnabled()) { + log.debug("Time based backlog quota exceeded, quota {}[ms], age of ledger " + + "slowest cursor currently on {}[ms]", backlogQuotaLimitInSecond * 1000, + estimateMsgAgeMs); } + + return new EstimateTimeBasedBacklogQuotaCheckResult( + shouldTruncateBacklog, + positionToCheckLedgerInfo.getTimestamp()); } else { - return CompletableFuture.completedFuture(false); + return new EstimateTimeBasedBacklogQuotaCheckResult(false, null); } } @@ -3000,7 +3669,7 @@ public void terminateComplete(Position lastCommittedPosition, Object ctx) { producers.values().forEach(Producer::disconnect); subscriptions.forEach((name, sub) -> sub.topicTerminated()); - PositionImpl lastPosition = (PositionImpl) lastCommittedPosition; + Position lastPosition = lastCommittedPosition; MessageId messageId = new MessageIdImpl(lastPosition.getLedgerId(), lastPosition.getEntryId(), -1); log.info("[{}] Topic terminated at {}", getName(), messageId); @@ -3033,7 +3702,7 @@ public boolean isOldestMessageExpired(ManagedCursor cursor, int messageTTLInSeco // if AutoSkipNonRecoverableData is set to true, just return true here. return true; } else { - log.warn("[{}] Error while getting the oldest message", topic, e); + log.warn("[{}] [{}] Error while getting the oldest message", topic, cursor.toString(), e); } } finally { if (entry != null) { @@ -3052,9 +3721,9 @@ public boolean isOldestMessageExpired(ManagedCursor cursor, int messageTTLInSeco public CompletableFuture clearBacklog() { log.info("[{}] Clearing backlog on all cursors in the topic.", topic); List> futures = new ArrayList<>(); - List cursors = getSubscriptions().keys(); - cursors.addAll(getReplicators().keys()); - cursors.addAll(getShadowReplicators().keys()); + List cursors = new ArrayList<>(getSubscriptions().keySet()); + cursors.addAll(getReplicators().keySet()); + cursors.addAll(getShadowReplicators().keySet()); for (String cursor : cursors) { futures.add(clearBacklog(cursor)); } @@ -3113,10 +3782,61 @@ public Position getLastPosition() { return ledger.getLastConfirmedEntry(); } + @Override + public CompletableFuture getLastDispatchablePosition() { + if (lastDispatchablePosition != null) { + return CompletableFuture.completedFuture(lastDispatchablePosition); + } + return ledger.getLastDispatchablePosition(entry -> { + MessageMetadata md = Commands.parseMessageMetadata(entry.getDataBuffer()); + // If a messages has marker will filter by AbstractBaseDispatcher.filterEntriesForConsumer + if (Markers.isServerOnlyMarker(md)) { + return false; + } else if (md.hasTxnidMostBits() && md.hasTxnidLeastBits()) { + // Filter-out transaction aborted messages. + TxnID txnID = new TxnID(md.getTxnidMostBits(), md.getTxnidLeastBits()); + return !isTxnAborted(txnID, entry.getPosition()); + } + return true; + }, getMaxReadPosition()).thenApply(position -> { + // Update lastDispatchablePosition to the given position + updateLastDispatchablePosition(position); + return position; + }); + } + + /** + * Update lastDispatchablePosition if the given position is greater than the lastDispatchablePosition. + * + * @param position + */ + public synchronized void updateLastDispatchablePosition(Position position) { + // Update lastDispatchablePosition to null if the position is null, fallback to + // ManagedLedgerImplUtils#asyncGetLastValidPosition + if (position == null) { + lastDispatchablePosition = null; + return; + } + + // If the position is greater than the maxReadPosition, ignore + if (position.compareTo(getMaxReadPosition()) > 0) { + return; + } + // If the lastDispatchablePosition is null, set it to the position + if (lastDispatchablePosition == null) { + lastDispatchablePosition = position; + return; + } + // If the position is greater than the lastDispatchablePosition, update it + if (position.compareTo(lastDispatchablePosition) > 0) { + lastDispatchablePosition = position; + } + } + @Override public CompletableFuture getLastMessageId() { CompletableFuture completableFuture = new CompletableFuture<>(); - PositionImpl position = (PositionImpl) ledger.getLastConfirmedEntry(); + Position position = ledger.getLastConfirmedEntry(); String name = getName(); int partitionIndex = TopicName.getPartitionIndex(name); if (log.isDebugEnabled()) { @@ -3127,13 +3847,13 @@ public CompletableFuture getLastMessageId() { .complete(new MessageIdImpl(position.getLedgerId(), position.getEntryId(), partitionIndex)); return completableFuture; } - ManagedLedgerImpl ledgerImpl = (ManagedLedgerImpl) ledger; - if (!ledgerImpl.ledgerExists(position.getLedgerId())) { + + if (!ledger.getLedgersInfo().containsKey(position.getLedgerId())) { completableFuture .complete(MessageId.earliest); return completableFuture; } - ledgerImpl.asyncReadEntry(position, new AsyncCallbacks.ReadEntryCallback() { + ledger.asyncReadEntry(position, new AsyncCallbacks.ReadEntryCallback() { @Override public void readEntryComplete(Entry entry, Object ctx) { try { @@ -3162,17 +3882,29 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { public synchronized void triggerCompaction() throws PulsarServerException, AlreadyRunningException { if (currentCompaction.isDone()) { + if (!lock.readLock().tryLock()) { + log.info("[{}] Conflict topic-close, topic-delete, skip triggering compaction", topic); + return; + } + try { + if (isClosingOrDeleting) { + log.info("[{}] Topic is closing or deleting, skip triggering compaction", topic); + return; + } - if (strategicCompactionMap.containsKey(topic)) { - currentCompaction = brokerService.pulsar().getStrategicCompactor() - .compact(topic, strategicCompactionMap.get(topic)); - } else { - currentCompaction = brokerService.pulsar().getCompactor().compact(topic); + if (strategicCompactionMap.containsKey(topic)) { + currentCompaction = brokerService.pulsar().getStrategicCompactor() + .compact(topic, strategicCompactionMap.get(topic)); + } else { + currentCompaction = topicCompactionService.compact().thenApply(x -> null); + } + } finally { + lock.readLock().unlock(); } currentCompaction.whenComplete((ignore, ex) -> { - if (ex != null){ - log.warn("[{}] Compaction failure.", topic, ex); - } + if (ex != null) { + log.warn("[{}] Compaction failure.", topic, ex); + } }); } else { throw new AlreadyRunningException("Compaction already in progress"); @@ -3188,7 +3920,7 @@ public synchronized LongRunningProcessStatus compactionStatus() { return LongRunningProcessStatus.forStatus(LongRunningProcessStatus.Status.RUNNING); } else { try { - if (current.join() == COMPACTION_NEVER_RUN) { + if (Objects.equals(current.join(), COMPACTION_NEVER_RUN)) { return LongRunningProcessStatus.forStatus(LongRunningProcessStatus.Status.NOT_RUN); } else { return LongRunningProcessStatus.forStatus(LongRunningProcessStatus.Status.SUCCESS); @@ -3204,11 +3936,11 @@ public synchronized void triggerOffload(MessageIdImpl messageId) throws AlreadyR CompletableFuture promise = currentOffload = new CompletableFuture<>(); log.info("[{}] Starting offload operation at messageId {}", topic, messageId); getManagedLedger().asyncOffloadPrefix( - PositionImpl.get(messageId.getLedgerId(), messageId.getEntryId()), + PositionFactory.create(messageId.getLedgerId(), messageId.getEntryId()), new OffloadCallback() { @Override public void offloadComplete(Position pos, Object ctx) { - PositionImpl impl = (PositionImpl) pos; + Position impl = pos; log.info("[{}] Completed successfully offload operation at messageId {}", topic, messageId); promise.complete(new MessageIdImpl(impl.getLedgerId(), impl.getEntryId(), -1)); } @@ -3252,10 +3984,19 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem .toList().size()) .sum(); if (hasSchema - || (!producers.isEmpty()) + || (userCreatedProducerCount > 0) || (numActiveConsumersWithoutAutoSchema != 0) || (ledger.getTotalSize() != 0)) { - return checkSchemaCompatibleForConsumer(schema); + return checkSchemaCompatibleForConsumer(schema) + .exceptionally(ex -> { + Throwable realCause = FutureUtil.unwrapCompletionException(ex); + if (realCause instanceof NotExistSchemaException) { + throw FutureUtil.wrapToCompletionException( + new IncompatibleSchemaException("Failed to add schema to an active topic" + + " with empty(BYTES) schema: new schema type " + schema.getType())); + } + throw FutureUtil.wrapToCompletionException(realCause); + }); } else { return addSchema(schema).thenCompose(schemaVersion -> CompletableFuture.completedFuture(null)); @@ -3284,12 +4025,14 @@ private synchronized void checkReplicatedSubscriptionControllerState(boolean sho boolean isCurrentlyEnabled = replicatedSubscriptionsController.isPresent(); boolean isEnableReplicatedSubscriptions = brokerService.pulsar().getConfiguration().isEnableReplicatedSubscriptions(); + boolean replicationEnabled = this.topicPolicies.getReplicationClusters().get().size() > 1; - if (shouldBeEnabled && !isCurrentlyEnabled && isEnableReplicatedSubscriptions) { + if (shouldBeEnabled && !isCurrentlyEnabled && isEnableReplicatedSubscriptions && replicationEnabled) { log.info("[{}] Enabling replicated subscriptions controller", topic); replicatedSubscriptionsController = Optional.of(new ReplicatedSubscriptionsController(this, brokerService.pulsar().getConfiguration().getClusterName())); - } else if (isCurrentlyEnabled && !shouldBeEnabled || !isEnableReplicatedSubscriptions) { + } else if (isCurrentlyEnabled && (!shouldBeEnabled || !isEnableReplicatedSubscriptions + || !replicationEnabled)) { log.info("[{}] Disabled replicated subscriptions controller", topic); replicatedSubscriptionsController.ifPresent(ReplicatedSubscriptionsController::close); replicatedSubscriptionsController = Optional.empty(); @@ -3311,8 +4054,8 @@ public Optional getReplicatedSubscriptionCont return replicatedSubscriptionsController; } - public CompactedTopic getCompactedTopic() { - return compactedTopic; + public TopicCompactionService getTopicCompactionService() { + return this.topicCompactionService; } @Override @@ -3339,7 +4082,7 @@ private synchronized void fence() { final int timeout = brokerService.pulsar().getConfiguration().getTopicFencingTimeoutSeconds(); if (timeout > 0) { this.fencedTopicMonitoringTask = brokerService.executor().schedule(this::closeFencedTopicForcefully, - timeout, TimeUnit.SECONDS); + timeout, SECONDS); } } } @@ -3369,17 +4112,32 @@ private void fenceTopicToCloseOrDelete() { } private void unfenceTopicToResume() { - subscriptions.values().forEach(sub -> sub.resumeAfterFence()); isFenced = false; isClosingOrDeleting = false; + subscriptions.values().forEach(sub -> sub.resumeAfterFence()); + unfenceReplicatorsToResume(); + } + + private void unfenceReplicatorsToResume() { + checkReplication(); + checkShadowReplication(); + } + + private void removeTerminatedReplicators(Map replicators) { + Map terminatedReplicators = new HashMap<>(); + replicators.forEach((cluster, replicator) -> { + if (replicator.isTerminated()) { + terminatedReplicators.put(cluster, replicator); + } + }); + terminatedReplicators.entrySet().forEach(entry -> { + replicators.remove(entry.getKey(), entry.getValue()); + }); } @Override public void publishTxnMessage(TxnID txnID, ByteBuf headersAndPayload, PublishContext publishContext) { pendingWriteOps.incrementAndGet(); - // in order to avoid the opAddEntry retain - - // in order to promise the publish txn message orderly, we should change the transactionCompletableFuture if (isFenced) { publishContext.completed(new TopicFencedException("fenced"), -1, -1); @@ -3391,6 +4149,14 @@ public void publishTxnMessage(TxnID txnID, ByteBuf headersAndPayload, PublishCon decrementPendingWriteOpsAndCheck(); return; } + if (isExceedMaximumDeliveryDelay(headersAndPayload)) { + publishContext.completed( + new NotAllowedException( + String.format("Exceeds max allowed delivery delay of %s milliseconds", + getDelayedDeliveryMaxDelayInMillis())), -1, -1); + decrementPendingWriteOpsAndCheck(); + return; + } MessageDeduplication.MessageDupStatus status = messageDeduplication.isDuplicate(publishContext, headersAndPayload); @@ -3400,23 +4166,21 @@ public void publishTxnMessage(TxnID txnID, ByteBuf headersAndPayload, PublishCon .thenAccept(position -> { // Message has been successfully persisted messageDeduplication.recordMessagePersisted(publishContext, - (PositionImpl) position); + position); publishContext.setProperty("txn_id", txnID.toString()); - publishContext.completed(null, ((PositionImpl) position).getLedgerId(), - ((PositionImpl) position).getEntryId()); + publishContext.completed(null, position.getLedgerId(), + position.getEntryId()); decrementPendingWriteOpsAndCheck(); }) .exceptionally(throwable -> { - throwable = throwable.getCause(); + throwable = FutureUtil.unwrapCompletionException(throwable); if (throwable instanceof NotAllowedException) { publishContext.completed((NotAllowedException) throwable, -1, -1); decrementPendingWriteOpsAndCheck(); - return null; - } else if (!(throwable instanceof ManagedLedgerException)) { - throwable = new ManagedLedgerException(throwable); + } else { + addFailed(ManagedLedgerException.getManagedLedgerException(throwable), publishContext); } - addFailed((ManagedLedgerException) throwable, publishContext); return null; }); break; @@ -3457,10 +4221,33 @@ public boolean isDelayedDeliveryEnabled() { return topicPolicies.getDelayedDeliveryEnabled().get(); } + public long getDelayedDeliveryMaxDelayInMillis() { + return topicPolicies.getDelayedDeliveryMaxDelayInMillis().get(); + } + public int getMaxUnackedMessagesOnSubscription() { return topicPolicies.getMaxUnackedMessagesOnSubscription().get(); } + public boolean isDispatcherPauseOnAckStatePersistentEnabled() { + Boolean b = topicPolicies.getDispatcherPauseOnAckStatePersistentEnabled().get(); + return b == null ? false : b.booleanValue(); + } + + @Override + public void updateBrokerDispatchPauseOnAckStatePersistentEnabled() { + super.updateBrokerDispatchPauseOnAckStatePersistentEnabled(); + // Trigger new read if subscriptions has been paused before. + if (!topicPolicies.getDispatcherPauseOnAckStatePersistentEnabled().get()) { + getSubscriptions().forEach((sName, subscription) -> { + if (subscription.getDispatcher() == null) { + return; + } + subscription.getDispatcher().checkAndResumeIfPaused(); + }); + } + } + @Override public void onUpdate(TopicPolicies policies) { if (log.isDebugEnabled()) { @@ -3469,51 +4256,42 @@ public void onUpdate(TopicPolicies policies) { if (policies == null) { return; } + // Update props. + // The component "EntryFilters" is update in the method "updateTopicPolicy(data)". + // see more detail: https://github.com/apache/pulsar/pull/19364. updateTopicPolicy(policies); shadowTopics = policies.getShadowTopics(); - updateDispatchRateLimiter(); - updateSubscriptionsDispatcherRateLimiter().thenRun(() -> { - updatePublishDispatcher(); - updateSubscribeRateLimiter(); - replicators.forEach((name, replicator) -> replicator.updateRateLimiter()); - shadowReplicators.forEach((name, replicator) -> replicator.updateRateLimiter()); - checkMessageExpiry(); - checkReplicationAndRetryOnFailure(); - - checkDeduplicationStatus(); - - preCreateSubscriptionForCompactionIfNeeded(); + checkReplicatedSubscriptionControllerState(); - // update managed ledger config - checkPersistencePolicies(); - }).exceptionally(e -> { - Throwable t = e instanceof CompletionException ? e.getCause() : e; - log.error("[{}] update topic policy error: {}", topic, t.getMessage(), t); - return null; - }); + // Apply policies for components(not contains the specified policies which only defined in namespace policies). + FutureUtil.waitForAll(applyUpdatedTopicPolicies()) + .thenAccept(__ -> log.info("[{}] topic-level policies updated successfully", topic)) + .exceptionally(e -> { + Throwable t = FutureUtil.unwrapCompletionException(e); + log.error("[{}] update topic-level policy error: {}", topic, t.getMessage(), t); + return null; + }); } - private CompletableFuture updateSubscriptionsDispatcherRateLimiter() { - List> subscriptionCheckFutures = new ArrayList<>((int) subscriptions.size()); + private void updateSubscriptionsDispatcherRateLimiter() { subscriptions.forEach((subName, sub) -> { - List> consumerCheckFutures = new ArrayList<>(sub.getConsumers().size()); - sub.getConsumers().forEach(consumer -> consumerCheckFutures.add(consumer.checkPermissionsAsync())); - subscriptionCheckFutures.add(FutureUtil.waitForAll(consumerCheckFutures).thenRun(() -> { - Dispatcher dispatcher = sub.getDispatcher(); - if (dispatcher != null) { - dispatcher.updateRateLimiter(); - } - })); + Dispatcher dispatcher = sub.getDispatcher(); + if (dispatcher != null) { + dispatcher.updateRateLimiter(); + } }); - return FutureUtil.waitForAll(subscriptionCheckFutures); } protected CompletableFuture initTopicPolicy() { - if (brokerService.pulsar().getConfig().isSystemTopicEnabled() - && brokerService.pulsar().getConfig().isTopicLevelPoliciesEnabled()) { - return CompletableFuture.completedFuture(null).thenRunAsync(() -> onUpdate( - brokerService.getPulsar().getTopicPoliciesService() - .getTopicPoliciesIfExists(TopicName.getPartitionedTopicName(topic))), + final var topicPoliciesService = brokerService.pulsar().getTopicPoliciesService(); + final var partitionedTopicName = TopicName.getPartitionedTopicName(topic); + if (topicPoliciesService.registerListener(partitionedTopicName, this)) { + if (ExtensibleLoadManagerImpl.isInternalTopic(topic)) { + return CompletableFuture.completedFuture(null); + } + return topicPoliciesService.getTopicPoliciesAsync(partitionedTopicName, + TopicPoliciesService.GetType.DEFAULT + ).thenAcceptAsync(optionalPolicies -> optionalPolicies.ifPresent(this::onUpdate), brokerService.getTopicOrderedExecutor()); } return CompletableFuture.completedFuture(null); @@ -3548,18 +4326,22 @@ public boolean checkSubscriptionTypesEnable(SubType subType) { } public TransactionBufferStats getTransactionBufferStats(boolean lowWaterMarks) { - return this.transactionBuffer.getStats(lowWaterMarks); + return getTransactionBufferStats(lowWaterMarks, false); + } + + public TransactionBufferStats getTransactionBufferStats(boolean lowWaterMarks, boolean segmentStats) { + return this.transactionBuffer.getStats(lowWaterMarks, segmentStats); } public TransactionPendingAckStats getTransactionPendingAckStats(String subName, boolean lowWaterMarks) { return this.subscriptions.get(subName).getTransactionPendingAckStats(lowWaterMarks); } - public PositionImpl getMaxReadPosition() { + public Position getMaxReadPosition() { return this.transactionBuffer.getMaxReadPosition(); } - public boolean isTxnAborted(TxnID txnID, PositionImpl readPosition) { + public boolean isTxnAborted(TxnID txnID, Position readPosition) { return this.transactionBuffer.isTxnAborted(txnID, readPosition); } @@ -3577,6 +4359,10 @@ public boolean isMigrated() { return ledger.isMigrated(); } + public boolean isDeduplicationEnabled() { + return getHierarchyTopicPolicies().getDeduplicationEnabled().get(); + } + public TransactionInPendingAckStats getTransactionInPendingAckStats(TxnID txnID, String subName) { return this.subscriptions.get(subName).getTransactionInPendingAckStats(txnID); } @@ -3594,11 +4380,30 @@ private CompletableFuture transactionBufferCleanupAndClose() { return transactionBuffer.clearSnapshot().thenCompose(__ -> transactionBuffer.closeAsync()); } - public long getLastDataMessagePublishedTimestamp() { - return lastDataMessagePublishedTimestamp; - } - public Optional getShadowSourceTopic() { return Optional.ofNullable(shadowSourceTopic); } + + protected boolean isExceedMaximumDeliveryDelay(ByteBuf headersAndPayload) { + if (isDelayedDeliveryEnabled()) { + long maxDeliveryDelayInMs = getDelayedDeliveryMaxDelayInMillis(); + if (maxDeliveryDelayInMs > 0) { + headersAndPayload.markReaderIndex(); + MessageMetadata msgMetadata = Commands.parseMessageMetadata(headersAndPayload); + headersAndPayload.resetReaderIndex(); + return msgMetadata.hasDeliverAtTime() + && msgMetadata.getDeliverAtTime() - msgMetadata.getPublishTime() > maxDeliveryDelayInMs; + } + } + return false; + } + + @Override + public PersistentTopicAttributes getTopicAttributes() { + if (persistentTopicAttributes != null) { + return persistentTopicAttributes; + } + return PERSISTENT_TOPIC_ATTRIBUTES_FIELD_UPDATER.updateAndGet(this, + old -> old != null ? old : new PersistentTopicAttributes(TopicName.get(topic))); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopicMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopicMetrics.java new file mode 100644 index 0000000000000..d8ebece7a51cb --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopicMetrics.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service.persistent; + +import java.util.concurrent.atomic.LongAdder; +import lombok.Getter; + +@Getter +public class PersistentTopicMetrics { + + private final BacklogQuotaMetrics backlogQuotaMetrics = new BacklogQuotaMetrics(); + + private final TransactionBufferClientMetrics transactionBufferClientMetrics = new TransactionBufferClientMetrics(); + + public static class BacklogQuotaMetrics { + private final LongAdder timeBasedBacklogQuotaExceededEvictionCount = new LongAdder(); + private final LongAdder sizeBasedBacklogQuotaExceededEvictionCount = new LongAdder(); + + public void recordTimeBasedBacklogEviction() { + timeBasedBacklogQuotaExceededEvictionCount.increment(); + } + + public void recordSizeBasedBacklogEviction() { + sizeBasedBacklogQuotaExceededEvictionCount.increment(); + } + + public long getSizeBasedBacklogQuotaExceededEvictionCount() { + return sizeBasedBacklogQuotaExceededEvictionCount.longValue(); + } + + public long getTimeBasedBacklogQuotaExceededEvictionCount() { + return timeBasedBacklogQuotaExceededEvictionCount.longValue(); + } + } + + @Getter + public static class TransactionBufferClientMetrics { + private final LongAdder commitSucceededCount = new LongAdder(); + private final LongAdder commitFailedCount = new LongAdder(); + + private final LongAdder abortSucceededCount = new LongAdder(); + private final LongAdder abortFailedCount = new LongAdder(); + } +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/CompactorSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PulsarCompactorSubscription.java similarity index 81% rename from pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/CompactorSubscription.java rename to pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PulsarCompactorSubscription.java index ec34aeffbec4c..fe13aeb572e2e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/CompactorSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PulsarCompactorSubscription.java @@ -22,21 +22,24 @@ import static org.apache.pulsar.broker.service.AbstractBaseDispatcher.checkAndApplyReachedEndOfTopicOrTopicMigration; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; import org.apache.pulsar.common.api.proto.CommandAck.AckType; import org.apache.pulsar.compaction.CompactedTopic; +import org.apache.pulsar.compaction.CompactedTopicContext; +import org.apache.pulsar.compaction.CompactedTopicImpl; import org.apache.pulsar.compaction.Compactor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class CompactorSubscription extends PersistentSubscription { +public class PulsarCompactorSubscription extends PersistentSubscription { private final CompactedTopic compactedTopic; - public CompactorSubscription(PersistentTopic topic, CompactedTopic compactedTopic, - String subscriptionName, ManagedCursor cursor) { + public PulsarCompactorSubscription(PersistentTopic topic, CompactedTopic compactedTopic, + String subscriptionName, ManagedCursor cursor) { super(topic, subscriptionName, cursor, false); checkArgument(subscriptionName.equals(Compactor.COMPACTION_SUBSCRIPTION)); this.compactedTopic = compactedTopic; @@ -106,5 +109,19 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { } } - private static final Logger log = LoggerFactory.getLogger(CompactorSubscription.class); + CompletableFuture cleanCompactedLedger() { + final CompletableFuture compactedTopicContextFuture = + ((CompactedTopicImpl) compactedTopic).getCompactedTopicContextFuture(); + if (compactedTopicContextFuture != null) { + return compactedTopicContextFuture.thenCompose(context -> { + long compactedLedgerId = context.getLedger().getId(); + ((CompactedTopicImpl) compactedTopic).reset(); + return compactedTopic.deleteCompactedLedger(compactedLedgerId); + }); + } else { + return CompletableFuture.completedFuture(null); + } + } + + private static final Logger log = LoggerFactory.getLogger(PulsarCompactorSubscription.class); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionSnapshotCache.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionSnapshotCache.java index 63af5a1e484af..f78aabfd821c3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionSnapshotCache.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionSnapshotCache.java @@ -21,7 +21,8 @@ import java.util.NavigableMap; import java.util.TreeMap; import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.pulsar.common.api.proto.MarkersMessageIdData; import org.apache.pulsar.common.api.proto.ReplicatedSubscriptionsSnapshot; @@ -31,7 +32,7 @@ @Slf4j public class ReplicatedSubscriptionSnapshotCache { private final String subscription; - private final NavigableMap snapshots; + private final NavigableMap snapshots; private final int maxSnapshotToCache; public ReplicatedSubscriptionSnapshotCache(String subscription, int maxSnapshotToCache) { @@ -42,7 +43,7 @@ public ReplicatedSubscriptionSnapshotCache(String subscription, int maxSnapshotT public synchronized void addNewSnapshot(ReplicatedSubscriptionsSnapshot snapshot) { MarkersMessageIdData msgId = snapshot.getLocalMessageId(); - PositionImpl position = new PositionImpl(msgId.getLedgerId(), msgId.getEntryId()); + Position position = PositionFactory.create(msgId.getLedgerId(), msgId.getEntryId()); if (log.isDebugEnabled()) { log.debug("[{}] Added new replicated-subscription snapshot at {} -- {}", subscription, position, @@ -61,10 +62,10 @@ public synchronized void addNewSnapshot(ReplicatedSubscriptionsSnapshot snapshot * Signal that the mark-delete position on the subscription has been advanced. If there is a snapshot that * correspond to this position, it will returned, other it will return null. */ - public synchronized ReplicatedSubscriptionsSnapshot advancedMarkDeletePosition(PositionImpl pos) { + public synchronized ReplicatedSubscriptionsSnapshot advancedMarkDeletePosition(Position pos) { ReplicatedSubscriptionsSnapshot snapshot = null; while (!snapshots.isEmpty()) { - PositionImpl first = snapshots.firstKey(); + Position first = snapshots.firstKey(); if (first.compareTo(pos) > 0) { // Snapshot is associated which an higher position, so it cannot be used now break; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java index 335f2cf8eec08..f56cf9de66b75 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java @@ -35,10 +35,11 @@ import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.pulsar.broker.service.Replicator; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.stats.OpenTelemetryReplicatedSubscriptionStats; import org.apache.pulsar.common.api.proto.ClusterMessageId; import org.apache.pulsar.common.api.proto.CommandAck.AckType; import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition; @@ -49,6 +50,7 @@ import org.apache.pulsar.common.api.proto.ReplicatedSubscriptionsSnapshotResponse; import org.apache.pulsar.common.api.proto.ReplicatedSubscriptionsUpdate; import org.apache.pulsar.common.protocol.Markers; +import org.apache.pulsar.opentelemetry.annotations.PulsarDeprecatedMetric; /** * Encapsulate all the logic of replicated subscriptions tracking for a given topic. @@ -70,24 +72,28 @@ public class ReplicatedSubscriptionsController implements AutoCloseable, Topic.P private final ConcurrentMap pendingSnapshots = new ConcurrentHashMap<>(); + @PulsarDeprecatedMetric( + newMetricName = OpenTelemetryReplicatedSubscriptionStats.SNAPSHOT_OPERATION_COUNT_METRIC_NAME) + @Deprecated private static final Gauge pendingSnapshotsMetric = Gauge .build("pulsar_replicated_subscriptions_pending_snapshots", "Counter of currently pending snapshots") .register(); + private final OpenTelemetryReplicatedSubscriptionStats stats; + public ReplicatedSubscriptionsController(PersistentTopic topic, String localCluster) { this.topic = topic; this.localCluster = localCluster; - timer = topic.getBrokerService().pulsar().getExecutor() + var pulsar = topic.getBrokerService().pulsar(); + timer = pulsar.getExecutor() .scheduleAtFixedRate(catchingAndLoggingThrowables(this::startNewSnapshot), 0, - topic.getBrokerService().pulsar().getConfiguration() - .getReplicatedSubscriptionsSnapshotFrequencyMillis(), + pulsar.getConfiguration().getReplicatedSubscriptionsSnapshotFrequencyMillis(), TimeUnit.MILLISECONDS); + stats = pulsar.getOpenTelemetryReplicatedSubscriptionStats(); } public void receivedReplicatedSubscriptionMarker(Position position, int markerType, ByteBuf payload) { - MarkerType m = null; - try { switch (markerType) { case MarkerType.REPLICATED_SUBSCRIPTION_SNAPSHOT_REQUEST_VALUE: @@ -105,7 +111,6 @@ public void receivedReplicatedSubscriptionMarker(Position position, int markerTy default: // Ignore } - } catch (IOException e) { log.warn("[{}] Failed to parse marker: {}", topic.getName(), e); } @@ -140,7 +145,7 @@ private void receivedSnapshotRequest(ReplicatedSubscriptionsSnapshotRequest requ // Send response containing the current last written message id. The response // marker we're publishing locally and then replicating will have a higher // message id. - PositionImpl lastMsgId = (PositionImpl) topic.getLastPosition(); + Position lastMsgId = topic.getLastPosition(); if (log.isDebugEnabled()) { log.debug("[{}] Received snapshot request. Last msg id: {}", topic.getName(), lastMsgId); } @@ -181,7 +186,7 @@ private void receiveSubscriptionUpdated(ReplicatedSubscriptionsUpdate update) { return; } - Position pos = new PositionImpl(updatedMessageId.getLedgerId(), updatedMessageId.getEntryId()); + Position pos = PositionFactory.create(updatedMessageId.getLedgerId(), updatedMessageId.getEntryId()); if (log.isDebugEnabled()) { log.debug("[{}][{}] Received update for subscription to {}", topic, update.getSubscriptionName(), pos); @@ -191,19 +196,23 @@ private void receiveSubscriptionUpdated(ReplicatedSubscriptionsUpdate update) { if (sub != null) { sub.acknowledgeMessage(Collections.singletonList(pos), AckType.Cumulative, Collections.emptyMap()); } else { - // Subscription doesn't exist. We need to force the creation of the subscription in this cluster, because - log.info("[{}][{}] Creating subscription at {}:{} after receiving update from replicated subcription", + // Subscription doesn't exist. We need to force the creation of the subscription in this cluster. + log.info("[{}][{}] Creating subscription at {}:{} after receiving update from replicated subscription", topic, update.getSubscriptionName(), updatedMessageId.getLedgerId(), pos); - topic.createSubscription(update.getSubscriptionName(), - InitialPosition.Latest, true /* replicateSubscriptionState */, null); + topic.createSubscription(update.getSubscriptionName(), InitialPosition.Earliest, + true /* replicateSubscriptionState */, Collections.emptyMap()) + .thenAccept(subscriptionCreated -> { + subscriptionCreated.acknowledgeMessage(Collections.singletonList(pos), + AckType.Cumulative, Collections.emptyMap()); + }); } } private void startNewSnapshot() { cleanupTimedOutSnapshots(); - if (topic.getLastDataMessagePublishedTimestamp() < lastCompletedSnapshotStartTime - || topic.getLastDataMessagePublishedTimestamp() == 0) { + if (topic.getLastMaxReadPositionMovedForwardTimestamp() < lastCompletedSnapshotStartTime + || topic.getLastMaxReadPositionMovedForwardTimestamp() == 0) { // There was no message written since the last snapshot, we can skip creating a new snapshot if (log.isDebugEnabled()) { log.debug("[{}] There is no new data in topic. Skipping snapshot creation.", topic.getName()); @@ -232,11 +241,12 @@ private void startNewSnapshot() { } pendingSnapshotsMetric.inc(); + stats.recordSnapshotStarted(); ReplicatedSubscriptionsSnapshotBuilder builder = new ReplicatedSubscriptionsSnapshotBuilder(this, - topic.getReplicators().keys(), topic.getBrokerService().pulsar().getConfiguration(), Clock.systemUTC()); + topic.getReplicators().keySet(), topic.getBrokerService().pulsar().getConfiguration(), + Clock.systemUTC()); pendingSnapshots.put(builder.getSnapshotId(), builder); builder.start(); - } public Optional getLastCompletedSnapshotId() { @@ -253,6 +263,8 @@ private void cleanupTimedOutSnapshots() { } pendingSnapshotsMetric.dec(); + var latencyMillis = entry.getValue().getDurationMillis(); + stats.recordSnapshotTimedOut(latencyMillis); it.remove(); } } @@ -260,11 +272,15 @@ private void cleanupTimedOutSnapshots() { void snapshotCompleted(String snapshotId) { ReplicatedSubscriptionsSnapshotBuilder snapshot = pendingSnapshots.remove(snapshotId); - pendingSnapshotsMetric.dec(); lastCompletedSnapshotId = snapshotId; if (snapshot != null) { lastCompletedSnapshotStartTime = snapshot.getStartTimeMillis(); + + pendingSnapshotsMetric.dec(); + var latencyMillis = snapshot.getDurationMillis(); + ReplicatedSubscriptionsSnapshotBuilder.SNAPSHOT_METRIC.observe(latencyMillis); + stats.recordSnapshotCompleted(latencyMillis); } } @@ -287,7 +303,7 @@ public void completed(Exception e, long ledgerId, long entryId) { log.debug("[{}] Published marker at {}:{}. Exception: {}", topic.getName(), ledgerId, entryId, e); } - this.positionOfLastLocalMarker = new PositionImpl(ledgerId, entryId); + this.positionOfLastLocalMarker = PositionFactory.create(ledgerId, entryId); } PersistentTopic topic() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsSnapshotBuilder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsSnapshotBuilder.java index 53ba7193dc696..e08b549f8aec9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsSnapshotBuilder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsSnapshotBuilder.java @@ -20,7 +20,6 @@ import io.prometheus.client.Summary; import java.time.Clock; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; @@ -29,11 +28,12 @@ import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.stats.OpenTelemetryReplicatedSubscriptionStats; import org.apache.pulsar.common.api.proto.MarkersMessageIdData; import org.apache.pulsar.common.api.proto.ReplicatedSubscriptionsSnapshotResponse; import org.apache.pulsar.common.protocol.Markers; +import org.apache.pulsar.opentelemetry.annotations.PulsarDeprecatedMetric; @Slf4j public class ReplicatedSubscriptionsSnapshotBuilder { @@ -42,7 +42,7 @@ public class ReplicatedSubscriptionsSnapshotBuilder { private final ReplicatedSubscriptionsController controller; private final Map responses = new TreeMap<>(); - private final List remoteClusters; + private final Set remoteClusters; private final Set missingClusters; private final boolean needTwoRounds; @@ -53,11 +53,13 @@ public class ReplicatedSubscriptionsSnapshotBuilder { private final Clock clock; - private static final Summary snapshotMetric = Summary.build("pulsar_replicated_subscriptions_snapshot_ms", + @PulsarDeprecatedMetric(newMetricName = OpenTelemetryReplicatedSubscriptionStats.SNAPSHOT_DURATION_METRIC_NAME) + @Deprecated + public static final Summary SNAPSHOT_METRIC = Summary.build("pulsar_replicated_subscriptions_snapshot_ms", "Time taken to create a consistent snapshot across clusters").register(); public ReplicatedSubscriptionsSnapshotBuilder(ReplicatedSubscriptionsController controller, - List remoteClusters, ServiceConfiguration conf, Clock clock) { + Set remoteClusters, ServiceConfiguration conf, Clock clock) { this.snapshotId = UUID.randomUUID().toString(); this.controller = controller; this.remoteClusters = remoteClusters; @@ -118,14 +120,12 @@ synchronized void receivedSnapshotResponse(Position position, ReplicatedSubscrip log.debug("[{}] Snapshot is complete {}", controller.topic().getName(), snapshotId); } // Snapshot is now complete, store it in the local topic - PositionImpl p = (PositionImpl) position; + Position p = position; controller.writeMarker( Markers.newReplicatedSubscriptionsSnapshot(snapshotId, controller.localCluster(), p.getLedgerId(), p.getEntryId(), responses)); controller.snapshotCompleted(snapshotId); - double latencyMillis = clock.millis() - startTimeMillis; - snapshotMetric.observe(latencyMillis); } boolean isTimedOut() { @@ -135,4 +135,8 @@ boolean isTimedOut() { long getStartTimeMillis() { return startTimeMillis; } + + long getDurationMillis() { + return clock.millis() - startTimeMillis; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java index fb306348bcdbb..65bcbfd131f12 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java @@ -67,7 +67,7 @@ protected boolean replicateEntries(List entries) { ByteBuf headersAndPayload = entry.getDataBuffer(); MessageImpl msg; try { - msg = MessageImpl.deserializeSkipBrokerEntryMetaData(headersAndPayload); + msg = MessageImpl.deserializeMetadataWithEmptyPayload(headersAndPayload); } catch (Throwable t) { log.error("[{}] Failed to deserialize message at {} (buffer size: {}): {}", replicatorId, entry.getPosition(), length, t.getMessage(), t); @@ -76,18 +76,6 @@ protected boolean replicateEntries(List entries) { continue; } - if (msg.isExpired(messageTTLInSeconds)) { - msgExpired.recordEvent(0 /* no value stat */); - if (log.isDebugEnabled()) { - log.debug("[{}] Discarding expired message at position {}, replicateTo {}", - replicatorId, entry.getPosition(), msg.getReplicateTo()); - } - cursor.asyncDelete(entry.getPosition(), this, entry.getPosition()); - entry.release(); - msg.recycle(); - continue; - } - if (STATE_UPDATER.get(this) != State.Started || isLocalMessageSkippedOnce) { // The producer is not ready yet after having stopped/restarted. Drop the message because it will // recovered when the producer is ready @@ -101,9 +89,11 @@ protected boolean replicateEntries(List entries) { continue; } - dispatchRateLimiter.ifPresent(rateLimiter -> rateLimiter.tryDispatchPermit(1, entry.getLength())); + dispatchRateLimiter.ifPresent(rateLimiter -> rateLimiter.consumeDispatchQuota(1, entry.getLength())); - msgOut.recordEvent(headersAndPayload.readableBytes()); + msgOut.recordEvent(msg.getDataBuffer().readableBytes()); + stats.incrementMsgOutCounter(); + stats.incrementBytesOutCounter(msg.getDataBuffer().readableBytes()); msg.setReplicatedFrom(localCluster); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SubscribeRateLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SubscribeRateLimiter.java index a052f1f6abf33..b1de10e73b76f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SubscribeRateLimiter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SubscribeRateLimiter.java @@ -19,39 +19,32 @@ package org.apache.pulsar.broker.service.persistent; -import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.base.MoreObjects; import java.util.Objects; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import org.apache.pulsar.broker.qos.AsyncTokenBucket; import org.apache.pulsar.broker.service.BrokerService; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.SubscribeRate; -import org.apache.pulsar.common.util.RateLimiter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class SubscribeRateLimiter { - private final String topicName; private final BrokerService brokerService; - private ConcurrentHashMap subscribeRateLimiter; - private final ScheduledExecutorService executorService; - private ScheduledFuture resetTask; + private ConcurrentHashMap subscribeRateLimiter; private SubscribeRate subscribeRate; public SubscribeRateLimiter(PersistentTopic topic) { this.topicName = topic.getName(); this.brokerService = topic.getBrokerService(); subscribeRateLimiter = new ConcurrentHashMap<>(); - this.executorService = brokerService.pulsar().getExecutor(); // get subscribeRate from topic level policies this.subscribeRate = topic.getSubscribeRate(); if (isSubscribeRateEnabled(this.subscribeRate)) { - resetTask = createTask(); log.info("[{}] configured subscribe-dispatch rate at broker {}", this.topicName, subscribeRate); } } @@ -63,7 +56,7 @@ public SubscribeRateLimiter(PersistentTopic topic) { */ public long getAvailableSubscribeRateLimit(ConsumerIdentifier consumerIdentifier) { return subscribeRateLimiter.get(consumerIdentifier) - == null ? -1 : subscribeRateLimiter.get(consumerIdentifier).getAvailablePermits(); + == null ? -1 : subscribeRateLimiter.get(consumerIdentifier).getTokens(); } /** @@ -73,8 +66,15 @@ public long getAvailableSubscribeRateLimit(ConsumerIdentifier consumerIdentifier */ public synchronized boolean tryAcquire(ConsumerIdentifier consumerIdentifier) { addSubscribeLimiterIfAbsent(consumerIdentifier); - return subscribeRateLimiter.get(consumerIdentifier) - == null || subscribeRateLimiter.get(consumerIdentifier).tryAcquire(); + AsyncTokenBucket tokenBucket = subscribeRateLimiter.get(consumerIdentifier); + if (tokenBucket == null) { + return true; + } + if (!tokenBucket.containsTokens(true)) { + return false; + } + tokenBucket.consumeTokens(1); + return true; } /** @@ -85,7 +85,7 @@ public synchronized boolean tryAcquire(ConsumerIdentifier consumerIdentifier) { */ public boolean subscribeAvailable(ConsumerIdentifier consumerIdentifier) { return (subscribeRateLimiter.get(consumerIdentifier) - == null || subscribeRateLimiter.get(consumerIdentifier).getAvailablePermits() > 0); + == null || subscribeRateLimiter.get(consumerIdentifier).containsTokens()); } /** @@ -97,15 +97,11 @@ private synchronized void addSubscribeLimiterIfAbsent(ConsumerIdentifier consume if (subscribeRateLimiter.get(consumerIdentifier) != null || !isSubscribeRateEnabled(this.subscribeRate)) { return; } - updateSubscribeRate(consumerIdentifier, this.subscribeRate); } private synchronized void removeSubscribeLimiter(ConsumerIdentifier consumerIdentifier) { - if (this.subscribeRateLimiter.get(consumerIdentifier) != null) { - this.subscribeRateLimiter.get(consumerIdentifier).close(); - this.subscribeRateLimiter.remove(consumerIdentifier); - } + this.subscribeRateLimiter.remove(consumerIdentifier); } /** @@ -116,23 +112,13 @@ private synchronized void removeSubscribeLimiter(ConsumerIdentifier consumerIden */ private synchronized void updateSubscribeRate(ConsumerIdentifier consumerIdentifier, SubscribeRate subscribeRate) { long ratePerConsumer = subscribeRate.subscribeThrottlingRatePerConsumer; - long ratePeriod = subscribeRate.ratePeriodInSecond; + long ratePeriodNanos = TimeUnit.SECONDS.toNanos(Math.max(subscribeRate.ratePeriodInSecond, 1)); // update subscribe-rateLimiter if (ratePerConsumer > 0) { - if (this.subscribeRateLimiter.get(consumerIdentifier) == null) { - this.subscribeRateLimiter.put(consumerIdentifier, - RateLimiter.builder() - .scheduledExecutorService(brokerService.pulsar().getExecutor()) - .permits(ratePerConsumer) - .rateTime(ratePeriod) - .timeUnit(TimeUnit.SECONDS) - .build()); - } else { - this.subscribeRateLimiter.get(consumerIdentifier) - .setRate(ratePerConsumer, ratePeriod, TimeUnit.SECONDS, - null); - } + AsyncTokenBucket tokenBucket = + AsyncTokenBucket.builder().rate(ratePerConsumer).ratePeriodNanos(ratePeriodNanos).build(); + this.subscribeRateLimiter.put(consumerIdentifier, tokenBucket); } else { // subscribe-rate should be disable and close removeSubscribeLimiter(consumerIdentifier); @@ -144,7 +130,6 @@ public void onSubscribeRateUpdate(SubscribeRate subscribeRate) { return; } this.subscribeRate = subscribeRate; - stopResetTask(); for (ConsumerIdentifier consumerIdentifier : this.subscribeRateLimiter.keySet()) { if (!isSubscribeRateEnabled(this.subscribeRate)) { removeSubscribeLimiter(consumerIdentifier); @@ -153,20 +138,26 @@ public void onSubscribeRateUpdate(SubscribeRate subscribeRate) { } } if (isSubscribeRateEnabled(this.subscribeRate)) { - this.resetTask = createTask(); log.info("[{}] configured subscribe-dispatch rate at broker {}", this.topicName, subscribeRate); } } /** - * Gets configured subscribe-rate from namespace policies. Returns null if subscribe-rate is not configured - * - * @return + * @deprecated Avoid using the deprecated method + * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and blocking + * call. */ + @Deprecated public SubscribeRate getPoliciesSubscribeRate() { return getPoliciesSubscribeRate(brokerService, topicName); } + /** + * @deprecated Avoid using the deprecated method + * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and blocking + * call. + */ + @Deprecated public static SubscribeRate getPoliciesSubscribeRate(BrokerService brokerService, final String topicName) { final String cluster = brokerService.pulsar().getConfiguration().getClusterName(); final Optional policies = DispatchRateLimiter.getPolicies(brokerService, topicName); @@ -201,32 +192,9 @@ public static boolean isSubscribeRateEnabled(SubscribeRate subscribeRate) { } public void close() { - closeAndClearRateLimiters(); - stopResetTask(); - } - private ScheduledFuture createTask() { - return executorService.scheduleAtFixedRate(catchingAndLoggingThrowables(this::closeAndClearRateLimiters), - this.subscribeRate.ratePeriodInSecond, - this.subscribeRate.ratePeriodInSecond, - TimeUnit.SECONDS); } - private void stopResetTask() { - if (this.resetTask != null) { - this.resetTask.cancel(false); - } - } - - private synchronized void closeAndClearRateLimiters() { - // close rate-limiter - this.subscribeRateLimiter.values().forEach(rateLimiter -> { - if (rateLimiter != null) { - rateLimiter.close(); - } - }); - this.subscribeRateLimiter.clear(); - } public SubscribeRate getSubscribeRate() { return subscribeRate; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SystemTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SystemTopic.java index 395a8c9075eb3..f2cec2138a3a0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SystemTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SystemTopic.java @@ -18,13 +18,16 @@ */ package org.apache.pulsar.broker.service.persistent; +import java.util.List; import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerService; +import org.apache.pulsar.broker.service.plugin.EntryFilter; import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.EntryFilters; public class SystemTopic extends PersistentTopic { @@ -77,9 +80,35 @@ public boolean isCompactionEnabled() { return !NamespaceService.isHeartbeatNamespace(TopicName.get(topic)); } + @Override + public boolean isDeduplicationEnabled() { + /* + Disable deduplication on system topic to avoid recovering deduplication WAL + (especially from offloaded topic). + Because the system topic usually is a precondition of other topics. therefore, + we should pay attention on topic loading time. + + Note: If the system topic loading timeout may cause dependent topics to fail to run. + + Dependency diagram: normal topic --rely on--> system topic --rely on--> deduplication recover + --may rely on--> (tiered storage) + */ + return false; + } + @Override public boolean isEncryptionRequired() { // System topics are only written by the broker that can't know the encryption context. return false; } + + @Override + public EntryFilters getEntryFiltersPolicy() { + return null; + } + + @Override + public List getEntryFilters() { + return null; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/plugin/EntryFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/plugin/EntryFilter.java index 2e5a590fa1976..686c72df8c24b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/plugin/EntryFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/plugin/EntryFilter.java @@ -49,7 +49,7 @@ enum FilterResult { */ REJECT, /** - * postpone message, it should not go to this conmumer. + * postpone message, it should not go to this consumer. */ RESCHEDULE } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/plugin/EntryFilterProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/plugin/EntryFilterProvider.java index f93e561542eeb..53418744b5486 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/plugin/EntryFilterProvider.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/plugin/EntryFilterProvider.java @@ -197,7 +197,8 @@ protected EntryFilter load(EntryFilterMetaData metadata) + " does not implement entry filter interface"); } EntryFilter pi = (EntryFilter) filter; - return new EntryFilterWithClassLoader(pi, ncl); + // the classloader is shared with the broker, the instance doesn't own it + return new EntryFilterWithClassLoader(pi, ncl, false); } catch (Throwable e) { if (e instanceof IOException) { throw (IOException) e; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/plugin/EntryFilterWithClassLoader.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/plugin/EntryFilterWithClassLoader.java index c5c5721087788..aab46c62acdb4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/plugin/EntryFilterWithClassLoader.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/plugin/EntryFilterWithClassLoader.java @@ -30,15 +30,23 @@ public class EntryFilterWithClassLoader implements EntryFilter { private final EntryFilter entryFilter; private final NarClassLoader classLoader; + private final boolean classLoaderOwned; - public EntryFilterWithClassLoader(EntryFilter entryFilter, NarClassLoader classLoader) { + public EntryFilterWithClassLoader(EntryFilter entryFilter, NarClassLoader classLoader, boolean classLoaderOwned) { this.entryFilter = entryFilter; this.classLoader = classLoader; + this.classLoaderOwned = classLoaderOwned; } @Override public FilterResult filterEntry(Entry entry, FilterContext context) { - return entryFilter.filterEntry(entry, context); + ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(classLoader); + return entryFilter.filterEntry(entry, context); + } finally { + Thread.currentThread().setContextClassLoader(currentClassLoader); + } } @VisibleForTesting @@ -48,11 +56,20 @@ public EntryFilter getEntryFilter() { @Override public void close() { - entryFilter.close(); + ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader(); try { - classLoader.close(); - } catch (IOException e) { - log.error("close EntryFilterWithClassLoader failed", e); + Thread.currentThread().setContextClassLoader(classLoader); + entryFilter.close(); + } finally { + Thread.currentThread().setContextClassLoader(currentClassLoader); + } + if (classLoaderOwned) { + log.info("Closing classloader {} for EntryFilter {}", classLoader, entryFilter.getClass().getName()); + try { + classLoader.close(); + } catch (IOException e) { + log.error("close EntryFilterWithClassLoader failed", e); + } } } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java index 1e75834a12988..e5fc7800c5170 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java @@ -64,8 +64,10 @@ public void checkCompatible(Iterable from, SchemaData to, SchemaComp log.warn("Error during schema parsing: {}", e.getMessage()); throw new IncompatibleSchemaException(e); } catch (SchemaValidationException e) { - log.warn("Error during schema compatibility check: {}", e.getMessage()); - throw new IncompatibleSchemaException(e); + String msg = String.format("Error during schema compatibility check with strategy %s: %s: %s", + strategy, e.getClass().getName(), e.getMessage()); + log.warn(msg); + throw new IncompatibleSchemaException(msg, e); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java index a8fc15f296598..f68cdd6473e48 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java @@ -52,7 +52,11 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.service.schema.SchemaStorageFormat.IndexEntry; +import org.apache.pulsar.broker.service.schema.SchemaStorageFormat.SchemaLocator; +import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; import org.apache.pulsar.broker.service.schema.exceptions.SchemaException; +import org.apache.pulsar.common.policies.data.SchemaMetadata; import org.apache.pulsar.common.protocol.schema.SchemaStorage; import org.apache.pulsar.common.protocol.schema.SchemaVersion; import org.apache.pulsar.common.protocol.schema.StoredSchema; @@ -109,7 +113,7 @@ public void start() throws IOException { pulsar.getIoEventLoopGroup(), Optional.empty(), null - ); + ).join(); } @Override @@ -472,18 +476,20 @@ private CompletableFuture updateSchemaLocator( .build() , locatorEntry.version ).thenApply(ignore -> nextVersion).whenComplete((__, ex) -> { - Throwable cause = FutureUtil.unwrapCompletionException(ex); - log.warn("[{}] Failed to update schema locator with position {}", schemaId, position, cause); - if (cause instanceof AlreadyExistsException || cause instanceof BadVersionException) { - bookKeeper.asyncDeleteLedger(position.getLedgerId(), new AsyncCallback.DeleteCallback() { - @Override - public void deleteComplete(int rc, Object ctx) { - if (rc != BKException.Code.OK) { - log.warn("[{}] Failed to delete ledger {} after updating schema locator failed, rc: {}", + if (ex != null) { + Throwable cause = FutureUtil.unwrapCompletionException(ex); + log.warn("[{}] Failed to update schema locator with position {}", schemaId, position, cause); + if (cause instanceof AlreadyExistsException || cause instanceof BadVersionException) { + bookKeeper.asyncDeleteLedger(position.getLedgerId(), new AsyncCallback.DeleteCallback() { + @Override + public void deleteComplete(int rc, Object ctx) { + if (rc != BKException.Code.OK) { + log.warn("[{}] Failed to delete ledger {} after updating schema locator failed, rc: {}", schemaId, position.getLedgerId(), rc); + } } - } - }, null); + }, null); + } } }); } @@ -525,7 +531,7 @@ private CompletableFuture readSchemaEntry( return openLedger(position.getLedgerId()) .thenCompose((ledger) -> - Functions.getLedgerEntry(ledger, position.getEntryId()) + Functions.getLedgerEntry(ledger, position.getEntryId(), config.isSchemaLedgerForceRecovery()) .thenCompose(entry -> closeLedger(ledger) .thenApply(ignore -> entry) ) @@ -551,13 +557,31 @@ private CompletableFuture> getSchemaLocator(String schema o.map(r -> new LocatorEntry(r.getValue(), r.getStat().getVersion()))); } + public CompletableFuture getSchemaMetadata(String schema) { + return getLocator(schema).thenApply(locator -> { + if (!locator.isPresent()) { + return null; + } + SchemaLocator sl = locator.get().locator; + SchemaMetadata metadata = new SchemaMetadata(); + IndexEntry info = sl.getInfo(); + metadata.info = new SchemaMetadata.Entry(info.getPosition().getLedgerId(), info.getPosition().getEntryId(), + info.getVersion()); + metadata.index = sl.getIndexList() == null ? null + : sl.getIndexList().stream().map(i -> new SchemaMetadata.Entry(i.getPosition().getLedgerId(), + i.getPosition().getEntryId(), i.getVersion())).collect(Collectors.toList()); + return metadata; + }); + } + @NotNull private CompletableFuture addEntry(LedgerHandle ledgerHandle, SchemaStorageFormat.SchemaEntry entry) { final CompletableFuture future = new CompletableFuture<>(); ledgerHandle.asyncAddEntry(entry.toByteArray(), (rc, handle, entryId, ctx) -> { if (rc != BKException.Code.OK) { - future.completeExceptionally(bkException("Failed to add entry", rc, ledgerHandle.getId(), -1)); + future.completeExceptionally(bkException("Failed to add entry", rc, ledgerHandle.getId(), -1, + config.isSchemaLedgerForceRecovery())); } else { future.complete(entryId); } @@ -579,7 +603,8 @@ private CompletableFuture createLedger(String schemaId) { LedgerPassword, (rc, handle, ctx) -> { if (rc != BKException.Code.OK) { - future.completeExceptionally(bkException("Failed to create ledger", rc, -1, -1)); + future.completeExceptionally(bkException("Failed to create ledger", rc, -1, -1, + config.isSchemaLedgerForceRecovery())); } else { future.complete(handle); } @@ -600,7 +625,8 @@ private CompletableFuture openLedger(Long ledgerId) { LedgerPassword, (rc, handle, ctx) -> { if (rc != BKException.Code.OK) { - future.completeExceptionally(bkException("Failed to open ledger", rc, ledgerId, -1)); + future.completeExceptionally(bkException("Failed to open ledger", rc, ledgerId, -1, + config.isSchemaLedgerForceRecovery())); } else { future.complete(handle); } @@ -614,7 +640,8 @@ private CompletableFuture closeLedger(LedgerHandle ledgerHandle) { CompletableFuture future = new CompletableFuture<>(); ledgerHandle.asyncClose((rc, handle, ctx) -> { if (rc != BKException.Code.OK) { - future.completeExceptionally(bkException("Failed to close ledger", rc, ledgerHandle.getId(), -1)); + future.completeExceptionally(bkException("Failed to close ledger", rc, ledgerHandle.getId(), -1, + config.isSchemaLedgerForceRecovery())); } else { future.complete(null); } @@ -645,12 +672,14 @@ public CompletableFuture> getStoreLedgerIdsBySchemaId(String schemaId } interface Functions { - static CompletableFuture getLedgerEntry(LedgerHandle ledger, long entry) { + static CompletableFuture getLedgerEntry(LedgerHandle ledger, long entry, + boolean forceRecovery) { final CompletableFuture future = new CompletableFuture<>(); ledger.asyncReadEntries(entry, entry, (rc, handle, entries, ctx) -> { if (rc != BKException.Code.OK) { - future.completeExceptionally(bkException("Failed to read entry", rc, ledger.getId(), entry)); + future.completeExceptionally(bkException("Failed to read entry", rc, ledger.getId(), entry, + forceRecovery)); } else { future.complete(entries.nextElement()); } @@ -697,7 +726,8 @@ static class LocatorEntry { } } - public static Exception bkException(String operation, int rc, long ledgerId, long entryId) { + public static Exception bkException(String operation, int rc, long ledgerId, long entryId, + boolean forceRecovery) { String message = org.apache.bookkeeper.client.api.BKException.getMessage(rc) + " - ledger=" + ledgerId + " - operation=" + operation; @@ -705,7 +735,11 @@ public static Exception bkException(String operation, int rc, long ledgerId, lon message += " - entry=" + entryId; } boolean recoverable = rc != BKException.Code.NoSuchLedgerExistsException - && rc != BKException.Code.NoSuchEntryException; + && rc != BKException.Code.NoSuchEntryException + && rc != BKException.Code.NoSuchLedgerExistsOnMetadataServerException + // if force-recovery is enabled then made it non-recoverable exception + // and force schema to skip this exception and recover immediately + && !forceRecovery; return new SchemaException(recoverable, message); } @@ -713,8 +747,10 @@ public static CompletableFuture ignoreUnrecoverableBKException(Completabl return source.exceptionally(t -> { if (t.getCause() != null && (t.getCause() instanceof SchemaException) + && !(t.getCause() instanceof IncompatibleSchemaException) && !((SchemaException) t.getCause()).isRecoverable()) { - // Meeting NoSuchLedgerExistsException or NoSuchEntryException when reading schemas in + // Meeting NoSuchLedgerExistsException, NoSuchEntryException or + // NoSuchLedgerExistsOnMetadataServerException when reading schemas in // bookkeeper. This also means that the data has already been deleted by other operations // in deleting schema. if (log.isDebugEnabled()) { @@ -726,4 +762,4 @@ public static CompletableFuture ignoreUnrecoverableBKException(Completabl throw t instanceof CompletionException ? (CompletionException) t : new CompletionException(t); }); } -} +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/ProtobufNativeSchemaCompatibilityCheck.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/ProtobufNativeSchemaCompatibilityCheck.java index 16b3b33ec7894..fc935e80dca36 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/ProtobufNativeSchemaCompatibilityCheck.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/ProtobufNativeSchemaCompatibilityCheck.java @@ -67,7 +67,9 @@ public void checkCompatible(Iterable from, SchemaData to, SchemaComp private void checkRootMessageChange(Descriptor fromDescriptor, Descriptor toDescriptor, SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException { if (!fromDescriptor.getFullName().equals(toDescriptor.getFullName())) { - throw new IncompatibleSchemaException("Protobuf root message isn't allow change!"); + throw new IncompatibleSchemaException("Protobuf root message change is not allowed under the '" + + strategy + "' strategy. Original message name: '" + fromDescriptor.getFullName() + + "', new message name: '" + toDescriptor.getFullName() + "'."); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryService.java index 3c5e3aae7ff5d..2a2467d3947ee 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryService.java @@ -21,7 +21,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; -import java.util.concurrent.ScheduledExecutorService; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.schema.validator.SchemaRegistryServiceWithSchemaDataValidator; import org.apache.pulsar.common.protocol.schema.SchemaStorage; import org.apache.pulsar.common.schema.SchemaType; @@ -44,13 +44,13 @@ static Map getCheckers(Set checker } static SchemaRegistryService create(SchemaStorage schemaStorage, Set schemaRegistryCompatibilityCheckers, - ScheduledExecutorService scheduler) { + PulsarService pulsarService) { if (schemaStorage != null) { try { Map checkers = getCheckers(schemaRegistryCompatibilityCheckers); checkers.put(SchemaType.KEY_VALUE, new KeyValueSchemaCompatibilityCheck(checkers)); return SchemaRegistryServiceWithSchemaDataValidator.of( - new SchemaRegistryServiceImpl(schemaStorage, checkers, scheduler)); + new SchemaRegistryServiceImpl(schemaStorage, checkers, pulsarService)); } catch (Exception e) { LOG.warn("Unable to create schema registry storage, defaulting to empty storage", e); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java index ae56df248d85d..c1a394dcfbbb7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java @@ -38,7 +38,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ScheduledExecutorService; import java.util.stream.Collectors; import javax.validation.constraints.NotNull; import lombok.extern.slf4j.Slf4j; @@ -47,7 +46,9 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.mutable.MutableLong; import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; +import org.apache.pulsar.broker.service.schema.exceptions.NotExistSchemaException; import org.apache.pulsar.broker.service.schema.exceptions.SchemaException; import org.apache.pulsar.broker.service.schema.proto.SchemaRegistryFormat; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; @@ -70,19 +71,19 @@ public class SchemaRegistryServiceImpl implements SchemaRegistryService { @VisibleForTesting SchemaRegistryServiceImpl(SchemaStorage schemaStorage, - Map compatibilityChecks, Clock clock, - ScheduledExecutorService scheduler) { + Map compatibilityChecks, + Clock clock, + PulsarService pulsarService) { this.schemaStorage = schemaStorage; this.compatibilityChecks = compatibilityChecks; this.clock = clock; - this.stats = SchemaRegistryStats.getInstance(scheduler); + this.stats = new SchemaRegistryStats(pulsarService); } - @VisibleForTesting SchemaRegistryServiceImpl(SchemaStorage schemaStorage, Map compatibilityChecks, - ScheduledExecutorService scheduler) { - this(schemaStorage, compatibilityChecks, Clock.systemUTC(), scheduler); + PulsarService pulsarService) { + this(schemaStorage, compatibilityChecks, Clock.systemUTC(), pulsarService); } @Override @@ -136,16 +137,17 @@ public CompletableFuture getSchema(String schemaId, SchemaVer } }) .whenComplete((v, t) -> { + var latencyMs = this.clock.millis() - start; if (t != null) { if (log.isDebugEnabled()) { log.debug("[{}] Get schema failed", schemaId); } - this.stats.recordGetFailed(schemaId); + this.stats.recordGetFailed(schemaId, latencyMs); } else { if (log.isDebugEnabled()) { log.debug(null == v ? "[{}] Schema not found" : "[{}] Schema is present", schemaId); } - this.stats.recordGetLatency(schemaId, this.clock.millis() - start); + this.stats.recordGetLatency(schemaId, latencyMs); } }); } @@ -157,10 +159,11 @@ public CompletableFuture>> getAllSchem return schemaStorage.getAll(schemaId) .thenCompose(schemas -> convertToSchemaAndMetadata(schemaId, schemas)) .whenComplete((v, t) -> { + var latencyMs = this.clock.millis() - start; if (t != null) { - this.stats.recordGetFailed(schemaId); + this.stats.recordListFailed(schemaId, latencyMs); } else { - this.stats.recordGetLatency(schemaId, this.clock.millis() - start); + this.stats.recordListLatency(schemaId, latencyMs); } }); } @@ -228,10 +231,11 @@ public CompletableFuture putSchemaIfAbsent(String schemaId, Schem return CompletableFuture.completedFuture(Pair.of(info.toByteArray(), context)); }); }))).whenComplete((v, ex) -> { + var latencyMs = this.clock.millis() - start.getValue(); if (ex != null) { log.error("[{}] Put schema failed", schemaId, ex); if (start.getValue() != 0) { - this.stats.recordPutFailed(schemaId); + this.stats.recordPutFailed(schemaId, latencyMs); } promise.completeExceptionally(ex); } else { @@ -261,14 +265,15 @@ public CompletableFuture deleteSchema(String schemaId, String use return schemaStorage .put(schemaId, deletedEntry, new byte[]{}) .whenComplete((v, t) -> { + var latencyMs = this.clock.millis() - start; if (t != null) { log.error("[{}] User {} delete schema failed", schemaId, user); - this.stats.recordDelFailed(schemaId); + this.stats.recordDelFailed(schemaId, latencyMs); } else { if (log.isDebugEnabled()) { log.debug("[{}] User {} delete schema finished", schemaId, user); } - this.stats.recordDelLatency(schemaId, this.clock.millis() - start); + this.stats.recordDelLatency(schemaId, latencyMs); } }); } @@ -284,11 +289,12 @@ public CompletableFuture deleteSchemaStorage(String schemaId, boo return schemaStorage.delete(schemaId, forcefully) .whenComplete((v, t) -> { + var latencyMs = this.clock.millis() - start; if (t != null) { - this.stats.recordDelFailed(schemaId); + this.stats.recordDelFailed(schemaId, latencyMs); log.error("[{}] Delete schema storage failed", schemaId); } else { - this.stats.recordDelLatency(schemaId, this.clock.millis() - start); + this.stats.recordDelLatency(schemaId, latencyMs); if (log.isDebugEnabled()) { log.debug("[{}] Delete schema storage finished", schemaId); } @@ -393,7 +399,7 @@ public CompletableFuture checkConsumerCompatibility(String schemaId, Schem return checkCompatibilityWithAll(schemaId, schemaData, strategy); } } else { - return FutureUtil.failedFuture(new IncompatibleSchemaException("Topic does not have schema to check")); + return FutureUtil.failedFuture(new NotExistSchemaException("Topic does not have schema to check")); } }); } @@ -473,7 +479,7 @@ private CompletableFuture checkCompatibilityWithLatest(String schemaId, Sc } return result; } else { - return FutureUtils.exception(new IncompatibleSchemaException("Do not have existing schema.")); + return CompletableFuture.completedFuture(null); } }); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryStats.java index 32e9e36853026..b1a7dc2a54133 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryStats.java @@ -18,69 +18,111 @@ */ package org.apache.pulsar.broker.service.schema; -import io.prometheus.client.CollectorRegistry; +import com.google.common.annotations.VisibleForTesting; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.LongCounter; import io.prometheus.client.Counter; import io.prometheus.client.Summary; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.stats.MetricsUtil; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; +import org.apache.pulsar.opentelemetry.annotations.PulsarDeprecatedMetric; class SchemaRegistryStats implements AutoCloseable, Runnable { private static final String NAMESPACE = "namespace"; private static final double[] QUANTILES = {0.50, 0.75, 0.95, 0.99, 0.999, 0.9999, 1}; - private static final AtomicBoolean CLOSED = new AtomicBoolean(false); - private final Counter getOpsFailedCounter; - private final Counter putOpsFailedCounter; - private final Counter deleteOpsFailedCounter; + public static final AttributeKey REQUEST_TYPE_KEY = + AttributeKey.stringKey("pulsar.schema_registry.request"); + @VisibleForTesting + enum RequestType { + GET, + LIST, + PUT, + DELETE; - private final Counter compatibleCounter; - private final Counter incompatibleCounter; - - private final Summary deleteOpsLatency; - private final Summary getOpsLatency; - private final Summary putOpsLatency; + public final Attributes attributes = Attributes.of(REQUEST_TYPE_KEY, name().toLowerCase()); + } - private final Map namespaceAccess = new ConcurrentHashMap<>(); - private ScheduledFuture future; + public static final AttributeKey RESPONSE_TYPE_KEY = + AttributeKey.stringKey("pulsar.schema_registry.response"); + @VisibleForTesting + enum ResponseType { + SUCCESS, + FAILURE; - private static volatile SchemaRegistryStats instance; + public final Attributes attributes = Attributes.of(RESPONSE_TYPE_KEY, name().toLowerCase()); + } - static synchronized SchemaRegistryStats getInstance(ScheduledExecutorService scheduler) { - if (null == instance) { - instance = new SchemaRegistryStats(scheduler); - } + public static final AttributeKey COMPATIBILITY_CHECK_RESPONSE_KEY = + AttributeKey.stringKey("pulsar.schema_registry.compatibility_check.response"); + @VisibleForTesting + enum CompatibilityCheckResponse { + COMPATIBLE, + INCOMPATIBLE; - return instance; + public final Attributes attributes = Attributes.of(COMPATIBILITY_CHECK_RESPONSE_KEY, name().toLowerCase()); } - private SchemaRegistryStats(ScheduledExecutorService scheduler) { - this.deleteOpsFailedCounter = Counter.build("pulsar_schema_del_ops_failed_total", "-") - .labelNames(NAMESPACE).create().register(); - this.getOpsFailedCounter = Counter.build("pulsar_schema_get_ops_failed_total", "-") - .labelNames(NAMESPACE).create().register(); - this.putOpsFailedCounter = Counter.build("pulsar_schema_put_ops_failed_total", "-") - .labelNames(NAMESPACE).create().register(); + public static final String SCHEMA_REGISTRY_REQUEST_DURATION_METRIC_NAME = + "pulsar.broker.request.schema_registry.duration"; + private final DoubleHistogram latencyHistogram; - this.compatibleCounter = Counter.build("pulsar_schema_compatible_total", "-") - .labelNames(NAMESPACE).create().register(); - this.incompatibleCounter = Counter.build("pulsar_schema_incompatible_total", "-") - .labelNames(NAMESPACE).create().register(); + public static final String COMPATIBLE_COUNTER_METRIC_NAME = + "pulsar.broker.operation.schema_registry.compatibility_check.count"; + private final LongCounter schemaCompatibilityCounter; - this.deleteOpsLatency = this.buildSummary("pulsar_schema_del_ops_latency", "-"); - this.getOpsLatency = this.buildSummary("pulsar_schema_get_ops_latency", "-"); - this.putOpsLatency = this.buildSummary("pulsar_schema_put_ops_latency", "-"); + @PulsarDeprecatedMetric(newMetricName = SCHEMA_REGISTRY_REQUEST_DURATION_METRIC_NAME) + private static final Counter getOpsFailedCounter = + Counter.build("pulsar_schema_get_ops_failed_total", "-").labelNames(NAMESPACE).create().register(); + @PulsarDeprecatedMetric(newMetricName = SCHEMA_REGISTRY_REQUEST_DURATION_METRIC_NAME) + private static final Counter putOpsFailedCounter = + Counter.build("pulsar_schema_put_ops_failed_total", "-").labelNames(NAMESPACE).create().register(); + @PulsarDeprecatedMetric(newMetricName = SCHEMA_REGISTRY_REQUEST_DURATION_METRIC_NAME) + private static final Counter deleteOpsFailedCounter = + Counter.build("pulsar_schema_del_ops_failed_total", "-").labelNames(NAMESPACE).create().register(); - if (null != scheduler) { - this.future = scheduler.scheduleAtFixedRate(this, 1, 1, TimeUnit.MINUTES); - } + @PulsarDeprecatedMetric(newMetricName = COMPATIBLE_COUNTER_METRIC_NAME) + private static final Counter compatibleCounter = + Counter.build("pulsar_schema_compatible_total", "-").labelNames(NAMESPACE).create().register(); + @PulsarDeprecatedMetric(newMetricName = COMPATIBLE_COUNTER_METRIC_NAME) + private static final Counter incompatibleCounter = + Counter.build("pulsar_schema_incompatible_total", "-").labelNames(NAMESPACE).create().register(); + + @PulsarDeprecatedMetric(newMetricName = SCHEMA_REGISTRY_REQUEST_DURATION_METRIC_NAME) + private static final Summary deleteOpsLatency = buildSummary("pulsar_schema_del_ops_latency", "-"); + + @PulsarDeprecatedMetric(newMetricName = SCHEMA_REGISTRY_REQUEST_DURATION_METRIC_NAME) + private static final Summary getOpsLatency = buildSummary("pulsar_schema_get_ops_latency", "-"); + + @PulsarDeprecatedMetric(newMetricName = SCHEMA_REGISTRY_REQUEST_DURATION_METRIC_NAME) + private static final Summary putOpsLatency = buildSummary("pulsar_schema_put_ops_latency", "-"); + + private final Map namespaceAccess = new ConcurrentHashMap<>(); + private final ScheduledFuture future; + + public SchemaRegistryStats(PulsarService pulsarService) { + this.future = pulsarService.getExecutor().scheduleAtFixedRate(this, 1, 1, TimeUnit.MINUTES); + + var meter = pulsarService.getOpenTelemetry().getMeter(); + latencyHistogram = meter.histogramBuilder(SCHEMA_REGISTRY_REQUEST_DURATION_METRIC_NAME) + .setDescription("The duration of Schema Registry requests.") + .setUnit("s") + .build(); + schemaCompatibilityCounter = meter.counterBuilder(COMPATIBLE_COUNTER_METRIC_NAME) + .setDescription("The number of Schema Registry compatibility check operations performed by the broker.") + .setUnit("{operation}") + .build(); } - private Summary buildSummary(String name, String help) { + private static Summary buildSummary(String name, String help) { Summary.Builder builder = Summary.build(name, help).labelNames(NAMESPACE); for (double quantile : QUANTILES) { @@ -90,38 +132,77 @@ private Summary buildSummary(String name, String help) { return builder.create().register(); } - void recordDelFailed(String schemaId) { - this.deleteOpsFailedCounter.labels(getNamespace(schemaId)).inc(); + void recordDelFailed(String schemaId, long millis) { + deleteOpsFailedCounter.labels(getNamespace(schemaId)).inc(); + recordOperationLatency(schemaId, millis, RequestType.DELETE, ResponseType.FAILURE); + } + + void recordGetFailed(String schemaId, long millis) { + getOpsFailedCounter.labels(getNamespace(schemaId)).inc(); + recordOperationLatency(schemaId, millis, RequestType.GET, ResponseType.FAILURE); } - void recordGetFailed(String schemaId) { - this.getOpsFailedCounter.labels(getNamespace(schemaId)).inc(); + void recordListFailed(String schemaId, long millis) { + getOpsFailedCounter.labels(getNamespace(schemaId)).inc(); + recordOperationLatency(schemaId, millis, RequestType.LIST, ResponseType.FAILURE); } - void recordPutFailed(String schemaId) { - this.putOpsFailedCounter.labels(getNamespace(schemaId)).inc(); + void recordPutFailed(String schemaId, long millis) { + putOpsFailedCounter.labels(getNamespace(schemaId)).inc(); + recordOperationLatency(schemaId, millis, RequestType.PUT, ResponseType.FAILURE); } void recordDelLatency(String schemaId, long millis) { - this.deleteOpsLatency.labels(getNamespace(schemaId)).observe(millis); + deleteOpsLatency.labels(getNamespace(schemaId)).observe(millis); + recordOperationLatency(schemaId, millis, RequestType.DELETE, ResponseType.SUCCESS); } void recordGetLatency(String schemaId, long millis) { - this.getOpsLatency.labels(getNamespace(schemaId)).observe(millis); + getOpsLatency.labels(getNamespace(schemaId)).observe(millis); + recordOperationLatency(schemaId, millis, RequestType.GET, ResponseType.SUCCESS); + } + + void recordListLatency(String schemaId, long millis) { + getOpsLatency.labels(getNamespace(schemaId)).observe(millis); + recordOperationLatency(schemaId, millis, RequestType.LIST, ResponseType.SUCCESS); } void recordPutLatency(String schemaId, long millis) { - this.putOpsLatency.labels(getNamespace(schemaId)).observe(millis); + putOpsLatency.labels(getNamespace(schemaId)).observe(millis); + recordOperationLatency(schemaId, millis, RequestType.PUT, ResponseType.SUCCESS); + } + + private void recordOperationLatency(String schemaId, long millis, + RequestType requestType, ResponseType responseType) { + var duration = MetricsUtil.convertToSeconds(millis, TimeUnit.MILLISECONDS); + var namespace = getNamespace(schemaId); + var attributes = Attributes.builder() + .put(OpenTelemetryAttributes.PULSAR_NAMESPACE, namespace) + .putAll(requestType.attributes) + .putAll(responseType.attributes) + .build(); + latencyHistogram.record(duration, attributes); } void recordSchemaIncompatible(String schemaId) { - this.incompatibleCounter.labels(getNamespace(schemaId)).inc(); + var namespace = getNamespace(schemaId); + incompatibleCounter.labels(namespace).inc(); + recordSchemaCompabilityResult(namespace, CompatibilityCheckResponse.INCOMPATIBLE); } void recordSchemaCompatible(String schemaId) { - this.compatibleCounter.labels(getNamespace(schemaId)).inc(); + var namespace = getNamespace(schemaId); + compatibleCounter.labels(namespace).inc(); + recordSchemaCompabilityResult(namespace, CompatibilityCheckResponse.COMPATIBLE); } + private void recordSchemaCompabilityResult(String namespace, CompatibilityCheckResponse result) { + var attributes = Attributes.builder() + .put(OpenTelemetryAttributes.PULSAR_NAMESPACE, namespace) + .putAll(result.attributes) + .build(); + schemaCompatibilityCounter.add(1, attributes); + } private String getNamespace(String schemaId) { String namespace; @@ -148,20 +229,9 @@ private void removeChild(String namespace) { } @Override - public void close() throws Exception { - if (CLOSED.compareAndSet(false, true)) { - CollectorRegistry.defaultRegistry.unregister(this.deleteOpsFailedCounter); - CollectorRegistry.defaultRegistry.unregister(this.getOpsFailedCounter); - CollectorRegistry.defaultRegistry.unregister(this.putOpsFailedCounter); - CollectorRegistry.defaultRegistry.unregister(this.compatibleCounter); - CollectorRegistry.defaultRegistry.unregister(this.incompatibleCounter); - CollectorRegistry.defaultRegistry.unregister(this.deleteOpsLatency); - CollectorRegistry.defaultRegistry.unregister(this.getOpsLatency); - CollectorRegistry.defaultRegistry.unregister(this.putOpsLatency); - if (null != this.future) { - this.future.cancel(false); - } - } + public synchronized void close() throws Exception { + namespaceAccess.keySet().forEach(this::removeChild); + future.cancel(false); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/exceptions/IncompatibleSchemaException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/exceptions/IncompatibleSchemaException.java index c1a2d9fd703fd..bbe2f4111d759 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/exceptions/IncompatibleSchemaException.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/exceptions/IncompatibleSchemaException.java @@ -33,6 +33,10 @@ public IncompatibleSchemaException(String message) { super(message); } + public IncompatibleSchemaException(String message, Throwable e) { + super(message, e); + } + public IncompatibleSchemaException(Throwable e) { super(e); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/exceptions/NotExistSchemaException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/exceptions/NotExistSchemaException.java new file mode 100644 index 0000000000000..2fe0a09237545 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/exceptions/NotExistSchemaException.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service.schema.exceptions; + +/** + * Exception is thrown when an schema not exist. + */ +public class NotExistSchemaException extends SchemaException { + + private static final long serialVersionUID = -8342983749283749283L; + + public NotExistSchemaException() { + super("The schema does not exist"); + } + + public NotExistSchemaException(String message) { + super(message); + } + + public NotExistSchemaException(String message, Throwable e) { + super(message, e); + } + + public NotExistSchemaException(Throwable e) { + super(e); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/AllocatorStatsGenerator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/AllocatorStatsGenerator.java index 677b04d8a74ca..d20aef90adc33 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/AllocatorStatsGenerator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/AllocatorStatsGenerator.java @@ -54,6 +54,8 @@ public static AllocatorStats generate(String allocatorName) { stats.numDirectArenas = allocator.metric().numDirectArenas(); stats.numHeapArenas = allocator.metric().numHeapArenas(); stats.numThreadLocalCaches = allocator.metric().numThreadLocalCaches(); + stats.usedHeapMemory = allocator.metric().usedHeapMemory(); + stats.usedDirectMemory = allocator.metric().usedDirectMemory(); stats.normalCacheSize = allocator.metric().normalCacheSize(); stats.smallCacheSize = allocator.metric().smallCacheSize(); return stats; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java index 909133338719f..1855e1798b465 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java @@ -18,37 +18,81 @@ */ package org.apache.pulsar.broker.stats; +import io.opentelemetry.api.metrics.ObservableLongCounter; +import io.prometheus.client.Counter; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.common.stats.Metrics; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.ConnectionCreateStatus; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.ConnectionStatus; /** */ -public class BrokerOperabilityMetrics { +public class BrokerOperabilityMetrics implements AutoCloseable { + private static final Counter TOPIC_LOAD_FAILED = Counter.build("topic_load_failed", "-").register(); private final List metricsList; private final String localCluster; private final DimensionStats topicLoadStats; private final String brokerName; private final LongAdder connectionTotalCreatedCount; - private final LongAdder connectionCreateSuccessCount; - private final LongAdder connectionCreateFailCount; private final LongAdder connectionTotalClosedCount; private final LongAdder connectionActive; + private volatile int healthCheckStatus; // 1=success, 0=failure, -1=unknown + + private final LongAdder connectionCreateSuccessCount; + private final LongAdder connectionCreateFailCount; + + public static final String CONNECTION_COUNTER_METRIC_NAME = "pulsar.broker.connection.count"; + private final ObservableLongCounter connectionCounter; - public BrokerOperabilityMetrics(String localCluster, String brokerName) { + public static final String CONNECTION_CREATE_COUNTER_METRIC_NAME = + "pulsar.broker.connection.create.operation.count"; + private final ObservableLongCounter connectionCreateCounter; + + public BrokerOperabilityMetrics(PulsarService pulsar) { this.metricsList = new ArrayList<>(); - this.localCluster = localCluster; - this.topicLoadStats = new DimensionStats("topic_load_times", 60); - this.brokerName = brokerName; + this.localCluster = pulsar.getConfiguration().getClusterName(); + this.topicLoadStats = new DimensionStats("pulsar_topic_load_times", 60); + this.brokerName = pulsar.getAdvertisedAddress(); this.connectionTotalCreatedCount = new LongAdder(); - this.connectionCreateSuccessCount = new LongAdder(); - this.connectionCreateFailCount = new LongAdder(); this.connectionTotalClosedCount = new LongAdder(); this.connectionActive = new LongAdder(); + this.healthCheckStatus = -1; + this.connectionCreateSuccessCount = new LongAdder(); + this.connectionCreateFailCount = new LongAdder(); + + connectionCounter = pulsar.getOpenTelemetry().getMeter() + .counterBuilder(CONNECTION_COUNTER_METRIC_NAME) + .setDescription("The number of connections.") + .setUnit("{connection}") + .buildWithCallback(measurement -> { + var closedConnections = connectionTotalClosedCount.sum(); + var openedConnections = connectionTotalCreatedCount.sum(); + var activeConnections = openedConnections - closedConnections; + measurement.record(activeConnections, ConnectionStatus.ACTIVE.attributes); + measurement.record(openedConnections, ConnectionStatus.OPEN.attributes); + measurement.record(closedConnections, ConnectionStatus.CLOSE.attributes); + }); + + connectionCreateCounter = pulsar.getOpenTelemetry().getMeter() + .counterBuilder(CONNECTION_CREATE_COUNTER_METRIC_NAME) + .setDescription("The number of connection create operations.") + .setUnit("{operation}") + .buildWithCallback(measurement -> { + measurement.record(connectionCreateSuccessCount.sum(), ConnectionCreateStatus.SUCCESS.attributes); + measurement.record(connectionCreateFailCount.sum(), ConnectionCreateStatus.FAILURE.attributes); + }); + } + + @Override + public void close() throws Exception { + connectionCounter.close(); + connectionCreateCounter.close(); } public List getMetrics() { @@ -57,8 +101,10 @@ public List getMetrics() { } private void generate() { + reset(); metricsList.add(getTopicLoadMetrics()); metricsList.add(getConnectionMetrics()); + metricsList.add(getHealthMetrics()); } public Metrics generateConnectionMetrics() { @@ -75,6 +121,12 @@ Metrics getConnectionMetrics() { return rMetrics; } + Metrics getHealthMetrics() { + Metrics rMetrics = Metrics.create(getDimensionMap("broker_health")); + rMetrics.put("brk_health", healthCheckStatus); + return rMetrics; + } + Map getDimensionMap(String metricsName) { Map dimensionMap = new HashMap<>(); dimensionMap.put("broker", brokerName); @@ -84,7 +136,9 @@ Map getDimensionMap(String metricsName) { } Metrics getTopicLoadMetrics() { - return getDimensionMetrics("topic_load_times", "topic_load", topicLoadStats); + Metrics metrics = getDimensionMetrics("pulsar_topic_load_times", "topic_load", topicLoadStats); + metrics.put("brk_topic_load_failed_count", TOPIC_LOAD_FAILED.get()); + return metrics; } Metrics getDimensionMetrics(String metricsName, String dimensionName, DimensionStats stats) { @@ -112,6 +166,10 @@ public void recordTopicLoadTimeValue(long topicLoadLatencyMs) { topicLoadStats.recordDimensionTimeValue(topicLoadLatencyMs, TimeUnit.MILLISECONDS); } + public void recordTopicLoadFailed() { + this.TOPIC_LOAD_FAILED.inc(); + } + public void recordConnectionCreate() { this.connectionTotalCreatedCount.increment(); this.connectionActive.increment(); @@ -129,4 +187,12 @@ public void recordConnectionCreateSuccess() { public void recordConnectionCreateFail() { this.connectionCreateFailCount.increment(); } + + public void recordHealthCheckStatusSuccess() { + this.healthCheckStatus = 1; + } + + public void recordHealthCheckStatusFail() { + this.healthCheckStatus = 0; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerStats.java index 84d5432fb9e19..04926b6cf1c72 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerStats.java @@ -20,8 +20,8 @@ public class BrokerStats extends NamespaceStats { - public int bundleCount; - public int topics; + public long bundleCount; + public long topics; public BrokerStats(int ratePeriodInSeconds) { super(ratePeriodInSeconds); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/ClusterReplicationMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/ClusterReplicationMetrics.java index 6b274b26b57fb..828cb48be429d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/ClusterReplicationMetrics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/ClusterReplicationMetrics.java @@ -20,23 +20,22 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import org.apache.pulsar.common.stats.Metrics; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; /** */ public class ClusterReplicationMetrics { private final List metricsList; private final String localCluster; - private final ConcurrentOpenHashMap metricsMap; + private final Map metricsMap = new ConcurrentHashMap<>(); public static final String SEPARATOR = "_"; public final boolean metricsEnabled; public ClusterReplicationMetrics(String localCluster, boolean metricsEnabled) { metricsList = new ArrayList<>(); this.localCluster = localCluster; - metricsMap = ConcurrentOpenHashMap.newBuilder() - .build(); this.metricsEnabled = metricsEnabled; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/DimensionStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/DimensionStats.java index 604265b554050..1b6f981ca4e21 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/DimensionStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/DimensionStats.java @@ -64,7 +64,7 @@ public DimensionStats(String name, long updateDurationInSec) { defaultRegistry.register(summary); } catch (IllegalArgumentException ie) { // it only happens in test-cases when try to register summary multiple times in registry - log.warn("{} is already registred {}", name, ie.getMessage()); + log.warn("{} is already registered {}", name, ie.getMessage()); } } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/NamespaceStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/NamespaceStats.java index e531139d4212b..afff2ec15eb90 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/NamespaceStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/NamespaceStats.java @@ -37,7 +37,7 @@ public class NamespaceStats { public int consumerCount; public int producerCount; public int replicatorCount; - public int subsCount; + public long subsCount; public static final String BRK_ADD_ENTRY_LATENCY_PREFIX = "brk_AddEntryLatencyBuckets"; public long[] addLatencyBucket = new long[ENTRY_LATENCY_BUCKETS_USEC.length + 1]; public static final String[] ADD_LATENCY_BUCKET_KEYS = new String[ENTRY_LATENCY_BUCKETS_USEC.length + 1]; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/OpenTelemetryConsumerStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/OpenTelemetryConsumerStats.java new file mode 100644 index 0000000000000..09b487a8fa2c3 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/OpenTelemetryConsumerStats.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats; + +import io.opentelemetry.api.metrics.BatchCallback; +import io.opentelemetry.api.metrics.ObservableLongMeasurement; +import java.util.Collection; +import java.util.Optional; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.service.Consumer; +import org.apache.pulsar.broker.service.Subscription; +import org.apache.pulsar.broker.service.Topic; + +public class OpenTelemetryConsumerStats implements AutoCloseable { + + // Replaces pulsar_consumer_msg_rate_out + public static final String MESSAGE_OUT_COUNTER = "pulsar.broker.consumer.message.outgoing.count"; + private final ObservableLongMeasurement messageOutCounter; + + // Replaces pulsar_consumer_msg_throughput_out + public static final String BYTES_OUT_COUNTER = "pulsar.broker.consumer.message.outgoing.size"; + private final ObservableLongMeasurement bytesOutCounter; + + // Replaces pulsar_consumer_msg_ack_rate + public static final String MESSAGE_ACK_COUNTER = "pulsar.broker.consumer.message.ack.count"; + private final ObservableLongMeasurement messageAckCounter; + + // Replaces pulsar_consumer_msg_rate_redeliver + public static final String MESSAGE_REDELIVER_COUNTER = "pulsar.broker.consumer.message.redeliver.count"; + private final ObservableLongMeasurement messageRedeliverCounter; + + // Replaces pulsar_consumer_unacked_messages + public static final String MESSAGE_UNACKNOWLEDGED_COUNTER = "pulsar.broker.consumer.message.unack.count"; + private final ObservableLongMeasurement messageUnacknowledgedCounter; + + public static final String CONSUMER_BLOCKED_COUNTER = "pulsar.broker.consumer.blocked"; + private final ObservableLongMeasurement consumerBlockedCounter; + + // Replaces pulsar_consumer_available_permits + public static final String MESSAGE_PERMITS_COUNTER = "pulsar.broker.consumer.permit.count"; + private final ObservableLongMeasurement messagePermitsCounter; + + private final BatchCallback batchCallback; + + public OpenTelemetryConsumerStats(PulsarService pulsar) { + var meter = pulsar.getOpenTelemetry().getMeter(); + + messageOutCounter = meter + .counterBuilder(MESSAGE_OUT_COUNTER) + .setUnit("{message}") + .setDescription("The total number of messages dispatched to this consumer.") + .buildObserver(); + + bytesOutCounter = meter + .counterBuilder(BYTES_OUT_COUNTER) + .setUnit("By") + .setDescription("The total number of messages bytes dispatched to this consumer.") + .buildObserver(); + + messageAckCounter = meter + .counterBuilder(MESSAGE_ACK_COUNTER) + .setUnit("{ack}") + .setDescription("The total number of message acknowledgments received from this consumer.") + .buildObserver(); + + messageRedeliverCounter = meter + .counterBuilder(MESSAGE_REDELIVER_COUNTER) + .setUnit("{message}") + .setDescription("The total number of messages that have been redelivered to this consumer.") + .buildObserver(); + + messageUnacknowledgedCounter = meter + .upDownCounterBuilder(MESSAGE_UNACKNOWLEDGED_COUNTER) + .setUnit("{message}") + .setDescription("The total number of messages unacknowledged by this consumer.") + .buildObserver(); + + consumerBlockedCounter = meter + .upDownCounterBuilder(CONSUMER_BLOCKED_COUNTER) + .setUnit("1") + .setDescription("Indicates whether the consumer is currently blocked due to unacknowledged messages.") + .buildObserver(); + + messagePermitsCounter = meter + .upDownCounterBuilder(MESSAGE_PERMITS_COUNTER) + .setUnit("{permit}") + .setDescription("The number of permits currently available for this consumer.") + .buildObserver(); + + batchCallback = meter.batchCallback(() -> pulsar.getBrokerService() + .getTopics() + .values() + .stream() + .map(topicFuture -> topicFuture.getNow(Optional.empty())) + .filter(Optional::isPresent) + .map(Optional::get) + .map(Topic::getSubscriptions) + .flatMap(s -> s.values().stream()) + .map(Subscription::getConsumers) + .flatMap(Collection::stream) + .forEach(this::recordMetricsForConsumer), + messageOutCounter, + bytesOutCounter, + messageAckCounter, + messageRedeliverCounter, + messageUnacknowledgedCounter, + consumerBlockedCounter, + messagePermitsCounter); + } + + @Override + public void close() { + batchCallback.close(); + } + + private void recordMetricsForConsumer(Consumer consumer) { + var attributes = consumer.getOpenTelemetryAttributes(); + messageOutCounter.record(consumer.getMsgOutCounter(), attributes); + bytesOutCounter.record(consumer.getBytesOutCounter(), attributes); + messageAckCounter.record(consumer.getMessageAckCounter(), attributes); + messageRedeliverCounter.record(consumer.getMessageRedeliverCounter(), attributes); + messageUnacknowledgedCounter.record(consumer.getUnackedMessages(), attributes); + consumerBlockedCounter.record(consumer.isBlocked() ? 1 : 0, attributes); + messagePermitsCounter.record(consumer.getAvailablePermits(), attributes); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/OpenTelemetryProducerStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/OpenTelemetryProducerStats.java new file mode 100644 index 0000000000000..9c09804554c31 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/OpenTelemetryProducerStats.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats; + +import io.opentelemetry.api.metrics.BatchCallback; +import io.opentelemetry.api.metrics.ObservableLongMeasurement; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.service.Producer; +import org.apache.pulsar.common.policies.data.stats.NonPersistentPublisherStatsImpl; + +public class OpenTelemetryProducerStats implements AutoCloseable { + + // Replaces pulsar_producer_msg_rate_in + public static final String MESSAGE_IN_COUNTER = "pulsar.broker.producer.message.incoming.count"; + private final ObservableLongMeasurement messageInCounter; + + // Replaces pulsar_producer_msg_throughput_in + public static final String BYTES_IN_COUNTER = "pulsar.broker.producer.message.incoming.size"; + private final ObservableLongMeasurement bytesInCounter; + + public static final String MESSAGE_DROP_COUNTER = "pulsar.broker.producer.message.drop.count"; + private final ObservableLongMeasurement messageDropCounter; + + private final BatchCallback batchCallback; + + public OpenTelemetryProducerStats(PulsarService pulsar) { + var meter = pulsar.getOpenTelemetry().getMeter(); + + messageInCounter = meter + .counterBuilder(MESSAGE_IN_COUNTER) + .setUnit("{message}") + .setDescription("The total number of messages received from this producer.") + .buildObserver(); + + bytesInCounter = meter + .counterBuilder(BYTES_IN_COUNTER) + .setUnit("By") + .setDescription("The total number of messages bytes received from this producer.") + .buildObserver(); + + messageDropCounter = meter + .counterBuilder(MESSAGE_DROP_COUNTER) + .setUnit("{message}") + .setDescription("The total number of messages dropped from this producer.") + .buildObserver(); + + batchCallback = meter.batchCallback(() -> pulsar.getBrokerService() + .getTopics() + .values() + .stream() + .filter(future -> future.isDone() && !future.isCompletedExceptionally()) + .map(CompletableFuture::join) + .filter(Optional::isPresent) + .flatMap(topic -> topic.get().getProducers().values().stream()) + .forEach(this::recordMetricsForProducer), + messageInCounter, + bytesInCounter, + messageDropCounter); + } + + @Override + public void close() { + batchCallback.close(); + } + + private void recordMetricsForProducer(Producer producer) { + var attributes = producer.getOpenTelemetryAttributes(); + var stats = producer.getStats(); + + messageInCounter.record(stats.getMsgInCounter(), attributes); + bytesInCounter.record(stats.getBytesInCounter(), attributes); + + if (stats instanceof NonPersistentPublisherStatsImpl nonPersistentStats) { + messageDropCounter.record(nonPersistentStats.getMsgDropCount(), attributes); + } + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/OpenTelemetryReplicatedSubscriptionStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/OpenTelemetryReplicatedSubscriptionStats.java new file mode 100644 index 0000000000000..55982eba24312 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/OpenTelemetryReplicatedSubscriptionStats.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.LongCounter; +import java.util.concurrent.TimeUnit; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.common.stats.MetricsUtil; + +public class OpenTelemetryReplicatedSubscriptionStats { + + public static final AttributeKey SNAPSHOT_OPERATION_RESULT = + AttributeKey.stringKey("pulsar.replication.subscription.snapshot.operation.result"); + public enum SnapshotOperationResult { + SUCCESS, + TIMEOUT; + private final Attributes attributes = Attributes.of(SNAPSHOT_OPERATION_RESULT, name().toLowerCase()); + } + + public static final String SNAPSHOT_OPERATION_COUNT_METRIC_NAME = + "pulsar.broker.replication.subscription.snapshot.operation.count"; + private final LongCounter snapshotOperationCounter; + + public static final String SNAPSHOT_DURATION_METRIC_NAME = + "pulsar.broker.replication.subscription.snapshot.operation.duration"; + private final DoubleHistogram snapshotDuration; + + public OpenTelemetryReplicatedSubscriptionStats(PulsarService pulsar) { + var meter = pulsar.getOpenTelemetry().getMeter(); + snapshotOperationCounter = meter.counterBuilder(SNAPSHOT_OPERATION_COUNT_METRIC_NAME) + .setDescription("The number of snapshot operations attempted") + .setUnit("{operation}") + .build(); + snapshotDuration = meter.histogramBuilder(SNAPSHOT_DURATION_METRIC_NAME) + .setDescription("Time taken to complete a consistent snapshot operation across clusters") + .setUnit("s") + .build(); + } + + public void recordSnapshotStarted() { + snapshotOperationCounter.add(1); + } + + public void recordSnapshotTimedOut(long durationMs) { + snapshotDuration.record(MetricsUtil.convertToSeconds(durationMs, TimeUnit.MILLISECONDS), + SnapshotOperationResult.TIMEOUT.attributes); + } + + public void recordSnapshotCompleted(long durationMs) { + snapshotDuration.record(MetricsUtil.convertToSeconds(durationMs, TimeUnit.MILLISECONDS), + SnapshotOperationResult.SUCCESS.attributes); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/OpenTelemetryReplicatorStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/OpenTelemetryReplicatorStats.java new file mode 100644 index 0000000000000..04bc805a64bbf --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/OpenTelemetryReplicatorStats.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats; + +import io.opentelemetry.api.metrics.BatchCallback; +import io.opentelemetry.api.metrics.ObservableDoubleMeasurement; +import io.opentelemetry.api.metrics.ObservableLongMeasurement; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.service.AbstractReplicator; +import org.apache.pulsar.broker.service.nonpersistent.NonPersistentReplicator; +import org.apache.pulsar.broker.service.persistent.PersistentReplicator; +import org.apache.pulsar.common.stats.MetricsUtil; + +public class OpenTelemetryReplicatorStats implements AutoCloseable { + + // Replaces pulsar_replication_rate_in + public static final String MESSAGE_IN_COUNTER = "pulsar.broker.replication.message.incoming.count"; + private final ObservableLongMeasurement messageInCounter; + + // Replaces pulsar_replication_rate_out + public static final String MESSAGE_OUT_COUNTER = "pulsar.broker.replication.message.outgoing.count"; + private final ObservableLongMeasurement messageOutCounter; + + // Replaces pulsar_replication_throughput_in + public static final String BYTES_IN_COUNTER = "pulsar.broker.replication.message.incoming.size"; + private final ObservableLongMeasurement bytesInCounter; + + // Replaces pulsar_replication_throughput_out + public static final String BYTES_OUT_COUNTER = "pulsar.broker.replication.message.outgoing.size"; + private final ObservableLongMeasurement bytesOutCounter; + + // Replaces pulsar_replication_backlog + public static final String BACKLOG_COUNTER = "pulsar.broker.replication.message.backlog.count"; + private final ObservableLongMeasurement backlogCounter; + + // Replaces pulsar_replication_delay_in_seconds + public static final String DELAY_GAUGE = "pulsar.broker.replication.message.backlog.age"; + private final ObservableDoubleMeasurement delayGauge; + + // Replaces pulsar_replication_rate_expired + public static final String EXPIRED_COUNTER = "pulsar.broker.replication.message.expired.count"; + private final ObservableLongMeasurement expiredCounter; + + public static final String DROPPED_COUNTER = "pulsar.broker.replication.message.dropped.count"; + private final ObservableLongMeasurement droppedCounter; + + private final BatchCallback batchCallback; + + public OpenTelemetryReplicatorStats(PulsarService pulsar) { + var meter = pulsar.getOpenTelemetry().getMeter(); + + messageInCounter = meter + .upDownCounterBuilder(MESSAGE_IN_COUNTER) + .setUnit("{message}") + .setDescription( + "The total number of messages received from the remote cluster through this replicator.") + .buildObserver(); + + messageOutCounter = meter + .upDownCounterBuilder(MESSAGE_OUT_COUNTER) + .setUnit("{message}") + .setDescription("The total number of messages sent to the remote cluster through this replicator.") + .buildObserver(); + + bytesInCounter = meter + .upDownCounterBuilder(BYTES_IN_COUNTER) + .setUnit("{By}") + .setDescription( + "The total number of messages bytes received from the remote cluster through this replicator.") + .buildObserver(); + + bytesOutCounter = meter + .upDownCounterBuilder(BYTES_OUT_COUNTER) + .setUnit("{By}") + .setDescription( + "The total number of messages bytes sent to the remote cluster through this replicator.") + .buildObserver(); + + backlogCounter = meter + .upDownCounterBuilder(BACKLOG_COUNTER) + .setUnit("{message}") + .setDescription("The total number of messages in the backlog for this replicator.") + .buildObserver(); + + delayGauge = meter + .gaugeBuilder(DELAY_GAUGE) + .setUnit("s") + .setDescription("The age of the oldest message in the replicator backlog.") + .buildObserver(); + + expiredCounter = meter + .upDownCounterBuilder(EXPIRED_COUNTER) + .setUnit("{message}") + .setDescription("The total number of messages that expired for this replicator.") + .buildObserver(); + + droppedCounter = meter + .upDownCounterBuilder(DROPPED_COUNTER) + .setUnit("{message}") + .setDescription("The total number of messages dropped by this replicator.") + .buildObserver(); + + batchCallback = meter.batchCallback(() -> pulsar.getBrokerService() + .getTopics() + .values() + .stream() + .filter(topicFuture -> topicFuture.isDone() && !topicFuture.isCompletedExceptionally()) + .map(CompletableFuture::join) + .filter(Optional::isPresent) + .map(Optional::get) + .flatMap(topic -> topic.getReplicators().values().stream()) + .map(AbstractReplicator.class::cast) + .forEach(this::recordMetricsForReplicator), + messageInCounter, + messageOutCounter, + bytesInCounter, + bytesOutCounter, + backlogCounter, + delayGauge, + expiredCounter, + droppedCounter); + } + + @Override + public void close() { + batchCallback.close(); + } + + private void recordMetricsForReplicator(AbstractReplicator replicator) { + var attributes = replicator.getAttributes(); + var stats = replicator.getStats(); + + messageInCounter.record(stats.getMsgInCount(), attributes); + messageOutCounter.record(stats.getMsgOutCount(), attributes); + bytesInCounter.record(stats.getBytesInCount(), attributes); + bytesOutCounter.record(stats.getBytesOutCount(), attributes); + var delaySeconds = MetricsUtil.convertToSeconds(replicator.getReplicationDelayMs(), TimeUnit.MILLISECONDS); + delayGauge.record(delaySeconds, attributes); + + if (replicator instanceof PersistentReplicator persistentReplicator) { + expiredCounter.record(persistentReplicator.getMessageExpiredCount(), attributes); + backlogCounter.record(persistentReplicator.getNumberOfEntriesInBacklog(), attributes); + } else if (replicator instanceof NonPersistentReplicator nonPersistentReplicator) { + droppedCounter.record(nonPersistentReplicator.getStats().getMsgDropCount(), attributes); + } + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/OpenTelemetryTopicStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/OpenTelemetryTopicStats.java new file mode 100644 index 0000000000000..0274cb7a7d4a6 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/OpenTelemetryTopicStats.java @@ -0,0 +1,487 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats; + +import io.opentelemetry.api.metrics.BatchCallback; +import io.opentelemetry.api.metrics.ObservableDoubleMeasurement; +import io.opentelemetry.api.metrics.ObservableLongMeasurement; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.service.AbstractTopic; +import org.apache.pulsar.broker.service.Dispatcher; +import org.apache.pulsar.broker.service.Subscription; +import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.common.policies.data.BacklogQuota; +import org.apache.pulsar.common.stats.MetricsUtil; +import org.apache.pulsar.compaction.CompactedTopicContext; +import org.apache.pulsar.compaction.Compactor; + +public class OpenTelemetryTopicStats implements AutoCloseable { + + // Replaces pulsar_subscriptions_count + public static final String SUBSCRIPTION_COUNTER = "pulsar.broker.topic.subscription.count"; + private final ObservableLongMeasurement subscriptionCounter; + + // Replaces pulsar_producers_count + public static final String PRODUCER_COUNTER = "pulsar.broker.topic.producer.count"; + private final ObservableLongMeasurement producerCounter; + + // Replaces pulsar_consumers_count + public static final String CONSUMER_COUNTER = "pulsar.broker.topic.consumer.count"; + private final ObservableLongMeasurement consumerCounter; + + // Replaces ['pulsar_rate_in', 'pulsar_in_messages_total'] + public static final String MESSAGE_IN_COUNTER = "pulsar.broker.topic.message.incoming.count"; + private final ObservableLongMeasurement messageInCounter; + + // Replaces ['pulsar_rate_out', 'pulsar_out_messages_total'] + public static final String MESSAGE_OUT_COUNTER = "pulsar.broker.topic.message.outgoing.count"; + private final ObservableLongMeasurement messageOutCounter; + + // Replaces ['pulsar_throughput_in', 'pulsar_in_bytes_total'] + public static final String BYTES_IN_COUNTER = "pulsar.broker.topic.message.incoming.size"; + private final ObservableLongMeasurement bytesInCounter; + + // Replaces ['pulsar_throughput_out', 'pulsar_out_bytes_total'] + public static final String BYTES_OUT_COUNTER = "pulsar.broker.topic.message.outgoing.size"; + private final ObservableLongMeasurement bytesOutCounter; + + // Replaces pulsar_publish_rate_limit_times + public static final String PUBLISH_RATE_LIMIT_HIT_COUNTER = "pulsar.broker.topic.publish.rate.limit.count"; + private final ObservableLongMeasurement publishRateLimitHitCounter; + + // Omitted: pulsar_consumer_msg_ack_rate + + // Replaces pulsar_storage_size + public static final String STORAGE_COUNTER = "pulsar.broker.topic.storage.size"; + private final ObservableLongMeasurement storageCounter; + + // Replaces pulsar_storage_logical_size + public static final String STORAGE_LOGICAL_COUNTER = "pulsar.broker.topic.storage.logical.size"; + private final ObservableLongMeasurement storageLogicalCounter; + + // Replaces pulsar_storage_backlog_size + public static final String STORAGE_BACKLOG_COUNTER = "pulsar.broker.topic.storage.backlog.size"; + private final ObservableLongMeasurement storageBacklogCounter; + + // Replaces pulsar_storage_offloaded_size + public static final String STORAGE_OFFLOADED_COUNTER = "pulsar.broker.topic.storage.offloaded.size"; + private final ObservableLongMeasurement storageOffloadedCounter; + + // Replaces pulsar_storage_backlog_quota_limit + public static final String BACKLOG_QUOTA_LIMIT_SIZE = "pulsar.broker.topic.storage.backlog.quota.limit.size"; + private final ObservableLongMeasurement backlogQuotaLimitSize; + + // Replaces pulsar_storage_backlog_quota_limit_time + public static final String BACKLOG_QUOTA_LIMIT_TIME = "pulsar.broker.topic.storage.backlog.quota.limit.time"; + private final ObservableLongMeasurement backlogQuotaLimitTime; + + // Replaces pulsar_storage_backlog_quota_exceeded_evictions_total + public static final String BACKLOG_EVICTION_COUNTER = "pulsar.broker.topic.storage.backlog.quota.eviction.count"; + private final ObservableLongMeasurement backlogEvictionCounter; + + // Replaces pulsar_storage_backlog_age_seconds + public static final String BACKLOG_QUOTA_AGE = "pulsar.broker.topic.storage.backlog.age"; + private final ObservableLongMeasurement backlogQuotaAge; + + // Replaces pulsar_storage_write_rate + public static final String STORAGE_OUT_COUNTER = "pulsar.broker.topic.storage.entry.outgoing.count"; + private final ObservableLongMeasurement storageOutCounter; + + // Replaces pulsar_storage_read_rate + public static final String STORAGE_IN_COUNTER = "pulsar.broker.topic.storage.entry.incoming.count"; + private final ObservableLongMeasurement storageInCounter; + + // Omitted: pulsar_storage_write_latency_le_* + + // Omitted: pulsar_entry_size_le_* + + // Replaces pulsar_compaction_removed_event_count + public static final String COMPACTION_REMOVED_COUNTER = "pulsar.broker.topic.compaction.removed.message.count"; + private final ObservableLongMeasurement compactionRemovedCounter; + + // Replaces ['pulsar_compaction_succeed_count', 'pulsar_compaction_failed_count'] + public static final String COMPACTION_OPERATION_COUNTER = "pulsar.broker.topic.compaction.operation.count"; + private final ObservableLongMeasurement compactionOperationCounter; + + // Replaces pulsar_compaction_duration_time_in_mills + public static final String COMPACTION_DURATION_SECONDS = "pulsar.broker.topic.compaction.duration"; + private final ObservableDoubleMeasurement compactionDurationSeconds; + + // Replaces pulsar_compaction_read_throughput + public static final String COMPACTION_BYTES_IN_COUNTER = "pulsar.broker.topic.compaction.incoming.size"; + private final ObservableLongMeasurement compactionBytesInCounter; + + // Replaces pulsar_compaction_write_throughput + public static final String COMPACTION_BYTES_OUT_COUNTER = "pulsar.broker.topic.compaction.outgoing.size"; + private final ObservableLongMeasurement compactionBytesOutCounter; + + // Omitted: pulsar_compaction_latency_le_* + + // Replaces pulsar_compaction_compacted_entries_count + public static final String COMPACTION_ENTRIES_COUNTER = "pulsar.broker.topic.compaction.compacted.entry.count"; + private final ObservableLongMeasurement compactionEntriesCounter; + + // Replaces pulsar_compaction_compacted_entries_size + public static final String COMPACTION_BYTES_COUNTER = "pulsar.broker.topic.compaction.compacted.entry.size"; + private final ObservableLongMeasurement compactionBytesCounter; + + // Replaces ['pulsar_txn_tb_active_total', 'pulsar_txn_tb_aborted_total', 'pulsar_txn_tb_committed_total'] + public static final String TRANSACTION_COUNTER = "pulsar.broker.topic.transaction.count"; + private final ObservableLongMeasurement transactionCounter; + + // Replaces ['pulsar_txn_tb_client_abort_failed_total', 'pulsar_txn_tb_client_commit_failed_total', + // 'pulsar_txn_tb_client_abort_latency', 'pulsar_txn_tb_client_commit_latency'] + public static final String TRANSACTION_BUFFER_CLIENT_OPERATION_COUNTER = + "pulsar.broker.topic.transaction.buffer.client.operation.count"; + private final ObservableLongMeasurement transactionBufferClientOperationCounter; + + // Replaces pulsar_subscription_delayed + public static final String DELAYED_SUBSCRIPTION_COUNTER = "pulsar.broker.topic.subscription.delayed.entry.count"; + private final ObservableLongMeasurement delayedSubscriptionCounter; + + // Omitted: pulsar_delayed_message_index_size_bytes + + // Omitted: pulsar_delayed_message_index_bucket_total + + // Omitted: pulsar_delayed_message_index_loaded + + // Omitted: pulsar_delayed_message_index_bucket_snapshot_size_bytes + + // Omitted: pulsar_delayed_message_index_bucket_op_count + + // Omitted: pulsar_delayed_message_index_bucket_op_latency_ms + + + private final BatchCallback batchCallback; + private final PulsarService pulsar; + + public OpenTelemetryTopicStats(PulsarService pulsar) { + this.pulsar = pulsar; + var meter = pulsar.getOpenTelemetry().getMeter(); + + subscriptionCounter = meter + .upDownCounterBuilder(SUBSCRIPTION_COUNTER) + .setUnit("{subscription}") + .setDescription("The number of Pulsar subscriptions of the topic served by this broker.") + .buildObserver(); + + producerCounter = meter + .upDownCounterBuilder(PRODUCER_COUNTER) + .setUnit("{producer}") + .setDescription("The number of active producers of the topic connected to this broker.") + .buildObserver(); + + consumerCounter = meter + .upDownCounterBuilder(CONSUMER_COUNTER) + .setUnit("{consumer}") + .setDescription("The number of active consumers of the topic connected to this broker.") + .buildObserver(); + + messageInCounter = meter + .counterBuilder(MESSAGE_IN_COUNTER) + .setUnit("{message}") + .setDescription("The total number of messages received for this topic.") + .buildObserver(); + + messageOutCounter = meter + .counterBuilder(MESSAGE_OUT_COUNTER) + .setUnit("{message}") + .setDescription("The total number of messages read from this topic.") + .buildObserver(); + + bytesInCounter = meter + .counterBuilder(BYTES_IN_COUNTER) + .setUnit("By") + .setDescription("The total number of messages bytes received for this topic.") + .buildObserver(); + + bytesOutCounter = meter + .counterBuilder(BYTES_OUT_COUNTER) + .setUnit("By") + .setDescription("The total number of messages bytes read from this topic.") + .buildObserver(); + + publishRateLimitHitCounter = meter + .counterBuilder(PUBLISH_RATE_LIMIT_HIT_COUNTER) + .setUnit("{event}") + .setDescription("The number of times the publish rate limit is triggered.") + .buildObserver(); + + storageCounter = meter + .upDownCounterBuilder(STORAGE_COUNTER) + .setUnit("By") + .setDescription( + "The total storage size of the messages in this topic, including storage used by replicas.") + .buildObserver(); + + storageLogicalCounter = meter + .upDownCounterBuilder(STORAGE_LOGICAL_COUNTER) + .setUnit("By") + .setDescription("The storage size of the messages in this topic, excluding storage used by replicas.") + .buildObserver(); + + storageBacklogCounter = meter + .upDownCounterBuilder(STORAGE_BACKLOG_COUNTER) + .setUnit("By") + .setDescription("The size of the backlog storage for this topic.") + .buildObserver(); + + storageOffloadedCounter = meter + .upDownCounterBuilder(STORAGE_OFFLOADED_COUNTER) + .setUnit("By") + .setDescription("The total amount of the data in this topic offloaded to the tiered storage.") + .buildObserver(); + + backlogQuotaLimitSize = meter + .upDownCounterBuilder(BACKLOG_QUOTA_LIMIT_SIZE) + .setUnit("By") + .setDescription("The size based backlog quota limit for this topic.") + .buildObserver(); + + backlogQuotaLimitTime = meter + .gaugeBuilder(BACKLOG_QUOTA_LIMIT_TIME) + .ofLongs() + .setUnit("s") + .setDescription("The time based backlog quota limit for this topic.") + .buildObserver(); + + backlogEvictionCounter = meter + .counterBuilder(BACKLOG_EVICTION_COUNTER) + .setUnit("{eviction}") + .setDescription("The number of times a backlog was evicted since it has exceeded its quota.") + .buildObserver(); + + backlogQuotaAge = meter + .gaugeBuilder(BACKLOG_QUOTA_AGE) + .ofLongs() + .setUnit("s") + .setDescription("The age of the oldest unacknowledged message (backlog).") + .buildObserver(); + + storageOutCounter = meter + .counterBuilder(STORAGE_OUT_COUNTER) + .setUnit("{entry}") + .setDescription("The total message batches (entries) written to the storage for this topic.") + .buildObserver(); + + storageInCounter = meter + .counterBuilder(STORAGE_IN_COUNTER) + .setUnit("{entry}") + .setDescription("The total message batches (entries) read from the storage for this topic.") + .buildObserver(); + + compactionRemovedCounter = meter + .counterBuilder(COMPACTION_REMOVED_COUNTER) + .setUnit("{message}") + .setDescription("The total number of messages removed by compaction.") + .buildObserver(); + + compactionOperationCounter = meter + .counterBuilder(COMPACTION_OPERATION_COUNTER) + .setUnit("{operation}") + .setDescription("The total number of compaction operations.") + .buildObserver(); + + compactionDurationSeconds = meter + .upDownCounterBuilder(COMPACTION_DURATION_SECONDS) + .ofDoubles() + .setUnit("s") + .setDescription("The total time duration of compaction operations on the topic.") + .buildObserver(); + + compactionBytesInCounter = meter + .counterBuilder(COMPACTION_BYTES_IN_COUNTER) + .setUnit("By") + .setDescription("The total count of bytes read by the compaction process for this topic.") + .buildObserver(); + + compactionBytesOutCounter = meter + .counterBuilder(COMPACTION_BYTES_OUT_COUNTER) + .setUnit("By") + .setDescription("The total count of bytes written by the compaction process for this topic.") + .buildObserver(); + + compactionEntriesCounter = meter + .counterBuilder(COMPACTION_ENTRIES_COUNTER) + .setUnit("{entry}") + .setDescription("The total number of compacted entries.") + .buildObserver(); + + compactionBytesCounter = meter + .counterBuilder(COMPACTION_BYTES_COUNTER) + .setUnit("By") + .setDescription("The total size of the compacted entries.") + .buildObserver(); + + transactionCounter = meter + .upDownCounterBuilder(TRANSACTION_COUNTER) + .setUnit("{transaction}") + .setDescription("The number of transactions on this topic.") + .buildObserver(); + + transactionBufferClientOperationCounter = meter + .counterBuilder(TRANSACTION_BUFFER_CLIENT_OPERATION_COUNTER) + .setUnit("{operation}") + .setDescription("The number of operations on the transaction buffer client.") + .buildObserver(); + + delayedSubscriptionCounter = meter + .upDownCounterBuilder(DELAYED_SUBSCRIPTION_COUNTER) + .setUnit("{entry}") + .setDescription("The total number of message batches (entries) delayed for dispatching.") + .buildObserver(); + + batchCallback = meter.batchCallback(() -> pulsar.getBrokerService() + .getTopics() + .values() + .stream() + .map(topicFuture -> topicFuture.getNow(Optional.empty())) + .forEach(topic -> topic.ifPresent(this::recordMetricsForTopic)), + subscriptionCounter, + producerCounter, + consumerCounter, + messageInCounter, + messageOutCounter, + bytesInCounter, + bytesOutCounter, + publishRateLimitHitCounter, + storageCounter, + storageLogicalCounter, + storageBacklogCounter, + storageOffloadedCounter, + backlogQuotaLimitSize, + backlogQuotaLimitTime, + backlogEvictionCounter, + backlogQuotaAge, + storageOutCounter, + storageInCounter, + compactionRemovedCounter, + compactionOperationCounter, + compactionDurationSeconds, + compactionBytesInCounter, + compactionBytesOutCounter, + compactionEntriesCounter, + compactionBytesCounter, + transactionCounter, + transactionBufferClientOperationCounter, + delayedSubscriptionCounter); + } + + @Override + public void close() { + batchCallback.close(); + } + + private void recordMetricsForTopic(Topic topic) { + var topicAttributes = topic.getTopicAttributes(); + var attributes = topicAttributes.getCommonAttributes(); + + if (topic instanceof AbstractTopic abstractTopic) { + subscriptionCounter.record(abstractTopic.getSubscriptions().size(), attributes); + producerCounter.record(abstractTopic.getProducers().size(), attributes); + consumerCounter.record(abstractTopic.getNumberOfConsumers(), attributes); + + messageInCounter.record(abstractTopic.getMsgInCounter(), attributes); + messageOutCounter.record(abstractTopic.getMsgOutCounter(), attributes); + bytesInCounter.record(abstractTopic.getBytesInCounter(), attributes); + bytesOutCounter.record(abstractTopic.getBytesOutCounter(), attributes); + + publishRateLimitHitCounter.record(abstractTopic.getTotalPublishRateLimitCounter(), attributes); + + // Omitted: consumerMsgAckCounter + } + + if (topic instanceof PersistentTopic persistentTopic) { + var persistentTopicMetrics = persistentTopic.getPersistentTopicMetrics(); + + var persistentTopicAttributes = persistentTopic.getTopicAttributes(); + var managedLedger = persistentTopic.getManagedLedger(); + var managedLedgerStats = persistentTopic.getManagedLedger().getStats(); + storageCounter.record(managedLedgerStats.getStoredMessagesSize(), attributes); + storageLogicalCounter.record(managedLedgerStats.getStoredMessagesLogicalSize(), attributes); + storageBacklogCounter.record(managedLedger.getEstimatedBacklogSize(), attributes); + storageOffloadedCounter.record(managedLedger.getOffloadedSize(), attributes); + storageInCounter.record(managedLedgerStats.getReadEntriesSucceededTotal(), attributes); + storageOutCounter.record(managedLedgerStats.getAddEntrySucceedTotal(), attributes); + + backlogQuotaLimitSize.record( + topic.getBacklogQuota(BacklogQuota.BacklogQuotaType.destination_storage).getLimitSize(), + attributes); + backlogQuotaLimitTime.record( + topic.getBacklogQuota(BacklogQuota.BacklogQuotaType.message_age).getLimitTime(), + attributes); + backlogQuotaAge.record(topic.getBestEffortOldestUnacknowledgedMessageAgeSeconds(), attributes); + var backlogQuotaMetrics = persistentTopicMetrics.getBacklogQuotaMetrics(); + backlogEvictionCounter.record(backlogQuotaMetrics.getSizeBasedBacklogQuotaExceededEvictionCount(), + persistentTopicAttributes.getSizeBasedQuotaAttributes()); + backlogEvictionCounter.record(backlogQuotaMetrics.getTimeBasedBacklogQuotaExceededEvictionCount(), + persistentTopicAttributes.getTimeBasedQuotaAttributes()); + + var txnBuffer = persistentTopic.getTransactionBuffer(); + transactionCounter.record(txnBuffer.getOngoingTxnCount(), + persistentTopicAttributes.getTransactionActiveAttributes()); + transactionCounter.record(txnBuffer.getCommittedTxnCount(), + persistentTopicAttributes.getTransactionCommittedAttributes()); + transactionCounter.record(txnBuffer.getAbortedTxnCount(), + persistentTopicAttributes.getTransactionAbortedAttributes()); + + var txnBufferClientMetrics = persistentTopicMetrics.getTransactionBufferClientMetrics(); + transactionBufferClientOperationCounter.record(txnBufferClientMetrics.getCommitSucceededCount().sum(), + persistentTopicAttributes.getTransactionBufferClientCommitSucceededAttributes()); + transactionBufferClientOperationCounter.record(txnBufferClientMetrics.getCommitFailedCount().sum(), + persistentTopicAttributes.getTransactionBufferClientCommitFailedAttributes()); + transactionBufferClientOperationCounter.record(txnBufferClientMetrics.getAbortSucceededCount().sum(), + persistentTopicAttributes.getTransactionBufferClientAbortSucceededAttributes()); + transactionBufferClientOperationCounter.record(txnBufferClientMetrics.getAbortFailedCount().sum(), + persistentTopicAttributes.getTransactionBufferClientAbortFailedAttributes()); + + Optional.ofNullable(pulsar.getNullableCompactor()) + .map(Compactor::getStats) + .flatMap(compactorMXBean -> compactorMXBean.getCompactionRecordForTopic(topic.getName())) + .ifPresent(compactionRecord -> { + compactionRemovedCounter.record(compactionRecord.getCompactionRemovedEventCount(), attributes); + compactionOperationCounter.record(compactionRecord.getCompactionSucceedCount(), + persistentTopicAttributes.getCompactionSuccessAttributes()); + compactionOperationCounter.record(compactionRecord.getCompactionFailedCount(), + persistentTopicAttributes.getCompactionFailureAttributes()); + compactionDurationSeconds.record(MetricsUtil.convertToSeconds( + compactionRecord.getCompactionDurationTimeInMills(), TimeUnit.MILLISECONDS), attributes); + compactionBytesInCounter.record(compactionRecord.getCompactionReadBytes(), attributes); + compactionBytesOutCounter.record(compactionRecord.getCompactionWriteBytes(), attributes); + + persistentTopic.getCompactedTopicContext().map(CompactedTopicContext::getLedger) + .ifPresent(ledger -> { + compactionEntriesCounter.record(ledger.getLastAddConfirmed() + 1, attributes); + compactionBytesCounter.record(ledger.getLength(), attributes); + }); + }); + + var delayedMessages = topic.getSubscriptions().values().stream() + .map(Subscription::getDispatcher) + .filter(Objects::nonNull) + .mapToLong(Dispatcher::getNumberOfDelayedMessages) + .sum(); + delayedSubscriptionCounter.record(delayedMessages, attributes); + } + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/OpenTelemetryTransactionCoordinatorStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/OpenTelemetryTransactionCoordinatorStats.java new file mode 100644 index 0000000000000..ab73b2390b37d --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/OpenTelemetryTransactionCoordinatorStats.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats; + +import io.opentelemetry.api.metrics.BatchCallback; +import io.opentelemetry.api.metrics.ObservableLongMeasurement; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.transaction.coordinator.TransactionMetadataStore; + +public class OpenTelemetryTransactionCoordinatorStats implements AutoCloseable { + + // Replaces ['pulsar_txn_aborted_total', + // 'pulsar_txn_committed_total', + // 'pulsar_txn_created_total', + // 'pulsar_txn_timeout_total', + // 'pulsar_txn_active_count'] + public static final String TRANSACTION_COUNTER = "pulsar.broker.transaction.coordinator.transaction.count"; + private final ObservableLongMeasurement transactionCounter; + + // Replaces pulsar_txn_append_log_total + public static final String APPEND_LOG_COUNTER = "pulsar.broker.transaction.coordinator.append.log.count"; + private final ObservableLongMeasurement appendLogCounter; + + private final BatchCallback batchCallback; + + public OpenTelemetryTransactionCoordinatorStats(PulsarService pulsar) { + var meter = pulsar.getOpenTelemetry().getMeter(); + + transactionCounter = meter + .upDownCounterBuilder(TRANSACTION_COUNTER) + .setUnit("{transaction}") + .setDescription("The number of transactions handled by the coordinator.") + .buildObserver(); + + appendLogCounter = meter + .counterBuilder(APPEND_LOG_COUNTER) + .setUnit("{entry}") + .setDescription("The number of transaction metadata entries appended by the coordinator.") + .buildObserver(); + + batchCallback = meter.batchCallback(() -> { + var transactionMetadataStoreService = pulsar.getTransactionMetadataStoreService(); + // Avoid NPE during Pulsar shutdown. + if (transactionMetadataStoreService != null) { + transactionMetadataStoreService.getStores() + .values() + .forEach(this::recordMetricsForTransactionMetadataStore); + } + }, + transactionCounter, + appendLogCounter); + } + + @Override + public void close() { + batchCallback.close(); + } + + private void recordMetricsForTransactionMetadataStore(TransactionMetadataStore transactionMetadataStore) { + var attributes = transactionMetadataStore.getAttributes(); + var stats = transactionMetadataStore.getMetadataStoreStats(); + + transactionCounter.record(stats.getAbortedCount(), attributes.getTxnAbortedAttributes()); + transactionCounter.record(stats.getActives(), attributes.getTxnActiveAttributes()); + transactionCounter.record(stats.getCommittedCount(), attributes.getTxnCommittedAttributes()); + transactionCounter.record(stats.getCreatedCount(), attributes.getTxnCreatedAttributes()); + transactionCounter.record(stats.getTimeoutCount(), attributes.getTxnTimeoutAttributes()); + + appendLogCounter.record(stats.getAppendLogCount(), attributes.getCommonAttributes()); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/OpenTelemetryTransactionPendingAckStoreStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/OpenTelemetryTransactionPendingAckStoreStats.java new file mode 100644 index 0000000000000..562ad56e44db4 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/OpenTelemetryTransactionPendingAckStoreStats.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats; + +import io.opentelemetry.api.metrics.ObservableLongCounter; +import io.opentelemetry.api.metrics.ObservableLongMeasurement; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.service.Subscription; +import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; + +public class OpenTelemetryTransactionPendingAckStoreStats implements AutoCloseable { + + // Replaces ['pulsar_txn_tp_committed_count_total', 'pulsar_txn_tp_aborted_count_total'] + public static final String ACK_COUNTER = "pulsar.broker.transaction.pending.ack.store.transaction.count"; + private final ObservableLongCounter ackCounter; + + public OpenTelemetryTransactionPendingAckStoreStats(PulsarService pulsar) { + var meter = pulsar.getOpenTelemetry().getMeter(); + + ackCounter = meter + .counterBuilder(ACK_COUNTER) + .setUnit("{transaction}") + .setDescription("The number of transactions handled by the persistent ack store.") + .buildWithCallback(measurement -> pulsar.getBrokerService() + .getTopics() + .values() + .stream() + .filter(topicFuture -> topicFuture.isDone() && !topicFuture.isCompletedExceptionally()) + .map(CompletableFuture::join) + .filter(Optional::isPresent) + .map(Optional::get) + .filter(Topic::isPersistent) + .map(Topic::getSubscriptions) + .forEach(subs -> subs.forEach((__, sub) -> recordMetricsForSubscription(measurement, sub)))); + } + + @Override + public void close() { + ackCounter.close(); + } + + private void recordMetricsForSubscription(ObservableLongMeasurement measurement, Subscription subscription) { + assert subscription instanceof PersistentSubscription; // The topics have already been filtered for persistence. + var stats = ((PersistentSubscription) subscription).getPendingAckHandle().getPendingAckHandleStats(); + if (stats != null) { + var attributes = stats.getAttributes(); + measurement.record(stats.getCommitSuccessCount(), attributes.getCommitSuccessAttributes()); + measurement.record(stats.getCommitFailedCount(), attributes.getCommitFailureAttributes()); + measurement.record(stats.getAbortSuccessCount(), attributes.getAbortSuccessAttributes()); + measurement.record(stats.getAbortFailedCount(), attributes.getAbortFailureAttributes()); + } + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/PulsarBrokerOpenTelemetry.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/PulsarBrokerOpenTelemetry.java new file mode 100644 index 0000000000000..065b03e6454ea --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/PulsarBrokerOpenTelemetry.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats; + +import com.google.common.annotations.VisibleForTesting; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdkBuilder; +import java.io.Closeable; +import java.util.function.Consumer; +import lombok.Getter; +import org.apache.pulsar.PulsarVersion; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.opentelemetry.Constants; +import org.apache.pulsar.opentelemetry.OpenTelemetryService; + +public class PulsarBrokerOpenTelemetry implements Closeable { + + public static final String SERVICE_NAME = "pulsar-broker"; + + @Getter + private final OpenTelemetryService openTelemetryService; + + @Getter + private final Meter meter; + + public PulsarBrokerOpenTelemetry(ServiceConfiguration config, + @VisibleForTesting Consumer builderCustomizer) { + openTelemetryService = OpenTelemetryService.builder() + .clusterName(config.getClusterName()) + .serviceName(SERVICE_NAME) + .serviceVersion(PulsarVersion.getVersion()) + .builderCustomizer(builderCustomizer) + .build(); + meter = openTelemetryService.getOpenTelemetry().getMeter(Constants.BROKER_INSTRUMENTATION_SCOPE_NAME); + } + + public OpenTelemetry getOpenTelemetry() { + return openTelemetryService.getOpenTelemetry(); + } + + @Override + public void close() { + openTelemetryService.close(); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/TimeWindow.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/TimeWindow.java deleted file mode 100644 index 08730189322ee..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/TimeWindow.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.stats; - -import java.util.concurrent.atomic.AtomicReferenceArray; -import java.util.function.Function; - -public final class TimeWindow { - private final int interval; - private final int sampleCount; - private final AtomicReferenceArray> array; - - public TimeWindow(int sampleCount, int interval) { - this.sampleCount = sampleCount; - this.interval = interval; - this.array = new AtomicReferenceArray<>(sampleCount); - } - - /** - * return current time window data. - * - * @param function generate data. - * @return - */ - public synchronized WindowWrap current(Function function) { - long millis = System.currentTimeMillis(); - - if (millis < 0) { - return null; - } - int idx = calculateTimeIdx(millis); - long windowStart = calculateWindowStart(millis); - while (true) { - WindowWrap old = array.get(idx); - if (old == null) { - WindowWrap window = new WindowWrap<>(interval, windowStart, null); - if (array.compareAndSet(idx, null, window)) { - T value = null == function ? null : function.apply(null); - window.value(value); - return window; - } else { - Thread.yield(); - } - } else if (windowStart == old.start()) { - return old; - } else if (windowStart > old.start()) { - T value = null == function ? null : function.apply(old.value()); - old.value(value); - old.resetWindowStart(windowStart); - return old; - } else { - //it should never goes here - throw new IllegalStateException(); - } - } - } - - private int calculateTimeIdx(long timeMillis) { - long timeId = timeMillis / this.interval; - return (int) (timeId % sampleCount); - } - - private long calculateWindowStart(long timeMillis) { - return timeMillis - timeMillis % this.interval; - } - - public int sampleCount() { - return sampleCount; - } - - public int interval() { - return interval; - } - - public long currentWindowStart(long millis) { - return this.calculateWindowStart(millis); - } -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/WindowWrap.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/WindowWrap.java deleted file mode 100644 index 12869b82921e5..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/WindowWrap.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.stats; - -public final class WindowWrap { - private final long interval; - private long start; - private T value; - - public WindowWrap(long interval, long windowStart, T value) { - this.interval = interval; - this.start = windowStart; - this.value = value; - } - - public long interval() { - return this.interval; - } - - public long start() { - return this.start; - } - - public T value() { - return value; - } - - public void value(T value) { - this.value = value; - } - - public WindowWrap resetWindowStart(long startTime) { - this.start = startTime; - return this; - } - - public boolean isTimeInWindow(long timeMillis) { - return start <= timeMillis && timeMillis < start + interval; - } -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/AbstractMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/AbstractMetrics.java index f87abcf495308..489d37dd0a307 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/AbstractMetrics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/AbstractMetrics.java @@ -24,9 +24,8 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerFactoryMXBean; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerMBeanImpl; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.common.policies.data.TopicStats; @@ -132,7 +131,7 @@ protected Metrics createMetrics(Map dimensionMap) { * @return */ protected ManagedLedgerFactoryMXBean getManagedLedgerCacheStats() { - return ((ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory()).getCacheStats(); + return pulsar.getManagedLedgerFactory().getCacheStats(); } /** @@ -140,8 +139,8 @@ protected ManagedLedgerFactoryMXBean getManagedLedgerCacheStats() { * * @return */ - protected Map getManagedLedgers() { - return ((ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory()).getManagedLedgers(); + protected Map getManagedLedgers() { + return pulsar.getManagedLedgerFactory().getManagedLedgers(); } protected String getLocalClusterName() { @@ -235,8 +234,8 @@ protected void populateMaxMap(Map map, String mkey, long value) { * @param metrics * @param ledger */ - protected void populateDimensionMap(Map> ledgersByDimensionMap, Metrics metrics, - ManagedLedgerImpl ledger) { + protected void populateDimensionMap(Map> ledgersByDimensionMap, Metrics metrics, + ManagedLedger ledger) { ledgersByDimensionMap.computeIfAbsent(metrics, __ -> new ArrayList<>()).add(ledger); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedCursorMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedCursorMetrics.java index 424a7cb2f81ac..639f51ead6cee 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedCursorMetrics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedCursorMetrics.java @@ -25,9 +25,7 @@ import java.util.Map; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedCursorMXBean; -import org.apache.bookkeeper.mledger.impl.ManagedCursorContainer; -import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.common.stats.Metrics; @@ -55,16 +53,15 @@ public synchronized List generate() { */ private List aggregate() { metricsCollection.clear(); - for (Map.Entry e : getManagedLedgers().entrySet()) { + for (Map.Entry e : getManagedLedgers().entrySet()) { String ledgerName = e.getKey(); - ManagedLedgerImpl ledger = e.getValue(); + ManagedLedger ledger = e.getValue(); String namespace = parseNamespaceFromLedgerName(ledgerName); - ManagedCursorContainer cursorContainer = ledger.getCursors(); - Iterator cursorIterator = cursorContainer.iterator(); + Iterator cursorIterator = ledger.getCursors().iterator(); while (cursorIterator.hasNext()) { - ManagedCursorImpl cursor = (ManagedCursorImpl) cursorIterator.next(); + ManagedCursor cursor = cursorIterator.next(); ManagedCursorMXBean cStats = cursor.getStats(); dimensionMap.clear(); dimensionMap.put("namespace", namespace); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerCacheMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerCacheMetrics.java index 890a37aa2d877..9eb4beb72fbf2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerCacheMetrics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerCacheMetrics.java @@ -18,13 +18,10 @@ */ package org.apache.pulsar.broker.stats.metrics; -import io.netty.buffer.PoolArenaMetric; -import io.netty.buffer.PoolChunkListMetric; -import io.netty.buffer.PoolChunkMetric; -import io.netty.buffer.PooledByteBufAllocator; import java.util.ArrayList; import java.util.List; import org.apache.bookkeeper.mledger.ManagedLedgerFactoryMXBean; +import org.apache.bookkeeper.mledger.impl.cache.PooledByteBufAllocatorStats; import org.apache.bookkeeper.mledger.impl.cache.RangeEntryCacheImpl; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.common.stats.Metrics; @@ -57,37 +54,13 @@ public synchronized List generate() { m.put("brk_ml_cache_hits_throughput", mlCacheStats.getCacheHitsThroughput()); m.put("brk_ml_cache_misses_throughput", mlCacheStats.getCacheMissesThroughput()); - PooledByteBufAllocator allocator = RangeEntryCacheImpl.ALLOCATOR; - long activeAllocations = 0; - long activeAllocationsSmall = 0; - long activeAllocationsNormal = 0; - long activeAllocationsHuge = 0; - long totalAllocated = 0; - long totalUsed = 0; - - for (PoolArenaMetric arena : allocator.metric().directArenas()) { - activeAllocations += arena.numActiveAllocations(); - activeAllocationsSmall += arena.numActiveSmallAllocations(); - activeAllocationsNormal += arena.numActiveNormalAllocations(); - activeAllocationsHuge += arena.numActiveHugeAllocations(); - - for (PoolChunkListMetric list : arena.chunkLists()) { - for (PoolChunkMetric chunk : list) { - int size = chunk.chunkSize(); - int used = size - chunk.freeBytes(); - - totalAllocated += size; - totalUsed += used; - } - } - } - - m.put("brk_ml_cache_pool_allocated", totalAllocated); - m.put("brk_ml_cache_pool_used", totalUsed); - m.put("brk_ml_cache_pool_active_allocations", activeAllocations); - m.put("brk_ml_cache_pool_active_allocations_small", activeAllocationsSmall); - m.put("brk_ml_cache_pool_active_allocations_normal", activeAllocationsNormal); - m.put("brk_ml_cache_pool_active_allocations_huge", activeAllocationsHuge); + var allocatorStats = new PooledByteBufAllocatorStats(RangeEntryCacheImpl.ALLOCATOR); + m.put("brk_ml_cache_pool_allocated", allocatorStats.totalAllocated); + m.put("brk_ml_cache_pool_used", allocatorStats.totalUsed); + m.put("brk_ml_cache_pool_active_allocations", allocatorStats.activeAllocations); + m.put("brk_ml_cache_pool_active_allocations_small", allocatorStats.activeAllocationsSmall); + m.put("brk_ml_cache_pool_active_allocations_normal", allocatorStats.activeAllocationsNormal); + m.put("brk_ml_cache_pool_active_allocations_huge", allocatorStats.activeAllocationsHuge); metrics.clear(); metrics.add(m); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerMetrics.java index 36004bc1281bb..52c69265c2f1f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerMetrics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerMetrics.java @@ -23,16 +23,15 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerMXBean; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.common.stats.Metrics; public class ManagedLedgerMetrics extends AbstractMetrics { private List metricsCollection; - private Map> ledgersByDimensionMap; + private Map> ledgersByDimensionMap; // temp map to prepare aggregation metrics private Map tempAggregatedMetricsMap; private static final Buckets @@ -53,7 +52,7 @@ public ManagedLedgerMetrics(PulsarService pulsar) { this.metricsCollection = new ArrayList<>(); this.ledgersByDimensionMap = new HashMap<>(); this.tempAggregatedMetricsMap = new HashMap<>(); - this.statsPeriodSeconds = ((ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory()) + this.statsPeriodSeconds = pulsar.getManagedLedgerFactory() .getConfig().getStatsPeriodSeconds(); } @@ -71,20 +70,20 @@ public synchronized List generate() { * @param ledgersByDimension * @return */ - private List aggregate(Map> ledgersByDimension) { + private List aggregate(Map> ledgersByDimension) { metricsCollection.clear(); - for (Entry> e : ledgersByDimension.entrySet()) { + for (Entry> e : ledgersByDimension.entrySet()) { Metrics metrics = e.getKey(); - List ledgers = e.getValue(); + List ledgers = e.getValue(); // prepare aggregation map tempAggregatedMetricsMap.clear(); // generate the collections by each metrics and then apply the aggregation - for (ManagedLedgerImpl ledger : ledgers) { + for (ManagedLedger ledger : ledgers) { ManagedLedgerMXBean lStats = ledger.getStats(); populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_AddEntryBytesRate", @@ -151,17 +150,17 @@ private List aggregate(Map> ledgersByD * * @return */ - private Map> groupLedgersByDimension() { + private Map> groupLedgersByDimension() { ledgersByDimensionMap.clear(); // get the current topics statistics from StatsBrokerFilter // Map : topic-name->dest-stat - for (Entry e : getManagedLedgers().entrySet()) { + for (Entry e : getManagedLedgers().entrySet()) { String ledgerName = e.getKey(); - ManagedLedgerImpl ledger = e.getValue(); + ManagedLedger ledger = e.getValue(); // we want to aggregate by NS dimension diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedBrokerStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedBrokerStats.java index 715231d3c6ee1..85096be9b00f4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedBrokerStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedBrokerStats.java @@ -33,7 +33,14 @@ public class AggregatedBrokerStats { public double storageReadRate; public double storageReadCacheMissesRate; public long msgBacklog; + public long sizeBasedBacklogQuotaExceededEvictionCount; + public long timeBasedBacklogQuotaExceededEvictionCount; + public long bytesInCounter; + public long bytesOutCounter; + public long systemTopicBytesInCounter; + public long bytesOutInternalCounter; + @SuppressWarnings("DuplicatedCode") void updateStats(TopicStats stats) { topicsCount++; subscriptionsCount += stats.subscriptionsCount; @@ -49,8 +56,15 @@ void updateStats(TopicStats stats) { storageReadRate += stats.managedLedgerStats.storageReadRate; storageReadCacheMissesRate += stats.managedLedgerStats.storageReadCacheMissesRate; msgBacklog += stats.msgBacklog; + timeBasedBacklogQuotaExceededEvictionCount += stats.timeBasedBacklogQuotaExceededEvictionCount; + sizeBasedBacklogQuotaExceededEvictionCount += stats.sizeBasedBacklogQuotaExceededEvictionCount; + bytesInCounter += stats.bytesInCounter; + bytesOutCounter += stats.bytesOutCounter; + systemTopicBytesInCounter += stats.systemTopicBytesInCounter; + bytesOutInternalCounter += stats.bytesOutInternalCounter; } + @SuppressWarnings("DuplicatedCode") public void reset() { topicsCount = 0; subscriptionsCount = 0; @@ -66,5 +80,11 @@ public void reset() { storageReadRate = 0; storageReadCacheMissesRate = 0; msgBacklog = 0; + sizeBasedBacklogQuotaExceededEvictionCount = 0; + timeBasedBacklogQuotaExceededEvictionCount = 0; + bytesInCounter = 0; + bytesOutCounter = 0; + systemTopicBytesInCounter = 0; + bytesOutInternalCounter = 0; } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java index 9fe5588044d2f..aaaea7b493e45 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java @@ -34,7 +34,7 @@ public class AggregatedNamespaceStats { public double throughputIn; public double throughputOut; - public long messageAckRate; + public double messageAckRate; public long bytesInCounter; public long msgInCounter; public long bytesOutCounter; @@ -43,6 +43,7 @@ public class AggregatedNamespaceStats { public ManagedLedgerStats managedLedgerStats = new ManagedLedgerStats(); public long msgBacklog; public long msgDelayed; + public long msgInReplay; public long ongoingTxnCount; public long abortedTxnCount; @@ -51,6 +52,9 @@ public class AggregatedNamespaceStats { long backlogQuotaLimit; long backlogQuotaLimitTime; + public long sizeBasedBacklogQuotaExceededEvictionCount; + public long timeBasedBacklogQuotaExceededEvictionCount; + public Map replicationStats = new HashMap<>(); public Map subscriptionStats = new HashMap<>(); @@ -64,10 +68,11 @@ public class AggregatedNamespaceStats { long compactionCompactedEntriesCount; long compactionCompactedEntriesSize; StatsBuckets compactionLatencyBuckets = new StatsBuckets(CompactionRecord.WRITE_LATENCY_BUCKETS_USEC); - int delayedMessageIndexSizeInBytes; + long delayedMessageIndexSizeInBytes; Map bucketDelayedIndexStats = new HashMap<>(); + @SuppressWarnings("DuplicatedCode") void updateStats(TopicStats stats) { topicsCount++; @@ -105,6 +110,9 @@ void updateStats(TopicStats stats) { backlogQuotaLimit = Math.max(backlogQuotaLimit, stats.backlogQuotaLimit); backlogQuotaLimitTime = Math.max(backlogQuotaLimitTime, stats.backlogQuotaLimitTime); + sizeBasedBacklogQuotaExceededEvictionCount += stats.sizeBasedBacklogQuotaExceededEvictionCount; + timeBasedBacklogQuotaExceededEvictionCount += stats.timeBasedBacklogQuotaExceededEvictionCount; + managedLedgerStats.storageWriteRate += stats.managedLedgerStats.storageWriteRate; managedLedgerStats.storageReadRate += stats.managedLedgerStats.storageReadRate; managedLedgerStats.storageReadCacheMissesRate += stats.managedLedgerStats.storageReadCacheMissesRate; @@ -126,6 +134,7 @@ void updateStats(TopicStats stats) { replStats.replicationBacklog += as.replicationBacklog; replStats.msgRateExpired += as.msgRateExpired; replStats.connectedCount += as.connectedCount; + replStats.disconnectedCount += as.disconnectedCount; replStats.replicationDelayInSeconds += as.replicationDelayInSeconds; }); @@ -133,10 +142,12 @@ void updateStats(TopicStats stats) { AggregatedSubscriptionStats subsStats = subscriptionStats.computeIfAbsent(n, k -> new AggregatedSubscriptionStats()); msgDelayed += as.msgDelayed; + msgInReplay += as.msgInReplay; subsStats.blockedSubscriptionOnUnackedMsgs = as.blockedSubscriptionOnUnackedMsgs; subsStats.msgBacklog += as.msgBacklog; subsStats.msgBacklogNoDelayed += as.msgBacklogNoDelayed; subsStats.msgDelayed += as.msgDelayed; + subsStats.msgInReplay += as.msgInReplay; subsStats.msgRateRedeliver += as.msgRateRedeliver; subsStats.unackedMessages += as.unackedMessages; subsStats.filterProcessedMsgCount += as.filterProcessedMsgCount; @@ -172,6 +183,7 @@ void updateStats(TopicStats stats) { compactionLatencyBuckets.addAll(stats.compactionLatencyBuckets); } + @SuppressWarnings("DuplicatedCode") public void reset() { managedLedgerStats.reset(); topicsCount = 0; @@ -182,14 +194,38 @@ public void reset() { rateOut = 0; throughputIn = 0; throughputOut = 0; + messageAckRate = 0; + bytesInCounter = 0; + msgInCounter = 0; + + bytesOutCounter = 0; + msgOutCounter = 0; msgBacklog = 0; msgDelayed = 0; + msgInReplay = 0; + ongoingTxnCount = 0; + abortedTxnCount = 0; + committedTxnCount = 0; + backlogQuotaLimit = 0; backlogQuotaLimitTime = -1; replicationStats.clear(); subscriptionStats.clear(); + + sizeBasedBacklogQuotaExceededEvictionCount = 0; + timeBasedBacklogQuotaExceededEvictionCount = 0; + + compactionRemovedEventCount = 0; + compactionSucceedCount = 0; + compactionFailedCount = 0; + compactionDurationTimeInMills = 0; + compactionReadThroughput = 0; + compactionWriteThroughput = 0; + compactionCompactedEntriesCount = 0; + compactionCompactedEntriesSize = 0; + delayedMessageIndexSizeInBytes = 0; bucketDelayedIndexStats.clear(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedReplicationStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedReplicationStats.java index 78f33f874e998..82668de6c35f7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedReplicationStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedReplicationStats.java @@ -41,6 +41,9 @@ public class AggregatedReplicationStats { /** The count of replication-subscriber up and running to replicate to remote cluster. */ public long connectedCount; + /** The count of replication-subscriber that failed to start to replicate to remote cluster. */ + public long disconnectedCount; + /** Time in seconds from the time a message was produced to the time when it is about to be replicated. */ public long replicationDelayInSeconds; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java index da0324c55655c..b713146f58bac 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java @@ -43,6 +43,8 @@ public class AggregatedSubscriptionStats { public long msgDelayed; + public long msgInReplay; + long msgOutCounter; long bytesOutCounter; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/MetricsExports.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/MetricsExports.java new file mode 100644 index 0000000000000..b80e5747d8a5a --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/MetricsExports.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats.prometheus; + +import static org.apache.pulsar.common.stats.JvmMetrics.getJvmDirectMemoryUsed; +import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.Gauge; +import io.prometheus.client.hotspot.DefaultExports; +import org.apache.pulsar.PulsarVersion; +import org.apache.pulsar.common.util.DirectMemoryUtils; + +public class MetricsExports { + private static boolean initialized = false; + + private MetricsExports() { + } + + public static synchronized void initialize() { + if (!initialized) { + DefaultExports.initialize(); + register(CollectorRegistry.defaultRegistry); + initialized = true; + } + } + + public static void register(CollectorRegistry registry) { + Gauge.build("jvm_memory_direct_bytes_used", "-").create().setChild(new Gauge.Child() { + @Override + public double get() { + return getJvmDirectMemoryUsed(); + } + }).register(registry); + + Gauge.build("jvm_memory_direct_bytes_max", "-").create().setChild(new Gauge.Child() { + @Override + public double get() { + return DirectMemoryUtils.jvmMaxDirectMemory(); + } + }).register(registry); + + // metric to export pulsar version info + Gauge.build("pulsar_version_info", "-") + .labelNames("version", "commit").create() + .setChild(new Gauge.Child() { + @Override + public double get() { + return 1.0; + } + }, PulsarVersion.getVersion(), PulsarVersion.getGitSha()) + .register(registry); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java index 4e72fa0d72b16..110a8aa82f112 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java @@ -32,7 +32,10 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; -import org.apache.pulsar.common.policies.data.BacklogQuota; +import org.apache.pulsar.broker.service.persistent.PersistentTopicMetrics; +import org.apache.pulsar.broker.service.persistent.PersistentTopicMetrics.BacklogQuotaMetrics; +import org.apache.pulsar.broker.stats.prometheus.metrics.PrometheusLabels; +import org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType; import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; import org.apache.pulsar.common.policies.data.stats.NonPersistentSubscriptionStatsImpl; import org.apache.pulsar.common.policies.data.stats.NonPersistentTopicStatsImpl; @@ -80,7 +83,7 @@ public static void generate(PulsarService pulsar, boolean includeTopicMetrics, b Optional compactorMXBean = getCompactorMXBean(pulsar); LongAdder topicsCount = new LongAdder(); Map localNamespaceTopicCount = new HashMap<>(); - pulsar.getBrokerService().getMultiLayerTopicMap().forEach((namespace, bundlesMap) -> { + pulsar.getBrokerService().getMultiLayerTopicsMap().forEach((namespace, bundlesMap) -> { namespaceStats.reset(); topicsCount.reset(); @@ -131,6 +134,7 @@ private static void aggregateTopicStats(TopicStats stats, SubscriptionStatsImpl subsStats.msgOutCounter = subscriptionStats.msgOutCounter; subsStats.msgBacklog = subscriptionStats.msgBacklog; subsStats.msgDelayed = subscriptionStats.msgDelayed; + subsStats.msgInReplay = subscriptionStats.msgInReplay; subsStats.msgRateExpired = subscriptionStats.msgRateExpired; subsStats.totalMsgExpired = subscriptionStats.totalMsgExpired; subsStats.msgBacklogNoDelayed = subsStats.msgBacklog - subsStats.msgDelayed; @@ -159,14 +163,15 @@ private static void aggregateTopicStats(TopicStats stats, SubscriptionStatsImpl subsStats.bucketDelayedIndexStats = subscriptionStats.bucketDelayedIndexStats; } + @SuppressWarnings("OptionalUsedAsFieldOrParameterType") private static void getTopicStats(Topic topic, TopicStats stats, boolean includeConsumerMetrics, boolean includeProducerMetrics, boolean getPreciseBacklog, boolean subscriptionBacklogSize, Optional compactorMXBean) { stats.reset(); - if (topic instanceof PersistentTopic) { + if (topic instanceof PersistentTopic persistentTopic) { // Managed Ledger stats - ManagedLedger ml = ((PersistentTopic) topic).getManagedLedger(); + ManagedLedger ml = persistentTopic.getManagedLedger(); ManagedLedgerMBeanImpl mlStats = (ManagedLedgerMBeanImpl) ml.getStats(); stats.managedLedgerStats.storageSize = mlStats.getStoredMessagesSize(); @@ -174,9 +179,10 @@ private static void getTopicStats(Topic topic, TopicStats stats, boolean include stats.managedLedgerStats.backlogSize = ml.getEstimatedBacklogSize(); stats.managedLedgerStats.offloadedStorageUsed = ml.getOffloadedSize(); stats.backlogQuotaLimit = topic - .getBacklogQuota(BacklogQuota.BacklogQuotaType.destination_storage).getLimitSize(); + .getBacklogQuota(BacklogQuotaType.destination_storage).getLimitSize(); stats.backlogQuotaLimitTime = topic - .getBacklogQuota(BacklogQuota.BacklogQuotaType.message_age).getLimitTime(); + .getBacklogQuota(BacklogQuotaType.message_age).getLimitTime(); + stats.backlogAgeSeconds = topic.getBestEffortOldestUnacknowledgedMessageAgeSeconds(); stats.managedLedgerStats.storageWriteLatencyBuckets .addAll(mlStats.getInternalAddEntryLatencyBuckets()); @@ -191,11 +197,23 @@ private static void getTopicStats(Topic topic, TopicStats stats, boolean include stats.managedLedgerStats.storageWriteRate = mlStats.getAddEntryMessagesRate(); stats.managedLedgerStats.storageReadRate = mlStats.getReadEntriesRate(); stats.managedLedgerStats.storageReadCacheMissesRate = mlStats.getReadEntriesOpsCacheMissesRate(); + + // Topic Stats + PersistentTopicMetrics persistentTopicMetrics = persistentTopic.getPersistentTopicMetrics(); + + BacklogQuotaMetrics backlogQuotaMetrics = persistentTopicMetrics.getBacklogQuotaMetrics(); + stats.sizeBasedBacklogQuotaExceededEvictionCount = + backlogQuotaMetrics.getSizeBasedBacklogQuotaExceededEvictionCount(); + stats.timeBasedBacklogQuotaExceededEvictionCount = + backlogQuotaMetrics.getTimeBasedBacklogQuotaExceededEvictionCount(); } + TopicStatsImpl tStatus = topic.getStats(getPreciseBacklog, subscriptionBacklogSize, false); stats.msgInCounter = tStatus.msgInCounter; stats.bytesInCounter = tStatus.bytesInCounter; stats.msgOutCounter = tStatus.msgOutCounter; + stats.systemTopicBytesInCounter = tStatus.systemTopicBytesInCounter; + stats.bytesOutInternalCounter = tStatus.getBytesOutInternalCounter(); stats.bytesOutCounter = tStatus.bytesOutCounter; stats.averageMsgSize = tStatus.averageMsgSize; stats.publishRateLimitedTimes = tStatus.publishRateLimitedTimes; @@ -273,7 +291,7 @@ private static void getTopicStats(Topic topic, TopicStats stats, boolean include } topic.getReplicators().forEach((cluster, replicator) -> { - ReplicatorStatsImpl replStats = replicator.getStats(); + ReplicatorStatsImpl replStats = replicator.computeStats(); AggregatedReplicationStats aggReplStats = stats.replicationStats.get(replicator.getRemoteCluster()); if (aggReplStats == null) { aggReplStats = new AggregatedReplicationStats(); @@ -286,7 +304,11 @@ private static void getTopicStats(Topic topic, TopicStats stats, boolean include aggReplStats.msgThroughputOut += replStats.msgThroughputOut; aggReplStats.replicationBacklog += replStats.replicationBacklog; aggReplStats.msgRateExpired += replStats.msgRateExpired; - aggReplStats.connectedCount += replStats.connected ? 1 : 0; + if (replStats.connected) { + aggReplStats.connectedCount += 1; + } else { + aggReplStats.disconnectedCount += 1; + } aggReplStats.replicationDelayInSeconds += replStats.replicationDelayInSeconds; }); @@ -334,7 +356,25 @@ private static void printBrokerStats(PrometheusMetricStreams stream, String clus writeMetric(stream, "pulsar_broker_storage_read_rate", brokerStats.storageReadRate, cluster); writeMetric(stream, "pulsar_broker_storage_read_cache_misses_rate", brokerStats.storageReadCacheMissesRate, cluster); + + writePulsarBacklogQuotaMetricBrokerLevel(stream, + "pulsar_broker_storage_backlog_quota_exceeded_evictions_total", + brokerStats.sizeBasedBacklogQuotaExceededEvictionCount, cluster, BacklogQuotaType.destination_storage); + writePulsarBacklogQuotaMetricBrokerLevel(stream, + "pulsar_broker_storage_backlog_quota_exceeded_evictions_total", + brokerStats.timeBasedBacklogQuotaExceededEvictionCount, cluster, BacklogQuotaType.message_age); + writeMetric(stream, "pulsar_broker_msg_backlog", brokerStats.msgBacklog, cluster); + long userOutBytes = brokerStats.bytesOutCounter - brokerStats.bytesOutInternalCounter; + writeMetric(stream, "pulsar_broker_out_bytes_total", + userOutBytes, cluster, "system_subscription", "false"); + writeMetric(stream, "pulsar_broker_out_bytes_total", + brokerStats.bytesOutInternalCounter, cluster, "system_subscription", "true"); + long userTopicInBytes = brokerStats.bytesInCounter - brokerStats.systemTopicBytesInCounter; + writeMetric(stream, "pulsar_broker_in_bytes_total", + userTopicInBytes, cluster, "system_topic", "false"); + writeMetric(stream, "pulsar_broker_in_bytes_total", + brokerStats.systemTopicBytesInCounter, cluster, "system_topic", "true"); } private static void printTopicsCountStats(PrometheusMetricStreams stream, Map namespaceTopicsCount, @@ -372,6 +412,7 @@ private static void printNamespaceStats(PrometheusMetricStreams stream, Aggregat stats.managedLedgerStats.storageLogicalSize, cluster, namespace); writeMetric(stream, "pulsar_storage_backlog_size", stats.managedLedgerStats.backlogSize, cluster, namespace); + writeMetric(stream, "pulsar_storage_offloaded_size", stats.managedLedgerStats.offloadedStorageUsed, cluster, namespace); @@ -384,14 +425,25 @@ private static void printNamespaceStats(PrometheusMetricStreams stream, Aggregat writeMetric(stream, "pulsar_subscription_delayed", stats.msgDelayed, cluster, namespace); + writeMetric(stream, "pulsar_subscription_in_replay", stats.msgInReplay, cluster, namespace); + writeMetric(stream, "pulsar_delayed_message_index_size_bytes", stats.delayedMessageIndexSizeInBytes, cluster, namespace); stats.bucketDelayedIndexStats.forEach((k, metric) -> { - writeMetric(stream, metric.name, metric.value, cluster, namespace, metric.labelsAndValues); + String[] labels = ArrayUtils.addAll(new String[]{"namespace", namespace}, metric.labelsAndValues); + writeMetric(stream, metric.name, metric.value, cluster, labels); }); writePulsarMsgBacklog(stream, stats.msgBacklog, cluster, namespace); + writePulsarBacklogQuotaMetricNamespaceLevel(stream, + "pulsar_storage_backlog_quota_exceeded_evictions_total", + stats.sizeBasedBacklogQuotaExceededEvictionCount, cluster, namespace, + BacklogQuotaType.destination_storage); + writePulsarBacklogQuotaMetricNamespaceLevel(stream, + "pulsar_storage_backlog_quota_exceeded_evictions_total", + stats.timeBasedBacklogQuotaExceededEvictionCount, cluster, namespace, + BacklogQuotaType.message_age); stats.managedLedgerStats.storageWriteLatencyBuckets.refresh(); long[] latencyBuckets = stats.managedLedgerStats.storageWriteLatencyBuckets.getBuckets(); @@ -465,12 +517,33 @@ private static void printNamespaceStats(PrometheusMetricStreams stream, Aggregat replStats -> replStats.replicationBacklog, cluster, namespace); writeReplicationStat(stream, "pulsar_replication_connected_count", stats, replStats -> replStats.connectedCount, cluster, namespace); + writeReplicationStat(stream, "pulsar_replication_disconnected_count", stats, + replStats -> replStats.disconnectedCount, cluster, namespace); writeReplicationStat(stream, "pulsar_replication_rate_expired", stats, replStats -> replStats.msgRateExpired, cluster, namespace); writeReplicationStat(stream, "pulsar_replication_delay_in_seconds", stats, replStats -> replStats.replicationDelayInSeconds, cluster, namespace); } + @SuppressWarnings("SameParameterValue") + private static void writePulsarBacklogQuotaMetricBrokerLevel(PrometheusMetricStreams stream, String metricName, + Number value, String cluster, + BacklogQuotaType backlogQuotaType) { + String quotaTypeLabelValue = PrometheusLabels.backlogQuotaTypeLabel(backlogQuotaType); + stream.writeSample(metricName, value, "cluster", cluster, + "quota_type", quotaTypeLabelValue); + } + + @SuppressWarnings("SameParameterValue") + private static void writePulsarBacklogQuotaMetricNamespaceLevel(PrometheusMetricStreams stream, String metricName, + Number value, String cluster, String namespace, + BacklogQuotaType backlogQuotaType) { + String quotaTypeLabelValue = PrometheusLabels.backlogQuotaTypeLabel(backlogQuotaType); + stream.writeSample(metricName, value, "cluster", cluster, + "namespace", namespace, + "quota_type", quotaTypeLabelValue); + } + private static void writePulsarMsgBacklog(PrometheusMetricStreams stream, Number value, String cluster, String namespace) { stream.writeSample("pulsar_msg_backlog", value, "cluster", cluster, "namespace", namespace, @@ -483,13 +556,21 @@ private static void writeMetric(PrometheusMetricStreams stream, String metricNam stream.writeSample(metricName, value, "cluster", cluster); } + private static void writeMetric(PrometheusMetricStreams stream, String metricName, Number value, + String cluster, String... extraLabelsAndValues) { + String[] labels = ArrayUtils.addAll(new String[]{"cluster", cluster}, extraLabelsAndValues); + stream.writeSample(metricName, value, labels); + } + + private static void writeMetric(PrometheusMetricStreams stream, String metricName, Number value, String cluster, - String namespace, String... extraLabelsAndValues) { - String[] labelsAndValues = new String[]{"cluster", cluster, "namespace", namespace}; - String[] labels = ArrayUtils.addAll(labelsAndValues, extraLabelsAndValues); + String namespace) { + String[] labels = new String[]{"cluster", cluster, "namespace", namespace}; stream.writeSample(metricName, value, labels); } + + private static void writeReplicationStat(PrometheusMetricStreams stream, String metricName, AggregatedNamespaceStats namespaceStats, Function sampleValueFunction, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java index 93cbad4e19503..5a5a61404b87f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java @@ -42,7 +42,7 @@ void writeSample(String metricName, Number value, String... labelsAndValuesArray stream.write(metricName).write('{'); for (int i = 0; i < labelsAndValuesArray.length; i += 2) { String labelValue = labelsAndValuesArray[i + 1]; - if (labelValue != null) { + if (labelValue != null && labelValue.indexOf('"') > -1) { labelValue = labelValue.replace("\"", "\\\""); } stream.write(labelsAndValuesArray[i]).write("=\"").write(labelValue).write('\"'); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java index 501bfbbb16331..6b4d08c359d42 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGenerator.java @@ -20,41 +20,44 @@ import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGeneratorUtils.generateSystemMetrics; import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGeneratorUtils.getTypeStr; -import static org.apache.pulsar.common.stats.JvmMetrics.getJvmDirectMemoryUsed; import io.netty.buffer.ByteBuf; -import io.netty.buffer.UnpooledByteBufAllocator; +import io.netty.buffer.ByteBufAllocator; +import io.netty.buffer.CompositeByteBuf; +import io.netty.buffer.PooledByteBufAllocator; +import io.netty.buffer.Unpooled; import io.prometheus.client.Collector; -import io.prometheus.client.CollectorRegistry; -import io.prometheus.client.Gauge; -import io.prometheus.client.Gauge.Child; -import io.prometheus.client.hotspot.DefaultExports; +import java.io.BufferedOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.io.StringWriter; +import java.io.OutputStreamWriter; import java.io.Writer; import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.CharBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Clock; import java.util.Collection; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import java.util.zip.CRC32; +import java.util.zip.Deflater; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.stats.NullStatsProvider; import org.apache.bookkeeper.stats.StatsProvider; -import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.broker.PulsarService; -import org.apache.pulsar.broker.stats.TimeWindow; -import org.apache.pulsar.broker.stats.WindowWrap; import org.apache.pulsar.broker.stats.metrics.ManagedCursorMetrics; import org.apache.pulsar.broker.stats.metrics.ManagedLedgerCacheMetrics; import org.apache.pulsar.broker.stats.metrics.ManagedLedgerMetrics; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.stats.Metrics; -import org.apache.pulsar.common.util.DirectMemoryUtils; import org.apache.pulsar.common.util.SimpleTextOutputStream; -import org.eclipse.jetty.server.HttpOutput; /** * Generate metrics aggregated at the namespace level and optionally at a topic level and formats them out @@ -63,123 +66,254 @@ * href="https://prometheus.io/docs/instrumenting/exposition_formats/">Exposition Formats */ @Slf4j -public class PrometheusMetricsGenerator { - private static volatile TimeWindow timeWindow; - private static final int MAX_COMPONENTS = 64; +public class PrometheusMetricsGenerator implements AutoCloseable { + private static final int DEFAULT_INITIAL_BUFFER_SIZE = 1024 * 1024; // 1MB + private static final int MINIMUM_FOR_MAX_COMPONENTS = 64; + + private volatile MetricsBuffer metricsBuffer; + private static AtomicReferenceFieldUpdater metricsBufferFieldUpdater = + AtomicReferenceFieldUpdater.newUpdater(PrometheusMetricsGenerator.class, MetricsBuffer.class, + "metricsBuffer"); + private volatile boolean closed; + + public static class MetricsBuffer { + private final CompletableFuture bufferFuture; + private final long createTimeslot; + private final AtomicInteger refCnt = new AtomicInteger(2); + + MetricsBuffer(long timeslot) { + bufferFuture = new CompletableFuture<>(); + createTimeslot = timeslot; + } - static { - DefaultExports.initialize(); + public CompletableFuture getBufferFuture() { + return bufferFuture; + } - Gauge.build("jvm_memory_direct_bytes_used", "-").create().setChild(new Child() { - @Override - public double get() { - return getJvmDirectMemoryUsed(); - } - }).register(CollectorRegistry.defaultRegistry); + long getCreateTimeslot() { + return createTimeslot; + } - Gauge.build("jvm_memory_direct_bytes_max", "-").create().setChild(new Child() { - @Override - public double get() { - return DirectMemoryUtils.jvmMaxDirectMemory(); - } - }).register(CollectorRegistry.defaultRegistry); - - // metric to export pulsar version info - Gauge.build("pulsar_version_info", "-") - .labelNames("version", "commit").create() - .setChild(new Child() { - @Override - public double get() { - return 1.0; + /** + * Retain the buffer. This is allowed, only when the buffer is not already released. + * + * @return true if the buffer is retained successfully, false otherwise. + */ + boolean retain() { + return refCnt.updateAndGet(x -> x > 0 ? x + 1 : x) > 0; + } + + /** + * Release the buffer. + */ + public void release() { + int newValue = refCnt.decrementAndGet(); + if (newValue == 0) { + bufferFuture.whenComplete((byteBuf, throwable) -> { + if (byteBuf != null) { + byteBuf.release(); } - }, PulsarVersion.getVersion(), PulsarVersion.getGitSha()) - .register(CollectorRegistry.defaultRegistry); + }); + } + } } - public static void generate(PulsarService pulsar, boolean includeTopicMetrics, boolean includeConsumerMetrics, - boolean includeProducerMetrics, OutputStream out) throws IOException { - generate(pulsar, includeTopicMetrics, includeConsumerMetrics, includeProducerMetrics, false, out, null); - } + /** + * A wraps the response buffer and asynchronously provides a gzip compressed buffer when requested. + */ + public static class ResponseBuffer { + private final ByteBuf uncompressedBuffer; + private boolean released = false; + private CompletableFuture compressedBuffer; - public static void generate(PulsarService pulsar, boolean includeTopicMetrics, boolean includeConsumerMetrics, - boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel, - OutputStream out) throws IOException { - generate(pulsar, includeTopicMetrics, includeConsumerMetrics, includeProducerMetrics, - splitTopicAndPartitionIndexLabel, out, null); - } + private ResponseBuffer(final ByteBuf uncompressedBuffer) { + this.uncompressedBuffer = uncompressedBuffer; + } - public static synchronized void generate(PulsarService pulsar, boolean includeTopicMetrics, - boolean includeConsumerMetrics, boolean includeProducerMetrics, - boolean splitTopicAndPartitionIndexLabel, OutputStream out, - List metricsProviders) throws IOException { - ByteBuf buffer; - boolean exposeBufferMetrics = pulsar.getConfiguration().isMetricsBufferResponse(); + public ByteBuf getUncompressedBuffer() { + return uncompressedBuffer; + } - if (!exposeBufferMetrics) { - buffer = generate0(pulsar, includeTopicMetrics, includeConsumerMetrics, includeProducerMetrics, - splitTopicAndPartitionIndexLabel, metricsProviders); - } else { - if (null == timeWindow) { - int period = pulsar.getConfiguration().getManagedLedgerStatsPeriodSeconds(); - timeWindow = new TimeWindow<>(1, (int) TimeUnit.SECONDS.toMillis(period)); + public synchronized CompletableFuture getCompressedBuffer(Executor executor) { + if (released) { + throw new IllegalStateException("Already released!"); } - WindowWrap window = timeWindow.current(oldBuf -> { - // release expired buffer, in case of memory leak - if (oldBuf != null && oldBuf.refCnt() > 0) { - oldBuf.release(); - log.debug("Cached metrics buffer released"); - } + if (compressedBuffer == null) { + compressedBuffer = new CompletableFuture<>(); + ByteBuf retainedDuplicate = uncompressedBuffer.retainedDuplicate(); + executor.execute(() -> { + try { + compressedBuffer.complete(compress(retainedDuplicate)); + } catch (Exception e) { + compressedBuffer.completeExceptionally(e); + } finally { + retainedDuplicate.release(); + } + }); + } + return compressedBuffer; + } - try { - ByteBuf buf = generate0(pulsar, includeTopicMetrics, includeConsumerMetrics, includeProducerMetrics, - splitTopicAndPartitionIndexLabel, metricsProviders); - log.debug("Generated metrics buffer size {}", buf.readableBytes()); - return buf; - } catch (IOException e) { - log.error("Generate metrics failed", e); - //return empty buffer if exception happens - return PulsarByteBufAllocator.DEFAULT.heapBuffer(0); - } - }); + private ByteBuf compress(ByteBuf uncompressedBuffer) { + GzipByteBufferWriter gzipByteBufferWriter = new GzipByteBufferWriter(uncompressedBuffer.alloc(), + uncompressedBuffer.readableBytes()); + return gzipByteBufferWriter.compress(uncompressedBuffer); + } - if (null == window || null == window.value()) { - return; + public synchronized void release() { + released = true; + uncompressedBuffer.release(); + if (compressedBuffer != null) { + compressedBuffer.whenComplete((byteBuf, throwable) -> { + if (byteBuf != null) { + byteBuf.release(); + } + }); } - buffer = window.value(); - log.debug("Current window start {}, current cached buf size {}", window.start(), buffer.readableBytes()); } + } - try { - if (out instanceof HttpOutput) { - HttpOutput output = (HttpOutput) out; - //no mem_copy and memory allocations here - ByteBuffer[] buffers = buffer.nioBuffers(); - for (ByteBuffer buffer0 : buffers) { - output.write(buffer0); + /** + * Compress input nio buffers into gzip format with output in a Netty composite ByteBuf. + */ + private static class GzipByteBufferWriter { + private static final byte[] GZIP_HEADER = + new byte[] {(byte) 0x1f, (byte) 0x8b, Deflater.DEFLATED, 0, 0, 0, 0, 0, 0, 0}; + private final ByteBufAllocator bufAllocator; + private final Deflater deflater; + private final CRC32 crc; + private final int bufferSize; + private final CompositeByteBuf resultBuffer; + private ByteBuf backingCompressBuffer; + private ByteBuffer compressBuffer; + + GzipByteBufferWriter(ByteBufAllocator bufAllocator, int readableBytes) { + deflater = new Deflater(Deflater.DEFAULT_COMPRESSION, true); + crc = new CRC32(); + this.bufferSize = Math.max(Math.min(resolveChunkSize(bufAllocator), readableBytes), 8192); + this.bufAllocator = bufAllocator; + this.resultBuffer = bufAllocator.compositeDirectBuffer(readableBytes / bufferSize + 2); + allocateCompressBuffer(); + } + + /** + * Compress the input Netty buffer and append it to the result buffer in gzip format. + * @param uncompressedBuffer + */ + public ByteBuf compress(ByteBuf uncompressedBuffer) { + try { + ByteBuffer[] nioBuffers = uncompressedBuffer.nioBuffers(); + for (int i = 0, nioBuffersLength = nioBuffers.length; i < nioBuffersLength; i++) { + ByteBuffer nioBuffer = nioBuffers[i]; + compressAndAppend(nioBuffer, i == 0, i == nioBuffersLength - 1); } - } else { - //read data from buffer and write it to output stream, with no more heap buffer(byte[]) allocation. - //not modify buffer readIndex/writeIndex here. - int readIndex = buffer.readerIndex(); - int readableBytes = buffer.readableBytes(); - for (int i = 0; i < readableBytes; i++) { - out.write(buffer.getByte(readIndex + i)); + return resultBuffer; + } finally { + close(); + } + } + + private void compressAndAppend(ByteBuffer nioBuffer, boolean isFirst, boolean isLast) { + if (isFirst) { + // write gzip header + compressBuffer.put(GZIP_HEADER); + } + // update the CRC32 checksum calculation + nioBuffer.mark(); + crc.update(nioBuffer); + nioBuffer.reset(); + // pass the input buffer to the deflater + deflater.setInput(nioBuffer); + // when the input buffer is the last one, set the flag to finish the deflater + if (isLast) { + deflater.finish(); + } + int written = -1; + // the deflater may need multiple calls to deflate the input buffer + // the completion is checked by the deflater.needsInput() method for buffers that aren't the last buffer + // for the last buffer, the completion is checked by the deflater.finished() method + while (!isLast && !deflater.needsInput() || isLast && !deflater.finished()) { + // when the previous deflater.deflate call returns 0 (and needsInput/finished returns false), + // it means that the output buffer is full. + // append the compressed buffer to the result buffer and allocate a new buffer. + if (written == 0) { + if (compressBuffer.position() > 0) { + appendCompressBufferToResultBuffer(); + allocateCompressBuffer(); + } else { + // this is an unexpected case, throw an exception to prevent an infinite loop + throw new IllegalStateException( + "Deflater didn't write any bytes while the compress buffer is empty."); + } } + written = deflater.deflate(compressBuffer); } - } finally { - if (!exposeBufferMetrics && buffer.refCnt() > 0) { - buffer.release(); - log.debug("Metrics buffer released."); + if (isLast) { + // append the last compressed buffer when it is not empty + if (compressBuffer.position() > 0) { + appendCompressBufferToResultBuffer(); + } else { + // release an unused empty buffer + backingCompressBuffer.release(); + } + backingCompressBuffer = null; + compressBuffer = null; + + // write gzip trailer, 2 integers (CRC32 checksum and uncompressed size) + ByteBuffer trailerBuf = ByteBuffer.allocate(2 * Integer.BYTES); + // integer values are in little endian byte order + trailerBuf.order(ByteOrder.LITTLE_ENDIAN); + // write CRC32 checksum + trailerBuf.putInt((int) crc.getValue()); + // write uncompressed size + trailerBuf.putInt(deflater.getTotalIn()); + trailerBuf.flip(); + resultBuffer.addComponent(true, Unpooled.wrappedBuffer(trailerBuf)); } } + + private void appendCompressBufferToResultBuffer() { + backingCompressBuffer.setIndex(0, compressBuffer.position()); + resultBuffer.addComponent(true, backingCompressBuffer); + } + + private void allocateCompressBuffer() { + backingCompressBuffer = bufAllocator.directBuffer(bufferSize); + compressBuffer = backingCompressBuffer.nioBuffer(0, bufferSize); + } + + private void close() { + if (deflater != null) { + deflater.end(); + } + if (backingCompressBuffer != null) { + backingCompressBuffer.release(); + } + } + } + + private final PulsarService pulsar; + private final boolean includeTopicMetrics; + private final boolean includeConsumerMetrics; + private final boolean includeProducerMetrics; + private final boolean splitTopicAndPartitionIndexLabel; + private final Clock clock; + + private volatile int initialBufferSize = DEFAULT_INITIAL_BUFFER_SIZE; + + public PrometheusMetricsGenerator(PulsarService pulsar, boolean includeTopicMetrics, + boolean includeConsumerMetrics, boolean includeProducerMetrics, + boolean splitTopicAndPartitionIndexLabel, Clock clock) { + this.pulsar = pulsar; + this.includeTopicMetrics = includeTopicMetrics; + this.includeConsumerMetrics = includeConsumerMetrics; + this.includeProducerMetrics = includeProducerMetrics; + this.splitTopicAndPartitionIndexLabel = splitTopicAndPartitionIndexLabel; + this.clock = clock; } - private static ByteBuf generate0(PulsarService pulsar, boolean includeTopicMetrics, boolean includeConsumerMetrics, - boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel, - List metricsProviders) throws IOException { - //Use unpooled buffers here to avoid direct buffer usage increasing. - //when write out 200MB data, MAX_COMPONENTS = 64 needn't mem_copy. see: CompositeByteBuf#consolidateIfNeeded() - ByteBuf buf = UnpooledByteBufAllocator.DEFAULT.compositeDirectBuffer(MAX_COMPONENTS); + protected ByteBuf generateMetrics(List metricsProviders) { + ByteBuf buf = allocateMultipartCompositeDirectBuffer(); boolean exceptionHappens = false; //Used in namespace/topic and transaction aggregators as share metric names PrometheusMetricStreams metricStreams = new PrometheusMetricStreams(); @@ -221,10 +355,41 @@ private static ByteBuf generate0(PulsarService pulsar, boolean includeTopicMetri //if exception happens, release buffer if (exceptionHappens) { buf.release(); + } else { + // for the next time, the initial buffer size will be suggested by the last buffer size + initialBufferSize = Math.max(DEFAULT_INITIAL_BUFFER_SIZE, buf.readableBytes()); } } } + private ByteBuf allocateMultipartCompositeDirectBuffer() { + // use composite buffer with pre-allocated buffers to ensure that the pooled allocator can be used + // for allocating the buffers + ByteBufAllocator byteBufAllocator = PulsarByteBufAllocator.DEFAULT; + int chunkSize = resolveChunkSize(byteBufAllocator); + CompositeByteBuf buf = byteBufAllocator.compositeDirectBuffer( + Math.max(MINIMUM_FOR_MAX_COMPONENTS, (initialBufferSize / chunkSize) + 1)); + int totalLen = 0; + while (totalLen < initialBufferSize) { + totalLen += chunkSize; + // increase the capacity in increments of chunkSize to preallocate the buffers + // in the composite buffer + buf.capacity(totalLen); + } + return buf; + } + + private static int resolveChunkSize(ByteBufAllocator byteBufAllocator) { + int chunkSize; + if (byteBufAllocator instanceof PooledByteBufAllocator) { + PooledByteBufAllocator pooledByteBufAllocator = (PooledByteBufAllocator) byteBufAllocator; + chunkSize = Math.max(pooledByteBufAllocator.metric().chunkSize(), DEFAULT_INITIAL_BUFFER_SIZE); + } else { + chunkSize = DEFAULT_INITIAL_BUFFER_SIZE; + } + return chunkSize; + } + private static void generateBrokerBasicMetrics(PulsarService pulsar, SimpleTextOutputStream stream) { String clusterName = pulsar.getConfiguration().getClusterName(); // generate managedLedgerCache metrics @@ -243,8 +408,8 @@ private static void generateBrokerBasicMetrics(PulsarService pulsar, SimpleTextO clusterName, Collector.Type.GAUGE, stream); } - parseMetricsToPrometheusMetrics(Collections.singletonList(pulsar.getBrokerService() - .getPulsarStats().getBrokerOperabilityMetrics().generateConnectionMetrics()), + parseMetricsToPrometheusMetrics(pulsar.getBrokerService() + .getPulsarStats().getBrokerOperabilityMetrics().getMetrics(), clusterName, Collector.Type.GAUGE, stream); // generate loadBalance metrics @@ -270,12 +435,13 @@ private static void parseMetricsToPrometheusMetrics(Collection metrics, String name = key.substring(0, nameIndex); value = key.substring(nameIndex + 1); if (!names.contains(name)) { - stream.write("# TYPE ").write(name.replace("brk_", "pulsar_")).write(' ') - .write(getTypeStr(metricType)).write("\n"); + stream.write("# TYPE "); + writeNameReplacingBrkPrefix(stream, name); + stream.write(' ').write(getTypeStr(metricType)).write("\n"); names.add(name); } - stream.write(name.replace("brk_", "pulsar_")) - .write("{cluster=\"").write(cluster).write('"'); + writeNameReplacingBrkPrefix(stream, name); + stream.write("{cluster=\"").write(cluster).write('"'); } catch (Exception e) { continue; } @@ -284,12 +450,13 @@ private static void parseMetricsToPrometheusMetrics(Collection metrics, String name = entry.getKey(); if (!names.contains(name)) { - stream.write("# TYPE ").write(entry.getKey().replace("brk_", "pulsar_")).write(' ') - .write(getTypeStr(metricType)).write('\n'); + stream.write("# TYPE "); + writeNameReplacingBrkPrefix(stream, entry.getKey()); + stream.write(' ').write(getTypeStr(metricType)).write('\n'); names.add(name); } - stream.write(name.replace("brk_", "pulsar_")) - .write("{cluster=\"").write(cluster).write('"'); + writeNameReplacingBrkPrefix(stream, name); + stream.write("{cluster=\"").write(cluster).write('"'); } //to avoid quantile label duplicated @@ -309,18 +476,98 @@ private static void parseMetricsToPrometheusMetrics(Collection metrics, } } + private static SimpleTextOutputStream writeNameReplacingBrkPrefix(SimpleTextOutputStream stream, String name) { + if (name.startsWith("brk_")) { + return stream.write("pulsar_").write(CharBuffer.wrap(name).position("brk_".length())); + } else { + return stream.write(name); + } + } + private static void generateManagedLedgerBookieClientMetrics(PulsarService pulsar, SimpleTextOutputStream stream) { StatsProvider statsProvider = pulsar.getManagedLedgerClientFactory().getStatsProvider(); if (statsProvider instanceof NullStatsProvider) { return; } - try { - Writer writer = new StringWriter(); + try (Writer writer = new OutputStreamWriter(new BufferedOutputStream(new OutputStream() { + @Override + public void write(int b) throws IOException { + stream.writeByte(b); + } + + @Override + public void write(byte b[], int off, int len) throws IOException { + stream.write(b, off, len); + } + }), StandardCharsets.UTF_8)) { statsProvider.writeAllMetrics(writer); - stream.write(writer.toString()); } catch (IOException e) { - // nop + log.error("Failed to write managed ledger bookie client metrics", e); + } + } + + public MetricsBuffer renderToBuffer(Executor executor, List metricsProviders) { + boolean cacheMetricsResponse = pulsar.getConfiguration().isMetricsBufferResponse(); + while (!closed && !Thread.currentThread().isInterrupted()) { + long currentTimeSlot = cacheMetricsResponse ? calculateCurrentTimeSlot() : 0; + MetricsBuffer currentMetricsBuffer = metricsBuffer; + if (currentMetricsBuffer == null || currentMetricsBuffer.getBufferFuture().isCompletedExceptionally() + || (currentMetricsBuffer.getBufferFuture().isDone() + && (currentMetricsBuffer.getCreateTimeslot() != 0 + && currentTimeSlot > currentMetricsBuffer.getCreateTimeslot()))) { + MetricsBuffer newMetricsBuffer = new MetricsBuffer(currentTimeSlot); + if (metricsBufferFieldUpdater.compareAndSet(this, currentMetricsBuffer, newMetricsBuffer)) { + if (currentMetricsBuffer != null) { + currentMetricsBuffer.release(); + } + CompletableFuture bufferFuture = newMetricsBuffer.getBufferFuture(); + executor.execute(() -> { + try { + bufferFuture.complete(new ResponseBuffer(generateMetrics(metricsProviders))); + } catch (Exception e) { + bufferFuture.completeExceptionally(e); + } finally { + if (currentTimeSlot == 0) { + // if the buffer is not cached, release it after the future is completed + metricsBufferFieldUpdater.compareAndSet(this, newMetricsBuffer, null); + newMetricsBuffer.release(); + } + } + }); + // no need to retain before returning since the new buffer starts with refCnt 2 + return newMetricsBuffer; + } else { + currentMetricsBuffer = metricsBuffer; + } + } + // retain the buffer before returning + // if the buffer is already released, retaining won't succeed, retry in that case + if (currentMetricsBuffer != null && currentMetricsBuffer.retain()) { + return currentMetricsBuffer; + } + } + return null; + } + + /** + * Calculate the current time slot based on the current time. + * This is to ensure that cached metrics are refreshed consistently at a fixed interval regardless of the request + * time. + */ + private long calculateCurrentTimeSlot() { + long cacheTimeoutMillis = + TimeUnit.SECONDS.toMillis(Math.max(1, pulsar.getConfiguration().getManagedLedgerStatsPeriodSeconds())); + long now = clock.millis(); + return now / cacheTimeoutMillis; + } + + @Override + public void close() { + closed = true; + MetricsBuffer buffer = metricsBufferFieldUpdater.getAndSet(this, null); + if (buffer != null) { + buffer.release(); } } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PulsarPrometheusMetricsServlet.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PulsarPrometheusMetricsServlet.java index 42bd2652883b6..43514d481dcab 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PulsarPrometheusMetricsServlet.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PulsarPrometheusMetricsServlet.java @@ -18,34 +18,168 @@ */ package org.apache.pulsar.broker.stats.prometheus; +import static org.apache.pulsar.broker.web.GzipHandlerUtil.isGzipCompressionEnabledForEndpoint; +import java.io.EOFException; import java.io.IOException; +import java.nio.ByteBuffer; +import java.time.Clock; +import java.util.Arrays; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.servlet.AsyncContext; +import javax.servlet.AsyncEvent; +import javax.servlet.AsyncListener; import javax.servlet.ServletOutputStream; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.PulsarService; +import org.eclipse.jetty.server.HttpOutput; +@Slf4j public class PulsarPrometheusMetricsServlet extends PrometheusMetricsServlet { - private static final long serialVersionUID = 1L; + private static final int EXECUTOR_MAX_THREADS = 4; - private final PulsarService pulsar; - private final boolean shouldExportTopicMetrics; - private final boolean shouldExportConsumerMetrics; - private final boolean shouldExportProducerMetrics; - private final boolean splitTopicAndPartitionLabel; + private final PrometheusMetricsGenerator prometheusMetricsGenerator; + private final boolean gzipCompressionEnabledForMetrics; public PulsarPrometheusMetricsServlet(PulsarService pulsar, boolean includeTopicMetrics, - boolean includeConsumerMetrics, boolean shouldExportProducerMetrics, + boolean includeConsumerMetrics, boolean includeProducerMetrics, boolean splitTopicAndPartitionLabel) { - super(pulsar.getConfiguration().getMetricsServletTimeoutMs(), pulsar.getConfiguration().getClusterName()); - this.pulsar = pulsar; - this.shouldExportTopicMetrics = includeTopicMetrics; - this.shouldExportConsumerMetrics = includeConsumerMetrics; - this.shouldExportProducerMetrics = shouldExportProducerMetrics; - this.splitTopicAndPartitionLabel = splitTopicAndPartitionLabel; + super(pulsar.getConfiguration().getMetricsServletTimeoutMs(), pulsar.getConfiguration().getClusterName(), + EXECUTOR_MAX_THREADS); + MetricsExports.initialize(); + prometheusMetricsGenerator = + new PrometheusMetricsGenerator(pulsar, includeTopicMetrics, includeConsumerMetrics, + includeProducerMetrics, splitTopicAndPartitionLabel, Clock.systemUTC()); + gzipCompressionEnabledForMetrics = isGzipCompressionEnabledForEndpoint( + pulsar.getConfiguration().getHttpServerGzipCompressionExcludedPaths(), DEFAULT_METRICS_PATH); } + @Override - protected void generateMetrics(String cluster, ServletOutputStream outputStream) throws IOException { - PrometheusMetricsGenerator.generate(pulsar, shouldExportTopicMetrics, shouldExportConsumerMetrics, - shouldExportProducerMetrics, splitTopicAndPartitionLabel, outputStream, metricsProviders); + public void destroy() { + super.destroy(); + prometheusMetricsGenerator.close(); + } + + protected void doGet(HttpServletRequest request, HttpServletResponse response) { + AsyncContext context = request.startAsync(); + // set hard timeout to 2 * timeout + if (metricsServletTimeoutMs > 0) { + context.setTimeout(metricsServletTimeoutMs * 2); + } + long startNanos = System.nanoTime(); + AtomicBoolean skipWritingResponse = new AtomicBoolean(false); + context.addListener(new AsyncListener() { + @Override + public void onComplete(AsyncEvent event) throws IOException { + } + + @Override + public void onTimeout(AsyncEvent event) throws IOException { + log.warn("Prometheus metrics request timed out"); + skipWritingResponse.set(true); + HttpServletResponse res = (HttpServletResponse) context.getResponse(); + if (!res.isCommitted()) { + res.setStatus(HTTP_STATUS_INTERNAL_SERVER_ERROR_500); + } + context.complete(); + } + + @Override + public void onError(AsyncEvent event) throws IOException { + skipWritingResponse.set(true); + } + + @Override + public void onStartAsync(AsyncEvent event) throws IOException { + } + }); + PrometheusMetricsGenerator.MetricsBuffer metricsBuffer = + prometheusMetricsGenerator.renderToBuffer(executor, metricsProviders); + if (metricsBuffer == null) { + log.info("Service is closing, skip writing metrics."); + response.setStatus(HTTP_STATUS_INTERNAL_SERVER_ERROR_500); + context.complete(); + return; + } + boolean compressOutput = gzipCompressionEnabledForMetrics && isGzipAccepted(request); + metricsBuffer.getBufferFuture().thenCompose(responseBuffer -> { + if (compressOutput) { + return responseBuffer.getCompressedBuffer(executor); + } else { + return CompletableFuture.completedFuture(responseBuffer.getUncompressedBuffer()); + } + }).whenComplete((buffer, ex) -> executor.execute(() -> { + try { + long elapsedNanos = System.nanoTime() - startNanos; + // check if the request has been timed out, implement a soft timeout + // so that response writing can continue to up to 2 * timeout + if (metricsServletTimeoutMs > 0 && elapsedNanos > TimeUnit.MILLISECONDS.toNanos( + metricsServletTimeoutMs)) { + log.warn("Prometheus metrics request was too long in queue ({}ms). Skipping sending metrics.", + TimeUnit.NANOSECONDS.toMillis(elapsedNanos)); + if (!response.isCommitted() && !skipWritingResponse.get()) { + response.setStatus(HTTP_STATUS_INTERNAL_SERVER_ERROR_500); + } + return; + } + if (skipWritingResponse.get()) { + log.warn("Response has timed or failed, skip writing metrics."); + return; + } + if (response.isCommitted()) { + log.warn("Response is already committed, cannot write metrics"); + return; + } + if (ex != null) { + log.error("Failed to generate metrics", ex); + response.setStatus(HTTP_STATUS_INTERNAL_SERVER_ERROR_500); + return; + } + if (buffer == null) { + log.error("Failed to generate metrics, buffer is null"); + response.setStatus(HTTP_STATUS_INTERNAL_SERVER_ERROR_500); + } else { + response.setStatus(HTTP_STATUS_OK_200); + response.setContentType("text/plain;charset=utf-8"); + if (compressOutput) { + response.setHeader("Content-Encoding", "gzip"); + } + ServletOutputStream outputStream = response.getOutputStream(); + if (outputStream instanceof HttpOutput) { + HttpOutput output = (HttpOutput) outputStream; + for (ByteBuffer nioBuffer : buffer.nioBuffers()) { + output.write(nioBuffer); + } + } else { + int length = buffer.readableBytes(); + if (length > 0) { + buffer.duplicate().readBytes(outputStream, length); + } + } + } + } catch (EOFException e) { + log.error("Failed to write metrics to response due to EOFException"); + } catch (IOException e) { + log.error("Failed to write metrics to response", e); + } finally { + metricsBuffer.release(); + context.complete(); + } + })); + } + + private boolean isGzipAccepted(HttpServletRequest request) { + String acceptEncoding = request.getHeader("Accept-Encoding"); + if (acceptEncoding != null) { + return Arrays.stream(acceptEncoding.split(",")) + .map(String::trim) + .anyMatch(str -> "gzip".equalsIgnoreCase(str)); + } + return false; } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java index dda03e3e59dd4..e54a3710e1294 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java @@ -25,26 +25,47 @@ import org.apache.bookkeeper.mledger.util.StatsBuckets; import org.apache.commons.lang3.ArrayUtils; import org.apache.pulsar.broker.service.Consumer; +import org.apache.pulsar.broker.stats.OpenTelemetryTopicStats; +import org.apache.pulsar.broker.stats.prometheus.metrics.PrometheusLabels; +import org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType; import org.apache.pulsar.common.policies.data.stats.TopicMetricBean; import org.apache.pulsar.compaction.CompactionRecord; import org.apache.pulsar.compaction.CompactorMXBean; +import org.apache.pulsar.opentelemetry.annotations.PulsarDeprecatedMetric; class TopicStats { + @PulsarDeprecatedMetric(newMetricName = OpenTelemetryTopicStats.SUBSCRIPTION_COUNTER) int subscriptionsCount; + @PulsarDeprecatedMetric(newMetricName = OpenTelemetryTopicStats.PRODUCER_COUNTER) int producersCount; + @PulsarDeprecatedMetric(newMetricName = OpenTelemetryTopicStats.CONSUMER_COUNTER) int consumersCount; + @PulsarDeprecatedMetric(newMetricName = OpenTelemetryTopicStats.MESSAGE_IN_COUNTER) double rateIn; + @PulsarDeprecatedMetric(newMetricName = OpenTelemetryTopicStats.MESSAGE_OUT_COUNTER) double rateOut; + @PulsarDeprecatedMetric(newMetricName = OpenTelemetryTopicStats.BYTES_IN_COUNTER) double throughputIn; + @PulsarDeprecatedMetric(newMetricName = OpenTelemetryTopicStats.BYTES_OUT_COUNTER) double throughputOut; + @PulsarDeprecatedMetric(newMetricName = OpenTelemetryTopicStats.MESSAGE_IN_COUNTER) long msgInCounter; + @PulsarDeprecatedMetric(newMetricName = OpenTelemetryTopicStats.BYTES_IN_COUNTER) long bytesInCounter; + @PulsarDeprecatedMetric(newMetricName = OpenTelemetryTopicStats.MESSAGE_OUT_COUNTER) long msgOutCounter; + @PulsarDeprecatedMetric(newMetricName = OpenTelemetryTopicStats.BYTES_OUT_COUNTER) long bytesOutCounter; + long systemTopicBytesInCounter; + long bytesOutInternalCounter; + @PulsarDeprecatedMetric // Can be derived from MESSAGE_IN_COUNTER and BYTES_IN_COUNTER double averageMsgSize; + @PulsarDeprecatedMetric(newMetricName = OpenTelemetryTopicStats.TRANSACTION_COUNTER) long ongoingTxnCount; + @PulsarDeprecatedMetric(newMetricName = OpenTelemetryTopicStats.TRANSACTION_COUNTER) long abortedTxnCount; + @PulsarDeprecatedMetric(newMetricName = OpenTelemetryTopicStats.TRANSACTION_COUNTER) long committedTxnCount; public long msgBacklog; @@ -52,6 +73,7 @@ class TopicStats { long backlogQuotaLimit; long backlogQuotaLimitTime; + long backlogAgeSeconds; ManagedLedgerStats managedLedgerStats = new ManagedLedgerStats(); @@ -73,6 +95,11 @@ class TopicStats { Map bucketDelayedIndexStats = new HashMap<>(); + public long sizeBasedBacklogQuotaExceededEvictionCount; + public long timeBasedBacklogQuotaExceededEvictionCount; + + + @SuppressWarnings("DuplicatedCode") public void reset() { subscriptionsCount = 0; producersCount = 0; @@ -111,8 +138,13 @@ public void reset() { compactionLatencyBuckets.reset(); delayedMessageIndexSizeInBytes = 0; bucketDelayedIndexStats.clear(); + + timeBasedBacklogQuotaExceededEvictionCount = 0; + sizeBasedBacklogQuotaExceededEvictionCount = 0; + backlogAgeSeconds = -1; } + @SuppressWarnings("OptionalUsedAsFieldOrParameterType") public static void printTopicStats(PrometheusMetricStreams stream, TopicStats stats, Optional compactorMXBean, String cluster, String namespace, String topic, boolean splitTopicAndPartitionIndexLabel) { @@ -165,6 +197,14 @@ public static void printTopicStats(PrometheusMetricStreams stream, TopicStats st cluster, namespace, topic, splitTopicAndPartitionIndexLabel); writeMetric(stream, "pulsar_storage_backlog_quota_limit_time", stats.backlogQuotaLimitTime, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_storage_backlog_age_seconds", stats.backlogAgeSeconds, + cluster, namespace, topic, splitTopicAndPartitionIndexLabel); + writeBacklogQuotaMetric(stream, "pulsar_storage_backlog_quota_exceeded_evictions_total", + stats.sizeBasedBacklogQuotaExceededEvictionCount, cluster, namespace, topic, + splitTopicAndPartitionIndexLabel, BacklogQuotaType.destination_storage); + writeBacklogQuotaMetric(stream, "pulsar_storage_backlog_quota_exceeded_evictions_total", + stats.timeBasedBacklogQuotaExceededEvictionCount, cluster, namespace, topic, + splitTopicAndPartitionIndexLabel, BacklogQuotaType.message_age); writeMetric(stream, "pulsar_delayed_message_index_size_bytes", stats.delayedMessageIndexSizeInBytes, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); @@ -270,6 +310,8 @@ public static void printTopicStats(PrometheusMetricStreams stream, TopicStats st subsStats.msgBacklogNoDelayed, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); writeSubscriptionMetric(stream, "pulsar_subscription_delayed", subsStats.msgDelayed, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + writeSubscriptionMetric(stream, "pulsar_subscription_in_replay", + subsStats.msgInReplay, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); writeSubscriptionMetric(stream, "pulsar_subscription_msg_rate_redeliver", subsStats.msgRateRedeliver, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); writeSubscriptionMetric(stream, "pulsar_subscription_unacked_messages", @@ -368,6 +410,8 @@ public static void printTopicStats(PrometheusMetricStreams stream, TopicStats st cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); writeMetric(stream, "pulsar_replication_connected_count", replStats.connectedCount, cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); + writeMetric(stream, "pulsar_replication_disconnected_count", replStats.disconnectedCount, + cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); writeMetric(stream, "pulsar_replication_rate_expired", replStats.msgRateExpired, cluster, namespace, topic, remoteCluster, splitTopicAndPartitionIndexLabel); writeMetric(stream, "pulsar_replication_delay_in_seconds", replStats.replicationDelayInSeconds, @@ -442,6 +486,17 @@ private static void writeMetric(PrometheusMetricStreams stream, String metricNam writeTopicMetric(stream, metricName, value, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); } + @SuppressWarnings("SameParameterValue") + private static void writeBacklogQuotaMetric(PrometheusMetricStreams stream, String metricName, Number value, + String cluster, String namespace, String topic, + boolean splitTopicAndPartitionIndexLabel, + BacklogQuotaType backlogQuotaType) { + + String quotaTypeLabelValue = PrometheusLabels.backlogQuotaTypeLabel(backlogQuotaType); + writeTopicMetric(stream, metricName, value, cluster, namespace, topic, splitTopicAndPartitionIndexLabel, + "quota_type", quotaTypeLabelValue); + } + private static void writeMetric(PrometheusMetricStreams stream, String metricName, Number value, String cluster, String namespace, String topic, String remoteCluster, boolean splitTopicAndPartitionIndexLabel) { @@ -475,7 +530,9 @@ private static void writeConsumerMetric(PrometheusMetricStreams stream, String m static void writeTopicMetric(PrometheusMetricStreams stream, String metricName, Number value, String cluster, String namespace, String topic, boolean splitTopicAndPartitionIndexLabel, String... extraLabelsAndValues) { - String[] labelsAndValues = new String[splitTopicAndPartitionIndexLabel ? 8 : 6]; + int baseLabelCount = splitTopicAndPartitionIndexLabel ? 8 : 6; + String[] labelsAndValues = + new String[baseLabelCount + (extraLabelsAndValues != null ? extraLabelsAndValues.length : 0)]; labelsAndValues[0] = "cluster"; labelsAndValues[1] = cluster; labelsAndValues[2] = "namespace"; @@ -495,7 +552,11 @@ static void writeTopicMetric(PrometheusMetricStreams stream, String metricName, } else { labelsAndValues[5] = topic; } - String[] labels = ArrayUtils.addAll(labelsAndValues, extraLabelsAndValues); - stream.writeSample(metricName, value, labels); + if (extraLabelsAndValues != null) { + for (int i = 0; i < extraLabelsAndValues.length; i++) { + labelsAndValues[baseLabelCount + i] = extraLabelsAndValues[i]; + } + } + stream.writeSample(metricName, value, labelsAndValues); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java index 3da061f6ffef2..df2638b3bb810 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TransactionAggregator.java @@ -56,7 +56,7 @@ public static void generate(PulsarService pulsar, PrometheusMetricStreams stream if (includeTopicMetrics) { - pulsar.getBrokerService().getMultiLayerTopicMap().forEach((namespace, bundlesMap) -> + pulsar.getBrokerService().getMultiLayerTopicsMap().forEach((namespace, bundlesMap) -> bundlesMap.forEach((bundle, topicsMap) -> topicsMap.forEach((name, topic) -> { if (topic instanceof PersistentTopic) { topic.getSubscriptions().values().forEach(subscription -> { diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarPlugin.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusLabels.java similarity index 65% rename from pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarPlugin.java rename to pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusLabels.java index 7936423f71d0b..9a2c520731468 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarPlugin.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusLabels.java @@ -16,18 +16,17 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.sql.presto; +package org.apache.pulsar.broker.stats.prometheus.metrics; -import com.google.common.collect.ImmutableList; -import io.trino.spi.Plugin; -import io.trino.spi.connector.ConnectorFactory; +import org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType; -/** - * Implementation of the Pulsar plugin for Pesto. - */ -public class PulsarPlugin implements Plugin { - @Override - public Iterable getConnectorFactories() { - return ImmutableList.of(new PulsarConnectorFactory()); +public class PrometheusLabels { + + public static String backlogQuotaTypeLabel(BacklogQuotaType backlogQuotaType) { + if (backlogQuotaType == BacklogQuotaType.message_age) { + return "time"; + } else /* destination_storage */ { + return "size"; + } } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/Summary.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/Summary.java index ba6407612428b..cb4a33a729484 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/Summary.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/Summary.java @@ -45,7 +45,7 @@ public Summary create() { } } - static class Child { + public static class Child { private final DataSketchesSummaryLogger logger; private final List quantiles; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/storage/ManagedLedgerStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/storage/ManagedLedgerStorage.java index 0b5a102eed1e0..944d2badf75f2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/storage/ManagedLedgerStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/storage/ManagedLedgerStorage.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.storage; import io.netty.channel.EventLoopGroup; +import io.opentelemetry.api.OpenTelemetry; import java.io.IOException; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.mledger.ManagedLedgerFactory; @@ -47,7 +48,8 @@ public interface ManagedLedgerStorage extends AutoCloseable { void initialize(ServiceConfiguration conf, MetadataStoreExtended metadataStore, BookKeeperClientFactory bookkeeperProvider, - EventLoopGroup eventLoopGroup) throws Exception; + EventLoopGroup eventLoopGroup, + OpenTelemetry openTelemetry) throws Exception; /** * Return the factory to create {@link ManagedLedgerFactory}. @@ -87,11 +89,12 @@ void initialize(ServiceConfiguration conf, static ManagedLedgerStorage create(ServiceConfiguration conf, MetadataStoreExtended metadataStore, BookKeeperClientFactory bkProvider, - EventLoopGroup eventLoopGroup) throws Exception { + EventLoopGroup eventLoopGroup, + OpenTelemetry openTelemetry) throws Exception { ManagedLedgerStorage storage = Reflections.createInstance(conf.getManagedLedgerStorageClassName(), ManagedLedgerStorage.class, Thread.currentThread().getContextClassLoader()); - storage.initialize(conf, metadataStore, bkProvider, eventLoopGroup); + storage.initialize(conf, metadataStore, bkProvider, eventLoopGroup, openTelemetry); return storage; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TopicPoliciesSystemTopicClient.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TopicPoliciesSystemTopicClient.java index 3fd8921c15efa..ea3ac507d1128 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TopicPoliciesSystemTopicClient.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TopicPoliciesSystemTopicClient.java @@ -30,8 +30,11 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.TypedMessageBuilder; +import org.apache.pulsar.client.api.schema.SchemaDefinition; +import org.apache.pulsar.client.internal.DefaultImplementation; import org.apache.pulsar.common.events.ActionType; import org.apache.pulsar.common.events.PulsarEvent; +import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,13 +44,17 @@ */ public class TopicPoliciesSystemTopicClient extends SystemTopicClientBase { + static Schema avroSchema = DefaultImplementation.getDefaultImplementation() + .newAvroSchema(SchemaDefinition.builder().withPojo(PulsarEvent.class).build()); + public TopicPoliciesSystemTopicClient(PulsarClient client, TopicName topicName) { super(client, topicName); + } @Override protected CompletableFuture> newWriterAsyncInternal() { - return client.newProducer(Schema.AVRO(PulsarEvent.class)) + return client.newProducer(avroSchema) .topic(topicName.toString()) .enableBatching(false) .createAsync() @@ -61,8 +68,9 @@ protected CompletableFuture> newWriterAsyncInternal() { @Override protected CompletableFuture> newReaderAsyncInternal() { - return client.newReader(Schema.AVRO(PulsarEvent.class)) + return client.newReader(avroSchema) .topic(topicName.toString()) + .subscriptionRolePrefix(SystemTopicNames.SYSTEM_READER_PREFIX) .startMessageId(MessageId.earliest) .readCompacted(true) .createAsync() diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TransactionBufferSnapshotBaseSystemTopicClient.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TransactionBufferSnapshotBaseSystemTopicClient.java index 8efa983a64d73..4023cd88bef55 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TransactionBufferSnapshotBaseSystemTopicClient.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TransactionBufferSnapshotBaseSystemTopicClient.java @@ -28,6 +28,7 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicName; @Slf4j @@ -201,6 +202,7 @@ protected CompletableFuture> newWriterAsyncInternal() { protected CompletableFuture> newReaderAsyncInternal() { return client.newReader(Schema.AVRO(schemaType)) .topic(topicName.toString()) + .subscriptionRolePrefix(SystemTopicNames.SYSTEM_READER_PREFIX) .startMessageId(MessageId.earliest) .readCompacted(true) .createAsync() diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/BrokerTool.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/BrokerTool.java index 2a479ce4b90c8..980c92fee8a5e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/BrokerTool.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/BrokerTool.java @@ -18,30 +18,32 @@ */ package org.apache.pulsar.broker.tools; -import org.apache.bookkeeper.tools.framework.Cli; -import org.apache.bookkeeper.tools.framework.CliFlags; -import org.apache.bookkeeper.tools.framework.CliSpec; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.ScopeType; /** * broker-tool is used for operations on a specific broker. */ +@Command(name = "broker-tool", description = "broker-tool is used for operations on a specific broker", + showDefaultValues = true, scope = ScopeType.INHERIT) public class BrokerTool { - public static final String NAME = "broker-tool"; + @Option( + names = {"-h", "--help"}, + description = "Display help information", + usageHelp = true + ) + public boolean help = false; public static int run(String[] args) { - CliSpec.Builder specBuilder = CliSpec.newBuilder() - .withName(NAME) - .withUsage(NAME + " [flags] [commands]") - .withDescription(NAME + " is used for operations on a specific broker") - .withFlags(new CliFlags()) - .withConsole(System.out) - .addCommand(new LoadReportCommand()) - .addCommand(new GenerateDocsCommand()); - - CliSpec spec = specBuilder.build(); - - return Cli.runCli(spec, args); + BrokerTool brokerTool = new BrokerTool(); + CommandLine commander = new CommandLine(brokerTool); + GenerateDocsCommand generateDocsCommand = new GenerateDocsCommand(commander); + commander.addSubcommand(LoadReportCommand.class) + .addSubcommand(generateDocsCommand); + return commander.execute(args); } public static void main(String[] args) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/GenerateDocsCommand.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/GenerateDocsCommand.java index 1819a56baa12a..b0ed54bc53fd0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/GenerateDocsCommand.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/GenerateDocsCommand.java @@ -18,63 +18,42 @@ */ package org.apache.pulsar.broker.tools; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; import java.util.ArrayList; import java.util.List; -import org.apache.bookkeeper.tools.framework.Cli; -import org.apache.bookkeeper.tools.framework.CliCommand; -import org.apache.bookkeeper.tools.framework.CliFlags; -import org.apache.bookkeeper.tools.framework.CliSpec; -import org.apache.pulsar.common.util.CmdGenerateDocs; +import java.util.concurrent.Callable; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; /** * The command to generate documents of broker-tool. */ -public class GenerateDocsCommand extends CliCommand { +@Command(name = "gen-doc", description = "Generate documents of broker-tool") +public class GenerateDocsCommand implements Callable { + @Option( + names = {"-n", "--command-names"}, + description = "List of command names", + arity = "0..1" + ) + private List commandNames = new ArrayList<>(); + private final CommandLine rootCmd; - private static final String NAME = "gen-doc"; - private static final String DESC = "Generate documents of broker-tool"; - - /** - * The CLI flags of gen docs command. - */ - protected static class GenDocFlags extends CliFlags { - @Parameter( - names = {"-n", "--command-names"}, - description = "List of command names" - ) - private List commandNames = new ArrayList<>(); - } - - public GenerateDocsCommand() { - super(CliSpec.newBuilder() - .withName(NAME) - .withDescription(DESC) - .withFlags(new GenDocFlags()) - .build()); + public GenerateDocsCommand(CommandLine rootCmd) { + this.rootCmd = rootCmd; } @Override - public Boolean apply(CliFlags globalFlags, String[] args) { - CliSpec newSpec = CliSpec.newBuilder(spec) - .withRunFunc(cmdFlags -> apply(cmdFlags)) - .build(); - return 0 == Cli.runCli(newSpec, args); - } - - private boolean apply(GenerateDocsCommand.GenDocFlags flags) { + public Integer call() throws Exception { CmdGenerateDocs cmd = new CmdGenerateDocs("pulsar"); - JCommander commander = new JCommander(); - commander.addCommand("load-report", new LoadReportCommand.Flags()); - cmd.addCommand("broker-tool", commander); - if (flags.commandNames.isEmpty()) { + cmd.addCommand("broker-tool", rootCmd); + if (commandNames.isEmpty()) { cmd.run(null); } else { - ArrayList args = new ArrayList(flags.commandNames); + ArrayList args = new ArrayList(commandNames); args.add(0, "-n"); cmd.run(args.toArray(new String[0])); } - return true; + return 0; } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/LoadReportCommand.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/LoadReportCommand.java index 935e3a9f2fa1a..f1f4a917571be 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/LoadReportCommand.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/LoadReportCommand.java @@ -18,76 +18,48 @@ */ package org.apache.pulsar.broker.tools; -import com.beust.jcommander.Parameter; import java.util.Optional; +import java.util.concurrent.Callable; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import org.apache.bookkeeper.tools.framework.Cli; -import org.apache.bookkeeper.tools.framework.CliCommand; -import org.apache.bookkeeper.tools.framework.CliFlags; -import org.apache.bookkeeper.tools.framework.CliSpec; import org.apache.commons.lang3.SystemUtils; import org.apache.pulsar.broker.loadbalance.BrokerHostUsage; import org.apache.pulsar.broker.loadbalance.impl.GenericBrokerHostUsageImpl; import org.apache.pulsar.broker.loadbalance.impl.LinuxBrokerHostUsageImpl; -import org.apache.pulsar.broker.tools.LoadReportCommand.Flags; import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; +import picocli.CommandLine.Command; +import picocli.CommandLine.Model.CommandSpec; +import picocli.CommandLine.Option; +import picocli.CommandLine.Spec; /** * The command to collect the load report of a specific broker. */ -public class LoadReportCommand extends CliCommand { +@Command(name = "load-report", description = "Collect the load report of a specific broker") +public class LoadReportCommand implements Callable { - private static final String NAME = "load-report"; - private static final String DESC = "Collect the load report of a specific broker"; + @Option(names = {"-i", "--interval-ms"}, description = "Interval to collect load report, in milliseconds") + public int intervalMilliseconds = 100; - /** - * The CLI flags of load report command. - */ - public static class Flags extends CliFlags { - - @Parameter( - names = { - "-i", "--interval-ms" - }, - description = "Interval to collect load report, in milliseconds" - ) - public int intervalMilliseconds = 100; - - } - - public LoadReportCommand() { - super(CliSpec.newBuilder() - .withName(NAME) - .withDescription(DESC) - .withFlags(new Flags()) - .build()); - } + @Spec + CommandSpec spec; @Override - public Boolean apply(CliFlags globalFlags, String[] args) { - CliSpec newSpec = CliSpec.newBuilder(spec) - .withRunFunc(cmdFlags -> apply(cmdFlags)) - .build(); - return 0 == Cli.runCli(newSpec, args); - } - - private boolean apply(Flags flags) { - + public Integer call() throws Exception { boolean isLinux = SystemUtils.IS_OS_LINUX; - spec.console().println("OS ARCH: " + SystemUtils.OS_ARCH); - spec.console().println("OS NAME: " + SystemUtils.OS_NAME); - spec.console().println("OS VERSION: " + SystemUtils.OS_VERSION); - spec.console().println("Linux: " + isLinux); - spec.console().println("--------------------------------------"); - spec.console().println(); - spec.console().println("Load Report Interval : " + flags.intervalMilliseconds + " ms"); - spec.console().println(); - spec.console().println("--------------------------------------"); - spec.console().println(); + spec.commandLine().getOut().println("OS ARCH: " + SystemUtils.OS_ARCH); + spec.commandLine().getOut().println("OS NAME: " + SystemUtils.OS_NAME); + spec.commandLine().getOut().println("OS VERSION: " + SystemUtils.OS_VERSION); + spec.commandLine().getOut().println("Linux: " + isLinux); + spec.commandLine().getOut().println("--------------------------------------"); + spec.commandLine().getOut().println(); + spec.commandLine().getOut().println("Load Report Interval : " + intervalMilliseconds + " ms"); + spec.commandLine().getOut().println(); + spec.commandLine().getOut().println("--------------------------------------"); + spec.commandLine().getOut().println(); ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor( new ExecutorProvider.ExtendedThreadFactory("load-report")); @@ -105,7 +77,7 @@ private boolean apply(Flags flags) { hostUsage.calculateBrokerHostUsage(); try { - TimeUnit.MILLISECONDS.sleep(flags.intervalMilliseconds); + TimeUnit.MILLISECONDS.sleep(intervalMilliseconds); } catch (InterruptedException e) { } hostUsage.calculateBrokerHostUsage(); @@ -117,13 +89,13 @@ private boolean apply(Flags flags) { printResourceUsage("Bandwidth In", usage.bandwidthIn); printResourceUsage("Bandwidth Out", usage.bandwidthOut); - return true; + return 0; } finally { scheduler.shutdown(); } } private void printResourceUsage(String name, ResourceUsage usage) { - spec.console().println(name + " : usage = " + usage.usage + ", limit = " + usage.limit); + spec.commandLine().getOut().println(name + " : usage = " + usage.usage + ", limit = " + usage.limit); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java index 8223aa12b75ae..b5ec70fda9774 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java @@ -19,19 +19,25 @@ package org.apache.pulsar.broker.transaction.buffer; import java.util.concurrent.CompletableFuture; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.common.policies.data.TransactionBufferStats; public interface AbortedTxnProcessor { + enum SnapshotType { + Single, + Segment, + } + /** * After the transaction buffer writes a transaction aborted marker to the topic, * the transaction buffer will put the aborted txnID and the aborted marker position to AbortedTxnProcessor. * @param txnID aborted transaction ID. * @param abortedMarkerPersistentPosition the position of the abort txn marker. */ - void putAbortedTxnAndPosition(TxnID txnID, PositionImpl abortedMarkerPersistentPosition); + void putAbortedTxnAndPosition(TxnID txnID, Position abortedMarkerPersistentPosition); /** * Clean up invalid aborted transactions. @@ -50,7 +56,7 @@ public interface AbortedTxnProcessor { * @return a Position (startReadCursorPosition) determiner where to start to recover in the original topic. */ - CompletableFuture recoverFromSnapshot(); + CompletableFuture recoverFromSnapshot(); /** * Delete the transaction buffer aborted transaction snapshot. @@ -62,13 +68,14 @@ public interface AbortedTxnProcessor { * Take aborted transactions snapshot. * @return a completableFuture. */ - CompletableFuture takeAbortedTxnsSnapshot(PositionImpl maxReadPosition); + CompletableFuture takeAbortedTxnsSnapshot(Position maxReadPosition); /** * Get the lastSnapshotTimestamps. - * @return the lastSnapshotTimestamps. + * + * @return a transactionBufferStats with the stats in the abortedTxnProcessor. */ - long getLastSnapshotTimestamps(); + TransactionBufferStats generateSnapshotStats(boolean segmentStats); CompletableFuture closeAsync(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBuffer.java index 99093e42fd74c..874f4c1c28a02 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBuffer.java @@ -23,7 +23,8 @@ import java.util.List; import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.pulsar.broker.transaction.exception.TransactionException; +import org.apache.pulsar.broker.transaction.exception.buffer.TransactionBufferException; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.policies.data.TransactionBufferStats; import org.apache.pulsar.common.policies.data.TransactionInBufferStats; @@ -56,8 +57,7 @@ public interface TransactionBuffer { * * @param txnID the transaction id * @return a future represents the result of the operation - * @throws org.apache.pulsar.broker.transaction.buffer.exceptions.TransactionNotFoundException if the transaction - * is not in the buffer. + * @throws TransactionBufferException.TransactionNotFoundException if the transaction is not in the buffer. */ CompletableFuture getTransactionMeta(TxnID txnID); @@ -70,8 +70,7 @@ public interface TransactionBuffer { * @param sequenceId the sequence id of the entry in this transaction buffer. * @param buffer the entry buffer * @return a future represents the result of the operation. - * @throws org.apache.pulsar.broker.transaction.buffer.exceptions.TransactionSealedException if the transaction - * has been sealed. + * @throws TransactionException.TransactionSealedException if the transaction has been sealed. */ CompletableFuture appendBufferToTxn(TxnID txnId, long sequenceId, ByteBuf buffer); @@ -82,8 +81,7 @@ public interface TransactionBuffer { * @param txnID transaction id * @param startSequenceId the sequence id to start read * @return a future represents the result of open operation. - * @throws org.apache.pulsar.broker.transaction.buffer.exceptions.TransactionNotFoundException if the transaction - * is not in the buffer. + * @throws TransactionBufferException.TransactionNotFoundException if the transaction is not in the buffer. */ CompletableFuture openTransactionBufferReader(TxnID txnID, long startSequenceId); @@ -95,8 +93,7 @@ public interface TransactionBuffer { * @param txnID the transaction id * @param lowWaterMark the low water mark of this transaction * @return a future represents the result of commit operation. - * @throws org.apache.pulsar.broker.transaction.buffer.exceptions.TransactionNotFoundException if the transaction - * is not in the buffer. + * @throws TransactionBufferException.TransactionNotFoundException if the transaction is not in the buffer. */ CompletableFuture commitTxn(TxnID txnID, long lowWaterMark); @@ -107,8 +104,7 @@ public interface TransactionBuffer { * @param txnID the transaction id * @param lowWaterMark the low water mark of this transaction * @return a future represents the result of abort operation. - * @throws org.apache.pulsar.broker.transaction.buffer.exceptions.TransactionNotFoundException if the transaction - * is not in the buffer. + * @throws TransactionBufferException.TransactionNotFoundException if the transaction is not in the buffer. */ CompletableFuture abortTxn(TxnID txnID, long lowWaterMark); @@ -139,24 +135,38 @@ public interface TransactionBuffer { CompletableFuture closeAsync(); /** - * Close the buffer asynchronously. + * Check if the txn is aborted. + * TODO: To avoid broker oom, we will load the aborted txn from snapshot on demand. + * So we need the readPosition to check if the txn is loaded. * @param txnID {@link TxnID} txnId. - * @param readPosition the persitent position of the txn message. - * @return the txnId is aborted. + * @param readPosition the persistent position of the txn message. + * @return whether the txn is aborted. */ - boolean isTxnAborted(TxnID txnID, PositionImpl readPosition); + boolean isTxnAborted(TxnID txnID, Position readPosition); /** * Sync max read position for normal publish. - * @param position {@link PositionImpl} the position to sync. + * @param position {@link Position} the position to sync. + * @param isMarkerMessage whether the message is marker message. */ - void syncMaxReadPositionForNormalPublish(PositionImpl position); + void syncMaxReadPositionForNormalPublish(Position position, boolean isMarkerMessage); /** * Get the can read max position. * @return the stable position. */ - PositionImpl getMaxReadPosition(); + Position getMaxReadPosition(); + + /** + * Get the snapshot type. + * + * The snapshot type can be either "Single" or "Segment". In "Single" mode, a single snapshot log is used + * to record the transaction buffer stats. In "Segment" mode, a snapshot segment topic is used to record + * the stats, and a separate snapshot segment index topic is used to index these stats. + * + * @return the snapshot type + */ + AbortedTxnProcessor.SnapshotType getSnapshotType(); /** * Get transaction in buffer stats. @@ -168,17 +178,20 @@ public interface TransactionBuffer { * Get transaction stats in buffer. * @return the transaction stats in buffer. */ - TransactionBufferStats getStats(boolean lowWaterMarks); + TransactionBufferStats getStats(boolean lowWaterMarks, boolean segmentStats); /** - * Wait TransactionBuffer Recovers completely. - * Take snapshot after TB Recovers completely. - * @param isTxn - * @return a future which has completely if isTxn = false. Or a future return by takeSnapshot. + * Get transaction stats in buffer. + * @return the transaction stats in buffer. */ - CompletableFuture checkIfTBRecoverCompletely(boolean isTxn); - + TransactionBufferStats getStats(boolean lowWaterMarks); + /** + * Wait TransactionBuffer recovers completely. + * + * @return a future that will be completed after the transaction buffer recover completely. + */ + CompletableFuture checkIfTBRecoverCompletely(); long getOngoingTxnCount(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientStats.java index 8fda233ff1dfa..c21b212f981dd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientStats.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.transaction.buffer; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferClientStatsImpl; import org.apache.pulsar.client.impl.transaction.TransactionBufferHandler; @@ -34,10 +35,10 @@ public interface TransactionBufferClientStats { void close(); - static TransactionBufferClientStats create(boolean exposeTopicMetrics, TransactionBufferHandler handler, - boolean enableTxnCoordinator) { + static TransactionBufferClientStats create(PulsarService pulsarService, boolean exposeTopicMetrics, + TransactionBufferHandler handler, boolean enableTxnCoordinator) { return enableTxnCoordinator - ? TransactionBufferClientStatsImpl.getInstance(exposeTopicMetrics, handler) : NOOP; + ? TransactionBufferClientStatsImpl.getInstance(pulsarService, exposeTopicMetrics, handler) : NOOP; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java index 56b49f98efe8e..4da7a48e96c51 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java @@ -31,8 +31,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; import org.apache.pulsar.broker.transaction.buffer.TransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.TransactionBufferReader; import org.apache.pulsar.broker.transaction.buffer.TransactionMeta; @@ -211,9 +212,18 @@ public TransactionBufferReader newReader(long sequenceId) throws final ConcurrentMap buffers; final Map> txnIndex; + private final Topic topic; + private final TopicTransactionBuffer.MaxReadPositionCallBack maxReadPositionCallBack; + public InMemTransactionBuffer(Topic topic) { this.buffers = new ConcurrentHashMap<>(); this.txnIndex = new HashMap<>(); + this.topic = topic; + if (topic instanceof PersistentTopic) { + this.maxReadPositionCallBack = ((PersistentTopic) topic).getMaxReadPositionCallBack(); + } else { + this.maxReadPositionCallBack = null; + } } @Override @@ -360,18 +370,28 @@ public CompletableFuture closeAsync() { } @Override - public boolean isTxnAborted(TxnID txnID, PositionImpl readPosition) { + public boolean isTxnAborted(TxnID txnID, Position readPosition) { return false; } @Override - public void syncMaxReadPositionForNormalPublish(PositionImpl position) { - //no-op + public void syncMaxReadPositionForNormalPublish(Position position, boolean isMarkerMessage) { + if (!isMarkerMessage) { + updateLastDispatchablePosition(position); + if (maxReadPositionCallBack != null) { + maxReadPositionCallBack.maxReadPositionMovedForward(null, position); + } + } } @Override - public PositionImpl getMaxReadPosition() { - return PositionImpl.LATEST; + public Position getMaxReadPosition() { + return topic.getLastPosition(); + } + + @Override + public AbortedTxnProcessor.SnapshotType getSnapshotType() { + return null; } @Override @@ -380,12 +400,18 @@ public TransactionInBufferStats getTransactionInBufferStats(TxnID txnID) { } @Override - public TransactionBufferStats getStats(boolean lowWaterMarks) { + public TransactionBufferStats getStats(boolean lowWaterMarks, boolean segmentStats) { return null; } @Override - public CompletableFuture checkIfTBRecoverCompletely(boolean isTxn) { + public TransactionBufferStats getStats(boolean lowWaterMarks) { + return getStats(lowWaterMarks, false); + } + + + @Override + public CompletableFuture checkIfTBRecoverCompletely() { return CompletableFuture.completedFuture(null); } @@ -412,4 +438,11 @@ public long getCommittedTxnCount() { .filter(txnBuffer -> txnBuffer.status.equals(TxnStatus.COMMITTED)) .count(); } + + // ThreadSafe + private void updateLastDispatchablePosition(Position position) { + if (topic instanceof PersistentTopic t) { + t.updateLastDispatchablePosition(position); + } + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index 5d582d564eadd..a0ffa121b8999 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -21,24 +21,18 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.commons.collections4.map.LinkedMap; -import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService.ReferenceCountedWriter; import org.apache.pulsar.broker.service.persistent.PersistentTopic; -import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; import org.apache.pulsar.broker.transaction.buffer.metadata.AbortTxnMetadata; import org.apache.pulsar.broker.transaction.buffer.metadata.TransactionBufferSnapshot; -import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.transaction.TxnID; -import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.policies.data.TransactionBufferStats; @Slf4j public class SingleSnapshotAbortedTxnProcessorImpl implements AbortedTxnProcessor { @@ -48,7 +42,7 @@ public class SingleSnapshotAbortedTxnProcessorImpl implements AbortedTxnProcesso * Aborts, map for jude message is aborted, linked for remove abort txn in memory when this * position have been deleted. */ - private final LinkedMap aborts = new LinkedMap<>(); + private final LinkedMap aborts = new LinkedMap<>(); private volatile long lastSnapshotTimestamps; @@ -67,15 +61,15 @@ public SingleSnapshotAbortedTxnProcessorImpl(PersistentTopic topic) { } @Override - public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl abortedMarkerPersistentPosition) { + public void putAbortedTxnAndPosition(TxnID abortedTxnId, Position abortedMarkerPersistentPosition) { aborts.put(abortedTxnId, abortedMarkerPersistentPosition); } //In this implementation we clear the invalid aborted txn ID one by one. @Override public void trimExpiredAbortedTxns() { - while (!aborts.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) - .ledgerExists(aborts.get(aborts.firstKey()).getLedgerId())) { + while (!aborts.isEmpty() && !topic.getManagedLedger().getLedgersInfo() + .containsKey(aborts.get(aborts.firstKey()).getLedgerId())) { if (log.isDebugEnabled()) { log.debug("[{}] Topic transaction buffer clear aborted transaction, TxnId : {}, Position : {}", topic.getName(), aborts.firstKey(), aborts.get(aborts.firstKey())); @@ -89,48 +83,27 @@ public boolean checkAbortedTransaction(TxnID txnID) { return aborts.containsKey(txnID); } - private long getSystemClientOperationTimeoutMs() throws Exception { - PulsarClientImpl pulsarClient = (PulsarClientImpl) topic.getBrokerService().getPulsar().getClient(); - return pulsarClient.getConfiguration().getOperationTimeoutMs(); - } - @Override - public CompletableFuture recoverFromSnapshot() { - return topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory() - .getTxnBufferSnapshotService() - .createReader(TopicName.get(topic.getName())).thenComposeAsync(reader -> { - try { - PositionImpl startReadCursorPosition = null; - while (reader.hasMoreEvents()) { - Message message = reader.readNextAsync() - .get(getSystemClientOperationTimeoutMs(), TimeUnit.MILLISECONDS); - if (topic.getName().equals(message.getKey())) { - TransactionBufferSnapshot transactionBufferSnapshot = message.getValue(); - if (transactionBufferSnapshot != null) { - handleSnapshot(transactionBufferSnapshot); - startReadCursorPosition = PositionImpl.get( - transactionBufferSnapshot.getMaxReadPositionLedgerId(), - transactionBufferSnapshot.getMaxReadPositionEntryId()); - } - } - } - return CompletableFuture.completedFuture(startReadCursorPosition); - } catch (TimeoutException ex) { - Throwable t = FutureUtil.unwrapCompletionException(ex); - String errorMessage = String.format("[%s] Transaction buffer recover fail by read " - + "transactionBufferSnapshot timeout!", topic.getName()); - log.error(errorMessage, t); - return FutureUtil.failedFuture( - new BrokerServiceException.ServiceUnitNotReadyException(errorMessage, t)); - } catch (Exception ex) { - log.error("[{}] Transaction buffer recover fail when read " - + "transactionBufferSnapshot!", topic.getName(), ex); - return FutureUtil.failedFuture(ex); - } finally { - closeReader(reader); - } - }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider() - .getExecutor(this)); + public CompletableFuture recoverFromSnapshot() { + final var future = new CompletableFuture(); + final var pulsar = topic.getBrokerService().getPulsar(); + pulsar.getTransactionExecutorProvider().getExecutor(this).execute(() -> { + try { + final var snapshot = pulsar.getTransactionBufferSnapshotServiceFactory().getTxnBufferSnapshotService() + .getTableView().readLatest(topic.getName()); + if (snapshot != null) { + handleSnapshot(snapshot); + final var startReadCursorPosition = PositionFactory.create(snapshot.getMaxReadPositionLedgerId(), + snapshot.getMaxReadPositionEntryId()); + future.complete(startReadCursorPosition); + } else { + future.complete(null); + } + } catch (Throwable e) { + future.completeExceptionally(e); + } + }); + return future; } @Override @@ -143,7 +116,7 @@ public CompletableFuture clearAbortedTxnSnapshot() { } @Override - public CompletableFuture takeAbortedTxnsSnapshot(PositionImpl maxReadPosition) { + public CompletableFuture takeAbortedTxnsSnapshot(Position maxReadPosition) { return takeSnapshotWriter.getFuture().thenCompose(writer -> { TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot(); snapshot.setTopicName(topic.getName()); @@ -173,8 +146,11 @@ public CompletableFuture takeAbortedTxnsSnapshot(PositionImpl maxReadPosit } @Override - public long getLastSnapshotTimestamps() { - return this.lastSnapshotTimestamps; + public TransactionBufferStats generateSnapshotStats(boolean segmentStats) { + TransactionBufferStats transactionBufferStats = new TransactionBufferStats(); + transactionBufferStats.lastSnapshotTimestamps = this.lastSnapshotTimestamps; + transactionBufferStats.totalAbortedTransactions = aborts.size(); + return transactionBufferStats; } @Override @@ -186,19 +162,12 @@ public synchronized CompletableFuture closeAsync() { return CompletableFuture.completedFuture(null); } - private void closeReader(SystemTopicClient.Reader reader) { - reader.closeAsync().exceptionally(e -> { - log.error("[{}]Transaction buffer reader close error!", topic.getName(), e); - return null; - }); - } - private void handleSnapshot(TransactionBufferSnapshot snapshot) { if (snapshot.getAborts() != null) { snapshot.getAborts().forEach(abortTxnMetadata -> aborts.put(new TxnID(abortTxnMetadata.getTxnIdMostBits(), abortTxnMetadata.getTxnIdLeastBits()), - PositionImpl.get(abortTxnMetadata.getLedgerId(), + PositionFactory.create(abortTxnMetadata.getLedgerId(), abortTxnMetadata.getEntryId()))); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 4f4e58ac3f55b..88a3968b7b430 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -24,11 +24,11 @@ import java.util.LinkedList; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.CompletionStage; import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.function.Supplier; @@ -36,9 +36,9 @@ import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedLedgerException; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.bookkeeper.mledger.impl.ReadOnlyManagedLedgerImpl; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.mledger.ReadOnlyManagedLedger; import org.apache.commons.collections4.map.LinkedMap; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.commons.lang3.tuple.Pair; @@ -53,14 +53,17 @@ import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexesMetadata; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotSegment; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TxnIDData; -import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.SegmentStats; +import org.apache.pulsar.common.policies.data.SegmentsStats; +import org.apache.pulsar.common.policies.data.TransactionBufferStats; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.util.FutureUtil; @@ -76,7 +79,7 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess /** * The map is used to clear the aborted transaction IDs persistent in the expired ledger. *

- * The key PositionImpl {@link PositionImpl} is the persistent position of + * The key Position {@link Position} is the persistent position of * the latest transaction of a segment. * The value TxnID {@link TxnID} is the latest Transaction ID in a segment. *

@@ -90,7 +93,7 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess * the positions. *

*/ - private final LinkedMap segmentIndex = new LinkedMap<>(); + private final LinkedMap segmentIndex = new LinkedMap<>(); /** * This map is used to check whether a transaction is an aborted transaction. @@ -109,12 +112,14 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess * indexes of the snapshot segment. *

*/ - private final LinkedMap indexes = new LinkedMap<>(); + private final LinkedMap indexes = new LinkedMap<>(); private final PersistentTopic topic; private volatile long lastSnapshotTimestamps; + private volatile long lastTakedSnapshotSegmentTimestamp; + /** * The number of the aborted transaction IDs in a segment. * This is calculated according to the configured memory size. @@ -151,7 +156,7 @@ public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { } @Override - public void putAbortedTxnAndPosition(TxnID txnID, PositionImpl position) { + public void putAbortedTxnAndPosition(TxnID txnID, Position position) { unsealedTxnIds.add(txnID); aborts.put(txnID, txnID); /* @@ -174,7 +179,7 @@ public boolean checkAbortedTransaction(TxnID txnID) { } /** - * Check werther the position in segmentIndex {@link SnapshotSegmentAbortedTxnProcessorImpl#segmentIndex} + * Check whether the position in segmentIndex {@link SnapshotSegmentAbortedTxnProcessorImpl#segmentIndex} * is expired. If the position is not exist in the original topic, the according transaction is an invalid * transaction. And the according segment is invalid, too. The transaction IDs before the transaction ID * in the aborts are invalid, too. @@ -182,14 +187,14 @@ public boolean checkAbortedTransaction(TxnID txnID) { @Override public void trimExpiredAbortedTxns() { //Checking whether there are some segment expired. - List positionsNeedToDelete = new ArrayList<>(); - while (!segmentIndex.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) - .ledgerExists(segmentIndex.firstKey().getLedgerId())) { + List positionsNeedToDelete = new ArrayList<>(); + while (!segmentIndex.isEmpty() && !topic.getManagedLedger().getLedgersInfo() + .containsKey(segmentIndex.firstKey().getLedgerId())) { if (log.isDebugEnabled()) { log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", topic.getName(), segmentIndex.firstKey()); } - PositionImpl positionNeedToDelete = segmentIndex.firstKey(); + Position positionNeedToDelete = segmentIndex.firstKey(); positionsNeedToDelete.add(positionNeedToDelete); TxnID theLatestDeletedTxnID = segmentIndex.remove(0); @@ -210,7 +215,7 @@ private String buildKey(long sequenceId) { } @Override - public CompletableFuture takeAbortedTxnsSnapshot(PositionImpl maxReadPosition) { + public CompletableFuture takeAbortedTxnsSnapshot(Position maxReadPosition) { //Store the latest aborted transaction IDs in unsealedTxnIDs and the according the latest max read position. TransactionBufferSnapshotIndexesMetadata metadata = new TransactionBufferSnapshotIndexesMetadata( maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), @@ -220,213 +225,130 @@ public CompletableFuture takeAbortedTxnsSnapshot(PositionImpl maxReadPosit } @Override - public CompletableFuture recoverFromSnapshot() { - return topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory() - .getTxnBufferSnapshotIndexService() - .createReader(TopicName.get(topic.getName())).thenComposeAsync(reader -> { - PositionImpl startReadCursorPosition = null; - TransactionBufferSnapshotIndexes persistentSnapshotIndexes = null; - try { - /* - Read the transaction snapshot segment index. -

- The processor can get the sequence ID, unsealed transaction IDs, - segment index list and max read position in the snapshot segment index. - Then we can traverse the index list to read all aborted transaction IDs - in segments to aborts. -

- */ - while (reader.hasMoreEvents()) { - Message message = reader.readNextAsync() - .get(getSystemClientOperationTimeoutMs(), TimeUnit.MILLISECONDS); - if (topic.getName().equals(message.getKey())) { - TransactionBufferSnapshotIndexes transactionBufferSnapshotIndexes = message.getValue(); - if (transactionBufferSnapshotIndexes != null) { - persistentSnapshotIndexes = transactionBufferSnapshotIndexes; - startReadCursorPosition = PositionImpl.get( - transactionBufferSnapshotIndexes.getSnapshot().getMaxReadPositionLedgerId(), - transactionBufferSnapshotIndexes.getSnapshot().getMaxReadPositionEntryId()); - } - } - } - } catch (TimeoutException ex) { - Throwable t = FutureUtil.unwrapCompletionException(ex); - String errorMessage = String.format("[%s] Transaction buffer recover fail by read " - + "transactionBufferSnapshot timeout!", topic.getName()); - log.error(errorMessage, t); - return FutureUtil.failedFuture( - new BrokerServiceException.ServiceUnitNotReadyException(errorMessage, t)); - } catch (Exception ex) { - log.error("[{}] Transaction buffer recover fail when read " - + "transactionBufferSnapshot!", topic.getName(), ex); - return FutureUtil.failedFuture(ex); - } finally { - closeReader(reader); - } - PositionImpl finalStartReadCursorPosition = startReadCursorPosition; - TransactionBufferSnapshotIndexes finalPersistentSnapshotIndexes = persistentSnapshotIndexes; - if (persistentSnapshotIndexes == null) { - return recoverOldSnapshot(); - } else { - this.unsealedTxnIds = convertTypeToTxnID(persistentSnapshotIndexes - .getSnapshot().getAborts()); - } - //Read snapshot segment to recover aborts. - ArrayList> completableFutures = new ArrayList<>(); - CompletableFuture openManagedLedgerAndHandleSegmentsFuture = new CompletableFuture<>(); - AtomicBoolean hasInvalidIndex = new AtomicBoolean(false); - AsyncCallbacks.OpenReadOnlyManagedLedgerCallback callback = new AsyncCallbacks - .OpenReadOnlyManagedLedgerCallback() { - @Override - public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl readOnlyManagedLedger, - Object ctx) { - finalPersistentSnapshotIndexes.getIndexList().forEach(index -> { - CompletableFuture handleSegmentFuture = new CompletableFuture<>(); - completableFutures.add(handleSegmentFuture); - readOnlyManagedLedger.asyncReadEntry( - new PositionImpl(index.getSegmentLedgerID(), - index.getSegmentEntryID()), - new AsyncCallbacks.ReadEntryCallback() { - @Override - public void readEntryComplete(Entry entry, Object ctx) { - handleSnapshotSegmentEntry(entry); - indexes.put(new PositionImpl( - index.abortedMarkLedgerID, - index.abortedMarkEntryID), - index); - entry.release(); - handleSegmentFuture.complete(null); - } - - @Override - public void readEntryFailed(ManagedLedgerException exception, Object ctx) { - /* - The logic flow of deleting expired segment is: -

- 1. delete segment - 2. update segment index -

- If the worker delete segment successfully - but failed to update segment index, - the segment can not be read according to the index. - We update index again if there are invalid indexes. - */ - if (((ManagedLedgerImpl) topic.getManagedLedger()) - .ledgerExists(index.getAbortedMarkLedgerID())) { - log.error("[{}] Failed to read snapshot segment [{}:{}]", - topic.getName(), index.segmentLedgerID, - index.segmentEntryID, exception); - handleSegmentFuture.completeExceptionally(exception); - } else { - hasInvalidIndex.set(true); - } - } - }, null); - }); - openManagedLedgerAndHandleSegmentsFuture.complete(null); - } + public CompletableFuture recoverFromSnapshot() { + final var pulsar = topic.getBrokerService().getPulsar(); + final var future = new CompletableFuture(); + pulsar.getTransactionExecutorProvider().getExecutor(this).execute(() -> { + try { + final var indexes = pulsar.getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotIndexService().getTableView().readLatest(topic.getName()); + if (indexes == null) { + // Try recovering from the old format snapshot + future.complete(recoverOldSnapshot()); + return; + } + final var snapshot = indexes.getSnapshot(); + final var startReadCursorPosition = PositionFactory.create(snapshot.getMaxReadPositionLedgerId(), + snapshot.getMaxReadPositionEntryId()); + this.unsealedTxnIds = convertTypeToTxnID(snapshot.getAborts()); + // Read snapshot segment to recover aborts + final var snapshotSegmentTopicName = TopicName.get(TopicDomain.persistent.toString(), + TopicName.get(topic.getName()).getNamespaceObject(), + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS); + readSegmentEntries(snapshotSegmentTopicName, indexes); + if (!this.indexes.isEmpty()) { + // If there is no segment index, the persistent worker will write segment begin from 0. + persistentWorker.sequenceID.set(this.indexes.get(this.indexes.lastKey()).sequenceID + 1); + } + unsealedTxnIds.forEach(txnID -> aborts.put(txnID, txnID)); + future.complete(startReadCursorPosition); + } catch (Throwable throwable) { + future.completeExceptionally(throwable); + } + }); + return future; + } - @Override - public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Object ctx) { - log.error("[{}] Failed to open readOnly managed ledger", topic, exception); - openManagedLedgerAndHandleSegmentsFuture.completeExceptionally(exception); - } - }; - - TopicName snapshotSegmentTopicName = TopicName.get(TopicDomain.persistent.toString(), - TopicName.get(topic.getName()).getNamespaceObject(), - SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS); - this.topic.getBrokerService().getPulsar().getManagedLedgerFactory() - .asyncOpenReadOnlyManagedLedger(snapshotSegmentTopicName - .getPersistenceNamingEncoding(), callback, - topic.getManagedLedger().getConfig(), - null); - /* - Wait the processor recover completely and then allow TB - to recover the messages after the startReadCursorPosition. - */ - return openManagedLedgerAndHandleSegmentsFuture - .thenCompose((ignore) -> FutureUtil.waitForAll(completableFutures)) - .thenCompose((i) -> { - /* - Update the snapshot segment index if there exist invalid indexes. - */ - if (hasInvalidIndex.get()) { - persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, - () -> persistentWorker.updateSnapshotIndex( - finalPersistentSnapshotIndexes.getSnapshot())); - } - /* - If there is no segment index, the persistent worker will write segment begin from 0. - */ - if (indexes.size() != 0) { - persistentWorker.sequenceID.set(indexes.get(indexes.lastKey()).sequenceID + 1); - } - /* - Append the aborted txn IDs in the index metadata - can keep the order of the aborted txn in the aborts. - So that we can trim the expired snapshot segment in aborts - according to the latest transaction IDs in the segmentIndex. - */ - unsealedTxnIds.forEach(txnID -> aborts.put(txnID, txnID)); - return CompletableFuture.completedFuture(finalStartReadCursorPosition); - }).exceptionally(ex -> { - log.error("[{}] Failed to recover snapshot segment", this.topic.getName(), ex); - return null; - }); - - }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider() - .getExecutor(this)); + private void readSegmentEntries(TopicName topicName, TransactionBufferSnapshotIndexes indexes) throws Exception { + final var managedLedger = openReadOnlyManagedLedger(topicName); + boolean hasInvalidIndex = false; + for (var index : indexes.getIndexList()) { + final var position = PositionFactory.create(index.getSegmentLedgerID(), index.getSegmentEntryID()); + final var abortedPosition = PositionFactory.create(index.abortedMarkLedgerID, index.abortedMarkEntryID); + try { + final var entry = readEntry(managedLedger, position); + try { + handleSnapshotSegmentEntry(entry); + this.indexes.put(abortedPosition, index); + } finally { + entry.release(); + } + } catch (Throwable throwable) { + if (topic.getManagedLedger().getLedgersInfo() + .containsKey(index.getAbortedMarkLedgerID())) { + log.error("[{}] Failed to read snapshot segment [{}:{}]", + topic.getName(), index.segmentLedgerID, + index.segmentEntryID, throwable); + throw throwable; + } else { + hasInvalidIndex = true; + } + } + } + if (hasInvalidIndex) { + // Update the snapshot segment index if there exist invalid indexes. + persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, + () -> persistentWorker.updateSnapshotIndex(indexes.getSnapshot())); + } + } + + private ReadOnlyManagedLedger openReadOnlyManagedLedger(TopicName topicName) throws Exception { + final var future = new CompletableFuture(); + final var callback = new AsyncCallbacks.OpenReadOnlyManagedLedgerCallback() { + @Override + public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedger managedLedger, Object ctx) { + future.complete(managedLedger); + } + + @Override + public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Object ctx) { + future.completeExceptionally(exception); + } + + @Override + public String toString() { + return String.format("Transaction buffer [%s] recover from snapshot", + SnapshotSegmentAbortedTxnProcessorImpl.this.topic.getName()); + } + }; + topic.getBrokerService().getPulsar().getManagedLedgerFactory().asyncOpenReadOnlyManagedLedger( + topicName.getPersistenceNamingEncoding(), callback, topic.getManagedLedger().getConfig(), null); + return wait(future, "open read only ml for " + topicName); + } + + private Entry readEntry(ReadOnlyManagedLedger managedLedger, Position position) throws Exception { + final var future = new CompletableFuture(); + managedLedger.asyncReadEntry(position, new AsyncCallbacks.ReadEntryCallback() { + @Override + public void readEntryComplete(Entry entry, Object ctx) { + future.complete(entry); + } + + @Override + public void readEntryFailed(ManagedLedgerException exception, Object ctx) { + future.completeExceptionally(exception); + } + }, null); + return wait(future, "read entry from " + position); } // This method will be deprecated and removed in version 4.x.0 - private CompletableFuture recoverOldSnapshot() { - return topic.getBrokerService().getTopic(TopicName.get(topic.getName()).getNamespace() + "/" - + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT, false) - .thenCompose(topicOption -> { - if (!topicOption.isPresent()) { - return CompletableFuture.completedFuture(null); - } else { - return topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory() - .getTxnBufferSnapshotService() - .createReader(TopicName.get(topic.getName())).thenComposeAsync(snapshotReader -> { - PositionImpl startReadCursorPositionInOldSnapshot = null; - try { - while (snapshotReader.hasMoreEvents()) { - Message message = snapshotReader.readNextAsync() - .get(getSystemClientOperationTimeoutMs(), TimeUnit.MILLISECONDS); - if (topic.getName().equals(message.getKey())) { - TransactionBufferSnapshot transactionBufferSnapshot = - message.getValue(); - if (transactionBufferSnapshot != null) { - handleOldSnapshot(transactionBufferSnapshot); - startReadCursorPositionInOldSnapshot = PositionImpl.get( - transactionBufferSnapshot.getMaxReadPositionLedgerId(), - transactionBufferSnapshot.getMaxReadPositionEntryId()); - } - } - } - } catch (TimeoutException ex) { - Throwable t = FutureUtil.unwrapCompletionException(ex); - String errorMessage = String.format("[%s] Transaction buffer recover fail by " - + "read transactionBufferSnapshot timeout!", topic.getName()); - log.error(errorMessage, t); - return FutureUtil.failedFuture(new BrokerServiceException - .ServiceUnitNotReadyException(errorMessage, t)); - } catch (Exception ex) { - log.error("[{}] Transaction buffer recover fail when read " - + "transactionBufferSnapshot!", topic.getName(), ex); - return FutureUtil.failedFuture(ex); - } finally { - assert snapshotReader != null; - closeReader(snapshotReader); - } - return CompletableFuture.completedFuture(startReadCursorPositionInOldSnapshot); - }, - topic.getBrokerService().getPulsar().getTransactionExecutorProvider() - .getExecutor(this)); - } - }); + private Position recoverOldSnapshot() throws Exception { + final var pulsar = topic.getBrokerService().getPulsar(); + final var topicName = TopicName.get(topic.getName()); + final var topics = wait(pulsar.getPulsarResources().getTopicResources().listPersistentTopicsAsync( + NamespaceName.get(topicName.getNamespace())), "list persistent topics"); + if (!topics.contains(TopicDomain.persistent + "://" + topicName.getNamespace() + "/" + + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT)) { + return null; + } + final var snapshot = pulsar.getTransactionBufferSnapshotServiceFactory().getTxnBufferSnapshotService() + .getTableView().readLatest(topic.getName()); + if (snapshot == null) { + return null; + } + handleOldSnapshot(snapshot); + return PositionFactory.create(snapshot.getMaxReadPositionLedgerId(), snapshot.getMaxReadPositionEntryId()); } // This method will be deprecated and removed in version 4.x.0 @@ -448,9 +370,25 @@ public CompletableFuture clearAbortedTxnSnapshot() { persistentWorker::clearSnapshotSegmentAndIndexes); } - @Override - public long getLastSnapshotTimestamps() { - return this.lastSnapshotTimestamps; + public TransactionBufferStats generateSnapshotStats(boolean segmentStats) { + TransactionBufferStats transactionBufferStats = new TransactionBufferStats(); + transactionBufferStats.totalAbortedTransactions = this.aborts.size(); + transactionBufferStats.lastSnapshotTimestamps = this.lastSnapshotTimestamps; + SegmentsStats segmentsStats = new SegmentsStats(); + segmentsStats.currentSegmentCapacity = this.snapshotSegmentCapacity; + segmentsStats.lastTookSnapshotSegmentTimestamp = this.lastTakedSnapshotSegmentTimestamp; + segmentsStats.unsealedAbortTxnIDSize = this.unsealedTxnIds.size(); + segmentsStats.segmentsSize = indexes.size(); + if (segmentStats) { + List statsList = new ArrayList<>(); + segmentIndex.forEach((position, txnID) -> { + SegmentStats stats = new SegmentStats(txnID.toString(), position.toString()); + statsList.add(stats); + }); + segmentsStats.segmentStats = statsList; + } + transactionBufferStats.segmentsStats = segmentsStats; + return transactionBufferStats; } @Override @@ -467,7 +405,7 @@ private void handleSnapshotSegmentEntry(Entry entry) { .decode(Unpooled.wrappedBuffer(headersAndPayload).nioBuffer()); TxnIDData lastTxn = snapshotSegment.getAborts().get(snapshotSegment.getAborts().size() - 1); - segmentIndex.put(new PositionImpl(snapshotSegment.getPersistentPositionLedgerId(), + segmentIndex.put(PositionFactory.create(snapshotSegment.getPersistentPositionLedgerId(), snapshotSegment.getPersistentPositionEntryId()), new TxnID(lastTxn.getMostSigBits(), lastTxn.getLeastSigBits())); convertTypeToTxnID(snapshotSegment.getAborts()).forEach(txnID -> aborts.put(txnID, txnID)); @@ -478,9 +416,17 @@ private long getSystemClientOperationTimeoutMs() throws Exception { return pulsarClient.getConfiguration().getOperationTimeoutMs(); } - private void closeReader(SystemTopicClient.Reader reader) { + private R wait(CompletableFuture future, String msg) throws Exception { + try { + return future.get(getSystemClientOperationTimeoutMs(), TimeUnit.MILLISECONDS); + } catch (ExecutionException e) { + throw new CompletionException("Failed to " + msg, e.getCause()); + } + } + + private void closeReader(SystemTopicClient.Reader reader) { reader.closeAsync().exceptionally(e -> { - log.error("[{}]Transaction buffer snapshot reader close error!", topic.getName(), e); + log.warn("[{}] Failed to close reader: {}", topic.getName(), e.getMessage()); return null; }); } @@ -666,7 +612,7 @@ private void executeTask() { } private CompletableFuture takeSnapshotSegmentAsync(LinkedList sealedAbortedTxnIdSegment, - PositionImpl abortedMarkerPersistentPosition) { + Position abortedMarkerPersistentPosition) { CompletableFuture res = writeSnapshotSegmentAsync(sealedAbortedTxnIdSegment, abortedMarkerPersistentPosition).thenRun(() -> { if (log.isDebugEnabled()) { @@ -690,7 +636,7 @@ private CompletableFuture takeSnapshotSegmentAsync(LinkedList seale } private CompletableFuture writeSnapshotSegmentAsync(LinkedList segment, - PositionImpl abortedMarkerPersistentPosition) { + Position abortedMarkerPersistentPosition) { TransactionBufferSnapshotSegment transactionBufferSnapshotSegment = new TransactionBufferSnapshotSegment(); transactionBufferSnapshotSegment.setAborts(convertTypeToTxnIDData(segment)); transactionBufferSnapshotSegment.setTopicName(this.topic.getName()); @@ -702,6 +648,7 @@ private CompletableFuture writeSnapshotSegmentAsync(LinkedList segm transactionBufferSnapshotSegment.setSequenceId(this.sequenceID.get()); return segmentWriter.writeAsync(buildKey(this.sequenceID.get()), transactionBufferSnapshotSegment); }).thenCompose((messageId) -> { + lastTakedSnapshotSegmentTimestamp = System.currentTimeMillis(); //Build index for this segment TransactionBufferSnapshotIndex index = new TransactionBufferSnapshotIndex(); index.setSequenceID(transactionBufferSnapshotSegment.getSequenceId()); @@ -720,7 +667,7 @@ private CompletableFuture writeSnapshotSegmentAsync(LinkedList segm } private CompletionStage updateIndexWhenExecuteTheLatestTask() { - PositionImpl maxReadPosition = topic.getMaxReadPosition(); + Position maxReadPosition = topic.getMaxReadPosition(); List aborts = convertTypeToTxnIDData(unsealedTxnIds); if (taskQueue.size() != 1) { return CompletableFuture.completedFuture(null); @@ -731,9 +678,9 @@ private CompletionStage updateIndexWhenExecuteTheLatestTask() { } // update index after delete all segment. - private CompletableFuture deleteSnapshotSegment(List positionNeedToDeletes) { + private CompletableFuture deleteSnapshotSegment(List positionNeedToDeletes) { List> results = new ArrayList<>(); - for (PositionImpl positionNeedToDelete : positionNeedToDeletes) { + for (Position positionNeedToDelete : positionNeedToDeletes) { long sequenceIdNeedToDelete = indexes.get(positionNeedToDelete).getSequenceID(); CompletableFuture res = snapshotSegmentsWriter.getFuture() .thenCompose(writer -> writer.deleteAsync(buildKey(sequenceIdNeedToDelete), null)) @@ -806,25 +753,37 @@ private CompletableFuture clearSnapshotSegmentAndIndexes() { *

*/ private CompletableFuture clearAllSnapshotSegments() { - return topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory() - .getTxnBufferSnapshotSegmentService() - .createReader(TopicName.get(topic.getName())).thenComposeAsync(reader -> { - try { - while (reader.hasMoreEvents()) { - Message message = reader.readNextAsync() - .get(getSystemClientOperationTimeoutMs(), TimeUnit.MILLISECONDS); - if (topic.getName().equals(message.getValue().getTopicName())) { - snapshotSegmentsWriter.getFuture().get().write(message.getKey(), null); - } + final var future = new CompletableFuture(); + final var pulsar = topic.getBrokerService().getPulsar(); + pulsar.getTransactionExecutorProvider().getExecutor(this).execute(() -> { + try { + final var reader = wait(pulsar.getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotSegmentService().createReader(TopicName.get(topic.getName())) + , "create reader"); + try { + while (wait(reader.hasMoreEventsAsync(), "has more events")) { + final var message = wait(reader.readNextAsync(), "read next"); + if (topic.getName().equals(message.getValue().getTopicName())) { + snapshotSegmentsWriter.getFuture().get().write(message.getKey(), null); } - return CompletableFuture.completedFuture(null); - } catch (Exception ex) { - log.error("[{}] Transaction buffer clear snapshot segments fail!", topic.getName(), ex); - return FutureUtil.failedFuture(ex); - } finally { - closeReader(reader); } - }); + future.complete(null); + } finally { + closeReader(reader); + } + } catch (Throwable throwable) { + future.completeExceptionally(throwable); + } + }); + return future; + } + + private R wait(CompletableFuture future, String msg) throws Exception { + try { + return future.get(getSystemClientOperationTimeoutMs(), TimeUnit.MILLISECONDS); + } catch (ExecutionException e) { + throw new CompletionException("Failed to " + msg, e.getCause()); + } } synchronized CompletableFuture closeAsync() { @@ -850,4 +809,4 @@ private List convertTypeToTxnIDData(List abortedTxns) { return segment; } -} \ No newline at end of file +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TableView.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TableView.java new file mode 100644 index 0000000000000..7608a393cc980 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TableView.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.buffer.impl; + +import static org.apache.pulsar.broker.systopic.SystemTopicClient.Reader; +import com.google.common.annotations.VisibleForTesting; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.utils.SimpleCache; + +/** + * Compared with the more generic {@link org.apache.pulsar.client.api.TableView}, this table view + * - Provides just a single public method that reads the latest value synchronously. + * - Maintains multiple long-lived readers that will be expired after some time (1 minute by default). + */ +@Slf4j +public class TableView { + + // Remove the cached reader and snapshots if there is no refresh request in 1 minute + private static final long CACHE_EXPIRE_TIMEOUT_MS = 60 * 1000L; + private static final long CACHE_EXPIRE_CHECK_FREQUENCY_MS = 3000L; + @VisibleForTesting + protected final Function>> readerCreator; + private final Map snapshots = new ConcurrentHashMap<>(); + private final long clientOperationTimeoutMs; + private final SimpleCache> readers; + + public TableView(Function>> readerCreator, long clientOperationTimeoutMs, + ScheduledExecutorService executor) { + this.readerCreator = readerCreator; + this.clientOperationTimeoutMs = clientOperationTimeoutMs; + this.readers = new SimpleCache<>(executor, CACHE_EXPIRE_TIMEOUT_MS, CACHE_EXPIRE_CHECK_FREQUENCY_MS); + } + + public T readLatest(String topic) throws Exception { + final var reader = getReader(topic); + while (wait(reader.hasMoreEventsAsync(), "has more events")) { + final var msg = wait(reader.readNextAsync(), "read message"); + if (msg.getKey() != null) { + if (msg.getValue() != null) { + snapshots.put(msg.getKey(), msg.getValue()); + } else { + snapshots.remove(msg.getKey()); + } + } + } + return snapshots.get(topic); + } + + @VisibleForTesting + protected Reader getReader(String topic) { + final var topicName = TopicName.get(topic); + return readers.get(topicName.getNamespaceObject(), () -> { + try { + return wait(readerCreator.apply(topicName), "create reader"); + } catch (Exception e) { + throw new RuntimeException(e); + } + }, __ -> __.closeAsync().exceptionally(e -> { + log.warn("Failed to close reader {}", e.getMessage()); + return null; + })); + } + + private R wait(CompletableFuture future, String msg) throws Exception { + try { + return future.get(clientOperationTimeoutMs, TimeUnit.MILLISECONDS); + } catch (ExecutionException e) { + throw new CompletionException("Failed to " + msg, e.getCause()); + } + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 89a8e95afba1f..41977e6b61d88 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -37,7 +37,7 @@ import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.commons.collections4.map.LinkedMap; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException; @@ -56,6 +56,7 @@ import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.Markers; import org.apache.pulsar.common.util.Codec; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.RecoverTimeRecord; import org.jctools.queues.MessagePassingQueue; import org.jctools.queues.SpscArrayQueue; @@ -68,15 +69,15 @@ public class TopicTransactionBuffer extends TopicTransactionBufferState implemen private final PersistentTopic topic; - private volatile PositionImpl maxReadPosition; + private volatile Position maxReadPosition; /** * Ongoing transaction, map for remove txn stable position, linked for find max read position. */ - private final LinkedMap ongoingTxns = new LinkedMap<>(); + private final LinkedMap ongoingTxns = new LinkedMap<>(); - // when add abort or change max read position, the count will +1. Take snapshot will set 0 into it. - private final AtomicLong changeMaxReadPositionAndAddAbortTimes = new AtomicLong(); + // when change max read position, the count will +1. Take snapshot will reset the count. + private final AtomicLong changeMaxReadPositionCount = new AtomicLong(); private final LongAdder txnCommittedCounter = new LongAdder(); @@ -88,8 +89,12 @@ public class TopicTransactionBuffer extends TopicTransactionBufferState implemen private final int takeSnapshotIntervalTime; + private final CompletableFuture transactionBufferFuture = new CompletableFuture<>(); + private CompletableFuture publishFuture = getTransactionBufferFuture() + .thenApply(__ -> PositionFactory.EARLIEST); + /** * The map is used to store the lowWaterMarks which key is TC ID and value is lowWaterMark of the TC. */ @@ -101,6 +106,9 @@ public class TopicTransactionBuffer extends TopicTransactionBufferState implemen private final AbortedTxnProcessor snapshotAbortedTxnProcessor; + private final AbortedTxnProcessor.SnapshotType snapshotType; + private final MaxReadPositionCallBack maxReadPositionCallBack; + public TopicTransactionBuffer(PersistentTopic topic) { super(State.None); this.topic = topic; @@ -109,12 +117,15 @@ public TopicTransactionBuffer(PersistentTopic topic) { .getConfiguration().getTransactionBufferSnapshotMaxTransactionCount(); this.takeSnapshotIntervalTime = topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); - this.maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); + this.maxReadPosition = topic.getManagedLedger().getLastConfirmedEntry(); if (topic.getBrokerService().getPulsar().getConfiguration().isTransactionBufferSegmentedSnapshotEnabled()) { snapshotAbortedTxnProcessor = new SnapshotSegmentAbortedTxnProcessorImpl(topic); + snapshotType = AbortedTxnProcessor.SnapshotType.Segment; } else { snapshotAbortedTxnProcessor = new SingleSnapshotAbortedTxnProcessorImpl(topic); + snapshotType = AbortedTxnProcessor.SnapshotType.Single; } + this.maxReadPositionCallBack = topic.getMaxReadPositionCallBack(); this.recover(); } @@ -126,19 +137,19 @@ private void recover() { public void recoverComplete() { synchronized (TopicTransactionBuffer.this) { if (ongoingTxns.isEmpty()) { - maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); + maxReadPosition = topic.getManagedLedger().getLastConfirmedEntry(); } if (!changeToReadyState()) { log.error("[{}]Transaction buffer recover fail, current state: {}", topic.getName(), getState()); - transactionBufferFuture.completeExceptionally + getTransactionBufferFuture().completeExceptionally (new BrokerServiceException.ServiceUnitNotReadyException( "Transaction buffer recover failed to change the status to Ready," + "current state is: " + getState())); } else { timer.newTimeout(TopicTransactionBuffer.this, takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); - transactionBufferFuture.complete(null); + getTransactionBufferFuture().complete(null); recoverTime.setRecoverEndTime(System.currentTimeMillis()); } } @@ -147,11 +158,11 @@ public void recoverComplete() { @Override public void noNeedToRecover() { synchronized (TopicTransactionBuffer.this) { - maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); + maxReadPosition = topic.getManagedLedger().getLastConfirmedEntry(); if (!changeToNoSnapshotState()) { log.error("[{}]Transaction buffer recover fail", topic.getName()); } else { - transactionBufferFuture.complete(null); + getTransactionBufferFuture().complete(null); recoverTime.setRecoverEndTime(System.currentTimeMillis()); } } @@ -164,14 +175,16 @@ public void handleTxnEntry(Entry entry) { TopicTransactionBufferRecover.SUBSCRIPTION_NAME, -1); if (msgMetadata != null && msgMetadata.hasTxnidMostBits() && msgMetadata.hasTxnidLeastBits()) { TxnID txnID = new TxnID(msgMetadata.getTxnidMostBits(), msgMetadata.getTxnidLeastBits()); - PositionImpl position = PositionImpl.get(entry.getLedgerId(), entry.getEntryId()); - if (Markers.isTxnMarker(msgMetadata)) { - if (Markers.isTxnAbortMarker(msgMetadata)) { - snapshotAbortedTxnProcessor.putAbortedTxnAndPosition(txnID, position); + Position position = PositionFactory.create(entry.getLedgerId(), entry.getEntryId()); + synchronized (TopicTransactionBuffer.this) { + if (Markers.isTxnMarker(msgMetadata)) { + if (Markers.isTxnAbortMarker(msgMetadata)) { + snapshotAbortedTxnProcessor.putAbortedTxnAndPosition(txnID, position); + } + removeTxnAndUpdateMaxReadPosition(txnID); + } else { + handleTransactionMessage(txnID, position); } - updateMaxReadPosition(txnID); - } else { - handleTransactionMessage(txnID, position); } } } @@ -187,10 +200,10 @@ public void recoverExceptionally(Throwable e) { // if transaction buffer recover fail throw PulsarClientException, // we need to change the PulsarClientException to ServiceUnitNotReadyException, // the tc do op will retry - transactionBufferFuture.completeExceptionally + getTransactionBufferFuture().completeExceptionally (new BrokerServiceException.ServiceUnitNotReadyException(e.getMessage(), e)); } else { - transactionBufferFuture.completeExceptionally(e); + getTransactionBufferFuture().completeExceptionally(e); } recoverTime.setRecoverEndTime(System.currentTimeMillis()); topic.close(true); @@ -203,35 +216,19 @@ public CompletableFuture getTransactionMeta(TxnID txnID) { return CompletableFuture.completedFuture(null); } + @VisibleForTesting + public CompletableFuture getPublishFuture() { + return publishFuture; + } + + @VisibleForTesting + public CompletableFuture getTransactionBufferFuture() { + return transactionBufferFuture; + } + @Override - public CompletableFuture checkIfTBRecoverCompletely(boolean isTxnEnabled) { - if (!isTxnEnabled) { - return CompletableFuture.completedFuture(null); - } else { - CompletableFuture completableFuture = new CompletableFuture<>(); - transactionBufferFuture.thenRun(() -> { - if (checkIfNoSnapshot()) { - snapshotAbortedTxnProcessor.takeAbortedTxnsSnapshot(maxReadPosition).thenRun(() -> { - if (changeToReadyStateFromNoSnapshot()) { - timer.newTimeout(TopicTransactionBuffer.this, - takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); - } - completableFuture.complete(null); - }).exceptionally(exception -> { - log.error("Topic {} failed to take snapshot", this.topic.getName()); - completableFuture.completeExceptionally(exception); - return null; - }); - } else { - completableFuture.complete(null); - } - }).exceptionally(exception -> { - log.error("Topic {}: TransactionBuffer recover failed", this.topic.getName(), exception.getCause()); - completableFuture.completeExceptionally(exception.getCause()); - return null; - }); - return completableFuture; - } + public CompletableFuture checkIfTBRecoverCompletely() { + return getTransactionBufferFuture(); } @Override @@ -251,6 +248,45 @@ public long getCommittedTxnCount() { @Override public CompletableFuture appendBufferToTxn(TxnID txnId, long sequenceId, ByteBuf buffer) { + // Method `takeAbortedTxnsSnapshot` will be executed in the different thread. + // So we need to retain the buffer in this thread. It will be released after message persistent. + buffer.retain(); + CompletableFuture future = getPublishFuture().thenCompose(ignore -> { + if (checkIfNoSnapshot()) { + CompletableFuture completableFuture = new CompletableFuture<>(); + // `publishFuture` will be completed after message persistent, so there will not be two threads + // writing snapshots at the same time. + snapshotAbortedTxnProcessor.takeAbortedTxnsSnapshot(maxReadPosition).thenRun(() -> { + if (changeToReadyStateFromNoSnapshot()) { + timer.newTimeout(TopicTransactionBuffer.this, + takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); + completableFuture.complete(null); + } else { + log.error("[{}]Failed to change state of transaction buffer to Ready from NoSnapshot", + topic.getName()); + completableFuture.completeExceptionally(new BrokerServiceException.ServiceUnitNotReadyException( + "Transaction Buffer take first snapshot failed, the current state is: " + getState())); + } + }).exceptionally(exception -> { + log.error("Topic {} failed to take snapshot", this.topic.getName()); + completableFuture.completeExceptionally(exception); + return null; + }); + return completableFuture.thenCompose(__ -> internalAppendBufferToTxn(txnId, buffer)); + } else if (checkIfReady()) { + return internalAppendBufferToTxn(txnId, buffer); + } else { + // `publishFuture` will be completed after transaction buffer recover completely + // during initializing, so this case should not happen. + return FutureUtil.failedFuture(new BrokerServiceException.ServiceUnitNotReadyException( + "Transaction Buffer recover failed, the current state is: " + getState())); + } + }).whenComplete(((position, throwable) -> buffer.release())); + publishFuture = future; + return future; + } + + private CompletableFuture internalAppendBufferToTxn(TxnID txnId, ByteBuf buffer) { CompletableFuture completableFuture = new CompletableFuture<>(); Long lowWaterMark = lowWaterMarks.get(txnId.getMostSigBits()); if (lowWaterMark != null && lowWaterMark >= txnId.getLeastSigBits()) { @@ -280,13 +316,19 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { private void handleTransactionMessage(TxnID txnId, Position position) { if (!ongoingTxns.containsKey(txnId) && !this.snapshotAbortedTxnProcessor .checkAbortedTransaction(txnId)) { - ongoingTxns.put(txnId, (PositionImpl) position); - PositionImpl firstPosition = ongoingTxns.get(ongoingTxns.firstKey()); - //max read position is less than first ongoing transaction message position, so entryId -1 - maxReadPosition = PositionImpl.get(firstPosition.getLedgerId(), firstPosition.getEntryId() - 1); + ongoingTxns.put(txnId, position); + Position firstPosition = ongoingTxns.get(ongoingTxns.firstKey()); + // max read position is less than first ongoing transaction message position + updateMaxReadPosition(topic.getManagedLedger().getPreviousPosition(firstPosition), + false); } } + // ThreadSafe + private void updateLastDispatchablePosition(Position position) { + topic.updateLastDispatchablePosition(position); + } + @Override public CompletableFuture openTransactionBufferReader(TxnID txnID, long startSequenceId) { return null; @@ -299,7 +341,7 @@ public CompletableFuture commitTxn(TxnID txnID, long lowWaterMark) { } CompletableFuture completableFuture = new CompletableFuture<>(); //Wait TB recover completely. - transactionBufferFuture.thenRun(() -> { + getTransactionBufferFuture().thenRun(() -> { ByteBuf commitMarker = Markers.newTxnCommitMarker(-1L, txnID.getMostSigBits(), txnID.getLeastSigBits()); try { @@ -307,7 +349,7 @@ public CompletableFuture commitTxn(TxnID txnID, long lowWaterMark) { @Override public void addComplete(Position position, ByteBuf entryData, Object ctx) { synchronized (TopicTransactionBuffer.this) { - updateMaxReadPosition(txnID); + removeTxnAndUpdateMaxReadPosition(txnID); handleLowWaterMark(txnID, lowWaterMark); snapshotAbortedTxnProcessor.trimExpiredAbortedTxns(); takeSnapshotByChangeTimes(); @@ -341,7 +383,7 @@ public CompletableFuture abortTxn(TxnID txnID, long lowWaterMark) { } CompletableFuture completableFuture = new CompletableFuture<>(); //Wait TB recover completely. - transactionBufferFuture.thenRun(() -> { + getTransactionBufferFuture().thenRun(() -> { //no message sent, need not to add abort mark by txn timeout. if (!checkIfReady()) { completableFuture.complete(null); @@ -353,14 +395,14 @@ public CompletableFuture abortTxn(TxnID txnID, long lowWaterMark) { @Override public void addComplete(Position position, ByteBuf entryData, Object ctx) { synchronized (TopicTransactionBuffer.this) { - snapshotAbortedTxnProcessor.putAbortedTxnAndPosition(txnID, (PositionImpl) position); - updateMaxReadPosition(txnID); + snapshotAbortedTxnProcessor.putAbortedTxnAndPosition(txnID, position); + removeTxnAndUpdateMaxReadPosition(txnID); snapshotAbortedTxnProcessor.trimExpiredAbortedTxns(); takeSnapshotByChangeTimes(); + txnAbortedCounter.increment(); + completableFuture.complete(null); + handleLowWaterMark(txnID, lowWaterMark); } - txnAbortedCounter.increment(); - completableFuture.complete(null); - handleLowWaterMark(txnID, lowWaterMark); } @Override @@ -422,33 +464,56 @@ private void handleLowWaterMark(TxnID txnID, long lowWaterMark) { } private void takeSnapshotByChangeTimes() { - if (changeMaxReadPositionAndAddAbortTimes.get() >= takeSnapshotIntervalNumber) { - this.changeMaxReadPositionAndAddAbortTimes.set(0); + if (changeMaxReadPositionCount.get() >= takeSnapshotIntervalNumber) { + this.changeMaxReadPositionCount.set(0); this.snapshotAbortedTxnProcessor.takeAbortedTxnsSnapshot(this.maxReadPosition); } } private void takeSnapshotByTimeout() { - if (changeMaxReadPositionAndAddAbortTimes.get() > 0) { - this.changeMaxReadPositionAndAddAbortTimes.set(0); + if (changeMaxReadPositionCount.get() > 0) { + this.changeMaxReadPositionCount.set(0); this.snapshotAbortedTxnProcessor.takeAbortedTxnsSnapshot(this.maxReadPosition); } this.timer.newTimeout(TopicTransactionBuffer.this, takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); } - void updateMaxReadPosition(TxnID txnID) { - PositionImpl preMaxReadPosition = this.maxReadPosition; + /** + * remove the specified transaction from ongoing transaction list and update the max read position. + * @param txnID + */ + void removeTxnAndUpdateMaxReadPosition(TxnID txnID) { ongoingTxns.remove(txnID); if (!ongoingTxns.isEmpty()) { - PositionImpl position = ongoingTxns.get(ongoingTxns.firstKey()); - //max read position is less than first ongoing transaction message position, so entryId -1 - maxReadPosition = PositionImpl.get(position.getLedgerId(), position.getEntryId() - 1); + Position position = ongoingTxns.get(ongoingTxns.firstKey()); + updateMaxReadPosition(topic.getManagedLedger().getPreviousPosition(position), false); } else { - maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); + updateMaxReadPosition(topic.getManagedLedger().getLastConfirmedEntry(), false); } - if (preMaxReadPosition.compareTo(this.maxReadPosition) != 0) { - this.changeMaxReadPositionAndAddAbortTimes.getAndIncrement(); + // Update the last dispatchable position to null if there is a TXN finished. + updateLastDispatchablePosition(null); + } + + /** + * update the max read position. if the new position is greater than the current max read position, + * we will trigger the callback, unless the disableCallback is true. + * Currently, we only use the callback to update the lastMaxReadPositionMovedForwardTimestamp. + * For non-transactional production, some marker messages will be sent to the topic, in which case we don't need + * to trigger the callback. + * @param newPosition new max read position to update. + * @param disableCallback whether disable the callback. + */ + void updateMaxReadPosition(Position newPosition, boolean disableCallback) { + Position preMaxReadPosition = this.maxReadPosition; + this.maxReadPosition = newPosition; + if (preMaxReadPosition.compareTo(this.maxReadPosition) < 0) { + if (!checkIfNoSnapshot()) { + this.changeMaxReadPositionCount.getAndIncrement(); + } + if (!disableCallback) { + maxReadPositionCallBack.maxReadPositionMovedForward(preMaxReadPosition, this.maxReadPosition); + } } } @@ -469,49 +534,66 @@ public CompletableFuture closeAsync() { } @Override - public boolean isTxnAborted(TxnID txnID, PositionImpl readPosition) { + public synchronized boolean isTxnAborted(TxnID txnID, Position readPosition) { return snapshotAbortedTxnProcessor.checkAbortedTransaction(txnID); } + /** + * Sync max read position for normal publish. + * @param position {@link Position} the position to sync. + * @param isMarkerMessage whether the message is marker message, in such case, we + * don't need to trigger the callback to update lastMaxReadPositionMovedForwardTimestamp. + */ @Override - public void syncMaxReadPositionForNormalPublish(PositionImpl position) { + public void syncMaxReadPositionForNormalPublish(Position position, boolean isMarkerMessage) { // when ongoing transaction is empty, proved that lastAddConfirm is can read max position, because callback // thread is the same tread, in this time the lastAddConfirm don't content transaction message. synchronized (TopicTransactionBuffer.this) { if (checkIfNoSnapshot()) { - this.maxReadPosition = position; + updateMaxReadPosition(position, isMarkerMessage); } else if (checkIfReady()) { if (ongoingTxns.isEmpty()) { - maxReadPosition = position; - changeMaxReadPositionAndAddAbortTimes.incrementAndGet(); + updateMaxReadPosition(position, isMarkerMessage); } } } + // If the message is a normal message, update the last dispatchable position. + if (!isMarkerMessage) { + updateLastDispatchablePosition(position); + } } @Override - public PositionImpl getMaxReadPosition() { + public AbortedTxnProcessor.SnapshotType getSnapshotType() { + return snapshotType; + } + + @Override + public Position getMaxReadPosition() { if (checkIfReady() || checkIfNoSnapshot()) { return this.maxReadPosition; } else { - return PositionImpl.EARLIEST; + return PositionFactory.EARLIEST; } } @Override public TransactionInBufferStats getTransactionInBufferStats(TxnID txnID) { TransactionInBufferStats transactionInBufferStats = new TransactionInBufferStats(); - transactionInBufferStats.aborted = isTxnAborted(txnID, null); - if (ongoingTxns.containsKey(txnID)) { - transactionInBufferStats.startPosition = ongoingTxns.get(txnID).toString(); + synchronized (this) { + transactionInBufferStats.aborted = isTxnAborted(txnID, null); + if (ongoingTxns.containsKey(txnID)) { + transactionInBufferStats.startPosition = ongoingTxns.get(txnID).toString(); + } } return transactionInBufferStats; } @Override - public TransactionBufferStats getStats(boolean lowWaterMarks) { - TransactionBufferStats transactionBufferStats = new TransactionBufferStats(); - transactionBufferStats.lastSnapshotTimestamps = this.snapshotAbortedTxnProcessor.getLastSnapshotTimestamps(); + public TransactionBufferStats getStats(boolean lowWaterMarks, boolean segmentStats) { + TransactionBufferStats transactionBufferStats = this.snapshotAbortedTxnProcessor + .generateSnapshotStats(segmentStats); + transactionBufferStats.snapshotType = snapshotType.toString(); transactionBufferStats.state = this.getState().name(); transactionBufferStats.maxReadPosition = this.maxReadPosition.toString(); if (lowWaterMarks) { @@ -524,6 +606,11 @@ public TransactionBufferStats getStats(boolean lowWaterMarks) { return transactionBufferStats; } + @Override + public TransactionBufferStats getStats(boolean lowWaterMarks) { + return getStats(lowWaterMarks, false); + } + @Override public void run(Timeout timeout) { if (checkIfReady()) { @@ -542,7 +629,7 @@ public static class TopicTransactionBufferRecover implements Runnable { private final TopicTransactionBufferRecoverCallBack callBack; - private Position startReadCursorPosition = PositionImpl.EARLIEST; + private Position startReadCursorPosition = PositionFactory.EARLIEST; private final SpscArrayQueue entryQueue; @@ -572,7 +659,7 @@ public void run() { this, topic.getName()); return; } - abortedTxnProcessor.recoverFromSnapshot().thenAcceptAsync(startReadCursorPosition -> { + abortedTxnProcessor.recoverFromSnapshot().thenAccept(startReadCursorPosition -> { //Transaction is not use for this topic, so just make maxReadPosition as LAC. if (startReadCursorPosition == null) { callBack.noNeedToRecover(); @@ -589,9 +676,9 @@ public void run() { log.error("[{}]Transaction buffer recover fail when open cursor!", topic.getName(), e); return; } - PositionImpl lastConfirmedEntry = - (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); - PositionImpl currentLoadPosition = (PositionImpl) this.startReadCursorPosition; + Position lastConfirmedEntry = + topic.getManagedLedger().getLastConfirmedEntry(); + Position currentLoadPosition = this.startReadCursorPosition; FillEntryQueueCallback fillEntryQueueCallback = new FillEntryQueueCallback(entryQueue, managedCursor, TopicTransactionBufferRecover.this); if (lastConfirmedEntry.getEntryId() != -1) { @@ -600,7 +687,7 @@ public void run() { Entry entry = entryQueue.poll(); if (entry != null) { try { - currentLoadPosition = PositionImpl.get(entry.getLedgerId(), + currentLoadPosition = PositionFactory.create(entry.getLedgerId(), entry.getEntryId()); callBack.handleTxnEntry(entry); } finally { @@ -618,8 +705,7 @@ public void run() { closeCursor(SUBSCRIPTION_NAME); callBack.recoverComplete(); - }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider() - .getExecutor(this)).exceptionally(e -> { + }).exceptionally(e -> { callBack.recoverExceptionally(e.getCause()); log.error("[{}]Transaction buffer failed to recover snapshot!", topic.getName(), e); return null; @@ -655,6 +741,18 @@ private void closeReader(SystemTopicClient.Reader rea } } + /** + * A functional interface to handle the max read position move forward. + */ + public interface MaxReadPositionCallBack { + /** + * callback method when max read position move forward. + * @param oldPosition the old max read position. + * @param newPosition the new max read position. + */ + void maxReadPositionMovedForward(Position oldPosition, Position newPosition); + } + static class FillEntryQueueCallback implements AsyncCallbacks.ReadEntriesCallback { private final AtomicLong outstandingReadsRequests = new AtomicLong(0); @@ -681,7 +779,7 @@ boolean fillQueue() { if (cursor.hasMoreEntries()) { outstandingReadsRequests.incrementAndGet(); cursor.asyncReadEntries(NUMBER_OF_PER_READ_ENTRY, - this, System.nanoTime(), PositionImpl.LATEST); + this, System.nanoTime(), PositionFactory.LATEST); } else { if (entryQueue.size() == 0) { isReadable = false; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferClientImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferClientImpl.java index 382d640ca8658..96ad020390055 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferClientImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferClientImpl.java @@ -39,10 +39,11 @@ public class TransactionBufferClientImpl implements TransactionBufferClient { private final TransactionBufferHandler tbHandler; private final TransactionBufferClientStats stats; - private TransactionBufferClientImpl(TransactionBufferHandler tbHandler, boolean exposeTopicLevelMetrics, - boolean enableTxnCoordinator) { + private TransactionBufferClientImpl(PulsarService pulsarService, TransactionBufferHandler tbHandler, + boolean exposeTopicLevelMetrics, boolean enableTxnCoordinator) { this.tbHandler = tbHandler; - this.stats = TransactionBufferClientStats.create(exposeTopicLevelMetrics, tbHandler, enableTxnCoordinator); + this.stats = TransactionBufferClientStats.create(pulsarService, exposeTopicLevelMetrics, tbHandler, + enableTxnCoordinator); } public static TransactionBufferClient create(PulsarService pulsarService, HashedWheelTimer timer, @@ -53,7 +54,7 @@ public static TransactionBufferClient create(PulsarService pulsarService, Hashed ServiceConfiguration config = pulsarService.getConfig(); boolean exposeTopicLevelMetrics = config.isExposeTopicLevelMetricsInPrometheus(); boolean enableTxnCoordinator = config.isTransactionCoordinatorEnabled(); - return new TransactionBufferClientImpl(handler, exposeTopicLevelMetrics, enableTxnCoordinator); + return new TransactionBufferClientImpl(pulsarService, handler, exposeTopicLevelMetrics, enableTxnCoordinator); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferClientStatsImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferClientStatsImpl.java index a447f70789311..4f1c2ca30cf54 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferClientStatsImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferClientStatsImpl.java @@ -18,31 +18,55 @@ */ package org.apache.pulsar.broker.transaction.buffer.impl; +import io.opentelemetry.api.metrics.ObservableLongUpDownCounter; import io.prometheus.client.CollectorRegistry; import io.prometheus.client.Counter; import io.prometheus.client.Gauge; import io.prometheus.client.Summary; +import java.util.Objects; +import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.LongAdder; +import lombok.NonNull; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.service.BrokerService; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.service.persistent.PersistentTopicMetrics; +import org.apache.pulsar.broker.stats.OpenTelemetryTopicStats; import org.apache.pulsar.broker.transaction.buffer.TransactionBufferClientStats; import org.apache.pulsar.client.impl.transaction.TransactionBufferHandler; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.opentelemetry.annotations.PulsarDeprecatedMetric; public final class TransactionBufferClientStatsImpl implements TransactionBufferClientStats { private static final double[] QUANTILES = {0.50, 0.75, 0.95, 0.99, 0.999, 0.9999, 1}; private final AtomicBoolean closed = new AtomicBoolean(false); + @PulsarDeprecatedMetric(newMetricName = OpenTelemetryTopicStats.TRANSACTION_BUFFER_CLIENT_OPERATION_COUNTER) private final Counter abortFailed; + @PulsarDeprecatedMetric(newMetricName = OpenTelemetryTopicStats.TRANSACTION_BUFFER_CLIENT_OPERATION_COUNTER) private final Counter commitFailed; + @PulsarDeprecatedMetric(newMetricName = OpenTelemetryTopicStats.TRANSACTION_BUFFER_CLIENT_OPERATION_COUNTER) private final Summary abortLatency; + @PulsarDeprecatedMetric(newMetricName = OpenTelemetryTopicStats.TRANSACTION_BUFFER_CLIENT_OPERATION_COUNTER) private final Summary commitLatency; + + public static final String PENDING_TRANSACTION_COUNTER = "pulsar.broker.transaction.buffer.client.pending.count"; + private final ObservableLongUpDownCounter pendingTransactionCounter; + + @PulsarDeprecatedMetric(newMetricName = PENDING_TRANSACTION_COUNTER) private final Gauge pendingRequests; private final boolean exposeTopicLevelMetrics; + private final BrokerService brokerService; + private static TransactionBufferClientStats instance; - private TransactionBufferClientStatsImpl(boolean exposeTopicLevelMetrics, - TransactionBufferHandler handler) { + private TransactionBufferClientStatsImpl(@NonNull PulsarService pulsarService, + boolean exposeTopicLevelMetrics, + @NonNull TransactionBufferHandler handler) { + this.brokerService = Objects.requireNonNull(pulsarService.getBrokerService()); this.exposeTopicLevelMetrics = exposeTopicLevelMetrics; String[] labelNames = exposeTopicLevelMetrics ? new String[]{"namespace", "topic"} : new String[]{"namespace"}; @@ -63,9 +87,14 @@ private TransactionBufferClientStatsImpl(boolean exposeTopicLevelMetrics, .setChild(new Gauge.Child() { @Override public double get() { - return null == handler ? 0 : handler.getPendingRequestsCount(); + return handler.getPendingRequestsCount(); } }); + this.pendingTransactionCounter = pulsarService.getOpenTelemetry().getMeter() + .upDownCounterBuilder(PENDING_TRANSACTION_COUNTER) + .setDescription("The number of pending transactions in the transaction buffer client.") + .setUnit("{transaction}") + .buildWithCallback(measurement -> measurement.record(handler.getPendingRequestsCount())); } private Summary buildSummary(String name, String help, String[] labelNames) { @@ -77,33 +106,52 @@ private Summary buildSummary(String name, String help, String[] labelNames) { return builder.register(); } - public static synchronized TransactionBufferClientStats getInstance(boolean exposeTopicLevelMetrics, + public static synchronized TransactionBufferClientStats getInstance(PulsarService pulsarService, + boolean exposeTopicLevelMetrics, TransactionBufferHandler handler) { if (null == instance) { - instance = new TransactionBufferClientStatsImpl(exposeTopicLevelMetrics, handler); + instance = new TransactionBufferClientStatsImpl(pulsarService, exposeTopicLevelMetrics, handler); } - return instance; } @Override public void recordAbortFailed(String topic) { this.abortFailed.labels(labelValues(topic)).inc(); + getTransactionBufferClientMetrics(topic) + .map(PersistentTopicMetrics.TransactionBufferClientMetrics::getAbortFailedCount) + .ifPresent(LongAdder::increment); } @Override public void recordCommitFailed(String topic) { this.commitFailed.labels(labelValues(topic)).inc(); + getTransactionBufferClientMetrics(topic) + .map(PersistentTopicMetrics.TransactionBufferClientMetrics::getCommitFailedCount) + .ifPresent(LongAdder::increment); } @Override public void recordAbortLatency(String topic, long nanos) { this.abortLatency.labels(labelValues(topic)).observe(nanos); + getTransactionBufferClientMetrics(topic) + .map(PersistentTopicMetrics.TransactionBufferClientMetrics::getAbortSucceededCount) + .ifPresent(LongAdder::increment); } @Override public void recordCommitLatency(String topic, long nanos) { this.commitLatency.labels(labelValues(topic)).observe(nanos); + getTransactionBufferClientMetrics(topic) + .map(PersistentTopicMetrics.TransactionBufferClientMetrics::getCommitSucceededCount) + .ifPresent(LongAdder::increment); + } + + private Optional getTransactionBufferClientMetrics( + String topic) { + return brokerService.getTopicReference(topic) + .filter(t -> t instanceof PersistentTopic) + .map(t -> ((PersistentTopic) t).getPersistentTopicMetrics().getTransactionBufferClientMetrics()); } private String[] labelValues(String topic) { @@ -125,6 +173,7 @@ public void close() { CollectorRegistry.defaultRegistry.unregister(this.abortLatency); CollectorRegistry.defaultRegistry.unregister(this.commitLatency); CollectorRegistry.defaultRegistry.unregister(this.pendingRequests); + pendingTransactionCounter.close(); } } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferDisable.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferDisable.java index 22ba8e2d2e8ef..d4fd071fef8a7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferDisable.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferDisable.java @@ -23,8 +23,10 @@ import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.service.BrokerServiceException; +import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; import org.apache.pulsar.broker.transaction.buffer.TransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.TransactionBufferReader; import org.apache.pulsar.broker.transaction.buffer.TransactionMeta; @@ -39,6 +41,17 @@ @Slf4j public class TransactionBufferDisable implements TransactionBuffer { + private final Topic topic; + private final TopicTransactionBuffer.MaxReadPositionCallBack maxReadPositionCallBack; + public TransactionBufferDisable(Topic topic) { + this.topic = topic; + if (topic instanceof PersistentTopic) { + this.maxReadPositionCallBack = ((PersistentTopic) topic).getMaxReadPositionCallBack(); + } else { + this.maxReadPositionCallBack = null; + } + } + @Override public CompletableFuture getTransactionMeta(TxnID txnID) { return CompletableFuture.completedFuture(null); @@ -79,18 +92,28 @@ public CompletableFuture closeAsync() { } @Override - public boolean isTxnAborted(TxnID txnID, PositionImpl readPosition) { + public boolean isTxnAborted(TxnID txnID, Position readPosition) { return false; } @Override - public void syncMaxReadPositionForNormalPublish(PositionImpl position) { - //no-op + public void syncMaxReadPositionForNormalPublish(Position position, boolean isMarkerMessage) { + if (!isMarkerMessage) { + updateLastDispatchablePosition(position); + if (maxReadPositionCallBack != null) { + maxReadPositionCallBack.maxReadPositionMovedForward(null, position); + } + } + } + + @Override + public Position getMaxReadPosition() { + return topic.getLastPosition(); } @Override - public PositionImpl getMaxReadPosition() { - return PositionImpl.LATEST; + public AbortedTxnProcessor.SnapshotType getSnapshotType() { + return null; } @Override @@ -98,13 +121,18 @@ public TransactionInBufferStats getTransactionInBufferStats(TxnID txnID) { return null; } + @Override + public TransactionBufferStats getStats(boolean lowWaterMarks, boolean segmentStats) { + return null; + } + @Override public TransactionBufferStats getStats(boolean lowWaterMarks) { return null; } @Override - public CompletableFuture checkIfTBRecoverCompletely(boolean isTxn) { + public CompletableFuture checkIfTBRecoverCompletely() { return CompletableFuture.completedFuture(null); } @@ -122,4 +150,11 @@ public long getAbortedTxnCount() { public long getCommittedTxnCount() { return 0; } + + // ThreadSafe + private void updateLastDispatchablePosition(Position position) { + if (topic instanceof PersistentTopic t) { + t.updateLastDispatchablePosition(position); + } + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java index 48dcf259edb1b..34ee28693b4fc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java @@ -31,6 +31,7 @@ import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicLong; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.namespace.NamespaceEphemeralData; @@ -61,6 +62,8 @@ public class TransactionBufferHandlerImpl implements TransactionBufferHandler { private final PulsarService pulsarService; private final PulsarClientImpl pulsarClient; + private final int randomKeyForSelectConnection; + private static final AtomicIntegerFieldUpdater REQUEST_CREDITS_UPDATER = AtomicIntegerFieldUpdater.newUpdater(TransactionBufferHandlerImpl.class, "requestCredits"); private volatile int requestCredits; @@ -74,6 +77,7 @@ public TransactionBufferHandlerImpl(PulsarService pulsarService, HashedWheelTime this.operationTimeoutInMills = operationTimeoutInMills; this.timer = timer; this.requestCredits = Math.max(100, maxConcurrentRequests); + this.randomKeyForSelectConnection = pulsarClient.getCnxPool().genRandomKeyToSelectCon(); } @Override @@ -134,8 +138,9 @@ public void endTxn(OpRequestSend op) { if (clientCnx.ctx().channel().isActive()) { clientCnx.registerTransactionBufferHandler(TransactionBufferHandlerImpl.this); outstandingRequests.put(op.requestId, op); + final long requestId = op.requestId; timer.newTimeout(timeout -> { - OpRequestSend peek = outstandingRequests.remove(op.requestId); + OpRequestSend peek = outstandingRequests.remove(requestId); if (peek != null && !peek.cb.isDone() && !peek.cb.isCompletedExceptionally()) { peek.cb.completeExceptionally(new TransactionBufferClientException .RequestTimeoutException()); @@ -296,7 +301,7 @@ protected OpRequestSend newObject(Handle handle) { } public CompletableFuture getClientCnxWithLookup(String topic) { - return pulsarClient.getConnection(topic); + return pulsarClient.getConnection(topic, randomKeyForSelectConnection).thenApply(Pair::getLeft); } public CompletableFuture getClientCnx(String topic) { @@ -317,7 +322,8 @@ public CompletableFuture getClientCnx(String topic) { } InetSocketAddress brokerAddress = InetSocketAddress.createUnresolved(uri.getHost(), uri.getPort()); - return pulsarClient.getConnection(brokerAddress, brokerAddress); + return pulsarClient.getConnection(brokerAddress, brokerAddress, + randomKeyForSelectConnection); } else { // Bundle is unloading, lookup topic return getClientCnxWithLookup(topic); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java index a7892a56f0bd5..dcebbb2829eec 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java @@ -22,7 +22,6 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException; import org.apache.pulsar.broker.service.Consumer; @@ -54,9 +53,9 @@ public interface PendingAckHandle { * @return the future of this operation. * @throws TransactionConflictException if the ack with transaction is conflict with pending ack. * @throws NotAllowedException if Use this method incorrectly eg. not use - * PositionImpl or cumulative ack with a list of positions. + * Position or cumulative ack with a list of positions. */ - CompletableFuture individualAcknowledgeMessage(TxnID txnID, List individualAcknowledgeMessage(TxnID txnID, List> positions); /** * Acknowledge message(s) for an ongoing transaction. @@ -78,9 +77,9 @@ CompletableFuture individualAcknowledgeMessage(TxnID txnID, List cumulativeAcknowledgeMessage(TxnID txnID, List positions); + CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, List positions); /** * Commit a transaction. @@ -108,14 +107,14 @@ CompletableFuture individualAcknowledgeMessage(TxnID txnID, List individualAcknowledgeMessage(TxnID txnID, List individualAcknowledgeMessage(TxnID txnID, List individualAcknowledgeMessage(TxnID txnID, List appendIndividualAck(TxnID txnID, List> positions); + CompletableFuture appendIndividualAck(TxnID txnID, List> positions); /** * Append the cumulative pending ack operation to the ack persistent store. * * @param txnID {@link TxnID} transaction id. - * @param position {@link PositionImpl} the pending ack position. + * @param position {@link Position} the pending ack position. * @return a future represents the result of this operation */ - CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position); + CompletableFuture appendCumulativeAck(TxnID txnID, Position position); /** * Append the pending ack commit mark to the ack persistent store. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java index 0840e2c2f45dd..e022dba09028c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java @@ -21,7 +21,7 @@ import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; import org.apache.pulsar.client.api.transaction.TxnID; @@ -44,12 +44,12 @@ public CompletableFuture closeAsync() { @Override public CompletableFuture appendIndividualAck(TxnID txnID, - List> positions) { + List> positions) { return CompletableFuture.completedFuture(null); } @Override - public CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position) { + public CompletableFuture appendCumulativeAck(TxnID txnID, Position position) { return CompletableFuture.completedFuture(null); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java index 9900d29725f21..b32dcbf3101a9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java @@ -21,7 +21,9 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.mledger.impl.AckSetStateUtil; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.transaction.pendingack.PendingAckReplyCallBack; @@ -87,23 +89,26 @@ public void handleMetadataEntry(PendingAckMetadataEntry pendingAckMetadataEntry) PendingAckMetadata pendingAckMetadata = pendingAckMetadataEntry.getPendingAckMetadatasList().get(0); pendingAckHandle.handleCumulativeAckRecover(txnID, - PositionImpl.get(pendingAckMetadata.getLedgerId(), pendingAckMetadata.getEntryId())); + PositionFactory.create(pendingAckMetadata.getLedgerId(), pendingAckMetadata.getEntryId())); } else { - List> positions = new ArrayList<>(); + List> positions = new ArrayList<>(); pendingAckMetadataEntry.getPendingAckMetadatasList().forEach(pendingAckMetadata -> { if (pendingAckMetadata.getAckSetsCount() == 0) { - positions.add(new MutablePair<>(PositionImpl.get(pendingAckMetadata.getLedgerId(), + positions.add(new MutablePair<>(PositionFactory.create(pendingAckMetadata.getLedgerId(), pendingAckMetadata.getEntryId()), pendingAckMetadata.getBatchSize())); } else { - PositionImpl position = - PositionImpl.get(pendingAckMetadata.getLedgerId(), pendingAckMetadata.getEntryId()); + long[] ackSets = null; if (pendingAckMetadata.getAckSetsCount() > 0) { - long[] ackSets = new long[pendingAckMetadata.getAckSetsCount()]; + ackSets = new long[pendingAckMetadata.getAckSetsCount()]; for (int i = 0; i < pendingAckMetadata.getAckSetsCount(); i++) { ackSets[i] = pendingAckMetadata.getAckSetAt(i); } - position.setAckSet(ackSets); + } else { + ackSets = new long[0]; } + Position position = + AckSetStateUtil.createPositionWithAckSet(pendingAckMetadata.getLedgerId(), + pendingAckMetadata.getEntryId(), ackSets); positions.add(new MutablePair<>(position, pendingAckMetadata.getBatchSize())); } }); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java index 4dce8b9a0fcf4..25c7727259db3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java @@ -32,6 +32,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Predicate; @@ -42,8 +43,8 @@ import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.mledger.impl.AckSetStateUtil; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException; import org.apache.pulsar.broker.transaction.pendingack.PendingAckReplyCallBack; @@ -80,20 +81,20 @@ public class MLPendingAckStore implements PendingAckStore { private final SpscArrayQueue entryQueue; //this is for replay - private final PositionImpl lastConfirmedEntry; + private final Position lastConfirmedEntry; - private PositionImpl currentLoadPosition; + private Position currentLoadPosition; private final AtomicLong currentIndexLag = new AtomicLong(0); private volatile long maxIndexLag; - protected PositionImpl maxAckPosition = PositionImpl.EARLIEST; + protected Position maxAckPosition = PositionFactory.EARLIEST; private final LogIndexLagBackoff logIndexBackoff; /** - * If the Batch feature is enabled by {@link #bufferedWriter}, {@link #handleMetadataEntry(PositionImpl, List)} is + * If the Batch feature is enabled by {@link #bufferedWriter}, {@link #handleMetadataEntry(Position, List)} is * executed after all data in the batch is written, instead of - * {@link #handleMetadataEntry(PositionImpl, PendingAckMetadataEntry)} after each data is written. This is because + * {@link #handleMetadataEntry(Position, PendingAckMetadataEntry)} after each data is written. This is because * method {@link #clearUselessLogData()} deletes the data in the unit of Entry. */ private final ArrayList batchedPendingAckLogsWaitingForHandle; @@ -111,7 +112,7 @@ public class MLPendingAckStore implements PendingAckStore { * If the max position (key) is smaller than the subCursor mark delete position, * the log cursor will mark delete the position before log position (value). */ - final ConcurrentSkipListMap pendingAckLogIndex; + final ConcurrentSkipListMap pendingAckLogIndex; private final ManagedCursor subManagedCursor; @@ -120,17 +121,17 @@ public class MLPendingAckStore implements PendingAckStore { public MLPendingAckStore(ManagedLedger managedLedger, ManagedCursor cursor, ManagedCursor subManagedCursor, long transactionPendingAckLogIndexMinLag, TxnLogBufferedWriterConfig bufferedWriterConfig, - Timer timer, TxnLogBufferedWriterMetricsStats bufferedWriterMetrics) { + Timer timer, TxnLogBufferedWriterMetricsStats bufferedWriterMetrics, Executor executor) { this.managedLedger = managedLedger; this.cursor = cursor; - this.currentLoadPosition = (PositionImpl) this.cursor.getMarkDeletedPosition(); + this.currentLoadPosition = this.cursor.getMarkDeletedPosition(); this.entryQueue = new SpscArrayQueue<>(2000); - this.lastConfirmedEntry = (PositionImpl) managedLedger.getLastConfirmedEntry(); + this.lastConfirmedEntry = managedLedger.getLastConfirmedEntry(); this.pendingAckLogIndex = new ConcurrentSkipListMap<>(); this.subManagedCursor = subManagedCursor; this.logIndexBackoff = new LogIndexLagBackoff(transactionPendingAckLogIndexMinLag, Long.MAX_VALUE, 1); this.maxIndexLag = logIndexBackoff.next(0); - this.bufferedWriter = new TxnLogBufferedWriter(managedLedger, ((ManagedLedgerImpl) managedLedger).getExecutor(), + this.bufferedWriter = new TxnLogBufferedWriter(managedLedger, executor, timer, PendingAckLogSerializer.INSTANCE, bufferedWriterConfig.getBatchedWriteMaxRecords(), bufferedWriterConfig.getBatchedWriteMaxSize(), bufferedWriterConfig.getBatchedWriteMaxDelayInMillis(), bufferedWriterConfig.isBatchEnabled(), @@ -147,7 +148,7 @@ public void replayAsync(PendingAckHandleImpl pendingAckHandle, ExecutorService t //TODO can control the number of entry to read private void readAsync(int numberOfEntriesToRead, AsyncCallbacks.ReadEntriesCallback readEntriesCallback) { - cursor.asyncReadEntries(numberOfEntriesToRead, readEntriesCallback, System.nanoTime(), PositionImpl.LATEST); + cursor.asyncReadEntries(numberOfEntriesToRead, readEntriesCallback, System.nanoTime(), PositionFactory.LATEST); } @Override @@ -186,17 +187,18 @@ public void closeFailed(ManagedLedgerException exception, Object ctx) { @Override public CompletableFuture appendIndividualAck(TxnID txnID, - List> positions) { + List> positions) { PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry(); pendingAckMetadataEntry.setPendingAckOp(PendingAckOp.ACK); pendingAckMetadataEntry.setAckType(AckType.Individual); List pendingAckMetadataList = new ArrayList<>(); positions.forEach(positionIntegerMutablePair -> { PendingAckMetadata pendingAckMetadata = new PendingAckMetadata(); - PositionImpl position = positionIntegerMutablePair.getLeft(); + Position position = positionIntegerMutablePair.getLeft(); int batchSize = positionIntegerMutablePair.getRight(); - if (positionIntegerMutablePair.getLeft().getAckSet() != null) { - for (long l : position.getAckSet()) { + long[] positionAckSet = AckSetStateUtil.getAckSetArrayOrNull(position); + if (positionAckSet != null) { + for (long l : positionAckSet) { pendingAckMetadata.addAckSet(l); } } @@ -210,13 +212,14 @@ public CompletableFuture appendIndividualAck(TxnID txnID, } @Override - public CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position) { + public CompletableFuture appendCumulativeAck(TxnID txnID, Position position) { PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry(); pendingAckMetadataEntry.setPendingAckOp(PendingAckOp.ACK); pendingAckMetadataEntry.setAckType(AckType.Cumulative); PendingAckMetadata pendingAckMetadata = new PendingAckMetadata(); - if (position.getAckSet() != null) { - for (long l : position.getAckSet()) { + long[] positionAckSet = AckSetStateUtil.getAckSetArrayOrNull(position); + if (positionAckSet != null) { + for (long l : positionAckSet) { pendingAckMetadata.addAckSet(l); } } @@ -257,8 +260,8 @@ public void addComplete(Position position, Object ctx) { currentIndexLag.incrementAndGet(); /** * If the Batch feature is enabled by {@link #bufferedWriter}, - * {@link #handleMetadataEntry(PositionImpl, List)} is executed after all data in the batch is written, - * instead of {@link #handleMetadataEntry(PositionImpl, PendingAckMetadataEntry)} after each data is + * {@link #handleMetadataEntry(Position, List)} is executed after all data in the batch is written, + * instead of {@link #handleMetadataEntry(Position, PendingAckMetadataEntry)} after each data is * written. This is because method {@link #clearUselessLogData()} deletes the data in the unit of Entry. * {@link TxnLogBufferedWriter.AddDataCallback#addComplete} for elements in a batch is executed * simultaneously and in strict order, so when the last element in a batch is complete, the whole @@ -267,11 +270,11 @@ public void addComplete(Position position, Object ctx) { if (position instanceof TxnBatchedPositionImpl batchedPosition){ batchedPendingAckLogsWaitingForHandle.add(pendingAckMetadataEntry); if (batchedPosition.getBatchIndex() == batchedPosition.getBatchSize() - 1){ - handleMetadataEntry((PositionImpl) position, batchedPendingAckLogsWaitingForHandle); + handleMetadataEntry(position, batchedPendingAckLogsWaitingForHandle); batchedPendingAckLogsWaitingForHandle.clear(); } } else { - handleMetadataEntry((PositionImpl) position, pendingAckMetadataEntry); + handleMetadataEntry(position, pendingAckMetadataEntry); } completableFuture.complete(null); clearUselessLogData(); @@ -301,7 +304,7 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { * @param logPosition The position of batch log Entry. * @param logList Pending ack log records in a batch log Entry. */ - private void handleMetadataEntry(PositionImpl logPosition, + private void handleMetadataEntry(Position logPosition, List logList) { Stream pendingAckMetaStream = logList.stream() .filter(log -> bothNotAbortAndCommitPredicate.test(log)) @@ -313,7 +316,7 @@ private void handleMetadataEntry(PositionImpl logPosition, pendingAckLog.getPendingAckOp() != PendingAckOp.ABORT && pendingAckLog.getPendingAckOp() != PendingAckOp.COMMIT; - private void handleMetadataEntry(PositionImpl logPosition, + private void handleMetadataEntry(Position logPosition, PendingAckMetadataEntry pendingAckMetadataEntry) { // store the persistent position in to memory // store the max position of this entry retain @@ -322,14 +325,14 @@ private void handleMetadataEntry(PositionImpl logPosition, } } - private void handleMetadataEntry(PositionImpl logPosition, Stream pendingAckListStream) { + private void handleMetadataEntry(Position logPosition, Stream pendingAckListStream) { // store the persistent position in to memory // store the max position of this entry retain Optional optional = pendingAckListStream .max((o1, o2) -> ComparisonChain.start().compare(o1.getLedgerId(), o2.getLedgerId()).compare(o1.getEntryId(), o2.getEntryId()).result()); optional.ifPresent(pendingAckMetadata -> { - PositionImpl nowPosition = PositionImpl.get(pendingAckMetadata.getLedgerId(), + Position nowPosition = PositionFactory.create(pendingAckMetadata.getLedgerId(), pendingAckMetadata.getEntryId()); if (nowPosition.compareTo(maxAckPosition) > 0) { maxAckPosition = nowPosition; @@ -346,18 +349,18 @@ private void handleMetadataEntry(PositionImpl logPosition, Stream 0 && fillEntryQueueCallback.fillQueue()) { Entry entry = entryQueue.poll(); if (entry != null) { - currentLoadPosition = PositionImpl.get(entry.getLedgerId(), entry.getEntryId()); + currentLoadPosition = PositionFactory.create(entry.getLedgerId(), entry.getEntryId()); List logs = deserializeEntry(entry); if (logs.isEmpty()){ continue; } else if (logs.size() == 1){ currentIndexLag.incrementAndGet(); PendingAckMetadataEntry log = logs.get(0); - handleMetadataEntry(new PositionImpl(entry.getLedgerId(), entry.getEntryId()), log); + handleMetadataEntry(PositionFactory.create(entry.getLedgerId(), entry.getEntryId()), log); pendingAckReplyCallBack.handleMetadataEntry(log); } else { int batchSize = logs.size(); @@ -419,7 +422,7 @@ public void run() { pendingAckReplyCallBack.handleMetadataEntry(log); } currentIndexLag.addAndGet(batchSize); - handleMetadataEntry(new PositionImpl(entry.getLedgerId(), entry.getEntryId()), logs); + handleMetadataEntry(PositionFactory.create(entry.getLedgerId(), entry.getEntryId()), logs); } entry.release(); clearUselessLogData(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java index ecc6599ce52b5..6fc61d423ce85 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java @@ -134,7 +134,12 @@ public void openCursorComplete(ManagedCursor cursor, Object ctx) { .getConfiguration() .getTransactionPendingAckLogIndexMinLag(), txnLogBufferedWriterConfig, - brokerClientSharedTimer, bufferedWriterMetrics)); + brokerClientSharedTimer, bufferedWriterMetrics, + originPersistentTopic + .getBrokerService() + .getPulsar() + .getOrderedExecutor() + .chooseThread())); if (log.isDebugEnabled()) { log.debug("{},{} open MLPendingAckStore cursor success", originPersistentTopic.getName(), @@ -159,7 +164,7 @@ public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { , originPersistentTopic.getName(), subscription.getName(), exception); pendingAckStoreFuture.completeExceptionally(exception); } - }, () -> true, null); + }, () -> CompletableFuture.completedFuture(true), null); }).exceptionally(e -> { Throwable t = FutureUtil.unwrapCompletionException(e); log.error("[{}] [{}] Failed to get managedLedger config when init pending ack store!", diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java index 0fc528f880070..fb633f7af65dd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java @@ -22,11 +22,11 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException; import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.broker.transaction.pendingack.PendingAckHandle; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckHandleStats; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.policies.data.TransactionInPendingAckStats; import org.apache.pulsar.common.policies.data.TransactionPendingAckStats; @@ -43,12 +43,12 @@ public class PendingAckHandleDisabled implements PendingAckHandle { @Override public CompletableFuture individualAcknowledgeMessage(TxnID txnID, - List> positions) { + List> positions) { return FutureUtil.failedFuture(new NotAllowedException("The transaction is disabled")); } @Override - public CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, List positions) { + public CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, List positions) { return FutureUtil.failedFuture(new NotAllowedException("The transaction is disabled")); } @@ -63,12 +63,12 @@ public CompletableFuture abortTxn(TxnID txnId, Consumer consumer, long low } @Override - public void syncBatchPositionAckSetForTransaction(PositionImpl position) { + public void syncBatchPositionAckSetForTransaction(Position position) { //no operation } @Override - public boolean checkIsCanDeleteConsumerPendingAck(PositionImpl position) { + public boolean checkIsCanDeleteConsumerPendingAck(Position position) { return false; } @@ -92,6 +92,11 @@ public TransactionPendingAckStats getStats(boolean lowWaterMarks) { return null; } + @Override + public PendingAckHandleStats getPendingAckHandleStats() { + return null; + } + @Override public CompletableFuture closeAsync() { return CompletableFuture.completedFuture(null); @@ -103,10 +108,10 @@ public boolean checkIfPendingAckStoreInit() { } @Override - public PositionImpl getPositionInPendingAck(PositionImpl position) { + public Position getPositionInPendingAck(Position position) { return null; } - public PositionInPendingAckStats checkPositionInPendingAckState(PositionImpl position, Integer batchIndex) { + public PositionInPendingAckStats checkPositionInPendingAckState(Position position, Integer batchIndex) { return null; } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java index 7dbe0385fd7e9..591842927f35b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java @@ -18,10 +18,13 @@ */ package org.apache.pulsar.broker.transaction.pendingack.impl; +import static org.apache.bookkeeper.mledger.impl.AckSetStateUtil.createPositionWithAckSet; +import static org.apache.bookkeeper.mledger.impl.AckSetStateUtil.getAckSetArrayOrNull; import static org.apache.bookkeeper.mledger.util.PositionAckSetUtil.andAckSet; import static org.apache.bookkeeper.mledger.util.PositionAckSetUtil.compareToWithAckSet; import static org.apache.bookkeeper.mledger.util.PositionAckSetUtil.isAckSetOverlap; import com.google.common.annotations.VisibleForTesting; +import io.netty.util.Timer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -35,16 +38,19 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.client.api.BKException; import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.impl.AckSetState; +import org.apache.bookkeeper.mledger.impl.AckSetStateUtil; import org.apache.commons.collections4.map.LinkedMap; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.commons.lang3.tuple.Pair; -import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException; import org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException; import org.apache.pulsar.broker.service.Consumer; @@ -53,14 +59,17 @@ import org.apache.pulsar.broker.transaction.pendingack.PendingAckHandleStats; import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.api.proto.CommandAck.AckType; import org.apache.pulsar.common.policies.data.TransactionInPendingAckStats; import org.apache.pulsar.common.policies.data.TransactionPendingAckStats; import org.apache.pulsar.common.stats.PositionInPendingAckStats; +import org.apache.pulsar.common.util.Backoff; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.RecoverTimeRecord; import org.apache.pulsar.common.util.collections.BitSetRecyclable; +import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.transaction.common.exception.TransactionConflictException; /** @@ -78,7 +87,7 @@ public class PendingAckHandleImpl extends PendingAckHandleState implements Pendi * If the position is batch position and it exits the map, will do operation `and` for this * two positions bit set. */ - private LinkedMap> individualAckOfTransaction; + private LinkedMap> individualAckOfTransaction; /** * The map is for individual ack of positions for transaction. @@ -98,13 +107,13 @@ public class PendingAckHandleImpl extends PendingAckHandleState implements Pendi *

* If it does not exits the map, the position will be added to the map. */ - private ConcurrentSkipListMap> individualAckPositions; + private ConcurrentSkipListMap> individualAckPositions; /** * The map is for transaction with position witch was cumulative acked by this transaction. * Only one cumulative ack position was acked by one transaction at the same time. */ - private Pair cumulativeAckOfTransaction; + private Pair cumulativeAckOfTransaction; private final String topicName; @@ -134,26 +143,30 @@ public class PendingAckHandleImpl extends PendingAckHandleState implements Pendi public final RecoverTimeRecord recoverTime = new RecoverTimeRecord(); + private final long pendingAckInitFailureBackoffInitialTimeInMs = 100; + + public final Backoff backoff = new Backoff(pendingAckInitFailureBackoffInitialTimeInMs, TimeUnit.MILLISECONDS, + 1, TimeUnit.MINUTES, 0, TimeUnit.MILLISECONDS); + + private final Timer transactionOpTimer; public PendingAckHandleImpl(PersistentSubscription persistentSubscription) { super(State.None); this.topicName = persistentSubscription.getTopicName(); this.subName = persistentSubscription.getName(); this.persistentSubscription = persistentSubscription; - internalPinnedExecutor = persistentSubscription - .getTopic() - .getBrokerService() - .getPulsar() - .getTransactionExecutorProvider() - .getExecutor(this); + var pulsar = persistentSubscription.getTopic().getBrokerService().getPulsar(); + internalPinnedExecutor = pulsar.getTransactionExecutorProvider().getExecutor(this); - ServiceConfiguration config = persistentSubscription.getTopic().getBrokerService().pulsar().getConfig(); - boolean exposeTopicLevelMetrics = config.isExposeTopicLevelMetricsInPrometheus(); - this.handleStats = PendingAckHandleStats.create(topicName, subName, exposeTopicLevelMetrics); + this.handleStats = PendingAckHandleStats.create( + topicName, subName, pulsar.getConfig().isExposeTopicLevelMetricsInPrometheus()); - this.pendingAckStoreProvider = this.persistentSubscription.getTopic() - .getBrokerService().getPulsar().getTransactionPendingAckStoreProvider(); + this.pendingAckStoreProvider = pulsar.getTransactionPendingAckStoreProvider(); + transactionOpTimer = pulsar.getTransactionTimer(); + init(); + } + private void init() { pendingAckStoreProvider.checkInitializedBefore(persistentSubscription) .thenAcceptAsync(init -> { if (init) { @@ -164,9 +177,9 @@ public PendingAckHandleImpl(PersistentSubscription persistentSubscription) { }, internalPinnedExecutor) .exceptionallyAsync(e -> { Throwable t = FutureUtil.unwrapCompletionException(e); - changeToErrorState(); + // Handling the exceptions in `exceptionHandleFuture`, + // it will be helpful to make the exception handling clearer. exceptionHandleFuture(t); - this.pendingAckStoreFuture.completeExceptionally(t); return null; }, internalPinnedExecutor); } @@ -180,9 +193,8 @@ private void initPendingAckStore() { recoverTime.setRecoverStartTime(System.currentTimeMillis()); pendingAckStore.replayAsync(this, internalPinnedExecutor); }).exceptionallyAsync(e -> { - handleCacheRequest(); - changeToErrorState(); - log.error("PendingAckHandleImpl init fail! TopicName : {}, SubName: {}", topicName, subName, e); + // Handling the exceptions in `exceptionHandleFuture`, + // it will be helpful to make the exception handling clearer. exceptionHandleFuture(e.getCause()); return null; }, internalPinnedExecutor); @@ -191,12 +203,12 @@ private void initPendingAckStore() { } private void addIndividualAcknowledgeMessageRequest(TxnID txnID, - List> positions, + List> positions, CompletableFuture completableFuture) { acceptQueue.add(() -> internalIndividualAcknowledgeMessage(txnID, positions, completableFuture)); } - public void internalIndividualAcknowledgeMessage(TxnID txnID, List> positions, + public void internalIndividualAcknowledgeMessage(TxnID txnID, List> positions, CompletableFuture completableFuture) { if (txnID == null) { completableFuture.completeExceptionally(new NotAllowedException("txnID can not be null.")); @@ -211,19 +223,18 @@ public void internalIndividualAcknowledgeMessage(TxnID txnID, List pendingAckStore.appendIndividualAck(txnID, positions).thenAccept(v -> { synchronized (org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl.this) { - for (MutablePair positionIntegerMutablePair : positions) { + for (MutablePair positionIntegerMutablePair : positions) { if (log.isDebugEnabled()) { log.debug("[{}] individualAcknowledgeMessage position: [{}], " + "txnId: [{}], subName: [{}]", topicName, positionIntegerMutablePair.left, txnID, subName); } - PositionImpl position = positionIntegerMutablePair.left; + Position position = positionIntegerMutablePair.left; // If try to ack message already acked by committed transaction or // normal acknowledge,throw exception. - if (((ManagedCursorImpl) persistentSubscription.getCursor()) - .isMessageDeleted(position)) { + if (persistentSubscription.getCursor().isMessageDeleted(position)) { String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to ack message:" + position + " already acked before."; log.error(errorMsg); @@ -232,20 +243,20 @@ public void internalIndividualAcknowledgeMessage(TxnID txnID, List bitSetRecyclable.size()) { bitSetRecyclable.set(positionIntegerMutablePair.right); } bitSetRecyclable.set(positionIntegerMutablePair.right, bitSetRecyclable.size()); long[] ackSetOverlap = bitSetRecyclable.toLongArray(); bitSetRecyclable.recycle(); - if (isAckSetOverlap(ackSetOverlap, - ((ManagedCursorImpl) persistentSubscription.getCursor()) + if (isAckSetOverlap(ackSetOverlap, persistentSubscription.getCursor() .getBatchPositionAckSet(position))) { String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to ack message:" @@ -258,8 +269,8 @@ public void internalIndividualAcknowledgeMessage(TxnID txnID, List individualAcknowledgeMessage(TxnID txnID, - List> positions) { + List> positions) { CompletableFuture completableFuture = new CompletableFuture<>(); internalPinnedExecutor.execute(() -> { if (!checkIfReady()) { @@ -333,13 +344,13 @@ public CompletableFuture individualAcknowledgeMessage(TxnID txnID, } private void addCumulativeAcknowledgeMessageRequest(TxnID txnID, - List positions, + List positions, CompletableFuture completableFuture) { acceptQueue.add(() -> internalCumulativeAcknowledgeMessage(txnID, positions, completableFuture)); } public void internalCumulativeAcknowledgeMessage(TxnID txnID, - List positions, + List positions, CompletableFuture completableFuture) { if (txnID == null) { completableFuture.completeExceptionally(new NotAllowedException("TransactionID can not be null.")); @@ -358,7 +369,7 @@ public void internalCumulativeAcknowledgeMessage(TxnID txnID, return; } - PositionImpl position = positions.get(0); + Position position = positions.get(0); this.pendingAckStoreFuture.thenAccept(pendingAckStore -> pendingAckStore.appendCumulativeAck(txnID, position).thenAccept(v -> { @@ -367,7 +378,7 @@ public void internalCumulativeAcknowledgeMessage(TxnID txnID, + "txnID:[{}], subName: [{}].", topicName, txnID, position, subName); } - if (position.compareTo((PositionImpl) persistentSubscription.getCursor() + if (position.compareTo(persistentSubscription.getCursor() .getMarkDeletedPosition()) <= 0) { String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to cumulative ack position: " + position + " within range of cursor's " @@ -405,7 +416,7 @@ public void internalCumulativeAcknowledgeMessage(TxnID txnID, } @Override - public CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, List positions) { + public CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, List positions) { CompletableFuture completableFuture = new CompletableFuture<>(); internalPinnedExecutor.execute(() -> { if (!checkIfReady()) { @@ -474,7 +485,7 @@ private void internalCommitTxn(TxnID txnID, Map properties, long l pendingAckStore.appendCommitMark(txnID, AckType.Individual).thenAccept(v -> { synchronized (PendingAckHandleImpl.this) { if (individualAckOfTransaction != null && individualAckOfTransaction.containsKey(txnID)) { - HashMap pendingAckMessageForCurrentTxn = + HashMap pendingAckMessageForCurrentTxn = individualAckOfTransaction.get(txnID); if (log.isDebugEnabled()) { log.debug("[{}] Transaction pending ack store commit txnId : " @@ -566,7 +577,7 @@ public CompletableFuture internalAbortTxn(TxnID txnId, Consumer consumer, pendingAckStoreFuture.thenAccept(pendingAckStore -> pendingAckStore.appendAbortMark(txnId, AckType.Individual).thenAccept(v -> { synchronized (PendingAckHandleImpl.this) { - HashMap pendingAckMessageForCurrentTxn = + HashMap pendingAckMessageForCurrentTxn = individualAckOfTransaction.get(txnId); if (pendingAckMessageForCurrentTxn != null) { if (log.isDebugEnabled()) { @@ -661,7 +672,7 @@ private void handleLowWaterMark(TxnID txnID, long lowWaterMark) { } @Override - public synchronized void syncBatchPositionAckSetForTransaction(PositionImpl position) { + public synchronized void syncBatchPositionAckSetForTransaction(Position position) { if (individualAckPositions == null) { individualAckPositions = new ConcurrentSkipListMap<>(); } @@ -675,13 +686,14 @@ public synchronized void syncBatchPositionAckSetForTransaction(PositionImpl posi } @Override - public synchronized boolean checkIsCanDeleteConsumerPendingAck(PositionImpl position) { + public synchronized boolean checkIsCanDeleteConsumerPendingAck(Position position) { if (!individualAckPositions.containsKey(position)) { return true; } else { position = individualAckPositions.get(position).left; - if (position.hasAckSet()) { - BitSetRecyclable bitSetRecyclable = BitSetRecyclable.valueOf(position.getAckSet()); + long[] positionAckSet = getAckSetArrayOrNull(position); + if (positionAckSet != null) { + BitSetRecyclable bitSetRecyclable = BitSetRecyclable.valueOf(positionAckSet); if (bitSetRecyclable.isEmpty()) { bitSetRecyclable.recycle(); return true; @@ -700,7 +712,7 @@ protected void handleAbort(TxnID txnID, AckType ackType) { this.cumulativeAckOfTransaction = null; } else { if (this.individualAckOfTransaction != null) { - HashMap pendingAckMessageForCurrentTxn = + HashMap pendingAckMessageForCurrentTxn = individualAckOfTransaction.get(txnID); if (pendingAckMessageForCurrentTxn != null) { individualAckAbortCommon(txnID, pendingAckMessageForCurrentTxn); @@ -709,24 +721,25 @@ protected void handleAbort(TxnID txnID, AckType ackType) { } } - private void individualAckAbortCommon(TxnID txnID, HashMap currentTxn) { - for (Map.Entry entry : + private void individualAckAbortCommon(TxnID txnID, HashMap currentTxn) { + for (Map.Entry entry : currentTxn.entrySet()) { - if (entry.getValue().hasAckSet() + long[] entryValueAckSet = getAckSetArrayOrNull(entry.getValue()); + if (entryValueAckSet != null && individualAckPositions.containsKey(entry.getValue())) { BitSetRecyclable thisBitSet = - BitSetRecyclable.valueOf(entry.getValue().getAckSet()); + BitSetRecyclable.valueOf(entryValueAckSet); int batchSize = individualAckPositions.get(entry.getValue()).right; thisBitSet.flip(0, batchSize); + AckSetState individualAckPositionAckSetState = + AckSetStateUtil.getAckSetState(individualAckPositions.get(entry.getValue()).left); BitSetRecyclable otherBitSet = - BitSetRecyclable.valueOf(individualAckPositions - .get(entry.getValue()).left.getAckSet()); + BitSetRecyclable.valueOf(individualAckPositionAckSetState.getAckSet()); otherBitSet.or(thisBitSet); if (otherBitSet.cardinality() == batchSize) { individualAckPositions.remove(entry.getValue()); } else { - individualAckPositions.get(entry.getKey()) - .left.setAckSet(otherBitSet.toLongArray()); + individualAckPositionAckSetState.setAckSet(otherBitSet.toLongArray()); } otherBitSet.recycle(); thisBitSet.recycle(); @@ -747,7 +760,7 @@ protected void handleCommit(TxnID txnID, AckType ackType, Map prop this.cumulativeAckOfTransaction = null; } else { if (this.individualAckOfTransaction != null) { - HashMap pendingAckMessageForCurrentTxn = + HashMap pendingAckMessageForCurrentTxn = individualAckOfTransaction.get(txnID); if (pendingAckMessageForCurrentTxn != null) { individualAckCommitCommon(txnID, pendingAckMessageForCurrentTxn, null); @@ -757,7 +770,7 @@ protected void handleCommit(TxnID txnID, AckType ackType, Map prop } private void individualAckCommitCommon(TxnID txnID, - HashMap currentTxn, + HashMap currentTxn, Map properties) { if (currentTxn != null) { persistentSubscription.acknowledgeMessage(new ArrayList<>(currentTxn.values()), @@ -766,7 +779,7 @@ private void individualAckCommitCommon(TxnID txnID, } } - private void handleIndividualAck(TxnID txnID, List> positions) { + private void handleIndividualAck(TxnID txnID, List> positions) { for (int i = 0; i < positions.size(); i++) { if (log.isDebugEnabled()) { log.debug("[{}][{}] TxnID:[{}] Individual acks on {}", topicName, @@ -780,11 +793,11 @@ private void handleIndividualAck(TxnID txnID, List(); } - PositionImpl position = positions.get(i).left; - - if (position.hasAckSet()) { + Position position = positions.get(i).left; + long[] positionAckSet = getAckSetArrayOrNull(position); + if (positionAckSet != null) { - HashMap pendingAckMessageForCurrentTxn = + HashMap pendingAckMessageForCurrentTxn = individualAckOfTransaction.computeIfAbsent(txnID, txn -> new HashMap<>()); if (pendingAckMessageForCurrentTxn.containsKey(position)) { @@ -803,20 +816,21 @@ private void handleIndividualAck(TxnID txnID, List positionPair = positions.get(i); - positionPair.left = PositionImpl.get(positionPair.getLeft().getLedgerId(), - positionPair.getLeft().getEntryId(), - Arrays.copyOf(positionPair.left.getAckSet(), positionPair.left.getAckSet().length)); - this.individualAckPositions.put(position, positions.get(i)); + MutablePair positionPair = positions.get(i); + long[] positionPairLeftAckSet = getAckSetArrayOrNull(positionPair.left); + positionPair.left = createPositionWithAckSet(positionPair.left.getLedgerId(), + positionPair.left.getEntryId(), + Arrays.copyOf(positionPairLeftAckSet, positionPairLeftAckSet.length)); + this.individualAckPositions.put(position, positionPair); } else { - MutablePair positionPair = + MutablePair positionPair = this.individualAckPositions.get(position); positionPair.setRight(positions.get(i).right); andAckSet(positionPair.getLeft(), position); } } else { - HashMap pendingAckMessageForCurrentTxn = + HashMap pendingAckMessageForCurrentTxn = individualAckOfTransaction.computeIfAbsent(txnID, txn -> new HashMap<>()); pendingAckMessageForCurrentTxn.put(position, position); this.individualAckPositions.putIfAbsent(position, positions.get(i)); @@ -824,7 +838,7 @@ private void handleIndividualAck(TxnID txnID, List 0 } } - protected void handleCumulativeAckRecover(TxnID txnID, PositionImpl position) { - if ((position.compareTo((PositionImpl) persistentSubscription.getCursor() + protected void handleCumulativeAckRecover(TxnID txnID, Position position) { + if ((position.compareTo(persistentSubscription.getCursor() .getMarkDeletedPosition()) > 0) && (cumulativeAckOfTransaction == null || (cumulativeAckOfTransaction.getKey().equals(txnID) && compareToWithAckSet(position, cumulativeAckOfTransaction.getValue()) > 0))) { @@ -842,23 +856,24 @@ && compareToWithAckSet(position, cumulativeAckOfTransaction.getValue()) > 0))) { } } - protected void handleIndividualAckRecover(TxnID txnID, List> positions) { - for (MutablePair positionIntegerMutablePair : positions) { - PositionImpl position = positionIntegerMutablePair.left; + protected void handleIndividualAckRecover(TxnID txnID, List> positions) { + for (MutablePair positionIntegerMutablePair : positions) { + Position position = positionIntegerMutablePair.left; // If try to ack message already acked by committed transaction or // normal acknowledge,throw exception. - if (((ManagedCursorImpl) persistentSubscription.getCursor()) - .isMessageDeleted(position)) { + if (persistentSubscription.getCursor().isMessageDeleted(position)) { return; } - if (position.hasAckSet()) { + long[] positionAckSet = getAckSetArrayOrNull(position); + + if (positionAckSet != null) { //in order to jude the bit set is over lap, so set the covering // the batch size bit to 1,should know the two // bit set don't have the same point is 0 BitSetRecyclable bitSetRecyclable = - BitSetRecyclable.valueOf(position.getAckSet()); + BitSetRecyclable.valueOf(positionAckSet); if (positionIntegerMutablePair.right > bitSetRecyclable.size()) { bitSetRecyclable.set(positionIntegerMutablePair.right); } @@ -866,15 +881,13 @@ protected void handleIndividualAckRecover(TxnID txnID, List init(), retryTime, TimeUnit.MILLISECONDS); + return; + } + log.error("[{}] [{}] PendingAckHandleImpl init fail!", topicName, subName, t); + handleCacheRequest(); + changeToErrorState(); + // ToDo: Add a new serverError `TransactionComponentLoadFailedException` + // and before that a `Unknown` will be returned first. + this.pendingAckStoreFuture = FutureUtil.failedFuture(new BrokerServiceException( + String.format("[%s][%s] Failed to init transaction pending ack.", topicName, subName))); + final boolean completedNow = this.pendingAckHandleCompletableFuture.completeExceptionally( + new BrokerServiceException( + String.format("[%s][%s] Failed to init transaction pending ack.", topicName, subName))); if (completedNow) { recoverTime.setRecoverEndTime(System.currentTimeMillis()); } } + private static boolean isRetryableException(Throwable ex) { + Throwable realCause = FutureUtil.unwrapCompletionException(ex); + return (realCause instanceof ManagedLedgerException + && !(realCause instanceof ManagedLedgerException.ManagedLedgerFencedException) + && !(realCause instanceof ManagedLedgerException.NonRecoverableLedgerException)) + || realCause instanceof PulsarClientException.BrokerPersistenceException + || realCause instanceof PulsarClientException.LookupException + || realCause instanceof PulsarClientException.ConnectException + || realCause instanceof MetadataStoreException + || realCause instanceof BKException; + } + @Override public TransactionInPendingAckStats getTransactionInPendingAckStats(TxnID txnID) { TransactionInPendingAckStats transactionInPendingAckStats = new TransactionInPendingAckStats(); if (cumulativeAckOfTransaction != null && cumulativeAckOfTransaction.getLeft().equals(txnID)) { - PositionImpl position = cumulativeAckOfTransaction.getRight(); + Position position = cumulativeAckOfTransaction.getRight(); StringBuilder stringBuilder = new StringBuilder() .append(position.getLedgerId()) .append(':') .append(position.getEntryId()); - if (cumulativeAckOfTransaction.getRight().hasAckSet()) { + long[] positionAckSet = getAckSetArrayOrNull(position); + if (positionAckSet != null) { BitSetRecyclable bitSetRecyclable = - BitSetRecyclable.valueOf(cumulativeAckOfTransaction.getRight().getAckSet()); + BitSetRecyclable.valueOf(positionAckSet); if (!bitSetRecyclable.isEmpty()) { stringBuilder.append(":").append(bitSetRecyclable.nextSetBit(0) - 1); } @@ -972,6 +1013,11 @@ public TransactionInPendingAckStats getTransactionInPendingAckStats(TxnID txnID) return transactionInPendingAckStats; } + @Override + public PendingAckHandleStats getPendingAckHandleStats() { + return handleStats; + } + @Override public CompletableFuture closeAsync() { changeToCloseState(); @@ -1018,17 +1064,17 @@ public CompletableFuture getStoreManageLedger() { } @Override - public PositionInPendingAckStats checkPositionInPendingAckState(PositionImpl position, Integer batchIndex) { + public PositionInPendingAckStats checkPositionInPendingAckState(Position position, Integer batchIndex) { if (!state.equals(State.Ready)) { return new PositionInPendingAckStats(PositionInPendingAckStats.State.PendingAckNotReady); } if (persistentSubscription.getCursor().getPersistentMarkDeletedPosition() != null && position.compareTo( - (PositionImpl) persistentSubscription.getCursor().getPersistentMarkDeletedPosition()) <= 0) { + persistentSubscription.getCursor().getPersistentMarkDeletedPosition()) <= 0) { return new PositionInPendingAckStats(PositionInPendingAckStats.State.MarkDelete); } else if (individualAckPositions == null) { return new PositionInPendingAckStats(PositionInPendingAckStats.State.NotInPendingAck); } - MutablePair positionIntegerMutablePair = individualAckPositions.get(position); + MutablePair positionIntegerMutablePair = individualAckPositions.get(position); if (positionIntegerMutablePair != null) { if (batchIndex == null) { return new PositionInPendingAckStats(PositionInPendingAckStats.State.PendingAck); @@ -1037,7 +1083,7 @@ public PositionInPendingAckStats checkPositionInPendingAckState(PositionImpl pos return new PositionInPendingAckStats(PositionInPendingAckStats.State.InvalidPosition); } BitSetRecyclable bitSetRecyclable = BitSetRecyclable - .valueOf(positionIntegerMutablePair.left.getAckSet()); + .valueOf(getAckSetArrayOrNull(positionIntegerMutablePair.left)); if (bitSetRecyclable.get(batchIndex)) { bitSetRecyclable.recycle(); return new PositionInPendingAckStats(PositionInPendingAckStats.State.NotInPendingAck); @@ -1052,7 +1098,7 @@ public PositionInPendingAckStats checkPositionInPendingAckState(PositionImpl pos } @VisibleForTesting - public Map> getIndividualAckPositions() { + public Map> getIndividualAckPositions() { return individualAckPositions; } @@ -1062,9 +1108,9 @@ public boolean checkIfPendingAckStoreInit() { } @Override - public PositionImpl getPositionInPendingAck(PositionImpl position) { + public Position getPositionInPendingAck(Position position) { if (individualAckPositions != null) { - MutablePair positionPair = this.individualAckPositions.get(position); + MutablePair positionPair = this.individualAckPositions.get(position); if (positionPair != null) { return positionPair.getLeft(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleStatsImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleStatsImpl.java index f30c233af5993..a89b582b838dd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleStatsImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleStatsImpl.java @@ -22,7 +22,10 @@ import io.prometheus.client.Summary; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import java.util.concurrent.atomic.LongAdder; import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckHandleAttributes; import org.apache.pulsar.broker.transaction.pendingack.PendingAckHandleStats; import org.apache.pulsar.common.naming.TopicName; @@ -37,6 +40,19 @@ public class PendingAckHandleStatsImpl implements PendingAckHandleStats { private final String[] labelFailed; private final String[] commitLatencyLabel; + private final String topic; + private final String subscription; + + private final LongAdder commitTxnSucceedCounter = new LongAdder(); + private final LongAdder commitTxnFailedCounter = new LongAdder(); + private final LongAdder abortTxnSucceedCounter = new LongAdder(); + private final LongAdder abortTxnFailedCounter = new LongAdder(); + + private volatile PendingAckHandleAttributes attributes = null; + private static final AtomicReferenceFieldUpdater + ATTRIBUTES_UPDATER = AtomicReferenceFieldUpdater.newUpdater( + PendingAckHandleStatsImpl.class, PendingAckHandleAttributes.class, "attributes"); + public PendingAckHandleStatsImpl(String topic, String subscription, boolean exposeTopicLevelMetrics) { initialize(exposeTopicLevelMetrics); @@ -51,6 +67,9 @@ public PendingAckHandleStatsImpl(String topic, String subscription, boolean expo } } + this.topic = topic; + this.subscription = subscription; + labelSucceed = exposeTopicLevelMetrics0 ? new String[]{namespace, topic, subscription, "succeed"} : new String[]{namespace, "succeed"}; labelFailed = exposeTopicLevelMetrics0 @@ -62,18 +81,24 @@ public PendingAckHandleStatsImpl(String topic, String subscription, boolean expo @Override public void recordCommitTxn(boolean success, long nanos) { String[] labels; + LongAdder counter; if (success) { labels = labelSucceed; + counter = commitTxnSucceedCounter; commitTxnLatency.labels(commitLatencyLabel).observe(TimeUnit.NANOSECONDS.toMicros(nanos)); } else { labels = labelFailed; + counter = commitTxnFailedCounter; } commitTxnCounter.labels(labels).inc(); + counter.increment(); } @Override public void recordAbortTxn(boolean success) { abortTxnCounter.labels(success ? labelSucceed : labelFailed).inc(); + var counter = success ? abortTxnSucceedCounter : abortTxnFailedCounter; + counter.increment(); } @Override @@ -81,11 +106,40 @@ public void close() { if (exposeTopicLevelMetrics0) { commitTxnCounter.remove(this.labelSucceed); commitTxnCounter.remove(this.labelFailed); + abortTxnCounter.remove(this.labelSucceed); abortTxnCounter.remove(this.labelFailed); - abortTxnCounter.remove(this.labelFailed); } } + @Override + public long getCommitSuccessCount() { + return commitTxnSucceedCounter.sum(); + } + + @Override + public long getCommitFailedCount() { + return commitTxnFailedCounter.sum(); + } + + @Override + public long getAbortSuccessCount() { + return abortTxnSucceedCounter.sum(); + } + + @Override + public long getAbortFailedCount() { + return abortTxnFailedCounter.sum(); + } + + @Override + public PendingAckHandleAttributes getAttributes() { + if (attributes != null) { + return attributes; + } + return ATTRIBUTES_UPDATER.updateAndGet(PendingAckHandleStatsImpl.this, + old -> old != null ? old : new PendingAckHandleAttributes(topic, subscription)); + } + static void initialize(boolean exposeTopicLevelMetrics) { if (INITIALIZED.compareAndSet(false, true)) { exposeTopicLevelMetrics0 = exposeTopicLevelMetrics; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/timeout/TransactionTimeoutTrackerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/timeout/TransactionTimeoutTrackerImpl.java index 7b3bf744a82ed..110d912385e48 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/timeout/TransactionTimeoutTrackerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/timeout/TransactionTimeoutTrackerImpl.java @@ -22,7 +22,6 @@ import io.netty.util.Timer; import io.netty.util.TimerTask; import java.time.Clock; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.TransactionMetadataStoreService; @@ -57,10 +56,10 @@ public class TransactionTimeoutTrackerImpl implements TransactionTimeoutTracker, } @Override - public CompletableFuture addTransaction(long sequenceId, long timeout) { + public void addTransaction(long sequenceId, long timeout) { if (timeout < tickTimeMillis) { this.transactionMetadataStoreService.endTransactionForTimeout(new TxnID(tcId, sequenceId)); - return CompletableFuture.completedFuture(false); + return; } synchronized (this){ long nowTime = clock.millis(); @@ -79,7 +78,6 @@ public CompletableFuture addTransaction(long sequenceId, long timeout) nowTaskTimeoutTime = transactionTimeoutTime; } } - return CompletableFuture.completedFuture(false); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/ExceptionHandler.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/ExceptionHandler.java index b11ec3a8a98db..205e02ed75a2e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/ExceptionHandler.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/ExceptionHandler.java @@ -24,6 +24,7 @@ import javax.servlet.ServletResponse; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.core.Response; +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.common.intercept.InterceptException; import org.apache.pulsar.common.policies.data.ErrorData; import org.apache.pulsar.common.util.ObjectMapperFactory; @@ -36,6 +37,7 @@ /** * Exception handler for handle exception. */ +@Slf4j public class ExceptionHandler { public void handle(ServletResponse response, Exception ex) throws IOException { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index 3d23d7812543a..2e198eb99752e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -56,12 +56,15 @@ import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.broker.authorization.AuthorizationService; +import org.apache.pulsar.broker.loadbalance.LoadManager; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.namespace.LookupOptions; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.resources.BookieResources; import org.apache.pulsar.broker.resources.ClusterResources; import org.apache.pulsar.broker.resources.DynamicConfigurationResources; +import org.apache.pulsar.broker.resources.LoadBalanceResources; import org.apache.pulsar.broker.resources.LocalPoliciesResources; import org.apache.pulsar.broker.resources.NamespaceResources; import org.apache.pulsar.broker.resources.NamespaceResources.IsolationPolicyResources; @@ -92,6 +95,7 @@ import org.apache.pulsar.common.policies.path.PolicyPath; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.metadata.api.coordination.LockManager; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; @@ -309,7 +313,7 @@ protected CompletableFuture validateAdminAccessForTenantAsync( } return pulsar.getPulsarResources().getTenantResources().getTenantAsync(tenant) .thenCompose(tenantInfoOptional -> { - if (!tenantInfoOptional.isPresent()) { + if (tenantInfoOptional.isEmpty()) { throw new RestException(Status.NOT_FOUND, "Tenant does not exist"); } TenantInfo tenantInfo = tenantInfoOptional.get(); @@ -380,9 +384,11 @@ protected CompletableFuture validateAdminAccessForTenantAsync( /** * It validates that peer-clusters can't coexist in replication-clusters. * - * @clusterName: given cluster whose peer-clusters can't be present into replication-cluster list - * @replicationClusters: replication-cluster list + * @param clusterName given cluster whose peer-clusters can't be present into replication-cluster list + * @param replicationClusters replication-cluster list + * @deprecated use {@link #validatePeerClusterConflictAsync(String, Set)} instead */ + @Deprecated protected void validatePeerClusterConflict(String clusterName, Set replicationClusters) { try { ClusterData clusterData = clusterResources().getCluster(clusterName).orElseThrow( @@ -453,7 +459,7 @@ protected void validateClusterForTenant(String tenant, String cluster) { protected CompletableFuture validateClusterForTenantAsync(String tenant, String cluster) { return pulsar().getPulsarResources().getTenantResources().getTenantAsync(tenant) .thenAccept(tenantInfo -> { - if (!tenantInfo.isPresent()) { + if (tenantInfo.isEmpty()) { throw new RestException(Status.NOT_FOUND, "Tenant does not exist"); } if (!tenantInfo.get().getAllowedClusters().contains(cluster)) { @@ -488,7 +494,7 @@ protected CompletableFuture validateClusterOwnershipAsync(String cluster) * Check if the cluster exists and redirect the call to the owning cluster. * * @param cluster Cluster name - * @throws Exception In case the redirect happens + * @throws WebApplicationException In case the redirect happens */ protected void validateClusterOwnership(String cluster) throws WebApplicationException { sync(()-> validateClusterOwnershipAsync(cluster)); @@ -528,6 +534,7 @@ protected static CompletableFuture getClusterDataIfDifferentCluster pulsar.getPulsarResources().getClusterResources().getClusterAsync(cluster) .whenComplete((clusterDataResult, ex) -> { if (ex != null) { + log.warn("[{}] Load cluster data failed: requested={}", clientAppId, cluster); clusterDataFuture.completeExceptionally(FutureUtil.unwrapCompletionException(ex)); return; } @@ -550,11 +557,8 @@ static boolean isValidCluster(PulsarService pulsarService, String cluster) {// I return true; } - if (!pulsarService.getConfiguration().isAuthorizationEnabled()) { - // Without authorization, any cluster name should be valid and accepted by the broker - return true; - } - return false; + // Without authorization, any cluster name should be valid and accepted by the broker + return !pulsarService.getConfiguration().isAuthorizationEnabled(); } protected void validateBundleOwnership(String tenant, String cluster, String namespace, boolean authoritative, @@ -606,12 +610,17 @@ protected CompletableFuture isBundleOwnedByAnyBroker(NamespaceName fqnn NamespaceBundle nsBundle = validateNamespaceBundleRange(fqnn, bundles, bundleRange); NamespaceService nsService = pulsar().getNamespaceService(); + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar)) { + return nsService.checkOwnershipPresentAsync(nsBundle); + } + LookupOptions options = LookupOptions.builder() .authoritative(false) .requestHttps(isRequestHttps()) .readOnly(true) .loadTopicsInBundle(false).build(); - return nsService.getWebServiceUrlAsync(nsBundle, options).thenApply(optionUrl -> optionUrl.isPresent()); + + return nsService.getWebServiceUrlAsync(nsBundle, options).thenApply(Optional::isPresent); } protected NamespaceBundle validateNamespaceBundleOwnership(NamespaceName fqnn, BundlesData bundles, @@ -664,7 +673,7 @@ public void validateBundleOwnership(NamespaceBundle bundle, boolean authoritativ .loadTopicsInBundle(false).build(); Optional webUrl = nsService.getWebServiceUrl(bundle, options); // Ensure we get a url - if (webUrl == null || !webUrl.isPresent()) { + if (webUrl.isEmpty()) { log.warn("Unable to get web service url"); throw new RestException(Status.PRECONDITION_FAILED, "Failed to find ownership for ServiceUnit:" + bundle.toString()); @@ -697,8 +706,6 @@ public void validateBundleOwnership(NamespaceBundle bundle, boolean authoritativ } catch (NullPointerException e) { log.warn("Unable to get web service url"); throw new RestException(Status.PRECONDITION_FAILED, "Failed to find ownership for ServiceUnit:" + bundle); - } catch (WebApplicationException wae) { - throw wae; } } @@ -712,24 +719,24 @@ public CompletableFuture validateBundleOwnershipAsync(NamespaceBundle bund .loadTopicsInBundle(false).build(); return nsService.getWebServiceUrlAsync(bundle, options) .thenCompose(webUrl -> { - if (webUrl == null || !webUrl.isPresent()) { + if (webUrl.isEmpty()) { log.warn("Unable to get web service url"); throw new RestException(Status.PRECONDITION_FAILED, "Failed to find ownership for ServiceUnit:" + bundle.toString()); } - // If the load manager is extensible load manager, we don't need check the authoritative. - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config())) { - return CompletableFuture.completedFuture(null); - } return nsService.isServiceUnitOwnedAsync(bundle) .thenAccept(owned -> { if (!owned) { boolean newAuthoritative = this.isLeaderBroker(); // Replace the host and port of the current request and redirect - URI redirect = UriBuilder.fromUri(uri.getRequestUri()).host(webUrl.get().getHost()) - .port(webUrl.get().getPort()).replaceQueryParam("authoritative", - newAuthoritative).replaceQueryParam("destinationBroker", - null).build(); + UriBuilder uriBuilder = UriBuilder.fromUri(uri.getRequestUri()) + .host(webUrl.get().getHost()) + .port(webUrl.get().getPort()) + .replaceQueryParam("authoritative", newAuthoritative); + if (!ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar)) { + uriBuilder.replaceQueryParam("destinationBroker", null); + } + URI redirect = uriBuilder.build(); log.debug("{} is not a service unit owned", bundle); // Redirect log.debug("Redirecting the rest call to {}", redirect); @@ -762,15 +769,13 @@ protected CompletableFuture validateTopicOwnershipAsync(TopicName topicNam .build(); return nsService.getWebServiceUrlAsync(topicName, options) - .thenApply(webUrl -> { - // Ensure we get a url - if (webUrl == null || !webUrl.isPresent()) { - log.info("Unable to get web service url"); - throw new RestException(Status.PRECONDITION_FAILED, - "Failed to find ownership for topic:" + topicName); - } - return webUrl.get(); - }).thenCompose(webUrl -> nsService.isServiceUnitOwnedAsync(topicName) + .thenApply(webUrl -> + webUrl.orElseThrow(() -> { + log.info("Unable to get web service url"); + throw new RestException(Status.PRECONDITION_FAILED, + "Failed to find ownership for topic:" + topicName); + }) + ).thenCompose(webUrl -> nsService.isServiceUnitOwnedAsync(topicName) .thenApply(isTopicOwned -> Pair.of(webUrl, isTopicOwned)) ).thenAccept(pair -> { URL webUrl = pair.getLeft(); @@ -896,14 +901,16 @@ public static CompletableFuture checkLocalOrGetPeerReplicationC log.warn(msg); validationFuture.completeExceptionally(new RestException(Status.NOT_FOUND, "Namespace is deleted")); - } else if (policies.replication_clusters.isEmpty()) { + } else if (policies.replication_clusters.isEmpty() && policies.allowed_clusters.isEmpty()) { String msg = String.format( "Namespace does not have any clusters configured : local_cluster=%s ns=%s", localCluster, namespace.toString()); log.warn(msg); validationFuture.completeExceptionally(new RestException(Status.PRECONDITION_FAILED, msg)); - } else if (!policies.replication_clusters.contains(localCluster)) { - getOwnerFromPeerClusterListAsync(pulsarService, policies.replication_clusters) + } else if (!policies.replication_clusters.contains(localCluster) && !policies.allowed_clusters + .contains(localCluster)) { + getOwnerFromPeerClusterListAsync(pulsarService, policies.replication_clusters, + policies.allowed_clusters) .thenAccept(ownerPeerCluster -> { if (ownerPeerCluster != null) { // found a peer that own this namespace @@ -943,9 +950,9 @@ public static CompletableFuture checkLocalOrGetPeerReplicationC } private static CompletableFuture getOwnerFromPeerClusterListAsync(PulsarService pulsar, - Set replicationClusters) { + Set replicationClusters, Set allowedClusters) { String currentCluster = pulsar.getConfiguration().getClusterName(); - if (replicationClusters == null || replicationClusters.isEmpty() || isBlank(currentCluster)) { + if (replicationClusters.isEmpty() && allowedClusters.isEmpty() || isBlank(currentCluster)) { return CompletableFuture.completedFuture(null); } @@ -955,7 +962,8 @@ private static CompletableFuture getOwnerFromPeerClusterListAsy return CompletableFuture.completedFuture(null); } for (String peerCluster : cluster.get().getPeerClusterNames()) { - if (replicationClusters.contains(peerCluster)) { + if (replicationClusters.contains(peerCluster) + || allowedClusters.contains(peerCluster)) { return pulsar.getPulsarResources().getClusterResources().getClusterAsync(peerCluster) .thenApply(ret -> { if (!ret.isPresent()) { @@ -1001,7 +1009,7 @@ protected boolean isLeaderBroker() { protected static boolean isLeaderBroker(PulsarService pulsar) { // For extensible load manager, it doesn't have leader election service on pulsar broker. - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar.getConfig())) { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar)) { return true; } return pulsar.getLeaderElectionService().isLeader(); @@ -1116,6 +1124,10 @@ protected NamespaceResources namespaceResources() { return pulsar().getPulsarResources().getNamespaceResources(); } + protected LoadBalanceResources loadBalanceResources() { + return pulsar().getPulsarResources().getLoadBalanceResources(); + } + protected ResourceGroupResources resourceGroupResources() { return pulsar().getPulsarResources().getResourcegroupResources(); } @@ -1198,24 +1210,43 @@ protected CompletableFuture canUpdateCluster(String tenant, Set ol /** * Redirect the call to the specified broker. * - * @param broker - * Broker name + * @param brokerId broker's id (lookup service address) */ - protected void validateBrokerName(String broker) { - String brokerUrl = String.format("http://%s", broker); - String brokerUrlTls = String.format("https://%s", broker); - if (!brokerUrl.equals(pulsar().getSafeWebServiceAddress()) - && !brokerUrlTls.equals(pulsar().getWebServiceAddressTls())) { - String[] parts = broker.split(":"); - checkArgument(parts.length == 2, String.format("Invalid broker url %s", broker)); - String host = parts[0]; - int port = Integer.parseInt(parts[1]); - - URI redirect = UriBuilder.fromUri(uri.getRequestUri()).host(host).port(port).build(); - log.debug("[{}] Redirecting the rest call to {}: broker={}", clientAppId(), redirect, broker); - throw new WebApplicationException(Response.temporaryRedirect(redirect).build()); - + protected CompletableFuture maybeRedirectToBroker(String brokerId) { + // backwards compatibility + String cleanedBrokerId = brokerId.replaceFirst("http[s]?://", ""); + if (pulsar.getBrokerId().equals(cleanedBrokerId) + // backwards compatibility + || ("http://" + cleanedBrokerId).equals(pulsar().getWebServiceAddress()) + || ("https://" + cleanedBrokerId).equals(pulsar().getWebServiceAddressTls())) { + // no need to redirect, the current broker matches the given broker id + return CompletableFuture.completedFuture(null); } + LockManager brokerLookupDataLockManager = + pulsar().getCoordinationService().getLockManager(BrokerLookupData.class); + return brokerLookupDataLockManager.readLock(LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + cleanedBrokerId) + .thenAccept(brokerLookupDataOptional -> { + if (brokerLookupDataOptional.isEmpty()) { + throw new RestException(Status.NOT_FOUND, + "Broker id '" + brokerId + "' not found in available brokers."); + } + brokerLookupDataOptional.ifPresent(brokerLookupData -> { + URI targetBrokerUri; + if ((isRequestHttps() || StringUtils.isBlank(brokerLookupData.getWebServiceUrl())) + && StringUtils.isNotBlank(brokerLookupData.getWebServiceUrlTls())) { + targetBrokerUri = URI.create(brokerLookupData.getWebServiceUrlTls()); + } else { + targetBrokerUri = URI.create(brokerLookupData.getWebServiceUrl()); + } + URI redirect = UriBuilder.fromUri(uri.getRequestUri()) + .scheme(targetBrokerUri.getScheme()) + .host(targetBrokerUri.getHost()) + .port(targetBrokerUri.getPort()).build(); + log.debug("[{}] Redirecting the rest call to {}: broker={}", clientAppId(), redirect, + cleanedBrokerId); + throw new WebApplicationException(Response.temporaryRedirect(redirect).build()); + }); + }); } public void validateTopicPolicyOperation(TopicName topicName, PolicyName policy, PolicyOperation operation) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RequestWrapper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RequestWrapper.java index 16d87baedbe3c..afebbd276eba8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RequestWrapper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RequestWrapper.java @@ -62,7 +62,7 @@ public void setReadListener(ReadListener readListener) { } - public int read() throws IOException { + public int read() { return byteArrayInputStream.read(); } }; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/ResponseHandlerFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/ResponseHandlerFilter.java index efed614039566..3fa00beea1f92 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/ResponseHandlerFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/ResponseHandlerFilter.java @@ -76,24 +76,24 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha if (request.isAsyncSupported() && request.isAsyncStarted()) { request.getAsyncContext().addListener(new AsyncListener() { @Override - public void onComplete(AsyncEvent asyncEvent) throws IOException { + public void onComplete(AsyncEvent asyncEvent) { handleInterceptor(request, response); } @Override - public void onTimeout(AsyncEvent asyncEvent) throws IOException { + public void onTimeout(AsyncEvent asyncEvent) { LOG.warn("Http request {} async context timeout.", request); handleInterceptor(request, response); } @Override - public void onError(AsyncEvent asyncEvent) throws IOException { + public void onError(AsyncEvent asyncEvent) { LOG.warn("Http request {} async context error.", request, asyncEvent.getThrowable()); handleInterceptor(request, response); } @Override - public void onStartAsync(AsyncEvent asyncEvent) throws IOException { + public void onStartAsync(AsyncEvent asyncEvent) { // nothing to do } }); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RestException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RestException.java index b18aa1c787a7f..c3ae3a495cfd5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RestException.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RestException.java @@ -30,7 +30,6 @@ /** * Exception used to provide better error messages to clients of the REST API. */ -@SuppressWarnings("serial") public class RestException extends WebApplicationException { private Throwable cause = null; static String getExceptionData(Throwable t) { @@ -75,8 +74,7 @@ public RestException(PulsarAdminException cae) { } private static Response getResponse(Throwable t) { - if (t instanceof WebApplicationException) { - WebApplicationException e = (WebApplicationException) t; + if (t instanceof WebApplicationException e) { return e.getResponse(); } else { return Response diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebExecutorStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebExecutorStats.java index 585df813027d7..28cfa7430cbe6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebExecutorStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebExecutorStats.java @@ -21,14 +21,21 @@ import io.prometheus.client.CollectorRegistry; import io.prometheus.client.Gauge; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.pulsar.opentelemetry.annotations.PulsarDeprecatedMetric; +@Deprecated class WebExecutorStats implements AutoCloseable { private static final AtomicBoolean CLOSED = new AtomicBoolean(false); + @PulsarDeprecatedMetric(newMetricName = WebExecutorThreadPoolStats.LIMIT_COUNTER) private final Gauge maxThreads; + @PulsarDeprecatedMetric(newMetricName = WebExecutorThreadPoolStats.LIMIT_COUNTER) private final Gauge minThreads; + @PulsarDeprecatedMetric(newMetricName = WebExecutorThreadPoolStats.USAGE_COUNTER) private final Gauge idleThreads; + @PulsarDeprecatedMetric(newMetricName = WebExecutorThreadPoolStats.USAGE_COUNTER) private final Gauge activeThreads; + @PulsarDeprecatedMetric(newMetricName = WebExecutorThreadPoolStats.USAGE_COUNTER) private final Gauge currentThreads; private final WebExecutorThreadPool executor; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java index 2d6a6af58477e..5f5e260890a02 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java @@ -20,23 +20,42 @@ import io.prometheus.client.CollectorRegistry; import io.prometheus.client.jetty.JettyStatisticsCollector; +import java.io.IOException; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import javax.servlet.DispatcherType; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletResponse; import lombok.Getter; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.apache.pulsar.jetty.tls.JettySslContextFactory; +import org.eclipse.jetty.server.ConnectionFactory; import org.eclipse.jetty.server.ConnectionLimit; +import org.eclipse.jetty.server.ForwardedRequestCustomizer; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.ProxyConnectionFactory; +import org.eclipse.jetty.server.RequestLog; +import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.server.handler.ContextHandler; import org.eclipse.jetty.server.handler.ContextHandlerCollection; import org.eclipse.jetty.server.handler.DefaultHandler; @@ -69,13 +88,17 @@ public class WebService implements AutoCloseable { private final PulsarService pulsar; private final Server server; private final List handlers; + @Deprecated private final WebExecutorStats executorStats; + private final WebExecutorThreadPoolStats webExecutorThreadPoolStats; private final WebExecutorThreadPool webServiceExecutor; private final ServerConnector httpConnector; private final ServerConnector httpsConnector; private final FilterInitializer filterInitializer; private JettyStatisticsCollector jettyStatisticsCollector; + private PulsarSslFactory sslFactory; + private ScheduledFuture sslContextRefreshTask; @Getter private static final DynamicSkipUnknownPropertyHandler sharedUnknownPropertyHandler = @@ -95,6 +118,8 @@ public WebService(PulsarService pulsar) throws PulsarServerException { "pulsar-web", config.getHttpServerThreadPoolQueueSize()); this.executorStats = WebExecutorStats.getStats(webServiceExecutor); + this.webExecutorThreadPoolStats = + new WebExecutorThreadPoolStats(pulsar.getOpenTelemetry().getMeter(), webServiceExecutor); this.server = new Server(webServiceExecutor); if (config.getMaxHttpServerConnections() > 0) { server.addBean(new ConnectionLimit(config.getMaxHttpServerConnections(), server)); @@ -103,9 +128,18 @@ public WebService(PulsarService pulsar) throws PulsarServerException { Optional port = config.getWebServicePort(); HttpConfiguration httpConfig = new HttpConfiguration(); + if (config.isWebServiceTrustXForwardedFor()) { + httpConfig.addCustomizer(new ForwardedRequestCustomizer()); + } httpConfig.setRequestHeaderSize(pulsar.getConfig().getHttpMaxRequestHeaderSize()); + HttpConnectionFactory httpConnectionFactory = new HttpConnectionFactory(httpConfig); if (port.isPresent()) { - httpConnector = new ServerConnector(server, new HttpConnectionFactory(httpConfig)); + List connectionFactories = new ArrayList<>(); + if (config.isWebServiceHaProxyProtocolEnabled()) { + connectionFactories.add(new ProxyConnectionFactory()); + } + connectionFactories.add(httpConnectionFactory); + httpConnector = new ServerConnector(server, connectionFactories.toArray(new ConnectionFactory[0])); httpConnector.setPort(port.get()); httpConnector.setHost(pulsar.getBindAddress()); connectors.add(httpConnector); @@ -116,35 +150,34 @@ public WebService(PulsarService pulsar) throws PulsarServerException { Optional tlsPort = config.getWebServicePortTls(); if (tlsPort.isPresent()) { try { - SslContextFactory sslCtxFactory; - if (config.isTlsEnabledWithKeyStore()) { - sslCtxFactory = JettySslContextFactory.createServerSslContextWithKeystore( - config.getWebServiceTlsProvider(), - config.getTlsKeyStoreType(), - config.getTlsKeyStore(), - config.getTlsKeyStorePassword(), - config.isTlsAllowInsecureConnection(), - config.getTlsTrustStoreType(), - config.getTlsTrustStore(), - config.getTlsTrustStorePassword(), - config.isTlsRequireTrustedClientCertOnConnect(), - config.getWebServiceTlsCiphers(), - config.getWebServiceTlsProtocols(), - config.getTlsCertRefreshCheckDurationSec() - ); - } else { - sslCtxFactory = JettySslContextFactory.createServerSslContext( - config.getWebServiceTlsProvider(), - config.isTlsAllowInsecureConnection(), - config.getTlsTrustCertsFilePath(), - config.getTlsCertificateFilePath(), - config.getTlsKeyFilePath(), - config.isTlsRequireTrustedClientCertOnConnect(), - config.getWebServiceTlsCiphers(), - config.getWebServiceTlsProtocols(), - config.getTlsCertRefreshCheckDurationSec()); + PulsarSslConfiguration sslConfiguration = buildSslConfiguration(config); + this.sslFactory = (PulsarSslFactory) Class.forName(config.getSslFactoryPlugin()) + .getConstructor().newInstance(); + this.sslFactory.initialize(sslConfiguration); + this.sslFactory.createInternalSslContext(); + if (config.getTlsCertRefreshCheckDurationSec() > 0) { + this.sslContextRefreshTask = this.pulsar.getExecutor() + .scheduleWithFixedDelay(this::refreshSslContext, + config.getTlsCertRefreshCheckDurationSec(), + config.getTlsCertRefreshCheckDurationSec(), + TimeUnit.SECONDS); + } + SslContextFactory sslCtxFactory = + JettySslContextFactory.createSslContextFactory(config.getWebServiceTlsProvider(), + this.sslFactory, config.isTlsRequireTrustedClientCertOnConnect(), + config.getTlsCiphers(), config.getTlsProtocols()); + List connectionFactories = new ArrayList<>(); + if (config.isWebServiceHaProxyProtocolEnabled()) { + connectionFactories.add(new ProxyConnectionFactory()); + } + connectionFactories.add(new SslConnectionFactory(sslCtxFactory, httpConnectionFactory.getProtocol())); + connectionFactories.add(httpConnectionFactory); + // org.eclipse.jetty.server.AbstractConnectionFactory.getFactories contains similar logic + // this is needed for TLS authentication + if (httpConfig.getCustomizer(SecureRequestCustomizer.class) == null) { + httpConfig.addCustomizer(new SecureRequestCustomizer()); } - httpsConnector = new ServerConnector(server, sslCtxFactory, new HttpConnectionFactory(httpConfig)); + httpsConnector = new ServerConnector(server, connectionFactories.toArray(new ConnectionFactory[0])); httpsConnector.setPort(tlsPort.get()); httpsConnector.setHost(pulsar.getBindAddress()); connectors.add(httpsConnector); @@ -202,6 +235,7 @@ private static class FilterInitializer { private final FilterHolder authenticationFilterHolder; FilterInitializer(PulsarService pulsarService) { ServiceConfiguration config = pulsarService.getConfiguration(); + if (config.getMaxConcurrentHttpRequests() > 0) { FilterHolder filterHolder = new FilterHolder(QoSFilter.class); filterHolder.setInitParameter("maxRequests", String.valueOf(config.getMaxConcurrentHttpRequests())); @@ -210,11 +244,15 @@ private static class FilterInitializer { if (config.isHttpRequestsLimitEnabled()) { filterHolders.add(new FilterHolder( - new RateLimitingFilter(config.getHttpRequestsMaxPerSecond()))); + new RateLimitingFilter(config.getHttpRequestsMaxPerSecond(), + pulsarService.getOpenTelemetry().getMeter()))); } - boolean brokerInterceptorEnabled = - pulsarService.getBrokerInterceptor() != null && !config.isDisableBrokerInterceptors(); + // wait until the PulsarService is ready to serve incoming requests + filterHolders.add( + new FilterHolder(new WaitUntilPulsarServiceIsReadyForIncomingRequestsFilter(pulsarService))); + + boolean brokerInterceptorEnabled = pulsarService.getBrokerInterceptor() != null; if (brokerInterceptorEnabled) { ExceptionHandler handler = new ExceptionHandler(); // Enable PreInterceptFilter only when interceptors are enabled @@ -255,21 +293,56 @@ public void addFilters(ServletContextHandler context, boolean requiresAuthentica } } + // Filter that waits until the PulsarService is ready to serve incoming requests + private static class WaitUntilPulsarServiceIsReadyForIncomingRequestsFilter implements Filter { + private final PulsarService pulsarService; + + public WaitUntilPulsarServiceIsReadyForIncomingRequestsFilter(PulsarService pulsarService) { + this.pulsarService = pulsarService; + } + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + try { + // Wait until the PulsarService is ready to serve incoming requests + pulsarService.waitUntilReadyForIncomingRequests(); + } catch (ExecutionException e) { + ((HttpServletResponse) response).sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE, + "PulsarService failed to start."); + return; + } catch (InterruptedException e) { + ((HttpServletResponse) response).sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE, + "PulsarService is not ready."); + return; + } + chain.doFilter(request, response); + } + + @Override + public void destroy() { + + } + } } public void addServlet(String path, ServletHolder servletHolder, boolean requiresAuthentication, Map attributeMap) { - ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS); + ServletContextHandler servletContextHandler = new ServletContextHandler(ServletContextHandler.SESSIONS); // Notice: each context path should be unique, but there's nothing here to verify that - context.setContextPath(path); - context.addServlet(servletHolder, MATCH_ALL); + servletContextHandler.setContextPath(path); + servletContextHandler.addServlet(servletHolder, MATCH_ALL); if (attributeMap != null) { - attributeMap.forEach((key, value) -> { - context.setAttribute(key, value); - }); + attributeMap.forEach(servletContextHandler::setAttribute); } - filterInitializer.addFilters(context, requiresAuthentication); - handlers.add(context); + filterInitializer.addFilters(servletContextHandler, requiresAuthentication); + + handlers.add(servletContextHandler); } public void addStaticResources(String basePath, String resourcePath) { @@ -286,15 +359,22 @@ public void addStaticResources(String basePath, String resourcePath) { public void start() throws PulsarServerException { try { RequestLogHandler requestLogHandler = new RequestLogHandler(); - requestLogHandler.setRequestLog(JettyRequestLogFactory.createRequestLogger()); + boolean showDetailedAddresses = pulsar.getConfiguration().getWebServiceLogDetailedAddresses() != null + ? pulsar.getConfiguration().getWebServiceLogDetailedAddresses() : + (pulsar.getConfiguration().isWebServiceHaProxyProtocolEnabled() + || pulsar.getConfiguration().isWebServiceTrustXForwardedFor()); + RequestLog requestLogger = JettyRequestLogFactory.createRequestLogger(showDetailedAddresses, server); + requestLogHandler.setRequestLog(requestLogger); handlers.add(0, new ContextHandlerCollection()); handlers.add(requestLogHandler); ContextHandlerCollection contexts = new ContextHandlerCollection(); contexts.setHandlers(handlers.toArray(new Handler[handlers.size()])); + Handler handlerForContexts = GzipHandlerUtil.wrapWithGzipHandler(contexts, + pulsar.getConfig().getHttpServerGzipCompressionExcludedPaths()); HandlerCollection handlerCollection = new HandlerCollection(); - handlerCollection.setHandlers(new Handler[] { contexts, new DefaultHandler(), requestLogHandler }); + handlerCollection.setHandlers(new Handler[] {handlerForContexts, new DefaultHandler(), requestLogHandler}); // Metrics handler StatisticsHandler stats = new StatisticsHandler(); @@ -305,7 +385,6 @@ public void start() throws PulsarServerException { } catch (IllegalArgumentException e) { // Already registered. Eg: in unit tests } - handlers.add(stats); server.setHandler(stats); server.start(); @@ -346,6 +425,10 @@ public void close() throws PulsarServerException { jettyStatisticsCollector = null; } webServiceExecutor.join(); + if (this.sslContextRefreshTask != null) { + this.sslContextRefreshTask.cancel(true); + } + webExecutorThreadPoolStats.close(); this.executorStats.close(); log.info("Web service closed"); } catch (Exception e) { @@ -369,5 +452,35 @@ public Optional getListenPortHTTPS() { } } + protected PulsarSslConfiguration buildSslConfiguration(ServiceConfiguration serviceConfig) { + return PulsarSslConfiguration.builder() + .tlsKeyStoreType(serviceConfig.getTlsKeyStoreType()) + .tlsKeyStorePath(serviceConfig.getTlsKeyStore()) + .tlsKeyStorePassword(serviceConfig.getTlsKeyStorePassword()) + .tlsTrustStoreType(serviceConfig.getTlsTrustStoreType()) + .tlsTrustStorePath(serviceConfig.getTlsTrustStore()) + .tlsTrustStorePassword(serviceConfig.getTlsTrustStorePassword()) + .tlsCiphers(serviceConfig.getTlsCiphers()) + .tlsProtocols(serviceConfig.getTlsProtocols()) + .tlsTrustCertsFilePath(serviceConfig.getTlsTrustCertsFilePath()) + .tlsCertificateFilePath(serviceConfig.getTlsCertificateFilePath()) + .tlsKeyFilePath(serviceConfig.getTlsKeyFilePath()) + .allowInsecureConnection(serviceConfig.isTlsAllowInsecureConnection()) + .requireTrustedClientCertOnConnect(serviceConfig.isTlsRequireTrustedClientCertOnConnect()) + .tlsEnabledWithKeystore(serviceConfig.isTlsEnabledWithKeyStore()) + .tlsCustomParams(serviceConfig.getSslFactoryPluginParams()) + .serverMode(true) + .isHttps(true) + .build(); + } + + protected void refreshSslContext() { + try { + this.sslFactory.update(); + } catch (Exception e) { + log.error("Failed to refresh SSL context", e); + } + } + private static final Logger log = LoggerFactory.getLogger(WebService.class); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/api/RawReader.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/api/RawReader.java index 92a2c89f9bc9c..55483708fdf6a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/api/RawReader.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/api/RawReader.java @@ -22,6 +22,7 @@ import java.util.concurrent.CompletableFuture; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.RawReaderImpl; +import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; /** * Topic reader which receives raw messages (i.e. as they are stored in the managed ledger). @@ -32,11 +33,27 @@ public interface RawReader { */ static CompletableFuture create(PulsarClient client, String topic, String subscription) { + return create(client, topic, subscription, true); + } + + static CompletableFuture create(PulsarClient client, String topic, String subscription, + boolean createTopicIfDoesNotExist) { CompletableFuture> future = new CompletableFuture<>(); - RawReader r = new RawReaderImpl((PulsarClientImpl) client, topic, subscription, future); + RawReader r = + new RawReaderImpl((PulsarClientImpl) client, topic, subscription, future, createTopicIfDoesNotExist); return future.thenApply(__ -> r); } + static CompletableFuture create(PulsarClient client, + ConsumerConfigurationData consumerConfiguration, + boolean createTopicIfDoesNotExist) { + CompletableFuture> future = new CompletableFuture<>(); + RawReader r = new RawReaderImpl((PulsarClientImpl) client, + consumerConfiguration, future, createTopicIfDoesNotExist); + return future.thenApply(__ -> r); + } + + /** * Get the topic for the reader. * diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java index 4809ce1a04807..d8c491dab2906 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java @@ -38,21 +38,30 @@ import org.apache.pulsar.common.compression.CompressionCodec; import org.apache.pulsar.common.compression.CompressionCodecProvider; import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.compaction.MessageCompactionData; public class RawBatchConverter { public static boolean isReadableBatch(RawMessage msg) { ByteBuf payload = msg.getHeadersAndPayload(); MessageMetadata metadata = Commands.parseMessageMetadata(payload); + return isReadableBatch(metadata); + } + + public static boolean isReadableBatch(MessageMetadata metadata) { return metadata.hasNumMessagesInBatch() && metadata.getEncryptionKeysCount() == 0; } - public static List> extractIdsAndKeysAndSize(RawMessage msg) - throws IOException { + public static List extractMessageCompactionData(RawMessage msg, MessageMetadata metadata) + throws IOException { checkArgument(msg.getMessageIdData().getBatchIndex() == -1); ByteBuf payload = msg.getHeadersAndPayload(); - MessageMetadata metadata = Commands.parseMessageMetadata(payload); + if (metadata == null) { + metadata = Commands.parseMessageMetadata(payload); + } else { + Commands.skipMessageMetadata(payload); + } int batchSize = metadata.getNumMessagesInBatch(); CompressionType compressionType = metadata.getCompression(); @@ -60,28 +69,52 @@ public static List> extractIdsAndKey int uncompressedSize = metadata.getUncompressedSize(); ByteBuf uncompressedPayload = codec.decode(payload, uncompressedSize); - List> idsAndKeysAndSize = new ArrayList<>(); + List messageCompactionDataList = new ArrayList<>(); SingleMessageMetadata smm = new SingleMessageMetadata(); for (int i = 0; i < batchSize; i++) { ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload, - smm, - 0, batchSize); + smm, + 0, batchSize); MessageId id = new BatchMessageIdImpl(msg.getMessageIdData().getLedgerId(), - msg.getMessageIdData().getEntryId(), - msg.getMessageIdData().getPartition(), - i); + msg.getMessageIdData().getEntryId(), + msg.getMessageIdData().getPartition(), + i); if (!smm.isCompactedOut()) { - idsAndKeysAndSize.add(ImmutableTriple.of(id, - smm.hasPartitionKey() ? smm.getPartitionKey() : null, - smm.hasPayloadSize() ? smm.getPayloadSize() : 0)); + messageCompactionDataList.add(new MessageCompactionData(id, + smm.hasPartitionKey() ? smm.getPartitionKey() : null, + smm.hasPayloadSize() ? smm.getPayloadSize() : 0, smm.getEventTime())); } singleMessagePayload.release(); } uncompressedPayload.release(); + return messageCompactionDataList; + } + + public static List> extractIdsAndKeysAndSize( + RawMessage msg) + throws IOException { + List> idsAndKeysAndSize = new ArrayList<>(); + for (MessageCompactionData mcd : extractMessageCompactionData(msg, null)) { + idsAndKeysAndSize.add(ImmutableTriple.of(mcd.messageId(), mcd.key(), mcd.payloadSize())); + } + return idsAndKeysAndSize; + } + + public static List> extractIdsAndKeysAndSize( + RawMessage msg, MessageMetadata metadata) throws IOException { + List> idsAndKeysAndSize = new ArrayList<>(); + for (MessageCompactionData mcd : extractMessageCompactionData(msg, metadata)) { + idsAndKeysAndSize.add(ImmutableTriple.of(mcd.messageId(), mcd.key(), mcd.payloadSize())); + } return idsAndKeysAndSize; } + public static Optional rebatchMessage(RawMessage msg, + BiPredicate filter) throws IOException { + return rebatchMessage(msg, null, filter, true); + } + /** * Take a batched message and a filter, and returns a message with the only the sub-messages * which match the filter. Returns an empty optional if no messages match. @@ -89,7 +122,9 @@ public static List> extractIdsAndKey * NOTE: this message does not alter the reference count of the RawMessage argument. */ public static Optional rebatchMessage(RawMessage msg, - BiPredicate filter) + MessageMetadata metadata, + BiPredicate filter, + boolean retainNullKey) throws IOException { checkArgument(msg.getMessageIdData().getBatchIndex() == -1); @@ -100,9 +135,13 @@ public static Optional rebatchMessage(RawMessage msg, payload.skipBytes(Short.BYTES); int brokerEntryMetadataSize = payload.readInt(); payload.readerIndex(readerIndex); - brokerMeta = payload.readRetainedSlice(brokerEntryMetadataSize + Short.BYTES + Integer.BYTES); + brokerMeta = payload.readSlice(brokerEntryMetadataSize + Short.BYTES + Integer.BYTES); + } + if (metadata == null) { + metadata = Commands.parseMessageMetadata(payload); + } else { + Commands.skipMessageMetadata(payload); } - MessageMetadata metadata = Commands.parseMessageMetadata(payload); ByteBuf batchBuffer = PulsarByteBufAllocator.DEFAULT.buffer(payload.capacity()); CompressionType compressionType = metadata.getCompression(); @@ -124,10 +163,19 @@ public static Optional rebatchMessage(RawMessage msg, msg.getMessageIdData().getEntryId(), msg.getMessageIdData().getPartition(), i); - if (!singleMessageMetadata.hasPartitionKey()) { - messagesRetained++; - Commands.serializeSingleMessageInBatchWithPayload(singleMessageMetadata, - singleMessagePayload, batchBuffer); + if (singleMessageMetadata.isCompactedOut()) { + // we may read compacted out message from the compacted topic + Commands.serializeSingleMessageInBatchWithPayload(emptyMetadata, + Unpooled.EMPTY_BUFFER, batchBuffer); + } else if (!singleMessageMetadata.hasPartitionKey()) { + if (retainNullKey) { + messagesRetained++; + Commands.serializeSingleMessageInBatchWithPayload(singleMessageMetadata, + singleMessagePayload, batchBuffer); + } else { + Commands.serializeSingleMessageInBatchWithPayload(emptyMetadata, + Unpooled.EMPTY_BUFFER, batchBuffer); + } } else if (filter.test(singleMessageMetadata.getPartitionKey(), id) && singleMessagePayload.readableBytes() > 0) { messagesRetained++; @@ -152,7 +200,7 @@ public static Optional rebatchMessage(RawMessage msg, if (brokerMeta != null) { CompositeByteBuf compositeByteBuf = PulsarByteBufAllocator.DEFAULT.compositeDirectBuffer(); - compositeByteBuf.addComponents(true, brokerMeta, metadataAndPayload); + compositeByteBuf.addComponents(true, brokerMeta.retain(), metadataAndPayload); metadataAndPayload = compositeByteBuf; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java index 7e1c2cd5e3fe3..374f1e30c0a89 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java @@ -23,6 +23,7 @@ import java.util.Set; import org.apache.pulsar.client.api.CryptoKeyReader; import org.apache.pulsar.client.api.MessageCrypto; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.impl.crypto.MessageCryptoBc; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; @@ -44,17 +45,17 @@ * [(k1, v1), (k2, v1), (k3, v1), (k1, v2), (k2, v2), (k3, v2), (k1, v3), (k2, v3), (k3, v3)] */ public class RawBatchMessageContainerImpl extends BatchMessageContainerImpl { - MessageCrypto msgCrypto; - Set encryptionKeys; - CryptoKeyReader cryptoKeyReader; + private MessageCrypto msgCrypto; + private Set encryptionKeys; + private CryptoKeyReader cryptoKeyReader; + private MessageIdAdv lastAddedMessageId; - public RawBatchMessageContainerImpl(int maxNumMessagesInBatch, int maxBytesInBatch) { + public RawBatchMessageContainerImpl() { super(); this.compressionType = CompressionType.NONE; this.compressor = new CompressionCodecNone(); - this.maxNumMessagesInBatch = maxNumMessagesInBatch; - this.maxBytesInBatch = maxBytesInBatch; } + private ByteBuf encrypt(ByteBuf compressedPayload) { if (msgCrypto == null) { return compressedPayload; @@ -90,6 +91,28 @@ public void setCryptoKeyReader(CryptoKeyReader cryptoKeyReader) { this.cryptoKeyReader = cryptoKeyReader; } + @Override + public boolean add(MessageImpl msg, SendCallback callback) { + this.lastAddedMessageId = (MessageIdAdv) msg.getMessageId(); + return super.add(msg, callback); + } + + @Override + protected boolean isBatchFull() { + return false; + } + + @Override + public boolean haveEnoughSpace(MessageImpl msg) { + if (lastAddedMessageId == null) { + return true; + } + // Keep same batch compact to same batch. + MessageIdAdv msgId = (MessageIdAdv) msg.getMessageId(); + return msgId.getLedgerId() == lastAddedMessageId.getLedgerId() + && msgId.getEntryId() == lastAddedMessageId.getEntryId(); + } + /** * Serializes the batched messages and return the ByteBuf. * It sets the CompressionType and Encryption Keys from the batched messages. @@ -164,8 +187,15 @@ public ByteBuf toByteBuf() { idData.writeTo(buf); buf.writeInt(metadataAndPayload.readableBytes()); buf.writeBytes(metadataAndPayload); + metadataAndPayload.release(); encryptedPayload.release(); clear(); return buf; } + + @Override + public void clear() { + this.lastAddedMessageId = null; + super.clear(); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java index 9a1c972b2cc98..5ac051d227119 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java @@ -51,7 +51,8 @@ public class RawReaderImpl implements RawReader { private RawConsumerImpl consumer; public RawReaderImpl(PulsarClientImpl client, String topic, String subscription, - CompletableFuture> consumerFuture) { + CompletableFuture> consumerFuture, + boolean createTopicIfDoesNotExist) { consumerConfiguration = new ConsumerConfigurationData<>(); consumerConfiguration.getTopicNames().add(topic); consumerConfiguration.setSubscriptionName(subscription); @@ -59,11 +60,19 @@ public RawReaderImpl(PulsarClientImpl client, String topic, String subscription, consumerConfiguration.setReceiverQueueSize(DEFAULT_RECEIVER_QUEUE_SIZE); consumerConfiguration.setReadCompacted(true); consumerConfiguration.setSubscriptionInitialPosition(SubscriptionInitialPosition.Earliest); + consumerConfiguration.setAckReceiptEnabled(true); - consumer = new RawConsumerImpl(client, consumerConfiguration, - consumerFuture); + consumer = new RawConsumerImpl(client, consumerConfiguration, consumerFuture, createTopicIfDoesNotExist); } + public RawReaderImpl(PulsarClientImpl client, ConsumerConfigurationData consumerConfiguration, + CompletableFuture> consumerFuture, + boolean createTopicIfDoesNotExist) { + this.consumerConfiguration = consumerConfiguration; + consumer = new RawConsumerImpl(client, consumerConfiguration, consumerFuture, createTopicIfDoesNotExist); + } + + @Override public String getTopic() { return consumerConfiguration.getTopicNames().stream() @@ -110,7 +119,7 @@ static class RawConsumerImpl extends ConsumerImpl { final Queue> pendingRawReceives; RawConsumerImpl(PulsarClientImpl client, ConsumerConfigurationData conf, - CompletableFuture> consumerFuture) { + CompletableFuture> consumerFuture, boolean createTopicIfDoesNotExist) { super(client, conf.getSingleTopic(), conf, @@ -122,7 +131,7 @@ static class RawConsumerImpl extends ConsumerImpl { MessageId.earliest, 0 /* startMessageRollbackDurationInSec */, Schema.BYTES, null, - true + createTopicIfDoesNotExist ); incomingRawMessages = new GrowableArrayBlockingQueue<>(); pendingRawReceives = new ConcurrentLinkedQueue<>(); @@ -151,14 +160,13 @@ void tryCompletePending() { // TODO message validation numMsg = 1; } + MessageIdData messageId = messageAndCnx.msg.getMessageIdData(); + lastDequeuedMessageId = new BatchMessageIdImpl(messageId.getLedgerId(), messageId.getEntryId(), + messageId.getPartition(), numMsg - 1); if (!future.complete(messageAndCnx.msg)) { messageAndCnx.msg.close(); closeAsync(); } - MessageIdData messageId = messageAndCnx.msg.getMessageIdData(); - lastDequeuedMessageId = new BatchMessageIdImpl(messageId.getLedgerId(), messageId.getEntryId(), - messageId.getPartition(), numMsg - 1); - ClientCnx currentCnx = cnx(); if (currentCnx == messageAndCnx.cnx) { increaseAvailablePermits(currentCnx, numMsg); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/ConsistentHashingTopicBundleAssigner.java b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/ConsistentHashingTopicBundleAssigner.java new file mode 100644 index 0000000000000..1e8b0d03392cc --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/ConsistentHashingTopicBundleAssigner.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.naming; + +import com.google.common.hash.Hashing; +import java.nio.charset.StandardCharsets; +import org.apache.pulsar.broker.PulsarService; + +public class ConsistentHashingTopicBundleAssigner implements TopicBundleAssignmentStrategy { + @Override + public NamespaceBundle findBundle(TopicName topicName, NamespaceBundles namespaceBundles) { + long hashCode = Hashing.crc32().hashString(topicName.toString(), StandardCharsets.UTF_8).padToLong(); + NamespaceBundle bundle = namespaceBundles.getBundle(hashCode); + if (topicName.getDomain().equals(TopicDomain.non_persistent)) { + bundle.setHasNonPersistentTopic(true); + } + return bundle; + } + + @Override + public void init(PulsarService pulsarService) { + } + +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java index 937d2763767b6..2b285cbb0e2ab 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java @@ -43,6 +43,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; +import lombok.Getter; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.PulsarService; @@ -50,10 +51,10 @@ import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerWrapper; import org.apache.pulsar.broker.resources.LocalPoliciesResources; import org.apache.pulsar.broker.resources.PulsarResources; -import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.common.policies.data.BundlesData; import org.apache.pulsar.common.policies.data.LocalPolicies; import org.apache.pulsar.common.policies.data.Policies; +import org.apache.pulsar.common.util.Backoff; import org.apache.pulsar.metadata.api.Notification; import org.apache.pulsar.policies.data.loadbalancer.BundleData; import org.apache.pulsar.stats.CacheMetricsCollector; @@ -68,6 +69,10 @@ public class NamespaceBundleFactory { private final AsyncLoadingCache bundlesCache; private final PulsarService pulsar; + + @Getter + private final TopicBundleAssignmentStrategy topicBundleAssignmentStrategy; + private final Duration maxRetryDuration = Duration.ofSeconds(10); public NamespaceBundleFactory(PulsarService pulsar, HashFunction hashFunc) { @@ -82,6 +87,8 @@ public NamespaceBundleFactory(PulsarService pulsar, HashFunction hashFunc) { pulsar.getLocalMetadataStore().registerListener(this::handleMetadataStoreNotification); this.pulsar = pulsar; + + topicBundleAssignmentStrategy = TopicBundleAssignmentFactory.create(pulsar); } private CompletableFuture loadBundles(NamespaceName namespace, Executor executor) { @@ -418,4 +425,4 @@ public static Range getRange(Long lowerEndpoint, Long upperEndpoint) { (upperEndpoint.equals(NamespaceBundles.FULL_UPPER_BOUND)) ? BoundType.CLOSED : BoundType.OPEN); } -} \ No newline at end of file +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundles.java b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundles.java index cb7e135662c01..fa7baeaa6067b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundles.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundles.java @@ -45,8 +45,8 @@ public class NamespaceBundles { public static final Long FULL_LOWER_BOUND = 0x00000000L; public static final Long FULL_UPPER_BOUND = 0xffffffffL; - private final NamespaceBundle fullBundle; + private final NamespaceBundle fullBundle; private final Optional> localPolicies; public NamespaceBundles(NamespaceName nsname, NamespaceBundleFactory factory, @@ -94,13 +94,8 @@ public NamespaceBundles(NamespaceName nsname, NamespaceBundleFactory factory, } public NamespaceBundle findBundle(TopicName topicName) { - checkArgument(this.nsname.equals(topicName.getNamespaceObject())); - long hashCode = factory.getLongHashCode(topicName.toString()); - NamespaceBundle bundle = getBundle(hashCode); - if (topicName.getDomain().equals(TopicDomain.non_persistent)) { - bundle.setHasNonPersistentTopic(true); - } - return bundle; + checkArgument(nsname.equals(topicName.getNamespaceObject())); + return factory.getTopicBundleAssignmentStrategy().findBundle(topicName, this); } public List getBundles() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/TopicBundleAssignmentFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/TopicBundleAssignmentFactory.java new file mode 100644 index 0000000000000..164d9a3f1deef --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/TopicBundleAssignmentFactory.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.naming; + +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.common.util.Reflections; + +public class TopicBundleAssignmentFactory { + + public static final String DEFAULT_TOPIC_BUNDLE_ASSIGNMENT_STRATEGY = + "org.apache.pulsar.common.naming.ConsistentHashingTopicBundleAssigner"; + + private static volatile TopicBundleAssignmentStrategy strategy; + + public static TopicBundleAssignmentStrategy create(PulsarService pulsar) { + if (strategy != null) { + return strategy; + } + synchronized (TopicBundleAssignmentFactory.class) { + if (strategy != null) { + return strategy; + } + String topicBundleAssignmentStrategy = getTopicBundleAssignmentStrategy(pulsar); + try { + TopicBundleAssignmentStrategy tempStrategy = + Reflections.createInstance(topicBundleAssignmentStrategy, + TopicBundleAssignmentStrategy.class, Thread.currentThread().getContextClassLoader()); + tempStrategy.init(pulsar); + strategy = tempStrategy; + return strategy; + } catch (Exception e) { + throw new RuntimeException( + "Could not load TopicBundleAssignmentStrategy:" + topicBundleAssignmentStrategy, e); + } + } + } + + private static String getTopicBundleAssignmentStrategy(PulsarService pulsar) { + if (pulsar == null || pulsar.getConfiguration() == null) { + return DEFAULT_TOPIC_BUNDLE_ASSIGNMENT_STRATEGY; + } + return pulsar.getConfiguration().getTopicBundleAssignmentStrategy(); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/TopicBundleAssignmentStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/TopicBundleAssignmentStrategy.java new file mode 100644 index 0000000000000..b43ca4afa440e --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/TopicBundleAssignmentStrategy.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.naming; + +import org.apache.pulsar.broker.PulsarService; + +public interface TopicBundleAssignmentStrategy { + NamespaceBundle findBundle(TopicName topicName, NamespaceBundles namespaceBundles); + + void init(PulsarService pulsarService); +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/AbstractTwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/AbstractTwoPhaseCompactor.java new file mode 100644 index 0000000000000..ddfe8825a8888 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/AbstractTwoPhaseCompactor.java @@ -0,0 +1,439 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.compaction; + +import io.netty.buffer.ByteBuf; +import java.io.IOException; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.function.BiPredicate; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.mledger.impl.LedgerMetadataUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.RawMessage; +import org.apache.pulsar.client.api.RawReader; +import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.client.impl.RawBatchConverter; +import org.apache.pulsar.common.api.proto.MessageMetadata; +import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.protocol.Markers; +import org.apache.pulsar.common.util.FutureUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Compaction will go through the topic in two passes. The first pass + * selects latest offset for each key in the topic. Then the second pass + * writes these values to a ledger. + * + *

The two passes are required to avoid holding the payloads of each of + * the latest values in memory, as the payload can be many orders of + * magnitude larger than a message id. + */ +public abstract class AbstractTwoPhaseCompactor extends Compactor { + + private static final Logger log = LoggerFactory.getLogger(AbstractTwoPhaseCompactor.class); + protected static final int MAX_OUTSTANDING = 500; + protected final Duration phaseOneLoopReadTimeout; + protected final boolean topicCompactionRetainNullKey; + + public AbstractTwoPhaseCompactor(ServiceConfiguration conf, + PulsarClient pulsar, + BookKeeper bk, + ScheduledExecutorService scheduler) { + super(conf, pulsar, bk, scheduler); + phaseOneLoopReadTimeout = Duration.ofSeconds( + conf.getBrokerServiceCompactionPhaseOneLoopTimeInSeconds()); + topicCompactionRetainNullKey = conf.isTopicCompactionRetainNullKey(); + } + + protected abstract Map toLatestMessageIdForKey(Map latestForKey); + + protected abstract boolean compactMessage(String topic, Map latestForKey, + RawMessage m, MessageMetadata metadata, MessageId id); + + + protected abstract boolean compactBatchMessage(String topic, Map latestForKey, + RawMessage m, + MessageMetadata metadata, MessageId id); + + @Override + protected CompletableFuture doCompaction(RawReader reader, BookKeeper bk) { + return reader.hasMessageAvailableAsync() + .thenCompose(available -> { + if (available) { + return phaseOne(reader).thenCompose( + (r) -> phaseTwo(reader, r.from, r.to, r.lastReadId, toLatestMessageIdForKey(r.latestForKey), bk)); + } else { + log.info("Skip compaction of the empty topic {}", reader.getTopic()); + return CompletableFuture.completedFuture(-1L); + } + }); + } + + private CompletableFuture phaseOne(RawReader reader) { + Map latestForKey = new HashMap<>(); + CompletableFuture loopPromise = new CompletableFuture<>(); + + reader.getLastMessageIdAsync() + .thenAccept(lastMessageId -> { + log.info("Commencing phase one of compaction for {}, reading to {}", + reader.getTopic(), lastMessageId); + // Each entry is processed as a whole, discard the batchIndex part deliberately. + MessageIdImpl lastImpl = (MessageIdImpl) lastMessageId; + MessageIdImpl lastEntryMessageId = new MessageIdImpl(lastImpl.getLedgerId(), + lastImpl.getEntryId(), + lastImpl.getPartitionIndex()); + phaseOneLoop(reader, Optional.empty(), Optional.empty(), lastEntryMessageId, latestForKey, + loopPromise); + }).exceptionally(ex -> { + loopPromise.completeExceptionally(ex); + return null; + }); + + return loopPromise; + } + + private void phaseOneLoop(RawReader reader, + Optional firstMessageId, + Optional toMessageId, + MessageId lastMessageId, + Map latestForKey, + CompletableFuture loopPromise) { + if (loopPromise.isDone()) { + return; + } + CompletableFuture future = reader.readNextAsync(); + FutureUtil.addTimeoutHandling(future, + phaseOneLoopReadTimeout, scheduler, + () -> FutureUtil.createTimeoutException("Timeout", getClass(), "phaseOneLoop(...)")); + + future.thenAcceptAsync(m -> { + try (m) { + MessageId id = m.getMessageId(); + boolean deletedMessage = false; + mxBean.addCompactionReadOp(reader.getTopic(), m.getHeadersAndPayload().readableBytes()); + MessageMetadata metadata = Commands.parseMessageMetadata(m.getHeadersAndPayload()); + if (Markers.isServerOnlyMarker(metadata)) { + mxBean.addCompactionRemovedEvent(reader.getTopic()); + deletedMessage = true; + } else if (RawBatchConverter.isReadableBatch(metadata)) { + deletedMessage = compactBatchMessage(reader.getTopic(), latestForKey, m, metadata, id); + } else { + deletedMessage = compactMessage(reader.getTopic(), latestForKey, m, metadata, id); + } + MessageId first = firstMessageId.orElse(deletedMessage ? null : id); + MessageId to = deletedMessage ? toMessageId.orElse(null) : id; + if (id.compareTo(lastMessageId) == 0) { + loopPromise.complete(new PhaseOneResult(first == null ? id : first, to == null ? id : to, + lastMessageId, latestForKey)); + } else { + phaseOneLoop(reader, + Optional.ofNullable(first), + Optional.ofNullable(to), + lastMessageId, + latestForKey, loopPromise); + } + } + }, scheduler).exceptionally(ex -> { + loopPromise.completeExceptionally(ex); + return null; + }); + } + + private CompletableFuture phaseTwo(RawReader reader, MessageId from, MessageId to, + MessageId lastReadId, + Map latestForKey, BookKeeper bk) { + Map metadata = + LedgerMetadataUtils.buildMetadataForCompactedLedger(reader.getTopic(), to.toByteArray()); + return createLedger(bk, metadata).thenCompose((ledger) -> { + log.info( + "Commencing phase two of compaction for {}, from {} to {}, compacting {} keys to ledger {}", + reader.getTopic(), from, to, latestForKey.size(), ledger.getId()); + return phaseTwoSeekThenLoop(reader, from, to, lastReadId, latestForKey, bk, ledger); + }); + } + + private CompletableFuture phaseTwoSeekThenLoop(RawReader reader, MessageId from, + MessageId to, + MessageId lastReadId, Map latestForKey, BookKeeper bk, + LedgerHandle ledger) { + CompletableFuture promise = new CompletableFuture<>(); + + reader.seekAsync(from).thenCompose((v) -> { + Semaphore outstanding = new Semaphore(MAX_OUTSTANDING); + CompletableFuture loopPromise = new CompletableFuture<>(); + phaseTwoLoop(reader, to, latestForKey, ledger, outstanding, loopPromise, MessageId.earliest); + return loopPromise; + }).thenCompose((v) -> closeLedger(ledger)) + .thenCompose((v) -> reader.acknowledgeCumulativeAsync(lastReadId, + Map.of(COMPACTED_TOPIC_LEDGER_PROPERTY, ledger.getId()))) + .whenComplete((res, exception) -> { + if (exception != null) { + deleteLedger(bk, ledger).whenComplete((res2, exception2) -> { + if (exception2 != null) { + log.warn("Cleanup of ledger {} for failed", ledger, exception2); + } + // complete with original exception + promise.completeExceptionally(exception); + }); + } else { + promise.complete(ledger.getId()); + } + }); + return promise; + } + + private void phaseTwoLoop(RawReader reader, MessageId to, Map latestForKey, + LedgerHandle lh, Semaphore outstanding, CompletableFuture promise, + MessageId lastCompactedMessageId) { + if (promise.isDone()) { + return; + } + reader.readNextAsync().thenAcceptAsync(m -> { + if (promise.isDone()) { + m.close(); + return; + } + + if (m.getMessageId().compareTo(lastCompactedMessageId) <= 0) { + m.close(); + phaseTwoLoop(reader, to, latestForKey, lh, outstanding, promise, lastCompactedMessageId); + return; + } + + try { + MessageId id = m.getMessageId(); + Optional messageToAdd = Optional.empty(); + mxBean.addCompactionReadOp(reader.getTopic(), m.getHeadersAndPayload().readableBytes()); + MessageMetadata metadata = Commands.parseMessageMetadata(m.getHeadersAndPayload()); + if (Markers.isServerOnlyMarker(metadata)) { + messageToAdd = Optional.empty(); + } else if (RawBatchConverter.isReadableBatch(metadata)) { + try { + messageToAdd = rebatchMessage(reader.getTopic(), + m, metadata, (key, subid) -> subid.equals(latestForKey.get(key)), + topicCompactionRetainNullKey); + } catch (IOException ioe) { + log.info("Error decoding batch for message {}. Whole batch will be included in output", + id, ioe); + messageToAdd = Optional.of(m); + } + } else { + Pair keyAndSize = extractKeyAndSize(m, metadata); + MessageId msg; + if (keyAndSize == null) { + messageToAdd = topicCompactionRetainNullKey ? Optional.of(m) : Optional.empty(); + } else if ((msg = latestForKey.get(keyAndSize.getLeft())) != null + && msg.equals(id)) { // consider message only if present into latestForKey map + if (keyAndSize.getRight() <= 0) { + promise.completeExceptionally(new IllegalArgumentException( + "Compaction phase found empty record from sorted key-map")); + } + messageToAdd = Optional.of(m); + } + } + + if (messageToAdd.isPresent()) { + RawMessage message = messageToAdd.get(); + try { + outstanding.acquire(); + CompletableFuture addFuture = addToCompactedLedger(lh, message, reader.getTopic()) + .whenComplete((res, exception2) -> { + outstanding.release(); + if (exception2 != null) { + promise.completeExceptionally(exception2); + } + }); + if (to.equals(id)) { + // make sure all inflight writes have finished + outstanding.acquire(MAX_OUTSTANDING); + addFuture.whenComplete((res, exception2) -> { + if (exception2 == null) { + promise.complete(null); + } + }); + return; + } + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + promise.completeExceptionally(ie); + } finally { + if (message != m) { + message.close(); + } + } + } else if (to.equals(id)) { + // Reached to last-id and phase-one found it deleted-message while iterating on ledger so, + // not present under latestForKey. Complete the compaction. + try { + // make sure all inflight writes have finished + outstanding.acquire(MAX_OUTSTANDING); + promise.complete(null); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + promise.completeExceptionally(e); + } + return; + } + phaseTwoLoop(reader, to, latestForKey, lh, outstanding, promise, m.getMessageId()); + } finally { + m.close(); + } + }, scheduler).exceptionally(ex -> { + promise.completeExceptionally(ex); + return null; + }); + } + + protected CompletableFuture createLedger(BookKeeper bk, + Map metadata) { + CompletableFuture bkf = new CompletableFuture<>(); + + try { + bk.asyncCreateLedger(conf.getManagedLedgerDefaultEnsembleSize(), + conf.getManagedLedgerDefaultWriteQuorum(), + conf.getManagedLedgerDefaultAckQuorum(), + Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, + Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD, + (rc, ledger, ctx) -> { + if (rc != BKException.Code.OK) { + bkf.completeExceptionally(BKException.create(rc)); + } else { + bkf.complete(ledger); + } + }, null, metadata); + } catch (Throwable t) { + log.error("Encountered unexpected error when creating compaction ledger", t); + return FutureUtil.failedFuture(t); + } + return bkf; + } + + protected CompletableFuture deleteLedger(BookKeeper bk, LedgerHandle lh) { + CompletableFuture bkf = new CompletableFuture<>(); + try { + bk.asyncDeleteLedger(lh.getId(), + (rc, ctx) -> { + if (rc != BKException.Code.OK) { + bkf.completeExceptionally(BKException.create(rc)); + } else { + bkf.complete(null); + } + }, null); + } catch (Throwable t) { + return FutureUtil.failedFuture(t); + } + return bkf; + } + + protected CompletableFuture closeLedger(LedgerHandle lh) { + CompletableFuture bkf = new CompletableFuture<>(); + try { + lh.asyncClose((rc, ledger, ctx) -> { + if (rc != BKException.Code.OK) { + bkf.completeExceptionally(BKException.create(rc)); + } else { + bkf.complete(null); + } + }, null); + } catch (Throwable t) { + return FutureUtil.failedFuture(t); + } + return bkf; + } + + private CompletableFuture addToCompactedLedger(LedgerHandle lh, RawMessage m, + String topic) { + CompletableFuture bkf = new CompletableFuture<>(); + ByteBuf serialized = m.serialize(); + try { + mxBean.addCompactionWriteOp(topic, m.getHeadersAndPayload().readableBytes()); + long start = System.nanoTime(); + lh.asyncAddEntry(serialized, + (rc, ledger, eid, ctx) -> { + mxBean.addCompactionLatencyOp(topic, System.nanoTime() - start, TimeUnit.NANOSECONDS); + if (rc != BKException.Code.OK) { + bkf.completeExceptionally(BKException.create(rc)); + } else { + bkf.complete(null); + } + }, null); + } catch (Throwable t) { + return FutureUtil.failedFuture(t); + } + return bkf; + } + + protected Pair extractKeyAndSize(RawMessage m, MessageMetadata msgMetadata) { + ByteBuf headersAndPayload = m.getHeadersAndPayload(); + if (msgMetadata.hasPartitionKey()) { + int size = headersAndPayload.readableBytes(); + if (msgMetadata.hasUncompressedSize()) { + size = msgMetadata.getUncompressedSize(); + } + return Pair.of(msgMetadata.getPartitionKey(), size); + } else { + return null; + } + } + + + protected Optional rebatchMessage(String topic, RawMessage msg, + MessageMetadata metadata, + BiPredicate filter, + boolean retainNullKey) + throws IOException { + if (log.isDebugEnabled()) { + log.debug("Rebatching message {} for topic {}", msg.getMessageId(), topic); + } + return RawBatchConverter.rebatchMessage(msg, metadata, filter, retainNullKey); + } + + protected static class PhaseOneResult { + + final MessageId from; + final MessageId to; // last undeleted messageId + final MessageId lastReadId; // last read messageId + final Map latestForKey; + + PhaseOneResult(MessageId from, MessageId to, MessageId lastReadId, + Map latestForKey) { + this.from = from; + this.to = to; + this.lastReadId = lastReadId; + this.latestForKey = latestForKey; + } + } + + public long getPhaseOneLoopReadTimeoutInSeconds() { + return phaseOneLoopReadTimeout.getSeconds(); + } +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopic.java index e1a10b3bbb212..563b826ac74c7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopic.java @@ -29,8 +29,19 @@ public interface CompactedTopic { CompletableFuture newCompactedLedger(Position p, long compactedLedgerId); CompletableFuture deleteCompactedLedger(long compactedLedgerId); + + /** + * Read entries from compacted topic. + * + * @deprecated Use {@link CompactedTopicUtils#asyncReadCompactedEntries(TopicCompactionService, ManagedCursor, + * int, long, org.apache.bookkeeper.mledger.Position, boolean, ReadEntriesCallback, boolean, Consumer)} + * instead. + */ + @Deprecated void asyncReadEntriesOrWait(ManagedCursor cursor, - int numberOfEntriesToRead, + int maxEntries, + long bytesToRead, + Position maxReadPosition, boolean isFirstRead, ReadEntriesCallback callback, Consumer consumer); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java index c8114f9adb652..141320b54e7a4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java @@ -32,6 +32,10 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Predicate; +import javax.annotation.Nullable; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.LedgerEntry; @@ -41,8 +45,8 @@ import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.EntryImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherSingleActiveConsumer.ReadEntriesCtx; import org.apache.pulsar.client.api.MessageId; @@ -52,15 +56,19 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Note: If you want to guarantee that strong consistency between `compactionHorizon` and `compactedTopicContext`, + * you need to call getting them method in "synchronized(CompactedTopicImpl){ ... }" lock block. + */ public class CompactedTopicImpl implements CompactedTopic { static final long NEWER_THAN_COMPACTED = -0xfeed0fbaL; static final long COMPACT_LEDGER_EMPTY = -0xfeed0fbbL; - static final int DEFAULT_STARTPOINT_CACHE_SIZE = 100; + static final int DEFAULT_MAX_CACHE_SIZE = 100; private final BookKeeper bk; - private PositionImpl compactionHorizon = null; - private CompletableFuture compactedTopicContext = null; + private volatile Position compactionHorizon = null; + private volatile CompletableFuture compactedTopicContext = null; public CompactedTopicImpl(BookKeeper bk) { this.bk = bk; @@ -69,14 +77,14 @@ public CompactedTopicImpl(BookKeeper bk) { @Override public CompletableFuture newCompactedLedger(Position p, long compactedLedgerId) { synchronized (this) { - compactionHorizon = (PositionImpl) p; - CompletableFuture previousContext = compactedTopicContext; compactedTopicContext = openCompactedLedger(bk, compactedLedgerId); + compactionHorizon = p; + // delete the ledger from the old context once the new one is open - return compactedTopicContext.thenCompose(__ -> - previousContext != null ? previousContext : CompletableFuture.completedFuture(null)); + return compactedTopicContext.thenCompose( + __ -> previousContext != null ? previousContext : CompletableFuture.completedFuture(null)); } } @@ -86,48 +94,55 @@ public CompletableFuture deleteCompactedLedger(long compactedLedgerId) { } @Override + @Deprecated public void asyncReadEntriesOrWait(ManagedCursor cursor, - int numberOfEntriesToRead, + int maxEntries, + long bytesToRead, + Position maxReadPosition, boolean isFirstRead, ReadEntriesCallback callback, Consumer consumer) { - synchronized (this) { - PositionImpl cursorPosition; - if (isFirstRead && MessageId.earliest.equals(consumer.getStartMessageId())){ - cursorPosition = PositionImpl.EARLIEST; + Position cursorPosition; + boolean readFromEarliest = isFirstRead && MessageId.earliest.equals(consumer.getStartMessageId()) + && (!cursor.isDurable() || cursor.getName().equals(Compactor.COMPACTION_SUBSCRIPTION) + || cursor.getMarkDeletedPosition() == null + || cursor.getMarkDeletedPosition().getEntryId() == -1L); + if (readFromEarliest){ + cursorPosition = PositionFactory.EARLIEST; } else { - cursorPosition = (PositionImpl) cursor.getReadPosition(); + cursorPosition = cursor.getReadPosition(); } // TODO: redeliver epoch link https://github.com/apache/pulsar/issues/13690 ReadEntriesCtx readEntriesCtx = ReadEntriesCtx.create(consumer, DEFAULT_CONSUMER_EPOCH); - if (compactionHorizon == null - || compactionHorizon.compareTo(cursorPosition) < 0) { - cursor.asyncReadEntriesOrWait(numberOfEntriesToRead, callback, readEntriesCtx, PositionImpl.LATEST); + + final Position currentCompactionHorizon = compactionHorizon; + + if (currentCompactionHorizon == null + || currentCompactionHorizon.compareTo(cursorPosition) < 0) { + cursor.asyncReadEntriesOrWait(maxEntries, bytesToRead, callback, readEntriesCtx, maxReadPosition); } else { + int numberOfEntriesToRead = cursor.applyMaxSizeCap(maxEntries, bytesToRead); + compactedTopicContext.thenCompose( (context) -> findStartPoint(cursorPosition, context.ledger.getLastAddConfirmed(), context.cache) .thenCompose((startPoint) -> { // do not need to read the compaction ledger if it is empty. // the cursor just needs to be set to the compaction horizon - if (startPoint == COMPACT_LEDGER_EMPTY) { - cursor.seek(compactionHorizon.getNext()); + if (startPoint == COMPACT_LEDGER_EMPTY || startPoint == NEWER_THAN_COMPACTED) { + cursor.seek(currentCompactionHorizon.getNext()); callback.readEntriesComplete(Collections.emptyList(), readEntriesCtx); return CompletableFuture.completedFuture(null); - } - if (startPoint == NEWER_THAN_COMPACTED && compactionHorizon.compareTo(cursorPosition) < 0) { - cursor.asyncReadEntriesOrWait(numberOfEntriesToRead, callback, readEntriesCtx, - PositionImpl.LATEST); - return CompletableFuture.completedFuture(null); } else { long endPoint = Math.min(context.ledger.getLastAddConfirmed(), - startPoint + numberOfEntriesToRead); - if (startPoint == NEWER_THAN_COMPACTED) { - cursor.seek(compactionHorizon.getNext()); - callback.readEntriesComplete(Collections.emptyList(), readEntriesCtx); - return CompletableFuture.completedFuture(null); - } + startPoint + (numberOfEntriesToRead - 1)); return readEntries(context.ledger, startPoint, endPoint) .thenAccept((entries) -> { + long entriesSize = 0; + for (Entry entry : entries) { + entriesSize += entry.getLength(); + } + cursor.updateReadStats(entries.size(), entriesSize); + Entry lastEntry = entries.get(entries.size() - 1); // The compaction task depends on the last snapshot and the incremental // entries to build the new snapshot. So for the compaction cursor, we @@ -141,7 +156,7 @@ public void asyncReadEntriesOrWait(ManagedCursor cursor, })) .exceptionally((exception) -> { if (exception.getCause() instanceof NoSuchElementException) { - cursor.seek(compactionHorizon.getNext()); + cursor.seek(currentCompactionHorizon.getNext()); callback.readEntriesComplete(Collections.emptyList(), readEntriesCtx); } else { callback.readEntriesFailed(new ManagedLedgerException(exception), readEntriesCtx); @@ -149,10 +164,9 @@ public void asyncReadEntriesOrWait(ManagedCursor cursor, return null; }); } - } } - static CompletableFuture findStartPoint(PositionImpl p, + static CompletableFuture findStartPoint(Position p, long lastEntryId, AsyncLoadingCache cache) { CompletableFuture promise = new CompletableFuture<>(); @@ -166,7 +180,7 @@ static CompletableFuture findStartPoint(PositionImpl p, } @VisibleForTesting - static void findStartPointLoop(PositionImpl p, long start, long end, + static void findStartPointLoop(Position p, long start, long end, CompletableFuture promise, AsyncLoadingCache cache) { long midpoint = start + ((end - start) / 2); @@ -240,7 +254,7 @@ private static CompletableFuture openCompactedLedger(Book } }, null); return promise.thenApply((ledger) -> new CompactedTopicContext( - ledger, createCache(ledger, DEFAULT_STARTPOINT_CACHE_SIZE))); + ledger, createCache(ledger, DEFAULT_MAX_CACHE_SIZE))); } private static CompletableFuture tryDeleteCompactedLedger(BookKeeper bk, long id) { @@ -258,7 +272,7 @@ private static CompletableFuture tryDeleteCompactedLedger(BookKeeper bk, l return promise; } - private static CompletableFuture> readEntries(LedgerHandle lh, long from, long to) { + static CompletableFuture> readEntries(LedgerHandle lh, long from, long to) { CompletableFuture> promise = new CompletableFuture<>(); lh.asyncReadEntries(from, to, @@ -290,8 +304,10 @@ private static CompletableFuture> readEntries(LedgerHandle lh, long * Getter for CompactedTopicContext. * @return CompactedTopicContext */ - public Optional getCompactedTopicContext() throws ExecutionException, InterruptedException { - return compactedTopicContext == null ? Optional.empty() : Optional.of(compactedTopicContext.get()); + public Optional getCompactedTopicContext() throws ExecutionException, InterruptedException, + TimeoutException { + return compactedTopicContext == null ? Optional.empty() : + Optional.of(compactedTopicContext.get(30, TimeUnit.SECONDS)); } @Override @@ -311,15 +327,74 @@ public CompletableFuture readLastEntryOfCompactedLedger() { }); } - private static int comparePositionAndMessageId(PositionImpl p, MessageIdData m) { + CompletableFuture findFirstMatchEntry(final Predicate predicate) { + var compactedTopicContextFuture = this.getCompactedTopicContextFuture(); + + if (compactedTopicContextFuture == null) { + return CompletableFuture.completedFuture(null); + } + return compactedTopicContextFuture.thenCompose(compactedTopicContext -> { + LedgerHandle lh = compactedTopicContext.getLedger(); + CompletableFuture promise = new CompletableFuture<>(); + findFirstMatchIndexLoop(predicate, 0L, lh.getLastAddConfirmed(), promise, null, lh); + return promise.thenCompose(index -> { + if (index == null) { + return CompletableFuture.completedFuture(null); + } + return readEntries(lh, index, index).thenApply(entries -> entries.get(0)); + }); + }); + } + private static void findFirstMatchIndexLoop(final Predicate predicate, + final long start, final long end, + final CompletableFuture promise, + final Long lastMatchIndex, + final LedgerHandle lh) { + if (start > end) { + promise.complete(lastMatchIndex); + return; + } + + long mid = (start + end) / 2; + readEntries(lh, mid, mid).thenAccept(entries -> { + Entry entry = entries.get(0); + final boolean isMatch; + try { + isMatch = predicate.test(entry); + } finally { + entry.release(); + } + + if (isMatch) { + findFirstMatchIndexLoop(predicate, start, mid - 1, promise, mid, lh); + } else { + findFirstMatchIndexLoop(predicate, mid + 1, end, promise, lastMatchIndex, lh); + } + }).exceptionally(ex -> { + promise.completeExceptionally(ex); + return null; + }); + } + + private static int comparePositionAndMessageId(Position p, MessageIdData m) { return ComparisonChain.start() .compare(p.getLedgerId(), m.getLedgerId()) .compare(p.getEntryId(), m.getEntryId()).result(); } - public synchronized Optional getCompactionHorizon() { + public Optional getCompactionHorizon() { return Optional.ofNullable(this.compactionHorizon); } + + public void reset() { + this.compactionHorizon = null; + this.compactedTopicContext = null; + } + + @Nullable + public CompletableFuture getCompactedTopicContextFuture() { + return compactedTopicContext; + } private static final Logger log = LoggerFactory.getLogger(CompactedTopicImpl.class); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicUtils.java new file mode 100644 index 0000000000000..a7a5fd4ef1113 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicUtils.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.compaction; + +import static com.google.common.base.Preconditions.checkArgument; +import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; +import com.google.common.annotations.Beta; +import java.util.Collections; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nullable; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.pulsar.broker.service.Consumer; +import org.apache.pulsar.broker.service.persistent.PersistentDispatcherSingleActiveConsumer; +import org.apache.pulsar.common.util.FutureUtil; + +public class CompactedTopicUtils { + + @Beta + public static void asyncReadCompactedEntries(TopicCompactionService topicCompactionService, + ManagedCursor cursor, int maxEntries, + long bytesToRead, Position maxReadPosition, + boolean readFromEarliest, AsyncCallbacks.ReadEntriesCallback callback, + boolean wait, @Nullable Consumer consumer) { + Objects.requireNonNull(topicCompactionService); + Objects.requireNonNull(cursor); + checkArgument(maxEntries > 0); + Objects.requireNonNull(callback); + + final Position readPosition; + if (readFromEarliest) { + readPosition = PositionFactory.EARLIEST; + } else { + readPosition = cursor.getReadPosition(); + } + + // TODO: redeliver epoch link https://github.com/apache/pulsar/issues/13690 + PersistentDispatcherSingleActiveConsumer.ReadEntriesCtx readEntriesCtx = + PersistentDispatcherSingleActiveConsumer.ReadEntriesCtx.create(consumer, DEFAULT_CONSUMER_EPOCH); + + CompletableFuture lastCompactedPositionFuture = topicCompactionService.getLastCompactedPosition(); + + lastCompactedPositionFuture.thenCompose(lastCompactedPosition -> { + if (lastCompactedPosition == null + || readPosition.compareTo( + lastCompactedPosition.getLedgerId(), lastCompactedPosition.getEntryId()) > 0) { + if (wait) { + cursor.asyncReadEntriesOrWait(maxEntries, bytesToRead, callback, readEntriesCtx, maxReadPosition); + } else { + cursor.asyncReadEntries(maxEntries, bytesToRead, callback, readEntriesCtx, maxReadPosition); + } + return CompletableFuture.completedFuture(null); + } + + int numberOfEntriesToRead = cursor.applyMaxSizeCap(maxEntries, bytesToRead); + + return topicCompactionService.readCompactedEntries(readPosition, numberOfEntriesToRead) + .thenAccept(entries -> { + if (CollectionUtils.isEmpty(entries)) { + Position seekToPosition = lastCompactedPosition.getNext(); + if (readPosition.compareTo(seekToPosition.getLedgerId(), seekToPosition.getEntryId()) > 0) { + seekToPosition = readPosition; + } + cursor.seek(seekToPosition); + callback.readEntriesComplete(Collections.emptyList(), readEntriesCtx); + return; + } + + long entriesSize = 0; + for (Entry entry : entries) { + entriesSize += entry.getLength(); + } + cursor.updateReadStats(entries.size(), entriesSize); + + Entry lastEntry = entries.get(entries.size() - 1); + cursor.seek(lastEntry.getPosition().getNext(), true); + callback.readEntriesComplete(entries, readEntriesCtx); + }); + }).exceptionally((exception) -> { + exception = FutureUtil.unwrapCompletionException(exception); + callback.readEntriesFailed(ManagedLedgerException.getManagedLedgerException(exception), readEntriesCtx); + return null; + }); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactionRecord.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactionRecord.java index 09f9f9b00abab..cea005d51b82c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactionRecord.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactionRecord.java @@ -45,18 +45,12 @@ public class CompactionRecord { private final LongAdder compactionSucceedCount = new LongAdder(); private final LongAdder compactionFailedCount = new LongAdder(); private final LongAdder compactionDurationTimeInMills = new LongAdder(); + private final LongAdder compactionReadBytes = new LongAdder(); + private final LongAdder compactionWriteBytes = new LongAdder(); public final StatsBuckets writeLatencyStats = new StatsBuckets(WRITE_LATENCY_BUCKETS_USEC); public final Rate writeRate = new Rate(); public final Rate readRate = new Rate(); - public void reset() { - compactionRemovedEventCount.reset(); - compactionSucceedCount.reset(); - compactionFailedCount.reset(); - compactionDurationTimeInMills.reset(); - writeLatencyStats.reset(); - } - public void addCompactionRemovedEvent() { lastCompactionRemovedEventCountOp.increment(); compactionRemovedEventCount.increment(); @@ -83,10 +77,12 @@ public void addCompactionEndOp(boolean succeed) { public void addCompactionReadOp(long readableBytes) { readRate.recordEvent(readableBytes); + compactionReadBytes.add(readableBytes); } public void addCompactionWriteOp(long writeableBytes) { writeRate.recordEvent(writeableBytes); + compactionWriteBytes.add(writeableBytes); } public void addCompactionLatencyOp(long latency, TimeUnit unit) { @@ -123,8 +119,16 @@ public double getCompactionReadThroughput() { return readRate.getValueRate(); } + public long getCompactionReadBytes() { + return compactionReadBytes.sum(); + } + public double getCompactionWriteThroughput() { writeRate.calculateRate(); return writeRate.getValueRate(); } + + public long getCompactionWriteBytes() { + return compactionWriteBytes.sum(); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactionServiceFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactionServiceFactory.java new file mode 100644 index 0000000000000..7bb30372e45bf --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactionServiceFactory.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.compaction; + +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.common.classification.InterfaceAudience; +import org.apache.pulsar.common.classification.InterfaceStability; + +@InterfaceAudience.Public +@InterfaceStability.Evolving +public interface CompactionServiceFactory extends AutoCloseable { + + /** + * Initialize the compaction service factory. + * + * @param pulsarService + * the pulsar service instance + * @return a future represents the initialization result + */ + CompletableFuture initialize(@Nonnull PulsarService pulsarService); + + /** + * Create a new topic compaction service for topic. + * + * @param topic + * the topic name + * @return a future represents the topic compaction service + */ + CompletableFuture newTopicCompactionService(@Nonnull String topic); +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/Compactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/Compactor.java index e93a642c76e4d..983443432ff49 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/Compactor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/Compactor.java @@ -56,7 +56,7 @@ public Compactor(ServiceConfiguration conf, } public CompletableFuture compact(String topic) { - return RawReader.create(pulsar, topic, COMPACTION_SUBSCRIPTION).thenComposeAsync( + return RawReader.create(pulsar, topic, COMPACTION_SUBSCRIPTION, false).thenComposeAsync( this::compactAndCloseReader, scheduler); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorMXBeanImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorMXBeanImpl.java index 64b91d17d2508..8a9d266b56e26 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorMXBeanImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorMXBeanImpl.java @@ -53,10 +53,6 @@ public Set getTopics() { return compactionRecordOps.keySet(); } - public void reset() { - compactionRecordOps.values().forEach(CompactionRecord::reset); - } - public void addCompactionReadOp(String topic, long readableBytes) { compactionRecordOps.computeIfAbsent(topic, k -> new CompactionRecord()).addCompactionReadOp(readableBytes); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java index 2539c306500a2..ba68e07cf5b0d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java @@ -20,8 +20,6 @@ import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.netty.channel.EventLoopGroup; import io.netty.util.concurrent.DefaultThreadFactory; @@ -41,27 +39,32 @@ import org.apache.pulsar.client.api.SizeUnit; import org.apache.pulsar.client.internal.PropertiesUtils; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; -import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.netty.EventLoopUtil; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.ScopeType; +@Command(name = "compact-topic", showDefaultValues = true, scope = ScopeType.INHERIT) public class CompactorTool { private static class Arguments { - @Parameter(names = {"-c", "--broker-conf"}, description = "Configuration file for Broker") + @Option(names = {"-c", "--broker-conf"}, description = "Configuration file for Broker") private String brokerConfigFile = "conf/broker.conf"; - @Parameter(names = {"-t", "--topic"}, description = "Topic to compact", required = true) + @Option(names = {"-t", "--topic"}, description = "Topic to compact", required = true) private String topic; - @Parameter(names = {"-h", "--help"}, description = "Show this help message") + @Option(names = {"-h", "--help"}, usageHelp = true, description = "Show this help message") private boolean help = false; - @Parameter(names = {"-g", "--generate-docs"}, description = "Generate docs") + @Option(names = {"-g", "--generate-docs"}, description = "Generate docs") private boolean generateDocs = false; } @@ -83,7 +86,9 @@ public static PulsarClient createClient(ServiceConfiguration brokerConfig) throw if (internalListener.getBrokerServiceUrlTls() != null && brokerConfig.isBrokerClientTlsEnabled()) { clientBuilder.serviceUrl(internalListener.getBrokerServiceUrlTls().toString()) .allowTlsInsecureConnection(brokerConfig.isTlsAllowInsecureConnection()) - .enableTlsHostnameVerification(brokerConfig.isTlsHostnameVerificationEnabled()); + .enableTlsHostnameVerification(brokerConfig.isTlsHostnameVerificationEnabled()) + .sslFactoryPlugin(brokerConfig.getBrokerClientSslFactoryPlugin()) + .sslFactoryPluginParams(brokerConfig.getBrokerClientSslFactoryPluginParams()); if (brokerConfig.isBrokerClientTlsEnabledWithKeyStore()) { clientBuilder.useKeyStoreTls(true) .tlsKeyStoreType(brokerConfig.getBrokerClientTlsKeyStoreType()) @@ -107,13 +112,11 @@ public static PulsarClient createClient(ServiceConfiguration brokerConfig) throw public static void main(String[] args) throws Exception { Arguments arguments = new Arguments(); - JCommander jcommander = new JCommander(arguments); - jcommander.setProgramName("PulsarTopicCompactor"); - - // parse args by JCommander - jcommander.parse(args); + CommandLine commander = new CommandLine(arguments); + commander.setCommandName("PulsarTopicCompactor"); + commander.parseArgs(args); if (arguments.help) { - jcommander.usage(); + commander.usage(commander.getOut()); System.exit(0); } @@ -126,7 +129,7 @@ public static void main(String[] args) throws Exception { // init broker config if (isBlank(arguments.brokerConfigFile)) { - jcommander.usage(); + commander.usage(commander.getOut()); throw new IllegalArgumentException("Need to specify a configuration file for broker"); } @@ -164,12 +167,12 @@ public static void main(String[] args) throws Exception { new DefaultThreadFactory("compactor-io")); @Cleanup - BookKeeper bk = bkClientFactory.create(brokerConfig, store, eventLoopGroup, Optional.empty(), null); + BookKeeper bk = bkClientFactory.create(brokerConfig, store, eventLoopGroup, Optional.empty(), null).get(); @Cleanup PulsarClient pulsar = createClient(brokerConfig); - Compactor compactor = new TwoPhaseCompactor(brokerConfig, pulsar, bk, scheduler); + Compactor compactor = new PublishingOrderCompactor(brokerConfig, pulsar, bk, scheduler); long ledgerId = compactor.compact(arguments.topic).get(); log.info("Compaction of topic {} complete. Compacted to ledger {}", arguments.topic, ledgerId); } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/util/CacheSizeAllocator.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/EventTimeCompactionServiceFactory.java similarity index 59% rename from pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/util/CacheSizeAllocator.java rename to pulsar-broker/src/main/java/org/apache/pulsar/compaction/EventTimeCompactionServiceFactory.java index 387076cc6d7f7..383c7b1aeedd6 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/util/CacheSizeAllocator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/EventTimeCompactionServiceFactory.java @@ -16,32 +16,18 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.sql.presto.util; +package org.apache.pulsar.compaction; -/** - * Cache size allocator. - */ -public interface CacheSizeAllocator { - - /** - * Get available cache size. - * - * @return available cache size - */ - long getAvailableCacheSize(); - - /** - * Cost available cache. - * - * @param size allocate size - */ - void allocate(long size); +import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.PulsarService; - /** - * Release allocated cache size. - * - * @param size release size - */ - void release(long size); +public class EventTimeCompactionServiceFactory extends PulsarCompactionServiceFactory { + @Override + protected Compactor newCompactor() throws PulsarServerException { + PulsarService pulsarService = getPulsarService(); + return new EventTimeOrderCompactor(pulsarService.getConfiguration(), + pulsarService.getClient(), pulsarService.getBookKeeperClient(), + pulsarService.getCompactorExecutor()); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/EventTimeOrderCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/EventTimeOrderCompactor.java new file mode 100644 index 0000000000000..db129b54533a8 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/EventTimeOrderCompactor.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.compaction; + +import io.netty.buffer.ByteBuf; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ScheduledExecutorService; +import java.util.stream.Collectors; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.RawMessage; +import org.apache.pulsar.client.impl.RawBatchConverter; +import org.apache.pulsar.common.api.proto.MessageMetadata; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class EventTimeOrderCompactor extends AbstractTwoPhaseCompactor> { + + private static final Logger log = LoggerFactory.getLogger(EventTimeOrderCompactor.class); + + public EventTimeOrderCompactor(ServiceConfiguration conf, + PulsarClient pulsar, + BookKeeper bk, + ScheduledExecutorService scheduler) { + super(conf, pulsar, bk, scheduler); + } + + @Override + protected Map toLatestMessageIdForKey( + Map> latestForKey) { + return latestForKey.entrySet() + .stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> entry.getValue().getLeft())); + } + + @Override + protected boolean compactMessage(String topic, Map> latestForKey, + RawMessage m, MessageMetadata metadata, MessageId id) { + boolean deletedMessage = false; + boolean replaceMessage = false; + MessageCompactionData mcd = extractMessageCompactionData(m, metadata); + + if (mcd != null) { + boolean newer = Optional.ofNullable(latestForKey.get(mcd.key())) + .map(Pair::getRight) + .map(latestEventTime -> mcd.eventTime() != null + && mcd.eventTime() >= latestEventTime).orElse(true); + if (newer) { + if (mcd.payloadSize() > 0) { + Pair old = latestForKey.put(mcd.key(), + new ImmutablePair<>(mcd.messageId(), mcd.eventTime())); + replaceMessage = old != null; + } else { + deletedMessage = true; + latestForKey.remove(mcd.key()); + } + } + } else { + if (!topicCompactionRetainNullKey) { + deletedMessage = true; + } + } + if (replaceMessage || deletedMessage) { + mxBean.addCompactionRemovedEvent(topic); + } + return deletedMessage; + } + + @Override + protected boolean compactBatchMessage(String topic, Map> latestForKey, RawMessage m, + MessageMetadata metadata, MessageId id) { + boolean deletedMessage = false; + try { + int numMessagesInBatch = metadata.getNumMessagesInBatch(); + int deleteCnt = 0; + + for (MessageCompactionData mcd : extractMessageCompactionDataFromBatch(m, metadata)) { + if (mcd.key() == null) { + if (!topicCompactionRetainNullKey) { + // record delete null-key message event + deleteCnt++; + mxBean.addCompactionRemovedEvent(topic); + } + continue; + } + + boolean newer = Optional.ofNullable(latestForKey.get(mcd.key())) + .map(Pair::getRight) + .map(latestEventTime -> mcd.eventTime() != null + && mcd.eventTime() > latestEventTime).orElse(true); + if (newer) { + if (mcd.payloadSize() > 0) { + Pair old = latestForKey.put(mcd.key(), + new ImmutablePair<>(mcd.messageId(), mcd.eventTime())); + if (old != null) { + mxBean.addCompactionRemovedEvent(topic); + } + } else { + latestForKey.remove(mcd.key()); + deleteCnt++; + mxBean.addCompactionRemovedEvent(topic); + } + } + } + + if (deleteCnt == numMessagesInBatch) { + deletedMessage = true; + } + } catch (IOException ioe) { + log.info("Error decoding batch for message {}. Whole batch will be included in output", + id, ioe); + } + return deletedMessage; + } + + protected MessageCompactionData extractMessageCompactionData(RawMessage m, MessageMetadata metadata) { + ByteBuf headersAndPayload = m.getHeadersAndPayload(); + if (metadata.hasPartitionKey()) { + int size = headersAndPayload.readableBytes(); + if (metadata.hasUncompressedSize()) { + size = metadata.getUncompressedSize(); + } + return new MessageCompactionData(m.getMessageId(), metadata.getPartitionKey(), + size, metadata.getEventTime()); + } else { + return null; + } + } + + private List extractMessageCompactionDataFromBatch(RawMessage msg, MessageMetadata metadata) + throws IOException { + return RawBatchConverter.extractMessageCompactionData(msg, metadata); + } +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/MessageCompactionData.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/MessageCompactionData.java new file mode 100644 index 0000000000000..03800273a806e --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/MessageCompactionData.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.compaction; + +import org.apache.pulsar.client.api.MessageId; + +public record MessageCompactionData (MessageId messageId, String key, Integer payloadSize, Long eventTime) {} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/PublishingOrderCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/PublishingOrderCompactor.java new file mode 100644 index 0000000000000..223e8c421a5ef --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/PublishingOrderCompactor.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.compaction; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ScheduledExecutorService; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.commons.lang3.tuple.ImmutableTriple; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.RawMessage; +import org.apache.pulsar.client.impl.RawBatchConverter; +import org.apache.pulsar.common.api.proto.MessageMetadata; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class PublishingOrderCompactor extends AbstractTwoPhaseCompactor { + + private static final Logger log = LoggerFactory.getLogger(PublishingOrderCompactor.class); + + public PublishingOrderCompactor(ServiceConfiguration conf, + PulsarClient pulsar, + BookKeeper bk, + ScheduledExecutorService scheduler) { + super(conf, pulsar, bk, scheduler); + } + + @Override + protected Map toLatestMessageIdForKey(Map latestForKey) { + return latestForKey; + } + + @Override + protected boolean compactMessage(String topic, Map latestForKey, + RawMessage m, MessageMetadata metadata, MessageId id) { + boolean deletedMessage = false; + boolean replaceMessage = false; + Pair keyAndSize = extractKeyAndSize(m, metadata); + if (keyAndSize != null) { + if (keyAndSize.getRight() > 0) { + MessageId old = latestForKey.put(keyAndSize.getLeft(), id); + replaceMessage = old != null; + } else { + deletedMessage = true; + latestForKey.remove(keyAndSize.getLeft()); + } + } else { + if (!topicCompactionRetainNullKey) { + deletedMessage = true; + } + } + if (replaceMessage || deletedMessage) { + mxBean.addCompactionRemovedEvent(topic); + } + return deletedMessage; + } + + @Override + protected boolean compactBatchMessage(String topic, Map latestForKey, + RawMessage m, MessageMetadata metadata, MessageId id) { + boolean deletedMessage = false; + try { + int numMessagesInBatch = metadata.getNumMessagesInBatch(); + int deleteCnt = 0; + for (ImmutableTriple e : extractIdsAndKeysAndSizeFromBatch( + m, metadata)) { + if (e != null) { + if (e.getMiddle() == null) { + if (!topicCompactionRetainNullKey) { + // record delete null-key message event + deleteCnt++; + mxBean.addCompactionRemovedEvent(topic); + } + continue; + } + if (e.getRight() > 0) { + MessageId old = latestForKey.put(e.getMiddle(), e.getLeft()); + if (old != null) { + mxBean.addCompactionRemovedEvent(topic); + } + } else { + latestForKey.remove(e.getMiddle()); + deleteCnt++; + mxBean.addCompactionRemovedEvent(topic); + } + } + } + if (deleteCnt == numMessagesInBatch) { + deletedMessage = true; + } + } catch (IOException ioe) { + log.info( + "Error decoding batch for message {}. Whole batch will be included in output", + id, ioe); + } + + return deletedMessage; + } + + protected List> extractIdsAndKeysAndSizeFromBatch( + RawMessage msg, MessageMetadata metadata) + throws IOException { + return RawBatchConverter.extractIdsAndKeysAndSize(msg, metadata); + } + +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/PulsarCompactionServiceFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/PulsarCompactionServiceFactory.java new file mode 100644 index 0000000000000..90132461b4c4a --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/PulsarCompactionServiceFactory.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.compaction; + +import com.google.common.annotations.VisibleForTesting; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.AccessLevel; +import lombok.Getter; +import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.PulsarService; + +public class PulsarCompactionServiceFactory implements CompactionServiceFactory { + + @Getter(AccessLevel.PROTECTED) + private PulsarService pulsarService; + + private volatile Compactor compactor; + + @VisibleForTesting + public Compactor getCompactor() throws PulsarServerException { + if (compactor == null) { + synchronized (this) { + if (compactor == null) { + compactor = newCompactor(); + } + } + } + return compactor; + } + + @Nullable + public Compactor getNullableCompactor() { + return compactor; + } + + protected Compactor newCompactor() throws PulsarServerException { + return new PublishingOrderCompactor(pulsarService.getConfiguration(), + pulsarService.getClient(), pulsarService.getBookKeeperClient(), + pulsarService.getCompactorExecutor()); + } + + @Override + public CompletableFuture initialize(@Nonnull PulsarService pulsarService) { + Objects.requireNonNull(pulsarService); + this.pulsarService = pulsarService; + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture newTopicCompactionService(@Nonnull String topic) { + Objects.requireNonNull(topic); + PulsarTopicCompactionService pulsarTopicCompactionService = + new PulsarTopicCompactionService(topic, pulsarService.getBookKeeperClient(), () -> { + try { + return this.getCompactor(); + } catch (Throwable e) { + throw new CompletionException(e); + } + }); + return CompletableFuture.completedFuture(pulsarTopicCompactionService); + } + + @Override + public void close() throws Exception { + // noop + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/PulsarTopicCompactionService.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/PulsarTopicCompactionService.java new file mode 100644 index 0000000000000..27efcf9524f8f --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/PulsarTopicCompactionService.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.compaction; + +import static com.google.common.base.Preconditions.checkArgument; +import static org.apache.pulsar.compaction.CompactedTopicImpl.COMPACT_LEDGER_EMPTY; +import static org.apache.pulsar.compaction.CompactedTopicImpl.NEWER_THAN_COMPACTED; +import static org.apache.pulsar.compaction.CompactedTopicImpl.findStartPoint; +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.function.Predicate; +import java.util.function.Supplier; +import javax.annotation.Nonnull; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.Position; +import org.apache.pulsar.common.api.proto.BrokerEntryMetadata; +import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.util.FutureUtil; + + +public class PulsarTopicCompactionService implements TopicCompactionService { + + private final String topic; + + private final CompactedTopicImpl compactedTopic; + + private final Supplier compactorSupplier; + + public PulsarTopicCompactionService(String topic, BookKeeper bookKeeper, + Supplier compactorSupplier) { + this.topic = topic; + this.compactedTopic = new CompactedTopicImpl(bookKeeper); + this.compactorSupplier = compactorSupplier; + } + + @Override + public CompletableFuture compact() { + Compactor compactor; + try { + compactor = compactorSupplier.get(); + } catch (Throwable e) { + return CompletableFuture.failedFuture(e); + } + return compactor.compact(topic).thenApply(x -> null); + } + + @Override + public CompletableFuture> readCompactedEntries(@Nonnull Position startPosition, + int numberOfEntriesToRead) { + Objects.requireNonNull(startPosition); + checkArgument(numberOfEntriesToRead > 0); + + CompletableFuture> resultFuture = new CompletableFuture<>(); + + Objects.requireNonNull(compactedTopic.getCompactedTopicContextFuture()).thenCompose( + (context) -> findStartPoint(startPosition, context.ledger.getLastAddConfirmed(), + context.cache).thenCompose((startPoint) -> { + if (startPoint == COMPACT_LEDGER_EMPTY || startPoint == NEWER_THAN_COMPACTED) { + return CompletableFuture.completedFuture(Collections.emptyList()); + } + long endPoint = + Math.min(context.ledger.getLastAddConfirmed(), startPoint + (numberOfEntriesToRead - 1)); + return CompactedTopicImpl.readEntries(context.ledger, startPoint, endPoint); + })).whenComplete((result, ex) -> { + if (ex == null) { + resultFuture.complete(result); + } else { + ex = FutureUtil.unwrapCompletionException(ex); + if (ex instanceof NoSuchElementException) { + resultFuture.complete(Collections.emptyList()); + } else { + resultFuture.completeExceptionally(ex); + } + } + }); + + return resultFuture; + } + + @Override + public CompletableFuture readLastCompactedEntry() { + return compactedTopic.readLastEntryOfCompactedLedger(); + } + + @Override + public CompletableFuture getLastCompactedPosition() { + return CompletableFuture.completedFuture(compactedTopic.getCompactionHorizon().orElse(null)); + } + + @Override + public CompletableFuture findEntryByPublishTime(long publishTime) { + final Predicate predicate = entry -> { + return Commands.parseMessageMetadata(entry.getDataBuffer()).getPublishTime() >= publishTime; + }; + return compactedTopic.findFirstMatchEntry(predicate); + } + + @Override + public CompletableFuture findEntryByEntryIndex(long entryIndex) { + final Predicate predicate = entry -> { + BrokerEntryMetadata brokerEntryMetadata = Commands.parseBrokerEntryMetadataIfExist(entry.getDataBuffer()); + if (brokerEntryMetadata == null || !brokerEntryMetadata.hasIndex()) { + return false; + } + return brokerEntryMetadata.getIndex() >= entryIndex; + }; + return compactedTopic.findFirstMatchEntry(predicate); + } + + public CompactedTopicImpl getCompactedTopic() { + return compactedTopic; + } + + @Override + public void close() throws IOException { + // noop + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java index a6b0942742763..1b54092d9aa4f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.compaction; -import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.ByteBuf; import java.time.Duration; import java.util.Iterator; @@ -60,42 +59,22 @@ *

As the first pass caches the entire message(not just offset) for each key into a map, * this compaction could be memory intensive if the message payload is large. */ -public class StrategicTwoPhaseCompactor extends TwoPhaseCompactor { +public class StrategicTwoPhaseCompactor extends PublishingOrderCompactor { private static final Logger log = LoggerFactory.getLogger(StrategicTwoPhaseCompactor.class); private static final int MAX_OUTSTANDING = 500; - private static final int MAX_NUM_MESSAGES_IN_BATCH = 1000; - private static final int MAX_BYTES_IN_BATCH = 128 * 1024; private static final int MAX_READER_RECONNECT_WAITING_TIME_IN_MILLIS = 20 * 1000; private final Duration phaseOneLoopReadTimeout; private final RawBatchMessageContainerImpl batchMessageContainer; - @VisibleForTesting public StrategicTwoPhaseCompactor(ServiceConfiguration conf, PulsarClient pulsar, BookKeeper bk, - ScheduledExecutorService scheduler, - int maxNumMessagesInBatch) { - this(conf, pulsar, bk, scheduler, maxNumMessagesInBatch, MAX_BYTES_IN_BATCH); - } - - private StrategicTwoPhaseCompactor(ServiceConfiguration conf, - PulsarClient pulsar, - BookKeeper bk, - ScheduledExecutorService scheduler, - int maxNumMessagesInBatch, - int maxBytesInBatch) { + ScheduledExecutorService scheduler) { super(conf, pulsar, bk, scheduler); - batchMessageContainer = new RawBatchMessageContainerImpl(maxNumMessagesInBatch, maxBytesInBatch); + batchMessageContainer = new RawBatchMessageContainerImpl(); phaseOneLoopReadTimeout = Duration.ofSeconds(conf.getBrokerServiceCompactionPhaseOneLoopTimeInSeconds()); } - public StrategicTwoPhaseCompactor(ServiceConfiguration conf, - PulsarClient pulsar, - BookKeeper bk, - ScheduledExecutorService scheduler) { - this(conf, pulsar, bk, scheduler, MAX_NUM_MESSAGES_IN_BATCH, MAX_BYTES_IN_BATCH); - } - public CompletableFuture compact(String topic) { throw new UnsupportedOperationException(); } @@ -418,7 +397,6 @@ private void phaseTwoLoop(String topic, Iterator> reader, .whenComplete((res, exception2) -> { if (exception2 != null) { promise.completeExceptionally(exception2); - return; } }); phaseTwoLoop(topic, reader, lh, outstanding, promise); @@ -443,35 +421,45 @@ private void phaseTwoLoop(String topic, Iterator> reader, CompletableFuture addToCompactedLedger( LedgerHandle lh, Message m, String topic, Semaphore outstanding) { + if (m == null) { + return flushBatchMessage(lh, topic, outstanding); + } + if (batchMessageContainer.haveEnoughSpace((MessageImpl) m)) { + batchMessageContainer.add((MessageImpl) m, null); + return CompletableFuture.completedFuture(false); + } + CompletableFuture f = flushBatchMessage(lh, topic, outstanding); + batchMessageContainer.add((MessageImpl) m, null); + return f; + } + + private CompletableFuture flushBatchMessage(LedgerHandle lh, String topic, + Semaphore outstanding) { + if (batchMessageContainer.getNumMessagesInBatch() <= 0) { + return CompletableFuture.completedFuture(false); + } CompletableFuture bkf = new CompletableFuture<>(); - if (m == null || batchMessageContainer.add((MessageImpl) m, null)) { - if (batchMessageContainer.getNumMessagesInBatch() > 0) { - try { - ByteBuf serialized = batchMessageContainer.toByteBuf(); - outstanding.acquire(); - mxBean.addCompactionWriteOp(topic, serialized.readableBytes()); - long start = System.nanoTime(); - lh.asyncAddEntry(serialized, - (rc, ledger, eid, ctx) -> { - outstanding.release(); - mxBean.addCompactionLatencyOp(topic, System.nanoTime() - start, TimeUnit.NANOSECONDS); - if (rc != BKException.Code.OK) { - bkf.completeExceptionally(BKException.create(rc)); - } else { - bkf.complete(true); - } - }, null); + try { + ByteBuf serialized = batchMessageContainer.toByteBuf(); + outstanding.acquire(); + mxBean.addCompactionWriteOp(topic, serialized.readableBytes()); + long start = System.nanoTime(); + lh.asyncAddEntry(serialized, + (rc, ledger, eid, ctx) -> { + outstanding.release(); + mxBean.addCompactionLatencyOp(topic, System.nanoTime() - start, TimeUnit.NANOSECONDS); + if (rc != BKException.Code.OK) { + bkf.completeExceptionally(BKException.create(rc)); + } else { + bkf.complete(true); + } + }, null); - } catch (Throwable t) { - log.error("Failed to add entry", t); - batchMessageContainer.discard((Exception) t); - return FutureUtil.failedFuture(t); - } - } else { - bkf.complete(false); - } - } else { - bkf.complete(false); + } catch (Throwable t) { + log.error("Failed to add entry", t); + batchMessageContainer.discard((Exception) t); + bkf.completeExceptionally(t); + return bkf; } return bkf; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TopicCompactionService.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TopicCompactionService.java new file mode 100644 index 0000000000000..fdd6bebbdec33 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TopicCompactionService.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.compaction; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.Position; +import org.apache.pulsar.common.classification.InterfaceAudience; +import org.apache.pulsar.common.classification.InterfaceStability; + +@InterfaceAudience.Public +@InterfaceStability.Evolving +public interface TopicCompactionService extends AutoCloseable { + /** + * Compact the topic. + * Topic Compaction is a key-based retention mechanism. It keeps the most recent value for a given key and + * user reads compacted data from TopicCompactionService. + * + * @return a future that will be completed when the compaction is done. + */ + CompletableFuture compact(); + + /** + * Read the compacted entries from the TopicCompactionService. + * + * @param startPosition the position to start reading from. + * @param numberOfEntriesToRead the maximum number of entries to read. + * @return a future that will be completed with the list of entries, this list can be null. + */ + CompletableFuture> readCompactedEntries(@Nonnull Position startPosition, int numberOfEntriesToRead); + + /** + * Read the last compacted entry from the TopicCompactionService. + * + * @return a future that will be completed with the compacted last entry, this entry can be null. + */ + CompletableFuture readLastCompactedEntry(); + + /** + * Get the last compacted position from the TopicCompactionService. + * + * @return a future that will be completed with the last compacted position, this position can be null. + */ + CompletableFuture getLastCompactedPosition(); + + /** + * Find the first entry that greater or equal to target publishTime. + * + * @param publishTime the publish time of entry. + * @return the first entry metadata that greater or equal to target publishTime, this entry can be null. + */ + CompletableFuture findEntryByPublishTime(long publishTime); + + /** + * Find the first entry that greater or equal to target entryIndex, + * if an entry that broker entry metadata is missed, then it will be skipped and find the next match entry. + * + * @param entryIndex the index of entry. + * @return the first entry that greater or equal to target entryIndex, this entry can be null. + */ + CompletableFuture findEntryByEntryIndex(long entryIndex); +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java deleted file mode 100644 index 821dd9c0c9d23..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java +++ /dev/null @@ -1,420 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.compaction; - -import io.netty.buffer.ByteBuf; -import java.io.IOException; -import java.time.Duration; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.Semaphore; -import java.util.concurrent.TimeUnit; -import org.apache.bookkeeper.client.BKException; -import org.apache.bookkeeper.client.BookKeeper; -import org.apache.bookkeeper.client.LedgerHandle; -import org.apache.bookkeeper.mledger.impl.LedgerMetadataUtils; -import org.apache.commons.lang3.tuple.ImmutableTriple; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.client.api.MessageId; -import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.RawMessage; -import org.apache.pulsar.client.api.RawReader; -import org.apache.pulsar.client.impl.MessageIdImpl; -import org.apache.pulsar.client.impl.RawBatchConverter; -import org.apache.pulsar.common.api.proto.MessageMetadata; -import org.apache.pulsar.common.protocol.Commands; -import org.apache.pulsar.common.util.FutureUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Compaction will go through the topic in two passes. The first pass - * selects latest offset for each key in the topic. Then the second pass - * writes these values to a ledger. - * - *

The two passes are required to avoid holding the payloads of each of - * the latest values in memory, as the payload can be many orders of - * magnitude larger than a message id. -*/ -public class TwoPhaseCompactor extends Compactor { - private static final Logger log = LoggerFactory.getLogger(TwoPhaseCompactor.class); - private static final int MAX_OUTSTANDING = 500; - protected static final String COMPACTED_TOPIC_LEDGER_PROPERTY = "CompactedTopicLedger"; - private final Duration phaseOneLoopReadTimeout; - - public TwoPhaseCompactor(ServiceConfiguration conf, - PulsarClient pulsar, - BookKeeper bk, - ScheduledExecutorService scheduler) { - super(conf, pulsar, bk, scheduler); - phaseOneLoopReadTimeout = Duration.ofSeconds(conf.getBrokerServiceCompactionPhaseOneLoopTimeInSeconds()); - } - - @Override - protected CompletableFuture doCompaction(RawReader reader, BookKeeper bk) { - return reader.hasMessageAvailableAsync() - .thenCompose(available -> { - if (available) { - return phaseOne(reader).thenCompose( - (r) -> phaseTwo(reader, r.from, r.to, r.lastReadId, r.latestForKey, bk)); - } else { - log.info("Skip compaction of the empty topic {}", reader.getTopic()); - return CompletableFuture.completedFuture(-1L); - } - }); - } - - private CompletableFuture phaseOne(RawReader reader) { - Map latestForKey = new HashMap<>(); - CompletableFuture loopPromise = new CompletableFuture<>(); - - reader.getLastMessageIdAsync() - .thenAccept(lastMessageId -> { - log.info("Commencing phase one of compaction for {}, reading to {}", - reader.getTopic(), lastMessageId); - // Each entry is processed as a whole, discard the batchIndex part deliberately. - MessageIdImpl lastImpl = (MessageIdImpl) lastMessageId; - MessageIdImpl lastEntryMessageId = new MessageIdImpl(lastImpl.getLedgerId(), lastImpl.getEntryId(), - lastImpl.getPartitionIndex()); - phaseOneLoop(reader, Optional.empty(), Optional.empty(), lastEntryMessageId, latestForKey, - loopPromise); - }).exceptionally(ex -> { - loopPromise.completeExceptionally(ex); - return null; - }); - - return loopPromise; - } - - private void phaseOneLoop(RawReader reader, - Optional firstMessageId, - Optional toMessageId, - MessageId lastMessageId, - Map latestForKey, - CompletableFuture loopPromise) { - if (loopPromise.isDone()) { - return; - } - CompletableFuture future = reader.readNextAsync(); - FutureUtil.addTimeoutHandling(future, - phaseOneLoopReadTimeout, scheduler, - () -> FutureUtil.createTimeoutException("Timeout", getClass(), "phaseOneLoop(...)")); - - future.thenAcceptAsync(m -> { - try { - MessageId id = m.getMessageId(); - boolean deletedMessage = false; - boolean replaceMessage = false; - mxBean.addCompactionReadOp(reader.getTopic(), m.getHeadersAndPayload().readableBytes()); - if (RawBatchConverter.isReadableBatch(m)) { - try { - for (ImmutableTriple e : RawBatchConverter - .extractIdsAndKeysAndSize(m)) { - if (e != null) { - if (e.getRight() > 0) { - MessageId old = latestForKey.put(e.getMiddle(), e.getLeft()); - replaceMessage = old != null; - } else { - deletedMessage = true; - latestForKey.remove(e.getMiddle()); - } - } - if (replaceMessage || deletedMessage) { - mxBean.addCompactionRemovedEvent(reader.getTopic()); - } - } - } catch (IOException ioe) { - log.info("Error decoding batch for message {}. Whole batch will be included in output", - id, ioe); - } - } else { - Pair keyAndSize = extractKeyAndSize(m); - if (keyAndSize != null) { - if (keyAndSize.getRight() > 0) { - MessageId old = latestForKey.put(keyAndSize.getLeft(), id); - replaceMessage = old != null; - } else { - deletedMessage = true; - latestForKey.remove(keyAndSize.getLeft()); - } - } - if (replaceMessage || deletedMessage) { - mxBean.addCompactionRemovedEvent(reader.getTopic()); - } - } - MessageId first = firstMessageId.orElse(deletedMessage ? null : id); - MessageId to = deletedMessage ? toMessageId.orElse(null) : id; - if (id.compareTo(lastMessageId) == 0) { - loopPromise.complete(new PhaseOneResult(first == null ? id : first, to == null ? id : to, - lastMessageId, latestForKey)); - } else { - phaseOneLoop(reader, - Optional.ofNullable(first), - Optional.ofNullable(to), - lastMessageId, - latestForKey, loopPromise); - } - } finally { - m.close(); - } - }, scheduler).exceptionally(ex -> { - loopPromise.completeExceptionally(ex); - return null; - }); - } - - private CompletableFuture phaseTwo(RawReader reader, MessageId from, MessageId to, MessageId lastReadId, - Map latestForKey, BookKeeper bk) { - Map metadata = - LedgerMetadataUtils.buildMetadataForCompactedLedger(reader.getTopic(), to.toByteArray()); - return createLedger(bk, metadata).thenCompose((ledger) -> { - log.info("Commencing phase two of compaction for {}, from {} to {}, compacting {} keys to ledger {}", - reader.getTopic(), from, to, latestForKey.size(), ledger.getId()); - return phaseTwoSeekThenLoop(reader, from, to, lastReadId, latestForKey, bk, ledger); - }); - } - - private CompletableFuture phaseTwoSeekThenLoop(RawReader reader, MessageId from, MessageId to, - MessageId lastReadId, Map latestForKey, BookKeeper bk, LedgerHandle ledger) { - CompletableFuture promise = new CompletableFuture<>(); - - reader.seekAsync(from).thenCompose((v) -> { - Semaphore outstanding = new Semaphore(MAX_OUTSTANDING); - CompletableFuture loopPromise = new CompletableFuture<>(); - phaseTwoLoop(reader, to, latestForKey, ledger, outstanding, loopPromise); - return loopPromise; - }).thenCompose((v) -> closeLedger(ledger)) - .thenCompose((v) -> reader.acknowledgeCumulativeAsync(lastReadId, - Map.of(COMPACTED_TOPIC_LEDGER_PROPERTY, ledger.getId()))) - .whenComplete((res, exception) -> { - if (exception != null) { - deleteLedger(bk, ledger).whenComplete((res2, exception2) -> { - if (exception2 != null) { - log.warn("Cleanup of ledger {} for failed", ledger, exception2); - } - // complete with original exception - promise.completeExceptionally(exception); - }); - } else { - promise.complete(ledger.getId()); - } - }); - return promise; - } - - private void phaseTwoLoop(RawReader reader, MessageId to, Map latestForKey, - LedgerHandle lh, Semaphore outstanding, CompletableFuture promise) { - if (promise.isDone()) { - return; - } - reader.readNextAsync().thenAcceptAsync(m -> { - if (promise.isDone()) { - m.close(); - return; - } - try { - MessageId id = m.getMessageId(); - Optional messageToAdd = Optional.empty(); - mxBean.addCompactionReadOp(reader.getTopic(), m.getHeadersAndPayload().readableBytes()); - if (RawBatchConverter.isReadableBatch(m)) { - try { - messageToAdd = RawBatchConverter.rebatchMessage( - m, (key, subid) -> subid.equals(latestForKey.get(key))); - } catch (IOException ioe) { - log.info("Error decoding batch for message {}. Whole batch will be included in output", - id, ioe); - messageToAdd = Optional.of(m); - } - } else { - Pair keyAndSize = extractKeyAndSize(m); - MessageId msg; - if (keyAndSize == null) { // pass through messages without a key - messageToAdd = Optional.of(m); - } else if ((msg = latestForKey.get(keyAndSize.getLeft())) != null - && msg.equals(id)) { // consider message only if present into latestForKey map - if (keyAndSize.getRight() <= 0) { - promise.completeExceptionally(new IllegalArgumentException( - "Compaction phase found empty record from sorted key-map")); - } - messageToAdd = Optional.of(m); - } - } - - if (messageToAdd.isPresent()) { - RawMessage message = messageToAdd.get(); - try { - outstanding.acquire(); - CompletableFuture addFuture = addToCompactedLedger(lh, message, reader.getTopic()) - .whenComplete((res, exception2) -> { - outstanding.release(); - if (exception2 != null) { - promise.completeExceptionally(exception2); - } - }); - if (to.equals(id)) { - addFuture.whenComplete((res, exception2) -> { - if (exception2 == null) { - promise.complete(null); - } - }); - } - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - promise.completeExceptionally(ie); - } finally { - if (message != m) { - message.close(); - } - } - } else if (to.equals(id)) { - // Reached to last-id and phase-one found it deleted-message while iterating on ledger so, - // not present under latestForKey. Complete the compaction. - try { - // make sure all inflight writes have finished - outstanding.acquire(MAX_OUTSTANDING); - promise.complete(null); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - promise.completeExceptionally(e); - } - return; - } - phaseTwoLoop(reader, to, latestForKey, lh, outstanding, promise); - } finally { - m.close(); - } - }, scheduler).exceptionally(ex -> { - promise.completeExceptionally(ex); - return null; - }); - } - - protected CompletableFuture createLedger(BookKeeper bk, Map metadata) { - CompletableFuture bkf = new CompletableFuture<>(); - - try { - bk.asyncCreateLedger(conf.getManagedLedgerDefaultEnsembleSize(), - conf.getManagedLedgerDefaultWriteQuorum(), - conf.getManagedLedgerDefaultAckQuorum(), - Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, - Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD, - (rc, ledger, ctx) -> { - if (rc != BKException.Code.OK) { - bkf.completeExceptionally(BKException.create(rc)); - } else { - bkf.complete(ledger); - } - }, null, metadata); - } catch (Throwable t) { - log.error("Encountered unexpected error when creating compaction ledger", t); - return FutureUtil.failedFuture(t); - } - return bkf; - } - - protected CompletableFuture deleteLedger(BookKeeper bk, LedgerHandle lh) { - CompletableFuture bkf = new CompletableFuture<>(); - try { - bk.asyncDeleteLedger(lh.getId(), - (rc, ctx) -> { - if (rc != BKException.Code.OK) { - bkf.completeExceptionally(BKException.create(rc)); - } else { - bkf.complete(null); - } - }, null); - } catch (Throwable t) { - return FutureUtil.failedFuture(t); - } - return bkf; - } - - protected CompletableFuture closeLedger(LedgerHandle lh) { - CompletableFuture bkf = new CompletableFuture<>(); - try { - lh.asyncClose((rc, ledger, ctx) -> { - if (rc != BKException.Code.OK) { - bkf.completeExceptionally(BKException.create(rc)); - } else { - bkf.complete(null); - } - }, null); - } catch (Throwable t) { - return FutureUtil.failedFuture(t); - } - return bkf; - } - - private CompletableFuture addToCompactedLedger(LedgerHandle lh, RawMessage m, String topic) { - CompletableFuture bkf = new CompletableFuture<>(); - ByteBuf serialized = m.serialize(); - try { - mxBean.addCompactionWriteOp(topic, m.getHeadersAndPayload().readableBytes()); - long start = System.nanoTime(); - lh.asyncAddEntry(serialized, - (rc, ledger, eid, ctx) -> { - mxBean.addCompactionLatencyOp(topic, System.nanoTime() - start, TimeUnit.NANOSECONDS); - if (rc != BKException.Code.OK) { - bkf.completeExceptionally(BKException.create(rc)); - } else { - bkf.complete(null); - } - }, null); - } catch (Throwable t) { - return FutureUtil.failedFuture(t); - } - return bkf; - } - - private static Pair extractKeyAndSize(RawMessage m) { - ByteBuf headersAndPayload = m.getHeadersAndPayload(); - MessageMetadata msgMetadata = Commands.parseMessageMetadata(headersAndPayload); - if (msgMetadata.hasPartitionKey()) { - int size = headersAndPayload.readableBytes(); - if (msgMetadata.hasUncompressedSize()) { - size = msgMetadata.getUncompressedSize(); - } - return Pair.of(msgMetadata.getPartitionKey(), size); - } else { - return null; - } - } - - private static class PhaseOneResult { - final MessageId from; - final MessageId to; // last undeleted messageId - final MessageId lastReadId; // last read messageId - final Map latestForKey; - - PhaseOneResult(MessageId from, MessageId to, MessageId lastReadId, Map latestForKey) { - this.from = from; - this.to = to; - this.lastReadId = lastReadId; - this.latestForKey = latestForKey; - } - } - - public long getPhaseOneLoopReadTimeoutInSeconds() { - return phaseOneLoopReadTimeout.getSeconds(); - } -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdGenerateDocumentation.java b/pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdGenerateDocumentation.java deleted file mode 100644 index 4a8639c060603..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdGenerateDocumentation.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.utils; - -import com.beust.jcommander.Parameters; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.client.impl.conf.ClientConfigurationData; -import org.apache.pulsar.common.util.BaseGenerateDocumentation; -import org.apache.pulsar.websocket.service.WebSocketProxyConfiguration; - -@Data -@Parameters(commandDescription = "Generate documentation automatically.") -@Slf4j -public class CmdGenerateDocumentation extends BaseGenerateDocumentation { - - @Override - public String generateDocumentByClassName(String className) throws Exception { - StringBuilder sb = new StringBuilder(); - if (ServiceConfiguration.class.getName().equals(className)) { - return generateDocByFieldContext(className, "Broker", sb); - } else if (ClientConfigurationData.class.getName().equals(className)) { - return generateDocByApiModelProperty(className, "Client", sb); - } else if (WebSocketProxyConfiguration.class.getName().equals(className)) { - return generateDocByFieldContext(className, "WebSocket", sb); - } - - return "Class [" + className + "] not found"; - } - - public static void main(String[] args) throws Exception { - CmdGenerateDocumentation generateDocumentation = new CmdGenerateDocumentation(); - generateDocumentation.run(args); - } -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdUtility.java b/pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdUtility.java deleted file mode 100644 index f0f88562953e5..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdUtility.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.utils; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.Writer; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class CmdUtility { - private static final Logger LOG = LoggerFactory.getLogger(CmdUtility.class); - private static final Charset UTF_8 = StandardCharsets.UTF_8; - - /** - * Executes the specified string command in a separate process. STDOUT and STDERR output will be buffered to the - * given writer ( {@link Writer}) argument. - * - * @param writer - * stdout and stderr output - * @param command - * a specified system command - * @return exitValue 0: success, Non-zero: failure - * @throws IOException - */ - public static int exec(Writer writer, String... command) throws IOException { - if (LOG.isDebugEnabled()) { - StringBuilder sb = new StringBuilder(); - for (String str : command) { - sb.append(str).append(' '); - } - LOG.debug("command={}", sb); - } - - ProcessBuilder pb = new ProcessBuilder(command); - pb.redirectErrorStream(true); - Process proc = null; - BufferedReader reader = null; - try { - proc = pb.start(); - reader = new BufferedReader(new InputStreamReader(proc.getInputStream(), UTF_8)); - String line = null; - while ((line = reader.readLine()) != null) { - if (writer != null) { - writer.write(line); - writer.write('\n'); - } - } - LOG.debug("sending the command to the host"); - int exitValue = proc.waitFor(); - if (LOG.isDebugEnabled()) { - LOG.debug("command exit value={}", exitValue); - } - return exitValue; - } catch (InterruptedException e) { - throw new RuntimeException(e); - } finally { - if (reader != null) { - reader.close(); - } - if (proc != null) { - proc.destroy(); - } - } - } - -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSet.java b/pulsar-broker/src/main/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSet.java index e42cae2580b78..cc1eae475fa2d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSet.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSet.java @@ -22,10 +22,12 @@ import java.util.Map; import java.util.NavigableMap; import java.util.NavigableSet; +import java.util.Optional; import java.util.TreeMap; import java.util.TreeSet; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.commons.lang3.mutable.MutableObject; import org.apache.pulsar.common.util.collections.LongPairSet; import org.roaringbitmap.RoaringBitmap; @@ -93,25 +95,51 @@ public void removeUpTo(long item1, long item2) { } } + public > Optional first(LongPairSet.LongPairFunction longPairConverter) { + MutableObject> result = new MutableObject<>(Optional.empty()); + processItems(longPairConverter, item -> { + result.setValue(Optional.of(item)); + return false; + }); + return result.getValue(); + } public > NavigableSet items(int numberOfItems, LongPairSet.LongPairFunction longPairConverter) { NavigableSet items = new TreeSet<>(); + processItems(longPairConverter, item -> { + items.add(item); + return items.size() < numberOfItems; + }); + return items; + } + + public interface ItemProcessor> { + /** + * @param item + * @return false if there is no further processing required + */ + boolean process(T item); + } + + public > void processItems(LongPairSet.LongPairFunction longPairConverter, + ItemProcessor itemProcessor) { lock.readLock().lock(); try { for (Map.Entry entry : map.entrySet()) { Iterator iterator = entry.getValue().stream().iterator(); - while (iterator.hasNext() && items.size() < numberOfItems) { - items.add(longPairConverter.apply(entry.getKey(), iterator.next())); + boolean continueProcessing = true; + while (continueProcessing && iterator.hasNext()) { + T item = longPairConverter.apply(entry.getKey(), iterator.next()); + continueProcessing = itemProcessor.process(item); } - if (items.size() == numberOfItems) { + if (!continueProcessing) { break; } } } finally { lock.readLock().unlock(); } - return items; } public boolean isEmpty() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/utils/SimpleCache.java b/pulsar-broker/src/main/java/org/apache/pulsar/utils/SimpleCache.java new file mode 100644 index 0000000000000..6a3a6721198e1 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/utils/SimpleCache.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.utils; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.function.Supplier; +import lombok.RequiredArgsConstructor; + +public class SimpleCache { + + private final Map> cache = new HashMap<>(); + private final long timeoutMs; + + @RequiredArgsConstructor + private class ExpirableValue { + + private final V value; + private final Consumer expireCallback; + private long deadlineMs; + + boolean tryExpire() { + if (System.currentTimeMillis() >= deadlineMs) { + expireCallback.accept(value); + return true; + } else { + return false; + } + } + + void updateDeadline() { + deadlineMs = System.currentTimeMillis() + timeoutMs; + } + } + + public SimpleCache(final ScheduledExecutorService scheduler, final long timeoutMs, final long frequencyMs) { + this.timeoutMs = timeoutMs; + scheduler.scheduleAtFixedRate(() -> { + synchronized (SimpleCache.this) { + final var keys = new HashSet(); + cache.forEach((key, value) -> { + if (value.tryExpire()) { + keys.add(key); + } + }); + cache.keySet().removeAll(keys); + } + }, frequencyMs, frequencyMs, TimeUnit.MILLISECONDS); + } + + public synchronized V get(final K key, final Supplier valueSupplier, final Consumer expireCallback) { + final var value = cache.get(key); + if (value != null) { + value.updateDeadline(); + return value.value; + } + + final var newValue = new ExpirableValue<>(valueSupplier.get(), expireCallback); + newValue.updateDeadline(); + cache.put(key, newValue); + return newValue.value; + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/utils/StatsOutputStream.java b/pulsar-broker/src/main/java/org/apache/pulsar/utils/StatsOutputStream.java index 1cd4afa650ec2..f0a9bf5fc6952 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/utils/StatsOutputStream.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/utils/StatsOutputStream.java @@ -19,11 +19,12 @@ package org.apache.pulsar.utils; import io.netty.buffer.ByteBuf; -import java.util.Stack; +import java.util.ArrayDeque; +import java.util.Deque; import org.apache.pulsar.common.util.SimpleTextOutputStream; public class StatsOutputStream extends SimpleTextOutputStream { - private final Stack separators = new Stack<>(); + private final Deque separators = new ArrayDeque<>(); public StatsOutputStream(ByteBuf buffer) { super(buffer); @@ -31,7 +32,7 @@ public StatsOutputStream(ByteBuf buffer) { public StatsOutputStream startObject() { checkSeparator(); - separators.push(Boolean.FALSE); + separators.addLast(Boolean.FALSE); write('{'); return this; } @@ -39,19 +40,19 @@ public StatsOutputStream startObject() { public StatsOutputStream startObject(String key) { checkSeparator(); write('"').writeEncoded(key).write("\":{"); - separators.push(Boolean.FALSE); + separators.addLast(Boolean.FALSE); return this; } public StatsOutputStream endObject() { - separators.pop(); + separators.removeLast(); write('}'); return this; } public StatsOutputStream startList() { checkSeparator(); - separators.push(Boolean.FALSE); + separators.addLast(Boolean.FALSE); write('['); return this; } @@ -59,12 +60,12 @@ public StatsOutputStream startList() { public StatsOutputStream startList(String key) { checkSeparator(); write('"').writeEncoded(key).write("\":["); - separators.push(Boolean.FALSE); + separators.addLast(Boolean.FALSE); return this; } public StatsOutputStream endList() { - separators.pop(); + separators.removeLast(); write(']'); return this; } @@ -121,10 +122,11 @@ StatsOutputStream writeItem(String s) { private void checkSeparator() { if (separators.isEmpty()) { return; - } else if (separators.peek() == Boolean.TRUE) { + } else if (separators.peekLast() == Boolean.TRUE) { write(","); } else { - separators.set(separators.size() - 1, Boolean.TRUE); + separators.pollLast(); + separators.addLast(Boolean.TRUE); } } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtils.java index 2477e1ad2bb1b..6f71860164638 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtils.java @@ -18,12 +18,7 @@ */ package org.apache.pulsar.utils.auth.tokens; -import com.beust.jcommander.DefaultUsageFormatter; -import com.beust.jcommander.IUsageFormatter; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; +import com.google.common.annotations.VisibleForTesting; import io.jsonwebtoken.Claims; import io.jsonwebtoken.Jwt; import io.jsonwebtoken.Jwts; @@ -32,7 +27,6 @@ import io.jsonwebtoken.io.Encoders; import io.jsonwebtoken.security.Keys; import java.io.BufferedReader; -import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -40,36 +34,44 @@ import java.security.Key; import java.security.KeyPair; import java.util.Date; +import java.util.Map; import java.util.Optional; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.Callable; import javax.crypto.SecretKey; import lombok.Cleanup; import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; -import org.apache.pulsar.common.util.CmdGenerateDocs; -import org.apache.pulsar.common.util.RelativeTimeUtil; - +import org.apache.pulsar.cli.converters.picocli.TimeUnitToMillisConverter; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; +import picocli.CommandLine.ScopeType; + +@Command(name = "tokens", showDefaultValues = true, scope = ScopeType.INHERIT) public class TokensCliUtils { - public static class Arguments { - @Parameter(names = {"-h", "--help"}, description = "Show this help message") - private boolean help = false; - } + private final CommandLine commander; - @Parameters(commandDescription = "Create a new secret key") - public static class CommandCreateSecretKey { - @Parameter(names = {"-a", + @Option(names = {"-h", "--help"}, usageHelp = true, description = "Show this help message") + private boolean help; + + @Command(description = "Create a new secret key") + public static class CommandCreateSecretKey implements Callable { + @Option(names = {"-a", "--signature-algorithm"}, description = "The signature algorithm for the new secret key.") SignatureAlgorithm algorithm = SignatureAlgorithm.HS256; - @Parameter(names = {"-o", + @Option(names = {"-o", "--output"}, description = "Write the secret key to a file instead of stdout") String outputFile; - @Parameter(names = { + @Option(names = { "-b", "--base64"}, description = "Encode the key in base64") boolean base64 = false; - public void run() throws IOException { + @Override + public Integer call() throws Exception { SecretKey secretKey = AuthTokenUtils.createSecretKey(algorithm); byte[] encoded = secretKey.getEncoded(); @@ -82,66 +84,78 @@ public void run() throws IOException { } else { System.out.write(encoded); } + + return 0; } } - @Parameters(commandDescription = "Create a new or pair of keys public/private") - public static class CommandCreateKeyPair { - @Parameter(names = {"-a", + @Command(description = "Create a new or pair of keys public/private") + public static class CommandCreateKeyPair implements Callable { + @Option(names = {"-a", "--signature-algorithm"}, description = "The signature algorithm for the new key pair.") SignatureAlgorithm algorithm = SignatureAlgorithm.RS256; - @Parameter(names = { + @Option(names = { "--output-private-key"}, description = "File where to write the private key", required = true) String privateKeyFile; - @Parameter(names = { + @Option(names = { "--output-public-key"}, description = "File where to write the public key", required = true) String publicKeyFile; - public void run() throws IOException { + @Override + public Integer call() throws Exception { KeyPair pair = Keys.keyPairFor(algorithm); Files.write(Paths.get(publicKeyFile), pair.getPublic().getEncoded()); Files.write(Paths.get(privateKeyFile), pair.getPrivate().getEncoded()); + + return 0; } } - @Parameters(commandDescription = "Create a new token") - public static class CommandCreateToken { - @Parameter(names = {"-a", + @Command(description = "Create a new token") + public static class CommandCreateToken implements Callable { + @Option(names = {"-a", "--signature-algorithm"}, description = "The signature algorithm for the new key pair.") SignatureAlgorithm algorithm = SignatureAlgorithm.RS256; - @Parameter(names = {"-s", + @Option(names = {"-s", "--subject"}, description = "Specify the 'subject' or 'principal' associate with this token", required = true) private String subject; - @Parameter(names = {"-e", + @Option(names = {"-e", "--expiry-time"}, description = "Relative expiry time for the token (eg: 1h, 3d, 10y)." - + " (m=minutes) Default: no expiration") - private String expiryTime; + + " (m=minutes) Default: no expiration", + converter = TimeUnitToMillisConverter.class) + private Long expiryTime = null; - @Parameter(names = {"-sk", + @Option(names = {"-sk", "--secret-key"}, description = "Pass the secret key for signing the token. This can either be: data:, file:, etc..") private String secretKey; - @Parameter(names = {"-pk", + @Option(names = {"-pk", "--private-key"}, description = "Pass the private key for signing the token. This can either be: data:, file:, etc..") private String privateKey; - public void run() throws Exception { + @Option(names = {"-hs", + "--headers"}, + description = "Additional headers to token. Format: --headers key1=value1") + private Map headers; + + @Override + public Integer call() throws Exception { if (secretKey == null && privateKey == null) { System.err.println( "Either --secret-key or --private-key needs to be passed for signing a token"); - System.exit(1); + return 1; } else if (secretKey != null && privateKey != null) { System.err.println( "Only one of --secret-key and --private-key needs to be passed for signing a token"); - System.exit(1); + return 1; } Key signingKey; @@ -154,41 +168,36 @@ public void run() throws Exception { signingKey = AuthTokenUtils.decodeSecretKey(encodedKey); } - Optional optExpiryTime = Optional.empty(); - if (expiryTime != null) { - long relativeTimeMillis; - try { - relativeTimeMillis = TimeUnit.SECONDS.toMillis( - RelativeTimeUtil.parseRelativeTimeInSeconds(expiryTime)); - } catch (IllegalArgumentException exception) { - throw new ParameterException(exception.getMessage()); - } - optExpiryTime = Optional.of(new Date(System.currentTimeMillis() + relativeTimeMillis)); - } + Optional optExpiryTime = (expiryTime == null) + ? Optional.empty() + : Optional.of(new Date(System.currentTimeMillis() + expiryTime)); - String token = AuthTokenUtils.createToken(signingKey, subject, optExpiryTime); + String token = AuthTokenUtils.createToken(signingKey, subject, optExpiryTime, Optional.ofNullable(headers)); System.out.println(token); + + return 0; } } - @Parameters(commandDescription = "Show the content of token") - public static class CommandShowToken { + @Command(description = "Show the content of token") + public static class CommandShowToken implements Callable { - @Parameter(description = "The token string", arity = 1) - private java.util.List args; + @Parameters(description = "The token string", arity = "0..1") + private String args; - @Parameter(names = {"-i", + @Option(names = {"-i", "--stdin"}, description = "Read token from standard input") private Boolean stdin = false; - @Parameter(names = {"-f", + @Option(names = {"-f", "--token-file"}, description = "Read token from a file") private String tokenFile; - public void run() throws Exception { + @Override + public Integer call() throws Exception { String token; if (args != null) { - token = args.get(0); + token = args; } else if (stdin) { @Cleanup BufferedReader r = new BufferedReader(new InputStreamReader(System.in)); @@ -201,59 +210,61 @@ public void run() throws Exception { System.err.println( "Token needs to be either passed as an argument or through `--stdin`," + " `--token-file` or by the `TOKEN` environment variable"); - System.exit(1); - return; + return 1; } String[] parts = token.split("\\."); System.out.println(new String(Decoders.BASE64URL.decode(parts[0]))); System.out.println("---"); System.out.println(new String(Decoders.BASE64URL.decode(parts[1]))); + + return 0; } } - @Parameters(commandDescription = "Validate a token against a key") - public static class CommandValidateToken { + @Command(description = "Validate a token against a key") + public static class CommandValidateToken implements Callable { - @Parameter(names = {"-a", + @Option(names = {"-a", "--signature-algorithm"}, description = "The signature algorithm for the key pair if using public key.") SignatureAlgorithm algorithm = SignatureAlgorithm.RS256; - @Parameter(description = "The token string", arity = 1) - private java.util.List args; + @Parameters(description = "The token string", arity = "0..1") + private String args; - @Parameter(names = {"-i", + @Option(names = {"-i", "--stdin"}, description = "Read token from standard input") private Boolean stdin = false; - @Parameter(names = {"-f", + @Option(names = {"-f", "--token-file"}, description = "Read token from a file") private String tokenFile; - @Parameter(names = {"-sk", + @Option(names = {"-sk", "--secret-key"}, description = "Pass the secret key for validating the token. This can either be: data:, file:, etc..") private String secretKey; - @Parameter(names = {"-pk", + @Option(names = {"-pk", "--public-key"}, description = "Pass the public key for validating the token. This can either be: data:, file:, etc..") private String publicKey; - public void run() throws Exception { + @Override + public Integer call() throws Exception { if (secretKey == null && publicKey == null) { System.err.println( "Either --secret-key or --public-key needs to be passed for signing a token"); - System.exit(1); + return 1; } else if (secretKey != null && publicKey != null) { System.err.println( "Only one of --secret-key and --public-key needs to be passed for signing a token"); - System.exit(1); + return 1; } String token; if (args != null) { - token = args.get(0); + token = args; } else if (stdin) { @Cleanup BufferedReader r = new BufferedReader(new InputStreamReader(System.in)); @@ -266,8 +277,7 @@ public void run() throws Exception { System.err.println( "Token needs to be either passed as an argument or through `--stdin`," + " `--token-file` or by the `TOKEN` environment variable"); - System.exit(1); - return; + return 1; } Key validationKey; @@ -281,71 +291,52 @@ public void run() throws Exception { } // Validate the token - @SuppressWarnings("unchecked") Jwt jwt = Jwts.parserBuilder() .setSigningKey(validationKey) .build() - .parse(token); + .parseClaimsJws(token); System.out.println(jwt.getBody()); + return 0; } } - public static void main(String[] args) throws Exception { - Arguments arguments = new Arguments(); - JCommander jcommander = new JCommander(arguments); - IUsageFormatter usageFormatter = new DefaultUsageFormatter(jcommander); - - CommandCreateSecretKey commandCreateSecretKey = new CommandCreateSecretKey(); - jcommander.addCommand("create-secret-key", commandCreateSecretKey); - - CommandCreateKeyPair commandCreateKeyPair = new CommandCreateKeyPair(); - jcommander.addCommand("create-key-pair", commandCreateKeyPair); - - CommandCreateToken commandCreateToken = new CommandCreateToken(); - jcommander.addCommand("create", commandCreateToken); + @Command + static class GenDoc implements Callable { - CommandShowToken commandShowToken = new CommandShowToken(); - jcommander.addCommand("show", commandShowToken); + private final CommandLine rootCmd; - CommandValidateToken commandValidateToken = new CommandValidateToken(); - jcommander.addCommand("validate", commandValidateToken); - - jcommander.addCommand("gen-doc", new Object()); - - try { - jcommander.parse(args); - - if (arguments.help || jcommander.getParsedCommand() == null) { - jcommander.usage(); - System.exit(1); - } - } catch (Exception e) { - System.err.println(e); - String chosenCommand = jcommander.getParsedCommand(); - usageFormatter.usage(chosenCommand); - System.exit(1); + public GenDoc(CommandLine rootCmd) { + this.rootCmd = rootCmd; } - String cmd = jcommander.getParsedCommand(); - - if (cmd.equals("create-secret-key")) { - commandCreateSecretKey.run(); - } else if (cmd.equals("create-key-pair")) { - commandCreateKeyPair.run(); - } else if (cmd.equals("create")) { - commandCreateToken.run(); - } else if (cmd.equals("show")) { - commandShowToken.run(); - } else if (cmd.equals("validate")) { - commandValidateToken.run(); - } else if (cmd.equals("gen-doc")) { + @Override + public Integer call() throws Exception { CmdGenerateDocs genDocCmd = new CmdGenerateDocs("pulsar"); - genDocCmd.addCommand("tokens", jcommander); + genDocCmd.addCommand("tokens", rootCmd); genDocCmd.run(null); - } else { - System.err.println("Invalid command: " + cmd); - System.exit(1); + + return 0; } } + + TokensCliUtils() { + commander = new CommandLine(this); + commander.addSubcommand("create-secret-key", CommandCreateSecretKey.class); + commander.addSubcommand("create-key-pair", CommandCreateKeyPair.class); + commander.addSubcommand("create", CommandCreateToken.class); + commander.addSubcommand("show", CommandShowToken.class); + commander.addSubcommand("validate", CommandValidateToken.class); + commander.addSubcommand("gen-doc", new GenDoc(commander)); + } + + @VisibleForTesting + int execute(String[] args) { + return commander.execute(args); + } + + public static void main(String[] args) throws Exception { + TokensCliUtils tokensCliUtils = new TokensCliUtils(); + System.exit(tokensCliUtils.execute(args)); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java b/pulsar-broker/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java index d73d1d7ed6bed..de3077959a444 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java @@ -52,7 +52,6 @@ import org.apache.bookkeeper.clients.exceptions.NamespaceExistsException; import org.apache.bookkeeper.clients.exceptions.NamespaceNotFoundException; import org.apache.bookkeeper.common.allocator.PoolingPolicy; -import org.apache.bookkeeper.common.component.ComponentStarter; import org.apache.bookkeeper.common.component.LifecycleComponent; import org.apache.bookkeeper.common.component.LifecycleComponentStack; import org.apache.bookkeeper.common.concurrent.FutureUtils; @@ -132,7 +131,7 @@ public LocalBookkeeperEnsemble(int numberOfBookies, boolean clearOldData, String advertisedAddress) { this(numberOfBookies, zkPort, streamStoragePort, zkDataDirName, bkDataDirName, clearOldData, advertisedAddress, - new BasePortManager(bkBasePort)); + bkBasePort != 0 ? new BasePortManager(bkBasePort) : () -> 0); } public LocalBookkeeperEnsemble(int numberOfBookies, @@ -195,6 +194,7 @@ private void runZookeeper(int maxCC) throws IOException { : createTempDirectory("zktest"); if (this.clearOldData) { + LOG.info("Wiping Zookeeper data directory at {}", zkDataDir.getAbsolutePath()); cleanDirectory(zkDataDir); } @@ -292,6 +292,7 @@ private void runBookies(ServerConfiguration baseConf) throws Exception { : createTempDirectory("bk" + i + "test"); if (this.clearOldData) { + LOG.info("Wiping Bookie data directory at {}", bkDataDir.getAbsolutePath()); cleanDirectory(bkDataDir); } @@ -311,6 +312,7 @@ private void runBookies(ServerConfiguration baseConf) throws Exception { bsConfs[i] = new ServerConfiguration(baseConf); // override settings bsConfs[i].setBookiePort(bookiePort); + bsConfs[i].setBookieId("bk" + i + "test"); String zkServers = "127.0.0.1:" + zkPort; String metadataServiceUriStr = "zk://" + zkServers + "/ledgers"; @@ -358,7 +360,7 @@ public void runStreamStorage(CompositeConfiguration conf) throws Exception { // create a default namespace try (StorageAdminClient admin = StorageClientBuilder.newBuilder() .withSettings(StorageClientSettings.newBuilder() - .serviceUri("bk://localhost:4181") + .serviceUri("bk://localhost:" + streamStoragePort) .backoffPolicy(Backoff.Jitter.of( Type.EXPONENTIAL, 1000, @@ -455,8 +457,10 @@ public void startBK(int i) throws Exception { try { bookieComponents[i] = org.apache.bookkeeper.server.Main .buildBookieServer(new BookieConfiguration(bsConfs[i])); - ComponentStarter.startComponent(bookieComponents[i]); + bookieComponents[i].start(); } catch (BookieException.InvalidCookieException ice) { + LOG.warn("Invalid cookie found for bookie {}", i, ice); + // InvalidCookieException can happen if the machine IP has changed // Since we are running here a local bookie that is always accessed // from localhost, we can ignore the error @@ -473,7 +477,7 @@ public void startBK(int i) throws Exception { bookieComponents[i] = org.apache.bookkeeper.server.Main .buildBookieServer(new BookieConfiguration(bsConfs[i])); - ComponentStarter.startComponent(bookieComponents[i]); + bookieComponents[i].start(); } @@ -496,7 +500,9 @@ public void stop() throws Exception { LOG.debug("Local ZK/BK stopping ..."); for (LifecycleComponent bookie : bookieComponents) { try { - bookie.close(); + if (bookie != null) { + bookie.close(); + } } catch (Exception e) { LOG.warn("failed to shutdown bookie", e); } diff --git a/pulsar-broker/src/test/java/org/apache/bookkeeper/mledger/impl/MangedLedgerInterceptorImplTest2.java b/pulsar-broker/src/test/java/org/apache/bookkeeper/mledger/impl/MangedLedgerInterceptorImpl2Test.java similarity index 96% rename from pulsar-broker/src/test/java/org/apache/bookkeeper/mledger/impl/MangedLedgerInterceptorImplTest2.java rename to pulsar-broker/src/test/java/org/apache/bookkeeper/mledger/impl/MangedLedgerInterceptorImpl2Test.java index 080ef9ea4c5fe..1be66a7f9d8f5 100644 --- a/pulsar-broker/src/test/java/org/apache/bookkeeper/mledger/impl/MangedLedgerInterceptorImplTest2.java +++ b/pulsar-broker/src/test/java/org/apache/bookkeeper/mledger/impl/MangedLedgerInterceptorImpl2Test.java @@ -39,9 +39,9 @@ */ @Slf4j @Test(groups = "broker") -public class MangedLedgerInterceptorImplTest2 extends MockedBookKeeperTestCase { +public class MangedLedgerInterceptorImpl2Test extends MockedBookKeeperTestCase { - public static void switchLedgerManually(ManagedLedgerImpl ledger){ + private static void switchLedgerManually(ManagedLedgerImpl ledger){ LedgerHandle originalLedgerHandle = ledger.currentLedger; ledger.ledgerClosed(ledger.currentLedger); ledger.createLedgerAfterClosed(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/PrometheusMetricsTestUtil.java b/pulsar-broker/src/test/java/org/apache/pulsar/PrometheusMetricsTestUtil.java new file mode 100644 index 0000000000000..68826372b7bd6 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/PrometheusMetricsTestUtil.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar; + +import com.google.common.util.concurrent.MoreExecutors; +import io.netty.buffer.ByteBuf; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.time.Clock; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGenerator; +import org.apache.pulsar.broker.stats.prometheus.PrometheusRawMetricsProvider; +import org.eclipse.jetty.server.HttpOutput; + +public class PrometheusMetricsTestUtil { + public static void generate(PulsarService pulsar, boolean includeTopicMetrics, boolean includeConsumerMetrics, + boolean includeProducerMetrics, OutputStream out) throws IOException { + generate(new PrometheusMetricsGenerator(pulsar, includeTopicMetrics, includeConsumerMetrics, + includeProducerMetrics, false, Clock.systemUTC()), out, null); + } + + public static void generate(PulsarService pulsar, boolean includeTopicMetrics, boolean includeConsumerMetrics, + boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel, + OutputStream out) throws IOException { + generate(new PrometheusMetricsGenerator(pulsar, includeTopicMetrics, includeConsumerMetrics, + includeProducerMetrics, splitTopicAndPartitionIndexLabel, Clock.systemUTC()), out, null); + } + + public static void generate(PrometheusMetricsGenerator metricsGenerator, OutputStream out, + List metricsProviders) throws IOException { + PrometheusMetricsGenerator.MetricsBuffer metricsBuffer = + metricsGenerator.renderToBuffer(MoreExecutors.directExecutor(), metricsProviders); + try { + ByteBuf buffer = null; + try { + buffer = metricsBuffer.getBufferFuture().get(5, TimeUnit.SECONDS).getUncompressedBuffer(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException(e); + } catch (ExecutionException | TimeoutException e) { + throw new IOException(e); + } + if (buffer == null) { + return; + } + if (out instanceof HttpOutput) { + HttpOutput output = (HttpOutput) out; + ByteBuffer[] nioBuffers = buffer.nioBuffers(); + for (ByteBuffer nioBuffer : nioBuffers) { + output.write(nioBuffer); + } + } else { + int length = buffer.readableBytes(); + if (length > 0) { + buffer.duplicate().readBytes(out, length); + } + } + } finally { + metricsBuffer.release(); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/PulsarBrokerStarterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/PulsarBrokerStarterTest.java index c6945169512b5..4c05a991b7ffe 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/PulsarBrokerStarterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/PulsarBrokerStarterTest.java @@ -22,7 +22,6 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -import com.beust.jcommander.Parameter; import com.google.common.collect.Sets; import java.io.ByteArrayOutputStream; import java.io.File; @@ -32,14 +31,16 @@ import java.io.OutputStreamWriter; import java.io.PrintStream; import java.io.PrintWriter; -import java.lang.reflect.Constructor; +import java.io.StringWriter; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Arrays; +import lombok.Cleanup; +import org.apache.pulsar.PulsarBrokerStarter.BrokerStarter; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.common.util.CmdGenerateDocs; import org.testng.annotations.Test; +import picocli.CommandLine.Option; @Test(groups = "broker") public class PulsarBrokerStarterTest { @@ -282,12 +283,14 @@ public void testGlobalZooKeeperConfig() throws SecurityException, NoSuchMethodEx */ @Test public void testMainWithNoArgument() throws Exception { - try { - PulsarBrokerStarter.main(new String[0]); - fail("No argument to main should've raised FileNotFoundException for no broker config!"); - } catch (FileNotFoundException e) { - // code should reach here. - } + BrokerStarter brokerStarter = new BrokerStarter(); + @Cleanup + StringWriter err = new StringWriter(); + @Cleanup + PrintWriter printWriter = new PrintWriter(err); + brokerStarter.getCommander().setErr(printWriter); + assertEquals(brokerStarter.start(new String[0]), 1); + assertTrue(err.toString().contains("FileNotFoundException")); } /** @@ -296,16 +299,16 @@ public void testMainWithNoArgument() throws Exception { */ @Test public void testMainRunBookieAndAutoRecoveryNoConfig() throws Exception { - try { - File testConfigFile = createValidBrokerConfigFile(); - String[] args = {"-c", testConfigFile.getAbsolutePath(), "-rb", "-ra", "-bc", ""}; - PulsarBrokerStarter.main(args); - fail("No Config file for bookie auto recovery should've raised IllegalArgumentException!"); - } catch (IllegalArgumentException e) { - // code should reach here. - e.printStackTrace(); - assertEquals(e.getMessage(), "No configuration file for Bookie"); - } + File testConfigFile = createValidBrokerConfigFile(); + String[] args = {"-c", testConfigFile.getAbsolutePath(), "-rb", "-ra", "-bc", ""}; + BrokerStarter starter = new BrokerStarter(); + @Cleanup + StringWriter err = new StringWriter(); + @Cleanup + PrintWriter printWriter = new PrintWriter(err); + starter.getCommander().setErr(printWriter); + assertEquals(starter.start(args), 1); + assertTrue(err.toString().contains("No configuration file for Bookie")); } /** @@ -314,15 +317,16 @@ public void testMainRunBookieAndAutoRecoveryNoConfig() throws Exception { */ @Test public void testMainRunBookieRecoveryNoConfig() throws Exception { - try { - File testConfigFile = createValidBrokerConfigFile(); - String[] args = {"-c", testConfigFile.getAbsolutePath(), "-ra", "-bc", ""}; - PulsarBrokerStarter.main(args); - fail("No Config file for bookie auto recovery should've raised IllegalArgumentException!"); - } catch (IllegalArgumentException e) { - // code should reach here. - assertEquals(e.getMessage(), "No configuration file for Bookie"); - } + File testConfigFile = createValidBrokerConfigFile(); + String[] args = {"-c", testConfigFile.getAbsolutePath(), "-ra", "-bc", ""}; + BrokerStarter starter = new BrokerStarter(); + @Cleanup + StringWriter err = new StringWriter(); + @Cleanup + PrintWriter printWriter = new PrintWriter(err); + starter.getCommander().setErr(printWriter); + assertEquals(starter.start(args), 1); + assertTrue(err.toString().contains("No configuration file for Bookie")); } /** @@ -330,15 +334,16 @@ public void testMainRunBookieRecoveryNoConfig() throws Exception { */ @Test public void testMainRunBookieNoConfig() throws Exception { - try { - File testConfigFile = createValidBrokerConfigFile(); - String[] args = {"-c", testConfigFile.getAbsolutePath(), "-rb", "-bc", ""}; - PulsarBrokerStarter.main(args); - fail("No Config file for bookie should've raised IllegalArgumentException!"); - } catch (IllegalArgumentException e) { - // code should reach here - assertEquals(e.getMessage(), "No configuration file for Bookie"); - } + File testConfigFile = createValidBrokerConfigFile(); + String[] args = {"-c", testConfigFile.getAbsolutePath(), "-rb", "-bc", ""}; + BrokerStarter starter = new BrokerStarter(); + @Cleanup + StringWriter err = new StringWriter(); + @Cleanup + PrintWriter printWriter = new PrintWriter(err); + starter.getCommander().setErr(printWriter); + assertEquals(starter.start(args), 1); + assertTrue(err.toString().contains("No configuration file for Bookie")); } /** @@ -346,14 +351,16 @@ public void testMainRunBookieNoConfig() throws Exception { */ @Test public void testMainEnableRunBookieThroughBrokerConfig() throws Exception { - try { - File testConfigFile = createValidBrokerConfigFile(); - String[] args = {"-c", testConfigFile.getAbsolutePath()}; - PulsarBrokerStarter.main(args); - fail("No argument to main should've raised IllegalArgumentException for no bookie config!"); - } catch (IllegalArgumentException e) { - // code should reach here. - } + File testConfigFile = createValidBrokerConfigFile(); + String[] args = {"-c", testConfigFile.getAbsolutePath()}; + BrokerStarter starter = new BrokerStarter(); + @Cleanup + StringWriter err = new StringWriter(); + @Cleanup + PrintWriter printWriter = new PrintWriter(err); + starter.getCommander().setErr(printWriter); + assertEquals(starter.start(args), 1); + assertTrue(err.toString().contains("IllegalArgumentException")); } @Test @@ -364,21 +371,15 @@ public void testMainGenerateDocs() throws Exception { System.setOut(new PrintStream(baoStream)); Class argumentsClass = Class.forName("org.apache.pulsar.PulsarBrokerStarter$StarterArguments"); - Constructor constructor = argumentsClass.getDeclaredConstructor(); - constructor.setAccessible(true); - Object obj = constructor.newInstance(); - - CmdGenerateDocs cmd = new CmdGenerateDocs("pulsar"); - cmd.addCommand("broker", obj); - cmd.run(null); + PulsarBrokerStarter.main(new String[]{"-g"}); String message = baoStream.toString(); Field[] fields = argumentsClass.getDeclaredFields(); for (Field field : fields) { - boolean fieldHasAnno = field.isAnnotationPresent(Parameter.class); + boolean fieldHasAnno = field.isAnnotationPresent(Option.class); if (fieldHasAnno) { - Parameter fieldAnno = field.getAnnotation(Parameter.class); + Option fieldAnno = field.getAnnotation(Option.class); String[] names = fieldAnno.names(); String nameStr = Arrays.asList(names).toString(); nameStr = nameStr.substring(1, nameStr.length() - 1); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/PulsarClusterMetadataSetupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/PulsarClusterMetadataSetupTest.java index 6196f66698869..710e040f8df1c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/PulsarClusterMetadataSetupTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/PulsarClusterMetadataSetupTest.java @@ -19,13 +19,15 @@ package org.apache.pulsar; import static org.testng.Assert.assertTrue; -import com.beust.jcommander.Parameter; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.lang.reflect.Field; import java.util.Arrays; +import lombok.extern.slf4j.Slf4j; import org.testng.annotations.Test; +import picocli.CommandLine.Option; +@Slf4j public class PulsarClusterMetadataSetupTest { @Test public void testMainGenerateDocs() throws Exception { @@ -43,16 +45,16 @@ public void testMainGenerateDocs() throws Exception { Field[] fields = argumentsClass.getDeclaredFields(); for (Field field : fields) { - boolean fieldHasAnno = field.isAnnotationPresent(Parameter.class); + boolean fieldHasAnno = field.isAnnotationPresent(Option.class); if (fieldHasAnno) { - Parameter fieldAnno = field.getAnnotation(Parameter.class); + Option fieldAnno = field.getAnnotation(Option.class); String[] names = fieldAnno.names(); - if (names.length == 0) { + if (names.length == 0 || fieldAnno.hidden()) { continue; } String nameStr = Arrays.asList(names).toString(); nameStr = nameStr.substring(1, nameStr.length() - 1); - assertTrue(message.indexOf(nameStr) > 0); + assertTrue(message.indexOf(nameStr) > 0, nameStr); } } } finally { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/PulsarClusterMetadataTeardownTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/PulsarClusterMetadataTeardownTest.java index f6a388dac76e3..95d12f378c55f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/PulsarClusterMetadataTeardownTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/PulsarClusterMetadataTeardownTest.java @@ -19,12 +19,12 @@ package org.apache.pulsar; import static org.testng.Assert.assertTrue; -import com.beust.jcommander.Parameter; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.lang.reflect.Field; import java.util.Arrays; import org.testng.annotations.Test; +import picocli.CommandLine.Option; public class PulsarClusterMetadataTeardownTest { @Test @@ -43,9 +43,9 @@ public void testMainGenerateDocs() throws Exception { Field[] fields = argumentsClass.getDeclaredFields(); for (Field field : fields) { - boolean fieldHasAnno = field.isAnnotationPresent(Parameter.class); + boolean fieldHasAnno = field.isAnnotationPresent(Option.class); if (fieldHasAnno) { - Parameter fieldAnno = field.getAnnotation(Parameter.class); + Option fieldAnno = field.getAnnotation(Option.class); String[] names = fieldAnno.names(); if (names.length == 0) { continue; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/PulsarInitialNamespaceSetupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/PulsarInitialNamespaceSetupTest.java index c1ad8c621c46d..0c6ba05b460e7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/PulsarInitialNamespaceSetupTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/PulsarInitialNamespaceSetupTest.java @@ -19,12 +19,12 @@ package org.apache.pulsar; import static org.testng.Assert.assertTrue; -import com.beust.jcommander.Parameter; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.lang.reflect.Field; import java.util.Arrays; import org.testng.annotations.Test; +import picocli.CommandLine.Option; public class PulsarInitialNamespaceSetupTest { @Test @@ -43,9 +43,9 @@ public void testMainGenerateDocs() throws Exception { Field[] fields = argumentsClass.getDeclaredFields(); for (Field field : fields) { - boolean fieldHasAnno = field.isAnnotationPresent(Parameter.class); + boolean fieldHasAnno = field.isAnnotationPresent(Option.class); if (fieldHasAnno) { - Parameter fieldAnno = field.getAnnotation(Parameter.class); + Option fieldAnno = field.getAnnotation(Option.class); String[] names = fieldAnno.names(); if (names.length == 0) { continue; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/PulsarStandaloneTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/PulsarStandaloneTest.java index 6ed93a75a3fb5..3d22feb822e32 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/PulsarStandaloneTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/PulsarStandaloneTest.java @@ -31,6 +31,7 @@ import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.metadata.bookkeeper.BKCluster; import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -46,12 +47,15 @@ public Object[][] enableBrokerClientAuth() { @Test public void testStandaloneWithRocksDB() throws Exception { String[] args = new String[]{"--config", - "./src/test/resources/configurations/pulsar_broker_test_standalone_with_rocksdb.conf"}; + "./src/test/resources/configurations/pulsar_broker_test_standalone_with_rocksdb.conf", + "-nss", + "-nfw"}; final int bookieNum = 3; final File tempDir = IOUtils.createTempDir("standalone", "test"); PulsarStandaloneStarter standalone = new PulsarStandaloneStarter(args); standalone.setBkDir(tempDir.getAbsolutePath()); + standalone.setBkPort(0); standalone.setNumOfBk(bookieNum); standalone.startBookieWithMetadataStore(); @@ -90,11 +94,12 @@ public void testMetadataInitialization(boolean enableBrokerClientAuth) throws Ex } final File bkDir = IOUtils.createTempDir("standalone", "bk"); standalone.setNumOfBk(1); + standalone.setBkPort(0); standalone.setBkDir(bkDir.getAbsolutePath()); standalone.start(); @Cleanup PulsarAdmin admin = PulsarAdmin.builder() - .serviceHttpUrl("http://localhost:8080") + .serviceHttpUrl(standalone.getWebServiceUrl()) .authentication(new MockTokenAuthenticationProvider.MockAuthentication()) .build(); if (enableBrokerClientAuth) { @@ -104,8 +109,8 @@ public void testMetadataInitialization(boolean enableBrokerClientAuth) throws Ex } else { assertTrue(admin.clusters().getClusters().isEmpty()); admin.clusters().createCluster("test_cluster", ClusterData.builder() - .serviceUrl("http://localhost:8080/") - .brokerServiceUrl("pulsar://localhost:6650/") + .serviceUrl(standalone.getWebServiceUrl()) + .brokerServiceUrl(standalone.getBrokerServiceUrl()) .build()); assertTrue(admin.tenants().getTenants().isEmpty()); admin.tenants().createTenant("public", TenantInfo.builder() @@ -125,4 +130,39 @@ public void testMetadataInitialization(boolean enableBrokerClientAuth) throws Ex cleanDirectory(bkDir); cleanDirectory(metadataDir); } + + + @Test + public void testShutdownHookClosesBkCluster() throws Exception { + File dataDir = IOUtils.createTempDir("data", ""); + File metadataDir = new File(dataDir, "metadata"); + File bkDir = new File(dataDir, "bookkeeper"); + @Cleanup + PulsarStandaloneStarter standalone = new PulsarStandaloneStarter(new String[] { + "--config", + "./src/test/resources/configurations/pulsar_broker_test_standalone_with_rocksdb.conf", + "-nss", + "-nfw", + "--metadata-dir", + metadataDir.getAbsolutePath(), + "--bookkeeper-dir", + bkDir.getAbsolutePath() + }); + standalone.setTestMode(true); + standalone.setBkPort(0); + standalone.start(); + BKCluster bkCluster = standalone.bkCluster; + standalone.runShutdownHook(); + assertTrue(bkCluster.isClosed()); + } + + @Test + public void testWipeData() throws Exception { + PulsarStandaloneStarter standalone = new PulsarStandaloneStarter(new String[] { + "--config", + "./src/test/resources/configurations/standalone_no_client_auth.conf", + "--wipe-data" + }); + assertTrue(standalone.isWipeData()); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/PulsarTransactionCoordinatorMetadataSetupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/PulsarTransactionCoordinatorMetadataSetupTest.java index 70c7c7bd62ee9..6ff055385b2a4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/PulsarTransactionCoordinatorMetadataSetupTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/PulsarTransactionCoordinatorMetadataSetupTest.java @@ -19,12 +19,12 @@ package org.apache.pulsar; import static org.testng.Assert.assertTrue; -import com.beust.jcommander.Parameter; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.lang.reflect.Field; import java.util.Arrays; import org.testng.annotations.Test; +import picocli.CommandLine.Option; public class PulsarTransactionCoordinatorMetadataSetupTest { @Test @@ -43,9 +43,9 @@ public void testMainGenerateDocs() throws Exception { Field[] fields = argumentsClass.getDeclaredFields(); for (Field field : fields) { - boolean fieldHasAnno = field.isAnnotationPresent(Parameter.class); + boolean fieldHasAnno = field.isAnnotationPresent(Option.class); if (fieldHasAnno) { - Parameter fieldAnno = field.getAnnotation(Parameter.class); + Option fieldAnno = field.getAnnotation(Option.class); String[] names = fieldAnno.names(); if (names.length == 0) { continue; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/PulsarVersionStarterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/PulsarVersionStarterTest.java index 219e3b80cd308..b921c3d384315 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/PulsarVersionStarterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/PulsarVersionStarterTest.java @@ -19,12 +19,12 @@ package org.apache.pulsar; import static org.testng.Assert.assertTrue; -import com.beust.jcommander.Parameter; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.lang.reflect.Field; import java.util.Arrays; import org.testng.annotations.Test; +import picocli.CommandLine.Option; public class PulsarVersionStarterTest { @Test @@ -43,9 +43,9 @@ public void testMainGenerateDocs() throws Exception { Field[] fields = argumentsClass.getDeclaredFields(); for (Field field : fields) { - boolean fieldHasAnno = field.isAnnotationPresent(Parameter.class); + boolean fieldHasAnno = field.isAnnotationPresent(Option.class); if (fieldHasAnno) { - Parameter fieldAnno = field.getAnnotation(Parameter.class); + Option fieldAnno = field.getAnnotation(Option.class); String[] names = fieldAnno.names(); if (names.length == 0) { continue; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/TestNGInstanceOrder.java b/pulsar-broker/src/test/java/org/apache/pulsar/TestNGInstanceOrder.java new file mode 100644 index 0000000000000..50c9863d586ec --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/TestNGInstanceOrder.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar; + +import java.util.Comparator; +import java.util.List; +import org.testng.IMethodInstance; +import org.testng.IMethodInterceptor; +import org.testng.ITestContext; + +// Sorts the test methods by test object instance hashcode, then priority, then method name. Useful when Factory +// generated tests interfere with each other. +public class TestNGInstanceOrder implements IMethodInterceptor { + @Override + public List intercept(List methods, ITestContext context) { + return methods.stream().sorted(Comparator.comparingInt(o -> o.getInstance().hashCode()) + .thenComparingInt(o -> o.getMethod().getInterceptedPriority()) + .thenComparingInt(o -> o.getMethod().getPriority()) + .thenComparing(o -> o.getMethod().getMethodName())) + .toList(); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/BookKeeperClientFactoryImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/BookKeeperClientFactoryImplTest.java index a02689dc9763a..3c0e4d0c409df 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/BookKeeperClientFactoryImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/BookKeeperClientFactoryImplTest.java @@ -41,6 +41,7 @@ import org.apache.pulsar.bookie.rackawareness.BookieRackAffinityMapping; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.apache.pulsar.zookeeper.ZkIsolatedBookieEnsemblePlacementPolicy; import org.testng.annotations.Test; /** @@ -152,6 +153,24 @@ public void testSetDefaultEnsemblePlacementPolicyRackAwareEnabledChangedValues() assertEquals(20, bkConf.getMinNumRacksPerWriteQuorum()); } + @Test + public void testSetEnsemblePlacementPolicys() { + ClientConfiguration bkConf = new ClientConfiguration(); + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setBookkeeperClientMinNumRacksPerWriteQuorum(3); + conf.setBookkeeperClientEnforceMinNumRacksPerWriteQuorum(true); + + MetadataStore store = mock(MetadataStore.class); + + BookKeeperClientFactoryImpl.setEnsemblePlacementPolicy( + bkConf, + conf, + store, + ZkIsolatedBookieEnsemblePlacementPolicy.class); + assertEquals(bkConf.getMinNumRacksPerWriteQuorum(), 3); + assertTrue(bkConf.getEnforceMinNumRacksPerWriteQuorum()); + } + @Test public void testSetDiskWeightBasedPlacementEnabled() { BookKeeperClientFactoryImpl factory = new BookKeeperClientFactoryImpl(); @@ -303,22 +322,22 @@ public void testBookKeeperIoThreadsConfiguration() throws Exception { public void testBookKeeperLimitStatsLoggingConfiguration() throws Exception { BookKeeperClientFactoryImpl factory = new BookKeeperClientFactoryImpl(); ServiceConfiguration conf = new ServiceConfiguration(); - assertFalse( - factory.createBkClientConfiguration(mock(MetadataStoreExtended.class), conf).getLimitStatsLogging()); + assertTrue(factory.createBkClientConfiguration(mock(MetadataStoreExtended.class), conf) + .getLimitStatsLogging()); EventLoopGroup eventLoopGroup = mock(EventLoopGroup.class); BookKeeper.Builder builder = factory.getBookKeeperBuilder(conf, eventLoopGroup, mock(StatsLogger.class), factory.createBkClientConfiguration(mock(MetadataStoreExtended.class), conf)); ClientConfiguration clientConfiguration = (ClientConfiguration) FieldUtils.readField(builder, "conf", true); - assertFalse(clientConfiguration.getLimitStatsLogging()); + assertTrue(clientConfiguration.getLimitStatsLogging()); - conf.setBookkeeperClientLimitStatsLogging(true); - assertTrue(factory.createBkClientConfiguration(mock(MetadataStoreExtended.class), conf) - .getLimitStatsLogging()); + conf.setBookkeeperClientLimitStatsLogging(false); + assertFalse( + factory.createBkClientConfiguration(mock(MetadataStoreExtended.class), conf).getLimitStatsLogging()); builder = factory.getBookKeeperBuilder(conf, eventLoopGroup, mock(StatsLogger.class), factory.createBkClientConfiguration(mock(MetadataStoreExtended.class), conf)); clientConfiguration = (ClientConfiguration) FieldUtils.readField(builder, "conf", true); - assertTrue(clientConfiguration.getLimitStatsLogging()); + assertFalse(clientConfiguration.getLimitStatsLogging()); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/BrokerTestUtil.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/BrokerTestUtil.java index bfb172d0711d4..5641816ee0b80 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/BrokerTestUtil.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/BrokerTestUtil.java @@ -18,8 +18,31 @@ */ package org.apache.pulsar.broker; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectWriter; +import java.io.IOException; +import java.io.StringWriter; +import java.io.UncheckedIOException; +import java.time.Duration; +import java.util.Arrays; import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.ObjectMapperFactory; import org.mockito.Mockito; +import org.slf4j.Logger; /** * Holds util methods used in test. @@ -77,4 +100,138 @@ public static T spyWithoutRecordingInvocations(T object) { .defaultAnswer(Mockito.CALLS_REAL_METHODS) .stubOnly()); } + + /** + * Uses Jackson to create a JSON string for the given object + * @param object to convert to JSON + * @return JSON string + */ + public static String toJson(Object object) { + ObjectWriter writer = ObjectMapperFactory.getMapper().writer(); + StringWriter stringWriter = new StringWriter(); + try (JsonGenerator generator = writer.createGenerator(stringWriter).useDefaultPrettyPrinter()) { + generator.writeObject(object); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return stringWriter.toString(); + } + + /** + * Logs the topic stats and internal stats for the given topic + * @param logger logger to use + * @param pulsarAdmin PulsarAdmin client to use + * @param topic topic name + */ + public static void logTopicStats(Logger logger, PulsarAdmin pulsarAdmin, String topic) { + try { + logger.info("[{}] stats: {}", topic, toJson(pulsarAdmin.topics().getStats(topic))); + logger.info("[{}] internalStats: {}", topic, + toJson(pulsarAdmin.topics().getInternalStats(topic, true))); + } catch (PulsarAdminException e) { + logger.warn("Failed to get stats for topic {}", topic, e); + } + } + + /** + * Receive messages concurrently from multiple consumers and handles them using the provided message handler. + * The message handler should return true if it wants to continue receiving more messages, false otherwise. + * + * @param messageHandler the message handler + * @param quietTimeout the duration of quiet time after which the method will stop waiting for more messages + * @param consumers the consumers to receive messages from + * @param the message value type + */ + public static void receiveMessages(BiFunction, Message, Boolean> messageHandler, + Duration quietTimeout, + Consumer... consumers) { + FutureUtil.waitForAll(Arrays.stream(consumers) + .map(consumer -> receiveMessagesAsync(consumer, quietTimeout, messageHandler)).toList()).join(); + } + + // asynchronously receive messages from a consumer and handle them using the provided message handler + // the benefit is that multiple consumers can be concurrently consumed without the need to have multiple threads + // this is useful in tests where multiple consumers are needed to test the functionality + private static CompletableFuture receiveMessagesAsync(Consumer consumer, Duration quietTimeout, + BiFunction, Message, Boolean> + messageHandler) { + CompletableFuture> receiveFuture = consumer.receiveAsync(); + return receiveFuture + .orTimeout(quietTimeout.toMillis(), TimeUnit.MILLISECONDS) + .handle((msg, t) -> { + if (t != null) { + if (t instanceof TimeoutException) { + // cancel the receive future so that Pulsar client can clean up the resources + receiveFuture.cancel(false); + return false; + } else { + throw FutureUtil.wrapToCompletionException(t); + } + } + return messageHandler.apply(consumer, msg); + }).thenComposeAsync(receiveMore -> { + if (receiveMore) { + return receiveMessagesAsync(consumer, quietTimeout, messageHandler); + } else { + return CompletableFuture.completedFuture(null); + } + }); + } + + /** + * Receive messages concurrently from multiple consumers and handles them using the provided message handler. + * The messages are received until the quiet timeout is reached or the maximum number of messages is received. + * + * @param messageHandler the message handler + * @param quietTimeout the duration of quiet time after which the method will stop waiting for more messages + * @param maxMessages the maximum number of messages to receive + * @param consumers the consumers to receive messages from + * @param the message value type + */ + public static void receiveMessagesN(BiConsumer, Message> messageHandler, + Duration quietTimeout, + int maxMessages, + Consumer... consumers) + throws ExecutionException, InterruptedException { + AtomicInteger messagesReceived = new AtomicInteger(); + receiveMessages( + (consumer, message) -> { + messageHandler.accept(consumer, message); + return messagesReceived.incrementAndGet() < maxMessages; + }, quietTimeout, consumers); + } + + /** + * Receive messages concurrently from multiple consumers and handles them using the provided message handler. + * + * @param messageHandler the message handler + * @param quietTimeout the duration of quiet time after which the method will stop waiting for more messages + * @param consumers the consumers to receive messages from + * @param the message value type + */ + public static void receiveMessagesInThreads(BiFunction, Message, Boolean> messageHandler, + final Duration quietTimeout, + Consumer... consumers) { + FutureUtil.waitForAll(Arrays.stream(consumers).sequential().map(consumer -> { + return CompletableFuture.runAsync(() -> { + try { + while (!Thread.currentThread().isInterrupted()) { + Message msg = consumer.receive((int) quietTimeout.toMillis(), TimeUnit.MILLISECONDS); + if (msg != null) { + if (!messageHandler.apply(consumer, msg)) { + break; + } + } else { + break; + } + } + } catch (PulsarClientException e) { + throw new CompletionException(e); + } + }, runnable -> { + Thread thread = new Thread(runnable, "Consumer-" + consumer.getConsumerName()); + thread.start(); + }); + }).toList()).join(); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/EmbeddedPulsarCluster.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/EmbeddedPulsarCluster.java index eac76a3a80ede..ff454f9c566c3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/EmbeddedPulsarCluster.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/EmbeddedPulsarCluster.java @@ -24,11 +24,11 @@ import java.util.Optional; import java.util.stream.Collectors; import lombok.Builder; +import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.metadata.bookkeeper.BKCluster; -import org.junit.platform.commons.util.StringUtils; public class EmbeddedPulsarCluster implements AutoCloseable { @@ -112,7 +112,7 @@ private ServiceConfiguration getConf() { conf.setWebServicePort(Optional.of(0)); conf.setNumExecutorThreadPoolSize(1); conf.setNumCacheExecutorThreadPoolSize(1); - conf.setNumWorkerThreadsForNonPersistentTopic(1); + conf.setTopicOrderedExecutorThreadNum(1); conf.setNumIOThreads(1); conf.setNumOrderedExecutorThreads(1); conf.setBookkeeperClientNumWorkerThreads(1); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java new file mode 100644 index 0000000000000..dbaaee8f48cd5 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.BatchMessageIdImpl; +import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.common.util.FutureUtil; +import org.awaitility.Awaitility; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class LedgerLostAndSkipNonRecoverableTest extends ProducerConsumerBase { + + private static final String DEFAULT_NAMESPACE = "my-property/my-ns"; + + @BeforeClass + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + protected void doInitConf() throws Exception { + conf.setAutoSkipNonRecoverableData(true); + } + + @DataProvider(name = "batchEnabled") + public Object[][] batchEnabled(){ + return new Object[][]{ + {true}, + {false} + }; + } + + @Test(timeOut = 30000, dataProvider = "batchEnabled") + public void testMarkDeletedPositionCanForwardAfterTopicLedgerLost(boolean enabledBatch) throws Exception { + String topicSimpleName = UUID.randomUUID().toString().replaceAll("-", ""); + String subName = UUID.randomUUID().toString().replaceAll("-", ""); + String topicName = String.format("persistent://%s/%s", DEFAULT_NAMESPACE, topicSimpleName); + + log.info("create topic and subscription."); + Consumer sub = createConsumer(topicName, subName, enabledBatch); + sub.redeliverUnacknowledgedMessages(); + sub.close(); + + log.info("send many messages."); + int ledgerCount = 3; + int messageCountPerLedger = enabledBatch ? 25 : 5; + int messageCountPerEntry = enabledBatch ? 5 : 1; + List[] sendMessages = + sendManyMessages(topicName, ledgerCount, messageCountPerLedger, messageCountPerEntry); + int sendMessageCount = Arrays.asList(sendMessages).stream() + .flatMap(s -> s.stream()).collect(Collectors.toList()).size(); + log.info("send {} messages", sendMessageCount); + + log.info("make individual ack."); + ConsumerAndReceivedMessages consumerAndReceivedMessages1 = + waitConsumeAndAllMessages(topicName, subName, enabledBatch,false); + List[] messageIds = consumerAndReceivedMessages1.messageIds; + Consumer consumer = consumerAndReceivedMessages1.consumer; + MessageIdImpl individualPosition = messageIds[1].get(messageCountPerEntry - 1); + MessageIdImpl expectedMarkDeletedPosition = + new MessageIdImpl(messageIds[0].get(0).getLedgerId(), messageIds[0].get(0).getEntryId(), -1); + MessageIdImpl lastPosition = + new MessageIdImpl(messageIds[2].get(4).getLedgerId(), messageIds[2].get(4).getEntryId(), -1); + consumer.acknowledge(individualPosition); + consumer.acknowledge(expectedMarkDeletedPosition); + waitPersistentCursorLedger(topicName, subName, expectedMarkDeletedPosition.getLedgerId(), + expectedMarkDeletedPosition.getEntryId()); + consumer.close(); + + log.info("Make lost ledger [{}].", individualPosition.getLedgerId()); + pulsar.getBrokerService().getTopic(topicName, false).get().get().close(false); + pulsarTestContext.getMockBookKeeper().deleteLedger(individualPosition.getLedgerId()); + + log.info("send some messages."); + sendManyMessages(topicName, 3, messageCountPerEntry); + + log.info("receive all messages then verify mark deleted position"); + ConsumerAndReceivedMessages consumerAndReceivedMessages2 = + waitConsumeAndAllMessages(topicName, subName, enabledBatch, true); + waitMarkDeleteLargeAndEquals(topicName, subName, lastPosition.getLedgerId(), lastPosition.getEntryId()); + + // cleanup + consumerAndReceivedMessages2.consumer.close(); + admin.topics().delete(topicName); + } + + private ManagedCursorImpl getCursor(String topicName, String subName) throws Exception { + PersistentSubscription subscription_ = + (PersistentSubscription) pulsar.getBrokerService().getTopic(topicName, false) + .get().get().getSubscription(subName); + return (ManagedCursorImpl) subscription_.getCursor(); + } + + private void waitMarkDeleteLargeAndEquals(String topicName, String subName, final long markDeletedLedgerId, + final long markDeletedEntryId) throws Exception { + Awaitility.await().atMost(Duration.ofSeconds(45)).untilAsserted(() -> { + Position persistentMarkDeletedPosition = getCursor(topicName, subName).getMarkDeletedPosition(); + log.info("markDeletedPosition {}:{}, expected {}:{}", persistentMarkDeletedPosition.getLedgerId(), + persistentMarkDeletedPosition.getEntryId(), markDeletedLedgerId, markDeletedEntryId); + Assert.assertTrue(persistentMarkDeletedPosition.getLedgerId() >= markDeletedLedgerId); + if (persistentMarkDeletedPosition.getLedgerId() > markDeletedLedgerId){ + return; + } + Assert.assertTrue(persistentMarkDeletedPosition.getEntryId() >= markDeletedEntryId); + }); + } + + private void waitPersistentCursorLedger(String topicName, String subName, final long markDeletedLedgerId, + final long markDeletedEntryId) throws Exception { + Awaitility.await().untilAsserted(() -> { + Position persistentMarkDeletedPosition = getCursor(topicName, subName).getPersistentMarkDeletedPosition(); + Assert.assertEquals(persistentMarkDeletedPosition.getLedgerId(), markDeletedLedgerId); + Assert.assertEquals(persistentMarkDeletedPosition.getEntryId(), markDeletedEntryId); + }); + } + + private List[] sendManyMessages(String topicName, int ledgerCount, int messageCountPerLedger, + int messageCountPerEntry) throws Exception { + List[] messageIds = new List[ledgerCount]; + for (int i = 0; i < ledgerCount; i++){ + admin.topics().unload(topicName); + if (messageCountPerEntry == 1) { + messageIds[i] = sendManyMessages(topicName, messageCountPerLedger); + } else { + messageIds[i] = sendManyBatchedMessages(topicName, messageCountPerEntry, + messageCountPerLedger / messageCountPerEntry); + } + } + return messageIds; + } + + private List sendManyMessages(String topicName, int messageCountPerLedger, + int messageCountPerEntry) throws Exception { + if (messageCountPerEntry == 1) { + return sendManyMessages(topicName, messageCountPerLedger); + } else { + return sendManyBatchedMessages(topicName, messageCountPerEntry, + messageCountPerLedger / messageCountPerEntry); + } + } + + private List sendManyMessages(String topicName, int msgCount) throws Exception { + List messageIdList = new ArrayList<>(); + final Producer producer = pulsarClient.newProducer(Schema.JSON(String.class)) + .topic(topicName) + .enableBatching(false) + .create(); + long timestamp = System.currentTimeMillis(); + for (int i = 0; i < msgCount; i++){ + String messageSuffix = String.format("%s-%s", timestamp, i); + MessageIdImpl messageIdSent = (MessageIdImpl) producer.newMessage() + .key(String.format("Key-%s", messageSuffix)) + .value(String.format("Msg-%s", messageSuffix)) + .send(); + messageIdList.add(messageIdSent); + } + producer.close(); + return messageIdList; + } + + private List sendManyBatchedMessages(String topicName, int msgCountPerEntry, int entryCount) + throws Exception { + Producer producer = pulsarClient.newProducer(Schema.JSON(String.class)) + .topic(topicName) + .enableBatching(true) + .batchingMaxPublishDelay(Integer.MAX_VALUE, TimeUnit.SECONDS) + .batchingMaxMessages(Integer.MAX_VALUE) + .create(); + List> messageIdFutures = new ArrayList<>(); + for (int i = 0; i < entryCount; i++){ + for (int j = 0; j < msgCountPerEntry; j++){ + CompletableFuture messageIdFuture = + producer.newMessage().value(String.format("entry-seq[%s], batch_index[%s]", i, j)).sendAsync(); + messageIdFutures.add(messageIdFuture); + } + producer.flush(); + } + producer.close(); + FutureUtil.waitForAll(messageIdFutures).get(); + return messageIdFutures.stream().map(f -> (MessageIdImpl)f.join()).collect(Collectors.toList()); + } + + private ConsumerAndReceivedMessages waitConsumeAndAllMessages(String topicName, String subName, + final boolean enabledBatch, + boolean ack) throws Exception { + List messageIds = new ArrayList<>(); + final Consumer consumer = createConsumer(topicName, subName, enabledBatch); + while (true){ + Message message = consumer.receive(5, TimeUnit.SECONDS); + if (message != null){ + messageIds.add((MessageIdImpl) message.getMessageId()); + if (ack) { + consumer.acknowledge(message); + } + } else { + break; + } + } + log.info("receive {} messages", messageIds.size()); + return new ConsumerAndReceivedMessages(consumer, sortMessageId(messageIds, enabledBatch)); + } + + @AllArgsConstructor + private static class ConsumerAndReceivedMessages { + private Consumer consumer; + private List[] messageIds; + } + + private List[] sortMessageId(List messageIds, boolean enabledBatch){ + Map> map = messageIds.stream().collect(Collectors.groupingBy(v -> v.getLedgerId())); + TreeMap> sortedMap = new TreeMap<>(map); + List[] res = new List[sortedMap.size()]; + Iterator>> iterator = sortedMap.entrySet().iterator(); + for (int i = 0; i < sortedMap.size(); i++){ + res[i] = iterator.next().getValue(); + } + for (List list : res){ + list.sort((m1, m2) -> { + if (enabledBatch){ + BatchMessageIdImpl mb1 = (BatchMessageIdImpl) m1; + BatchMessageIdImpl mb2 = (BatchMessageIdImpl) m2; + return (int) (mb1.getLedgerId() * 1000000 + mb1.getEntryId() * 1000 + mb1.getBatchIndex() - + mb2.getLedgerId() * 1000000 + mb2.getEntryId() * 1000 + mb2.getBatchIndex()); + } + return (int) (m1.getLedgerId() * 1000 + m1.getEntryId() - + m2.getLedgerId() * 1000 + m2.getEntryId()); + }); + } + return res; + } + + private Consumer createConsumer(String topicName, String subName, boolean enabledBatch) throws Exception { + final Consumer consumer = pulsarClient.newConsumer(Schema.JSON(String.class)) + .autoScaledReceiverQueueSizeEnabled(false) + .subscriptionType(SubscriptionType.Failover) + .isAckReceiptEnabled(true) + .enableBatchIndexAcknowledgment(enabledBatch) + .receiverQueueSize(1000) + .topic(topicName) + .subscriptionName(subName) + .subscribe(); + return consumer; + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/MockedBookKeeperClientFactory.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/MockedBookKeeperClientFactory.java index 6d65687a501df..887e35e2774f7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/MockedBookKeeperClientFactory.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/MockedBookKeeperClientFactory.java @@ -19,9 +19,9 @@ package org.apache.pulsar.broker; import io.netty.channel.EventLoopGroup; -import java.io.IOException; import java.util.Map; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.EnsemblePlacementPolicy; @@ -51,19 +51,19 @@ public MockedBookKeeperClientFactory() { } @Override - public BookKeeper create(ServiceConfiguration conf, MetadataStoreExtended store, - EventLoopGroup eventLoopGroup, - Optional> ensemblePlacementPolicyClass, - Map properties) throws IOException { - return mockedBk; + public CompletableFuture create(ServiceConfiguration conf, MetadataStoreExtended store, + EventLoopGroup eventLoopGroup, + Optional> ensemblePlacementPolicyClass, + Map properties) { + return CompletableFuture.completedFuture(mockedBk); } @Override - public BookKeeper create(ServiceConfiguration conf, MetadataStoreExtended store, + public CompletableFuture create(ServiceConfiguration conf, MetadataStoreExtended store, EventLoopGroup eventLoopGroup, Optional> ensemblePlacementPolicyClass, - Map properties, StatsLogger statsLogger) throws IOException { - return mockedBk; + Map properties, StatsLogger statsLogger) { + return CompletableFuture.completedFuture(mockedBk); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/MultiBrokerTestZKBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/MultiBrokerTestZKBaseTest.java index 0cd5bce5d51cf..a78254df4aae0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/MultiBrokerTestZKBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/MultiBrokerTestZKBaseTest.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.broker; +import java.util.ArrayList; +import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.metadata.TestZKServer; @@ -32,6 +34,7 @@ @Slf4j public abstract class MultiBrokerTestZKBaseTest extends MultiBrokerBaseTest { TestZKServer testZKServer; + List storesToClose = new ArrayList<>(); @Override protected void doInitConf() throws Exception { @@ -42,18 +45,28 @@ protected void doInitConf() throws Exception { @Override protected void onCleanup() { super.onCleanup(); + for (MetadataStoreExtended store : storesToClose) { + try { + store.close(); + } catch (Exception e) { + log.error("Error in closing metadata store", e); + } + } + storesToClose.clear(); if (testZKServer != null) { try { testZKServer.close(); } catch (Exception e) { log.error("Error in stopping ZK server", e); } + testZKServer = null; } } @Override protected PulsarTestContext.Builder createPulsarTestContextBuilder(ServiceConfiguration conf) { return super.createPulsarTestContextBuilder(conf) + .spyNoneByDefault() .localMetadataStore(createMetadataStore(MetadataStoreConfig.METADATA_STORE)) .configurationMetadataStore(createMetadataStore(MetadataStoreConfig.CONFIGURATION_METADATA_STORE)); } @@ -61,8 +74,11 @@ protected PulsarTestContext.Builder createPulsarTestContextBuilder(ServiceConfig @NotNull protected MetadataStoreExtended createMetadataStore(String metadataStoreName) { try { - return MetadataStoreExtended.create(testZKServer.getConnectionString(), - MetadataStoreConfig.builder().metadataStoreName(metadataStoreName).build()); + MetadataStoreExtended store = + MetadataStoreExtended.create(testZKServer.getConnectionString(), + MetadataStoreConfig.builder().metadataStoreName(metadataStoreName).build()); + storesToClose.add(store); + return store; } catch (MetadataStoreException e) { throw new RuntimeException(e); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/PulsarServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/PulsarServiceTest.java index 37a7310ae17ca..dd2f9288071a5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/PulsarServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/PulsarServiceTest.java @@ -24,11 +24,14 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import static org.testng.AssertJUnit.assertSame; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerService; import org.testng.annotations.AfterMethod; @@ -54,12 +57,19 @@ protected void cleanup() throws Exception { @Override protected void doInitConf() throws Exception { super.doInitConf(); + conf.setBrokerServicePortTls(Optional.of(0)); + conf.setWebServicePortTls(Optional.of(0)); + conf.setTopicNameCacheMaxCapacity(5000); + conf.setMaxSecondsToClearTopicNameCache(5); if (useStaticPorts) { conf.setBrokerServicePortTls(Optional.of(6651)); conf.setBrokerServicePort(Optional.of(6660)); conf.setWebServicePort(Optional.of(8081)); conf.setWebServicePortTls(Optional.of(8082)); } + conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); } @Test @@ -186,7 +196,35 @@ public void testDynamicBrokerPort() throws Exception { } @Test - public void testBacklogAndRetentionCheck() { + public void testTopicCacheConfiguration() throws Exception { + cleanup(); + setup(); + assertEquals(conf.getTopicNameCacheMaxCapacity(), 5000); + assertEquals(conf.getMaxSecondsToClearTopicNameCache(), 5); + + List topicNameCached = new ArrayList<>(); + for (int i = 0; i < 20; i++) { + topicNameCached.add(TopicName.get("public/default/tp_" + i)); + } + + // Verify: the cache does not clear since it is not larger than max capacity. + Thread.sleep(10 * 1000); + for (int i = 0; i < 20; i++) { + assertTrue(topicNameCached.get(i) == TopicName.get("public/default/tp_" + i)); + } + + // Update max capacity. + admin.brokers().updateDynamicConfiguration("topicNameCacheMaxCapacity", "10"); + + // Verify: the cache were cleared. + Thread.sleep(10 * 1000); + for (int i = 0; i < 20; i++) { + assertFalse(topicNameCached.get(i) == TopicName.get("public/default/tp_" + i)); + } + } + + @Test + public void testBacklogAndRetentionCheck() throws PulsarServerException { ServiceConfiguration config = new ServiceConfiguration(); config.setClusterName("test"); config.setMetadataStoreUrl("memory:local"); @@ -198,6 +236,8 @@ public void testBacklogAndRetentionCheck() { pulsarService.start(); } catch (Exception e) { assertFalse(e.getCause() instanceof IllegalArgumentException); + } finally { + pulsarService.close(); } // Only set retention @@ -210,6 +250,8 @@ public void testBacklogAndRetentionCheck() { pulsarService.start(); } catch (Exception e) { assertFalse(e.getCause() instanceof IllegalArgumentException); + } finally { + pulsarService.close(); } // Set both retention and backlog quota @@ -222,6 +264,8 @@ public void testBacklogAndRetentionCheck() { pulsarService.start(); } catch (Exception e) { assertFalse(e.getCause() instanceof IllegalArgumentException); + } finally { + pulsarService.close(); } // Set invalidated retention and backlog quota @@ -233,6 +277,8 @@ public void testBacklogAndRetentionCheck() { pulsarService.start(); } catch (Exception e) { assertTrue(e.getCause() instanceof IllegalArgumentException); + } finally { + pulsarService.close(); } config.setBacklogQuotaDefaultLimitBytes(4 * 1024 * 1024); @@ -244,6 +290,8 @@ public void testBacklogAndRetentionCheck() { pulsarService.start(); } catch (Exception e) { assertTrue(e.getCause() instanceof IllegalArgumentException); + } finally { + pulsarService.close(); } // Only set backlog quota @@ -256,6 +304,8 @@ public void testBacklogAndRetentionCheck() { pulsarService.start(); } catch (Exception e) { assertFalse(e.getCause() instanceof IllegalArgumentException); + } finally { + pulsarService.close(); } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SLAMonitoringTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SLAMonitoringTest.java index 47949d7312b88..941229fc3d96c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SLAMonitoringTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SLAMonitoringTest.java @@ -102,9 +102,9 @@ void setup() throws Exception { createTenant(pulsarAdmins[BROKER_COUNT - 1]); for (int i = 0; i < BROKER_COUNT; i++) { - String topic = String.format("%s/%s/%s:%s", NamespaceService.SLA_NAMESPACE_PROPERTY, "my-cluster", - pulsarServices[i].getAdvertisedAddress(), brokerWebServicePorts[i]); - pulsarAdmins[0].namespaces().createNamespace(topic); + var namespaceName = NamespaceService.getSLAMonitorNamespace(pulsarServices[i].getBrokerId(), + pulsarServices[i].getConfig()); + pulsarAdmins[0].namespaces().createNamespace(namespaceName.toString()); } } @@ -126,15 +126,26 @@ private void createTenant(PulsarAdmin pulsarAdmin) @AfterClass(alwaysRun = true) public void shutdown() throws Exception { log.info("--- Shutting down ---"); - executor.shutdownNow(); - executor = null; + if (executor != null) { + executor.shutdownNow(); + executor = null; + } for (int i = 0; i < BROKER_COUNT; i++) { - pulsarAdmins[i].close(); - pulsarServices[i].close(); + if (pulsarAdmins[i] != null) { + pulsarAdmins[i].close(); + pulsarAdmins[i] = null; + } + if (pulsarServices[i] != null) { + pulsarServices[i].close(); + pulsarServices[i] = null; + } } - bkEnsemble.stop(); + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; + } } @Test @@ -173,9 +184,9 @@ public void testOwnedNamespaces() { public void testOwnershipViaAdminAfterSetup() { for (int i = 0; i < BROKER_COUNT; i++) { try { - String topic = String.format("persistent://%s/%s/%s:%s/%s", - NamespaceService.SLA_NAMESPACE_PROPERTY, "my-cluster", pulsarServices[i].getAdvertisedAddress(), - brokerWebServicePorts[i], "my-topic"); + String topic = String.format("persistent://%s/%s/%s/%s", + NamespaceService.SLA_NAMESPACE_PROPERTY, "my-cluster", + pulsarServices[i].getBrokerId(), "my-topic"); assertEquals(pulsarAdmins[0].lookups().lookupTopic(topic), "pulsar://" + pulsarServices[i].getAdvertisedAddress() + ":" + brokerNativeBrokerPorts[i]); } catch (Exception e) { @@ -199,8 +210,8 @@ public void testUnloadIfBrokerCrashes() { fail("Should be a able to close the broker index " + crashIndex + " Exception: " + e); } - String topic = String.format("persistent://%s/%s/%s:%s/%s", NamespaceService.SLA_NAMESPACE_PROPERTY, - "my-cluster", pulsarServices[crashIndex].getAdvertisedAddress(), brokerWebServicePorts[crashIndex], + String topic = String.format("persistent://%s/%s/%s/%s", NamespaceService.SLA_NAMESPACE_PROPERTY, + "my-cluster", pulsarServices[crashIndex].getBrokerId(), "my-topic"); log.info("Lookup for namespace {}", topic); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java new file mode 100644 index 0000000000000..b39f8135e0e0c --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker; + +import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.CA_CERT_FILE_PATH; +import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.getTlsFileForClient; +import static org.apache.pulsar.client.impl.SameAuthParamsLookupAutoClusterFailover.PulsarServiceState; +import io.netty.channel.EventLoopGroup; +import java.net.ServerSocket; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import org.apache.pulsar.broker.service.NetworkErrorTestBase; +import org.apache.pulsar.broker.service.OneWayReplicatorTestBase; +import org.apache.pulsar.client.api.ClientBuilder; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.SameAuthParamsLookupAutoClusterFailover; +import org.apache.pulsar.client.impl.auth.AuthenticationTls; +import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class SameAuthParamsLookupAutoClusterFailoverTest extends OneWayReplicatorTestBase { + + public void setup() throws Exception { + super.setup(); + } + + @Override + @AfterMethod(alwaysRun = true, timeOut = 300000) + public void cleanup() throws Exception { + super.cleanup(); + } + + @DataProvider(name = "enabledTls") + public Object[][] enabledTls () { + return new Object[][] { + {true}, + {false} + }; + } + + @Test(dataProvider = "enabledTls", timeOut = 240 * 1000) + public void testAutoClusterFailover(boolean enabledTls) throws Exception { + // Start clusters. + setup(); + ServerSocket dummyServer = new ServerSocket(NetworkErrorTestBase.getOneFreePort()); + + // Initialize client. + String urlProxy = enabledTls ? "pulsar+tls://127.0.0.1:" + dummyServer.getLocalPort() + : "pulsar://127.0.0.1:" + dummyServer.getLocalPort(); + String url1 = enabledTls ? pulsar1.getBrokerServiceUrlTls() : pulsar1.getBrokerServiceUrl(); + String url2 = enabledTls ? pulsar2.getBrokerServiceUrlTls() : pulsar2.getBrokerServiceUrl(); + final String[] urlArray = new String[]{url1, urlProxy, url2}; + final SameAuthParamsLookupAutoClusterFailover failover = SameAuthParamsLookupAutoClusterFailover.builder() + .pulsarServiceUrlArray(urlArray) + .failoverThreshold(5) + .recoverThreshold(5) + .checkHealthyIntervalMs(300) + .testTopic("a/b/c") + .markTopicNotFoundAsAvailable(true) + .build(); + ClientBuilder clientBuilder = PulsarClient.builder().serviceUrlProvider(failover); + if (enabledTls) { + Map authParams = new HashMap<>(); + authParams.put("tlsCertFile", getTlsFileForClient("admin.cert")); + authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8")); + clientBuilder.authentication(AuthenticationTls.class.getName(), authParams) + .enableTls(true) + .allowTlsInsecureConnection(false) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH); + } + final PulsarClient client = clientBuilder.build(); + failover.initialize(client); + final EventLoopGroup executor = WhiteboxImpl.getInternalState(failover, "executor"); + final PulsarServiceState[] stateArray = + WhiteboxImpl.getInternalState(failover, "pulsarServiceStateArray"); + + // Test all things is fine. + final String tp = BrokerTestUtil.newUniqueName(nonReplicatedNamespace + "/tp"); + final Producer producer = client.newProducer(Schema.STRING).topic(tp).create(); + producer.send("0"); + Assert.assertEquals(failover.getCurrentPulsarServiceIndex(), 0); + + CompletableFuture checkStatesFuture1 = new CompletableFuture<>(); + executor.submit(() -> { + boolean res = stateArray[0] == PulsarServiceState.Healthy; + res = res & stateArray[1] == PulsarServiceState.Healthy; + res = res & stateArray[2] == PulsarServiceState.Healthy; + checkStatesFuture1.complete(res); + }); + Assert.assertTrue(checkStatesFuture1.join()); + + // Test failover 0 --> 3. + pulsar1.close(); + Awaitility.await().atMost(60, TimeUnit.SECONDS).untilAsserted(() -> { + CompletableFuture checkStatesFuture2 = new CompletableFuture<>(); + executor.submit(() -> { + boolean res = stateArray[0] == PulsarServiceState.Failed; + res = res & stateArray[1] == PulsarServiceState.Failed; + res = res & stateArray[2] == PulsarServiceState.Healthy; + checkStatesFuture2.complete(res); + }); + Assert.assertTrue(checkStatesFuture2.join()); + producer.send("0->2"); + Assert.assertEquals(failover.getCurrentPulsarServiceIndex(), 2); + }); + + // Test recover 2 --> 1. + executor.execute(() -> { + urlArray[1] = url2; + }); + Awaitility.await().atMost(60, TimeUnit.SECONDS).untilAsserted(() -> { + CompletableFuture checkStatesFuture3 = new CompletableFuture<>(); + executor.submit(() -> { + boolean res = stateArray[0] == PulsarServiceState.Failed; + res = res & stateArray[1] == PulsarServiceState.Healthy; + res = res & stateArray[2] == PulsarServiceState.Healthy; + checkStatesFuture3.complete(res); + }); + Assert.assertTrue(checkStatesFuture3.join()); + producer.send("2->1"); + Assert.assertEquals(failover.getCurrentPulsarServiceIndex(), 1); + }); + + // Test recover 1 --> 0. + executor.execute(() -> { + urlArray[0] = url2; + }); + Awaitility.await().atMost(60, TimeUnit.SECONDS).untilAsserted(() -> { + CompletableFuture checkStatesFuture4 = new CompletableFuture<>(); + executor.submit(() -> { + boolean res = stateArray[0] == PulsarServiceState.Healthy; + res = res & stateArray[1] == PulsarServiceState.Healthy; + res = res & stateArray[2] == PulsarServiceState.Healthy; + checkStatesFuture4.complete(res); + }); + Assert.assertTrue(checkStatesFuture4.join()); + producer.send("1->0"); + Assert.assertEquals(failover.getCurrentPulsarServiceIndex(), 0); + }); + + // cleanup. + producer.close(); + client.close(); + dummyServer.close(); + } + + @Override + protected void cleanupPulsarResources() { + // Nothing to do. + } + +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/TopicEventsListenerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/TopicEventsListenerTest.java index e6459bbf74c31..ceb3c1d0d9335 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/TopicEventsListenerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/TopicEventsListenerTest.java @@ -126,7 +126,7 @@ public void testEvents(String topicTypePersistence, String topicTypePartitioned, boolean forceDelete) throws Exception { String topicName = topicTypePersistence + "://" + namespace + "/" + "topic-" + UUID.randomUUID(); - createTopicAndVerifyEvents(topicTypePartitioned, topicName); + createTopicAndVerifyEvents(topicTypePersistence, topicTypePartitioned, topicName); events.clear(); if (topicTypePartitioned.equals("partitioned")) { @@ -150,7 +150,7 @@ public void testEventsWithUnload(String topicTypePersistence, String topicTypePa boolean forceDelete) throws Exception { String topicName = topicTypePersistence + "://" + namespace + "/" + "topic-" + UUID.randomUUID(); - createTopicAndVerifyEvents(topicTypePartitioned, topicName); + createTopicAndVerifyEvents(topicTypePersistence, topicTypePartitioned, topicName); events.clear(); admin.topics().unload(topicName); @@ -182,7 +182,7 @@ public void testEventsActiveSub(String topicTypePersistence, String topicTypePar boolean forceDelete) throws Exception { String topicName = topicTypePersistence + "://" + namespace + "/" + "topic-" + UUID.randomUUID(); - createTopicAndVerifyEvents(topicTypePartitioned, topicName); + createTopicAndVerifyEvents(topicTypePersistence, topicTypePartitioned, topicName); Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("sub").subscribe(); Producer producer = pulsarClient.newProducer().topic(topicName).create(); @@ -238,7 +238,7 @@ public void testEventsActiveSub(String topicTypePersistence, String topicTypePar public void testTopicAutoGC(String topicTypePersistence, String topicTypePartitioned) throws Exception { String topicName = topicTypePersistence + "://" + namespace + "/" + "topic-" + UUID.randomUUID(); - createTopicAndVerifyEvents(topicTypePartitioned, topicName); + createTopicAndVerifyEvents(topicTypePersistence, topicTypePartitioned, topicName); admin.namespaces().setInactiveTopicPolicies(namespace, new InactiveTopicPolicies(InactiveTopicDeleteMode.delete_when_no_subscriptions, 1, true)); @@ -262,25 +262,21 @@ public void testTopicAutoGC(String topicTypePersistence, String topicTypePartiti ); } - private void createTopicAndVerifyEvents(String topicTypePartitioned, String topicName) throws Exception { + private void createTopicAndVerifyEvents(String topicDomain, String topicTypePartitioned, String topicName) throws Exception { final String[] expectedEvents; - if (topicTypePartitioned.equals("partitioned")) { - topicNameToWatch = topicName + "-partition-1"; - admin.topics().createPartitionedTopic(topicName, 2); - triggerPartitionsCreation(topicName); - + if (topicDomain.equalsIgnoreCase("persistent") || topicTypePartitioned.equals("partitioned")) { expectedEvents = new String[]{ "LOAD__BEFORE", "CREATE__BEFORE", "CREATE__SUCCESS", "LOAD__SUCCESS" }; - } else { - topicNameToWatch = topicName; - admin.topics().createNonPartitionedTopic(topicName); - expectedEvents = new String[]{ + // Before https://github.com/apache/pulsar/pull/21995, Pulsar will skip create topic if the topic + // was already exists, and the action "check topic exists" will try to load Managed ledger, + // the check triggers two exrtra events: [LOAD__BEFORE, LOAD__FAILURE]. + // #21995 fixed this wrong behavior, so remove these two events. "LOAD__BEFORE", "LOAD__FAILURE", "LOAD__BEFORE", @@ -288,7 +284,14 @@ private void createTopicAndVerifyEvents(String topicTypePartitioned, String topi "CREATE__SUCCESS", "LOAD__SUCCESS" }; - + } + if (topicTypePartitioned.equals("partitioned")) { + topicNameToWatch = topicName + "-partition-1"; + admin.topics().createPartitionedTopic(topicName, 2); + triggerPartitionsCreation(topicName); + } else { + topicNameToWatch = topicName; + admin.topics().createNonPartitionedTopic(topicName); } Awaitility.waitAtMost(10, TimeUnit.SECONDS).untilAsserted(() -> diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java index 7ed5fe34ea4f7..900babbecf4ad 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java @@ -18,7 +18,13 @@ */ package org.apache.pulsar.broker.admin; +import static java.util.concurrent.TimeUnit.MINUTES; import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.pulsar.broker.BrokerTestUtil.newUniqueName; +import static org.apache.pulsar.broker.resources.LoadBalanceResources.BUNDLE_DATA_BASE_PATH; +import static org.apache.pulsar.common.policies.data.NamespaceIsolationPolicyUnloadScope.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -35,6 +41,7 @@ import java.lang.reflect.Field; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.time.Clock; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -47,7 +54,9 @@ import java.util.TreeSet; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import javax.ws.rs.NotAcceptableException; import javax.ws.rs.core.Response.Status; import lombok.AllArgsConstructor; @@ -64,7 +73,9 @@ import org.apache.pulsar.broker.admin.AdminApiTest.MockedPulsarService; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerWrapper; import org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl; +import org.apache.pulsar.broker.service.AbstractTopic; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -100,33 +111,18 @@ import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.AutoFailoverPolicyData; -import org.apache.pulsar.common.policies.data.AutoFailoverPolicyType; -import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; -import org.apache.pulsar.common.policies.data.BacklogQuota; -import org.apache.pulsar.common.policies.data.BrokerNamespaceIsolationData; -import org.apache.pulsar.common.policies.data.BrokerNamespaceIsolationDataImpl; -import org.apache.pulsar.common.policies.data.BundlesData; -import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.ConsumerStats; -import org.apache.pulsar.common.policies.data.EntryFilters; -import org.apache.pulsar.common.policies.data.FailureDomain; -import org.apache.pulsar.common.policies.data.NamespaceIsolationData; -import org.apache.pulsar.common.policies.data.NonPersistentTopicStats; -import org.apache.pulsar.common.policies.data.PartitionedTopicStats; -import org.apache.pulsar.common.policies.data.PersistencePolicies; -import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; -import org.apache.pulsar.common.policies.data.RetentionPolicies; -import org.apache.pulsar.common.policies.data.SubscriptionStats; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; -import org.apache.pulsar.common.policies.data.TopicStats; -import org.apache.pulsar.common.policies.data.TopicType; +import org.apache.pulsar.common.policies.data.*; import org.apache.pulsar.common.policies.data.impl.BacklogQuotaImpl; +import org.apache.pulsar.common.protocol.schema.SchemaData; import org.awaitility.Awaitility; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -135,6 +131,10 @@ public class AdminApi2Test extends MockedPulsarServiceBaseTest { private MockedPulsarService mockPulsarSetup; + private boolean restartClusterAfterTest; + private int usageCount; + private String defaultNamespace; + private String defaultTenant; @BeforeClass @Override @@ -149,13 +149,49 @@ public void setup() throws Exception { setupClusters(); } + @Test + public void testExceptionOfMaxTopicsPerNamespaceCanBeHanle() throws Exception { + super.internalCleanup(); + conf.setMaxTopicsPerNamespace(3); + super.internalSetup(); + String topic = "persistent://testTenant/ns1/test_create_topic_v"; + TenantInfoImpl tenantInfo = new TenantInfoImpl(Sets.newHashSet("role1", "role2"), + Sets.newHashSet("test")); + // check producer/consumer auto create non-partitioned topic + conf.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED); + admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(brokerUrl.toString()).build()); + admin.tenants().createTenant("testTenant", tenantInfo); + admin.namespaces().createNamespace("testTenant/ns1", Sets.newHashSet("test")); + + pulsarClient.newProducer().topic(topic + "1").create().close(); + pulsarClient.newProducer().topic(topic + "2").create().close(); + pulsarClient.newConsumer().topic(topic + "3").subscriptionName("test_sub").subscribe().close(); + try { + pulsarClient.newConsumer().topic(topic + "4").subscriptionName("test_sub") + .subscribeAsync().get(5, TimeUnit.SECONDS); + Assert.fail(); + } catch (Exception e) { + assertTrue(e.getCause() instanceof PulsarClientException.NotAllowedException); + } + + // reset configuration + conf.setMaxTopicsPerNamespace(0); + conf.setDefaultNumPartitions(1); + } + @Override protected ServiceConfiguration getDefaultConf() { ServiceConfiguration conf = super.getDefaultConf(); + configureDefaults(conf); + return conf; + } + + void configureDefaults(ServiceConfiguration conf) { conf.setForceDeleteNamespaceAllowed(true); conf.setLoadBalancerEnabled(true); - conf.setEnableNamespaceIsolationUpdateOnTime(true); - return conf; + conf.setAllowOverrideEntryFilters(true); + conf.setEntryFilterNames(List.of()); + conf.setMaxNumPartitionsPerPartitionedTopic(0); } @AfterClass(alwaysRun = true) @@ -171,6 +207,20 @@ public void cleanup() throws Exception { @AfterMethod(alwaysRun = true) public void resetClusters() throws Exception { + if (restartClusterAfterTest) { + restartClusterAndResetUsageCount(); + } else { + try { + cleanupCluster(); + } catch (Exception e) { + log.error("Failed to clean up state by deleting namespaces and tenants after test. " + + "Restarting the test broker.", e); + restartClusterAndResetUsageCount(); + } + } + } + + private void cleanupCluster() throws Exception { pulsar.getConfiguration().setForceDeleteTenantAllowed(true); pulsar.getConfiguration().setForceDeleteNamespaceAllowed(true); for (String tenant : admin.tenants().getTenants()) { @@ -178,22 +228,59 @@ public void resetClusters() throws Exception { deleteNamespaceWithRetry(namespace, true, admin, pulsar, mockPulsarSetup.getPulsar()); } - admin.tenants().deleteTenant(tenant, true); + try { + admin.tenants().deleteTenant(tenant, true); + } catch (Exception e) { + log.error("Failed to delete tenant {} after test", tenant, e); + String zkDirectory = "/managed-ledgers/" + tenant; + try { + log.info("Listing {} to see if existing keys are preventing deletion.", zkDirectory); + pulsar.getPulsarResources().getLocalMetadataStore().get().getChildren(zkDirectory) + .get(5, TimeUnit.SECONDS).forEach(key -> log.info("Child key '{}'", key)); + } catch (Exception ignore) { + log.error("Failed to list tenant {} ZK directory {} after test", tenant, zkDirectory, e); + } + throw e; + } } for (String cluster : admin.clusters().getClusters()) { admin.clusters().deleteCluster(cluster); } - resetConfig(); + configureDefaults(conf); setupClusters(); } + private void restartClusterAfterTest() { + restartClusterAfterTest = true; + } + + private void restartClusterAndResetUsageCount() throws Exception { + cleanup(); + restartClusterAfterTest = false; + usageCount = 0; + setup(); + } + + private void restartClusterIfReused() throws Exception { + if (usageCount > 1) { + restartClusterAndResetUsageCount(); + } + } + + @BeforeMethod + public void increaseUsageCount() { + usageCount++; + } + private void setupClusters() throws PulsarAdminException { admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test")); - admin.tenants().createTenant("prop-xyz", tenantInfo); - admin.namespaces().createNamespace("prop-xyz/ns1", Set.of("test")); + defaultTenant = newUniqueName("prop-xyz"); + admin.tenants().createTenant(defaultTenant, tenantInfo); + defaultNamespace = defaultTenant + "/ns1"; + admin.namespaces().createNamespace(defaultNamespace, Set.of("test")); } @DataProvider(name = "topicType") @@ -250,7 +337,7 @@ public void testIncrementPartitionsOfTopic() throws Exception { final String subName2 = topicName + "-my-sub-2/encode"; final int startPartitions = 4; final int newPartitions = 8; - final String partitionedTopicName = "persistent://prop-xyz/ns1/" + topicName; + final String partitionedTopicName = "persistent://" + defaultNamespace + "/" + topicName; URL pulsarUrl = new URL(pulsar.getWebServiceAddress()); @@ -299,7 +386,7 @@ public void testIncrementPartitionsOfTopic() throws Exception { assertEquals(new HashSet<>(admin.topics().getSubscriptions(newPartitionTopicName)), Set.of(subName1, subName2)); - assertEquals(new HashSet<>(admin.topics().getList("prop-xyz/ns1")).size(), newPartitions); + assertEquals(new HashSet<>(admin.topics().getList(defaultNamespace)).size(), newPartitions); // test cumulative stats for partitioned topic PartitionedTopicStats topicStats = admin.topics().getPartitionedStats(partitionedTopicName, false); @@ -333,27 +420,22 @@ public void testIncrementPartitionsOfTopic() throws Exception { @Test public void testTopicPoliciesWithMultiBroker() throws Exception { + restartClusterAfterTest(); + //setup cluster with 3 broker - cleanup(); - setup(); admin.clusters().updateCluster("test", ClusterData.builder().serviceUrl((pulsar.getWebServiceAddress() + ",localhost:1026," + "localhost:2050")).build()); TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test")); - admin.tenants().createTenant("prop-xyz2", tenantInfo); - admin.namespaces().createNamespace("prop-xyz2/ns1", Set.of("test")); - conf.setBrokerServicePort(Optional.of(1024)); - conf.setBrokerServicePortTls(Optional.of(1025)); - conf.setWebServicePort(Optional.of(1026)); - conf.setWebServicePortTls(Optional.of(1027)); + String tenantName = newUniqueName("prop-xyz2"); + admin.tenants().createTenant(tenantName, tenantInfo); + admin.namespaces().createNamespace(tenantName + "/ns1", Set.of("test")); + ServiceConfiguration config2 = super.getDefaultConf(); @Cleanup - PulsarTestContext pulsarTestContext2 = createAdditionalPulsarTestContext(conf); + PulsarTestContext pulsarTestContext2 = createAdditionalPulsarTestContext(config2); PulsarService pulsar2 = pulsarTestContext2.getPulsarService(); - conf.setBrokerServicePort(Optional.of(2048)); - conf.setBrokerServicePortTls(Optional.of(2049)); - conf.setWebServicePort(Optional.of(2050)); - conf.setWebServicePortTls(Optional.of(2051)); + ServiceConfiguration config3 = super.getDefaultConf(); @Cleanup - PulsarTestContext pulsarTestContext3 = createAdditionalPulsarTestContext(conf); + PulsarTestContext pulsarTestContext3 = createAdditionalPulsarTestContext(config3); PulsarService pulsar3 = pulsarTestContext.getPulsarService(); @Cleanup PulsarAdmin admin2 = PulsarAdmin.builder().serviceHttpUrl(pulsar2.getWebServiceAddress()).build(); @@ -361,14 +443,14 @@ public void testTopicPoliciesWithMultiBroker() throws Exception { PulsarAdmin admin3 = PulsarAdmin.builder().serviceHttpUrl(pulsar3.getWebServiceAddress()).build(); //for partitioned topic, we can get topic policies from every broker - final String topic = "persistent://prop-xyz2/ns1/" + BrokerTestUtil.newUniqueName("test"); + final String topic = "persistent://" + tenantName + "/ns1/" + newUniqueName("test"); int partitionNum = 3; admin.topics().createPartitionedTopic(topic, partitionNum); pulsarClient.newConsumer().topic(topic).subscriptionName("sub").subscribe().close(); setTopicPoliciesAndValidate(admin2, admin3, topic); //for non-partitioned topic, we can get topic policies from every broker - final String topic2 = "persistent://prop-xyz2/ns1/" + BrokerTestUtil.newUniqueName("test"); + final String topic2 = "persistent://" + tenantName + "/ns1/" + newUniqueName("test"); pulsarClient.newConsumer().topic(topic2).subscriptionName("sub").subscribe().close(); setTopicPoliciesAndValidate(admin2, admin3, topic2); } @@ -402,7 +484,7 @@ private void setTopicPoliciesAndValidate(PulsarAdmin admin2 public void nonPersistentTopics() throws Exception { final String topicName = "nonPersistentTopic"; - final String nonPersistentTopicName = "non-persistent://prop-xyz/ns1/" + topicName; + final String nonPersistentTopicName = "non-persistent://" + defaultNamespace + "/" + topicName; // Force to create a topic publishMessagesOnTopic(nonPersistentTopicName, 0, 0); @@ -423,8 +505,7 @@ public void nonPersistentTopics() throws Exception { assertEquals(topicStats.getSubscriptions().get("my-sub").getMsgDropRate(), 0); assertEquals(topicStats.getPublishers().size(), 0); assertEquals(topicStats.getMsgDropRate(), 0); - assertEquals(topicStats.getOwnerBroker(), - pulsar.getAdvertisedAddress() + ":" + pulsar.getConfiguration().getWebServicePort().get()); + assertEquals(topicStats.getOwnerBroker(), pulsar.getBrokerId()); PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(nonPersistentTopicName, false); assertEquals(internalStats.cursors.keySet(), Set.of("my-sub")); @@ -434,7 +515,7 @@ public void nonPersistentTopics() throws Exception { assertFalse(topicStats.getSubscriptions().containsKey("my-sub")); assertEquals(topicStats.getPublishers().size(), 0); // test partitioned-topic - final String partitionedTopicName = "non-persistent://prop-xyz/ns1/paritioned"; + final String partitionedTopicName = "non-persistent://" + defaultNamespace + "/paritioned"; admin.topics().createPartitionedTopic(partitionedTopicName, 5); assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 5); } @@ -462,7 +543,7 @@ private void publishMessagesOnTopic(String topicName, int messages, int startIdx @Test public void testSetPersistencePolicies() throws Exception { - final String namespace = "prop-xyz/ns2"; + final String namespace = newUniqueName(defaultTenant + "/ns2"); admin.namespaces().createNamespace(namespace, Set.of("test")); assertNull(admin.namespaces().getPersistence(namespace)); @@ -500,7 +581,7 @@ public void testSetPersistencePolicies() throws Exception { @Test public void testUpdatePersistencePolicyUpdateManagedCursor() throws Exception { - final String namespace = "prop-xyz/ns2"; + final String namespace = newUniqueName(defaultTenant + "/ns2"); final String topicName = "persistent://" + namespace + "/topic1"; admin.namespaces().createNamespace(namespace, Set.of("test")); @@ -541,7 +622,7 @@ public void testUpdatePersistencePolicyUpdateManagedCursor() throws Exception { @Test(dataProvider = "topicType") public void testUnloadTopic(final String topicType) throws Exception { - final String namespace = "prop-xyz/ns2"; + final String namespace = newUniqueName(defaultTenant + "/ns2"); final String topicName = topicType + "://" + namespace + "/topic1"; admin.namespaces().createNamespace(namespace, Set.of("test")); @@ -591,11 +672,12 @@ private void unloadTopic(String topicName) throws Exception { */ @Test(dataProvider = "namespaceNames", timeOut = 30000) public void testResetCursorOnPosition(String namespaceName) throws Exception { - final String topicName = "persistent://prop-xyz/use/" + namespaceName + "/resetPosition"; + restartClusterAfterTest(); + final String topicName = "persistent://" + defaultTenant + "/use/" + namespaceName + "/resetPosition"; final int totalProducedMessages = 50; // set retention - admin.namespaces().setRetention("prop-xyz/ns1", new RetentionPolicies(10, 10)); + admin.namespaces().setRetention(defaultNamespace, new RetentionPolicies(10, 10)); // create consumer and subscription Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub") @@ -674,14 +756,11 @@ public void testResetCursorOnPosition(String namespaceName) throws Exception { } consumer.close(); - - cleanup(); - setup(); } @Test public void shouldNotSupportResetOnPartitionedTopic() throws PulsarAdminException, PulsarClientException { - final String partitionedTopicName = "persistent://prop-xyz/ns1/" + BrokerTestUtil.newUniqueName("parttopic"); + final String partitionedTopicName = "persistent://" + defaultNamespace + "/" + newUniqueName("parttopic"); admin.topics().createPartitionedTopic(partitionedTopicName, 4); @Cleanup Consumer consumer = pulsarClient.newConsumer().topic(partitionedTopicName).subscriptionName("my-sub") @@ -713,7 +792,9 @@ private void publishMessagesOnPersistentTopic(String topicName, int messages, in @Test(timeOut = 20000) public void testMaxConsumersOnSubApi() throws Exception { - final String namespace = "prop-xyz/ns1"; + final String namespace = newUniqueName(defaultTenant + "/ns"); + admin.namespaces().createNamespace(namespace, Set.of("test")); + assertNull(admin.namespaces().getMaxConsumersPerSubscription(namespace)); admin.namespaces().setMaxConsumersPerSubscription(namespace, 10); Awaitility.await().untilAsserted(() -> { @@ -732,7 +813,7 @@ public void testMaxConsumersOnSubApi() throws Exception { */ @Test public void testLoadReportApi() throws Exception { - + restartClusterAfterTest(); this.conf.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName()); @Cleanup("cleanup") MockedPulsarService mockPulsarSetup1 = new MockedPulsarService(this.conf); @@ -795,6 +876,8 @@ public void testPeerCluster() throws Exception { */ @Test public void testReplicationPeerCluster() throws Exception { + restartClusterAfterTest(); + admin.clusters().createCluster("us-west1", ClusterData.builder().serviceUrl("http://broker.messaging.west1.example.com:8080").build()); admin.clusters().createCluster("us-west2", @@ -814,7 +897,7 @@ public void testReplicationPeerCluster() throws Exception { assertEquals(allClusters, List.of("test", "us-east1", "us-east2", "us-west1", "us-west2", "us-west3", "us-west4")); - final String property = "peer-prop"; + final String property = newUniqueName("peer-prop"); Set allowedClusters = Set.of("us-west1", "us-west2", "us-west3", "us-west4", "us-east1", "us-east2", "global"); TenantInfoImpl propConfig = new TenantInfoImpl(Set.of("test"), allowedClusters); @@ -848,9 +931,6 @@ public void testReplicationPeerCluster() throws Exception { clusterIds = Set.of("us-west1", "us-west4"); // no peer coexist in replication clusters admin.namespaces().setNamespaceReplicationClusters(namespace, clusterIds); - - cleanup(); - setup(); } @Test @@ -889,7 +969,7 @@ public void clusterFailureDomain() throws PulsarAdminException { @Test public void namespaceAntiAffinity() throws PulsarAdminException { - final String namespace = "prop-xyz/ns1"; + final String namespace = defaultNamespace; final String antiAffinityGroup = "group"; assertTrue(isBlank(admin.namespaces().getNamespaceAntiAffinityGroup(namespace))); admin.namespaces().setNamespaceAntiAffinityGroup(namespace, antiAffinityGroup); @@ -897,9 +977,9 @@ public void namespaceAntiAffinity() throws PulsarAdminException { admin.namespaces().deleteNamespaceAntiAffinityGroup(namespace); assertTrue(isBlank(admin.namespaces().getNamespaceAntiAffinityGroup(namespace))); - final String ns1 = "prop-xyz/antiAG1"; - final String ns2 = "prop-xyz/antiAG2"; - final String ns3 = "prop-xyz/antiAG3"; + final String ns1 = defaultTenant + "/antiAG1"; + final String ns2 = defaultTenant + "/antiAG2"; + final String ns3 = defaultTenant + "/antiAG3"; admin.namespaces().createNamespace(ns1, Set.of("test")); admin.namespaces().createNamespace(ns2, Set.of("test")); admin.namespaces().createNamespace(ns3, Set.of("test")); @@ -908,19 +988,19 @@ public void namespaceAntiAffinity() throws PulsarAdminException { admin.namespaces().setNamespaceAntiAffinityGroup(ns3, antiAffinityGroup); Set namespaces = new HashSet<>( - admin.namespaces().getAntiAffinityNamespaces("prop-xyz", "test", antiAffinityGroup)); + admin.namespaces().getAntiAffinityNamespaces(defaultTenant, "test", antiAffinityGroup)); assertEquals(namespaces.size(), 3); assertTrue(namespaces.contains(ns1)); assertTrue(namespaces.contains(ns2)); assertTrue(namespaces.contains(ns3)); - List namespaces2 = admin.namespaces().getAntiAffinityNamespaces("prop-xyz", "test", "invalid-group"); + List namespaces2 = admin.namespaces().getAntiAffinityNamespaces(defaultTenant, "test", "invalid-group"); assertEquals(namespaces2.size(), 0); } @Test public void testPersistentTopicList() throws Exception { - final String namespace = "prop-xyz/ns2"; + final String namespace = newUniqueName(defaultTenant + "/ns2"); final String topicName = "non-persistent://" + namespace + "/bundle-topic"; admin.namespaces().createNamespace(namespace, 20); admin.namespaces().setNamespaceReplicationClusters(namespace, Set.of("test")); @@ -954,7 +1034,7 @@ public void testPersistentTopicList() throws Exception { @Test public void testCreateAndGetTopicProperties() throws Exception { - final String namespace = "prop-xyz/ns2"; + final String namespace = newUniqueName(defaultTenant + "/ns2"); final String nonPartitionedTopicName = "persistent://" + namespace + "/non-partitioned-TopicProperties"; admin.namespaces().createNamespace(namespace, 20); Map nonPartitionedTopicProperties = new HashMap<>(); @@ -975,7 +1055,7 @@ public void testCreateAndGetTopicProperties() throws Exception { @Test public void testUpdatePartitionedTopicProperties() throws Exception { - final String namespace = "prop-xyz/ns2"; + final String namespace = newUniqueName(defaultTenant + "/ns2"); final String topicName = "persistent://" + namespace + "/testUpdatePartitionedTopicProperties"; final String topicNameTwo = "persistent://" + namespace + "/testUpdatePartitionedTopicProperties2"; admin.namespaces().createNamespace(namespace, 20); @@ -1033,7 +1113,7 @@ public void testUpdatePartitionedTopicProperties() throws Exception { @Test public void testUpdateNonPartitionedTopicProperties() throws Exception { - final String namespace = "prop-xyz/ns2"; + final String namespace = newUniqueName(defaultTenant + "/ns2"); final String topicName = "persistent://" + namespace + "/testUpdateNonPartitionedTopicProperties"; admin.namespaces().createNamespace(namespace, 20); @@ -1068,7 +1148,7 @@ public void testUpdateNonPartitionedTopicProperties() throws Exception { @Test public void testNonPersistentTopics() throws Exception { - final String namespace = "prop-xyz/ns2"; + final String namespace = newUniqueName(defaultTenant + "/ns2"); final String topicName = "non-persistent://" + namespace + "/topic"; admin.namespaces().createNamespace(namespace, 20); admin.namespaces().setNamespaceReplicationClusters(namespace, Set.of("test")); @@ -1096,7 +1176,7 @@ public void testNonPersistentTopics() throws Exception { public void testPublishConsumerStats() throws Exception { final String topicName = "statTopic"; final String subscriberName = topicName + "-my-sub-1"; - final String topic = "persistent://prop-xyz/ns1/" + topicName; + final String topic = "persistent://" + defaultNamespace + "/" + topicName; final String producerName = "myProducer"; @Cleanup @@ -1143,6 +1223,8 @@ public void testPublishConsumerStats() throws Exception { @Test public void testTenantNameWithUnderscore() throws Exception { + restartClusterAfterTest(); + TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test")); admin.tenants().createTenant("prop_xyz", tenantInfo); @@ -1150,15 +1232,12 @@ public void testTenantNameWithUnderscore() throws Exception { String topic = "persistent://prop_xyz/use/my-namespace/my-topic"; + @Cleanup Producer producer = pulsarClient.newProducer().topic(topic) .create(); TopicStats stats = admin.topics().getStats(topic); assertEquals(stats.getPublishers().size(), 1); - producer.close(); - - cleanup(); - setup(); } @Test @@ -1200,11 +1279,11 @@ public void testTenantWithNonexistentClusters() throws Exception { assertFalse(admin.tenants().getTenants().contains("test-tenant")); // Check existing tenant - assertTrue(admin.tenants().getTenants().contains("prop-xyz")); + assertTrue(admin.tenants().getTenants().contains(defaultTenant)); // If we try to update existing tenant with nonexistent clusters, it should fail immediately try { - admin.tenants().updateTenant("prop-xyz", tenantInfo); + admin.tenants().updateTenant(defaultTenant, tenantInfo); fail("Should have failed"); } catch (PulsarAdminException e) { assertEquals(e.getStatusCode(), Status.PRECONDITION_FAILED.getStatusCode()); @@ -1219,7 +1298,7 @@ public void brokerNamespaceIsolationPolicies() throws Exception { String cluster = pulsar.getConfiguration().getClusterName(); String namespaceRegex = "other/" + cluster + "/other.*"; String brokerName = pulsar.getAdvertisedAddress(); - String brokerAddress = brokerName + ":" + pulsar.getConfiguration().getWebServicePort().get(); + String brokerAddress = pulsar.getBrokerId(); Map parameters1 = new HashMap<>(); parameters1.put("min_limit", "1"); @@ -1227,7 +1306,7 @@ public void brokerNamespaceIsolationPolicies() throws Exception { NamespaceIsolationData nsPolicyData1 = NamespaceIsolationData.builder() .namespaces(Collections.singletonList(namespaceRegex)) - .primary(Collections.singletonList(brokerName + ":[0-9]*")) + .primary(Collections.singletonList(brokerName)) .secondary(Collections.singletonList(brokerName + ".*")) .autoFailoverPolicy(AutoFailoverPolicyData.builder() .policyType(AutoFailoverPolicyType.min_available) @@ -1264,7 +1343,7 @@ public void brokerNamespaceIsolationPolicies() throws Exception { @Test public void brokerNamespaceIsolationPoliciesUpdateOnTime() throws Exception { String brokerName = pulsar.getAdvertisedAddress(); - String ns1Name = "prop-xyz/test_ns1_iso_" + System.currentTimeMillis(); + String ns1Name = defaultTenant + "/test_ns1_iso_" + System.currentTimeMillis(); admin.namespaces().createNamespace(ns1Name, Set.of("test")); // 0. without isolation policy configured, lookup will success. @@ -1304,6 +1383,7 @@ public void brokerNamespaceIsolationPoliciesUpdateOnTime() throws Exception { try { admin.lookups().lookupTopic(ns1Name + "/topic3"); + fail(); } catch (Exception e) { // expected lookup fail, because no brokers matched the policy. log.info(" 2 expected fail lookup"); @@ -1311,6 +1391,7 @@ public void brokerNamespaceIsolationPoliciesUpdateOnTime() throws Exception { try { admin.lookups().lookupTopic(ns1Name + "/topic1"); + fail(); } catch (Exception e) { // expected lookup fail, because no brokers matched the policy. log.info(" 22 expected fail lookup"); @@ -1335,36 +1416,25 @@ public void clustersList() throws PulsarAdminException { */ @Test public void testClusterIsReadyBeforeCreateTopic() throws Exception { + restartClusterAfterTest(); final String topicName = "partitionedTopic"; final int partitions = 4; - final String persistentPartitionedTopicName = "persistent://prop-xyz/ns2/" + topicName; - final String NonPersistentPartitionedTopicName = "non-persistent://prop-xyz/ns2/" + topicName; + final String persistentPartitionedTopicName = "persistent://" + defaultTenant + "/ns2/" + topicName; + final String NonPersistentPartitionedTopicName = "non-persistent://" + defaultTenant + "/ns2/" + topicName; - admin.namespaces().createNamespace("prop-xyz/ns2"); + admin.namespaces().createNamespace(defaultTenant + "/ns2"); // By default the cluster will configure as configuration file. So the create topic operation // will never throw exception except there is no cluster. - admin.namespaces().setNamespaceReplicationClusters("prop-xyz/ns2", new HashSet()); - - try { - admin.topics().createPartitionedTopic(persistentPartitionedTopicName, partitions); - Assert.fail("should have failed due to Namespace does not have any clusters configured"); - } catch (PulsarAdminException.PreconditionFailedException ignored) { - } - - try { - admin.topics().createPartitionedTopic(NonPersistentPartitionedTopicName, partitions); - Assert.fail("should have failed due to Namespace does not have any clusters configured"); - } catch (PulsarAdminException.PreconditionFailedException ignored) { - } + admin.namespaces().setNamespaceReplicationClusters(defaultTenant + "/ns2", Sets.newHashSet(configClusterName)); - cleanup(); - setup(); + admin.topics().createPartitionedTopic(persistentPartitionedTopicName, partitions); + admin.topics().createPartitionedTopic(NonPersistentPartitionedTopicName, partitions); } @Test public void testCreateNamespaceWithNoClusters() throws PulsarAdminException { String localCluster = pulsar.getConfiguration().getClusterName(); - String namespace = "prop-xyz/test-ns-with-no-clusters"; + String namespace = newUniqueName(defaultTenant + "/test-ns-with-no-clusters"); admin.namespaces().createNamespace(namespace); // Global cluster, if there, should be omitted from the results @@ -1377,7 +1447,7 @@ public void testConsumerStatsLastTimestamp() throws PulsarClientException, Pulsa long timestamp = System.currentTimeMillis(); final String topicName = "consumer-stats-" + timestamp; final String subscribeName = topicName + "-test-stats-sub"; - final String topic = "persistent://prop-xyz/ns1/" + topicName; + final String topic = "persistent://" + defaultNamespace + "/" + topicName; final String producerName = "producer-" + topicName; @Cleanup @@ -1480,10 +1550,9 @@ public void testConsumerStatsLastTimestamp() throws PulsarClientException, Pulsa @Test(timeOut = 30000) public void testPreciseBacklog() throws Exception { - cleanup(); - setup(); + restartClusterIfReused(); - final String topic = "persistent://prop-xyz/ns1/precise-back-log"; + final String topic = "persistent://" + defaultNamespace + "/precise-back-log"; final String subName = "sub-name"; @Cleanup @@ -1533,6 +1602,7 @@ public void testPreciseBacklog() throws Exception { @Test public void testDeleteTenant() throws Exception { + restartClusterAfterTest(); // Disabled conf: systemTopicEnabled. see: https://github.com/apache/pulsar/pull/17070 boolean originalSystemTopicEnabled = conf.isSystemTopicEnabled(); if (originalSystemTopicEnabled) { @@ -1542,7 +1612,7 @@ public void testDeleteTenant() throws Exception { } pulsar.getConfiguration().setForceDeleteNamespaceAllowed(false); - String tenant = "test-tenant-1"; + String tenant = newUniqueName("test-tenant-1"); assertFalse(admin.tenants().getTenants().contains(tenant)); // create tenant @@ -1583,17 +1653,11 @@ public void testDeleteTenant() throws Exception { final String managedLedgersPath = "/managed-ledgers/" + tenant; final String partitionedTopicPath = "/admin/partitioned-topics/" + tenant; final String localPoliciesPath = "/admin/local-policies/" + tenant; - final String bundleDataPath = "/loadbalance/bundle-data/" + tenant; + final String bundleDataPath = BUNDLE_DATA_BASE_PATH + "/" + tenant; assertFalse(pulsar.getLocalMetadataStore().exists(managedLedgersPath).join()); assertFalse(pulsar.getLocalMetadataStore().exists(partitionedTopicPath).join()); assertFalse(pulsar.getLocalMetadataStore().exists(localPoliciesPath).join()); assertFalse(pulsar.getLocalMetadataStore().exists(bundleDataPath).join()); - // Reset conf: systemTopicEnabled - if (originalSystemTopicEnabled) { - cleanup(); - conf.setSystemTopicEnabled(true); - setup(); - } } @Data @@ -1628,13 +1692,16 @@ private void setNamespaceAttr(NamespaceAttr namespaceAttr){ @Test(dataProvider = "namespaceAttrs") public void testDeleteNamespace(NamespaceAttr namespaceAttr) throws Exception { + restartClusterAfterTest(); + // Set conf. cleanup(); - NamespaceAttr originalNamespaceAttr = markOriginalNamespaceAttr(); setNamespaceAttr(namespaceAttr); + this.conf.setMetadataStoreUrl("127.0.0.1:2181"); + this.conf.setConfigurationMetadataStoreUrl("127.0.0.1:2182"); setup(); - String tenant = "test-tenant"; + String tenant = newUniqueName("test-tenant"); assertFalse(admin.tenants().getTenants().contains(tenant)); // create tenant @@ -1652,6 +1719,28 @@ public void testDeleteNamespace(NamespaceAttr namespaceAttr) throws Exception { admin.topics().createPartitionedTopic(topic, 10); assertFalse(admin.topics().getList(namespace).isEmpty()); + final String managedLedgersPath = "/managed-ledgers/" + namespace; + final String bundleDataPath = BUNDLE_DATA_BASE_PATH + "/" + namespace; + // Trigger bundle owned by brokers. + pulsarClient.newProducer().topic(topic).create().close(); + // Trigger bundle data write to ZK. + Awaitility.await().untilAsserted(() -> { + boolean bundleDataWereWriten = false; + for (PulsarService ps : new PulsarService[]{pulsar, mockPulsarSetup.getPulsar()}) { + ModularLoadManagerWrapper loadManager = (ModularLoadManagerWrapper) ps.getLoadManager().get(); + ModularLoadManagerImpl loadManagerImpl = (ModularLoadManagerImpl) loadManager.getLoadManager(); + ps.getBrokerService().updateRates(); + loadManagerImpl.updateLocalBrokerData(); + loadManagerImpl.writeBundleDataOnZooKeeper(); + bundleDataWereWriten = bundleDataWereWriten || ps.getLocalMetadataStore().exists(bundleDataPath).join(); + } + assertTrue(bundleDataWereWriten); + }); + + // assert znode exists in metadata store + assertTrue(pulsar.getLocalMetadataStore().exists(bundleDataPath).join()); + assertTrue(pulsar.getLocalMetadataStore().exists(managedLedgersPath).join()); + try { admin.namespaces().deleteNamespace(namespace, false); fail("should have failed due to namespace not empty"); @@ -1668,23 +1757,14 @@ public void testDeleteNamespace(NamespaceAttr namespaceAttr) throws Exception { assertFalse(admin.namespaces().getNamespaces(tenant).contains(namespace)); assertTrue(admin.namespaces().getNamespaces(tenant).isEmpty()); - - final String managedLedgersPath = "/managed-ledgers/" + namespace; + // assert znode deleted in metadata store assertFalse(pulsar.getLocalMetadataStore().exists(managedLedgersPath).join()); - - - final String bundleDataPath = "/loadbalance/bundle-data/" + namespace; assertFalse(pulsar.getLocalMetadataStore().exists(bundleDataPath).join()); - - // Reset config - cleanup(); - setNamespaceAttr(originalNamespaceAttr); - setup(); } @Test public void testDeleteNamespaceWithTopicPolicies() throws Exception { - String tenant = "test-tenant"; + String tenant = newUniqueName("test-tenant"); assertFalse(admin.tenants().getTenants().contains(tenant)); // create tenant @@ -1732,7 +1812,7 @@ public void testDeleteNamespaceWithTopicPolicies() throws Exception { @Test(timeOut = 30000) public void testBacklogNoDelayed() throws PulsarClientException, PulsarAdminException, InterruptedException { - final String topic = "persistent://prop-xyz/ns1/precise-back-log-no-delayed-" + UUID.randomUUID().toString(); + final String topic = "persistent://" + defaultNamespace + "/precise-back-log-no-delayed-" + UUID.randomUUID().toString(); final String subName = "sub-name"; @Cleanup @@ -1788,7 +1868,7 @@ public void testBacklogNoDelayed() throws PulsarClientException, PulsarAdminExce @Test public void testPreciseBacklogForPartitionedTopic() throws PulsarClientException, PulsarAdminException { - final String topic = "persistent://prop-xyz/ns1/precise-back-log-for-partitioned-topic"; + final String topic = "persistent://" + defaultNamespace + "/precise-back-log-for-partitioned-topic"; admin.topics().createPartitionedTopic(topic, 2); final String subName = "sub-name"; @@ -1830,7 +1910,7 @@ public void testPreciseBacklogForPartitionedTopic() throws PulsarClientException @Test(timeOut = 30000) public void testBacklogNoDelayedForPartitionedTopic() throws PulsarClientException, PulsarAdminException, InterruptedException { - final String topic = "persistent://prop-xyz/ns1/precise-back-log-no-delayed-partitioned-topic"; + final String topic = "persistent://" + defaultNamespace + "/precise-back-log-no-delayed-partitioned-topic"; admin.topics().createPartitionedTopic(topic, 2); final String subName = "sub-name"; @@ -1845,6 +1925,8 @@ public void testBacklogNoDelayedForPartitionedTopic() throws PulsarClientExcepti .acknowledgmentGroupTime(0, TimeUnit.SECONDS) .subscribe(); + long start1 = 0; + long start2 = 0; @Cleanup Producer producer = client.newProducer() .topic(topic) @@ -1852,6 +1934,12 @@ public void testBacklogNoDelayedForPartitionedTopic() throws PulsarClientExcepti .create(); for (int i = 0; i < 10; i++) { + if (i == 0) { + start1 = Clock.systemUTC().millis(); + } + if (i == 5) { + start2 = Clock.systemUTC().millis(); + } if (i > 4) { producer.newMessage() .value("message-1".getBytes(StandardCharsets.UTF_8)) @@ -1862,29 +1950,34 @@ public void testBacklogNoDelayedForPartitionedTopic() throws PulsarClientExcepti } } // wait until the message add to delay queue. + long finalStart1 = start1; Awaitility.await().untilAsserted(() -> { TopicStats topicStats = admin.topics().getPartitionedStats(topic, false, true, true, true); assertEquals(topicStats.getSubscriptions().get(subName).getMsgBacklog(), 10); assertEquals(topicStats.getSubscriptions().get(subName).getBacklogSize(), 440); assertEquals(topicStats.getSubscriptions().get(subName).getMsgBacklogNoDelayed(), 5); + assertTrue(topicStats.getSubscriptions().get(subName).getEarliestMsgPublishTimeInBacklog() >= finalStart1); }); for (int i = 0; i < 5; i++) { consumer.acknowledge(consumer.receive()); } // Wait the ack send. - Awaitility.await().untilAsserted(() -> { + long finalStart2 = start2; + Awaitility.await().timeout(1, MINUTES).untilAsserted(() -> { TopicStats topicStats2 = admin.topics().getPartitionedStats(topic, false, true, true, true); assertEquals(topicStats2.getSubscriptions().get(subName).getMsgBacklog(), 5); assertEquals(topicStats2.getSubscriptions().get(subName).getBacklogSize(), 223); assertEquals(topicStats2.getSubscriptions().get(subName).getMsgBacklogNoDelayed(), 0); + assertTrue(topicStats2.getSubscriptions().get(subName).getEarliestMsgPublishTimeInBacklog() >= finalStart2); }); } @Test public void testMaxNumPartitionsPerPartitionedTopicSuccess() { - final String topic = "persistent://prop-xyz/ns1/max-num-partitions-per-partitioned-topic-success"; + restartClusterAfterTest(); + final String topic = "persistent://" + defaultNamespace + "/max-num-partitions-per-partitioned-topic-success"; pulsar.getConfiguration().setMaxNumPartitionsPerPartitionedTopic(3); try { @@ -1899,7 +1992,8 @@ public void testMaxNumPartitionsPerPartitionedTopicSuccess() { @Test public void testMaxNumPartitionsPerPartitionedTopicFailure() { - final String topic = "persistent://prop-xyz/ns1/max-num-partitions-per-partitioned-topic-failure"; + restartClusterAfterTest(); + final String topic = "persistent://" + defaultNamespace + "/max-num-partitions-per-partitioned-topic-failure"; pulsar.getConfiguration().setMaxNumPartitionsPerPartitionedTopic(2); try { @@ -1916,21 +2010,24 @@ public void testMaxNumPartitionsPerPartitionedTopicFailure() { @Test public void testListOfNamespaceBundles() throws Exception { TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test")); - admin.tenants().createTenant("prop-xyz2", tenantInfo); - admin.namespaces().createNamespace("prop-xyz2/ns1", 10); - admin.namespaces().setNamespaceReplicationClusters("prop-xyz2/ns1", Set.of("test")); - admin.namespaces().createNamespace("prop-xyz2/test/ns2", 10); - assertEquals(admin.namespaces().getBundles("prop-xyz2/ns1").getNumBundles(), 10); - assertEquals(admin.namespaces().getBundles("prop-xyz2/test/ns2").getNumBundles(), 10); + String tenantName = newUniqueName("prop-xyz2"); + admin.tenants().createTenant(tenantName, tenantInfo); + admin.namespaces().createNamespace(tenantName + "/ns1", 10); + admin.namespaces().setNamespaceReplicationClusters(tenantName + "/ns1", Set.of("test")); + admin.namespaces().createNamespace(tenantName + "/test/ns2", 10); + assertEquals(admin.namespaces().getBundles(tenantName + "/ns1").getNumBundles(), 10); + assertEquals(admin.namespaces().getBundles(tenantName + "/test/ns2").getNumBundles(), 10); - admin.namespaces().deleteNamespace("prop-xyz2/test/ns2"); + admin.namespaces().deleteNamespace(tenantName + "/test/ns2"); } @Test public void testForceDeleteNamespace() throws Exception { - final String namespaceName = "prop-xyz2/ns1"; + restartClusterAfterTest(); + String tenantName = newUniqueName("prop-xyz2"); + final String namespaceName = tenantName + "/ns1"; TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test")); - admin.tenants().createTenant("prop-xyz2", tenantInfo); + admin.tenants().createTenant(tenantName, tenantInfo); admin.namespaces().createNamespace(namespaceName, 1); final String topic = "persistent://" + namespaceName + "/test" + UUID.randomUUID(); pulsarClient.newProducer(Schema.DOUBLE).topic(topic).create().close(); @@ -1942,17 +2039,15 @@ public void testForceDeleteNamespace() throws Exception { } catch (PulsarAdminException e) { assertEquals(e.getStatusCode(), 404); } - - cleanup(); - setup(); } @Test public void testForceDeleteNamespaceWithAutomaticTopicCreation() throws Exception { conf.setForceDeleteNamespaceAllowed(true); - final String namespaceName = "prop-xyz2/ns1"; + String tenantName = newUniqueName("prop-xyz2"); + final String namespaceName = tenantName + "/ns1"; TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test")); - admin.tenants().createTenant("prop-xyz2", tenantInfo); + admin.tenants().createTenant(tenantName, tenantInfo); admin.namespaces().createNamespace(namespaceName, 1); admin.namespaces().setAutoTopicCreation(namespaceName, AutoTopicCreationOverride.builder() @@ -1977,7 +2072,7 @@ public void testForceDeleteNamespaceWithAutomaticTopicCreation() throws Exceptio // the consumer will try to re-create the partitions admin.namespaces().deleteNamespace(namespaceName, true); - assertFalse(admin.namespaces().getNamespaces("prop-xyz2").contains("ns1")); + assertFalse(admin.namespaces().getNamespaces(tenantName).contains("ns1")); } } @@ -2000,6 +2095,7 @@ public void testUpdateClusterWithProxyUrl() throws Exception { @Test public void testMaxNamespacesPerTenant() throws Exception { + restartClusterAfterTest(); cleanup(); conf.setMaxNamespacesPerTenant(2); setup(); @@ -2013,19 +2109,11 @@ public void testMaxNamespacesPerTenant() throws Exception { Assert.assertEquals(e.getStatusCode(), 412); Assert.assertEquals(e.getHttpError(), "Exceed the maximum number of namespace in tenant :testTenant"); } - - //unlimited - cleanup(); - conf.setMaxNamespacesPerTenant(0); - setup(); - admin.tenants().createTenant("testTenant", tenantInfo); - for (int i = 0; i < 10; i++) { - admin.namespaces().createNamespace("testTenant/ns-" + i, Set.of("test")); - } } @Test public void testAutoTopicCreationOverrideWithMaxNumPartitionsLimit() throws Exception{ + restartClusterAfterTest(); cleanup(); conf.setMaxNumPartitionsPerPartitionedTopic(10); setup(); @@ -2067,6 +2155,7 @@ public void testAutoTopicCreationOverrideWithMaxNumPartitionsLimit() throws Exce } @Test public void testMaxTopicsPerNamespace() throws Exception { + restartClusterAfterTest(); cleanup(); conf.setMaxTopicsPerNamespace(10); setup(); @@ -2158,16 +2247,12 @@ public void testMaxTopicsPerNamespace() throws Exception { } catch (PulsarClientException e) { log.info("Exception: ", e); } - - // reset configuration - conf.setMaxTopicsPerNamespace(0); - conf.setDefaultNumPartitions(1); } @Test public void testInvalidBundleErrorResponse() throws Exception { try { - admin.namespaces().deleteNamespaceBundle("prop-xyz/ns1", "invalid-bundle"); + admin.namespaces().deleteNamespaceBundle(defaultNamespace, "invalid-bundle"); fail("should have failed due to invalid bundle"); } catch (PreconditionFailedException e) { assertTrue(e.getMessage().startsWith("Invalid bundle range")); @@ -2176,6 +2261,7 @@ public void testInvalidBundleErrorResponse() throws Exception { @Test public void testMaxSubscriptionsPerTopic() throws Exception { + restartClusterAfterTest(); cleanup(); conf.setMaxSubscriptionsPerTopic(2); setup(); @@ -2258,7 +2344,7 @@ public void testMaxSubscriptionsPerTopic() throws Exception { @Test(timeOut = 30000) public void testMaxSubPerTopicApi() throws Exception { - final String myNamespace = "prop-xyz/ns" + UUID.randomUUID(); + final String myNamespace = newUniqueName(defaultTenant + "/ns"); admin.namespaces().createNamespace(myNamespace, Set.of("test")); assertNull(admin.namespaces().getMaxSubscriptionsPerTopic(myNamespace)); @@ -2281,8 +2367,11 @@ public void testMaxSubPerTopicApi() throws Exception { } } - @Test(timeOut = 30000) + @Test(timeOut = 60000) public void testSetNamespaceEntryFilters() throws Exception { + restartClusterAfterTest(); + restartClusterIfReused(); + @Cleanup final MockEntryFilterProvider testEntryFilterProvider = new MockEntryFilterProvider(conf); @@ -2299,7 +2388,7 @@ public void testSetNamespaceEntryFilters() throws Exception { try { EntryFilters entryFilters = new EntryFilters("test"); - final String myNamespace = "prop-xyz/ns" + UUID.randomUUID(); + final String myNamespace = newUniqueName(defaultTenant + "/ns"); admin.namespaces().createNamespace(myNamespace, Sets.newHashSet("test")); final String topicName = myNamespace + "/topic"; admin.topics().createNonPartitionedTopic(topicName); @@ -2359,6 +2448,9 @@ public void testSetNamespaceEntryFilters() throws Exception { @Test(dataProvider = "topicType") public void testSetTopicLevelEntryFilters(String topicType) throws Exception { + restartClusterAfterTest(); + restartClusterIfReused(); + @Cleanup final MockEntryFilterProvider testEntryFilterProvider = new MockEntryFilterProvider(conf); @@ -2373,7 +2465,7 @@ public void testSetTopicLevelEntryFilters(String topicType) throws Exception { "entryFilterProvider", testEntryFilterProvider, true); try { EntryFilters entryFilters = new EntryFilters("test"); - final String topic = topicType + "://prop-xyz/ns1/test-schema-validation-enforced"; + final String topic = topicType + "://" + defaultNamespace + "/test-schema-validation-enforced"; admin.topics().createPartitionedTopic(topic, 1); final String fullTopicName = topic + TopicName.PARTITIONED_TOPIC_SUFFIX + 0; @Cleanup @@ -2431,13 +2523,13 @@ public void testSetTopicLevelEntryFilters(String topicType) throws Exception { } } - @Test(timeOut = 30000) + @Test(timeOut = 60000) public void testSetEntryFiltersHierarchy() throws Exception { + restartClusterAfterTest(); + restartClusterIfReused(); + @Cleanup final MockEntryFilterProvider testEntryFilterProvider = new MockEntryFilterProvider(conf); - conf.setEntryFilterNames(List.of("test", "test1")); - conf.setAllowOverrideEntryFilters(true); - testEntryFilterProvider.setMockEntryFilters(new EntryFilterDefinition( "test", null, @@ -2450,9 +2542,11 @@ public void testSetEntryFiltersHierarchy() throws Exception { final EntryFilterProvider oldEntryFilterProvider = pulsar.getBrokerService().getEntryFilterProvider(); FieldUtils.writeField(pulsar.getBrokerService(), "entryFilterProvider", testEntryFilterProvider, true); + conf.setEntryFilterNames(List.of("test", "test1")); + conf.setAllowOverrideEntryFilters(true); try { - final String topic = "persistent://prop-xyz/ns1/test-schema-validation-enforced"; + final String topic = "persistent://" + defaultNamespace + "/test-schema-validation-enforced"; admin.topics().createPartitionedTopic(topic, 1); final String fullTopicName = topic + TopicName.PARTITIONED_TOPIC_SUFFIX + 0; @Cleanup @@ -2460,8 +2554,9 @@ public void testSetEntryFiltersHierarchy() throws Exception { .topic(fullTopicName) .create(); assertNull(admin.topicPolicies().getEntryFiltersPerTopic(topic, false)); - assertEquals(admin.topicPolicies().getEntryFiltersPerTopic(topic, true), - new EntryFilters("test,test1")); + Awaitility.await().untilAsserted(() -> + assertEquals(admin.topicPolicies().getEntryFiltersPerTopic(topic, true), + new EntryFilters("test,test1"))); assertEquals(pulsar .getBrokerService() .getTopic(fullTopicName, false) @@ -2471,10 +2566,11 @@ public void testSetEntryFiltersHierarchy() throws Exception { .size(), 2); EntryFilters nsEntryFilters = new EntryFilters("test"); - admin.namespaces().setNamespaceEntryFilters("prop-xyz/ns1", nsEntryFilters); - assertEquals(admin.namespaces().getNamespaceEntryFilters("prop-xyz/ns1"), nsEntryFilters); - assertEquals(admin.topicPolicies().getEntryFiltersPerTopic(topic, true), - new EntryFilters("test")); + admin.namespaces().setNamespaceEntryFilters(defaultNamespace, nsEntryFilters); + assertEquals(admin.namespaces().getNamespaceEntryFilters(defaultNamespace), nsEntryFilters); + Awaitility.await().untilAsserted(() -> + assertEquals(admin.topicPolicies().getEntryFiltersPerTopic(topic, true), + new EntryFilters("test"))); Awaitility.await().untilAsserted(() -> { assertEquals(pulsar .getBrokerService() @@ -2503,8 +2599,9 @@ public void testSetEntryFiltersHierarchy() throws Exception { admin.topicPolicies().setEntryFiltersPerTopic(topic, topicEntryFilters); Awaitility.await().untilAsserted(() -> assertEquals(admin.topicPolicies().getEntryFiltersPerTopic(topic, false), topicEntryFilters)); - assertEquals(admin.topicPolicies().getEntryFiltersPerTopic(topic, true), - new EntryFilters("test1")); + Awaitility.await().untilAsserted(() -> + assertEquals(admin.topicPolicies().getEntryFiltersPerTopic(topic, true), + new EntryFilters("test1"))); Awaitility.await().untilAsserted(() -> { assertEquals(pulsar .getBrokerService() @@ -2530,8 +2627,11 @@ public void testSetEntryFiltersHierarchy() throws Exception { } } - @Test(timeOut = 30000) + @Test(timeOut = 60000) public void testValidateNamespaceEntryFilters() throws Exception { + restartClusterAfterTest(); + restartClusterIfReused(); + @Cleanup final MockEntryFilterProvider testEntryFilterProvider = new MockEntryFilterProvider(conf); @@ -2546,7 +2646,7 @@ public void testValidateNamespaceEntryFilters() throws Exception { "entryFilterProvider", testEntryFilterProvider, true); try { - final String myNamespace = "prop-xyz/ns" + UUID.randomUUID(); + final String myNamespace = newUniqueName(defaultTenant + "/ns"); admin.namespaces().createNamespace(myNamespace, Sets.newHashSet("test")); try { admin.namespaces().setNamespaceEntryFilters(myNamespace, new EntryFilters("notexists")); @@ -2585,8 +2685,11 @@ public void testValidateNamespaceEntryFilters() throws Exception { } } - @Test(timeOut = 30000) + @Test(timeOut = 60000) public void testValidateTopicEntryFilters() throws Exception { + restartClusterAfterTest(); + restartClusterIfReused(); + @Cleanup final MockEntryFilterProvider testEntryFilterProvider = new MockEntryFilterProvider(conf); @@ -2601,7 +2704,7 @@ public void testValidateTopicEntryFilters() throws Exception { "entryFilterProvider", testEntryFilterProvider, true); try { - final String myNamespace = "prop-xyz/ns" + UUID.randomUUID(); + final String myNamespace = newUniqueName(defaultTenant + "/ns"); admin.namespaces().createNamespace(myNamespace, Sets.newHashSet("test")); final String topicName = myNamespace + "/topic"; admin.topics().createNonPartitionedTopic(topicName); @@ -2648,8 +2751,9 @@ public void testValidateTopicEntryFilters() throws Exception { @Test(timeOut = 30000) public void testMaxSubPerTopic() throws Exception { + restartClusterAfterTest(); pulsar.getConfiguration().setMaxSubscriptionsPerTopic(0); - final String myNamespace = "prop-xyz/ns" + UUID.randomUUID(); + final String myNamespace = newUniqueName(defaultTenant + "/ns"); admin.namespaces().createNamespace(myNamespace, Set.of("test")); final String topic = "persistent://" + myNamespace + "/testMaxSubPerTopic"; pulsarClient.newProducer().topic(topic).create().close(); @@ -2689,12 +2793,13 @@ public void testMaxSubPerTopic() throws Exception { @Test(timeOut = 30000) public void testMaxSubPerTopicPriority() throws Exception { + restartClusterAfterTest(); final int brokerLevelMaxSub = 2; cleanup(); conf.setMaxSubscriptionsPerTopic(brokerLevelMaxSub); setup(); - final String myNamespace = "prop-xyz/ns" + UUID.randomUUID(); + final String myNamespace = newUniqueName(defaultTenant + "/ns"); admin.namespaces().createNamespace(myNamespace, Set.of("test")); final String topic = "persistent://" + myNamespace + "/testMaxSubPerTopic"; //Create a client that can fail quickly @@ -2746,115 +2851,155 @@ public void testMaxSubPerTopicPriority() throws Exception { @Test public void testMaxProducersPerTopicUnlimited() throws Exception { + restartClusterAfterTest(); final int maxProducersPerTopic = 1; cleanup(); conf.setMaxProducersPerTopic(maxProducersPerTopic); setup(); //init namespace - final String myNamespace = "prop-xyz/ns" + UUID.randomUUID(); + final String myNamespace = newUniqueName(defaultTenant + "/ns"); admin.namespaces().createNamespace(myNamespace, Set.of("test")); final String topic = "persistent://" + myNamespace + "/testMaxProducersPerTopicUnlimited"; + admin.topics().createNonPartitionedTopic(topic); + AtomicInteger schemaOpsCounter = injectSchemaCheckCounterForTopic(topic); //the policy is set to 0, so there will be no restrictions admin.namespaces().setMaxProducersPerTopic(myNamespace, 0); Awaitility.await().until(() -> admin.namespaces().getMaxProducersPerTopic(myNamespace) == 0); - List> producers = new ArrayList<>(); + List> producers = new ArrayList<>(); for (int i = 0; i < maxProducersPerTopic + 1; i++) { - Producer producer = pulsarClient.newProducer().topic(topic).create(); + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); producers.add(producer); } + assertEquals(schemaOpsCounter.get(), maxProducersPerTopic + 1); admin.namespaces().removeMaxProducersPerTopic(myNamespace); Awaitility.await().until(() -> admin.namespaces().getMaxProducersPerTopic(myNamespace) == null); + try { @Cleanup - Producer producer = pulsarClient.newProducer().topic(topic).create(); + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); fail("should fail"); } catch (PulsarClientException e) { - assertTrue(e.getMessage().contains("Topic reached max producers limit")); + String expectMsg = "Topic '" + topic + "' reached max producers limit"; + assertTrue(e.getMessage().contains(expectMsg)); + assertEquals(schemaOpsCounter.get(), maxProducersPerTopic + 1); } //set the limit to 3 admin.namespaces().setMaxProducersPerTopic(myNamespace, 3); Awaitility.await().until(() -> admin.namespaces().getMaxProducersPerTopic(myNamespace) == 3); // should success - Producer producer = pulsarClient.newProducer().topic(topic).create(); + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); producers.add(producer); + assertEquals(schemaOpsCounter.get(), maxProducersPerTopic + 2); try { @Cleanup Producer producer1 = pulsarClient.newProducer().topic(topic).create(); fail("should fail"); } catch (PulsarClientException e) { - assertTrue(e.getMessage().contains("Topic reached max producers limit")); + String expectMsg = "Topic '" + topic + "' reached max producers limit"; + assertTrue(e.getMessage().contains(expectMsg)); + assertEquals(schemaOpsCounter.get(), maxProducersPerTopic + 2); } //clean up - for (Producer tempProducer : producers) { + for (Producer tempProducer : producers) { tempProducer.close(); } } + private AtomicInteger injectSchemaCheckCounterForTopic(String topicName) { + final var topics = pulsar.getBrokerService().getTopics(); + AbstractTopic topic = (AbstractTopic) topics.get(topicName).join().get(); + AbstractTopic spyTopic = Mockito.spy(topic); + AtomicInteger counter = new AtomicInteger(); + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + counter.incrementAndGet(); + return invocation.callRealMethod(); + } + }).when(spyTopic).addSchema(any(SchemaData.class)); + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + counter.incrementAndGet(); + return invocation.callRealMethod(); + } + }).when(spyTopic).addSchemaIfIdleOrCheckCompatible(any(SchemaData.class)); + topics.put(topicName, CompletableFuture.completedFuture(Optional.of(spyTopic))); + return counter; + } + @Test public void testMaxConsumersPerTopicUnlimited() throws Exception { + restartClusterAfterTest(); final int maxConsumersPerTopic = 1; cleanup(); conf.setMaxConsumersPerTopic(maxConsumersPerTopic); setup(); //init namespace - final String myNamespace = "prop-xyz/ns" + UUID.randomUUID(); + final String myNamespace = newUniqueName(defaultTenant + "/ns"); admin.namespaces().createNamespace(myNamespace, Set.of("test")); final String topic = "persistent://" + myNamespace + "/testMaxConsumersPerTopicUnlimited"; + admin.topics().createNonPartitionedTopic(topic); + AtomicInteger schemaOpsCounter = injectSchemaCheckCounterForTopic(topic); assertNull(admin.namespaces().getMaxConsumersPerTopic(myNamespace)); //the policy is set to 0, so there will be no restrictions admin.namespaces().setMaxConsumersPerTopic(myNamespace, 0); Awaitility.await().until(() -> admin.namespaces().getMaxConsumersPerTopic(myNamespace) == 0); - List> consumers = new ArrayList<>(); + List> consumers = new ArrayList<>(); for (int i = 0; i < maxConsumersPerTopic + 1; i++) { - Consumer consumer = - pulsarClient.newConsumer().subscriptionName(UUID.randomUUID().toString()).topic(topic).subscribe(); + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .subscriptionName(UUID.randomUUID().toString()).topic(topic).subscribe(); consumers.add(consumer); } + assertEquals(schemaOpsCounter.get(), maxConsumersPerTopic + 2); admin.namespaces().removeMaxConsumersPerTopic(myNamespace); Awaitility.await().until(() -> admin.namespaces().getMaxConsumersPerTopic(myNamespace) == null); try { @Cleanup - Consumer subscribe = - pulsarClient.newConsumer().subscriptionName(UUID.randomUUID().toString()).topic(topic).subscribe(); + Consumer subscribe = pulsarClient.newConsumer(Schema.STRING) + .subscriptionName(UUID.randomUUID().toString()).topic(topic).subscribe(); fail("should fail"); } catch (PulsarClientException e) { assertTrue(e.getMessage().contains("Topic reached max consumers limit")); + assertEquals(schemaOpsCounter.get(), maxConsumersPerTopic + 2); } //set the limit to 3 admin.namespaces().setMaxConsumersPerTopic(myNamespace, 3); Awaitility.await().until(() -> admin.namespaces().getMaxConsumersPerTopic(myNamespace) == 3); // should success - Consumer consumer = - pulsarClient.newConsumer().subscriptionName(UUID.randomUUID().toString()).topic(topic).subscribe(); + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .subscriptionName(UUID.randomUUID().toString()).topic(topic).subscribe(); consumers.add(consumer); + assertEquals(schemaOpsCounter.get(), maxConsumersPerTopic + 3); try { @Cleanup - Consumer subscribe = - pulsarClient.newConsumer().subscriptionName(UUID.randomUUID().toString()).topic(topic).subscribe(); + Consumer subscribe = pulsarClient.newConsumer(Schema.STRING) + .subscriptionName(UUID.randomUUID().toString()).topic(topic).subscribe(); fail("should fail"); } catch (PulsarClientException e) { assertTrue(e.getMessage().contains("Topic reached max consumers limit")); + assertEquals(schemaOpsCounter.get(), maxConsumersPerTopic + 3); } //clean up - for (Consumer subConsumer : consumers) { + for (Consumer subConsumer : consumers) { subConsumer.close(); } } @Test public void testClearBacklogForTheSubscriptionThatNoConsumers() throws Exception { - final String topic = "persistent://prop-xyz/ns1/clear_backlog_no_consumers" + UUID.randomUUID().toString(); + final String topic = "persistent://" + defaultNamespace + "/clear_backlog_no_consumers" + UUID.randomUUID().toString(); final String sub = "my-sub"; admin.topics().createNonPartitionedTopic(topic); admin.topics().createSubscription(topic, sub, MessageId.earliest); @@ -2863,7 +3008,10 @@ public void testClearBacklogForTheSubscriptionThatNoConsumers() throws Exception @Test(timeOut = 200000) public void testCompactionApi() throws Exception { - final String namespace = "prop-xyz/ns1"; + final String namespace = newUniqueName(defaultTenant + "/ns"); + admin.namespaces().createNamespace(namespace, Set.of("test")); + + assertNull(admin.namespaces().getCompactionThreshold(namespace)); assertEquals(pulsar.getConfiguration().getBrokerServiceCompactionThresholdInBytes(), 0); @@ -2878,11 +3026,13 @@ public void testCompactionApi() throws Exception { @Test(timeOut = 200000) public void testCompactionPriority() throws Exception { + restartClusterAfterTest(); cleanup(); conf.setBrokerServiceCompactionMonitorIntervalInSeconds(10000); setup(); - final String topic = "persistent://prop-xyz/ns1/topic" + UUID.randomUUID(); - final String namespace = "prop-xyz/ns1"; + final String namespace = newUniqueName(defaultTenant + "/ns"); + admin.namespaces().createNamespace(namespace, Set.of("test")); + final String topic = "persistent://" + namespace + "/topic" + UUID.randomUUID(); pulsarClient.newProducer().topic(topic).create().close(); TopicName topicName = TopicName.get(topic); PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopicIfExists(topic).get().get(); @@ -2921,7 +3071,8 @@ public void testCompactionPriority() throws Exception { @Test public void testProperties() throws Exception { - final String namespace = "prop-xyz/ns1"; + final String namespace = newUniqueName(defaultTenant + "/ns"); + admin.namespaces().createNamespace(namespace, Set.of("test")); admin.namespaces().setProperty(namespace, "a", "a"); assertEquals("a", admin.namespaces().getProperty(namespace, "a")); assertNull(admin.namespaces().getProperty(namespace, "b")); @@ -2944,7 +3095,7 @@ public void testProperties() throws Exception { @Test public void testGetListInBundle() throws Exception { - final String namespace = "prop-xyz/ns11"; + final String namespace = defaultTenant + "/ns11"; admin.namespaces().createNamespace(namespace, 3); final String persistentTopicName = TopicName.get( @@ -2978,7 +3129,8 @@ public void testGetListInBundle() throws Exception { @Test public void testGetTopicsWithDifferentMode() throws Exception { - final String namespace = "prop-xyz/ns1"; + final String namespace = newUniqueName(defaultTenant + "/ns"); + admin.namespaces().createNamespace(namespace, Set.of("test")); final String persistentTopicName = TopicName .get("persistent", NamespaceName.get(namespace), "get_topics_mode_" + UUID.randomUUID().toString()) @@ -3021,16 +3173,14 @@ public void testGetTopicsWithDifferentMode() throws Exception { @Test(dataProvider = "isV1") public void testNonPartitionedTopic(boolean isV1) throws Exception { - String tenant = "prop-xyz"; + restartClusterAfterTest(); + String tenant = defaultTenant; String cluster = "test"; String namespace = tenant + "/" + (isV1 ? cluster + "/" : "") + "n1" + isV1; String topic = "persistent://" + namespace + "/t1" + isV1; admin.namespaces().createNamespace(namespace, Set.of(cluster)); admin.topics().createNonPartitionedTopic(topic); assertTrue(admin.topics().getList(namespace).contains(topic)); - - cleanup(); - setup(); } /** @@ -3043,7 +3193,7 @@ public void testFailedUpdatePartitionedTopic() throws Exception { final String subName1 = topicName + "-my-sub-1"; final int startPartitions = 4; final int newPartitions = 8; - final String partitionedTopicName = "persistent://prop-xyz/ns1/" + topicName; + final String partitionedTopicName = "persistent://" + defaultNamespace + "/" + topicName; URL pulsarUrl = new URL(pulsar.getWebServiceAddress()); @@ -3062,7 +3212,7 @@ public void testFailedUpdatePartitionedTopic() throws Exception { admin.topics().createSubscription(partitionedTopicName + "-partition-" + startPartitions, subName1, MessageId.earliest); fail("Unexpected behaviour"); - } catch (PulsarAdminException.PreconditionFailedException ex) { + } catch (PulsarAdminException.ConflictException ex) { // OK } @@ -3077,13 +3227,37 @@ public void testFailedUpdatePartitionedTopic() throws Exception { assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, newPartitions); } + /** + * Validate retring failed partitioned topic should succeed. + * @throws Exception + */ + @Test + public void testTopicStatsWithEarliestTimeInBacklogIfNoBacklog() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp_"); + final String subscriptionName = "s1"; + admin.topics().createNonPartitionedTopic(topicName); + admin.topics().createSubscription(topicName, subscriptionName, MessageId.earliest); + + // Send one message. + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topicName).enableBatching(false) + .create(); + MessageIdImpl messageId = (MessageIdImpl) producer.send("123"); + // Catch up. + admin.topics().skipAllMessages(topicName, subscriptionName); + // Get topic stats with earliestTimeInBacklog + TopicStats topicStats = admin.topics().getStats(topicName, false, false, true); + assertEquals(topicStats.getSubscriptions().get(subscriptionName).getEarliestMsgPublishTimeInBacklog(), -1L); + + // cleanup. + producer.close(); + admin.topics().delete(topicName); + } + @Test(dataProvider = "topicType") public void testPartitionedStatsAggregationByProducerName(String topicType) throws Exception { - cleanup(); - setup(); - + restartClusterIfReused(); conf.setAggregatePublisherStatsByProducerName(true); - final String topic = topicType + "://prop-xyz/ns1/test-partitioned-stats-aggregation-by-producer-name"; + final String topic = topicType + "://" + defaultNamespace + "/test-partitioned-stats-aggregation-by-producer-name"; admin.topics().createPartitionedTopic(topic, 10); @Cleanup @@ -3137,8 +3311,9 @@ public int choosePartition(Message msg, TopicMetadata metadata) { @Test(dataProvider = "topicType") public void testPartitionedStatsAggregationByProducerNamePerPartition(String topicType) throws Exception { + restartClusterIfReused(); conf.setAggregatePublisherStatsByProducerName(true); - final String topic = topicType + "://prop-xyz/ns1/test-partitioned-stats-aggregation-by-producer-name-per-pt"; + final String topic = topicType + "://" + defaultNamespace + "/test-partitioned-stats-aggregation-by-producer-name-per-pt"; admin.topics().createPartitionedTopic(topic, 2); @Cleanup @@ -3161,7 +3336,7 @@ public void testPartitionedStatsAggregationByProducerNamePerPartition(String top @Test(dataProvider = "topicType") public void testSchemaValidationEnforced(String topicType) throws Exception { - final String topic = topicType + "://prop-xyz/ns1/test-schema-validation-enforced"; + final String topic = topicType + "://" + defaultNamespace + "/test-schema-validation-enforced"; admin.topics().createPartitionedTopic(topic, 1); @Cleanup Producer producer1 = pulsarClient.newProducer() @@ -3177,31 +3352,31 @@ public void testSchemaValidationEnforced(String topicType) throws Exception { @Test public void testGetNamespaceTopicList() throws Exception { - final String persistentTopic = "persistent://prop-xyz/ns1/testGetNamespaceTopicList"; - final String nonPersistentTopic = "non-persistent://prop-xyz/ns1/non-testGetNamespaceTopicList"; - final String eventTopic = "persistent://prop-xyz/ns1/__change_events"; + final String persistentTopic = "persistent://" + defaultNamespace + "/testGetNamespaceTopicList"; + final String nonPersistentTopic = "non-persistent://" + defaultNamespace + "/non-testGetNamespaceTopicList"; + final String eventTopic = "persistent://" + defaultNamespace + "/__change_events"; admin.topics().createNonPartitionedTopic(persistentTopic); Awaitility.await().untilAsserted(() -> - admin.namespaces().getTopics("prop-xyz/ns1", + admin.namespaces().getTopics(defaultNamespace, ListNamespaceTopicsOptions.builder().mode(Mode.PERSISTENT).includeSystemTopic(true).build()) .contains(eventTopic)); - List notIncludeSystemTopics = admin.namespaces().getTopics("prop-xyz/ns1", + List notIncludeSystemTopics = admin.namespaces().getTopics(defaultNamespace, ListNamespaceTopicsOptions.builder().includeSystemTopic(false).build()); Assert.assertFalse(notIncludeSystemTopics.contains(eventTopic)); @Cleanup Producer producer = pulsarClient.newProducer() .topic(nonPersistentTopic) .create(); - List notPersistentTopics = admin.namespaces().getTopics("prop-xyz/ns1", + List notPersistentTopics = admin.namespaces().getTopics(defaultNamespace, ListNamespaceTopicsOptions.builder().mode(Mode.NON_PERSISTENT).build()); Assert.assertTrue(notPersistentTopics.contains(nonPersistentTopic)); } @Test private void testTerminateSystemTopic() throws Exception { - final String topic = "persistent://prop-xyz/ns1/testTerminateSystemTopic"; + final String topic = "persistent://" + defaultNamespace + "/testTerminateSystemTopic"; admin.topics().createNonPartitionedTopic(topic); - final String eventTopic = "persistent://prop-xyz/ns1/__change_events"; + final String eventTopic = "persistent://" + defaultNamespace + "/__change_events"; admin.topicPolicies().setMaxConsumers(topic, 2); Awaitility.await().untilAsserted(() -> { Assert.assertEquals(admin.topicPolicies().getMaxConsumers(topic), Integer.valueOf(2)); @@ -3213,12 +3388,301 @@ private void testTerminateSystemTopic() throws Exception { @Test private void testDeleteNamespaceForciblyWithManyTopics() throws Exception { - final String ns = "prop-xyz/ns-testDeleteNamespaceForciblyWithManyTopics"; + final String ns = defaultTenant + "/ns-testDeleteNamespaceForciblyWithManyTopics"; admin.namespaces().createNamespace(ns, 2); for (int i = 0; i < 100; i++) { admin.topics().createPartitionedTopic(String.format("persistent://%s", ns + "/topic" + i), 3); } admin.namespaces().deleteNamespace(ns, true); - Assert.assertFalse(admin.namespaces().getNamespaces("prop-xyz").contains(ns)); + Assert.assertFalse(admin.namespaces().getNamespaces(defaultTenant).contains(ns)); + } + + @Test + private void testSetBacklogQuotasNamespaceLevelIfRetentionExists() throws Exception { + final String ns = defaultTenant + "/ns-testSetBacklogQuotasNamespaceLevel"; + final long backlogQuotaLimitSize = 100000002; + final int backlogQuotaLimitTime = 2; + admin.namespaces().createNamespace(ns, 2); + // create retention. + admin.namespaces().setRetention(ns, new RetentionPolicies(1800, 10000)); + // set backlog quota. + admin.namespaces().setBacklogQuota(ns, BacklogQuota.builder() + .limitSize(backlogQuotaLimitSize).limitTime(backlogQuotaLimitTime).build()); + // Verify result. + Map map = admin.namespaces().getBacklogQuotaMap(ns); + assertEquals(map.size(), 1); + assertTrue(map.containsKey(BacklogQuota.BacklogQuotaType.destination_storage)); + BacklogQuota backlogQuota = map.get(BacklogQuota.BacklogQuotaType.destination_storage); + assertEquals(backlogQuota.getLimitSize(), backlogQuotaLimitSize); + assertEquals(backlogQuota.getLimitTime(), backlogQuotaLimitTime); + // cleanup. + admin.namespaces().deleteNamespace(ns); + } + + @Test + private void testAnalyzeSubscriptionBacklogNotCauseStuck() throws Exception { + final String topic = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp"); + final String subscription = "s1"; + admin.topics().createNonPartitionedTopic(topic); + // Send 10 messages. + Consumer consumer = pulsarClient.newConsumer(Schema.STRING).topic(topic).subscriptionName(subscription) + .receiverQueueSize(0).subscribe(); + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + for (int i = 0; i < 10; i++) { + producer.send(i + ""); + } + + // Verify consumer can receive all messages after calling "analyzeSubscriptionBacklog". + admin.topics().analyzeSubscriptionBacklog(topic, subscription, Optional.of(MessageIdImpl.earliest)); + for (int i = 0; i < 10; i++) { + Awaitility.await().untilAsserted(() -> { + Message m = consumer.receive(); + assertNotNull(m); + consumer.acknowledge(m); + }); + } + + // cleanup. + consumer.close(); + producer.close(); + admin.topics().delete(topic); + } + + @Test + public void testGetStatsIfPartitionNotExists() throws Exception { + // create topic. + final String partitionedTp = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp"); + admin.topics().createPartitionedTopic(partitionedTp, 1); + TopicName partition0 = TopicName.get(partitionedTp).getPartition(0); + boolean topicExists1 = pulsar.getBrokerService().getTopic(partition0.toString(), false).join().isPresent(); + assertTrue(topicExists1); + // Verify topics-stats works. + TopicStats topicStats = admin.topics().getStats(partition0.toString()); + assertNotNull(topicStats); + + // Delete partition and call topic-stats again. + admin.topics().delete(partition0.toString()); + boolean topicExists2 = pulsar.getBrokerService().getTopic(partition0.toString(), false).join().isPresent(); + assertFalse(topicExists2); + // Verify: respond 404. + try { + admin.topics().getStats(partition0.toString()); + fail("Should respond 404 after the partition was deleted"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("Topic partitions were not yet created")); + } + + // cleanup. + admin.topics().deletePartitionedTopic(partitionedTp); + } + + private NamespaceIsolationData createPolicyData(NamespaceIsolationPolicyUnloadScope scope, List namespaces, + List primaryBrokers + ) { + // setup ns-isolation-policy in both the clusters. + Map parameters1 = new HashMap<>(); + parameters1.put("min_limit", "1"); + parameters1.put("usage_threshold", "100"); + List nsRegexList = new ArrayList<>(namespaces); + + NamespaceIsolationData.Builder build = NamespaceIsolationData.builder() + // "prop-ig/ns1" is present in test cluster, policy set on test2 should work + .namespaces(nsRegexList) + .primary(primaryBrokers) + .secondary(Collections.singletonList("")) + .autoFailoverPolicy(AutoFailoverPolicyData.builder() + .policyType(AutoFailoverPolicyType.min_available) + .parameters(parameters1) + .build()); + if (scope != null) { + build.unloadScope(scope); + } + return build.build(); + } + + private boolean allTopicsUnloaded(List topics) { + for (String topic : topics) { + if (pulsar.getBrokerService().getTopicReference(topic).isPresent()) { + return false; + } + } + return true; + } + + private void loadTopics(List topics) throws PulsarClientException, ExecutionException, InterruptedException { + // create a topic by creating a producer so that the topic is present on the broker + for (String topic : topics) { + Producer producer = pulsarClient.newProducer().topic(topic).create(); + producer.close(); + pulsar.getBrokerService().getTopicIfExists(topic).get(); + } + + // All namespaces are loaded onto broker. Assert that + for (String topic : topics) { + assertTrue(pulsar.getBrokerService().getTopicReference(topic).isPresent()); + } + } + + /** + * Validates that the namespace isolation policy set and update is unloading only the relevant namespaces based on + * the unload scope provided. + * + * @param topicType persistent or non persistent. + * @param policyName policy name. + * @param nsPrefix unique namespace prefix. + * @param totalNamespaces total namespaces to create. Only the end part. Each namespace also gets a topic t1. + * @param initialScope unload scope while creating the policy. + * @param initialNamespaceRegex namespace regex while creating the policy. + * @param initialLoadedNS expected namespaces to be still loaded after the policy create call. Remaining namespaces + * will be asserted to be unloaded within 20 seconds. + * @param updatedScope unload scope while updating the policy. + * @param updatedNamespaceRegex namespace regex while updating the policy. + * @param updatedLoadedNS expected namespaces to be loaded after policy update call. Remaining namespaces will be + * asserted to be unloaded within 20 seconds. + * @throws PulsarAdminException + * @throws PulsarClientException + * @throws ExecutionException + * @throws InterruptedException + */ + private void testIsolationPolicyUnloadsNSWithScope(String topicType, String policyName, String nsPrefix, + List totalNamespaces, + NamespaceIsolationPolicyUnloadScope initialScope, + List initialNamespaceRegex, List initialLoadedNS, + NamespaceIsolationPolicyUnloadScope updatedScope, + List updatedNamespaceRegex, List updatedLoadedNS, + List updatedBrokerRegex) + throws PulsarAdminException, PulsarClientException, ExecutionException, InterruptedException { + + // Create all namespaces + List allTopics = new ArrayList<>(); + for (String namespacePart: totalNamespaces) { + admin.namespaces().createNamespace(nsPrefix + namespacePart, Set.of("test")); + allTopics.add(topicType + "://" + nsPrefix + namespacePart + "/t1"); + } + // Load all topics so that they are present. Assume topic t1 under each namespace + loadTopics(allTopics); + + // Create the policy + NamespaceIsolationData nsPolicyData1 = createPolicyData( + initialScope, initialNamespaceRegex, Collections.singletonList(".*") + ); + admin.clusters().createNamespaceIsolationPolicy("test", policyName, nsPolicyData1); + + List initialLoadedTopics = new ArrayList<>(); + for (String namespacePart: initialLoadedNS) { + initialLoadedTopics.add(topicType + "://" + nsPrefix + namespacePart + "/t1"); + } + + List initialUnloadedTopics = new ArrayList<>(allTopics); + initialUnloadedTopics.removeAll(initialLoadedTopics); + + // Assert that all topics (and thus ns) not under initialLoadedNS namespaces are unloaded + if (initialUnloadedTopics.isEmpty()) { + // Just wait a bit to ensure we don't miss lazy unloading of topics we expect not to unload + TimeUnit.SECONDS.sleep(5); + } else { + Awaitility.await() + .atMost(10, TimeUnit.SECONDS) + .until(() -> allTopicsUnloaded(initialUnloadedTopics)); + } + // Assert that all topics under initialLoadedNS are still present + initialLoadedTopics.forEach(t -> assertTrue(pulsar.getBrokerService().getTopicReference(t).isPresent())); + + // Load the topics again + loadTopics(allTopics); + + // Update policy using updatedScope with updated namespace regex + nsPolicyData1 = createPolicyData(updatedScope, updatedNamespaceRegex, updatedBrokerRegex); + admin.clusters().updateNamespaceIsolationPolicy("test", policyName, nsPolicyData1); + + List updatedLoadedTopics = new ArrayList<>(); + for (String namespacePart : updatedLoadedNS) { + updatedLoadedTopics.add(topicType + "://" + nsPrefix + namespacePart + "/t1"); + } + + List updatedUnloadedTopics = new ArrayList<>(allTopics); + updatedUnloadedTopics.removeAll(updatedLoadedTopics); + + // Assert that all topics (and thus ns) not under updatedLoadedNS namespaces are unloaded + if (updatedUnloadedTopics.isEmpty()) { + // Just wait a bit to ensure we don't miss lazy unloading of topics we expect not to unload + TimeUnit.SECONDS.sleep(5); + } else { + Awaitility.await() + .atMost(10, TimeUnit.SECONDS) + .until(() -> allTopicsUnloaded(updatedUnloadedTopics)); + } + // Assert that all topics under updatedLoadedNS are still present + updatedLoadedTopics.forEach(t -> assertTrue(pulsar.getBrokerService().getTopicReference(t).isPresent())); + + } + + @Test(dataProvider = "topicType") + public void testIsolationPolicyUnloadsNSWithAllScope(final String topicType) throws Exception { + String nsPrefix = newUniqueName(defaultTenant + "/") + "-unload-test-"; + testIsolationPolicyUnloadsNSWithScope( + topicType, "policy-all", nsPrefix, List.of("a1", "a2", "b1", "b2", "c1"), + all_matching, List.of(".*-unload-test-a.*"), List.of("b1", "b2", "c1"), + all_matching, List.of(".*-unload-test-c.*"), List.of("b1", "b2"), + Collections.singletonList(".*") + ); + } + + @Test(dataProvider = "topicType") + public void testIsolationPolicyUnloadsNSWithChangedScope1(final String topicType) throws Exception { + String nsPrefix1 = newUniqueName(defaultTenant + "/") + "-unload-test-"; + // Addition case + testIsolationPolicyUnloadsNSWithScope( + topicType, "policy-changed1", nsPrefix1, List.of("a1", "a2", "b1", "b2", "c1"), + all_matching, List.of(".*-unload-test-a.*"), List.of("b1", "b2", "c1"), + changed, List.of(".*-unload-test-a.*", ".*-unload-test-c.*"), List.of("a1", "a2", "b1", "b2"), + Collections.singletonList(".*") + ); + } + + @Test(dataProvider = "topicType") + public void testIsolationPolicyUnloadsNSWithChangedScope2(final String topicType) throws Exception { + String nsPrefix2 = newUniqueName(defaultTenant + "/") + "-unload-test-"; + // removal case + testIsolationPolicyUnloadsNSWithScope( + topicType, "policy-changed2", nsPrefix2, List.of("a1", "a2", "b1", "b2", "c1"), + all_matching, List.of(".*-unload-test-a.*", ".*-unload-test-c.*"), List.of("b1", "b2"), + changed, List.of(".*-unload-test-c.*"), List.of("b1", "b2", "c1"), + Collections.singletonList(".*") + ); + } + + @Test(dataProvider = "topicType") + public void testIsolationPolicyUnloadsNSWithScopeMissing(final String topicType) throws Exception { + String nsPrefix = newUniqueName(defaultTenant + "/") + "-unload-test-"; + testIsolationPolicyUnloadsNSWithScope( + topicType, "policy-changed", nsPrefix, List.of("a1", "a2", "b1", "b2", "c1"), + all_matching, List.of(".*-unload-test-a.*"), List.of("b1", "b2", "c1"), + null, List.of(".*-unload-test-a.*", ".*-unload-test-c.*"), List.of("a1", "a2", "b1", "b2"), + Collections.singletonList(".*") + ); + } + + @Test(dataProvider = "topicType") + public void testIsolationPolicyUnloadsNSWithNoneScope(final String topicType) throws Exception { + String nsPrefix = newUniqueName(defaultTenant + "/") + "-unload-test-"; + testIsolationPolicyUnloadsNSWithScope( + topicType, "policy-none", nsPrefix, List.of("a1", "a2", "b1", "b2", "c1"), + all_matching, List.of(".*-unload-test-a.*"), List.of("b1", "b2", "c1"), + none, List.of(".*-unload-test-a.*", ".*-unload-test-c.*"), List.of("a1", "a2", "b1", "b2", "c1"), + Collections.singletonList(".*") + ); + } + + @Test(dataProvider = "topicType") + public void testIsolationPolicyUnloadsNSWithPrimaryChanged(final String topicType) throws Exception { + String nsPrefix = newUniqueName(defaultTenant + "/") + "-unload-test-"; + // As per changed flag, only c1 should unload, but due to primary change, both a* and c* will. + testIsolationPolicyUnloadsNSWithScope( + topicType, "policy-primary-changed", nsPrefix, List.of("a1", "a2", "b1", "b2", "c1"), + all_matching, List.of(".*-unload-test-a.*"), List.of("b1", "b2", "c1"), + changed, List.of(".*-unload-test-a.*", ".*-unload-test-c.*"), List.of("b1", "b2"), + List.of(".*", "broker.*") + ); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiDelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiDelayedDeliveryTest.java index 90af0e963fe8b..c9752750a8d7a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiDelayedDeliveryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiDelayedDeliveryTest.java @@ -66,6 +66,7 @@ public void testDisableDelayedDelivery() throws Exception { DelayedDeliveryPolicies delayedDeliveryPolicies = DelayedDeliveryPolicies.builder() .tickTime(2000) .active(false) + .maxDeliveryDelayInMillis(10_000) .build(); admin.namespaces().setDelayedDeliveryMessages(namespace, delayedDeliveryPolicies); //zk update takes time @@ -124,6 +125,7 @@ public void testNamespaceDelayedDeliveryPolicyApi() throws Exception { DelayedDeliveryPolicies delayedDeliveryPolicies = DelayedDeliveryPolicies.builder() .tickTime(3) .active(true) + .maxDeliveryDelayInMillis(5000) .build(); admin.namespaces().setDelayedDeliveryMessages(namespace, delayedDeliveryPolicies); Awaitility.await().untilAsserted(() @@ -151,12 +153,14 @@ public void testDelayedDeliveryApplied() throws Exception { DelayedDeliveryPolicies.builder() .tickTime(conf.getDelayedDeliveryTickTimeMillis()) .active(conf.isDelayedDeliveryEnabled()) + .maxDeliveryDelayInMillis(conf.getDelayedDeliveryMaxDelayInMillis()) .build(); assertEquals(admin.topics().getDelayedDeliveryPolicy(topic, true), brokerLevelPolicy); //set namespace-level policy DelayedDeliveryPolicies namespaceLevelPolicy = DelayedDeliveryPolicies.builder() .tickTime(100) .active(true) + .maxDeliveryDelayInMillis(4000) .build(); admin.namespaces().setDelayedDeliveryMessages(namespace, namespaceLevelPolicy); Awaitility.await().untilAsserted(() @@ -164,10 +168,12 @@ public void testDelayedDeliveryApplied() throws Exception { DelayedDeliveryPolicies policyFromBroker = admin.topics().getDelayedDeliveryPolicy(topic, true); assertEquals(policyFromBroker.getTickTime(), 100); assertTrue(policyFromBroker.isActive()); + assertEquals(policyFromBroker.getMaxDeliveryDelayInMillis(), 4000); // set topic-level policy DelayedDeliveryPolicies topicLevelPolicy = DelayedDeliveryPolicies.builder() .tickTime(200) .active(true) + .maxDeliveryDelayInMillis(5000) .build(); admin.topics().setDelayedDeliveryPolicy(topic, topicLevelPolicy); Awaitility.await().untilAsserted(() @@ -175,6 +181,7 @@ public void testDelayedDeliveryApplied() throws Exception { policyFromBroker = admin.topics().getDelayedDeliveryPolicy(topic, true); assertEquals(policyFromBroker.getTickTime(), 200); assertTrue(policyFromBroker.isActive()); + assertEquals(policyFromBroker.getMaxDeliveryDelayInMillis(), 5000); //remove topic-level policy admin.topics().removeDelayedDeliveryPolicy(topic); Awaitility.await().untilAsserted(() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiDynamicConfigurationsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiDynamicConfigurationsTest.java index c9a07dc966de6..aa7c2d720e353 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiDynamicConfigurationsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiDynamicConfigurationsTest.java @@ -18,14 +18,21 @@ */ package org.apache.pulsar.broker.admin; +import static org.apache.pulsar.common.naming.NamespaceName.SYSTEM_NAMESPACE; +import static org.assertj.core.api.Assertions.assertThat; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertThrows; import static org.testng.Assert.fail; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; import javax.ws.rs.core.Response; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -69,4 +76,103 @@ public void TestDeleteInvalidDynamicConfiguration() { } } } + + @Test + public void testRegisterCustomDynamicConfiguration() throws PulsarAdminException { + String key = "my-broker-config-key-1"; + String invalidValue = "invalid-value"; + + // register + pulsar.getBrokerService().registerCustomDynamicConfiguration(key, value -> !value.equals(invalidValue)); + assertThrows(IllegalArgumentException.class, + () -> pulsar.getBrokerService().registerCustomDynamicConfiguration(key, null)); + Map allDynamicConfigurations = admin.brokers().getAllDynamicConfigurations(); + assertThat(allDynamicConfigurations).doesNotContainKey(key); + + // update with listener + AtomicReference changeValue = new AtomicReference<>(null); + pulsar.getBrokerService().registerConfigurationListener(key, changeValue::set); + String newValue = "my-broker-config-value-1"; + admin.brokers().updateDynamicConfiguration(key, newValue); + allDynamicConfigurations = admin.brokers().getAllDynamicConfigurations(); + assertThat(allDynamicConfigurations.get(key)).isEqualTo(newValue); + + Awaitility.await().untilAsserted(() -> { + assertThat(changeValue.get()).isEqualTo(newValue); + }); + + // update with invalid value + assertThrows(PulsarAdminException.PreconditionFailedException.class, + () -> admin.brokers().updateDynamicConfiguration(key, invalidValue)); + + // delete + admin.brokers().deleteDynamicConfiguration(key); + allDynamicConfigurations = admin.brokers().getAllDynamicConfigurations(); + assertThat(allDynamicConfigurations).doesNotContainKey(key); + } + + @Test + public void testDeleteStringDynamicConfig() throws PulsarAdminException { + String syncEventTopic = BrokerTestUtil.newUniqueName(SYSTEM_NAMESPACE + "/tp"); + // The default value is null; + Awaitility.await().untilAsserted(() -> { + assertNull(pulsar.getConfig().getConfigurationMetadataSyncEventTopic()); + }); + // Set dynamic config. + admin.brokers().updateDynamicConfiguration("configurationMetadataSyncEventTopic", syncEventTopic); + Awaitility.await().untilAsserted(() -> { + assertEquals(pulsar.getConfig().getConfigurationMetadataSyncEventTopic(), syncEventTopic); + }); + // Remove dynamic config. + admin.brokers().deleteDynamicConfiguration("configurationMetadataSyncEventTopic"); + Awaitility.await().untilAsserted(() -> { + assertNull(pulsar.getConfig().getConfigurationMetadataSyncEventTopic()); + }); + } + + @Test + public void testDeleteIntDynamicConfig() throws PulsarAdminException { + // Record the default value; + int defaultValue = pulsar.getConfig().getMaxConcurrentTopicLoadRequest(); + // Set dynamic config. + int newValue = defaultValue + 1000; + admin.brokers().updateDynamicConfiguration("maxConcurrentTopicLoadRequest", newValue + ""); + Awaitility.await().untilAsserted(() -> { + assertEquals(pulsar.getConfig().getMaxConcurrentTopicLoadRequest(), newValue); + }); + // Verify: it has been reverted to the default value. + admin.brokers().deleteDynamicConfiguration("maxConcurrentTopicLoadRequest"); + Awaitility.await().untilAsserted(() -> { + assertEquals(pulsar.getConfig().getMaxConcurrentTopicLoadRequest(), defaultValue); + }); + } + + @Test + public void testDeleteCustomizedDynamicConfig() throws PulsarAdminException { + // Record the default value; + String customizedConfigName = "a123"; + pulsar.getBrokerService().registerCustomDynamicConfiguration(customizedConfigName, v -> true); + + AtomicReference currentValue = new AtomicReference<>(); + pulsar.getBrokerService().registerConfigurationListener(customizedConfigName, v -> { + currentValue.set(v); + }); + + // The default value is null; + Awaitility.await().untilAsserted(() -> { + assertNull(currentValue.get()); + }); + + // Set dynamic config. + admin.brokers().updateDynamicConfiguration(customizedConfigName, "xxx"); + Awaitility.await().untilAsserted(() -> { + assertEquals(currentValue.get(), "xxx"); + }); + + // Remove dynamic config. + admin.brokers().deleteDynamicConfiguration(customizedConfigName); + Awaitility.await().untilAsserted(() -> { + assertNull(currentValue.get()); + }); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiGetLastMessageIdTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiGetLastMessageIdTest.java index 3d2a6b934f847..27d72f98c2c49 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiGetLastMessageIdTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiGetLastMessageIdTest.java @@ -168,7 +168,7 @@ public Map, Collection>> register(Object callback, Object... c testNamespace, "my-topic", true); } catch (Exception e) { //System.out.println(e.getMessage()); - Assert.assertEquals("Topic not found", e.getMessage()); + Assert.assertTrue(e.getMessage().contains("Topic not found")); } String key = "legendtkl"; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiHealthCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiHealthCheckTest.java index a780f889de85f..618e023ccbf25 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiHealthCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiHealthCheckTest.java @@ -18,11 +18,13 @@ */ package org.apache.pulsar.broker.admin; +import static org.apache.pulsar.broker.admin.impl.BrokersBase.HEALTH_CHECK_TOPIC_SUFFIX; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import java.lang.management.ManagementFactory; import java.lang.management.ThreadMXBean; +import java.lang.reflect.Field; import java.time.Duration; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -31,16 +33,27 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.ProducerBuilderImpl; +import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicVersion; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.compaction.Compactor; import org.awaitility.Awaitility; +import org.mockito.Mockito; import org.springframework.util.CollectionUtils; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @Test(groups = "broker-admin") @@ -69,14 +82,27 @@ public void cleanup() throws Exception { super.internalCleanup(); } - @Test - public void testHealthCheckup() throws Exception { + @DataProvider(name = "topicVersion") + public static Object[][] topicVersions() { + return new Object[][] { + { null }, + { TopicVersion.V1 }, + { TopicVersion.V2 }, + }; + } + + @Test(dataProvider = "topicVersion") + public void testHealthCheckup(TopicVersion topicVersion) throws Exception { final int times = 30; CompletableFuture future = new CompletableFuture<>(); pulsar.getExecutor().execute(() -> { try { for (int i = 0; i < times; i++) { - admin.brokers().healthcheck(); + if (topicVersion == null) { + admin.brokers().healthcheck(); + } else { + admin.brokers().healthcheck(topicVersion); + } } future.complete(null); }catch (PulsarAdminException e) { @@ -84,11 +110,18 @@ public void testHealthCheckup() throws Exception { } }); for (int i = 0; i < times; i++) { - admin.brokers().healthcheck(); + if (topicVersion == null) { + admin.brokers().healthcheck(); + } else { + admin.brokers().healthcheck(topicVersion); + } } // To ensure we don't have any subscription - final String testHealthCheckTopic = String.format("persistent://pulsar/test/localhost:%s/healthcheck", - pulsar.getConfig().getWebServicePort().get()); + String brokerId = pulsar.getBrokerId(); + NamespaceName namespaceName = (topicVersion == TopicVersion.V2) + ? NamespaceService.getHeartbeatNamespaceV2(brokerId, pulsar.getConfiguration()) + : NamespaceService.getHeartbeatNamespace(brokerId, pulsar.getConfiguration()); + final String testHealthCheckTopic = String.format("persistent://%s/%s", namespaceName, HEALTH_CHECK_TOPIC_SUFFIX); Awaitility.await().untilAsserted(() -> { assertFalse(future.isCompletedExceptionally()); }); @@ -171,62 +204,49 @@ public void testDeadlockDetectionOverhead() { } } - @Test - public void testHealthCheckupV1() throws Exception { - final int times = 30; - CompletableFuture future = new CompletableFuture<>(); - pulsar.getExecutor().execute(() -> { - try { - for (int i = 0; i < times; i++) { - admin.brokers().healthcheck(TopicVersion.V1); - } - future.complete(null); - }catch (PulsarAdminException e) { - future.completeExceptionally(e); - } - }); - for (int i = 0; i < times; i++) { - admin.brokers().healthcheck(TopicVersion.V1); + class DummyProducerBuilder extends ProducerBuilderImpl { + // This is a dummy producer builder to test the health check timeout + // the producer constructed by this builder will not send any message + public DummyProducerBuilder(PulsarClientImpl client, Schema schema) { + super(client, schema); + } + + @Override + public CompletableFuture> createAsync() { + CompletableFuture> future = new CompletableFuture<>(); + super.createAsync().thenAccept(producer -> { + Producer spyProducer = Mockito.spy(producer); + Mockito.doReturn(CompletableFuture.completedFuture(MessageId.earliest)) + .when(spyProducer).sendAsync(Mockito.any()); + future.complete(spyProducer); + }).exceptionally(ex -> { + future.completeExceptionally(ex); + return null; + }); + return future; } - final String testHealthCheckTopic = String.format("persistent://pulsar/test/localhost:%s/healthcheck", - pulsar.getConfig().getWebServicePort().get()); - Awaitility.await().untilAsserted(() -> { - assertFalse(future.isCompletedExceptionally()); - }); - // To ensure we don't have any subscription - Awaitility.await().untilAsserted(() -> - assertTrue(CollectionUtils.isEmpty(admin.topics() - .getSubscriptions(testHealthCheckTopic).stream() - // All system topics are using compaction, even though is not explicitly set in the policies. - .filter(v -> !v.equals(Compactor.COMPACTION_SUBSCRIPTION)) - .collect(Collectors.toList()) - )) - ); } @Test - public void testHealthCheckupV2() throws Exception { - final int times = 30; - CompletableFuture future = new CompletableFuture<>(); - pulsar.getExecutor().execute(() -> { - try { - for (int i = 0; i < times; i++) { - admin.brokers().healthcheck(TopicVersion.V2); - } - future.complete(null); - }catch (PulsarAdminException e) { - future.completeExceptionally(e); - } - }); - for (int i = 0; i < times; i++) { - admin.brokers().healthcheck(TopicVersion.V2); - } + public void testHealthCheckTimeOut() throws Exception { final String testHealthCheckTopic = String.format("persistent://pulsar/localhost:%s/healthcheck", pulsar.getConfig().getWebServicePort().get()); - Awaitility.await().untilAsserted(() -> { - assertFalse(future.isCompletedExceptionally()); - }); - // To ensure we don't have any subscription + PulsarClient client = pulsar.getClient(); + PulsarClient spyClient = Mockito.spy(client); + Mockito.doReturn(new DummyProducerBuilder<>((PulsarClientImpl) spyClient, Schema.BYTES)) + .when(spyClient).newProducer(Schema.STRING); + // use reflection to replace the client in the broker + Field field = PulsarService.class.getDeclaredField("client"); + field.setAccessible(true); + field.set(pulsar, spyClient); + try { + admin.brokers().healthcheck(TopicVersion.V2); + throw new Exception("Should not reach here"); + } catch (PulsarAdminException e) { + log.info("Exception caught", e); + assertTrue(e.getMessage().contains("LowOverheadTimeoutException")); + } + // To ensure we don't have any subscription, the producers and readers are closed. Awaitility.await().untilAsserted(() -> assertTrue(CollectionUtils.isEmpty(admin.topics() .getSubscriptions(testHealthCheckTopic).stream() @@ -236,4 +256,5 @@ public void testHealthCheckupV2() throws Exception { )) ); } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiMaxUnackedMessagesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiMaxUnackedMessagesTest.java index 4fb268fc9713f..9b95e41bb64e7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiMaxUnackedMessagesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiMaxUnackedMessagesTest.java @@ -197,7 +197,7 @@ public void testMaxUnackedMessagesPerConsumerPriority() throws Exception { private List consumeMsg(Consumer consumer, int msgNum) throws Exception { List list = new ArrayList<>(); for (int i = 0; i replicationClusters = localAdmin.namespaces().getPolicies("prop-ig/ns1").replication_clusters; + Assert.assertFalse(replicationClusters.contains("cluster-2")); + + // setup ns-isolation-policy in both the clusters. + String policyName1 = "policy-1"; + Map parameters1 = new HashMap<>(); + parameters1.put("min_limit", "1"); + parameters1.put("usage_threshold", "100"); + List nsRegexList = new ArrayList<>(Arrays.asList("prop-ig/.*")); + + NamespaceIsolationData nsPolicyData1 = NamespaceIsolationData.builder() + // "prop-ig/ns1" is present in test cluster, policy set on test2 should work + .namespaces(nsRegexList) + .primary(Collections.singletonList(".*")) + .secondary(Collections.singletonList("")) + .autoFailoverPolicy(AutoFailoverPolicyData.builder() + .policyType(AutoFailoverPolicyType.min_available) + .parameters(parameters1) + .build()) + .build(); + + localAdmin.clusters().createNamespaceIsolationPolicy("test", policyName1, nsPolicyData1); + // verify policy is present in local cluster + Map policiesMap = + localAdmin.clusters().getNamespaceIsolationPolicies("test"); + assertEquals(policiesMap.get(policyName1), nsPolicyData1); + + remoteAdmin.clusters().createNamespaceIsolationPolicy("cluster-2", policyName1, nsPolicyData1); + // verify policy is present in remote cluster + policiesMap = remoteAdmin.clusters().getNamespaceIsolationPolicies("cluster-2"); + assertEquals(policiesMap.get(policyName1), nsPolicyData1); + + } + +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java index c3265897b8767..1ea29c9d431bd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java @@ -37,6 +37,7 @@ */ package org.apache.pulsar.broker.admin; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -48,6 +49,7 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import io.opentelemetry.api.common.Attributes; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -59,6 +61,9 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.resources.NamespaceResources; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil; +import org.apache.pulsar.broker.stats.OpenTelemetryTopicStats; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.admin.LongRunningProcessStatus; import org.apache.pulsar.client.admin.PulsarAdminException.ConflictException; import org.apache.pulsar.client.api.MessageId; @@ -71,6 +76,7 @@ import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; import org.apache.pulsar.common.policies.data.OffloadedReadPriority; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -100,6 +106,12 @@ public void setup() throws Exception { admin.namespaces().createNamespace(myNamespace, Set.of("test")); } + @Override + protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder pulsarTestContextBuilder) { + super.customizeMainPulsarTestContextBuilder(pulsarTestContextBuilder); + pulsarTestContextBuilder.enableOpenTelemetry(true); + } + @AfterMethod(alwaysRun = true) @Override public void cleanup() throws Exception { @@ -114,6 +126,7 @@ private void testOffload(String topicName, String mlName) throws Exception { CompletableFuture promise = new CompletableFuture<>(); doReturn(promise).when(offloader).offload(any(), any(), any()); + doReturn(true).when(offloader).isAppendable(); MessageId currentId = MessageId.latest; try (Producer p = pulsarClient.newProducer().topic(topicName).enableBatching(false).create()) { @@ -125,8 +138,18 @@ private void testOffload(String topicName, String mlName) throws Exception { ManagedLedgerInfo info = pulsar.getManagedLedgerFactory().getManagedLedgerInfo(mlName); assertEquals(info.ledgers.size(), 2); - assertEquals(admin.topics().offloadStatus(topicName).getStatus(), - LongRunningProcessStatus.Status.NOT_RUN); + assertEquals(admin.topics().offloadStatus(topicName).getStatus(), LongRunningProcessStatus.Status.NOT_RUN); + var topicNameObject = TopicName.get(topicName); + var attributes = Attributes.builder() + .put(OpenTelemetryAttributes.PULSAR_DOMAIN, topicNameObject.getDomain().toString()) + .put(OpenTelemetryAttributes.PULSAR_TENANT, topicNameObject.getTenant()) + .put(OpenTelemetryAttributes.PULSAR_NAMESPACE, topicNameObject.getNamespace()) + .put(OpenTelemetryAttributes.PULSAR_TOPIC, topicNameObject.getPartitionedTopicName()) + .build(); + // Verify the respective metric is 0 before the offload begins. + var metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + BrokerOpenTelemetryTestUtil.assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.STORAGE_OFFLOADED_COUNTER, + attributes, actual -> assertThat(actual).isZero()); admin.topics().triggerOffload(topicName, currentId); @@ -164,6 +187,11 @@ private void testOffload(String topicName, String mlName) throws Exception { assertEquals(firstUnoffloadedMessage.getEntryId(), 0); verify(offloader, times(2)).offload(any(), any(), any()); + + // Verify the metrics have been updated. + metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + BrokerOpenTelemetryTestUtil.assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.STORAGE_OFFLOADED_COUNTER, + attributes, actual -> assertThat(actual).isPositive()); } @@ -213,6 +241,8 @@ public void testOffloadPoliciesApi() throws Exception { OffloadPoliciesImpl offloadPolicies = (OffloadPoliciesImpl) admin.topics().getOffloadPolicies(topicName); assertNull(offloadPolicies); OffloadPoliciesImpl offload = new OffloadPoliciesImpl(); + offload.setManagedLedgerOffloadDriver("S3"); + offload.setManagedLedgerOffloadBucket("bucket"); String path = "fileSystemPath"; offload.setFileSystemProfilePath(path); admin.topics().setOffloadPolicies(topicName, offload); @@ -404,12 +434,13 @@ private void testOffload(boolean isPartitioned) throws Exception { //3 construct a topic level offloadPolicies OffloadPoliciesImpl offloadPolicies = new OffloadPoliciesImpl(); offloadPolicies.setOffloadersDirectory("."); - offloadPolicies.setManagedLedgerOffloadDriver("mock"); + offloadPolicies.setManagedLedgerOffloadDriver("S3"); + offloadPolicies.setManagedLedgerOffloadBucket("bucket"); offloadPolicies.setManagedLedgerOffloadPrefetchRounds(10); offloadPolicies.setManagedLedgerOffloadThresholdInBytes(1024L); LedgerOffloader topicOffloader = mock(LedgerOffloader.class); - when(topicOffloader.getOffloadDriverName()).thenReturn("mock"); + when(topicOffloader.getOffloadDriverName()).thenReturn("S3"); doReturn(topicOffloader).when(pulsar).createManagedLedgerOffloader(any()); //4 set topic level offload policies @@ -423,18 +454,18 @@ private void testOffload(boolean isPartitioned) throws Exception { .getTopic(TopicName.get(topicName).getPartition(i).toString(), false).get().get(); assertNotNull(topic.getManagedLedger().getConfig().getLedgerOffloader()); assertEquals(topic.getManagedLedger().getConfig().getLedgerOffloader().getOffloadDriverName() - , "mock"); + , "S3"); } } else { PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService() .getTopic(topicName, false).get().get(); assertNotNull(topic.getManagedLedger().getConfig().getLedgerOffloader()); assertEquals(topic.getManagedLedger().getConfig().getLedgerOffloader().getOffloadDriverName() - , "mock"); + , "S3"); } //6 remove topic level offload policy, offloader should become namespaceOffloader LedgerOffloader namespaceOffloader = mock(LedgerOffloader.class); - when(namespaceOffloader.getOffloadDriverName()).thenReturn("s3"); + when(namespaceOffloader.getOffloadDriverName()).thenReturn("S3"); Map map = new HashMap<>(); map.put(TopicName.get(topicName).getNamespaceObject(), namespaceOffloader); doReturn(map).when(pulsar).getLedgerOffloaderMap(); @@ -450,14 +481,14 @@ private void testOffload(boolean isPartitioned) throws Exception { .getTopicIfExists(TopicName.get(topicName).getPartition(i).toString()).get().get(); assertNotNull(topic.getManagedLedger().getConfig().getLedgerOffloader()); assertEquals(topic.getManagedLedger().getConfig().getLedgerOffloader().getOffloadDriverName() - , "s3"); + , "S3"); } } else { PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService() .getTopic(topicName, false).get().get(); assertNotNull(topic.getManagedLedger().getConfig().getLedgerOffloader()); assertEquals(topic.getManagedLedger().getConfig().getLedgerOffloader().getOffloadDriverName() - , "s3"); + , "S3"); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java index f67bd6fcfce5b..34d7dbeb8183c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java @@ -467,4 +467,34 @@ public void testCompatibility() throws Exception { assertTrue(e.getMessage().contains("Incompatible schema: exists schema type STRING, new schema type INT8")); } } + + @Test + public void testCompatibilityWithEmpty() throws Exception { + List> checkSchemas = List.of( + Schema.STRING, + Schema.JSON(SchemaDefinition.builder().withPojo(Foo.class).withProperties(PROPS).build()), + Schema.AVRO(SchemaDefinition.builder().withPojo(Foo.class).withProperties(PROPS).build()), + Schema.KeyValue(Schema.STRING, Schema.STRING) + ); + for (Schema schema : checkSchemas) { + SchemaInfo schemaInfo = schema.getSchemaInfo(); + String topicName = schemaCompatibilityNamespace + "/testCompatibilityWithEmpty"; + PostSchemaPayload postSchemaPayload = new PostSchemaPayload(schemaInfo.getType().toString(), + schemaInfo.getSchemaDefinition(), new HashMap<>()); + + // check compatibility with empty schema + IsCompatibilityResponse isCompatibilityResponse = + admin.schemas().testCompatibility(topicName, postSchemaPayload); + assertTrue(isCompatibilityResponse.isCompatibility()); + assertEquals(isCompatibilityResponse.getSchemaCompatibilityStrategy(), SchemaCompatibilityStrategy.FULL.name()); + + // set schema compatibility strategy is FULL_TRANSITIVE to cover checkCompatibilityWithAll + admin.namespaces().setSchemaCompatibilityStrategy(schemaCompatibilityNamespace, SchemaCompatibilityStrategy.FULL_TRANSITIVE); + isCompatibilityResponse = admin.schemas().testCompatibility(topicName, postSchemaPayload); + assertTrue(isCompatibilityResponse.isCompatibility()); + assertEquals(isCompatibilityResponse.getSchemaCompatibilityStrategy(), SchemaCompatibilityStrategy.FULL_TRANSITIVE.name()); + // set back to FULL + admin.namespaces().setSchemaCompatibilityStrategy(schemaCompatibilityNamespace, SchemaCompatibilityStrategy.FULL); + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaWithAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaWithAuthTest.java index 5159d7b714195..2dcb930fbe719 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaWithAuthTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaWithAuthTest.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Set; import javax.crypto.SecretKey; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; @@ -81,6 +82,7 @@ public void setup() throws Exception { ? brokerUrl.toString() : brokerUrlTls.toString()) .authentication(AuthenticationToken.class.getName(), ADMIN_TOKEN); + closeAdmin(); admin = Mockito.spy(pulsarAdminBuilder.build()); // Setup namespaces @@ -99,22 +101,26 @@ public void cleanup() throws Exception { @Test public void testGetCreateDeleteSchema() throws Exception { String topicName = "persistent://schematest/test/testCreateSchema"; + @Cleanup PulsarAdmin adminWithoutPermission = PulsarAdmin.builder() .serviceHttpUrl(brokerUrl != null ? brokerUrl.toString() : brokerUrlTls.toString()) .build(); + @Cleanup PulsarAdmin adminWithAdminPermission = PulsarAdmin.builder() .serviceHttpUrl(brokerUrl != null ? brokerUrl.toString() : brokerUrlTls.toString()) .authentication(AuthenticationToken.class.getName(), ADMIN_TOKEN) .build(); + @Cleanup PulsarAdmin adminWithConsumePermission = PulsarAdmin.builder() .serviceHttpUrl(brokerUrl != null ? brokerUrl.toString() : brokerUrlTls.toString()) .authentication(AuthenticationToken.class.getName(), CONSUME_TOKEN) .build(); - + @Cleanup PulsarAdmin adminWithProducePermission = PulsarAdmin.builder() .serviceHttpUrl(brokerUrl != null ? brokerUrl.toString() : brokerUrlTls.toString()) .authentication(AuthenticationToken.class.getName(), PRODUCE_TOKEN) .build(); + admin.topics().createNonPartitionedTopic(topicName); admin.topics().grantPermission(topicName, "consumer", EnumSet.of(AuthAction.consume)); admin.topics().grantPermission(topicName, "producer", EnumSet.of(AuthAction.produce)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTenantTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTenantTest.java index f883417614229..0cd9d9b737eba 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTenantTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTenantTest.java @@ -21,7 +21,6 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; - import java.util.Collections; import java.util.List; import java.util.UUID; @@ -30,6 +29,7 @@ import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfo; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -46,7 +46,7 @@ public void setup() throws Exception { .createCluster(CLUSTER, ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); } - @BeforeClass(alwaysRun = true) + @AfterClass(alwaysRun = true) @Override public void cleanup() throws Exception { super.internalCleanup(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java index 855343e18a24b..4a1dbface2c63 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.broker.admin; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; @@ -55,15 +57,19 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; import javax.ws.rs.client.InvocationCallback; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response.Status; import lombok.Builder; import lombok.Cleanup; +import lombok.SneakyThrows; import lombok.Value; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedLedgerInfo; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; @@ -73,8 +79,11 @@ import org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl; import org.apache.pulsar.broker.namespace.NamespaceEphemeralData; import org.apache.pulsar.broker.namespace.NamespaceService; +import org.apache.pulsar.broker.service.StickyKeyConsumerSelector; +import org.apache.pulsar.broker.service.persistent.PersistentStickyKeyDispatcherMultipleConsumers; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.broker.testcontext.SpyConfig; +import org.apache.pulsar.client.admin.GetStatsOptions; import org.apache.pulsar.client.admin.LongRunningProcessStatus; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -90,6 +99,7 @@ import org.apache.pulsar.client.api.ConsumerCryptoFailureAction; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageListener; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; @@ -135,8 +145,12 @@ import org.apache.pulsar.common.policies.data.TopicHashPositions; import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.util.Codec; +import org.apache.pulsar.common.util.Murmur3_32Hash; import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.apache.pulsar.common.util.collections.ConcurrentOpenLongPairRangeSet; +import org.apache.pulsar.common.util.collections.LongPairRangeSet; import org.apache.pulsar.compaction.Compactor; +import org.apache.pulsar.compaction.PulsarCompactionServiceFactory; import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -246,7 +260,9 @@ private void setupClusters() throws PulsarAdminException { @AfterClass(alwaysRun = true) @Override public void cleanup() throws Exception { + pulsar.getConfiguration().setBrokerShutdownTimeoutMs(0); adminTls.close(); + otheradmin.close(); super.internalCleanup(); mockPulsarSetup.cleanup(); } @@ -326,10 +342,12 @@ public void clusters() throws Exception { } catch (PulsarAdminException e) { assertTrue(e instanceof PreconditionFailedException); } + + restartBroker(); } @Test - public void clusterNamespaceIsolationPolicies() throws PulsarAdminException { + public void clusterNamespaceIsolationPolicies() throws Exception { try { // create String policyName1 = "policy-1"; @@ -476,7 +494,7 @@ public void clusterNamespaceIsolationPolicies() throws PulsarAdminException { throw e; } - // validate regex: invlid regex for primary and seconday + // validate regex: invalid regex for primary and secondary Map parameters = new HashMap<>(); parameters.put("min_limit", "1"); parameters.put("usage_threshold", "100"); @@ -505,6 +523,7 @@ public void clusterNamespaceIsolationPolicies() throws PulsarAdminException { // Ok } + restartBroker(); } @Test @@ -522,7 +541,8 @@ public void brokers() throws Exception { Assert.assertEquals(list2.size(), 1); BrokerInfo leaderBroker = admin.brokers().getLeaderBroker(); - Assert.assertEquals(leaderBroker.getServiceUrl(), pulsar.getLeaderElectionService().getCurrentLeader().map(LeaderBroker::getServiceUrl).get()); + Assert.assertEquals(leaderBroker.getBrokerId(), + pulsar.getLeaderElectionService().getCurrentLeader().map(LeaderBroker::getBrokerId).get()); Map nsMap = admin.brokers().getOwnedNamespaces("test", list.get(0)); // since sla-monitor ns is not created nsMap.size() == 1 (for HeartBeat Namespace) @@ -530,7 +550,7 @@ public void brokers() throws Exception { for (String ns : nsMap.keySet()) { NamespaceOwnershipStatus nsStatus = nsMap.get(ns); if (ns.equals( - NamespaceService.getHeartbeatNamespace(pulsar.getAdvertisedAddress(), pulsar.getConfiguration()) + NamespaceService.getHeartbeatNamespace(pulsar.getBrokerId(), pulsar.getConfiguration()) + "/0x00000000_0xffffffff")) { assertEquals(nsStatus.broker_assignment, BrokerAssignment.shared); assertFalse(nsStatus.is_controlled); @@ -538,10 +558,7 @@ public void brokers() throws Exception { } } - String[] parts = list.get(0).split(":"); - Assert.assertEquals(parts.length, 2); - Map nsMap2 = adminTls.brokers().getOwnedNamespaces("test", - String.format("%s:%d", parts[0], pulsar.getListenPortHTTPS().get())); + Map nsMap2 = adminTls.brokers().getOwnedNamespaces("test", list.get(0)); Assert.assertEquals(nsMap2.size(), 2); deleteNamespaceWithRetry("prop-xyz/ns1", false); @@ -696,6 +713,10 @@ public void testInvalidDynamicConfigContentInMetadata() throws Exception { Awaitility.await().until(() -> pulsar.getConfiguration().getBrokerShutdownTimeoutMs() == newValue); // verify value is updated assertEquals(pulsar.getConfiguration().getBrokerShutdownTimeoutMs(), newValue); + // reset config + pulsar.getConfiguration().setBrokerShutdownTimeoutMs(0L); + // restart broker + restartBroker(); } /** @@ -794,6 +815,8 @@ public void namespaces() throws Exception { TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test", "usw")); admin.tenants().updateTenant("prop-xyz", tenantInfo); + Awaitility.await().untilAsserted(() -> + assertEquals(admin.tenants().getTenantInfo("prop-xyz").getAllowedClusters(), Set.of("test", "usw"))); assertEquals(admin.namespaces().getPolicies("prop-xyz/ns1").bundles, PoliciesUtil.defaultBundle()); @@ -926,8 +949,7 @@ public void persistentTopics(String topicName) throws Exception { assertEquals(topicStats.getSubscriptions().get(subName).getConsumers().size(), 1); assertEquals(topicStats.getSubscriptions().get(subName).getMsgBacklog(), 10); assertEquals(topicStats.getPublishers().size(), 0); - assertEquals(topicStats.getOwnerBroker(), - pulsar.getAdvertisedAddress() + ":" + pulsar.getConfiguration().getWebServicePort().get()); + assertEquals(topicStats.getOwnerBroker(), pulsar.getBrokerId()); PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(persistentTopicName, false); assertEquals(internalStats.cursors.keySet(), Set.of(Codec.encode(subName))); @@ -993,6 +1015,135 @@ public void persistentTopics(String topicName) throws Exception { assertEquals(admin.topics().getList("prop-xyz/ns1"), new ArrayList<>()); } + @Test(dataProvider = "topicName") + public void testSkipHoleMessages(String topicName) throws Exception { + final String subName = topicName; + assertEquals(admin.topics().getList("prop-xyz/ns1"), new ArrayList<>()); + + final String persistentTopicName = "persistent://prop-xyz/ns1/" + topicName; + // Force to create a topic + publishMessagesOnPersistentTopic("persistent://prop-xyz/ns1/" + topicName, 0); + assertEquals(admin.topics().getList("prop-xyz/ns1"), + List.of("persistent://prop-xyz/ns1/" + topicName)); + + // create consumer and subscription + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsar.getWebServiceAddress()) + .statsInterval(0, TimeUnit.SECONDS) + .build(); + AtomicInteger total = new AtomicInteger(); + Consumer consumer = client.newConsumer().topic(persistentTopicName) + .messageListener(new MessageListener() { + @SneakyThrows + @Override + public void received(Consumer consumer, Message msg) { + if (total.get() %2 !=0){ + // artificially created 50 hollow messages + consumer.acknowledge(msg); + } + total.incrementAndGet(); + } + }) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Exclusive).subscribe(); + + assertEquals(admin.topics().getSubscriptions(persistentTopicName), List.of(subName)); + + publishMessagesOnPersistentTopic("persistent://prop-xyz/ns1/" + topicName, 100); + TimeUnit.SECONDS.sleep(2); + TopicStats topicStats = admin.topics().getStats(persistentTopicName); + long msgBacklog = topicStats.getSubscriptions().get(subName).getMsgBacklog(); + log.info("back={}",msgBacklog); + int skipNumber = 20; + admin.topics().skipMessages(persistentTopicName, subName, skipNumber); + topicStats = admin.topics().getStats(persistentTopicName); + assertEquals(topicStats.getSubscriptions().get(subName).getMsgBacklog(), msgBacklog - skipNumber); + } + + @Test(dataProvider = "topicType") + public void testPartitionState(String topicType) throws Exception { + final String namespace = "prop-xyz/ns1"; + final String partitionedTopicName = topicType + "://" + namespace + "/ds1"; + + admin.topics().createPartitionedTopic(partitionedTopicName, 4); + + // create consumer and subscription + URL pulsarUrl = new URL(pulsar.getWebServiceAddress()); + @Cleanup + PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS) + .build(); + Consumer consumer = client.newConsumer().topic(partitionedTopicName) + .subscriptionName("my-sub").subscribe(); + + Producer producer = client.newProducer(Schema.BYTES) + .topic(partitionedTopicName) + .enableBatching(false) + .create(); + for (int i = 0; i < 10; i++) { + String message = "message-" + i; + producer.send(message.getBytes()); + } + + GetStatsOptions getStatsOptions = new GetStatsOptions(false, false, false, true, true); + PartitionedTopicStats topicStats = admin.topics().getPartitionedStats(partitionedTopicName, + true, getStatsOptions); + assertEquals(topicStats.getPublishers().size(), 0); + topicStats.getPartitions().forEach((k, v)-> { + assertEquals(v.getPublishers().size(), 0); + v.getSubscriptions().forEach((k1, v1)-> { + assertEquals(v1.getConsumers().size(), 0); + }); + }); + + topicStats.getSubscriptions().forEach((k, v)-> { + assertEquals(v.getConsumers().size(), 0); + }); + + producer.close(); + consumer.close(); + client.close(); + } + + + @Test(dataProvider = "topicType") + public void testNonPartitionState(String topicType) throws Exception { + final String namespace = "prop-xyz/ns1"; + final String topicName = topicType + "://" + namespace + "/ds1"; + + admin.topics().createNonPartitionedTopic(topicName); + + // create consumer and subscription + URL pulsarUrl = new URL(pulsar.getWebServiceAddress()); + @Cleanup + PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS) + .build(); + Consumer consumer = client.newConsumer().topic(topicName) + .subscriptionName("my-sub").subscribe(); + + Producer producer = client.newProducer(Schema.BYTES) + .topic(topicName) + .enableBatching(false) + .create(); + for (int i = 0; i < 10; i++) { + String message = "message-" + i; + producer.send(message.getBytes()); + } + + GetStatsOptions getStatsOptions = new GetStatsOptions(false, false, false, true, true); + TopicStats topicStats = admin.topics().getStats(topicName, getStatsOptions); + + assertEquals(topicStats.getPublishers().size(), 0); + + topicStats.getSubscriptions().forEach((k, v)-> { + assertEquals(v.getConsumers().size(), 0); + }); + + producer.close(); + consumer.close(); + client.close(); + } + @Test(dataProvider = "topicNamesForAllTypes") public void partitionedTopics(String topicType, String topicName) throws Exception { final String namespace = "prop-xyz/ns1"; @@ -1239,7 +1390,7 @@ public void testGetStats() throws Exception { TopicStats topicStats = admin.topics().getStats(topic, false, false, true); assertEquals(topicStats.getEarliestMsgPublishTimeInBacklogs(), 0); - assertEquals(topicStats.getSubscriptions().get(subName).getEarliestMsgPublishTimeInBacklog(), 0); + assertEquals(topicStats.getSubscriptions().get(subName).getEarliestMsgPublishTimeInBacklog(), -1); assertEquals(topicStats.getSubscriptions().get(subName).getBacklogSize(), -1); // publish several messages @@ -1259,7 +1410,7 @@ public void testGetStats() throws Exception { topicStats = admin.topics().getStats(topic, false, true, true); assertEquals(topicStats.getEarliestMsgPublishTimeInBacklogs(), 0); - assertEquals(topicStats.getSubscriptions().get(subName).getEarliestMsgPublishTimeInBacklog(), 0); + assertEquals(topicStats.getSubscriptions().get(subName).getEarliestMsgPublishTimeInBacklog(), -1); assertEquals(topicStats.getSubscriptions().get(subName).getBacklogSize(), 0); } @@ -2164,8 +2315,8 @@ public void testUnsubscribeOnNamespace(Integer numBundles) throws Exception { admin.namespaces().unsubscribeNamespace("prop-xyz/ns1-bundles", "my-sub"); - assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/ns1-bundles/ds2"), - List.of("my-sub-1", "my-sub-2")); + assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/ns1-bundles/ds2").stream().sorted() + .toList(), List.of("my-sub-1", "my-sub-2")); assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/ns1-bundles/ds1"), List.of("my-sub-1")); @@ -3055,6 +3206,9 @@ public void testTopicBundleRangeLookup() throws PulsarAdminException, PulsarServ TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test", "usw")); admin.tenants().updateTenant("prop-xyz", tenantInfo); + Awaitility.await().untilAsserted(() -> + assertEquals(admin.tenants().getTenantInfo("prop-xyz").getAllowedClusters(), + tenantInfo.getAllowedClusters())); admin.namespaces().createNamespace("prop-xyz/getBundleNs", 100); assertEquals(admin.namespaces().getPolicies("prop-xyz/getBundleNs").bundles.getNumBundles(), 100); @@ -3074,7 +3228,7 @@ public void testTriggerCompaction() throws Exception { // mock actual compaction, we don't need to really run it CompletableFuture promise = new CompletableFuture(); - Compactor compactor = pulsar.getCompactor(); + Compactor compactor = ((PulsarCompactionServiceFactory)pulsar.getCompactionServiceFactory()).getCompactor(); doReturn(promise).when(compactor).compact(topicName); admin.topics().triggerCompaction(topicName); @@ -3110,7 +3264,7 @@ public void testTriggerCompactionPartitionedTopic() throws Exception { // mock actual compaction, we don't need to really run it CompletableFuture promise = new CompletableFuture<>(); - Compactor compactor = pulsar.getCompactor(); + Compactor compactor = ((PulsarCompactionServiceFactory)pulsar.getCompactionServiceFactory()).getCompactor(); doReturn(promise).when(compactor).compact(topicName + "-partition-0"); CompletableFuture promise1 = new CompletableFuture<>(); @@ -3154,7 +3308,7 @@ public void testCompactionStatus() throws Exception { // mock actual compaction, we don't need to really run it CompletableFuture promise = new CompletableFuture(); - Compactor compactor = pulsar.getCompactor(); + Compactor compactor = ((PulsarCompactionServiceFactory)pulsar.getCompactionServiceFactory()).getCompactor(); doReturn(promise).when(compactor).compact(topicName); admin.topics().triggerCompaction(topicName); @@ -3248,6 +3402,9 @@ public void testCreateAndDeleteNamespaceWithBundles() throws Exception { TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test", "usw")); admin.tenants().updateTenant("prop-xyz", tenantInfo); + Awaitility.await().untilAsserted(() -> + assertEquals(admin.tenants().getTenantInfo("prop-xyz").getAllowedClusters(), + tenantInfo.getAllowedClusters())); String ns = BrokerTestUtil.newUniqueName("prop-xyz/ns"); @@ -3301,8 +3458,8 @@ public void testGetTtlDurationDefaultInSeconds() throws Exception { } @Test - public void testGetReadPositionWhenJoining() throws Exception { - final String topic = "persistent://prop-xyz/ns1/testGetReadPositionWhenJoining-" + UUID.randomUUID().toString(); + public void testGetLastSentPositionWhenJoining() throws Exception { + final String topic = "persistent://prop-xyz/ns1/testGetLastSentPositionWhenJoining-" + UUID.randomUUID().toString(); final String subName = "my-sub"; @Cleanup Producer producer = pulsarClient.newProducer() @@ -3310,34 +3467,189 @@ public void testGetReadPositionWhenJoining() throws Exception { .enableBatching(false) .create(); + @Cleanup + final Consumer consumer1 = pulsarClient.newConsumer() + .topic(topic) + .subscriptionType(SubscriptionType.Key_Shared) + .subscriptionName(subName) + .subscribe(); + final int messages = 10; MessageIdImpl messageId = null; for (int i = 0; i < messages; i++) { messageId = (MessageIdImpl) producer.send(("Hello Pulsar - " + i).getBytes()); + consumer1.receive(); } - List> consumers = new ArrayList<>(); - for (int i = 0; i < 2; i++) { - Consumer consumer = pulsarClient.newConsumer() - .topic(topic) - .subscriptionType(SubscriptionType.Key_Shared) - .subscriptionName(subName) - .subscribe(); - consumers.add(consumer); - } + @Cleanup + final Consumer consumer2 = pulsarClient.newConsumer() + .topic(topic) + .subscriptionType(SubscriptionType.Key_Shared) + .subscriptionName(subName) + .subscribe(); TopicStats stats = admin.topics().getStats(topic); Assert.assertEquals(stats.getSubscriptions().size(), 1); SubscriptionStats subStats = stats.getSubscriptions().get(subName); Assert.assertNotNull(subStats); Assert.assertEquals(subStats.getConsumers().size(), 2); - ConsumerStats consumerStats = subStats.getConsumers().get(0); - Assert.assertEquals(consumerStats.getReadPositionWhenJoining(), - PositionImpl.get(messageId.getLedgerId(), messageId.getEntryId() + 1).toString()); + ConsumerStats consumerStats = subStats.getConsumers().stream() + .filter(s -> s.getConsumerName().equals(consumer2.getConsumerName())).findFirst().get(); + Assert.assertEquals(consumerStats.getLastSentPositionWhenJoining(), + PositionFactory.create(messageId.getLedgerId(), messageId.getEntryId()).toString()); + } + + @Test + public void testGetLastSentPosition() throws Exception { + final String topic = "persistent://prop-xyz/ns1/testGetLastSentPosition-" + UUID.randomUUID().toString(); + final String subName = "my-sub"; + @Cleanup + final Producer producer = pulsarClient.newProducer() + .topic(topic) + .enableBatching(false) + .create(); + final AtomicInteger counter = new AtomicInteger(); + @Cleanup + final Consumer consumer = pulsarClient.newConsumer() + .topic(topic) + .subscriptionType(SubscriptionType.Key_Shared) + .subscriptionName(subName) + .messageListener((c, msg) -> { + try { + c.acknowledge(msg); + counter.getAndIncrement(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .subscribe(); - for (Consumer consumer : consumers) { - consumer.close(); + TopicStats stats = admin.topics().getStats(topic); + Assert.assertEquals(stats.getSubscriptions().size(), 1); + SubscriptionStats subStats = stats.getSubscriptions().get(subName); + Assert.assertNotNull(subStats); + Assert.assertNull(subStats.getLastSentPosition()); + + final int messages = 10; + MessageIdImpl messageId = null; + for (int i = 0; i < messages; i++) { + messageId = (MessageIdImpl) producer.send(("Hello Pulsar - " + i).getBytes()); } + + Awaitility.await().untilAsserted(() -> assertEquals(counter.get(), messages)); + + stats = admin.topics().getStats(topic); + Assert.assertEquals(stats.getSubscriptions().size(), 1); + subStats = stats.getSubscriptions().get(subName); + Assert.assertNotNull(subStats); + Assert.assertEquals(subStats.getLastSentPosition(), PositionFactory.create(messageId.getLedgerId(), messageId.getEntryId()).toString()); + } + + @Test + public void testGetIndividuallySentPositions() throws Exception { + // The producer sends messages with two types of keys. + // The dispatcher sends keyA messages to consumer1. + // Consumer1 will not receive any messages. Its receiver queue size is 1. + // Consumer2 will receive and ack any messages immediately. + + final String topic = "persistent://prop-xyz/ns1/testGetIndividuallySentPositions-" + UUID.randomUUID().toString(); + final String subName = "my-sub"; + @Cleanup + final Producer producer = pulsarClient.newProducer() + .topic(topic) + .enableBatching(false) + .create(); + + final String consumer1Name = "c1"; + final String consumer2Name = "c2"; + + @Cleanup + final Consumer consumer1 = pulsarClient.newConsumer() + .topic(topic) + .consumerName(consumer1Name) + .receiverQueueSize(1) + .subscriptionType(SubscriptionType.Key_Shared) + .subscriptionName(subName) + .subscribe(); + + final PersistentStickyKeyDispatcherMultipleConsumers dispatcher = + (PersistentStickyKeyDispatcherMultipleConsumers) pulsar.getBrokerService().getTopicIfExists(topic).get().get().getSubscription(subName).getDispatcher(); + final String keyA = "key-a"; + final String keyB = "key-b"; + final int hashA = Murmur3_32Hash.getInstance().makeHash(keyA.getBytes()); + + final Field selectorField = PersistentStickyKeyDispatcherMultipleConsumers.class.getDeclaredField("selector"); + selectorField.setAccessible(true); + final StickyKeyConsumerSelector selector = spy((StickyKeyConsumerSelector) selectorField.get(dispatcher)); + selectorField.set(dispatcher, selector); + + // the selector returns consumer1 if keyA + doAnswer((invocationOnMock -> { + final int hash = invocationOnMock.getArgument(0); + + final String consumerName = hash == hashA ? consumer1Name : consumer2Name; + return dispatcher.getConsumers().stream().filter(consumer -> consumer.consumerName().equals(consumerName)).findFirst().get(); + })).when(selector).select(anyInt()); + + final AtomicInteger consumer2AckCounter = new AtomicInteger(); + @Cleanup + final Consumer consumer2 = pulsarClient.newConsumer() + .topic(topic) + .consumerName(consumer2Name) + .subscriptionType(SubscriptionType.Key_Shared) + .subscriptionName(subName) + .messageListener((c, msg) -> { + try { + c.acknowledge(msg); + consumer2AckCounter.getAndIncrement(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .subscribe(); + + final LongPairRangeSet.LongPairConsumer positionRangeConverter = PositionFactory::create; + final LongPairRangeSet expectedIndividuallySentPositions = new ConcurrentOpenLongPairRangeSet<>(4096, positionRangeConverter); + + TopicStats stats = admin.topics().getStats(topic); + Assert.assertEquals(stats.getSubscriptions().size(), 1); + SubscriptionStats subStats = stats.getSubscriptions().get(subName); + Assert.assertNotNull(subStats); + Assert.assertEquals(subStats.getIndividuallySentPositions(), expectedIndividuallySentPositions.toString()); + + final Function sendFn = (key) -> { + try { + return (MessageIdImpl) producer.newMessage().key(key).value(("msg").getBytes()).send(); + } catch (PulsarClientException e) { + throw new RuntimeException(e); + } + }; + final List messageIdList = new ArrayList<>(); + + // the dispatcher can send keyA message, but then consumer1's receiver queue will be full + messageIdList.add(sendFn.apply(keyA)); + + // the dispatcher can send messages other than keyA + messageIdList.add(sendFn.apply(keyA)); + messageIdList.add(sendFn.apply(keyB)); + messageIdList.add(sendFn.apply(keyA)); + messageIdList.add(sendFn.apply(keyB)); + messageIdList.add(sendFn.apply(keyB)); + + assertEquals(messageIdList.size(), 6); + Awaitility.await().untilAsserted(() -> assertEquals(consumer2AckCounter.get(), 3)); + + // set expected value + expectedIndividuallySentPositions.addOpenClosed(messageIdList.get(1).getLedgerId(), messageIdList.get(1).getEntryId(), + messageIdList.get(2).getLedgerId(), messageIdList.get(2).getEntryId()); + expectedIndividuallySentPositions.addOpenClosed(messageIdList.get(3).getLedgerId(), messageIdList.get(3).getEntryId(), + messageIdList.get(5).getLedgerId(), messageIdList.get(5).getEntryId()); + + stats = admin.topics().getStats(topic); + Assert.assertEquals(stats.getSubscriptions().size(), 1); + subStats = stats.getSubscriptions().get(subName); + Assert.assertNotNull(subStats); + Assert.assertEquals(subStats.getIndividuallySentPositions(), expectedIndividuallySentPositions.toString()); } @Test @@ -3550,4 +3862,16 @@ public void testRetentionAndBacklogQuotaCheck() throws PulsarAdminException { }); } + + @Test + @SneakyThrows + public void testPermissions() { + String namespace = "prop-xyz/ns1/"; + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://" + namespace + random; + final String subject = UUID.randomUUID().toString(); + assertThrows(NotFoundException.class, () -> admin.topics().getPermissions(topic)); + assertThrows(NotFoundException.class, () -> admin.topics().grantPermission(topic, subject, Set.of(AuthAction.produce))); + assertThrows(NotFoundException.class, () -> admin.topics().revokePermissions(topic, subject)); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTlsAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTlsAuthTest.java index 2b7b9101a81f2..f5d35d6baad8c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTlsAuthTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTlsAuthTest.java @@ -35,6 +35,7 @@ import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.GenericType; import javax.ws.rs.core.MediaType; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.mutable.MutableBoolean; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; @@ -197,6 +198,7 @@ public void testSuperUserCanUpdateScaleOfTransactionCoordinators() throws Except .getPartitionedTopicResources() .createPartitionedTopic(SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN, new PartitionedTopicMetadata(3)); + @Cleanup PulsarAdmin admin = buildAdminClient("admin"); admin.transactions().scaleTransactionCoordinators(4); int partitions = pulsar.getPulsarResources() @@ -472,6 +474,7 @@ public void testCertRefreshForPulsarAdmin() throws Exception { int autoCertRefreshTimeSec = 1; try { Files.copy(Paths.get(getTlsFileForClient(user2 + ".key-pk8")), keyFilePath, StandardCopyOption.REPLACE_EXISTING); + @Cleanup PulsarAdmin admin = PulsarAdmin.builder() .allowTlsInsecureConnection(false) .enableTlsHostnameVerification(false) @@ -506,8 +509,7 @@ public void testCertRefreshForPulsarAdmin() throws Exception { }, 5, 1000); Assert.assertTrue(success.booleanValue()); Assert.assertEquals(Set.of("tenantX"), admin.tenants().getTenants()); - admin.close(); - }finally { + } finally { Files.delete(keyFile.toPath()); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java index 046f2b4cf14c6..2894903c0d0c1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java @@ -302,7 +302,7 @@ public void clusters() throws Exception { NamespaceIsolationDataImpl policyData = NamespaceIsolationDataImpl.builder() .namespaces(Collections.singletonList("dummy/colo/ns")) - .primary(Collections.singletonList("localhost" + ":" + pulsar.getListenPortHTTP())) + .primary(Collections.singletonList(pulsar.getAdvertisedAddress())) .autoFailoverPolicy(AutoFailoverPolicyData.builder() .policyType(AutoFailoverPolicyType.min_available) .parameters(parameters1) @@ -722,11 +722,12 @@ public void brokers() throws Exception { assertTrue(res instanceof Set); Set activeBrokers = (Set) res; assertEquals(activeBrokers.size(), 1); - assertEquals(activeBrokers, Set.of(pulsar.getAdvertisedAddress() + ":" + pulsar.getListenPortHTTP().get())); + assertEquals(activeBrokers, Set.of(pulsar.getBrokerId())); Object leaderBrokerRes = asyncRequests(ctx -> brokers.getLeaderBroker(ctx)); assertTrue(leaderBrokerRes instanceof BrokerInfo); BrokerInfo leaderBroker = (BrokerInfo)leaderBrokerRes; - assertEquals(leaderBroker.getServiceUrl(), pulsar.getLeaderElectionService().getCurrentLeader().map(LeaderBroker::getServiceUrl).get()); + assertEquals(leaderBroker.getBrokerId(), + pulsar.getLeaderElectionService().getCurrentLeader().map(LeaderBroker::getBrokerId).get()); } @Test @@ -868,10 +869,10 @@ public void persistentTopics() throws Exception { Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); // verify permission response = mock(AsyncResponse.class); - responseCaptor = ArgumentCaptor.forClass(Response.class); + ArgumentCaptor>> permissionsCaptor = ArgumentCaptor.forClass(Map.class); persistentTopics.getPermissionsOnTopic(response, property, cluster, namespace, topic); - verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); - Map> permission = (Map>) responseCaptor.getValue(); + verify(response, timeout(5000).times(1)).resume(permissionsCaptor.capture()); + Map> permission = permissionsCaptor.getValue(); assertEquals(permission.get(role), actions); // remove permission response = mock(AsyncResponse.class); @@ -882,10 +883,10 @@ public void persistentTopics() throws Exception { // verify removed permission Awaitility.await().untilAsserted(() -> { AsyncResponse response1 = mock(AsyncResponse.class); - ArgumentCaptor responseCaptor1 = ArgumentCaptor.forClass(Response.class); + ArgumentCaptor>> permissionsCaptor1 = ArgumentCaptor.forClass(Map.class); persistentTopics.getPermissionsOnTopic(response1, property, cluster, namespace, topic); - verify(response1, timeout(5000).times(1)).resume(responseCaptor1.capture()); - Map> p = (Map>) responseCaptor1.getValue(); + verify(response1, timeout(5000).times(1)).resume(permissionsCaptor1.capture()); + Map> p = permissionsCaptor1.getValue(); assertTrue(p.isEmpty()); }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTopicApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTopicApiTest.java index 93bf2349103c3..0a334cd7e819e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTopicApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTopicApiTest.java @@ -19,21 +19,38 @@ package org.apache.pulsar.broker.admin; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import lombok.Cleanup; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerConsumerBase; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.policies.data.stats.NonPersistentTopicStatsImpl; import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; +import org.apache.pulsar.common.util.FutureUtil; +import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -59,6 +76,62 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + @Test + public void testDeleteNonExistTopic() throws Exception { + // Case 1: call delete for a partitioned topic. + final String topic1 = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createPartitionedTopic(topic1, 2); + admin.schemas().createSchemaAsync(topic1, Schema.STRING.getSchemaInfo()); + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.schemas().getAllSchemas(topic1).size(), 1); + }); + try { + admin.topics().delete(topic1); + fail("expected a 409 error"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("please call delete-partitioned-topic")); + } + Awaitility.await().pollDelay(Duration.ofSeconds(2)).untilAsserted(() -> { + assertEquals(admin.schemas().getAllSchemas(topic1).size(), 1); + }); + // cleanup. + admin.topics().deletePartitionedTopic(topic1, false); + + // Case 2: call delete-partitioned-topi for a non-partitioned topic. + final String topic2 = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createNonPartitionedTopic(topic2); + admin.schemas().createSchemaAsync(topic2, Schema.STRING.getSchemaInfo()); + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.schemas().getAllSchemas(topic2).size(), 1); + }); + try { + admin.topics().deletePartitionedTopic(topic2); + fail("expected a 409 error"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("Instead of calling delete-partitioned-topic please call delete")); + } + Awaitility.await().pollDelay(Duration.ofSeconds(2)).untilAsserted(() -> { + assertEquals(admin.schemas().getAllSchemas(topic2).size(), 1); + }); + // cleanup. + admin.topics().delete(topic2, false); + + // Case 3: delete topic does not exist. + final String topic3 = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + try { + admin.topics().delete(topic3); + fail("expected a 404 error"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("not found")); + } + try { + admin.topics().deletePartitionedTopic(topic3); + fail("expected a 404 error"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("not found")); + } + } + @Test public void testPeekMessages() throws Exception { @Cleanup @@ -140,4 +213,56 @@ public void testGetStats(String topic) throws Exception { } assertTrue(stats.getSubscriptions().containsKey(subscriptionName)); } + + @Test + public void testGetMessagesId() throws PulsarClientException, ExecutionException, InterruptedException { + String topic = newTopicName(); + + int numMessages = 10; + int batchingMaxMessages = numMessages / 2; + + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topic) + .enableBatching(true) + .batchingMaxMessages(batchingMaxMessages) + .batchingMaxPublishDelay(60, TimeUnit.SECONDS) + .create(); + + List> futures = new ArrayList<>(); + for (int i = 0; i < numMessages; i++) { + futures.add(producer.sendAsync(("msg-" + i).getBytes(UTF_8))); + } + FutureUtil.waitForAll(futures).get(); + + Map messageIdMap = new HashMap<>(); + futures.forEach(n -> { + try { + MessageId messageId = n.get(); + if (messageId instanceof MessageIdImpl impl) { + MessageIdImpl key = new MessageIdImpl(impl.getLedgerId(), impl.getEntryId(), -1); + Integer i = messageIdMap.computeIfAbsent(key, __ -> 0); + messageIdMap.put(key, i + 1); + } + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + }); + + messageIdMap.forEach((key, value) -> { + assertEquals(value, batchingMaxMessages); + try { + List> messages = admin.topics().getMessagesById(topic, + key.getLedgerId(), key.getEntryId()); + assertNotNull(messages); + assertEquals(messages.size(), batchingMaxMessages); + } catch (PulsarAdminException e) { + throw new RuntimeException(e); + } + }); + + // The message id doesn't exist. + assertThrows(PulsarAdminException.NotFoundException.class, () -> admin.topics() + .getMessagesById(topic, 1024, 2048)); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AnalyzeBacklogSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AnalyzeBacklogSubscriptionTest.java index 64b2a58ab86e8..f8aa3dc355d92 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AnalyzeBacklogSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AnalyzeBacklogSubscriptionTest.java @@ -154,17 +154,17 @@ private void verifyBacklog(String topic, String subscription, int numEntries, in AnalyzeSubscriptionBacklogResult analyzeSubscriptionBacklogResult = admin.topics().analyzeSubscriptionBacklog(topic, subscription, Optional.empty()); - assertEquals(numEntries, analyzeSubscriptionBacklogResult.getEntries()); - assertEquals(numEntries, analyzeSubscriptionBacklogResult.getFilterAcceptedEntries()); - assertEquals(0, analyzeSubscriptionBacklogResult.getFilterRejectedEntries()); - assertEquals(0, analyzeSubscriptionBacklogResult.getFilterRescheduledEntries()); - assertEquals(0, analyzeSubscriptionBacklogResult.getFilterRescheduledEntries()); + assertEquals(analyzeSubscriptionBacklogResult.getEntries(), numEntries); + assertEquals(analyzeSubscriptionBacklogResult.getFilterAcceptedEntries(), numEntries); + assertEquals(analyzeSubscriptionBacklogResult.getFilterRejectedEntries(), 0); + assertEquals(analyzeSubscriptionBacklogResult.getFilterRescheduledEntries(), 0); + assertEquals(analyzeSubscriptionBacklogResult.getFilterRescheduledEntries(), 0); - assertEquals(numMessages, analyzeSubscriptionBacklogResult.getMessages()); - assertEquals(numMessages, analyzeSubscriptionBacklogResult.getFilterAcceptedMessages()); - assertEquals(0, analyzeSubscriptionBacklogResult.getFilterRejectedMessages()); + assertEquals(analyzeSubscriptionBacklogResult.getMessages(), numMessages); + assertEquals(analyzeSubscriptionBacklogResult.getFilterAcceptedMessages(), numMessages); + assertEquals(analyzeSubscriptionBacklogResult.getFilterRejectedMessages(), 0); - assertEquals(0, analyzeSubscriptionBacklogResult.getFilterRescheduledMessages()); + assertEquals(analyzeSubscriptionBacklogResult.getFilterRescheduledMessages(), 0); assertFalse(analyzeSubscriptionBacklogResult.isAborted()); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AuthZTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AuthZTest.java new file mode 100644 index 0000000000000..3816b9a7a7ed0 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AuthZTest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin; + +import io.jsonwebtoken.Jwts; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.pulsar.broker.authorization.AuthorizationService; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.common.policies.data.NamespaceOperation; +import org.apache.pulsar.common.policies.data.TopicOperation; +import org.apache.pulsar.security.MockedPulsarStandalone; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import static org.mockito.Mockito.doReturn; + +public class AuthZTest extends MockedPulsarStandalone { + + protected PulsarAdmin superUserAdmin; + + protected PulsarAdmin tenantManagerAdmin; + + protected AuthorizationService authorizationService; + + protected AuthorizationService orignalAuthorizationService; + + protected static final String TENANT_ADMIN_SUBJECT = UUID.randomUUID().toString(); + protected static final String TENANT_ADMIN_TOKEN = Jwts.builder() + .claim("sub", TENANT_ADMIN_SUBJECT).signWith(SECRET_KEY).compact(); + + @Override + public void close() throws Exception { + if (superUserAdmin != null) { + superUserAdmin.close(); + superUserAdmin = null; + } + if (tenantManagerAdmin != null) { + tenantManagerAdmin.close(); + tenantManagerAdmin = null; + } + authorizationService = null; + orignalAuthorizationService = null; + super.close(); + } + + @BeforeMethod(alwaysRun = true) + public void before() throws IllegalAccessException { + orignalAuthorizationService = getPulsarService().getBrokerService().getAuthorizationService(); + authorizationService = Mockito.spy(orignalAuthorizationService); + FieldUtils.writeField(getPulsarService().getBrokerService(), "authorizationService", + authorizationService, true); + } + + @AfterMethod(alwaysRun = true) + public void after() throws IllegalAccessException { + FieldUtils.writeField(getPulsarService().getBrokerService(), "authorizationService", + orignalAuthorizationService, true); + } + + protected AtomicBoolean setAuthorizationTopicOperationChecker(String role, Object operation) { + AtomicBoolean execFlag = new AtomicBoolean(false); + if (operation instanceof TopicOperation) { + Mockito.doAnswer(invocationOnMock -> { + String role_ = invocationOnMock.getArgument(2); + if (role.equals(role_)) { + TopicOperation operation_ = invocationOnMock.getArgument(1); + Assert.assertEquals(operation_, operation); + } + execFlag.set(true); + return invocationOnMock.callRealMethod(); + }).when(authorizationService).allowTopicOperationAsync(Mockito.any(), Mockito.any(), Mockito.any(), + Mockito.any(), Mockito.any()); + } else if (operation instanceof NamespaceOperation) { + doReturn(true) + .when(authorizationService).isValidOriginalPrincipal(Mockito.any(), Mockito.any(), Mockito.any()); + Mockito.doAnswer(invocationOnMock -> { + String role_ = invocationOnMock.getArgument(2); + if (role.equals(role_)) { + TopicOperation operation_ = invocationOnMock.getArgument(1); + Assert.assertEquals(operation_, operation); + } + execFlag.set(true); + return invocationOnMock.callRealMethod(); + }).when(authorizationService).allowNamespaceOperationAsync(Mockito.any(), Mockito.any(), Mockito.any(), + Mockito.any(), Mockito.any()); + } else { + throw new IllegalArgumentException(""); + } + + + return execFlag; + } + + protected void createTopic(String topic, boolean partitioned) throws Exception { + if (partitioned) { + superUserAdmin.topics().createPartitionedTopic(topic, 2); + } else { + superUserAdmin.topics().createNonPartitionedTopic(topic); + } + } + + protected void deleteTopic(String topic, boolean partitioned) throws Exception { + if (partitioned) { + superUserAdmin.topics().deletePartitionedTopic(topic, true); + } else { + superUserAdmin.topics().delete(topic, true); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BookiesApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BookiesApiTest.java index 7b9d4344a4ce5..1bd1de2130b11 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BookiesApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/BookiesApiTest.java @@ -66,6 +66,16 @@ public void testBasic() throws Exception { fail("should not reach here"); } catch (PulsarAdminException pae) { assertEquals(404, pae.getStatusCode()); + assertEquals(pae.getHttpError(), "Bookie rack placement configuration not found: " + bookie0); + } + + // delete bookie doesn't exist + try { + admin.bookies().deleteBookieRackInfo(bookie0); + fail("should not reach here"); + } catch (PulsarAdminException pae) { + assertEquals(404, pae.getStatusCode()); + assertEquals(pae.getHttpError(), "Bookie rack placement configuration not found: " + bookie0); } // update the bookie info diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/CreateSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/CreateSubscriptionTest.java index 2064559888286..c121ea44387fe 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/CreateSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/CreateSubscriptionTest.java @@ -31,6 +31,7 @@ import java.util.concurrent.TimeUnit; import javax.ws.rs.ClientErrorException; import javax.ws.rs.core.Response.Status; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.http.HttpResponse; @@ -431,6 +432,7 @@ public void createSubscriptionBySpecifyingStringPosition() throws IOException, P final int numberOfMessages = 5; String topic = "persistent://my-property/my-ns/my-topic"; RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(30 * 1000).build(); + @Cleanup CloseableHttpClient httpClient = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).build(); // Produce some messages to pulsar diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/GetPartitionMetadataMultiBrokerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/GetPartitionMetadataMultiBrokerTest.java new file mode 100644 index 0000000000000..60691203e777d --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/GetPartitionMetadataMultiBrokerTest.java @@ -0,0 +1,310 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import java.lang.reflect.Field; +import java.net.URL; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.impl.ClientCnx; +import org.apache.pulsar.client.impl.ConnectionPool; +import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.common.naming.TopicDomain; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.partition.PartitionedTopicMetadata; +import org.apache.pulsar.common.policies.data.TopicType; +import org.apache.pulsar.common.util.FutureUtil; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Test(groups = "broker-admin") +@Slf4j +public class GetPartitionMetadataMultiBrokerTest extends GetPartitionMetadataTest { + + private PulsarService pulsar2; + private URL url2; + private PulsarAdmin admin2; + private PulsarClientImpl clientWithHttpLookup2; + private PulsarClientImpl clientWitBinaryLookup2; + + @BeforeClass(alwaysRun = true) + protected void setup() throws Exception { + super.setup(); + } + + @Override + @AfterClass(alwaysRun = true) + protected void cleanup() throws Exception { + super.cleanup(); + } + + @Override + protected void cleanupBrokers() throws Exception { + // Cleanup broker2. + if (clientWithHttpLookup2 != null) { + clientWithHttpLookup2.close(); + clientWithHttpLookup2 = null; + } + if (clientWitBinaryLookup2 != null) { + clientWitBinaryLookup2.close(); + clientWitBinaryLookup2 = null; + } + if (admin2 != null) { + admin2.close(); + admin2 = null; + } + if (pulsar2 != null) { + pulsar2.close(); + pulsar2 = null; + } + + // Super cleanup. + super.cleanupBrokers(); + } + + @Override + protected void setupBrokers() throws Exception { + super.setupBrokers(); + doInitConf(); + pulsar2 = new PulsarService(conf); + pulsar2.start(); + url2 = new URL(pulsar2.getWebServiceAddress()); + admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build(); + clientWithHttpLookup2 = + (PulsarClientImpl) PulsarClient.builder().serviceUrl(pulsar2.getWebServiceAddress()).build(); + clientWitBinaryLookup2 = + (PulsarClientImpl) PulsarClient.builder().serviceUrl(pulsar2.getBrokerServiceUrl()).build(); + } + + @Override + protected PulsarClientImpl[] getClientsToTest() { + return new PulsarClientImpl[] {clientWithHttpLookup1, clientWitBinaryLookup1, + clientWithHttpLookup2, clientWitBinaryLookup2}; + } + + protected PulsarClientImpl[] getClientsToTest(boolean isUsingHttpLookup) { + if (isUsingHttpLookup) { + return new PulsarClientImpl[]{clientWithHttpLookup1, clientWithHttpLookup2}; + } else { + return new PulsarClientImpl[]{clientWitBinaryLookup1, clientWitBinaryLookup2}; + } + } + + @Override + protected int getLookupRequestPermits() { + return pulsar1.getBrokerService().getLookupRequestSemaphore().availablePermits() + + pulsar2.getBrokerService().getLookupRequestSemaphore().availablePermits(); + } + + protected void verifyPartitionsNeverCreated(String topicNameStr) throws Exception { + TopicName topicName = TopicName.get(topicNameStr); + try { + List topicList = admin1.topics().getList("public/default"); + for (int i = 0; i < 3; i++) { + assertFalse(topicList.contains(topicName.getPartition(i))); + } + } catch (Exception ex) { + // If the namespace bundle has not been loaded yet, it means no non-persistent topic was created. So + // this behavior is also correct. + // This error is not expected, a seperated PR is needed to fix this issue. + assertTrue(ex.getMessage().contains("Failed to find ownership for")); + } + } + + protected void verifyNonPartitionedTopicNeverCreated(String topicNameStr) throws Exception { + TopicName topicName = TopicName.get(topicNameStr); + try { + List topicList = admin1.topics().getList("public/default"); + assertFalse(topicList.contains(topicName.getPartitionedTopicName())); + } catch (Exception ex) { + // If the namespace bundle has not been loaded yet, it means no non-persistent topic was created. So + // this behavior is also correct. + // This error is not expected, a seperated PR is needed to fix this issue. + assertTrue(ex.getMessage().contains("Failed to find ownership for")); + } + } + + protected void modifyTopicAutoCreation(boolean allowAutoTopicCreation, + TopicType allowAutoTopicCreationType, + int defaultNumPartitions) throws Exception { + doModifyTopicAutoCreation(admin1, pulsar1, allowAutoTopicCreation, allowAutoTopicCreationType, + defaultNumPartitions); + doModifyTopicAutoCreation(admin2, pulsar2, allowAutoTopicCreation, allowAutoTopicCreationType, + defaultNumPartitions); + } + + /** + * {@inheritDoc} + */ + @Test(dataProvider = "topicDomains") + public void testAutoCreatingMetadataWhenCallingOldAPI(TopicDomain topicDomain) throws Exception { + super.testAutoCreatingMetadataWhenCallingOldAPI(topicDomain); + } + + /** + * {@inheritDoc} + */ + @Test(dataProvider = "autoCreationParamsAll", enabled = false) + public void testGetMetadataIfNonPartitionedTopicExists(boolean configAllowAutoTopicCreation, + boolean paramMetadataAutoCreationEnabled, + boolean isUsingHttpLookup, + TopicDomain topicDomain) throws Exception { + super.testGetMetadataIfNonPartitionedTopicExists(configAllowAutoTopicCreation, paramMetadataAutoCreationEnabled, + isUsingHttpLookup, topicDomain); + } + + /** + * {@inheritDoc} + */ + @Test(dataProvider = "autoCreationParamsAll") + public void testGetMetadataIfPartitionedTopicExists(boolean configAllowAutoTopicCreation, + boolean paramMetadataAutoCreationEnabled, + boolean isUsingHttpLookup, + TopicDomain topicDomain) throws Exception { + super.testGetMetadataIfNonPartitionedTopicExists(configAllowAutoTopicCreation, paramMetadataAutoCreationEnabled, + isUsingHttpLookup, topicDomain); + } + + /** + * {@inheritDoc} + */ + @Test(dataProvider = "clients") + public void testAutoCreatePartitionedTopic(boolean isUsingHttpLookup, TopicDomain topicDomain) throws Exception { + super.testAutoCreatePartitionedTopic(isUsingHttpLookup, topicDomain); + } + + /** + * {@inheritDoc} + */ + @Test(dataProvider = "clients") + public void testAutoCreateNonPartitionedTopic(boolean isUsingHttpLookup, TopicDomain topicDomain) throws Exception { + super.testAutoCreateNonPartitionedTopic(isUsingHttpLookup, topicDomain); + } + + /** + * {@inheritDoc} + */ + @Test(dataProvider = "autoCreationParamsNotAllow") + public void testGetMetadataIfNotAllowedCreate(boolean configAllowAutoTopicCreation, + boolean paramMetadataAutoCreationEnabled, + boolean isUsingHttpLookup) throws Exception { + super.testGetMetadataIfNotAllowedCreate(configAllowAutoTopicCreation, paramMetadataAutoCreationEnabled, + isUsingHttpLookup); + } + + /** + * {@inheritDoc} + */ + @Test(dataProvider = "autoCreationParamsNotAllow") + public void testGetMetadataIfNotAllowedCreateOfNonPersistentTopic(boolean configAllowAutoTopicCreation, + boolean paramMetadataAutoCreationEnabled, + boolean isUsingHttpLookup) throws Exception { + super.testGetMetadataIfNotAllowedCreateOfNonPersistentTopic(configAllowAutoTopicCreation, + paramMetadataAutoCreationEnabled, isUsingHttpLookup); + } + + @DataProvider(name = "autoCreationParamsAllForNonPersistentTopic") + public Object[][] autoCreationParamsAllForNonPersistentTopic(){ + return new Object[][]{ + // configAllowAutoTopicCreation, paramCreateIfAutoCreationEnabled, isUsingHttpLookup. + {true, true, true}, + {true, true, false}, + {true, false, true}, + {true, false, false}, + {false, true, true}, + {false, true, false}, + {false, false, true}, + {false, false, false} + }; + } + + @Test(dataProvider = "autoCreationParamsAllForNonPersistentTopic", priority = Integer.MAX_VALUE) + public void testCompatibilityDifferentBrokersForNonPersistentTopic(boolean configAllowAutoTopicCreation, + boolean paramMetadataAutoCreationEnabled, + boolean isUsingHttpLookup) throws Exception { + modifyTopicAutoCreation(configAllowAutoTopicCreation, TopicType.PARTITIONED, 3); + + // Initialize the connections of internal Pulsar Client. + PulsarClientImpl client1 = (PulsarClientImpl) pulsar1.getClient(); + PulsarClientImpl client2 = (PulsarClientImpl) pulsar2.getClient(); + client1.getLookup(pulsar2.getBrokerServiceUrl()).getBroker(TopicName.get(DEFAULT_NS + "/tp1")); + client2.getLookup(pulsar1.getBrokerServiceUrl()).getBroker(TopicName.get(DEFAULT_NS + "/tp1")); + + // Inject a not support flag into the connections initialized. + Field field = ClientCnx.class.getDeclaredField("supportsGetPartitionedMetadataWithoutAutoCreation"); + field.setAccessible(true); + for (PulsarClientImpl client : Arrays.asList(client1, client2)) { + ConnectionPool pool = client.getCnxPool(); + for (CompletableFuture connectionFuture : pool.getConnections()) { + ClientCnx clientCnx = connectionFuture.join(); + clientCnx.isSupportsGetPartitionedMetadataWithoutAutoCreation(); + field.set(clientCnx, false); + } + } + // Verify: the method "getPartitionsForTopic(topic, false, true)" will fallback + // to "getPartitionsForTopic(topic, true)" behavior. + int lookupPermitsBefore = getLookupRequestPermits(); + + // Verify: we will not get an un-support error. + PulsarClientImpl[] clientArray = getClientsToTest(isUsingHttpLookup); + for (PulsarClientImpl client : clientArray) { + final String topicNameStr = BrokerTestUtil.newUniqueName("non-persistent://" + DEFAULT_NS + "/tp"); + try { + PartitionedTopicMetadata topicMetadata = client + .getPartitionedTopicMetadata(topicNameStr, paramMetadataAutoCreationEnabled, false) + .join(); + log.info("Get topic metadata: {}", topicMetadata.partitions); + } catch (Exception ex) { + Throwable unwrapEx = FutureUtil.unwrapCompletionException(ex); + assertTrue(unwrapEx instanceof PulsarClientException.TopicDoesNotExistException + || unwrapEx instanceof PulsarClientException.NotFoundException); + assertFalse(ex.getMessage().contains("getting partitions without auto-creation is not supported from" + + " the broker")); + } + } + + // Verify: lookup semaphore has been releases. + Awaitility.await().untilAsserted(() -> { + assertEquals(getLookupRequestPermits(), lookupPermitsBefore); + }); + + // reset clients. + for (PulsarClientImpl client : Arrays.asList(client1, client2)) { + ConnectionPool pool = client.getCnxPool(); + for (CompletableFuture connectionFuture : pool.getConnections()) { + ClientCnx clientCnx = connectionFuture.join(); + clientCnx.isSupportsGetPartitionedMetadataWithoutAutoCreation(); + field.set(clientCnx, true); + } + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/GetPartitionMetadataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/GetPartitionMetadataTest.java new file mode 100644 index 0000000000000..e9a639697d9ff --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/GetPartitionMetadataTest.java @@ -0,0 +1,632 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import com.google.common.collect.Sets; +import java.lang.reflect.Field; +import java.net.URL; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.impl.ClientCnx; +import org.apache.pulsar.client.impl.ConnectionPool; +import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicDomain; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.partition.PartitionedTopicMetadata; +import org.apache.pulsar.common.policies.data.ClusterDataImpl; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.policies.data.TopicType; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Test(groups = "broker-admin") +@Slf4j +public class GetPartitionMetadataTest { + + protected static final String DEFAULT_NS = "public/default"; + + protected String clusterName = "c1"; + + protected LocalBookkeeperEnsemble bkEnsemble; + + protected ServiceConfiguration conf = new ServiceConfiguration(); + + protected PulsarService pulsar1; + protected URL url1; + protected PulsarAdmin admin1; + protected PulsarClientImpl clientWithHttpLookup1; + protected PulsarClientImpl clientWitBinaryLookup1; + + @BeforeClass(alwaysRun = true) + protected void setup() throws Exception { + bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); + bkEnsemble.start(); + // Start broker. + setupBrokers(); + // Create default NS. + admin1.clusters().createCluster(clusterName, new ClusterDataImpl()); + admin1.tenants().createTenant(NamespaceName.get(DEFAULT_NS).getTenant(), + new TenantInfoImpl(Collections.emptySet(), Sets.newHashSet(clusterName))); + admin1.namespaces().createNamespace(DEFAULT_NS); + } + + @AfterClass(alwaysRun = true) + protected void cleanup() throws Exception { + cleanupBrokers(); + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; + } + } + + protected void cleanupBrokers() throws Exception { + // Cleanup broker2. + if (clientWithHttpLookup1 != null) { + clientWithHttpLookup1.close(); + clientWithHttpLookup1 = null; + } + if (clientWitBinaryLookup1 != null) { + clientWitBinaryLookup1.close(); + clientWitBinaryLookup1 = null; + } + if (admin1 != null) { + admin1.close(); + admin1 = null; + } + if (pulsar1 != null) { + pulsar1.close(); + pulsar1 = null; + } + // Reset configs. + conf = new ServiceConfiguration(); + } + + protected void setupBrokers() throws Exception { + doInitConf(); + // Start broker. + pulsar1 = new PulsarService(conf); + pulsar1.start(); + url1 = new URL(pulsar1.getWebServiceAddress()); + admin1 = PulsarAdmin.builder().serviceHttpUrl(url1.toString()).build(); + clientWithHttpLookup1 = + (PulsarClientImpl) PulsarClient.builder().serviceUrl(pulsar1.getWebServiceAddress()).build(); + clientWitBinaryLookup1 = + (PulsarClientImpl) PulsarClient.builder().serviceUrl(pulsar1.getBrokerServiceUrl()).build(); + } + + protected void doInitConf() { + conf.setClusterName(clusterName); + conf.setAdvertisedAddress("localhost"); + conf.setBrokerServicePort(Optional.of(0)); + conf.setWebServicePort(Optional.of(0)); + conf.setMetadataStoreUrl("zk:127.0.0.1:" + bkEnsemble.getZookeeperPort()); + conf.setConfigurationMetadataStoreUrl("zk:127.0.0.1:" + bkEnsemble.getZookeeperPort() + "/foo"); + conf.setBrokerDeleteInactiveTopicsEnabled(false); + conf.setBrokerShutdownTimeoutMs(0L); + conf.setLoadBalancerSheddingEnabled(false); + } + + protected PulsarClientImpl[] getClientsToTest() { + return new PulsarClientImpl[] {clientWithHttpLookup1, clientWitBinaryLookup1}; + } + + protected PulsarClientImpl[] getClientsToTest(boolean isUsingHttpLookup) { + if (isUsingHttpLookup) { + return new PulsarClientImpl[] {clientWithHttpLookup1}; + } else { + return new PulsarClientImpl[] {clientWitBinaryLookup1}; + } + + } + + protected int getLookupRequestPermits() { + return pulsar1.getBrokerService().getLookupRequestSemaphore().availablePermits(); + } + + protected void verifyPartitionsNeverCreated(String topicNameStr) throws Exception { + TopicName topicName = TopicName.get(topicNameStr); + List topicList = admin1.topics().getList("public/default"); + for (int i = 0; i < 3; i++) { + assertFalse(topicList.contains(topicName.getPartition(i))); + } + } + + protected void verifyNonPartitionedTopicNeverCreated(String topicNameStr) throws Exception { + TopicName topicName = TopicName.get(topicNameStr); + List topicList = admin1.topics().getList("public/default"); + assertFalse(topicList.contains(topicName.getPartitionedTopicName())); + } + + @DataProvider(name = "topicDomains") + public Object[][] topicDomains() { + return new Object[][]{ + {TopicDomain.persistent}, + {TopicDomain.non_persistent} + }; + } + + protected static void doModifyTopicAutoCreation(PulsarAdmin admin1, PulsarService pulsar1, + boolean allowAutoTopicCreation, TopicType allowAutoTopicCreationType, + int defaultNumPartitions) throws Exception { + admin1.brokers().updateDynamicConfiguration( + "allowAutoTopicCreation", allowAutoTopicCreation + ""); + admin1.brokers().updateDynamicConfiguration( + "allowAutoTopicCreationType", allowAutoTopicCreationType + ""); + admin1.brokers().updateDynamicConfiguration( + "defaultNumPartitions", defaultNumPartitions + ""); + Awaitility.await().untilAsserted(() -> { + assertEquals(pulsar1.getConfiguration().isAllowAutoTopicCreation(), allowAutoTopicCreation); + assertEquals(pulsar1.getConfiguration().getAllowAutoTopicCreationType(), allowAutoTopicCreationType); + assertEquals(pulsar1.getConfiguration().getDefaultNumPartitions(), defaultNumPartitions); + }); + } + + protected void modifyTopicAutoCreation(boolean allowAutoTopicCreation, + TopicType allowAutoTopicCreationType, + int defaultNumPartitions) throws Exception { + doModifyTopicAutoCreation(admin1, pulsar1, allowAutoTopicCreation, allowAutoTopicCreationType, + defaultNumPartitions); + } + + @Test(dataProvider = "topicDomains") + public void testAutoCreatingMetadataWhenCallingOldAPI(TopicDomain topicDomain) throws Exception { + modifyTopicAutoCreation(true, TopicType.PARTITIONED, 3); + + int lookupPermitsBefore = getLookupRequestPermits(); + + for (PulsarClientImpl client : getClientsToTest()) { + // Verify: the behavior of topic creation. + final String tp = BrokerTestUtil.newUniqueName(topicDomain.value() + "://" + DEFAULT_NS + "/tp"); + client.getPartitionsForTopic(tp).join(); + Optional metadata1 = pulsar1.getPulsarResources().getNamespaceResources() + .getPartitionedTopicResources() + .getPartitionedTopicMetadataAsync(TopicName.get(tp), true).join(); + assertTrue(metadata1.isPresent()); + assertEquals(metadata1.get().partitions, 3); + + // Verify: lookup semaphore has been releases. + Awaitility.await().untilAsserted(() -> { + assertEquals(getLookupRequestPermits(), lookupPermitsBefore); + }); + + // Cleanup. + admin1.topics().deletePartitionedTopic(tp, false); + } + } + + @Test(dataProvider = "topicDomains", priority = Integer.MAX_VALUE) + public void testCompatibilityForNewClientAndOldBroker(TopicDomain topicDomain) throws Exception { + modifyTopicAutoCreation(true, TopicType.PARTITIONED, 3); + // Initialize connections. + String pulsarUrl = pulsar1.getBrokerServiceUrl(); + PulsarClientImpl[] clients = getClientsToTest(false); + for (PulsarClientImpl client : clients) { + client.getLookup(pulsarUrl).getBroker(TopicName.get(DEFAULT_NS + "/tp1")); + } + // Inject a not support flag into the connections initialized. + Field field = ClientCnx.class.getDeclaredField("supportsGetPartitionedMetadataWithoutAutoCreation"); + field.setAccessible(true); + for (PulsarClientImpl client : clients) { + ConnectionPool pool = client.getCnxPool(); + for (CompletableFuture connectionFuture : pool.getConnections()) { + ClientCnx clientCnx = connectionFuture.join(); + clientCnx.isSupportsGetPartitionedMetadataWithoutAutoCreation(); + field.set(clientCnx, false); + } + } + + // Verify: the method "getPartitionsForTopic(topic, false, true)" will fallback to + // "getPartitionsForTopic(topic)" behavior. + int lookupPermitsBefore = getLookupRequestPermits(); + for (PulsarClientImpl client : clients) { + // Verify: the behavior of topic creation. + final String tp = BrokerTestUtil.newUniqueName(topicDomain.value() + "://" + DEFAULT_NS + "/tp"); + client.getPartitionedTopicMetadata(tp, false, true).join(); + Optional metadata1 = pulsar1.getPulsarResources().getNamespaceResources() + .getPartitionedTopicResources() + .getPartitionedTopicMetadataAsync(TopicName.get(tp), true).join(); + assertTrue(metadata1.isPresent()); + assertEquals(metadata1.get().partitions, 3); + + // Verify: lookup semaphore has been releases. + Awaitility.await().untilAsserted(() -> { + assertEquals(getLookupRequestPermits(), lookupPermitsBefore); + }); + + // Cleanup. + admin1.topics().deletePartitionedTopic(tp, false); + } + + // reset clients. + for (PulsarClientImpl client : clients) { + ConnectionPool pool = client.getCnxPool(); + for (CompletableFuture connectionFuture : pool.getConnections()) { + ClientCnx clientCnx = connectionFuture.join(); + clientCnx.isSupportsGetPartitionedMetadataWithoutAutoCreation(); + field.set(clientCnx, true); + } + } + } + + @DataProvider(name = "autoCreationParamsAll") + public Object[][] autoCreationParamsAll(){ + return new Object[][]{ + // configAllowAutoTopicCreation, paramCreateIfAutoCreationEnabled, isUsingHttpLookup. + {true, true, true, TopicDomain.persistent}, + {true, true, false, TopicDomain.persistent}, + {true, false, true, TopicDomain.persistent}, + {true, false, false, TopicDomain.persistent}, + {false, true, true, TopicDomain.persistent}, + {false, true, false, TopicDomain.persistent}, + {false, false, true, TopicDomain.persistent}, + {false, false, false, TopicDomain.persistent}, + {true, true, true, TopicDomain.non_persistent}, + {true, true, false, TopicDomain.non_persistent}, + {true, false, true, TopicDomain.non_persistent}, + {true, false, false, TopicDomain.non_persistent}, + {false, true, true, TopicDomain.non_persistent}, + {false, true, false, TopicDomain.non_persistent}, + {false, false, true, TopicDomain.non_persistent}, + {false, false, false, TopicDomain.non_persistent} + }; + } + + @Test(dataProvider = "autoCreationParamsAll") + public void testGetMetadataIfNonPartitionedTopicExists(boolean configAllowAutoTopicCreation, + boolean paramMetadataAutoCreationEnabled, + boolean isUsingHttpLookup, + TopicDomain topicDomain) throws Exception { + modifyTopicAutoCreation(configAllowAutoTopicCreation, TopicType.PARTITIONED, 3); + + int lookupPermitsBefore = getLookupRequestPermits(); + + // Create topic. + final String topicNameStr = BrokerTestUtil.newUniqueName(topicDomain.value() + "://" + DEFAULT_NS + "/tp_"); + admin1.topics().createNonPartitionedTopic(topicNameStr); + + PulsarClientImpl[] clientArray = getClientsToTest(isUsingHttpLookup); + for (PulsarClientImpl client : clientArray) { + // Verify: the result of get partitioned topic metadata. + PartitionedTopicMetadata response = + client.getPartitionedTopicMetadata(topicNameStr, paramMetadataAutoCreationEnabled, false).join(); + assertEquals(response.partitions, 0); + List partitionedTopics = admin1.topics().getPartitionedTopicList("public/default"); + assertFalse(partitionedTopics.contains(topicNameStr)); + verifyPartitionsNeverCreated(topicNameStr); + + // Verify: lookup semaphore has been releases. + Awaitility.await().untilAsserted(() -> { + assertEquals(getLookupRequestPermits(), lookupPermitsBefore); + }); + } + + // Cleanup. + admin1.topics().delete(topicNameStr, false); + } + + @Test(dataProvider = "autoCreationParamsAll") + public void testGetMetadataIfPartitionedTopicExists(boolean configAllowAutoTopicCreation, + boolean paramMetadataAutoCreationEnabled, + boolean isUsingHttpLookup, + TopicDomain topicDomain) throws Exception { + modifyTopicAutoCreation(configAllowAutoTopicCreation, TopicType.PARTITIONED, 3); + + int lookupPermitsBefore = getLookupRequestPermits(); + + // Create topic. + final String topicNameStr = BrokerTestUtil.newUniqueName(topicDomain.value() + "://" + DEFAULT_NS + "/tp"); + admin1.topics().createPartitionedTopic(topicNameStr, 3); + + PulsarClientImpl[] clientArray = getClientsToTest(isUsingHttpLookup); + for (PulsarClientImpl client : clientArray) { + // Verify: the result of get partitioned topic metadata. + PartitionedTopicMetadata response = + client.getPartitionedTopicMetadata(topicNameStr, paramMetadataAutoCreationEnabled, false).join(); + assertEquals(response.partitions, 3); + verifyNonPartitionedTopicNeverCreated(topicNameStr); + + // Verify: lookup semaphore has been releases. + Awaitility.await().untilAsserted(() -> { + assertEquals(getLookupRequestPermits(), lookupPermitsBefore); + }); + } + + // Cleanup. + admin1.topics().deletePartitionedTopic(topicNameStr, false); + } + + @DataProvider(name = "clients") + public Object[][] clients(){ + return new Object[][]{ + // isUsingHttpLookup. + {true, TopicDomain.persistent}, + {false, TopicDomain.non_persistent} + }; + } + + @Test(dataProvider = "clients") + public void testAutoCreatePartitionedTopic(boolean isUsingHttpLookup, TopicDomain topicDomain) throws Exception { + modifyTopicAutoCreation(true, TopicType.PARTITIONED, 3); + + int lookupPermitsBefore = getLookupRequestPermits(); + + PulsarClientImpl[] clientArray = getClientsToTest(isUsingHttpLookup); + for (PulsarClientImpl client : clientArray) { + // Case-1: normal topic. + final String topicNameStr = BrokerTestUtil.newUniqueName(topicDomain.value() + "://" + DEFAULT_NS + "/tp"); + // Verify: the result of get partitioned topic metadata. + PartitionedTopicMetadata response = client.getPartitionedTopicMetadata(topicNameStr, true, false).join(); + assertEquals(response.partitions, 3); + // Verify: the behavior of topic creation. + List partitionedTopics = admin1.topics().getPartitionedTopicList("public/default"); + assertTrue(partitionedTopics.contains(topicNameStr)); + verifyNonPartitionedTopicNeverCreated(topicNameStr); + // The API "getPartitionedTopicMetadata" only creates the partitioned metadata, it will not create the + // partitions. + verifyPartitionsNeverCreated(topicNameStr); + + // Case-2: topic with suffix "-partition-1". + final String topicNameStrWithSuffix = BrokerTestUtil.newUniqueName( + topicDomain.value() + "://" + DEFAULT_NS + "/tp") + "-partition-1"; + // Verify: the result of get partitioned topic metadata. + PartitionedTopicMetadata response2 = + client.getPartitionedTopicMetadata(topicNameStrWithSuffix, true, false).join(); + assertEquals(response2.partitions, 0); + // Verify: the behavior of topic creation. + List partitionedTopics2 = + admin1.topics().getPartitionedTopicList("public/default"); + assertFalse(partitionedTopics2.contains(topicNameStrWithSuffix)); + assertFalse(partitionedTopics2.contains( + TopicName.get(topicNameStrWithSuffix).getPartitionedTopicName())); + + // Verify: lookup semaphore has been releases. + Awaitility.await().untilAsserted(() -> { + assertEquals(getLookupRequestPermits(), lookupPermitsBefore); + }); + // Cleanup. + admin1.topics().deletePartitionedTopic(topicNameStr, false); + try { + admin1.topics().delete(topicNameStrWithSuffix, false); + } catch (Exception ex) {} + } + + } + + @Test(dataProvider = "clients") + public void testAutoCreateNonPartitionedTopic(boolean isUsingHttpLookup, TopicDomain topicDomain) throws Exception { + modifyTopicAutoCreation(true, TopicType.NON_PARTITIONED, 3); + + int lookupPermitsBefore = getLookupRequestPermits(); + + PulsarClientImpl[] clientArray = getClientsToTest(isUsingHttpLookup); + for (PulsarClientImpl client : clientArray) { + // Case 1: normal topic. + final String topicNameStr = BrokerTestUtil.newUniqueName(topicDomain.value() + "://" + DEFAULT_NS + "/tp"); + // Verify: the result of get partitioned topic metadata. + PartitionedTopicMetadata response = client.getPartitionedTopicMetadata(topicNameStr, true, false).join(); + assertEquals(response.partitions, 0); + // Verify: the behavior of topic creation. + List partitionedTopics = admin1.topics().getPartitionedTopicList("public/default"); + assertFalse(partitionedTopics.contains(topicNameStr)); + verifyPartitionsNeverCreated(topicNameStr); + + // Case-2: topic with suffix "-partition-1". + final String topicNameStrWithSuffix = BrokerTestUtil.newUniqueName( + topicDomain.value() + "://" + DEFAULT_NS + "/tp") + "-partition-1"; + // Verify: the result of get partitioned topic metadata. + PartitionedTopicMetadata response2 = + client.getPartitionedTopicMetadata(topicNameStrWithSuffix, true, false).join(); + assertEquals(response2.partitions, 0); + // Verify: the behavior of topic creation. + List partitionedTopics2 = + admin1.topics().getPartitionedTopicList("public/default"); + assertFalse(partitionedTopics2.contains(topicNameStrWithSuffix)); + assertFalse(partitionedTopics2.contains( + TopicName.get(topicNameStrWithSuffix).getPartitionedTopicName())); + + // Verify: lookup semaphore has been releases. + Awaitility.await().untilAsserted(() -> { + assertEquals(getLookupRequestPermits(), lookupPermitsBefore); + }); + // Cleanup. + try { + admin1.topics().delete(topicNameStr, false); + } catch (Exception ex) {} + try { + admin1.topics().delete(topicNameStrWithSuffix, false); + } catch (Exception ex) {} + } + } + + @DataProvider(name = "autoCreationParamsNotAllow") + public Object[][] autoCreationParamsNotAllow(){ + return new Object[][]{ + // configAllowAutoTopicCreation, paramCreateIfAutoCreationEnabled, isUsingHttpLookup. + {true, false, true}, + {true, false, false}, + {false, false, true}, + {false, false, false}, + {false, true, true}, + {false, true, false}, + }; + } + + @Test(dataProvider = "autoCreationParamsNotAllow") + public void testGetMetadataIfNotAllowedCreate(boolean configAllowAutoTopicCreation, + boolean paramMetadataAutoCreationEnabled, + boolean isUsingHttpLookup) throws Exception { + modifyTopicAutoCreation(configAllowAutoTopicCreation, TopicType.PARTITIONED, 3); + + int lookupPermitsBefore = getLookupRequestPermits(); + + PulsarClientImpl[] clientArray = getClientsToTest(isUsingHttpLookup); + for (PulsarClientImpl client : clientArray) { + // Define topic. + final String topicNameStr = BrokerTestUtil.newUniqueName("persistent://" + DEFAULT_NS + "/tp"); + final TopicName topicName = TopicName.get(topicNameStr); + // Verify: the result of get partitioned topic metadata. + try { + client.getPartitionedTopicMetadata(topicNameStr, paramMetadataAutoCreationEnabled, false) + .join(); + fail("Expect a not found exception"); + } catch (Exception e) { + Throwable unwrapEx = FutureUtil.unwrapCompletionException(e); + assertTrue(unwrapEx instanceof PulsarClientException.TopicDoesNotExistException + || unwrapEx instanceof PulsarClientException.NotFoundException); + } + // Verify: the behavior of topic creation. + List partitionedTopics = admin1.topics().getPartitionedTopicList("public/default"); + pulsar1.getPulsarResources().getNamespaceResources().getPartitionedTopicResources() + .partitionedTopicExists(topicName); + assertFalse(partitionedTopics.contains(topicNameStr)); + verifyNonPartitionedTopicNeverCreated(topicNameStr); + verifyPartitionsNeverCreated(topicNameStr); + + // Verify: lookup semaphore has been releases. + Awaitility.await().untilAsserted(() -> { + assertEquals(getLookupRequestPermits(), lookupPermitsBefore); + }); + } + } + + /** + * Regarding the API "get partitioned metadata" about non-persistent topic. + * The original behavior is: + * param-auto-create = true, broker-config-auto-create = true + * HTTP API: default configuration {@link ServiceConfiguration#getDefaultNumPartitions()} + * binary API: default configuration {@link ServiceConfiguration#getDefaultNumPartitions()} + * param-auto-create = true, broker-config-auto-create = false + * HTTP API: {partitions: 0} + * binary API: {partitions: 0} + * param-auto-create = false + * HTTP API: not found error + * binary API: not support + * After PIP-344, the behavior will be the same as persistent topics, which was described in PIP-344. + */ + @Test(dataProvider = "autoCreationParamsNotAllow") + public void testGetMetadataIfNotAllowedCreateOfNonPersistentTopic(boolean configAllowAutoTopicCreation, + boolean paramMetadataAutoCreationEnabled, + boolean isUsingHttpLookup) throws Exception { + modifyTopicAutoCreation(configAllowAutoTopicCreation, TopicType.PARTITIONED, 3); + + int lookupPermitsBefore = getLookupRequestPermits(); + + PulsarClientImpl[] clientArray = getClientsToTest(isUsingHttpLookup); + for (PulsarClientImpl client : clientArray) { + // Define topic. + final String topicNameStr = BrokerTestUtil.newUniqueName("non-persistent://" + DEFAULT_NS + "/tp"); + final TopicName topicName = TopicName.get(topicNameStr); + // Verify: the result of get partitioned topic metadata. + try { + PartitionedTopicMetadata topicMetadata = client + .getPartitionedTopicMetadata(topicNameStr, paramMetadataAutoCreationEnabled, false) + .join(); + log.info("Get topic metadata: {}", topicMetadata.partitions); + fail("Expected a not found ex"); + } catch (Exception ex) { + Throwable unwrapEx = FutureUtil.unwrapCompletionException(ex); + assertTrue(unwrapEx instanceof PulsarClientException.TopicDoesNotExistException + || unwrapEx instanceof PulsarClientException.NotFoundException); + } + + // Verify: the behavior of topic creation. + List partitionedTopics = admin1.topics().getPartitionedTopicList("public/default"); + pulsar1.getPulsarResources().getNamespaceResources().getPartitionedTopicResources() + .partitionedTopicExists(topicName); + assertFalse(partitionedTopics.contains(topicNameStr)); + verifyNonPartitionedTopicNeverCreated(topicNameStr); + verifyPartitionsNeverCreated(topicNameStr); + } + + // Verify: lookup semaphore has been releases. + Awaitility.await().untilAsserted(() -> { + assertEquals(getLookupRequestPermits(), lookupPermitsBefore); + }); + } + + @Test(dataProvider = "topicDomains") + public void testNamespaceNotExist(TopicDomain topicDomain) throws Exception { + int lookupPermitsBefore = getLookupRequestPermits(); + final String namespaceNotExist = BrokerTestUtil.newUniqueName("public/ns"); + final String topicNameStr = BrokerTestUtil.newUniqueName(topicDomain.toString() + "://" + namespaceNotExist + "/tp"); + PulsarClientImpl[] clientArray = getClientsToTest(false); + for (PulsarClientImpl client : clientArray) { + try { + PartitionedTopicMetadata topicMetadata = client + .getPartitionedTopicMetadata(topicNameStr, true, true) + .join(); + log.info("Get topic metadata: {}", topicMetadata.partitions); + fail("Expected a not found ex"); + } catch (Exception ex) { + Throwable unwrapEx = FutureUtil.unwrapCompletionException(ex); + assertTrue(unwrapEx instanceof PulsarClientException.BrokerMetadataException || + unwrapEx instanceof PulsarClientException.TopicDoesNotExistException); + } + } + // Verify: lookup semaphore has been releases. + Awaitility.await().untilAsserted(() -> { + assertEquals(getLookupRequestPermits(), lookupPermitsBefore); + }); + } + + @Test(dataProvider = "topicDomains") + public void testTenantNotExist(TopicDomain topicDomain) throws Exception { + int lookupPermitsBefore = getLookupRequestPermits(); + final String tenantNotExist = BrokerTestUtil.newUniqueName("tenant"); + final String namespaceNotExist = BrokerTestUtil.newUniqueName(tenantNotExist + "/default"); + final String topicNameStr = BrokerTestUtil.newUniqueName(topicDomain.toString() + "://" + namespaceNotExist + "/tp"); + PulsarClientImpl[] clientArray = getClientsToTest(false); + for (PulsarClientImpl client : clientArray) { + try { + PartitionedTopicMetadata topicMetadata = client + .getPartitionedTopicMetadata(topicNameStr, true, true) + .join(); + log.info("Get topic metadata: {}", topicMetadata.partitions); + fail("Expected a not found ex"); + } catch (Exception ex) { + Throwable unwrapEx = FutureUtil.unwrapCompletionException(ex); + assertTrue(unwrapEx instanceof PulsarClientException.BrokerMetadataException || + unwrapEx instanceof PulsarClientException.TopicDoesNotExistException); + } + } + // Verify: lookup semaphore has been releases. + Awaitility.await().untilAsserted(() -> { + assertEquals(getLookupRequestPermits(), lookupPermitsBefore); + }); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespaceAuthZTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespaceAuthZTest.java new file mode 100644 index 0000000000000..66e13ef59f0ef --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespaceAuthZTest.java @@ -0,0 +1,2097 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.admin; + +import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.deleteNamespaceWithRetry; +import static org.apache.pulsar.common.policies.data.SchemaAutoUpdateCompatibilityStrategy.AutoUpdateDisabled; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import com.google.common.collect.Sets; +import io.jsonwebtoken.Jwts; +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import lombok.Cleanup; +import lombok.SneakyThrows; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.pulsar.broker.authorization.AuthorizationService; +import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.MessageRoutingMode; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; +import org.apache.pulsar.common.policies.data.AuthAction; +import org.apache.pulsar.common.policies.data.AutoSubscriptionCreationOverride; +import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; +import org.apache.pulsar.common.policies.data.BacklogQuota; +import org.apache.pulsar.common.policies.data.BookieAffinityGroupData; +import org.apache.pulsar.common.policies.data.BundlesData; +import org.apache.pulsar.common.policies.data.DispatchRate; +import org.apache.pulsar.common.policies.data.EntryFilters; +import org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode; +import org.apache.pulsar.common.policies.data.InactiveTopicPolicies; +import org.apache.pulsar.common.policies.data.NamespaceOperation; +import org.apache.pulsar.common.policies.data.OffloadPolicies; +import org.apache.pulsar.common.policies.data.PersistencePolicies; +import org.apache.pulsar.common.policies.data.Policies; +import org.apache.pulsar.common.policies.data.PolicyName; +import org.apache.pulsar.common.policies.data.PolicyOperation; +import org.apache.pulsar.common.policies.data.PublishRate; +import org.apache.pulsar.common.policies.data.RetentionPolicies; +import org.apache.pulsar.common.policies.data.SubscribeRate; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.packages.management.core.MockedPackagesStorageProvider; +import org.apache.pulsar.packages.management.core.common.PackageMetadata; +import org.apache.pulsar.security.MockedPulsarStandalone; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Test(groups = "broker-admin") +public class NamespaceAuthZTest extends MockedPulsarStandalone { + + private PulsarAdmin superUserAdmin; + + private PulsarAdmin tenantManagerAdmin; + + private PulsarClient pulsarClient; + + private AuthorizationService authorizationService; + + private static final String TENANT_ADMIN_SUBJECT = UUID.randomUUID().toString(); + private static final String TENANT_ADMIN_TOKEN = Jwts.builder() + .claim("sub", TENANT_ADMIN_SUBJECT).signWith(SECRET_KEY).compact(); + + @SneakyThrows + @BeforeClass + public void setup() { + getServiceConfiguration().setEnablePackagesManagement(true); + getServiceConfiguration().setPackagesManagementStorageProvider(MockedPackagesStorageProvider.class.getName()); + getServiceConfiguration().setDefaultNumberOfNamespaceBundles(1); + getServiceConfiguration().setForceDeleteNamespaceAllowed(true); + configureTokenAuthentication(); + configureDefaultAuthorization(); + start(); + this.superUserAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(SUPER_USER_TOKEN)) + .build(); + final TenantInfo tenantInfo = superUserAdmin.tenants().getTenantInfo("public"); + tenantInfo.getAdminRoles().add(TENANT_ADMIN_SUBJECT); + superUserAdmin.tenants().updateTenant("public", tenantInfo); + this.tenantManagerAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(TENANT_ADMIN_TOKEN)) + .build(); + this.pulsarClient = super.getPulsarService().getClient(); + this.authorizationService = Mockito.spy(getPulsarService().getBrokerService().getAuthorizationService()); + FieldUtils.writeField(getPulsarService().getBrokerService(), "authorizationService", + authorizationService, true); + } + + + @SneakyThrows + @AfterClass + public void cleanup() { + if (superUserAdmin != null) { + superUserAdmin.close(); + superUserAdmin = null; + } + if (tenantManagerAdmin != null) { + tenantManagerAdmin.close(); + tenantManagerAdmin = null; + } + pulsarClient = null; + authorizationService = null; + close(); + } + + @AfterMethod + public void after() throws Exception { + deleteNamespaceWithRetry("public/default", true, superUserAdmin); + superUserAdmin.namespaces().createNamespace("public/default"); + } + + private AtomicBoolean setAuthorizationOperationChecker(String role, NamespaceOperation operation) { + AtomicBoolean execFlag = new AtomicBoolean(false); + Mockito.doAnswer(invocationOnMock -> { + String role_ = invocationOnMock.getArgument(2); + if (role.equals(role_)) { + NamespaceOperation operation_ = invocationOnMock.getArgument(1); + Assert.assertEquals(operation_, operation); + } + execFlag.set(true); + return invocationOnMock.callRealMethod(); + }).when(authorizationService).allowNamespaceOperationAsync(Mockito.any(), Mockito.any(), Mockito.any(), + Mockito.any()); + return execFlag; + } + + private void clearAuthorizationOperationChecker() { + Mockito.doAnswer(InvocationOnMock::callRealMethod).when(authorizationService) + .allowNamespaceOperationAsync(Mockito.any(), Mockito.any(), Mockito.any(), + Mockito.any()); + } + + private AtomicBoolean setAuthorizationPolicyOperationChecker(String role, Object policyName, Object operation) { + AtomicBoolean execFlag = new AtomicBoolean(false); + if (operation instanceof PolicyOperation) { + Mockito.doAnswer(invocationOnMock -> { + String role_ = invocationOnMock.getArgument(3); + if (role.equals(role_)) { + PolicyName policyName_ = invocationOnMock.getArgument(1); + PolicyOperation operation_ = invocationOnMock.getArgument(2); + assertEquals(operation_, operation); + assertEquals(policyName_, policyName); + } + execFlag.set(true); + return invocationOnMock.callRealMethod(); + }).when(authorizationService).allowNamespacePolicyOperationAsync(Mockito.any(), Mockito.any(), Mockito.any(), + Mockito.any(), Mockito.any()); + } else { + throw new IllegalArgumentException(""); + } + return execFlag; + } + + @SneakyThrows + @Test + public void testProperties() { + final String random = UUID.randomUUID().toString(); + final String namespace = "public/default"; + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createNonPartitionedTopic(topic); + + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // test superuser + Map properties = new HashMap<>(); + properties.put("key1", "value1"); + superUserAdmin.namespaces().setProperties(namespace, properties); + superUserAdmin.namespaces().setProperty(namespace, "key2", "value2"); + superUserAdmin.namespaces().getProperties(namespace); + superUserAdmin.namespaces().getProperty(namespace, "key2"); + superUserAdmin.namespaces().removeProperty(namespace, "key2"); + superUserAdmin.namespaces().clearProperties(namespace); + + // test tenant manager + tenantManagerAdmin.namespaces().setProperties(namespace, properties); + tenantManagerAdmin.namespaces().setProperty(namespace, "key2", "value2"); + tenantManagerAdmin.namespaces().getProperties(namespace); + tenantManagerAdmin.namespaces().getProperty(namespace, "key2"); + tenantManagerAdmin.namespaces().removeProperty(namespace, "key2"); + tenantManagerAdmin.namespaces().clearProperties(namespace); + + // test nobody + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setProperties(namespace, properties)); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setProperty(namespace, "key2", "value2")); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getProperties(namespace)); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getProperty(namespace, "key2")); + + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeProperty(namespace, "key2")); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().clearProperties(namespace)); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(action)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setProperties(namespace, properties)); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setProperty(namespace, "key2", "value2")); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getProperties(namespace)); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getProperty(namespace, "key2")); + + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeProperty(namespace, "key2")); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().clearProperties(namespace)); + + superUserAdmin.namespaces().revokePermissionsOnNamespace(namespace, subject); + } + superUserAdmin.topics().delete(topic, true); + } + + @Test + public void testTopics() throws Exception { + final String random = UUID.randomUUID().toString(); + final String namespace = "public/default"; + final String topic = "persistent://" + namespace + "/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createNonPartitionedTopic(topic); + + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + + // test super admin + superUserAdmin.namespaces().getTopics(namespace); + + // test tenant manager + tenantManagerAdmin.namespaces().getTopics(namespace); + + AtomicBoolean execFlag = setAuthorizationOperationChecker(subject, NamespaceOperation.GET_TOPICS); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getTopics(namespace)); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(action)); + if (AuthAction.consume == action || AuthAction.produce == action) { + subAdmin.namespaces().getTopics(namespace); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getTopics(namespace)); + } + superUserAdmin.namespaces().revokePermissionsOnNamespace(namespace, subject); + } + + superUserAdmin.topics().delete(topic, true); + } + + @Test + public void testBookieAffinityGroup() throws Exception { + final String random = UUID.randomUUID().toString(); + final String namespace = "public/default"; + final String topic = "persistent://" + namespace + "/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createNonPartitionedTopic(topic); + + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + + // test super admin + BookieAffinityGroupData bookieAffinityGroupData = BookieAffinityGroupData.builder() + .bookkeeperAffinityGroupPrimary("aaa") + .bookkeeperAffinityGroupSecondary("bbb") + .build(); + superUserAdmin.namespaces().setBookieAffinityGroup(namespace, bookieAffinityGroupData); + BookieAffinityGroupData bookieAffinityGroup = superUserAdmin.namespaces().getBookieAffinityGroup(namespace); + Assert.assertEquals(bookieAffinityGroupData, bookieAffinityGroup); + superUserAdmin.namespaces().deleteBookieAffinityGroup(namespace); + bookieAffinityGroup = superUserAdmin.namespaces().getBookieAffinityGroup(namespace); + Assert.assertNull(bookieAffinityGroup); + + // test tenant manager + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> tenantManagerAdmin.namespaces().setBookieAffinityGroup(namespace, bookieAffinityGroupData)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> tenantManagerAdmin.namespaces().getBookieAffinityGroup(namespace)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> tenantManagerAdmin.namespaces().deleteBookieAffinityGroup(namespace)); + + // test nobody + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setBookieAffinityGroup(namespace, bookieAffinityGroupData)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getBookieAffinityGroup(namespace)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().deleteBookieAffinityGroup(namespace)); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(action)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setBookieAffinityGroup(namespace, bookieAffinityGroupData)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getBookieAffinityGroup(namespace)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().deleteBookieAffinityGroup(namespace)); + superUserAdmin.namespaces().revokePermissionsOnNamespace(namespace, subject); + } + + superUserAdmin.topics().delete(topic, true); + } + + + @Test + public void testGetBundles() throws Exception { + final String random = UUID.randomUUID().toString(); + final String namespace = "public/default"; + final String topic = "persistent://" + namespace + "/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createNonPartitionedTopic(topic); + + Producer producer = pulsarClient.newProducer(Schema.BYTES) + .topic(topic) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + producer.send("message".getBytes()); + + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // test super admin + superUserAdmin.namespaces().getBundles(namespace); + + // test tenant manager + tenantManagerAdmin.namespaces().getBundles(namespace); + + // test nobody + AtomicBoolean execFlag = setAuthorizationOperationChecker(subject, NamespaceOperation.GET_BUNDLE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getBundles(namespace)); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(action)); + if (AuthAction.consume == action || AuthAction.produce == action) { + subAdmin.namespaces().getBundles(namespace); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getBundles(namespace)); + } + superUserAdmin.namespaces().revokePermissionsOnNamespace(namespace, subject); + } + + superUserAdmin.topics().delete(topic, true); + } + + @Test + public void testUnloadBundles() throws Exception { + final String random = UUID.randomUUID().toString(); + final String namespace = "public/default"; + final String topic = "persistent://" + namespace + "/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createNonPartitionedTopic(topic); + + Producer producer = pulsarClient.newProducer(Schema.BYTES) + .topic(topic) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + producer.send("message".getBytes()); + + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + + final String defaultBundle = "0x00000000_0xffffffff"; + + // test super admin + superUserAdmin.namespaces().unloadNamespaceBundle(namespace, defaultBundle); + + // test tenant manager + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> tenantManagerAdmin.namespaces().unloadNamespaceBundle(namespace, defaultBundle)); + + // test nobody + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().unloadNamespaceBundle(namespace, defaultBundle)); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(action)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().unloadNamespaceBundle(namespace, defaultBundle)); + superUserAdmin.namespaces().revokePermissionsOnNamespace(namespace, subject); + } + + superUserAdmin.topics().delete(topic, true); + } + + @Test + public void testSplitBundles() throws Exception { + final String random = UUID.randomUUID().toString(); + final String namespace = "public/default"; + final String topic = "persistent://" + namespace + "/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createNonPartitionedTopic(topic); + + Producer producer = pulsarClient.newProducer(Schema.BYTES) + .topic(topic) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + producer.send("message".getBytes()); + + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + + final String defaultBundle = "0x00000000_0xffffffff"; + + // test super admin + superUserAdmin.namespaces().splitNamespaceBundle(namespace, defaultBundle, false, null); + + // test tenant manager + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> tenantManagerAdmin.namespaces().splitNamespaceBundle(namespace, defaultBundle, false, null)); + + // test nobody + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().splitNamespaceBundle(namespace, defaultBundle, false, null)); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(action)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().splitNamespaceBundle(namespace, defaultBundle, false, null)); + superUserAdmin.namespaces().revokePermissionsOnNamespace(namespace, subject); + } + + superUserAdmin.topics().delete(topic, true); + } + + @Test + public void testDeleteBundles() throws Exception { + final String random = UUID.randomUUID().toString(); + final String namespace = "public/default"; + final String topic = "persistent://" + namespace + "/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createNonPartitionedTopic(topic); + + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + + Producer producer = pulsarClient.newProducer(Schema.BYTES) + .topic(topic) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + producer.send("message".getBytes()); + + for (int i = 0; i < 3; i++) { + superUserAdmin.namespaces() + .splitNamespaceBundle(namespace, Policies.BundleType.LARGEST.toString(), false, null); + } + + BundlesData bundles = superUserAdmin.namespaces().getBundles(namespace); + Assert.assertEquals(bundles.getNumBundles(), 4); + List boundaries = bundles.getBoundaries(); + Assert.assertEquals(boundaries.size(), 5); + + List bundleRanges = new ArrayList<>(); + for (int i = 0; i < boundaries.size() - 1; i++) { + String bundleRange = boundaries.get(i) + "_" + boundaries.get(i + 1); + List allTopicsFromNamespaceBundle = getPulsarService().getBrokerService() + .getAllTopicsFromNamespaceBundle(namespace, namespace + "/" + bundleRange); + System.out.println(StringUtils.join(allTopicsFromNamespaceBundle)); + if (allTopicsFromNamespaceBundle.isEmpty()) { + bundleRanges.add(bundleRange); + } + } + + // test super admin + superUserAdmin.namespaces().deleteNamespaceBundle(namespace, bundleRanges.get(0)); + + // test tenant manager + tenantManagerAdmin.namespaces().deleteNamespaceBundle(namespace, bundleRanges.get(1)); + + // test nobody + AtomicBoolean execFlag = setAuthorizationOperationChecker(subject, NamespaceOperation.DELETE_BUNDLE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().deleteNamespaceBundle(namespace, bundleRanges.get(1))); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(action)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().deleteNamespaceBundle(namespace, bundleRanges.get(1))); + superUserAdmin.namespaces().revokePermissionsOnNamespace(namespace, subject); + } + } + + @Test + public void testPermission() throws Exception { + final String random = UUID.randomUUID().toString(); + final String namespace = "public/default"; + final String topic = "persistent://" + namespace + "/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createNonPartitionedTopic(topic); + + final String role = "sub"; + final AuthAction testAction = AuthAction.consume; + + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + + // test super admin + superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, role, Set.of(testAction)); + Map> permissions = superUserAdmin.namespaces().getPermissions(namespace); + Assert.assertEquals(permissions.get(role), Set.of(testAction)); + superUserAdmin.namespaces().revokePermissionsOnNamespace(namespace, role); + permissions = superUserAdmin.namespaces().getPermissions(namespace); + Assert.assertTrue(permissions.isEmpty()); + + // test tenant manager + tenantManagerAdmin.namespaces().grantPermissionOnNamespace(namespace, role, Set.of(testAction)); + permissions = tenantManagerAdmin.namespaces().getPermissions(namespace); + Assert.assertEquals(permissions.get(role), Set.of(testAction)); + tenantManagerAdmin.namespaces().revokePermissionsOnNamespace(namespace, role); + permissions = tenantManagerAdmin.namespaces().getPermissions(namespace); + Assert.assertTrue(permissions.isEmpty()); + + // test nobody + AtomicBoolean execFlag = + setAuthorizationOperationChecker(subject, NamespaceOperation.GRANT_PERMISSION); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().grantPermissionOnNamespace(namespace, role, Set.of(testAction))); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationOperationChecker(subject, NamespaceOperation.GET_PERMISSION); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getPermissions(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = + setAuthorizationOperationChecker(subject, NamespaceOperation.REVOKE_PERMISSION); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().revokePermissionsOnNamespace(namespace, role)); + Assert.assertTrue(execFlag.get()); + + clearAuthorizationOperationChecker(); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(action)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().grantPermissionOnNamespace(namespace, role, Set.of(testAction))); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getPermissions(namespace)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().revokePermissionsOnNamespace(namespace, role)); + superUserAdmin.namespaces().revokePermissionsOnNamespace(namespace, subject); + } + + superUserAdmin.topics().delete(topic, true); + } + + @Test + public void testPermissionOnSubscription() throws Exception { + final String random = UUID.randomUUID().toString(); + final String namespace = "public/default"; + final String topic = "persistent://" + namespace + "/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createNonPartitionedTopic(topic); + + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + + final String subscription = "my-sub"; + final String role = "sub"; + pulsarClient.newConsumer().topic(topic) + .subscriptionName(subscription) + .subscribe().close(); + + + // test super admin + superUserAdmin.namespaces().grantPermissionOnSubscription(namespace, subscription, Set.of(role)); + Map> permissionOnSubscription = + superUserAdmin.namespaces().getPermissionOnSubscription(namespace); + Assert.assertEquals(permissionOnSubscription.get(subscription), Set.of(role)); + superUserAdmin.namespaces().revokePermissionOnSubscription(namespace, subscription, role); + permissionOnSubscription = superUserAdmin.namespaces().getPermissionOnSubscription(namespace); + Assert.assertTrue(permissionOnSubscription.isEmpty()); + + // test tenant manager + tenantManagerAdmin.namespaces().grantPermissionOnSubscription(namespace, subscription, Set.of(role)); + permissionOnSubscription = tenantManagerAdmin.namespaces().getPermissionOnSubscription(namespace); + Assert.assertEquals(permissionOnSubscription.get(subscription), Set.of(role)); + tenantManagerAdmin.namespaces().revokePermissionOnSubscription(namespace, subscription, role); + permissionOnSubscription = tenantManagerAdmin.namespaces().getPermissionOnSubscription(namespace); + Assert.assertTrue(permissionOnSubscription.isEmpty()); + + // test nobody + AtomicBoolean execFlag = setAuthorizationOperationChecker(subject, NamespaceOperation.GRANT_PERMISSION); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().grantPermissionOnSubscription(namespace, subscription, Set.of(role))); + Assert.assertTrue(execFlag.get()); + execFlag = setAuthorizationOperationChecker(subject, NamespaceOperation.GET_PERMISSION); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getPermissionOnSubscription(namespace)); + Assert.assertTrue(execFlag.get()); + execFlag = setAuthorizationOperationChecker(subject, NamespaceOperation.REVOKE_PERMISSION); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().revokePermissionOnSubscription(namespace, subscription, role)); + Assert.assertTrue(execFlag.get()); + + clearAuthorizationOperationChecker(); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(action)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().grantPermissionOnSubscription(namespace, subscription, Set.of(role))); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getPermissionOnSubscription(namespace)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().revokePermissionOnSubscription(namespace, subscription, role)); + superUserAdmin.namespaces().revokePermissionsOnNamespace(namespace, subject); + } + + superUserAdmin.topics().delete(topic, true); + } + + @Test + public void testClearBacklog() throws Exception { + final String random = UUID.randomUUID().toString(); + final String namespace = "public/default"; + final String topic = "persistent://" + namespace + "/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createNonPartitionedTopic(topic); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // test super admin + superUserAdmin.namespaces().clearNamespaceBacklog(namespace); + + // test tenant manager + tenantManagerAdmin.namespaces().clearNamespaceBacklog(namespace); + + // test nobody + AtomicBoolean execFlag = setAuthorizationOperationChecker(subject, NamespaceOperation.CLEAR_BACKLOG); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().clearNamespaceBacklog(namespace)); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(action)); + if (AuthAction.consume == action) { + subAdmin.namespaces().clearNamespaceBacklog(namespace); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().clearNamespaceBacklog(namespace)); + } + superUserAdmin.namespaces().revokePermissionsOnNamespace(namespace, subject); + } + + superUserAdmin.topics().delete(topic, true); + } + + @Test + public void testClearNamespaceBundleBacklog() throws Exception { + final String random = UUID.randomUUID().toString(); + final String namespace = "public/default"; + final String topic = "persistent://" + namespace + "/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createNonPartitionedTopic(topic); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + + @Cleanup + Producer batchProducer = pulsarClient.newProducer().topic(topic) + .enableBatching(false) + .create(); + + final String defaultBundle = "0x00000000_0xffffffff"; + + // test super admin + superUserAdmin.namespaces().clearNamespaceBundleBacklog(namespace, defaultBundle); + + // test tenant manager + tenantManagerAdmin.namespaces().clearNamespaceBundleBacklog(namespace, defaultBundle); + + // test nobody + AtomicBoolean execFlag = setAuthorizationOperationChecker(subject, NamespaceOperation.CLEAR_BACKLOG); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().clearNamespaceBundleBacklog(namespace, defaultBundle)); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(action)); + if (AuthAction.consume == action) { + subAdmin.namespaces().clearNamespaceBundleBacklog(namespace, defaultBundle); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().clearNamespaceBundleBacklog(namespace, defaultBundle)); + } + superUserAdmin.namespaces().revokePermissionsOnNamespace(namespace, subject); + } + + superUserAdmin.topics().delete(topic, true); + } + + @Test + public void testUnsubscribeNamespace() throws Exception { + final String random = UUID.randomUUID().toString(); + final String namespace = "public/default"; + final String topic = "persistent://" + namespace + "/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createNonPartitionedTopic(topic); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + + @Cleanup + Producer batchProducer = pulsarClient.newProducer().topic(topic) + .enableBatching(false) + .create(); + + pulsarClient.newConsumer().topic(topic) + .subscriptionName("sub") + .subscribe().close(); + + // test super admin + superUserAdmin.namespaces().unsubscribeNamespace(namespace, "sub"); + + // test tenant manager + tenantManagerAdmin.namespaces().unsubscribeNamespace(namespace, "sub"); + + // test nobody + AtomicBoolean execFlag = setAuthorizationOperationChecker(subject, NamespaceOperation.UNSUBSCRIBE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().unsubscribeNamespace(namespace, "sub")); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(action)); + if (AuthAction.consume == action) { + subAdmin.namespaces().unsubscribeNamespace(namespace, "sub"); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().unsubscribeNamespace(namespace, "sub")); + } + superUserAdmin.namespaces().revokePermissionsOnNamespace(namespace, subject); + } + + superUserAdmin.topics().delete(topic, true); + } + + @Test + public void testUnsubscribeNamespaceBundle() throws Exception { + final String random = UUID.randomUUID().toString(); + final String namespace = "public/default"; + final String topic = "persistent://" + namespace + "/" + random ; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createNonPartitionedTopic(topic); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + + @Cleanup + Producer batchProducer = pulsarClient.newProducer().topic(topic) + .enableBatching(false) + .create(); + + pulsarClient.newConsumer().topic(topic) + .subscriptionName("sub") + .subscribe().close(); + + final String defaultBundle = "0x00000000_0xffffffff"; + + // test super admin + superUserAdmin.namespaces().unsubscribeNamespaceBundle(namespace, defaultBundle, "sub"); + + // test tenant manager + tenantManagerAdmin.namespaces().unsubscribeNamespaceBundle(namespace, defaultBundle, "sub"); + + // test nobody + AtomicBoolean execFlag = setAuthorizationOperationChecker(subject, NamespaceOperation.UNSUBSCRIBE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().unsubscribeNamespaceBundle(namespace, defaultBundle, "sub")); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(action)); + if (AuthAction.consume == action) { + subAdmin.namespaces().unsubscribeNamespaceBundle(namespace, defaultBundle, "sub"); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().unsubscribeNamespaceBundle(namespace, defaultBundle, "sub")); + } + superUserAdmin.namespaces().revokePermissionsOnNamespace(namespace, subject); + } + + superUserAdmin.topics().delete(topic, true); + } + + @Test + public void testPackageAPI() throws Exception { + final String namespace = "public/default"; + + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + + + File file = File.createTempFile("package-api-test", ".package"); + + // testing upload api + String packageName = "function://public/default/test@v1"; + PackageMetadata originalMetadata = PackageMetadata.builder().description("test").build(); + superUserAdmin.packages().upload(originalMetadata, packageName, file.getPath()); + + // testing download api + String downloadPath = new File(file.getParentFile(), "package-api-test-download.package").getPath(); + superUserAdmin.packages().download(packageName, downloadPath); + File downloadFile = new File(downloadPath); + assertTrue(downloadFile.exists()); + downloadFile.delete(); + + // testing list packages api + List packages = superUserAdmin.packages().listPackages("function", "public/default"); + assertEquals(packages.size(), 1); + assertEquals(packages.get(0), "test"); + + // testing list versions api + List versions = superUserAdmin.packages().listPackageVersions(packageName); + assertEquals(versions.size(), 1); + assertEquals(versions.get(0), "v1"); + + // testing get packages api + PackageMetadata metadata = superUserAdmin.packages().getMetadata(packageName); + assertEquals(metadata.getDescription(), originalMetadata.getDescription()); + assertNull(metadata.getContact()); + assertTrue(metadata.getModificationTime() > 0); + assertTrue(metadata.getCreateTime() > 0); + assertNull(metadata.getProperties()); + + // testing update package metadata api + PackageMetadata updatedMetadata = originalMetadata; + updatedMetadata.setContact("test@apache.org"); + updatedMetadata.setProperties(Collections.singletonMap("key", "value")); + superUserAdmin.packages().updateMetadata(packageName, updatedMetadata); + + superUserAdmin.packages().getMetadata(packageName); + + // ---- test tenant manager --- + + file = File.createTempFile("package-api-test", ".package"); + + // test tenant manager + packageName = "function://public/default/test@v2"; + originalMetadata = PackageMetadata.builder().description("test").build(); + tenantManagerAdmin.packages().upload(originalMetadata, packageName, file.getPath()); + + // testing download api + downloadPath = new File(file.getParentFile(), "package-api-test-download.package").getPath(); + tenantManagerAdmin.packages().download(packageName, downloadPath); + downloadFile = new File(downloadPath); + assertTrue(downloadFile.exists()); + downloadFile.delete(); + + // testing list packages api + packages = tenantManagerAdmin.packages().listPackages("function", "public/default"); + assertEquals(packages.size(), 1); + assertEquals(packages.get(0), "test"); + + // testing list versions api + tenantManagerAdmin.packages().listPackageVersions(packageName); + + // testing get packages api + tenantManagerAdmin.packages().getMetadata(packageName); + + // testing update package metadata api + updatedMetadata = originalMetadata; + updatedMetadata.setContact("test@apache.org"); + updatedMetadata.setProperties(Collections.singletonMap("key", "value")); + tenantManagerAdmin.packages().updateMetadata(packageName, updatedMetadata); + + // ---- test nobody --- + AtomicBoolean execFlag = setAuthorizationOperationChecker(subject, NamespaceOperation.PACKAGES); + + File file3 = File.createTempFile("package-api-test", ".package"); + + // test tenant manager + String packageName3 = "function://public/default/test@v3"; + PackageMetadata originalMetadata3 = PackageMetadata.builder().description("test").build(); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.packages().upload(originalMetadata3, packageName3, file3.getPath())); + + + // testing download api + String downloadPath3 = new File(file3.getParentFile(), "package-api-test-download.package").getPath(); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.packages().download(packageName3, downloadPath3)); + + // testing list packages api + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.packages().listPackages("function", "public/default")); + + // testing list versions api + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.packages().listPackageVersions(packageName3)); + + // testing get packages api + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.packages().getMetadata(packageName3)); + + // testing update package metadata api + PackageMetadata updatedMetadata3 = originalMetadata; + updatedMetadata3.setContact("test@apache.org"); + updatedMetadata3.setProperties(Collections.singletonMap("key", "value")); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.packages().updateMetadata(packageName3, updatedMetadata3)); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(action)); + File file4 = File.createTempFile("package-api-test", ".package"); + String packageName4 = "function://public/default/test@v4"; + PackageMetadata originalMetadata4 = PackageMetadata.builder().description("test").build(); + String downloadPath4 = new File(file3.getParentFile(), "package-api-test-download.package").getPath(); + if (AuthAction.packages == action) { + subAdmin.packages().upload(originalMetadata4, packageName4, file.getPath()); + + // testing download api + subAdmin.packages().download(packageName4, downloadPath4); + downloadFile = new File(downloadPath4); + assertTrue(downloadFile.exists()); + downloadFile.delete(); + + // testing list packages api + packages = subAdmin.packages().listPackages("function", "public/default"); + assertEquals(packages.size(), 1); + assertEquals(packages.get(0), "test"); + + // testing list versions api + subAdmin.packages().listPackageVersions(packageName4); + + // testing get packages api + subAdmin.packages().getMetadata(packageName4); + + // testing update package metadata api + PackageMetadata updatedMetadata4 = originalMetadata; + updatedMetadata4.setContact("test@apache.org"); + updatedMetadata4.setProperties(Collections.singletonMap("key", "value")); + subAdmin.packages().updateMetadata(packageName, updatedMetadata4); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.packages().upload(originalMetadata4, packageName4, file4.getPath())); + + // testing download api + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.packages().download(packageName4, downloadPath4)); + + // testing list packages api + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.packages().listPackages("function", "public/default")); + + // testing list versions api + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.packages().listPackageVersions(packageName4)); + + // testing get packages api + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.packages().getMetadata(packageName4)); + + // testing update package metadata api + PackageMetadata updatedMetadata4 = originalMetadata; + updatedMetadata4.setContact("test@apache.org"); + updatedMetadata4.setProperties(Collections.singletonMap("key", "value")); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.packages().updateMetadata(packageName4, updatedMetadata4)); + } + superUserAdmin.namespaces().revokePermissionsOnNamespace(namespace, subject); + } + } + + @Test + @SneakyThrows + public void testDispatchRate() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RATE, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getDispatchRate(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RATE, PolicyOperation.WRITE); + DispatchRate dispatchRate = + DispatchRate.builder().dispatchThrottlingRateInByte(10).dispatchThrottlingRateInMsg(10).build(); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setDispatchRate(namespace, dispatchRate)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RATE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeDispatchRate(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testSubscribeRate() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RATE, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getSubscribeRate(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RATE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setSubscribeRate(namespace, new SubscribeRate())); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RATE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeSubscribeRate(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testPublishRate() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RATE, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getPublishRate(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RATE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setPublishRate(namespace, new PublishRate(10, 10))); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RATE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removePublishRate(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testSubscriptionDispatchRate() { + final String random = UUID.randomUUID().toString(); + final String namespace = "public/default"; + final String topic = "persistent://" + namespace + "/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createNonPartitionedTopic(topic); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RATE, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getSubscriptionDispatchRate(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RATE, PolicyOperation.WRITE); + DispatchRate dispatchRate = DispatchRate.builder().dispatchThrottlingRateInMsg(10).dispatchThrottlingRateInByte(10).build(); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setSubscriptionDispatchRate(namespace, dispatchRate)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RATE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeSubscriptionDispatchRate(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testCompactionThreshold() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = + setAuthorizationPolicyOperationChecker(subject, PolicyName.COMPACTION, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getCompactionThreshold(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.COMPACTION, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setCompactionThreshold(namespace, 100L * 1024L *1024L)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.COMPACTION, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeCompactionThreshold(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testAutoTopicCreation() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = + setAuthorizationPolicyOperationChecker(subject, PolicyName.AUTO_TOPIC_CREATION, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getAutoTopicCreation(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = + setAuthorizationPolicyOperationChecker(subject, PolicyName.AUTO_TOPIC_CREATION, PolicyOperation.WRITE); + AutoTopicCreationOverride build = AutoTopicCreationOverride.builder().allowAutoTopicCreation(true).build(); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setAutoTopicCreation(namespace, build)); + Assert.assertTrue(execFlag.get()); + + execFlag = + setAuthorizationPolicyOperationChecker(subject, PolicyName.AUTO_TOPIC_CREATION, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeAutoTopicCreation(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testAutoSubscriptionCreation() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.AUTO_SUBSCRIPTION_CREATION, + PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getAutoSubscriptionCreation(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.AUTO_SUBSCRIPTION_CREATION, + PolicyOperation.WRITE); + AutoSubscriptionCreationOverride build = + AutoSubscriptionCreationOverride.builder().allowAutoSubscriptionCreation(true).build(); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setAutoSubscriptionCreation(namespace, build)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.AUTO_SUBSCRIPTION_CREATION, + PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeAutoSubscriptionCreation(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testMaxUnackedMessagesPerConsumer() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_UNACKED, + PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getMaxUnackedMessagesPerConsumer(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_UNACKED, + PolicyOperation.WRITE); + AutoSubscriptionCreationOverride build = + AutoSubscriptionCreationOverride.builder().allowAutoSubscriptionCreation(true).build(); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setMaxUnackedMessagesPerConsumer(namespace, 100)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_UNACKED, + PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeMaxUnackedMessagesPerConsumer(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testMaxUnackedMessagesPerSubscription() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_UNACKED, + PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getMaxUnackedMessagesPerSubscription(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_UNACKED, + PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setMaxUnackedMessagesPerSubscription(namespace, 100)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_UNACKED, + PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeMaxUnackedMessagesPerSubscription(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testNamespaceResourceGroup() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RESOURCEGROUP, + PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getNamespaceResourceGroup(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RESOURCEGROUP, + PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setNamespaceResourceGroup(namespace, "test-group")); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RESOURCEGROUP, + PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeNamespaceResourceGroup(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testDispatcherPauseOnAckStatePersistent() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = + setAuthorizationPolicyOperationChecker(subject, PolicyName.DISPATCHER_PAUSE_ON_ACK_STATE_PERSISTENT, + PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getDispatcherPauseOnAckStatePersistent(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.DISPATCHER_PAUSE_ON_ACK_STATE_PERSISTENT, + PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setDispatcherPauseOnAckStatePersistent(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.DISPATCHER_PAUSE_ON_ACK_STATE_PERSISTENT, + PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeDispatcherPauseOnAckStatePersistent(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testBacklogQuota() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.BACKLOG, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getBacklogQuotaMap(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.BACKLOG, PolicyOperation.WRITE); + BacklogQuota backlogQuota = BacklogQuota.builder().limitTime(10).limitSize(10).build(); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setBacklogQuota(namespace, backlogQuota)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.BACKLOG, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeBacklogQuota(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testDeduplicationSnapshotInterval() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.DEDUPLICATION_SNAPSHOT, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getDeduplicationSnapshotInterval(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.DEDUPLICATION_SNAPSHOT, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setDeduplicationSnapshotInterval(namespace, 100)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.DEDUPLICATION_SNAPSHOT, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeDeduplicationSnapshotInterval(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testMaxSubscriptionsPerTopic() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = + setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_SUBSCRIPTIONS, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getMaxSubscriptionsPerTopic(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_SUBSCRIPTIONS, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setMaxSubscriptionsPerTopic(namespace, 10)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_SUBSCRIPTIONS, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeMaxSubscriptionsPerTopic(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testMaxProducersPerTopic() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = + setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_PRODUCERS, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getMaxProducersPerTopic(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_PRODUCERS, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setMaxProducersPerTopic(namespace, 10)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_PRODUCERS, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeMaxProducersPerTopic(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testMaxConsumersPerTopic() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = + setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_CONSUMERS, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getMaxConsumersPerTopic(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_CONSUMERS, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setMaxConsumersPerTopic(namespace, 10)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_CONSUMERS, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeMaxConsumersPerTopic(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testNamespaceReplicationClusters() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = + setAuthorizationPolicyOperationChecker(subject, PolicyName.REPLICATION, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getNamespaceReplicationClusters(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.REPLICATION, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("test"))); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testReplicatorDispatchRate() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = + setAuthorizationPolicyOperationChecker(subject, PolicyName.REPLICATION_RATE, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getReplicatorDispatchRate(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.REPLICATION_RATE, PolicyOperation.WRITE); + DispatchRate build = + DispatchRate.builder().dispatchThrottlingRateInByte(10).dispatchThrottlingRateInMsg(10).build(); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setReplicatorDispatchRate(namespace, build)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.REPLICATION_RATE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeReplicatorDispatchRate(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testMaxConsumersPerSubscription() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = + setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_CONSUMERS, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getMaxConsumersPerSubscription(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_CONSUMERS, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setMaxConsumersPerSubscription(namespace, 10)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_CONSUMERS, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeMaxConsumersPerSubscription(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testOffloadThreshold() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.OFFLOAD, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getOffloadThreshold(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.OFFLOAD, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setOffloadThreshold(namespace, 10)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testOffloadPolicies() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.OFFLOAD, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getOffloadPolicies(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.OFFLOAD, PolicyOperation.WRITE); + OffloadPolicies offloadPolicies = OffloadPolicies.builder().managedLedgerOffloadThresholdInBytes(10L).build(); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setOffloadPolicies(namespace, offloadPolicies)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.OFFLOAD, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeOffloadPolicies(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testMaxTopicsPerNamespace() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_TOPICS, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getMaxTopicsPerNamespace(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_TOPICS, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setMaxTopicsPerNamespace(namespace, 10)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_TOPICS, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeMaxTopicsPerNamespace(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testDeduplicationStatus() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = + setAuthorizationPolicyOperationChecker(subject, PolicyName.DEDUPLICATION, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getDeduplicationStatus(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.DEDUPLICATION, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setDeduplicationStatus(namespace, true)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.DEDUPLICATION, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeDeduplicationStatus(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testPersistence() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = + setAuthorizationPolicyOperationChecker(subject, PolicyName.PERSISTENCE, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getPersistence(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.PERSISTENCE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setPersistence(namespace, new PersistencePolicies(10, 10, 10, 10))); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.PERSISTENCE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removePersistence(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testNamespaceMessageTTL() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.TTL, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getNamespaceMessageTTL(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.TTL, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setNamespaceMessageTTL(namespace, 10)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.TTL, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeNamespaceMessageTTL(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testSubscriptionExpirationTime() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = + setAuthorizationPolicyOperationChecker(subject, PolicyName.SUBSCRIPTION_EXPIRATION_TIME, + PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getSubscriptionExpirationTime(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.SUBSCRIPTION_EXPIRATION_TIME, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setSubscriptionExpirationTime(namespace, 10)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.SUBSCRIPTION_EXPIRATION_TIME, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeSubscriptionExpirationTime(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testDelayedDeliveryMessages() { + final String random = UUID.randomUUID().toString(); + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = + setAuthorizationPolicyOperationChecker(subject, PolicyName.DELAYED_DELIVERY, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getDelayedDelivery(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testRetention() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RETENTION, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getRetention(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RETENTION, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setRetention(namespace, new RetentionPolicies(10, 10))); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RETENTION, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeRetention(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testInactiveTopicPolicies() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.INACTIVE_TOPIC, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getInactiveTopicPolicies(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.INACTIVE_TOPIC, PolicyOperation.WRITE); + InactiveTopicPolicies inactiveTopicPolicies = new InactiveTopicPolicies( + InactiveTopicDeleteMode.delete_when_no_subscriptions, 10, false); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setInactiveTopicPolicies(namespace, inactiveTopicPolicies)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.INACTIVE_TOPIC, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeInactiveTopicPolicies(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testNamespaceAntiAffinityGroup() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.ANTI_AFFINITY, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getNamespaceAntiAffinityGroup(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.ANTI_AFFINITY, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setNamespaceAntiAffinityGroup(namespace, "invalid-group")); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testOffloadDeleteLagMs() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.OFFLOAD, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getOffloadDeleteLagMs(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.OFFLOAD, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setOffloadDeleteLag(namespace, 100, TimeUnit.HOURS)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testOffloadThresholdInSeconds() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = + setAuthorizationPolicyOperationChecker(subject, PolicyName.OFFLOAD, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getOffloadThresholdInSeconds(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.OFFLOAD, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setOffloadThresholdInSeconds(namespace, 10000)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testNamespaceEntryFilters() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.ENTRY_FILTERS, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getNamespaceEntryFilters(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.ENTRY_FILTERS, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setNamespaceEntryFilters(namespace, new EntryFilters("filter1"))); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.ENTRY_FILTERS, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeNamespaceEntryFilters(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testEncryptionRequiredStatus() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.ENCRYPTION, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getEncryptionRequiredStatus(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.ENCRYPTION, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setEncryptionRequiredStatus(namespace, false)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testSubscriptionTypesEnabled() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.SUBSCRIPTION_AUTH_MODE, + PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getSubscriptionTypesEnabled(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.SUBSCRIPTION_AUTH_MODE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setSubscriptionTypesEnabled(namespace, Sets.newHashSet(SubscriptionType.Failover))); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.SUBSCRIPTION_AUTH_MODE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().removeSubscriptionTypesEnabled(namespace)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testIsAllowAutoUpdateSchema() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.SCHEMA_COMPATIBILITY_STRATEGY, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getIsAllowAutoUpdateSchema(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.SCHEMA_COMPATIBILITY_STRATEGY, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setIsAllowAutoUpdateSchema(namespace, true)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testSchemaAutoUpdateCompatibilityStrategy() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.SCHEMA_COMPATIBILITY_STRATEGY, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getSchemaAutoUpdateCompatibilityStrategy(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.SCHEMA_COMPATIBILITY_STRATEGY, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setSchemaAutoUpdateCompatibilityStrategy(namespace, AutoUpdateDisabled)); + Assert.assertTrue(execFlag.get()); + } + + @Test + @SneakyThrows + public void testSchemaValidationEnforced() { + final String namespace = "public/default"; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.SCHEMA_COMPATIBILITY_STRATEGY, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().getSchemaValidationEnforced(namespace)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.SCHEMA_COMPATIBILITY_STRATEGY, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.namespaces().setSchemaValidationEnforced(namespace, true)); + Assert.assertTrue(execFlag.get()); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java index 5644be406a7ee..f294866095250 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java @@ -32,6 +32,7 @@ import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import java.lang.reflect.Field; @@ -78,6 +79,7 @@ import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.namespace.OwnershipCache; import org.apache.pulsar.broker.service.AbstractTopic; +import org.apache.pulsar.broker.service.TopicPolicyTestUtils; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.web.PulsarWebResource; import org.apache.pulsar.broker.web.RestException; @@ -112,7 +114,6 @@ import org.apache.pulsar.common.policies.data.SubscribeRate; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.policies.data.TenantInfoImpl; -import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.common.policies.data.impl.DispatchRateImpl; import org.apache.pulsar.common.util.FutureUtil; @@ -215,9 +216,9 @@ private void initAndStartBroker() throws Exception { doReturn(null).when(namespaces).clientAuthData(); doReturn(Set.of("use", "usw", "usc", "global")).when(namespaces).clusters(); - admin.clusters().createCluster("use", ClusterData.builder().serviceUrl("http://broker-use.com:8080").build()); - admin.clusters().createCluster("usw", ClusterData.builder().serviceUrl("http://broker-usw.com:8080").build()); - admin.clusters().createCluster("usc", ClusterData.builder().serviceUrl("http://broker-usc.com:8080").build()); + admin.clusters().createCluster("use", ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); + admin.clusters().createCluster("usw", ClusterData.builder().serviceUrl("http://127.0.0.2:8082").build()); + admin.clusters().createCluster("usc", ClusterData.builder().serviceUrl("http://127.0.0.3:8083").build()); admin.tenants().createTenant(this.testTenant, new TenantInfoImpl(Set.of("role1", "role2"), Set.of("use", "usc", "usw"))); admin.tenants().createTenant(this.testOtherTenant, @@ -317,9 +318,9 @@ public void testGetNamespaces() throws Exception { expectedList.sort(null); AsyncResponse response = mock(AsyncResponse.class); namespaces.getTenantNamespaces(response, this.testTenant); - ArgumentCaptor captor = ArgumentCaptor.forClass(Response.class); - verify(response, timeout(5000).times(1)).resume(captor.capture()); - List namespacesList = (List) captor.getValue(); + ArgumentCaptor> listOfStringsCaptor = ArgumentCaptor.forClass(List.class); + verify(response, timeout(5000).times(1)).resume(listOfStringsCaptor.capture()); + List namespacesList = listOfStringsCaptor.getValue(); namespacesList.sort(null); assertEquals(namespacesList, expectedList); @@ -713,7 +714,7 @@ public void testNamespacesApiRedirects() throws Exception { verify(response, timeout(5000).times(1)).resume(captor.capture()); assertEquals(captor.getValue().getResponse().getStatus(), Status.TEMPORARY_REDIRECT.getStatusCode()); assertEquals(captor.getValue().getResponse().getLocation().toString(), - UriBuilder.fromUri(uri).host("broker-usc.com").port(8080).toString()); + UriBuilder.fromUri(uri).host("127.0.0.3").port(8083).toString()); uri = URI.create(pulsar.getWebServiceAddress() + "/admin/namespace/" + this.testLocalNamespaces.get(2).toString() + "/unload"); @@ -725,7 +726,7 @@ public void testNamespacesApiRedirects() throws Exception { verify(response, timeout(5000).atLeast(1)).resume(captor.capture()); assertEquals(captor.getValue().getResponse().getStatus(), Status.TEMPORARY_REDIRECT.getStatusCode()); assertEquals(captor.getValue().getResponse().getLocation().toString(), - UriBuilder.fromUri(uri).host("broker-usc.com").port(8080).toString()); + UriBuilder.fromUri(uri).host("127.0.0.3").port(8083).toString()); // check the bundle should not unload to an inactive destination broker namespaces.unloadNamespaceBundle(response, this.testTenant, this.testOtherCluster, @@ -762,7 +763,7 @@ public boolean matches(NamespaceName nsname) { verify(response, timeout(5000).times(1)).resume(captor.capture()); assertEquals(captor.getValue().getResponse().getStatus(), Status.TEMPORARY_REDIRECT.getStatusCode()); assertEquals(captor.getValue().getResponse().getLocation().toString(), - UriBuilder.fromUri(uri).host("broker-usc.com").port(8080).toString()); + UriBuilder.fromUri(uri).host("127.0.0.3").port(8083).toString()); // cleanup resetBroker(); @@ -1294,6 +1295,14 @@ public void testForceDeleteNamespace() throws Exception { pulsar.getConfiguration().setForceDeleteNamespaceAllowed(false); } + @Test + public void testSetNamespaceReplicationCluters() throws Exception { + String namespace = BrokerTestUtil.newUniqueName(this.testTenant + "/namespace"); + admin.namespaces().createNamespace(namespace, 100); + assertThrows(PulsarAdminException.PreconditionFailedException.class, + () -> admin.namespaces().setNamespaceReplicationClusters(namespace, Set.of())); + } + @Test public void testForceDeleteNamespaceNotAllowed() throws Exception { assertFalse(pulsar.getConfiguration().isForceDeleteNamespaceAllowed()); @@ -2094,16 +2103,96 @@ public void testDeleteTopicPolicyWhenDeleteSystemTopic() throws Exception { Producer producer = pulsarClient.newProducer(Schema.STRING) .topic(systemTopic).create(); admin.topicPolicies().setMaxConsumers(systemTopic, 5); + Awaitility.await().atMost(3, TimeUnit.SECONDS).untilAsserted(() -> { + final var policies = TopicPolicyTestUtils.getTopicPoliciesBypassCache(pulsar.getTopicPoliciesService(), + TopicName.get(systemTopic)); + Assert.assertTrue(policies.isPresent()); + Assert.assertEquals(policies.get().getMaxConsumerPerTopic(), 5); + }); - Integer maxConsumerPerTopic = pulsar - .getTopicPoliciesService() - .getTopicPoliciesBypassCacheAsync(TopicName.get(systemTopic)).get() - .getMaxConsumerPerTopic(); - - assertEquals(maxConsumerPerTopic, 5); admin.topics().delete(systemTopic, true); - TopicPolicies topicPolicies = pulsar.getTopicPoliciesService() - .getTopicPoliciesBypassCacheAsync(TopicName.get(systemTopic)).get(5, TimeUnit.SECONDS); - assertNull(topicPolicies); + Awaitility.await().atMost(3, TimeUnit.SECONDS).untilAsserted(() -> assertTrue( + TopicPolicyTestUtils.getTopicPoliciesBypassCache(pulsar.getTopicPoliciesService(), TopicName.get(systemTopic)) + .isEmpty())); + } + + @Test + public void testCreateNamespacesWithPolicy() throws Exception { + try { + asyncRequests(response -> namespaces.createNamespace(response, this.testTenant, "other-colo", "my-namespace", + new Policies())); + fail("should have failed"); + } catch (RestException e) { + // Ok, cluster doesn't exist + assertEquals(e.getResponse().getStatus(), Status.FORBIDDEN.getStatusCode()); + } + + List nsnames = new ArrayList<>(); + nsnames.add(NamespaceName.get(this.testTenant, "use", "create-namespace-1")); + nsnames.add(NamespaceName.get(this.testTenant, "use", "create-namespace-2")); + nsnames.add(NamespaceName.get(this.testTenant, "usc", "create-other-namespace-1")); + createTestNamespaces(nsnames, BundlesData.builder().build()); + + try { + asyncRequests(response -> namespaces.createNamespace(response, this.testTenant, "use", "create-namespace-1", + new Policies())); + fail("should have failed"); + } catch (RestException e) { + // Ok, namespace already exists + assertEquals(e.getResponse().getStatus(), Status.CONFLICT.getStatusCode()); + + } + + try { + asyncRequests(response -> namespaces.createNamespace(response,"non-existing-tenant", "use", "create-namespace-1", + new Policies())); + fail("should have failed"); + } catch (RestException e) { + // Ok, tenant doesn't exist + assertEquals(e.getResponse().getStatus(), Status.NOT_FOUND.getStatusCode()); + } + + try { + asyncRequests(response -> namespaces.createNamespace(response, this.testTenant, "use", "create-namespace-#", + new Policies())); + fail("should have failed"); + } catch (RestException e) { + // Ok, invalid namespace name + assertEquals(e.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode()); + } + + mockZooKeeperGlobal.failConditional(Code.SESSIONEXPIRED, (op, path) -> { + return op == MockZooKeeper.Op.CREATE + && path.equals("/admin/policies/my-tenant/use/my-namespace-3"); + }); + try { + asyncRequests(response -> namespaces.createNamespace(response, this.testTenant, "use", "my-namespace-3", new Policies())); + fail("should have failed"); + } catch (RestException e) { + // Ok + assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode()); + } + } + + private void createTestNamespaces(List nsnames, Policies policies) throws Exception { + for (NamespaceName nsName : nsnames) { + asyncRequests(ctx -> namespaces.createNamespace(ctx, nsName.getTenant(), nsName.getCluster(), nsName.getLocalName(), policies)); + } + } + + @Test + public void testDispatcherPauseOnAckStatePersistent() throws Exception { + String namespace = BrokerTestUtil.newUniqueName(this.testTenant + "/namespace"); + + admin.namespaces().createNamespace(namespace, Set.of(testLocalCluster)); + + assertFalse(admin.namespaces().getDispatcherPauseOnAckStatePersistent(namespace)); + // should pass + admin.namespaces().setDispatcherPauseOnAckStatePersistent(namespace); + assertTrue(admin.namespaces().getDispatcherPauseOnAckStatePersistent(namespace)); + admin.namespaces().removeDispatcherPauseOnAckStatePersistent(namespace); + assertFalse(admin.namespaces().getDispatcherPauseOnAckStatePersistent(namespace)); + + admin.namespaces().deleteNamespace(namespace); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesV2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesV2Test.java index cec3076219445..c1e8dfa30994a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesV2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesV2Test.java @@ -39,6 +39,7 @@ import org.apache.pulsar.broker.web.RestException; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies; import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.PolicyName; import org.apache.pulsar.common.policies.data.PolicyOperation; @@ -196,4 +197,43 @@ public void testOperationDispatchRate() throws Exception { this.testTenant, this.testNamespace)); assertTrue(Objects.isNull(dispatchRate)); } + + @Test + public void testOperationDelayedDelivery() throws Exception { + boolean isActive = true; + long tickTime = 1000; + long maxDeliveryDelayInMillis = 5000; + // 1. set delayed delivery policy + namespaces.setDelayedDeliveryPolicies(this.testTenant, this.testNamespace, + DelayedDeliveryPolicies.builder() + .active(isActive) + .tickTime(tickTime) + .maxDeliveryDelayInMillis(maxDeliveryDelayInMillis) + .build()); + + // 2. query delayed delivery policy & check + DelayedDeliveryPolicies policy = + (DelayedDeliveryPolicies) asyncRequests(response -> namespaces.getDelayedDeliveryPolicies(response, + this.testTenant, this.testNamespace)); + assertEquals(policy.isActive(), isActive); + assertEquals(policy.getTickTime(), tickTime); + assertEquals(policy.getMaxDeliveryDelayInMillis(), maxDeliveryDelayInMillis); + + // 3. remove & check + namespaces.removeDelayedDeliveryPolicies(this.testTenant, this.testNamespace); + policy = + (DelayedDeliveryPolicies) asyncRequests(response -> namespaces.getDelayedDeliveryPolicies(response, + this.testTenant, this.testNamespace)); + assertTrue(Objects.isNull(policy)); + + // 4. invalid namespace check + String invalidNamespace = this.testNamespace + "/"; + try { + namespaces.setDelayedDeliveryPolicies(this.testTenant, invalidNamespace, + DelayedDeliveryPolicies.builder().build()); + fail("should have failed"); + } catch (RestException e) { + assertEquals(e.getResponse().getStatus(), Response.Status.PRECONDITION_FAILED.getStatusCode()); + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 284e50c830286..18fd3dd1c8bb3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -31,6 +31,11 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertSame; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.HashMap; @@ -50,10 +55,12 @@ import javax.ws.rs.core.UriInfo; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.commons.collections4.MapUtils; import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.admin.v2.ExtPersistentTopics; import org.apache.pulsar.broker.admin.v2.NonPersistentTopics; import org.apache.pulsar.broker.admin.v2.PersistentTopics; -import org.apache.pulsar.broker.admin.v2.ExtPersistentTopics; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationDataHttps; import org.apache.pulsar.broker.namespace.NamespaceService; @@ -62,17 +69,23 @@ import org.apache.pulsar.broker.resources.TopicResources; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.web.PulsarWebResource; import org.apache.pulsar.broker.web.RestException; +import org.apache.pulsar.client.admin.LongRunningProcessStatus; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.admin.Topics; import org.apache.pulsar.client.admin.internal.TopicsImpl; import org.apache.pulsar.client.api.CompressionType; +import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.interceptor.ProducerInterceptor; import org.apache.pulsar.client.impl.BatchMessageIdImpl; import org.apache.pulsar.client.impl.MessageIdImpl; @@ -86,6 +99,7 @@ import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.AuthAction; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.TenantInfoImpl; @@ -126,6 +140,7 @@ public void initPersistentTopics() throws Exception { @Override @BeforeMethod protected void setup() throws Exception { + conf.setTopicLevelPoliciesEnabled(false); super.internalSetup(); persistentTopics = spy(PersistentTopics.class); persistentTopics.setServletContext(new MockServletContext()); @@ -163,8 +178,8 @@ protected void setup() throws Exception { doReturn(spy(new TopicResources(pulsar.getLocalMetadataStore()))).when(resources).getTopicResources(); doReturn(resources).when(pulsar).getPulsarResources(); - admin.clusters().createCluster("use", ClusterData.builder().serviceUrl("http://broker-use.com:8080").build()); - admin.clusters().createCluster("test", ClusterData.builder().serviceUrl("http://broker-use.com:8080").build()); + admin.clusters().createCluster("use", ClusterData.builder().serviceUrl("http://127.0.0.3:8082").build()); + admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); admin.tenants().createTenant(this.testTenant, new TenantInfoImpl(Set.of("role1", "role2"), Set.of(testLocalCluster, "test"))); admin.tenants().createTenant("pulsar", @@ -236,7 +251,7 @@ public void testGetSubscriptions() { response = mock(AsyncResponse.class); persistentTopics.getSubscriptions(response, testTenant, testNamespace, testLocalTopicName + "-partition-0", true); - verify(response, timeout(5000).times(1)).resume(List.of("test")); + verify(response, timeout(5000).times(1)).resume(Set.of("test")); // 6) Delete the subscription response = mock(AsyncResponse.class); @@ -250,7 +265,7 @@ public void testGetSubscriptions() { response = mock(AsyncResponse.class); persistentTopics.getSubscriptions(response, testTenant, testNamespace, testLocalTopicName + "-partition-0", true); - verify(response, timeout(5000).times(1)).resume(new ArrayList<>()); + verify(response, timeout(5000).times(1)).resume(Set.of()); // 8) Create a sub of partitioned-topic response = mock(AsyncResponse.class); @@ -264,16 +279,16 @@ public void testGetSubscriptions() { response = mock(AsyncResponse.class); persistentTopics.getSubscriptions(response, testTenant, testNamespace, testLocalTopicName + "-partition-1", true); - verify(response, timeout(5000).times(1)).resume(List.of("test")); + verify(response, timeout(5000).times(1)).resume(Set.of("test")); // response = mock(AsyncResponse.class); persistentTopics.getSubscriptions(response, testTenant, testNamespace, testLocalTopicName + "-partition-0", true); - verify(response, timeout(5000).times(1)).resume(new ArrayList<>()); + verify(response, timeout(5000).times(1)).resume(Set.of()); // response = mock(AsyncResponse.class); persistentTopics.getSubscriptions(response, testTenant, testNamespace, testLocalTopicName, true); - verify(response, timeout(5000).times(1)).resume(List.of("test")); + verify(response, timeout(5000).times(1)).resume(Set.of("test")); // 9) Delete the partitioned topic response = mock(AsyncResponse.class); @@ -313,7 +328,7 @@ public void testCreateSubscriptions() throws Exception { Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); response = mock(AsyncResponse.class); - persistentTopics.getStats(response, testTenant, testNamespace, testLocalTopicName, true, true, false, false); + persistentTopics.getStats(response, testTenant, testNamespace, testLocalTopicName, true, true, false, false, false, false); ArgumentCaptor statCaptor = ArgumentCaptor.forClass(TopicStats.class); verify(response, timeout(5000).times(1)).resume(statCaptor.capture()); TopicStats topicStats = statCaptor.getValue(); @@ -331,7 +346,7 @@ public void testCreateSubscriptions() throws Exception { Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); response = mock(AsyncResponse.class); - persistentTopics.getStats(response, testTenant, testNamespace, testLocalTopicName, true, true, false, false); + persistentTopics.getStats(response, testTenant, testNamespace, testLocalTopicName, true, true, false, false, false, false); statCaptor = ArgumentCaptor.forClass(TopicStats.class); verify(response, timeout(5000).times(1)).resume(statCaptor.capture()); topicStats = statCaptor.getValue(); @@ -350,7 +365,7 @@ public void testCreateSubscriptions() throws Exception { Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); response = mock(AsyncResponse.class); - persistentTopics.getStats(response, testTenant, testNamespace, testLocalTopicName, true, true, false, false); + persistentTopics.getStats(response, testTenant, testNamespace, testLocalTopicName, true, true, false, false, false, false); statCaptor = ArgumentCaptor.forClass(TopicStats.class); verify(response, timeout(5000).times(1)).resume(statCaptor.capture()); topicStats = statCaptor.getValue(); @@ -369,7 +384,7 @@ public void testCreateSubscriptions() throws Exception { Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); response = mock(AsyncResponse.class); - persistentTopics.getStats(response, testTenant, testNamespace, testLocalTopicName, true, true, false, false); + persistentTopics.getStats(response, testTenant, testNamespace, testLocalTopicName, true, true, false, false, false, false); statCaptor = ArgumentCaptor.forClass(TopicStats.class); verify(response, timeout(5000).times(1)).resume(statCaptor.capture()); topicStats = statCaptor.getValue(); @@ -427,6 +442,7 @@ public void testTerminate() { persistentTopics.createNonPartitionedTopic(response, testTenant, testNamespace, testLocalTopicName, true, null); // 2) Create a subscription + response = mock(AsyncResponse.class); persistentTopics.createSubscription(response, testTenant, testNamespace, testLocalTopicName, "test", true, new ResetCursorData(MessageId.earliest), false); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); @@ -537,12 +553,13 @@ public void testCreateNonPartitionedTopic() { @Test public void testCreatePartitionedTopic() { - AsyncResponse response = mock(AsyncResponse.class); - ArgumentCaptor responseCaptor = - ArgumentCaptor.forClass(PartitionedTopicMetadata.class); final String topicName = "standard-partitioned-topic-a"; - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, 2, true); + persistentTopics.createPartitionedTopic(mock(AsyncResponse.class), testTenant, testNamespace, topicName, 2, + true); Awaitility.await().untilAsserted(() -> { + ArgumentCaptor responseCaptor = + ArgumentCaptor.forClass(PartitionedTopicMetadata.class); + AsyncResponse response = mock(AsyncResponse.class); persistentTopics.getPartitionedMetadata(response, testTenant, testNamespace, topicName, true, false); verify(response, timeout(5000).atLeast(1)).resume(responseCaptor.capture()); @@ -789,26 +806,26 @@ public void testGetPartitionedTopicsList() throws KeeperException, InterruptedEx Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); response = mock(AsyncResponse.class); - responseCaptor = ArgumentCaptor.forClass(Response.class); + ArgumentCaptor> listOfStringsCaptor = ArgumentCaptor.forClass(List.class); persistentTopics.getPartitionedTopicList(response, testTenant, testNamespace, false); - verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); - List persistentPartitionedTopics = (List) responseCaptor.getValue(); + verify(response, timeout(5000).times(1)).resume(listOfStringsCaptor.capture()); + List persistentPartitionedTopics = listOfStringsCaptor.getValue(); Assert.assertEquals(persistentPartitionedTopics.size(), 1); Assert.assertEquals(TopicName.get(persistentPartitionedTopics.get(0)).getDomain().value(), TopicDomain.persistent.value()); response = mock(AsyncResponse.class); - responseCaptor = ArgumentCaptor.forClass(Response.class); + listOfStringsCaptor = ArgumentCaptor.forClass(List.class); persistentTopics.getPartitionedTopicList(response, testTenant, testNamespace, true); - verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); - persistentPartitionedTopics = (List) responseCaptor.getValue(); + verify(response, timeout(5000).times(1)).resume(listOfStringsCaptor.capture()); + persistentPartitionedTopics = listOfStringsCaptor.getValue(); Assert.assertEquals(persistentPartitionedTopics.size(), 2); response = mock(AsyncResponse.class); - responseCaptor = ArgumentCaptor.forClass(Response.class); + listOfStringsCaptor = ArgumentCaptor.forClass(List.class); nonPersistentTopic.getPartitionedTopicList(response, testTenant, testNamespace, false); - verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); - List nonPersistentPartitionedTopics = (List) responseCaptor.getValue(); + verify(response, timeout(5000).times(1)).resume(listOfStringsCaptor.capture()); + List nonPersistentPartitionedTopics = listOfStringsCaptor.getValue(); Assert.assertEquals(nonPersistentPartitionedTopics.size(), 1); Assert.assertEquals(TopicName.get(nonPersistentPartitionedTopics.get(0)).getDomain().value(), TopicDomain.non_persistent.value()); @@ -829,17 +846,17 @@ public void testGetList() throws Exception { Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); response = mock(AsyncResponse.class); - responseCaptor = ArgumentCaptor.forClass(Response.class); + ArgumentCaptor> listOfStringsCaptor = ArgumentCaptor.forClass(List.class); persistentTopics.getList(response, testTenant, testNamespace, null, false); - verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); - List topics = (List) responseCaptor.getValue(); + verify(response, timeout(5000).times(1)).resume(listOfStringsCaptor.capture()); + List topics = listOfStringsCaptor.getValue(); Assert.assertEquals(topics.size(), 1); response = mock(AsyncResponse.class); - responseCaptor = ArgumentCaptor.forClass(Response.class); + listOfStringsCaptor = ArgumentCaptor.forClass(List.class); persistentTopics.getList(response, testTenant, testNamespace, null, true); - verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); - topics = (List) responseCaptor.getValue(); + verify(response, timeout(5000).times(1)).resume(listOfStringsCaptor.capture()); + topics = listOfStringsCaptor.getValue(); Assert.assertEquals(topics.size(), 2); response = mock(AsyncResponse.class); @@ -855,17 +872,17 @@ public void testGetList() throws Exception { Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); response = mock(AsyncResponse.class); - responseCaptor = ArgumentCaptor.forClass(Response.class); + listOfStringsCaptor = ArgumentCaptor.forClass(List.class); nonPersistentTopic.getList(response, testTenant, testNamespace, null, false); - verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); - topics = (List) responseCaptor.getValue(); + verify(response, timeout(5000).times(1)).resume(listOfStringsCaptor.capture()); + topics = listOfStringsCaptor.getValue(); Assert.assertEquals(topics.size(), 1); response = mock(AsyncResponse.class); - responseCaptor = ArgumentCaptor.forClass(Response.class); + listOfStringsCaptor = ArgumentCaptor.forClass(List.class); nonPersistentTopic.getList(response, testTenant, testNamespace, null, true); - verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); - topics = (List) responseCaptor.getValue(); + verify(response, timeout(5000).times(1)).resume(listOfStringsCaptor.capture()); + topics = listOfStringsCaptor.getValue(); Assert.assertEquals(topics.size(), 2); } @@ -873,34 +890,41 @@ public void testGetList() throws Exception { public void testGrantNonPartitionedTopic() { final String topicName = "non-partitioned-topic"; AsyncResponse response = mock(AsyncResponse.class); + ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); persistentTopics.createNonPartitionedTopic(response, testTenant, testNamespace, topicName, true, null); + verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); + Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); String role = "role"; Set expectActions = new HashSet<>(); expectActions.add(AuthAction.produce); response = mock(AsyncResponse.class); - ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); + responseCaptor = ArgumentCaptor.forClass(Response.class); persistentTopics.grantPermissionsOnTopic(response, testTenant, testNamespace, topicName, role, expectActions); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); response = mock(AsyncResponse.class); - responseCaptor = ArgumentCaptor.forClass(Response.class); + ArgumentCaptor>> permissionsCaptor = ArgumentCaptor.forClass(Map.class); persistentTopics.getPermissionsOnTopic(response, testTenant, testNamespace, topicName); - verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); - Map> permissions = (Map>) responseCaptor.getValue(); + verify(response, timeout(5000).times(1)).resume(permissionsCaptor.capture()); + Map> permissions = permissionsCaptor.getValue(); Assert.assertEquals(permissions.get(role), expectActions); } @Test - public void testCreateExistedPartition() { - final AsyncResponse response = mock(AsyncResponse.class); - final String topicName = "test-create-existed-partition"; + public void testCreateExistedPartition() throws InterruptedException { + AsyncResponse response = mock(AsyncResponse.class); + ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); + final String topicName = "testcreateexisted"; persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, 3, true); + verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); + Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); final String partitionName = TopicName.get(topicName).getPartition(0).getLocalName(); - ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(RestException.class); - persistentTopics.createNonPartitionedTopic(response, testTenant, testNamespace, partitionName, false, null); - verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); - Assert.assertEquals(responseCaptor.getValue().getResponse().getStatus(), + response = mock(AsyncResponse.class); + ArgumentCaptor restExceptionCaptor = ArgumentCaptor.forClass(RestException.class); + persistentTopics.createNonPartitionedTopic(response, testTenant, testNamespace, partitionName, true, null); + verify(response, timeout(5000).times(1)).resume(restExceptionCaptor.capture()); + Assert.assertEquals(restExceptionCaptor.getValue().getResponse().getStatus(), Response.Status.CONFLICT.getStatusCode()); } @@ -925,36 +949,26 @@ public void testGrantPartitionedTopic() { verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); response = mock(AsyncResponse.class); - responseCaptor = ArgumentCaptor.forClass(Response.class); + ArgumentCaptor>> permissionsCaptor = ArgumentCaptor.forClass(Map.class); persistentTopics.getPermissionsOnTopic(response, testTenant, testNamespace, partitionedTopicName); - verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); - Map> permissions = (Map>) responseCaptor.getValue(); + verify(response, timeout(5000).times(1)).resume(permissionsCaptor.capture()); + Map> permissions = permissionsCaptor.getValue(); Assert.assertEquals(permissions.get(role), expectActions); - TopicName topicName = TopicName.get(TopicDomain.persistent.value(), testTenant, testNamespace, - partitionedTopicName); - for (int i = 0; i < numPartitions; i++) { - TopicName partition = topicName.getPartition(i); - response = mock(AsyncResponse.class); - responseCaptor = ArgumentCaptor.forClass(Response.class); - persistentTopics.getPermissionsOnTopic(response, testTenant, testNamespace, - partition.getEncodedLocalName()); - verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); - Map> partitionPermissions = - (Map>) responseCaptor.getValue(); - Assert.assertEquals(partitionPermissions.get(role), expectActions); - } } @Test public void testRevokeNonPartitionedTopic() { final String topicName = "non-partitioned-topic"; AsyncResponse response = mock(AsyncResponse.class); + ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); persistentTopics.createNonPartitionedTopic(response, testTenant, testNamespace, topicName, true, null); + verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); + Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); String role = "role"; Set expectActions = new HashSet<>(); expectActions.add(AuthAction.produce); response = mock(AsyncResponse.class); - ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); + responseCaptor = ArgumentCaptor.forClass(Response.class); persistentTopics.grantPermissionsOnTopic(response, testTenant, testNamespace, topicName, role, expectActions); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); @@ -964,10 +978,10 @@ public void testRevokeNonPartitionedTopic() { verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); response = mock(AsyncResponse.class); - responseCaptor = ArgumentCaptor.forClass(Response.class); + ArgumentCaptor>> permissionsCaptor = ArgumentCaptor.forClass(Map.class); persistentTopics.getPermissionsOnTopic(response, testTenant, testNamespace, topicName); - verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); - Map> permissions = (Map>) responseCaptor.getValue(); + verify(response, timeout(5000).times(1)).resume(permissionsCaptor.capture()); + Map> permissions = permissionsCaptor.getValue(); Assert.assertEquals(permissions.get(role), null); } @@ -996,22 +1010,22 @@ public void testRevokePartitionedTopic() { verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); response = mock(AsyncResponse.class); - responseCaptor = ArgumentCaptor.forClass(Response.class); + ArgumentCaptor>> permissionsCaptor = ArgumentCaptor.forClass(Map.class); persistentTopics.getPermissionsOnTopic(response, testTenant, testNamespace, partitionedTopicName); - verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); - Map> permissions = (Map>) responseCaptor.getValue(); + verify(response, timeout(5000).times(1)).resume(permissionsCaptor.capture()); + Map> permissions = permissionsCaptor.getValue(); Assert.assertEquals(permissions.get(role), null); TopicName topicName = TopicName.get(TopicDomain.persistent.value(), testTenant, testNamespace, partitionedTopicName); for (int i = 0; i < numPartitions; i++) { TopicName partition = topicName.getPartition(i); response = mock(AsyncResponse.class); - responseCaptor = ArgumentCaptor.forClass(Response.class); + permissionsCaptor = ArgumentCaptor.forClass(Map.class); persistentTopics.getPermissionsOnTopic(response, testTenant, testNamespace, partition.getEncodedLocalName()); - verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); + verify(response, timeout(5000).times(1)).resume(permissionsCaptor.capture()); Map> partitionPermissions = - (Map>) responseCaptor.getValue(); + permissionsCaptor.getValue(); Assert.assertEquals(partitionPermissions.get(role), null); } } @@ -1369,20 +1383,36 @@ public void testGetMessageById() throws Exception { Message message2 = admin.topics().getMessageById(topicName2, id2.getLedgerId(), id2.getEntryId()); Assert.assertEquals(message2.getData(), data2.getBytes()); - Message message3 = null; - try { - message3 = admin.topics().getMessageById(topicName2, id1.getLedgerId(), id1.getEntryId()); - Assert.fail(); - } catch (Exception e) { - Assert.assertNull(message3); - } + Assert.expectThrows(PulsarAdminException.NotFoundException.class, () -> { + admin.topics().getMessageById(topicName2, id1.getLedgerId(), id1.getEntryId()); + }); + Assert.expectThrows(PulsarAdminException.NotFoundException.class, () -> { + admin.topics().getMessageById(topicName1, id2.getLedgerId(), id2.getEntryId()); + }); + } - Message message4 = null; - try { - message4 = admin.topics().getMessageById(topicName1, id2.getLedgerId(), id2.getEntryId()); - Assert.fail(); - } catch (Exception e) { - Assert.assertNull(message4); + @Test + public void testGetMessageById4SpecialPropsInMsg() throws Exception { + TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test")); + admin.tenants().createTenant("tenant-xyz", tenantInfo); + admin.namespaces().createNamespace("tenant-xyz/ns-abc", Set.of("test")); + final String topicName1 = "persistent://tenant-xyz/ns-abc/testGetMessageById1"; + admin.topics().createNonPartitionedTopic(topicName1); + Map inSpecialProps = new HashMap<>(); + inSpecialProps.put("city=shanghai", "tag"); + inSpecialProps.put("city,beijing", "haidian"); + @Cleanup + ProducerBase producer1 = (ProducerBase) pulsarClient.newProducer().topic(topicName1) + .enableBatching(false).create(); + String data1 = "test1"; + MessageIdImpl id1 = (MessageIdImpl) producer1.newMessage().value(data1.getBytes()).properties(inSpecialProps) + .send(); + + Message message1 = admin.topics().getMessageById(topicName1, id1.getLedgerId(), id1.getEntryId()); + Assert.assertEquals(message1.getData(), data1.getBytes()); + Map outSpecialProps = message1.getProperties(); + for (String k : inSpecialProps.keySet()) { + Assert.assertEquals(inSpecialProps.get(k), outSpecialProps.get(k)); } } @@ -1439,6 +1469,69 @@ public void onSendAcknowledgement(Producer producer, Message message, MessageId .compareTo(id2) > 0); } + @Test + public void testGetMessageIdByTimestampWithCompaction() throws Exception { + TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test")); + admin.tenants().createTenant("tenant-xyz", tenantInfo); + admin.namespaces().createNamespace("tenant-xyz/ns-abc", Set.of("test")); + final String topicName = "persistent://tenant-xyz/ns-abc/testGetMessageIdByTimestampWithCompaction"; + admin.topics().createNonPartitionedTopic(topicName); + + Map publishTimeMap = new ConcurrentHashMap<>(); + @Cleanup + ProducerBase producer = (ProducerBase) pulsarClient.newProducer().topic(topicName) + .enableBatching(false) + .intercept(new ProducerInterceptor() { + @Override + public void close() { + + } + + @Override + public boolean eligible(Message message) { + return true; + } + + @Override + public Message beforeSend(Producer producer, Message message) { + return message; + } + + @Override + public void onSendAcknowledgement(Producer producer, Message message, MessageId msgId, + Throwable exception) { + publishTimeMap.put(message.getMessageId(), message.getPublishTime()); + } + }) + .create(); + + MessageId id1 = producer.newMessage().key("K1").value("test1".getBytes()).send(); + MessageId id2 = producer.newMessage().key("K2").value("test2".getBytes()).send(); + + long publish1 = publishTimeMap.get(id1); + long publish2 = publishTimeMap.get(id2); + Assert.assertTrue(publish1 < publish2); + + admin.topics().triggerCompaction(topicName); + Awaitility.await().untilAsserted(() -> + assertSame(admin.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS)); + + admin.topics().unload(topicName); + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(topicName, false); + assertEquals(internalStats.ledgers.size(), 1); + assertEquals(internalStats.ledgers.get(0).entries, 0); + }); + + Assert.assertEquals(admin.topics().getMessageIdByTimestamp(topicName, publish1 - 1), id1); + Assert.assertEquals(admin.topics().getMessageIdByTimestamp(topicName, publish1), id1); + Assert.assertEquals(admin.topics().getMessageIdByTimestamp(topicName, publish1 + 1), id2); + Assert.assertEquals(admin.topics().getMessageIdByTimestamp(topicName, publish2), id2); + Assert.assertTrue(admin.topics().getMessageIdByTimestamp(topicName, publish2 + 1) + .compareTo(id2) > 0); + } + @Test public void testGetBatchMessageIdByTimestamp() throws Exception { TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test")); @@ -1647,7 +1740,7 @@ public void testUpdatePartitionedTopic() true, 3); verify(response, timeout(5000).times(1)).resume(throwableCaptor.capture()); Assert.assertEquals(throwableCaptor.getValue().getMessage(), - "Expect partitions 3 can't less than current partitions 4."); + "Desired partitions 3 can't be less than the current partitions 4."); response = mock(AsyncResponse.class); metaCaptor = ArgumentCaptor.forClass(PartitionedTopicMetadata.class); @@ -1655,6 +1748,51 @@ public void testUpdatePartitionedTopic() verify(response, timeout(5000).times(1)).resume(metaCaptor.capture()); partitionedTopicMetadata = metaCaptor.getValue(); Assert.assertEquals(partitionedTopicMetadata.partitions, 4); + + // test for configuration maxNumPartitionsPerPartitionedTopic + conf.setMaxNumPartitionsPerPartitionedTopic(4); + response = mock(AsyncResponse.class); + throwableCaptor = ArgumentCaptor.forClass(Throwable.class); + persistentTopics.updatePartitionedTopic(response, testTenant, testNamespaceLocal, topicName, false, true, + true, 5); + verify(response, timeout(5000).times(1)).resume(throwableCaptor.capture()); + Assert.assertEquals(throwableCaptor.getValue().getMessage(), + "Desired partitions 5 can't be greater than the maximum partitions per topic 4."); + + response = mock(AsyncResponse.class); + metaCaptor = ArgumentCaptor.forClass(PartitionedTopicMetadata.class); + persistentTopics.getPartitionedMetadata(response, testTenant, testNamespaceLocal, topicName, true, false); + verify(response, timeout(5000).times(1)).resume(metaCaptor.capture()); + partitionedTopicMetadata = metaCaptor.getValue(); + Assert.assertEquals(partitionedTopicMetadata.partitions, 4); + + conf.setMaxNumPartitionsPerPartitionedTopic(-1); + response = mock(AsyncResponse.class); + responseCaptor = ArgumentCaptor.forClass(Response.class); + persistentTopics.updatePartitionedTopic(response, testTenant, testNamespaceLocal, topicName, false, true, + true, 5); + verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); + + response = mock(AsyncResponse.class); + metaCaptor = ArgumentCaptor.forClass(PartitionedTopicMetadata.class); + persistentTopics.getPartitionedMetadata(response, testTenant, testNamespaceLocal, topicName, true, false); + verify(response, timeout(5000).times(1)).resume(metaCaptor.capture()); + partitionedTopicMetadata = metaCaptor.getValue(); + Assert.assertEquals(partitionedTopicMetadata.partitions, 5); + + conf.setMaxNumPartitionsPerPartitionedTopic(0); + response = mock(AsyncResponse.class); + responseCaptor = ArgumentCaptor.forClass(Response.class); + persistentTopics.updatePartitionedTopic(response, testTenant, testNamespaceLocal, topicName, false, true, + true, 6); + verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); + + response = mock(AsyncResponse.class); + metaCaptor = ArgumentCaptor.forClass(PartitionedTopicMetadata.class); + persistentTopics.getPartitionedMetadata(response, testTenant, testNamespaceLocal, topicName, true, false); + verify(response, timeout(5000).times(1)).resume(metaCaptor.capture()); + partitionedTopicMetadata = metaCaptor.getValue(); + Assert.assertEquals(partitionedTopicMetadata.partitions, 6); } @Test @@ -1667,7 +1805,7 @@ public void testInternalGetReplicatedSubscriptionStatusFromLocal() throws Except admin.topics().createSubscription(topicName, subName, MessageId.latest); // partition-0 call from local and partition-1 call from admin. - NamespaceService namespaceService = spy(pulsar.getNamespaceService()); + NamespaceService namespaceService = pulsar.getNamespaceService(); doReturn(CompletableFuture.completedFuture(true)) .when(namespaceService).isServiceUnitOwnedAsync(topic.getPartition(0)); doReturn(CompletableFuture.completedFuture(false)) @@ -1688,4 +1826,83 @@ public void testInternalGetReplicatedSubscriptionStatusFromLocal() throws Except // verify we only call getReplicatedSubscriptionStatusAsync once. verify(topics, times(1)).getReplicatedSubscriptionStatusAsync(any(), any()); } + + @Test + public void testNamespaceResources() throws Exception { + String ns1V1 = "test/" + testNamespace + "v1"; + String ns1V2 = testNamespace + "v2"; + admin.namespaces().createNamespace(testTenant+"/"+ns1V1); + admin.namespaces().createNamespace(testTenant+"/"+ns1V2); + + List namespaces = pulsar.getPulsarResources().getNamespaceResources().listNamespacesAsync(testTenant) + .get(); + assertTrue(namespaces.contains(ns1V2)); + assertTrue(namespaces.contains(ns1V1)); + } + + @Test + public void testCreateMissingPartitions() throws Exception { + String topicName = "persistent://" + testTenant + "/" + testNamespaceLocal + "/testCreateMissingPartitions"; + assertThrows(PulsarAdminException.NotFoundException.class, () -> admin.topics().createMissedPartitions(topicName)); + } + + @Test + public void testForceDeleteSubscription() throws Exception { + try { + pulsar.getConfiguration().setAllowAutoSubscriptionCreation(false); + String topicName = "persistent://" + testTenant + "/" + testNamespaceLocal + "/testForceDeleteSubscription"; + String subName = "sub1"; + admin.topics().createNonPartitionedTopic(topicName); + admin.topics().createSubscription(topicName, subName, MessageId.latest); + + @Cleanup + Consumer c0 = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Shared) + .subscribe(); + @Cleanup + Consumer c1 = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Shared) + .subscribe(); + + admin.topics().deleteSubscription(topicName, subName, true); + } finally { + pulsar.getConfiguration().setAllowAutoSubscriptionCreation(true); + } + } + + @Test + public void testUpdatePropertiesOnNonDurableSub() throws Exception { + String topic = "persistent://" + testTenant + "/" + testNamespaceLocal + "/testUpdatePropertiesOnNonDurableSub"; + String subscription = "sub"; + admin.topics().createNonPartitionedTopic(topic); + + @Cleanup + Reader __ = pulsarClient.newReader(Schema.STRING) + .startMessageId(MessageId.earliest) + .subscriptionName(subscription) + .topic(topic) + .create(); + + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topic, false).get().get(); + PersistentSubscription subscription1 = persistentTopic.getSubscriptions().get(subscription); + assertNotNull(subscription1); + ManagedCursor cursor = subscription1.getCursor(); + + Map properties = admin.topics().getSubscriptionProperties(topic, subscription); + assertEquals(properties.size(), 0); + assertTrue(MapUtils.isEmpty(cursor.getCursorProperties())); + + admin.topics().updateSubscriptionProperties(topic, subscription, Map.of("foo", "bar")); + properties = admin.topics().getSubscriptionProperties(topic, subscription); + assertEquals(properties.size(), 1); + assertEquals(properties.get("foo"), "bar"); + + assertEquals(cursor.getCursorProperties().size(), 1); + assertEquals(cursor.getCursorProperties().get("foo"), "bar"); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PulsarClientImplMultiBrokersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PulsarClientImplMultiBrokersTest.java new file mode 100644 index 0000000000000..29604d0440b05 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PulsarClientImplMultiBrokersTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin; + +import static org.testng.Assert.fail; +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertTrue; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.MultiBrokerBaseTest; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.client.impl.LookupService; +import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.awaitility.reflect.WhiteboxImpl; +import org.testng.annotations.Test; + +/** + * Test multi-broker admin api. + */ +@Slf4j +@Test(groups = "broker-admin") +public class PulsarClientImplMultiBrokersTest extends MultiBrokerBaseTest { + @Override + protected int numberOfAdditionalBrokers() { + return 3; + } + + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + this.conf.setManagedLedgerMaxEntriesPerLedger(10); + } + + @Override + protected void onCleanup() { + super.onCleanup(); + } + + @Test(timeOut = 30 * 1000) + public void testReleaseUrlLookupServices() throws Exception { + PulsarClientImpl pulsarClient = (PulsarClientImpl) additionalBrokerClients.get(0); + Map urlLookupMap = WhiteboxImpl.getInternalState(pulsarClient, "urlLookupMap"); + assertEquals(urlLookupMap.size(), 0); + for (PulsarService pulsar : additionalBrokers) { + pulsarClient.getLookup(pulsar.getBrokerServiceUrl()); + pulsarClient.getLookup(pulsar.getWebServiceAddress()); + } + assertEquals(urlLookupMap.size(), additionalBrokers.size() * 2); + // Verify: lookup services will be release. + pulsarClient.close(); + assertEquals(urlLookupMap.size(), 0); + try { + for (PulsarService pulsar : additionalBrokers) { + pulsarClient.getLookup(pulsar.getBrokerServiceUrl()); + pulsarClient.getLookup(pulsar.getWebServiceAddress()); + } + fail("Expected a error when calling pulsarClient.getLookup if getLookup was closed"); + } catch (IllegalStateException illegalArgumentException) { + assertTrue(illegalArgumentException.getMessage().contains("has been closed")); + } + assertEquals(urlLookupMap.size(), 0); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAuthZTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAuthZTest.java new file mode 100644 index 0000000000000..2e05b28e747e4 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAuthZTest.java @@ -0,0 +1,2143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.admin; + +import static org.mockito.Mockito.doReturn; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import io.jsonwebtoken.Jwts; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.Cleanup; +import lombok.SneakyThrows; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.pulsar.broker.service.plugin.EntryFilterDefinition; +import org.apache.pulsar.broker.service.plugin.EntryFilterProvider; +import org.apache.pulsar.broker.service.plugin.EntryFilterTest; +import org.apache.pulsar.broker.testcontext.MockEntryFilterProvider; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.AuthAction; +import org.apache.pulsar.common.policies.data.AutoSubscriptionCreationOverride; +import org.apache.pulsar.common.policies.data.BacklogQuota; +import org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies; +import org.apache.pulsar.common.policies.data.DispatchRate; +import org.apache.pulsar.common.policies.data.EntryFilters; +import org.apache.pulsar.common.policies.data.InactiveTopicPolicies; +import org.apache.pulsar.common.policies.data.NamespaceOperation; +import org.apache.pulsar.common.policies.data.OffloadPolicies; +import org.apache.pulsar.common.policies.data.PersistencePolicies; +import org.apache.pulsar.common.policies.data.PolicyName; +import org.apache.pulsar.common.policies.data.PolicyOperation; +import org.apache.pulsar.common.policies.data.PublishRate; +import org.apache.pulsar.common.policies.data.RetentionPolicies; +import org.apache.pulsar.common.policies.data.SubscribeRate; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.common.policies.data.TopicOperation; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Test(groups = "broker-admin") +public class TopicAuthZTest extends AuthZTest { + + @SneakyThrows + @BeforeClass(alwaysRun = true) + public void setup() { + configureTokenAuthentication(); + configureDefaultAuthorization(); + start(); + this.superUserAdmin =PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(SUPER_USER_TOKEN)) + .build(); + final TenantInfo tenantInfo = superUserAdmin.tenants().getTenantInfo("public"); + tenantInfo.getAdminRoles().add(TENANT_ADMIN_SUBJECT); + superUserAdmin.tenants().updateTenant("public", tenantInfo); + this.tenantManagerAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(TENANT_ADMIN_TOKEN)) + .build(); + + } + + @SneakyThrows + @AfterClass(alwaysRun = true) + public void cleanup() { + close(); + } + + private AtomicBoolean setAuthorizationPolicyOperationChecker(String role, Object policyName, Object operation) { + AtomicBoolean execFlag = new AtomicBoolean(false); + if (operation instanceof PolicyOperation ) { + + doReturn(true) + .when(authorizationService).isValidOriginalPrincipal(Mockito.any(), Mockito.any(), Mockito.any()); + + Mockito.doAnswer(invocationOnMock -> { + String role_ = invocationOnMock.getArgument(4); + if (role.equals(role_)) { + PolicyName policyName_ = invocationOnMock.getArgument(1); + PolicyOperation operation_ = invocationOnMock.getArgument(2); + Assert.assertEquals(operation_, operation); + Assert.assertEquals(policyName_, policyName); + } + execFlag.set(true); + return invocationOnMock.callRealMethod(); + }).when(authorizationService).allowTopicPolicyOperationAsync(Mockito.any(), Mockito.any(), Mockito.any(), + Mockito.any(), Mockito.any(), Mockito.any()); + } else { + throw new IllegalArgumentException(""); + } + return execFlag; + } + + @DataProvider(name = "partitioned") + public static Object[][] partitioned() { + return new Object[][] { + {true}, + {false} + }; + } + + + @SneakyThrows + @Test + public void testUnloadAndCompactAndTrim() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createPartitionedTopic(topic, 2); + + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // test superuser + superUserAdmin.topics().unload(topic); + superUserAdmin.topics().triggerCompaction(topic); + superUserAdmin.topics().trimTopic(TopicName.get(topic).getPartition(0).getLocalName()); + superUserAdmin.topicPolicies().getSchemaCompatibilityStrategy(topic, false); + + // test tenant manager + tenantManagerAdmin.topics().unload(topic); + tenantManagerAdmin.topics().triggerCompaction(topic); + tenantManagerAdmin.topics().trimTopic(TopicName.get(topic).getPartition(0).getLocalName()); + tenantManagerAdmin.topicPolicies().getSchemaCompatibilityStrategy(topic, false); + + // test nobody + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().unload(topic)); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().triggerCompaction(topic)); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().trimTopic(TopicName.get(topic).getPartition(0).getLocalName())); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getSchemaCompatibilityStrategy(topic, false)); + + // Test only super/admin can do the operation, other auth are not permitted. + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().unload(topic)); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().triggerCompaction(topic)); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().trimTopic(topic)); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getSchemaCompatibilityStrategy(topic, false)); + + superUserAdmin.topics().revokePermissions(topic, subject); + } + superUserAdmin.topics().deletePartitionedTopic(topic, true); + } + + @Test + @SneakyThrows + public void testGetManagedLedgerInfo() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createPartitionedTopic(topic, 2); + + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // test superuser + superUserAdmin.topics().getInternalInfo(topic); + + // test tenant manager + tenantManagerAdmin.topics().getInternalInfo(topic); + + // test nobody + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getInternalInfo(topic)); + + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.GET_STATS); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + if (action == AuthAction.produce || action == AuthAction.consume) { + subAdmin.topics().getInternalInfo(topic); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getInternalInfo(topic)); + } + superUserAdmin.topics().revokePermissions(topic, subject); + } + + Assert.assertTrue(execFlag.get()); + + superUserAdmin.topics().deletePartitionedTopic(topic, true); + } + + @Test + @SneakyThrows + public void testGetPartitionedStatsAndInternalStats() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createPartitionedTopic(topic, 2); + + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // test superuser + superUserAdmin.topics().getPartitionedStats(topic, false); + superUserAdmin.topics().getPartitionedInternalStats(topic); + + // test tenant manager + tenantManagerAdmin.topics().getPartitionedStats(topic, false); + tenantManagerAdmin.topics().getPartitionedInternalStats(topic); + + // test nobody + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getPartitionedStats(topic, false)); + + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.GET_STATS); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getPartitionedInternalStats(topic)); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + if (action == AuthAction.produce || action == AuthAction.consume) { + subAdmin.topics().getPartitionedStats(topic, false); + subAdmin.topics().getPartitionedInternalStats(topic); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getPartitionedStats(topic, false)); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getPartitionedInternalStats(topic)); + } + superUserAdmin.topics().revokePermissions(topic, subject); + } + superUserAdmin.topics().deletePartitionedTopic(topic, true); + } + + @Test + @SneakyThrows + public void testCreateSubscriptionAndUpdateSubscriptionPropertiesAndAnalyzeSubscriptionBacklog() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createPartitionedTopic(topic, 2); + AtomicInteger suffix = new AtomicInteger(1); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + superUserAdmin.topics().createSubscription(topic, "test-sub" + suffix.incrementAndGet(), MessageId.earliest); + + // test tenant manager + tenantManagerAdmin.topics().createSubscription(topic, "test-sub" + suffix.incrementAndGet(), MessageId.earliest); + + // test nobody + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().createSubscription(topic, "test-sub" + suffix.incrementAndGet(), MessageId.earliest)); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + if (action == AuthAction.consume) { + subAdmin.topics().createSubscription(topic, "test-sub" + suffix.incrementAndGet(), MessageId.earliest); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().createSubscription(topic, "test-sub" + suffix.incrementAndGet(), MessageId.earliest)); + } + superUserAdmin.topics().revokePermissions(topic, subject); + } + // test UpdateSubscriptionProperties + Map properties = new HashMap<>(); + superUserAdmin.topics().createSubscription(topic, "test-sub", MessageId.earliest); + // test superuser + superUserAdmin.topics().updateSubscriptionProperties(topic, "test-sub" , properties); + superUserAdmin.topics().getSubscriptionProperties(topic, "test-sub"); + superUserAdmin.topics().analyzeSubscriptionBacklog(TopicName.get(topic).getPartition(0).getLocalName(), "test-sub", Optional.empty()); + + // test tenant manager + tenantManagerAdmin.topics().updateSubscriptionProperties(topic, "test-sub" , properties); + tenantManagerAdmin.topics().getSubscriptionProperties(topic, "test-sub"); + tenantManagerAdmin.topics().analyzeSubscriptionBacklog(TopicName.get(topic).getPartition(0).getLocalName(), "test-sub", Optional.empty()); + + // test nobody + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.CONSUME); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().updateSubscriptionProperties(topic, "test-sub", properties)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.CONSUME); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getSubscriptionProperties(topic, "test-sub")); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.CONSUME); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().analyzeSubscriptionBacklog(TopicName.get(topic).getPartition(0).getLocalName(), "test-sub", Optional.empty())); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + if (action == AuthAction.consume) { + subAdmin.topics().updateSubscriptionProperties(topic, "test-sub", properties); + subAdmin.topics().getSubscriptionProperties(topic, "test-sub"); + subAdmin.topics().analyzeSubscriptionBacklog(TopicName.get(topic).getPartition(0).getLocalName(), "test-sub", Optional.empty()); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().updateSubscriptionProperties(topic, "test-sub", properties)); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getSubscriptionProperties(topic, "test-sub")); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().analyzeSubscriptionBacklog(TopicName.get(topic).getPartition(0).getLocalName(), "test-sub", Optional.empty())); + } + superUserAdmin.topics().revokePermissions(topic, subject); + } + superUserAdmin.topics().deletePartitionedTopic(topic, true); + } + + @Test + @SneakyThrows + public void testCreateMissingPartition() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createPartitionedTopic(topic, 2); + AtomicInteger suffix = new AtomicInteger(1); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + superUserAdmin.topics().createMissedPartitions(topic); + + // test tenant manager + + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, NamespaceOperation.CREATE_TOPIC); + tenantManagerAdmin.topics().createMissedPartitions(topic); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.LOOKUP); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().createMissedPartitions(topic)); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().createMissedPartitions(topic)); + superUserAdmin.topics().revokePermissions(topic, subject); + } + superUserAdmin.topics().deletePartitionedTopic(topic, true); + } + + @Test(dataProvider = "partitioned") + @SneakyThrows + public void testPartitionedTopicMetadata(boolean partitioned) { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, partitioned); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + superUserAdmin.topics().getPartitionedTopicMetadata(topic); + + // test tenant manager + tenantManagerAdmin.topics().getPartitionedTopicMetadata(topic); + + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.LOOKUP); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getPartitionedTopicMetadata(topic)); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + if (AuthAction.produce == action || AuthAction.consume == action) { + subAdmin.topics().getPartitionedTopicMetadata(topic); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getPartitionedTopicMetadata(topic)); + } + superUserAdmin.topics().revokePermissions(topic, subject); + } + deleteTopic(topic, partitioned); + } + + @Test(dataProvider = "partitioned") + @SneakyThrows + public void testGetProperties(boolean partitioned) { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, partitioned); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + Map properties = new HashMap<>(); + properties.put("key1", "value1"); + superUserAdmin.topics().getProperties(topic); + + // test tenant manager + tenantManagerAdmin.topics().getProperties(topic); + + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.GET_METADATA); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getProperties(topic)); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + if (AuthAction.produce == action || AuthAction.consume == action) { + subAdmin.topics().getProperties(topic); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getProperties(topic)); + } + superUserAdmin.topics().revokePermissions(topic, subject); + } + deleteTopic(topic, partitioned); + } + + @Test(dataProvider = "partitioned") + @SneakyThrows + public void testUpdateProperties(boolean partitioned) { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, partitioned); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + Map properties = new HashMap<>(); + properties.put("key1", "value1"); + superUserAdmin.topics().updateProperties(topic, properties); + + // test tenant manager + tenantManagerAdmin.topics().updateProperties(topic, properties); + + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.UPDATE_METADATA); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().updateProperties(topic, properties)); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().updateProperties(topic, properties)); + superUserAdmin.topics().revokePermissions(topic, subject); + } + deleteTopic(topic, partitioned); + } + + @Test(dataProvider = "partitioned") + @SneakyThrows + public void testRemoveProperties(boolean partitioned) { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, partitioned); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + superUserAdmin.topics().removeProperties(topic, "key1"); + + // test tenant manager + tenantManagerAdmin.topics().removeProperties(topic, "key1"); + + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.DELETE_METADATA); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().removeProperties(topic, "key1")); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().removeProperties(topic, "key1")); + superUserAdmin.topics().revokePermissions(topic, subject); + } + deleteTopic(topic, partitioned); + } + + @Test + @SneakyThrows + public void testDeletePartitionedTopic() { + final String random = UUID.randomUUID().toString(); + String ns = "public/default/"; + final String topic = "persistent://" + ns + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + createTopic(topic , true); + superUserAdmin.topics().deletePartitionedTopic(topic); + + // test tenant manager + createTopic(topic, true); + tenantManagerAdmin.topics().deletePartitionedTopic(topic); + + createTopic(topic, true); + + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, NamespaceOperation.DELETE_TOPIC); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().deletePartitionedTopic(topic)); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.namespaces().grantPermissionOnNamespace(ns, subject, Set.of(action)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().deletePartitionedTopic(topic)); + superUserAdmin.namespaces().revokePermissionsOnNamespace(ns, subject); + } + } + + @Test(dataProvider = "partitioned") + @SneakyThrows + public void testGetSubscription(boolean partitioned) { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, partitioned); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + superUserAdmin.topics().getSubscriptions(topic); + + // test tenant manager + tenantManagerAdmin.topics().getSubscriptions(topic); + + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.GET_SUBSCRIPTIONS); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getSubscriptions(topic)); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + if (AuthAction.consume == action) { + subAdmin.topics().getSubscriptions(topic); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getSubscriptions(topic)); + } + superUserAdmin.topics().revokePermissions(topic, subject); + } + deleteTopic(topic, partitioned); + } + + @Test(dataProvider = "partitioned") + @SneakyThrows + public void testGetInternalStats(boolean partitioned) { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, partitioned); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + if (partitioned) { + superUserAdmin.topics().getPartitionedInternalStats(topic); + } else { + superUserAdmin.topics().getInternalStats(topic); + } + + // test tenant manager + if (partitioned) { + tenantManagerAdmin.topics().getPartitionedInternalStats(topic); + } else { + tenantManagerAdmin.topics().getInternalStats(topic); + + } + + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.GET_STATS); + if (partitioned) { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getPartitionedInternalStats(topic)); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getInternalStats(topic)); + } + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + if (AuthAction.produce == action || AuthAction.consume == action) { + if (partitioned) { + subAdmin.topics().getPartitionedInternalStats(topic); + } else { + subAdmin.topics().getInternalStats(topic); + } + } else { + if (partitioned) { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getPartitionedInternalStats(topic)); + + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getInternalStats(topic)); + } + } + superUserAdmin.topics().revokePermissions(topic, subject); + } + deleteTopic(topic, partitioned); + } + + @Test(dataProvider = "partitioned") + @SneakyThrows + public void testDeleteSubscription(boolean partitioned) { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, partitioned); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + String subName = "test-sub"; + superUserAdmin.topics().createSubscription(topic, subName, MessageId.latest); + superUserAdmin.topics().deleteSubscription(topic, subName); + + // test tenant manager + superUserAdmin.topics().createSubscription(topic, subName, MessageId.latest); + tenantManagerAdmin.topics().deleteSubscription(topic, subName); + + superUserAdmin.topics().createSubscription(topic, subName, MessageId.latest); + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.UNSUBSCRIBE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().deleteSubscription(topic, subName)); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + if (AuthAction.consume == action) { + subAdmin.topics().deleteSubscription(topic, "test-sub"); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().deleteSubscription(topic, "test-sub")); + } + superUserAdmin.topics().revokePermissions(topic, subject); + } + deleteTopic(topic, partitioned); + } + + @Test(dataProvider = "partitioned") + @SneakyThrows + public void testSkipAllMessage(boolean partitioned) { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, partitioned); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + String subName = "test-sub"; + superUserAdmin.topics().createSubscription(topic, subName, MessageId.latest); + superUserAdmin.topics().skipAllMessages(topic, subName); + + // test tenant manager + tenantManagerAdmin.topics().skipAllMessages(topic, subName); + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.SKIP); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().skipAllMessages(topic, subName)); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + if (AuthAction.consume == action) { + subAdmin.topics().skipAllMessages(topic,subName); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().skipAllMessages(topic, subName)); + } + superUserAdmin.topics().revokePermissions(topic, subject); + } + deleteTopic(topic, partitioned); + } + + @Test + @SneakyThrows + public void testSkipMessage() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + String subName = "test-sub"; + superUserAdmin.topics().createSubscription(topic, subName, MessageId.latest); + superUserAdmin.topics().skipMessages(topic, subName, 1); + + // test tenant manager + tenantManagerAdmin.topics().skipMessages(topic, subName, 1); + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.SKIP); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().skipMessages(topic, subName, 1)); + Assert.assertTrue(execFlag.get()); + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + if (AuthAction.consume == action) { + subAdmin.topics().skipMessages(topic, subName, 1); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().skipMessages(topic, subName, 1)); + } + superUserAdmin.topics().revokePermissions(topic, subject); + } + deleteTopic(topic, false); + } + + @Test(dataProvider = "partitioned") + @SneakyThrows + public void testExpireMessagesForAllSubscriptions(boolean partitioned) { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, partitioned); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + superUserAdmin.topics().expireMessagesForAllSubscriptions(topic, 1); + + // test tenant manager + tenantManagerAdmin.topics().expireMessagesForAllSubscriptions(topic, 1); + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.EXPIRE_MESSAGES); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().expireMessagesForAllSubscriptions(topic, 1)); + Assert.assertTrue(execFlag.get()); + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + if (AuthAction.consume == action) { + subAdmin.topics().expireMessagesForAllSubscriptions(topic, 1); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().expireMessagesForAllSubscriptions(topic, 1)); + } + superUserAdmin.topics().revokePermissions(topic, subject); + } + deleteTopic(topic, partitioned); + } + + @Test(dataProvider = "partitioned") + @SneakyThrows + public void testResetCursor(boolean partitioned) { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, partitioned); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + String subName = "test-sub"; + superUserAdmin.topics().createSubscription(topic, subName, MessageId.latest); + superUserAdmin.topics().resetCursor(topic, subName, System.currentTimeMillis()); + + // test tenant manager + tenantManagerAdmin.topics().resetCursor(topic, subName, System.currentTimeMillis()); + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.RESET_CURSOR); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().resetCursor(topic, subName, System.currentTimeMillis())); + Assert.assertTrue(execFlag.get()); + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + if (AuthAction.consume == action) { + subAdmin.topics().resetCursor(topic, subName, System.currentTimeMillis()); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().resetCursor(topic, subName, System.currentTimeMillis())); + } + superUserAdmin.topics().revokePermissions(topic, subject); + } + deleteTopic(topic, partitioned); + } + + @Test + @SneakyThrows + public void testResetCursorOnPosition() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + String subName = "test-sub"; + superUserAdmin.topics().createSubscription(topic, subName, MessageId.latest); + superUserAdmin.topics().resetCursor(topic, subName, MessageId.latest); + + // test tenant manager + tenantManagerAdmin.topics().resetCursor(topic, subName, MessageId.latest); + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.RESET_CURSOR); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().resetCursor(topic, subName, MessageId.latest)); + Assert.assertTrue(execFlag.get()); + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + if (AuthAction.consume == action) { + subAdmin.topics().resetCursor(topic, subName, MessageId.latest); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().resetCursor(topic, subName, MessageId.latest)); + } + superUserAdmin.topics().revokePermissions(topic, subject); + } + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testGetMessageById() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + @Cleanup + final PulsarClient pulsarClient = PulsarClient.builder() + .serviceUrl(getPulsarService().getBrokerServiceUrl()) + .authentication(new AuthenticationToken(TENANT_ADMIN_TOKEN)) + .build(); + @Cleanup + final Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + final MessageIdImpl messageId = (MessageIdImpl) producer.send("test"); + superUserAdmin.topics().getMessagesById(topic, messageId.getLedgerId(), messageId.getEntryId()); + + // test tenant manager + tenantManagerAdmin.topics().getMessagesById(topic, messageId.getLedgerId(), messageId.getEntryId()); + + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.PEEK_MESSAGES); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getMessagesById(topic, messageId.getLedgerId(), messageId.getEntryId())); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + if (AuthAction.consume == action) { + subAdmin.topics().getMessagesById(topic, messageId.getLedgerId(), messageId.getEntryId()); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getMessagesById(topic, messageId.getLedgerId(), messageId.getEntryId())); + } + superUserAdmin.topics().revokePermissions(topic, subject); + } + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testPeekNthMessage() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + @Cleanup + final PulsarClient pulsarClient = PulsarClient.builder() + .serviceUrl(getPulsarService().getBrokerServiceUrl()) + .authentication(new AuthenticationToken(TENANT_ADMIN_TOKEN)) + .build(); + String subName = "test-sub"; + @Cleanup + final Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + producer.send("test"); + superUserAdmin.topics().peekMessages(topic, subName, 1); + + // test tenant manager + tenantManagerAdmin.topics().peekMessages(topic, subName, 1); + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.PEEK_MESSAGES); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().peekMessages(topic, subName, 1)); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + if (AuthAction.consume == action) { + subAdmin.topics().peekMessages(topic, subName, 1); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().peekMessages(topic, subName, 1)); + } + superUserAdmin.topics().revokePermissions(topic, subject); + } + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testExamineMessage() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + @Cleanup + final PulsarClient pulsarClient = PulsarClient.builder() + .serviceUrl(getPulsarService().getBrokerServiceUrl()) + .authentication(new AuthenticationToken(TENANT_ADMIN_TOKEN)) + .build(); + @Cleanup + final Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + producer.send("test"); + superUserAdmin.topics().examineMessage(topic, "latest", 1); + + // test tenant manager + tenantManagerAdmin.topics().examineMessage(topic, "latest", 1); + + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.PEEK_MESSAGES); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().examineMessage(topic, "latest", 1)); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + if (AuthAction.consume == action) { + subAdmin.topics().examineMessage(topic, "latest", 1); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().examineMessage(topic, "latest", 1)); + } + superUserAdmin.topics().revokePermissions(topic, subject); + } + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testExpireMessage() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + @Cleanup + final PulsarClient pulsarClient = PulsarClient.builder() + .serviceUrl(getPulsarService().getBrokerServiceUrl()) + .authentication(new AuthenticationToken(TENANT_ADMIN_TOKEN)) + .build(); + String subName = "test-sub"; + superUserAdmin.topics().createSubscription(topic, subName, MessageId.latest); + @Cleanup + final Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + producer.send("test1"); + producer.send("test2"); + producer.send("test3"); + producer.send("test4"); + superUserAdmin.topics().expireMessages(topic, subName, 1); + + // test tenant manager + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.EXPIRE_MESSAGES); + tenantManagerAdmin.topics().expireMessages(topic, subName, 1); + Assert.assertTrue(execFlag.get()); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().expireMessages(topic, subName, 1)); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + if (AuthAction.consume == action) { + subAdmin.topics().expireMessages(topic, subName, 1); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().expireMessages(topic, subName, 1)); + } + superUserAdmin.topics().revokePermissions(topic, subject); + } + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testExpireMessageByPosition() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + @Cleanup + final PulsarClient pulsarClient = PulsarClient.builder() + .serviceUrl(getPulsarService().getBrokerServiceUrl()) + .authentication(new AuthenticationToken(TENANT_ADMIN_TOKEN)) + .build(); + String subName = "test-sub"; + superUserAdmin.topics().createSubscription(topic, subName, MessageId.latest); + @Cleanup + final Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + producer.send("test1"); + producer.send("test2"); + producer.send("test3"); + producer.send("test4"); + superUserAdmin.topics().expireMessages(topic, subName, MessageId.earliest, false); + + // test tenant manager + tenantManagerAdmin.topics().expireMessages(topic, subName, MessageId.earliest, false); + + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.EXPIRE_MESSAGES); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().expireMessages(topic, subName, MessageId.earliest, false)); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + if (AuthAction.consume == action) { + subAdmin.topics().expireMessages(topic, subName, MessageId.earliest, false); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().expireMessages(topic, subName, MessageId.earliest, false)); + } + superUserAdmin.topics().revokePermissions(topic, subject); + } + deleteTopic(topic, false); + } + + + + + @Test + @SneakyThrows + public void testSchemaCompatibility() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + superUserAdmin.topicPolicies().getSchemaCompatibilityStrategy(topic, true); + + // test tenant manager + tenantManagerAdmin.topicPolicies().getSchemaCompatibilityStrategy(topic, true); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getSchemaCompatibilityStrategy(topic, false)); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getSchemaCompatibilityStrategy(topic, false)); + superUserAdmin.topics().revokePermissions(topic, subject); + } + deleteTopic(topic, false); + } + + + + @Test + @SneakyThrows + public void testGetEntryFilter() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + superUserAdmin.topicPolicies().getEntryFiltersPerTopic(topic, true); + + // test tenant manager + tenantManagerAdmin.topicPolicies().getEntryFiltersPerTopic(topic, true); + + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.ENTRY_FILTERS, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getEntryFiltersPerTopic(topic, false)); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getEntryFiltersPerTopic(topic, false)); + superUserAdmin.topics().revokePermissions(topic, subject); + } + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testSetEntryFilter() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + final EntryFilterProvider oldEntryFilterProvider = getPulsarService().getBrokerService().getEntryFilterProvider(); + @Cleanup + final MockEntryFilterProvider testEntryFilterProvider = + new MockEntryFilterProvider(getServiceConfiguration()); + + testEntryFilterProvider + .setMockEntryFilters(new EntryFilterDefinition( + "test", + null, + EntryFilterTest.class.getName() + )); + FieldUtils.writeField(getPulsarService().getBrokerService(), + "entryFilterProvider", testEntryFilterProvider, true); + final EntryFilters entryFilter = new EntryFilters("test"); + superUserAdmin.topicPolicies().setEntryFiltersPerTopic(topic, entryFilter); + + // test tenant manager + tenantManagerAdmin.topicPolicies().setEntryFiltersPerTopic(topic, entryFilter); + + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.ENTRY_FILTERS, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setEntryFiltersPerTopic(topic, entryFilter)); + Assert.assertTrue(execFlag.get()); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setEntryFiltersPerTopic(topic, entryFilter)); + superUserAdmin.topics().revokePermissions(topic, subject); + } + deleteTopic(topic, false); + FieldUtils.writeField(getPulsarService().getBrokerService(), + "entryFilterProvider", oldEntryFilterProvider, true); + } + + @Test + @SneakyThrows + public void testRemoveEntryFilter() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + final EntryFilterProvider oldEntryFilterProvider = getPulsarService().getBrokerService().getEntryFilterProvider(); + @Cleanup + final MockEntryFilterProvider testEntryFilterProvider = + new MockEntryFilterProvider(getServiceConfiguration()); + + testEntryFilterProvider + .setMockEntryFilters(new EntryFilterDefinition( + "test", + null, + EntryFilterTest.class.getName() + )); + FieldUtils.writeField(getPulsarService().getBrokerService(), + "entryFilterProvider", testEntryFilterProvider, true); + final EntryFilters entryFilter = new EntryFilters("test"); + superUserAdmin.topicPolicies().removeEntryFiltersPerTopic(topic); + // test tenant manager + tenantManagerAdmin.topicPolicies().removeEntryFiltersPerTopic(topic); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removeEntryFiltersPerTopic(topic)); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removeEntryFiltersPerTopic(topic)); + superUserAdmin.topics().revokePermissions(topic, subject); + } + deleteTopic(topic, false); + FieldUtils.writeField(getPulsarService().getBrokerService(), + "entryFilterProvider", oldEntryFilterProvider, true); + } + + @Test + @SneakyThrows + public void testShadowTopic() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + + String shadowTopic = topic + "-shadow-topic"; + superUserAdmin.topics().createShadowTopic(shadowTopic, topic); + superUserAdmin.topics().setShadowTopics(topic, Lists.newArrayList(shadowTopic)); + superUserAdmin.topics().getShadowTopics(topic); + superUserAdmin.topics().removeShadowTopics(topic); + + + // test tenant manager + tenantManagerAdmin.topics().setShadowTopics(topic, Lists.newArrayList(shadowTopic)); + tenantManagerAdmin.topics().getShadowTopics(topic); + tenantManagerAdmin.topics().removeShadowTopics(topic); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().setShadowTopics(topic, Lists.newArrayList(shadowTopic))); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getShadowTopics(topic)); + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(topic, subject, Set.of(action)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().setShadowTopics(topic, Lists.newArrayList(shadowTopic))); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getShadowTopics(topic)); + superUserAdmin.topics().revokePermissions(topic, subject); + } + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testList() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationTopicOperationChecker(subject, NamespaceOperation.GET_TOPICS); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getList("public/default")); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationTopicOperationChecker(subject, NamespaceOperation.GET_TOPICS); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getPartitionedTopicList("public/default")); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testPermissionsOnTopic() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // + superUserAdmin.topics().getPermissions(topic); + superUserAdmin.topics().grantPermission(topic, subject, Sets.newHashSet(AuthAction.functions)); + superUserAdmin.topics().revokePermissions(topic, subject); + + // test tenant manager + tenantManagerAdmin.topics().getPermissions(topic); + tenantManagerAdmin.topics().grantPermission(topic, subject, Sets.newHashSet(AuthAction.functions)); + tenantManagerAdmin.topics().revokePermissions(topic, subject); + + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getPermissions(topic)); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().grantPermission(topic, subject, Sets.newHashSet(AuthAction.functions))); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().revokePermissions(topic, subject)); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testOffloadPolicies() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.OFFLOAD, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getOffloadPolicies(topic)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.OFFLOAD, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setOffloadPolicies(topic, OffloadPolicies.builder().build())); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.OFFLOAD, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removeOffloadPolicies(topic)); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testMaxUnackedMessagesOnConsumer() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_UNACKED, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getMaxUnackedMessagesOnConsumer(topic)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_UNACKED, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setMaxUnackedMessagesOnConsumer(topic, 2)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_UNACKED, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removeMaxUnackedMessagesOnConsumer(topic)); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testDeduplicationSnapshotInterval() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.DEDUPLICATION_SNAPSHOT, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getDeduplicationSnapshotInterval(topic)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.DEDUPLICATION_SNAPSHOT, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setDeduplicationSnapshotInterval(topic, 2)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.DEDUPLICATION_SNAPSHOT, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removeDeduplicationSnapshotInterval(topic)); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testInactiveTopicPolicies() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.INACTIVE_TOPIC, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getInactiveTopicPolicies(topic)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.INACTIVE_TOPIC, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setInactiveTopicPolicies(topic, new InactiveTopicPolicies())); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.INACTIVE_TOPIC, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removeInactiveTopicPolicies(topic)); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testMaxUnackedMessagesOnSubscription() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_UNACKED, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getMaxUnackedMessagesOnSubscription(topic)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_UNACKED, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setMaxUnackedMessagesOnSubscription(topic, 2)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_UNACKED, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removeMaxUnackedMessagesOnSubscription(topic)); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testDelayedDeliveryPolicies() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.DELAYED_DELIVERY, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getDelayedDeliveryPolicy(topic)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.DELAYED_DELIVERY, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setDelayedDeliveryPolicy(topic, DelayedDeliveryPolicies.builder().build())); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.DELAYED_DELIVERY, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removeDelayedDeliveryPolicy(topic)); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testAutoSubscriptionCreation() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.AUTO_SUBSCRIPTION_CREATION, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getAutoSubscriptionCreation(topic, false)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.AUTO_SUBSCRIPTION_CREATION, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setAutoSubscriptionCreation(topic, AutoSubscriptionCreationOverride.builder().build())); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.AUTO_SUBSCRIPTION_CREATION, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removeAutoSubscriptionCreation(topic)); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testSubscribeRate() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RATE, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getSubscribeRate(topic)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RATE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setSubscribeRate(topic, new SubscribeRate())); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RATE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removeSubscribeRate(topic)); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testSubscriptionTypesEnabled() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.SUBSCRIPTION_AUTH_MODE, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getSubscriptionTypesEnabled(topic)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.SUBSCRIPTION_AUTH_MODE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setSubscriptionTypesEnabled(topic, new HashSet<>())); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.SUBSCRIPTION_AUTH_MODE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removeSubscriptionTypesEnabled(topic)); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testPublishRate() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RATE, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getPublishRate(topic)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RATE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setPublishRate(topic, new PublishRate())); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RATE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removePublishRate(topic)); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testMaxConsumersPerSubscription() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_CONSUMERS, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getMaxConsumersPerSubscription(topic)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_CONSUMERS, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setMaxConsumersPerSubscription(topic, 2)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_CONSUMERS, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removeMaxConsumersPerSubscription(topic)); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testCompactionThreshold() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.COMPACTION, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getCompactionThreshold(topic)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.COMPACTION, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setCompactionThreshold(topic, 20000)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.COMPACTION, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removeCompactionThreshold(topic)); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testDispatchRate() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RATE, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getDispatchRate(topic)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RATE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setDispatchRate(topic, DispatchRate.builder().build())); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RATE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removeDispatchRate(topic)); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testMaxConsumers() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_CONSUMERS, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getMaxConsumers(topic)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_CONSUMERS, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setMaxConsumers(topic, 2)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_CONSUMERS, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removeMaxConsumers(topic)); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testMaxProducers() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_PRODUCERS, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getMaxProducers(topic)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_PRODUCERS, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setMaxProducers(topic, 2)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.MAX_PRODUCERS, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removeMaxProducers(topic)); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testReplicatorDispatchRate() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.REPLICATION_RATE, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getReplicatorDispatchRate(topic)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.REPLICATION_RATE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setReplicatorDispatchRate(topic, DispatchRate.builder().build())); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.REPLICATION_RATE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removeReplicatorDispatchRate(topic)); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testPersistence() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.PERSISTENCE, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getPersistence(topic)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.PERSISTENCE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setPersistence(topic, new PersistencePolicies())); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.PERSISTENCE, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removePersistence(topic)); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testRetention() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RETENTION, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getRetention(topic, false)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RETENTION, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setRetention(topic, new RetentionPolicies())); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.RETENTION, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removeRetention(topic)); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testDeduplication() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.DEDUPLICATION, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getDeduplicationStatus(topic, false)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.DEDUPLICATION, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setDeduplicationStatus(topic, false)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.DEDUPLICATION, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removeDeduplicationStatus(topic)); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testMessageTTL() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.TTL, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getMessageTTL(topic, false)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.TTL, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setMessageTTL(topic, 2)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.TTL, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removeMessageTTL(topic)); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testBacklogQuota() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.BACKLOG, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().getBacklogQuotaMap(topic, false)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.BACKLOG, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().setBacklogQuota(topic, BacklogQuota.builder().build())); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.BACKLOG, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topicPolicies().removeBacklogQuota(topic)); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } + + @Test + @SneakyThrows + public void testReplicationClusters() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + createTopic(topic, false); + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + AtomicBoolean execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.REPLICATION, PolicyOperation.READ); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().getReplicationClusters(topic, false)); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.REPLICATION, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().setReplicationClusters(topic, new ArrayList<>())); + Assert.assertTrue(execFlag.get()); + + execFlag = setAuthorizationPolicyOperationChecker(subject, PolicyName.REPLICATION, PolicyOperation.WRITE); + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> subAdmin.topics().removeReplicationClusters(topic)); + Assert.assertTrue(execFlag.get()); + + deleteTopic(topic, false); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAutoCreationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAutoCreationTest.java index d447787d1b7cb..c90ad15242c85 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAutoCreationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAutoCreationTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.admin; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; @@ -27,9 +28,11 @@ import java.util.List; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.Consumer; @@ -37,9 +40,11 @@ import org.apache.pulsar.client.api.ProducerConsumerBase; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.impl.LookupService; +import org.apache.pulsar.client.impl.LookupTopicResult; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; +import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; import org.apache.pulsar.common.policies.data.TopicType; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -55,6 +60,7 @@ protected void setup() throws Exception { conf.setAllowAutoTopicCreationType(TopicType.PARTITIONED); conf.setAllowAutoTopicCreation(true); conf.setDefaultNumPartitions(3); + conf.setForceDeleteNamespaceAllowed(true); super.internalSetup(); super.producerBaseSetup(); } @@ -128,13 +134,17 @@ public void testPartitionedTopicAutoCreationForbiddenDuringNamespaceDeletion() // we want to skip the "lookup" phase, because it is blocked by the HTTP API LookupService mockLookup = mock(LookupService.class); ((PulsarClientImpl) pulsarClient).setLookup(mockLookup); - when(mockLookup.getPartitionedTopicMetadata(any())).thenAnswer( + when(mockLookup.getPartitionedTopicMetadata(any(), anyBoolean())).thenAnswer( i -> CompletableFuture.completedFuture(new PartitionedTopicMetadata(0))); - when(mockLookup.getBroker(any())).thenAnswer(i -> { + when(mockLookup.getPartitionedTopicMetadata(any(), anyBoolean(), anyBoolean())).thenAnswer( + i -> CompletableFuture.completedFuture(new PartitionedTopicMetadata(0))); + when(mockLookup.getBroker(any())).thenAnswer(ignored -> { InetSocketAddress brokerAddress = new InetSocketAddress(pulsar.getAdvertisedAddress(), pulsar.getBrokerListenPort().get()); - return CompletableFuture.completedFuture(Pair.of(brokerAddress, brokerAddress)); + return CompletableFuture.completedFuture(new LookupTopicResult(brokerAddress, brokerAddress, false)); }); + final String topicPoliciesServiceInitException + = "Topic creation encountered an exception by initialize topic policies service"; // Creating a producer and creating a Consumer may trigger automatic topic // creation, let's try to create a Producer and a Consumer @@ -142,20 +152,24 @@ public void testPartitionedTopicAutoCreationForbiddenDuringNamespaceDeletion() .sendTimeout(1, TimeUnit.SECONDS) .topic(topic) .create()) { - } catch (PulsarClientException.LookupException expected) { - String msg = "Namespace bundle for topic (%s) not served by this instance"; + } catch (PulsarClientException.TopicDoesNotExistException expected) { + // Since the "policies.deleted" is "true", the value of "isAllowAutoTopicCreationAsync" will be false, + // so the "TopicDoesNotExistException" is expected. log.info("Expected error", expected); - assertTrue(expected.getMessage().contains(String.format(msg, topic))); + assertTrue(expected.getMessage().contains(topic) + || expected.getMessage().contains(topicPoliciesServiceInitException)); } try (Consumer ignored = pulsarClient.newConsumer() .topic(topic) .subscriptionName("test") .subscribe()) { - } catch (PulsarClientException.LookupException expected) { - String msg = "Namespace bundle for topic (%s) not served by this instance"; + } catch (PulsarClientException.TopicDoesNotExistException expected) { + // Since the "policies.deleted" is "true", the value of "isAllowAutoTopicCreationAsync" will be false, + // so the "TopicDoesNotExistException" is expected. log.info("Expected error", expected); - assertTrue(expected.getMessage().contains(String.format(msg, topic))); + assertTrue(expected.getMessage().contains(topic) + || expected.getMessage().contains(topicPoliciesServiceInitException)); } @@ -182,4 +196,56 @@ public void testPartitionedTopicAutoCreationForbiddenDuringNamespaceDeletion() } } + + @Test + public void testClientWithAutoCreationGotNotFoundException() throws PulsarAdminException, PulsarClientException { + final String namespace = "public/test_1"; + final String topicName = "persistent://public/test_1/test_auto_creation_got_not_found" + + System.currentTimeMillis(); + final int retryTimes = 30; + admin.namespaces().createNamespace(namespace); + admin.namespaces().setAutoTopicCreation(namespace, AutoTopicCreationOverride.builder() + .allowAutoTopicCreation(true) + .topicType("non-partitioned") + .build()); + + @Cleanup("shutdown") + final ExecutorService executor1 = Executors.newSingleThreadExecutor(); + + @Cleanup("shutdown") + final ExecutorService executor2 = Executors.newSingleThreadExecutor(); + + for (int i = 0; i < retryTimes; i++) { + final CompletableFuture adminListSub = CompletableFuture.runAsync(() -> { + try { + admin.topics().getSubscriptions(topicName); + } catch (PulsarAdminException e) { + throw new RuntimeException(e); + } + }, executor1); + + final CompletableFuture> consumerSub = CompletableFuture.supplyAsync(() -> { + try { + return pulsarClient.newConsumer() + .topic(topicName) + .subscriptionName("sub-1") + .subscribe(); + } catch (PulsarClientException e) { + throw new RuntimeException(e); + } + }, executor2); + + try { + adminListSub.join(); + } catch (Throwable ex) { + // we don't care the exception. + } + + consumerSub.join().close(); + admin.topics().delete(topicName, true); + } + + admin.namespaces().deleteNamespace(namespace, true); + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesAuthZTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesAuthZTest.java new file mode 100644 index 0000000000000..002ba2cbfcf9d --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesAuthZTest.java @@ -0,0 +1,491 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin; + +import static org.awaitility.Awaitility.await; +import io.jsonwebtoken.Jwts; +import java.util.Set; +import java.util.UUID; +import lombok.Cleanup; +import lombok.SneakyThrows; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; +import org.apache.pulsar.common.policies.data.AuthAction; +import org.apache.pulsar.common.policies.data.OffloadPolicies; +import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; +import org.apache.pulsar.common.policies.data.RetentionPolicies; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.security.MockedPulsarStandalone; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + + +public final class TopicPoliciesAuthZTest extends MockedPulsarStandalone { + + private PulsarAdmin superUserAdmin; + + private PulsarAdmin tenantManagerAdmin; + + private static final String TENANT_ADMIN_SUBJECT = UUID.randomUUID().toString(); + private static final String TENANT_ADMIN_TOKEN = Jwts.builder() + .claim("sub", TENANT_ADMIN_SUBJECT).signWith(SECRET_KEY).compact(); + + @SneakyThrows + @BeforeClass + public void before() { + configureTokenAuthentication(); + configureDefaultAuthorization(); + start(); + this.superUserAdmin =PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(SUPER_USER_TOKEN)) + .build(); + final TenantInfo tenantInfo = superUserAdmin.tenants().getTenantInfo("public"); + tenantInfo.getAdminRoles().add(TENANT_ADMIN_SUBJECT); + superUserAdmin.tenants().updateTenant("public", tenantInfo); + this.tenantManagerAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(TENANT_ADMIN_TOKEN)) + .build(); + } + + + @SneakyThrows + @AfterClass + public void after() { + if (superUserAdmin != null) { + superUserAdmin.close(); + superUserAdmin = null; + } + if (tenantManagerAdmin != null) { + tenantManagerAdmin.close(); + tenantManagerAdmin = null; + } + close(); + } + + + @SneakyThrows + @Test + public void testRetention() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createNonPartitionedTopic(topic); + + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + final RetentionPolicies definedRetentionPolicy = new RetentionPolicies(1, 1); + // test superuser + superUserAdmin.topicPolicies().setRetention(topic, definedRetentionPolicy); + + // because the topic policies is eventual consistency, we should wait here + await().untilAsserted(() -> { + final RetentionPolicies receivedRetentionPolicy = superUserAdmin.topicPolicies().getRetention(topic); + Assert.assertEquals(receivedRetentionPolicy, definedRetentionPolicy); + }); + superUserAdmin.topicPolicies().removeRetention(topic); + + await().untilAsserted(() -> { + final RetentionPolicies retention = superUserAdmin.topicPolicies().getRetention(topic); + Assert.assertNull(retention); + }); + + // test tenant manager + + tenantManagerAdmin.topicPolicies().setRetention(topic, definedRetentionPolicy); + await().untilAsserted(() -> { + final RetentionPolicies receivedRetentionPolicy = tenantManagerAdmin.topicPolicies().getRetention(topic); + Assert.assertEquals(receivedRetentionPolicy, definedRetentionPolicy); + }); + tenantManagerAdmin.topicPolicies().removeRetention(topic); + await().untilAsserted(() -> { + final RetentionPolicies retention = tenantManagerAdmin.topicPolicies().getRetention(topic); + Assert.assertNull(retention); + }); + + // test nobody + + try { + subAdmin.topicPolicies().getRetention(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + + subAdmin.topicPolicies().setRetention(topic, definedRetentionPolicy); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + subAdmin.topicPolicies().removeRetention(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + // test sub user with permissions + for (AuthAction action : AuthAction.values()) { + superUserAdmin.namespaces().grantPermissionOnNamespace("public/default", + subject, Set.of(action)); + try { + subAdmin.topicPolicies().getRetention(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + + subAdmin.topicPolicies().setRetention(topic, definedRetentionPolicy); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + subAdmin.topicPolicies().removeRetention(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + superUserAdmin.namespaces().revokePermissionsOnNamespace("public/default", subject); + } + } + + @SneakyThrows + @Test + public void testOffloadPolicy() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createNonPartitionedTopic(topic); + + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + + // mocked data + final OffloadPoliciesImpl definedOffloadPolicies = new OffloadPoliciesImpl(); + definedOffloadPolicies.setManagedLedgerOffloadThresholdInBytes(100L); + definedOffloadPolicies.setManagedLedgerOffloadThresholdInSeconds(100L); + definedOffloadPolicies.setManagedLedgerOffloadDeletionLagInMillis(200L); + definedOffloadPolicies.setManagedLedgerOffloadDriver("s3"); + definedOffloadPolicies.setManagedLedgerOffloadBucket("buck"); + + // test superuser + superUserAdmin.topicPolicies().setOffloadPolicies(topic, definedOffloadPolicies); + + // because the topic policies is eventual consistency, we should wait here + await().untilAsserted(() -> { + final OffloadPolicies offloadPolicy = superUserAdmin.topicPolicies().getOffloadPolicies(topic); + Assert.assertEquals(offloadPolicy, definedOffloadPolicies); + }); + superUserAdmin.topicPolicies().removeOffloadPolicies(topic); + + await().untilAsserted(() -> { + final OffloadPolicies offloadPolicy = superUserAdmin.topicPolicies().getOffloadPolicies(topic); + Assert.assertNull(offloadPolicy); + }); + + // test tenant manager + + tenantManagerAdmin.topicPolicies().setOffloadPolicies(topic, definedOffloadPolicies); + await().untilAsserted(() -> { + final OffloadPolicies offloadPolicy = tenantManagerAdmin.topicPolicies().getOffloadPolicies(topic); + Assert.assertEquals(offloadPolicy, definedOffloadPolicies); + }); + tenantManagerAdmin.topicPolicies().removeOffloadPolicies(topic); + await().untilAsserted(() -> { + final OffloadPolicies offloadPolicy = tenantManagerAdmin.topicPolicies().getOffloadPolicies(topic); + Assert.assertNull(offloadPolicy); + }); + + // test nobody + + try { + subAdmin.topicPolicies().getOffloadPolicies(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + + subAdmin.topicPolicies().setOffloadPolicies(topic, definedOffloadPolicies); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + subAdmin.topicPolicies().removeOffloadPolicies(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + // test sub user with permissions + for (AuthAction action : AuthAction.values()) { + superUserAdmin.namespaces().grantPermissionOnNamespace("public/default", + subject, Set.of(action)); + try { + subAdmin.topicPolicies().getOffloadPolicies(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + + subAdmin.topicPolicies().setOffloadPolicies(topic, definedOffloadPolicies); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + subAdmin.topicPolicies().removeOffloadPolicies(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + superUserAdmin.namespaces().revokePermissionsOnNamespace("public/default", subject); + } + } + + @SneakyThrows + @Test + public void testMaxUnackedMessagesOnConsumer() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createNonPartitionedTopic(topic); + + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + + // mocked data + int definedUnackedMessagesOnConsumer = 100; + + // test superuser + superUserAdmin.topicPolicies().setMaxUnackedMessagesOnConsumer(topic, definedUnackedMessagesOnConsumer); + + // because the topic policies is eventual consistency, we should wait here + await().untilAsserted(() -> { + final int unackedMessagesOnConsumer = superUserAdmin.topicPolicies() + .getMaxUnackedMessagesOnConsumer(topic); + Assert.assertEquals(unackedMessagesOnConsumer, definedUnackedMessagesOnConsumer); + }); + superUserAdmin.topicPolicies().removeMaxUnackedMessagesOnConsumer(topic); + + await().untilAsserted(() -> { + final Integer unackedMessagesOnConsumer = superUserAdmin.topicPolicies().getMaxUnackedMessagesOnConsumer(topic); + Assert.assertNull(unackedMessagesOnConsumer); + }); + + // test tenant manager + + tenantManagerAdmin.topicPolicies().setMaxUnackedMessagesOnConsumer(topic, definedUnackedMessagesOnConsumer); + await().untilAsserted(() -> { + final int unackedMessagesOnConsumer = tenantManagerAdmin.topicPolicies().getMaxUnackedMessagesOnConsumer(topic); + Assert.assertEquals(unackedMessagesOnConsumer, definedUnackedMessagesOnConsumer); + }); + tenantManagerAdmin.topicPolicies().removeMaxUnackedMessagesOnConsumer(topic); + await().untilAsserted(() -> { + final Integer unackedMessagesOnConsumer = tenantManagerAdmin.topicPolicies().getMaxUnackedMessagesOnConsumer(topic); + Assert.assertNull(unackedMessagesOnConsumer); + }); + + // test nobody + + try { + subAdmin.topicPolicies().getMaxUnackedMessagesOnConsumer(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + + subAdmin.topicPolicies().setMaxUnackedMessagesOnConsumer(topic, definedUnackedMessagesOnConsumer); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + subAdmin.topicPolicies().removeMaxUnackedMessagesOnConsumer(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + // test sub user with permissions + for (AuthAction action : AuthAction.values()) { + superUserAdmin.namespaces().grantPermissionOnNamespace("public/default", + subject, Set.of(action)); + try { + subAdmin.topicPolicies().getMaxUnackedMessagesOnConsumer(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + + subAdmin.topicPolicies().setMaxUnackedMessagesOnConsumer(topic, definedUnackedMessagesOnConsumer); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + subAdmin.topicPolicies().removeMaxUnackedMessagesOnConsumer(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + superUserAdmin.namespaces().revokePermissionsOnNamespace("public/default", subject); + } + } + + @SneakyThrows + @Test + public void testMaxUnackedMessagesOnSubscription() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createNonPartitionedTopic(topic); + + @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + + // mocked data + int definedUnackedMessagesOnConsumer = 100; + + // test superuser + superUserAdmin.topicPolicies().setMaxUnackedMessagesOnSubscription(topic, definedUnackedMessagesOnConsumer); + + // because the topic policies is eventual consistency, we should wait here + await().untilAsserted(() -> { + final int unackedMessagesOnConsumer = superUserAdmin.topicPolicies() + .getMaxUnackedMessagesOnSubscription(topic); + Assert.assertEquals(unackedMessagesOnConsumer, definedUnackedMessagesOnConsumer); + }); + superUserAdmin.topicPolicies().removeMaxUnackedMessagesOnSubscription(topic); + + await().untilAsserted(() -> { + final Integer unackedMessagesOnConsumer = superUserAdmin.topicPolicies() + .getMaxUnackedMessagesOnSubscription(topic); + Assert.assertNull(unackedMessagesOnConsumer); + }); + + // test tenant manager + + tenantManagerAdmin.topicPolicies().setMaxUnackedMessagesOnSubscription(topic, definedUnackedMessagesOnConsumer); + await().untilAsserted(() -> { + final int unackedMessagesOnConsumer = tenantManagerAdmin.topicPolicies().getMaxUnackedMessagesOnSubscription(topic); + Assert.assertEquals(unackedMessagesOnConsumer, definedUnackedMessagesOnConsumer); + }); + tenantManagerAdmin.topicPolicies().removeMaxUnackedMessagesOnSubscription(topic); + await().untilAsserted(() -> { + final Integer unackedMessagesOnConsumer = tenantManagerAdmin.topicPolicies() + .getMaxUnackedMessagesOnSubscription(topic); + Assert.assertNull(unackedMessagesOnConsumer); + }); + + // test nobody + + try { + subAdmin.topicPolicies().getMaxUnackedMessagesOnSubscription(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + + subAdmin.topicPolicies().setMaxUnackedMessagesOnSubscription(topic, definedUnackedMessagesOnConsumer); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + subAdmin.topicPolicies().removeMaxUnackedMessagesOnSubscription(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + // test sub user with permissions + for (AuthAction action : AuthAction.values()) { + superUserAdmin.namespaces().grantPermissionOnNamespace("public/default", + subject, Set.of(action)); + try { + subAdmin.topicPolicies().getMaxUnackedMessagesOnSubscription(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + + subAdmin.topicPolicies().setMaxUnackedMessagesOnSubscription(topic, definedUnackedMessagesOnConsumer); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + subAdmin.topicPolicies().removeMaxUnackedMessagesOnSubscription(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + superUserAdmin.namespaces().revokePermissionsOnNamespace("public/default", subject); + } + + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java index 4ff2917105246..dc9a7ec4429fc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java @@ -23,6 +23,7 @@ import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import java.lang.reflect.Field; @@ -41,14 +42,18 @@ import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.ConfigHelper; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.AbstractTopic; import org.apache.pulsar.broker.service.PublishRateLimiterImpl; import org.apache.pulsar.broker.service.SystemTopicBasedTopicPoliciesService; +import org.apache.pulsar.broker.service.TopicPolicyTestUtils; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.service.persistent.SubscribeRateLimiter; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -70,6 +75,7 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies; import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.HierarchyTopicPolicies; import org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode; @@ -84,6 +90,7 @@ import org.apache.pulsar.common.policies.data.TopicStats; import org.assertj.core.api.Assertions; import org.awaitility.Awaitility; +import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -176,11 +183,11 @@ public void testTopicPolicyInitialValueWithNamespaceAlreadyLoaded() throws Excep //load the nameserver, but topic is not init. log.info("lookup:{}",admin.lookups().lookupTopic(topic)); - assertTrue(pulsar.getBrokerService().isTopicNsOwnedByBroker(topicName)); + assertTrue(pulsar.getBrokerService().isTopicNsOwnedByBrokerAsync(topicName).join()); assertFalse(pulsar.getBrokerService().getTopics().containsKey(topic)); //make sure namespace policy reader is fully started. Awaitility.await().untilAsserted(()-> { - assertTrue(policyService.getPoliciesCacheInit(topicName.getNamespaceObject())); + assertTrue(policyService.getPoliciesCacheInit(topicName.getNamespaceObject()).isDone()); }); //load the topic. @@ -2022,16 +2029,12 @@ public void testPublishRateInDifferentLevelPolicy() throws Exception { final String topicName = "persistent://" + myNamespace + "/test-" + UUID.randomUUID(); pulsarClient.newProducer().topic(topicName).create().close(); - Field publishMaxMessageRate = PublishRateLimiterImpl.class.getDeclaredField("publishMaxMessageRate"); - publishMaxMessageRate.setAccessible(true); - Field publishMaxByteRate = PublishRateLimiterImpl.class.getDeclaredField("publishMaxByteRate"); - publishMaxByteRate.setAccessible(true); //1 use broker-level policy by default PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicIfExists(topicName).get().get(); PublishRateLimiterImpl publishRateLimiter = (PublishRateLimiterImpl) topic.getTopicPublishRateLimiter(); - Assert.assertEquals(publishMaxMessageRate.get(publishRateLimiter), 5); - Assert.assertEquals(publishMaxByteRate.get(publishRateLimiter), 50L); + Assert.assertEquals(publishRateLimiter.getTokenBucketOnMessage().getRate(), 5); + Assert.assertEquals(publishRateLimiter.getTokenBucketOnByte().getRate(), 50L); //2 set namespace-level policy PublishRate publishMsgRate = new PublishRate(10, 100L); @@ -2040,12 +2043,12 @@ public void testPublishRateInDifferentLevelPolicy() throws Exception { Awaitility.await() .until(() -> { PublishRateLimiterImpl limiter = (PublishRateLimiterImpl) topic.getTopicPublishRateLimiter(); - return (int)publishMaxMessageRate.get(limiter) == 10; + return (int)limiter.getTokenBucketOnMessage().getRate() == 10; }); publishRateLimiter = (PublishRateLimiterImpl) topic.getTopicPublishRateLimiter(); - Assert.assertEquals(publishMaxMessageRate.get(publishRateLimiter), 10); - Assert.assertEquals(publishMaxByteRate.get(publishRateLimiter), 100L); + Assert.assertEquals(publishRateLimiter.getTokenBucketOnMessage().getRate(), 10); + Assert.assertEquals(publishRateLimiter.getTokenBucketOnByte().getRate(), 100L); //3 set topic-level policy, namespace-level policy should be overwritten PublishRate publishMsgRate2 = new PublishRate(11, 101L); @@ -2055,8 +2058,8 @@ public void testPublishRateInDifferentLevelPolicy() throws Exception { .until(() -> admin.topicPolicies().getPublishRate(topicName) != null); publishRateLimiter = (PublishRateLimiterImpl) topic.getTopicPublishRateLimiter(); - Assert.assertEquals(publishMaxMessageRate.get(publishRateLimiter), 11); - Assert.assertEquals(publishMaxByteRate.get(publishRateLimiter), 101L); + Assert.assertEquals(publishRateLimiter.getTokenBucketOnMessage().getRate(), 11); + Assert.assertEquals(publishRateLimiter.getTokenBucketOnByte().getRate(), 101L); //4 remove topic-level policy, namespace-level policy will take effect admin.topicPolicies().removePublishRate(topicName); @@ -2065,8 +2068,8 @@ public void testPublishRateInDifferentLevelPolicy() throws Exception { .until(() -> admin.topicPolicies().getPublishRate(topicName) == null); publishRateLimiter = (PublishRateLimiterImpl) topic.getTopicPublishRateLimiter(); - Assert.assertEquals(publishMaxMessageRate.get(publishRateLimiter), 10); - Assert.assertEquals(publishMaxByteRate.get(publishRateLimiter), 100L); + Assert.assertEquals(publishRateLimiter.getTokenBucketOnMessage().getRate(), 10); + Assert.assertEquals(publishRateLimiter.getTokenBucketOnByte().getRate(), 100L); //5 remove namespace-level policy, broker-level policy will take effect admin.namespaces().removePublishRate(myNamespace); @@ -2074,12 +2077,12 @@ public void testPublishRateInDifferentLevelPolicy() throws Exception { Awaitility.await() .until(() -> { PublishRateLimiterImpl limiter = (PublishRateLimiterImpl) topic.getTopicPublishRateLimiter(); - return (int)publishMaxMessageRate.get(limiter) == 5; + return (int)limiter.getTokenBucketOnMessage().getRate() == 5; }); publishRateLimiter = (PublishRateLimiterImpl) topic.getTopicPublishRateLimiter(); - Assert.assertEquals(publishMaxMessageRate.get(publishRateLimiter), 5); - Assert.assertEquals(publishMaxByteRate.get(publishRateLimiter), 50L); + Assert.assertEquals(publishRateLimiter.getTokenBucketOnMessage().getRate(), 5); + Assert.assertEquals(publishRateLimiter.getTokenBucketOnByte().getRate(), 50L); } @Test(timeOut = 20000) @@ -2088,7 +2091,7 @@ public void testTopicMaxMessageSizeApi() throws Exception{ assertNull(admin.topicPolicies().getMaxMessageSize(persistenceTopic)); admin.topicPolicies().setMaxMessageSize(persistenceTopic,10); Awaitility.await().until(() - -> pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(persistenceTopic)) != null); + -> TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), TopicName.get(persistenceTopic)) != null); assertEquals(admin.topicPolicies().getMaxMessageSize(persistenceTopic).intValue(),10); admin.topicPolicies().removeMaxMessageSize(persistenceTopic); @@ -2134,7 +2137,7 @@ public void testTopicMaxMessageSize(TopicDomain topicDomain, boolean isPartition assertNull(admin.topicPolicies().getMaxMessageSize(topic)); // set msg size admin.topicPolicies().setMaxMessageSize(topic, 10); - Awaitility.await().until(() -> pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic)) != null); + Awaitility.await().until(() -> TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), TopicName.get(topic)) != null); if (isPartitioned) { for (int i = 0; i <3; i++) { String partitionName = TopicName.get(topic).getPartition(i).toString(); @@ -2251,7 +2254,7 @@ public void testMaxSubscriptionsPerTopicApi() throws Exception { // set max subscriptions admin.topicPolicies().setMaxSubscriptionsPerTopic(topic, 10); Awaitility.await().until(() - -> pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic)) != null); + -> TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), TopicName.get(topic)) != null); assertEquals(admin.topicPolicies().getMaxSubscriptionsPerTopic(topic).intValue(), 10); // remove max subscriptions admin.topicPolicies().removeMaxSubscriptionsPerTopic(topic); @@ -2274,7 +2277,7 @@ public void testMaxSubscriptionsPerTopicWithExistingSubs() throws Exception { final int topicLevelMaxSubNum = 2; admin.topicPolicies().setMaxSubscriptionsPerTopic(topic, topicLevelMaxSubNum); Awaitility.await().until(() - -> pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic)) != null); + -> TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), TopicName.get(topic)) != null); List> consumerList = new ArrayList<>(); String subName = "my-sub-"; for (int i = 0; i < topicLevelMaxSubNum; i++) { @@ -2406,7 +2409,7 @@ public void testMaxSubscriptionsPerTopic() throws Exception { final int topicLevelMaxSubNum = 2; admin.topicPolicies().setMaxSubscriptionsPerTopic(topic, topicLevelMaxSubNum); Awaitility.await().until(() - -> pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic)) != null); + -> TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), TopicName.get(topic)) != null); List> consumerList = new ArrayList<>(); for (int i = 0; i < topicLevelMaxSubNum; i++) { @@ -2609,7 +2612,7 @@ public void testSubscriptionTypesEnabled() throws Exception { admin.topicPolicies().setSubscriptionTypesEnabled(topic, subscriptionTypeSet); Awaitility.await().until(() - -> pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic)) != null); + -> TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), TopicName.get(topic)) != null); waitTopicPoliciesApplied(topic, 0, hierarchyTopicPolicies -> { assertTrue(hierarchyTopicPolicies.getSubscriptionTypesEnabled().get() .contains(CommandSubscribe.SubType.Failover)); @@ -2832,7 +2835,7 @@ public void testPolicyIsDeleteTogetherManually() throws Exception { pulsarClient.newProducer().topic(topic).create().close(); Awaitility.await().untilAsserted(() -> - Assertions.assertThat(pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic))) + Assertions.assertThat(TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), TopicName.get(topic))) .isNull()); int maxConsumersPerSubscription = 10; @@ -2841,7 +2844,7 @@ public void testPolicyIsDeleteTogetherManually() throws Exception { Awaitility.await().untilAsserted(() -> Assertions.assertThat(pulsar.getBrokerService().getTopic(topic, false).get().isPresent()).isTrue()); Awaitility.await().untilAsserted(() -> - Assertions.assertThat(pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic))) + Assertions.assertThat(TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), TopicName.get(topic))) .isNotNull()); admin.topics().delete(topic); @@ -2849,7 +2852,7 @@ public void testPolicyIsDeleteTogetherManually() throws Exception { Awaitility.await().untilAsserted(() -> Assertions.assertThat(pulsar.getBrokerService().getTopic(topic, false).get().isPresent()).isFalse()); Awaitility.await().untilAsserted(() -> - Assertions.assertThat(pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic))) + Assertions.assertThat(TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), TopicName.get(topic))) .isNull()); } @@ -2861,8 +2864,8 @@ public void testPoliciesCanBeDeletedWithTopic() throws Exception { pulsarClient.newProducer().topic(topic2).create().close(); Awaitility.await().untilAsserted(() -> { - Assertions.assertThat(pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic))).isNull(); - Assertions.assertThat(pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic2))).isNull(); + Assertions.assertThat(TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), TopicName.get(topic))).isNull(); + Assertions.assertThat(TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), TopicName.get(topic2))).isNull(); }); // Init Topic Policies. Send 4 messages in a row, there should be only 2 messages left after compression admin.topicPolicies().setMaxConsumersPerSubscription(topic, 1); @@ -2870,8 +2873,8 @@ public void testPoliciesCanBeDeletedWithTopic() throws Exception { admin.topicPolicies().setMaxConsumersPerSubscription(topic, 3); admin.topicPolicies().setMaxConsumersPerSubscription(topic2, 4); Awaitility.await().untilAsserted(() -> { - Assertions.assertThat(pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic))).isNotNull(); - Assertions.assertThat(pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic2))).isNotNull(); + Assertions.assertThat(TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), TopicName.get(topic))).isNotNull(); + Assertions.assertThat(TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), TopicName.get(topic2))).isNotNull(); }); String topicPoliciesTopic = "persistent://" + myNamespace + "/" + SystemTopicNames.NAMESPACE_EVENTS_LOCAL_NAME; PersistentTopic persistentTopic = @@ -2904,7 +2907,7 @@ public void testPoliciesCanBeDeletedWithTopic() throws Exception { admin.topics().delete(topic, true); Awaitility.await().untilAsserted(() -> - assertNull(pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic)))); + assertNull(TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), TopicName.get(topic)))); persistentTopic.triggerCompaction(); field = PersistentTopic.class.getDeclaredField("currentCompaction"); field.setAccessible(true); @@ -2936,7 +2939,7 @@ public void testPolicyIsDeleteTogetherAutomatically() throws Exception { pulsarClient.newProducer().topic(topic).create().close(); Awaitility.await().untilAsserted(() -> - Assertions.assertThat(pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic))) + Assertions.assertThat(TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), TopicName.get(topic))) .isNull()); int maxConsumersPerSubscription = 10; @@ -2945,7 +2948,7 @@ public void testPolicyIsDeleteTogetherAutomatically() throws Exception { Awaitility.await().untilAsserted(() -> Assertions.assertThat(pulsar.getBrokerService().getTopic(topic, false).get().isPresent()).isTrue()); Awaitility.await().untilAsserted(() -> - Assertions.assertThat(pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic))) + Assertions.assertThat(TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), TopicName.get(topic))) .isNotNull()); InactiveTopicPolicies inactiveTopicPolicies = @@ -2959,7 +2962,7 @@ public void testPolicyIsDeleteTogetherAutomatically() throws Exception { Awaitility.await().untilAsserted(() -> Assertions.assertThat(pulsar.getBrokerService().getTopic(topic, false).get().isPresent()).isFalse()); Awaitility.await().untilAsserted(() -> - Assertions.assertThat(pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic))) + Assertions.assertThat(TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), TopicName.get(topic))) .isNull()); } @@ -2990,6 +2993,10 @@ public void testReplicatorClusterApi() throws Exception { admin.topics().removeReplicationClusters(topic); Awaitility.await().untilAsserted(() -> assertNull(admin.topics().getReplicationClusters(topic, false))); + + assertThrows(PulsarAdminException.PreconditionFailedException.class, () -> admin.topics().setReplicationClusters(topic, List.of())); + assertThrows(PulsarAdminException.PreconditionFailedException.class, () -> admin.topics().setReplicationClusters(topic, null)); + } @Test @@ -3001,62 +3008,64 @@ public void testLoopCreateAndDeleteTopicPolicies() throws Exception { n++; pulsarClient.newProducer().topic(topic).create().close(); Awaitility.await().untilAsserted(() -> { - Assertions.assertThat(pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic))).isNull(); + Assertions.assertThat(TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), TopicName.get(topic))).isNull(); }); admin.topicPolicies().setMaxConsumersPerSubscription(topic, 1); Awaitility.await().untilAsserted(() -> { - Assertions.assertThat(pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic))).isNotNull(); + Assertions.assertThat(TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), TopicName.get(topic))).isNotNull(); }); admin.topics().delete(topic); Awaitility.await().untilAsserted(() -> { - Assertions.assertThat(pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic))).isNull(); + Assertions.assertThat(TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), TopicName.get(topic))).isNull(); }); } } + @Test public void testGlobalTopicPolicies() throws Exception { final String topic = testTopic + UUID.randomUUID(); pulsarClient.newProducer().topic(topic).create().close(); Awaitility.await().untilAsserted(() -> - Assertions.assertThat(pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic))) + Assertions.assertThat(TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), TopicName.get(topic))) .isNull()); admin.topicPolicies(true).setRetention(topic, new RetentionPolicies(1, 2)); SystemTopicBasedTopicPoliciesService topicPoliciesService = (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); // check global topic policies can be added correctly. - Awaitility.await().untilAsserted(() -> assertNotNull(topicPoliciesService.getTopicPolicies(TopicName.get(topic), true))); - TopicPolicies topicPolicies = topicPoliciesService.getTopicPolicies(TopicName.get(topic), true); - assertNull(topicPoliciesService.getTopicPolicies(TopicName.get(topic))); + Awaitility.await().untilAsserted(() -> assertNotNull( + TopicPolicyTestUtils.getGlobalTopicPolicies(topicPoliciesService, TopicName.get(topic)))); + TopicPolicies topicPolicies = TopicPolicyTestUtils.getGlobalTopicPolicies(topicPoliciesService, TopicName.get(topic)); + assertNull(TopicPolicyTestUtils.getLocalTopicPolicies(topicPoliciesService, TopicName.get(topic))); assertEquals(topicPolicies.getRetentionPolicies().getRetentionTimeInMinutes(), 1); assertEquals(topicPolicies.getRetentionPolicies().getRetentionSizeInMB(), 2); // check global topic policies can be updated correctly. admin.topicPolicies(true).setRetention(topic, new RetentionPolicies(3, 4)); Awaitility.await().untilAsserted(() -> { - TopicPolicies tempPolicies = topicPoliciesService.getTopicPolicies(TopicName.get(topic), true); - assertNull(topicPoliciesService.getTopicPolicies(TopicName.get(topic))); + TopicPolicies tempPolicies = TopicPolicyTestUtils.getGlobalTopicPolicies(topicPoliciesService, TopicName.get(topic)); + assertNull(TopicPolicyTestUtils.getLocalTopicPolicies(topicPoliciesService, (TopicName.get(topic)))); assertEquals(tempPolicies.getRetentionPolicies().getRetentionTimeInMinutes(), 3); assertEquals(tempPolicies.getRetentionPolicies().getRetentionSizeInMB(), 4); }); //Local topic policies and global topic policies can exist together. admin.topicPolicies().setRetention(topic, new RetentionPolicies(10, 20)); - Awaitility.await().untilAsserted(() -> assertNotNull(topicPoliciesService.getTopicPolicies(TopicName.get(topic)))); - TopicPolicies tempPolicies = topicPoliciesService.getTopicPolicies(TopicName.get(topic), true); + Awaitility.await().untilAsserted(() -> assertNotNull(TopicPolicyTestUtils.getTopicPolicies(topicPoliciesService, (TopicName.get(topic))))); + TopicPolicies tempPolicies = TopicPolicyTestUtils.getGlobalTopicPolicies(topicPoliciesService, TopicName.get(topic)); assertEquals(tempPolicies.getRetentionPolicies().getRetentionTimeInMinutes(), 3); assertEquals(tempPolicies.getRetentionPolicies().getRetentionSizeInMB(), 4); - tempPolicies = topicPoliciesService.getTopicPolicies(TopicName.get(topic)); + tempPolicies = TopicPolicyTestUtils.getTopicPolicies(topicPoliciesService, (TopicName.get(topic))); assertEquals(tempPolicies.getRetentionPolicies().getRetentionTimeInMinutes(), 10); assertEquals(tempPolicies.getRetentionPolicies().getRetentionSizeInMB(), 20); // check remove global topic policies can be removed correctly. admin.topicPolicies(true).removeRetention(topic); - Awaitility.await().untilAsserted(() -> - assertNull(topicPoliciesService.getTopicPolicies(TopicName.get(topic), true).getRetentionPolicies())); + Awaitility.await().untilAsserted(() -> assertNull(TopicPolicyTestUtils.getGlobalTopicPolicies(topicPoliciesService, + TopicName.get(topic)).getRetentionPolicies())); } @@ -3100,7 +3109,7 @@ public void testShadowTopics() throws Exception { pulsarClient.newProducer().topic(sourceTopic).create().close(); Awaitility.await().untilAsserted(() -> - Assert.assertNull(pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(sourceTopic)))); + Assert.assertNull(TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), TopicName.get(sourceTopic)))); //shadow topic must exist Assert.expectThrows(PulsarAdminException.PreconditionFailedException.class, ()-> @@ -3130,16 +3139,13 @@ public void testGetTopicPoliciesWhenDeleteTopicPolicy() throws Exception { admin.topics().createNonPartitionedTopic(persistenceTopic); admin.topicPolicies().setMaxConsumers(persistenceTopic, 5); - Integer maxConsumerPerTopic = pulsar - .getTopicPoliciesService() - .getTopicPoliciesBypassCacheAsync(TopicName.get(persistenceTopic)).get() - .getMaxConsumerPerTopic(); + Integer maxConsumerPerTopic = TopicPolicyTestUtils.getTopicPoliciesBypassCache(pulsar.getTopicPoliciesService(), + TopicName.get(persistenceTopic)).orElseThrow().getMaxConsumerPerTopic(); assertEquals(maxConsumerPerTopic, 5); admin.topics().delete(persistenceTopic, true); - TopicPolicies topicPolicies =pulsar.getTopicPoliciesService() - .getTopicPoliciesBypassCacheAsync(TopicName.get(persistenceTopic)).get(5, TimeUnit.SECONDS); - assertNull(topicPolicies); + assertTrue(TopicPolicyTestUtils.getTopicPoliciesBypassCache(pulsar.getTopicPoliciesService(), + TopicName.get(persistenceTopic)).isEmpty()); } @Test @@ -3155,4 +3161,72 @@ public void testProduceChangesWithEncryptionRequired() throws Exception { }); } + @Test + public void testDelayedDeliveryPolicy() throws Exception { + final String topic = testTopic + UUID.randomUUID(); + admin.topics().createNonPartitionedTopic(topic); + + boolean isActive = true; + long tickTime = 1000; + long maxDeliveryDelayInMillis = 5000; + DelayedDeliveryPolicies policy = DelayedDeliveryPolicies.builder() + .active(isActive) + .tickTime(tickTime) + .maxDeliveryDelayInMillis(maxDeliveryDelayInMillis) + .build(); + + admin.topicPolicies().setDelayedDeliveryPolicy(topic, policy); + Awaitility.await() + .untilAsserted(() -> Assert.assertEquals(admin.topicPolicies().getDelayedDeliveryPolicy(topic), policy)); + + admin.topicPolicies().removeDelayedDeliveryPolicy(topic); + Awaitility.await() + .untilAsserted(() -> Assert.assertNull(admin.topicPolicies().getDelayedDeliveryPolicy(topic))); + + admin.topics().delete(topic, true); + } + + @Test + public void testUpdateRetentionWithPartialFailure() throws Exception { + String tpName = BrokerTestUtil.newUniqueName("persistent://" + myNamespace + "/tp"); + admin.topics().createNonPartitionedTopic(tpName); + + // Load topic up. + admin.topics().getInternalStats(tpName); + + // Inject an error that makes dispatch rate update fail. + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(tpName, false).join().get(); + final var subscriptions = persistentTopic.getSubscriptions(); + PersistentSubscription mockedSubscription = Mockito.mock(PersistentSubscription.class); + Mockito.when(mockedSubscription.getDispatcher()).thenThrow(new RuntimeException("Mocked error: getDispatcher")); + subscriptions.put("mockedSubscription", mockedSubscription); + + // Update namespace-level retention policies. + RetentionPolicies retentionPolicies1 = new RetentionPolicies(1, 1); + admin.namespaces().setRetentionAsync(myNamespace, retentionPolicies1); + + // Verify: update retention will be success even if other component update throws exception. + Awaitility.await().untilAsserted(() -> { + ManagedLedgerImpl ML = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + assertEquals(ML.getConfig().getRetentionSizeInMB(), 1); + assertEquals(ML.getConfig().getRetentionTimeMillis(), 1 * 60 * 1000); + }); + + // Update topic-level retention policies. + RetentionPolicies retentionPolicies2 = new RetentionPolicies(2, 2); + admin.topics().setRetentionAsync(tpName, retentionPolicies2); + + // Verify: update retention will be success even if other component update throws exception. + Awaitility.await().untilAsserted(() -> { + ManagedLedgerImpl ML = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + assertEquals(ML.getConfig().getRetentionSizeInMB(), 2); + assertEquals(ML.getConfig().getRetentionTimeMillis(), 2 * 60 * 1000); + }); + + // Cleanup. + subscriptions.clear(); + admin.namespaces().removeRetention(myNamespace); + admin.topics().delete(tpName, false); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesWithBrokerRestartTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesWithBrokerRestartTest.java new file mode 100644 index 0000000000000..672fc2c95f890 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesWithBrokerRestartTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin; + +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.common.policies.data.RetentionPolicies; +import org.awaitility.Awaitility; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; + +@Slf4j +@Test(groups = "broker-admin") +public class TopicPoliciesWithBrokerRestartTest extends MockedPulsarServiceBaseTest { + + @Override + @BeforeClass(alwaysRun = true) + protected void setup() throws Exception { + super.internalSetup(); + setupDefaultTenantAndNamespace(); + } + + @Override + @AfterClass(alwaysRun = true) + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + + @Test + public void testRetentionWithBrokerRestart() throws Exception { + final int messages = 1_000; + final int topicNum = 500; + // (1) Init topic + admin.namespaces().createNamespace("public/retention"); + final String topicName = "persistent://public/retention/retention_with_broker_restart"; + admin.topics().createNonPartitionedTopic(topicName); + for (int i = 0; i < topicNum; i++) { + final String shadowTopicNames = topicName + "_" + i; + admin.topics().createNonPartitionedTopic(shadowTopicNames); + } + // (2) Set retention + final RetentionPolicies retentionPolicies = new RetentionPolicies(20, 20); + for (int i = 0; i < topicNum; i++) { + final String shadowTopicNames = topicName + "_" + i; + admin.topicPolicies().setRetention(shadowTopicNames, retentionPolicies); + } + admin.topicPolicies().setRetention(topicName, retentionPolicies); + // (3) Send messages + @Cleanup + final Producer publisher = pulsarClient.newProducer() + .topic(topicName) + .create(); + for (int i = 0; i < messages; i++) { + publisher.send((i + "").getBytes(StandardCharsets.UTF_8)); + } + // (4) Check configuration + Awaitility.await().untilAsserted(() -> { + final PersistentTopic persistentTopic1 = (PersistentTopic) + pulsar.getBrokerService().getTopic(topicName, true).join().get(); + final ManagedLedgerImpl managedLedger1 = (ManagedLedgerImpl) persistentTopic1.getManagedLedger(); + Assert.assertEquals(managedLedger1.getConfig().getRetentionSizeInMB(), 20); + Assert.assertEquals(managedLedger1.getConfig().getRetentionTimeMillis(), + TimeUnit.MINUTES.toMillis(20)); + }); + // (5) Restart broker + restartBroker(); + // (6) Check configuration again + for (int i = 0; i < topicNum; i++) { + final String shadowTopicNames = topicName + "_" + i; + admin.lookups().lookupTopic(shadowTopicNames); + final PersistentTopic persistentTopicTmp = (PersistentTopic) + pulsar.getBrokerService().getTopic(shadowTopicNames, true).join().get(); + final ManagedLedgerImpl managedLedgerTemp = (ManagedLedgerImpl) persistentTopicTmp.getManagedLedger(); + Assert.assertEquals(managedLedgerTemp.getConfig().getRetentionSizeInMB(), 20); + Assert.assertEquals(managedLedgerTemp.getConfig().getRetentionTimeMillis(), + TimeUnit.MINUTES.toMillis(20)); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsAuthTest.java index efd8b66d754ac..463addf9eaed4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsAuthTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsAuthTest.java @@ -84,11 +84,14 @@ protected void setup() throws Exception { Set providers = new HashSet<>(); providers.add(AuthenticationProviderToken.class.getName()); conf.setAuthenticationProviders(providers); + conf.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); + conf.setBrokerClientAuthenticationParameters("token:" + ADMIN_TOKEN); super.internalSetup(); PulsarAdminBuilder pulsarAdminBuilder = PulsarAdmin.builder().serviceHttpUrl(brokerUrl != null ? brokerUrl.toString() : brokerUrlTls.toString()) .authentication(AuthenticationToken.class.getName(), ADMIN_TOKEN); + closeAdmin(); admin = Mockito.spy(pulsarAdminBuilder.build()); admin.clusters().createCluster(testLocalCluster, new ClusterDataImpl()); admin.tenants().createTenant(testTenant, new TenantInfoImpl(Set.of("role1", "role2"), diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsTest.java index 9aa29f08c5ce8..8940fe4a1f3c3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsTest.java @@ -56,6 +56,7 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationDataHttps; import org.apache.pulsar.broker.namespace.NamespaceService; +import org.apache.pulsar.broker.namespace.TopicExistsInfo; import org.apache.pulsar.broker.rest.Topics; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.BrokerServiceException; @@ -325,6 +326,9 @@ public void testLookUpWithRedirect() throws Exception { conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePort(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); + conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); @Cleanup PulsarTestContext pulsarTestContext2 = createAdditionalPulsarTestContext(conf); PulsarService pulsar2 = pulsarTestContext2.getPulsarService(); @@ -357,9 +361,12 @@ public void testLookUpWithException() throws Exception { CompletableFuture future = new CompletableFuture(); future.completeExceptionally(new BrokerServiceException("Fake Exception")); CompletableFuture existFuture = new CompletableFuture(); - existFuture.complete(true); + existFuture.complete(TopicExistsInfo.newNonPartitionedTopicExists()); doReturn(future).when(nameSpaceService).getBrokerServiceUrlAsync(any(), any()); doReturn(existFuture).when(nameSpaceService).checkTopicExists(any()); + CompletableFuture existBooleanFuture = new CompletableFuture(); + existBooleanFuture.complete(false); + doReturn(existBooleanFuture).when(nameSpaceService).checkNonPartitionedTopicExists(any()); doReturn(nameSpaceService).when(pulsar).getNamespaceService(); AsyncResponse asyncResponse = mock(AsyncResponse.class); ProducerMessages producerMessages = new ProducerMessages(); @@ -370,7 +377,7 @@ public void testLookUpWithException() throws Exception { topics.produceOnPersistentTopic(asyncResponse, testTenant, testNamespace, testTopicName, false, producerMessages); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(RestException.class); verify(asyncResponse, timeout(5000).times(1)).resume(responseCaptor.capture()); - Assert.assertEquals(responseCaptor.getValue().getMessage(), "Can't find owner of given topic."); + Assert.assertTrue(responseCaptor.getValue().getMessage().contains(topicName + " not found")); } @Test @@ -378,8 +385,11 @@ public void testLookUpTopicNotExist() throws Exception { String topicName = "persistent://" + testTenant + "/" + testNamespace + "/" + testTopicName; NamespaceService nameSpaceService = mock(NamespaceService.class); CompletableFuture existFuture = new CompletableFuture(); - existFuture.complete(false); + existFuture.complete(TopicExistsInfo.newTopicNotExists()); + CompletableFuture existBooleanFuture = new CompletableFuture(); + existBooleanFuture.complete(false); doReturn(existFuture).when(nameSpaceService).checkTopicExists(any()); + doReturn(existBooleanFuture).when(nameSpaceService).checkNonPartitionedTopicExists(any()); doReturn(nameSpaceService).when(pulsar).getNamespaceService(); AsyncResponse asyncResponse = mock(AsyncResponse.class); ProducerMessages producerMessages = new ProducerMessages(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsWithoutTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsWithoutTlsTest.java new file mode 100644 index 0000000000000..88bf2f8f42108 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsWithoutTlsTest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import lombok.Cleanup; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.authentication.AuthenticationDataHttps; +import org.apache.pulsar.broker.rest.Topics; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.naming.TopicDomain; +import org.apache.pulsar.common.policies.data.ClusterDataImpl; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.apache.pulsar.websocket.data.ProducerMessage; +import org.apache.pulsar.websocket.data.ProducerMessages; +import org.mockito.ArgumentCaptor; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import javax.ws.rs.container.AsyncResponse; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; +import java.net.URI; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; + +public class TopicsWithoutTlsTest extends MockedPulsarServiceBaseTest { + + private Topics topics; + private final String testLocalCluster = "test"; + private final String testTenant = "my-tenant"; + private final String testNamespace = "my-namespace"; + private final String testTopicName = "my-topic"; + + @Override + @BeforeMethod + protected void setup() throws Exception { + super.internalSetup(); + topics = spy(new Topics()); + topics.setPulsar(pulsar); + doReturn(TopicDomain.persistent.value()).when(topics).domain(); + doReturn("test-app").when(topics).clientAppId(); + doReturn(mock(AuthenticationDataHttps.class)).when(topics).clientAuthData(); + admin.clusters().createCluster(testLocalCluster, new ClusterDataImpl()); + admin.tenants().createTenant(testTenant, new TenantInfoImpl(Set.of("role1", "role2"), Set.of(testLocalCluster))); + admin.namespaces().createNamespace(testTenant + "/" + testNamespace, + Set.of(testLocalCluster)); + } + + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + this.conf.setBrokerServicePortTls(Optional.empty()); + this.conf.setWebServicePortTls(Optional.empty()); + } + + @Override + @AfterMethod + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testLookUpWithRedirect() throws Exception { + String topicName = "persistent://" + testTenant + "/" + testNamespace + "/" + testTopicName; + URI requestPath = URI.create(pulsar.getWebServiceAddress() + "/topics/my-tenant/my-namespace/my-topic"); + //create topic on one broker + admin.topics().createNonPartitionedTopic(topicName); + conf.setBrokerServicePort(Optional.of(0)); + conf.setWebServicePort(Optional.of(0)); + @Cleanup + PulsarTestContext pulsarTestContext2 = createAdditionalPulsarTestContext(conf); + PulsarService pulsar2 = pulsarTestContext2.getPulsarService(); + doReturn(false).when(topics).isRequestHttps(); + UriInfo uriInfo = mock(UriInfo.class); + doReturn(requestPath).when(uriInfo).getRequestUri(); + FieldUtils.writeField(topics, "uri", uriInfo, true); + //do produce on another broker + topics.setPulsar(pulsar2); + AsyncResponse asyncResponse = mock(AsyncResponse.class); + ProducerMessages producerMessages = new ProducerMessages(); + producerMessages.setValueSchema(ObjectMapperFactory.getMapper().getObjectMapper(). + writeValueAsString(Schema.INT64.getSchemaInfo())); + String message = "[]"; + producerMessages.setMessages(createMessages(message)); + topics.produceOnPersistentTopic(asyncResponse, testTenant, testNamespace, testTopicName, false, producerMessages); + ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); + verify(asyncResponse, timeout(5000).times(1)).resume(responseCaptor.capture()); + // Verify got redirect response + Assert.assertEquals(responseCaptor.getValue().getStatusInfo(), Response.Status.TEMPORARY_REDIRECT); + // Verify URI point to address of broker the topic was created on + Assert.assertEquals(responseCaptor.getValue().getLocation().toString(), requestPath.toString()); + } + + private static List createMessages(String message) throws JsonProcessingException { + return ObjectMapperFactory.getMapper().reader() + .forType(new TypeReference>() { + }).readValue(message); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TransactionAndSchemaAuthZTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TransactionAndSchemaAuthZTest.java new file mode 100644 index 0000000000000..f52d6dae9bb23 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TransactionAndSchemaAuthZTest.java @@ -0,0 +1,353 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin; + +import io.jsonwebtoken.Jwts; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import lombok.Cleanup; +import lombok.SneakyThrows; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.transaction.Transaction; +import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; +import org.apache.pulsar.common.naming.SystemTopicNames; +import org.apache.pulsar.common.partition.PartitionedTopicMetadata; +import org.apache.pulsar.common.policies.data.AuthAction; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.common.policies.data.TopicOperation; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.schema.SchemaType; +import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Test(groups = "broker-admin") +public class TransactionAndSchemaAuthZTest extends AuthZTest { + + @SneakyThrows + @BeforeClass(alwaysRun = true) + public void setup() { + configureTokenAuthentication(); + configureDefaultAuthorization(); + enableTransaction(); + start(); + createTransactionCoordinatorAssign(16); + this.superUserAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(SUPER_USER_TOKEN)) + .build(); + final TenantInfo tenantInfo = superUserAdmin.tenants().getTenantInfo("public"); + tenantInfo.getAdminRoles().add(TENANT_ADMIN_SUBJECT); + superUserAdmin.tenants().updateTenant("public", tenantInfo); + this.tenantManagerAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(TENANT_ADMIN_TOKEN)) + .build(); + + superUserAdmin.tenants().createTenant("pulsar", tenantInfo); + superUserAdmin.namespaces().createNamespace("pulsar/system"); + } + + @SneakyThrows + @AfterClass(alwaysRun = true) + public void cleanup() { + close(); + } + + @BeforeMethod + public void before() throws IllegalAccessException { + orignalAuthorizationService = getPulsarService().getBrokerService().getAuthorizationService(); + authorizationService = Mockito.spy(orignalAuthorizationService); + FieldUtils.writeField(getPulsarService().getBrokerService(), "authorizationService", + authorizationService, true); + } + + @AfterMethod + public void after() throws IllegalAccessException { + FieldUtils.writeField(getPulsarService().getBrokerService(), "authorizationService", + orignalAuthorizationService, true); + } + + protected void createTransactionCoordinatorAssign(int numPartitionsOfTC) throws MetadataStoreException { + getPulsarService().getPulsarResources() + .getNamespaceResources() + .getPartitionedTopicResources() + .createPartitionedTopic(SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN, + new PartitionedTopicMetadata(numPartitionsOfTC)); + } + + public enum OperationAuthType { + Lookup, + Produce, + Consume, + AdminOrSuperUser, + NOAuth + } + + private final String testTopic = "persistent://public/default/" + UUID.randomUUID().toString(); + @FunctionalInterface + public interface ThrowingBiConsumer { + void accept(T t) throws PulsarAdminException; + } + + @DataProvider(name = "authFunction") + public Object[][] authFunction () throws Exception { + String sub = "my-sub"; + createTopic(testTopic, false); + @Cleanup final PulsarClient pulsarClient = PulsarClient.builder() + .serviceUrl(getPulsarService().getBrokerServiceUrl()) + .authentication(new AuthenticationToken(SUPER_USER_TOKEN)) + .enableTransaction(true) + .build(); + @Cleanup final Producer producer = pulsarClient.newProducer(Schema.STRING).topic(testTopic).create(); + + @Cleanup final Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(testTopic) + .subscriptionName(sub) + .subscribe(); + + Transaction transaction = pulsarClient.newTransaction().withTransactionTimeout(5, TimeUnit.MINUTES) + .build().get(); + MessageIdImpl messageId = (MessageIdImpl) producer.newMessage().value("test message").send(); + + consumer.acknowledgeAsync(messageId, transaction).get(); + + return new Object[][]{ + // SCHEMA + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.schemas().getSchemaInfo(testTopic), + OperationAuthType.Lookup + }, + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.schemas().getSchemaInfo( + testTopic, 0), + OperationAuthType.Lookup + }, + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.schemas().getAllSchemas( + testTopic), + OperationAuthType.Lookup + }, + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.schemas().createSchema(testTopic, + SchemaInfo.builder().type(SchemaType.STRING).build()), + OperationAuthType.Produce + }, + // TODO: improve the authorization check for testCompatibility and deleteSchema + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.schemas().testCompatibility( + testTopic, SchemaInfo.builder().type(SchemaType.STRING).build()), + OperationAuthType.AdminOrSuperUser + }, + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.schemas().deleteSchema( + testTopic), + OperationAuthType.AdminOrSuperUser + }, + + // TRANSACTION + + // Modify transaction coordinator + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.transactions() + .abortTransaction(transaction.getTxnID()), + OperationAuthType.AdminOrSuperUser + }, + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.transactions() + .scaleTransactionCoordinators(17), + OperationAuthType.AdminOrSuperUser + }, + // TODO: fix authorization check of check transaction coordinator stats. + // Check transaction coordinator stats + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.transactions() + .getCoordinatorInternalStats(1, false), + OperationAuthType.NOAuth + }, + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.transactions() + .getCoordinatorStats(), + OperationAuthType.AdminOrSuperUser + }, + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.transactions() + .getSlowTransactionsByCoordinatorId(1, 5, TimeUnit.SECONDS), + OperationAuthType.NOAuth + }, + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.transactions() + .getTransactionMetadata(transaction.getTxnID()), + OperationAuthType.NOAuth + }, + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.transactions() + .listTransactionCoordinators(), + OperationAuthType.NOAuth + }, + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.transactions() + .getSlowTransactions(5, TimeUnit.SECONDS), + OperationAuthType.AdminOrSuperUser + }, + + // TODO: Check the authorization of the topic when get stats of TB or TP + // Check stats related to transaction buffer and transaction pending ack + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.transactions() + .getPendingAckInternalStats(testTopic, sub, false), + OperationAuthType.NOAuth + }, + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.transactions() + .getPendingAckStats(testTopic, sub, false), + OperationAuthType.NOAuth + }, + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.transactions() + .getPositionStatsInPendingAck(testTopic, sub, messageId.getLedgerId(), + messageId.getEntryId(), null), + OperationAuthType.NOAuth + }, + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.transactions() + .getTransactionBufferInternalStats(testTopic, false), + OperationAuthType.NOAuth + }, + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.transactions() + .getTransactionBufferStats(testTopic, false), + OperationAuthType.NOAuth + }, + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.transactions() + .getTransactionBufferStats(testTopic, false), + OperationAuthType.NOAuth + }, + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.transactions() + .getTransactionInBufferStats(transaction.getTxnID(), testTopic), + OperationAuthType.NOAuth + }, + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.transactions() + .getTransactionInBufferStats(transaction.getTxnID(), testTopic), + OperationAuthType.NOAuth + }, + new Object[] { + (ThrowingBiConsumer) (admin) -> admin.transactions() + .getTransactionInPendingAckStats(transaction.getTxnID(), testTopic, sub), + OperationAuthType.NOAuth + }, + }; + } + + @Test(dataProvider = "authFunction") + public void testSchemaAndTransactionAuthorization(ThrowingBiConsumer adminConsumer, OperationAuthType topicOpType) + throws Exception { + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + + + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + // test tenant manager + if (topicOpType != OperationAuthType.AdminOrSuperUser) { + adminConsumer.accept(tenantManagerAdmin); + } + + if (topicOpType != OperationAuthType.NOAuth) { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> adminConsumer.accept(subAdmin)); + } + + AtomicBoolean execFlag = null; + if (topicOpType == OperationAuthType.Lookup) { + execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.LOOKUP); + } else if (topicOpType == OperationAuthType.Produce) { + execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.PRODUCE); + } else if (topicOpType == OperationAuthType.Consume) { + execFlag = setAuthorizationTopicOperationChecker(subject, TopicOperation.CONSUME); + } + + for (AuthAction action : AuthAction.values()) { + superUserAdmin.topics().grantPermission(testTopic, subject, Set.of(action)); + + if (authActionMatchOperation(topicOpType, action)) { + adminConsumer.accept(subAdmin); + } else { + Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class, + () -> adminConsumer.accept(subAdmin)); + } + superUserAdmin.topics().revokePermissions(testTopic, subject); + } + + if (execFlag != null) { + Assert.assertTrue(execFlag.get()); + } + + } + + private boolean authActionMatchOperation(OperationAuthType operationAuthType, AuthAction action) { + switch (operationAuthType) { + case Lookup -> { + if (AuthAction.consume == action || AuthAction.produce == action) { + return true; + } + } + case Consume -> { + if (AuthAction.consume == action) { + return true; + } + } + case Produce -> { + if (AuthAction.produce == action) { + return true; + } + } + case AdminOrSuperUser -> { + return false; + } + case NOAuth -> { + return true; + } + } + return false; + } + +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApi2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApi2Test.java index cd08977a09b2a..38ccdf6f9c89a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApi2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApi2Test.java @@ -451,7 +451,7 @@ public void testResetCursorOnPosition(String namespaceName) throws Exception { } } - // close consumer which will clean up intenral-receive-queue + // close consumer which will clean up internal-receive-queue consumer.close(); // messages should still be available due to retention diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest.java index e720c9b7613eb..d92c3126c5404 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest.java @@ -111,6 +111,7 @@ import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.compaction.Compactor; +import org.apache.pulsar.compaction.PulsarCompactionServiceFactory; import org.apache.pulsar.metadata.cache.impl.MetadataCacheImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -172,7 +173,9 @@ public void setup() throws Exception { @AfterClass(alwaysRun = true) @Override public void cleanup() throws Exception { + pulsar.getConfiguration().setBrokerShutdownTimeoutMs(0); adminTls.close(); + otheradmin.close(); super.internalCleanup(); mockPulsarSetup.cleanup(); } @@ -449,7 +452,7 @@ public void brokers() throws Exception { for (String ns : nsMap.keySet()) { NamespaceOwnershipStatus nsStatus = nsMap.get(ns); if (ns.equals( - NamespaceService.getHeartbeatNamespace(pulsar.getAdvertisedAddress(), pulsar.getConfiguration()) + NamespaceService.getHeartbeatNamespace(pulsar.getBrokerId(), pulsar.getConfiguration()) + "/0x00000000_0xffffffff")) { assertEquals(nsStatus.broker_assignment, BrokerAssignment.shared); assertFalse(nsStatus.is_controlled); @@ -457,10 +460,7 @@ public void brokers() throws Exception { } } - String[] parts = list.get(0).split(":"); - Assert.assertEquals(parts.length, 2); - Map nsMap2 = adminTls.brokers().getOwnedNamespaces("use", - String.format("%s:%d", parts[0], pulsar.getListenPortHTTPS().get())); + Map nsMap2 = adminTls.brokers().getOwnedNamespaces("use", list.get(0)); Assert.assertEquals(nsMap2.size(), 2); admin.namespaces().deleteNamespace("prop-xyz/use/ns1"); @@ -1380,8 +1380,8 @@ public void testUnsubscribeOnNamespace(Integer numBundles) throws Exception { admin.namespaces().unsubscribeNamespace("prop-xyz/use/ns1-bundles", "my-sub"); - assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/use/ns1-bundles/ds2"), - List.of("my-sub-1", "my-sub-2")); + assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/use/ns1-bundles/ds2").stream() + .sorted().toList(), List.of("my-sub-1", "my-sub-2")); assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/use/ns1-bundles/ds1"), List.of("my-sub-1")); @@ -2075,7 +2075,7 @@ public void testTriggerCompaction() throws Exception { // mock actual compaction, we don't need to really run it CompletableFuture promise = new CompletableFuture<>(); - Compactor compactor = pulsar.getCompactor(); + Compactor compactor = ((PulsarCompactionServiceFactory)pulsar.getCompactionServiceFactory()).getCompactor(); doReturn(promise).when(compactor).compact(topicName); admin.topics().triggerCompaction(topicName); @@ -2112,7 +2112,7 @@ public void testCompactionStatus() throws Exception { // mock actual compaction, we don't need to really run it CompletableFuture promise = new CompletableFuture<>(); - Compactor compactor = pulsar.getCompactor(); + Compactor compactor = ((PulsarCompactionServiceFactory)pulsar.getCompactionServiceFactory()).getCompactor(); doReturn(promise).when(compactor).compact(topicName); admin.topics().triggerCompaction(topicName); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java index 52aadde7b2621..113937c2558d9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java @@ -19,19 +19,28 @@ package org.apache.pulsar.broker.admin.v3; import static org.mockito.Mockito.spy; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; import java.util.Map; import java.util.concurrent.TimeUnit; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.transaction.TransactionTestBase; +import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.naming.SystemTopicNames; +import org.apache.pulsar.common.policies.data.TransactionBufferInternalStats; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @Slf4j -@Test(groups = "broker-admin") +@Test(groups = "broker-admin-isolated") public class AdminApiTransactionMultiBrokerTest extends TransactionTestBase { private static final int NUM_BROKERS = 16; @@ -65,6 +74,9 @@ public void testRedirectOfGetCoordinatorInternalStats() throws Exception { for (int i = 0; map.containsValue(getPulsarServiceList().get(i).getBrokerServiceUrl()); i++) { if (!map.containsValue(getPulsarServiceList().get(i + 1).getBrokerServiceUrl())) + if (localAdmin != null) { + localAdmin.close(); + } localAdmin = spy(createNewPulsarAdmin(PulsarAdmin.builder() .serviceHttpUrl(pulsarServiceList.get(i + 1).getWebServiceAddress()))); } @@ -80,5 +92,43 @@ public void testRedirectOfGetCoordinatorInternalStats() throws Exception { for (int i = 0; i < NUM_PARTITIONS; i++) { localAdmin.transactions().getCoordinatorInternalStats(i, false); } + localAdmin.close(); + } + + @Test + public void testGetTransactionBufferInternalStatsInMultiBroker() throws Exception { + for (int i = 0; i < super.getBrokerCount(); i++) { + getPulsarServiceList().get(i).getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); + } + String topic1 = NAMESPACE1 + "/testGetTransactionBufferInternalStatsInMultiBroker"; + assertTrue(admin.namespaces().getBundles(NAMESPACE1).getNumBundles() > 1); + for (int i = 0; true ; i++) { + topic1 = topic1 + i; + admin.topics().createNonPartitionedTopic(topic1); + String segmentTopicBroker = admin.lookups() + .lookupTopic(NAMESPACE1 + "/" + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS); + String indexTopicBroker = admin.lookups() + .lookupTopic(NAMESPACE1 + "/" + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_INDEXES); + if (segmentTopicBroker.equals(indexTopicBroker)) { + String topicBroker = admin.lookups().lookupTopic(topic1); + if (!topicBroker.equals(segmentTopicBroker)) { + break; + } + } else { + break; + } + } + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.BYTES).topic(topic1).create(); + TransactionBufferInternalStats stats = admin.transactions() + .getTransactionBufferInternalStatsAsync(topic1, true).get(); + assertEquals(stats.snapshotType, AbortedTxnProcessor.SnapshotType.Segment.toString()); + assertNull(stats.singleSnapshotSystemTopicInternalStats); + assertNotNull(stats.segmentInternalStats); + assertTrue(stats.segmentInternalStats.managedLedgerName + .contains(SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS)); + assertNotNull(stats.segmentIndexInternalStats); + assertTrue(stats.segmentIndexInternalStats.managedLedgerName + .contains(SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_INDEXES)); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java index 0e51470da75a5..e32af29c7e962 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java @@ -36,10 +36,13 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import lombok.Cleanup; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.http.HttpStatus; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -48,11 +51,15 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.TransactionIsolationLevel; import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.BatchMessageIdImpl; import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.client.impl.transaction.TransactionImpl; +import org.apache.pulsar.common.api.proto.MarkerType; +import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicDomain; @@ -60,7 +67,9 @@ import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats; +import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.policies.data.TransactionBufferInternalStats; import org.apache.pulsar.common.policies.data.TransactionBufferStats; import org.apache.pulsar.common.policies.data.TransactionCoordinatorInfo; import org.apache.pulsar.common.policies.data.TransactionCoordinatorInternalStats; @@ -72,7 +81,10 @@ import org.apache.pulsar.common.policies.data.TransactionPendingAckStats; import org.apache.pulsar.common.stats.PositionInPendingAckStats; import org.apache.pulsar.packages.management.core.MockedPackagesStorageProvider; +import org.apache.pulsar.transaction.coordinator.TxnMeta; +import org.apache.pulsar.transaction.coordinator.exceptions.CoordinatorException; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl; +import org.apache.pulsar.transaction.coordinator.proto.TxnStatus; import org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -160,7 +172,7 @@ public void testGetTransactionInBufferStats() throws Exception { } catch (ExecutionException ex) { assertTrue(ex.getCause() instanceof PulsarAdminException.NotFoundException); PulsarAdminException.NotFoundException cause = (PulsarAdminException.NotFoundException)ex.getCause(); - assertEquals(cause.getMessage(), "Topic not found"); + assertTrue(cause.getMessage().contains("Topic not found")); } try { pulsar.getBrokerService().getTopic(topic, false); @@ -170,7 +182,7 @@ public void testGetTransactionInBufferStats() throws Exception { } catch (ExecutionException ex) { assertTrue(ex.getCause() instanceof PulsarAdminException.NotFoundException); PulsarAdminException.NotFoundException cause = (PulsarAdminException.NotFoundException)ex.getCause(); - assertEquals(cause.getMessage(), "Topic not found"); + assertTrue(cause.getMessage().contains("Topic not found")); } admin.topics().createNonPartitionedTopic(topic); Producer producer = pulsarClient.newProducer(Schema.BYTES).topic(topic).sendTimeout(0, TimeUnit.SECONDS).create(); @@ -178,8 +190,8 @@ public void testGetTransactionInBufferStats() throws Exception { TransactionInBufferStats transactionInBufferStats = admin.transactions() .getTransactionInBufferStatsAsync(new TxnID(transaction.getTxnIdMostBits(), transaction.getTxnIdLeastBits()), topic).get(); - PositionImpl position = - PositionImpl.get(((MessageIdImpl) messageId).getLedgerId(), ((MessageIdImpl) messageId).getEntryId()); + Position position = + PositionFactory.create(((MessageIdImpl) messageId).getLedgerId(), ((MessageIdImpl) messageId).getEntryId()); assertEquals(transactionInBufferStats.startPosition, position.toString()); assertFalse(transactionInBufferStats.aborted); @@ -205,7 +217,7 @@ public void testGetTransactionInPendingAckStats() throws Exception { } catch (ExecutionException ex) { assertTrue(ex.getCause() instanceof PulsarAdminException.NotFoundException); PulsarAdminException.NotFoundException cause = (PulsarAdminException.NotFoundException)ex.getCause(); - assertEquals(cause.getMessage(), "Topic not found"); + assertTrue(cause.getMessage().contains("Topic not found")); } try { pulsar.getBrokerService().getTopic(topic, false); @@ -216,7 +228,7 @@ public void testGetTransactionInPendingAckStats() throws Exception { } catch (ExecutionException ex) { assertTrue(ex.getCause() instanceof PulsarAdminException.NotFoundException); PulsarAdminException.NotFoundException cause = (PulsarAdminException.NotFoundException)ex.getCause(); - assertEquals(cause.getMessage(), "Topic not found"); + assertTrue(cause.getMessage().contains("Topic not found")); } admin.topics().createNonPartitionedTopic(topic); Producer producer = pulsarClient.newProducer(Schema.BYTES).topic(topic).create(); @@ -299,10 +311,10 @@ public void testGetTransactionMetadata() throws Exception { Map producedPartitions = transactionMetadata.producedPartitions; Map> ackedPartitions = transactionMetadata.ackedPartitions; - PositionImpl position1 = getPositionByMessageId(messageId1); - PositionImpl position2 = getPositionByMessageId(messageId2); - PositionImpl position3 = getPositionByMessageId(messageId3); - PositionImpl position4 = getPositionByMessageId(messageId4); + Position position1 = getPositionByMessageId(messageId1); + Position position2 = getPositionByMessageId(messageId2); + Position position3 = getPositionByMessageId(messageId3); + Position position4 = getPositionByMessageId(messageId4); assertFalse(producedPartitions.get(topic1).aborted); assertFalse(producedPartitions.get(topic2).aborted); @@ -331,7 +343,7 @@ public void testGetTransactionBufferStats() throws Exception { } catch (ExecutionException ex) { assertTrue(ex.getCause() instanceof PulsarAdminException.NotFoundException); PulsarAdminException.NotFoundException cause = (PulsarAdminException.NotFoundException)ex.getCause(); - assertEquals(cause.getMessage(), "Topic not found"); + assertTrue(cause.getMessage().contains("Topic not found")); } try { pulsar.getBrokerService().getTopic(topic, false); @@ -341,7 +353,7 @@ public void testGetTransactionBufferStats() throws Exception { } catch (ExecutionException ex) { assertTrue(ex.getCause() instanceof PulsarAdminException.NotFoundException); PulsarAdminException.NotFoundException cause = (PulsarAdminException.NotFoundException)ex.getCause(); - assertEquals(cause.getMessage(), "Topic not found"); + assertTrue(cause.getMessage().contains("Topic not found")); } admin.topics().createNonPartitionedTopic(topic); Producer producer = pulsarClient.newProducer(Schema.BYTES) @@ -364,7 +376,7 @@ public void testGetTransactionBufferStats() throws Exception { assertEquals(transactionBufferStats.state, "Ready"); assertEquals(transactionBufferStats.maxReadPosition, - PositionImpl.get(((MessageIdImpl) messageId).getLedgerId(), + PositionFactory.create(((MessageIdImpl) messageId).getLedgerId(), ((MessageIdImpl) messageId).getEntryId() + 1).toString()); assertTrue(transactionBufferStats.lastSnapshotTimestamps > currentTime); assertNull(transactionBufferStats.lowWaterMarks); @@ -389,7 +401,7 @@ public void testGetPendingAckStats(String ackType) throws Exception { } catch (ExecutionException ex) { assertTrue(ex.getCause() instanceof PulsarAdminException.NotFoundException); PulsarAdminException.NotFoundException cause = (PulsarAdminException.NotFoundException)ex.getCause(); - assertEquals(cause.getMessage(), "Topic not found"); + assertTrue(cause.getMessage().contains("Topic not found")); } try { pulsar.getBrokerService().getTopic(topic, false); @@ -399,7 +411,7 @@ public void testGetPendingAckStats(String ackType) throws Exception { } catch (ExecutionException ex) { assertTrue(ex.getCause() instanceof PulsarAdminException.NotFoundException); PulsarAdminException.NotFoundException cause = (PulsarAdminException.NotFoundException)ex.getCause(); - assertEquals(cause.getMessage(), "Topic not found"); + assertTrue(cause.getMessage().contains("Topic not found")); } admin.topics().createNonPartitionedTopic(topic); @@ -498,8 +510,8 @@ public void testGetSlowTransactions() throws Exception { assertEquals(transactionMetadata.timeoutAt, 60000); } - private static PositionImpl getPositionByMessageId(MessageId messageId) { - return PositionImpl.get(((MessageIdImpl) messageId).getLedgerId(), ((MessageIdImpl) messageId).getEntryId()); + private static Position getPositionByMessageId(MessageId messageId) { + return PositionFactory.create(((MessageIdImpl) messageId).getLedgerId(), ((MessageIdImpl) messageId).getEntryId()); } @Test(timeOut = 20000) @@ -510,7 +522,7 @@ public void testGetCoordinatorInternalStats() throws Exception { TransactionCoordinatorInternalStats stats = admin.transactions() .getCoordinatorInternalStatsAsync(0, true).get(); - verifyManagedLegerInternalStats(stats.transactionLogStats.managedLedgerInternalStats, 26); + verifyManagedLedgerInternalStats(stats.transactionLogStats.managedLedgerInternalStats, 26); assertEquals(TopicName.get(TopicDomain.persistent.toString(), NamespaceName.SYSTEM_NAMESPACE, MLTransactionLogImpl.TRANSACTION_LOG_PREFIX + "0").getPersistenceNamingEncoding(), stats.transactionLogStats.managedLedgerName); @@ -538,7 +550,7 @@ public void testGetPendingAckInternalStats() throws Exception { } catch (ExecutionException ex) { assertTrue(ex.getCause() instanceof PulsarAdminException.NotFoundException); PulsarAdminException.NotFoundException cause = (PulsarAdminException.NotFoundException)ex.getCause(); - assertEquals(cause.getMessage(), "Topic not found"); + assertTrue(cause.getMessage().contains("Topic not found")); } try { pulsar.getBrokerService().getTopic(topic, false); @@ -548,7 +560,7 @@ public void testGetPendingAckInternalStats() throws Exception { } catch (ExecutionException ex) { assertTrue(ex.getCause() instanceof PulsarAdminException.NotFoundException); PulsarAdminException.NotFoundException cause = (PulsarAdminException.NotFoundException)ex.getCause(); - assertEquals(cause.getMessage(), "Topic not found"); + assertTrue(cause.getMessage().contains("Topic not found")); } admin.topics().createNonPartitionedTopic(topic); Producer producer = pulsarClient.newProducer(Schema.BYTES).topic(topic).create(); @@ -565,7 +577,7 @@ public void testGetPendingAckInternalStats() throws Exception { + subName + SystemTopicNames.PENDING_ACK_STORE_SUFFIX).getPersistenceNamingEncoding(), stats.pendingAckLogStats.managedLedgerName); - verifyManagedLegerInternalStats(managedLedgerInternalStats, 16); + verifyManagedLedgerInternalStats(managedLedgerInternalStats, 16); ManagedLedgerInternalStats finalManagedLedgerInternalStats = managedLedgerInternalStats; managedLedgerInternalStats.cursors.forEach((s, cursorStats) -> { @@ -584,6 +596,93 @@ public void testGetPendingAckInternalStats() throws Exception { assertNull(managedLedgerInternalStats.ledgers.get(0).metadata); } + @Test(timeOut = 20000) + public void testGetTransactionBufferInternalStats() throws Exception { + // Initialize transaction + initTransaction(1); + + // Create topics + final String topic1 = "persistent://public/default/testGetTransactionBufferInternalStats-1"; + final String topic2 = "persistent://public/default/testGetTransactionBufferInternalStats-2"; + final String topic3 = "persistent://public/default/testGetTransactionBufferInternalStats-3"; + pulsar.getConfig().setTransactionCoordinatorEnabled(false); + admin.topics().createNonPartitionedTopic(topic1); + + // Verify NotFoundException when transaction coordinator is disabled + try { + admin.transactions().getTransactionBufferInternalStatsAsync(topic1, true).get(); + fail(); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof PulsarAdminException.NotFoundException); + } + + // Enable transaction coordinator and disable segmented snapshot + pulsar.getConfig().setTransactionCoordinatorEnabled(true); + pulsar.getConfig().setTransactionBufferSegmentedSnapshotEnabled(false); + + // Send a message with a transaction and abort it + Producer producer = pulsarClient.newProducer(Schema.BYTES).topic(topic2).create(); + TransactionImpl transaction = (TransactionImpl) getTransaction(); + producer.newMessage(transaction).send(); + transaction.abort().get(); + + Awaitility.await().untilAsserted(() -> { + // Get transaction buffer internal stats and verify single snapshot stats + TransactionBufferInternalStats stats = admin.transactions() + .getTransactionBufferInternalStatsAsync(topic2, true).get(); + assertEquals(stats.snapshotType, AbortedTxnProcessor.SnapshotType.Single.toString()); + assertNotNull(stats.singleSnapshotSystemTopicInternalStats); + + // Get managed ledger internal stats for the transaction buffer snapshot topic + PersistentTopicInternalStats internalStats = admin.topics().getInternalStats( + TopicName.get(topic2).getNamespace() + "/" + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT); + verifyManagedLedgerInternalStats(stats.singleSnapshotSystemTopicInternalStats.managedLedgerInternalStats, + internalStats); + assertTrue(stats.singleSnapshotSystemTopicInternalStats.managedLedgerName + .contains(SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT)); + assertNull(stats.segmentInternalStats); + assertNull(stats.segmentIndexInternalStats); + }); + + // Configure segmented snapshot and set segment size + pulsar.getConfig().setTransactionBufferSnapshotSegmentSize(9); + pulsar.getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); + + // Send a message with a transaction and abort it + producer = pulsarClient.newProducer(Schema.BYTES).topic(topic3).create(); + transaction = (TransactionImpl) getTransaction(); + producer.newMessage(transaction).send(); + transaction.abort().get(); + + Awaitility.await().untilAsserted(() -> { + // Get transaction buffer internal stats and verify segmented snapshot stats + TransactionBufferInternalStats stats = + admin.transactions().getTransactionBufferInternalStatsAsync(topic3, true).get(); + assertEquals(stats.snapshotType, AbortedTxnProcessor.SnapshotType.Segment.toString()); + assertNull(stats.singleSnapshotSystemTopicInternalStats); + assertNotNull(stats.segmentInternalStats); + + // Get managed ledger internal stats for the transaction buffer segments topic + PersistentTopicInternalStats internalStats = admin.topics().getInternalStats( + TopicName.get(topic2).getNamespace() + "/" + + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS); + verifyManagedLedgerInternalStats(stats.segmentInternalStats.managedLedgerInternalStats, internalStats); + assertTrue(stats.segmentInternalStats.managedLedgerName + .contains(SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS)); + + // Get managed ledger internal stats for the transaction buffer indexes topic + assertNotNull(stats.segmentIndexInternalStats); + internalStats = admin.topics().getInternalStats( + TopicName.get(topic2).getNamespace() + "/" + + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_INDEXES); + verifyManagedLedgerInternalStats(stats.segmentIndexInternalStats.managedLedgerInternalStats, internalStats); + assertTrue(stats.segmentIndexInternalStats.managedLedgerName + .contains(SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_INDEXES)); + }); + } + + + @Test(timeOut = 20000) public void testTransactionNotEnabled() throws Exception { cleanup(); @@ -809,6 +908,147 @@ public void testGetPositionStatsInPendingAckStatsFroBatch() throws Exception { } + @Test + public void testAbortTransaction() throws Exception { + initTransaction(1); + + Transaction transaction = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.MINUTES).build().get(); + + TxnMeta txnMeta = pulsar.getTransactionMetadataStoreService().getTxnMeta(transaction.getTxnID()).get(); + assertEquals(txnMeta.status(), TxnStatus.OPEN); + + // abort + admin.transactions().abortTransaction(transaction.getTxnID()); + try { + pulsar.getTransactionMetadataStoreService().getTxnMeta(transaction.getTxnID()).get(); + fail(); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof CoordinatorException.TransactionNotFoundException); + } + } + + @Test + public void testPeekMessageForSkipTxnMarker() throws Exception { + initTransaction(1); + + final String topic = BrokerTestUtil.newUniqueName("persistent://public/default/peek_marker"); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + int n = 10; + for (int i = 0; i < n; i++) { + Transaction txn = pulsarClient.newTransaction().build().get(); + producer.newMessage(txn).value("msg").send(); + txn.commit().get(); + } + + List> peekMsgs = admin.topics().peekMessages(topic, "t-sub", n, + false, TransactionIsolationLevel.READ_UNCOMMITTED); + assertEquals(peekMsgs.size(), n); + for (Message peekMsg : peekMsgs) { + assertEquals(new String(peekMsg.getValue()), "msg"); + } + } + + @Test + public void testPeekMessageFoReadCommittedMessages() throws Exception { + initTransaction(1); + + final String topic = BrokerTestUtil.newUniqueName("persistent://public/default/peek_txn"); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + int n = 10; + // Alternately sends `n` committed transactional messages and `n` abort transactional messages. + for (int i = 0; i < 2 * n; i++) { + Transaction txn = pulsarClient.newTransaction().build().get(); + if (i % 2 == 0) { + producer.newMessage(txn).value("msg").send(); + txn.commit().get(); + } else { + producer.newMessage(txn).value("msg-aborted").send(); + txn.abort(); + } + } + // Then sends 1 uncommitted transactional messages. + Transaction txn = pulsarClient.newTransaction().build().get(); + producer.newMessage(txn).value("msg-uncommitted").send(); + // Then sends n-1 no transaction messages. + for (int i = 0; i < n - 1; i++) { + producer.newMessage().value("msg-after-uncommitted").send(); + } + + // peek n message, all messages value should be "msg" + { + List> peekMsgs = admin.topics().peekMessages(topic, "t-sub", n, + false, TransactionIsolationLevel.READ_COMMITTED); + assertEquals(peekMsgs.size(), n); + for (Message peekMsg : peekMsgs) { + assertEquals(new String(peekMsg.getValue()), "msg"); + } + } + + // peek 3 * n message, and still get n message, all messages value should be "msg" + { + List> peekMsgs = admin.topics().peekMessages(topic, "t-sub", 2 * n, + false, TransactionIsolationLevel.READ_COMMITTED); + assertEquals(peekMsgs.size(), n); + for (Message peekMsg : peekMsgs) { + assertEquals(new String(peekMsg.getValue()), "msg"); + } + } + } + + @Test + public void testPeekMessageForShowAllMessages() throws Exception { + initTransaction(1); + + final String topic = BrokerTestUtil.newUniqueName("persistent://public/default/peek_all"); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + int n = 10; + // Alternately sends `n` committed transactional messages and `n` abort transactional messages. + for (int i = 0; i < 2 * n; i++) { + Transaction txn = pulsarClient.newTransaction().build().get(); + if (i % 2 == 0) { + producer.newMessage(txn).value("msg").send(); + txn.commit().get(); + } else { + producer.newMessage(txn).value("msg-aborted").send(); + txn.abort(); + } + } + // Then sends `n` uncommitted transactional messages. + Transaction txn = pulsarClient.newTransaction().build().get(); + for (int i = 0; i < n; i++) { + producer.newMessage(txn).value("msg-uncommitted").send(); + } + + // peek 5 * n message, will get 5 * n msg. + List> peekMsgs = admin.topics().peekMessages(topic, "t-sub", 5 * n, + true, TransactionIsolationLevel.READ_UNCOMMITTED); + assertEquals(peekMsgs.size(), 5 * n); + + for (int i = 0; i < 4 * n; i++) { + Message peekMsg = peekMsgs.get(i); + MessageImpl peekMsgImpl = (MessageImpl) peekMsg; + MessageMetadata metadata = peekMsgImpl.getMessageBuilder(); + if (metadata.hasMarkerType()) { + assertTrue(metadata.getMarkerType() == MarkerType.TXN_COMMIT_VALUE || + metadata.getMarkerType() == MarkerType.TXN_ABORT_VALUE); + } else { + String value = new String(peekMsg.getValue()); + assertTrue(value.equals("msg") || value.equals("msg-aborted")); + } + } + for (int i = 4 * n; i < peekMsgs.size(); i++) { + Message peekMsg = peekMsgs.get(i); + assertEquals(new String(peekMsg.getValue()), "msg-uncommitted"); + } + } + private static void verifyCoordinatorStats(String state, long sequenceId, long lowWaterMark) { assertEquals(state, "Ready"); @@ -836,7 +1076,7 @@ private Transaction getTransaction() throws Exception { .withTransactionTimeout(5, TimeUnit.SECONDS).build().get(); } - private static void verifyManagedLegerInternalStats(ManagedLedgerInternalStats managedLedgerInternalStats, + private static void verifyManagedLedgerInternalStats(ManagedLedgerInternalStats managedLedgerInternalStats, long totalSize) { assertEquals(managedLedgerInternalStats.entriesAddedCounter, 1); assertEquals(managedLedgerInternalStats.numberOfEntries, 1); @@ -851,4 +1091,20 @@ private static void verifyManagedLegerInternalStats(ManagedLedgerInternalStats m assertNotNull(managedLedgerInternalStats.ledgers.get(0).metadata); assertEquals(managedLedgerInternalStats.cursors.size(), 1); } + + private static void verifyManagedLedgerInternalStats(ManagedLedgerInternalStats internalStats, + ManagedLedgerInternalStats persistentTopicStats) { + assertEquals(persistentTopicStats.entriesAddedCounter, internalStats.entriesAddedCounter); + assertEquals(persistentTopicStats.numberOfEntries, internalStats.numberOfEntries); + assertEquals(persistentTopicStats.totalSize, internalStats.totalSize); + assertEquals(persistentTopicStats.currentLedgerEntries, internalStats.currentLedgerEntries); + assertEquals(persistentTopicStats.currentLedgerSize, internalStats.currentLedgerSize); + assertEquals(persistentTopicStats.lastLedgerCreationFailureTimestamp, internalStats.lastLedgerCreationFailureTimestamp); + assertEquals(persistentTopicStats.waitingCursorsCount, internalStats.waitingCursorsCount); + assertEquals(persistentTopicStats.pendingAddEntriesCount, internalStats.pendingAddEntriesCount); + assertEquals(persistentTopicStats.lastConfirmedEntry, internalStats.lastConfirmedEntry); + assertNotNull(internalStats.ledgers.get(0).metadata); + assertEquals(persistentTopicStats.ledgers.size(), internalStats.ledgers.size()); + assertEquals(persistentTopicStats.cursors.size(), internalStats.cursors.size()); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/PackagesApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/PackagesApiTest.java index 7085ed178fc28..221e2d47bfa1a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/PackagesApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/PackagesApiTest.java @@ -56,6 +56,23 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + @Test + public void testRepeatUploadThrowConflictException() throws Exception { + // create a temp file for testing + File file = File.createTempFile("package-api-test", ".package"); + + // testing upload api + String packageName = "function://public/default/test@v1"; + PackageMetadata originalMetadata = PackageMetadata.builder().description("test").build(); + admin.packages().upload(originalMetadata, packageName, file.getPath()); + try { + admin.packages().upload(originalMetadata, packageName, file.getPath()); + fail(); + } catch (PulsarAdminException e) { + assertEquals(e.getStatusCode(), 409); + } + } + @Test(timeOut = 60000) public void testPackagesOperations() throws Exception { // create a temp file for testing diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthLogsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthLogsTest.java index 6ffcecbeb9f8b..942a42fa7aaa1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthLogsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthLogsTest.java @@ -60,6 +60,8 @@ public void setup() throws Exception { conf.setAuthorizationEnabled(true); conf.setAuthorizationAllowWildcardsMatching(true); conf.setSuperUserRoles(Sets.newHashSet("super")); + conf.setBrokerClientAuthenticationPlugin(MockAuthentication.class.getName()); + conf.setBrokerClientAuthenticationParameters("user:pass.pass"); internalSetup(); try (PulsarAdmin admin = PulsarAdmin.builder() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java index 58cf4ee418ea4..6b0ff3333bbc7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.auth; -import static org.mockito.Mockito.when; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -26,12 +25,14 @@ import java.net.SocketAddress; import java.util.Collections; import java.util.EnumSet; +import lombok.Cleanup; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.authorization.AuthorizationService; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminBuilder; +import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.AuthAction; @@ -55,12 +56,18 @@ public AuthorizationTest() { @Override public void setup() throws Exception { conf.setClusterName("c1"); + conf.setSystemTopicEnabled(false); + conf.setForceDeleteNamespaceAllowed(true); conf.setAuthenticationEnabled(true); + conf.setForceDeleteNamespaceAllowed(true); + conf.setForceDeleteTenantAllowed(true); conf.setAuthenticationProviders( Sets.newHashSet("org.apache.pulsar.broker.auth.MockAuthenticationProvider")); conf.setAuthorizationEnabled(true); conf.setAuthorizationAllowWildcardsMatching(true); conf.setSuperUserRoles(Sets.newHashSet("pulsar.super_user", "pass.pass")); + conf.setBrokerClientAuthenticationPlugin(MockAuthentication.class.getName()); + conf.setBrokerClientAuthenticationParameters("user:pass.pass"); internalSetup(); } @@ -69,6 +76,11 @@ protected void customizeNewPulsarAdminBuilder(PulsarAdminBuilder pulsarAdminBuil pulsarAdminBuilder.authentication(new MockAuthentication("pass.pass")); } + @Override + protected void customizeNewPulsarClientBuilder(ClientBuilder clientBuilder) { + clientBuilder.authentication(new MockAuthentication("pass.pass")); + } + @AfterClass(alwaysRun = true) @Override public void cleanup() throws Exception { @@ -95,8 +107,9 @@ public void simple() throws Exception { assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); assertTrue(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); - admin.topics().grantPermission("persistent://p1/c1/ns1/ds2", "other-role", - EnumSet.of(AuthAction.consume)); + String topic = "persistent://p1/c1/ns1/ds2"; + admin.topics().createNonPartitionedTopic(topic); + admin.topics().grantPermission(topic, "other-role", EnumSet.of(AuthAction.consume)); waitForChange(); assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds2"), "other-role", null)); @@ -166,8 +179,9 @@ public void simple() throws Exception { assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds2"), "my.role.1", null)); assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds2"), "my.role.2", null)); - admin.topics().grantPermission("persistent://p1/c1/ns1/ds1", "my.*", - EnumSet.of(AuthAction.produce)); + String topic1 = "persistent://p1/c1/ns1/ds1"; + admin.topics().createNonPartitionedTopic(topic1); + admin.topics().grantPermission(topic1, "my.*", EnumSet.of(AuthAction.produce)); waitForChange(); assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my.role.1", null)); @@ -230,8 +244,26 @@ public void simple() throws Exception { assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role2", null, "role2-sub2")); assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "pulsar.super_user", null, "role3-sub1")); - admin.namespaces().deleteNamespace("p1/c1/ns1"); + admin.namespaces().deleteNamespace("p1/c1/ns1", true); admin.tenants().deleteTenant("p1"); + + admin.clusters().deleteCluster("c1"); + } + + @Test + public void testDeleteV1Tenant() throws Exception { + admin.clusters().createCluster("c1", ClusterData.builder().build()); + admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet("role1"), Sets.newHashSet("c1"))); + waitForChange(); + admin.namespaces().createNamespace("p1/c1/ns1"); + waitForChange(); + + + String topic = "persistent://p1/c1/ns1/ds2"; + admin.topics().createNonPartitionedTopic(topic); + + admin.namespaces().deleteNamespace("p1/c1/ns1", true); + admin.tenants().deleteTenant("p1", true); admin.clusters().deleteCluster("c1"); } @@ -283,12 +315,12 @@ public void testGetListWithGetBundleOp() throws Exception { admin.namespaces().grantPermissionOnNamespace(namespaceV1, "pass.pass2", EnumSet.of(AuthAction.produce)); admin.namespaces().createNamespace(namespaceV2, Sets.newHashSet("c1")); admin.namespaces().grantPermissionOnNamespace(namespaceV2, "pass.pass2", EnumSet.of(AuthAction.produce)); + @Cleanup PulsarAdmin admin2 = PulsarAdmin.builder().serviceHttpUrl(brokerUrl != null ? brokerUrl.toString() : brokerUrlTls.toString()) .authentication(new MockAuthentication("pass.pass2")) .build(); - when(pulsar.getAdminClient()).thenReturn(admin2); Assert.assertEquals(admin2.topics().getList(namespaceV1, TopicDomain.non_persistent).size(), 0); Assert.assertEquals(admin2.topics().getList(namespaceV2, TopicDomain.non_persistent).size(), 0); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationWithAuthDataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationWithAuthDataTest.java index b69ccc40607f1..bb0461d321cb8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationWithAuthDataTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationWithAuthDataTest.java @@ -270,6 +270,7 @@ public void testAdmin() throws PulsarAdminException { admin.topics().createNonPartitionedTopic(nonPartitionedTopic); admin.lookups().lookupPartitionedTopic(partitionedTopic); admin.lookups().lookupTopic(nonPartitionedTopic); + admin.topics().delete(nonPartitionedTopic); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/InvalidBrokerConfigForAuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/InvalidBrokerConfigForAuthorizationTest.java index f6e6619f7c497..065a0a8e1bd17 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/InvalidBrokerConfigForAuthorizationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/InvalidBrokerConfigForAuthorizationTest.java @@ -21,6 +21,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.fail; import org.apache.pulsar.broker.PulsarServerException; +import org.testng.annotations.AfterMethod; import org.testng.annotations.Test; public class InvalidBrokerConfigForAuthorizationTest extends MockedPulsarServiceBaseTest { @@ -47,6 +48,8 @@ protected void setup() throws Exception { } + + @AfterMethod(alwaysRun = true) @Override protected void cleanup() throws Exception { internalCleanup(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthentication.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthentication.java index 0b1726617f71f..25ac59796b02c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthentication.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthentication.java @@ -29,7 +29,10 @@ public class MockAuthentication implements Authentication { private static final Logger log = LoggerFactory.getLogger(MockAuthentication.class); - private final String user; + private String user; + + public MockAuthentication() { + } public MockAuthentication(String user) { this.user = user; @@ -67,6 +70,7 @@ public String getCommandData() { @Override public void configure(Map authParams) { + this.user = authParams.get("user"); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthorizationProvider.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthorizationProvider.java index 1b2a6322cba3d..de5117c0187a7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthorizationProvider.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthorizationProvider.java @@ -48,11 +48,6 @@ public CompletableFuture isSuperUser(String role, return roleAuthorizedAsync(role); } - @Override - public CompletableFuture isSuperUser(String role, ServiceConfiguration serviceConfiguration) { - return roleAuthorizedAsync(role); - } - @Override public CompletableFuture isTenantAdmin(String tenant, String role, TenantInfo tenantInfo, AuthenticationDataSource authenticationData) { @@ -128,12 +123,6 @@ public CompletableFuture allowTenantOperationAsync(String tenantName, S return roleAuthorizedAsync(role); } - @Override - public Boolean allowTenantOperation(String tenantName, String role, TenantOperation operation, - AuthenticationDataSource authData) { - return roleAuthorized(role); - } - @Override public CompletableFuture allowNamespaceOperationAsync(NamespaceName namespaceName, String role, @@ -142,15 +131,6 @@ public CompletableFuture allowNamespaceOperationAsync(NamespaceName nam return roleAuthorizedAsync(role); } - @Override - public Boolean allowNamespaceOperation(NamespaceName namespaceName, - String role, - NamespaceOperation operation, - AuthenticationDataSource authData) { - return roleAuthorized(role); - } - - @Override public CompletableFuture allowNamespacePolicyOperationAsync(NamespaceName namespaceName, PolicyName policy, @@ -160,15 +140,6 @@ public CompletableFuture allowNamespacePolicyOperationAsync(NamespaceNa return roleAuthorizedAsync(role); } - @Override - public Boolean allowNamespacePolicyOperation(NamespaceName namespaceName, - PolicyName policy, - PolicyOperation operation, - String role, - AuthenticationDataSource authData) { - return roleAuthorized(role); - } - @Override public CompletableFuture allowTopicOperationAsync(TopicName topic, String role, @@ -177,14 +148,6 @@ public CompletableFuture allowTopicOperationAsync(TopicName topic, return roleAuthorizedAsync(role); } - @Override - public Boolean allowTopicOperation(TopicName topicName, - String role, - TopicOperation operation, - AuthenticationDataSource authData) { - return roleAuthorized(role); - } - CompletableFuture roleAuthorizedAsync(String role) { CompletableFuture promise = new CompletableFuture<>(); try { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockOIDCIdentityProvider.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockOIDCIdentityProvider.java new file mode 100644 index 0000000000000..5d29c443d2bde --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockOIDCIdentityProvider.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.auth; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.matching; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.common.FileSource; +import com.github.tomakehurst.wiremock.extension.Parameters; +import com.github.tomakehurst.wiremock.extension.ResponseTransformer; +import com.github.tomakehurst.wiremock.http.Request; +import com.github.tomakehurst.wiremock.http.Response; +import io.jsonwebtoken.SignatureAlgorithm; +import io.jsonwebtoken.impl.DefaultJwtBuilder; +import io.jsonwebtoken.security.Keys; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.security.KeyPair; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.util.Base64; +import java.util.Date; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Mock OIDC (and therefore OAuth2) server for testing. Note that the client_id is mapped to the token's subject claim. + */ +public class MockOIDCIdentityProvider { + private final WireMockServer server; + private final PublicKey publicKey; + private final String audience; + public MockOIDCIdentityProvider(String clientSecret, String audience, long tokenTTLMillis) { + this.audience = audience; + KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256); + publicKey = keyPair.getPublic(); + server = new WireMockServer(wireMockConfig().port(0) + .extensions(new OAuth2Transformer(keyPair, tokenTTLMillis))); + server.start(); + + // Set up a correct openid-configuration that points to the next stub + server.stubFor( + get(urlEqualTo("/.well-known/openid-configuration")) + .willReturn(aResponse() + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "issuer": "%s", + "token_endpoint": "%s/oauth/token" + } + """.replace("%s", server.baseUrl())))); + + // Only respond when the client sends the expected request body + server.stubFor(post(urlEqualTo("/oauth/token")) + .withRequestBody(matching(".*grant_type=client_credentials.*")) + .withRequestBody(matching(".*audience=" + URLEncoder.encode(audience, StandardCharsets.UTF_8) + ".*")) + .withRequestBody(matching(".*client_id=.*")) + .withRequestBody(matching(".*client_secret=" + clientSecret + "(&.*|$)")) + .willReturn(aResponse().withTransformers("o-auth-token-transformer").withStatus(200))); + } + + public void stop() { + server.stop(); + } + + public String getBase64EncodedPublicKey() { + return Base64.getEncoder().encodeToString(publicKey.getEncoded()); + } + + public String getIssuer() { + return server.baseUrl(); + } + + class OAuth2Transformer extends ResponseTransformer { + + private final PrivateKey privateKey; + private final long tokenTTL; + + private final Pattern clientIdToRolePattern = Pattern.compile("client_id=([A-Za-z0-9-]*)(&|$)"); + + OAuth2Transformer(KeyPair key, long tokenTTLMillis) { + this.privateKey = key.getPrivate(); + this.tokenTTL = tokenTTLMillis; + } + + @Override + public Response transform(Request request, Response response, FileSource files, Parameters parameters) { + Matcher m = clientIdToRolePattern.matcher(request.getBodyAsString()); + if (m.find()) { + String role = m.group(1); + return Response.Builder.like(response).but().body(""" + { + "access_token": "%s", + "expires_in": %d, + "token_type":"Bearer" + } + """.formatted(generateToken(role), + TimeUnit.MILLISECONDS.toSeconds(tokenTTL))).build(); + } else { + return Response.Builder.like(response).but().body("Invalid request").status(400).build(); + } + } + + @Override + public String getName() { + return "o-auth-token-transformer"; + } + + @Override + public boolean applyGlobally() { + return false; + } + + private String generateToken(String role) { + long now = System.currentTimeMillis(); + DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); + defaultJwtBuilder.setHeaderParam("typ", "JWT"); + defaultJwtBuilder.setHeaderParam("alg", "RS256"); + defaultJwtBuilder.setIssuer(server.baseUrl()); + defaultJwtBuilder.setSubject(role); + defaultJwtBuilder.setAudience(audience); + defaultJwtBuilder.setIssuedAt(new Date(now)); + defaultJwtBuilder.setNotBefore(new Date(now)); + defaultJwtBuilder.setExpiration(new Date(now + tokenTTL)); + defaultJwtBuilder.signWith(privateKey); + return defaultJwtBuilder.compact(); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java index b688d5fbf24d8..8dd2fc1c3c26d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java @@ -18,16 +18,22 @@ */ package org.apache.pulsar.broker.auth; -import static org.apache.pulsar.broker.BrokerTestUtil.*; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithoutRecordingInvocations; +import static org.assertj.core.api.Assertions.assertThat; +import static org.testng.Assert.assertEquals; import com.google.common.collect.Sets; +import io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.net.InetSocketAddress; import java.net.URI; import java.net.URL; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Date; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -37,10 +43,15 @@ import java.util.function.Predicate; import javax.ws.rs.container.AsyncResponse; import javax.ws.rs.container.TimeoutHandler; +import lombok.AllArgsConstructor; +import lombok.Data; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.authentication.AuthenticationProviderTls; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminBuilder; @@ -48,6 +59,11 @@ import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.impl.ConnectionPool; +import org.apache.pulsar.client.impl.ProducerImpl; +import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.client.impl.auth.AuthenticationDisabled; +import org.apache.pulsar.client.impl.auth.AuthenticationTls; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.policies.data.TenantInfoImpl; @@ -55,11 +71,12 @@ import org.apache.pulsar.tests.TestRetrySupport; import org.apache.pulsar.utils.ResourceUtils; import org.apache.zookeeper.MockZooKeeper; +import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; import org.mockito.Mockito; import org.mockito.internal.util.MockUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testcontainers.shaded.org.awaitility.Awaitility; import org.testng.annotations.DataProvider; /** @@ -68,7 +85,7 @@ public abstract class MockedPulsarServiceBaseTest extends TestRetrySupport { // All certificate-authority files are copied from the tests/certificate-authority directory and all share the same // root CA. - protected static String getTlsFileForClient(String name) { + public static String getTlsFileForClient(String name) { return ResourceUtils.getAbsolutePath(String.format("certificate-authority/client-keys/%s.pem", name)); } public final static String CA_CERT_FILE_PATH = @@ -131,6 +148,8 @@ protected static String getTlsFileForClient(String name) { protected boolean enableBrokerInterceptor = false; + private final List closeables = new ArrayList<>(); + public MockedPulsarServiceBaseTest() { resetConfig(); } @@ -150,7 +169,6 @@ protected final void resetConfig() { } protected final void internalSetup() throws Exception { - incrementSetupNumber(); init(); lookupUrl = new URI(brokerUrl.toString()); if (isTcpLookup) { @@ -221,16 +239,34 @@ protected void doInitConf() throws Exception { this.conf.setBrokerShutdownTimeoutMs(0L); this.conf.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); this.conf.setBrokerServicePort(Optional.of(0)); - this.conf.setBrokerServicePortTls(Optional.of(0)); this.conf.setAdvertisedAddress("localhost"); this.conf.setWebServicePort(Optional.of(0)); - this.conf.setWebServicePortTls(Optional.of(0)); this.conf.setNumExecutorThreadPoolSize(5); this.conf.setExposeBundlesMetricsInPrometheus(true); + // Disable the dispatcher retry backoff in tests by default + this.conf.setDispatcherRetryBackoffInitialTimeInMs(0); + this.conf.setDispatcherRetryBackoffMaxTimeInMs(0); } protected final void init() throws Exception { + incrementSetupNumber(); doInitConf(); + // trying to config the broker internal client + if (conf.getWebServicePortTls().isPresent() + && conf.getAuthenticationProviders().contains(AuthenticationProviderTls.class.getName()) + && !conf.isTlsEnabledWithKeyStore()) { + // enabled TLS + if (conf.getBrokerClientAuthenticationPlugin() == null + || conf.getBrokerClientAuthenticationPlugin().equals(AuthenticationDisabled.class.getName())) { + conf.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); + conf.setBrokerClientAuthenticationParameters("tlsCertFile:" + BROKER_CERT_FILE_PATH + + ",tlsKeyFile:" + BROKER_KEY_FILE_PATH); + conf.setBrokerClientTlsEnabled(true); + conf.setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH); + conf.setBrokerClientCertificateFilePath(BROKER_CERT_FILE_PATH); + conf.setBrokerClientKeyFilePath(BROKER_KEY_FILE_PATH); + } + } startBroker(); } @@ -238,32 +274,60 @@ protected final void internalCleanup() throws Exception { markCurrentSetupNumberCleaned(); // if init fails, some of these could be null, and if so would throw // an NPE in shutdown, obscuring the real error - if (admin != null) { - admin.close(); - if (MockUtil.isMock(admin)) { - Mockito.reset(admin); - } - admin = null; - } + closeAdmin(); if (pulsarClient != null) { pulsarClient.shutdown(); pulsarClient = null; } if (brokerGateway != null) { brokerGateway.close(); + brokerGateway = null; } if (pulsarTestContext != null) { pulsarTestContext.close(); pulsarTestContext = null; } + resetConfig(); + callCloseables(closeables); + closeables.clear(); onCleanup(); + + // clear fields to avoid test runtime memory leak, pulsarTestContext already handles closing of these instances + pulsar = null; + mockZooKeeper = null; + mockZooKeeperGlobal = null; + } + + protected void closeAdmin() { + if (admin != null) { + admin.close(); + if (MockUtil.isMock(admin)) { + Mockito.reset(admin); + } + admin = null; + } } protected void onCleanup() { } + protected T registerCloseable(T closeable) { + closeables.add(closeable); + return closeable; + } + + private static void callCloseables(List closeables) { + for (int i = closeables.size() - 1; i >= 0; i--) { + try { + closeables.get(i).close(); + } catch (Exception e) { + log.error("Failure in calling close method", e); + } + } + } + protected abstract void setup() throws Exception; protected abstract void cleanup() throws Exception; @@ -397,19 +461,27 @@ protected PulsarTestContext.Builder createPulsarTestContextBuilder(ServiceConfig return builder; } + protected PulsarTestContext createAdditionalPulsarTestContext(ServiceConfiguration conf) throws Exception { + return createAdditionalPulsarTestContext(conf, null); + } /** * This method can be used in test classes for creating additional PulsarTestContext instances * that share the same mock ZooKeeper and BookKeeper instances as the main PulsarTestContext instance. * * @param conf the ServiceConfiguration instance to use + * @param builderCustomizer a consumer that can be used to customize the builder configuration * @return the PulsarTestContext instance * @throws Exception if an error occurs */ - protected PulsarTestContext createAdditionalPulsarTestContext(ServiceConfiguration conf) throws Exception { - return createPulsarTestContextBuilder(conf) + protected PulsarTestContext createAdditionalPulsarTestContext(ServiceConfiguration conf, + Consumer builderCustomizer) throws Exception { + var builder = createPulsarTestContextBuilder(conf) .reuseMockBookkeeperAndMetadataStores(pulsarTestContext) - .reuseSpyConfig(pulsarTestContext) - .build(); + .reuseSpyConfig(pulsarTestContext); + if (builderCustomizer != null) { + builderCustomizer.accept(builder); + } + return builder.build(); } protected void waitForZooKeeperWatchers() { @@ -464,9 +536,7 @@ protected ServiceConfiguration getDefaultConf() { configuration.setBrokerShutdownTimeoutMs(0L); configuration.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); configuration.setBrokerServicePort(Optional.of(0)); - configuration.setBrokerServicePortTls(Optional.of(0)); configuration.setWebServicePort(Optional.of(0)); - configuration.setWebServicePortTls(Optional.of(0)); configuration.setBookkeeperClientExposeStatsToPrometheus(true); configuration.setNumExecutorThreadPoolSize(5); configuration.setBrokerMaxConnections(0); @@ -615,12 +685,18 @@ public static void deleteNamespaceWithRetry(String ns, boolean force, PulsarAdmi */ public static void deleteNamespaceWithRetry(String ns, boolean force, PulsarAdmin admin, Collection pulsars) throws Exception { - Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> { + Awaitility.await() + .pollDelay(500, TimeUnit.MILLISECONDS) + .until(() -> { try { // Maybe fail by race-condition with create topics, just retry. admin.namespaces().deleteNamespace(ns, force); return true; - } catch (Exception ex) { + } catch (PulsarAdminException.NotFoundException ex) { + // namespace was already deleted, ignore exception + return true; + } catch (Exception e) { + log.warn("Failed to delete namespace {} (force={})", ns, force, e); return false; } }); @@ -638,5 +714,55 @@ public Object[][] incorrectPersistentPolicies() { }; } + protected ServiceProducer getServiceProducer(ProducerImpl clientProducer, String topicName) { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).join().get(); + org.apache.pulsar.broker.service.Producer serviceProducer = + persistentTopic.getProducers().get(clientProducer.getProducerName()); + long clientProducerId = WhiteboxImpl.getInternalState(clientProducer, "producerId"); + assertEquals(serviceProducer.getProducerId(), clientProducerId); + assertEquals(serviceProducer.getEpoch(), clientProducer.getConnectionHandler().getEpoch()); + return new ServiceProducer(serviceProducer, persistentTopic); + } + + @Data + @AllArgsConstructor + public static class ServiceProducer { + private org.apache.pulsar.broker.service.Producer serviceProducer; + private PersistentTopic persistentTopic; + } + + protected void sleepSeconds(int seconds){ + try { + Thread.sleep(1000 * seconds); + } catch (InterruptedException e) { + log.warn("This thread has been interrupted", e); + Thread.currentThread().interrupt(); + } + } + + private static void reconnectAllConnections(PulsarClientImpl c) throws Exception { + ConnectionPool pool = c.getCnxPool(); + Method closeAllConnections = ConnectionPool.class.getDeclaredMethod("closeAllConnections", new Class[]{}); + closeAllConnections.setAccessible(true); + closeAllConnections.invoke(pool, new Object[]{}); + } + + protected void reconnectAllConnections() throws Exception { + reconnectAllConnections((PulsarClientImpl) pulsarClient); + } + + protected void assertOtelMetricLongSumValue(String metricName, int value) { + assertThat(pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics()) + .anySatisfy(metric -> OpenTelemetryAssertions.assertThat(metric) + .hasName(metricName) + .hasLongSumSatisfying( + sum -> sum.hasPointsSatisfying(point -> point.hasValue(value)))); + } + + protected void logTopicStats(String topic) { + BrokerTestUtil.logTopicStats(log, admin, topic); + } + private static final Logger log = LoggerFactory.getLogger(MockedPulsarServiceBaseTest.class); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/cache/BundlesQuotasTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/cache/BundlesQuotasTest.java index d78e8c0914c7e..079ce25318a6a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/cache/BundlesQuotasTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/cache/BundlesQuotasTest.java @@ -24,6 +24,8 @@ import com.google.common.collect.Range; import com.google.common.hash.Hashing; import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.resources.LoadBalanceResources; +import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceBundleFactory; import org.apache.pulsar.common.naming.NamespaceName; @@ -41,14 +43,24 @@ public class BundlesQuotasTest { private MetadataStore store; private NamespaceBundleFactory bundleFactory; + private PulsarService pulsar; @BeforeMethod public void setup() throws Exception { store = MetadataStoreFactory.create("memory:local", MetadataStoreConfig.builder().build()); + LoadBalanceResources.QuotaResources quotaResources = new LoadBalanceResources.QuotaResources(store, 30000); - PulsarService pulsar = mock(PulsarService.class); + pulsar = mock(PulsarService.class); when(pulsar.getLocalMetadataStore()).thenReturn(mock(MetadataStoreExtended.class)); when(pulsar.getConfigurationMetadataStore()).thenReturn(mock(MetadataStoreExtended.class)); + + LoadBalanceResources loadBalanceResources = mock(LoadBalanceResources.class); + when(loadBalanceResources.getQuotaResources()).thenReturn(quotaResources); + + PulsarResources pulsarResources = mock(PulsarResources.class); + when(pulsarResources.getLoadBalanceResources()).thenReturn(loadBalanceResources); + + when(pulsar.getPulsarResources()).thenReturn(pulsarResources); bundleFactory = new NamespaceBundleFactory(pulsar, Hashing.crc32()); } @@ -59,7 +71,7 @@ public void teardown() throws Exception { @Test public void testGetSetDefaultQuota() throws Exception { - BundlesQuotas bundlesQuotas = new BundlesQuotas(store); + BundlesQuotas bundlesQuotas = new BundlesQuotas(pulsar); ResourceQuota quota2 = new ResourceQuota(); quota2.setMsgRateIn(10); quota2.setMsgRateOut(20); @@ -75,7 +87,7 @@ public void testGetSetDefaultQuota() throws Exception { @Test public void testGetSetBundleQuota() throws Exception { - BundlesQuotas bundlesQuotas = new BundlesQuotas(store); + BundlesQuotas bundlesQuotas = new BundlesQuotas(pulsar); NamespaceBundle testBundle = new NamespaceBundle(NamespaceName.get("pulsar/test/ns-2"), Range.closedOpen(0L, (long) Integer.MAX_VALUE), bundleFactory); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/AbstractDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/AbstractDeliveryTrackerTest.java index 1d166a8db5c9e..8b72411329c65 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/AbstractDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/AbstractDeliveryTrackerTest.java @@ -37,7 +37,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.awaitility.Awaitility; import org.testng.annotations.AfterClass; @@ -82,7 +82,7 @@ public void test(DelayedDeliveryTracker tracker) throws Exception { assertEquals(tracker.getNumberOfDelayedMessages(), 5); assertTrue(tracker.hasMessageAvailable()); - Set scheduled = tracker.getScheduledMessages(10); + Set scheduled = tracker.getScheduledMessages(10); assertEquals(scheduled.size(), 1); // Move time forward diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTrackerFactoryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTrackerFactoryTest.java new file mode 100644 index 0000000000000..9861ab5723732 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTrackerFactoryTest.java @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed; + +import lombok.Cleanup; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.broker.delayed.bucket.RecoverDelayedDeliveryTrackerException; +import org.apache.pulsar.broker.service.BrokerService; +import org.apache.pulsar.broker.service.Dispatcher; +import org.apache.pulsar.broker.service.Subscription; +import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.api.*; +import org.awaitility.Awaitility; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.lang.reflect.Field; +import java.time.Duration; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +public class DelayedDeliveryTrackerFactoryTest extends ProducerConsumerBase { + @BeforeClass + @Override + public void setup() throws Exception { + conf.setDelayedDeliveryTrackerFactoryClassName(BucketDelayedDeliveryTrackerFactory.class.getName()); + conf.setDelayedDeliveryMaxNumBuckets(10); + conf.setDelayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds(1); + conf.setDelayedDeliveryMaxIndexesPerBucketSnapshotSegment(10); + conf.setDelayedDeliveryMinIndexCountPerBucket(50); + conf.setDelayedDeliveryTickTimeMillis(1024); + conf.setDispatcherReadFailureBackoffInitialTimeInMs(1000); + super.internalSetup(); + super.producerBaseSetup(); + } + + @Override + @AfterClass(alwaysRun = true) + public void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testFallbackToInMemoryTracker() throws Exception { + Pair pair = + mockDelayedDeliveryTrackerFactoryAndDispatcher(); + BrokerService brokerService = pair.getLeft(); + PersistentDispatcherMultipleConsumers dispatcher = pair.getRight(); + + // Since Mocked BucketDelayedDeliveryTrackerFactory.newTracker0() throws RecoverDelayedDeliveryTrackerException, + // the factory should be fallback to InMemoryDelayedDeliveryTrackerFactory + @Cleanup + DelayedDeliveryTracker tracker = brokerService.getDelayedDeliveryTrackerFactory().newTracker(dispatcher); + Assert.assertTrue(tracker instanceof InMemoryDelayedDeliveryTracker); + + DelayedDeliveryTrackerFactory fallbackFactory = brokerService.getFallbackDelayedDeliveryTrackerFactory(); + Assert.assertTrue(fallbackFactory instanceof InMemoryDelayedDeliveryTrackerFactory); + } + + + private Pair mockDelayedDeliveryTrackerFactoryAndDispatcher() + throws Exception { + BrokerService brokerService = Mockito.spy(pulsar.getBrokerService()); + + // Mock dispatcher + PersistentDispatcherMultipleConsumers dispatcher = Mockito.mock(PersistentDispatcherMultipleConsumers.class); + Mockito.doReturn("test").when(dispatcher).getName(); + // Mock BucketDelayedDeliveryTrackerFactory + @Cleanup + BucketDelayedDeliveryTrackerFactory factory = new BucketDelayedDeliveryTrackerFactory(); + factory = Mockito.spy(factory); + factory.initialize(pulsar); + Mockito.doThrow(new RecoverDelayedDeliveryTrackerException(new RuntimeException())) + .when(factory).newTracker0(Mockito.eq(dispatcher)); + // Mock brokerService + Mockito.doReturn(factory).when(brokerService).getDelayedDeliveryTrackerFactory(); + // Mock topic and subscription + PersistentTopic topic = Mockito.mock(PersistentTopic.class); + Mockito.doReturn(brokerService).when(topic).getBrokerService(); + Subscription subscription = Mockito.mock(Subscription.class); + Mockito.doReturn("topic").when(topic).getName(); + Mockito.doReturn("sub").when(subscription).getName(); + Mockito.doReturn(topic).when(dispatcher).getTopic(); + Mockito.doReturn(subscription).when(dispatcher).getSubscription(); + + return Pair.of(brokerService, dispatcher); + } + + @Test + public void testFallbackToInMemoryTrackerFactoryFailed() throws Exception { + Pair pair = + mockDelayedDeliveryTrackerFactoryAndDispatcher(); + BrokerService brokerService = pair.getLeft(); + PersistentDispatcherMultipleConsumers dispatcher = pair.getRight(); + + // Mock InMemoryDelayedDeliveryTrackerFactory + @Cleanup + InMemoryDelayedDeliveryTrackerFactory factory = new InMemoryDelayedDeliveryTrackerFactory(); + factory = Mockito.spy(factory); + factory.initialize(pulsar); + // Mock InMemoryDelayedDeliveryTrackerFactory.newTracker0() throws RuntimeException + Mockito.doThrow(new RuntimeException()).when(factory).newTracker0(Mockito.eq(dispatcher)); + + // Mock brokerService to return mocked InMemoryDelayedDeliveryTrackerFactory + Mockito.doAnswer(inv -> null).when(brokerService).initializeFallbackDelayedDeliveryTrackerFactory(); + Mockito.doReturn(factory).when(brokerService).getFallbackDelayedDeliveryTrackerFactory(); + + // Since Mocked BucketDelayedDeliveryTrackerFactory.newTracker0() throws RecoverDelayedDeliveryTrackerException, + // and Mocked InMemoryDelayedDeliveryTrackerFactory.newTracker0() throws RuntimeException, + // the tracker instance should be DelayedDeliveryTracker.DISABLE + @Cleanup + DelayedDeliveryTracker tracker = brokerService.getDelayedDeliveryTrackerFactory().newTracker(dispatcher); + Assert.assertEquals(tracker, DelayedDeliveryTracker.DISABLE); + } + + // 1. Create BucketDelayedDeliveryTracker failed, fallback to InMemoryDelayedDeliveryTracker, + // 2. Publish delay messages + @Test(timeOut = 60_000) + public void testPublishDelayMessagesAndCreateBucketDelayDeliveryTrackerFailed() throws Exception { + String topicName = "persistent://public/default/" + UUID.randomUUID(); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .enableBatching(false) + .create(); + + // Mock BucketDelayedDeliveryTrackerFactory.newTracker0() throws RecoverDelayedDeliveryTrackerException + PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get(); + topic = Mockito.spy(topic); + BrokerService brokerService = Mockito.spy(pulsar.getBrokerService()); + BucketDelayedDeliveryTrackerFactory factory = + (BucketDelayedDeliveryTrackerFactory) Mockito.spy(brokerService.getDelayedDeliveryTrackerFactory()); + Mockito.doThrow(new RecoverDelayedDeliveryTrackerException(new RuntimeException())) + .when(factory).newTracker0(Mockito.any()); + Mockito.doReturn(factory).when(brokerService).getDelayedDeliveryTrackerFactory(); + + // Return mocked BrokerService + Mockito.doReturn(brokerService).when(topic).getBrokerService(); + + // Set Mocked topic to BrokerService + final var topicMap = brokerService.getTopics(); + topicMap.put(topicName, CompletableFuture.completedFuture(Optional.of(topic))); + + // Create consumer + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName("sub") + .subscriptionType(SubscriptionType.Shared) + .messageListener((c, msg) -> { + try { + c.acknowledge(msg); + } catch (PulsarClientException e) { + throw new RuntimeException(e); + } + }) + .subscribe(); + + PersistentSubscription subscription = topic.getSubscription("sub"); + Dispatcher dispatcher = subscription.getDispatcher(); + Assert.assertTrue(dispatcher instanceof PersistentDispatcherMultipleConsumers); + + // Publish a delay message to initialize DelayedDeliveryTracker + producer.newMessage().value("test").deliverAfter(10_000, TimeUnit.MILLISECONDS).send(); + + // Get DelayedDeliveryTracker from Dispatcher + PersistentDispatcherMultipleConsumers dispatcher0 = (PersistentDispatcherMultipleConsumers) dispatcher; + Field trackerField = + PersistentDispatcherMultipleConsumers.class.getDeclaredField("delayedDeliveryTracker"); + trackerField.setAccessible(true); + + AtomicReference> reference = new AtomicReference<>(); + // Wait until DelayedDeliveryTracker is initialized + Awaitility.await().atMost(Duration.ofSeconds(20)).until(() -> { + @SuppressWarnings("unchecked") + Optional optional = + (Optional) trackerField.get(dispatcher0); + if (optional.isPresent()) { + reference.set(optional); + return true; + } + return false; + }); + + Optional optional = reference.get(); + Assert.assertTrue(optional.get() instanceof InMemoryDelayedDeliveryTracker); + + // Mock DelayedDeliveryTracker and Count the number of addMessage() calls + AtomicInteger counter = new AtomicInteger(0); + InMemoryDelayedDeliveryTracker tracker = (InMemoryDelayedDeliveryTracker) optional.get(); + tracker = Mockito.spy(tracker); + Mockito.doAnswer(inv -> { + counter.incrementAndGet(); + return inv.callRealMethod(); + }).when(tracker).addMessage(Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + // Set Mocked InMemoryDelayedDeliveryTracker back to Dispatcher + trackerField.set(dispatcher0, Optional.of(tracker)); + + // Publish 10 delay messages, so the counter should be 10 + for (int i = 0; i < 10; i++) { + producer.newMessage().value("test") + .deliverAfter(10_000, TimeUnit.MILLISECONDS).send(); + } + + try { + Awaitility.await().atMost(Duration.ofSeconds(20)).until(() -> counter.get() == 10); + } finally { + consumer.close(); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java index 499262c1e60b9..cf0b3c45d7023 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java @@ -24,6 +24,7 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Predicate; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedCursor; @@ -31,7 +32,7 @@ import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats; public class MockManagedCursor implements ManagedCursor { @@ -102,13 +103,13 @@ public List readEntries(int numberOfEntriesToRead) throws InterruptedExce @Override public void asyncReadEntries(int numberOfEntriesToRead, AsyncCallbacks.ReadEntriesCallback callback, Object ctx, - PositionImpl maxPosition) { + Position maxPosition) { } @Override public void asyncReadEntries(int numberOfEntriesToRead, long maxSizeBytes, - AsyncCallbacks.ReadEntriesCallback callback, Object ctx, PositionImpl maxPosition) { + AsyncCallbacks.ReadEntriesCallback callback, Object ctx, Position maxPosition) { } @@ -138,13 +139,13 @@ public List readEntriesOrWait(int maxEntries, long maxSizeBytes) @Override public void asyncReadEntriesOrWait(int numberOfEntriesToRead, AsyncCallbacks.ReadEntriesCallback callback, - Object ctx, PositionImpl maxPosition) { + Object ctx, Position maxPosition) { } @Override public void asyncReadEntriesOrWait(int maxEntries, long maxSizeBytes, AsyncCallbacks.ReadEntriesCallback callback, - Object ctx, PositionImpl maxPosition) { + Object ctx, Position maxPosition) { } @@ -276,6 +277,11 @@ public void asyncFindNewestMatching(FindPositionConstraint constraint, } + @Override + public void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate condition, + AsyncCallbacks.FindEntryCallback callback, Object ctx, boolean isFindFromLedger) { + } + @Override public void resetCursor(Position position) throws InterruptedException, ManagedLedgerException { @@ -381,7 +387,7 @@ public ManagedLedger getManagedLedger() { } @Override - public Range getLastIndividualDeletedRange() { + public Range getLastIndividualDeletedRange() { return null; } @@ -391,7 +397,7 @@ public void trimDeletedEntries(List entries) { } @Override - public long[] getDeletedBatchIndexesAsLongArray(PositionImpl position) { + public long[] getDeletedBatchIndexesAsLongArray(Position position) { return new long[0]; } @@ -409,4 +415,34 @@ public boolean checkAndUpdateReadPositionChanged() { public boolean isClosed() { return false; } + + @Override + public ManagedLedgerInternalStats.CursorStats getCursorStats() { + return null; + } + + @Override + public boolean isMessageDeleted(Position position) { + return false; + } + + @Override + public ManagedCursor duplicateNonDurableCursor(String nonDurableCursorName) throws ManagedLedgerException { + return null; + } + + @Override + public long[] getBatchPositionAckSet(Position position) { + return new long[0]; + } + + @Override + public int applyMaxSizeCap(int maxEntries, long maxSizeBytes) { + return 0; + } + + @Override + public void updateReadStats(int readEntriesCount, long readEntriesSize) { + + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java index 39b3992fbd195..bf5a282a4ee6d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java @@ -45,15 +45,16 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; import org.apache.bookkeeper.mledger.ManagedCursor; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.commons.lang3.mutable.MutableLong; import org.apache.pulsar.broker.delayed.AbstractDeliveryTrackerTest; import org.apache.pulsar.broker.delayed.MockBucketSnapshotStorage; import org.apache.pulsar.broker.delayed.MockManagedCursor; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; +import org.awaitility.Awaitility; import org.roaringbitmap.RoaringBitmap; import org.roaringbitmap.buffer.ImmutableRoaringBitmap; -import org.testcontainers.shaded.org.apache.commons.lang3.mutable.MutableLong; -import org.testcontainers.shaded.org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.DataProvider; @@ -165,7 +166,7 @@ public void testContainsMessage(BucketDelayedDeliveryTracker tracker) { assertTrue(tracker.containsMessage(1, 1)); clockTime.set(20); - Set scheduledMessages = tracker.getScheduledMessages(1); + Set scheduledMessages = tracker.getScheduledMessages(1); assertEquals(scheduledMessages.stream().findFirst().get().getEntryId(), 1); tracker.addMessage(3, 3, 30); @@ -182,7 +183,7 @@ public void testContainsMessage(BucketDelayedDeliveryTracker tracker) { } @Test(dataProvider = "delayedTracker", invocationCount = 10) - public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) { + public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) throws Exception { for (int i = 1; i <= 100; i++) { tracker.addMessage(i, i, i * 10); } @@ -198,7 +199,7 @@ public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) { }); assertTrue(tracker.hasMessageAvailable()); - Set scheduledMessages = new TreeSet<>(); + Set scheduledMessages = new TreeSet<>(); Awaitility.await().untilAsserted(() -> { scheduledMessages.addAll(tracker.getScheduledMessages(100)); assertEquals(scheduledMessages.size(), 1); @@ -219,7 +220,7 @@ public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) { clockTime.set(100 * 10); assertTrue(tracker2.hasMessageAvailable()); - Set scheduledMessages2 = new TreeSet<>(); + Set scheduledMessages2 = new TreeSet<>(); Awaitility.await().untilAsserted(() -> { scheduledMessages2.addAll(tracker2.getScheduledMessages(70)); @@ -227,8 +228,8 @@ public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) { }); int i = 31; - for (PositionImpl scheduledMessage : scheduledMessages2) { - assertEquals(scheduledMessage, PositionImpl.get(i, i)); + for (Position scheduledMessage : scheduledMessages2) { + assertEquals(scheduledMessage, PositionFactory.create(i, i)); i++; } @@ -265,7 +266,7 @@ public void testRoaringBitmapSerialize() { } @Test(dataProvider = "delayedTracker") - public void testMergeSnapshot(final BucketDelayedDeliveryTracker tracker) { + public void testMergeSnapshot(final BucketDelayedDeliveryTracker tracker) throws Exception { for (int i = 1; i <= 110; i++) { tracker.addMessage(i, i, i * 10); Awaitility.await().untilAsserted(() -> { @@ -304,21 +305,21 @@ public void testMergeSnapshot(final BucketDelayedDeliveryTracker tracker) { clockTime.set(110 * 10); - NavigableSet scheduledMessages = new TreeSet<>(); + NavigableSet scheduledMessages = new TreeSet<>(); Awaitility.await().untilAsserted(() -> { scheduledMessages.addAll(tracker2.getScheduledMessages(110)); assertEquals(scheduledMessages.size(), 110); }); for (int i = 1; i <= 110; i++) { - PositionImpl position = scheduledMessages.pollFirst(); - assertEquals(position, PositionImpl.get(i, i)); + Position position = scheduledMessages.pollFirst(); + assertEquals(position, PositionFactory.create(i, i)); } tracker2.close(); } @Test(dataProvider = "delayedTracker") - public void testWithBkException(final BucketDelayedDeliveryTracker tracker) { + public void testWithBkException(final BucketDelayedDeliveryTracker tracker) throws Exception { MockBucketSnapshotStorage mockBucketSnapshotStorage = (MockBucketSnapshotStorage) bucketSnapshotStorage; mockBucketSnapshotStorage.injectCreateException( new BucketSnapshotPersistenceException("Bookie operation timeout, op: Create entry")); @@ -380,7 +381,7 @@ public void testWithBkException(final BucketDelayedDeliveryTracker tracker) { assertEquals(tracker2.getScheduledMessages(100).size(), 0); - Set scheduledMessages = new TreeSet<>(); + Set scheduledMessages = new TreeSet<>(); Awaitility.await().untilAsserted(() -> { scheduledMessages.addAll(tracker2.getScheduledMessages(100)); assertEquals(scheduledMessages.size(), delayedMessagesInSnapshotValue); @@ -418,10 +419,10 @@ public void testWithCreateFailDowngrade(BucketDelayedDeliveryTracker tracker) { assertEquals(6, tracker.getNumberOfDelayedMessages()); - NavigableSet scheduledMessages = tracker.getScheduledMessages(5); + NavigableSet scheduledMessages = tracker.getScheduledMessages(5); for (int i = 1; i <= 5; i++) { - PositionImpl position = scheduledMessages.pollFirst(); - assertEquals(position, PositionImpl.get(i, i)); + Position position = scheduledMessages.pollFirst(); + assertEquals(position, PositionFactory.create(i, i)); } } @@ -439,7 +440,7 @@ public void testMaxIndexesPerSegment(BucketDelayedDeliveryTracker tracker) { tracker.close(); } - + @Test(dataProvider = "delayedTracker") public void testClear(BucketDelayedDeliveryTracker tracker) throws ExecutionException, InterruptedException, TimeoutException { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java index d1cf91635f992..8ac101b2ced5b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java @@ -30,6 +30,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import lombok.Cleanup; import okhttp3.Call; import okhttp3.Callback; @@ -69,7 +70,6 @@ public class BrokerInterceptorTest extends ProducerConsumerBase { public void setup() throws Exception { conf.setSystemTopicEnabled(false); conf.setTopicLevelPoliciesEnabled(false); - this.conf.setDisableBrokerInterceptors(false); this.listener1 = mock(BrokerInterceptor.class); this.ncl1 = mock(NarClassLoader.class); @@ -91,7 +91,20 @@ public void setup() throws Exception { @Override protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder pulsarTestContextBuilder) { - pulsarTestContextBuilder.brokerInterceptor(new CounterBrokerInterceptor()); + HashMap brokerInterceptorWithClassLoaderHashMap = new HashMap<>(); + NarClassLoader narClassLoader = mock(NarClassLoader.class); + BrokerInterceptorWithClassLoader counterBrokerInterceptor + = new BrokerInterceptorWithClassLoader(new CounterBrokerInterceptor(), narClassLoader); + brokerInterceptorWithClassLoaderHashMap.put(CounterBrokerInterceptor.NAME, counterBrokerInterceptor); + BrokerInterceptors brokerInterceptors = new BrokerInterceptors(brokerInterceptorWithClassLoaderHashMap); + pulsarTestContextBuilder.brokerInterceptor(brokerInterceptors); + } + + private CounterBrokerInterceptor getCounterBrokerInterceptor() { + BrokerInterceptor brokerInterceptor = pulsar.getBrokerInterceptor(); + BrokerInterceptorWithClassLoader brokerInterceptorWithClassLoader = + ((BrokerInterceptors) brokerInterceptor).getInterceptors().get(CounterBrokerInterceptor.NAME); + return (CounterBrokerInterceptor) brokerInterceptorWithClassLoader.getInterceptor(); } @Override @@ -119,93 +132,83 @@ public void testInitialize() throws Exception { @Test public void testWebserviceRequest() throws PulsarAdminException { - BrokerInterceptor listener = pulsar.getBrokerInterceptor(); - Assert.assertTrue(listener instanceof CounterBrokerInterceptor); admin.namespaces().createNamespace("public/test", 4); - Awaitility.await().until(() -> ((CounterBrokerInterceptor) listener).getCount() >= 1); + Awaitility.await().until(() -> getCounterBrokerInterceptor().getCount() >= 1); } @Test public void testPulsarCommand() throws PulsarClientException { - BrokerInterceptor listener = pulsar.getBrokerInterceptor(); - Assert.assertTrue(listener instanceof CounterBrokerInterceptor); pulsarClient.newProducer(Schema.BOOL).topic("test").create(); // CONNECT and PRODUCER - Awaitility.await().until(() -> ((CounterBrokerInterceptor) listener).getCount() >= 2); + Awaitility.await().until(() -> getCounterBrokerInterceptor().getCount() >= 2); } @Test public void testConnectionCreation() throws PulsarClientException { - BrokerInterceptor listener = pulsar.getBrokerInterceptor(); - Assert.assertTrue(listener instanceof CounterBrokerInterceptor); pulsarClient.newProducer(Schema.BOOL).topic("test").create(); pulsarClient.newConsumer(Schema.STRING).topic("test1").subscriptionName("test-sub").subscribe(); // single connection for both producer and consumer - Awaitility.await().until(() -> ((CounterBrokerInterceptor) listener).getConnectionCreationCount() == 1); + Awaitility.await().until(() -> getCounterBrokerInterceptor().getConnectionCreationCount() == 1); } @Test public void testProducerCreation() throws PulsarClientException { - BrokerInterceptor listener = pulsar.getBrokerInterceptor(); - Assert.assertTrue(listener instanceof CounterBrokerInterceptor); - assertEquals(((CounterBrokerInterceptor) listener).getProducerCount(), 0); + CounterBrokerInterceptor counterBrokerInterceptor = getCounterBrokerInterceptor(); + assertEquals(counterBrokerInterceptor.getProducerCount(), 0); pulsarClient.newProducer(Schema.BOOL).topic("test").create(); - Awaitility.await().until(() -> ((CounterBrokerInterceptor) listener).getProducerCount() == 1); + Awaitility.await().until(() -> counterBrokerInterceptor.getProducerCount() == 1); } @Test public void testProducerClose() throws PulsarClientException { - BrokerInterceptor listener = pulsar.getBrokerInterceptor(); - Assert.assertTrue(listener instanceof CounterBrokerInterceptor); - assertEquals(((CounterBrokerInterceptor) listener).getProducerCount(), 0); + CounterBrokerInterceptor counterBrokerInterceptor = getCounterBrokerInterceptor(); + assertEquals(counterBrokerInterceptor.getProducerCount(), 0); Producer producer = pulsarClient.newProducer(Schema.BOOL).topic("test").create(); - Awaitility.await().until(() -> ((CounterBrokerInterceptor) listener).getProducerCount() == 1); + Awaitility.await().until(() -> counterBrokerInterceptor.getProducerCount() == 1); producer.close(); - Awaitility.await().until(() -> ((CounterBrokerInterceptor) listener).getProducerCount() == 0); + Awaitility.await().until(() -> counterBrokerInterceptor.getProducerCount() == 0); } @Test public void testConsumerCreation() throws PulsarClientException { - BrokerInterceptor listener = pulsar.getBrokerInterceptor(); - Assert.assertTrue(listener instanceof CounterBrokerInterceptor); - assertEquals(((CounterBrokerInterceptor) listener).getConsumerCount(), 0); + CounterBrokerInterceptor counterBrokerInterceptor = getCounterBrokerInterceptor(); + assertEquals(counterBrokerInterceptor.getConsumerCount(), 0); pulsarClient.newConsumer(Schema.STRING).topic("test1").subscriptionName("test-sub").subscribe(); - Awaitility.await().until(() -> ((CounterBrokerInterceptor) listener).getConsumerCount() == 1); + Awaitility.await().until(() -> counterBrokerInterceptor.getConsumerCount() == 1); } @Test public void testConsumerClose() throws PulsarClientException { - BrokerInterceptor listener = pulsar.getBrokerInterceptor(); - Assert.assertTrue(listener instanceof CounterBrokerInterceptor); - assertEquals(((CounterBrokerInterceptor) listener).getConsumerCount(), 0); + CounterBrokerInterceptor counterBrokerInterceptor = getCounterBrokerInterceptor(); + assertEquals(counterBrokerInterceptor.getConsumerCount(), 0); Consumer consumer = pulsarClient .newConsumer(Schema.STRING).topic("test1").subscriptionName("test-sub").subscribe(); - Awaitility.await().until(() -> ((CounterBrokerInterceptor) listener).getConsumerCount() == 1); + Awaitility.await().until(() -> counterBrokerInterceptor.getConsumerCount() == 1); consumer.close(); - Awaitility.await().until(() -> ((CounterBrokerInterceptor) listener).getConsumerCount() == 0); + Awaitility.await().until(() -> counterBrokerInterceptor.getConsumerCount() == 0); } @Test public void testMessagePublishAndProduced() throws PulsarClientException { - BrokerInterceptor listener = pulsar.getBrokerInterceptor(); - Assert.assertTrue(listener instanceof CounterBrokerInterceptor); + CounterBrokerInterceptor counterBrokerInterceptor = getCounterBrokerInterceptor(); @Cleanup Producer producer = pulsarClient.newProducer(Schema.STRING) .topic("test-before-send-message") .create(); - assertEquals(((CounterBrokerInterceptor)listener).getMessagePublishCount(),0); - assertEquals(((CounterBrokerInterceptor)listener).getMessageProducedCount(),0); + assertEquals(counterBrokerInterceptor.getMessagePublishCount(), 0); + assertEquals(counterBrokerInterceptor.getMessageProducedCount(), 0); producer.send("hello world"); - assertEquals(((CounterBrokerInterceptor)listener).getMessagePublishCount(),1); - assertEquals(((CounterBrokerInterceptor)listener).getMessageProducedCount(),1); + Awaitility.await().untilAsserted(() -> { + assertEquals(counterBrokerInterceptor.getMessagePublishCount(), 1); + assertEquals(counterBrokerInterceptor.getMessageProducedCount(), 1); + }); } @Test public void testBeforeSendMessage() throws PulsarClientException { - BrokerInterceptor listener = pulsar.getBrokerInterceptor(); - Assert.assertTrue(listener instanceof CounterBrokerInterceptor); + CounterBrokerInterceptor counterBrokerInterceptor = getCounterBrokerInterceptor(); @Cleanup Producer producer = pulsarClient.newProducer(Schema.STRING) @@ -217,26 +220,22 @@ public void testBeforeSendMessage() throws PulsarClientException { .subscriptionName("test") .subscribe(); - assertEquals(((CounterBrokerInterceptor)listener).getMessageProducedCount(),0); - assertEquals(((CounterBrokerInterceptor)listener).getMessageDispatchCount(),0); + assertEquals(counterBrokerInterceptor.getMessageProducedCount(), 0); + assertEquals(counterBrokerInterceptor.getMessageDispatchCount(), 0); producer.send("hello world"); - assertEquals(((CounterBrokerInterceptor)listener).getMessageProducedCount(),1); - + Awaitility.await().until(() -> counterBrokerInterceptor.getMessageProducedCount() == 1); Message msg = consumer.receive(); assertEquals(msg.getValue(), "hello world"); - Awaitility.await().until(() -> ((CounterBrokerInterceptor) listener).getBeforeSendCount() == 1); - Awaitility.await().until(() -> ((CounterBrokerInterceptor) listener).getBeforeSendCountAtConsumerLevel() == 1); - Awaitility.await().until(() -> ((CounterBrokerInterceptor) listener).getMessageDispatchCount() == 1); + Awaitility.await().until(() -> counterBrokerInterceptor.getBeforeSendCount() == 1); + Awaitility.await().until(() -> counterBrokerInterceptor.getBeforeSendCountAtConsumerLevel() == 1); + Awaitility.await().until(() -> counterBrokerInterceptor.getMessageDispatchCount() == 1); } @Test public void testInterceptAck() throws Exception { final String topic = "test-intercept-ack" + UUID.randomUUID(); - BrokerInterceptor interceptor = pulsar.getBrokerInterceptor(); - Assert.assertTrue(interceptor instanceof CounterBrokerInterceptor); - try (Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); Consumer consumer = pulsarClient.newConsumer(Schema.STRING).topic(topic) .subscriptionName("test-sub").subscribe()) { @@ -244,13 +243,12 @@ public void testInterceptAck() throws Exception { Message message = consumer.receive(); consumer.acknowledge(message); } - Awaitility.await().until(() -> ((CounterBrokerInterceptor) interceptor).getHandleAckCount() == 1); + Awaitility.await().until(() -> getCounterBrokerInterceptor().getHandleAckCount() == 1); } @Test public void asyncResponseFilterTest() throws Exception { - Assert.assertTrue(pulsar.getBrokerInterceptor() instanceof CounterBrokerInterceptor); - CounterBrokerInterceptor interceptor = (CounterBrokerInterceptor) pulsar.getBrokerInterceptor(); + CounterBrokerInterceptor interceptor = getCounterBrokerInterceptor(); interceptor.clearResponseList(); OkHttpClient client = new OkHttpClient(); @@ -307,4 +305,19 @@ public void requestInterceptorFailedTest() { } } + @Test + public void testInterceptNack() throws Exception { + final String topic = "test-intercept-nack" + UUID.randomUUID(); + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .negativeAckRedeliveryDelay(1, TimeUnit.SECONDS) + .topic(topic) + .subscriptionName("test-sub").subscribe(); + producer.send("test intercept nack message"); + Message message = consumer.receive(); + consumer.negativeAcknowledge(message); + Awaitility.await().until(() -> getCounterBrokerInterceptor().getHandleNackCount().get() == 1); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorUtilsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorUtilsTest.java index 709e43b37286c..979bf6cd0d5db 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorUtilsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorUtilsTest.java @@ -65,13 +65,13 @@ public void testLoadBrokerEventListener() throws Exception { BrokerInterceptorWithClassLoader returnedPhWithCL = BrokerInterceptorUtils.load(metadata, ""); BrokerInterceptor returnedPh = returnedPhWithCL.getInterceptor(); - assertSame(mockLoader, returnedPhWithCL.getClassLoader()); + assertSame(mockLoader, returnedPhWithCL.getNarClassLoader()); assertTrue(returnedPh instanceof MockBrokerInterceptor); } } @Test(expectedExceptions = IOException.class) - public void testLoadBrokerEventListenerWithBlankListerClass() throws Exception { + public void testLoadBrokerEventListenerWithBlankListenerClass() throws Exception { BrokerInterceptorDefinition def = new BrokerInterceptorDefinition(); def.setDescription("test-broker-listener"); @@ -98,7 +98,7 @@ public void testLoadBrokerEventListenerWithBlankListerClass() throws Exception { } @Test(expectedExceptions = IOException.class) - public void testLoadBrokerEventListenerWithWrongListerClass() throws Exception { + public void testLoadBrokerEventListenerWithWrongListenerClass() throws Exception { BrokerInterceptorDefinition def = new BrokerInterceptorDefinition(); def.setInterceptorClass(Runnable.class.getName()); def.setDescription("test-broker-listener"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoaderTest.java index a2f97e16a76ae..64d4b5ee6cca5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoaderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoaderTest.java @@ -135,7 +135,7 @@ public void close() { new BrokerInterceptorWithClassLoader(interceptor, narLoader); ClassLoader curClassLoader = Thread.currentThread().getContextClassLoader(); // test class loader - assertEquals(brokerInterceptorWithClassLoader.getClassLoader(), narLoader); + assertEquals(brokerInterceptorWithClassLoader.getNarClassLoader(), narLoader); // test initialize brokerInterceptorWithClassLoader.initialize(mock(PulsarService.class)); assertEquals(Thread.currentThread().getContextClassLoader(), curClassLoader); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/CounterBrokerInterceptor.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/CounterBrokerInterceptor.java index 9c327a0ea6e78..91bfa8185e0f0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/CounterBrokerInterceptor.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/CounterBrokerInterceptor.java @@ -31,6 +31,7 @@ import javax.servlet.http.HttpServletRequest; import lombok.AllArgsConstructor; import lombok.Data; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Entry; import org.apache.http.HttpStatus; @@ -62,9 +63,12 @@ public class CounterBrokerInterceptor implements BrokerInterceptor { private final AtomicInteger messageDispatchCount = new AtomicInteger(); private final AtomicInteger messageAckCount = new AtomicInteger(); private final AtomicInteger handleAckCount = new AtomicInteger(); + @Getter + private final AtomicInteger handleNackCount = new AtomicInteger(); private final AtomicInteger txnCount = new AtomicInteger(); private final AtomicInteger committedTxnCount = new AtomicInteger(); private final AtomicInteger abortedTxnCount = new AtomicInteger(); + public static final String NAME = "COUNTER-BROKER-INTERCEPTOR"; public void reset() { beforeSendCount.set(0); @@ -79,6 +83,7 @@ public void reset() { txnCount.set(0); committedTxnCount.set(0); abortedTxnCount.set(0); + handleNackCount.set(0); } private final List responseList = new CopyOnWriteArrayList<>(); @@ -209,6 +214,9 @@ public void onPulsarCommand(BaseCommand command, ServerCnx cnx) { if (command.getType().equals(BaseCommand.Type.ACK)) { handleAckCount.incrementAndGet(); } + if(command.getType().equals(BaseCommand.Type.REDELIVER_UNACKNOWLEDGED_MESSAGES)) { + handleNackCount.incrementAndGet(); + } count.incrementAndGet(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/ExceptionsBrokerInterceptorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/ExceptionsBrokerInterceptorTest.java index aa254a8ac168a..007378493cb0a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/ExceptionsBrokerInterceptorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/ExceptionsBrokerInterceptorTest.java @@ -22,7 +22,6 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; - import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; @@ -35,6 +34,7 @@ import org.apache.pulsar.client.impl.ConsumerImpl; import org.apache.pulsar.common.nar.NarClassLoader; import org.awaitility.Awaitility; +import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -46,7 +46,6 @@ public class ExceptionsBrokerInterceptorTest extends ProducerConsumerBase { public void setup() throws Exception { conf.setSystemTopicEnabled(false); conf.setTopicLevelPoliciesEnabled(false); - this.conf.setDisableBrokerInterceptors(false); this.enableBrokerInterceptor = true; @@ -54,6 +53,7 @@ public void setup() throws Exception { super.producerBaseSetup(); } + @AfterMethod(alwaysRun = true) @Override protected void cleanup() throws Exception { super.internalCleanup(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/MangedLedgerInterceptorImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/MangedLedgerInterceptorImplTest.java index 7d164b68147ab..74a88382b0e0e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/MangedLedgerInterceptorImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/MangedLedgerInterceptorImplTest.java @@ -18,8 +18,15 @@ */ package org.apache.pulsar.broker.intercept; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertNotNull; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.function.Predicate; @@ -34,8 +41,6 @@ import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.OpAddEntry; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.bookkeeper.mledger.intercept.ManagedLedgerInterceptor; import org.apache.bookkeeper.test.MockedBookKeeperTestCase; import org.apache.pulsar.common.api.proto.BrokerEntryMetadata; @@ -49,15 +54,6 @@ import org.testng.Assert; import org.testng.annotations.Test; -import java.nio.charset.StandardCharsets; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotEquals; -import static org.testng.Assert.assertNotNull; - @Test(groups = "broker") public class MangedLedgerInterceptorImplTest extends MockedBookKeeperTestCase { private static final Logger log = LoggerFactory.getLogger(MangedLedgerInterceptorImplTest.class); @@ -264,9 +260,9 @@ public void testFindPositionByIndex() throws Exception { assertEquals(((ManagedLedgerInterceptorImpl) ledger.getManagedLedgerInterceptor()).getIndex(), 9); - PositionImpl position = null; + Position position = null; for (int index = 0; index <= ((ManagedLedgerInterceptorImpl) ledger.getManagedLedgerInterceptor()).getIndex(); index ++) { - position = (PositionImpl) ledger.asyncFindPosition(new IndexSearchPredicate(index)).get(); + position = ledger.asyncFindPosition(new IndexSearchPredicate(index)).get(); assertEquals(position.getEntryId(), (index % maxSequenceIdPerLedger) / MOCK_BATCH_SIZE); } @@ -279,7 +275,7 @@ public void testFindPositionByIndex() throws Exception { assertNotEquals(firstLedgerId, secondLedgerId); for (int index = 0; index <= ((ManagedLedgerInterceptorImpl) ledger.getManagedLedgerInterceptor()).getIndex(); index ++) { - position = (PositionImpl) ledger.asyncFindPosition(new IndexSearchPredicate(index)).get(); + position = ledger.asyncFindPosition(new IndexSearchPredicate(index)).get(); assertEquals(position.getEntryId(), (index % maxSequenceIdPerLedger) / MOCK_BATCH_SIZE); } @@ -298,7 +294,7 @@ public void testFindPositionByIndex() throws Exception { assertNotEquals(secondLedgerId, thirdLedgerId); for (int index = 0; index <= ((ManagedLedgerInterceptorImpl) ledger.getManagedLedgerInterceptor()).getIndex(); index ++) { - position = (PositionImpl) ledger.asyncFindPosition(new IndexSearchPredicate(index)).get(); + position = ledger.asyncFindPosition(new IndexSearchPredicate(index)).get(); assertEquals(position.getEntryId(), (index % maxSequenceIdPerLedger) / MOCK_BATCH_SIZE); } cursor.close(); @@ -394,16 +390,15 @@ public MockManagedLedgerInterceptorImpl( } @Override - public OpAddEntry beforeAddEntry(OpAddEntry op, int numberOfMessages) { + public void beforeAddEntry(AddEntryOperation op, int numberOfMessages) { if (op == null || numberOfMessages <= 0) { - return op; + return; } op.setData(Commands.addBrokerEntryMetadata(op.getData(), brokerEntryMetadataInterceptors, numberOfMessages)); if (op != null) { throw new RuntimeException("throw exception before add entry for test"); } - return op; } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AdvertisedListenersMultiBrokerLeaderElectionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AdvertisedListenersMultiBrokerLeaderElectionTest.java new file mode 100644 index 0000000000000..5adc78b2c5212 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AdvertisedListenersMultiBrokerLeaderElectionTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance; + +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class AdvertisedListenersMultiBrokerLeaderElectionTest extends MultiBrokerLeaderElectionTest { + @Override + protected PulsarTestContext.Builder createPulsarTestContextBuilder(ServiceConfiguration conf) { + conf.setWebServicePortTls(Optional.of(0)); + return super.createPulsarTestContextBuilder(conf).preallocatePorts(true).configOverride(config -> { + // use advertised address that is different than the name used in the advertised listeners + config.setAdvertisedAddress("localhost"); + config.setAdvertisedListeners( + "public_pulsar:pulsar://127.0.0.1:" + config.getBrokerServicePort().get() + + ",public_http:http://127.0.0.1:" + config.getWebServicePort().get() + + ",public_https:https://127.0.0.1:" + config.getWebServicePortTls().get()); + }); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AdvertisedListenersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AdvertisedListenersTest.java index 7a8154312e4dc..a88ccd60ae4c4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AdvertisedListenersTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AdvertisedListenersTest.java @@ -78,7 +78,6 @@ private void updateConfig(ServiceConfiguration conf, String advertisedAddress) { ",public_https:https://localhost:" + httpsPort); conf.setBrokerServicePort(Optional.of(pulsarPort)); conf.setWebServicePort(Optional.of(httpPort)); - conf.setWebServicePortTls(Optional.of(httpsPort)); } @Test @@ -101,7 +100,6 @@ public void testLookup() throws Exception { assertEquals(new URI(ld.getBrokerUrl()).getHost(), "localhost"); assertEquals(new URI(ld.getHttpUrl()).getHost(), "localhost"); - assertEquals(new URI(ld.getHttpUrlTls()).getHost(), "localhost"); // Produce data diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java index 560cfa9216a02..f1e462c4ec784 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java @@ -26,6 +26,7 @@ import com.google.common.collect.Range; import com.google.common.collect.Sets; import com.google.common.hash.Hashing; +import io.opentelemetry.api.OpenTelemetry; import java.lang.reflect.Field; import java.util.Collections; import java.util.HashMap; @@ -39,6 +40,7 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; +import org.apache.pulsar.broker.loadbalance.impl.BundleRangeCache; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl; import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerWrapper; @@ -58,8 +60,6 @@ import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.policies.data.TenantInfoImpl; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.awaitility.Awaitility; import org.testng.annotations.AfterClass; @@ -103,14 +103,14 @@ public void setup() throws Exception { setupConfigs(conf); super.internalSetup(conf); pulsar1 = pulsar; - primaryHost = String.format("%s:%d", "localhost", pulsar1.getListenPortHTTP().get()); + primaryHost = pulsar1.getBrokerId(); admin1 = admin; var config2 = getDefaultConf(); setupConfigs(config2); additionalPulsarTestContext = createAdditionalPulsarTestContext(config2); pulsar2 = additionalPulsarTestContext.getPulsarService(); - secondaryHost = String.format("%s:%d", "localhost", pulsar2.getListenPortHTTP().get()); + secondaryHost = pulsar2.getBrokerId(); primaryLoadManager = getField(pulsar1.getLoadManager().get(), "loadManager"); secondaryLoadManager = getField(pulsar2.getLoadManager().get(), "loadManager"); @@ -136,8 +136,9 @@ protected void cleanup() throws Exception { protected void beforePulsarStart(PulsarService pulsar) throws Exception { if (resources == null) { - MetadataStoreExtended localStore = pulsar.createLocalMetadataStore(null); - MetadataStoreExtended configStore = (MetadataStoreExtended) pulsar.createConfigurationMetadataStore(null); + MetadataStoreExtended localStore = pulsar.createLocalMetadataStore(null, OpenTelemetry.noop()); + MetadataStoreExtended configStore = + (MetadataStoreExtended) pulsar.createConfigurationMetadataStore(null, OpenTelemetry.noop()); resources = new PulsarResources(localStore, configStore); } this.createNamespaceIfNotExists(resources, NamespaceName.SYSTEM_NAMESPACE.getTenant(), @@ -165,12 +166,10 @@ protected void createNamespaceIfNotExists(PulsarResources resources, } } - protected Object getBundleOwnershipData(){ - return ConcurrentOpenHashMap.>>newBuilder().build(); + return new BundleRangeCache(); } - protected String getLoadManagerClassName() { return ModularLoadManagerImpl.class.getName(); } @@ -364,17 +363,8 @@ protected void selectBrokerForNamespace( Object ownershipData, String broker, String namespace, String assignedBundleName) { - ConcurrentOpenHashMap>> - brokerToNamespaceToBundleRange = - (ConcurrentOpenHashMap>>) ownershipData; - ConcurrentOpenHashSet bundleSet = - ConcurrentOpenHashSet.newBuilder().build(); - bundleSet.add(assignedBundleName); - ConcurrentOpenHashMap> nsToBundleMap = - ConcurrentOpenHashMap.>newBuilder().build(); - nsToBundleMap.put(namespace, bundleSet); - brokerToNamespaceToBundleRange.put(broker, nsToBundleMap); + final var brokerToNamespaceToBundleRange = (BundleRangeCache) ownershipData; + brokerToNamespaceToBundleRange.add(broker, namespace, assignedBundleName); } /** @@ -560,10 +550,9 @@ private static void filterAntiAffinityGroupOwnedBrokers( if (ownershipData instanceof Set) { LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar, assignedNamespace, brokers, (Set>) ownershipData, brokerToDomainMap); - } else if (ownershipData instanceof ConcurrentOpenHashMap) { + } else if (ownershipData instanceof BundleRangeCache) { LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar, assignedNamespace, brokers, - (ConcurrentOpenHashMap>>) - ownershipData, brokerToDomainMap); + (BundleRangeCache) ownershipData, brokerToDomainMap); } else { throw new RuntimeException("Unknown ownershipData class type"); } @@ -580,11 +569,9 @@ private static boolean shouldAntiAffinityNamespaceUnload( if (ownershipData instanceof Set) { return LoadManagerShared.shouldAntiAffinityNamespaceUnload(namespace, bundle, currentBroker, pulsar, (Set>) ownershipData, candidate); - } else if (ownershipData instanceof ConcurrentOpenHashMap) { - return LoadManagerShared.shouldAntiAffinityNamespaceUnload(namespace, bundle, - currentBroker, pulsar, - (ConcurrentOpenHashMap>>) - ownershipData, candidate); + } else if (ownershipData instanceof BundleRangeCache) { + return LoadManagerShared.shouldAntiAffinityNamespaceUnload(namespace, currentBroker, pulsar, + (BundleRangeCache) ownershipData, candidate); } else { throw new RuntimeException("Unknown ownershipData class type"); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java index 62faa70bbcb76..358410f1f28e3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java @@ -59,7 +59,10 @@ public void setup() throws Exception { @AfterMethod(alwaysRun = true) void shutdown() throws Exception { - bkEnsemble.stop(); + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; + } log.info("---- bk stopped ----"); } @@ -115,7 +118,8 @@ public void anErrorShouldBeThrowBeforeLeaderElected() throws PulsarServerExcepti checkLookupException(tenant, namespace, client); // broker, webService and leaderElectionService is started, and elect is done; - leaderBrokerReference.set(new LeaderBroker(pulsar.getWebServiceAddress())); + leaderBrokerReference.set( + new LeaderBroker(pulsar.getBrokerId(), pulsar.getSafeWebServiceAddress())); Producer producer = client.newProducer() .topic("persistent://" + tenant + "/" + namespace + "/1p") @@ -129,10 +133,10 @@ private void checkLookupException(String tenant, String namespace, PulsarClient .topic("persistent://" + tenant + "/" + namespace + "/1p") .create(); } catch (PulsarClientException t) { - Assert.assertTrue(t instanceof PulsarClientException.LookupException); + Assert.assertTrue(t instanceof PulsarClientException.BrokerMetadataException + || t instanceof PulsarClientException.LookupException); Assert.assertTrue( - t.getMessage().contains( - "java.lang.IllegalStateException: The leader election has not yet been completed!")); + t.getMessage().contains("The leader election has not yet been completed")); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtilsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtilsTest.java new file mode 100644 index 0000000000000..ac21b30bdde51 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtilsTest.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mockStatic; +import static org.testng.Assert.assertEquals; +import java.nio.file.Files; +import java.util.stream.Stream; +import lombok.extern.slf4j.Slf4j; +import org.mockito.MockedStatic; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class LinuxInfoUtilsTest { + + /** + * simulate reading first line of /proc/stat to get total cpu usage. + */ + @Test + public void testGetCpuUsageForEntireHost(){ + try (MockedStatic filesMockedStatic = mockStatic(Files.class)) { + filesMockedStatic.when(() -> Files.lines(any())).thenReturn( + Stream.generate(() -> "cpu 317808 128 58637 2503692 7634 0 13472 0 0 0")); + long idle = 2503692 + 7634, total = 2901371; + LinuxInfoUtils.ResourceUsage resourceUsage = LinuxInfoUtils.ResourceUsage.builder() + .usage(total - idle) + .idle(idle) + .total(total).build(); + assertEquals(LinuxInfoUtils.getCpuUsageForEntireHost(), resourceUsage); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadBalancerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadBalancerTest.java index 68902c73e5717..acd918b55fe1b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadBalancerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadBalancerTest.java @@ -34,10 +34,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import lombok.SneakyThrows; @@ -66,6 +63,7 @@ import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; +import org.apache.pulsar.utils.ResourceUtils; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.ZooDefs; @@ -86,9 +84,15 @@ */ @Test(groups = "broker") public class LoadBalancerTest { - LocalBookkeeperEnsemble bkEnsemble; - ExecutorService executor = new ThreadPoolExecutor(5, 20, 30, TimeUnit.SECONDS, new LinkedBlockingQueue<>()); + public final static String CA_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem"); + public final static String BROKER_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem"); + public final static String BROKER_KEY_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem"); + + LocalBookkeeperEnsemble bkEnsemble; private static final Logger log = LoggerFactory.getLogger(LoadBalancerTest.class); @@ -124,7 +128,6 @@ void setup() throws Exception { config.setAdvertisedAddress("localhost"); config.setWebServicePort(Optional.of(0)); config.setBrokerServicePortTls(Optional.of(0)); - config.setWebServicePortTls(Optional.of(0)); config.setMetadataStoreUrl("zk:127.0.0.1:" + bkEnsemble.getZookeeperPort()); config.setBrokerShutdownTimeoutMs(0L); config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); @@ -132,6 +135,9 @@ void setup() throws Exception { config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName()); config.setAdvertisedAddress(localhost+i); config.setLoadBalancerEnabled(false); + config.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + config.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + config.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); pulsarServices[i] = new PulsarService(config); pulsarServices[i].start(); @@ -139,7 +145,7 @@ void setup() throws Exception { brokerNativeBrokerPorts[i] = pulsarServices[i].getBrokerListenPort().get(); brokerUrls[i] = new URL("http://127.0.0.1" + ":" + brokerWebServicePorts[i]); - lookupAddresses[i] = pulsarServices[i].getAdvertisedAddress() + ":" + pulsarServices[i].getListenPortHTTP().get(); + lookupAddresses[i] = pulsarServices[i].getBrokerId(); pulsarAdmins[i] = PulsarAdmin.builder().serviceHttpUrl(brokerUrls[i].toString()).build(); } @@ -151,16 +157,22 @@ void setup() throws Exception { @AfterMethod(alwaysRun = true) void shutdown() throws Exception { log.info("--- Shutting down ---"); - executor.shutdownNow(); for (int i = 0; i < BROKER_COUNT; i++) { - pulsarAdmins[i].close(); + if (pulsarAdmins[i] != null) { + pulsarAdmins[i].close(); + pulsarAdmins[i] = null; + } if (pulsarServices[i] != null) { pulsarServices[i].close(); + pulsarServices[i] = null; } } - bkEnsemble.stop(); + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; + } } private void loopUntilLeaderChangesForAllBroker(List activePulsars, LeaderBroker oldLeader) { @@ -410,7 +422,7 @@ public void testTopicAssignmentWithExistingBundles() throws Exception { double expectedMaxVariation = 10.0; for (int i = 0; i < BROKER_COUNT; i++) { long actualValue = 0; - String resourceId = "http://" + lookupAddresses[i]; + String resourceId = lookupAddresses[i]; if (namespaceOwner.containsKey(resourceId)) { actualValue = namespaceOwner.get(resourceId); } @@ -671,32 +683,26 @@ public void testNamespaceBundleAutoSplit() throws Exception { */ @Test public void testLeaderElection() throws Exception { - // this.pulsarServices is the reference to all of the PulsarServices - // it is used in order to clean up the resources - PulsarService[] allServices = new PulsarService[pulsarServices.length]; - System.arraycopy(pulsarServices, 0, allServices, 0, pulsarServices.length); for (int i = 0; i < BROKER_COUNT - 1; i++) { List activePulsar = new ArrayList<>(); List followerPulsar = new ArrayList<>(); LeaderBroker oldLeader = null; PulsarService leaderPulsar = null; for (int j = 0; j < BROKER_COUNT; j++) { - if (allServices[j].getState() != PulsarService.State.Closed) { - activePulsar.add(allServices[j]); - LeaderElectionService les = allServices[j].getLeaderElectionService(); + PulsarService pulsarService = pulsarServices[j]; + if (pulsarService.getState() != PulsarService.State.Closed) { + activePulsar.add(pulsarService); + LeaderElectionService les = pulsarService.getLeaderElectionService(); if (les.isLeader()) { oldLeader = les.getCurrentLeader().get(); - leaderPulsar = allServices[j]; - // set the refence to null in the main array, - // in order to prevent closing this PulsarService twice - pulsarServices[i] = null; + leaderPulsar = pulsarService; } else { - followerPulsar.add(allServices[j]); + followerPulsar.add(pulsarService); } } } // Make sure all brokers see the same leader - log.info("Old leader is : {}", oldLeader.getServiceUrl()); + log.info("Old leader is : {}", oldLeader.getBrokerId()); for (PulsarService pulsar : activePulsar) { log.info("Current leader for {} is : {}", pulsar.getWebServiceAddress(), pulsar.getLeaderElectionService().getCurrentLeader()); assertEquals(pulsar.getLeaderElectionService().readCurrentLeader().join(), Optional.of(oldLeader)); @@ -706,7 +712,7 @@ public void testLeaderElection() throws Exception { leaderPulsar.close(); loopUntilLeaderChangesForAllBroker(followerPulsar, oldLeader); LeaderBroker newLeader = followerPulsar.get(0).getLeaderElectionService().readCurrentLeader().join().get(); - log.info("New leader is : {}", newLeader.getServiceUrl()); + log.info("New leader is : {}", newLeader.getBrokerId()); Assert.assertNotEquals(newLeader, oldLeader); } } @@ -744,7 +750,7 @@ private void createNamespacePolicies(PulsarService pulsar) throws Exception { // set up policy that use this broker as secondary policyData = NamespaceIsolationData.builder() .namespaces(Collections.singletonList("pulsar/use/secondary-ns.*")) - .primary(Collections.singletonList(pulsarServices[0].getWebServiceAddress())) + .primary(Collections.singletonList(pulsarServices[0].getAdvertisedAddress())) .secondary(allExceptFirstBroker) .autoFailoverPolicy(AutoFailoverPolicyData.builder() .policyType(AutoFailoverPolicyType.min_available) @@ -756,7 +762,7 @@ private void createNamespacePolicies(PulsarService pulsar) throws Exception { // set up policy that do not use this broker (neither primary nor secondary) policyData = NamespaceIsolationData.builder() .namespaces(Collections.singletonList("pulsar/use/shared-ns.*")) - .primary(Collections.singletonList(pulsarServices[0].getWebServiceAddress())) + .primary(Collections.singletonList(pulsarServices[0].getAdvertisedAddress())) .secondary(allExceptFirstBroker) .autoFailoverPolicy(AutoFailoverPolicyData.builder() .policyType(AutoFailoverPolicyType.min_available) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerStrategyTest.java index c64c9950a95a9..53ddde8856c63 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerStrategyTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerStrategyTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.loadbalance; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; import java.lang.reflect.Field; import java.util.Arrays; @@ -34,6 +35,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.impl.AvgShedder; import org.apache.pulsar.broker.loadbalance.impl.LeastLongTermMessageRate; import org.apache.pulsar.broker.loadbalance.impl.LeastResourceUsageWithWeight; import org.apache.pulsar.broker.loadbalance.impl.RoundRobinBrokerSelector; @@ -47,6 +49,47 @@ @Test(groups = "broker") public class ModularLoadManagerStrategyTest { + public void testAvgShedderWithPreassignedBroker() throws Exception { + ModularLoadManagerStrategy strategy = new AvgShedder(); + Field field = AvgShedder.class.getDeclaredField("bundleBrokerMap"); + field.setAccessible(true); + Map bundleBrokerMap = (Map) field.get(strategy); + BundleData bundleData = new BundleData(); + // assign bundle to broker1 in bundleBrokerMap. + bundleBrokerMap.put(bundleData, "1"); + assertEquals(strategy.selectBroker(Set.of("1", "2", "3"), bundleData, null, null), Optional.of("1")); + assertEquals(bundleBrokerMap.get(bundleData), "1"); + + // remove broker1 in candidates, only broker2 is candidate. + assertEquals(strategy.selectBroker(Set.of("2"), bundleData, null, null), Optional.of("2")); + assertEquals(bundleBrokerMap.get(bundleData), "2"); + } + + public void testAvgShedderWithoutPreassignedBroker() throws Exception { + ModularLoadManagerStrategy strategy = new AvgShedder(); + Field field = AvgShedder.class.getDeclaredField("bundleBrokerMap"); + field.setAccessible(true); + Map bundleBrokerMap = (Map) field.get(strategy); + BundleData bundleData = new BundleData(); + Set candidates = new HashSet<>(); + candidates.add("1"); + candidates.add("2"); + candidates.add("3"); + + // select broker from candidates randomly. + Optional selectedBroker = strategy.selectBroker(candidates, bundleData, null, null); + assertTrue(selectedBroker.isPresent()); + assertTrue(candidates.contains(selectedBroker.get())); + assertEquals(bundleBrokerMap.get(bundleData), selectedBroker.get()); + + // remove original broker in candidates + candidates.remove(selectedBroker.get()); + selectedBroker = strategy.selectBroker(candidates, bundleData, null, null); + assertTrue(selectedBroker.isPresent()); + assertTrue(candidates.contains(selectedBroker.get())); + assertEquals(bundleBrokerMap.get(bundleData), selectedBroker.get()); + } + // Test that least long term message rate works correctly. public void testLeastLongTermMessageRate() { BundleData bundleData = new BundleData(); @@ -68,6 +111,9 @@ public void testLeastLongTermMessageRate() { assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("2")); brokerData2.getLocalData().setCpu(new ResourceUsage(90, 100)); assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("3")); + // disable considering cpu usage to avoid broker2 being overloaded. + conf.setLoadBalancerCPUResourceWeight(0); + assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("2")); } // Test that least resource usage with weight works correctly. @@ -88,8 +134,8 @@ public void testLeastResourceUsageWithWeight() { conf.setLoadBalancerCPUResourceWeight(1.0); conf.setLoadBalancerMemoryResourceWeight(0.1); conf.setLoadBalancerDirectMemoryResourceWeight(0.1); - conf.setLoadBalancerBandwithInResourceWeight(1.0); - conf.setLoadBalancerBandwithOutResourceWeight(1.0); + conf.setLoadBalancerBandwidthInResourceWeight(1.0); + conf.setLoadBalancerBandwidthOutResourceWeight(1.0); conf.setLoadBalancerHistoryResourcePercentage(0.5); conf.setLoadBalancerAverageResourceUsageDifferenceThresholdPercentage(5); @@ -167,8 +213,8 @@ public void testLeastResourceUsageWithWeightWithArithmeticException() conf.setLoadBalancerCPUResourceWeight(1.0); conf.setLoadBalancerMemoryResourceWeight(0.1); conf.setLoadBalancerDirectMemoryResourceWeight(0.1); - conf.setLoadBalancerBandwithInResourceWeight(1.0); - conf.setLoadBalancerBandwithOutResourceWeight(1.0); + conf.setLoadBalancerBandwidthInResourceWeight(1.0); + conf.setLoadBalancerBandwidthOutResourceWeight(1.0); conf.setLoadBalancerHistoryResourcePercentage(0.5); conf.setLoadBalancerAverageResourceUsageDifferenceThresholdPercentage(5); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionTest.java index 5c840129dd8d5..32f3acf42142e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/MultiBrokerLeaderElectionTest.java @@ -20,6 +20,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -36,14 +37,24 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.MultiBrokerTestZKBaseTest; import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.client.admin.Lookup; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.impl.LookupService; +import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.common.naming.TopicName; import org.awaitility.Awaitility; import org.testng.annotations.Test; @Slf4j @Test(groups = "broker") public class MultiBrokerLeaderElectionTest extends MultiBrokerTestZKBaseTest { + public MultiBrokerLeaderElectionTest() { + super(); + this.isTcpLookup = true; + } + @Override protected int numberOfAdditionalBrokers() { return 9; @@ -88,39 +99,82 @@ public void shouldAllBrokersBeAbleToGetTheLeader() { }); } - @Test - public void shouldProvideConsistentAnswerToTopicLookups() + @Test(timeOut = 120000L) + public void shouldProvideConsistentAnswerToTopicLookupsUsingAdminApi() throws PulsarAdminException, ExecutionException, InterruptedException { - String topicNameBase = "persistent://public/default/lookuptest" + UUID.randomUUID() + "-"; + String namespace = "public/ns" + UUID.randomUUID(); + admin.namespaces().createNamespace(namespace, 256); + String topicNameBase = "persistent://" + namespace + "/lookuptest-"; List topicNames = IntStream.range(0, 500).mapToObj(i -> topicNameBase + i) .collect(Collectors.toList()); List allAdmins = getAllAdmins(); - @Cleanup("shutdown") + @Cleanup("shutdownNow") ExecutorService executorService = Executors.newFixedThreadPool(allAdmins.size()); List>> resultFutures = new ArrayList<>(); - String leaderBrokerUrl = admin.brokers().getLeaderBroker().getServiceUrl(); - log.info("LEADER is {}", leaderBrokerUrl); // use Phaser to increase the chances of a race condition by triggering all threads once - // they are waiting just before the lookupTopic call + // they are waiting just before each lookupTopic call + @Cleanup("forceTermination") final Phaser phaser = new Phaser(1); for (PulsarAdmin brokerAdmin : allAdmins) { - if (!leaderBrokerUrl.equals(brokerAdmin.getServiceUrl())) { - phaser.register(); - log.info("Doing lookup to broker {}", brokerAdmin.getServiceUrl()); - resultFutures.add(executorService.submit(() -> { - phaser.arriveAndAwaitAdvance(); - return topicNames.stream().map(topicName -> { - try { - return brokerAdmin.lookups().lookupTopic(topicName); - } catch (PulsarAdminException e) { - log.error("Error looking up topic {} in {}", topicName, brokerAdmin.getServiceUrl()); - throw new RuntimeException(e); - } - }).collect(Collectors.toList()); - })); + phaser.register(); + Lookup lookups = brokerAdmin.lookups(); + log.info("Doing lookup to broker {}", brokerAdmin.getServiceUrl()); + resultFutures.add(executorService.submit(() -> topicNames.stream().map(topicName -> { + phaser.arriveAndAwaitAdvance(); + try { + return lookups.lookupTopic(topicName); + } catch (PulsarAdminException e) { + log.error("Error looking up topic {} in {}", topicName, brokerAdmin.getServiceUrl()); + throw new RuntimeException(e); + } + }).collect(Collectors.toList()))); + } + phaser.arriveAndDeregister(); + List firstResult = null; + for (Future> resultFuture : resultFutures) { + List result = resultFuture.get(); + if (firstResult == null) { + firstResult = result; + } else { + assertEquals(result, firstResult, "The lookup results weren't consistent."); } } - phaser.arriveAndAwaitAdvance(); + } + + @Test(timeOut = 60000L) + public void shouldProvideConsistentAnswerToTopicLookupsUsingClient() + throws PulsarAdminException, ExecutionException, InterruptedException { + String namespace = "public/ns" + UUID.randomUUID(); + admin.namespaces().createNamespace(namespace, 256); + String topicNameBase = "persistent://" + namespace + "/lookuptest-"; + List topicNames = IntStream.range(0, 500).mapToObj(i -> topicNameBase + i) + .collect(Collectors.toList()); + List allClients = getAllClients(); + @Cleanup("shutdownNow") + ExecutorService executorService = Executors.newFixedThreadPool(allClients.size()); + List>> resultFutures = new ArrayList<>(); + // use Phaser to increase the chances of a race condition by triggering all threads once + // they are waiting just before each lookupTopic call + @Cleanup("forceTermination") + final Phaser phaser = new Phaser(1); + for (PulsarClient brokerClient : allClients) { + phaser.register(); + String serviceUrl = ((PulsarClientImpl) brokerClient).getConfiguration().getServiceUrl(); + LookupService lookupService = ((PulsarClientImpl) brokerClient).getLookup(); + log.info("Doing lookup to broker {}", serviceUrl); + resultFutures.add(executorService.submit(() -> topicNames.stream().map(topicName -> { + phaser.arriveAndAwaitAdvance(); + try { + InetSocketAddress logicalAddress = + lookupService.getBroker(TopicName.get(topicName)).get().getLogicalAddress(); + return logicalAddress.getHostString() + ":" + logicalAddress.getPort(); + } catch (InterruptedException | ExecutionException e) { + log.error("Error looking up topic {} in {}", topicName, serviceUrl); + throw new RuntimeException(e); + } + }).collect(Collectors.toList()))); + } + phaser.arriveAndDeregister(); List firstResult = null; for (Future> resultFuture : resultFutures) { List result = resultFuture.get(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java index 28dde8b7f559d..31c8c9f3bccc1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java @@ -20,6 +20,7 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import com.google.common.io.Resources; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Optional; @@ -37,6 +38,13 @@ @Test(groups = "broker") public class SimpleBrokerStartTest { + final static String caCertPath = Resources.getResource("certificate-authority/certs/ca.cert.pem") + .getPath(); + final static String brokerCertPath = + Resources.getResource("certificate-authority/server-keys/broker.cert.pem").getPath(); + final static String brokerKeyPath = + Resources.getResource("certificate-authority/server-keys/broker.key-pk8.pem").getPath(); + public void testHasNICSpeed() throws Exception { if (!LinuxInfoUtils.isLinux()) { return; @@ -57,6 +65,9 @@ public void testHasNICSpeed() throws Exception { config.setBrokerServicePortTls(Optional.of(0)); config.setWebServicePortTls(Optional.of(0)); config.setAdvertisedAddress("localhost"); + config.setTlsTrustCertsFilePath(caCertPath); + config.setTlsCertificateFilePath(brokerCertPath); + config.setTlsKeyFilePath(brokerKeyPath); boolean hasNicSpeeds = LinuxInfoUtils.checkHasNicSpeeds(); if (hasNicSpeeds) { @Cleanup @@ -85,6 +96,9 @@ public void testNoNICSpeed() throws Exception { config.setBrokerServicePortTls(Optional.of(0)); config.setWebServicePortTls(Optional.of(0)); config.setAdvertisedAddress("localhost"); + config.setTlsTrustCertsFilePath(caCertPath); + config.setTlsCertificateFilePath(brokerCertPath); + config.setTlsKeyFilePath(brokerKeyPath); boolean hasNicSpeeds = LinuxInfoUtils.checkHasNicSpeeds(); if (!hasNicSpeeds) { @Cleanup diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java index c4898786e3e03..1e91230559b0a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java @@ -20,6 +20,7 @@ import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgsRecordingInvocations; import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -41,7 +42,9 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; import org.apache.commons.lang3.SystemUtils; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; @@ -55,12 +58,16 @@ import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceUnit; import org.apache.pulsar.client.admin.BrokerStats; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.policies.data.AutoFailoverPolicyData; import org.apache.pulsar.common.policies.data.AutoFailoverPolicyType; +import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.NamespaceIsolationData; import org.apache.pulsar.common.policies.data.ResourceQuota; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.policies.impl.NamespaceIsolationPolicies; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.metadata.api.MetadataStoreException.BadVersionException; @@ -70,7 +77,9 @@ import org.apache.pulsar.policies.data.loadbalancer.ResourceUnitRanking; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; +import org.apache.pulsar.utils.ResourceUtils; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -78,6 +87,14 @@ @Slf4j @Test(groups = "broker") public class SimpleLoadManagerImplTest { + + public final static String CA_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem"); + public final static String BROKER_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem"); + public final static String BROKER_KEY_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem"); + LocalBookkeeperEnsemble bkEnsemble; URL url1; @@ -92,8 +109,14 @@ public class SimpleLoadManagerImplTest { BrokerStats brokerStatsClient2; String primaryHost; + + String primaryTlsHost; + String secondaryHost; + private String defaultNamespace; + private String defaultTenant; + ExecutorService executor = new ThreadPoolExecutor(5, 20, 30, TimeUnit.SECONDS, new LinkedBlockingQueue<>()); @BeforeMethod @@ -107,14 +130,17 @@ void setup() throws Exception { ServiceConfiguration config1 = new ServiceConfiguration(); config1.setClusterName("use"); config1.setWebServicePort(Optional.of(0)); + config1.setWebServicePortTls(Optional.of(0)); config1.setMetadataStoreUrl("zk:127.0.0.1:" + bkEnsemble.getZookeeperPort()); config1.setBrokerShutdownTimeoutMs(0L); config1.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); config1.setBrokerServicePort(Optional.of(0)); config1.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName()); config1.setBrokerServicePortTls(Optional.of(0)); - config1.setWebServicePortTls(Optional.of(0)); config1.setAdvertisedAddress("localhost"); + config1.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + config1.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + config1.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); pulsar1 = new PulsarService(config1); pulsar1.start(); @@ -122,11 +148,13 @@ void setup() throws Exception { admin1 = PulsarAdmin.builder().serviceHttpUrl(url1.toString()).build(); brokerStatsClient1 = admin1.brokerStats(); primaryHost = pulsar1.getWebServiceAddress(); + primaryTlsHost = pulsar1.getWebServiceAddressTls(); // Start broker 2 ServiceConfiguration config2 = new ServiceConfiguration(); config2.setClusterName("use"); config2.setWebServicePort(Optional.of(0)); + config2.setWebServicePortTls(Optional.of(0)); config2.setMetadataStoreUrl("zk:127.0.0.1:" + bkEnsemble.getZookeeperPort()); config2.setBrokerShutdownTimeoutMs(0L); config2.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); @@ -134,6 +162,9 @@ void setup() throws Exception { config2.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName()); config2.setBrokerServicePortTls(Optional.of(0)); config2.setWebServicePortTls(Optional.of(0)); + config2.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + config2.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + config2.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); config2.setAdvertisedAddress("localhost"); pulsar2 = new PulsarService(config2); pulsar2.start(); @@ -143,20 +174,40 @@ void setup() throws Exception { brokerStatsClient2 = admin2.brokerStats(); secondaryHost = pulsar2.getWebServiceAddress(); Thread.sleep(100); + + setupClusters(); } @AfterMethod(alwaysRun = true) void shutdown() throws Exception { log.info("--- Shutting down ---"); - executor.shutdownNow(); + if (executor != null) { + executor.shutdownNow(); + executor = null; + } - admin1.close(); - admin2.close(); + if (admin1 != null) { + admin1.close(); + admin1 = null; + } + if (admin2 != null) { + admin2.close(); + admin2 = null; + } - pulsar2.close(); - pulsar1.close(); + if (pulsar2 != null) { + pulsar2.close(); + pulsar2 = null; + } + if (pulsar1 != null) { + pulsar1.close(); + pulsar1 = null; + } - bkEnsemble.stop(); + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; + } } private void createNamespacePolicies(PulsarService pulsar) throws Exception { @@ -189,6 +240,7 @@ private void createNamespacePolicies(PulsarService pulsar) throws Exception { @Test public void testBasicBrokerSelection() throws Exception { + @Cleanup("stop") SimpleLoadManagerImpl loadManager = new SimpleLoadManagerImpl(pulsar1); PulsarResourceDescription rd = new PulsarResourceDescription(); rd.put("memory", new ResourceUsage(1024, 4096)); @@ -196,7 +248,7 @@ public void testBasicBrokerSelection() throws Exception { rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024)); rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024)); - ResourceUnit ru1 = new SimpleResourceUnit("http://prod2-broker7.messaging.usw.example.com:8080", rd); + ResourceUnit ru1 = new SimpleResourceUnit("prod2-broker7.messaging.usw.example.com:8080", rd); Set rus = new HashSet<>(); rus.add(ru1); LoadRanker lr = new ResourceAvailabilityRanker(); @@ -224,6 +276,7 @@ private void setObjectField(Class objClass, Object objInstance, String fieldN @Test public void testPrimary() throws Exception { createNamespacePolicies(pulsar1); + @Cleanup("stop") SimpleLoadManagerImpl loadManager = new SimpleLoadManagerImpl(pulsar1); PulsarResourceDescription rd = new PulsarResourceDescription(); rd.put("memory", new ResourceUsage(1024, 4096)); @@ -231,15 +284,15 @@ public void testPrimary() throws Exception { rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024)); rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024)); - ResourceUnit ru1 = new SimpleResourceUnit( - "http://" + pulsar1.getAdvertisedAddress() + ":" + pulsar1.getConfiguration().getWebServicePort().get(), rd); + ResourceUnit ru1 = new SimpleResourceUnit(pulsar1.getBrokerId(), rd); Set rus = new HashSet<>(); rus.add(ru1); LoadRanker lr = new ResourceAvailabilityRanker(); // inject the load report and rankings Map loadReports = new HashMap<>(); - org.apache.pulsar.policies.data.loadbalancer.LoadReport loadReport = new org.apache.pulsar.policies.data.loadbalancer.LoadReport(); + org.apache.pulsar.policies.data.loadbalancer.LoadReport loadReport = + new org.apache.pulsar.policies.data.loadbalancer.LoadReport(); loadReport.setSystemResourceUsage(new SystemResourceUsage()); loadReports.put(ru1, loadReport); setObjectField(SimpleLoadManagerImpl.class, loadManager, "currentLoadReports", loadReports); @@ -254,16 +307,15 @@ public void testPrimary() throws Exception { sortedRankingsInstance.get().put(lr.getRank(rd), rus); setObjectField(SimpleLoadManagerImpl.class, loadManager, "sortedRankings", sortedRankingsInstance); - ResourceUnit found = loadManager - .getLeastLoaded(NamespaceName.get("pulsar/use/primary-ns.10")).get(); - // broker is not active so found should be null + ResourceUnit found = loadManager.getLeastLoaded(NamespaceName.get("pulsar/use/primary-ns.10")).get(); + // TODO: this test doesn't make sense. This was the original assertion. assertNotEquals(found, null, "did not find a broker when expected one to be found"); - } @Test(enabled = false) public void testPrimarySecondary() throws Exception { createNamespacePolicies(pulsar1); + @Cleanup("stop") SimpleLoadManagerImpl loadManager = new SimpleLoadManagerImpl(pulsar1); PulsarResourceDescription rd = new PulsarResourceDescription(); @@ -272,7 +324,7 @@ public void testPrimarySecondary() throws Exception { rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024)); rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024)); - ResourceUnit ru1 = new SimpleResourceUnit("http://prod2-broker7.messaging.usw.example.com:8080", rd); + ResourceUnit ru1 = new SimpleResourceUnit("prod2-broker7.messaging.usw.example.com:8080", rd); Set rus = new HashSet<>(); rus.add(ru1); LoadRanker lr = new ResourceAvailabilityRanker(); @@ -334,6 +386,7 @@ public void testLoadReportParsing() throws Exception { @Test(enabled = true) public void testDoLoadShedding() throws Exception { + @Cleanup("stop") SimpleLoadManagerImpl loadManager = spyWithClassAndConstructorArgsRecordingInvocations(SimpleLoadManagerImpl.class, pulsar1); PulsarResourceDescription rd = new PulsarResourceDescription(); rd.put("memory", new ResourceUsage(1024, 4096)); @@ -341,8 +394,8 @@ public void testDoLoadShedding() throws Exception { rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024)); rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024)); - ResourceUnit ru1 = new SimpleResourceUnit("http://pulsar-broker1.com:8080", rd); - ResourceUnit ru2 = new SimpleResourceUnit("http://pulsar-broker2.com:8080", rd); + ResourceUnit ru1 = new SimpleResourceUnit("pulsar-broker1.com:8080", rd); + ResourceUnit ru2 = new SimpleResourceUnit("pulsar-broker2.com:8080", rd); Set rus = new HashSet<>(); rus.add(ru1); rus.add(ru2); @@ -395,14 +448,14 @@ public void testEvenBundleDistribution() throws Exception { final SimpleLoadManagerImpl loadManager = (SimpleLoadManagerImpl) pulsar1.getLoadManager().get(); for (final NamespaceBundle bundle : bundles) { - if (loadManager.getLeastLoaded(bundle).get().getResourceId().equals(primaryHost)) { + if (loadManager.getLeastLoaded(bundle).get().getResourceId().equals(pulsar1.getBrokerId())) { ++numAssignedToPrimary; } else { ++numAssignedToSecondary; } // Check that number of assigned bundles are equivalent when an even number have been assigned. if ((numAssignedToPrimary + numAssignedToSecondary) % 2 == 0) { - assert (numAssignedToPrimary == numAssignedToSecondary); + assertEquals(numAssignedToPrimary, numAssignedToSecondary); } } } @@ -453,7 +506,21 @@ public void testTask() throws Exception { task1.run(); verify(loadManager, times(1)).writeResourceQuotasToZooKeeper(); - LoadSheddingTask task2 = new LoadSheddingTask(atomicLoadManager, null, null); + LoadSheddingTask task2 = new LoadSheddingTask(atomicLoadManager, null, null, null); + task2.run(); + verify(loadManager, times(1)).doLoadShedding(); + } + + @Test + public void testMetadataServiceNotAvailable() { + LoadManager loadManager = mock(LoadManager.class); + AtomicReference atomicLoadManager = new AtomicReference<>(loadManager); + ManagedLedgerFactoryImpl factory = mock(ManagedLedgerFactoryImpl.class); + doReturn(false).when(factory).isMetadataServiceAvailable(); + LoadSheddingTask task2 = new LoadSheddingTask(atomicLoadManager, null, null, factory); + task2.run(); + verify(loadManager, times(0)).doLoadShedding(); + doReturn(true).when(factory).isMetadataServiceAvailable(); task2.run(); verify(loadManager, times(1)).doLoadShedding(); } @@ -475,4 +542,34 @@ public void testUsage() { assertEquals(usage.getBandwidthIn().usage, usageLimit); } + @Test + public void testGetWebSerUrl() throws PulsarAdminException { + String webServiceUrl = admin1.brokerStats().getLoadReport().getWebServiceUrl(); + Assert.assertEquals(webServiceUrl, pulsar1.getWebServiceAddress()); + + String webServiceUrl2 = admin2.brokerStats().getLoadReport().getWebServiceUrl(); + Assert.assertEquals(webServiceUrl2, pulsar2.getWebServiceAddress()); + } + + @Test + public void testRedirectOwner() throws PulsarAdminException { + final String topicName = "persistent://" + defaultNamespace + "/" + "test-topic"; + admin1.topics().createNonPartitionedTopic(topicName); + TopicStats stats = admin1.topics().getStats(topicName); + Assert.assertNotNull(stats); + + TopicStats stats2 = admin2.topics().getStats(topicName); + Assert.assertNotNull(stats2); + } + + private void setupClusters() throws PulsarAdminException { + admin1.clusters().createCluster("use", ClusterData.builder().serviceUrl(pulsar1.getWebServiceAddress()) + .brokerServiceUrl(pulsar1.getBrokerServiceUrl()).build()); + TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("use")); + defaultTenant = "prop-xyz"; + admin1.tenants().createTenant(defaultTenant, tenantInfo); + defaultNamespace = defaultTenant + "/ns1"; + admin1.namespaces().createNamespace(defaultNamespace, Set.of("use")); + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java index 3469dbe5a5499..cd653a964be36 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java @@ -32,17 +32,18 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import lombok.Cleanup; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.loadbalance.AntiAffinityNamespaceGroupTest; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; import org.apache.pulsar.broker.loadbalance.extensions.filter.AntiAffinityGroupPolicyFilter; import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper; +import org.apache.pulsar.broker.namespace.LookupOptions; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.naming.ServiceUnitId; -import org.testcontainers.shaded.org.apache.commons.lang3.reflect.FieldUtils; import org.testng.annotations.Test; @Test(groups = "broker") @@ -61,7 +62,8 @@ protected String getLoadManagerClassName() { protected String selectBroker(ServiceUnitId serviceUnit, Object loadManager) { try { - return ((ExtensibleLoadManagerImpl) loadManager).assign(Optional.empty(), serviceUnit).get() + return ((ExtensibleLoadManagerImpl) loadManager) + .assign(Optional.empty(), serviceUnit, LookupOptions.builder().build()).get() .get().getPulsarServiceUrl(); } catch (Throwable e) { throw new RuntimeException(e); @@ -109,6 +111,7 @@ public void testAntiAffinityGroupPolicyFilter() final String antiAffinityEnabledNameSpace = namespace + nsSuffix; admin.namespaces().createNamespace(antiAffinityEnabledNameSpace); admin.namespaces().setNamespaceAntiAffinityGroup(antiAffinityEnabledNameSpace, namespaceAntiAffinityGroup); + @Cleanup PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsar.getSafeWebServiceAddress()).build(); @Cleanup Producer producer = pulsarClient.newProducer().topic( @@ -130,8 +133,8 @@ public void testAntiAffinityGroupPolicyFilter() doReturn(namespace + "/" + bundle).when(namespaceBundle).toString(); var expected = new HashMap<>(brokers); - var actual = antiAffinityGroupPolicyFilter.filter( - brokers, namespaceBundle, context); + var actual = antiAffinityGroupPolicyFilter.filterAsync( + brokers, namespaceBundle, context).get(); assertEquals(actual, expected); doReturn(antiAffinityEnabledNameSpace + "/" + bundle).when(namespaceBundle).toString(); @@ -141,8 +144,8 @@ public void testAntiAffinityGroupPolicyFilter() var srcBroker = serviceUnitStateChannel.getOwnerAsync(namespaceBundle.toString()) .get(5, TimeUnit.SECONDS).get(); expected.remove(srcBroker); - actual = antiAffinityGroupPolicyFilter.filter( - brokers, namespaceBundle, context); + actual = antiAffinityGroupPolicyFilter.filterAsync( + brokers, namespaceBundle, context).get(); assertEquals(actual, expected); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryIntegrationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryIntegrationTest.java new file mode 100644 index 0000000000000..e975671fa12e8 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryIntegrationTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions; + +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.util.PortManager; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.LoadManager; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.awaitility.Awaitility; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class BrokerRegistryIntegrationTest { + + private static final String clusterName = "test"; + private final int zkPort = PortManager.nextFreePort(); + private final LocalBookkeeperEnsemble bk = new LocalBookkeeperEnsemble(2, zkPort, PortManager::nextFreePort); + private PulsarService pulsar; + private BrokerRegistry brokerRegistry; + private String brokerMetadataPath; + + @BeforeClass + protected void setup() throws Exception { + bk.start(); + pulsar = new PulsarService(brokerConfig()); + pulsar.start(); + final var admin = pulsar.getAdminClient(); + admin.clusters().createCluster(clusterName, ClusterData.builder().build()); + admin.tenants().createTenant("public", TenantInfo.builder() + .allowedClusters(Collections.singleton(clusterName)).build()); + admin.namespaces().createNamespace("public/default"); + brokerRegistry = ((ExtensibleLoadManagerWrapper) pulsar.getLoadManager().get()).get().getBrokerRegistry(); + brokerMetadataPath = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + pulsar.getBrokerId(); + } + + @AfterClass(alwaysRun = true) + protected void cleanup() throws Exception { + final var startMs = System.currentTimeMillis(); + if (pulsar != null) { + pulsar.close(); + } + final var elapsedMs = System.currentTimeMillis() - startMs; + bk.stop(); + if (elapsedMs > 5000) { + throw new RuntimeException("Broker took " + elapsedMs + "ms to close"); + } + } + + @Test + public void testRecoverFromNodeDeletion() throws Exception { + // Simulate the case that the node was somehow deleted (e.g. by session timeout) + Awaitility.await().atMost(Duration.ofSeconds(3)).untilAsserted(() -> Assert.assertEquals( + brokerRegistry.getAvailableBrokersAsync().join(), List.of(pulsar.getBrokerId()))); + pulsar.getLocalMetadataStore().delete(brokerMetadataPath, Optional.empty()); + Awaitility.await().atMost(Duration.ofSeconds(10)).untilAsserted(() -> Assert.assertEquals( + brokerRegistry.getAvailableBrokersAsync().join(), List.of(pulsar.getBrokerId()))); + + // If the node is deleted by unregister(), it should not recreate the path + brokerRegistry.unregister(); + Thread.sleep(3000); + Assert.assertTrue(brokerRegistry.getAvailableBrokersAsync().get().isEmpty()); + + // Restore the normal state + brokerRegistry.registerAsync().get(); + Assert.assertEquals(brokerRegistry.getAvailableBrokersAsync().get(), List.of(pulsar.getBrokerId())); + } + + @Test + public void testRegisterAgain() throws Exception { + Awaitility.await().atMost(Duration.ofSeconds(3)).untilAsserted(() -> Assert.assertEquals( + brokerRegistry.getAvailableBrokersAsync().join(), List.of(pulsar.getBrokerId()))); + final var metadataStore = pulsar.getLocalMetadataStore(); + final var oldResult = metadataStore.get(brokerMetadataPath).get().orElseThrow(); + log.info("Old result: {} {}", new String(oldResult.getValue()), oldResult.getStat().getVersion()); + brokerRegistry.registerAsync().get(); + + Awaitility.await().atMost(Duration.ofSeconds(3)).untilAsserted(() -> { + final var newResult = metadataStore.get(brokerMetadataPath).get().orElseThrow(); + log.info("New result: {} {}", new String(newResult.getValue()), newResult.getStat().getVersion()); + Assert.assertTrue(newResult.getStat().getVersion() > oldResult.getStat().getVersion()); + Assert.assertEquals(newResult.getValue(), oldResult.getValue()); + }); + } + + protected ServiceConfiguration brokerConfig() { + final var config = new ServiceConfiguration(); + config.setClusterName(clusterName); + config.setAdvertisedAddress("localhost"); + config.setBrokerServicePort(Optional.of(0)); + config.setWebServicePort(Optional.of(0)); + config.setMetadataStoreUrl("zk:127.0.0.1:" + bk.getZookeeperPort()); + config.setManagedLedgerDefaultWriteQuorum(1); + config.setManagedLedgerDefaultAckQuorum(1); + config.setManagedLedgerDefaultEnsembleSize(1); + config.setDefaultNumberOfNamespaceBundles(16); + config.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); + config.setLoadBalancerDebugModeEnabled(true); + config.setBrokerShutdownTimeoutMs(100); + return config; + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryMetadataStoreIntegrationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryMetadataStoreIntegrationTest.java new file mode 100644 index 0000000000000..3e01b1fad0f21 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryMetadataStoreIntegrationTest.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions; + +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateMetadataStoreTableViewImpl; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class BrokerRegistryMetadataStoreIntegrationTest extends BrokerRegistryIntegrationTest { + + @Override + protected ServiceConfiguration brokerConfig() { + final var config = super.brokerConfig(); + config.setLoadManagerServiceUnitStateTableViewClassName( + ServiceUnitStateMetadataStoreTableViewImpl.class.getName()); + return config; + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java index 26986a494f0be..941d0e4cbc3a0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java @@ -19,13 +19,15 @@ package org.apache.pulsar.broker.loadbalance.extensions; import static org.apache.pulsar.broker.loadbalance.LoadManager.LOADBALANCE_BROKERS_ROOT; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - import java.time.Duration; import java.util.HashSet; import java.util.List; @@ -37,6 +39,7 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.BiConsumer; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -49,12 +52,13 @@ import org.apache.pulsar.common.naming.ServiceUnitId; import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.Notification; import org.apache.pulsar.metadata.api.NotificationType; import org.apache.pulsar.policies.data.loadbalancer.LoadManagerReport; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.awaitility.Awaitility; -import org.testcontainers.shaded.org.awaitility.reflect.WhiteboxImpl; +import org.awaitility.reflect.WhiteboxImpl; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; @@ -76,7 +80,7 @@ public class BrokerRegistryTest { private LocalBookkeeperEnsemble bkEnsemble; - // Make sure the load manager don't register itself to `/loadbalance/brokers/{lookupServiceAddress}` + // Make sure the load manager don't register itself to `/loadbalance/brokers/{brokerId}`. public static class MockLoadManager implements LoadManager { @Override @@ -193,8 +197,14 @@ private BrokerRegistryImpl createBrokerRegistryImpl(PulsarService pulsar) { @AfterClass(alwaysRun = true) void shutdown() throws Exception { - executor.shutdownNow(); - bkEnsemble.stop(); + if (executor != null) { + executor.shutdownNow(); + executor = null; + } + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; + } } @AfterMethod(alwaysRun = true) @@ -286,24 +296,20 @@ public void testRegisterAndLookup() throws Exception { } @Test - public void testRegisterFailWithSameBrokerId() throws Exception { + public void testRegisterWithSameBrokerId() throws Exception { PulsarService pulsar1 = createPulsarService(); PulsarService pulsar2 = createPulsarService(); pulsar1.start(); pulsar2.start(); - doReturn(pulsar1.getLookupServiceAddress()).when(pulsar2).getLookupServiceAddress(); + doReturn(pulsar1.getBrokerId()).when(pulsar2).getBrokerId(); BrokerRegistryImpl brokerRegistry1 = createBrokerRegistryImpl(pulsar1); BrokerRegistryImpl brokerRegistry2 = createBrokerRegistryImpl(pulsar2); brokerRegistry1.start(); - try { - brokerRegistry2.start(); - fail(); - } catch (Exception ex) { - log.info("Broker registry start failed.", ex); - assertTrue(ex instanceof PulsarServerException); - assertTrue(ex.getMessage().contains("LockBusyException")); - } + brokerRegistry2.start(); + + pulsar1.close(); + pulsar2.close(); } @Test @@ -331,7 +337,7 @@ public void testCloseRegister() throws Exception { assertEquals(getState(brokerRegistry), BrokerRegistryImpl.State.Started); // Check state after re-register. - brokerRegistry.register(); + brokerRegistry.registerAsync().get(); assertEquals(getState(brokerRegistry), BrokerRegistryImpl.State.Registered); // Check state after close. @@ -395,8 +401,36 @@ public void testKeyPath() { assertEquals(keyPath, LOADBALANCE_BROKERS_ROOT + "/brokerId"); } - public BrokerRegistryImpl.State getState(BrokerRegistryImpl brokerRegistry) { - return WhiteboxImpl.getInternalState(brokerRegistry, BrokerRegistryImpl.State.class); + @Test + public void testRegisterAsyncTimeout() throws Exception { + var pulsar1 = createPulsarService(); + pulsar1.start(); + pulsar1.getConfiguration().setMetadataStoreOperationTimeoutSeconds(1); + var metadataCache = mock(MetadataCache.class); + var brokerRegistry = new BrokerRegistryImpl(pulsar1, metadataCache); + + // happy case + doReturn(CompletableFuture.completedFuture(null)).when(metadataCache).put(any(), any(), any()); + brokerRegistry.start(); + + // unhappy case (timeout) + doAnswer(invocationOnMock -> { + return CompletableFuture.supplyAsync(() -> null, CompletableFuture.delayedExecutor(5, TimeUnit.SECONDS)); + }).when(metadataCache).put(any(), any(), any()); + try { + brokerRegistry.registerAsync().join(); + } catch (Exception e) { + assertTrue(e.getCause() instanceof TimeoutException); + } + + // happy case again + doReturn(CompletableFuture.completedFuture(null)).when(metadataCache).put(any(), any(), any()); + brokerRegistry.registerAsync().join(); + } + + + private static BrokerRegistryImpl.State getState(BrokerRegistryImpl brokerRegistry) { + return brokerRegistry.state.get(); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java new file mode 100644 index 0000000000000..fa63ce566c603 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class ExtensibleLoadManagerCloseTest { + + private static final String clusterName = "test"; + private final List brokers = new ArrayList<>(); + private LocalBookkeeperEnsemble bk; + + @BeforeClass(alwaysRun = true) + public void setup() throws Exception { + bk = new LocalBookkeeperEnsemble(1, 0, () -> 0); + bk.start(); + } + + private void setupBrokers(int numBrokers) throws Exception { + brokers.clear(); + for (int i = 0; i < numBrokers; i++) { + final var broker = new PulsarService(brokerConfig()); + broker.start(); + brokers.add(broker); + } + final var admin = brokers.get(0).getAdminClient(); + if (!admin.clusters().getClusters().contains(clusterName)) { + admin.clusters().createCluster(clusterName, ClusterData.builder().build()); + admin.tenants().createTenant("public", TenantInfo.builder() + .allowedClusters(Collections.singleton(clusterName)).build()); + admin.namespaces().createNamespace("public/default"); + } + } + + + @AfterClass(alwaysRun = true, timeOut = 30000) + public void cleanup() throws Exception { + bk.stop(); + } + + private ServiceConfiguration brokerConfig() { + final var config = new ServiceConfiguration(); + config.setClusterName(clusterName); + config.setAdvertisedAddress("localhost"); + config.setBrokerServicePort(Optional.of(0)); + config.setWebServicePort(Optional.of(0)); + config.setMetadataStoreUrl("zk:127.0.0.1:" + bk.getZookeeperPort()); + config.setManagedLedgerDefaultWriteQuorum(1); + config.setManagedLedgerDefaultAckQuorum(1); + config.setManagedLedgerDefaultEnsembleSize(1); + config.setDefaultNumberOfNamespaceBundles(16); + config.setLoadBalancerAutoBundleSplitEnabled(false); + config.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); + config.setLoadBalancerDebugModeEnabled(true); + config.setBrokerShutdownTimeoutMs(100); + return config; + } + + + @Test + public void testCloseAfterLoadingBundles() throws Exception { + setupBrokers(3); + final var topic = "test"; + final var admin = brokers.get(0).getAdminClient(); + admin.topics().createPartitionedTopic(topic, 20); + admin.lookups().lookupPartitionedTopic(topic); + final var client = PulsarClient.builder().serviceUrl(brokers.get(0).getBrokerServiceUrl()).build(); + final var producer = client.newProducer().topic(topic).create(); + producer.close(); + client.close(); + + final var closeTimeMsList = new ArrayList(); + for (var broker : brokers) { + final var startTimeMs = System.currentTimeMillis(); + broker.close(); + closeTimeMsList.add(System.currentTimeMillis() - startTimeMs); + } + log.info("Brokers close time: {}", closeTimeMsList); + for (var closeTimeMs : closeTimeMsList) { + Assert.assertTrue(closeTimeMs < 5000L); + } + } + + @Test + public void testLookup() throws Exception { + setupBrokers(1); + final var topic = "test-lookup"; + final var numPartitions = 16; + final var admin = brokers.get(0).getAdminClient(); + admin.topics().createPartitionedTopic(topic, numPartitions); + + final var futures = new ArrayList>(); + for (int i = 0; i < numPartitions; i++) { + futures.add(admin.lookups().lookupTopicAsync(topic + TopicName.PARTITIONED_TOPIC_SUFFIX + i)); + } + FutureUtil.waitForAll(futures).get(); + + final var start = System.currentTimeMillis(); + brokers.get(0).close(); + final var closeTimeMs = System.currentTimeMillis() - start; + log.info("Broker close time: {}", closeTimeMs); + Assert.assertTrue(closeTimeMs < 5000L); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java new file mode 100644 index 0000000000000..bb224cdf7c40e --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions; + +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; +import com.google.common.collect.Sets; +import com.google.common.io.Resources; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateMetadataStoreTableViewImpl; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateTableViewImpl; +import org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.impl.LookupService; +import org.apache.pulsar.common.naming.NamespaceBundle; +import org.apache.pulsar.common.naming.SystemTopicNames; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.util.FutureUtil; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; + +public abstract class ExtensibleLoadManagerImplBaseTest extends MockedPulsarServiceBaseTest { + + final static String caCertPath = Resources.getResource("certificate-authority/certs/ca.cert.pem").getPath(); + final static String brokerCertPath = + Resources.getResource("certificate-authority/server-keys/broker.cert.pem").getPath(); + final static String brokerKeyPath = + Resources.getResource("certificate-authority/server-keys/broker.key-pk8.pem").getPath(); + + protected PulsarService pulsar1; + protected PulsarService pulsar2; + + protected PulsarTestContext additionalPulsarTestContext; + + protected ExtensibleLoadManagerImpl primaryLoadManager; + + protected ExtensibleLoadManagerImpl secondaryLoadManager; + + protected ServiceUnitStateChannelImpl channel1; + protected ServiceUnitStateChannelImpl channel2; + + protected final String defaultTestNamespace; + + protected LookupService lookupService; + + protected String serviceUnitStateTableViewClassName; + + protected ArrayList clients = new ArrayList<>(); + + @DataProvider(name = "serviceUnitStateTableViewClassName") + public static Object[][] serviceUnitStateTableViewClassName() { + return new Object[][]{ + {ServiceUnitStateTableViewImpl.class.getName()}, + {ServiceUnitStateMetadataStoreTableViewImpl.class.getName()} + }; + } + + protected ExtensibleLoadManagerImplBaseTest(String defaultTestNamespace, String serviceUnitStateTableViewClassName) { + this.defaultTestNamespace = defaultTestNamespace; + this.serviceUnitStateTableViewClassName = serviceUnitStateTableViewClassName; + } + + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + updateConfig(conf); + } + + + protected ServiceConfiguration updateConfig(ServiceConfiguration conf) { + conf.setForceDeleteNamespaceAllowed(true); + conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); + conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); + conf.setLoadManagerServiceUnitStateTableViewClassName(serviceUnitStateTableViewClassName); + conf.setLoadBalancerReportUpdateMaxIntervalMinutes(1); + conf.setLoadBalancerSheddingEnabled(false); + conf.setLoadBalancerDebugModeEnabled(true); + conf.setWebServicePortTls(Optional.of(0)); + conf.setBrokerServicePortTls(Optional.of(0)); + conf.setTlsCertificateFilePath(brokerCertPath); + conf.setTlsKeyFilePath(brokerKeyPath); + conf.setTlsTrustCertsFilePath(caCertPath); + return conf; + } + + @Override + @BeforeClass(alwaysRun = true) + protected void setup() throws Exception { + super.internalSetup(conf); + pulsar1 = pulsar; + var conf2 = updateConfig(getDefaultConf()); + additionalPulsarTestContext = createAdditionalPulsarTestContext(conf2); + pulsar2 = additionalPulsarTestContext.getPulsarService(); + + setPrimaryLoadManager(); + setSecondaryLoadManager(); + + admin.clusters().createCluster(this.conf.getClusterName(), + ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); + admin.tenants().createTenant("public", + new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), + Sets.newHashSet(this.conf.getClusterName()))); + admin.namespaces().createNamespace("public/default"); + admin.namespaces().setNamespaceReplicationClusters("public/default", + Sets.newHashSet(this.conf.getClusterName())); + + admin.namespaces().createNamespace(defaultTestNamespace, 128); + admin.namespaces().setNamespaceReplicationClusters(defaultTestNamespace, + Sets.newHashSet(this.conf.getClusterName())); + lookupService = (LookupService) FieldUtils.readDeclaredField(pulsarClient, "lookup", true); + + for (int i = 0; i < 4; i++) { + clients.add(pulsarClient(lookupUrl.toString(), 100)); + } + } + + private static PulsarClient pulsarClient(String url, int intervalInMillis) throws PulsarClientException { + return + PulsarClient.builder() + .serviceUrl(url) + .statsInterval(intervalInMillis, TimeUnit.MILLISECONDS).build(); + } + + + @Override + @AfterClass(alwaysRun = true) + protected void cleanup() throws Exception { + List> futures = new ArrayList<>(); + for (PulsarClient client : clients) { + futures.add(client.closeAsync()); + } + futures.add(pulsar2.closeAsync()); + + if (additionalPulsarTestContext != null) { + additionalPulsarTestContext.close(); + additionalPulsarTestContext = null; + } + super.internalCleanup(); + try { + FutureUtil.waitForAll(futures).join(); + } catch (Throwable e) { + // skip error + } + pulsar1 = pulsar2 = null; + primaryLoadManager = secondaryLoadManager = null; + channel1 = channel2 = null; + lookupService = null; + + } + + @BeforeMethod(alwaysRun = true) + protected void initializeState() throws PulsarAdminException, IllegalAccessException { + admin.namespaces().unload(defaultTestNamespace); + reset(primaryLoadManager, secondaryLoadManager); + FieldUtils.writeDeclaredField(pulsarClient, "lookup", lookupService, true); + pulsar1.getConfig().setLoadBalancerMultiPhaseBundleUnload(true); + pulsar2.getConfig().setLoadBalancerMultiPhaseBundleUnload(true); + } + + protected void setPrimaryLoadManager() throws IllegalAccessException { + ExtensibleLoadManagerWrapper wrapper = + (ExtensibleLoadManagerWrapper) pulsar1.getLoadManager().get(); + primaryLoadManager = spy((ExtensibleLoadManagerImpl) + FieldUtils.readField(wrapper, "loadManager", true)); + FieldUtils.writeField(wrapper, "loadManager", primaryLoadManager, true); + channel1 = (ServiceUnitStateChannelImpl) + FieldUtils.readField(primaryLoadManager, "serviceUnitStateChannel", true); + } + + private void setSecondaryLoadManager() throws IllegalAccessException { + ExtensibleLoadManagerWrapper wrapper = + (ExtensibleLoadManagerWrapper) pulsar2.getLoadManager().get(); + secondaryLoadManager = spy((ExtensibleLoadManagerImpl) + FieldUtils.readField(wrapper, "loadManager", true)); + FieldUtils.writeField(wrapper, "loadManager", secondaryLoadManager, true); + channel2 = (ServiceUnitStateChannelImpl) + FieldUtils.readField(secondaryLoadManager, "serviceUnitStateChannel", true); + } + + protected static CompletableFuture getBundleAsync(PulsarService pulsar, TopicName topic) { + return pulsar.getNamespaceService().getBundleAsync(topic); + } + + protected Pair getBundleIsNotOwnByChangeEventTopic(String topicNamePrefix) + throws Exception { + TopicName changeEventsTopicName = + TopicName.get(defaultTestNamespace + "/" + SystemTopicNames.NAMESPACE_EVENTS_LOCAL_NAME); + NamespaceBundle changeEventsBundle = getBundleAsync(pulsar1, changeEventsTopicName).get(); + int i = 0; + while(true) { + TopicName topicName = TopicName.get(defaultTestNamespace + "/" + topicNamePrefix + "-" + i); + NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); + if (!bundle.equals(changeEventsBundle)) { + return Pair.of(topicName, bundle); + } + i++; + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index b131aff68d909..d8d3e5bb44ffb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.loadbalance.extensions; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateTableViewImpl.TOPIC; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Admin; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Bandwidth; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.MsgRate; @@ -35,44 +36,61 @@ import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Overloaded; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Underloaded; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown; +import static org.apache.pulsar.broker.namespace.NamespaceService.getHeartbeatNamespace; +import static org.apache.pulsar.broker.namespace.NamespaceService.getHeartbeatNamespaceV2; +import static org.apache.pulsar.broker.namespace.NamespaceService.getSLAMonitorNamespace; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - import com.google.common.collect.Sets; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; -import java.net.URL; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; +import org.apache.commons.lang3.mutable.MutableInt; import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.LeaderBroker; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateMetadataStoreTableViewImpl; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateTableViewImpl; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; @@ -83,105 +101,70 @@ import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter; import org.apache.pulsar.broker.loadbalance.extensions.reporter.BrokerLoadDataReporter; import org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder; -import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; +import org.apache.pulsar.broker.loadbalance.extensions.store.TableViewLoadDataStoreImpl; import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl; import org.apache.pulsar.broker.lookup.LookupResult; import org.apache.pulsar.broker.namespace.LookupOptions; -import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.apache.pulsar.broker.namespace.NamespaceBundleOwnershipListener; +import org.apache.pulsar.broker.namespace.NamespaceBundleSplitListener; +import org.apache.pulsar.broker.namespace.NamespaceEphemeralData; +import org.apache.pulsar.broker.namespace.NamespaceService; +import org.apache.pulsar.broker.service.BrokerServiceException; +import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.impl.TableViewImpl; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.LookupService; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.ServiceUnitId; +import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.naming.TopicVersion; +import org.apache.pulsar.common.policies.data.BrokerAssignment; import org.apache.pulsar.common.policies.data.BundlesData; -import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.policies.data.NamespaceOwnershipStatus; import org.apache.pulsar.common.stats.Metrics; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; -import org.testcontainers.shaded.org.awaitility.Awaitility; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.BeforeMethod; +import org.awaitility.Awaitility; +import org.testng.AssertJUnit; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Factory; import org.testng.annotations.Test; /** * Unit test for {@link ExtensibleLoadManagerImpl}. */ @Slf4j -@Test(groups = "broker") -public class ExtensibleLoadManagerImplTest extends MockedPulsarServiceBaseTest { - - private PulsarService pulsar1; - private PulsarService pulsar2; - - private PulsarTestContext additionalPulsarTestContext; - - private ExtensibleLoadManagerImpl primaryLoadManager; - - private ExtensibleLoadManagerImpl secondaryLoadManager; - - private ServiceUnitStateChannelImpl channel1; - private ServiceUnitStateChannelImpl channel2; - - @BeforeClass - @Override - public void setup() throws Exception { - conf.setForceDeleteNamespaceAllowed(true); - conf.setAllowAutoTopicCreation(true); - conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); - conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); - conf.setLoadBalancerSheddingEnabled(false); - conf.setLoadBalancerDebugModeEnabled(true); - super.internalSetup(conf); - pulsar1 = pulsar; - ServiceConfiguration defaultConf = getDefaultConf(); - defaultConf.setAllowAutoTopicCreation(true); - defaultConf.setForceDeleteNamespaceAllowed(true); - defaultConf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); - defaultConf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); - defaultConf.setLoadBalancerSheddingEnabled(false); - additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf); - pulsar2 = additionalPulsarTestContext.getPulsarService(); +@Test(groups = "flaky") +@SuppressWarnings("unchecked") +public class ExtensibleLoadManagerImplTest extends ExtensibleLoadManagerImplBaseTest { - setPrimaryLoadManager(); - - setSecondaryLoadManager(); - - admin.clusters().createCluster(this.conf.getClusterName(), - ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); - admin.tenants().createTenant("public", - new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), - Sets.newHashSet(this.conf.getClusterName()))); - admin.namespaces().createNamespace("public/default"); - admin.namespaces().setNamespaceReplicationClusters("public/default", - Sets.newHashSet(this.conf.getClusterName())); - } - - @Override - @AfterClass - protected void cleanup() throws Exception { - pulsar1 = null; - pulsar2.close(); - super.internalCleanup(); - this.additionalPulsarTestContext.close(); - } - - @BeforeMethod(alwaysRun = true) - protected void initializeState() throws PulsarAdminException { - admin.namespaces().unload("public/default"); - reset(primaryLoadManager, secondaryLoadManager); + @Factory(dataProvider = "serviceUnitStateTableViewClassName") + public ExtensibleLoadManagerImplTest(String serviceUnitStateTableViewClassName) { + super("public/test", serviceUnitStateTableViewClassName); } @Test public void testAssignInternalTopic() throws Exception { Optional brokerLookupData1 = primaryLoadManager.assign( - Optional.of(TopicName.get(ServiceUnitStateChannelImpl.TOPIC)), - getBundleAsync(pulsar1, TopicName.get(ServiceUnitStateChannelImpl.TOPIC)).get()).get(); + Optional.of(TopicName.get(TOPIC)), + getBundleAsync(pulsar1, TopicName.get(TOPIC)).get(), + LookupOptions.builder().build()).get(); Optional brokerLookupData2 = secondaryLoadManager.assign( - Optional.of(TopicName.get(ServiceUnitStateChannelImpl.TOPIC)), - getBundleAsync(pulsar1, TopicName.get(ServiceUnitStateChannelImpl.TOPIC)).get()).get(); + Optional.of(TopicName.get(TOPIC)), + getBundleAsync(pulsar1, TopicName.get(TOPIC)).get(), + LookupOptions.builder().build()).get(); assertEquals(brokerLookupData1, brokerLookupData2); assertTrue(brokerLookupData1.isPresent()); @@ -189,25 +172,25 @@ public void testAssignInternalTopic() throws Exception { FieldUtils.readField(channel1, "leaderElectionService", true); Optional currentLeader = leaderElectionService.getCurrentLeader(); assertTrue(currentLeader.isPresent()); - assertEquals(brokerLookupData1.get().getWebServiceUrl(), currentLeader.get().getServiceUrl()); + assertEquals(brokerLookupData1.get().getWebServiceUrlTls(), currentLeader.get().getServiceUrl()); } @Test public void testAssign() throws Exception { - TopicName topicName = TopicName.get("test-assign"); - NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); - Optional brokerLookupData = primaryLoadManager.assign(Optional.empty(), bundle).get(); + Pair topicAndBundle = getBundleIsNotOwnByChangeEventTopic("test-assign"); + TopicName topicName = topicAndBundle.getLeft(); + NamespaceBundle bundle = topicAndBundle.getRight(); + Optional brokerLookupData = primaryLoadManager.assign(Optional.empty(), bundle, + LookupOptions.builder().build()).get(); assertTrue(brokerLookupData.isPresent()); log.info("Assign the bundle {} to {}", bundle, brokerLookupData); // Should get owner info from channel. - Optional brokerLookupData1 = secondaryLoadManager.assign(Optional.empty(), bundle).get(); + Optional brokerLookupData1 = secondaryLoadManager.assign(Optional.empty(), bundle, + LookupOptions.builder().build()).get(); assertEquals(brokerLookupData, brokerLookupData1); - verify(primaryLoadManager, times(1)).getBrokerSelectionStrategy(); - verify(secondaryLoadManager, times(0)).getBrokerSelectionStrategy(); - Optional lookupResult = pulsar2.getNamespaceService() - .getBrokerServiceUrlAsync(topicName, null).get(); + .getBrokerServiceUrlAsync(topicName, LookupOptions.builder().build()).get(); assertTrue(lookupResult.isPresent()); assertEquals(lookupResult.get().getLookupData().getHttpUrl(), brokerLookupData.get().getWebServiceUrl()); @@ -217,15 +200,61 @@ public void testAssign() throws Exception { assertEquals(webServiceUrl.get().toString(), brokerLookupData.get().getWebServiceUrl()); } + @Test + public void testLookupOptions() throws Exception { + Pair topicAndBundle = + getBundleIsNotOwnByChangeEventTopic("test-lookup-options"); + TopicName topicName = topicAndBundle.getLeft(); + NamespaceBundle bundle = topicAndBundle.getRight(); + + admin.topics().createPartitionedTopic(topicName.toString(), 1); + + // Test LookupOptions.readOnly = true when the bundle is not owned by any broker. + Optional webServiceUrlReadOnlyTrue = pulsar1.getNamespaceService() + .getWebServiceUrl(bundle, LookupOptions.builder().readOnly(true).requestHttps(false).build()); + assertTrue(webServiceUrlReadOnlyTrue.isEmpty()); + + // Test LookupOptions.readOnly = false and the bundle assign to some broker. + Optional webServiceUrlReadOnlyFalse = pulsar1.getNamespaceService() + .getWebServiceUrl(bundle, LookupOptions.builder().readOnly(false).requestHttps(false).build()); + assertTrue(webServiceUrlReadOnlyFalse.isPresent()); + + // Test LookupOptions.requestHttps = true + Optional webServiceUrlHttps = pulsar2.getNamespaceService() + .getWebServiceUrl(bundle, LookupOptions.builder().requestHttps(true).build()); + assertTrue(webServiceUrlHttps.isPresent()); + assertTrue(webServiceUrlHttps.get().toString().startsWith("https")); + + // TODO: Support LookupOptions.loadTopicsInBundle = true + + // Test LookupOptions.advertisedListenerName = internal but the broker do not have internal listener. + try { + pulsar2.getNamespaceService() + .getWebServiceUrl(bundle, LookupOptions.builder().advertisedListenerName("internal").build()); + fail(); + } catch (Exception e) { + assertTrue(e.getMessage().contains("the broker do not have internal listener")); + } + } + @Test public void testCheckOwnershipAsync() throws Exception { - NamespaceBundle bundle = getBundleAsync(pulsar1, TopicName.get("test-check-ownership")).get(); + Pair topicAndBundle = getBundleIsNotOwnByChangeEventTopic("test-check-ownership"); + NamespaceBundle bundle = topicAndBundle.getRight(); // 1. The bundle is never assigned. + retryStrategically((test) -> { + try { + return !primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get() + && !secondaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get(); + } catch (Exception e) { + return false; + } + }, 5, 200); assertFalse(primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); assertFalse(secondaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); // 2. Assign the bundle to a broker. - Optional lookupData = primaryLoadManager.assign(Optional.empty(), bundle).get(); + Optional lookupData = primaryLoadManager.assign(Optional.empty(), bundle, LookupOptions.builder().build()).get(); assertTrue(lookupData.isPresent()); if (lookupData.get().getPulsarServiceUrl().equals(pulsar1.getBrokerServiceUrl())) { assertTrue(primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); @@ -239,8 +268,8 @@ public void testCheckOwnershipAsync() throws Exception { @Test public void testFilter() throws Exception { - TopicName topicName = TopicName.get("test-filter"); - NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); + Pair topicAndBundle = getBundleIsNotOwnByChangeEventTopic("test-filter"); + NamespaceBundle bundle = topicAndBundle.getRight(); doReturn(List.of(new BrokerFilter() { @Override @@ -249,59 +278,124 @@ public String name() { } @Override - public Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) throws BrokerFilterException { - brokers.remove(pulsar1.getLookupServiceAddress()); - return brokers; + public CompletableFuture> filterAsync(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { + brokers.remove(pulsar1.getBrokerId()); + return CompletableFuture.completedFuture(brokers); } })).when(primaryLoadManager).getBrokerFilterPipeline(); - Optional brokerLookupData = primaryLoadManager.assign(Optional.empty(), bundle).get(); + Optional brokerLookupData = primaryLoadManager.assign(Optional.empty(), bundle, LookupOptions.builder().build()).get(); assertTrue(brokerLookupData.isPresent()); assertEquals(brokerLookupData.get().getWebServiceUrl(), pulsar2.getWebServiceAddress()); } @Test public void testFilterHasException() throws Exception { - TopicName topicName = TopicName.get("test-filter-has-exception"); - NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); + Pair topicAndBundle = getBundleIsNotOwnByChangeEventTopic("test-filter-has-exception"); + NamespaceBundle bundle = topicAndBundle.getRight(); doReturn(List.of(new MockBrokerFilter() { @Override - public Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) throws BrokerFilterException { - brokers.clear(); - throw new BrokerFilterException("Test"); + public CompletableFuture> filterAsync(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { + brokers.remove(brokers.keySet().iterator().next()); + return FutureUtil.failedFuture(new BrokerFilterException("Test")); } })).when(primaryLoadManager).getBrokerFilterPipeline(); - Optional brokerLookupData = primaryLoadManager.assign(Optional.empty(), bundle).get(); + Optional brokerLookupData = primaryLoadManager.assign(Optional.empty(), bundle, LookupOptions.builder().build()).get(); assertTrue(brokerLookupData.isPresent()); } + @Test(timeOut = 30 * 1000) + public void testUnloadUponTopicLookupFailure() throws Exception { + TopicName topicName = + TopicName.get("public/test/testUnloadUponTopicLookupFailure"); + NamespaceBundle bundle = pulsar1.getNamespaceService().getBundle(topicName); + primaryLoadManager.assign(Optional.empty(), bundle, LookupOptions.builder().build()).get(); + + CompletableFuture future1 = new CompletableFuture(); + CompletableFuture future2 = new CompletableFuture(); + try { + pulsar1.getBrokerService().getTopics().put(topicName.toString(), future1); + pulsar2.getBrokerService().getTopics().put(topicName.toString(), future2); + CompletableFuture.delayedExecutor(1, TimeUnit.SECONDS).execute(() -> { + future1.completeExceptionally(new CompletionException( + new BrokerServiceException.ServiceUnitNotReadyException("Please redo the lookup"))); + future2.completeExceptionally(new CompletionException( + new BrokerServiceException.ServiceUnitNotReadyException("Please redo the lookup"))); + }); + admin.namespaces().unloadNamespaceBundle(bundle.getNamespaceObject().toString(), bundle.getBundleRange()); + } finally { + pulsar1.getBrokerService().getTopics().remove(topicName.toString()); + pulsar2.getBrokerService().getTopics().remove(topicName.toString()); + } + } + + @Test(timeOut = 30 * 1000) public void testUnloadAdminAPI() throws Exception { - TopicName topicName = TopicName.get("test-unload"); - NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); + Pair topicAndBundle = getBundleIsNotOwnByChangeEventTopic("test-unload"); + TopicName topicName = topicAndBundle.getLeft(); + NamespaceBundle bundle = topicAndBundle.getRight(); + + AtomicInteger onloadCount = new AtomicInteger(0); + AtomicInteger unloadCount = new AtomicInteger(0); + + NamespaceBundleOwnershipListener listener = new NamespaceBundleOwnershipListener() { + @Override + public void onLoad(NamespaceBundle bundle) { + onloadCount.incrementAndGet(); + } + @Override + public void unLoad(NamespaceBundle bundle) { + unloadCount.incrementAndGet(); + } + + @Override + public boolean test(NamespaceBundle namespaceBundle) { + return namespaceBundle.equals(bundle); + } + }; + pulsar1.getNamespaceService().addNamespaceBundleOwnershipListener(listener); + pulsar2.getNamespaceService().addNamespaceBundleOwnershipListener(listener); String broker = admin.lookups().lookupTopic(topicName.toString()); log.info("Assign the bundle {} to {}", bundle, broker); checkOwnershipState(broker, bundle); + Awaitility.await().untilAsserted(() -> { + assertEquals(onloadCount.get(), 1); + assertEquals(unloadCount.get(), 0); + }); + admin.namespaces().unloadNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange()); assertFalse(primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); assertFalse(secondaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); + Awaitility.await().untilAsserted(() -> { + assertEquals(onloadCount.get(), 1); + assertEquals(unloadCount.get(), 1); + }); broker = admin.lookups().lookupTopic(topicName.toString()); log.info("Assign the bundle {} to {}", bundle, broker); - String dstBrokerUrl = pulsar1.getLookupServiceAddress(); + String finalBroker = broker; + Awaitility.await().untilAsserted(() -> { + checkOwnershipState(finalBroker, bundle); + assertEquals(onloadCount.get(), 2); + assertEquals(unloadCount.get(), 1); + }); + + + String dstBrokerUrl = pulsar1.getBrokerId(); String dstBrokerServiceUrl; if (broker.equals(pulsar1.getBrokerServiceUrl())) { - dstBrokerUrl = pulsar2.getLookupServiceAddress(); + dstBrokerUrl = pulsar2.getBrokerId(); dstBrokerServiceUrl = pulsar2.getBrokerServiceUrl(); } else { dstBrokerServiceUrl = pulsar1.getBrokerServiceUrl(); @@ -309,6 +403,10 @@ public void testUnloadAdminAPI() throws Exception { checkOwnershipState(broker, bundle); admin.namespaces().unloadNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange(), dstBrokerUrl); + Awaitility.await().untilAsserted(() -> { + assertEquals(onloadCount.get(), 3); + assertEquals(unloadCount.get(), 3); //one from releasing and one from owned + }); assertEquals(admin.lookups().lookupTopic(topicName.toString()), dstBrokerServiceUrl); @@ -322,7 +420,432 @@ public void testUnloadAdminAPI() throws Exception { } } - private void checkOwnershipState(String broker, NamespaceBundle bundle) + @Test(timeOut = 30 * 1000, priority = 1000) + public void testNamespaceOwnershipListener() throws Exception { + Pair topicAndBundle = + getBundleIsNotOwnByChangeEventTopic("test-namespace-ownership-listener"); + TopicName topicName = topicAndBundle.getLeft(); + NamespaceBundle bundle = topicAndBundle.getRight(); + + String broker = admin.lookups().lookupTopic(topicName.toString()); + log.info("Assign the bundle {} to {}", bundle, broker); + + checkOwnershipState(broker, bundle); + + AtomicInteger onloadCount = new AtomicInteger(0); + AtomicInteger unloadCount = new AtomicInteger(0); + + NamespaceBundleOwnershipListener listener = new NamespaceBundleOwnershipListener() { + @Override + public void onLoad(NamespaceBundle bundle) { + onloadCount.incrementAndGet(); + } + + @Override + public void unLoad(NamespaceBundle bundle) { + unloadCount.incrementAndGet(); + } + + @Override + public boolean test(NamespaceBundle namespaceBundle) { + return namespaceBundle.equals(bundle); + } + }; + pulsar1.getNamespaceService().addNamespaceBundleOwnershipListener(listener); + pulsar2.getNamespaceService().addNamespaceBundleOwnershipListener(listener); + + // There are a service unit state channel already started, when add listener, it will trigger the onload event. + Awaitility.await().untilAsserted(() -> { + assertEquals(onloadCount.get(), 1); + assertEquals(unloadCount.get(), 0); + }); + + @Cleanup + ServiceUnitStateChannelImpl channel3 = new ServiceUnitStateChannelImpl(pulsar1); + channel3.start(); + @Cleanup + ServiceUnitStateChannelImpl channel4 = new ServiceUnitStateChannelImpl(pulsar2); + channel4.start(); + Awaitility.await().untilAsserted(() -> { + assertEquals(onloadCount.get(), 2); + assertEquals(unloadCount.get(), 0); + }); + + } + + @DataProvider(name = "isPersistentTopicSubscriptionTypeTest") + public Object[][] isPersistentTopicSubscriptionTypeTest() { + return new Object[][]{ + {TopicDomain.persistent, SubscriptionType.Exclusive}, + {TopicDomain.persistent, SubscriptionType.Shared}, + {TopicDomain.persistent, SubscriptionType.Failover}, + {TopicDomain.persistent, SubscriptionType.Key_Shared}, + {TopicDomain.non_persistent, SubscriptionType.Exclusive}, + {TopicDomain.non_persistent, SubscriptionType.Shared}, + {TopicDomain.non_persistent, SubscriptionType.Failover}, + {TopicDomain.non_persistent, SubscriptionType.Key_Shared}, + }; + } + + @Test(timeOut = 30_000, dataProvider = "isPersistentTopicSubscriptionTypeTest") + public void testTransferClientReconnectionWithoutLookup(TopicDomain topicDomain, SubscriptionType subscriptionType) + throws Exception { + testTransferClientReconnectionWithoutLookup(clients, topicDomain, subscriptionType, defaultTestNamespace, + admin, lookupUrl.toString(), pulsar1, pulsar2, primaryLoadManager, secondaryLoadManager); + } + + @Test(enabled = false) + public static void testTransferClientReconnectionWithoutLookup( + List clients, + TopicDomain topicDomain, + SubscriptionType subscriptionType, + String defaultTestNamespace, + PulsarAdmin admin, String brokerServiceUrl, + PulsarService pulsar1, PulsarService pulsar2, + ExtensibleLoadManager primaryLoadManager, + ExtensibleLoadManager secondaryLoadManager) + throws Exception { + var id = String.format("test-tx-client-reconnect-%s-%s", subscriptionType, UUID.randomUUID()); + var topic = String.format("%s://%s/%s", topicDomain.toString(), defaultTestNamespace, id); + var topicName = TopicName.get(topic); + var timeoutMs = 30_000; + + var consumers = new ArrayList>(); + var lookups = new ArrayList>(); + int clientId = 0; + try { + var pulsarClient = clients.get(clientId++); + @Cleanup + var producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + lookups.add(spyLookupService(pulsarClient)); + + var consumerCount = subscriptionType == SubscriptionType.Exclusive ? 1 : 3; + + for (int i = 0; i < consumerCount; i++) { + var client = clients.get(clientId++); + var consumer = client.newConsumer(Schema.STRING). + subscriptionName(id). + subscriptionType(subscriptionType). + subscriptionInitialPosition(SubscriptionInitialPosition.Earliest). + ackTimeout(1000, TimeUnit.MILLISECONDS). + topic(topic). + subscribe(); + consumers.add(consumer); + lookups.add(spyLookupService(client)); + } + + Awaitility.await() + .until(() -> producer.isConnected() && consumers.stream().allMatch(Consumer::isConnected)); + + NamespaceBundle bundle = getBundleAsync(pulsar1, TopicName.get(topic)).get(); + String broker = admin.lookups().lookupTopic(topic); + final String dstBrokerUrl; + final String dstBrokerServiceUrl; + if (broker.equals(pulsar1.getBrokerServiceUrl())) { + dstBrokerUrl = pulsar2.getBrokerId(); + dstBrokerServiceUrl = pulsar2.getBrokerServiceUrl(); + } else { + dstBrokerUrl = pulsar1.getBrokerId(); + dstBrokerServiceUrl = pulsar1.getBrokerServiceUrl(); + } + checkOwnershipState(broker, bundle, primaryLoadManager, secondaryLoadManager, pulsar1); + + var messageCountBeforeUnloading = 10; + var messageCountAfterUnloading = 10; + var messageCount = messageCountBeforeUnloading + messageCountAfterUnloading; + + var semMessagesReadyToSend = new Semaphore(0); + var cdlStart = new CountDownLatch(1); + + @Cleanup(value = "shutdown") + var executor = Executors.newFixedThreadPool(1 /* bundle unload */ + 1 /* producer */ + consumers.size()); + + var futures = new ArrayList>(); + futures.add(CompletableFuture.runAsync(() -> { + try { + cdlStart.await(); + semMessagesReadyToSend.release(messageCountBeforeUnloading); + admin.namespaces() + .unloadNamespaceBundle(defaultTestNamespace, bundle.getBundleRange(), dstBrokerUrl); + //log.info("### unloaded."); + semMessagesReadyToSend.release(messageCountAfterUnloading); + } catch (InterruptedException | PulsarAdminException e) { + fail(); + } + }, executor)); + + var pendingMessages = Collections.synchronizedSet(new HashSet<>(messageCount)); + var producerFuture = CompletableFuture.runAsync(() -> { + try { + cdlStart.await(); + for (int i = 0; i < messageCount; i++) { + semMessagesReadyToSend.acquire(); + String message = String.format("message-%d", i); + if (topicDomain == TopicDomain.persistent) { + // Only verify receipt of persistent topic messages. + pendingMessages.add(message); + } + producer.send(message); + //log.info("### producer sent: {}", message); + } + } catch (PulsarClientException | InterruptedException e) { + fail(); + } + }, executor); + futures.add(producerFuture); + + consumers.stream().map(consumer -> CompletableFuture.runAsync(() -> { + try { + cdlStart.await(); + } catch (InterruptedException e) { + fail(); + } + while (!producerFuture.isDone() || !pendingMessages.isEmpty()) { + try { + var message = consumer.receive(200, TimeUnit.MILLISECONDS); + if (message != null) { + consumer.acknowledge(message); + pendingMessages.remove(message.getValue()); + //log.info("### consumer received: {}", message.getValue()); + } + } catch (PulsarClientException e) { + // Retry read + } + } + }, executor)).forEach(futures::add); + + var asyncTasks = FutureUtil.waitForAllAndSupportCancel(futures).orTimeout(timeoutMs, TimeUnit.MILLISECONDS); + + cdlStart.countDown(); + Awaitility.await().atMost(timeoutMs, TimeUnit.MILLISECONDS).ignoreExceptions().until( + () -> dstBrokerServiceUrl.equals(admin.lookups().lookupTopic(topic))); + + asyncTasks.get(); + + assertTrue(futures.stream().allMatch(CompletableFuture::isDone)); + assertTrue(futures.stream().noneMatch(CompletableFuture::isCompletedExceptionally)); + assertTrue(pendingMessages.isEmpty()); + + assertTrue(producer.isConnected()); + assertTrue(consumers.stream().allMatch(Consumer::isConnected)); + + for (var lookupService : lookups) { + verify(lookupService.getRight(), never()).getBroker(topicName); + } + } finally { + for (var consumer: consumers) { + consumer.close(); + } + + clientId = 0; + for (var lookup : lookups) { + resetLookupService(clients.get(clientId++), lookup.getLeft()); + } + } + } + + @Test(timeOut = 30 * 1000, dataProvider = "isPersistentTopicSubscriptionTypeTest") + public void testUnloadClientReconnectionWithLookup(TopicDomain topicDomain, + SubscriptionType subscriptionType) throws Exception { + testUnloadClientReconnectionWithLookup(clients, topicDomain, subscriptionType, defaultTestNamespace, + admin, lookupUrl.toString(), pulsar1); + } + + @Test(enabled = false) + public static void testUnloadClientReconnectionWithLookup(List clients, + TopicDomain topicDomain, + SubscriptionType subscriptionType, + String defaultTestNamespace, + PulsarAdmin admin, + String brokerServiceUrl, + PulsarService pulsar1) throws Exception { + var id = String.format("test-unload-%s-client-reconnect-%s-%s", + topicDomain, subscriptionType, UUID.randomUUID()); + var topic = String.format("%s://%s/%s", topicDomain, defaultTestNamespace, id); + var topicName = TopicName.get(topic); + + var consumers = new ArrayList>(); + Pair lookup = null; + PulsarClient pulsarClient = null; + try { + pulsarClient = clients.get(0); + var producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + + var consumerCount = subscriptionType == SubscriptionType.Exclusive ? 1 : 3; + for (int i = 0; i < consumerCount; i++) { + consumers.add(pulsarClient.newConsumer(Schema.STRING). + subscriptionName(id).subscriptionType(subscriptionType).topic(topic).subscribe()); + } + Awaitility.await() + .until(() -> producer.isConnected() && consumers.stream().allMatch(Consumer::isConnected)); + + lookup = spyLookupService(pulsarClient); + + final CountDownLatch cdl = new CountDownLatch(3); + + NamespaceBundle bundle = getBundleAsync(pulsar1, TopicName.get(topic)).get(); + CompletableFuture unloadNamespaceBundle = CompletableFuture.runAsync(() -> { + try { + cdl.await(); + admin.namespaces().unloadNamespaceBundle(defaultTestNamespace, bundle.getBundleRange()); + } catch (InterruptedException | PulsarAdminException e) { + fail(); + } + }); + + MutableInt sendCount = new MutableInt(); + Awaitility.await().atMost(20, TimeUnit.SECONDS).ignoreExceptions().until(() -> { + var message = String.format("message-%d", sendCount.getValue()); + + boolean messageSent = false; + while (true) { + var recvFutures = consumers.stream(). + map(consumer -> consumer.receiveAsync().orTimeout(200, TimeUnit.MILLISECONDS)). + collect(Collectors.toList()); + + if (!messageSent) { + producer.send(message); + messageSent = true; + } + + if (topicDomain == TopicDomain.non_persistent) { + // No need to wait for message receipt, we're only trying to stress the consumer lookup pathway. + break; + } + var msg = (Message) FutureUtil.waitForAny(recvFutures, __ -> true).get().get(); + if (Objects.equals(msg.getValue(), message)) { + break; + } + } + + cdl.countDown(); + return sendCount.incrementAndGet() == 10; + }); + + assertTrue(producer.isConnected()); + assertTrue(consumers.stream().allMatch(Consumer::isConnected)); + assertTrue(unloadNamespaceBundle.isDone()); + verify(lookup.getRight(), times(1 + consumerCount)).getBroker(topicName); + } finally { + for (var consumer : consumers) { + consumer.close(); + } + resetLookupService(pulsarClient, lookup.getLeft()); + } + } + + @DataProvider(name = "isPersistentTopicTest") + public Object[][] isPersistentTopicTest() { + return new Object[][]{{TopicDomain.persistent}, {TopicDomain.non_persistent}}; + } + + @Test(timeOut = 30 * 1000, dataProvider = "isPersistentTopicTest") + public void testOptimizeUnloadDisable(TopicDomain topicDomain) throws Exception { + testOptimizeUnloadDisable(clients, topicDomain, defaultTestNamespace, admin, lookupUrl.toString(), pulsar1, + pulsar2); + } + + @Test(enabled = false) + public static void testOptimizeUnloadDisable(List clients, + TopicDomain topicDomain, + String defaultTestNamespace, + PulsarAdmin admin, + String brokerServiceUrl, + PulsarService pulsar1, + PulsarService pulsar2) throws Exception { + var id = String.format("test-optimize-unload-disable-%s-%s", topicDomain, UUID.randomUUID()); + var topic = String.format("%s://%s/%s", topicDomain, defaultTestNamespace, id); + var topicName = TopicName.get(topic); + + pulsar1.getConfig().setLoadBalancerMultiPhaseBundleUnload(false); + pulsar2.getConfig().setLoadBalancerMultiPhaseBundleUnload(false); + + var pulsarClient = clients.get(0); + Pair lookup = null; + @Cleanup + var producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + + @Cleanup + var consumer = pulsarClient.newConsumer(Schema.STRING).subscriptionName(id).topic(topic).subscribe(); + + Awaitility.await().until(() -> producer.isConnected() && consumer.isConnected()); + + try { + lookup = spyLookupService(pulsarClient); + + final CountDownLatch cdl = new CountDownLatch(3); + + NamespaceBundle bundle = getBundleAsync(pulsar1, TopicName.get(topic)).get(); + var srcBrokerServiceUrl = admin.lookups().lookupTopic(topic); + var dstBroker = srcBrokerServiceUrl.equals(pulsar1.getBrokerServiceUrl()) ? pulsar2 : pulsar1; + + CompletableFuture unloadNamespaceBundle = CompletableFuture.runAsync(() -> { + try { + cdl.await(); + admin.namespaces().unloadNamespaceBundle(defaultTestNamespace, bundle.getBundleRange(), + dstBroker.getBrokerId()); + } catch (InterruptedException | PulsarAdminException e) { + fail(); + } + }); + + MutableInt sendCount = new MutableInt(); + Awaitility.await().atMost(20, TimeUnit.SECONDS).ignoreExceptions().until(() -> { + var message = String.format("message-%d", sendCount.getValue()); + + AtomicBoolean messageSent = new AtomicBoolean(false); + while (true) { + var recvFuture = consumer.receiveAsync().orTimeout(200, TimeUnit.MILLISECONDS); + if (!messageSent.get()) { + producer.sendAsync(message).thenAccept(messageId -> { + if (messageId != null) { + messageSent.set(true); + } + }).get(200, TimeUnit.MILLISECONDS); + } + + if (topicDomain == TopicDomain.non_persistent) { + // No need to wait for message receipt, we're only trying to stress the consumer lookup pathway. + break; + } + var msg = recvFuture.get(); + if (Objects.equals(msg.getValue(), message)) { + break; + } + } + + cdl.countDown(); + return sendCount.incrementAndGet() == 10; + }); + + Pair finalLookup = lookup; + Awaitility.await() + .atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + assertTrue(producer.isConnected()); + assertTrue(consumer.isConnected()); + assertTrue(unloadNamespaceBundle.isDone()); + verify(finalLookup.getRight(), times(2)).getBroker(topicName); + }); + } finally { + resetLookupService(pulsarClient, lookup.getLeft()); + } + } + + protected static Pair spyLookupService(PulsarClient client) throws IllegalAccessException { + LookupService svc = (LookupService) FieldUtils.readDeclaredField(client, "lookup", true); + var lookup = spy(svc); + FieldUtils.writeDeclaredField(client, "lookup", lookup, true); + return Pair.of(svc, lookup); + } + + protected static void resetLookupService(PulsarClient client, LookupService lookup) throws IllegalAccessException { + FieldUtils.writeDeclaredField(client, "lookup", lookup, true); + } + + protected static void checkOwnershipState(String broker, NamespaceBundle bundle, + ExtensibleLoadManager primaryLoadManager, + ExtensibleLoadManager secondaryLoadManager, PulsarService pulsar1) throws ExecutionException, InterruptedException { var targetLoadManager = secondaryLoadManager; var otherLoadManager = primaryLoadManager; @@ -334,29 +857,60 @@ private void checkOwnershipState(String broker, NamespaceBundle bundle) assertFalse(otherLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); } + protected void checkOwnershipState(String broker, NamespaceBundle bundle) + throws ExecutionException, InterruptedException { + checkOwnershipState(broker, bundle, primaryLoadManager, secondaryLoadManager, pulsar1); + } + @Test(timeOut = 30 * 1000) public void testSplitBundleAdminAPI() throws Exception { - String namespace = "public/default"; - String topic = "persistent://" + namespace + "/test-split"; - admin.topics().createPartitionedTopic(topic, 10); + final String namespace = "public/testSplitBundleAdminAPI"; + admin.namespaces().createNamespace(namespace, 1); + Pair topicAndBundle = getBundleIsNotOwnByChangeEventTopic("test-split"); + TopicName topicName = topicAndBundle.getLeft(); + admin.topics().createPartitionedTopic(topicName.toString(), 10); BundlesData bundles = admin.namespaces().getBundles(namespace); int numBundles = bundles.getNumBundles(); var bundleRanges = bundles.getBoundaries().stream().map(Long::decode).sorted().toList(); String firstBundle = bundleRanges.get(0) + "_" + bundleRanges.get(1); + AtomicInteger splitCount = new AtomicInteger(0); + NamespaceBundleSplitListener namespaceBundleSplitListener = new NamespaceBundleSplitListener() { + @Override + public void onSplit(NamespaceBundle bundle) { + splitCount.incrementAndGet(); + } + + @Override + public boolean test(NamespaceBundle namespaceBundle) { + return namespaceBundle + .toString() + .equals(String.format(namespace + "/0x%08x_0x%08x", bundleRanges.get(0), bundleRanges.get(1))); + } + }; + pulsar1.getNamespaceService().addNamespaceBundleSplitListener(namespaceBundleSplitListener); + pulsar2.getNamespaceService().addNamespaceBundleSplitListener(namespaceBundleSplitListener); + long mid = bundleRanges.get(0) + (bundleRanges.get(1) - bundleRanges.get(0)) / 2; admin.namespaces().splitNamespaceBundle(namespace, firstBundle, true, null); - BundlesData bundlesData = admin.namespaces().getBundles(namespace); - assertEquals(bundlesData.getNumBundles(), numBundles + 1); - String lowBundle = String.format("0x%08x", bundleRanges.get(0)); - String midBundle = String.format("0x%08x", mid); - String highBundle = String.format("0x%08x", bundleRanges.get(1)); - assertTrue(bundlesData.getBoundaries().contains(lowBundle)); - assertTrue(bundlesData.getBoundaries().contains(midBundle)); - assertTrue(bundlesData.getBoundaries().contains(highBundle)); + + Awaitility.await() + .atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + BundlesData bundlesData = admin.namespaces().getBundles(namespace); + assertEquals(bundlesData.getNumBundles(), numBundles + 1); + String lowBundle = String.format("0x%08x", bundleRanges.get(0)); + String midBundle = String.format("0x%08x", mid); + String highBundle = String.format("0x%08x", bundleRanges.get(1)); + assertTrue(bundlesData.getBoundaries().contains(lowBundle)); + assertTrue(bundlesData.getBoundaries().contains(midBundle)); + assertTrue(bundlesData.getBoundaries().contains(highBundle)); + assertEquals(splitCount.get(), 1); + }); + // Test split bundle with invalid bundle range. try { @@ -365,13 +919,37 @@ public void testSplitBundleAdminAPI() throws Exception { } catch (PulsarAdminException ex) { assertTrue(ex.getMessage().contains("Invalid bundle range")); } + + + // delete and retry + Awaitility.await() + .atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + admin.namespaces().deleteNamespace(namespace); + }); + admin.namespaces().createNamespace(namespace, 1); + admin.namespaces().splitNamespaceBundle(namespace, firstBundle, true, null); + + Awaitility.await() + .atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + BundlesData bundlesData = admin.namespaces().getBundles(namespace); + assertEquals(bundlesData.getNumBundles(), numBundles + 1); + String lowBundle = String.format("0x%08x", bundleRanges.get(0)); + String midBundle = String.format("0x%08x", mid); + String highBundle = String.format("0x%08x", bundleRanges.get(1)); + assertTrue(bundlesData.getBoundaries().contains(lowBundle)); + assertTrue(bundlesData.getBoundaries().contains(midBundle)); + assertTrue(bundlesData.getBoundaries().contains(highBundle)); + assertEquals(splitCount.get(), 2); + }); } @Test(timeOut = 30 * 1000) public void testSplitBundleWithSpecificPositionAdminAPI() throws Exception { - String namespace = "public/default"; + String namespace = defaultTestNamespace; String topic = "persistent://" + namespace + "/test-split-with-specific-position"; - admin.topics().createPartitionedTopic(topic, 10); + admin.topics().createPartitionedTopic(topic, 1024); BundlesData bundles = admin.namespaces().getBundles(namespace); int numBundles = bundles.getNumBundles(); @@ -386,7 +964,8 @@ public void testSplitBundleWithSpecificPositionAdminAPI() throws Exception { "specified_positions_divide", List.of(bundleRanges.get(0), bundleRanges.get(1), splitPosition)); BundlesData bundlesData = admin.namespaces().getBundles(namespace); - assertEquals(bundlesData.getNumBundles(), numBundles + 1); + Awaitility.waitAtMost(15, TimeUnit.SECONDS) + .untilAsserted(() -> assertEquals(bundlesData.getNumBundles(), numBundles + 1)); String lowBundle = String.format("0x%08x", bundleRanges.get(0)); String midBundle = String.format("0x%08x", splitPosition); String highBundle = String.format("0x%08x", bundleRanges.get(1)); @@ -396,16 +975,28 @@ public void testSplitBundleWithSpecificPositionAdminAPI() throws Exception { } @Test(timeOut = 30 * 1000) public void testDeleteNamespaceBundle() throws Exception { - TopicName topicName = TopicName.get("test-delete-namespace-bundle"); - NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); - - String broker = admin.lookups().lookupTopic(topicName.toString()); - log.info("Assign the bundle {} to {}", bundle, broker); - - checkOwnershipState(broker, bundle); - - admin.namespaces().deleteNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange()); - assertFalse(primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); + final String namespace = "public/testDeleteNamespaceBundle"; + admin.namespaces().createNamespace(namespace, 3); + TopicName topicName = TopicName.get(namespace + "/test-delete-namespace-bundle"); + + Awaitility.await() + .atMost(15, TimeUnit.SECONDS) + .ignoreExceptions() + .untilAsserted(() -> { + NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); + String broker = admin.lookups().lookupTopic(topicName.toString()); + log.info("Assign the bundle {} to {}", bundle, broker); + checkOwnershipState(broker, bundle); + admin.namespaces().deleteNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange(), true); + // this could fail if the system topic lookup asynchronously happens before this. + // we will retry if it fails. + assertFalse(primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); + }); + + Awaitility.await() + .atMost(15, TimeUnit.SECONDS) + .ignoreExceptions() + .untilAsserted(() -> admin.namespaces().deleteNamespace(namespace, true)); } @Test(timeOut = 30 * 1000) @@ -445,34 +1036,42 @@ public void testCheckOwnershipPresentWithSystemNamespace() throws Exception { @Test public void testMoreThenOneFilter() throws Exception { - TopicName topicName = TopicName.get("test-filter-has-exception"); - NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); - - String lookupServiceAddress1 = pulsar1.getLookupServiceAddress(); + // Use a different namespace to avoid flaky test failures + // from unloading the default namespace and the following topic policy lookups at the init state step + Pair topicAndBundle = + getBundleIsNotOwnByChangeEventTopic("test-filter-has-exception"); + TopicName topicName = topicAndBundle.getLeft(); + NamespaceBundle bundle = topicAndBundle.getRight(); + + String brokerId1 = pulsar1.getBrokerId(); doReturn(List.of(new MockBrokerFilter() { @Override - public Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) throws BrokerFilterException { - brokers.remove(lookupServiceAddress1); - return brokers; + public CompletableFuture> filterAsync(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { + brokers.remove(brokerId1); + return CompletableFuture.completedFuture(brokers); } },new MockBrokerFilter() { @Override - public Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) throws BrokerFilterException { - brokers.clear(); - throw new BrokerFilterException("Test"); + public CompletableFuture> filterAsync(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { + return FutureUtil.failedFuture(new BrokerFilterException("Test")); } })).when(primaryLoadManager).getBrokerFilterPipeline(); - - Optional brokerLookupData = primaryLoadManager.assign(Optional.empty(), bundle).get(); - assertTrue(brokerLookupData.isPresent()); - assertEquals(brokerLookupData.get().getWebServiceUrl(), pulsar2.getWebServiceAddress()); + Optional brokerLookupData = primaryLoadManager.assign(Optional.empty(), bundle, LookupOptions.builder().build()).get(); + Awaitility.waitAtMost(5, TimeUnit.SECONDS).untilAsserted(() -> { + assertTrue(brokerLookupData.isPresent()); + assertEquals(brokerLookupData.get().getWebServiceUrl(), pulsar2.getWebServiceAddress()); + assertEquals(brokerLookupData.get().getPulsarServiceUrl(), + pulsar1.getAdminClient().lookups().lookupTopic(topicName.toString())); + assertEquals(brokerLookupData.get().getPulsarServiceUrl(), + pulsar2.getAdminClient().lookups().lookupTopic(topicName.toString())); + }); } - @Test + @Test(priority = 200) public void testDeployAndRollbackLoadManager() throws Exception { // Test rollback to modular load manager. ServiceConfiguration defaultConf = getDefaultConf(); @@ -482,8 +1081,13 @@ public void testDeployAndRollbackLoadManager() throws Exception { defaultConf.setLoadBalancerSheddingEnabled(false); try (var additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf)) { // start pulsar3 with old load manager + @Cleanup var pulsar3 = additionalPulsarTestContext.getPulsarService(); - String topic = "persistent://public/default/test"; + Pair topicAndBundle = + getBundleIsNotOwnByChangeEventTopic("testDeployAndRollbackLoadManager"); + TopicName topicName = topicAndBundle.getLeft(); + NamespaceBundle bundle = topicAndBundle.getRight(); + String topic = topicName.toString(); String lookupResult1 = pulsar3.getAdminClient().lookups().lookupTopic(topic); assertEquals(lookupResult1, pulsar3.getBrokerServiceUrl()); @@ -493,7 +1097,6 @@ public void testDeployAndRollbackLoadManager() throws Exception { assertEquals(lookupResult1, lookupResult2); assertEquals(lookupResult1, lookupResult3); - NamespaceBundle bundle = getBundleAsync(pulsar1, TopicName.get("test")).get(); LookupOptions options = LookupOptions.builder() .authoritative(false) .requestHttps(false) @@ -514,13 +1117,29 @@ public void testDeployAndRollbackLoadManager() throws Exception { assertTrue(webServiceUrl3.isPresent()); assertEquals(webServiceUrl3.get().toString(), webServiceUrl1.get().toString()); + List pulsarServices = List.of(pulsar1, pulsar2, pulsar3); + for (PulsarService pulsarService : pulsarServices) { + // Test lookup heartbeat namespace's topic + for (PulsarService pulsar : pulsarServices) { + assertLookupHeartbeatOwner(pulsarService, + pulsar.getBrokerId(), pulsar.getBrokerServiceUrl()); + } + // Test lookup SLA namespace's topic + for (PulsarService pulsar : pulsarServices) { + assertLookupSLANamespaceOwner(pulsarService, + pulsar.getBrokerId(), pulsar.getBrokerServiceUrl()); + } + } + // Test deploy new broker with new load manager ServiceConfiguration conf = getDefaultConf(); conf.setAllowAutoTopicCreation(true); conf.setForceDeleteNamespaceAllowed(true); conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); + conf.setLoadManagerServiceUnitStateTableViewClassName(serviceUnitStateTableViewClassName); try (var additionPulsarTestContext = createAdditionalPulsarTestContext(conf)) { + @Cleanup var pulsar4 = additionPulsarTestContext.getPulsarService(); Set availableCandidates = Sets.newHashSet(pulsar1.getBrokerServiceUrl(), @@ -562,62 +1181,355 @@ public void testDeployAndRollbackLoadManager() throws Exception { assertTrue(webServiceUrl4.isPresent()); assertEquals(webServiceUrl4.get().toString(), webServiceUrl1.get().toString()); + pulsarServices = List.of(pulsar1, pulsar2, pulsar3, pulsar4); + for (PulsarService pulsarService : pulsarServices) { + // Test lookup heartbeat namespace's topic + for (PulsarService pulsar : pulsarServices) { + assertLookupHeartbeatOwner(pulsarService, + pulsar.getBrokerId(), pulsar.getBrokerServiceUrl()); + } + // Test lookup SLA namespace's topic + for (PulsarService pulsar : pulsarServices) { + assertLookupSLANamespaceOwner(pulsarService, + pulsar.getBrokerId(), pulsar.getBrokerServiceUrl()); + } + } + // Check if the broker is available + var wrapper = (ExtensibleLoadManagerWrapper) pulsar4.getLoadManager().get(); + var loadManager4 = spy((ExtensibleLoadManagerImpl) + FieldUtils.readField(wrapper, "loadManager", true)); + loadManager4.getBrokerRegistry().unregister(); + + NamespaceName slaMonitorNamespace = + getSLAMonitorNamespace(pulsar4.getBrokerId(), pulsar.getConfiguration()); + String slaMonitorTopic = slaMonitorNamespace.getPersistentTopicName("test"); + String result = pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic); + assertNotNull(result); + log.info("{} Namespace is re-owned by {}", slaMonitorTopic, result); + assertNotEquals(result, pulsar4.getBrokerServiceUrl()); + + Producer producer = pulsar.getClient().newProducer(Schema.STRING).topic(slaMonitorTopic).create(); + producer.send("t1"); + + // Test re-register broker and check the lookup result + loadManager4.getBrokerRegistry().registerAsync().get(); + + result = pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic); + assertNotNull(result); + log.info("{} Namespace is re-owned by {}", slaMonitorTopic, result); + assertEquals(result, pulsar4.getBrokerServiceUrl()); + + producer.send("t2"); + Producer producer1 = pulsar.getClient().newProducer(Schema.STRING).topic(slaMonitorTopic).create(); + producer1.send("t3"); + + producer.close(); + producer1.close(); + @Cleanup + Consumer consumer = pulsar.getClient().newConsumer(Schema.STRING) + .topic(slaMonitorTopic) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionName("test") + .subscribe(); + // receive message t1 t2 t3 + assertEquals(consumer.receive().getValue(), "t1"); + assertEquals(consumer.receive().getValue(), "t2"); + assertEquals(consumer.receive().getValue(), "t3"); } } } - @Test - public void testTopBundlesLoadDataStoreTableViewFromChannelOwner() throws Exception { - var topBundlesLoadDataStorePrimary = - (LoadDataStore) FieldUtils.readDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", true); - var serviceUnitStateChannelPrimary = - (ServiceUnitStateChannelImpl) FieldUtils.readDeclaredField(primaryLoadManager, - "serviceUnitStateChannel", true); - var tvPrimary = - (TableViewImpl) FieldUtils.readDeclaredField(topBundlesLoadDataStorePrimary, "tableView", true); + @Test(priority = 200) + public void testLoadBalancerServiceUnitTableViewSyncer() throws Exception { + + Pair topicAndBundle = + getBundleIsNotOwnByChangeEventTopic("testLoadBalancerServiceUnitTableViewSyncer"); + TopicName topicName = topicAndBundle.getLeft(); + NamespaceBundle bundle = topicAndBundle.getRight(); + String topic = topicName.toString(); + + String lookupResultBefore1 = pulsar1.getAdminClient().lookups().lookupTopic(topic); + String lookupResultBefore2 = pulsar2.getAdminClient().lookups().lookupTopic(topic); + assertEquals(lookupResultBefore1, lookupResultBefore2); + + LookupOptions options = LookupOptions.builder() + .authoritative(false) + .requestHttps(false) + .readOnly(false) + .loadTopicsInBundle(false).build(); + Optional webServiceUrlBefore1 = + pulsar1.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webServiceUrlBefore1.isPresent()); + + Optional webServiceUrlBefore2 = + pulsar2.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webServiceUrlBefore2.isPresent()); + assertEquals(webServiceUrlBefore2.get().toString(), webServiceUrlBefore1.get().toString()); + + + String syncerTyp = serviceUnitStateTableViewClassName.equals(ServiceUnitStateTableViewImpl.class.getName()) ? + "SystemTopicToMetadataStoreSyncer" : "MetadataStoreToSystemTopicSyncer"; + pulsar.getAdminClient().brokers() + .updateDynamicConfiguration("loadBalancerServiceUnitTableViewSyncer", syncerTyp); + makeSecondaryAsLeader(); + makePrimaryAsLeader(); + Awaitility.waitAtMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> assertTrue(primaryLoadManager.getServiceUnitStateTableViewSyncer().isActive())); + Awaitility.waitAtMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> assertFalse(secondaryLoadManager.getServiceUnitStateTableViewSyncer().isActive())); + ServiceConfiguration defaultConf = getDefaultConf(); + defaultConf.setAllowAutoTopicCreation(true); + defaultConf.setForceDeleteNamespaceAllowed(true); + defaultConf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getCanonicalName()); + defaultConf.setLoadBalancerSheddingEnabled(false); + defaultConf.setLoadManagerServiceUnitStateTableViewClassName(ServiceUnitStateTableViewImpl.class.getName()); + try (var additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf)) { + // start pulsar3 with ServiceUnitStateTableViewImpl + @Cleanup + var pulsar3 = additionalPulsarTestContext.getPulsarService(); - var topBundlesLoadDataStoreSecondary = - (LoadDataStore) FieldUtils.readDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", true); - var tvSecondary = - (TableViewImpl) FieldUtils.readDeclaredField(topBundlesLoadDataStoreSecondary, "tableView", true); + String lookupResult1 = pulsar3.getAdminClient().lookups().lookupTopic(topic); + String lookupResult2 = pulsar1.getAdminClient().lookups().lookupTopic(topic); + String lookupResult3 = pulsar2.getAdminClient().lookups().lookupTopic(topic); + assertEquals(lookupResult1, lookupResult2); + assertEquals(lookupResult1, lookupResult3); + assertEquals(lookupResult1, lookupResultBefore1); - if (serviceUnitStateChannelPrimary.isChannelOwnerAsync().get(5, TimeUnit.SECONDS)) { - assertNotNull(tvPrimary); - assertNull(tvSecondary); - } else { - assertNull(tvPrimary); - assertNotNull(tvSecondary); - } + Optional webServiceUrl1 = + pulsar1.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webServiceUrl1.isPresent()); + + Optional webServiceUrl2 = + pulsar2.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webServiceUrl2.isPresent()); + assertEquals(webServiceUrl2.get().toString(), webServiceUrl1.get().toString()); - restartBroker(); - pulsar1 = pulsar; - setPrimaryLoadManager(); - admin.namespaces().setNamespaceReplicationClusters("public/default", - Sets.newHashSet(this.conf.getClusterName())); - - var serviceUnitStateChannelPrimaryNew = - (ServiceUnitStateChannelImpl) FieldUtils.readDeclaredField(primaryLoadManager, - "serviceUnitStateChannel", true); - var topBundlesLoadDataStorePrimaryNew = - (LoadDataStore) FieldUtils.readDeclaredField(primaryLoadManager, "topBundlesLoadDataStore" - , true); - Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> { - assertFalse(serviceUnitStateChannelPrimaryNew.isChannelOwnerAsync().get(5, TimeUnit.SECONDS)); - assertNotNull(FieldUtils.readDeclaredField(topBundlesLoadDataStoreSecondary, "tableView" - , true)); - assertNull(FieldUtils.readDeclaredField(topBundlesLoadDataStorePrimaryNew, "tableView" - , true)); + Optional webServiceUrl3 = + pulsar3.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webServiceUrl3.isPresent()); + assertEquals(webServiceUrl3.get().toString(), webServiceUrl1.get().toString()); + + assertEquals(webServiceUrl3.get().toString(), webServiceUrlBefore1.get().toString()); + + List pulsarServices = List.of(pulsar1, pulsar2, pulsar3); + for (PulsarService pulsarService : pulsarServices) { + // Test lookup heartbeat namespace's topic + for (PulsarService pulsar : pulsarServices) { + assertLookupHeartbeatOwner(pulsarService, + pulsar.getBrokerId(), pulsar.getBrokerServiceUrl()); + } + // Test lookup SLA namespace's topic + for (PulsarService pulsar : pulsarServices) { + assertLookupSLANamespaceOwner(pulsarService, + pulsar.getBrokerId(), pulsar.getBrokerServiceUrl()); } - ); + } + + // Start broker4 with ServiceUnitStateMetadataStoreTableViewImpl + ServiceConfiguration conf = getDefaultConf(); + conf.setAllowAutoTopicCreation(true); + conf.setForceDeleteNamespaceAllowed(true); + conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getCanonicalName()); + conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); + conf.setLoadManagerServiceUnitStateTableViewClassName( + ServiceUnitStateMetadataStoreTableViewImpl.class.getName()); + try (var additionPulsarTestContext = createAdditionalPulsarTestContext(conf)) { + @Cleanup + var pulsar4 = additionPulsarTestContext.getPulsarService(); + + Set availableCandidates = Sets.newHashSet( + pulsar1.getBrokerServiceUrl(), + pulsar2.getBrokerServiceUrl(), + pulsar3.getBrokerServiceUrl(), + pulsar4.getBrokerServiceUrl()); + String lookupResult4 = pulsar4.getAdminClient().lookups().lookupTopic(topic); + assertTrue(availableCandidates.contains(lookupResult4)); + + String lookupResult5 = pulsar1.getAdminClient().lookups().lookupTopic(topic); + String lookupResult6 = pulsar2.getAdminClient().lookups().lookupTopic(topic); + String lookupResult7 = pulsar3.getAdminClient().lookups().lookupTopic(topic); + assertEquals(lookupResult4, lookupResult5); + assertEquals(lookupResult4, lookupResult6); + assertEquals(lookupResult4, lookupResult7); + assertEquals(lookupResult4, lookupResultBefore1); + + + Pair topicAndBundle2 = + getBundleIsNotOwnByChangeEventTopic("testLoadBalancerServiceUnitTableViewSyncer2"); + String topic2 = topicAndBundle2.getLeft().toString(); + + String lookupResult8 = pulsar1.getAdminClient().lookups().lookupTopic(topic2); + String lookupResult9 = pulsar2.getAdminClient().lookups().lookupTopic(topic2); + String lookupResult10 = pulsar3.getAdminClient().lookups().lookupTopic(topic2); + String lookupResult11 = pulsar4.getAdminClient().lookups().lookupTopic(topic2); + assertEquals(lookupResult9, lookupResult8); + assertEquals(lookupResult10, lookupResult8); + assertEquals(lookupResult11, lookupResult8); + + Set availableWebUrlCandidates = Sets.newHashSet( + pulsar1.getWebServiceAddress(), + pulsar2.getWebServiceAddress(), + pulsar3.getWebServiceAddress(), + pulsar4.getWebServiceAddress()); + + webServiceUrl1 = + pulsar1.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webServiceUrl1.isPresent()); + assertTrue(availableWebUrlCandidates.contains(webServiceUrl1.get().toString())); + + webServiceUrl2 = + pulsar2.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webServiceUrl2.isPresent()); + assertEquals(webServiceUrl2.get().toString(), webServiceUrl1.get().toString()); + + webServiceUrl3 = + pulsar3.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webServiceUrl3.isPresent()); + assertTrue(availableWebUrlCandidates.contains(webServiceUrl3.get().toString())); + + var webServiceUrl4 = + pulsar4.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webServiceUrl4.isPresent()); + assertEquals(webServiceUrl4.get().toString(), webServiceUrl1.get().toString()); + assertEquals(webServiceUrl4.get().toString(), webServiceUrlBefore1.get().toString()); + + pulsarServices = List.of(pulsar1, pulsar2, pulsar3, pulsar4); + for (PulsarService pulsarService : pulsarServices) { + // Test lookup heartbeat namespace's topic + for (PulsarService pulsar : pulsarServices) { + assertLookupHeartbeatOwner(pulsarService, + pulsar.getBrokerId(), pulsar.getBrokerServiceUrl()); + } + // Test lookup SLA namespace's topic + for (PulsarService pulsar : pulsarServices) { + assertLookupSLANamespaceOwner(pulsarService, + pulsar.getBrokerId(), pulsar.getBrokerServiceUrl()); + } + } + // Check if the broker is available + var wrapper = (ExtensibleLoadManagerWrapper) pulsar4.getLoadManager().get(); + var loadManager4 = spy((ExtensibleLoadManagerImpl) + FieldUtils.readField(wrapper, "loadManager", true)); + loadManager4.getBrokerRegistry().unregister(); + + NamespaceName slaMonitorNamespace = + getSLAMonitorNamespace(pulsar4.getBrokerId(), pulsar.getConfiguration()); + String slaMonitorTopic = slaMonitorNamespace.getPersistentTopicName("test"); + String result = pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic); + assertNotNull(result); + log.info("{} Namespace is re-owned by {}", slaMonitorTopic, result); + assertNotEquals(result, pulsar4.getBrokerServiceUrl()); + + Producer producer = pulsar.getClient().newProducer(Schema.STRING).topic(slaMonitorTopic).create(); + producer.send("t1"); + + // Test re-register broker and check the lookup result + loadManager4.getBrokerRegistry().registerAsync().get(); + + result = pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic); + assertNotNull(result); + log.info("{} Namespace is re-owned by {}", slaMonitorTopic, result); + assertEquals(result, pulsar4.getBrokerServiceUrl()); + + producer.send("t2"); + Producer producer1 = pulsar.getClient().newProducer(Schema.STRING).topic(slaMonitorTopic).create(); + producer1.send("t3"); + + producer.close(); + producer1.close(); + @Cleanup + Consumer consumer = pulsar.getClient().newConsumer(Schema.STRING) + .topic(slaMonitorTopic) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionName("test") + .subscribe(); + // receive message t1 t2 t3 + assertEquals(consumer.receive().getValue(), "t1"); + assertEquals(consumer.receive().getValue(), "t2"); + assertEquals(consumer.receive().getValue(), "t3"); + } + } + + pulsar.getAdminClient().brokers() + .deleteDynamicConfiguration("loadBalancerServiceUnitTableViewSyncer"); + makeSecondaryAsLeader(); + Awaitility.waitAtMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> assertFalse(primaryLoadManager.getServiceUnitStateTableViewSyncer().isActive())); + Awaitility.waitAtMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> assertFalse(secondaryLoadManager.getServiceUnitStateTableViewSyncer().isActive())); } - @Test - public void testRoleChange() - throws Exception { + private void assertLookupHeartbeatOwner(PulsarService pulsar, + String brokerId, + String expectedBrokerServiceUrl) throws Exception { + NamespaceName heartbeatNamespaceV1 = + getHeartbeatNamespace(brokerId, pulsar.getConfiguration()); + + String heartbeatV1Topic = heartbeatNamespaceV1.getPersistentTopicName("test"); + assertEquals(pulsar.getAdminClient().lookups().lookupTopic(heartbeatV1Topic), expectedBrokerServiceUrl); + + NamespaceName heartbeatNamespaceV2 = + getHeartbeatNamespaceV2(brokerId, pulsar.getConfiguration()); + + String heartbeatV2Topic = heartbeatNamespaceV2.getPersistentTopicName("test"); + assertEquals(pulsar.getAdminClient().lookups().lookupTopic(heartbeatV2Topic), expectedBrokerServiceUrl); + } + + private void assertLookupSLANamespaceOwner(PulsarService pulsar, + String brokerId, + String expectedBrokerServiceUrl) throws Exception { + NamespaceName slaMonitorNamespace = getSLAMonitorNamespace(brokerId, pulsar.getConfiguration()); + String slaMonitorTopic = slaMonitorNamespace.getPersistentTopicName("test"); + String result = pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic); + log.info("Topic {} Lookup result: {}", slaMonitorTopic, result); + assertNotNull(result); + assertEquals(result, expectedBrokerServiceUrl); + } + + + private void makePrimaryAsLeader() throws Exception { + log.info("makePrimaryAsLeader"); + if (channel2.isChannelOwner()) { + pulsar2.getLeaderElectionService().close(); + Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + assertTrue(channel1.isChannelOwner()); + }); + pulsar2.getLeaderElectionService().start(); + } + Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + assertTrue(channel1.isChannelOwner()); + }); + Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + assertFalse(channel2.isChannelOwner()); + }); + } - var topBundlesLoadDataStorePrimary = (LoadDataStore) - FieldUtils.readDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", true); + private void makeSecondaryAsLeader() throws Exception { + log.info("makeSecondaryAsLeader"); + if (channel1.isChannelOwner()) { + pulsar1.getLeaderElectionService().close(); + Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + assertTrue(channel2.isChannelOwner()); + }); + pulsar1.getLeaderElectionService().start(); + } + + Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + assertTrue(channel2.isChannelOwner()); + }); + Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + assertFalse(channel1.isChannelOwner()); + }); + } + + @Test(timeOut = 30 * 1000, priority = 2100) + public void testRoleChangeIdempotency() throws Exception { + + makePrimaryAsLeader(); + + var topBundlesLoadDataStorePrimary = + (TableViewLoadDataStoreImpl) primaryLoadManager.getTopBundlesLoadDataStore(); var topBundlesLoadDataStorePrimarySpy = spy(topBundlesLoadDataStorePrimary); AtomicInteger countPri = new AtomicInteger(3); AtomicInteger countPri2 = new AtomicInteger(3); @@ -626,21 +1538,18 @@ public void testRoleChange() throw new RuntimeException(); } // Call the real method - reset(); - return null; + return invocationOnMock.callRealMethod(); }).when(topBundlesLoadDataStorePrimarySpy).startTableView(); doAnswer(invocationOnMock -> { if (countPri2.decrementAndGet() > 0) { throw new RuntimeException(); } // Call the real method - reset(); - return null; + return invocationOnMock.callRealMethod(); }).when(topBundlesLoadDataStorePrimarySpy).closeTableView(); - FieldUtils.writeDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", topBundlesLoadDataStorePrimarySpy, true); - var topBundlesLoadDataStoreSecondary = (LoadDataStore) - FieldUtils.readDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", true); + var topBundlesLoadDataStoreSecondary = + (TableViewLoadDataStoreImpl) secondaryLoadManager.getTopBundlesLoadDataStore(); var topBundlesLoadDataStoreSecondarySpy = spy(topBundlesLoadDataStoreSecondary); AtomicInteger countSec = new AtomicInteger(3); AtomicInteger countSec2 = new AtomicInteger(3); @@ -648,53 +1557,137 @@ public void testRoleChange() if (countSec.decrementAndGet() > 0) { throw new RuntimeException(); } - // Call the real method - reset(); - return null; + return invocationOnMock.callRealMethod(); }).when(topBundlesLoadDataStoreSecondarySpy).startTableView(); doAnswer(invocationOnMock -> { if (countSec2.decrementAndGet() > 0) { throw new RuntimeException(); } // Call the real method - reset(); - return null; + return invocationOnMock.callRealMethod(); }).when(topBundlesLoadDataStoreSecondarySpy).closeTableView(); - FieldUtils.writeDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", topBundlesLoadDataStoreSecondarySpy, true); - if (channel1.isChannelOwnerAsync().get(5, TimeUnit.SECONDS)) { - primaryLoadManager.playFollower(); - primaryLoadManager.playFollower(); - secondaryLoadManager.playLeader(); - secondaryLoadManager.playLeader(); - primaryLoadManager.playLeader(); + try { + FieldUtils.writeDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", + topBundlesLoadDataStorePrimarySpy, true); + FieldUtils.writeDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", + topBundlesLoadDataStoreSecondarySpy, true); + primaryLoadManager.playLeader(); secondaryLoadManager.playFollower(); + verify(topBundlesLoadDataStorePrimarySpy, times(3)).startTableView(); + verify(topBundlesLoadDataStorePrimarySpy, times(5)).closeTableView(); + verify(topBundlesLoadDataStoreSecondarySpy, times(0)).startTableView(); + verify(topBundlesLoadDataStoreSecondarySpy, times(3)).closeTableView(); + + } finally { + FieldUtils.writeDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", + topBundlesLoadDataStorePrimary, true); + FieldUtils.writeDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", + topBundlesLoadDataStoreSecondary, true); + } + + + primaryLoadManager.playFollower(); secondaryLoadManager.playFollower(); - } else { - primaryLoadManager.playLeader(); + assertEquals(ExtensibleLoadManagerImpl.Role.Leader, + primaryLoadManager.getRole()); + assertEquals(ExtensibleLoadManagerImpl.Role.Follower, + secondaryLoadManager.getRole()); + + primaryLoadManager.playLeader(); - secondaryLoadManager.playFollower(); - secondaryLoadManager.playFollower(); - primaryLoadManager.playFollower(); - primaryLoadManager.playFollower(); - secondaryLoadManager.playLeader(); secondaryLoadManager.playLeader(); - } + assertEquals(ExtensibleLoadManagerImpl.Role.Leader, + primaryLoadManager.getRole()); + assertEquals(ExtensibleLoadManagerImpl.Role.Follower, + secondaryLoadManager.getRole()); + + + } + @Test(timeOut = 30 * 1000, priority = 2000) + public void testRoleChange() throws Exception { + makePrimaryAsLeader(); + + var leader = primaryLoadManager; + var follower = secondaryLoadManager; + + BrokerLoadData brokerLoadExpected = new BrokerLoadData(); + SystemResourceUsage usage = new SystemResourceUsage(); + var cpu = new ResourceUsage(1.0, 100.0); + String key = "b1"; + usage.setCpu(cpu); + brokerLoadExpected.update(usage, 0, 0, 0, 0, 0, 0, conf); + String bundle = "public/default/0x00000000_0xffffffff"; + TopBundlesLoadData topBundlesExpected = new TopBundlesLoadData(); + topBundlesExpected.getTopBundlesLoadData().clear(); + topBundlesExpected.getTopBundlesLoadData().add(new TopBundlesLoadData.BundleLoadData(bundle, new NamespaceBundleStats())); + + Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + + assertNotNull(FieldUtils.readDeclaredField(leader.getTopBundlesLoadDataStore(), "tableView", true)); + assertNull(FieldUtils.readDeclaredField(follower.getTopBundlesLoadDataStore(), "tableView", true)); + + + for (String internalTopic : ExtensibleLoadManagerImpl.INTERNAL_TOPICS) { + if (serviceUnitStateTableViewClassName + .equals(ServiceUnitStateMetadataStoreTableViewImpl.class.getCanonicalName()) + && internalTopic.equals(TOPIC)) { + continue; + } + assertTrue(leader.pulsar.getBrokerService().getTopicReference(internalTopic) + .isPresent()); + assertTrue(follower.pulsar.getBrokerService().getTopicReference(internalTopic) + .isEmpty()); + + assertTrue(leader.pulsar.getNamespaceService() + .isServiceUnitOwnedAsync(TopicName.get(internalTopic)).get()); + assertFalse(follower.pulsar.getNamespaceService() + .isServiceUnitOwnedAsync(TopicName.get(internalTopic)).get()); + } + }); + follower.getBrokerLoadDataStore().pushAsync(key, brokerLoadExpected).get(3, TimeUnit.SECONDS); + follower.getTopBundlesLoadDataStore().pushAsync(bundle, topBundlesExpected).get(3, TimeUnit.SECONDS); + + makeSecondaryAsLeader(); + + var leader2 = secondaryLoadManager; + var follower2 = primaryLoadManager; + brokerLoadExpected.update(usage, 1, 0, 0, 0, 0, 0, conf); + topBundlesExpected.getTopBundlesLoadData().get(0).stats().msgRateIn = 1; - verify(topBundlesLoadDataStorePrimarySpy, times(3)).startTableView(); - verify(topBundlesLoadDataStorePrimarySpy, times(3)).closeTableView(); - verify(topBundlesLoadDataStoreSecondarySpy, times(3)).startTableView(); - verify(topBundlesLoadDataStoreSecondarySpy, times(3)).closeTableView(); + Awaitility.await().atMost(30, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> { + assertNotNull(FieldUtils.readDeclaredField(leader2.getTopBundlesLoadDataStore(), "tableView", true)); + assertNull(FieldUtils.readDeclaredField(follower2.getTopBundlesLoadDataStore(), "tableView", true)); + + for (String internalTopic : ExtensibleLoadManagerImpl.INTERNAL_TOPICS) { + if (serviceUnitStateTableViewClassName + .equals(ServiceUnitStateMetadataStoreTableViewImpl.class.getCanonicalName()) + && internalTopic.equals(TOPIC)) { + continue; + } + assertTrue(leader2.pulsar.getBrokerService().getTopicReference(internalTopic) + .isPresent()); + assertTrue(follower2.pulsar.getBrokerService().getTopicReference(internalTopic) + .isEmpty()); + + assertTrue(leader2.pulsar.getNamespaceService() + .isServiceUnitOwnedAsync(TopicName.get(internalTopic)).get()); + assertFalse(follower2.pulsar.getNamespaceService() + .isServiceUnitOwnedAsync(TopicName.get(internalTopic)).get()); + } + }); - FieldUtils.writeDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", topBundlesLoadDataStorePrimary, true); - FieldUtils.writeDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", topBundlesLoadDataStoreSecondary, true); + follower2.getBrokerLoadDataStore().pushAsync(key, brokerLoadExpected).get(3, TimeUnit.SECONDS); + follower2.getTopBundlesLoadDataStore().pushAsync(bundle, topBundlesExpected).get(3, TimeUnit.SECONDS); } @Test public void testGetMetrics() throws Exception { { + ServiceConfiguration conf = getDefaultConf(); + conf.setLoadBalancerMemoryResourceWeight(1); var brokerLoadDataReporter = mock(BrokerLoadDataReporter.class); FieldUtils.writeDeclaredField(primaryLoadManager, "brokerLoadDataReporter", brokerLoadDataReporter, true); BrokerLoadData loadData = new BrokerLoadData(); @@ -805,6 +1798,12 @@ SplitDecision.Reason.Unknown, new AtomicLong(6)) FieldUtils.writeDeclaredField(channel1, "handlerCounters", handlerCounters, true); } + primaryLoadManager.getIgnoredSendMsgCount().incrementAndGet(); + primaryLoadManager.getIgnoredSendMsgCount().incrementAndGet(); + primaryLoadManager.getIgnoredAckCount().incrementAndGet(); + primaryLoadManager.getIgnoredAckCount().incrementAndGet(); + primaryLoadManager.getIgnoredAckCount().incrementAndGet(); + var expected = Set.of( """ dimensions=[{broker=localhost, metric=loadBalancing}], metrics=[{brk_lb_bandwidth_in_usage=3.0, brk_lb_bandwidth_out_usage=4.0, brk_lb_cpu_usage=1.0, brk_lb_directMemory_usage=2.0, brk_lb_memory_usage=400.0}] @@ -875,31 +1874,37 @@ SplitDecision.Reason.Unknown, new AtomicLong(6)) dimensions=[{broker=localhost, metric=sunitStateChn, result=Schedule}], metrics=[{brk_sunit_state_chn_inactive_broker_cleanup_ops_total=5}] dimensions=[{broker=localhost, metric=sunitStateChn, result=Success}], metrics=[{brk_sunit_state_chn_inactive_broker_cleanup_ops_total=1}] dimensions=[{broker=localhost, metric=sunitStateChn}], metrics=[{brk_sunit_state_chn_orphan_su_cleanup_ops_total=3, brk_sunit_state_chn_owned_su_total=10, brk_sunit_state_chn_su_tombstone_cleanup_ops_total=2}] + dimensions=[{broker=localhost, metric=bundleUnloading}], metrics=[{brk_lb_ignored_ack_total=3, brk_lb_ignored_send_total=2}] """.split("\n")); - var actual = primaryLoadManager.getMetrics().stream().map(m -> m.toString()).collect(Collectors.toSet()); + var actual = primaryLoadManager.getMetrics().stream().map(Metrics::toString).collect(Collectors.toSet()); assertEquals(actual, expected); } - @Test + @Test(priority = 100) public void testDisableBroker() throws Exception { // Test rollback to modular load manager. ServiceConfiguration defaultConf = getDefaultConf(); defaultConf.setAllowAutoTopicCreation(true); defaultConf.setForceDeleteNamespaceAllowed(true); defaultConf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); + defaultConf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); defaultConf.setLoadBalancerSheddingEnabled(false); + defaultConf.setLoadBalancerDebugModeEnabled(true); + defaultConf.setTopicLevelPoliciesEnabled(false); + defaultConf.setLoadManagerServiceUnitStateTableViewClassName(serviceUnitStateTableViewClassName); try (var additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf)) { + @Cleanup var pulsar3 = additionalPulsarTestContext.getPulsarService(); ExtensibleLoadManagerImpl ternaryLoadManager = spy((ExtensibleLoadManagerImpl) FieldUtils.readField(pulsar3.getLoadManager().get(), "loadManager", true)); - String topic = "persistent://public/default/test"; + String topic = "persistent://" + defaultTestNamespace +"/test"; String lookupResult1 = pulsar3.getAdminClient().lookups().lookupTopic(topic); - TopicName topicName = TopicName.get("test"); + TopicName topicName = TopicName.get(topic); NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); if (!pulsar3.getBrokerServiceUrl().equals(lookupResult1)) { admin.namespaces().unloadNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange(), - pulsar3.getLookupServiceAddress()); + pulsar3.getBrokerId()); lookupResult1 = pulsar2.getAdminClient().lookups().lookupTopic(topic); } String lookupResult2 = pulsar1.getAdminClient().lookups().lookupTopic(topic); @@ -925,36 +1930,203 @@ public void testDisableBroker() throws Exception { } } - private static abstract class MockBrokerFilter implements BrokerFilter { + @Test(timeOut = 30 * 1000) + public void testListTopic() throws Exception { + final String namespace = "public/testListTopic"; + admin.namespaces().createNamespace(namespace, 9); + + final String persistentTopicName = TopicName.get( + "persistent", NamespaceName.get(namespace), + "get_topics_mode_" + UUID.randomUUID()).toString(); + + final String nonPersistentTopicName = TopicName.get( + "non-persistent", NamespaceName.get(namespace), + "get_topics_mode_" + UUID.randomUUID()).toString(); + admin.topics().createPartitionedTopic(persistentTopicName, 9); + admin.topics().createPartitionedTopic(nonPersistentTopicName, 9); + pulsarClient.newProducer().topic(persistentTopicName).create().close(); + pulsarClient.newProducer().topic(nonPersistentTopicName).create().close(); - @Override - public String name() { - return "Mock-broker-filter"; + BundlesData bundlesData = admin.namespaces().getBundles(namespace); + List boundaries = bundlesData.getBoundaries(); + int topicNum = 0; + for (int i = 0; i < boundaries.size() - 1; i++) { + String bundle = String.format("%s_%s", boundaries.get(i), boundaries.get(i + 1)); + List topic = admin.topics().getListInBundle(namespace, bundle); + if (topic == null) { + continue; + } + topicNum += topic.size(); + for (String s : topic) { + assertFalse(TopicName.get(s).isPersistent()); + } } + assertEquals(topicNum, 9); + List list = admin.topics().getList(namespace); + assertEquals(list.size(), 18); + admin.namespaces().deleteNamespace(namespace, true); } - private void setPrimaryLoadManager() throws IllegalAccessException { - ExtensibleLoadManagerWrapper wrapper = - (ExtensibleLoadManagerWrapper) pulsar1.getLoadManager().get(); - primaryLoadManager = spy((ExtensibleLoadManagerImpl) - FieldUtils.readField(wrapper, "loadManager", true)); - FieldUtils.writeField(wrapper, "loadManager", primaryLoadManager, true); - channel1 = (ServiceUnitStateChannelImpl) - FieldUtils.readField(primaryLoadManager, "serviceUnitStateChannel", true); + @Test(timeOut = 30 * 1000, priority = -1) + public void testGetOwnedServiceUnitsAndGetOwnedNamespaceStatus() throws Exception { + NamespaceName heartbeatNamespacePulsar1V1 = + getHeartbeatNamespace(pulsar1.getBrokerId(), pulsar1.getConfiguration()); + NamespaceName heartbeatNamespacePulsar1V2 = + NamespaceService.getHeartbeatNamespaceV2(pulsar1.getBrokerId(), pulsar1.getConfiguration()); + + NamespaceName heartbeatNamespacePulsar2V1 = + getHeartbeatNamespace(pulsar2.getBrokerId(), pulsar2.getConfiguration()); + NamespaceName heartbeatNamespacePulsar2V2 = + NamespaceService.getHeartbeatNamespaceV2(pulsar2.getBrokerId(), pulsar2.getConfiguration()); + + NamespaceName slaMonitorNamespacePulsar1 = + getSLAMonitorNamespace(pulsar1.getBrokerId(), pulsar1.getConfiguration()); + + NamespaceName slaMonitorNamespacePulsar2 = + getSLAMonitorNamespace(pulsar2.getBrokerId(), pulsar2.getConfiguration()); + + NamespaceBundle bundle1 = pulsar1.getNamespaceService().getNamespaceBundleFactory() + .getFullBundle(heartbeatNamespacePulsar1V1); + NamespaceBundle bundle2 = pulsar1.getNamespaceService().getNamespaceBundleFactory() + .getFullBundle(heartbeatNamespacePulsar1V2); + + NamespaceBundle bundle3 = pulsar2.getNamespaceService().getNamespaceBundleFactory() + .getFullBundle(heartbeatNamespacePulsar2V1); + NamespaceBundle bundle4 = pulsar2.getNamespaceService().getNamespaceBundleFactory() + .getFullBundle(heartbeatNamespacePulsar2V2); + + NamespaceBundle slaBundle1 = pulsar1.getNamespaceService().getNamespaceBundleFactory() + .getFullBundle(slaMonitorNamespacePulsar1); + NamespaceBundle slaBundle2 = pulsar2.getNamespaceService().getNamespaceBundleFactory() + .getFullBundle(slaMonitorNamespacePulsar2); + + + Set ownedServiceUnitsByPulsar1 = primaryLoadManager.getOwnedServiceUnits(); + log.info("Owned service units: {}", ownedServiceUnitsByPulsar1); + // heartbeat namespace bundle will own by pulsar1 + assertTrue(ownedServiceUnitsByPulsar1.contains(bundle1)); + assertTrue(ownedServiceUnitsByPulsar1.contains(bundle2)); + assertTrue(ownedServiceUnitsByPulsar1.contains(slaBundle1)); + Set ownedServiceUnitsByPulsar2 = secondaryLoadManager.getOwnedServiceUnits(); + log.info("Owned service units: {}", ownedServiceUnitsByPulsar2); + assertTrue(ownedServiceUnitsByPulsar2.contains(bundle3)); + assertTrue(ownedServiceUnitsByPulsar2.contains(bundle4)); + assertTrue(ownedServiceUnitsByPulsar2.contains(slaBundle2)); + Map ownedNamespacesByPulsar1 = + admin.brokers().getOwnedNamespaces(conf.getClusterName(), pulsar1.getBrokerId()); + Map ownedNamespacesByPulsar2 = + admin.brokers().getOwnedNamespaces(conf.getClusterName(), pulsar2.getBrokerId()); + assertTrue(ownedNamespacesByPulsar1.containsKey(bundle1.toString())); + assertTrue(ownedNamespacesByPulsar1.containsKey(bundle2.toString())); + assertTrue(ownedNamespacesByPulsar1.containsKey(slaBundle1.toString())); + + assertTrue(ownedNamespacesByPulsar2.containsKey(bundle3.toString())); + assertTrue(ownedNamespacesByPulsar2.containsKey(bundle4.toString())); + assertTrue(ownedNamespacesByPulsar2.containsKey(slaBundle2.toString())); + + String topic = "persistent://" + defaultTestNamespace + "/test-get-owned-service-units"; + admin.topics().createPartitionedTopic(topic, 1); + NamespaceBundle bundle = getBundleAsync(pulsar1, TopicName.get(topic)).join(); + CompletableFuture> owner = primaryLoadManager.assign(Optional.empty(), bundle, LookupOptions.builder().build()); + assertFalse(owner.join().isEmpty()); + + BrokerLookupData brokerLookupData = owner.join().get(); + if (brokerLookupData.getWebServiceUrl().equals(pulsar1.getWebServiceAddress())) { + assertOwnedServiceUnits(pulsar1, primaryLoadManager, bundle); + } else { + assertOwnedServiceUnits(pulsar2, secondaryLoadManager, bundle); + } } - private void setSecondaryLoadManager() throws IllegalAccessException { - ExtensibleLoadManagerWrapper wrapper = - (ExtensibleLoadManagerWrapper) pulsar2.getLoadManager().get(); - secondaryLoadManager = spy((ExtensibleLoadManagerImpl) - FieldUtils.readField(wrapper, "loadManager", true)); - FieldUtils.writeField(wrapper, "loadManager", secondaryLoadManager, true); - channel2 = (ServiceUnitStateChannelImpl) - FieldUtils.readField(secondaryLoadManager, "serviceUnitStateChannel", true); + private void assertOwnedServiceUnits( + PulsarService pulsar, + ExtensibleLoadManagerImpl extensibleLoadManager, + NamespaceBundle bundle) throws PulsarAdminException { + Awaitility.await().untilAsserted(() -> { + Set ownedBundles = extensibleLoadManager.getOwnedServiceUnits(); + assertTrue(ownedBundles.contains(bundle)); + }); + Map ownedNamespaces = + admin.brokers().getOwnedNamespaces(conf.getClusterName(), pulsar.getBrokerId()); + assertTrue(ownedNamespaces.containsKey(bundle.toString())); + NamespaceOwnershipStatus status = ownedNamespaces.get(bundle.toString()); + assertTrue(status.is_active); + assertFalse(status.is_controlled); + assertEquals(status.broker_assignment, BrokerAssignment.shared); } - private CompletableFuture getBundleAsync(PulsarService pulsar, TopicName topic) { - return pulsar.getNamespaceService().getBundleAsync(topic); + @Test(timeOut = 30 * 1000) + public void testGetOwnedServiceUnitsWhenLoadManagerNotStart() + throws Exception { + ExtensibleLoadManagerImpl loadManager = new ExtensibleLoadManagerImpl(); + Set ownedServiceUnits = loadManager.getOwnedServiceUnits(); + assertNotNull(ownedServiceUnits); + assertTrue(ownedServiceUnits.isEmpty()); + } + + @Test(timeOut = 30 * 1000) + public void testTryAcquiringOwnership() + throws PulsarAdminException, ExecutionException, InterruptedException { + final String namespace = "public/testTryAcquiringOwnership"; + admin.namespaces().createNamespace(namespace, 1); + String topic = "persistent://" + namespace + "/test"; + NamespaceBundle bundle = getBundleAsync(pulsar1, TopicName.get(topic)).get(); + NamespaceEphemeralData namespaceEphemeralData = primaryLoadManager.tryAcquiringOwnership(bundle).get(); + assertTrue(Set.of(pulsar1.getBrokerServiceUrl(), pulsar2.getBrokerServiceUrl()) + .contains(namespaceEphemeralData.getNativeUrl())); + Awaitility.await() + .atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + admin.namespaces().deleteNamespace(namespace, true); + }); + } + + @Test(timeOut = 30 * 1000) + public void testHealthcheck() throws PulsarAdminException { + admin.brokers().healthcheck(TopicVersion.V2); + } + + @Test(timeOut = 30 * 1000) + public void compactionScheduleTest() { + if (serviceUnitStateTableViewClassName.equals( + ServiceUnitStateMetadataStoreTableViewImpl.class.getCanonicalName())) { + // no topic compaction happens + return; + } + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(30, TimeUnit.SECONDS) + .ignoreExceptions() + .untilAsserted(() -> { // wait until true + primaryLoadManager.monitor(); + secondaryLoadManager.monitor(); + var threshold = admin.topicPolicies() + .getCompactionThreshold(TOPIC, false); + AssertJUnit.assertEquals(5 * 1024 * 1024, threshold == null ? 0 : threshold.longValue()); + }); + } + + @Test(timeOut = 30 * 1000) + public void testMonitorBrokerRegistry() throws MetadataStoreException { + primaryLoadManager.getBrokerRegistry().unregister(); + assertFalse(primaryLoadManager.getBrokerRegistry().isRegistered()); + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(30, TimeUnit.SECONDS) + .ignoreExceptions() + .untilAsserted(() -> { // wait until true + primaryLoadManager.monitor(); + assertTrue(primaryLoadManager.getBrokerRegistry().isRegistered()); + }); + } + + private static abstract class MockBrokerFilter implements BrokerFilter { + + @Override + public String name() { + return "Mock-broker-filter"; + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplWithAdvertisedListenersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplWithAdvertisedListenersTest.java new file mode 100644 index 0000000000000..b9c945fe81571 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplWithAdvertisedListenersTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions; + +import static org.apache.pulsar.common.util.PortManager.nextLockedFreePort; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.common.naming.TopicDomain; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Factory; +import org.testng.annotations.Test; + +/** + * Unit test for {@link ExtensibleLoadManagerImpl with AdvertisedListeners broker configs}. + */ +@Slf4j +@Test(groups = "flaky") +@SuppressWarnings("unchecked") +public class ExtensibleLoadManagerImplWithAdvertisedListenersTest extends ExtensibleLoadManagerImplBaseTest { + + public String brokerServiceUrl; + + @Factory(dataProvider = "serviceUnitStateTableViewClassName") + public ExtensibleLoadManagerImplWithAdvertisedListenersTest(String serviceUnitStateTableViewClassName) { + super("public/test", serviceUnitStateTableViewClassName); + } + + @Override + protected ServiceConfiguration updateConfig(ServiceConfiguration conf) { + super.updateConfig(conf); + int privatePulsarPort = nextLockedFreePort(); + int publicPulsarPort = nextLockedFreePort(); + conf.setInternalListenerName("internal"); + conf.setBindAddresses("external:pulsar://localhost:" + publicPulsarPort); + conf.setAdvertisedListeners( + "external:pulsar://localhost:" + publicPulsarPort + + ",internal:pulsar://localhost:" + privatePulsarPort); + conf.setWebServicePortTls(Optional.empty()); + conf.setBrokerServicePortTls(Optional.empty()); + conf.setBrokerServicePort(Optional.of(privatePulsarPort)); + conf.setWebServicePort(Optional.of(0)); + brokerServiceUrl = conf.getBindAddresses().replaceAll("external:", ""); + return conf; + } + + @DataProvider(name = "isPersistentTopicSubscriptionTypeTest") + public Object[][] isPersistentTopicSubscriptionTypeTest() { + return new Object[][]{ + {TopicDomain.non_persistent, SubscriptionType.Exclusive}, + {TopicDomain.persistent, SubscriptionType.Key_Shared} + }; + } + + @Test(timeOut = 30_000, dataProvider = "isPersistentTopicSubscriptionTypeTest") + public void testTransferClientReconnectionWithoutLookup(TopicDomain topicDomain, SubscriptionType subscriptionType) + throws Exception { + ExtensibleLoadManagerImplTest.testTransferClientReconnectionWithoutLookup( + clients, + topicDomain, subscriptionType, + defaultTestNamespace, admin, + brokerServiceUrl, + pulsar1, pulsar2, primaryLoadManager, secondaryLoadManager); + } + + @Test(timeOut = 30 * 1000, dataProvider = "isPersistentTopicSubscriptionTypeTest") + public void testUnloadClientReconnectionWithLookup(TopicDomain topicDomain, + SubscriptionType subscriptionType) throws Exception { + ExtensibleLoadManagerImplTest.testUnloadClientReconnectionWithLookup( + clients, + topicDomain, subscriptionType, + defaultTestNamespace, admin, + brokerServiceUrl, + pulsar1); + } + + @DataProvider(name = "isPersistentTopicTest") + public Object[][] isPersistentTopicTest() { + return new Object[][]{{TopicDomain.persistent}, {TopicDomain.non_persistent}}; + } + + @Test(timeOut = 30 * 1000, dataProvider = "isPersistentTopicTest") + public void testOptimizeUnloadDisable(TopicDomain topicDomain) throws Exception { + ExtensibleLoadManagerImplTest.testOptimizeUnloadDisable( + clients, + topicDomain, defaultTestNamespace, admin, + brokerServiceUrl, pulsar1, pulsar2); + } + +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplWithTransactionCoordinatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplWithTransactionCoordinatorTest.java new file mode 100644 index 0000000000000..1d3f02f4e717d --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplWithTransactionCoordinatorTest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions; + +import static org.testng.Assert.assertEquals; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.awaitility.Awaitility; +import org.testng.annotations.Factory; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class ExtensibleLoadManagerImplWithTransactionCoordinatorTest extends ExtensibleLoadManagerImplBaseTest { + + @Factory(dataProvider = "serviceUnitStateTableViewClassName") + public ExtensibleLoadManagerImplWithTransactionCoordinatorTest(String serviceUnitStateTableViewClassName) { + super("public/test-elb-with-tx", serviceUnitStateTableViewClassName); + } + + @Override + protected ServiceConfiguration updateConfig(ServiceConfiguration conf) { + conf = super.updateConfig(conf); + conf.setTransactionCoordinatorEnabled(true); + return conf; + } + + @Test(timeOut = 30 * 1000) + public void testUnloadAdminAPI() throws Exception { + var topicAndBundle = getBundleIsNotOwnByChangeEventTopic("test-unload"); + var topicName = topicAndBundle.getLeft(); + var bundle = topicAndBundle.getRight(); + + var srcBroker = admin.lookups().lookupTopic(topicName.toString()); + var dstBroker = srcBroker.equals(pulsar1.getBrokerServiceUrl()) ? pulsar2 : pulsar1; + var dstBrokerUrl = dstBroker.getBrokerId(); + var dstBrokerServiceUrl = dstBroker.getBrokerServiceUrl(); + + admin.namespaces().unloadNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange(), dstBrokerUrl); + Awaitility.await().untilAsserted( + () -> assertEquals(admin.lookups().lookupTopic(topicName.toString()), dstBrokerServiceUrl)); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/LoadManagerFailFastTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/LoadManagerFailFastTest.java new file mode 100644 index 0000000000000..a400bf733e557 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/LoadManagerFailFastTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions; + +import java.util.Optional; +import lombok.Cleanup; +import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.LoadManager; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; +import org.apache.pulsar.common.util.PortManager; +import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.awaitility.Awaitility; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +public class LoadManagerFailFastTest { + + private static final String cluster = "test"; + private final int zkPort = PortManager.nextLockedFreePort(); + private final LocalBookkeeperEnsemble bk = new LocalBookkeeperEnsemble(2, zkPort, PortManager::nextLockedFreePort); + private final ServiceConfiguration config = new ServiceConfiguration(); + + @BeforeClass + protected void setup() throws Exception { + bk.start(); + config.setClusterName(cluster); + config.setAdvertisedAddress("localhost"); + config.setBrokerServicePort(Optional.of(0)); + config.setWebServicePort(Optional.of(0)); + config.setMetadataStoreUrl("zk:localhost:" + zkPort); + } + + @AfterClass + protected void cleanup() throws Exception { + bk.stop(); + } + + @Test(timeOut = 30000) + public void testBrokerRegistryFailure() throws Exception { + config.setLoadManagerClassName(BrokerRegistryLoadManager.class.getName()); + @Cleanup final var pulsar = new PulsarService(config); + try { + pulsar.start(); + Assert.fail(); + } catch (PulsarServerException e) { + Assert.assertNull(e.getCause()); + Assert.assertEquals(e.getMessage(), "Cannot start BrokerRegistry"); + } + Assert.assertTrue(pulsar.getLocalMetadataStore().getChildren(LoadManager.LOADBALANCE_BROKERS_ROOT).get() + .isEmpty()); + } + + @Test(timeOut = 30000) + public void testServiceUnitStateChannelFailure() throws Exception { + config.setLoadManagerClassName(ChannelLoadManager.class.getName()); + @Cleanup final var pulsar = new PulsarService(config); + try { + pulsar.start(); + Assert.fail(); + } catch (PulsarServerException e) { + Assert.assertNull(e.getCause()); + Assert.assertEquals(e.getMessage(), "Cannot start ServiceUnitStateChannel"); + } + Awaitility.await().untilAsserted(() -> Assert.assertTrue(pulsar.getLocalMetadataStore() + .getChildren(LoadManager.LOADBALANCE_BROKERS_ROOT).get().isEmpty())); + } + + private static class BrokerRegistryLoadManager extends ExtensibleLoadManagerImpl { + + @Override + protected BrokerRegistry createBrokerRegistry(PulsarService pulsar) { + final var mockBrokerRegistry = Mockito.mock(BrokerRegistryImpl.class); + try { + Mockito.doThrow(new PulsarServerException("Cannot start BrokerRegistry")).when(mockBrokerRegistry) + .start(); + } catch (PulsarServerException e) { + throw new RuntimeException(e); + } + return mockBrokerRegistry; + } + } + + private static class ChannelLoadManager extends ExtensibleLoadManagerImpl { + + @Override + protected ServiceUnitStateChannel createServiceUnitStateChannel(PulsarService pulsar) { + final var channel = Mockito.mock(ServiceUnitStateChannelImpl.class); + try { + Mockito.doThrow(new PulsarServerException("Cannot start ServiceUnitStateChannel")).when(channel) + .start(); + } catch (PulsarServerException e) { + throw new RuntimeException(e); + } + Mockito.doAnswer(__ -> null).when(channel).listen(Mockito.any()); + return channel; + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java index cb26c460f0a03..b6e38d4f6956c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java @@ -30,6 +30,7 @@ import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.EventType.Unload; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MAX_CLEAN_UP_DELAY_TIME_IN_SECS; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateTableViewImpl.TOPIC; import static org.apache.pulsar.metadata.api.extended.SessionEvent.ConnectionLost; import static org.apache.pulsar.metadata.api.extended.SessionEvent.Reconnected; import static org.apache.pulsar.metadata.api.extended.SessionEvent.SessionLost; @@ -37,9 +38,6 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; @@ -51,14 +49,21 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; +import static org.testng.Assert.fail; +import static org.testng.AssertJUnit.assertEquals; import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.AssertJUnit.assertNull; import java.lang.reflect.Field; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -66,12 +71,14 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import lombok.Cleanup; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistryImpl; @@ -81,13 +88,16 @@ import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; import org.apache.pulsar.broker.namespace.NamespaceService; +import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.testcontext.PulsarTestContext; -import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.client.api.TypedMessageBuilder; -import org.apache.pulsar.client.impl.TableViewImpl; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.apache.pulsar.client.admin.Brokers; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.TableView; +import org.apache.pulsar.common.naming.NamespaceBundle; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.metadata.api.MetadataStoreTableView; import org.apache.pulsar.metadata.api.NotificationType; import org.apache.pulsar.metadata.api.coordination.LeaderElectionState; import org.apache.pulsar.metadata.api.extended.SessionEvent; @@ -95,17 +105,23 @@ import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Factory; import org.testng.annotations.Test; @Test(groups = "broker") +@SuppressWarnings("unchecked") public class ServiceUnitStateChannelTest extends MockedPulsarServiceBaseTest { private PulsarService pulsar1; private PulsarService pulsar2; private ServiceUnitStateChannel channel1; private ServiceUnitStateChannel channel2; - private String lookupServiceAddress1; - private String lookupServiceAddress2; + private String namespaceName; + private String namespaceName2; + private String brokerId1; + private String brokerId2; + private String brokerId3; private String bundle; private String bundle1; private String bundle2; @@ -123,28 +139,58 @@ public class ServiceUnitStateChannelTest extends MockedPulsarServiceBaseTest { private BrokerRegistryImpl registry; + private PulsarAdmin pulsarAdmin; + private ExtensibleLoadManagerImpl loadManager; - @BeforeClass - @Override - protected void setup() throws Exception { + private final String serviceUnitStateTableViewClassName; + + private Brokers brokers; + + @DataProvider(name = "serviceUnitStateTableViewClassName") + public static Object[][] serviceUnitStateTableViewClassName() { + return new Object[][]{ + {ServiceUnitStateTableViewImpl.class.getName()}, + {ServiceUnitStateMetadataStoreTableViewImpl.class.getName()} + }; + } + + @Factory(dataProvider = "serviceUnitStateTableViewClassName") + public ServiceUnitStateChannelTest(String serviceUnitStateTableViewClassName) { + this.serviceUnitStateTableViewClassName = serviceUnitStateTableViewClassName; + } + + private void updateConfig(ServiceConfiguration conf) { conf.setAllowAutoTopicCreation(true); + conf.setAllowAutoTopicCreationType(TopicType.PARTITIONED); conf.setLoadBalancerDebugModeEnabled(true); conf.setBrokerServiceCompactionMonitorIntervalInSeconds(10); + conf.setLoadManagerServiceUnitStateTableViewClassName(serviceUnitStateTableViewClassName); + } + + @BeforeClass + @Override + protected void setup() throws Exception { + updateConfig(conf); super.internalSetup(conf); - admin.tenants().createTenant("pulsar", createDefaultTenantInfo()); - admin.namespaces().createNamespace("pulsar/system"); - admin.tenants().createTenant("public", createDefaultTenantInfo()); - admin.namespaces().createNamespace("public/default"); + namespaceName = "my-tenant/my-ns"; + namespaceName2 = "my-tenant/my-ns2"; + admin.tenants().createTenant("my-tenant", createDefaultTenantInfo()); + admin.namespaces().createNamespace(namespaceName); + admin.namespaces().createNamespace(namespaceName2); pulsar1 = pulsar; - registry = new BrokerRegistryImpl(pulsar); + registry = spy(new BrokerRegistryImpl(pulsar1)); + registry.start(); + pulsarAdmin = spy(pulsar.getAdminClient()); loadManagerContext = mock(LoadManagerContext.class); doReturn(mock(LoadDataStore.class)).when(loadManagerContext).brokerLoadDataStore(); doReturn(mock(LoadDataStore.class)).when(loadManagerContext).topBundleLoadDataStore(); loadManager = mock(ExtensibleLoadManagerImpl.class); - additionalPulsarTestContext = createAdditionalPulsarTestContext(getDefaultConf()); + var conf2 = getDefaultConf(); + updateConfig(conf2); + additionalPulsarTestContext = createAdditionalPulsarTestContext(conf2); pulsar2 = additionalPulsarTestContext.getPulsarService(); channel1 = createChannel(pulsar1); @@ -152,27 +198,33 @@ protected void setup() throws Exception { channel2 = createChannel(pulsar2); channel2.start(); - lookupServiceAddress1 = (String) - FieldUtils.readDeclaredField(channel1, "lookupServiceAddress", true); - lookupServiceAddress2 = (String) - FieldUtils.readDeclaredField(channel2, "lookupServiceAddress", true); - - bundle = "public/default/0x00000000_0xffffffff"; - bundle1 = "public/default/0x00000000_0xfffffff0"; - bundle2 = "public/default/0xfffffff0_0xffffffff"; - bundle3 = "public/default3/0x00000000_0xffffffff"; + brokerId1 = (String) + FieldUtils.readDeclaredField(channel1, "brokerId", true); + brokerId2 = (String) + FieldUtils.readDeclaredField(channel2, "brokerId", true); + brokerId3 = "broker-3"; + + bundle = namespaceName + "/0x00000000_0xffffffff"; + bundle1 = namespaceName + "/0x00000000_0xfffffff0"; + bundle2 = namespaceName + "/0xfffffff0_0xffffffff"; + bundle3 = namespaceName2 + "/0x00000000_0xffffffff"; childBundle1Range = "0x7fffffff_0xffffffff"; childBundle2Range = "0x00000000_0x7fffffff"; - childBundle11 = "public/default/" + childBundle1Range; - childBundle12 = "public/default/" + childBundle2Range; + childBundle11 = namespaceName + "/" + childBundle1Range; + childBundle12 = namespaceName + "/" + childBundle2Range; + + childBundle31 = namespaceName2 + "/" + childBundle1Range; + childBundle32 = namespaceName2 + "/" + childBundle2Range; - childBundle31 = "public/default3/" + childBundle1Range; - childBundle32 = "public/default3/" + childBundle2Range; + brokers = mock(Brokers.class); + doReturn(CompletableFuture.failedFuture(new RuntimeException("failed"))).when(brokers) + .healthcheckAsync(any(), any()); } @BeforeMethod protected void initChannels() throws Exception { + disableChannels(); cleanTableViews(); cleanOwnershipMonitorCounters(channel1); cleanOwnershipMonitorCounters(channel2); @@ -180,6 +232,8 @@ protected void initChannels() throws Exception { cleanOpsCounters(channel2); cleanMetadataState(channel1); cleanMetadataState(channel2); + enableChannels(); + reset(pulsarAdmin); } @@ -215,7 +269,7 @@ public void channelOwnerTest() throws Exception { assertEquals(newChannelOwner1, newChannelOwner2); assertNotEquals(channelOwner1, newChannelOwner1); - if (newChannelOwner1.equals(Optional.of(lookupServiceAddress1))) { + if (newChannelOwner1.equals(Optional.of(brokerId1))) { assertTrue(channel1.isChannelOwnerAsync().get(2, TimeUnit.SECONDS)); assertFalse(channel2.isChannelOwnerAsync().get(2, TimeUnit.SECONDS)); } else { @@ -224,13 +278,14 @@ public void channelOwnerTest() throws Exception { } } - @Test(priority = 0) + @Test(priority = 100) public void channelValidationTest() throws ExecutionException, InterruptedException, IllegalAccessException, PulsarServerException, TimeoutException { var channel = createChannel(pulsar); int errorCnt = validateChannelStart(channel); assertEquals(6, errorCnt); + @Cleanup("shutdownNow") ExecutorService executor = Executors.newSingleThreadExecutor(); Future startFuture = executor.submit(() -> { try { @@ -247,7 +302,7 @@ public void channelValidationTest() ServiceUnitStateChannelImpl.ChannelState.LeaderElectionServiceStarted, true); assertNotNull(channel.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get()); - Future closeFuture = executor.submit(()->{ + Future closeFuture = executor.submit(() -> { try { channel.close(); } catch (PulsarServerException e) { @@ -280,7 +335,7 @@ private int validateChannelStart(ServiceUnitStateChannelImpl channel) try { channel.isChannelOwnerAsync().get(2, TimeUnit.SECONDS); } catch (ExecutionException e) { - if(e.getCause() instanceof IllegalStateException){ + if (e.getCause() instanceof IllegalStateException) { errorCnt++; } } @@ -299,7 +354,7 @@ private int validateChannelStart(ServiceUnitStateChannelImpl channel) } } try { - channel.publishAssignEventAsync(bundle, lookupServiceAddress1).get(2, TimeUnit.SECONDS); + channel.publishAssignEventAsync(bundle, brokerId1).get(2, TimeUnit.SECONDS); } catch (ExecutionException e) { if (e.getCause() instanceof IllegalStateException) { errorCnt++; @@ -307,7 +362,7 @@ private int validateChannelStart(ServiceUnitStateChannelImpl channel) } try { channel.publishUnloadEventAsync( - new Unload(lookupServiceAddress1, bundle, Optional.of(lookupServiceAddress2))) + new Unload(brokerId1, bundle, Optional.of(brokerId2))) .get(2, TimeUnit.SECONDS); } catch (ExecutionException e) { if (e.getCause() instanceof IllegalStateException) { @@ -315,7 +370,7 @@ private int validateChannelStart(ServiceUnitStateChannelImpl channel) } } try { - Split split = new Split(bundle, lookupServiceAddress1, Map.of( + Split split = new Split(bundle, brokerId1, Map.of( childBundle1Range, Optional.empty(), childBundle2Range, Optional.empty())); channel.publishSplitEventAsync(split) .get(2, TimeUnit.SECONDS); @@ -327,23 +382,6 @@ private int validateChannelStart(ServiceUnitStateChannelImpl channel) return errorCnt; } - @Test(priority = 1) - public void compactionScheduleTest() { - - Awaitility.await() - .pollInterval(200, TimeUnit.MILLISECONDS) - .atMost(5, TimeUnit.SECONDS) - .untilAsserted(() -> { // wait until true - try { - var threshold = admin.topicPolicies() - .getCompactionThreshold(ServiceUnitStateChannelImpl.TOPIC, false).longValue(); - assertEquals(5 * 1024 * 1024, threshold); - } catch (Exception e) { - ; - } - }); - } - @Test(priority = 2) public void assignmentTest() throws ExecutionException, InterruptedException, IllegalAccessException, TimeoutException { @@ -356,8 +394,8 @@ public void assignmentTest() assertTrue(owner1.get().isEmpty()); assertTrue(owner2.get().isEmpty()); - var assigned1 = channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); - var assigned2 = channel2.publishAssignEventAsync(bundle, lookupServiceAddress2); + var assigned1 = channel1.publishAssignEventAsync(bundle, brokerId1); + var assigned2 = channel2.publishAssignEventAsync(bundle, brokerId2); assertNotNull(assigned1); assertNotNull(assigned2); waitUntilOwnerChanges(channel1, bundle, null); @@ -366,8 +404,8 @@ public void assignmentTest() String assignedAddr2 = assigned2.get(5, TimeUnit.SECONDS); assertEquals(assignedAddr1, assignedAddr2); - assertTrue(assignedAddr1.equals(lookupServiceAddress1) - || assignedAddr1.equals(lookupServiceAddress2), assignedAddr1); + assertTrue(assignedAddr1.equals(brokerId1) + || assignedAddr1.equals(brokerId2), assignedAddr1); var ownerAddr1 = channel1.getOwnerAsync(bundle).get(); var ownerAddr2 = channel2.getOwnerAsync(bundle).get(); @@ -390,35 +428,33 @@ public void assignmentTestWhenOneAssignmentFails() assertEquals(0, getOwnerRequests1.size()); assertEquals(0, getOwnerRequests2.size()); - var producer = (Producer) FieldUtils.readDeclaredField(channel1, - "producer", true); - var spyProducer = spy(producer); - var msg = mock(TypedMessageBuilder.class); - var future = spy(CompletableFuture.failedFuture(new RuntimeException())); - doReturn(msg).when(spyProducer).newMessage(); - doReturn(msg).when(msg).key(any()); - doReturn(msg).when(msg).value(any()); - doReturn(future).when(msg).sendAsync(); - - FieldUtils.writeDeclaredField(channel1, "producer", spyProducer, true); - - var owner1 = channel1.getOwnerAsync(bundle); - var owner2 = channel2.getOwnerAsync(bundle); - - assertTrue(owner1.get().isEmpty()); - assertTrue(owner2.get().isEmpty()); + var tableView = getTableView(channel1); + var spyTableView = spy(tableView); + var future = CompletableFuture.failedFuture(new RuntimeException()); + doReturn(future).when(spyTableView).put(any(), any()); - var owner3 = channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); - var owner4 = channel2.publishAssignEventAsync(bundle, lookupServiceAddress2); - assertTrue(owner3.isCompletedExceptionally()); - assertNotNull(owner4); - String ownerAddrOpt2 = owner4.get(5, TimeUnit.SECONDS); - assertEquals(ownerAddrOpt2, lookupServiceAddress2); - waitUntilNewOwner(channel1, bundle, lookupServiceAddress2); - assertEquals(0, getOwnerRequests1.size()); - assertEquals(0, getOwnerRequests2.size()); + try { + setTableView(channel1, spyTableView); + + var owner1 = channel1.getOwnerAsync(bundle); + var owner2 = channel2.getOwnerAsync(bundle); + + assertTrue(owner1.get().isEmpty()); + assertTrue(owner2.get().isEmpty()); + var owner3 = channel1.publishAssignEventAsync(bundle, brokerId1); + var owner4 = channel2.publishAssignEventAsync(bundle, brokerId2); + + assertTrue(owner3.isCompletedExceptionally()); + assertNotNull(owner4); + String ownerAddrOpt2 = owner4.get(5, TimeUnit.SECONDS); + assertEquals(ownerAddrOpt2, brokerId2); + waitUntilNewOwner(channel1, bundle, brokerId2); + assertEquals(0, getOwnerRequests1.size()); + assertEquals(0, getOwnerRequests2.size()); + } finally { + setTableView(channel1, tableView); + } - FieldUtils.writeDeclaredField(channel1, "producer", producer, true); } @Test(priority = 4) @@ -431,26 +467,25 @@ public void transferTest() assertTrue(owner1.get().isEmpty()); assertTrue(owner2.get().isEmpty()); - - channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); - waitUntilNewOwner(channel1, bundle, lookupServiceAddress1); - waitUntilNewOwner(channel2, bundle, lookupServiceAddress1); + channel1.publishAssignEventAsync(bundle, brokerId1); + waitUntilNewOwner(channel1, bundle, brokerId1); + waitUntilNewOwner(channel2, bundle, brokerId1); var ownerAddr1 = channel1.getOwnerAsync(bundle).get(); var ownerAddr2 = channel2.getOwnerAsync(bundle).get(); assertEquals(ownerAddr1, ownerAddr2); - assertEquals(ownerAddr1, Optional.of(lookupServiceAddress1)); + assertEquals(ownerAddr1, Optional.of(brokerId1)); - Unload unload = new Unload(lookupServiceAddress1, bundle, Optional.of(lookupServiceAddress2)); + Unload unload = new Unload(brokerId1, bundle, Optional.of(brokerId2)); channel1.publishUnloadEventAsync(unload); - waitUntilNewOwner(channel1, bundle, lookupServiceAddress2); - waitUntilNewOwner(channel2, bundle, lookupServiceAddress2); + waitUntilNewOwner(channel1, bundle, brokerId2); + waitUntilNewOwner(channel2, bundle, brokerId2); ownerAddr1 = channel1.getOwnerAsync(bundle).get(5, TimeUnit.SECONDS); ownerAddr2 = channel2.getOwnerAsync(bundle).get(5, TimeUnit.SECONDS); assertEquals(ownerAddr1, ownerAddr2); - assertEquals(ownerAddr1, Optional.of(lookupServiceAddress2)); + assertEquals(ownerAddr1, Optional.of(brokerId2)); validateHandlerCounters(channel1, 2, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0); validateHandlerCounters(channel2, 2, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0); @@ -467,104 +502,102 @@ public void transferTestWhenDestBrokerFails() assertEquals(0, getOwnerRequests1.size()); assertEquals(0, getOwnerRequests2.size()); - channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); - waitUntilNewOwner(channel1, bundle, lookupServiceAddress1); - waitUntilNewOwner(channel2, bundle, lookupServiceAddress1); + channel1.publishAssignEventAsync(bundle, brokerId1); + waitUntilNewOwner(channel1, bundle, brokerId1); + waitUntilNewOwner(channel2, bundle, brokerId1); var ownerAddr1 = channel1.getOwnerAsync(bundle).get(); var ownerAddr2 = channel2.getOwnerAsync(bundle).get(); assertEquals(ownerAddr1, ownerAddr2); - assertEquals(ownerAddr1, Optional.of(lookupServiceAddress1)); + assertEquals(ownerAddr1, Optional.of(brokerId1)); - var producer = (Producer) FieldUtils.readDeclaredField(channel1, - "producer", true); - var spyProducer = spy(producer); - var msg = mock(TypedMessageBuilder.class); + var tableView = getTableView(channel2); + var spyTableView = spy(tableView); var future = CompletableFuture.failedFuture(new RuntimeException()); - doReturn(msg).when(spyProducer).newMessage(); - doReturn(msg).when(msg).key(any()); - doReturn(msg).when(msg).value(any()); - doReturn(future).when(msg).sendAsync(); - FieldUtils.writeDeclaredField(channel2, "producer", spyProducer, true); - FieldUtils.writeDeclaredField(channel1, - "inFlightStateWaitingTimeInMillis", 3 * 1000, true); - FieldUtils.writeDeclaredField(channel2, - "inFlightStateWaitingTimeInMillis", 3 * 1000, true); - Unload unload = new Unload(lookupServiceAddress1, bundle, Optional.of(lookupServiceAddress2)); - channel1.publishUnloadEventAsync(unload); - // channel1 is broken. the ownership transfer won't be complete. - waitUntilState(channel1, bundle); - waitUntilState(channel2, bundle); - var owner1 = channel1.getOwnerAsync(bundle); - var owner2 = channel2.getOwnerAsync(bundle); - - assertFalse(owner1.isDone()); - assertFalse(owner2.isDone()); - - assertEquals(1, getOwnerRequests1.size()); - assertEquals(1, getOwnerRequests2.size()); - - // In 5 secs, the getOwnerAsync requests(lookup requests) should time out. - Awaitility.await().atMost(5, TimeUnit.SECONDS) - .untilAsserted(() -> assertTrue(owner1.isCompletedExceptionally())); - Awaitility.await().atMost(5, TimeUnit.SECONDS) - .untilAsserted(() -> assertTrue(owner2.isCompletedExceptionally())); - - assertEquals(0, getOwnerRequests1.size()); - assertEquals(0, getOwnerRequests2.size()); - - // recovered, check the monitor update state : Assigned -> Owned - doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress1))) - .when(loadManager).selectAsync(any(), any()); - FieldUtils.writeDeclaredField(channel2, "producer", producer, true); - FieldUtils.writeDeclaredField(channel1, - "inFlightStateWaitingTimeInMillis", 1 , true); - FieldUtils.writeDeclaredField(channel2, - "inFlightStateWaitingTimeInMillis", 1 , true); - - ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( - List.of(lookupServiceAddress1, lookupServiceAddress2)); - ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( - List.of(lookupServiceAddress1, lookupServiceAddress2)); - - - waitUntilNewOwner(channel1, bundle, lookupServiceAddress1); - waitUntilNewOwner(channel2, bundle, lookupServiceAddress1); - ownerAddr1 = channel1.getOwnerAsync(bundle).get(); - ownerAddr2 = channel2.getOwnerAsync(bundle).get(); - - assertEquals(ownerAddr1, ownerAddr2); - assertEquals(ownerAddr1, Optional.of(lookupServiceAddress1)); - - var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; - validateMonitorCounters(leader, - 0, - 0, - 1, - 0, - 0, - 0, - 0); + doReturn(future).when(spyTableView).put(any(), any()); + try { + setTableView(channel2, spyTableView); + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 3 * 1000, true); + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 3 * 1000, true); + Unload unload = new Unload(brokerId1, bundle, Optional.of(brokerId2)); + channel1.publishUnloadEventAsync(unload); + // channel2 is broken. the ownership transfer won't be complete. + waitUntilState(channel1, bundle); + waitUntilState(channel2, bundle); + var owner1 = channel1.getOwnerAsync(bundle); + var owner2 = channel2.getOwnerAsync(bundle); + + assertTrue(owner1.isDone()); + assertEquals(brokerId2, owner1.get().get()); + assertFalse(owner2.isDone()); + + assertEquals(0, getOwnerRequests1.size()); + assertEquals(1, getOwnerRequests2.size()); + + // In 10 secs, the getOwnerAsync requests(lookup requests) should time out. + Awaitility.await().atMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> assertTrue(owner2.isCompletedExceptionally())); + + assertEquals(0, getOwnerRequests2.size()); + + // recovered, check the monitor update state : Assigned -> Owned + doReturn(CompletableFuture.completedFuture(Optional.of(brokerId1))) + .when(loadManager).selectAsync(any(), any(), any()); + } finally { + setTableView(channel2, tableView); + } - FieldUtils.writeDeclaredField(channel1, - "inFlightStateWaitingTimeInMillis", 30 * 1000, true); - FieldUtils.writeDeclaredField(channel2, - "inFlightStateWaitingTimeInMillis", 30 * 1000, true); + try { + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 1, true); + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 1, true); + ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( + List.of(brokerId1, brokerId2)); + ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( + List.of(brokerId1, brokerId2)); + + + waitUntilNewOwner(channel1, bundle, brokerId1); + waitUntilNewOwner(channel2, bundle, brokerId1); + ownerAddr1 = channel1.getOwnerAsync(bundle).get(); + ownerAddr2 = channel2.getOwnerAsync(bundle).get(); + + assertEquals(ownerAddr1, ownerAddr2); + assertEquals(ownerAddr1, Optional.of(brokerId1)); + + var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; + validateMonitorCounters(leader, + 0, + 0, + 1, + 0, + 0, + 0, + 0); + } finally { + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); + } } @Test(priority = 6) public void splitAndRetryTest() throws Exception { - channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); - waitUntilNewOwner(channel1, bundle, lookupServiceAddress1); - waitUntilNewOwner(channel2, bundle, lookupServiceAddress1); + channel1.publishAssignEventAsync(bundle, brokerId1); + waitUntilNewOwner(channel1, bundle, brokerId1); + waitUntilNewOwner(channel2, bundle, brokerId1); var ownerAddr1 = channel1.getOwnerAsync(bundle).get(); var ownerAddr2 = channel2.getOwnerAsync(bundle).get(); - assertEquals(ownerAddr1, Optional.of(lookupServiceAddress1)); - assertEquals(ownerAddr2, Optional.of(lookupServiceAddress1)); + assertEquals(ownerAddr1, Optional.of(brokerId1)); + assertEquals(ownerAddr2, Optional.of(brokerId1)); assertTrue(ownerAddr1.isPresent()); - NamespaceService namespaceService = spy(pulsar1.getNamespaceService()); + NamespaceService namespaceService = pulsar1.getNamespaceService(); CompletableFuture future = new CompletableFuture<>(); int badVersionExceptionCount = 3; AtomicInteger count = new AtomicInteger(badVersionExceptionCount); @@ -573,11 +606,7 @@ public void splitAndRetryTest() throws Exception { if (count.decrementAndGet() > 0) { return future; } - // Call the real method - reset(namespaceService); - doReturn(CompletableFuture.completedFuture(List.of("test-topic-1", "test-topic-2"))) - .when(namespaceService).getOwnedTopicListForNamespaceBundle(any()); - return future; + return invocationOnMock.callRealMethod(); }).when(namespaceService).updateNamespaceBundles(any(), any()); doReturn(namespaceService).when(pulsar1).getNamespaceService(); doReturn(CompletableFuture.completedFuture(List.of("test-topic-1", "test-topic-2"))) @@ -589,76 +618,81 @@ public void splitAndRetryTest() throws Exception { childBundle1Range, Optional.empty(), childBundle2Range, Optional.empty())); channel1.publishSplitEventAsync(split); - waitUntilState(channel1, bundle, Deleted); - waitUntilState(channel2, bundle, Deleted); + waitUntilState(channel1, bundle, Init); + waitUntilState(channel2, bundle, Init); - validateHandlerCounters(channel1, 1, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0); - validateHandlerCounters(channel2, 1, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0); + validateHandlerCounters(channel1, 1, 0, 3, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0); + validateHandlerCounters(channel2, 1, 0, 3, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0); validateEventCounters(channel1, 1, 0, 1, 0, 0, 0); validateEventCounters(channel2, 0, 0, 0, 0, 0, 0); // Verify the retry count - verify(((ServiceUnitStateChannelImpl) channel1), times(badVersionExceptionCount + 1)) + verify(((ServiceUnitStateChannelImpl) channel1), times(badVersionExceptionCount)) .splitServiceUnitOnceAndRetry(any(), any(), any(), any(), any(), any(), any(), any(), anyLong(), any()); - - waitUntilNewOwner(channel1, childBundle11, lookupServiceAddress1); - waitUntilNewOwner(channel1, childBundle12, lookupServiceAddress1); - waitUntilNewOwner(channel2, childBundle11, lookupServiceAddress1); - waitUntilNewOwner(channel2, childBundle12, lookupServiceAddress1); - assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(childBundle11).get()); - assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(childBundle12).get()); - assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle11).get()); - assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle12).get()); + waitUntilNewOwner(channel1, childBundle11, brokerId1); + waitUntilNewOwner(channel1, childBundle12, brokerId1); + waitUntilNewOwner(channel2, childBundle11, brokerId1); + waitUntilNewOwner(channel2, childBundle12, brokerId1); + assertEquals(Optional.of(brokerId1), channel1.getOwnerAsync(childBundle11).get()); + assertEquals(Optional.of(brokerId1), channel1.getOwnerAsync(childBundle12).get()); + assertEquals(Optional.of(brokerId1), channel2.getOwnerAsync(childBundle11).get()); + assertEquals(Optional.of(brokerId1), channel2.getOwnerAsync(childBundle12).get()); // try the monitor and check the monitor moves `Deleted` -> `Init` FieldUtils.writeDeclaredField(channel1, - "inFlightStateWaitingTimeInMillis", 1 , true); + "inFlightStateWaitingTimeInMillis", 1, true); FieldUtils.writeDeclaredField(channel1, - "semiTerminalStateWaitingTimeInMillis", 1, true); + "stateTombstoneDelayTimeInMillis", 1, true); FieldUtils.writeDeclaredField(channel2, - "inFlightStateWaitingTimeInMillis", 1 , true); + "inFlightStateWaitingTimeInMillis", 1, true); FieldUtils.writeDeclaredField(channel2, - "semiTerminalStateWaitingTimeInMillis", 1, true); + "stateTombstoneDelayTimeInMillis", 1, true); ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( - List.of(lookupServiceAddress1, lookupServiceAddress2)); + List.of(brokerId1, brokerId2)); ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( - List.of(lookupServiceAddress1, lookupServiceAddress2)); + List.of(brokerId1, brokerId2)); waitUntilState(channel1, bundle, Init); waitUntilState(channel2, bundle, Init); var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; validateMonitorCounters(leader, 0, - 1, + 0, 0, 0, 0, 0, 0); - cleanTableView(channel1, childBundle11); - cleanTableView(channel2, childBundle11); - cleanTableView(channel1, childBundle12); - cleanTableView(channel2, childBundle12); + try { + disableChannels(); + overrideTableView(channel1, childBundle11, null); + overrideTableView(channel2, childBundle11, null); + overrideTableView(channel1, childBundle12, null); + overrideTableView(channel2, childBundle12, null); + } finally { + enableChannels(); + } FieldUtils.writeDeclaredField(channel1, "inFlightStateWaitingTimeInMillis", 30 * 1000, true); FieldUtils.writeDeclaredField(channel1, - "semiTerminalStateWaitingTimeInMillis", 300 * 1000, true); + "stateTombstoneDelayTimeInMillis", 300 * 1000, true); FieldUtils.writeDeclaredField(channel2, "inFlightStateWaitingTimeInMillis", 30 * 1000, true); FieldUtils.writeDeclaredField(channel2, - "semiTerminalStateWaitingTimeInMillis", 300 * 1000, true); + "stateTombstoneDelayTimeInMillis", 300 * 1000, true); } @Test(priority = 7) public void handleMetadataSessionEventTest() throws IllegalAccessException { var ts = System.currentTimeMillis(); + ServiceUnitStateChannelImpl channel1 = (ServiceUnitStateChannelImpl) this.channel1; channel1.handleMetadataSessionEvent(SessionReestablished); var lastMetadataSessionEvent = getLastMetadataSessionEvent(channel1); var lastMetadataSessionEventTimestamp = getLastMetadataSessionEventTimestamp(channel1); @@ -699,30 +733,32 @@ public void handleMetadataSessionEventTest() throws IllegalAccessException { @Test(priority = 8) public void handleBrokerCreationEventTest() throws IllegalAccessException { var cleanupJobs = getCleanupJobs(channel1); - String broker = "broker-1"; + String broker = brokerId2; var future = new CompletableFuture(); cleanupJobs.put(broker, future); - channel1.handleBrokerRegistrationEvent(broker, NotificationType.Created); - assertEquals(0, cleanupJobs.size()); - assertTrue(future.isCancelled()); + ((ServiceUnitStateChannelImpl) channel1).handleBrokerRegistrationEvent(broker, NotificationType.Created); + Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> { + assertEquals(0, cleanupJobs.size()); + assertTrue(future.isCancelled()); + }); + } @Test(priority = 9) - public void handleBrokerDeletionEventTest() - throws IllegalAccessException, ExecutionException, InterruptedException, TimeoutException { + public void handleBrokerDeletionEventTest() throws Exception { var cleanupJobs1 = getCleanupJobs(channel1); var cleanupJobs2 = getCleanupJobs(channel2); var leaderCleanupJobsTmp = spy(cleanupJobs1); var followerCleanupJobsTmp = spy(cleanupJobs2); - var leaderChannel = channel1; - var followerChannel = channel2; + ServiceUnitStateChannelImpl leaderChannel = (ServiceUnitStateChannelImpl) channel1; + ServiceUnitStateChannelImpl followerChannel = (ServiceUnitStateChannelImpl) channel2; String leader = channel1.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); String leader2 = channel2.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); assertEquals(leader, leader2); - if (leader.equals(lookupServiceAddress2)) { - leaderChannel = channel2; - followerChannel = channel1; + if (leader.equals(brokerId2)) { + leaderChannel = (ServiceUnitStateChannelImpl) channel2; + followerChannel = (ServiceUnitStateChannelImpl) channel1; var tmp = followerCleanupJobsTmp; followerCleanupJobsTmp = leaderCleanupJobsTmp; leaderCleanupJobsTmp = tmp; @@ -736,22 +772,24 @@ public void handleBrokerDeletionEventTest() var owner1 = channel1.getOwnerAsync(bundle1); var owner2 = channel2.getOwnerAsync(bundle2); - doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))) - .when(loadManager).selectAsync(any(), any()); + doReturn(CompletableFuture.completedFuture(Optional.of(brokerId2))) + .when(loadManager).selectAsync(any(), any(), any()); assertTrue(owner1.get().isEmpty()); assertTrue(owner2.get().isEmpty()); - String broker = lookupServiceAddress1; + String broker = brokerId1; channel1.publishAssignEventAsync(bundle1, broker); channel2.publishAssignEventAsync(bundle2, broker); + waitUntilNewOwner(channel1, bundle1, broker); waitUntilNewOwner(channel2, bundle1, broker); waitUntilNewOwner(channel1, bundle2, broker); waitUntilNewOwner(channel2, bundle2, broker); - channel1.publishUnloadEventAsync(new Unload(broker, bundle1, Optional.of(lookupServiceAddress2))); - waitUntilNewOwner(channel1, bundle1, lookupServiceAddress2); - waitUntilNewOwner(channel2, bundle1, lookupServiceAddress2); + // Verify to transfer the ownership to the other broker. + channel1.publishUnloadEventAsync(new Unload(broker, bundle1, Optional.of(brokerId2))); + waitUntilNewOwner(channel1, bundle1, brokerId2); + waitUntilNewOwner(channel2, bundle1, brokerId2); // test stable metadata state leaderChannel.handleMetadataSessionEvent(SessionReestablished); @@ -760,13 +798,21 @@ public void handleBrokerDeletionEventTest() System.currentTimeMillis() - (MAX_CLEAN_UP_DELAY_TIME_IN_SECS * 1000 + 1000), true); FieldUtils.writeDeclaredField(followerChannel, "lastMetadataSessionEventTimestamp", System.currentTimeMillis() - (MAX_CLEAN_UP_DELAY_TIME_IN_SECS * 1000 + 1000), true); + + doReturn(brokers).when(pulsarAdmin).brokers(); leaderChannel.handleBrokerRegistrationEvent(broker, NotificationType.Deleted); followerChannel.handleBrokerRegistrationEvent(broker, NotificationType.Deleted); - waitUntilNewOwner(channel1, bundle1, lookupServiceAddress2); - waitUntilNewOwner(channel2, bundle1, lookupServiceAddress2); - waitUntilNewOwner(channel1, bundle2, lookupServiceAddress2); - waitUntilNewOwner(channel2, bundle2, lookupServiceAddress2); + + leaderChannel.handleBrokerRegistrationEvent(brokerId2, + NotificationType.Deleted); + followerChannel.handleBrokerRegistrationEvent(brokerId2, + NotificationType.Deleted); + + waitUntilNewOwner(channel1, bundle1, brokerId2); + waitUntilNewOwner(channel2, bundle1, brokerId2); + waitUntilNewOwner(channel1, bundle2, brokerId2); + waitUntilNewOwner(channel2, bundle2, brokerId2); verify(leaderCleanupJobs, times(1)).computeIfAbsent(eq(broker), any()); verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); @@ -777,18 +823,18 @@ public void handleBrokerDeletionEventTest() }); validateMonitorCounters(leaderChannel, - 1, + 2, 0, - 1, + 3, 0, - 1, + 2, 0, 0); // test jittery metadata state - channel1.publishUnloadEventAsync(new Unload(lookupServiceAddress2, bundle1, Optional.of(broker))); - channel1.publishUnloadEventAsync(new Unload(lookupServiceAddress2, bundle2, Optional.of(broker))); + channel1.publishUnloadEventAsync(new Unload(brokerId2, bundle1, Optional.of(broker))); + channel1.publishUnloadEventAsync(new Unload(brokerId2, bundle2, Optional.of(broker))); waitUntilNewOwner(channel1, bundle1, broker); waitUntilNewOwner(channel2, bundle1, broker); waitUntilNewOwner(channel1, bundle2, broker); @@ -808,13 +854,14 @@ public void handleBrokerDeletionEventTest() }); validateMonitorCounters(leaderChannel, - 1, + 2, 0, - 1, + 3, 0, - 2, + 3, 0, 0); + reset(pulsarAdmin); // broker is back online leaderChannel.handleBrokerRegistrationEvent(broker, NotificationType.Created); @@ -829,16 +876,17 @@ public void handleBrokerDeletionEventTest() }); validateMonitorCounters(leaderChannel, - 1, + 2, 0, - 1, + 3, 0, - 2, + 3, 0, 1); // broker is offline again + doReturn(brokers).when(pulsarAdmin).brokers(); FieldUtils.writeDeclaredField(leaderChannel, "maxCleanupDelayTimeInSecs", 3, true); leaderChannel.handleBrokerRegistrationEvent(broker, NotificationType.Deleted); followerChannel.handleBrokerRegistrationEvent(broker, NotificationType.Deleted); @@ -851,19 +899,19 @@ public void handleBrokerDeletionEventTest() }); validateMonitorCounters(leaderChannel, - 1, - 0, - 1, + 2, 0, 3, 0, + 4, + 0, 1); // finally cleanup - waitUntilNewOwner(channel1, bundle1, lookupServiceAddress2); - waitUntilNewOwner(channel2, bundle1, lookupServiceAddress2); - waitUntilNewOwner(channel1, bundle2, lookupServiceAddress2); - waitUntilNewOwner(channel2, bundle2, lookupServiceAddress2); + waitUntilNewOwner(channel1, bundle1, brokerId2); + waitUntilNewOwner(channel2, bundle1, brokerId2); + waitUntilNewOwner(channel1, bundle2, brokerId2); + waitUntilNewOwner(channel2, bundle2, brokerId2); verify(leaderCleanupJobs, times(3)).computeIfAbsent(eq(broker), any()); verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); @@ -873,17 +921,18 @@ public void handleBrokerDeletionEventTest() }); validateMonitorCounters(leaderChannel, - 2, - 0, 3, 0, - 3, + 5, + 0, + 4, 0, 1); + reset(pulsarAdmin); // test unstable state - channel1.publishUnloadEventAsync(new Unload(lookupServiceAddress2, bundle1, Optional.of(broker))); - channel1.publishUnloadEventAsync(new Unload(lookupServiceAddress2, bundle2, Optional.of(broker))); + channel1.publishUnloadEventAsync(new Unload(brokerId2, bundle1, Optional.of(broker))); + channel1.publishUnloadEventAsync(new Unload(brokerId2, bundle2, Optional.of(broker))); waitUntilNewOwner(channel1, bundle1, broker); waitUntilNewOwner(channel2, bundle1, broker); waitUntilNewOwner(channel1, bundle2, broker); @@ -902,11 +951,11 @@ public void handleBrokerDeletionEventTest() }); validateMonitorCounters(leaderChannel, - 2, - 0, 3, 0, - 3, + 5, + 0, + 4, 1, 1); @@ -918,26 +967,25 @@ public void handleBrokerDeletionEventTest() true); } - @Test(priority = 10) - public void conflictAndCompactionTest() throws ExecutionException, InterruptedException, TimeoutException, - IllegalAccessException, PulsarClientException, PulsarServerException { + @Test(priority = 2000) + public void conflictAndCompactionTest() throws Exception { String bundle = String.format("%s/%s", "public/default", "0x0000000a_0xffffffff"); var owner1 = channel1.getOwnerAsync(bundle); var owner2 = channel2.getOwnerAsync(bundle); assertTrue(owner1.get().isEmpty()); assertTrue(owner2.get().isEmpty()); - var assigned1 = channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); + var assigned1 = channel1.publishAssignEventAsync(bundle, brokerId1); assertNotNull(assigned1); - waitUntilNewOwner(channel1, bundle, lookupServiceAddress1); - waitUntilNewOwner(channel2, bundle, lookupServiceAddress1); + waitUntilNewOwner(channel1, bundle, brokerId1); + waitUntilNewOwner(channel2, bundle, brokerId1); String assignedAddr1 = assigned1.get(5, TimeUnit.SECONDS); - assertEquals(lookupServiceAddress1, assignedAddr1); + assertEquals(brokerId1, assignedAddr1); FieldUtils.writeDeclaredField(channel2, "inFlightStateWaitingTimeInMillis", 3 * 1000, true); - var assigned2 = channel2.publishAssignEventAsync(bundle, lookupServiceAddress2); + var assigned2 = channel2.publishAssignEventAsync(bundle, brokerId2); assertNotNull(assigned2); Exception ex = null; try { @@ -945,69 +993,92 @@ public void conflictAndCompactionTest() throws ExecutionException, InterruptedEx } catch (CompletionException e) { ex = e; } - assertNotNull(ex); - assertEquals(TimeoutException.class, ex.getCause().getClass()); - assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(bundle).get()); - assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(bundle).get()); + assertNull(ex); + assertEquals(Optional.of(brokerId1), channel2.getOwnerAsync(bundle).get()); + assertEquals(Optional.of(brokerId1), channel1.getOwnerAsync(bundle).get()); + if (serviceUnitStateTableViewClassName.equals( + ServiceUnitStateMetadataStoreTableViewImpl.class.getCanonicalName())) { + // no compaction + return; + } - var compactor = spy (pulsar1.getStrategicCompactor()); + var compactor = spy(pulsar1.getStrategicCompactor()); Field strategicCompactorField = FieldUtils.getDeclaredField(PulsarService.class, "strategicCompactor", true); FieldUtils.writeField(strategicCompactorField, pulsar1, compactor, true); FieldUtils.writeField(strategicCompactorField, pulsar2, compactor, true); - Awaitility.await() - .pollInterval(200, TimeUnit.MILLISECONDS) - .atMost(140, TimeUnit.SECONDS) - .untilAsserted(() -> { - channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); - verify(compactor, times(1)) - .compact(eq(ServiceUnitStateChannelImpl.TOPIC), any()); - }); + + var threshold = admin.topicPolicies() + .getCompactionThreshold(TOPIC); + admin.topicPolicies() + .setCompactionThreshold(TOPIC, 0); + + try { + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(140, TimeUnit.SECONDS) + .untilAsserted(() -> { + channel1.publishAssignEventAsync(bundle, brokerId1); + verify(compactor, times(1)) + .compact(eq(TOPIC), any()); + }); + + + var channel3 = createChannel(pulsar); + channel3.start(); + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> assertEquals( + channel3.getOwnerAsync(bundle).get(), Optional.of(brokerId1))); + channel3.close(); + } finally { + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); + if (threshold != null) { + admin.topicPolicies() + .setCompactionThreshold(TOPIC, threshold); + } + } - var channel3 = createChannel(pulsar); - channel3.start(); - Awaitility.await() - .pollInterval(200, TimeUnit.MILLISECONDS) - .atMost(5, TimeUnit.SECONDS) - .untilAsserted(() -> assertEquals( - channel3.getOwnerAsync(bundle).get(), Optional.of(lookupServiceAddress1))); - channel3.close(); - FieldUtils.writeDeclaredField(channel2, - "inFlightStateWaitingTimeInMillis", 30 * 1000, true); } @Test(priority = 11) public void ownerLookupCountTests() throws IllegalAccessException { - - overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigning, "b1", 1)); - channel1.getOwnerAsync(bundle); - channel1.getOwnerAsync(bundle); - - overrideTableView(channel1, bundle, new ServiceUnitStateData(Owned, "b1", 1)); - channel1.getOwnerAsync(bundle); - channel1.getOwnerAsync(bundle); - channel1.getOwnerAsync(bundle); - - overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, "b1", 1)); - channel1.getOwnerAsync(bundle); - channel1.getOwnerAsync(bundle); - - overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, null, "b1", 1)); - channel1.getOwnerAsync(bundle); - - overrideTableView(channel1, bundle, new ServiceUnitStateData(Free, "b1", 1)); - channel1.getOwnerAsync(bundle); - - overrideTableView(channel1, bundle, new ServiceUnitStateData(Deleted, "b1", 1)); - channel1.getOwnerAsync(bundle); - channel1.getOwnerAsync(bundle); - - overrideTableView(channel1, bundle, null); - channel1.getOwnerAsync(bundle); - channel1.getOwnerAsync(bundle); - channel1.getOwnerAsync(bundle); - - validateOwnerLookUpCounters(channel1, 2, 3, 2, 1, 1, 2, 3); + try { + disableChannels(); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigning, "b1", 1)); + channel1.getOwnerAsync(bundle); + channel1.getOwnerAsync(bundle); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Owned, "b1", 1)); + channel1.getOwnerAsync(bundle); + channel1.getOwnerAsync(bundle); + channel1.getOwnerAsync(bundle); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, "b1", 1)); + channel1.getOwnerAsync(bundle); + channel1.getOwnerAsync(bundle); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, null, "b1", 1)); + channel1.getOwnerAsync(bundle); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Free, "b1", 1)); + channel1.getOwnerAsync(bundle); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Deleted, "b1", 1)); + channel1.getOwnerAsync(bundle); + channel1.getOwnerAsync(bundle); + + overrideTableView(channel1, bundle, null); + channel1.getOwnerAsync(bundle); + channel1.getOwnerAsync(bundle); + channel1.getOwnerAsync(bundle); + + validateOwnerLookUpCounters(channel1, 2, 3, 2, 1, 1, 2, 3); + } finally { + enableChannels(); + } } @@ -1015,16 +1086,16 @@ public void ownerLookupCountTests() throws IllegalAccessException { public void unloadTest() throws ExecutionException, InterruptedException, IllegalAccessException { - channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); + channel1.publishAssignEventAsync(bundle, brokerId1); - waitUntilNewOwner(channel1, bundle, lookupServiceAddress1); - waitUntilNewOwner(channel2, bundle, lookupServiceAddress1); + waitUntilNewOwner(channel1, bundle, brokerId1); + waitUntilNewOwner(channel2, bundle, brokerId1); var ownerAddr1 = channel1.getOwnerAsync(bundle).get(); var ownerAddr2 = channel2.getOwnerAsync(bundle).get(); assertEquals(ownerAddr1, ownerAddr2); - assertEquals(ownerAddr1, Optional.of(lookupServiceAddress1)); - Unload unload = new Unload(lookupServiceAddress1, bundle, Optional.empty()); + assertEquals(ownerAddr1, Optional.of(brokerId1)); + Unload unload = new Unload(brokerId1, bundle, Optional.empty()); channel1.publishUnloadEventAsync(unload); @@ -1036,17 +1107,17 @@ public void unloadTest() assertEquals(Optional.empty(), owner1.get()); assertEquals(Optional.empty(), owner2.get()); - channel2.publishAssignEventAsync(bundle, lookupServiceAddress2); + channel2.publishAssignEventAsync(bundle, brokerId2); - waitUntilNewOwner(channel1, bundle, lookupServiceAddress2); - waitUntilNewOwner(channel2, bundle, lookupServiceAddress2); + waitUntilNewOwner(channel1, bundle, brokerId2); + waitUntilNewOwner(channel2, bundle, brokerId2); ownerAddr1 = channel1.getOwnerAsync(bundle).get(); ownerAddr2 = channel2.getOwnerAsync(bundle).get(); assertEquals(ownerAddr1, ownerAddr2); - assertEquals(ownerAddr1, Optional.of(lookupServiceAddress2)); - Unload unload2 = new Unload(lookupServiceAddress2, bundle, Optional.empty()); + assertEquals(ownerAddr1, Optional.of(brokerId2)); + Unload unload2 = new Unload(brokerId2, bundle, Optional.empty()); channel2.publishUnloadEventAsync(unload2); @@ -1055,19 +1126,19 @@ public void unloadTest() // test monitor if Free -> Init FieldUtils.writeDeclaredField(channel1, - "inFlightStateWaitingTimeInMillis", 1 , true); + "inFlightStateWaitingTimeInMillis", 1, true); FieldUtils.writeDeclaredField(channel1, - "semiTerminalStateWaitingTimeInMillis", 1, true); + "stateTombstoneDelayTimeInMillis", 1, true); FieldUtils.writeDeclaredField(channel2, - "inFlightStateWaitingTimeInMillis", 1 , true); + "inFlightStateWaitingTimeInMillis", 1, true); FieldUtils.writeDeclaredField(channel2, - "semiTerminalStateWaitingTimeInMillis", 1, true); + "stateTombstoneDelayTimeInMillis", 1, true); ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( - List.of(lookupServiceAddress1, lookupServiceAddress2)); + List.of(brokerId1, brokerId2)); ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( - List.of(lookupServiceAddress1, lookupServiceAddress2)); + List.of(brokerId1, brokerId2)); waitUntilState(channel1, bundle, Init); waitUntilState(channel2, bundle, Init); @@ -1085,19 +1156,19 @@ public void unloadTest() FieldUtils.writeDeclaredField(channel1, "inFlightStateWaitingTimeInMillis", 30 * 1000, true); FieldUtils.writeDeclaredField(channel1, - "semiTerminalStateWaitingTimeInMillis", 30 * 1000, true); + "stateTombstoneDelayTimeInMillis", 30 * 1000, true); FieldUtils.writeDeclaredField(channel2, "inFlightStateWaitingTimeInMillis", 300 * 1000, true); FieldUtils.writeDeclaredField(channel2, - "semiTerminalStateWaitingTimeInMillis", 300 * 1000, true); + "stateTombstoneDelayTimeInMillis", 300 * 1000, true); } @Test(priority = 13) public void assignTestWhenDestBrokerProducerFails() throws ExecutionException, InterruptedException, IllegalAccessException { - Unload unload = new Unload(lookupServiceAddress1, bundle, Optional.empty()); + Unload unload = new Unload(brokerId1, bundle, Optional.empty()); channel1.publishUnloadEventAsync(unload); @@ -1107,58 +1178,51 @@ public void assignTestWhenDestBrokerProducerFails() assertEquals(Optional.empty(), channel1.getOwnerAsync(bundle).get()); assertEquals(Optional.empty(), channel2.getOwnerAsync(bundle).get()); - var producer = (Producer) FieldUtils.readDeclaredField(channel1, - "producer", true); - var spyProducer = spy(producer); - var msg = mock(TypedMessageBuilder.class); + var tableview = getTableView(channel1); + var tableviewSpy = spy(tableview); var future = CompletableFuture.failedFuture(new RuntimeException()); - doReturn(msg).when(spyProducer).newMessage(); - doReturn(msg).when(msg).key(any()); - doReturn(msg).when(msg).value(any()); - doReturn(future).when(msg).sendAsync(); - FieldUtils.writeDeclaredField(channel2, "producer", spyProducer, true); + doReturn(future).when(tableviewSpy).put(any(), any()); + setTableView(channel2, tableviewSpy); FieldUtils.writeDeclaredField(channel1, "inFlightStateWaitingTimeInMillis", 3 * 1000, true); FieldUtils.writeDeclaredField(channel2, "inFlightStateWaitingTimeInMillis", 3 * 1000, true); - doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))) - .when(loadManager).selectAsync(any(), any()); - channel1.publishAssignEventAsync(bundle, lookupServiceAddress2); + doReturn(CompletableFuture.completedFuture(Optional.of(brokerId2))) + .when(loadManager).selectAsync(any(), any(), any()); + channel1.publishAssignEventAsync(bundle, brokerId2); // channel1 is broken. the assign won't be complete. waitUntilState(channel1, bundle); waitUntilState(channel2, bundle); var owner1 = channel1.getOwnerAsync(bundle); var owner2 = channel2.getOwnerAsync(bundle); - assertFalse(owner1.isDone()); + assertTrue(owner1.isDone()); assertFalse(owner2.isDone()); - // In 5 secs, the getOwnerAsync requests(lookup requests) should time out. - Awaitility.await().atMost(5, TimeUnit.SECONDS) - .untilAsserted(() -> assertTrue(owner1.isCompletedExceptionally())); - Awaitility.await().atMost(5, TimeUnit.SECONDS) + // In 10 secs, the getOwnerAsync requests(lookup requests) should time out. + Awaitility.await().atMost(10, TimeUnit.SECONDS) .untilAsserted(() -> assertTrue(owner2.isCompletedExceptionally())); // recovered, check the monitor update state : Assigned -> Owned - FieldUtils.writeDeclaredField(channel2, "producer", producer, true); + setTableView(channel2, tableview); FieldUtils.writeDeclaredField(channel1, - "inFlightStateWaitingTimeInMillis", 1 , true); + "inFlightStateWaitingTimeInMillis", 1, true); FieldUtils.writeDeclaredField(channel2, - "inFlightStateWaitingTimeInMillis", 1 , true); + "inFlightStateWaitingTimeInMillis", 1, true); ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( - List.of(lookupServiceAddress1, lookupServiceAddress2)); + List.of(brokerId1, brokerId2)); ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( - List.of(lookupServiceAddress1, lookupServiceAddress2)); + List.of(brokerId1, brokerId2)); - waitUntilNewOwner(channel1, bundle, lookupServiceAddress2); - waitUntilNewOwner(channel2, bundle, lookupServiceAddress2); + waitUntilNewOwner(channel1, bundle, brokerId2); + waitUntilNewOwner(channel2, bundle, brokerId2); var ownerAddr1 = channel1.getOwnerAsync(bundle).get(); var ownerAddr2 = channel2.getOwnerAsync(bundle).get(); assertEquals(ownerAddr1, ownerAddr2); - assertEquals(ownerAddr1, Optional.of(lookupServiceAddress2)); + assertEquals(ownerAddr1, Optional.of(brokerId2)); var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; validateMonitorCounters(leader, @@ -1178,35 +1242,30 @@ public void assignTestWhenDestBrokerProducerFails() } @Test(priority = 14) - public void splitTestWhenProducerFails() + public void splitTestWhenTableViewPutFails() throws ExecutionException, InterruptedException, IllegalAccessException { - Unload unload = new Unload(lookupServiceAddress1, bundle, Optional.empty()); + Unload unload = new Unload(brokerId1, bundle, Optional.empty()); channel1.publishUnloadEventAsync(unload); waitUntilState(channel1, bundle, Free); waitUntilState(channel2, bundle, Free); - channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); + channel1.publishAssignEventAsync(bundle, brokerId1); waitUntilState(channel1, bundle, Owned); waitUntilState(channel2, bundle, Owned); - assertEquals(lookupServiceAddress1, channel1.getOwnerAsync(bundle).get().get()); - assertEquals(lookupServiceAddress1, channel2.getOwnerAsync(bundle).get().get()); + assertEquals(brokerId1, channel1.getOwnerAsync(bundle).get().get()); + assertEquals(brokerId1, channel2.getOwnerAsync(bundle).get().get()); - var producer = (Producer) FieldUtils.readDeclaredField(channel1, - "producer", true); - var spyProducer = spy(producer); - var msg = mock(TypedMessageBuilder.class); + var tableview = getTableView(channel1); + var tableviewSpy = spy(tableview); var future = CompletableFuture.failedFuture(new RuntimeException()); - doReturn(msg).when(spyProducer).newMessage(); - doReturn(msg).when(msg).key(any()); - doReturn(msg).when(msg).value(any()); - doReturn(future).when(msg).sendAsync(); - FieldUtils.writeDeclaredField(channel1, "producer", spyProducer, true); + doReturn(future).when(tableviewSpy).put(any(), any()); + setTableView(channel1, tableviewSpy); FieldUtils.writeDeclaredField(channel1, "inFlightStateWaitingTimeInMillis", 3 * 1000, true); FieldUtils.writeDeclaredField(channel2, @@ -1214,7 +1273,7 @@ public void splitTestWhenProducerFails() // Assert child bundle ownerships in the channels. - Split split = new Split(bundle, lookupServiceAddress1, Map.of( + Split split = new Split(bundle, brokerId1, Map.of( childBundle1Range, Optional.empty(), childBundle2Range, Optional.empty())); channel2.publishSplitEventAsync(split); // channel1 is broken. the split won't be complete. @@ -1225,24 +1284,25 @@ public void splitTestWhenProducerFails() // recovered, check the monitor update state : Splitting -> Owned - FieldUtils.writeDeclaredField(channel1, "producer", producer, true); + setTableView(channel1, tableview); FieldUtils.writeDeclaredField(channel1, - "inFlightStateWaitingTimeInMillis", 1 , true); + "inFlightStateWaitingTimeInMillis", 1, true); FieldUtils.writeDeclaredField(channel2, - "inFlightStateWaitingTimeInMillis", 1 , true); + "inFlightStateWaitingTimeInMillis", 1, true); var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; - - waitUntilStateWithMonitor(leader, bundle, Deleted); - waitUntilStateWithMonitor(channel1, bundle, Deleted); - waitUntilStateWithMonitor(channel2, bundle, Deleted); + doReturn(CompletableFuture.completedFuture(Optional.of(brokerId1))) + .when(loadManager).selectAsync(any(), any(), any()); + waitUntilStateWithMonitor(leader, bundle, Init); + waitUntilStateWithMonitor(channel1, bundle, Init); + waitUntilStateWithMonitor(channel2, bundle, Init); var ownerAddr1 = channel1.getOwnerAsync(bundle); var ownerAddr2 = channel2.getOwnerAsync(bundle); - assertTrue(ownerAddr1.isCompletedExceptionally()); - assertTrue(ownerAddr2.isCompletedExceptionally()); + assertTrue(ownerAddr1.get().isEmpty()); + assertTrue(ownerAddr2.get().isEmpty()); FieldUtils.writeDeclaredField(channel1, @@ -1255,19 +1315,20 @@ public void splitTestWhenProducerFails() @Test(priority = 15) public void testIsOwner() throws IllegalAccessException { + var owner1 = channel1.isOwner(bundle); var owner2 = channel2.isOwner(bundle); assertFalse(owner1); assertFalse(owner2); - owner1 = channel1.isOwner(bundle, lookupServiceAddress2); - owner2 = channel2.isOwner(bundle, lookupServiceAddress1); + owner1 = channel1.isOwner(bundle, brokerId2); + owner2 = channel2.isOwner(bundle, brokerId1); assertFalse(owner1); assertFalse(owner2); - channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); + channel1.publishAssignEventAsync(bundle, brokerId1); owner2 = channel2.isOwner(bundle); assertFalse(owner2); @@ -1280,53 +1341,127 @@ public void testIsOwner() throws IllegalAccessException { assertTrue(owner1); assertFalse(owner2); - owner1 = channel1.isOwner(bundle, lookupServiceAddress1); - owner2 = channel2.isOwner(bundle, lookupServiceAddress2); + owner1 = channel1.isOwner(bundle, brokerId1); + owner2 = channel2.isOwner(bundle, brokerId2); assertTrue(owner1); assertFalse(owner2); - owner1 = channel2.isOwner(bundle, lookupServiceAddress1); - owner2 = channel1.isOwner(bundle, lookupServiceAddress2); + owner1 = channel2.isOwner(bundle, brokerId1); + owner2 = channel1.isOwner(bundle, brokerId2); assertTrue(owner1); assertFalse(owner2); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigning, lookupServiceAddress1, 1)); - assertFalse(channel1.isOwner(bundle)); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Owned, lookupServiceAddress1, 1)); - assertTrue(channel1.isOwner(bundle)); + try { + disableChannels(); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigning, brokerId1, 1)); + assertFalse(channel1.isOwner(bundle)); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Owned, brokerId1, 1)); + assertTrue(channel1.isOwner(bundle)); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, null, lookupServiceAddress1, 1)); - assertFalse(channel1.isOwner(bundle)); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, null, brokerId1, 1)); + assertFalse(channel1.isOwner(bundle)); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, null, lookupServiceAddress1, 1)); - assertTrue(channel1.isOwner(bundle)); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, null, brokerId1, 1)); + assertTrue(channel1.isOwner(bundle)); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Free, null, lookupServiceAddress1, 1)); - assertFalse(channel1.isOwner(bundle)); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Free, null, brokerId1, 1)); + assertFalse(channel1.isOwner(bundle)); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Deleted, null, lookupServiceAddress1, 1)); - assertFalse(channel1.isOwner(bundle)); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Deleted, null, brokerId1, 1)); + assertFalse(channel1.isOwner(bundle)); - overrideTableView(channel1, bundle, null); - assertFalse(channel1.isOwner(bundle)); + overrideTableView(channel1, bundle, null); + assertFalse(channel1.isOwner(bundle)); + } finally { + enableChannels(); + } } @Test(priority = 16) + public void testGetOwnerAsync() throws Exception { + try { + disableChannels(); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Owned, brokerId1, 1)); + var owner = channel1.getOwnerAsync(bundle); + //assertTrue(owner.isDone()); + assertEquals(brokerId1, owner.get().get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Owned, brokerId2, 1)); + owner = channel1.getOwnerAsync(bundle); + //assertTrue(owner.isDone()); + assertEquals(brokerId2, owner.get().get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigning, brokerId1, 1)); + owner = channel1.getOwnerAsync(bundle); + assertFalse(owner.isDone()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigning, brokerId2, 1)); + owner = channel1.getOwnerAsync(bundle); + //assertTrue(owner.isDone()); + assertEquals(brokerId2, owner.get().get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, brokerId1, 1)); + owner = channel1.getOwnerAsync(bundle); + assertFalse(owner.isDone()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, brokerId2, 1)); + owner = channel1.getOwnerAsync(bundle); + //assertTrue(owner.isDone()); + assertEquals(brokerId2, owner.get().get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, null, brokerId1, 1)); + owner = channel1.getOwnerAsync(bundle); + //assertTrue(owner.isDone()); + assertEquals(Optional.empty(), owner.get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, null, brokerId1, 1)); + owner = channel1.getOwnerAsync(bundle); + //assertTrue(owner.isDone()); + assertEquals(brokerId1, owner.get().get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, null, brokerId2, 1)); + owner = channel1.getOwnerAsync(bundle); + //assertTrue(owner.isDone()); + assertEquals(brokerId2, owner.get().get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Free, null, brokerId1, 1)); + owner = channel1.getOwnerAsync(bundle); + //assertTrue(owner.isDone()); + assertEquals(Optional.empty(), owner.get()); + + overrideTableView(channel1, bundle, null); + owner = channel1.getOwnerAsync(bundle); + //assertTrue(owner.isDone()); + assertEquals(Optional.empty(), owner.get()); + + overrideTableView(channel1, bundle1, new ServiceUnitStateData(Deleted, null, brokerId1, 1)); + owner = channel1.getOwnerAsync(bundle1); + //assertTrue(owner.isDone()); + assertTrue(owner.isCompletedExceptionally()); + } finally { + enableChannels(); + } + + } + + @Test(priority = 17) public void splitAndRetryFailureTest() throws Exception { - channel1.publishAssignEventAsync(bundle3, lookupServiceAddress1); - waitUntilNewOwner(channel1, bundle3, lookupServiceAddress1); - waitUntilNewOwner(channel2, bundle3, lookupServiceAddress1); + channel1.publishAssignEventAsync(bundle3, brokerId1); + waitUntilNewOwner(channel1, bundle3, brokerId1); + waitUntilNewOwner(channel2, bundle3, brokerId1); var ownerAddr1 = channel1.getOwnerAsync(bundle3).get(); var ownerAddr2 = channel2.getOwnerAsync(bundle3).get(); - assertEquals(ownerAddr1, Optional.of(lookupServiceAddress1)); - assertEquals(ownerAddr2, Optional.of(lookupServiceAddress1)); + assertEquals(ownerAddr1, Optional.of(brokerId1)); + assertEquals(ownerAddr2, Optional.of(brokerId1)); assertTrue(ownerAddr1.isPresent()); - NamespaceService namespaceService = spy(pulsar1.getNamespaceService()); + NamespaceService namespaceService = pulsar1.getNamespaceService(); CompletableFuture future = new CompletableFuture<>(); + int badVersionExceptionCount = 10; AtomicInteger count = new AtomicInteger(badVersionExceptionCount); future.completeExceptionally(new MetadataStoreException.BadVersionException("BadVersion")); @@ -1334,12 +1469,8 @@ public void splitAndRetryFailureTest() throws Exception { if (count.decrementAndGet() > 0) { return future; } - // Call the real method - reset(namespaceService); - doReturn(CompletableFuture.completedFuture(List.of("test-topic-1", "test-topic-2"))) - .when(namespaceService).getOwnedTopicListForNamespaceBundle(any()); - return future; - }).when(namespaceService).updateNamespaceBundlesForPolicies(any(), any()); + return invocationOnMock.callRealMethod(); + }).when(namespaceService).updateNamespaceBundles(any(), any()); doReturn(namespaceService).when(pulsar1).getNamespaceService(); doReturn(CompletableFuture.completedFuture(List.of("test-topic-1", "test-topic-2"))) .when(namespaceService).getOwnedTopicListForNamespaceBundle(any()); @@ -1351,9 +1482,9 @@ public void splitAndRetryFailureTest() throws Exception { channel1.publishSplitEventAsync(split); FieldUtils.writeDeclaredField(channel1, - "inFlightStateWaitingTimeInMillis", 1 , true); + "inFlightStateWaitingTimeInMillis", 1, true); FieldUtils.writeDeclaredField(channel2, - "inFlightStateWaitingTimeInMillis", 1 , true); + "inFlightStateWaitingTimeInMillis", 1, true); Awaitility.await() .pollInterval(200, TimeUnit.MILLISECONDS) @@ -1361,46 +1492,50 @@ public void splitAndRetryFailureTest() throws Exception { .untilAsserted(() -> { assertEquals(3, count.get()); }); - var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; - ((ServiceUnitStateChannelImpl) leader) - .monitorOwnerships(List.of(lookupServiceAddress1, lookupServiceAddress2)); - waitUntilState(leader, bundle3, Deleted); - waitUntilState(channel1, bundle3, Deleted); - waitUntilState(channel2, bundle3, Deleted); + ServiceUnitStateChannelImpl leader = + (ServiceUnitStateChannelImpl) (channel1.isChannelOwnerAsync().get() ? channel1 : channel2); + doReturn(CompletableFuture.completedFuture(Optional.of(brokerId1))) + .when(loadManager).selectAsync(any(), any(), any()); + leader.monitorOwnerships(List.of(brokerId1, brokerId2)); + + waitUntilState(leader, bundle3, Init); + waitUntilState(channel1, bundle3, Init); + waitUntilState(channel2, bundle3, Init); + + waitUntilNewOwner(channel1, childBundle31, brokerId1); + waitUntilNewOwner(channel1, childBundle32, brokerId1); + waitUntilNewOwner(channel2, childBundle31, brokerId1); + waitUntilNewOwner(channel2, childBundle32, brokerId1); + + assertEquals(Optional.of(brokerId1), channel1.getOwnerAsync(childBundle31).get()); + assertEquals(Optional.of(brokerId1), channel1.getOwnerAsync(childBundle32).get()); + assertEquals(Optional.of(brokerId1), channel2.getOwnerAsync(childBundle31).get()); + assertEquals(Optional.of(brokerId1), channel2.getOwnerAsync(childBundle32).get()); - validateHandlerCounters(channel1, 1, 0, 3, 0, 0, 0, 2, 1, 0, 0, 0, 0, 1, 0); - validateHandlerCounters(channel2, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0); + validateHandlerCounters(channel1, 1, 0, 3, 0, 0, 0, 2, 1, 0, 0, 1, 0, 1, 0); + validateHandlerCounters(channel2, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1, 0); validateEventCounters(channel1, 1, 0, 1, 0, 0, 0); validateEventCounters(channel2, 0, 0, 0, 0, 0, 0); - waitUntilNewOwner(channel1, childBundle31, lookupServiceAddress1); - waitUntilNewOwner(channel1, childBundle32, lookupServiceAddress1); - waitUntilNewOwner(channel2, childBundle31, lookupServiceAddress1); - waitUntilNewOwner(channel2, childBundle32, lookupServiceAddress1); - assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(childBundle31).get()); - assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(childBundle32).get()); - assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle31).get()); - assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle32).get()); - // try the monitor and check the monitor moves `Deleted` -> `Init` FieldUtils.writeDeclaredField(channel1, - "semiTerminalStateWaitingTimeInMillis", 1, true); + "stateTombstoneDelayTimeInMillis", 1, true); FieldUtils.writeDeclaredField(channel2, - "semiTerminalStateWaitingTimeInMillis", 1, true); + "stateTombstoneDelayTimeInMillis", 1, true); ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( - List.of(lookupServiceAddress1, lookupServiceAddress2)); + List.of(brokerId1, brokerId2)); ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( - List.of(lookupServiceAddress1, lookupServiceAddress2)); + List.of(brokerId1, brokerId2)); waitUntilState(channel1, bundle3, Init); waitUntilState(channel2, bundle3, Init); validateMonitorCounters(leader, 0, - 1, + 0, 1, 0, 0, @@ -1413,29 +1548,29 @@ public void splitAndRetryFailureTest() throws Exception { FieldUtils.writeDeclaredField(channel1, "inFlightStateWaitingTimeInMillis", 30 * 1000, true); FieldUtils.writeDeclaredField(channel1, - "semiTerminalStateWaitingTimeInMillis", 300 * 1000, true); + "stateTombstoneDelayTimeInMillis", 300 * 1000, true); FieldUtils.writeDeclaredField(channel2, "inFlightStateWaitingTimeInMillis", 30 * 1000, true); FieldUtils.writeDeclaredField(channel2, - "semiTerminalStateWaitingTimeInMillis", 300 * 1000, true); + "stateTombstoneDelayTimeInMillis", 300 * 1000, true); } - @Test(priority = 17) + @Test(priority = 18) public void testOverrideInactiveBrokerStateData() throws IllegalAccessException, ExecutionException, InterruptedException, TimeoutException { - var leaderChannel = channel1; - var followerChannel = channel2; + ServiceUnitStateChannelImpl leaderChannel = (ServiceUnitStateChannelImpl) channel1; + ServiceUnitStateChannelImpl followerChannel = (ServiceUnitStateChannelImpl) channel2; String leader = channel1.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); String leader2 = channel2.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); assertEquals(leader, leader2); - if (leader.equals(lookupServiceAddress2)) { - leaderChannel = channel2; - followerChannel = channel1; + if (leader.equals(brokerId2)) { + leaderChannel = (ServiceUnitStateChannelImpl) channel2; + followerChannel = (ServiceUnitStateChannelImpl) channel1; } - String broker = lookupServiceAddress1; + String broker = brokerId1; // test override states String releasingBundle = "public/releasing/0xfffffff0_0xffffffff"; @@ -1444,49 +1579,57 @@ public void testOverrideInactiveBrokerStateData() String freeBundle = "public/free/0xfffffff0_0xffffffff"; String deletedBundle = "public/deleted/0xfffffff0_0xffffffff"; String ownedBundle = "public/owned/0xfffffff0_0xffffffff"; - overrideTableViews(releasingBundle, - new ServiceUnitStateData(Releasing, null, broker, 1)); - overrideTableViews(splittingBundle, - new ServiceUnitStateData(Splitting, null, broker, - Map.of(childBundle1Range, Optional.empty(), - childBundle2Range, Optional.empty()), 1)); - overrideTableViews(assigningBundle, - new ServiceUnitStateData(Assigning, broker, null, 1)); - overrideTableViews(freeBundle, - new ServiceUnitStateData(Free, null, broker, 1)); - overrideTableViews(deletedBundle, - new ServiceUnitStateData(Deleted, null, broker, 1)); - overrideTableViews(ownedBundle, - new ServiceUnitStateData(Owned, broker, null, 1)); + try { + disableChannels(); + overrideTableViews(releasingBundle, + new ServiceUnitStateData(Releasing, null, broker, 1)); + overrideTableViews(splittingBundle, + new ServiceUnitStateData(Splitting, null, broker, + Map.of(childBundle1Range, Optional.empty(), + childBundle2Range, Optional.empty()), 1)); + overrideTableViews(assigningBundle, + new ServiceUnitStateData(Assigning, broker, null, 1)); + overrideTableViews(freeBundle, + new ServiceUnitStateData(Free, null, broker, 1)); + overrideTableViews(deletedBundle, + new ServiceUnitStateData(Deleted, null, broker, 1)); + overrideTableViews(ownedBundle, + new ServiceUnitStateData(Owned, broker, null, 1)); + } finally { + enableChannels(); + } // test stable metadata state - doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))) - .when(loadManager).selectAsync(any(), any()); + doReturn(CompletableFuture.completedFuture(Optional.of(brokerId2))) + .when(loadManager).selectAsync(any(), any(), any()); leaderChannel.handleMetadataSessionEvent(SessionReestablished); followerChannel.handleMetadataSessionEvent(SessionReestablished); FieldUtils.writeDeclaredField(leaderChannel, "lastMetadataSessionEventTimestamp", System.currentTimeMillis() - (MAX_CLEAN_UP_DELAY_TIME_IN_SECS * 1000 + 1000), true); FieldUtils.writeDeclaredField(followerChannel, "lastMetadataSessionEventTimestamp", System.currentTimeMillis() - (MAX_CLEAN_UP_DELAY_TIME_IN_SECS * 1000 + 1000), true); + + doReturn(brokers).when(pulsarAdmin).brokers(); leaderChannel.handleBrokerRegistrationEvent(broker, NotificationType.Deleted); followerChannel.handleBrokerRegistrationEvent(broker, NotificationType.Deleted); - waitUntilNewOwner(channel2, releasingBundle, lookupServiceAddress2); - waitUntilNewOwner(channel2, childBundle11, lookupServiceAddress2); - waitUntilNewOwner(channel2, childBundle12, lookupServiceAddress2); - waitUntilNewOwner(channel2, assigningBundle, lookupServiceAddress2); - waitUntilNewOwner(channel2, ownedBundle, lookupServiceAddress2); + + waitUntilNewOwner(channel2, releasingBundle, brokerId2); + waitUntilNewOwner(channel2, childBundle11, brokerId2); + waitUntilNewOwner(channel2, childBundle12, brokerId2); + waitUntilNewOwner(channel2, assigningBundle, brokerId2); + waitUntilNewOwner(channel2, ownedBundle, brokerId2); assertEquals(Optional.empty(), channel2.getOwnerAsync(freeBundle).get()); assertTrue(channel2.getOwnerAsync(deletedBundle).isCompletedExceptionally()); - assertTrue(channel2.getOwnerAsync(splittingBundle).isCompletedExceptionally()); + assertTrue(channel2.getOwnerAsync(splittingBundle).get().isEmpty()); // clean-up FieldUtils.writeDeclaredField(leaderChannel, "maxCleanupDelayTimeInSecs", 3 * 60, true); cleanTableViews(); - + reset(pulsarAdmin); } - @Test(priority = 18) + @Test(priority = 19) public void testOverrideOrphanStateData() throws IllegalAccessException, ExecutionException, InterruptedException, TimeoutException { @@ -1495,53 +1638,89 @@ public void testOverrideOrphanStateData() String leader = channel1.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); String leader2 = channel2.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); assertEquals(leader, leader2); - if (leader.equals(lookupServiceAddress2)) { + if (leader.equals(brokerId2)) { leaderChannel = channel2; followerChannel = channel1; } - String broker = lookupServiceAddress1; + String broker = brokerId1; // test override states - String releasingBundle = "public/releasing/0xfffffff0_0xffffffff"; + String releasingBundle1 = "public/releasing1/0xfffffff0_0xffffffff"; + String releasingBundle2 = "public/releasing2/0xfffffff0_0xffffffff"; String splittingBundle = bundle; - String assigningBundle = "public/assigning/0xfffffff0_0xffffffff"; + String assigningBundle1 = "public/assigning1/0xfffffff0_0xffffffff"; + String assigningBundle2 = "public/assigning2/0xfffffff0_0xffffffff"; String freeBundle = "public/free/0xfffffff0_0xffffffff"; String deletedBundle = "public/deleted/0xfffffff0_0xffffffff"; - String ownedBundle = "public/owned/0xfffffff0_0xffffffff"; - overrideTableViews(releasingBundle, - new ServiceUnitStateData(Releasing, null, broker, 1)); - overrideTableViews(splittingBundle, - new ServiceUnitStateData(Splitting, null, broker, - Map.of(childBundle1Range, Optional.empty(), - childBundle2Range, Optional.empty()), 1)); - overrideTableViews(assigningBundle, - new ServiceUnitStateData(Assigning, broker, null, 1)); - overrideTableViews(freeBundle, - new ServiceUnitStateData(Free, null, broker, 1)); - overrideTableViews(deletedBundle, - new ServiceUnitStateData(Deleted, null, broker, 1)); - overrideTableViews(ownedBundle, - new ServiceUnitStateData(Owned, broker, null, 1)); + String ownedBundle1 = "public/owned1/0xfffffff0_0xffffffff"; + String ownedBundle2 = "public/owned2SourceBundle/0xfffffff0_0xffffffff"; + String ownedBundle3 = "public/owned3/0xfffffff0_0xffffffff"; + String inactiveBroker = "broker-inactive-1"; + try { + disableChannels(); + overrideTableViews(releasingBundle1, + new ServiceUnitStateData(Releasing, broker, brokerId2, 1)); + overrideTableViews(releasingBundle2, + new ServiceUnitStateData(Releasing, brokerId2, brokerId3, 1)); + overrideTableViews(splittingBundle, + new ServiceUnitStateData(Splitting, null, broker, + Map.of(childBundle1Range, Optional.empty(), + childBundle2Range, Optional.empty()), 1)); + overrideTableViews(assigningBundle1, + new ServiceUnitStateData(Assigning, broker, null, 1)); + overrideTableViews(assigningBundle2, + new ServiceUnitStateData(Assigning, broker, brokerId2, 1)); + overrideTableViews(freeBundle, + new ServiceUnitStateData(Free, null, broker, 1)); + overrideTableViews(deletedBundle, + new ServiceUnitStateData(Deleted, null, broker, 1)); + overrideTableViews(ownedBundle1, + new ServiceUnitStateData(Owned, broker, null, 1)); + overrideTableViews(ownedBundle2, + new ServiceUnitStateData(Owned, broker, inactiveBroker, 1)); + overrideTableViews(ownedBundle3, + new ServiceUnitStateData(Owned, inactiveBroker, broker, 1)); + } finally { + enableChannels(); + } + // test stable metadata state - doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))) - .when(loadManager).selectAsync(any(), any()); + doReturn(CompletableFuture.completedFuture(Optional.of(brokerId2))) + .when(loadManager).selectAsync(any(), any(), any()); FieldUtils.writeDeclaredField(leaderChannel, "inFlightStateWaitingTimeInMillis", -1, true); FieldUtils.writeDeclaredField(followerChannel, "inFlightStateWaitingTimeInMillis", -1, true); ((ServiceUnitStateChannelImpl) leaderChannel) - .monitorOwnerships(List.of(lookupServiceAddress1, lookupServiceAddress2)); + .monitorOwnerships(List.of(brokerId1, brokerId2, "broker-3")); - waitUntilNewOwner(channel2, releasingBundle, broker); - waitUntilNewOwner(channel2, childBundle11, broker); - waitUntilNewOwner(channel2, childBundle12, broker); - waitUntilNewOwner(channel2, assigningBundle, lookupServiceAddress2); - waitUntilNewOwner(channel2, ownedBundle, broker); - assertEquals(Optional.empty(), channel2.getOwnerAsync(freeBundle).get()); + ServiceUnitStateChannel finalLeaderChannel = leaderChannel; + Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> getCleanupJobs(finalLeaderChannel).isEmpty()); + + + waitUntilNewOwner(channel2, releasingBundle1, brokerId2); + waitUntilNewOwner(channel2, releasingBundle2, brokerId2); + assertTrue(channel2.getOwnerAsync(splittingBundle).get().isEmpty()); + waitUntilNewOwner(channel2, childBundle11, brokerId2); + waitUntilNewOwner(channel2, childBundle12, brokerId2); + waitUntilNewOwner(channel2, assigningBundle1, brokerId2); + waitUntilNewOwner(channel2, assigningBundle2, brokerId2); + assertTrue(channel2.getOwnerAsync(freeBundle).get().isEmpty()); assertTrue(channel2.getOwnerAsync(deletedBundle).isCompletedExceptionally()); - assertTrue(channel2.getOwnerAsync(splittingBundle).isCompletedExceptionally()); + waitUntilNewOwner(channel2, ownedBundle1, broker); + waitUntilNewOwner(channel2, ownedBundle2, broker); + waitUntilNewOwner(channel2, ownedBundle3, brokerId2); + + validateMonitorCounters(leaderChannel, + 1, + 0, + 6, + 0, + 1, + 0, + 0); // clean-up FieldUtils.writeDeclaredField(channel1, @@ -1551,10 +1730,177 @@ public void testOverrideOrphanStateData() cleanTableViews(); } + @Test(priority = 20) + public void testActiveGetOwner() throws Exception { + + // case 1: the bundle owner is empty + String broker = brokerId2; + String bundle = "public/owned/0xfffffff0_0xffffffff"; + try { + disableChannels(); + overrideTableViews(bundle, null); + assertEquals(Optional.empty(), channel1.getOwnerAsync(bundle).get()); + + // case 2: the bundle ownership is transferring, and the dst broker is not the channel owner + overrideTableViews(bundle, + new ServiceUnitStateData(Releasing, broker, brokerId1, 1)); + assertEquals(Optional.of(broker), channel1.getOwnerAsync(bundle).get()); + + + // case 3: the bundle ownership is transferring, and the dst broker is the channel owner + overrideTableViews(bundle, + new ServiceUnitStateData(Assigning, brokerId1, brokerId2, 1)); + assertFalse(channel1.getOwnerAsync(bundle).isDone()); + + // case 4: the bundle ownership is found + overrideTableViews(bundle, + new ServiceUnitStateData(Owned, broker, null, 1)); + var owner = channel1.getOwnerAsync(bundle).get(5, TimeUnit.SECONDS).get(); + assertEquals(owner, broker); + } finally { + enableChannels(); + } + + // case 5: the owner lookup gets delayed + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 1000, true); + var delayedFuture = new CompletableFuture(); + doReturn(delayedFuture).when(registry).lookupAsync(eq(broker)); + CompletableFuture.runAsync(() -> { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + delayedFuture.complete(Optional.of(broker)); + }); + + // verify the owner eventually returns in inFlightStateWaitingTimeInMillis. + long start = System.currentTimeMillis(); + assertEquals(broker, channel1.getOwnerAsync(bundle).get().get()); + long elapsed = System.currentTimeMillis() - start; + assertTrue(elapsed < 1000); - private static ConcurrentOpenHashMap>> getOwnerRequests( + // case 6: the owner is inactive + doReturn(CompletableFuture.completedFuture(Optional.empty())) + .when(registry).lookupAsync(eq(broker)); + + // verify getOwnerAsync times out + start = System.currentTimeMillis(); + var ex = expectThrows(ExecutionException.class, () -> channel1.getOwnerAsync(bundle).get()); + assertTrue(ex.getCause() instanceof IllegalStateException); + assertTrue(System.currentTimeMillis() - start >= 1000); + + try { + // verify getOwnerAsync returns immediately when not registered + registry.unregister(); + start = System.currentTimeMillis(); + assertEquals(broker, channel1.getOwnerAsync(bundle).get().get()); + elapsed = System.currentTimeMillis() - start; + assertTrue(elapsed < 1000); + } finally { + registry.registerAsync().join(); + } + + + // case 7: the ownership cleanup(no new owner) by the leader channel + doReturn(CompletableFuture.completedFuture(Optional.empty())) + .when(loadManager).selectAsync(any(), any(), any()); + ServiceUnitStateChannelImpl leaderChannel = (ServiceUnitStateChannelImpl) channel1; + String leader1 = channel1.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); + String leader2 = channel2.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); + assertEquals(leader1, leader2); + if (leader1.equals(brokerId2)) { + leaderChannel = (ServiceUnitStateChannelImpl) channel2; + } + leaderChannel.handleMetadataSessionEvent(SessionReestablished); + FieldUtils.writeDeclaredField(leaderChannel, "lastMetadataSessionEventTimestamp", + System.currentTimeMillis() - (MAX_CLEAN_UP_DELAY_TIME_IN_SECS * 1000 + 1000), true); + doReturn(brokers).when(pulsarAdmin).brokers(); + leaderChannel.handleBrokerRegistrationEvent(broker, NotificationType.Deleted); + + // verify the ownership cleanup, and channel's getOwnerAsync returns empty result without timeout + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 20 * 1000, true); + start = System.currentTimeMillis(); + assertTrue(channel1.getOwnerAsync(bundle).get().isEmpty()); + waitUntilState(channel1, bundle, Init); + waitUntilState(channel2, bundle, Init); + + assertTrue(System.currentTimeMillis() - start < 20_000); + reset(pulsarAdmin); + // case 8: simulate ownership cleanup(brokerId1 as the new owner) by the leader channel + try { + disableChannels(); + overrideTableViews(bundle, + new ServiceUnitStateData(Owned, broker, null, 1)); + } finally { + enableChannels(); + } + doReturn(CompletableFuture.completedFuture(Optional.of(brokerId1))) + .when(loadManager).selectAsync(any(), any(), any()); + leaderChannel.handleMetadataSessionEvent(SessionReestablished); + FieldUtils.writeDeclaredField(leaderChannel, "lastMetadataSessionEventTimestamp", + System.currentTimeMillis() - (MAX_CLEAN_UP_DELAY_TIME_IN_SECS * 1000 + 1000), true); + getCleanupJobs(leaderChannel).clear(); + doReturn(brokers).when(pulsarAdmin).brokers(); + leaderChannel.handleBrokerRegistrationEvent(broker, NotificationType.Deleted); + + // verify the ownership cleanup, and channel's getOwnerAsync returns brokerId1 without timeout + start = System.currentTimeMillis(); + assertEquals(brokerId1, channel1.getOwnerAsync(bundle).get().get()); + assertTrue(System.currentTimeMillis() - start < 20_000); + + // test clean-up + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); + cleanTableViews(); + reset(pulsarAdmin); + } + + @Test(priority = 21) + public void testGetOwnershipEntrySetBeforeChannelStart() { + var tmpChannel = new ServiceUnitStateChannelImpl(pulsar1); + try { + tmpChannel.getOwnershipEntrySet(); + fail(); + } catch (Exception e) { + assertTrue(e instanceof IllegalStateException); + assertEquals("Invalid channel state:Constructed", e.getMessage()); + } + } + + @Test(priority = 22) + public void unloadTimeoutCheckTest() + throws Exception { + + String topic = "persistent://" + namespaceName + "/test-topic"; + NamespaceBundle bundleName = pulsar.getNamespaceService().getBundle(TopicName.get(topic)); + var releasing = new ServiceUnitStateData(Releasing, pulsar2.getBrokerId(), pulsar1.getBrokerId(), 1); + + try { + disableChannels(); + overrideTableView(channel1, bundleName.toString(), releasing); + var topicFuture = pulsar1.getBrokerService().getOrCreateTopic(topic); + topicFuture.get(1, TimeUnit.SECONDS); + } catch (Exception e) { + if (!(e.getCause() instanceof BrokerServiceException.ServiceUnitNotReadyException && e.getMessage() + .contains("Please redo the lookup"))) { + fail(); + } + } finally { + enableChannels(); + } + + pulsar1.getBrokerService() + .unloadServiceUnit(bundleName, true, true, 5, + TimeUnit.SECONDS).get(2, TimeUnit.SECONDS); + } + + + private static ConcurrentHashMap>> getOwnerRequests( ServiceUnitStateChannel channel) throws IllegalAccessException { - return (ConcurrentOpenHashMap>>) + return (ConcurrentHashMap>>) FieldUtils.readDeclaredField(channel, "getOwnerRequests", true); } @@ -1571,9 +1917,9 @@ private static long getLastMetadataSessionEventTimestamp(ServiceUnitStateChannel FieldUtils.readField(channel, "lastMetadataSessionEventTimestamp", true); } - private static ConcurrentOpenHashMap> getCleanupJobs( + private static ConcurrentHashMap> getCleanupJobs( ServiceUnitStateChannel channel) throws IllegalAccessException { - return (ConcurrentOpenHashMap>) + return (ConcurrentHashMap>) FieldUtils.readField(channel, "cleanupJobs", true); } @@ -1622,10 +1968,21 @@ private static void waitUntilNewOwner(ServiceUnitStateChannel channel, String se }); } - private static void waitUntilState(ServiceUnitStateChannel channel, String key) + private static ServiceUnitStateTableView getTableView(ServiceUnitStateChannel channel) throws IllegalAccessException { - TableViewImpl tv = (TableViewImpl) + return (ServiceUnitStateTableView) FieldUtils.readField(channel, "tableview", true); + } + + private static void setTableView(ServiceUnitStateChannel channel, + ServiceUnitStateTableView tableView) + throws IllegalAccessException { + FieldUtils.writeField(channel, "tableview", tableView, true); + } + + private static void waitUntilState(ServiceUnitStateChannel channel, String key) + throws IllegalAccessException { + var tv = getTableView(channel); Awaitility.await() .pollInterval(200, TimeUnit.MILLISECONDS) .atMost(10, TimeUnit.SECONDS) @@ -1641,8 +1998,7 @@ private static void waitUntilState(ServiceUnitStateChannel channel, String key) private static void waitUntilState(ServiceUnitStateChannel channel, String key, ServiceUnitState expected) throws IllegalAccessException { - TableViewImpl tv = (TableViewImpl) - FieldUtils.readField(channel, "tableview", true); + var tv = getTableView(channel); Awaitility.await() .pollInterval(200, TimeUnit.MILLISECONDS) .atMost(10, TimeUnit.SECONDS) @@ -1655,42 +2011,63 @@ private static void waitUntilState(ServiceUnitStateChannel channel, String key, private void waitUntilStateWithMonitor(ServiceUnitStateChannel channel, String key, ServiceUnitState expected) throws IllegalAccessException { - TableViewImpl tv = (TableViewImpl) - FieldUtils.readField(channel, "tableview", true); + var tv = getTableView(channel); Awaitility.await() .pollInterval(200, TimeUnit.MILLISECONDS) .atMost(10, TimeUnit.SECONDS) .until(() -> { // wait until true ((ServiceUnitStateChannelImpl) channel) - .monitorOwnerships(List.of(lookupServiceAddress1, lookupServiceAddress2)); + .monitorOwnerships(List.of(brokerId1, brokerId2)); ServiceUnitStateData data = tv.get(key); ServiceUnitState actual = state(data); return actual == expected; }); } - private static void cleanTableView(ServiceUnitStateChannel channel, String serviceUnit) + private void cleanTableViews() throws IllegalAccessException { - var tv = (TableViewImpl) - FieldUtils.readField(channel, "tableview", true); - var cache = (ConcurrentMap) - FieldUtils.readField(tv, "data", true); - cache.remove(serviceUnit); + cleanTableView(channel1); + cleanTableView(channel2); } - private void cleanTableViews() - throws IllegalAccessException { - var tv1 = (TableViewImpl) - FieldUtils.readField(channel1, "tableview", true); - var cache1 = (ConcurrentMap) - FieldUtils.readField(tv1, "data", true); - cache1.clear(); - - var tv2 = (TableViewImpl) - FieldUtils.readField(channel2, "tableview", true); - var cache2 = (ConcurrentMap) - FieldUtils.readField(tv2, "data", true); - cache2.clear(); + private void cleanTableView(ServiceUnitStateChannel channel) throws IllegalAccessException { + var getOwnerRequests = (Map>) + FieldUtils.readField(channel, "getOwnerRequests", true); + getOwnerRequests.clear(); + var tv = getTableView(channel); + if (serviceUnitStateTableViewClassName.equals(ServiceUnitStateTableViewImpl.class.getCanonicalName())) { + var tableview = (TableView) + FieldUtils.readField(tv, "tableview", true); + var cache = (ConcurrentMap) + FieldUtils.readField(tableview, "data", true); + cache.clear(); + } else { + var tableview = (MetadataStoreTableView) + FieldUtils.readField(tv, "tableview", true); + var handlerCounters = + (Map) + FieldUtils.readDeclaredField(channel, "handlerCounters", true); + var initCounter = handlerCounters.get(Init).getTotal(); + var deletedCounter = new AtomicLong(initCounter.get()); + try { + var set = tableview.entrySet(); + for (var e : set) { + try { + tableview.delete(e.getKey()).join(); + deletedCounter.incrementAndGet(); + } catch (CompletionException ex) { + if (!(ex.getCause() instanceof MetadataStoreException.NotFoundException)) { + throw ex; + } + } + } + Awaitility.await().ignoreNoExceptions().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> { + assertEquals(initCounter.get(), deletedCounter.get()); + }); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } private void overrideTableViews(String serviceUnit, ServiceUnitStateData val) throws IllegalAccessException { @@ -1698,17 +2075,55 @@ private void overrideTableViews(String serviceUnit, ServiceUnitStateData val) th overrideTableView(channel2, serviceUnit, val); } - private static void overrideTableView(ServiceUnitStateChannel channel, String serviceUnit, ServiceUnitStateData val) - throws IllegalAccessException { - var tv = (TableViewImpl) - FieldUtils.readField(channel, "tableview", true); - var cache = (ConcurrentMap) - FieldUtils.readField(tv, "data", true); - if(val == null){ - cache.remove(serviceUnit); - } else { - cache.put(serviceUnit, val); + @Test(enabled = false) + public static void overrideTableView(ServiceUnitStateChannel channel, + String serviceUnit, ServiceUnitStateData val) throws IllegalAccessException { + var getOwnerRequests = (Map>) + FieldUtils.readField(channel, "getOwnerRequests", true); + getOwnerRequests.clear(); + var tv = getTableView(channel); + + var handlerCounters = + (Map) + FieldUtils.readDeclaredField(channel, "handlerCounters", true); + + var cur = tv.get(serviceUnit); + if (cur != null) { + long intCountStart = handlerCounters.get(Init).getTotal().get(); + var deletedCount = new AtomicLong(0); + tv.delete(serviceUnit).join(); + deletedCount.incrementAndGet(); + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(3, TimeUnit.SECONDS) + .untilAsserted(() -> { + assertEquals( + handlerCounters.get(Init).getTotal().get() + - intCountStart, deletedCount.get()); + assertNull(tv.get(serviceUnit)); + }); } + + + + if (val != null) { + long stateCountStart = handlerCounters.get(state(val)).getTotal().get(); + var stateCount = new AtomicLong(0); + tv.put(serviceUnit, val).join(); + stateCount.incrementAndGet(); + + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(3, TimeUnit.SECONDS) + .untilAsserted(() -> { + assertEquals( + handlerCounters.get(state(val)).getTotal().get() + - stateCountStart, stateCount.get()); + assertEquals(val, tv.get(serviceUnit)); + }); + } + + } private static void cleanOpsCounters(ServiceUnitStateChannel channel) @@ -1717,7 +2132,7 @@ private static void cleanOpsCounters(ServiceUnitStateChannel channel) (Map) FieldUtils.readDeclaredField(channel, "handlerCounters", true); - for(var val : handlerCounters.values()){ + for (var val : handlerCounters.values()) { val.getFailure().set(0); val.getTotal().set(0); } @@ -1726,7 +2141,7 @@ private static void cleanOpsCounters(ServiceUnitStateChannel channel) (Map) FieldUtils.readDeclaredField(channel, "eventCounters", true); - for(var val : eventCounters.values()){ + for (var val : eventCounters.values()) { val.getFailure().set(0); val.getTotal().set(0); } @@ -1735,7 +2150,7 @@ private static void cleanOpsCounters(ServiceUnitStateChannel channel) (Map) FieldUtils.readDeclaredField(channel, "ownerLookUpCounters", true); - for(var val : ownerLookUpCounters.values()){ + for (var val : ownerLookUpCounters.values()) { val.getFailure().set(0); val.getTotal().set(0); } @@ -1752,7 +2167,7 @@ private void cleanOwnershipMonitorCounters(ServiceUnitStateChannel channel) thro } private void cleanMetadataState(ServiceUnitStateChannel channel) throws IllegalAccessException { - channel.handleMetadataSessionEvent(SessionReestablished); + ((ServiceUnitStateChannelImpl) channel).handleMetadataSessionEvent(SessionReestablished); FieldUtils.writeDeclaredField(channel, "lastMetadataSessionEventTimestamp", 0L, true); } @@ -1830,7 +2245,7 @@ private static void validateOwnerLookUpCounters(ServiceUnitStateChannel channel, long free, long deleted, long init - ) + ) throws IllegalAccessException { var ownerLookUpCounters = (Map) @@ -1873,7 +2288,7 @@ private static void validateMonitorCounters(ServiceUnitStateChannel channel, } ServiceUnitStateChannelImpl createChannel(PulsarService pulsar) - throws IllegalAccessException { + throws IllegalAccessException, PulsarServerException { var tmpChannel = new ServiceUnitStateChannelImpl(pulsar); FieldUtils.writeDeclaredField(tmpChannel, "ownershipMonitorDelayTimeInSecs", 5, true); var channel = spy(tmpChannel); @@ -1881,10 +2296,11 @@ ServiceUnitStateChannelImpl createChannel(PulsarService pulsar) doReturn(loadManagerContext).when(channel).getContext(); doReturn(registry).when(channel).getBrokerRegistry(); doReturn(loadManager).when(channel).getLoadManager(); + doReturn(pulsarAdmin).when(channel).getPulsarAdmin(); var leaderElectionService = new LeaderElectionService( - pulsar.getCoordinationService(), pulsar.getSafeWebServiceAddress(), + pulsar.getCoordinationService(), pulsar.getBrokerId(), pulsar.getSafeWebServiceAddress(), state -> { if (state == LeaderElectionState.Leading) { channel.scheduleOwnershipMonitor(); @@ -1898,4 +2314,14 @@ ServiceUnitStateChannelImpl createChannel(PulsarService pulsar) return channel; } + + private void disableChannels() { + ((ServiceUnitStateChannelImpl) channel1).disable(); + ((ServiceUnitStateChannelImpl) channel2).disable(); + } + + private void enableChannels() { + ((ServiceUnitStateChannelImpl) channel1).enable(); + ((ServiceUnitStateChannelImpl) channel2).enable(); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataConflictResolverTest.java similarity index 88% rename from pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java rename to pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataConflictResolverTest.java index 62de91dab292b..d336e8918ec5e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataConflictResolverTest.java @@ -25,13 +25,15 @@ import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Releasing; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; -import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.StorageType.MetadataStore; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.StorageType.SystemTopic; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import org.testng.annotations.Test; @Test(groups = "broker") -public class ServiceUnitStateCompactionStrategyTest { - ServiceUnitStateCompactionStrategy strategy = new ServiceUnitStateCompactionStrategy(); +public class ServiceUnitStateDataConflictResolverTest { + ServiceUnitStateDataConflictResolver strategy = new ServiceUnitStateDataConflictResolver(); String dst = "dst"; String src = "src"; @@ -91,6 +93,32 @@ public void testVersionId(){ } + @Test + public void testStoreType(){ + ServiceUnitStateDataConflictResolver strategy = new ServiceUnitStateDataConflictResolver(); + strategy.setStorageType(SystemTopic); + assertFalse(strategy.shouldKeepLeft( + null, + new ServiceUnitStateData(Owned, dst, 1))); + assertFalse(strategy.shouldKeepLeft( + null, + new ServiceUnitStateData(Owned, dst, 2))); + assertFalse(strategy.shouldKeepLeft( + new ServiceUnitStateData(Owned, dst, 1), + new ServiceUnitStateData(Owned, dst, 3))); + + strategy.setStorageType(MetadataStore); + assertFalse(strategy.shouldKeepLeft( + null, + new ServiceUnitStateData(Owned, dst, 1))); + assertTrue(strategy.shouldKeepLeft( + null, + new ServiceUnitStateData(Owned, dst, 2))); + assertTrue(strategy.shouldKeepLeft( + new ServiceUnitStateData(Owned, dst, 1), + new ServiceUnitStateData(Owned, dst, 3))); + } + @Test public void testForce(){ assertFalse(strategy.shouldKeepLeft( diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTest.java index 620266aee46a1..0a5f012ad40a7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTest.java @@ -25,6 +25,8 @@ import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Releasing; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.StorageType.MetadataStore; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.StorageType.SystemTopic; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import org.testng.annotations.Test; @@ -54,63 +56,123 @@ public void testActive() { } @Test - public void testTransitions() { - - assertFalse(ServiceUnitState.isValidTransition(Init, Init)); - assertTrue(ServiceUnitState.isValidTransition(Init, Free)); - assertTrue(ServiceUnitState.isValidTransition(Init, Owned)); - assertTrue(ServiceUnitState.isValidTransition(Init, Assigning)); - assertTrue(ServiceUnitState.isValidTransition(Init, Releasing)); - assertTrue(ServiceUnitState.isValidTransition(Init, Splitting)); - assertTrue(ServiceUnitState.isValidTransition(Init, Deleted)); - - assertTrue(ServiceUnitState.isValidTransition(Free, Init)); - assertFalse(ServiceUnitState.isValidTransition(Free, Free)); - assertFalse(ServiceUnitState.isValidTransition(Free, Owned)); - assertTrue(ServiceUnitState.isValidTransition(Free, Assigning)); - assertFalse(ServiceUnitState.isValidTransition(Free, Releasing)); - assertFalse(ServiceUnitState.isValidTransition(Free, Splitting)); - assertFalse(ServiceUnitState.isValidTransition(Free, Deleted)); - - assertFalse(ServiceUnitState.isValidTransition(Assigning, Init)); - assertFalse(ServiceUnitState.isValidTransition(Assigning, Free)); - assertFalse(ServiceUnitState.isValidTransition(Assigning, Assigning)); - assertTrue(ServiceUnitState.isValidTransition(Assigning, Owned)); - assertFalse(ServiceUnitState.isValidTransition(Assigning, Releasing)); - assertFalse(ServiceUnitState.isValidTransition(Assigning, Splitting)); - assertFalse(ServiceUnitState.isValidTransition(Assigning, Deleted)); - - assertFalse(ServiceUnitState.isValidTransition(Owned, Init)); - assertFalse(ServiceUnitState.isValidTransition(Owned, Free)); - assertFalse(ServiceUnitState.isValidTransition(Owned, Assigning)); - assertFalse(ServiceUnitState.isValidTransition(Owned, Owned)); - assertTrue(ServiceUnitState.isValidTransition(Owned, Releasing)); - assertTrue(ServiceUnitState.isValidTransition(Owned, Splitting)); - assertFalse(ServiceUnitState.isValidTransition(Owned, Deleted)); - - assertFalse(ServiceUnitState.isValidTransition(Releasing, Init)); - assertTrue(ServiceUnitState.isValidTransition(Releasing, Free)); - assertTrue(ServiceUnitState.isValidTransition(Releasing, Assigning)); - assertFalse(ServiceUnitState.isValidTransition(Releasing, Owned)); - assertFalse(ServiceUnitState.isValidTransition(Releasing, Releasing)); - assertFalse(ServiceUnitState.isValidTransition(Releasing, Splitting)); - assertFalse(ServiceUnitState.isValidTransition(Releasing, Deleted)); - - assertFalse(ServiceUnitState.isValidTransition(Splitting, Init)); - assertFalse(ServiceUnitState.isValidTransition(Splitting, Free)); - assertFalse(ServiceUnitState.isValidTransition(Splitting, Assigning)); - assertFalse(ServiceUnitState.isValidTransition(Splitting, Owned)); - assertFalse(ServiceUnitState.isValidTransition(Splitting, Releasing)); - assertFalse(ServiceUnitState.isValidTransition(Splitting, Splitting)); - assertTrue(ServiceUnitState.isValidTransition(Splitting, Deleted)); - - assertTrue(ServiceUnitState.isValidTransition(Deleted, Init)); - assertFalse(ServiceUnitState.isValidTransition(Deleted, Free)); - assertFalse(ServiceUnitState.isValidTransition(Deleted, Assigning)); - assertFalse(ServiceUnitState.isValidTransition(Deleted, Owned)); - assertFalse(ServiceUnitState.isValidTransition(Deleted, Releasing)); - assertFalse(ServiceUnitState.isValidTransition(Deleted, Splitting)); - assertFalse(ServiceUnitState.isValidTransition(Deleted, Deleted)); + public void testTransitionsOverSystemTopic() { + + assertFalse(ServiceUnitState.isValidTransition(Init, Init, SystemTopic)); + assertTrue(ServiceUnitState.isValidTransition(Init, Free, SystemTopic)); + assertTrue(ServiceUnitState.isValidTransition(Init, Owned, SystemTopic)); + assertTrue(ServiceUnitState.isValidTransition(Init, Assigning, SystemTopic)); + assertTrue(ServiceUnitState.isValidTransition(Init, Releasing, SystemTopic)); + assertTrue(ServiceUnitState.isValidTransition(Init, Splitting, SystemTopic)); + assertTrue(ServiceUnitState.isValidTransition(Init, Deleted, SystemTopic)); + + assertTrue(ServiceUnitState.isValidTransition(Free, Init, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Free, Free, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Free, Owned, SystemTopic)); + assertTrue(ServiceUnitState.isValidTransition(Free, Assigning, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Free, Releasing, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Free, Splitting, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Free, Deleted, SystemTopic)); + + assertFalse(ServiceUnitState.isValidTransition(Assigning, Init, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Assigning, Free, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Assigning, Assigning, SystemTopic)); + assertTrue(ServiceUnitState.isValidTransition(Assigning, Owned, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Assigning, Releasing, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Assigning, Splitting, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Assigning, Deleted, SystemTopic)); + + assertFalse(ServiceUnitState.isValidTransition(Owned, Init, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Owned, Free, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Owned, Assigning, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Owned, Owned, SystemTopic)); + assertTrue(ServiceUnitState.isValidTransition(Owned, Releasing, SystemTopic)); + assertTrue(ServiceUnitState.isValidTransition(Owned, Splitting, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Owned, Deleted, SystemTopic)); + + assertFalse(ServiceUnitState.isValidTransition(Releasing, Init, SystemTopic)); + assertTrue(ServiceUnitState.isValidTransition(Releasing, Free, SystemTopic)); + assertTrue(ServiceUnitState.isValidTransition(Releasing, Assigning, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Releasing, Owned, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Releasing, Releasing, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Releasing, Splitting, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Releasing, Deleted, SystemTopic)); + + assertFalse(ServiceUnitState.isValidTransition(Splitting, Init, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Splitting, Free, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Splitting, Assigning, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Splitting, Owned, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Splitting, Releasing, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Splitting, Splitting, SystemTopic)); + assertTrue(ServiceUnitState.isValidTransition(Splitting, Deleted, SystemTopic)); + + assertTrue(ServiceUnitState.isValidTransition(Deleted, Init, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Free, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Assigning, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Owned, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Releasing, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Splitting, SystemTopic)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Deleted, SystemTopic)); + } + + @Test + public void testTransitionsOverMetadataStore() { + + assertFalse(ServiceUnitState.isValidTransition(Init, Init, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Init, Free, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Init, Owned, MetadataStore)); + assertTrue(ServiceUnitState.isValidTransition(Init, Assigning, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Init, Releasing, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Init, Splitting, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Init, Deleted, MetadataStore)); + + assertFalse(ServiceUnitState.isValidTransition(Free, Init, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Free, Free, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Free, Owned, MetadataStore)); + assertTrue(ServiceUnitState.isValidTransition(Free, Assigning, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Free, Releasing, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Free, Splitting, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Free, Deleted, MetadataStore)); + + assertFalse(ServiceUnitState.isValidTransition(Assigning, Init, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Assigning, Free, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Assigning, Assigning, MetadataStore)); + assertTrue(ServiceUnitState.isValidTransition(Assigning, Owned, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Assigning, Releasing, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Assigning, Splitting, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Assigning, Deleted, MetadataStore)); + + assertFalse(ServiceUnitState.isValidTransition(Owned, Init, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Owned, Free, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Owned, Assigning, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Owned, Owned, MetadataStore)); + assertTrue(ServiceUnitState.isValidTransition(Owned, Releasing, MetadataStore)); + assertTrue(ServiceUnitState.isValidTransition(Owned, Splitting, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Owned, Deleted, MetadataStore)); + + assertFalse(ServiceUnitState.isValidTransition(Releasing, Init, MetadataStore)); + assertTrue(ServiceUnitState.isValidTransition(Releasing, Free, MetadataStore)); + assertTrue(ServiceUnitState.isValidTransition(Releasing, Assigning, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Releasing, Owned, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Releasing, Releasing, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Releasing, Splitting, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Releasing, Deleted, MetadataStore)); + + assertFalse(ServiceUnitState.isValidTransition(Splitting, Init, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Splitting, Free, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Splitting, Assigning, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Splitting, Owned, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Splitting, Releasing, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Splitting, Splitting, MetadataStore)); + assertTrue(ServiceUnitState.isValidTransition(Splitting, Deleted, MetadataStore)); + + assertTrue(ServiceUnitState.isValidTransition(Deleted, Init, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Free, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Assigning, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Owned, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Releasing, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Splitting, MetadataStore)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Deleted, MetadataStore)); } } \ No newline at end of file diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java index 85792a7ba9387..295c157e3596a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java @@ -41,8 +41,8 @@ public void testUpdateBySystemResourceUsage() { conf.setLoadBalancerCPUResourceWeight(0.5); conf.setLoadBalancerMemoryResourceWeight(0.5); conf.setLoadBalancerDirectMemoryResourceWeight(0.5); - conf.setLoadBalancerBandwithInResourceWeight(0.5); - conf.setLoadBalancerBandwithOutResourceWeight(0.5); + conf.setLoadBalancerBandwidthInResourceWeight(0.5); + conf.setLoadBalancerBandwidthOutResourceWeight(0.5); conf.setLoadBalancerHistoryResourcePercentage(0.75); BrokerLoadData data = new BrokerLoadData(); @@ -108,9 +108,9 @@ public void testUpdateBySystemResourceUsage() { assertThat(data.getUpdatedAt(), greaterThanOrEqualTo(now)); assertEquals(data.getReportedAt(), 0l); assertEquals(data.toString(conf), "cpu= 300.00%, memory= 100.00%, directMemory= 2.00%, " - + "bandwithIn= 3.00%, bandwithOut= 4.00%, " + + "bandwidthIn= 3.00%, bandwidthOut= 4.00%, " + "cpuWeight= 0.500000, memoryWeight= 0.500000, directMemoryWeight= 0.500000, " - + "bandwithInResourceWeight= 0.500000, bandwithOutResourceWeight= 0.500000, " + + "bandwidthInResourceWeight= 0.500000, bandwidthOutResourceWeight= 0.500000, " + "msgThroughputIn= 5.00, msgThroughputOut= 6.00, " + "msgRateIn= 7.00, msgRateOut= 8.00, bundleCount= 9, " + "maxResourceUsage= 300.00%, weightedMaxEMA= 187.50%, msgThroughputEMA= 5.00, " @@ -126,8 +126,8 @@ public void testUpdateByBrokerLoadData() { conf.setLoadBalancerCPUResourceWeight(0.5); conf.setLoadBalancerMemoryResourceWeight(0.5); conf.setLoadBalancerDirectMemoryResourceWeight(0.5); - conf.setLoadBalancerBandwithInResourceWeight(0.5); - conf.setLoadBalancerBandwithOutResourceWeight(0.5); + conf.setLoadBalancerBandwidthInResourceWeight(0.5); + conf.setLoadBalancerBandwidthOutResourceWeight(0.5); conf.setLoadBalancerHistoryResourcePercentage(0.75); BrokerLoadData data = new BrokerLoadData(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupDataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupDataTest.java index 7bcd0687f0008..0a9742fd76175 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupDataTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupDataTest.java @@ -18,47 +18,76 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.data; +import static org.testng.Assert.fail; +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertTrue; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.lookup.LookupResult; +import org.apache.pulsar.broker.namespace.LookupOptions; import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener; -import org.junit.Assert; import org.testng.annotations.Test; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; @Test(groups = "broker") public class BrokerLookupDataTest { @Test - public void testConstructors() { + public void testConstructors() throws PulsarServerException, URISyntaxException { String webServiceUrl = "http://localhost:8080"; String webServiceUrlTls = "https://localhoss:8081"; String pulsarServiceUrl = "pulsar://localhost:6650"; String pulsarServiceUrlTls = "pulsar+ssl://localhost:6651"; - Map advertisedListeners = new HashMap<>(); + final String listenerUrl = "pulsar://gateway:7000"; + final String listenerUrlTls = "pulsar://gateway:8000"; + final String listener = "internal"; + Map advertisedListeners = new HashMap<>(){{ + put(listener, AdvertisedListener.builder() + .brokerServiceUrl(new URI(listenerUrl)) + .brokerServiceUrlTls(new URI(listenerUrlTls)) + .build()); + }}; Map protocols = new HashMap<>(){{ put("kafka", "9092"); }}; BrokerLookupData lookupData = new BrokerLookupData( webServiceUrl, webServiceUrlTls, pulsarServiceUrl, pulsarServiceUrlTls, advertisedListeners, protocols, true, true, - ExtensibleLoadManagerImpl.class.getName(), System.currentTimeMillis(),"3.0"); - Assert.assertEquals(webServiceUrl, lookupData.webServiceUrl()); - Assert.assertEquals(webServiceUrlTls, lookupData.webServiceUrlTls()); - Assert.assertEquals(pulsarServiceUrl, lookupData.pulsarServiceUrl()); - Assert.assertEquals(pulsarServiceUrlTls, lookupData.pulsarServiceUrlTls()); - Assert.assertEquals(Optional.of("9092"), lookupData.getProtocol("kafka")); - Assert.assertEquals(Optional.empty(), lookupData.getProtocol("echo")); - Assert.assertTrue(lookupData.persistentTopicsEnabled()); - Assert.assertTrue(lookupData.nonPersistentTopicsEnabled()); - Assert.assertEquals("3.0", lookupData.brokerVersion()); + ExtensibleLoadManagerImpl.class.getName(), System.currentTimeMillis(),"3.0", + Collections.emptyMap()); + assertEquals(webServiceUrl, lookupData.webServiceUrl()); + assertEquals(webServiceUrlTls, lookupData.webServiceUrlTls()); + assertEquals(pulsarServiceUrl, lookupData.pulsarServiceUrl()); + assertEquals(pulsarServiceUrlTls, lookupData.pulsarServiceUrlTls()); + assertEquals(Optional.of("9092"), lookupData.getProtocol("kafka")); + assertEquals(Optional.empty(), lookupData.getProtocol("echo")); + assertTrue(lookupData.persistentTopicsEnabled()); + assertTrue(lookupData.nonPersistentTopicsEnabled()); + assertEquals("3.0", lookupData.brokerVersion()); + + LookupResult lookupResult = lookupData.toLookupResult(LookupOptions.builder().build()); + assertEquals(webServiceUrl, lookupResult.getLookupData().getHttpUrl()); + assertEquals(webServiceUrlTls, lookupResult.getLookupData().getHttpUrlTls()); + assertEquals(pulsarServiceUrl, lookupResult.getLookupData().getBrokerUrl()); + assertEquals(pulsarServiceUrlTls, lookupResult.getLookupData().getBrokerUrlTls()); - LookupResult lookupResult = lookupData.toLookupResult(); - Assert.assertEquals(webServiceUrl, lookupResult.getLookupData().getHttpUrl()); - Assert.assertEquals(webServiceUrlTls, lookupResult.getLookupData().getHttpUrlTls()); - Assert.assertEquals(pulsarServiceUrl, lookupResult.getLookupData().getBrokerUrl()); - Assert.assertEquals(pulsarServiceUrlTls, lookupResult.getLookupData().getBrokerUrlTls()); + try { + lookupData.toLookupResult(LookupOptions.builder().advertisedListenerName("others").build()); + fail(); + } catch (PulsarServerException ex) { + assertTrue(ex.getMessage().contains("the broker do not have others listener")); + } + lookupResult = lookupData.toLookupResult(LookupOptions.builder().advertisedListenerName(listener).build()); + assertEquals(listenerUrl, lookupResult.getLookupData().getBrokerUrl()); + assertEquals(listenerUrlTls, lookupResult.getLookupData().getBrokerUrlTls()); + assertEquals(webServiceUrl, lookupResult.getLookupData().getHttpUrl()); + assertEquals(webServiceUrlTls, lookupResult.getLookupData().getHttpUrlTls()); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java index 68bd7b29094cd..ab0065e0aa5ba 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.mock; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -90,10 +91,25 @@ public void closeTableView() throws IOException { } + @Override + public void start() throws LoadDataStoreException { + + } + + @Override + public void init() throws IOException { + + } + @Override public void startTableView() throws LoadDataStoreException { } + + @Override + public void startProducer() throws LoadDataStoreException { + + } }; configuration.setPreferLaterVersions(true); doReturn(configuration).when(mockContext).brokerConfiguration(); @@ -121,6 +137,6 @@ public BrokerLookupData getLookupData(String version, String loadManagerClassNam return new BrokerLookupData( webServiceUrl, webServiceUrlTls, pulsarServiceUrl, pulsarServiceUrlTls, advertisedListeners, protocols, true, true, - loadManagerClassName, -1, version); + loadManagerClassName, -1, version, Collections.emptyMap()); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java index c2c534f72e9db..d3553bd25d1fa 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java @@ -28,9 +28,13 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; + import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; @@ -67,7 +71,7 @@ public class BrokerIsolationPoliciesFilterTest { */ @Test public void testFilterWithNamespaceIsolationPoliciesForPrimaryAndSecondaryBrokers() - throws IllegalAccessException, BrokerFilterException { + throws IllegalAccessException, BrokerFilterException, ExecutionException, InterruptedException { var namespace = "my-tenant/my-ns"; NamespaceName namespaceName = NamespaceName.get(namespace); @@ -80,48 +84,48 @@ public void testFilterWithNamespaceIsolationPoliciesForPrimaryAndSecondaryBroker BrokerIsolationPoliciesFilter filter = new BrokerIsolationPoliciesFilter(isolationPoliciesHelper); // a. available-brokers: broker1, broker2, broker3 => result: broker1 - Map result = filter.filter(new HashMap<>(Map.of( - "broker1", getLookupData(), - "broker2", getLookupData(), - "broker3", getLookupData())), namespaceName, getContext()); - assertEquals(result.keySet(), Set.of("broker1")); + Map result = filter.filterAsync(new HashMap<>(Map.of( + "broker1:8080", getLookupData(), + "broker2:8080", getLookupData(), + "broker3:8080", getLookupData())), namespaceName, getContext()).get(); + assertEquals(result.keySet(), Set.of("broker1:8080")); // b. available-brokers: broker2, broker3 => result: broker2 - result = filter.filter(new HashMap<>(Map.of( - "broker2", getLookupData(), - "broker3", getLookupData())), namespaceName, getContext()); - assertEquals(result.keySet(), Set.of("broker2")); + result = filter.filterAsync(new HashMap<>(Map.of( + "broker2:8080", getLookupData(), + "broker3:8080", getLookupData())), namespaceName, getContext()).get(); + assertEquals(result.keySet(), Set.of("broker2:8080")); // c. available-brokers: broker3 => result: NULL - result = filter.filter(new HashMap<>(Map.of( - "broker3", getLookupData())), namespaceName, getContext()); + result = filter.filterAsync(new HashMap<>(Map.of( + "broker3:8080", getLookupData())), namespaceName, getContext()).get(); assertTrue(result.isEmpty()); // 2. Namespace: primary=broker1, secondary=broker2, shared=broker3, min_limit = 2 setIsolationPolicies(policies, namespaceName, Set.of("broker1"), Set.of("broker2"), Set.of("broker3"), 2); // a. available-brokers: broker1, broker2, broker3 => result: broker1, broker2 - result = filter.filter(new HashMap<>(Map.of( - "broker1", getLookupData(), - "broker2", getLookupData(), - "broker3", getLookupData())), namespaceName, getContext()); - assertEquals(result.keySet(), Set.of("broker1", "broker2")); + result = filter.filterAsync(new HashMap<>(Map.of( + "broker1:8080", getLookupData(), + "broker2:8080", getLookupData(), + "broker3:8080", getLookupData())), namespaceName, getContext()).get(); + assertEquals(result.keySet(), Set.of("broker1:8080", "broker2:8080")); // b. available-brokers: broker2, broker3 => result: broker2 - result = filter.filter(new HashMap<>(Map.of( - "broker2", getLookupData(), - "broker3", getLookupData())), namespaceName, getContext()); - assertEquals(result.keySet(), Set.of("broker2")); + result = filter.filterAsync(new HashMap<>(Map.of( + "broker2:8080", getLookupData(), + "broker3:8080", getLookupData())), namespaceName, getContext()).get(); + assertEquals(result.keySet(), Set.of("broker2:8080")); // c. available-brokers: broker3 => result: NULL - result = filter.filter(new HashMap<>(Map.of( - "broker3", getLookupData())), namespaceName, getContext()); + result = filter.filterAsync(new HashMap<>(Map.of( + "broker3:8080", getLookupData())), namespaceName, getContext()).get(); assertTrue(result.isEmpty()); } @Test public void testFilterWithPersistentOrNonPersistentDisabled() - throws IllegalAccessException, BrokerFilterException { + throws IllegalAccessException, BrokerFilterException, ExecutionException, InterruptedException { var namespace = "my-tenant/my-ns"; NamespaceName namespaceName = NamespaceName.get(namespace); NamespaceBundle namespaceBundle = mock(NamespaceBundle.class); @@ -129,7 +133,8 @@ public void testFilterWithPersistentOrNonPersistentDisabled() doReturn(namespaceName).when(namespaceBundle).getNamespaceObject(); var policies = mock(SimpleResourceAllocationPolicies.class); - doReturn(false).when(policies).areIsolationPoliciesPresent(eq(namespaceName)); + doReturn(CompletableFuture.completedFuture(false)) + .when(policies).areIsolationPoliciesPresentAsync(eq(namespaceName)); doReturn(true).when(policies).isSharedBroker(any()); IsolationPoliciesHelper isolationPoliciesHelper = new IsolationPoliciesHelper(policies); @@ -137,32 +142,32 @@ public void testFilterWithPersistentOrNonPersistentDisabled() - Map result = filter.filter(new HashMap<>(Map.of( - "broker1", getLookupData(), - "broker2", getLookupData(), - "broker3", getLookupData())), namespaceBundle, getContext()); - assertEquals(result.keySet(), Set.of("broker1", "broker2", "broker3")); + Map result = filter.filterAsync(new HashMap<>(Map.of( + "broker1:8080", getLookupData(), + "broker2:8080", getLookupData(), + "broker3:8080", getLookupData())), namespaceBundle, getContext()).get(); + assertEquals(result.keySet(), Set.of("broker1:8080", "broker2:8080", "broker3:8080")); - result = filter.filter(new HashMap<>(Map.of( - "broker1", getLookupData(true, false), - "broker2", getLookupData(true, false), - "broker3", getLookupData())), namespaceBundle, getContext()); - assertEquals(result.keySet(), Set.of("broker3")); + result = filter.filterAsync(new HashMap<>(Map.of( + "broker1:8080", getLookupData(true, false), + "broker2:8080", getLookupData(true, false), + "broker3:8080", getLookupData())), namespaceBundle, getContext()).get(); + assertEquals(result.keySet(), Set.of("broker3:8080")); doReturn(false).when(namespaceBundle).hasNonPersistentTopic(); - result = filter.filter(new HashMap<>(Map.of( - "broker1", getLookupData(), - "broker2", getLookupData(), - "broker3", getLookupData())), namespaceBundle, getContext()); - assertEquals(result.keySet(), Set.of("broker1", "broker2", "broker3")); - - result = filter.filter(new HashMap<>(Map.of( - "broker1", getLookupData(false, true), - "broker2", getLookupData(), - "broker3", getLookupData())), namespaceBundle, getContext()); - assertEquals(result.keySet(), Set.of("broker2", "broker3")); + result = filter.filterAsync(new HashMap<>(Map.of( + "broker1:8080", getLookupData(), + "broker2:8080", getLookupData(), + "broker3:8080", getLookupData())), namespaceBundle, getContext()).get(); + assertEquals(result.keySet(), Set.of("broker1:8080", "broker2:8080", "broker3:8080")); + + result = filter.filterAsync(new HashMap<>(Map.of( + "broker1:8080", getLookupData(false, true), + "broker2:8080", getLookupData(), + "broker3:8080", getLookupData())), namespaceBundle, getContext()).get(); + assertEquals(result.keySet(), Set.of("broker2:8080", "broker3:8080")); } private void setIsolationPolicies(SimpleResourceAllocationPolicies policies, @@ -172,7 +177,8 @@ private void setIsolationPolicies(SimpleResourceAllocationPolicies policies, Set shared, int min_limit) { reset(policies); - doReturn(true).when(policies).areIsolationPoliciesPresent(eq(namespaceName)); + doReturn(CompletableFuture.completedFuture(true)) + .when(policies).areIsolationPoliciesPresentAsync(eq(namespaceName)); doReturn(false).when(policies).isPrimaryBroker(eq(namespaceName), any()); doReturn(false).when(policies).isSecondaryBroker(eq(namespaceName), any()); doReturn(false).when(policies).isSharedBroker(any()); @@ -213,7 +219,7 @@ public BrokerLookupData getLookupData(boolean persistentTopicsEnabled, webServiceUrl, webServiceUrlTls, pulsarServiceUrl, pulsarServiceUrlTls, advertisedListeners, protocols, persistentTopicsEnabled, nonPersistentTopicsEnabled, - ExtensibleLoadManagerImpl.class.getName(), System.currentTimeMillis(), "3.0.0"); + ExtensibleLoadManagerImpl.class.getName(), System.currentTimeMillis(), "3.0.0", Collections.emptyMap()); } public LoadManagerContext getContext() { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java index 4aef87cf63aa8..17475b419576f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java @@ -27,6 +27,7 @@ import org.testng.annotations.Test; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ExecutionException; /** @@ -35,7 +36,7 @@ public class BrokerLoadManagerClassFilterTest extends BrokerFilterTestBase { @Test - public void test() throws BrokerFilterException { + public void test() throws BrokerFilterException, ExecutionException, InterruptedException { LoadManagerContext context = getContext(); context.brokerConfiguration().setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); BrokerLoadManagerClassFilter filter = new BrokerLoadManagerClassFilter(); @@ -48,14 +49,14 @@ public void test() throws BrokerFilterException { "broker5", getLookupData("3.0.0", null) ); - Map result = filter.filter(new HashMap<>(originalBrokers), null, context); + Map result = filter.filterAsync(new HashMap<>(originalBrokers), null, context).get(); assertEquals(result, Map.of( "broker1", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()), "broker2", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()) )); context.brokerConfiguration().setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); - result = filter.filter(new HashMap<>(originalBrokers), null, context); + result = filter.filterAsync(new HashMap<>(originalBrokers), null, context).get(); assertEquals(result, Map.of( "broker3", getLookupData("3.0.0", ModularLoadManagerImpl.class.getName()), diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java index da13a9526a881..9d000cb91a155 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ExecutionException; import static org.testng.Assert.assertEquals; @@ -38,7 +39,7 @@ public class BrokerMaxTopicCountFilterTest extends BrokerFilterTestBase { @Test - public void test() throws IllegalAccessException, BrokerFilterException { + public void test() throws IllegalAccessException, BrokerFilterException, ExecutionException, InterruptedException { LoadManagerContext context = getContext(); LoadDataStore store = context.brokerLoadDataStore(); BrokerLoadData maxTopicLoadData = new BrokerLoadData(); @@ -58,7 +59,8 @@ public void test() throws IllegalAccessException, BrokerFilterException { "broker3", getLookupData(), "broker4", getLookupData() ); - Map result = filter.filter(new HashMap<>(originalBrokers), null, context); + Map result = + filter.filterAsync(new HashMap<>(originalBrokers), null, context).get(); assertEquals(result, Map.of( "broker2", getLookupData(), "broker4", getLookupData() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java index cafd8f0ea7a4c..2aa7faeb599fd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java @@ -21,9 +21,11 @@ import static org.mockito.Mockito.doReturn; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ExecutionException; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.BrokerFilterBadVersionException; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; @@ -39,14 +41,14 @@ public class BrokerVersionFilterTest extends BrokerFilterTestBase { @Test - public void testFilterEmptyBrokerList() throws BrokerFilterException { + public void testFilterEmptyBrokerList() throws BrokerFilterException, ExecutionException, InterruptedException { BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - Map result = brokerVersionFilter.filter(new HashMap<>(), null, getContext()); + Map result = brokerVersionFilter.filterAsync(new HashMap<>(), null, getContext()).get(); assertTrue(result.isEmpty()); } @Test - public void testDisabledFilter() throws BrokerFilterException { + public void testDisabledFilter() throws BrokerFilterException, ExecutionException, InterruptedException { LoadManagerContext context = getContext(); ServiceConfiguration configuration = new ServiceConfiguration(); configuration.setPreferLaterVersions(false); @@ -58,12 +60,12 @@ public void testDisabledFilter() throws BrokerFilterException { ); Map brokers = new HashMap<>(originalBrokers); BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - Map result = brokerVersionFilter.filter(brokers, null, context); + Map result = brokerVersionFilter.filterAsync(brokers, null, context).get(); assertEquals(result, originalBrokers); } @Test - public void testFilter() throws BrokerFilterException { + public void testFilter() throws BrokerFilterException, ExecutionException, InterruptedException { Map originalBrokers = Map.of( "localhost:6650", getLookupData("2.10.0"), "localhost:6651", getLookupData("2.10.1"), @@ -71,8 +73,8 @@ public void testFilter() throws BrokerFilterException { "localhost:6653", getLookupData("2.10.1") ); BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - Map result = brokerVersionFilter.filter( - new HashMap<>(originalBrokers), null, getContext()); + Map result = brokerVersionFilter.filterAsync( + new HashMap<>(originalBrokers), null, getContext()).get(); assertEquals(result, Map.of( "localhost:6651", getLookupData("2.10.1"), "localhost:6652", getLookupData("2.10.1"), @@ -85,7 +87,7 @@ public void testFilter() throws BrokerFilterException { "localhost:6652", getLookupData("2.10.1"), "localhost:6653", getLookupData("2.10.1") ); - result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()); + result = brokerVersionFilter.filterAsync(new HashMap<>(originalBrokers), null, getContext()).get(); assertEquals(result, Map.of( "localhost:6652", getLookupData("2.10.1"), @@ -99,19 +101,24 @@ public void testFilter() throws BrokerFilterException { "localhost:6653", getLookupData("2.10.2-SNAPSHOT") ); - result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()); + result = brokerVersionFilter.filterAsync(new HashMap<>(originalBrokers), null, getContext()).get(); assertEquals(result, Map.of( "localhost:6653", getLookupData("2.10.2-SNAPSHOT") )); } - @Test(expectedExceptions = BrokerFilterBadVersionException.class) - public void testInvalidVersionString() throws BrokerFilterException { + @Test + public void testInvalidVersionString() { Map originalBrokers = Map.of( "localhost:6650", getLookupData("xxx") ); BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()); + try { + brokerVersionFilter.filterAsync(new HashMap<>(originalBrokers), null, getContext()).get(); + fail(); + } catch (Exception ex) { + assertEquals(ex.getCause().getClass(), BrokerFilterBadVersionException.class); + } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManagerTest.java index cbf77b59d5ad6..f2e9cf86868e2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManagerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManagerTest.java @@ -33,6 +33,8 @@ import org.apache.pulsar.broker.lookup.LookupResult; import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener; import org.testng.annotations.Test; + +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -106,6 +108,6 @@ public BrokerLookupData getLookupData(String broker, String loadManagerClassName return new BrokerLookupData( webServiceUrl, webServiceUrlTls, pulsarServiceUrl, pulsarServiceUrlTls, advertisedListeners, protocols, true, true, - loadManagerClassName, startTimeStamp, "3.0.0"); + loadManagerClassName, startTimeStamp, "3.0.0", Collections.emptyMap()); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManagerTest.java index 3287306ab48ba..57b7830214b92 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManagerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManagerTest.java @@ -123,40 +123,23 @@ public void testSuccess() throws IllegalAccessException, ExecutionException, Int manager.handleEvent(bundle, new ServiceUnitStateData(ServiceUnitState.Free, dstBroker, VERSION_ID_INIT), null); assertEquals(inFlightUnloadRequests.size(), 1); - assertEquals(counter.toMetrics(null).toString(), - counterExpected.toMetrics(null).toString()); manager.handleEvent(bundle, new ServiceUnitStateData(ServiceUnitState.Deleted, dstBroker, VERSION_ID_INIT), null); - counterExpected.update(SplitDecision.Label.Success, Sessions); - assertEquals(inFlightUnloadRequests.size(), 0); - assertEquals(counter.toMetrics(null).toString(), - counterExpected.toMetrics(null).toString()); + assertEquals(inFlightUnloadRequests.size(), 1); - // Success with Init state. - future = manager.waitAsync(CompletableFuture.completedFuture(null), - bundle, decision, 5, TimeUnit.SECONDS); - inFlightUnloadRequests = getinFlightUnloadRequests(manager); + manager.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Owned, dstBroker, VERSION_ID_INIT), null); assertEquals(inFlightUnloadRequests.size(), 1); + + // Success with Init state. manager.handleEvent(bundle, new ServiceUnitStateData(ServiceUnitState.Init, dstBroker, VERSION_ID_INIT), null); assertEquals(inFlightUnloadRequests.size(), 0); counterExpected.update(SplitDecision.Label.Success, Sessions); assertEquals(counter.toMetrics(null).toString(), counterExpected.toMetrics(null).toString()); - future.get(); - // Success with Owned state. - future = manager.waitAsync(CompletableFuture.completedFuture(null), - bundle, decision, 5, TimeUnit.SECONDS); - inFlightUnloadRequests = getinFlightUnloadRequests(manager); - assertEquals(inFlightUnloadRequests.size(), 1); - manager.handleEvent(bundle, - new ServiceUnitStateData(ServiceUnitState.Owned, dstBroker, VERSION_ID_INIT), null); - assertEquals(inFlightUnloadRequests.size(), 0); - counterExpected.update(SplitDecision.Label.Success, Sessions); - assertEquals(counter.toMetrics(null).toString(), - counterExpected.toMetrics(null).toString()); future.get(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java index 6a2ae1cc562cc..be78cfcb595c5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java @@ -26,7 +26,6 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -49,7 +48,7 @@ public class UnloadManagerTest { @Test public void testEventPubFutureHasException() { UnloadCounter counter = new UnloadCounter(); - UnloadManager manager = new UnloadManager(counter); + UnloadManager manager = new UnloadManager(counter, "mockBrokerId"); var unloadDecision = new UnloadDecision(new Unload("broker-1", "bundle-1"), Success, Admin); CompletableFuture future = @@ -69,7 +68,7 @@ public void testEventPubFutureHasException() { @Test public void testTimeout() throws IllegalAccessException { UnloadCounter counter = new UnloadCounter(); - UnloadManager manager = new UnloadManager(counter); + UnloadManager manager = new UnloadManager(counter, "mockBrokerId"); var unloadDecision = new UnloadDecision(new Unload("broker-1", "bundle-1"), Success, Admin); CompletableFuture future = @@ -93,61 +92,80 @@ public void testTimeout() throws IllegalAccessException { @Test public void testSuccess() throws IllegalAccessException, ExecutionException, InterruptedException { UnloadCounter counter = new UnloadCounter(); - UnloadManager manager = new UnloadManager(counter); + UnloadManager manager = new UnloadManager(counter, "mockBrokerId"); + String dstBroker = "broker-2"; + String srcBroker = "broker-1"; + String bundle = "bundle-1"; var unloadDecision = - new UnloadDecision(new Unload("broker-1", "bundle-1"), Success, Admin); + new UnloadDecision(new Unload(srcBroker, bundle), Success, Admin); CompletableFuture future = manager.waitAsync(CompletableFuture.completedFuture(null), - "bundle-1", unloadDecision, 5, TimeUnit.SECONDS); + bundle, unloadDecision, 5, TimeUnit.SECONDS); Map> inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager); assertEquals(inFlightUnloadRequestMap.size(), 1); - manager.handleEvent("bundle-1", - new ServiceUnitStateData(ServiceUnitState.Assigning, "broker-1", VERSION_ID_INIT), null); + manager.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Assigning, null, srcBroker, VERSION_ID_INIT), null); assertEquals(inFlightUnloadRequestMap.size(), 1); - manager.handleEvent("bundle-1", - new ServiceUnitStateData(ServiceUnitState.Deleted, "broker-1", VERSION_ID_INIT), null); + manager.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Deleted, null, srcBroker, VERSION_ID_INIT), null); assertEquals(inFlightUnloadRequestMap.size(), 1); - manager.handleEvent("bundle-1", - new ServiceUnitStateData(ServiceUnitState.Splitting, "broker-1", VERSION_ID_INIT), null); + manager.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Splitting, null, srcBroker, VERSION_ID_INIT), null); assertEquals(inFlightUnloadRequestMap.size(), 1); - manager.handleEvent("bundle-1", - new ServiceUnitStateData(ServiceUnitState.Releasing, "broker-1", VERSION_ID_INIT), null); + manager.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Releasing, null, srcBroker, VERSION_ID_INIT), null); assertEquals(inFlightUnloadRequestMap.size(), 1); - manager.handleEvent("bundle-1", - new ServiceUnitStateData(ServiceUnitState.Init, "broker-1", VERSION_ID_INIT), null); + manager.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Free, null, srcBroker, true, VERSION_ID_INIT), null); assertEquals(inFlightUnloadRequestMap.size(), 1); - manager.handleEvent("bundle-1", - new ServiceUnitStateData(ServiceUnitState.Free, "broker-1", VERSION_ID_INIT), null); + // Success with Init state. + manager.handleEvent(bundle, null, null); assertEquals(inFlightUnloadRequestMap.size(), 0); future.get(); assertEquals(counter.getBreakdownCounters().get(Success).get(Admin).get(), 1); // Success with Owned state. future = manager.waitAsync(CompletableFuture.completedFuture(null), - "bundle-1", unloadDecision, 5, TimeUnit.SECONDS); + bundle, unloadDecision, 5, TimeUnit.SECONDS); inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager); - assertEquals(inFlightUnloadRequestMap.size(), 1); + manager.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Owned, dstBroker, null, VERSION_ID_INIT), null); + assertEquals(inFlightUnloadRequestMap.size(), 1); + manager.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Owned, dstBroker, srcBroker, VERSION_ID_INIT), null); + assertEquals(inFlightUnloadRequestMap.size(), 0); + future.get(); + assertEquals(counter.getBreakdownCounters().get(Success).get(Admin).get(), 2); - manager.handleEvent("bundle-1", - new ServiceUnitStateData(ServiceUnitState.Owned, "broker-1", VERSION_ID_INIT), null); + // Success with Free state. + future = manager.waitAsync(CompletableFuture.completedFuture(null), + bundle, unloadDecision, 5, TimeUnit.SECONDS); + inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager); + assertEquals(inFlightUnloadRequestMap.size(), 1); + manager.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Free, dstBroker, srcBroker, true, VERSION_ID_INIT), null); + assertEquals(inFlightUnloadRequestMap.size(), 1); + manager.handleEvent(bundle, + new ServiceUnitStateData(ServiceUnitState.Free, dstBroker, srcBroker, false, VERSION_ID_INIT), null); assertEquals(inFlightUnloadRequestMap.size(), 0); future.get(); + assertEquals(counter.getBreakdownCounters().get(Success).get(Admin).get(), 3); + - assertEquals(counter.getBreakdownCounters().get(Success).get(Admin).get(), 2); } @Test public void testFailedStage() throws IllegalAccessException { UnloadCounter counter = new UnloadCounter(); - UnloadManager manager = new UnloadManager(counter); + UnloadManager manager = new UnloadManager(counter, "mockBrokerId"); var unloadDecision = new UnloadDecision(new Unload("broker-1", "bundle-1"), Success, Admin); CompletableFuture future = @@ -158,7 +176,7 @@ public void testFailedStage() throws IllegalAccessException { assertEquals(inFlightUnloadRequestMap.size(), 1); manager.handleEvent("bundle-1", - new ServiceUnitStateData(ServiceUnitState.Owned, "broker-1", VERSION_ID_INIT), + new ServiceUnitStateData(ServiceUnitState.Owned, null, "broker-1", VERSION_ID_INIT), new IllegalStateException("Failed stage.")); try { @@ -176,7 +194,7 @@ public void testFailedStage() throws IllegalAccessException { @Test public void testClose() throws IllegalAccessException { UnloadCounter counter = new UnloadCounter(); - UnloadManager manager = new UnloadManager(counter); + UnloadManager manager = new UnloadManager(counter, "mockBrokerId"); var unloadDecision = new UnloadDecision(new Unload("broker-1", "bundle-1"), Success, Admin); CompletableFuture future = diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java index 93ab35981e1c4..9b0530349d036 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java @@ -45,9 +45,9 @@ import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; +import org.awaitility.Awaitility; import org.mockito.MockedStatic; import org.mockito.Mockito; -import org.testcontainers.shaded.org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java index be8c6af2b0404..344387b293004 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java @@ -48,7 +48,7 @@ import org.apache.pulsar.broker.service.PulsarStats; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; -import org.testcontainers.shaded.org.awaitility.Awaitility; +import org.awaitility.Awaitility; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java index af5890fcacbb2..48bef15b5f80a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java @@ -46,6 +46,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -100,6 +101,7 @@ import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; +import org.assertj.core.api.Assertions; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -168,18 +170,18 @@ public LoadManagerContext setupContext(){ var ctx = getContext(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000)); - topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 1000000, 3000000)); - topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("my-tenant/my-namespaceC", 2000000, 4000000)); - topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD", 2000000, 6000000)); - topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 2000000, 7000000)); + topBundlesLoadDataStore.pushAsync("broker1:8080", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000)); + topBundlesLoadDataStore.pushAsync("broker2:8080", getTopBundlesLoad("my-tenant/my-namespaceB", 1000000, 3000000)); + topBundlesLoadDataStore.pushAsync("broker3:8080", getTopBundlesLoad("my-tenant/my-namespaceC", 2000000, 4000000)); + topBundlesLoadDataStore.pushAsync("broker4:8080", getTopBundlesLoad("my-tenant/my-namespaceD", 2000000, 6000000)); + topBundlesLoadDataStore.pushAsync("broker5:8080", getTopBundlesLoad("my-tenant/my-namespaceE", 2000000, 7000000)); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 2, "broker1")); - brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 4, "broker2")); - brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx, 6, "broker3")); - brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 80, "broker4")); - brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 90, "broker5")); + brokerLoadDataStore.pushAsync("broker1:8080", getCpuLoad(ctx, 2, "broker1:8080")); + brokerLoadDataStore.pushAsync("broker2:8080", getCpuLoad(ctx, 4, "broker2:8080")); + brokerLoadDataStore.pushAsync("broker3:8080", getCpuLoad(ctx, 6, "broker3:8080")); + brokerLoadDataStore.pushAsync("broker4:8080", getCpuLoad(ctx, 80, "broker4:8080")); + brokerLoadDataStore.pushAsync("broker5:8080", getCpuLoad(ctx, 90, "broker5:8080")); return ctx; } @@ -192,9 +194,9 @@ public LoadManagerContext setupContext(int clusterSize) { Random rand = new Random(); for (int i = 0; i < clusterSize; i++) { int brokerLoad = rand.nextInt(1000); - brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx, brokerLoad, "broker" + i)); + brokerLoadDataStore.pushAsync("broker" + i + ":8080", getCpuLoad(ctx, brokerLoad, "broker" + i + ":8080")); int bundleLoad = rand.nextInt(brokerLoad + 1); - topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i, + topBundlesLoadDataStore.pushAsync("broker" + i + ":8080", getTopBundlesLoad("my-tenant/my-namespace" + i, bundleLoad, brokerLoad - bundleLoad)); } return ctx; @@ -209,14 +211,14 @@ public LoadManagerContext setupContextLoadSkewedOverload(int clusterSize) { int i = 0; for (; i < clusterSize-1; i++) { int brokerLoad = 1; - topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i, + topBundlesLoadDataStore.pushAsync("broker" + i + ":8080", getTopBundlesLoad("my-tenant/my-namespace" + i, 300_000, 700_000)); - brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx, brokerLoad, "broker" + i)); + brokerLoadDataStore.pushAsync("broker" + i + ":8080", getCpuLoad(ctx, brokerLoad, "broker" + i + ":8080")); } int brokerLoad = 100; - topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i, + topBundlesLoadDataStore.pushAsync("broker" + i + ":8080", getTopBundlesLoad("my-tenant/my-namespace" + i, 30_000_000, 70_000_000)); - brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx, brokerLoad, "broker" + i)); + brokerLoadDataStore.pushAsync("broker" + i + ":8080", getCpuLoad(ctx, brokerLoad, "broker" + i + ":8080")); return ctx; } @@ -230,21 +232,21 @@ public LoadManagerContext setupContextLoadSkewedUnderload(int clusterSize) { int i = 0; for (; i < clusterSize-2; i++) { int brokerLoad = 98; - topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i, + topBundlesLoadDataStore.pushAsync("broker" + i + ":8080", getTopBundlesLoad("my-tenant/my-namespace" + i, 30_000_000, 70_000_000)); - brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx, brokerLoad, "broker" + i)); + brokerLoadDataStore.pushAsync("broker" + i + ":8080", getCpuLoad(ctx, brokerLoad, "broker" + i + ":8080")); } int brokerLoad = 99; - topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i, + topBundlesLoadDataStore.pushAsync("broker" + i + ":8080", getTopBundlesLoad("my-tenant/my-namespace" + i, 30_000_000, 70_000_000)); - brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx, brokerLoad, "broker" + i)); + brokerLoadDataStore.pushAsync("broker" + i + ":8080", getCpuLoad(ctx, brokerLoad, "broker" + i + ":8080")); i++; brokerLoad = 1; - topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i, + topBundlesLoadDataStore.pushAsync("broker" + i + ":8080", getTopBundlesLoad("my-tenant/my-namespace" + i, 300_000, 700_000)); - brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx, brokerLoad, "broker" + i)); + brokerLoadDataStore.pushAsync("broker" + i + ":8080", getCpuLoad(ctx, brokerLoad, "broker" + i + ":8080")); return ctx; } @@ -383,10 +385,25 @@ public void closeTableView() throws IOException { } + @Override + public void start() throws LoadDataStoreException { + + } + + @Override + public void init() throws IOException { + + } + @Override public void startTableView() throws LoadDataStoreException { } + + @Override + public void startProducer() throws LoadDataStoreException { + + } }; var topBundleLoadDataStore = new LoadDataStore() { @@ -436,19 +453,34 @@ public void closeTableView() throws IOException { } + @Override + public void start() throws LoadDataStoreException { + + } + + @Override + public void init() throws IOException { + + } + @Override public void startTableView() throws LoadDataStoreException { } + + @Override + public void startProducer() throws LoadDataStoreException { + + } }; BrokerRegistry brokerRegistry = mock(BrokerRegistry.class); doReturn(CompletableFuture.completedFuture(Map.of( - "broker1", getMockBrokerLookupData(), - "broker2", getMockBrokerLookupData(), - "broker3", getMockBrokerLookupData(), - "broker4", getMockBrokerLookupData(), - "broker5", getMockBrokerLookupData() + "broker1:8080", getMockBrokerLookupData(), + "broker2:8080", getMockBrokerLookupData(), + "broker3:8080", getMockBrokerLookupData(), + "broker4:8080", getMockBrokerLookupData(), + "broker5:8080", getMockBrokerLookupData() ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync(); doReturn(conf).when(ctx).brokerConfiguration(); doReturn(brokerLoadDataStore).when(ctx).brokerLoadDataStore(); @@ -496,11 +528,11 @@ public void testEmptyTopBundlesLoadData() { var ctx = getContext(); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 2, "broker1")); - brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 4, "broker2")); - brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx, 6, "broker3")); - brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 80, "broker4")); - brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 90, "broker5")); + brokerLoadDataStore.pushAsync("broker1:8080", getCpuLoad(ctx, 2, "broker1:8080")); + brokerLoadDataStore.pushAsync("broker2:8080", getCpuLoad(ctx, 4, "broker2:8080")); + brokerLoadDataStore.pushAsync("broker3:8080", getCpuLoad(ctx, 6, "broker3:8080")); + brokerLoadDataStore.pushAsync("broker4:8080", getCpuLoad(ctx, 80, "broker4:8080")); + brokerLoadDataStore.pushAsync("broker5:8080", getCpuLoad(ctx, 90, "broker5:8080")); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); @@ -520,11 +552,11 @@ public void testOutDatedLoadData() throws IllegalAccessException { assertEquals(res.size(), 2); - FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker1").get(), "updatedAt", 0, true); - FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker2").get(), "updatedAt", 0, true); - FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker3").get(), "updatedAt", 0, true); - FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker4").get(), "updatedAt", 0, true); - FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker5").get(), "updatedAt", 0, true); + FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker1:8080").get(), "updatedAt", 0, true); + FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker2:8080").get(), "updatedAt", 0, true); + FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker3:8080").get(), "updatedAt", 0, true); + FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker4:8080").get(), "updatedAt", 0, true); + FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker5:8080").get(), "updatedAt", 0, true); res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); @@ -541,20 +573,20 @@ public void testRecentlyUnloadedBrokers() { Map recentlyUnloadedBrokers = new HashMap<>(); var oldTS = System.currentTimeMillis() - ctx.brokerConfiguration() .getLoadBalancerBrokerLoadDataTTLInSeconds() * 1001; - recentlyUnloadedBrokers.put("broker1", oldTS); + recentlyUnloadedBrokers.put("broker1:8080", oldTS); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), recentlyUnloadedBrokers); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + expected.add(new UnloadDecision(new Unload("broker5:8080", bundleE1, Optional.of("broker1:8080")), Success, Overloaded)); - expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + expected.add(new UnloadDecision(new Unload("broker4:8080", bundleD1, Optional.of("broker2:8080")), Success, Overloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), setupLoadAvg); assertEquals(counter.getLoadStd(), setupLoadStd); var now = System.currentTimeMillis(); - recentlyUnloadedBrokers.put("broker1", now); + recentlyUnloadedBrokers.put("broker1:8080", now); res = transferShedder.findBundlesForUnloading(ctx, Map.of(), recentlyUnloadedBrokers); assertTrue(res.isEmpty()); @@ -574,9 +606,9 @@ public void testRecentlyUnloadedBundles() { recentlyUnloadedBundles.put(bundleD2, now); var res = transferShedder.findBundlesForUnloading(ctx, recentlyUnloadedBundles, Map.of()); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker3", + expected.add(new UnloadDecision(new Unload("broker3:8080", "my-tenant/my-namespaceC/0x00000000_0x0FFFFFFF", - Optional.of("broker1")), + Optional.of("broker1:8080")), Success, Overloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), setupLoadAvg); @@ -609,13 +641,13 @@ public void testBundlesWithIsolationPolicies() { isolationPoliciesHelper, antiAffinityGroupPolicyHelper)); setIsolationPolicies(allocationPoliciesSpy, "my-tenant/my-namespaceE", - Set.of("broker5"), Set.of(), Set.of(), 1); + Set.of("broker5:8080"), Set.of(), Set.of(), 1); var ctx = setupContext(); ctx.brokerConfiguration().setLoadBalancerSheddingBundlesWithPoliciesEnabled(true); doReturn(ctx.brokerConfiguration()).when(pulsar).getConfiguration(); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker1")), + expected.add(new UnloadDecision(new Unload("broker4:8080", bundleD1, Optional.of("broker1:8080")), Success, Overloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), setupLoadAvg); @@ -626,7 +658,7 @@ public void testBundlesWithIsolationPolicies() { res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); expected = new HashSet<>(); - expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.empty()), + expected.add(new UnloadDecision(new Unload("broker4:8080", bundleD1, Optional.empty()), Success, Overloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), setupLoadAvg); @@ -634,7 +666,8 @@ public void testBundlesWithIsolationPolicies() { // test setLoadBalancerSheddingBundlesWithPoliciesEnabled=false; - doReturn(true).when(allocationPoliciesSpy).areIsolationPoliciesPresent(any()); + doReturn(CompletableFuture.completedFuture(true)) + .when(allocationPoliciesSpy).areIsolationPoliciesPresentAsync(any()); ctx.brokerConfiguration().setLoadBalancerTransferEnabled(true); ctx.brokerConfiguration().setLoadBalancerSheddingBundlesWithPoliciesEnabled(false); res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); @@ -665,7 +698,7 @@ public BrokerLookupData getLookupData() { webServiceUrl, webServiceUrlTls, pulsarServiceUrl, pulsarServiceUrlTls, advertisedListeners, protocols, true, true, - conf.getLoadManagerClassName(), System.currentTimeMillis(), "3.0.0"); + conf.getLoadManagerClassName(), System.currentTimeMillis(), "3.0.0", Collections.emptyMap()); } private void setIsolationPolicies(SimpleResourceAllocationPolicies policies, @@ -679,12 +712,14 @@ private void setIsolationPolicies(SimpleResourceAllocationPolicies policies, NamespaceBundle namespaceBundle = mock(NamespaceBundle.class); doReturn(true).when(namespaceBundle).hasNonPersistentTopic(); doReturn(namespaceName).when(namespaceBundle).getNamespaceObject(); - doReturn(false).when(policies).areIsolationPoliciesPresent(any()); + doReturn(CompletableFuture.completedFuture(false)) + .when(policies).areIsolationPoliciesPresentAsync(any()); doReturn(false).when(policies).isPrimaryBroker(any(), any()); doReturn(false).when(policies).isSecondaryBroker(any(), any()); doReturn(true).when(policies).isSharedBroker(any()); - doReturn(true).when(policies).areIsolationPoliciesPresent(eq(namespaceName)); + doReturn(CompletableFuture.completedFuture(true)) + .when(policies).areIsolationPoliciesPresentAsync(eq(namespaceName)); primary.forEach(broker -> { doReturn(true).when(policies).isPrimaryBroker(eq(namespaceName), eq(broker)); @@ -723,8 +758,8 @@ public void testBundlesWithAntiAffinityGroup() throws MetadataStoreException { doAnswer(invocationOnMock -> { Map brokers = invocationOnMock.getArgument(0); brokers.clear(); - return brokers; - }).when(antiAffinityGroupPolicyHelper).filter(any(), any()); + return CompletableFuture.completedFuture(brokers); + }).when(antiAffinityGroupPolicyHelper).filterAsync(any(), any()); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); assertTrue(res.isEmpty()); @@ -737,14 +772,14 @@ public void testBundlesWithAntiAffinityGroup() throws MetadataStoreException { String bundle = invocationOnMock.getArgument(1, String.class); if (bundle.equalsIgnoreCase(bundleE1)) { - return brokers; + return CompletableFuture.completedFuture(brokers); } brokers.clear(); - return brokers; - }).when(antiAffinityGroupPolicyHelper).filter(any(), any()); + return CompletableFuture.completedFuture(brokers); + }).when(antiAffinityGroupPolicyHelper).filterAsync(any(), any()); var res2 = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected2 = new HashSet<>(); - expected2.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + expected2.add(new UnloadDecision(new Unload("broker5:8080", bundleE1, Optional.of("broker1:8080")), Success, Overloaded)); assertEquals(res2, expected2); assertEquals(counter.getLoadAvg(), setupLoadAvg); @@ -761,10 +796,10 @@ public String name() { } @Override - public Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) throws BrokerFilterException { - throw new BrokerFilterException("test"); + public CompletableFuture> filterAsync(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { + return FutureUtil.failedFuture(new BrokerFilterException("test")); } }; filters.add(filter); @@ -820,22 +855,22 @@ public void testTargetStd() { var ctx = getContext(); BrokerRegistry brokerRegistry = mock(BrokerRegistry.class); doReturn(CompletableFuture.completedFuture(Map.of( - "broker1", mock(BrokerLookupData.class), - "broker2", mock(BrokerLookupData.class), - "broker3", mock(BrokerLookupData.class) + "broker1:8080", mock(BrokerLookupData.class), + "broker2:8080", mock(BrokerLookupData.class), + "broker3:8080", mock(BrokerLookupData.class) ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync(); doReturn(brokerRegistry).when(ctx).brokerRegistry(); ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 10, "broker1")); - brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 20, "broker2")); - brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx, 30, "broker3")); + brokerLoadDataStore.pushAsync("broker1:8080", getCpuLoad(ctx, 10, "broker1:8080")); + brokerLoadDataStore.pushAsync("broker2:8080", getCpuLoad(ctx, 20, "broker2:8080")); + brokerLoadDataStore.pushAsync("broker3:8080", getCpuLoad(ctx, 30, "broker3:8080")); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 30, 30)); - topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 40, 40)); - topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("my-tenant/my-namespaceC", 50, 50)); + topBundlesLoadDataStore.pushAsync("broker1:8080", getTopBundlesLoad("my-tenant/my-namespaceA", 30, 30)); + topBundlesLoadDataStore.pushAsync("broker2:8080", getTopBundlesLoad("my-tenant/my-namespaceB", 40, 40)); + topBundlesLoadDataStore.pushAsync("broker3:8080", getTopBundlesLoad("my-tenant/my-namespaceC", 50, 50)); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); @@ -851,11 +886,11 @@ public void testSingleTopBundlesLoadData() { TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContext(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1)); - topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 2)); - topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("my-tenant/my-namespaceC", 6)); - topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD", 10)); - topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 70)); + topBundlesLoadDataStore.pushAsync("broker1:8080", getTopBundlesLoad("my-tenant/my-namespaceA", 1)); + topBundlesLoadDataStore.pushAsync("broker2:8080", getTopBundlesLoad("my-tenant/my-namespaceB", 2)); + topBundlesLoadDataStore.pushAsync("broker3:8080", getTopBundlesLoad("my-tenant/my-namespaceC", 6)); + topBundlesLoadDataStore.pushAsync("broker4:8080", getTopBundlesLoad("my-tenant/my-namespaceD", 10)); + topBundlesLoadDataStore.pushAsync("broker5:8080", getTopBundlesLoad("my-tenant/my-namespaceE", 70)); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); assertTrue(res.isEmpty()); @@ -870,14 +905,14 @@ public void testBundleThroughputLargerThanOffloadThreshold() { TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContext(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD", 1000000000, 1000000000)); - topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 1000000000, 1000000000)); + topBundlesLoadDataStore.pushAsync("broker4:8080", getTopBundlesLoad("my-tenant/my-namespaceD", 1000000000, 1000000000)); + topBundlesLoadDataStore.pushAsync("broker5:8080", getTopBundlesLoad("my-tenant/my-namespaceE", 1000000000, 1000000000)); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker3", + expected.add(new UnloadDecision(new Unload("broker3:8080", "my-tenant/my-namespaceC/0x00000000_0x0FFFFFFF", - Optional.of("broker1")), + Optional.of("broker1:8080")), Success, Overloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), setupLoadAvg); @@ -891,12 +926,12 @@ public void testTargetStdAfterTransfer() { TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContext(); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 55, "broker4")); - brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 65, "broker5")); + brokerLoadDataStore.pushAsync("broker4:8080", getCpuLoad(ctx, 55, "broker4:8080")); + brokerLoadDataStore.pushAsync("broker5:8080", getCpuLoad(ctx, 65, "broker5:8080")); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + expected.add(new UnloadDecision(new Unload("broker5:8080", bundleE1, Optional.of("broker1:8080")), Success, Overloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), 0.26400000000000007); @@ -912,43 +947,43 @@ public void testUnloadBundlesGreaterThanTargetThroughput() throws IllegalAccessE var brokerRegistry = mock(BrokerRegistry.class); doReturn(brokerRegistry).when(ctx).brokerRegistry(); doReturn(CompletableFuture.completedFuture(Map.of( - "broker1", mock(BrokerLookupData.class), - "broker2", mock(BrokerLookupData.class) + "broker1:8080", mock(BrokerLookupData.class), + "broker2:8080", mock(BrokerLookupData.class) ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000, 3000000, 4000000, 5000000)); - topBundlesLoadDataStore.pushAsync("broker2", + topBundlesLoadDataStore.pushAsync("broker1:8080", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000, 3000000, 4000000, 5000000)); + topBundlesLoadDataStore.pushAsync("broker2:8080", getTopBundlesLoad("my-tenant/my-namespaceB", 100000000, 180000000, 220000000, 250000000, 250000000)); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 10, "broker1")); - brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 1000, "broker2")); + brokerLoadDataStore.pushAsync("broker1:8080", getCpuLoad(ctx, 10, "broker1:8080")); + brokerLoadDataStore.pushAsync("broker2:8080", getCpuLoad(ctx, 1000, "broker2:8080")); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); expected.add(new UnloadDecision( - new Unload("broker2", "my-tenant/my-namespaceB/0x00000000_0x1FFFFFFF", Optional.of("broker1")), + new Unload("broker2:8080", "my-tenant/my-namespaceB/0x00000000_0x1FFFFFFF", Optional.of("broker1:8080")), Success, Overloaded)); expected.add(new UnloadDecision( - new Unload("broker2", "my-tenant/my-namespaceB/0x1FFFFFFF_0x2FFFFFFF", Optional.of("broker1")), + new Unload("broker2:8080", "my-tenant/my-namespaceB/0x1FFFFFFF_0x2FFFFFFF", Optional.of("broker1:8080")), Success, Overloaded)); expected.add(new UnloadDecision( - new Unload("broker2", "my-tenant/my-namespaceB/0x2FFFFFFF_0x3FFFFFFF", Optional.of("broker1")), + new Unload("broker2:8080", "my-tenant/my-namespaceB/0x2FFFFFFF_0x3FFFFFFF", Optional.of("broker1:8080")), Success, Overloaded)); expected.add(new UnloadDecision( - new Unload("broker1", "my-tenant/my-namespaceA/0x00000000_0x1FFFFFFF", Optional.of("broker2")), + new Unload("broker1:8080", "my-tenant/my-namespaceA/0x00000000_0x1FFFFFFF", Optional.of("broker2:8080")), Success, Overloaded)); expected.add(new UnloadDecision( - new Unload("broker1", "my-tenant/my-namespaceA/0x1FFFFFFF_0x2FFFFFFF", Optional.of("broker2")), + new Unload("broker1:8080", "my-tenant/my-namespaceA/0x1FFFFFFF_0x2FFFFFFF", Optional.of("broker2:8080")), Success, Overloaded)); expected.add(new UnloadDecision( - new Unload("broker1","my-tenant/my-namespaceA/0x2FFFFFFF_0x3FFFFFFF", Optional.of("broker2")), + new Unload("broker1:8080","my-tenant/my-namespaceA/0x2FFFFFFF_0x3FFFFFFF", Optional.of("broker2:8080")), Success, Overloaded)); expected.add(new UnloadDecision( - new Unload("broker1","my-tenant/my-namespaceA/0x3FFFFFFF_0x4FFFFFFF", Optional.of("broker2")), + new Unload("broker1:8080","my-tenant/my-namespaceA/0x3FFFFFFF_0x4FFFFFFF", Optional.of("broker2:8080")), Success, Overloaded)); assertEquals(counter.getLoadAvg(), 5.05); assertEquals(counter.getLoadStd(), 4.95); @@ -968,20 +1003,20 @@ public void testSkipBundlesGreaterThanTargetThroughputAfterSplit() { var brokerRegistry = mock(BrokerRegistry.class); doReturn(brokerRegistry).when(ctx).brokerRegistry(); doReturn(CompletableFuture.completedFuture(Map.of( - "broker1", mock(BrokerLookupData.class), - "broker2", mock(BrokerLookupData.class) + "broker1:8080", mock(BrokerLookupData.class), + "broker2:8080", mock(BrokerLookupData.class) ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker1", + topBundlesLoadDataStore.pushAsync("broker1:8080", getTopBundlesLoad("my-tenant/my-namespaceA", 1, 500000000)); - topBundlesLoadDataStore.pushAsync("broker2", + topBundlesLoadDataStore.pushAsync("broker2:8080", getTopBundlesLoad("my-tenant/my-namespaceB", 500000000, 500000000)); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 50, "broker1")); - brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 100, "broker2")); + brokerLoadDataStore.pushAsync("broker1:8080", getCpuLoad(ctx, 50, "broker1:8080")); + brokerLoadDataStore.pushAsync("broker2:8080", getCpuLoad(ctx, 100, "broker2:8080")); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); @@ -999,24 +1034,24 @@ public void testUnloadBundlesLessThanTargetThroughputAfterSplit() throws Illegal var brokerRegistry = mock(BrokerRegistry.class); doReturn(brokerRegistry).when(ctx).brokerRegistry(); doReturn(CompletableFuture.completedFuture(Map.of( - "broker1", mock(BrokerLookupData.class), - "broker2", mock(BrokerLookupData.class) + "broker1:8080", mock(BrokerLookupData.class), + "broker2:8080", mock(BrokerLookupData.class) ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000, 3000000, 4000000, 5000000)); - topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 490000000, 510000000)); + topBundlesLoadDataStore.pushAsync("broker1:8080", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000, 3000000, 4000000, 5000000)); + topBundlesLoadDataStore.pushAsync("broker2:8080", getTopBundlesLoad("my-tenant/my-namespaceB", 490000000, 510000000)); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 10, "broker1")); - brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 1000, "broker2")); + brokerLoadDataStore.pushAsync("broker1:8080", getCpuLoad(ctx, 10, "broker1:8080")); + brokerLoadDataStore.pushAsync("broker2:8080", getCpuLoad(ctx, 1000, "broker2:8080")); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); expected.add(new UnloadDecision( - new Unload("broker2", "my-tenant/my-namespaceB/0x00000000_0x0FFFFFFF", Optional.of("broker1")), + new Unload("broker2:8080", "my-tenant/my-namespaceB/0x00000000_0x0FFFFFFF", Optional.of("broker1:8080")), Success, Overloaded)); assertEquals(counter.getLoadAvg(), 5.05); assertEquals(counter.getLoadStd(), 4.95); @@ -1037,30 +1072,30 @@ public void testUnloadBundlesGreaterThanTargetThroughputAfterSplit() throws Ille var brokerRegistry = mock(BrokerRegistry.class); doReturn(brokerRegistry).when(ctx).brokerRegistry(); doReturn(CompletableFuture.completedFuture(Map.of( - "broker1", mock(BrokerLookupData.class), - "broker2", mock(BrokerLookupData.class) + "broker1:8080", mock(BrokerLookupData.class), + "broker2:8080", mock(BrokerLookupData.class) ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 2400000, 2400000)); - topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 5000000, 5000000)); + topBundlesLoadDataStore.pushAsync("broker1:8080", getTopBundlesLoad("my-tenant/my-namespaceA", 2400000, 2400000)); + topBundlesLoadDataStore.pushAsync("broker2:8080", getTopBundlesLoad("my-tenant/my-namespaceB", 5000000, 5000000)); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 48, "broker1")); - brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 100, "broker2")); + brokerLoadDataStore.pushAsync("broker1:8080", getCpuLoad(ctx, 48, "broker1:8080")); + brokerLoadDataStore.pushAsync("broker2:8080", getCpuLoad(ctx, 100, "broker2:8080")); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); expected.add(new UnloadDecision( - new Unload("broker1", - res.stream().filter(x -> x.getUnload().sourceBroker().equals("broker1")).findFirst().get() - .getUnload().serviceUnit(), Optional.of("broker2")), + new Unload("broker1:8080", + res.stream().filter(x -> x.getUnload().sourceBroker().equals("broker1:8080")).findFirst().get() + .getUnload().serviceUnit(), Optional.of("broker2:8080")), Success, Overloaded)); expected.add(new UnloadDecision( - new Unload("broker2", - res.stream().filter(x -> x.getUnload().sourceBroker().equals("broker2")).findFirst().get() - .getUnload().serviceUnit(), Optional.of("broker1")), + new Unload("broker2:8080", + res.stream().filter(x -> x.getUnload().sourceBroker().equals("broker2:8080")).findFirst().get() + .getUnload().serviceUnit(), Optional.of("broker1:8080")), Success, Overloaded)); assertEquals(counter.getLoadAvg(), 0.74); assertEquals(counter.getLoadStd(), 0.26); @@ -1070,26 +1105,27 @@ public void testUnloadBundlesGreaterThanTargetThroughputAfterSplit() throws Ille assertEquals(stats.std(), 2.5809568279517847E-8); } - @Test - public void testMinBrokerWithZeroTraffic() throws IllegalAccessException { + public void testMinBrokerWithLowTraffic() throws IllegalAccessException { UnloadCounter counter = new UnloadCounter(); TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContext(); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - var load = getCpuLoad(ctx, 4, "broker2"); - FieldUtils.writeDeclaredField(load,"msgThroughputEMA", 0, true); - brokerLoadDataStore.pushAsync("broker2", load); - brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 55, "broker4")); - brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 65, "broker5")); + var load = getCpuLoad(ctx, 4, "broker2:8080"); + FieldUtils.writeDeclaredField(load, "msgThroughputEMA", 10, true); + + + brokerLoadDataStore.pushAsync("broker2:8080", load); + brokerLoadDataStore.pushAsync("broker4:8080", getCpuLoad(ctx, 55, "broker4:8080")); + brokerLoadDataStore.pushAsync("broker5:8080", getCpuLoad(ctx, 65, "broker5:8080")); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + expected.add(new UnloadDecision(new Unload("broker5:8080", bundleE1, Optional.of("broker1:8080")), Success, Overloaded)); - expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + expected.add(new UnloadDecision(new Unload("broker4:8080", bundleD1, Optional.of("broker2:8080")), Success, Underloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), 0.26400000000000007); @@ -1103,17 +1139,17 @@ public void testMinBrokerWithLowerLoadThanAvg() throws IllegalAccessException { var ctx = setupContext(); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - var load = getCpuLoad(ctx, 3 , "broker2"); - brokerLoadDataStore.pushAsync("broker2", load); - brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 55, "broker4")); - brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 65, "broker5")); + var load = getCpuLoad(ctx, 3 , "broker2:8080"); + brokerLoadDataStore.pushAsync("broker2:8080", load); + brokerLoadDataStore.pushAsync("broker4:8080", getCpuLoad(ctx, 55, "broker4:8080")); + brokerLoadDataStore.pushAsync("broker5:8080", getCpuLoad(ctx, 65, "broker5:8080")); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + expected.add(new UnloadDecision(new Unload("broker5:8080", bundleE1, Optional.of("broker1:8080")), Success, Overloaded)); - expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + expected.add(new UnloadDecision(new Unload("broker4:8080", bundleD1, Optional.of("broker2:8080")), Success, Underloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), 0.262); @@ -1129,9 +1165,9 @@ public void testMaxNumberOfTransfersPerShedderCycle() { .setLoadBalancerMaxNumberOfBrokerSheddingPerCycle(10); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + expected.add(new UnloadDecision(new Unload("broker5:8080", bundleE1, Optional.of("broker1:8080")), Success, Overloaded)); - expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + expected.add(new UnloadDecision(new Unload("broker4:8080", bundleD1, Optional.of("broker2:8080")), Success, Overloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), setupLoadAvg); @@ -1155,9 +1191,9 @@ public void testLoadBalancerSheddingConditionHitCountThreshold() { } var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + expected.add(new UnloadDecision(new Unload("broker5:8080", bundleE1, Optional.of("broker1:8080")), Success, Overloaded)); - expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + expected.add(new UnloadDecision(new Unload("broker4:8080", bundleD1, Optional.of("broker2:8080")), Success, Overloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), setupLoadAvg); @@ -1170,13 +1206,13 @@ public void testRemainingTopBundles() { TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContext(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 2000000, 3000000)); + topBundlesLoadDataStore.pushAsync("broker5:8080", getTopBundlesLoad("my-tenant/my-namespaceE", 2000000, 3000000)); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + expected.add(new UnloadDecision(new Unload("broker5:8080", bundleE1, Optional.of("broker1:8080")), Success, Overloaded)); - expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + expected.add(new UnloadDecision(new Unload("broker4:8080", bundleD1, Optional.of("broker2:8080")), Success, Overloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), setupLoadAvg); @@ -1190,14 +1226,14 @@ public void testLoadMoreThan100() throws IllegalAccessException { var ctx = setupContext(); var brokerLoadDataStore = ctx.brokerLoadDataStore(); - brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 200, "broker4")); - brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 1000, "broker5")); + brokerLoadDataStore.pushAsync("broker4:8080", getCpuLoad(ctx, 200, "broker4:8080")); + brokerLoadDataStore.pushAsync("broker5:8080", getCpuLoad(ctx, 1000, "broker5:8080")); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); - expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), + expected.add(new UnloadDecision(new Unload("broker5:8080", bundleE1, Optional.of("broker1:8080")), Success, Overloaded)); - expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")), + expected.add(new UnloadDecision(new Unload("broker4:8080", bundleD1, Optional.of("broker2:8080")), Success, Overloaded)); assertEquals(res, expected); assertEquals(counter.getLoadAvg(), 2.4240000000000004); @@ -1231,13 +1267,16 @@ public void testOverloadOutlier() { TransferShedder transferShedder = new TransferShedder(counter); var ctx = setupContextLoadSkewedOverload(100); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); - var expected = new HashSet(); - expected.add(new UnloadDecision( - new Unload("broker99", "my-tenant/my-namespace99/0x00000000_0x0FFFFFFF", - Optional.of("broker52")), Success, Overloaded)); - assertEquals(res, expected); - assertEquals(counter.getLoadAvg(), 0.019900000000000008); - assertEquals(counter.getLoadStd(), 0.09850375627355534); + Assertions.assertThat(res).isIn( + Set.of(new UnloadDecision( + new Unload("broker99:8080", "my-tenant/my-namespace99/0x00000000_0x0FFFFFFF", + Optional.of("broker52:8080")), Success, Underloaded)), + Set.of(new UnloadDecision( + new Unload("broker99:8080", "my-tenant/my-namespace99/0x00000000_0x0FFFFFFF", + Optional.of("broker83:8080")), Success, Underloaded)) + ); + assertEquals(counter.getLoadAvg(), 0.019900000000000008, 0.00001); + assertEquals(counter.getLoadStd(), 0.09850375627355534, 0.00001); } @Test @@ -1248,11 +1287,11 @@ public void testUnderloadOutlier() { var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new HashSet(); expected.add(new UnloadDecision( - new Unload("broker98", "my-tenant/my-namespace98/0x00000000_0x0FFFFFFF", - Optional.of("broker99")), Success, Underloaded)); + new Unload("broker98:8080", "my-tenant/my-namespace98/0x00000000_0x0FFFFFFF", + Optional.of("broker99:8080")), Success, Underloaded)); assertEquals(res, expected); - assertEquals(counter.getLoadAvg(), 0.9704000000000005); - assertEquals(counter.getLoadStd(), 0.09652895938523735); + assertEquals(counter.getLoadAvg(), 0.9704000000000005, 0.00001); + assertEquals(counter.getLoadStd(), 0.09652895938523735, 0.00001); } @Test @@ -1268,13 +1307,13 @@ public void testRandomLoadStats() { double[] loads = new double[numBrokers]; final Map availableBrokers = new HashMap<>(); for (int i = 0; i < loads.length; i++) { - availableBrokers.put("broker" + i, mock(BrokerLookupData.class)); + availableBrokers.put("broker" + i + ":8080", mock(BrokerLookupData.class)); } stats.update(loadStore, availableBrokers, Map.of(), conf); var brokerLoadDataStore = ctx.brokerLoadDataStore(); for (int i = 0; i < loads.length; i++) { - loads[i] = loadStore.get("broker" + i).get().getWeightedMaxEMA(); + loads[i] = loadStore.get("broker" + i + ":8080").get().getWeightedMaxEMA(); } int i = 0; int j = loads.length - 1; @@ -1309,8 +1348,8 @@ public void testHighVarianceLoadStats() { var conf = ctx.brokerConfiguration(); final Map availableBrokers = new HashMap<>(); for (int i = 0; i < loads.length; i++) { - availableBrokers.put("broker" + i, mock(BrokerLookupData.class)); - loadStore.pushAsync("broker" + i, getCpuLoad(ctx, loads[i], "broker" + i)); + availableBrokers.put("broker" + i + ":8080", mock(BrokerLookupData.class)); + loadStore.pushAsync("broker" + i + ":8080", getCpuLoad(ctx, loads[i], "broker" + i + ":8080")); } stats.update(loadStore, availableBrokers, Map.of(), conf); @@ -1328,8 +1367,8 @@ public void testLowVarianceLoadStats() { var conf = ctx.brokerConfiguration(); final Map availableBrokers = new HashMap<>(); for (int i = 0; i < loads.length; i++) { - availableBrokers.put("broker" + i, mock(BrokerLookupData.class)); - loadStore.pushAsync("broker" + i, getCpuLoad(ctx, loads[i], "broker" + i)); + availableBrokers.put("broker" + i + ":8080", mock(BrokerLookupData.class)); + loadStore.pushAsync("broker" + i + ":8080", getCpuLoad(ctx, loads[i], "broker" + i + ":8080")); } stats.update(loadStore, availableBrokers, Map.of(), conf); assertEquals(stats.avg(), 3.9449999999999994); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java index 38d4e9904e649..1fd89ba882ba9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java @@ -28,6 +28,7 @@ import static org.mockito.Mockito.verify; import com.google.common.collect.Lists; +import lombok.Cleanup; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry; @@ -127,6 +128,8 @@ public void testExecuteMoreThenOnceWhenFirstNotDone() throws InterruptedExceptio PulsarService pulsar = mock(PulsarService.class); NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class); doReturn(CompletableFuture.completedFuture(true)).when(channel).isChannelOwnerAsync(); + @Cleanup("shutdownNow") + ExecutorService executor = Executors.newFixedThreadPool(1); doAnswer(__ -> CompletableFuture.supplyAsync(() -> { try { // Delay 5 seconds to finish. @@ -135,9 +138,10 @@ public void testExecuteMoreThenOnceWhenFirstNotDone() throws InterruptedExceptio throw new RuntimeException(e); } return Lists.newArrayList("broker-1", "broker-2"); - }, Executors.newFixedThreadPool(1))).when(registry).getAvailableBrokersAsync(); + }, executor)).when(registry).getAvailableBrokersAsync(); UnloadScheduler scheduler = new UnloadScheduler(pulsar, loadManagerExecutor, unloadManager, context, channel, unloadStrategy, counter, reference); + @Cleanup("shutdownNow") ExecutorService executorService = Executors.newFixedThreadPool(5); CountDownLatch latch = new CountDownLatch(5); for (int i = 0; i < 5; i++) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java index 184c337a47c80..820307637be67 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java @@ -18,9 +18,12 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.store; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertThrows; import static org.testng.AssertJUnit.assertTrue; import com.google.common.collect.Sets; @@ -28,11 +31,13 @@ import lombok.Cleanup; import lombok.Data; import lombok.NoArgsConstructor; +import org.apache.commons.lang.reflect.FieldUtils; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.awaitility.Awaitility; +import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -74,8 +79,7 @@ public void testPushGetAndRemove() throws Exception { @Cleanup LoadDataStore loadDataStore = - LoadDataStoreFactory.create(pulsar.getClient(), topic, MyClass.class); - loadDataStore.startTableView(); + LoadDataStoreFactory.create(pulsar, topic, MyClass.class); MyClass myClass1 = new MyClass("1", 1); loadDataStore.pushAsync("key1", myClass1).get(); @@ -107,8 +111,7 @@ public void testForEach() throws Exception { @Cleanup LoadDataStore loadDataStore = - LoadDataStoreFactory.create(pulsar.getClient(), topic, Integer.class); - loadDataStore.startTableView(); + LoadDataStoreFactory.create(pulsar, topic, Integer.class); Map map = new HashMap<>(); for (int i = 0; i < 10; i++) { @@ -131,24 +134,79 @@ public void testForEach() throws Exception { public void testTableViewRestart() throws Exception { String topic = TopicDomain.persistent + "://" + NamespaceName.SYSTEM_NAMESPACE + "/" + UUID.randomUUID(); LoadDataStore loadDataStore = - LoadDataStoreFactory.create(pulsar.getClient(), topic, Integer.class); - - loadDataStore.startTableView(); + LoadDataStoreFactory.create(pulsar, topic, Integer.class); loadDataStore.pushAsync("1", 1).get(); Awaitility.await().untilAsserted(() -> assertEquals(loadDataStore.size(), 1)); assertEquals(loadDataStore.get("1").get(), 1); loadDataStore.closeTableView(); loadDataStore.pushAsync("1", 2).get(); - Exception ex = null; - try { - loadDataStore.get("1"); - } catch (IllegalStateException e) { - ex = e; - } - assertNotNull(ex); - loadDataStore.startTableView(); Awaitility.await().untilAsserted(() -> assertEquals(loadDataStore.get("1").get(), 2)); + + loadDataStore.pushAsync("1", 3).get(); + FieldUtils.writeField(loadDataStore, "tableViewLastUpdateTimestamp", 0 , true); + Awaitility.await().untilAsserted(() -> assertEquals(loadDataStore.get("1").get(), 3)); + } + + @Test + public void testProducerRestart() throws Exception { + String topic = TopicDomain.persistent + "://" + NamespaceName.SYSTEM_NAMESPACE + "/" + UUID.randomUUID(); + var loadDataStore = + (TableViewLoadDataStoreImpl) spy(LoadDataStoreFactory.create(pulsar, topic, Integer.class)); + + // happy case + loadDataStore.pushAsync("1", 1).get(); + Awaitility.await().untilAsserted(() -> assertEquals(loadDataStore.size(), 1)); + assertEquals(loadDataStore.get("1").get(), 1); + verify(loadDataStore, times(1)).startProducer(); + + // loadDataStore will restart producer if null. + FieldUtils.writeField(loadDataStore, "producer", null, true); + loadDataStore.pushAsync("1", 2).get(); + Awaitility.await().untilAsserted(() -> assertEquals(loadDataStore.get("1").get(), 2)); + verify(loadDataStore, times(2)).startProducer(); + + // loadDataStore will restart producer if too slow. + FieldUtils.writeField(loadDataStore, "producerLastPublishTimestamp", 0 , true); + loadDataStore.pushAsync("1", 3).get(); + Awaitility.await().untilAsserted(() -> assertEquals(loadDataStore.get("1").get(), 3)); + verify(loadDataStore, times(3)).startProducer(); + } + + @Test + public void testProducerStop() throws Exception { + String topic = TopicDomain.persistent + "://" + NamespaceName.SYSTEM_NAMESPACE + "/" + UUID.randomUUID(); + LoadDataStore loadDataStore = + LoadDataStoreFactory.create(pulsar, topic, Integer.class); + loadDataStore.startProducer(); + loadDataStore.pushAsync("1", 1).get(); + loadDataStore.removeAsync("1").get(); + + loadDataStore.close(); + + loadDataStore.pushAsync("2", 2).get(); + loadDataStore.removeAsync("2").get(); + } + + @Test + public void testShutdown() throws Exception { + String topic = TopicDomain.persistent + "://" + NamespaceName.SYSTEM_NAMESPACE + "/" + UUID.randomUUID(); + LoadDataStore loadDataStore = + LoadDataStoreFactory.create(pulsar, topic, Integer.class); + loadDataStore.start(); + loadDataStore.shutdown(); + + Assert.assertTrue(loadDataStore.pushAsync("2", 2).isCompletedExceptionally()); + Assert.assertTrue(loadDataStore.removeAsync("2").isCompletedExceptionally()); + assertTrue(loadDataStore.get("2").isEmpty()); + assertThrows(IllegalStateException.class, loadDataStore::size); + assertThrows(IllegalStateException.class, loadDataStore::entrySet); + assertThrows(IllegalStateException.class, () -> loadDataStore.forEach((k, v) -> {})); + assertThrows(IllegalStateException.class, loadDataStore::init); + assertThrows(IllegalStateException.class, loadDataStore::start); + assertThrows(IllegalStateException.class, loadDataStore::startProducer); + assertThrows(IllegalStateException.class, loadDataStore::startTableView); + assertThrows(IllegalStateException.class, loadDataStore::closeTableView); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/CustomBrokerSelectionStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/CustomBrokerSelectionStrategyTest.java new file mode 100644 index 0000000000000..3ac6df2595109 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/CustomBrokerSelectionStrategyTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.strategy; + +import java.util.Comparator; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import lombok.Cleanup; +import org.apache.pulsar.broker.MultiBrokerBaseTest; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder; +import org.apache.pulsar.client.impl.PartitionedProducerImpl; +import org.apache.pulsar.client.impl.ProducerImpl; +import org.testng.Assert; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class CustomBrokerSelectionStrategyTest extends MultiBrokerBaseTest { + + @Override + protected void startBroker() throws Exception { + addCustomConfigs(conf); + super.startBroker(); + } + + @Override + protected ServiceConfiguration createConfForAdditionalBroker(int additionalBrokerIndex) { + return addCustomConfigs(getDefaultConf()); + } + + private static ServiceConfiguration addCustomConfigs(ServiceConfiguration conf) { + conf.setLoadManagerClassName(CustomExtensibleLoadManager.class.getName()); + conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); + conf.setLoadBalancerAutoBundleSplitEnabled(false); + conf.setDefaultNumberOfNamespaceBundles(8); + // Don't consider broker's load so the broker will be selected randomly with the default strategy + conf.setLoadBalancerAverageResourceUsageDifferenceThresholdPercentage(100); + return conf; + } + + @Test + public void testSingleBrokerSelected() throws Exception { + final var topic = "test-single-broker-selected"; + getAllAdmins().get(0).topics().createPartitionedTopic(topic, 16); + @Cleanup final var producer = (PartitionedProducerImpl) getAllClients().get(0).newProducer() + .topic(topic).create(); + Assert.assertNotNull(producer); + final var connections = producer.getProducers().stream().map(ProducerImpl::getClientCnx) + .collect(Collectors.toSet()); + Assert.assertEquals(connections.size(), 1); + final var port = Integer.parseInt(connections.stream().findFirst().orElseThrow().ctx().channel() + .remoteAddress().toString().replaceAll(".*:", "")); + final var expectedPort = Stream.concat(Stream.of(pulsar), additionalBrokers.stream()) + .min(Comparator.comparingInt(o -> o.getListenPortHTTP().orElseThrow())) + .map(PulsarService::getBrokerListenPort) + .orElseThrow().orElseThrow(); + Assert.assertEquals(port, expectedPort); + } + + public static class CustomExtensibleLoadManager extends ExtensibleLoadManagerImpl { + + @Override + public BrokerSelectionStrategy createBrokerSelectionStrategy() { + // The smallest HTTP port will always be selected because the host parts are all "localhost" + return (brokers, __, ___) -> brokers.stream().sorted().findFirst(); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java index 0eea1d87513bf..5f3a08d493bc3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java @@ -196,8 +196,8 @@ public static LoadManagerContext getContext() { conf.setLoadBalancerCPUResourceWeight(1.0); conf.setLoadBalancerMemoryResourceWeight(0.1); conf.setLoadBalancerDirectMemoryResourceWeight(0.1); - conf.setLoadBalancerBandwithInResourceWeight(1.0); - conf.setLoadBalancerBandwithOutResourceWeight(1.0); + conf.setLoadBalancerBandwidthInResourceWeight(1.0); + conf.setLoadBalancerBandwidthOutResourceWeight(1.0); conf.setLoadBalancerHistoryResourcePercentage(0.5); conf.setLoadBalancerAverageResourceUsageDifferenceThresholdPercentage(5); var brokerLoadDataStore = new LoadDataStore() { @@ -252,10 +252,25 @@ public void closeTableView() throws IOException { } + @Override + public void start() throws LoadDataStoreException { + + } + + @Override + public void init() throws IOException { + + } + @Override public void startTableView() throws LoadDataStoreException { } + + @Override + public void startProducer() throws LoadDataStoreException { + + } }; doReturn(conf).when(ctx).brokerConfiguration(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/AvgShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/AvgShedderTest.java new file mode 100644 index 0000000000000..215e3d766a927 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/AvgShedderTest.java @@ -0,0 +1,283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.impl; + +import com.google.common.collect.Multimap; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.LoadData; +import org.apache.pulsar.policies.data.loadbalancer.BrokerData; +import org.apache.pulsar.policies.data.loadbalancer.BundleData; +import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; +import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; +import org.apache.pulsar.policies.data.loadbalancer.TimeAverageBrokerData; +import org.apache.pulsar.policies.data.loadbalancer.TimeAverageMessageData; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.fail; + +@Test(groups = "broker") +public class AvgShedderTest { + private AvgShedder avgShedder; + private final ServiceConfiguration conf; + + public AvgShedderTest() { + conf = new ServiceConfiguration(); + } + + @BeforeMethod + public void setup() { + avgShedder = new AvgShedder(); + } + + private BrokerData initBrokerData() { + LocalBrokerData localBrokerData = new LocalBrokerData(); + localBrokerData.setCpu(new ResourceUsage()); + localBrokerData.setMemory(new ResourceUsage()); + localBrokerData.setBandwidthIn(new ResourceUsage()); + localBrokerData.setBandwidthOut(new ResourceUsage()); + BrokerData brokerData = new BrokerData(localBrokerData); + TimeAverageBrokerData timeAverageBrokerData = new TimeAverageBrokerData(); + brokerData.setTimeAverageData(timeAverageBrokerData); + return brokerData; + } + + @Test + public void testHitHighThreshold() { + LoadData loadData = new LoadData(); + BrokerData brokerData1 = initBrokerData(); + BrokerData brokerData2 = initBrokerData(); + BrokerData brokerData3 = initBrokerData(); + loadData.getBrokerData().put("broker1", brokerData1); + loadData.getBrokerData().put("broker2", brokerData2); + loadData.getBrokerData().put("broker3", brokerData3); + // AvgShedder will distribute the load evenly between the highest and lowest brokers + conf.setMaxUnloadPercentage(0.5); + + // Set the high threshold to 40% and hit count high threshold to 2 + int hitCountForHighThreshold = 2; + conf.setLoadBalancerAvgShedderHighThreshold(40); + conf.setLoadBalancerAvgShedderHitCountHighThreshold(hitCountForHighThreshold); + brokerData1.getLocalData().setCpu(new ResourceUsage(80, 100)); + brokerData2.getLocalData().setCpu(new ResourceUsage(30, 100)); + brokerData1.getLocalData().setMsgRateIn(10000); + brokerData1.getLocalData().setMsgRateOut(10000); + brokerData2.getLocalData().setMsgRateIn(1000); + brokerData2.getLocalData().setMsgRateOut(1000); + + // broker3 is in the middle + brokerData3.getLocalData().setCpu(new ResourceUsage(50, 100)); + brokerData3.getLocalData().setMsgRateIn(5000); + brokerData3.getLocalData().setMsgRateOut(5000); + + // expect to shed bundles with message rate(in+out) ((10000+10000)-(1000+1000))/2 = 9000 + // each bundle with 450 msg rate in and 450 msg rate out + // so 9000/(450+450)=10 bundles will be shed + for (int i = 0; i < 11; i++) { + brokerData1.getLocalData().getBundles().add("bundle-" + i); + BundleData bundle = new BundleData(); + TimeAverageMessageData timeAverageMessageData = new TimeAverageMessageData(); + timeAverageMessageData.setMsgRateIn(450); + timeAverageMessageData.setMsgRateOut(450); + // as AvgShedder map BundleData to broker, the hashCode of different BundleData should be different + // so we need to set some different fields to make the hashCode different + timeAverageMessageData.setNumSamples(i); + bundle.setShortTermData(timeAverageMessageData); + loadData.getBundleData().put("bundle-" + i, bundle); + } + + // do shedding for the first time, expect to shed nothing because hit count is not enough + Multimap bundlesToUnload = avgShedder.findBundlesForUnloading(loadData, conf); + assertEquals(bundlesToUnload.size(), 0); + + // do shedding for the second time, expect to shed 10 bundles + bundlesToUnload = avgShedder.findBundlesForUnloading(loadData, conf); + assertEquals(bundlesToUnload.size(), 10); + + // assert that all the bundles are shed from broker1 + for (String broker : bundlesToUnload.keys()) { + assertEquals(broker, "broker1"); + } + // assert that all the bundles are shed to broker2 + for (String bundle : bundlesToUnload.values()) { + BundleData bundleData = loadData.getBundleData().get(bundle); + assertEquals(avgShedder.selectBroker(loadData.getBrokerData().keySet(), bundleData, loadData, conf).get(), "broker2"); + } + } + + @Test + public void testHitLowThreshold() { + LoadData loadData = new LoadData(); + BrokerData brokerData1 = initBrokerData(); + BrokerData brokerData2 = initBrokerData(); + BrokerData brokerData3 = initBrokerData(); + loadData.getBrokerData().put("broker1", brokerData1); + loadData.getBrokerData().put("broker2", brokerData2); + loadData.getBrokerData().put("broker3", brokerData3); + // AvgShedder will distribute the load evenly between the highest and lowest brokers + conf.setMaxUnloadPercentage(0.5); + + // Set the low threshold to 20% and hit count low threshold to 6 + int hitCountForLowThreshold = 6; + conf.setLoadBalancerAvgShedderLowThreshold(20); + conf.setLoadBalancerAvgShedderHitCountLowThreshold(hitCountForLowThreshold); + brokerData1.getLocalData().setCpu(new ResourceUsage(60, 100)); + brokerData2.getLocalData().setCpu(new ResourceUsage(40, 100)); + brokerData1.getLocalData().setMsgRateIn(10000); + brokerData1.getLocalData().setMsgRateOut(10000); + brokerData2.getLocalData().setMsgRateIn(1000); + brokerData2.getLocalData().setMsgRateOut(1000); + + // broker3 is in the middle + brokerData3.getLocalData().setCpu(new ResourceUsage(50, 100)); + brokerData3.getLocalData().setMsgRateIn(5000); + brokerData3.getLocalData().setMsgRateOut(5000); + + // expect to shed bundles with message rate(in+out) ((10000+10000)-(1000+1000))/2 = 9000 + // each bundle with 450 msg rate in and 450 msg rate out + // so 9000/(450+450)=10 bundles will be shed + for (int i = 0; i < 11; i++) { + brokerData1.getLocalData().getBundles().add("bundle-" + i); + BundleData bundle = new BundleData(); + TimeAverageMessageData timeAverageMessageData = new TimeAverageMessageData(); + timeAverageMessageData.setMsgRateIn(450); + timeAverageMessageData.setMsgRateOut(450); + // as AvgShedder map BundleData to broker, the hashCode of different BundleData should be different + // so we need to set some different fields to make the hashCode different + timeAverageMessageData.setNumSamples(i); + bundle.setShortTermData(timeAverageMessageData); + loadData.getBundleData().put("bundle-" + i, bundle); + } + + // do shedding for (lowCountForHighThreshold - 1) times, expect to shed nothing because hit count is not enough + Multimap bundlesToUnload; + for (int i = 0; i < hitCountForLowThreshold - 1; i++) { + bundlesToUnload = avgShedder.findBundlesForUnloading(loadData, conf); + assertEquals(bundlesToUnload.size(), 0); + } + + // do shedding for the last time, expect to shed 10 bundles + bundlesToUnload = avgShedder.findBundlesForUnloading(loadData, conf); + assertEquals(bundlesToUnload.size(), 10); + + // assert that all the bundles are shed from broker1 + for (String broker : bundlesToUnload.keys()) { + assertEquals(broker, "broker1"); + } + // assert that all the bundles are shed to broker2 + for (String bundle : bundlesToUnload.values()) { + BundleData bundleData = loadData.getBundleData().get(bundle); + assertEquals(avgShedder.selectBroker(loadData.getBrokerData().keySet(), bundleData, loadData, conf).get(), "broker2"); + } + } + + @Test + public void testSheddingMultiplePairs() { + LoadData loadData = new LoadData(); + BrokerData brokerData1 = initBrokerData(); + BrokerData brokerData2 = initBrokerData(); + BrokerData brokerData3 = initBrokerData(); + BrokerData brokerData4 = initBrokerData(); + loadData.getBrokerData().put("broker1", brokerData1); + loadData.getBrokerData().put("broker2", brokerData2); + loadData.getBrokerData().put("broker3", brokerData3); + loadData.getBrokerData().put("broker4", brokerData4); + // AvgShedder will distribute the load evenly between the highest and lowest brokers + conf.setMaxUnloadPercentage(0.5); + + // Set the high threshold to 40% and hit count high threshold to 2 + int hitCountForHighThreshold = 2; + conf.setLoadBalancerAvgShedderHighThreshold(40); + conf.setLoadBalancerAvgShedderHitCountHighThreshold(hitCountForHighThreshold); + + // pair broker1 and broker2 + brokerData1.getLocalData().setCpu(new ResourceUsage(80, 100)); + brokerData2.getLocalData().setCpu(new ResourceUsage(30, 100)); + brokerData1.getLocalData().setMsgRateIn(10000); + brokerData1.getLocalData().setMsgRateOut(10000); + brokerData2.getLocalData().setMsgRateIn(1000); + brokerData2.getLocalData().setMsgRateOut(1000); + + // pair broker3 and broker4 + brokerData3.getLocalData().setCpu(new ResourceUsage(75, 100)); + brokerData3.getLocalData().setMsgRateIn(10000); + brokerData3.getLocalData().setMsgRateOut(10000); + brokerData4.getLocalData().setCpu(new ResourceUsage(35, 100)); + brokerData4.getLocalData().setMsgRateIn(1000); + brokerData4.getLocalData().setMsgRateOut(1000); + + // expect to shed bundles with message rate(in+out) ((10000+10000)-(1000+1000))/2 = 9000 + // each bundle with 450 msg rate in and 450 msg rate out + // so 9000/(450+450)=10 bundles will be shed + for (int i = 0; i < 11; i++) { + brokerData1.getLocalData().getBundles().add("bundle1-" + i); + brokerData3.getLocalData().getBundles().add("bundle3-" + i); + + BundleData bundle = new BundleData(); + TimeAverageMessageData timeAverageMessageData = new TimeAverageMessageData(); + timeAverageMessageData.setMsgRateIn(450); + timeAverageMessageData.setMsgRateOut(450); + // as AvgShedder map BundleData to broker, the hashCode of different BundleData should be different + // so we need to set some different fields to make the hashCode different + timeAverageMessageData.setNumSamples(i); + bundle.setShortTermData(timeAverageMessageData); + loadData.getBundleData().put("bundle1-" + i, bundle); + + bundle = new BundleData(); + timeAverageMessageData = new TimeAverageMessageData(); + timeAverageMessageData.setMsgRateIn(450); + timeAverageMessageData.setMsgRateOut(450); + timeAverageMessageData.setNumSamples(i+11); + bundle.setShortTermData(timeAverageMessageData); + loadData.getBundleData().put("bundle3-" + i, bundle); + } + + // do shedding for the first time, expect to shed nothing because hit count is not enough + Multimap bundlesToUnload = avgShedder.findBundlesForUnloading(loadData, conf); + assertEquals(bundlesToUnload.size(), 0); + + // do shedding for the second time, expect to shed 10*2=20 bundles + bundlesToUnload = avgShedder.findBundlesForUnloading(loadData, conf); + assertEquals(bundlesToUnload.size(), 20); + + // assert that half of the bundles are shed from broker1, and the other half are shed from broker3 + for (String broker : bundlesToUnload.keys()) { + if (broker.equals("broker1")) { + assertEquals(bundlesToUnload.get(broker).size(), 10); + } else if (broker.equals("broker3")) { + assertEquals(bundlesToUnload.get(broker).size(), 10); + } else { + fail(); + } + } + + // assert that all the bundles from broker1 are shed to broker2, and all the bundles from broker3 are shed to broker4 + for (String bundle : bundlesToUnload.values()) { + BundleData bundleData = loadData.getBundleData().get(bundle); + if (bundle.startsWith("bundle1-")) { + assertEquals(avgShedder.selectBroker(loadData.getBrokerData().keySet(), bundleData, loadData, conf).get(), "broker2"); + } else if (bundle.startsWith("bundle3-")) { + assertEquals(avgShedder.selectBroker(loadData.getBrokerData().keySet(), bundleData, loadData, conf).get(), "broker4"); + } else { + fail(); + } + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTaskTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTaskTest.java index 3173987a3c8a8..74e692e3d7de0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTaskTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTaskTest.java @@ -28,6 +28,7 @@ import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.policies.data.loadbalancer.TimeAverageMessageData; +import org.apache.pulsar.utils.ResourceUtils; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -45,6 +46,13 @@ @Test(groups = "broker") public class BundleSplitterTaskTest { + public final static String CA_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem"); + public final static String BROKER_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem"); + public final static String BROKER_KEY_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem"); + private LocalBookkeeperEnsemble bkEnsemble; private PulsarService pulsar; @@ -67,6 +75,9 @@ void setup() throws Exception { config.setBrokerServicePort(Optional.of(0)); config.setBrokerServicePortTls(Optional.of(0)); config.setWebServicePortTls(Optional.of(0)); + config.setTlsCertificateFilePath(CA_CERT_FILE_PATH); + config.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + config.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); pulsar = new PulsarService(config); pulsar.start(); } @@ -150,8 +161,14 @@ public void testLoadBalancerNamespaceMaximumBundles() throws Exception { @AfterMethod(alwaysRun = true) void shutdown() throws Exception { log.info("--- Shutting down ---"); - pulsar.close(); - bkEnsemble.stop(); + if (pulsar != null) { + pulsar.close(); + pulsar = null; + } + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; + } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerSharedTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerSharedTest.java index 8bc097779b00a..465e8e2d85246 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerSharedTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerSharedTest.java @@ -18,13 +18,9 @@ */ package org.apache.pulsar.broker.loadbalance.impl; +import com.google.common.collect.Sets; import java.util.HashSet; import java.util.Set; - -import com.google.common.collect.Sets; - -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet; import org.testng.Assert; import org.testng.annotations.Test; @@ -37,59 +33,44 @@ public void testRemoveMostServicingBrokersForNamespace() { String assignedBundle = namespace + "/0x00000000_0x40000000"; Set candidates = new HashSet<>(); - ConcurrentOpenHashMap>> map = - ConcurrentOpenHashMap.>>newBuilder() - .build(); - LoadManagerShared.removeMostServicingBrokersForNamespace(assignedBundle, candidates, map); + final var cache = new BundleRangeCache(); + LoadManagerShared.removeMostServicingBrokersForNamespace(assignedBundle, candidates, cache); Assert.assertEquals(candidates.size(), 0); candidates = Sets.newHashSet("broker1"); - LoadManagerShared.removeMostServicingBrokersForNamespace(assignedBundle, candidates, map); + LoadManagerShared.removeMostServicingBrokersForNamespace(assignedBundle, candidates, cache); Assert.assertEquals(candidates.size(), 1); Assert.assertTrue(candidates.contains("broker1")); candidates = Sets.newHashSet("broker1"); - fillBrokerToNamespaceToBundleMap(map, "broker1", namespace, "0x40000000_0x80000000"); - LoadManagerShared.removeMostServicingBrokersForNamespace(assignedBundle, candidates, map); + cache.add("broker1", namespace, "0x40000000_0x80000000"); + LoadManagerShared.removeMostServicingBrokersForNamespace(assignedBundle, candidates, cache); Assert.assertEquals(candidates.size(), 1); Assert.assertTrue(candidates.contains("broker1")); candidates = Sets.newHashSet("broker1", "broker2"); - LoadManagerShared.removeMostServicingBrokersForNamespace(assignedBundle, candidates, map); + LoadManagerShared.removeMostServicingBrokersForNamespace(assignedBundle, candidates, cache); Assert.assertEquals(candidates.size(), 1); Assert.assertTrue(candidates.contains("broker2")); candidates = Sets.newHashSet("broker1", "broker2"); - fillBrokerToNamespaceToBundleMap(map, "broker2", namespace, "0x80000000_0xc0000000"); - LoadManagerShared.removeMostServicingBrokersForNamespace(assignedBundle, candidates, map); + cache.add("broker2", namespace, "0x80000000_0xc0000000"); + LoadManagerShared.removeMostServicingBrokersForNamespace(assignedBundle, candidates, cache); Assert.assertEquals(candidates.size(), 2); Assert.assertTrue(candidates.contains("broker1")); Assert.assertTrue(candidates.contains("broker2")); candidates = Sets.newHashSet("broker1", "broker2"); - fillBrokerToNamespaceToBundleMap(map, "broker2", namespace, "0xc0000000_0xd0000000"); - LoadManagerShared.removeMostServicingBrokersForNamespace(assignedBundle, candidates, map); + cache.add("broker2", namespace, "0xc0000000_0xd0000000"); + LoadManagerShared.removeMostServicingBrokersForNamespace(assignedBundle, candidates, cache); Assert.assertEquals(candidates.size(), 1); Assert.assertTrue(candidates.contains("broker1")); candidates = Sets.newHashSet("broker1", "broker2", "broker3"); - fillBrokerToNamespaceToBundleMap(map, "broker3", namespace, "0xd0000000_0xffffffff"); - LoadManagerShared.removeMostServicingBrokersForNamespace(assignedBundle, candidates, map); + cache.add("broker3", namespace, "0xd0000000_0xffffffff"); + LoadManagerShared.removeMostServicingBrokersForNamespace(assignedBundle, candidates, cache); Assert.assertEquals(candidates.size(), 2); Assert.assertTrue(candidates.contains("broker1")); Assert.assertTrue(candidates.contains("broker3")); } - - private static void fillBrokerToNamespaceToBundleMap( - ConcurrentOpenHashMap>> map, - String broker, String namespace, String bundle) { - map.computeIfAbsent(broker, - k -> ConcurrentOpenHashMap.>newBuilder().build()) - .computeIfAbsent(namespace, - k -> ConcurrentOpenHashSet.newBuilder().build()) - .add(bundle); - } - } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java similarity index 59% rename from pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java rename to pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java index 4b9f679f19d1f..aceeefe304b62 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java @@ -16,11 +16,14 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.broker.loadbalance; +package org.apache.pulsar.broker.loadbalance.impl; import static java.lang.Thread.sleep; -import static org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl.TIME_AVERAGE_BROKER_ZPATH; +import static org.apache.pulsar.broker.resources.LoadBalanceResources.BROKER_TIME_AVERAGE_BASE_PATH; +import static org.apache.pulsar.broker.resources.LoadBalanceResources.BUNDLE_DATA_BASE_PATH; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -28,6 +31,7 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.BoundType; import com.google.common.collect.Range; @@ -37,13 +41,16 @@ import java.lang.reflect.Method; import java.net.URL; import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Random; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -55,46 +62,64 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; +import org.apache.pulsar.broker.loadbalance.LoadBalancerTestingUtils; +import org.apache.pulsar.broker.loadbalance.LoadData; +import org.apache.pulsar.broker.loadbalance.LoadManager; +import org.apache.pulsar.broker.loadbalance.ResourceUnit; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared.BrokerTopicLoadingPredicate; -import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl; -import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerWrapper; -import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; import org.apache.pulsar.client.admin.Namespaces; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceBundleFactory; +import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm; import org.apache.pulsar.common.naming.NamespaceBundles; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.ServiceUnitId; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.BundlesData; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.NamespaceIsolationDataImpl; +import org.apache.pulsar.common.policies.data.ResourceQuota; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.apache.pulsar.common.util.PortManager; +import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.Notification; import org.apache.pulsar.metadata.api.NotificationType; import org.apache.pulsar.metadata.api.extended.CreateOption; +import org.apache.pulsar.policies.data.loadbalancer.BrokerData; +import org.apache.pulsar.policies.data.loadbalancer.BundleData; import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; -import org.apache.pulsar.policies.data.loadbalancer.BrokerData; -import org.apache.pulsar.policies.data.loadbalancer.BundleData; import org.apache.pulsar.policies.data.loadbalancer.TimeAverageBrokerData; import org.apache.pulsar.policies.data.loadbalancer.TimeAverageMessageData; +import org.apache.pulsar.utils.ResourceUtils; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.awaitility.Awaitility; import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @Slf4j @Test(groups = "broker") public class ModularLoadManagerImplTest { + + public final static String CA_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem"); + public final static String BROKER_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem"); + public final static String BROKER_KEY_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem"); + private LocalBookkeeperEnsemble bkEnsemble; private URL url1; @@ -107,8 +132,9 @@ public class ModularLoadManagerImplTest { private PulsarService pulsar3; - private String primaryHost; - private String secondaryHost; + private String primaryBrokerId; + + private String secondaryBrokerId; private NamespaceBundleFactory nsFactory; @@ -155,6 +181,7 @@ void setup() throws Exception { config1.setLoadBalancerLoadSheddingStrategy("org.apache.pulsar.broker.loadbalance.impl.OverloadShedder"); config1.setClusterName("use"); config1.setWebServicePort(Optional.of(0)); + config1.setWebServicePortTls(Optional.of(0)); config1.setMetadataStoreUrl("zk:127.0.0.1:" + bkEnsemble.getZookeeperPort()); config1.setAdvertisedAddress("localhost"); @@ -162,11 +189,13 @@ void setup() throws Exception { config1.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); config1.setBrokerServicePort(Optional.of(0)); config1.setBrokerServicePortTls(Optional.of(0)); - config1.setWebServicePortTls(Optional.of(0)); + config1.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + config1.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + config1.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); pulsar1 = new PulsarService(config1); pulsar1.start(); - primaryHost = String.format("%s:%d", "localhost", pulsar1.getListenPortHTTP().get()); + primaryBrokerId = pulsar1.getBrokerId(); url1 = new URL(pulsar1.getWebServiceAddress()); admin1 = PulsarAdmin.builder().serviceHttpUrl(url1.toString()).build(); @@ -176,13 +205,16 @@ void setup() throws Exception { config2.setLoadBalancerLoadSheddingStrategy("org.apache.pulsar.broker.loadbalance.impl.OverloadShedder"); config2.setClusterName("use"); config2.setWebServicePort(Optional.of(0)); + config2.setWebServicePortTls(Optional.of(0)); config2.setMetadataStoreUrl("zk:127.0.0.1:" + bkEnsemble.getZookeeperPort()); config2.setAdvertisedAddress("localhost"); config2.setBrokerShutdownTimeoutMs(0L); config2.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); config2.setBrokerServicePort(Optional.of(0)); config2.setBrokerServicePortTls(Optional.of(0)); - config2.setWebServicePortTls(Optional.of(0)); + config2.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + config2.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + config2.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); pulsar2 = new PulsarService(config2); pulsar2.start(); @@ -191,16 +223,19 @@ void setup() throws Exception { config.setLoadBalancerLoadSheddingStrategy("org.apache.pulsar.broker.loadbalance.impl.OverloadShedder"); config.setClusterName("use"); config.setWebServicePort(Optional.of(0)); + config.setWebServicePortTls(Optional.of(0)); config.setMetadataStoreUrl("zk:127.0.0.1:" + bkEnsemble.getZookeeperPort()); config.setAdvertisedAddress("localhost"); config.setBrokerShutdownTimeoutMs(0L); config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); config.setBrokerServicePort(Optional.of(0)); config.setBrokerServicePortTls(Optional.of(0)); - config.setWebServicePortTls(Optional.of(0)); + config.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + config.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + config.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); pulsar3 = new PulsarService(config); - secondaryHost = String.format("%s:%d", "localhost", pulsar2.getListenPortHTTP().get()); + secondaryBrokerId = pulsar2.getBrokerId(); url2 = new URL(pulsar2.getWebServiceAddress()); admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build(); @@ -213,19 +248,36 @@ void setup() throws Exception { @AfterMethod(alwaysRun = true) void shutdown() throws Exception { log.info("--- Shutting down ---"); - executor.shutdownNow(); + if (executor != null) { + executor.shutdownNow(); + executor = null; + } - admin1.close(); - admin2.close(); + if (admin1 != null) { + admin1.close(); + admin1 = null; + } + if (admin2 != null) { + admin2.close(); + admin2 = null; + } - pulsar2.close(); - pulsar1.close(); - - if (pulsar3.isRunning()) { + if (pulsar2 != null) { + pulsar2.close(); + pulsar2 = null; + } + if (pulsar1 != null) { + pulsar1.close(); + pulsar1 = null; + } + if (pulsar3 != null && pulsar3.isRunning()) { pulsar3.close(); } - - bkEnsemble.stop(); + pulsar3 = null; + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; + } } private NamespaceBundle makeBundle(final String property, final String cluster, final String namespace) { @@ -251,7 +303,7 @@ public void testCandidateConsistency() throws Exception { for (int i = 0; i < 2; ++i) { final ServiceUnitId serviceUnit = makeBundle(Integer.toString(i)); final String broker = primaryLoadManager.selectBrokerForAssignment(serviceUnit).get(); - if (broker.equals(primaryHost)) { + if (broker.equals(primaryBrokerId)) { foundFirst = true; } else { foundSecond = true; @@ -267,12 +319,12 @@ public void testCandidateConsistency() throws Exception { LoadData loadData = (LoadData) getField(primaryLoadManager, "loadData"); // Make sure the second broker is not in the internal map. - Awaitility.await().untilAsserted(() -> assertFalse(loadData.getBrokerData().containsKey(secondaryHost))); + Awaitility.await().untilAsserted(() -> assertFalse(loadData.getBrokerData().containsKey(secondaryBrokerId))); // Try 5 more selections, ensure they all go to the first broker. for (int i = 2; i < 7; ++i) { final ServiceUnitId serviceUnit = makeBundle(Integer.toString(i)); - assertEquals(primaryLoadManager.selectBrokerForAssignment(serviceUnit), primaryHost); + assertEquals(primaryLoadManager.selectBrokerForAssignment(serviceUnit), primaryBrokerId); } } @@ -288,13 +340,13 @@ public void testEvenBundleDistribution() throws Exception { final TimeAverageMessageData longTermMessageData = new TimeAverageMessageData(1000); longTermMessageData.setMsgRateIn(1000); bundleData.setLongTermData(longTermMessageData); - final String firstBundleDataPath = String.format("%s/%s", ModularLoadManagerImpl.BUNDLE_DATA_PATH, bundles[0]); + final String firstBundleDataPath = String.format("%s/%s", BUNDLE_DATA_BASE_PATH, bundles[0]); // Write long message rate for first bundle to ensure that even bundle distribution is not a coincidence of // balancing by message rate. If we were balancing by message rate, one of the brokers should only have this // one bundle. pulsar1.getLocalMetadataStore().getMetadataCache(BundleData.class).create(firstBundleDataPath, bundleData).join(); for (final NamespaceBundle bundle : bundles) { - if (primaryLoadManager.selectBrokerForAssignment(bundle).equals(primaryHost)) { + if (primaryLoadManager.selectBrokerForAssignment(bundle).equals(primaryBrokerId)) { ++numAssignedToPrimary; } else { ++numAssignedToSecondary; @@ -308,52 +360,52 @@ public void testEvenBundleDistribution() throws Exception { } - + @Test public void testBrokerAffinity() throws Exception { // Start broker 3 pulsar3.start(); - + final String tenant = "test"; final String cluster = "test"; String namespace = tenant + "/" + cluster + "/" + "test"; String topic = "persistent://" + namespace + "/my-topic1"; - admin1.clusters().createCluster(cluster, ClusterData.builder().serviceUrl("http://" + pulsar1.getAdvertisedAddress()).build()); + admin1.clusters().createCluster(cluster, ClusterData.builder().serviceUrl(pulsar1.getWebServiceAddress()).build()); admin1.tenants().createTenant(tenant, new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet(cluster))); admin1.namespaces().createNamespace(namespace, 16); - + String topicLookup = admin1.lookups().lookupTopic(topic); String bundleRange = admin1.lookups().getBundleRange(topic); - + String brokerServiceUrl = pulsar1.getBrokerServiceUrl(); - String brokerUrl = pulsar1.getSafeWebServiceAddress(); + String brokerId = pulsar1.getBrokerId(); log.debug("initial broker service url - {}", topicLookup); Random rand=new Random(); - + if (topicLookup.equals(brokerServiceUrl)) { int x = rand.nextInt(2); if (x == 0) { - brokerUrl = pulsar2.getSafeWebServiceAddress(); + brokerId = pulsar2.getBrokerId(); brokerServiceUrl = pulsar2.getBrokerServiceUrl(); } else { - brokerUrl = pulsar3.getSafeWebServiceAddress(); + brokerId = pulsar3.getBrokerId(); brokerServiceUrl = pulsar3.getBrokerServiceUrl(); } } - brokerUrl = brokerUrl.replaceFirst("http[s]?://", ""); - log.debug("destination broker service url - {}, broker url - {}", brokerServiceUrl, brokerUrl); - String leaderServiceUrl = admin1.brokers().getLeaderBroker().getServiceUrl(); - log.debug("leader serviceUrl - {}, broker1 service url - {}", leaderServiceUrl, pulsar1.getSafeWebServiceAddress()); - //Make a call to broker which is not a leader - if (!leaderServiceUrl.equals(pulsar1.getSafeWebServiceAddress())) { - admin1.namespaces().unloadNamespaceBundle(namespace, bundleRange, brokerUrl); + log.debug("destination broker service url - {}, broker url - {}", brokerServiceUrl, brokerId); + String leaderBrokerId = admin1.brokers().getLeaderBroker().getBrokerId(); + log.debug("leader lookup address - {}, broker1 lookup address - {}", leaderBrokerId, + pulsar1.getBrokerId()); + // Make a call to broker which is not a leader + if (!leaderBrokerId.equals(pulsar1.getBrokerId())) { + admin1.namespaces().unloadNamespaceBundle(namespace, bundleRange, brokerId); } else { - admin2.namespaces().unloadNamespaceBundle(namespace, bundleRange, brokerUrl); + admin2.namespaces().unloadNamespaceBundle(namespace, bundleRange, brokerId); } - + sleep(2000); String topicLookupAfterUnload = admin1.lookups().lookupTopic(topic); log.debug("final broker service url - {}", topicLookupAfterUnload); @@ -384,7 +436,7 @@ public void testMaxTopicDistributionToBroker() throws Exception { final TimeAverageMessageData longTermMessageData = new TimeAverageMessageData(1000); longTermMessageData.setMsgRateIn(1000); bundleData.setLongTermData(longTermMessageData); - final String firstBundleDataPath = String.format("%s/%s", ModularLoadManagerImpl.BUNDLE_DATA_PATH, bundles[0]); + final String firstBundleDataPath = String.format("%s/%s", BUNDLE_DATA_BASE_PATH, bundles[0]); pulsar1.getLocalMetadataStore().getMetadataCache(BundleData.class).create(firstBundleDataPath, bundleData).join(); String maxTopicOwnedBroker = primaryLoadManager.selectBrokerForAssignment(bundles[0]).get(); @@ -393,6 +445,61 @@ public void testMaxTopicDistributionToBroker() throws Exception { } } + /** + * It verifies that the load-manager of leader broker only write topK * brokerCount bundles to zk. + */ + @Test + public void testFilterBundlesWhileWritingToMetadataStore() throws Exception { + Map pulsarServices = new HashMap<>(); + pulsarServices.put(pulsar1.getWebServiceAddress(), pulsar1); + pulsarServices.put(pulsar2.getWebServiceAddress(), pulsar2); + MetadataCache metadataCache = pulsar1.getLocalMetadataStore().getMetadataCache(BundleData.class); + String protocol = "http://"; + PulsarService leaderBroker = pulsarServices.get(protocol + pulsar1.getLeaderElectionService().getCurrentLeader().get().getBrokerId()); + ModularLoadManagerImpl loadManager = (ModularLoadManagerImpl) getField( + leaderBroker.getLoadManager().get(), "loadManager"); + int topK = 1; + leaderBroker.getConfiguration().setLoadBalancerMaxNumberOfBundlesInBundleLoadReport(topK); + // there are two broker in cluster, so total bundle count will be topK * 2 + int exportBundleCount = topK * 2; + + // create and configure bundle-data + final int totalBundles = 5; + final NamespaceBundle[] bundles = LoadBalancerTestingUtils.makeBundles( + nsFactory, "test", "test", "test", totalBundles); + LoadData loadData = (LoadData) getField(loadManager, "loadData"); + for (int i = 0; i < totalBundles; i++) { + final BundleData bundleData = new BundleData(10, 1000); + final String bundleDataPath = String.format("%s/%s", BUNDLE_DATA_BASE_PATH, bundles[i]); + final TimeAverageMessageData longTermMessageData = new TimeAverageMessageData(1000); + longTermMessageData.setMsgThroughputIn(1000 * i); + longTermMessageData.setMsgThroughputOut(1000 * i); + longTermMessageData.setMsgRateIn(1000 * i); + longTermMessageData.setNumSamples(1000); + bundleData.setLongTermData(longTermMessageData); + loadData.getBundleData().put(bundles[i].toString(), bundleData); + loadData.getBrokerData().get(leaderBroker.getWebServiceAddress().substring(protocol.length())) + .getLocalData().getLastStats().put(bundles[i].toString(), new NamespaceBundleStats()); + metadataCache.create(bundleDataPath, bundleData).join(); + } + for (int i = 0; i < totalBundles; i++) { + final String bundleDataPath = String.format("%s/%s", BUNDLE_DATA_BASE_PATH, bundles[i]); + assertEquals(metadataCache.getWithStats(bundleDataPath).get().get().getStat().getVersion(), 0); + } + + // update bundle data to zk and verify + loadManager.writeBundleDataOnZooKeeper(); + int filterBundleCount = totalBundles - exportBundleCount; + for (int i = 0; i < filterBundleCount; i++) { + final String bundleDataPath = String.format("%s/%s", BUNDLE_DATA_BASE_PATH, bundles[i]); + assertEquals(metadataCache.getWithStats(bundleDataPath).get().get().getStat().getVersion(), 0); + } + for (int i = filterBundleCount; i < totalBundles; i++) { + final String bundleDataPath = String.format("%s/%s", BUNDLE_DATA_BASE_PATH, bundles[i]); + assertEquals(metadataCache.getWithStats(bundleDataPath).get().get().getStat().getVersion(), 1); + } + } + // Test that load shedding works @Test public void testLoadShedding() throws Exception { @@ -410,39 +517,127 @@ public void testLoadShedding() throws Exception { doAnswer(invocation -> { bundleReference.set(invocation.getArguments()[0].toString() + '/' + invocation.getArguments()[1]); return null; - }).when(namespacesSpy1).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString()); + }).when(namespacesSpy1).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); + + AtomicReference> selectedBrokerRef = new AtomicReference<>(); + ModularLoadManagerImpl primaryLoadManagerSpy = spy(primaryLoadManager); + doAnswer(invocation -> { + ServiceUnitId serviceUnitId = (ServiceUnitId) invocation.getArguments()[0]; + Optional broker = primaryLoadManager.selectBroker(serviceUnitId); + selectedBrokerRef.set(broker); + return broker; + }).when(primaryLoadManagerSpy).selectBroker(any()); + setField(pulsar1.getAdminClient(), "namespaces", namespacesSpy1); pulsar1.getConfiguration().setLoadBalancerEnabled(true); - final LoadData loadData = (LoadData) getField(primaryLoadManager, "loadData"); + final LoadData loadData = (LoadData) getField(primaryLoadManagerSpy, "loadData"); final Map brokerDataMap = loadData.getBrokerData(); - final BrokerData brokerDataSpy1 = spy(brokerDataMap.get(primaryHost)); + final BrokerData brokerDataSpy1 = spy(brokerDataMap.get(primaryBrokerId)); when(brokerDataSpy1.getLocalData()).thenReturn(localBrokerData); - brokerDataMap.put(primaryHost, brokerDataSpy1); + brokerDataMap.put(primaryBrokerId, brokerDataSpy1); // Need to update all the bundle data for the shredder to see the spy. - primaryLoadManager.handleDataNotification(new Notification(NotificationType.Created, LoadManager.LOADBALANCE_BROKERS_ROOT + "/broker:8080")); + primaryLoadManagerSpy.handleDataNotification(new Notification(NotificationType.Created, LoadManager.LOADBALANCE_BROKERS_ROOT + "/broker:8080")); sleep(100); localBrokerData.setCpu(new ResourceUsage(80, 100)); - primaryLoadManager.doLoadShedding(); + primaryLoadManagerSpy.doLoadShedding(); // 80% is below overload threshold: verify nothing is unloaded. - verify(namespacesSpy1, Mockito.times(0)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString()); + verify(namespacesSpy1, Mockito.times(0)) + .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); localBrokerData.setCpu(new ResourceUsage(90, 100)); - primaryLoadManager.doLoadShedding(); + primaryLoadManagerSpy.doLoadShedding(); // Most expensive bundle will be unloaded. - verify(namespacesSpy1, Mockito.times(1)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString()); + verify(namespacesSpy1, Mockito.times(1)) + .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); assertEquals(bundleReference.get(), mockBundleName(2)); + assertEquals(selectedBrokerRef.get().get(), secondaryBrokerId); - primaryLoadManager.doLoadShedding(); + primaryLoadManagerSpy.doLoadShedding(); // Now less expensive bundle will be unloaded (normally other bundle would move off and nothing would be // unloaded, but this is not the case due to the spy's behavior). - verify(namespacesSpy1, Mockito.times(2)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString()); + verify(namespacesSpy1, Mockito.times(2)) + .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); assertEquals(bundleReference.get(), mockBundleName(1)); + assertEquals(selectedBrokerRef.get().get(), secondaryBrokerId); - primaryLoadManager.doLoadShedding(); + primaryLoadManagerSpy.doLoadShedding(); // Now both are in grace period: neither should be unloaded. - verify(namespacesSpy1, Mockito.times(2)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString()); + verify(namespacesSpy1, Mockito.times(2)) + .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); + assertEquals(selectedBrokerRef.get().get(), secondaryBrokerId); + + // Test bundle transfer to same broker + + loadData.getRecentlyUnloadedBundles().clear(); + primaryLoadManagerSpy.doLoadShedding(); + verify(namespacesSpy1, Mockito.times(3)) + .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); + + loadData.getRecentlyUnloadedBundles().clear(); + primaryLoadManagerSpy.doLoadShedding(); + // The bundle shouldn't be unloaded because the broker is the same. + verify(namespacesSpy1, Mockito.times(4)) + .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); + } + + @Test + public void testUnloadBundleMetric() throws Exception { + final NamespaceBundleStats stats1 = new NamespaceBundleStats(); + final NamespaceBundleStats stats2 = new NamespaceBundleStats(); + stats1.msgRateIn = 100; + stats2.msgRateIn = 200; + final Map statsMap = new ConcurrentHashMap<>(); + statsMap.put(mockBundleName(1), stats1); + statsMap.put(mockBundleName(2), stats2); + final LocalBrokerData localBrokerData = new LocalBrokerData(); + localBrokerData.update(new SystemResourceUsage(), statsMap); + final Namespaces namespacesSpy1 = spy(pulsar1.getAdminClient().namespaces()); + doNothing().when(namespacesSpy1).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); + setField(pulsar1.getAdminClient(), "namespaces", namespacesSpy1); + ModularLoadManagerImpl primaryLoadManagerSpy = spy(primaryLoadManager); + + pulsar1.getConfiguration().setLoadBalancerEnabled(true); + final LoadData loadData = (LoadData) getField(primaryLoadManagerSpy, "loadData"); + + final Map brokerDataMap = loadData.getBrokerData(); + final BrokerData brokerDataSpy1 = spy(brokerDataMap.get(primaryBrokerId)); + when(brokerDataSpy1.getLocalData()).thenReturn(localBrokerData); + brokerDataMap.put(primaryBrokerId, brokerDataSpy1); + // Need to update all the bundle data for the shredder to see the spy. + primaryLoadManagerSpy.handleDataNotification(new Notification(NotificationType.Created, LoadManager.LOADBALANCE_BROKERS_ROOT + "/broker:8080")); + + sleep(100); + + // Most expensive bundle will be unloaded. + localBrokerData.setCpu(new ResourceUsage(90, 100)); + primaryLoadManagerSpy.doLoadShedding(); + assertEquals(getField(primaryLoadManagerSpy, "unloadBundleCount"), 1l); + assertEquals(getField(primaryLoadManagerSpy, "unloadBrokerCount"), 1l); + + // Now less expensive bundle will be unloaded + primaryLoadManagerSpy.doLoadShedding(); + assertEquals(getField(primaryLoadManagerSpy, "unloadBundleCount"), 2l); + assertEquals(getField(primaryLoadManagerSpy, "unloadBrokerCount"), 2l); + + // Now both are in grace period: neither should be unloaded. + primaryLoadManagerSpy.doLoadShedding(); + assertEquals(getField(primaryLoadManagerSpy, "unloadBundleCount"), 2l); + assertEquals(getField(primaryLoadManagerSpy, "unloadBrokerCount"), 2l); + + // clear the recently unloaded bundles to avoid the grace period + loadData.getRecentlyUnloadedBundles().clear(); + + // Test bundle to be unloaded is filtered. + doAnswer(invocation -> { + // return empty broker to avoid unloading the bundle + return Optional.empty(); + }).when(primaryLoadManagerSpy).selectBroker(any()); + primaryLoadManagerSpy.doLoadShedding(); + + assertEquals(getField(primaryLoadManagerSpy, "unloadBundleCount"), 2l); + assertEquals(getField(primaryLoadManagerSpy, "unloadBrokerCount"), 2l); } // Test that ModularLoadManagerImpl will determine that writing local data to ZooKeeper is necessary if certain @@ -507,6 +702,10 @@ public void testNeedBrokerDataUpdate() throws Exception { currentData.setCpu(new ResourceUsage(206, 1000)); assert (needUpdate.get()); + // set the resource weight of cpu to 0, so that it should not trigger an update + conf.setLoadBalancerCPUResourceWeight(0); + assert (!needUpdate.get()); + lastData.setCpu(new ResourceUsage()); currentData.setCpu(new ResourceUsage()); @@ -559,10 +758,12 @@ public void testNamespaceIsolationPoliciesForPrimaryAndSecondaryBrokers() throws final String tenant = "my-property"; final String cluster = "use"; final String namespace = "my-ns"; - final String broker1Address = pulsar1.getAdvertisedAddress() + "0"; - final String broker2Address = pulsar2.getAdvertisedAddress() + "1"; - final String sharedBroker = "broker3"; - admin1.clusters().createCluster(cluster, ClusterData.builder().serviceUrl("http://" + pulsar1.getAdvertisedAddress()).build()); + String broker1Host = pulsar1.getAdvertisedAddress() + "0"; + final String broker1Address = broker1Host + ":8080"; + String broker2Host = pulsar2.getAdvertisedAddress() + "1"; + final String broker2Address = broker2Host + ":8080"; + final String sharedBroker = "broker3:8080"; + admin1.clusters().createCluster(cluster, ClusterData.builder().serviceUrl(pulsar1.getWebServiceAddress()).build()); admin1.tenants().createTenant(tenant, new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet(cluster))); admin1.namespaces().createNamespace(tenant + "/" + cluster + "/" + namespace); @@ -570,8 +771,8 @@ public void testNamespaceIsolationPoliciesForPrimaryAndSecondaryBrokers() throws // set a new policy String newPolicyJsonTemplate = "{\"namespaces\":[\"%s/%s/%s.*\"],\"primary\":[\"%s\"]," + "\"secondary\":[\"%s\"],\"auto_failover_policy\":{\"policy_type\":\"min_available\",\"parameters\":{\"min_limit\":%s,\"usage_threshold\":80}}}"; - String newPolicyJson = String.format(newPolicyJsonTemplate, tenant, cluster, namespace, broker1Address, - broker2Address, 1); + String newPolicyJson = String.format(newPolicyJsonTemplate, tenant, cluster, namespace, broker1Host, + broker2Host, 1); String newPolicyName = "my-ns-isolation-policies"; ObjectMapper jsonMapper = ObjectMapperFactory.create(); NamespaceIsolationDataImpl nsPolicyData = jsonMapper.readValue(newPolicyJson.getBytes(), @@ -583,12 +784,12 @@ public void testNamespaceIsolationPoliciesForPrimaryAndSecondaryBrokers() throws ServiceUnitId serviceUnit = LoadBalancerTestingUtils.makeBundles(nsFactory, tenant, cluster, namespace, 1)[0]; BrokerTopicLoadingPredicate brokerTopicLoadingPredicate = new BrokerTopicLoadingPredicate() { @Override - public boolean isEnablePersistentTopics(String brokerUrl) { + public boolean isEnablePersistentTopics(String brokerId) { return true; } @Override - public boolean isEnableNonPersistentTopics(String brokerUrl) { + public boolean isEnableNonPersistentTopics(String brokerId) { return true; } }; @@ -620,8 +821,8 @@ public boolean isEnableNonPersistentTopics(String brokerUrl) { // (2) now we will have isolation policy : primary=broker1, secondary=broker2, minLimit=2 - newPolicyJson = String.format(newPolicyJsonTemplate, tenant, cluster, namespace, broker1Address, - broker2Address, 2); + newPolicyJson = String.format(newPolicyJsonTemplate, tenant, cluster, namespace, broker1Host, + broker2Host, 2); nsPolicyData = jsonMapper.readValue(newPolicyJson.getBytes(), NamespaceIsolationDataImpl.class); admin1.clusters().createNamespaceIsolationPolicy("use", newPolicyName, nsPolicyData); @@ -658,16 +859,18 @@ public void testLoadSheddingWithNamespaceIsolationPolicies() throws Exception { final String tenant = "my-tenant"; final String namespace = "my-tenant/use/my-ns"; final String bundle = "0x00000000_0xffffffff"; - final String brokerAddress = pulsar1.getAdvertisedAddress(); - final String broker1Address = pulsar1.getAdvertisedAddress() + 1; + final String brokerHost = pulsar1.getAdvertisedAddress(); + final String brokerAddress = brokerHost + ":8080"; + final String broker1Host = pulsar1.getAdvertisedAddress() + "1"; + final String broker1Address = broker1Host + ":8080"; - admin1.clusters().createCluster(cluster, ClusterData.builder().serviceUrl("http://" + pulsar1.getAdvertisedAddress()).build()); + admin1.clusters().createCluster(cluster, ClusterData.builder().serviceUrl(pulsar1.getWebServiceAddress()).build()); admin1.tenants().createTenant(tenant, new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet(cluster))); admin1.namespaces().createNamespace(namespace); @Cleanup - PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsar1.getSafeWebServiceAddress()).build(); + PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsar1.getWebServiceAddress()).build(); Producer producer = pulsarClient.newProducer().topic("persistent://" + namespace + "/my-topic1") .create(); ModularLoadManagerImpl loadManager = (ModularLoadManagerImpl) ((ModularLoadManagerWrapper) pulsar1 @@ -676,12 +879,13 @@ public void testLoadSheddingWithNamespaceIsolationPolicies() throws Exception { loadManager.updateAll(); // test1: no isolation policy - assertTrue(loadManager.shouldNamespacePoliciesUnload(namespace, bundle, primaryHost)); + assertTrue(loadManager.shouldNamespacePoliciesUnload(namespace, bundle, primaryBrokerId)); // test2: as isolation policy, there are not another broker to load the bundle. String newPolicyJsonTemplate = "{\"namespaces\":[\"%s.*\"],\"primary\":[\"%s\"]," + "\"secondary\":[\"%s\"],\"auto_failover_policy\":{\"policy_type\":\"min_available\",\"parameters\":{\"min_limit\":%s,\"usage_threshold\":80}}}"; - String newPolicyJson = String.format(newPolicyJsonTemplate, namespace, broker1Address,broker1Address, 1); + + String newPolicyJson = String.format(newPolicyJsonTemplate, namespace, broker1Host, broker1Host, 1); String newPolicyName = "my-ns-isolation-policies"; ObjectMapper jsonMapper = ObjectMapperFactory.create(); NamespaceIsolationDataImpl nsPolicyData = jsonMapper.readValue(newPolicyJson.getBytes(), @@ -690,11 +894,11 @@ public void testLoadSheddingWithNamespaceIsolationPolicies() throws Exception { assertFalse(loadManager.shouldNamespacePoliciesUnload(namespace, bundle, broker1Address)); // test3: as isolation policy, there are another can load the bundle. - String newPolicyJson1 = String.format(newPolicyJsonTemplate, namespace, brokerAddress,brokerAddress, 1); + String newPolicyJson1 = String.format(newPolicyJsonTemplate, namespace, brokerHost, brokerHost, 1); NamespaceIsolationDataImpl nsPolicyData1 = jsonMapper.readValue(newPolicyJson1.getBytes(), NamespaceIsolationDataImpl.class); admin1.clusters().updateNamespaceIsolationPolicy(cluster, newPolicyName, nsPolicyData1); - assertTrue(loadManager.shouldNamespacePoliciesUnload(namespace, bundle, primaryHost)); + assertTrue(loadManager.shouldNamespacePoliciesUnload(namespace, bundle, primaryBrokerId)); producer.close(); } @@ -711,7 +915,7 @@ public void testOwnBrokerZnodeByMultipleBroker() throws Exception { ServiceConfiguration config = new ServiceConfiguration(); config.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); config.setClusterName("use"); - config.setWebServicePort(Optional.of(0)); + config.setWebServicePort(Optional.of(PortManager.nextLockedFreePort())); config.setMetadataStoreUrl("zk:127.0.0.1:" + bkEnsemble.getZookeeperPort()); config.setBrokerShutdownTimeoutMs(0L); config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); @@ -719,10 +923,12 @@ public void testOwnBrokerZnodeByMultipleBroker() throws Exception { PulsarService pulsar = new PulsarService(config); // create znode using different zk-session final String brokerZnode = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + pulsar.getAdvertisedAddress() + ":" - + config.getWebServicePort(); - pulsar1.getLocalMetadataStore().put(brokerZnode, new byte[0], Optional.empty(), EnumSet.of(CreateOption.Ephemeral)).join(); + + config.getWebServicePort().get(); + pulsar1.getLocalMetadataStore() + .put(brokerZnode, new byte[0], Optional.empty(), EnumSet.of(CreateOption.Ephemeral)).join(); try { pulsar.start(); + fail("should have failed"); } catch (PulsarServerException e) { //Ok. } @@ -743,10 +949,174 @@ public void testRemoveDeadBrokerTimeAverageData() throws Exception { List data = pulsar1.getLocalMetadataStore() .getMetadataCache(TimeAverageBrokerData.class) - .getChildren(TIME_AVERAGE_BROKER_ZPATH) + .getChildren(BROKER_TIME_AVERAGE_BASE_PATH) .join(); Awaitility.await().untilAsserted(() -> assertTrue(pulsar1.getLeaderElectionService().isLeader())); assertEquals(data.size(), 1); } + + @DataProvider(name = "isV1") + public Object[][] isV1() { + return new Object[][] {{true}, {false}}; + } + + @Test(dataProvider = "isV1") + public void testBundleDataDefaultValue(boolean isV1) throws Exception { + final String cluster = "use"; + final String tenant = "my-tenant"; + final String namespace = "my-ns"; + NamespaceName ns = isV1 ? NamespaceName.get(tenant, cluster, namespace) : NamespaceName.get(tenant, namespace); + admin1.clusters().createCluster(cluster, ClusterData.builder().serviceUrl(pulsar1.getWebServiceAddress()).build()); + admin1.tenants().createTenant(tenant, + new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet(cluster))); + admin1.namespaces().createNamespace(ns.toString(), 16); + + // set resourceQuota to the first bundle range. + BundlesData bundlesData = admin1.namespaces().getBundles(ns.toString()); + NamespaceBundle namespaceBundle = nsFactory.getBundle(ns, + Range.range(Long.decode(bundlesData.getBoundaries().get(0)), BoundType.CLOSED, Long.decode(bundlesData.getBoundaries().get(1)), + BoundType.OPEN)); + ResourceQuota quota = new ResourceQuota(); + quota.setMsgRateIn(1024.1); + quota.setMsgRateOut(1024.2); + quota.setBandwidthIn(1024.3); + quota.setBandwidthOut(1024.4); + quota.setMemory(1024.0); + admin1.resourceQuotas().setNamespaceBundleResourceQuota(ns.toString(), namespaceBundle.getBundleRange(), quota); + + ModularLoadManagerWrapper loadManagerWrapper = (ModularLoadManagerWrapper) pulsar1.getLoadManager().get(); + ModularLoadManagerImpl lm = (ModularLoadManagerImpl) loadManagerWrapper.getLoadManager(); + + // get the bundleData of the first bundle range. + // The default value of the bundleData be the same as resourceQuota because the resourceQuota is present. + BundleData defaultBundleData = lm.getBundleDataOrDefault(namespaceBundle.toString()); + + TimeAverageMessageData shortTermData = defaultBundleData.getShortTermData(); + TimeAverageMessageData longTermData = defaultBundleData.getLongTermData(); + assertEquals(shortTermData.getMsgRateIn(), 1024.1); + assertEquals(shortTermData.getMsgRateOut(), 1024.2); + assertEquals(shortTermData.getMsgThroughputIn(), 1024.3); + assertEquals(shortTermData.getMsgThroughputOut(), 1024.4); + + assertEquals(longTermData.getMsgRateIn(), 1024.1); + assertEquals(longTermData.getMsgRateOut(), 1024.2); + assertEquals(longTermData.getMsgThroughputIn(), 1024.3); + assertEquals(longTermData.getMsgThroughputOut(), 1024.4); + } + + + @Test + public void testRemoveNonExistBundleData() + throws PulsarAdminException, InterruptedException, + PulsarClientException, PulsarServerException, NoSuchFieldException, IllegalAccessException { + final String cluster = "use"; + final String tenant = "my-tenant"; + final String namespace = "remove-non-exist-bundle-data-ns"; + final String topicName = tenant + "/" + namespace + "/" + "topic"; + int bundleNumbers = 8; + + admin1.clusters().createCluster(cluster, ClusterData.builder().serviceUrl(pulsar1.getWebServiceAddress()).build()); + admin1.tenants().createTenant(tenant, + new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet(cluster))); + admin1.namespaces().createNamespace(tenant + "/" + namespace, bundleNumbers); + + @Cleanup + PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsar1.getBrokerServiceUrl()).build(); + + // create a lot of topic to fully distributed among bundles. + for (int i = 0; i < 10; i++) { + String topicNameI = topicName + i; + admin1.topics().createPartitionedTopic(topicNameI, 20); + // trigger bundle assignment + + pulsarClient.newConsumer().topic(topicNameI) + .subscriptionName("my-subscriber-name2").subscribe(); + } + + ModularLoadManagerWrapper loadManagerWrapper = (ModularLoadManagerWrapper) pulsar1.getLoadManager().get(); + ModularLoadManagerImpl lm1 = (ModularLoadManagerImpl) loadManagerWrapper.getLoadManager(); + ModularLoadManagerWrapper loadManager2 = (ModularLoadManagerWrapper) pulsar2.getLoadManager().get(); + ModularLoadManagerImpl lm2 = (ModularLoadManagerImpl) loadManager2.getLoadManager(); + + Field executors = lm1.getClass().getDeclaredField("executors"); + executors.setAccessible(true); + ExecutorService executorService = (ExecutorService) executors.get(lm1); + + assertEquals(lm1.getAvailableBrokers().size(), 2); + + pulsar1.getBrokerService().updateRates(); + pulsar2.getBrokerService().updateRates(); + + lm1.writeBrokerDataOnZooKeeper(true); + lm2.writeBrokerDataOnZooKeeper(true); + + // wait for metadata store notification finish + CountDownLatch latch = new CountDownLatch(1); + executorService.submit(latch::countDown); + latch.await(); + + loadManagerWrapper.writeResourceQuotasToZooKeeper(); + + MetadataCache bundlesCache = pulsar1.getLocalMetadataStore().getMetadataCache(BundleData.class); + + // trigger bundle split + String topicToFindBundle = topicName + 0; + NamespaceBundle bundleWillBeSplit = pulsar1.getNamespaceService().getBundle(TopicName.get(topicToFindBundle)); + + final Optional leastLoaded = loadManagerWrapper.getLeastLoaded(bundleWillBeSplit); + assertFalse(leastLoaded.isEmpty()); + + String bundleDataPath = BUNDLE_DATA_BASE_PATH + "/" + tenant + "/" + namespace; + CompletableFuture> children = bundlesCache.getChildren(bundleDataPath); + List bundles = children.join(); + assertTrue(bundles.contains(bundleWillBeSplit.getBundleRange())); + + // after updateAll no namespace bundle data is deleted from metadata store. + lm1.updateAll(); + + children = bundlesCache.getChildren(bundleDataPath); + bundles = children.join(); + assertFalse(bundles.isEmpty()); + assertEquals(bundleNumbers, bundles.size()); + + NamespaceName namespaceName = NamespaceName.get(tenant, namespace); + pulsar1.getAdminClient().namespaces().splitNamespaceBundle(tenant + "/" + namespace, + bundleWillBeSplit.getBundleRange(), + false, NamespaceBundleSplitAlgorithm.RANGE_EQUALLY_DIVIDE_NAME); + + NamespaceBundles allBundlesAfterSplit = + pulsar1.getNamespaceService().getNamespaceBundleFactory() + .getBundles(namespaceName); + + assertFalse(allBundlesAfterSplit.getBundles().contains(bundleWillBeSplit)); + + // the bundle data should be deleted + + pulsar1.getBrokerService().updateRates(); + pulsar2.getBrokerService().updateRates(); + + lm1.writeBrokerDataOnZooKeeper(true); + lm2.writeBrokerDataOnZooKeeper(true); + + latch = new CountDownLatch(1); + // wait for metadata store notification finish + CountDownLatch finalLatch = latch; + executorService.submit(finalLatch::countDown); + latch.await(); + + loadManagerWrapper.writeResourceQuotasToZooKeeper(); + + lm1.updateAll(); + + log.info("update all triggered."); + + // check bundle data should be deleted from metadata store. + + CompletableFuture> childrenAfterSplit = bundlesCache.getChildren(bundleDataPath); + List bundlesAfterSplit = childrenAfterSplit.join(); + + assertFalse(bundlesAfterSplit.contains(bundleWillBeSplit.getBundleRange())); + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedderTest.java index 1f0962f2e44c3..a05c5d4a9e0c2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/OverloadShedderTest.java @@ -149,6 +149,44 @@ public void testBrokerWithMultipleBundles() { assertEquals(bundlesToUnload.get("broker-1"), List.of("bundle-10", "bundle-9")); } + @Test + public void testBrokerWithResourceWeight() { + int numBundles = 10; + LoadData loadData = new LoadData(); + LocalBrokerData broker1 = new LocalBrokerData(); + + double brokerThroghput = 0; + for (int i = 1; i <= numBundles; i++) { + broker1.getBundles().add("bundle-" + i); + BundleData bundle = new BundleData(); + TimeAverageMessageData db = new TimeAverageMessageData(); + double throughput = i * 1024 * 1024; + db.setMsgThroughputIn(throughput); + db.setMsgThroughputOut(throughput); + bundle.setShortTermData(db); + loadData.getBundleData().put("bundle-" + i, bundle); + brokerThroghput += throughput; + } + broker1.setMsgThroughputIn(brokerThroghput); + broker1.setMsgThroughputOut(brokerThroghput); + + loadData.getBrokerData().put("broker-1", new BrokerData(broker1)); + + // set bandwidth usage to 99.9% so that it is considered overloaded + broker1.setBandwidthIn(new ResourceUsage(999, 1000)); + broker1.setBandwidthOut(new ResourceUsage(999, 1000)); + assertFalse(os.findBundlesForUnloading(loadData, conf).isEmpty()); + + // set bandwidth resource weight to 0 so that it is not considered overloaded + conf.setLoadBalancerBandwidthInResourceWeight(0); + conf.setLoadBalancerBandwidthOutResourceWeight(0); + assertTrue(os.findBundlesForUnloading(loadData, conf).isEmpty()); + + // set bandwidth resource weight back to 1, or it will affect other tests + conf.setLoadBalancerBandwidthInResourceWeight(1); + conf.setLoadBalancerBandwidthOutResourceWeight(1); + } + @Test public void testFilterRecentlyUnloaded() { int numBundles = 10; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/UniformLoadShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/UniformLoadShedderTest.java index 00182fffb8a31..4b4042cf31a72 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/UniformLoadShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/UniformLoadShedderTest.java @@ -26,6 +26,7 @@ import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; @Test(groups = "broker") public class UniformLoadShedderTest { @@ -119,4 +120,81 @@ public void testBrokerWithMultipleBundles() { assertFalse(bundlesToUnload.isEmpty()); } + @Test + public void testOverloadBrokerSelect() { + conf.setMaxUnloadBundleNumPerShedding(1); + conf.setMaxUnloadPercentage(0.5); + int numBrokers = 5; + int numBundles = 5; + LoadData loadData = new LoadData(); + + LocalBrokerData[] localBrokerDatas = new LocalBrokerData[]{ + new LocalBrokerData(), + new LocalBrokerData(), + new LocalBrokerData(), + new LocalBrokerData(), + new LocalBrokerData()}; + + String[] brokerNames = new String[]{"broker0", "broker1", "broker2", "broker3", "broker4"}; + + double[] brokerMsgRates = new double[]{ + 50000, // broker0 + 60000, // broker1 + 70000, // broker2 + 10000, // broker3 + 20000};// broker4 + + double[] brokerMsgThroughputs = new double[]{ + 50 * 1024 * 1024, // broker0 + 60 * 1024 * 1024, // broker1 + 70 * 1024 * 1024, // broker2 + 80 * 1024 * 1024, // broker3 + 10 * 1024 * 1024};// broker4 + + + for (int brokerId = 0; brokerId < numBrokers; brokerId++) { + double msgRate = brokerMsgRates[brokerId] / numBundles; + double throughput = brokerMsgThroughputs[brokerId] / numBundles; + for (int i = 0; i < numBundles; ++i) { + String bundleName = "broker-" + brokerId + "-bundle-" + i; + localBrokerDatas[brokerId].getBundles().add(bundleName); + localBrokerDatas[brokerId].setMsgRateIn(brokerMsgRates[brokerId]); + localBrokerDatas[brokerId].setMsgThroughputIn(brokerMsgThroughputs[brokerId]); + BundleData bundle = new BundleData(); + + TimeAverageMessageData timeAverageMessageData = new TimeAverageMessageData(); + timeAverageMessageData.setMsgRateIn(msgRate); + timeAverageMessageData.setMsgThroughputIn(throughput); + bundle.setShortTermData(timeAverageMessageData); + loadData.getBundleData().put(bundleName, bundle); + } + loadData.getBrokerData().put(brokerNames[brokerId], new BrokerData(localBrokerDatas[brokerId])); + } + + // disable throughput based load shedding, enable rate based load shedding only + conf.setLoadBalancerMsgRateDifferenceShedderThreshold(50); + conf.setLoadBalancerMsgThroughputMultiplierDifferenceShedderThreshold(0); + + Multimap bundlesToUnload = uniformLoadShedder.findBundlesForUnloading(loadData, conf); + assertEquals(bundlesToUnload.size(), 1); + assertTrue(bundlesToUnload.containsKey("broker2")); + + + // disable rate based load shedding, enable throughput based load shedding only + conf.setLoadBalancerMsgRateDifferenceShedderThreshold(0); + conf.setLoadBalancerMsgThroughputMultiplierDifferenceShedderThreshold(2); + + bundlesToUnload = uniformLoadShedder.findBundlesForUnloading(loadData, conf); + assertEquals(bundlesToUnload.size(), 1); + assertTrue(bundlesToUnload.containsKey("broker3")); + + // enable both rate and throughput based load shedding, but rate based load shedding has higher priority + conf.setLoadBalancerMsgRateDifferenceShedderThreshold(50); + conf.setLoadBalancerMsgThroughputMultiplierDifferenceShedderThreshold(2); + + bundlesToUnload = uniformLoadShedder.findBundlesForUnloading(loadData, conf); + assertEquals(bundlesToUnload.size(), 1); + assertTrue(bundlesToUnload.containsKey("broker2")); + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/lookup/http/HttpTopicLookupv2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/lookup/http/HttpTopicLookupv2Test.java index 6a6065bc289f6..ab492de055ba5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/lookup/http/HttpTopicLookupv2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/lookup/http/HttpTopicLookupv2Test.java @@ -44,6 +44,7 @@ import org.apache.pulsar.broker.lookup.RedirectData; import org.apache.pulsar.broker.lookup.v1.TopicLookup; import org.apache.pulsar.broker.namespace.NamespaceService; +import org.apache.pulsar.broker.namespace.TopicExistsInfo; import org.apache.pulsar.broker.resources.ClusterResources; import org.apache.pulsar.broker.resources.NamespaceResources; import org.apache.pulsar.broker.resources.PulsarResources; @@ -149,9 +150,12 @@ public void testLookupTopicNotExist() throws Exception { config.setAuthorizationEnabled(true); NamespaceService namespaceService = pulsar.getNamespaceService(); - CompletableFuture future = new CompletableFuture<>(); - future.complete(false); + CompletableFuture future = new CompletableFuture<>(); + future.complete(TopicExistsInfo.newTopicNotExists()); doReturn(future).when(namespaceService).checkTopicExists(any(TopicName.class)); + CompletableFuture booleanFuture = new CompletableFuture<>(); + booleanFuture.complete(false); + doReturn(booleanFuture).when(namespaceService).checkNonPartitionedTopicExists(any(TopicName.class)); AsyncResponse asyncResponse1 = mock(AsyncResponse.class); destLookup.lookupTopicAsync(asyncResponse1, TopicDomain.persistent.value(), "myprop", "usc", "ns2", "topic_not_exist", false, null, null); @@ -260,9 +264,12 @@ public void testValidateReplicationSettingsOnNamespace() throws Exception { policies3Future.complete(Optional.of(policies3)); doReturn(policies3Future).when(namespaceResources).getPoliciesAsync(namespaceName2); NamespaceService namespaceService = pulsar.getNamespaceService(); - CompletableFuture future = new CompletableFuture<>(); - future.complete(false); + CompletableFuture future = new CompletableFuture<>(); + future.complete(TopicExistsInfo.newTopicNotExists()); doReturn(future).when(namespaceService).checkTopicExists(any(TopicName.class)); + CompletableFuture booleanFuture = new CompletableFuture<>(); + booleanFuture.complete(false); + doReturn(future).when(namespaceService).checkNonPartitionedTopicExists(any(TopicName.class)); destLookup.lookupTopicAsync(asyncResponse, TopicDomain.persistent.value(), property, cluster, ns2, "invalid-localCluster", false, null, null); verify(asyncResponse).resume(arg.capture()); @@ -278,4 +285,36 @@ public void testDataPojo() { assertEquals(data2.getRedirectLookupAddress(), url); } + @Test + public void topicNotFound() throws Exception { + MockTopicLookup destLookup = spy(MockTopicLookup.class); + doReturn(false).when(destLookup).isRequestHttps(); + BrokerService brokerService = pulsar.getBrokerService(); + doReturn(new Semaphore(1000,true)).when(brokerService).getLookupRequestSemaphore(); + destLookup.setPulsar(pulsar); + doReturn("null").when(destLookup).clientAppId(); + Field uriField = PulsarWebResource.class.getDeclaredField("uri"); + uriField.setAccessible(true); + UriInfo uriInfo = mock(UriInfo.class); + uriField.set(destLookup, uriInfo); + URI uri = URI.create("http://localhost:8080/lookup/v2/destination/topic/myprop/usc/ns2/topic1"); + doReturn(uri).when(uriInfo).getRequestUri(); + config.setAuthorizationEnabled(true); + NamespaceService namespaceService = pulsar.getNamespaceService(); + CompletableFuture future = new CompletableFuture<>(); + future.complete(TopicExistsInfo.newTopicNotExists()); + doReturn(future).when(namespaceService).checkTopicExists(any(TopicName.class)); + + // Get the current semaphore first + Integer state1 = pulsar.getBrokerService().getLookupRequestSemaphore().availablePermits(); + AsyncResponse asyncResponse1 = mock(AsyncResponse.class); + // We used a nonexistent topic to test + destLookup.lookupTopicAsync(asyncResponse1, TopicDomain.persistent.value(), "myprop", "usc", "ns2", "topic2", false, null, null); + // Gets semaphore status + Integer state2 = pulsar.getBrokerService().getLookupRequestSemaphore().availablePermits(); + // If it is successfully released, it should be equal + assertEquals(state1, state2); + + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceCreateBundlesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceCreateBundlesTest.java index 43d37466918ce..73cfaf1b0d96b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceCreateBundlesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceCreateBundlesTest.java @@ -20,15 +20,21 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +import lombok.Cleanup; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; +import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.policies.data.BookieAffinityGroupData; import org.apache.pulsar.common.policies.data.Policies; +import org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -81,4 +87,31 @@ public void testSplitBundleUpdatesLocalPoliciesWithoutOverwriting() throws Excep assertNotNull(admin.namespaces().getBookieAffinityGroup(namespaceName)); producer.close(); } + + @Test + public void testBundleSplitListener() throws Exception { + String namespaceName = "prop/" + UUID.randomUUID().toString(); + String topicName = "persistent://" + namespaceName + "/my-topic5"; + admin.namespaces().createNamespace(namespaceName); + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topicName).sendTimeout(1, + TimeUnit.SECONDS).create(); + producer.send(new byte[1]); + String bundleRange = admin.lookups().getBundleRange(topicName); + AtomicBoolean isTriggered = new AtomicBoolean(false); + pulsar.getNamespaceService().addNamespaceBundleSplitListener(new NamespaceBundleSplitListener() { + @Override + public void onSplit(NamespaceBundle bundle) { + assertEquals(bundleRange, bundle.getBundleRange()); + isTriggered.set(true); + } + + @Override + public boolean test(NamespaceBundle namespaceBundle) { + return true; + } + }); + admin.namespaces().splitNamespaceBundle(namespaceName, bundleRange, false, null); + Awaitility.await().untilAsserted(() -> assertTrue(isTriggered.get())); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceOwnershipListenerTests.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceOwnershipListenerTests.java index 02787aa14358c..8fc19432eb320 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceOwnershipListenerTests.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceOwnershipListenerTests.java @@ -35,6 +35,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.spy; import static org.testng.Assert.assertTrue; @Test(groups = "broker") @@ -102,6 +104,25 @@ public void unLoad(NamespaceBundle bundle) { deleteNamespaceWithRetry(namespace, false); } + @Test + public void testAddNamespaceBundleOwnershipListenerBeforeLBStart() { + NamespaceService namespaceService = spy(new NamespaceService(pulsar)); + doThrow(new IllegalStateException("The LM is not initialized")) + .when(namespaceService).getOwnedServiceUnits(); + namespaceService.addNamespaceBundleOwnershipListener(new NamespaceBundleOwnershipListener() { + @Override + public void onLoad(NamespaceBundle bundle) {} + + @Override + public void unLoad(NamespaceBundle bundle) {} + + @Override + public boolean test(NamespaceBundle namespaceBundle) { + return false; + } + }); + } + @Test public void testGetAllPartitions() throws Exception { final String namespace = "prop/" + UUID.randomUUID().toString(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java index ac5d92c880227..951247bd68861 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.namespace; +import static org.apache.pulsar.broker.resources.LoadBalanceResources.BUNDLE_DATA_BASE_PATH; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; @@ -32,6 +33,7 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.github.benmanes.caffeine.cache.AsyncLoadingCache; +import com.google.common.collect.Sets; import com.google.common.hash.Hashing; import java.lang.reflect.Field; import java.lang.reflect.Method; @@ -40,16 +42,19 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import lombok.Cleanup; import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.commons.lang3.tuple.Pair; @@ -63,6 +68,7 @@ import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.admin.Namespaces; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.PulsarClient; @@ -72,12 +78,15 @@ import org.apache.pulsar.common.naming.NamespaceBundles; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.ServiceUnitId; +import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.BundlesData; +import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.LocalPolicies; import org.apache.pulsar.common.policies.data.Policies; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.util.ObjectMapperFactory; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.metadata.api.GetResult; import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.Notification; @@ -94,6 +103,7 @@ import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @Test(groups = "flaky") @@ -188,6 +198,7 @@ public void testSplitMapWithRefreshedStatMap() throws Exception { ManagedLedger ledger = mock(ManagedLedger.class); when(ledger.getCursors()).thenReturn(new ArrayList<>()); + when(ledger.getConfig()).thenReturn(new ManagedLedgerConfig()); doReturn(CompletableFuture.completedFuture(null)).when(MockOwnershipCache).disableOwnership(any(NamespaceBundle.class)); Field ownership = NamespaceService.class.getDeclaredField("ownershipCache"); @@ -288,8 +299,7 @@ public void testUnloadNamespaceBundleFailure() throws Exception { final String topicName = "persistent://my-property/use/my-ns/my-topic1"; pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name").subscribe(); - ConcurrentOpenHashMap>> topics = pulsar.getBrokerService() - .getTopics(); + final var topics = pulsar.getBrokerService().getTopics(); Topic spyTopic = spy(topics.get(topicName).get().get()); topics.clear(); CompletableFuture> topicFuture = CompletableFuture.completedFuture(Optional.of(spyTopic)); @@ -319,7 +329,7 @@ public void testUnloadNamespaceBundleWithStuckTopic() throws Exception { final String topicName = "persistent://my-property/use/my-ns/my-topic1"; Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name") .subscribe(); - ConcurrentOpenHashMap>> topics = pulsar.getBrokerService().getTopics(); + final var topics = pulsar.getBrokerService().getTopics(); Topic spyTopic = spy(topics.get(topicName).get().get()); topics.clear(); CompletableFuture> topicFuture = CompletableFuture.completedFuture(Optional.of(spyTopic)); @@ -349,14 +359,14 @@ public void testUnloadNamespaceBundleWithStuckTopic() throws Exception { @Test public void testLoadReportDeserialize() throws Exception { - final String candidateBroker1 = "http://localhost:8000"; - final String candidateBroker2 = "http://localhost:3000"; - LoadReport lr = new LoadReport(null, null, candidateBroker1, null); - LocalBrokerData ld = new LocalBrokerData(null, null, candidateBroker2, null); - URI uri1 = new URI(candidateBroker1); - URI uri2 = new URI(candidateBroker2); - String path1 = String.format("%s/%s:%s", LoadManager.LOADBALANCE_BROKERS_ROOT, uri1.getHost(), uri1.getPort()); - String path2 = String.format("%s/%s:%s", LoadManager.LOADBALANCE_BROKERS_ROOT, uri2.getHost(), uri2.getPort()); + final String candidateBroker1 = "localhost:8000"; + String broker1Url = "pulsar://localhost:6650"; + final String candidateBroker2 = "localhost:3000"; + String broker2Url = "pulsar://localhost:6660"; + LoadReport lr = new LoadReport("http://" + candidateBroker1, null, broker1Url, null); + LocalBrokerData ld = new LocalBrokerData("http://" + candidateBroker2, null, broker2Url, null); + String path1 = String.format("%s/%s", LoadManager.LOADBALANCE_BROKERS_ROOT, candidateBroker1); + String path2 = String.format("%s/%s", LoadManager.LOADBALANCE_BROKERS_ROOT, candidateBroker2); pulsar.getLocalMetadataStore().put(path1, ObjectMapperFactory.getMapper().writer().writeValueAsBytes(lr), @@ -375,23 +385,23 @@ public void testLoadReportDeserialize() throws Exception { .getAndSet(new ModularLoadManagerWrapper(new ModularLoadManagerImpl())); oldLoadManager.stop(); LookupResult result2 = pulsar.getNamespaceService().createLookupResult(candidateBroker2, false, null).get(); - Assert.assertEquals(result1.getLookupData().getBrokerUrl(), candidateBroker1); - Assert.assertEquals(result2.getLookupData().getBrokerUrl(), candidateBroker2); + Assert.assertEquals(result1.getLookupData().getBrokerUrl(), broker1Url); + Assert.assertEquals(result2.getLookupData().getBrokerUrl(), broker2Url); System.out.println(result2); } @Test public void testCreateLookupResult() throws Exception { - final String candidateBroker = "pulsar://localhost:6650"; + final String candidateBroker = "localhost:8080"; + final String brokerUrl = "pulsar://localhost:6650"; final String listenerUrl = "pulsar://localhost:7000"; final String listenerUrlTls = "pulsar://localhost:8000"; final String listener = "listenerName"; Map advertisedListeners = new HashMap<>(); advertisedListeners.put(listener, AdvertisedListener.builder().brokerServiceUrl(new URI(listenerUrl)).brokerServiceUrlTls(new URI(listenerUrlTls)).build()); - LocalBrokerData ld = new LocalBrokerData(null, null, candidateBroker, null, advertisedListeners); - URI uri = new URI(candidateBroker); - String path = String.format("%s/%s:%s", LoadManager.LOADBALANCE_BROKERS_ROOT, uri.getHost(), uri.getPort()); + LocalBrokerData ld = new LocalBrokerData("http://" + candidateBroker, null, brokerUrl, null, advertisedListeners); + String path = String.format("%s/%s", LoadManager.LOADBALANCE_BROKERS_ROOT, candidateBroker); pulsar.getLocalMetadataStore().put(path, ObjectMapperFactory.getMapper().writer().writeValueAsBytes(ld), @@ -401,7 +411,7 @@ public void testCreateLookupResult() throws Exception { LookupResult noListener = pulsar.getNamespaceService().createLookupResult(candidateBroker, false, null).get(); LookupResult withListener = pulsar.getNamespaceService().createLookupResult(candidateBroker, false, listener).get(); - Assert.assertEquals(noListener.getLookupData().getBrokerUrl(), candidateBroker); + Assert.assertEquals(noListener.getLookupData().getBrokerUrl(), brokerUrl); Assert.assertEquals(withListener.getLookupData().getBrokerUrl(), listenerUrl); Assert.assertEquals(withListener.getLookupData().getBrokerUrlTls(), listenerUrlTls); System.out.println(withListener); @@ -649,7 +659,7 @@ public void testSplitBundleWithHighestThroughput() throws Exception { NamespaceBundle targetNamespaceBundle = bundles.findBundle(TopicName.get(topic + "0")); String bundle = targetNamespaceBundle.getBundleRange(); - String path = ModularLoadManagerImpl.getBundleDataPath(namespace + "/" + bundle); + String path = BUNDLE_DATA_BASE_PATH + "/" + namespace + "/" + bundle; NamespaceBundleStats defaultStats = new NamespaceBundleStats(); defaultStats.msgThroughputIn = 100000; defaultStats.msgThroughputOut = 100000; @@ -683,7 +693,7 @@ public void testSplitBundleWithHighestThroughput() throws Exception { @Test public void testHeartbeatNamespaceMatch() throws Exception { - NamespaceName namespaceName = NamespaceService.getHeartbeatNamespace(pulsar.getAdvertisedAddress(), conf); + NamespaceName namespaceName = NamespaceService.getHeartbeatNamespace(pulsar.getBrokerId(), conf); NamespaceBundle namespaceBundle = pulsar.getNamespaceService().getNamespaceBundleFactory().getFullBundle(namespaceName); assertTrue(NamespaceService.isSystemServiceNamespace( NamespaceBundle.getBundleNamespace(namespaceBundle.toString()))); @@ -691,7 +701,6 @@ public void testHeartbeatNamespaceMatch() throws Exception { @Test public void testModularLoadManagerRemoveInactiveBundleFromLoadData() throws Exception { - final String BUNDLE_DATA_PATH = "/loadbalance/bundle-data"; final String namespace = "pulsar/test/ns1"; final String topic1 = "persistent://" + namespace + "/topic1"; final String topic2 = "persistent://" + namespace + "/topic2"; @@ -704,7 +713,7 @@ public void testModularLoadManagerRemoveInactiveBundleFromLoadData() throws Exce Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager"); loadManagerField.setAccessible(true); doReturn(true).when(loadManager).isCentralized(); - SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getSafeWebServiceAddress(), null); + SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getBrokerId(), null); Optional res = Optional.of(resourceUnit); doReturn(res).when(loadManager).getLeastLoaded(any(ServiceUnitId.class)); loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager)); @@ -742,13 +751,12 @@ public void testModularLoadManagerRemoveInactiveBundleFromLoadData() throws Exce Awaitility.await().untilAsserted(() -> { assertNull(loadData.getBundleData().get(oldBundle.toString())); - assertFalse(bundlesCache.exists(BUNDLE_DATA_PATH + "/" + oldBundle.toString()).get()); + assertFalse(bundlesCache.exists(BUNDLE_DATA_BASE_PATH + "/" + oldBundle.toString()).get()); }); } @Test public void testModularLoadManagerRemoveBundleAndLoad() throws Exception { - final String BUNDLE_DATA_PATH = "/loadbalance/bundle-data"; final String namespace = "prop/ns-abc"; final String bundleName = namespace + "/0x00000000_0xffffffff"; final String topic1 = "persistent://" + namespace + "/topic1"; @@ -783,7 +791,7 @@ public void testModularLoadManagerRemoveBundleAndLoad() throws Exception { pulsar.getBrokerService().updateRates(); waitResourceDataUpdateToZK(loadManager); - String path = BUNDLE_DATA_PATH + "/" + bundleName; + String path = BUNDLE_DATA_BASE_PATH + "/" + bundleName; Optional getResult = pulsar.getLocalMetadataStore().get(path).get(); assertTrue(getResult.isPresent()); @@ -800,6 +808,152 @@ public void testModularLoadManagerRemoveBundleAndLoad() throws Exception { assertFalse(getResult.isPresent()); } + @DataProvider(name = "topicDomain") + public Object[] topicDomain() { + return new Object[]{ + TopicDomain.persistent.value(), + TopicDomain.non_persistent.value() + }; + } + + @Test(dataProvider = "topicDomain") + public void testCheckTopicExists(String topicDomain) throws Exception { + String topic = topicDomain + "://prop/ns-abc/" + UUID.randomUUID(); + admin.topics().createNonPartitionedTopic(topic); + Awaitility.await().untilAsserted(() -> { + assertTrue(pulsar.getNamespaceService().checkTopicExists(TopicName.get(topic)).get().isExists()); + }); + + String partitionedTopic = topicDomain + "://prop/ns-abc/" + UUID.randomUUID(); + admin.topics().createPartitionedTopic(partitionedTopic, 5); + Awaitility.await().untilAsserted(() -> { + assertTrue(pulsar.getNamespaceService().checkTopicExists(TopicName.get(partitionedTopic)).get().isExists()); + assertTrue(pulsar.getNamespaceService() + .checkTopicExists(TopicName.get(partitionedTopic + "-partition-2")).get().isExists()); + }); + } + + @Test + public void testAllowedClustersAtNamespaceLevelShouldBeIncludedInAllowedClustersAtTenantLevel() throws Exception { + // 1. Setup + pulsar.getConfiguration().setForceDeleteNamespaceAllowed(true); + pulsar.getConfiguration().setForceDeleteTenantAllowed(true); + Set tenantAllowedClusters = Set.of("test", "r1", "r2"); + Set allowedClusters1 = Set.of("test", "r1", "r2", "r3"); + Set allowedClusters2 = Set.of("test", "r1", "r2"); + Set clusters = Set.of("r1", "r2", "r3", "r4"); + final String tenant = "my-tenant"; + final String namespace = tenant + "/testAllowedCluster"; + admin.tenants().createTenant(tenant, + new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet("test"))); + admin.namespaces().createNamespace(namespace); + pulsar.getPulsarResources().getTenantResources().updateTenantAsync(tenant, tenantInfo -> + TenantInfo.builder().allowedClusters(tenantAllowedClusters).build()); + for (String cluster : clusters) { + pulsar.getPulsarResources().getClusterResources().createCluster(cluster, ClusterData.builder().build()); + } + // 2. Verify + admin.namespaces().setNamespaceAllowedClusters(namespace, allowedClusters2); + + try { + admin.namespaces().setNamespaceAllowedClusters(namespace, allowedClusters1); + fail(); + } catch (PulsarAdminException e) { + assertEquals(e.getStatusCode(), 403); + assertEquals(e.getMessage(), + "Cluster [r3] is not in the list of allowed clusters list for tenant [my-tenant]"); + } + // 3. Clean up + admin.namespaces().deleteNamespace(namespace, true); + admin.tenants().deleteTenant(tenant, true); + for (String cluster : clusters) { + pulsar.getPulsarResources().getClusterResources().deleteCluster(cluster); + } + pulsar.getConfiguration().setForceDeleteNamespaceAllowed(false); + pulsar.getConfiguration().setForceDeleteTenantAllowed(false); + } + + /** + * Test case: + * 1. Replication clusters should be included in the allowed clusters. For compatibility, the replication + * clusters could be set before the allowed clusters are set. + * 2. Peer cluster can not be a part of the allowed clusters. + */ + @Test + public void testNewAllowedClusterAdminAPIAndItsImpactOnReplicationClusterAPI() throws Exception { + // 1. Setup + pulsar.getConfiguration().setForceDeleteNamespaceAllowed(true); + pulsar.getConfiguration().setForceDeleteTenantAllowed(true); + // Setup: Prepare cluster resource, tenant and namespace + Set replicationClusters = Set.of("test", "r1", "r2"); + Set tenantAllowedClusters = Set.of("test", "r1", "r2", "r3"); + Set allowedClusters = Set.of("test", "r1", "r2", "r3"); + Set clusters = Set.of("r1", "r2", "r3", "r4"); + final String tenant = "my-tenant"; + final String namespace = tenant + "/testAllowedCluster"; + admin.tenants().createTenant(tenant, + new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet("test"))); + admin.namespaces().createNamespace(namespace); + pulsar.getPulsarResources().getTenantResources().updateTenantAsync(tenant, tenantInfo -> + TenantInfo.builder().allowedClusters(tenantAllowedClusters).build()); + + Namespaces namespaces = admin.namespaces(); + for (String cluster : clusters) { + pulsar.getPulsarResources().getClusterResources().createCluster(cluster, ClusterData.builder().build()); + } + // 2. Verify + // 2.1 Replication clusters should be included in the allowed clusters. + + // SUCCESS + // 2.1.1. Set replication clusters without allowed clusters at namespace level. + namespaces.setNamespaceReplicationClusters(namespace, replicationClusters); + // 2..1.2 Set allowed clusters. + namespaces.setNamespaceAllowedClusters(namespace, allowedClusters); + // 2.1.3. Get allowed clusters and replication clusters. + List allowedClustersResponse = namespaces.getNamespaceAllowedClusters(namespace); + + List replicationClustersResponse = namespaces.getNamespaceReplicationClusters(namespace); + + assertEquals(replicationClustersResponse.size(), replicationClusters.size()); + assertEquals(allowedClustersResponse.size(), allowedClusters.size()); + + // FAIL + // 2.1.4. Fail: Set allowed clusters whose scope is smaller than replication clusters. + Set allowedClustersSmallScope = Set.of("r1", "r3"); + try { + namespaces.setNamespaceAllowedClusters(namespace, allowedClustersSmallScope); + fail(); + } catch (PulsarAdminException ignore) {} + // 2.1.5. Fail: Set replication clusters whose scope is excel the allowed clusters. + Set replicationClustersExcel = Set.of("r1", "r4"); + try { + namespaces.setNamespaceReplicationClusters(namespace, replicationClustersExcel); + fail(); + //Todo: The status code in the old implementation is confused. + } catch (PulsarAdminException.NotAuthorizedException ignore) {} + + // 2.2 Peer cluster can not be a part of the allowed clusters. + LinkedHashSet peerCluster = new LinkedHashSet<>(); + peerCluster.add("r2"); + pulsar.getPulsarResources().getClusterResources().deleteCluster("r1"); + pulsar.getPulsarResources().getClusterResources().createCluster("r1", + ClusterData.builder().peerClusterNames(peerCluster).build()); + try { + namespaces.setNamespaceAllowedClusters(namespace, Set.of("test", "r1", "r2", "r3")); + fail(); + } catch (PulsarAdminException.ConflictException ignore) {} + + // CleanUp: Namespace with replication clusters can not be deleted by force. + namespaces.setNamespaceReplicationClusters(namespace, Set.of(conf.getClusterName())); + admin.namespaces().deleteNamespace(namespace, true); + admin.tenants().deleteTenant(tenant, true); + for (String cluster : clusters) { + pulsar.getPulsarResources().getClusterResources().deleteCluster(cluster); + } + pulsar.getConfiguration().setForceDeleteNamespaceAllowed(false); + pulsar.getConfiguration().setForceDeleteTenantAllowed(false); + } + /** * 1. Manually trigger "LoadReportUpdaterTask" * 2. Registry another new zk-node-listener "waitForBrokerChangeNotice". @@ -834,10 +988,7 @@ private void waitResourceDataUpdateToZK(LoadManager loadManager) throws Exceptio public CompletableFuture registryBrokerDataChangeNotice() { CompletableFuture completableFuture = new CompletableFuture<>(); - String lookupServiceAddress = pulsar.getAdvertisedAddress() + ":" - + (conf.getWebServicePort().isPresent() ? conf.getWebServicePort().get() - : conf.getWebServicePortTls().get()); - String brokerDataPath = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + lookupServiceAddress; + String brokerDataPath = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + pulsar.getBrokerId(); pulsar.getLocalMetadataStore().registerListener(notice -> { if (brokerDataPath.equals(notice.getPath())){ if (!completableFuture.isDone()) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceUnloadingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceUnloadingTest.java index 0dbfe1760879a..1526611874a62 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceUnloadingTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceUnloadingTest.java @@ -22,8 +22,12 @@ import com.google.common.collect.Sets; +import lombok.Cleanup; import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -34,6 +38,9 @@ public class NamespaceUnloadingTest extends BrokerTestBase { @BeforeMethod @Override protected void setup() throws Exception { + conf.setTopicLevelPoliciesEnabled(true); + conf.setForceDeleteNamespaceAllowed(true); + conf.setTopicLoadTimeoutSeconds(Integer.MAX_VALUE); super.baseSetup(); } @@ -68,4 +75,26 @@ public void testUnloadPartiallyLoadedNamespace() throws Exception { producer.close(); } + @Test + public void testUnloadWithTopicCreation() throws PulsarAdminException, PulsarClientException { + final String namespaceName = "prop/ns_unloading"; + final String topicName = "persistent://prop/ns_unloading/with_topic_creation"; + final int partitions = 5; + admin.namespaces().createNamespace(namespaceName, 1); + admin.topics().createPartitionedTopic(topicName, partitions); + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.INT32) + .topic(topicName) + .create(); + + for (int i = 0; i < 100; i++) { + admin.namespaces().unloadNamespaceBundle(namespaceName, "0x00000000_0xffffffff"); + } + + for (int i = 0; i < partitions; i++) { + producer.send(i); + } + admin.namespaces().deleteNamespace(namespaceName, true); + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnerShipCacheForCurrentServerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnerShipCacheForCurrentServerTest.java index e53d5c25bd2b5..22fa2c32b5655 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnerShipCacheForCurrentServerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnerShipCacheForCurrentServerTest.java @@ -76,7 +76,7 @@ public void testCreateTopicWithNotTopicNsOwnedBroker() { int verifiedBrokerNum = 0; for (PulsarService pulsarService : this.getPulsarServiceList()) { BrokerService bs = pulsarService.getBrokerService(); - if (bs.isTopicNsOwnedByBroker(TopicName.get(topicName))) { + if (bs.isTopicNsOwnedByBrokerAsync(TopicName.get(topicName)).join()) { continue; } verifiedBrokerNum ++; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnerShipForCurrentServerTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnerShipForCurrentServerTestBase.java index 8dd4f53db8240..46e8989ac3df4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnerShipForCurrentServerTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnerShipForCurrentServerTestBase.java @@ -80,10 +80,8 @@ protected void startBroker() throws Exception { conf.setBrokerShutdownTimeoutMs(0L); conf.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); conf.setBrokerServicePort(Optional.of(0)); - conf.setBrokerServicePortTls(Optional.of(0)); conf.setAdvertisedAddress("localhost"); conf.setWebServicePort(Optional.of(0)); - conf.setWebServicePortTls(Optional.of(0)); serviceConfigurationList.add(conf); PulsarTestContext.Builder testContextBuilder = diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java index 9e3d9e3a41340..2c3182659f022 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java @@ -55,15 +55,12 @@ import org.apache.pulsar.metadata.coordination.impl.CoordinationServiceImpl; import org.apache.pulsar.zookeeper.ZookeeperServerTest; import org.awaitility.Awaitility; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @Test(groups = "broker") public class OwnershipCacheTest { - private static final Logger log = LoggerFactory.getLogger(OwnershipCacheTest.class); private PulsarService pulsar; private ServiceConfiguration config; @@ -102,7 +99,7 @@ public void setup() throws Exception { nsService = mock(NamespaceService.class); brokerService = mock(BrokerService.class); doReturn(CompletableFuture.completedFuture(1)).when(brokerService) - .unloadServiceUnit(any(), anyBoolean(), anyLong(), any()); + .unloadServiceUnit(any(), anyBoolean(), anyBoolean(), anyLong(), any()); doReturn(config).when(pulsar).getConfiguration(); doReturn(nsService).when(pulsar).getNamespaceService(); @@ -123,14 +120,14 @@ public void teardown() throws Exception { @Test public void testConstructor() { - OwnershipCache cache = new OwnershipCache(this.pulsar, bundleFactory, nsService); + OwnershipCache cache = new OwnershipCache(this.pulsar, nsService); assertNotNull(cache); assertNotNull(cache.getOwnedBundles()); } @Test public void testDisableOwnership() throws Exception { - OwnershipCache cache = new OwnershipCache(this.pulsar, bundleFactory, nsService); + OwnershipCache cache = new OwnershipCache(this.pulsar, nsService); NamespaceBundle testBundle = new NamespaceBundle(NamespaceName.get("pulsar/test/ns-1"), Range.closedOpen(0L, (long) Integer.MAX_VALUE), @@ -148,7 +145,7 @@ public void testDisableOwnership() throws Exception { @Test public void testGetOrSetOwner() throws Exception { - OwnershipCache cache = new OwnershipCache(this.pulsar, bundleFactory, nsService); + OwnershipCache cache = new OwnershipCache(this.pulsar, nsService); NamespaceBundle testFullBundle = new NamespaceBundle(NamespaceName.get("pulsar/test/ns-2"), Range.closedOpen(0L, (long) Integer.MAX_VALUE), bundleFactory); @@ -194,7 +191,7 @@ public void testGetOrSetOwner() throws Exception { @Test public void testGetOwner() throws Exception { - OwnershipCache cache = new OwnershipCache(this.pulsar, bundleFactory, nsService); + OwnershipCache cache = new OwnershipCache(this.pulsar, nsService); NamespaceBundle testBundle = new NamespaceBundle(NamespaceName.get("pulsar/test/ns-3"), Range.closedOpen(0L, (long) Integer.MAX_VALUE), bundleFactory); @@ -241,7 +238,7 @@ public void testGetOwner() throws Exception { @Test public void testGetOwnedServiceUnit() throws Exception { - OwnershipCache cache = new OwnershipCache(this.pulsar, bundleFactory, nsService); + OwnershipCache cache = new OwnershipCache(this.pulsar, nsService); NamespaceName testNs = NamespaceName.get("pulsar/test/ns-5"); NamespaceBundle testBundle = new NamespaceBundle(testNs, Range.closedOpen(0L, (long) Integer.MAX_VALUE), @@ -301,7 +298,7 @@ public void testGetOwnedServiceUnit() throws Exception { @Test public void testGetOwnedServiceUnits() throws Exception { - OwnershipCache cache = new OwnershipCache(this.pulsar, bundleFactory, nsService); + OwnershipCache cache = new OwnershipCache(this.pulsar, nsService); NamespaceName testNs = NamespaceName.get("pulsar/test/ns-6"); NamespaceBundle testBundle = new NamespaceBundle(testNs, Range.closedOpen(0L, (long) Integer.MAX_VALUE), @@ -347,7 +344,7 @@ public void testGetOwnedServiceUnits() throws Exception { @Test public void testRemoveOwnership() throws Exception { - OwnershipCache cache = new OwnershipCache(this.pulsar, bundleFactory, nsService); + OwnershipCache cache = new OwnershipCache(this.pulsar, nsService); NamespaceName testNs = NamespaceName.get("pulsar/test/ns-7"); NamespaceBundle bundle = new NamespaceBundle(testNs, Range.closedOpen(0L, (long) Integer.MAX_VALUE), @@ -373,7 +370,7 @@ public void testRemoveOwnership() throws Exception { @Test public void testReestablishOwnership() throws Exception { - OwnershipCache cache = new OwnershipCache(this.pulsar, bundleFactory, nsService); + OwnershipCache cache = new OwnershipCache(this.pulsar, nsService); NamespaceBundle testFullBundle = new NamespaceBundle(NamespaceName.get("pulsar/test/ns-8"), Range.closedOpen(0L, (long) Integer.MAX_VALUE), bundleFactory); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/PulsarClientBasedHandler.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/PulsarClientBasedHandler.java new file mode 100644 index 0000000000000..ed9881a8cadb9 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/PulsarClientBasedHandler.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.protocol; + +import io.netty.channel.ChannelInitializer; +import io.netty.channel.socket.SocketChannel; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.service.BrokerService; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Reader; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfo; + +public class PulsarClientBasedHandler implements ProtocolHandler { + + static final String PROTOCOL = "test"; + + private String topic; + private int partitions; + private String cluster; + private PulsarClient client; + private List> readers; + private ExecutorService executor; + private volatile boolean running = false; + volatile long closeTimeMs; + + @Override + public String protocolName() { + return PROTOCOL; + } + + @Override + public boolean accept(String protocol) { + return protocol.equals(PROTOCOL); + } + + @Override + public void initialize(ServiceConfiguration conf) throws Exception { + final var properties = conf.getProperties(); + topic = (String) properties.getOrDefault("metadata.topic", "metadata-topic"); + partitions = (Integer) properties.getOrDefault("metadata.partitions", 1); + cluster = conf.getClusterName(); + } + + @Override + public String getProtocolDataToAdvertise() { + return ""; + } + + @Override + public void start(BrokerService service) { + try { + final var port = service.getPulsar().getListenPortHTTP().orElseThrow(); + @Cleanup final var admin = PulsarAdmin.builder().serviceHttpUrl("http://localhost:" + port).build(); + try { + admin.clusters().createCluster(cluster, ClusterData.builder() + .serviceUrl(service.getPulsar().getWebServiceAddress()) + .serviceUrlTls(service.getPulsar().getWebServiceAddressTls()) + .brokerServiceUrl(service.getPulsar().getBrokerServiceUrl()) + .brokerServiceUrlTls(service.getPulsar().getBrokerServiceUrlTls()) + .build()); + } catch (PulsarAdminException ignored) { + } + try { + admin.tenants().createTenant("public", TenantInfo.builder().allowedClusters(Set.of(cluster)).build()); + } catch (PulsarAdminException ignored) { + } + try { + admin.namespaces().createNamespace("public/default"); + } catch (PulsarAdminException ignored) { + } + } catch (PulsarClientException e) { + throw new RuntimeException(e); + } + try { + final var port = service.getListenPort().orElseThrow(); + client = PulsarClient.builder().serviceUrl("pulsar://localhost:" + port).build(); + readers = new ArrayList<>(); + for (int i = 0; i < partitions; i++) { + readers.add(client.newReader().topic(topic + TopicName.PARTITIONED_TOPIC_SUFFIX + i) + .startMessageId(MessageId.earliest).create()); + } + running = true; + executor = Executors.newSingleThreadExecutor(); + executor.execute(() -> { + while (running) { + readers.forEach(reader -> { + try { + reader.readNext(1, TimeUnit.MILLISECONDS); + } catch (PulsarClientException ignored) { + } + }); + } + }); + } catch (PulsarClientException e) { + throw new RuntimeException(e); + } + } + + @Override + public Map> newChannelInitializers() { + return Map.of(); + } + + @Override + public void close() { + final var start = System.currentTimeMillis(); + running = false; + if (client != null) { + try { + client.close(); + } catch (PulsarClientException ignored) { + } + client = null; + } + if (executor != null) { + executor.shutdown(); + executor = null; + } + closeTimeMs = System.currentTimeMillis() - start; + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/PulsarClientBasedHandlerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/PulsarClientBasedHandlerTest.java new file mode 100644 index 0000000000000..bdaddf9afb1da --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/PulsarClientBasedHandlerTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.protocol; + +import java.io.File; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.util.PortManager; +import org.apache.commons.io.FileUtils; +import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +public class PulsarClientBasedHandlerTest { + + private final static String clusterName = "cluster"; + private final static int shutdownTimeoutMs = 100; + private final int zkPort = PortManager.nextFreePort(); + private final LocalBookkeeperEnsemble bk = new LocalBookkeeperEnsemble(2, zkPort, PortManager::nextFreePort); + private File tempDirectory; + private PulsarService pulsar; + + @BeforeClass + public void setup() throws Exception { + bk.start(); + final var config = new ServiceConfiguration(); + config.setClusterName(clusterName); + config.setAdvertisedAddress("localhost"); + config.setBrokerServicePort(Optional.of(0)); + config.setWebServicePort(Optional.of(0)); + config.setMetadataStoreUrl("zk:127.0.0.1:" + zkPort); + + tempDirectory = SimpleProtocolHandlerTestsBase.configureProtocolHandler(config, + PulsarClientBasedHandler.class.getName(), true); + + config.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); + config.setLoadBalancerDebugModeEnabled(true); + config.setBrokerShutdownTimeoutMs(shutdownTimeoutMs); + + pulsar = new PulsarService(config); + pulsar.start(); + } + + @Test(timeOut = 30000) + public void testStopBroker() throws PulsarServerException { + final var beforeStop = System.currentTimeMillis(); + final var handler = (PulsarClientBasedHandler) pulsar.getProtocolHandlers() + .protocol(PulsarClientBasedHandler.PROTOCOL); + pulsar.close(); + final var elapsedMs = System.currentTimeMillis() - beforeStop; + log.info("It spends {} ms to stop the broker ({} for protocol handler)", elapsedMs, handler.closeTimeMs); + Assert.assertTrue(elapsedMs < + + handler.closeTimeMs + shutdownTimeoutMs + 1000); // tolerate 1 more second for other processes + } + + @AfterClass(alwaysRun = true) + public void cleanup() throws Exception { + bk.stop(); + if (tempDirectory != null) { + FileUtils.deleteDirectory(tempDirectory); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/SimpleProtocolHandlerTestsBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/SimpleProtocolHandlerTestsBase.java index c894b7d77c43a..6c80f220c3ddb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/SimpleProtocolHandlerTestsBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/SimpleProtocolHandlerTestsBase.java @@ -127,12 +127,18 @@ public SimpleProtocolHandlerTestsBase(boolean useSeparateThreadPool) { @BeforeClass @Override protected void setup() throws Exception { - tempDirectory = Files.createTempDirectory("SimpleProtocolHandlerTest").toFile(); + tempDirectory = configureProtocolHandler(conf, MyProtocolHandler.class.getName(), useSeparateThreadPool); + super.baseSetup(); + } + + static File configureProtocolHandler(ServiceConfiguration conf, String className, boolean useSeparateThreadPool) + throws Exception { + final var tempDirectory = Files.createTempDirectory("SimpleProtocolHandlerTest").toFile(); conf.setUseSeparateThreadPoolForProtocolHandlers(useSeparateThreadPool); conf.setProtocolHandlerDirectory(tempDirectory.getAbsolutePath()); conf.setMessagingProtocols(Collections.singleton("test")); - buildMockNarFile(tempDirectory); - super.baseSetup(); + buildMockNarFile(tempDirectory, className); + return tempDirectory; } @Test @@ -163,7 +169,7 @@ protected void cleanup() throws Exception { } } - private static void buildMockNarFile(File tempDirectory) throws Exception { + private static void buildMockNarFile(File tempDirectory, String className) throws Exception { File file = new File(tempDirectory, "temp.nar"); try (ZipOutputStream zipfile = new ZipOutputStream(new FileOutputStream(file))) { @@ -176,7 +182,7 @@ private static void buildMockNarFile(File tempDirectory) throws Exception { zipfile.putNextEntry(manifest); String yaml = "name: test\n" + "description: this is a test\n" + - "handlerClass: " + MyProtocolHandler.class.getName() + "\n"; + "handlerClass: " + className + "\n"; zipfile.write(yaml.getBytes(StandardCharsets.UTF_8)); zipfile.closeEntry(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/qos/AsyncTokenBucketTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/qos/AsyncTokenBucketTest.java new file mode 100644 index 0000000000000..b446f9e902f2a --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/qos/AsyncTokenBucketTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.qos; + +import static org.testng.Assert.assertEquals; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + + +public class AsyncTokenBucketTest { + private AtomicLong manualClockSource; + private MonotonicSnapshotClock clockSource; + + private AsyncTokenBucket asyncTokenBucket; + + @BeforeMethod + public void setup() { + manualClockSource = new AtomicLong(TimeUnit.SECONDS.toNanos(100)); + clockSource = requestSnapshot -> manualClockSource.get(); + } + + + private void incrementSeconds(int seconds) { + manualClockSource.addAndGet(TimeUnit.SECONDS.toNanos(seconds)); + } + + private void incrementMillis(long millis) { + manualClockSource.addAndGet(TimeUnit.MILLISECONDS.toNanos(millis)); + } + + @Test + void shouldAddTokensWithConfiguredRate() { + asyncTokenBucket = + AsyncTokenBucket.builder().capacity(100).rate(10).initialTokens(0).clock(clockSource).build(); + incrementSeconds(5); + assertEquals(asyncTokenBucket.getTokens(), 50); + incrementSeconds(1); + assertEquals(asyncTokenBucket.getTokens(), 60); + incrementSeconds(4); + assertEquals(asyncTokenBucket.getTokens(), 100); + + // No matter how long the period is, tokens do not go above capacity + incrementSeconds(5); + assertEquals(asyncTokenBucket.getTokens(), 100); + + // Consume all and verify none available and then wait 1 period and check replenished + asyncTokenBucket.consumeTokens(100); + assertEquals(asyncTokenBucket.tokens(true), 0); + incrementSeconds(1); + assertEquals(asyncTokenBucket.getTokens(), 10); + } + + @Test + void shouldCalculatePauseCorrectly() { + asyncTokenBucket = + AsyncTokenBucket.builder().capacity(100).rate(10).initialTokens(0).clock(clockSource) + .build(); + incrementSeconds(5); + asyncTokenBucket.consumeTokens(100); + assertEquals(asyncTokenBucket.getTokens(), -50); + assertEquals(TimeUnit.NANOSECONDS.toMillis(asyncTokenBucket.calculateThrottlingDuration()), 5100); + } + + @Test + void shouldSupportFractionsWhenUpdatingTokens() { + asyncTokenBucket = + AsyncTokenBucket.builder().capacity(100).rate(10).initialTokens(0).clock(clockSource).build(); + incrementMillis(100); + assertEquals(asyncTokenBucket.getTokens(), 1); + } + + @Test + void shouldSupportFractionsAndRetainLeftoverWhenUpdatingTokens() { + asyncTokenBucket = + AsyncTokenBucket.builder().capacity(100).rate(10).initialTokens(0).clock(clockSource).build(); + for (int i = 0; i < 150; i++) { + incrementMillis(1); + } + assertEquals(asyncTokenBucket.getTokens(), 1); + incrementMillis(150); + assertEquals(asyncTokenBucket.getTokens(), 3); + } + +} \ No newline at end of file diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/RGUsageMTAggrWaitForAllMsgsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/RGUsageMTAggrWaitForAllMsgsTest.java index 9bf7e3c5325d9..392ec0d3ff46f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/RGUsageMTAggrWaitForAllMsgsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/RGUsageMTAggrWaitForAllMsgsTest.java @@ -20,6 +20,10 @@ import com.google.common.collect.Sets; import io.prometheus.client.Summary; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Map; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.resourcegroup.ResourceGroup.BytesAndMessagesCount; import org.apache.pulsar.broker.resourcegroup.ResourceGroup.ResourceGroupMonitoringClass; @@ -45,11 +49,6 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Map; -import java.util.concurrent.TimeUnit; - // The tests implement a set of producer/consumer operations on a set of topics. // [A thread is started for each producer, and each consumer in the test.] @@ -57,6 +56,7 @@ // After sending/receiving all the messages, traffic usage statistics, and Prometheus-metrics // are verified on the RGs. @Slf4j +@Test(groups = "flaky") public class RGUsageMTAggrWaitForAllMsgsTest extends ProducerConsumerBase { @BeforeClass @Override @@ -119,9 +119,9 @@ private class ProduceMessages implements Runnable { private final int numMesgsToProduce; private final String myProduceTopic; - private int sentNumBytes = 0; - private int sentNumMsgs = 0; - private int numExceptions = 0; + private volatile int sentNumBytes = 0; + private volatile int sentNumMsgs = 0; + private volatile int numExceptions = 0; ProduceMessages(int prodId, int nMesgs, String[] topics) { producerId = prodId; @@ -202,9 +202,9 @@ private class ConsumeMessages implements Runnable { private final int recvTimeoutMilliSecs = 1000; private final int ackTimeoutMilliSecs = 1100; // has to be more than 1 second - private int recvdNumBytes = 0; - private int recvdNumMsgs = 0; - private int numExceptions = 0; + private volatile int recvdNumBytes = 0; + private volatile int recvdNumMsgs = 0; + private volatile int numExceptions = 0; private volatile boolean allMessagesReceived = false; private volatile boolean consumerIsReady = false; @@ -494,15 +494,15 @@ private void testProduceConsumeUsageOnRG(String[] topicStrings) throws Exception while (numConsumersDone < NUM_CONSUMERS) { for (int ix = 0; ix < NUM_CONSUMERS; ix++) { if (!joinedConsumers[ix]) { + consThr[ix].thread.join(); + joinedConsumers[ix] = true; + log.debug("Joined consumer={}", ix); + recvdBytes = consThr[ix].consumer.getNumBytesRecvd(); recvdMsgs = consThr[ix].consumer.getNumMessagesRecvd(); numConsumerExceptions += consThr[ix].consumer.getNumExceptions(); log.debug("Consumer={} received {} mesgs and {} bytes", ix, recvdMsgs, recvdBytes); - consThr[ix].thread.join(); - joinedConsumers[ix] = true; - log.debug("Joined consumer={}", ix); - recvdNumBytes += recvdBytes; recvdNumMsgs += recvdMsgs; numConsumersDone++; @@ -682,17 +682,11 @@ private void verifyRGMetrics(int sentNumBytes, int sentNumMsgs, for (ResourceGroupMonitoringClass mc : ResourceGroupMonitoringClass.values()) { String mcName = mc.name(); int mcIndex = mc.ordinal(); - double quotaBytes = ResourceGroupService.getRgQuotaByteCount(rgName, mcName); - totalQuotaBytes[mcIndex] += quotaBytes; - double quotaMesgs = ResourceGroupService.getRgQuotaMessageCount(rgName, mcName); - totalQuotaMessages[mcIndex] += quotaMesgs; - double usedBytes = ResourceGroupService.getRgLocalUsageByteCount(rgName, mcName); - totalUsedBytes[mcIndex] += usedBytes; - double usedMesgs = ResourceGroupService.getRgLocalUsageMessageCount(rgName, mcName); - totalUsedMessages[mcIndex] += usedMesgs; - - double usageReportedCount = ResourceGroup.getRgUsageReportedCount(rgName, mcName); - totalUsageReportCounts[mcIndex] += usageReportedCount; + totalQuotaBytes[mcIndex] += ResourceGroupService.getRgQuotaByteCount(rgName, mcName); + totalQuotaMessages[mcIndex] += ResourceGroupService.getRgQuotaMessageCount(rgName, mcName); + totalUsedBytes[mcIndex] += ResourceGroupService.getRgLocalUsageByteCount(rgName, mcName); + totalUsedMessages[mcIndex] += ResourceGroupService.getRgLocalUsageMessageCount(rgName, mcName); + totalUsageReportCounts[mcIndex] += ResourceGroup.getRgUsageReportedCount(rgName, mcName); } totalTenantRegisters += ResourceGroupService.getRgTenantRegistersCount(rgName); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupConfigListenerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupConfigListenerTest.java index 90c26530850a3..4010635ed9952 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupConfigListenerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupConfigListenerTest.java @@ -18,20 +18,31 @@ */ package org.apache.pulsar.broker.resourcegroup; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertThrows; import com.google.common.collect.Sets; +import java.util.ArrayList; import java.util.Random; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.broker.resources.ResourceGroupResources; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.ResourceGroup; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.metadata.api.MetadataStore; import org.awaitility.Awaitility; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -288,4 +299,41 @@ private void prepareData() throws PulsarAdminException { testAddRg.setDispatchRateInBytes(200L); } + + @Test + public void testNewResourceGroupNamespaceConfigListener() { + PulsarService pulsarService = mock(PulsarService.class); + PulsarResources pulsarResources = mock(PulsarResources.class); + doReturn(pulsarResources).when(pulsarService).getPulsarResources(); + ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + doReturn(scheduledExecutorService).when(pulsarService).getExecutor(); + + ResourceGroupService resourceGroupService = mock(ResourceGroupService.class); + ResourceGroupResources resourceGroupResources = mock(ResourceGroupResources.class); + RuntimeException exception = new RuntimeException("listResourceGroupsAsync error"); + doReturn(CompletableFuture.failedFuture(exception)) + .when(resourceGroupResources).listResourceGroupsAsync(); + doReturn(mock(MetadataStore.class)) + .when(resourceGroupResources).getStore(); + doReturn(resourceGroupResources).when(pulsarResources).getResourcegroupResources(); + + ServiceConfiguration ServiceConfiguration = new ServiceConfiguration(); + doReturn(ServiceConfiguration).when(pulsarService).getConfiguration(); + + ResourceGroupConfigListener resourceGroupConfigListener = + new ResourceGroupConfigListener(resourceGroupService, pulsarService); + + // getResourcegroupResources() returns an error, ResourceGroupNamespaceConfigListener doesn't be created. + Awaitility.await().pollDelay(3, TimeUnit.SECONDS).untilAsserted(() -> { + assertNull(resourceGroupConfigListener.getRgNamespaceConfigListener()); + }); + + // ResourceGroupNamespaceConfigListener will be created, and uses real pulsar resource. + doReturn(CompletableFuture.completedFuture(new ArrayList())) + .when(resourceGroupResources).listResourceGroupsAsync(); + doReturn(pulsar.getPulsarResources()).when(pulsarService).getPulsarResources(); + Awaitility.await().untilAsserted(() -> { + assertNotNull(resourceGroupConfigListener.getRgNamespaceConfigListener()); + }); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterTest.java index fed827b1517e6..3e88af9ec0be7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterTest.java @@ -21,7 +21,9 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; - +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.MessageId; @@ -34,12 +36,9 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - public class ResourceGroupRateLimiterTest extends BrokerTestBase { + private static final long MESSAGE_SIZE_SERIALIZED = 41L; final String rgName = "testRG"; org.apache.pulsar.common.policies.data.ResourceGroup testAddRg = new org.apache.pulsar.common.policies.data.ResourceGroup(); @@ -53,7 +52,6 @@ protected void setup() throws Exception { conf.setMaxPendingPublishRequestsPerConnection(0); super.baseSetup(); prepareData(); - } @AfterClass(alwaysRun = true) @@ -76,7 +74,7 @@ public void createResourceGroup(String rgName, org.apache.pulsar.common.policies public void deleteResourceGroup(String rgName) throws PulsarAdminException { admin.resourcegroups().deleteResourceGroup(rgName); - Awaitility.await().atMost(1, TimeUnit.SECONDS) + Awaitility.await().atMost(10, TimeUnit.SECONDS) .untilAsserted(() -> assertNull(pulsar.getResourceGroupServiceManager().resourceGroupGet(rgName))); } @@ -123,7 +121,7 @@ private void testRateLimit() throws PulsarAdminException, PulsarClientException, try { // third one should succeed - messageId = producer.sendAsync(new byte[MESSAGE_SIZE]).get(100, TimeUnit.MILLISECONDS); + messageId = producer.sendAsync(new byte[MESSAGE_SIZE]).get(300, TimeUnit.MILLISECONDS); Assert.assertNotNull(messageId); } catch (TimeoutException e) { Assert.fail("should not fail"); @@ -133,6 +131,8 @@ private void testRateLimit() throws PulsarAdminException, PulsarClientException, admin.namespaces().removeNamespaceResourceGroup(namespaceName); deleteResourceGroup(rgName); + Thread.sleep(2000); + // No rate limits should be applied. for (int i = 0; i < 5; i++) { messageId = producer.sendAsync(new byte[MESSAGE_SIZE]).get(100, TimeUnit.MILLISECONDS); @@ -148,7 +148,7 @@ public void testResourceGroupPublishRateLimit() throws Exception { } private void prepareData() { - testAddRg.setPublishRateInBytes(Long.valueOf(MESSAGE_SIZE)); + testAddRg.setPublishRateInBytes(Long.valueOf(MESSAGE_SIZE_SERIALIZED)); testAddRg.setPublishRateInMsgs(1); testAddRg.setDispatchRateInMsgs(-1); testAddRg.setDispatchRateInBytes(Long.valueOf(-1)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupReportLocalUsageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupReportLocalUsageTest.java new file mode 100644 index 0000000000000..139d19886c7d1 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupReportLocalUsageTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.resourcegroup; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.resourcegroup.ResourceGroup.BytesAndMessagesCount; +import org.apache.pulsar.broker.resourcegroup.ResourceGroup.PerMonitoringClassFields; +import org.apache.pulsar.broker.resourcegroup.ResourceGroup.ResourceGroupMonitoringClass; +import org.apache.pulsar.broker.service.resource.usage.ResourceUsage; +import org.apache.pulsar.common.policies.data.ResourceGroup; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +public class ResourceGroupReportLocalUsageTest extends MockedPulsarServiceBaseTest { + + @BeforeClass + @Override + protected void setup() throws Exception { + super.internalSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + + @Test + public void testRgFillResourceUsage() throws Exception { + pulsar.getResourceGroupServiceManager().close(); + AtomicBoolean needReport = new AtomicBoolean(false); + ResourceGroupService service = new ResourceGroupService(pulsar, TimeUnit.HOURS, null, + new ResourceQuotaCalculator() { + @Override + public boolean needToReportLocalUsage(long currentBytesUsed, long lastReportedBytes, + long currentMessagesUsed, long lastReportedMessages, + long lastReportTimeMSecsSinceEpoch) { + return needReport.get(); + } + + @Override + public long computeLocalQuota(long confUsage, long myUsage, long[] allUsages) { + return 0; + } + }); + String rgName = "rg-1"; + ResourceGroup rgConfig = new ResourceGroup(); + rgConfig.setPublishRateInBytes(1000L); + rgConfig.setPublishRateInMsgs(2000); + service.resourceGroupCreate(rgName, rgConfig); + + BytesAndMessagesCount bytesAndMessagesCount = new BytesAndMessagesCount(); + bytesAndMessagesCount.bytes = 20; + bytesAndMessagesCount.messages = 10; + + org.apache.pulsar.broker.resourcegroup.ResourceGroup resourceGroup = service.resourceGroupGet(rgName); + for (ResourceGroupMonitoringClass value : ResourceGroupMonitoringClass.values()) { + resourceGroup.incrementLocalUsageStats(value, bytesAndMessagesCount); + } + + // Case1: Suppress report ResourceUsage. + needReport.set(false); + ResourceUsage resourceUsage = new ResourceUsage(); + resourceGroup.rgFillResourceUsage(resourceUsage); + assertFalse(resourceUsage.hasDispatch()); + assertFalse(resourceUsage.hasPublish()); + for (ResourceGroupMonitoringClass value : ResourceGroupMonitoringClass.values()) { + PerMonitoringClassFields monitoredEntity = + resourceGroup.getMonitoredEntity(value); + assertEquals(monitoredEntity.usedLocallySinceLastReport.messages, 0); + assertEquals(monitoredEntity.usedLocallySinceLastReport.bytes, 0); + assertEquals(monitoredEntity.totalUsedLocally.messages, 0); + assertEquals(monitoredEntity.totalUsedLocally.bytes, 0); + assertEquals(monitoredEntity.lastReportedValues.messages, 0); + assertEquals(monitoredEntity.lastReportedValues.bytes, 0); + } + + // Case2: Report ResourceUsage. + for (ResourceGroupMonitoringClass value : ResourceGroupMonitoringClass.values()) { + resourceGroup.incrementLocalUsageStats(value, bytesAndMessagesCount); + } + needReport.set(true); + resourceUsage = new ResourceUsage(); + resourceGroup.rgFillResourceUsage(resourceUsage); + assertTrue(resourceUsage.hasDispatch()); + assertTrue(resourceUsage.hasPublish()); + for (ResourceGroupMonitoringClass value : ResourceGroupMonitoringClass.values()) { + PerMonitoringClassFields monitoredEntity = + resourceGroup.getMonitoredEntity(value); + assertEquals(monitoredEntity.usedLocallySinceLastReport.messages, 0); + assertEquals(monitoredEntity.usedLocallySinceLastReport.bytes, 0); + assertEquals(monitoredEntity.totalUsedLocally.messages, bytesAndMessagesCount.messages); + assertEquals(monitoredEntity.totalUsedLocally.bytes, bytesAndMessagesCount.bytes); + assertEquals(monitoredEntity.lastReportedValues.messages, bytesAndMessagesCount.messages); + assertEquals(monitoredEntity.lastReportedValues.bytes, bytesAndMessagesCount.bytes); + } + } +} \ No newline at end of file diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractBaseDispatcherTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractBaseDispatcherTest.java index 332cccc2d2c6a..6866e09731301 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractBaseDispatcherTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractBaseDispatcherTest.java @@ -33,10 +33,11 @@ import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.EntryImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -117,7 +118,7 @@ public void testFilterEntriesForConsumerOfEntryFilter() throws Exception { int size = this.helper.filterEntriesForConsumer(entries, batchSizes, sendMessageInfo, null, cursor, false, null); assertEquals(size, 0); - verify(subscriptionDispatchRateLimiter).tryDispatchPermit(1, expectedBytePermits); + verify(subscriptionDispatchRateLimiter).consumeDispatchQuota(1, expectedBytePermits); } @Test @@ -279,7 +280,8 @@ public boolean canUnsubscribe(Consumer consumer) { } @Override - public CompletableFuture close() { + public CompletableFuture close(boolean disconnectConsumers, + Optional assignedBrokerLookupData) { return null; } @@ -294,7 +296,8 @@ public CompletableFuture disconnectActiveConsumers(boolean isResetCursor) } @Override - public CompletableFuture disconnectAllConsumers(boolean isResetCursor) { + public CompletableFuture disconnectAllConsumers(boolean isResetCursor, + Optional assignedBrokerLookupData) { return null; } @@ -314,7 +317,7 @@ public void redeliverUnacknowledgedMessages(Consumer consumer, long consumerEpoc } @Override - public void redeliverUnacknowledgedMessages(Consumer consumer, List positions) { + public void redeliverUnacknowledgedMessages(Consumer consumer, List positions) { } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractReplicatorTest.java new file mode 100644 index 0000000000000..374296e68671d --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractReplicatorTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyBoolean; +import static org.mockito.Mockito.anyInt; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import io.netty.channel.DefaultEventLoop; +import io.netty.util.internal.DefaultPriorityQueue; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.Cleanup; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.ConnectionPool; +import org.apache.pulsar.client.impl.ProducerBuilderImpl; +import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; +import org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl; +import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; +import org.testng.Assert; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class AbstractReplicatorTest { + + @Test + public void testRetryStartProducerStoppedByTopicRemove() throws Exception { + final String localCluster = "localCluster"; + final String remoteCluster = "remoteCluster"; + final String topicName = "remoteTopicName"; + final String replicatorPrefix = "pulsar.repl"; + @Cleanup("shutdownNow") + final DefaultEventLoop eventLoopGroup = new DefaultEventLoop(); + // Mock services. + final ServiceConfiguration pulsarConfig = mock(ServiceConfiguration.class); + final PulsarService pulsar = mock(PulsarService.class); + final BrokerService broker = mock(BrokerService.class); + final Topic localTopic = mock(Topic.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + final PulsarClientImpl localClient = mock(PulsarClientImpl.class); + when(localClient.getCnxPool()).thenReturn(connectionPool); + final PulsarClientImpl remoteClient = mock(PulsarClientImpl.class); + when(remoteClient.getCnxPool()).thenReturn(connectionPool); + final ProducerConfigurationData producerConf = new ProducerConfigurationData(); + final ProducerBuilderImpl producerBuilder = mock(ProducerBuilderImpl.class); + final var topics = new ConcurrentHashMap>>(); + when(broker.executor()).thenReturn(eventLoopGroup); + when(broker.getTopics()).thenReturn(topics); + when(remoteClient.newProducer(any(Schema.class))).thenReturn(producerBuilder); + when(broker.pulsar()).thenReturn(pulsar); + when(pulsar.getClient()).thenReturn(localClient); + when(pulsar.getConfiguration()).thenReturn(pulsarConfig); + when(pulsarConfig.getReplicationProducerQueueSize()).thenReturn(100); + when(localTopic.getName()).thenReturn(topicName); + when(producerBuilder.topic(any())).thenReturn(producerBuilder); + when(producerBuilder.messageRoutingMode(any())).thenReturn(producerBuilder); + when(producerBuilder.enableBatching(anyBoolean())).thenReturn(producerBuilder); + when(producerBuilder.sendTimeout(anyInt(), any())).thenReturn(producerBuilder); + when(producerBuilder.maxPendingMessages(anyInt())).thenReturn(producerBuilder); + when(producerBuilder.producerName(anyString())).thenReturn(producerBuilder); + when(producerBuilder.getConf()).thenReturn(producerConf); + // Mock create producer fail. + when(producerBuilder.create()).thenThrow(new RuntimeException("mocked ex")); + when(producerBuilder.createAsync()) + .thenReturn(CompletableFuture.failedFuture(new RuntimeException("mocked ex"))); + // Make race condition: "retry start producer" and "close replicator". + final ReplicatorInTest replicator = new ReplicatorInTest(localCluster, localTopic, remoteCluster, topicName, + replicatorPrefix, broker, remoteClient); + replicator.startProducer(); + replicator.terminate(); + + // Verify task will done. + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + AtomicInteger taskCounter = new AtomicInteger(); + CountDownLatch checkTaskFinished = new CountDownLatch(1); + eventLoopGroup.execute(() -> { + synchronized (replicator) { + LinkedBlockingQueue taskQueue = WhiteboxImpl.getInternalState(eventLoopGroup, "taskQueue"); + DefaultPriorityQueue scheduledTaskQueue = + WhiteboxImpl.getInternalState(eventLoopGroup, "scheduledTaskQueue"); + taskCounter.set(taskQueue.size() + scheduledTaskQueue.size()); + checkTaskFinished.countDown(); + } + }); + checkTaskFinished.await(); + Assert.assertEquals(taskCounter.get(), 0); + }); + } + + private static class ReplicatorInTest extends AbstractReplicator { + + public ReplicatorInTest(String localCluster, Topic localTopic, String remoteCluster, String remoteTopicName, + String replicatorPrefix, BrokerService brokerService, + PulsarClientImpl replicationClient) throws PulsarServerException { + super(localCluster, localTopic, remoteCluster, remoteTopicName, replicatorPrefix, brokerService, + replicationClient); + } + + @Override + protected String getProducerName() { + return "pulsar.repl.producer"; + } + + @Override + protected void setProducerAndTriggerReadEntries(Producer producer) { + + } + + @Override + protected Position getReplicatorReadPosition() { + return PositionFactory.EARLIEST; + } + + @Override + public ReplicatorStatsImpl computeStats() { + return null; + } + + @Override + public ReplicatorStatsImpl getStats() { + return null; + } + + @Override + public void updateRates() { + + } + + @Override + public boolean isConnected() { + return false; + } + + @Override + public long getNumberOfEntriesInBacklog() { + return 0; + } + + @Override + protected void disableReplicatorRead() { + + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java index 7f5d3e0edf178..337717ed97b1b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractTopicTest.java @@ -19,13 +19,15 @@ package org.apache.pulsar.broker.service; import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.mockito.Mockito.withSettings; import static org.testng.Assert.assertEquals; +import java.util.concurrent.ConcurrentHashMap; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.apache.pulsar.broker.qos.AsyncTokenBucket; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -43,18 +45,16 @@ public void beforeMethod() { subscription = mock(AbstractSubscription.class); when(brokerService.pulsar()).thenReturn(pulsarService); + doReturn(pulsarService).when(brokerService).getPulsar(); when(pulsarService.getConfiguration()).thenReturn(serviceConfiguration); when(brokerService.getBacklogQuotaManager()).thenReturn(backlogQuotaManager); + doReturn(AsyncTokenBucket.DEFAULT_SNAPSHOT_CLOCK).when(pulsarService).getMonotonicSnapshotClock(); topic = mock(AbstractTopic.class, withSettings() .useConstructor("topic", brokerService) .defaultAnswer(CALLS_REAL_METHODS)); - ConcurrentOpenHashMap subscriptions = - ConcurrentOpenHashMap.newBuilder() - .expectedItems(16) - .concurrencyLevel(1) - .build(); + final var subscriptions = new ConcurrentHashMap(); subscriptions.put("subscription", subscription); when(topic.getSubscriptions()).thenAnswer(invocation -> subscriptions); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AdvertisedAddressTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AdvertisedAddressTest.java index 554c663850fd9..a60d6599e8f76 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AdvertisedAddressTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AdvertisedAddressTest.java @@ -66,8 +66,14 @@ public void setup() throws Exception { @AfterMethod(alwaysRun = true) public void shutdown() throws Exception { - pulsar.close(); - bkEnsemble.stop(); + if (pulsar != null) { + pulsar.close(); + pulsar = null; + } + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; + } } @Test @@ -75,7 +81,7 @@ public void testAdvertisedAddress() throws Exception { Assert.assertEquals( pulsar.getAdvertisedAddress(), advertisedAddress ); Assert.assertEquals( pulsar.getBrokerServiceUrl(), String.format("pulsar://%s:%d", advertisedAddress, pulsar.getBrokerListenPort().get()) ); Assert.assertEquals( pulsar.getSafeWebServiceAddress(), String.format("http://%s:%d", advertisedAddress, pulsar.getListenPortHTTP().get()) ); - String brokerZkPath = String.format("/loadbalance/brokers/%s:%d", pulsar.getAdvertisedAddress(), pulsar.getListenPortHTTP().get()); + String brokerZkPath = String.format("/loadbalance/brokers/%s", pulsar.getBrokerId()); String bkBrokerData = new String(bkEnsemble.getZkClient().getData(brokerZkPath, false, new Stat()), StandardCharsets.UTF_8); JsonObject jsonBkBrokerData = new Gson().fromJson(bkBrokerData, JsonObject.class); Assert.assertEquals( jsonBkBrokerData.get("pulsarServiceUrl").getAsString(), pulsar.getBrokerServiceUrl() ); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java index f3463ee121d75..56f9f4f91246e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java @@ -18,11 +18,23 @@ */ package org.apache.pulsar.broker.service; +import static java.util.Map.entry; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongGaugeValue; +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongSumValue; +import static org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType.destination_storage; +import static org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType.message_age; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.AssertionsForClassTypes.within; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.google.common.collect.Sets; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; import java.net.URL; import java.time.Duration; import java.util.ArrayList; @@ -33,15 +45,21 @@ import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import lombok.Cleanup; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil; +import org.apache.pulsar.broker.stats.OpenTelemetryTopicStats; +import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient; +import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.Metric; +import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.Metrics; import org.apache.pulsar.client.admin.GetStatsOptions; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -60,6 +78,8 @@ import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.common.policies.data.impl.BacklogQuotaImpl; +import org.apache.pulsar.functions.worker.WorkerConfig; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.awaitility.Awaitility; import org.slf4j.Logger; @@ -73,6 +93,9 @@ @Test(groups = "broker") public class BacklogQuotaManagerTest { + private static final Logger log = LoggerFactory.getLogger(BacklogQuotaManagerTest.class); + + public static final String CLUSTER_NAME = "usc"; PulsarService pulsar; ServiceConfiguration config; @@ -80,6 +103,8 @@ public class BacklogQuotaManagerTest { PulsarAdmin admin; LocalBookkeeperEnsemble bkEnsemble; + PrometheusMetricsClient prometheusMetricsClient; + InMemoryMetricReader openTelemetryMetricReader; private static final int TIME_TO_CHECK_BACKLOG_QUOTA = 2; private static final int MAX_ENTRIES_PER_LEDGER = 5; @@ -117,7 +142,7 @@ void setup() throws Exception { config.setMetadataStoreUrl("zk:127.0.0.1:" + bkEnsemble.getZookeeperPort()); config.setAdvertisedAddress("localhost"); config.setWebServicePort(Optional.of(0)); - config.setClusterName("usc"); + config.setClusterName(CLUSTER_NAME); config.setBrokerShutdownTimeoutMs(0L); config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); config.setBrokerServicePort(Optional.of(0)); @@ -127,15 +152,18 @@ void setup() throws Exception { config.setManagedLedgerMaxEntriesPerLedger(MAX_ENTRIES_PER_LEDGER); config.setManagedLedgerMinLedgerRolloverTimeMinutes(0); config.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED); - config.setSystemTopicEnabled(false); - config.setTopicLevelPoliciesEnabled(false); + config.setSystemTopicEnabled(true); + config.setTopicLevelPoliciesEnabled(true); config.setForceDeleteNamespaceAllowed(true); - pulsar = new PulsarService(config); + openTelemetryMetricReader = InMemoryMetricReader.create(); + pulsar = new PulsarService(config, new WorkerConfig(), Optional.empty(), exitCode -> { + }, BrokerOpenTelemetryTestUtil.getOpenTelemetrySdkBuilderConsumer(openTelemetryMetricReader)); pulsar.start(); adminUrl = new URL("http://127.0.0.1" + ":" + pulsar.getListenPortHTTP().get()); admin = PulsarAdmin.builder().serviceHttpUrl(adminUrl.toString()).build(); + prometheusMetricsClient = new PrometheusMetricsClient("127.0.0.1", pulsar.getListenPortHTTP().get()); admin.clusters().createCluster("usc", ClusterData.builder().serviceUrl(adminUrl.toString()).build()); admin.tenants().createTenant("prop", @@ -190,7 +218,7 @@ private void rolloverStats() { } /** - * Readers should not effect backlog quota + * Readers should not affect backlog quota */ @Test public void testBacklogQuotaWithReader() throws Exception { @@ -202,18 +230,18 @@ public void testBacklogQuotaWithReader() throws Exception { .limitTime(TIME_TO_CHECK_BACKLOG_QUOTA) .retentionPolicy(BacklogQuota.RetentionPolicy.producer_exception) .build()); - try (PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();) { + try (PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, SECONDS).build()) { final String topic1 = "persistent://prop/ns-quota/topic1" + UUID.randomUUID(); final int numMsgs = 20; Reader reader = client.newReader().topic(topic1).receiverQueueSize(1).startMessageId(MessageId.latest).create(); - org.apache.pulsar.client.api.Producer producer = createProducer(client, topic1); + Producer producer = createProducer(client, topic1); byte[] content = new byte[1024]; for (int i = 0; i < numMsgs; i++) { content[0] = (byte) (content[0] + 1); - MessageId msgId = producer.send(content); + producer.send(content); } Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000); @@ -228,7 +256,7 @@ public void testBacklogQuotaWithReader() throws Exception { // non-durable mes should still assertEquals(stats.getSubscriptions().size(), 1); long nonDurableSubscriptionBacklog = stats.getSubscriptions().values().iterator().next().getMsgBacklog(); - assertEquals(nonDurableSubscriptionBacklog, MAX_ENTRIES_PER_LEDGER, + assertEquals(nonDurableSubscriptionBacklog, 0, "non-durable subscription backlog is [" + nonDurableSubscriptionBacklog + "]"); MessageIdImpl msgId = null; @@ -254,15 +282,12 @@ public void testBacklogQuotaWithReader() throws Exception { // check there is only one ledger left assertEquals(internalStats.ledgers.size(), 1); - - // check if its the expected ledger id given MAX_ENTRIES_PER_LEDGER - assertEquals(internalStats.ledgers.get(0).ledgerId, finalMsgId.getLedgerId()); }); // check reader can still read with out error while (true) { - Message msg = reader.readNext(5, TimeUnit.SECONDS); + Message msg = reader.readNext(5, SECONDS); if (msg == null) { break; } @@ -272,8 +297,12 @@ public void testBacklogQuotaWithReader() throws Exception { } private TopicStats getTopicStats(String topic1) throws PulsarAdminException { + return getTopicStats(topic1, true); + } + + private TopicStats getTopicStats(String topic1, boolean getPreciseBacklog) throws PulsarAdminException { TopicStats stats = - admin.topics().getStats(topic1, GetStatsOptions.builder().getPreciseBacklog(true).build()); + admin.topics().getStats(topic1, GetStatsOptions.builder().getPreciseBacklog(getPreciseBacklog).build()); return stats; } @@ -287,10 +316,11 @@ public void testTriggerBacklogQuotaSizeWithReader() throws Exception { .limitTime(TIME_TO_CHECK_BACKLOG_QUOTA) .retentionPolicy(BacklogQuota.RetentionPolicy.producer_exception) .build()); - try (PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();) { + try (PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, SECONDS).build();) { final String topic1 = "persistent://prop/ns-quota/topic1" + UUID.randomUUID(); final int numMsgs = 20; Reader reader = client.newReader().topic(topic1).receiverQueueSize(1).startMessageId(MessageId.latest).create(); + Producer producer = createProducer(client, topic1); byte[] content = new byte[1024]; for (int i = 0; i < numMsgs; i++) { @@ -303,10 +333,10 @@ public void testTriggerBacklogQuotaSizeWithReader() throws Exception { TopicStats stats = getTopicStats(topic1); // overall backlogSize should be zero because we only have readers assertEquals(stats.getBacklogSize(), 0, "backlog size is [" + stats.getBacklogSize() + "]"); - // non-durable mes should still assertEquals(stats.getSubscriptions().size(), 1); long nonDurableSubscriptionBacklog = stats.getSubscriptions().values().iterator().next().getMsgBacklog(); - assertEquals(nonDurableSubscriptionBacklog, MAX_ENTRIES_PER_LEDGER, + // All the full ledgers should be deleted. + assertEquals(nonDurableSubscriptionBacklog, 0, "non-durable subscription backlog is [" + nonDurableSubscriptionBacklog + "]"); MessageIdImpl messageId = null; try { @@ -327,13 +357,13 @@ public void testTriggerBacklogQuotaSizeWithReader() throws Exception { // check there is only one ledger left assertEquals(internalStats.ledgers.size(), 1); - // check if its the expected ledger id given MAX_ENTRIES_PER_LEDGER - assertEquals(internalStats.ledgers.get(0).ledgerId, finalMessageId.getLedgerId()); + // check if it's the expected ledger id given MAX_ENTRIES_PER_LEDGER + assertEquals(internalStats.ledgers.get(0).ledgerId, finalMessageId.getLedgerId() + 1); }); - // check reader can still read with out error + // check reader can still read without error while (true) { - Message msg = reader.readNext(5, TimeUnit.SECONDS); + Message msg = reader.readNext(5, SECONDS); if (msg == null) { break; } @@ -344,6 +374,455 @@ public void testTriggerBacklogQuotaSizeWithReader() throws Exception { } } + @Test + public void backlogsStatsPrecise() throws PulsarAdminException, PulsarClientException, InterruptedException { + config.setPreciseTimeBasedBacklogQuotaCheck(true); + final String namespace = "prop/ns-quota"; + assertEquals(admin.namespaces().getBacklogQuotaMap(namespace), new HashMap<>()); + final int sizeLimitBytes = 15 * 1024 * 1024; + final int timeLimitSeconds = 123; + admin.namespaces().setBacklogQuota( + namespace, + BacklogQuota.builder() + .limitSize(sizeLimitBytes) + .retentionPolicy(BacklogQuota.RetentionPolicy.producer_exception) + .build(), + destination_storage); + admin.namespaces().setBacklogQuota( + namespace, + BacklogQuota.builder() + .limitTime(timeLimitSeconds) + .retentionPolicy(BacklogQuota.RetentionPolicy.producer_exception) + .build(), + message_age); + + try (PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()) + .statsInterval(0, SECONDS).build()) { + final String topic1 = "persistent://prop/ns-quota/topic2" + UUID.randomUUID(); + + final String subName1 = "c1"; + final String subName2 = "c2"; + final int numMsgs = 4; + + Consumer consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1) + .acknowledgmentGroupTime(0, SECONDS) + .subscribe(); + Consumer consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2) + .acknowledgmentGroupTime(0, SECONDS) + .subscribe(); + Producer producer = createProducer(client, topic1); + + byte[] content = new byte[1024]; + for (int i = 0; i < numMsgs; i++) { + Thread.sleep(3000); // Guarantees if we use wrong message in age, to show up in failed test + producer.send(content); + } + + String c1MarkDeletePositionBefore = + admin.topics().getInternalStats(topic1).cursors.get(subName1).markDeletePosition; + + // Move subscription 1, one message, such that subscription 2 is the oldest + // S2 S1 + // 0 1 + Message oldestMessage = consumer1.receive(); + consumer1.acknowledge(oldestMessage); + + log.info("Subscription 1 moved 1 message. Now subscription 2 is the oldest. Oldest message:"+ + oldestMessage.getMessageId()); + + c1MarkDeletePositionBefore = waitForMarkDeletePositionToChange(topic1, subName1, c1MarkDeletePositionBefore); + waitForQuotaCheckToRunTwice(); + + Metrics metrics = prometheusMetricsClient.getMetrics(); + TopicStats topicStats = getTopicStats(topic1); + + assertThat(topicStats.getBacklogQuotaLimitSize()).isEqualTo(sizeLimitBytes); + assertThat(topicStats.getBacklogQuotaLimitTime()).isEqualTo(timeLimitSeconds); + long expectedMessageAgeSeconds = MILLISECONDS.toSeconds(System.currentTimeMillis() - oldestMessage.getPublishTime()); + assertThat(topicStats.getOldestBacklogMessageAgeSeconds()) + .isCloseTo(expectedMessageAgeSeconds, within(1L)); + assertThat(topicStats.getOldestBacklogMessageSubscriptionName()).isEqualTo(subName2); + + Metric backlogAgeMetric = + metrics.findSingleMetricByNameAndLabels("pulsar_storage_backlog_age_seconds", + Pair.of("topic", topic1)); + assertThat(backlogAgeMetric.tags).containsExactly( + entry("cluster", CLUSTER_NAME), + entry("namespace", namespace), + entry("topic", topic1)); + assertThat((long) backlogAgeMetric.value).isCloseTo(expectedMessageAgeSeconds, within(2L)); + + // Move subscription 2 away from being the oldest mark delete + // S2/S1 + // 0 1 + Message firstOldestMessage = consumer2.receive(); + consumer2.acknowledge(firstOldestMessage); + // We only read and not ack, since we just need its publish-timestamp for later assert + Message secondOldestMessage = consumer2.receive(); + + // Switch subscription 1 to be where subscription 2 was in terms of oldest mark delete + // S1 S2 + // 0 1 + consumer1.seek(MessageId.earliest); + + log.info("Subscription 1 moved to be the oldest"); + + c1MarkDeletePositionBefore = waitForMarkDeletePositionToChange(topic1, subName1, c1MarkDeletePositionBefore); + waitForQuotaCheckToRunTwice(); + + metrics = prometheusMetricsClient.getMetrics(); + long actualAge = (long) metrics.findByNameAndLabels( + "pulsar_storage_backlog_age_seconds", "topic", topic1) + .get(0).value; + + expectedMessageAgeSeconds = MILLISECONDS.toSeconds(System.currentTimeMillis() - oldestMessage.getPublishTime()); + assertThat(actualAge).isCloseTo(expectedMessageAgeSeconds, within(2L)); + + topicStats = getTopicStats(topic1); + assertThat(topicStats.getOldestBacklogMessageSubscriptionName()).isEqualTo(subName1); + + long entriesReadBefore = getReadEntries(topic1); + + // Move subscription 1 passed subscription 2 + for (int i = 0; i < 3; i++) { + Message message = consumer1.receive(); + log.info("Subscription 1 about to ack message ID {}", message.getMessageId()); + consumer1.acknowledge(message); + } + + log.info("Subscription 1 moved 3 messages. Now subscription 2 is the oldest"); + waitForMarkDeletePositionToChange(topic1, subName1, c1MarkDeletePositionBefore); + waitForQuotaCheckToRunTwice(); + + // Cache shouldn't be used, since position has changed + long readEntries = getReadEntries(topic1); + assertThat(readEntries).isGreaterThan(entriesReadBefore); + + topicStats = getTopicStats(topic1); + expectedMessageAgeSeconds = MILLISECONDS.toSeconds(System.currentTimeMillis() - secondOldestMessage.getPublishTime()); + assertThat(topicStats.getOldestBacklogMessageAgeSeconds()).isCloseTo(expectedMessageAgeSeconds, within(2L)); + assertThat(topicStats.getOldestBacklogMessageSubscriptionName()).isEqualTo(subName2); + + waitForQuotaCheckToRunTwice(); + + // Cache should be used, since position hasn't changed + assertThat(getReadEntries(topic1)).isEqualTo(readEntries); + + // Move subscription 1 and 2 to end + Message msg = consumer1.receive(); + consumer1.acknowledge(msg); + consumer2.acknowledge(secondOldestMessage); + for (int i = 0; i < 2; i++) { + Message message = consumer2.receive(); + log.info("Subscription 2 about to ack message ID {}", message.getMessageId()); + consumer2.acknowledge(message); + } + + log.info("Subscription 1 and 2 moved to end. Now should not backlog"); + waitForMarkDeletePositionToChange(topic1, subName1, c1MarkDeletePositionBefore); + waitForQuotaCheckToRunTwice(); + + topicStats = getTopicStats(topic1); + assertThat(topicStats.getBacklogSize()).isEqualTo(0); + assertThat(topicStats.getSubscriptions().get(subName1).getMsgBacklog()).isEqualTo(0); + assertThat(topicStats.getSubscriptions().get(subName2).getMsgBacklog()).isEqualTo(0); + assertThat(topicStats.getOldestBacklogMessageAgeSeconds()).isEqualTo(0); + assertThat(topicStats.getOldestBacklogMessageSubscriptionName()).isNull(); + + metrics = prometheusMetricsClient.getMetrics(); + backlogAgeMetric = + metrics.findSingleMetricByNameAndLabels("pulsar_storage_backlog_age_seconds", + Pair.of("topic", topic1)); + assertThat(backlogAgeMetric.tags).containsExactly( + entry("cluster", CLUSTER_NAME), + entry("namespace", namespace), + entry("topic", topic1)); + assertThat((long) backlogAgeMetric.value).isEqualTo(0); + + // producer should create success. + Producer producer2 = createProducer(client, topic1); + assertNotNull(producer2); + } + } + + @Test + public void backlogsStatsPreciseWithNoBacklog() throws PulsarAdminException, PulsarClientException, InterruptedException { + config.setPreciseTimeBasedBacklogQuotaCheck(true); + config.setExposePreciseBacklogInPrometheus(true); + final String namespace = "prop/ns-quota"; + assertEquals(admin.namespaces().getBacklogQuotaMap(namespace), new HashMap<>()); + final int timeLimitSeconds = 2; + admin.namespaces().setBacklogQuota( + namespace, + BacklogQuota.builder() + .limitTime(timeLimitSeconds) + .retentionPolicy(BacklogQuota.RetentionPolicy.producer_exception) + .build(), + message_age); + + try (PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()) + .maxBackoffInterval(5, SECONDS) + .statsInterval(0, SECONDS).build()) { + final String topic1 = "persistent://prop/ns-quota/topic2" + UUID.randomUUID(); + + final String subName1 = "c1"; + final int numMsgs = 4; + + Consumer consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1) + .acknowledgmentGroupTime(0, SECONDS) + .subscribe(); + Producer producer = createProducer(client, topic1); + + byte[] content = new byte[1024]; + for (int i = 0; i < numMsgs; i++) { + MessageId send = producer.send(content); + System.out.println(i + ":msg:" + MILLISECONDS.toSeconds(System.currentTimeMillis())); + } + + String c1MarkDeletePositionBefore = + admin.topics().getInternalStats(topic1).cursors.get(subName1).markDeletePosition; + + // Move subscription 1 to end + for (int i = 0; i < numMsgs; i++) { + Message message1 = consumer1.receive(); + consumer1.acknowledge(message1); + } + + // This code will wait about 4~5 Seconds, to make sure the oldest message is 4~5 seconds old + c1MarkDeletePositionBefore = waitForMarkDeletePositionToChange(topic1, subName1, c1MarkDeletePositionBefore); + waitForQuotaCheckToRunTwice(); + + Metrics metrics = prometheusMetricsClient.getMetrics(); + TopicStats topicStats = getTopicStats(topic1); + + assertThat(topicStats.getBacklogQuotaLimitTime()).isEqualTo(timeLimitSeconds); + assertThat(topicStats.getBacklogSize()).isEqualTo(0); + assertThat(topicStats.getSubscriptions().get(subName1).getMsgBacklog()).isEqualTo(0); + assertThat(topicStats.getOldestBacklogMessageAgeSeconds()).isEqualTo(0); + assertThat(topicStats.getOldestBacklogMessageSubscriptionName()).isNull(); + + Metric backlogAgeMetric = + metrics.findSingleMetricByNameAndLabels("pulsar_storage_backlog_age_seconds", + Pair.of("topic", topic1)); + assertThat(backlogAgeMetric.tags).containsExactly( + entry("cluster", CLUSTER_NAME), + entry("namespace", namespace), + entry("topic", topic1)); + assertThat((long) backlogAgeMetric.value).isEqualTo(0); + + // producer should create success. + Producer producer2 = createProducer(client, topic1); + assertNotNull(producer2); + } + config.setPreciseTimeBasedBacklogQuotaCheck(false); + config.setExposePreciseBacklogInPrometheus(false); + } + + private long getReadEntries(String topic1) { + return ((PersistentTopic) pulsar.getBrokerService().getTopicReference(topic1).get()) + .getManagedLedger().getStats().getEntriesReadTotalCount(); + } + + @Test + public void backlogsStatsNotPrecise() throws PulsarAdminException, PulsarClientException, InterruptedException { + config.setPreciseTimeBasedBacklogQuotaCheck(false); + config.setManagedLedgerMaxEntriesPerLedger(6); + final String namespace = "prop/ns-quota"; + assertEquals(admin.namespaces().getBacklogQuotaMap(namespace), new HashMap<>()); + final int sizeLimitBytes = 15 * 1024 * 1024; + final int timeLimitSeconds = 123; + admin.namespaces().setBacklogQuota( + namespace, + BacklogQuota.builder() + .limitSize(sizeLimitBytes) + .retentionPolicy(BacklogQuota.RetentionPolicy.producer_exception) + .build(), + destination_storage); + admin.namespaces().setBacklogQuota( + namespace, + BacklogQuota.builder() + .limitTime(timeLimitSeconds) + .retentionPolicy(BacklogQuota.RetentionPolicy.producer_exception) + .build(), + message_age); + + try (PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()) + .statsInterval(0, SECONDS).build()) { + final String topic1 = "persistent://prop/ns-quota/topic2" + UUID.randomUUID(); + + final String subName1 = "brandNewC1"; + final String subName2 = "brandNewC2"; + final int numMsgs = 5; + + Consumer consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1) + .acknowledgmentGroupTime(0, SECONDS) + .isAckReceiptEnabled(true) + .subscribe(); + Consumer consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2) + .acknowledgmentGroupTime(0, SECONDS) + .isAckReceiptEnabled(true) + .subscribe(); + Producer producer = createProducer(client, topic1); + + byte[] content = new byte[1024]; + for (int i = 0; i < numMsgs; i++) { + Thread.sleep(500); + producer.send(content); + } + + String c1MarkDeletePositionBefore = + admin.topics().getInternalStats(topic1).cursors.get(subName1).markDeletePosition; + + consumer1.acknowledge(consumer1.receive()); + log.info("Moved subscription 1, by 1 message"); + c1MarkDeletePositionBefore = waitForMarkDeletePositionToChange(topic1, subName1, c1MarkDeletePositionBefore); + waitForQuotaCheckToRunTwice(); + + TopicStats topicStats = getTopicStats(topic1); + + // We have only one ledger, and it is not closed yet, so we can't tell the age until it is closed + assertThat(topicStats.getOldestBacklogMessageSubscriptionName()).isNull(); + + Metrics metrics = prometheusMetricsClient.getMetrics(); + Metric backlogAgeMetric = + metrics.findSingleMetricByNameAndLabels("pulsar_storage_backlog_age_seconds", + Pair.of("topic", topic1)); + assertThat(backlogAgeMetric.value).isEqualTo(-1); + + unloadAndLoadTopic(topic1, producer); + long unloadTime = System.currentTimeMillis(); + + waitForQuotaCheckToRunTwice(); + + topicStats = getTopicStats(topic1); + assertThat(topicStats.getOldestBacklogMessageSubscriptionName()).isEqualTo(subName2); + // age is measured against the ledger closing time + long expectedAge = MILLISECONDS.toSeconds(System.currentTimeMillis() - unloadTime); + assertThat(topicStats.getOldestBacklogMessageAgeSeconds()).isCloseTo(expectedAge, within(1L)); + + String c2MarkDeletePositionBefore = + admin.topics().getInternalStats(topic1).cursors.get(subName2).markDeletePosition; + Message message; + for (int i = 0; i < numMsgs-1; i++) { + consumer1.acknowledge(consumer1.receive()); + message = consumer2.receive(); + consumer2.acknowledge(message); + } + // At this point subscription 2 is the oldest + + waitForMarkDeletePositionToChange(topic1, subName1, c1MarkDeletePositionBefore); + waitForMarkDeletePositionToChange(topic1, subName2, c2MarkDeletePositionBefore); + waitForQuotaCheckToRunTwice(); + + topicStats = getTopicStats(topic1); + assertThat(topicStats.getOldestBacklogMessageSubscriptionName()).isEqualTo(subName2); + expectedAge = MILLISECONDS.toSeconds(System.currentTimeMillis() - unloadTime); + assertThat(topicStats.getOldestBacklogMessageAgeSeconds()).isCloseTo(expectedAge, within(1L)); + config.setManagedLedgerMaxEntriesPerLedger(MAX_ENTRIES_PER_LEDGER); + } + } + + @Test + public void backlogsStatsNotPreciseWithNoBacklog() throws PulsarAdminException, PulsarClientException, InterruptedException { + config.setPreciseTimeBasedBacklogQuotaCheck(false); + config.setExposePreciseBacklogInPrometheus(false); + config.setManagedLedgerMaxEntriesPerLedger(6); + final String namespace = "prop/ns-quota"; + assertEquals(admin.namespaces().getBacklogQuotaMap(namespace), new HashMap<>()); + final int timeLimitSeconds = 2; + admin.namespaces().setBacklogQuota( + namespace, + BacklogQuota.builder() + .limitTime(timeLimitSeconds) + .retentionPolicy(BacklogQuota.RetentionPolicy.producer_exception) + .build(), + message_age); + + try (PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()) + .maxBackoffInterval(3, SECONDS) + .statsInterval(0, SECONDS).build()) { + final String topic1 = "persistent://prop/ns-quota/topic2" + UUID.randomUUID(); + + final String subName1 = "brandNewC1"; + final int numMsgs = 5; + + Consumer consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1) + .acknowledgmentGroupTime(0, SECONDS) + .isAckReceiptEnabled(true) + .subscribe(); + Producer producer = createProducer(client, topic1); + + byte[] content = new byte[1024]; + for (int i = 0; i < numMsgs; i++) { + producer.send(content); + } + + String c1MarkDeletePositionBefore = + admin.topics().getInternalStats(topic1).cursors.get(subName1).markDeletePosition; + + log.info("Moved subscription 1 to end"); + for (int i = 0; i < numMsgs; i++) { + consumer1.acknowledge(consumer1.receive()); + } + + c1MarkDeletePositionBefore = waitForMarkDeletePositionToChange(topic1, subName1, c1MarkDeletePositionBefore); + waitForQuotaCheckToRunTwice(); + + // backlog and backlogAceSeconds should be 0 + TopicStats topicStats = getTopicStats(topic1, false); + Metrics metrics = prometheusMetricsClient.getMetrics(); + assertEquals(topicStats.getSubscriptions().get(subName1).getMsgBacklog(), 0); + assertThat(topicStats.getOldestBacklogMessageSubscriptionName()).isNull(); + assertThat(topicStats.getOldestBacklogMessageAgeSeconds()).isEqualTo(0); + Metric backlogAgeMetric = + metrics.findSingleMetricByNameAndLabels("pulsar_storage_backlog_age_seconds", + Pair.of("topic", topic1)); + assertThat(backlogAgeMetric.value).isEqualTo(0); + + // producer should create success. + Producer producer2 = createProducer(client, topic1); + assertNotNull(producer2); + + config.setManagedLedgerMaxEntriesPerLedger(MAX_ENTRIES_PER_LEDGER); + } + } + + private void unloadAndLoadTopic(String topic, Producer producer) throws PulsarAdminException, + PulsarClientException { + admin.topics().unload(topic); + // This will load the topic + producer.send("Bla".getBytes()); + Awaitility.await().pollInterval(100, MILLISECONDS).atMost(5, SECONDS) + .until(() -> admin.topics().getInternalStats(topic).numberOfEntries > 0); + } + + private void waitForQuotaCheckToRunTwice() { + final long initialQuotaCheckCount = getQuotaCheckCount(); + Awaitility.await() + .pollInterval(1, SECONDS) + .atMost(TIME_TO_CHECK_BACKLOG_QUOTA*3, SECONDS) + .until(() -> getQuotaCheckCount() > initialQuotaCheckCount + 1); + } + + /** + * @return The new mark delete position + */ + private String waitForMarkDeletePositionToChange(String topic, + String subscriptionName, + String previousMarkDeletePosition) { + return Awaitility.await().pollInterval(1, SECONDS).atMost(5, SECONDS).until( + () -> admin.topics().getInternalStats(topic).cursors.get(subscriptionName).markDeletePosition, + markDeletePosition -> markDeletePosition != null && !markDeletePosition.equals(previousMarkDeletePosition)); + } + + private long getQuotaCheckCount() { + Metrics metrics = prometheusMetricsClient.getMetrics(); + return (long) metrics.findByNameAndLabels( + "pulsar_storage_backlog_quota_check_duration_seconds_count", + "cluster", CLUSTER_NAME) + .get(0).value; + } + /** * Time based backlog quota won't affect reader since broker doesn't keep track of consuming position for reader * and can't do message age check against the quota. @@ -359,7 +838,7 @@ public void testTriggerBacklogTimeQuotaWithReader() throws Exception { .limitTime(TIME_TO_CHECK_BACKLOG_QUOTA) .retentionPolicy(BacklogQuota.RetentionPolicy.producer_exception) .build()); - try (PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();) { + try (PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, SECONDS).build();) { final String topic1 = "persistent://prop/ns-quota/topic2" + UUID.randomUUID(); final int numMsgs = 9; Reader reader = client.newReader().topic(topic1).receiverQueueSize(1).startMessageId(MessageId.latest).create(); @@ -405,7 +884,7 @@ public void testTriggerBacklogTimeQuotaWithReader() throws Exception { // check reader can still read without error while (true) { - Message msg = reader.readNext(5, TimeUnit.SECONDS); + Message msg = reader.readNext(5, SECONDS); if (msg == null) { break; } @@ -420,23 +899,24 @@ public void testTriggerBacklogTimeQuotaWithReader() throws Exception { public void testConsumerBacklogEvictionSizeQuota() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/ns-quota"), new HashMap<>()); + var backlogSizeLimit = 10 * 1024; admin.namespaces().setBacklogQuota("prop/ns-quota", BacklogQuota.builder() - .limitSize(10 * 1024) + .limitSize(backlogSizeLimit) .retentionPolicy(BacklogQuota.RetentionPolicy.consumer_backlog_eviction) .build()); @Cleanup - PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS) + PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, SECONDS) .build(); - final String topic1 = "persistent://prop/ns-quota/topic2" + UUID.randomUUID(); + final String topic1 = BrokerTestUtil.newUniqueName("persistent://prop/ns-quota/topic2"); final String subName1 = "c1"; final String subName2 = "c2"; final int numMsgs = 20; Consumer consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe(); Consumer consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2).subscribe(); - org.apache.pulsar.client.api.Producer producer = createProducer(client, topic1); + Producer producer = createProducer(client, topic1); byte[] content = new byte[1024]; for (int i = 0; i < numMsgs; i++) { producer.send(content); @@ -449,6 +929,23 @@ public void testConsumerBacklogEvictionSizeQuota() throws Exception { TopicStats stats = getTopicStats(topic1); assertTrue(stats.getBacklogSize() < 10 * 1024, "Storage size is [" + stats.getStorageSize() + "]"); + assertThat(evictionCountMetric("prop/ns-quota", topic1, "size")).isEqualTo(1); + assertThat(evictionCountMetric("size")).isEqualTo(1); + + var attributes = Attributes.builder() + .put(OpenTelemetryAttributes.PULSAR_DOMAIN, "persistent") + .put(OpenTelemetryAttributes.PULSAR_TENANT, "prop") + .put(OpenTelemetryAttributes.PULSAR_NAMESPACE, "prop/ns-quota") + .put(OpenTelemetryAttributes.PULSAR_TOPIC, topic1) + .build(); + var metrics = openTelemetryMetricReader.collectAllMetrics(); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.BACKLOG_QUOTA_LIMIT_SIZE, attributes, + backlogSizeLimit); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.BACKLOG_EVICTION_COUNTER, Attributes.builder() + .putAll(attributes) + .put(OpenTelemetryAttributes.PULSAR_BACKLOG_QUOTA_TYPE, "size") + .build(), + 1); } @Test @@ -459,10 +956,10 @@ public void testConsumerBacklogEvictionTimeQuotaPrecise() throws Exception { BacklogQuota.builder() .limitTime(TIME_TO_CHECK_BACKLOG_QUOTA) .retentionPolicy(BacklogQuota.RetentionPolicy.consumer_backlog_eviction) - .build(), BacklogQuota.BacklogQuotaType.message_age); + .build(), message_age); config.setPreciseTimeBasedBacklogQuotaCheck(true); @Cleanup - PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS) + PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, SECONDS) .build(); final String topic1 = "persistent://prop/ns-quota/topic3" + UUID.randomUUID(); @@ -472,7 +969,7 @@ public void testConsumerBacklogEvictionTimeQuotaPrecise() throws Exception { Consumer consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe(); Consumer consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2).subscribe(); - org.apache.pulsar.client.api.Producer producer = createProducer(client, topic1); + Producer producer = createProducer(client, topic1); byte[] content = new byte[1024]; for (int i = 0; i < numMsgs; i++) { producer.send(content); @@ -491,29 +988,54 @@ public void testConsumerBacklogEvictionTimeQuotaPrecise() throws Exception { // All messages for both subscription should be cleaned up from backlog by backlog monitor task. assertEquals(stats.getSubscriptions().get(subName1).getMsgBacklog(), 0); assertEquals(stats.getSubscriptions().get(subName2).getMsgBacklog(), 0); + assertThat(evictionCountMetric("prop/ns-quota", topic1, "time")).isEqualTo(1); + assertThat(evictionCountMetric("time")).isEqualTo(1); + } + + @SuppressWarnings("SameParameterValue") + private long evictionCountMetric(String namespace, String topic, String quotaType) { + Metrics metrics = prometheusMetricsClient.getMetrics(); + Metric topicEvictionsTotal = metrics.findSingleMetricByNameAndLabels( + "pulsar_storage_backlog_quota_exceeded_evictions_total", + Pair.of("topic", topic), + Pair.of("quota_type", quotaType), + Pair.of("namespace", namespace), + Pair.of("cluster", CLUSTER_NAME)); + return (long) topicEvictionsTotal.value; } + private long evictionCountMetric(String quotaType) { + Metrics metrics = prometheusMetricsClient.getMetrics(); + Metric topicEvictionsTotal = metrics.findSingleMetricByNameAndLabels( + "pulsar_broker_storage_backlog_quota_exceeded_evictions_total", + Pair.of("quota_type", quotaType), + Pair.of("cluster", CLUSTER_NAME)); + return (long) topicEvictionsTotal.value; + } + + @Test(timeOut = 60000) public void testConsumerBacklogEvictionTimeQuota() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/ns-quota"), new HashMap<>()); + var backlogTimeLimit = TIME_TO_CHECK_BACKLOG_QUOTA; admin.namespaces().setBacklogQuota("prop/ns-quota", BacklogQuota.builder() - .limitTime(TIME_TO_CHECK_BACKLOG_QUOTA) + .limitTime(backlogTimeLimit) .retentionPolicy(BacklogQuota.RetentionPolicy.consumer_backlog_eviction) - .build(), BacklogQuota.BacklogQuotaType.message_age); + .build(), message_age); @Cleanup - PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS) + PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, SECONDS) .build(); - final String topic1 = "persistent://prop/ns-quota/topic3" + UUID.randomUUID(); + final String topic1 = BrokerTestUtil.newUniqueName("persistent://prop/ns-quota/topic3"); final String subName1 = "c1"; final String subName2 = "c2"; final int numMsgs = 14; Consumer consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe(); Consumer consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2).subscribe(); - org.apache.pulsar.client.api.Producer producer = createProducer(client, topic1); + Producer producer = createProducer(client, topic1); byte[] content = new byte[1024]; for (int i = 0; i < numMsgs; i++) { producer.send(content); @@ -529,7 +1051,8 @@ public void testConsumerBacklogEvictionTimeQuota() throws Exception { ManagedLedgerImpl ml = (ManagedLedgerImpl) topic1Reference.getManagedLedger(); Position slowConsumerReadPos = ml.getSlowestConsumer().getReadPosition(); - Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA * 2) * 1000); + var delaySeconds = backlogTimeLimit * 2; + Thread.sleep(delaySeconds * 1000); rolloverStats(); TopicStats stats2 = getTopicStats(topic1); @@ -541,6 +1064,23 @@ public void testConsumerBacklogEvictionTimeQuota() throws Exception { }); assertEquals(ml.getSlowestConsumer().getReadPosition(), slowConsumerReadPos); + + var attributes = Attributes.builder() + .put(OpenTelemetryAttributes.PULSAR_DOMAIN, "persistent") + .put(OpenTelemetryAttributes.PULSAR_TENANT, "prop") + .put(OpenTelemetryAttributes.PULSAR_NAMESPACE, "prop/ns-quota") + .put(OpenTelemetryAttributes.PULSAR_TOPIC, topic1) + .build(); + var metrics = openTelemetryMetricReader.collectAllMetrics(); + assertMetricLongGaugeValue(metrics, OpenTelemetryTopicStats.BACKLOG_QUOTA_LIMIT_TIME, attributes, + backlogTimeLimit); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.BACKLOG_EVICTION_COUNTER, Attributes.builder() + .putAll(attributes) + .put(OpenTelemetryAttributes.PULSAR_BACKLOG_QUOTA_TYPE, "time") + .build(), + 1); + assertMetricLongGaugeValue(metrics, OpenTelemetryTopicStats.BACKLOG_QUOTA_AGE, attributes, + value -> assertThat(value).isGreaterThanOrEqualTo(delaySeconds)); } @Test(timeOut = 60000) @@ -551,9 +1091,9 @@ public void testConsumerBacklogEvictionTimeQuotaWithPartEviction() throws Except BacklogQuota.builder() .limitTime(5) // set limit time as 5 seconds .retentionPolicy(BacklogQuota.RetentionPolicy.consumer_backlog_eviction) - .build(), BacklogQuota.BacklogQuotaType.message_age); + .build(), message_age); @Cleanup - PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS) + PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, SECONDS) .build(); final String topic1 = "persistent://prop/ns-quota/topic3" + UUID.randomUUID(); @@ -563,7 +1103,7 @@ public void testConsumerBacklogEvictionTimeQuotaWithPartEviction() throws Except Consumer consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe(); Consumer consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2).subscribe(); - org.apache.pulsar.client.api.Producer producer = createProducer(client, topic1); + Producer producer = createProducer(client, topic1); byte[] content = new byte[1024]; for (int i = 0; i < numMsgs; i++) { producer.send(content); @@ -605,17 +1145,17 @@ public void testConsumerBacklogEvictionTimeQuotaWithEmptyLedger() throws Excepti BacklogQuota.builder() .limitTime(TIME_TO_CHECK_BACKLOG_QUOTA) .retentionPolicy(BacklogQuota.RetentionPolicy.consumer_backlog_eviction) - .build(), BacklogQuota.BacklogQuotaType.message_age); + .build(), message_age); config.setPreciseTimeBasedBacklogQuotaCheck(true); @Cleanup - PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS) + PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, SECONDS) .build(); final String topic = "persistent://prop/ns-quota/topic4" + UUID.randomUUID(); final String subName = "c1"; Consumer consumer = client.newConsumer().topic(topic).subscriptionName(subName).subscribe(); - org.apache.pulsar.client.api.Producer producer = createProducer(client, topic); + Producer producer = createProducer(client, topic); producer.send(new byte[1024]); consumer.receive(); @@ -663,7 +1203,7 @@ public void testConsumerBacklogEvictionWithAckSizeQuota() throws Exception { Consumer consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe(); Consumer consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2).subscribe(); - org.apache.pulsar.client.api.Producer producer = createProducer(client, topic1); + Producer producer = createProducer(client, topic1); byte[] content = new byte[1024]; for (int i = 0; i < numMsgs; i++) { producer.send(content); @@ -687,7 +1227,7 @@ public void testConsumerBacklogEvictionWithAckTimeQuotaPrecise() throws Exceptio BacklogQuota.builder() .limitTime(TIME_TO_CHECK_BACKLOG_QUOTA) .retentionPolicy(BacklogQuota.RetentionPolicy.consumer_backlog_eviction) - .build(), BacklogQuota.BacklogQuotaType.message_age); + .build(), message_age); config.setPreciseTimeBasedBacklogQuotaCheck(true); @Cleanup PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).build(); @@ -699,7 +1239,7 @@ public void testConsumerBacklogEvictionWithAckTimeQuotaPrecise() throws Exceptio Consumer consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe(); Consumer consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2).subscribe(); - org.apache.pulsar.client.api.Producer producer = createProducer(client, topic1); + Producer producer = createProducer(client, topic1); byte[] content = new byte[1024]; for (int i = 0; i < numMsgs; i++) { @@ -737,7 +1277,7 @@ private Producer createProducer(PulsarClient client, String topic) throws PulsarClientException { return client.newProducer() .enableBatching(false) - .sendTimeout(2, TimeUnit.SECONDS) + .sendTimeout(2, SECONDS) .topic(topic) .create(); } @@ -756,7 +1296,7 @@ public void testConsumerBacklogEvictionWithAckTimeQuota() throws Exception { Consumer consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe(); Consumer consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2).subscribe(); - org.apache.pulsar.client.api.Producer producer = createProducer(client, topic1); + Producer producer = createProducer(client, topic1); byte[] content = new byte[1024]; List> messagesToAcknowledge = new ArrayList<>(); @@ -797,7 +1337,7 @@ public void testConsumerBacklogEvictionWithAckTimeQuota() throws Exception { BacklogQuota.builder() .limitTime(2 * TIME_TO_CHECK_BACKLOG_QUOTA) .retentionPolicy(BacklogQuota.RetentionPolicy.consumer_backlog_eviction) - .build(), BacklogQuota.BacklogQuotaType.message_age); + .build(), message_age); Awaitility.await() .pollInterval(Duration.ofSeconds(1)) @@ -831,10 +1371,10 @@ public void testConcurrentAckAndEviction() throws Exception { final CountDownLatch counter = new CountDownLatch(2); final AtomicBoolean gotException = new AtomicBoolean(false); @Cleanup - PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS) + PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, SECONDS) .build(); @Cleanup - PulsarClient client2 = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS) + PulsarClient client2 = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, SECONDS) .build(); Consumer consumer1 = client2.newConsumer().topic(topic1).subscriptionName(subName1).subscribe(); Consumer consumer2 = client2.newConsumer().topic(topic1).subscriptionName(subName2).subscribe(); @@ -874,7 +1414,7 @@ public void testConcurrentAckAndEviction() throws Exception { consumerThread.start(); // test hangs without timeout since there is nothing to consume due to eviction - counter.await(20, TimeUnit.SECONDS); + counter.await(20, SECONDS); assertFalse(gotException.get()); Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000); rolloverStats(); @@ -903,13 +1443,13 @@ public void testNoEviction() throws Exception { final AtomicBoolean gotException = new AtomicBoolean(false); @Cleanup final PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()) - .statsInterval(0, TimeUnit.SECONDS).build(); + .statsInterval(0, SECONDS).build(); final Consumer consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe(); final Consumer consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2).subscribe(); @Cleanup final PulsarClient client2 = PulsarClient.builder().serviceUrl(adminUrl.toString()) - .statsInterval(0, TimeUnit.SECONDS).build(); + .statsInterval(0, SECONDS).build(); Thread producerThread = new Thread(() -> { try { @@ -967,16 +1507,16 @@ public void testEvictionMulti() throws Exception { final AtomicBoolean gotException = new AtomicBoolean(false); @Cleanup final PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()) - .statsInterval(0, TimeUnit.SECONDS).build(); + .statsInterval(0, SECONDS).build(); final Consumer consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe(); final Consumer consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2).subscribe(); @Cleanup final PulsarClient client3 = PulsarClient.builder().serviceUrl(adminUrl.toString()) - .statsInterval(0, TimeUnit.SECONDS).build(); + .statsInterval(0, SECONDS).build(); @Cleanup final PulsarClient client2 = PulsarClient.builder().serviceUrl(adminUrl.toString()) - .statsInterval(0, TimeUnit.SECONDS).build(); + .statsInterval(0, SECONDS).build(); Thread producerThread1 = new Thread(() -> { try { @@ -1040,7 +1580,7 @@ public void testEvictionMulti() throws Exception { producerThread2.start(); consumerThread1.start(); consumerThread2.start(); - counter.await(20, TimeUnit.SECONDS); + counter.await(20, SECONDS); assertFalse(gotException.get()); Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000); rolloverStats(); @@ -1060,7 +1600,7 @@ public void testAheadProducerOnHold() throws Exception { .build()); @Cleanup final PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()) - .statsInterval(0, TimeUnit.SECONDS).build(); + .statsInterval(0, SECONDS).build(); final String topic1 = "persistent://prop/quotahold/hold"; final String subName1 = "c1hold"; final int numMsgs = 10; @@ -1102,7 +1642,7 @@ public void testAheadProducerOnHoldTimeout() throws Exception { .build()); @Cleanup final PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()) - .statsInterval(0, TimeUnit.SECONDS).build(); + .statsInterval(0, SECONDS).build(); final String topic1 = "persistent://prop/quotahold/holdtimeout"; final String subName1 = "c1holdtimeout"; boolean gotException = false; @@ -1140,7 +1680,7 @@ public void testProducerException() throws Exception { .build()); @Cleanup final PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()) - .statsInterval(0, TimeUnit.SECONDS).build(); + .statsInterval(0, SECONDS).build(); final String topic1 = "persistent://prop/quotahold/except"; final String subName1 = "c1except"; boolean gotException = false; @@ -1169,8 +1709,13 @@ public void testProducerException() throws Exception { assertTrue(gotException, "backlog exceeded exception did not occur"); } - @Test - public void testProducerExceptionAndThenUnblockSizeQuota() throws Exception { + @DataProvider(name = "dedupTestSet") + public static Object[][] dedupTestSet() { + return new Object[][] { { Boolean.TRUE }, { Boolean.FALSE } }; + } + + @Test(dataProvider = "dedupTestSet") + public void testProducerExceptionAndThenUnblockSizeQuota(boolean dedupTestSet) throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/quotahold"), new HashMap<>()); admin.namespaces().setBacklogQuota("prop/quotahold", @@ -1180,15 +1725,18 @@ public void testProducerExceptionAndThenUnblockSizeQuota() throws Exception { .build()); @Cleanup final PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()) - .statsInterval(0, TimeUnit.SECONDS).build(); + .statsInterval(0, SECONDS).build(); final String topic1 = "persistent://prop/quotahold/exceptandunblock"; final String subName1 = "c1except"; boolean gotException = false; Consumer consumer = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe(); - byte[] content = new byte[1024]; Producer producer = createProducer(client, topic1); + + admin.topicPolicies().setDeduplicationStatus(topic1, dedupTestSet); + Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000); + for (int i = 0; i < 10; i++) { producer.send(content); } @@ -1207,6 +1755,7 @@ public void testProducerExceptionAndThenUnblockSizeQuota() throws Exception { } assertTrue(gotException, "backlog exceeded exception did not occur"); + assertFalse(producer.isConnected()); // now remove backlog and ensure that producer is unblocked; TopicStats stats = getTopicStats(topic1); @@ -1223,14 +1772,33 @@ public void testProducerExceptionAndThenUnblockSizeQuota() throws Exception { Exception sendException = null; gotException = false; try { - for (int i = 0; i < 5; i++) { + for (int i = 0; i < 10; i++) { producer.send(content); + Message msg = consumer.receive(); + consumer.acknowledge(msg); } } catch (Exception e) { gotException = true; sendException = e; } + Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000); assertFalse(gotException, "unable to publish due to " + sendException); + + gotException = false; + long lastDisconnectedTimestamp = producer.getLastDisconnectedTimestamp(); + try { + // try to send over backlog quota and make sure it passes + producer.send(content); + producer.send(content); + } catch (PulsarClientException ce) { + assertTrue(ce instanceof PulsarClientException.ProducerBlockedQuotaExceededException + || ce instanceof PulsarClientException.TimeoutException, ce.getMessage()); + gotException = true; + sendException = ce; + } + assertFalse(gotException, "unable to publish due to " + sendException); + assertEquals(lastDisconnectedTimestamp, producer.getLastDisconnectedTimestamp()); + } @Test @@ -1241,11 +1809,11 @@ public void testProducerExceptionAndThenUnblockTimeQuotaPrecise() throws Excepti BacklogQuota.builder() .limitTime(TIME_TO_CHECK_BACKLOG_QUOTA) .retentionPolicy(BacklogQuota.RetentionPolicy.producer_exception) - .build(), BacklogQuota.BacklogQuotaType.message_age); + .build(), message_age); config.setPreciseTimeBasedBacklogQuotaCheck(true); @Cleanup final PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()) - .statsInterval(0, TimeUnit.SECONDS).build(); + .statsInterval(0, SECONDS).build(); final String topic1 = "persistent://prop/quotahold/exceptandunblock2"; final String subName1 = "c1except"; boolean gotException = false; @@ -1307,10 +1875,10 @@ public void testProducerExceptionAndThenUnblockTimeQuota() throws Exception { BacklogQuota.builder() .limitTime(TIME_TO_CHECK_BACKLOG_QUOTA) .retentionPolicy(BacklogQuota.RetentionPolicy.producer_exception) - .build(), BacklogQuota.BacklogQuotaType.message_age); + .build(), message_age); @Cleanup final PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()) - .statsInterval(0, TimeUnit.SECONDS).build(); + .statsInterval(0, SECONDS).build(); final String topic1 = "persistent://prop/quotahold/exceptandunblock2"; final String subName1 = "c1except"; boolean gotException = false; @@ -1378,10 +1946,13 @@ public void testBacklogQuotaInGB(boolean backlogQuotaSizeGB) throws Exception { config.setBacklogQuotaDefaultRetentionPolicy(BacklogQuota.RetentionPolicy.consumer_backlog_eviction); pulsar = new PulsarService(config); pulsar.start(); + if (admin != null) { + admin.close(); + } admin = PulsarAdmin.builder().serviceHttpUrl(adminUrl.toString()).build(); @Cleanup - PulsarClient client = PulsarClient.builder().serviceUrl(pulsar.getBrokerServiceUrl()).statsInterval(0, TimeUnit.SECONDS) + PulsarClient client = PulsarClient.builder().serviceUrl(pulsar.getBrokerServiceUrl()).statsInterval(0, SECONDS) .build(); final String topic1 = "persistent://prop/ns-quota/topic2" + UUID.randomUUID(); @@ -1391,7 +1962,7 @@ public void testBacklogQuotaInGB(boolean backlogQuotaSizeGB) throws Exception { Consumer consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe(); Consumer consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2).subscribe(); - org.apache.pulsar.client.api.Producer producer = createProducer(client, topic1); + Producer producer = createProducer(client, topic1); byte[] content = new byte[1024]; for (int i = 0; i < numMsgs; i++) { producer.send(content); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java index a2d80b2ba600b..c66eff2c8a180 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java @@ -768,6 +768,7 @@ public void testConcurrentBatchMessageAck(BatcherBuilder builder) throws Excepti final Consumer myConsumer = pulsarClient.newConsumer().topic(topicName) .subscriptionName(subscriptionName).subscriptionType(SubscriptionType.Shared).subscribe(); // assertEquals(dispatcher.getTotalUnackedMessages(), 1); + @Cleanup("shutdownNow") ExecutorService executor = Executors.newFixedThreadPool(10); final CountDownLatch latch = new CountDownLatch(numMsgs); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java index 433f5e56d952d..ed7f6974dd26f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java @@ -18,8 +18,17 @@ */ package org.apache.pulsar.broker.service; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; +import com.carrotsearch.hppc.ObjectSet; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -27,19 +36,32 @@ import java.util.concurrent.TimeUnit; import lombok.Cleanup; import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.impl.AckSetStateUtil; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.ConsumerBuilder; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.BatchMessageIdImpl; +import org.apache.pulsar.client.impl.ConsumerImpl; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.collections.BitSetRecyclable; import org.awaitility.Awaitility; +import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +@Slf4j @Test(groups = "broker") public class BatchMessageWithBatchIndexLevelTest extends BatchMessageTest { @@ -280,4 +302,370 @@ public void testAckMessageWithNotOwnerConsumerUnAckMessageCount() throws Excepti Awaitility.await().until(() -> getPulsar().getBrokerService().getTopic(topicName, false) .get().get().getSubscription(subName).getConsumers().get(0).getUnackedMessages() == 0); } + + @Test + public void testNegativeAckAndLongAckDelayWillNotLeadRepeatConsume() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://prop/ns-abc/tp_"); + final String subscriptionName = "s1"; + final int redeliveryDelaySeconds = 2; + + // Create producer and consumer. + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .enableBatching(true) + .batchingMaxMessages(1000) + .batchingMaxPublishDelay(1, TimeUnit.HOURS) + .create(); + ConsumerImpl consumer = (ConsumerImpl) pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName(subscriptionName) + .subscriptionType(SubscriptionType.Shared) + .negativeAckRedeliveryDelay(redeliveryDelaySeconds, TimeUnit.SECONDS) + .enableBatchIndexAcknowledgment(true) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .acknowledgmentGroupTime(1, TimeUnit.HOURS) + .subscribe(); + + // Send 10 messages in batch. + ArrayList messagesSent = new ArrayList<>(); + List> sendTasks = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + String msg = Integer.valueOf(i).toString(); + sendTasks.add(producer.sendAsync(Integer.valueOf(i).toString())); + messagesSent.add(msg); + } + producer.flush(); + FutureUtil.waitForAll(sendTasks).join(); + + // Receive messages. + ArrayList messagesReceived = new ArrayList<>(); + // NegativeAck "batchMessageIdIndex1" once. + boolean index1HasBeenNegativeAcked = false; + while (true) { + Message message = consumer.receive(2, TimeUnit.SECONDS); + if (message == null) { + break; + } + if (index1HasBeenNegativeAcked) { + messagesReceived.add(message.getValue()); + consumer.acknowledge(message); + continue; + } + if (((MessageIdAdv) message.getMessageId()).getBatchIndex() == 1) { + consumer.negativeAcknowledge(message); + index1HasBeenNegativeAcked = true; + continue; + } + messagesReceived.add(message.getValue()); + consumer.acknowledge(message); + } + + // Receive negative acked messages. + // Wait the message negative acknowledgment finished. + int tripleRedeliveryDelaySeconds = redeliveryDelaySeconds * 3; + while (true) { + Message message = consumer.receive(tripleRedeliveryDelaySeconds, TimeUnit.SECONDS); + if (message == null) { + break; + } + messagesReceived.add(message.getValue()); + consumer.acknowledge(message); + } + + log.info("messagesSent: {}, messagesReceived: {}", messagesSent, messagesReceived); + Assert.assertEquals(messagesReceived.size(), messagesSent.size()); + + // cleanup. + producer.close(); + consumer.close(); + admin.topics().delete(topicName); + } + + @Test + public void testMixIndexAndNonIndexUnAckMessageCount() throws Exception { + final String topicName = "persistent://prop/ns-abc/testMixIndexAndNonIndexUnAckMessageCount-"; + + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topicName) + .enableBatching(true) + .batchingMaxPublishDelay(100, TimeUnit.MILLISECONDS) + .create(); + @Cleanup + Consumer consumer = pulsarClient.newConsumer() + .topic(topicName) + .subscriptionName("sub") + .subscriptionType(SubscriptionType.Shared) + .acknowledgmentGroupTime(100, TimeUnit.MILLISECONDS) + .enableBatchIndexAcknowledgment(true) + .isAckReceiptEnabled(true) + .subscribe(); + + // send two batch messages: [(1), (2,3)] + producer.send("1".getBytes()); + producer.sendAsync("2".getBytes()); + producer.send("3".getBytes()); + + Message message1 = consumer.receive(); + Message message2 = consumer.receive(); + Message message3 = consumer.receive(); + consumer.acknowledgeAsync(message1); + consumer.acknowledge(message2); // send group ack: non-index ack for 1, index ack for 2 + consumer.acknowledge(message3); // index ack for 3 + + assertEquals(admin.topics().getStats(topicName).getSubscriptions() + .get("sub").getUnackedMessages(), 0); + } + + @Test + public void testUnAckMessagesWhenConcurrentDeliveryAndAck() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://prop/ns-abc/tp"); + final String subName = "s1"; + final int receiverQueueSize = 500; + admin.topics().createNonPartitionedTopic(topicName); + admin.topics().createSubscription(topicName, subName, MessageId.earliest); + ConsumerBuilder consumerBuilder = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .receiverQueueSize(receiverQueueSize) + .subscriptionName(subName) + .enableBatchIndexAcknowledgment(true) + .subscriptionType(SubscriptionType.Shared) + .isAckReceiptEnabled(true); + + // Send 100 messages. + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .enableBatching(true) + .batchingMaxPublishDelay(1, TimeUnit.HOURS) + .create(); + CompletableFuture lastSent = null; + for (int i = 0; i < 100; i++) { + lastSent = producer.sendAsync(i + ""); + } + producer.flush(); + lastSent.join(); + + // When consumer1 is closed, may some messages are in the client memory(it they are being acked now). + Consumer consumer1 = consumerBuilder.consumerName("c1").subscribe(); + Message[] messagesInClientMemory = new Message[2]; + for (int i = 0; i < 2; i++) { + Message msg = consumer1.receive(2, TimeUnit.SECONDS); + assertNotNull(msg); + messagesInClientMemory[i] = msg; + } + ConsumerImpl consumer2 = (ConsumerImpl) consumerBuilder.consumerName("c2").subscribe(); + Awaitility.await().until(() -> consumer2.isConnected()); + + // The consumer2 will receive messages after consumer1 closed. + // Insert a delay mechanism to make the flow like below: + // 1. Close consumer1, then the 100 messages will be redelivered. + // 2. Read redeliver messages. No messages were acked at this time. + // 3. The in-flight ack of two messages is finished. + // 4. Send the messages to consumer2, consumer2 will get all the 100 messages. + CompletableFuture receiveMessageSignal2 = new CompletableFuture<>(); + org.apache.pulsar.broker.service.Consumer serviceConsumer2 = + makeConsumerReceiveMessagesDelay(topicName, subName, "c2", receiveMessageSignal2); + // step 1: close consumer. + consumer1.close(); + // step 2: wait for read messages from replay queue. + Thread.sleep(2 * 1000); + // step 3: wait for the in-flight ack. + BitSetRecyclable bitSetRecyclable = createBitSetRecyclable(100); + long ledgerId = 0, entryId = 0; + for (Message message : messagesInClientMemory) { + BatchMessageIdImpl msgId = (BatchMessageIdImpl) message.getMessageId(); + bitSetRecyclable.clear(msgId.getBatchIndex()); + ledgerId = msgId.getLedgerId(); + entryId = msgId.getEntryId(); + } + getCursor(topicName, subName).delete(AckSetStateUtil.createPositionWithAckSet(ledgerId, entryId, bitSetRecyclable.toLongArray())); + // step 4: send messages to consumer2. + receiveMessageSignal2.complete(null); + // Verify: Consumer2 will get all the 100 messages, and "unAckMessages" is 100. + List messages2 = new ArrayList<>(); + while (true) { + Message msg = consumer2.receive(2, TimeUnit.SECONDS); + if (msg == null) { + break; + } + messages2.add(msg); + } + assertEquals(messages2.size(), 100); + assertEquals(serviceConsumer2.getUnackedMessages(), 100); + // After the messages were pop out, the permits in the client memory went to 100. + Awaitility.await().untilAsserted(() -> { + assertEquals(serviceConsumer2.getAvailablePermits() + consumer2.getAvailablePermits(), + receiverQueueSize); + }); + + // cleanup. + producer.close(); + consumer2.close(); + admin.topics().delete(topicName, false); + } + + private BitSetRecyclable createBitSetRecyclable(int batchSize) { + BitSetRecyclable bitSetRecyclable = new BitSetRecyclable(batchSize); + for (int i = 0; i < batchSize; i++) { + bitSetRecyclable.set(i); + } + return bitSetRecyclable; + } + + private ManagedCursorImpl getCursor(String topic, String sub) { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topic, false).join().get(); + PersistentDispatcherMultipleConsumers dispatcher = + (PersistentDispatcherMultipleConsumers) persistentTopic.getSubscription(sub).getDispatcher(); + return (ManagedCursorImpl) dispatcher.getCursor(); + } + + /*** + * After {@param signal} complete, the consumer({@param consumerName}) start to receive messages. + */ + private org.apache.pulsar.broker.service.Consumer makeConsumerReceiveMessagesDelay(String topic, String sub, + String consumerName, + CompletableFuture signal) throws Exception { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topic, false).join().get(); + PersistentDispatcherMultipleConsumers dispatcher = + (PersistentDispatcherMultipleConsumers) persistentTopic.getSubscription(sub).getDispatcher(); + org.apache.pulsar.broker.service.Consumer serviceConsumer = null; + for (org.apache.pulsar.broker.service.Consumer c : dispatcher.getConsumers()){ + if (c.consumerName().equals(consumerName)) { + serviceConsumer = c; + break; + } + } + final org.apache.pulsar.broker.service.Consumer originalConsumer = serviceConsumer; + + // Insert a delay signal. + org.apache.pulsar.broker.service.Consumer spyServiceConsumer = spy(originalConsumer); + doAnswer(invocation -> { + List entries = (List) invocation.getArguments()[0]; + EntryBatchSizes batchSizes = (EntryBatchSizes) invocation.getArguments()[1]; + EntryBatchIndexesAcks batchIndexesAcks = (EntryBatchIndexesAcks) invocation.getArguments()[2]; + int totalMessages = (int) invocation.getArguments()[3]; + long totalBytes = (long) invocation.getArguments()[4]; + long totalChunkedMessages = (long) invocation.getArguments()[5]; + RedeliveryTracker redeliveryTracker = (RedeliveryTracker) invocation.getArguments()[6]; + return signal.thenApply(__ -> originalConsumer.sendMessages(entries, batchSizes, batchIndexesAcks, totalMessages, totalBytes, + totalChunkedMessages, redeliveryTracker)).join(); + }).when(spyServiceConsumer) + .sendMessages(anyList(), any(), any(), anyInt(), anyLong(), anyLong(), any()); + doAnswer(invocation -> { + List entries = (List) invocation.getArguments()[0]; + EntryBatchSizes batchSizes = (EntryBatchSizes) invocation.getArguments()[1]; + EntryBatchIndexesAcks batchIndexesAcks = (EntryBatchIndexesAcks) invocation.getArguments()[2]; + int totalMessages = (int) invocation.getArguments()[3]; + long totalBytes = (long) invocation.getArguments()[4]; + long totalChunkedMessages = (long) invocation.getArguments()[5]; + RedeliveryTracker redeliveryTracker = (RedeliveryTracker) invocation.getArguments()[6]; + long epoch = (long) invocation.getArguments()[7]; + return signal.thenApply(__ -> originalConsumer.sendMessages(entries, batchSizes, batchIndexesAcks, totalMessages, totalBytes, + totalChunkedMessages, redeliveryTracker, epoch)).join(); + }).when(spyServiceConsumer) + .sendMessages(anyList(), any(), any(), anyInt(), anyLong(), anyLong(), any(), anyLong()); + + // Replace the consumer. + Field fConsumerList = AbstractDispatcherMultipleConsumers.class.getDeclaredField("consumerList"); + Field fConsumerSet = AbstractDispatcherMultipleConsumers.class.getDeclaredField("consumerSet"); + fConsumerList.setAccessible(true); + fConsumerSet.setAccessible(true); + List consumerList = + (List) fConsumerList.get(dispatcher); + ObjectSet consumerSet = + (ObjectSet) fConsumerSet.get(dispatcher); + + consumerList.remove(originalConsumer); + consumerSet.removeAll(originalConsumer); + consumerList.add(spyServiceConsumer); + consumerSet.add(spyServiceConsumer); + return originalConsumer; + } + + /*** + * 1. Send a batch message contains 100 single messages. + * 2. Ack 2 messages. + * 3. Redeliver the batch message and ack them. + * 4. Verify: the permits is correct. + */ + @Test + public void testPermitsIfHalfAckBatchMessage() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://prop/ns-abc/tp"); + final String subName = "s1"; + final int receiverQueueSize = 1000; + final int ackedMessagesCountInTheFistStep = 2; + admin.topics().createNonPartitionedTopic(topicName); + admin.topics(). createSubscription(topicName, subName, MessageId.earliest); + ConsumerBuilder consumerBuilder = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .receiverQueueSize(receiverQueueSize) + .subscriptionName(subName) + .enableBatchIndexAcknowledgment(true) + .subscriptionType(SubscriptionType.Shared) + .isAckReceiptEnabled(true); + + // Send 100 messages. + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .enableBatching(true) + .batchingMaxPublishDelay(1, TimeUnit.HOURS) + .create(); + CompletableFuture lastSent = null; + for (int i = 1; i <= 100; i++) { + lastSent = producer. sendAsync(i + ""); + } + producer.flush(); + lastSent.join(); + + // Ack 2 messages, and trigger a redelivery. + Consumer consumer1 = consumerBuilder.subscribe(); + for (int i = 0; i < ackedMessagesCountInTheFistStep; i++) { + Message msg = consumer1. receive(2, TimeUnit.SECONDS); + assertNotNull(msg); + consumer1.acknowledge(msg); + } + consumer1.close(); + + // Receive the left 98 messages, and ack them. + // Verify the permits is correct. + ConsumerImpl consumer2 = (ConsumerImpl) consumerBuilder.subscribe(); + Awaitility.await().until(() -> consumer2.isConnected()); + List messages = new ArrayList<>(); + int nextMessageValue = ackedMessagesCountInTheFistStep + 1; + while (true) { + Message msg = consumer2.receive(2, TimeUnit.SECONDS); + if (msg == null) { + break; + } + assertEquals(msg.getValue(), nextMessageValue + ""); + messages.add(msg.getMessageId()); + nextMessageValue++; + } + assertEquals(messages.size(), 98); + consumer2.acknowledge(messages); + + org.apache.pulsar.broker.service.Consumer serviceConsumer2 = + getTheUniqueServiceConsumer(topicName, subName); + Awaitility.await().untilAsserted(() -> { + // After the messages were pop out, the permits in the client memory went to 98. + int permitsInClientMemory = consumer2.getAvailablePermits(); + int permitsInBroker = serviceConsumer2.getAvailablePermits(); + assertEquals(permitsInClientMemory + permitsInBroker, receiverQueueSize); + }); + + // cleanup. + producer.close(); + consumer2.close(); + admin.topics().delete(topicName, false); + } + + private org.apache.pulsar.broker.service.Consumer getTheUniqueServiceConsumer(String topic, String sub) { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService(). getTopic(topic, false).join().get(); + PersistentDispatcherMultipleConsumers dispatcher = + (PersistentDispatcherMultipleConsumers) persistentTopic.getSubscription(sub).getDispatcher(); + return dispatcher.getConsumers().iterator().next(); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BkEnsemblesChaosTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BkEnsemblesChaosTest.java new file mode 100644 index 0000000000000..d49489d8a84b0 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BkEnsemblesChaosTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.client.api.Producer; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class BkEnsemblesChaosTest extends CanReconnectZKClientPulsarServiceBaseTest { + + @Override + @BeforeClass(alwaysRun = true, timeOut = 300000) + public void setup() throws Exception { + super.setup(); + } + + @Override + @AfterClass(alwaysRun = true, timeOut = 300000) + public void cleanup() throws Exception { + super.cleanup(); + } + + @Test + public void testBookieInfoIsCorrectEvenIfLostNotificationDueToZKClientReconnect() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp_"); + final byte[] msgValue = "test".getBytes(); + admin.topics().createNonPartitionedTopic(topicName); + // Ensure broker works. + Producer producer1 = client.newProducer().topic(topicName).create(); + producer1.send(msgValue); + producer1.close(); + admin.topics().unload(topicName); + + // Restart some bookies, which triggers the ZK node of Bookie deleted and created. + // And make the local metadata store reconnect to lose some notification of the ZK node change. + for (int i = 0; i < numberOfBookies - 1; i++){ + bkEnsemble.stopBK(i); + } + makeLocalMetadataStoreKeepReconnect(); + for (int i = 0; i < numberOfBookies - 1; i++){ + bkEnsemble.startBK(i); + } + // Sleep 100ms to lose the notifications of ZK node create. + Thread.sleep(100); + stopLocalMetadataStoreAlwaysReconnect(); + + // Ensure broker still works. + admin.topics().unload(topicName); + Producer producer2 = client.newProducer().topic(topicName).create(); + producer2.send(msgValue); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BkEnsemblesTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BkEnsemblesTestBase.java index 9b3d145dea230..71c5a995643c6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BkEnsemblesTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BkEnsemblesTestBase.java @@ -100,6 +100,9 @@ protected void setup() throws Exception { pulsar = new PulsarService(config); pulsar.start(); + if (admin != null) { + admin.close(); + } admin = PulsarAdmin.builder().serviceHttpUrl(pulsar.getWebServiceAddress()).build(); admin.clusters().createCluster("usc", ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); @@ -116,9 +119,18 @@ protected void setup() throws Exception { protected void cleanup() throws Exception { config = null; markCurrentSetupNumberCleaned(); - admin.close(); - pulsar.close(); - bkEnsemble.stop(); + if (admin != null) { + admin.close(); + admin = null; + } + if (pulsar != null) { + pulsar.close(); + pulsar = null; + } + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; + } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java index e69714e539be6..82892ad353aa1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java @@ -21,8 +21,8 @@ import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.retryStrategically; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertThrows; import static org.testng.Assert.fail; - import java.lang.reflect.Field; import java.util.Map.Entry; import java.util.NavigableMap; @@ -31,10 +31,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; - import com.google.common.collect.Sets; import lombok.Cleanup; - import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; @@ -47,6 +45,7 @@ import org.apache.bookkeeper.util.StringUtils; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Producer; @@ -127,7 +126,7 @@ public void testCrashBrokerWithoutCursorLedgerLeak() throws Exception { // (3) remove topic and managed-ledger from broker which means topic is not closed gracefully consumer.close(); producer.close(); - pulsar.getBrokerService().removeTopicFromCache(topic1); + pulsar.getBrokerService().removeTopicFromCache(topic); ManagedLedgerFactoryImpl factory = (ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory(); Field field = ManagedLedgerFactoryImpl.class.getDeclaredField("ledgers"); field.setAccessible(true); @@ -192,9 +191,9 @@ public void testSkipCorruptDataLedger() throws Exception { .build(); final String ns1 = "prop/usc/crash-broker"; - final int totalMessages = 100; + final int totalMessages = 99; final int totalDataLedgers = 5; - final int entriesPerLedger = totalMessages / totalDataLedgers; + final int entriesPerLedger = 20; try { admin.namespaces().createNamespace(ns1); @@ -211,10 +210,8 @@ public void testSkipCorruptDataLedger() throws Exception { PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topic1).get(); ManagedLedgerImpl ml = (ManagedLedgerImpl) topic.getManagedLedger(); ManagedCursorImpl cursor = (ManagedCursorImpl) ml.getCursors().iterator().next(); - Field configField = ManagedCursorImpl.class.getDeclaredField("config"); - configField.setAccessible(true); // Create multiple data-ledger - ManagedLedgerConfig config = (ManagedLedgerConfig) configField.get(cursor); + ManagedLedgerConfig config = ml.getConfig(); config.setMaxEntriesPerLedger(entriesPerLedger); config.setMinimumRolloverTime(1, TimeUnit.MILLISECONDS); // bookkeeper client @@ -252,7 +249,7 @@ public void testSkipCorruptDataLedger() throws Exception { // clean managed-ledger and recreate topic to clean any data from the cache producer.close(); - pulsar.getBrokerService().removeTopicFromCache(topic1); + pulsar.getBrokerService().removeTopicFromCache(topic); ManagedLedgerFactoryImpl factory = (ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory(); Field field = ManagedLedgerFactoryImpl.class.getDeclaredField("ledgers"); field.setAccessible(true); @@ -274,9 +271,9 @@ public void testSkipCorruptDataLedger() throws Exception { retryStrategically((test) -> config.isAutoSkipNonRecoverableData(), 5, 100); - // (5) consumer will be able to consume 20 messages from last non-deleted ledger + // (5) consumer will be able to consume 19 messages from last non-deleted ledger consumer = client.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe(); - for (int i = 0; i < entriesPerLedger; i++) { + for (int i = 0; i < entriesPerLedger - 1; i++) { msg = consumer.receive(); System.out.println(i); consumer.acknowledge(msg); @@ -297,9 +294,9 @@ public void testTruncateCorruptDataLedger() throws Exception { .statsInterval(0, TimeUnit.SECONDS) .build(); - final int totalMessages = 100; + final int totalMessages = 99; final int totalDataLedgers = 5; - final int entriesPerLedger = totalMessages / totalDataLedgers; + final int entriesPerLedger = 20; final String tenant = "prop"; try { @@ -324,10 +321,8 @@ public void testTruncateCorruptDataLedger() throws Exception { PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topic1).get(); ManagedLedgerImpl ml = (ManagedLedgerImpl) topic.getManagedLedger(); ManagedCursorImpl cursor = (ManagedCursorImpl) ml.getCursors().iterator().next(); - Field configField = ManagedCursorImpl.class.getDeclaredField("config"); - configField.setAccessible(true); // Create multiple data-ledger - ManagedLedgerConfig config = (ManagedLedgerConfig) configField.get(cursor); + ManagedLedgerConfig config = ml.getConfig(); config.setMaxEntriesPerLedger(entriesPerLedger); config.setMinimumRolloverTime(1, TimeUnit.MILLISECONDS); // bookkeeper client @@ -497,10 +492,31 @@ public void testDeleteTopicWithMissingData() throws Exception { // Expected } - // Deletion must succeed - admin.topics().delete(topic); + assertThrows(PulsarAdminException.ServerSideErrorException.class, () -> admin.topics().delete(topic)); + } + + @Test + public void testDeleteTopicWithoutTopicLoaded() throws Exception { + String namespace = BrokerTestUtil.newUniqueName("prop/usc"); + admin.namespaces().createNamespace(namespace); + + String topic = BrokerTestUtil.newUniqueName(namespace + "/my-topic"); + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsar.getBrokerServiceUrl()) + .statsInterval(0, TimeUnit.SECONDS) + .build(); - // Topic will not be there after + @Cleanup + Producer producer = client.newProducer(Schema.STRING) + .topic(topic) + .create(); + + producer.close(); + admin.topics().unload(topic); + + admin.topics().delete(topic); assertEquals(pulsar.getBrokerService().getTopicIfExists(topic).join(), Optional.empty()); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java index 951892f4ebfbc..d7272fcffa964 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java @@ -105,8 +105,12 @@ protected void setup() throws Exception { protected void cleanup() throws Exception { if (pulsarService != null) { pulsarService.close(); + pulsarService = null; + } + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; } - bkEnsemble.stop(); } /** @@ -171,7 +175,7 @@ public void testBookieIsolation() throws Exception { pulsarService = new PulsarService(config); pulsarService.start(); - + @Cleanup PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarService.getWebServiceAddress()).build(); ClusterData clusterData = ClusterData.builder().serviceUrl(pulsarService.getWebServiceAddress()).build(); @@ -231,28 +235,43 @@ public void testBookieIsolation() throws Exception { LedgerManager ledgerManager = getLedgerManager(bookie1); // namespace: ns1 - ManagedLedgerImpl ml = (ManagedLedgerImpl) topic1.getManagedLedger(); - assertEquals(ml.getLedgersInfoAsList().size(), totalLedgers); + ManagedLedgerImpl ml1 = (ManagedLedgerImpl) topic1.getManagedLedger(); + // totalLedgers = totalPublish / totalEntriesPerLedger. (totalPublish = 100, totalEntriesPerLedger = 20.) + // The last ledger is full, a new empty ledger will be created. + // The ledger is created async, so adding a wait is needed. + Awaitility.await().untilAsserted(() -> { + assertEquals(ml1.getLedgersInfoAsList().size(), totalLedgers + 1); + assertEquals(ml1.getCurrentLedgerEntries(), 0); + }); // validate ledgers' ensemble with affinity bookies - assertAffinityBookies(ledgerManager, ml.getLedgersInfoAsList(), defaultBookies); + assertAffinityBookies(ledgerManager, ml1.getLedgersInfoAsList(), defaultBookies); // namespace: ns2 - ml = (ManagedLedgerImpl) topic2.getManagedLedger(); - assertEquals(ml.getLedgersInfoAsList().size(), totalLedgers); + ManagedLedgerImpl ml2 = (ManagedLedgerImpl) topic2.getManagedLedger(); + Awaitility.await().untilAsserted(() -> { + assertEquals(ml2.getLedgersInfoAsList().size(), totalLedgers + 1); + assertEquals(ml2.getCurrentLedgerEntries(), 0); + }); // validate ledgers' ensemble with affinity bookies - assertAffinityBookies(ledgerManager, ml.getLedgersInfoAsList(), isolatedBookies); + assertAffinityBookies(ledgerManager, ml2.getLedgersInfoAsList(), isolatedBookies); // namespace: ns3 - ml = (ManagedLedgerImpl) topic3.getManagedLedger(); - assertEquals(ml.getLedgersInfoAsList().size(), totalLedgers); + ManagedLedgerImpl ml3 = (ManagedLedgerImpl) topic3.getManagedLedger(); + Awaitility.await().untilAsserted(() -> { + assertEquals(ml3.getLedgersInfoAsList().size(), totalLedgers + 1); + assertEquals(ml3.getCurrentLedgerEntries(), 0); + }); // validate ledgers' ensemble with affinity bookies - assertAffinityBookies(ledgerManager, ml.getLedgersInfoAsList(), isolatedBookies); + assertAffinityBookies(ledgerManager, ml3.getLedgersInfoAsList(), isolatedBookies); // namespace: ns4 - ml = (ManagedLedgerImpl) topic4.getManagedLedger(); - assertEquals(ml.getLedgersInfoAsList().size(), totalLedgers); + ManagedLedgerImpl ml4 = (ManagedLedgerImpl) topic4.getManagedLedger(); + Awaitility.await().untilAsserted(() -> { + assertEquals(ml4.getLedgersInfoAsList().size(), totalLedgers + 1); + assertEquals(ml4.getCurrentLedgerEntries(), 0); + }); // validate ledgers' ensemble with affinity bookies - assertAffinityBookies(ledgerManager, ml.getLedgersInfoAsList(), isolatedBookies); + assertAffinityBookies(ledgerManager, ml4.getLedgersInfoAsList(), isolatedBookies); ManagedLedgerClientFactory mlFactory = (ManagedLedgerClientFactory) pulsarService.getManagedLedgerClientFactory(); @@ -304,6 +323,7 @@ public void testSetRackInfoAndAffinityGroupDuringProduce() throws Exception { bookies[3].getBookieId()); ServiceConfiguration config = new ServiceConfiguration(); + config.setTopicLevelPoliciesEnabled(false); config.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); config.setClusterName(cluster); config.setWebServicePort(Optional.of(0)); @@ -328,6 +348,7 @@ public void testSetRackInfoAndAffinityGroupDuringProduce() throws Exception { pulsarService = new PulsarService(config); pulsarService.start(); + @Cleanup PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarService.getWebServiceAddress()).build(); ClusterData clusterData = ClusterData.builder().serviceUrl(pulsarService.getWebServiceAddress()).build(); @@ -387,11 +408,14 @@ public void testSetRackInfoAndAffinityGroupDuringProduce() throws Exception { ManagedLedgerImpl ml2 = (ManagedLedgerImpl) topic2.getManagedLedger(); // namespace: ns2 - assertEquals(ml2.getLedgersInfoAsList().size(), totalLedgers); - + Awaitility.await().untilAsserted(() -> { + assertEquals(ml2.getLedgersInfoAsList().size(), totalLedgers + 1); + assertEquals(ml2.getCurrentLedgerEntries(), 0); + }); List ledgers = ml2.getLedgersInfoAsList(); // validate ledgers' ensemble with affinity bookies - for (int i=1; i> ledgerMetaFuture = ledgerManager.readLedgerMetadata(ledgerId); @@ -471,7 +495,7 @@ public void testStrictBookieIsolation() throws Exception { pulsarService = new PulsarService(config); pulsarService.start(); - + @Cleanup PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarService.getWebServiceAddress()).build(); ClusterData clusterData = ClusterData.builder().serviceUrl(pulsarService.getWebServiceAddress()).build(); @@ -528,28 +552,40 @@ public void testStrictBookieIsolation() throws Exception { LedgerManager ledgerManager = getLedgerManager(bookie1); // namespace: ns1 - ManagedLedgerImpl ml = (ManagedLedgerImpl) topic1.getManagedLedger(); - assertEquals(ml.getLedgersInfoAsList().size(), totalLedgers); + ManagedLedgerImpl ml1 = (ManagedLedgerImpl) topic1.getManagedLedger(); + Awaitility.await().untilAsserted(() -> { + assertEquals(ml1.getLedgersInfoAsList().size(), totalLedgers + 1); + assertEquals(ml1.getCurrentLedgerEntries(), 0); + }); // validate ledgers' ensemble with affinity bookies - assertAffinityBookies(ledgerManager, ml.getLedgersInfoAsList(), defaultBookies); + assertAffinityBookies(ledgerManager, ml1.getLedgersInfoAsList(), defaultBookies); // namespace: ns2 - ml = (ManagedLedgerImpl) topic2.getManagedLedger(); - assertEquals(ml.getLedgersInfoAsList().size(), totalLedgers); + ManagedLedgerImpl ml2 = (ManagedLedgerImpl) topic2.getManagedLedger(); + Awaitility.await().untilAsserted(() -> { + assertEquals(ml2.getLedgersInfoAsList().size(), totalLedgers + 1); + assertEquals(ml2.getCurrentLedgerEntries(), 0); + }); // validate ledgers' ensemble with affinity bookies - assertAffinityBookies(ledgerManager, ml.getLedgersInfoAsList(), isolatedBookies); + assertAffinityBookies(ledgerManager, ml2.getLedgersInfoAsList(), isolatedBookies); // namespace: ns3 - ml = (ManagedLedgerImpl) topic3.getManagedLedger(); - assertEquals(ml.getLedgersInfoAsList().size(), totalLedgers); + ManagedLedgerImpl ml3 = (ManagedLedgerImpl) topic3.getManagedLedger(); + Awaitility.await().untilAsserted(() -> { + assertEquals(ml3.getLedgersInfoAsList().size(), totalLedgers + 1); + assertEquals(ml3.getCurrentLedgerEntries(), 0); + }); // validate ledgers' ensemble with affinity bookies - assertAffinityBookies(ledgerManager, ml.getLedgersInfoAsList(), isolatedBookies); + assertAffinityBookies(ledgerManager, ml3.getLedgersInfoAsList(), isolatedBookies); // namespace: ns4 - ml = (ManagedLedgerImpl) topic4.getManagedLedger(); - assertEquals(ml.getLedgersInfoAsList().size(), totalLedgers); + ManagedLedgerImpl ml4 = (ManagedLedgerImpl) topic4.getManagedLedger(); + Awaitility.await().untilAsserted(() -> { + assertEquals(ml4.getLedgersInfoAsList().size(), totalLedgers + 1); + assertEquals(ml4.getCurrentLedgerEntries(), 0); + }); // validate ledgers' ensemble with affinity bookies - assertAffinityBookies(ledgerManager, ml.getLedgersInfoAsList(), isolatedBookies); + assertAffinityBookies(ledgerManager, ml4.getLedgersInfoAsList(), isolatedBookies); ManagedLedgerClientFactory mlFactory = (ManagedLedgerClientFactory) pulsarService.getManagedLedgerClientFactory(); @@ -612,9 +648,9 @@ public void testBookieIsolationWithSecondaryGroup() throws Exception { config.setBrokerShutdownTimeoutMs(0L); config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); config.setBrokerServicePort(Optional.of(0)); + config.setTopicLevelPoliciesEnabled(false); config.setAdvertisedAddress("localhost"); config.setBookkeeperClientIsolationGroups(brokerBookkeeperClientIsolationGroups); - config.setManagedLedgerDefaultEnsembleSize(2); config.setManagedLedgerDefaultWriteQuorum(2); config.setManagedLedgerDefaultAckQuorum(2); @@ -627,6 +663,7 @@ public void testBookieIsolationWithSecondaryGroup() throws Exception { pulsarService = new PulsarService(config); pulsarService.start(); + @Cleanup PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarService.getWebServiceAddress()).build(); ClusterData clusterData = ClusterData.builder().serviceUrl(pulsarService.getWebServiceAddress()).build(); @@ -686,22 +723,32 @@ public void testBookieIsolationWithSecondaryGroup() throws Exception { LedgerManager ledgerManager = getLedgerManager(bookie1); // namespace: ns1 - ManagedLedgerImpl ml = (ManagedLedgerImpl) topic1.getManagedLedger(); - assertEquals(ml.getLedgersInfoAsList().size(), totalLedgers); + ManagedLedgerImpl ml1 = (ManagedLedgerImpl) topic1.getManagedLedger(); + Awaitility.await().untilAsserted(() -> { + assertEquals(ml1.getLedgersInfoAsList().size(), totalLedgers + 1); + assertEquals(ml1.getCurrentLedgerEntries(), 0); + }); + // validate ledgers' ensemble with affinity bookies - assertAffinityBookies(ledgerManager, ml.getLedgersInfoAsList(), defaultBookies); + assertAffinityBookies(ledgerManager, ml1.getLedgersInfoAsList(), defaultBookies); // namespace: ns2 - ml = (ManagedLedgerImpl) topic2.getManagedLedger(); - assertEquals(ml.getLedgersInfoAsList().size(), totalLedgers); + ManagedLedgerImpl ml2 = (ManagedLedgerImpl) topic2.getManagedLedger(); + Awaitility.await().untilAsserted(() -> { + assertEquals(ml2.getLedgersInfoAsList().size(), totalLedgers + 1); + assertEquals(ml2.getCurrentLedgerEntries(), 0); + }); // validate ledgers' ensemble with affinity bookies - assertAffinityBookies(ledgerManager, ml.getLedgersInfoAsList(), isolatedBookies); + assertAffinityBookies(ledgerManager, ml2.getLedgersInfoAsList(), isolatedBookies); // namespace: ns3 - ml = (ManagedLedgerImpl) topic3.getManagedLedger(); - assertEquals(ml.getLedgersInfoAsList().size(), totalLedgers); + ManagedLedgerImpl ml3 = (ManagedLedgerImpl) topic3.getManagedLedger(); + Awaitility.await().untilAsserted(() -> { + assertEquals(ml3.getLedgersInfoAsList().size(), totalLedgers + 1); + assertEquals(ml3.getCurrentLedgerEntries(), 0); + }); // validate ledgers' ensemble with affinity bookies - assertAffinityBookies(ledgerManager, ml.getLedgersInfoAsList(), isolatedBookies); + assertAffinityBookies(ledgerManager, ml3.getLedgersInfoAsList(), isolatedBookies); ManagedLedgerClientFactory mlFactory = (ManagedLedgerClientFactory) pulsarService.getManagedLedgerClientFactory(); @@ -765,6 +812,7 @@ public void testDeleteIsolationGroup() throws Exception { pulsarService = new PulsarService(config); pulsarService.start(); + @Cleanup PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarService.getWebServiceAddress()).build(); ClusterData clusterData = ClusterData.builder().serviceUrl(pulsarService.getWebServiceAddress()).build(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerInternalClientConfigurationOverrideTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerInternalClientConfigurationOverrideTest.java index 1b1b383e930e3..f33202c3c4033 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerInternalClientConfigurationOverrideTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerInternalClientConfigurationOverrideTest.java @@ -18,17 +18,21 @@ */ package org.apache.pulsar.broker.service; +import static org.testng.Assert.assertEquals; import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.internal.PulsarAdminImpl; +import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.ClusterDataImpl; +import org.apache.pulsar.common.policies.data.Policies; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; - +import lombok.Cleanup; import java.util.Optional; import java.util.Properties; @@ -112,4 +116,40 @@ public void testNamespaceServicePulsarClientConfiguration() { Assert.assertEquals(clientConf.getMemoryLimitBytes(), 100000); } + @Test + public void testOldNamespacePolicy() throws Exception { + + String ns = "prop/oldNsWithDefaultNonNullValues"; + String topic = "persistent://" + ns + "/t1"; + Policies policies = new Policies(); + policies.max_consumers_per_subscription = -1; + policies.max_consumers_per_topic = -1; + policies.max_producers_per_topic = -1; + policies.max_subscriptions_per_topic = -1; + policies.max_topics_per_namespace = -1; + policies.max_unacked_messages_per_consumer = -1; + policies.max_unacked_messages_per_subscription = -1; + admin.namespaces().createNamespace(ns, policies); + + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topic).create(); + PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get(); + assertEquals(topicRef.topicPolicies.getMaxUnackedMessagesOnSubscription().get(), + conf.getMaxUnackedMessagesPerSubscription()); + assertEquals(topicRef.topicPolicies.getMaxConsumersPerSubscription().get(), + conf.getMaxConsumersPerSubscription()); + assertEquals(topicRef.topicPolicies.getMaxConsumerPerTopic().get(), + conf.getMaxConsumersPerTopic()); + assertEquals(topicRef.topicPolicies.getMaxProducersPerTopic().get(), + conf.getMaxProducersPerTopic()); + assertEquals(topicRef.topicPolicies.getMaxSubscriptionsPerTopic().get(), + conf.getMaxSubscriptionsPerTopic()); + assertEquals(topicRef.topicPolicies.getTopicMaxMessageSize().get(), + conf.getMaxMessageSize()); + assertEquals(topicRef.topicPolicies.getMaxUnackedMessagesOnConsumer().get(), + conf.getMaxUnackedMessagesPerConsumer()); + + + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoSubscriptionCreationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoSubscriptionCreationTest.java index 3f2e182874e1d..f1128e389ca45 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoSubscriptionCreationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoSubscriptionCreationTest.java @@ -28,6 +28,7 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.AutoSubscriptionCreationOverride; +import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; @@ -159,15 +160,19 @@ public void testDynamicConfigurationTopicAutoSubscriptionCreation() throws PulsarAdminException, PulsarClientException { pulsar.getConfiguration().setAllowAutoTopicCreation(false); pulsar.getConfiguration().setAllowAutoSubscriptionCreation(true); - admin.brokers().updateDynamicConfiguration("allowAutoSubscriptionCreation", "false"); + String allowAutoSubscriptionCreation = "allowAutoSubscriptionCreation"; + admin.brokers().updateDynamicConfiguration(allowAutoSubscriptionCreation, "false"); String topicString = "persistent://prop/ns-abc/non-partitioned-topic" + UUID.randomUUID(); String subscriptionName = "non-partitioned-topic-sub"; admin.topics().createNonPartitionedTopic(topicString); Assert.assertThrows(PulsarClientException.class, ()-> pulsarClient.newConsumer().topic(topicString).subscriptionName(subscriptionName).subscribe()); - admin.brokers().updateDynamicConfiguration("allowAutoSubscriptionCreation", "true"); - pulsarClient.newConsumer().topic(topicString).subscriptionName(subscriptionName).subscribe(); - assertTrue(admin.topics().getSubscriptions(topicString).contains(subscriptionName)); + admin.brokers().updateDynamicConfiguration(allowAutoSubscriptionCreation, "true"); + Awaitility.await().untilAsserted(() -> { + Assert.assertEquals(admin.brokers().getAllDynamicConfigurations().get(allowAutoSubscriptionCreation), "true"); + pulsarClient.newConsumer().topic(topicString).subscriptionName(subscriptionName).subscribe(); + assertTrue(admin.topics().getSubscriptions(topicString).contains(subscriptionName)); + }); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoTopicCreationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoTopicCreationTest.java index a28b60bbae354..3e735ee4c85b8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoTopicCreationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceAutoTopicCreationTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.service; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateTableViewImpl.TOPIC; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -25,17 +26,24 @@ import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import lombok.Cleanup; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.client.admin.ListNamespaceTopicsOptions; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TopicType; import org.awaitility.Awaitility; import org.testng.Assert; @@ -526,4 +534,57 @@ public void testDynamicConfigurationTopicAutoCreationPartitionedWhenDefaultMoreT } } + @Test + public void testExtensibleLoadManagerImplInternalTopicAutoCreations() + throws PulsarAdminException, PulsarClientException { + pulsar.getConfiguration().setAllowAutoTopicCreation(true); + pulsar.getConfiguration().setAllowAutoTopicCreationType(TopicType.PARTITIONED); + pulsar.getConfiguration().setDefaultNumPartitions(3); + pulsar.getConfiguration().setMaxNumPartitionsPerPartitionedTopic(5); + final String namespaceName = NamespaceName.SYSTEM_NAMESPACE.toString(); + TenantInfoImpl tenantInfo = new TenantInfoImpl(); + tenantInfo.setAllowedClusters(Set.of(configClusterName)); + admin.tenants().createTenant("pulsar", tenantInfo); + admin.namespaces().createNamespace(namespaceName); + admin.topics().createNonPartitionedTopic(TOPIC); + admin.topics().createNonPartitionedTopic(ExtensibleLoadManagerImpl.BROKER_LOAD_DATA_STORE_TOPIC); + admin.topics().createNonPartitionedTopic(ExtensibleLoadManagerImpl.TOP_BUNDLES_LOAD_DATA_STORE_TOPIC); + + // clear the topics to test the auto creation of non-persistent topics. + final var topics = pulsar.getBrokerService().getTopics(); + final var oldTopics = topics.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, + Map.Entry::getValue)); + topics.clear(); + + // The created persistent topic correctly can be found by + // pulsar.getPulsarResources().getTopicResources().persistentTopicExists(topic); + Producer producer = pulsarClient.newProducer().topic(TOPIC).create(); + + // The created non-persistent topics cannot be found, as we did topics.clear() + try { + pulsarClient.newProducer().topic(ExtensibleLoadManagerImpl.BROKER_LOAD_DATA_STORE_TOPIC).create(); + Assert.fail("Create should have failed."); + } catch (PulsarClientException.TopicDoesNotExistException | PulsarClientException.NotFoundException e) { + // expected + } + try { + pulsarClient.newProducer().topic(ExtensibleLoadManagerImpl.TOP_BUNDLES_LOAD_DATA_STORE_TOPIC).create(); + Assert.fail("Create should have failed."); + } catch (PulsarClientException.TopicDoesNotExistException | PulsarClientException.NotFoundException e) { + // expected + } + + oldTopics.forEach((key, val) -> topics.put(key, val)); + + Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> { + List partitionedTopicList = admin.topics().getPartitionedTopicList(namespaceName); + assertEquals(partitionedTopicList.size(), 0); + }); + + producer.close(); + admin.namespaces().deleteNamespace(namespaceName); + admin.tenants().deleteTenant("pulsar"); + + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceChaosTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceChaosTest.java new file mode 100644 index 0000000000000..4187364e46f65 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceChaosTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.testng.Assert.assertEquals; +import java.nio.charset.StandardCharsets; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.partition.PartitionedTopicMetadata; +import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; +import org.apache.pulsar.common.policies.data.TopicType; +import org.apache.pulsar.common.policies.data.impl.AutoTopicCreationOverrideImpl; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.ZooKeeper; +import org.awaitility.reflect.WhiteboxImpl; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class BrokerServiceChaosTest extends CanReconnectZKClientPulsarServiceBaseTest { + + @Override + @BeforeClass(alwaysRun = true, timeOut = 300000) + public void setup() throws Exception { + super.setup(); + } + + @Override + @AfterClass(alwaysRun = true, timeOut = 300000) + public void cleanup() throws Exception { + super.cleanup(); + } + + @Test + public void testFetchPartitionedTopicMetadataWithCacheRefresh() throws Exception { + final String configMetadataStoreConnectString = + WhiteboxImpl.getInternalState(pulsar.getConfigurationMetadataStore(), "zkConnectString"); + final ZooKeeper anotherZKCli = new ZooKeeper(configMetadataStoreConnectString, 5000, null); + // Set policy of auto create topic to PARTITIONED. + final String ns = defaultTenant + "/ns_" + UUID.randomUUID().toString().replaceAll("-", ""); + final TopicName topicName1 = TopicName.get("persistent://" + ns + "/tp1"); + final TopicName topicName2 = TopicName.get("persistent://" + ns + "/tp2"); + admin.namespaces().createNamespace(ns); + AutoTopicCreationOverride autoTopicCreationOverride = + new AutoTopicCreationOverrideImpl.AutoTopicCreationOverrideImplBuilder().allowAutoTopicCreation(true) + .topicType(TopicType.PARTITIONED.toString()) + .defaultNumPartitions(3).build(); + admin.namespaces().setAutoTopicCreationAsync(ns, autoTopicCreationOverride); + // Make the cache of namespace policy is valid. + admin.namespaces().getAutoSubscriptionCreation(ns); + // Trigger the zk node "/admin/partitioned-topics/{namespace}/persistent" created. + admin.topics().createPartitionedTopic(topicName1.toString(), 2); + admin.topics().deletePartitionedTopic(topicName1.toString()); + + // Since there is no partitioned metadata created, the partitions count of metadata will be 0. + PartitionedTopicMetadata partitionedTopicMetadata1 = + pulsar.getBrokerService().fetchPartitionedTopicMetadataAsync(topicName2).get(); + assertEquals(partitionedTopicMetadata1.partitions, 0); + + // Create the partitioned metadata by another zk client. + // Make a error to make the cache could not update. + makeLocalMetadataStoreKeepReconnect(); + anotherZKCli.create("/admin/partitioned-topics/" + ns + "/persistent/" + topicName2.getLocalName(), + "{\"partitions\":3}".getBytes(StandardCharsets.UTF_8), + ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + stopLocalMetadataStoreAlwaysReconnect(); + + // Get the partitioned metadata from cache, there is 90% chance that partitions count of metadata is 0. + PartitionedTopicMetadata partitionedTopicMetadata2 = + pulsar.getBrokerService().fetchPartitionedTopicMetadataAsync(topicName2).get(); + // Note: If you want to reproduce the issue, you can perform validation on the next line. + // assertEquals(partitionedTopicMetadata2.partitions, 0); + + // Verify the new method will return a correct result. + PartitionedTopicMetadata partitionedTopicMetadata3 = + pulsar.getBrokerService().fetchPartitionedTopicMetadataAsync(topicName2, true).get(); + assertEquals(partitionedTopicMetadata3.partitions, 3); + + // cleanup. + admin.topics().deletePartitionedTopic(topicName2.toString()); + anotherZKCli.close(); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java index e9b7ddb991e57..17209c83c13ea 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java @@ -18,17 +18,23 @@ */ package org.apache.pulsar.broker.service; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateTableViewImpl.TOPIC; import static org.apache.pulsar.common.naming.SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN; import static org.apache.pulsar.common.naming.SystemTopicNames.TRANSACTION_COORDINATOR_LOG; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -40,6 +46,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.lang.reflect.Array; import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -59,22 +66,28 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.client.api.ReadHandle; +import org.apache.bookkeeper.mledger.LedgerOffloader; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.bookkeeper.mledger.impl.NullLedgerOffloader; +import org.apache.bookkeeper.mledger.impl.NonAppendableLedgerOffloader; import org.apache.http.HttpResponse; -import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient; import org.apache.pulsar.broker.stats.prometheus.PrometheusRawMetricsProvider; import org.apache.pulsar.client.admin.BrokerStats; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -88,12 +101,14 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionInitialPosition; +import org.apache.pulsar.client.api.SubscriptionMode; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.impl.ClientCnx; import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.PulsarServiceNameResolver; import org.apache.pulsar.client.impl.auth.AuthenticationTls; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; import org.apache.pulsar.common.api.proto.CommandLookupTopicResponse; import org.apache.pulsar.common.api.proto.CommandPartitionedTopicMetadataResponse; import org.apache.pulsar.common.naming.NamespaceBundle; @@ -102,11 +117,20 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.BundlesData; import org.apache.pulsar.common.policies.data.LocalPolicies; +import org.apache.pulsar.common.policies.data.OffloadPolicies; +import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; +import org.apache.pulsar.common.policies.data.OffloadedReadPriority; import org.apache.pulsar.common.policies.data.SubscriptionStats; import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.util.netty.EventLoopUtil; +import org.apache.pulsar.compaction.Compactor; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.MockZooKeeper; import org.awaitility.Awaitility; +import org.glassfish.jersey.client.JerseyClient; +import org.glassfish.jersey.client.JerseyClientBuilder; +import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -383,18 +407,28 @@ public void testConnectionController2() throws Exception { } private void createNewConnectionAndCheckFail(String topicName, ClientBuilder builder) throws Exception { + PulsarClient client = null; try { - createNewConnection(topicName, builder); + client = createNewConnection(topicName, builder); fail("should fail"); } catch (Exception e) { assertTrue(e.getMessage().contains("Reached the maximum number of connections")); + } finally { + if (client != null) { + client.close(); + } } } private PulsarClient createNewConnection(String topicName, ClientBuilder clientBuilder) throws PulsarClientException { PulsarClient client1 = clientBuilder.build(); - client1.newProducer().topic(topicName).create().close(); - return client1; + try { + client1.newProducer().topic(topicName).create().close(); + return client1; + } catch (PulsarClientException e) { + client1.close(); + throw e; + } } private void cleanClient(List clients) throws Exception { @@ -715,7 +749,7 @@ public void testTlsEnabled() throws Exception { fail("should fail"); } catch (Exception e) { - assertTrue(e.getMessage().contains("General OpenSslEngine problem")); + assertTrue(e.getMessage().contains("unable to find valid certification path to requested target")); } finally { pulsarClient.close(); } @@ -752,6 +786,8 @@ public void testTlsEnabledWithoutNonTlsServicePorts() throws Exception { conf.setNumExecutorThreadPoolSize(5); restartBroker(); + PulsarClient pulsarClient = null; + // Access with TLS (Allow insecure TLS connection) try { pulsarClient = PulsarClient.builder().serviceUrl(brokerUrlTls.toString()).enableTls(true) @@ -971,13 +1007,15 @@ public void testLookupThrottlingForClientByClient() throws Exception { conf.setConcurrentLookupRequest(1); conf.setMaxLookupRequest(2); + @Cleanup("shutdownNow") EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(20, false, new DefaultThreadFactory("test-pool", Thread.currentThread().isDaemon())); long reqId = 0xdeadbeef; // Using an AtomicReference in order to reset a new CountDownLatch AtomicReference latchRef = new AtomicReference<>(); latchRef.set(new CountDownLatch(1)); - try (ConnectionPool pool = new ConnectionPool(conf, eventLoop, () -> new ClientCnx(conf, eventLoop) { + try (ConnectionPool pool = new ConnectionPool(InstrumentProvider.NOOP, conf, eventLoop, + () -> new ClientCnx(InstrumentProvider.NOOP, conf, eventLoop) { @Override protected void handleLookupResponse(CommandLookupTopicResponse lookupResult) { try { @@ -997,16 +1035,16 @@ protected void handlePartitionResponse(CommandPartitionedTopicMetadataResponse l } super.handlePartitionResponse(lookupResult); } - })) { + }, null)) { // for PMR // 2 lookup will succeed long reqId1 = reqId++; - ByteBuf request1 = Commands.newPartitionMetadataRequest(topicName, reqId1); + ByteBuf request1 = Commands.newPartitionMetadataRequest(topicName, reqId1, true); CompletableFuture f1 = pool.getConnection(resolver.resolveHost()) .thenCompose(clientCnx -> clientCnx.newLookup(request1, reqId1)); long reqId2 = reqId++; - ByteBuf request2 = Commands.newPartitionMetadataRequest(topicName, reqId2); + ByteBuf request2 = Commands.newPartitionMetadataRequest(topicName, reqId2, true); CompletableFuture f2 = pool.getConnection(resolver.resolveHost()) .thenCompose(clientCnx -> { CompletableFuture future = clientCnx.newLookup(request2, reqId2); @@ -1021,17 +1059,17 @@ protected void handlePartitionResponse(CommandPartitionedTopicMetadataResponse l // 3 lookup will fail latchRef.set(new CountDownLatch(1)); long reqId3 = reqId++; - ByteBuf request3 = Commands.newPartitionMetadataRequest(topicName, reqId3); + ByteBuf request3 = Commands.newPartitionMetadataRequest(topicName, reqId3, true); f1 = pool.getConnection(resolver.resolveHost()) .thenCompose(clientCnx -> clientCnx.newLookup(request3, reqId3)); long reqId4 = reqId++; - ByteBuf request4 = Commands.newPartitionMetadataRequest(topicName, reqId4); + ByteBuf request4 = Commands.newPartitionMetadataRequest(topicName, reqId4, true); f2 = pool.getConnection(resolver.resolveHost()) .thenCompose(clientCnx -> clientCnx.newLookup(request4, reqId4)); long reqId5 = reqId++; - ByteBuf request5 = Commands.newPartitionMetadataRequest(topicName, reqId5); + ByteBuf request5 = Commands.newPartitionMetadataRequest(topicName, reqId5, true); CompletableFuture f3 = pool.getConnection(resolver.resolveHost()) .thenCompose(clientCnx -> { CompletableFuture future = clientCnx.newLookup(request5, reqId5); @@ -1139,7 +1177,7 @@ public void testTopicLoadingOnDisableNamespaceBundle() throws Exception { // try to create topic which should fail as bundle is disable CompletableFuture> futureResult = pulsar.getBrokerService() - .loadOrCreatePersistentTopic(topicName, true, null); + .loadOrCreatePersistentTopic(topicName, true, null, null); try { futureResult.get(); @@ -1152,6 +1190,192 @@ public void testTopicLoadingOnDisableNamespaceBundle() throws Exception { } } + @Test + public void testConcurrentLoadTopicExceedLimitShouldNotBeAutoCreated() throws Exception { + boolean needDeleteTopic = false; + final String namespace = "prop/concurrentLoad"; + try { + // set up broker disable auto create and set concurrent load to 1 qps. + cleanup(); + conf.setMaxConcurrentTopicLoadRequest(1); + conf.setAllowAutoTopicCreation(false); + setup(); + + try { + admin.namespaces().createNamespace(namespace); + } catch (PulsarAdminException.ConflictException e) { + // Ok.. (if test fails intermittently and namespace is already created) + } + + // create 3 topic + String topicName = "persistent://" + namespace + "/my-topic"; + + for (int i = 0; i < 3; i++) { + admin.topics().createNonPartitionedTopic(topicName + "_" + i); + } + + needDeleteTopic = true; + + // try to load 10 topic + ArrayList>> loadFutures = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + // try to create topic which should fail as bundle is disable + CompletableFuture> futureResult = pulsar.getBrokerService() + .loadOrCreatePersistentTopic(topicName + "_" + i, false, null, null); + loadFutures.add(futureResult); + } + + CompletableFuture[] o = (CompletableFuture[]) Array.newInstance(CompletableFuture.class, 10); + CompletableFuture[] completableFutures = loadFutures.toArray(o); + CompletableFuture.allOf(completableFutures).get(); + + // check topic load CompletableFuture. only first three topic should be success. + for (int i = 0; i < 10; i++) { + CompletableFuture> load = loadFutures.get(i); + if (i < 3) { + Assert.assertTrue(load.isDone()); + Assert.assertFalse(load.isCompletedExceptionally()); + } else { + // check topic should not be created if disable autoCreateTopic. + Assert.assertTrue(load.isDone()); + Assert.assertTrue(load.get().isEmpty()); + } + } + } finally { + if (needDeleteTopic) { + String topicName = "persistent://" + namespace + "/my-topic"; + + for (int i = 0; i < 3; i++) { + admin.topics().delete(topicName + "_" + i); + } + } + } + } + + @Test + public void testCheckInactiveSubscriptionsShouldNotDeleteCompactionCursor() throws Exception { + String namespace = "prop/test"; + + // set up broker set compaction threshold. + cleanup(); + conf.setBrokerServiceCompactionThresholdInBytes(8); + setup(); + + try { + admin.namespaces().createNamespace(namespace); + } catch (PulsarAdminException.ConflictException e) { + // Ok.. (if test fails intermittently and namespace is already created) + } + + // set enable subscription expiration. + admin.namespaces().setSubscriptionExpirationTime(namespace, 1); + + String compactionInactiveTestTopic = "persistent://prop/test/testCompactionCursorShouldNotDelete"; + + admin.topics().createNonPartitionedTopic(compactionInactiveTestTopic); + + CompletableFuture> topicCf = + pulsar.getBrokerService().getTopic(compactionInactiveTestTopic, true); + + Optional topicOptional = topicCf.get(); + assertTrue(topicOptional.isPresent()); + + PersistentTopic topic = (PersistentTopic) topicOptional.get(); + + PersistentSubscription sub = (PersistentSubscription) topic.getSubscription(Compactor.COMPACTION_SUBSCRIPTION); + assertNotNull(sub); + + topic.checkCompaction(); + + Field currentCompaction = PersistentTopic.class.getDeclaredField("currentCompaction"); + currentCompaction.setAccessible(true); + CompletableFuture compactionFuture = (CompletableFuture)currentCompaction.get(topic); + + compactionFuture.get(); + + ManagedCursorImpl cursor = (ManagedCursorImpl) sub.getCursor(); + + // make cursor last active time to very small to check if it will be deleted + Field cursorLastActiveField = ManagedCursorImpl.class.getDeclaredField("lastActive"); + cursorLastActiveField.setAccessible(true); + cursorLastActiveField.set(cursor, 0); + + // replace origin object. so we can check if subscription is deleted. + PersistentSubscription spySubscription = Mockito.spy(sub); + topic.getSubscriptions().put(Compactor.COMPACTION_SUBSCRIPTION, spySubscription); + + // trigger inactive check. + topic.checkInactiveSubscriptions(); + + // Compaction subscription should not call delete method. + Mockito.verify(spySubscription, Mockito.never()).delete(); + + // check if the subscription exist. + assertNotNull(topic.getSubscription(Compactor.COMPACTION_SUBSCRIPTION)); + + } + + @Test + public void testCheckInactiveSubscriptionWhenNoMessageToAck() throws Exception { + String namespace = "prop/testInactiveSubscriptionWhenNoMessageToAck"; + + try { + admin.namespaces().createNamespace(namespace); + } catch (PulsarAdminException.ConflictException e) { + // Ok.. (if test fails intermittently and namespace is already created) + } + + String topic = "persistent://" + namespace + "/my-topic"; + Producer producer = pulsarClient.newProducer().topic(topic).create(); + producer.send("test".getBytes()); + producer.close(); + + // create consumer to consume all messages + Consumer consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1") + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe(); + consumer.acknowledge(consumer.receive()); + + Optional topicOptional = pulsar.getBrokerService().getTopic(topic, true).get(); + assertTrue(topicOptional.isPresent()); + PersistentTopic persistentTopic = (PersistentTopic) topicOptional.get(); + + // wait for 1s, but consumer is still connected all the time. + // so subscription should not be deleted. + Thread.sleep(1000); + persistentTopic.checkInactiveSubscriptions(1000); + PersistentTopic finalPersistentTopic = persistentTopic; + Awaitility.await().pollDelay(3, TimeUnit.SECONDS).until(() -> + finalPersistentTopic.getSubscriptions().containsKey("sub1")); + PersistentSubscription sub = persistentTopic.getSubscription("sub1"); + + // shutdown pulsar ungracefully + // disable the updateLastActive method to simulate the ungraceful shutdown + ManagedCursorImpl cursor = (ManagedCursorImpl) sub.getCursor(); + ManagedCursorImpl spyCursor = Mockito.spy(cursor); + doNothing().when(spyCursor).updateLastActive(); + Field cursorField = PersistentSubscription.class.getDeclaredField("cursor"); + cursorField.setAccessible(true); + cursorField.set(sub, spyCursor); + + // restart pulsar + consumer.close(); + restartBroker(); + + admin.lookups().lookupTopic(topic); + topicOptional = pulsar.getBrokerService().getTopic(topic, true).get(); + assertTrue(topicOptional.isPresent()); + persistentTopic = (PersistentTopic) topicOptional.get(); + persistentTopic.checkInactiveSubscriptions(1000); + + // check if subscription is still present + PersistentTopic finalPersistentTopic1 = persistentTopic; + Awaitility.await().pollDelay(3, TimeUnit.SECONDS).until(() -> + finalPersistentTopic1.getSubscriptions().containsKey("sub1")); + sub = persistentTopic.getSubscription("sub1"); + assertNotNull(sub); + } + + /** * Verifies brokerService should not have deadlock and successfully remove topic from topicMap on topic-failure and * it should not introduce deadlock while performing it. @@ -1176,7 +1400,7 @@ public void testTopicFailureShouldNotHaveDeadLock() { BrokerService service = spy(pulsar.getBrokerService()); // create topic will fail to get managedLedgerConfig CompletableFuture failedManagedLedgerConfig = new CompletableFuture<>(); - failedManagedLedgerConfig.completeExceptionally(new NullPointerException("failed to peristent policy")); + failedManagedLedgerConfig.completeExceptionally(new NullPointerException("failed to persistent policy")); doReturn(failedManagedLedgerConfig).when(service).getManagedLedgerConfig(any()); CompletableFuture topicCreation = new CompletableFuture(); @@ -1321,7 +1545,8 @@ public void testStuckTopicUnloading() throws Exception { public void testMetricsProvider() throws IOException { PrometheusRawMetricsProvider rawMetricsProvider = stream -> stream.write("test_metrics{label1=\"xyz\"} 10 \n"); getPulsar().addPrometheusRawMetricsProvider(rawMetricsProvider); - HttpClient httpClient = HttpClientBuilder.create().build(); + @Cleanup + CloseableHttpClient httpClient = HttpClientBuilder.create().build(); final String metricsEndPoint = getPulsar().getWebServiceAddress() + "/metrics"; HttpResponse response = httpClient.execute(new HttpGet(metricsEndPoint)); InputStream inputStream = response.getEntity().getContent(); @@ -1335,81 +1560,6 @@ public void testMetricsProvider() throws IOException { Assert.assertTrue(sb.toString().contains("test_metrics")); } - @Test - public void testPublishRateLimiterMonitor() { - BrokerService.PublishRateLimiterMonitor monitor = new BrokerService.PublishRateLimiterMonitor("test"); - AtomicInteger checkCnt = new AtomicInteger(0); - AtomicInteger refreshCnt = new AtomicInteger(0); - monitor.startOrUpdate(100, checkCnt::incrementAndGet, refreshCnt::incrementAndGet); - Assert.assertEquals(monitor.getTickTimeMs(), 100); - Awaitility.await().until(() -> checkCnt.get() > 0); - Awaitility.await().until(() -> refreshCnt.get() > 0); - - monitor.startOrUpdate(500, checkCnt::incrementAndGet, refreshCnt::incrementAndGet); - Assert.assertEquals(monitor.getTickTimeMs(), 500); - checkCnt.set(0); - refreshCnt.set(0); - Awaitility.await().until(() -> checkCnt.get() > 0); - Awaitility.await().until(() -> refreshCnt.get() > 0); - - monitor.stop(); - Assert.assertEquals(monitor.getTickTimeMs(), 0); - } - - @Test - public void testDynamicBrokerPublisherThrottlingTickTimeMillis() throws Exception { - cleanup(); - conf.setBrokerPublisherThrottlingMaxMessageRate(1000); - conf.setBrokerPublisherThrottlingTickTimeMillis(100); - setup(); - - int prevTickMills = 100; - BrokerService.PublishRateLimiterMonitor monitor = pulsar.getBrokerService().brokerPublishRateLimiterMonitor; - Awaitility.await().until(() -> monitor.getTickTimeMs() == prevTickMills); - - int newTickMills = prevTickMills * 2; - admin.brokers().updateDynamicConfiguration("brokerPublisherThrottlingTickTimeMillis", - String.valueOf(newTickMills)); - Awaitility.await().until(() -> monitor.getTickTimeMs() == newTickMills); - - admin.brokers().updateDynamicConfiguration("brokerPublisherThrottlingTickTimeMillis", - String.valueOf(0)); - Awaitility.await().until(() -> monitor.getTickTimeMs() == 0); - - admin.brokers().updateDynamicConfiguration("brokerPublisherThrottlingTickTimeMillis", - String.valueOf(prevTickMills)); - Awaitility.await().until(() -> monitor.getTickTimeMs() == prevTickMills); - } - - @Test - public void testDynamicTopicPublisherThrottlingTickTimeMillis() throws Exception { - cleanup(); - conf.setPreciseTopicPublishRateLimiterEnable(false); - conf.setMaxPublishRatePerTopicInMessages(1000); - conf.setTopicPublisherThrottlingTickTimeMillis(100); - setup(); - - @Cleanup - Producer producer = pulsarClient.newProducer().topic("persistent://prop/ns-abc/test-topic").create(); - - int prevTickMills = 100; - BrokerService.PublishRateLimiterMonitor monitor = pulsar.getBrokerService().topicPublishRateLimiterMonitor; - Awaitility.await().until(() -> monitor.getTickTimeMs() == prevTickMills); - - int newTickMills = prevTickMills * 2; - admin.brokers().updateDynamicConfiguration("topicPublisherThrottlingTickTimeMillis", - String.valueOf(newTickMills)); - Awaitility.await().until(() -> monitor.getTickTimeMs() == newTickMills); - - admin.brokers().updateDynamicConfiguration("topicPublisherThrottlingTickTimeMillis", - String.valueOf(0)); - Awaitility.await().until(() -> monitor.getTickTimeMs() == 0); - - admin.brokers().updateDynamicConfiguration("topicPublisherThrottlingTickTimeMillis", - String.valueOf(prevTickMills)); - Awaitility.await().until(() -> monitor.getTickTimeMs() == prevTickMills); - } - @Test public void shouldNotPreventCreatingTopicWhenNonexistingTopicIsCached() throws Exception { // run multiple iterations to increase the chance of reproducing a race condition in the topic cache @@ -1462,8 +1612,10 @@ public void testIsSystemTopic() { assertTrue(brokerService.isSystemTopic(TRANSACTION_COORDINATOR_ASSIGN)); assertTrue(brokerService.isSystemTopic(TRANSACTION_COORDINATOR_LOG)); - NamespaceName heartbeatNamespaceV1 = NamespaceService.getHeartbeatNamespace(pulsar.getAdvertisedAddress(), pulsar.getConfig()); - NamespaceName heartbeatNamespaceV2 = NamespaceService.getHeartbeatNamespaceV2(pulsar.getAdvertisedAddress(), pulsar.getConfig()); + NamespaceName heartbeatNamespaceV1 = NamespaceService + .getHeartbeatNamespace(pulsar.getBrokerId(), pulsar.getConfig()); + NamespaceName heartbeatNamespaceV2 = NamespaceService + .getHeartbeatNamespaceV2(pulsar.getBrokerId(), pulsar.getConfig()); assertTrue(brokerService.isSystemTopic("persistent://" + heartbeatNamespaceV1.toString() + "/healthcheck")); assertTrue(brokerService.isSystemTopic(heartbeatNamespaceV2.toString() + "/healthcheck")); } @@ -1478,7 +1630,7 @@ public void testGetTopic() throws Exception { producer1.close(); PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName.toString(), false).get().get(); persistentTopic.close().join(); - List topics = new ArrayList<>(pulsar.getBrokerService().getTopics().keys()); + List topics = new ArrayList<>(pulsar.getBrokerService().getTopics().keySet()); topics.removeIf(item -> item.contains(SystemTopicNames.NAMESPACE_EVENTS_LOCAL_NAME)); Assert.assertEquals(topics.size(), 0); @Cleanup @@ -1513,4 +1665,257 @@ public void testDynamicConfigurationsForceDeleteTenantAllowed() throws Exception assertTrue(conf.isForceDeleteTenantAllowed()); }); } + + @Test + public void testMetricsPersistentTopicLoadFails() throws Exception { + final String namespace = "prop/" + UUID.randomUUID().toString().replaceAll("-", ""); + String topic = "persistent://" + namespace + "/topic1_" + UUID.randomUUID(); + admin.namespaces().createNamespace(namespace); + admin.topics().createNonPartitionedTopic(topic); + admin.topics().unload(topic); + + // Inject an error that makes the topic load fails. + AtomicBoolean failMarker = new AtomicBoolean(true); + mockZooKeeper.failConditional(KeeperException.Code.NODEEXISTS, (op, path) -> { + if (failMarker.get() && op.equals(MockZooKeeper.Op.SET) && + path.endsWith(TopicName.get(topic).getPersistenceNamingEncoding())) { + return true; + } + return false; + }); + + // Do test + CompletableFuture> producer = pulsarClient.newProducer().topic(topic).createAsync(); + JerseyClient httpClient = JerseyClientBuilder.createClient(); + Awaitility.await().until(() -> { + String response = httpClient.target(pulsar.getWebServiceAddress()).path("/metrics/") + .request().get(String.class); + Multimap metricMap = PrometheusMetricsClient.parseMetrics(response); + if (!metricMap.containsKey("pulsar_topic_load_failed_count")) { + return false; + } + double topic_load_failed_count = 0; + for (PrometheusMetricsClient.Metric metric : metricMap.get("pulsar_topic_load_failed_count")) { + topic_load_failed_count += metric.value; + } + return topic_load_failed_count >= 1D; + }); + + // Remove the injection. + failMarker.set(false); + // cleanup. + httpClient.close(); + producer.join().close(); + admin.topics().delete(topic); + admin.namespaces().deleteNamespace(namespace); + } + + @Test + public void testMetricsNonPersistentTopicLoadFails() throws Exception { + final String namespace = "prop/" + UUID.randomUUID().toString().replaceAll("-", ""); + String topic = "non-persistent://" + namespace + "/topic1_" + UUID.randomUUID(); + admin.namespaces().createNamespace(namespace); + + // Inject an error that makes the topic load fails. + // Since we did not set a topic factory name, the "topicFactory" variable is null, inject a mocked + // "topicFactory". + Field fieldTopicFactory = BrokerService.class.getDeclaredField("topicFactory"); + fieldTopicFactory.setAccessible(true); + TopicFactory originalTopicFactory = (TopicFactory) fieldTopicFactory.get(pulsar.getBrokerService()); + assertNull(originalTopicFactory); + TopicFactory mockedTopicFactory = mock(TopicFactory.class); + when(mockedTopicFactory.create(anyString(), any(), any(), any())) + .thenThrow(new RuntimeException("mocked error")); + fieldTopicFactory.set(pulsar.getBrokerService(), mockedTopicFactory); + + // Do test. + CompletableFuture> producer = pulsarClient.newProducer().topic(topic).createAsync(); + JerseyClient httpClient = JerseyClientBuilder.createClient(); + Awaitility.await().until(() -> { + String response = httpClient.target(pulsar.getWebServiceAddress()).path("/metrics/") + .request().get(String.class); + Multimap metricMap = PrometheusMetricsClient.parseMetrics(response); + if (!metricMap.containsKey("pulsar_topic_load_failed_count")) { + return false; + } + double topic_load_failed_count = 0; + for (PrometheusMetricsClient.Metric metric : metricMap.get("pulsar_topic_load_failed_count")) { + topic_load_failed_count += metric.value; + } + return topic_load_failed_count >= 1D; + }); + + // Remove the injection. + fieldTopicFactory.set(pulsar.getBrokerService(), null); + + // cleanup. + httpClient.close(); + producer.join().close(); + admin.topics().delete(topic); + admin.namespaces().deleteNamespace(namespace); + } + + @Test + public void testIsSystemTopicAllowAutoTopicCreationAsync() throws Exception { + BrokerService brokerService = pulsar.getBrokerService(); + assertFalse(brokerService.isAllowAutoTopicCreationAsync( + TOPIC).get()); + assertTrue(brokerService.isAllowAutoTopicCreationAsync( + "persistent://pulsar/system/my-system-topic").get()); + } + + @Test + public void testDuplicateAcknowledgement() throws Exception { + final String ns = "prop/ns-test"; + + admin.namespaces().createNamespace(ns, 2); + final String topicName = "persistent://prop/ns-test/duplicated-acknowledgement-test"; + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topicName) + .create(); + @Cleanup + Consumer consumer1 = pulsarClient.newConsumer() + .topic(topicName) + .subscriptionName("sub-1") + .acknowledgmentGroupTime(0, TimeUnit.SECONDS) + .subscriptionType(SubscriptionType.Shared) + .isAckReceiptEnabled(true) + .subscribe(); + producer.send("1".getBytes(StandardCharsets.UTF_8)); + Message message = consumer1.receive(); + consumer1.acknowledge(message); + consumer1.acknowledge(message); + assertEquals(admin.topics().getStats(topicName).getSubscriptions() + .get("sub-1").getUnackedMessages(), 0); + } + + @Test + public void testUnsubscribeNonDurableSub() throws Exception { + final String ns = "prop/ns-test"; + final String topic = ns + "/testUnsubscribeNonDurableSub"; + + admin.namespaces().createNamespace(ns, 2); + admin.topics().createPartitionedTopic(String.format("persistent://%s", topic), 1); + + pulsarClient.newProducer(Schema.STRING).topic(topic).create().close(); + @Cleanup + Consumer consumer = pulsarClient + .newConsumer(Schema.STRING) + .topic(topic) + .subscriptionMode(SubscriptionMode.NonDurable) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionName("sub1") + .subscriptionType(SubscriptionType.Shared) + .subscribe(); + try { + consumer.unsubscribe(); + } catch (Exception ex) { + fail("Unsubscribe failed"); + } + } + + + @Test + public void testOffloadConfShouldNotAppliedForSystemTopic() throws PulsarAdminException { + final String driver = "aws-s3"; + final String region = "test-region"; + final String bucket = "test-bucket"; + final String role = "test-role"; + final String roleSessionName = "test-role-session-name"; + final String credentialId = "test-credential-id"; + final String credentialSecret = "test-credential-secret"; + final String endPoint = "test-endpoint"; + final Integer maxBlockSizeInBytes = 5; + final Integer readBufferSizeInBytes = 2; + final Long offloadThresholdInBytes = 10L; + final Long offloadThresholdInSeconds = 1000L; + final Long offloadDeletionLagInMillis = 5L; + + final OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.create( + driver, + region, + bucket, + endPoint, + role, + roleSessionName, + credentialId, + credentialSecret, + maxBlockSizeInBytes, + readBufferSizeInBytes, + offloadThresholdInBytes, + offloadThresholdInSeconds, + offloadDeletionLagInMillis, + OffloadedReadPriority.TIERED_STORAGE_FIRST + ); + + var fakeOffloader = new LedgerOffloader() { + @Override + public String getOffloadDriverName() { + return driver; + } + + @Override + public CompletableFuture offload(ReadHandle ledger, UUID uid, Map extraMetadata) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture readOffloaded(long ledgerId, UUID uid, Map offloadDriverMetadata) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture deleteOffloaded(long ledgerId, UUID uid, Map offloadDriverMetadata) { + return CompletableFuture.completedFuture(null); + } + + @Override + public OffloadPolicies getOffloadPolicies() { + return offloadPolicies; + } + + @Override + public void close() { + } + }; + + final BrokerService brokerService = pulsar.getBrokerService(); + final String namespace = "prop/" + UUID.randomUUID(); + admin.namespaces().createNamespace(namespace); + admin.namespaces().setOffloadPolicies(namespace, offloadPolicies); + Awaitility.await().untilAsserted(() -> { + OffloadPolicies policiesGot = admin.namespaces().getOffloadPolicies(namespace); + assertNotNull(policiesGot); + }); + + // Inject the cache to avoid real load off-loader jar + final Map ledgerOffloaderMap = pulsar.getLedgerOffloaderMap(); + ledgerOffloaderMap.put(NamespaceName.get(namespace), fakeOffloader); + + // (1) test normal topic + final String normalTopic = "persistent://" + namespace + "/" + UUID.randomUUID(); + var managedLedgerConfig = brokerService.getManagedLedgerConfig(TopicName.get(normalTopic)).join(); + + Assert.assertEquals(managedLedgerConfig.getLedgerOffloader(), fakeOffloader); + + // (2) test system topic + for (String eventTopicName : SystemTopicNames.EVENTS_TOPIC_NAMES) { + boolean offloadPoliciesExists = false; + try { + OffloadPolicies policiesGot = + admin.namespaces().getOffloadPolicies(TopicName.get(eventTopicName).getNamespace()); + offloadPoliciesExists = policiesGot != null; + } catch (PulsarAdminException.NotFoundException notFoundException) { + offloadPoliciesExists = false; + } + var managedLedgerConfig2 = brokerService.getManagedLedgerConfig(TopicName.get(eventTopicName)).join(); + if (offloadPoliciesExists) { + Assert.assertTrue(managedLedgerConfig2.getLedgerOffloader() instanceof NonAppendableLedgerOffloader); + } else { + Assert.assertEquals(managedLedgerConfig2.getLedgerOffloader(), NullLedgerOffloader.INSTANCE); + } + } + } } + diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceThrottlingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceThrottlingTest.java index b2cfe63e2e5b4..ddf0fae13545e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceThrottlingTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceThrottlingTest.java @@ -18,8 +18,9 @@ */ package org.apache.pulsar.broker.service; +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat; +import static org.awaitility.Awaitility.waitAtMost; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import io.netty.buffer.ByteBuf; @@ -38,6 +39,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import lombok.Cleanup; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; @@ -45,6 +47,7 @@ import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.PulsarServiceNameResolver; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.util.netty.EventLoopUtil; import org.testng.annotations.AfterMethod; @@ -66,18 +69,46 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + @Override + protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder builder) { + super.customizeMainPulsarTestContextBuilder(builder); + builder.enableOpenTelemetry(true); + } + /** - * Verifies: updating zk-throttling node reflects broker-maxConcurrentLookupRequest and updates semaphore. - * - * @throws Exception + * Verifies: updating zk-throttling node reflects broker-maxConcurrentLookupRequest and updates semaphore, as well + * as the related limit metric value. */ @Test public void testThrottlingLookupRequestSemaphore() throws Exception { - BrokerService service = pulsar.getBrokerService(); - assertNotEquals(service.lookupRequestSemaphore.get().availablePermits(), 0); - admin.brokers().updateDynamicConfiguration("maxConcurrentLookupRequest", Integer.toString(0)); - Thread.sleep(1000); - assertEquals(service.lookupRequestSemaphore.get().availablePermits(), 0); + var lookupRequestSemaphore = pulsar.getBrokerService().lookupRequestSemaphore; + var configName = "maxConcurrentLookupRequest"; + var metricName = BrokerService.TOPIC_LOOKUP_LIMIT_METRIC_NAME; + // Validate that the configuration has not been overridden. + assertThat(admin.brokers().getAllDynamicConfigurations()).doesNotContainKey(configName); + assertOtelMetricLongSumValue(metricName, 50_000); + assertThat(lookupRequestSemaphore.get().availablePermits()).isNotEqualTo(0); + admin.brokers().updateDynamicConfiguration(configName, Integer.toString(0)); + waitAtMost(1, TimeUnit.SECONDS).until(() -> lookupRequestSemaphore.get().availablePermits() == 0); + assertOtelMetricLongSumValue(metricName, 0); + } + + /** + * Verifies: updating zk-throttling node reflects broker-maxConcurrentTopicLoadRequest and updates semaphore, as + * well as the related limit metric value. + */ + @Test + public void testThrottlingTopicLoadRequestSemaphore() throws Exception { + var topicLoadRequestSemaphore = pulsar.getBrokerService().topicLoadRequestSemaphore; + var configName = "maxConcurrentTopicLoadRequest"; + var metricName = BrokerService.TOPIC_LOAD_LIMIT_METRIC_NAME; + // Validate that the configuration has not been overridden. + assertThat(admin.brokers().getAllDynamicConfigurations()).doesNotContainKey(configName); + assertOtelMetricLongSumValue(metricName, 5_000); + assertThat(topicLoadRequestSemaphore.get().availablePermits()).isNotEqualTo(0); + admin.brokers().updateDynamicConfiguration(configName, Integer.toString(0)); + waitAtMost(1, TimeUnit.SECONDS).until(() -> topicLoadRequestSemaphore.get().availablePermits() == 0); + assertOtelMetricLongSumValue(metricName, 0); } /** @@ -159,7 +190,7 @@ public void testLookupThrottlingForClientByBroker() throws Exception { EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(20, false, new DefaultThreadFactory("test-pool", Thread.currentThread().isDaemon())); ExecutorService executor = Executors.newFixedThreadPool(10); - try (ConnectionPool pool = new ConnectionPool(conf, eventLoop)) { + try (ConnectionPool pool = new ConnectionPool(InstrumentProvider.NOOP, conf, eventLoop, null)) { final int totalConsumers = 20; List> futures = new ArrayList<>(); @@ -167,7 +198,7 @@ public void testLookupThrottlingForClientByBroker() throws Exception { for (int i = 0; i < totalConsumers; i++) { long reqId = 0xdeadbeef + i; Future f = executor.submit(() -> { - ByteBuf request = Commands.newPartitionMetadataRequest(topicName, reqId); + ByteBuf request = Commands.newPartitionMetadataRequest(topicName, reqId, true); pool.getConnection(resolver.resolveHost()) .thenCompose(clientCnx -> clientCnx.newLookup(request, reqId)) .get(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java new file mode 100644 index 0000000000000..787b4d3154e90 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import com.google.common.collect.Sets; +import com.google.common.io.Resources; +import io.netty.channel.Channel; +import java.net.URL; +import java.nio.channels.SelectionKey; +import java.util.Collections; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.policies.data.TopicType; +import org.apache.pulsar.metadata.impl.ZKMetadataStore; +import org.apache.pulsar.tests.TestRetrySupport; +import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.apache.pulsar.zookeeper.ZookeeperServerTest; +import org.apache.zookeeper.ClientCnxn; +import org.apache.zookeeper.ZooKeeper; +import org.awaitility.reflect.WhiteboxImpl; + +@Slf4j +public abstract class CanReconnectZKClientPulsarServiceBaseTest extends TestRetrySupport { + protected final String defaultTenant = "public"; + protected final String defaultNamespace = defaultTenant + "/default"; + final static String caCertPath = Resources.getResource("certificate-authority/certs/ca.cert.pem") + .getPath(); + final static String brokerCertPath = + Resources.getResource("certificate-authority/server-keys/broker.cert.pem").getPath(); + final static String brokerKeyPath = + Resources.getResource("certificate-authority/server-keys/broker.key-pk8.pem").getPath(); + protected int numberOfBookies = 3; + protected final String clusterName = "r1"; + protected URL url; + protected URL urlTls; + protected ServiceConfiguration config = new ServiceConfiguration(); + protected ZookeeperServerTest brokerConfigZk; + protected LocalBookkeeperEnsemble bkEnsemble; + protected PulsarService pulsar; + protected BrokerService broker; + protected PulsarAdmin admin; + protected PulsarClient client; + protected ZooKeeper localZkOfBroker; + protected Object localMetaDataStoreClientCnx; + protected final AtomicBoolean LocalMetadataStoreInReconnectFinishSignal = new AtomicBoolean(); + + protected void startZKAndBK() throws Exception { + // Start ZK. + brokerConfigZk = new ZookeeperServerTest(0); + brokerConfigZk.start(); + + // Start BK. + bkEnsemble = new LocalBookkeeperEnsemble(numberOfBookies, 0, () -> 0); + bkEnsemble.start(); + } + + protected void startBrokers() throws Exception { + // Start brokers. + setConfigDefaults(config, clusterName, bkEnsemble, brokerConfigZk); + pulsar = new PulsarService(config); + pulsar.start(); + broker = pulsar.getBrokerService(); + ZKMetadataStore zkMetadataStore = (ZKMetadataStore) pulsar.getLocalMetadataStore(); + localZkOfBroker = zkMetadataStore.getZkClient(); + ClientCnxn cnxn = WhiteboxImpl.getInternalState(localZkOfBroker, "cnxn"); + Object sendThread = WhiteboxImpl.getInternalState(cnxn, "sendThread"); + localMetaDataStoreClientCnx = WhiteboxImpl.getInternalState(sendThread, "clientCnxnSocket"); + + url = new URL(pulsar.getWebServiceAddress()); + urlTls = new URL(pulsar.getWebServiceAddressTls()); + admin = PulsarAdmin.builder().serviceHttpUrl(url.toString()).build(); + client = PulsarClient.builder().serviceUrl(url.toString()).build(); + } + + protected void makeLocalMetadataStoreKeepReconnect() throws Exception { + if (!LocalMetadataStoreInReconnectFinishSignal.compareAndSet(false, true)) { + throw new RuntimeException("Local metadata store is already keeping reconnect"); + } + if (localMetaDataStoreClientCnx.getClass().getSimpleName().equals("ClientCnxnSocketNIO")) { + makeLocalMetadataStoreKeepReconnectNIO(); + } else { + // ClientCnxnSocketNetty. + makeLocalMetadataStoreKeepReconnectNetty(); + } + } + + protected void makeLocalMetadataStoreKeepReconnectNIO() { + new Thread(() -> { + while (LocalMetadataStoreInReconnectFinishSignal.get()) { + try { + SelectionKey sockKey = WhiteboxImpl.getInternalState(localMetaDataStoreClientCnx, "sockKey"); + if (sockKey != null) { + sockKey.channel().close(); + } + // Prevents high cpu usage. + Thread.sleep(5); + } catch (Exception e) { + log.error("Try close the ZK connection of local metadata store failed: {}", e.toString()); + } + } + }).start(); + } + + protected void makeLocalMetadataStoreKeepReconnectNetty() { + new Thread(() -> { + while (LocalMetadataStoreInReconnectFinishSignal.get()) { + try { + Channel channel = WhiteboxImpl.getInternalState(localMetaDataStoreClientCnx, "channel"); + if (channel != null) { + channel.close(); + } + // Prevents high cpu usage. + Thread.sleep(5); + } catch (Exception e) { + log.error("Try close the ZK connection of local metadata store failed: {}", e.toString()); + } + } + }).start(); + } + + protected void stopLocalMetadataStoreAlwaysReconnect() { + LocalMetadataStoreInReconnectFinishSignal.set(false); + } + + protected void createDefaultTenantsAndClustersAndNamespace() throws Exception { + admin.clusters().createCluster(clusterName, ClusterData.builder() + .serviceUrl(url.toString()) + .serviceUrlTls(urlTls.toString()) + .brokerServiceUrl(pulsar.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar.getBrokerServiceUrlTls()) + .brokerClientTlsEnabled(false) + .build()); + + admin.tenants().createTenant(defaultTenant, new TenantInfoImpl(Collections.emptySet(), + Sets.newHashSet(clusterName))); + + admin.namespaces().createNamespace(defaultNamespace, Sets.newHashSet(clusterName)); + } + + @Override + protected void setup() throws Exception { + incrementSetupNumber(); + + log.info("--- Starting OneWayReplicatorTestBase::setup ---"); + + startZKAndBK(); + + startBrokers(); + + createDefaultTenantsAndClustersAndNamespace(); + + Thread.sleep(100); + log.info("--- OneWayReplicatorTestBase::setup completed ---"); + } + + private void setConfigDefaults(ServiceConfiguration config, String clusterName, + LocalBookkeeperEnsemble bookkeeperEnsemble, ZookeeperServerTest brokerConfigZk) { + config.setClusterName(clusterName); + config.setAdvertisedAddress("localhost"); + config.setWebServicePort(Optional.of(0)); + config.setWebServicePortTls(Optional.of(0)); + config.setMetadataStoreUrl("zk:127.0.0.1:" + bookkeeperEnsemble.getZookeeperPort()); + config.setConfigurationMetadataStoreUrl("zk:127.0.0.1:" + brokerConfigZk.getZookeeperPort() + "/foo"); + config.setBrokerDeleteInactiveTopicsEnabled(false); + config.setBrokerDeleteInactiveTopicsFrequencySeconds(60); + config.setBrokerShutdownTimeoutMs(0L); + config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); + config.setBrokerServicePort(Optional.of(0)); + config.setBrokerServicePortTls(Optional.of(0)); + config.setBacklogQuotaCheckIntervalInSeconds(5); + config.setDefaultNumberOfNamespaceBundles(1); + config.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED); + config.setEnableReplicatedSubscriptions(true); + config.setReplicatedSubscriptionsSnapshotFrequencyMillis(1000); + config.setTlsTrustCertsFilePath(caCertPath); + config.setTlsCertificateFilePath(brokerCertPath); + config.setTlsKeyFilePath(brokerKeyPath); + } + + @Override + protected void cleanup() throws Exception { + markCurrentSetupNumberCleaned(); + log.info("--- Shutting down ---"); + + stopLocalMetadataStoreAlwaysReconnect(); + + // Stop brokers. + if (client != null) { + client.close(); + client = null; + } + if (admin != null) { + admin.close(); + admin = null; + } + if (pulsar != null) { + pulsar.close(); + pulsar = null; + } + + // Stop ZK and BK. + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; + } + if (brokerConfigZk != null) { + brokerConfigZk.stop(); + brokerConfigZk = null; + } + + // Reset configs. + config = new ServiceConfiguration(); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java index df4f66c43d2b4..e56a3495600f0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java @@ -25,42 +25,46 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; - +import com.google.common.collect.Sets; import java.lang.reflect.Method; import java.net.URL; +import java.util.List; +import java.util.Optional; import java.util.concurrent.TimeUnit; - +import lombok.Cleanup; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.ClusterData.ClusterUrl; +import org.apache.pulsar.common.policies.data.ClusterPolicies.ClusterUrl; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; +import org.testng.annotations.Factory; import org.testng.annotations.Test; -import com.google.common.collect.Sets; - -import lombok.Cleanup; - -@Test(groups = "broker") +@Test(groups = "cluster-migration") public class ClusterMigrationTest { private static final Logger log = LoggerFactory.getLogger(ClusterMigrationTest.class); protected String methodName; String namespace = "pulsar/migrationNs"; + String namespaceNotToMigrate = "pulsar/notToMigrateNs"; TestBroker broker1, broker2, broker3, broker4; URL url1; @@ -84,30 +88,39 @@ public class ClusterMigrationTest { PulsarService pulsar4; PulsarAdmin admin4; - @DataProvider(name = "TopicsubscriptionTypes") - public Object[][] subscriptionTypes() { + String loadManagerClassName; + + @DataProvider(name="NamespaceMigrationTopicSubscriptionTypes") + public Object[][] namespaceMigrationSubscriptionTypes() { return new Object[][] { - {true, SubscriptionType.Shared}, - {true, SubscriptionType.Key_Shared}, - {true, SubscriptionType.Shared}, - {true, SubscriptionType.Key_Shared}, - - {false, SubscriptionType.Shared}, - {false, SubscriptionType.Key_Shared}, - {false, SubscriptionType.Shared}, - {false, SubscriptionType.Key_Shared}, + {SubscriptionType.Shared, true, false}, + {SubscriptionType.Shared, false, true}, + {SubscriptionType.Shared, true, true}, + }; + } + + @DataProvider(name = "loadManagerClassName") + public static Object[][] loadManagerClassName() { + return new Object[][]{ + {ModularLoadManagerImpl.class.getName()}, + {ExtensibleLoadManagerImpl.class.getName()} }; } + @Factory(dataProvider = "loadManagerClassName") + public ClusterMigrationTest(String loadManagerClassName) { + this.loadManagerClassName = loadManagerClassName; + } + @BeforeMethod(alwaysRun = true, timeOut = 300000) public void setup() throws Exception { log.info("--- Starting ReplicatorTestBase::setup ---"); - broker1 = new TestBroker("r1"); - broker2 = new TestBroker("r2"); - broker3 = new TestBroker("r3"); - broker4 = new TestBroker("r4"); + broker1 = new TestBroker("r1", loadManagerClassName); + broker2 = new TestBroker("r2", loadManagerClassName); + broker3 = new TestBroker("r3", loadManagerClassName); + broker4 = new TestBroker("r4", loadManagerClassName); pulsar1 = broker1.getPulsarService(); url1 = new URL(pulsar1.getWebServiceAddress()); @@ -167,22 +180,28 @@ public void setup() throws Exception { .brokerServiceUrlTls(pulsar4.getBrokerServiceUrlTls()).build()); // Setting r3 as replication cluster for r1 - admin1.tenants().createTenant("pulsar", + updateTenantInfo(admin1, "pulsar", new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet("r1", "r3"))); - admin3.tenants().createTenant("pulsar", + updateTenantInfo(admin3, "pulsar", new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet("r1", "r3"))); admin1.namespaces().createNamespace(namespace, Sets.newHashSet("r1", "r3")); admin3.namespaces().createNamespace(namespace); admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r3")); + admin1.namespaces().createNamespace(namespaceNotToMigrate, Sets.newHashSet("r1", "r3")); + admin3.namespaces().createNamespace(namespaceNotToMigrate); + admin1.namespaces().setNamespaceReplicationClusters(namespaceNotToMigrate, Sets.newHashSet("r1", "r3")); // Setting r4 as replication cluster for r2 - admin2.tenants().createTenant("pulsar", + updateTenantInfo(admin2, "pulsar", new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet("r2", "r4"))); - admin4.tenants().createTenant("pulsar", + updateTenantInfo(admin4,"pulsar", new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet("r2", "r4"))); admin2.namespaces().createNamespace(namespace, Sets.newHashSet("r2", "r4")); admin4.namespaces().createNamespace(namespace); admin2.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r2", "r4")); + admin2.namespaces().createNamespace(namespaceNotToMigrate, Sets.newHashSet("r2", "r4")); + admin4.namespaces().createNamespace(namespaceNotToMigrate); + admin2.namespaces().setNamespaceReplicationClusters(namespaceNotToMigrate, Sets.newHashSet("r2", "r4")); assertEquals(admin1.clusters().getCluster("r1").getServiceUrl(), url1.toString()); assertEquals(admin2.clusters().getCluster("r2").getServiceUrl(), url2.toString()); @@ -198,9 +217,21 @@ public void setup() throws Exception { } + protected void updateTenantInfo(PulsarAdmin admin, String tenant, TenantInfoImpl tenantInfo) throws Exception { + if (!admin.tenants().getTenants().contains(tenant)) { + admin.tenants().createTenant(tenant, tenantInfo); + } else { + admin.tenants().updateTenant(tenant, tenantInfo); + } + } + @AfterMethod(alwaysRun = true, timeOut = 300000) protected void cleanup() throws Exception { log.info("--- Shutting down ---"); + admin1.close(); + admin2.close(); + admin3.close(); + admin4.close(); broker1.cleanup(); broker2.cleanup(); broker3.cleanup(); @@ -229,11 +260,11 @@ public void beforeMethod(Method m) throws Exception { * (11) Restart Broker-1 and connect producer/consumer on cluster-1 * @throws Exception */ - @Test(dataProvider = "TopicsubscriptionTypes") - public void testClusterMigration(boolean persistent, SubscriptionType subType) throws Exception { + @Test + public void testClusterMigration() throws Exception { log.info("--- Starting ReplicatorTest::testClusterMigration ---"); final String topicName = BrokerTestUtil - .newUniqueName((persistent ? "persistent" : "non-persistent") + "://" + namespace + "/migrationTopic"); + .newUniqueName("persistent://" + namespace + "/migrationTopic"); @Cleanup PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS) @@ -241,7 +272,7 @@ public void testClusterMigration(boolean persistent, SubscriptionType subType) t // cluster-1 producer/consumer Producer producer1 = client1.newProducer().topic(topicName).enableBatching(false) .producerName("cluster1-1").messageRoutingMode(MessageRoutingMode.SinglePartition).create(); - Consumer consumer1 = client1.newConsumer().topic(topicName).subscriptionType(subType) + Consumer consumer1 = client1.newConsumer().topic(topicName).subscriptionType(SubscriptionType.Shared) .subscriptionName("s1").subscribe(); AbstractTopic topic1 = (AbstractTopic) pulsar1.getBrokerService().getTopic(topicName, false).getNow(null).get(); retryStrategically((test) -> !topic1.getProducers().isEmpty(), 5, 500); @@ -265,8 +296,10 @@ public void testClusterMigration(boolean persistent, SubscriptionType subType) t AbstractTopic topic2 = (AbstractTopic) pulsar2.getBrokerService().getTopic(topicName, false).getNow(null).get(); assertFalse(topic2.getProducers().isEmpty()); - ClusterUrl migratedUrl = new ClusterUrl(pulsar2.getBrokerServiceUrl(), pulsar2.getBrokerServiceUrlTls()); + ClusterUrl migratedUrl = new ClusterUrl(pulsar2.getWebServiceAddress(), pulsar2.getWebServiceAddressTls(), + pulsar2.getBrokerServiceUrl(), pulsar2.getBrokerServiceUrlTls()); admin1.clusters().updateClusterMigration("r1", true, migratedUrl); + assertEquals(admin1.clusters().getClusterMigration("r1").getMigratedClusterUrl(), migratedUrl); retryStrategically((test) -> { try { @@ -278,7 +311,6 @@ public void testClusterMigration(boolean persistent, SubscriptionType subType) t return false; }, 10, 500); - topic1.checkClusterMigration().get(); log.info("before sending message"); @@ -300,24 +332,51 @@ public void testClusterMigration(boolean persistent, SubscriptionType subType) t // try to consume backlog messages from cluster-1 consumer1 = client1.newConsumer().topic(topicName).subscriptionName("s1").subscribe(); - if (persistent) { - for (int i = 0; i < n; i++) { - Message msg = consumer1.receive(); - assertEquals(msg.getData(), "test1".getBytes()); - consumer1.acknowledge(msg); - } + for (int i = 0; i < n; i++) { + Message msg = consumer1.receive(); + assertEquals(msg.getData(), "test1".getBytes()); + consumer1.acknowledge(msg); } // after consuming all messages, consumer should have disconnected // from cluster-1 and reconnect with cluster-2 retryStrategically((test) -> !topic2.getSubscriptions().isEmpty(), 10, 500); assertFalse(topic2.getSubscriptions().isEmpty()); + topic1.checkClusterMigration().get(); + final var replicators = topic1.getReplicators(); + replicators.forEach((r, replicator) -> { + assertFalse(replicator.isConnected()); + }); + + assertTrue(topic1.getSubscriptions().isEmpty()); + // not also create a new consumer which should also reconnect to cluster-2 - Consumer consumer2 = client1.newConsumer().topic(topicName).subscriptionType(subType) + Consumer consumer2 = client1.newConsumer().topic(topicName).subscriptionType(SubscriptionType.Shared) .subscriptionName("s2").subscribe(); retryStrategically((test) -> topic2.getSubscription("s2") != null, 10, 500); assertFalse(topic2.getSubscription("s2").getConsumers().isEmpty()); + // new sub on migration topic must be redirected immediately + Consumer consumerM = client1.newConsumer().topic(topicName).subscriptionType(SubscriptionType.Shared) + .subscriptionName("sM").subscribe(); + assertFalse(pulsar2.getBrokerService().getTopicReference(topicName).get().getSubscription("sM").getConsumers() + .isEmpty()); + consumerM.close(); + + // migrate topic after creating subscription + String newTopicName = topicName + "-new"; + consumerM = client1.newConsumer().topic(newTopicName).subscriptionType(SubscriptionType.Shared) + .subscriptionName("sM").subscribe(); + retryStrategically((t) -> pulsar2.getBrokerService().getTopicReference(newTopicName).isPresent(), 5, 100); + pulsar2.getBrokerService().getTopicReference(newTopicName).get().checkClusterMigration().get(); + retryStrategically((t) -> + pulsar2.getBrokerService().getTopicReference(newTopicName).isPresent() && + pulsar2.getBrokerService().getTopicReference(newTopicName).get().getSubscription("sM") + .getConsumers().isEmpty(), 5, 100); + assertFalse(pulsar2.getBrokerService().getTopicReference(newTopicName).get().getSubscription("sM").getConsumers() + .isEmpty()); + consumerM.close(); + // publish messages to cluster-2 and consume them for (int i = 0; i < n; i++) { producer1.send("test2".getBytes()); @@ -333,12 +392,12 @@ public void testClusterMigration(boolean persistent, SubscriptionType subType) t // create non-migrated topic which should connect to cluster-1 String diffTopic = BrokerTestUtil - .newUniqueName((persistent ? "persistent" : "non-persistent") + "://" + namespace + "/migrationTopic"); - Consumer consumerDiff = client1.newConsumer().topic(diffTopic).subscriptionType(subType) + .newUniqueName("persistent://" + namespace + "/migrationTopic"); + Consumer consumerDiff = client1.newConsumer().topic(diffTopic).subscriptionType(SubscriptionType.Shared) .subscriptionName("s1-d").subscribe(); Producer producerDiff = client1.newProducer().topic(diffTopic).enableBatching(false) .producerName("cluster1-d").messageRoutingMode(MessageRoutingMode.SinglePartition).create(); - AbstractTopic topicDiff = (AbstractTopic) pulsar1.getBrokerService().getTopic(diffTopic, false).getNow(null).get(); + AbstractTopic topicDiff = (AbstractTopic) pulsar2.getBrokerService().getTopic(diffTopic, false).getNow(null).get(); assertNotNull(topicDiff); for (int i = 0; i < n; i++) { producerDiff.send("diff".getBytes()); @@ -349,7 +408,7 @@ public void testClusterMigration(boolean persistent, SubscriptionType subType) t broker1.restart(); Producer producer4 = client1.newProducer().topic(topicName).enableBatching(false) .producerName("cluster1-4").messageRoutingMode(MessageRoutingMode.SinglePartition).create(); - Consumer consumer3 = client1.newConsumer().topic(topicName).subscriptionType(subType) + Consumer consumer3 = client1.newConsumer().topic(topicName).subscriptionType(SubscriptionType.Shared) .subscriptionName("s3").subscribe(); retryStrategically((test) -> topic2.getProducers().size() == 4, 10, 500); assertTrue(topic2.getProducers().size() == 4); @@ -362,15 +421,16 @@ public void testClusterMigration(boolean persistent, SubscriptionType subType) t assertEquals(consumer3.receive(2, TimeUnit.SECONDS).getData(), "test3".getBytes()); } + client1.close(); + client2.close(); log.info("Successfully consumed messages by migrated consumers"); } - @Test(dataProvider = "TopicsubscriptionTypes") - public void testClusterMigrationWithReplicationBacklog(boolean persistent, SubscriptionType subType) throws Exception { + @Test + public void testClusterMigrationWithReplicationBacklog() throws Exception { log.info("--- Starting ReplicatorTest::testClusterMigrationWithReplicationBacklog ---"); - persistent = true; final String topicName = BrokerTestUtil - .newUniqueName((persistent ? "persistent" : "non-persistent") + "://" + namespace + "/migrationTopic"); + .newUniqueName("persistent://" + namespace + "/migrationTopic"); @Cleanup PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS) @@ -381,11 +441,11 @@ public void testClusterMigrationWithReplicationBacklog(boolean persistent, Subsc // cluster-1 producer/consumer Producer producer1 = client1.newProducer().topic(topicName).enableBatching(false) .producerName("cluster1-1").messageRoutingMode(MessageRoutingMode.SinglePartition).create(); - Consumer consumer1 = client1.newConsumer().topic(topicName).subscriptionType(subType) + Consumer consumer1 = client1.newConsumer().topic(topicName).subscriptionType(SubscriptionType.Shared) .subscriptionName("s1").subscribe(); // cluster-3 consumer - Consumer consumer3 = client3.newConsumer().topic(topicName).subscriptionType(subType) + Consumer consumer3 = client3.newConsumer().topic(topicName).subscriptionType(SubscriptionType.Shared) .subscriptionName("s1").subscribe(); AbstractTopic topic1 = (AbstractTopic) pulsar1.getBrokerService().getTopic(topicName, false).getNow(null).get(); retryStrategically((test) -> !topic1.getProducers().isEmpty(), 5, 500); @@ -399,7 +459,7 @@ public void testClusterMigrationWithReplicationBacklog(boolean persistent, Subsc assertEquals(topic1.getReplicators().size(), 1); // stop service in the replication cluster to build replication backlog - broker3.cleanup(); + broker3.stop(); retryStrategically((test) -> broker3.getPulsarService() == null, 10, 1000); assertNull(pulsar3.getBrokerService()); @@ -423,9 +483,12 @@ public void testClusterMigrationWithReplicationBacklog(boolean persistent, Subsc retryStrategically((test) -> topic2.getReplicators().size() == 1, 10, 2000); log.info("replicators should be ready"); - ClusterUrl migratedUrl = new ClusterUrl(pulsar2.getBrokerServiceUrl(), pulsar2.getBrokerServiceUrlTls()); + + ClusterUrl migratedUrl = new ClusterUrl(pulsar2.getWebServiceAddress(), pulsar2.getWebServiceAddressTls(), + pulsar2.getBrokerServiceUrl(), pulsar2.getBrokerServiceUrlTls()); admin1.clusters().updateClusterMigration("r1", true, migratedUrl); log.info("update cluster migration called"); + retryStrategically((test) -> { try { topic1.checkClusterMigration().get(); @@ -458,17 +521,574 @@ public void testClusterMigrationWithReplicationBacklog(boolean persistent, Subsc retryStrategically((test) -> !topic1.isReplicationBacklogExist(), 10, 1000); assertFalse(topic1.isReplicationBacklogExist()); + // verify that the producer1 is now connected to migrated cluster "r2" since backlog is cleared. + topic1.checkClusterMigration().get(); + // verify that the producer1 is now is now connected to migrated cluster "r2" since backlog is cleared. retryStrategically((test) -> topic2.getProducers().size()==2, 10, 500); assertEquals(topic2.getProducers().size(), 2); + + client1.close(); + client2.close(); + client3.close(); + } + + /** + * This test validates that blue cluster first creates list of subscriptions into green cluster so, green cluster + * will not lose the data if producer migrates. + * + * @throws Exception + */ + @Test + public void testClusterMigrationWithResourceCreated() throws Exception { + log.info("--- Starting testClusterMigrationWithResourceCreated ---"); + + String tenant = "pulsar2"; + String namespace = tenant + "/migration"; + String greenClusterName = pulsar2.getConfig().getClusterName(); + String blueClusterName = pulsar1.getConfig().getClusterName(); + admin1.clusters().createCluster(greenClusterName, + ClusterData.builder().serviceUrl(url2.toString()).serviceUrlTls(urlTls2.toString()) + .brokerServiceUrl(pulsar2.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar2.getBrokerServiceUrlTls()).build()); + admin2.clusters().createCluster(blueClusterName, + ClusterData.builder().serviceUrl(url1.toString()).serviceUrlTls(urlTls1.toString()) + .brokerServiceUrl(pulsar1.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar1.getBrokerServiceUrlTls()).build()); + + admin1.tenants().createTenant(tenant, new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), + Sets.newHashSet("r1", greenClusterName))); + // broker should handle already tenant creation + admin2.tenants().createTenant(tenant, new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), + Sets.newHashSet("r1", greenClusterName))); + admin1.namespaces().createNamespace(namespace, Sets.newHashSet("r1", greenClusterName)); + + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + namespace + "/migrationTopic"); + + broker1.getPulsarService().getConfig().setClusterMigrationAutoResourceCreation(true); + @Cleanup + PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS) + .build(); + // cluster-1 producer/consumer + Producer producer1 = client1.newProducer().topic(topicName).enableBatching(false) + .producerName("cluster1-1").messageRoutingMode(MessageRoutingMode.SinglePartition).create(); + // create subscriptions + admin1.topics().createSubscription(topicName, "s1", MessageId.earliest); + admin1.topics().createSubscription(topicName, "s2", MessageId.earliest); + + ClusterUrl migratedUrl = new ClusterUrl(pulsar2.getWebServiceAddress(), pulsar2.getWebServiceAddressTls(), + pulsar2.getBrokerServiceUrl(), pulsar2.getBrokerServiceUrlTls()); + admin1.clusters().updateClusterMigration("r1", true, migratedUrl); + + PersistentTopic topic1 = (PersistentTopic) pulsar1.getBrokerService().getTopic(topicName, false).getNow(null) + .get(); + retryStrategically((test) -> { + try { + topic1.checkClusterMigration().get(); + return true; + } catch (Exception e) { + // ok + } + return false; + }, 10, 500); + + assertNotNull(admin2.tenants().getTenantInfo(tenant)); + assertNotNull(admin2.namespaces().getPolicies(namespace)); + List subLists = admin2.topics().getSubscriptions(topicName); + assertTrue(subLists.contains("s1")); + assertTrue(subLists.contains("s2")); + + int n = 5; + for (int i = 0; i < n; i++) { + producer1.send("test1".getBytes()); + } + + Consumer consumer1 = client1.newConsumer().topic(topicName).subscriptionName("s1").subscribe(); + for (int i = 0; i < n; i++) { + assertNotNull(consumer1.receive()); + } + + consumer1.close(); + producer1.close(); + + // publish to new topic which should be redirected immediately + String newTopic = topicName+"-new"; + producer1 = client1.newProducer().topic(newTopic).enableBatching(false) + .producerName("cluster1-1").messageRoutingMode(MessageRoutingMode.SinglePartition).create(); + retryStrategically((test) -> { + try { + pulsar2.getBrokerService().getTopic(newTopic, false).getNow(null).get(); + return true; + } catch (Exception e) { + // ok + } + return false; + }, 10, 500); + PersistentTopic pulsar2Topic = (PersistentTopic) pulsar2.getBrokerService().getTopic(newTopic, false).getNow(null) + .get(); + retryStrategically((test) -> { + try { + return !pulsar2Topic.getProducers().isEmpty(); + } catch (Exception e) { + return false; + } + }, 10, 500); + assertFalse(pulsar2Topic.getProducers().isEmpty()); + consumer1 = client1.newConsumer().topic(newTopic).subscriptionName("s1").subscribe(); + retryStrategically((test) -> { + try { + return !pulsar2Topic.getSubscription("s1").getConsumers().isEmpty(); + } catch (Exception e) { + return false; + } + }, 10, 500); + assertFalse(pulsar2Topic.getSubscription("s1").getConsumers().isEmpty()); + + client1.close(); + } + + @Test(dataProvider = "NamespaceMigrationTopicSubscriptionTypes") + public void testNamespaceMigration(SubscriptionType subType, boolean isClusterMigrate, boolean isNamespaceMigrate) throws Exception { + log.info("--- Starting Test::testNamespaceMigration ---"); + // topic for the namespace1 (to be migrated) + final String topicName = BrokerTestUtil + .newUniqueName("persistent://" + namespace + "/migrationTopic"); + // topic for namespace2 (not to be migrated) + final String topicName2 = BrokerTestUtil + .newUniqueName("persistent://" + namespaceNotToMigrate + "/migrationTopic"); + + @Cleanup + PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS) + .build(); + // blue cluster - namespace1 - producer/consumer + Producer blueProducerNs1_1 = client1.newProducer().topic(topicName).enableBatching(false) + .producerName("blue-producer-ns1-1").messageRoutingMode(MessageRoutingMode.SinglePartition).create(); + Consumer blueConsumerNs1_1 = client1.newConsumer().topic(topicName).subscriptionType(subType) + .subscriptionName("s1").subscribe(); + AbstractTopic blueTopicNs1_1 = (AbstractTopic) pulsar1.getBrokerService().getTopic(topicName, false).getNow(null).get(); + retryStrategically((test) -> !blueTopicNs1_1.getProducers().isEmpty(), 5, 500); + retryStrategically((test) -> !blueTopicNs1_1.getSubscriptions().isEmpty(), 5, 500); + assertFalse(blueTopicNs1_1.getProducers().isEmpty()); + assertFalse(blueTopicNs1_1.getSubscriptions().isEmpty()); + + // blue cluster - namespace2 - producer/consumer + Producer blueProducerNs2_1 = client1.newProducer().topic(topicName2).enableBatching(false) + .producerName("blue-producer-ns2-1").messageRoutingMode(MessageRoutingMode.SinglePartition).create(); + Consumer blueConsumerNs2_1 = client1.newConsumer().topic(topicName2).subscriptionType(subType) + .subscriptionName("s1").subscribe(); + AbstractTopic blueTopicNs2_1 = (AbstractTopic) pulsar1.getBrokerService().getTopic(topicName2, false).getNow(null).get(); + retryStrategically((test) -> !blueTopicNs2_1.getProducers().isEmpty(), 5, 500); + retryStrategically((test) -> !blueTopicNs2_1.getSubscriptions().isEmpty(), 5, 500); + assertFalse(blueTopicNs2_1.getProducers().isEmpty()); + assertFalse(blueTopicNs2_1.getSubscriptions().isEmpty()); + + // build backlog on the blue cluster + blueConsumerNs1_1.close(); + blueConsumerNs2_1.close(); + int n = 5; + for (int i = 0; i < n; i++) { + blueProducerNs1_1.send("test1".getBytes()); + blueProducerNs2_1.send("test1".getBytes()); + } + + @Cleanup + PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()).statsInterval(0, TimeUnit.SECONDS) + .build(); + // green cluster - namespace1 - producer/consumer + Producer greenProducerNs1_1 = client2.newProducer().topic(topicName).enableBatching(false) + .producerName("green-producer-ns1-1").messageRoutingMode(MessageRoutingMode.SinglePartition).create(); + AbstractTopic greenTopicNs1_1 = (AbstractTopic) pulsar2.getBrokerService().getTopic(topicName, false).getNow(null).get(); + assertFalse(greenTopicNs1_1.getProducers().isEmpty()); + + // green cluster - namespace2 - producer/consumer + Producer greenProducerNs2_1 = client2.newProducer().topic(topicName2).enableBatching(false) + .producerName("cluster2-nm1").messageRoutingMode(MessageRoutingMode.SinglePartition).create(); + AbstractTopic greenTopicNs2_1 = (AbstractTopic) pulsar2.getBrokerService().getTopic(topicName2, false).getNow(null).get(); + assertFalse(greenTopicNs2_1.getProducers().isEmpty()); + + // blue - green cluster migration + ClusterUrl migratedUrl = new ClusterUrl(pulsar2.getWebServiceAddress(), pulsar2.getWebServiceAddressTls(), + pulsar2.getBrokerServiceUrl(), pulsar2.getBrokerServiceUrlTls()); + admin1.clusters().updateClusterMigration("r1", isClusterMigrate, migratedUrl); + admin1.namespaces().updateMigrationState(namespace, isNamespaceMigrate); + + retryStrategically((test) -> { + try { + blueTopicNs1_1.checkClusterMigration().get(); + if (isClusterMigrate) { + blueTopicNs2_1.checkClusterMigration().get(); + } + return true; + } catch (Exception e) { + // ok + } + return false; + }, 10, 500); + + + blueTopicNs1_1.checkClusterMigration().get(); + if (isClusterMigrate) { + blueTopicNs2_1.checkClusterMigration().get(); + } + + log.info("before sending message"); + sleep(1000); + blueProducerNs1_1.sendAsync("test1".getBytes()); + blueProducerNs2_1.sendAsync("test1".getBytes()); + + // producer is disconnected from blue for namespace1 as cluster or namespace migration is enabled + retryStrategically((test) -> blueTopicNs1_1.getProducers().isEmpty(), 10, 500); + assertTrue(blueTopicNs1_1.getProducers().isEmpty()); + + if(isClusterMigrate){ + // producer is disconnected from blue for namespace2 if cluster migration is enabled + retryStrategically((test) -> blueTopicNs2_1.getProducers().isEmpty(), 10, 500); + assertTrue(blueTopicNs2_1.getProducers().isEmpty()); + } else { + // producer is not disconnected from blue for namespace2 if namespace migration is disabled + retryStrategically((test) -> !blueTopicNs2_1.getProducers().isEmpty(), 10, 500); + assertTrue(!blueTopicNs2_1.getProducers().isEmpty()); + } + + // create producer on blue which should be redirected to green + Producer blueProducerNs1_2 = client1.newProducer().topic(topicName).enableBatching(false) + .producerName("blue-producer-ns1-2").messageRoutingMode(MessageRoutingMode.SinglePartition).create(); + + // producer is connected with green + retryStrategically((test) -> greenTopicNs1_1.getProducers().size() == 3, 10, 500); + assertTrue(greenTopicNs1_1.getProducers().size() == 3); + + // blueProducerNs2_1 should be migrated to green if the cluster migration is enabled + // should not be migrated if the namespace migration is disabled for namespace2 + if (isClusterMigrate) { + retryStrategically((test) -> greenTopicNs2_1.getProducers().size() == 2, 10, 500); + assertTrue(greenTopicNs2_1.getProducers().size() == 2); + } else{ + retryStrategically((test) -> greenTopicNs2_1.getProducers().size() == 1, 10, 500); + assertTrue(greenTopicNs2_1.getProducers().size() == 1); + } + + // try to consume backlog messages from cluster-1 + blueConsumerNs1_1 = client1.newConsumer().topic(topicName).subscriptionName("s1").subscribe(); + blueConsumerNs2_1 = client1.newConsumer().topic(topicName2).subscriptionName("s1").subscribe(); + for (int i = 0; i < n; i++) { + Message msg = blueConsumerNs1_1.receive(); + assertEquals(msg.getData(), "test1".getBytes()); + blueConsumerNs1_1.acknowledge(msg); + + Message msg2 = blueConsumerNs2_1.receive(); + assertEquals(msg2.getData(), "test1".getBytes()); + blueConsumerNs2_1.acknowledge(msg2); + } + // after consuming all messages, consumer should have disconnected + // from blue and reconnect with green + retryStrategically((test) -> !greenTopicNs1_1.getSubscriptions().isEmpty(), 10, 500); + assertFalse(greenTopicNs1_1.getSubscriptions().isEmpty()); + if (isClusterMigrate) { + retryStrategically((test) -> !greenTopicNs2_1.getSubscriptions().isEmpty(), 10, 500); + assertFalse(greenTopicNs2_1.getSubscriptions().isEmpty()); + } else { + retryStrategically((test) -> greenTopicNs2_1.getSubscriptions().isEmpty(), 10, 500); + assertTrue(greenTopicNs2_1.getSubscriptions().isEmpty()); + } + + blueTopicNs1_1.checkClusterMigration().get(); + if (isClusterMigrate) { + blueTopicNs2_1.checkClusterMigration().get(); + } + + final var replicators = blueTopicNs1_1.getReplicators(); + replicators.forEach((r, replicator) -> { + assertFalse(replicator.isConnected()); + }); + assertTrue(blueTopicNs1_1.getSubscriptions().isEmpty()); + + if (isClusterMigrate) { + final var replicatorsNm = blueTopicNs2_1.getReplicators(); + replicatorsNm.forEach((r, replicator) -> { + assertFalse(replicator.isConnected()); + }); + assertTrue(blueTopicNs2_1.getSubscriptions().isEmpty()); + } else { + final var replicatorsNm = blueTopicNs2_1.getReplicators(); + replicatorsNm.forEach((r, replicator) -> { + assertTrue(replicator.isConnected()); + }); + assertFalse(blueTopicNs2_1.getSubscriptions().isEmpty()); + } + + // create a new consumer on blue which should also reconnect to green + Consumer blueConsumerNs1_2 = client1.newConsumer().topic(topicName).subscriptionType(subType) + .subscriptionName("s2").subscribe(); + Consumer blueConsumerNs2_2 = client1.newConsumer().topic(topicName2).subscriptionType(subType) + .subscriptionName("s2").subscribe(); + retryStrategically((test) -> greenTopicNs1_1.getSubscription("s2") != null, 10, 500); + assertFalse(greenTopicNs1_1.getSubscription("s2").getConsumers().isEmpty()); + if (isClusterMigrate) { + retryStrategically((test) -> greenTopicNs2_1.getSubscription("s2") != null, 10, 500); + assertFalse(greenTopicNs2_1.getSubscription("s2").getConsumers().isEmpty()); + } else { + retryStrategically((test) -> greenTopicNs2_1.getSubscription("s2") == null, 10, 500); + } + + // new sub on migration topic must be redirected immediately + Consumer consumerM = client1.newConsumer().topic(topicName).subscriptionType(subType) + .subscriptionName("sM").subscribe(); + assertFalse(pulsar2.getBrokerService().getTopicReference(topicName).get().getSubscription("sM").getConsumers() + .isEmpty()); + consumerM.close(); + + // migrate topic after creating subscription + String newTopicName = topicName + "-new"; + consumerM = client1.newConsumer().topic(newTopicName).subscriptionType(subType) + .subscriptionName("sM").subscribe(); + retryStrategically((t) -> pulsar1.getBrokerService().getTopicReference(newTopicName).isPresent(), 5, 100); + pulsar2.getBrokerService().getTopicReference(newTopicName).get().checkClusterMigration().get(); + retryStrategically((t) -> + pulsar2.getBrokerService().getTopicReference(newTopicName).isPresent() && + pulsar2.getBrokerService().getTopicReference(newTopicName).get().getSubscription("sM") + .getConsumers().isEmpty(), 5, 100); + assertFalse(pulsar2.getBrokerService().getTopicReference(newTopicName).get().getSubscription("sM").getConsumers() + .isEmpty()); + consumerM.close(); + + // publish messages to cluster-2 and consume them + for (int i = 0; i < n; i++) { + blueProducerNs1_1.send("test2".getBytes()); + blueProducerNs1_2.send("test2".getBytes()); + greenProducerNs1_1.send("test2".getBytes()); + } + log.info("Successfully published messages by migrated producers"); + for (int i = 0; i < n * 3; i++) { + assertEquals(blueConsumerNs1_1.receive(2, TimeUnit.SECONDS).getData(), "test2".getBytes()); + assertEquals(blueConsumerNs1_2.receive(2, TimeUnit.SECONDS).getData(), "test2".getBytes()); + + } + + // create non-migrated topic which should connect to blue + String diffTopic = BrokerTestUtil + .newUniqueName("persistent://" + namespace + "/migrationTopic"); + Consumer consumerDiff = client1.newConsumer().topic(diffTopic).subscriptionType(subType) + .subscriptionName("s1-d").subscribe(); + Producer producerDiff = client1.newProducer().topic(diffTopic).enableBatching(false) + .producerName("cluster1-d").messageRoutingMode(MessageRoutingMode.SinglePartition).create(); + AbstractTopic topicDiff = (AbstractTopic) pulsar2.getBrokerService().getTopic(diffTopic, false).getNow(null).get(); + assertNotNull(topicDiff); + for (int i = 0; i < n; i++) { + producerDiff.send("diff".getBytes()); + assertEquals(consumerDiff.receive(2, TimeUnit.SECONDS).getData(), "diff".getBytes()); + } + + // restart broker-1 + broker1.restart(); + Producer producer4 = client1.newProducer().topic(topicName).enableBatching(false) + .producerName("cluster1-4").messageRoutingMode(MessageRoutingMode.SinglePartition).create(); + Consumer consumer3 = client1.newConsumer().topic(topicName).subscriptionType(subType) + .subscriptionName("s3").subscribe(); + retryStrategically((test) -> greenTopicNs1_1.getProducers().size() == 4, 10, 500); + assertTrue(greenTopicNs1_1.getProducers().size() == 4); + retryStrategically((test) -> greenTopicNs1_1.getSubscription("s3") != null, 10, 500); + assertFalse(greenTopicNs1_1.getSubscription("s3").getConsumers().isEmpty()); + for (int i = 0; i < n; i++) { + producer4.send("test3".getBytes()); + assertEquals(blueConsumerNs1_1.receive(2, TimeUnit.SECONDS).getData(), "test3".getBytes()); + assertEquals(blueConsumerNs1_2.receive(2, TimeUnit.SECONDS).getData(), "test3".getBytes()); + assertEquals(consumer3.receive(2, TimeUnit.SECONDS).getData(), "test3".getBytes()); + } + + log.info("Successfully consumed messages by migrated consumers"); + + // clean up + blueConsumerNs1_1.close(); + blueConsumerNs1_2.close(); + blueConsumerNs2_1.close(); + blueProducerNs1_1.close(); + blueProducerNs1_2.close(); + blueProducerNs2_1.close(); + greenProducerNs1_1.close(); + greenProducerNs2_1.close(); + client1.close(); + client2.close(); + } + + @Test(dataProvider = "NamespaceMigrationTopicSubscriptionTypes") + public void testNamespaceMigrationWithReplicationBacklog(SubscriptionType subType, boolean isClusterMigrate, boolean isNamespaceMigrate) throws Exception { + log.info("--- Starting ReplicatorTest::testNamespaceMigrationWithReplicationBacklog ---"); + // topic for namespace1 (to be migrated) + final String topicName = BrokerTestUtil + .newUniqueName("persistent://" + namespace + "/migrationTopic"); + // topic for namespace2 (not to be migrated) + final String topicName2 = BrokerTestUtil + .newUniqueName("persistent://" + namespaceNotToMigrate + "/migrationTopic"); + + @Cleanup + PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS) + .build(); + @Cleanup + PulsarClient client3 = PulsarClient.builder().serviceUrl(url3.toString()).statsInterval(0, TimeUnit.SECONDS) + .build(); + // blue cluster - namespace1 - producer/consumer + Producer blueProducerNs1_1 = client1.newProducer().topic(topicName).enableBatching(false) + .producerName("blue-producer-ns1-1").messageRoutingMode(MessageRoutingMode.SinglePartition).create(); + Consumer blueConsumerNs1_1 = client1.newConsumer().topic(topicName).subscriptionType(subType) + .subscriptionName("s1").subscribe(); + + // blue cluster - namespace2 - producer/consumer + Producer blueProducerNs2_1 = client1.newProducer().topic(topicName2).enableBatching(false) + .producerName("blue-producer-ns1-1").messageRoutingMode(MessageRoutingMode.SinglePartition).create(); + Consumer blueConsumerNs2_1 = client1.newConsumer().topic(topicName2).subscriptionType(subType) + .subscriptionName("s1").subscribe(); + + // blue cluster replication consumer namespace1 + Consumer blueConsumerReplicationNs1 = client3.newConsumer().topic(topicName).subscriptionType(subType) + .subscriptionName("s1").subscribe(); + + // blue cluster replication consumer namespace2 + Consumer blueConsumerReplicationNs2 = client3.newConsumer().topic(topicName2).subscriptionType(subType) + .subscriptionName("s1").subscribe(); + + + AbstractTopic blueTopicNs1 = (AbstractTopic) pulsar1.getBrokerService().getTopic(topicName, false).getNow(null).get(); + retryStrategically((test) -> !blueTopicNs1.getProducers().isEmpty(), 5, 500); + retryStrategically((test) -> !blueTopicNs1.getSubscriptions().isEmpty(), 5, 500); + assertFalse(blueTopicNs1.getProducers().isEmpty()); + assertFalse(blueTopicNs1.getSubscriptions().isEmpty()); + + AbstractTopic blueTopicNs2 = (AbstractTopic) pulsar1.getBrokerService().getTopic(topicName2, false).getNow(null).get(); + retryStrategically((test) -> !blueTopicNs2.getProducers().isEmpty(), 5, 500); + retryStrategically((test) -> !blueTopicNs2.getSubscriptions().isEmpty(), 5, 500); + assertFalse(blueTopicNs2.getProducers().isEmpty()); + assertFalse(blueTopicNs2.getSubscriptions().isEmpty()); + + // build backlog + blueConsumerNs1_1.close(); + blueConsumerNs2_1.close(); + retryStrategically((test) -> blueTopicNs1.getReplicators().size() == 1, 10, 3000); + assertEquals(blueTopicNs1.getReplicators().size(), 1); + retryStrategically((test) -> blueTopicNs2.getReplicators().size() == 1, 10, 3000); + assertEquals(blueTopicNs2.getReplicators().size(), 1); + + // stop service in the replication cluster to build replication backlog + broker3.stop(); + retryStrategically((test) -> broker3.getPulsarService() == null, 10, 1000); + assertNull(pulsar3.getBrokerService()); + + //publish messages into topic in blue cluster + int n = 5; + for (int i = 0; i < n; i++) { + blueProducerNs1_1.send("test1".getBytes()); + blueProducerNs2_1.send("test1".getBytes()); + } + retryStrategically((test) -> blueTopicNs1.isReplicationBacklogExist(), 10, 1000); + assertTrue(blueTopicNs1.isReplicationBacklogExist()); + retryStrategically((test) -> blueTopicNs2.isReplicationBacklogExist(), 10, 1000); + assertTrue(blueTopicNs2.isReplicationBacklogExist()); + + @Cleanup + PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()).statsInterval(0, TimeUnit.SECONDS) + .build(); + // green cluster - namespace1 - producer/consumer + Producer greenProducerNs1_1 = client2.newProducer().topic(topicName).enableBatching(false) + .producerName("green-producer-ns1-1").messageRoutingMode(MessageRoutingMode.SinglePartition).create(); + AbstractTopic greenTopicNs1 = (AbstractTopic) pulsar2.getBrokerService().getTopic(topicName, false).getNow(null).get(); + Producer greenProducerNs2_1 = client2.newProducer().topic(topicName2).enableBatching(false) + .producerName("green-producer-ns2-1").messageRoutingMode(MessageRoutingMode.SinglePartition).create(); + AbstractTopic greenTopicNs2 = (AbstractTopic) pulsar2.getBrokerService().getTopic(topicName2, false).getNow(null).get(); + log.info("name of topic 2 - {}", greenTopicNs1.getName()); + assertFalse(greenTopicNs1.getProducers().isEmpty()); + + retryStrategically((test) -> greenTopicNs1.getReplicators().size() == 1, 10, 2000); + log.info("replicators should be ready"); + + ClusterUrl migratedUrl = new ClusterUrl(pulsar2.getWebServiceAddress(), pulsar2.getWebServiceAddressTls(), + pulsar2.getBrokerServiceUrl(), pulsar2.getBrokerServiceUrlTls()); + admin1.clusters().updateClusterMigration("r1", isClusterMigrate, migratedUrl); + admin1.namespaces().updateMigrationState(namespace, isNamespaceMigrate); + assertEquals(admin1.namespaces().getPolicies(namespace).migrated, isNamespaceMigrate); + log.info("update cluster migration called"); + + retryStrategically((test) -> { + try { + blueTopicNs1.checkClusterMigration().get(); + if (isClusterMigrate) { + blueTopicNs2.checkClusterMigration().get(); + } + return true; + } catch (Exception e) { + // ok + } + return false; + }, 10, 500); + + blueTopicNs1.checkClusterMigration().get(); + if (isClusterMigrate) { + blueTopicNs2.checkClusterMigration().get(); + } + + blueProducerNs1_1.sendAsync("test1".getBytes()); + blueProducerNs2_1.sendAsync("test1".getBytes()); + + // producer is disconnected from blue + retryStrategically((test) -> blueTopicNs1.getProducers().isEmpty(), 10, 500); + assertTrue(blueTopicNs1.getProducers().isEmpty()); + if (isClusterMigrate) { + retryStrategically((test) -> blueTopicNs2.getProducers().isEmpty(), 10, 500); + assertTrue(blueTopicNs2.getProducers().isEmpty()); + } else { + retryStrategically((test) -> !blueTopicNs2.getProducers().isEmpty(), 10, 500); + assertFalse(blueTopicNs2.getProducers().isEmpty()); + } + + // verify that the disconnected producer is not redirected + // to replication cluster since there is replication backlog. + assertEquals(greenTopicNs1.getProducers().size(), 1); + + // Restart the service in cluster "r3". + broker3.restart(); + retryStrategically((test) -> broker3.getPulsarService() != null, 10, 1000); + assertNotNull(broker3.getPulsarService()); + pulsar3 = broker3.getPulsarService(); + + // verify that the replication backlog drains once service in cluster "r3" is restarted. + retryStrategically((test) -> !blueTopicNs1.isReplicationBacklogExist(), 10, 1000); + assertFalse(blueTopicNs1.isReplicationBacklogExist()); + retryStrategically((test) -> !blueTopicNs2.isReplicationBacklogExist(), 10, 1000); + assertFalse(blueTopicNs2.isReplicationBacklogExist()); + + blueTopicNs1.checkClusterMigration().get(); + blueTopicNs2.checkClusterMigration().get(); + + // verify that the producer1 is now is now connected to migrated cluster green since backlog is cleared. + retryStrategically((test) -> greenTopicNs1.getProducers().size()==2, 10, 500); + assertEquals(greenTopicNs1.getProducers().size(), 2); + if (isClusterMigrate) { + retryStrategically((test) -> greenTopicNs2.getProducers().size()==2, 10, 500); + assertEquals(greenTopicNs2.getProducers().size(), 2); + } else { + retryStrategically((test) -> greenTopicNs2.getProducers().size()==1, 10, 500); + assertEquals(greenTopicNs2.getProducers().size(), 1); + } + + // clean up + blueProducerNs1_1.close(); + blueProducerNs2_1.close(); + blueConsumerNs1_1.close(); + blueConsumerNs2_1.close(); + greenProducerNs1_1.close(); + greenProducerNs2_1.close(); + client1.close(); + client2.close(); + client3.close(); } static class TestBroker extends MockedPulsarServiceBaseTest { private String clusterName; + private String loadManagerClassName; - public TestBroker(String clusterName) throws Exception { + public TestBroker(String clusterName, String loadManagerClassName) throws Exception { this.clusterName = clusterName; + this.loadManagerClassName = loadManagerClassName; setup(); } @@ -477,6 +1097,18 @@ protected void setup() throws Exception { super.setupWithClusterName(clusterName); } + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + this.conf.setLoadManagerClassName(loadManagerClassName); + this.conf.setWebServicePortTls(Optional.of(0)); + this.conf.setBrokerServicePortTls(Optional.of(0)); + this.conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + this.conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + this.conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); + } + + public PulsarService getPulsarService() { return pulsar; } @@ -485,9 +1117,13 @@ public String getClusterName() { return configClusterName; } + public void stop() throws Exception { + stopBroker(); + } + @Override protected void cleanup() throws Exception { - stopBroker(); + internalCleanup(); } public void restart() throws Exception { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelectorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelectorTest.java index dbca31416bb9d..04aafc49b47e6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelectorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelectorTest.java @@ -18,21 +18,29 @@ */ package org.apache.pulsar.broker.service; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; - - -import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerAssignException; -import org.apache.pulsar.client.api.Range; -import org.testng.Assert; -import org.testng.annotations.Test; - +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerAssignException; +import org.apache.pulsar.client.api.Range; +import org.assertj.core.data.Offset; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.Test; @Test(groups = "broker") public class ConsistentHashingStickyKeyConsumerSelectorTest { @@ -40,7 +48,7 @@ public class ConsistentHashingStickyKeyConsumerSelectorTest { @Test public void testConsumerSelect() throws ConsumerAssignException { - ConsistentHashingStickyKeyConsumerSelector selector = new ConsistentHashingStickyKeyConsumerSelector(100); + ConsistentHashingStickyKeyConsumerSelector selector = new ConsistentHashingStickyKeyConsumerSelector(200); String key1 = "anyKey"; Assert.assertNull(selector.select(key1.getBytes())); @@ -146,30 +154,396 @@ public void testGetConsumerKeyHashRanges() throws BrokerServiceException.Consume ConsistentHashingStickyKeyConsumerSelector selector = new ConsistentHashingStickyKeyConsumerSelector(3); List consumerName = Arrays.asList("consumer1", "consumer2", "consumer3"); List consumers = new ArrayList<>(); + long id=0; for (String s : consumerName) { - Consumer consumer = mock(Consumer.class); - when(consumer.consumerName()).thenReturn(s); + Consumer consumer = createMockConsumer(s, s, id++); selector.addConsumer(consumer); consumers.add(consumer); } + + // check that results are the same when called multiple times + assertThat(selector.getConsumerKeyHashRanges()) + .containsExactlyEntriesOf(selector.getConsumerKeyHashRanges()); + Map> expectedResult = new HashMap<>(); + assertThat(consumers.get(0).consumerName()).isEqualTo("consumer1"); expectedResult.put(consumers.get(0), Arrays.asList( - Range.of(0, 330121749), - Range.of(330121750, 618146114), - Range.of(1797637922, 1976098885))); + Range.of(95615213, 440020355), + Range.of(440020356, 455987436), + Range.of(1189794593, 1264144431))); + assertThat(consumers.get(1).consumerName()).isEqualTo("consumer2"); expectedResult.put(consumers.get(1), Arrays.asList( - Range.of(938427576, 1094135919), - Range.of(1138613629, 1342907082), - Range.of(1342907083, 1797637921))); + Range.of(939655188, 1189794592), + Range.of(1314727625, 1977451233), + Range.of(1977451234, 2016237253))); + assertThat(consumers.get(2).consumerName()).isEqualTo("consumer3"); expectedResult.put(consumers.get(2), Arrays.asList( - Range.of(618146115, 772640562), - Range.of(772640563, 938427575), - Range.of(1094135920, 1138613628))); + Range.of(0, 95615212), + Range.of(455987437, 939655187), + Range.of(1264144432, 1314727624), + Range.of(2016237254, 2147483646))); + Map> consumerKeyHashRanges = selector.getConsumerKeyHashRanges(); + assertThat(consumerKeyHashRanges).containsExactlyInAnyOrderEntriesOf(expectedResult); + + // check that ranges are continuous and cover the whole range + List allRanges = + consumerKeyHashRanges.values().stream().flatMap(List::stream).sorted().collect(Collectors.toList()); + Range previousRange = null; + for (Range range : allRanges) { + if (previousRange != null) { + assertThat(range.getStart()).isEqualTo(previousRange.getEnd() + 1); + } + previousRange = range; + } + assertThat(allRanges.stream().mapToInt(r -> r.getEnd() - r.getStart() + 1).sum()).isEqualTo(Integer.MAX_VALUE); + } + + @Test + public void testConsumersGetSufficientlyAccuratelyEvenlyMapped() + throws BrokerServiceException.ConsumerAssignException { + ConsistentHashingStickyKeyConsumerSelector selector = new ConsistentHashingStickyKeyConsumerSelector(200); + List consumers = new ArrayList<>(); + for (int i = 0; i < 20; i++) { + // use the same name for all consumers, use toString to distinguish them + Consumer consumer = createMockConsumer("consumer", String.format("index %02d", i), i); + selector.addConsumer(consumer); + consumers.add(consumer); + } + printConsumerRangesStats(selector); + + int totalSelections = 10000; + + Map consumerSelectionCount = new HashMap<>(); + for (int i = 0; i < totalSelections; i++) { + Consumer selectedConsumer = selector.select(("key " + i).getBytes(StandardCharsets.UTF_8)); + consumerSelectionCount.computeIfAbsent(selectedConsumer, c -> new MutableInt()).increment(); + } + + printSelectionCountStats(consumerSelectionCount); + + int averageCount = totalSelections / consumers.size(); + int allowedVariance = (int) (0.2d * averageCount); + System.out.println("averageCount: " + averageCount + " allowedVariance: " + allowedVariance); + + for (Map.Entry entry : consumerSelectionCount.entrySet()) { + assertThat(entry.getValue().intValue()).describedAs("consumer: %s", entry.getKey()) + .isCloseTo(averageCount, Offset.offset(allowedVariance)); + } + + consumers.forEach(selector::removeConsumer); + assertThat(selector.getConsumerKeyHashRanges()).isEmpty(); + } + + private static void printSelectionCountStats(Map consumerSelectionCount) { + int totalSelections = consumerSelectionCount.values().stream().mapToInt(MutableInt::intValue).sum(); + consumerSelectionCount.entrySet().stream() + .sorted(Map.Entry.comparingByKey(Comparator.comparing(Consumer::toString))) + .forEach(entry -> System.out.println( + String.format("consumer: %s got selected %d times. ratio: %.2f%%", entry.getKey(), + entry.getValue().intValue(), + ((double) entry.getValue().intValue() / totalSelections) * 100.0d))); + } + + private static void printConsumerRangesStats(ConsistentHashingStickyKeyConsumerSelector selector) { + selector.getConsumerKeyHashRanges().entrySet().stream() + .map(entry -> Map.entry(entry.getKey(), + entry.getValue().stream().mapToInt(r -> r.getEnd() - r.getStart() + 1).sum())) + .sorted(Map.Entry.comparingByKey(Comparator.comparing(Consumer::toString))) + .forEach(entry -> System.out.println( + String.format("consumer: %s total ranges size: %d ratio: %.2f%%", entry.getKey(), + entry.getValue(), + ((double) entry.getValue() / (Integer.MAX_VALUE - 1)) * 100.0d))); + } + + private static Consumer createMockConsumer(String consumerName, String toString, long id) { + // without stubOnly, the mock will record method invocations and run into OOME + Consumer consumer = mock(Consumer.class, Mockito.withSettings().stubOnly()); + when(consumer.consumerName()).thenReturn(consumerName); + when(consumer.getPriorityLevel()).thenReturn(0); + when(consumer.toString()).thenReturn(toString); + when(consumer.consumerId()).thenReturn(id); + return consumer; + } + + // reproduces https://github.com/apache/pulsar/issues/22050 + @Test + public void shouldNotCollideWithConsumerNameEndsWithNumber() { + ConsistentHashingStickyKeyConsumerSelector selector = new ConsistentHashingStickyKeyConsumerSelector(12); + List consumerName = Arrays.asList("consumer1", "consumer11"); + List consumers = new ArrayList<>(); + for (String s : consumerName) { + Consumer consumer = mock(Consumer.class); + when(consumer.consumerName()).thenReturn(s); + selector.addConsumer(consumer); + consumers.add(consumer); + } + Map rangeToConsumer = new HashMap<>(); for (Map.Entry> entry : selector.getConsumerKeyHashRanges().entrySet()) { - System.out.println(entry.getValue()); - Assert.assertEquals(entry.getValue(), expectedResult.get(entry.getKey())); - expectedResult.remove(entry.getKey()); + for (Range range : entry.getValue()) { + Consumer previous = rangeToConsumer.put(range, entry.getKey()); + if (previous != null) { + Assert.fail("Ranges are colliding between " + previous.consumerName() + " and " + entry.getKey() + .consumerName()); + } + } + } + } + + @Test + public void shouldRemoveConsumersFromConsumerKeyHashRanges() { + ConsistentHashingStickyKeyConsumerSelector selector = new ConsistentHashingStickyKeyConsumerSelector(12); + List consumers = IntStream.range(1, 100).mapToObj(i -> "consumer" + i) + .map(consumerName -> { + Consumer consumer = mock(Consumer.class); + when(consumer.consumerName()).thenReturn(consumerName); + return consumer; + }).collect(Collectors.toList()); + + // when consumers are added + consumers.forEach(selector::addConsumer); + // then each consumer should have a range + Assert.assertEquals(selector.getConsumerKeyHashRanges().size(), consumers.size()); + // when consumers are removed + consumers.forEach(selector::removeConsumer); + // then there should be no mapping remaining + Assert.assertEquals(selector.getConsumerKeyHashRanges().size(), 0); + // when consumers are removed again, should not fail + consumers.forEach(selector::removeConsumer); + } + + @Test + public void testShouldNotChangeSelectedConsumerWhenConsumerIsRemoved() { + final ConsistentHashingStickyKeyConsumerSelector selector = new ConsistentHashingStickyKeyConsumerSelector(100); + final String consumerName = "consumer"; + final int numOfInitialConsumers = 100; + List consumers = new ArrayList<>(); + for (int i = 0; i < numOfInitialConsumers; i++) { + final Consumer consumer = createMockConsumer(consumerName, "index " + i, i); + consumers.add(consumer); + selector.addConsumer(consumer); + } + + int hashRangeSize = Integer.MAX_VALUE; + int validationPointCount = 200; + int increment = hashRangeSize / (validationPointCount + 1); + List selectedConsumerBeforeRemoval = new ArrayList<>(); + + for (int i = 0; i < validationPointCount; i++) { + selectedConsumerBeforeRemoval.add(selector.select(i * increment)); + } + + for (int i = 0; i < validationPointCount; i++) { + Consumer selected = selector.select(i * increment); + Consumer expected = selectedConsumerBeforeRemoval.get(i); + assertThat(selected.consumerId()).as("validationPoint %d", i).isEqualTo(expected.consumerId()); + } + + Set removedConsumers = new HashSet<>(); + for (Consumer removedConsumer : consumers) { + selector.removeConsumer(removedConsumer); + removedConsumers.add(removedConsumer); + for (int i = 0; i < validationPointCount; i++) { + int hash = i * increment; + Consumer selected = selector.select(hash); + Consumer expected = selectedConsumerBeforeRemoval.get(i); + if (!removedConsumers.contains(expected)) { + assertThat(selected.consumerId()).as("validationPoint %d, removed %s, hash %d ranges %s", i, + removedConsumer.toString(), hash, selector.getConsumerKeyHashRanges()).isEqualTo(expected.consumerId()); + } + } + } + } + + @Test + public void testShouldNotChangeSelectedConsumerWhenConsumerIsRemovedCheckHashRanges() { + final ConsistentHashingStickyKeyConsumerSelector selector = new ConsistentHashingStickyKeyConsumerSelector(100); + final String consumerName = "consumer"; + final int numOfInitialConsumers = 25; + List consumers = new ArrayList<>(); + for (int i = 0; i < numOfInitialConsumers; i++) { + final Consumer consumer = createMockConsumer(consumerName, "index " + i, i); + consumers.add(consumer); + selector.addConsumer(consumer); + } + + Map> expected = selector.getConsumerKeyHashRanges(); + assertThat(selector.getConsumerKeyHashRanges()).as("sanity check").containsExactlyInAnyOrderEntriesOf(expected); + System.out.println(expected); + + for (Consumer removedConsumer : consumers) { + selector.removeConsumer(removedConsumer); + for (Map.Entry> entry : expected.entrySet()) { + if (entry.getKey() == removedConsumer) { + continue; + } + for (Range range : entry.getValue()) { + Consumer rangeStartConsumer = selector.select(range.getStart()); + assertThat(rangeStartConsumer).as("removed %s, range %s", removedConsumer, range) + .isEqualTo(entry.getKey()); + Consumer rangeEndConsumer = selector.select(range.getEnd()); + assertThat(rangeEndConsumer).as("removed %s, range %s", removedConsumer, range) + .isEqualTo(entry.getKey()); + assertThat(rangeStartConsumer).isSameAs(rangeEndConsumer); + } + } + expected = selector.getConsumerKeyHashRanges(); + } + } + + @Test + public void testShouldNotChangeSelectedConsumerUnnecessarilyWhenConsumerIsAddedCheckHashRanges() { + final ConsistentHashingStickyKeyConsumerSelector selector = new ConsistentHashingStickyKeyConsumerSelector(100); + final String consumerName = "consumer"; + final int numOfInitialConsumers = 25; + List consumers = new ArrayList<>(); + for (int i = 0; i < numOfInitialConsumers; i++) { + final Consumer consumer = createMockConsumer(consumerName, "index " + i, i); + consumers.add(consumer); + selector.addConsumer(consumer); + } + + Map> expected = selector.getConsumerKeyHashRanges(); + assertThat(selector.getConsumerKeyHashRanges()).as("sanity check").containsExactlyInAnyOrderEntriesOf(expected); + + for (int i = numOfInitialConsumers; i < numOfInitialConsumers * 2; i++) { + final Consumer addedConsumer = createMockConsumer(consumerName, "index " + i, i); + selector.addConsumer(addedConsumer); + for (Map.Entry> entry : expected.entrySet()) { + if (entry.getKey() == addedConsumer) { + continue; + } + for (Range range : entry.getValue()) { + Consumer rangeStartConsumer = selector.select(range.getStart()); + if (rangeStartConsumer != addedConsumer) { + assertThat(rangeStartConsumer).as("added %s, range start %s", addedConsumer, range) + .isEqualTo(entry.getKey()); + } + Consumer rangeEndConsumer = selector.select(range.getStart()); + if (rangeEndConsumer != addedConsumer) { + assertThat(rangeEndConsumer).as("added %s, range end %s", addedConsumer, range) + .isEqualTo(entry.getKey()); + } + } + } + expected = selector.getConsumerKeyHashRanges(); + } + } + + @Test + public void testShouldNotChangeSelectedConsumerWhenConsumerIsAdded() { + final ConsistentHashingStickyKeyConsumerSelector selector = new ConsistentHashingStickyKeyConsumerSelector(100); + final String consumerName = "consumer"; + final int numOfInitialConsumers = 50; + List consumers = new ArrayList<>(); + for (int i = 0; i < numOfInitialConsumers; i++) { + final Consumer consumer = createMockConsumer(consumerName, "index " + i, i); + consumers.add(consumer); + selector.addConsumer(consumer); + } + + int hashRangeSize = Integer.MAX_VALUE; + int validationPointCount = 200; + int increment = hashRangeSize / (validationPointCount + 1); + List selectedConsumerBeforeRemoval = new ArrayList<>(); + + for (int i = 0; i < validationPointCount; i++) { + selectedConsumerBeforeRemoval.add(selector.select(i * increment)); + } + + for (int i = 0; i < validationPointCount; i++) { + Consumer selected = selector.select(i * increment); + Consumer expected = selectedConsumerBeforeRemoval.get(i); + assertThat(selected.consumerId()).as("validationPoint %d", i).isEqualTo(expected.consumerId()); + } + + Set addedConsumers = new HashSet<>(); + for (int i = numOfInitialConsumers; i < numOfInitialConsumers * 2; i++) { + final Consumer addedConsumer = createMockConsumer(consumerName, "index " + i, i); + selector.addConsumer(addedConsumer); + addedConsumers.add(addedConsumer); + for (int j = 0; j < validationPointCount; j++) { + int hash = j * increment; + Consumer selected = selector.select(hash); + Consumer expected = selectedConsumerBeforeRemoval.get(j); + if (!addedConsumers.contains(addedConsumer)) { + assertThat(selected.consumerId()).as("validationPoint %d, hash %d", j, hash).isEqualTo(expected.consumerId()); + } + } + } + } + + @Test + public void testShouldNotChangeMappingWhenConsumerLeavesAndRejoins() { + final ConsistentHashingStickyKeyConsumerSelector selector = new ConsistentHashingStickyKeyConsumerSelector(100); + final String consumerName = "consumer"; + final int numOfInitialConsumers = 25; + List consumers = new ArrayList<>(); + for (int i = 0; i < numOfInitialConsumers; i++) { + final Consumer consumer = createMockConsumer(consumerName, "index " + i, i); + consumers.add(consumer); + selector.addConsumer(consumer); + } + + Map> expected = selector.getConsumerKeyHashRanges(); + assertThat(selector.getConsumerKeyHashRanges()).as("sanity check").containsExactlyInAnyOrderEntriesOf(expected); + + selector.removeConsumer(consumers.get(0)); + selector.removeConsumer(consumers.get(numOfInitialConsumers / 2)); + selector.addConsumer(consumers.get(0)); + selector.addConsumer(consumers.get(numOfInitialConsumers / 2)); + + assertThat(selector.getConsumerKeyHashRanges()).as("ranges shouldn't change").containsExactlyInAnyOrderEntriesOf(expected); + } + + @Test + public void testConsumersReconnect() { + final ConsistentHashingStickyKeyConsumerSelector selector = new ConsistentHashingStickyKeyConsumerSelector(100); + final String consumerName = "consumer"; + final int numOfInitialConsumers = 50; + final int validationPointCount = 200; + final List pointsToTest = pointsToTest(validationPointCount); + List consumers = new ArrayList<>(); + for (int i = 0; i < numOfInitialConsumers; i++) { + final Consumer consumer = createMockConsumer(consumerName, "index " + i, i); + consumers.add(consumer); + selector.addConsumer(consumer); + } + + // Mark original results. + List selectedConsumersBeforeRemove = new ArrayList<>(); + for (int i = 0; i < validationPointCount; i++) { + int point = pointsToTest.get(i); + selectedConsumersBeforeRemove.add(selector.select(point)); + } + + // All consumers leave (in any order) + List randomOrderConsumers = new ArrayList<>(consumers); + Collections.shuffle(randomOrderConsumers); + for (Consumer c : randomOrderConsumers) { + selector.removeConsumer(c); + } + + // All consumers reconnect in the same order as originally + for (Consumer c : consumers) { + selector.addConsumer(c); + } + + // Check that the same consumers are selected as before + for (int j = 0; j < validationPointCount; j++) { + int point = pointsToTest.get(j); + Consumer selected = selector.select(point); + Consumer expected = selectedConsumersBeforeRemove.get(j); + assertThat(selected.consumerId()).as("validationPoint %d, hash %d", j, point).isEqualTo(expected.consumerId()); + } + } + + private List pointsToTest(int validationPointCount) { + List res = new ArrayList<>(); + int hashRangeSize = Integer.MAX_VALUE; + final int increment = hashRangeSize / (validationPointCount + 1); + for (int i = 0; i < validationPointCount; i++) { + res.add(i * increment); } - Assert.assertEquals(expectedResult.size(), 0); + return res; } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumedLedgersTrimTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumedLedgersTrimTest.java index 80db4c30f454d..30867dd2cb44d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumedLedgersTrimTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumedLedgersTrimTest.java @@ -97,11 +97,13 @@ public void TestConsumedLedgersTrim() throws Exception { } ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); - Assert.assertEquals(managedLedger.getLedgersInfoAsList().size(), msgNum / 2); + Awaitility.await().untilAsserted(() -> { + Assert.assertEquals(managedLedger.getLedgersInfoAsList().size() - 1, msgNum / 2); + }); //no traffic, unconsumed ledger will be retained Thread.sleep(1200); - Assert.assertEquals(managedLedger.getLedgersInfoAsList().size(), msgNum / 2); + Assert.assertEquals(managedLedger.getLedgersInfoAsList().size() - 1, msgNum / 2); for (int i = 0; i < msgNum; i++) { Message msg = consumer.receive(2, TimeUnit.SECONDS); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerIdentityWrapperTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerIdentityWrapperTest.java new file mode 100644 index 0000000000000..75c8e6db5d2a0 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerIdentityWrapperTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class ConsumerIdentityWrapperTest { + private static Consumer mockConsumer() { + return mockConsumer("consumer"); + } + + private static Consumer mockConsumer(String consumerName) { + Consumer consumer = mock(Consumer.class); + when(consumer.consumerName()).thenReturn(consumerName); + return consumer; + } + + @Test + public void testEquals() { + Consumer consumer = mockConsumer(); + assertEquals(new ConsumerIdentityWrapper(consumer), new ConsumerIdentityWrapper(consumer)); + } + + @Test + public void testHashCode() { + Consumer consumer = mockConsumer(); + assertEquals(new ConsumerIdentityWrapper(consumer).hashCode(), + new ConsumerIdentityWrapper(consumer).hashCode()); + } + + @Test + public void testEqualsAndHashCode() { + Consumer consumer1 = mockConsumer(); + Consumer consumer2 = mockConsumer(); + ConsumerIdentityWrapper wrapper1 = new ConsumerIdentityWrapper(consumer1); + ConsumerIdentityWrapper wrapper2 = new ConsumerIdentityWrapper(consumer1); + ConsumerIdentityWrapper wrapper3 = new ConsumerIdentityWrapper(consumer2); + + // Test equality + assertEquals(wrapper1, wrapper2); + assertNotEquals(wrapper1, wrapper3); + + // Test hash code + assertEquals(wrapper1.hashCode(), wrapper2.hashCode()); + assertNotEquals(wrapper1.hashCode(), wrapper3.hashCode()); + } +} \ No newline at end of file diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerNameIndexTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerNameIndexTrackerTest.java new file mode 100644 index 0000000000000..0f18ecce2ffb4 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerNameIndexTrackerTest.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class ConsumerNameIndexTrackerTest { + private ConsumerNameIndexTracker tracker; + + @BeforeMethod + public void setUp() { + tracker = new ConsumerNameIndexTracker(); + } + + private static Consumer mockConsumer() { + return mockConsumer("consumer"); + } + + + private static Consumer mockConsumer(String consumerName) { + Consumer consumer = mock(Consumer.class); + when(consumer.consumerName()).thenReturn(consumerName); + return consumer; + } + + @Test + public void testIncreaseConsumerRefCountAndReturnIndex() { + Consumer consumer1 = mockConsumer(); + Consumer consumer2 = mockConsumer(); + ConsumerIdentityWrapper wrapper1 = new ConsumerIdentityWrapper(consumer1); + ConsumerIdentityWrapper wrapper2 = new ConsumerIdentityWrapper(consumer2); + int index1 = tracker.increaseConsumerRefCountAndReturnIndex(wrapper1); + int index2 = tracker.increaseConsumerRefCountAndReturnIndex(wrapper2); + assertNotEquals(index1, index2); + assertEquals(index1, tracker.getTrackedIndex(wrapper1)); + assertEquals(index2, tracker.getTrackedIndex(wrapper2)); + } + + @Test + public void testTrackingReturnsStableIndexWhenRemovedAndAddedInSameOrder() { + List consumerIdentityWrappers = + IntStream.range(0, 100).mapToObj(i -> mockConsumer()).map(ConsumerIdentityWrapper::new).toList(); + Map trackedIndexes = + consumerIdentityWrappers.stream().collect(Collectors.toMap( + wrapper -> wrapper, wrapper -> tracker.increaseConsumerRefCountAndReturnIndex(wrapper))); + // stop tracking every other consumer + for (int i = 0; i < consumerIdentityWrappers.size(); i++) { + if (i % 2 == 0) { + tracker.decreaseConsumerRefCount(consumerIdentityWrappers.get(i)); + } + } + // check that others are tracked + for (int i = 0; i < consumerIdentityWrappers.size(); i++) { + ConsumerIdentityWrapper wrapper = consumerIdentityWrappers.get(i); + int trackedIndex = tracker.getTrackedIndex(wrapper); + assertEquals(trackedIndex, i % 2 == 0 ? -1 : trackedIndexes.get(wrapper)); + } + // check that new consumers are tracked with the same index + for (int i = 0; i < consumerIdentityWrappers.size(); i++) { + ConsumerIdentityWrapper wrapper = consumerIdentityWrappers.get(i); + if (i % 2 == 0) { + int trackedIndex = tracker.increaseConsumerRefCountAndReturnIndex(wrapper); + assertEquals(trackedIndex, trackedIndexes.get(wrapper)); + } + } + // check that all consumers are tracked with the original indexes + for (int i = 0; i < consumerIdentityWrappers.size(); i++) { + ConsumerIdentityWrapper wrapper = consumerIdentityWrappers.get(i); + int trackedIndex = tracker.getTrackedIndex(wrapper); + assertEquals(trackedIndex, trackedIndexes.get(wrapper)); + } + } + + @Test + public void testTrackingMultipleTimes() { + List consumerIdentityWrappers = + IntStream.range(0, 100).mapToObj(i -> mockConsumer()).map(ConsumerIdentityWrapper::new).toList(); + Map trackedIndexes = + consumerIdentityWrappers.stream().collect(Collectors.toMap( + wrapper -> wrapper, wrapper -> tracker.increaseConsumerRefCountAndReturnIndex(wrapper))); + Map trackedIndexes2 = + consumerIdentityWrappers.stream().collect(Collectors.toMap( + wrapper -> wrapper, wrapper -> tracker.increaseConsumerRefCountAndReturnIndex(wrapper))); + assertThat(tracker.getTrackedConsumerNamesCount()).isEqualTo(1); + assertThat(trackedIndexes).containsExactlyInAnyOrderEntriesOf(trackedIndexes2); + consumerIdentityWrappers.forEach(wrapper -> tracker.decreaseConsumerRefCount(wrapper)); + for (ConsumerIdentityWrapper wrapper : consumerIdentityWrappers) { + int trackedIndex = tracker.getTrackedIndex(wrapper); + assertEquals(trackedIndex, trackedIndexes.get(wrapper)); + } + consumerIdentityWrappers.forEach(wrapper -> tracker.decreaseConsumerRefCount(wrapper)); + assertThat(tracker.getTrackedConsumersCount()).isEqualTo(0); + assertThat(tracker.getTrackedConsumerNamesCount()).isEqualTo(0); + } + + @Test + public void testDecreaseConsumerRefCount() { + Consumer consumer1 = mockConsumer(); + ConsumerIdentityWrapper wrapper1 = new ConsumerIdentityWrapper(consumer1); + int index1 = tracker.increaseConsumerRefCountAndReturnIndex(wrapper1); + assertNotEquals(index1, -1); + tracker.decreaseConsumerRefCount(wrapper1); + assertEquals(tracker.getTrackedIndex(wrapper1), -1); + } + + @Test + public void testGetTrackedIndex() { + Consumer consumer1 = mockConsumer(); + Consumer consumer2 = mockConsumer(); + ConsumerIdentityWrapper wrapper1 = new ConsumerIdentityWrapper(consumer1); + ConsumerIdentityWrapper wrapper2 = new ConsumerIdentityWrapper(consumer2); + int index1 = tracker.increaseConsumerRefCountAndReturnIndex(wrapper1); + int index2 = tracker.increaseConsumerRefCountAndReturnIndex(wrapper2); + assertEquals(index1, tracker.getTrackedIndex(wrapper1)); + assertEquals(index2, tracker.getTrackedIndex(wrapper2)); + } + + @Test + public void testTrackingMultipleNames() { + List consumerIdentityWrappers = + IntStream.range(0, 100).mapToObj(i -> mockConsumer("consumer" + i)).map(ConsumerIdentityWrapper::new) + .toList(); + consumerIdentityWrappers.forEach(wrapper -> tracker.increaseConsumerRefCountAndReturnIndex(wrapper)); + assertThat(tracker.getTrackedConsumerNamesCount()).isEqualTo(100); + assertThat(tracker.getTrackedConsumersCount()).isEqualTo(100); + consumerIdentityWrappers.forEach(wrapper -> tracker.decreaseConsumerRefCount(wrapper)); + assertThat(tracker.getTrackedConsumersCount()).isEqualTo(0); + assertThat(tracker.getTrackedConsumerNamesCount()).isEqualTo(0); + } +} \ No newline at end of file diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CurrentLedgerRolloverIfFullTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CurrentLedgerRolloverIfFullTest.java index 375fe41f143cd..4f83d25a29210 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CurrentLedgerRolloverIfFullTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CurrentLedgerRolloverIfFullTest.java @@ -81,7 +81,9 @@ public void testCurrentLedgerRolloverIfFull() throws Exception { } ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); - Assert.assertEquals(managedLedger.getLedgersInfoAsList().size(), msgNum / 2); + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + Assert.assertEquals(managedLedger.getLedgersInfoAsList().size(), msgNum / 2 + 1); + }); for (int i = 0; i < msgNum; i++) { Message msg = consumer.receive(2, TimeUnit.SECONDS); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/DeduplicationDisabledBrokerLevelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/DeduplicationDisabledBrokerLevelTest.java new file mode 100644 index 0000000000000..195d0155a31c6 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/DeduplicationDisabledBrokerLevelTest.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.service.persistent.MessageDeduplication; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.Schema; +import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +public class DeduplicationDisabledBrokerLevelTest extends ProducerConsumerBase { + + private int deduplicationSnapshotFrequency = 5; + private int brokerDeduplicationEntriesInterval = 1000; + + @BeforeClass + @Override + protected void setup() throws Exception { + super.internalSetup(); + producerBaseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + protected void doInitConf() throws Exception { + this.conf.setBrokerDeduplicationEnabled(false); + this.conf.setBrokerDeduplicationSnapshotFrequencyInSeconds(deduplicationSnapshotFrequency); + this.conf.setBrokerDeduplicationEntriesInterval(brokerDeduplicationEntriesInterval); + } + + @Test + public void testNoBacklogOnDeduplication() throws Exception { + final String topic = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createNonPartitionedTopic(topic); + final PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topic, false).join().get(); + final ManagedLedgerImpl ml = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + // deduplication enabled: + // broker level: "false" + // topic level: "true". + // So it is enabled. + admin.topicPolicies().setDeduplicationStatus(topic, true); + Awaitility.await().untilAsserted(() -> { + ManagedCursorImpl cursor = + (ManagedCursorImpl) ml.getCursors().get(PersistentTopic.DEDUPLICATION_CURSOR_NAME); + assertNotNull(cursor); + }); + + // Verify: regarding deduplication cursor, messages will be acknowledged automatically. + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + producer.send("1"); + producer.send("2"); + producer.send("3"); + producer.close(); + ManagedCursorImpl cursor = (ManagedCursorImpl) ml.getCursors().get(PersistentTopic.DEDUPLICATION_CURSOR_NAME); + Awaitility.await().atMost(Duration.ofSeconds(deduplicationSnapshotFrequency * 3)).untilAsserted(() -> { + Position LAC = ml.getLastConfirmedEntry(); + Position cursorMD = cursor.getMarkDeletedPosition(); + assertTrue(LAC.compareTo(cursorMD) <= 0); + }); + + // cleanup. + admin.topics().delete(topic); + } + + @Test + public void testSnapshotCounterAfterUnload() throws Exception { + final int originalDeduplicationSnapshotFrequency = deduplicationSnapshotFrequency; + deduplicationSnapshotFrequency = 3600; + cleanup(); + setup(); + + // Create a topic and wait deduplication is started. + final String topic = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createNonPartitionedTopic(topic); + final PersistentTopic persistentTopic1 = + (PersistentTopic) pulsar.getBrokerService().getTopic(topic, false).join().get(); + final ManagedLedgerImpl ml1 = (ManagedLedgerImpl) persistentTopic1.getManagedLedger(); + admin.topicPolicies().setDeduplicationStatus(topic, true); + Awaitility.await().untilAsserted(() -> { + ManagedCursorImpl cursor1 = + (ManagedCursorImpl) ml1.getCursors().get(PersistentTopic.DEDUPLICATION_CURSOR_NAME); + assertNotNull(cursor1); + }); + final MessageDeduplication deduplication1 = persistentTopic1.getMessageDeduplication(); + + // 1. Send 999 messages, it is less than "brokerDeduplicationEntriesIntervaddl". + // 2. Unload topic. + // 3. Send 1 messages, there are 1099 messages have not been snapshot now. + // 4. Verify the snapshot has been taken. + // step 1. + final Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + for (int i = 0; i < brokerDeduplicationEntriesInterval - 1; i++) { + producer.send(i + ""); + } + int snapshotCounter1 = WhiteboxImpl.getInternalState(deduplication1, "snapshotCounter"); + assertEquals(snapshotCounter1, brokerDeduplicationEntriesInterval - 1); + admin.topics().unload(topic); + PersistentTopic persistentTopic2 = + (PersistentTopic) pulsar.getBrokerService().getTopic(topic, false).join().get(); + ManagedLedgerImpl ml2 = (ManagedLedgerImpl) persistentTopic2.getManagedLedger(); + MessageDeduplication deduplication2 = persistentTopic2.getMessageDeduplication(); + admin.topicPolicies().setDeduplicationStatus(topic, true); + Awaitility.await().untilAsserted(() -> { + ManagedCursorImpl cursor = + (ManagedCursorImpl) ml2.getCursors().get(PersistentTopic.DEDUPLICATION_CURSOR_NAME); + assertNotNull(cursor); + }); + // step 3. + producer.send("last message"); + ml2.trimConsumedLedgersInBackground(new CompletableFuture<>()); + // step 4. + Awaitility.await().untilAsserted(() -> { + int snapshotCounter3 = WhiteboxImpl.getInternalState(deduplication2, "snapshotCounter"); + assertTrue(snapshotCounter3 < brokerDeduplicationEntriesInterval); + // Verify: the previous ledger will be removed because all messages have been acked. + assertEquals(ml2.getLedgersInfo().size(), 1); + }); + + // cleanup. + producer.close(); + admin.topics().delete(topic); + deduplicationSnapshotFrequency = originalDeduplicationSnapshotFrequency; + cleanup(); + setup(); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/DisabledCreateTopicToRemoteClusterForReplicationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/DisabledCreateTopicToRemoteClusterForReplicationTest.java new file mode 100644 index 0000000000000..0f8db4aaa7316 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/DisabledCreateTopicToRemoteClusterForReplicationTest.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import java.util.Arrays; +import java.util.HashSet; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.RetentionPolicies; +import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.apache.pulsar.zookeeper.ZookeeperServerTest; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class DisabledCreateTopicToRemoteClusterForReplicationTest extends OneWayReplicatorTestBase { + + @Override + @BeforeClass(alwaysRun = true, timeOut = 300000) + public void setup() throws Exception { + super.setup(); + admin1.namespaces().setRetention(replicatedNamespace, new RetentionPolicies(300, 1024)); + admin2.namespaces().setRetention(replicatedNamespace, new RetentionPolicies(300, 1024)); + admin1.namespaces().setRetention(nonReplicatedNamespace, new RetentionPolicies(300, 1024)); + admin2.namespaces().setRetention(nonReplicatedNamespace, new RetentionPolicies(300, 1024)); + } + + @Override + @AfterClass(alwaysRun = true, timeOut = 300000) + public void cleanup() throws Exception { + super.cleanup(); + } + + @Override + protected void setConfigDefaults(ServiceConfiguration config, String clusterName, + LocalBookkeeperEnsemble bookkeeperEnsemble, ZookeeperServerTest brokerConfigZk) { + super.setConfigDefaults(config, clusterName, bookkeeperEnsemble, brokerConfigZk); + config.setCreateTopicToRemoteClusterForReplication(false); + config.setReplicationStartAt("earliest"); + } + + @Test + public void testCreatePartitionedTopicWithNsReplication() throws Exception { + String ns = defaultTenant + "/" + UUID.randomUUID().toString().replace("-", ""); + admin1.namespaces().createNamespace(ns); + admin2.namespaces().createNamespace(ns); + admin1.namespaces().setRetention(ns, new RetentionPolicies(3600, -1)); + admin2.namespaces().setRetention(ns, new RetentionPolicies(3600, -1)); + + // Create non-partitioned topic. + // Enable replication. + final String tp = BrokerTestUtil.newUniqueName("persistent://" + ns + "/tp_"); + final String part1 = TopicName.get(tp).getPartition(0).toString(); + admin1.topics().createPartitionedTopic(tp, 1); + admin1.namespaces().setNamespaceReplicationClusters(ns, new HashSet<>(Arrays.asList(cluster1, cluster2))); + + // Trigger and wait for replicator starts. + String msgValue = "msg-1"; + Producer producer1 = client1.newProducer(Schema.STRING).topic(tp).create(); + producer1.send(msgValue); + producer1.close(); + Awaitility.await().untilAsserted(() -> { + PersistentTopic topicPart1 = (PersistentTopic) broker1.getTopic(part1, false).join().get(); + assertFalse(topicPart1.getReplicators().isEmpty()); + }); + + // Verify: there is no topic with the same name on the remote cluster. + try { + admin2.topics().getPartitionedTopicMetadata(tp); + fail("Expected a not found ex"); + } catch (PulsarAdminException.NotFoundException ex) { + // expected. + } + + // Verify: after creating the topic on the remote cluster, all things are fine. + admin2.topics().createPartitionedTopic(tp, 1); + Consumer consumer2 = client2.newConsumer(Schema.STRING).topic(tp).isAckReceiptEnabled(true) + .subscriptionName("s1").subscribe(); + assertEquals(consumer2.receive(10, TimeUnit.SECONDS).getValue(), msgValue); + consumer2.close(); + + // cleanup. + admin1.namespaces().setNamespaceReplicationClusters(ns, new HashSet<>(Arrays.asList(cluster1))); + Awaitility.await().untilAsserted(() -> { + PersistentTopic topicPart1 = (PersistentTopic) broker1.getTopic(part1, false).join().get(); + assertTrue(topicPart1.getReplicators().isEmpty()); + }); + admin1.topics().deletePartitionedTopic(tp, false); + admin2.topics().deletePartitionedTopic(tp, false); + admin1.namespaces().deleteNamespace(ns); + admin2.namespaces().deleteNamespace(ns); + } + + @Test + public void testEnableTopicReplication() throws Exception { + String ns = nonReplicatedNamespace; + + // Create non-partitioned topic. + // Enable replication. + final String tp = BrokerTestUtil.newUniqueName("persistent://" + ns + "/tp_"); + final String part1 = TopicName.get(tp).getPartition(0).toString(); + admin1.topics().createPartitionedTopic(tp, 1); + admin1.topics().setReplicationClusters(tp, Arrays.asList(cluster1, cluster2)); + + // Trigger and wait for replicator starts. + Producer p1 = client1.newProducer(Schema.STRING).topic(tp).create(); + p1.send("msg-1"); + p1.close(); + Awaitility.await().untilAsserted(() -> { + PersistentTopic topicPart1 = (PersistentTopic) broker1.getTopic(part1, false).join().get(); + assertFalse(topicPart1.getReplicators().isEmpty()); + }); + + // Verify: there is no topic with the same name on the remote cluster. + try { + admin2.topics().getPartitionedTopicMetadata(tp); + fail("Expected a not found ex"); + } catch (PulsarAdminException.NotFoundException ex) { + // expected. + } + + // Verify: after creating the topic on the remote cluster, all things are fine. + admin2.topics().createPartitionedTopic(tp, 1); + waitReplicatorStarted(part1); + + // cleanup. + admin1.topics().setReplicationClusters(tp, Arrays.asList(cluster1)); + Awaitility.await().untilAsserted(() -> { + PersistentTopic topicPart1 = (PersistentTopic) broker1.getTopic(part1, false).join().get(); + assertTrue(topicPart1.getReplicators().isEmpty()); + }); + admin1.topics().deletePartitionedTopic(tp, false); + admin2.topics().deletePartitionedTopic(tp, false); + } + + @Test + public void testNonPartitionedTopic() throws Exception { + String ns = nonReplicatedNamespace; + + // Create non-partitioned topic. + // Enable replication. + final String tp = BrokerTestUtil.newUniqueName("persistent://" + ns + "/tp_"); + admin1.topics().createNonPartitionedTopic(tp); + admin1.topics().setReplicationClusters(tp, Arrays.asList(cluster1, cluster2)); + + // Trigger and wait for replicator starts. + Producer p1 = client1.newProducer(Schema.STRING).topic(tp).create(); + p1.send("msg-1"); + p1.close(); + Awaitility.await().untilAsserted(() -> { + PersistentTopic topicPart1 = (PersistentTopic) broker1.getTopic(tp, false).join().get(); + assertFalse(topicPart1.getReplicators().isEmpty()); + }); + + // Verify: there is no topic with the same name on the remote cluster. + try { + admin2.topics().getPartitionedTopicMetadata(tp); + fail("Expected a not found ex"); + } catch (PulsarAdminException.NotFoundException ex) { + // expected. + } + + // Verify: after creating the topic on the remote cluster, all things are fine. + admin2.topics().createNonPartitionedTopic(tp); + waitReplicatorStarted(tp); + + // cleanup. + admin1.topics().setReplicationClusters(tp, Arrays.asList(cluster1)); + Awaitility.await().untilAsserted(() -> { + PersistentTopic topicPart1 = (PersistentTopic) broker1.getTopic(tp, false).join().get(); + assertTrue(topicPart1.getReplicators().isEmpty()); + }); + admin1.topics().delete(tp, false); + admin2.topics().delete(tp, false); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/EnableProxyProtocolTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/EnableProxyProtocolTest.java index 2f128fe6270a5..725b895fe6e14 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/EnableProxyProtocolTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/EnableProxyProtocolTest.java @@ -19,11 +19,19 @@ package org.apache.pulsar.broker.service; import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelHandlerContext; +import java.util.concurrent.TimeUnit; import lombok.Cleanup; -import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.client.api.InjectedClientCnxClientBuilder; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.ClientBuilderImpl; import org.apache.pulsar.client.impl.ClientCnx; import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; import org.apache.pulsar.common.policies.data.SubscriptionStats; import org.apache.pulsar.common.policies.data.TopicStats; import org.awaitility.Awaitility; @@ -32,10 +40,6 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import java.net.InetSocketAddress; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; - @Test(groups = "broker") public class EnableProxyProtocolTest extends BrokerTestBase { @@ -53,7 +57,7 @@ protected void cleanup() throws Exception { } @Test - public void testSimpleProduceAndConsume() throws PulsarClientException { + public void testSimpleProduceAndConsume() throws Exception { final String namespace = "prop/ns-abc"; final String topicName = "persistent://" + namespace + "/testSimpleProduceAndConsume"; final String subName = "my-subscriber-name"; @@ -76,30 +80,106 @@ public void testSimpleProduceAndConsume() throws PulsarClientException { } Assert.assertEquals(received, messages); + + // cleanup. + org.apache.pulsar.broker.service.Consumer serverConsumer = pulsar.getBrokerService().getTopicReference(topicName) + .get().getSubscription(subName).getConsumers().get(0); + ((ServerCnx) serverConsumer.cnx()).close(); + consumer.close(); + producer.close(); + admin.topics().delete(topicName); } @Test - public void testProxyProtocol() throws PulsarClientException, ExecutionException, InterruptedException, PulsarAdminException { + public void testProxyProtocol() throws Exception { final String namespace = "prop/ns-abc"; final String topicName = "persistent://" + namespace + "/testProxyProtocol"; final String subName = "my-subscriber-name"; - PulsarClientImpl client = (PulsarClientImpl) pulsarClient; - CompletableFuture cnx = client.getCnxPool().getConnection(InetSocketAddress.createUnresolved("localhost", pulsar.getBrokerListenPort().get())); - // Simulate the proxy protcol message - cnx.get().ctx().channel().writeAndFlush(Unpooled.copiedBuffer("PROXY TCP4 198.51.100.22 203.0.113.7 35646 80\r\n".getBytes())); - pulsarClient.newConsumer().topic(topicName).subscriptionName(subName) - .subscribe(); - org.apache.pulsar.broker.service.Consumer c = pulsar.getBrokerService().getTopicReference(topicName).get().getSubscription(subName).getConsumers().get(0); - Awaitility.await().untilAsserted(() -> Assert.assertTrue(c.cnx().hasHAProxyMessage())); + + // Create a client that injected the protocol implementation. + ClientBuilderImpl clientBuilder = (ClientBuilderImpl) PulsarClient.builder().serviceUrl(lookupUrl.toString()); + @Cleanup + PulsarClientImpl protocolClient = InjectedClientCnxClientBuilder.create(clientBuilder, + (conf, eventLoopGroup) -> new ClientCnx(InstrumentProvider.NOOP, conf, eventLoopGroup) { + public void channelActive(ChannelHandlerContext ctx) throws Exception { + byte[] bs = "PROXY TCP4 198.51.100.22 203.0.113.7 35646 80\r\n".getBytes(); + ctx.writeAndFlush(Unpooled.copiedBuffer(bs)); + super.channelActive(ctx); + } + }); + + // Verify the addr can be handled correctly. + testPubAndSub(topicName, subName, "198.51.100.22:35646", protocolClient); + + // cleanup. + admin.topics().delete(topicName); + } + + @Test(timeOut = 10000) + public void testPubSubWhenSlowNetwork() throws Exception { + final String namespace = "prop/ns-abc"; + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + namespace + "/tp"); + final String subName = "my-subscriber-name"; + + // Create a client that injected the protocol implementation. + ClientBuilderImpl clientBuilder = (ClientBuilderImpl) PulsarClient.builder().serviceUrl(lookupUrl.toString()); + @Cleanup + PulsarClientImpl protocolClient = InjectedClientCnxClientBuilder.create(clientBuilder, + (conf, eventLoopGroup) -> new ClientCnx(InstrumentProvider.NOOP, conf, eventLoopGroup) { + public void channelActive(ChannelHandlerContext ctx) throws Exception { + Thread task = new Thread(() -> { + try { + byte[] bs1 = "PROXY".getBytes(); + byte[] bs2 = " TCP4 198.51.100.22 203.0.113.7 35646 80\r\n".getBytes(); + ctx.writeAndFlush(Unpooled.copiedBuffer(bs1)); + Thread.sleep(100); + ctx.writeAndFlush(Unpooled.copiedBuffer(bs2)); + super.channelActive(ctx); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + task.start(); + } + }); + + // Verify the addr can be handled correctly. + testPubAndSub(topicName, subName, "198.51.100.22:35646", protocolClient); + + // cleanup. + admin.topics().delete(topicName); + } + + private void testPubAndSub(String topicName, String subName, String expectedHostAndPort, + PulsarClientImpl pulsarClient) throws Exception { + // Verify: subscribe + org.apache.pulsar.client.api.Consumer clientConsumer = pulsarClient.newConsumer(Schema.STRING).topic(topicName) + .subscriptionName(subName).subscribe(); + org.apache.pulsar.broker.service.Consumer serverConsumer = pulsar.getBrokerService() + .getTopicReference(topicName).get().getSubscription(subName).getConsumers().get(0); + Awaitility.await().untilAsserted(() -> Assert.assertTrue(serverConsumer.cnx().hasHAProxyMessage())); TopicStats topicStats = admin.topics().getStats(topicName); Assert.assertEquals(topicStats.getSubscriptions().size(), 1); SubscriptionStats subscriptionStats = topicStats.getSubscriptions().get(subName); Assert.assertEquals(subscriptionStats.getConsumers().size(), 1); - Assert.assertEquals(subscriptionStats.getConsumers().get(0).getAddress(), "198.51.100.22:35646"); + Assert.assertEquals(subscriptionStats.getConsumers().get(0).getAddress(), expectedHostAndPort); + + // Verify: producer register. + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topicName).create(); + TopicStats topicStats2 = admin.topics().getStats(topicName); + Assert.assertEquals(topicStats2.getPublishers().size(), 1); + Assert.assertEquals(topicStats2.getPublishers().get(0).getAddress(), expectedHostAndPort); + + // Verify: Pub & Sub + producer.send("1"); + Message msg = clientConsumer.receive(2, TimeUnit.SECONDS); + Assert.assertNotNull(msg); + Assert.assertEquals(msg.getValue(), "1"); + clientConsumer.acknowledge(msg); - pulsarClient.newProducer().topic(topicName).create(); - topicStats = admin.topics().getStats(topicName); - Assert.assertEquals(topicStats.getPublishers().size(), 1); - Assert.assertEquals(topicStats.getPublishers().get(0).getAddress(), "198.51.100.22:35646"); + // cleanup. + ((ServerCnx) serverConsumer.cnx()).close(); + producer.close(); + clientConsumer.close(); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/GeoReplicationWithConfigurationSyncTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/GeoReplicationWithConfigurationSyncTestBase.java new file mode 100644 index 0000000000000..1362a046247d8 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/GeoReplicationWithConfigurationSyncTestBase.java @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import com.google.common.collect.Sets; +import com.google.common.io.Resources; +import java.net.URL; +import java.util.Collections; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.policies.data.TopicType; +import org.apache.pulsar.tests.TestRetrySupport; +import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.apache.pulsar.zookeeper.ZookeeperServerTest; + +@Slf4j +public abstract class GeoReplicationWithConfigurationSyncTestBase extends TestRetrySupport { + + protected final String defaultTenant = "public"; + protected final String defaultNamespace = defaultTenant + "/default"; + final static String caCertPath = Resources.getResource("certificate-authority/certs/ca.cert.pem") + .getPath(); + final static String brokerCertPath = + Resources.getResource("certificate-authority/server-keys/broker.cert.pem").getPath(); + final static String brokerKeyPath = + Resources.getResource("certificate-authority/server-keys/broker.key-pk8.pem").getPath(); + + protected final String cluster1 = "r1"; + protected URL url1; + protected URL urlTls1; + protected ServiceConfiguration config1 = new ServiceConfiguration(); + protected ZookeeperServerTest brokerConfigZk1; + protected LocalBookkeeperEnsemble bkEnsemble1; + protected PulsarService pulsar1; + protected BrokerService ns1; + protected PulsarAdmin admin1; + protected PulsarClient client1; + + protected URL url2; + protected URL urlTls2; + protected final String cluster2 = "r2"; + protected ServiceConfiguration config2 = new ServiceConfiguration(); + protected ZookeeperServerTest brokerConfigZk2; + protected LocalBookkeeperEnsemble bkEnsemble2; + protected PulsarService pulsar2; + protected BrokerService ns2; + protected PulsarAdmin admin2; + protected PulsarClient client2; + + protected void startZKAndBK() throws Exception { + // Start ZK. + brokerConfigZk1 = new ZookeeperServerTest(0); + brokerConfigZk1.start(); + brokerConfigZk2 = new ZookeeperServerTest(0); + brokerConfigZk2.start(); + + // Start BK. + bkEnsemble1 = new LocalBookkeeperEnsemble(3, 0, () -> 0); + bkEnsemble1.start(); + bkEnsemble2 = new LocalBookkeeperEnsemble(3, 0, () -> 0); + bkEnsemble2.start(); + } + + protected void startBrokers() throws Exception { + // Start brokers. + setConfigDefaults(config1, cluster1, bkEnsemble1, brokerConfigZk1); + pulsar1 = new PulsarService(config1); + pulsar1.start(); + ns1 = pulsar1.getBrokerService(); + + url1 = new URL(pulsar1.getWebServiceAddress()); + urlTls1 = new URL(pulsar1.getWebServiceAddressTls()); + admin1 = PulsarAdmin.builder().serviceHttpUrl(url1.toString()).build(); + client1 = PulsarClient.builder().serviceUrl(url1.toString()).build(); + + // Start region 2 + setConfigDefaults(config2, cluster2, bkEnsemble2, brokerConfigZk2); + pulsar2 = new PulsarService(config2); + pulsar2.start(); + ns2 = pulsar2.getBrokerService(); + + url2 = new URL(pulsar2.getWebServiceAddress()); + urlTls2 = new URL(pulsar2.getWebServiceAddressTls()); + admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build(); + client2 = PulsarClient.builder().serviceUrl(url2.toString()).build(); + } + + protected void createDefaultTenantsAndClustersAndNamespace() throws Exception { + admin1.clusters().createCluster(cluster1, ClusterData.builder() + .serviceUrl(url1.toString()) + .serviceUrlTls(urlTls1.toString()) + .brokerServiceUrl(pulsar1.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar1.getBrokerServiceUrlTls()) + .brokerClientTlsEnabled(false) + .build()); + admin1.clusters().createCluster(cluster2, ClusterData.builder() + .serviceUrl(url2.toString()) + .serviceUrlTls(urlTls2.toString()) + .brokerServiceUrl(pulsar2.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar2.getBrokerServiceUrlTls()) + .brokerClientTlsEnabled(false) + .build()); + admin2.clusters().createCluster(cluster1, ClusterData.builder() + .serviceUrl(url1.toString()) + .serviceUrlTls(urlTls1.toString()) + .brokerServiceUrl(pulsar1.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar1.getBrokerServiceUrlTls()) + .brokerClientTlsEnabled(false) + .build()); + admin2.clusters().createCluster(cluster2, ClusterData.builder() + .serviceUrl(url2.toString()) + .serviceUrlTls(urlTls2.toString()) + .brokerServiceUrl(pulsar2.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar2.getBrokerServiceUrlTls()) + .brokerClientTlsEnabled(false) + .build()); + + admin1.tenants().createTenant(defaultTenant, new TenantInfoImpl(Collections.emptySet(), + Sets.newHashSet(cluster1, cluster2))); + admin2.tenants().createTenant(defaultTenant, new TenantInfoImpl(Collections.emptySet(), + Sets.newHashSet(cluster1, cluster2))); + + admin1.namespaces().createNamespace(defaultNamespace); + admin2.namespaces().createNamespace(defaultNamespace); + } + + @Override + protected void setup() throws Exception { + incrementSetupNumber(); + + log.info("--- Starting OneWayReplicatorTestBase::setup ---"); + + startZKAndBK(); + + startBrokers(); + + createDefaultTenantsAndClustersAndNamespace(); + + Thread.sleep(100); + log.info("--- OneWayReplicatorTestBase::setup completed ---"); + } + + protected void setConfigDefaults(ServiceConfiguration config, String clusterName, + LocalBookkeeperEnsemble bookkeeperEnsemble, ZookeeperServerTest brokerConfigZk) { + config.setClusterName(clusterName); + config.setAdvertisedAddress("localhost"); + config.setWebServicePort(Optional.of(0)); + config.setWebServicePortTls(Optional.of(0)); + config.setMetadataStoreUrl("zk:127.0.0.1:" + bookkeeperEnsemble.getZookeeperPort()); + config.setConfigurationMetadataStoreUrl("zk:127.0.0.1:" + brokerConfigZk.getZookeeperPort() + "/foo"); + config.setBrokerDeleteInactiveTopicsEnabled(false); + config.setBrokerDeleteInactiveTopicsFrequencySeconds(60); + config.setBrokerShutdownTimeoutMs(0L); + config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); + config.setBrokerServicePort(Optional.of(0)); + config.setBrokerServicePortTls(Optional.of(0)); + config.setBacklogQuotaCheckIntervalInSeconds(5); + config.setDefaultNumberOfNamespaceBundles(1); + config.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED); + config.setEnableReplicatedSubscriptions(true); + config.setReplicatedSubscriptionsSnapshotFrequencyMillis(1000); + config.setLoadBalancerSheddingEnabled(false); + config.setTlsTrustCertsFilePath(caCertPath); + config.setTlsCertificateFilePath(brokerCertPath); + config.setTlsKeyFilePath(brokerKeyPath); + } + + @Override + protected void cleanup() throws Exception { + // shutdown. + markCurrentSetupNumberCleaned(); + log.info("--- Shutting down ---"); + + // Stop brokers. + if (client1 != null) { + client1.close(); + client1 = null; + } + if (client2 != null) { + client2.close(); + client2 = null; + } + if (admin1 != null) { + admin1.close(); + admin1 = null; + } + if (admin2 != null) { + admin2.close(); + admin2 = null; + } + if (pulsar2 != null) { + pulsar2.close(); + pulsar2 = null; + } + if (pulsar1 != null) { + pulsar1.close(); + pulsar1 = null; + } + + // Stop ZK and BK. + if (bkEnsemble1 != null) { + bkEnsemble1.stop(); + bkEnsemble1 = null; + } + if (bkEnsemble2 != null) { + bkEnsemble2.stop(); + bkEnsemble2 = null; + } + if (brokerConfigZk1 != null) { + brokerConfigZk1.stop(); + brokerConfigZk1 = null; + } + if (brokerConfigZk2 != null) { + brokerConfigZk2.stop(); + brokerConfigZk2 = null; + } + + // Reset configs. + config1 = new ServiceConfiguration(); + config2 = new ServiceConfiguration(); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/InactiveTopicDeleteTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/InactiveTopicDeleteTest.java index ebd53d65a7465..1549ba8d81c09 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/InactiveTopicDeleteTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/InactiveTopicDeleteTest.java @@ -35,8 +35,6 @@ import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.impl.ConsumerImpl; -import org.apache.pulsar.client.impl.MultiTopicsConsumerImpl; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.naming.TopicVersion; @@ -307,7 +305,8 @@ public void testDeleteWhenNoSubscriptionsWithMultiConfig() throws Exception { } Awaitility.await() .untilAsserted(() -> Assert.assertFalse(admin.topics().getList(namespace).contains(topic))); - Assert.assertFalse(admin.topics().getList(namespace3).contains(topic3)); + Awaitility.await() + .untilAsserted(() -> Assert.assertFalse(admin.topics().getList(namespace3).contains(topic3))); } @Test @@ -356,12 +355,9 @@ public void testDeleteWhenNoBacklogs() throws Exception { admin.topics().skipAllMessages(topic, "sub"); Awaitility.await().untilAsserted(() -> { - Assert.assertFalse(consumer.isConnected()); - final List consumers = ((MultiTopicsConsumerImpl) consumer2).getConsumers(); - consumers.forEach(c -> Assert.assertFalse(c.isConnected())); - Assert.assertFalse(consumer2.isConnected()); - Assert.assertFalse(admin.topics().getList("prop/ns-abc").contains(topic)); - Assert.assertFalse(admin.topics().getList("prop/ns-abc").contains(topic2)); + final List topics = admin.topics().getList("prop/ns-abc"); + Assert.assertFalse(topics.contains(topic)); + Assert.assertFalse(topics.contains(topic2)); }); consumer.close(); consumer2.close(); @@ -602,10 +598,12 @@ public void testHealthTopicInactiveNotClean() throws Exception { conf.setBrokerDeleteInactiveTopicsFrequencySeconds(1); super.baseSetup(); // init topic - NamespaceName heartbeatNamespaceV1 = NamespaceService.getHeartbeatNamespace(pulsar.getAdvertisedAddress(), pulsar.getConfig()); + NamespaceName heartbeatNamespaceV1 = NamespaceService + .getHeartbeatNamespace(pulsar.getBrokerId(), pulsar.getConfig()); final String healthCheckTopicV1 = "persistent://" + heartbeatNamespaceV1 + "/healthcheck"; - NamespaceName heartbeatNamespaceV2 = NamespaceService.getHeartbeatNamespaceV2(pulsar.getAdvertisedAddress(), pulsar.getConfig()); + NamespaceName heartbeatNamespaceV2 = NamespaceService + .getHeartbeatNamespaceV2(pulsar.getBrokerId(), pulsar.getConfig()); final String healthCheckTopicV2 = "persistent://" + heartbeatNamespaceV2 + "/healthcheck"; admin.brokers().healthcheck(TopicVersion.V1); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/InmemoryTopicPoliciesService.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/InmemoryTopicPoliciesService.java new file mode 100644 index 0000000000000..88a75fe8f0387 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/InmemoryTopicPoliciesService.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TopicPolicies; + +public class InmemoryTopicPoliciesService implements TopicPoliciesService { + + private final Map cache = new HashMap<>(); + private final Map> listeners = new HashMap<>(); + + @Override + public synchronized CompletableFuture deleteTopicPoliciesAsync(TopicName topicName) { + cache.remove(topicName); + return CompletableFuture.completedFuture(null); + } + + @Override + public synchronized CompletableFuture updateTopicPoliciesAsync(TopicName topicName, TopicPolicies policies) { + final var existingPolicies = cache.get(topicName); + if (existingPolicies != policies) { + cache.put(topicName, policies); + CompletableFuture.runAsync(() -> { + final TopicPolicies latestPolicies; + final List listeners; + synchronized (InmemoryTopicPoliciesService.this) { + latestPolicies = cache.get(topicName); + listeners = this.listeners.getOrDefault(topicName, List.of()); + } + for (var listener : listeners) { + listener.onUpdate(latestPolicies); + } + }); + } + return CompletableFuture.completedFuture(null); + } + + @Override + public synchronized CompletableFuture> getTopicPoliciesAsync( + TopicName topicName, GetType type) { + return CompletableFuture.completedFuture(Optional.ofNullable(cache.get(topicName))); + } + + @Override + public synchronized boolean registerListener(TopicName topicName, TopicPolicyListener listener) { + listeners.computeIfAbsent(topicName, __ -> new ArrayList<>()).add(listener); + return true; + } + + @Override + public synchronized void unregisterListener(TopicName topicName, TopicPolicyListener listener) { + listeners.get(topicName).remove(listener); + } + + synchronized boolean containsKey(TopicName topicName) { + return cache.containsKey(topicName); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/InmemoryTopicPoliciesServiceServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/InmemoryTopicPoliciesServiceServiceTest.java new file mode 100644 index 0000000000000..9ec16405ba853 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/InmemoryTopicPoliciesServiceServiceTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import com.google.common.collect.Lists; +import java.util.List; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.naming.TopicName; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class InmemoryTopicPoliciesServiceServiceTest extends MockedPulsarServiceBaseTest { + + @BeforeClass + @Override + protected void setup() throws Exception { + conf.setTopicPoliciesServiceClassName(InmemoryTopicPoliciesService.class.getName()); + conf.setSystemTopicEnabled(false); // verify topic policies don't rely on system topics + super.internalSetup(); + super.setupDefaultTenantAndNamespace(); + } + + @AfterClass + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + // Shadow replicator is created by the topic policies update, this test verifies the listener can be triggered + @Test + public void testShadowReplicator() throws Exception { + final var sourceTopic = TopicName.get("test-shadow-replicator").toString(); + final var shadowTopic = sourceTopic + "-shadow"; + + admin.topics().createNonPartitionedTopic(sourceTopic); + admin.topics().createShadowTopic(shadowTopic, sourceTopic); + admin.topics().setShadowTopics(sourceTopic, Lists.newArrayList(shadowTopic)); + + @Cleanup final var producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create(); + @Cleanup final var consumer = pulsarClient.newConsumer(Schema.STRING).topic(shadowTopic) + .subscriptionName("sub").subscribe(); + producer.send("msg"); + final var msg = consumer.receive(5, TimeUnit.SECONDS); + Assert.assertNotNull(msg); + Assert.assertEquals(msg.getValue(), "msg"); + + final var persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopicIfExists(sourceTopic).get() + .orElseThrow(); + Assert.assertEquals(TopicPolicyTestUtils.getTopicPolicies(persistentTopic).getShadowTopics(), List.of(shadowTopic)); + Assert.assertEquals(persistentTopic.getShadowReplicators().size(), 1); + } + + @Test + public void testTopicPoliciesAdmin() throws Exception { + final var topic = "test-topic-policies-admin"; + admin.topics().createNonPartitionedTopic(topic); + + Assert.assertNull(admin.topicPolicies().getCompactionThreshold(topic)); + admin.topicPolicies().setCompactionThreshold(topic, 1000); + Assert.assertEquals(admin.topicPolicies().getCompactionThreshold(topic).intValue(), 1000); + // Sleep here because "Directory not empty error" might occur if deleting the topic immediately + Thread.sleep(1000); + final var topicPoliciesService = (InmemoryTopicPoliciesService) pulsar.getTopicPoliciesService(); + Assert.assertTrue(topicPoliciesService.containsKey(TopicName.get(topic))); + admin.topics().delete(topic); + Assert.assertFalse(topicPoliciesService.containsKey(TopicName.get(topic))); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/Ipv4Proxy.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/Ipv4Proxy.java new file mode 100644 index 0000000000000..a84dab4d17dff --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/Ipv4Proxy.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import io.netty.bootstrap.Bootstrap; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.buffer.Unpooled; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.handler.logging.LogLevel; +import io.netty.handler.logging.LoggingHandler; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import lombok.Getter; + +public class Ipv4Proxy { + @Getter + private final int localPort; + private final String backendServerHost; + private final int backendServerPort; + private final EventLoopGroup serverGroup = new NioEventLoopGroup(1); + private final EventLoopGroup workerGroup = new NioEventLoopGroup(); + private ChannelFuture localServerChannel; + private ServerBootstrap serverBootstrap = new ServerBootstrap(); + private List frontChannels = Collections.synchronizedList(new ArrayList<>()); + private AtomicBoolean rejectAllConnections = new AtomicBoolean(); + + public Ipv4Proxy(int localPort, String backendServerHost, int backendServerPort) { + this.localPort = localPort; + this.backendServerHost = backendServerHost; + this.backendServerPort = backendServerPort; + } + + public synchronized void startup() throws InterruptedException { + localServerChannel = serverBootstrap.group(serverGroup, workerGroup) + .channel(NioServerSocketChannel.class) + .handler(new LoggingHandler(LogLevel.INFO)) + .childHandler(new ChannelInitializer() { + @Override + protected void initChannel(SocketChannel ch) { + ch.pipeline().addLast(new FrontendHandler()); + } + }).childOption(ChannelOption.AUTO_READ, false) + .bind(localPort).sync(); + } + + public synchronized void stop() throws InterruptedException{ + localServerChannel.channel().close().sync(); + serverGroup.shutdownGracefully(); + workerGroup.shutdownGracefully(); + } + + private static void closeOnFlush(Channel ch) { + if (ch.isActive()) { + ch.writeAndFlush(Unpooled.EMPTY_BUFFER).addListener(ChannelFutureListener.CLOSE); + } + } + + public void disconnectFrontChannels() throws InterruptedException { + for (Channel channel : frontChannels) { + channel.close(); + } + } + + public void rejectAllConnections() throws InterruptedException { + rejectAllConnections.set(true); + } + + public void unRejectAllConnections() throws InterruptedException { + rejectAllConnections.set(false); + } + + private class FrontendHandler extends ChannelInboundHandlerAdapter { + + private Channel backendChannel; + + @Override + public void channelActive(ChannelHandlerContext ctx) { + if (rejectAllConnections.get()) { + ctx.close(); + return; + } + final Channel frontendChannel = ctx.channel(); + frontChannels.add(frontendChannel); + Bootstrap backendBootstrap = new Bootstrap(); + backendBootstrap.group(frontendChannel.eventLoop()) + .channel(ctx.channel().getClass()) + .handler(new BackendHandler(frontendChannel)) + .option(ChannelOption.AUTO_READ, false); + ChannelFuture backendChannelFuture = + backendBootstrap.connect(Ipv4Proxy.this.backendServerHost, Ipv4Proxy.this.backendServerPort); + backendChannel = backendChannelFuture.channel(); + backendChannelFuture.addListener((ChannelFutureListener) future -> { + if (future.isSuccess()) { + frontendChannel.read(); + } else { + frontChannels.remove(frontendChannel); + frontendChannel.close(); + } + }); + } + + @Override + public void channelRead(final ChannelHandlerContext ctx, Object msg) { + if (backendChannel.isActive()) { + backendChannel.writeAndFlush(msg).addListener((ChannelFutureListener) future -> { + if (future.isSuccess()) { + ctx.channel().read(); + } else { + future.channel().close(); + } + }); + } + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) { + frontChannels.remove(ctx.channel()); + if (backendChannel != null) { + closeOnFlush(backendChannel); + } + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + cause.printStackTrace(); + closeOnFlush(ctx.channel()); + } + } + + private class BackendHandler extends ChannelInboundHandlerAdapter { + + private final Channel frontendChannel; + + public BackendHandler(Channel inboundChannel) { + this.frontendChannel = inboundChannel; + } + + @Override + public void channelActive(ChannelHandlerContext ctx) { + if (!frontendChannel.isActive()) { + closeOnFlush(ctx.channel()); + } else { + ctx.read(); + } + } + + @Override + public void channelRead(final ChannelHandlerContext ctx, Object msg) { + frontendChannel.writeAndFlush(msg).addListener((ChannelFutureListener) future -> { + if (future.isSuccess()) { + ctx.channel().read(); + } else { + future.channel().close(); + } + }); + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) { + closeOnFlush(frontendChannel); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + cause.printStackTrace(); + closeOnFlush(ctx.channel()); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MaxMessageSizeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MaxMessageSizeTest.java index fb15661fddf2d..84543a82d7725 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MaxMessageSizeTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MaxMessageSizeTest.java @@ -91,8 +91,18 @@ void setup() { @AfterMethod(alwaysRun = true) void shutdown() { try { - pulsar.close(); - bkEnsemble.stop(); + if (admin != null) { + admin.close(); + admin = null; + } + if (pulsar != null) { + pulsar.close(); + pulsar = null; + } + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; + } } catch (Throwable t) { t.printStackTrace(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageCumulativeAckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageCumulativeAckTest.java index f3fe26af4b968..cc4fe22962484 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageCumulativeAckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageCumulativeAckTest.java @@ -37,6 +37,7 @@ import io.netty.channel.ChannelHandlerContext; import java.net.InetSocketAddress; import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; @@ -74,7 +75,9 @@ public void setup() throws Exception { .when(serverCnx).getCommandSender(); String topicName = TopicName.get("MessageCumulativeAckTest").toString(); - PersistentTopic persistentTopic = new PersistentTopic(topicName, mock(ManagedLedger.class), pulsarTestContext.getBrokerService()); + var mockManagedLedger = mock(ManagedLedger.class); + when(mockManagedLedger.getConfig()).thenReturn(new ManagedLedgerConfig()); + var persistentTopic = new PersistentTopic(topicName, mockManagedLedger, pulsarTestContext.getBrokerService()); sub = spy(new PersistentSubscription(persistentTopic, "sub-1", mock(ManagedCursorImpl.class), false)); doNothing().when(sub).acknowledgeMessage(any(), any(), any()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessagePublishBufferThrottleTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessagePublishBufferThrottleTest.java index 173f772a7316f..0faae14da08ba 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessagePublishBufferThrottleTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessagePublishBufferThrottleTest.java @@ -18,12 +18,18 @@ */ package org.apache.pulsar.broker.service; -import static org.testng.Assert.assertEquals; +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat; +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongSumValue; import static org.testng.Assert.fail; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import lombok.Cleanup; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.ConnectionRateLimitOperationName; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -43,6 +49,12 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + @Override + protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder pulsarTestContextBuilder) { + super.customizeMainPulsarTestContextBuilder(pulsarTestContextBuilder); + pulsarTestContextBuilder.enableOpenTelemetry(true); + } + @Test public void testMessagePublishBufferThrottleDisabled() throws Exception { conf.setMaxMessagePublishBufferSizeInMB(-1); @@ -52,7 +64,8 @@ public void testMessagePublishBufferThrottleDisabled() throws Exception { .topic(topic) .producerName("producer-name") .create(); - assertEquals(pulsar.getBrokerService().getPausedConnections(), 0); + assertRateLimitCounter(ConnectionRateLimitOperationName.PAUSED, 0); + assertRateLimitCounter(ConnectionRateLimitOperationName.RESUMED, 0); pulsarTestContext.getMockBookKeeper().addEntryDelay(1, TimeUnit.SECONDS); @@ -63,7 +76,8 @@ public void testMessagePublishBufferThrottleDisabled() throws Exception { } producer.flush(); - assertEquals(pulsar.getBrokerService().getPausedConnections(), 0); + assertRateLimitCounter(ConnectionRateLimitOperationName.PAUSED, 0); + assertRateLimitCounter(ConnectionRateLimitOperationName.RESUMED, 0); } @Test @@ -71,14 +85,14 @@ public void testMessagePublishBufferThrottleEnable() throws Exception { conf.setMaxMessagePublishBufferSizeInMB(1); super.baseSetup(); - assertEquals(pulsar.getBrokerService().getPausedConnections(), 0); final String topic = "persistent://prop/ns-abc/testMessagePublishBufferThrottleEnable"; Producer producer = pulsarClient.newProducer() .topic(topic) .producerName("producer-name") .create(); - assertEquals(pulsar.getBrokerService().getPausedConnections(), 0); + assertRateLimitCounter(ConnectionRateLimitOperationName.PAUSED, 0); + assertRateLimitCounter(ConnectionRateLimitOperationName.RESUMED, 0); pulsarTestContext.getMockBookKeeper().addEntryDelay(1, TimeUnit.SECONDS); @@ -87,23 +101,27 @@ public void testMessagePublishBufferThrottleEnable() throws Exception { producer.sendAsync(payload); } - Awaitility.await().untilAsserted( - () -> Assert.assertEquals(pulsar.getBrokerService().getPausedConnections(), 1L)); - assertEquals(pulsar.getBrokerService().getPausedConnections(), 1); + Awaitility.await().untilAsserted(() -> { + assertRateLimitCounter(ConnectionRateLimitOperationName.PAUSED, 0); + assertRateLimitCounter(ConnectionRateLimitOperationName.RESUMED, 0); + }); producer.flush(); - Awaitility.await().untilAsserted( - () -> Assert.assertEquals(pulsar.getBrokerService().getPausedConnections(), 0L)); - - assertEquals(pulsar.getBrokerService().getPausedConnections(), 0); + Awaitility.await().untilAsserted(() -> { + assertRateLimitCounter(ConnectionRateLimitOperationName.PAUSED, 0); + assertRateLimitCounter(ConnectionRateLimitOperationName.RESUMED, 0); + }); } @Test public void testBlockByPublishRateLimiting() throws Exception { conf.setMaxMessagePublishBufferSizeInMB(1); super.baseSetup(); - assertEquals(pulsar.getBrokerService().getPausedConnections(), 0); + + assertRateLimitCounter(ConnectionRateLimitOperationName.PAUSED, 0); + assertRateLimitCounter(ConnectionRateLimitOperationName.RESUMED, 0); + final String topic = "persistent://prop/ns-abc/testBlockByPublishRateLimiting"; Producer producer = pulsarClient.newProducer() .topic(topic) @@ -111,7 +129,8 @@ public void testBlockByPublishRateLimiting() throws Exception { .create(); Topic topicRef = pulsar.getBrokerService().getTopicReference(topic).get(); Assert.assertNotNull(topicRef); - assertEquals(pulsar.getBrokerService().getPausedConnections(), 0); + assertRateLimitCounter(ConnectionRateLimitOperationName.PAUSED, 0); + assertRateLimitCounter(ConnectionRateLimitOperationName.RESUMED, 0); pulsarTestContext.getMockBookKeeper().addEntryDelay(5, TimeUnit.SECONDS); @@ -121,13 +140,15 @@ public void testBlockByPublishRateLimiting() throws Exception { producer.sendAsync(payload); } - Awaitility.await().untilAsserted(() -> assertEquals(pulsar.getBrokerService().getPausedConnections(), 1)); + Awaitility.await().untilAsserted(() -> assertRateLimitCounter(ConnectionRateLimitOperationName.PAUSED, 1)); CompletableFuture flushFuture = producer.flushAsync(); // Block by publish rate. // After 1 second, the message buffer throttling will be lifted, but the rate limiting will still be in place. - assertEquals(pulsar.getBrokerService().getPausedConnections(), 1); + assertRateLimitCounter(ConnectionRateLimitOperationName.PAUSED, 1); + assertRateLimitCounter(ConnectionRateLimitOperationName.RESUMED, 0); + try { flushFuture.get(2, TimeUnit.SECONDS); fail("Should have timed out"); @@ -137,14 +158,52 @@ public void testBlockByPublishRateLimiting() throws Exception { flushFuture.join(); - Awaitility.await().untilAsserted(() -> - assertEquals(pulsar.getBrokerService().getPausedConnections(), 0)); + Awaitility.await().untilAsserted(() -> { + assertRateLimitCounter(ConnectionRateLimitOperationName.PAUSED, 10); + assertRateLimitCounter(ConnectionRateLimitOperationName.RESUMED, 10); + }); + } + + @Test + public void testConnectionThrottled() throws Exception { + super.baseSetup(); + + var topic = BrokerTestUtil.newUniqueName("persistent://prop/ns-abc/testSendThrottled"); + + assertRateLimitCounter(ConnectionRateLimitOperationName.THROTTLED, 0); + assertRateLimitCounter(ConnectionRateLimitOperationName.UNTHROTTLED, 0); + + @Cleanup + var producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(false) + .topic(topic) + .create(); + final int messages = 2000; + for (int i = 0; i < messages; i++) { + producer.sendAsync("Message - " + i); + } + producer.flush(); - // Resume message publish. - ((AbstractTopic)topicRef).producers.get("producer-name").getCnx().enableCnxAutoRead(); + // Wait for the connection to be throttled and unthrottled. + Awaitility.await().untilAsserted(() -> { + var metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + assertMetricLongSumValue(metrics, BrokerService.CONNECTION_RATE_LIMIT_COUNT_METRIC_NAME, + ConnectionRateLimitOperationName.THROTTLED.attributes, value -> assertThat(value).isPositive()); + assertMetricLongSumValue(metrics, BrokerService.CONNECTION_RATE_LIMIT_COUNT_METRIC_NAME, + ConnectionRateLimitOperationName.UNTHROTTLED.attributes, value -> assertThat(value).isPositive()); + }); + } - flushFuture.get(); - Awaitility.await().untilAsserted(() -> - assertEquals(pulsar.getBrokerService().getPausedConnections(), 0)); + private void assertRateLimitCounter(ConnectionRateLimitOperationName connectionRateLimitState, int expectedCount) { + var metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + if (expectedCount == 0) { + assertThat(metrics).noneSatisfy(metricData -> assertThat(metricData) + .hasName(BrokerService.CONNECTION_RATE_LIMIT_COUNT_METRIC_NAME) + .hasLongSumSatisfying(sum -> sum.hasPointsSatisfying( + points -> points.hasAttributes(connectionRateLimitState.attributes)))); + } else { + assertMetricLongSumValue(metrics, BrokerService.CONNECTION_RATE_LIMIT_COUNT_METRIC_NAME, + connectionRateLimitState.attributes, expectedCount); + } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageTTLTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageTTLTest.java index 68a9a769ac1fe..909702445f715 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageTTLTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageTTLTest.java @@ -23,15 +23,23 @@ import static org.mockito.Mockito.verify; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import lombok.Cleanup; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.CursorStats; import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; @@ -100,7 +108,7 @@ public void testMessageExpiryAfterTopicUnload() throws Exception { CursorStats statsBeforeExpire = internalStatsBeforeExpire.cursors.get(subscriptionName); log.info("markDeletePosition before expire {}", statsBeforeExpire.markDeletePosition); assertEquals(statsBeforeExpire.markDeletePosition, - PositionImpl.get(firstMessageId.getLedgerId(), -1).toString()); + PositionFactory.create(firstMessageId.getLedgerId(), -1).toString()); Awaitility.await().timeout(30, TimeUnit.SECONDS) .pollDelay(3, TimeUnit.SECONDS).untilAsserted(() -> { @@ -110,7 +118,7 @@ public void testMessageExpiryAfterTopicUnload() throws Exception { PersistentTopicInternalStats internalStatsAfterExpire = admin.topics().getInternalStats(topicName); CursorStats statsAfterExpire = internalStatsAfterExpire.cursors.get(subscriptionName); log.info("markDeletePosition after expire {}", statsAfterExpire.markDeletePosition); - assertEquals(statsAfterExpire.markDeletePosition, PositionImpl.get(lastMessageId.getLedgerId(), + assertEquals(statsAfterExpire.markDeletePosition, PositionFactory.create(lastMessageId.getLedgerId(), lastMessageId.getEntryId() ).toString()); }); } @@ -138,4 +146,92 @@ public void testTTLPoliciesUpdate() throws Exception { topicRefMock.onUpdate(topicPolicies); verify(topicRefMock, times(2)).checkMessageExpiry(); } + + @Test + public void testTtlFilteredByIgnoreSubscriptions() throws Exception { + String topicName = "persistent://prop/ns-abc/testTTLFilteredByIgnoreSubscriptions"; + String subName = "__SUB_FILTER"; + cleanup(); + Set ignoredSubscriptions = new HashSet<>(); + ignoredSubscriptions.add(subName); + int defaultTtl = 5; + conf.setAdditionalSystemCursorNames(ignoredSubscriptions); + conf.setTtlDurationDefaultInSeconds(defaultTtl); + super.baseSetup(); + + pulsarClient.newConsumer(Schema.STRING).topic(topicName).subscriptionName(subName) + .subscribe().close(); + + @Cleanup + org.apache.pulsar.client.api.Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(false).topic(topicName).create(); + + final int messages = 10; + + for (int i = 0; i < messages; i++) { + String message = "my-message-" + i; + producer.send(message); + } + producer.close(); + + Optional topic = pulsar.getBrokerService().getTopicReference(topicName); + assertTrue(topic.isPresent()); + PersistentSubscription subscription = (PersistentSubscription) topic.get().getSubscription(subName); + + Thread.sleep((defaultTtl - 1) * 1000); + topic.get().checkMessageExpiry(); + // Wait the message expire task done and make sure the message does not expire early. + Thread.sleep(1000); + assertEquals(subscription.getNumberOfEntriesInBacklog(false), 10); + Thread.sleep(2000); + topic.get().checkMessageExpiry(); + // Wait the message expire task done. + retryStrategically((test) -> subscription.getNumberOfEntriesInBacklog(false) == 0, 5, 200); + // The message should not expire because the subscription is ignored. + assertEquals(subscription.getNumberOfEntriesInBacklog(false), 10); + + conf.setAdditionalSystemCursorNames(new TreeSet<>()); + } + + @Test + public void testTtlWithoutIgnoreSubscriptions() throws Exception { + String topicName = "persistent://prop/ns-abc/testTTLWithoutIgnoreSubscriptions"; + String subName = "__SUB_FILTER"; + cleanup(); + int defaultTtl = 5; + conf.setTtlDurationDefaultInSeconds(defaultTtl); + conf.setBrokerDeleteInactiveTopicsEnabled(false); + super.baseSetup(); + + pulsarClient.newConsumer(Schema.STRING).topic(topicName).subscriptionName(subName) + .subscribe().close(); + + @Cleanup + org.apache.pulsar.client.api.Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(false).topic(topicName).create(); + + final int messages = 10; + + for (int i = 0; i < messages; i++) { + String message = "my-message-" + i; + producer.send(message); + } + producer.close(); + + Optional topic = pulsar.getBrokerService().getTopicReference(topicName); + assertTrue(topic.isPresent()); + PersistentSubscription subscription = (PersistentSubscription) topic.get().getSubscription(subName); + + Thread.sleep((defaultTtl - 1) * 1000); + topic.get().checkMessageExpiry(); + // Wait the message expire task done and make sure the message does not expire early. + Thread.sleep(1000); + assertEquals(subscription.getNumberOfEntriesInBacklog(false), 10); + Thread.sleep(2000); + topic.get().checkMessageExpiry(); + // Wait the message expire task done and make sure the message expired. + retryStrategically((test) -> subscription.getNumberOfEntriesInBacklog(false) == 0, 5, 200); + assertEquals(subscription.getNumberOfEntriesInBacklog(false), 0); + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/NetworkErrorTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/NetworkErrorTestBase.java new file mode 100644 index 0000000000000..0161a4a63cfc6 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/NetworkErrorTestBase.java @@ -0,0 +1,307 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import com.google.common.collect.Sets; +import java.io.IOException; +import java.net.ServerSocket; +import java.net.URL; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl; +import org.apache.pulsar.broker.namespace.LookupOptions; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.ClientBuilder; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.common.naming.ServiceUnitId; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.policies.data.TopicType; +import org.apache.pulsar.tests.TestRetrySupport; +import org.apache.pulsar.utils.ResourceUtils; +import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.apache.pulsar.zookeeper.ZookeeperServerTest; +import org.awaitility.reflect.WhiteboxImpl; + +@Slf4j +public abstract class NetworkErrorTestBase extends TestRetrySupport { + + protected final static String CA_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem"); + protected final static String BROKER_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem"); + protected final static String BROKER_KEY_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem"); + protected final String defaultTenant = "public"; + protected final String defaultNamespace = defaultTenant + "/default"; + protected final String cluster1 = "r1"; + protected URL url1; + protected URL urlTls1; + protected URL url2; + protected URL urlTls2; + protected ServiceConfiguration config1 = new ServiceConfiguration(); + protected ServiceConfiguration config2 = new ServiceConfiguration(); + protected ZookeeperServerTest brokerConfigZk1; + protected Ipv4Proxy metadataZKProxy; + protected LocalBookkeeperEnsemble bkEnsemble1; + protected PulsarService pulsar1; + protected PulsarService pulsar2; + protected BrokerService broker1; + protected BrokerService broker2; + protected PulsarAdmin admin1; + protected PulsarAdmin admin2; + protected PulsarClient client1; + protected PulsarClient client2; + + private final static AtomicReference preferBroker = new AtomicReference<>(); + + protected void startZKAndBK() throws Exception { + // Start ZK & BK. + bkEnsemble1 = new LocalBookkeeperEnsemble(3, 0, () -> 0); + bkEnsemble1.start(); + + metadataZKProxy = new Ipv4Proxy(getOneFreePort(), "127.0.0.1", bkEnsemble1.getZookeeperPort()); + metadataZKProxy.startup(); + } + + protected void startBrokers() throws Exception { + // Start brokers. + setConfigDefaults(config1, cluster1, metadataZKProxy.getLocalPort()); + pulsar1 = new PulsarService(config1); + pulsar1.start(); + broker1 = pulsar1.getBrokerService(); + url1 = new URL(pulsar1.getWebServiceAddress()); + urlTls1 = new URL(pulsar1.getWebServiceAddressTls()); + + setConfigDefaults(config2, cluster1, bkEnsemble1.getZookeeperPort()); + pulsar2 = new PulsarService(config2); + pulsar2.start(); + broker2 = pulsar2.getBrokerService(); + url2 = new URL(pulsar2.getWebServiceAddress()); + urlTls2 = new URL(pulsar2.getWebServiceAddressTls()); + + log.info("broker-1: {}, broker-2: {}", broker1.getListenPort(), broker2.getListenPort()); + } + + public static int getOneFreePort() throws IOException { + ServerSocket serverSocket = new ServerSocket(0); + int port = serverSocket.getLocalPort(); + serverSocket.close(); + return port; + } + + protected void startAdminClient() throws Exception { + admin1 = PulsarAdmin.builder().serviceHttpUrl(url1.toString()).build(); + admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build(); + } + + protected void startPulsarClient() throws Exception{ + ClientBuilder clientBuilder1 = PulsarClient.builder().serviceUrl(url1.toString()); + client1 = initClient(clientBuilder1); + ClientBuilder clientBuilder2 = PulsarClient.builder().serviceUrl(url2.toString()); + client2 = initClient(clientBuilder2); + } + + protected void createDefaultTenantsAndClustersAndNamespace() throws Exception { + admin1.clusters().createCluster(cluster1, ClusterData.builder() + .serviceUrl(url1.toString()) + .serviceUrlTls(urlTls1.toString()) + .brokerServiceUrl(pulsar1.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar1.getBrokerServiceUrlTls()) + .brokerClientTlsEnabled(false) + .build()); + admin1.tenants().createTenant(defaultTenant, new TenantInfoImpl(Collections.emptySet(), + Sets.newHashSet(cluster1))); + admin1.namespaces().createNamespace(defaultNamespace, Sets.newHashSet(cluster1)); + } + + @Override + protected void setup() throws Exception { + incrementSetupNumber(); + + log.info("--- Starting OneWayReplicatorTestBase::setup ---"); + + startZKAndBK(); + + startBrokers(); + + startAdminClient(); + + createDefaultTenantsAndClustersAndNamespace(); + + startPulsarClient(); + + Thread.sleep(100); + log.info("--- OneWayReplicatorTestBase::setup completed ---"); + } + + protected void setConfigDefaults(ServiceConfiguration config, String clusterName, int zkPort) { + config.setClusterName(clusterName); + config.setAdvertisedAddress("localhost"); + config.setWebServicePort(Optional.of(0)); + config.setWebServicePortTls(Optional.of(0)); + config.setMetadataStoreUrl("zk:127.0.0.1:" + zkPort); + config.setConfigurationMetadataStoreUrl("zk:127.0.0.1:" + zkPort + "/config_meta"); + config.setBrokerDeleteInactiveTopicsEnabled(false); + config.setBrokerDeleteInactiveTopicsFrequencySeconds(60); + config.setBrokerShutdownTimeoutMs(0L); + config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); + config.setBrokerServicePort(Optional.of(0)); + config.setBrokerServicePortTls(Optional.of(0)); + config.setBacklogQuotaCheckIntervalInSeconds(5); + config.setDefaultNumberOfNamespaceBundles(1); + config.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED); + config.setEnableReplicatedSubscriptions(true); + config.setReplicatedSubscriptionsSnapshotFrequencyMillis(1000); + config.setLoadBalancerSheddingEnabled(false); + config.setForceDeleteNamespaceAllowed(true); + config.setLoadManagerClassName(PreferBrokerModularLoadManager.class.getName()); + config.setMetadataStoreSessionTimeoutMillis(5000); + config.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + config.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + config.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); + } + + @Override + protected void cleanup() throws Exception { + // shutdown. + markCurrentSetupNumberCleaned(); + log.info("--- Shutting down ---"); + + // Stop brokers. + if (client1 != null) { + client1.close(); + client1 = null; + } + if (admin1 != null) { + admin1.close(); + admin1 = null; + } + if (client2 != null) { + client2.close(); + client2 = null; + } + if (admin2 != null) { + admin2.close(); + admin2 = null; + } + if (pulsar1 != null) { + pulsar1.close(); + pulsar1 = null; + } + if (pulsar2 != null) { + pulsar2.close(); + pulsar2 = null; + } + + // Stop ZK and BK. + if (bkEnsemble1 != null) { + bkEnsemble1.stop(); + bkEnsemble1 = null; + } + if (metadataZKProxy != null) { + metadataZKProxy.stop(); + } + if (brokerConfigZk1 != null) { + brokerConfigZk1.stop(); + brokerConfigZk1 = null; + } + + // Reset configs. + config1 = new ServiceConfiguration(); + preferBroker.set(null); + } + + protected PulsarClient initClient(ClientBuilder clientBuilder) throws Exception { + return clientBuilder.build(); + } + + protected static class PreferBrokerModularLoadManager extends ModularLoadManagerImpl { + + @Override + public String setNamespaceBundleAffinity(String bundle, String broker) { + if (StringUtils.isNotBlank(broker)) { + return broker; + } + Set availableBrokers = NetworkErrorTestBase.getAvailableBrokers(super.pulsar); + String prefer = preferBroker.get(); + if (availableBrokers.contains(prefer)) { + return prefer; + } else { + return null; + } + } + } + + protected static class PreferExtensibleLoadManager extends ExtensibleLoadManagerImpl { + + @Override + public CompletableFuture> selectAsync(ServiceUnitId bundle, + Set excludeBrokerSet, + LookupOptions options) { + Set availableBrokers = NetworkErrorTestBase.getAvailableBrokers(super.pulsar); + String prefer = preferBroker.get(); + if (availableBrokers.contains(prefer)) { + return CompletableFuture.completedFuture(Optional.of(prefer)); + } else { + return super.selectAsync(bundle, excludeBrokerSet, options); + } + } + } + + public void setPreferBroker(PulsarService target) { + for (PulsarService pulsar : Arrays.asList(pulsar1, pulsar2)) { + for (String broker : getAvailableBrokers(pulsar)) { + if (broker.endsWith(target.getBrokerListenPort().orElse(-1) + "") + || broker.endsWith(target.getListenPortHTTPS().orElse(-1) + "") + || broker.endsWith(target.getListenPortHTTP().orElse(-1) + "") + || broker.endsWith(target.getBrokerListenPortTls().orElse(-1) + "")) { + preferBroker.set(broker); + } + } + } + } + + public static Set getAvailableBrokers(PulsarService pulsar) { + Object loadManagerWrapper = pulsar.getLoadManager().get(); + Object loadManager = WhiteboxImpl.getInternalState(loadManagerWrapper, "loadManager"); + if (loadManager instanceof ModularLoadManagerImpl) { + return ((ModularLoadManagerImpl) loadManager).getAvailableBrokers(); + } else if (loadManager instanceof ExtensibleLoadManagerImpl) { + return new HashSet<>(((ExtensibleLoadManagerImpl) loadManager).getBrokerRegistry() + .getAvailableBrokersAsync().join()); + } else { + throw new RuntimeException("Not support for the load manager: " + loadManager.getClass().getName()); + } + } + + public void clearPreferBroker() { + preferBroker.set(null); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java index e8a21502fb1af..a8f8d7ecbbd47 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java @@ -18,18 +18,92 @@ */ package org.apache.pulsar.broker.service; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.benmanes.caffeine.cache.AsyncLoadingCache; +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; +import io.netty.util.concurrent.FastThreadLocalThread; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.resources.ClusterResources; +import org.apache.pulsar.broker.service.persistent.GeoPersistentReplicator; +import org.apache.pulsar.broker.service.persistent.PersistentReplicator; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient; +import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerBuilder; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.ProducerBuilderImpl; +import org.apache.pulsar.client.impl.ProducerImpl; +import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.partition.PartitionedTopicMetadata; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; +import org.apache.pulsar.common.policies.data.RetentionPolicies; +import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.policies.data.TopicStats; -import org.junit.Assert; +import org.apache.pulsar.common.policies.data.impl.AutoTopicCreationOverrideImpl; +import org.apache.pulsar.common.util.FutureUtil; +import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; +import org.glassfish.jersey.client.JerseyClient; +import org.glassfish.jersey.client.JerseyClientBuilder; +import org.mockito.Mockito; +import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +@Slf4j @Test(groups = "broker") public class OneWayReplicatorTest extends OneWayReplicatorTestBase { @@ -45,34 +119,1209 @@ public void cleanup() throws Exception { super.cleanup(); } - @Test + private void waitReplicatorStopped(String topicName) { + Awaitility.await().untilAsserted(() -> { + Optional topicOptional2 = pulsar2.getBrokerService().getTopic(topicName, false).get(); + assertTrue(topicOptional2.isPresent()); + PersistentTopic persistentTopic2 = (PersistentTopic) topicOptional2.get(); + assertTrue(persistentTopic2.getProducers().isEmpty()); + Optional topicOptional1 = pulsar2.getBrokerService().getTopic(topicName, false).get(); + assertTrue(topicOptional1.isPresent()); + PersistentTopic persistentTopic1 = (PersistentTopic) topicOptional2.get(); + assertTrue(persistentTopic1.getReplicators().isEmpty() + || !persistentTopic1.getReplicators().get(cluster2).isConnected()); + }); + } + + /** + * Override "AbstractReplicator.producer" by {@param producer} and return the original value. + */ + private ProducerImpl overrideProducerForReplicator(AbstractReplicator replicator, ProducerImpl newProducer) + throws Exception { + Field producerField = AbstractReplicator.class.getDeclaredField("producer"); + producerField.setAccessible(true); + ProducerImpl originalValue = (ProducerImpl) producerField.get(replicator); + synchronized (replicator) { + producerField.set(replicator, newProducer); + } + return originalValue; + } + + @Test(timeOut = 45 * 1000) public void testReplicatorProducerStatInTopic() throws Exception { - final String topicName = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp_"); + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); final String subscribeName = "subscribe_1"; final byte[] msgValue = "test".getBytes(); - admin1.topics().createNonPartitionedTopic(topicName); - admin2.topics().createNonPartitionedTopic(topicName); - admin1.topics().createSubscription(topicName, subscribeName, MessageId.earliest); - admin2.topics().createSubscription(topicName, subscribeName, MessageId.earliest); - // Verify replicator works. Producer producer1 = client1.newProducer().topic(topicName).create(); + Producer producer2 = client2.newProducer().topic(topicName).create(); // Do not publish messages Consumer consumer2 = client2.newConsumer().topic(topicName).subscriptionName(subscribeName).subscribe(); producer1.newMessage().value(msgValue).send(); pulsar1.getBrokerService().checkReplicationPolicies(); assertEquals(consumer2.receive(10, TimeUnit.SECONDS).getValue(), msgValue); - // Verify there has one item in the attribute "publishers" or "replications" + // Verify that the "publishers" field does not include the producer for replication TopicStats topicStats2 = admin2.topics().getStats(topicName); - Assert.assertTrue(topicStats2.getPublishers().size() + topicStats2.getReplication().size() > 0); + assertEquals(topicStats2.getPublishers().size(), 1); + assertFalse(topicStats2.getPublishers().get(0).getProducerName().startsWith(config1.getReplicatorPrefix())); + + // Update broker stats immediately (usually updated every minute) + pulsar2.getBrokerService().updateRates(); + String brokerStats2 = admin2.brokerStats().getTopics(); + + boolean found = false; + ObjectMapper mapper = new ObjectMapper(); + JsonNode rootNode = mapper.readTree(brokerStats2); + if (rootNode.hasNonNull(replicatedNamespace)) { + Iterator bundleNodes = rootNode.get(replicatedNamespace).elements(); + while (bundleNodes.hasNext()) { + JsonNode bundleNode = bundleNodes.next(); + if (bundleNode.hasNonNull("persistent") && bundleNode.get("persistent").hasNonNull(topicName)) { + found = true; + JsonNode topicNode = bundleNode.get("persistent").get(topicName); + // Verify that the "publishers" field does not include the producer for replication + assertEquals(topicNode.get("publishers").size(), 1); + assertEquals(topicNode.get("producerCount").intValue(), 1); + Iterator publisherNodes = topicNode.get("publishers").elements(); + while (publisherNodes.hasNext()) { + JsonNode publisherNode = publisherNodes.next(); + assertFalse(publisherNode.get("producerName").textValue() + .startsWith(config1.getReplicatorPrefix())); + } + break; + } + } + } + assertTrue(found); + + // cleanup. + consumer2.unsubscribe(); + producer2.close(); + producer1.close(); + cleanupTopics(() -> { + admin1.topics().delete(topicName); + admin2.topics().delete(topicName); + }); + } + @Test(timeOut = 45 * 1000) + public void testCreateRemoteConsumerFirst() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); + Producer producer1 = client1.newProducer(Schema.STRING).topic(topicName).create(); + + // The topic in cluster2 has a replicator created producer(schema Auto_Produce), but does not have any schema。 + // Verify: the consumer of this cluster2 can create successfully. + Consumer consumer2 = client2.newConsumer(Schema.STRING).topic(topicName).subscriptionName("s1") + .subscribe();; + // Wait for replicator started. + waitReplicatorStarted(topicName); // cleanup. + producer1.close(); consumer2.close(); + cleanupTopics(() -> { + admin1.topics().delete(topicName); + admin2.topics().delete(topicName); + }); + } + + @Test(timeOut = 45 * 1000) + public void testTopicCloseWhenInternalProducerCloseErrorOnce() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); + admin1.topics().createNonPartitionedTopic(topicName); + // Wait for replicator started. + waitReplicatorStarted(topicName); + PersistentTopic topic1 = + (PersistentTopic) pulsar1.getBrokerService().getTopic(topicName, false).join().get(); + PersistentReplicator replicator1 = + (PersistentReplicator) topic1.getReplicators().values().iterator().next(); + // Mock an error when calling "replicator.disconnect()" + AtomicBoolean closeFailed = new AtomicBoolean(true); + final ProducerImpl mockProducer = Mockito.mock(ProducerImpl.class); + final AtomicReference originalProducer1 = new AtomicReference(); + doAnswer(invocation -> { + if (closeFailed.get()) { + return CompletableFuture.failedFuture(new Exception("mocked ex")); + } else { + return originalProducer1.get().closeAsync(); + } + }).when(mockProducer).closeAsync(); + originalProducer1.set(overrideProducerForReplicator(replicator1, mockProducer)); + // Verify: since the "replicator.producer.closeAsync()" will retry after it failed, the topic unload should be + // successful. + admin1.topics().unload(topicName); + // Verify: After "replicator.producer.closeAsync()" retry again, the "replicator.producer" will be closed + // successful. + closeFailed.set(false); + AtomicReference topic2 = new AtomicReference(); + AtomicReference replicator2 = new AtomicReference(); + Awaitility.await().untilAsserted(() -> { + topic2.set((PersistentTopic) pulsar1.getBrokerService().getTopic(topicName, false).join().get()); + replicator2.set((PersistentReplicator) topic2.get().getReplicators().values().iterator().next()); + // It is a new Topic after reloading. + assertNotEquals(topic2.get(), topic1); + assertNotEquals(replicator2.get(), replicator1); + }); + Awaitility.await().untilAsserted(() -> { + // Old replicator should be closed. + Assert.assertFalse(replicator1.isConnected()); + Assert.assertFalse(originalProducer1.get().isConnected()); + // New replicator should be connected. + Assert.assertTrue(replicator2.get().isConnected()); + }); + // cleanup. + cleanupTopics(() -> { + admin1.topics().delete(topicName); + admin2.topics().delete(topicName); + }); + } + + private Runnable injectMockReplicatorProducerBuilder( + BiFunction producerDecorator) + throws Exception { + String cluster2 = pulsar2.getConfig().getClusterName(); + BrokerService brokerService = pulsar1.getBrokerService(); + // Wait for the internal client created. + final String topicNameTriggerInternalClientCreate = + BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); + admin1.topics().createNonPartitionedTopic(topicNameTriggerInternalClientCreate); + waitReplicatorStarted(topicNameTriggerInternalClientCreate); + cleanupTopics(() -> { + admin1.topics().delete(topicNameTriggerInternalClientCreate); + admin2.topics().delete(topicNameTriggerInternalClientCreate); + }); + + // Inject spy client. + final var replicationClients = brokerService.getReplicationClients(); + PulsarClientImpl internalClient = (PulsarClientImpl) replicationClients.get(cluster2); + PulsarClient spyClient = spy(internalClient); + assertTrue(replicationClients.remove(cluster2, internalClient)); + assertNull(replicationClients.putIfAbsent(cluster2, spyClient)); + + // Inject producer decorator. + doAnswer(invocation -> { + Schema schema = (Schema) invocation.getArguments()[0]; + ProducerBuilderImpl producerBuilder = (ProducerBuilderImpl) internalClient.newProducer(schema); + ProducerBuilder spyProducerBuilder = spy(producerBuilder); + doAnswer(ignore -> { + CompletableFuture producerFuture = new CompletableFuture<>(); + producerBuilder.createAsync().whenComplete((p, t) -> { + if (t != null) { + producerFuture.completeExceptionally(t); + return; + } + ProducerImpl pImpl = (ProducerImpl) p; + new FastThreadLocalThread(() -> { + try { + ProducerImpl newProducer = producerDecorator.apply(producerBuilder.getConf(), pImpl); + producerFuture.complete(newProducer); + } catch (Exception ex) { + producerFuture.completeExceptionally(ex); + } + }).start(); + }); + + return producerFuture; + }).when(spyProducerBuilder).createAsync(); + return spyProducerBuilder; + }).when(spyClient).newProducer(any(Schema.class)); + + // Return a cleanup injection task; + return () -> { + assertTrue(replicationClients.remove(cluster2, spyClient)); + assertNull(replicationClients.putIfAbsent(cluster2, internalClient)); + }; + } + + private SpyCursor spyCursor(PersistentTopic persistentTopic, String cursorName) throws Exception { + ManagedLedgerImpl ml = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + ManagedCursorImpl cursor = (ManagedCursorImpl) ml.getCursors().get(cursorName); + ManagedCursorImpl spyCursor = spy(cursor); + // remove cursor. + ml.getCursors().removeCursor(cursorName); + ml.deactivateCursor(cursor); + // Add the spy one. addCursor(ManagedCursorImpl cursor) + Method m = ManagedLedgerImpl.class.getDeclaredMethod("addCursor", new Class[]{ManagedCursorImpl.class}); + m.setAccessible(true); + m.invoke(ml, new Object[]{spyCursor}); + return new SpyCursor(cursor, spyCursor); + } + + @Data + @AllArgsConstructor + static class SpyCursor { + ManagedCursorImpl original; + ManagedCursorImpl spy; + } + + private CursorCloseSignal makeCursorClosingDelay(SpyCursor spyCursor) throws Exception { + CountDownLatch startCloseSignal = new CountDownLatch(1); + CountDownLatch startCallbackSignal = new CountDownLatch(1); + doAnswer(invocation -> { + AsyncCallbacks.CloseCallback originalCallback = (AsyncCallbacks.CloseCallback) invocation.getArguments()[0]; + Object ctx = invocation.getArguments()[1]; + AsyncCallbacks.CloseCallback newCallback = new AsyncCallbacks.CloseCallback() { + @Override + public void closeComplete(Object ctx) { + new FastThreadLocalThread(new Runnable() { + @Override + @SneakyThrows + public void run() { + startCallbackSignal.await(); + originalCallback.closeComplete(ctx); + } + }).start(); + } + + @Override + public void closeFailed(ManagedLedgerException exception, Object ctx) { + new FastThreadLocalThread(new Runnable() { + @Override + @SneakyThrows + public void run() { + startCallbackSignal.await(); + originalCallback.closeFailed(exception, ctx); + } + }).start(); + } + }; + startCloseSignal.await(); + spyCursor.original.asyncClose(newCallback, ctx); + return null; + }).when(spyCursor.spy).asyncClose(any(AsyncCallbacks.CloseCallback.class), any()); + return new CursorCloseSignal(startCloseSignal, startCallbackSignal); + } + + @AllArgsConstructor + static class CursorCloseSignal { + CountDownLatch startCloseSignal; + CountDownLatch startCallbackSignal; + + void startClose() { + startCloseSignal.countDown(); + } + + void startCallback() { + startCallbackSignal.countDown(); + } + } + + /** + * See the description and execution flow: https://github.com/apache/pulsar/pull/21946. + * Steps: + * - Create topic, but the internal producer of Replicator created failed. + * - Unload bundle, the Replicator will be closed, but the internal producer creation retry has not executed yet. + * - The internal producer creation retry execute successfully, the "repl.cursor" has not been closed yet. + * - The topic is wholly closed. + * - Verify: the delayed created internal producer will be closed. + */ + @Test(timeOut = 120 * 1000) + public void testConcurrencyOfUnloadBundleAndRecreateProducer() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); + // Inject an error for "replicator.producer" creation. + // The delay time of next retry to create producer is below: + // 0.1s, 0.2, 0.4, 0.8, 1.6s, 3.2s, 6.4s... + // If the retry counter is larger than 6, the next creation will be slow enough to close Replicator. + final AtomicInteger createProducerCounter = new AtomicInteger(); + final int failTimes = 6; + Runnable taskToClearInjection = injectMockReplicatorProducerBuilder((producerCnf, originalProducer) -> { + if (topicName.equals(producerCnf.getTopicName())) { + // There is a switch to determine create producer successfully or not. + if (createProducerCounter.incrementAndGet() > failTimes) { + return originalProducer; + } + log.info("Retry create replicator.producer count: {}", createProducerCounter); + // Release producer and fail callback. + originalProducer.closeAsync(); + throw new RuntimeException("mock error"); + } + return originalProducer; + }); + + // Create topic. + admin1.topics().createNonPartitionedTopic(topicName); + PersistentTopic persistentTopic = + (PersistentTopic) pulsar1.getBrokerService().getTopic(topicName, false).join().get(); + PersistentReplicator replicator = + (PersistentReplicator) persistentTopic.getReplicators().values().iterator().next(); + // Since we inject a producer creation error, the replicator can not start successfully. + assertFalse(replicator.isConnected()); + + // Stuck the closing of the cursor("pulsar.repl"), until the internal producer of the replicator started. + SpyCursor spyCursor = + spyCursor(persistentTopic, "pulsar.repl." + pulsar2.getConfig().getClusterName()); + CursorCloseSignal cursorCloseSignal = makeCursorClosingDelay(spyCursor); + + // Unload bundle: call "topic.close(false)". + // Stuck start new producer, until the state of replicator change to Stopped. + // The next once of "createProducerSuccessAfterFailTimes" to create producer will be successfully. + Awaitility.await().pollInterval(Duration.ofMillis(100)).atMost(Duration.ofSeconds(60)).untilAsserted(() -> { + assertTrue(createProducerCounter.get() >= failTimes, + "count of retry to create producer is " + createProducerCounter.get()); + }); + CompletableFuture topicCloseFuture = persistentTopic.close(true); + Awaitility.await().atMost(Duration.ofSeconds(30)).untilAsserted(() -> { + String state = String.valueOf(replicator.getState()); + assertTrue(state.equals("Stopped") || state.equals("Terminated")); + }); + + // Delay close cursor, until "replicator.producer" create successfully. + // The next once retry time of create "replicator.producer" will be 3.2s. + Thread.sleep(4 * 1000); + log.info("Replicator.state: {}", replicator.getState()); + cursorCloseSignal.startClose(); + cursorCloseSignal.startCallback(); + + // Wait for topic close successfully. + // Verify there is no orphan producer on the remote cluster. + topicCloseFuture.join(); + Awaitility.await().pollInterval(Duration.ofSeconds(1)).untilAsserted(() -> { + PersistentTopic persistentTopic2 = + (PersistentTopic) pulsar2.getBrokerService().getTopic(topicName, false).join().get(); + assertEquals(persistentTopic2.getProducers().size(), 0); + Assert.assertFalse(replicator.isConnected()); + }); + + // cleanup. + taskToClearInjection.run(); + cleanupTopics(() -> { + admin1.topics().delete(topicName); + admin2.topics().delete(topicName); + }); + } + + @Test + public void testPartitionedTopicLevelReplication() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + nonReplicatedNamespace + "/tp_"); + final String partition0 = TopicName.get(topicName).getPartition(0).toString(); + final String partition1 = TopicName.get(topicName).getPartition(1).toString(); + admin1.topics().createPartitionedTopic(topicName, 2); + admin1.topics().setReplicationClusters(topicName, Arrays.asList(cluster1, cluster2)); + // Check the partitioned topic has been created at the remote cluster. + PartitionedTopicMetadata topicMetadata2 = admin2.topics().getPartitionedTopicMetadata(topicName); + assertEquals(topicMetadata2.partitions, 2); + // cleanup. + admin1.topics().setReplicationClusters(topicName, Arrays.asList(cluster1)); + waitReplicatorStopped(partition0); + waitReplicatorStopped(partition1); + admin1.topics().deletePartitionedTopic(topicName); + admin2.topics().deletePartitionedTopic(topicName); + } + + // https://github.com/apache/pulsar/issues/22967 + @Test + public void testPartitionedTopicWithTopicPolicyAndNoReplicationClusters() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); + admin1.topics().createPartitionedTopic(topicName, 2); + try { + admin1.topicPolicies().setMessageTTL(topicName, 5); + Awaitility.await().ignoreExceptions().untilAsserted(() -> { + assertEquals(admin2.topics().getPartitionedTopicMetadata(topicName).partitions, 2); + }); + admin1.topics().updatePartitionedTopic(topicName, 3, false); + Awaitility.await().ignoreExceptions().untilAsserted(() -> { + assertEquals(admin2.topics().getPartitionedTopicMetadata(topicName).partitions, 3); + }); + } finally { + // cleanup. + admin1.topics().deletePartitionedTopic(topicName, true); + if (!usingGlobalZK) { + admin2.topics().deletePartitionedTopic(topicName, true); + } + } + } + + @Test + public void testPartitionedTopicLevelReplicationRemoteTopicExist() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + nonReplicatedNamespace + "/tp_"); + final String partition0 = TopicName.get(topicName).getPartition(0).toString(); + final String partition1 = TopicName.get(topicName).getPartition(1).toString(); + admin1.topics().createPartitionedTopic(topicName, 2); + admin2.topics().createPartitionedTopic(topicName, 2); + admin1.topics().setReplicationClusters(topicName, Arrays.asList(cluster1, cluster2)); + // Check the partitioned topic has been created at the remote cluster. + Awaitility.await().untilAsserted(() -> { + PartitionedTopicMetadata topicMetadata2 = admin2.topics().getPartitionedTopicMetadata(topicName); + assertEquals(topicMetadata2.partitions, 2); + }); + + // Expand partitions + admin2.topics().updatePartitionedTopic(topicName, 3); + Awaitility.await().untilAsserted(() -> { + PartitionedTopicMetadata topicMetadata2 = admin2.topics().getPartitionedTopicMetadata(topicName); + assertEquals(topicMetadata2.partitions, 3); + }); + // cleanup. + admin1.topics().setReplicationClusters(topicName, Arrays.asList(cluster1)); + waitReplicatorStopped(partition0); + waitReplicatorStopped(partition1); + admin1.topics().deletePartitionedTopic(topicName); + admin2.topics().deletePartitionedTopic(topicName); + } + + @Test + public void testPartitionedTopicLevelReplicationRemoteConflictTopicExist() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + nonReplicatedNamespace + "/tp_"); + admin2.topics().createPartitionedTopic(topicName, 3); + admin1.topics().createPartitionedTopic(topicName, 2); + try { + admin1.topics().setReplicationClusters(topicName, Arrays.asList(cluster1, cluster2)); + fail("Expected error due to a conflict partitioned topic already exists."); + } catch (Exception ex) { + Throwable unWrapEx = FutureUtil.unwrapCompletionException(ex); + assertTrue(unWrapEx.getMessage().contains("with different partitions")); + } + // Check nothing changed. + PartitionedTopicMetadata topicMetadata2 = admin2.topics().getPartitionedTopicMetadata(topicName); + assertEquals(topicMetadata2.partitions, 3); + assertEquals(admin1.topics().getReplicationClusters(topicName, true).size(), 1); + // cleanup. + admin1.topics().deletePartitionedTopic(topicName); + admin2.topics().deletePartitionedTopic(topicName); + } + + /** + * See the description and execution flow: https://github.com/apache/pulsar/pull/21948. + * Steps: + * 1.Create topic, does not enable replication now. + * - The topic will be loaded in the memory. + * 2.Enable namespace level replication. + * - Broker creates a replicator, and the internal producer of replicator is starting. + * - We inject an error to make the internal producer fail to connect，after few seconds, it will retry to start. + * 3.Unload bundle. + * - Starting to close the topic. + * - The replicator will be closed, but it will not close the internal producer, because the producer has not + * been created successfully. + * - We inject a sleeping into the progress of closing the "repl.cursor" to make it stuck. So the topic is still + * in the process of being closed now. + * 4.Internal producer retry to connect. + * - At the next retry, it connected successful. Since the state of "repl.cursor" is not "Closed", this producer + * will not be closed now. + * 5.Topic closed. + * - Cancel the stuck of closing the "repl.cursor". + * - The topic is wholly closed. + * 6.Verify: the delayed created internal producer will be closed. In other words, there is no producer is connected + * to the remote cluster. + */ + @Test + public void testConcurrencyOfUnloadBundleAndRecreateProducer2() throws Exception { + final String namespaceName = defaultTenant + "/" + UUID.randomUUID().toString().replaceAll("-", ""); + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + namespaceName + "/tp_"); + // 1.Create topic, does not enable replication now. + admin1.namespaces().createNamespace(namespaceName); + admin2.namespaces().createNamespace(namespaceName); + admin1.topics().createNonPartitionedTopic(topicName); + PersistentTopic persistentTopic = + (PersistentTopic) pulsar1.getBrokerService().getTopic(topicName, false).join().get(); + + // We inject an error to make the internal producer fail to connect. + // The delay time of next retry to create producer is below: + // 0.1s, 0.2, 0.4, 0.8, 1.6s, 3.2s, 6.4s... + // If the retry counter is larger than 6, the next creation will be slow enough to close Replicator. + final AtomicInteger createProducerCounter = new AtomicInteger(); + final int failTimes = 6; + Runnable taskToClearInjection = injectMockReplicatorProducerBuilder((producerCnf, originalProducer) -> { + if (topicName.equals(producerCnf.getTopicName())) { + // There is a switch to determine create producer successfully or not. + if (createProducerCounter.incrementAndGet() > failTimes) { + return originalProducer; + } + log.info("Retry create replicator.producer count: {}", createProducerCounter); + // Release producer and fail callback. + originalProducer.closeAsync(); + throw new RuntimeException("mock error"); + } + return originalProducer; + }); + + // 2.Enable namespace level replication. + admin1.namespaces().setNamespaceReplicationClusters(namespaceName, Sets.newHashSet(cluster1, cluster2)); + AtomicReference replicator = new AtomicReference(); + Awaitility.await().untilAsserted(() -> { + assertFalse(persistentTopic.getReplicators().isEmpty()); + replicator.set( + (PersistentReplicator) persistentTopic.getReplicators().values().iterator().next()); + // Since we inject a producer creation error, the replicator can not start successfully. + assertFalse(replicator.get().isConnected()); + }); + + // We inject a sleeping into the progress of closing the "repl.cursor" to make it stuck, until the internal + // producer of the replicator started. + SpyCursor spyCursor = + spyCursor(persistentTopic, "pulsar.repl." + pulsar2.getConfig().getClusterName()); + CursorCloseSignal cursorCloseSignal = makeCursorClosingDelay(spyCursor); + + // 3.Unload bundle: call "topic.close(false)". + // Stuck start new producer, until the state of replicator change to Stopped. + // The next once of "createProducerSuccessAfterFailTimes" to create producer will be successfully. + Awaitility.await().pollInterval(Duration.ofMillis(100)).atMost(Duration.ofSeconds(60)).untilAsserted(() -> { + assertTrue(createProducerCounter.get() >= failTimes); + }); + CompletableFuture topicCloseFuture = persistentTopic.close(true); + Awaitility.await().atMost(Duration.ofSeconds(30)).untilAsserted(() -> { + String state = String.valueOf(replicator.get().getState()); + log.error("replicator state: {}", state); + assertTrue(state.equals("Disconnected") || state.equals("Terminated")); + }); + + // 5.Delay close cursor, until "replicator.producer" create successfully. + // The next once retry time of create "replicator.producer" will be 3.2s. + Thread.sleep(4 * 1000); + log.info("Replicator.state: {}", replicator.get().getState()); + cursorCloseSignal.startClose(); + cursorCloseSignal.startCallback(); + // Wait for topic close successfully. + topicCloseFuture.join(); + + // 6. Verify there is no orphan producer on the remote cluster. + Awaitility.await().pollInterval(Duration.ofSeconds(1)).untilAsserted(() -> { + PersistentTopic persistentTopic2 = + (PersistentTopic) pulsar2.getBrokerService().getTopic(topicName, false).join().get(); + assertEquals(persistentTopic2.getProducers().size(), 0); + Assert.assertFalse(replicator.get().isConnected()); + }); + + // cleanup. + taskToClearInjection.run(); + cleanupTopics(namespaceName, () -> { + admin1.topics().delete(topicName); + admin2.topics().delete(topicName); + }); + admin1.namespaces().setNamespaceReplicationClusters(namespaceName, Sets.newHashSet(cluster1)); + admin1.namespaces().deleteNamespace(namespaceName); + admin2.namespaces().deleteNamespace(namespaceName); + } + + @Test + public void testUnFenceTopicToReuse() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp"); + // Wait for replicator started. + Producer producer1 = client1.newProducer(Schema.STRING).topic(topicName).create(); + waitReplicatorStarted(topicName); + + // Inject an error to make topic close fails. + final String mockProducerName = UUID.randomUUID().toString(); + final org.apache.pulsar.broker.service.Producer mockProducer = + mock(org.apache.pulsar.broker.service.Producer.class); + doAnswer(invocation -> CompletableFuture.failedFuture(new RuntimeException("mocked error"))) + .when(mockProducer).disconnect(any()); + doAnswer(invocation -> CompletableFuture.failedFuture(new RuntimeException("mocked error"))) + .when(mockProducer).disconnect(); + PersistentTopic persistentTopic = + (PersistentTopic) pulsar1.getBrokerService().getTopic(topicName, false).join().get(); + persistentTopic.getProducers().put(mockProducerName, mockProducer); + + // Do close. + GeoPersistentReplicator replicator1 = + (GeoPersistentReplicator) persistentTopic.getReplicators().values().iterator().next(); + try { + persistentTopic.close(true, false).join(); + fail("Expected close fails due to a producer close fails"); + } catch (Exception ex) { + log.info("Expected error: {}", ex.getMessage()); + } + + // Broker will call `topic.unfenceTopicToResume` if close clients fails. + // Verify: the replicator will be re-created. + Awaitility.await().untilAsserted(() -> { + assertTrue(producer1.isConnected()); + GeoPersistentReplicator replicator2 = + (GeoPersistentReplicator) persistentTopic.getReplicators().values().iterator().next(); + assertNotEquals(replicator1, replicator2); + assertFalse(replicator1.isConnected()); + assertFalse(replicator1.producer != null && replicator1.producer.isConnected()); + assertTrue(replicator2.isConnected()); + assertTrue(replicator2.producer != null && replicator2.producer.isConnected()); + }); + + // cleanup the injection. + persistentTopic.getProducers().remove(mockProducerName, mockProducer); + // cleanup. producer1.close(); cleanupTopics(() -> { admin1.topics().delete(topicName); admin2.topics().delete(topicName); }); } + + @Test + public void testDeleteNonPartitionedTopic() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); + admin1.topics().createNonPartitionedTopic(topicName); + + // Verify replicator works. + verifyReplicationWorks(topicName); + + // Disable replication. + setTopicLevelClusters(topicName, Arrays.asList(cluster1), admin1, pulsar1); + setTopicLevelClusters(topicName, Arrays.asList(cluster2), admin2, pulsar2); + + // Delete topic. + admin1.topics().delete(topicName); + admin2.topics().delete(topicName); + + // Verify the topic was deleted. + assertFalse(pulsar1.getPulsarResources().getTopicResources() + .persistentTopicExists(TopicName.get(topicName)).join()); + assertFalse(pulsar2.getPulsarResources().getTopicResources() + .persistentTopicExists(TopicName.get(topicName)).join()); + } + + @Test + public void testDeletePartitionedTopic() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); + admin1.topics().createPartitionedTopic(topicName, 2); + + // Verify replicator works. + verifyReplicationWorks(topicName); + + // Disable replication. + setTopicLevelClusters(topicName, Arrays.asList(cluster1), admin1, pulsar1); + setTopicLevelClusters(topicName, Arrays.asList(cluster2), admin2, pulsar2); + + // Delete topic. + admin1.topics().deletePartitionedTopic(topicName); + if (!usingGlobalZK) { + admin2.topics().deletePartitionedTopic(topicName); + } + + // Verify the topic was deleted. + assertFalse(pulsar1.getPulsarResources().getNamespaceResources().getPartitionedTopicResources() + .partitionedTopicExists(TopicName.get(topicName))); + assertFalse(pulsar2.getPulsarResources().getNamespaceResources().getPartitionedTopicResources() + .partitionedTopicExists(TopicName.get(topicName))); + if (!usingGlobalZK) { + // So far, the topic partitions on the remote cluster are needed to delete manually when using global ZK. + assertFalse(pulsar1.getPulsarResources().getTopicResources() + .persistentTopicExists(TopicName.get(topicName).getPartition(0)).join()); + assertFalse(pulsar2.getPulsarResources().getTopicResources() + .persistentTopicExists(TopicName.get(topicName).getPartition(0)).join()); + assertFalse(pulsar1.getPulsarResources().getTopicResources() + .persistentTopicExists(TopicName.get(topicName).getPartition(1)).join()); + assertFalse(pulsar2.getPulsarResources().getTopicResources() + .persistentTopicExists(TopicName.get(topicName).getPartition(1)).join()); + } + } + + @Test + public void testNoExpandTopicPartitionsWhenDisableTopicLevelReplication() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); + admin1.topics().createPartitionedTopic(topicName, 2); + + // Verify replicator works. + verifyReplicationWorks(topicName); + + // Disable topic level replication. + setTopicLevelClusters(topicName, Arrays.asList(cluster1), admin1, pulsar1); + setTopicLevelClusters(topicName, Arrays.asList(cluster2), admin2, pulsar2); + + // Expand topic. + admin1.topics().updatePartitionedTopic(topicName, 3); + assertEquals(admin1.topics().getPartitionedTopicMetadata(topicName).partitions, 3); + + // Wait for async tasks that were triggered by expanding topic partitions. + Thread.sleep(3 * 1000); + + + // Verify: the topics on the remote cluster did not been expanded. + assertEquals(admin2.topics().getPartitionedTopicMetadata(topicName).partitions, 2); + + cleanupTopics(() -> { + admin1.topics().deletePartitionedTopic(topicName, false); + admin2.topics().deletePartitionedTopic(topicName, false); + }); + } + + @Test + public void testExpandTopicPartitionsOnNamespaceLevelReplication() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); + admin1.topics().createPartitionedTopic(topicName, 2); + + // Verify replicator works. + verifyReplicationWorks(topicName); + + // Expand topic. + admin1.topics().updatePartitionedTopic(topicName, 3); + assertEquals(admin1.topics().getPartitionedTopicMetadata(topicName).partitions, 3); + + // Verify: the topics on the remote cluster will be expanded. + Awaitility.await().untilAsserted(() -> { + assertEquals(admin2.topics().getPartitionedTopicMetadata(topicName).partitions, 3); + }); + + cleanupTopics(() -> { + admin1.topics().deletePartitionedTopic(topicName, false); + admin2.topics().deletePartitionedTopic(topicName, false); + }); + } + + private String getTheLatestMessage(String topic, PulsarClient client, PulsarAdmin admin) throws Exception { + String dummySubscription = "s_" + UUID.randomUUID().toString().replace("-", ""); + admin.topics().createSubscription(topic, dummySubscription, MessageId.earliest); + Consumer c = client.newConsumer(Schema.STRING).topic(topic).subscriptionName(dummySubscription) + .subscribe(); + String lastMsgValue = null; + while (true) { + Message msg = c.receive(2, TimeUnit.SECONDS); + if (msg == null) { + break; + } + lastMsgValue = msg.getValue(); + } + c.unsubscribe(); + return lastMsgValue; + } + + enum ReplicationLevel { + TOPIC_LEVEL, + NAMESPACE_LEVEL; + } + + @DataProvider(name = "replicationLevels") + public Object[][] replicationLevels() { + return new Object[][]{ + {ReplicationLevel.TOPIC_LEVEL}, + {ReplicationLevel.NAMESPACE_LEVEL} + }; + } + + @Test(dataProvider = "replicationLevels") + public void testReloadWithTopicLevelGeoReplication(ReplicationLevel replicationLevel) throws Exception { + final String topicName = ((Supplier) () -> { + if (replicationLevel.equals(ReplicationLevel.TOPIC_LEVEL)) { + return BrokerTestUtil.newUniqueName("persistent://" + nonReplicatedNamespace + "/tp_"); + } else { + return BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); + } + }).get(); + admin1.topics().createNonPartitionedTopic(topicName); + admin2.topics().createNonPartitionedTopic(topicName); + admin2.topics().createSubscription(topicName, "s1", MessageId.earliest); + if (replicationLevel.equals(ReplicationLevel.TOPIC_LEVEL)) { + admin1.topics().setReplicationClusters(topicName, Arrays.asList(cluster1, cluster2)); + } else { + pulsar1.getConfig().setTopicLevelPoliciesEnabled(false); + } + verifyReplicationWorks(topicName); + + /** + * Verify: + * 1. Inject an error to make the replicator is not able to work. + * 2. Send one message, since the replicator does not work anymore, this message will not be replicated. + * 3. Unload topic, the replicator will be re-created. + * 4. Verify: the message can be replicated to the remote cluster. + */ + // Step 1: Inject an error to make the replicator is not able to work. + Replicator replicator = broker1.getTopic(topicName, false).join().get().getReplicators().get(cluster2); + replicator.terminate(); + + // Step 2: Send one message, since the replicator does not work anymore, this message will not be replicated. + String msg = UUID.randomUUID().toString(); + Producer p1 = client1.newProducer(Schema.STRING).topic(topicName).create(); + p1.send(msg); + p1.close(); + // The result of "peek message" will be the messages generated, so it is not the same as the message just sent. + Thread.sleep(3000); + assertNotEquals(getTheLatestMessage(topicName, client2, admin2), msg); + assertEquals(admin1.topics().getStats(topicName).getReplication().get(cluster2).getReplicationBacklog(), 1); + + // Step 3: Unload topic, the replicator will be re-created. + admin1.topics().unload(topicName); + + // Step 4. Verify: the message can be replicated to the remote cluster. + Awaitility.await().atMost(Duration.ofSeconds(300)).untilAsserted(() -> { + log.info("replication backlog: {}", + admin1.topics().getStats(topicName).getReplication().get(cluster2).getReplicationBacklog()); + assertEquals(admin1.topics().getStats(topicName).getReplication().get(cluster2).getReplicationBacklog(), 0); + assertEquals(getTheLatestMessage(topicName, client2, admin2), msg); + }); + + // Cleanup. + if (replicationLevel.equals(ReplicationLevel.TOPIC_LEVEL)) { + admin1.topics().setReplicationClusters(topicName, Arrays.asList(cluster1)); + Awaitility.await().untilAsserted(() -> { + assertEquals(broker1.getTopic(topicName, false).join().get().getReplicators().size(), 0); + }); + admin1.topics().delete(topicName, false); + admin2.topics().delete(topicName, false); + } else { + pulsar1.getConfig().setTopicLevelPoliciesEnabled(true); + cleanupTopics(() -> { + admin1.topics().delete(topicName); + admin2.topics().delete(topicName); + }); + } + } + + protected void enableReplication(String topic) throws Exception { + admin1.topics().setReplicationClusters(topic, Arrays.asList(cluster1, cluster2)); + } + + protected void disableReplication(String topic) throws Exception { + admin1.topics().setReplicationClusters(topic, Arrays.asList(cluster1, cluster2)); + } + + @Test(timeOut = 30 * 1000) + public void testCreateRemoteAdminFailed() throws Exception { + final TenantInfo tenantInfo = admin1.tenants().getTenantInfo(defaultTenant); + final String ns1 = defaultTenant + "/ns_" + UUID.randomUUID().toString().replace("-", ""); + final String randomClusterName = "c_" + UUID.randomUUID().toString().replace("-", ""); + final String topic = BrokerTestUtil.newUniqueName(ns1 + "/tp"); + admin1.namespaces().createNamespace(ns1); + admin1.topics().createPartitionedTopic(topic, 2); + + // Inject a wrong cluster data which with empty fields. + ClusterResources clusterResources = broker1.getPulsar().getPulsarResources().getClusterResources(); + clusterResources.createCluster(randomClusterName, ClusterData.builder().build()); + Set allowedClusters = new HashSet<>(tenantInfo.getAllowedClusters()); + allowedClusters.add(randomClusterName); + admin1.tenants().updateTenant(defaultTenant, TenantInfo.builder().adminRoles(tenantInfo.getAdminRoles()) + .allowedClusters(allowedClusters).build()); + + // Verify. + try { + admin1.topics().setReplicationClusters(topic, Arrays.asList(cluster1, randomClusterName)); + fail("Expected a error due to empty fields"); + } catch (Exception ex) { + // Expected an error. + } + + // cleanup. + admin1.topics().deletePartitionedTopic(topic); + admin1.tenants().updateTenant(defaultTenant, tenantInfo); + } + + @Test + public void testConfigReplicationStartAt() throws Exception { + // Initialize. + String ns1 = defaultTenant + "/ns_" + UUID.randomUUID().toString().replace("-", ""); + String subscription1 = "s1"; + admin1.namespaces().createNamespace(ns1); + if (!usingGlobalZK) { + admin2.namespaces().createNamespace(ns1); + } + + RetentionPolicies retentionPolicies = new RetentionPolicies(60 * 24, 1024); + admin1.namespaces().setRetention(ns1, retentionPolicies); + admin2.namespaces().setRetention(ns1, retentionPolicies); + + // 1. default config. + // Enable replication for topic1. + final String topic1 = BrokerTestUtil.newUniqueName("persistent://" + ns1 + "/tp_"); + admin1.topics().createNonPartitionedTopicAsync(topic1); + admin1.topics().createSubscription(topic1, subscription1, MessageId.earliest); + Producer p1 = client1.newProducer(Schema.STRING).topic(topic1).create(); + p1.send("msg-1"); + p1.close(); + enableReplication(topic1); + // Verify: since the replication was started at latest, there is no message to consume. + Consumer c1 = client2.newConsumer(Schema.STRING).topic(topic1).subscriptionName(subscription1) + .subscribe(); + Message msg1 = c1.receive(2, TimeUnit.SECONDS); + assertNull(msg1); + c1.close(); + disableReplication(topic1); + + // 2.Update config: start at "earliest". + admin1.brokers().updateDynamicConfiguration("replicationStartAt", MessageId.earliest.toString()); + Awaitility.await().untilAsserted(() -> { + pulsar1.getConfiguration().getReplicationStartAt().equalsIgnoreCase("earliest"); + }); + + final String topic2 = BrokerTestUtil.newUniqueName("persistent://" + ns1 + "/tp_"); + admin1.topics().createNonPartitionedTopicAsync(topic2); + admin1.topics().createSubscription(topic2, subscription1, MessageId.earliest); + Producer p2 = client1.newProducer(Schema.STRING).topic(topic2).create(); + p2.send("msg-1"); + p2.close(); + enableReplication(topic2); + // Verify: since the replication was started at earliest, there is one message to consume. + Consumer c2 = client2.newConsumer(Schema.STRING).topic(topic2).subscriptionName(subscription1) + .subscribe(); + Message msg2 = c2.receive(2, TimeUnit.SECONDS); + assertNotNull(msg2); + assertEquals(msg2.getValue(), "msg-1"); + c2.close(); + disableReplication(topic2); + + // 2.Update config: start at "latest". + admin1.brokers().updateDynamicConfiguration("replicationStartAt", MessageId.latest.toString()); + Awaitility.await().untilAsserted(() -> { + pulsar1.getConfiguration().getReplicationStartAt().equalsIgnoreCase("latest"); + }); + + final String topic3 = BrokerTestUtil.newUniqueName("persistent://" + ns1 + "/tp_"); + admin1.topics().createNonPartitionedTopicAsync(topic3); + admin1.topics().createSubscription(topic3, subscription1, MessageId.earliest); + Producer p3 = client1.newProducer(Schema.STRING).topic(topic3).create(); + p3.send("msg-1"); + p3.close(); + enableReplication(topic3); + // Verify: since the replication was started at latest, there is no message to consume. + Consumer c3 = client2.newConsumer(Schema.STRING).topic(topic3).subscriptionName(subscription1) + .subscribe(); + Message msg3 = c3.receive(2, TimeUnit.SECONDS); + assertNull(msg3); + c3.close(); + disableReplication(topic3); + + // cleanup. + // There is no good way to delete topics when using global ZK, skip cleanup. + admin1.namespaces().setNamespaceReplicationClusters(ns1, Collections.singleton(cluster1)); + admin1.namespaces().unload(ns1); + admin2.namespaces().setNamespaceReplicationClusters(ns1, Collections.singleton(cluster2)); + admin2.namespaces().unload(ns1); + admin1.topics().delete(topic1, false); + admin2.topics().delete(topic1, false); + admin1.topics().delete(topic2, false); + admin2.topics().delete(topic2, false); + admin1.topics().delete(topic3, false); + admin2.topics().delete(topic3, false); + } + + @DataProvider(name = "replicationModes") + public Object[][] replicationModes() { + return new Object[][]{ + {ReplicationMode.OneWay}, + {ReplicationMode.DoubleWay} + }; + } + + protected enum ReplicationMode { + OneWay, + DoubleWay; + } + + @Test(dataProvider = "replicationModes") + public void testDifferentTopicCreationRule(ReplicationMode replicationMode) throws Exception { + String ns = defaultTenant + "/" + UUID.randomUUID().toString().replace("-", ""); + admin1.namespaces().createNamespace(ns); + admin2.namespaces().createNamespace(ns); + + // Set topic auto-creation rule. + // c1: no-partitioned topic + // c2: partitioned topic with 2 partitions. + AutoTopicCreationOverride autoTopicCreation = + AutoTopicCreationOverrideImpl.builder().allowAutoTopicCreation(true) + .topicType("partitioned").defaultNumPartitions(2).build(); + admin2.namespaces().setAutoTopicCreation(ns, autoTopicCreation); + Awaitility.await().untilAsserted(() -> { + assertEquals(admin2.namespaces().getAutoTopicCreationAsync(ns).join().getDefaultNumPartitions(), 2); + // Trigger system topic __change_event's initialize. + pulsar2.getTopicPoliciesService().getTopicPoliciesAsync(TopicName.get("persistent://" + ns + "/1"), + TopicPoliciesService.GetType.DEFAULT); + }); + + // Create non-partitioned topic. + // Enable replication. + final String tp = BrokerTestUtil.newUniqueName("persistent://" + ns + "/tp_"); + admin1.topics().createNonPartitionedTopic(tp); + admin1.namespaces().setNamespaceReplicationClusters(ns, new HashSet<>(Arrays.asList(cluster1, cluster2))); + if (replicationMode.equals(ReplicationMode.DoubleWay)) { + admin2.namespaces().setNamespaceReplicationClusters(ns, new HashSet<>(Arrays.asList(cluster1, cluster2))); + } + + // Trigger and wait for replicator starts. + Producer p1 = client1.newProducer(Schema.STRING).topic(tp).create(); + p1.send("msg-1"); + p1.close(); + Awaitility.await().untilAsserted(() -> { + PersistentTopic persistentTopic = (PersistentTopic) broker1.getTopic(tp, false).join().get(); + assertFalse(persistentTopic.getReplicators().isEmpty()); + }); + + // Verify: the topics are the same between two clusters. + Predicate topicNameFilter = t -> { + TopicName topicName = TopicName.get(t); + if (!topicName.getNamespace().equals(ns)) { + return false; + } + return t.startsWith(tp); + }; + Awaitility.await().untilAsserted(() -> { + List topics1 = pulsar1.getBrokerService().getTopics().keySet() + .stream().filter(topicNameFilter).collect(Collectors.toList()); + List topics2 = pulsar2.getBrokerService().getTopics().keySet() + .stream().filter(topicNameFilter).collect(Collectors.toList()); + Collections.sort(topics1); + Collections.sort(topics2); + assertEquals(topics1, topics2); + }); + + // cleanup. + admin1.namespaces().setNamespaceReplicationClusters(ns, new HashSet<>(Arrays.asList(cluster1))); + if (replicationMode.equals(ReplicationMode.DoubleWay)) { + admin2.namespaces().setNamespaceReplicationClusters(ns, new HashSet<>(Arrays.asList(cluster2))); + } + Awaitility.await().untilAsserted(() -> { + PersistentTopic persistentTopic = (PersistentTopic) broker1.getTopic(tp, false).join().get(); + assertTrue(persistentTopic.getReplicators().isEmpty()); + if (replicationMode.equals(ReplicationMode.DoubleWay)) { + assertTrue(persistentTopic.getReplicators().isEmpty()); + } + }); + admin1.topics().delete(tp, false); + admin2.topics().delete(tp, false); + admin1.namespaces().deleteNamespace(ns); + admin2.namespaces().deleteNamespace(ns); + } + + @Test + public void testReplicationCountMetrics() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + nonReplicatedNamespace + "/tp_"); + // 1.Create topic, does not enable replication now. + admin1.topics().createNonPartitionedTopic(topicName); + PersistentTopic persistentTopic = + (PersistentTopic) pulsar1.getBrokerService().getTopic(topicName, false).join().get(); + + // We inject an error to make the internal producer fail to connect. + final AtomicInteger createProducerCounter = new AtomicInteger(); + final AtomicBoolean failedCreateProducer = new AtomicBoolean(true); + Runnable taskToClearInjection = injectMockReplicatorProducerBuilder((producerCnf, originalProducer) -> { + if (topicName.equals(producerCnf.getTopicName())) { + // There is a switch to determine create producer successfully or not. + if (failedCreateProducer.get()) { + log.info("Retry create replicator.producer count: {}", createProducerCounter); + // Release producer and fail callback. + originalProducer.closeAsync(); + throw new RuntimeException("mock error"); + } + return originalProducer; + } + return originalProducer; + }); + + // 2.Enable replication. + admin1.topics().setReplicationClusters(topicName, Arrays.asList(cluster1, cluster2)); + + // Verify: metrics. + // Cluster level: + // - pulsar_replication_connected_count + // - pulsar_replication_disconnected_count + // Namespace level: + // - pulsar_replication_connected_count + // - pulsar_replication_disconnected_count + // Topic level: + // - pulsar_replication_connected_count + // - pulsar_replication_disconnected_count + JerseyClient httpClient = JerseyClientBuilder.createClient(); + Awaitility.await().untilAsserted(() -> { + int topicConnected = 0; + int topicDisconnected = 0; + + String response = httpClient.target(pulsar1.getWebServiceAddress()).path("/metrics/") + .request().get(String.class); + Multimap metricMap = PrometheusMetricsClient.parseMetrics(response); + if (!metricMap.containsKey("pulsar_replication_disconnected_count")) { + fail("Expected 1 disconnected replicator."); + } + for (PrometheusMetricsClient.Metric metric : metricMap.get("pulsar_replication_connected_count")) { + if (cluster1.equals(metric.tags.get("cluster")) + && nonReplicatedNamespace.equals(metric.tags.get("namespace")) + && topicName.equals(metric.tags.get("topic"))) { + topicConnected += Double.valueOf(metric.value).intValue(); + } + } + for (PrometheusMetricsClient.Metric metric : metricMap.get("pulsar_replication_disconnected_count")) { + if (cluster1.equals(metric.tags.get("cluster")) + && nonReplicatedNamespace.equals(metric.tags.get("namespace")) + && topicName.equals(metric.tags.get("topic"))) { + topicDisconnected += Double.valueOf(metric.value).intValue(); + } + } + log.info("{}, {},", topicConnected, topicDisconnected); + assertEquals(topicConnected, 0); + assertEquals(topicDisconnected, 1); + }); + + // Let replicator connect successfully. + failedCreateProducer.set(false); + // Verify: metrics. + // Cluster level: + // - pulsar_replication_connected_count + // - pulsar_replication_disconnected_count + // Namespace level: + // - pulsar_replication_connected_count + // - pulsar_replication_disconnected_count + // Topic level: + // - pulsar_replication_connected_count + // - pulsar_replication_disconnected_count + Awaitility.await().atMost(Duration.ofSeconds(130)).untilAsserted(() -> { + int topicConnected = 0; + int topicDisconnected = 0; + + String response = httpClient.target(pulsar1.getWebServiceAddress()).path("/metrics/") + .request().get(String.class); + Multimap metricMap = PrometheusMetricsClient.parseMetrics(response); + if (!metricMap.containsKey("pulsar_replication_disconnected_count")) { + fail("Expected 1 disconnected replicator."); + } + for (PrometheusMetricsClient.Metric metric : metricMap.get("pulsar_replication_connected_count")) { + if (cluster1.equals(metric.tags.get("cluster")) + && nonReplicatedNamespace.equals(metric.tags.get("namespace")) + && topicName.equals(metric.tags.get("topic"))) { + topicConnected += Double.valueOf(metric.value).intValue(); + } + } + for (PrometheusMetricsClient.Metric metric : metricMap.get("pulsar_replication_disconnected_count")) { + if (cluster1.equals(metric.tags.get("cluster")) + && nonReplicatedNamespace.equals(metric.tags.get("namespace")) + && topicName.equals(metric.tags.get("topic"))) { + topicDisconnected += Double.valueOf(metric.value).intValue(); + } + } + log.info("{}, {}", topicConnected, topicDisconnected); + assertEquals(topicConnected, 1); + assertEquals(topicDisconnected, 0); + }); + + // cleanup. + taskToClearInjection.run(); + admin1.topics().setReplicationClusters(topicName, Arrays.asList(cluster1)); + waitReplicatorStopped(topicName); + admin1.topics().delete(topicName, false); + admin2.topics().delete(topicName, false); + } + + /** + * This test used to confirm the "start replicator retry task" will be skipped after the topic is closed. + */ + @Test + public void testCloseTopicAfterStartReplicationFailed() throws Exception { + Field fieldTopicNameCache = TopicName.class.getDeclaredField("cache"); + fieldTopicNameCache.setAccessible(true); + ConcurrentHashMap topicNameCache = + (ConcurrentHashMap) fieldTopicNameCache.get(null); + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + nonReplicatedNamespace + "/tp_"); + // 1.Create topic, does not enable replication now. + admin1.topics().createNonPartitionedTopic(topicName); + Producer producer1 = client1.newProducer().topic(topicName).create(); + PersistentTopic persistentTopic = + (PersistentTopic) pulsar1.getBrokerService().getTopic(topicName, false).join().get(); + + // We inject an error to make "start replicator" to fail. + AsyncLoadingCache existsCache = + WhiteboxImpl.getInternalState(pulsar1.getConfigurationMetadataStore(), "existsCache"); + String path = "/admin/partitioned-topics/" + TopicName.get(topicName).getPersistenceNamingEncoding(); + existsCache.put(path, CompletableFuture.completedFuture(true)); + + // 2.Enable replication and unload topic after failed to start replicator. + admin1.topics().setReplicationClusters(topicName, Arrays.asList(cluster1, cluster2)); + Thread.sleep(3000); + producer1.close(); + existsCache.synchronous().invalidate(path); + admin1.topics().unload(topicName); + // Verify: the "start replicator retry task" will be skipped after the topic is closed. + // - Retry delay is "PersistentTopic.POLICY_UPDATE_FAILURE_RETRY_TIME_SECONDS": 60s, so wait for 70s. + // - Since the topic should not be touched anymore, we use "TopicName" to confirm whether it be used by + // Replication again. + Thread.sleep(10 * 1000); + topicNameCache.remove(topicName); + Thread.sleep(60 * 1000); + assertTrue(!topicNameCache.containsKey(topicName)); + + // cleanup. + admin1.topics().setReplicationClusters(topicName, Arrays.asList(cluster1)); + admin1.topics().delete(topicName, false); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTestBase.java index 33620716288af..200c8dd3b3d9f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTestBase.java @@ -18,36 +18,65 @@ */ package org.apache.pulsar.broker.service; +import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.BROKER_CERT_FILE_PATH; +import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.BROKER_KEY_FILE_PATH; +import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.CA_CERT_FILE_PATH; +import static org.apache.pulsar.compaction.Compactor.COMPACTION_SUBSCRIPTION; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.assertEquals; import com.google.common.collect.Sets; import java.net.URL; +import java.time.Duration; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; +import java.util.List; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.ClientBuilder; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.naming.SystemTopicNames; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.policies.data.TopicType; +import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; import org.apache.pulsar.tests.TestRetrySupport; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.apache.pulsar.zookeeper.ZookeeperServerTest; +import org.awaitility.Awaitility; +import org.testng.Assert; @Slf4j public abstract class OneWayReplicatorTestBase extends TestRetrySupport { protected final String defaultTenant = "public"; - protected final String defaultNamespace = defaultTenant + "/default"; + protected final String replicatedNamespace = defaultTenant + "/default"; + protected final String nonReplicatedNamespace = defaultTenant + "/ns1"; protected final String cluster1 = "r1"; + + protected boolean usingGlobalZK = false; + protected URL url1; protected URL urlTls1; protected ServiceConfiguration config1 = new ServiceConfiguration(); protected ZookeeperServerTest brokerConfigZk1; protected LocalBookkeeperEnsemble bkEnsemble1; protected PulsarService pulsar1; - protected BrokerService ns1; + protected BrokerService broker1; protected PulsarAdmin admin1; protected PulsarClient client1; @@ -58,7 +87,7 @@ public abstract class OneWayReplicatorTestBase extends TestRetrySupport { protected ZookeeperServerTest brokerConfigZk2; protected LocalBookkeeperEnsemble bkEnsemble2; protected PulsarService pulsar2; - protected BrokerService ns2; + protected BrokerService broker2; protected PulsarAdmin admin2; protected PulsarClient client2; @@ -66,8 +95,12 @@ protected void startZKAndBK() throws Exception { // Start ZK. brokerConfigZk1 = new ZookeeperServerTest(0); brokerConfigZk1.start(); - brokerConfigZk2 = new ZookeeperServerTest(0); - brokerConfigZk2.start(); + if (usingGlobalZK) { + brokerConfigZk2 = brokerConfigZk1; + } else { + brokerConfigZk2 = new ZookeeperServerTest(0); + brokerConfigZk2.start(); + } // Start BK. bkEnsemble1 = new LocalBookkeeperEnsemble(3, 0, () -> 0); @@ -81,23 +114,29 @@ protected void startBrokers() throws Exception { setConfigDefaults(config1, cluster1, bkEnsemble1, brokerConfigZk1); pulsar1 = new PulsarService(config1); pulsar1.start(); - ns1 = pulsar1.getBrokerService(); - + broker1 = pulsar1.getBrokerService(); url1 = new URL(pulsar1.getWebServiceAddress()); urlTls1 = new URL(pulsar1.getWebServiceAddressTls()); - admin1 = PulsarAdmin.builder().serviceHttpUrl(url1.toString()).build(); - client1 = PulsarClient.builder().serviceUrl(url1.toString()).build(); // Start region 2 setConfigDefaults(config2, cluster2, bkEnsemble2, brokerConfigZk2); pulsar2 = new PulsarService(config2); pulsar2.start(); - ns2 = pulsar2.getBrokerService(); - + broker2 = pulsar2.getBrokerService(); url2 = new URL(pulsar2.getWebServiceAddress()); urlTls2 = new URL(pulsar2.getWebServiceAddressTls()); + } + + protected void startAdminClient() throws Exception { + admin1 = PulsarAdmin.builder().serviceHttpUrl(url1.toString()).build(); admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build(); - client2 = PulsarClient.builder().serviceUrl(url2.toString()).build(); + } + + protected void startPulsarClient() throws Exception{ + ClientBuilder clientBuilder1 = PulsarClient.builder().serviceUrl(url1.toString()); + client1 = initClient(clientBuilder1); + ClientBuilder clientBuilder2 = PulsarClient.builder().serviceUrl(url2.toString()); + client2 = initClient(clientBuilder2); } protected void createDefaultTenantsAndClustersAndNamespace() throws Exception { @@ -115,35 +154,75 @@ protected void createDefaultTenantsAndClustersAndNamespace() throws Exception { .brokerServiceUrlTls(pulsar2.getBrokerServiceUrlTls()) .brokerClientTlsEnabled(false) .build()); - admin2.clusters().createCluster(cluster1, ClusterData.builder() - .serviceUrl(url1.toString()) - .serviceUrlTls(urlTls1.toString()) - .brokerServiceUrl(pulsar1.getBrokerServiceUrl()) - .brokerServiceUrlTls(pulsar1.getBrokerServiceUrlTls()) - .brokerClientTlsEnabled(false) - .build()); - admin2.clusters().createCluster(cluster2, ClusterData.builder() - .serviceUrl(url2.toString()) - .serviceUrlTls(urlTls2.toString()) - .brokerServiceUrl(pulsar2.getBrokerServiceUrl()) - .brokerServiceUrlTls(pulsar2.getBrokerServiceUrlTls()) - .brokerClientTlsEnabled(false) - .build()); - admin1.tenants().createTenant(defaultTenant, new TenantInfoImpl(Collections.emptySet(), Sets.newHashSet(cluster1, cluster2))); - admin2.tenants().createTenant(defaultTenant, new TenantInfoImpl(Collections.emptySet(), - Sets.newHashSet(cluster1, cluster2))); + admin1.namespaces().createNamespace(replicatedNamespace, Sets.newHashSet(cluster1, cluster2)); + admin1.namespaces().createNamespace(nonReplicatedNamespace); + + if (!usingGlobalZK) { + admin2.clusters().createCluster(cluster1, ClusterData.builder() + .serviceUrl(url1.toString()) + .serviceUrlTls(urlTls1.toString()) + .brokerServiceUrl(pulsar1.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar1.getBrokerServiceUrlTls()) + .brokerClientTlsEnabled(false) + .build()); + admin2.clusters().createCluster(cluster2, ClusterData.builder() + .serviceUrl(url2.toString()) + .serviceUrlTls(urlTls2.toString()) + .brokerServiceUrl(pulsar2.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar2.getBrokerServiceUrlTls()) + .brokerClientTlsEnabled(false) + .build()); + admin2.tenants().createTenant(defaultTenant, new TenantInfoImpl(Collections.emptySet(), + Sets.newHashSet(cluster1, cluster2))); + admin2.namespaces().createNamespace(replicatedNamespace); + admin2.namespaces().createNamespace(nonReplicatedNamespace); + } - admin1.namespaces().createNamespace(defaultNamespace, Sets.newHashSet(cluster1, cluster2)); - admin2.namespaces().createNamespace(defaultNamespace); } protected void cleanupTopics(CleanupTopicAction cleanupTopicAction) throws Exception { - admin1.namespaces().setNamespaceReplicationClusters(defaultNamespace, Collections.singleton(cluster1)); - admin1.namespaces().unload(defaultNamespace); + cleanupTopics(replicatedNamespace, cleanupTopicAction); + } + + protected void cleanupTopics(String namespace, CleanupTopicAction cleanupTopicAction) throws Exception { + if (usingGlobalZK) { + throw new IllegalArgumentException("The method cleanupTopics does not support for global ZK"); + } + waitChangeEventsInit(namespace); + admin1.namespaces().setNamespaceReplicationClusters(namespace, Collections.singleton(cluster1)); + admin1.namespaces().unload(namespace); cleanupTopicAction.run(); - admin1.namespaces().setNamespaceReplicationClusters(defaultNamespace, Sets.newHashSet(cluster1, cluster2)); + admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet(cluster1, cluster2)); + waitChangeEventsInit(namespace); + } + + protected void waitChangeEventsInit(String namespace) { + CompletableFuture> future = pulsar1.getBrokerService() + .getTopic(namespace + "/" + SystemTopicNames.NAMESPACE_EVENTS_LOCAL_NAME, false); + if (future == null) { + return; + } + Optional optional = future.join(); + if (!optional.isPresent()) { + return; + } + PersistentTopic topic = (PersistentTopic) optional.get(); + Awaitility.await().atMost(Duration.ofSeconds(180)).untilAsserted(() -> { + TopicStatsImpl topicStats = topic.getStats(true, false, false); + topicStats.getSubscriptions().entrySet().forEach(entry -> { + // No wait for compaction. + if (COMPACTION_SUBSCRIPTION.equals(entry.getKey())) { + return; + } + // No wait for durable cursor. + if (entry.getValue().isDurable()) { + return; + } + Assert.assertTrue(entry.getValue().getMsgBacklog() == 0, entry.getKey()); + }); + }); } protected interface CleanupTopicAction { @@ -160,13 +239,17 @@ protected void setup() throws Exception { startBrokers(); + startAdminClient(); + createDefaultTenantsAndClustersAndNamespace(); + startPulsarClient(); + Thread.sleep(100); log.info("--- OneWayReplicatorTestBase::setup completed ---"); } - private void setConfigDefaults(ServiceConfiguration config, String clusterName, + protected void setConfigDefaults(ServiceConfiguration config, String clusterName, LocalBookkeeperEnsemble bookkeeperEnsemble, ZookeeperServerTest brokerConfigZk) { config.setClusterName(clusterName); config.setAdvertisedAddress("localhost"); @@ -185,35 +268,216 @@ private void setConfigDefaults(ServiceConfiguration config, String clusterName, config.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED); config.setEnableReplicatedSubscriptions(true); config.setReplicatedSubscriptionsSnapshotFrequencyMillis(1000); + config.setLoadBalancerSheddingEnabled(false); + config.setForceDeleteNamespaceAllowed(true); + config.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + config.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); + config.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + config.setClusterName(clusterName); + config.setTlsRequireTrustedClientCertOnConnect(false); + Set tlsProtocols = Sets.newConcurrentHashSet(); + tlsProtocols.add("TLSv1.3"); + tlsProtocols.add("TLSv1.2"); + config.setTlsProtocols(tlsProtocols); + } + + protected void cleanupPulsarResources() throws Exception { + // delete namespaces. + waitChangeEventsInit(replicatedNamespace); + admin1.namespaces().setNamespaceReplicationClusters(replicatedNamespace, Sets.newHashSet(cluster1)); + if (!usingGlobalZK) { + admin2.namespaces().setNamespaceReplicationClusters(replicatedNamespace, Sets.newHashSet(cluster2)); + } + admin1.namespaces().deleteNamespace(replicatedNamespace, true); + admin1.namespaces().deleteNamespace(nonReplicatedNamespace, true); + if (!usingGlobalZK) { + admin2.namespaces().deleteNamespace(replicatedNamespace, true); + admin2.namespaces().deleteNamespace(nonReplicatedNamespace, true); + } } @Override protected void cleanup() throws Exception { + // cleanup pulsar resources. + cleanupPulsarResources(); + + // shutdown. markCurrentSetupNumberCleaned(); log.info("--- Shutting down ---"); // Stop brokers. - client1.close(); - client2.close(); - admin1.close(); - admin2.close(); + if (client1 != null) { + client1.close(); + client1 = null; + } + if (client2 != null) { + client2.close(); + client2 = null; + } + if (admin1 != null) { + admin1.close(); + admin1 = null; + } + if (admin2 != null) { + admin2.close(); + admin2 = null; + } if (pulsar2 != null) { pulsar2.close(); + pulsar2 = null; } if (pulsar1 != null) { pulsar1.close(); + pulsar1 = null; } // Stop ZK and BK. - bkEnsemble1.stop(); - bkEnsemble2.stop(); - brokerConfigZk1.stop(); - brokerConfigZk2.stop(); + if (bkEnsemble1 != null) { + bkEnsemble1.stop(); + bkEnsemble1 = null; + } + if (bkEnsemble2 != null) { + bkEnsemble2.stop(); + bkEnsemble2 = null; + } + if (brokerConfigZk1 != null) { + brokerConfigZk1.stop(); + brokerConfigZk1 = null; + } + if (!usingGlobalZK && brokerConfigZk2 != null) { + brokerConfigZk2.stop(); + brokerConfigZk2 = null; + } // Reset configs. config1 = new ServiceConfiguration(); - setConfigDefaults(config1, cluster1, bkEnsemble1, brokerConfigZk1); config2 = new ServiceConfiguration(); - setConfigDefaults(config2, cluster2, bkEnsemble2, brokerConfigZk2); + } + + protected void waitReplicatorStarted(String topicName) { + Awaitility.await().untilAsserted(() -> { + Optional topicOptional2 = pulsar2.getBrokerService().getTopic(topicName, false).get(); + assertTrue(topicOptional2.isPresent()); + PersistentTopic persistentTopic2 = (PersistentTopic) topicOptional2.get(); + assertFalse(persistentTopic2.getProducers().isEmpty()); + }); + } + + protected PulsarClient initClient(ClientBuilder clientBuilder) throws Exception { + return clientBuilder.build(); + } + + protected void verifyReplicationWorks(String topic) throws Exception { + // Wait for replicator starting. + Awaitility.await().until(() -> { + try { + PersistentTopic persistentTopic = (PersistentTopic) pulsar1.getBrokerService() + .getTopic(topic, false).join().get(); + if (persistentTopic.getReplicators().size() > 0) { + return true; + } + } catch (Exception ex) {} + + try { + String partition0 = TopicName.get(topic).getPartition(0).toString(); + PersistentTopic persistentTopic = (PersistentTopic) pulsar1.getBrokerService() + .getTopic(partition0, false).join().get(); + if (persistentTopic.getReplicators().size() > 0) { + return true; + } + } catch (Exception ex) {} + + return false; + }); + // Verify: pub & sub. + final String subscription = "__subscribe_1"; + final String msgValue = "__msg1"; + Producer producer1 = client1.newProducer(Schema.STRING).topic(topic).create(); + Consumer consumer2 = client2.newConsumer(Schema.STRING).topic(topic).isAckReceiptEnabled(true) + .subscriptionName(subscription).subscribe(); + producer1.newMessage().value(msgValue).send(); + pulsar1.getBrokerService().checkReplicationPolicies(); + assertEquals(consumer2.receive(10, TimeUnit.SECONDS).getValue(), msgValue); + consumer2.unsubscribe(); + producer1.close(); + } + + protected void setTopicLevelClusters(String topic, List clusters, PulsarAdmin admin, + PulsarService pulsar) throws Exception { + Set expected = new HashSet<>(clusters); + TopicName topicName = TopicName.get(TopicName.get(topic).getPartitionedTopicName()); + int partitions = ensurePartitionsAreSame(topic); + admin.topics().setReplicationClusters(topic, clusters); + Awaitility.await().untilAsserted(() -> { + TopicPolicies policies = TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), topicName); + assertEquals(new HashSet<>(policies.getReplicationClusters()), expected); + if (partitions == 0) { + checkNonPartitionedTopicLevelClusters(topicName.toString(), clusters, admin, pulsar.getBrokerService()); + } else { + for (int i = 0; i < partitions; i++) { + checkNonPartitionedTopicLevelClusters(topicName.getPartition(i).toString(), clusters, admin, + pulsar.getBrokerService()); + } + } + }); + } + + protected void checkNonPartitionedTopicLevelClusters(String topic, List clusters, PulsarAdmin admin, + BrokerService broker) throws Exception { + CompletableFuture> future = broker.getTopic(topic, false); + if (future == null) { + return; + } + Optional optional = future.join(); + if (optional == null || !optional.isPresent()) { + return; + } + PersistentTopic persistentTopic = (PersistentTopic) optional.get(); + Set expected = new HashSet<>(clusters); + Set act = new HashSet<>(TopicPolicyTestUtils.getTopicPolicies(persistentTopic).getReplicationClusters()); + assertEquals(act, expected); + } + + protected int ensurePartitionsAreSame(String topic) throws Exception { + TopicName topicName = TopicName.get(TopicName.get(topic).getPartitionedTopicName()); + boolean isPartitionedTopic1 = pulsar1.getPulsarResources().getNamespaceResources() + .getPartitionedTopicResources().partitionedTopicExists(topicName); + boolean isPartitionedTopic2 = pulsar2.getPulsarResources().getNamespaceResources() + .getPartitionedTopicResources().partitionedTopicExists(topicName); + if (isPartitionedTopic1 != isPartitionedTopic2) { + throw new IllegalArgumentException(String.format("Can not delete topic." + + " isPartitionedTopic1: %s, isPartitionedTopic2: %s", + isPartitionedTopic1, isPartitionedTopic2)); + } + if (!isPartitionedTopic1) { + return 0; + } + int partitions1 = pulsar1.getPulsarResources().getNamespaceResources() + .getPartitionedTopicResources().getPartitionedTopicMetadataAsync(topicName).join().get().partitions; + int partitions2 = pulsar2.getPulsarResources().getNamespaceResources() + .getPartitionedTopicResources().getPartitionedTopicMetadataAsync(topicName).join().get().partitions; + if (partitions1 != partitions2) { + throw new IllegalArgumentException(String.format("Can not delete topic." + + " partitions1: %s, partitions2: %s", + partitions1, partitions2)); + } + return partitions1; + } + + protected void deleteTopicAfterDisableTopicLevelReplication(String topic) throws Exception { + setTopicLevelClusters(topic, Arrays.asList(cluster1), admin1, pulsar1); + setTopicLevelClusters(topic, Arrays.asList(cluster1), admin2, pulsar2); + admin2.topics().setReplicationClusters(topic, Arrays.asList(cluster2)); + + int partitions = ensurePartitionsAreSame(topic); + + TopicName topicName = TopicName.get(TopicName.get(topic).getPartitionedTopicName()); + if (partitions != 0) { + admin1.topics().deletePartitionedTopic(topicName.toString()); + admin2.topics().deletePartitionedTopic(topicName.toString()); + } else { + admin1.topics().delete(topicName.toString()); + admin2.topics().delete(topicName.toString()); + } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java new file mode 100644 index 0000000000000..d99969fbaa7e5 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import java.util.Arrays; +import java.util.HashSet; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.policies.data.RetentionPolicies; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class OneWayReplicatorUsingGlobalZKTest extends OneWayReplicatorTest { + + @Override + @BeforeClass(alwaysRun = true, timeOut = 300000) + public void setup() throws Exception { + super.usingGlobalZK = true; + super.setup(); + } + + @Override + @AfterClass(alwaysRun = true, timeOut = 300000) + public void cleanup() throws Exception { + super.cleanup(); + } + + @Test(enabled = false) + public void testReplicatorProducerStatInTopic() throws Exception { + super.testReplicatorProducerStatInTopic(); + } + + @Test(enabled = false) + public void testCreateRemoteConsumerFirst() throws Exception { + super.testReplicatorProducerStatInTopic(); + } + + @Test(enabled = false) + public void testTopicCloseWhenInternalProducerCloseErrorOnce() throws Exception { + super.testReplicatorProducerStatInTopic(); + } + + @Test(enabled = false) + public void testConcurrencyOfUnloadBundleAndRecreateProducer() throws Exception { + super.testConcurrencyOfUnloadBundleAndRecreateProducer(); + } + + @Test(enabled = false) + public void testPartitionedTopicLevelReplication() throws Exception { + super.testPartitionedTopicLevelReplication(); + } + + @Test(enabled = false) + public void testPartitionedTopicLevelReplicationRemoteTopicExist() throws Exception { + super.testPartitionedTopicLevelReplicationRemoteTopicExist(); + } + + @Test(enabled = false) + public void testPartitionedTopicLevelReplicationRemoteConflictTopicExist() throws Exception { + super.testPartitionedTopicLevelReplicationRemoteConflictTopicExist(); + } + + @Test(enabled = false) + public void testConcurrencyOfUnloadBundleAndRecreateProducer2() throws Exception { + super.testConcurrencyOfUnloadBundleAndRecreateProducer2(); + } + + @Test(enabled = false) + public void testUnFenceTopicToReuse() throws Exception { + super.testUnFenceTopicToReuse(); + } + + @Test + public void testDeleteNonPartitionedTopic() throws Exception { + super.testDeleteNonPartitionedTopic(); + } + + @Test + public void testDeletePartitionedTopic() throws Exception { + super.testDeletePartitionedTopic(); + } + + @Test(enabled = false) + public void testNoExpandTopicPartitionsWhenDisableTopicLevelReplication() throws Exception { + super.testNoExpandTopicPartitionsWhenDisableTopicLevelReplication(); + } + + @Test(enabled = false) + public void testExpandTopicPartitionsOnNamespaceLevelReplication() throws Exception { + super.testExpandTopicPartitionsOnNamespaceLevelReplication(); + } + + @Test(enabled = false) + public void testReloadWithTopicLevelGeoReplication(ReplicationLevel replicationLevel) throws Exception { + super.testReloadWithTopicLevelGeoReplication(replicationLevel); + } + + @Test + @Override + public void testConfigReplicationStartAt() throws Exception { + // Initialize. + String ns1 = defaultTenant + "/ns_" + UUID.randomUUID().toString().replace("-", ""); + String subscription1 = "s1"; + admin1.namespaces().createNamespace(ns1); + RetentionPolicies retentionPolicies = new RetentionPolicies(60 * 24, 1024); + admin1.namespaces().setRetention(ns1, retentionPolicies); + admin2.namespaces().setRetention(ns1, retentionPolicies); + + // Update config: start at "earliest". + admin1.brokers().updateDynamicConfiguration("replicationStartAt", MessageId.earliest.toString()); + Awaitility.await().untilAsserted(() -> { + pulsar1.getConfiguration().getReplicationStartAt().equalsIgnoreCase("earliest"); + }); + + // Verify: since the replication was started at earliest, there is one message to consume. + final String topic1 = BrokerTestUtil.newUniqueName("persistent://" + ns1 + "/tp_"); + admin1.topics().createNonPartitionedTopicAsync(topic1); + admin1.topics().createSubscription(topic1, subscription1, MessageId.earliest); + org.apache.pulsar.client.api.Producer p1 = client1.newProducer(Schema.STRING).topic(topic1).create(); + p1.send("msg-1"); + p1.close(); + + admin1.namespaces().setNamespaceReplicationClusters(ns1, new HashSet<>(Arrays.asList(cluster1, cluster2))); + org.apache.pulsar.client.api.Consumer c1 = client2.newConsumer(Schema.STRING).topic(topic1) + .subscriptionName(subscription1).subscribe(); + Message msg2 = c1.receive(2, TimeUnit.SECONDS); + assertNotNull(msg2); + assertEquals(msg2.getValue(), "msg-1"); + c1.close(); + + // cleanup. + admin1.brokers().updateDynamicConfiguration("replicationStartAt", MessageId.latest.toString()); + Awaitility.await().untilAsserted(() -> { + pulsar1.getConfiguration().getReplicationStartAt().equalsIgnoreCase("latest"); + }); + } + + @Test(enabled = false) + @Override + public void testDifferentTopicCreationRule(ReplicationMode replicationMode) throws Exception { + super.testDifferentTopicCreationRule(replicationMode); + } + + @Test(enabled = false) + @Override + public void testReplicationCountMetrics() throws Exception { + super.testReplicationCountMetrics(); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java index 631b702dfbd08..000ea7af91525 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java @@ -51,6 +51,7 @@ import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; +import lombok.Cleanup; import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteLedgerCallback; @@ -60,15 +61,15 @@ import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherSingleActiveConsumer; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.common.api.proto.BaseCommand; import org.apache.pulsar.common.api.proto.CommandActiveConsumerChange; @@ -157,8 +158,7 @@ public void setup() throws Exception { doReturn(new PulsarCommandSenderImpl(null, serverCnxWithOldVersion)) .when(serverCnxWithOldVersion).getCommandSender(); - NamespaceService nsSvc = mock(NamespaceService.class); - doReturn(nsSvc).when(pulsarTestContext.getPulsarService()).getNamespaceService(); + NamespaceService nsSvc = pulsarTestContext.getPulsarService().getNamespaceService(); doReturn(true).when(nsSvc).isServiceUnitOwned(any(NamespaceBundle.class)); doReturn(true).when(nsSvc).isServiceUnitActive(any(TopicName.class)); doReturn(CompletableFuture.completedFuture(true)).when(nsSvc).checkTopicOwnership(any(TopicName.class)); @@ -180,6 +180,7 @@ void setupMLAsyncCallbackMocks() { cursorMock = mock(ManagedCursorImpl.class); doReturn(new ArrayList<>()).when(ledgerMock).getCursors(); + doReturn(new ManagedLedgerConfig()).when(ledgerMock).getConfig(); doReturn("mockCursor").when(cursorMock).getName(); // call openLedgerComplete with ledgerMock on ML factory asyncOpen @@ -202,7 +203,7 @@ void setupMLAsyncCallbackMocks() { // call addComplete on ledger asyncAddEntry doAnswer(invocationOnMock -> { ((AddEntryCallback) invocationOnMock.getArguments()[1]).addComplete( - new PositionImpl(1, 1), null, null); + PositionFactory.create(1, 1), null, null); return null; }).when(ledgerMock).asyncAddEntry(any(byte[].class), any(AddEntryCallback.class), any()); @@ -457,11 +458,12 @@ private CommandActiveConsumerChange waitActiveChangeEvent(int consumerId) return res.get(); } - @Test(invocationCount = 100) + @Test public void testAddRemoveConsumerNonPartitionedTopic() throws Exception { - log.info("--- Starting PersistentDispatcherFailoverConsumerTest::testAddConsumer ---"); + log.info("--- Starting PersistentDispatcherFailoverConsumerTest::testAddRemoveConsumerNonPartitionedTopic ---"); String[] sortedConsumerNameByHashSelector = sortConsumerNameByHashSelector("Cons1", "Cons2"); - BrokerService spyBrokerService = spy(pulsarTestContext.getBrokerService()); + BrokerService spyBrokerService = pulsarTestContext.getBrokerService(); + @Cleanup("shutdownNow") final EventLoopGroup singleEventLoopGroup = EventLoopUtil.newEventLoopGroup(1, pulsarTestContext.getBrokerService().getPulsar().getConfig().isEnableBusyWait(), new DefaultThreadFactory("pulsar-io")); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java index 48798f0020f01..176a799292ac3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.service; -import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.retryStrategically; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.doAnswer; @@ -33,10 +32,8 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; - import io.netty.buffer.ByteBuf; import io.netty.buffer.UnpooledByteBufAllocator; - import java.lang.reflect.Field; import java.util.ArrayList; import java.util.HashSet; @@ -46,7 +43,9 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; - +import java.util.concurrent.atomic.AtomicReference; +import javax.ws.rs.client.Entity; +import javax.ws.rs.core.MediaType; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedCursor; @@ -54,14 +53,16 @@ import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo; import org.apache.bookkeeper.test.MockedBookKeeperTestCase; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.service.persistent.PersistentMessageExpiryMonitor; import org.apache.pulsar.broker.service.persistent.PersistentMessageFinder; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.impl.ResetCursorData; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.api.proto.BrokerEntryMetadata; @@ -71,17 +72,19 @@ import org.apache.pulsar.common.protocol.ByteBufPair; import org.apache.pulsar.common.protocol.Commands; import org.awaitility.Awaitility; +import org.mockito.Mockito; +import org.testng.Assert; import org.testng.annotations.Test; -import javax.ws.rs.client.Entity; -import javax.ws.rs.core.MediaType; - @Test(groups = "broker") public class PersistentMessageFinderTest extends MockedBookKeeperTestCase { public static byte[] createMessageWrittenToLedger(String msg) { + return createMessageWrittenToLedger(msg, System.currentTimeMillis()); + } + public static byte[] createMessageWrittenToLedger(String msg, long messageTimestamp) { MessageMetadata messageMetadata = new MessageMetadata() - .setPublishTime(System.currentTimeMillis()) + .setPublishTime(messageTimestamp) .setProducerName("createMessageWrittenToLedger") .setSequenceId(1); ByteBuf data = UnpooledByteBufAllocator.DEFAULT.heapBuffer().writeBytes(msg.getBytes()); @@ -230,7 +233,11 @@ public void findEntryFailed(ManagedLedgerException exception, Optional }); assertTrue(ex.get()); - PersistentMessageExpiryMonitor monitor = new PersistentMessageExpiryMonitor("topicname", c1.getName(), c1, null); + PersistentTopic mock = mock(PersistentTopic.class); + when(mock.getName()).thenReturn("topicname"); + when(mock.getLastPosition()).thenReturn(PositionFactory.EARLIEST); + + PersistentMessageExpiryMonitor monitor = new PersistentMessageExpiryMonitor(mock, c1.getName(), c1, null); monitor.findEntryFailed(new ManagedLedgerException.ConcurrentFindCursorPositionException("failed"), Optional.empty(), null); Field field = monitor.getClass().getDeclaredField("expirationCheckInProgress"); @@ -243,6 +250,39 @@ public void findEntryFailed(ManagedLedgerException exception, Optional factory.shutdown(); } + @Test + void testPersistentMessageFinderWhenLastMessageDelete() throws Exception { + final String ledgerAndCursorName = "testPersistentMessageFinderWhenLastMessageDelete"; + + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setRetentionSizeInMB(10); + config.setMaxEntriesPerLedger(10); + config.setRetentionTime(1, TimeUnit.HOURS); + ManagedLedger ledger = factory.open(ledgerAndCursorName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor(ledgerAndCursorName); + + ledger.addEntry(createMessageWrittenToLedger("msg1")); + ledger.addEntry(createMessageWrittenToLedger("msg2")); + ledger.addEntry(createMessageWrittenToLedger("msg3")); + Position lastPosition = ledger.addEntry(createMessageWrittenToLedger("last-message")); + + long endTimestamp = System.currentTimeMillis() + 1000; + + Result result = new Result(); + // delete last position message + cursor.delete(lastPosition); + CompletableFuture future = findMessage(result, cursor, endTimestamp); + future.get(); + assertNull(result.exception); + assertNotEquals(result.position, null); + assertEquals(result.position, lastPosition); + + result.reset(); + cursor.close(); + ledger.close(); + factory.shutdown(); + } + @Test void testPersistentMessageFinderWithBrokerTimestampForMessage() throws Exception { @@ -361,11 +401,15 @@ void testMessageExpiryWithTimestampNonRecoverableException() throws Exception { for (int i = 0; i < totalEntries; i++) { ledger.addEntry(createMessageWrittenToLedger("msg" + i)); } + Awaitility.await().untilAsserted(() -> + assertEquals(ledger.getState(), ManagedLedgerImpl.State.LedgerOpened)); List ledgers = ledger.getLedgersInfoAsList(); LedgerInfo lastLedgerInfo = ledgers.get(ledgers.size() - 1); - - assertEquals(ledgers.size(), totalEntries / entriesPerLedger); + // The `lastLedgerInfo` should be newly opened, and it does not contain any entries. + // Please refer to: https://github.com/apache/pulsar/pull/22034 + assertEquals(lastLedgerInfo.getEntries(), 0); + assertEquals(ledgers.size(), totalEntries / entriesPerLedger + 1); // this will make sure that all entries should be deleted Thread.sleep(TimeUnit.SECONDS.toMillis(ttlSeconds)); @@ -374,20 +418,18 @@ void testMessageExpiryWithTimestampNonRecoverableException() throws Exception { bkc.deleteLedger(ledgers.get(1).getLedgerId()); bkc.deleteLedger(ledgers.get(2).getLedgerId()); - PersistentMessageExpiryMonitor monitor = new PersistentMessageExpiryMonitor("topicname", c1.getName(), c1, null); - Position previousMarkDelete = null; - for (int i = 0; i < totalEntries; i++) { - monitor.expireMessages(1); - Position previousPos = previousMarkDelete; - retryStrategically( - (test) -> c1.getMarkDeletedPosition() != null && !c1.getMarkDeletedPosition().equals(previousPos), - 5, 100); - previousMarkDelete = c1.getMarkDeletedPosition(); - } - - PositionImpl markDeletePosition = (PositionImpl) c1.getMarkDeletedPosition(); - assertEquals(lastLedgerInfo.getLedgerId(), markDeletePosition.getLedgerId()); - assertEquals(lastLedgerInfo.getEntries() - 1, markDeletePosition.getEntryId()); + PersistentTopic mock = mock(PersistentTopic.class); + when(mock.getName()).thenReturn("topicname"); + when(mock.getLastPosition()).thenReturn(PositionFactory.EARLIEST); + + PersistentMessageExpiryMonitor monitor = new PersistentMessageExpiryMonitor(mock, c1.getName(), c1, null); + assertTrue(monitor.expireMessages(ttlSeconds)); + Awaitility.await().untilAsserted(() -> { + Position markDeletePosition = c1.getMarkDeletedPosition(); + // The markDeletePosition points to the last entry of the previous ledger in lastLedgerInfo. + assertEquals(markDeletePosition.getLedgerId(), lastLedgerInfo.getLedgerId() - 1); + assertEquals(markDeletePosition.getEntryId(), entriesPerLedger - 1); + }); c1.close(); ledger.close(); @@ -395,6 +437,73 @@ void testMessageExpiryWithTimestampNonRecoverableException() throws Exception { } + @Test + public void testIncorrectClientClock() throws Exception { + final String ledgerAndCursorName = "testIncorrectClientClock"; + int maxTTLSeconds = 1; + int entriesNum = 10; + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setMaxEntriesPerLedger(1); + ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open(ledgerAndCursorName, config); + ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor(ledgerAndCursorName); + // set client clock to 10 days later + long incorrectPublishTimestamp = System.currentTimeMillis() + TimeUnit.DAYS.toMillis(10); + for (int i = 0; i < entriesNum; i++) { + ledger.addEntry(createMessageWrittenToLedger("msg" + i, incorrectPublishTimestamp)); + } + Awaitility.await().untilAsserted(() -> + assertEquals(ledger.getState(), ManagedLedgerImpl.State.LedgerOpened)); + // The number of ledgers should be (entriesNum / MaxEntriesPerLedger) + 1 + // Please refer to: https://github.com/apache/pulsar/pull/22034 + assertEquals(ledger.getLedgersInfoAsList().size(), entriesNum + 1); + PersistentTopic mock = mock(PersistentTopic.class); + when(mock.getName()).thenReturn("topicname"); + when(mock.getLastPosition()).thenReturn(PositionFactory.EARLIEST); + PersistentMessageExpiryMonitor monitor = new PersistentMessageExpiryMonitor(mock, c1.getName(), c1, null); + Thread.sleep(TimeUnit.SECONDS.toMillis(maxTTLSeconds)); + monitor.expireMessages(maxTTLSeconds); + assertEquals(c1.getNumberOfEntriesInBacklog(true), 0); + } + + @Test + public void testCheckExpiryByLedgerClosureTimeWithAckUnclosedLedger() throws Throwable { + final String ledgerAndCursorName = "testCheckExpiryByLedgerClosureTimeWithAckUnclosedLedger"; + int maxTTLSeconds = 1; + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setMaxEntriesPerLedger(5); + ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open(ledgerAndCursorName, config); + ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor(ledgerAndCursorName); + // set client clock to 10 days later + long incorrectPublishTimestamp = System.currentTimeMillis() + TimeUnit.DAYS.toMillis(10); + for (int i = 0; i < 7; i++) { + ledger.addEntry(createMessageWrittenToLedger("msg" + i, incorrectPublishTimestamp)); + } + assertEquals(ledger.getLedgersInfoAsList().size(), 2); + PersistentTopic mock = mock(PersistentTopic.class); + when(mock.getName()).thenReturn("topicname"); + when(mock.getLastPosition()).thenReturn(PositionFactory.EARLIEST); + PersistentMessageExpiryMonitor monitor = new PersistentMessageExpiryMonitor(mock, c1.getName(), c1, null); + AsyncCallbacks.MarkDeleteCallback markDeleteCallback = + (AsyncCallbacks.MarkDeleteCallback) spy( + FieldUtils.readDeclaredField(monitor, "markDeleteCallback", true)); + FieldUtils.writeField(monitor, "markDeleteCallback", markDeleteCallback, true); + + AtomicReference throwableAtomicReference = new AtomicReference<>(); + Mockito.doAnswer(invocation -> { + ManagedLedgerException argument = invocation.getArgument(0, ManagedLedgerException.class); + throwableAtomicReference.set(argument); + return invocation.callRealMethod(); + }).when(markDeleteCallback).markDeleteFailed(any(), any()); + + Position position = ledger.getLastConfirmedEntry(); + c1.markDelete(position); + Thread.sleep(TimeUnit.SECONDS.toMillis(maxTTLSeconds)); + monitor.expireMessages(maxTTLSeconds); + assertEquals(c1.getNumberOfEntriesInBacklog(true), 0); + + Assert.assertNull(throwableAtomicReference.get()); + } + @Test void testMessageExpiryWithPosition() throws Exception { final String ledgerAndCursorName = "testPersistentMessageExpiryWithPositionNonRecoverableLedgers"; @@ -411,54 +520,55 @@ void testMessageExpiryWithPosition() throws Exception { ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor(ledgerAndCursorName); PersistentSubscription subscription = mock(PersistentSubscription.class); - Topic topic = mock(Topic.class); + PersistentTopic topic = mock(PersistentTopic.class); when(subscription.getTopic()).thenReturn(topic); + when(topic.getName()).thenReturn("topicname"); for (int i = 0; i < totalEntries; i++) { positions.add(ledger.addEntry(createMessageWrittenToLedger("msg" + i))); } when(topic.getLastPosition()).thenReturn(positions.get(positions.size() - 1)); - PersistentMessageExpiryMonitor monitor = spy(new PersistentMessageExpiryMonitor("topicname", + PersistentMessageExpiryMonitor monitor = spy(new PersistentMessageExpiryMonitor(topic, cursor.getName(), cursor, subscription)); - assertEquals(cursor.getMarkDeletedPosition(), PositionImpl.get(positions.get(0).getLedgerId(), -1)); + assertEquals(cursor.getMarkDeletedPosition(), PositionFactory.create(positions.get(0).getLedgerId(), -1)); boolean issued; // Expire by position and verify mark delete position of cursor. issued = monitor.expireMessages(positions.get(15)); Awaitility.await().untilAsserted(() -> verify(monitor, times(1)).findEntryComplete(any(), any())); - assertEquals(cursor.getMarkDeletedPosition(), PositionImpl.get(positions.get(15).getLedgerId(), positions.get(15).getEntryId())); + assertEquals(cursor.getMarkDeletedPosition(), PositionFactory.create(positions.get(15).getLedgerId(), positions.get(15).getEntryId())); assertTrue(issued); clearInvocations(monitor); // Expire by position beyond last position and nothing should happen. - issued = monitor.expireMessages(PositionImpl.get(100, 100)); - assertEquals(cursor.getMarkDeletedPosition(), PositionImpl.get(positions.get(15).getLedgerId(), positions.get(15).getEntryId())); + issued = monitor.expireMessages(PositionFactory.create(100, 100)); + assertEquals(cursor.getMarkDeletedPosition(), PositionFactory.create(positions.get(15).getLedgerId(), positions.get(15).getEntryId())); assertFalse(issued); // Expire by position again and verify mark delete position of cursor didn't change. issued = monitor.expireMessages(positions.get(15)); Awaitility.await().untilAsserted(() -> verify(monitor, times(1)).findEntryComplete(any(), any())); - assertEquals(cursor.getMarkDeletedPosition(), PositionImpl.get(positions.get(15).getLedgerId(), positions.get(15).getEntryId())); + assertEquals(cursor.getMarkDeletedPosition(), PositionFactory.create(positions.get(15).getLedgerId(), positions.get(15).getEntryId())); assertTrue(issued); clearInvocations(monitor); // Expire by position before current mark delete position and verify mark delete position of cursor didn't change. issued = monitor.expireMessages(positions.get(10)); Awaitility.await().untilAsserted(() -> verify(monitor, times(1)).findEntryComplete(any(), any())); - assertEquals(cursor.getMarkDeletedPosition(), PositionImpl.get(positions.get(15).getLedgerId(), positions.get(15).getEntryId())); + assertEquals(cursor.getMarkDeletedPosition(), PositionFactory.create(positions.get(15).getLedgerId(), positions.get(15).getEntryId())); assertTrue(issued); clearInvocations(monitor); // Expire by position after current mark delete position and verify mark delete position of cursor move to new position. issued = monitor.expireMessages(positions.get(16)); Awaitility.await().untilAsserted(() -> verify(monitor, times(1)).findEntryComplete(any(), any())); - assertEquals(cursor.getMarkDeletedPosition(), PositionImpl.get(positions.get(16).getLedgerId(), positions.get(16).getEntryId())); + assertEquals(cursor.getMarkDeletedPosition(), PositionFactory.create(positions.get(16).getLedgerId(), positions.get(16).getEntryId())); assertTrue(issued); clearInvocations(monitor); ManagedCursorImpl mockCursor = mock(ManagedCursorImpl.class); - PersistentMessageExpiryMonitor mockMonitor = spy(new PersistentMessageExpiryMonitor("topicname", + PersistentMessageExpiryMonitor mockMonitor = spy(new PersistentMessageExpiryMonitor(topic, cursor.getName(), mockCursor, subscription)); // Not calling findEntryComplete to clear expirationCheckInProgress condition, so following call to // expire message shouldn't issue. diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java index 6ce6de1916f87..85e0887465db2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java @@ -53,7 +53,6 @@ import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.BeforeMethod; @@ -119,6 +118,7 @@ public void setup(Method m) throws Exception { public void testConcurrentTopicAndSubscriptionDelete() throws Exception { // create topic final PersistentTopic topic = (PersistentTopic) brokerService.getOrCreateTopic(successTopicName).get(); + topic.initialize().join(); CommandSubscribe cmd = new CommandSubscribe() .setConsumerId(1) .setTopic(successTopicName) @@ -153,7 +153,7 @@ public void testConcurrentTopicAndSubscriptionDelete() throws Exception { try { barrier.await(); // do subscription delete - ConcurrentOpenHashMap subscriptions = topic.getSubscriptions(); + final var subscriptions = topic.getSubscriptions(); PersistentSubscription ps = subscriptions.get(successSubName); // Thread.sleep(2,0); log.info("unsubscriber outcome is {}", ps.doUnsubscribe(ps.getConsumers().get(0)).get()); @@ -177,6 +177,7 @@ public void testConcurrentTopicAndSubscriptionDelete() throws Exception { public void testConcurrentTopicGCAndSubscriptionDelete() throws Exception { // create topic final PersistentTopic topic = (PersistentTopic) brokerService.getOrCreateTopic(successTopicName).get(); + topic.initialize().join(); CommandSubscribe cmd = new CommandSubscribe() .setConsumerId(1) .setTopic(successTopicName) @@ -217,7 +218,7 @@ public void testConcurrentTopicGCAndSubscriptionDelete() throws Exception { try { barrier.await(); // do subscription delete - ConcurrentOpenHashMap subscriptions = topic.getSubscriptions(); + final var subscriptions = topic.getSubscriptions(); PersistentSubscription ps = subscriptions.get(successSubName); // Thread.sleep(2,0); log.info("unsubscriber outcome is {}", ps.doUnsubscribe(ps.getConsumers().get(0)).get()); @@ -241,6 +242,7 @@ public void testConcurrentTopicGCAndSubscriptionDelete() throws Exception { public void testConcurrentTopicDeleteAndUnsubscribe() throws Exception { // create topic final PersistentTopic topic = (PersistentTopic) brokerService.getOrCreateTopic(successTopicName).get(); + topic.initialize().join(); CommandSubscribe cmd = new CommandSubscribe() .setConsumerId(1) .setTopic(successTopicName) @@ -275,7 +277,7 @@ public void testConcurrentTopicDeleteAndUnsubscribe() throws Exception { barrier.await(); // Thread.sleep(2,0); // assertTrue(topic.unsubscribe(successSubName).isDone()); - ConcurrentOpenHashMap subscriptions = topic.getSubscriptions(); + final var subscriptions = topic.getSubscriptions(); PersistentSubscription ps = subscriptions.get(successSubName); log.info("unsubscribe result : {}", topic.unsubscribe(successSubName).get()); log.info("closing consumer.."); @@ -299,6 +301,7 @@ public void testConcurrentTopicDeleteAndUnsubscribe() throws Exception { public void testConcurrentTopicDeleteAndSubsUnsubscribe() throws Exception { // create topic final PersistentTopic topic = (PersistentTopic) brokerService.getOrCreateTopic(successTopicName).get(); + topic.initialize().join(); CommandSubscribe cmd = new CommandSubscribe() .setConsumerId(1) .setTopic(successTopicName) @@ -335,7 +338,7 @@ public void testConcurrentTopicDeleteAndSubsUnsubscribe() throws Exception { log.info("&&&&&&&&& UNSUBSCRIBER TH"); // Thread.sleep(2,0); // assertTrue(topic.unsubscribe(successSubName).isDone()); - ConcurrentOpenHashMap subscriptions = topic.getSubscriptions(); + final var subscriptions = topic.getSubscriptions(); PersistentSubscription ps = subscriptions.get(successSubName); log.info("unsubscribe result : " + ps.doUnsubscribe(ps.getConsumers().get(0)).get()); } catch (Exception e) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicE2ETest.java index 63f80911ae62b..36e741f8fa9cd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicE2ETest.java @@ -1499,7 +1499,7 @@ public void testBrokerTopicStats() throws Exception { } Metrics metric = null; - // sleep 1 sec to caclulate metrics per second + // sleep 1 sec to calculate metrics per second Thread.sleep(1000); brokerService.updateRates(); List metrics = brokerService.getTopicMetrics(); @@ -1607,6 +1607,38 @@ public void testBrokerConnectionStats() throws Exception { assertEquals((long) map.get("brk_connection_create_fail_count"), 1); } + /** + * There is detailed info about this test. + * see: https://github.com/apache/pulsar/issues/10150#issuecomment-1112380074 + */ + @Test + public void testBrokerHealthCheckStatus() throws Exception { + + cleanup(); + conf.setSystemTopicEnabled(false); + conf.setTopicLevelPoliciesEnabled(false); + conf.setHealthCheckMetricsUpdateTimeInSeconds(60); + setup(); + BrokerService brokerService = this.pulsar.getBrokerService(); + + Map map = null; + + brokerService.checkHealth().get(); + brokerService.updateRates(); + Awaitility.await().until(() -> this.activeCount.get() == 1); + List metrics = brokerService.getTopicMetrics(); + System.out.println(metrics); + + for (int i = 0; i < metrics.size(); i++) { + if (metrics.get(i).getDimensions().containsValue("broker_health")) { + map = metrics.get(i).getMetrics(); + break; + } + } + assertNotNull(map); + assertEquals(map.get("brk_health"), 1); + } + @Test public void testPayloadCorruptionDetection() throws Exception { final String topicName = "persistent://prop/ns-abc/topic1"; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicInitializeDelayTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicInitializeDelayTest.java new file mode 100644 index 0000000000000..a563077e012da --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicInitializeDelayTest.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.Policies; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +@Test(groups = "broker") +@Slf4j +public class PersistentTopicInitializeDelayTest extends BrokerTestBase { + + @BeforeMethod + @Override + protected void setup() throws Exception { + conf.setTopicFactoryClassName(MyTopicFactory.class.getName()); + conf.setAllowAutoTopicCreation(true); + conf.setManagedLedgerMaxEntriesPerLedger(1); + conf.setBrokerDeleteInactiveTopicsEnabled(false); + conf.setTransactionCoordinatorEnabled(false); + conf.setTopicLoadTimeoutSeconds(30); + super.baseSetup(); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test(timeOut = 30 * 1000) + public void testTopicInitializeDelay() throws Exception { + admin.tenants().createTenant("public", TenantInfo.builder().allowedClusters(Set.of(configClusterName)).build()); + String namespace = "public/initialize-delay"; + admin.namespaces().createNamespace(namespace); + final String topicName = "persistent://" + namespace + "/testTopicInitializeDelay"; + admin.topics().createNonPartitionedTopic(topicName); + + admin.topicPolicies().setMaxConsumers(topicName, 10); + Awaitility.await().untilAsserted(() -> assertEquals(admin.topicPolicies().getMaxConsumers(topicName), 10)); + admin.topics().unload(topicName); + CompletableFuture> optionalFuture = pulsar.getBrokerService().getTopic(topicName, true); + + Optional topic = optionalFuture.get(15, TimeUnit.SECONDS); + assertTrue(topic.isPresent()); + } + + public static class MyTopicFactory implements TopicFactory { + @Override + public T create(String topic, ManagedLedger ledger, BrokerService brokerService, + Class topicClazz) { + try { + if (topicClazz == NonPersistentTopic.class) { + return (T) new NonPersistentTopic(topic, brokerService); + } else { + return (T) new MyPersistentTopic(topic, ledger, brokerService); + } + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + @Override + public void close() throws IOException { + // No-op + } + } + + public static class MyPersistentTopic extends PersistentTopic { + + private static AtomicInteger checkReplicationInvocationCount = new AtomicInteger(0); + + public MyPersistentTopic(String topic, ManagedLedger ledger, BrokerService brokerService) { + super(topic, ledger, brokerService); + SystemTopicBasedTopicPoliciesService topicPoliciesService = + (SystemTopicBasedTopicPoliciesService) brokerService.getPulsar().getTopicPoliciesService(); + if (topicPoliciesService.getListeners().containsKey(TopicName.get(topic)) ) { + brokerService.getPulsar().getTopicPoliciesService().getTopicPoliciesAsync(TopicName.get(topic), + TopicPoliciesService.GetType.DEFAULT + ).thenAccept(optionalPolicies -> optionalPolicies.ifPresent(this::onUpdate)); + } + } + + protected void updateTopicPolicyByNamespacePolicy(Policies namespacePolicies) { + try { + Thread.sleep(10 * 1000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + super.updateTopicPolicyByNamespacePolicy(namespacePolicies); + } + + public CompletableFuture checkReplication() { + if (TopicName.get(topic).getLocalName().equalsIgnoreCase("testTopicInitializeDelay")) { + checkReplicationInvocationCount.incrementAndGet(); + log.info("checkReplication, count = {}", checkReplicationInvocationCount.get()); + List configuredClusters = topicPolicies.getReplicationClusters().get(); + if (!(configuredClusters.size() == 1 && configuredClusters.contains(brokerService.pulsar().getConfiguration().getClusterName()))) { + try { + // this will cause the get topic timeout. + Thread.sleep(8 * 1000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new RuntimeException("checkReplication error"); + } + } + return super.checkReplication(); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index 45ef58bb7038c..81c12df4f3918 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -21,8 +21,10 @@ import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgsRecordingInvocations; import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.atLeast; @@ -46,12 +48,14 @@ import io.netty.buffer.Unpooled; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.DefaultEventLoop; +import io.netty.channel.DefaultEventLoopGroup; +import io.netty.channel.EventLoopGroup; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.URL; +import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -60,6 +64,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutionException; @@ -86,25 +91,26 @@ import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.bookkeeper.test.MockedBookKeeperTestCase; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.namespace.NamespaceService; -import org.apache.pulsar.broker.service.persistent.CompactorSubscription; import org.apache.pulsar.broker.service.persistent.GeoPersistentReplicator; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherSingleActiveConsumer; import org.apache.pulsar.broker.service.persistent.PersistentReplicator; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.service.persistent.PulsarCompactorSubscription; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.ProducerBuilderImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; @@ -127,10 +133,11 @@ import org.apache.pulsar.common.protocol.schema.SchemaVersion; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.compaction.CompactedTopic; import org.apache.pulsar.compaction.CompactedTopicContext; import org.apache.pulsar.compaction.Compactor; +import org.apache.pulsar.compaction.CompactorMXBean; +import org.apache.pulsar.compaction.PulsarCompactionServiceFactory; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.impl.FaultInjectionMetadataStore; import org.awaitility.Awaitility; @@ -161,6 +168,7 @@ public class PersistentTopicTest extends MockedBookKeeperTestCase { private BrokerService brokerService; + private EventLoopGroup eventLoopGroup; @BeforeMethod(alwaysRun = true) public void setup() throws Exception { @@ -172,11 +180,13 @@ public void setup() throws Exception { svcConfig.setClusterName("pulsar-cluster"); svcConfig.setTopicLevelPoliciesEnabled(false); svcConfig.setSystemTopicEnabled(false); + Compactor compactor = mock(Compactor.class); + when(compactor.getStats()).thenReturn(mock(CompactorMXBean.class)); pulsarTestContext = PulsarTestContext.builderForNonStartableContext() .config(svcConfig) .spyByDefault() .useTestPulsarResources(metadataStore) - .compactor(mock(Compactor.class)) + .compactor(compactor) .build(); brokerService = pulsarTestContext.getBrokerService(); @@ -197,14 +207,16 @@ public void setup() throws Exception { .when(serverCnx).getCommandSender(); ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); Channel channel = mock(Channel.class); - doReturn(spy(DefaultEventLoop.class)).when(channel).eventLoop(); + + eventLoopGroup = new DefaultEventLoopGroup(); + doReturn(eventLoopGroup.next()).when(channel).eventLoop(); doReturn(channel).when(ctx).channel(); doReturn(ctx).when(serverCnx).ctx(); + doReturn(CompletableFuture.completedFuture(Optional.of(true))).when(serverCnx).checkConnectionLiveness(); - NamespaceService nsSvc = mock(NamespaceService.class); + NamespaceService nsSvc = pulsarTestContext.getPulsarService().getNamespaceService(); NamespaceBundle bundle = mock(NamespaceBundle.class); doReturn(CompletableFuture.completedFuture(bundle)).when(nsSvc).getBundleAsync(any()); - doReturn(nsSvc).when(pulsarTestContext.getPulsarService()).getNamespaceService(); doReturn(true).when(nsSvc).isServiceUnitOwned(any()); doReturn(true).when(nsSvc).isServiceUnitActive(any()); doReturn(CompletableFuture.completedFuture(true)).when(nsSvc).isServiceUnitActiveAsync(any()); @@ -219,11 +231,16 @@ public void teardown() throws Exception { pulsarTestContext.close(); pulsarTestContext = null; } + if (eventLoopGroup != null) { + eventLoopGroup.shutdownNow(); + eventLoopGroup = null; + } } @Test public void testCreateTopic() { final ManagedLedger ledgerMock = mock(ManagedLedger.class); + doReturn(new ManagedLedgerConfig()).when(ledgerMock).getConfig(); doReturn(new ArrayList<>()).when(ledgerMock).getCursors(); final String topicName = "persistent://prop/use/ns-abc/topic1"; @@ -278,13 +295,15 @@ public void testPublishMessage() throws Exception { doAnswer(invocationOnMock -> { final ByteBuf payload = (ByteBuf) invocationOnMock.getArguments()[0]; - final AddEntryCallback callback = (AddEntryCallback) invocationOnMock.getArguments()[1]; - final Topic.PublishContext ctx = (Topic.PublishContext) invocationOnMock.getArguments()[2]; - callback.addComplete(PositionImpl.LATEST, payload, ctx); + final AddEntryCallback callback = (AddEntryCallback) invocationOnMock.getArguments()[2]; + final Topic.PublishContext ctx = (Topic.PublishContext) invocationOnMock.getArguments()[3]; + callback.addComplete(PositionFactory.LATEST, payload, ctx); return null; - }).when(ledgerMock).asyncAddEntry(any(ByteBuf.class), any(AddEntryCallback.class), any()); + }).when(ledgerMock).asyncAddEntry(any(ByteBuf.class), anyInt(), any(AddEntryCallback.class), any()); PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService); + long lastMaxReadPositionMovedForwardTimestamp = topic.getLastMaxReadPositionMovedForwardTimestamp(); + /* * MessageMetadata.Builder messageMetadata = MessageMetadata.newBuilder(); * messageMetadata.setPublishTime(System.currentTimeMillis()); messageMetadata.setProducerName("producer-name"); @@ -297,8 +316,8 @@ public void testPublishMessage() throws Exception { final Topic.PublishContext publishContext = new Topic.PublishContext() { @Override public void completed(Exception e, long ledgerId, long entryId) { - assertEquals(ledgerId, PositionImpl.LATEST.getLedgerId()); - assertEquals(entryId, PositionImpl.LATEST.getEntryId()); + assertEquals(ledgerId, PositionFactory.LATEST.getLedgerId()); + assertEquals(entryId, PositionFactory.LATEST.getEntryId()); latch.countDown(); } @@ -309,10 +328,10 @@ public void setMetadataFromEntryData(ByteBuf entryData) { assertEquals(entryData.array(), payload.array()); } }; - topic.publishMessage(payload, publishContext); assertTrue(latch.await(1, TimeUnit.SECONDS)); + assertTrue(topic.getLastMaxReadPositionMovedForwardTimestamp() > lastMaxReadPositionMovedForwardTimestamp); } @Test @@ -349,6 +368,7 @@ public void testPublishMessageMLFailure() throws Exception { final String successTopicName = "persistent://prop/use/ns-abc/successTopic"; final ManagedLedger ledgerMock = mock(ManagedLedger.class); + doReturn(new ManagedLedgerConfig()).when(ledgerMock).getConfig(); doReturn(new ArrayList<>()).when(ledgerMock).getCursors(); PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService); @@ -358,10 +378,10 @@ public void testPublishMessageMLFailure() throws Exception { // override asyncAddEntry callback to return error doAnswer((Answer) invocationOnMock -> { - ((AddEntryCallback) invocationOnMock.getArguments()[1]).addFailed( - new ManagedLedgerException("Managed ledger failure"), invocationOnMock.getArguments()[2]); + ((AddEntryCallback) invocationOnMock.getArguments()[2]).addFailed( + new ManagedLedgerException("Managed ledger failure"), invocationOnMock.getArguments()[3]); return null; - }).when(ledgerMock).asyncAddEntry(any(ByteBuf.class), any(AddEntryCallback.class), any()); + }).when(ledgerMock).asyncAddEntry(any(ByteBuf.class), anyInt(), any(AddEntryCallback.class), any()); topic.publishMessage(payload, (exception, ledgerId, entryId) -> { if (exception == null) { @@ -494,51 +514,6 @@ public void testProducerOverwrite() { topic.getProducers().values().forEach(producer -> Assert.assertEquals(producer.getEpoch(), 3)); } - private void testMaxProducers() { - PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService); - topic.initialize().join(); - String role = "appid1"; - // 1. add producer1 - Producer producer = new Producer(topic, serverCnx, 1 /* producer id */, "prod-name1", role, - false, null, SchemaVersion.Latest, 0, false, ProducerAccessMode.Shared, Optional.empty(), true); - topic.addProducer(producer, new CompletableFuture<>()); - assertEquals(topic.getProducers().size(), 1); - - // 2. add producer2 - Producer producer2 = new Producer(topic, serverCnx, 2 /* producer id */, "prod-name2", role, - false, null, SchemaVersion.Latest, 0, false, ProducerAccessMode.Shared, Optional.empty(), true); - topic.addProducer(producer2, new CompletableFuture<>()); - assertEquals(topic.getProducers().size(), 2); - - // 3. add producer3 but reached maxProducersPerTopic - try { - Producer producer3 = new Producer(topic, serverCnx, 3 /* producer id */, "prod-name3", role, - false, null, SchemaVersion.Latest, 0, false, ProducerAccessMode.Shared, Optional.empty(), true); - topic.addProducer(producer3, new CompletableFuture<>()).join(); - fail("should have failed"); - } catch (Exception e) { - assertEquals(e.getCause().getClass(), BrokerServiceException.ProducerBusyException.class); - } - } - - @Test - public void testMaxProducersForBroker() { - // set max clients - pulsarTestContext.getConfig().setMaxProducersPerTopic(2); - testMaxProducers(); - } - - @Test - public void testMaxProducersForNamespace() throws Exception { - // set max clients - Policies policies = new Policies(); - policies.max_producers_per_topic = 2; - pulsarTestContext.getPulsarResources().getNamespaceResources() - .createPolicies(TopicName.get(successTopicName).getNamespaceObject(), - policies); - testMaxProducers(); - } - private Producer getMockedProducerWithSpecificAddress(Topic topic, long producerId, InetAddress address) { final String producerNameBase = "producer"; final String role = "appid1"; @@ -627,6 +602,7 @@ public void testMaxSameAddressProducers() throws Exception { @Test public void testSubscribeFail() throws Exception { PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService); + topic.initialize().join(); // Empty subscription name CommandSubscribe cmd = new CommandSubscribe() @@ -663,6 +639,7 @@ private SubscriptionOption getSubscriptionOption(CommandSubscribe cmd) { @Test public void testSubscribeUnsubscribe() throws Exception { PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService); + topic.initialize().join(); CommandSubscribe cmd = new CommandSubscribe() .setConsumerId(1) @@ -678,7 +655,15 @@ public void testSubscribeUnsubscribe() throws Exception { f1.get(); // 2. duplicate subscribe - Future f2 = topic.subscribe(getSubscriptionOption(cmd)); + CommandSubscribe cmd2 = new CommandSubscribe() + .setConsumerId(2) + .setTopic(successTopicName) + .setSubscription(successSubName) + .setConsumerName("consumer-name") + .setReadCompacted(false) + .setRequestId(2) + .setSubType(SubType.Exclusive); + Future f2 = topic.subscribe(getSubscriptionOption(cmd2)); try { f2.get(); fail("should fail with exception"); @@ -743,19 +728,11 @@ public void testAddRemoveConsumer() throws Exception { sub.addConsumer(consumer); assertTrue(sub.getDispatcher().isConsumerConnected()); - // 2. duplicate add consumer - try { - sub.addConsumer(consumer).get(); - fail("Should fail with ConsumerBusyException"); - } catch (Exception e) { - assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerBusyException); - } - - // 3. simple remove consumer + // 2. simple remove consumer sub.removeConsumer(consumer); assertFalse(sub.getDispatcher().isConsumerConnected()); - // 4. duplicate remove consumer + // 3. duplicate remove consumer try { sub.removeConsumer(consumer); fail("Should fail with ServerMetadataException"); @@ -800,11 +777,7 @@ private void testMaxConsumersShared() throws Exception { addConsumerToSubscription.setAccessible(true); // for count consumers on topic - ConcurrentOpenHashMap subscriptions = - ConcurrentOpenHashMap.newBuilder() - .expectedItems(16) - .concurrencyLevel(1) - .build(); + final var subscriptions = new ConcurrentHashMap(); subscriptions.put("sub-1", sub); subscriptions.put("sub-2", sub2); Field field = topic.getClass().getDeclaredField("subscriptions"); @@ -896,11 +869,7 @@ private void testMaxConsumersFailover() throws Exception { addConsumerToSubscription.setAccessible(true); // for count consumers on topic - ConcurrentOpenHashMap subscriptions = - ConcurrentOpenHashMap.newBuilder() - .expectedItems(16) - .concurrencyLevel(1) - .build(); + final var subscriptions = new ConcurrentHashMap(); subscriptions.put("sub-1", sub); subscriptions.put("sub-2", sub2); Field field = topic.getClass().getDeclaredField("subscriptions"); @@ -1015,11 +984,7 @@ public void testMaxSameAddressConsumers() throws Exception { addConsumerToSubscription.setAccessible(true); // for count consumers on topic - ConcurrentOpenHashMap subscriptions = - ConcurrentOpenHashMap.newBuilder() - .expectedItems(16) - .concurrencyLevel(1) - .build(); + final var subscriptions = new ConcurrentHashMap(); subscriptions.put("sub1", sub1); subscriptions.put("sub2", sub2); Field field = topic.getClass().getDeclaredField("subscriptions"); @@ -1235,6 +1200,7 @@ public void testDeleteTopic() throws Exception { public void testDeleteAndUnsubscribeTopic() throws Exception { // create topic final PersistentTopic topic = (PersistentTopic) brokerService.getOrCreateTopic(successTopicName).get(); + topic.initialize().join(); CommandSubscribe cmd = new CommandSubscribe() .setConsumerId(1) .setTopic(successTopicName) @@ -1321,7 +1287,7 @@ public void testConcurrentTopicAndSubscriptionDelete() throws Exception { try { barrier.await(); // do subscription delete - ConcurrentOpenHashMap subscriptions = topic.getSubscriptions(); + final var subscriptions = topic.getSubscriptions(); PersistentSubscription ps = subscriptions.get(successSubName); // Thread.sleep(5,0); log.info("unsubscriber outcome is {}", ps.doUnsubscribe(ps.getConsumers().get(0)).get()); @@ -1344,6 +1310,7 @@ public void testConcurrentTopicAndSubscriptionDelete() throws Exception { @Test public void testDeleteTopicRaceConditions() throws Exception { PersistentTopic topic = (PersistentTopic) brokerService.getOrCreateTopic(successTopicName).get(); + topic.initialize().join(); // override ledger deletion callback to slow down deletion doAnswer(invocationOnMock -> { @@ -1398,6 +1365,7 @@ void setupMLAsyncCallbackMocks() { final CompletableFuture closeFuture = new CompletableFuture<>(); doReturn(new ArrayList<>()).when(ledgerMock).getCursors(); + doReturn(new ManagedLedgerConfig()).when(ledgerMock).getConfig(); doReturn("mockCursor").when(cursorMock).getName(); doReturn(true).when(cursorMock).isDurable(); // doNothing().when(cursorMock).asyncClose(new CloseCallback() { @@ -1442,11 +1410,11 @@ public void closeFailed(ManagedLedgerException exception, Object ctx) { // call addComplete on ledger asyncAddEntry doAnswer(invocationOnMock -> { - ((AddEntryCallback) invocationOnMock.getArguments()[1]).addComplete(new PositionImpl(1, 1), + ((AddEntryCallback) invocationOnMock.getArguments()[2]).addComplete(PositionFactory.create(1, 1), null, - invocationOnMock.getArguments()[2]); + invocationOnMock.getArguments()[3]); return null; - }).when(ledgerMock).asyncAddEntry(any(ByteBuf.class), any(AddEntryCallback.class), any()); + }).when(ledgerMock).asyncAddEntry(any(ByteBuf.class), anyInt(), any(AddEntryCallback.class), any()); // call openCursorComplete on cursor asyncOpen doAnswer(invocationOnMock -> { @@ -1534,6 +1502,7 @@ public void testFailoverSubscription() throws Exception { .setSubType(SubType.Failover); // 1. Subscribe with non partition topic + topic1.initialize().join(); Future f1 = topic1.subscribe(getSubscriptionOption(cmd1)); f1.get(); @@ -1549,6 +1518,7 @@ public void testFailoverSubscription() throws Exception { .setRequestId(1) .setSubType(SubType.Failover); + topic2.initialize(); Future f2 = topic2.subscribe(getSubscriptionOption(cmd2)); f2.get(); @@ -1693,16 +1663,19 @@ public void testAtomicReplicationRemoval() throws Exception { String remoteCluster = "remote"; final ManagedLedger ledgerMock = mock(ManagedLedger.class); doNothing().when(ledgerMock).asyncDeleteCursor(any(), any(), any()); + doReturn(new ManagedLedgerConfig()).when(ledgerMock).getConfig(); doReturn(new ArrayList<>()).when(ledgerMock).getCursors(); PersistentTopic topic = new PersistentTopic(globalTopicName, ledgerMock, brokerService); topic.initialize().join(); String remoteReplicatorName = topic.getReplicatorPrefix() + "." + remoteCluster; - ConcurrentOpenHashMap replicatorMap = topic.getReplicators(); + final var replicatorMap = topic.getReplicators(); ManagedCursor cursor = mock(ManagedCursorImpl.class); doReturn(remoteCluster).when(cursor).getName(); PulsarClientImpl pulsarClientMock = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(pulsarClientMock.getCnxPool()).thenReturn(connectionPool); when(pulsarClientMock.newProducer(any())).thenAnswer( invocation -> { ProducerBuilderImpl producerBuilder = @@ -1750,6 +1723,7 @@ public void testClosingReplicationProducerTwice() throws Exception { final ManagedLedger ledgerMock = mock(ManagedLedger.class); doNothing().when(ledgerMock).asyncDeleteCursor(any(), any(), any()); doReturn(new ArrayList<>()).when(ledgerMock).getCursors(); + doReturn(new ManagedLedgerConfig()).when(ledgerMock).getConfig(); PersistentTopic topic = new PersistentTopic(globalTopicName, ledgerMock, brokerService); @@ -1778,12 +1752,12 @@ public void testClosingReplicationProducerTwice() throws Exception { any(), eq(null) ); - replicator.disconnect(false); - replicator.disconnect(false); + replicator.terminate(); + replicator.terminate(); replicator.startProducer(); - verify(clientImpl, Mockito.times(2)).createProducerAsync(any(), any(), any()); + verify(clientImpl, Mockito.times(1)).createProducerAsync(any(), any(), any()); } @Test @@ -1792,10 +1766,10 @@ public void testCompactorSubscription() { CompactedTopic compactedTopic = mock(CompactedTopic.class); when(compactedTopic.newCompactedLedger(any(Position.class), anyLong())) .thenReturn(CompletableFuture.completedFuture(mock(CompactedTopicContext.class))); - PersistentSubscription sub = new CompactorSubscription(topic, compactedTopic, + PersistentSubscription sub = new PulsarCompactorSubscription(topic, compactedTopic, Compactor.COMPACTION_SUBSCRIPTION, cursorMock); - PositionImpl position = new PositionImpl(1, 1); + Position position = PositionFactory.create(1, 1); long ledgerId = 0xc0bfefeL; sub.acknowledgeMessage(Collections.singletonList(position), AckType.Cumulative, Map.of(Compactor.COMPACTED_TOPIC_LEDGER_PROPERTY, ledgerId)); @@ -1807,7 +1781,7 @@ public void testCompactorSubscription() { public void testCompactorSubscriptionUpdatedOnInit() { long ledgerId = 0xc0bfefeL; Map properties = Map.of(Compactor.COMPACTED_TOPIC_LEDGER_PROPERTY, ledgerId); - PositionImpl position = new PositionImpl(1, 1); + Position position = PositionFactory.create(1, 1); doAnswer((invokactionOnMock) -> properties).when(cursorMock).getProperties(); doAnswer((invokactionOnMock) -> position).when(cursorMock).getMarkDeletedPosition(); @@ -1816,14 +1790,15 @@ public void testCompactorSubscriptionUpdatedOnInit() { CompactedTopic compactedTopic = mock(CompactedTopic.class); when(compactedTopic.newCompactedLedger(any(Position.class), anyLong())) .thenReturn(CompletableFuture.completedFuture(null)); - new CompactorSubscription(topic, compactedTopic, Compactor.COMPACTION_SUBSCRIPTION, cursorMock); + new PulsarCompactorSubscription(topic, compactedTopic, Compactor.COMPACTION_SUBSCRIPTION, cursorMock); verify(compactedTopic, Mockito.times(1)).newCompactedLedger(position, ledgerId); } @Test public void testCompactionTriggeredAfterThresholdFirstInvocation() throws Exception { CompletableFuture compactPromise = new CompletableFuture<>(); - Compactor compactor = pulsarTestContext.getPulsarService().getCompactor(); + Compactor compactor = ((PulsarCompactionServiceFactory) pulsarTestContext.getPulsarService() + .getCompactionServiceFactory()).getCompactor(); doReturn(compactPromise).when(compactor).compact(anyString()); Policies policies = new Policies(); @@ -1854,7 +1829,8 @@ public void testCompactionTriggeredAfterThresholdFirstInvocation() throws Except @Test public void testCompactionTriggeredAfterThresholdSecondInvocation() throws Exception { CompletableFuture compactPromise = new CompletableFuture<>(); - Compactor compactor = pulsarTestContext.getPulsarService().getCompactor(); + Compactor compactor = ((PulsarCompactionServiceFactory) pulsarTestContext.getPulsarService() + .getCompactionServiceFactory()).getCompactor(); doReturn(compactPromise).when(compactor).compact(anyString()); ManagedCursor subCursor = mock(ManagedCursor.class); @@ -1888,7 +1864,8 @@ public void testCompactionTriggeredAfterThresholdSecondInvocation() throws Excep @Test public void testCompactionDisabledWithZeroThreshold() throws Exception { CompletableFuture compactPromise = new CompletableFuture<>(); - Compactor compactor = pulsarTestContext.getPulsarService().getCompactor(); + Compactor compactor = ((PulsarCompactionServiceFactory) pulsarTestContext.getPulsarService() + .getCompactionServiceFactory()).getCompactor(); doReturn(compactPromise).when(compactor).compact(anyString()); Policies policies = new Policies(); @@ -2029,11 +2006,7 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { public void testCheckInactiveSubscriptions() throws Exception { PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService); - ConcurrentOpenHashMap subscriptions = - ConcurrentOpenHashMap.newBuilder() - .expectedItems(16) - .concurrencyLevel(1) - .build(); + final var subscriptions = new ConcurrentHashMap(); // This subscription is connected by consumer. PersistentSubscription nonDeletableSubscription1 = spyWithClassAndConstructorArgsRecordingInvocations(PersistentSubscription.class, topic, @@ -2137,6 +2110,7 @@ public void testTopicCloseFencingTimeout() throws Exception { @Test public void testGetDurableSubscription() throws Exception { ManagedLedger mockLedger = mock(ManagedLedger.class); + doReturn(new ManagedLedgerConfig()).when(mockLedger).getConfig(); ManagedCursor mockCursor = mock(ManagedCursorImpl.class); Position mockPosition = mock(Position.class); doReturn("test").when(mockCursor).getName(); @@ -2158,6 +2132,7 @@ public void testGetDurableSubscription() throws Exception { return null; }).when(mockLedger).asyncOpenCursor(any(), any(), any(), any(), any(), any()); PersistentTopic topic = new PersistentTopic(successTopicName, mockLedger, brokerService); + topic.initialize().join(); CommandSubscribe cmd = new CommandSubscribe() .setConsumerId(1) @@ -2205,9 +2180,14 @@ public void testKeySharedMetadataExposedToStats() throws Exception { sub1.addConsumer(consumer1); consumer1.close(); - SubscriptionStatsImpl stats1 = sub1.getStats(false, false, false); - assertEquals(stats1.keySharedMode, "AUTO_SPLIT"); - assertFalse(stats1.allowOutOfOrderDelivery); + CompletableFuture stats1Async = + sub1.getStatsAsync(new GetStatsOptions(false, false, false, false, false)); + assertThat(stats1Async).succeedsWithin(Duration.ofSeconds(3)) + .matches(stats1 -> { + assertEquals(stats1.keySharedMode, "AUTO_SPLIT"); + assertFalse(stats1.allowOutOfOrderDelivery); + return true; + }); Consumer consumer2 = new Consumer(sub2, SubType.Key_Shared, topic.getName(), 2, 0, "Cons2", true, serverCnx, "myrole-1", Collections.emptyMap(), false, @@ -2216,9 +2196,14 @@ public void testKeySharedMetadataExposedToStats() throws Exception { sub2.addConsumer(consumer2); consumer2.close(); - SubscriptionStatsImpl stats2 = sub2.getStats(false, false, false); - assertEquals(stats2.keySharedMode, "AUTO_SPLIT"); - assertTrue(stats2.allowOutOfOrderDelivery); + CompletableFuture stats2Async = + sub2.getStatsAsync(new GetStatsOptions(false, false, false, false, false)); + assertThat(stats2Async).succeedsWithin(Duration.ofSeconds(3)) + .matches(stats2 -> { + assertEquals(stats2.keySharedMode, "AUTO_SPLIT"); + assertTrue(stats2.allowOutOfOrderDelivery); + return true; + }); KeySharedMeta ksm = new KeySharedMeta().setKeySharedMode(KeySharedMode.STICKY) .setAllowOutOfOrderDelivery(false); @@ -2228,9 +2213,13 @@ public void testKeySharedMetadataExposedToStats() throws Exception { sub3.addConsumer(consumer3); consumer3.close(); - SubscriptionStatsImpl stats3 = sub3.getStats(false, false, false); - assertEquals(stats3.keySharedMode, "STICKY"); - assertFalse(stats3.allowOutOfOrderDelivery); + CompletableFuture stats3Async = sub3.getStatsAsync(new GetStatsOptions(false, false, false, false, false)); + assertThat(stats3Async).succeedsWithin(Duration.ofSeconds(3)) + .matches(stats3 -> { + assertEquals(stats3.keySharedMode, "STICKY"); + assertFalse(stats3.allowOutOfOrderDelivery); + return true; + }); } private ByteBuf getMessageWithMetadata(byte[] data) { @@ -2274,7 +2263,8 @@ public void testGetReplicationClusters() throws MetadataStoreException { topicPolicies.setReplicationClusters(topicClusters); Optional optionalTopicPolicies = Optional.of(topicPolicies); topicPoliciesFuture.complete(optionalTopicPolicies); - when(topicPoliciesService.getTopicPoliciesIfExists(any())).thenReturn(topicPolicies); + when(topicPoliciesService.getTopicPoliciesAsync(any(), any())) + .thenReturn(CompletableFuture.completedFuture(Optional.of(topicPolicies))); topic = new PersistentTopic(successTopicName, ledgerMock, brokerService); topic.initialize().join(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisePublishLimiterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisePublishLimiterTest.java deleted file mode 100644 index 73cb43d52b112..0000000000000 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisePublishLimiterTest.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.service; - -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertTrue; -import org.apache.pulsar.common.policies.data.PublishRate; -import org.testng.annotations.Test; - -public class PrecisePublishLimiterTest { - - @Test - void shouldResetMsgLimitAfterUpdate() { - PrecisePublishLimiter precisePublishLimiter = new PrecisePublishLimiter(new PublishRate(), () -> { - }); - precisePublishLimiter.update(new PublishRate(1, 1)); - assertFalse(precisePublishLimiter.tryAcquire(99, 99)); - precisePublishLimiter.update(new PublishRate(-1, 100)); - assertTrue(precisePublishLimiter.tryAcquire(99, 99)); - } - - @Test - void shouldResetBytesLimitAfterUpdate() { - PrecisePublishLimiter precisePublishLimiter = new PrecisePublishLimiter(new PublishRate(), () -> { - }); - precisePublishLimiter.update(new PublishRate(1, 1)); - assertFalse(precisePublishLimiter.tryAcquire(99, 99)); - precisePublishLimiter.update(new PublishRate(100, -1)); - assertTrue(precisePublishLimiter.tryAcquire(99, 99)); - } - - @Test - void shouldCloseResources() throws Exception { - for (int i = 0; i < 20000; i++) { - PrecisePublishLimiter precisePublishLimiter = new PrecisePublishLimiter(new PublishRate(100, 100), () -> { - }); - precisePublishLimiter.tryAcquire(99, 99); - precisePublishLimiter.close(); - } - } -} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterDisableTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterDisableTest.java index 8dfed77814b35..ec952a7ca7734 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterDisableTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterDisableTest.java @@ -18,7 +18,10 @@ */ package org.apache.pulsar.broker.service; -import static org.testng.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import org.apache.pulsar.broker.qos.AsyncTokenBucket; import org.testng.annotations.Test; public class PublishRateLimiterDisableTest { @@ -26,7 +29,9 @@ public class PublishRateLimiterDisableTest { // GH issue #10603 @Test void shouldAlwaysAllowAcquire() { - PublishRateLimiterDisable publishRateLimiter = PublishRateLimiterDisable.DISABLED_RATE_LIMITER; - assertTrue(publishRateLimiter.tryAcquire(Integer.MAX_VALUE, Long.MAX_VALUE)); + PublishRateLimiter publishRateLimiter = new PublishRateLimiterImpl(AsyncTokenBucket.DEFAULT_SNAPSHOT_CLOCK); + Producer producer = mock(Producer.class); + publishRateLimiter.handlePublishThrottling(producer, Integer.MAX_VALUE, Long.MAX_VALUE); + verify(producer, never()).incrementThrottleCount(); } } \ No newline at end of file diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java index b934ced08c5db..2c44ba7e23004 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java @@ -16,145 +16,105 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.pulsar.broker.service; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import io.netty.channel.EventLoopGroup; +import java.util.HashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.PublishRate; -import org.apache.pulsar.common.util.RateLimiter; +import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.util.HashMap; -import java.util.concurrent.ScheduledFuture; - -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertTrue; - @Test(groups = "broker") public class PublishRateLimiterTest { private final String CLUSTER_NAME = "clusterName"; private final Policies policies = new Policies(); private final PublishRate publishRate = new PublishRate(10, 100); private final PublishRate newPublishRate = new PublishRate(20, 200); + private AtomicLong manualClockSource; - private PrecisePublishLimiter precisePublishLimiter; + private Producer producer; private PublishRateLimiterImpl publishRateLimiter; + private AtomicInteger throttleCount = new AtomicInteger(0); + @BeforeMethod public void setup() throws Exception { policies.publishMaxMessageRate = new HashMap<>(); policies.publishMaxMessageRate.put(CLUSTER_NAME, publishRate); + manualClockSource = new AtomicLong(TimeUnit.SECONDS.toNanos(100)); + publishRateLimiter = new PublishRateLimiterImpl(requestSnapshot -> manualClockSource.get()); + publishRateLimiter.update(policies, CLUSTER_NAME); + producer = mock(Producer.class); + throttleCount.set(0); + doAnswer(a -> { + throttleCount.incrementAndGet(); + return null; + }).when(producer).incrementThrottleCount(); + doAnswer(a -> { + throttleCount.decrementAndGet(); + return null; + }).when(producer).decrementThrottleCount(); + TransportCnx transportCnx = mock(TransportCnx.class); + when(producer.getCnx()).thenReturn(transportCnx); + BrokerService brokerService = mock(BrokerService.class); + when(transportCnx.getBrokerService()).thenReturn(brokerService); + EventLoopGroup eventLoopGroup = mock(EventLoopGroup.class); + when(brokerService.executor()).thenReturn(eventLoopGroup); + doReturn(null).when(eventLoopGroup).schedule(any(Runnable.class), anyLong(), any()); + incrementSeconds(1); + } + + @AfterMethod + public void cleanup() throws Exception { + policies.publishMaxMessageRate.clear(); + policies.publishMaxMessageRate = null; + } - precisePublishLimiter = new PrecisePublishLimiter(policies, CLUSTER_NAME, () -> System.out.print("Refresh permit")); - publishRateLimiter = new PublishRateLimiterImpl(policies, CLUSTER_NAME); + private void incrementSeconds(int seconds) { + manualClockSource.addAndGet(TimeUnit.SECONDS.toNanos(seconds)); } @Test public void testPublishRateLimiterImplExceed() throws Exception { // increment not exceed - publishRateLimiter.incrementPublishCount(5, 50); - publishRateLimiter.checkPublishRate(); - assertFalse(publishRateLimiter.isPublishRateExceeded()); - publishRateLimiter.resetPublishCount(); + publishRateLimiter.handlePublishThrottling(producer, 5, 50); + assertEquals(throttleCount.get(), 0); + + incrementSeconds(1); // numOfMessages increment exceeded - publishRateLimiter.incrementPublishCount(11, 100); - publishRateLimiter.checkPublishRate(); - assertTrue(publishRateLimiter.isPublishRateExceeded()); - publishRateLimiter.resetPublishCount(); + publishRateLimiter.handlePublishThrottling(producer, 11, 100); + assertEquals(throttleCount.get(), 1); - // msgSizeInBytes increment exceeded - publishRateLimiter.incrementPublishCount(9, 110); - publishRateLimiter.checkPublishRate(); - assertTrue(publishRateLimiter.isPublishRateExceeded()); + incrementSeconds(1); + // msgSizeInBytes increment exceeded + publishRateLimiter.handlePublishThrottling(producer, 9, 110); + assertEquals(throttleCount.get(), 2); } @Test public void testPublishRateLimiterImplUpdate() { - publishRateLimiter.incrementPublishCount(11, 110); - publishRateLimiter.checkPublishRate(); - assertTrue(publishRateLimiter.isPublishRateExceeded()); + publishRateLimiter.handlePublishThrottling(producer, 11, 110); + assertEquals(throttleCount.get(), 1); // update + throttleCount.set(0); publishRateLimiter.update(newPublishRate); - publishRateLimiter.incrementPublishCount(11, 110); - publishRateLimiter.checkPublishRate(); - assertFalse(publishRateLimiter.isPublishRateExceeded()); - - } - - @Test - public void testPrecisePublishRateLimiterUpdate() { - assertFalse(precisePublishLimiter.tryAcquire(15, 150)); - - //update - precisePublishLimiter.update(newPublishRate); - assertTrue(precisePublishLimiter.tryAcquire(15, 150)); - } - - @Test - public void testPrecisePublishRateLimiterAcquire() throws Exception { - Class precisePublishLimiterClass = Class.forName("org.apache.pulsar.broker.service.PrecisePublishLimiter"); - Field topicPublishRateLimiterOnMessageField = precisePublishLimiterClass.getDeclaredField("topicPublishRateLimiterOnMessage"); - Field topicPublishRateLimiterOnByteField = precisePublishLimiterClass.getDeclaredField("topicPublishRateLimiterOnByte"); - topicPublishRateLimiterOnMessageField.setAccessible(true); - topicPublishRateLimiterOnByteField.setAccessible(true); - - RateLimiter topicPublishRateLimiterOnMessage = (RateLimiter)topicPublishRateLimiterOnMessageField.get( - precisePublishLimiter); - RateLimiter topicPublishRateLimiterOnByte = (RateLimiter)topicPublishRateLimiterOnByteField.get( - precisePublishLimiter); - - Method renewTopicPublishRateLimiterOnMessageMethod = topicPublishRateLimiterOnMessage.getClass().getDeclaredMethod("renew", null); - Method renewTopicPublishRateLimiterOnByteMethod = topicPublishRateLimiterOnByte.getClass().getDeclaredMethod("renew", null); - renewTopicPublishRateLimiterOnMessageMethod.setAccessible(true); - renewTopicPublishRateLimiterOnByteMethod.setAccessible(true); - - // running tryAcquire in order to lazyInit the renewTask - precisePublishLimiter.tryAcquire(1, 10); - - Field onMessageRenewTaskField = topicPublishRateLimiterOnMessage.getClass().getDeclaredField("renewTask"); - Field onByteRenewTaskField = topicPublishRateLimiterOnByte.getClass().getDeclaredField("renewTask"); - onMessageRenewTaskField.setAccessible(true); - onByteRenewTaskField.setAccessible(true); - ScheduledFuture onMessageRenewTask = (ScheduledFuture) onMessageRenewTaskField.get(topicPublishRateLimiterOnMessage); - ScheduledFuture onByteRenewTask = (ScheduledFuture) onByteRenewTaskField.get(topicPublishRateLimiterOnByte); - - onMessageRenewTask.cancel(false); - onByteRenewTask.cancel(false); - - // renewing the permits from previous tests - renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage); - renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte); - - // tryAcquire not exceeded - assertTrue(precisePublishLimiter.tryAcquire(1, 10)); - renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage); - renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte); - - // tryAcquire numOfMessages exceeded - assertFalse(precisePublishLimiter.tryAcquire(11, 100)); - renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage); - renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte); - - // tryAcquire msgSizeInBytes exceeded - assertFalse(precisePublishLimiter.tryAcquire(10, 101)); - renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage); - renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte); - renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage); - renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte); - - // tryAcquire exceeded exactly - assertFalse(precisePublishLimiter.tryAcquire(10, 100)); - renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage); - renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte); - renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage); - renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte); - - // tryAcquire not exceeded - assertTrue(precisePublishLimiter.tryAcquire(9, 99)); + publishRateLimiter.handlePublishThrottling(producer, 11, 110); + assertEquals(throttleCount.get(), 0); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatedSubscriptionTest.java similarity index 54% rename from pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorSubscriptionTest.java rename to pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatedSubscriptionTest.java index 3982f44905d6b..5b896a22baa33 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatedSubscriptionTest.java @@ -18,18 +18,28 @@ */ package org.apache.pulsar.broker.service; +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongSumValue; +import static org.apache.pulsar.broker.stats.OpenTelemetryReplicatedSubscriptionStats.SNAPSHOT_DURATION_METRIC_NAME; +import static org.apache.pulsar.broker.stats.OpenTelemetryReplicatedSubscriptionStats.SNAPSHOT_OPERATION_COUNT_METRIC_NAME; +import static org.assertj.core.api.Assertions.assertThat; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import com.google.common.collect.Sets; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions; +import java.lang.reflect.Field; import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -37,30 +47,42 @@ import lombok.Cleanup; import org.apache.bookkeeper.mledger.Position; import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.service.persistent.ReplicatedSubscriptionsController; +import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferState; +import org.apache.pulsar.client.admin.LongRunningProcessStatus; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; +import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.common.policies.data.PartitionedTopicStats; +import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TopicStats; import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; /** * Tests replicated subscriptions (PIP-33) */ @Test(groups = "broker") -public class ReplicatorSubscriptionTest extends ReplicatorTestBase { - private static final Logger log = LoggerFactory.getLogger(ReplicatorSubscriptionTest.class); +public class ReplicatedSubscriptionTest extends ReplicatorTestBase { + private static final Logger log = LoggerFactory.getLogger(ReplicatedSubscriptionTest.class); @Override @BeforeClass(timeOut = 300000) @@ -123,7 +145,6 @@ public void testReplicatedSubscriptionAcrossTwoRegions() throws Exception { producer.send(body.getBytes(StandardCharsets.UTF_8)); sentMessages.add(body); } - producer.close(); } Set receivedMessages = new LinkedHashSet<>(); @@ -152,6 +173,180 @@ public void testReplicatedSubscriptionAcrossTwoRegions() throws Exception { // assert that all messages have been received assertEquals(new ArrayList<>(sentMessages), new ArrayList<>(receivedMessages), "Sent and received " + "messages don't match."); + + var metrics1 = metricReader1.collectAllMetrics(); + assertMetricLongSumValue(metrics1, SNAPSHOT_OPERATION_COUNT_METRIC_NAME, + Attributes.empty(),value -> assertThat(value).isPositive()); + assertMetricLongSumValue(metrics1, SNAPSHOT_OPERATION_COUNT_METRIC_NAME, + Attributes.empty(), value -> assertThat(value).isPositive()); + assertThat(metrics1) + .anySatisfy(metric -> OpenTelemetryAssertions.assertThat(metric) + .hasName(SNAPSHOT_DURATION_METRIC_NAME) + .hasHistogramSatisfying(histogram -> histogram.hasPointsSatisfying( + histogramPoint -> histogramPoint.hasSumGreaterThan(0.0)))); + } + + /** + * Tests replicated subscriptions across two regions and can read successful. + */ + @Test + public void testReplicatedSubscriptionAcrossTwoRegionsGetLastMessage() throws Exception { + String namespace = BrokerTestUtil.newUniqueName("pulsar/replicatedsubscriptionlastmessage"); + String topicName = "persistent://" + namespace + "/mytopic"; + String subscriptionName = "cluster-subscription"; + // this setting can be used to manually run the test with subscription replication disabled + // it shows that subscription replication has no impact in behavior for this test case + boolean replicateSubscriptionState = true; + + admin1.namespaces().createNamespace(namespace); + admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2")); + + @Cleanup + PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()) + .statsInterval(0, TimeUnit.SECONDS) + .build(); + + // create subscription in r1 + createReplicatedSubscription(client1, topicName, subscriptionName, replicateSubscriptionState); + + @Cleanup + PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()) + .statsInterval(0, TimeUnit.SECONDS) + .build(); + + // create subscription in r2 + createReplicatedSubscription(client2, topicName, subscriptionName, replicateSubscriptionState); + + Set sentMessages = new LinkedHashSet<>(); + + // send messages in r1 + @Cleanup + Producer producer = client1.newProducer().topic(topicName) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + int numMessages = 6; + for (int i = 0; i < numMessages; i++) { + String body = "message" + i; + producer.send(body.getBytes(StandardCharsets.UTF_8)); + sentMessages.add(body); + } + producer.close(); + + + // consume 3 messages in r1 + Set receivedMessages = new LinkedHashSet<>(); + try (Consumer consumer1 = client1.newConsumer() + .topic(topicName) + .subscriptionName(subscriptionName) + .replicateSubscriptionState(replicateSubscriptionState) + .subscribe()) { + readMessages(consumer1, receivedMessages, 3, false); + } + + // wait for subscription to be replicated + Thread.sleep(2 * config1.getReplicatedSubscriptionsSnapshotFrequencyMillis()); + + // create a reader in r2 + Reader reader = client2.newReader().topic(topicName) + .subscriptionName("new-sub") + .startMessageId(MessageId.earliest) + .create(); + int readNum = 0; + while (reader.hasMessageAvailable()) { + Message message = reader.readNext(10, TimeUnit.SECONDS); + assertNotNull(message); + log.info("Receive message: " + new String(message.getValue()) + " msgId: " + message.getMessageId()); + readNum++; + } + assertEquals(readNum, numMessages); + } + + @Test + public void testReplicatedSubscribeAndSwitchToStandbyCluster() throws Exception { + final String namespace = BrokerTestUtil.newUniqueName("pulsar/ns_"); + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + namespace + "/tp_"); + final String subscriptionName = "s1"; + final boolean isReplicatedSubscription = true; + final int messagesCount = 20; + final LinkedHashSet sentMessages = new LinkedHashSet<>(); + final Set receivedMessages = Collections.synchronizedSet(new LinkedHashSet<>()); + admin1.namespaces().createNamespace(namespace); + admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2")); + admin1.topics().createNonPartitionedTopic(topicName); + admin1.topics().createSubscription(topicName, subscriptionName, MessageId.earliest, isReplicatedSubscription); + final PersistentTopic topic1 = + (PersistentTopic) pulsar1.getBrokerService().getTopic(topicName, false).join().get(); + + // Send messages + // Wait for the topic created on the cluster2. + // Wait for the snapshot created. + final PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).build(); + Producer producer1 = client1.newProducer(Schema.STRING).topic(topicName).enableBatching(false).create(); + Consumer consumer1 = client1.newConsumer(Schema.STRING).topic(topicName) + .subscriptionName(subscriptionName).replicateSubscriptionState(isReplicatedSubscription).subscribe(); + for (int i = 0; i < messagesCount / 2; i++) { + String msg = i + ""; + producer1.send(msg); + sentMessages.add(msg); + } + Awaitility.await().untilAsserted(() -> { + final var replicators = topic1.getReplicators(); + assertTrue(replicators != null && replicators.size() == 1, "Replicator should started"); + assertTrue(replicators.values().iterator().next().isConnected(), "Replicator should be connected"); + assertTrue(topic1.getReplicatedSubscriptionController().get().getLastCompletedSnapshotId().isPresent(), + "One snapshot should be finished"); + }); + final PersistentTopic topic2 = + (PersistentTopic) pulsar2.getBrokerService().getTopic(topicName, false).join().get(); + Awaitility.await().untilAsserted(() -> { + assertTrue(topic2.getReplicatedSubscriptionController().isPresent(), + "Replicated subscription controller should created"); + }); + for (int i = messagesCount / 2; i < messagesCount; i++) { + String msg = i + ""; + producer1.send(msg); + sentMessages.add(msg); + } + + // Consume half messages and wait the subscription created on the cluster2. + for (int i = 0; i < messagesCount / 2; i++){ + Message message = consumer1.receive(2, TimeUnit.SECONDS); + if (message == null) { + fail("Should not receive null."); + } + receivedMessages.add(message.getValue()); + consumer1.acknowledge(message); + } + Awaitility.await().untilAsserted(() -> { + assertNotNull(topic2.getSubscriptions().get(subscriptionName), "Subscription should created"); + }); + + // Switch client to cluster2. + // Since the cluster1 was not crash, all messages will be replicated to the cluster2. + consumer1.close(); + final PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()).build(); + final Consumer consumer2 = client2.newConsumer(Schema.AUTO_CONSUME()).topic(topicName) + .subscriptionName(subscriptionName).replicateSubscriptionState(isReplicatedSubscription).subscribe(); + + // Verify all messages will be consumed. + Awaitility.await().untilAsserted(() -> { + while (true) { + Message message = consumer2.receive(2, TimeUnit.SECONDS); + if (message != null) { + receivedMessages.add(message.getValue().toString()); + consumer2.acknowledge(message); + } else { + break; + } + } + assertEquals(receivedMessages.size(), sentMessages.size()); + }); + + consumer2.close(); + producer1.close(); + client1.close(); + client2.close(); } /** @@ -530,6 +725,42 @@ public void testReplicatedSubscriptionRestApi2() throws Exception { String.format("numReceivedMessages2 (%d) should be less than %d", numReceivedMessages2, numMessages)); } + @Test(timeOut = 30000) + public void testReplicatedSubscriptionRestApi3() throws Exception { + final String namespace = BrokerTestUtil.newUniqueName("geo/replicatedsubscription"); + final String topicName = "persistent://" + namespace + "/topic-rest-api3"; + final String subName = "sub"; + admin4.tenants().createTenant("geo", + new TenantInfoImpl(Sets.newHashSet("appid1", "appid4"), Sets.newHashSet(cluster1, cluster4))); + admin4.namespaces().createNamespace(namespace); + admin4.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet(cluster1, cluster4)); + admin4.topics().createPartitionedTopic(topicName, 2); + + @Cleanup + final PulsarClient client4 = PulsarClient.builder().serviceUrl(url4.toString()) + .statsInterval(0, TimeUnit.SECONDS).build(); + + Consumer consumer4 = client4.newConsumer().topic(topicName).subscriptionName(subName).subscribe(); + Assert.expectThrows(PulsarAdminException.class, () -> + admin4.topics().setReplicatedSubscriptionStatus(topicName, subName, true)); + consumer4.close(); + } + + /** + * before sending message, we should wait for transaction buffer recover complete, + * or the MaxReadPosition will not move forward when the message is sent, and the + * MaxReadPositionMovedForwardTimestamp will not be updated, then the replication will not be triggered. + * @param topicName + * @throws Exception + */ + private void waitTBRecoverComplete(PulsarService pulsarService, String topicName) throws Exception { + TopicTransactionBufferState buffer = (TopicTransactionBufferState) ((PersistentTopic) pulsarService.getBrokerService() + .getTopic(topicName, false).get().get()).getTransactionBuffer(); + Field stateField = TopicTransactionBufferState.class.getDeclaredField("state"); + stateField.setAccessible(true); + Awaitility.await().until(() -> !stateField.get(buffer).toString().equals("Initializing")); + } + /** * Tests replicated subscriptions when replicator producer is closed */ @@ -557,6 +788,9 @@ public void testReplicatedSubscriptionWhenReplicatorProducerIsClosed() throws Ex .subscribe(); // send one message to trigger replication + if (config1.isTransactionCoordinatorEnabled()) { + waitTBRecoverComplete(pulsar1, topicName); + } @Cleanup Producer producer = client1.newProducer().topic(topicName) .enableBatching(false) @@ -613,6 +847,278 @@ public void testReplicatedSubscriptionWhenReplicatorProducerIsClosed() throws Ex Awaitility.await().untilAsserted(() -> assertNotNull(topic2.getSubscription(subscriptionName))); } + @DataProvider(name = "isTopicPolicyEnabled") + private Object[][] isTopicPolicyEnabled() { + // Todo: fix replication can not be enabled at topic level. + return new Object[][] { { Boolean.FALSE } }; + } + + /** + * Test the replication subscription can work normal in the following cases: + *

+ * 1. Do not write data into the original topic when the topic does not configure a remote cluster. {topic1} + * 1. Publish message to the topic and then wait a moment, + * the backlog will not increase after publishing completely. + * 2. Acknowledge the messages, the last confirm entry does not change. + * 2. Snapshot and mark will be written after topic configure a remote cluster. {topic2} + * 1. publish message to topic. After publishing completely, the backlog of the topic keep increase. + * 2. Wait the snapshot complete, the backlog stop changing. + * 3. Publish messages to wait another snapshot complete. + * 4. Ack messages to move the mark delete position after the position record in the first snapshot. + * 5. Check new entry (a mark) appending to the original topic. + * 3. Stopping writing snapshot and mark after remove the remote cluster of the topic. {topic2} + * similar to step 1. + *

+ */ + // TODO: this test causes OOME in the CI, need to investigate + @Test(dataProvider = "isTopicPolicyEnabled", enabled = false) + public void testWriteMarkerTaskOfReplicateSubscriptions(boolean isTopicPolicyEnabled) throws Exception { + // 1. Prepare resource and use proper configuration. + String namespace = BrokerTestUtil.newUniqueName("pulsar/testReplicateSubBackLog"); + String topic1 = "persistent://" + namespace + "/replication-enable"; + String topic2 = "persistent://" + namespace + "/replication-disable"; + String subName = "sub"; + + admin1.namespaces().createNamespace(namespace); + pulsar1.getConfiguration().setTopicLevelPoliciesEnabled(isTopicPolicyEnabled); + pulsar1.getConfiguration().setReplicationPolicyCheckDurationSeconds(1); + pulsar1.getConfiguration().setReplicatedSubscriptionsSnapshotFrequencyMillis(1000); + // 2. Build Producer and Consumer. + @Cleanup + PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()) + .statsInterval(0, TimeUnit.SECONDS) + .build(); + @Cleanup + Consumer consumer1 = client1.newConsumer() + .topic(topic1) + .subscriptionName(subName) + .ackTimeout(5, TimeUnit.SECONDS) + .subscriptionType(SubscriptionType.Shared) + .replicateSubscriptionState(true) + .subscribe(); + @Cleanup + Producer producer1 = client1.newProducer() + .topic(topic1) + .create(); + // 3. Test replication subscription work as expected. + // Test case 1: disable replication, backlog will not increase. + testReplicatedSubscriptionWhenDisableReplication(producer1, consumer1, topic1); + + // Test case 2: enable replication, mark and snapshot work as expected. + if (isTopicPolicyEnabled) { + admin1.topics().createNonPartitionedTopic(topic2); + admin1.topics().setReplicationClusters(topic2, List.of("r1", "r2")); + } else { + admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2")); + } + @Cleanup + Consumer consumer2 = client1.newConsumer() + .topic(topic2) + .subscriptionName(subName) + .ackTimeout(5, TimeUnit.SECONDS) + .subscriptionType(SubscriptionType.Shared) + .replicateSubscriptionState(true) + .subscribe(); + @Cleanup + Producer producer2 = client1.newProducer() + .topic(topic2) + .create(); + testReplicatedSubscriptionWhenEnableReplication(producer2, consumer2, topic2); + + // Test case 3: enable replication, mark and snapshot work as expected. + if (isTopicPolicyEnabled) { + admin1.topics().setReplicationClusters(topic2, List.of("r1")); + } else { + admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1")); + } + testReplicatedSubscriptionWhenDisableReplication(producer2, consumer2, topic2); + // 4. Clear resource. + pulsar1.getConfiguration().setForceDeleteNamespaceAllowed(true); + admin1.namespaces().deleteNamespace(namespace, true); + pulsar1.getConfiguration().setForceDeleteNamespaceAllowed(false); + } + + @Test + public void testReplicatedSubscriptionWithCompaction() throws Exception { + final String namespace = BrokerTestUtil.newUniqueName("pulsar/replicatedsubscription"); + final String topicName = "persistent://" + namespace + "/testReplicatedSubscriptionWithCompaction"; + final String subName = "sub"; + + admin1.namespaces().createNamespace(namespace); + admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2")); + admin1.topics().createNonPartitionedTopic(topicName); + admin1.topicPolicies().setCompactionThreshold(topicName, 100 * 1024 * 1024L); + + @Cleanup final PulsarClient client = PulsarClient.builder().serviceUrl(url1.toString()) + .statsInterval(0, TimeUnit.SECONDS).build(); + + Producer producer = client.newProducer(Schema.STRING).topic(topicName).create(); + if (config1.isTransactionCoordinatorEnabled()) { + waitTBRecoverComplete(pulsar1, topicName); + } + producer.newMessage().key("K1").value("V1").send(); + producer.newMessage().key("K1").value("V2").send(); + producer.close(); + + createReplicatedSubscription(client, topicName, subName, true); + Awaitility.await().untilAsserted(() -> { + Map status = admin1.topics().getReplicatedSubscriptionStatus(topicName, subName); + assertTrue(status.get(topicName)); + }); + + Awaitility.await().untilAsserted(() -> { + PersistentTopic t1 = (PersistentTopic) pulsar1.getBrokerService() + .getTopic(topicName, false).get().get(); + ReplicatedSubscriptionsController rsc1 = t1.getReplicatedSubscriptionController().get(); + Assert.assertTrue(rsc1.getLastCompletedSnapshotId().isPresent()); + assertEquals(t1.getPendingWriteOps().get(), 0L); + }); + + admin1.topics().triggerCompaction(topicName); + + Awaitility.await().untilAsserted(() -> { + assertEquals(admin1.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS); + }); + + @Cleanup + Consumer consumer = client.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName("sub2") + .subscriptionType(SubscriptionType.Exclusive) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .readCompacted(true) + .subscribe(); + List result = new ArrayList<>(); + while (true) { + Message receive = consumer.receive(2, TimeUnit.SECONDS); + if (receive == null) { + break; + } + + result.add(receive.getValue()); + } + + Assert.assertEquals(result, List.of("V2")); + } + + /** + * Disable replication subscription. + * Test scheduled task case. + * 1. Send three messages |1:0|1:1|1:2|. + * 2. Get topic backlog, as backlog1. + * 3. Wait a moment. + * 4. Get the topic backlog again, the backlog will not increase. + * Test acknowledge messages case. + * 1. Get the last confirm entry, as LAC1. + * 2. Acknowledge these messages |1:0|1:1|. + * 3. wait a moment. + * 4. Get the last confirm entry, as LAC2. LAC1 is equal to LAC2. + * Clear environment. + * 1. Ack all the retained messages. |1:2| + * 2. Wait for the backlog to return to zero. + */ + private void testReplicatedSubscriptionWhenDisableReplication(Producer producer, Consumer consumer, + String topic) throws Exception { + final int messageSum = 3; + // Test scheduled task case. + for (int i = 0; i < messageSum; i++) { + producer.newMessage().send(); + } + long backlog1 = admin1.topics().getStats(topic, false).getBacklogSize(); + Thread.sleep(3000); + long backlog2 = admin1.topics().getStats(topic, false).getBacklogSize(); + assertEquals(backlog1, backlog2); + // Test acknowledge messages case. + String lastConfirmEntry1 = admin1.topics().getInternalStats(topic).lastConfirmedEntry; + for (int i = 0; i < messageSum - 1; i++) { + consumer.acknowledge(consumer.receive(5, TimeUnit.SECONDS)); + } + Awaitility.await().untilAsserted(() -> { + String lastConfirmEntry2 = admin1.topics().getInternalStats(topic).lastConfirmedEntry; + assertEquals(lastConfirmEntry1, lastConfirmEntry2); + }); + // Clear environment. + consumer.acknowledge(consumer.receive(5, TimeUnit.SECONDS)); + Awaitility.await().untilAsserted(() -> { + long backlog4 = admin1.topics().getStats(topic, false).getBacklogSize(); + assertEquals(backlog4, 0); + }); + } + + /** + * Enable replication subscription. + * Test scheduled task case. + * 1. Wait replicator connected. + * 2. Send three messages |1:0|1:1|1:2|. + * 3. Get topic backlog, as backlog1. + * 4. Wait a moment. + * 5. Get the topic backlog again, as backlog2. The backlog2 is bigger than backlog1. |1:0|1:1|1:2|mark|. + * 6. Wait the snapshot complete. + * Test acknowledge messages case. + * 1. Write messages and wait another snapshot complete. |1:0|1:1|1:2|mark|1:3|1:4|1:5|mark| + * 2. Ack message |1:0|1:1|1:2|1:3|1:4|. + * 3. Get last confirm entry, as LAC1. + * 2. Wait a moment. + * 3. Get Last confirm entry, as LAC2. LAC2 different to LAC1. |1:5|mark|mark| + * Clear environment. + * 1. Ack all the retained message |1:5|. + * 2. Wait for the backlog to return to zero. + */ + private void testReplicatedSubscriptionWhenEnableReplication(Producer producer, Consumer consumer, + String topic) throws Exception { + final int messageSum = 3; + Awaitility.await().untilAsserted(() -> { + List keys = pulsar1.getBrokerService() + .getTopic(topic, false).get().get() + .getReplicators().keySet().stream().toList(); + assertEquals(keys.size(), 1); + assertTrue(pulsar1.getBrokerService() + .getTopic(topic, false).get().get() + .getReplicators().get(keys.get(0)).isConnected()); + }); + // Test scheduled task case. + sendMessageAndWaitSnapshotComplete(producer, topic, messageSum); + // Test acknowledge messages case. + // After snapshot write completely, acknowledging message to move the mark delete position + // after the position recorded in the snapshot will trigger to write a new marker. + sendMessageAndWaitSnapshotComplete(producer, topic, messageSum); + String lastConfirmedEntry3 = admin1.topics().getInternalStats(topic, false).lastConfirmedEntry; + for (int i = 0; i < messageSum * 2 - 1; i++) { + consumer.acknowledge(consumer.receive(5, TimeUnit.SECONDS)); + } + Awaitility.await().untilAsserted(() -> { + String lastConfirmedEntry4 = admin1.topics().getInternalStats(topic, false).lastConfirmedEntry; + assertNotEquals(lastConfirmedEntry3, lastConfirmedEntry4); + }); + // Clear environment. + consumer.acknowledge(consumer.receive(5, TimeUnit.SECONDS)); + Awaitility.await().untilAsserted(() -> { + long backlog4 = admin1.topics().getStats(topic, false).getBacklogSize(); + assertEquals(backlog4, 0); + }); + } + + private void sendMessageAndWaitSnapshotComplete(Producer producer, String topic, + int messageSum) throws Exception { + for (int i = 0; i < messageSum; i++) { + producer.newMessage().send(); + } + long backlog1 = admin1.topics().getStats(topic, false).getBacklogSize(); + Awaitility.await().untilAsserted(() -> { + long backlog2 = admin1.topics().getStats(topic, false).getBacklogSize(); + assertTrue(backlog2 > backlog1); + }); + // Wait snapshot write completely, stop writing marker into topic. + Awaitility.await().untilAsserted(() -> { + String lastConfirmedEntry1 = admin1.topics().getInternalStats(topic, false).lastConfirmedEntry; + PersistentTopicInternalStats persistentTopicInternalStats = admin1.topics().getInternalStats(topic, false); + Thread.sleep(1000); + String lastConfirmedEntry2 = admin1.topics().getInternalStats(topic, false).lastConfirmedEntry; + assertEquals(lastConfirmedEntry1, lastConfirmedEntry2); + }); + } + void publishMessages(Producer producer, int startIndex, int numMessages, Set sentMessages) throws PulsarClientException { for (int i = startIndex; i < startIndex + numMessages; i++) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatedSubscriptionWithTransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatedSubscriptionWithTransactionTest.java new file mode 100644 index 0000000000000..9494247794420 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatedSubscriptionWithTransactionTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +/** + * Tests replicated subscriptions with transaction (PIP-33) + */ +@Test(groups = "broker") +public class ReplicatedSubscriptionWithTransactionTest extends ReplicatedSubscriptionTest { + + @Override + @BeforeClass(timeOut = 300000) + public void setup() throws Exception { + config1.setTransactionCoordinatorEnabled(true); + config2.setTransactionCoordinatorEnabled(true); + config3.setTransactionCoordinatorEnabled(true); + config4.setTransactionCoordinatorEnabled(true); + super.setup(); + } + + @Override + @AfterClass(alwaysRun = true, timeOut = 300000) + public void cleanup() throws Exception { + super.cleanup(); + } + + @DataProvider(name = "isTopicPolicyEnabled") + private Object[][] isTopicPolicyEnabled() { + // Todo: fix replication can not be enabled at topic level. + return new Object[][] { { Boolean.FALSE } }; + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicationTxnTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicationTxnTest.java new file mode 100644 index 0000000000000..3caf4a1f2398c --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicationTxnTest.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.apache.pulsar.common.naming.NamespaceName.SYSTEM_NAMESPACE; +import static org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl.TRANSACTION_LOG_PREFIX; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import com.google.common.collect.Sets; +import java.util.Collections; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.service.persistent.GeoPersistentReplicator; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore; +import org.apache.pulsar.client.api.ClientBuilder; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.transaction.Transaction; +import org.apache.pulsar.common.api.proto.MessageMetadata; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.SystemTopicNames; +import org.apache.pulsar.common.naming.TopicDomain; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.partition.PartitionedTopicMetadata; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.apache.pulsar.zookeeper.ZookeeperServerTest; +import org.awaitility.reflect.WhiteboxImpl; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class ReplicationTxnTest extends OneWayReplicatorTestBase { + + private boolean transactionBufferSegmentedSnapshotEnabled = false; + private int txnLogPartitions = 4; + + @Override + @BeforeClass(alwaysRun = true, timeOut = 300000) + public void setup() throws Exception { + super.setup(); + } + + @Override + @AfterClass(alwaysRun = true, timeOut = 300000) + public void cleanup() throws Exception { + super.cleanup(); + } + + @Override + protected PulsarClient initClient(ClientBuilder clientBuilder) throws Exception { + return clientBuilder.enableTransaction(true).build(); + } + + @Override + protected void setConfigDefaults(ServiceConfiguration config, String clusterName, + LocalBookkeeperEnsemble bookkeeperEnsemble, ZookeeperServerTest brokerConfigZk) { + super.setConfigDefaults(config, clusterName, bookkeeperEnsemble, brokerConfigZk); + config.setSystemTopicEnabled(true); + config.setTopicLevelPoliciesEnabled(true); + config.setTransactionCoordinatorEnabled(true); + config.setTransactionLogBatchedWriteEnabled(true); + config.setTransactionPendingAckBatchedWriteEnabled(true); + config.setTransactionBufferSegmentedSnapshotEnabled(transactionBufferSegmentedSnapshotEnabled); + } + + @Override + protected void createDefaultTenantsAndClustersAndNamespace() throws Exception { + super.createDefaultTenantsAndClustersAndNamespace(); + + // Create resource that transaction function relies on. + admin1.tenants().createTenant(SYSTEM_NAMESPACE.getTenant(), new TenantInfoImpl(Collections.emptySet(), + Sets.newHashSet(cluster1, cluster2))); + admin1.namespaces().createNamespace(SYSTEM_NAMESPACE.toString(), 4); + pulsar1.getPulsarResources().getNamespaceResources().getPartitionedTopicResources().createPartitionedTopic( + SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN, new PartitionedTopicMetadata(txnLogPartitions)); + //admin1.topics().createPartitionedTopic(SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN.toString(), 4); + + admin2.tenants().createTenant(SYSTEM_NAMESPACE.getTenant(), new TenantInfoImpl(Collections.emptySet(), + Sets.newHashSet(cluster1, cluster2))); + admin2.namespaces().createNamespace(SYSTEM_NAMESPACE.toString(), 4); + pulsar2.getPulsarResources().getNamespaceResources().getPartitionedTopicResources().createPartitionedTopic( + SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN, new PartitionedTopicMetadata(txnLogPartitions)); + } + + private void pubAndSubOneMsg(String topic, String subscription) throws Exception { + Consumer consumer1 = client1.newConsumer(Schema.STRING).topic(topic).subscriptionName(subscription) + .isAckReceiptEnabled(true).subscribe(); + Producer producer1 = client1.newProducer(Schema.STRING).topic(topic).create(); + producer1.newMessage().value("msg1").send(); + // start txn. + Transaction txn = client1.newTransaction().withTransactionTimeout(1, TimeUnit.MINUTES).build().get(); + // consume. + Message c1Msg1 = consumer1.receive(5, TimeUnit.SECONDS); + assertNotNull(c1Msg1); + assertEquals(c1Msg1.getValue(), "msg1"); + consumer1.acknowledgeAsync(c1Msg1.getMessageId(), txn).join(); + // send. + producer1.newMessage(txn).value("msg2").send(); + // commit. + txn.commit().get(); + + // Consume the msg with TXN. + Message c1Msg2 = consumer1.receive(5, TimeUnit.SECONDS); + assertNotNull(c1Msg2); + assertEquals(c1Msg2.getValue(), "msg2"); + consumer1.acknowledgeAsync(c1Msg2.getMessageId()).join(); + + // Consume messages on the remote cluster. + Consumer consumer2 = client2.newConsumer(Schema.STRING).topic(topic).subscriptionName(subscription).subscribe(); + Message c2Msg1 = consumer2.receive(15, TimeUnit.SECONDS); + assertNotNull(c2Msg1); + MessageMetadata msgMetadata1 = WhiteboxImpl.getInternalState(c2Msg1, "msgMetadata"); + // Verify: the messages replicated has no TXN id. + assertFalse(msgMetadata1.hasTxnidMostBits()); + assertFalse(msgMetadata1.hasTxnidLeastBits()); + consumer2.acknowledge(c2Msg1); + Message c2Msg2 = consumer2.receive(15, TimeUnit.SECONDS); + assertNotNull(c2Msg2); + MessageMetadata msgMetadata2 = WhiteboxImpl.getInternalState(c2Msg2, "msgMetadata"); + // Verify: the messages replicated has no TXN id. + assertFalse(msgMetadata2.hasTxnidMostBits()); + assertFalse(msgMetadata2.hasTxnidLeastBits()); + consumer2.acknowledge(c2Msg2); + + // cleanup. + producer1.close(); + consumer1.close(); + consumer2.close(); + } + + private void verifyNoReplicator(BrokerService broker, TopicName topicName) throws Exception { + String tpStr = topicName.toString(); + CompletableFuture> future = broker.getTopic(tpStr, true); + if (future == null) { + return; + } + PersistentTopic persistentTopic = (PersistentTopic) future.join().get(); + assertTrue(persistentTopic.getReplicators().isEmpty()); + } + + @Test + public void testTxnLogNotBeReplicated() throws Exception { + final String topic = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp"); + final String subscription = "s1"; + admin1.topics().createNonPartitionedTopic(topic); + waitReplicatorStarted(topic); + admin1.topics().createSubscription(topic, subscription, MessageId.earliest); + admin2.topics().createSubscription(topic, subscription, MessageId.earliest); + // Pub & Sub. + pubAndSubOneMsg(topic, subscription); + // To cover more cases, sleep 3s. + Thread.sleep(3000); + + // Verify: messages on the TXN system topic did not been replicated. + // __transaction_log_: it only uses ML, will not create topic. + for (int i = 0; i < txnLogPartitions; i++) { + TopicName txnLog = TopicName.get(TopicDomain.persistent.value(), + NamespaceName.SYSTEM_NAMESPACE, TRANSACTION_LOG_PREFIX + i); + assertNotNull(pulsar1.getManagedLedgerFactory() + .getManagedLedgerInfo(txnLog.getPersistenceNamingEncoding())); + assertFalse(broker1.getTopics().containsKey(txnLog.toString())); + } + // __transaction_pending_ack: it only uses ML, will not create topic. + TopicName pendingAck = TopicName.get( + MLPendingAckStore.getTransactionPendingAckStoreSuffix(topic, subscription)); + assertNotNull(pulsar1.getManagedLedgerFactory() + .getManagedLedgerInfo(pendingAck.getPersistenceNamingEncoding())); + assertFalse(broker1.getTopics().containsKey(pendingAck.toString())); + // __transaction_buffer_snapshot. + verifyNoReplicator(broker1, TopicName.get(TopicDomain.persistent.value(), + TopicName.get(topic).getNamespaceObject(), + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT)); + verifyNoReplicator(broker1, TopicName.get(TopicDomain.persistent.value(), + TopicName.get(topic).getNamespaceObject(), + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS)); + verifyNoReplicator(broker1, TopicName.get(TopicDomain.persistent.value(), + TopicName.get(topic).getNamespaceObject(), + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_INDEXES)); + + // cleanup. + cleanupTopics(() -> { + admin1.topics().delete(topic); + admin2.topics().delete(topic); + try { + admin1.topics().delete(pendingAck.toString()); + } catch (Exception ex) {} + try { + admin2.topics().delete(pendingAck.toString()); + } catch (Exception ex) {} + }); + } + + @Test + public void testOngoingMessagesWillNotBeReplicated() throws Exception { + final String topic = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp"); + final String subscription = "s1"; + admin1.topics().createNonPartitionedTopic(topic); + waitReplicatorStarted(topic); + admin1.topics().createSubscription(topic, subscription, MessageId.earliest); + admin2.topics().createSubscription(topic, subscription, MessageId.earliest); + // Pub without commit. + Producer producer1 = client1.newProducer(Schema.STRING).topic(topic).create(); + Transaction txn = client1.newTransaction().withTransactionTimeout(1, TimeUnit.HOURS).build().get(); + producer1.newMessage(txn).value("msg1").send(); + // Verify: receive nothing on the remote cluster. + Consumer consumer2 = client2.newConsumer(Schema.STRING).topic(topic).subscriptionName(subscription).subscribe(); + Message msg = consumer2.receive(15, TimeUnit.SECONDS); + assertNull(msg); + // Verify: the repl cursor is not end of the topic. + PersistentTopic persistentTopic = (PersistentTopic) broker1.getTopic(topic, false).join().get(); + GeoPersistentReplicator replicator = + (GeoPersistentReplicator) persistentTopic.getReplicators().values().iterator().next(); + assertTrue(replicator.getCursor().hasMoreEntries()); + + // cleanup. + producer1.close(); + consumer2.close(); + cleanupTopics(() -> { + admin1.topics().delete(topic); + admin2.topics().delete(topic); + TopicName pendingAck = TopicName.get( + MLPendingAckStore.getTransactionPendingAckStoreSuffix(topic, subscription)); + try { + admin1.topics().delete(pendingAck.toString()); + } catch (Exception ex) {} + try { + admin2.topics().delete(pendingAck.toString()); + } catch (Exception ex) {} + }); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsTest.java index a5d14ca0487dc..c214378fd94a3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsTest.java @@ -23,10 +23,8 @@ import static org.testng.Assert.assertTrue; import java.util.List; import java.util.Optional; -import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.internal.PulsarAdminImpl; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -55,7 +53,7 @@ public void testReplicationAdmin() throws Exception { ns.getClusterPulsarAdmin(cluster3, Optional.of(admin1.clusters().getCluster(cluster3))); // verify the admin - ConcurrentOpenHashMap clusterAdmins = ns.getClusterAdmins(); + final var clusterAdmins = ns.getClusterAdmins(); assertFalse(clusterAdmins.isEmpty()); clusterAdmins.forEach((cluster, admin) -> { ClientConfigurationData clientConfigData = ((PulsarAdminImpl) admin).getClientConfigData(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsWithKeyStoreTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsWithKeyStoreTest.java index 3d3eb3faa727f..451bdd10eb106 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsWithKeyStoreTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsWithKeyStoreTest.java @@ -23,10 +23,8 @@ import static org.testng.Assert.assertTrue; import java.util.List; import java.util.Optional; -import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.internal.PulsarAdminImpl; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -56,7 +54,7 @@ public void testReplicationAdmin() throws Exception { ns.getClusterPulsarAdmin(cluster3, Optional.of(admin1.clusters().getCluster(cluster3))); // verify the admin - ConcurrentOpenHashMap clusterAdmins = ns.getClusterAdmins(); + final var clusterAdmins = ns.getClusterAdmins(); assertFalse(clusterAdmins.isEmpty()); clusterAdmins.forEach((cluster, admin) -> { ClientConfigurationData clientConfigData = ((PulsarAdminImpl) admin).getClientConfigData(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorGlobalNSTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorGlobalNSTest.java index 62ba0e83c8600..a1f147cbb6273 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorGlobalNSTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorGlobalNSTest.java @@ -18,28 +18,55 @@ */ package org.apache.pulsar.broker.service; +import static org.testng.Assert.fail; import com.google.common.collect.Sets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.Future; import lombok.Cleanup; -import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.impl.ConsumerImpl; import org.apache.pulsar.client.impl.ProducerImpl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.pulsar.common.naming.TopicName; +import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Factory; import org.testng.annotations.Test; import java.lang.reflect.Method; import java.util.concurrent.TimeUnit; +/** + * The tests in this class should be denied in a production pulsar cluster. they are very dangerous, which leads to + * a lot of topic deletion and makes namespace policies being incorrect. + */ +@Slf4j @Test(groups = "broker-impl") public class ReplicatorGlobalNSTest extends ReplicatorTestBase { protected String methodName; + @DataProvider(name = "loadManagerClassName") + public static Object[][] loadManagerClassName() { + return new Object[][]{ + {ModularLoadManagerImpl.class.getName()}, + {ExtensibleLoadManagerImpl.class.getName()} + }; + } + + @Factory(dataProvider = "loadManagerClassName") + public ReplicatorGlobalNSTest(String loadManagerClassName) { + this.loadManagerClassName = loadManagerClassName; + } @BeforeMethod public void beforeMethod(Method m) { @@ -64,7 +91,7 @@ public void cleanup() throws Exception { * * @throws Exception */ - @Test + @Test(priority = Integer.MAX_VALUE) public void testRemoveLocalClusterOnGlobalNamespace() throws Exception { log.info("--- Starting ReplicatorTest::testRemoveLocalClusterOnGlobalNamespace ---"); @@ -90,42 +117,96 @@ public void testRemoveLocalClusterOnGlobalNamespace() throws Exception { admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r2", "r3")); - MockedPulsarServiceBaseTest - .retryStrategically((test) -> !pulsar1.getBrokerService().getTopics().containsKey(topicName), 50, 150); - - Assert.assertFalse(pulsar1.getBrokerService().getTopics().containsKey(topicName)); - Assert.assertFalse(producer1.isConnected()); - Assert.assertFalse(consumer1.isConnected()); - Assert.assertTrue(consumer2.isConnected()); - + Awaitility.await().atMost(1, TimeUnit.MINUTES).untilAsserted(() -> { + Assert.assertFalse(pulsar1.getBrokerService().getTopics().containsKey(topicName)); + Assert.assertFalse(producer1.isConnected()); + Assert.assertFalse(consumer1.isConnected()); + Assert.assertTrue(consumer2.isConnected()); + }); } - @Test - public void testForcefullyTopicDeletion() throws Exception { - log.info("--- Starting ReplicatorTest::testForcefullyTopicDeletion ---"); - - final String namespace = "pulsar/removeClusterTest"; - admin1.namespaces().createNamespace(namespace); - admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1")); - - final String topicName = "persistent://" + namespace + "/topic"; - - @Cleanup - PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS) - .build(); - - ProducerImpl producer1 = (ProducerImpl) client1.newProducer().topic(topicName) - .enableBatching(false).messageRoutingMode(MessageRoutingMode.SinglePartition).create(); - producer1.close(); - - admin1.topics().delete(topicName, true); - - MockedPulsarServiceBaseTest - .retryStrategically((test) -> !pulsar1.getBrokerService().getTopics().containsKey(topicName), 50, 150); - - Assert.assertFalse(pulsar1.getBrokerService().getTopics().containsKey(topicName)); + /** + * This is not a formal operation and can cause serious problems if call it in a production environment. + */ + @Test(priority = Integer.MAX_VALUE - 1) + public void testConfigChange() throws Exception { + log.info("--- Starting ReplicatorTest::testConfigChange ---"); + // This test is to verify that the config change on global namespace is successfully applied in broker during + // runtime. + // Run a set of producer tasks to create the topics + List> results = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + final TopicName dest = TopicName.get(BrokerTestUtil.newUniqueName("persistent://pulsar/ns/topic-" + i)); + + results.add(executor.submit(new Callable() { + @Override + public Void call() throws Exception { + + @Cleanup + MessageProducer producer = new MessageProducer(url1, dest); + log.info("--- Starting producer --- " + url1); + + @Cleanup + MessageConsumer consumer = new MessageConsumer(url1, dest); + log.info("--- Starting Consumer --- " + url1); + + producer.produce(2); + consumer.receive(2); + return null; + } + })); + } + + for (Future result : results) { + try { + result.get(); + } catch (Exception e) { + log.error("exception in getting future result ", e); + fail(String.format("replication test failed with %s exception", e.getMessage())); + } + } + + Thread.sleep(1000L); + // Make sure that the internal replicators map contains remote cluster info + final var replicationClients1 = ns1.getReplicationClients(); + final var replicationClients2 = ns2.getReplicationClients(); + final var replicationClients3 = ns3.getReplicationClients(); + + Assert.assertNotNull(replicationClients1.get("r2")); + Assert.assertNotNull(replicationClients1.get("r3")); + Assert.assertNotNull(replicationClients2.get("r1")); + Assert.assertNotNull(replicationClients2.get("r3")); + Assert.assertNotNull(replicationClients3.get("r1")); + Assert.assertNotNull(replicationClients3.get("r2")); + + // Case 1: Update the global namespace replication configuration to only contains the local cluster itself + admin1.namespaces().setNamespaceReplicationClusters("pulsar/ns", Sets.newHashSet("r1")); + + // Wait for config changes to be updated. + Thread.sleep(1000L); + + // Make sure that the internal replicators map still contains remote cluster info + Assert.assertNotNull(replicationClients1.get("r2")); + Assert.assertNotNull(replicationClients1.get("r3")); + Assert.assertNotNull(replicationClients2.get("r1")); + Assert.assertNotNull(replicationClients2.get("r3")); + Assert.assertNotNull(replicationClients3.get("r1")); + Assert.assertNotNull(replicationClients3.get("r2")); + + // Case 2: Update the configuration back + admin1.namespaces().setNamespaceReplicationClusters("pulsar/ns", Sets.newHashSet("r1", "r2", "r3")); + + // Wait for config changes to be updated. + Thread.sleep(1000L); + + // Make sure that the internal replicators map still contains remote cluster info + Assert.assertNotNull(replicationClients1.get("r2")); + Assert.assertNotNull(replicationClients1.get("r3")); + Assert.assertNotNull(replicationClients2.get("r1")); + Assert.assertNotNull(replicationClients2.get("r3")); + Assert.assertNotNull(replicationClients3.get("r1")); + Assert.assertNotNull(replicationClients3.get("r2")); + + // Case 3: TODO: Once automatic cleanup is implemented, add tests case to verify auto removal of clusters } - - private static final Logger log = LoggerFactory.getLogger(ReplicatorGlobalNSTest.class); - } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRateLimiterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRateLimiterTest.java index b97210b009a3b..2e0dd0a90e8a6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRateLimiterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRateLimiterTest.java @@ -18,21 +18,25 @@ */ package org.apache.pulsar.broker.service; +import static org.assertj.core.api.Assertions.assertThat; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.AssertJUnit.assertFalse; import com.google.common.collect.Sets; import java.lang.reflect.Method; +import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import lombok.Cleanup; +import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.common.policies.data.DispatchRate; +import org.apache.pulsar.broker.qos.AsyncTokenBucket; import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,6 +63,7 @@ public void beforeMethod(Method m) throws Exception { @Override @BeforeClass(timeOut = 300000) public void setup() throws Exception { + AsyncTokenBucket.switchToConsistentTokensView(); super.setup(); } @@ -66,6 +71,7 @@ public void setup() throws Exception { @AfterClass(alwaysRun = true, timeOut = 300000) public void cleanup() throws Exception { super.cleanup(); + AsyncTokenBucket.resetToDefaultEventualConsistentTokensView(); } enum DispatchRateType { @@ -97,7 +103,7 @@ public void testReplicatorRateLimiterWithOnlyTopicLevel() throws Exception { PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getOrCreateTopic(topicName).get(); // rate limiter disable by default - assertFalse(topic.getReplicators().values().get(0).getRateLimiter().isPresent()); + assertFalse(getRateLimiter(topic).isPresent()); //set topic-level policy, which should take effect DispatchRate topicRate = DispatchRate.builder() @@ -108,16 +114,16 @@ public void testReplicatorRateLimiterWithOnlyTopicLevel() throws Exception { admin1.topics().setReplicatorDispatchRate(topicName, topicRate); Awaitility.await().untilAsserted(() -> assertEquals(admin1.topics().getReplicatorDispatchRate(topicName), topicRate)); - assertTrue(topic.getReplicators().values().get(0).getRateLimiter().isPresent()); - assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnMsg(), 10); - assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnByte(), 20L); + assertTrue(getRateLimiter(topic).isPresent()); + assertEquals(getRateLimiter(topic).get().getDispatchRateOnMsg(), 10); + assertEquals(getRateLimiter(topic).get().getDispatchRateOnByte(), 20L); //remove topic-level policy admin1.topics().removeReplicatorDispatchRate(topicName); Awaitility.await().untilAsserted(() -> assertNull(admin1.topics().getReplicatorDispatchRate(topicName))); - assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnMsg(), -1); - assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnByte(), + assertEquals(getRateLimiter(topic).get().getDispatchRateOnMsg(), -1); + assertEquals(getRateLimiter(topic).get().getDispatchRateOnByte(), -1L); } @@ -141,7 +147,7 @@ public void testReplicatorRateLimiterWithOnlyNamespaceLevel() throws Exception { PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getOrCreateTopic(topicName).get(); // rate limiter disable by default - assertFalse(topic.getReplicators().values().get(0).getRateLimiter().isPresent()); + assertFalse(getRateLimiter(topic).isPresent()); //set namespace-level policy, which should take effect DispatchRate topicRate = DispatchRate.builder() @@ -152,16 +158,16 @@ public void testReplicatorRateLimiterWithOnlyNamespaceLevel() throws Exception { admin1.namespaces().setReplicatorDispatchRate(namespace, topicRate); Awaitility.await().untilAsserted(() -> assertEquals(admin1.namespaces().getReplicatorDispatchRate(namespace), topicRate)); - assertTrue(topic.getReplicators().values().get(0).getRateLimiter().isPresent()); - assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnMsg(), 10); - assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnByte(), 20L); + assertTrue(getRateLimiter(topic).isPresent()); + assertEquals(getRateLimiter(topic).get().getDispatchRateOnMsg(), 10); + assertEquals(getRateLimiter(topic).get().getDispatchRateOnByte(), 20L); //remove topic-level policy admin1.namespaces().removeReplicatorDispatchRate(namespace); Awaitility.await().untilAsserted(() -> assertNull(admin1.namespaces().getReplicatorDispatchRate(namespace))); - assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnMsg(), -1); - assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnByte(), + assertEquals(getRateLimiter(topic).get().getDispatchRateOnMsg(), -1); + assertEquals(getRateLimiter(topic).get().getDispatchRateOnByte(), -1L); } @@ -185,7 +191,7 @@ public void testReplicatorRateLimiterWithOnlyBrokerLevel() throws Exception { PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getOrCreateTopic(topicName).get(); // rate limiter disable by default - assertFalse(topic.getReplicators().values().get(0).getRateLimiter().isPresent()); + assertFalse(getRateLimiter(topic).isPresent()); //set broker-level policy, which should take effect admin1.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerReplicatorInMsg", "10"); @@ -199,9 +205,9 @@ public void testReplicatorRateLimiterWithOnlyBrokerLevel() throws Exception { .getAllDynamicConfigurations().get("dispatchThrottlingRatePerReplicatorInByte"), "20"); }); - assertTrue(topic.getReplicators().values().get(0).getRateLimiter().isPresent()); - assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnMsg(), 10); - assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnByte(), 20L); + assertTrue(getRateLimiter(topic).isPresent()); + assertEquals(getRateLimiter(topic).get().getDispatchRateOnMsg(), 10); + assertEquals(getRateLimiter(topic).get().getDispatchRateOnByte(), 20L); } @Test @@ -224,9 +230,9 @@ public void testReplicatorRatePriority() throws Exception { PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getOrCreateTopic(topicName).get(); //use broker-level by default - assertTrue(topic.getReplicators().values().get(0).getRateLimiter().isPresent()); - assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnMsg(), 100); - assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnByte(), 200L); + assertTrue(getRateLimiter(topic).isPresent()); + assertEquals(getRateLimiter(topic).get().getDispatchRateOnMsg(), 100); + assertEquals(getRateLimiter(topic).get().getDispatchRateOnByte(), 200L); //set namespace-level policy, which should take effect DispatchRate nsDispatchRate = DispatchRate.builder() @@ -237,8 +243,8 @@ public void testReplicatorRatePriority() throws Exception { admin1.namespaces().setReplicatorDispatchRate(namespace, nsDispatchRate); Awaitility.await() .untilAsserted(() -> assertEquals(admin1.namespaces().getReplicatorDispatchRate(namespace), nsDispatchRate)); - assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnMsg(), 50); - assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnByte(), 60L); + assertEquals(getRateLimiter(topic).get().getDispatchRateOnMsg(), 50); + assertEquals(getRateLimiter(topic).get().getDispatchRateOnByte(), 60L); //set topic-level policy, which should take effect DispatchRate topicRate = DispatchRate.builder() @@ -249,8 +255,8 @@ public void testReplicatorRatePriority() throws Exception { admin1.topics().setReplicatorDispatchRate(topicName, topicRate); Awaitility.await().untilAsserted(() -> assertEquals(admin1.topics().getReplicatorDispatchRate(topicName), topicRate)); - assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnMsg(), 10); - assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnByte(), 20L); + assertEquals(getRateLimiter(topic).get().getDispatchRateOnMsg(), 10); + assertEquals(getRateLimiter(topic).get().getDispatchRateOnByte(), 20L); //Set the namespace-level policy, which should not take effect DispatchRate nsDispatchRate2 = DispatchRate.builder() @@ -261,21 +267,21 @@ public void testReplicatorRatePriority() throws Exception { admin1.namespaces().setReplicatorDispatchRate(namespace, nsDispatchRate2); Awaitility.await() .untilAsserted(() -> assertEquals(admin1.namespaces().getReplicatorDispatchRate(namespace), nsDispatchRate2)); - assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnByte(), 20L); + assertEquals(getRateLimiter(topic).get().getDispatchRateOnByte(), 20L); //remove topic-level policy, namespace-level should take effect admin1.topics().removeReplicatorDispatchRate(topicName); Awaitility.await().untilAsserted(() -> assertNull(admin1.topics().getReplicatorDispatchRate(topicName))); - assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnMsg(), 500); - assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnByte(), + assertEquals(getRateLimiter(topic).get().getDispatchRateOnMsg(), 500); + assertEquals(getRateLimiter(topic).get().getDispatchRateOnByte(), 600L); //remove namespace-level policy, broker-level should take effect admin1.namespaces().setReplicatorDispatchRate(namespace, null); Awaitility.await().untilAsserted(() -> - assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnMsg(), 100)); - assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnByte(), + assertEquals(getRateLimiter(topic).get().getDispatchRateOnMsg(), 100)); + assertEquals(getRateLimiter(topic).get().getDispatchRateOnByte(), 200L); } @@ -311,7 +317,7 @@ public void testReplicatorRateLimiterDynamicallyChange() throws Exception { PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getOrCreateTopic(topicName).get(); // 1. default replicator throttling not configured - Assert.assertFalse(topic.getReplicators().values().get(0).getRateLimiter().isPresent()); + Assert.assertFalse(getRateLimiter(topic).isPresent()); // 2. change namespace setting of replicator dispatchRateMsg, verify topic changed. int messageRate = 100; @@ -325,7 +331,7 @@ public void testReplicatorRateLimiterDynamicallyChange() throws Exception { boolean replicatorUpdated = false; int retry = 5; for (int i = 0; i < retry; i++) { - if (topic.getReplicators().values().get(0).getRateLimiter().isPresent()) { + if (getRateLimiter(topic).isPresent()) { replicatorUpdated = true; break; } else { @@ -335,7 +341,7 @@ public void testReplicatorRateLimiterDynamicallyChange() throws Exception { } } Assert.assertTrue(replicatorUpdated); - Assert.assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnMsg(), messageRate); + Assert.assertEquals(getRateLimiter(topic).get().getDispatchRateOnMsg(), messageRate); // 3. change namespace setting of replicator dispatchRateByte, verify topic changed. messageRate = 500; @@ -347,7 +353,7 @@ public void testReplicatorRateLimiterDynamicallyChange() throws Exception { admin1.namespaces().setReplicatorDispatchRate(namespace, dispatchRateByte); replicatorUpdated = false; for (int i = 0; i < retry; i++) { - if (topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnByte() == messageRate) { + if (getRateLimiter(topic).get().getDispatchRateOnByte() == messageRate) { replicatorUpdated = true; break; } else { @@ -410,7 +416,7 @@ public void testReplicatorRateLimiterMessageNotReceivedAllMessages(DispatchRateT boolean replicatorUpdated = false; int retry = 5; for (int i = 0; i < retry; i++) { - if (topic.getReplicators().values().get(0).getRateLimiter().isPresent()) { + if (getRateLimiter(topic).isPresent()) { replicatorUpdated = true; break; } else { @@ -421,9 +427,9 @@ public void testReplicatorRateLimiterMessageNotReceivedAllMessages(DispatchRateT } Assert.assertTrue(replicatorUpdated); if (DispatchRateType.messageRate.equals(dispatchRateType)) { - Assert.assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnMsg(), messageRate); + Assert.assertEquals(getRateLimiter(topic).get().getDispatchRateOnMsg(), messageRate); } else { - Assert.assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnByte(), messageRate); + Assert.assertEquals(getRateLimiter(topic).get().getDispatchRateOnByte(), messageRate); } @Cleanup @@ -495,7 +501,7 @@ public void testReplicatorRateLimiterMessageReceivedAllMessages() throws Excepti boolean replicatorUpdated = false; int retry = 5; for (int i = 0; i < retry; i++) { - if (topic.getReplicators().values().get(0).getRateLimiter().isPresent()) { + if (getRateLimiter(topic).isPresent()) { replicatorUpdated = true; break; } else { @@ -505,7 +511,7 @@ public void testReplicatorRateLimiterMessageReceivedAllMessages() throws Excepti } } Assert.assertTrue(replicatorUpdated); - Assert.assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnMsg(), messageRate); + Assert.assertEquals(getRateLimiter(topic).get().getDispatchRateOnMsg(), messageRate); @Cleanup PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()).statsInterval(0, TimeUnit.SECONDS) @@ -545,5 +551,68 @@ public void testReplicatorRateLimiterMessageReceivedAllMessages() throws Excepti producer.close(); } + @Test + public void testReplicatorRateLimiterByBytes() throws Exception { + final String namespace = "pulsar/replicatormsg-" + System.currentTimeMillis(); + final String topicName = "persistent://" + namespace + "/RateLimiterByBytes"; + + admin1.namespaces().createNamespace(namespace); + // 0. set 2 clusters, there will be 1 replicator in each topic + admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2")); + + final int byteRate = 400; + final int payloadSize = 100; + DispatchRate dispatchRate = DispatchRate.builder() + .dispatchThrottlingRateInMsg(-1) + .dispatchThrottlingRateInByte(byteRate) + .ratePeriodInSecond(360) + .build(); + admin1.namespaces().setReplicatorDispatchRate(namespace, dispatchRate); + + @Cleanup + PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).build(); + @Cleanup + Producer producer = client1.newProducer().topic(topicName) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + + PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getOrCreateTopic(topicName).get(); + + Awaitility.await() + .untilAsserted(() -> assertTrue(getRateLimiter(topic).isPresent())); + assertEquals(getRateLimiter(topic).get().getDispatchRateOnByte(), byteRate); + + @Cleanup + PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()) + .build(); + final AtomicInteger totalReceived = new AtomicInteger(0); + + @Cleanup + Consumer ignored = client2.newConsumer().topic(topicName).subscriptionName("sub2-in-cluster2") + .messageListener((c1, msg) -> { + Assert.assertNotNull(msg, "Message cannot be null"); + String receivedMessage = new String(msg.getData()); + log.debug("Received message [{}] in the listener", receivedMessage); + totalReceived.incrementAndGet(); + }).subscribe(); + + // The total bytes is 5 times the rate limit value. + int numMessages = byteRate / payloadSize * 5; + for (int i = 0; i < numMessages * payloadSize; i++) { + producer.send(new byte[payloadSize]); + } + + Awaitility.await().pollDelay(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // The rate limit occurs in the next reading cycle, so a value fault tolerance needs to be added. + assertThat(totalReceived.get()).isLessThan((byteRate / payloadSize) + 2); + }); + } + + private static Optional getRateLimiter(PersistentTopic topic) { + return topic.getReplicators().values().stream().findFirst().map(Replicator::getRateLimiter).orElseThrow(); + } + private static final Logger log = LoggerFactory.getLogger(ReplicatorRateLimiterTest.class); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java index 176eab0e94b3d..aac7a85f477c5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java @@ -18,13 +18,15 @@ */ package org.apache.pulsar.broker.service; +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat; import static org.apache.pulsar.broker.BrokerTestUtil.newUniqueName; -import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.retryStrategically; +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricDoubleGaugeValue; +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongSumValue; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; @@ -33,6 +35,7 @@ import com.google.common.collect.Sets; import com.scurrilous.circe.checksum.Crc32cIntChecksum; import io.netty.buffer.ByteBuf; +import io.opentelemetry.api.common.Attributes; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; @@ -44,13 +47,11 @@ import java.util.SortedSet; import java.util.TreeSet; import java.util.UUID; -import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -62,15 +63,18 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerException.CursorAlreadyClosedException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.lang3.RandomUtils; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.service.BrokerServiceException.NamingException; import org.apache.pulsar.broker.service.persistent.PersistentReplicator; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.stats.OpenTelemetryReplicatorStats; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; @@ -83,6 +87,7 @@ import org.apache.pulsar.client.api.RawMessage; import org.apache.pulsar.client.api.RawReader; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.TypedMessageBuilder; import org.apache.pulsar.client.api.schema.GenericRecord; @@ -104,7 +109,7 @@ import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.protocol.Commands; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; import org.apache.pulsar.schema.Schemas; import org.awaitility.Awaitility; import org.awaitility.reflect.WhiteboxImpl; @@ -129,9 +134,11 @@ public class ReplicatorTest extends ReplicatorTestBase { @BeforeMethod(alwaysRun = true) public void beforeMethod(Method m) throws Exception { methodName = m.getName(); - admin1.namespaces().removeBacklogQuota("pulsar/ns"); - admin1.namespaces().removeBacklogQuota("pulsar/ns1"); - admin1.namespaces().removeBacklogQuota("pulsar/global/ns"); + if (admin1 != null) { + admin1.namespaces().removeBacklogQuota("pulsar/ns"); + admin1.namespaces().removeBacklogQuota("pulsar/ns1"); + admin1.namespaces().removeBacklogQuota("pulsar/global/ns"); + } } @Override @@ -151,107 +158,49 @@ public Object[][] partitionedTopicProvider() { return new Object[][] { { Boolean.TRUE }, { Boolean.FALSE } }; } - @Test - public void testConfigChange() throws Exception { - log.info("--- Starting ReplicatorTest::testConfigChange ---"); - // This test is to verify that the config change on global namespace is successfully applied in broker during - // runtime. - // Run a set of producer tasks to create the topics - List> results = new ArrayList<>(); - for (int i = 0; i < 10; i++) { - final TopicName dest = TopicName.get(BrokerTestUtil.newUniqueName("persistent://pulsar/ns/topic-" + i)); - - results.add(executor.submit(new Callable() { - @Override - public Void call() throws Exception { - - @Cleanup - MessageProducer producer = new MessageProducer(url1, dest); - log.info("--- Starting producer --- " + url1); - - @Cleanup - MessageConsumer consumer = new MessageConsumer(url1, dest); - log.info("--- Starting Consumer --- " + url1); - - producer.produce(2); - consumer.receive(2); - return null; - } - })); - } - - for (Future result : results) { - try { - result.get(); - } catch (Exception e) { - log.error("exception in getting future result ", e); - fail(String.format("replication test failed with %s exception", e.getMessage())); - } - } - - Thread.sleep(1000L); - // Make sure that the internal replicators map contains remote cluster info - ConcurrentOpenHashMap replicationClients1 = ns1.getReplicationClients(); - ConcurrentOpenHashMap replicationClients2 = ns2.getReplicationClients(); - ConcurrentOpenHashMap replicationClients3 = ns3.getReplicationClients(); - - Assert.assertNotNull(replicationClients1.get("r2")); - Assert.assertNotNull(replicationClients1.get("r3")); - Assert.assertNotNull(replicationClients2.get("r1")); - Assert.assertNotNull(replicationClients2.get("r3")); - Assert.assertNotNull(replicationClients3.get("r1")); - Assert.assertNotNull(replicationClients3.get("r2")); - - // Case 1: Update the global namespace replication configuration to only contains the local cluster itself - admin1.namespaces().setNamespaceReplicationClusters("pulsar/ns", Sets.newHashSet("r1")); - - // Wait for config changes to be updated. - Thread.sleep(1000L); - - // Make sure that the internal replicators map still contains remote cluster info - Assert.assertNotNull(replicationClients1.get("r2")); - Assert.assertNotNull(replicationClients1.get("r3")); - Assert.assertNotNull(replicationClients2.get("r1")); - Assert.assertNotNull(replicationClients2.get("r3")); - Assert.assertNotNull(replicationClients3.get("r1")); - Assert.assertNotNull(replicationClients3.get("r2")); - - // Case 2: Update the configuration back - admin1.namespaces().setNamespaceReplicationClusters("pulsar/ns", Sets.newHashSet("r1", "r2", "r3")); - - // Wait for config changes to be updated. - Thread.sleep(1000L); - - // Make sure that the internal replicators map still contains remote cluster info - Assert.assertNotNull(replicationClients1.get("r2")); - Assert.assertNotNull(replicationClients1.get("r3")); - Assert.assertNotNull(replicationClients2.get("r1")); - Assert.assertNotNull(replicationClients2.get("r3")); - Assert.assertNotNull(replicationClients3.get("r1")); - Assert.assertNotNull(replicationClients3.get("r2")); - - // Case 3: TODO: Once automatic cleanup is implemented, add tests case to verify auto removal of clusters - } - @Test(timeOut = 10000) public void activeBrokerParse() throws Exception { pulsar1.getConfiguration().setAuthorizationEnabled(true); //init clusterData - String cluster2ServiceUrls = String.format("%s,localhost:1234,localhost:5678,localhost:5677,localhost:5676", - pulsar2.getWebServiceAddress()); - ClusterData cluster2Data = ClusterData.builder().serviceUrl(cluster2ServiceUrls).build(); + ClusterData cluster2Data = ClusterData.builder().serviceUrl(pulsar2.getWebServiceAddress()).build(); String cluster2 = "activeCLuster2"; admin2.clusters().createCluster(cluster2, cluster2Data); Awaitility.await().until(() -> admin2.clusters().getCluster(cluster2) != null); List list = admin1.brokers().getActiveBrokers(cluster2); - assertEquals(list.get(0), url2.toString().replace("http://", "")); + assertEquals(list.get(0), pulsar2.getBrokerId()); //restore configuration pulsar1.getConfiguration().setAuthorizationEnabled(false); } + @Test + public void testForcefullyTopicDeletion() throws Exception { + log.info("--- Starting ReplicatorTest::testForcefullyTopicDeletion ---"); + + final String namespace = BrokerTestUtil.newUniqueName("pulsar/removeClusterTest"); + admin1.namespaces().createNamespace(namespace); + admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1")); + + final String topicName = "persistent://" + namespace + "/topic"; + + @Cleanup + PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS) + .build(); + + ProducerImpl producer1 = (ProducerImpl) client1.newProducer().topic(topicName) + .enableBatching(false).messageRoutingMode(MessageRoutingMode.SinglePartition).create(); + producer1.close(); + + admin1.topics().delete(topicName, true); + + MockedPulsarServiceBaseTest + .retryStrategically((test) -> !pulsar1.getBrokerService().getTopics().containsKey(topicName), 50, 150); + + Assert.assertFalse(pulsar1.getBrokerService().getTopics().containsKey(topicName)); + } + @SuppressWarnings("unchecked") @Test(timeOut = 30000) public void testConcurrentReplicator() throws Exception { @@ -284,11 +233,7 @@ public void testConcurrentReplicator() throws Exception { final Method startRepl = PersistentTopic.class.getDeclaredMethod("startReplicator", String.class); startRepl.setAccessible(true); - Field replClientField = BrokerService.class.getDeclaredField("replicationClients"); - replClientField.setAccessible(true); - ConcurrentOpenHashMap replicationClients = - (ConcurrentOpenHashMap) replClientField - .get(pulsar1.getBrokerService()); + final var replicationClients = pulsar1.getBrokerService().getReplicationClients(); replicationClients.put("r3", pulsarClient); admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2", "r3")); @@ -666,8 +611,8 @@ public void testFailures() { .get(BrokerTestUtil.newUniqueName("persistent://pulsar/ns/res-cons-id-")); // Create another consumer using replication prefix as sub id + @Cleanup MessageConsumer consumer = new MessageConsumer(url2, dest, "pulsar.repl."); - consumer.close(); } catch (Exception e) { // SUCCESS @@ -691,7 +636,7 @@ public void testReplicatePeekAndSkip() throws Exception { producer1.produce(2); PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString()).get(); PersistentReplicator replicator = (PersistentReplicator) topic.getReplicators() - .get(topic.getReplicators().keys().get(0)); + .get(topic.getReplicators().keySet().stream().toList().get(0)); replicator.skipMessages(2); CompletableFuture result = replicator.peekNthMessage(1); Entry entry = result.get(50, TimeUnit.MILLISECONDS); @@ -718,13 +663,43 @@ public void testReplicatorClearBacklog() throws Exception { producer1.produce(2); PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString()).get(); PersistentReplicator replicator = (PersistentReplicator) spy( - topic.getReplicators().get(topic.getReplicators().keys().get(0))); + topic.getReplicators().get(topic.getReplicators().keySet().stream().toList().get(0))); replicator.readEntriesFailed(new ManagedLedgerException.InvalidCursorPositionException("failed"), null); replicator.clearBacklog().get(); Thread.sleep(100); replicator.updateRates(); // for code-coverage replicator.expireMessages(1); // for code-coverage - ReplicatorStats status = replicator.getStats(); + ReplicatorStats status = replicator.computeStats(); + assertEquals(status.getReplicationBacklog(), 0); + } + + + @Test(timeOut = 30000) + public void testResetReplicatorSubscriptionPosition() throws Exception { + final TopicName dest = TopicName + .get(BrokerTestUtil.newUniqueName("persistent://pulsar/ns/resetReplicatorSubscription")); + + @Cleanup + MessageProducer producer1 = new MessageProducer(url1, dest); + + // Produce from cluster1 and consume from the rest + for (int i = 0; i < 10; i++) { + producer1.produce(2); + } + + PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString()).get(); + + PersistentReplicator replicator = (PersistentReplicator) spy( + topic.getReplicators().get(topic.getReplicators().keySet().stream().toList().get(0))); + + MessageId id = topic.getLastMessageId().get(); + admin1.topics().expireMessages(dest.getPartitionedTopicName(), + replicator.getCursor().getName(), + id,false); + + replicator.updateRates(); + + ReplicatorStats status = replicator.computeStats(); assertEquals(status.getReplicationBacklog(), 0); } @@ -819,7 +794,7 @@ public void testDeleteReplicatorFailure() throws Exception { @Cleanup MessageProducer producer1 = new MessageProducer(url1, dest); PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(topicName).get(); - final String replicatorClusterName = topic.getReplicators().keys().get(0); + final String replicatorClusterName = topic.getReplicators().keySet().stream().toList().get(0); ManagedLedgerImpl ledger = (ManagedLedgerImpl) topic.getManagedLedger(); CountDownLatch latch = new CountDownLatch(1); // delete cursor already : so next time if topic.removeReplicator will get exception but then it should @@ -860,13 +835,13 @@ public void testReplicatorProducerClosing() throws Exception { @Cleanup MessageProducer producer1 = new MessageProducer(url1, dest); PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(topicName).get(); - final String replicatorClusterName = topic.getReplicators().keys().get(0); + final String replicatorClusterName = topic.getReplicators().keySet().stream().toList().get(0); Replicator replicator = topic.getPersistentReplicator(replicatorClusterName); pulsar2.close(); pulsar2 = null; pulsar3.close(); pulsar3 = null; - replicator.disconnect(false); + replicator.terminate(); Thread.sleep(100); Field field = AbstractReplicator.class.getDeclaredField("producer"); field.setAccessible(true); @@ -1024,14 +999,28 @@ public void testResumptionAfterBacklogRelaxed() throws Exception { Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000); - assertEquals(replicator.getStats().replicationBacklog, 0); + assertEquals(replicator.computeStats().replicationBacklog, 0); + var attributes = Attributes.of( + OpenTelemetryAttributes.PULSAR_DOMAIN, dest.getDomain().value(), + OpenTelemetryAttributes.PULSAR_TENANT, dest.getTenant(), + OpenTelemetryAttributes.PULSAR_NAMESPACE, dest.getNamespace(), + OpenTelemetryAttributes.PULSAR_TOPIC, dest.getPartitionedTopicName(), + OpenTelemetryAttributes.PULSAR_REPLICATION_REMOTE_CLUSTER_NAME, cluster2 + ); + var metrics = metricReader1.collectAllMetrics(); + assertMetricLongSumValue(metrics, OpenTelemetryReplicatorStats.BACKLOG_COUNTER, attributes, 0); + assertMetricDoubleGaugeValue(metrics, OpenTelemetryReplicatorStats.DELAY_GAUGE, attributes, 0.0); // Next message will not be replicated, because r2 has reached the quota producer1.produce(1); Thread.sleep(500); - assertEquals(replicator.getStats().replicationBacklog, 1); + assertEquals(replicator.computeStats().replicationBacklog, 1); + metrics = metricReader1.collectAllMetrics(); + assertMetricLongSumValue(metrics, OpenTelemetryReplicatorStats.BACKLOG_COUNTER, attributes, 1); + assertMetricDoubleGaugeValue(metrics, OpenTelemetryReplicatorStats.DELAY_GAUGE, attributes, + aDouble -> assertThat(aDouble).isPositive()); // Consumer will now drain 1 message and the replication backlog will be cleared consumer2.receive(1); @@ -1040,13 +1029,16 @@ public void testResumptionAfterBacklogRelaxed() throws Exception { consumer2.receive(1); int retry = 10; - for (int i = 0; i < retry && replicator.getStats().replicationBacklog > 0; i++) { + for (int i = 0; i < retry && replicator.computeStats().replicationBacklog > 0; i++) { if (i != retry - 1) { Thread.sleep(100); } } - assertEquals(replicator.getStats().replicationBacklog, 0); + assertEquals(replicator.computeStats().replicationBacklog, 0); + metrics = metricReader1.collectAllMetrics(); + assertMetricLongSumValue(metrics, OpenTelemetryReplicatorStats.BACKLOG_COUNTER, attributes, 0); + assertMetricDoubleGaugeValue(metrics, OpenTelemetryReplicatorStats.DELAY_GAUGE, attributes, 0.0); } } @@ -1239,7 +1231,7 @@ public void testReplicatedCluster() throws Exception { log.info("--- Starting ReplicatorTest::testReplicatedCluster ---"); - final String namespace = "pulsar/global/repl"; + final String namespace = BrokerTestUtil.newUniqueName("pulsar/global/repl"); final String topicName = BrokerTestUtil.newUniqueName("persistent://" + namespace + "/topic1"); admin1.namespaces().createNamespace(namespace); admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2", "r3")); @@ -1460,13 +1452,6 @@ public void testCleanupTopic() throws Exception { // Ok } - final CompletableFuture> timedOutTopicFuture = topicFuture; - // timeout topic future should be removed from cache - retryStrategically((test) -> pulsar1.getBrokerService().getTopic(topicName, false) != timedOutTopicFuture, 5, - 1000); - - assertNotEquals(timedOutTopicFuture, pulsar1.getBrokerService().getTopics().get(topicName)); - try { Consumer consumer = client1.newConsumer().topic(topicName).subscriptionType(SubscriptionType.Shared) .subscriptionName("my-subscriber-name").subscribeAsync().get(100, TimeUnit.MILLISECONDS); @@ -1478,6 +1463,7 @@ public void testCleanupTopic() throws Exception { ManagedLedgerImpl ml = (ManagedLedgerImpl) mlFactory.open(topicMlName + "-2"); mlFuture.complete(ml); + // Re-create topic will success. Consumer consumer = client1.newConsumer().topic(topicName).subscriptionName("my-subscriber-name") .subscriptionType(SubscriptionType.Shared).subscribeAsync() .get(2 * topicLoadTimeoutSeconds, TimeUnit.SECONDS); @@ -1646,7 +1632,7 @@ public void testReplicatorWithFailedAck() throws Exception { log.info("--- Starting ReplicatorTest::testReplication ---"); - String namespace = "pulsar/global/ns2"; + String namespace = BrokerTestUtil.newUniqueName("pulsar/global/ns"); admin1.namespaces().createNamespace(namespace, Sets.newHashSet("r1")); final TopicName dest = TopicName .get(BrokerTestUtil.newUniqueName("persistent://" + namespace + "/ackFailedTopic")); @@ -1660,7 +1646,7 @@ public void testReplicatorWithFailedAck() throws Exception { final ManagedCursorImpl cursor = (ManagedCursorImpl) managedLedger.openCursor("pulsar.repl.r2"); final ManagedCursorImpl spyCursor = spy(cursor); managedLedger.getCursors().removeCursor(cursor.getName()); - managedLedger.getCursors().add(spyCursor, PositionImpl.EARLIEST); + managedLedger.getCursors().add(spyCursor, PositionFactory.EARLIEST); AtomicBoolean isMakeAckFail = new AtomicBoolean(false); doAnswer(invocation -> { Position pos = (Position) invocation.getArguments()[0]; @@ -1683,14 +1669,16 @@ public void testReplicatorWithFailedAck() throws Exception { producer1.produce(2); MessageIdImpl lastMessageId = (MessageIdImpl) topic.getLastMessageId().get(); - Position lastPosition = PositionImpl.get(lastMessageId.getLedgerId(), lastMessageId.getEntryId()); - ConcurrentOpenHashMap replicators = topic.getReplicators(); - PersistentReplicator replicator = (PersistentReplicator) replicators.get("r2"); + Position lastPosition = PositionFactory.create(lastMessageId.getLedgerId(), lastMessageId.getEntryId()); Awaitility.await().pollInterval(1, TimeUnit.SECONDS).timeout(30, TimeUnit.SECONDS) - .untilAsserted(() -> assertEquals(org.apache.pulsar.broker.service.AbstractReplicator.State.Started, - replicator.getState())); - assertEquals(replicator.getState(), org.apache.pulsar.broker.service.AbstractReplicator.State.Started); + .ignoreExceptions() + .untilAsserted(() -> { + final var replicators = topic.getReplicators(); + PersistentReplicator replicator = (PersistentReplicator) replicators.get("r2"); + assertEquals(org.apache.pulsar.broker.service.AbstractReplicator.State.Started, + replicator.getState()); + }); // Make sure all the data has replicated to the remote cluster before close the cursor. Awaitility.await().untilAsserted(() -> assertEquals(cursor.getMarkDeletedPosition(), lastPosition)); @@ -1716,7 +1704,7 @@ public void testReplicatorWithFailedAck() throws Exception { @Test public void testWhenUpdateReplicationCluster() throws Exception { log.info("--- testWhenUpdateReplicationCluster ---"); - String namespace = "pulsar/ns2"; + String namespace = BrokerTestUtil.newUniqueName("pulsar/ns");; admin1.namespaces().createNamespace(namespace); admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2")); final TopicName dest = TopicName.get( @@ -1745,12 +1733,12 @@ public void testWhenUpdateReplicationCluster() throws Exception { @Test public void testReplicatorProducerNotExceed() throws Exception { log.info("--- testReplicatorProducerNotExceed ---"); - String namespace1 = "pulsar/ns11"; + String namespace1 = BrokerTestUtil.newUniqueName("pulsar/ns1"); admin1.namespaces().createNamespace(namespace1); admin1.namespaces().setNamespaceReplicationClusters(namespace1, Sets.newHashSet("r1", "r2")); final TopicName dest1 = TopicName.get( BrokerTestUtil.newUniqueName("persistent://" + namespace1 + "/testReplicatorProducerNotExceed1")); - String namespace2 = "pulsar/ns22"; + String namespace2 = BrokerTestUtil.newUniqueName("pulsar/ns2"); admin2.namespaces().createNamespace(namespace2); admin2.namespaces().setNamespaceReplicationClusters(namespace2, Sets.newHashSet("r1", "r2")); final TopicName dest2 = TopicName.get( @@ -1773,4 +1761,205 @@ public void testReplicatorProducerNotExceed() throws Exception { Assert.assertThrows(PulsarClientException.ProducerBusyException.class, () -> new MessageProducer(url2, dest2)); } + + @Test + public void testReplicatorWithTTL() throws Exception { + log.info("--- Starting ReplicatorTest::testReplicatorWithTTL ---"); + + final String cluster1 = pulsar1.getConfig().getClusterName(); + final String cluster2 = pulsar2.getConfig().getClusterName(); + final String namespace = BrokerTestUtil.newUniqueName("pulsar/ns"); + final TopicName topic = TopicName + .get(BrokerTestUtil.newUniqueName("persistent://" + namespace + "/testReplicatorWithTTL")); + admin1.namespaces().createNamespace(namespace, Sets.newHashSet(cluster1, cluster2)); + admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet(cluster1, cluster2)); + admin1.topics().createNonPartitionedTopic(topic.toString()); + admin1.topicPolicies().setMessageTTL(topic.toString(), 1); + + @Cleanup + PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS) + .build(); + + @Cleanup + Producer persistentProducer1 = client1.newProducer().topic(topic.toString()).create(); + persistentProducer1.send("V1".getBytes()); + + waitReplicateFinish(topic, admin1); + + PersistentTopic persistentTopic = + (PersistentTopic) pulsar1.getBrokerService().getTopicReference(topic.toString()).get(); + persistentTopic.getReplicators().forEach((cluster, replicator) -> { + PersistentReplicator persistentReplicator = (PersistentReplicator) replicator; + // Pause replicator + pauseReplicator(persistentReplicator); + }); + + persistentProducer1.send("V2".getBytes()); + persistentProducer1.send("V3".getBytes()); + + Thread.sleep(1000); + + admin1.topics().expireMessagesForAllSubscriptions(topic.toString(), 1); + + persistentTopic.getReplicators().forEach((cluster, replicator) -> { + PersistentReplicator persistentReplicator = (PersistentReplicator) replicator; + persistentReplicator.startProducer(); + }); + + waitReplicateFinish(topic, admin1); + + persistentProducer1.send("V4".getBytes()); + + waitReplicateFinish(topic, admin1); + + @Cleanup + PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()).statsInterval(0, TimeUnit.SECONDS) + .build(); + + @Cleanup + Consumer consumer = client2.newConsumer().topic(topic.toString()) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscriptionName("sub").subscribe(); + + List result = new ArrayList<>(); + while (true) { + Message receive = consumer.receive(2, TimeUnit.SECONDS); + if (receive == null) { + break; + } + result.add(new String(receive.getValue())); + } + + assertEquals(result, Lists.newArrayList("V1", "V2", "V3", "V4")); + } + + @Test + public void testReplicationMetrics() throws Exception { + var destTopicName = TopicName.get(BrokerTestUtil.newUniqueName("persistent://pulsar/ns/replicationMetrics")); + + @Cleanup + var producer1 = new MessageProducer(url1, destTopicName); + + @Cleanup + var consumer1 = new MessageConsumer(url1, destTopicName); + + @Cleanup + var consumer2 = new MessageConsumer(url2, destTopicName); + + // Produce from cluster 1 and consume from the 1 and 2. + producer1.produce(3); + consumer1.receive(2); + consumer2.receive(1); + + { + // Validate replicator metrics on cluster 1 from cluster 2 + var attributes = Attributes.of( + OpenTelemetryAttributes.PULSAR_DOMAIN, destTopicName.getDomain().value(), + OpenTelemetryAttributes.PULSAR_TENANT, destTopicName.getTenant(), + OpenTelemetryAttributes.PULSAR_NAMESPACE, destTopicName.getNamespace(), + OpenTelemetryAttributes.PULSAR_TOPIC, destTopicName.getPartitionedTopicName(), + OpenTelemetryAttributes.PULSAR_REPLICATION_REMOTE_CLUSTER_NAME, cluster2 + ); + var metrics = metricReader1.collectAllMetrics(); + assertMetricLongSumValue(metrics, OpenTelemetryReplicatorStats.MESSAGE_OUT_COUNTER, attributes, 3); + assertMetricLongSumValue(metrics, OpenTelemetryReplicatorStats.BYTES_OUT_COUNTER, attributes, + aLong -> assertThat(aLong).isPositive()); + + var topicOpt = pulsar1.getBrokerService().getTopicReference(destTopicName.toString()); + assertThat(topicOpt).isPresent(); + var topic = topicOpt.get(); + var persistentReplicators = topic.getReplicators() + .values() + .stream() + .map(PersistentReplicator.class::cast) + .toList(); + persistentReplicators.forEach(this::pauseReplicator); + producer1.produce(5); + Awaitility.await().untilAsserted(() -> { + persistentReplicators.forEach(repl -> repl.expireMessages(1)); + assertMetricLongSumValue(metricReader1.collectAllMetrics(), + OpenTelemetryReplicatorStats.EXPIRED_COUNTER, + attributes, 5); + }); + } + + { + // Validate replicator metrics on cluster 2 from cluster 1 + var attributes = Attributes.of( + OpenTelemetryAttributes.PULSAR_DOMAIN, destTopicName.getDomain().value(), + OpenTelemetryAttributes.PULSAR_TENANT, destTopicName.getTenant(), + OpenTelemetryAttributes.PULSAR_NAMESPACE, destTopicName.getNamespace(), + OpenTelemetryAttributes.PULSAR_TOPIC, destTopicName.getPartitionedTopicName(), + OpenTelemetryAttributes.PULSAR_REPLICATION_REMOTE_CLUSTER_NAME, cluster1 + ); + var metrics = metricReader2.collectAllMetrics(); + assertMetricLongSumValue(metrics, OpenTelemetryReplicatorStats.MESSAGE_IN_COUNTER, attributes, 3); + assertMetricLongSumValue(metrics, OpenTelemetryReplicatorStats.BYTES_IN_COUNTER, attributes, + aLong -> assertThat(aLong).isPositive()); + } + } + + @Test + public void testEnableReplicationWithNamespaceAllowedClustersPolices() throws Exception { + log.info("--- testEnableReplicationWithNamespaceAllowedClustersPolices ---"); + String namespace1 = "pulsar/ns" + RandomUtils.nextLong(); + admin1.namespaces().createNamespace(namespace1); + admin2.namespaces().createNamespace(namespace1 + "init_cluster_node"); + admin1.namespaces().setNamespaceAllowedClusters(namespace1, Sets.newHashSet("r1", "r2")); + final TopicName topicName = TopicName.get( + BrokerTestUtil.newUniqueName("persistent://" + namespace1 + "/testReplicatorProducerNotExceed1")); + + @Cleanup PulsarClient client1 = PulsarClient + .builder() + .serviceUrl(pulsar1.getBrokerServiceUrl()) + .build(); + @Cleanup Producer producer = client1 + .newProducer() + .topic(topicName.toString()) + .create(); + producer.newMessage().send(); + // Enable replication at the topic level in the cluster1. + admin1.topics().setReplicationClusters(topicName.toString(), List.of("r1", "r2")); + + PersistentTopic persistentTopic1 = (PersistentTopic) pulsar1.getBrokerService().getTopic(topicName.toString(), + false) + .get() + .get(); + // Verify the replication from cluster1 to cluster2 is ready, but the replication form the cluster2 to cluster1 + // is not ready. + Awaitility.await().untilAsserted(() -> { + final var replicatorMap = persistentTopic1.getReplicators(); + assertEquals(replicatorMap.size(), 1); + Replicator replicator = replicatorMap.get(replicatorMap.keySet().stream().toList().get(0)); + assertTrue(replicator.isConnected()); + }); + + PersistentTopic persistentTopic2 = (PersistentTopic) pulsar2.getBrokerService().getTopic(topicName.toString(), + false) + .get() + .get(); + + Awaitility.await().untilAsserted(() -> { + final var replicatorMap = persistentTopic2.getReplicators(); + assertEquals(replicatorMap.size(), 0); + }); + // Enable replication at the topic level in the cluster2. + admin2.topics().setReplicationClusters(topicName.toString(), List.of("r1", "r2")); + // Verify the replication between cluster1 and cluster2 is ready. + Awaitility.await().untilAsserted(() -> { + final var replicatorMap = persistentTopic2.getReplicators(); + assertEquals(replicatorMap.size(), 1); + Replicator replicator = replicatorMap.get(replicatorMap.keySet().stream().toList().get(0)); + assertTrue(replicator.isConnected()); + }); + } + + private void pauseReplicator(PersistentReplicator replicator) { + Awaitility.await().untilAsserted(() -> { + assertTrue(replicator.isConnected()); + }); + replicator.closeProducerAsync(true); + Awaitility.await().untilAsserted(() -> { + assertFalse(replicator.isConnected()); + }); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTestBase.java index b83e8ac9d2dbf..33877b681184f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTestBase.java @@ -20,12 +20,11 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; - -import com.google.common.io.Resources; +import static org.testng.Assert.assertNull; import com.google.common.collect.Sets; - +import com.google.common.io.Resources; import io.netty.util.concurrent.DefaultThreadFactory; - +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; import java.net.URL; import java.util.Optional; import java.util.Set; @@ -34,12 +33,9 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; - import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.TopicType; -import org.apache.pulsar.tests.TestRetrySupport; +import org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -50,7 +46,11 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.TypedMessageBuilder; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.policies.data.TopicType; +import org.apache.pulsar.functions.worker.WorkerConfig; +import org.apache.pulsar.tests.TestRetrySupport; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.apache.pulsar.zookeeper.ZookeeperServerTest; import org.slf4j.Logger; @@ -62,6 +62,7 @@ public abstract class ReplicatorTestBase extends TestRetrySupport { ServiceConfiguration config1 = new ServiceConfiguration(); PulsarService pulsar1; BrokerService ns1; + protected InMemoryMetricReader metricReader1; PulsarAdmin admin1; LocalBookkeeperEnsemble bkEnsemble1; @@ -73,6 +74,7 @@ public abstract class ReplicatorTestBase extends TestRetrySupport { BrokerService ns2; PulsarAdmin admin2; LocalBookkeeperEnsemble bkEnsemble2; + protected InMemoryMetricReader metricReader2; URL url3; URL urlTls3; @@ -81,6 +83,15 @@ public abstract class ReplicatorTestBase extends TestRetrySupport { BrokerService ns3; PulsarAdmin admin3; LocalBookkeeperEnsemble bkEnsemble3; + protected InMemoryMetricReader metricReader3; + + URL url4; + URL urlTls4; + ServiceConfiguration config4 = new ServiceConfiguration(); + PulsarService pulsar4; + PulsarAdmin admin4; + LocalBookkeeperEnsemble bkEnsemble4; + protected InMemoryMetricReader metricReader4; ZookeeperServerTest globalZkS; @@ -111,6 +122,12 @@ public abstract class ReplicatorTestBase extends TestRetrySupport { protected final String cluster1 = "r1"; protected final String cluster2 = "r2"; protected final String cluster3 = "r3"; + protected final String cluster4 = "r4"; + protected String loadManagerClassName; + + protected String getLoadManagerClassName() { + return loadManagerClassName; + } // Default frequency public int getBrokerServicePurgeInactiveFrequency() { @@ -140,7 +157,8 @@ protected void setup() throws Exception { // completely // independent config objects instead of referring to the same properties object setConfig1DefaultValue(); - pulsar1 = new PulsarService(config1); + metricReader1 = InMemoryMetricReader.create(); + pulsar1 = buildPulsarService(config1, metricReader1); pulsar1.start(); ns1 = pulsar1.getBrokerService(); @@ -155,7 +173,8 @@ protected void setup() throws Exception { bkEnsemble2.start(); setConfig2DefaultValue(); - pulsar2 = new PulsarService(config2); + metricReader2 = InMemoryMetricReader.create(); + pulsar2 = buildPulsarService(config2, metricReader2); pulsar2.start(); ns2 = pulsar2.getBrokerService(); @@ -170,7 +189,8 @@ protected void setup() throws Exception { bkEnsemble3.start(); setConfig3DefaultValue(); - pulsar3 = new PulsarService(config3); + metricReader3 = InMemoryMetricReader.create(); + pulsar3 = buildPulsarService(config3, metricReader3); pulsar3.start(); ns3 = pulsar3.getBrokerService(); @@ -178,6 +198,22 @@ protected void setup() throws Exception { urlTls3 = new URL(pulsar3.getWebServiceAddressTls()); admin3 = PulsarAdmin.builder().serviceHttpUrl(url3.toString()).build(); + // Start region 4 + + // Start zk & bks + bkEnsemble4 = new LocalBookkeeperEnsemble(3, 0, () -> 0); + bkEnsemble4.start(); + + setConfig4DefaultValue(); + metricReader4 = InMemoryMetricReader.create(); + pulsar4 = buildPulsarService(config4, metricReader4); + pulsar4.start(); + + url4 = new URL(pulsar4.getWebServiceAddress()); + urlTls4 = new URL(pulsar4.getWebServiceAddressTls()); + admin4 = PulsarAdmin.builder().serviceHttpUrl(url4.toString()).build(); + + // Provision the global namespace admin1.clusters().createCluster(cluster1, ClusterData.builder() .serviceUrl(url1.toString()) @@ -230,18 +266,45 @@ protected void setup() throws Exception { .brokerClientTlsTrustStorePassword(keyStorePassword) .brokerClientTlsTrustStoreType(keyStoreType) .build()); + admin4.clusters().createCluster(cluster4, ClusterData.builder() + .serviceUrlTls(urlTls4.toString()) + .brokerServiceUrlTls(pulsar4.getBrokerServiceUrlTls()) + .brokerClientTlsEnabled(true) + .brokerClientCertificateFilePath(clientCertFilePath) + .brokerClientKeyFilePath(clientKeyFilePath) + .brokerClientTrustCertsFilePath(caCertFilePath) + .brokerClientTlsEnabledWithKeyStore(tlsWithKeyStore) + .brokerClientTlsKeyStore(clientKeyStorePath) + .brokerClientTlsKeyStorePassword(keyStorePassword) + .brokerClientTlsKeyStoreType(keyStoreType) + .brokerClientTlsTrustStore(clientTrustStorePath) + .brokerClientTlsTrustStorePassword(keyStorePassword) + .brokerClientTlsTrustStoreType(keyStoreType) + .build()); - admin1.tenants().createTenant("pulsar", - new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet("r1", "r2", "r3"))); + updateTenantInfo("pulsar", + new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), + Sets.newHashSet("r1", "r2", "r3"))); admin1.namespaces().createNamespace("pulsar/ns", Sets.newHashSet("r1", "r2", "r3")); admin1.namespaces().createNamespace("pulsar/ns1", Sets.newHashSet("r1", "r2")); assertEquals(admin2.clusters().getCluster(cluster1).getServiceUrl(), url1.toString()); assertEquals(admin2.clusters().getCluster(cluster2).getServiceUrl(), url2.toString()); assertEquals(admin2.clusters().getCluster(cluster3).getServiceUrl(), url3.toString()); + assertNull(admin2.clusters().getCluster(cluster4).getServiceUrl()); assertEquals(admin2.clusters().getCluster(cluster1).getBrokerServiceUrl(), pulsar1.getBrokerServiceUrl()); assertEquals(admin2.clusters().getCluster(cluster2).getBrokerServiceUrl(), pulsar2.getBrokerServiceUrl()); assertEquals(admin2.clusters().getCluster(cluster3).getBrokerServiceUrl(), pulsar3.getBrokerServiceUrl()); + assertNull(admin2.clusters().getCluster(cluster4).getBrokerServiceUrl()); + + assertEquals(admin2.clusters().getCluster(cluster1).getServiceUrlTls(), urlTls1.toString()); + assertEquals(admin2.clusters().getCluster(cluster2).getServiceUrlTls(), urlTls2.toString()); + assertEquals(admin2.clusters().getCluster(cluster3).getServiceUrlTls(), urlTls3.toString()); + assertEquals(admin2.clusters().getCluster(cluster4).getServiceUrlTls(), urlTls4.toString()); + assertEquals(admin2.clusters().getCluster(cluster1).getBrokerServiceUrlTls(), pulsar1.getBrokerServiceUrlTls()); + assertEquals(admin2.clusters().getCluster(cluster2).getBrokerServiceUrlTls(), pulsar2.getBrokerServiceUrlTls()); + assertEquals(admin2.clusters().getCluster(cluster3).getBrokerServiceUrlTls(), pulsar3.getBrokerServiceUrlTls()); + assertEquals(admin2.clusters().getCluster(cluster4).getBrokerServiceUrlTls(), pulsar4.getBrokerServiceUrlTls()); // Also create V1 namespace for compatibility check admin1.clusters().createCluster("global", ClusterData.builder() @@ -256,8 +319,16 @@ protected void setup() throws Exception { } + private PulsarService buildPulsarService(ServiceConfiguration config, InMemoryMetricReader metricReader) { + return new PulsarService(config, + new WorkerConfig(), + Optional.empty(), + exitCode -> log.info("Pulsar service finished with exit code {}", exitCode), + BrokerOpenTelemetryTestUtil.getOpenTelemetrySdkBuilderConsumer(metricReader)); + } + public void setConfig3DefaultValue() { - setConfigDefaults(config3, "r3", bkEnsemble3); + setConfigDefaults(config3, cluster3, bkEnsemble3); config3.setTlsEnabled(true); } @@ -269,6 +340,11 @@ public void setConfig2DefaultValue() { setConfigDefaults(config2, cluster2, bkEnsemble2); } + public void setConfig4DefaultValue() { + setConfigDefaults(config4, cluster4, bkEnsemble4); + config4.setEnableReplicatedSubscriptions(false); + } + private void setConfigDefaults(ServiceConfiguration config, String clusterName, LocalBookkeeperEnsemble bookkeeperEnsemble) { config.setClusterName(clusterName); @@ -299,21 +375,23 @@ private void setConfigDefaults(ServiceConfiguration config, String clusterName, config.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED); config.setEnableReplicatedSubscriptions(true); config.setReplicatedSubscriptionsSnapshotFrequencyMillis(1000); + config.setLoadManagerClassName(getLoadManagerClassName()); } public void resetConfig1() { config1 = new ServiceConfiguration(); - setConfig1DefaultValue(); } public void resetConfig2() { config2 = new ServiceConfiguration(); - setConfig2DefaultValue(); } public void resetConfig3() { config3 = new ServiceConfiguration(); - setConfig3DefaultValue(); + } + + public void resetConfig4() { + config4 = new ServiceConfiguration(); } private int inSec(int time, TimeUnit unit) { @@ -329,28 +407,90 @@ protected void cleanup() throws Exception { executor = null; } - admin1.close(); - admin2.close(); - admin3.close(); + if (admin1 != null) { + admin1.close(); + admin1 = null; + } + if (admin2 != null) { + admin2.close(); + admin2 = null; + } + if (admin3 != null) { + admin3.close(); + admin3 = null; + } + if (admin4 != null) { + admin4.close(); + admin4 = null; + } + if (metricReader4 != null) { + metricReader4.close(); + metricReader4 = null; + } + if (metricReader3 != null) { + metricReader3.close(); + metricReader3 = null; + } + if (metricReader2 != null) { + metricReader2.close(); + metricReader2 = null; + } + if (metricReader1 != null) { + metricReader1.close(); + metricReader1 = null; + } + + if (pulsar4 != null) { + pulsar4.close(); + pulsar4 = null; + } if (pulsar3 != null) { pulsar3.close(); + pulsar3 = null; } if (pulsar2 != null) { pulsar2.close(); + pulsar2 = null; } if (pulsar1 != null) { pulsar1.close(); + pulsar1 = null; } - bkEnsemble1.stop(); - bkEnsemble2.stop(); - bkEnsemble3.stop(); - globalZkS.stop(); + if (bkEnsemble1 != null) { + bkEnsemble1.stop(); + bkEnsemble1 = null; + } + if (bkEnsemble2 != null) { + bkEnsemble2.stop(); + bkEnsemble2 = null; + } + if (bkEnsemble3 != null) { + bkEnsemble3.stop(); + bkEnsemble3 = null; + } + if (bkEnsemble4 != null) { + bkEnsemble4.stop(); + bkEnsemble4 = null; + } + if (globalZkS != null) { + globalZkS.stop(); + globalZkS = null; + } resetConfig1(); resetConfig2(); resetConfig3(); + resetConfig4(); + } + + protected void updateTenantInfo(String tenant, TenantInfoImpl tenantInfo) throws Exception { + if (!admin1.tenants().getTenants().contains(tenant)) { + admin1.tenants().createTenant(tenant, tenantInfo); + } else { + admin1.tenants().updateTenant(tenant, tenantInfo); + } } static class MessageProducer implements AutoCloseable { @@ -365,12 +505,16 @@ static class MessageProducer implements AutoCloseable { this.namespace = dest.getNamespace(); this.topicName = dest.toString(); client = PulsarClient.builder().serviceUrl(url.toString()).statsInterval(0, TimeUnit.SECONDS).build(); - producer = client.newProducer() - .topic(topicName) - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.SinglePartition) - .create(); - + try { + producer = client.newProducer() + .topic(topicName) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + } catch (Exception e) { + client.close(); + throw e; + } } MessageProducer(URL url, final TopicName dest, boolean batch) throws Exception { @@ -383,8 +527,12 @@ static class MessageProducer implements AutoCloseable { .enableBatching(batch) .batchingMaxPublishDelay(1, TimeUnit.SECONDS) .batchingMaxMessages(5); - producer = producerBuilder.create(); - + try { + producer = producerBuilder.create(); + } catch (Exception e) { + client.close(); + throw e; + } } void produceBatch(int messages) throws Exception { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTopicPoliciesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTopicPoliciesTest.java index c0281f073cfd4..ab1f0c0ece2e2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTopicPoliciesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTopicPoliciesTest.java @@ -29,12 +29,9 @@ import java.util.List; import java.util.Set; import java.util.UUID; -import java.util.stream.Collectors; -import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.AutoSubscriptionCreationOverride; @@ -750,7 +747,7 @@ public void testRemoveReplicationClusters() throws Exception { assertNotNull(topicRef); Awaitility.await().untilAsserted(() -> { - List replicaClusters = topicRef.getReplicators().keys().stream().sorted().collect(Collectors.toList()); + List replicaClusters = topicRef.getReplicators().keySet().stream().sorted().toList(); assertEquals(replicaClusters.size(), 1); assertEquals(replicaClusters.toString(), "[r2]"); }); @@ -758,7 +755,7 @@ public void testRemoveReplicationClusters() throws Exception { // removing topic replica cluster policy, so namespace policy should take effect admin1.topics().removeReplicationClusters(persistentTopicName); Awaitility.await().untilAsserted(() -> { - List replicaClusters = topicRef.getReplicators().keys().stream().sorted().collect(Collectors.toList()); + List replicaClusters = topicRef.getReplicators().keySet().stream().sorted().toList(); assertEquals(replicaClusters.size(), 2); assertEquals(replicaClusters.toString(), "[r2, r3]"); }); @@ -792,8 +789,7 @@ public void testReplicateAutoSubscriptionCreation() throws Exception { assertNull(admin3.topicPolicies(true).getAutoSubscriptionCreation(topic, false))); } - private void init(String namespace, String topic) - throws PulsarAdminException, PulsarClientException, PulsarServerException { + private void init(String namespace, String topic) throws Exception { final String cluster2 = pulsar2.getConfig().getClusterName(); final String cluster1 = pulsar1.getConfig().getClusterName(); final String cluster3 = pulsar3.getConfig().getClusterName(); @@ -817,11 +813,9 @@ private void init(String namespace, String topic) pulsar3.getClient().newProducer().topic(topic).create().close(); //init topic policies server - Awaitility.await().ignoreExceptions().untilAsserted(() -> { - assertNull(pulsar1.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic))); - assertNull(pulsar2.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic))); - assertNull(pulsar3.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic))); - }); + TopicPolicyTestUtils.getTopicPolicies(pulsar1.getTopicPoliciesService(), TopicName.get(topic)); + TopicPolicyTestUtils.getTopicPolicies(pulsar2.getTopicPoliciesService(), TopicName.get(topic)); + TopicPolicyTestUtils.getTopicPolicies(pulsar3.getTopicPoliciesService(), TopicName.get(topic)); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ResendRequestTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ResendRequestTest.java index 113d766a8ab64..ee58111ad77ea 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ResendRequestTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ResendRequestTest.java @@ -147,7 +147,7 @@ public void testExclusiveSingleAckedNormalTopic() throws Exception { assertEquals(messageDataHashSet.size(), totalMessages); printIncomingMessageQueue(consumer); - // 9. Calling resend after acking all messages - expectin 0 messages + // 9. Calling resend after acking all messages - expecting 0 messages consumer.redeliverUnacknowledgedMessages(); assertNull(consumer.receive(2000, TimeUnit.MILLISECONDS)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxNonInjectionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxNonInjectionTest.java new file mode 100644 index 0000000000000..3acc941a2c8c2 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxNonInjectionTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.service; + +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.Schema; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class ServerCnxNonInjectionTest extends ProducerConsumerBase { + + @BeforeClass + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test(timeOut = 60 * 1000) + public void testCheckConnectionLivenessAfterClosed() throws Exception { + // Create a ServerCnx + final String tp = BrokerTestUtil.newUniqueName("public/default/tp"); + Producer p = pulsarClient.newProducer(Schema.STRING).topic(tp).create(); + ServerCnx serverCnx = (ServerCnx) pulsar.getBrokerService().getTopic(tp, false).join().get() + .getProducers().values().iterator().next().getCnx(); + // Call "CheckConnectionLiveness" after serverCnx is closed. The resulted future should be done eventually. + p.close(); + serverCnx.close(); + Thread.sleep(1000); + serverCnx.checkConnectionLiveness().join(); + } + +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index c3bab634a42c1..42b52d901e32f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -21,6 +21,7 @@ import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgsRecordingInvocations; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; @@ -45,8 +46,11 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.ChannelHandler; +import io.netty.channel.DefaultChannelId; import io.netty.channel.embedded.EmbeddedChannel; import io.netty.handler.codec.LengthFieldBasedFrameDecoder; +import io.vertx.core.impl.ConcurrentHashSet; +import java.io.Closeable; import java.io.IOException; import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; @@ -60,10 +64,16 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; import java.util.function.Supplier; +import java.util.stream.Collectors; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback; @@ -73,17 +83,18 @@ import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.ManagedLedgerException; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.commons.lang3.mutable.MutableInt; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.TransactionMetadataStoreService; import org.apache.pulsar.broker.auth.MockAlwaysExpiredAuthenticationProvider; +import org.apache.pulsar.broker.auth.MockAuthenticationProvider; import org.apache.pulsar.broker.auth.MockAuthorizationProvider; +import org.apache.pulsar.broker.auth.MockMultiStageAuthenticationProvider; import org.apache.pulsar.broker.auth.MockMutableAuthenticationProvider; import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription; -import org.apache.pulsar.broker.testcontext.PulsarTestContext; -import org.apache.pulsar.broker.TransactionMetadataStoreService; -import org.apache.pulsar.broker.auth.MockAuthenticationProvider; -import org.apache.pulsar.broker.auth.MockMultiStageAuthenticationProvider; import org.apache.pulsar.broker.authentication.AuthenticationProvider; import org.apache.pulsar.broker.authentication.AuthenticationService; import org.apache.pulsar.broker.authorization.AuthorizationService; @@ -93,7 +104,10 @@ import org.apache.pulsar.broker.service.ServerCnx.State; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.service.utils.ClientChannelHelper; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.apache.pulsar.client.api.ProducerAccessMode; import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.client.util.ConsumerName; import org.apache.pulsar.common.api.AuthData; import org.apache.pulsar.common.api.proto.AuthMethod; import org.apache.pulsar.common.api.proto.BaseCommand; @@ -113,6 +127,7 @@ import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespaceResponse; import org.apache.pulsar.common.api.proto.CommandLookupTopicResponse; import org.apache.pulsar.common.api.proto.CommandPartitionedTopicMetadataResponse; +import org.apache.pulsar.common.api.proto.CommandPing; import org.apache.pulsar.common.api.proto.CommandProducerSuccess; import org.apache.pulsar.common.api.proto.CommandSendError; import org.apache.pulsar.common.api.proto.CommandSendReceipt; @@ -135,6 +150,7 @@ import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.Commands.ChecksumType; import org.apache.pulsar.common.protocol.PulsarHandler; +import org.apache.pulsar.common.protocol.schema.EmptyVersion; import org.apache.pulsar.common.topics.TopicList; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap; @@ -149,6 +165,7 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +@Slf4j @SuppressWarnings("unchecked") @Test(groups = "broker") public class ServerCnxTest { @@ -184,10 +201,12 @@ public class ServerCnxTest { private ManagedLedger ledgerMock; private ManagedCursor cursorMock; + private ConcurrentHashSet channelsStoppedAnswerHealthCheck = new ConcurrentHashSet<>(); @BeforeMethod(alwaysRun = true) public void setup() throws Exception { + channelsStoppedAnswerHealthCheck.clear(); svcConfig = new ServiceConfiguration(); svcConfig.setBrokerShutdownTimeoutMs(0L); svcConfig.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); @@ -210,6 +229,8 @@ public void setup() throws Exception { doReturn(CompletableFuture.completedFuture(true)).when(namespaceService).checkTopicOwnership(any()); doReturn(CompletableFuture.completedFuture(topics)).when(namespaceService).getListOfTopics( NamespaceName.get("use", "ns-abc"), CommandGetTopicsOfNamespace.Mode.ALL); + doReturn(CompletableFuture.completedFuture(topics)).when(namespaceService).getListOfUserTopics( + NamespaceName.get("use", "ns-abc"), CommandGetTopicsOfNamespace.Mode.ALL); doReturn(CompletableFuture.completedFuture(topics)).when(namespaceService).getListOfPersistentTopics( NamespaceName.get("use", "ns-abc")); @@ -496,6 +517,41 @@ public void testConnectCommandWithPassingOriginalAuthData() throws Exception { channel.finish(); } + @Test(timeOut = 30000) + public void testConnectCommandWithPassingOriginalAuthDataAndSetAnonymousUserRole() throws Exception { + AuthenticationService authenticationService = mock(AuthenticationService.class); + AuthenticationProvider authenticationProvider = new MockAuthenticationProvider(); + String authMethodName = authenticationProvider.getAuthMethodName(); + + String anonymousUserRole = "admin"; + when(brokerService.getAuthenticationService()).thenReturn(authenticationService); + when(authenticationService.getAuthenticationProvider(authMethodName)).thenReturn(authenticationProvider); + when(authenticationService.getAnonymousUserRole()).thenReturn(Optional.of(anonymousUserRole)); + svcConfig.setAuthenticationEnabled(true); + svcConfig.setAuthenticateOriginalAuthData(true); + svcConfig.setProxyRoles(Collections.singleton("pass.proxy")); + svcConfig.setAnonymousUserRole(anonymousUserRole); + + resetChannel(); + assertTrue(channel.isActive()); + assertEquals(serverCnx.getState(), State.Start); + + // When both the proxy and the broker set the anonymousUserRole option + // the proxy will use anonymousUserRole to delegate the client's role when connecting. + ByteBuf clientCommand = Commands.newConnect(authMethodName, "pass.proxy", 1, null, + null, anonymousUserRole, null, null); + channel.writeInbound(clientCommand); + + Object response1 = getResponse(); + assertTrue(response1 instanceof CommandConnected); + assertEquals(serverCnx.getState(), State.Connected); + assertEquals(serverCnx.getAuthRole(), anonymousUserRole); + assertEquals(serverCnx.getPrincipal(), anonymousUserRole); + assertEquals(serverCnx.getOriginalPrincipal(), anonymousUserRole); + assertTrue(serverCnx.isActive()); + channel.finish(); + } + @Test(timeOut = 30000) public void testConnectCommandWithPassingOriginalPrincipal() throws Exception { AuthenticationService authenticationService = mock(AuthenticationService.class); @@ -927,6 +983,324 @@ public void testVerifyOriginalPrincipalWithAuthDataForwardedFromProxy() throws E })); } + @Test + public void testDuplicateProducer() throws Exception { + final String tName = successTopicName; + final long producerId = 1; + final MutableInt requestId = new MutableInt(1); + final MutableInt epoch = new MutableInt(1); + final Map metadata = Collections.emptyMap(); + final String pName = "p1"; + resetChannel(); + setChannelConnected(); + + // The producer register using the first connection. + ByteBuf cmdProducer1 = Commands.newProducer(tName, producerId, requestId.incrementAndGet(), + pName, false, metadata, null, epoch.incrementAndGet(), false, + ProducerAccessMode.Shared, Optional.empty(), false); + channel.writeInbound(cmdProducer1); + assertTrue(getResponse() instanceof CommandProducerSuccess); + PersistentTopic topicRef = (PersistentTopic) brokerService.getTopicReference(tName).get(); + assertNotNull(topicRef); + assertEquals(topicRef.getProducers().size(), 1); + + // Verify the second producer will be reject due to the previous one still is active. + // Every second try once, total 10 times, all requests should fail. + ClientChannel channel2 = new ClientChannel(); + BackGroundExecutor backGroundExecutor1 = startBackgroundExecutorForEmbeddedChannel(channel); + BackGroundExecutor autoResponseForHeartBeat = autoResponseForHeartBeat(channel, clientChannelHelper); + BackGroundExecutor backGroundExecutor2 = startBackgroundExecutorForEmbeddedChannel(channel2.channel); + setChannelConnected(channel2.serverCnx); + + for (int i = 0; i < 10; i++) { + ByteBuf cmdProducer2 = Commands.newProducer(tName, producerId, requestId.incrementAndGet(), + pName, false, metadata, null, epoch.incrementAndGet(), false, + ProducerAccessMode.Shared, Optional.empty(), false); + channel2.channel.writeInbound(cmdProducer2); + Object response2 = getResponse(channel2.channel, channel2.clientChannelHelper); + assertTrue(response2 instanceof CommandError); + assertEquals(topicRef.getProducers().size(), 1); + assertTrue(channel.isActive()); + Thread.sleep(500); + } + + // cleanup. + autoResponseForHeartBeat.close(); + backGroundExecutor1.close(); + backGroundExecutor2.close(); + channel.finish(); + channel2.close(); + } + + @Test + public void testProducerChangeSocket() throws Exception { + final String tName = successTopicName; + final long producerId = 1; + final MutableInt requestId = new MutableInt(1); + final MutableInt epoch = new MutableInt(1); + final Map metadata = Collections.emptyMap(); + final String pName = "p1"; + resetChannel(); + setChannelConnected(); + + // The producer register using the first connection. + ByteBuf cmdProducer1 = Commands.newProducer(tName, producerId, requestId.incrementAndGet(), + pName, false, metadata, null, epoch.incrementAndGet(), false, + ProducerAccessMode.Shared, Optional.empty(), false); + channel.writeInbound(cmdProducer1); + assertTrue(getResponse() instanceof CommandProducerSuccess); + PersistentTopic topicRef = (PersistentTopic) brokerService.getTopicReference(tName).get(); + assertNotNull(topicRef); + assertEquals(topicRef.getProducers().size(), 1); + + // Verify the second producer using a new connection will override the producer who using a stopped channel. + channelsStoppedAnswerHealthCheck.add(channel); + ClientChannel channel2 = new ClientChannel(); + BackGroundExecutor backGroundExecutor1 = startBackgroundExecutorForEmbeddedChannel(channel); + BackGroundExecutor backGroundExecutor2 = startBackgroundExecutorForEmbeddedChannel(channel2.channel); + setChannelConnected(channel2.serverCnx); + + ByteBuf cmdProducer2 = Commands.newProducer(tName, producerId, requestId.incrementAndGet(), + pName, false, metadata, null, epoch.incrementAndGet(), false, + ProducerAccessMode.Shared, Optional.empty(), false); + channel2.channel.writeInbound(cmdProducer2); + Object response2 = getResponse(channel2.channel, channel2.clientChannelHelper); + assertTrue(response2 instanceof CommandProducerSuccess); + assertEquals(topicRef.getProducers().size(), 1); + + // cleanup. + channelsStoppedAnswerHealthCheck.clear(); + backGroundExecutor1.close(); + backGroundExecutor2.close(); + channel.finish(); + channel2.close(); + } + + @Test + public void testHandleConsumerAfterClientChannelInactive() throws Exception { + final String tName = successTopicName; + final long consumerId = 1; + final MutableInt requestId = new MutableInt(1); + final String sName = successSubName; + final String cName1 = ConsumerName.generateRandomName(); + final String cName2 = ConsumerName.generateRandomName(); + resetChannel(); + setChannelConnected(); + + // The producer register using the first connection. + ByteBuf cmdSubscribe1 = Commands.newSubscribe(tName, sName, consumerId, requestId.incrementAndGet(), + SubType.Exclusive, 0, cName1, 0); + channel.writeInbound(cmdSubscribe1); + assertTrue(getResponse() instanceof CommandSuccess); + PersistentTopic topicRef = (PersistentTopic) brokerService.getTopicReference(tName).get(); + assertNotNull(topicRef); + assertNotNull(topicRef.getSubscription(sName).getConsumers()); + assertEquals(topicRef.getSubscription(sName).getConsumers().size(), 1); + assertEquals(topicRef.getSubscription(sName).getConsumers().iterator().next().consumerName(), cName1); + + // Verify the second producer using a new connection will override the consumer who using a stopped channel. + channelsStoppedAnswerHealthCheck.add(channel); + ClientChannel channel2 = new ClientChannel(); + setChannelConnected(channel2.serverCnx); + ByteBuf cmdSubscribe2 = Commands.newSubscribe(tName, sName, consumerId, requestId.incrementAndGet(), + CommandSubscribe.SubType.Exclusive, 0, cName2, 0); + channel2.channel.writeInbound(cmdSubscribe2); + BackGroundExecutor backGroundExecutor = startBackgroundExecutorForEmbeddedChannel(channel); + + assertTrue(getResponse(channel2.channel, channel2.clientChannelHelper) instanceof CommandSuccess); + assertEquals(topicRef.getSubscription(sName).getConsumers().size(), 1); + assertEquals(topicRef.getSubscription(sName).getConsumers().iterator().next().consumerName(), cName2); + backGroundExecutor.close(); + + // cleanup. + channel.finish(); + channel2.close(); + } + + @Test + public void test2ndSubFailedIfDisabledConCheck() + throws Exception { + final String tName = successTopicName; + final long consumerId = 1; + final MutableInt requestId = new MutableInt(1); + final String sName = successSubName; + final String cName1 = ConsumerName.generateRandomName(); + final String cName2 = ConsumerName.generateRandomName(); + // Disabled connection check. + pulsar.getConfig().setConnectionLivenessCheckTimeoutMillis(-1); + resetChannel(); + setChannelConnected(); + + // The consumer register using the first connection. + ByteBuf cmdSubscribe1 = Commands.newSubscribe(tName, sName, consumerId, requestId.incrementAndGet(), + SubType.Exclusive, 0, cName1, 0); + channel.writeInbound(cmdSubscribe1); + assertTrue(getResponse() instanceof CommandSuccess); + PersistentTopic topicRef = (PersistentTopic) brokerService.getTopicReference(tName).orElse(null); + assertNotNull(topicRef); + assertNotNull(topicRef.getSubscription(sName).getConsumers()); + assertEquals(topicRef.getSubscription(sName).getConsumers().stream().map(Consumer::consumerName) + .collect(Collectors.toList()), Collections.singletonList(cName1)); + + // Verify the consumer using a new connection will override the consumer who using a stopped channel. + channelsStoppedAnswerHealthCheck.add(channel); + ClientChannel channel2 = new ClientChannel(); + setChannelConnected(channel2.serverCnx); + ByteBuf cmdSubscribe2 = Commands.newSubscribe(tName, sName, consumerId, requestId.incrementAndGet(), + CommandSubscribe.SubType.Exclusive, 0, cName2, 0); + channel2.channel.writeInbound(cmdSubscribe2); + BackGroundExecutor backGroundExecutor = startBackgroundExecutorForEmbeddedChannel(channel); + + // Since the feature "ConnectionLiveness" has been disabled, the fix + // by https://github.com/apache/pulsar/pull/21183 will not be affected, so the client will still get an error. + Object responseOfConnection2 = getResponse(channel2.channel, channel2.clientChannelHelper); + assertTrue(responseOfConnection2 instanceof CommandError); + assertTrue(((CommandError) responseOfConnection2).getMessage() + .contains("Exclusive consumer is already connected")); + assertEquals(topicRef.getSubscription(sName).getConsumers().size(), 1); + assertEquals(topicRef.getSubscription(sName).getConsumers().iterator().next().consumerName(), cName1); + backGroundExecutor.close(); + + // cleanup. + channel.finish(); + channel2.close(); + // Reset configuration. + pulsar.getConfig().setConnectionLivenessCheckTimeoutMillis(5000); + } + + /** + * When a channel typed "EmbeddedChannel", once we call channel.execute(runnable), there is no background thread + * to run it. + * So starting a background thread to trigger the tasks in the queue. + */ + private BackGroundExecutor startBackgroundExecutorForEmbeddedChannel(final EmbeddedChannel channel) { + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + ScheduledFuture scheduledFuture = executor.scheduleWithFixedDelay(() -> { + channel.runPendingTasks(); + }, 100, 100, TimeUnit.MILLISECONDS); + return new BackGroundExecutor(executor, scheduledFuture); + } + + /** + * Auto answer `Pong` for the `Cmd-Ping`. + * Node: This will result in additional threads pop Command from the Command queue, so do not call this + * method if the channel needs to accept other Command. + */ + private BackGroundExecutor autoResponseForHeartBeat(EmbeddedChannel channel, + ClientChannelHelper clientChannelHelper) { + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + ScheduledFuture scheduledFuture = executor.scheduleWithFixedDelay(() -> { + tryPeekResponse(channel, clientChannelHelper); + }, 100, 100, TimeUnit.MILLISECONDS); + return new BackGroundExecutor(executor, scheduledFuture); + } + + @AllArgsConstructor + private static class BackGroundExecutor implements Closeable { + + private ScheduledExecutorService executor; + + private ScheduledFuture scheduledFuture; + + @Override + public void close() throws IOException { + if (scheduledFuture != null) { + scheduledFuture.cancel(true); + } + executor.shutdown(); + } + } + + private class ClientChannel implements Closeable { + private ClientChannelHelper clientChannelHelper = new ClientChannelHelper(); + private ServerCnx serverCnx = new ServerCnx(pulsar); + private EmbeddedChannel channel = new EmbeddedChannel(DefaultChannelId.newInstance(), + new LengthFieldBasedFrameDecoder( + 5 * 1024 * 1024, + 0, + 4, + 0, + 4), + serverCnx); + public ClientChannel() { + serverCnx.setAuthRole(""); + } + public void close(){ + if (channel != null && channel.isActive()) { + serverCnx.close(); + channel.close(); + } + } + } + + @Test + public void testHandleProducer() throws Exception { + final String tName = "persistent://public/default/test-topic"; + final long producerId = 1; + final MutableInt requestId = new MutableInt(1); + final MutableInt epoch = new MutableInt(1); + final Map metadata = Collections.emptyMap(); + final String pName = "p1"; + resetChannel(); + assertTrue(channel.isActive()); + assertEquals(serverCnx.getState(), State.Start); + + // connect. + ByteBuf cConnect = Commands.newConnect("none", "", null); + channel.writeInbound(cConnect); + assertEquals(serverCnx.getState(), State.Connected); + assertTrue(getResponse() instanceof CommandConnected); + + // There is an in-progress producer registration. + ByteBuf cProducer1 = Commands.newProducer(tName, producerId, requestId.incrementAndGet(), + pName, false, metadata, null, epoch.incrementAndGet(), false, + ProducerAccessMode.Shared, Optional.empty(), false); + CompletableFuture existingFuture1 = new CompletableFuture(); + serverCnx.getProducers().put(producerId, existingFuture1); + channel.writeInbound(cProducer1); + Object response1 = getResponse(); + assertTrue(response1 instanceof CommandError); + CommandError error1 = (CommandError) response1; + assertEquals(error1.getError().toString(), ServerError.ServiceNotReady.toString()); + assertTrue(error1.getMessage().contains("already present on the connection")); + + // There is a failed registration. + ByteBuf cProducer2 = Commands.newProducer(tName, producerId, requestId.incrementAndGet(), + pName, false, metadata, null, epoch.incrementAndGet(), false, + ProducerAccessMode.Shared, Optional.empty(), false); + CompletableFuture existingFuture2 = new CompletableFuture(); + existingFuture2.completeExceptionally(new BrokerServiceException.ProducerBusyException("123")); + serverCnx.getProducers().put(producerId, existingFuture2); + + channel.writeInbound(cProducer2); + Object response2 = getResponse(); + assertTrue(response2 instanceof CommandError); + CommandError error2 = (CommandError) response2; + assertEquals(error2.getError().toString(), ServerError.ProducerBusy.toString()); + assertTrue(error2.getMessage().contains("already failed to register present on the connection")); + + // There is an successful registration. + ByteBuf cProducer3 = Commands.newProducer(tName, producerId, requestId.incrementAndGet(), + pName, false, metadata, null, epoch.incrementAndGet(), false, + ProducerAccessMode.Shared, Optional.empty(), false); + CompletableFuture existingFuture3 = new CompletableFuture(); + org.apache.pulsar.broker.service.Producer serviceProducer = + mock(org.apache.pulsar.broker.service.Producer.class); + when(serviceProducer.getProducerName()).thenReturn(pName); + when(serviceProducer.getSchemaVersion()).thenReturn(new EmptyVersion()); + existingFuture3.complete(serviceProducer); + serverCnx.getProducers().put(producerId, existingFuture3); + + channel.writeInbound(cProducer3); + Object response3 = getResponse(); + assertTrue(response3 instanceof CommandProducerSuccess); + CommandProducerSuccess cProducerSuccess = (CommandProducerSuccess) response3; + assertEquals(cProducerSuccess.getProducerName(), pName); + + // cleanup. + channel.finish(); + } + // This test used to be in the ServerCnxAuthorizationTest class, but it was migrated here because the mocking // in that class was too extensive. There is some overlap with this test and other tests in this class. The primary // role of this test is verifying that the correct role and AuthenticationDataSource are passed to the @@ -1474,14 +1848,14 @@ public void testBrokerClosedProducerClientRecreatesProducerThenSendCommand() thr ByteBuf clientCommand1 = Commands.newProducer(successTopicName, 1 /* producer id */, 1 /* request id */, producerName, Collections.emptyMap(), false); channel.writeInbound(clientCommand1); - assertTrue(getResponse() instanceof CommandProducerSuccess); + assertThat(getResponse()).isInstanceOf(CommandProducerSuccess.class); // Call disconnect method on producer to trigger activity similar to unloading Producer producer = serverCnx.getProducers().get(1).get(); assertNotNull(producer); producer.disconnect(); channel.runPendingTasks(); - assertTrue(getResponse() instanceof CommandCloseProducer); + assertThat(getResponse()).isInstanceOf(CommandCloseProducer.class); // Send message and expect no response sendMessage(); @@ -1637,9 +2011,11 @@ public void testDuplicateConcurrentSubscribeCommand() throws Exception { "test" /* consumer name */, 0 /* avoid reseting cursor */); channel.writeInbound(clientCommand); + BackGroundExecutor backGroundExecutor = startBackgroundExecutorForEmbeddedChannel(channel); + // Create producer second time clientCommand = Commands.newSubscribe(successTopicName, // - successSubName, 2 /* consumer id */, 1 /* request id */, SubType.Exclusive, 0, + successSubName, 2 /* consumer id */, 2 /* request id */, SubType.Exclusive, 0, "test" /* consumer name */, 0 /* avoid reseting cursor */); channel.writeInbound(clientCommand); @@ -1649,6 +2025,9 @@ public void testDuplicateConcurrentSubscribeCommand() throws Exception { CommandError error = (CommandError) response; assertEquals(error.getError(), ServerError.ConsumerBusy); }); + + // cleanup. + backGroundExecutor.close(); channel.finish(); } @@ -2038,7 +2417,8 @@ public void testSubscribeBookieTimeout() throws Exception { "test" /* consumer name */, 0 /* avoid reseting cursor */); channel.writeInbound(subscribe1); - ByteBuf closeConsumer = Commands.newCloseConsumer(1 /* consumer id */, 2 /* request id */); + ByteBuf closeConsumer = Commands.newCloseConsumer(1 /* consumer id */, 2 /* request id */, + null /* assignedBrokerServiceUrl */, null /* assignedBrokerServiceUrlTls */); channel.writeInbound(closeConsumer); ByteBuf subscribe2 = Commands.newSubscribe(successTopicName, // @@ -2212,7 +2592,7 @@ public void testAckCommand() throws Exception { channel.writeInbound(clientCommand); assertTrue(getResponse() instanceof CommandSuccess); - PositionImpl pos = new PositionImpl(0, 0); + Position pos = PositionFactory.create(0, 0); clientCommand = Commands.newAck(1 /* consumer id */, pos.getLedgerId(), pos.getEntryId(), null, AckType.Individual, @@ -2471,6 +2851,10 @@ protected void resetChannel() throws Exception { } protected void setChannelConnected() throws Exception { + setChannelConnected(serverCnx); + } + + protected void setChannelConnected(ServerCnx serverCnx) throws Exception { Field channelState = ServerCnx.class.getDeclaredField("state"); channelState.setAccessible(true); channelState.set(serverCnx, State.Connected); @@ -2484,13 +2868,25 @@ private void setConnectionVersion(int version) throws Exception { } protected Object getResponse() throws Exception { + return getResponse(channel, clientChannelHelper); + } + + protected Object getResponse(EmbeddedChannel channel, ClientChannelHelper clientChannelHelper) throws Exception { // Wait at most for 10s to get a response final long sleepTimeMs = 10; final long iterations = TimeUnit.SECONDS.toMillis(10) / sleepTimeMs; for (int i = 0; i < iterations; i++) { if (!channel.outboundMessages().isEmpty()) { Object outObject = channel.outboundMessages().remove(); - return clientChannelHelper.getCommand(outObject); + Object cmd = clientChannelHelper.getCommand(outObject); + if (cmd instanceof CommandPing) { + if (channelsStoppedAnswerHealthCheck.contains(channel)) { + continue; + } + channel.writeInbound(Commands.newPong()); + continue; + } + return cmd; } else { Thread.sleep(sleepTimeMs); } @@ -2499,10 +2895,31 @@ protected Object getResponse() throws Exception { throw new IOException("Failed to get response from socket within 10s"); } + protected Object tryPeekResponse(EmbeddedChannel channel, ClientChannelHelper clientChannelHelper) { + while (true) { + if (channel.outboundMessages().isEmpty()) { + return null; + } else { + Object outObject = channel.outboundMessages().peek(); + Object cmd = clientChannelHelper.getCommand(outObject); + if (cmd instanceof CommandPing) { + if (channelsStoppedAnswerHealthCheck.contains(channel)) { + continue; + } + channel.writeInbound(Commands.newPong()); + channel.outboundMessages().remove(); + continue; + } + return cmd; + } + } + } + private void setupMLAsyncCallbackMocks() { ledgerMock = mock(ManagedLedger.class); cursorMock = mock(ManagedCursor.class); doReturn(new ArrayList<>()).when(ledgerMock).getCursors(); + doReturn(new ManagedLedgerConfig()).when(ledgerMock).getConfig(); // call openLedgerComplete with ledgerMock on ML factory asyncOpen doAnswer((Answer) invocationOnMock -> { @@ -2526,12 +2943,12 @@ private void setupMLAsyncCallbackMocks() { // call addComplete on ledger asyncAddEntry doAnswer((Answer) invocationOnMock -> { - ((AddEntryCallback) invocationOnMock.getArguments()[1]).addComplete( - new PositionImpl(-1, -1), + ((AddEntryCallback) invocationOnMock.getArguments()[2]).addComplete( + PositionFactory.create(-1, -1), null, - invocationOnMock.getArguments()[2]); + invocationOnMock.getArguments()[3]); return null; - }).when(ledgerMock).asyncAddEntry(any(ByteBuf.class), any(AddEntryCallback.class), any()); + }).when(ledgerMock).asyncAddEntry(any(ByteBuf.class), anyInt(), any(AddEntryCallback.class), any()); doAnswer((Answer) invocationOnMock -> true).when(cursorMock).isDurable(); @@ -2969,8 +3386,9 @@ public boolean isCompletedExceptionally() { }; // assert error response assertTrue(responseAssert.test(responseAssert)); - // assert consumer-delete event occur - assertEquals(1L, + // The delete event will only occur after the future is completed. + // assert consumer-delete event will not occur. + assertEquals(0L, deleteTimesMark.getAllValues().stream().filter(f -> f == existingConsumerFuture).count()); // Server will not close the connection assertTrue(channel.isOpen()); @@ -3157,7 +3575,7 @@ public void handlePartitionMetadataRequestWithServiceNotReady() throws Exception doReturn(false).when(pulsar).isRunning(); assertTrue(channel.isActive()); - ByteBuf clientCommand = Commands.newPartitionMetadataRequest(successTopicName, 1); + ByteBuf clientCommand = Commands.newPartitionMetadataRequest(successTopicName, 1, true); channel.writeInbound(clientCommand); Object response = getResponse(); assertTrue(response instanceof CommandPartitionedTopicMetadataResponse); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/StandaloneTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/StandaloneTest.java index 5307e1a9ee874..541408b781be2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/StandaloneTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/StandaloneTest.java @@ -19,7 +19,6 @@ package org.apache.pulsar.broker.service; import org.apache.pulsar.PulsarStandaloneStarter; -import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.testng.annotations.Test; import static org.testng.AssertJUnit.assertEquals; @@ -27,34 +26,45 @@ import static org.testng.AssertJUnit.assertNull; @Test(groups = "broker") -public class StandaloneTest extends MockedPulsarServiceBaseTest { +public class StandaloneTest { - @Override - protected void setup() throws Exception { + static class TestPulsarStandaloneStarter extends PulsarStandaloneStarter { + public TestPulsarStandaloneStarter(String[] args) throws Exception { + super(args); + } - } - - @Override - protected void cleanup() throws Exception { + @Override + protected void registerShutdownHook() { + // ignore to prevent memory leaks + } + @Override + protected void exit(int status) { + // don't ever call System.exit in tests + throw new RuntimeException("Exited with status " + status); + } } @Test public void testWithoutMetadataStoreUrlInConfFile() throws Exception { String[] args = new String[]{"--config", "../conf/standalone.conf"}; - PulsarStandaloneStarter standalone = new PulsarStandaloneStarter(args); + PulsarStandaloneStarter standalone = new TestPulsarStandaloneStarter(args); assertNotNull(standalone.getConfig().getProperties().getProperty("metadataStoreUrl")); assertNotNull(standalone.getConfig().getProperties().getProperty("configurationMetadataStoreUrl")); } @Test - public void testAdvertised() throws Exception { + public void testInitialize() throws Exception { String[] args = new String[]{"--config", "./src/test/resources/configurations/pulsar_broker_test_standalone.conf"}; - PulsarStandaloneStarter standalone = new PulsarStandaloneStarter(args); + PulsarStandaloneStarter standalone = new TestPulsarStandaloneStarter(args); assertNull(standalone.getConfig().getAdvertisedAddress()); assertEquals(standalone.getConfig().getAdvertisedListeners(), "internal:pulsar://192.168.1.11:6660,internal:pulsar+ssl://192.168.1.11:6651"); + assertEquals(standalone.getConfig().isDispatcherPauseOnAckStatePersistentEnabled(), true); + assertEquals(standalone.getConfig().getMaxSecondsToClearTopicNameCache(), 1); + assertEquals(standalone.getConfig().getTopicNameCacheMaxCapacity(), 200); + assertEquals(standalone.getConfig().isCreateTopicToRemoteClusterForReplication(), true); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java index b11946069c9dd..582d10294a5a4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.service; +import static org.assertj.core.api.Assertions.assertThat; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; @@ -33,12 +34,14 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.InjectedClientCnxClientBuilder; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageRoutingMode; @@ -49,8 +52,13 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.impl.BatchMessageIdImpl; +import org.apache.pulsar.client.impl.ClientBuilderImpl; +import org.apache.pulsar.client.impl.ClientCnx; import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; +import org.apache.pulsar.common.api.proto.CommandError; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.util.RelativeTimeUtil; import org.awaitility.Awaitility; import org.testng.annotations.AfterClass; @@ -78,9 +86,11 @@ protected void cleanup() throws Exception { public void testSeek() throws Exception { final String topicName = "persistent://prop/use/ns-abc/testSeek"; + @Cleanup Producer producer = pulsarClient.newProducer().topic(topicName).create(); // Disable pre-fetch in consumer to track the messages received + @Cleanup org.apache.pulsar.client.api.Consumer consumer = pulsarClient.newConsumer().topic(topicName) .subscriptionName("my-subscription").receiverQueueSize(0).subscribe(); @@ -128,11 +138,37 @@ public void testSeek() throws Exception { assertEquals(sub.getNumberOfEntriesInBacklog(false), 0); } + @Test + public void testSeekIsByReceive() throws PulsarClientException { + final String topicName = "persistent://prop/use/ns-abc/testSeekIsByReceive"; + + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topicName).create(); + + String subscriptionName = "my-subscription"; + @Cleanup + org.apache.pulsar.client.api.Consumer consumer = pulsarClient.newConsumer().topic(topicName) + .subscriptionName(subscriptionName) + .subscribe(); + + List messageIds = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + String message = "my-message-" + i; + MessageId msgId = producer.send(message.getBytes()); + messageIds.add(msgId); + } + + consumer.seek(messageIds.get(5)); + Message message = consumer.receive(); + assertThat(message.getMessageId()).isEqualTo(messageIds.get(6)); + } + @Test public void testSeekForBatch() throws Exception { final String topicName = "persistent://prop/use/ns-abcd/testSeekForBatch"; String subscriptionName = "my-subscription-batch"; + @Cleanup Producer producer = pulsarClient.newProducer(Schema.STRING) .enableBatching(true) .batchingMaxMessages(3) @@ -159,6 +195,7 @@ public void testSeekForBatch() throws Exception { producer.close(); + @Cleanup org.apache.pulsar.client.api.Consumer consumer = pulsarClient.newConsumer(Schema.STRING) .topic(topicName) .subscriptionName(subscriptionName) @@ -189,6 +226,7 @@ public void testSeekForBatchMessageAndSpecifiedBatchIndex() throws Exception { final String topicName = "persistent://prop/use/ns-abcd/testSeekForBatchMessageAndSpecifiedBatchIndex"; String subscriptionName = "my-subscription-batch"; + @Cleanup Producer producer = pulsarClient.newProducer(Schema.STRING) .enableBatching(true) .batchingMaxMessages(3) @@ -233,6 +271,7 @@ public void testSeekForBatchMessageAndSpecifiedBatchIndex() throws Exception { .serviceUrl(lookupUrl.toString()) .build(); + @Cleanup org.apache.pulsar.client.api.Consumer consumer = newPulsarClient.newConsumer(Schema.STRING) .topic(topicName) .subscriptionName(subscriptionName) @@ -269,6 +308,7 @@ public void testSeekForBatchByAdmin() throws PulsarClientException, ExecutionExc final String topicName = "persistent://prop/use/ns-abcd/testSeekForBatchByAdmin-" + UUID.randomUUID().toString(); String subscriptionName = "my-subscription-batch"; + @Cleanup Producer producer = pulsarClient.newProducer(Schema.STRING) .enableBatching(true) .batchingMaxMessages(3) @@ -294,7 +334,7 @@ public void testSeekForBatchByAdmin() throws PulsarClientException, ExecutionExc producer.close(); - + @Cleanup org.apache.pulsar.client.api.Consumer consumer = pulsarClient.newConsumer(Schema.STRING) .topic(topicName) .subscriptionName(subscriptionName) @@ -350,6 +390,7 @@ public void testConcurrentResetCursor() throws Exception { final String topicName = "persistent://prop/use/ns-abc/testConcurrentReset_" + System.currentTimeMillis(); final String subscriptionName = "test-sub-name"; + @Cleanup Producer producer = pulsarClient.newProducer().topic(topicName).create(); admin.topics().createSubscription(topicName, subscriptionName, MessageId.earliest); @@ -399,6 +440,7 @@ public void testSeekOnPartitionedTopic() throws Exception { final String topicName = "persistent://prop/use/ns-abc/testSeekPartitions"; admin.topics().createPartitionedTopic(topicName, 2); + @Cleanup org.apache.pulsar.client.api.Consumer consumer = pulsarClient.newConsumer().topic(topicName) .subscriptionName("my-subscription").subscribe(); @@ -416,9 +458,11 @@ public void testSeekTime() throws Exception { long resetTimeInMillis = TimeUnit.SECONDS .toMillis(RelativeTimeUtil.parseRelativeTimeInSeconds(resetTimeStr)); + @Cleanup Producer producer = pulsarClient.newProducer().topic(topicName).create(); // Disable pre-fetch in consumer to track the messages received + @Cleanup org.apache.pulsar.client.api.Consumer consumer = pulsarClient.newConsumer().topic(topicName) .subscriptionName("my-subscription").receiverQueueSize(0).subscribe(); @@ -452,6 +496,7 @@ public void testSeekTimeByFunction() throws Exception { int msgNum = 20; admin.topics().createPartitionedTopic(topicName, partitionNum); creatProducerAndSendMsg(topicName, msgNum); + @Cleanup org.apache.pulsar.client.api.Consumer consumer = pulsarClient .newConsumer(Schema.STRING).startMessageIdInclusive() .topic(topicName).subscriptionName("my-sub").subscribe(); @@ -499,6 +544,7 @@ public void testSeekTimeOnPartitionedTopic() throws Exception { long resetTimeInMillis = TimeUnit.SECONDS .toMillis(RelativeTimeUtil.parseRelativeTimeInSeconds(resetTimeStr)); admin.topics().createPartitionedTopic(topicName, partitions); + @Cleanup Producer producer = pulsarClient.newProducer().topic(topicName).create(); // Disable pre-fetch in consumer to track the messages received org.apache.pulsar.client.api.Consumer consumer = pulsarClient.newConsumer().topic(topicName) @@ -552,12 +598,14 @@ public void testSeekTimeOnPartitionedTopic() throws Exception { public void testShouldCloseAllConsumersForMultipleConsumerDispatcherWhenSeek() throws Exception { final String topicName = "persistent://prop/use/ns-abc/testShouldCloseAllConsumersForMultipleConsumerDispatcherWhenSeek"; // Disable pre-fetch in consumer to track the messages received + @Cleanup org.apache.pulsar.client.api.Consumer consumer1 = pulsarClient.newConsumer() .topic(topicName) .subscriptionType(SubscriptionType.Shared) .subscriptionName("my-subscription") .subscribe(); + @Cleanup org.apache.pulsar.client.api.Consumer consumer2 = pulsarClient.newConsumer() .topic(topicName) .subscriptionType(SubscriptionType.Shared) @@ -584,20 +632,20 @@ public void testShouldCloseAllConsumersForMultipleConsumerDispatcherWhenSeek() t for (Consumer consumer : consumers) { assertFalse(connectedSinceSet.contains(consumer.getStats().getConnectedSince())); } - consumer1.close(); - consumer2.close(); } @Test public void testOnlyCloseActiveConsumerForSingleActiveConsumerDispatcherWhenSeek() throws Exception { final String topicName = "persistent://prop/use/ns-abc/testOnlyCloseActiveConsumerForSingleActiveConsumerDispatcherWhenSeek"; // Disable pre-fetch in consumer to track the messages received + @Cleanup org.apache.pulsar.client.api.Consumer consumer1 = pulsarClient.newConsumer() .topic(topicName) .subscriptionType(SubscriptionType.Failover) .subscriptionName("my-subscription") .subscribe(); + @Cleanup org.apache.pulsar.client.api.Consumer consumer2 = pulsarClient.newConsumer() .topic(topicName) .subscriptionType(SubscriptionType.Failover) @@ -637,11 +685,13 @@ public void testSeekByFunction() throws Exception { int msgNum = 160; admin.topics().createPartitionedTopic(topicName, partitionNum); creatProducerAndSendMsg(topicName, msgNum); + @Cleanup org.apache.pulsar.client.api.Consumer consumer = pulsarClient .newConsumer(Schema.STRING).startMessageIdInclusive() .topic(topicName).subscriptionName("my-sub").subscribe(); TopicName partitionedTopic = TopicName.get(topicName); + @Cleanup Reader reader = pulsarClient.newReader(Schema.STRING) .startMessageId(MessageId.earliest) .topic(partitionedTopic.getPartition(0).toString()).create(); @@ -690,12 +740,11 @@ public void testSeekByFunction() throws Exception { for (MessageId messageId : msgNotIn) { assertFalse(received.contains(messageId)); } - reader.close(); - consumer.close(); } private List creatProducerAndSendMsg(String topic, int msgNum) throws Exception { List messageIds = new ArrayList<>(); + @Cleanup Producer producer = pulsarClient .newProducer(Schema.STRING) .enableBatching(false) @@ -704,7 +753,6 @@ private List creatProducerAndSendMsg(String topic, int msgNum) throws for (int i = 0; i < msgNum; i++) { messageIds.add(producer.send("msg" + i)); } - producer.close(); return messageIds; } @@ -725,6 +773,7 @@ public void testSeekByFunctionAndMultiTopic() throws Exception { MessageId msgIdInTopic2Partition0 = admin.topics().getLastMessageId(topic2.getPartition(0).toString()); MessageId msgIdInTopic2Partition2 = admin.topics().getLastMessageId(topic2.getPartition(2).toString()); + @Cleanup org.apache.pulsar.client.api.Consumer consumer = pulsarClient .newConsumer(Schema.STRING).startMessageIdInclusive() .topics(Arrays.asList(topicName, topicName2)).subscriptionName("my-sub").subscribe(); @@ -757,10 +806,73 @@ public void testSeekByFunctionAndMultiTopic() throws Exception { assertEquals(count, (msgInTopic1Partition0 + msgInTopic1Partition1 + msgInTopic1Partition2) * 2); } + @Test + public void testSeekWillNotEncounteredFencedError() throws Exception { + String topicName = "persistent://prop/ns-abc/my-topic2"; + admin.topics().createNonPartitionedTopic(topicName); + admin.topicPolicies().setRetention(topicName, new RetentionPolicies(3600, 0)); + // Create a pulsar client with a subscription fenced counter. + ClientBuilderImpl clientBuilder = (ClientBuilderImpl) PulsarClient.builder().serviceUrl(lookupUrl.toString()); + AtomicInteger receivedFencedErrorCounter = new AtomicInteger(); + @Cleanup + PulsarClient client = InjectedClientCnxClientBuilder.create(clientBuilder, (conf, eventLoopGroup) -> + new ClientCnx(InstrumentProvider.NOOP, conf, eventLoopGroup) { + protected void handleError(CommandError error) { + if (error.getMessage() != null && error.getMessage().contains("Subscription is fenced")) { + receivedFencedErrorCounter.incrementAndGet(); + } + super.handleError(error); + } + }); + + // publish some messages. + @Cleanup + org.apache.pulsar.client.api.Consumer consumer = client.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName("s1") + .subscribe(); + + @Cleanup + Producer producer = client.newProducer(Schema.STRING) + .topic(topicName).create(); + MessageIdImpl msgId1 = (MessageIdImpl) producer.send("0"); + for (int i = 1; i < 11; i++) { + admin.topics().unload(topicName); + producer.send(i + ""); + } + + // Inject a delay for reset-cursor. + mockZooKeeper.delay(3000, (op, path) -> { + if (path.equals("/managed-ledgers/prop/ns-abc/persistent/my-topic2/s1")) { + return op.toString().equalsIgnoreCase("SET"); + } + return false; + }); + + // Verify: consumer will not receive "subscription fenced" error after a seek. + for (int i = 1; i < 11; i++) { + Message msg = consumer.receive(2, TimeUnit.SECONDS); + assertNotNull(msg); + consumer.acknowledge(msg); + } + consumer.seek(msgId1); + Awaitility.await().untilAsserted(() -> { + assertTrue(consumer.isConnected()); + }); + assertEquals(receivedFencedErrorCounter.get(), 0); + + // cleanup. + producer.close(); + consumer.close(); + client.close(); + admin.topics().delete(topicName); + } + @Test public void testExceptionBySeekFunction() throws Exception { final String topicName = "persistent://prop/use/ns-abc/test" + UUID.randomUUID(); creatProducerAndSendMsg(topicName,10); + @Cleanup org.apache.pulsar.client.api.Consumer consumer = pulsarClient .newConsumer() .topic(topicName).subscriptionName("my-sub").subscribe(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SyncConfigStoreTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SyncConfigStoreTest.java new file mode 100644 index 0000000000000..577725f96ed34 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SyncConfigStoreTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.apache.pulsar.common.naming.NamespaceName.SYSTEM_NAMESPACE; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import java.util.Arrays; +import java.util.HashSet; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.common.naming.TopicDomain; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.metadata.api.MetadataEvent; +import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.apache.pulsar.zookeeper.ZookeeperServerTest; +import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class SyncConfigStoreTest extends GeoReplicationWithConfigurationSyncTestBase { + + private static final String CONF_NAME_SYNC_EVENT_TOPIC = "configurationMetadataSyncEventTopic"; + private static final String SYNC_EVENT_TOPIC = TopicDomain.persistent.value() + "://" + SYSTEM_NAMESPACE + + "/__sync_config_meta"; + + @Override + @BeforeClass(alwaysRun = true, timeOut = 300000) + public void setup() throws Exception { + super.setup(); + TenantInfoImpl tenantInfo = new TenantInfoImpl(); + tenantInfo.setAllowedClusters(new HashSet<>(Arrays.asList(cluster1, cluster2))); + admin1.tenants().createTenant(TopicName.get(SYNC_EVENT_TOPIC).getTenant(), tenantInfo); + admin1.namespaces().createNamespace(TopicName.get(SYNC_EVENT_TOPIC).getNamespace()); + } + + @Override + @AfterClass(alwaysRun = true, timeOut = 300000) + public void cleanup() throws Exception { + super.cleanup(); + } + + protected void setConfigDefaults(ServiceConfiguration config, String clusterName, + LocalBookkeeperEnsemble bookkeeperEnsemble, ZookeeperServerTest brokerConfigZk) { + super.setConfigDefaults(config, clusterName, bookkeeperEnsemble, brokerConfigZk); + } + + @Test + public void testDynamicEnableConfigurationMetadataSyncEventTopic() throws Exception { + // Verify the condition that supports synchronizer: the metadata store is a different one. + Awaitility.await().untilAsserted(() -> { + boolean shouldShutdownConfigurationMetadataStore = + WhiteboxImpl.getInternalState(pulsar1, "shouldShutdownConfigurationMetadataStore"); + assertTrue(shouldShutdownConfigurationMetadataStore); + }); + + // Verify the synchronizer will be created dynamically. + admin1.brokers().updateDynamicConfiguration(CONF_NAME_SYNC_EVENT_TOPIC, SYNC_EVENT_TOPIC); + Awaitility.await().untilAsserted(() -> { + assertEquals(pulsar1.getConfig().getConfigurationMetadataSyncEventTopic(), SYNC_EVENT_TOPIC); + PulsarMetadataEventSynchronizer synchronizer = + WhiteboxImpl.getInternalState(pulsar1, "configMetadataSynchronizer"); + assertNotNull(synchronizer); + assertEquals(synchronizer.getState(), PulsarMetadataEventSynchronizer.State.Started); + assertTrue(synchronizer.isStarted()); + }); + + PulsarMetadataEventSynchronizer synchronizerStarted = + WhiteboxImpl.getInternalState(pulsar1, "configMetadataSynchronizer"); + Producer producerStarted = + WhiteboxImpl.getInternalState(synchronizerStarted, "producer"); + Consumer consumerStarted = + WhiteboxImpl.getInternalState(synchronizerStarted, "consumer"); + + // Verify the synchronizer will be closed dynamically. + admin1.brokers().deleteDynamicConfiguration(CONF_NAME_SYNC_EVENT_TOPIC); + Awaitility.await().untilAsserted(() -> { + // The synchronizer that was started will be closed. + assertEquals(synchronizerStarted.getState(), PulsarMetadataEventSynchronizer.State.Closed); + assertTrue(synchronizerStarted.isClosingOrClosed()); + assertFalse(producerStarted.isConnected()); + assertFalse(consumerStarted.isConnected()); + // The synchronizer in memory will be null. + assertNull(pulsar1.getConfig().getConfigurationMetadataSyncEventTopic()); + PulsarMetadataEventSynchronizer synchronizer = + WhiteboxImpl.getInternalState(pulsar1, "configMetadataSynchronizer"); + assertNull(synchronizer); + }); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index f9fc717a817af..7e3f4e14daa6d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -18,55 +18,53 @@ */ package org.apache.pulsar.broker.service; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.testng.AssertJUnit.assertEquals; import static org.testng.AssertJUnit.assertNotNull; import static org.testng.AssertJUnit.assertNull; -import static org.testng.AssertJUnit.assertTrue; -import java.lang.reflect.Field; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; -import org.apache.pulsar.broker.service.BrokerServiceException.TopicPoliciesCacheNotInitException; -import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; import org.apache.pulsar.broker.systopic.SystemTopicClient; -import org.apache.pulsar.broker.systopic.TopicPoliciesSystemTopicClient; import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.client.impl.Backoff; -import org.apache.pulsar.client.impl.BackoffBuilder; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.events.PulsarEvent; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TopicPolicies; -import org.apache.pulsar.common.util.FutureUtil; +import org.assertj.core.api.Assertions; import org.awaitility.Awaitility; +import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @Test(groups = "broker") +@Slf4j public class SystemTopicBasedTopicPoliciesServiceTest extends MockedPulsarServiceBaseTest { private static final String NAMESPACE1 = "system-topic/namespace-1"; private static final String NAMESPACE2 = "system-topic/namespace-2"; private static final String NAMESPACE3 = "system-topic/namespace-3"; + private static final String NAMESPACE4 = "system-topic/namespace-4"; + + private static final String NAMESPACE5 = "system-topic/namespace-5"; + private static final TopicName TOPIC1 = TopicName.get("persistent", NamespaceName.get(NAMESPACE1), "topic-1"); private static final TopicName TOPIC2 = TopicName.get("persistent", NamespaceName.get(NAMESPACE1), "topic-2"); private static final TopicName TOPIC3 = TopicName.get("persistent", NamespaceName.get(NAMESPACE2), "topic-1"); @@ -92,7 +90,7 @@ protected void cleanup() throws Exception { @Test public void testConcurrentlyRegisterUnregisterListeners() throws ExecutionException, InterruptedException { TopicName topicName = TopicName.get("test"); - class TopicPolicyListenerImpl implements TopicPolicyListener { + class TopicPolicyListenerImpl implements TopicPolicyListener { @Override public void onUpdate(TopicPolicies data) { @@ -102,7 +100,7 @@ public void onUpdate(TopicPolicies data) { CompletableFuture f = CompletableFuture.completedFuture(null).thenRunAsync(() -> { for (int i = 0; i < 100; i++) { - TopicPolicyListener listener = new TopicPolicyListenerImpl(); + TopicPolicyListener listener = new TopicPolicyListenerImpl(); systemTopicBasedTopicPoliciesService.registerListener(topicName, listener); Assert.assertNotNull(systemTopicBasedTopicPoliciesService.listeners.get(topicName)); Assert.assertTrue(systemTopicBasedTopicPoliciesService.listeners.get(topicName).size() >= 1); @@ -111,7 +109,7 @@ public void onUpdate(TopicPolicies data) { }); for (int i = 0; i < 100; i++) { - TopicPolicyListener listener = new TopicPolicyListenerImpl(); + TopicPolicyListener listener = new TopicPolicyListenerImpl(); systemTopicBasedTopicPoliciesService.registerListener(topicName, listener); Assert.assertNotNull(systemTopicBasedTopicPoliciesService.listeners.get(topicName)); Assert.assertTrue(systemTopicBasedTopicPoliciesService.listeners.get(topicName).size() >= 1); @@ -124,7 +122,7 @@ public void onUpdate(TopicPolicies data) { } @Test - public void testGetPolicy() throws ExecutionException, InterruptedException, TopicPoliciesCacheNotInitException { + public void testGetPolicy() throws Exception { // Init topic policies TopicPolicies initPolicy = TopicPolicies.builder() @@ -135,11 +133,11 @@ public void testGetPolicy() throws ExecutionException, InterruptedException, Top // Wait for all topic policies updated. Awaitility.await().untilAsserted(() -> Assert.assertTrue(systemTopicBasedTopicPoliciesService - .getPoliciesCacheInit(TOPIC1.getNamespaceObject()))); + .getPoliciesCacheInit(TOPIC1.getNamespaceObject()).isDone())); // Assert broker is cache all topic policies Awaitility.await().untilAsserted(() -> - Assert.assertEquals(systemTopicBasedTopicPoliciesService.getTopicPolicies(TOPIC1) + Assert.assertEquals(TopicPolicyTestUtils.getTopicPolicies(systemTopicBasedTopicPoliciesService, TOPIC1) .getMaxConsumerPerTopic().intValue(), 10)); // Update policy for TOPIC1 @@ -179,12 +177,12 @@ public void testGetPolicy() throws ExecutionException, InterruptedException, Top systemTopicBasedTopicPoliciesService.updateTopicPoliciesAsync(TOPIC6, policies6).get(); Awaitility.await().untilAsserted(() -> { - TopicPolicies policiesGet1 = systemTopicBasedTopicPoliciesService.getTopicPolicies(TOPIC1); - TopicPolicies policiesGet2 = systemTopicBasedTopicPoliciesService.getTopicPolicies(TOPIC2); - TopicPolicies policiesGet3 = systemTopicBasedTopicPoliciesService.getTopicPolicies(TOPIC3); - TopicPolicies policiesGet4 = systemTopicBasedTopicPoliciesService.getTopicPolicies(TOPIC4); - TopicPolicies policiesGet5 = systemTopicBasedTopicPoliciesService.getTopicPolicies(TOPIC5); - TopicPolicies policiesGet6 = systemTopicBasedTopicPoliciesService.getTopicPolicies(TOPIC6); + TopicPolicies policiesGet1 = TopicPolicyTestUtils.getTopicPolicies(systemTopicBasedTopicPoliciesService, TOPIC1); + TopicPolicies policiesGet2 = TopicPolicyTestUtils.getTopicPolicies(systemTopicBasedTopicPoliciesService, TOPIC2); + TopicPolicies policiesGet3 = TopicPolicyTestUtils.getTopicPolicies(systemTopicBasedTopicPoliciesService, TOPIC3); + TopicPolicies policiesGet4 = TopicPolicyTestUtils.getTopicPolicies(systemTopicBasedTopicPoliciesService, TOPIC4); + TopicPolicies policiesGet5 = TopicPolicyTestUtils.getTopicPolicies(systemTopicBasedTopicPoliciesService, TOPIC5); + TopicPolicies policiesGet6 = TopicPolicyTestUtils.getTopicPolicies(systemTopicBasedTopicPoliciesService, TOPIC6); Assert.assertEquals(policiesGet1, policies1); Assert.assertEquals(policiesGet2, policies2); @@ -217,8 +215,8 @@ public void testGetPolicy() throws ExecutionException, InterruptedException, Top // reader for NAMESPACE1 will back fill the reader cache Awaitility.await().untilAsserted(() -> { - TopicPolicies policiesGet1 = systemTopicBasedTopicPoliciesService.getTopicPolicies(TOPIC1); - TopicPolicies policiesGet2 = systemTopicBasedTopicPoliciesService.getTopicPolicies(TOPIC2); + TopicPolicies policiesGet1 = TopicPolicyTestUtils.getTopicPolicies(systemTopicBasedTopicPoliciesService, TOPIC1); + TopicPolicies policiesGet2 = TopicPolicyTestUtils.getTopicPolicies(systemTopicBasedTopicPoliciesService, TOPIC2); Assert.assertEquals(policies1, policiesGet1); Assert.assertEquals(policies2, policiesGet2); }); @@ -229,7 +227,8 @@ public void testGetPolicy() throws ExecutionException, InterruptedException, Top Assert.assertTrue(systemTopicBasedTopicPoliciesService.checkReaderIsCached(NamespaceName.get(NAMESPACE3))); // Check get without cache - TopicPolicies policiesGet1 = systemTopicBasedTopicPoliciesService.getTopicPoliciesBypassCacheAsync(TOPIC1).get(); + TopicPolicies policiesGet1 = TopicPolicyTestUtils.getTopicPoliciesBypassCache(systemTopicBasedTopicPoliciesService, + TOPIC1).orElseThrow(); Assert.assertEquals(policies1, policiesGet1); } @@ -243,7 +242,7 @@ public void testCacheCleanup() throws Exception { Awaitility.await().untilAsserted(() -> assertNotNull(admin.topics().getMaxConsumers(topic))); Map map = systemTopicBasedTopicPoliciesService.getPoliciesCache(); - Map>> listMap = + Map> listMap = systemTopicBasedTopicPoliciesService.getListeners(); assertNotNull(map.get(topicName)); assertEquals(map.get(topicName).getMaxConsumerPerTopic().intValue(), 1000); @@ -262,7 +261,7 @@ public void testListenerCleanupByPartition() throws Exception { admin.topics().createPartitionedTopic(topic, 3); pulsarClient.newProducer().topic(topic).create().close(); - Map>> listMap = + Map> listMap = systemTopicBasedTopicPoliciesService.getListeners(); Awaitility.await().untilAsserted(() -> { // all 3 topic partition have registered the topic policy listeners. @@ -296,92 +295,125 @@ private void prepareData() throws PulsarAdminException { } @Test - public void testGetPolicyTimeout() throws Exception { + public void testHandleNamespaceBeingDeleted() throws Exception { SystemTopicBasedTopicPoliciesService service = (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); - Awaitility.await().untilAsserted(() -> assertTrue(service.policyCacheInitMap.get(TOPIC1.getNamespaceObject()))); - service.policyCacheInitMap.put(TOPIC1.getNamespaceObject(), false); - long start = System.currentTimeMillis(); - Backoff backoff = new BackoffBuilder() - .setInitialTime(500, TimeUnit.MILLISECONDS) - .setMandatoryStop(5000, TimeUnit.MILLISECONDS) - .setMax(1000, TimeUnit.MILLISECONDS) - .create(); - try { - service.getTopicPoliciesAsyncWithRetry(TOPIC1, backoff, pulsar.getExecutor(), false).get(); - } catch (Exception e) { - assertTrue(e.getCause() instanceof TopicPoliciesCacheNotInitException); - } - long cost = System.currentTimeMillis() - start; - assertTrue("actual:" + cost, cost >= 5000 - 1000); + pulsar.getPulsarResources().getNamespaceResources().setPolicies(NamespaceName.get(NAMESPACE1), + old -> { + old.deleted = true; + return old; + }); + service.deleteTopicPoliciesAsync(TOPIC1).get(); } @Test - public void testCreatSystemTopicClientWithRetry() throws Exception { - SystemTopicBasedTopicPoliciesService service = - spy((SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService()); - Field field = SystemTopicBasedTopicPoliciesService.class - .getDeclaredField("namespaceEventsSystemTopicFactory"); - field.setAccessible(true); - NamespaceEventsSystemTopicFactory factory = spy((NamespaceEventsSystemTopicFactory) field.get(service)); - SystemTopicClient client = mock(TopicPoliciesSystemTopicClient.class); - doReturn(client).when(factory).createTopicPoliciesSystemTopicClient(any()); - field.set(service, factory); - - SystemTopicClient.Reader reader = mock(SystemTopicClient.Reader.class); - // Throw an exception first, create successfully after retrying - doReturn(FutureUtil.failedFuture(new PulsarClientException("test"))) - .doReturn(CompletableFuture.completedFuture(reader)).when(client).newReaderAsync(); - - SystemTopicClient.Reader reader1 = service.createSystemTopicClientWithRetry(null).get(); - - assertEquals(reader1, reader); - } + public void testGetTopicPoliciesWithCleanCache() throws Exception { + final String topic = "persistent://" + NAMESPACE1 + "/test" + UUID.randomUUID(); + pulsarClient.newProducer().topic(topic).create().close(); - @Test - public void testGetTopicPoliciesWithRetry() throws Exception { - Field initMapField = SystemTopicBasedTopicPoliciesService.class.getDeclaredField("policyCacheInitMap"); - initMapField.setAccessible(true); - Map initMap = (Map)initMapField.get(systemTopicBasedTopicPoliciesService); - initMap.remove(NamespaceName.get(NAMESPACE1)); - Field readerCaches = SystemTopicBasedTopicPoliciesService.class.getDeclaredField("readerCaches"); - readerCaches.setAccessible(true); - Map>> readers = (Map)readerCaches.get(systemTopicBasedTopicPoliciesService); - readers.remove(NamespaceName.get(NAMESPACE1)); - Backoff backoff = new BackoffBuilder() - .setInitialTime(500, TimeUnit.MILLISECONDS) - .setMandatoryStop(5000, TimeUnit.MILLISECONDS) - .setMax(1000, TimeUnit.MILLISECONDS) - .create(); - TopicPolicies initPolicy = TopicPolicies.builder() - .maxConsumerPerTopic(10) - .build(); - ScheduledExecutorService executors = Executors.newScheduledThreadPool(1); - executors.schedule(new Runnable() { - @Override - public void run() { - try { - systemTopicBasedTopicPoliciesService.updateTopicPoliciesAsync(TOPIC1, initPolicy).get(); - } catch (Exception ignore) {} - } - }, 2000, TimeUnit.MILLISECONDS); + SystemTopicBasedTopicPoliciesService topicPoliciesService = + (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + + ConcurrentHashMap spyPoliciesCache = spy(new ConcurrentHashMap()); + FieldUtils.writeDeclaredField(topicPoliciesService, "policiesCache", spyPoliciesCache, true); + + Awaitility.await().untilAsserted(() -> Assertions.assertThat( + TopicPolicyTestUtils.getTopicPolicies(topicPoliciesService, TopicName.get(topic))).isNull()); + + admin.topicPolicies().setMaxConsumersPerSubscription(topic, 1); Awaitility.await().untilAsserted(() -> { - Optional topicPolicies = systemTopicBasedTopicPoliciesService - .getTopicPoliciesAsyncWithRetry(TOPIC1, backoff, pulsar.getExecutor(), false).get(); - Assert.assertTrue(topicPolicies.isPresent()); - if (topicPolicies.isPresent()) { - Assert.assertEquals(topicPolicies.get(), initPolicy); + Assertions.assertThat(TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), + TopicName.get(topic))).isNotNull(); + }); + + Map>> readers = + (Map>>) + FieldUtils.readDeclaredField(topicPoliciesService, "readerCaches", true); + + Mockito.doAnswer(invocation -> { + Thread.sleep(1000); + return invocation.callRealMethod(); + }).when(spyPoliciesCache).get(Mockito.any()); + + CompletableFuture result = new CompletableFuture<>(); + Thread thread = new Thread(() -> { + try { + for (int i = 0; i < 10; i++) { + final var policies = TopicPolicyTestUtils.getTopicPolicies(topicPoliciesService, + TopicName.get(topic)); + if (policies == null) { + throw new Exception("null policies for " + i + "th get"); + } + } + result.complete(null); + } catch (Exception e) { + result.completeExceptionally(e); + } + }); + + Thread thread2 = new Thread(() -> { + for (int i = 0; i < 10; i++) { + CompletableFuture> readerCompletableFuture = + readers.get(TopicName.get(topic).getNamespaceObject()); + if (readerCompletableFuture != null) { + readerCompletableFuture.join().closeAsync().join(); + } } }); + + thread.start(); + thread2.start(); + + thread.join(); + thread2.join(); + + result.join(); } @Test - public void testHandleNamespaceBeingDeleted() throws Exception { + public void testWriterCache() throws Exception { + admin.namespaces().createNamespace(NAMESPACE4); + for (int i = 1; i <= 5; i ++) { + final String topicName = "persistent://" + NAMESPACE4 + "/testWriterCache" + i; + admin.topics().createNonPartitionedTopic(topicName); + pulsarClient.newProducer(Schema.STRING).topic(topicName).create().close(); + } + @Cleanup("shutdown") + ExecutorService executorService = Executors.newFixedThreadPool(5); + for (int i = 1; i <= 5; i ++) { + int finalI = i; + executorService.execute(() -> { + final String topicName = "persistent://" + NAMESPACE4 + "/testWriterCache" + finalI; + try { + admin.topicPolicies().setMaxConsumers(topicName, 2); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } SystemTopicBasedTopicPoliciesService service = (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); - pulsar.getPulsarResources().getNamespaceResources().setPolicies(NamespaceName.get(NAMESPACE1), + Assert.assertNotNull(service.getWriterCaches().synchronous().get(NamespaceName.get(NAMESPACE4))); + for (int i = 1; i <= 5; i ++) { + final String topicName = "persistent://" + NAMESPACE4 + "/testWriterCache" + i; + admin.topics().delete(topicName); + } + admin.namespaces().deleteNamespace(NAMESPACE4); + Assert.assertNull(service.getWriterCaches().synchronous().getIfPresent(NamespaceName.get(NAMESPACE4))); + } + + @Test + public void testPrepareInitPoliciesCacheAsyncWhenNamespaceBeingDeleted() throws Exception { + SystemTopicBasedTopicPoliciesService service = (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + admin.namespaces().createNamespace(NAMESPACE5); + + NamespaceName namespaceName = NamespaceName.get(NAMESPACE5); + pulsar.getPulsarResources().getNamespaceResources().setPolicies(namespaceName, old -> { old.deleted = true; return old; - }); - service.deleteTopicPoliciesAsync(TOPIC1).get(); + }); + + assertNull(service.getPoliciesCacheInit(namespaceName)); + service.prepareInitPoliciesCacheAsync(namespaceName).get(); + admin.namespaces().deleteNamespace(NAMESPACE5); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicGCTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicGCTest.java new file mode 100644 index 0000000000000..172bd3702e129 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicGCTest.java @@ -0,0 +1,381 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; +import lombok.EqualsAndHashCode; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.ConsumerBuilder; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.TopicMessageId; +import org.apache.pulsar.client.impl.ConsumerImpl; +import org.apache.pulsar.client.impl.PatternMultiTopicsConsumerImpl; +import org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode; +import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class TopicGCTest extends ProducerConsumerBase { + + @BeforeClass(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @EqualsAndHashCode.Include + protected void doInitConf() throws Exception { + super.doInitConf(); + conf.setBrokerDeleteInactiveTopicsEnabled(true); + conf.setBrokerDeleteInactiveTopicsMode( + InactiveTopicDeleteMode.delete_when_subscriptions_caught_up); + conf.setBrokerDeleteInactiveTopicsFrequencySeconds(10); + } + + private enum SubscribeTopicType { + MULTI_PARTITIONED_TOPIC, + REGEX_TOPIC; + } + + @DataProvider(name = "subscribeTopicTypes") + public Object[][] subTopicTypes() { + return new Object[][]{ + {SubscribeTopicType.MULTI_PARTITIONED_TOPIC}, + {SubscribeTopicType.REGEX_TOPIC} + }; + } + + private void setSubscribeTopic(ConsumerBuilder consumerBuilder, SubscribeTopicType subscribeTopicType, + String topicName, String topicPattern) { + if (subscribeTopicType.equals(SubscribeTopicType.MULTI_PARTITIONED_TOPIC)) { + consumerBuilder.topic(topicName); + } else { + consumerBuilder.topicsPattern(Pattern.compile(topicPattern)); + } + } + + @Test(dataProvider = "subscribeTopicTypes", timeOut = 300 * 1000) + public void testRecreateConsumerAfterOnePartGc(SubscribeTopicType subscribeTopicType) throws Exception { + final String topic = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final String topicPattern = "persistent://public/default/tp.*"; + final String partition0 = topic + "-partition-0"; + final String partition1 = topic + "-partition-1"; + final String subscription = "s1"; + admin.topics().createPartitionedTopic(topic, 2); + admin.topics().createSubscription(topic, subscription, MessageId.earliest); + + // create consumers and producers. + Producer producer0 = pulsarClient.newProducer(Schema.STRING).topic(partition0) + .enableBatching(false).create(); + Producer producer1 = pulsarClient.newProducer(Schema.STRING).topic(partition1) + .enableBatching(false).create(); + ConsumerBuilder consumerBuilder1 = pulsarClient.newConsumer(Schema.STRING) + .subscriptionName(subscription) + .isAckReceiptEnabled(true) + .subscriptionType(SubscriptionType.Shared); + setSubscribeTopic(consumerBuilder1, subscribeTopicType, topic, topicPattern); + Consumer consumer1 = consumerBuilder1.subscribe(); + + // Make consume all messages for one topic, do not consume any messages for another one. + producer0.send("1"); + producer1.send("2"); + admin.topics().skipAllMessages(partition0, subscription); + + // Wait for topic GC. + // Partition 0 will be deleted about 20s later, left 2min to avoid flaky. + producer0.close(); + consumer1.close(); + Awaitility.await().atMost(2, TimeUnit.MINUTES).untilAsserted(() -> { + CompletableFuture> tp1 = pulsar.getBrokerService().getTopic(partition0, false); + CompletableFuture> tp2 = pulsar.getBrokerService().getTopic(partition1, false); + assertTrue(tp1 == null || !tp1.get().isPresent()); + assertTrue(tp2 != null && tp2.get().isPresent()); + }); + + // Verify that the consumer subscribed with partitioned topic can be created successful. + ConsumerBuilder consumerBuilder2 = pulsarClient.newConsumer(Schema.STRING) + .subscriptionName(subscription) + .isAckReceiptEnabled(true) + .subscriptionType(SubscriptionType.Shared); + setSubscribeTopic(consumerBuilder2, subscribeTopicType, topic, topicPattern); + Consumer consumer2 = consumerBuilder2.subscribe(); + Message msg = consumer2.receive(2, TimeUnit.SECONDS); + String receivedMsgValue = msg.getValue(); + log.info("received msg: {}", receivedMsgValue); + consumer2.acknowledge(msg); + + // cleanup. + consumer2.close(); + producer0.close(); + producer1.close(); + admin.topics().deletePartitionedTopic(topic); + } + + @Test(dataProvider = "subscribeTopicTypes", timeOut = 300 * 1000) + public void testAppendCreateConsumerAfterOnePartGc(SubscribeTopicType subscribeTopicType) throws Exception { + final String topic = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final String topicPattern = "persistent://public/default/tp.*"; + final String partition0 = topic + "-partition-0"; + final String partition1 = topic + "-partition-1"; + final String subscription = "s1"; + admin.topics().createPartitionedTopic(topic, 2); + admin.topics().createSubscription(topic, subscription, MessageId.earliest); + + // create consumers and producers. + Producer producer0 = pulsarClient.newProducer(Schema.STRING).topic(partition0) + .enableBatching(false).create(); + Producer producer1 = pulsarClient.newProducer(Schema.STRING).topic(partition1) + .enableBatching(false).create(); + ConsumerBuilder consumerBuilder1 = pulsarClient.newConsumer(Schema.STRING) + .subscriptionName(subscription) + .isAckReceiptEnabled(true) + .subscriptionType(SubscriptionType.Shared); + setSubscribeTopic(consumerBuilder1, subscribeTopicType, topic, topicPattern); + Consumer consumer1 = consumerBuilder1.subscribe(); + + // Make consume all messages for one topic, do not consume any messages for another one. + producer0.send("partition-0-1"); + producer1.send("partition-1-1"); + producer1.send("partition-1-2"); + producer1.send("partition-1-4"); + admin.topics().skipAllMessages(partition0, subscription); + + // Wait for topic GC. + // Partition 0 will be deleted about 20s later, left 2min to avoid flaky. + producer0.close(); + Awaitility.await().atMost(2, TimeUnit.MINUTES).untilAsserted(() -> { + CompletableFuture> tp1 = pulsar.getBrokerService().getTopic(partition0, false); + CompletableFuture> tp2 = pulsar.getBrokerService().getTopic(partition1, false); + assertTrue(tp1 == null || !tp1.get().isPresent()); + assertTrue(tp2 != null && tp2.get().isPresent()); + }); + + // Verify that the messages under "partition-1" still can be ack. + for (int i = 0; i < 2; i++) { + Message msg = consumer1.receive(2, TimeUnit.SECONDS); + assertNotNull(msg, "Expected at least received 2 messages."); + log.info("received msg[{}]: {}", i, msg.getValue()); + TopicMessageId messageId = (TopicMessageId) msg.getMessageId(); + if (messageId.getOwnerTopic().equals(partition1)) { + consumer1.acknowledgeAsync(msg); + } + } + consumer1.close(); + + // Verify that the consumer subscribed with partitioned topic can be created successful. + ConsumerBuilder consumerBuilder2 = pulsarClient.newConsumer(Schema.STRING) + .subscriptionName(subscription) + .isAckReceiptEnabled(true) + .subscriptionType(SubscriptionType.Shared); + setSubscribeTopic(consumerBuilder2, subscribeTopicType, topic, topicPattern); + Consumer consumer2 = consumerBuilder2.subscribe(); + producer1.send("partition-1-5"); + Message msg = consumer2.receive(2, TimeUnit.SECONDS); + assertNotNull(msg); + String receivedMsgValue = msg.getValue(); + log.info("received msg: {}", receivedMsgValue); + consumer2.acknowledge(msg); + + // cleanup. + consumer2.close(); + producer0.close(); + producer1.close(); + admin.topics().deletePartitionedTopic(topic); + } + + @Test(timeOut = 180 * 1000) + public void testPhasePartDeletion() throws Exception { + final String topic = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final String topicPattern = "persistent://public/default/tp.*"; + final String partition0 = topic + "-partition-0"; + final String partition1 = topic + "-partition-1"; + final String partition2 = topic + "-partition-2"; + final String subscription = "s1"; + admin.topics().createPartitionedTopic(topic, 3); + // Create consumer. + PatternMultiTopicsConsumerImpl c1 = (PatternMultiTopicsConsumerImpl) pulsarClient + .newConsumer(Schema.STRING) + .subscriptionName(subscription) + .isAckReceiptEnabled(true) + .subscriptionType(SubscriptionType.Shared) + .topicsPattern(Pattern.compile(topicPattern)).subscribe(); + // Check subscriptions. + Awaitility.await().untilAsserted(() -> { + ConcurrentHashMap> consumers + = WhiteboxImpl.getInternalState(c1, "consumers"); + ConcurrentHashMap partitionedTopics + = WhiteboxImpl.getInternalState(c1, "partitionedTopics"); + assertEquals(partitionedTopics.size(), 1); + assertEquals(partitionedTopics.get(topic), 3); + assertEquals(consumers.size(), 3); + assertTrue(consumers.containsKey(partition0)); + assertTrue(consumers.containsKey(partition1)); + assertTrue(consumers.containsKey(partition2)); + }); + // Delete partitions the first time. + admin.topics().delete(partition0, true); + // Check subscriptions. + Awaitility.await().untilAsserted(() -> { + ConcurrentHashMap> consumers + = WhiteboxImpl.getInternalState(c1, "consumers"); + ConcurrentHashMap partitionedTopics + = WhiteboxImpl.getInternalState(c1, "partitionedTopics"); + assertEquals(partitionedTopics.size(), 1); + assertEquals(partitionedTopics.get(topic), 3); + assertEquals(consumers.size(), 2); + assertTrue(consumers.containsKey(partition1)); + assertTrue(consumers.containsKey(partition2)); + }); + // Delete partitions the second time. + admin.topics().delete(partition1, true); + // Check subscriptions. + Awaitility.await().untilAsserted(() -> { + ConcurrentHashMap> consumers + = WhiteboxImpl.getInternalState(c1, "consumers"); + ConcurrentHashMap partitionedTopics + = WhiteboxImpl.getInternalState(c1, "partitionedTopics"); + assertEquals(partitionedTopics.size(), 1); + assertEquals(partitionedTopics.get(topic), 3); + assertEquals(consumers.size(), 1); + assertTrue(consumers.containsKey(partition2)); + }); + // Delete partitions the third time. + admin.topics().delete(partition2, true); + // Check subscriptions. + Awaitility.await().untilAsserted(() -> { + ConcurrentHashMap> consumers + = WhiteboxImpl.getInternalState(c1, "consumers"); + ConcurrentHashMap partitionedTopics + = WhiteboxImpl.getInternalState(c1, "partitionedTopics"); + assertEquals(partitionedTopics.size(), 0); + assertEquals(consumers.size(), 0); + }); + + // cleanup. + c1.close(); + admin.topics().deletePartitionedTopic(topic); + } + + @Test(timeOut = 180 * 1000) + public void testExpandPartitions() throws Exception { + final String topic = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final String topicPattern = "persistent://public/default/tp.*"; + final String partition0 = topic + "-partition-0"; + final String partition1 = topic + "-partition-1"; + final String subscription = "s1"; + admin.topics().createPartitionedTopic(topic, 2); + // Delete partitions. + admin.topics().delete(partition0, true); + admin.topics().delete(partition1, true); + // Create consumer. + PatternMultiTopicsConsumerImpl c1 = (PatternMultiTopicsConsumerImpl) pulsarClient + .newConsumer(Schema.STRING) + .subscriptionName(subscription) + .isAckReceiptEnabled(true) + .subscriptionType(SubscriptionType.Shared) + .topicsPattern(Pattern.compile(topicPattern)).subscribe(); + // Check subscriptions. + Awaitility.await().untilAsserted(() -> { + ConcurrentHashMap> consumers + = WhiteboxImpl.getInternalState(c1, "consumers"); + ConcurrentHashMap partitionedTopics + = WhiteboxImpl.getInternalState(c1, "partitionedTopics"); + assertEquals(partitionedTopics.size(), 0); + assertEquals(consumers.size(), 0); + }); + // Trigger partitions creation. + pulsarClient.newConsumer(Schema.STRING).subscriptionName(subscription) + .subscriptionType(SubscriptionType.Shared).topic(topic).subscribe().close(); + // Check subscriptions. + Awaitility.await().untilAsserted(() -> { + ConcurrentHashMap> consumers + = WhiteboxImpl.getInternalState(c1, "consumers"); + ConcurrentHashMap partitionedTopics + = WhiteboxImpl.getInternalState(c1, "partitionedTopics"); + assertEquals(partitionedTopics.size(), 1); + assertEquals(partitionedTopics.get(topic), 2); + assertEquals(consumers.size(), 2); + assertTrue(consumers.containsKey(partition0)); + assertTrue(consumers.containsKey(partition1)); + }); + // Expand partitions the first time. + admin.topics().updatePartitionedTopic(topic, 3); + final String partition2 = topic + "-partition-2"; + // Check subscriptions. + Awaitility.await().untilAsserted(() -> { + ConcurrentHashMap> consumers + = WhiteboxImpl.getInternalState(c1, "consumers"); + ConcurrentHashMap partitionedTopics + = WhiteboxImpl.getInternalState(c1, "partitionedTopics"); + assertEquals(partitionedTopics.size(), 1); + assertEquals(partitionedTopics.get(topic), 3); + assertEquals(consumers.size(), 3); + assertTrue(consumers.containsKey(partition0)); + assertTrue(consumers.containsKey(partition1)); + assertTrue(consumers.containsKey(partition2)); + }); + // Expand partitions the second time. + admin.topics().updatePartitionedTopic(topic, 4); + final String partition3 = topic + "-partition-3"; + // Check subscriptions. + Awaitility.await().untilAsserted(() -> { + ConcurrentHashMap> consumers + = WhiteboxImpl.getInternalState(c1, "consumers"); + ConcurrentHashMap partitionedTopics + = WhiteboxImpl.getInternalState(c1, "partitionedTopics"); + assertEquals(partitionedTopics.size(), 1); + assertEquals(partitionedTopics.get(topic), 4); + assertEquals(consumers.size(), 4); + assertTrue(consumers.containsKey(partition0)); + assertTrue(consumers.containsKey(partition1)); + assertTrue(consumers.containsKey(partition2)); + assertTrue(consumers.containsKey(partition3)); + }); + + // cleanup. + c1.close(); + admin.topics().deletePartitionedTopic(topic); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicListServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicListServiceTest.java index 2b0b852a27375..069794ec504dd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicListServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicListServiceTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.service; +import com.google.re2j.Pattern; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.namespace.NamespaceService; @@ -43,7 +44,6 @@ import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Semaphore; -import java.util.regex.Pattern; public class TopicListServiceTest { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicListWatcherTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicListWatcherTest.java index c232675779fca..641b1bd4e74b3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicListWatcherTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicListWatcherTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.service; +import com.google.re2j.Pattern; import org.apache.pulsar.common.topics.TopicList; import org.apache.pulsar.metadata.api.NotificationType; import static org.mockito.Mockito.mock; @@ -29,7 +30,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.regex.Pattern; public class TopicListWatcherTest { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicOwnerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicOwnerTest.java index 5a8fd34c9cdba..521d68cebe599 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicOwnerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicOwnerTest.java @@ -117,10 +117,19 @@ void setup() throws Exception { @AfterMethod(alwaysRun = true) void tearDown() throws Exception { for (int i = 0; i < BROKER_COUNT; i++) { - pulsarServices[i].close(); - pulsarAdmins[i].close(); + if (pulsarAdmins[i] != null) { + pulsarAdmins[i].close(); + pulsarAdmins[i] = null; + } + if (pulsarServices[i] != null) { + pulsarServices[i].close(); + pulsarServices[i] = null; + } + } + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; } - bkEnsemble.stop(); } @SuppressWarnings("unchecked") diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyTestUtils.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyTestUtils.java new file mode 100644 index 0000000000000..9cf688d62edc6 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyTestUtils.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import lombok.Cleanup; +import org.apache.pulsar.common.events.PulsarEvent; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TopicPolicies; + +public class TopicPolicyTestUtils { + + public static TopicPolicies getTopicPolicies(AbstractTopic topic) { + final TopicPolicies topicPolicies; + try { + topicPolicies = getTopicPolicies(topic.brokerService.getPulsar().getTopicPoliciesService(), + TopicName.get(topic.topic)); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException(e); + } + if (topicPolicies == null) { + throw new RuntimeException("No topic policies for " + topic); + } + return topicPolicies; + } + + public static TopicPolicies getTopicPolicies(TopicPoliciesService topicPoliciesService, TopicName topicName) + throws ExecutionException, InterruptedException { + return topicPoliciesService.getTopicPoliciesAsync(topicName, TopicPoliciesService.GetType.DEFAULT).get() + .orElse(null); + } + + public static TopicPolicies getLocalTopicPolicies(TopicPoliciesService topicPoliciesService, TopicName topicName) + throws ExecutionException, InterruptedException { + return topicPoliciesService.getTopicPoliciesAsync(topicName, TopicPoliciesService.GetType.LOCAL_ONLY).get() + .orElse(null); + } + + public static TopicPolicies getGlobalTopicPolicies(TopicPoliciesService topicPoliciesService, TopicName topicName) + throws ExecutionException, InterruptedException { + return topicPoliciesService.getTopicPoliciesAsync(topicName, TopicPoliciesService.GetType.GLOBAL_ONLY).get() + .orElse(null); + } + + public static Optional getTopicPoliciesBypassCache(TopicPoliciesService topicPoliciesService, + TopicName topicName) throws Exception { + @Cleanup final var reader = ((SystemTopicBasedTopicPoliciesService) topicPoliciesService) + .getNamespaceEventsSystemTopicFactory() + .createTopicPoliciesSystemTopicClient(topicName.getNamespaceObject()) + .newReader(); + PulsarEvent event = null; + while (reader.hasMoreEvents()) { + event = reader.readNext().getValue(); + } + return Optional.ofNullable(event).map(e -> e.getTopicPoliciesEvent().getPolicies()); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisTopicPublishRateThrottleTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPublishRateThrottleTest.java similarity index 72% rename from pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisTopicPublishRateThrottleTest.java rename to pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPublishRateThrottleTest.java index c22ed41fc1533..721d049342552 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisTopicPublishRateThrottleTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPublishRateThrottleTest.java @@ -18,63 +18,31 @@ */ package org.apache.pulsar.broker.service; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.common.policies.data.PublishRate; +import org.apache.pulsar.broker.qos.AsyncTokenBucket; import org.awaitility.Awaitility; import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - @Test(groups = "broker") -public class PrecisTopicPublishRateThrottleTest extends BrokerTestBase{ +public class TopicPublishRateThrottleTest extends BrokerTestBase{ + @BeforeMethod(alwaysRun = true) @Override protected void setup() throws Exception { - //No-op + AsyncTokenBucket.switchToConsistentTokensView(); } + @AfterMethod(alwaysRun = true) @Override protected void cleanup() throws Exception { - //No-op - } - - @Test - public void testPrecisTopicPublishRateLimitingDisabled() throws Exception { - PublishRate publishRate = new PublishRate(1,10); - // disable precis topic publish rate limiting - conf.setPreciseTopicPublishRateLimiterEnable(false); - conf.setMaxPendingPublishRequestsPerConnection(0); - super.baseSetup(); - admin.namespaces().setPublishRate("prop/ns-abc", publishRate); - final String topic = "persistent://prop/ns-abc/testPrecisTopicPublishRateLimiting"; - org.apache.pulsar.client.api.Producer producer = pulsarClient.newProducer() - .topic(topic) - .producerName("producer-name") - .create(); - - Topic topicRef = pulsar.getBrokerService().getTopicReference(topic).get(); - Assert.assertNotNull(topicRef); - MessageId messageId = null; - try { - // first will be success - messageId = producer.sendAsync(new byte[10]).get(500, TimeUnit.MILLISECONDS); - Assert.assertNotNull(messageId); - // second will be success - messageId = producer.sendAsync(new byte[10]).get(500, TimeUnit.MILLISECONDS); - Assert.assertNotNull(messageId); - } catch (TimeoutException e) { - // No-op - } - Thread.sleep(1000); - try { - messageId = producer.sendAsync(new byte[10]).get(1, TimeUnit.SECONDS); - } catch (TimeoutException e) { - // No-op - } - Assert.assertNotNull(messageId); super.internalCleanup(); + AsyncTokenBucket.resetToDefaultEventualConsistentTokensView(); } @Test @@ -103,7 +71,6 @@ public void testProducerBlockedByPrecisTopicPublishRateLimiting() throws Excepti } catch (TimeoutException e) { // No-op } - super.internalCleanup(); } @Test @@ -139,7 +106,6 @@ public void testPrecisTopicPublishRateLimitingProduceRefresh() throws Exception // No-op } Assert.assertNotNull(messageId); - super.internalCleanup(); } @Test @@ -164,9 +130,10 @@ public void testBrokerLevelPublishRateDynamicUpdate() throws Exception{ "" + rateInMsg)); Topic topicRef = pulsar.getBrokerService().getTopicReference(topic).get(); Assert.assertNotNull(topicRef); - PrecisePublishLimiter limiter = ((PrecisePublishLimiter) ((AbstractTopic) topicRef).topicPublishRateLimiter); - Awaitility.await().untilAsserted(() -> Assert.assertEquals(limiter.publishMaxMessageRate, rateInMsg)); - Assert.assertEquals(limiter.publishMaxByteRate, 0); + PublishRateLimiterImpl limiter = ((PublishRateLimiterImpl) ((AbstractTopic) topicRef).topicPublishRateLimiter); + Awaitility.await() + .untilAsserted(() -> Assert.assertEquals(limiter.getTokenBucketOnMessage().getRate(), rateInMsg)); + Assert.assertNull(limiter.getTokenBucketOnByte()); // maxPublishRatePerTopicInBytes admin.brokers().updateDynamicConfiguration("maxPublishRatePerTopicInBytes", "" + rateInByte); @@ -174,10 +141,10 @@ public void testBrokerLevelPublishRateDynamicUpdate() throws Exception{ .untilAsserted(() -> Assert.assertEquals(admin.brokers().getAllDynamicConfigurations().get("maxPublishRatePerTopicInBytes"), "" + rateInByte)); - Awaitility.await().untilAsserted(() -> Assert.assertEquals(limiter.publishMaxByteRate, rateInByte)); - Assert.assertEquals(limiter.publishMaxMessageRate, rateInMsg); + Awaitility.await() + .untilAsserted(() -> Assert.assertEquals(limiter.getTokenBucketOnByte().getRate(), rateInByte)); + Assert.assertEquals(limiter.getTokenBucketOnMessage().getRate(), rateInMsg); producer.close(); - super.internalCleanup(); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java index 72f940d238d8c..7e8454f6c7eef 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java @@ -33,7 +33,7 @@ import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; @@ -124,7 +124,7 @@ public void testMarkerDelete() throws Exception { // maxReadPosition move to msgId1, msgId2 have not be committed assertEquals(admin.topics().getInternalStats(topicName).cursors.get(subName).markDeletePosition, - PositionImpl.get(msgId1.getLedgerId(), msgId1.getEntryId()).toString()); + PositionFactory.create(msgId1.getLedgerId(), msgId1.getEntryId()).toString()); MessageIdImpl msgId3 = (MessageIdImpl) producer.newMessage(txn3).send(); txn2.commit().get(); @@ -135,7 +135,7 @@ public void testMarkerDelete() throws Exception { // maxReadPosition move to txn1 marker, so entryId is msgId2.getEntryId() + 1, // because send msgId2 before commit txn1 assertEquals(admin.topics().getInternalStats(topicName).cursors.get(subName).markDeletePosition, - PositionImpl.get(msgId2.getLedgerId(), msgId2.getEntryId() + 1).toString()); + PositionFactory.create(msgId2.getLedgerId(), msgId2.getEntryId() + 1).toString()); MessageIdImpl msgId4 = (MessageIdImpl) producer.newMessage(txn4).send(); txn3.commit().get(); @@ -145,13 +145,13 @@ public void testMarkerDelete() throws Exception { // maxReadPosition move to txn2 marker, because msgId4 have not be committed assertEquals(admin.topics().getInternalStats(topicName).cursors.get(subName).markDeletePosition, - PositionImpl.get(msgId3.getLedgerId(), msgId3.getEntryId() + 1).toString()); + PositionFactory.create(msgId3.getLedgerId(), msgId3.getEntryId() + 1).toString()); txn4.abort().get(); // maxReadPosition move to txn4 abort marker, so entryId is msgId4.getEntryId() + 2 Awaitility.await().untilAsserted(() -> assertEquals(admin.topics().getInternalStats(topicName) - .cursors.get(subName).markDeletePosition, PositionImpl.get(msgId4.getLedgerId(), + .cursors.get(subName).markDeletePosition, PositionFactory.create(msgId4.getLedgerId(), msgId4.getEntryId() + 2).toString())); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionalReplicateSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionalReplicateSubscriptionTest.java new file mode 100644 index 0000000000000..aa39e859a8c3d --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionalReplicateSubscriptionTest.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import com.google.common.collect.Sets; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.transaction.Transaction; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.SystemTopicNames; +import org.apache.pulsar.common.partition.PartitionedTopicMetadata; +import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +public class TransactionalReplicateSubscriptionTest extends ReplicatorTestBase { + @Override + @BeforeClass(timeOut = 300000) + public void setup() throws Exception { + super.setup(); + admin1.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString()); + createTransactionCoordinatorAssign(16, pulsar1); + } + + @Override + @AfterClass(alwaysRun = true, timeOut = 300000) + public void cleanup() throws Exception { + super.cleanup(); + } + + /** + * enable transaction coordinator for the cluster1 + */ + @Override + public void setConfig1DefaultValue(){ + super.setConfig1DefaultValue(); + config1.setTransactionCoordinatorEnabled(true); + } + + protected void createTransactionCoordinatorAssign(int numPartitionsOfTC, PulsarService pulsarService) throws MetadataStoreException { + pulsarService.getPulsarResources() + .getNamespaceResources() + .getPartitionedTopicResources() + .createPartitionedTopic(SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN, + new PartitionedTopicMetadata(numPartitionsOfTC)); + } + + /** + * Test replicated subscription with transaction. + * @throws Exception + */ + @Test + public void testReplicatedSubscribeAndSwitchToStandbyClusterWithTransaction() throws Exception { + final String namespace = BrokerTestUtil.newUniqueName("pulsar/ns_"); + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + namespace + "/tp_"); + final String subscriptionName = "s1"; + final boolean isReplicatedSubscription = true; + final int messagesCount = 20; + final LinkedHashSet sentMessages = new LinkedHashSet<>(); + final Set receivedMessages = Collections.synchronizedSet(new LinkedHashSet<>()); + admin1.namespaces().createNamespace(namespace); + admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2")); + admin1.topics().createNonPartitionedTopic(topicName); + admin1.topics().createSubscription(topicName, subscriptionName, MessageId.earliest, isReplicatedSubscription); + final PersistentTopic topic1 = + (PersistentTopic) pulsar1.getBrokerService().getTopic(topicName, false).join().get(); + + // Send messages + // Wait for the topic created on the cluster2. + // Wait for the snapshot created. + final PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).enableTransaction(true).build(); + Producer producer1 = client1.newProducer(Schema.STRING).topic(topicName).enableBatching(false).create(); + Consumer consumer1 = client1.newConsumer(Schema.STRING).topic(topicName) + .subscriptionName(subscriptionName).replicateSubscriptionState(isReplicatedSubscription).subscribe(); + Transaction txn1 = client1.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build().get(); + for (int i = 0; i < messagesCount / 2; i++) { + String msg = i + ""; + producer1.newMessage(txn1).value(msg).send(); + sentMessages.add(msg); + } + txn1.commit().get(); + Awaitility.await().untilAsserted(() -> { + final var replicators = topic1.getReplicators(); + assertTrue(replicators != null && replicators.size() == 1, "Replicator should started"); + assertTrue(replicators.values().iterator().next().isConnected(), "Replicator should be connected"); + assertTrue(topic1.getReplicatedSubscriptionController().get().getLastCompletedSnapshotId().isPresent(), + "One snapshot should be finished"); + }); + final PersistentTopic topic2 = + (PersistentTopic) pulsar2.getBrokerService().getTopic(topicName, false).join().get(); + Awaitility.await().untilAsserted(() -> { + assertTrue(topic2.getReplicatedSubscriptionController().isPresent(), + "Replicated subscription controller should created"); + }); + Transaction txn2 = client1.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build().get(); + for (int i = messagesCount / 2; i < messagesCount; i++) { + String msg = i + ""; + producer1.newMessage(txn2).value(msg).send(); + sentMessages.add(msg); + } + txn2.commit().get(); + + // Consume half messages and wait the subscription created on the cluster2. + for (int i = 0; i < messagesCount / 2; i++){ + Message message = consumer1.receive(2, TimeUnit.SECONDS); + if (message == null) { + fail("Should not receive null."); + } + receivedMessages.add(message.getValue()); + consumer1.acknowledge(message); + } + Awaitility.await().untilAsserted(() -> { + assertNotNull(topic2.getSubscriptions().get(subscriptionName), "Subscription should created"); + }); + + // Switch client to cluster2. + // Since the cluster1 was not crash, all messages will be replicated to the cluster2. + consumer1.close(); + final PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()).build(); + final Consumer consumer2 = client2.newConsumer(Schema.AUTO_CONSUME()).topic(topicName) + .subscriptionName(subscriptionName).replicateSubscriptionState(isReplicatedSubscription).subscribe(); + + // Verify all messages will be consumed. + Awaitility.await().untilAsserted(() -> { + while (true) { + Message message = consumer2.receive(2, TimeUnit.SECONDS); + if (message != null) { + receivedMessages.add(message.getValue().toString()); + consumer2.acknowledge(message); + } else { + break; + } + } + assertEquals(receivedMessages.size(), sentMessages.size()); + }); + + consumer2.close(); + producer1.close(); + client1.close(); + client2.close(); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ZkSessionExpireTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ZkSessionExpireTest.java new file mode 100644 index 0000000000000..143557b008b23 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ZkSessionExpireTest.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.ProducerImpl; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class ZkSessionExpireTest extends NetworkErrorTestBase { + + private java.util.function.Consumer settings; + + @AfterMethod(alwaysRun = true) + @Override + public void cleanup() throws Exception { + super.cleanup(); + } + + private void setupWithSettings(java.util.function.Consumer settings) throws Exception { + this.settings = settings; + super.setup(); + } + + protected void setConfigDefaults(ServiceConfiguration config, String clusterName, int zkPort) { + super.setConfigDefaults(config, clusterName, zkPort); + settings.accept(config); + } + + @DataProvider(name = "settings") + public Object[][] settings() { + return new Object[][]{ + {false, NetworkErrorTestBase.PreferBrokerModularLoadManager.class}, + {true, NetworkErrorTestBase.PreferBrokerModularLoadManager.class} + // Create a separate PR to add this test case. + // {true, NetworkErrorTestBase.PreferExtensibleLoadManager.class}. + }; + } + + @Test(timeOut = 600 * 1000, dataProvider = "settings") + public void testTopicUnloadAfterSessionRebuild(boolean enableSystemTopic, Class loadManager) throws Exception { + // Setup. + setupWithSettings(config -> { + config.setManagedLedgerMaxEntriesPerLedger(1); + config.setSystemTopicEnabled(enableSystemTopic); + config.setTopicLevelPoliciesEnabled(enableSystemTopic); + if (loadManager != null) { + config.setLoadManagerClassName(loadManager.getName()); + } + }); + + // Init topic. + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp"); + admin1.topics().createNonPartitionedTopic(topicName); + admin1.topics().createSubscription(topicName, "s1", MessageId.earliest); + + // Inject a prefer mechanism, so that all topics will be assigned to broker1, which can be injected a ZK + // session expire error. + setPreferBroker(pulsar1); + admin1.namespaces().unload(defaultNamespace); + admin2.namespaces().unload(defaultNamespace); + + // Confirm all brokers registered. + Awaitility.await().untilAsserted(() -> { + assertEquals(getAvailableBrokers(pulsar1).size(), 2); + assertEquals(getAvailableBrokers(pulsar2).size(), 2); + }); + + // Load up a topic, and it will be assigned to broker1. + ProducerImpl p1 = (ProducerImpl) client1.newProducer(Schema.STRING).topic(topicName) + .sendTimeout(10, TimeUnit.SECONDS).create(); + Topic broker1Topic1 = pulsar1.getBrokerService().getTopic(topicName, false).join().get(); + assertNotNull(broker1Topic1); + clearPreferBroker(); + + // Inject a ZK session expire error, and wait for broker1 to offline. + metadataZKProxy.rejectAllConnections(); + metadataZKProxy.disconnectFrontChannels(); + Awaitility.await().untilAsserted(() -> { + assertEquals(getAvailableBrokers(pulsar2).size(), 1); + }); + + // Send messages continuously. + // Verify: the topic was transferred to broker2. + CompletableFuture broker1Send1 = p1.sendAsync("broker1_msg1"); + Producer p2 = client2.newProducer(Schema.STRING).topic(topicName) + .sendTimeout(10, TimeUnit.SECONDS).create(); + CompletableFuture broker2Send1 = p2.sendAsync("broker2_msg1"); + Awaitility.await().untilAsserted(() -> { + CompletableFuture> future = pulsar2.getBrokerService().getTopic(topicName, false); + assertNotNull(future); + assertTrue(future.isDone() && !future.isCompletedExceptionally()); + Optional optional = future.join(); + assertTrue(optional != null && !optional.isEmpty()); + }); + + // Both two brokers assumed they are the owner of the topic. + Topic broker1Topic2 = pulsar1.getBrokerService().getTopic(topicName, false).join().get(); + Topic broker2Topic2 = pulsar2.getBrokerService().getTopic(topicName, false).join().get(); + assertNotNull(broker1Topic2); + assertNotNull(broker2Topic2); + + // Send messages continuously. + // Publishing to broker-1 will fail. + // Publishing to broker-2 will success. + CompletableFuture broker1Send2 = p1.sendAsync("broker1_msg2"); + CompletableFuture broker2Send2 = p2.sendAsync("broker2_msg2"); + try { + broker1Send1.join(); + broker1Send2.join(); + p1.getClientCnx(); + fail("expected a publish error"); + } catch (Exception ex) { + // Expected. + } + broker2Send1.join(); + broker2Send2.join(); + + // Broker rebuild ZK session. + metadataZKProxy.unRejectAllConnections(); + Awaitility.await().untilAsserted(() -> { + assertEquals(getAvailableBrokers(pulsar1).size(), 2); + assertEquals(getAvailableBrokers(pulsar2).size(), 2); + }); + + // Verify: the topic on broker-1 will be unloaded. + // Verify: the topic on broker-2 is fine. + Awaitility.await().atMost(Duration.ofSeconds(10)).untilAsserted(() -> { + CompletableFuture> future = pulsar1.getBrokerService().getTopic(topicName, false); + assertTrue(future == null || future.isCompletedExceptionally()); + }); + Topic broker2Topic3 = pulsar2.getBrokerService().getTopic(topicName, false).join().get(); + assertNotNull(broker2Topic3); + + // Send messages continuously. + // Verify: p1.send will success(it will connect to broker-2). + // Verify: p2.send will success. + CompletableFuture broker1Send3 = p1.sendAsync("broker1_msg3"); + CompletableFuture broker2Send3 = p2.sendAsync("broker2_msg3"); + broker1Send3.join(); + broker2Send3.join(); + + long msgBacklog = admin2.topics().getStats(topicName).getSubscriptions().get("s1").getMsgBacklog(); + log.info("msgBacklog: {}", msgBacklog); + + // cleanup. + p1.close(); + p2.close(); + admin2.topics().delete(topicName, false); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumersTest.java index b2638d53ab1c3..6b0f48a57cfe3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumersTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentStickyKeyDispatcherMultipleConsumersTest.java @@ -128,15 +128,15 @@ public void testSendMessage() throws BrokerServiceException { assertEquals(byteBuf.toString(UTF_8), "message" + index); }; return mockPromise; - }).when(consumerMock).sendMessages(any(List.class), any(EntryBatchSizes.class), any(), - anyInt(), anyLong(), anyLong(), any(RedeliveryTracker.class)); + }).when(consumerMock).sendMessages(any(List.class), any(List.class), any(EntryBatchSizes.class), any(), + anyInt(), anyLong(), anyLong(), any(RedeliveryTracker.class), anyLong()); try { nonpersistentDispatcher.sendMessages(entries); } catch (Exception e) { fail("Failed to sendMessages.", e); } - verify(consumerMock, times(1)).sendMessages(any(List.class), any(EntryBatchSizes.class), - eq(null), anyInt(), anyLong(), anyLong(), any(RedeliveryTracker.class)); + verify(consumerMock, times(1)).sendMessages(any(List.class), any(List.class), any(EntryBatchSizes.class), + eq(null), anyInt(), anyLong(), anyLong(), any(RedeliveryTracker.class), anyLong()); } @Test(timeOut = 10000) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java index eb25489076b22..e0d6a432bdad2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java @@ -18,11 +18,13 @@ */ package org.apache.pulsar.broker.service.nonpersistent; +import java.lang.reflect.Field; import java.util.Optional; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import lombok.Cleanup; +import org.apache.pulsar.broker.service.AbstractTopic; import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.broker.service.SubscriptionOption; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -35,9 +37,7 @@ import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.TopicStats; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.awaitility.Awaitility; -import org.junit.Assert; import org.mockito.Mockito; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -126,7 +126,7 @@ public void testCreateNonExistentPartitions() throws PulsarAdminException, Pulsa } catch (PulsarClientException.TopicDoesNotExistException ignored) { } - Assert.assertEquals(admin.topics().getPartitionedTopicMetadata(topicName).partitions, 4); + assertEquals(admin.topics().getPartitionedTopicMetadata(topicName).partitions, 4); } @@ -211,44 +211,64 @@ public void testSubscriptionsOnNonPersistentTopic() throws Exception { .subscriptionMode(SubscriptionMode.Durable) .subscribe(); - ConcurrentOpenHashMap subscriptionMap = mockTopic.getSubscriptions(); - Assert.assertEquals(subscriptionMap.size(), 4); + final var subscriptionMap = mockTopic.getSubscriptions(); + assertEquals(subscriptionMap.size(), 4); // Check exclusive subscription NonPersistentSubscription exclusiveSub = subscriptionMap.get(exclusiveSubName); - Assert.assertNotNull(exclusiveSub); + assertNotNull(exclusiveSub); exclusiveConsumer.close(); Awaitility.waitAtMost(10, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS) .until(() -> subscriptionMap.get(exclusiveSubName) == null); // Check failover subscription NonPersistentSubscription failoverSub = subscriptionMap.get(failoverSubName); - Assert.assertNotNull(failoverSub); + assertNotNull(failoverSub); failoverConsumer1.close(); failoverSub = subscriptionMap.get(failoverSubName); - Assert.assertNotNull(failoverSub); + assertNotNull(failoverSub); failoverConsumer2.close(); Awaitility.waitAtMost(10, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS) .until(() -> subscriptionMap.get(failoverSubName) == null); // Check shared subscription NonPersistentSubscription sharedSub = subscriptionMap.get(sharedSubName); - Assert.assertNotNull(sharedSub); + assertNotNull(sharedSub); sharedConsumer1.close(); sharedSub = subscriptionMap.get(sharedSubName); - Assert.assertNotNull(sharedSub); + assertNotNull(sharedSub); sharedConsumer2.close(); Awaitility.waitAtMost(10, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS) .until(() -> subscriptionMap.get(sharedSubName) == null); // Check KeyShared subscription NonPersistentSubscription keySharedSub = subscriptionMap.get(keySharedSubName); - Assert.assertNotNull(keySharedSub); + assertNotNull(keySharedSub); keySharedConsumer1.close(); keySharedSub = subscriptionMap.get(keySharedSubName); - Assert.assertNotNull(keySharedSub); + assertNotNull(keySharedSub); keySharedConsumer2.close(); Awaitility.waitAtMost(10, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS) .until(() -> subscriptionMap.get(keySharedSubName) == null); } + + + @Test + public void testRemoveProducerOnNonPersistentTopic() throws Exception { + final String topicName = "non-persistent://prop/ns-abc/topic_" + UUID.randomUUID(); + + Producer producer = pulsarClient.newProducer() + .topic(topicName) + .create(); + + NonPersistentTopic topic = (NonPersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get(); + Field field = AbstractTopic.class.getDeclaredField("userCreatedProducerCount"); + field.setAccessible(true); + int userCreatedProducerCount = (int) field.get(topic); + assertEquals(userCreatedProducerCount, 1); + + producer.close(); + userCreatedProducerCount = (int) field.get(topic); + assertEquals(userCreatedProducerCount, 0); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java index 0a82b2b4c3cb0..20ea33fb3e1ed 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java @@ -18,12 +18,17 @@ */ package org.apache.pulsar.broker.service.persistent; -import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX; +import static org.apache.bookkeeper.mledger.ManagedCursor.CURSOR_INTERNAL_PROPERTY_PREFIX; +import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.Metric; +import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.parseMetrics; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import com.google.common.collect.Multimap; import java.io.ByteArrayOutputStream; import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -32,14 +37,14 @@ import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.LedgerHandle; -import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.pulsar.PrometheusMetricsTestUtil; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory; import org.apache.pulsar.broker.service.Dispatcher; -import org.apache.pulsar.broker.stats.PrometheusMetricsTest; -import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGenerator; import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; @@ -47,6 +52,7 @@ import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @Test(groups = "broker") @@ -97,7 +103,7 @@ public void testBucketDelayedDeliveryWithAllConsumersDisconnecting() throws Exce Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher.getNumberOfDelayedMessages(), 1000)); List bucketKeys = ((PersistentDispatcherMultipleConsumers) dispatcher).getCursor().getCursorProperties().keySet().stream() - .filter(x -> x.startsWith(ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX)).toList(); + .filter(x -> x.startsWith(CURSOR_INTERNAL_PROPERTY_PREFIX)).toList(); c1.close(); @@ -112,7 +118,7 @@ public void testBucketDelayedDeliveryWithAllConsumersDisconnecting() throws Exce Dispatcher dispatcher2 = pulsar.getBrokerService().getTopicReference(topic).get().getSubscription("sub").getDispatcher(); List bucketKeys2 = ((PersistentDispatcherMultipleConsumers) dispatcher2).getCursor().getCursorProperties().keySet().stream() - .filter(x -> x.startsWith(ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX)).toList(); + .filter(x -> x.startsWith(CURSOR_INTERNAL_PROPERTY_PREFIX)).toList(); Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher2.getNumberOfDelayedMessages(), 1000)); Assert.assertEquals(bucketKeys, bucketKeys2); @@ -211,11 +217,11 @@ public void testBucketDelayedIndexMetrics() throws Exception { Thread.sleep(2000); ByteArrayOutputStream output = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, true, true, output); + PrometheusMetricsTestUtil.generate(pulsar, true, true, true, output); String metricsStr = output.toString(StandardCharsets.UTF_8); - Multimap metricsMap = PrometheusMetricsTest.parseMetrics(metricsStr); + Multimap metricsMap = parseMetrics(metricsStr); - List bucketsMetrics = + List bucketsMetrics = metricsMap.get("pulsar_delayed_message_index_bucket_total").stream() .filter(metric -> metric.tags.get("topic").equals(topic)).toList(); MutableInt bucketsSum = new MutableInt(); @@ -224,12 +230,12 @@ public void testBucketDelayedIndexMetrics() throws Exception { bucketsSum.add(metric.value); }); assertEquals(6, bucketsSum.intValue()); - Optional bucketsTopicMetric = + Optional bucketsTopicMetric = bucketsMetrics.stream().filter(metric -> !metric.tags.containsKey("subscription")).findFirst(); assertTrue(bucketsTopicMetric.isPresent()); assertEquals(bucketsSum.intValue(), bucketsTopicMetric.get().value); - List loadedIndexMetrics = + List loadedIndexMetrics = metricsMap.get("pulsar_delayed_message_index_loaded").stream() .filter(metric -> metric.tags.get("topic").equals(topic)).toList(); MutableInt loadedIndexSum = new MutableInt(); @@ -238,12 +244,12 @@ public void testBucketDelayedIndexMetrics() throws Exception { loadedIndexSum.add(metric.value); }).count(); assertEquals(2, count); - Optional loadedIndexTopicMetrics = + Optional loadedIndexTopicMetrics = bucketsMetrics.stream().filter(metric -> !metric.tags.containsKey("subscription")).findFirst(); assertTrue(loadedIndexTopicMetrics.isPresent()); assertEquals(loadedIndexSum.intValue(), loadedIndexTopicMetrics.get().value); - List snapshotSizeBytesMetrics = + List snapshotSizeBytesMetrics = metricsMap.get("pulsar_delayed_message_index_bucket_snapshot_size_bytes").stream() .filter(metric -> metric.tags.get("topic").equals(topic)).toList(); MutableInt snapshotSizeBytesSum = new MutableInt(); @@ -253,12 +259,12 @@ public void testBucketDelayedIndexMetrics() throws Exception { snapshotSizeBytesSum.add(metric.value); }).count(); assertEquals(2, count); - Optional snapshotSizeBytesTopicMetrics = + Optional snapshotSizeBytesTopicMetrics = snapshotSizeBytesMetrics.stream().filter(metric -> !metric.tags.containsKey("subscription")).findFirst(); assertTrue(snapshotSizeBytesTopicMetrics.isPresent()); assertEquals(snapshotSizeBytesSum.intValue(), snapshotSizeBytesTopicMetrics.get().value); - List opCountMetrics = + List opCountMetrics = metricsMap.get("pulsar_delayed_message_index_bucket_op_count").stream() .filter(metric -> metric.tags.get("topic").equals(topic)).toList(); MutableInt opCountMetricsSum = new MutableInt(); @@ -270,14 +276,14 @@ public void testBucketDelayedIndexMetrics() throws Exception { opCountMetricsSum.add(metric.value); }).count(); assertEquals(2, count); - Optional opCountTopicMetrics = + Optional opCountTopicMetrics = opCountMetrics.stream() .filter(metric -> metric.tags.get("state").equals("succeed") && metric.tags.get("type") .equals("create") && !metric.tags.containsKey("subscription")).findFirst(); assertTrue(opCountTopicMetrics.isPresent()); assertEquals(opCountMetricsSum.intValue(), opCountTopicMetrics.get().value); - List opLatencyMetrics = + List opLatencyMetrics = metricsMap.get("pulsar_delayed_message_index_bucket_op_latency_ms").stream() .filter(metric -> metric.tags.get("topic").equals(topic)).toList(); MutableInt opLatencyMetricsSum = new MutableInt(); @@ -289,7 +295,7 @@ public void testBucketDelayedIndexMetrics() throws Exception { opLatencyMetricsSum.add(metric.value); }).count(); assertTrue(count >= 2); - Optional opLatencyTopicMetrics = + Optional opLatencyTopicMetrics = opCountMetrics.stream() .filter(metric -> metric.tags.get("type").equals("create") && !metric.tags.containsKey("subscription")).findFirst(); @@ -297,10 +303,10 @@ public void testBucketDelayedIndexMetrics() throws Exception { assertEquals(opLatencyMetricsSum.intValue(), opLatencyTopicMetrics.get().value); ByteArrayOutputStream namespaceOutput = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, false, true, true, namespaceOutput); - Multimap namespaceMetricsMap = PrometheusMetricsTest.parseMetrics(namespaceOutput.toString(StandardCharsets.UTF_8)); + PrometheusMetricsTestUtil.generate(pulsar, false, true, true, namespaceOutput); + Multimap namespaceMetricsMap = parseMetrics(namespaceOutput.toString(StandardCharsets.UTF_8)); - Optional namespaceMetric = + Optional namespaceMetric = namespaceMetricsMap.get("pulsar_delayed_message_index_bucket_total").stream().findFirst(); assertTrue(namespaceMetric.isPresent()); assertEquals(6, namespaceMetric.get().value); @@ -353,4 +359,121 @@ public void testDelete() throws Exception { } } } + + @DataProvider(name = "subscriptionTypes") + public Object[][] subscriptionTypes() { + return new Object[][]{ + {SubscriptionType.Shared}, + {SubscriptionType.Key_Shared}, + {SubscriptionType.Failover}, + {SubscriptionType.Exclusive}, + }; + } + + /** + * see: https://github.com/apache/pulsar/pull/21595. + */ + @Test(dataProvider = "subscriptionTypes") + public void testDeleteTopicIfCursorPropsEmpty(SubscriptionType subscriptionType) throws Exception { + final String topic = BrokerTestUtil.newUniqueName("persistent://my-property/my-ns/tp_"); + final String subscriptionName = "s1"; + // create a topic. + admin.topics().createNonPartitionedTopic(topic); + // create a subscription without props. + admin.topics().createSubscription(topic, subscriptionName, MessageId.earliest); + pulsarClient.newConsumer().topic(topic).subscriptionName(subscriptionName) + .subscriptionType(subscriptionType).subscribe().close(); + ManagedCursor cursor = findCursor(topic, subscriptionName); + assertNotNull(cursor); + assertTrue(cursor.getCursorProperties() == null || cursor.getCursorProperties().isEmpty()); + // Test topic deletion is successful. + admin.topics().delete(topic); + } + + /** + * see: https://github.com/apache/pulsar/pull/21595. + */ + @Test(dataProvider = "subscriptionTypes") + public void testDeletePartitionedTopicIfCursorPropsEmpty(SubscriptionType subscriptionType) throws Exception { + final String topic = BrokerTestUtil.newUniqueName("persistent://my-property/my-ns/tp_"); + final String subscriptionName = "s1"; + // create a topic. + admin.topics().createPartitionedTopic(topic, 2); + // create a subscription without props. + admin.topics().createSubscription(topic, subscriptionName, MessageId.earliest); + pulsarClient.newConsumer().topic(topic).subscriptionName(subscriptionName) + .subscriptionType(subscriptionType).subscribe().close(); + ManagedCursor cursor = findCursor(topic + "-partition-0", subscriptionName); + assertNotNull(cursor); + assertTrue(cursor.getCursorProperties() == null || cursor.getCursorProperties().isEmpty()); + // Test topic deletion is successful. + admin.topics().deletePartitionedTopic(topic); + } + + /** + * see: https://github.com/apache/pulsar/pull/21595. + */ + @Test(dataProvider = "subscriptionTypes") + public void testDeleteTopicIfCursorPropsNotEmpty(SubscriptionType subscriptionType) throws Exception { + final String topic = BrokerTestUtil.newUniqueName("persistent://my-property/my-ns/tp_"); + final String subscriptionName = "s1"; + // create a topic. + admin.topics().createNonPartitionedTopic(topic); + // create a subscription without props. + admin.topics().createSubscription(topic, subscriptionName, MessageId.earliest); + pulsarClient.newConsumer().topic(topic).subscriptionName(subscriptionName) + .subscriptionType(subscriptionType).subscribe().close(); + ManagedCursor cursor = findCursor(topic, subscriptionName); + assertNotNull(cursor); + assertTrue(cursor.getCursorProperties() == null || cursor.getCursorProperties().isEmpty()); + // Put a subscription prop. + Map properties = new HashMap<>(); + properties.put("ignore", "ignore"); + admin.topics().updateSubscriptionProperties(topic, subscriptionName, properties); + assertTrue(cursor.getCursorProperties() != null && !cursor.getCursorProperties().isEmpty()); + // Test topic deletion is successful. + admin.topics().delete(topic); + } + + /** + * see: https://github.com/apache/pulsar/pull/21595. + */ + @Test(dataProvider = "subscriptionTypes") + public void testDeletePartitionedTopicIfCursorPropsNotEmpty(SubscriptionType subscriptionType) throws Exception { + final String topic = BrokerTestUtil.newUniqueName("persistent://my-property/my-ns/tp_"); + final String subscriptionName = "s1"; + // create a topic. + admin.topics().createPartitionedTopic(topic, 2); + pulsarClient.newProducer().topic(topic).create().close(); + // create a subscription without props. + admin.topics().createSubscription(topic, subscriptionName, MessageId.earliest); + pulsarClient.newConsumer().topic(topic).subscriptionName(subscriptionName) + .subscriptionType(subscriptionType).subscribe().close(); + + ManagedCursor cursor = findCursor(topic + "-partition-0", subscriptionName); + assertNotNull(cursor); + assertTrue(cursor.getCursorProperties() == null || cursor.getCursorProperties().isEmpty()); + // Put a subscription prop. + Map properties = new HashMap<>(); + properties.put("ignore", "ignore"); + admin.topics().updateSubscriptionProperties(topic, subscriptionName, properties); + assertTrue(cursor.getCursorProperties() != null && !cursor.getCursorProperties().isEmpty()); + // Test topic deletion is successful. + admin.topics().deletePartitionedTopic(topic); + } + + + private ManagedCursor findCursor(String topic, String subscriptionName) { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topic, false).join().get(); + Iterator cursorIterator = persistentTopic.getManagedLedger().getCursors().iterator(); + while (cursorIterator.hasNext()) { + ManagedCursor managedCursor = cursorIterator.next(); + if (managedCursor == null || !managedCursor.getName().equals(subscriptionName)) { + continue; + } + return managedCursor; + } + return null; + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/DelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/DelayedDeliveryTest.java index 2db404ed90de5..3ca966d210886 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/DelayedDeliveryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/DelayedDeliveryTest.java @@ -23,6 +23,8 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import io.opentelemetry.api.common.Attributes; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -35,6 +37,9 @@ import org.apache.bookkeeper.client.BKException; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.service.Dispatcher; +import org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil; +import org.apache.pulsar.broker.stats.OpenTelemetryTopicStats; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -44,6 +49,7 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterClass; @@ -68,6 +74,12 @@ public void cleanup() throws Exception { super.internalCleanup(); } + @Override + protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder pulsarTestContextBuilder) { + super.customizeMainPulsarTestContextBuilder(pulsarTestContextBuilder); + pulsarTestContextBuilder.enableOpenTelemetry(true); + } + @Test public void testDelayedDelivery() throws Exception { String topic = BrokerTestUtil.newUniqueName("testNegativeAcks"); @@ -105,6 +117,16 @@ public void testDelayedDelivery() throws Exception { Message msg = sharedConsumer.receive(100, TimeUnit.MILLISECONDS); assertNull(msg); + var attributes = Attributes.builder() + .put(OpenTelemetryAttributes.PULSAR_DOMAIN, "persistent") + .put(OpenTelemetryAttributes.PULSAR_TENANT, "public") + .put(OpenTelemetryAttributes.PULSAR_NAMESPACE, "public/default") + .put(OpenTelemetryAttributes.PULSAR_TOPIC, "persistent://public/default/" + topic) + .build(); + var metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + BrokerOpenTelemetryTestUtil.assertMetricLongSumValue(metrics, + OpenTelemetryTopicStats.DELAYED_SUBSCRIPTION_COUNTER, attributes, 10); + for (int i = 0; i < 10; i++) { msg = failoverConsumer.receive(100, TimeUnit.MILLISECONDS); assertEquals(msg.getValue(), "msg-" + i); @@ -233,7 +255,7 @@ public void testDelayedDeliveryWithMultipleConcurrentReadEntries() .topic(topic) .subscriptionName("shared-sub") .subscriptionType(SubscriptionType.Shared) - .receiverQueueSize(1) // Use small prefecthing to simulate the multiple read batches + .receiverQueueSize(1) // Use small prefetching to simulate the multiple read batches .subscribe(); // Simulate race condition with high frequency of calls to dispatcher.readMoreEntries() @@ -337,6 +359,7 @@ public void testEnableAndDisableTopicDelayedDelivery() throws Exception { DelayedDeliveryPolicies delayedDeliveryPolicies = DelayedDeliveryPolicies.builder() .tickTime(2000) .active(false) + .maxDeliveryDelayInMillis(5000) .build(); admin.topics().setDelayedDeliveryPolicy(topicName, delayedDeliveryPolicies); //wait for update @@ -349,6 +372,7 @@ public void testEnableAndDisableTopicDelayedDelivery() throws Exception { assertFalse(admin.topics().getDelayedDeliveryPolicy(topicName).isActive()); assertEquals(2000, admin.topics().getDelayedDeliveryPolicy(topicName).getTickTime()); + assertEquals(5000, admin.topics().getDelayedDeliveryPolicy(topicName).getMaxDeliveryDelayInMillis()); admin.topics().removeDelayedDeliveryPolicy(topicName); //wait for update @@ -622,4 +646,42 @@ public void testDispatcherReadFailure() throws Exception { } } + @Test + public void testDelayedDeliveryExceedsMaxDelay() throws Exception { + long maxDeliveryDelayInMillis = 5000; + String topic = BrokerTestUtil.newUniqueName("testDelayedDeliveryExceedsMaxDelay"); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .create(); + + admin.topicPolicies().setDelayedDeliveryPolicy(topic, + DelayedDeliveryPolicies.builder() + .active(true) + .tickTime(100L) + .maxDeliveryDelayInMillis(maxDeliveryDelayInMillis) + .build()); + + //wait for update + for (int i = 0; i < 50; i++) { + Thread.sleep(100); + if (admin.topics().getDelayedDeliveryPolicy(topic) != null) { + break; + } + } + + try { + producer.newMessage() + .value("msg") + .deliverAfter(6, TimeUnit.SECONDS) + .send(); + + producer.flush(); + fail("Should have thrown NotAllowedException due to exceeding maxDeliveryDelayInMillis"); + } catch (PulsarClientException.NotAllowedException ex) { + assertEquals(ex.getMessage(), "Exceeds max allowed delivery delay of " + + maxDeliveryDelayInMillis + " milliseconds"); + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/KeySharedLookAheadConfigTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/KeySharedLookAheadConfigTest.java new file mode 100644 index 0000000000000..cf028cf369d7b --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/KeySharedLookAheadConfigTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service.persistent; + +import static org.apache.pulsar.broker.service.persistent.PersistentStickyKeyDispatcherMultipleConsumers.getEffectiveLookAheadLimit; +import static org.testng.Assert.assertEquals; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.testng.annotations.Test; + +public class KeySharedLookAheadConfigTest { + + @Test + public void testGetEffectiveLookAheadLimit() { + ServiceConfiguration config = new ServiceConfiguration(); + + config.setKeySharedLookAheadMsgInReplayThresholdPerConsumer(5); + config.setKeySharedLookAheadMsgInReplayThresholdPerSubscription(100); + assertEquals(getEffectiveLookAheadLimit(config, 5), 25); + assertEquals(getEffectiveLookAheadLimit(config, 100), 100); + + config.setKeySharedLookAheadMsgInReplayThresholdPerConsumer(5); + config.setKeySharedLookAheadMsgInReplayThresholdPerSubscription(0); + assertEquals(getEffectiveLookAheadLimit(config, 100), 500); + + config.setKeySharedLookAheadMsgInReplayThresholdPerConsumer(0); + config.setKeySharedLookAheadMsgInReplayThresholdPerSubscription(6000); + assertEquals(getEffectiveLookAheadLimit(config, 100), 6000); + + config.setKeySharedLookAheadMsgInReplayThresholdPerConsumer(0); + config.setKeySharedLookAheadMsgInReplayThresholdPerSubscription(0); + config.setMaxUnackedMessagesPerConsumer(0); + config.setMaxUnackedMessagesPerSubscription(0); + assertEquals(getEffectiveLookAheadLimit(config, 100), Integer.MAX_VALUE); + + config.setKeySharedLookAheadMsgInReplayThresholdPerConsumer(0); + config.setKeySharedLookAheadMsgInReplayThresholdPerSubscription(0); + config.setMaxUnackedMessagesPerConsumer(1); + config.setMaxUnackedMessagesPerSubscription(10); + assertEquals(getEffectiveLookAheadLimit(config, 100), 10); + + config.setKeySharedLookAheadMsgInReplayThresholdPerConsumer(0); + config.setKeySharedLookAheadMsgInReplayThresholdPerSubscription(0); + config.setMaxUnackedMessagesPerConsumer(22); + config.setMaxUnackedMessagesPerSubscription(0); + assertEquals(getEffectiveLookAheadLimit(config, 100), 2200); + } +} \ No newline at end of file diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java index b4152b143ab6c..5b1c78574b462 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java @@ -21,6 +21,7 @@ import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.apache.pulsar.common.protocol.Commands.serializeMetadataAndPayload; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; @@ -32,29 +33,43 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; import io.netty.buffer.ByteBuf; import io.netty.channel.EventLoopGroup; import java.lang.reflect.Field; import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerException; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.broker.service.BacklogQuotaManager; import org.apache.pulsar.broker.service.BrokerService; +import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.api.proto.MessageMetadata; +import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.protocol.Commands; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.apache.pulsar.broker.qos.AsyncTokenBucket; +import org.apache.pulsar.compaction.CompactionServiceFactory; +import org.awaitility.Awaitility; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @Slf4j @Test(groups = "broker") -public class MessageDuplicationTest { +public class MessageDuplicationTest extends BrokerTestBase { private static final int BROKER_DEDUPLICATION_ENTRIES_INTERVAL = 10; private static final int BROKER_DEDUPLICATION_MAX_NUMBER_PRODUCERS = 10; @@ -174,7 +189,7 @@ public void testInactiveProducerRemove() throws Exception { Field field = MessageDeduplication.class.getDeclaredField("inactiveProducers"); field.setAccessible(true); - Map inactiveProducers = (Map) field.get(messageDeduplication); + Map inactiveProducers = (ConcurrentHashMap) field.get(messageDeduplication); String producerName1 = "test1"; when(publishContext.getHighestSequenceId()).thenReturn(2L); @@ -214,9 +229,7 @@ public void testInactiveProducerRemove() throws Exception { messageDeduplication.purgeInactiveProducers(); assertFalse(inactiveProducers.containsKey(producerName2)); assertFalse(inactiveProducers.containsKey(producerName3)); - field = MessageDeduplication.class.getDeclaredField("highestSequencedPushed"); - field.setAccessible(true); - ConcurrentOpenHashMap highestSequencedPushed = (ConcurrentOpenHashMap) field.get(messageDeduplication); + final var highestSequencedPushed = messageDeduplication.highestSequencedPushed; assertEquals((long) highestSequencedPushed.get(producerName1), 2L); assertFalse(highestSequencedPushed.containsKey(producerName2)); @@ -234,6 +247,7 @@ public void testIsDuplicateWithFailure() { doReturn(serviceConfiguration).when(pulsarService).getConfiguration(); doReturn(mock(PulsarResources.class)).when(pulsarService).getPulsarResources(); + doReturn(mock(CompactionServiceFactory.class)).when(pulsarService).getCompactionServiceFactory(); ManagedLedger managedLedger = mock(ManagedLedger.class); MessageDeduplication messageDeduplication = spy(new MessageDeduplication(pulsarService, mock(PersistentTopic.class), managedLedger)); @@ -252,7 +266,9 @@ public void testIsDuplicateWithFailure() { BrokerService brokerService = mock(BrokerService.class); doReturn(eventLoopGroup).when(brokerService).executor(); doReturn(pulsarService).when(brokerService).pulsar(); + doReturn(pulsarService).when(brokerService).getPulsar(); doReturn(new BacklogQuotaManager(pulsarService)).when(brokerService).getBacklogQuotaManager(); + doReturn(AsyncTokenBucket.DEFAULT_SNAPSHOT_CLOCK).when(pulsarService).getMonotonicSnapshotClock(); PersistentTopic persistentTopic = spyWithClassAndConstructorArgs(PersistentTopic.class, "topic-1", brokerService, managedLedger, messageDeduplication); @@ -265,8 +281,8 @@ public void testIsDuplicateWithFailure() { Topic.PublishContext publishContext2 = getPublishContext(producerName2, 1); persistentTopic.publishMessage(byteBuf1, publishContext1); - persistentTopic.addComplete(new PositionImpl(0, 1), null, publishContext1); - verify(managedLedger, times(1)).asyncAddEntry(any(ByteBuf.class), any(), any()); + persistentTopic.addComplete(PositionFactory.create(0, 1), null, publishContext1); + verify(managedLedger, times(1)).asyncAddEntry(any(ByteBuf.class), anyInt(), any(), any()); Long lastSequenceIdPushed = messageDeduplication.highestSequencedPushed.get(producerName1); assertNotNull(lastSequenceIdPushed); assertEquals(lastSequenceIdPushed.longValue(), 0); @@ -275,8 +291,8 @@ public void testIsDuplicateWithFailure() { assertEquals(lastSequenceIdPushed.longValue(), 0); persistentTopic.publishMessage(byteBuf2, publishContext2); - persistentTopic.addComplete(new PositionImpl(0, 2), null, publishContext2); - verify(managedLedger, times(2)).asyncAddEntry(any(ByteBuf.class), any(), any()); + persistentTopic.addComplete(PositionFactory.create(0, 2), null, publishContext2); + verify(managedLedger, times(2)).asyncAddEntry(any(ByteBuf.class), anyInt(), any(), any()); lastSequenceIdPushed = messageDeduplication.highestSequencedPushed.get(producerName2); assertNotNull(lastSequenceIdPushed); assertEquals(lastSequenceIdPushed.longValue(), 1); @@ -287,8 +303,8 @@ public void testIsDuplicateWithFailure() { byteBuf1 = getMessage(producerName1, 1); publishContext1 = getPublishContext(producerName1, 1); persistentTopic.publishMessage(byteBuf1, publishContext1); - persistentTopic.addComplete(new PositionImpl(0, 3), null, publishContext1); - verify(managedLedger, times(3)).asyncAddEntry(any(ByteBuf.class), any(), any()); + persistentTopic.addComplete(PositionFactory.create(0, 3), null, publishContext1); + verify(managedLedger, times(3)).asyncAddEntry(any(ByteBuf.class), anyInt(), any(), any()); lastSequenceIdPushed = messageDeduplication.highestSequencedPushed.get(producerName1); assertNotNull(lastSequenceIdPushed); assertEquals(lastSequenceIdPushed.longValue(), 1); @@ -299,8 +315,8 @@ public void testIsDuplicateWithFailure() { byteBuf1 = getMessage(producerName1, 5); publishContext1 = getPublishContext(producerName1, 5); persistentTopic.publishMessage(byteBuf1, publishContext1); - persistentTopic.addComplete(new PositionImpl(0, 4), null, publishContext1); - verify(managedLedger, times(4)).asyncAddEntry(any(ByteBuf.class), any(), any()); + persistentTopic.addComplete(PositionFactory.create(0, 4), null, publishContext1); + verify(managedLedger, times(4)).asyncAddEntry(any(ByteBuf.class), anyInt(), any(), any()); lastSequenceIdPushed = messageDeduplication.highestSequencedPushed.get(producerName1); assertNotNull(lastSequenceIdPushed); assertEquals(lastSequenceIdPushed.longValue(), 5); @@ -312,7 +328,7 @@ public void testIsDuplicateWithFailure() { byteBuf1 = getMessage(producerName1, 0); publishContext1 = getPublishContext(producerName1, 0); persistentTopic.publishMessage(byteBuf1, publishContext1); - verify(managedLedger, times(4)).asyncAddEntry(any(ByteBuf.class), any(), any()); + verify(managedLedger, times(4)).asyncAddEntry(any(ByteBuf.class), anyInt(), any(), any()); lastSequenceIdPushed = messageDeduplication.highestSequencedPushed.get(producerName1); assertNotNull(lastSequenceIdPushed); assertEquals(lastSequenceIdPushed.longValue(), 5); @@ -323,7 +339,7 @@ public void testIsDuplicateWithFailure() { publishContext1 = getPublishContext(producerName1, 6); // don't complete message persistentTopic.publishMessage(byteBuf1, publishContext1); - verify(managedLedger, times(5)).asyncAddEntry(any(ByteBuf.class), any(), any()); + verify(managedLedger, times(5)).asyncAddEntry(any(ByteBuf.class), anyInt(), any(), any()); lastSequenceIdPushed = messageDeduplication.highestSequencedPushed.get(producerName1); assertNotNull(lastSequenceIdPushed); assertEquals(lastSequenceIdPushed.longValue(), 6); @@ -335,17 +351,17 @@ public void testIsDuplicateWithFailure() { byteBuf1 = getMessage(producerName1, 6); publishContext1 = getPublishContext(producerName1, 6); persistentTopic.publishMessage(byteBuf1, publishContext1); - verify(managedLedger, times(5)).asyncAddEntry(any(ByteBuf.class), any(), any()); + verify(managedLedger, times(5)).asyncAddEntry(any(ByteBuf.class), anyInt(), any(), any()); verify(publishContext1, times(1)).completed(any(MessageDeduplication.MessageDupUnknownException.class), eq(-1L), eq(-1L)); // complete seq 6 message eventually - persistentTopic.addComplete(new PositionImpl(0, 5), null, publishContext1); + persistentTopic.addComplete(PositionFactory.create(0, 5), null, publishContext1); // simulate failure byteBuf1 = getMessage(producerName1, 7); publishContext1 = getPublishContext(producerName1, 7); persistentTopic.publishMessage(byteBuf1, publishContext1); - verify(managedLedger, times(6)).asyncAddEntry(any(ByteBuf.class), any(), any()); + verify(managedLedger, times(6)).asyncAddEntry(any(ByteBuf.class), anyInt(), any(), any()); persistentTopic.addFailed(new ManagedLedgerException("test"), publishContext1); // check highestSequencedPushed is reset @@ -365,7 +381,7 @@ public void testIsDuplicateWithFailure() { byteBuf1 = getMessage(producerName1, 6); publishContext1 = getPublishContext(producerName1, 6); persistentTopic.publishMessage(byteBuf1, publishContext1); - verify(managedLedger, times(6)).asyncAddEntry(any(ByteBuf.class), any(), any()); + verify(managedLedger, times(6)).asyncAddEntry(any(ByteBuf.class), anyInt(), any(), any()); verify(publishContext1, times(1)).completed(eq(null), eq(-1L), eq(-1L)); lastSequenceIdPushed = messageDeduplication.highestSequencedPushed.get(producerName1); assertNotNull(lastSequenceIdPushed); @@ -375,8 +391,8 @@ public void testIsDuplicateWithFailure() { byteBuf1 = getMessage(producerName1, 8); publishContext1 = getPublishContext(producerName1, 8); persistentTopic.publishMessage(byteBuf1, publishContext1); - verify(managedLedger, times(7)).asyncAddEntry(any(ByteBuf.class), any(), any()); - persistentTopic.addComplete(new PositionImpl(0, 5), null, publishContext1); + verify(managedLedger, times(7)).asyncAddEntry(any(ByteBuf.class), anyInt(), any(), any()); + persistentTopic.addComplete(PositionFactory.create(0, 5), null, publishContext1); lastSequenceIdPushed = messageDeduplication.highestSequencedPushed.get(producerName1); assertNotNull(lastSequenceIdPushed); assertEquals(lastSequenceIdPushed.longValue(), 8); @@ -437,4 +453,71 @@ public void completed(Exception e, long ledgerId, long entryId) { } }); } + + @BeforeMethod(alwaysRun = true) + @Override + protected void setup() throws Exception { + this.conf.setBrokerDeduplicationEnabled(true); + super.baseSetup(); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testMessageDeduplication() throws Exception { + String topicName = "persistent://prop/ns-abc/testMessageDeduplication"; + String producerName = "test-producer"; + Producer producer = pulsarClient + .newProducer(Schema.STRING) + .producerName(producerName) + .topic(topicName) + .create(); + final PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService() + .getTopicIfExists(topicName).get().orElse(null); + assertNotNull(persistentTopic); + final MessageDeduplication messageDeduplication = persistentTopic.getMessageDeduplication(); + assertFalse(messageDeduplication.getInactiveProducers().containsKey(producerName)); + producer.close(); + Awaitility.await().untilAsserted(() -> assertTrue(messageDeduplication.getInactiveProducers().containsKey(producerName))); + admin.topicPolicies().setDeduplicationStatus(topicName, false); + Awaitility.await().untilAsserted(() -> { + final Boolean deduplicationStatus = admin.topicPolicies().getDeduplicationStatus(topicName); + Assert.assertNotNull(deduplicationStatus); + Assert.assertFalse(deduplicationStatus); + }); + messageDeduplication.purgeInactiveProducers(); + assertTrue(messageDeduplication.getInactiveProducers().isEmpty()); + } + + + @Test + public void testMessageDeduplicationShouldNotWorkForSystemTopic() throws PulsarAdminException { + final String localName = UUID.randomUUID().toString(); + final String namespace = "prop/ns-abc"; + final String prefix = "persistent://%s/".formatted(namespace); + final String topic = prefix + localName; + admin.topics().createNonPartitionedTopic(topic); + + // broker level policies + final String eventSystemTopic = prefix + SystemTopicNames.NAMESPACE_EVENTS_LOCAL_NAME; + final Optional optionalTopic = pulsar.getBrokerService().getTopic(eventSystemTopic, true).join(); + assertTrue(optionalTopic.isPresent()); + final Topic ptRef = optionalTopic.get(); + assertTrue(ptRef.isSystemTopic()); + assertFalse(ptRef.isDeduplicationEnabled()); + + // namespace level policies + admin.namespaces().setDeduplicationStatus(namespace, true); + assertTrue(ptRef.isSystemTopic()); + assertFalse(ptRef.isDeduplicationEnabled()); + + // topic level policies + admin.topicPolicies().setDeduplicationStatus(eventSystemTopic, true); + assertTrue(ptRef.isSystemTopic()); + assertFalse(ptRef.isDeduplicationEnabled()); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryControllerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryControllerTest.java index be5294d1c0f63..1708dc7bc2536 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryControllerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryControllerTest.java @@ -27,10 +27,11 @@ import java.lang.reflect.Field; import java.util.Set; import java.util.TreeSet; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.util.collections.ConcurrentLongLongHashMap; -import org.apache.pulsar.utils.ConcurrentBitmapSortedLongPairSet; import org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap; +import org.apache.pulsar.utils.ConcurrentBitmapSortedLongPairSet; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -224,19 +225,19 @@ public void testGetMessagesToReplayNow(boolean allowOutOfOrderDelivery) throws E if (allowOutOfOrderDelivery) { // The entries are sorted by ledger ID but not by entry ID - PositionImpl[] actual1 = controller.getMessagesToReplayNow(3).toArray(new PositionImpl[3]); - PositionImpl[] expected1 = { PositionImpl.get(1, 1), PositionImpl.get(1, 2), PositionImpl.get(1, 3) }; + Position[] actual1 = controller.getMessagesToReplayNow(3, item -> true).toArray(new Position[3]); + Position[] expected1 = { PositionFactory.create(1, 1), PositionFactory.create(1, 2), PositionFactory.create(1, 3) }; assertEqualsNoOrder(actual1, expected1); } else { // The entries are completely sorted - Set actual2 = controller.getMessagesToReplayNow(6); - Set expected2 = new TreeSet<>(); - expected2.add(PositionImpl.get(1, 1)); - expected2.add(PositionImpl.get(1, 2)); - expected2.add(PositionImpl.get(1, 3)); - expected2.add(PositionImpl.get(2, 1)); - expected2.add(PositionImpl.get(2, 2)); - expected2.add(PositionImpl.get(3, 1)); + Set actual2 = controller.getMessagesToReplayNow(6, item -> true); + Set expected2 = new TreeSet<>(); + expected2.add(PositionFactory.create(1, 1)); + expected2.add(PositionFactory.create(1, 2)); + expected2.add(PositionFactory.create(1, 3)); + expected2.add(PositionFactory.create(2, 1)); + expected2.add(PositionFactory.create(2, 2)); + expected2.add(PositionFactory.create(3, 1)); assertEquals(actual2, expected2); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PartitionKeywordCompatibilityTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PartitionKeywordCompatibilityTest.java index 86a5fcdc05aec..3eabb12b88921 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PartitionKeywordCompatibilityTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PartitionKeywordCompatibilityTest.java @@ -28,6 +28,7 @@ import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; +import org.apache.pulsar.common.policies.data.PartitionedTopicStats; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -70,6 +71,8 @@ public void testAutoCreatePartitionTopicWithKeywordAndDeleteIt() Assert.assertTrue(topics.contains(TopicName.get(topicName).getPartition(0).toString())); Assert.assertTrue(partitionedTopicList.contains(topicName)); consumer.close(); + PartitionedTopicStats stats = admin.topics().getPartitionedStats(topicName, false); + Assert.assertEquals(stats.getSubscriptions().size(), 1); admin.topics().deletePartitionedTopic(topicName); topics = admin.topics().getList("public/default"); partitionedTopicList = admin.topics().getPartitionedTopicList("public/default"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersTest.java new file mode 100644 index 0000000000000..a03ed92b81590 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersTest.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service.persistent; + +import com.carrotsearch.hppc.ObjectSet; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.service.Dispatcher; +import org.apache.pulsar.broker.service.Subscription; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionType; +import org.awaitility.reflect.WhiteboxImpl; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker-api") +public class PersistentDispatcherMultipleConsumersTest extends ProducerConsumerBase { + + @BeforeClass(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test(timeOut = 30 * 1000) + public void testTopicDeleteIfConsumerSetMismatchConsumerList() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final String subscription = "s1"; + admin.topics().createNonPartitionedTopic(topicName); + admin.topics().createSubscription(topicName, subscription, MessageId.earliest); + + Consumer consumer = pulsarClient.newConsumer(Schema.STRING).topic(topicName).subscriptionName(subscription) + .subscriptionType(SubscriptionType.Shared).subscribe(); + // Make an error that "consumerSet" is mismatch with "consumerList". + Dispatcher dispatcher = pulsar.getBrokerService() + .getTopic(topicName, false).join().get() + .getSubscription(subscription).getDispatcher(); + ObjectSet consumerSet = + WhiteboxImpl.getInternalState(dispatcher, "consumerSet"); + List consumerList = + WhiteboxImpl.getInternalState(dispatcher, "consumerList"); + + org.apache.pulsar.broker.service.Consumer serviceConsumer = consumerList.get(0); + consumerSet.add(serviceConsumer); + consumerList.add(serviceConsumer); + + // Verify: the topic can be deleted successfully. + consumer.close(); + admin.topics().delete(topicName, false); + } + + @Test(timeOut = 30 * 1000) + public void testTopicDeleteIfConsumerSetMismatchConsumerList2() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final String subscription = "s1"; + admin.topics().createNonPartitionedTopic(topicName); + admin.topics().createSubscription(topicName, subscription, MessageId.earliest); + + Consumer consumer = pulsarClient.newConsumer(Schema.STRING).topic(topicName).subscriptionName(subscription) + .subscriptionType(SubscriptionType.Shared).subscribe(); + // Make an error that "consumerSet" is mismatch with "consumerList". + Dispatcher dispatcher = pulsar.getBrokerService() + .getTopic(topicName, false).join().get() + .getSubscription(subscription).getDispatcher(); + ObjectSet consumerSet = + WhiteboxImpl.getInternalState(dispatcher, "consumerSet"); + consumerSet.clear(); + + // Verify: the topic can be deleted successfully. + consumer.close(); + admin.topics().delete(topicName, false); + } + + @Test + public void testSkipReadEntriesFromCloseCursor() throws Exception { + final String topicName = + BrokerTestUtil.newUniqueName("persistent://public/default/testSkipReadEntriesFromCloseCursor"); + final String subscription = "s1"; + admin.topics().createNonPartitionedTopic(topicName); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topicName).create(); + for (int i = 0; i < 10; i++) { + producer.send("message-" + i); + } + producer.close(); + + // Get the dispatcher of the topic. + PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService() + .getTopic(topicName, false).join().get(); + + ManagedCursor cursor = Mockito.mock(ManagedCursorImpl.class); + Mockito.doReturn(subscription).when(cursor).getName(); + Subscription sub = Mockito.mock(PersistentSubscription.class); + Mockito.doReturn(topic).when(sub).getTopic(); + // Mock the dispatcher. + PersistentDispatcherMultipleConsumers dispatcher = + Mockito.spy(new PersistentDispatcherMultipleConsumers(topic, cursor, sub)); + // Return 10 permits to make the dispatcher can read more entries. + Mockito.doReturn(10).when(dispatcher).getFirstAvailableConsumerPermits(); + + // Make the count + 1 when call the scheduleReadEntriesWithDelay(...). + AtomicInteger callScheduleReadEntriesWithDelayCnt = new AtomicInteger(0); + Mockito.doAnswer(inv -> { + callScheduleReadEntriesWithDelayCnt.getAndIncrement(); + return inv.callRealMethod(); + }).when(dispatcher).scheduleReadEntriesWithDelay(Mockito.any(), Mockito.any(), Mockito.anyLong()); + + // Make the count + 1 when call the readEntriesFailed(...). + AtomicInteger callReadEntriesFailed = new AtomicInteger(0); + Mockito.doAnswer(inv -> { + callReadEntriesFailed.getAndIncrement(); + return inv.callRealMethod(); + }).when(dispatcher).readEntriesFailed(Mockito.any(), Mockito.any()); + + Mockito.doReturn(false).when(cursor).isClosed(); + + // Mock the readEntriesOrWait(...) to simulate the cursor is closed. + Mockito.doAnswer(inv -> { + PersistentDispatcherMultipleConsumers dispatcher1 = inv.getArgument(2); + dispatcher1.readEntriesFailed(new ManagedLedgerException.CursorAlreadyClosedException("cursor closed"), + null); + return null; + }).when(cursor).asyncReadEntriesOrWait(Mockito.anyInt(), Mockito.anyLong(), Mockito.eq(dispatcher), + Mockito.any(), Mockito.any()); + + dispatcher.readMoreEntries(); + + // Verify: the readEntriesFailed should be called once and the scheduleReadEntriesWithDelay should not be called. + Assert.assertTrue(callReadEntriesFailed.get() == 1 && callScheduleReadEntriesWithDelayCnt.get() == 0); + + // Verify: the topic can be deleted successfully. + admin.topics().delete(topicName, false); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumerTest.java new file mode 100644 index 0000000000000..a4c9e26ffb853 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumerTest.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service.persistent; + +import java.util.concurrent.atomic.AtomicInteger; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.service.Consumer; +import org.apache.pulsar.broker.service.Subscription; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.api.proto.CommandSubscribe; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker-api") +public class PersistentDispatcherSingleActiveConsumerTest extends ProducerConsumerBase { + @BeforeClass(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testSkipReadEntriesFromCloseCursor() throws Exception { + final String topicName = + BrokerTestUtil.newUniqueName("persistent://public/default/testSkipReadEntriesFromCloseCursor"); + final String subscription = "s1"; + admin.topics().createNonPartitionedTopic(topicName); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topicName).create(); + for (int i = 0; i < 10; i++) { + producer.send("message-" + i); + } + producer.close(); + + // Get the dispatcher of the topic. + PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService() + .getTopic(topicName, false).join().get(); + + ManagedCursor cursor = Mockito.mock(ManagedCursorImpl.class); + Mockito.doReturn(subscription).when(cursor).getName(); + Subscription sub = Mockito.mock(PersistentSubscription.class); + Mockito.doReturn(topic).when(sub).getTopic(); + // Mock the dispatcher. + PersistentDispatcherSingleActiveConsumer dispatcher = + Mockito.spy(new PersistentDispatcherSingleActiveConsumer(cursor, CommandSubscribe.SubType.Exclusive,0, topic, sub)); + + // Mock a consumer + Consumer consumer = Mockito.mock(Consumer.class); + consumer.getAvailablePermits(); + Mockito.doReturn(10).when(consumer).getAvailablePermits(); + Mockito.doReturn(10).when(consumer).getAvgMessagesPerEntry(); + Mockito.doReturn("test").when(consumer).consumerName(); + Mockito.doReturn(true).when(consumer).isWritable(); + Mockito.doReturn(false).when(consumer).readCompacted(); + + // Make the consumer as the active consumer. + Mockito.doReturn(consumer).when(dispatcher).getActiveConsumer(); + + // Make the count + 1 when call the scheduleReadEntriesWithDelay(...). + AtomicInteger callScheduleReadEntriesWithDelayCnt = new AtomicInteger(0); + Mockito.doAnswer(inv -> { + callScheduleReadEntriesWithDelayCnt.getAndIncrement(); + return inv.callRealMethod(); + }).when(dispatcher).scheduleReadEntriesWithDelay(Mockito.eq(consumer), Mockito.anyLong()); + + // Make the count + 1 when call the readEntriesFailed(...). + AtomicInteger callReadEntriesFailed = new AtomicInteger(0); + Mockito.doAnswer(inv -> { + callReadEntriesFailed.getAndIncrement(); + return inv.callRealMethod(); + }).when(dispatcher).readEntriesFailed(Mockito.any(), Mockito.any()); + + Mockito.doReturn(false).when(cursor).isClosed(); + + // Mock the readEntriesOrWait(...) to simulate the cursor is closed. + Mockito.doAnswer(inv -> { + PersistentDispatcherSingleActiveConsumer dispatcher1 = inv.getArgument(2); + dispatcher1.readEntriesFailed(new ManagedLedgerException.CursorAlreadyClosedException("cursor closed"), + null); + return null; + }).when(cursor).asyncReadEntriesOrWait(Mockito.anyInt(), Mockito.anyLong(), Mockito.eq(dispatcher), + Mockito.any(), Mockito.any()); + + dispatcher.readMoreEntries(consumer); + + // Verify: the readEntriesFailed should be called once and the scheduleReadEntriesWithDelay should not be called. + Assert.assertTrue(callReadEntriesFailed.get() == 1 && callScheduleReadEntriesWithDelayCnt.get() == 0); + + // Verify: the topic can be deleted successfully. + admin.topics().delete(topicName, false); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java index 48a4bfc923608..a0054f7e71425 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java @@ -20,6 +20,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.pulsar.common.protocol.Commands.serializeMetadataAndPayload; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyBoolean; import static org.mockito.Mockito.anyInt; @@ -35,28 +36,39 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import com.google.common.collect.BoundType; +import com.google.common.collect.Range; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.ChannelPromise; import io.netty.channel.EventLoopGroup; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.GenericFutureListener; import java.lang.reflect.Field; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Queue; import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.bookkeeper.common.util.OrderedExecutor; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.EntryImpl; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.BrokerService; @@ -65,16 +77,22 @@ import org.apache.pulsar.broker.service.EntryBatchSizes; import org.apache.pulsar.broker.service.RedeliveryTracker; import org.apache.pulsar.broker.service.StickyKeyConsumerSelector; +import org.apache.pulsar.broker.service.TransportCnx; +import org.apache.pulsar.broker.service.plugin.EntryFilterProvider; import org.apache.pulsar.common.api.proto.KeySharedMeta; import org.apache.pulsar.common.api.proto.KeySharedMode; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.policies.data.HierarchyTopicPolicies; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.Markers; +import org.apache.pulsar.common.util.collections.ConcurrentOpenLongPairRangeSet; +import org.apache.pulsar.common.util.collections.LongPairRangeSet; import org.awaitility.Awaitility; import org.mockito.ArgumentCaptor; import org.testng.Assert; +import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @Test(groups = "broker") @@ -82,6 +100,7 @@ public class PersistentStickyKeyDispatcherMultipleConsumersTest { private PulsarService pulsarMock; private BrokerService brokerMock; + private ManagedLedgerImpl ledgerMock; private ManagedCursorImpl cursorMock; private Consumer consumerMock; private PersistentTopic topicMock; @@ -94,6 +113,9 @@ public class PersistentStickyKeyDispatcherMultipleConsumersTest { final String topicName = "persistent://public/default/testTopic"; final String subscriptionName = "testSubscription"; + private AtomicInteger consumerMockAvailablePermits; + int retryBackoffInitialTimeInMs = 10; + int retryBackoffMaxTimeInMs = 50; @BeforeMethod public void setup() throws Exception { @@ -102,13 +124,19 @@ public void setup() throws Exception { doReturn(100).when(configMock).getDispatcherMaxReadBatchSize(); doReturn(true).when(configMock).isSubscriptionKeySharedUseConsistentHashing(); doReturn(1).when(configMock).getSubscriptionKeySharedConsistentHashingReplicaPoints(); - doReturn(true).when(configMock).isDispatcherDispatchMessagesInSubscriptionThread(); - + doReturn(false).when(configMock).isDispatcherDispatchMessagesInSubscriptionThread(); + doReturn(false).when(configMock).isAllowOverrideEntryFilters(); + doAnswer(invocation -> retryBackoffInitialTimeInMs).when(configMock).getDispatcherRetryBackoffInitialTimeInMs(); + doAnswer(invocation -> retryBackoffMaxTimeInMs).when(configMock).getDispatcherRetryBackoffMaxTimeInMs(); pulsarMock = mock(PulsarService.class); doReturn(configMock).when(pulsarMock).getConfiguration(); + EntryFilterProvider mockEntryFilterProvider = mock(EntryFilterProvider.class); + when(mockEntryFilterProvider.getBrokerEntryFilters()).thenReturn(Collections.emptyList()); + brokerMock = mock(BrokerService.class); doReturn(pulsarMock).when(brokerMock).pulsar(); + when(brokerMock.getEntryFilterProvider()).thenReturn(mockEntryFilterProvider); HierarchyTopicPolicies topicPolicies = new HierarchyTopicPolicies(); topicPolicies.getMaxConsumersPerSubscription().updateBrokerValue(0); @@ -128,14 +156,50 @@ public void setup() throws Exception { doReturn(topicName).when(topicMock).getName(); doReturn(topicPolicies).when(topicMock).getHierarchyTopicPolicies(); + ledgerMock = mock(ManagedLedgerImpl.class); + doAnswer((invocationOnMock -> { + final Position position = invocationOnMock.getArgument(0); + if (position.getEntryId() > 0) { + return PositionFactory.create(position.getLedgerId(), position.getEntryId() - 1); + } else { + fail("Undefined behavior on mock"); + return PositionFactory.EARLIEST; + } + })).when(ledgerMock).getPreviousPosition(any(Position.class)); + doAnswer((invocationOnMock -> { + final Position position = invocationOnMock.getArgument(0); + return PositionFactory.create(position.getLedgerId(), position.getEntryId() < 0 ? 0 : position.getEntryId() + 1); + })).when(ledgerMock).getNextValidPosition(any(Position.class)); + doAnswer((invocationOnMock -> { + final Range range = invocationOnMock.getArgument(0); + Position fromPosition = range.lowerEndpoint(); + boolean fromIncluded = range.lowerBoundType() == BoundType.CLOSED; + Position toPosition = range.upperEndpoint(); + boolean toIncluded = range.upperBoundType() == BoundType.CLOSED; + + long count = 0; + + if (fromPosition.getLedgerId() == toPosition.getLedgerId()) { + // If the 2 positions are in the same ledger + count = toPosition.getEntryId() - fromPosition.getEntryId() - 1; + count += fromIncluded ? 1 : 0; + count += toIncluded ? 1 : 0; + } else { + fail("Undefined behavior on mock"); + } + return count; + })).when(ledgerMock).getNumberOfEntries(any()); + cursorMock = mock(ManagedCursorImpl.class); doReturn(null).when(cursorMock).getLastIndividualDeletedRange(); doReturn(subscriptionName).when(cursorMock).getName(); + doReturn(ledgerMock).when(cursorMock).getManagedLedger(); - consumerMock = mock(Consumer.class); + consumerMock = createMockConsumer(); channelMock = mock(ChannelPromise.class); doReturn("consumer1").when(consumerMock).consumerName(); - doReturn(1000).when(consumerMock).getAvailablePermits(); + consumerMockAvailablePermits = new AtomicInteger(1000); + doAnswer(invocation -> consumerMockAvailablePermits.get()).when(consumerMock).getAvailablePermits(); doReturn(true).when(consumerMock).isWritable(); doReturn(channelMock).when(consumerMock).sendMessages( anyList(), @@ -148,14 +212,27 @@ public void setup() throws Exception { ); subscriptionMock = mock(PersistentSubscription.class); + when(subscriptionMock.getTopic()).thenReturn(topicMock); persistentDispatcher = new PersistentStickyKeyDispatcherMultipleConsumers( topicMock, cursorMock, subscriptionMock, configMock, new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT)); } + protected static Consumer createMockConsumer() { + Consumer consumerMock = mock(Consumer.class); + TransportCnx transportCnx = mock(TransportCnx.class); + doReturn(transportCnx).when(consumerMock).cnx(); + doReturn(true).when(transportCnx).isActive(); + return consumerMock; + } + + @AfterMethod(alwaysRun = true) public void cleanup() { + if (persistentDispatcher != null && !persistentDispatcher.isClosed()) { + persistentDispatcher.close(); + } if (orderedExecutor != null) { - orderedExecutor.shutdown(); + orderedExecutor.shutdownNow(); orderedExecutor = null; } } @@ -163,7 +240,7 @@ public void cleanup() { @Test(timeOut = 10000) public void testAddConsumerWhenClosed() throws Exception { persistentDispatcher.close().get(); - Consumer consumer = mock(Consumer.class); + Consumer consumer = createMockConsumer(); persistentDispatcher.addConsumer(consumer); verify(consumer, times(1)).disconnect(); assertEquals(0, persistentDispatcher.getConsumers().size()); @@ -221,7 +298,7 @@ public void testSendMessage() { .setStart(0) .setEnd(9); - Consumer consumerMock = mock(Consumer.class); + Consumer consumerMock = createMockConsumer(); doReturn(keySharedMeta).when(consumerMock).getKeySharedMeta(); persistentDispatcher.addConsumer(consumerMock); persistentDispatcher.consumerFlow(consumerMock, 1000); @@ -243,7 +320,7 @@ public void testSendMessage() { @Test public void testSkipRedeliverTemporally() { - final Consumer slowConsumerMock = mock(Consumer.class); + final Consumer slowConsumerMock = createMockConsumer(); final ChannelPromise slowChannelMock = mock(ChannelPromise.class); // add entries to redeliver and read target final List redeliverEntries = new ArrayList<>(); @@ -271,7 +348,6 @@ public void testSkipRedeliverTemporally() { // Create 2Consumers try { doReturn("consumer2").when(slowConsumerMock).consumerName(); - // Change slowConsumer availablePermits to 0 and back to normal when(slowConsumerMock.getAvailablePermits()) .thenReturn(0) .thenReturn(1); @@ -297,28 +373,24 @@ public void testSkipRedeliverTemporally() { // Change slowConsumer availablePermits to 1 // run PersistentStickyKeyDispatcherMultipleConsumers#sendMessagesToConsumers internally // and then stop to dispatch to slowConsumer - if (persistentDispatcher.sendMessagesToConsumers(PersistentStickyKeyDispatcherMultipleConsumers.ReadType.Normal, - redeliverEntries, true)) { - persistentDispatcher.readMoreEntriesAsync(); - } + persistentDispatcher.readEntriesComplete(redeliverEntries, + PersistentDispatcherMultipleConsumers.ReadType.Replay); - Awaitility.await().untilAsserted(() -> { - verify(consumerMock, times(1)).sendMessages( - argThat(arg -> { - assertEquals(arg.size(), 1); - Entry entry = arg.get(0); - assertEquals(entry.getLedgerId(), 1); - assertEquals(entry.getEntryId(), 3); - return true; - }), - any(EntryBatchSizes.class), - any(EntryBatchIndexesAcks.class), - anyInt(), - anyLong(), - anyLong(), - any(RedeliveryTracker.class) - ); - }); + verify(consumerMock, times(1)).sendMessages( + argThat(arg -> { + assertEquals(arg.size(), 1); + Entry entry = arg.get(0); + assertEquals(entry.getLedgerId(), 1); + assertEquals(entry.getEntryId(), 3); + return true; + }), + any(EntryBatchSizes.class), + any(EntryBatchIndexesAcks.class), + anyInt(), + anyLong(), + anyLong(), + any(RedeliveryTracker.class) + ); verify(slowConsumerMock, times(0)).sendMessages( anyList(), any(EntryBatchSizes.class), @@ -336,10 +408,10 @@ public void testMessageRedelivery() throws Exception { final Queue actualEntriesToConsumer2 = new ConcurrentLinkedQueue<>(); final Queue expectedEntriesToConsumer1 = new ConcurrentLinkedQueue<>(); - expectedEntriesToConsumer1.add(PositionImpl.get(1, 1)); + expectedEntriesToConsumer1.add(PositionFactory.create(1, 1)); final Queue expectedEntriesToConsumer2 = new ConcurrentLinkedQueue<>(); - expectedEntriesToConsumer2.add(PositionImpl.get(1, 2)); - expectedEntriesToConsumer2.add(PositionImpl.get(1, 3)); + expectedEntriesToConsumer2.add(PositionFactory.create(1, 2)); + expectedEntriesToConsumer2.add(PositionFactory.create(1, 3)); final AtomicInteger remainingEntriesNum = new AtomicInteger( expectedEntriesToConsumer1.size() + expectedEntriesToConsumer2.size()); @@ -356,7 +428,7 @@ public void testMessageRedelivery() throws Exception { final List readEntries = new ArrayList<>(); readEntries.add(allEntries.get(2)); // message3 - final Consumer consumer1 = mock(Consumer.class); + final Consumer consumer1 = createMockConsumer(); doReturn("consumer1").when(consumer1).consumerName(); // Change availablePermits of consumer1 to 0 and then back to normal when(consumer1.getAvailablePermits()).thenReturn(0).thenReturn(10); @@ -372,7 +444,7 @@ public void testMessageRedelivery() throws Exception { }).when(consumer1).sendMessages(anyList(), any(EntryBatchSizes.class), any(EntryBatchIndexesAcks.class), anyInt(), anyLong(), anyLong(), any(RedeliveryTracker.class)); - final Consumer consumer2 = mock(Consumer.class); + final Consumer consumer2 = createMockConsumer(); doReturn("consumer2").when(consumer2).consumerName(); when(consumer2.getAvailablePermits()).thenReturn(10); doReturn(true).when(consumer2).isWritable(); @@ -447,12 +519,566 @@ public void testMessageRedelivery() throws Exception { persistentDispatcher.readMoreEntries(); } - assertEquals(actualEntriesToConsumer1, expectedEntriesToConsumer1); - assertEquals(actualEntriesToConsumer2, expectedEntriesToConsumer2); + assertThat(actualEntriesToConsumer1).containsExactlyElementsOf(expectedEntriesToConsumer1); + assertThat(actualEntriesToConsumer2).containsExactlyElementsOf(expectedEntriesToConsumer2); allEntries.forEach(entry -> entry.release()); } + @DataProvider(name = "initializeLastSentPosition") + private Object[][] initialLastSentPositionProvider() { + return new Object[][] { { false }, { true } }; + } + + @Test(dataProvider = "initializeLastSentPosition") + public void testLastSentPositionAndIndividuallySentPositions(final boolean initializeLastSentPosition) throws Exception { + final Position initialLastSentPosition = PositionFactory.create(1, 10); + final LongPairRangeSet expectedIndividuallySentPositions + = new ConcurrentOpenLongPairRangeSet<>(4096, PositionFactory::create); + + final Field lastSentPositionField = PersistentStickyKeyDispatcherMultipleConsumers.class + .getDeclaredField("lastSentPosition"); + lastSentPositionField.setAccessible(true); + final LongPairRangeSet individuallySentPositions = persistentDispatcher.getIndividuallySentPositionsField(); + final Supplier clearPosition = () -> { + try { + lastSentPositionField.set(persistentDispatcher, initializeLastSentPosition ? initialLastSentPosition : null); + individuallySentPositions.clear(); + expectedIndividuallySentPositions.clear(); + } catch (Throwable e) { + return e; + } + return null; + }; + if (!initializeLastSentPosition) { + doReturn(initialLastSentPosition).when(cursorMock).getMarkDeletedPosition(); + doAnswer(invocationOnMock -> { + // skip copy operation + return initialLastSentPosition; + }).when(cursorMock).processIndividuallyDeletedMessagesAndGetMarkDeletedPosition(any()); + } + + // Assume the range sequence is [1:0, 1:19], [2:0, 2:19], ..., [10:0, 10:19] + doAnswer((invocationOnMock -> { + final Position position = invocationOnMock.getArgument(0); + if (position.getEntryId() > 0) { + return PositionFactory.create(position.getLedgerId(), position.getEntryId() - 1); + } else if (position.getLedgerId() > 0) { + return PositionFactory.create(position.getLedgerId() - 1, 19); + } else { + throw new NullPointerException(); + } + })).when(ledgerMock).getPreviousPosition(any(Position.class)); + doAnswer((invocationOnMock -> { + final Position position = invocationOnMock.getArgument(0); + if (position.getEntryId() < 19) { + return PositionFactory.create(position.getLedgerId(), position.getEntryId() + 1); + } else { + return PositionFactory.create(position.getLedgerId() + 1, 0); + } + })).when(ledgerMock).getNextValidPosition(any(Position.class)); + doReturn(PositionFactory.create(10, 19)).when(ledgerMock).getLastConfirmedEntry(); + doAnswer((invocationOnMock -> { + final Range range = invocationOnMock.getArgument(0); + Position fromPosition = range.lowerEndpoint(); + boolean fromIncluded = range.lowerBoundType() == BoundType.CLOSED; + Position toPosition = range.upperEndpoint(); + boolean toIncluded = range.upperBoundType() == BoundType.CLOSED; + + if (fromPosition.getLedgerId() == toPosition.getLedgerId()) { + // If the 2 positions are in the same ledger + long count = toPosition.getEntryId() - fromPosition.getEntryId() - 1; + count += fromIncluded ? 1 : 0; + count += toIncluded ? 1 : 0; + return count; + } else { + long count = 0; + // If the from & to are pointing to different ledgers, then we need to : + // 1. Add the entries in the ledger pointed by toPosition + count += toPosition.getEntryId(); + count += toIncluded ? 1 : 0; + + // 2. Add the entries in the ledger pointed by fromPosition + count += 20 - (fromPosition.getEntryId() + 1); + count += fromIncluded ? 1 : 0; + + // 3. Add the whole ledgers entries in between + for (long i = fromPosition.getLedgerId() + 1; i < toPosition.getLedgerId(); i++) { + count += 20; + } + + return count; + } + })).when(ledgerMock).getNumberOfEntries(any()); + assertEquals(ledgerMock.getNextValidPosition(PositionFactory.create(1, 0)), PositionFactory.create(1, 1)); + assertEquals(ledgerMock.getNextValidPosition(PositionFactory.create(1, 19)), PositionFactory.create(2, 0)); + assertEquals(ledgerMock.getPreviousPosition(PositionFactory.create(2, 0)), PositionFactory.create(1, 19)); + assertThrows(NullPointerException.class, () -> ledgerMock.getPreviousPosition(PositionFactory.create(0, 0))); + assertEquals(ledgerMock.getNumberOfEntries(Range.openClosed( + PositionFactory.create(1, 0), PositionFactory.create(1, 0))), 0); + assertEquals(ledgerMock.getNumberOfEntries(Range.openClosed( + PositionFactory.create(1, -1), PositionFactory.create(1, 9))), 10); + assertEquals(ledgerMock.getNumberOfEntries(Range.openClosed( + PositionFactory.create(1, 19), PositionFactory.create(2, -1))), 0); + assertEquals(ledgerMock.getNumberOfEntries(Range.openClosed( + PositionFactory.create(1, 19), PositionFactory.create(2, 9))), 10); + assertEquals(ledgerMock.getNumberOfEntries(Range.openClosed( + PositionFactory.create(1, -1), PositionFactory.create(3, 19))), 60); + + // Add a consumer + final Consumer consumer1 = createMockConsumer(); + doReturn("consumer1").when(consumer1).consumerName(); + when(consumer1.getAvailablePermits()).thenReturn(1000); + doReturn(true).when(consumer1).isWritable(); + doReturn(channelMock).when(consumer1).sendMessages(anyList(), any(EntryBatchSizes.class), + any(EntryBatchIndexesAcks.class), anyInt(), anyLong(), anyLong(), any(RedeliveryTracker.class)); + persistentDispatcher.addConsumer(consumer1); + + /* + On single ledger + */ + + // Expected individuallySentPositions (isp): [(1:-1, 1:8]] (init) -> [(1:-1, 1:9]] (update) -> [] (remove) + // Expected lastSentPosition (lsp): 1:10 (init) -> 1:10 (remove) + // upper bound and the new entry are less than initial last sent position + assertNull(clearPosition.get()); + individuallySentPositions.addOpenClosed(1, -1, 1, 8); + persistentDispatcher.sendMessagesToConsumers(PersistentDispatcherMultipleConsumers.ReadType.Normal, + Arrays.asList(EntryImpl.create(1, 9, createMessage("test", 1))), true); + assertTrue(individuallySentPositions.isEmpty()); + assertEquals(persistentDispatcher.getLastSentPosition(), initialLastSentPosition.toString()); + + // isp: [(1:-1, 1:9]] -> [(1:-1, 1:10]] -> [] + // lsp: 1:10 -> 1:10 + // upper bound is less than initial last sent position + // upper bound and the new entry are less than or equal to initial last sent position + assertNull(clearPosition.get()); + individuallySentPositions.addOpenClosed(1, -1, 1, 9); + persistentDispatcher.sendMessagesToConsumers(PersistentDispatcherMultipleConsumers.ReadType.Normal, + Arrays.asList(EntryImpl.create(1, 10, createMessage("test", 1))), true); + assertTrue(individuallySentPositions.isEmpty()); + assertEquals(persistentDispatcher.getLastSentPosition(), initialLastSentPosition.toString()); + + // isp: [(1:-1, 1:2], (1:3, 1:4], (1:5, 1:6]] -> [(1:-1, 1:2], (1:3, 1:4], (1:5, 1:6], (1:9, 1:10]] -> [] + // lsp: 1:10 -> 1:10 + // upper bound and the new entry are less than or equal to initial last sent position + // individually sent positions has multiple ranges + assertNull(clearPosition.get()); + individuallySentPositions.addOpenClosed(1, -1, 1, 2); + individuallySentPositions.addOpenClosed(1, 3, 1, 4); + individuallySentPositions.addOpenClosed(1, 5, 1, 6); + persistentDispatcher.sendMessagesToConsumers(PersistentDispatcherMultipleConsumers.ReadType.Normal, + Arrays.asList(EntryImpl.create(1, 10, createMessage("test", 1))), true); + assertTrue(individuallySentPositions.isEmpty()); + assertEquals(persistentDispatcher.getLastSentPosition(), initialLastSentPosition.toString()); + + // isp: [(1:-1, 1:10]] -> [(1:-1, 1:11]] -> [] + // lsp: 1:10 -> 1:11 + // upper bound is less than or equal to initial last sent position + // the new entry is next position of initial last sent position + assertNull(clearPosition.get()); + individuallySentPositions.addOpenClosed(1, -1, 1, 10); + persistentDispatcher.sendMessagesToConsumers(PersistentDispatcherMultipleConsumers.ReadType.Normal, + Arrays.asList(EntryImpl.create(1, 11, createMessage("test", 1))), true); + assertTrue(individuallySentPositions.isEmpty()); + assertEquals(persistentDispatcher.getLastSentPosition(), PositionFactory.create(1, 11).toString()); + + // isp: [(1:-1, 1:9]] -> [(1:-1, 1:9], (1:10, 1:11]] -> [] + // lsp: 1:10 -> 1:11 + // upper bound is less than initial last sent position + // the new entry is next position of initial last sent position + assertNull(clearPosition.get()); + individuallySentPositions.addOpenClosed(1, -1, 1, 9); + persistentDispatcher.sendMessagesToConsumers(PersistentDispatcherMultipleConsumers.ReadType.Normal, + Arrays.asList(EntryImpl.create(1, 11, createMessage("test", 1))), true); + assertTrue(individuallySentPositions.isEmpty()); + assertEquals(persistentDispatcher.getLastSentPosition(), PositionFactory.create(1, 11).toString()); + + // isp: [(1:11, 1:15]] -> [(1:10, 1:15]] -> [] + // lsp: 1:10 -> 1:15 + // upper bound is greater than initial last sent position + // the range doesn't contain next position of initial last sent position + // the new entry is next position of initial last sent position + assertNull(clearPosition.get()); + individuallySentPositions.addOpenClosed(1, 11, 1, 15); + persistentDispatcher.sendMessagesToConsumers(PersistentDispatcherMultipleConsumers.ReadType.Normal, + Arrays.asList(EntryImpl.create(1, 11, createMessage("test", 1))), true); + assertTrue(individuallySentPositions.isEmpty()); + assertEquals(persistentDispatcher.getLastSentPosition(), PositionFactory.create(1, 15).toString()); + + // isp: [(1:11, 1:15]] -> [(1:10, 1:16]] -> [] + // lsp: 1:10 -> 1:16 + // upper bound is greater than initial last sent position + // the range doesn't contain next position of initial last sent position + // the new entries contain next position of initial last sent position + // first of the new entries is less than initial last sent position + assertNull(clearPosition.get()); + individuallySentPositions.addOpenClosed(1, 11, 1, 15); + persistentDispatcher.sendMessagesToConsumers(PersistentDispatcherMultipleConsumers.ReadType.Normal, + Arrays.asList(EntryImpl.create(1, 9, createMessage("test", 1)), + EntryImpl.create(1, 11, createMessage("test", 2)), + EntryImpl.create(1, 16, createMessage("test", 3))), true); + assertTrue(individuallySentPositions.isEmpty()); + assertEquals(persistentDispatcher.getLastSentPosition(), PositionFactory.create(1, 16).toString()); + + // isp: [(1:11, 1:15]] -> [(1:11, 1:15]] -> [(1:11, 1:15]] + // lsp: 1:10 -> 1:10 + // upper bound is greater than initial last sent position + // the range doesn't contain next position of initial last sent position + // the new entry isn't next position of initial last sent position + // the range contains the new entry + assertNull(clearPosition.get()); + individuallySentPositions.addOpenClosed(1, 11, 1, 15); + expectedIndividuallySentPositions.addOpenClosed(1, 11, 1, 15); + persistentDispatcher.sendMessagesToConsumers(PersistentDispatcherMultipleConsumers.ReadType.Normal, + Arrays.asList(EntryImpl.create(1, 15, createMessage("test", 1))), true); + assertEquals(individuallySentPositions.toString(), expectedIndividuallySentPositions.toString()); + assertEquals(persistentDispatcher.getLastSentPosition(), initialLastSentPosition.toString()); + + // isp: [(1:11, 1:15]] -> [(1:11, 1:16]] -> [(1:11, 1:16]] + // lsp: 1:10 -> 1:10 + // upper bound is greater than initial last sent position + // the range doesn't contain next position of initial last sent position + // the new entry isn't next position of initial last sent position + // the range doesn't contain the new entry + // the new entry is next position of upper bound + assertNull(clearPosition.get()); + individuallySentPositions.addOpenClosed(1, 11, 1, 15); + expectedIndividuallySentPositions.addOpenClosed(1, 11, 1, 16); + persistentDispatcher.sendMessagesToConsumers(PersistentDispatcherMultipleConsumers.ReadType.Normal, + Arrays.asList(EntryImpl.create(1, 16, createMessage("test", 1))), true); + assertEquals(individuallySentPositions.toString(), expectedIndividuallySentPositions.toString()); + assertEquals(persistentDispatcher.getLastSentPosition(), initialLastSentPosition.toString()); + + // isp: [(1:11, 1:15]] -> [(1:11, 1:15], (1:16, 1:17]] -> [(1:11, 1:15], (1:16, 1:17]] + // lsp: 1:10 -> 1:10 + // upper bound is greater than initial last sent position + // the range doesn't contain next position of initial last sent position + // the new entry isn't next position of initial last sent position + // the range doesn't contain the new entry + // the new entry isn't next position of upper bound + // the new entry is same ledger + assertNull(clearPosition.get()); + individuallySentPositions.addOpenClosed(1, 11, 1, 15); + expectedIndividuallySentPositions.addOpenClosed(1, 11, 1, 15); + expectedIndividuallySentPositions.addOpenClosed(1, 16, 1, 17); + persistentDispatcher.sendMessagesToConsumers(PersistentDispatcherMultipleConsumers.ReadType.Normal, + Arrays.asList(EntryImpl.create(1, 17, createMessage("test", 1))), true); + assertEquals(individuallySentPositions.toString(), expectedIndividuallySentPositions.toString()); + assertEquals(persistentDispatcher.getLastSentPosition(), initialLastSentPosition.toString()); + + /* + On multiple contiguous ledgers + */ + + // isp: [(1:11, 1:18]] -> [(1:11, 1:18], (2:-1, 2:0]] -> [(1:11, 1:18], (2:-1, 2:0]] + // lsp: 1:10 -> 1:10 + // upper bound is greater than initial last sent position + // the range doesn't contain next position of initial last sent position + // the new entry isn't next position of initial last sent position + // the range doesn't contain the new entry + // the new entry isn't next position of upper bound + // the new entry isn't same ledger + assertNull(clearPosition.get()); + individuallySentPositions.addOpenClosed(1, 11, 1, 18); + expectedIndividuallySentPositions.addOpenClosed(1, 11, 1, 18); + expectedIndividuallySentPositions.addOpenClosed(2, -1, 2, 0); + persistentDispatcher.sendMessagesToConsumers(PersistentDispatcherMultipleConsumers.ReadType.Normal, + Arrays.asList(EntryImpl.create(2, 0, createMessage("test", 1))), true); + assertEquals(individuallySentPositions.toString(), expectedIndividuallySentPositions.toString()); + assertEquals(persistentDispatcher.getLastSentPosition(), initialLastSentPosition.toString()); + + // isp: [(1:11, 1:19], (2:-1, 2:0]] -> [(1:10, 1:19], (2:-1, 2:0]] -> [] + // lsp: 1:10 -> 2:0 + // upper bound is greater than initial last sent position + // the range doesn't contain next position of initial last sent position + // the new entry is next position of initial last sent position + // the new entry isn't same ledger + assertNull(clearPosition.get()); + individuallySentPositions.addOpenClosed(1, 11, 1, 19); + individuallySentPositions.addOpenClosed(2, -1, 2, 0); + persistentDispatcher.sendMessagesToConsumers(PersistentDispatcherMultipleConsumers.ReadType.Normal, + Arrays.asList(EntryImpl.create(1, 11, createMessage("test", 1))), true); + assertTrue(individuallySentPositions.isEmpty()); + assertEquals(persistentDispatcher.getLastSentPosition(), PositionFactory.create(2, 0).toString()); + + // isp: [(1:11, 1:19], (2:-1, 2:19], (3:-1, 3:0]] -> [(1:10, 1:19], (2:-1, 2:19], (3:-1, 3:0]] -> [] + // lsp: 1:10 -> 3:0 + // upper bound is greater than initial last sent position + // the range doesn't contain next position of initial last sent position + // the new entry is next position of initial last sent position + // the new entry isn't same ledger + assertNull(clearPosition.get()); + individuallySentPositions.addOpenClosed(1, 11, 1, 19); + individuallySentPositions.addOpenClosed(2, -1, 2, 19); + individuallySentPositions.addOpenClosed(3, -1, 3, 0); + persistentDispatcher.sendMessagesToConsumers(PersistentDispatcherMultipleConsumers.ReadType.Normal, + Arrays.asList(EntryImpl.create(1, 11, createMessage("test", 1))), true); + assertTrue(individuallySentPositions.isEmpty()); + assertEquals(persistentDispatcher.getLastSentPosition(), PositionFactory.create(3, 0).toString()); + + // isp: [(1:11, 1:19], (2:-1, 2:0]] -> [(1:11, 1:19], (2:-1, 2:1]] -> [(1:11, 1:19], (2:-1, 2:1]] + // lsp: 1:10 -> 1:10 + // upper bound is greater than initial last sent position + // the range doesn't contain next position of initial last sent position + // the new entry isn't next position of initial last sent position + // the new entry isn't same ledger + assertNull(clearPosition.get()); + individuallySentPositions.addOpenClosed(1, 11, 1, 19); + individuallySentPositions.addOpenClosed(2, -1, 2, 0); + expectedIndividuallySentPositions.addOpenClosed(1, 11, 1, 19); + expectedIndividuallySentPositions.addOpenClosed(2, -1, 2, 1); + persistentDispatcher.sendMessagesToConsumers(PersistentDispatcherMultipleConsumers.ReadType.Normal, + Arrays.asList(EntryImpl.create(2, 1, createMessage("test", 1))), true); + assertEquals(individuallySentPositions.toString(), expectedIndividuallySentPositions.toString()); + assertEquals(persistentDispatcher.getLastSentPosition(), initialLastSentPosition.toString()); + } + + @DataProvider(name = "testBackoffDelayWhenNoMessagesDispatched") + private Object[][] testBackoffDelayWhenNoMessagesDispatchedParams() { + return new Object[][] { { false, true }, { true, true }, { true, false }, { false, false } }; + } + + @Test(dataProvider = "testBackoffDelayWhenNoMessagesDispatched") + public void testBackoffDelayWhenNoMessagesDispatched(boolean dispatchMessagesInSubscriptionThread, boolean isKeyShared) + throws Exception { + persistentDispatcher.close(); + + List retryDelays = new CopyOnWriteArrayList<>(); + doReturn(dispatchMessagesInSubscriptionThread).when(configMock).isDispatcherDispatchMessagesInSubscriptionThread(); + + PersistentDispatcherMultipleConsumers dispatcher; + if (isKeyShared) { + dispatcher = new PersistentStickyKeyDispatcherMultipleConsumers( + topicMock, cursorMock, subscriptionMock, configMock, + new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT)) { + @Override + protected void reScheduleReadInMs(long readAfterMs) { + retryDelays.add(readAfterMs); + } + }; + } else { + dispatcher = new PersistentDispatcherMultipleConsumers(topicMock, cursorMock, subscriptionMock) { + @Override + protected void reScheduleReadInMs(long readAfterMs) { + retryDelays.add(readAfterMs); + } + }; + } + + // add a consumer without permits to trigger the retry behavior + consumerMockAvailablePermits.set(0); + dispatcher.addConsumer(consumerMock); + + // call "readEntriesComplete" directly to test the retry behavior + List entries = List.of(EntryImpl.create(1, 1, createMessage("message1", 1))); + dispatcher.readEntriesComplete(entries, PersistentDispatcherMultipleConsumers.ReadType.Normal); + Awaitility.await().untilAsserted(() -> { + assertEquals(retryDelays.size(), 1); + assertEquals(retryDelays.get(0), 10, "Initial retry delay should be 10ms"); + } + ); + // test the second retry delay + entries = List.of(EntryImpl.create(1, 1, createMessage("message1", 1))); + dispatcher.readEntriesComplete(entries, PersistentDispatcherMultipleConsumers.ReadType.Normal); + Awaitility.await().untilAsserted(() -> { + assertEquals(retryDelays.size(), 2); + double delay = retryDelays.get(1); + assertEquals(delay, 20.0, 2.0, "Second retry delay should be 20ms (jitter <-10%)"); + } + ); + // verify the max retry delay + for (int i = 0; i < 100; i++) { + entries = List.of(EntryImpl.create(1, 1, createMessage("message1", 1))); + dispatcher.readEntriesComplete(entries, PersistentDispatcherMultipleConsumers.ReadType.Normal); + } + Awaitility.await().untilAsserted(() -> { + assertEquals(retryDelays.size(), 102); + double delay = retryDelays.get(101); + assertEquals(delay, 50.0, 5.0, "Max delay should be 50ms (jitter <-10%)"); + } + ); + // unblock to check that the retry delay is reset + consumerMockAvailablePermits.set(1000); + entries = List.of(EntryImpl.create(1, 2, createMessage("message2", 1, "key2"))); + dispatcher.readEntriesComplete(entries, PersistentDispatcherMultipleConsumers.ReadType.Normal); + // wait that the possibly async handling has completed + Awaitility.await().untilAsserted(() -> assertFalse(dispatcher.isSendInProgress())); + + // now block again to check the next retry delay so verify it was reset + consumerMockAvailablePermits.set(0); + entries = List.of(EntryImpl.create(1, 3, createMessage("message3", 1, "key3"))); + dispatcher.readEntriesComplete(entries, PersistentDispatcherMultipleConsumers.ReadType.Normal); + Awaitility.await().untilAsserted(() -> { + assertEquals(retryDelays.size(), 103); + assertEquals(retryDelays.get(0), 10, "Resetted retry delay should be 10ms"); + } + ); + } + + @Test(dataProvider = "testBackoffDelayWhenNoMessagesDispatched") + public void testBackoffDelayWhenRetryDelayDisabled(boolean dispatchMessagesInSubscriptionThread, boolean isKeyShared) + throws Exception { + persistentDispatcher.close(); + + // it should be possible to disable the retry delay + // by setting retryBackoffInitialTimeInMs and retryBackoffMaxTimeInMs to 0 + retryBackoffInitialTimeInMs=0; + retryBackoffMaxTimeInMs=0; + + List retryDelays = new CopyOnWriteArrayList<>(); + doReturn(dispatchMessagesInSubscriptionThread).when(configMock) + .isDispatcherDispatchMessagesInSubscriptionThread(); + + PersistentDispatcherMultipleConsumers dispatcher; + if (isKeyShared) { + dispatcher = new PersistentStickyKeyDispatcherMultipleConsumers( + topicMock, cursorMock, subscriptionMock, configMock, + new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT)) { + @Override + protected void reScheduleReadInMs(long readAfterMs) { + retryDelays.add(readAfterMs); + } + }; + } else { + dispatcher = new PersistentDispatcherMultipleConsumers(topicMock, cursorMock, subscriptionMock) { + @Override + protected void reScheduleReadInMs(long readAfterMs) { + retryDelays.add(readAfterMs); + } + }; + } + + // add a consumer without permits to trigger the retry behavior + consumerMockAvailablePermits.set(0); + dispatcher.addConsumer(consumerMock); + + // call "readEntriesComplete" directly to test the retry behavior + List entries = List.of(EntryImpl.create(1, 1, createMessage("message1", 1))); + dispatcher.readEntriesComplete(entries, PersistentDispatcherMultipleConsumers.ReadType.Normal); + Awaitility.await().untilAsserted(() -> { + assertEquals(retryDelays.size(), 1); + assertEquals(retryDelays.get(0), 0, "Initial retry delay should be 0ms"); + } + ); + // test the second retry delay + entries = List.of(EntryImpl.create(1, 1, createMessage("message1", 1))); + dispatcher.readEntriesComplete(entries, PersistentDispatcherMultipleConsumers.ReadType.Normal); + Awaitility.await().untilAsserted(() -> { + assertEquals(retryDelays.size(), 2); + double delay = retryDelays.get(1); + assertEquals(delay, 0, 0, "Second retry delay should be 0ms"); + } + ); + // verify the max retry delay + for (int i = 0; i < 100; i++) { + entries = List.of(EntryImpl.create(1, 1, createMessage("message1", 1))); + dispatcher.readEntriesComplete(entries, PersistentDispatcherMultipleConsumers.ReadType.Normal); + } + Awaitility.await().untilAsserted(() -> { + assertEquals(retryDelays.size(), 102); + double delay = retryDelays.get(101); + assertEquals(delay, 0, 0, "Max delay should be 0ms"); + } + ); + // unblock to check that the retry delay is reset + consumerMockAvailablePermits.set(1000); + entries = List.of(EntryImpl.create(1, 2, createMessage("message2", 1, "key2"))); + dispatcher.readEntriesComplete(entries, PersistentDispatcherMultipleConsumers.ReadType.Normal); + // wait that the possibly async handling has completed + Awaitility.await().untilAsserted(() -> assertFalse(dispatcher.isSendInProgress())); + + // now block again to check the next retry delay so verify it was reset + consumerMockAvailablePermits.set(0); + entries = List.of(EntryImpl.create(1, 3, createMessage("message3", 1, "key3"))); + dispatcher.readEntriesComplete(entries, PersistentDispatcherMultipleConsumers.ReadType.Normal); + Awaitility.await().untilAsserted(() -> { + assertEquals(retryDelays.size(), 103); + assertEquals(retryDelays.get(0), 0, "Resetted retry delay should be 0ms"); + } + ); + } + + + @Test(dataProvider = "testBackoffDelayWhenNoMessagesDispatched") + public void testNoBackoffDelayWhenDelayedMessages(boolean dispatchMessagesInSubscriptionThread, boolean isKeyShared) + throws Exception { + persistentDispatcher.close(); + + doReturn(dispatchMessagesInSubscriptionThread).when(configMock) + .isDispatcherDispatchMessagesInSubscriptionThread(); + + AtomicInteger readMoreEntriesCalled = new AtomicInteger(0); + AtomicInteger reScheduleReadInMsCalled = new AtomicInteger(0); + AtomicBoolean delayAllMessages = new AtomicBoolean(true); + + PersistentDispatcherMultipleConsumers dispatcher; + if (isKeyShared) { + dispatcher = new PersistentStickyKeyDispatcherMultipleConsumers( + topicMock, cursorMock, subscriptionMock, configMock, + new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT)) { + @Override + protected void reScheduleReadInMs(long readAfterMs) { + reScheduleReadInMsCalled.incrementAndGet(); + } + + @Override + public synchronized void readMoreEntries() { + readMoreEntriesCalled.incrementAndGet(); + } + + @Override + public boolean trackDelayedDelivery(long ledgerId, long entryId, MessageMetadata msgMetadata) { + if (delayAllMessages.get()) { + // simulate delayed message + return true; + } + return super.trackDelayedDelivery(ledgerId, entryId, msgMetadata); + } + }; + } else { + dispatcher = new PersistentDispatcherMultipleConsumers(topicMock, cursorMock, subscriptionMock) { + @Override + protected void reScheduleReadInMs(long readAfterMs) { + reScheduleReadInMsCalled.incrementAndGet(); + } + + @Override + public synchronized void readMoreEntries() { + readMoreEntriesCalled.incrementAndGet(); + } + + @Override + public boolean trackDelayedDelivery(long ledgerId, long entryId, MessageMetadata msgMetadata) { + if (delayAllMessages.get()) { + // simulate delayed message + return true; + } + return super.trackDelayedDelivery(ledgerId, entryId, msgMetadata); + } + }; + } + + doAnswer(invocationOnMock -> { + GenericFutureListener> listener = invocationOnMock.getArgument(0); + Future future = mock(Future.class); + when(future.isDone()).thenReturn(true); + listener.operationComplete(future); + return channelMock; + }).when(channelMock).addListener(any()); + + // add a consumer with permits + consumerMockAvailablePermits.set(1000); + dispatcher.addConsumer(consumerMock); + + List entries = new ArrayList<>(List.of(EntryImpl.create(1, 1, createMessage("message1", 1)))); + dispatcher.readEntriesComplete(entries, PersistentDispatcherMultipleConsumers.ReadType.Normal); + Awaitility.await().untilAsserted(() -> { + assertEquals(reScheduleReadInMsCalled.get(), 0, "reScheduleReadInMs should not be called"); + assertTrue(readMoreEntriesCalled.get() >= 1); + }); + } + private ByteBuf createMessage(String message, int sequenceId) { return createMessage(message, sequenceId, "testKey"); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java index 401f52daa6291..309cd7b55ac0c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java @@ -40,10 +40,10 @@ import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedCursorContainer; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.resources.NamespaceResources; import org.apache.pulsar.broker.service.Consumer; @@ -92,7 +92,12 @@ public class PersistentSubscriptionTest { public void setup() throws Exception { pulsarTestContext = PulsarTestContext.builderForNonStartableContext() .spyByDefault() - .configCustomizer(config -> config.setTransactionCoordinatorEnabled(true)) + .configCustomizer(config -> { + config.setTransactionCoordinatorEnabled(true); + config.setTransactionPendingAckStoreProviderClassName( + CustomTransactionPendingAckStoreProvider.class.getName()); + config.setTransactionBufferProviderClassName(InMemTransactionBufferProvider.class.getName()); + }) .useTestPulsarResources() .build(); @@ -100,62 +105,12 @@ public void setup() throws Exception { doReturn(Optional.of(new Policies())).when(namespaceResources) .getPoliciesIfCached(any()); - doReturn(new InMemTransactionBufferProvider()).when(pulsarTestContext.getPulsarService()) - .getTransactionBufferProvider(); - doReturn(new TransactionPendingAckStoreProvider() { - @Override - public CompletableFuture newPendingAckStore(PersistentSubscription subscription) { - return CompletableFuture.completedFuture(new PendingAckStore() { - @Override - public void replayAsync(PendingAckHandleImpl pendingAckHandle, ExecutorService executorService) { - try { - Field field = PendingAckHandleState.class.getDeclaredField("state"); - field.setAccessible(true); - field.set(pendingAckHandle, PendingAckHandleState.State.Ready); - } catch (NoSuchFieldException | IllegalAccessException e) { - fail(); - } - } - - @Override - public CompletableFuture closeAsync() { - return CompletableFuture.completedFuture(null); - } - - @Override - public CompletableFuture appendIndividualAck(TxnID txnID, List> positions) { - return CompletableFuture.completedFuture(null); - } - - @Override - public CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position) { - return CompletableFuture.completedFuture(null); - } - - @Override - public CompletableFuture appendCommitMark(TxnID txnID, AckType ackType) { - return CompletableFuture.completedFuture(null); - } - - @Override - public CompletableFuture appendAbortMark(TxnID txnID, AckType ackType) { - return CompletableFuture.completedFuture(null); - } - }); - } - - @Override - public CompletableFuture checkInitializedBefore(PersistentSubscription subscription) { - return CompletableFuture.completedFuture(true); - } - }).when(pulsarTestContext.getPulsarService()).getTransactionPendingAckStoreProvider(); - ledgerMock = mock(ManagedLedgerImpl.class); cursorMock = mock(ManagedCursorImpl.class); managedLedgerConfigMock = mock(ManagedLedgerConfig.class); doReturn(new ManagedCursorContainer()).when(ledgerMock).getCursors(); doReturn("mockCursor").when(cursorMock).getName(); - doReturn(new PositionImpl(1, 50)).when(cursorMock).getMarkDeletedPosition(); + doReturn(PositionFactory.create(1, 50)).when(cursorMock).getMarkDeletedPosition(); doReturn(ledgerMock).when(cursorMock).getManagedLedger(); doReturn(managedLedgerConfigMock).when(ledgerMock).getConfig(); doReturn(false).when(managedLedgerConfigMock).isAutoSkipNonRecoverableData(); @@ -177,10 +132,10 @@ public void teardown() throws Exception { @Test public void testCanAcknowledgeAndAbortForTransaction() throws Exception { - List> positionsPair = new ArrayList<>(); - positionsPair.add(new MutablePair<>(new PositionImpl(2, 1), 0)); - positionsPair.add(new MutablePair<>(new PositionImpl(2, 3), 0)); - positionsPair.add(new MutablePair<>(new PositionImpl(2, 5), 0)); + List> positionsPair = new ArrayList<>(); + positionsPair.add(new MutablePair<>(PositionFactory.create(2, 1), 0)); + positionsPair.add(new MutablePair<>(PositionFactory.create(2, 3), 0)); + positionsPair.add(new MutablePair<>(PositionFactory.create(2, 5), 0)); doAnswer((invocationOnMock) -> { ((AsyncCallbacks.DeleteCallback) invocationOnMock.getArguments()[1]) @@ -201,14 +156,14 @@ public void testCanAcknowledgeAndAbortForTransaction() throws Exception { // Single ack for txn1 persistentSubscription.transactionIndividualAcknowledge(txnID1, positionsPair); - List positions = new ArrayList<>(); - positions.add(new PositionImpl(1, 100)); + List positions = new ArrayList<>(); + positions.add(PositionFactory.create(1, 100)); // Cumulative ack for txn1 persistentSubscription.transactionCumulativeAcknowledge(txnID1, positions).get(); positions.clear(); - positions.add(new PositionImpl(2, 1)); + positions.add(PositionFactory.create(2, 1)); // Can not single ack message already acked. try { @@ -220,7 +175,7 @@ public void testCanAcknowledgeAndAbortForTransaction() throws Exception { } positions.clear(); - positions.add(new PositionImpl(2, 50)); + positions.add(PositionFactory.create(2, 50)); // Can not cumulative ack message for another txn. try { @@ -234,12 +189,12 @@ public void testCanAcknowledgeAndAbortForTransaction() throws Exception { } List positionList = new ArrayList<>(); - positionList.add(new PositionImpl(1, 1)); - positionList.add(new PositionImpl(1, 3)); - positionList.add(new PositionImpl(1, 5)); - positionList.add(new PositionImpl(3, 1)); - positionList.add(new PositionImpl(3, 3)); - positionList.add(new PositionImpl(3, 5)); + positionList.add(PositionFactory.create(1, 1)); + positionList.add(PositionFactory.create(1, 3)); + positionList.add(PositionFactory.create(1, 5)); + positionList.add(PositionFactory.create(3, 1)); + positionList.add(PositionFactory.create(3, 3)); + positionList.add(PositionFactory.create(3, 5)); // Acknowledge from normal consumer will succeed ignoring message acked by ongoing transaction. persistentSubscription.acknowledgeMessage(positionList, AckType.Individual, Collections.emptyMap()); @@ -248,13 +203,13 @@ public void testCanAcknowledgeAndAbortForTransaction() throws Exception { persistentSubscription.endTxn(txnID1.getMostSigBits(), txnID2.getLeastSigBits(), TxnAction.ABORT_VALUE, -1); positions.clear(); - positions.add(new PositionImpl(2, 50)); + positions.add(PositionFactory.create(2, 50)); // Retry above ack, will succeed. As abort has clear pending_ack for those messages. persistentSubscription.transactionCumulativeAcknowledge(txnID2, positions); positionsPair.clear(); - positionsPair.add(new MutablePair(new PositionImpl(2, 1), 0)); + positionsPair.add(new MutablePair(PositionFactory.create(2, 1), 0)); persistentSubscription.transactionIndividualAcknowledge(txnID2, positionsPair); } @@ -271,7 +226,7 @@ public void testAcknowledgeUpdateCursorLastActive() throws Exception { doCallRealMethod().when(cursorMock).getLastActive(); List positionList = new ArrayList<>(); - positionList.add(new PositionImpl(1, 1)); + positionList.add(PositionFactory.create(1, 1)); long beforeAcknowledgeTimestamp = System.currentTimeMillis(); Thread.sleep(1); persistentSubscription.acknowledgeMessage(positionList, AckType.Individual, Collections.emptyMap()); @@ -279,4 +234,53 @@ public void testAcknowledgeUpdateCursorLastActive() throws Exception { // `acknowledgeMessage` should update cursor last active assertTrue(persistentSubscription.cursor.getLastActive() > beforeAcknowledgeTimestamp); } + + public static class CustomTransactionPendingAckStoreProvider implements TransactionPendingAckStoreProvider { + @Override + public CompletableFuture newPendingAckStore(PersistentSubscription subscription) { + return CompletableFuture.completedFuture(new PendingAckStore() { + @Override + public void replayAsync(PendingAckHandleImpl pendingAckHandle, ExecutorService executorService) { + try { + Field field = PendingAckHandleState.class.getDeclaredField("state"); + field.setAccessible(true); + field.set(pendingAckHandle, PendingAckHandleState.State.Ready); + } catch (NoSuchFieldException | IllegalAccessException e) { + fail(); + } + } + + @Override + public CompletableFuture closeAsync() { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendIndividualAck(TxnID txnID, + List> positions) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendCumulativeAck(TxnID txnID, Position position) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendCommitMark(TxnID txnID, AckType ackType) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendAbortMark(TxnID txnID, AckType ackType) { + return CompletableFuture.completedFuture(null); + } + }); + } + + @Override + public CompletableFuture checkInitializedBefore(PersistentSubscription subscription) { + return CompletableFuture.completedFuture(true); + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java index 412b8207e34c7..903443d37bb07 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.broker.service.persistent; +import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.Metric; +import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.parseMetrics; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.doAnswer; @@ -45,6 +47,9 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -53,15 +58,23 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; +import java.util.stream.Collectors; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.mledger.impl.ManagedCursorContainer; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.pulsar.PrometheusMetricsTestUtil; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.BrokerTestBase; -import org.apache.pulsar.broker.stats.PrometheusMetricsTest; -import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGenerator; +import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.TopicPoliciesService; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -72,15 +85,19 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; +import org.apache.pulsar.client.api.SubscriptionMode; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.Policies; +import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.policies.data.TopicStats; import org.awaitility.Awaitility; -import org.junit.Assert; +import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; @@ -102,6 +119,11 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + @Override protected void doInitConf() throws Exception { + super.doInitConf(); + this.conf.setManagedLedgerCursorBackloggedThreshold(10); + } + /** * Test validates that broker cleans up topic which failed to unload while bundle unloading. * @@ -291,7 +313,8 @@ public void testPersistentPartitionedTopicUnload() throws Exception { assertFalse(pulsar.getBrokerService().getTopics().containsKey(topicName)); pulsar.getBrokerService().getTopicIfExists(topicName).get(); - assertTrue(pulsar.getBrokerService().getTopics().containsKey(topicName)); + // The map topics should only contain partitions, does not contain partitioned topic. + assertFalse(pulsar.getBrokerService().getTopics().containsKey(topicName)); // ref of partitioned-topic name should be empty assertFalse(pulsar.getBrokerService().getTopicReference(topicName).isPresent()); @@ -304,6 +327,83 @@ public void testPersistentPartitionedTopicUnload() throws Exception { } } + @DataProvider(name = "closeWithoutWaitingClientDisconnectInFirstBatch") + public Object[][] closeWithoutWaitingClientDisconnectInFirstBatch() { + return new Object[][]{ + new Object[] {true}, + new Object[] {false}, + }; + } + + @Test(dataProvider = "closeWithoutWaitingClientDisconnectInFirstBatch") + public void testConcurrentClose(boolean closeWithoutWaitingClientDisconnectInFirstBatch) throws Exception { + final String topicName = "persistent://prop/ns/concurrentClose"; + final String ns = "prop/ns"; + admin.namespaces().createNamespace(ns, 1); + admin.topics().createNonPartitionedTopic(topicName); + final Topic topic = pulsar.getBrokerService().getTopicIfExists(topicName).get().get(); + List> futureList = + make2ConcurrentBatchesOfClose(topic, 10, closeWithoutWaitingClientDisconnectInFirstBatch); + Map>> futureMap = + futureList.stream().collect(Collectors.groupingBy(Objects::hashCode)); + /** + * The first call: get the return value of "topic.close". + * The other 19 calls: get the cached value which related {@link PersistentTopic#closeFutures}. + */ + assertTrue(futureMap.size() <= 3); + for (List list : futureMap.values()){ + if (list.size() == 1){ + // This is the first call, the future is the return value of `topic.close`. + } else { + // Two types future list: wait client close or not. + assertTrue(list.size() >= 9 && list.size() <= 10); + } + } + } + + private List> make2ConcurrentBatchesOfClose(Topic topic, int tryTimes, + boolean closeWithoutWaitingClientDisconnectInFirstBatch){ + final List> futureList = Collections.synchronizedList(new ArrayList<>()); + final List taskList = new ArrayList<>(); + CountDownLatch allTaskBeginLatch = new CountDownLatch(1); + // Call a batch of close. + for (int i = 0; i < tryTimes; i++) { + Thread thread = new Thread(() -> { + try { + allTaskBeginLatch.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + futureList.add(topic.close(closeWithoutWaitingClientDisconnectInFirstBatch)); + }); + thread.start(); + taskList.add(thread); + } + // Call another batch of close. + for (int i = 0; i < tryTimes; i++) { + Thread thread = new Thread(() -> { + try { + allTaskBeginLatch.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + futureList.add(topic.close(!closeWithoutWaitingClientDisconnectInFirstBatch)); + }); + thread.start(); + taskList.add(thread); + } + // Wait close task executed. + allTaskBeginLatch.countDown(); + Awaitility.await().atMost(5, TimeUnit.SECONDS).until(()->{ + for (Thread thread : taskList){ + if (thread.isAlive()){ + return false; + } + } + return true; + }); + return futureList; + } @DataProvider(name = "topicAndMetricsLevel") public Object[][] indexPatternTestData() { @@ -313,7 +413,6 @@ public Object[][] indexPatternTestData() { }; } - @Test(dataProvider = "topicAndMetricsLevel") public void testDelayedDeliveryTrackerMemoryUsageMetric(String topic, boolean exposeTopicLevelMetrics) throws Exception { PulsarClient client = pulsar.getClient(); @@ -349,17 +448,17 @@ public void testDelayedDeliveryTrackerMemoryUsageMetric(String topic, boolean ex latch.await(10, TimeUnit.SECONDS); ByteArrayOutputStream output = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, exposeTopicLevelMetrics, true, true, output); + PrometheusMetricsTestUtil.generate(pulsar, exposeTopicLevelMetrics, true, true, output); String metricsStr = output.toString(StandardCharsets.UTF_8); - Multimap metricsMap = PrometheusMetricsTest.parseMetrics(metricsStr); - Collection metrics = metricsMap.get("pulsar_delayed_message_index_size_bytes"); + Multimap metricsMap = parseMetrics(metricsStr); + Collection metrics = metricsMap.get("pulsar_delayed_message_index_size_bytes"); Assert.assertTrue(metrics.size() > 0); int topicLevelNum = 0; int namespaceLevelNum = 0; int subscriptionLevelNum = 0; - for (PrometheusMetricsTest.Metric metric : metrics) { + for (Metric metric : metrics) { if (exposeTopicLevelMetrics && metric.tags.get("topic").equals(topic)) { Assert.assertTrue(metric.value > 0); topicLevelNum++; @@ -453,8 +552,7 @@ public void testCreateNonExistentPartitions() throws PulsarAdminException, Pulsa .topic(partition.toString()) .create(); fail("unexpected behaviour"); - } catch (PulsarClientException.TopicDoesNotExistException ignored) { - + } catch (PulsarClientException.NotAllowedException ex) { } Assert.assertEquals(admin.topics().getPartitionedTopicMetadata(topicName).partitions, 4); } @@ -603,4 +701,171 @@ public void testCreateTopicWithZombieReplicatorCursor(boolean topicLevelPolicy) return !topic.getManagedLedger().getCursors().iterator().hasNext(); }); } + + @Test + public void testCheckPersistencePolicies() throws Exception { + final String myNamespace = "prop/ns"; + admin.namespaces().createNamespace(myNamespace, Sets.newHashSet("test")); + final String topic = "persistent://" + myNamespace + "/testConfig" + UUID.randomUUID(); + conf.setForceDeleteNamespaceAllowed(true); + pulsarClient.newProducer().topic(topic).create().close(); + RetentionPolicies retentionPolicies = new RetentionPolicies(1, 1); + PersistentTopic persistentTopic = spy((PersistentTopic) pulsar.getBrokerService().getTopicIfExists(topic).get().get()); + TopicPoliciesService policiesService = spy(pulsar.getTopicPoliciesService()); + doReturn(policiesService).when(pulsar).getTopicPoliciesService(); + TopicPolicies policies = new TopicPolicies(); + policies.setRetentionPolicies(retentionPolicies); + doReturn(CompletableFuture.completedFuture(Optional.of(policies))).when(policiesService) + .getTopicPoliciesAsync(TopicName.get(topic), TopicPoliciesService.GetType.DEFAULT); + persistentTopic.onUpdate(policies); + verify(persistentTopic, times(1)).checkPersistencePolicies(); + Awaitility.await().untilAsserted(() -> { + assertEquals(persistentTopic.getManagedLedger().getConfig().getRetentionSizeInMB(), 1L); + assertEquals(persistentTopic.getManagedLedger().getConfig().getRetentionTimeMillis(), TimeUnit.MINUTES.toMillis(1)); + }); + // throw exception + doReturn(CompletableFuture.failedFuture(new RuntimeException())).when(persistentTopic).checkPersistencePolicies(); + policies.setRetentionPolicies(new RetentionPolicies(2, 2)); + persistentTopic.onUpdate(policies); + assertEquals(persistentTopic.getManagedLedger().getConfig().getRetentionSizeInMB(), 1L); + assertEquals(persistentTopic.getManagedLedger().getConfig().getRetentionTimeMillis(), TimeUnit.MINUTES.toMillis(1)); + } + + @Test + public void testDynamicConfigurationAutoSkipNonRecoverableData() throws Exception { + pulsar.getConfiguration().setAutoSkipNonRecoverableData(false); + final String topicName = "persistent://prop/ns-abc/testAutoSkipNonRecoverableData"; + final String subName = "test_sub"; + + Consumer subscribe = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe(); + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).join().get(); + PersistentSubscription subscription = persistentTopic.getSubscription(subName); + + assertFalse(persistentTopic.ledger.getConfig().isAutoSkipNonRecoverableData()); + assertFalse(subscription.getExpiryMonitor().isAutoSkipNonRecoverableData()); + + String key = "autoSkipNonRecoverableData"; + admin.brokers().updateDynamicConfiguration(key, "true"); + Awaitility.await() + .untilAsserted(() -> assertEquals(admin.brokers().getAllDynamicConfigurations().get(key), "true")); + + assertTrue(persistentTopic.ledger.getConfig().isAutoSkipNonRecoverableData()); + assertTrue(subscription.getExpiryMonitor().isAutoSkipNonRecoverableData()); + + subscribe.close(); + admin.topics().delete(topicName); + } + + @Test + public void testCursorGetConfigAfterTopicPoliciesChanged() throws Exception { + final String topicName = "persistent://prop/ns-abc/" + UUID.randomUUID(); + final String subName = "test_sub"; + + @Cleanup + Consumer subscribe = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe(); + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).join().get(); + PersistentSubscription subscription = persistentTopic.getSubscription(subName); + + int maxConsumers = 100; + admin.topicPolicies().setMaxConsumers(topicName, 100); + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topicPolicies().getMaxConsumers(topicName, false), maxConsumers); + }); + + ManagedCursorImpl cursor = (ManagedCursorImpl) subscription.getCursor(); + assertEquals(cursor.getConfig(), persistentTopic.getManagedLedger().getConfig()); + + subscribe.close(); + admin.topics().delete(topicName); + } + + @Test + public void testAddWaitingCursorsForNonDurable() throws Exception { + final String ns = "prop/ns-test"; + admin.namespaces().createNamespace(ns, 2); + final String topicName = "persistent://prop/ns-test/testAddWaitingCursors"; + admin.topics().createNonPartitionedTopic(topicName); + final Optional topic = pulsar.getBrokerService().getTopic(topicName, false).join(); + assertNotNull(topic.get()); + PersistentTopic persistentTopic = (PersistentTopic) topic.get(); + ManagedLedgerImpl ledger = (ManagedLedgerImpl)persistentTopic.getManagedLedger(); + final ManagedCursor spyCursor= spy(ledger.newNonDurableCursor(PositionFactory.LATEST, "sub-2")); + doAnswer((invocation) -> { + Thread.sleep(5_000); + invocation.callRealMethod(); + return null; + }).when(spyCursor).asyncReadEntriesOrWait(any(int.class), any(long.class), + any(AsyncCallbacks.ReadEntriesCallback.class), any(Object.class), any(Position.class)); + Field cursorField = ManagedLedgerImpl.class.getDeclaredField("cursors"); + cursorField.setAccessible(true); + ManagedCursorContainer container = (ManagedCursorContainer) cursorField.get(ledger); + container.removeCursor("sub-2"); + container.add(spyCursor, null); + final Consumer consumer = pulsarClient.newConsumer(Schema.STRING).topic(topicName) + .subscriptionMode(SubscriptionMode.NonDurable) + .subscriptionType(SubscriptionType.Exclusive) + .subscriptionName("sub-2").subscribe(); + final Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topicName).create(); + producer.send("test"); + producer.close(); + final Message receive = consumer.receive(); + assertEquals("test", receive.getValue()); + consumer.close(); + Awaitility.await() + .pollDelay(5, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .untilAsserted(() -> { + assertEquals(ledger.getWaitingCursorsCount(), 0); + }); + } + + @Test + public void testAddWaitingCursorsForNonDurable2() throws Exception { + final String ns = "prop/ns-test"; + admin.namespaces().createNamespace(ns, 2); + final String topicName = "persistent://prop/ns-test/testAddWaitingCursors2"; + admin.topics().createNonPartitionedTopic(topicName); + pulsarClient.newConsumer(Schema.STRING).topic(topicName) + .subscriptionMode(SubscriptionMode.Durable) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub-1").subscribe().close(); + @Cleanup + final Producer producer = pulsarClient.newProducer(Schema.STRING).enableBatching(false).topic(topicName).create(); + for (int i = 0; i < 100; i ++) { + producer.sendAsync("test-" + i); + } + @Cleanup + final Consumer consumer = pulsarClient.newConsumer(Schema.STRING).topic(topicName) + .subscriptionMode(SubscriptionMode.NonDurable) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionType(SubscriptionType.Exclusive) + .subscriptionName("sub-2").subscribe(); + int count = 0; + while(true) { + final Message msg = consumer.receive(3, TimeUnit.SECONDS); + if (msg != null) { + consumer.acknowledge(msg); + count++; + } else { + break; + } + } + Assert.assertEquals(count, 100); + Thread.sleep(3_000); + for (int i = 0; i < 100; i ++) { + producer.sendAsync("test-" + i); + } + while(true) { + final Message msg = consumer.receive(5, TimeUnit.SECONDS); + if (msg != null) { + consumer.acknowledge(msg); + count++; + } else { + break; + } + } + Assert.assertEquals(count, 200); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionConfigTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionConfigTest.java index aa0015742f662..604326203e876 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionConfigTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionConfigTest.java @@ -20,10 +20,9 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; - import lombok.Cleanup; - import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.ProducerConsumerBase; import org.apache.pulsar.client.api.Schema; @@ -48,6 +47,12 @@ public void cleanup() throws Exception { super.internalCleanup(); } + @Override + protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder pulsarTestContextBuilder) { + super.customizeMainPulsarTestContextBuilder(pulsarTestContextBuilder); + pulsarTestContextBuilder.enableOpenTelemetry(true); + } + @Test public void createReplicatedSubscription() throws Exception { this.conf.setEnableReplicatedSubscriptions(true); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionSnapshotCacheTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionSnapshotCacheTest.java index c269b098c6b88..1587c4965c388 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionSnapshotCacheTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionSnapshotCacheTest.java @@ -22,7 +22,7 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.pulsar.common.api.proto.ReplicatedSubscriptionsSnapshot; import org.testng.annotations.Test; @@ -33,8 +33,8 @@ public class ReplicatedSubscriptionSnapshotCacheTest { public void testSnapshotCache() { ReplicatedSubscriptionSnapshotCache cache = new ReplicatedSubscriptionSnapshotCache("my-subscription", 10); - assertNull(cache.advancedMarkDeletePosition(new PositionImpl(0, 0))); - assertNull(cache.advancedMarkDeletePosition(new PositionImpl(100, 0))); + assertNull(cache.advancedMarkDeletePosition(PositionFactory.create(0, 0))); + assertNull(cache.advancedMarkDeletePosition(PositionFactory.create(100, 0))); ReplicatedSubscriptionsSnapshot s1 = new ReplicatedSubscriptionsSnapshot() .setSnapshotId("snapshot-1"); @@ -57,19 +57,19 @@ public void testSnapshotCache() { cache.addNewSnapshot(s5); cache.addNewSnapshot(s7); - assertNull(cache.advancedMarkDeletePosition(new PositionImpl(0, 0))); - assertNull(cache.advancedMarkDeletePosition(new PositionImpl(1, 0))); - ReplicatedSubscriptionsSnapshot snapshot = cache.advancedMarkDeletePosition(new PositionImpl(1, 1)); + assertNull(cache.advancedMarkDeletePosition(PositionFactory.create(0, 0))); + assertNull(cache.advancedMarkDeletePosition(PositionFactory.create(1, 0))); + ReplicatedSubscriptionsSnapshot snapshot = cache.advancedMarkDeletePosition(PositionFactory.create(1, 1)); assertNotNull(snapshot); assertEquals(snapshot.getSnapshotId(), "snapshot-1"); - snapshot = cache.advancedMarkDeletePosition(new PositionImpl(5, 6)); + snapshot = cache.advancedMarkDeletePosition(PositionFactory.create(5, 6)); assertNotNull(snapshot); assertEquals(snapshot.getSnapshotId(), "snapshot-5"); // Snapshots should have been now removed - assertNull(cache.advancedMarkDeletePosition(new PositionImpl(2, 2))); - assertNull(cache.advancedMarkDeletePosition(new PositionImpl(5, 5))); + assertNull(cache.advancedMarkDeletePosition(PositionFactory.create(2, 2))); + assertNull(cache.advancedMarkDeletePosition(PositionFactory.create(5, 5))); } @Test @@ -98,12 +98,12 @@ public void testSnapshotCachePruning() { cache.addNewSnapshot(s4); // Snapshot-1 was already pruned - assertNull(cache.advancedMarkDeletePosition(new PositionImpl(1, 1))); - ReplicatedSubscriptionsSnapshot snapshot = cache.advancedMarkDeletePosition(new PositionImpl(2, 2)); + assertNull(cache.advancedMarkDeletePosition(PositionFactory.create(1, 1))); + ReplicatedSubscriptionsSnapshot snapshot = cache.advancedMarkDeletePosition(PositionFactory.create(2, 2)); assertNotNull(snapshot); assertEquals(snapshot.getSnapshotId(), "snapshot-2"); - snapshot = cache.advancedMarkDeletePosition(new PositionImpl(5, 5)); + snapshot = cache.advancedMarkDeletePosition(PositionFactory.create(5, 5)); assertNotNull(snapshot); assertEquals(snapshot.getSnapshotId(), "snapshot-4"); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsSnapshotBuilderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsSnapshotBuilderTest.java index 4ae923bd2443c..fa409832fc17b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsSnapshotBuilderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsSnapshotBuilderTest.java @@ -25,16 +25,12 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; - import io.netty.buffer.ByteBuf; - import java.time.Clock; - import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.List; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import java.util.Set; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.common.api.proto.ReplicatedSubscriptionsSnapshot; import org.apache.pulsar.common.api.proto.ReplicatedSubscriptionsSnapshotRequest; @@ -71,16 +67,16 @@ public void setup() { Commands.skipMessageMetadata(marker); markers.add(marker); return null; - }).when(controller) + }) + .when(controller) .writeMarker(any(ByteBuf.class)); } @Test public void testBuildSnapshotWith2Clusters() throws Exception { - List remoteClusters = Collections.singletonList("b"); - ReplicatedSubscriptionsSnapshotBuilder builder = new ReplicatedSubscriptionsSnapshotBuilder(controller, - remoteClusters, conf, clock); + Set.of("b"), + conf, clock); assertTrue(markers.isEmpty()); @@ -93,14 +89,14 @@ public void testBuildSnapshotWith2Clusters() throws Exception { assertEquals(request.getSourceCluster(), localCluster); // Simulate the responses coming back - ReplicatedSubscriptionsSnapshotResponse response = new ReplicatedSubscriptionsSnapshotResponse() - .setSnapshotId("snapshot-1"); + ReplicatedSubscriptionsSnapshotResponse response = new ReplicatedSubscriptionsSnapshotResponse().setSnapshotId( + "snapshot-1"); response.setCluster() .setCluster("b") .setMessageId() .setLedgerId(11) .setEntryId(11); - builder.receivedSnapshotResponse(new PositionImpl(1, 1), response); + builder.receivedSnapshotResponse(PositionFactory.create(1, 1), response); // At this point the snapshot should be created assertEquals(markers.size(), 1); @@ -116,10 +112,9 @@ public void testBuildSnapshotWith2Clusters() throws Exception { @Test public void testBuildSnapshotWith3Clusters() throws Exception { - List remoteClusters = Arrays.asList("b", "c"); - ReplicatedSubscriptionsSnapshotBuilder builder = new ReplicatedSubscriptionsSnapshotBuilder(controller, - remoteClusters, conf, clock); + Set.of("b", "c"), + conf, clock); assertTrue(markers.isEmpty()); @@ -132,26 +127,26 @@ public void testBuildSnapshotWith3Clusters() throws Exception { assertEquals(request.getSourceCluster(), localCluster); // Simulate the responses coming back - ReplicatedSubscriptionsSnapshotResponse response1 = new ReplicatedSubscriptionsSnapshotResponse() - .setSnapshotId("snapshot-1"); + ReplicatedSubscriptionsSnapshotResponse response1 = new ReplicatedSubscriptionsSnapshotResponse().setSnapshotId( + "snapshot-1"); response1.setCluster() .setCluster("b") .setMessageId() .setLedgerId(11) .setEntryId(11); - builder.receivedSnapshotResponse(new PositionImpl(1, 1), response1); + builder.receivedSnapshotResponse(PositionFactory.create(1, 1), response1); // No markers should be sent out assertTrue(markers.isEmpty()); - ReplicatedSubscriptionsSnapshotResponse response2 = new ReplicatedSubscriptionsSnapshotResponse() - .setSnapshotId("snapshot-1"); + ReplicatedSubscriptionsSnapshotResponse response2 = new ReplicatedSubscriptionsSnapshotResponse().setSnapshotId( + "snapshot-1"); response2.setCluster() .setCluster("c") .setMessageId() .setLedgerId(22) .setEntryId(22); - builder.receivedSnapshotResponse(new PositionImpl(2, 2), response2); + builder.receivedSnapshotResponse(PositionFactory.create(2, 2), response2); // Since we have 2 remote clusters, a 2nd round of snapshot will be taken assertEquals(markers.size(), 1); @@ -159,26 +154,26 @@ public void testBuildSnapshotWith3Clusters() throws Exception { assertEquals(request.getSourceCluster(), localCluster); // Responses coming back - ReplicatedSubscriptionsSnapshotResponse response3 = new ReplicatedSubscriptionsSnapshotResponse() - .setSnapshotId("snapshot-1"); + ReplicatedSubscriptionsSnapshotResponse response3 = new ReplicatedSubscriptionsSnapshotResponse().setSnapshotId( + "snapshot-1"); response3.setCluster() .setCluster("b") .setMessageId() .setLedgerId(33) .setEntryId(33); - builder.receivedSnapshotResponse(new PositionImpl(3, 3), response3); + builder.receivedSnapshotResponse(PositionFactory.create(3, 3), response3); // No markers should be sent out assertTrue(markers.isEmpty()); - ReplicatedSubscriptionsSnapshotResponse response4 = new ReplicatedSubscriptionsSnapshotResponse() - .setSnapshotId("snapshot-1"); + ReplicatedSubscriptionsSnapshotResponse response4 = new ReplicatedSubscriptionsSnapshotResponse().setSnapshotId( + "snapshot-1"); response4.setCluster() .setCluster("c") .setMessageId() .setLedgerId(44) .setEntryId(44); - builder.receivedSnapshotResponse(new PositionImpl(4, 4), response4); + builder.receivedSnapshotResponse(PositionFactory.create(4, 4), response4); // At this point the snapshot should be created assertEquals(markers.size(), 1); @@ -198,10 +193,9 @@ public void testBuildSnapshotWith3Clusters() throws Exception { @Test public void testBuildTimeout() { - List remoteClusters = Collections.singletonList("b"); - ReplicatedSubscriptionsSnapshotBuilder builder = new ReplicatedSubscriptionsSnapshotBuilder(controller, - remoteClusters, conf, clock); + Set.of("b"), + conf, clock); assertFalse(builder.isTimedOut()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ShadowReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ShadowReplicatorTest.java index 9f1885e034def..511cf87133a0d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ShadowReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ShadowReplicatorTest.java @@ -142,8 +142,8 @@ public void testShadowReplication() throws Exception { Assert.assertEquals(shadowMessage.getBrokerPublishTime(), sourceMessage.getBrokerPublishTime()); Assert.assertEquals(shadowMessage.getIndex(), sourceMessage.getIndex()); - //`replicatedFrom` is set as localClusterName in shadow topic. - Assert.assertNotEquals(shadowMessage.getReplicatedFrom(), sourceMessage.getReplicatedFrom()); + Assert.assertEquals(replicator.stats.getBytesOutCount(), 0); + Assert.assertEquals(shadowMessage.getMessageId(), sourceMessage.getMessageId()); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ShadowTopicRealBkTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ShadowTopicRealBkTest.java new file mode 100644 index 0000000000000..b0e572a826c47 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ShadowTopicRealBkTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service.persistent; + +import com.google.common.collect.Lists; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.apache.bookkeeper.mledger.impl.ShadowManagedLedgerImpl; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.common.util.PortManager; +import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.awaitility.Awaitility; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +public class ShadowTopicRealBkTest { + + private static final String cluster = "test"; + private final int zkPort = PortManager.nextLockedFreePort(); + private final LocalBookkeeperEnsemble bk = new LocalBookkeeperEnsemble(2, zkPort, PortManager::nextLockedFreePort); + private PulsarService pulsar; + private PulsarAdmin admin; + + @BeforeClass + public void setup() throws Exception { + bk.start(); + final var config = new ServiceConfiguration(); + config.setClusterName(cluster); + config.setAdvertisedAddress("localhost"); + config.setBrokerServicePort(Optional.of(0)); + config.setWebServicePort(Optional.of(0)); + config.setMetadataStoreUrl("zk:localhost:" + zkPort); + pulsar = new PulsarService(config); + pulsar.start(); + admin = pulsar.getAdminClient(); + admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()) + .brokerServiceUrl(pulsar.getBrokerServiceUrl()).build()); + admin.tenants().createTenant("public", TenantInfo.builder().allowedClusters(Set.of(cluster)).build()); + admin.namespaces().createNamespace("public/default"); + } + + @AfterClass(alwaysRun = true) + public void cleanup() throws Exception { + if (pulsar != null) { + pulsar.close(); + } + bk.stop(); + } + + @Test + public void testReadFromStorage() throws Exception { + final var sourceTopic = TopicName.get("test-read-from-source" + UUID.randomUUID()).toString(); + final var shadowTopic = sourceTopic + "-shadow"; + + admin.topics().createNonPartitionedTopic(sourceTopic); + admin.topics().createShadowTopic(shadowTopic, sourceTopic); + admin.topics().setShadowTopics(sourceTopic, Lists.newArrayList(shadowTopic)); + + Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(()->{ + final var sourcePersistentTopic = (PersistentTopic) pulsar.getBrokerService() + .getTopicIfExists(sourceTopic).get().orElseThrow(); + final var replicator = (ShadowReplicator) sourcePersistentTopic.getShadowReplicators().get(shadowTopic); + Assert.assertNotNull(replicator); + Assert.assertEquals(String.valueOf(replicator.getState()), "Started"); + }); + + final var client = pulsar.getClient(); + // When the message was sent, there is no cursor, so it will read from the cache + final var producer = client.newProducer().topic(sourceTopic).create(); + producer.send("message".getBytes()); + // 1. Verify RangeEntryCacheImpl#readFromStorage + final var consumer = client.newConsumer().topic(shadowTopic).subscriptionName("sub") + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe(); + final var msg = consumer.receive(5, TimeUnit.SECONDS); + Assert.assertNotNull(msg); + Assert.assertEquals(msg.getValue(), "message".getBytes()); + + // 2. Verify EntryCache#asyncReadEntry + final var shadowManagedLedger = ((PersistentTopic) pulsar.getBrokerService().getTopicIfExists(shadowTopic).get() + .orElseThrow()).getManagedLedger(); + Assert.assertTrue(shadowManagedLedger instanceof ShadowManagedLedgerImpl); + shadowManagedLedger.getEarliestMessagePublishTimeInBacklog().get(3, TimeUnit.SECONDS); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ShadowTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ShadowTopicTest.java index 1dbfe109a93d4..5334339ae5b62 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ShadowTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ShadowTopicTest.java @@ -21,6 +21,8 @@ import com.google.common.collect.Lists; import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.Set; import java.util.concurrent.TimeUnit; import lombok.AllArgsConstructor; import lombok.Cleanup; @@ -113,6 +115,35 @@ public void testPartitionedShadowTopicSetup() throws Exception { Assert.assertEquals(brokerShadowTopic.getShadowSourceTopic().get().toString(), sourceTopicPartition); } + @Test + public void testPartitionedShadowTopicProduceAndConsume() throws Exception { + String sourceTopic = newShadowSourceTopicName(); + String shadowTopic = sourceTopic + "-shadow"; + admin.topics().createPartitionedTopic(sourceTopic, 3); + admin.topics().createShadowTopic(shadowTopic, sourceTopic); + + admin.topics().setShadowTopics(sourceTopic, Lists.newArrayList(shadowTopic)); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create(); + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.STRING).topic(shadowTopic).subscriptionName("test") + .subscribe(); + + for (int i = 0; i < 10; i++) { + producer.send("msg-" + i); + } + + Set set = new HashSet<>(); + for (int i = 0; i < 10; i++) { + Message msg = consumer.receive(); + set.add(msg.getValue()); + } + for (int i = 0; i < 10; i++) { + Assert.assertTrue(set.contains("msg-" + i)); + } + } + @Test public void testShadowTopicNotWritable() throws Exception { String sourceTopic = newShadowSourceTopicName(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/TopicDuplicationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/TopicDuplicationTest.java index e57092d02dd5d..f1940a2899978 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/TopicDuplicationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/TopicDuplicationTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.service.persistent; +import static org.apache.pulsar.broker.service.persistent.PersistentTopic.DEDUPLICATION_CURSOR_NAME; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; @@ -25,6 +26,7 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; + import java.util.Optional; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -32,13 +34,18 @@ import java.util.concurrent.TimeUnit; import lombok.Cleanup; import org.apache.bookkeeper.mledger.ManagedCursor; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerConsumerBase; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.naming.TopicName; import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -244,16 +251,16 @@ public void testTopicPolicyTakeSnapshot() throws Exception { countDownLatch.await(); PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopicIfExists(topicName).get().get(); long seqId = persistentTopic.getMessageDeduplication().highestSequencedPersisted.get(producerName); - PositionImpl position = (PositionImpl) persistentTopic.getMessageDeduplication().getManagedCursor() + Position position = persistentTopic.getMessageDeduplication().getManagedCursor() .getManagedLedger().getLastConfirmedEntry(); assertEquals(seqId, msgNum - 1); assertEquals(position.getEntryId(), msgNum - 1); //The first time, use topic-leve policies, 1 second delay + 3 second interval Awaitility.await() - .until(() -> ((PositionImpl) persistentTopic.getMessageDeduplication().getManagedCursor() + .until(() -> (persistentTopic.getMessageDeduplication().getManagedCursor() .getMarkDeletedPosition()).getEntryId() == msgNum - 1); ManagedCursor managedCursor = persistentTopic.getMessageDeduplication().getManagedCursor(); - PositionImpl markDeletedPosition = (PositionImpl) managedCursor.getMarkDeletedPosition(); + Position markDeletedPosition = managedCursor.getMarkDeletedPosition(); assertEquals(position, markDeletedPosition); //remove topic-level policies, namespace-level should be used, interval becomes 5 seconds @@ -261,10 +268,10 @@ public void testTopicPolicyTakeSnapshot() throws Exception { producer.newMessage().value("msg").send(); //zk update time + 5 second interval time Awaitility.await() - .until(() -> ((PositionImpl) persistentTopic.getMessageDeduplication().getManagedCursor() + .until(() -> (persistentTopic.getMessageDeduplication().getManagedCursor() .getMarkDeletedPosition()).getEntryId() == msgNum); - markDeletedPosition = (PositionImpl) managedCursor.getMarkDeletedPosition(); - position = (PositionImpl) persistentTopic.getMessageDeduplication().getManagedCursor().getManagedLedger().getLastConfirmedEntry(); + markDeletedPosition = managedCursor.getMarkDeletedPosition(); + position = persistentTopic.getMessageDeduplication().getManagedCursor().getManagedLedger().getLastConfirmedEntry(); assertEquals(msgNum, markDeletedPosition.getEntryId()); assertEquals(position, markDeletedPosition); @@ -275,17 +282,17 @@ public void testTopicPolicyTakeSnapshot() throws Exception { producer.newMessage().value("msg").send(); //ensure that the time exceeds the scheduling interval of ns and topic, but no snapshot is generated Thread.sleep(3000); - markDeletedPosition = (PositionImpl) managedCursor.getMarkDeletedPosition(); - position = (PositionImpl) persistentTopic.getMessageDeduplication().getManagedCursor().getManagedLedger().getLastConfirmedEntry(); + markDeletedPosition = managedCursor.getMarkDeletedPosition(); + position = persistentTopic.getMessageDeduplication().getManagedCursor().getManagedLedger().getLastConfirmedEntry(); // broker-level interval is 7 seconds, so 3 seconds will not take a snapshot assertNotEquals(msgNum + 1, markDeletedPosition.getEntryId()); assertNotEquals(position, markDeletedPosition); // wait for scheduler Awaitility.await() - .until(() -> ((PositionImpl) persistentTopic.getMessageDeduplication().getManagedCursor() + .until(() -> (persistentTopic.getMessageDeduplication().getManagedCursor() .getMarkDeletedPosition()).getEntryId() == msgNum + 1); - markDeletedPosition = (PositionImpl) managedCursor.getMarkDeletedPosition(); - position = (PositionImpl) persistentTopic.getMessageDeduplication().getManagedCursor().getManagedLedger().getLastConfirmedEntry(); + markDeletedPosition = managedCursor.getMarkDeletedPosition(); + position = persistentTopic.getMessageDeduplication().getManagedCursor().getManagedLedger().getLastConfirmedEntry(); assertEquals(msgNum + 1, markDeletedPosition.getEntryId()); assertEquals(position, markDeletedPosition); } @@ -347,13 +354,13 @@ private void testTakeSnapshot(boolean enabledSnapshot) throws Exception { countDownLatch.await(); PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopicIfExists(topicName).get().get(); long seqId = persistentTopic.getMessageDeduplication().highestSequencedPersisted.get(producerName); - PositionImpl position = (PositionImpl) persistentTopic.getMessageDeduplication().getManagedCursor().getManagedLedger().getLastConfirmedEntry(); + Position position = persistentTopic.getMessageDeduplication().getManagedCursor().getManagedLedger().getLastConfirmedEntry(); assertEquals(seqId, msgNum - 1); assertEquals(position.getEntryId(), msgNum - 1); Thread.sleep(2000); ManagedCursor managedCursor = persistentTopic.getMessageDeduplication().getManagedCursor(); - PositionImpl markDeletedPosition = (PositionImpl) managedCursor.getMarkDeletedPosition(); + Position markDeletedPosition = managedCursor.getMarkDeletedPosition(); if (enabledSnapshot) { assertEquals(position, markDeletedPosition); } else { @@ -362,14 +369,14 @@ private void testTakeSnapshot(boolean enabledSnapshot) throws Exception { } producer.newMessage().value("msg").send(); - markDeletedPosition = (PositionImpl) managedCursor.getMarkDeletedPosition(); - position = (PositionImpl) persistentTopic.getMessageDeduplication().getManagedCursor().getManagedLedger().getLastConfirmedEntry(); + markDeletedPosition = managedCursor.getMarkDeletedPosition(); + position = persistentTopic.getMessageDeduplication().getManagedCursor().getManagedLedger().getLastConfirmedEntry(); assertNotEquals(msgNum, markDeletedPosition.getEntryId()); assertNotNull(position); Thread.sleep(2000); - markDeletedPosition = (PositionImpl) managedCursor.getMarkDeletedPosition(); - position = (PositionImpl) persistentTopic.getMessageDeduplication().getManagedCursor().getManagedLedger().getLastConfirmedEntry(); + markDeletedPosition = managedCursor.getMarkDeletedPosition(); + position = persistentTopic.getMessageDeduplication().getManagedCursor().getManagedLedger().getLastConfirmedEntry(); if (enabledSnapshot) { assertEquals(msgNum, markDeletedPosition.getEntryId()); assertEquals(position, markDeletedPosition); @@ -424,27 +431,27 @@ public void testNamespacePolicyTakeSnapshot() throws Exception { countDownLatch.await(); PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopicIfExists(topicName).get().get(); long seqId = persistentTopic.getMessageDeduplication().highestSequencedPersisted.get(producerName); - PositionImpl position = (PositionImpl) persistentTopic.getMessageDeduplication().getManagedCursor() + Position position = persistentTopic.getMessageDeduplication().getManagedCursor() .getManagedLedger().getLastConfirmedEntry(); assertEquals(seqId, msgNum - 1); assertEquals(position.getEntryId(), msgNum - 1); //The first time, 1 second delay + 1 second interval - Awaitility.await().until(()-> ((PositionImpl) persistentTopic + Awaitility.await().until(()-> (persistentTopic .getMessageDeduplication().getManagedCursor().getMarkDeletedPosition()).getEntryId() == msgNum -1); ManagedCursor managedCursor = persistentTopic.getMessageDeduplication().getManagedCursor(); - PositionImpl markDeletedPosition = (PositionImpl) managedCursor.getMarkDeletedPosition(); + Position markDeletedPosition = managedCursor.getMarkDeletedPosition(); assertEquals(position, markDeletedPosition); //remove namespace-level policies, broker-level should be used admin.namespaces().removeDeduplicationSnapshotInterval(myNamespace); Thread.sleep(2000); - markDeletedPosition = (PositionImpl) managedCursor.getMarkDeletedPosition(); - position = (PositionImpl) persistentTopic.getMessageDeduplication().getManagedCursor().getManagedLedger().getLastConfirmedEntry(); + markDeletedPosition = managedCursor.getMarkDeletedPosition(); + position = persistentTopic.getMessageDeduplication().getManagedCursor().getManagedLedger().getLastConfirmedEntry(); assertNotEquals(msgNum - 1, markDeletedPosition.getEntryId()); assertNotEquals(position, markDeletedPosition.getEntryId()); //3 seconds total Thread.sleep(1000); - markDeletedPosition = (PositionImpl) managedCursor.getMarkDeletedPosition(); - position = (PositionImpl) persistentTopic.getMessageDeduplication().getManagedCursor().getManagedLedger().getLastConfirmedEntry(); + markDeletedPosition = managedCursor.getMarkDeletedPosition(); + position = persistentTopic.getMessageDeduplication().getManagedCursor().getManagedLedger().getLastConfirmedEntry(); assertEquals(msgNum - 1, markDeletedPosition.getEntryId()); assertEquals(position, markDeletedPosition); @@ -475,14 +482,14 @@ public void testDisableNamespacePolicyTakeSnapshot() throws Exception { countDownLatch.await(); PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopicIfExists(topicName).get().get(); ManagedCursor managedCursor = persistentTopic.getMessageDeduplication().getManagedCursor(); - PositionImpl markDeletedPosition = (PositionImpl) managedCursor.getMarkDeletedPosition(); + Position markDeletedPosition = managedCursor.getMarkDeletedPosition(); long seqId = persistentTopic.getMessageDeduplication().highestSequencedPersisted.get(producerName); - PositionImpl position = (PositionImpl) persistentTopic.getMessageDeduplication().getManagedCursor() + Position position = persistentTopic.getMessageDeduplication().getManagedCursor() .getManagedLedger().getLastConfirmedEntry(); assertEquals(seqId, msgNum - 1); assertEquals(position.getEntryId(), msgNum - 1); - Awaitility.await().until(()-> ((PositionImpl) persistentTopic + Awaitility.await().until(()-> (persistentTopic .getMessageDeduplication().getManagedCursor().getMarkDeletedPosition()).getEntryId() == -1); // take snapshot is disabled, so markDeletedPosition should not change @@ -492,6 +499,133 @@ public void testDisableNamespacePolicyTakeSnapshot() throws Exception { } + @Test(timeOut = 30000) + public void testDisableNamespacePolicyTakeSnapshotShouldNotThrowException() throws Exception { + cleanup(); + conf.setBrokerDeduplicationEnabled(true); + conf.setBrokerDeduplicationSnapshotFrequencyInSeconds(1); + conf.setBrokerDeduplicationSnapshotIntervalSeconds(1); + conf.setBrokerDeduplicationEntriesInterval(20000); + setup(); + + final String topicName = testTopic + UUID.randomUUID().toString(); + final String producerName = "my-producer"; + @Cleanup + Producer producer = pulsarClient + .newProducer(Schema.STRING).topic(topicName).enableBatching(false).producerName(producerName).create(); + + // disable deduplication + admin.namespaces().setDeduplicationStatus(myNamespace, false); + + int msgNum = 50; + CountDownLatch countDownLatch = new CountDownLatch(msgNum); + for (int i = 0; i < msgNum; i++) { + producer.newMessage().value("msg" + i).sendAsync().whenComplete((res, e) -> countDownLatch.countDown()); + } + countDownLatch.await(); + PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService() + .getTopicIfExists(topicName).get().get(); + ManagedCursor managedCursor = persistentTopic.getMessageDeduplication().getManagedCursor(); + + // when disable topic deduplication the cursor should be deleted. + assertNull(managedCursor); + + // this method will be called at brokerService forEachTopic. + // if topic level disable deduplication. + // this method should be skipped without throw exception. + persistentTopic.checkDeduplicationSnapshot(); + } + + @Test + public void testFinishTakeSnapshotWhenTopicLoading() throws Exception { + cleanup(); + setup(); + + // Create a topic and wait deduplication is started. + int brokerDeduplicationEntriesInterval = 1000; + pulsar.getConfiguration().setBrokerDeduplicationEnabled(true); + pulsar.getConfiguration().setBrokerDeduplicationEntriesInterval(brokerDeduplicationEntriesInterval); + final String topic = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createNonPartitionedTopic(topic); + final PersistentTopic persistentTopic1 = + (PersistentTopic) pulsar.getBrokerService().getTopic(topic, false).join().get(); + final ManagedLedgerImpl ml1 = (ManagedLedgerImpl) persistentTopic1.getManagedLedger(); + Awaitility.await().untilAsserted(() -> { + ManagedCursorImpl cursor1 = + (ManagedCursorImpl) ml1.getCursors().get(PersistentTopic.DEDUPLICATION_CURSOR_NAME); + assertNotNull(cursor1); + }); + final MessageDeduplication deduplication1 = persistentTopic1.getMessageDeduplication(); + + + // Send 999 messages, it is less than "brokerDeduplicationEntriesInterval". + // So it would not trigger takeSnapshot + final Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic).enableBatching(false).create(); + for (int i = 0; i < brokerDeduplicationEntriesInterval - 1; i++) { + producer.send(i + ""); + } + producer.close(); + int snapshotCounter1 = WhiteboxImpl.getInternalState(deduplication1, "snapshotCounter"); + assertEquals(snapshotCounter1, brokerDeduplicationEntriesInterval - 1); + + + // Unload and load topic, simulate topic load is timeout. + // SetBrokerDeduplicationEntriesInterval to 10, therefore recoverSequenceIdsMap#takeSnapshot + // would trigger and should update the snapshot position. + // However, if topic close and takeSnapshot are concurrent, + // it would result in takeSnapshot throw exception + admin.topics().unload(topic); + pulsar.getConfiguration().setBrokerDeduplicationEntriesInterval(10); + + // Mock message deduplication recovery speed topicLoadTimeoutSeconds + pulsar.getConfiguration().setTopicLoadTimeoutSeconds(1); + String mlPath = BrokerService.MANAGED_LEDGER_PATH_ZNODE + "/" + + TopicName.get(topic).getPersistenceNamingEncoding() + "/" + DEDUPLICATION_CURSOR_NAME; + mockZooKeeper.delay(2 * 1000, (op, path) -> { + if (mlPath.equals(path)) { + return true; + } + return false; + }); + + final var topics = pulsar.getBrokerService().getTopics(); + try { + pulsar.getBrokerService().getTopic(topic, false).join().get(); + Assert.fail(); + } catch (Exception e) { + // topic loading should timeout. + } + Awaitility.await().untilAsserted(() -> { + // topic loading timeout then close topic and remove from topicsMap + Assert.assertFalse(topics.containsKey(topic)); + }); + + + // Load topic again, setBrokerDeduplicationEntriesInterval to 10000, + // make recoverSequenceIdsMap#takeSnapshot not trigger takeSnapshot. + // But actually it should not replay again in recoverSequenceIdsMap, + // since previous topic loading should finish the replay process. + pulsar.getConfiguration().setBrokerDeduplicationEntriesInterval(10000); + pulsar.getConfiguration().setTopicLoadTimeoutSeconds(60); + PersistentTopic persistentTopic2 = + (PersistentTopic) pulsar.getBrokerService().getTopic(topic, false).join().get(); + ManagedLedgerImpl ml2 = (ManagedLedgerImpl) persistentTopic2.getManagedLedger(); + MessageDeduplication deduplication2 = persistentTopic2.getMessageDeduplication(); + + Awaitility.await().untilAsserted(() -> { + int snapshotCounter3 = WhiteboxImpl.getInternalState(deduplication2, "snapshotCounter"); + Assert.assertEquals(snapshotCounter3, 0); + Assert.assertEquals(ml2.getLedgersInfo().size(), 1); + }); + + + // cleanup. + admin.topics().delete(topic); + cleanup(); + setup(); + } + private void waitCacheInit(String topicName) throws Exception { pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe().close(); TopicName topic = TopicName.get(topicName); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java index b868858646c50..e5ebf5b884477 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java @@ -22,6 +22,7 @@ import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgsRecordingInvocations; import static org.apache.pulsar.client.api.SubscriptionInitialPosition.Earliest; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -30,8 +31,9 @@ import static org.testng.Assert.assertTrue; import static org.testng.AssertJUnit.assertEquals; import static org.testng.AssertJUnit.assertNotNull; - +import io.netty.buffer.ByteBuf; import java.lang.reflect.Field; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -39,12 +41,11 @@ import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; - import lombok.Cleanup; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Entry; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; import org.apache.commons.lang.reflect.FieldUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.service.AbstractTopic; @@ -58,11 +59,15 @@ import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.RawMessage; +import org.apache.pulsar.client.api.RawReader; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.stats.AnalyzeSubscriptionBacklogResult; +import org.apache.pulsar.compaction.Compactor; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -150,6 +155,58 @@ public void testOverride() throws Exception { consumer.close(); } + @Test + public void testEntryFilterWithCompactor() throws Exception { + conf.setAllowOverrideEntryFilters(true); + String topic = "persistent://prop/ns-abc/topic" + UUID.randomUUID(); + + List messages = new ArrayList<>(); + Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(false).topic(topic).create(); + producer.newMessage().key("K1").value("V1").send(); + producer.newMessage().key("K2").value("V2").send(); + producer.newMessage().key("K3").value("V3").send(); + producer.newMessage().key("K4").value("V4").send(); + messages.add("V2"); + messages.add("V4"); + + PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get(); + + // set topic level entry filters + EntryFilter mockFilter = mock(EntryFilter.class); + doAnswer(invocationOnMock -> { + FilterContext filterContext = invocationOnMock.getArgument(1); + String partitionKey = filterContext.getMsgMetadata().getPartitionKey(); + if (partitionKey.equals("K1") || partitionKey.equals("K3")) { + return EntryFilter.FilterResult.REJECT; + } else { + return EntryFilter.FilterResult.ACCEPT; + } + }).when(mockFilter).filterEntry(any(Entry.class), any(FilterContext.class)); + setMockFilterToTopic(topicRef, List.of(mockFilter)); + + List results = new ArrayList<>(); + RawReader rawReader = RawReader.create(pulsarClient, topic, Compactor.COMPACTION_SUBSCRIPTION).get(); + while (true) { + boolean hasMsg = rawReader.hasMessageAvailableAsync().get(); + if (hasMsg) { + try (RawMessage m = rawReader.readNextAsync().get()) { + ByteBuf headersAndPayload = m.getHeadersAndPayload(); + Commands.skipMessageMetadata(headersAndPayload); + byte[] bytes = new byte[headersAndPayload.readableBytes()]; + headersAndPayload.readBytes(bytes); + + results.add(new String(bytes)); + } + } else { + break; + } + } + rawReader.closeAsync().get(); + + Assert.assertEquals(messages, results); + } + @SneakyThrows private void setMockFilterToTopic(PersistentTopic topicRef, List mockFilter) { FieldUtils.writeField(topicRef, "entryFilters", Pair.of(null, mockFilter), true); @@ -182,9 +239,9 @@ public void testFilter() throws Exception { hasFilterField.setAccessible(true); NarClassLoader narClassLoader = mock(NarClassLoader.class); EntryFilter filter1 = new EntryFilterTest(); - EntryFilterWithClassLoader loader1 = spyWithClassAndConstructorArgsRecordingInvocations(EntryFilterWithClassLoader.class, filter1, narClassLoader); + EntryFilterWithClassLoader loader1 = spyWithClassAndConstructorArgsRecordingInvocations(EntryFilterWithClassLoader.class, filter1, narClassLoader, false); EntryFilter filter2 = new EntryFilter2Test(); - EntryFilterWithClassLoader loader2 = spyWithClassAndConstructorArgsRecordingInvocations(EntryFilterWithClassLoader.class, filter2, narClassLoader); + EntryFilterWithClassLoader loader2 = spyWithClassAndConstructorArgsRecordingInvocations(EntryFilterWithClassLoader.class, filter2, narClassLoader, false); field.set(dispatcher, List.of(loader1, loader2)); hasFilterField.set(dispatcher, true); @@ -198,7 +255,7 @@ public void testFilter() throws Exception { int counter = 0; while (true) { - Message message = consumer.receive(1, TimeUnit.SECONDS); + Message message = consumer.receive(5, TimeUnit.SECONDS); if (message != null) { counter++; consumer.acknowledge(message); @@ -232,7 +289,7 @@ public void testFilter() throws Exception { counter = 0; while (true) { - Message message = consumer.receive(1, TimeUnit.SECONDS); + Message message = consumer.receive(5, TimeUnit.SECONDS); if (message != null) { counter++; consumer.acknowledge(message); @@ -251,7 +308,7 @@ public void testFilter() throws Exception { assertNotNull(lastMsgId); MessageIdImpl finalLastMsgId = lastMsgId; Awaitility.await().untilAsserted(() -> { - PositionImpl position = (PositionImpl) subscription.getCursor().getMarkDeletedPosition(); + Position position = subscription.getCursor().getMarkDeletedPosition(); assertEquals(position.getLedgerId(), finalLastMsgId.getLedgerId()); assertEquals(position.getEntryId(), finalLastMsgId.getEntryId()); }); @@ -264,7 +321,7 @@ public void testFilter() throws Exception { } counter = 0; while (true) { - Message message = consumer.receive(1, TimeUnit.SECONDS); + Message message = consumer.receive(5, TimeUnit.SECONDS); if (message != null) { counter++; consumer.acknowledge(message); @@ -314,9 +371,9 @@ public void testFilteredMsgCount(String topic) throws Throwable { hasFilterField.setAccessible(true); NarClassLoader narClassLoader = mock(NarClassLoader.class); EntryFilter filter1 = new EntryFilterTest(); - EntryFilterWithClassLoader loader1 = spyWithClassAndConstructorArgs(EntryFilterWithClassLoader.class, filter1, narClassLoader); + EntryFilterWithClassLoader loader1 = spyWithClassAndConstructorArgs(EntryFilterWithClassLoader.class, filter1, narClassLoader, false); EntryFilter filter2 = new EntryFilter2Test(); - EntryFilterWithClassLoader loader2 = spyWithClassAndConstructorArgs(EntryFilterWithClassLoader.class, filter2, narClassLoader); + EntryFilterWithClassLoader loader2 = spyWithClassAndConstructorArgs(EntryFilterWithClassLoader.class, filter2, narClassLoader, false); field.set(dispatcher, List.of(loader1, loader2)); hasFilterField.set(dispatcher, true); @@ -406,10 +463,10 @@ public void testEntryFilterRescheduleMessageDependingOnConsumerSharedSubscriptio NarClassLoader narClassLoader = mock(NarClassLoader.class); EntryFilter filter1 = new EntryFilterTest(); EntryFilterWithClassLoader loader1 = - spyWithClassAndConstructorArgs(EntryFilterWithClassLoader.class, filter1, narClassLoader); + spyWithClassAndConstructorArgs(EntryFilterWithClassLoader.class, filter1, narClassLoader, false); EntryFilter filter2 = new EntryFilterTest(); EntryFilterWithClassLoader loader2 = - spyWithClassAndConstructorArgs(EntryFilterWithClassLoader.class, filter2, narClassLoader); + spyWithClassAndConstructorArgs(EntryFilterWithClassLoader.class, filter2, narClassLoader, false); field.set(dispatcher, List.of(loader1, loader2)); hasFilterField.set(dispatcher, true); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorageTest.java index d0c2e149bf438..3653c01daec37 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorageTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorageTest.java @@ -21,6 +21,7 @@ import java.nio.ByteBuffer; import org.apache.bookkeeper.client.api.BKException; import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.service.schema.exceptions.SchemaException; import org.apache.pulsar.common.schema.LongSchemaVersion; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.testng.annotations.Test; @@ -29,23 +30,29 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; @Test(groups = "broker") public class BookkeeperSchemaStorageTest { @Test public void testBkException() { - Exception ex = bkException("test", BKException.Code.ReadException, 1, -1); + Exception ex = bkException("test", BKException.Code.ReadException, 1, -1, false); assertEquals("Error while reading ledger - ledger=1 - operation=test", ex.getMessage()); - ex = bkException("test", BKException.Code.ReadException, 1, 0); + ex = bkException("test", BKException.Code.ReadException, 1, 0, false); assertEquals("Error while reading ledger - ledger=1 - operation=test - entry=0", ex.getMessage()); - ex = bkException("test", BKException.Code.QuorumException, 1, -1); + ex = bkException("test", BKException.Code.QuorumException, 1, -1, false); assertEquals("Invalid quorum size on ensemble size - ledger=1 - operation=test", ex.getMessage()); - ex = bkException("test", BKException.Code.QuorumException, 1, 0); + ex = bkException("test", BKException.Code.QuorumException, 1, 0, false); assertEquals("Invalid quorum size on ensemble size - ledger=1 - operation=test - entry=0", ex.getMessage()); + SchemaException sc = (SchemaException) bkException("test", BKException.Code.BookieHandleNotAvailableException, 1, 0, false); + assertTrue(sc.isRecoverable()); + sc = (SchemaException) bkException("test", BKException.Code.BookieHandleNotAvailableException, 1, 0, true); + assertFalse(sc.isRecoverable()); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/ClientGetSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/ClientGetSchemaTest.java index 970e6b2712981..ec81f39fef92c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/ClientGetSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/ClientGetSchemaTest.java @@ -20,8 +20,9 @@ import static org.apache.pulsar.common.naming.TopicName.PUBLIC_TENANT; import static org.apache.pulsar.schema.compatibility.SchemaCompatibilityCheckTest.randomName; -import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertNotNull; import java.util.ArrayList; import java.util.List; @@ -177,4 +178,36 @@ public void testSchemaFailure() throws Exception { producer.close(); consumer.close(); } + + @Test + public void testAddProducerOnDeletedSchemaLedgerTopic() throws Exception { + final String tenant = PUBLIC_TENANT; + final String namespace = "test-namespace-" + randomName(16); + final String topicOne = "test-deleted-schema-ledger"; + final String fqtnOne = TopicName.get(TopicDomain.persistent.value(), tenant, namespace, topicOne).toString(); + + //pulsar.getConfig().setManagedLedgerForceRecovery(true); + admin.namespaces().createNamespace(tenant + "/" + namespace, Sets.newHashSet("test")); + + // (1) create topic with schema + Producer producer = pulsarClient + .newProducer(Schema.AVRO(SchemaDefinition. builder().withAlwaysAllowNull(false) + .withSupportSchemaVersioning(true).withPojo(Schemas.PersonTwo.class).build())) + .topic(fqtnOne).create(); + + producer.close(); + + String key = TopicName.get(fqtnOne).getSchemaName(); + BookkeeperSchemaStorage schemaStrogate = (BookkeeperSchemaStorage) pulsar.getSchemaStorage(); + long schemaLedgerId = schemaStrogate.getSchemaLedgerList(key).get(0); + + // (2) break schema locator by deleting schema-ledger + schemaStrogate.getBookKeeper().deleteLedger(schemaLedgerId); + + admin.topics().unload(fqtnOne); + + Producer producerWihtoutSchema = pulsarClient.newProducer().topic(fqtnOne).create(); + + assertNotNull(producerWihtoutSchema); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java index c7e30d5c3fc37..658ea268c644c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java @@ -18,17 +18,23 @@ */ package org.apache.pulsar.broker.service.schema; +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat; +import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.Metric; +import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.parseMetrics; +import static org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy.BACKWARD; +import static org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy.BACKWARD_TRANSITIVE; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.testng.Assert.fail; import static org.testng.AssertJUnit.assertEquals; -import static org.testng.AssertJUnit.assertFalse; import static org.testng.AssertJUnit.assertNull; -import static org.testng.AssertJUnit.assertTrue; import com.google.common.collect.Multimap; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; +import io.opentelemetry.api.common.Attributes; import java.io.ByteArrayOutputStream; import java.nio.charset.StandardCharsets; import java.time.Clock; +import java.time.Duration; import java.time.Instant; import java.time.ZoneId; import java.util.ArrayList; @@ -36,21 +42,31 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.pulsar.PrometheusMetricsTestUtil; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.service.schema.SchemaRegistry.SchemaAndMetadata; -import org.apache.pulsar.broker.stats.PrometheusMetricsTest; -import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGenerator; +import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.apache.pulsar.client.impl.schema.KeyValueSchemaInfo; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; +import org.apache.pulsar.common.protocol.schema.IsCompatibilityResponse; import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.protocol.schema.SchemaVersion; +import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.common.schema.LongSchemaVersion; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.schema.SchemaInfoWithVersion; import org.apache.pulsar.common.schema.SchemaType; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; +import org.assertj.core.api.InstanceOfAssertFactories; +import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -59,7 +75,7 @@ @Test(groups = "broker") public class SchemaServiceTest extends MockedPulsarServiceBaseTest { - private static final Clock MockClock = Clock.fixed(Instant.EPOCH, ZoneId.systemDefault()); + private static final Clock MockClock = Clock.fixed(Instant.now(), ZoneId.systemDefault()); private final String schemaId1 = "1/2/3/4"; private static final String userId = "user"; @@ -92,7 +108,21 @@ protected void setup() throws Exception { storage.start(); Map checkMap = new HashMap<>(); checkMap.put(SchemaType.AVRO, new AvroSchemaCompatibilityCheck()); - schemaRegistryService = new SchemaRegistryServiceImpl(storage, checkMap, MockClock, null); + schemaRegistryService = new SchemaRegistryServiceImpl(storage, checkMap, MockClock, pulsar); + + var schemaRegistryStats = + Mockito.spy((SchemaRegistryStats) FieldUtils.readField(schemaRegistryService, "stats", true)); + // Disable periodic cleanup of Prometheus entries. + Mockito.doNothing().when(schemaRegistryStats).run(); + FieldUtils.writeField(schemaRegistryService, "stats", schemaRegistryStats, true); + + setupDefaultTenantAndNamespace(); + } + + @Override + protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder pulsarTestContextBuilder) { + super.customizeMainPulsarTestContextBuilder(pulsarTestContextBuilder); + pulsarTestContextBuilder.enableOpenTelemetry(true); } @AfterMethod(alwaysRun = true) @@ -110,33 +140,59 @@ public void testSchemaRegistryMetrics() throws Exception { getSchema(schemaId, version(0)); deleteSchema(schemaId, version(1)); + var otelMetrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + assertThat(otelMetrics).anySatisfy(metric -> assertThat(metric) + .hasName(SchemaRegistryStats.SCHEMA_REGISTRY_REQUEST_DURATION_METRIC_NAME) + .hasHistogramSatisfying(histogram -> histogram.hasPointsSatisfying( + point -> point + .hasAttributes(Attributes.of(OpenTelemetryAttributes.PULSAR_NAMESPACE, "tenant/ns", + SchemaRegistryStats.REQUEST_TYPE_KEY, "delete", + SchemaRegistryStats.RESPONSE_TYPE_KEY, "success")) + .hasCount(1), + point -> point + .hasAttributes(Attributes.of(OpenTelemetryAttributes.PULSAR_NAMESPACE, "tenant/ns", + SchemaRegistryStats.REQUEST_TYPE_KEY, "put", + SchemaRegistryStats.RESPONSE_TYPE_KEY, "success")) + .hasCount(1), + point -> point + .hasAttributes(Attributes.of(OpenTelemetryAttributes.PULSAR_NAMESPACE, "tenant/ns", + SchemaRegistryStats.REQUEST_TYPE_KEY, "list", + SchemaRegistryStats.RESPONSE_TYPE_KEY, "success")) + .hasCount(1), + point -> point + .hasAttributes(Attributes.of(OpenTelemetryAttributes.PULSAR_NAMESPACE, "tenant/ns", + SchemaRegistryStats.REQUEST_TYPE_KEY, "get", + SchemaRegistryStats.RESPONSE_TYPE_KEY, "success")) + .hasCount(1) + ))); + ByteArrayOutputStream output = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, false, false, false, output); + PrometheusMetricsTestUtil.generate(pulsar, false, false, false, output); output.flush(); String metricsStr = output.toString(StandardCharsets.UTF_8); - Multimap metrics = PrometheusMetricsTest.parseMetrics(metricsStr); + Multimap metrics = parseMetrics(metricsStr); - Collection delMetrics = metrics.get("pulsar_schema_del_ops_failed_total"); + Collection delMetrics = metrics.get("pulsar_schema_del_ops_failed_total"); Assert.assertEquals(delMetrics.size(), 0); - Collection getMetrics = metrics.get("pulsar_schema_get_ops_failed_total"); + Collection getMetrics = metrics.get("pulsar_schema_get_ops_failed_total"); Assert.assertEquals(getMetrics.size(), 0); - Collection putMetrics = metrics.get("pulsar_schema_put_ops_failed_total"); + Collection putMetrics = metrics.get("pulsar_schema_put_ops_failed_total"); Assert.assertEquals(putMetrics.size(), 0); - Collection deleteLatency = metrics.get("pulsar_schema_del_ops_latency_count"); - for (PrometheusMetricsTest.Metric metric : deleteLatency) { + Collection deleteLatency = metrics.get("pulsar_schema_del_ops_latency_count"); + for (Metric metric : deleteLatency) { Assert.assertEquals(metric.tags.get("namespace"), namespace); Assert.assertTrue(metric.value > 0); } - Collection getLatency = metrics.get("pulsar_schema_get_ops_latency_count"); - for (PrometheusMetricsTest.Metric metric : getLatency) { + Collection getLatency = metrics.get("pulsar_schema_get_ops_latency_count"); + for (Metric metric : getLatency) { Assert.assertEquals(metric.tags.get("namespace"), namespace); Assert.assertTrue(metric.value > 0); } - Collection putLatency = metrics.get("pulsar_schema_put_ops_latency_count"); - for (PrometheusMetricsTest.Metric metric : putLatency) { + Collection putLatency = metrics.get("pulsar_schema_put_ops_latency_count"); + for (Metric metric : putLatency) { Assert.assertEquals(metric.tags.get("namespace"), namespace); Assert.assertTrue(metric.value > 0); } @@ -301,16 +357,39 @@ public void dontReAddExistingSchemaInMiddle() throws Exception { putSchema(schemaId1, schemaData2, version(1)); } - @Test(expectedExceptions = ExecutionException.class) + @Test public void checkIsCompatible() throws Exception { - putSchema(schemaId1, schemaData1, version(0), SchemaCompatibilityStrategy.BACKWARD_TRANSITIVE); - putSchema(schemaId1, schemaData2, version(1), SchemaCompatibilityStrategy.BACKWARD_TRANSITIVE); - - assertTrue(schemaRegistryService.isCompatible(schemaId1, schemaData3, - SchemaCompatibilityStrategy.BACKWARD).get()); - assertFalse(schemaRegistryService.isCompatible(schemaId1, schemaData3, - SchemaCompatibilityStrategy.BACKWARD_TRANSITIVE).get()); - putSchema(schemaId1, schemaData3, version(2), SchemaCompatibilityStrategy.BACKWARD_TRANSITIVE); + var schemaId = BrokerTestUtil.newUniqueName("tenant/ns/topic"); + putSchema(schemaId, schemaData1, version(0), BACKWARD_TRANSITIVE); + putSchema(schemaId, schemaData2, version(1), BACKWARD_TRANSITIVE); + + var timeout = Duration.ofSeconds(1); + assertThat(schemaRegistryService.isCompatible(schemaId, schemaData3, BACKWARD)) + .succeedsWithin(timeout, InstanceOfAssertFactories.BOOLEAN) + .isTrue(); + assertThat(schemaRegistryService.isCompatible(schemaId, schemaData3, BACKWARD_TRANSITIVE)) + .failsWithin(timeout) + .withThrowableOfType(ExecutionException.class) + .withCauseInstanceOf(IncompatibleSchemaException.class); + assertThatThrownBy(() -> putSchema(schemaId, schemaData3, version(2), BACKWARD_TRANSITIVE)) + .isInstanceOf(ExecutionException.class) + .hasCauseInstanceOf(IncompatibleSchemaException.class); + + assertThat(pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics()) + .anySatisfy(metric -> assertThat(metric) + .hasName(SchemaRegistryStats.COMPATIBLE_COUNTER_METRIC_NAME) + .hasLongSumSatisfying( + sum -> sum.hasPointsSatisfying( + point -> point + .hasAttributes(Attributes.of( + OpenTelemetryAttributes.PULSAR_NAMESPACE, "tenant/ns", + SchemaRegistryStats.COMPATIBILITY_CHECK_RESPONSE_KEY, "compatible")) + .hasValue(2), + point -> point + .hasAttributes(Attributes.of( + OpenTelemetryAttributes.PULSAR_NAMESPACE, "tenant/ns", + SchemaRegistryStats.COMPATIBILITY_CHECK_RESPONSE_KEY, "incompatible")) + .hasValue(2)))); } @Test @@ -366,23 +445,44 @@ private void deleteSchema(String schemaId, SchemaVersion expectedVersion) throws assertEquals(expectedVersion, version); } - private SchemaData randomSchema() { - UUID randomString = UUID.randomUUID(); - return SchemaData.builder() - .user(userId) - .type(SchemaType.JSON) - .timestamp(MockClock.millis()) - .isDeleted(false) - .data(randomString.toString().getBytes()) - .props(new TreeMap<>()) - .build(); - } - private static SchemaData getSchemaData(String schemaJson) { - return SchemaData.builder().data(schemaJson.getBytes()).type(SchemaType.AVRO).user(userId).build(); + return SchemaData.builder() + .data(schemaJson.getBytes()) + .type(SchemaType.AVRO) + .user(userId) + .timestamp(MockClock.millis()) + .build(); } private SchemaVersion version(long version) { return new LongSchemaVersion(version); } + + @Test + public void testKeyValueSchema() throws Exception { + final String topicName = "persistent://public/default/testKeyValueSchema"; + admin.topics().createNonPartitionedTopic(BrokerTestUtil.newUniqueName(topicName)); + + final SchemaInfo schemaInfo = KeyValueSchemaInfo.encodeKeyValueSchemaInfo( + "keyValue", + SchemaInfo.builder().type(SchemaType.STRING).schema(new byte[0]) + .build(), + SchemaInfo.builder().type(SchemaType.BOOLEAN).schema(new byte[0]) + .build(), KeyValueEncodingType.SEPARATED); + Assert.assertTrue(admin.schemas().testCompatibility(topicName, schemaInfo).isCompatibility()); + admin.schemas().createSchema(topicName, schemaInfo); + + final IsCompatibilityResponse isCompatibilityResponse = admin.schemas().testCompatibility(topicName, schemaInfo); + Assert.assertTrue(isCompatibilityResponse.isCompatibility()); + + final SchemaInfoWithVersion schemaInfoWithVersion = admin.schemas().getSchemaInfoWithVersion(topicName); + Assert.assertEquals(schemaInfoWithVersion.getVersion(), 0); + + final Long version1 = admin.schemas().getVersionBySchema(topicName, schemaInfo); + Assert.assertEquals(version1, 0); + + final Long version2 = admin.schemas().getVersionBySchema(topicName, schemaInfoWithVersion.getSchemaInfo()); + Assert.assertEquals(version2, 0); + + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/TopicSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/TopicSchemaTest.java new file mode 100644 index 0000000000000..66bfd1c3ec2b0 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/TopicSchemaTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service.schema; + +import static org.apache.pulsar.broker.service.schema.SchemaRegistry.SchemaAndMetadata; +import static org.testng.Assert.assertTrue; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.naming.TopicDomain; +import org.apache.pulsar.common.naming.TopicName; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class TopicSchemaTest extends ProducerConsumerBase { + + @BeforeClass(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @DataProvider(name = "topicDomains") + public Object[][] topicDomains() { + return new Object[][]{ + {TopicDomain.non_persistent}, + {TopicDomain.persistent} + }; + } + + @Test(dataProvider = "topicDomains") + public void testDeleteNonPartitionedTopicWithSchema(TopicDomain topicDomain) throws Exception { + final String topic = BrokerTestUtil.newUniqueName(topicDomain.value() + "://public/default/tp"); + final String schemaId = TopicName.get(TopicName.get(topic).getPartitionedTopicName()).getSchemaName(); + admin.topics().createNonPartitionedTopic(topic); + + // Add schema. + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic) + .enableBatching(false).create(); + producer.close(); + List schemaList1 = pulsar.getSchemaRegistryService().getAllSchemas(schemaId).join() + .stream().map(s -> s.join()).filter(Objects::nonNull).collect(Collectors.toList()); + assertTrue(schemaList1 != null && schemaList1.size() > 0); + + // Verify the schema has been deleted with topic. + admin.topics().delete(topic, false); + List schemaList2 = pulsar.getSchemaRegistryService().getAllSchemas(schemaId).join() + .stream().map(s -> s.join()).filter(Objects::nonNull).collect(Collectors.toList()); + assertTrue(schemaList2 == null || schemaList2.isEmpty()); + } + + @Test + public void testDeletePartitionedTopicWithoutSchema() throws Exception { + // Non-persistent topic does not support partitioned topic now, so only write a test case for persistent topic. + TopicDomain topicDomain = TopicDomain.persistent; + final String topic = BrokerTestUtil.newUniqueName(topicDomain.value() + "://public/default/tp"); + final String partition0 = topic + "-partition-0"; + final String partition1 = topic + "-partition-1"; + final String schemaId = TopicName.get(TopicName.get(topic).getPartitionedTopicName()).getSchemaName(); + admin.topics().createPartitionedTopic(topic, 2); + + // Add schema. + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic) + .enableBatching(false).create(); + producer.close(); + List schemaList1 = pulsar.getSchemaRegistryService().getAllSchemas(schemaId).join() + .stream().map(s -> s.join()).filter(Objects::nonNull).collect(Collectors.toList()); + assertTrue(schemaList1 != null && schemaList1.size() > 0); + + // Verify the schema will not been deleted with partition-0. + admin.topics().delete(partition0, false); + List schemaList2 = pulsar.getSchemaRegistryService().getAllSchemas(schemaId).join() + .stream().map(s -> s.join()).filter(Objects::nonNull).collect(Collectors.toList()); + assertTrue(schemaList2 != null && schemaList2.size() > 0); + + // Verify the schema will not been deleted with partition-0 & partition-1. + admin.topics().delete(partition1, false); + List schemaList3 = pulsar.getSchemaRegistryService().getAllSchemas(schemaId).join() + .stream().map(s -> s.join()).filter(Objects::nonNull).collect(Collectors.toList()); + assertTrue(schemaList3 != null && schemaList3.size() > 0); + + // Verify the schema will be deleted with partitioned metadata. + admin.topics().deletePartitionedTopic(topic, false); + List schemaList4 = pulsar.getSchemaRegistryService().getAllSchemas(schemaId).join() + .stream().map(s -> s.join()).filter(Objects::nonNull).collect(Collectors.toList()); + assertTrue(schemaList4 == null || schemaList4.isEmpty()); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/utils/ClientChannelHelper.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/utils/ClientChannelHelper.java index bf0dd3aa9c1c5..c8fce32efc5f0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/utils/ClientChannelHelper.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/utils/ClientChannelHelper.java @@ -27,6 +27,8 @@ import org.apache.pulsar.common.api.proto.CommandEndTxnResponse; import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespaceResponse; import org.apache.pulsar.common.api.proto.CommandPartitionedTopicMetadataResponse; +import org.apache.pulsar.common.api.proto.CommandPing; +import org.apache.pulsar.common.api.proto.CommandPong; import org.apache.pulsar.common.api.proto.CommandWatchTopicListSuccess; import org.apache.pulsar.common.protocol.PulsarDecoder; import org.apache.pulsar.common.api.proto.CommandAck; @@ -207,6 +209,16 @@ protected void handleEndTxnOnSubscriptionResponse( CommandEndTxnOnSubscriptionResponse commandEndTxnOnSubscriptionResponse) { queue.offer(new CommandEndTxnOnSubscriptionResponse().copyFrom(commandEndTxnOnSubscriptionResponse)); } + + @Override + protected void handlePing(CommandPing ping) { + queue.offer(new CommandPing().copyFrom(ping)); + } + + @Override + protected void handlePong(CommandPong pong) { + return; + } }; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/AuthenticatedConsumerStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/AuthenticatedConsumerStatsTest.java new file mode 100644 index 0000000000000..e8cadb72e1e04 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/AuthenticatedConsumerStatsTest.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Sets; +import io.jsonwebtoken.Jwts; +import io.jsonwebtoken.SignatureAlgorithm; +import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; +import org.apache.pulsar.client.admin.PulsarAdminBuilder; +import org.apache.pulsar.client.api.AuthenticationFactory; +import org.apache.pulsar.client.api.ClientBuilder; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; +import org.apache.pulsar.common.policies.data.ConsumerStats; +import org.apache.pulsar.common.policies.data.TopicStats; +import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.testng.Assert; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.time.Duration; +import java.util.Base64; +import java.util.Date; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Properties; +import java.util.Set; + +public class AuthenticatedConsumerStatsTest extends ConsumerStatsTest{ + private final String ADMIN_TOKEN; + private final String TOKEN_PUBLIC_KEY; + private final KeyPair kp; + + AuthenticatedConsumerStatsTest() throws NoSuchAlgorithmException { + KeyPairGenerator kpg = KeyPairGenerator.getInstance("RSA"); + kp = kpg.generateKeyPair(); + + byte[] encodedPublicKey = kp.getPublic().getEncoded(); + TOKEN_PUBLIC_KEY = "data:;base64," + Base64.getEncoder().encodeToString(encodedPublicKey); + ADMIN_TOKEN = generateToken(kp, "admin"); + } + + + private String generateToken(KeyPair kp, String subject) { + PrivateKey pkey = kp.getPrivate(); + long expMillis = System.currentTimeMillis() + Duration.ofHours(1).toMillis(); + Date exp = new Date(expMillis); + + return Jwts.builder() + .setSubject(subject) + .setExpiration(exp) + .signWith(pkey, SignatureAlgorithm.forSigningKey(pkey)) + .compact(); + } + + @Override + protected void customizeNewPulsarClientBuilder(ClientBuilder clientBuilder) { + clientBuilder.authentication(AuthenticationFactory.token(ADMIN_TOKEN)); + } + + @Override + protected void customizeNewPulsarAdminBuilder(PulsarAdminBuilder pulsarAdminBuilder) { + pulsarAdminBuilder.authentication(AuthenticationFactory.token(ADMIN_TOKEN)); + } + + @BeforeMethod + @Override + protected void setup() throws Exception { + conf.setAuthenticationEnabled(true); + conf.setAuthorizationEnabled(true); + + Set superUserRoles = new HashSet<>(); + superUserRoles.add("admin"); + conf.setSuperUserRoles(superUserRoles); + + Set providers = new HashSet<>(); + providers.add(AuthenticationProviderToken.class.getName()); + conf.setAuthenticationProviders(providers); + conf.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); + conf.setBrokerClientAuthenticationParameters("token:" + ADMIN_TOKEN); + + conf.setClusterName("test"); + + // Set provider domain name + Properties properties = new Properties(); + properties.setProperty("tokenPublicKey", TOKEN_PUBLIC_KEY); + conf.setProperties(properties); + + super.internalSetup(); + super.producerBaseSetup(); + } + + @Test + public void testConsumerStatsOutput() throws Exception { + Set allowedFields = Sets.newHashSet( + "msgRateOut", + "msgThroughputOut", + "bytesOutCounter", + "msgOutCounter", + "messageAckRate", + "msgRateRedeliver", + "chunkedMessageRate", + "consumerName", + "availablePermits", + "unackedMessages", + "avgMessagesPerEntry", + "blockedConsumerOnUnackedMsgs", + "readPositionWhenJoining", + "lastAckedTime", + "lastAckedTimestamp", + "lastConsumedTime", + "lastConsumedTimestamp", + "lastConsumedFlowTimestamp", + "keyHashRanges", + "metadata", + "address", + "connectedSince", + "clientVersion", + "appId"); + + final String topicName = "persistent://public/default/testConsumerStatsOutput"; + final String subName = "my-subscription"; + + Consumer consumer = pulsarClient.newConsumer() + .topic(topicName) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName(subName) + .subscribe(); + + TopicStats stats = admin.topics().getStats(topicName); + ObjectMapper mapper = ObjectMapperFactory.create(); + ConsumerStats consumerStats = stats.getSubscriptions() + .get(subName).getConsumers().get(0); + Assert.assertTrue(consumerStats.getLastConsumedFlowTimestamp() > 0); + JsonNode node = mapper.readTree(mapper.writer().writeValueAsString(consumerStats)); + Iterator itr = node.fieldNames(); + while (itr.hasNext()) { + String field = itr.next(); + Assert.assertTrue(allowedFields.contains(field), field + " should not be exposed"); + } + // assert that role is exposed + Assert.assertEquals(consumerStats.getAppId(), "admin"); + consumer.close(); + } + +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/BrokerOpenTelemetryTestUtil.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/BrokerOpenTelemetryTestUtil.java new file mode 100644 index 0000000000000..d7ad0588201d4 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/BrokerOpenTelemetryTestUtil.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats; + +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdkBuilder; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; +import java.util.Collection; +import java.util.Map; +import java.util.function.Consumer; +import org.apache.pulsar.opentelemetry.OpenTelemetryService; + +public class BrokerOpenTelemetryTestUtil { + // Creates an OpenTelemetrySdkBuilder customizer for use in tests. + public static Consumer getOpenTelemetrySdkBuilderConsumer( + InMemoryMetricReader reader) { + return sdkBuilder -> { + sdkBuilder.addMeterProviderCustomizer( + (meterProviderBuilder, __) -> meterProviderBuilder.registerMetricReader(reader)); + sdkBuilder.addPropertiesSupplier( + () -> Map.of(OpenTelemetryService.OTEL_SDK_DISABLED_KEY, "false", + "otel.java.enabled.resource.providers", "none")); + }; + } + + public static void assertMetricDoubleSumValue(Collection metrics, String metricName, + Attributes attributes, Consumer valueConsumer) { + assertThat(metrics) + .anySatisfy(metric -> assertThat(metric) + .hasName(metricName) + .hasDoubleSumSatisfying(sum -> sum.satisfies( + sumData -> assertThat(sumData.getPoints()).anySatisfy( + point -> { + assertThat(point.getAttributes()).isEqualTo(attributes); + valueConsumer.accept(point.getValue()); + })))); + } + + public static void assertMetricLongSumValue(Collection metrics, String metricName, + Attributes attributes, long expected) { + assertMetricLongSumValue(metrics, metricName, attributes, actual -> assertThat(actual).isEqualTo(expected)); + } + + public static void assertMetricLongSumValue(Collection metrics, String metricName, + Attributes attributes, Consumer valueConsumer) { + assertThat(metrics) + .anySatisfy(metric -> assertThat(metric) + .hasName(metricName) + .hasLongSumSatisfying(sum -> sum.satisfies( + sumData -> assertThat(sumData.getPoints()).anySatisfy( + point -> { + assertThat(point.getAttributes()).isEqualTo(attributes); + valueConsumer.accept(point.getValue()); + })))); + } + + public static void assertMetricLongGaugeValue(Collection metrics, String metricName, + Attributes attributes, long expected) { + assertMetricLongGaugeValue(metrics, metricName, attributes, actual -> assertThat(actual).isEqualTo(expected)); + } + + public static void assertMetricLongGaugeValue(Collection metrics, String metricName, + Attributes attributes, Consumer valueConsumer) { + assertThat(metrics) + .anySatisfy(metric -> assertThat(metric) + .hasName(metricName) + .hasLongGaugeSatisfying(gauge -> gauge.satisfies( + pointData -> assertThat(pointData.getPoints()).anySatisfy( + point -> { + assertThat(point.getAttributes()).isEqualTo(attributes); + valueConsumer.accept(point.getValue()); + })))); + } + + public static void assertMetricDoubleGaugeValue(Collection metrics, String metricName, + Attributes attributes, double expected) { + assertMetricDoubleGaugeValue(metrics, metricName, attributes, actual -> assertThat(actual).isEqualTo(expected)); + } + + public static void assertMetricDoubleGaugeValue(Collection metrics, String metricName, + Attributes attributes, Consumer valueConsumer) { + assertThat(metrics) + .anySatisfy(metric -> assertThat(metric) + .hasName(metricName) + .hasDoubleGaugeSatisfying(gauge -> gauge.satisfies( + pointData -> assertThat(pointData.getPoints()).anySatisfy( + point -> { + assertThat(point.getAttributes()).isEqualTo(attributes); + valueConsumer.accept(point.getValue()); + })))); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ConsumerStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ConsumerStatsTest.java index bbeee9f5a497a..14403765105b9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ConsumerStatsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ConsumerStatsTest.java @@ -19,6 +19,8 @@ package org.apache.pulsar.broker.stats; import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; +import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.Metric; +import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.parseMetrics; import static org.mockito.Mockito.mock; import static org.testng.Assert.assertNotEquals; import static org.testng.AssertJUnit.assertEquals; @@ -43,6 +45,7 @@ import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.PrometheusMetricsTestUtil; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.Topic; @@ -50,7 +53,6 @@ import org.apache.pulsar.broker.service.plugin.EntryFilter; import org.apache.pulsar.broker.service.plugin.EntryFilterProducerTest; import org.apache.pulsar.broker.service.plugin.EntryFilterWithClassLoader; -import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGenerator; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -193,7 +195,7 @@ public void testAckStatsOnPartitionedTopicForExclusiveSubscription() throws Puls @Test public void testUpdateStatsForActiveConsumerAndSubscription() throws Exception { - final String topicName = "persistent://prop/use/ns-abc/testUpdateStatsForActiveConsumerAndSubscription"; + final String topicName = "persistent://public/default/testUpdateStatsForActiveConsumerAndSubscription"; pulsarClient.newConsumer() .topic(topicName) .subscriptionType(SubscriptionType.Shared) @@ -231,7 +233,7 @@ public void testConsumerStatsOutput() throws Exception { "unackedMessages", "avgMessagesPerEntry", "blockedConsumerOnUnackedMsgs", - "readPositionWhenJoining", + "lastSentPositionWhenJoining", "lastAckedTime", "lastAckedTimestamp", "lastConsumedTime", @@ -333,14 +335,14 @@ private void testMessageAckRateMetric(String topicName, boolean exposeTopicLevel consumer2.updateRates(); ByteArrayOutputStream output = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, exposeTopicLevelMetrics, true, true, output); + PrometheusMetricsTestUtil.generate(pulsar, exposeTopicLevelMetrics, true, true, output); String metricStr = output.toString(StandardCharsets.UTF_8); - Multimap metricsMap = PrometheusMetricsTest.parseMetrics(metricStr); - Collection ackRateMetric = metricsMap.get("pulsar_consumer_msg_ack_rate"); + Multimap metricsMap = parseMetrics(metricStr); + Collection ackRateMetric = metricsMap.get("pulsar_consumer_msg_ack_rate"); String rateOutMetricName = exposeTopicLevelMetrics ? "pulsar_consumer_msg_rate_out" : "pulsar_rate_out"; - Collection rateOutMetric = metricsMap.get(rateOutMetricName); + Collection rateOutMetric = metricsMap.get(rateOutMetricName); Assert.assertTrue(ackRateMetric.size() > 0); Assert.assertTrue(rateOutMetric.size() > 0); @@ -407,7 +409,7 @@ public void testAvgMessagesPerEntry() throws Exception { EntryFilter filter = new EntryFilterProducerTest(); EntryFilterWithClassLoader loader = spyWithClassAndConstructorArgs(EntryFilterWithClassLoader.class, filter, - narClassLoader); + narClassLoader, false); Pair> entryFilters = Pair.of("filter", List.of(loader)); PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService() @@ -446,4 +448,37 @@ public void testAvgMessagesPerEntry() throws Exception { int avgMessagesPerEntry = consumerStats.getAvgMessagesPerEntry(); assertEquals(3, avgMessagesPerEntry); } + + @Test() + public void testNonPersistentTopicSharedSubscriptionUnackedMessages() throws Exception { + final String topicName = "non-persistent://my-property/my-ns/my-topic" + UUID.randomUUID(); + final String subName = "my-sub"; + + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topicName) + .create(); + @Cleanup + Consumer consumer = pulsarClient.newConsumer() + .topic(topicName) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Shared) + .subscribe(); + + for (int i = 0; i < 5; i++) { + producer.send(("message-" + i).getBytes()); + } + for (int i = 0; i < 5; i++) { + Message msg = consumer.receive(5, TimeUnit.SECONDS); + consumer.acknowledge(msg); + } + TimeUnit.SECONDS.sleep(1); + + TopicStats topicStats = admin.topics().getStats(topicName); + assertEquals(1, topicStats.getSubscriptions().size()); + List consumers = topicStats.getSubscriptions().get(subName).getConsumers(); + assertEquals(1, consumers.size()); + assertEquals(0, consumers.get(0).getUnackedMessages()); + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedCursorMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedCursorMetricsTest.java index baa4bea570155..8ddb5320588da 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedCursorMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedCursorMetricsTest.java @@ -18,20 +18,24 @@ */ package org.apache.pulsar.broker.stats; +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongSumValue; +import static org.assertj.core.api.Assertions.assertThat; import java.util.ArrayList; import java.util.List; import java.util.UUID; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import lombok.Cleanup; import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedCursorMXBean; -import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.bookkeeper.mledger.ManagedCursorAttributes; +import org.apache.bookkeeper.mledger.impl.OpenTelemetryManagedCursorStats; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.stats.metrics.ManagedCursorMetrics; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.MessageId; @@ -80,6 +84,12 @@ protected PulsarClient createNewPulsarClient(ClientBuilder clientBuilder) throws return PulsarTestClient.create(clientBuilder); } + @Override + protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder pulsarTestContextBuilder) { + super.customizeMainPulsarTestContextBuilder(pulsarTestContextBuilder); + pulsarTestContextBuilder.enableOpenTelemetry(true); + } + /*** * This method has overridden these case: * brk_ml_cursor_persistLedgerSucceed @@ -115,10 +125,7 @@ public void testManagedCursorMetrics() throws Exception { .topic(topicName) .enableBatching(false) .create(); - final PersistentSubscription persistentSubscription = - (PersistentSubscription) pulsar.getBrokerService() - .getTopic(topicName, false).get().get().getSubscription(subName); - final ManagedCursorImpl managedCursor = (ManagedCursorImpl) persistentSubscription.getCursor(); + var managedCursor = getManagedCursor(topicName, subName); ManagedCursorMXBean managedCursorMXBean = managedCursor.getStats(); // Assert. metricsList = metrics.generate(); @@ -128,6 +135,19 @@ public void testManagedCursorMetrics() throws Exception { Assert.assertEquals(metricsList.get(0).getMetrics().get("brk_ml_cursor_persistZookeeperSucceed"), 0L); Assert.assertEquals(metricsList.get(0).getMetrics().get("brk_ml_cursor_persistZookeeperErrors"), 0L); Assert.assertEquals(metricsList.get(0).getMetrics().get("brk_ml_cursor_nonContiguousDeletedMessagesRange"), 0L); + // Validate OpenTelemetry metrics as well + var attributesSet = new ManagedCursorAttributes(managedCursor); + var otelMetrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.PERSIST_OPERATION_COUNTER, + attributesSet.getAttributesOperationSucceed(), 0); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.PERSIST_OPERATION_COUNTER, + attributesSet.getAttributesOperationFailure(), 0); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.PERSIST_OPERATION_METADATA_STORE_COUNTER, + attributesSet.getAttributesOperationSucceed(), 0); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.PERSIST_OPERATION_METADATA_STORE_COUNTER, + attributesSet.getAttributesOperationFailure(), 0); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.NON_CONTIGUOUS_MESSAGE_RANGE_COUNTER, + attributesSet.getAttributes(), 0); /** * 1. Send many messages, and only ack half. After the cursor data is written to BK, * verify "brk_ml_cursor_persistLedgerSucceed" and "brk_ml_cursor_nonContiguousDeletedMessagesRange". @@ -156,6 +176,17 @@ public void testManagedCursorMetrics() throws Exception { Assert.assertEquals(metricsList.get(0).getMetrics().get("brk_ml_cursor_persistZookeeperErrors"), 0L); Assert.assertNotEquals(metricsList.get(0).getMetrics().get("brk_ml_cursor_nonContiguousDeletedMessagesRange"), 0L); + otelMetrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.PERSIST_OPERATION_COUNTER, + attributesSet.getAttributesOperationSucceed(), value -> assertThat(value).isPositive()); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.PERSIST_OPERATION_COUNTER, + attributesSet.getAttributesOperationFailure(), 0); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.PERSIST_OPERATION_METADATA_STORE_COUNTER, + attributesSet.getAttributesOperationSucceed(), 0); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.PERSIST_OPERATION_METADATA_STORE_COUNTER, + attributesSet.getAttributesOperationFailure(), 0); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.NON_CONTIGUOUS_MESSAGE_RANGE_COUNTER, + attributesSet.getAttributes(), value -> assertThat(value).isPositive()); // Ack another half. for (MessageId messageId : keepsMessageIdList){ consumer.acknowledge(messageId); @@ -171,6 +202,17 @@ public void testManagedCursorMetrics() throws Exception { Assert.assertEquals(metricsList.get(0).getMetrics().get("brk_ml_cursor_persistZookeeperSucceed"), 0L); Assert.assertEquals(metricsList.get(0).getMetrics().get("brk_ml_cursor_persistZookeeperErrors"), 0L); Assert.assertEquals(metricsList.get(0).getMetrics().get("brk_ml_cursor_nonContiguousDeletedMessagesRange"), 0L); + otelMetrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.PERSIST_OPERATION_COUNTER, + attributesSet.getAttributesOperationSucceed(), value -> assertThat(value).isPositive()); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.PERSIST_OPERATION_COUNTER, + attributesSet.getAttributesOperationFailure(), 0); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.PERSIST_OPERATION_METADATA_STORE_COUNTER, + attributesSet.getAttributesOperationSucceed(), 0); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.PERSIST_OPERATION_METADATA_STORE_COUNTER, + attributesSet.getAttributesOperationFailure(), 0); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.NON_CONTIGUOUS_MESSAGE_RANGE_COUNTER, + attributesSet.getAttributes(), 0); /** * Make BK error, and send many message, then wait cursor persistent finish. * After the cursor data is written to ZK, verify "brk_ml_cursor_persistLedgerErrors" and @@ -196,6 +238,17 @@ public void testManagedCursorMetrics() throws Exception { Assert.assertNotEquals(metricsList.get(0).getMetrics().get("brk_ml_cursor_persistZookeeperSucceed"), 0L); Assert.assertEquals(metricsList.get(0).getMetrics().get("brk_ml_cursor_persistZookeeperErrors"), 0L); Assert.assertEquals(metricsList.get(0).getMetrics().get("brk_ml_cursor_nonContiguousDeletedMessagesRange"), 0L); + otelMetrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.PERSIST_OPERATION_COUNTER, + attributesSet.getAttributesOperationSucceed(), value -> assertThat(value).isPositive()); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.PERSIST_OPERATION_COUNTER, + attributesSet.getAttributesOperationFailure(), value -> assertThat(value).isPositive()); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.PERSIST_OPERATION_METADATA_STORE_COUNTER, + attributesSet.getAttributesOperationSucceed(), value -> assertThat(value).isPositive()); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.PERSIST_OPERATION_METADATA_STORE_COUNTER, + attributesSet.getAttributesOperationFailure(), 0); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.NON_CONTIGUOUS_MESSAGE_RANGE_COUNTER, + attributesSet.getAttributes(), 0); /** * TODO verify "brk_ml_cursor_persistZookeeperErrors". * This is not easy to implement, we can use {@link #mockZooKeeper} to fail ZK, but we cannot identify whether @@ -210,13 +263,16 @@ public void testManagedCursorMetrics() throws Exception { admin.topics().delete(topicName, true); } - private ManagedCursorMXBean getManagedCursorMXBean(String topicName, String subscriptionName) - throws ExecutionException, InterruptedException { + private ManagedCursorMXBean getManagedCursorMXBean(String topicName, String subscriptionName) throws Exception { + var managedCursor = getManagedCursor(topicName, subscriptionName); + return managedCursor.getStats(); + } + + private ManagedCursor getManagedCursor(String topicName, String subscriptionName) throws Exception { final PersistentSubscription persistentSubscription = (PersistentSubscription) pulsar.getBrokerService() .getTopic(topicName, false).get().get().getSubscription(subscriptionName); - final ManagedCursorImpl managedCursor = (ManagedCursorImpl) persistentSubscription.getCursor(); - return managedCursor.getStats(); + return persistentSubscription.getCursor(); } @Test @@ -265,9 +321,11 @@ public void testCursorReadWriteMetrics() throws Exception { } } + var managedCursor1 = getManagedCursor(topicName, subName1); + var cursorMXBean1 = managedCursor1.getStats(); + var managedCursor2 = getManagedCursor(topicName, subName2); + var cursorMXBean2 = managedCursor2.getStats(); // Wait for persistent cursor meta. - ManagedCursorMXBean cursorMXBean1 = getManagedCursorMXBean(topicName, subName1); - ManagedCursorMXBean cursorMXBean2 = getManagedCursorMXBean(topicName, subName2); Awaitility.await().until(() -> cursorMXBean1.getWriteCursorLedgerLogicalSize() > 0); Awaitility.await().until(() -> cursorMXBean2.getWriteCursorLedgerLogicalSize() > 0); @@ -281,6 +339,22 @@ public void testCursorReadWriteMetrics() throws Exception { Assert.assertNotEquals(metricsList.get(1).getMetrics().get("brk_ml_cursor_writeLedgerLogicalSize"), 0L); Assert.assertEquals(metricsList.get(1).getMetrics().get("brk_ml_cursor_readLedgerSize"), 0L); + var otelMetrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + var attributes1 = new ManagedCursorAttributes(managedCursor1).getAttributes(); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.OUTGOING_BYTE_COUNTER, + attributes1, value -> assertThat(value).isPositive()); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.OUTGOING_BYTE_LOGICAL_COUNTER, + attributes1, value -> assertThat(value).isPositive()); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.INCOMING_BYTE_COUNTER, + attributes1, 0); + + var attributes2 = new ManagedCursorAttributes(managedCursor2).getAttributes(); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.OUTGOING_BYTE_COUNTER, + attributes2, value -> assertThat(value).isPositive()); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.OUTGOING_BYTE_LOGICAL_COUNTER, + attributes2, value -> assertThat(value).isPositive()); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedCursorStats.INCOMING_BYTE_COUNTER, + attributes2, 0); // cleanup. consumer.close(); consumer2.close(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedLedgerMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedLedgerMetricsTest.java index bec73121e487a..d0fd384ba78fb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedLedgerMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedLedgerMetricsTest.java @@ -18,27 +18,38 @@ */ package org.apache.pulsar.broker.stats; +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongSumValue; import static org.apache.pulsar.transaction.coordinator.impl.DisabledTxnLogBufferedWriterMetricsStats.DISABLED_BUFFERED_WRITER_METRICS; +import static org.assertj.core.api.Assertions.assertThat; +import com.google.common.collect.Sets; import io.netty.util.HashedWheelTimer; import io.netty.util.concurrent.DefaultThreadFactory; +import io.opentelemetry.api.common.Attributes; import java.util.List; import java.util.Map.Entry; import java.util.concurrent.TimeUnit; -import com.google.common.collect.Sets; +import lombok.Cleanup; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerMBeanImpl; +import org.apache.bookkeeper.mledger.impl.OpenTelemetryManagedLedgerStats; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.broker.stats.metrics.ManagedLedgerMetrics; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.stats.Metrics; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl; import org.apache.pulsar.transaction.coordinator.impl.TxnLogBufferedWriterConfig; +import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -68,6 +79,12 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + @Override + protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder pulsarTestContextBuilder) { + super.customizeMainPulsarTestContextBuilder(pulsarTestContextBuilder); + pulsarTestContextBuilder.enableOpenTelemetry(true); + } + @Test public void testManagedLedgerMetrics() throws Exception { ManagedLedgerMetrics metrics = new ManagedLedgerMetrics(pulsar); @@ -76,15 +93,20 @@ public void testManagedLedgerMetrics() throws Exception { List list1 = metrics.generate(); Assert.assertTrue(list1.isEmpty()); - Producer producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1") - .create(); + var topicName = "persistent://my-property/use/my-ns/my-topic1"; + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topicName).create(); + + @Cleanup + var consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("sub1").subscribe(); + for (int i = 0; i < 10; i++) { String message = "my-message-" + i; producer.send(message.getBytes()); } - for (Entry ledger : ((ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory()) - .getManagedLedgers().entrySet()) { + var managedLedgerFactory = (ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory(); + for (Entry ledger : managedLedgerFactory.getManagedLedgers().entrySet()) { ManagedLedgerMBeanImpl stats = (ManagedLedgerMBeanImpl) ledger.getValue().getStats(); stats.refreshStats(1, TimeUnit.SECONDS); } @@ -96,14 +118,78 @@ public void testManagedLedgerMetrics() throws Exception { String message = "my-message-" + i; producer.send(message.getBytes()); } - for (Entry ledger : ((ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory()) - .getManagedLedgers().entrySet()) { + for (Entry ledger : managedLedgerFactory.getManagedLedgers().entrySet()) { ManagedLedgerMBeanImpl stats = (ManagedLedgerMBeanImpl) ledger.getValue().getStats(); stats.refreshStats(1, TimeUnit.SECONDS); } List list3 = metrics.generate(); Assert.assertEquals(list3.get(0).getMetrics().get(addEntryRateKey), 5.0D); + // Validate OpenTelemetry metrics. + var ledgers = managedLedgerFactory.getManagedLedgers(); + var topicNameObj = TopicName.get(topicName); + var mlName = topicNameObj.getPersistenceNamingEncoding(); + assertThat(ledgers).containsKey(mlName); + var ml = ledgers.get(mlName); + var attribCommon = Attributes.of( + OpenTelemetryAttributes.ML_NAME, mlName, + OpenTelemetryAttributes.PULSAR_NAMESPACE, topicNameObj.getNamespace() + ); + var metricReader = pulsarTestContext.getOpenTelemetryMetricReader(); + + Awaitility.await().untilAsserted(() -> { + var otelMetrics = metricReader.collectAllMetrics(); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedLedgerStats.BACKLOG_COUNTER, attribCommon, 15); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedLedgerStats.MARK_DELETE_COUNTER, attribCommon, 0); + }); + + for (int i = 0; i < 10; i++) { + var msg = consumer.receive(1, TimeUnit.SECONDS); + consumer.acknowledge(msg); + } + + Awaitility.await().untilAsserted(() -> { + var otelMetrics = metricReader.collectAllMetrics(); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedLedgerStats.BACKLOG_COUNTER, attribCommon, 5); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedLedgerStats.MARK_DELETE_COUNTER, attribCommon, + value -> assertThat(value).isPositive()); + }); + + Awaitility.await().untilAsserted(() -> { + @Cleanup + var cons = pulsarClient.newConsumer() + .topic(topicName) + .subscriptionName(BrokerTestUtil.newUniqueName("sub")) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + cons.receive(1, TimeUnit.SECONDS); + + var attribSucceed = Attributes.of( + OpenTelemetryAttributes.ML_NAME, mlName, + OpenTelemetryAttributes.PULSAR_NAMESPACE, topicNameObj.getNamespace(), + OpenTelemetryAttributes.ML_OPERATION_STATUS, "success" + ); + var attribFailed = Attributes.of( + OpenTelemetryAttributes.ML_NAME, mlName, + OpenTelemetryAttributes.PULSAR_NAMESPACE, topicNameObj.getNamespace(), + OpenTelemetryAttributes.ML_OPERATION_STATUS, "failure" + ); + var otelMetrics = metricReader.collectAllMetrics(); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedLedgerStats.ADD_ENTRY_COUNTER, attribSucceed, 15); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedLedgerStats.ADD_ENTRY_COUNTER, attribFailed, 0); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedLedgerStats.BYTES_OUT_COUNTER, attribCommon, + value -> assertThat(value).isPositive()); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedLedgerStats.BYTES_OUT_WITH_REPLICAS_COUNTER, + attribCommon, value -> assertThat(value).isPositive()); + + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedLedgerStats.READ_ENTRY_COUNTER, attribSucceed, + value -> assertThat(value).isPositive()); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedLedgerStats.READ_ENTRY_COUNTER, attribFailed, 0); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedLedgerStats.BYTES_IN_COUNTER, attribCommon, + value -> assertThat(value).isPositive()); + assertMetricLongSumValue(otelMetrics, OpenTelemetryManagedLedgerStats.READ_ENTRY_CACHE_MISS_COUNTER, + attribCommon, value -> assertThat(value).isPositive()); + }); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java index 8ae0242c6232a..27bdb2e3004ea 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.broker.stats; +import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.Metric; +import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.parseMetrics; import com.google.common.collect.Multimap; import java.io.ByteArrayOutputStream; import java.util.Collection; @@ -28,10 +30,10 @@ import java.util.concurrent.atomic.AtomicInteger; import lombok.Cleanup; import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.PrometheusMetricsTestUtil; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; +import org.apache.pulsar.broker.authentication.metrics.AuthenticationMetricsToken; import org.apache.pulsar.broker.service.BrokerTestBase; -import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGenerator; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Producer; @@ -50,7 +52,7 @@ public class MetadataStoreStatsTest extends BrokerTestBase { @Override protected void setup() throws Exception { super.baseSetup(); - AuthenticationProviderToken.resetMetrics(); + AuthenticationMetricsToken.reset(); } @Override @@ -99,14 +101,14 @@ public void testMetadataStoreStats() throws Exception { } ByteArrayOutputStream output = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, false, false, false, false, output); + PrometheusMetricsTestUtil.generate(pulsar, false, false, false, false, output); String metricsStr = output.toString(); - Multimap metricsMap = PrometheusMetricsTest.parseMetrics(metricsStr); + Multimap metricsMap = parseMetrics(metricsStr); String metricsDebugMessage = "Assertion failed with metrics:\n" + metricsStr + "\n"; - Collection opsLatency = metricsMap.get("pulsar_metadata_store_ops_latency_ms" + "_sum"); - Collection putBytes = metricsMap.get("pulsar_metadata_store_put_bytes" + "_total"); + Collection opsLatency = metricsMap.get("pulsar_metadata_store_ops_latency_ms" + "_sum"); + Collection putBytes = metricsMap.get("pulsar_metadata_store_put_bytes" + "_total"); Assert.assertTrue(opsLatency.size() > 1, metricsDebugMessage); Assert.assertTrue(putBytes.size() > 1, metricsDebugMessage); @@ -116,7 +118,7 @@ public void testMetadataStoreStats() throws Exception { expectedMetadataStoreName.add(MetadataStoreConfig.CONFIGURATION_METADATA_STORE); AtomicInteger matchCount = new AtomicInteger(0); - for (PrometheusMetricsTest.Metric m : opsLatency) { + for (Metric m : opsLatency) { Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage); String metadataStoreName = m.tags.get("name"); if (!isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) { @@ -150,7 +152,7 @@ public void testMetadataStoreStats() throws Exception { Assert.assertEquals(matchCount.get(), expectedMetadataStoreName.size() * 6); matchCount = new AtomicInteger(0); - for (PrometheusMetricsTest.Metric m : putBytes) { + for (Metric m : putBytes) { Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage); String metadataStoreName = m.tags.get("name"); if (!isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) { @@ -189,14 +191,14 @@ public void testBatchMetadataStoreMetrics() throws Exception { } ByteArrayOutputStream output = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, false, false, false, false, output); + PrometheusMetricsTestUtil.generate(pulsar, false, false, false, false, output); String metricsStr = output.toString(); - Multimap metricsMap = PrometheusMetricsTest.parseMetrics(metricsStr); + Multimap metricsMap = parseMetrics(metricsStr); - Collection executorQueueSize = metricsMap.get("pulsar_batch_metadata_store_executor_queue_size"); - Collection opsWaiting = metricsMap.get("pulsar_batch_metadata_store_queue_wait_time_ms" + "_sum"); - Collection batchExecuteTime = metricsMap.get("pulsar_batch_metadata_store_batch_execute_time_ms" + "_sum"); - Collection opsPerBatch = metricsMap.get("pulsar_batch_metadata_store_batch_size" + "_sum"); + Collection executorQueueSize = metricsMap.get("pulsar_batch_metadata_store_executor_queue_size"); + Collection opsWaiting = metricsMap.get("pulsar_batch_metadata_store_queue_wait_time_ms" + "_sum"); + Collection batchExecuteTime = metricsMap.get("pulsar_batch_metadata_store_batch_execute_time_ms" + "_sum"); + Collection opsPerBatch = metricsMap.get("pulsar_batch_metadata_store_batch_size" + "_sum"); String metricsDebugMessage = "Assertion failed with metrics:\n" + metricsStr + "\n"; @@ -210,7 +212,7 @@ public void testBatchMetadataStoreMetrics() throws Exception { expectedMetadataStoreName.add(MetadataStoreConfig.CONFIGURATION_METADATA_STORE); AtomicInteger matchCount = new AtomicInteger(0); - for (PrometheusMetricsTest.Metric m : executorQueueSize) { + for (Metric m : executorQueueSize) { Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage); String metadataStoreName = m.tags.get("name"); if (isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) { @@ -221,7 +223,7 @@ public void testBatchMetadataStoreMetrics() throws Exception { Assert.assertEquals(matchCount.get(), expectedMetadataStoreName.size()); matchCount = new AtomicInteger(0); - for (PrometheusMetricsTest.Metric m : opsWaiting) { + for (Metric m : opsWaiting) { Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage); String metadataStoreName = m.tags.get("name"); if (isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) { @@ -232,7 +234,7 @@ public void testBatchMetadataStoreMetrics() throws Exception { Assert.assertEquals(matchCount.get(), expectedMetadataStoreName.size()); matchCount = new AtomicInteger(0); - for (PrometheusMetricsTest.Metric m : batchExecuteTime) { + for (Metric m : batchExecuteTime) { Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage); String metadataStoreName = m.tags.get("name"); if (isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) { @@ -243,7 +245,7 @@ public void testBatchMetadataStoreMetrics() throws Exception { Assert.assertEquals(matchCount.get(), expectedMetadataStoreName.size()); matchCount = new AtomicInteger(0); - for (PrometheusMetricsTest.Metric m : opsPerBatch) { + for (Metric m : opsPerBatch) { Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage); String metadataStoreName = m.tags.get("name"); if (isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/OpenTelemetryAuthenticationStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/OpenTelemetryAuthenticationStatsTest.java new file mode 100644 index 0000000000000..4cde37b50ffc1 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/OpenTelemetryAuthenticationStatsTest.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats; + +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat; +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongSumValue; +import io.jsonwebtoken.SignatureAlgorithm; +import io.opentelemetry.api.common.Attributes; +import java.time.Duration; +import java.util.Date; +import java.util.Optional; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import javax.crypto.SecretKey; +import javax.naming.AuthenticationException; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationProvider; +import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; +import org.apache.pulsar.broker.authentication.metrics.AuthenticationMetrics; +import org.apache.pulsar.broker.authentication.metrics.AuthenticationMetricsToken; +import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; +import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class OpenTelemetryAuthenticationStatsTest extends BrokerTestBase { + + private static final Duration AUTHENTICATION_TIMEOUT = Duration.ofSeconds(1); + + private SecretKey secretKey; + private AuthenticationProvider provider; + + @BeforeMethod(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.baseSetup(); + + secretKey = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); + provider = new AuthenticationProviderToken(); + registerCloseable(provider); + + var properties = new Properties(); + properties.setProperty("tokenSecretKey", AuthTokenUtils.encodeKeyBase64(secretKey)); + + var conf = new ServiceConfiguration(); + conf.setProperties(properties); + + var authenticationProviderContext = AuthenticationProvider.Context.builder() + .config(conf) + .openTelemetry(pulsar.getOpenTelemetry().getOpenTelemetry()) + .build(); + provider.initialize(authenticationProviderContext); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Override + protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder builder) { + super.customizeMainPulsarTestContextBuilder(builder); + builder.enableOpenTelemetry(true); + } + + @Test + public void testAuthenticationSuccess() { + // Pulsar protocol auth + assertThat(provider.authenticateAsync(new TestAuthenticationDataSource(Optional.empty()))) + .succeedsWithin(AUTHENTICATION_TIMEOUT); + assertMetricLongSumValue(pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(), + AuthenticationMetrics.AUTHENTICATION_COUNTER_METRIC_NAME, + Attributes.of(AuthenticationMetrics.PROVIDER_KEY, "AuthenticationProviderToken", + AuthenticationMetrics.AUTH_RESULT_KEY, "success", + AuthenticationMetrics.AUTH_METHOD_KEY, "token"), + 1); + } + + @Test + public void testTokenDurationHistogram() { + // Token with expiry 15 seconds into the future + var expiryTime = Optional.of(new Date(System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(15))); + assertThat(provider.authenticateAsync(new TestAuthenticationDataSource(expiryTime))) + .succeedsWithin(AUTHENTICATION_TIMEOUT); + // Token without expiry + assertThat(provider.authenticateAsync(new TestAuthenticationDataSource(Optional.empty()))) + .succeedsWithin(AUTHENTICATION_TIMEOUT); + assertThat(pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics()) + .anySatisfy(metric -> assertThat(metric) + .hasName(AuthenticationMetricsToken.EXPIRING_TOKEN_HISTOGRAM_METRIC_NAME) + .hasHistogramSatisfying(histogram -> histogram.hasPointsSatisfying( + histogramPoint -> histogramPoint.hasCount(2).hasMax(Double.POSITIVE_INFINITY)))); + } + + @Test + public void testAuthenticationFailure() { + // Authentication should fail if credentials not passed. + assertThat(provider.authenticateAsync(new AuthenticationDataSource() { })) + .failsWithin(AUTHENTICATION_TIMEOUT) + .withThrowableThat() + .withRootCauseInstanceOf(AuthenticationException.class) + .withMessageContaining("No token credentials passed"); + assertMetricLongSumValue(pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(), + AuthenticationMetrics.AUTHENTICATION_COUNTER_METRIC_NAME, + Attributes.of(AuthenticationMetrics.PROVIDER_KEY, "AuthenticationProviderToken", + AuthenticationMetrics.AUTH_RESULT_KEY, "failure", + AuthenticationMetrics.AUTH_METHOD_KEY, "token", + AuthenticationMetrics.ERROR_CODE_KEY, "INVALID_AUTH_DATA"), + 1); + } + + @Test + public void testTokenExpired() { + var expiredDate = Optional.of(new Date(System.currentTimeMillis() - TimeUnit.HOURS.toMillis(1))); + assertThat(provider.authenticateAsync(new TestAuthenticationDataSource(expiredDate))) + .failsWithin(AUTHENTICATION_TIMEOUT) + .withThrowableThat() + .withRootCauseInstanceOf(AuthenticationException.class) + .withMessageContaining("JWT expired"); + assertMetricLongSumValue(pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(), + AuthenticationMetricsToken.EXPIRED_TOKEN_COUNTER_METRIC_NAME, Attributes.empty(), 1); + } + + private class TestAuthenticationDataSource implements AuthenticationDataSource { + private final String token; + + public TestAuthenticationDataSource(Optional expiryTime) { + token = AuthTokenUtils.createToken(secretKey, "subject", expiryTime); + } + + @Override + public boolean hasDataFromCommand() { + return true; + } + + @Override + public String getCommandData() { + return token; + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/OpenTelemetryBrokerOperabilityStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/OpenTelemetryBrokerOperabilityStatsTest.java new file mode 100644 index 0000000000000..4378e6b05b3ee --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/OpenTelemetryBrokerOperabilityStatsTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats; + +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongSumValue; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.ConnectionCreateStatus; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class OpenTelemetryBrokerOperabilityStatsTest extends BrokerTestBase { + + @BeforeMethod(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.baseSetup(); + } + + @Override + protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder pulsarTestContextBuilder) { + super.customizeMainPulsarTestContextBuilder(pulsarTestContextBuilder); + pulsarTestContextBuilder.enableOpenTelemetry(true); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testBrokerConnection() throws Exception { + var topicName = BrokerTestUtil.newUniqueName("persistent://my-namespace/use/my-ns/testBrokerConnection"); + + @Cleanup + var producer = pulsarClient.newProducer().topic(topicName).create(); + + var metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + assertMetricLongSumValue(metrics, BrokerOperabilityMetrics.CONNECTION_COUNTER_METRIC_NAME, + OpenTelemetryAttributes.ConnectionStatus.OPEN.attributes, 1); + assertMetricLongSumValue(metrics, BrokerOperabilityMetrics.CONNECTION_COUNTER_METRIC_NAME, + OpenTelemetryAttributes.ConnectionStatus.CLOSE.attributes, 0); + assertMetricLongSumValue(metrics, BrokerOperabilityMetrics.CONNECTION_COUNTER_METRIC_NAME, + OpenTelemetryAttributes.ConnectionStatus.ACTIVE.attributes, 1); + + assertMetricLongSumValue(metrics, BrokerOperabilityMetrics.CONNECTION_CREATE_COUNTER_METRIC_NAME, + ConnectionCreateStatus.SUCCESS.attributes, 1); + assertMetricLongSumValue(metrics, BrokerOperabilityMetrics.CONNECTION_CREATE_COUNTER_METRIC_NAME, + ConnectionCreateStatus.FAILURE.attributes, 0); + + pulsarClient.close(); + + metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + assertMetricLongSumValue(metrics, BrokerOperabilityMetrics.CONNECTION_COUNTER_METRIC_NAME, + OpenTelemetryAttributes.ConnectionStatus.CLOSE.attributes, 1); + + pulsar.getConfiguration().setAuthenticationEnabled(true); + + replacePulsarClient(PulsarClient.builder() + .serviceUrl(lookupUrl.toString()) + .operationTimeout(1, TimeUnit.MILLISECONDS)); + assertThatThrownBy(() -> pulsarClient.newProducer().topic(topicName).create()) + .isInstanceOf(PulsarClientException.AuthenticationException.class); + pulsarClient.close(); + + metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + assertMetricLongSumValue(metrics, BrokerOperabilityMetrics.CONNECTION_COUNTER_METRIC_NAME, + OpenTelemetryAttributes.ConnectionStatus.OPEN.attributes, 2); + assertMetricLongSumValue(metrics, BrokerOperabilityMetrics.CONNECTION_COUNTER_METRIC_NAME, + OpenTelemetryAttributes.ConnectionStatus.CLOSE.attributes, 2); + assertMetricLongSumValue(metrics, BrokerOperabilityMetrics.CONNECTION_COUNTER_METRIC_NAME, + OpenTelemetryAttributes.ConnectionStatus.ACTIVE.attributes, 0); + + assertMetricLongSumValue(metrics, BrokerOperabilityMetrics.CONNECTION_CREATE_COUNTER_METRIC_NAME, + ConnectionCreateStatus.SUCCESS.attributes, 1); + assertMetricLongSumValue(metrics, BrokerOperabilityMetrics.CONNECTION_CREATE_COUNTER_METRIC_NAME, + ConnectionCreateStatus.FAILURE.attributes, 1); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/OpenTelemetryConsumerStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/OpenTelemetryConsumerStatsTest.java new file mode 100644 index 0000000000000..a05d7075cf3d7 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/OpenTelemetryConsumerStatsTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats; + +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongSumValue; +import static org.assertj.core.api.Assertions.assertThat; +import io.opentelemetry.api.common.Attributes; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class OpenTelemetryConsumerStatsTest extends BrokerTestBase { + + @BeforeMethod(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.baseSetup(); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Override + protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder builder) { + super.customizeMainPulsarTestContextBuilder(builder); + builder.enableOpenTelemetry(true); + } + + @Test(timeOut = 30_000) + public void testMessagingMetrics() throws Exception { + var topicName = BrokerTestUtil.newUniqueName("persistent://prop/ns-abc/testConsumerMessagingMetrics"); + admin.topics().createNonPartitionedTopic(topicName); + + var messageCount = 5; + var ackCount = 3; + + var subscriptionName = BrokerTestUtil.newUniqueName("test"); + var receiverQueueSize = 100; + + @Cleanup + var consumer = pulsarClient.newConsumer() + .topic(topicName) + .subscriptionName(subscriptionName) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionType(SubscriptionType.Shared) + .ackTimeout(1, TimeUnit.SECONDS) + .receiverQueueSize(receiverQueueSize) + .subscribe(); + + @Cleanup + var producer = pulsarClient.newProducer() + .topic(topicName) + .create(); + for (int i = 0; i < messageCount; i++) { + producer.send(String.format("msg-%d", i).getBytes()); + var message = consumer.receive(); + if (i < ackCount) { + consumer.acknowledge(message); + } + } + + var attributes = Attributes.builder() + .put(OpenTelemetryAttributes.PULSAR_DOMAIN, "persistent") + .put(OpenTelemetryAttributes.PULSAR_TENANT, "prop") + .put(OpenTelemetryAttributes.PULSAR_NAMESPACE, "prop/ns-abc") + .put(OpenTelemetryAttributes.PULSAR_TOPIC, topicName) + .put(OpenTelemetryAttributes.PULSAR_SUBSCRIPTION_NAME, subscriptionName) + .put(OpenTelemetryAttributes.PULSAR_SUBSCRIPTION_TYPE, SubscriptionType.Shared.toString()) + .put(OpenTelemetryAttributes.PULSAR_CONSUMER_NAME, consumer.getConsumerName()) + .put(OpenTelemetryAttributes.PULSAR_CONSUMER_ID, 0) + .build(); + + Awaitility.await().untilAsserted(() -> { + var metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + + assertMetricLongSumValue(metrics, OpenTelemetryConsumerStats.MESSAGE_OUT_COUNTER, attributes, + actual -> assertThat(actual).isPositive()); + assertMetricLongSumValue(metrics, OpenTelemetryConsumerStats.BYTES_OUT_COUNTER, attributes, + actual -> assertThat(actual).isPositive()); + + assertMetricLongSumValue(metrics, OpenTelemetryConsumerStats.MESSAGE_ACK_COUNTER, attributes, ackCount); + assertMetricLongSumValue(metrics, OpenTelemetryConsumerStats.MESSAGE_PERMITS_COUNTER, attributes, + actual -> assertThat(actual).isGreaterThanOrEqualTo(receiverQueueSize - messageCount - ackCount)); + + var unAckCount = messageCount - ackCount; + assertMetricLongSumValue(metrics, OpenTelemetryConsumerStats.MESSAGE_UNACKNOWLEDGED_COUNTER, attributes, + unAckCount); + assertMetricLongSumValue(metrics, OpenTelemetryConsumerStats.CONSUMER_BLOCKED_COUNTER, attributes, 0); + assertMetricLongSumValue(metrics, OpenTelemetryConsumerStats.MESSAGE_REDELIVER_COUNTER, attributes, + actual -> assertThat(actual).isGreaterThanOrEqualTo(unAckCount)); + }); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/OpenTelemetryManagedLedgerCacheStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/OpenTelemetryManagedLedgerCacheStatsTest.java new file mode 100644 index 0000000000000..c3a4a2e054ef3 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/OpenTelemetryManagedLedgerCacheStatsTest.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats; + +import static org.apache.bookkeeper.mledger.OpenTelemetryManagedLedgerCacheStats.CACHE_ENTRY_COUNTER; +import static org.apache.bookkeeper.mledger.OpenTelemetryManagedLedgerCacheStats.CACHE_EVICTION_OPERATION_COUNTER; +import static org.apache.bookkeeper.mledger.OpenTelemetryManagedLedgerCacheStats.CACHE_OPERATION_BYTES_COUNTER; +import static org.apache.bookkeeper.mledger.OpenTelemetryManagedLedgerCacheStats.CACHE_OPERATION_COUNTER; +import static org.apache.bookkeeper.mledger.OpenTelemetryManagedLedgerCacheStats.CACHE_POOL_ACTIVE_ALLOCATION_COUNTER; +import static org.apache.bookkeeper.mledger.OpenTelemetryManagedLedgerCacheStats.CACHE_POOL_ACTIVE_ALLOCATION_SIZE_COUNTER; +import static org.apache.bookkeeper.mledger.OpenTelemetryManagedLedgerCacheStats.CACHE_SIZE_COUNTER; +import static org.apache.bookkeeper.mledger.OpenTelemetryManagedLedgerCacheStats.MANAGED_LEDGER_COUNTER; +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongSumValue; +import static org.assertj.core.api.Assertions.assertThat; +import io.opentelemetry.api.common.Attributes; +import lombok.Cleanup; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.CacheEntryStatus; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.CacheOperationStatus; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.PoolArenaType; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.PoolChunkAllocationType; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class OpenTelemetryManagedLedgerCacheStatsTest extends BrokerTestBase { + + @BeforeMethod(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.baseSetup(); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Override + protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder builder) { + super.customizeMainPulsarTestContextBuilder(builder); + builder.enableOpenTelemetry(true); + } + + @Test + public void testManagedLedgerCacheStats() throws Exception { + var topicName = BrokerTestUtil.newUniqueName("persistent://prop/ns-abc/testManagedLedgerCacheStats"); + + @Cleanup + var producer = pulsarClient.newProducer().topic(topicName).create(); + + @Cleanup + var consumer1 = pulsarClient.newConsumer() + .topic(topicName) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionName(BrokerTestUtil.newUniqueName("sub")) + .subscribe(); + + @Cleanup + var consumer2 = pulsarClient.newConsumer() + .topic(topicName) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionName(BrokerTestUtil.newUniqueName("sub")) + .subscribe(); + + producer.send("test".getBytes()); + consumer1.receive(); + + Awaitility.await().untilAsserted(() -> { + var metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + assertMetricLongSumValue(metrics, CACHE_ENTRY_COUNTER, CacheEntryStatus.ACTIVE.attributes, + value -> assertThat(value).isNotNegative()); + assertMetricLongSumValue(metrics, CACHE_ENTRY_COUNTER, CacheEntryStatus.INSERTED.attributes, + value -> assertThat(value).isPositive()); + assertMetricLongSumValue(metrics, CACHE_ENTRY_COUNTER, CacheEntryStatus.EVICTED.attributes, + value -> assertThat(value).isPositive()); + assertMetricLongSumValue(metrics, CACHE_SIZE_COUNTER, Attributes.empty(), + value -> assertThat(value).isNotNegative()); + }); + + var metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + + assertMetricLongSumValue(metrics, MANAGED_LEDGER_COUNTER, Attributes.empty(), 2); + assertMetricLongSumValue(metrics, CACHE_EVICTION_OPERATION_COUNTER, Attributes.empty(), 0); + + assertMetricLongSumValue(metrics, CACHE_OPERATION_COUNTER, CacheOperationStatus.HIT.attributes, + value -> assertThat(value).isPositive()); + assertMetricLongSumValue(metrics, CACHE_OPERATION_BYTES_COUNTER, CacheOperationStatus.HIT.attributes, + value -> assertThat(value).isPositive()); + assertMetricLongSumValue(metrics, CACHE_OPERATION_COUNTER, CacheOperationStatus.MISS.attributes, + value -> assertThat(value).isNotNegative()); + assertMetricLongSumValue(metrics, CACHE_OPERATION_BYTES_COUNTER, CacheOperationStatus.MISS.attributes, + value -> assertThat(value).isNotNegative()); + + assertMetricLongSumValue(metrics, CACHE_POOL_ACTIVE_ALLOCATION_COUNTER, PoolArenaType.SMALL.attributes, + value -> assertThat(value).isNotNegative()); + assertMetricLongSumValue(metrics, CACHE_POOL_ACTIVE_ALLOCATION_COUNTER, PoolArenaType.NORMAL.attributes, + value -> assertThat(value).isNotNegative()); + assertMetricLongSumValue(metrics, CACHE_POOL_ACTIVE_ALLOCATION_COUNTER, PoolArenaType.HUGE.attributes, + value -> assertThat(value).isNotNegative()); + assertMetricLongSumValue(metrics, CACHE_POOL_ACTIVE_ALLOCATION_SIZE_COUNTER, + PoolChunkAllocationType.ALLOCATED.attributes, value -> assertThat(value).isNotNegative()); + assertMetricLongSumValue(metrics, CACHE_POOL_ACTIVE_ALLOCATION_SIZE_COUNTER, + PoolChunkAllocationType.USED.attributes, value -> assertThat(value).isNotNegative()); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/OpenTelemetryMetadataStoreStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/OpenTelemetryMetadataStoreStatsTest.java new file mode 100644 index 0000000000000..15689fca5d7c0 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/OpenTelemetryMetadataStoreStatsTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats; + +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongSumValue; +import static org.assertj.core.api.Assertions.assertThat; +import io.opentelemetry.api.common.Attributes; +import java.util.concurrent.ExecutorService; +import lombok.Cleanup; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.broker.testcontext.NonClosingProxyHandler; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.apache.pulsar.metadata.api.MetadataStore; +import org.apache.pulsar.metadata.impl.stats.BatchMetadataStoreStats; +import org.apache.pulsar.metadata.impl.stats.MetadataStoreStats; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class OpenTelemetryMetadataStoreStatsTest extends BrokerTestBase { + + @BeforeMethod(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.baseSetup(); + setupDefaultTenantAndNamespace(); + + // In testing conditions, the metadata store gets initialized before Pulsar does, so the OpenTelemetry SDK is + // not yet initialized. Work around this issue by recreating the stats object once we have access to the SDK. + var localMetadataStore = (MetadataStore) NonClosingProxyHandler.getDelegate(pulsar.getLocalMetadataStore()); + var currentStats = (MetadataStoreStats) FieldUtils.readField(localMetadataStore, "metadataStoreStats", true); + var localMetadataStoreName = (String) FieldUtils.readField(currentStats, "metadataStoreName", true); + + currentStats.close(); + var newStats = new MetadataStoreStats( + localMetadataStoreName, pulsar.getOpenTelemetry().getOpenTelemetryService().getOpenTelemetry()); + FieldUtils.writeField(localMetadataStore, "metadataStoreStats", newStats, true); + + var currentBatchedStats = (BatchMetadataStoreStats) FieldUtils.readField(localMetadataStore, "batchMetadataStoreStats", true); + currentBatchedStats.close(); + var currentExecutor = (ExecutorService) FieldUtils.readField(currentBatchedStats, "executor", true); + var newBatchedStats = new BatchMetadataStoreStats( + localMetadataStoreName, currentExecutor, pulsar.getOpenTelemetry().getOpenTelemetryService().getOpenTelemetry()); + FieldUtils.writeField(localMetadataStore, "batchMetadataStoreStats", newBatchedStats, true); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Override + protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder pulsarTestContextBuilder) { + super.customizeMainPulsarTestContextBuilder(pulsarTestContextBuilder); + pulsarTestContextBuilder.enableOpenTelemetry(true); + } + + @Test + public void testMetadataStoreStats() throws Exception { + var topicName = BrokerTestUtil.newUniqueName("persistent://public/default/test-metadata-store-stats"); + + @Cleanup + var producer = pulsarClient.newProducer().topic(topicName).create(); + + producer.newMessage().value("test".getBytes()).send(); + + var attributes = Attributes.of(MetadataStoreStats.METADATA_STORE_NAME, "metadata-store"); + + var metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + assertMetricLongSumValue(metrics, MetadataStoreStats.METADATA_STORE_PUT_BYTES_COUNTER_METRIC_NAME, + attributes, value -> assertThat(value).isPositive()); + assertMetricLongSumValue(metrics, BatchMetadataStoreStats.EXECUTOR_QUEUE_SIZE_METRIC_NAME, attributes, + value -> assertThat(value).isPositive()); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/OpenTelemetryProducerStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/OpenTelemetryProducerStatsTest.java new file mode 100644 index 0000000000000..e273ac4446141 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/OpenTelemetryProducerStatsTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats; + +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongSumValue; +import static org.assertj.core.api.Assertions.assertThat; +import io.opentelemetry.api.common.Attributes; +import lombok.Cleanup; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class OpenTelemetryProducerStatsTest extends BrokerTestBase { + + @BeforeMethod(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.baseSetup(); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Override + protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder builder) { + super.customizeMainPulsarTestContextBuilder(builder); + builder.enableOpenTelemetry(true); + } + + + @Test(timeOut = 30_000) + public void testMessagingMetrics() throws Exception { + var topicName = BrokerTestUtil.newUniqueName("persistent://prop/ns-abc/testProducerMessagingMetrics"); + admin.topics().createNonPartitionedTopic(topicName); + + var messageCount = 5; + var producerName = BrokerTestUtil.newUniqueName("testProducerName"); + + @Cleanup + var producer = pulsarClient.newProducer() + .producerName(producerName) + .topic(topicName) + .create(); + for (int i = 0; i < messageCount; i++) { + producer.send(String.format("msg-%d", i).getBytes()); + } + + var attributes = Attributes.builder() + .put(OpenTelemetryAttributes.PULSAR_DOMAIN, "persistent") + .put(OpenTelemetryAttributes.PULSAR_TENANT, "prop") + .put(OpenTelemetryAttributes.PULSAR_NAMESPACE, "prop/ns-abc") + .put(OpenTelemetryAttributes.PULSAR_TOPIC, topicName) + .put(OpenTelemetryAttributes.PULSAR_PRODUCER_NAME, producerName) + .put(OpenTelemetryAttributes.PULSAR_PRODUCER_ID, 0) + .put(OpenTelemetryAttributes.PULSAR_PRODUCER_ACCESS_MODE, "shared") + .build(); + + var metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + + assertMetricLongSumValue(metrics, OpenTelemetryProducerStats.MESSAGE_IN_COUNTER, attributes, + actual -> assertThat(actual).isPositive()); + assertMetricLongSumValue(metrics, OpenTelemetryProducerStats.BYTES_IN_COUNTER, attributes, + actual -> assertThat(actual).isPositive()); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/OpenTelemetryTopicStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/OpenTelemetryTopicStatsTest.java new file mode 100644 index 0000000000000..c6d07c018c806 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/OpenTelemetryTopicStatsTest.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats; + +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongSumValue; +import static org.assertj.core.api.Assertions.assertThat; +import io.opentelemetry.api.common.Attributes; +import java.util.Objects; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.common.policies.data.PublishRate; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class OpenTelemetryTopicStatsTest extends BrokerTestBase { + + @BeforeMethod(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.baseSetup(); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Override + protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder builder) { + super.customizeMainPulsarTestContextBuilder(builder); + builder.enableOpenTelemetry(true); + } + + @Test(timeOut = 30_000) + public void testMessagingMetrics() throws Exception { + var topicName = BrokerTestUtil.newUniqueName("persistent://prop/ns-abc/testMessagingMetrics"); + admin.topics().createNonPartitionedTopic(topicName); + + var producerCount = 5; + var messagesPerProducer = 2; + var consumerCount = 3; + var messageCount = producerCount * messagesPerProducer; + + for (int i = 0; i < producerCount; i++) { + var producer = registerCloseable(pulsarClient.newProducer().topic(topicName).create()); + for (int j = 0; j < messagesPerProducer; j++) { + producer.send(String.format("producer-%d-msg-%d", i, j).getBytes()); + } + } + + var cdl = new CountDownLatch(consumerCount); + for (int i = 0; i < consumerCount; i++) { + var consumer = registerCloseable(pulsarClient.newConsumer().topic(topicName) + .subscriptionName("test") + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionType(SubscriptionType.Shared) + .subscribe()); + consumer.receiveAsync().orTimeout(100, TimeUnit.MILLISECONDS).handle((__, ex) -> { + cdl.countDown(); + return null; + }); + } + cdl.await(); + + var attributes = Attributes.builder() + .put(OpenTelemetryAttributes.PULSAR_DOMAIN, "persistent") + .put(OpenTelemetryAttributes.PULSAR_TENANT, "prop") + .put(OpenTelemetryAttributes.PULSAR_NAMESPACE, "prop/ns-abc") + .put(OpenTelemetryAttributes.PULSAR_TOPIC, topicName) + .build(); + + var metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.SUBSCRIPTION_COUNTER, attributes, 1); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.PRODUCER_COUNTER, attributes, producerCount); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.CONSUMER_COUNTER, attributes, consumerCount); + + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.MESSAGE_IN_COUNTER, attributes, messageCount); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.MESSAGE_OUT_COUNTER, attributes, messageCount); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.BYTES_IN_COUNTER, attributes, + actual -> assertThat(actual).isPositive()); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.BYTES_OUT_COUNTER, attributes, + actual -> assertThat(actual).isPositive()); + + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.STORAGE_COUNTER, attributes, + actual -> assertThat(actual).isPositive()); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.STORAGE_LOGICAL_COUNTER, attributes, + actual -> assertThat(actual).isPositive()); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.STORAGE_BACKLOG_COUNTER, attributes, + actual -> assertThat(actual).isPositive()); + + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.STORAGE_OUT_COUNTER, attributes, messageCount); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.STORAGE_IN_COUNTER, attributes, messageCount); + } + + @Test(timeOut = 30_000) + public void testPublishRateLimitMetric() throws Exception { + var topicName = BrokerTestUtil.newUniqueName("persistent://prop/ns-abc/testPublishRateLimitMetric"); + admin.topics().createNonPartitionedTopic(topicName); + + var publishRate = new PublishRate(1, -1); + admin.topicPolicies().setPublishRate(topicName, publishRate); + Awaitility.await().until(() -> Objects.equals(publishRate, admin.topicPolicies().getPublishRate(topicName))); + + @Cleanup + var producer = pulsarClient.newProducer().topic(topicName).create(); + producer.send("msg".getBytes()); + + var attributes = Attributes.builder() + .put(OpenTelemetryAttributes.PULSAR_DOMAIN, "persistent") + .put(OpenTelemetryAttributes.PULSAR_TENANT, "prop") + .put(OpenTelemetryAttributes.PULSAR_NAMESPACE, "prop/ns-abc") + .put(OpenTelemetryAttributes.PULSAR_TOPIC, topicName) + .build(); + + var metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.PUBLISH_RATE_LIMIT_HIT_COUNTER, attributes, 1); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java index a7a28afd8ac64..fa073d3694b26 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java @@ -19,22 +19,28 @@ package org.apache.pulsar.broker.stats; import static com.google.common.base.Preconditions.checkArgument; +import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.Metric; +import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.parseMetrics; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -import com.google.common.base.MoreObjects; import com.google.common.base.Splitter; -import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; import io.jsonwebtoken.SignatureAlgorithm; import io.prometheus.client.Collector; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.lang.reflect.Field; +import java.math.BigDecimal; import java.math.RoundingMode; import java.nio.charset.StandardCharsets; -import java.text.NumberFormat; +import java.time.Clock; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -49,9 +55,9 @@ import java.util.Properties; import java.util.Random; import java.util.Set; -import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -62,12 +68,18 @@ import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.PrometheusMetricsTestUtil; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationProvider; import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; +import org.apache.pulsar.broker.authentication.metrics.AuthenticationMetricsToken; import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; +import org.apache.pulsar.broker.loadbalance.extensions.manager.UnloadManager; import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerWrapper; import org.apache.pulsar.broker.service.AbstractTopic; +import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentMessageExpiryMonitor; @@ -80,10 +92,13 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.common.naming.SystemTopicNames; +import org.apache.pulsar.common.policies.data.RetentionPolicies; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.compaction.Compactor; +import org.apache.pulsar.compaction.PulsarCompactionServiceFactory; import org.apache.zookeeper.CreateMode; import org.awaitility.Awaitility; -import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -97,7 +112,7 @@ public class PrometheusMetricsTest extends BrokerTestBase { @Override protected void setup() throws Exception { super.baseSetup(); - AuthenticationProviderToken.resetMetrics(); + AuthenticationMetricsToken.reset(); } @Override @@ -119,25 +134,12 @@ protected void cleanup() throws Exception { @Test public void testPublishRateLimitedTimes() throws Exception { - checkPublishRateLimitedTimes(true); - checkPublishRateLimitedTimes(false); - } - - private void checkPublishRateLimitedTimes(boolean preciseRateLimit) throws Exception { cleanup(); - if (preciseRateLimit) { - conf.setBrokerPublisherThrottlingTickTimeMillis(10000000); - conf.setMaxPublishRatePerTopicInMessages(1); - conf.setMaxPublishRatePerTopicInBytes(1); - conf.setBrokerPublisherThrottlingMaxMessageRate(100000); - conf.setBrokerPublisherThrottlingMaxByteRate(10000000); - } else { - conf.setBrokerPublisherThrottlingTickTimeMillis(1); - conf.setBrokerPublisherThrottlingMaxMessageRate(1); - conf.setBrokerPublisherThrottlingMaxByteRate(1); - } + conf.setMaxPublishRatePerTopicInMessages(1); + conf.setMaxPublishRatePerTopicInBytes(1); + conf.setBrokerPublisherThrottlingMaxMessageRate(100000); + conf.setBrokerPublisherThrottlingMaxByteRate(10000000); conf.setStatsUpdateFrequencyInSecs(100000000); - conf.setPreciseTopicPublishRateLimiterEnable(preciseRateLimit); setup(); String ns1 = "prop/ns-abc1" + UUID.randomUUID(); admin.namespaces().createNamespace(ns1, 1); @@ -166,7 +168,7 @@ private void checkPublishRateLimitedTimes(boolean preciseRateLimit) throws Excep }); @Cleanup ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); assertTrue(metrics.containsKey("pulsar_publish_rate_limit_times")); @@ -179,15 +181,9 @@ private void checkPublishRateLimitedTimes(boolean preciseRateLimit) throws Excep assertEquals(item.value, 1); return; } else if (item.tags.get("topic").equals(topicName3)) { - //When using precise rate limiting, we only trigger the rate limiting of the topic, - // so if the topic is not using the same connection, the rate limiting times will be 0 - //When using asynchronous rate limiting, we will trigger the broker-level rate limiting, - // and all connections will be limited at this time. - if (preciseRateLimit) { - assertEquals(item.value, 0); - } else { - assertEquals(item.value, 1); - } + // We only trigger the rate limiting of the topic, so if the topic is not using + // the same connection, the rate limiting times will be 0 + assertEquals(item.value, 0); return; } fail("should not fail"); @@ -202,7 +198,7 @@ private void checkPublishRateLimitedTimes(boolean preciseRateLimit) throws Excep @Cleanup ByteArrayOutputStream statsOut2 = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut2); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut2); String metricsStr2 = statsOut2.toString(); Multimap metrics2 = parseMetrics(metricsStr2); assertTrue(metrics2.containsKey("pulsar_publish_rate_limit_times")); @@ -217,6 +213,110 @@ private void checkPublishRateLimitedTimes(boolean preciseRateLimit) throws Excep producer3.close(); } + @Test + public void testBrokerMetrics() throws Exception { + cleanup(); + conf.setAdditionalSystemCursorNames(Set.of("test-cursor")); + conf.setTopicLevelPoliciesEnabled(true); + conf.setSystemTopicEnabled(true); + setup(); + + admin.tenants().createTenant("test-tenant", + new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("test"))); + admin.namespaces().createNamespace("test-tenant/test-ns", 4); + Producer p1 = pulsarClient.newProducer().topic("persistent://test-tenant/test-ns/my-topic1").create(); + Producer p2 = pulsarClient.newProducer().topic("persistent://test-tenant/test-ns/my-topic2").create(); + // system topic + Producer p3 = pulsarClient.newProducer().topic("persistent://test-tenant/test-ns/__test-topic").create(); + + Consumer c1 = pulsarClient.newConsumer() + .topic("persistent://test-tenant/test-ns/my-topic1") + .subscriptionName("test") + .subscribe(); + + // additional system cursor + Consumer c2 = pulsarClient.newConsumer() + .topic("persistent://test-tenant/test-ns/my-topic2") + .subscriptionName("test-cursor") + .subscribe(); + + Consumer c3 = pulsarClient.newConsumer() + .topic("persistent://test-tenant/test-ns/__test-topic") + .subscriptionName("test-v1") + .subscribe(); + + final int messages = 10; + for (int i = 0; i < messages; i++) { + String message = "my-message-" + i; + p1.send(message.getBytes()); + p2.send(message.getBytes()); + p3.send(message.getBytes()); + } + + for (int i = 0; i < messages; i++) { + c1.acknowledge(c1.receive()); + c2.acknowledge(c2.receive()); + c3.acknowledge(c3.receive()); + } + + // unsubscribe to test remove cursor impact on metric + c1.unsubscribe(); + c2.unsubscribe(); + + admin.topicPolicies().setRetention("persistent://test-tenant/test-ns/my-topic2", + new RetentionPolicies(60, 1024)); + + ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); + String metricsStr = statsOut.toString(); + Multimap metrics = parseMetrics(metricsStr); + + metrics.entries().forEach(e -> { + System.out.println(e.getKey() + ": " + e.getValue()); + }); + + List bytesOutTotal = (List) metrics.get("pulsar_broker_out_bytes_total"); + List bytesInTotal = (List) metrics.get("pulsar_broker_in_bytes_total"); + List topicLevelBytesOutTotal = (List) metrics.get("pulsar_out_bytes_total"); + + assertEquals(bytesOutTotal.size(), 2); + assertEquals(bytesInTotal.size(), 2); + assertEquals(topicLevelBytesOutTotal.size(), 3); + + double systemOutBytes = 0.0; + double userOutBytes = 0.0; + double systemInBytes = 0.0; + double userInBytes = 0.0; + + for (Metric metric : bytesOutTotal) { + if (metric.tags.get("system_subscription").equals("true")) { + systemOutBytes = metric.value; + } else { + userOutBytes = metric.value; + } + } + + for (Metric metric : bytesInTotal) { + if (metric.tags.get("system_topic").equals("true")) { + systemInBytes = metric.value; + } else { + userInBytes = metric.value; + } + } + + double systemCursorOutBytes = 0.0; + for (Metric metric : topicLevelBytesOutTotal) { + if (metric.tags.get("subscription").startsWith(SystemTopicNames.SYSTEM_READER_PREFIX) + || metric.tags.get("subscription").equals(Compactor.COMPACTION_SUBSCRIPTION)) { + systemCursorOutBytes = metric.value; + } + } + + assertEquals(systemCursorOutBytes, systemInBytes); + assertEquals(userOutBytes / 2, systemOutBytes - systemCursorOutBytes); + assertEquals(userOutBytes + systemOutBytes, userInBytes + systemInBytes); + } + @Test public void testMetricsTopicCount() throws Exception { String ns1 = "prop/ns-abc1"; @@ -234,7 +334,7 @@ public void testMetricsTopicCount() throws Exception { Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT); @Cleanup ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); Collection metric = metrics.get("pulsar_topics_count"); @@ -246,6 +346,14 @@ public void testMetricsTopicCount() throws Exception { assertEquals(item.value, 3.0); } }); + Collection pulsarTopicLoadTimesMetrics = metrics.get("pulsar_topic_load_times"); + Collection pulsarTopicLoadTimesCountMetrics = metrics.get("pulsar_topic_load_times_count"); + assertEquals(pulsarTopicLoadTimesMetrics.size(), 6); + assertEquals(pulsarTopicLoadTimesCountMetrics.size(), 1); + Collection topicLoadTimeP999Metrics = metrics.get("pulsar_topic_load_time_99_9_percentile_ms"); + Collection topicLoadTimeFailedCountMetrics = metrics.get("pulsar_topic_load_failed_count"); + assertEquals(topicLoadTimeP999Metrics.size(), 1); + assertEquals(topicLoadTimeFailedCountMetrics.size(), 1); } @Test @@ -263,7 +371,7 @@ public void testMetricsAvgMsgSize2() throws Exception { producerInServer.getStats().msgThroughputIn = 100; @Cleanup ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, true, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, true, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); assertTrue(metrics.containsKey("pulsar_average_msg_size")); @@ -306,7 +414,7 @@ public void testPerTopicStats() throws Exception { } ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); @@ -317,51 +425,55 @@ public void testPerTopicStats() throws Exception { // There should be 2 metrics with different tags for each topic List cm = (List) metrics.get("pulsar_storage_write_latency_le_1"); assertEquals(cm.size(), 2); - assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); + assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns"); - assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); + assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns"); cm = (List) metrics.get("pulsar_producers_count"); assertEquals(cm.size(), 2); - assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); - assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns"); - assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); + assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); + assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns"); + assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns"); - cm = (List) metrics.get("topic_load_times_count"); + cm = (List) metrics.get("pulsar_topic_load_times_count"); + assertEquals(cm.size(), 1); + assertEquals(cm.get(0).tags.get("cluster"), "test"); + + cm = (List) metrics.get("topic_load_failed_total"); assertEquals(cm.size(), 1); assertEquals(cm.get(0).tags.get("cluster"), "test"); cm = (List) metrics.get("pulsar_in_bytes_total"); assertEquals(cm.size(), 2); - assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); + assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns"); - assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); + assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns"); cm = (List) metrics.get("pulsar_in_messages_total"); assertEquals(cm.size(), 2); - assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); + assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns"); - assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); + assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns"); cm = (List) metrics.get("pulsar_out_bytes_total"); assertEquals(cm.size(), 2); - assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); + assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns"); assertEquals(cm.get(0).tags.get("subscription"), "test"); - assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); + assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns"); assertEquals(cm.get(1).tags.get("subscription"), "test"); cm = (List) metrics.get("pulsar_out_messages_total"); assertEquals(cm.size(), 2); - assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); + assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns"); assertEquals(cm.get(0).tags.get("subscription"), "test"); - assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); + assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns"); assertEquals(cm.get(1).tags.get("subscription"), "test"); @@ -400,7 +512,7 @@ public void testPerBrokerStats() throws Exception { } ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); @@ -509,7 +621,7 @@ public void testPerTopicStatsReconnect() throws Exception { c2.close(); ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); @@ -587,7 +699,7 @@ public void testStorageReadCacheMissesRate(boolean cacheEnable) throws Exception // includeTopicMetric true ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); @@ -619,7 +731,7 @@ public void testStorageReadCacheMissesRate(boolean cacheEnable) throws Exception // includeTopicMetric false ByteArrayOutputStream statsOut2 = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut2); + PrometheusMetricsTestUtil.generate(pulsar, false, false, false, statsOut2); String metricsStr2 = statsOut2.toString(); Multimap metrics2 = parseMetrics(metricsStr2); @@ -703,7 +815,7 @@ public void testPerTopicExpiredStat() throws Exception { Awaitility.await().until(() -> sub2.getExpiredMessageRate() != 0.0); ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); // There should be 2 metrics with different tags for each topic @@ -732,14 +844,13 @@ public void testPerTopicExpiredStat() throws Exception { //check value field = PersistentSubscription.class.getDeclaredField("expiryMonitor"); field.setAccessible(true); - NumberFormat nf = NumberFormat.getNumberInstance(); - nf.setMaximumFractionDigits(3); - nf.setRoundingMode(RoundingMode.DOWN); for (int i = 0; i < topicList.size(); i++) { PersistentSubscription subscription = (PersistentSubscription) pulsar.getBrokerService() .getTopicIfExists(topicList.get(i)).get().get().getSubscription(subName); PersistentMessageExpiryMonitor monitor = (PersistentMessageExpiryMonitor) field.get(subscription); - assertEquals(Double.valueOf(nf.format(monitor.getMessageExpiryRate())).doubleValue(), cm.get(i).value); + BigDecimal bigDecimal = BigDecimal.valueOf(monitor.getMessageExpiryRate()); + bigDecimal = bigDecimal.setScale(3, RoundingMode.DOWN); + assertEquals(bigDecimal.doubleValue(), cm.get(i).value); } cm = (List) metrics.get("pulsar_subscription_total_msg_expired"); @@ -779,12 +890,22 @@ public void testBundlesMetrics() throws Exception { mockZooKeeper.create(mockedBroker, new byte[]{0}, Collections.emptyList(), CreateMode.EPHEMERAL); pulsar.getBrokerService().updateRates(); - Awaitility.await().untilAsserted(() -> assertTrue(pulsar.getBrokerService().getBundleStats().size() > 0)); + Awaitility.await().until(() -> !pulsar.getBrokerService().getBundleStats().isEmpty()); ModularLoadManagerWrapper loadManager = (ModularLoadManagerWrapper)pulsar.getLoadManager().get(); loadManager.getLoadManager().updateLocalBrokerData(); + // Force registration of UnloadManager load balance stats + for (var latencyMetric : UnloadManager.LatencyMetric.values()) { + var serviceUnit = "serviceUnit"; + var brokerLookupAddress = "lookupAddress"; + var serviceUnitStateData = mock(ServiceUnitStateData.class); + when(serviceUnitStateData.sourceBroker()).thenReturn(brokerLookupAddress); + when(serviceUnitStateData.dstBroker()).thenReturn(brokerLookupAddress); + latencyMetric.beginMeasurement(serviceUnit, brokerLookupAddress, serviceUnitStateData); + latencyMetric.endMeasurement(serviceUnit); + } ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, false, false, false, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); assertTrue(metrics.containsKey("pulsar_bundle_msg_rate_in")); @@ -803,6 +924,11 @@ public void testBundlesMetrics() throws Exception { assertTrue(metrics.containsKey("pulsar_lb_bundles_split_total")); + assertTrue(metrics.containsKey("brk_lb_unload_latency_ms_bucket")); + assertTrue(metrics.containsKey("brk_lb_release_latency_ms_bucket")); + assertTrue(metrics.containsKey("brk_lb_assign_latency_ms_bucket")); + assertTrue(metrics.containsKey("brk_lb_disconnect_latency_ms_bucket")); + // cleanup. mockZooKeeper.delete(mockedBroker, 0); } @@ -829,7 +955,7 @@ public void testNonPersistentSubMetrics() throws Exception { } ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); assertTrue(metrics.containsKey("pulsar_subscription_back_log")); @@ -876,7 +1002,7 @@ public void testPerNamespaceStats() throws Exception { } ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, false, false, false, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); @@ -949,7 +1075,7 @@ public void testPerProducerStats() throws Exception { } ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, true, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, true, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); @@ -961,26 +1087,26 @@ public void testPerProducerStats() throws Exception { List cm = (List) metrics.get("pulsar_producer_msg_rate_in"); assertEquals(cm.size(), 2); assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns"); - assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); - assertEquals(cm.get(0).tags.get("producer_name"), "producer2"); - assertEquals(cm.get(0).tags.get("producer_id"), "1"); + assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); + assertEquals(cm.get(0).tags.get("producer_name"), "producer1"); + assertEquals(cm.get(0).tags.get("producer_id"), "0"); assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns"); - assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); - assertEquals(cm.get(1).tags.get("producer_name"), "producer1"); - assertEquals(cm.get(1).tags.get("producer_id"), "0"); + assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); + assertEquals(cm.get(1).tags.get("producer_name"), "producer2"); + assertEquals(cm.get(1).tags.get("producer_id"), "1"); cm = (List) metrics.get("pulsar_producer_msg_throughput_in"); assertEquals(cm.size(), 2); assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns"); - assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); - assertEquals(cm.get(0).tags.get("producer_name"), "producer2"); - assertEquals(cm.get(0).tags.get("producer_id"), "1"); + assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); + assertEquals(cm.get(0).tags.get("producer_name"), "producer1"); + assertEquals(cm.get(0).tags.get("producer_id"), "0"); assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns"); - assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); - assertEquals(cm.get(1).tags.get("producer_name"), "producer1"); - assertEquals(cm.get(1).tags.get("producer_id"), "0"); + assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); + assertEquals(cm.get(1).tags.get("producer_name"), "producer2"); + assertEquals(cm.get(1).tags.get("producer_id"), "1"); p1.close(); p2.close(); @@ -1017,7 +1143,7 @@ public void testPerConsumerStats() throws Exception { } ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, true, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, true, false, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); @@ -1030,42 +1156,42 @@ public void testPerConsumerStats() throws Exception { List cm = (List) metrics.get("pulsar_out_bytes_total"); assertEquals(cm.size(), 4); assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns"); - assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); + assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); assertEquals(cm.get(0).tags.get("subscription"), "test"); assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns"); - assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); + assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); assertEquals(cm.get(1).tags.get("subscription"), "test"); - assertEquals(cm.get(1).tags.get("consumer_id"), "1"); + assertEquals(cm.get(1).tags.get("consumer_id"), "0"); assertEquals(cm.get(2).tags.get("namespace"), "my-property/use/my-ns"); - assertEquals(cm.get(2).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); + assertEquals(cm.get(2).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); assertEquals(cm.get(2).tags.get("subscription"), "test"); assertEquals(cm.get(3).tags.get("namespace"), "my-property/use/my-ns"); - assertEquals(cm.get(3).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); + assertEquals(cm.get(3).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); assertEquals(cm.get(3).tags.get("subscription"), "test"); - assertEquals(cm.get(3).tags.get("consumer_id"), "0"); + assertEquals(cm.get(3).tags.get("consumer_id"), "1"); cm = (List) metrics.get("pulsar_out_messages_total"); assertEquals(cm.size(), 4); assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns"); - assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); + assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); assertEquals(cm.get(0).tags.get("subscription"), "test"); assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns"); - assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); + assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); assertEquals(cm.get(1).tags.get("subscription"), "test"); - assertEquals(cm.get(1).tags.get("consumer_id"), "1"); + assertEquals(cm.get(1).tags.get("consumer_id"), "0"); assertEquals(cm.get(2).tags.get("namespace"), "my-property/use/my-ns"); - assertEquals(cm.get(2).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); + assertEquals(cm.get(2).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); assertEquals(cm.get(2).tags.get("subscription"), "test"); assertEquals(cm.get(3).tags.get("namespace"), "my-property/use/my-ns"); - assertEquals(cm.get(3).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1"); + assertEquals(cm.get(3).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2"); assertEquals(cm.get(3).tags.get("subscription"), "test"); - assertEquals(cm.get(3).tags.get("consumer_id"), "0"); + assertEquals(cm.get(3).tags.get("consumer_id"), "1"); p1.close(); p2.close(); @@ -1104,7 +1230,7 @@ public void testDuplicateMetricTypeDefinitions() throws Exception { } ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, false, false, false, statsOut); String metricsStr = statsOut.toString(); Map typeDefs = new HashMap<>(); @@ -1208,7 +1334,7 @@ public void testManagedLedgerCacheStats() throws Exception { } ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, false, false, false, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); @@ -1244,7 +1370,7 @@ public void testManagedLedgerStats() throws Exception { } ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, false, false, false, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); @@ -1322,7 +1448,7 @@ public void testManagedLedgerBookieClientStats() throws Exception { } ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, false, false, false, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); @@ -1377,7 +1503,7 @@ public void testAuthMetrics() throws IOException, AuthenticationException { ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); try { provider.authenticate(new AuthenticationDataSource() { @@ -1403,7 +1529,7 @@ public String getCommandData() { }); ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, false, false, false, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); List cm = (List) metrics.get("pulsar_authentication_success_total"); @@ -1441,7 +1567,7 @@ public void testExpiredTokenMetrics() throws Exception { ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); Date expiredDate = new Date(System.currentTimeMillis() - TimeUnit.HOURS.toMillis(1)); String expiredToken = AuthTokenUtils.createToken(secretKey, "subject", Optional.of(expiredDate)); @@ -1464,7 +1590,7 @@ public String getCommandData() { } ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, false, false, false, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); List cm = (List) metrics.get("pulsar_expired_token_total"); @@ -1477,6 +1603,7 @@ public String getCommandData() { public void testExpiringTokenMetrics() throws Exception { SecretKey secretKey = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); + @Cleanup AuthenticationProviderToken provider = new AuthenticationProviderToken(); Properties properties = new Properties(); @@ -1484,7 +1611,7 @@ public void testExpiringTokenMetrics() throws Exception { ServiceConfiguration conf = new ServiceConfiguration(); conf.setProperties(properties); - provider.initialize(conf); + provider.initialize(AuthenticationProvider.Context.builder().config(conf).build()); int[] tokenRemainTime = new int[]{3, 7, 40, 100, 400}; @@ -1505,33 +1632,25 @@ public String getCommandData() { } ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, false, false, false, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); Metric countMetric = ((List) metrics.get("pulsar_expiring_token_minutes_count")).get(0); assertEquals(countMetric.value, tokenRemainTime.length); List cm = (List) metrics.get("pulsar_expiring_token_minutes_bucket"); - assertEquals(cm.size(), 5); + var buckets = cm.stream().map(m -> m.tags.get("le")).collect(Collectors.toSet()); + assertThat(buckets).isEqualTo(Set.of("5.0", "10.0", "60.0", "240.0", "1440.0", "10080.0", "20160.0", "43200.0", + "129600.0", "259200.0", "388800.0", "525600.0", "+Inf")); cm.forEach((e) -> { - switch (e.tags.get("le")) { - case "5.0": - assertEquals(e.value, 1); - break; - case "10.0": - assertEquals(e.value, 2); - break; - case "60.0": - assertEquals(e.value, 3); - break; - case "240.0": - assertEquals(e.value, 4); - break; - default: - assertEquals(e.value, 5); - break; - } + var expectedValue = switch(e.tags.get("le")) { + case "5.0" -> 1; + case "10.0" -> 2; + case "60.0" -> 3; + case "240.0" -> 4; + default -> 5; + }; + assertEquals(e.value, expectedValue); }); - provider.close(); } @Test @@ -1579,7 +1698,7 @@ public void testManagedCursorPersistStats() throws Exception { // enable ExposeManagedCursorMetricsInPrometheus pulsar.getConfiguration().setExposeManagedCursorMetricsInPrometheus(true); ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); @@ -1592,7 +1711,7 @@ public void testManagedCursorPersistStats() throws Exception { // disable ExposeManagedCursorMetricsInPrometheus pulsar.getConfiguration().setExposeManagedCursorMetricsInPrometheus(false); ByteArrayOutputStream statsOut2 = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut2); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut2); String metricsStr2 = statsOut2.toString(); Multimap metrics2 = parseMetrics(metricsStr2); List cm2 = (List) metrics2.get("pulsar_ml_cursor_persistLedgerSucceed"); @@ -1611,7 +1730,7 @@ public void testBrokerConnection() throws Exception { .create(); ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); List cm = (List) metrics.get("pulsar_connection_created_total_count"); @@ -1628,7 +1747,7 @@ public void testBrokerConnection() throws Exception { pulsarClient.close(); statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); metricsStr = statsOut.toString(); metrics = parseMetrics(metricsStr); @@ -1651,7 +1770,7 @@ public void testBrokerConnection() throws Exception { pulsarClient.close(); statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); metricsStr = statsOut.toString(); metrics = parseMetrics(metricsStr); @@ -1671,6 +1790,20 @@ public void testBrokerConnection() throws Exception { compareBrokerConnectionStateCount(cm, 2.0); } + @Test + public void testBrokerHealthCheckMetric() throws Exception { + conf.setHealthCheckMetricsUpdateTimeInSeconds(60); + BrokerService brokerService = pulsar.getBrokerService(); + brokerService.checkHealth().get(); + brokerService.updateRates(); + ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); + String metricsStr = statsOut.toString(); + Multimap metrics = parseMetrics(metricsStr); + List cm = (List) metrics.get("pulsar_health"); + compareBrokerConnectionStateCount(cm, 1); + } + private void compareBrokerConnectionStateCount(List cm, double count) { assertEquals(cm.size(), 1); assertEquals(cm.get(0).tags.get("cluster"), "test"); @@ -1695,7 +1828,7 @@ public void testCompaction() throws Exception { .messageRoutingMode(MessageRoutingMode.SinglePartition) .create(); ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); List cm = (List) metrics.get("pulsar_compaction_removed_event_count"); @@ -1727,10 +1860,10 @@ public void testCompaction() throws Exception { .value(data) .send(); } - Compactor compactor = pulsar.getCompactor(); + Compactor compactor = ((PulsarCompactionServiceFactory)pulsar.getCompactionServiceFactory()).getCompactor(); compactor.compact(topicName).get(); statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); metricsStr = statsOut.toString(); metrics = parseMetrics(metricsStr); cm = (List) metrics.get("pulsar_compaction_removed_event_count"); @@ -1763,31 +1896,34 @@ public void testCompaction() throws Exception { @Test public void testMetricsWithCache() throws Throwable { - ServiceConfiguration configuration = Mockito.mock(ServiceConfiguration.class); - Mockito.when(configuration.getManagedLedgerStatsPeriodSeconds()).thenReturn(2); - Mockito.when(configuration.isMetricsBufferResponse()).thenReturn(true); - Mockito.when(configuration.getClusterName()).thenReturn(configClusterName); - Mockito.when(pulsar.getConfiguration()).thenReturn(configuration); - - int period = pulsar.getConfiguration().getManagedLedgerStatsPeriodSeconds(); - TimeWindow timeWindow = new TimeWindow<>(2, (int) TimeUnit.SECONDS.toMillis(period)); - + ServiceConfiguration configuration = pulsar.getConfiguration(); + configuration.setManagedLedgerStatsPeriodSeconds(2); + configuration.setMetricsBufferResponse(true); + configuration.setClusterName(configClusterName); + + // create a mock clock to control the time + AtomicLong currentTimeMillis = new AtomicLong(System.currentTimeMillis()); + Clock clock = mock(); + when(clock.millis()).thenAnswer(invocation -> currentTimeMillis.get()); + + PrometheusMetricsGenerator prometheusMetricsGenerator = + new PrometheusMetricsGenerator(pulsar, true, false, false, + false, clock); + String previousMetrics = null; for (int a = 0; a < 4; a++) { - long start = System.currentTimeMillis(); ByteArrayOutputStream statsOut1 = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, false, statsOut1, null); + PrometheusMetricsTestUtil.generate(prometheusMetricsGenerator, statsOut1, null); ByteArrayOutputStream statsOut2 = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, false, statsOut2, null); - long end = System.currentTimeMillis(); - - if (timeWindow.currentWindowStart(start) == timeWindow.currentWindowStart(end)) { - String metricsStr1 = statsOut1.toString(); - String metricsStr2 = statsOut2.toString(); - assertEquals(metricsStr1, metricsStr2); - Multimap metrics = parseMetrics(metricsStr1); - } - - Thread.sleep(TimeUnit.SECONDS.toMillis(period / 2)); + PrometheusMetricsTestUtil.generate(prometheusMetricsGenerator, statsOut2, null); + + String metricsStr1 = statsOut1.toString(); + String metricsStr2 = statsOut2.toString(); + assertTrue(metricsStr1.length() > 1000); + assertEquals(metricsStr1, metricsStr2); + assertNotEquals(metricsStr1, previousMetrics); + previousMetrics = metricsStr1; + // move time forward + currentTimeMillis.addAndGet(TimeUnit.SECONDS.toMillis(2)); } } @@ -1815,7 +1951,7 @@ public void testSplitTopicAndPartitionLabel() throws Exception { .subscribe(); @Cleanup ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, true, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, true, statsOut); String metricsStr = statsOut.toString(); Multimap metrics = parseMetrics(metricsStr); Collection metric = metrics.get("pulsar_consumers_count"); @@ -1833,13 +1969,6 @@ public void testSplitTopicAndPartitionLabel() throws Exception { consumer2.close(); } - private void compareCompactionStateCount(List cm, double count) { - assertEquals(cm.size(), 1); - assertEquals(cm.get(0).tags.get("cluster"), "test"); - assertEquals(cm.get(0).tags.get("broker"), "localhost"); - assertEquals(cm.get(0).value, count); - } - @Test public void testMetricsGroupedByTypeDefinitions() throws Exception { Producer p1 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create(); @@ -1851,7 +1980,7 @@ public void testMetricsGroupedByTypeDefinitions() throws Exception { } ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, false, false, false, statsOut); String metricsStr = statsOut.toString(); Pattern typePattern = Pattern.compile("^#\\s+TYPE\\s+(\\w+)\\s+(\\w+)"); @@ -1897,62 +2026,6 @@ public void testMetricsGroupedByTypeDefinitions() throws Exception { p2.close(); } - /** - * Hacky parsing of Prometheus text format. Should be good enough for unit tests - */ - public static Multimap parseMetrics(String metrics) { - Multimap parsed = ArrayListMultimap.create(); - - // Example of lines are - // jvm_threads_current{cluster="standalone",} 203.0 - // or - // pulsar_subscriptions_count{cluster="standalone", namespace="public/default", - // topic="persistent://public/default/test-2"} 0.0 - Pattern pattern = Pattern.compile("^(\\w+)\\{([^\\}]+)\\}\\s([+-]?[\\d\\w\\.-]+)$"); - Pattern tagsPattern = Pattern.compile("(\\w+)=\"([^\"]+)\"(,\\s?)?"); - - Splitter.on("\n").split(metrics).forEach(line -> { - if (line.isEmpty() || line.startsWith("#")) { - return; - } - - Matcher matcher = pattern.matcher(line); - assertTrue(matcher.matches(), "line " + line + " does not match pattern " + pattern); - String name = matcher.group(1); - - Metric m = new Metric(); - String numericValue = matcher.group(3); - if (numericValue.equalsIgnoreCase("-Inf")) { - m.value = Double.NEGATIVE_INFINITY; - } else if (numericValue.equalsIgnoreCase("+Inf")) { - m.value = Double.POSITIVE_INFINITY; - } else { - m.value = Double.parseDouble(numericValue); - } - String tags = matcher.group(2); - Matcher tagsMatcher = tagsPattern.matcher(tags); - while (tagsMatcher.find()) { - String tag = tagsMatcher.group(1); - String value = tagsMatcher.group(2); - m.tags.put(tag, value); - } - - parsed.put(name, m); - }); - - return parsed; - } - - public static class Metric { - public Map tags = new TreeMap<>(); - public double value; - - @Override - public String toString() { - return MoreObjects.toStringHelper(this).add("tags", tags).add("value", value).toString(); - } - } - @Test public void testEscapeLabelValue() throws Exception { String ns1 = "prop/ns-abc1"; @@ -1967,7 +2040,7 @@ public void testEscapeLabelValue() throws Exception { .subscribe(); @Cleanup ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); String metricsStr = statsOut.toString(); final List subCountLines = metricsStr.lines() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java index d5e0066a86f15..bc4cb73e5b6fe 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java @@ -19,6 +19,8 @@ package org.apache.pulsar.broker.stats; import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; +import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.Metric; +import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.parseMetrics; import static org.mockito.Mockito.mock; import com.google.common.collect.Multimap; import java.io.ByteArrayOutputStream; @@ -29,13 +31,13 @@ import java.util.concurrent.TimeUnit; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.PrometheusMetricsTestUtil; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.Dispatcher; import org.apache.pulsar.broker.service.EntryFilterSupport; import org.apache.pulsar.broker.service.plugin.EntryFilter; import org.apache.pulsar.broker.service.plugin.EntryFilterTest; import org.apache.pulsar.broker.service.plugin.EntryFilterWithClassLoader; -import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGenerator; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -84,7 +86,7 @@ protected void cleanup() throws Exception { @Test public void testConsumersAfterMarkDelete() throws PulsarClientException, PulsarAdminException { final String topicName = "persistent://my-property/my-ns/testConsumersAfterMarkDelete-" - + UUID.randomUUID().toString(); + + UUID.randomUUID(); final String subName = "my-sub"; Consumer consumer1 = pulsarClient.newConsumer() @@ -206,7 +208,7 @@ public void testSubscriptionStats(final String topic, final String subName, bool NarClassLoader narClassLoader = mock(NarClassLoader.class); EntryFilter filter1 = new EntryFilterTest(); EntryFilterWithClassLoader loader1 = - spyWithClassAndConstructorArgs(EntryFilterWithClassLoader.class, filter1, narClassLoader); + spyWithClassAndConstructorArgs(EntryFilterWithClassLoader.class, filter1, narClassLoader, false); field.set(dispatcher, List.of(loader1)); hasFilterField.set(dispatcher, true); } @@ -231,17 +233,17 @@ public void testSubscriptionStats(final String topic, final String subName, bool } ByteArrayOutputStream output = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, enableTopicStats, false, false, output); + PrometheusMetricsTestUtil.generate(pulsar, enableTopicStats, false, false, output); String metricsStr = output.toString(); - Multimap metrics = PrometheusMetricsTest.parseMetrics(metricsStr); + Multimap metrics = parseMetrics(metricsStr); - Collection throughFilterMetrics = + Collection throughFilterMetrics = metrics.get("pulsar_subscription_filter_processed_msg_count"); - Collection acceptedMetrics = + Collection acceptedMetrics = metrics.get("pulsar_subscription_filter_accepted_msg_count"); - Collection rejectedMetrics = + Collection rejectedMetrics = metrics.get("pulsar_subscription_filter_rejected_msg_count"); - Collection rescheduledMetrics = + Collection rescheduledMetrics = metrics.get("pulsar_subscription_filter_rescheduled_msg_count"); if (enableTopicStats) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TimeWindowTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TimeWindowTest.java deleted file mode 100644 index 89528c1965397..0000000000000 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TimeWindowTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.stats; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; -import org.testng.annotations.Test; - -public class TimeWindowTest { - - @Test - public void windowTest() throws Exception { - int intervalInMs = 1000; - int sampleCount = 2; - TimeWindow timeWindow = new TimeWindow<>(sampleCount, intervalInMs); - - WindowWrap expect1 = timeWindow.current(oldValue -> 1); - WindowWrap expect2 = timeWindow.current(oldValue -> null); - assertNotNull(expect1); - assertNotNull(expect2); - - if (expect1.start() == expect2.start()) { - assertEquals((int) expect1.value(), 1); - assertEquals(expect1, expect2); - assertEquals(expect1.value(), expect2.value()); - } - - Thread.sleep(intervalInMs); - - WindowWrap expect3 = timeWindow.current(oldValue -> 2); - WindowWrap expect4 = timeWindow.current(oldValue -> null); - assertNotNull(expect3); - assertNotNull(expect4); - - if (expect3.start() == expect4.start()) { - assertEquals((int) expect3.value(), 2); - assertEquals(expect3, expect4); - assertEquals(expect3.value(), expect4.value()); - } - - Thread.sleep(intervalInMs); - - WindowWrap expect5 = timeWindow.current(oldValue -> 3); - WindowWrap expect6 = timeWindow.current(oldValue -> null); - assertNotNull(expect5); - assertNotNull(expect6); - - if (expect5.start() == expect6.start()) { - assertEquals((int) expect5.value(), 3); - assertEquals(expect5, expect6); - assertEquals(expect5.value(), expect6.value()); - } - - Thread.sleep(intervalInMs); - - WindowWrap expect7 = timeWindow.current(oldValue -> 4); - WindowWrap expect8 = timeWindow.current(oldValue -> null); - assertNotNull(expect7); - assertNotNull(expect8); - - if (expect7.start() == expect8.start()) { - assertEquals((int) expect7.value(), 4); - assertEquals(expect7, expect8); - assertEquals(expect7.value(), expect8.value()); - } - } -} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java index 4d38f5fad5141..8d5cb9dc39148 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java @@ -19,7 +19,8 @@ package org.apache.pulsar.broker.stats; import static com.google.common.base.Preconditions.checkArgument; -import static org.apache.pulsar.broker.stats.PrometheusMetricsTest.parseMetrics; +import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.Metric; +import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.parseMetrics; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -37,9 +38,9 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.PrometheusMetricsTestUtil; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.BrokerTestBase; -import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGenerator; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; @@ -117,10 +118,10 @@ public void testTransactionCoordinatorMetrics() throws Exception { pulsar.getTransactionMetadataStoreService().getStores() .get(transactionCoordinatorIDTwo).newTransaction(timeout, null).get(); ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); String metricsStr = statsOut.toString(); - Multimap metrics = parseMetrics(metricsStr); - Collection metric = metrics.get("pulsar_txn_active_count"); + Multimap metrics = parseMetrics(metricsStr); + Collection metric = metrics.get("pulsar_txn_active_count"); assertEquals(metric.size(), 2); metric.forEach(item -> { if ("0".equals(item.tags.get("coordinator_id"))) { @@ -185,11 +186,11 @@ public void testTransactionCoordinatorRateMetrics() throws Exception { pulsar.getBrokerService().updateRates(); ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); String metricsStr = statsOut.toString(); - Multimap metrics = parseMetrics(metricsStr); + Multimap metrics = parseMetrics(metricsStr); - Collection metric = metrics.get("pulsar_txn_created_total"); + Collection metric = metrics.get("pulsar_txn_created_total"); assertEquals(metric.size(), 1); metric.forEach(item -> assertEquals(item.value, txnCount)); @@ -215,7 +216,7 @@ public void testTransactionCoordinatorRateMetrics() throws Exception { }); statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); metricsStr = statsOut.toString(); metrics = parseMetrics(metricsStr); @@ -271,12 +272,12 @@ public void testManagedLedgerMetrics() throws Exception { producer.send("hello pulsar".getBytes()); consumer.acknowledgeAsync(consumer.receive().getMessageId(), transaction).get(); ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); String metricsStr = statsOut.toString(); - Multimap metrics = parseMetrics(metricsStr); + Multimap metrics = parseMetrics(metricsStr); - Collection metric = metrics.get("pulsar_storage_size"); + Collection metric = metrics.get("pulsar_storage_size"); checkManagedLedgerMetrics(subName, 32, metric); checkManagedLedgerMetrics(MLTransactionLogImpl.TRANSACTION_SUBSCRIPTION_NAME, 252, metric); @@ -289,7 +290,7 @@ public void testManagedLedgerMetrics() throws Exception { checkManagedLedgerMetrics(MLTransactionLogImpl.TRANSACTION_SUBSCRIPTION_NAME, 126, metric); statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, false, false, false, statsOut); metricsStr = statsOut.toString(); metrics = parseMetrics(metricsStr); metric = metrics.get("pulsar_storage_size"); @@ -333,15 +334,15 @@ public void testManagedLedgerMetricsWhenPendingAckNotInit() throws Exception { producer.send("hello pulsar".getBytes()); consumer.acknowledgeAsync(consumer.receive().getMessageId(), transaction).get(); ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); String metricsStr = statsOut.toString(); - Multimap metrics = parseMetrics(metricsStr); + Multimap metrics = parseMetrics(metricsStr); - Collection metric = metrics.get("pulsar_storage_size"); + Collection metric = metrics.get("pulsar_storage_size"); checkManagedLedgerMetrics(subName, 32, metric); //No statistics of the pendingAck are generated when the pendingAck is not initialized. - for (PrometheusMetricsTest.Metric metric1 : metric) { + for (Metric metric1 : metric) { if (metric1.tags.containsValue(subName2)) { Assert.fail(); } @@ -358,7 +359,7 @@ public void testManagedLedgerMetricsWhenPendingAckNotInit() throws Exception { consumer.acknowledgeAsync(consumer.receive().getMessageId(), transaction).get(); statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, true, false, false, statsOut); metricsStr = statsOut.toString(); metrics = parseMetrics(metricsStr); metric = metrics.get("pulsar_storage_size"); @@ -392,7 +393,7 @@ public void testDuplicateMetricTypeDefinitions() throws Exception { .send(); } ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, false, false, false, statsOut); String metricsStr = statsOut.toString(); Map typeDefs = new HashMap<>(); @@ -431,9 +432,9 @@ public void testDuplicateMetricTypeDefinitions() throws Exception { } - private void checkManagedLedgerMetrics(String tag, double value, Collection metrics) { + private void checkManagedLedgerMetrics(String tag, double value, Collection metrics) { boolean exist = false; - for (PrometheusMetricsTest.Metric metric1 : metrics) { + for (Metric metric1 : metrics) { if (metric1.tags.containsValue(tag)) { assertEquals(metric1.value, value); exist = true; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStatsTest.java index b5933f9ecf529..11358eb1e2c1c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStatsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStatsTest.java @@ -20,10 +20,12 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; - +import java.util.HashMap; +import org.apache.bookkeeper.mledger.util.StatsBuckets; +import org.apache.pulsar.common.policies.data.stats.TopicMetricBean; import org.testng.annotations.Test; -@Test(groups = "broker") +@Test(groups = {"broker"}) public class AggregatedNamespaceStatsTest { @Test @@ -99,6 +101,7 @@ public void testSimpleAggregation() { replStats2.msgThroughputOut = 1536.0; replStats2.replicationBacklog = 99; replStats2.connectedCount = 1; + replStats2.disconnectedCount = 2; replStats2.msgRateExpired = 3.0; replStats2.replicationDelayInSeconds = 20; topicStats2.replicationStats.put(namespace, replStats2); @@ -146,6 +149,7 @@ public void testSimpleAggregation() { assertEquals(nsReplStats.msgThroughputOut, 1792.0); assertEquals(nsReplStats.replicationBacklog, 100); assertEquals(nsReplStats.connectedCount, 1); + assertEquals(nsReplStats.disconnectedCount, 2); assertEquals(nsReplStats.msgRateExpired, 6.0); assertEquals(nsReplStats.replicationDelayInSeconds, 40); @@ -157,4 +161,101 @@ public void testSimpleAggregation() { assertEquals(nsSubStats.unackedMessages, 2); } + + @Test + public void testReset() { + AggregatedNamespaceStats stats = new AggregatedNamespaceStats(); + stats.topicsCount = 8; + stats.subscriptionsCount = 3; + stats.producersCount = 1; + stats.consumersCount = 8; + stats.rateIn = 1.3; + stats.rateOut = 3.5; + stats.throughputIn = 3.2; + stats.throughputOut = 5.8; + stats.messageAckRate = 12; + stats.bytesInCounter = 1234; + stats.msgInCounter = 3889; + stats.bytesOutCounter = 89775; + stats.msgOutCounter = 28983; + stats.msgBacklog = 39; + stats.msgDelayed = 31; + + stats.ongoingTxnCount = 87; + stats.abortedTxnCount = 74; + stats.committedTxnCount = 34; + + stats.backlogQuotaLimit = 387; + stats.backlogQuotaLimitTime = 8771; + + stats.replicationStats = new HashMap<>(); + stats.replicationStats.put("r", new AggregatedReplicationStats()); + + stats.subscriptionStats = new HashMap<>(); + stats.subscriptionStats.put("r", new AggregatedSubscriptionStats()); + + stats.compactionRemovedEventCount = 124; + stats.compactionSucceedCount = 487; + stats.compactionFailedCount = 84857; + stats.compactionDurationTimeInMills = 2384; + stats.compactionReadThroughput = 355423; + stats.compactionWriteThroughput = 23299; + stats.compactionCompactedEntriesCount = 37522; + stats.compactionCompactedEntriesSize = 8475; + + stats.compactionLatencyBuckets = new StatsBuckets(5); + stats.compactionLatencyBuckets.addValue(3); + + stats.delayedMessageIndexSizeInBytes = 45223; + + stats.bucketDelayedIndexStats = new HashMap<>(); + stats.bucketDelayedIndexStats.put("t", new TopicMetricBean()); + + stats.reset(); + + assertEquals(stats.bytesOutCounter, 0); + assertEquals(stats.topicsCount, 0); + assertEquals(stats.subscriptionsCount, 0); + assertEquals(stats.producersCount, 0); + assertEquals(stats.consumersCount, 0); + assertEquals(stats.rateIn, 0); + assertEquals(stats.rateOut, 0); + assertEquals(stats.throughputIn, 0); + assertEquals(stats.throughputOut, 0); + assertEquals(stats.messageAckRate, 0); + assertEquals(stats.bytesInCounter, 0); + assertEquals(stats.msgInCounter, 0); + assertEquals(stats.bytesOutCounter, 0); + assertEquals(stats.msgOutCounter, 0); + + assertEquals(stats.managedLedgerStats.storageSize, 0); + + assertEquals(stats.msgBacklog, 0); + assertEquals(stats.msgDelayed, 0); + + assertEquals(stats.ongoingTxnCount, 0); + assertEquals(stats.abortedTxnCount, 0); + assertEquals(stats.committedTxnCount, 0); + + assertEquals(stats.backlogQuotaLimit, 0); + assertEquals(stats.backlogQuotaLimitTime, -1); + + assertEquals(stats.replicationStats.size(), 0); + assertEquals(stats.subscriptionStats.size(), 0); + + assertEquals(stats.compactionRemovedEventCount, 0); + assertEquals(stats.compactionSucceedCount, 0); + assertEquals(stats.compactionFailedCount, 0); + assertEquals(stats.compactionDurationTimeInMills, 0); + assertEquals(stats.compactionReadThroughput, 0); + assertEquals(stats.compactionWriteThroughput, 0); + assertEquals(stats.compactionCompactedEntriesCount, 0); + assertEquals(stats.compactionCompactedEntriesSize, 0); + + assertEquals(stats.compactionLatencyBuckets.getSum(), 0); + + assertEquals(stats.delayedMessageIndexSizeInBytes, 0); + assertEquals(stats.bucketDelayedIndexStats.size(), 0); + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregatorTest.java index e63f644f3d0e9..e091eee178d8c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregatorTest.java @@ -21,6 +21,8 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.when; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.impl.ManagedLedgerMBeanImpl; import org.apache.bookkeeper.mledger.util.StatsBuckets; @@ -28,15 +30,14 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.Consumer; -import org.apache.pulsar.broker.service.Replicator; -import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.service.persistent.PersistentTopicMetrics; import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; import org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl; import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.metadata.impl.ZKMetadataStore; import org.mockito.Mockito; import org.testng.annotations.BeforeMethod; @@ -46,17 +47,14 @@ public class NamespaceStatsAggregatorTest { protected PulsarService pulsar; private BrokerService broker; - private ConcurrentOpenHashMap>> - multiLayerTopicsMap; + private Map>> multiLayerTopicsMap; @BeforeMethod(alwaysRun = true) public void setup() throws Exception { - multiLayerTopicsMap = ConcurrentOpenHashMap.>>newBuilder() - .build(); + multiLayerTopicsMap = new ConcurrentHashMap<>(); pulsar = Mockito.mock(PulsarService.class); broker = Mockito.mock(BrokerService.class); - doReturn(multiLayerTopicsMap).when(broker).getMultiLayerTopicMap(); + doReturn(multiLayerTopicsMap).when(broker).getMultiLayerTopicsMap(); Mockito.when(pulsar.getLocalMetadataStore()).thenReturn(Mockito.mock(ZKMetadataStore.class)); ServiceConfiguration mockConfig = Mockito.mock(ServiceConfiguration.class); doReturn(mockConfig).when(pulsar).getConfiguration(); @@ -69,9 +67,9 @@ public void testGenerateSubscriptionsStats() { final String namespace = "tenant/cluster/ns"; // prepare multi-layer topic map - ConcurrentOpenHashMap bundlesMap = ConcurrentOpenHashMap.newBuilder().build(); - ConcurrentOpenHashMap topicsMap = ConcurrentOpenHashMap.newBuilder().build(); - ConcurrentOpenHashMap subscriptionsMaps = ConcurrentOpenHashMap.newBuilder().build(); + final var bundlesMap = new ConcurrentHashMap>(); + final var topicsMap = new ConcurrentHashMap(); + final var subscriptionsMaps = new ConcurrentHashMap(); bundlesMap.put("my-bundle", topicsMap); multiLayerTopicsMap.put(namespace, bundlesMap); @@ -87,7 +85,7 @@ public void testGenerateSubscriptionsStats() { // Prepare topic and subscription PersistentTopic topic = Mockito.mock(PersistentTopic.class); - Subscription subscription = Mockito.mock(Subscription.class); + PersistentSubscription subscription = Mockito.mock(PersistentSubscription.class); Consumer consumer = Mockito.mock(Consumer.class); ConsumerStatsImpl consumerStats = new ConsumerStatsImpl(); when(consumer.getStats()).thenReturn(consumerStats); @@ -99,9 +97,11 @@ public void testGenerateSubscriptionsStats() { when(topic.getStats(false, false, false)).thenReturn(topicStats); when(topic.getBrokerService()).thenReturn(broker); when(topic.getSubscriptions()).thenReturn(subscriptionsMaps); - when(topic.getReplicators()).thenReturn(ConcurrentOpenHashMap.newBuilder().build()); + when(topic.getReplicators()).thenReturn(new ConcurrentHashMap<>()); when(topic.getManagedLedger()).thenReturn(ml); when(topic.getBacklogQuota(Mockito.any())).thenReturn(Mockito.mock(BacklogQuota.class)); + PersistentTopicMetrics persistentTopicMetrics = new PersistentTopicMetrics(); + when(topic.getPersistentTopicMetrics()).thenReturn(persistentTopicMetrics); topicsMap.put("my-topic", topic); PrometheusMetricStreams metricStreams = Mockito.spy(new PrometheusMetricStreams()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorTest.java new file mode 100644 index 0000000000000..ed5c5a6335ceb --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats.prometheus; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import com.google.common.util.concurrent.MoreExecutors; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.time.Clock; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ExecutionException; +import java.util.zip.GZIPInputStream; +import org.apache.commons.io.IOUtils; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.testng.annotations.Test; + +public class PrometheusMetricsGeneratorTest { + + // reproduce issue #22575 + @Test + public void testReproducingBufferOverflowExceptionAndEOFExceptionBugsInGzipCompression() + throws ExecutionException, InterruptedException, IOException { + PulsarService pulsar = mock(PulsarService.class); + ServiceConfiguration serviceConfiguration = new ServiceConfiguration(); + when(pulsar.getConfiguration()).thenReturn(serviceConfiguration); + + // generate a random byte buffer which is 8 bytes less than the minimum compress buffer size limit + // this will trigger the BufferOverflowException bug in writing the gzip trailer + // it will also trigger another bug in finishing the gzip compression stream when the compress buffer is full + // which results in EOFException + Random random = new Random(); + byte[] inputBytes = new byte[8192 - 8]; + random.nextBytes(inputBytes); + ByteBuf byteBuf = Unpooled.wrappedBuffer(inputBytes); + + PrometheusMetricsGenerator generator = + new PrometheusMetricsGenerator(pulsar, false, false, false, false, Clock.systemUTC()) { + // override the generateMetrics method to return the random byte buffer for gzip compression + // instead of the actual metrics + @Override + protected ByteBuf generateMetrics(List metricsProviders) { + return byteBuf; + } + }; + + PrometheusMetricsGenerator.MetricsBuffer metricsBuffer = + generator.renderToBuffer(MoreExecutors.directExecutor(), Collections.emptyList()); + try { + PrometheusMetricsGenerator.ResponseBuffer responseBuffer = metricsBuffer.getBufferFuture().get(); + + ByteBuf compressed = responseBuffer.getCompressedBuffer(MoreExecutors.directExecutor()).get(); + byte[] compressedBytes = new byte[compressed.readableBytes()]; + compressed.readBytes(compressedBytes); + try (GZIPInputStream gzipInputStream = new GZIPInputStream(new ByteArrayInputStream(compressedBytes))) { + byte[] uncompressedBytes = IOUtils.toByteArray(gzipInputStream); + assertEquals(uncompressedBytes, inputBytes); + } + } finally { + metricsBuffer.release(); + } + } +} \ No newline at end of file diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicServiceTest.java index 44a4de5e8a923..e66140efb32bb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicServiceTest.java @@ -20,10 +20,13 @@ import static org.apache.pulsar.broker.service.SystemTopicBasedTopicPoliciesService.getEventKey; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import com.google.common.collect.Sets; import java.util.HashSet; import lombok.Cleanup; import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -99,7 +102,9 @@ public void testSystemTopicSchemaCompatibility() throws Exception { TopicPoliciesSystemTopicClient systemTopicClientForNamespace1 = systemTopicFactory .createTopicPoliciesSystemTopicClient(NamespaceName.get(NAMESPACE1)); String topicName = systemTopicClientForNamespace1.getTopicName().toString(); - SystemTopic topic = new SystemTopic(topicName, mock(ManagedLedger.class), pulsar.getBrokerService()); + final var mockManagedLedger = mock(ManagedLedger.class); + when(mockManagedLedger.getConfig()).thenReturn(new ManagedLedgerConfig()); + SystemTopic topic = new SystemTopic(topicName, mockManagedLedger, pulsar.getBrokerService()); Assert.assertEquals(SchemaCompatibilityStrategy.ALWAYS_COMPATIBLE, topic.getSchemaCompatibilityStrategy()); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java index 008c2143a3566..e7bfa3278e36d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.broker.systopic; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; import com.google.common.collect.Sets; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -35,8 +37,10 @@ import org.apache.pulsar.broker.admin.impl.BrokersBase; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.broker.service.TopicPolicyTestUtils; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.service.schema.SchemaRegistry; import org.apache.pulsar.client.admin.ListTopicsOptions; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; @@ -55,6 +59,7 @@ import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.common.util.FutureUtil; import org.awaitility.Awaitility; @@ -77,6 +82,7 @@ protected void setup() throws Exception { conf.setDefaultNumPartitions(PARTITIONS); conf.setManagedLedgerMaxEntriesPerLedger(1); conf.setBrokerDeleteInactiveTopicsEnabled(false); + conf.setTransactionCoordinatorEnabled(true); super.baseSetup(); } @@ -160,7 +166,7 @@ public void testProduceAndConsumeUnderSystemNamespace() throws Exception { @Test public void testHealthCheckTopicNotOffload() throws Exception { - NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getAdvertisedAddress(), + NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getBrokerId(), pulsar.getConfig()); TopicName topicName = TopicName.get("persistent", namespaceName, BrokersBase.HEALTH_CHECK_TOPIC_SUFFIX); PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService() @@ -180,18 +186,25 @@ public void testHealthCheckTopicNotOffload() throws Exception { @Test public void testSystemNamespaceNotCreateChangeEventsTopic() throws Exception { admin.brokers().healthcheck(TopicVersion.V2); - NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getAdvertisedAddress(), + NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getBrokerId(), pulsar.getConfig()); TopicName topicName = TopicName.get("persistent", namespaceName, SystemTopicNames.NAMESPACE_EVENTS_LOCAL_NAME); Optional optionalTopic = pulsar.getBrokerService() .getTopic(topicName.getPartition(1).toString(), false).join(); Assert.assertTrue(optionalTopic.isEmpty()); + + TopicName heartbeatTopicName = TopicName.get("persistent", + namespaceName, BrokersBase.HEALTH_CHECK_TOPIC_SUFFIX); + admin.topics().getRetention(heartbeatTopicName.toString()); + optionalTopic = pulsar.getBrokerService() + .getTopic(topicName.getPartition(1).toString(), false).join(); + Assert.assertTrue(optionalTopic.isEmpty()); } @Test public void testHeartbeatTopicNotAllowedToSendEvent() throws Exception { admin.brokers().healthcheck(TopicVersion.V2); - NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getAdvertisedAddress(), + NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getBrokerId(), pulsar.getConfig()); TopicName topicName = TopicName.get("persistent", namespaceName, SystemTopicNames.NAMESPACE_EVENTS_LOCAL_NAME); for (int partition = 0; partition < PARTITIONS; partition ++) { @@ -203,6 +216,40 @@ public void testHeartbeatTopicNotAllowedToSendEvent() throws Exception { }); } + @Test + public void testHeartbeatTopicBeDeleted() throws Exception { + admin.brokers().healthcheck(TopicVersion.V2); + NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getBrokerId(), + pulsar.getConfig()); + TopicName heartbeatTopicName = TopicName.get("persistent", namespaceName, BrokersBase.HEALTH_CHECK_TOPIC_SUFFIX); + + List topics = getPulsar().getNamespaceService().getListOfPersistentTopics(namespaceName).join(); + Assert.assertEquals(topics.size(), 1); + Assert.assertEquals(topics.get(0), heartbeatTopicName.toString()); + + admin.topics().delete(heartbeatTopicName.toString(), true); + topics = getPulsar().getNamespaceService().getListOfPersistentTopics(namespaceName).join(); + Assert.assertEquals(topics.size(), 0); + } + + @Test + public void testHeartbeatNamespaceNotCreateTransactionInternalTopic() throws Exception { + admin.brokers().healthcheck(TopicVersion.V2); + NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getBrokerId(), + pulsar.getConfig()); + TopicName topicName = TopicName.get("persistent", + namespaceName, SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT); + Optional optionalTopic = pulsar.getBrokerService() + .getTopic(topicName.getPartition(1).toString(), false).join(); + Assert.assertTrue(optionalTopic.isEmpty()); + + List topics = getPulsar().getNamespaceService().getListOfPersistentTopics(namespaceName).join(); + Assert.assertEquals(topics.size(), 1); + TopicName heartbeatTopicName = TopicName.get("persistent", + namespaceName, BrokersBase.HEALTH_CHECK_TOPIC_SUFFIX); + Assert.assertEquals(topics.get(0), heartbeatTopicName.toString()); + } + @Test public void testSetBacklogCausedCreatingProducerFailure() throws Exception { final String ns = "prop/ns-test"; @@ -299,4 +346,26 @@ public void testSystemTopicNotCheckExceed() throws Exception { writer1.get().close(); writer2.get().close(); } + + @Test + public void testDeleteTopicSchemaAndPolicyWhenTopicIsNotLoaded() throws Exception { + final String ns = "prop/ns-test"; + admin.namespaces().createNamespace(ns, 2); + final String topicName = "persistent://prop/ns-test/testDeleteTopicSchemaAndPolicyWhenTopicIsNotLoaded"; + admin.topics().createNonPartitionedTopic(topicName); + pulsarClient.newProducer(Schema.STRING).topic(topicName).create().close(); + admin.topicPolicies().setMaxConsumers(topicName, 2); + Awaitility.await().untilAsserted(() -> assertEquals(admin.topicPolicies().getMaxConsumers(topicName), 2)); + CompletableFuture> topic = pulsar.getBrokerService().getTopic(topicName, false); + PersistentTopic persistentTopic = (PersistentTopic) topic.join().get(); + persistentTopic.close(); + admin.topics().delete(topicName); + TopicPolicies topicPolicies = TopicPolicyTestUtils.getTopicPolicies(pulsar.getTopicPoliciesService(), + TopicName.get(topicName)); + assertNull(topicPolicies); + String base = TopicName.get(topicName).getPartitionedTopicName(); + String id = TopicName.get(base).getSchemaName(); + CompletableFuture schema = pulsar.getSchemaRegistryService().getSchema(id); + assertNull(schema.join()); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/AbstractTestPulsarService.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/AbstractTestPulsarService.java index a6861268b94d5..c67714484f442 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/AbstractTestPulsarService.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/AbstractTestPulsarService.java @@ -19,13 +19,20 @@ package org.apache.pulsar.broker.testcontext; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdkBuilder; +import java.io.IOException; +import java.util.Optional; +import java.util.function.Consumer; +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.BookKeeperClientFactory; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.intercept.BrokerInterceptor; import org.apache.pulsar.broker.service.PulsarMetadataEventSynchronizer; -import org.apache.pulsar.compaction.Compactor; +import org.apache.pulsar.compaction.CompactionServiceFactory; +import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; @@ -35,68 +42,61 @@ * {@link PulsarService} implementations for a PulsarService instance used in tests. * Please see {@link PulsarTestContext} for more details. */ + +@Slf4j abstract class AbstractTestPulsarService extends PulsarService { protected final SpyConfig spyConfig; - protected final MetadataStoreExtended localMetadataStore; - protected final MetadataStoreExtended configurationMetadataStore; - protected final Compactor compactor; - protected final BrokerInterceptor brokerInterceptor; - protected final BookKeeperClientFactory bookKeeperClientFactory; public AbstractTestPulsarService(SpyConfig spyConfig, ServiceConfiguration config, MetadataStoreExtended localMetadataStore, - MetadataStoreExtended configurationMetadataStore, Compactor compactor, + MetadataStoreExtended configurationMetadataStore, + CompactionServiceFactory compactionServiceFactory, BrokerInterceptor brokerInterceptor, - BookKeeperClientFactory bookKeeperClientFactory) { - super(config); + BookKeeperClientFactory bookKeeperClientFactory, + Consumer openTelemetrySdkBuilderCustomizer) { + super(config, new WorkerConfig(), Optional.empty(), + exitCode -> log.info("Pulsar process termination requested with code {}.", exitCode), + openTelemetrySdkBuilderCustomizer); + this.spyConfig = spyConfig; - this.localMetadataStore = - NonClosingProxyHandler.createNonClosingProxy(localMetadataStore, MetadataStoreExtended.class); - this.configurationMetadataStore = - NonClosingProxyHandler.createNonClosingProxy(configurationMetadataStore, MetadataStoreExtended.class); - this.compactor = compactor; - this.brokerInterceptor = brokerInterceptor; - this.bookKeeperClientFactory = bookKeeperClientFactory; + setLocalMetadataStore( + NonClosingProxyHandler.createNonClosingProxy(localMetadataStore, MetadataStoreExtended.class)); + setConfigurationMetadataStore( + NonClosingProxyHandler.createNonClosingProxy(configurationMetadataStore, MetadataStoreExtended.class)); + super.setCompactionServiceFactory(compactionServiceFactory); + setBrokerInterceptor(brokerInterceptor); + setBkClientFactory(bookKeeperClientFactory); } @Override - public MetadataStore createConfigurationMetadataStore(PulsarMetadataEventSynchronizer synchronizer) + public MetadataStore createConfigurationMetadataStore(PulsarMetadataEventSynchronizer synchronizer, + OpenTelemetry openTelemetry) throws MetadataStoreException { if (synchronizer != null) { - synchronizer.registerSyncListener(configurationMetadataStore::handleMetadataEvent); + synchronizer.registerSyncListener( + ((MetadataStoreExtended) getConfigurationMetadataStore())::handleMetadataEvent); } - return configurationMetadataStore; + return getConfigurationMetadataStore(); } @Override - public MetadataStoreExtended createLocalMetadataStore(PulsarMetadataEventSynchronizer synchronizer) + public MetadataStoreExtended createLocalMetadataStore(PulsarMetadataEventSynchronizer synchronizer, + OpenTelemetry openTelemetry) throws MetadataStoreException, PulsarServerException { if (synchronizer != null) { - synchronizer.registerSyncListener(localMetadataStore::handleMetadataEvent); + synchronizer.registerSyncListener( + getLocalMetadataStore()::handleMetadataEvent); } - return localMetadataStore; + return getLocalMetadataStore(); } @Override - public Compactor newCompactor() throws PulsarServerException { - if (compactor != null) { - return compactor; - } else { - return spyConfig.getCompactor().spy(super.newCompactor()); - } - } - - @Override - public BrokerInterceptor getBrokerInterceptor() { - if (brokerInterceptor != null) { - return brokerInterceptor; - } else { - return super.getBrokerInterceptor(); - } + public BookKeeperClientFactory newBookKeeperClientFactory() { + return getBkClientFactory(); } @Override - public BookKeeperClientFactory newBookKeeperClientFactory() { - return bookKeeperClientFactory; + protected BrokerInterceptor newBrokerInterceptor() throws IOException { + return getBrokerInterceptor() != null ? getBrokerInterceptor() : super.newBrokerInterceptor(); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/MockBookKeeperClientFactory.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/MockBookKeeperClientFactory.java index fd457687323bf..5f02fd7af48f0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/MockBookKeeperClientFactory.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/MockBookKeeperClientFactory.java @@ -21,6 +21,7 @@ import io.netty.channel.EventLoopGroup; import java.util.Map; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.EnsemblePlacementPolicy; import org.apache.bookkeeper.stats.StatsLogger; @@ -39,21 +40,21 @@ class MockBookKeeperClientFactory implements BookKeeperClientFactory { } @Override - public BookKeeper create(ServiceConfiguration conf, MetadataStoreExtended store, - EventLoopGroup eventLoopGroup, - Optional> ensemblePlacementPolicyClass, - Map properties) { + public CompletableFuture create(ServiceConfiguration conf, MetadataStoreExtended store, + EventLoopGroup eventLoopGroup, + Optional> ensemblePlacementPolicyClass, + Map properties) { // Always return the same instance (so that we don't loose the mock BK content on broker restart - return mockBookKeeper; + return CompletableFuture.completedFuture(mockBookKeeper); } @Override - public BookKeeper create(ServiceConfiguration conf, MetadataStoreExtended store, + public CompletableFuture create(ServiceConfiguration conf, MetadataStoreExtended store, EventLoopGroup eventLoopGroup, Optional> ensemblePlacementPolicyClass, Map properties, StatsLogger statsLogger) { // Always return the same instance (so that we don't loose the mock BK content on broker restart - return mockBookKeeper; + return CompletableFuture.completedFuture(mockBookKeeper); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/MockPulsarCompactionServiceFactory.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/MockPulsarCompactionServiceFactory.java new file mode 100644 index 0000000000000..77959fe6ce994 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/MockPulsarCompactionServiceFactory.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.testcontext; + +import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.compaction.Compactor; +import org.apache.pulsar.compaction.PulsarCompactionServiceFactory; + +public class MockPulsarCompactionServiceFactory extends PulsarCompactionServiceFactory { + private final Compactor compactor; + private final SpyConfig spyConfig; + + public MockPulsarCompactionServiceFactory(SpyConfig spyConfig, Compactor compactor) { + this.compactor = compactor; + this.spyConfig = spyConfig; + } + + @Override + protected Compactor newCompactor() throws PulsarServerException { + if (this.compactor != null) { + return this.compactor; + } else { + return spyConfig.getCompactor().spy(super.newCompactor()); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/NonStartableTestPulsarService.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/NonStartableTestPulsarService.java index 4b7762a2acfdf..7860b0708e35e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/NonStartableTestPulsarService.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/NonStartableTestPulsarService.java @@ -20,7 +20,9 @@ import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import io.netty.channel.EventLoopGroup; +import java.io.IOException; import java.util.Collections; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -37,14 +39,15 @@ import org.apache.pulsar.broker.resources.TopicResources; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.schema.DefaultSchemaRegistryService; -import org.apache.pulsar.broker.service.schema.SchemaRegistryService; import org.apache.pulsar.broker.storage.ManagedLedgerStorage; -import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.broker.transaction.buffer.TransactionBufferProvider; +import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.compaction.Compactor; +import org.apache.pulsar.compaction.CompactionServiceFactory; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; @@ -53,42 +56,53 @@ * for a "non-startable" PulsarService. Please see {@link PulsarTestContext} for more details. */ class NonStartableTestPulsarService extends AbstractTestPulsarService { - private final PulsarResources pulsarResources; - private final ManagedLedgerStorage managedLedgerClientFactory; - private final BrokerService brokerService; - - private final SchemaRegistryService schemaRegistryService; - - private final PulsarClientImpl pulsarClient; private final NamespaceService namespaceService; public NonStartableTestPulsarService(SpyConfig spyConfig, ServiceConfiguration config, MetadataStoreExtended localMetadataStore, MetadataStoreExtended configurationMetadataStore, - Compactor compactor, BrokerInterceptor brokerInterceptor, + CompactionServiceFactory compactionServiceFactory, + BrokerInterceptor brokerInterceptor, BookKeeperClientFactory bookKeeperClientFactory, PulsarResources pulsarResources, ManagedLedgerStorage managedLedgerClientFactory, Function brokerServiceCustomizer) { - super(spyConfig, config, localMetadataStore, configurationMetadataStore, compactor, brokerInterceptor, - bookKeeperClientFactory); - this.pulsarResources = pulsarResources; - this.managedLedgerClientFactory = managedLedgerClientFactory; + super(spyConfig, config, localMetadataStore, configurationMetadataStore, compactionServiceFactory, + brokerInterceptor, bookKeeperClientFactory, null); + setPulsarResources(pulsarResources); + setManagedLedgerClientFactory(managedLedgerClientFactory); try { - this.brokerService = brokerServiceCustomizer.apply( - spyConfig.getBrokerService().spy(TestBrokerService.class, this, getIoEventLoopGroup())); + setBrokerService(brokerServiceCustomizer.apply( + spyConfig.getBrokerService().spy(TestBrokerService.class, this, getIoEventLoopGroup()))); } catch (Exception e) { throw new RuntimeException(e); } - this.schemaRegistryService = spyWithClassAndConstructorArgs(DefaultSchemaRegistryService.class); - this.pulsarClient = mock(PulsarClientImpl.class); + setSchemaRegistryService(spyWithClassAndConstructorArgs(DefaultSchemaRegistryService.class)); + PulsarClientImpl mockClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(mockClient.getCnxPool()).thenReturn(connectionPool); + setClient(mockClient); this.namespaceService = mock(NamespaceService.class); try { startNamespaceService(); } catch (PulsarServerException e) { throw new RuntimeException(e); } + if (config.isTransactionCoordinatorEnabled()) { + try { + setTransactionBufferProvider(TransactionBufferProvider + .newProvider(config.getTransactionBufferProviderClassName())); + } catch (IOException e) { + throw new RuntimeException(e); + } + try { + setTransactionPendingAckStoreProvider(TransactionPendingAckStoreProvider + .newProvider(config.getTransactionPendingAckStoreProviderClassName())); + } catch (IOException e) { + throw new RuntimeException(e); + } + } } @Override @@ -101,63 +115,17 @@ public Supplier getNamespaceServiceProvider() throws PulsarSer return () -> namespaceService; } - @Override - public synchronized PulsarClient getClient() throws PulsarServerException { - return pulsarClient; - } - @Override public PulsarClientImpl createClientImpl(ClientConfigurationData clientConf) throws PulsarClientException { - return pulsarClient; - } - - @Override - public SchemaRegistryService getSchemaRegistryService() { - return schemaRegistryService; - } - - @Override - public PulsarResources getPulsarResources() { - return pulsarResources; - } - - public BrokerService getBrokerService() { - return brokerService; - } - - @Override - public MetadataStore getConfigurationMetadataStore() { - return configurationMetadataStore; - } - - @Override - public MetadataStoreExtended getLocalMetadataStore() { - return localMetadataStore; - } - - @Override - public ManagedLedgerStorage getManagedLedgerClientFactory() { - return managedLedgerClientFactory; - } - - @Override - protected PulsarResources newPulsarResources() { - return pulsarResources; - } - - @Override - protected ManagedLedgerStorage newManagedLedgerClientFactory() throws Exception { - return managedLedgerClientFactory; + try { + return (PulsarClientImpl) getClient(); + } catch (PulsarServerException e) { + throw new PulsarClientException(e); + } } - @Override protected BrokerService newBrokerService(PulsarService pulsar) throws Exception { - return brokerService; - } - - @Override - public BookKeeperClientFactory getBookKeeperClientFactory() { - return bookKeeperClientFactory; + return getBrokerService(); } static class TestBrokerService extends BrokerService { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java index d3d4b7cf9341c..3d79a17a90f50 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java @@ -21,6 +21,9 @@ import com.google.common.util.concurrent.MoreExecutors; import io.netty.channel.EventLoopGroup; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdkBuilder; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.Duration; @@ -52,9 +55,13 @@ import org.apache.pulsar.broker.resources.TopicResources; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.ServerCnx; +import org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil; import org.apache.pulsar.broker.storage.ManagedLedgerStorage; import org.apache.pulsar.common.util.GracefulExecutorServicesShutdown; +import org.apache.pulsar.common.util.PortManager; +import org.apache.pulsar.compaction.CompactionServiceFactory; import org.apache.pulsar.compaction.Compactor; +import org.apache.pulsar.compaction.PulsarCompactionServiceFactory; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.MetadataStoreException; @@ -83,7 +90,7 @@ * There are few motivations for PulsarTestContext: *
    *
  • It reduces the reliance on Mockito for hooking into the PulsarService for injecting mocks or customizing the behavior of some - * collaborators. Mockito is not thread-safe and some mocking operations get corrupted. Some examples of the issuess: https://github.com/apache/pulsar/issues/13620, https://github.com/apache/pulsar/issues/16444 and https://github.com/apache/pulsar/issues/16427.
    • + * collaborators. Mockito is not thread-safe and some mocking operations get corrupted. Some examples of the issues: https://github.com/apache/pulsar/issues/13620, https://github.com/apache/pulsar/issues/16444 and https://github.com/apache/pulsar/issues/16427. *
  • Since the Mockito issue causes test flakiness, this change will improve reliability.
    • *
  • It makes it possible to use composition over inheritance in test classes. This can help reduce the dependency on * deep test base cases hierarchies.
    • @@ -135,6 +142,8 @@ public class PulsarTestContext implements AutoCloseable { private final Compactor compactor; + private final CompactionServiceFactory compactionServiceFactory; + private final BrokerService brokerService; @Getter(AccessLevel.NONE) @@ -153,6 +162,10 @@ public class PulsarTestContext implements AutoCloseable { private final boolean startable; + private final boolean preallocatePorts; + + private final boolean enableOpenTelemetry; + private final InMemoryMetricReader openTelemetryMetricReader; public ManagedLedgerFactory getManagedLedgerFactory() { return managedLedgerClientFactory.getManagedLedgerFactory(); @@ -189,6 +202,10 @@ public static Builder builderForNonStartableContext() { * @throws Exception if there is an error closing the resources */ public void close() throws Exception { + callCloseables(closeables); + } + + private static void callCloseables(List closeables) { for (int i = closeables.size() - 1; i >= 0; i--) { try { closeables.get(i).close(); @@ -220,8 +237,11 @@ public static class Builder { protected SpyConfig.Builder spyConfigBuilder = SpyConfig.builder(SpyConfig.SpyType.NONE); protected Consumer pulsarServiceCustomizer; protected ServiceConfiguration svcConfig = initializeConfig(); - protected Consumer configOverrideCustomizer = this::defaultOverrideServiceConfiguration; + protected Consumer configOverrideCustomizer; + + protected boolean configOverrideCalled = false; protected Function brokerServiceCustomizer = Function.identity(); + protected PulsarTestContext otherContextToClose; /** * Initialize the ServiceConfiguration with default values. @@ -290,6 +310,10 @@ protected void defaultOverrideServiceConfiguration(ServiceConfiguration svcConfi if (svcConfig.getManagedLedgerCacheSizeMB() == unconfiguredDefaults.getManagedLedgerCacheSizeMB()) { svcConfig.setManagedLedgerCacheSizeMB(8); } + + if (svcConfig.getTopicLoadTimeoutSeconds() == unconfiguredDefaults.getTopicLoadTimeoutSeconds()) { + svcConfig.setTopicLoadTimeoutSeconds(10); + } } /** @@ -307,7 +331,17 @@ protected long resolveBrokerShutdownTimeoutMs() { * @return the builder */ public Builder spyByDefault() { - spyConfigBuilder = SpyConfig.builder(SpyConfig.SpyType.SPY); + spyConfigDefault(SpyConfig.SpyType.SPY); + return this; + } + + public Builder spyNoneByDefault() { + spyConfigDefault(SpyConfig.SpyType.NONE); + return this; + } + + public Builder spyConfigDefault(SpyConfig.SpyType spyType) { + spyConfigBuilder = SpyConfig.builder(spyType); return this; } @@ -341,6 +375,7 @@ public Builder configCustomizer(Consumer configCustomerize */ public Builder configOverride(Consumer configOverrideCustomizer) { this.configOverrideCustomizer = configOverrideCustomizer; + this.configOverrideCalled = true; return this; } @@ -393,7 +428,15 @@ public Builder reuseSpyConfig(PulsarTestContext otherContext) { * The other PulsarTestContext will be closed when this one is closed. */ public Builder chainClosing(PulsarTestContext otherContext) { - registerCloseable(otherContext); + otherContextToClose = otherContext; + return this; + } + + /** + * Registers a closeable to close as the last one by prepending it to the closeables list. + */ + public Builder prependCloseable(AutoCloseable closeable) { + closeables.add(0, closeable); return this; } @@ -517,12 +560,15 @@ public final PulsarTestContext build() { if (super.config == null) { config(svcConfig); } + handlePreallocatePorts(super.config); + if (configOverrideCustomizer != null || !configOverrideCalled) { + // call defaultOverrideServiceConfiguration if configOverrideCustomizer + // isn't explicitly set to null with `.configOverride(null)` call + defaultOverrideServiceConfiguration(super.config); + } if (configOverrideCustomizer != null) { configOverrideCustomizer.accept(super.config); } - if (super.brokerInterceptor != null) { - super.config.setDisableBrokerInterceptors(false); - } initializeCommonPulsarServices(spyConfig); initializePulsarServices(spyConfig, this); if (pulsarServiceCustomizer != null) { @@ -532,13 +578,49 @@ public final PulsarTestContext build() { try { super.pulsarService.start(); } catch (Exception e) { + callCloseables(super.closeables); + super.closeables.clear(); throw new RuntimeException(e); } } + if (otherContextToClose != null) { + prependCloseable(otherContextToClose); + } brokerService(super.pulsarService.getBrokerService()); return super.build(); } + protected void handlePreallocatePorts(ServiceConfiguration config) { + if (super.preallocatePorts) { + config.getBrokerServicePort().ifPresent(portNumber -> { + if (portNumber == 0) { + config.setBrokerServicePort(Optional.of(PortManager.nextLockedFreePort())); + } + }); + config.getBrokerServicePortTls().ifPresent(portNumber -> { + if (portNumber == 0) { + config.setBrokerServicePortTls(Optional.of(PortManager.nextLockedFreePort())); + } + }); + config.getWebServicePort().ifPresent(portNumber -> { + if (portNumber == 0) { + config.setWebServicePort(Optional.of(PortManager.nextLockedFreePort())); + } + }); + config.getWebServicePortTls().ifPresent(portNumber -> { + if (portNumber == 0) { + config.setWebServicePortTls(Optional.of(PortManager.nextLockedFreePort())); + } + }); + registerCloseable(() -> { + config.getBrokerServicePort().ifPresent(PortManager::releaseLockedPort); + config.getBrokerServicePortTls().ifPresent(PortManager::releaseLockedPort); + config.getWebServicePort().ifPresent(PortManager::releaseLockedPort); + config.getWebServicePortTls().ifPresent(PortManager::releaseLockedPort); + }); + } + } + private void initializeCommonPulsarServices(SpyConfig spyConfig) { if (super.bookKeeperClient == null && super.managedLedgerClientFactory == null) { if (super.executor == null) { @@ -584,7 +666,8 @@ private void initializeCommonPulsarServices(SpyConfig spyConfig) { } else { try { MetadataStoreExtended store = MetadataStoreFactoryImpl.createExtended("memory:local", - MetadataStoreConfig.builder().build()); + MetadataStoreConfig.builder() + .metadataStoreName(MetadataStoreConfig.METADATA_STORE).build()); registerCloseable(() -> { store.close(); resetSpyOrMock(store); @@ -655,10 +738,29 @@ public Builder pulsarResources(PulsarResources pulsarResources) { protected void initializePulsarServices(SpyConfig spyConfig, Builder builder) { BookKeeperClientFactory bookKeeperClientFactory = new MockBookKeeperClientFactory(builder.bookKeeperClient); + CompactionServiceFactory compactionServiceFactory = builder.compactionServiceFactory; + if (builder.compactionServiceFactory == null && builder.config.getCompactionServiceFactoryClassName() + .equals(PulsarCompactionServiceFactory.class.getName())) { + compactionServiceFactory = new MockPulsarCompactionServiceFactory(spyConfig, builder.compactor); + } + Consumer openTelemetrySdkBuilderCustomizer; + if (builder.enableOpenTelemetry) { + var reader = InMemoryMetricReader.create(); + openTelemetryMetricReader(reader); + registerCloseable(reader); + openTelemetrySdkBuilderCustomizer = BrokerOpenTelemetryTestUtil.getOpenTelemetrySdkBuilderConsumer(reader); + } else { + openTelemetrySdkBuilderCustomizer = null; + } PulsarService pulsarService = spyConfig.getPulsarService() .spy(StartableTestPulsarService.class, spyConfig, builder.config, builder.localMetadataStore, - builder.configurationMetadataStore, builder.compactor, builder.brokerInterceptor, - bookKeeperClientFactory, builder.brokerServiceCustomizer); + builder.configurationMetadataStore, compactionServiceFactory, + builder.brokerInterceptor, + bookKeeperClientFactory, builder.brokerServiceCustomizer, + openTelemetrySdkBuilderCustomizer); + if (compactionServiceFactory != null) { + compactionServiceFactory.initialize(pulsarService); + } registerCloseable(() -> { pulsarService.close(); resetSpyOrMock(pulsarService); @@ -698,7 +800,7 @@ protected void initializePulsarServices(SpyConfig spyConfig, Builder builder) { if (metadataStore == null) { metadataStore = builder.configurationMetadataStore; } - NamespaceResources nsr = spyConfigPulsarResources.spy(NamespaceResources.class, metadataStore, 30); + NamespaceResources nsr = spyConfigPulsarResources.spy(NamespaceResources.class,metadataStore, 30); TopicResources tsr = spyConfigPulsarResources.spy(TopicResources.class, metadataStore); pulsarResources( spyConfigPulsarResources.spy( @@ -713,11 +815,20 @@ protected void initializePulsarServices(SpyConfig spyConfig, Builder builder) { } BookKeeperClientFactory bookKeeperClientFactory = new MockBookKeeperClientFactory(builder.bookKeeperClient); + CompactionServiceFactory compactionServiceFactory = builder.compactionServiceFactory; + if (builder.compactionServiceFactory == null && builder.config.getCompactionServiceFactoryClassName() + .equals(PulsarCompactionServiceFactory.class.getName())) { + compactionServiceFactory = new MockPulsarCompactionServiceFactory(spyConfig, builder.compactor); + } PulsarService pulsarService = spyConfig.getPulsarService() .spy(NonStartableTestPulsarService.class, spyConfig, builder.config, builder.localMetadataStore, - builder.configurationMetadataStore, builder.compactor, builder.brokerInterceptor, + builder.configurationMetadataStore, compactionServiceFactory, + builder.brokerInterceptor, bookKeeperClientFactory, builder.pulsarResources, builder.managedLedgerClientFactory, builder.brokerServiceCustomizer); + if (compactionServiceFactory != null) { + compactionServiceFactory.initialize(pulsarService); + } registerCloseable(() -> { pulsarService.close(); resetSpyOrMock(pulsarService); @@ -733,9 +844,8 @@ private static ManagedLedgerStorage createManagedLedgerClientFactory(BookKeeper @Override public void initialize(ServiceConfiguration conf, MetadataStoreExtended metadataStore, - BookKeeperClientFactory bookkeeperProvider, EventLoopGroup eventLoopGroup) - throws Exception { - + BookKeeperClientFactory bookkeeperProvider, EventLoopGroup eventLoopGroup, + OpenTelemetry openTelemetry) { } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/SpyConfig.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/SpyConfig.java index de51cee8f2449..64789d1f0d487 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/SpyConfig.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/SpyConfig.java @@ -98,9 +98,15 @@ public T spy(Class clazz, Object... args) { */ private final SpyType bookKeeperClient; /** - * Spy configuration for {@link PulsarService#getCompactor()}. + * Spy configuration for {@link PulsarService#getCompactionServiceFactory#getCompactor()}. */ private final SpyType compactor; + + /** + * Spy configuration for {@link PulsarService#getCompactionServiceFactory()}. + */ + + private final SpyType compactedServiceFactory; /** * Spy configuration for {@link PulsarService#getNamespaceService()}. */ @@ -123,12 +129,17 @@ public static Builder builder() { */ public static Builder builder(SpyType defaultSpyType) { Builder spyConfigBuilder = new Builder(); + configureDefaults(spyConfigBuilder, defaultSpyType); + return spyConfigBuilder; + } + + public static void configureDefaults(Builder spyConfigBuilder, SpyType defaultSpyType) { spyConfigBuilder.pulsarService(defaultSpyType); spyConfigBuilder.pulsarResources(defaultSpyType); spyConfigBuilder.brokerService(defaultSpyType); spyConfigBuilder.bookKeeperClient(defaultSpyType); spyConfigBuilder.compactor(defaultSpyType); + spyConfigBuilder.compactedServiceFactory(defaultSpyType); spyConfigBuilder.namespaceService(defaultSpyType); - return spyConfigBuilder; } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/StartableTestPulsarService.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/StartableTestPulsarService.java index 8a485a07496aa..a0774414492dc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/StartableTestPulsarService.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/StartableTestPulsarService.java @@ -19,6 +19,8 @@ package org.apache.pulsar.broker.testcontext; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdkBuilder; +import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; import org.apache.pulsar.broker.BookKeeperClientFactory; @@ -28,7 +30,7 @@ import org.apache.pulsar.broker.intercept.BrokerInterceptor; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerService; -import org.apache.pulsar.compaction.Compactor; +import org.apache.pulsar.compaction.CompactionServiceFactory; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; /** @@ -39,14 +41,15 @@ class StartableTestPulsarService extends AbstractTestPulsarService { private final Function brokerServiceCustomizer; public StartableTestPulsarService(SpyConfig spyConfig, ServiceConfiguration config, - MetadataStoreExtended localMetadataStore, - MetadataStoreExtended configurationMetadataStore, - Compactor compactor, - BrokerInterceptor brokerInterceptor, - BookKeeperClientFactory bookKeeperClientFactory, - Function brokerServiceCustomizer) { - super(spyConfig, config, localMetadataStore, configurationMetadataStore, compactor, brokerInterceptor, - bookKeeperClientFactory); + MetadataStoreExtended localMetadataStore, + MetadataStoreExtended configurationMetadataStore, + CompactionServiceFactory compactionServiceFactory, + BrokerInterceptor brokerInterceptor, + BookKeeperClientFactory bookKeeperClientFactory, + Function brokerServiceCustomizer, + Consumer openTelemetrySdkBuilderCustomizer) { + super(spyConfig, config, localMetadataStore, configurationMetadataStore, compactionServiceFactory, + brokerInterceptor, bookKeeperClientFactory, openTelemetrySdkBuilderCustomizer); this.brokerServiceCustomizer = brokerServiceCustomizer; } @@ -59,4 +62,4 @@ protected BrokerService newBrokerService(PulsarService pulsar) throws Exception public Supplier getNamespaceServiceProvider() throws PulsarServerException { return () -> spyConfig.getNamespaceService().spy(NamespaceService.class, this); } -} +} \ No newline at end of file diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/tools/BrokerToolTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/tools/BrokerToolTest.java index ad2cf7784eb1f..063c041f2e0fe 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/tools/BrokerToolTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/tools/BrokerToolTest.java @@ -19,12 +19,12 @@ package org.apache.pulsar.broker.tools; import static org.testng.Assert.assertTrue; -import com.beust.jcommander.Parameter; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.lang.reflect.Field; import java.util.Arrays; import org.testng.annotations.Test; +import picocli.CommandLine.Option; /** * Broker Tool Tests. @@ -47,12 +47,12 @@ public void testGenerateDocs() throws Exception { String message = baoStream.toString(); - Class argumentsClass = Class.forName("org.apache.pulsar.broker.tools.LoadReportCommand$Flags"); + Class argumentsClass = Class.forName("org.apache.pulsar.broker.tools.LoadReportCommand"); Field[] fields = argumentsClass.getDeclaredFields(); for (Field field : fields) { - boolean fieldHasAnno = field.isAnnotationPresent(Parameter.class); + boolean fieldHasAnno = field.isAnnotationPresent(Option.class); if (fieldHasAnno) { - Parameter fieldAnno = field.getAnnotation(Parameter.class); + Option fieldAnno = field.getAnnotation(Option.class); String[] names = fieldAnno.names(); String nameStr = Arrays.asList(names).toString(); nameStr = nameStr.substring(1, nameStr.length() - 1); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java index cb15ab003f7b7..d9ba825f02e93 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java @@ -18,34 +18,41 @@ */ package org.apache.pulsar.broker.transaction; -import static org.junit.Assert.assertEquals; +import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertNotNull; import java.lang.reflect.Field; import java.util.LinkedList; import java.util.NavigableMap; +import java.util.Optional; import java.util.Queue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.commons.collections4.map.LinkedMap; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService.ReferenceCountedWriter; +import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; +import org.apache.pulsar.broker.transaction.buffer.impl.SingleSnapshotAbortedTxnProcessorImpl; import org.apache.pulsar.broker.transaction.buffer.impl.SnapshotSegmentAbortedTxnProcessorImpl; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexes; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotSegment; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.admin.Transactions; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Producer; @@ -55,7 +62,9 @@ import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.TopicStats; -import org.testcontainers.shaded.org.awaitility.Awaitility; +import org.apache.pulsar.common.policies.data.TransactionBufferStats; +import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -71,11 +80,13 @@ public class SegmentAbortedTxnProcessorTest extends TransactionTestBase { @Override @BeforeClass protected void setup() throws Exception { - setUpBase(1, 1, PROCESSOR_TOPIC, 0); + setUpBase(1, 1, null, 0); this.pulsarService = getPulsarServiceList().get(0); this.pulsarService.getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); this.pulsarService.getConfig().setTransactionBufferSnapshotSegmentSize(8 + PROCESSOR_TOPIC.length() + SEGMENT_SIZE * 3); + admin.topics().createNonPartitionedTopic(PROCESSOR_TOPIC); + assertTrue(getSnapshotAbortedTxnProcessor(PROCESSOR_TOPIC) instanceof SnapshotSegmentAbortedTxnProcessorImpl); } @Override @@ -102,20 +113,20 @@ public void testPutAbortedTxnIntoProcessor() throws Exception { //1.1 Put 10 aborted txn IDs to persistent two sealed segments. for (int i = 0; i < 10; i++) { TxnID txnID = new TxnID(0, i); - PositionImpl position = new PositionImpl(0, i); + Position position = PositionFactory.create(0, i); processor.putAbortedTxnAndPosition(txnID, position); } //1.2 Put 4 aborted txn IDs into the unsealed segment. for (int i = 10; i < 14; i++) { TxnID txnID = new TxnID(0, i); - PositionImpl position = new PositionImpl(0, i); + Position position = PositionFactory.create(0, i); processor.putAbortedTxnAndPosition(txnID, position); } //1.3 Verify the common data flow verifyAbortedTxnIDAndSegmentIndex(processor, 0, 14); //2. Take the latest snapshot and verify recover from snapshot AbortedTxnProcessor newProcessor = new SnapshotSegmentAbortedTxnProcessorImpl(persistentTopic); - PositionImpl maxReadPosition = new PositionImpl(0, 14); + Position maxReadPosition = PositionFactory.create(0, 14); //2.1 Avoid update operation being canceled. waitTaskExecuteCompletely(processor); //2.2 take the latest snapshot @@ -147,7 +158,7 @@ private void waitTaskExecuteCompletely(AbortedTxnProcessor processor) throws Exc .getDeclaredField("taskQueue"); taskQueueField.setAccessible(true); Queue queue = (Queue) taskQueueField.get(persistentWorker); - Awaitility.await().untilAsserted(() -> Assert.assertEquals(queue.size(), 0)); + Awaitility.await().untilAsserted(() -> assertEquals(queue.size(), 0)); } private void verifyAbortedTxnIDAndSegmentIndex(AbortedTxnProcessor processor, int begin, int txnIdSize) @@ -164,9 +175,9 @@ private void verifyAbortedTxnIDAndSegmentIndex(AbortedTxnProcessor processor, in unsealedSegmentField.setAccessible(true); indexField.setAccessible(true); LinkedList unsealedSegment = (LinkedList) unsealedSegmentField.get(processor); - LinkedMap indexes = (LinkedMap) indexField.get(processor); - Assert.assertEquals(unsealedSegment.size(), txnIdSize % SEGMENT_SIZE); - Assert.assertEquals(indexes.size(), txnIdSize / SEGMENT_SIZE); + LinkedMap indexes = (LinkedMap) indexField.get(processor); + assertEquals(unsealedSegment.size(), txnIdSize % SEGMENT_SIZE); + assertEquals(indexes.size(), txnIdSize / SEGMENT_SIZE); } // Verify the update index future can be completed when the queue has other tasks. @@ -187,7 +198,7 @@ public void testFuturesCanCompleteWhenItIsCanceled() throws Exception { queue.add(new MutablePair<>(SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker.OperationType.WriteSegment, new MutablePair<>(new CompletableFuture<>(), task))); try { - processor.takeAbortedTxnsSnapshot(new PositionImpl(1, 10)).get(2, TimeUnit.SECONDS); + processor.takeAbortedTxnsSnapshot(PositionFactory.create(1, 10)).get(2, TimeUnit.SECONDS); fail("The update index operation should fail."); } catch (Exception e) { Assert.assertTrue(e.getCause() instanceof BrokerServiceException.ServiceUnitNotReadyException); @@ -204,7 +215,7 @@ public void testClearSnapshotSegments() throws Exception { //1. Write two snapshot segment. for (int j = 0; j < SEGMENT_SIZE * 2; j++) { TxnID txnID = new TxnID(0, j); - PositionImpl position = new PositionImpl(0, j); + Position position = PositionFactory.create(0, j); processor.putAbortedTxnAndPosition(txnID, position); } Awaitility.await().untilAsserted(() -> verifySnapshotSegmentsSize(PROCESSOR_TOPIC, 2)); @@ -224,7 +235,7 @@ public void testClearSnapshotSegments() throws Exception { //3. Try to write a snapshot segment that will fail to update indexes. for (int j = 0; j < SEGMENT_SIZE; j++) { TxnID txnID = new TxnID(0, j); - PositionImpl position = new PositionImpl(0, j); + Position position = PositionFactory.create(0, j); processor.putAbortedTxnAndPosition(txnID, position); } //4. Wait writing segment completed. @@ -251,6 +262,85 @@ public void testClearSnapshotSegments() throws Exception { processor.closeAsync().get(5, TimeUnit.SECONDS); } + @Test + public void testTxnSegmentStats() throws Exception { + // Set up test environment + String namespace = TENANT + "/testTxnSegmentStats"; + String topic = "persistent://" + namespace + "/testTxnSegmentStats"; + this.pulsarService.getConfig().setTransactionBufferSnapshotSegmentSize(8 + topic.length() + SEGMENT_SIZE * 3); + + // Create necessary resources + Transactions transactions = admin.transactions(); + admin.namespaces().createNamespace(namespace); + admin.topics().createNonPartitionedTopic(topic); + + // Prepare topic, producer, and consumer + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topic).create(); + @Cleanup + Consumer consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("my-sub").subscribe(); + + // Record the start time of the test + long testStartTime = System.currentTimeMillis(); + + Transaction transaction = null; + // Send messages with transactions and abort them + for (int i = 0; i < SEGMENT_SIZE; i++) { + transaction = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.HOURS).build().get(); + producer.newMessage(transaction).send(); + transaction.abort().get(); + } + + Transaction txn = pulsarClient.newTransaction().withTransactionTimeout(5, TimeUnit.HOURS).build().get(); + producer.newMessage(txn).send(); + txn.abort().get(); + + // Get the transaction buffer stats without segment stats + TransactionBufferStats statsWithoutSegmentStats = transactions + .getTransactionBufferStats(topic, false, false); + assertNotNull(statsWithoutSegmentStats); + assertNotNull(statsWithoutSegmentStats.segmentsStats); + assertNull(statsWithoutSegmentStats.segmentsStats.segmentStats); + assertEquals(statsWithoutSegmentStats.snapshotType, AbortedTxnProcessor.SnapshotType.Segment.toString()); + + // Verify the segment stats + assertEquals(statsWithoutSegmentStats.segmentsStats.segmentsSize, 1L); + assertEquals(statsWithoutSegmentStats.segmentsStats.unsealedAbortTxnIDSize, 1L); + assertEquals(statsWithoutSegmentStats.segmentsStats.currentSegmentCapacity, SEGMENT_SIZE); + assertEquals(statsWithoutSegmentStats.totalAbortedTransactions, SEGMENT_SIZE + 1); + assertTrue(statsWithoutSegmentStats.segmentsStats.lastTookSnapshotSegmentTimestamp >= testStartTime); + + // Get the transaction buffer stats with segment stats + TransactionBufferStats statsWithSegmentStats = transactions + .getTransactionBufferStats(topic, false, true); + assertNotNull(statsWithSegmentStats); + assertNotNull(statsWithSegmentStats.segmentsStats.segmentStats); + + // Verify if the segment stats are present when requested + assertEquals(statsWithSegmentStats.segmentsStats.segmentStats.size(), 1); + assertEquals(statsWithSegmentStats.segmentsStats.segmentStats.get(0).lastTxnID, + transaction.getTxnID().toString()); + + // Verify multiple segments + for (int i = 0; i < SEGMENT_SIZE * 3; i++) { + transaction = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.HOURS).build().get(); + producer.newMessage(transaction).send(); + transaction.abort().get(); + } + statsWithSegmentStats = transactions.getTransactionBufferStats(topic, false, true); + + // Verify the segment stats + assertEquals(statsWithSegmentStats.segmentsStats.segmentsSize, 4L); + assertEquals(statsWithSegmentStats.segmentsStats.unsealedAbortTxnIDSize, 1L); + assertEquals(statsWithSegmentStats.totalAbortedTransactions, SEGMENT_SIZE * 4 + 1); + + // Reset the configuration + this.pulsarService.getConfig() + .setTransactionBufferSnapshotSegmentSize(8 + PROCESSOR_TOPIC.length() + SEGMENT_SIZE * 3); + } + private void verifySnapshotSegmentsSize(String topic, int size) throws Exception { SystemTopicClient.Reader reader = pulsarService.getTransactionBufferSnapshotServiceFactory() @@ -264,7 +354,7 @@ private void verifySnapshotSegmentsSize(String topic, int size) throws Exception segmentCount++; } } - Assert.assertEquals(segmentCount, size); + assertEquals(segmentCount, size); } private void verifySnapshotSegmentsIndexSize(String topic, int size) throws Exception { @@ -281,7 +371,7 @@ private void verifySnapshotSegmentsIndexSize(String topic, int size) throws Exce } System.out.printf("message.getValue().getTopicName() :" + message.getValue().getTopicName()); } - Assert.assertEquals(indexCount, size); + assertEquals(indexCount, size); } private void doCompaction(TopicName topic) throws Exception { @@ -311,15 +401,20 @@ private void doCompaction(TopicName topic) throws Exception { */ @Test public void testSnapshotProcessorUpgrade() throws Exception { + String NAMESPACE2 = TENANT + "/ns2"; + admin.namespaces().createNamespace(NAMESPACE2); this.pulsarService = getPulsarServiceList().get(0); this.pulsarService.getConfig().setTransactionBufferSegmentedSnapshotEnabled(false); // Create a topic, send 10 messages without using transactions, and send 10 messages using transactions. // Abort these transactions and verify the data. - final String topicName = "persistent://" + NAMESPACE1 + "/testSnapshotProcessorUpgrade"; + final String topicName = "persistent://" + NAMESPACE2 + "/testSnapshotProcessorUpgrade"; + @Cleanup Producer producer = pulsarClient.newProducer().topic(topicName).create(); + @Cleanup Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("test-sub").subscribe(); + assertTrue(getSnapshotAbortedTxnProcessor(topicName) instanceof SingleSnapshotAbortedTxnProcessorImpl); // Send 10 messages without using transactions for (int i = 0; i < 10; i++) { producer.send(("test-message-" + i).getBytes()); @@ -352,6 +447,7 @@ public void testSnapshotProcessorUpgrade() throws Exception { // Unload the topic admin.topics().unload(topicName); + assertTrue(getSnapshotAbortedTxnProcessor(topicName) instanceof SnapshotSegmentAbortedTxnProcessorImpl); // Sends a new message using a transaction and aborts it. Transaction txn = pulsarClient.newTransaction() @@ -362,7 +458,7 @@ public void testSnapshotProcessorUpgrade() throws Exception { // Verifies that the topic has exactly one segment. Awaitility.await().untilAsserted(() -> { - String segmentTopic = "persistent://" + NAMESPACE1 + "/" + + String segmentTopic = "persistent://" + NAMESPACE2 + "/" + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS; TopicStats topicStats = admin.topics().getStats(segmentTopic); assertEquals(1, topicStats.getMsgInCounter()); @@ -399,9 +495,10 @@ public void testSegmentedSnapshotWithoutCreatingOldSnapshotTopic() throws Except // Create a new topic in the namespace String topicName = "persistent://" + namespaceName + "/newTopic"; + @Cleanup Producer producer = pulsarClient.newProducer().topic(topicName).create(); producer.close(); - + assertTrue(getSnapshotAbortedTxnProcessor(topicName) instanceof SnapshotSegmentAbortedTxnProcessorImpl); // Check that the __transaction_buffer_snapshot topic is not created in the same namespace String transactionBufferSnapshotTopic = "persistent://" + namespaceName + "/" + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT; @@ -415,4 +512,21 @@ public void testSegmentedSnapshotWithoutCreatingOldSnapshotTopic() throws Except // Destroy the namespace after the test admin.namespaces().deleteNamespace(namespaceName, true); } + + private AbortedTxnProcessor getSnapshotAbortedTxnProcessor(String topicName) { + PersistentTopic persistentTopic = getPersistentTopic(topicName); + return WhiteboxImpl.getInternalState(persistentTopic.getTransactionBuffer(), "snapshotAbortedTxnProcessor"); + } + + private PersistentTopic getPersistentTopic(String topicName) { + for (PulsarService pulsar : getPulsarServiceList()) { + CompletableFuture> future = + pulsar.getBrokerService().getTopic(topicName, false); + if (future == null) { + continue; + } + return (PersistentTopic) future.join().get(); + } + throw new NullPointerException("topic[" + topicName + "] not found"); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index 2d6622571c033..f21e11b980209 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -47,9 +47,10 @@ import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.mledger.ReadOnlyManagedLedger; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.bookkeeper.mledger.impl.ReadOnlyManagedLedgerImpl; import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.commons.collections4.map.LinkedMap; import org.apache.commons.lang3.RandomUtils; @@ -65,6 +66,7 @@ import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; import org.apache.pulsar.broker.transaction.buffer.impl.SingleSnapshotAbortedTxnProcessorImpl; +import org.apache.pulsar.broker.transaction.buffer.impl.TableView; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.metadata.TransactionBufferSnapshot; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndex; @@ -89,10 +91,8 @@ import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.events.EventType; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -200,20 +200,14 @@ private void recoverTest(String testTopic) throws Exception { Awaitility.await().until(() -> { for (int i = 0; i < getPulsarServiceList().size(); i++) { - Field field = BrokerService.class.getDeclaredField("topics"); - field.setAccessible(true); - ConcurrentOpenHashMap>> topics = - (ConcurrentOpenHashMap>>) field - .get(getPulsarServiceList().get(i).getBrokerService()); + final var topics = getPulsarServiceList().get(i).getBrokerService().getTopics(); CompletableFuture> completableFuture = topics.get("persistent://" + testTopic); if (completableFuture != null) { Optional topic = completableFuture.get(); if (topic.isPresent()) { PersistentTopic persistentTopic = (PersistentTopic) topic.get(); - field = PersistentTopic.class.getDeclaredField("transactionBuffer"); - field.setAccessible(true); TopicTransactionBuffer topicTransactionBuffer = - (TopicTransactionBuffer) field.get(persistentTopic); + (TopicTransactionBuffer) persistentTopic.getTransactionBuffer(); if (topicTransactionBuffer.checkIfReady()) { return true; } else { @@ -454,17 +448,13 @@ private void testTopicTransactionBufferDeleteAbort(Boolean enableSnapshotSegment assertTrue(((MessageIdImpl) messageId2).getLedgerId() != ((MessageIdImpl) messageId1).getLedgerId()); boolean exist = false; for (int i = 0; i < getPulsarServiceList().size(); i++) { - Field field = BrokerService.class.getDeclaredField("topics"); - field.setAccessible(true); - ConcurrentOpenHashMap>> topics = - (ConcurrentOpenHashMap>>) field - .get(getPulsarServiceList().get(i).getBrokerService()); + final var topics = getPulsarServiceList().get(i).getBrokerService().getTopics(); CompletableFuture> completableFuture = topics.get("persistent://" + ABORT_DELETE); if (completableFuture != null) { Optional topic = completableFuture.get(); if (topic.isPresent()) { PersistentTopic persistentTopic = (PersistentTopic) topic.get(); - field = ManagedLedgerImpl.class.getDeclaredField("ledgers"); + var field = ManagedLedgerImpl.class.getDeclaredField("ledgers"); field.setAccessible(true); NavigableMap ledgers = (NavigableMap) field.get(persistentTopic.getManagedLedger()); @@ -491,8 +481,8 @@ private void testTopicTransactionBufferDeleteAbort(Boolean enableSnapshotSegment Field abortsField = SingleSnapshotAbortedTxnProcessorImpl.class.getDeclaredField("aborts"); abortsField.setAccessible(true); - LinkedMap linkedMap = - (LinkedMap) abortsField.get(abortedTxnProcessor); + LinkedMap linkedMap = + (LinkedMap) abortsField.get(abortedTxnProcessor); assertEquals(linkedMap.size(), 1); assertEquals(linkedMap.get(linkedMap.firstKey()).getLedgerId(), ((MessageIdImpl) message.getMessageId()).getLedgerId()); @@ -510,6 +500,7 @@ public void clearTransactionBufferSnapshotTest(Boolean enableSnapshotSegment) th getPulsarServiceList().get(0).getConfig().setTransactionBufferSegmentedSnapshotEnabled(enableSnapshotSegment); String topic = NAMESPACE1 + "/tb-snapshot-delete-" + RandomUtils.nextInt(); + @Cleanup Producer producer = pulsarClient .newProducer() .topic(topic) @@ -559,6 +550,7 @@ private void checkSnapshotCount(TopicName topicName, boolean hasSnapshot, CompletableFuture compactionFuture = (CompletableFuture) field.get(persistentTopic); Awaitility.await().untilAsserted(() -> assertTrue(compactionFuture.isDone())); + @Cleanup Reader reader = pulsarClient.newReader(Schema.AUTO_CONSUME()) .readCompacted(true) .startMessageId(MessageId.earliest) @@ -579,6 +571,19 @@ private void checkSnapshotCount(TopicName topicName, boolean hasSnapshot, reader.close(); } + static class MockTableView extends TableView { + + public MockTableView(PulsarService pulsar) { + super(topic -> pulsar.getTransactionBufferSnapshotServiceFactory().getTxnBufferSnapshotService() + .createReader(topic), 30000L, pulsar.getExecutor()); + } + + @Override + public SystemTopicClient.Reader getReader(String topic) { + return readerCreator.apply(TopicName.get(topic)).join(); + } + } + @Test(timeOut=30000) public void testTransactionBufferRecoverThrowException() throws Exception { String topic = NAMESPACE1 + "/testTransactionBufferRecoverThrowPulsarClientException"; @@ -609,6 +614,7 @@ public void testTransactionBufferRecoverThrowException() throws Exception { doReturn(CompletableFuture.completedFuture(reader)) .when(systemTopicTxnBufferSnapshotService).createReader(any()); doReturn(refCounterWriter).when(systemTopicTxnBufferSnapshotService).getReferenceWriter(any()); + doReturn(new MockTableView(pulsarServiceList.get(0))).when(systemTopicTxnBufferSnapshotService).getTableView(); TransactionBufferSnapshotServiceFactory transactionBufferSnapshotServiceFactory = mock(TransactionBufferSnapshotServiceFactory.class); doReturn(systemTopicTxnBufferSnapshotService) @@ -660,7 +666,8 @@ private void checkCloseTopic(PulsarClient pulsarClient, PersistentTopic originalTopic, Field field, Producer producer) throws Exception { - field.set(getPulsarServiceList().get(0), transactionBufferSnapshotServiceFactory); + final var pulsar = getPulsarServiceList().get(0); + field.set(pulsar, transactionBufferSnapshotServiceFactory); // recover again will throw then close topic new TopicTransactionBuffer(originalTopic); @@ -671,7 +678,7 @@ private void checkCloseTopic(PulsarClient pulsarClient, assertTrue((boolean) close.get(originalTopic)); }); - field.set(getPulsarServiceList().get(0), transactionBufferSnapshotServiceFactoryOriginal); + field.set(pulsar, transactionBufferSnapshotServiceFactoryOriginal); Transaction txn = pulsarClient.newTransaction() .withTransactionTimeout(5, TimeUnit.SECONDS) @@ -681,29 +688,11 @@ private void checkCloseTopic(PulsarClient pulsarClient, txn.commit().get(); } - - @Test - public void testTransactionBufferNoSnapshotCloseReader() throws Exception{ - String topic = NAMESPACE1 + "/test"; - @Cleanup - Producer producer = pulsarClient.newProducer(Schema.STRING).producerName("testTxnTimeOut_producer") - .topic(topic).sendTimeout(0, TimeUnit.SECONDS).enableBatching(false).create(); - - admin.topics().unload(topic); - - // unload success, all readers have been closed except for the compaction sub - producer.send("test"); - TopicStats stats = admin.topics().getStats(NAMESPACE1 + "/" + TRANSACTION_BUFFER_SNAPSHOT); - - // except for the compaction sub - assertEquals(stats.getSubscriptions().size(), 1); - assertTrue(stats.getSubscriptions().keySet().contains("__compaction")); - } - @Test public void testTransactionBufferIndexSystemTopic() throws Exception { + final var pulsar = pulsarServiceList.get(0); SystemTopicTxnBufferSnapshotService transactionBufferSnapshotIndexService = - new TransactionBufferSnapshotServiceFactory(pulsarClient).getTxnBufferSnapshotIndexService(); + new TransactionBufferSnapshotServiceFactory(pulsar).getTxnBufferSnapshotIndexService(); SystemTopicClient.Writer indexesWriter = transactionBufferSnapshotIndexService.getReferenceWriter( @@ -763,9 +752,10 @@ public void testTransactionBufferSegmentSystemTopic() throws Exception { BrokerService brokerService = pulsarService.getBrokerService(); // create snapshot segment writer + final var pulsar = pulsarServiceList.get(0); SystemTopicTxnBufferSnapshotService transactionBufferSnapshotSegmentService = - new TransactionBufferSnapshotServiceFactory(pulsarClient).getTxnBufferSnapshotSegmentService(); + new TransactionBufferSnapshotServiceFactory(pulsar).getTxnBufferSnapshotSegmentService(); SystemTopicClient.Writer segmentWriter = transactionBufferSnapshotSegmentService @@ -795,9 +785,9 @@ public void testTransactionBufferSegmentSystemTopic() throws Exception { AsyncCallbacks.OpenReadOnlyManagedLedgerCallback callback = new AsyncCallbacks .OpenReadOnlyManagedLedgerCallback() { @Override - public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl readOnlyManagedLedger, Object ctx) { + public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedger readOnlyManagedLedger, Object ctx) { readOnlyManagedLedger.asyncReadEntry( - new PositionImpl(messageId.getLedgerId(), messageId.getEntryId()), + PositionFactory.create(messageId.getLedgerId(), messageId.getEntryId()), new AsyncCallbacks.ReadEntryCallback() { @Override public void readEntryComplete(Entry entry, Object ctx) { @@ -853,11 +843,13 @@ public void testSnapshotSegment() throws Exception { this.getPulsarServiceList().get(0).getConfig() .setTransactionBufferSnapshotMaxTransactionCount(theCountOfSnapshotMaxTxnCount); // 1. Build producer and consumer + @Cleanup Producer producer = pulsarClient.newProducer(Schema.INT32) .topic(topic) .enableBatching(false) .create(); + @Cleanup Consumer consumer = pulsarClient.newConsumer(Schema.INT32) .topic(topic) .subscriptionName(subName) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java index 78846fb75922a..25479e657d456 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java @@ -19,9 +19,12 @@ package org.apache.pulsar.broker.transaction; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertTrue; import com.google.common.collect.Sets; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -30,17 +33,24 @@ import java.util.concurrent.TimeUnit; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.persistent.MessageRedeliveryController; +import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.client.impl.ChunkMessageIdImpl; import org.apache.pulsar.common.api.proto.MessageIdData; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.api.proto.TxnAction; @@ -224,6 +234,62 @@ public void sortedTest() throws Exception { log.info("TransactionConsumeTest sortedTest finish."); } + @Test + public void testMessageRedelivery() throws Exception { + int transactionMessageCnt = 10; + String subName = "shared-test"; + + @Cleanup + Consumer sharedConsumer = pulsarClient.newConsumer() + .topic(CONSUME_TOPIC) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Shared) + .subscribe(); + + Awaitility.await().until(sharedConsumer::isConnected); + + long mostSigBits = 2L; + long leastSigBits = 5L; + TxnID txnID = new TxnID(mostSigBits, leastSigBits); + + // produce batch message with txn and then abort + PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService() + .getTopic(CONSUME_TOPIC, false).get().get(); + + List sendMessageList = new ArrayList<>(); + List messageIdDataList = appendTransactionMessages(txnID, persistentTopic, transactionMessageCnt, sendMessageList); + + persistentTopic.endTxn(txnID, TxnAction.ABORT_VALUE, 0L).get(); + log.info("Abort txn."); + + // redeliver transaction messages to shared consumer + PersistentSubscription subRef = persistentTopic.getSubscription(subName); + PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) subRef + .getDispatcher(); + Field redeliveryMessagesField = PersistentDispatcherMultipleConsumers.class + .getDeclaredField("redeliveryMessages"); + redeliveryMessagesField.setAccessible(true); + MessageRedeliveryController redeliveryMessages = new MessageRedeliveryController(true); + + final Field totalAvailablePermitsField = PersistentDispatcherMultipleConsumers.class + .getDeclaredField("totalAvailablePermits"); + totalAvailablePermitsField.setAccessible(true); + totalAvailablePermitsField.set(dispatcher, 1000); + + for (MessageIdData messageIdData : messageIdDataList) { + redeliveryMessages.add(messageIdData.getLedgerId(), messageIdData.getEntryId()); + } + + redeliveryMessagesField.set(dispatcher, redeliveryMessages); + dispatcher.readMoreEntries(); + + // shared consumer should not receive the redelivered aborted transaction messages + Message message = sharedConsumer.receive(5, TimeUnit.SECONDS); + Assert.assertNull(message); + + log.info("TransactionConsumeTest testMessageRedelivery finish."); + } + private void sendNormalMessages(Producer producer, int startMsgCnt, int messageCnt, List sendMessageList) throws PulsarClientException { @@ -258,7 +324,7 @@ private List appendTransactionMessages( ByteBuf headerAndPayload = Commands.serializeMetadataAndPayload( Commands.ChecksumType.Crc32c, metadata, Unpooled.copiedBuffer(msg.getBytes(UTF_8))); - CompletableFuture completableFuture = new CompletableFuture<>(); + CompletableFuture completableFuture = new CompletableFuture<>(); topic.publishTxnMessage(txnID, headerAndPayload, new Topic.PublishContext() { @Override @@ -298,7 +364,7 @@ public long getNumberOfMessages() { @Override public void completed(Exception e, long ledgerId, long entryId) { - completableFuture.complete(PositionImpl.get(ledgerId, entryId)); + completableFuture.complete(PositionFactory.create(ledgerId, entryId)); } }); positionList.add(new MessageIdData().setLedgerId(completableFuture.get() @@ -308,4 +374,45 @@ public void completed(Exception e, long ledgerId, long entryId) { return positionList; } + @Test + public void testAckChunkMessage() throws Exception { + String producerName = "test-producer"; + String subName = "testAckChunkMessage"; + @Cleanup + PulsarClient pulsarClient1 = PulsarClient.builder().serviceUrl(pulsarServiceList.get(0).getBrokerServiceUrl()) + .enableTransaction(true).build(); + @Cleanup + Producer producer = pulsarClient1 + .newProducer(Schema.STRING) + .producerName(producerName) + .topic(CONSUME_TOPIC) + .enableChunking(true) + .enableBatching(false) + .create(); + Consumer consumer = pulsarClient1 + .newConsumer(Schema.STRING) + .subscriptionType(SubscriptionType.Shared) + .topic(CONSUME_TOPIC) + .subscriptionName(subName) + .subscribe(); + + int messageSize = 6000; // payload size in KB + String message = "a".repeat(messageSize * 1000); + MessageId messageId = producer.newMessage().value(message).send(); + assertTrue(messageId instanceof ChunkMessageIdImpl); + assertNotEquals(((ChunkMessageIdImpl) messageId).getLastChunkMessageId(), + ((ChunkMessageIdImpl) messageId).getFirstChunkMessageId()); + + Transaction transaction = pulsarClient1.newTransaction() + .withTransactionTimeout(5, TimeUnit.HOURS) + .build() + .get(); + + Message msg = consumer.receive(); + consumer.acknowledgeAsync(msg.getMessageId(), transaction); + transaction.commit().get(); + + Assert.assertEquals(admin.topics().getStats(CONSUME_TOPIC).getSubscriptions().get(subName) + .getUnackedMessages(), 0); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java index ddd8cf0790321..14b1d563c11ec 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java @@ -19,11 +19,13 @@ package org.apache.pulsar.broker.transaction; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.testng.Assert.assertTrue; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; @@ -34,11 +36,14 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.ReadOnlyCursor; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -180,6 +185,37 @@ private void produceTest(boolean endAction) throws Exception { log.info("produce and {} test finished.", endAction ? "commit" : "abort"); } + @Test + public void testUpdateLastMaxReadPositionMovedForwardTimestampForTransactionalPublish() throws Exception { + final String topic = NAMESPACE1 + "/testUpdateLastMaxReadPositionMovedForwardTimestampForTransactionalPublish"; + PulsarClient pulsarClient = this.pulsarClient; + Transaction txn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build().get(); + @Cleanup + Producer producer = pulsarClient + .newProducer() + .topic(topic) + .sendTimeout(0, TimeUnit.SECONDS) + .create(); + PersistentTopic persistentTopic = getTopic(topic); + long lastMaxReadPositionMovedForwardTimestamp = persistentTopic.getLastMaxReadPositionMovedForwardTimestamp(); + + // transactional publish will not update lastMaxReadPositionMovedForwardTimestamp + producer.newMessage(txn).value("hello world".getBytes()).send(); + assertTrue(persistentTopic.getLastMaxReadPositionMovedForwardTimestamp() == lastMaxReadPositionMovedForwardTimestamp); + + // commit transaction will update lastMaxReadPositionMovedForwardTimestamp + txn.commit().get(); + assertTrue(persistentTopic.getLastMaxReadPositionMovedForwardTimestamp() > lastMaxReadPositionMovedForwardTimestamp); + } + + private PersistentTopic getTopic(String topic) throws ExecutionException, InterruptedException { + Optional optionalTopic = getPulsarServiceList().get(0).getBrokerService() + .getTopic(topic, true).get(); + return (PersistentTopic) optionalTopic.get(); + } + private void checkMessageId(List> futureList, boolean isFinished) { futureList.forEach(messageIdFuture -> { try { @@ -213,7 +249,7 @@ private ReadOnlyCursor getOriginTopicCursor(String topic, int partition) { } return getPulsarServiceList().get(0).getManagedLedgerFactory().openReadOnlyCursor( TopicName.get(topic).getPersistenceNamingEncoding(), - PositionImpl.EARLIEST, new ManagedLedgerConfig()); + PositionFactory.EARLIEST, new ManagedLedgerConfig()); } catch (Exception e) { log.error("Failed to get origin topic readonly cursor.", e); Assert.fail("Failed to get origin topic readonly cursor."); @@ -230,6 +266,7 @@ public void ackCommitTest() throws Exception { .build().get(); log.info("init transaction {}.", txn); + @Cleanup Producer incomingProducer = pulsarClient.newProducer() .topic(ACK_COMMIT_TOPIC) .batchingMaxMessages(1) @@ -241,6 +278,7 @@ public void ackCommitTest() throws Exception { } log.info("prepare incoming messages finished."); + @Cleanup Consumer consumer = pulsarClient.newConsumer() .topic(ACK_COMMIT_TOPIC) .subscriptionName(subscriptionName) @@ -292,6 +330,7 @@ public void ackAbortTest() throws Exception { .build().get(); log.info("init transaction {}.", txn); + @Cleanup Producer incomingProducer = pulsarClient.newProducer() .topic(ACK_ABORT_TOPIC) .batchingMaxMessages(1) @@ -303,6 +342,7 @@ public void ackAbortTest() throws Exception { } log.info("prepare incoming messages finished."); + @Cleanup Consumer consumer = pulsarClient.newConsumer() .topic(ACK_ABORT_TOPIC) .subscriptionName(subscriptionName) @@ -350,7 +390,7 @@ private int getPendingAckCount(String topic, String subscriptionName) throws Exc int pendingAckCount = 0; for (PulsarService pulsarService : getPulsarServiceList()) { - for (String key : pulsarService.getBrokerService().getTopics().keys()) { + for (String key : pulsarService.getBrokerService().getTopics().keySet()) { if (key.contains(topic)) { Field field = clazz.getDeclaredField("pendingAckHandle"); field.setAccessible(true); @@ -361,8 +401,8 @@ private int getPendingAckCount(String topic, String subscriptionName) throws Exc field = PendingAckHandleImpl.class.getDeclaredField("individualAckPositions"); field.setAccessible(true); - Map> map = - (Map>) field.get(pendingAckHandle); + Map> map = + (Map>) field.get(pendingAckHandle); if (map != null) { pendingAckCount += map.size(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index c4ec2ec766e32..5480b1a21d5a0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -19,10 +19,11 @@ package org.apache.pulsar.broker.transaction; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongSumValue; import static org.apache.pulsar.common.naming.SystemTopicNames.PENDING_ACK_STORE_CURSOR_NAME; import static org.apache.pulsar.common.naming.SystemTopicNames.PENDING_ACK_STORE_SUFFIX; -import static org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl.TRANSACTION_LOG_PREFIX; import static org.apache.pulsar.transaction.coordinator.impl.DisabledTxnLogBufferedWriterMetricsStats.DISABLED_BUFFERED_WRITER_METRICS; +import static org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl.TRANSACTION_LOG_PREFIX; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.doAnswer; @@ -40,14 +41,15 @@ import io.netty.util.HashedWheelTimer; import io.netty.util.Timeout; import io.netty.util.concurrent.DefaultThreadFactory; +import io.opentelemetry.api.common.Attributes; import java.lang.reflect.Field; import java.lang.reflect.Method; -import java.util.ArrayList; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.Map; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -69,16 +71,19 @@ import org.apache.bookkeeper.common.util.Bytes; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerFactory; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedCursorContainer; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.commons.lang3.reflect.MethodUtils; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.intercept.CounterBrokerInterceptor; @@ -87,10 +92,12 @@ import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService; import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService.ReferenceCountedWriter; +import org.apache.pulsar.broker.service.TopicPolicyTestUtils; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.TransactionBufferSnapshotServiceFactory; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.stats.OpenTelemetryTopicStats; import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; @@ -99,13 +106,14 @@ import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferProvider; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferRecoverCallBack; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferState; -import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; import org.apache.pulsar.broker.transaction.buffer.metadata.TransactionBufferSnapshot; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckReplyCallBack; import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore; import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStoreProvider; import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; +import org.apache.pulsar.client.admin.LongRunningProcessStatus; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -124,28 +132,30 @@ import org.apache.pulsar.client.impl.ConsumerBase; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.impl.MessagesImpl; +import org.apache.pulsar.client.impl.transaction.TransactionImpl; import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.common.api.proto.CommandSubscribe; -import org.apache.pulsar.client.impl.transaction.TransactionImpl; import org.apache.pulsar.common.events.EventType; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies; import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats; import org.apache.pulsar.common.policies.data.RetentionPolicies; -import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.apache.pulsar.compaction.CompactionServiceFactory; +import org.apache.pulsar.compaction.PulsarCompactionServiceFactory; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; import org.apache.pulsar.transaction.coordinator.TransactionMetadataStore; import org.apache.pulsar.transaction.coordinator.TransactionMetadataStoreState; import org.apache.pulsar.transaction.coordinator.TransactionRecoverTracker; import org.apache.pulsar.transaction.coordinator.TransactionTimeoutTracker; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl; -import org.apache.pulsar.transaction.coordinator.impl.MLTransactionSequenceIdGenerator; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStore; +import org.apache.pulsar.transaction.coordinator.impl.MLTransactionSequenceIdGenerator; import org.apache.pulsar.transaction.coordinator.impl.TxnLogBufferedWriterConfig; import org.awaitility.Awaitility; import org.mockito.invocation.InvocationOnMock; @@ -178,7 +188,7 @@ protected void cleanup() throws Exception { @Test public void testTopicTransactionMetrics() throws Exception { - final String topic = "persistent://tnx/ns1/test_transaction_topic"; + final String topic = BrokerTestUtil.newUniqueName("persistent://tnx/ns1/test_transaction_topic"); @Cleanup Producer producer = this.pulsarClient.newProducer() @@ -212,6 +222,49 @@ public void testTopicTransactionMetrics() throws Exception { assertEquals(stats.committedTxnCount, 1); assertEquals(stats.abortedTxnCount, 1); assertEquals(stats.ongoingTxnCount, 1); + + var attributes = Attributes.builder() + .put(OpenTelemetryAttributes.PULSAR_DOMAIN, "persistent") + .put(OpenTelemetryAttributes.PULSAR_TENANT, "tnx") + .put(OpenTelemetryAttributes.PULSAR_NAMESPACE, "tnx/ns1") + .put(OpenTelemetryAttributes.PULSAR_TOPIC, topic) + .build(); + + var metrics = pulsarTestContexts.get(0).getOpenTelemetryMetricReader().collectAllMetrics(); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.TRANSACTION_BUFFER_CLIENT_OPERATION_COUNTER, + Attributes.builder() + .putAll(attributes) + .remove(OpenTelemetryAttributes.PULSAR_DOMAIN) + .putAll(OpenTelemetryAttributes.TransactionStatus.COMMITTED.attributes) + .putAll(OpenTelemetryAttributes.TransactionBufferClientOperationStatus.SUCCESS.attributes) + .build(), + 1); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.TRANSACTION_BUFFER_CLIENT_OPERATION_COUNTER, + Attributes.builder() + .putAll(attributes) + .remove(OpenTelemetryAttributes.PULSAR_DOMAIN) + .putAll(OpenTelemetryAttributes.TransactionStatus.ABORTED.attributes) + .putAll(OpenTelemetryAttributes.TransactionBufferClientOperationStatus.SUCCESS.attributes) + .build(), + 1); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.TRANSACTION_COUNTER, + Attributes.builder() + .putAll(attributes) + .put(OpenTelemetryAttributes.PULSAR_TRANSACTION_STATUS, "committed") + .build(), + 1); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.TRANSACTION_COUNTER, + Attributes.builder() + .putAll(attributes) + .put(OpenTelemetryAttributes.PULSAR_TRANSACTION_STATUS, "aborted") + .build(), + 1); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.TRANSACTION_COUNTER, + Attributes.builder() + .putAll(attributes) + .put(OpenTelemetryAttributes.PULSAR_TRANSACTION_STATUS, "active") + .build(), + 1); } @Test @@ -331,8 +384,7 @@ public void brokerNotInitTxnManagedLedgerTopic() throws Exception { return true; }); - ConcurrentOpenHashMap>> topics = - getPulsarServiceList().get(0).getBrokerService().getTopics(); + final var topics = getPulsarServiceList().get(0).getBrokerService().getTopics(); Assert.assertNull(topics.get(TopicName.get(TopicDomain.persistent.value(), NamespaceName.SYSTEM_NAMESPACE, TRANSACTION_LOG_PREFIX).toString() + 0)); @@ -356,15 +408,18 @@ public void testAsyncSendOrAckForSingleFuture() throws Exception { int threadSize = 30; String topicName = "subscription"; getPulsarServiceList().get(0).getConfig().setBrokerDeduplicationEnabled(false); + @Cleanup("shutdownNow") ExecutorService executorService = Executors.newFixedThreadPool(threadSize); //build producer/consumer + @Cleanup Producer producer = pulsarClient.newProducer() .topic(topic) .producerName("producer") .sendTimeout(0, TimeUnit.SECONDS) .create(); + @Cleanup Consumer consumer = pulsarClient.newConsumer() .topic(topic) .subscriptionType(SubscriptionType.Exclusive) @@ -469,7 +524,8 @@ public void testSubscriptionRecreateTopic() } admin.topics().setRetention(topic, new RetentionPolicies(retentionSizeInMinutesSetTopic, retentionSizeInMbSetTopic)); - pulsarClient.newConsumer().topic(topic) + @Cleanup + final Consumer subscribe = pulsarClient.newConsumer().topic(topic) .subscriptionName(subName) .subscribe(); pulsarService.getBrokerService().getTopicIfExists(topic).thenAccept(option -> { @@ -492,11 +548,7 @@ public void testSubscriptionRecreateTopic() .getSubscription(subName); subscription.getPendingAckManageLedger().thenAccept(managedLedger -> { long retentionSize = managedLedger.getConfig().getRetentionSizeInMB(); - if (!originPersistentTopic.getTopicPolicies().isPresent()) { - log.error("Failed to getTopicPolicies of :" + originPersistentTopic); - Assert.fail(); - } - TopicPolicies topicPolicies = originPersistentTopic.getTopicPolicies().get(); + TopicPolicyTestUtils.getTopicPolicies(originPersistentTopic); // verify the topic policies exist Assert.assertEquals(retentionSizeInMbSetTopic, retentionSize); MLPendingAckStoreProvider mlPendingAckStoreProvider = new MLPendingAckStoreProvider(); CompletableFuture future = mlPendingAckStoreProvider.newPendingAckStore(subscription); @@ -531,11 +583,17 @@ public void testTakeSnapshotBeforeBuildTxnProducer() throws Exception { .untilAsserted(() -> Assert.assertFalse(reader.hasMessageAvailable())); //test take snapshot by build producer by the transactionEnable client + @Cleanup Producer producer = pulsarClient.newProducer(Schema.STRING) .producerName("testSnapshot").sendTimeout(0, TimeUnit.SECONDS) .topic(topic).enableBatching(true) .create(); + Transaction transaction = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS).build().get(); + producer.newMessage(transaction).send(); + transaction.abort().get(); + Awaitility.await().untilAsserted(() -> { Message message1 = reader.readNext(); TransactionBufferSnapshot snapshot1 = message1.getValue(); @@ -549,7 +607,7 @@ public void testTakeSnapshotBeforeBuildTxnProducer() throws Exception { Awaitility.await().untilAsserted(() -> { Message message1 = reader.readNext(); TransactionBufferSnapshot snapshot1 = message1.getValue(); - Assert.assertEquals(snapshot1.getMaxReadPositionEntryId(), 1); + Assert.assertEquals(snapshot1.getMaxReadPositionEntryId(), 3); }); } @@ -645,32 +703,33 @@ public void testMaxReadPositionForNormalPublish() throws Exception { //test publishing normal messages will change maxReadPosition in the state of NoSnapshot. MessageIdImpl messageId = (MessageIdImpl) normalProducer.newMessage().value("normal message").send(); - PositionImpl position = topicTransactionBuffer.getMaxReadPosition(); + Position position = topicTransactionBuffer.getMaxReadPosition(); Assert.assertEquals(position.getLedgerId(), messageId.getLedgerId()); Assert.assertEquals(position.getEntryId(), messageId.getEntryId()); //test the state of TransactionBuffer is Ready after build Producer by pulsarClient that enables transaction. + @Cleanup Producer txnProducer = pulsarClient.newProducer(Schema.STRING) .producerName("testTransactionPublish") .topic(topic) .sendTimeout(0, TimeUnit.SECONDS) .create(); - Awaitility.await().untilAsserted(() -> Assert.assertTrue(topicTransactionBuffer.checkIfReady())); + Awaitility.await().untilAsserted(() -> Assert.assertTrue(topicTransactionBuffer.checkIfNoSnapshot())); //test publishing txn messages will not change maxReadPosition if don`t commit or abort. Transaction transaction = pulsarClient.newTransaction() .withTransactionTimeout(5, TimeUnit.SECONDS).build().get(); MessageIdImpl messageId1 = (MessageIdImpl) txnProducer.newMessage(transaction).value("txn message").send(); - PositionImpl position1 = topicTransactionBuffer.getMaxReadPosition(); + Position position1 = topicTransactionBuffer.getMaxReadPosition(); Assert.assertEquals(position1.getLedgerId(), messageId.getLedgerId()); Assert.assertEquals(position1.getEntryId(), messageId.getEntryId()); MessageIdImpl messageId2 = (MessageIdImpl) normalProducer.newMessage().value("normal message").send(); - PositionImpl position2 = topicTransactionBuffer.getMaxReadPosition(); + Position position2 = topicTransactionBuffer.getMaxReadPosition(); Assert.assertEquals(position2.getLedgerId(), messageId.getLedgerId()); Assert.assertEquals(position2.getEntryId(), messageId.getEntryId()); transaction.commit().get(); - PositionImpl position3 = topicTransactionBuffer.getMaxReadPosition(); + Position position3 = topicTransactionBuffer.getMaxReadPosition(); Assert.assertEquals(position3.getLedgerId(), messageId2.getLedgerId()); Assert.assertEquals(position3.getEntryId(), messageId2.getEntryId() + 1); @@ -678,7 +737,7 @@ public void testMaxReadPositionForNormalPublish() throws Exception { //test publishing normal messages will change maxReadPosition if the state of TB //is Ready and ongoingTxns is empty. MessageIdImpl messageId4 = (MessageIdImpl) normalProducer.newMessage().value("normal message").send(); - PositionImpl position4 = topicTransactionBuffer.getMaxReadPosition(); + Position position4 = topicTransactionBuffer.getMaxReadPosition(); Assert.assertEquals(position4.getLedgerId(), messageId4.getLedgerId()); Assert.assertEquals(position4.getEntryId(), messageId4.getEntryId()); @@ -692,7 +751,7 @@ public void testMaxReadPositionForNormalPublish() throws Exception { maxReadPositionField.setAccessible(true); field.set(topicTransactionBuffer, TopicTransactionBufferState.State.Initializing); MessageIdImpl messageId5 = (MessageIdImpl) normalProducer.newMessage().value("normal message").send(); - PositionImpl position5 = (PositionImpl) maxReadPositionField.get(topicTransactionBuffer); + Position position5 = (Position) maxReadPositionField.get(topicTransactionBuffer); Assert.assertEquals(position5.getLedgerId(), messageId4.getLedgerId()); Assert.assertEquals(position5.getEntryId(), messageId4.getEntryId()); } @@ -741,7 +800,7 @@ public void testEndTBRecoveringWhenManagerLedgerDisReadable() throws Exception{ TransactionBuffer buffer2 = new TopicTransactionBuffer(persistentTopic); Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> - assertEquals(buffer2.getStats(false).state, "Ready")); + assertEquals(buffer2.getStats(false, false).state, "Ready")); managedCursors.removeCursor("transaction-buffer-sub"); doAnswer(invocation -> { @@ -753,7 +812,7 @@ public void testEndTBRecoveringWhenManagerLedgerDisReadable() throws Exception{ managedCursors.add(managedCursor, managedCursor.getMarkDeletedPosition()); TransactionBuffer buffer3 = new TopicTransactionBuffer(persistentTopic); Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> - assertEquals(buffer3.getStats(false).state, "Ready")); + assertEquals(buffer3.getStats(false, false).state, "Ready")); persistentTopic.getInternalStats(false).thenAccept(internalStats -> { assertTrue(internalStats.cursors.isEmpty()); }); @@ -787,7 +846,7 @@ public void testEndTPRecoveringWhenManagerLedgerDisReadable() throws Exception{ ManagedCursorImpl managedCursor = mock(ManagedCursorImpl.class); doReturn(true).when(managedCursor).hasMoreEntries(); doReturn(false).when(managedCursor).isClosed(); - doReturn(new PositionImpl(-1, -1)).when(managedCursor).getMarkDeletedPosition(); + doReturn(PositionFactory.create(-1, -1)).when(managedCursor).getMarkDeletedPosition(); doAnswer(invocation -> { AsyncCallbacks.ReadEntriesCallback callback = invocation.getArgument(1); callback.readEntriesFailed(new ManagedLedgerException.NonRecoverableLedgerException("No ledger exist"), @@ -799,7 +858,8 @@ public void testEndTPRecoveringWhenManagerLedgerDisReadable() throws Exception{ doReturn(CompletableFuture.completedFuture( new MLPendingAckStore(persistentTopic.getManagedLedger(), managedCursor, null, 500, bufferedWriterConfig, transactionTimer, - DISABLED_BUFFERED_WRITER_METRICS))) + DISABLED_BUFFERED_WRITER_METRICS, persistentTopic.getBrokerService().getPulsar() + .getOrderedExecutor().chooseThread()))) .when(pendingAckStoreProvider).newPendingAckStore(any()); doReturn(CompletableFuture.completedFuture(true)).when(pendingAckStoreProvider).checkInitializedBefore(any()); @@ -866,9 +926,10 @@ public void testEndTCRecoveringWhenManagerLedgerDisReadable() throws Exception{ MLTransactionSequenceIdGenerator mlTransactionSequenceIdGenerator = new MLTransactionSequenceIdGenerator(); persistentTopic.getManagedLedger().getConfig().setManagedLedgerInterceptor(mlTransactionSequenceIdGenerator); MLTransactionLogImpl mlTransactionLog = - new MLTransactionLogImpl(new TransactionCoordinatorID(1), null, + spy(new MLTransactionLogImpl(new TransactionCoordinatorID(1), null, persistentTopic.getManagedLedger().getConfig(), new TxnLogBufferedWriterConfig(), - transactionTimer, DISABLED_BUFFERED_WRITER_METRICS); + transactionTimer, DISABLED_BUFFERED_WRITER_METRICS)); + doAnswer(__ -> CompletableFuture.completedFuture(null)).when(mlTransactionLog).closeAsync(); Class mlTransactionLogClass = MLTransactionLogImpl.class; Field field = mlTransactionLogClass.getDeclaredField("cursor"); field.setAccessible(true); @@ -882,6 +943,7 @@ public void testEndTCRecoveringWhenManagerLedgerDisReadable() throws Exception{ doNothing().when(transactionRecoverTracker).handleCommittingAndAbortingTransaction(); TransactionTimeoutTracker timeoutTracker = mock(TransactionTimeoutTracker.class); doNothing().when(timeoutTracker).start(); + @Cleanup("closeAsync") MLTransactionMetadataStore metadataStore1 = new MLTransactionMetadataStore(new TransactionCoordinatorID(1), mlTransactionLog, timeoutTracker, mlTransactionSequenceIdGenerator, 0L); @@ -895,6 +957,7 @@ public void testEndTCRecoveringWhenManagerLedgerDisReadable() throws Exception{ return null; }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any()); + @Cleanup("closeAsync") MLTransactionMetadataStore metadataStore2 = new MLTransactionMetadataStore(new TransactionCoordinatorID(1), @@ -909,6 +972,7 @@ public void testEndTCRecoveringWhenManagerLedgerDisReadable() throws Exception{ return null; }).when(managedCursor).asyncReadEntries(anyInt(), any(), any(), any()); + @Cleanup("closeAsync") MLTransactionMetadataStore metadataStore3 = new MLTransactionMetadataStore(new TransactionCoordinatorID(1), mlTransactionLog, timeoutTracker, mlTransactionSequenceIdGenerator, 0L); @@ -976,7 +1040,7 @@ public void testNoEntryCanBeReadWhenRecovery() throws Exception { topicTransactionBuffer.getMaxReadPosition()); completableFuture.get(); - doReturn(PositionImpl.LATEST).when(managedLedger).getLastConfirmedEntry(); + doReturn(PositionFactory.LATEST).when(managedLedger).getLastConfirmedEntry(); ManagedCursorImpl managedCursor = mock(ManagedCursorImpl.class); doReturn(false).when(managedCursor).hasMoreEntries(); doReturn(managedCursor).when(managedLedger).newNonDurableCursor(any(), any()); @@ -1049,10 +1113,10 @@ public void testCancelTxnTimeout() throws Exception{ } @Test - public void testNotChangeMaxReadPositionAndAddAbortTimesWhenCheckIfNoSnapshot() throws Exception { + public void testNotChangeMaxReadPositionCountWhenCheckIfNoSnapshot() throws Exception { PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0) .getBrokerService() - .getTopic(NAMESPACE1 + "/changeMaxReadPositionAndAddAbortTimes" + UUID.randomUUID(), true) + .getTopic(NAMESPACE1 + "/changeMaxReadPositionCount" + UUID.randomUUID(), true) .get().get(); TransactionBuffer buffer = persistentTopic.getTransactionBuffer(); Field processorField = TopicTransactionBuffer.class.getDeclaredField("snapshotAbortedTxnProcessor"); @@ -1060,9 +1124,9 @@ public void testNotChangeMaxReadPositionAndAddAbortTimesWhenCheckIfNoSnapshot() AbortedTxnProcessor abortedTxnProcessor = (AbortedTxnProcessor) processorField.get(buffer); Field changeTimeField = TopicTransactionBuffer - .class.getDeclaredField("changeMaxReadPositionAndAddAbortTimes"); + .class.getDeclaredField("changeMaxReadPositionCount"); changeTimeField.setAccessible(true); - AtomicLong changeMaxReadPositionAndAddAbortTimes = (AtomicLong) changeTimeField.get(buffer); + AtomicLong changeMaxReadPositionCount = (AtomicLong) changeTimeField.get(buffer); Field field1 = TopicTransactionBufferState.class.getDeclaredField("state"); field1.setAccessible(true); @@ -1071,10 +1135,10 @@ public void testNotChangeMaxReadPositionAndAddAbortTimesWhenCheckIfNoSnapshot() TopicTransactionBufferState.State state = (TopicTransactionBufferState.State) field1.get(buffer); Assert.assertEquals(state, TopicTransactionBufferState.State.NoSnapshot); }); - Assert.assertEquals(changeMaxReadPositionAndAddAbortTimes.get(), 0L); + Assert.assertEquals(changeMaxReadPositionCount.get(), 0L); - buffer.syncMaxReadPositionForNormalPublish(new PositionImpl(1, 1)); - Assert.assertEquals(changeMaxReadPositionAndAddAbortTimes.get(), 0L); + buffer.syncMaxReadPositionForNormalPublish(PositionFactory.create(1, 1), false); + Assert.assertEquals(changeMaxReadPositionCount.get(), 0L); } @@ -1366,7 +1430,7 @@ public void testGetConnectExceptionForAckMsgWhenCnxIsNull() throws Exception { producer.newMessage().value(Bytes.toBytes(i)).send(); } ClientCnx cnx = (ClientCnx) MethodUtils.invokeMethod(consumer, true, "cnx"); - MethodUtils.invokeMethod(consumer, true, "connectionClosed", cnx); + MethodUtils.invokeMethod(consumer, true, "connectionClosed", cnx, Optional.empty(), Optional.empty()); Message message = consumer.receive(); Transaction transaction = pulsarClient @@ -1444,6 +1508,7 @@ public void testPendingAckBatchMessageCommit() throws Exception { public void testPendingAckReplayChangeStateError() throws InterruptedException, TimeoutException { AtomicInteger atomicInteger = new AtomicInteger(1); // Create Executor + @Cleanup("shutdownNow") ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); // Mock serviceConfiguration. ServiceConfiguration serviceConfiguration = mock(ServiceConfiguration.class); @@ -1502,7 +1567,7 @@ public Object answer(InvocationOnMock invocation) throws Throwable { fail("Expect failure by PendingAckHandle closed, but success"); } catch (ExecutionException executionException){ Throwable t = executionException.getCause(); - Assert.assertTrue(t instanceof BrokerServiceException.ServiceUnitNotReadyException); + Assert.assertTrue(t instanceof BrokerServiceException); } } @@ -1571,6 +1636,9 @@ public void testTBRecoverChangeStateError() throws InterruptedException, Timeout when(pulsar.getTransactionBufferSnapshotServiceFactory()).thenReturn(transactionBufferSnapshotServiceFactory); TopicTransactionBufferProvider topicTransactionBufferProvider = new TopicTransactionBufferProvider(); when(pulsar.getTransactionBufferProvider()).thenReturn(topicTransactionBufferProvider); + CompactionServiceFactory compactionServiceFactory = new PulsarCompactionServiceFactory(); + compactionServiceFactory.initialize(pulsar); + when(pulsar.getCompactionServiceFactory()).thenReturn(compactionServiceFactory); // Mock BacklogQuotaManager BacklogQuotaManager backlogQuotaManager = mock(BacklogQuotaManager.class); // Mock brokerService. @@ -1581,14 +1649,15 @@ public void testTBRecoverChangeStateError() throws InterruptedException, Timeout // Mock managedLedger. ManagedLedgerImpl managedLedger = mock(ManagedLedgerImpl.class); ManagedCursorContainer managedCursors = new ManagedCursorContainer(); + when(managedLedger.getConfig()).thenReturn(new ManagedLedgerConfig()); when(managedLedger.getCursors()).thenReturn(managedCursors); - PositionImpl position = PositionImpl.EARLIEST; + Position position = PositionFactory.EARLIEST; when(managedLedger.getLastConfirmedEntry()).thenReturn(position); // Create topic. persistentTopic.set(new PersistentTopic("topic-a", managedLedger, brokerService)); try { // Do check. - persistentTopic.get().checkIfTransactionBufferRecoverCompletely(true).get(5, TimeUnit.SECONDS); + persistentTopic.get().checkIfTransactionBufferRecoverCompletely().get(5, TimeUnit.SECONDS); fail("Expect failure by TB closed, but it is finished."); } catch (ExecutionException executionException){ Throwable t = executionException.getCause(); @@ -1647,7 +1716,7 @@ public void testGetTxnState() throws Exception { @Test public void testEncryptionRequired() throws Exception { - final String namespace = "tnx/ns-prechecks"; + final String namespace = "tnx/testEncryptionRequired"; final String topic = "persistent://" + namespace + "/test_transaction_topic"; admin.namespaces().createNamespace(namespace); admin.namespaces().setEncryptionRequiredStatus(namespace, true); @@ -1746,8 +1815,6 @@ public void testTBSnapshotWriter() throws Exception { .createAsync(); getTopic("persistent://" + topic + "-partition-0"); Thread.sleep(3000); - // the producer shouldn't be created, because the transaction buffer snapshot writer future didn't finish. - assertFalse(producerFuture.isDone()); // The topic will be closed, because the transaction buffer snapshot writer future is failed, // the failed writer future will be removed, the producer will be reconnected and work well. @@ -1778,4 +1845,223 @@ private void getTopic(String topicName) { }); } + @Test + public void testReadCommittedWithReadCompacted() throws Exception{ + final String namespace = "tnx/ns-read-committed-compacted"; + final String topic = "persistent://" + namespace + "/test_transaction_topic"; + admin.namespaces().createNamespace(namespace); + admin.topics().createNonPartitionedTopic(topic); + + admin.topicPolicies().setCompactionThreshold(topic, 100 * 1024 * 1024); + + @Cleanup + Consumer consumer = this.pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName("sub") + .subscriptionType(SubscriptionType.Exclusive) + .readCompacted(true) + .subscribe(); + + @Cleanup + Producer producer = this.pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .create(); + + producer.newMessage().key("K1").value("V1").send(); + + Transaction txn = pulsarClient.newTransaction() + .withTransactionTimeout(1, TimeUnit.MINUTES).build().get(); + producer.newMessage(txn).key("K2").value("V2").send(); + producer.newMessage(txn).key("K3").value("V3").send(); + + List messages = new ArrayList<>(); + while (true) { + Message message = consumer.receive(5, TimeUnit.SECONDS); + if (message == null) { + break; + } + messages.add(message.getValue()); + } + + Assert.assertEquals(messages, List.of("V1")); + + txn.commit(); + + messages.clear(); + + while (true) { + Message message = consumer.receive(5, TimeUnit.SECONDS); + if (message == null) { + break; + } + messages.add(message.getValue()); + } + + Assert.assertEquals(messages, List.of("V2", "V3")); + } + + + @Test + public void testReadCommittedWithCompaction() throws Exception{ + final String namespace = "tnx/ns-read-committed-compaction"; + final String topic = "persistent://" + namespace + "/test_transaction_topic" + UUID.randomUUID(); + admin.namespaces().createNamespace(namespace); + admin.topics().createNonPartitionedTopic(topic); + + admin.topicPolicies().setCompactionThreshold(topic, 100 * 1024 * 1024); + + @Cleanup + Producer producer = this.pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .create(); + + producer.newMessage().key("K1").value("V1").send(); + + Transaction txn = pulsarClient.newTransaction() + .withTransactionTimeout(1, TimeUnit.MINUTES).build().get(); + producer.newMessage(txn).key("K2").value("V2").send(); + producer.newMessage(txn).key("K3").value("V3").send(); + txn.commit().get(); + + producer.newMessage().key("K1").value("V4").send(); + + Transaction txn2 = pulsarClient.newTransaction() + .withTransactionTimeout(1, TimeUnit.MINUTES).build().get(); + producer.newMessage(txn2).key("K2").value("V5").send(); + producer.newMessage(txn2).key("K3").value("V6").send(); + txn2.commit().get(); + + admin.topics().triggerCompaction(topic); + + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topics().compactionStatus(topic).status, + LongRunningProcessStatus.Status.SUCCESS); + }); + + @Cleanup + Consumer consumer = this.pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName("sub") + .subscriptionType(SubscriptionType.Exclusive) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .readCompacted(true) + .subscribe(); + List result = new ArrayList<>(); + while (true) { + Message receive = consumer.receive(2, TimeUnit.SECONDS); + if (receive == null) { + break; + } + + result.add(receive.getValue()); + } + + Assert.assertEquals(result, List.of("V4", "V5", "V6")); + } + + @Test + public void testDelayedDeliveryExceedsMaxDelay() throws Exception { + final long maxDeliveryDelayInMillis = 5000; + final String namespace = "tnx/testDelayedDeliveryExceedsMaxDelay"; + final String topic = "persistent://" + namespace + "/test_transaction_topic" + UUID.randomUUID(); + admin.namespaces().createNamespace(namespace); + admin.topics().createNonPartitionedTopic(topic); + admin.topicPolicies().setDelayedDeliveryPolicy(topic, + DelayedDeliveryPolicies.builder() + .active(true) + .tickTime(100L) + .maxDeliveryDelayInMillis(maxDeliveryDelayInMillis) + .build()); + + @Cleanup + Producer producer = this.pulsarClient.newProducer() + .topic(topic) + .sendTimeout(5, TimeUnit.SECONDS) + .addEncryptionKey("my-app-key") + .defaultCryptoKeyReader("file:./src/test/resources/certificate/public-key.client-rsa.pem") + .create(); + + try { + Transaction txn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS).build().get(); + producer.newMessage(txn) + .value(UUID.randomUUID().toString().getBytes(StandardCharsets.UTF_8)) + .deliverAfter(6, TimeUnit.SECONDS) + .send(); + txn.commit(); + fail("Should have thrown NotAllowedException due to exceeding maxDeliveryDelayInMillis"); + } catch (PulsarClientException.NotAllowedException ex) { + assertEquals(ex.getMessage(), "Exceeds max allowed delivery delay of " + + maxDeliveryDelayInMillis + " milliseconds"); + } + } + + @Test + public void testPersistentTopicGetLastDispatchablePositionWithTxn() throws Exception { + String topic = "persistent://" + NAMESPACE1 + "/testPersistentTopicGetLastDispatchablePositionWithTxn"; + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .enableBatching(false) + .create(); + + BrokerService brokerService = pulsarTestContexts.get(0).getBrokerService(); + PersistentTopic persistentTopic = (PersistentTopic) brokerService.getTopicReference(topic).get(); + + + // send a normal message + String body = UUID.randomUUID().toString(); + MessageIdImpl msgId = (MessageIdImpl) producer.send(body); + + // send 3 txn messages + Transaction txn = pulsarClient.newTransaction().build().get(); + producer.newMessage(txn).value(UUID.randomUUID().toString()).send(); + producer.newMessage(txn).value(UUID.randomUUID().toString()).send(); + producer.newMessage(txn).value(UUID.randomUUID().toString()).send(); + + // get last dispatchable position + Position lastDispatchablePosition = persistentTopic.getLastDispatchablePosition().get(); + // the last dispatchable position should be the message id of the normal message + assertEquals(lastDispatchablePosition, PositionFactory.create(msgId.getLedgerId(), msgId.getEntryId())); + + // abort the txn + txn.abort().get(5, TimeUnit.SECONDS); + + // get last dispatchable position + lastDispatchablePosition = persistentTopic.getLastDispatchablePosition().get(); + // the last dispatchable position should be the message id of the normal message + assertEquals(lastDispatchablePosition, PositionFactory.create(msgId.getLedgerId(), msgId.getEntryId())); + + + @Cleanup + Reader reader = pulsarClient.newReader(Schema.STRING) + .topic(topic) + .startMessageId(MessageId.earliest) + .create(); + Transaction txn1 = pulsarClient.newTransaction().build().get(); + producer.newMessage(txn1).value(UUID.randomUUID().toString()).send(); + producer.newMessage(txn1).value(UUID.randomUUID().toString()).send(); + producer.newMessage(txn1).value(UUID.randomUUID().toString()).send(); + List> messages = new ArrayList<>(); + while (reader.hasMessageAvailable()) { + messages.add(reader.readNext()); + } + assertEquals(messages.size(), 1); + assertEquals(messages.get(0).getValue(), body); + + txn1.abort().get(5, TimeUnit.SECONDS); + + @Cleanup + Reader reader1 = pulsarClient.newReader(Schema.STRING) + .topic(topic) + .startMessageId(MessageId.earliest) + .create(); + List> messages1 = new ArrayList<>(); + while (reader1.hasMessageAvailable()) { + messages1.add(reader1.readNext()); + } + assertEquals(messages1.size(), 1); + assertEquals(messages1.get(0).getValue(), body); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java index f45eda8d21fbe..34af94f2c3185 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java @@ -58,6 +58,7 @@ public abstract class TransactionTestBase extends TestRetrySupport { public static final String CLUSTER_NAME = "test"; @Setter + @Getter private int brokerCount = 3; @Getter private final List serviceConfigurationList = new ArrayList<>(); @@ -114,10 +115,10 @@ protected void setUpBase(int numBroker,int numPartitionsOfTC, String topic, int new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString()); createTransactionCoordinatorAssign(numPartitionsOfTC); + admin.tenants().createTenant(TENANT, + new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); + admin.namespaces().createNamespace(NAMESPACE1, 4); if (topic != null) { - admin.tenants().createTenant(TENANT, - new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); - admin.namespaces().createNamespace(NAMESPACE1); if (numPartitions == 0) { admin.topics().createNonPartitionedTopic(topic); } else { @@ -156,20 +157,22 @@ protected void startBroker() throws Exception { conf.setBrokerShutdownTimeoutMs(0L); conf.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); conf.setBrokerServicePort(Optional.of(0)); - conf.setBrokerServicePortTls(Optional.of(0)); conf.setAdvertisedAddress("localhost"); conf.setWebServicePort(Optional.of(0)); - conf.setWebServicePortTls(Optional.of(0)); conf.setTransactionCoordinatorEnabled(true); conf.setBrokerDeduplicationEnabled(true); conf.setTransactionBufferSnapshotMaxTransactionCount(2); conf.setTransactionBufferSnapshotMinTimeInMillis(2000); + // Disable the dispatcher retry backoff in tests by default + conf.setDispatcherRetryBackoffInitialTimeInMs(0); + conf.setDispatcherRetryBackoffMaxTimeInMs(0); serviceConfigurationList.add(conf); PulsarTestContext.Builder testContextBuilder = PulsarTestContext.builder() .brokerInterceptor(new CounterBrokerInterceptor()) .spyByDefault() + .enableOpenTelemetry(true) .config(conf); if (i > 0) { testContextBuilder.reuseMockBookkeeperAndMetadataStores(pulsarTestContexts.get(0)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TopicTransactionBufferTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TopicTransactionBufferTest.java index aa98fc7d70106..1ab97eb457a05 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TopicTransactionBufferTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TopicTransactionBufferTest.java @@ -18,21 +18,60 @@ */ package org.apache.pulsar.broker.transaction.buffer; +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.fail; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import lombok.Cleanup; +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongSumValue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +import static org.testng.AssertJUnit.assertTrue; +import io.opentelemetry.api.common.Attributes; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.commons.lang3.RandomUtils; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.stats.OpenTelemetryTopicStats; import org.apache.pulsar.broker.transaction.TransactionTestBase; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferState; +import org.apache.pulsar.broker.transaction.buffer.utils.TransactionBufferTestImpl; +import org.apache.pulsar.broker.transaction.buffer.utils.TransactionBufferTestProvider; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Reader; +import org.apache.pulsar.client.api.TopicMessageId; import org.apache.pulsar.client.api.transaction.Transaction; +import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.client.impl.transaction.TransactionImpl; +import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.client.impl.TopicMessageIdImpl; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; import org.apache.pulsar.transaction.coordinator.TransactionMetadataStore; import org.apache.pulsar.transaction.coordinator.TransactionMetadataStoreState; @@ -44,18 +83,8 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import java.time.Duration; -import java.util.Collections; -import java.util.Map; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; - public class TopicTransactionBufferTest extends TransactionTestBase { - @BeforeMethod(alwaysRun = true) protected void setup() throws Exception { setBrokerCount(1); @@ -86,10 +115,19 @@ protected void cleanup() throws Exception { @Test public void testTransactionBufferAppendMarkerWriteFailState() throws Exception { final String topic = "persistent://" + NAMESPACE1 + "/testPendingAckManageLedgerWriteFailState"; + var attributes = Attributes.builder() + .put(OpenTelemetryAttributes.PULSAR_TENANT, "tnx") + .put(OpenTelemetryAttributes.PULSAR_NAMESPACE, "tnx/ns1") + .put(OpenTelemetryAttributes.PULSAR_TOPIC, topic) + .putAll(OpenTelemetryAttributes.TransactionStatus.COMMITTED.attributes) + .putAll(OpenTelemetryAttributes.TransactionBufferClientOperationStatus.FAILURE.attributes) + .build(); + Transaction txn = pulsarClient.newTransaction() .withTransactionTimeout(5, TimeUnit.SECONDS) .build().get(); + @Cleanup Producer producer = pulsarClient .newProducer() .topic(topic) @@ -97,11 +135,19 @@ public void testTransactionBufferAppendMarkerWriteFailState() throws Exception { .enableBatching(false) .create(); + assertMetricLongSumValue( + pulsarTestContexts.get(0).getOpenTelemetryMetricReader().collectAllMetrics(), + OpenTelemetryTopicStats.TRANSACTION_BUFFER_CLIENT_OPERATION_COUNTER, attributes, 0); + producer.newMessage(txn).value("test".getBytes()).send(); PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0) .getBrokerService().getTopic(TopicName.get(topic).toString(), false).get().get(); FieldUtils.writeField(persistentTopic.getManagedLedger(), "state", ManagedLedgerImpl.State.WriteFailed, true); txn.commit().get(); + + assertMetricLongSumValue( + pulsarTestContexts.get(0).getOpenTelemetryMetricReader().collectAllMetrics(), + OpenTelemetryTopicStats.TRANSACTION_BUFFER_CLIENT_OPERATION_COUNTER, attributes, 1); } @Test @@ -147,13 +193,13 @@ public void testCheckDeduplicationFailedWhenCreatePersistentTopic() throws Excep @Test public void testCloseTransactionBufferWhenTimeout() throws Exception { - String topic = "persistent://" + NAMESPACE1 + "/test_" + UUID.randomUUID(); + String topic = "persistent://" + NAMESPACE1 + "/testCloseTransactionBufferWhenTimeout"; PulsarService pulsar = pulsarServiceList.get(0); BrokerService brokerService0 = pulsar.getBrokerService(); BrokerService brokerService = Mockito.spy(brokerService0); AtomicReference reference = new AtomicReference<>(); - pulsar.getConfiguration().setTopicLoadTimeoutSeconds(10); - long topicLoadTimeout = TimeUnit.SECONDS.toMillis(pulsar.getConfiguration().getTopicLoadTimeoutSeconds() + 1); + pulsar.getConfiguration().setTopicLoadTimeoutSeconds(5); + long topicLoadTimeout = TimeUnit.SECONDS.toMillis(pulsar.getConfiguration().getTopicLoadTimeoutSeconds() + 3); Mockito .doAnswer(inv -> { @@ -179,4 +225,363 @@ public void testCloseTransactionBufferWhenTimeout() throws Exception { Assert.assertTrue(f.isCompletedExceptionally()); } + /** + * This test mainly test the following two point: + * 1. `getLastMessageIds` will get max read position. + * Send two message |1:0|1:1|; mock max read position as |1:0|; `getLastMessageIds` will get |1:0|. + * 2. `getLastMessageIds` will wait Transaction buffer recover completely. + * Mock `checkIfTBRecoverCompletely` return an exception, `getLastMessageIds` will fail too. + * Mock `checkIfTBRecoverCompletely` return null, `getLastMessageIds` will get correct result. + */ + @Test + public void testGetMaxPositionAfterTBReady() throws Exception { + // 1. Prepare test environment. + String topic = "persistent://" + NAMESPACE1 + "/testGetMaxReadyPositionAfterTBReady"; + // 1.1 Mock component. + TransactionBuffer transactionBuffer = Mockito.spy(TransactionBuffer.class); + when(transactionBuffer.checkIfTBRecoverCompletely()) + // If the Transaction buffer failed to recover, we can not get the correct last max read id. + .thenReturn(CompletableFuture.failedFuture(new Throwable("Mock fail"))) + // If the transaction buffer recover successfully, the max read position can be acquired successfully. + .thenReturn(CompletableFuture.completedFuture(null)); + TransactionBufferProvider transactionBufferProvider = Mockito.spy(TransactionBufferProvider.class); + Mockito.doReturn(transactionBuffer).when(transactionBufferProvider).newTransactionBuffer(any()); + TransactionBufferProvider originalTBProvider = getPulsarServiceList().get(0).getTransactionBufferProvider(); + Mockito.doReturn(transactionBufferProvider).when(getPulsarServiceList().get(0)).getTransactionBufferProvider(); + // 2. Building producer and consumer. + admin.topics().createNonPartitionedTopic(topic); + @Cleanup + Consumer consumer = pulsarClient.newConsumer() + .topic(topic) + .subscriptionName("sub") + .subscribe(); + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topic) + .create(); + // 3. Send message and test the exception can be handled as expected. + MessageIdImpl messageId = (MessageIdImpl) producer.newMessage().send(); + producer.newMessage().send(); + Mockito.doReturn(PositionFactory.create(messageId.getLedgerId(), messageId.getEntryId())) + .when(transactionBuffer).getMaxReadPosition(); + try { + consumer.getLastMessageIds(); + fail(); + } catch (PulsarClientException exception) { + assertTrue(exception.getMessage().contains("Failed to recover Transaction Buffer.")); + } + List messageIdList = consumer.getLastMessageIds(); + assertEquals(messageIdList.size(), 1); + TopicMessageIdImpl actualMessageID = (TopicMessageIdImpl) messageIdList.get(0); + assertEquals(messageId.getLedgerId(), actualMessageID.getLedgerId()); + assertEquals(messageId.getEntryId(), actualMessageID.getEntryId()); + // 4. Clean resource + Mockito.doReturn(originalTBProvider).when(getPulsarServiceList().get(0)).getTransactionBufferProvider(); + } + + /** + * Add a E2E test for the get last message ID. It tests 4 cases. + *

    + * 1. Only normal messages in the topic. + * 2. There are ongoing transactions, last message ID will not be updated until transaction end. + * 3. Aborted transaction will make the last message ID be updated as expected. + * 4. Committed transaction will make the last message ID be updated as expected. + *

    + */ + @Test + public void testGetLastMessageIdsWithOngoingTransactions() throws Exception { + // 1. Prepare environment + String topic = "persistent://" + NAMESPACE1 + "/testGetLastMessageIdsWithOngoingTransactions"; + String subName = "my-subscription"; + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topic) + .create(); + Consumer consumer = pulsarClient.newConsumer() + .topic(topic) + .subscriptionName(subName) + .subscribe(); + + // 2. Test last max read position can be required correctly. + // 2.1 Case1: send 3 original messages. |1:0|1:1|1:2| + MessageIdImpl expectedLastMessageID = null; + for (int i = 0; i < 3; i++) { + expectedLastMessageID = (MessageIdImpl) producer.newMessage().send(); + } + assertGetLastMessageId(consumer, expectedLastMessageID); + // 2.2 Case2: send 2 ongoing transactional messages and 2 original messages. + // |1:0|1:1|1:2|txn1:start->1:3|1:4|txn2:start->1:5. + Transaction txn1 = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.HOURS) + .build() + .get(); + Transaction txn2 = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.HOURS) + .build() + .get(); + + // |1:0|1:1|1:2|txn1:1:3| + producer.newMessage(txn1).send(); + + // |1:0|1:1|1:2|txn1:1:3|1:4| + MessageIdImpl expectedLastMessageID1 = (MessageIdImpl) producer.newMessage().send(); + + // |1:0|1:1|1:2|txn1:1:3|1:4|txn2:1:5| + producer.newMessage(txn2).send(); + + // 2.2.1 Last message ID will not change when txn1 and txn2 do not end. + assertGetLastMessageId(consumer, expectedLastMessageID); + + // 2.2.2 Last message ID will update to 1:4 when txn1 committed. + // |1:0|1:1|1:2|txn1:1:3|1:4|txn2:1:5|tx1:commit->1:6| + txn1.commit().get(5, TimeUnit.SECONDS); + assertGetLastMessageId(consumer, expectedLastMessageID1); + + // 2.2.3 Last message ID will still to 1:4 when txn2 aborted. + // |1:0|1:1|1:2|txn1:1:3|1:4|txn2:1:5|tx1:commit->1:6|tx2:abort->1:7| + txn2.abort().get(5, TimeUnit.SECONDS); + assertGetLastMessageId(consumer, expectedLastMessageID1); + + // Handle the case of the maxReadPosition < lastPosition, but it's an aborted transactional message. + Transaction txn3 = pulsarClient.newTransaction() + .build() + .get(); + producer.newMessage(txn3).send(); + assertGetLastMessageId(consumer, expectedLastMessageID1); + txn3.abort().get(5, TimeUnit.SECONDS); + assertGetLastMessageId(consumer, expectedLastMessageID1); + } + + /** + * produce 3 messages and then trigger a ledger switch, + * then create a transaction and send a transactional message. + * As there are messages in the new ledger, the reader should be able to read the messages. + * But reader.hasMessageAvailable() returns false if the entry id of max read position is -1. + * @throws Exception + */ + @Test + public void testGetLastMessageIdsWithOpenTransactionAtLedgerHead() throws Exception { + String topic = "persistent://" + NAMESPACE1 + "/testGetLastMessageIdsWithOpenTransactionAtLedgerHead"; + String subName = "my-subscription"; + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topic) + .create(); + Consumer consumer = pulsarClient.newConsumer() + .topic(topic) + .subscriptionName(subName) + .subscribe(); + MessageId expectedLastMessageID = null; + for (int i = 0; i < 3; i++) { + expectedLastMessageID = producer.newMessage().value(String.valueOf(i).getBytes()).send(); + System.out.println("expectedLastMessageID: " + expectedLastMessageID); + } + triggerLedgerSwitch(topic); + Transaction txn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.HOURS) + .build() + .get(); + producer.newMessage(txn).send(); + + Reader reader = pulsarClient.newReader() + .topic(topic) + .startMessageId(MessageId.earliest) + .create(); + assertTrue(reader.hasMessageAvailable()); + } + + private void triggerLedgerSwitch(String topicName) throws Exception{ + admin.topics().unload(topicName); + Awaitility.await().until(() -> { + CompletableFuture> topicFuture = + getPulsarServiceList().get(0).getBrokerService().getTopic(topicName, false); + if (!topicFuture.isDone() || topicFuture.isCompletedExceptionally()){ + return false; + } + Optional topicOptional = topicFuture.join(); + if (!topicOptional.isPresent()){ + return false; + } + PersistentTopic persistentTopic = (PersistentTopic) topicOptional.get(); + ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + return managedLedger.getState() == ManagedLedgerImpl.State.LedgerOpened; + }); + } + + private void assertGetLastMessageId(Consumer consumer, MessageIdImpl expected) throws Exception { + TopicMessageIdImpl actual = (TopicMessageIdImpl) consumer.getLastMessageIds().get(0); + assertEquals(expected.getEntryId(), actual.getEntryId()); + assertEquals(expected.getLedgerId(), actual.getLedgerId()); + } + + /** + * This test verifies the state changes of a TransactionBuffer within a topic under different conditions. + * Initially, the TransactionBuffer is in a NoSnapshot state upon topic creation. + * It remains in the NoSnapshot state even after a normal message is sent. + * The state changes to Ready only after a transactional message is sent. + * The test also ensures that the TransactionBuffer can be correctly recovered after the topic is unloaded. + */ + @Test + public void testWriteSnapshotWhenFirstTxnMessageSend() throws Exception { + // 1. Prepare test environment. + String topic = "persistent://" + NAMESPACE1 + "/testWriteSnapshotWhenFirstTxnMessageSend"; + String txnMsg = "transaction message"; + String normalMsg = "normal message"; + admin.topics().createNonPartitionedTopic(topic); + PersistentTopic persistentTopic = (PersistentTopic) pulsarServiceList.get(0).getBrokerService() + .getTopic(topic, false) + .get() + .get(); + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .create(); + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName("my-sub") + .subscribe(); + // 2. Test the state of transaction buffer after building producer with no new messages. + // The TransactionBuffer should be in NoSnapshot state before transaction message sent. + TopicTransactionBuffer topicTransactionBuffer = (TopicTransactionBuffer) persistentTopic.getTransactionBuffer(); + Awaitility.await().untilAsserted(() -> { + Assert.assertEquals(topicTransactionBuffer.getState(), TopicTransactionBufferState.State.NoSnapshot); + }); + // 3. Test the state of transaction buffer after sending normal messages. + // The TransactionBuffer should still be in NoSnapshot state after a normal message is sent. + producer.newMessage().value(normalMsg).send(); + Assert.assertEquals(topicTransactionBuffer.getState(), TopicTransactionBufferState.State.NoSnapshot); + // 4. Test the state of transaction buffer after sending transaction messages. + // The transaction buffer should be in Ready state at this time. + Transaction transaction = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.HOURS) + .build() + .get(); + producer.newMessage(transaction).value(txnMsg).send(); + Assert.assertEquals(topicTransactionBuffer.getState(), TopicTransactionBufferState.State.Ready); + // 5. Test transaction buffer can be recovered correctly. + // There are 4 message sent to this topic, 2 normal message and 2 transaction message |m1|m2-txn1|m3-txn1|m4|. + // Aborting the transaction and unload the topic and then redelivering unacked messages, + // only normal messages can be received. + transaction.abort().get(5, TimeUnit.SECONDS); + producer.newMessage().value(normalMsg).send(); + admin.topics().unload(topic); + PersistentTopic persistentTopic2 = (PersistentTopic) pulsarServiceList.get(0).getBrokerService() + .getTopic(topic, false) + .get() + .get(); + TopicTransactionBuffer topicTransactionBuffer2 = (TopicTransactionBuffer) persistentTopic2 + .getTransactionBuffer(); + Awaitility.await().untilAsserted(() -> { + Assert.assertEquals(topicTransactionBuffer2.getState(), TopicTransactionBufferState.State.Ready); + }); + consumer.redeliverUnacknowledgedMessages(); + for (int i = 0; i < 2; i++) { + Message message = consumer.receive(5, TimeUnit.SECONDS); + Assert.assertEquals(message.getValue(), normalMsg); + } + Message message = consumer.receive(5, TimeUnit.SECONDS); + Assert.assertNull(message); + } + + /** + * Send some messages before transaction buffer ready and then send some messages after transaction buffer ready, + * these messages should be received in order. + */ + @Test + public void testMessagePublishInOrder() throws Exception { + // 1. Prepare test environment. + this.pulsarServiceList.forEach(pulsarService -> { + pulsarService.setTransactionBufferProvider(new TransactionBufferTestProvider()); + }); + String topic = "persistent://" + NAMESPACE1 + "/testMessagePublishInOrder" + RandomUtils.nextLong(); + admin.topics().createNonPartitionedTopic(topic); + PersistentTopic persistentTopic = (PersistentTopic) pulsarServiceList.get(0).getBrokerService() + .getTopic(topic, false) + .get() + .get(); + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.INT32) + .topic(topic) + .create(); + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.INT32) + .topic(topic) + .subscriptionName("sub") + .subscribe(); + Transaction transaction = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.HOURS) + .build().get(); + + // 2. Set a new future in transaction buffer as `transactionBufferFuture` to simulate whether the + // transaction buffer recover completely. + TransactionBufferTestImpl topicTransactionBuffer = (TransactionBufferTestImpl) persistentTopic + .getTransactionBuffer(); + CompletableFuture completableFuture = new CompletableFuture<>(); + CompletableFuture originalFuture = topicTransactionBuffer.getPublishFuture(); + topicTransactionBuffer.setPublishFuture(completableFuture); + topicTransactionBuffer.setState(TopicTransactionBufferState.State.Ready); + // Register this topic to the transaction in advance to avoid the sending request pending here. + ((TransactionImpl) transaction).registerProducedTopic(topic).get(5, TimeUnit.SECONDS); + // 3. Test the messages sent before transaction buffer ready is in order. + for (int i = 0; i < 50; i++) { + producer.newMessage(transaction).value(i).sendAsync(); + } + // 4. Test the messages sent after transaction buffer ready is in order. + completableFuture.complete(originalFuture.get()); + for (int i = 50; i < 100; i++) { + producer.newMessage(transaction).value(i).sendAsync(); + } + transaction.commit().get(); + for (int i = 0; i < 100; i++) { + Message message = consumer.receive(5, TimeUnit.SECONDS); + Assert.assertEquals(message.getValue(), i); + } + } + + /** + * Test `testMessagePublishInOrder` will test the ref count work as expected with no exception. + * And this test is used to test the memory leak due to ref count. + */ + @Test + public void testRefCountWhenAppendBufferToTxn() throws Exception { + // 1. Prepare test resource + this.pulsarServiceList.forEach(pulsarService -> { + pulsarService.setTransactionBufferProvider(new TransactionBufferTestProvider()); + }); + String topic = "persistent://" + NAMESPACE1 + "/testRefCountWhenAppendBufferToTxn"; + admin.topics().createNonPartitionedTopic(topic); + PersistentTopic persistentTopic = (PersistentTopic) pulsarServiceList.get(0).getBrokerService() + .getTopic(topic, false) + .get() + .get(); + TransactionBufferTestImpl topicTransactionBuffer = (TransactionBufferTestImpl) persistentTopic + .getTransactionBuffer(); + // 2. Test reference count does not change in the method `appendBufferToTxn`. + // 2.1 Test sending first transaction message, this will take a snapshot. + ByteBuf byteBuf1 = Unpooled.buffer(); + topicTransactionBuffer.appendBufferToTxn(new TxnID(1, 1), 1L, byteBuf1) + .get(5, TimeUnit.SECONDS); + Awaitility.await().untilAsserted(() -> Assert.assertEquals(byteBuf1.refCnt(), 1)); + // 2.2 Test send the second transaction message, this will not take snapshots. + ByteBuf byteBuf2 = Unpooled.buffer(); + topicTransactionBuffer.appendBufferToTxn(new TxnID(1, 1), 1L, byteBuf1) + .get(5, TimeUnit.SECONDS); + Awaitility.await().untilAsserted(() -> Assert.assertEquals(byteBuf2.refCnt(), 1)); + // 2.3 Test sending message failed. + topicTransactionBuffer.setPublishFuture(FutureUtil.failedFuture(new Exception("fail"))); + ByteBuf byteBuf3 = Unpooled.buffer(); + try { + topicTransactionBuffer.appendBufferToTxn(new TxnID(1, 1), 1L, byteBuf1) + .get(5, TimeUnit.SECONDS); + fail(); + } catch (Exception e) { + assertEquals(e.getCause().getMessage(), "fail"); + } + Awaitility.await().untilAsserted(() -> Assert.assertEquals(byteBuf3.refCnt(), 1)); + // 3. release resource + byteBuf1.release(); + byteBuf2.release(); + byteBuf3.release(); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java index 86606e3ae8eb1..1c3de777e9349 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java @@ -18,39 +18,57 @@ */ package org.apache.pulsar.broker.transaction.buffer; +import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.Metric; +import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.parseMetrics; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; import io.netty.util.HashedWheelTimer; import io.netty.util.concurrent.DefaultThreadFactory; -import lombok.Cleanup; import java.io.ByteArrayOutputStream; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.ExecutionException; - +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.PrometheusMetricsTestUtil; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; -import org.apache.pulsar.broker.stats.PrometheusMetricsTest; -import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGenerator; import org.apache.pulsar.broker.transaction.TransactionTestBase; import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferClientImpl; import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferHandlerImpl; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.api.transaction.TransactionBufferClient; import org.apache.pulsar.client.api.transaction.TransactionBufferClientException; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.ClientCnx; +import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.api.proto.TxnAction; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.awaitility.Awaitility; @@ -61,14 +79,6 @@ import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import java.lang.reflect.Field; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.TimeUnit; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; @Test(groups = "broker") public class TransactionBufferClientTest extends TransactionTestBase { @@ -115,6 +125,52 @@ public void testCommitOnTopic() throws ExecutionException, InterruptedException } } + @Test + public void testRecoveryTransactionBufferWhenCommonTopicAndSystemTopicAtDifferentBroker() throws Exception { + for (int i = 0; i < getPulsarServiceList().size(); i++) { + getPulsarServiceList().get(i).getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); + } + String topic1 = NAMESPACE1 + "/testRecoveryTransactionBufferWhenCommonTopicAndSystemTopicAtDifferentBroker"; + admin.tenants().createTenant(TENANT, + new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); + admin.namespaces().createNamespace(NAMESPACE1, 4); + admin.tenants().createTenant(NamespaceName.SYSTEM_NAMESPACE.getTenant(), + new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); + admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString()); + pulsarServiceList.get(0).getPulsarResources() + .getNamespaceResources() + .getPartitionedTopicResources() + .createPartitionedTopic(SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN, + new PartitionedTopicMetadata(3)); + assertTrue(admin.namespaces().getBundles(NAMESPACE1).getNumBundles() > 1); + for (int i = 0; true ; i++) { + topic1 = topic1 + i; + admin.topics().createNonPartitionedTopic(topic1); + String segmentTopicBroker = admin.lookups() + .lookupTopic(NAMESPACE1 + "/" + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT); + String indexTopicBroker = admin.lookups() + .lookupTopic(NAMESPACE1 + "/" + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_INDEXES); + if (segmentTopicBroker.equals(indexTopicBroker)) { + String topicBroker = admin.lookups().lookupTopic(topic1); + if (!topicBroker.equals(segmentTopicBroker)) { + break; + } + } else { + break; + } + } + @Cleanup + PulsarClient localPulsarClient = PulsarClient.builder() + .serviceUrl(pulsarServiceList.get(0).getBrokerServiceUrl()) + .enableTransaction(true).build(); + @Cleanup + Producer producer = localPulsarClient.newProducer(Schema.BYTES).topic(topic1).create(); + Transaction transaction = localPulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.HOURS) + .build().get(); + producer.newMessage(transaction).send(); + } + @Test public void testAbortOnTopic() throws ExecutionException, InterruptedException { List> futures = new ArrayList<>(); @@ -157,6 +213,8 @@ public void testAbortOnSubscription() throws ExecutionException, InterruptedExce @Test public void testTransactionBufferMetrics() throws Exception { + this.cleanup(); + this.setup(); //Test commit for (int i = 0; i < partitions; i++) { String topic = partitionedTopicName.getPartition(i).toString(); @@ -171,30 +229,30 @@ public void testTransactionBufferMetrics() throws Exception { @Cleanup ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsarServiceList.get(0), true, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsarServiceList.get(0), true, false, false, statsOut); String metricsStr = statsOut.toString(); - Multimap metricsMap = PrometheusMetricsTest.parseMetrics(metricsStr); + Multimap metricsMap = parseMetrics(metricsStr); - Collection abortFailed = metricsMap.get("pulsar_txn_tb_client_abort_failed_total"); - Collection commitFailed = metricsMap.get("pulsar_txn_tb_client_commit_failed_total"); - Collection abortLatencyCount = + Collection abortFailed = metricsMap.get("pulsar_txn_tb_client_abort_failed_total"); + Collection commitFailed = metricsMap.get("pulsar_txn_tb_client_commit_failed_total"); + Collection abortLatencyCount = metricsMap.get("pulsar_txn_tb_client_abort_latency_count"); - Collection commitLatencyCount = + Collection commitLatencyCount = metricsMap.get("pulsar_txn_tb_client_commit_latency_count"); - Collection pending = metricsMap.get("pulsar_txn_tb_client_pending_requests"); + Collection pending = metricsMap.get("pulsar_txn_tb_client_pending_requests"); assertEquals(abortFailed.stream().mapToDouble(metric -> metric.value).sum(), 0); assertEquals(commitFailed.stream().mapToDouble(metric -> metric.value).sum(), 0); for (int i = 0; i < partitions; i++) { String topic = partitionedTopicName.getPartition(i).toString(); - Optional optional = abortLatencyCount.stream() + Optional optional = abortLatencyCount.stream() .filter(metric -> metric.tags.get("topic").equals(topic)).findFirst(); assertTrue(optional.isPresent()); assertEquals(optional.get().value, 1D); - Optional optional1 = commitLatencyCount.stream() + Optional optional1 = commitLatencyCount.stream() .filter(metric -> metric.tags.get("topic").equals(topic)).findFirst(); assertTrue(optional1.isPresent()); assertEquals(optional1.get().value, 1D); @@ -203,14 +261,22 @@ public void testTransactionBufferMetrics() throws Exception { assertEquals(pending.size(), 1); } + /** + * This is a flaky test. + */ @Test public void testTransactionBufferClientTimeout() throws Exception { PulsarService pulsarService = pulsarServiceList.get(0); - PulsarClient mockClient = mock(PulsarClientImpl.class); + PulsarClientImpl mockClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(mockClient.getCnxPool()).thenReturn(connectionPool); CompletableFuture completableFuture = new CompletableFuture<>(); ClientCnx clientCnx = mock(ClientCnx.class); completableFuture.complete(clientCnx); - when(((PulsarClientImpl)mockClient).getConnection(anyString())).thenReturn(completableFuture); + when(mockClient.getConnection(anyString())).thenReturn(completableFuture); + when(mockClient.getConnection(anyString(), anyInt())).thenReturn( + CompletableFuture.completedFuture(Pair.of(clientCnx, false))); + when(mockClient.getConnection(any(), any(), anyInt())).thenReturn(completableFuture); ChannelHandlerContext cnx = mock(ChannelHandlerContext.class); when(clientCnx.ctx()).thenReturn(cnx); Channel channel = mock(Channel.class); @@ -237,7 +303,9 @@ public PulsarClient answer(InvocationOnMock invocation) throws Throwable { ConcurrentSkipListMap outstandingRequests = (ConcurrentSkipListMap) field.get(transactionBufferHandler); - assertEquals(outstandingRequests.size(), 1); + Awaitility.await().atMost(1, TimeUnit.SECONDS).untilAsserted(() -> { + assertEquals(outstandingRequests.size(), 1); + }); Awaitility.await().atLeast(2, TimeUnit.SECONDS).until(() -> { if (outstandingRequests.size() == 0) { @@ -257,11 +325,12 @@ public PulsarClient answer(InvocationOnMock invocation) throws Throwable { @Test public void testTransactionBufferChannelUnActive() throws PulsarServerException { PulsarService pulsarService = pulsarServiceList.get(0); - PulsarClient mockClient = mock(PulsarClientImpl.class); - CompletableFuture completableFuture = new CompletableFuture<>(); + PulsarClientImpl mockClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(mockClient.getCnxPool()).thenReturn(connectionPool); ClientCnx clientCnx = mock(ClientCnx.class); - completableFuture.complete(clientCnx); - when(((PulsarClientImpl)mockClient).getConnection(anyString())).thenReturn(completableFuture); + when(mockClient.getConnection(anyString(), anyInt())).thenReturn( + CompletableFuture.completedFuture(Pair.of(clientCnx, false))); ChannelHandlerContext cnx = mock(ChannelHandlerContext.class); when(clientCnx.ctx()).thenReturn(cnx); Channel channel = mock(Channel.class); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java index e92cf29521e34..d1784f6a392bf 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.transaction.buffer; -import com.google.common.collect.Sets; import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; @@ -30,7 +29,6 @@ import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClient; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -49,8 +47,6 @@ protected void setup() throws Exception { setUpBase(1, 16, null, 0); Awaitility.await().until(() -> ((PulsarClientImpl) pulsarClient) .getTcClient().getState() == TransactionCoordinatorClient.State.READY); - admin.tenants().createTenant(TENANT, - new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); } @AfterMethod(alwaysRun = true) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferHandlerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferHandlerImplTest.java index d6ec092c4456f..633671420e5fc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferHandlerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferHandlerImplTest.java @@ -18,12 +18,14 @@ */ package org.apache.pulsar.broker.transaction.buffer; +import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.namespace.NamespaceEphemeralData; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferHandlerImpl; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; @@ -31,8 +33,8 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; -import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.impl.ClientCnx; +import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.api.proto.TxnAction; import org.apache.pulsar.common.naming.NamespaceBundle; @@ -46,7 +48,9 @@ public class TransactionBufferHandlerImplTest { @Test public void testRequestCredits() throws PulsarServerException { - PulsarClient pulsarClient = mock(PulsarClientImpl.class); + PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(pulsarClient.getCnxPool()).thenReturn(connectionPool); PulsarService pulsarService = mock(PulsarService.class); NamespaceService namespaceService = mock(NamespaceService.class); when(pulsarService.getNamespaceService()).thenReturn(namespaceService); @@ -54,7 +58,10 @@ public void testRequestCredits() throws PulsarServerException { when(namespaceService.getBundleAsync(any())).thenReturn(CompletableFuture.completedFuture(mock(NamespaceBundle.class))); Optional opData = Optional.empty(); when(namespaceService.getOwnerAsync(any())).thenReturn(CompletableFuture.completedFuture(opData)); - when(((PulsarClientImpl)pulsarClient).getConnection(anyString())).thenReturn(CompletableFuture.completedFuture(mock(ClientCnx.class))); + when(pulsarClient.getConnection(anyString(), anyInt())) + .thenReturn(CompletableFuture.completedFuture(Pair.of(mock(ClientCnx.class), false))); + when(pulsarClient.getConnection(anyString())) + .thenReturn(CompletableFuture.completedFuture(mock(ClientCnx.class))); TransactionBufferHandlerImpl handler = spy(new TransactionBufferHandlerImpl(pulsarService, null, 1000, 3000)); doNothing().when(handler).endTxn(any()); doReturn(CompletableFuture.completedFuture(mock(ClientCnx.class))).when(handler).getClientCnx(anyString()); @@ -75,7 +82,9 @@ public void testRequestCredits() throws PulsarServerException { @Test public void testMinRequestCredits() throws PulsarServerException { - PulsarClient pulsarClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + when(pulsarClient.getCnxPool()).thenReturn(connectionPool); PulsarService pulsarService = mock(PulsarService.class); when(pulsarService.getClient()).thenReturn(pulsarClient); TransactionBufferHandlerImpl handler = spy(new TransactionBufferHandlerImpl(pulsarService, null, 50, 3000)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionLowWaterMarkTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionLowWaterMarkTest.java index 3901f186d81c7..818b854ffe941 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionLowWaterMarkTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionLowWaterMarkTest.java @@ -31,9 +31,8 @@ import java.util.concurrent.TimeUnit; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; import org.apache.commons.collections4.map.LinkedMap; -import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -54,7 +53,6 @@ import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; import org.apache.pulsar.transaction.coordinator.TransactionMetadataStore; import org.apache.pulsar.transaction.coordinator.TransactionMetadataStoreState; @@ -106,6 +104,7 @@ public void testTransactionBufferLowWaterMark() throws Exception { .withTransactionTimeout(5, TimeUnit.SECONDS) .build().get(); + @Cleanup Producer producer = pulsarClient .newProducer() .topic(TOPIC) @@ -113,6 +112,7 @@ public void testTransactionBufferLowWaterMark() throws Exception { .enableBatching(false) .create(); + @Cleanup Consumer consumer = pulsarClient.newConsumer() .topic(TOPIC) .subscriptionName("test") @@ -146,7 +146,8 @@ public void testTransactionBufferLowWaterMark() throws Exception { PartitionedTopicMetadata partitionedTopicMetadata = ((PulsarClientImpl) pulsarClient).getLookup() - .getPartitionedTopicMetadata(SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN).get(); + .getPartitionedTopicMetadata(SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN, false) + .get(); Transaction lowWaterMarkTxn = null; for (int i = 0; i < partitionedTopicMetadata.partitions; i++) { lowWaterMarkTxn = pulsarClient.newTransaction() @@ -209,27 +210,23 @@ public void testPendingAckLowWaterMark() throws Exception { Message message = consumer.receive(2, TimeUnit.SECONDS); assertEquals(new String(message.getData()), TEST1); consumer.acknowledgeAsync(message.getMessageId(), txn).get(); - LinkedMap> individualAckOfTransaction = null; + LinkedMap> individualAckOfTransaction = null; for (int i = 0; i < getPulsarServiceList().size(); i++) { - Field field = BrokerService.class.getDeclaredField("topics"); - field.setAccessible(true); - ConcurrentOpenHashMap>> topics = - (ConcurrentOpenHashMap>>) field - .get(getPulsarServiceList().get(i).getBrokerService()); + final var topics = getPulsarServiceList().get(i).getBrokerService().getTopics(); CompletableFuture> completableFuture = topics.get(TOPIC); if (completableFuture != null) { Optional topic = completableFuture.get(); if (topic.isPresent()) { PersistentSubscription persistentSubscription = (PersistentSubscription) topic.get() .getSubscription(subName); - field = PersistentSubscription.class.getDeclaredField("pendingAckHandle"); + var field = PersistentSubscription.class.getDeclaredField("pendingAckHandle"); field.setAccessible(true); PendingAckHandleImpl pendingAckHandle = (PendingAckHandleImpl) field.get(persistentSubscription); field = PendingAckHandleImpl.class.getDeclaredField("individualAckOfTransaction"); field.setAccessible(true); individualAckOfTransaction = - (LinkedMap>) field.get(pendingAckHandle); + (LinkedMap>) field.get(pendingAckHandle); } } } @@ -251,7 +248,8 @@ public void testPendingAckLowWaterMark() throws Exception { PartitionedTopicMetadata partitionedTopicMetadata = ((PulsarClientImpl) pulsarClient).getLookup() - .getPartitionedTopicMetadata(SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN).get(); + .getPartitionedTopicMetadata(SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN, false) + .get(); Transaction lowWaterMarkTxn = null; for (int i = 0; i < partitionedTopicMetadata.partitions; i++) { lowWaterMarkTxn = pulsarClient.newTransaction() @@ -448,8 +446,8 @@ private boolean checkTxnIsOngoingInTP(TxnID txnID, String subName) throws Except Field field2 = PendingAckHandleImpl.class.getDeclaredField("individualAckOfTransaction"); field2.setAccessible(true); - LinkedMap> individualAckOfTransaction = - (LinkedMap>) field2.get(pendingAckHandle); + LinkedMap> individualAckOfTransaction = + (LinkedMap>) field2.get(pendingAckHandle); return individualAckOfTransaction.containsKey(txnID); } @@ -463,8 +461,8 @@ private boolean checkTxnIsOngoingInTB(TxnID txnID) throws Exception { (TopicTransactionBuffer) persistentTopic.getTransactionBuffer(); Field field3 = TopicTransactionBuffer.class.getDeclaredField("ongoingTxns"); field3.setAccessible(true); - LinkedMap ongoingTxns = - (LinkedMap) field3.get(topicTransactionBuffer); + LinkedMap ongoingTxns = + (LinkedMap) field3.get(topicTransactionBuffer); return ongoingTxns.containsKey(txnID); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java index 55d115905a35d..0b50f91fd403c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java @@ -28,7 +28,7 @@ import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.transaction.TransactionTestBase; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer; @@ -78,6 +78,7 @@ public void commitTxnTest() throws Exception { .withTransactionTimeout(5, TimeUnit.SECONDS) .build().get(); + @Cleanup Producer producer = pulsarClient .newProducer() .topic(TOPIC) @@ -85,6 +86,7 @@ public void commitTxnTest() throws Exception { .enableBatching(false) .create(); + @Cleanup Consumer consumer = pulsarClient.newConsumer() .topic(TOPIC) .subscriptionName("test") @@ -124,6 +126,7 @@ public void abortTxnTest() throws Exception { .withTransactionTimeout(5, TimeUnit.SECONDS) .build().get(); + @Cleanup Producer producer = pulsarClient .newProducer() .topic(TOPIC) @@ -131,6 +134,7 @@ public void abortTxnTest() throws Exception { .enableBatching(false) .create(); + @Cleanup Consumer consumer = pulsarClient.newConsumer() .topic(TOPIC) .subscriptionName("test") @@ -191,6 +195,15 @@ public void testSyncNormalPositionWhenTBRecover(boolean clientEnableTransaction, .topic(topicName) .create(); + if (clientEnableTransaction) { + Transaction transaction = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.HOURS) + .build() + .get(); + producer.newMessage(transaction).send(); + transaction.commit().get(); + } + PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService() .getTopic(TopicName.get(topicName).toString(), false).get().get(); @@ -205,40 +218,42 @@ public void testSyncNormalPositionWhenTBRecover(boolean clientEnableTransaction, // init maxReadPosition is PositionImpl.EARLIEST Position position = topicTransactionBuffer.getMaxReadPosition(); - assertEquals(position, PositionImpl.EARLIEST); + assertEquals(position, PositionFactory.EARLIEST); MessageIdImpl messageId = (MessageIdImpl) producer.send("test".getBytes()); // send normal message can't change MaxReadPosition when state is None or Initializing position = topicTransactionBuffer.getMaxReadPosition(); - assertEquals(position, PositionImpl.EARLIEST); + assertEquals(position, PositionFactory.EARLIEST); + + // change to None state can recover + field.set(topicTransactionBuffer, TopicTransactionBufferState.State.None); // invoke recover Method method = TopicTransactionBuffer.class.getDeclaredMethod("recover"); method.setAccessible(true); method.invoke(topicTransactionBuffer); - // change to None state can recover - field.set(topicTransactionBuffer, TopicTransactionBufferState.State.None); - // recover success again checkTopicTransactionBufferState(clientEnableTransaction, topicTransactionBuffer); // change MaxReadPosition to normal message position - assertEquals(PositionImpl.get(messageId.getLedgerId(), messageId.getEntryId()), + assertEquals(PositionFactory.create(messageId.getLedgerId(), messageId.getEntryId()), topicTransactionBuffer.getMaxReadPosition()); } private void checkTopicTransactionBufferState(boolean clientEnableTransaction, TopicTransactionBuffer topicTransactionBuffer) { // recover success - Awaitility.await().until(() -> { + Awaitility.await().untilAsserted(() -> { if (clientEnableTransaction) { // recover success, client enable transaction will change to Ready State - return topicTransactionBuffer.getStats(false).state.equals(Ready.name()); + assertEquals(topicTransactionBuffer.getStats(false, false).state, + Ready.name()); } else { // recover success, client disable transaction will change to NoSnapshot State - return topicTransactionBuffer.getStats(false).state.equals(NoSnapshot.name()); + assertEquals(topicTransactionBuffer.getStats(false, false).state, + NoSnapshot.name()); } }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/utils/TransactionBufferTestImpl.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/utils/TransactionBufferTestImpl.java new file mode 100644 index 0000000000000..7ee14ffc3378e --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/utils/TransactionBufferTestImpl.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.buffer.utils; + +import lombok.Setter; +import org.apache.bookkeeper.mledger.Position; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer; + +import java.util.concurrent.CompletableFuture; + +public class TransactionBufferTestImpl extends TopicTransactionBuffer { + @Setter + public CompletableFuture transactionBufferFuture = null; + @Setter + public State state = null; + @Setter + public CompletableFuture publishFuture = null; + + public TransactionBufferTestImpl(PersistentTopic topic) { + super(topic); + } + + @Override + public CompletableFuture getTransactionBufferFuture() { + return transactionBufferFuture == null ? super.getTransactionBufferFuture() : transactionBufferFuture; + } + + @Override + public State getState() { + return state == null ? super.getState() : state; + } + + @Override + public CompletableFuture getPublishFuture() { + return publishFuture == null ? super.getPublishFuture() : publishFuture; + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/utils/TransactionBufferTestProvider.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/utils/TransactionBufferTestProvider.java new file mode 100644 index 0000000000000..7bc93c0e7cf25 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/utils/TransactionBufferTestProvider.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.buffer.utils; + +import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.transaction.buffer.TransactionBuffer; +import org.apache.pulsar.broker.transaction.buffer.TransactionBufferProvider; + +public class TransactionBufferTestProvider implements TransactionBufferProvider { + + @Override + public TransactionBuffer newTransactionBuffer(Topic originTopic) { + return new TransactionBufferTestImpl((PersistentTopic) originTopic); + } +} + diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionCoordinatorClientTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionCoordinatorClientTest.java index c442c3a901464..36bc0e522c210 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionCoordinatorClientTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionCoordinatorClientTest.java @@ -24,14 +24,18 @@ import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import lombok.Cleanup; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.TransactionMetadataStoreService; import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferClientImpl; +import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.transaction.TransactionBufferClient; +import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClient; import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClient.State; import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClientException; import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.client.impl.transaction.TransactionCoordinatorClientImpl; import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.Test; @@ -107,4 +111,24 @@ public void testTransactionCoordinatorExceptionUnwrap() { instanceof TransactionCoordinatorClientException.InvalidTxnStatusException); } } + + @Test + public void testClientStartWithRetry() throws Exception{ + String validBrokerServiceUrl = pulsarServices[0].getBrokerServiceUrl(); + String invalidBrokerServiceUrl = "localhost:0"; + String brokerServiceUrl = validBrokerServiceUrl + "," + invalidBrokerServiceUrl; + + @Cleanup + PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(brokerServiceUrl).build(); + @Cleanup + TransactionCoordinatorClient transactionCoordinatorClient = new TransactionCoordinatorClientImpl(pulsarClient); + + try { + transactionCoordinatorClient.start(); + }catch (TransactionCoordinatorClientException e) { + Assert.fail("Shouldn't have exception at here", e); + } + + Assert.assertEquals(transactionCoordinatorClient.getState(), State.READY); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionMetaStoreAssignmentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionMetaStoreAssignmentTest.java index ac1f570218704..25ce90e1cf091 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionMetaStoreAssignmentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionMetaStoreAssignmentTest.java @@ -61,6 +61,7 @@ public void testTransactionMetaStoreAssignAndFailover() throws Exception { pulsarServiceList.remove(crashedMetaStore); crashedMetaStore.close(); + pulsarClient.close(); pulsarClient = buildClient(); Awaitility.await().atMost(5, TimeUnit.SECONDS) .untilAsserted(() -> { @@ -90,7 +91,7 @@ public void testTransactionMetaStoreUnload() throws Exception { .removeTransactionMetadataStore(TransactionCoordinatorID.get(f))); } checkTransactionCoordinatorNum(0); - buildClient(); + pulsarClient = buildClient(); checkTransactionCoordinatorNum(16); pulsarClient.close(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionMetaStoreTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionMetaStoreTestBase.java index eb714dd848afc..5bf48932f3687 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionMetaStoreTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/coordinator/TransactionMetaStoreTestBase.java @@ -120,19 +120,30 @@ public final void shutdownAll() throws Exception { @Override protected void cleanup() throws Exception { - for (PulsarAdmin admin : pulsarAdmins) { - if (admin != null) { - admin.close(); + if (transactionCoordinatorClient != null) { + transactionCoordinatorClient.close(); + transactionCoordinatorClient = null; + } + for (int i = 0; i < BROKER_COUNT; i++) { + if (pulsarAdmins[i] != null) { + pulsarAdmins[i].close(); + pulsarAdmins[i] = null; } } if (pulsarClient != null) { pulsarClient.close(); + pulsarClient = null; } - for (PulsarService service : pulsarServices) { - if (service != null) { - service.close(); + for (int i = 0; i < BROKER_COUNT; i++) { + if (pulsarServices[i] != null) { + pulsarServices[i].close(); + pulsarServices[i] = null; } } + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; + } Mockito.reset(); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java index 1360dd7c4442b..58cf59aa6b3b9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java @@ -21,7 +21,6 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; -import java.lang.reflect.Field; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -30,11 +29,11 @@ import java.util.concurrent.TimeUnit; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.collections4.map.LinkedMap; import org.apache.commons.lang3.tuple.MutablePair; -import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.transaction.TransactionTestBase; @@ -47,7 +46,6 @@ import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.util.collections.BitSetRecyclable; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -119,24 +117,20 @@ public void txnAckTestNoBatchAndSharedSubMemoryDeleteTest() throws Exception { int count = 0; for (int i = 0; i < getPulsarServiceList().size(); i++) { - Field field = BrokerService.class.getDeclaredField("topics"); - field.setAccessible(true); - ConcurrentOpenHashMap>> topics = - (ConcurrentOpenHashMap>>) field - .get(getPulsarServiceList().get(i).getBrokerService()); + final var topics = getPulsarServiceList().get(i).getBrokerService().getTopics(); CompletableFuture> completableFuture = topics.get("persistent://" + normalTopic); if (completableFuture != null) { Optional topic = completableFuture.get(); if (topic.isPresent()) { PersistentSubscription persistentSubscription = (PersistentSubscription) topic.get() .getSubscription(subscriptionName); - field = PersistentSubscription.class.getDeclaredField("pendingAckHandle"); + var field = PersistentSubscription.class.getDeclaredField("pendingAckHandle"); field.setAccessible(true); PendingAckHandleImpl pendingAckHandle = (PendingAckHandleImpl) field.get(persistentSubscription); field = PendingAckHandleImpl.class.getDeclaredField("individualAckOfTransaction"); field.setAccessible(true); - LinkedMap> individualAckOfTransaction = - (LinkedMap>) field.get(pendingAckHandle); + LinkedMap> individualAckOfTransaction = + (LinkedMap>) field.get(pendingAckHandle); assertTrue(individualAckOfTransaction.isEmpty()); if (retryCnt == 0) { //one message are not ack @@ -176,7 +170,7 @@ public void txnAckTestBatchAndSharedSubMemoryDeleteTest() throws Exception { PendingAckHandleImpl pendingAckHandle = null; - LinkedMap> individualAckOfTransaction = null; + LinkedMap> individualAckOfTransaction = null; ManagedCursorImpl managedCursor = null; MessageId[] messageIds = new MessageId[2]; @@ -213,30 +207,26 @@ public void txnAckTestBatchAndSharedSubMemoryDeleteTest() throws Exception { commitTxn.commit().get(); int count = 0; for (int i = 0; i < getPulsarServiceList().size(); i++) { - Field field = BrokerService.class.getDeclaredField("topics"); - field.setAccessible(true); - ConcurrentOpenHashMap>> topics = - (ConcurrentOpenHashMap>>) field - .get(getPulsarServiceList().get(i).getBrokerService()); + final var topics = getPulsarServiceList().get(i).getBrokerService().getTopics(); CompletableFuture> completableFuture = topics.get("persistent://" + normalTopic); if (completableFuture != null) { Optional topic = completableFuture.get(); if (topic.isPresent()) { PersistentSubscription testPersistentSubscription = (PersistentSubscription) topic.get().getSubscription(subscriptionName); - field = PersistentSubscription.class.getDeclaredField("pendingAckHandle"); + var field = PersistentSubscription.class.getDeclaredField("pendingAckHandle"); field.setAccessible(true); pendingAckHandle = (PendingAckHandleImpl) field.get(testPersistentSubscription); field = PendingAckHandleImpl.class.getDeclaredField("individualAckOfTransaction"); field.setAccessible(true); individualAckOfTransaction = - (LinkedMap>) field.get(pendingAckHandle); + (LinkedMap>) field.get(pendingAckHandle); assertTrue(individualAckOfTransaction.isEmpty()); managedCursor = (ManagedCursorImpl) testPersistentSubscription.getCursor(); field = ManagedCursorImpl.class.getDeclaredField("batchDeletedIndexes"); field.setAccessible(true); - final ConcurrentSkipListMap batchDeletedIndexes = - (ConcurrentSkipListMap) field.get(managedCursor); + final ConcurrentSkipListMap batchDeletedIndexes = + (ConcurrentSkipListMap) field.get(managedCursor); if (retryCnt == 0) { //one message are not ack Awaitility.await().until(() -> { @@ -313,16 +303,16 @@ public void testPendingAckClearPositionIsSmallerThanMarkDelete() throws Exceptio .orElseThrow(); PersistentSubscription subscription = (PersistentSubscription) t.getSubscription(subscriptionName); PendingAckHandleImpl pendingAckHandle = (PendingAckHandleImpl) subscription.getPendingAckHandle(); - Map> individualAckPositions = + Map> individualAckPositions = pendingAckHandle.getIndividualAckPositions(); // one message in pending ack state assertEquals(1, individualAckPositions.size()); // put the PositionImpl.EARLIEST to the map - individualAckPositions.put(PositionImpl.EARLIEST, new MutablePair<>(PositionImpl.EARLIEST, 0)); + individualAckPositions.put(PositionFactory.EARLIEST, new MutablePair<>(PositionFactory.EARLIEST, 0)); // put the PositionImpl.LATEST to the map - individualAckPositions.put(PositionImpl.LATEST, new MutablePair<>(PositionImpl.EARLIEST, 0)); + individualAckPositions.put(PositionFactory.LATEST, new MutablePair<>(PositionFactory.EARLIEST, 0)); // three position in pending ack state assertEquals(3, individualAckPositions.size()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckMetadataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckMetadataTest.java index efe83cebc3661..f01ea7ac67ff9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckMetadataTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckMetadataTest.java @@ -18,9 +18,19 @@ */ package org.apache.pulsar.broker.transaction.pendingack; +import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.State.WriteFailed; +import static org.apache.pulsar.transaction.coordinator.impl.DisabledTxnLogBufferedWriterMetricsStats.DISABLED_BUFFERED_WRITER_METRICS; +import static org.testng.Assert.assertTrue; +import static org.testng.AssertJUnit.fail; import io.netty.util.HashedWheelTimer; import io.netty.util.concurrent.DefaultThreadFactory; +import java.lang.reflect.Field; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import lombok.Cleanup; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.ManagedCursor; @@ -36,14 +46,6 @@ import org.apache.pulsar.common.api.proto.CommandAck; import org.apache.pulsar.transaction.coordinator.impl.TxnLogBufferedWriterConfig; import org.testng.annotations.Test; -import java.lang.reflect.Field; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; -import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.State.WriteFailed; -import static org.apache.pulsar.transaction.coordinator.impl.DisabledTxnLogBufferedWriterMetricsStats.DISABLED_BUFFERED_WRITER_METRICS; -import static org.testng.Assert.assertTrue; -import static org.testng.AssertJUnit.fail; public class PendingAckMetadataTest extends MockedBookKeeperTestCase { @@ -80,9 +82,13 @@ public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { ManagedCursor cursor = completableFuture.get().openCursor("test"); ManagedCursor subCursor = completableFuture.get().openCursor("test"); + + @Cleanup("shutdownNow") + ExecutorService executorService = Executors.newSingleThreadExecutor(); + MLPendingAckStore pendingAckStore = new MLPendingAckStore(completableFuture.get(), cursor, subCursor, 500, - bufferedWriterConfig, transactionTimer, DISABLED_BUFFERED_WRITER_METRICS); + bufferedWriterConfig, transactionTimer, DISABLED_BUFFERED_WRITER_METRICS, executorService); Field field = MLPendingAckStore.class.getDeclaredField("managedLedger"); field.setAccessible(true); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java index bc537fb784f0e..72aa078d5da1e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java @@ -19,13 +19,20 @@ package org.apache.pulsar.broker.transaction.pendingack; +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongSumValue; +import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.Metric; +import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.parseMetrics; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; import static org.testng.AssertJUnit.assertFalse; import static org.testng.AssertJUnit.assertNotNull; import static org.testng.AssertJUnit.assertTrue; import static org.testng.AssertJUnit.fail; +import com.google.common.collect.Multimap; +import io.opentelemetry.api.common.Attributes; import java.io.ByteArrayOutputStream; import java.lang.reflect.Field; import java.lang.reflect.Method; @@ -38,19 +45,22 @@ import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import com.google.common.collect.Multimap; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.client.api.BKException; import org.apache.bookkeeper.mledger.ManagedCursor; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.Position; import org.apache.commons.collections4.map.LinkedMap; +import org.apache.pulsar.PrometheusMetricsTestUtil; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.AbstractTopic; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; -import org.apache.pulsar.broker.stats.PrometheusMetricsTest; -import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGenerator; +import org.apache.pulsar.broker.stats.OpenTelemetryTransactionPendingAckStoreStats; import org.apache.pulsar.broker.transaction.TransactionTestBase; import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore; import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; @@ -58,13 +68,14 @@ import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.transaction.Transaction; +import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.BatchMessageIdImpl; import org.apache.pulsar.client.impl.MessageIdImpl; -import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.transaction.TransactionImpl; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.SystemTopicNames; @@ -72,10 +83,13 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; /** @@ -99,6 +113,98 @@ protected void cleanup() { super.internalCleanup(); } + + @DataProvider(name = "retryableErrors") + public Object[][] retryableErrors() { + return new Object[][] { + {new ManagedLedgerException("mock retryable error")}, + {new MetadataStoreException("mock retryable error")}, + {new BKException(-1)}, + }; + } + + /** + * Test consumer can be built successfully with retryable exception + * and get correct error with no-retryable exception. + * @throws Exception + */ + @Test(timeOut = 60000, dataProvider = "retryableErrors") + public void testBuildConsumerEncounterPendingAckInitFailure(Exception retryableError) throws Exception { + // 1. Prepare and make sure the consumer can be built successfully. + String topic = BrokerTestUtil.newUniqueName(NAMESPACE1 + "/tp"); + admin.topics().createNonPartitionedTopic(topic); + Consumer consumer1 = pulsarClient.newConsumer() + .subscriptionName("subName1") + .topic(topic) + .subscribe(); + // 2. Mock a transactionPendingAckStoreProvider to test building consumer + // failing at transactionPendingAckStoreProvider::checkInitializedBefore. + Field transactionPendingAckStoreProviderField = PulsarService.class + .getDeclaredField("transactionPendingAckStoreProvider"); + transactionPendingAckStoreProviderField.setAccessible(true); + TransactionPendingAckStoreProvider pendingAckStoreProvider = + (TransactionPendingAckStoreProvider) transactionPendingAckStoreProviderField + .get(pulsarServiceList.get(0)); + TransactionPendingAckStoreProvider mockProvider = mock(pendingAckStoreProvider.getClass()); + // 3. Test retryable exception when checkInitializedBefore: + // The consumer will be built successfully after one time retry. + when(mockProvider.checkInitializedBefore(any())) + // First, the method checkInitializedBefore will fail with a retryable exception. + .thenReturn(FutureUtil.failedFuture(retryableError)) + // Then, the method will be executed successfully. + .thenReturn(CompletableFuture.completedFuture(false)); + transactionPendingAckStoreProviderField.set(pulsarServiceList.get(0), mockProvider); + Consumer consumer2 = pulsarClient.newConsumer() + .subscriptionName("subName2") + .topic(topic) + .subscribe(); + + // 4. Test retryable exception when newPendingAckStore: + // The consumer will be built successfully after one time retry. + when(mockProvider.checkInitializedBefore(any())) + .thenReturn(CompletableFuture.completedFuture(true)); + + when(mockProvider.newPendingAckStore(any())) + // First, the method newPendingAckStore will fail with a retryable exception. + .thenReturn(FutureUtil.failedFuture(new ManagedLedgerException("mock fail new store"))) + // Then, the method will be executed successfully. + .thenCallRealMethod(); + transactionPendingAckStoreProviderField.set(pulsarServiceList.get(0), mockProvider); + Consumer consumer3 = pulsarClient.newConsumer() + .subscriptionName("subName3") + .topic(topic) + .subscribe(); + + // 5. Test no-retryable exception: + // The consumer building will be failed without retrying. + when(mockProvider.checkInitializedBefore(any())) + // The method checkInitializedBefore will fail with a no-retryable exception without retrying. + .thenReturn(FutureUtil.failedFuture(new ManagedLedgerException + .NonRecoverableLedgerException("mock fail"))) + .thenReturn(CompletableFuture.completedFuture(false)); + PulsarClient pulsarClient = PulsarClient.builder() + .serviceUrl(pulsarServiceList.get(0).getBrokerServiceUrl()) + .operationTimeout(3, TimeUnit.SECONDS) + .build(); + try { + @Cleanup + Consumer consumer4 = pulsarClient.newConsumer() + .subscriptionName("subName4") + .topic(topic) + .subscribe(); + fail(); + } catch (Exception exception) { + assertTrue(exception.getMessage().contains("Failed to init transaction pending ack.")); + } + + // cleanup. + consumer1.close(); + consumer2.close(); + consumer3.close(); + pulsarClient.close(); + admin.topics().delete(topic, false); + } + @Test public void individualPendingAckReplayTest() throws Exception { int messageCount = 1000; @@ -210,8 +316,8 @@ public void individualPendingAckReplayTest() throws Exception { // in order to check out the pending ack cursor is clear whether or not. Awaitility.await() - .until(() -> ((PositionImpl) managedCursor.getMarkDeletedPosition()) - .compareTo((PositionImpl) managedCursor.getManagedLedger().getLastConfirmedEntry()) == -1); + .until(() -> (managedCursor.getMarkDeletedPosition()) + .compareTo(managedCursor.getManagedLedger().getLastConfirmedEntry()) == -1); } @Test @@ -254,35 +360,71 @@ public void testPendingAckMetrics() throws Exception { @Cleanup ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsarServiceList.get(0), true, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsarServiceList.get(0), true, false, false, statsOut); String metricsStr = statsOut.toString(); - Multimap metricsMap = PrometheusMetricsTest.parseMetrics(metricsStr); + Multimap metricsMap = parseMetrics(metricsStr); - Collection abortedCount = metricsMap.get("pulsar_txn_tp_aborted_count_total"); - Collection committedCount = metricsMap.get("pulsar_txn_tp_committed_count_total"); - Collection commitLatency = metricsMap.get("pulsar_txn_tp_commit_latency"); + Collection abortedCount = metricsMap.get("pulsar_txn_tp_aborted_count_total"); + Collection committedCount = metricsMap.get("pulsar_txn_tp_committed_count_total"); + Collection commitLatency = metricsMap.get("pulsar_txn_tp_commit_latency"); Assert.assertTrue(commitLatency.size() > 0); int count = 0; - for (PrometheusMetricsTest.Metric metric : commitLatency) { + for (Metric metric : commitLatency) { if (metric.tags.get("topic").endsWith(PENDING_ACK_REPLAY_TOPIC) && metric.value > 0) { count++; } } Assert.assertTrue(count > 0); - for (PrometheusMetricsTest.Metric metric : abortedCount) { + for (Metric metric : abortedCount) { if (metric.tags.get("subscription").equals(subName) && metric.tags.get("status").equals("succeed")) { assertTrue(metric.tags.get("topic").endsWith(PENDING_ACK_REPLAY_TOPIC)); assertTrue(metric.value > 0); } } - for (PrometheusMetricsTest.Metric metric : committedCount) { + for (Metric metric : committedCount) { if (metric.tags.get("subscription").equals(subName) && metric.tags.get("status").equals("succeed")) { assertTrue(metric.tags.get("topic").endsWith(PENDING_ACK_REPLAY_TOPIC)); assertTrue(metric.value > 0); } } + + var otelMetrics = pulsarTestContexts.get(0).getOpenTelemetryMetricReader().collectAllMetrics(); + var commonAttributes = Attributes.builder() + .put(OpenTelemetryAttributes.PULSAR_TENANT, "tnx") + .put(OpenTelemetryAttributes.PULSAR_NAMESPACE, "tnx/ns1") + .put(OpenTelemetryAttributes.PULSAR_TOPIC, TopicName.get(PENDING_ACK_REPLAY_TOPIC).toString()) + .put(OpenTelemetryAttributes.PULSAR_SUBSCRIPTION_NAME, subName) + .build(); + assertMetricLongSumValue(otelMetrics, OpenTelemetryTransactionPendingAckStoreStats.ACK_COUNTER, + Attributes.builder() + .putAll(commonAttributes) + .put(OpenTelemetryAttributes.PULSAR_TRANSACTION_STATUS, "committed") + .put(OpenTelemetryAttributes.PULSAR_TRANSACTION_ACK_STORE_OPERATION_STATUS, "success") + .build(), + 50); + assertMetricLongSumValue(otelMetrics, OpenTelemetryTransactionPendingAckStoreStats.ACK_COUNTER, + Attributes.builder() + .putAll(commonAttributes) + .put(OpenTelemetryAttributes.PULSAR_TRANSACTION_STATUS, "committed") + .put(OpenTelemetryAttributes.PULSAR_TRANSACTION_ACK_STORE_OPERATION_STATUS, "failure") + .build(), + 0); + assertMetricLongSumValue(otelMetrics, OpenTelemetryTransactionPendingAckStoreStats.ACK_COUNTER, + Attributes.builder() + .putAll(commonAttributes) + .put(OpenTelemetryAttributes.PULSAR_TRANSACTION_STATUS, "aborted") + .put(OpenTelemetryAttributes.PULSAR_TRANSACTION_ACK_STORE_OPERATION_STATUS, "success") + .build(), + 50); + assertMetricLongSumValue(otelMetrics, OpenTelemetryTransactionPendingAckStoreStats.ACK_COUNTER, + Attributes.builder() + .putAll(commonAttributes) + .put(OpenTelemetryAttributes.PULSAR_TRANSACTION_STATUS, "aborted") + .put(OpenTelemetryAttributes.PULSAR_TRANSACTION_ACK_STORE_OPERATION_STATUS, "failure") + .build(), + 0); } @Test @@ -375,8 +517,8 @@ public void cumulativePendingAckReplayTest() throws Exception { // in order to check out the pending ack cursor is clear whether or not. Awaitility.await() - .until(() -> ((PositionImpl) managedCursor.getMarkDeletedPosition()) - .compareTo((PositionImpl) managedCursor.getManagedLedger().getLastConfirmedEntry()) == 0); + .until(() -> (managedCursor.getMarkDeletedPosition()) + .compareTo(managedCursor.getManagedLedger().getLastConfirmedEntry()) == 0); } @Test @@ -492,8 +634,8 @@ public void testDeleteUselessLogDataWhenSubCursorMoved() throws Exception { field3.setAccessible(true); field4.setAccessible(true); - ConcurrentSkipListMap pendingAckLogIndex = - (ConcurrentSkipListMap) field3.get(pendingAckStore); + ConcurrentSkipListMap pendingAckLogIndex = + (ConcurrentSkipListMap) field3.get(pendingAckStore); long maxIndexLag = (long) field4.get(pendingAckStore); Assert.assertEquals(pendingAckLogIndex.size(), 0); Assert.assertEquals(maxIndexLag, 5); @@ -635,8 +777,8 @@ public void testPendingAckLowWaterMarkRemoveFirstTxn() throws Exception { PendingAckHandleImpl oldPendingAckHandle = (PendingAckHandleImpl) field1.get(persistentSubscription); Field field2 = PendingAckHandleImpl.class.getDeclaredField("individualAckOfTransaction"); field2.setAccessible(true); - LinkedMap> oldIndividualAckOfTransaction = - (LinkedMap>) field2.get(oldPendingAckHandle); + LinkedMap> oldIndividualAckOfTransaction = + (LinkedMap>) field2.get(oldPendingAckHandle); Awaitility.await().untilAsserted(() -> Assert.assertEquals(oldIndividualAckOfTransaction.size(), 0)); PendingAckHandleImpl pendingAckHandle = new PendingAckHandleImpl(persistentSubscription); @@ -656,8 +798,8 @@ public void testPendingAckLowWaterMarkRemoveFirstTxn() throws Exception { }); - LinkedMap> individualAckOfTransaction = - (LinkedMap>) field2.get(pendingAckHandle); + LinkedMap> individualAckOfTransaction = + (LinkedMap>) field2.get(pendingAckHandle); assertFalse(individualAckOfTransaction.containsKey(transaction1.getTxnID())); assertFalse(individualAckOfTransaction.containsKey(transaction2.getTxnID())); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreTest.java index 19d6cc85c9ff6..6dd3e6e7c7822 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreTest.java @@ -18,6 +18,10 @@ */ package org.apache.pulsar.broker.transaction.pendingack.impl; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedHashSet; @@ -30,7 +34,8 @@ import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; @@ -42,7 +47,6 @@ import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.transaction.coordinator.impl.TxnLogBufferedWriterConfig; -import static org.mockito.Mockito.*; import org.awaitility.Awaitility; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -172,7 +176,7 @@ public void testMainProcess(boolean writeWithBatch, boolean readWithBatch) throw List> futureList = new ArrayList<>(); for (int i = 0; i < 20; i++){ TxnID txnID = new TxnID(i, i); - PositionImpl position = PositionImpl.get(i, i); + Position position = PositionFactory.create(i, i); futureList.add(mlPendingAckStoreForWrite.appendCumulativeAck(txnID, position)); } for (int i = 0; i < 10; i++){ @@ -185,7 +189,7 @@ public void testMainProcess(boolean writeWithBatch, boolean readWithBatch) throw } for (int i = 40; i < 50; i++){ TxnID txnID = new TxnID(i, i); - PositionImpl position = PositionImpl.get(i, i); + Position position = PositionFactory.create(i, i); futureList.add(mlPendingAckStoreForWrite.appendCumulativeAck(txnID, position)); } FutureUtil.waitForAll(futureList).get(); @@ -210,7 +214,7 @@ public void testMainProcess(boolean writeWithBatch, boolean readWithBatch) throw LinkedHashSet expectedPositions = calculatePendingAckIndexes(positionList, skipSet); Assert.assertEquals( mlPendingAckStoreForWrite.pendingAckLogIndex.keySet().stream() - .map(PositionImpl::getEntryId).collect(Collectors.toList()), + .map(Position::getEntryId).collect(Collectors.toList()), new ArrayList<>(expectedPositions) ); // Replay. @@ -237,19 +241,19 @@ public Object answer(InvocationOnMock invocation) throws Throwable { // Verify build sparse indexes correct after replay. Assert.assertEquals(mlPendingAckStoreForRead.pendingAckLogIndex.size(), mlPendingAckStoreForWrite.pendingAckLogIndex.size()); - Iterator> iteratorReplay = + Iterator> iteratorReplay = mlPendingAckStoreForRead.pendingAckLogIndex.entrySet().iterator(); - Iterator> iteratorWrite = + Iterator> iteratorWrite = mlPendingAckStoreForWrite.pendingAckLogIndex.entrySet().iterator(); while (iteratorReplay.hasNext()){ - Map.Entry replayEntry = iteratorReplay.next(); - Map.Entry writeEntry = iteratorWrite.next(); + Map.Entry replayEntry = iteratorReplay.next(); + Map.Entry writeEntry = iteratorWrite.next(); Assert.assertEquals(replayEntry.getKey(), writeEntry.getKey()); Assert.assertEquals(replayEntry.getValue().getLedgerId(), writeEntry.getValue().getLedgerId()); Assert.assertEquals(replayEntry.getValue().getEntryId(), writeEntry.getValue().getEntryId()); } // Verify delete correct. - when(managedCursorMock.getPersistentMarkDeletedPosition()).thenReturn(PositionImpl.get(19, 19)); + when(managedCursorMock.getPersistentMarkDeletedPosition()).thenReturn(PositionFactory.create(19, 19)); mlPendingAckStoreForWrite.clearUselessLogData(); mlPendingAckStoreForRead.clearUselessLogData(); Assert.assertTrue(mlPendingAckStoreForWrite.pendingAckLogIndex.keySet().iterator().next().getEntryId() > 19); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/ProcessHandlerFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/ProcessHandlerFilterTest.java index cabde4af1f167..6d28aeb9f9dd1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/ProcessHandlerFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/ProcessHandlerFilterTest.java @@ -60,7 +60,6 @@ public void testChainDoFilter() throws ServletException, IOException { Mockito.doReturn(config).when(mockPulsarService).getConfig(); // request has MULTIPART_FORM_DATA content-type config.setBrokerInterceptors(Sets.newHashSet("Interceptor1","Interceptor2")); - config.setDisableBrokerInterceptors(false); HttpServletRequest mockHttpServletRequest2 = Mockito.mock(HttpServletRequest.class); Mockito.doReturn(MediaType.MULTIPART_FORM_DATA).when(mockHttpServletRequest2).getContentType(); ProcessHandlerFilter processHandlerFilter2 = new ProcessHandlerFilter(mockPulsarService.getBrokerInterceptor()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceOriginalClientIPTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceOriginalClientIPTest.java new file mode 100644 index 0000000000000..7f7fa85bd3bb4 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceOriginalClientIPTest.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.web; + +import static org.testng.Assert.assertTrue; +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import nl.altindag.console.ConsoleCaptor; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.assertj.core.api.ThrowingConsumer; +import org.awaitility.Awaitility; +import org.eclipse.jetty.client.HttpClient; +import org.eclipse.jetty.client.ProxyProtocolClientConnectionFactory.V2; +import org.eclipse.jetty.client.api.ContentResponse; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class WebServiceOriginalClientIPTest extends MockedPulsarServiceBaseTest { + HttpClient httpClient; + + @BeforeClass(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.internalSetup(); + httpClient = new HttpClient(new SslContextFactory(true)); + httpClient.start(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + if (httpClient != null) { + httpClient.stop(); + } + } + + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + conf.setWebServiceTrustXForwardedFor(true); + conf.setWebServiceHaProxyProtocolEnabled(true); + conf.setWebServicePortTls(Optional.of(0)); + conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); + conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + } + + @DataProvider(name = "tlsEnabled") + public Object[][] tlsEnabled() { + return new Object[][] { { true }, { false } }; + } + + @Test(dataProvider = "tlsEnabled") + public void testClientIPIsPickedFromXForwardedForHeaderAndLogged(boolean tlsEnabled) throws Exception { + String metricsUrl = + (tlsEnabled ? pulsar.getWebServiceAddressTls() : pulsar.getWebServiceAddress()) + "/metrics/"; + performLoggingTest(consoleCaptor -> { + // Send a GET request to the metrics URL + ContentResponse response = httpClient.newRequest(metricsUrl) + .header("X-Forwarded-For", "11.22.33.44:12345") + .send(); + + // Validate the response + assertTrue(response.getContentAsString().contains("process_cpu_seconds_total")); + + // Validate that the client IP passed in X-Forwarded-For is logged + assertTrue(consoleCaptor.getStandardOutput().stream() + .anyMatch(line -> line.contains("RequestLog") && line.contains("[R:11.22.33.44:12345 via "))); + }); + } + + @Test(dataProvider = "tlsEnabled") + public void testClientIPIsPickedFromForwardedHeaderAndLogged(boolean tlsEnabled) throws Exception { + String metricsUrl = + (tlsEnabled ? pulsar.getWebServiceAddressTls() : pulsar.getWebServiceAddress()) + "/metrics/"; + performLoggingTest(consoleCaptor -> { + // Send a GET request to the metrics URL + ContentResponse response = httpClient.newRequest(metricsUrl) + .header("Forwarded", "for=11.22.33.44:12345") + .send(); + + // Validate the response + assertTrue(response.getContentAsString().contains("process_cpu_seconds_total")); + + // Validate that the client IP passed in Forwarded is logged + assertTrue(consoleCaptor.getStandardOutput().stream() + .anyMatch(line -> line.contains("RequestLog") && line.contains("[R:11.22.33.44:12345 via "))); + }); + } + + @Test(dataProvider = "tlsEnabled") + public void testClientIPIsPickedFromHAProxyProtocolAndLogged(boolean tlsEnabled) throws Exception { + String metricsUrl = (tlsEnabled ? pulsar.getWebServiceAddressTls() : pulsar.getWebServiceAddress()) + "/metrics/"; + performLoggingTest(consoleCaptor -> { + // Send a GET request to the metrics URL + ContentResponse response = httpClient.newRequest(metricsUrl) + // Jetty client will add HA Proxy protocol header with the given IP to the request + .tag(new V2.Tag(V2.Tag.Command.PROXY, null, V2.Tag.Protocol.STREAM, + // source IP and port + "99.22.33.44", 1234, + // destination IP and port + "5.4.3.1", 4321, + null)) + .send(); + + // Validate the response + assertTrue(response.getContentAsString().contains("process_cpu_seconds_total")); + + // Validate that the client IP and destination IP passed in HA Proxy protocol is logged + assertTrue(consoleCaptor.getStandardOutput().stream() + .anyMatch(line -> line.contains("RequestLog") && line.contains("[R:99.22.33.44:1234 via ") + && line.contains(" dst 5.4.3.1:4321]"))); + }); + } + + void performLoggingTest(ThrowingConsumer testFunction) { + ConsoleCaptor consoleCaptor = new ConsoleCaptor(); + try { + Awaitility.await().atMost(Duration.of(2, ChronoUnit.SECONDS)).untilAsserted(() -> { + consoleCaptor.clearOutput(); + testFunction.accept(consoleCaptor); + }); + } finally { + consoleCaptor.close(); + System.out.println("--- Captured console output:"); + consoleCaptor.getStandardOutput().forEach(System.out::println); + consoleCaptor.getErrorOutput().forEach(System.err::println); + System.out.println("--- End of captured console output"); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java index b069d31dc6e0d..08041d72c7e44 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java @@ -18,6 +18,10 @@ */ package org.apache.pulsar.broker.web; +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat; +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongSumValue; +import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.Metric; +import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.parseMetrics; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -26,10 +30,12 @@ import com.google.common.io.CharStreams; import com.google.common.io.Closeables; import io.netty.handler.ssl.util.InsecureTrustManagerFactory; +import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.net.HttpURLConnection; import java.net.URL; import java.security.KeyStore; import java.security.PrivateKey; @@ -42,6 +48,8 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.zip.GZIPInputStream; +import java.util.zip.ZipException; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.KeyManager; import javax.net.ssl.KeyManagerFactory; @@ -49,11 +57,13 @@ import javax.net.ssl.TrustManager; import lombok.Cleanup; import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.PrometheusMetricsTestUtil; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.stats.PrometheusMetricsTest; -import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGenerator; import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.apache.pulsar.broker.web.RateLimitingFilter.Result; +import org.apache.pulsar.broker.web.WebExecutorThreadPoolStats.LimitType; +import org.apache.pulsar.broker.web.WebExecutorThreadPoolStats.UsageType; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminBuilder; import org.apache.pulsar.client.admin.PulsarAdminException.ConflictException; @@ -101,34 +111,47 @@ public class WebServiceTest { @Test public void testWebExecutorMetrics() throws Exception { setupEnv(true, false, false, false, -1, false); + + var otelMetrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + assertMetricLongSumValue(otelMetrics, WebExecutorThreadPoolStats.LIMIT_COUNTER, LimitType.MAX.attributes, + value -> assertThat(value).isPositive()); + assertMetricLongSumValue(otelMetrics, WebExecutorThreadPoolStats.LIMIT_COUNTER, LimitType.MIN.attributes, + value -> assertThat(value).isPositive()); + assertMetricLongSumValue(otelMetrics, WebExecutorThreadPoolStats.USAGE_COUNTER, UsageType.ACTIVE.attributes, + value -> assertThat(value).isNotNegative()); + assertMetricLongSumValue(otelMetrics, WebExecutorThreadPoolStats.USAGE_COUNTER, UsageType.CURRENT.attributes, + value -> assertThat(value).isPositive()); + assertMetricLongSumValue(otelMetrics, WebExecutorThreadPoolStats.USAGE_COUNTER, UsageType.IDLE.attributes, + value -> assertThat(value).isNotNegative()); + ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); - PrometheusMetricsGenerator.generate(pulsar, false, false, false, statsOut); + PrometheusMetricsTestUtil.generate(pulsar, false, false, false, statsOut); String metricsStr = statsOut.toString(); - Multimap metrics = PrometheusMetricsTest.parseMetrics(metricsStr); + Multimap metrics = parseMetrics(metricsStr); - Collection maxThreads = metrics.get("pulsar_web_executor_max_threads"); - Collection minThreads = metrics.get("pulsar_web_executor_min_threads"); - Collection activeThreads = metrics.get("pulsar_web_executor_active_threads"); - Collection idleThreads = metrics.get("pulsar_web_executor_idle_threads"); - Collection currentThreads = metrics.get("pulsar_web_executor_current_threads"); + Collection maxThreads = metrics.get("pulsar_web_executor_max_threads"); + Collection minThreads = metrics.get("pulsar_web_executor_min_threads"); + Collection activeThreads = metrics.get("pulsar_web_executor_active_threads"); + Collection idleThreads = metrics.get("pulsar_web_executor_idle_threads"); + Collection currentThreads = metrics.get("pulsar_web_executor_current_threads"); - for (PrometheusMetricsTest.Metric metric : maxThreads) { + for (Metric metric : maxThreads) { Assert.assertNotNull(metric.tags.get("cluster")); Assert.assertTrue(metric.value > 0); } - for (PrometheusMetricsTest.Metric metric : minThreads) { + for (Metric metric : minThreads) { Assert.assertNotNull(metric.tags.get("cluster")); Assert.assertTrue(metric.value > 0); } - for (PrometheusMetricsTest.Metric metric : activeThreads) { + for (Metric metric : activeThreads) { Assert.assertNotNull(metric.tags.get("cluster")); Assert.assertTrue(metric.value >= 0); } - for (PrometheusMetricsTest.Metric metric : idleThreads) { + for (Metric metric : idleThreads) { Assert.assertNotNull(metric.tags.get("cluster")); Assert.assertTrue(metric.value >= 0); } - for (PrometheusMetricsTest.Metric metric : currentThreads) { + for (Metric metric : currentThreads) { Assert.assertNotNull(metric.tags.get("cluster")); Assert.assertTrue(metric.value > 0); } @@ -248,12 +271,29 @@ public void testTlsAuthDisallowInsecure() throws Exception { public void testRateLimiting() throws Exception { setupEnv(false, false, false, false, 10.0, false); + // setupEnv makes a HTTP call to create the cluster. + var metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + assertMetricLongSumValue(metrics, RateLimitingFilter.RATE_LIMIT_REQUEST_COUNT_METRIC_NAME, + Result.ACCEPTED.attributes, 1); + assertThat(metrics).noneSatisfy(metricData -> assertThat(metricData) + .hasName(RateLimitingFilter.RATE_LIMIT_REQUEST_COUNT_METRIC_NAME) + .hasLongSumSatisfying( + sum -> sum.hasPointsSatisfying(point -> point.hasAttributes(Result.REJECTED.attributes)))); + // Make requests without exceeding the max rate for (int i = 0; i < 5; i++) { makeHttpRequest(false, false); Thread.sleep(200); } + metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + assertMetricLongSumValue(metrics, RateLimitingFilter.RATE_LIMIT_REQUEST_COUNT_METRIC_NAME, + Result.ACCEPTED.attributes, 6); + assertThat(metrics).noneSatisfy(metricData -> assertThat(metricData) + .hasName(RateLimitingFilter.RATE_LIMIT_REQUEST_COUNT_METRIC_NAME) + .hasLongSumSatisfying( + sum -> sum.hasPointsSatisfying(point -> point.hasAttributes(Result.REJECTED.attributes)))); + try { for (int i = 0; i < 500; i++) { makeHttpRequest(false, false); @@ -263,6 +303,12 @@ public void testRateLimiting() throws Exception { } catch (IOException e) { assertTrue(e.getMessage().contains("429")); } + + metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + assertMetricLongSumValue(metrics, RateLimitingFilter.RATE_LIMIT_REQUEST_COUNT_METRIC_NAME, + Result.ACCEPTED.attributes, value -> assertThat(value).isGreaterThan(6)); + assertMetricLongSumValue(metrics, RateLimitingFilter.RATE_LIMIT_REQUEST_COUNT_METRIC_NAME, + Result.REJECTED.attributes, value -> assertThat(value).isPositive()); } @Test @@ -353,6 +399,71 @@ public void testBrokerReady() throws Exception { assertEquals(res.getResponseBody(), "ok"); } + @Test + public void testCompressOutputMetricsInPrometheus() throws Exception { + setupEnv(true, false, false, false, -1, false); + + String metricsUrl = pulsar.getWebServiceAddress() + "/metrics/"; + + URL url = new URL(metricsUrl); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("GET"); + connection.setRequestProperty("Accept-Encoding", "gzip"); + + StringBuilder content = new StringBuilder(); + + try (InputStream inputStream = connection.getInputStream()) { + try (GZIPInputStream gzipInputStream = new GZIPInputStream(inputStream)) { + // Process the decompressed content + int data; + while ((data = gzipInputStream.read()) != -1) { + content.append((char) data); + } + } + + log.info("Response Content: {}", content); + assertTrue(content.toString().contains("process_cpu_seconds_total")); + } catch (IOException e) { + log.error("Failed to decompress the content, likely the content is not compressed ", e); + fail(); + } finally { + connection.disconnect(); + } + } + + @Test + public void testUnCompressOutputMetricsInPrometheus() throws Exception { + setupEnv(true, false, false, false, -1, false); + + String metricsUrl = pulsar.getWebServiceAddress() + "/metrics/"; + + URL url = new URL(metricsUrl); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("GET"); + + StringBuilder content = new StringBuilder(); + + try (InputStream inputStream = connection.getInputStream()) { + try (GZIPInputStream gzipInputStream = new GZIPInputStream(inputStream)) { + fail(); + } catch (IOException e) { + assertTrue(e instanceof ZipException); + } + + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); + String line; + while ((line = reader.readLine()) != null) { + content.append(line + "\n"); + } + } finally { + connection.disconnect(); + } + + log.info("Response Content: {}", content); + + assertTrue(content.toString().contains("process_cpu_seconds_total")); + } + private String makeHttpRequest(boolean useTls, boolean useAuth) throws Exception { InputStream response = null; try { @@ -428,6 +539,7 @@ private void setupEnv(boolean enableFilter, boolean enableTls, boolean enableAut pulsarTestContext = PulsarTestContext.builder() .spyByDefault() .config(config) + .enableOpenTelemetry(true) .build(); pulsar = pulsarTestContext.getPulsarService(); @@ -451,7 +563,7 @@ private void setupEnv(boolean enableFilter, boolean enableTls, boolean enableAut + "/lookup/v2/destination/persistent/my-property/local/my-namespace/my-topic"; BROKER_LOOKUP_URL_TLS = BROKER_URL_BASE_TLS + "/lookup/v2/destination/persistent/my-property/local/my-namespace/my-topic"; - + @Cleanup PulsarAdmin pulsarAdmin = adminBuilder.serviceHttpUrl(serviceUrl).build(); try { @@ -459,8 +571,6 @@ private void setupEnv(boolean enableFilter, boolean enableTls, boolean enableAut ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); } catch (ConflictException ce) { // This is OK. - } finally { - pulsarAdmin.close(); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/ClusterMetadataSetupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/ClusterMetadataSetupTest.java index da5914f60e2ac..0c402a83e4227 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/ClusterMetadataSetupTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/ClusterMetadataSetupTest.java @@ -44,6 +44,7 @@ import org.apache.pulsar.PulsarInitialNamespaceSetup; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.broker.resources.TenantResources; +import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.functions.worker.WorkerUtils; @@ -73,10 +74,11 @@ public void testReSetupClusterMetadata() throws Exception { "--cluster", "testReSetupClusterMetadata-cluster", "--zookeeper", "127.0.0.1:" + localZkS.getZookeeperPort(), "--configuration-store", "127.0.0.1:" + localZkS.getZookeeperPort(), + "--configuration-metadata-store-config-path", "src/test/resources/conf/zk_client_enable_sasl.conf", "--web-service-url", "http://127.0.0.1:8080", "--web-service-url-tls", "https://127.0.0.1:8443", "--broker-service-url", "pulsar://127.0.0.1:6650", - "--broker-service-url-tls","pulsar+ssl://127.0.0.1:6651" + "--broker-service-url-tls", "pulsar+ssl://127.0.0.1:6651" }; PulsarClusterMetadataSetup.main(args); SortedMap data1 = localZkS.dumpData(); @@ -86,6 +88,48 @@ public void testReSetupClusterMetadata() throws Exception { PulsarClusterMetadataSetup.main(args); SortedMap data3 = localZkS.dumpData(); assertEquals(data1, data3); + String clusterDataJson = data1.get("/admin/clusters/testReSetupClusterMetadata-cluster"); + assertNotNull(clusterDataJson); + ClusterData clusterData = ObjectMapperFactory + .getMapper() + .reader() + .readValue(clusterDataJson, ClusterData.class); + assertEquals(clusterData.getServiceUrl(), "http://127.0.0.1:8080"); + assertEquals(clusterData.getServiceUrlTls(), "https://127.0.0.1:8443"); + assertEquals(clusterData.getBrokerServiceUrl(), "pulsar://127.0.0.1:6650"); + assertEquals(clusterData.getBrokerServiceUrlTls(), "pulsar+ssl://127.0.0.1:6651"); + assertFalse(clusterData.isBrokerClientTlsEnabled()); + } + + public void testSetupClusterMetadataWithAuthEnabled() throws Exception { + String clusterName = "cluster-with-auth"; + String[] args = { + "--cluster", clusterName, + "--zookeeper", "127.0.0.1:" + localZkS.getZookeeperPort(), + "--configuration-store", "127.0.0.1:" + localZkS.getZookeeperPort(), + "--web-service-url", "http://127.0.0.1:8080", + "--web-service-url-tls", "https://127.0.0.1:8443", + "--broker-service-url", "pulsar://127.0.0.1:6650", + "--broker-service-url-tls", "pulsar+ssl://127.0.0.1:6651", + "--tls-enable", + "--auth-plugin", "org.apache.pulsar.client.impl.auth.AuthenticationToken", + "--auth-parameters", "token:my-token" + }; + PulsarClusterMetadataSetup.main(args); + SortedMap data = localZkS.dumpData(); + String clusterDataJson = data.get("/admin/clusters/" + clusterName); + assertNotNull(clusterDataJson); + ClusterData clusterData = ObjectMapperFactory + .getMapper() + .reader() + .readValue(clusterDataJson, ClusterData.class); + assertEquals(clusterData.getServiceUrl(), "http://127.0.0.1:8080"); + assertEquals(clusterData.getServiceUrlTls(), "https://127.0.0.1:8443"); + assertEquals(clusterData.getBrokerServiceUrl(), "pulsar://127.0.0.1:6650"); + assertEquals(clusterData.getBrokerServiceUrlTls(), "pulsar+ssl://127.0.0.1:6651"); + assertTrue(clusterData.isBrokerClientTlsEnabled()); + assertEquals(clusterData.getAuthenticationPlugin(), "org.apache.pulsar.client.impl.auth.AuthenticationToken"); + assertEquals(clusterData.getAuthenticationParameters(), "token:my-token"); } @DataProvider(name = "bundleNumberForDefaultNamespace") diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/ZKReconnectTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/ZKReconnectTest.java new file mode 100644 index 0000000000000..7b9e4beec6bdf --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/ZKReconnectTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.zookeeper; + +import com.google.common.collect.Sets; +import org.apache.pulsar.broker.MetadataSessionExpiredPolicy; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.zookeeper.KeeperException; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.util.concurrent.TimeUnit; + + +@Test +public class ZKReconnectTest extends MockedPulsarServiceBaseTest { + + @BeforeMethod(alwaysRun = true) + @Override + protected void setup() throws Exception { + this.conf.setZookeeperSessionExpiredPolicy(MetadataSessionExpiredPolicy.reconnect); + this.internalSetup(); + admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); + admin.tenants().createTenant("public", + new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("test"))); + admin.namespaces().createNamespace("public/default"); + admin.namespaces().setNamespaceReplicationClusters("public/default", Sets.newHashSet("test")); + } + + @Test + public void testGetPartitionMetadataFailAlsoCanProduceMessage() throws Exception { + + pulsarClient = PulsarClient.builder(). + serviceUrl(pulsar.getBrokerServiceUrl()) + .build(); + + String topic = "testGetPartitionMetadataFailAlsoCanProduceMessage"; + admin.topics().createPartitionedTopic(topic, 5); + Producer producer = pulsarClient.newProducer() + .autoUpdatePartitionsInterval(1, TimeUnit.SECONDS).topic(topic).create(); + + this.mockZooKeeper.setAlwaysFail(KeeperException.Code.SESSIONEXPIRED); + + // clear cache + pulsar.getPulsarResources().getNamespaceResources().getPartitionedTopicResources() + .getCache().delete("/admin/partitioned-topics/public/default/persistent" + + "/testGetPartitionMetadataFailAlsoCanProduceMessage"); + pulsar.getNamespaceService().getOwnershipCache().invalidateLocalOwnerCache(); + + // autoUpdatePartitions 1 second + TimeUnit.SECONDS.sleep(3); + + // also can send message + producer.send("test".getBytes()); + this.mockZooKeeper.unsetAlwaysFail(); + producer.send("test".getBytes()); + producer.close(); + } + + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + this.internalCleanup(); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java index 75ae91f18a305..c46f4744cd5df 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java @@ -49,6 +49,7 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.AuthAction; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.zookeeper.KeeperException.Code; import org.awaitility.Awaitility; @@ -120,6 +121,7 @@ protected void setup() throws Exception { } protected final void internalSetup(Authentication auth) throws Exception { + closeAdmin(); admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString()) .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).authentication(auth) .build()); @@ -258,10 +260,12 @@ public void testAnonymousSyncProducerAndConsumer(int batchMessageDelayMs) throws new TenantInfoImpl(Sets.newHashSet("anonymousUser"), Sets.newHashSet("test"))); // make a PulsarAdmin instance as "anonymousUser" for http request - admin.close(); + closeAdmin(); admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrl.toString()).build()); admin.namespaces().createNamespace("my-property/my-ns", Sets.newHashSet("test")); - admin.topics().grantPermission("persistent://my-property/my-ns/my-topic", "anonymousUser", + String topic = "persistent://my-property/my-ns/my-topic"; + admin.topics().createNonPartitionedTopic(topic); + admin.topics().grantPermission(topic, "anonymousUser", EnumSet.allOf(AuthAction.class)); // setup the client @@ -396,11 +400,6 @@ public void testDeleteAuthenticationPoliciesOfTopic() throws Exception { Awaitility.await().untilAsserted(() -> { assertTrue(pulsar.getPulsarResources().getNamespaceResources().getPolicies(NamespaceName.get("p1/ns1")) .get().auth_policies.getTopicAuthentication().containsKey(partitionedTopic)); - for (int i = 0; i < numPartitions; i++) { - assertTrue(pulsar.getPulsarResources().getNamespaceResources().getPolicies(NamespaceName.get("p1/ns1")) - .get().auth_policies.getTopicAuthentication() - .containsKey(TopicName.get(partitionedTopic).getPartition(i).toString())); - } }); admin.topics().deletePartitionedTopic("persistent://p1/ns1/partitioned-topic"); @@ -502,4 +501,48 @@ public void testCleanupEmptyTopicAuthenticationMap() throws Exception { .get().auth_policies.getTopicAuthentication().containsKey(topic)); }); } + + @Test + public void testCleanupEmptySubscriptionAuthenticationMap() throws Exception { + Map authParams = new HashMap<>(); + authParams.put("tlsCertFile", getTlsFileForClient("admin.cert")); + authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8")); + Authentication authTls = new AuthenticationTls(); + authTls.configure(authParams); + internalSetup(authTls); + + admin.clusters().createCluster("test", ClusterData.builder().build()); + admin.tenants().createTenant("p1", + new TenantInfoImpl(Collections.emptySet(), new HashSet<>(admin.clusters().getClusters()))); + String namespace = "p1/ns1"; + admin.namespaces().createNamespace("p1/ns1"); + + // grant permission1 and permission2 + String subscription = "test-sub-1"; + String role1 = "test-user-1"; + String role2 = "test-user-2"; + Set roles = new HashSet<>(); + roles.add(role1); + roles.add(role2); + admin.namespaces().grantPermissionOnSubscription(namespace, subscription, roles); + Optional policies = pulsar.getPulsarResources().getNamespaceResources().getPolicies(NamespaceName.get(namespace)); + assertTrue(policies.isPresent()); + assertTrue(policies.get().auth_policies.getSubscriptionAuthentication().containsKey(subscription)); + assertTrue(policies.get().auth_policies.getSubscriptionAuthentication().get(subscription).contains(role1)); + assertTrue(policies.get().auth_policies.getSubscriptionAuthentication().get(subscription).contains(role2)); + + // revoke permission1 + admin.namespaces().revokePermissionOnSubscription(namespace, subscription, role1); + policies = pulsar.getPulsarResources().getNamespaceResources().getPolicies(NamespaceName.get(namespace)); + assertTrue(policies.isPresent()); + assertTrue(policies.get().auth_policies.getSubscriptionAuthentication().containsKey(subscription)); + assertFalse(policies.get().auth_policies.getSubscriptionAuthentication().get(subscription).contains(role1)); + assertTrue(policies.get().auth_policies.getSubscriptionAuthentication().get(subscription).contains(role2)); + + // revoke permission2 + admin.namespaces().revokePermissionOnSubscription(namespace, subscription, role2); + policies = pulsar.getPulsarResources().getNamespaceResources().getPolicies(NamespaceName.get(namespace)); + assertTrue(policies.isPresent()); + assertFalse(policies.get().auth_policies.getSubscriptionAuthentication().containsKey(subscription)); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java index e3bd321d76332..042c9b328d58b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java @@ -30,6 +30,7 @@ import org.apache.pulsar.client.impl.auth.AuthenticationTls; import org.apache.pulsar.common.tls.PublicSuffixMatcher; import org.apache.pulsar.common.tls.TlsHostnameVerifier; +import org.assertj.core.util.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -141,6 +142,7 @@ public void testTlsSyncProducerAndConsumerWithInvalidBrokerHost(boolean hostname // setup broker cert which has CN = "pulsar" different than broker's hostname="localhost" conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); + conf.setAuthenticationProviders(Sets.newTreeSet(AuthenticationProviderTls.class.getName())); conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); conf.setTlsCertificateFilePath(TLS_MIM_SERVER_CERT_FILE_PATH); conf.setTlsKeyFilePath(TLS_MIM_SERVER_KEY_FILE_PATH); @@ -182,6 +184,7 @@ public void testTlsSyncProducerAndConsumerCorrectBrokerHost() throws Exception { // setup broker cert which has CN = "localhost" conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); + conf.setAuthenticationProviders(Sets.newTreeSet(AuthenticationProviderTls.class.getName())); conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java index 0ce3b7df07d1f..2638709abc5e2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java @@ -37,7 +37,6 @@ import java.util.Set; import java.util.TreeMap; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import javax.naming.AuthenticationException; import lombok.Cleanup; @@ -64,7 +63,6 @@ import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TenantOperation; import org.apache.pulsar.common.policies.data.TopicOperation; -import org.apache.pulsar.common.util.RestException; import org.apache.pulsar.packages.management.core.MockedPackagesStorageProvider; import org.awaitility.Awaitility; import org.slf4j.Logger; @@ -119,6 +117,7 @@ protected void cleanup() throws Exception { public void testProducerAndConsumerAuthorization() throws Exception { log.info("-- Starting {} test --", methodName); cleanup(); + conf.setTopicLevelPoliciesEnabled(false); conf.setAuthorizationProvider(TestAuthorizationProvider.class.getName()); setup(); @@ -179,6 +178,7 @@ public void testProducerAndConsumerAuthorization() throws Exception { public void testSubscriberPermission() throws Exception { log.info("-- Starting {} test --", methodName); cleanup(); + conf.setTopicLevelPoliciesEnabled(false); conf.setEnablePackagesManagement(true); conf.setPackagesManagementStorageProvider(MockedPackagesStorageProvider.class.getName()); conf.setAuthorizationProvider(PulsarAuthorizationProvider.class.getName()); @@ -234,6 +234,7 @@ public void testSubscriberPermission() throws Exception { } // grant topic consume authorization to the subscriptionRole + tenantAdmin.topics().createNonPartitionedTopic(topicName); tenantAdmin.topics().grantPermission(topicName, subscriptionRole, Collections.singleton(AuthAction.consume)); @@ -349,7 +350,8 @@ public void testSubscriberPermission() throws Exception { } catch (Exception e) { // my-sub1 has no msg backlog, so expire message won't be issued on that subscription assertTrue(e.getMessage().startsWith("Expire message by timestamp not issued on topic")); - } sub1Admin.topics().peekMessages(topicName, subscriptionName, 1); + } + sub1Admin.topics().peekMessages(topicName, subscriptionName, 1); sub1Admin.topics().resetCursor(topicName, subscriptionName, 10); sub1Admin.topics().resetCursor(topicName, subscriptionName, MessageId.earliest); @@ -369,6 +371,7 @@ public void testSubscriberPermission() throws Exception { public void testClearBacklogPermission() throws Exception { log.info("-- Starting {} test --", methodName); cleanup(); + conf.setTopicLevelPoliciesEnabled(false); conf.setAuthorizationProvider(PulsarAuthorizationProvider.class.getName()); setup(); @@ -610,6 +613,7 @@ public void testUpdateTopicPropertiesAuthorization() throws Exception { public void testSubscriptionPrefixAuthorization() throws Exception { log.info("-- Starting {} test --", methodName); cleanup(); + conf.setTopicLevelPoliciesEnabled(false); conf.setAuthorizationProvider(TestAuthorizationProviderWithSubscriptionPrefix.class.getName()); setup(); @@ -667,6 +671,61 @@ public void testGrantPermission() throws Exception { log.info("-- Exiting {} test --", methodName); } + @Test + public void testRevokePermission() throws Exception { + log.info("-- Starting {} test --", methodName); + cleanup(); + conf.setAuthorizationProvider(PulsarAuthorizationProvider.class.getName()); + setup(); + + Authentication adminAuthentication = new ClientAuthentication("superUser"); + + @Cleanup + PulsarAdmin admin = spy( + PulsarAdmin.builder().serviceHttpUrl(brokerUrl.toString()).authentication(adminAuthentication).build()); + + Authentication authentication = new ClientAuthentication(clientRole); + + replacePulsarClient(PulsarClient.builder() + .serviceUrl(pulsar.getBrokerServiceUrl()) + .authentication(authentication)); + + admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(brokerUrl.toString()).build()); + + admin.tenants().createTenant("public", + new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("test"))); + admin.namespaces().createNamespace("public/default", Sets.newHashSet("test")); + + AuthorizationService authorizationService = new AuthorizationService(conf, pulsar.getPulsarResources()); + TopicName topicName = TopicName.get("persistent://public/default/t1"); + NamespaceName namespaceName = NamespaceName.get("public/default"); + String role = "test-role"; + String role2 = "test-role-2"; + Set actions = Sets.newHashSet(AuthAction.produce, AuthAction.consume); + Assert.assertFalse(authorizationService.canProduce(topicName, role, null)); + Assert.assertFalse(authorizationService.canConsume(topicName, role, null, "sub1")); + authorizationService.grantPermissionAsync(topicName, actions, role, "auth-json").get(); + authorizationService.grantPermissionAsync(topicName, actions, role2, "auth-json").get(); + Assert.assertTrue(authorizationService.canProduce(topicName, role, null)); + Assert.assertTrue(authorizationService.canConsume(topicName, role, null, "sub1")); + + authorizationService.revokePermissionAsync(topicName, role).get(); + Assert.assertFalse(authorizationService.canProduce(topicName, role, null)); + Assert.assertFalse(authorizationService.canConsume(topicName, role, null, "sub1")); + Assert.assertTrue(authorizationService.canProduce(topicName, role2, null)); + Assert.assertTrue(authorizationService.canConsume(topicName, role2, null, "sub1")); + + authorizationService.revokePermissionAsync(topicName, role2).get(); + Assert.assertFalse(authorizationService.canProduce(topicName, role2, null)); + Assert.assertFalse(authorizationService.canConsume(topicName, role2, null, "sub1")); + + authorizationService.grantPermissionAsync(namespaceName, actions, role, null).get(); + Assert.assertTrue(authorizationService.allowNamespaceOperationAsync(namespaceName, NamespaceOperation.GET_TOPIC, role, null).get()); + authorizationService.revokePermissionAsync(namespaceName, role).get(); + Assert.assertFalse(authorizationService.allowNamespaceOperationAsync(namespaceName, NamespaceOperation.GET_TOPIC, role, null).get()); + log.info("-- Exiting {} test --", methodName); + } + @Test public void testAuthData() throws Exception { log.info("-- Starting {} test --", methodName); @@ -694,6 +753,7 @@ public void testAuthData() throws Exception { public void testPermissionForProducerCreateInitialSubscription() throws Exception { log.info("-- Starting {} test --", methodName); cleanup(); + conf.setTopicLevelPoliciesEnabled(false); conf.setAuthorizationProvider(PulsarAuthorizationProvider.class.getName()); setup(); @@ -715,6 +775,7 @@ public void testPermissionForProducerCreateInitialSubscription() throws Exceptio admin.tenants().createTenant("my-property", new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("test"))); admin.namespaces().createNamespace("my-property/my-ns", Sets.newHashSet("test")); + admin.topics().createNonPartitionedTopic(topic); admin.topics().grantPermission(topic, invalidRole, Collections.singleton(AuthAction.produce)); admin.topics().grantPermission(topic, producerRole, Sets.newHashSet(AuthAction.produce, AuthAction.consume)); @@ -836,13 +897,6 @@ public void close() throws IOException { // No-op } - @Override - public CompletableFuture isSuperUser(String role, - ServiceConfiguration serviceConfiguration) { - Set superUserRoles = serviceConfiguration.getSuperUserRoles(); - return CompletableFuture.completedFuture(role != null && superUserRoles.contains(role) ? true : false); - } - @Override public void initialize(ServiceConfiguration conf, PulsarResources pulsarResources) throws IOException { this.conf = conf; @@ -917,23 +971,12 @@ public CompletableFuture allowTenantOperationAsync( return CompletableFuture.completedFuture(true); } - @Override - public Boolean allowTenantOperation( - String tenantName, String role, TenantOperation operation, AuthenticationDataSource authData) { - return true; - } - @Override public CompletableFuture allowNamespaceOperationAsync( NamespaceName namespaceName, String role, NamespaceOperation operation, AuthenticationDataSource authData) { return CompletableFuture.completedFuture(true); } - @Override - public Boolean allowNamespaceOperation( - NamespaceName namespaceName, String role, NamespaceOperation operation, AuthenticationDataSource authData) { - return null; - } @Override public CompletableFuture allowTopicOperationAsync( @@ -948,43 +991,6 @@ public CompletableFuture allowTopicOperationAsync( return isAuthorizedFuture; } - - @Override - public Boolean allowTopicOperation( - TopicName topicName, String role, TopicOperation operation, AuthenticationDataSource authData) { - try { - return allowTopicOperationAsync(topicName, role, operation, authData).get(); - } catch (InterruptedException e) { - throw new RestException(e); - } catch (ExecutionException e) { - throw new RestException(e); - } - } - } - - /** - * This provider always fails authorization on consumer and passes on producer - * - */ - public static class TestAuthorizationProvider2 extends TestAuthorizationProvider { - - @Override - public CompletableFuture canProduceAsync(TopicName topicName, String role, - AuthenticationDataSource authenticationData) { - return CompletableFuture.completedFuture(true); - } - - @Override - public CompletableFuture canConsumeAsync(TopicName topicName, String role, - AuthenticationDataSource authenticationData, String subscription) { - return CompletableFuture.completedFuture(false); - } - - @Override - public CompletableFuture canLookupAsync(TopicName topicName, String role, - AuthenticationDataSource authenticationData) { - return CompletableFuture.completedFuture(true); - } } public static class TestAuthorizationProviderWithSubscriptionPrefix extends TestAuthorizationProvider { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java index 792f419ee997e..157df1185307a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java @@ -18,7 +18,10 @@ */ package org.apache.pulsar.client.api; +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat; +import static org.apache.pulsar.broker.namespace.NamespaceService.LOOKUP_REQUEST_DURATION_METRIC_NAME; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -27,10 +30,12 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import com.github.benmanes.caffeine.cache.AsyncLoadingCache; import com.google.common.collect.Sets; import com.google.common.util.concurrent.MoreExecutors; import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponse; +import io.prometheus.client.CollectorRegistry; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.Field; @@ -48,6 +53,7 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -69,9 +75,17 @@ import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl; import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerWrapper; import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceUnit; +import org.apache.pulsar.broker.namespace.NamespaceEphemeralData; import org.apache.pulsar.broker.namespace.NamespaceService; +import org.apache.pulsar.broker.namespace.OwnedBundle; +import org.apache.pulsar.broker.namespace.OwnershipCache; +import org.apache.pulsar.broker.namespace.ServiceUnitUtils; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.apache.pulsar.client.impl.BinaryProtoLookupService; +import org.apache.pulsar.client.impl.ClientCnx; +import org.apache.pulsar.client.impl.LookupService; +import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.ServiceUnitId; @@ -83,6 +97,7 @@ import org.apache.pulsar.common.util.SecurityUtility; import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; +import org.apache.zookeeper.KeeperException; import org.asynchttpclient.AsyncCompletionHandler; import org.asynchttpclient.AsyncHttpClient; import org.asynchttpclient.AsyncHttpClientConfig; @@ -94,6 +109,7 @@ import org.asynchttpclient.Response; import org.asynchttpclient.channel.DefaultKeepAliveStrategy; import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.AfterMethod; @@ -119,6 +135,12 @@ protected void cleanup() throws Exception { internalCleanup(); } + @Override + protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder builder) { + super.customizeMainPulsarTestContextBuilder(builder); + builder.enableOpenTelemetry(true); + } + /** * Usecase Multiple Broker => Lookup Redirection test * @@ -129,7 +151,7 @@ protected void cleanup() throws Exception { * * @throws Exception */ - @Test + @Test(timeOut = 30_000) public void testMultipleBrokerLookup() throws Exception { log.info("-- Starting {} test --", methodName); @@ -145,7 +167,8 @@ public void testMultipleBrokerLookup() throws Exception { conf2.setConfigurationMetadataStoreUrl("zk:localhost:3181"); @Cleanup - PulsarTestContext pulsarTestContext2 = createAdditionalPulsarTestContext(conf2); + PulsarTestContext pulsarTestContext2 = createAdditionalPulsarTestContext(conf2, + builder -> builder.enableOpenTelemetry(true)); PulsarService pulsar2 = pulsarTestContext2.getPulsarService(); pulsar.getLoadManager().get().writeLoadReportOnZookeeper(); pulsar2.getLoadManager().get().writeLoadReportOnZookeeper(); @@ -161,22 +184,82 @@ public void testMultipleBrokerLookup() throws Exception { // mock: return Broker2 as a Least-loaded broker when leader receives request [3] doReturn(true).when(loadManager1).isCentralized(); - SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getSafeWebServiceAddress(), null); + SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getBrokerId(), null); doReturn(Optional.of(resourceUnit)).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class)); doReturn(Optional.of(resourceUnit)).when(loadManager2).getLeastLoaded(any(ServiceUnitId.class)); loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager1)); - /**** started broker-2 ****/ + // Disable collecting topic stats during this test, as it deadlocks on access to map BrokerService.topics. + pulsar2.getOpenTelemetryTopicStats().close(); + pulsar2.getOpenTelemetryConsumerStats().close(); + pulsar2.getOpenTelemetryProducerStats().close(); + pulsar2.getOpenTelemetryReplicatorStats().close(); + + var metricReader = pulsarTestContext.getOpenTelemetryMetricReader(); + var lookupRequestSemaphoreField = BrokerService.class.getDeclaredField("lookupRequestSemaphore"); + lookupRequestSemaphoreField.setAccessible(true); + var lookupRequestSemaphoreSpy = spy(pulsar.getBrokerService().getLookupRequestSemaphore()); + var cdlAfterLookupSemaphoreAcquire = new CountDownLatch(1); + var cdlLookupSemaphoreVerification = new CountDownLatch(1); + doAnswer(invocation -> { + var ret = invocation.callRealMethod(); + if (Boolean.TRUE.equals(ret)) { + cdlAfterLookupSemaphoreAcquire.countDown(); + cdlLookupSemaphoreVerification.await(); + } + return ret; + }).doCallRealMethod().when(lookupRequestSemaphoreSpy).tryAcquire(); + lookupRequestSemaphoreField.set(pulsar.getBrokerService(), new AtomicReference<>(lookupRequestSemaphoreSpy)); + + var topicLoadRequestSemaphoreField = BrokerService.class.getDeclaredField("topicLoadRequestSemaphore"); + topicLoadRequestSemaphoreField.setAccessible(true); + var topicLoadRequestSemaphoreSpy = spy(pulsar2.getBrokerService().getTopicLoadRequestSemaphore().get()); + + var cdlAfterTopicLoadSemaphoreAcquire = new CountDownLatch(1); + var cdlTopicLoadSemaphoreVerification = new CountDownLatch(1); + + doAnswer(invocation -> { + var ret = invocation.callRealMethod(); + if (Boolean.TRUE.equals(ret)) { + cdlAfterTopicLoadSemaphoreAcquire.countDown(); + cdlTopicLoadSemaphoreVerification.await(); + } + return ret; + }).doCallRealMethod().when(topicLoadRequestSemaphoreSpy).tryAcquire(); + topicLoadRequestSemaphoreField.set(pulsar2.getBrokerService(), + new AtomicReference<>(topicLoadRequestSemaphoreSpy)); + + assertThat(pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics()) + .noneSatisfy(metric -> assertThat(metric).hasName(LOOKUP_REQUEST_DURATION_METRIC_NAME)); + /**** started broker-2 ****/ @Cleanup PulsarClient pulsarClient2 = PulsarClient.builder().serviceUrl(pulsar2.getBrokerServiceUrl()).build(); + var consumerFuture = pulsarClient2.newConsumer().topic("persistent://my-property/my-ns/my-topic1") + .subscriptionName("my-subscriber-name").subscribeAsync(); + + cdlAfterLookupSemaphoreAcquire.await(); + assertThat(metricReader.collectAllMetrics()) + .anySatisfy(metric -> assertThat(metric) + .hasName(BrokerService.TOPIC_LOOKUP_USAGE_METRIC_NAME) + .hasLongSumSatisfying( + sum -> sum.hasPointsSatisfying(point -> point.hasValue(1)))); + cdlLookupSemaphoreVerification.countDown(); + + cdlAfterTopicLoadSemaphoreAcquire.await(); + assertThat(pulsarTestContext2.getOpenTelemetryMetricReader().collectAllMetrics()) + .anySatisfy(metric -> assertThat(metric) + .hasName(BrokerService.TOPIC_LOAD_USAGE_METRIC_NAME) + .hasLongSumSatisfying( + sum -> sum.hasPointsSatisfying(point -> point.hasValue(1)))); + cdlTopicLoadSemaphoreVerification.countDown(); + // load namespace-bundle by calling Broker2 - Consumer consumer = pulsarClient2.newConsumer().topic("persistent://my-property/my-ns/my-topic1") - .subscriptionName("my-subscriber-name").subscribe(); - Producer producer = pulsarClient.newProducer(Schema.BYTES) - .topic("persistent://my-property/my-ns/my-topic1") - .create(); + @Cleanup + var consumer = consumerFuture.get(); + @Cleanup + var producer = pulsarClient.newProducer().topic("persistent://my-property/my-ns/my-topic1").create(); for (int i = 0; i < 10; i++) { String message = "my-message-" + i; @@ -192,11 +275,21 @@ public void testMultipleBrokerLookup() throws Exception { String expectedMessage = "my-message-" + i; testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage); } + + var metrics = metricReader.collectAllMetrics(); + assertThat(metrics) + .anySatisfy(metric -> assertThat(metric) + .hasName(LOOKUP_REQUEST_DURATION_METRIC_NAME) + .hasHistogramSatisfying(histogram -> histogram.hasPointsSatisfying( + point -> point + .hasAttributes(NamespaceService.PULSAR_LOOKUP_RESPONSE_REDIRECT_ATTRIBUTES) + .hasCount(1), + point -> point + .hasAttributes(NamespaceService.PULSAR_LOOKUP_RESPONSE_BROKER_ATTRIBUTES) + .hasCount(1)))); + // Acknowledge the consumption of all messages at once consumer.acknowledgeCumulative(msg); - consumer.close(); - producer.close(); - } @Test @@ -294,7 +387,7 @@ public void testMultipleBrokerDifferentClusterLookup() throws Exception { // mock: return Broker2 as a Least-loaded broker when leader receives request doReturn(true).when(loadManager2).isCentralized(); - SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getSafeWebServiceAddress(), null); + SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getBrokerId(), null); doReturn(Optional.of(resourceUnit)).when(loadManager2).getLeastLoaded(any(ServiceUnitId.class)); loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager2)); /**** started broker-2 ****/ @@ -474,7 +567,7 @@ public void testWebserviceServiceTls() throws Exception { // request [3] doReturn(true).when(loadManager1).isCentralized(); doReturn(true).when(loadManager2).isCentralized(); - SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getWebServiceAddress(), null); + SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getBrokerId(), null); doReturn(Optional.of(resourceUnit)).when(loadManager2).getLeastLoaded(any(ServiceUnitId.class)); doReturn(Optional.of(resourceUnit)).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class)); @@ -507,6 +600,9 @@ public void testWebserviceServiceTls() throws Exception { loadManager1 = null; loadManager2 = null; + + conf.setBrokerServicePortTls(Optional.empty()); + conf.setWebServicePortTls(Optional.empty()); } /** @@ -565,7 +661,7 @@ public void testSplitUnloadLookupTest() throws Exception { loadManagerField.set(pulsar2.getNamespaceService(), new AtomicReference<>(loadManager2)); // mock: return Broker1 as a Least-loaded broker when leader receives request [3] doReturn(true).when(loadManager1).isCentralized(); - SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getSafeWebServiceAddress(), null); + SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getBrokerId(), null); doReturn(Optional.of(resourceUnit)).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class)); doReturn(Optional.of(resourceUnit)).when(loadManager2).getLeastLoaded(any(ServiceUnitId.class)); loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager1)); @@ -680,7 +776,7 @@ public void testModularLoadManagerSplitBundle() throws Exception { loadManagerField.set(pulsar2.getNamespaceService(), new AtomicReference<>(loadManager2)); // mock: return Broker1 as a Least-loaded broker when leader receives request [3] doReturn(true).when(loadManager1).isCentralized(); - SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getSafeWebServiceAddress(), null); + SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getBrokerId(), null); Optional res = Optional.of(resourceUnit); doReturn(res).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class)); doReturn(res).when(loadManager2).getLeastLoaded(any(ServiceUnitId.class)); @@ -746,7 +842,7 @@ public void testModularLoadManagerSplitBundle() throws Exception { }); // Unload the NamespacePolicies and AntiAffinity check. - String currentBroker = String.format("%s:%d", "localhost", pulsar.getListenPortHTTP().get()); + String currentBroker = pulsar.getBrokerId(); assertTrue(loadManager.shouldNamespacePoliciesUnload(namespace,"0x00000000_0xffffffff", currentBroker)); assertTrue(loadManager.shouldAntiAffinityNamespaceUnload(namespace,"0x00000000_0xffffffff", currentBroker)); @@ -823,6 +919,104 @@ public void testSkipSplitBundleIfOnlyOneBroker() throws Exception { } } + @Test + public void testMergeGetPartitionedMetadataRequests() throws Exception { + // Assert the lookup service is a "BinaryProtoLookupService". + final PulsarClientImpl pulsarClientImpl = (PulsarClientImpl) pulsarClient; + final LookupService lookupService = pulsarClientImpl.getLookup(); + assertTrue(lookupService instanceof BinaryProtoLookupService); + + final String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final int topicPartitions = 10; + admin.topics().createPartitionedTopic(tpName, topicPartitions); + + // Verify the request is works after merge the requests. + List> futures = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + futures.add(lookupService.getPartitionedTopicMetadata(TopicName.get(tpName), false)); + } + for (CompletableFuture future : futures) { + assertEquals(future.join().partitions, topicPartitions); + } + + // cleanup. + admin.topics().deletePartitionedTopic(tpName); + } + + @Test + public void testMergeLookupRequests() throws Exception { + // Assert the lookup service is a "BinaryProtoLookupService". + final PulsarClientImpl pulsarClientImpl = (PulsarClientImpl) pulsarClient; + final LookupService lookupService = pulsarClientImpl.getLookup(); + assertTrue(lookupService instanceof BinaryProtoLookupService); + + final String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createNonPartitionedTopic(tpName); + + // Create 1 producer and 100 consumers. + List> producers = new ArrayList<>(); + List> consumers = new ArrayList<>(); + for (int i = 0; i < 20; i++) { + producers.add(pulsarClient.newProducer(Schema.STRING).topic(tpName).create()); + } + for (int i = 0; i < 20; i++) { + consumers.add(pulsarClient.newConsumer(Schema.STRING).topic(tpName).subscriptionName("s" + i).subscribe()); + } + + // Verify the lookup count will be smaller than before improve. + int lookupCountBeforeUnload = calculateLookupRequestCount(); + admin.namespaces().unload(TopicName.get(tpName).getNamespace()); + Awaitility.await().untilAsserted(() -> { + for (Producer p : producers) { + assertEquals(WhiteboxImpl.getInternalState(p, "state").toString(), "Ready"); + } + for (Consumer c : consumers) { + assertEquals(WhiteboxImpl.getInternalState(c, "state").toString(), "Ready"); + } + }); + int lookupCountAfterUnload = calculateLookupRequestCount(); + log.info("lookup count before unload: {}, after unload: {}", lookupCountBeforeUnload, lookupCountAfterUnload); + assertTrue(lookupCountAfterUnload < lookupCountBeforeUnload * 2, + "the lookup count should be smaller than before improve"); + + // Verify the producers and consumers is still works. + List messagesSent = new ArrayList<>(); + int index = 0; + for (Producer producer: producers) { + String message = Integer.valueOf(index++).toString(); + producer.send(message); + messagesSent.add(message); + } + HashSet messagesReceived = new HashSet<>(); + for (Consumer consumer : consumers) { + while (true) { + Message msg = consumer.receive(2, TimeUnit.SECONDS); + if (msg == null) { + break; + } + messagesReceived.add(msg.getValue()); + } + } + assertEquals(messagesReceived.size(), producers.size()); + + // cleanup. + for (Producer producer: producers) { + producer.close(); + } + for (Consumer consumer : consumers) { + consumer.close(); + } + admin.topics().delete(tpName); + } + + private int calculateLookupRequestCount() throws Exception { + int failures = CollectorRegistry.defaultRegistry.getSampleValue("pulsar_broker_lookup_failures_total") + .intValue(); + int answers = CollectorRegistry.defaultRegistry.getSampleValue("pulsar_broker_lookup_answers_total") + .intValue(); + return failures + answers; + } + @Test(timeOut = 10000) public void testPartitionedMetadataWithDeprecatedVersion() throws Exception { @@ -840,6 +1034,8 @@ public void testPartitionedMetadataWithDeprecatedVersion() throws Exception { admin.topics().createPartitionedTopic(dest.toString(), totalPartitions); stopBroker(); + conf.setBrokerServicePortTls(Optional.empty()); + conf.setWebServicePortTls(Optional.empty()); conf.setClientLibraryVersionCheckEnabled(true); startBroker(); @@ -997,4 +1193,159 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat return "invalid"; } } + + @Test + public void testLookupConnectionNotCloseIfGetUnloadingExOrMetadataEx() throws Exception { + String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createNonPartitionedTopic(tpName); + PulsarClientImpl pulsarClientImpl = (PulsarClientImpl) pulsarClient; + Producer producer = pulsarClientImpl.newProducer(Schema.STRING).topic(tpName).create(); + Consumer consumer = pulsarClientImpl.newConsumer(Schema.STRING).topic(tpName) + .subscriptionName("s1").isAckReceiptEnabled(true).subscribe(); + LookupService lookupService = pulsarClientImpl.getLookup(); + assertTrue(lookupService instanceof BinaryProtoLookupService); + ClientCnx lookupConnection = pulsarClientImpl.getCnxPool().getConnection(lookupService.resolveHost()).join(); + + var metricReader = pulsarTestContext.getOpenTelemetryMetricReader(); + assertThat(metricReader.collectAllMetrics()) + .noneSatisfy(metric -> assertThat(metric) + .hasName(LOOKUP_REQUEST_DURATION_METRIC_NAME) + .hasHistogramSatisfying(histogram -> histogram.hasPointsSatisfying( + point -> point + .hasAttributes(NamespaceService.PULSAR_LOOKUP_RESPONSE_FAILURE_ATTRIBUTES), + point -> point + .hasAttributes(NamespaceService.PULSAR_LOOKUP_RESPONSE_BROKER_ATTRIBUTES)))); + + // Verify the socket will not be closed if the bundle is unloading. + BundleOfTopic bundleOfTopic = new BundleOfTopic(tpName); + bundleOfTopic.setBundleIsUnloading(); + try { + lookupService.getBroker(TopicName.get(tpName)).get(); + fail("It should failed due to the namespace bundle is unloading."); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("is being unloaded")); + } + + assertThat(metricReader.collectAllMetrics()) + .anySatisfy(metric -> assertThat(metric) + .hasName(LOOKUP_REQUEST_DURATION_METRIC_NAME) + .hasHistogramSatisfying(histogram -> histogram.hasPointsSatisfying( + point -> point + .hasAttributes(NamespaceService.PULSAR_LOOKUP_RESPONSE_FAILURE_ATTRIBUTES) + .hasCount(1), + point -> point + .hasAttributes(NamespaceService.PULSAR_LOOKUP_RESPONSE_BROKER_ATTRIBUTES)))); + // Do unload topic, trigger producer & consumer reconnection. + pulsar.getBrokerService().getTopic(tpName, false).join().get().close(true); + assertTrue(lookupConnection.ctx().channel().isActive()); + bundleOfTopic.setBundleIsNotUnloading(); + // Assert producer & consumer could reconnect successful. + producer.send("1"); + HashSet messagesReceived = new HashSet<>(); + while (true) { + Message msg = consumer.receive(2, TimeUnit.SECONDS); + if (msg == null) { + break; + } + messagesReceived.add(msg.getValue()); + } + assertTrue(messagesReceived.contains("1")); + + // Verify the socket will not be closed if get a metadata ex. + bundleOfTopic.releaseBundleLockAndMakeAcquireFail(); + try { + lookupService.getBroker(TopicName.get(tpName)).get(); + fail("It should failed due to the acquire bundle lock fail."); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("OperationTimeout")); + } + // Do unload topic, trigger producer & consumer reconnection. + pulsar.getBrokerService().getTopic(tpName, false).join().get().close(true); + assertTrue(lookupConnection.ctx().channel().isActive()); + bundleOfTopic.makeAcquireBundleLockSuccess(); + // Assert producer could reconnect successful. + producer.send("2"); + while (true) { + Message msg = consumer.receive(2, TimeUnit.SECONDS); + if (msg == null) { + break; + } + messagesReceived.add(msg.getValue()); + } + assertTrue(messagesReceived.contains("2")); + + // cleanup. + producer.close(); + consumer.close(); + admin.topics().delete(tpName); + } + + private class BundleOfTopic { + + private NamespaceBundle namespaceBundle; + private OwnershipCache ownershipCache; + private AsyncLoadingCache ownedBundlesCache; + + public BundleOfTopic(String tpName) { + namespaceBundle = pulsar.getNamespaceService().getBundle(TopicName.get(tpName)); + ownershipCache = pulsar.getNamespaceService().getOwnershipCache(); + ownedBundlesCache = WhiteboxImpl.getInternalState(ownershipCache, "ownedBundlesCache"); + } + + private void setBundleIsUnloading() { + ownedBundlesCache.get(namespaceBundle).join().setActive(false); + } + + private void setBundleIsNotUnloading() { + ownedBundlesCache.get(namespaceBundle).join().setActive(true); + } + + private void releaseBundleLockAndMakeAcquireFail() throws Exception { + ownedBundlesCache.synchronous().invalidateAll(); + mockZooKeeper.delete(ServiceUnitUtils.path(namespaceBundle), -1); + mockZooKeeper.setAlwaysFail(KeeperException.Code.OPERATIONTIMEOUT); + } + + private void makeAcquireBundleLockSuccess() throws Exception { + mockZooKeeper.unsetAlwaysFail(); + } + } + + @Test(timeOut = 30000) + public void testLookupConnectionNotCloseIfFailedToAcquireOwnershipOfBundle() throws Exception { + String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createNonPartitionedTopic(tpName); + final var pulsarClientImpl = (PulsarClientImpl) pulsarClient; + final var cache = pulsar.getNamespaceService().getOwnershipCache(); + final var bundle = pulsar.getNamespaceService().getBundle(TopicName.get(tpName)); + final var value = cache.getOwnerAsync(bundle).get().orElse(null); + assertNotNull(value); + + cache.invalidateLocalOwnerCache(); + final var lock = pulsar.getCoordinationService().getLockManager(NamespaceEphemeralData.class) + .acquireLock(ServiceUnitUtils.path(bundle), new NamespaceEphemeralData()).join(); + lock.updateValue(null); + log.info("Updated bundle {} with null", bundle.getBundleRange()); + + // wait for the system topic reader to __change_events is closed, otherwise the test will be affected + Thread.sleep(500); + + final var future = pulsarClientImpl.getLookup().getBroker(TopicName.get(tpName)); + final var cnx = pulsarClientImpl.getCnxPool().getConnections().stream().findAny() + .map(CompletableFuture::join).orElse(null); + assertNotNull(cnx); + + try { + future.get(); + fail(); + } catch (ExecutionException e) { + log.info("getBroker failed with {}: {}", e.getCause().getClass().getName(), e.getMessage()); + assertTrue(e.getCause() instanceof PulsarClientException.BrokerMetadataException); + assertTrue(cnx.ctx().channel().isActive()); + lock.updateValue(value); + lock.release(); + assertTrue(e.getMessage().contains("Failed to acquire ownership")); + pulsarClientImpl.getLookup().getBroker(TopicName.get(tpName)).get(); + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientAuthenticationTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientAuthenticationTlsTest.java index c9b243257c4e1..d716d5a806392 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientAuthenticationTlsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientAuthenticationTlsTest.java @@ -22,6 +22,7 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.expectThrows; import java.util.HashSet; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -50,7 +51,8 @@ protected void doInitConf() throws Exception { Set providers = new HashSet<>(); providers.add(AuthenticationProviderTls.class.getName()); conf.setAuthenticationProviders(providers); - + conf.setWebServicePortTls(Optional.of(0)); + conf.setBrokerServicePortTls(Optional.of(0)); conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientDeduplicationFailureTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientDeduplicationFailureTest.java index 6b3b05405baea..601a8d76aaacd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientDeduplicationFailureTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientDeduplicationFailureTest.java @@ -127,10 +127,22 @@ void setup(Method method) throws Exception { @AfterMethod(alwaysRun = true) void shutdown() throws Exception { log.info("--- Shutting down ---"); - pulsarClient.close(); - admin.close(); - pulsar.close(); - bkEnsemble.stop(); + if (pulsarClient != null) { + pulsarClient.close(); + pulsar = null; + } + if (admin != null) { + admin.close(); + admin = null; + } + if (pulsar != null) { + pulsar.close(); + pulsar = null; + } + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; + } } private static class ProducerThread implements Runnable { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientDeduplicationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientDeduplicationTest.java index d2f9617a5faae..4e96252056d26 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientDeduplicationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientDeduplicationTest.java @@ -32,6 +32,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.impl.BatchMessageIdImpl; import org.apache.pulsar.client.impl.MessageIdImpl; @@ -382,6 +383,7 @@ public void testUpdateSequenceIdInSyncCodeSegment() throws Exception { int totalMessage = 200; int threadSize = 5; String topicName = "subscription"; + @Cleanup("shutdownNow") ExecutorService executorService = Executors.newFixedThreadPool(threadSize); conf.setBrokerDeduplicationEnabled(true); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java index 61c7a98602b69..705b171929be6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java @@ -270,7 +270,7 @@ private void consumerCreatedThenFailsRetryTimeout(String topic) throws Exception if (subscribeCount == 1) { ctx.writeAndFlush(Commands.newSuccess(subscribe.getRequestId())); // Trigger reconnect - ctx.writeAndFlush(Commands.newCloseConsumer(subscribe.getConsumerId(), -1)); + ctx.writeAndFlush(Commands.newCloseConsumer(subscribe.getConsumerId(), -1, null, null)); } else if (subscribeCount != 2) { // Respond to subsequent requests to prevent timeouts ctx.writeAndFlush(Commands.newSuccess(subscribe.getRequestId())); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerBatchReceiveTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerBatchReceiveTest.java index d54b1c99e3e13..974d25aad64db 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerBatchReceiveTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerBatchReceiveTest.java @@ -112,7 +112,7 @@ public Object[][] batchReceivePolicyProvider() { // Number of message limitation exceed receiverQueue size { BatchReceivePolicy.builder() - .maxNumMessages(70) + .maxNumMessages(50) .build(), true, 50, false }, // Number of message limitation exceed receiverQueue size and timeout limitation @@ -147,7 +147,7 @@ public Object[][] batchReceivePolicyProvider() { // Number of message limitation exceed receiverQueue size { BatchReceivePolicy.builder() - .maxNumMessages(70) + .maxNumMessages(50) .build(), false, 50, false }, // Number of message limitation exceed receiverQueue size and timeout limitation @@ -248,7 +248,7 @@ public Object[][] batchReceivePolicyProvider() { // Number of message limitation exceed receiverQueue size { BatchReceivePolicy.builder() - .maxNumMessages(70) + .maxNumMessages(50) .build(), true, 50, true }, // Number of message limitation exceed receiverQueue size and timeout limitation @@ -283,7 +283,7 @@ public Object[][] batchReceivePolicyProvider() { // Number of message limitation exceed receiverQueue size { BatchReceivePolicy.builder() - .maxNumMessages(70) + .maxNumMessages(50) .build(), false, 50, true }, // Number of message limitation exceed receiverQueue size and timeout limitation diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerRedeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerRedeliveryTest.java index 90114add25084..fcf1a638d5884 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerRedeliveryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerRedeliveryTest.java @@ -424,4 +424,28 @@ public void testAckNotSent(int numAcked, int batchSize, CommandAck.AckType ackTy assertTrue(values.isEmpty()); } } + + @Test + public void testRedeliverMessagesWithoutValue() throws Exception { + String topic = "persistent://my-property/my-ns/testRedeliverMessagesWithoutValue"; + @Cleanup Consumer consumer = pulsarClient.newConsumer(Schema.INT32) + .topic(topic) + .subscriptionName("sub") + .enableRetry(true) + .subscribe(); + @Cleanup Producer producer = pulsarClient.newProducer(Schema.INT32) + .topic(topic) + .enableBatching(true) + .create(); + for (int i = 0; i < 10; i++) { + producer.newMessage().key("messages without value").send(); + } + + Message message = consumer.receive(); + consumer.reconsumeLater(message, 2, TimeUnit.SECONDS); + for (int i = 0; i < 9; i++) { + assertNotNull(consumer.receive(5, TimeUnit.SECONDS)); + } + assertTrue(consumer.receive(5, TimeUnit.SECONDS).getTopicName().contains("sub-RETRY")); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicDefaultMultiPartitionsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicDefaultMultiPartitionsTest.java new file mode 100644 index 0000000000000..b8bccb793724b --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicDefaultMultiPartitionsTest.java @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api; + +import static org.apache.pulsar.client.util.RetryMessageUtil.DLQ_GROUP_TOPIC_SUFFIX; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.common.naming.TopicDomain; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TopicType; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker-impl") +public class DeadLetterTopicDefaultMultiPartitionsTest extends ProducerConsumerBase { + + @BeforeClass(alwaysRun = true) + @Override + protected void setup() throws Exception { + this.conf.setMaxMessageSize(5 * 1024); + this.conf.setAllowAutoTopicCreation(true); + this.conf.setDefaultNumPartitions(2); + this.conf.setAllowAutoTopicCreationType(TopicType.PARTITIONED); + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + private void triggerDLQGenerate(String topic, String subscription) throws Exception { + String DLQ = getDLQName(topic, subscription); + String p0OfDLQ = TopicName.get(DLQ).getPartition(0).toString(); + Consumer consumer = pulsarClient.newConsumer().topic(topic).subscriptionName(subscription) + .ackTimeout(1000, TimeUnit.MILLISECONDS) + .subscriptionType(SubscriptionType.Shared) + .receiverQueueSize(10) + .negativeAckRedeliveryDelay(100, TimeUnit.MILLISECONDS) + .deadLetterPolicy(DeadLetterPolicy.builder().maxRedeliverCount(1).build()) + .subscribe(); + Producer producer = pulsarClient.newProducer().topic(topic).create(); + producer.newMessage().value(new byte[]{1}).send(); + + Message message1 = consumer.receive(); + consumer.negativeAcknowledge(message1); + Message message2 = consumer.receive(); + consumer.negativeAcknowledge(message2); + + Awaitility.await().atMost(Duration.ofSeconds(1500)).until(() -> { + Message message3 = consumer.receive(2, TimeUnit.SECONDS); + if (message3 != null) { + log.info("===> {}", message3.getRedeliveryCount()); + consumer.negativeAcknowledge(message3); + } + List topicList = pulsar.getPulsarResources().getTopicResources() + .listPersistentTopicsAsync(TopicName.get(topic).getNamespaceObject()).join(); + if (topicList.contains(DLQ) || topicList.contains(p0OfDLQ)) { + return true; + } + int partitions = admin.topics().getPartitionedTopicMetadata(topic).partitions; + for (int i = 0; i < partitions; i++) { + for (int j = -1; j < pulsar.getConfig().getDefaultNumPartitions(); j++) { + String p0OfDLQ2; + if (j == -1) { + p0OfDLQ2 = TopicName + .get(getDLQName(TopicName.get(topic).getPartition(i).toString(), subscription)) + .toString(); + } else { + p0OfDLQ2 = TopicName + .get(getDLQName(TopicName.get(topic).getPartition(i).toString(), subscription)) + .getPartition(j).toString(); + } + if (topicList.contains(p0OfDLQ2)) { + return true; + } + } + } + return false; + }); + producer.close(); + consumer.close(); + admin.topics().unload(topic); + } + + private static String getDLQName(String primaryTopic, String subscription) { + String domain = TopicName.get(primaryTopic).getDomain().toString(); + return domain + "://" + TopicName.get(primaryTopic) + .toString().substring(( domain + "://").length()) + + "-" + subscription + DLQ_GROUP_TOPIC_SUFFIX; + } + + @DataProvider(name = "topicCreationTypes") + public Object[][] topicCreationTypes() { + return new Object[][]{ + //{TopicType.NON_PARTITIONED}, + {TopicType.PARTITIONED} + }; + } + + @Test(dataProvider = "topicCreationTypes") + public void testGenerateNonPartitionedDLQ(TopicType topicType) throws Exception { + final String topic = BrokerTestUtil.newUniqueName( "persistent://public/default/tp"); + final String subscription = "s1"; + switch (topicType) { + case PARTITIONED: { + admin.topics().createPartitionedTopic(topic, 2); + break; + } + case NON_PARTITIONED: { + admin.topics().createNonPartitionedTopic(topic); + } + } + + triggerDLQGenerate(topic, subscription); + + // Verify: no partitioned DLQ. + List partitionedTopics = pulsar.getPulsarResources().getNamespaceResources() + .getPartitionedTopicResources() + .listPartitionedTopicsAsync(TopicName.get(topic).getNamespaceObject(), TopicDomain.persistent).join(); + for (String tp : partitionedTopics) { + assertFalse(tp.endsWith("-DLQ")); + } + // Verify: non-partitioned DLQ exists. + List partitions = pulsar.getPulsarResources().getTopicResources() + .listPersistentTopicsAsync(TopicName.get(topic).getNamespaceObject()).join(); + List DLQCreated = new ArrayList<>(); + for (String tp : partitions) { + if (tp.endsWith("-DLQ")) { + DLQCreated.add(tp); + } + assertFalse(tp.endsWith("-partition-0-DLQ")); + } + assertTrue(!DLQCreated.isEmpty()); + + // cleanup. + switch (topicType) { + case PARTITIONED: { + admin.topics().deletePartitionedTopic(topic); + break; + } + case NON_PARTITIONED: { + admin.topics().delete(topic, false); + } + } + for (String t : DLQCreated) { + try { + admin.topics().delete(TopicName.get(t).getPartitionedTopicName(), false); + } catch (Exception ex) {} + try { + admin.topics().deletePartitionedTopic(TopicName.get(t).getPartitionedTopicName(), false); + } catch (Exception ex) {} + } + } + + @Test + public void testManuallyCreatePartitionedDLQ() throws Exception { + final String topic = BrokerTestUtil.newUniqueName( "persistent://public/default/tp"); + final String subscription = "s1"; + String DLQ = getDLQName(topic, subscription); + String p0OfDLQ = TopicName.get(DLQ).getPartition(0).toString(); + String p1OfDLQ = TopicName.get(DLQ).getPartition(1).toString(); + admin.topics().createNonPartitionedTopic(topic); + admin.topics().createPartitionedTopic(DLQ, 2); + + Awaitility.await().untilAsserted(() -> { + // Verify: partitioned DLQ exists. + List partitionedTopics = pulsar.getPulsarResources().getNamespaceResources() + .getPartitionedTopicResources() + .listPartitionedTopicsAsync(TopicName.get(topic).getNamespaceObject(), TopicDomain.persistent).join(); + assertTrue(partitionedTopics.contains(DLQ)); + assertFalse(partitionedTopics.contains(p0OfDLQ)); + // Verify: DLQ partitions exists. + List partitions = pulsar.getPulsarResources().getTopicResources() + .listPersistentTopicsAsync(TopicName.get(topic).getNamespaceObject()).join(); + assertFalse(partitions.contains(DLQ)); + assertTrue(partitions.contains(p0OfDLQ)); + assertTrue(partitions.contains(p1OfDLQ)); + }); + + // cleanup. + admin.topics().delete(topic, false); + admin.topics().deletePartitionedTopic(DLQ, false); + } + + @Test + public void testManuallyCreatePartitionedDLQ2() throws Exception { + final String topic = BrokerTestUtil.newUniqueName( "persistent://public/default/tp"); + final String subscription = "s1"; + final String p0OfTopic = TopicName.get(topic).getPartition(0).toString(); + String DLQ = getDLQName(p0OfTopic, subscription); + String p0OfDLQ = TopicName.get(DLQ).getPartition(0).toString(); + admin.topics().createPartitionedTopic(topic, 10); + try { + admin.topics().createPartitionedTopic(DLQ, 2); + } catch (Exception ex) { + // Keep multiple versions compatible. + if (ex.getMessage().contains("Partitioned Topic Name should not contain '-partition-'")){ + return; + } else { + fail("Failed to create partitioned DLQ"); + } + } + + Awaitility.await().untilAsserted(() -> { + // Verify: partitioned DLQ exists. + List partitionedTopics = pulsar.getPulsarResources().getNamespaceResources() + .getPartitionedTopicResources() + .listPartitionedTopicsAsync(TopicName.get(topic).getNamespaceObject(), TopicDomain.persistent).join(); + assertTrue(partitionedTopics.contains(DLQ)); + assertFalse(partitionedTopics.contains(p0OfDLQ)); + // Verify: DLQ partitions exists. + List partitions = pulsar.getPulsarResources().getTopicResources() + .listPersistentTopicsAsync(TopicName.get(topic).getNamespaceObject()).join(); + assertFalse(partitions.contains(DLQ)); + }); + + // cleanup. + admin.topics().deletePartitionedTopic(topic, false); + admin.topics().deletePartitionedTopic(DLQ, false); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java index 5e3731fbf24fc..f5a74dcd1661b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java @@ -29,6 +29,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; @@ -46,9 +47,10 @@ import org.slf4j.LoggerFactory; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -@Test(groups = "flaky") +@Test(groups = "broker-impl") public class DeadLetterTopicTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(DeadLetterTopicTest.class); @@ -56,6 +58,7 @@ public class DeadLetterTopicTest extends ProducerConsumerBase { @BeforeMethod(alwaysRun = true) @Override protected void setup() throws Exception { + this.conf.setMaxMessageSize(5 * 1024); super.internalSetup(); super.producerBaseSetup(); } @@ -66,6 +69,15 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + private String createMessagePayload(int size) { + StringBuilder str = new StringBuilder(); + Random rand = new Random(); + for (int i = 0; i < size; i++) { + str.append(rand.nextInt(10)); + } + return str.toString(); + } + @Test public void testDeadLetterTopicWithMessageKey() throws Exception { final String topic = "persistent://my-property/my-ns/dead-letter-topic"; @@ -125,12 +137,71 @@ public void testDeadLetterTopicWithMessageKey() throws Exception { consumer.close(); } + @Test + public void testDeadLetterTopicWithBinaryMessageKey() throws Exception { + final String topic = "persistent://my-property/my-ns/dead-letter-topic"; - @Test(groups = "quarantine") - public void testDeadLetterTopic() throws Exception { + final int maxRedeliveryCount = 1; + + final int sendMessages = 100; + + Consumer consumer = pulsarClient.newConsumer(Schema.BYTES) + .topic(topic) + .subscriptionName("my-subscription") + .subscriptionType(SubscriptionType.Shared) + .ackTimeout(1, TimeUnit.SECONDS) + .deadLetterPolicy(DeadLetterPolicy.builder().maxRedeliverCount(maxRedeliveryCount).build()) + .receiverQueueSize(100) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + @Cleanup + PulsarClient newPulsarClient = newPulsarClient(lookupUrl.toString(), 0);// Creates new client connection + Consumer deadLetterConsumer = newPulsarClient.newConsumer(Schema.BYTES) + .topic("persistent://my-property/my-ns/dead-letter-topic-my-subscription-DLQ") + .subscriptionName("my-subscription") + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + Producer producer = pulsarClient.newProducer(Schema.BYTES) + .topic(topic) + .create(); + + byte[] key = new byte[]{1, 2, 3, 4}; + for (int i = 0; i < sendMessages; i++) { + producer.newMessage() + .keyBytes(key) + .value(String.format("Hello Pulsar [%d]", i).getBytes()) + .send(); + } + + producer.close(); + + int totalReceived = 0; + do { + Message message = consumer.receive(); + log.info("consumer received message : {} {}", message.getMessageId(), new String(message.getData())); + totalReceived++; + } while (totalReceived < sendMessages * (maxRedeliveryCount + 1)); + + int totalInDeadLetter = 0; + do { + Message message = deadLetterConsumer.receive(); + assertEquals(message.getKeyBytes(), key); + log.info("dead letter consumer received message : {} {}", message.getMessageId(), new String(message.getData())); + deadLetterConsumer.acknowledge(message); + totalInDeadLetter++; + } while (totalInDeadLetter < sendMessages); + + deadLetterConsumer.close(); + consumer.close(); + } + + @Test + public void testDeadLetterTopicMessagesWithOrderingKey() throws Exception { final String topic = "persistent://my-property/my-ns/dead-letter-topic"; - final int maxRedeliveryCount = 2; + final int maxRedeliveryCount = 1; final int sendMessages = 100; @@ -156,8 +227,12 @@ public void testDeadLetterTopic() throws Exception { .topic(topic) .create(); + byte[] key = new byte[]{1, 2, 3, 4}; for (int i = 0; i < sendMessages; i++) { - producer.send(String.format("Hello Pulsar [%d]", i).getBytes()); + producer.newMessage() + .orderingKey(key) + .value(String.format("Hello Pulsar [%d]", i).getBytes()) + .send(); } producer.close(); @@ -172,6 +247,7 @@ public void testDeadLetterTopic() throws Exception { int totalInDeadLetter = 0; do { Message message = deadLetterConsumer.receive(); + assertEquals(message.getOrderingKey(), key); log.info("dead letter consumer received message : {} {}", message.getMessageId(), new String(message.getData())); deadLetterConsumer.acknowledge(message); totalInDeadLetter++; @@ -179,6 +255,143 @@ public void testDeadLetterTopic() throws Exception { deadLetterConsumer.close(); consumer.close(); + } + + public void testDeadLetterTopicWithProducerName() throws Exception { + final String topic = "persistent://my-property/my-ns/dead-letter-topic"; + final String subscription = "my-subscription"; + final String consumerName = "my-consumer"; + String deadLetterProducerName = String.format("%s-%s-%s-DLQ", topic, subscription, consumerName); + + final int maxRedeliveryCount = 1; + + final int sendMessages = 100; + + Consumer consumer = pulsarClient.newConsumer(Schema.BYTES) + .topic(topic) + .subscriptionName(subscription) + .consumerName(consumerName) + .subscriptionType(SubscriptionType.Shared) + .ackTimeout(1, TimeUnit.SECONDS) + .deadLetterPolicy(DeadLetterPolicy.builder().maxRedeliverCount(maxRedeliveryCount).build()) + .receiverQueueSize(100) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + @Cleanup + PulsarClient newPulsarClient = newPulsarClient(lookupUrl.toString(), 0);// Creates new client connection + Consumer deadLetterConsumer = newPulsarClient.newConsumer(Schema.BYTES) + .topic("persistent://my-property/my-ns/dead-letter-topic-my-subscription-DLQ") + .subscriptionName("my-subscription") + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + Producer producer = pulsarClient.newProducer(Schema.BYTES) + .topic(topic) + .create(); + + for (int i = 0; i < sendMessages; i++) { + producer.newMessage() + .value(String.format("Hello Pulsar [%d]", i).getBytes()) + .send(); + } + + producer.close(); + + int totalReceived = 0; + do { + Message message = consumer.receive(); + log.info("consumer received message : {} {}", message.getMessageId(), new String(message.getData())); + totalReceived++; + } while (totalReceived < sendMessages * (maxRedeliveryCount + 1)); + + int totalInDeadLetter = 0; + do { + Message message = deadLetterConsumer.receive(); + assertEquals(message.getProducerName(), deadLetterProducerName); + log.info("dead letter consumer received message : {} {}", message.getMessageId(), new String(message.getData())); + deadLetterConsumer.acknowledge(message); + totalInDeadLetter++; + } while (totalInDeadLetter < sendMessages); + + deadLetterConsumer.close(); + consumer.close(); + } + + @DataProvider(name = "produceLargeMessages") + public Object[][] produceLargeMessages() { + return new Object[][] { { false }, { true } }; + } + + @Test(dataProvider = "produceLargeMessages") + public void testDeadLetterTopic(boolean produceLargeMessages) throws Exception { + final String topic = "persistent://my-property/my-ns/dead-letter-topic"; + + final int maxRedeliveryCount = 2; + + final int sendMessages = 100; + + Consumer consumer = pulsarClient.newConsumer(Schema.BYTES) + .topic(topic) + .subscriptionName("my-subscription") + .subscriptionType(SubscriptionType.Shared) + .ackTimeout(1, TimeUnit.SECONDS) + .deadLetterPolicy(DeadLetterPolicy.builder().maxRedeliverCount(maxRedeliveryCount).build()) + .receiverQueueSize(100) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + @Cleanup + PulsarClient newPulsarClient = newPulsarClient(lookupUrl.toString(), 0);// Creates new client connection + Consumer deadLetterConsumer = newPulsarClient.newConsumer(Schema.BYTES) + .topic("persistent://my-property/my-ns/dead-letter-topic-my-subscription-DLQ") + .subscriptionName("my-subscription") + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + Producer producer = pulsarClient.newProducer(Schema.BYTES) + .topic(topic) + .enableChunking(produceLargeMessages) + .enableBatching(!produceLargeMessages) + .create(); + + Map messageContent = new HashMap<>(); + + for (int i = 0; i < sendMessages; i++) { + String data; + if (!produceLargeMessages) { + data = String.format("Hello Pulsar [%d]", i); + } else { + data = createMessagePayload(1024 * 10); + } + producer.newMessage().key(String.valueOf(i)).value(data.getBytes()).send(); + messageContent.put(i, data); + } + + producer.close(); + + int totalReceived = 0; + do { + Message message = consumer.receive(5, TimeUnit.SECONDS); + assertNotNull(message, "The consumer should be able to receive messages."); + log.info("consumer received message : {}", message.getMessageId()); + totalReceived++; + } while (totalReceived < sendMessages * (maxRedeliveryCount + 1)); + + int totalInDeadLetter = 0; + do { + Message message = deadLetterConsumer.receive(5, TimeUnit.SECONDS); + assertNotNull(message, "the deadLetterConsumer should receive messages."); + assertEquals(new String(message.getData()), messageContent.get(Integer.parseInt(message.getKey()))); + messageContent.remove(Integer.parseInt(message.getKey())); + log.info("dead letter consumer received message : {}", message.getMessageId()); + deadLetterConsumer.acknowledge(message); + totalInDeadLetter++; + } while (totalInDeadLetter < sendMessages); + assertTrue(messageContent.isEmpty()); + + deadLetterConsumer.close(); + consumer.close(); Consumer checkConsumer = this.pulsarClient.newConsumer(Schema.BYTES) .topic(topic) @@ -202,10 +415,11 @@ public void testDeadLetterTopicHasOriginalInfo() throws Exception { final int maxRedeliveryCount = 1; final int sendMessages = 10; + final String subscriptionName = "my-subscription"; Consumer consumer = pulsarClient.newConsumer(Schema.BYTES) .topic(topic) - .subscriptionName("my-subscription") + .subscriptionName(subscriptionName) .subscriptionType(SubscriptionType.Shared) .ackTimeout(1, TimeUnit.SECONDS) .deadLetterPolicy(DeadLetterPolicy.builder().maxRedeliverCount(maxRedeliveryCount).build()) @@ -241,6 +455,7 @@ public void testDeadLetterTopicHasOriginalInfo() throws Exception { Message message = deadLetterConsumer.receive(); //Original info should exists assertEquals(message.getProperties().get(RetryMessageUtil.SYSTEM_PROPERTY_REAL_TOPIC), topic); + assertEquals(message.getProperties().get(RetryMessageUtil.SYSTEM_PROPERTY_REAL_SUBSCRIPTION), subscriptionName); assertTrue(messageIds.contains(message.getProperties().get(RetryMessageUtil.SYSTEM_PROPERTY_ORIGIN_MESSAGE_ID))); deadLetterConsumer.acknowledge(message); totalInDeadLetter++; @@ -836,6 +1051,9 @@ public void testDeadLetterTopicWithInitialSubscriptionAndMultiConsumers() throws assertTrue(admin.topics().getSubscriptions(deadLetterTopic).contains(dlqInitialSub)); }); + // We should assert that all consumers are able to produce messages to DLQ + assertEquals(admin.topics().getStats(deadLetterTopic).getPublishers().size(), 2); + Consumer deadLetterConsumer = newPulsarClient.newConsumer(Schema.BYTES) .topic(deadLetterTopic) .subscriptionName(dlqInitialSub) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java index fc103a46027c0..88286af98ae5f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java @@ -29,7 +29,6 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Queues; import com.google.common.collect.Sets; -import java.lang.reflect.Field; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -49,14 +48,11 @@ import java.util.stream.Collectors; import lombok.Cleanup; import org.apache.pulsar.broker.namespace.NamespaceService; -import org.apache.pulsar.broker.service.BrokerService; -import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.impl.ConsumerImpl; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.common.policies.data.SubscriptionStats; import org.apache.pulsar.common.policies.data.TopicStats; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet; import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -692,8 +688,8 @@ public void testBlockBrokerDispatching() { try { final int waitMills = 500; final int maxUnAckPerBroker = 200; - final double unAckMsgPercentagePerDispatcher = 10; - int maxUnAckPerDispatcher = (int) ((maxUnAckPerBroker * unAckMsgPercentagePerDispatcher) / 100); // 200 * + final double unAckMsgPercentagePerDispatcher = 0.1; + int maxUnAckPerDispatcher = (int) (maxUnAckPerBroker * unAckMsgPercentagePerDispatcher); // 200 * // 10% = 20 // messages pulsar.getConfiguration().setMaxUnackedMessagesPerBroker(maxUnAckPerBroker); @@ -703,11 +699,7 @@ public void testBlockBrokerDispatching() { stopBroker(); startBroker(); - Field field = BrokerService.class.getDeclaredField("blockedDispatchers"); - field.setAccessible(true); - @SuppressWarnings("unchecked") - ConcurrentOpenHashSet blockedDispatchers = - (ConcurrentOpenHashSet) field.get(pulsar.getBrokerService()); + final var blockedDispatchers = pulsar.getBrokerService().getBlockedDispatchers(); final int receiverQueueSize = 10; final int totalProducedMsgs = maxUnAckPerBroker * 3; @@ -783,7 +775,7 @@ public void testBlockBrokerDispatching() { consumer2Sub1.close(); // (1.c) verify that dispatcher is part of blocked dispatcher assertEquals(blockedDispatchers.size(), 1); - String dispatcherName = blockedDispatchers.values().get(0).getName(); + String dispatcherName = blockedDispatchers.stream().findFirst().orElseThrow().getName(); String subName = dispatcherName.substring(dispatcherName.lastIndexOf("/") + 2, dispatcherName.length()); assertEquals(subName, subscriberName1); timestamps.add(System.currentTimeMillis()); @@ -907,8 +899,8 @@ public void testBrokerDispatchBlockAndSubAckBackRequiredMsgs() { .getMaxUnackedMessagesPerSubscriptionOnBrokerBlocked(); try { final int maxUnAckPerBroker = 200; - final double unAckMsgPercentagePerDispatcher = 10; - int maxUnAckPerDispatcher = (int) ((maxUnAckPerBroker * unAckMsgPercentagePerDispatcher) / 100); // 200 * + final double unAckMsgPercentagePerDispatcher = 0.1; + int maxUnAckPerDispatcher = (int) (maxUnAckPerBroker * unAckMsgPercentagePerDispatcher); // 200 * // 10% = 20 // messages pulsar.getConfiguration().setMaxUnackedMessagesPerBroker(maxUnAckPerBroker); @@ -918,10 +910,7 @@ public void testBrokerDispatchBlockAndSubAckBackRequiredMsgs() { stopBroker(); startBroker(); - Field field = BrokerService.class.getDeclaredField("blockedDispatchers"); - field.setAccessible(true); - ConcurrentOpenHashSet blockedDispatchers = - (ConcurrentOpenHashSet) field.get(pulsar.getBrokerService()); + final var blockedDispatchers = pulsar.getBrokerService().getBlockedDispatchers(); final int receiverQueueSize = 10; final int totalProducedMsgs = maxUnAckPerBroker * 3; @@ -992,7 +981,7 @@ public void testBrokerDispatchBlockAndSubAckBackRequiredMsgs() { consumer2Sub1.close(); // (1.c) verify that dispatcher is part of blocked dispatcher assertEquals(blockedDispatchers.size(), 1); - String dispatcherName = blockedDispatchers.values().get(0).getName(); + String dispatcherName = blockedDispatchers.stream().findFirst().orElseThrow().getName(); String subName = dispatcherName.substring(dispatcherName.lastIndexOf("/") + 2, dispatcherName.length()); assertEquals(subName, subscriberName1); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/InjectedClientCnxClientBuilder.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/InjectedClientCnxClientBuilder.java new file mode 100644 index 0000000000000..288bdba9b3846 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/InjectedClientCnxClientBuilder.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api; + +import io.netty.channel.EventLoopGroup; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.client.impl.ClientBuilderImpl; +import org.apache.pulsar.client.impl.ClientCnx; +import org.apache.pulsar.client.impl.ConnectionPool; +import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; +import org.apache.pulsar.client.util.ExecutorProvider; +import org.apache.pulsar.common.util.netty.EventLoopUtil; + +public class InjectedClientCnxClientBuilder { + + public static PulsarClientImpl create(final ClientBuilderImpl clientBuilder, + final ClientCnxFactory clientCnxFactory) throws Exception { + ClientConfigurationData conf = clientBuilder.getClientConfigurationData(); + ThreadFactory threadFactory = new ExecutorProvider + .ExtendedThreadFactory("pulsar-client-io", Thread.currentThread().isDaemon()); + EventLoopGroup eventLoopGroup = + EventLoopUtil.newEventLoopGroup(conf.getNumIoThreads(), conf.isEnableBusyWait(), threadFactory); + + // Inject into ClientCnx. + ConnectionPool pool = new ConnectionPool(InstrumentProvider.NOOP, conf, eventLoopGroup, + () -> clientCnxFactory.generate(conf, eventLoopGroup), null); + + return new InjectedClientCnxPulsarClientImpl(conf, eventLoopGroup, pool); + } + + public interface ClientCnxFactory { + + ClientCnx generate(ClientConfigurationData conf, EventLoopGroup eventLoopGroup); + } + + @Slf4j + private static class InjectedClientCnxPulsarClientImpl extends PulsarClientImpl { + + public InjectedClientCnxPulsarClientImpl(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, + ConnectionPool pool) + throws PulsarClientException { + super(conf, eventLoopGroup, pool); + } + + @Override + public CompletableFuture closeAsync() { + return super.closeAsync().handle((v, ex) -> { + try { + getCnxPool().close(); + } catch (Exception e) { + log.warn("Failed to close cnx pool", e); + } + try { + eventLoopGroup.shutdownGracefully().get(10, TimeUnit.SECONDS); + } catch (Exception e) { + log.warn("Failed to shutdown event loop group", e); + } + return null; + }); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/InterceptorsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/InterceptorsTest.java index f23d82b32cd43..8115f34121d3c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/InterceptorsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/InterceptorsTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.api; +import com.google.common.collect.Sets; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -29,11 +30,10 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; - -import com.google.common.collect.Sets; import lombok.Cleanup; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomUtils; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.client.impl.TopicMessageImpl; import org.apache.pulsar.common.api.proto.KeyValue; @@ -79,6 +79,12 @@ public Object[][] getTopicPartition() { return new Object[][] {{ 0 }, { 3 }}; } + @DataProvider(name = "topics") + public Object[][] getTopics() { + return new Object[][] {{ List.of("persistent://my-property/my-ns/my-topic") }, + { List.of("persistent://my-property/my-ns/my-topic", "persistent://my-property/my-ns/my-topic1") }}; + } + @Test public void testProducerInterceptor() throws Exception { Map> ackCallback = new HashMap<>(); @@ -403,9 +409,9 @@ public void close() { @Override public Message beforeConsume(Consumer consumer, Message message) { - MessageImpl msg = (MessageImpl) message; + MessageImpl msg = ((MessageImpl) ((TopicMessageImpl) message).getMessage()); msg.getMessageBuilder().addProperty().setKey("beforeConsumer").setValue("1"); - return msg; + return message; } @Override @@ -449,13 +455,19 @@ public void onAckTimeoutSend(Consumer consumer, Set messageId int keyCount = 0; for (int i = 0; i < 2; i++) { - Message received = consumer.receive(); + Message received; + if (i % 2 == 0) { + received = consumer.receive(); + } else { + received = consumer.receiveAsync().join(); + } MessageImpl msg = (MessageImpl) ((TopicMessageImpl) received).getMessage(); for (KeyValue keyValue : msg.getMessageBuilder().getPropertiesList()) { if ("beforeConsumer".equals(keyValue.getKey())) { keyCount++; } } + Assert.assertEquals(keyCount, i + 1); consumer.acknowledge(received); } Assert.assertEquals(2, keyCount); @@ -475,9 +487,9 @@ public void close() { @Override public Message beforeConsume(Consumer consumer, Message message) { - MessageImpl msg = (MessageImpl) message; + MessageImpl msg = ((MessageImpl) ((TopicMessageImpl) message).getMessage()); msg.getMessageBuilder().addProperty().setKey("beforeConsumer").setValue("1"); - return msg; + return message; } @Override @@ -612,8 +624,8 @@ public void onAckTimeoutSend(Consumer consumer, Set messageId consumer.close(); } - @Test - public void testConsumerInterceptorForNegativeAcksSend() throws PulsarClientException, InterruptedException { + @Test(dataProvider = "topics") + public void testConsumerInterceptorForNegativeAcksSend(List topics) throws PulsarClientException, InterruptedException { final int totalNumOfMessages = 100; CountDownLatch latch = new CountDownLatch(totalNumOfMessages / 2); @@ -640,6 +652,7 @@ public void onAcknowledgeCumulative(Consumer consumer, MessageId message @Override public void onNegativeAcksSend(Consumer consumer, Set messageIds) { + Assert.assertTrue(latch.getCount() > 0); messageIds.forEach(messageId -> latch.countDown()); } @@ -650,7 +663,7 @@ public void onAckTimeoutSend(Consumer consumer, Set messageId }; Consumer consumer = pulsarClient.newConsumer(Schema.STRING) - .topic("persistent://my-property/my-ns/my-topic") + .topics(topics) .subscriptionType(SubscriptionType.Failover) .intercept(interceptor) .negativeAckRedeliveryDelay(100, TimeUnit.MILLISECONDS) @@ -658,7 +671,7 @@ public void onAckTimeoutSend(Consumer consumer, Set messageId .subscribe(); Producer producer = pulsarClient.newProducer(Schema.STRING) - .topic("persistent://my-property/my-ns/my-topic") + .topic(topics.get(0)) .create(); for (int i = 0; i < totalNumOfMessages; i++) { @@ -682,8 +695,9 @@ public void onAckTimeoutSend(Consumer consumer, Set messageId consumer.close(); } - @Test - public void testConsumerInterceptorForAckTimeoutSend() throws PulsarClientException, InterruptedException { + @Test(dataProvider = "topics") + public void testConsumerInterceptorForAckTimeoutSend(List topics) throws PulsarClientException, + InterruptedException { final int totalNumOfMessages = 100; CountDownLatch latch = new CountDownLatch(totalNumOfMessages / 2); @@ -714,16 +728,17 @@ public void onNegativeAcksSend(Consumer consumer, Set message @Override public void onAckTimeoutSend(Consumer consumer, Set messageIds) { + Assert.assertTrue(latch.getCount() > 0); messageIds.forEach(messageId -> latch.countDown()); } }; Producer producer = pulsarClient.newProducer(Schema.STRING) - .topic("persistent://my-property/my-ns/my-topic") + .topic(topics.get(0)) .create(); Consumer consumer = pulsarClient.newConsumer(Schema.STRING) - .topic("persistent://my-property/my-ns/my-topic") + .topics(topics) .subscriptionName("foo") .intercept(interceptor) .ackTimeout(2, TimeUnit.SECONDS) @@ -856,6 +871,101 @@ public void onPartitionsChange(String topicName, int partitions) { Assert.assertNull(reader.readNext(3, TimeUnit.SECONDS)); } + @Test(dataProvider = "topicPartition") + public void testConsumerInterceptorForOnArrive(int topicPartition) throws PulsarClientException, + InterruptedException, PulsarAdminException { + String topicName = "persistent://my-property/my-ns/on-arrive"; + if (topicPartition > 0) { + admin.topics().createPartitionedTopic(topicName, topicPartition); + } + + final int receiveQueueSize = 100; + final int totalNumOfMessages = receiveQueueSize * 2; + + // The onArrival method is called for half of the receiveQueueSize messages before beforeConsume is called for all messages. + CountDownLatch latch = new CountDownLatch(receiveQueueSize / 2); + final AtomicInteger onArrivalCount = new AtomicInteger(0); + ConsumerInterceptor interceptor = new ConsumerInterceptor() { + @Override + public void close() {} + + @Override + public Message onArrival(Consumer consumer, Message message) { + MessageImpl msg = (MessageImpl) message; + msg.getMessageBuilder().addProperty().setKey("onArrival").setValue("1"); + latch.countDown(); + onArrivalCount.incrementAndGet(); + return msg; + } + + @Override + public Message beforeConsume(Consumer consumer, Message message) { + return message; + } + + @Override + public void onAcknowledge(Consumer consumer, MessageId messageId, Throwable cause) { + + } + + @Override + public void onAcknowledgeCumulative(Consumer consumer, MessageId messageId, Throwable cause) { + + } + + @Override + public void onNegativeAcksSend(Consumer consumer, Set messageIds) { + } + + @Override + public void onAckTimeoutSend(Consumer consumer, Set messageIds) { + + } + }; + + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .create(); + + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName("test-arrive") + .intercept(interceptor) + .receiverQueueSize(receiveQueueSize) + .subscribe(); + + for (int i = 0; i < totalNumOfMessages; i++) { + producer.send("Mock message"); + } + + // Not call receive message, just wait for onArrival interceptor. + latch.await(); + Assert.assertEquals(latch.getCount(), 0); + + for (int i = 0; i < totalNumOfMessages; i++) { + Message message = consumer.receive(); + MessageImpl msgImpl; + if (message instanceof MessageImpl) { + msgImpl = (MessageImpl) message; + } else if (message instanceof TopicMessageImpl) { + msgImpl = (MessageImpl) ((TopicMessageImpl) message).getMessage(); + } else { + throw new ClassCastException("Message type is not expected"); + } + boolean haveKey = false; + for (KeyValue keyValue : msgImpl.getMessageBuilder().getPropertiesList()) { + if ("onArrival".equals(keyValue.getKey())) { + haveKey = true; + } + } + Assert.assertTrue(haveKey); + } + Assert.assertEquals(totalNumOfMessages, onArrivalCount.get()); + + producer.close(); + consumer.close(); + } + private void produceAndConsume(int msgCount, Producer producer, Reader reader) throws PulsarClientException { for (int i = 0; i < msgCount; i++) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java index 18fb141be3178..c08c37b413f4f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java @@ -18,6 +18,12 @@ */ package org.apache.pulsar.client.api; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.pulsar.broker.BrokerTestUtil.receiveMessages; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; @@ -26,18 +32,23 @@ import static org.testng.Assert.fail; import com.google.common.collect.Lists; import java.io.IOException; +import java.lang.reflect.Field; import java.nio.file.Files; import java.nio.file.Paths; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Random; import java.util.Set; +import java.util.TreeSet; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListSet; @@ -48,19 +59,35 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; +import java.util.stream.Collectors; import lombok.Cleanup; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.service.StickyKeyConsumerSelector; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.nonpersistent.NonPersistentStickyKeyDispatcherMultipleConsumers; +import org.apache.pulsar.broker.service.persistent.MessageRedeliveryController; +import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.apache.pulsar.broker.service.persistent.PersistentStickyKeyDispatcherMultipleConsumers; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.impl.ConsumerImpl; +import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.common.api.proto.KeySharedMode; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.util.Murmur3_32Hash; +import org.apache.pulsar.common.util.collections.ConcurrentOpenLongPairRangeSet; +import org.apache.pulsar.common.util.collections.LongPairRangeSet; import org.awaitility.Awaitility; +import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -70,7 +97,7 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -@Test(groups = "flaky") +@Test(groups = "broker-impl") public class KeySharedSubscriptionTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(KeySharedSubscriptionTest.class); @@ -134,6 +161,12 @@ public void resetDefaultNamespace() throws Exception { admin.topics().delete(topicName, false); } } + // reset read ahead limits to defaults + ServiceConfiguration defaultConf = new ServiceConfiguration(); + conf.setKeySharedLookAheadMsgInReplayThresholdPerSubscription( + defaultConf.getKeySharedLookAheadMsgInReplayThresholdPerSubscription()); + conf.setKeySharedLookAheadMsgInReplayThresholdPerConsumer( + defaultConf.getKeySharedLookAheadMsgInReplayThresholdPerConsumer()); } private static final Random random = new Random(System.nanoTime()); @@ -306,11 +339,11 @@ public void testConsumerCrashSendAndReceiveWithHashRangeAutoSplitStickyKeyConsum } @Test(dataProvider = "data") - public void testNonKeySendAndReceiveWithHashRangeAutoSplitStickyKeyConsumerSelector( + public void testNoKeySendAndReceiveWithHashRangeAutoSplitStickyKeyConsumerSelector( String topicType, boolean enableBatch ) throws PulsarClientException { - String topic = topicType + "://public/default/key_shared_none_key-" + UUID.randomUUID(); + String topic = topicType + "://public/default/key_shared_no_key-" + UUID.randomUUID(); @Cleanup Consumer consumer1 = createConsumer(topic); @@ -330,13 +363,13 @@ public void testNonKeySendAndReceiveWithHashRangeAutoSplitStickyKeyConsumerSelec .send(); } - receive(Lists.newArrayList(consumer1, consumer2, consumer3)); + receiveAndCheckDistribution(Lists.newArrayList(consumer1, consumer2, consumer3), 100); } @Test(dataProvider = "batch") - public void testNonKeySendAndReceiveWithHashRangeExclusiveStickyKeyConsumerSelector(boolean enableBatch) + public void testNoKeySendAndReceiveWithHashRangeExclusiveStickyKeyConsumerSelector(boolean enableBatch) throws PulsarClientException { - String topic = "persistent://public/default/key_shared_none_key_exclusive-" + UUID.randomUUID(); + String topic = "persistent://public/default/key_shared_no_key_exclusive-" + UUID.randomUUID(); @Cleanup Consumer consumer1 = createConsumer(topic, KeySharedPolicy.stickyHashRange() @@ -353,21 +386,32 @@ public void testNonKeySendAndReceiveWithHashRangeExclusiveStickyKeyConsumerSelec @Cleanup Producer producer = createProducer(topic, enableBatch); + int consumer1ExpectMessages = 0; + int consumer2ExpectMessages = 0; + int consumer3ExpectMessages = 0; + for (int i = 0; i < 100; i++) { producer.newMessage() .value(i) .send(); + + String fallbackKey = producer.getProducerName() + "-" + producer.getLastSequenceId(); + int slot = Murmur3_32Hash.getInstance().makeHash(fallbackKey.getBytes()) + % KeySharedPolicy.DEFAULT_HASH_RANGE_SIZE; + if (slot <= 20000) { + consumer1ExpectMessages++; + } else if (slot <= 40000) { + consumer2ExpectMessages++; + } else { + consumer3ExpectMessages++; + } } - int slot = Murmur3_32Hash.getInstance().makeHash("NONE_KEY".getBytes()) - % KeySharedPolicy.DEFAULT_HASH_RANGE_SIZE; + List, Integer>> checkList = new ArrayList<>(); - if (slot <= 20000) { - checkList.add(new KeyValue<>(consumer1, 100)); - } else if (slot <= 40000) { - checkList.add(new KeyValue<>(consumer2, 100)); - } else { - checkList.add(new KeyValue<>(consumer3, 100)); - } + checkList.add(new KeyValue<>(consumer1, consumer1ExpectMessages)); + checkList.add(new KeyValue<>(consumer2, consumer2ExpectMessages)); + checkList.add(new KeyValue<>(consumer3, consumer3ExpectMessages)); + receiveAndCheck(checkList); } @@ -609,8 +653,11 @@ public void testOrderingWhenAddingConsumers() throws Exception { } @Test - public void testReadAheadWhenAddingConsumers() throws Exception { - String topic = "testReadAheadWhenAddingConsumers-" + UUID.randomUUID(); + public void testReadAheadWithConfiguredLookAheadLimit() throws Exception { + String topic = "testReadAheadWithConfiguredLookAheadLimit-" + UUID.randomUUID(); + + // Set the look ahead limit to 50 for subscriptions + conf.setKeySharedLookAheadMsgInReplayThresholdPerSubscription(50); @Cleanup Producer producer = createProducer(topic, false); @@ -657,8 +704,9 @@ public void testReadAheadWhenAddingConsumers() throws Exception { PersistentSubscription sub = (PersistentSubscription) t.getSubscription("key_shared"); // We need to ensure that dispatcher does not keep to look ahead in the topic, - PositionImpl readPosition = (PositionImpl) sub.getCursor().getReadPosition(); - assertTrue(readPosition.getEntryId() < 1000); + Position readPosition = sub.getCursor().getReadPosition(); + long entryId = readPosition.getEntryId(); + assertTrue(entryId < 100); } @Test @@ -1088,13 +1136,21 @@ public void testAllowOutOfOrderDeliveryChangedAfterAllConsumerDisconnected() thr final String topicName = "persistent://public/default/change-allow-ooo-delivery-" + UUID.randomUUID(); final String subName = "my-sub"; - Consumer consumer = pulsarClient.newConsumer() + final Consumer consumer1 = pulsarClient.newConsumer() .topic(topicName) .subscriptionName(subName) .subscriptionType(SubscriptionType.Key_Shared) .keySharedPolicy(KeySharedPolicy.autoSplitHashRange().setAllowOutOfOrderDelivery(true)) .subscribe(); + @Cleanup + final Producer producer = pulsarClient.newProducer() + .topic(topicName) + .enableBatching(false) + .create(); + producer.send("message".getBytes()); + Awaitility.await().untilAsserted(() -> assertNotNull(consumer1.receive(100, TimeUnit.MILLISECONDS))); + CompletableFuture> future = pulsar.getBrokerService().getTopicIfExists(topicName); assertTrue(future.isDone()); assertTrue(future.get().isPresent()); @@ -1102,14 +1158,18 @@ public void testAllowOutOfOrderDeliveryChangedAfterAllConsumerDisconnected() thr PersistentStickyKeyDispatcherMultipleConsumers dispatcher = (PersistentStickyKeyDispatcherMultipleConsumers) topic.getSubscription(subName).getDispatcher(); assertTrue(dispatcher.isAllowOutOfOrderDelivery()); - consumer.close(); + assertNull(dispatcher.getLastSentPositionField()); + assertNull(dispatcher.getIndividuallySentPositionsField()); + consumer1.close(); - consumer = pulsarClient.newConsumer() + final Consumer consumer2 = pulsarClient.newConsumer() .topic(topicName) .subscriptionName(subName) .subscriptionType(SubscriptionType.Key_Shared) .keySharedPolicy(KeySharedPolicy.autoSplitHashRange().setAllowOutOfOrderDelivery(false)) .subscribe(); + producer.send("message".getBytes()); + Awaitility.await().untilAsserted(() -> assertNotNull(consumer2.receive(100, TimeUnit.MILLISECONDS))); future = pulsar.getBrokerService().getTopicIfExists(topicName); assertTrue(future.isDone()); @@ -1117,7 +1177,9 @@ public void testAllowOutOfOrderDeliveryChangedAfterAllConsumerDisconnected() thr topic = future.get().get(); dispatcher = (PersistentStickyKeyDispatcherMultipleConsumers) topic.getSubscription(subName).getDispatcher(); assertFalse(dispatcher.isAllowOutOfOrderDelivery()); - consumer.close(); + assertNotNull(dispatcher.getLastSentPositionField()); + assertNotNull(dispatcher.getIndividuallySentPositionsField()); + consumer2.close(); } @Test(timeOut = 30_000) @@ -1191,6 +1253,370 @@ public void testCheckConsumersWithSameName() throws Exception { l.await(); } + @DataProvider(name = "preSend") + private Object[][] preSendProvider() { + return new Object[][] { { false }, { true } }; + } + + @Test(timeOut = 30_000, dataProvider = "preSend") + public void testCheckBetweenSkippingAndRecentlyJoinedConsumers(boolean preSend) throws Exception { + conf.setSubscriptionKeySharedUseConsistentHashing(true); + + final String topicName = "persistent://public/default/recently-joined-consumers-" + UUID.randomUUID(); + final String subName = "my-sub"; + + @Cleanup + final Producer p = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .create(); + if (preSend) { + // verify that the test succeeds even if the topic has a message + p.send("msg"); + } + + final Supplier> cb = () -> pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionInitialPosition(SubscriptionInitialPosition.Latest) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Key_Shared) + .keySharedPolicy(KeySharedPolicy.autoSplitHashRange() + .setAllowOutOfOrderDelivery(false)); + + // create 2 consumers + final String c1ConsumerName = "c1"; + @Cleanup + final Consumer c1 = cb.get().consumerName(c1ConsumerName).receiverQueueSize(1).subscribe(); + @Cleanup + final Consumer c2 = cb.get().consumerName("c2").receiverQueueSize(1000).subscribe(); + + final PersistentStickyKeyDispatcherMultipleConsumers dispatcher = + (PersistentStickyKeyDispatcherMultipleConsumers) pulsar.getBrokerService().getTopicIfExists(topicName).get().get().getSubscription(subName).getDispatcher(); + final Field recentlyJoinedConsumersField = PersistentStickyKeyDispatcherMultipleConsumers.class.getDeclaredField("recentlyJoinedConsumers"); + recentlyJoinedConsumersField.setAccessible(true); + final LinkedHashMap recentlyJoinedConsumers = (LinkedHashMap) recentlyJoinedConsumersField.get(dispatcher); + final String keyA = "key-a"; + final int hashA = Murmur3_32Hash.getInstance().makeHash(keyA.getBytes()); + final Map hashConsumerMap = new HashMap<>(); + hashConsumerMap.put(hashA, c1.getConsumerName()); + + // enforce the selector will return c1 if keyA + final Field selectorField = PersistentStickyKeyDispatcherMultipleConsumers.class.getDeclaredField("selector"); + selectorField.setAccessible(true); + final StickyKeyConsumerSelector selector = spy((StickyKeyConsumerSelector) selectorField.get(dispatcher)); + selectorField.set(dispatcher, selector); + doAnswer((invocationOnMock -> { + final int hash = invocationOnMock.getArgument(0); + final String consumerName = hashConsumerMap.getOrDefault(hash, c2.getConsumerName()); + return dispatcher.getConsumers().stream().filter(consumer -> consumer.consumerName().equals(consumerName)).findFirst().get(); + })).when(selector).select(anyInt()); + + // send and receive + Awaitility.await().untilAsserted(() -> assertEquals(admin.topics().getStats(topicName).getSubscriptions().get(subName).getConsumers().stream().filter(c -> c.getConsumerName().equals(c1ConsumerName)).findFirst().get().getAvailablePermits(), 1)); + final MessageIdImpl msg0Id = (MessageIdImpl) p.newMessage().key(keyA).value("msg-0").send(); + Awaitility.await().untilAsserted(() -> assertEquals(admin.topics().getStats(topicName).getSubscriptions().get(subName).getConsumers().stream().filter(c -> c.getConsumerName().equals(c1ConsumerName)).findFirst().get().getAvailablePermits(), 0)); + + final MessageIdImpl msg1Id = (MessageIdImpl) p.newMessage().key(keyA).value("msg-1").send(); + Awaitility.await().untilAsserted(() -> assertEquals(admin.topics().getStats(topicName).getSubscriptions().get(subName).getMsgBacklog(), 2)); + + final Field redeliveryMessagesField = PersistentDispatcherMultipleConsumers.class + .getDeclaredField("redeliveryMessages"); + redeliveryMessagesField.setAccessible(true); + final MessageRedeliveryController redeliveryMessages = (MessageRedeliveryController) redeliveryMessagesField.get(dispatcher); + + final Set replayMsgSet = redeliveryMessages.getMessagesToReplayNow(3, item -> true); + assertEquals(replayMsgSet.size(), 1); + final Position replayMsg = replayMsgSet.stream().findAny().get(); + assertEquals(replayMsg, PositionFactory.create(msg1Id.getLedgerId(), msg1Id.getEntryId())); + + // add c3 + final String c3ConsumerName = "c3"; + hashConsumerMap.put(hashA, c3ConsumerName); + @Cleanup + final Consumer c3 = cb.get().consumerName(c3ConsumerName).subscribe(); + final List> c3Msgs = new ArrayList<>(); + final org.apache.pulsar.broker.service.Consumer c3Broker = dispatcher.getConsumers().stream().filter(consumer -> consumer.consumerName().equals(c3ConsumerName)).findFirst().get(); + assertEquals(recentlyJoinedConsumers.get(c3Broker), PositionFactory.create(msg0Id.getLedgerId(), msg0Id.getEntryId())); + + // None of messages are sent to c3. + Message c3Msg = c3.receive(100, TimeUnit.MILLISECONDS); + assertNull(c3Msg); + + // Disconnect c1 + c1.close(); + + c3Msg = c3.receive(100, TimeUnit.MILLISECONDS); + assertNotNull(c3Msg); + c3Msgs.add(c3Msg); + // The mark delete position will move forward. Then remove c3 from recentlyJoinedConsumers. + c3.acknowledge(c3Msg); + Awaitility.await().untilAsserted(() -> assertNull(recentlyJoinedConsumers.get(c3Broker))); + c3Msg = c3.receive(100, TimeUnit.MILLISECONDS); + assertNotNull(c3Msg); + c3Msgs.add(c3Msg); + c3.acknowledge(c3Msg); + + // check ordering + assertTrue(c3Msgs.get(0).getMessageId().compareTo(c3Msgs.get(1).getMessageId()) < 0); + } + + @Test(timeOut = 30_000) + public void testLastSentPositionWhenRecreatingDispatcher() throws Exception { + // The lastSentPosition and individuallySentPositions should be initialized + // by the markDeletedPosition and individuallyDeletedMessages. + final String topicName = "persistent://public/default/rewind-" + UUID.randomUUID(); + final String subName = "my-sub"; + + final int numMessages = 9; + final List keys = Arrays.asList("key-a", "key-b", "key-c"); + final AtomicInteger receiveCounter = new AtomicInteger(); + final AtomicInteger ackCounter = new AtomicInteger(); + + @Cleanup + final Producer producer = pulsarClient.newProducer(Schema.INT32) + .topic(topicName) + .enableBatching(false) + .create(); + + final Supplier> cb = () -> pulsarClient.newConsumer(Schema.INT32) + .topic(topicName) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Key_Shared) + .keySharedPolicy(KeySharedPolicy.autoSplitHashRange() + .setAllowOutOfOrderDelivery(false)); + + @Cleanup + final Consumer c1 = cb.get().messageListener((c, msg) -> { + if (keys.get(0).equals(msg.getKey())) { + try { + c.acknowledge(msg); + ackCounter.getAndIncrement(); + } catch (PulsarClientException e) { + fail(e.getMessage()); + } + } + receiveCounter.getAndIncrement(); + }).subscribe(); + + PersistentStickyKeyDispatcherMultipleConsumers dispatcher = + (PersistentStickyKeyDispatcherMultipleConsumers) pulsar.getBrokerService().getTopicIfExists(topicName).get().get().getSubscription(subName).getDispatcher(); + LongPairRangeSet individuallySentPositionsField = dispatcher.getIndividuallySentPositionsField(); + final ManagedCursorImpl cursor = (ManagedCursorImpl) ((PersistentSubscription) pulsar.getBrokerService().getTopicIfExists(topicName).get().get().getSubscription(subName)).getCursor(); + final ManagedLedgerImpl ledger = (ManagedLedgerImpl) cursor.getManagedLedger(); + + MessageIdImpl msgId = null; + for (int i = 0; i < numMessages; i++) { + msgId = (MessageIdImpl) producer.newMessage().key(keys.get(i % keys.size())).value(i).send(); + } + + // wait for consumption + Awaitility.await().untilAsserted(() -> assertEquals(receiveCounter.get(), numMessages)); + assertEquals(ackCounter.get(), numMessages / keys.size()); + assertEquals(dispatcher.getLastSentPositionField(), PositionFactory.create(msgId.getLedgerId(), msgId.getEntryId())); + assertTrue(individuallySentPositionsField.isEmpty()); + receiveCounter.set(0); + ackCounter.set(0); + + // create expected values + final Position expectedLastSentPosition = ledger.getNextValidPosition(cursor.getMarkDeletedPosition()); + final ConcurrentOpenLongPairRangeSet + expectedIndividuallySentPositions = new ConcurrentOpenLongPairRangeSet<>(4096, PositionFactory::create); + cursor.getIndividuallyDeletedMessagesSet().forEach(range -> { + final Position lower = range.lowerEndpoint(); + final Position upper = range.upperEndpoint(); + expectedIndividuallySentPositions.addOpenClosed(lower.getLedgerId(), lower.getEntryId(), upper.getLedgerId(), upper.getEntryId()); + return true; + }); + + // modify subscription type to close current dispatcher + admin.topics().createSubscription(topicName, "sub-alt", MessageId.earliest); + c1.close(); + @Cleanup + final Consumer c2 = pulsarClient.newConsumer(Schema.INT32) + .topic(topicName) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Exclusive) + .subscribe(); + c2.close(); + assertEquals(admin.topics().getStats(topicName).getSubscriptions().get(subName).getType(), SubscriptionType.Exclusive.toString()); + + @Cleanup + final Consumer c3 = cb.get().receiverQueueSize(0).subscribe(); + dispatcher = (PersistentStickyKeyDispatcherMultipleConsumers) pulsar.getBrokerService().getTopicIfExists(topicName).get().get().getSubscription(subName).getDispatcher(); + individuallySentPositionsField = dispatcher.getIndividuallySentPositionsField(); + + assertNull(dispatcher.getLastSentPositionField()); + assertTrue(individuallySentPositionsField.isEmpty()); + + assertNotNull(c3.receive()); + + // validate the individuallySentPosition is initialized by the individuallyDeletedMessages + // if it is not initialized expectedly, it has sent-hole of key-c messages because key-c messages are not scheduled to be dispatched to some consumer(already acked). + assertEquals(dispatcher.getLastSentPositionField(), expectedLastSentPosition); + assertEquals(individuallySentPositionsField.toString(), expectedIndividuallySentPositions.toString()); + } + + @Test(timeOut = 30_000) + public void testLastSentPositionWhenResettingCursor() throws Exception { + // The lastSentPosition and individuallySentPositions should be cleared if reset-cursor operation is executed. + final String nsName = "public/default"; + final String topicName = "persistent://" + nsName + "/reset-cursor-" + UUID.randomUUID(); + final String subName = "my-sub"; + + final int numMessages = 10; + final List keys = Arrays.asList("key-a", "key-b"); + final AtomicInteger ackCounter = new AtomicInteger(); + + @Cleanup + final Producer producer = pulsarClient.newProducer(Schema.INT32) + .topic(topicName) + .enableBatching(false) + .create(); + + final Supplier> cb = () -> pulsarClient.newConsumer(Schema.INT32) + .topic(topicName) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Key_Shared) + .receiverQueueSize(0) + .keySharedPolicy(KeySharedPolicy.autoSplitHashRange() + .setAllowOutOfOrderDelivery(false)); + + @Cleanup + final Consumer c1 = cb.get().consumerName("c1").subscribe(); + @Cleanup + final Consumer c2 = cb.get().consumerName("c2").subscribe(); + + // set retention policy + admin.namespaces().setRetention(nsName, new RetentionPolicies(1, 1024 * 1024)); + + // enforce the selector will return c1 if keys.get(0) + final PersistentStickyKeyDispatcherMultipleConsumers dispatcher = + (PersistentStickyKeyDispatcherMultipleConsumers) pulsar.getBrokerService().getTopicIfExists(topicName).get().get().getSubscription(subName).getDispatcher(); + final int hashA = Murmur3_32Hash.getInstance().makeHash(keys.get(0).getBytes()); + final Map hashConsumerMap = new HashMap<>(); + hashConsumerMap.put(hashA, c1.getConsumerName()); + final Field selectorField = PersistentStickyKeyDispatcherMultipleConsumers.class.getDeclaredField("selector"); + selectorField.setAccessible(true); + final StickyKeyConsumerSelector selector = spy((StickyKeyConsumerSelector) selectorField.get(dispatcher)); + selectorField.set(dispatcher, selector); + doAnswer((invocationOnMock -> { + final int hash = invocationOnMock.getArgument(0); + final String consumerName = hashConsumerMap.getOrDefault(hash, c2.getConsumerName()); + return dispatcher.getConsumers().stream().filter(consumer -> consumer.consumerName().equals(consumerName)).findFirst().get(); + })).when(selector).select(anyInt()); + + for (int i = 0; i < numMessages; i++) { + producer.newMessage().key(keys.get(i % keys.size())).value(i).send(); + } + + // consume some messages + for (int i = 0; i < numMessages / keys.size(); i++) { + final Message msg = c2.receive(); + if (msg != null) { + c2.acknowledge(msg); + ackCounter.getAndIncrement(); + } + } + assertEquals(ackCounter.get(), numMessages / keys.size()); + + // store current lastSentPosition for comparison + final LongPairRangeSet individuallySentPositionsField = dispatcher.getIndividuallySentPositionsField(); + assertNotNull(dispatcher.getLastSentPositionField()); + assertFalse(individuallySentPositionsField.isEmpty()); + + // reset cursor and receive a message + admin.topics().resetCursor(topicName, subName, MessageId.earliest, true); + + // validate the lastSentPosition and individuallySentPositions are cleared after resetting cursor + assertNull(dispatcher.getLastSentPositionField()); + assertTrue(individuallySentPositionsField.isEmpty()); + } + + @Test(timeOut = 30_000) + public void testLastSentPositionWhenSkipping() throws Exception { + // The lastSentPosition and individuallySentPositions should be updated if skip operation is executed. + // There are updated to follow the new markDeletedPosition. + final String topicName = "persistent://public/default/skip-" + UUID.randomUUID(); + final String subName = "my-sub"; + + final int numMessages = 10; + final List keys = Arrays.asList("key-a", "key-b"); + final int numSkip = 2; + final AtomicInteger ackCounter = new AtomicInteger(); + + @Cleanup + final Producer producer = pulsarClient.newProducer(Schema.INT32) + .topic(topicName) + .enableBatching(false) + .create(); + + final Supplier> cb = () -> pulsarClient.newConsumer(Schema.INT32) + .topic(topicName) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Key_Shared) + .keySharedPolicy(KeySharedPolicy.autoSplitHashRange() + .setAllowOutOfOrderDelivery(false)) + .receiverQueueSize(0); + + @Cleanup + final Consumer c1 = cb.get().consumerName("c1").subscribe(); + @Cleanup + final Consumer c2 = cb.get().consumerName("c2").subscribe(); + + // enforce the selector will return c1 if keys.get(0) + final PersistentStickyKeyDispatcherMultipleConsumers dispatcher = + (PersistentStickyKeyDispatcherMultipleConsumers) pulsar.getBrokerService().getTopicIfExists(topicName).get().get().getSubscription(subName).getDispatcher(); + final int hashA = Murmur3_32Hash.getInstance().makeHash(keys.get(0).getBytes()); + final Map hashConsumerMap = new HashMap<>(); + hashConsumerMap.put(hashA, c1.getConsumerName()); + final Field selectorField = PersistentStickyKeyDispatcherMultipleConsumers.class.getDeclaredField("selector"); + selectorField.setAccessible(true); + final StickyKeyConsumerSelector selector = spy((StickyKeyConsumerSelector) selectorField.get(dispatcher)); + selectorField.set(dispatcher, selector); + doAnswer((invocationOnMock -> { + final int hash = invocationOnMock.getArgument(0); + final String consumerName = hashConsumerMap.getOrDefault(hash, c2.getConsumerName()); + return dispatcher.getConsumers().stream().filter(consumer -> consumer.consumerName().equals(consumerName)).findFirst().get(); + })).when(selector).select(anyInt()); + + final List positionList = new ArrayList<>(); + for (int i = 0; i < numMessages; i++) { + final MessageIdImpl msgId = (MessageIdImpl) producer.newMessage().key(keys.get(i % keys.size())).value(i).send(); + positionList.add(PositionFactory.create(msgId.getLedgerId(), msgId.getEntryId())); + } + + // consume some messages + for (int i = 0; i < numSkip; i++) { + final Message msg = c2.receive(); + if (msg != null) { + c2.acknowledge(msg); + ackCounter.getAndIncrement(); + } + } + assertEquals(ackCounter.get(), numSkip); + final ManagedCursorImpl managedCursor = ((ManagedCursorImpl) ((PersistentSubscription) pulsar.getBrokerService().getTopicIfExists(topicName).get().get().getSubscription(subName)).getCursor()); + Awaitility.await().untilAsserted(() -> assertEquals(managedCursor.getIndividuallyDeletedMessagesSet().size(), 2)); + + // store current lastSentPosition for comparison + final Position lastSentPositionBeforeSkip = dispatcher.getLastSentPositionField(); + final LongPairRangeSet individuallySentPositionsField = dispatcher.getIndividuallySentPositionsField(); + assertNotNull(lastSentPositionBeforeSkip); + assertFalse(individuallySentPositionsField.isEmpty()); + + // skip messages and receive a message + admin.topics().skipMessages(topicName, subName, numSkip); + final MessageIdImpl msgIdAfterSkip = (MessageIdImpl) c1.receive().getMessageId(); + final Position positionAfterSkip = PositionFactory.create(msgIdAfterSkip.getLedgerId(), + msgIdAfterSkip.getEntryId()); + assertEquals(positionAfterSkip, positionList.get(4)); + + // validate the lastSentPosition is updated to the new markDeletedPosition + // validate the individuallySentPositions is updated expectedly (removeAtMost the new markDeletedPosition) + final Position lastSentPosition = dispatcher.getLastSentPositionField(); + assertNotNull(lastSentPosition); + assertTrue(lastSentPosition.compareTo(lastSentPositionBeforeSkip) > 0); + assertEquals(lastSentPosition, positionList.get(4)); + assertTrue(individuallySentPositionsField.isEmpty()); + } private KeySharedMode getKeySharedModeOfSubscription(Topic topic, String subscription) { if (TopicName.get(topic.getName()).getDomain().equals(TopicDomain.persistent)) { @@ -1326,19 +1752,17 @@ private void receiveAndCheckDistribution(List> consumers, int expect private void receiveAndCheck(List, Integer>> checkList) throws PulsarClientException { Map> consumerKeys = new HashMap<>(); for (KeyValue, Integer> check : checkList) { - if (check.getValue() % 2 != 0) { - throw new IllegalArgumentException(); - } + Consumer consumer = check.getKey(); int received = 0; Map> lastMessageForKey = new HashMap<>(); for (Integer i = 0; i < check.getValue(); i++) { - Message message = check.getKey().receive(); + Message message = consumer.receive(); if (i % 2 == 0) { - check.getKey().acknowledge(message); + consumer.acknowledge(message); } String key = message.hasOrderingKey() ? new String(message.getOrderingKey()) : message.getKey(); log.info("[{}] Receive message key: {} value: {} messageId: {}", - check.getKey().getConsumerName(), key, message.getValue(), message.getMessageId()); + consumer.getConsumerName(), key, message.getValue(), message.getMessageId()); // check messages is order by key if (lastMessageForKey.get(key) == null) { Assert.assertNotNull(message); @@ -1347,8 +1771,8 @@ private void receiveAndCheck(List, Integer>> checkLis .compareTo(lastMessageForKey.get(key).getValue()) > 0); } lastMessageForKey.put(key, message); - consumerKeys.putIfAbsent(check.getKey(), new HashSet<>()); - consumerKeys.get(check.getKey()).add(key); + consumerKeys.putIfAbsent(consumer, new HashSet<>()); + consumerKeys.get(consumer).add(key); received++; } Assert.assertEquals(check.getValue().intValue(), received); @@ -1357,12 +1781,12 @@ private void receiveAndCheck(List, Integer>> checkLis // messages not acked, test redelivery lastMessageForKey = new HashMap<>(); for (int i = 0; i < redeliveryCount; i++) { - Message message = check.getKey().receive(); + Message message = consumer.receive(); received++; - check.getKey().acknowledge(message); + consumer.acknowledge(message); String key = message.hasOrderingKey() ? new String(message.getOrderingKey()) : message.getKey(); log.info("[{}] Receive redeliver message key: {} value: {} messageId: {}", - check.getKey().getConsumerName(), key, message.getValue(), message.getMessageId()); + consumer.getConsumerName(), key, message.getValue(), message.getMessageId()); // check redelivery messages is order by key if (lastMessageForKey.get(key) == null) { Assert.assertNotNull(message); @@ -1374,16 +1798,16 @@ private void receiveAndCheck(List, Integer>> checkLis } Message noMessages = null; try { - noMessages = check.getKey().receive(100, TimeUnit.MILLISECONDS); + noMessages = consumer.receive(100, TimeUnit.MILLISECONDS); } catch (PulsarClientException ignore) { } Assert.assertNull(noMessages, "redeliver too many messages."); Assert.assertEquals((check.getValue() + redeliveryCount), received); } Set allKeys = new HashSet<>(); - consumerKeys.forEach((k, v) -> v.forEach(key -> { + consumerKeys.forEach((k, v) -> v.stream().filter(Objects::nonNull).forEach(key -> { assertTrue(allKeys.add(key), - "Key "+ key + "is distributed to multiple consumers." ); + "Key " + key + " is distributed to multiple consumers." ); })); } @@ -1630,4 +2054,403 @@ public void testContinueDispatchMessagesWhenMessageDelayed() throws Exception { log.info("Got {} other messages...", sum); Assert.assertEquals(sum, delayedMessages + messages); } + + private AtomicInteger injectReplayReadCounter(String topicName, String cursorName) throws Exception { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).join().get(); + ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + ManagedCursorImpl cursor = (ManagedCursorImpl) managedLedger.openCursor(cursorName); + managedLedger.getCursors().removeCursor(cursor.getName()); + managedLedger.getActiveCursors().removeCursor(cursor.getName()); + ManagedCursorImpl spyCursor = Mockito.spy(cursor); + managedLedger.getCursors().add(spyCursor, PositionFactory.EARLIEST); + managedLedger.getActiveCursors().add(spyCursor, PositionFactory.EARLIEST); + AtomicInteger replyReadCounter = new AtomicInteger(); + Mockito.doAnswer(invocation -> { + if (!String.valueOf(invocation.getArguments()[2]).equals("Normal")) { + replyReadCounter.incrementAndGet(); + } + return invocation.callRealMethod(); + }).when(spyCursor).asyncReplayEntries(Mockito.anySet(), Mockito.any(), Mockito.any()); + Mockito.doAnswer(invocation -> { + if (!String.valueOf(invocation.getArguments()[2]).equals("Normal")) { + replyReadCounter.incrementAndGet(); + } + return invocation.callRealMethod(); + }).when(spyCursor).asyncReplayEntries(Mockito.anySet(), Mockito.any(), Mockito.any(), Mockito.anyBoolean()); + admin.topics().createSubscription(topicName, cursorName, MessageId.earliest); + return replyReadCounter; + } + + @Test + public void testNoRepeatedReadAndDiscard() throws Exception { + int delayedMessages = 100; + final String topic = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final String subName = "my-sub"; + admin.topics().createNonPartitionedTopic(topic); + AtomicInteger replyReadCounter = injectReplayReadCounter(topic, subName); + + // Send messages. + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.INT32).topic(topic).enableBatching(false).create(); + for (int i = 0; i < delayedMessages; i++) { + MessageId messageId = producer.newMessage() + .key(String.valueOf(random.nextInt(NUMBER_OF_KEYS))) + .value(100 + i) + .send(); + log.info("Published message :{}", messageId); + } + producer.close(); + + // Make ack holes. + Consumer consumer1 = pulsarClient.newConsumer(Schema.INT32) + .topic(topic) + .subscriptionName(subName) + .receiverQueueSize(10) + .subscriptionType(SubscriptionType.Key_Shared) + .subscribe(); + Consumer consumer2 = pulsarClient.newConsumer(Schema.INT32) + .topic(topic) + .subscriptionName(subName) + .receiverQueueSize(10) + .subscriptionType(SubscriptionType.Key_Shared) + .subscribe(); + List msgList1 = new ArrayList<>(); + List msgList2 = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + Message msg1 = consumer1.receive(1, TimeUnit.SECONDS); + if (msg1 != null) { + msgList1.add(msg1); + } + Message msg2 = consumer2.receive(1, TimeUnit.SECONDS); + if (msg2 != null) { + msgList2.add(msg2); + } + } + Consumer redeliverConsumer = null; + if (!msgList1.isEmpty()) { + msgList1.forEach(msg -> consumer1.acknowledgeAsync(msg)); + redeliverConsumer = consumer2; + } else { + msgList2.forEach(msg -> consumer2.acknowledgeAsync(msg)); + redeliverConsumer = consumer1; + } + + // consumer3 will be added to the "recentJoinedConsumers". + Consumer consumer3 = pulsarClient.newConsumer(Schema.INT32) + .topic(topic) + .subscriptionName(subName) + .receiverQueueSize(1000) + .subscriptionType(SubscriptionType.Key_Shared) + .subscribe(); + redeliverConsumer.close(); + + // Verify: no repeated Read-and-discard. + Thread.sleep(5 * 1000); + int maxReplayCount = delayedMessages * 2; + log.info("Reply read count: {}", replyReadCounter.get()); + assertTrue(replyReadCounter.get() < maxReplayCount); + + // cleanup. + consumer1.close(); + consumer2.close(); + consumer3.close(); + admin.topics().delete(topic, false); + } + + @DataProvider(name = "allowKeySharedOutOfOrder") + public Object[][] allowKeySharedOutOfOrder() { + return new Object[][]{ + {true}, + {false} + }; + } + + /** + * This test is in order to guarantee the feature added by https://github.com/apache/pulsar/pull/7105. + * 1. Start 3 consumers: + * - consumer1 will be closed and trigger a messages redeliver. + * - consumer2 will not ack any messages to make the new consumer joined late will be stuck due + * to the mechanism "recentlyJoinedConsumers". + * - consumer3 will always receive and ack messages. + * 2. Add consumer4 after consumer1 was close, and consumer4 will be stuck due to the mechanism + * "recentlyJoinedConsumers". + * 3. Verify: + * - (Main purpose) consumer3 can still receive messages util the cursor.readerPosition is larger than LAC. + * - no repeated Read-and-discard. + * - at last, all messages will be received. + */ + @Test(timeOut = 180 * 1000, dataProvider = "allowKeySharedOutOfOrder") // the test will be finished in 60s. + public void testRecentJoinedPosWillNotStuckOtherConsumer(boolean allowKeySharedOutOfOrder) throws Exception { + final int messagesSentPerTime = 100; + final Set totalReceivedMessages = new TreeSet<>(); + final String topic = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final String subName = "my-sub"; + admin.topics().createNonPartitionedTopic(topic); + AtomicInteger replyReadCounter = injectReplayReadCounter(topic, subName); + + // Send messages. + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.INT32).topic(topic).enableBatching(false).create(); + for (int i = 0; i < messagesSentPerTime; i++) { + MessageId messageId = producer.newMessage() + .key(String.valueOf(random.nextInt(NUMBER_OF_KEYS))) + .value(100 + i) + .send(); + log.info("Published message :{}", messageId); + } + + KeySharedPolicy keySharedPolicy = KeySharedPolicy.autoSplitHashRange() + .setAllowOutOfOrderDelivery(allowKeySharedOutOfOrder); + // 1. Start 3 consumers and make ack holes. + // - one consumer will be closed and trigger a messages redeliver. + // - one consumer will not ack any messages to make the new consumer joined late will be stuck due to the + // mechanism "recentlyJoinedConsumers". + // - one consumer will always receive and ack messages. + Consumer consumer1 = pulsarClient.newConsumer(Schema.INT32) + .topic(topic) + .subscriptionName(subName) + .receiverQueueSize(10) + .subscriptionType(SubscriptionType.Key_Shared) + .keySharedPolicy(keySharedPolicy) + .subscribe(); + Consumer consumer2 = pulsarClient.newConsumer(Schema.INT32) + .topic(topic) + .subscriptionName(subName) + .receiverQueueSize(10) + .subscriptionType(SubscriptionType.Key_Shared) + .keySharedPolicy(keySharedPolicy) + .subscribe(); + Consumer consumer3 = pulsarClient.newConsumer(Schema.INT32) + .topic(topic) + .subscriptionName(subName) + .receiverQueueSize(10) + .subscriptionType(SubscriptionType.Key_Shared) + .keySharedPolicy(keySharedPolicy) + .subscribe(); + List msgList1 = new ArrayList<>(); + List msgList2 = new ArrayList<>(); + List msgList3 = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + Message msg1 = consumer1.receive(1, TimeUnit.SECONDS); + if (msg1 != null) { + totalReceivedMessages.add(msg1.getValue()); + msgList1.add(msg1); + } + Message msg2 = consumer2.receive(1, TimeUnit.SECONDS); + if (msg2 != null) { + totalReceivedMessages.add(msg2.getValue()); + msgList2.add(msg2); + } + Message msg3 = consumer3.receive(1, TimeUnit.SECONDS); + if (msg2 != null) { + totalReceivedMessages.add(msg3.getValue()); + msgList3.add(msg3); + } + } + Consumer consumerWillBeClose = null; + Consumer consumerAlwaysAck = null; + Consumer consumerStuck = null; + if (!msgList1.isEmpty()) { + msgList1.forEach(msg -> consumer1.acknowledgeAsync(msg)); + consumerAlwaysAck = consumer1; + consumerWillBeClose = consumer2; + consumerStuck = consumer3; + } else if (!msgList2.isEmpty()){ + msgList2.forEach(msg -> consumer2.acknowledgeAsync(msg)); + consumerAlwaysAck = consumer2; + consumerWillBeClose = consumer3; + consumerStuck = consumer1; + } else { + msgList3.forEach(msg -> consumer3.acknowledgeAsync(msg)); + consumerAlwaysAck = consumer3; + consumerWillBeClose = consumer1; + consumerStuck = consumer2; + } + + // 2. Add consumer4 after "consumerWillBeClose" was close, and consumer4 will be stuck due to the mechanism + // "recentlyJoinedConsumers". + Consumer consumer4 = pulsarClient.newConsumer(Schema.INT32) + .topic(topic) + .subscriptionName(subName) + .receiverQueueSize(1000) + .subscriptionType(SubscriptionType.Key_Shared) + .keySharedPolicy(keySharedPolicy) + .subscribe(); + consumerWillBeClose.close(); + + Thread.sleep(2000); + + for (int i = messagesSentPerTime; i < messagesSentPerTime * 2; i++) { + MessageId messageId = producer.newMessage() + .key(String.valueOf(random.nextInt(NUMBER_OF_KEYS))) + .value(100 + i) + .send(); + log.info("Published message :{}", messageId); + } + + // Send messages again. + // Verify: "consumerAlwaysAck" can receive messages util the cursor.readerPosition is larger than LAC. + while (true) { + Message msg = consumerAlwaysAck.receive(2, TimeUnit.SECONDS); + if (msg == null) { + break; + } + totalReceivedMessages.add(msg.getValue()); + consumerAlwaysAck.acknowledge(msg); + } + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topic, false).join().get(); + ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + ManagedCursorImpl cursor = (ManagedCursorImpl) managedLedger.openCursor(subName); + log.info("cursor_readPosition {}, LAC {}", cursor.getReadPosition(), managedLedger.getLastConfirmedEntry()); + assertTrue((cursor.getReadPosition()) + .compareTo(managedLedger.getLastConfirmedEntry()) > 0); + + // Make all consumers to start to read and acknowledge messages. + // Verify: no repeated Read-and-discard. + Thread.sleep(5 * 1000); + int maxReplayCount = messagesSentPerTime * 2; + log.info("Reply read count: {}", replyReadCounter.get()); + assertTrue(replyReadCounter.get() < maxReplayCount); + // Verify: at last, all messages will be received. + ReceivedMessages receivedMessages = ackAllMessages(consumerAlwaysAck, consumerStuck, consumer4); + totalReceivedMessages.addAll(receivedMessages.messagesReceived.stream().map(p -> p.getRight()).collect( + Collectors.toList())); + assertEquals(totalReceivedMessages.size(), messagesSentPerTime * 2); + + // cleanup. + consumer1.close(); + consumer2.close(); + consumer3.close(); + consumer4.close(); + producer.close(); + admin.topics().delete(topic, false); + } + + @Test + public void testReadAheadLimit() throws Exception { + String topic = "testReadAheadLimit-" + UUID.randomUUID(); + int numberOfKeys = 1000; + long pauseTime = 100L; + int readAheadLimit = 20; + pulsar.getConfig().setKeySharedLookAheadMsgInReplayThresholdPerSubscription(readAheadLimit); + + @Cleanup + Producer producer = createProducer(topic, false); + + // create a consumer and close it to create a subscription + String subscriptionName = "key_shared"; + pulsarClient.newConsumer(Schema.INT32) + .topic(topic) + .subscriptionName(subscriptionName) + .subscriptionType(SubscriptionType.Key_Shared) + .subscribe() + .close(); + + Topic t = pulsar.getBrokerService().getTopicIfExists(topic).get().get(); + PersistentSubscription sub = (PersistentSubscription) t.getSubscription(subscriptionName); + // get the dispatcher reference + PersistentStickyKeyDispatcherMultipleConsumers dispatcher = + (PersistentStickyKeyDispatcherMultipleConsumers) sub.getDispatcher(); + + // create a function to use for checking the number of messages in replay + Runnable checkLimit = () -> { + assertThat(dispatcher.getNumberOfMessagesInReplay()).isLessThanOrEqualTo(readAheadLimit); + }; + + // Adding a new consumer. + @Cleanup + Consumer c1 = pulsarClient.newConsumer(Schema.INT32) + .topic(topic) + .consumerName("c1") + .subscriptionName(subscriptionName) + .subscriptionType(SubscriptionType.Key_Shared) + .receiverQueueSize(10) + .startPaused(true) // start paused + .subscribe(); + + @Cleanup + Consumer c2 = pulsarClient.newConsumer(Schema.INT32) + .topic(topic) + .consumerName("c2") + .subscriptionName(subscriptionName) + .subscriptionType(SubscriptionType.Key_Shared) + .receiverQueueSize(500) // use large receiver queue size + .subscribe(); + + @Cleanup + Consumer c3 = pulsarClient.newConsumer(Schema.INT32) + .topic(topic) + .consumerName("c3") + .subscriptionName(subscriptionName) + .subscriptionType(SubscriptionType.Key_Shared) + .receiverQueueSize(10) + .startPaused(true) // start paused + .subscribe(); + + // find keys that will be assigned to c2 + List keysForC2 = new ArrayList<>(); + for (int i = 0; i < numberOfKeys; i++) { + String key = String.valueOf(i); + byte[] keyBytes = key.getBytes(UTF_8); + int hash = StickyKeyConsumerSelector.makeStickyKeyHash(keyBytes); + if (dispatcher.getSelector().select(hash).consumerName().equals("c2")) { + keysForC2.add(key); + } + } + + Set remainingMessageValues = new HashSet<>(); + // produce messages with keys that all get assigned to c2 + for (int i = 0; i < 1000; i++) { + String key = keysForC2.get(random.nextInt(keysForC2.size())); + //log.info("Producing message with key: {} value: {}", key, i); + producer.newMessage() + .key(key) + .value(i) + .send(); + remainingMessageValues.add(i); + } + + checkLimit.run(); + + Thread.sleep(pauseTime); + checkLimit.run(); + + Thread.sleep(pauseTime); + checkLimit.run(); + + // resume c1 and c3 + c1.resume(); + c3.resume(); + + Thread.sleep(pauseTime); + checkLimit.run(); + + // produce more messages + for (int i = 1000; i < 2000; i++) { + String key = String.valueOf(random.nextInt(numberOfKeys)); + producer.newMessage() + .key(key) + .value(i) + .send(); + remainingMessageValues.add(i); + checkLimit.run(); + } + + // consume the messages + receiveMessages((consumer, msg) -> { + synchronized (this) { + try { + consumer.acknowledge(msg); + } catch (PulsarClientException e) { + throw new RuntimeException(e); + } + remainingMessageValues.remove(msg.getValue()); + checkLimit.run(); + return true; + } + }, Duration.ofSeconds(2), c1, c2, c3); + assertEquals(remainingMessageValues, Collections.emptySet()); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/LookupPropertiesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/LookupPropertiesTest.java new file mode 100644 index 0000000000000..768dc29731f49 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/LookupPropertiesTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api; + +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.MultiBrokerBaseTest; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.broker.namespace.LookupOptions; +import org.apache.pulsar.client.impl.LookupTopicResult; +import org.apache.pulsar.client.impl.PartitionedProducerImpl; +import org.apache.pulsar.client.impl.ProducerImpl; +import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.common.naming.ServiceUnitId; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.util.FutureUtil; +import org.testng.Assert; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker-api") +public class LookupPropertiesTest extends MultiBrokerBaseTest { + + private static final String BROKER_KEY = "lookup.broker.id"; + private static final String CLIENT_KEY = "broker.id"; + + @Override + protected void startBroker() throws Exception { + addCustomConfigs(conf, 0); + super.startBroker(); + } + + @Override + protected ServiceConfiguration createConfForAdditionalBroker(int additionalBrokerIndex) { + return addCustomConfigs(getDefaultConf(), additionalBrokerIndex + 10); + } + + private static ServiceConfiguration addCustomConfigs(ServiceConfiguration config, int index) { + config.setDefaultNumberOfNamespaceBundles(16); + config.setLoadBalancerAutoBundleSplitEnabled(false); + config.setLoadManagerClassName(BrokerIdAwareLoadManager.class.getName()); + config.setLoadBalancerAverageResourceUsageDifferenceThresholdPercentage(100); + config.setLoadBalancerDebugModeEnabled(true); + config.setBrokerShutdownTimeoutMs(1000); + final var properties = new Properties(); + properties.setProperty(BROKER_KEY, "broker-" + index); + config.setProperties(properties); + return config; + } + + @Test + public void testLookupProperty() throws Exception { + admin.namespaces().unload("public/default"); + final var topic = "test-lookup-property"; + admin.topics().createPartitionedTopic(topic, 16); + @Cleanup final var client = (PulsarClientImpl) PulsarClient.builder() + .serviceUrl(pulsar.getBrokerServiceUrl()) + .lookupProperties( + Collections.singletonMap(CLIENT_KEY, "broker-10")) // broker-10 refers to additionalBrokers[0] + .build(); + @Cleanup final var producer = (PartitionedProducerImpl) client.newProducer().topic(topic).create(); + Assert.assertNotNull(producer); + final var connections = producer.getProducers().stream().map(ProducerImpl::getClientCnx) + .collect(Collectors.toSet()); + Assert.assertEquals(connections.size(), 1); + final var port = ((InetSocketAddress) connections.stream().findAny().orElseThrow().ctx().channel() + .remoteAddress()).getPort(); + Assert.assertEquals(port, additionalBrokers.get(0).getBrokerListenPort().orElseThrow()); + } + + @Test + public void testConcurrentLookupProperties() throws Exception { + @Cleanup final var client = (PulsarClientImpl) PulsarClient.builder() + .serviceUrl(pulsar.getBrokerServiceUrl()) + .build(); + final var futures = new ArrayList>(); + BrokerIdAwareLoadManager.clientIdList.clear(); + + final var clientIdList = IntStream.range(0, 10).mapToObj(i -> "key-" + i).toList(); + for (var clientId : clientIdList) { + client.getConfiguration().setLookupProperties(Collections.singletonMap(CLIENT_KEY, clientId)); + futures.add(client.getLookup().getBroker(TopicName.get("test-concurrent-lookup-properties"))); + client.getConfiguration().setLookupProperties(Collections.emptyMap()); + } + FutureUtil.waitForAll(futures).get(); + Assert.assertEquals(clientIdList, BrokerIdAwareLoadManager.clientIdList); + } + + public static class BrokerIdAwareLoadManager extends ExtensibleLoadManagerImpl { + + static final List clientIdList = Collections.synchronizedList(new ArrayList<>()); + + @Override + public CompletableFuture> assign(Optional topic, + ServiceUnitId serviceUnit, LookupOptions options) { + getClientId(options).ifPresent(clientIdList::add); + return super.assign(topic, serviceUnit, options); + } + + @Override + public CompletableFuture> selectAsync(ServiceUnitId bundle, Set excludeBrokerSet, + LookupOptions options) { + final var clientId = options.getProperties() == null ? null : options.getProperties().get(CLIENT_KEY); + if (clientId == null) { + return super.selectAsync(bundle, excludeBrokerSet, options); + } + return getBrokerRegistry().getAvailableBrokerLookupDataAsync().thenCompose(brokerLookupDataMap -> { + final var optBroker = brokerLookupDataMap.entrySet().stream().filter(entry -> { + final var brokerId = entry.getValue().properties().get(BROKER_KEY); + return brokerId != null && brokerId.equals(clientId); + }).findAny(); + return optBroker.map(Map.Entry::getKey).map(Optional::of).map(CompletableFuture::completedFuture) + .orElseGet(() -> super.selectAsync(bundle, excludeBrokerSet, options)); + }); + } + + private static Optional getClientId(LookupOptions options) { + if (options.getProperties() == null) { + return Optional.empty(); + } + return Optional.ofNullable(options.getProperties().get(CLIENT_KEY)); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MaxProducerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MaxProducerTest.java new file mode 100644 index 0000000000000..a34b05280c4f5 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MaxProducerTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api; + +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import java.util.ArrayList; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker-api") +public class MaxProducerTest extends ProducerConsumerBase { + + @BeforeClass(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + conf.setMaxProducersPerTopic(2); + } + + @Test + public void testMaxProducersForBroker() throws Exception { + testMaxProducers(2); + } + + @Test + public void testMaxProducersForNamespace() throws Exception { + // set max clients + admin.namespaces().setMaxProducersPerTopic("public/default", 3); + testMaxProducers(3); + } + + private void testMaxProducers(int maxProducerExpected) throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createNonPartitionedTopic(topicName); + + List> producers = new ArrayList<>(); + for (int i = 0; i < maxProducerExpected; i++) { + producers.add(pulsarClient.newProducer().topic(topicName).create()); + } + + try { + pulsarClient.newProducer().topic(topicName).create(); + fail("should have failed"); + } catch (Exception e) { + assertTrue(e instanceof PulsarClientException.ProducerBusyException); + } + + // cleanup. + for (org.apache.pulsar.client.api.Producer p : producers) { + p.close(); + } + admin.topics().delete(topicName, false); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java index 4f4affc39d316..360d27f64133d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java @@ -49,6 +49,7 @@ import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.impl.DispatchRateImpl; +import org.apache.pulsar.broker.qos.AsyncTokenBucket; import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,6 +67,7 @@ public class MessageDispatchThrottlingTest extends ProducerConsumerBase { @BeforeClass @Override protected void setup() throws Exception { + AsyncTokenBucket.switchToConsistentTokensView(); this.conf.setClusterName("test"); super.internalSetup(); super.producerBaseSetup(); @@ -75,6 +77,7 @@ protected void setup() throws Exception { @Override protected void cleanup() throws Exception { super.internalCleanup(); + AsyncTokenBucket.resetToDefaultEventualConsistentTokensView(); } @AfterMethod(alwaysRun = true) @@ -246,20 +249,12 @@ public void testMessageRateLimitingNotReceiveAllMessages(SubscriptionType subscr // create producer and topic Producer producer = pulsarClient.newProducer().topic(topicName).create(); PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get(); - boolean isMessageRateUpdate = false; - int retry = 5; - for (int i = 0; i < retry; i++) { - if (topic.getDispatchRateLimiter().get().getDispatchRateOnMsg() > 0 - || topic.getDispatchRateLimiter().get().getDispatchRateOnByte() > 0) { - isMessageRateUpdate = true; - break; - } else { - if (i != retry - 1) { - Thread.sleep(100); - } - } - } - Assert.assertTrue(isMessageRateUpdate); + + Awaitility.await() + .ignoreExceptions() + .until(() -> topic.getDispatchRateLimiter().get().getDispatchRateOnMsg() > 0 + || topic.getDispatchRateLimiter().get().getDispatchRateOnByte() > 0); + Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate); int numMessages = 500; @@ -387,19 +382,11 @@ public void testMessageRateLimitingReceiveAllMessagesAfterThrottling(Subscriptio // create producer and topic Producer producer = pulsarClient.newProducer().topic(topicName).create(); PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get(); - boolean isMessageRateUpdate = false; - int retry = 5; - for (int i = 0; i < retry; i++) { - if (topic.getDispatchRateLimiter().get().getDispatchRateOnMsg() > 0) { - isMessageRateUpdate = true; - break; - } else { - if (i != retry - 1) { - Thread.sleep(100); - } - } - } - Assert.assertTrue(isMessageRateUpdate); + + Awaitility.await() + .ignoreExceptions() + .until(() -> topic.getDispatchRateLimiter().get().getDispatchRateOnMsg() > 0); + Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate); final int numProducedMessages = 20; @@ -498,7 +485,7 @@ public void testBytesRateLimitingReceiveAllMessagesAfterThrottling(SubscriptionT * * @throws Exception */ - @Test(timeOut = 5000) + @Test(timeOut = 10000) public void testRateLimitingMultipleConsumers() throws Exception { log.info("-- Starting {} test --", methodName); @@ -516,19 +503,11 @@ public void testRateLimitingMultipleConsumers() throws Exception { // create producer and topic Producer producer = pulsarClient.newProducer().topic(topicName).create(); PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get(); - boolean isMessageRateUpdate = false; - int retry = 5; - for (int i = 0; i < retry; i++) { - if (topic.getDispatchRateLimiter().get().getDispatchRateOnMsg() > 0) { - isMessageRateUpdate = true; - break; - } else { - if (i != retry - 1) { - Thread.sleep(100); - } - } - } - Assert.assertTrue(isMessageRateUpdate); + + Awaitility.await() + .ignoreExceptions() + .until(() -> topic.getDispatchRateLimiter().get().getDispatchRateOnMsg() > 0); + Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate); final int numProducedMessages = 500; @@ -536,11 +515,17 @@ public void testRateLimitingMultipleConsumers() throws Exception { final AtomicInteger totalReceived = new AtomicInteger(0); ConsumerBuilder consumerBuilder = pulsarClient.newConsumer().topic(topicName) + .receiverQueueSize(1) .subscriptionName("my-subscriber-name").subscriptionType(SubscriptionType.Shared).messageListener((c1, msg) -> { Assert.assertNotNull(msg, "Message cannot be null"); String receivedMessage = new String(msg.getData()); log.debug("Received message [{}] in the listener", receivedMessage); totalReceived.incrementAndGet(); + try { + c1.acknowledge(msg); + } catch (PulsarClientException e) { + throw new RuntimeException(e); + } }); Consumer consumer1 = consumerBuilder.subscribe(); Consumer consumer2 = consumerBuilder.subscribe(); @@ -558,10 +543,10 @@ public void testRateLimitingMultipleConsumers() throws Exception { } // it can make sure that consumer had enough time to consume message but couldn't consume due to throttling - Thread.sleep(500); + Thread.sleep(1000); - // consumer should not have received all published message due to message-rate throttling - Assert.assertNotEquals(totalReceived.get(), numProducedMessages); + // rate limiter should have limited messages with at least 10% accuracy (or 2 messages if messageRate is low) + Assert.assertEquals(totalReceived.get(), messageRate, Math.max(messageRate / 10, 2)); consumer1.close(); consumer2.close(); @@ -593,19 +578,11 @@ public void testRateLimitingWithBatchMsgEnabled() throws Exception { Producer producer = pulsarClient.newProducer().topic(topicName).enableBatching(true) .batchingMaxPublishDelay(1, TimeUnit.SECONDS).batchingMaxMessages(messagesPerBatch).create(); PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get(); - boolean isMessageRateUpdate = false; - int retry = 5; - for (int i = 0; i < retry; i++) { - if (topic.getDispatchRateLimiter().get().getDispatchRateOnMsg() > 0) { - isMessageRateUpdate = true; - break; - } else { - if (i != retry - 1) { - Thread.sleep(100); - } - } - } - Assert.assertTrue(isMessageRateUpdate); + + Awaitility.await() + .ignoreExceptions() + .until(() -> topic.getDispatchRateLimiter().get().getDispatchRateOnMsg() > 0); + Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate); final AtomicInteger totalReceived = new AtomicInteger(0); @@ -732,20 +709,12 @@ public void testMessageByteRateThrottlingCombined(SubscriptionType subscription) // create producer and topic Producer producer = pulsarClient.newProducer().topic(topicName).create(); PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get(); - boolean isMessageRateUpdate = false; - int retry = 5; - for (int i = 0; i < retry; i++) { - if (topic.getDispatchRateLimiter().get().getDispatchRateOnMsg() > 0 - && topic.getDispatchRateLimiter().get().getDispatchRateOnByte() > 0) { - isMessageRateUpdate = true; - break; - } else { - if (i != retry - 1) { - Thread.sleep(100); - } - } - } - Assert.assertTrue(isMessageRateUpdate); + + Awaitility.await() + .ignoreExceptions() + .until(() -> topic.getDispatchRateLimiter().get().getDispatchRateOnMsg() > 0 + || topic.getDispatchRateLimiter().get().getDispatchRateOnByte() > 0); + Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate); final int numProducedMessages = 200; @@ -813,20 +782,12 @@ public void testGlobalNamespaceThrottling() throws Exception { // create producer and topic Producer producer = pulsarClient.newProducer().topic(topicName).create(); PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get(); - boolean isMessageRateUpdate = false; - int retry = 5; - for (int i = 0; i < retry; i++) { - if (topic.getDispatchRateLimiter().get().getDispatchRateOnMsg() > 0 - || topic.getDispatchRateLimiter().get().getDispatchRateOnByte() > 0) { - isMessageRateUpdate = true; - break; - } else { - if (i != retry - 1) { - Thread.sleep(100); - } - } - } - Assert.assertTrue(isMessageRateUpdate); + + Awaitility.await() + .ignoreExceptions() + .until(() -> topic.getDispatchRateLimiter().get().getDispatchRateOnMsg() > 0 + || topic.getDispatchRateLimiter().get().getDispatchRateOnByte() > 0); + Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate); int numMessages = 500; @@ -1148,19 +1109,11 @@ public void testRelativeMessageRateLimitingThrottling(SubscriptionType subscript // create producer and topic Producer producer = pulsarClient.newProducer().topic(topicName).enableBatching(false).create(); PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get(); - boolean isMessageRateUpdate = false; - int retry = 10; - for (int i = 0; i < retry; i++) { - if (topic.getDispatchRateLimiter().get().getDispatchRateOnMsg() > 0) { - isMessageRateUpdate = true; - break; - } else { - if (i != retry - 1) { - Thread.sleep(100); - } - } - } - Assert.assertTrue(isMessageRateUpdate); + + Awaitility.await() + .ignoreExceptions() + .until(() -> topic.getDispatchRateLimiter().get().getDispatchRateOnMsg() > 0); + Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate); Thread.sleep(2000); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageListenerExecutorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageListenerExecutorTest.java new file mode 100644 index 0000000000000..9e148beb3045d --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageListenerExecutorTest.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api; + +import static org.testng.Assert.assertTrue; +import com.google.common.util.concurrent.Uninterruptibles; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicLong; +import lombok.Cleanup; +import org.apache.pulsar.client.util.ExecutorProvider; +import org.apache.pulsar.common.naming.TopicName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Test(groups = "broker-api") +public class MessageListenerExecutorTest extends ProducerConsumerBase { + private static final Logger log = LoggerFactory.getLogger(MessageListenerExecutorTest.class); + + @BeforeClass(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Override + protected void customizeNewPulsarClientBuilder(ClientBuilder clientBuilder) { + // Set listenerThreads to 1 to reproduce the pr more easily in #22861 + clientBuilder.listenerThreads(1); + } + + @Test + public void testConsumerMessageListenerExecutorIsolation() throws Exception { + log.info("-- Starting {} test --", methodName); + + @Cleanup("shutdownNow") + ExecutorService executor = Executors.newCachedThreadPool(); + List> maxConsumeDelayWithDisableIsolationFutures = new ArrayList<>(); + int loops = 5; + long consumeSleepTimeMs = 10000; + for (int i = 0; i < loops; i++) { + // The first consumer will consume messages with sleep block 1s, + // and the others will consume messages without sleep block. + // The maxConsumeDelayWithDisableIsolation of all consumers + // should be greater than sleepTimeMs cause by disable MessageListenerExecutor. + CompletableFuture maxConsumeDelayFuture = startConsumeAndComputeMaxConsumeDelay( + "persistent://my-property/my-ns/testConsumerMessageListenerDisableIsolation-" + i, + "my-sub-testConsumerMessageListenerDisableIsolation-" + i, + i == 0 ? Duration.ofMillis(consumeSleepTimeMs) : Duration.ofMillis(0), + false, + executor); + maxConsumeDelayWithDisableIsolationFutures.add(maxConsumeDelayFuture); + } + + // ensure all consumers consume messages delay more than consumeSleepTimeMs + boolean allDelayMoreThanConsumeSleepTimeMs = maxConsumeDelayWithDisableIsolationFutures.stream() + .map(CompletableFuture::join) + .allMatch(delay -> delay > consumeSleepTimeMs); + assertTrue(allDelayMoreThanConsumeSleepTimeMs); + + List> maxConsumeDelayWhitEnableIsolationFutures = new ArrayList<>(); + for (int i = 0; i < loops; i++) { + // The first consumer will consume messages with sleep block 1s, + // and the others will consume messages without sleep block. + // The maxConsumeDelayWhitEnableIsolation of the first consumer + // should be greater than sleepTimeMs, and the others should be + // less than sleepTimeMs, cause by enable MessageListenerExecutor. + CompletableFuture maxConsumeDelayFuture = startConsumeAndComputeMaxConsumeDelay( + "persistent://my-property/my-ns/testConsumerMessageListenerEnableIsolation-" + i, + "my-sub-testConsumerMessageListenerEnableIsolation-" + i, + i == 0 ? Duration.ofMillis(consumeSleepTimeMs) : Duration.ofMillis(0), + true, + executor); + maxConsumeDelayWhitEnableIsolationFutures.add(maxConsumeDelayFuture); + } + + assertTrue(maxConsumeDelayWhitEnableIsolationFutures.get(0).join() > consumeSleepTimeMs); + boolean remainingAlmostNoDelay = maxConsumeDelayWhitEnableIsolationFutures.stream() + .skip(1) + .map(CompletableFuture::join) + .allMatch(delay -> delay < 1000); + assertTrue(remainingAlmostNoDelay); + + log.info("-- Exiting {} test --", methodName); + } + + private CompletableFuture startConsumeAndComputeMaxConsumeDelay(String topic, String subscriptionName, + Duration consumeSleepTime, + boolean enableMessageListenerExecutorIsolation, + ExecutorService executorService) + throws Exception { + int numMessages = 2; + final CountDownLatch latch = new CountDownLatch(numMessages); + int numPartitions = 50; + TopicName nonIsolationTopicName = TopicName.get(topic); + admin.topics().createPartitionedTopic(nonIsolationTopicName.toString(), numPartitions); + + AtomicLong maxConsumeDelay = new AtomicLong(-1); + ConsumerBuilder consumerBuilder = + pulsarClient.newConsumer(Schema.INT64) + .topic(nonIsolationTopicName.toString()) + .subscriptionName(subscriptionName) + .messageListener((c1, msg) -> { + Assert.assertNotNull(msg, "Message cannot be null"); + log.debug("Received message [{}] in the listener", msg.getValue()); + c1.acknowledgeAsync(msg); + maxConsumeDelay.set(Math.max(maxConsumeDelay.get(), + System.currentTimeMillis() - msg.getValue())); + if (consumeSleepTime.toMillis() > 0) { + Uninterruptibles.sleepUninterruptibly(consumeSleepTime); + } + latch.countDown(); + }); + + ExecutorService executor = Executors.newSingleThreadExecutor( + new ExecutorProvider.ExtendedThreadFactory(subscriptionName + "listener-executor-", true)); + if (enableMessageListenerExecutorIsolation) { + consumerBuilder.messageListenerExecutor((message, runnable) -> executor.execute(runnable)); + } + + Consumer consumer = consumerBuilder.subscribe(); + ProducerBuilder producerBuilder = pulsarClient.newProducer(Schema.INT64) + .topic(nonIsolationTopicName.toString()); + + Producer producer = producerBuilder.create(); + List> futures = new ArrayList<>(); + + // Asynchronously produce messages + for (int i = 0; i < numMessages; i++) { + Future future = producer.sendAsync(System.currentTimeMillis()); + futures.add(future); + } + + log.info("Waiting for async publish to complete"); + for (Future future : futures) { + future.get(); + } + + CompletableFuture maxDelayFuture = new CompletableFuture<>(); + + CompletableFuture.runAsync(() -> { + try { + latch.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }, executorService).whenCompleteAsync((v, ex) -> { + maxDelayFuture.complete(maxConsumeDelay.get()); + try { + producer.close(); + consumer.close(); + executor.shutdownNow(); + } catch (PulsarClientException e) { + throw new RuntimeException(e); + } + }); + + return maxDelayFuture; + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiRolesTokenAuthorizationProviderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiRolesTokenAuthorizationProviderTest.java index 0445ad27ca8e7..ef775f8f81923 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiRolesTokenAuthorizationProviderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiRolesTokenAuthorizationProviderTest.java @@ -44,6 +44,7 @@ import org.apache.pulsar.client.impl.auth.AuthenticationToken; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfo; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -116,7 +117,7 @@ protected void setup() throws Exception { ); } - @BeforeClass + @AfterClass @Override protected void cleanup() throws Exception { super.internalCleanup(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiTopicsConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiTopicsConsumerTest.java index b8ea87ab4016e..7a12acd47edf9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiTopicsConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiTopicsConsumerTest.java @@ -26,9 +26,11 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import com.google.common.collect.Lists; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -42,6 +44,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import lombok.Cleanup; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.impl.ClientBuilderImpl; import org.apache.pulsar.client.impl.ConsumerImpl; @@ -50,6 +53,7 @@ import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.util.FutureUtil; import org.awaitility.Awaitility; import org.mockito.AdditionalAnswers; import org.mockito.Mockito; @@ -76,6 +80,11 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + @Override + protected void customizeNewPulsarClientBuilder(ClientBuilder clientBuilder) { + clientBuilder.ioThreads(4).connectionsPerBroker(4); + } + // test that reproduces the issue https://github.com/apache/pulsar/issues/12024 // where closing the consumer leads to an endless receive loop @Test @@ -351,4 +360,74 @@ public int choosePartition(Message msg, TopicMetadata metadata) { } consumer.close(); } + + @Test(invocationCount = 10, timeOut = 30000) + public void testMultipleIOThreads() throws PulsarAdminException, PulsarClientException { + final var topic = TopicName.get(newTopicName()).toString(); + final var numPartitions = 100; + admin.topics().createPartitionedTopic(topic, numPartitions); + for (int i = 0; i < 100; i++) { + admin.topics().createNonPartitionedTopic(topic + "-" + i); + } + @Cleanup + final var consumer = pulsarClient.newConsumer(Schema.INT32).topicsPattern(topic + ".*") + .subscriptionName("sub").subscribe(); + assertTrue(consumer instanceof MultiTopicsConsumerImpl); + assertTrue(consumer.isConnected()); + } + + @Test + public void testSameTopics() throws Exception { + final String topic1 = BrokerTestUtil.newUniqueName("public/default/tp"); + final String topic2 = "persistent://" + topic1; + admin.topics().createNonPartitionedTopic(topic2); + // Create consumer with two same topics. + try { + pulsarClient.newConsumer(Schema.INT32).topics(Arrays.asList(topic1, topic2)) + .subscriptionName("s1").subscribe(); + fail("Do not allow use two same topics."); + } catch (Exception e) { + if (e instanceof PulsarClientException && e.getCause() != null) { + e = (Exception) e.getCause(); + } + Throwable unwrapEx = FutureUtil.unwrapCompletionException(e); + assertTrue(unwrapEx instanceof IllegalArgumentException); + assertTrue(e.getMessage().contains( "Subscription topics include duplicate items" + + " or invalid names")); + } + // cleanup. + admin.topics().delete(topic2); + } + + @Test(timeOut = 30000) + public void testSubscriptionNotFound() throws PulsarAdminException, PulsarClientException { + final var topic1 = newTopicName(); + final var topic2 = newTopicName(); + + pulsar.getConfiguration().setAllowAutoSubscriptionCreation(false); + + try { + final var singleTopicConsumer = pulsarClient.newConsumer() + .topic(topic1) + .subscriptionName("sub-1") + .isAckReceiptEnabled(true) + .subscribe(); + assertTrue(singleTopicConsumer instanceof ConsumerImpl); + } catch (Throwable t) { + assertTrue(t.getCause().getCause() instanceof PulsarClientException.SubscriptionNotFoundException); + } + + try { + final var multiTopicsConsumer = pulsarClient.newConsumer() + .topics(List.of(topic1, topic2)) + .subscriptionName("sub-2") + .isAckReceiptEnabled(true) + .subscribe(); + assertTrue(multiTopicsConsumer instanceof MultiTopicsConsumerImpl); + } catch (Throwable t) { + assertTrue(t.getCause().getCause() instanceof PulsarClientException.SubscriptionNotFoundException); + } + + pulsar.getConfiguration().setAllowAutoSubscriptionCreation(true); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java index 2fc8aebf64a4a..81d65b192049b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java @@ -195,7 +195,7 @@ protected void setup() throws Exception { Set superUserRoles = new HashSet<>(); superUserRoles.add("admin"); conf.setSuperUserRoles(superUserRoles); - + conf.setTopicLevelPoliciesEnabled(false); conf.setAuthorizationEnabled(true); conf.setAuthenticationEnabled(true); Set providersClassNames = Sets.newHashSet(MutualAuthenticationProvider.class.getName()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonDurableSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonDurableSubscriptionTest.java index 6375f79bfbb6e..bbac688d9224c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonDurableSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonDurableSubscriptionTest.java @@ -19,17 +19,36 @@ package org.apache.pulsar.client.api; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.PulsarChannelInitializer; import org.apache.pulsar.broker.service.ServerCnx; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.impl.ConsumerImpl; +import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.common.api.proto.CommandFlow; +import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats; +import org.apache.pulsar.common.policies.data.SubscriptionStats; +import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -38,7 +57,7 @@ @Test(groups = "broker-api") @Slf4j -public class NonDurableSubscriptionTest extends ProducerConsumerBase { +public class NonDurableSubscriptionTest extends ProducerConsumerBase { private final AtomicInteger numFlow = new AtomicInteger(0); @@ -254,4 +273,400 @@ public void testFlowCountForMultiTopics() throws Exception { assertEquals(numFlow.get(), numPartitions); } + + private void trimLedgers(final String tpName) { + // Wait for topic loading. + org.awaitility.Awaitility.await().untilAsserted(() -> { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(tpName, false).join().get(); + assertNotNull(persistentTopic); + }); + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(tpName, false).join().get(); + ManagedLedgerImpl ml = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + CompletableFuture trimLedgersTask = new CompletableFuture<>(); + ml.trimConsumedLedgersInBackground(trimLedgersTask); + trimLedgersTask.join(); + } + + private void switchLedgerManually(final String tpName) throws Exception { + Method ledgerClosed = + ManagedLedgerImpl.class.getDeclaredMethod("ledgerClosed", new Class[]{LedgerHandle.class}); + Method createLedgerAfterClosed = + ManagedLedgerImpl.class.getDeclaredMethod("createLedgerAfterClosed", new Class[0]); + ledgerClosed.setAccessible(true); + createLedgerAfterClosed.setAccessible(true); + + // Wait for topic create. + org.awaitility.Awaitility.await().untilAsserted(() -> { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(tpName, false).join().get(); + assertNotNull(persistentTopic); + }); + + // Switch ledger. + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(tpName, false).join().get(); + ManagedLedgerImpl ml = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + LedgerHandle currentLedger1 = WhiteboxImpl.getInternalState(ml, "currentLedger"); + ledgerClosed.invoke(ml, new Object[]{currentLedger1}); + createLedgerAfterClosed.invoke(ml, new Object[0]); + Awaitility.await().untilAsserted(() -> { + LedgerHandle currentLedger2 = WhiteboxImpl.getInternalState(ml, "currentLedger"); + assertNotEquals(currentLedger1.getId(), currentLedger2.getId()); + }); + } + + @Test + public void testHasMessageAvailableIfIncomingQueueNotEmpty() throws Exception { + final String nonDurableCursor = "non-durable-cursor"; + final String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + Reader reader = pulsarClient.newReader(Schema.STRING).topic(topicName).receiverQueueSize(1) + .subscriptionName(nonDurableCursor).startMessageId(MessageIdImpl.earliest).create(); + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topicName).create(); + MessageIdImpl msgSent = (MessageIdImpl) producer.send("1"); + + // Trigger switch ledger. + // Trigger a trim ledgers task, and verify trim ledgers successful. + switchLedgerManually(topicName); + trimLedgers(topicName); + + // Since there is one message in the incoming queue, so the method "reader.hasMessageAvailable" should return + // true. + boolean hasMessageAvailable = reader.hasMessageAvailable(); + Message msgReceived = reader.readNext(2, TimeUnit.SECONDS); + if (msgReceived == null) { + assertFalse(hasMessageAvailable); + } else { + log.info("receive msg: {}", msgReceived.getValue()); + assertTrue(hasMessageAvailable); + assertEquals(msgReceived.getValue(), "1"); + } + + // cleanup. + reader.close(); + producer.close(); + admin.topics().delete(topicName); + } + + @Test + public void testInitReaderAtSpecifiedPosition() throws Exception { + String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createNonPartitionedTopic(topicName); + admin.topics().createSubscription(topicName, "s0", MessageId.earliest); + + // Trigger 5 ledgers. + ArrayList ledgers = new ArrayList<>(); + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topicName).create(); + for (int i = 0; i < 5; i++) { + MessageIdImpl msgId = (MessageIdImpl) producer.send("1"); + ledgers.add(msgId.getLedgerId()); + admin.topics().unload(topicName); + } + producer.close(); + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).join().get(); + ManagedLedgerImpl ml = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + LedgerHandle currentLedger = WhiteboxImpl.getInternalState(ml, "currentLedger"); + log.info("currentLedger: {}", currentLedger.getId()); + + // Less than the first ledger, and entry id is "-1". + log.info("start test s1"); + String s1 = "s1"; + MessageIdImpl startMessageId1 = new MessageIdImpl(ledgers.get(0) - 1, -1, -1); + Reader reader1 = pulsarClient.newReader(Schema.STRING).topic(topicName).subscriptionName(s1) + .receiverQueueSize(0).startMessageId(startMessageId1).create(); + ManagedLedgerInternalStats.CursorStats cursor1 = admin.topics().getInternalStats(topicName).cursors.get(s1); + log.info("cursor1 readPosition: {}, markDeletedPosition: {}", cursor1.readPosition, cursor1.markDeletePosition); + Position p1 = parseReadPosition(cursor1); + assertEquals(p1.getLedgerId(), ledgers.get(0)); + assertEquals(p1.getEntryId(), 0); + reader1.close(); + + // Less than the first ledger, and entry id is Long.MAX_VALUE. + log.info("start test s2"); + String s2 = "s2"; + MessageIdImpl startMessageId2 = new MessageIdImpl(ledgers.get(0) - 1, Long.MAX_VALUE, -1); + Reader reader2 = pulsarClient.newReader(Schema.STRING).topic(topicName).subscriptionName(s2) + .receiverQueueSize(0).startMessageId(startMessageId2).create(); + ManagedLedgerInternalStats.CursorStats cursor2 = admin.topics().getInternalStats(topicName).cursors.get(s2); + log.info("cursor2 readPosition: {}, markDeletedPosition: {}", cursor2.readPosition, cursor2.markDeletePosition); + Position p2 = parseReadPosition(cursor2); + assertEquals(p2.getLedgerId(), ledgers.get(0)); + assertEquals(p2.getEntryId(), 0); + reader2.close(); + + // Larger than the latest ledger, and entry id is "-1". + log.info("start test s3"); + String s3 = "s3"; + MessageIdImpl startMessageId3 = new MessageIdImpl(currentLedger.getId() + 1, -1, -1); + Reader reader3 = pulsarClient.newReader(Schema.STRING).topic(topicName).subscriptionName(s3) + .receiverQueueSize(0).startMessageId(startMessageId3).create(); + ManagedLedgerInternalStats.CursorStats cursor3 = admin.topics().getInternalStats(topicName).cursors.get(s3); + log.info("cursor3 readPosition: {}, markDeletedPosition: {}", cursor3.readPosition, cursor3.markDeletePosition); + Position p3 = parseReadPosition(cursor3); + assertEquals(p3.getLedgerId(), currentLedger.getId()); + assertEquals(p3.getEntryId(), 0); + reader3.close(); + + // Larger than the latest ledger, and entry id is Long.MAX_VALUE. + log.info("start test s4"); + String s4 = "s4"; + MessageIdImpl startMessageId4 = new MessageIdImpl(currentLedger.getId() + 1, Long.MAX_VALUE, -1); + Reader reader4 = pulsarClient.newReader(Schema.STRING).topic(topicName).subscriptionName(s4) + .receiverQueueSize(0).startMessageId(startMessageId4).create(); + ManagedLedgerInternalStats.CursorStats cursor4 = admin.topics().getInternalStats(topicName).cursors.get(s4); + log.info("cursor4 readPosition: {}, markDeletedPosition: {}", cursor4.readPosition, cursor4.markDeletePosition); + Position p4 = parseReadPosition(cursor4); + assertEquals(p4.getLedgerId(), currentLedger.getId()); + assertEquals(p4.getEntryId(), 0); + reader4.close(); + + // Ledger id and entry id both are Long.MAX_VALUE. + log.info("start test s5"); + String s5 = "s5"; + MessageIdImpl startMessageId5 = new MessageIdImpl(currentLedger.getId() + 1, Long.MAX_VALUE, -1); + Reader reader5 = pulsarClient.newReader(Schema.STRING).topic(topicName).subscriptionName(s5) + .receiverQueueSize(0).startMessageId(startMessageId5).create(); + ManagedLedgerInternalStats.CursorStats cursor5 = admin.topics().getInternalStats(topicName).cursors.get(s5); + log.info("cursor5 readPosition: {}, markDeletedPosition: {}", cursor5.readPosition, cursor5.markDeletePosition); + Position p5 = parseReadPosition(cursor5); + assertEquals(p5.getLedgerId(), currentLedger.getId()); + assertEquals(p5.getEntryId(), 0); + reader5.close(); + + // Ledger id equals LAC, and entry id is "-1". + log.info("start test s6"); + String s6 = "s6"; + MessageIdImpl startMessageId6 = new MessageIdImpl(ledgers.get(ledgers.size() - 1), -1, -1); + Reader reader6 = pulsarClient.newReader(Schema.STRING).topic(topicName).subscriptionName(s6) + .receiverQueueSize(0).startMessageId(startMessageId6).create(); + ManagedLedgerInternalStats.CursorStats cursor6 = admin.topics().getInternalStats(topicName).cursors.get(s6); + log.info("cursor6 readPosition: {}, markDeletedPosition: {}", cursor6.readPosition, cursor6.markDeletePosition); + Position p6 = parseReadPosition(cursor6); + assertEquals(p6.getLedgerId(), ledgers.get(ledgers.size() - 1)); + assertEquals(p6.getEntryId(), 0); + reader6.close(); + + // Larger than the latest ledger, and entry id is Long.MAX_VALUE. + log.info("start test s7"); + String s7 = "s7"; + MessageIdImpl startMessageId7 = new MessageIdImpl(ledgers.get(ledgers.size() - 1), Long.MAX_VALUE, -1); + Reader reader7 = pulsarClient.newReader(Schema.STRING).topic(topicName).subscriptionName(s7) + .receiverQueueSize(0).startMessageId(startMessageId7).create(); + ManagedLedgerInternalStats.CursorStats cursor7 = admin.topics().getInternalStats(topicName).cursors.get(s7); + log.info("cursor7 readPosition: {}, markDeletedPosition: {}", cursor7.readPosition, cursor7.markDeletePosition); + Position p7 = parseReadPosition(cursor7); + assertEquals(p7.getLedgerId(), currentLedger.getId()); + assertEquals(p7.getEntryId(), 0); + reader7.close(); + + // A middle ledger id, and entry id is "-1". + log.info("start test s8"); + String s8 = "s8"; + MessageIdImpl startMessageId8 = new MessageIdImpl(ledgers.get(2), 0, -1); + Reader reader8 = pulsarClient.newReader(Schema.STRING).topic(topicName).subscriptionName(s8) + .receiverQueueSize(0).startMessageId(startMessageId8).create(); + ManagedLedgerInternalStats.CursorStats cursor8 = admin.topics().getInternalStats(topicName).cursors.get(s8); + log.info("cursor8 readPosition: {}, markDeletedPosition: {}", cursor8.readPosition, cursor8.markDeletePosition); + Position p8 = parseReadPosition(cursor8); + assertEquals(p8.getLedgerId(), ledgers.get(2)); + assertEquals(p8.getEntryId(), 0); + reader8.close(); + + // Larger than the latest ledger, and entry id is Long.MAX_VALUE. + log.info("start test s9"); + String s9 = "s9"; + MessageIdImpl startMessageId9 = new MessageIdImpl(ledgers.get(2), Long.MAX_VALUE, -1); + Reader reader9 = pulsarClient.newReader(Schema.STRING).topic(topicName).subscriptionName(s9) + .receiverQueueSize(0).startMessageId(startMessageId9).create(); + ManagedLedgerInternalStats.CursorStats cursor9 = admin.topics().getInternalStats(topicName).cursors.get(s9); + log.info("cursor9 readPosition: {}, markDeletedPosition: {}", cursor9.readPosition, + cursor9.markDeletePosition); + Position p9 = parseReadPosition(cursor9); + assertEquals(p9.getLedgerId(), ledgers.get(3)); + assertEquals(p9.getEntryId(), 0); + reader9.close(); + + // Larger than the latest ledger, and entry id equals with the max entry id of this ledger. + log.info("start test s10"); + String s10 = "s10"; + MessageIdImpl startMessageId10 = new MessageIdImpl(ledgers.get(2), 0, -1); + Reader reader10 = pulsarClient.newReader(Schema.STRING).topic(topicName).subscriptionName(s10) + .receiverQueueSize(0).startMessageId(startMessageId10).create(); + ManagedLedgerInternalStats.CursorStats cursor10 = admin.topics().getInternalStats(topicName).cursors.get(s10); + log.info("cursor10 readPosition: {}, markDeletedPosition: {}", cursor10.readPosition, cursor10.markDeletePosition); + Position p10 = parseReadPosition(cursor10); + assertEquals(p10.getLedgerId(), ledgers.get(2)); + assertEquals(p10.getEntryId(), 0); + reader10.close(); + + // cleanup + admin.topics().delete(topicName, false); + } + + private Position parseReadPosition(ManagedLedgerInternalStats.CursorStats cursorStats) { + String[] ledgerIdAndEntryId = cursorStats.readPosition.split(":"); + return PositionFactory.create(Long.valueOf(ledgerIdAndEntryId[0]), Long.valueOf(ledgerIdAndEntryId[1])); + } + + @Test + public void testReaderInitAtDeletedPosition() throws Exception { + String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createNonPartitionedTopic(topicName); + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topicName).create(); + producer.send("1"); + producer.send("2"); + producer.send("3"); + MessageIdImpl msgIdInDeletedLedger4 = (MessageIdImpl) producer.send("4"); + MessageIdImpl msgIdInDeletedLedger5 = (MessageIdImpl) producer.send("5"); + + // Trigger a trim ledgers task, and verify trim ledgers successful. + admin.topics().unload(topicName); + trimLedgers(topicName); + List ledgers = admin.topics().getInternalStats(topicName).ledgers; + assertEquals(ledgers.size(), 1); + assertNotEquals(ledgers.get(0).ledgerId, msgIdInDeletedLedger5.getLedgerId()); + + // Start a reader at a deleted ledger. + MessageIdImpl startMessageId = + new MessageIdImpl(msgIdInDeletedLedger4.getLedgerId(), msgIdInDeletedLedger4.getEntryId(), -1); + Reader reader = pulsarClient.newReader(Schema.STRING).topic(topicName).subscriptionName("s1") + .startMessageId(startMessageId).create(); + Message msg1 = reader.readNext(2, TimeUnit.SECONDS); + Assert.assertNull(msg1); + + // Verify backlog and markDeletePosition is correct. + Awaitility.await().untilAsserted(() -> { + SubscriptionStats subscriptionStats = admin.topics() + .getStats(topicName, true, true, true).getSubscriptions().get("s1"); + log.info("backlog size: {}", subscriptionStats.getMsgBacklog()); + assertEquals(subscriptionStats.getMsgBacklog(), 0); + ManagedLedgerInternalStats.CursorStats cursorStats = + admin.topics().getInternalStats(topicName).cursors.get("s1"); + String[] ledgerIdAndEntryId = cursorStats.markDeletePosition.split(":"); + Position actMarkDeletedPos = + PositionFactory.create(Long.valueOf(ledgerIdAndEntryId[0]), Long.valueOf(ledgerIdAndEntryId[1])); + Position expectedMarkDeletedPos = + PositionFactory.create(msgIdInDeletedLedger5.getLedgerId(), msgIdInDeletedLedger5.getEntryId()); + log.info("Expected mark deleted position: {}", expectedMarkDeletedPos); + log.info("Actual mark deleted position: {}", cursorStats.markDeletePosition); + assertTrue(actMarkDeletedPos.compareTo(expectedMarkDeletedPos) >= 0); + }); + + // cleanup. + reader.close(); + producer.close(); + admin.topics().delete(topicName, false); + } + + @Test + public void testTrimLedgerIfNoDurableCursor() throws Exception { + final String nonDurableCursor = "non-durable-cursor"; + final String durableCursor = "durable-cursor"; + final String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createNonPartitionedTopic(topicName); + Reader reader = pulsarClient.newReader(Schema.STRING).topic(topicName).receiverQueueSize(1) + .subscriptionName(nonDurableCursor).startMessageId(MessageIdImpl.earliest).create(); + Consumer consumer = pulsarClient.newConsumer(Schema.STRING).topic(topicName).receiverQueueSize(1) + .subscriptionName(durableCursor).subscribe(); + consumer.close(); + + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topicName).create(); + producer.send("1"); + producer.send("2"); + producer.send("3"); + producer.send("4"); + MessageIdImpl msgIdInDeletedLedger5 = (MessageIdImpl) producer.send("5"); + + Message msg1 = reader.readNext(2, TimeUnit.SECONDS); + assertEquals(msg1.getValue(), "1"); + Message msg2 = reader.readNext(2, TimeUnit.SECONDS); + assertEquals(msg2.getValue(), "2"); + Message msg3 = reader.readNext(2, TimeUnit.SECONDS); + assertEquals(msg3.getValue(), "3"); + + // Unsubscribe durable cursor. + // Trigger a trim ledgers task, and verify trim ledgers successful. + admin.topics().unload(topicName); + Thread.sleep(3 * 1000); + admin.topics().deleteSubscription(topicName, durableCursor); + // Trim ledgers after release durable cursor. + trimLedgers(topicName); + List ledgers = admin.topics().getInternalStats(topicName).ledgers; + assertEquals(ledgers.size(), 1); + assertNotEquals(ledgers.get(0).ledgerId, msgIdInDeletedLedger5.getLedgerId()); + + // Verify backlog and markDeletePosition is correct. + Awaitility.await().untilAsserted(() -> { + SubscriptionStats subscriptionStats = admin.topics().getStats(topicName, true, true, true) + .getSubscriptions().get(nonDurableCursor); + log.info("backlog size: {}", subscriptionStats.getMsgBacklog()); + assertEquals(subscriptionStats.getMsgBacklog(), 0); + ManagedLedgerInternalStats.CursorStats cursorStats = + admin.topics().getInternalStats(topicName).cursors.get(nonDurableCursor); + String[] ledgerIdAndEntryId = cursorStats.markDeletePosition.split(":"); + Position actMarkDeletedPos = + PositionFactory.create(Long.valueOf(ledgerIdAndEntryId[0]), Long.valueOf(ledgerIdAndEntryId[1])); + Position expectedMarkDeletedPos = + PositionFactory.create(msgIdInDeletedLedger5.getLedgerId(), msgIdInDeletedLedger5.getEntryId()); + log.info("Expected mark deleted position: {}", expectedMarkDeletedPos); + log.info("Actual mark deleted position: {}", cursorStats.markDeletePosition); + Assert.assertTrue(actMarkDeletedPos.compareTo(expectedMarkDeletedPos) >= 0); + }); + + // Clear the incoming queue of the reader for next test. + while (true) { + Message msg = reader.readNext(2, TimeUnit.SECONDS); + if (msg == null) { + break; + } + log.info("clear msg: {}", msg.getValue()); + } + + // The following tests are designed to verify the api "getNumberOfEntries" and "consumedEntries" still work + // after changes.See the code-description added with the PR https://github.com/apache/pulsar/pull/10667. + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).join().get(); + ManagedLedgerImpl ml = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + ManagedCursorImpl cursor = (ManagedCursorImpl) ml.getCursors().get(nonDurableCursor); + + // Verify "getNumberOfEntries" if there is no entries to consume. + assertEquals(0, cursor.getNumberOfEntries()); + assertEquals(0, ml.getNumberOfEntries()); + + // Verify "getNumberOfEntries" if there is 1 entry to consume. + producer.send("6"); + producer.send("7"); + Awaitility.await().untilAsserted(() -> { + assertEquals(2, ml.getNumberOfEntries()); + // Since there is one message has been pulled into the incoming queue of reader. There is only one messages + // waiting to cursor read. + assertEquals(1, cursor.getNumberOfEntries()); + }); + + // Verify "consumedEntries" is correct. + ManagedLedgerInternalStats.CursorStats cursorStats = + admin.topics().getInternalStats(topicName).cursors.get(nonDurableCursor); + // "messagesConsumedCounter" should be 0 after unload the topic. + // Note: "topic_internal_stat.cursor.messagesConsumedCounter" means how many messages were acked on this + // cursor. The similar one "topic_stats.lastConsumedTimestamp" means the last time of sending messages to + // the consumer. + assertEquals(0, cursorStats.messagesConsumedCounter); + Message msg6 = reader.readNext(2, TimeUnit.SECONDS); + assertEquals(msg6.getValue(), "6"); + Message msg7 = reader.readNext(2, TimeUnit.SECONDS); + assertEquals(msg7.getValue(), "7"); + Awaitility.await().untilAsserted(() -> { + // "messagesConsumedCounter" should be 2 after consumed 2 message. + ManagedLedgerInternalStats.CursorStats cStat = + admin.topics().getInternalStats(topicName).cursors.get(nonDurableCursor); + assertEquals(2, cStat.messagesConsumedCounter); + }); + + // cleanup. + reader.close(); + producer.close(); + admin.topics().delete(topicName, false); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPartitionedTopicExpectedTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPartitionedTopicExpectedTest.java new file mode 100644 index 0000000000000..7b0edd314d055 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPartitionedTopicExpectedTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api; + +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.client.impl.ProducerBuilderImpl; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; +import org.apache.pulsar.common.policies.data.TopicType; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Slf4j +public class NonPartitionedTopicExpectedTest extends ProducerConsumerBase { + + @BeforeClass + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testWhenNonPartitionedTopicExists() throws Exception { + final String topic = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createNonPartitionedTopic(topic); + ProducerBuilderImpl producerBuilder = + (ProducerBuilderImpl) pulsarClient.newProducer(Schema.STRING).topic(topic); + producerBuilder.getConf().setNonPartitionedTopicExpected(true); + // Verify: create successfully. + Producer producer = producerBuilder.create(); + // cleanup. + producer.close(); + admin.topics().delete(topic, false); + } + + @Test + public void testWhenPartitionedTopicExists() throws Exception { + final String topic = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createPartitionedTopic(topic, 2); + ProducerBuilderImpl producerBuilder = + (ProducerBuilderImpl) pulsarClient.newProducer(Schema.STRING).topic(topic); + producerBuilder.getConf().setNonPartitionedTopicExpected(true); + // Verify: failed to create. + try { + producerBuilder.create(); + Assert.fail("expected an error since producer expected a non-partitioned topic"); + } catch (Exception ex) { + // expected an error. + log.error("expected error", ex); + } + // cleanup. + admin.topics().deletePartitionedTopic(topic, false); + } + + @DataProvider(name = "topicTypes") + public Object[][] topicTypes() { + return new Object[][]{ + {TopicType.PARTITIONED}, + {TopicType.NON_PARTITIONED} + }; + } + + @Test(dataProvider = "topicTypes") + public void testWhenTopicNotExists(TopicType topicType) throws Exception { + final String namespace = "public/default"; + final String topic = BrokerTestUtil.newUniqueName("persistent://" + namespace + "/tp"); + final TopicName topicName = TopicName.get(topic); + AutoTopicCreationOverride.Builder policyBuilder = AutoTopicCreationOverride.builder() + .topicType(topicType.toString()).allowAutoTopicCreation(true); + if (topicType.equals(TopicType.PARTITIONED)) { + policyBuilder.defaultNumPartitions(2); + } + AutoTopicCreationOverride policy = policyBuilder.build(); + admin.namespaces().setAutoTopicCreation(namespace, policy); + + ProducerBuilderImpl producerBuilder = + (ProducerBuilderImpl) pulsarClient.newProducer(Schema.STRING).topic(topic); + producerBuilder.getConf().setNonPartitionedTopicExpected(true); + // Verify: create successfully. + Producer producer = producerBuilder.create(); + // Verify: only create non-partitioned topic. + Assert.assertFalse(pulsar.getPulsarResources().getNamespaceResources().getPartitionedTopicResources() + .partitionedTopicExists(topicName)); + Assert.assertTrue(pulsar.getNamespaceService().checkNonPartitionedTopicExists(topicName).join()); + + // cleanup. + producer.close(); + admin.topics().delete(topic, false); + admin.namespaces().removeAutoTopicCreation(namespace); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java index 8527406496448..e5c992ec6f858 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.client.api; +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongSumValue; +import static org.assertj.core.api.Assertions.assertThat; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; @@ -27,6 +29,7 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.google.common.collect.Sets; +import io.opentelemetry.api.common.Attributes; import java.net.URL; import java.util.HashSet; import java.util.Optional; @@ -38,7 +41,9 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import lombok.Cleanup; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.LoadManager; @@ -48,8 +53,11 @@ import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.nonpersistent.NonPersistentReplicator; import org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic; +import org.apache.pulsar.broker.stats.OpenTelemetryProducerStats; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.impl.ConsumerImpl; +import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.impl.MultiTopicsConsumerImpl; import org.apache.pulsar.client.impl.PartitionedProducerImpl; import org.apache.pulsar.client.impl.ProducerImpl; @@ -62,8 +70,10 @@ import org.apache.pulsar.common.policies.data.SubscriptionStats; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TopicType; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.apache.pulsar.zookeeper.ZookeeperServerTest; +import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -101,6 +111,12 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + @Override + protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder pulsarTestContextBuilder) { + super.customizeMainPulsarTestContextBuilder(pulsarTestContextBuilder); + pulsarTestContextBuilder.enableOpenTelemetry(true); + } + @Test(timeOut = 90000 /* 1.5mn */) public void testNonPersistentPartitionsAreNotAutoCreatedWhenThePartitionedTopicDoesNotExist() throws Exception { final boolean defaultAllowAutoTopicCreation = conf.isAllowAutoTopicCreation(); @@ -353,9 +369,12 @@ public void testProducerRateLimit() throws Exception { @Cleanup("shutdownNow") ExecutorService executor = Executors.newFixedThreadPool(5); AtomicBoolean failed = new AtomicBoolean(false); + @Cleanup Consumer consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("subscriber-1") .subscribe(); - Producer producer = pulsarClient.newProducer().topic(topic).create(); + var producerName = BrokerTestUtil.newUniqueName("testProducerRateLimit"); + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topic).producerName(producerName).create(); byte[] msgData = "testData".getBytes(); final int totalProduceMessages = 10; CountDownLatch latch = new CountDownLatch(totalProduceMessages); @@ -388,7 +407,19 @@ public void testProducerRateLimit() throws Exception { // but as message should be dropped at broker: broker should not receive the message assertNotEquals(messageSet.size(), totalProduceMessages); - producer.close(); + // Verify the corresponding metric is updated + var attributes = Attributes.builder() + .put(OpenTelemetryAttributes.PULSAR_PRODUCER_NAME, producerName) + .put(OpenTelemetryAttributes.PULSAR_PRODUCER_ID, 0) + .put(OpenTelemetryAttributes.PULSAR_PRODUCER_ACCESS_MODE, "shared") + .put(OpenTelemetryAttributes.PULSAR_DOMAIN, "non-persistent") + .put(OpenTelemetryAttributes.PULSAR_TENANT, "my-property") + .put(OpenTelemetryAttributes.PULSAR_NAMESPACE, "my-property/my-ns") + .put(OpenTelemetryAttributes.PULSAR_TOPIC, topic) + .build(); + var metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + assertMetricLongSumValue(metrics, OpenTelemetryProducerStats.MESSAGE_DROP_COUNTER, attributes, + value -> assertThat(value).isPositive()); } finally { conf.setMaxConcurrentNonPersistentMessagePerConnection(defaultNonPersistentMessageRate); } @@ -818,17 +849,23 @@ public void testMsgDropStat() throws Exception { int defaultNonPersistentMessageRate = conf.getMaxConcurrentNonPersistentMessagePerConnection(); try { - final String topicName = "non-persistent://my-property/my-ns/stats-topic"; + final String topicName = BrokerTestUtil.newUniqueName("non-persistent://my-property/my-ns/stats-topic"); // restart broker with lower publish rate limit conf.setMaxConcurrentNonPersistentMessagePerConnection(1); stopBroker(); startBroker(); + + pulsar.getBrokerService().updateRates(); + + @Cleanup Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("subscriber-1") .receiverQueueSize(1).subscribe(); + @Cleanup Consumer consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName("subscriber-2") .receiverQueueSize(1).subscriptionType(SubscriptionType.Shared).subscribe(); + @Cleanup ProducerImpl producer = (ProducerImpl) pulsarClient.newProducer().topic(topicName) .enableBatching(false) .messageRoutingMode(MessageRoutingMode.SinglePartition) @@ -836,31 +873,41 @@ public void testMsgDropStat() throws Exception { @Cleanup("shutdownNow") ExecutorService executor = Executors.newFixedThreadPool(5); byte[] msgData = "testData".getBytes(); - final int totalProduceMessages = 200; - CountDownLatch latch = new CountDownLatch(totalProduceMessages); + final int totalProduceMessages = 1000; + CountDownLatch latch = new CountDownLatch(1); + AtomicInteger messagesSent = new AtomicInteger(0); for (int i = 0; i < totalProduceMessages; i++) { executor.submit(() -> { - producer.sendAsync(msgData).handle((msg, e) -> { - latch.countDown(); + producer.sendAsync(msgData).handle((msgId, e) -> { + int count = messagesSent.incrementAndGet(); + // process at least 20% of messages before signalling the latch + // a non-persistent message will return entryId as -1 when it has been dropped + // due to setMaxConcurrentNonPersistentMessagePerConnection limit + // also ensure that it has happened before the latch is signalled + if (count > totalProduceMessages * 0.2 && msgId != null + && ((MessageIdImpl) msgId).getEntryId() == -1) { + latch.countDown(); + } return null; }); }); } - latch.await(); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + + NonPersistentTopic topic = + (NonPersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get(); + + Awaitility.await().ignoreExceptions().untilAsserted(() -> { + pulsar.getBrokerService().updateRates(); + NonPersistentTopicStats stats = topic.getStats(false, false, false); + NonPersistentPublisherStats npStats = stats.getPublishers().get(0); + NonPersistentSubscriptionStats sub1Stats = stats.getSubscriptions().get("subscriber-1"); + NonPersistentSubscriptionStats sub2Stats = stats.getSubscriptions().get("subscriber-2"); + assertTrue(npStats.getMsgDropRate() > 0); + assertTrue(sub1Stats.getMsgDropRate() > 0); + assertTrue(sub2Stats.getMsgDropRate() > 0); + }); - NonPersistentTopic topic = (NonPersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get(); - pulsar.getBrokerService().updateRates(); - NonPersistentTopicStats stats = topic.getStats(false, false, false); - NonPersistentPublisherStats npStats = stats.getPublishers().get(0); - NonPersistentSubscriptionStats sub1Stats = stats.getSubscriptions().get("subscriber-1"); - NonPersistentSubscriptionStats sub2Stats = stats.getSubscriptions().get("subscriber-2"); - assertTrue(npStats.getMsgDropRate() > 0); - assertTrue(sub1Stats.getMsgDropRate() > 0); - assertTrue(sub2Stats.getMsgDropRate() > 0); - - producer.close(); - consumer.close(); - consumer2.close(); } finally { conf.setMaxConcurrentNonPersistentMessagePerConnection(defaultNonPersistentMessageRate); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/OrphanPersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/OrphanPersistentTopicTest.java new file mode 100644 index 0000000000000..9396a80cf2557 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/OrphanPersistentTopicTest.java @@ -0,0 +1,316 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api; + +import static org.apache.pulsar.broker.service.persistent.PersistentTopic.DEDUPLICATION_CURSOR_NAME; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; +import java.lang.reflect.Field; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.namespace.NamespaceService; +import org.apache.pulsar.broker.service.BrokerService; +import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.TopicPoliciesService; +import org.apache.pulsar.broker.service.TopicPolicyListener; +import org.apache.pulsar.broker.transaction.buffer.TransactionBuffer; +import org.apache.pulsar.broker.transaction.buffer.TransactionBufferProvider; +import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferDisable; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.compaction.CompactionServiceFactory; +import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker-api") +public class OrphanPersistentTopicTest extends ProducerConsumerBase { + + @BeforeClass(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testNoOrphanTopicAfterCreateTimeout() throws Exception { + // Make the topic loading timeout faster. + int topicLoadTimeoutSeconds = 2; + long originalTopicLoadTimeoutSeconds = pulsar.getConfig().getTopicLoadTimeoutSeconds(); + pulsar.getConfig().setTopicLoadTimeoutSeconds(2); + + String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + String mlPath = BrokerService.MANAGED_LEDGER_PATH_ZNODE + "/" + TopicName.get(tpName).getPersistenceNamingEncoding(); + + // Make topic load timeout 5 times. + AtomicInteger timeoutCounter = new AtomicInteger(); + for (int i = 0; i < 5; i++) { + mockZooKeeper.delay(topicLoadTimeoutSeconds * 2 * 1000, (op, path) -> { + if (mlPath.equals(path)) { + log.info("Topic load timeout: " + timeoutCounter.incrementAndGet()); + return true; + } + return false; + }); + } + + // Load topic. + CompletableFuture> consumer = pulsarClient.newConsumer() + .topic(tpName) + .subscriptionName("my-sub") + .subscribeAsync(); + + // After create timeout 5 times, the topic will be created successful. + Awaitility.await().ignoreExceptions().atMost(40, TimeUnit.SECONDS).untilAsserted(() -> { + CompletableFuture> future = pulsar.getBrokerService().getTopic(tpName, false); + assertTrue(future.isDone()); + Optional optional = future.get(); + assertTrue(optional.isPresent()); + }); + + // Assert only one PersistentTopic was not closed. + TopicPoliciesService topicPoliciesService = pulsar.getTopicPoliciesService(); + Map> listeners = + WhiteboxImpl.getInternalState(topicPoliciesService, "listeners"); + assertEquals(listeners.get(TopicName.get(tpName)).size(), 1); + + // cleanup. + consumer.join().close(); + admin.topics().delete(tpName, false); + pulsar.getConfig().setTopicLoadTimeoutSeconds(originalTopicLoadTimeoutSeconds); + } + + @Test + public void testCloseLedgerThatTopicAfterCreateTimeout() throws Exception { + // Make the topic loading timeout faster. + long originalTopicLoadTimeoutSeconds = pulsar.getConfig().getTopicLoadTimeoutSeconds(); + int topicLoadTimeoutSeconds = 1; + pulsar.getConfig().setTopicLoadTimeoutSeconds(topicLoadTimeoutSeconds); + pulsar.getConfig().setBrokerDeduplicationEnabled(true); + pulsar.getConfig().setTransactionCoordinatorEnabled(true); + String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp2"); + + // Mock message deduplication recovery speed topicLoadTimeoutSeconds + String mlPath = BrokerService.MANAGED_LEDGER_PATH_ZNODE + "/" + + TopicName.get(tpName).getPersistenceNamingEncoding() + "/" + DEDUPLICATION_CURSOR_NAME; + mockZooKeeper.delay(topicLoadTimeoutSeconds * 1000, (op, path) -> { + if (mlPath.equals(path)) { + log.info("Topic load timeout: " + path); + return true; + } + return false; + }); + + // First load topic will trigger timeout + // The first topic load will trigger a timeout. When the topic closes, it will call transactionBuffer.close. + // Here, we simulate a sleep to ensure that the ledger is not immediately closed. + TransactionBufferProvider mockTransactionBufferProvider = new TransactionBufferProvider() { + @Override + public TransactionBuffer newTransactionBuffer(Topic originTopic) { + return new TransactionBufferDisable(originTopic) { + @SneakyThrows + @Override + public CompletableFuture closeAsync() { + Thread.sleep(500); + return super.closeAsync(); + } + }; + } + }; + TransactionBufferProvider originalTransactionBufferProvider = pulsar.getTransactionBufferProvider(); + pulsar.setTransactionBufferProvider(mockTransactionBufferProvider); + CompletableFuture> firstLoad = pulsar.getBrokerService().getTopic(tpName, true); + Awaitility.await().ignoreExceptions().atMost(5, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + // assert first create topic timeout + .untilAsserted(() -> { + assertTrue(firstLoad.isCompletedExceptionally()); + }); + + // Once the first load topic times out, immediately to load the topic again. + Producer producer = pulsarClient.newProducer().topic(tpName).create(); + for (int i = 0; i < 10; i++) { + MessageId send = producer.send("msg".getBytes()); + Thread.sleep(100); + assertNotNull(send); + } + + // set to back + pulsar.setTransactionBufferProvider(originalTransactionBufferProvider); + pulsar.getConfig().setTopicLoadTimeoutSeconds(originalTopicLoadTimeoutSeconds); + pulsar.getConfig().setBrokerDeduplicationEnabled(false); + pulsar.getConfig().setTransactionCoordinatorEnabled(false); + } + + @Test + public void testNoOrphanTopicIfInitFailed() throws Exception { + String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createNonPartitionedTopic(tpName); + + // Load topic. + Consumer consumer = pulsarClient.newConsumer() + .topic(tpName) + .subscriptionName("my-sub") + .subscribe(); + + // Make the method `PersitentTopic.initialize` fail. + Field fieldCompactionServiceFactory = PulsarService.class.getDeclaredField("compactionServiceFactory"); + fieldCompactionServiceFactory.setAccessible(true); + CompactionServiceFactory compactionServiceFactory = + (CompactionServiceFactory) fieldCompactionServiceFactory.get(pulsar); + fieldCompactionServiceFactory.set(pulsar, null); + admin.topics().unload(tpName); + + // Wait for failed to create topic for several times. + Thread.sleep(5 * 1000); + + // Remove the injected error, the topic will be created successful. + fieldCompactionServiceFactory.set(pulsar, compactionServiceFactory); + // We do not know the next time of consumer reconnection, so wait for 2 minutes to avoid flaky. It will be + // very fast in normal. + Awaitility.await().ignoreExceptions().atMost(120, TimeUnit.SECONDS).untilAsserted(() -> { + CompletableFuture> future = pulsar.getBrokerService().getTopic(tpName, false); + assertTrue(future.isDone()); + Optional optional = future.get(); + assertTrue(optional.isPresent()); + }); + + // Assert only one PersistentTopic was not closed. + TopicPoliciesService topicPoliciesService = pulsar.getTopicPoliciesService(); + Map> listeners = + WhiteboxImpl.getInternalState(topicPoliciesService, "listeners"); + assertEquals(listeners.get(TopicName.get(tpName)).size(), 1); + + // cleanup. + consumer.close(); + admin.topics().delete(tpName, false); + } + + @DataProvider(name = "whetherTimeoutOrNot") + public Object[][] whetherTimeoutOrNot() { + return new Object[][] { + {true}, + {false} + }; + } + + @Test(timeOut = 60 * 1000, dataProvider = "whetherTimeoutOrNot") + public void testCheckOwnerShipFails(boolean injectTimeout) throws Exception { + if (injectTimeout) { + pulsar.getConfig().setTopicLoadTimeoutSeconds(5); + } + String ns = "public" + "/" + UUID.randomUUID().toString().replaceAll("-", ""); + String tpName = BrokerTestUtil.newUniqueName("persistent://" + ns + "/tp"); + admin.namespaces().createNamespace(ns); + admin.topics().createNonPartitionedTopic(tpName); + admin.namespaces().unload(ns); + + // Inject an error when calling "NamespaceService.isServiceUnitActiveAsync". + AtomicInteger failedTimes = new AtomicInteger(); + NamespaceService namespaceService = pulsar.getNamespaceService(); + doAnswer(invocation -> { + TopicName paramTp = (TopicName) invocation.getArguments()[0]; + if (paramTp.toString().equalsIgnoreCase(tpName) && failedTimes.incrementAndGet() <= 2) { + if (injectTimeout) { + Thread.sleep(10 * 1000); + } + log.info("Failed {} times", failedTimes.get()); + return CompletableFuture.failedFuture(new RuntimeException("mocked error")); + } + return invocation.callRealMethod(); + }).when(namespaceService).isServiceUnitActiveAsync(any(TopicName.class)); + + // Verify: the consumer can create successfully eventually. + Consumer consumer = pulsarClient.newConsumer().topic(tpName).subscriptionName("s1").subscribe(); + + // cleanup. + if (injectTimeout) { + pulsar.getConfig().setTopicLoadTimeoutSeconds(60); + } + consumer.close(); + admin.topics().delete(tpName); + } + + @Test(timeOut = 60 * 1000, dataProvider = "whetherTimeoutOrNot") + public void testTopicLoadAndDeleteAtTheSameTime(boolean injectTimeout) throws Exception { + if (injectTimeout) { + pulsar.getConfig().setTopicLoadTimeoutSeconds(5); + } + String ns = "public" + "/" + UUID.randomUUID().toString().replaceAll("-", ""); + String tpName = BrokerTestUtil.newUniqueName("persistent://" + ns + "/tp"); + admin.namespaces().createNamespace(ns); + admin.topics().createNonPartitionedTopic(tpName); + admin.namespaces().unload(ns); + + // Inject a race condition: load topic and delete topic execute at the same time. + AtomicInteger mockRaceConditionCounter = new AtomicInteger(); + NamespaceService namespaceService = pulsar.getNamespaceService(); + doAnswer(invocation -> { + TopicName paramTp = (TopicName) invocation.getArguments()[0]; + if (paramTp.toString().equalsIgnoreCase(tpName) && mockRaceConditionCounter.incrementAndGet() <= 1) { + if (injectTimeout) { + Thread.sleep(10 * 1000); + } + log.info("Race condition occurs {} times", mockRaceConditionCounter.get()); + pulsar.getManagedLedgerFactory().delete(TopicName.get(tpName).getPersistenceNamingEncoding()); + } + return invocation.callRealMethod(); + }).when(namespaceService).isServiceUnitActiveAsync(any(TopicName.class)); + + // Verify: the consumer create failed due to pulsar does not allow to create topic automatically. + try { + pulsar.getBrokerService().getTopic(tpName, false, Collections.emptyMap()).join(); + } catch (Exception ex) { + log.warn("Expected error", ex); + } + + // Verify: the consumer create successfully after allowing to create topic automatically. + Consumer consumer = pulsarClient.newConsumer().topic(tpName).subscriptionName("s1").subscribe(); + + // cleanup. + if (injectTimeout) { + pulsar.getConfig().setTopicLoadTimeoutSeconds(60); + } + consumer.close(); + admin.topics().delete(tpName); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PatternMultiTopicsConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PatternMultiTopicsConsumerTest.java index 00a47c3957150..475477ac52149 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PatternMultiTopicsConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PatternMultiTopicsConsumerTest.java @@ -18,11 +18,14 @@ */ package org.apache.pulsar.client.api; +import static org.testng.Assert.fail; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.client.impl.PatternMultiTopicsConsumerImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -95,4 +98,38 @@ private void testWithConsumer(Consumer consumer) throws Exception { consumer.close(); } + @Test(timeOut = 30000) + public void testFailedSubscribe() throws Exception { + final String topicName1 = BrokerTestUtil.newUniqueName("persistent://public/default/tp_test"); + final String topicName2 = BrokerTestUtil.newUniqueName("persistent://public/default/tp_test"); + final String topicName3 = BrokerTestUtil.newUniqueName("persistent://public/default/tp_test"); + final String subName = "s1"; + admin.topics().createPartitionedTopic(topicName1, 2); + admin.topics().createPartitionedTopic(topicName2, 3); + admin.topics().createNonPartitionedTopic(topicName3); + + // Register a exclusive consumer to makes the pattern consumer failed to subscribe. + Consumer c1 = pulsarClient.newConsumer(Schema.STRING).topic(topicName3).subscriptionType(SubscriptionType.Exclusive) + .subscriptionName(subName).subscribe(); + + try { + PatternMultiTopicsConsumerImpl consumer = + (PatternMultiTopicsConsumerImpl) pulsarClient.newConsumer(Schema.STRING) + .topicsPattern("persistent://public/default/tp_test.*") + .subscriptionType(SubscriptionType.Failover) + .subscriptionName(subName) + .subscribe(); + fail("Expected a consumer busy error."); + } catch (Exception ex) { + log.info("consumer busy", ex); + } + + c1.close(); + // Verify all internal consumer will be closed. + // If delete topic without "-f" work, it means the internal consumers were closed. + admin.topics().delete(topicName3); + admin.topics().deletePartitionedTopic(topicName2); + admin.topics().deletePartitionedTopic(topicName1); + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerConsumerBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerConsumerBase.java index f58c1fa26afc7..0cf2e49d35bee 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerConsumerBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerConsumerBase.java @@ -18,12 +18,17 @@ */ package org.apache.pulsar.client.api; +import static org.apache.pulsar.broker.BrokerTestUtil.receiveMessagesInThreads; import com.google.common.collect.Sets; - import java.lang.reflect.Method; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Random; import java.util.Set; - +import java.util.function.BiFunction; +import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; @@ -69,4 +74,49 @@ protected String newTopicName() { return "my-property/my-ns/topic-" + Long.toHexString(random.nextLong()); } + protected ReceivedMessages receiveAndAckMessages( + BiFunction ackPredicate, + Consumer...consumers) throws Exception { + ReceivedMessages receivedMessages = new ReceivedMessages(); + receiveMessagesInThreads((consumer, msg) -> { + T v = msg.getValue(); + MessageId messageId = msg.getMessageId(); + receivedMessages.messagesReceived.add(Pair.of(msg.getMessageId(), v)); + if (ackPredicate.apply(messageId, v)) { + consumer.acknowledgeAsync(msg); + receivedMessages.messagesAcked.add(Pair.of(msg.getMessageId(), v)); + } + return true; + }, Duration.ofSeconds(2), consumers); + return receivedMessages; + } + + protected ReceivedMessages ackAllMessages(Consumer...consumers) throws Exception { + return receiveAndAckMessages((msgId, msgV) -> true, consumers); + } + + protected static class ReceivedMessages { + + List> messagesReceived = Collections.synchronizedList(new ArrayList<>()); + + List> messagesAcked = Collections.synchronizedList(new ArrayList<>()); + + public boolean hasReceivedMessage(T v) { + for (Pair pair : messagesReceived) { + if (pair.getRight().equals(v)) { + return true; + } + } + return false; + } + + public boolean hasAckedMessage(T v) { + for (Pair pair : messagesAcked) { + if (pair.getRight().equals(v)) { + return true; + } + } + return false; + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PulsarMultiListenersWithInternalListenerNameTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PulsarMultiListenersWithInternalListenerNameTest.java index 8365b7a555703..a076e20b33218 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PulsarMultiListenersWithInternalListenerNameTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PulsarMultiListenersWithInternalListenerNameTest.java @@ -44,6 +44,7 @@ import org.apache.pulsar.client.impl.LookupService; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfo; @@ -137,24 +138,24 @@ private void doFindBrokerWithListenerName(boolean useHttp) throws Exception { conf.setMaxLookupRedirects(10); @Cleanup - LookupService lookupService = useHttp ? new HttpLookupService(conf, eventExecutors) : + LookupService lookupService = useHttp ? new HttpLookupService(InstrumentProvider.NOOP, conf, eventExecutors) : new BinaryProtoLookupService((PulsarClientImpl) this.pulsarClient, lookupUrl.toString(), "internal", false, this.executorService); + TopicName topicName = TopicName.get("persistent://public/default/test"); + // test request 1 { - CompletableFuture> future = - lookupService.getBroker(TopicName.get("persistent://public/default/test")); - Pair result = future.get(10, TimeUnit.SECONDS); - Assert.assertEquals(result.getKey(), brokerAddress); - Assert.assertEquals(result.getValue(), brokerAddress); + var result = lookupService.getBroker(topicName).get(10, TimeUnit.SECONDS); + Assert.assertEquals(result.getLogicalAddress(), brokerAddress); + Assert.assertEquals(result.getPhysicalAddress(), brokerAddress); + Assert.assertEquals(result.isUseProxy(), false); } // test request 2 { - CompletableFuture> future = - lookupService.getBroker(TopicName.get("persistent://public/default/test")); - Pair result = future.get(10, TimeUnit.SECONDS); - Assert.assertEquals(result.getKey(), brokerAddress); - Assert.assertEquals(result.getValue(), brokerAddress); + var result = lookupService.getBroker(topicName).get(10, TimeUnit.SECONDS); + Assert.assertEquals(result.getLogicalAddress(), brokerAddress); + Assert.assertEquals(result.getPhysicalAddress(), brokerAddress); + Assert.assertEquals(result.isUseProxy(), false); } } @@ -172,7 +173,7 @@ public void testHttpLookupRedirect() throws Exception { conf.setMaxLookupRedirects(10); @Cleanup - HttpLookupService lookupService = new HttpLookupService(conf, eventExecutors); + HttpLookupService lookupService = new HttpLookupService(InstrumentProvider.NOOP, conf, eventExecutors); NamespaceService namespaceService = pulsar.getNamespaceService(); LookupResult lookupResult = new LookupResult(pulsar.getWebServiceAddress(), null, @@ -187,12 +188,11 @@ public void testHttpLookupRedirect() throws Exception { doReturn(CompletableFuture.completedFuture(optional), CompletableFuture.completedFuture(optional2)) .when(namespaceService).getBrokerServiceUrlAsync(any(), any()); - CompletableFuture> future = - lookupService.getBroker(TopicName.get("persistent://public/default/test")); - - Pair result = future.get(10, TimeUnit.SECONDS); - Assert.assertEquals(result.getKey(), address); - Assert.assertEquals(result.getValue(), address); + var result = + lookupService.getBroker(TopicName.get("persistent://public/default/test")).get(10, TimeUnit.SECONDS); + Assert.assertEquals(result.getLogicalAddress(), address); + Assert.assertEquals(result.getPhysicalAddress(), address); + Assert.assertEquals(result.isUseProxy(), false); } @AfterMethod(alwaysRun = true) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/RetryTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/RetryTopicTest.java index 2ccae72143443..9cb82fde04118 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/RetryTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/RetryTopicTest.java @@ -257,6 +257,9 @@ public void testAutoConsumeSchemaRetryLetter() throws Exception { public void testRetryTopicProperties() throws Exception { final String topic = "persistent://my-property/my-ns/retry-topic"; + byte[] key = "key".getBytes(); + byte[] orderingKey = "orderingKey".getBytes(); + final int maxRedeliveryCount = 3; final int sendMessages = 10; @@ -285,7 +288,11 @@ public void testRetryTopicProperties() throws Exception { Set originMessageIds = new HashSet<>(); for (int i = 0; i < sendMessages; i++) { - MessageId msgId = producer.send(String.format("Hello Pulsar [%d]", i).getBytes()); + MessageId msgId = producer.newMessage() + .value(String.format("Hello Pulsar [%d]", i).getBytes()) + .keyBytes(key) + .orderingKey(orderingKey) + .send(); originMessageIds.add(msgId.toString()); } @@ -298,6 +305,10 @@ public void testRetryTopicProperties() throws Exception { if (message.hasProperty(RetryMessageUtil.SYSTEM_PROPERTY_RECONSUMETIMES)) { // check the REAL_TOPIC property assertEquals(message.getProperty(RetryMessageUtil.SYSTEM_PROPERTY_REAL_TOPIC), topic); + assertTrue(message.hasKey()); + assertEquals(message.getKeyBytes(), key); + assertTrue(message.hasOrderingKey()); + assertEquals(message.getOrderingKey(), orderingKey); retryMessageIds.add(message.getProperty(RetryMessageUtil.SYSTEM_PROPERTY_ORIGIN_MESSAGE_ID)); } consumer.reconsumeLater(message, 1, TimeUnit.SECONDS); @@ -317,6 +328,10 @@ public void testRetryTopicProperties() throws Exception { if (message.hasProperty(RetryMessageUtil.SYSTEM_PROPERTY_RECONSUMETIMES)) { // check the REAL_TOPIC property assertEquals(message.getProperty(RetryMessageUtil.SYSTEM_PROPERTY_REAL_TOPIC), topic); + assertTrue(message.hasKey()); + assertEquals(message.getKeyBytes(), key); + assertTrue(message.hasOrderingKey()); + assertEquals(message.getOrderingKey(), orderingKey); deadLetterMessageIds.add(message.getProperty(RetryMessageUtil.SYSTEM_PROPERTY_ORIGIN_MESSAGE_ID)); } deadLetterConsumer.acknowledge(message); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerDisallowAutoCreateTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerDisallowAutoCreateTopicTest.java new file mode 100644 index 0000000000000..728e556f0224a --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerDisallowAutoCreateTopicTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api; + +import static org.apache.pulsar.client.util.RetryMessageUtil.RETRY_GROUP_TOPIC_SUFFIX; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker-api") +public class SimpleProducerConsumerDisallowAutoCreateTopicTest extends ProducerConsumerBase { + + @BeforeClass(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + conf.setAllowAutoTopicCreation(false); + } + + @Test + public void testClearErrorIfRetryTopicNotExists() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp_"); + final String subName = "sub"; + final String retryTopicName = topicName + "-" + subName + RETRY_GROUP_TOPIC_SUFFIX; + admin.topics().createNonPartitionedTopic(topicName); + Consumer consumer = null; + try { + consumer = pulsarClient.newConsumer() + .topic(topicName) + .subscriptionName(subName) + .enableRetry(true) + .subscribe(); + fail(""); + } catch (Exception ex) { + log.info("got an expected error", ex); + assertTrue(ex.getMessage().contains("Not found:")); + assertTrue(ex.getMessage().contains(retryTopicName)); + } finally { + // cleanup. + if (consumer != null) { + consumer.close(); + } + admin.topics().delete(topicName); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerMLInitializeDelayTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerMLInitializeDelayTest.java new file mode 100644 index 0000000000000..7c7665a5bd3e4 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerMLInitializeDelayTest.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api; + +import com.carrotsearch.hppc.ObjectSet; +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.service.Dispatcher; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.common.naming.TopicName; +import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker-api") +public class SimpleProducerConsumerMLInitializeDelayTest extends ProducerConsumerBase { + + @BeforeClass(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + conf.setTopicLoadTimeoutSeconds(60 * 5); + } + + @Test(timeOut = 30 * 1000) + public void testConsumerListMatchesConsumerSet() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final String subName = "sub"; + final int clientOperationTimeout = 3; + final int loadMLDelayMillis = clientOperationTimeout * 3 * 1000; + final int clientMaxBackoffSeconds = clientOperationTimeout * 2; + admin.topics().createNonPartitionedTopic(topicName); + // Create a client with a low operation timeout. + PulsarClient client = PulsarClient.builder() + .serviceUrl(lookupUrl.toString()) + .operationTimeout(clientOperationTimeout, TimeUnit.SECONDS) + .maxBackoffInterval(clientMaxBackoffSeconds, TimeUnit.SECONDS) + .build(); + Consumer consumer = client.newConsumer() + .topic(topicName) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Shared) + .subscribe(); + // Inject a delay for the initialization of ML, to make the consumer to register twice. + // Consumer register twice: the first will be timeout, and try again. + AtomicInteger delayTimes = new AtomicInteger(); + mockZooKeeper.delay(loadMLDelayMillis, (op, s) -> { + if (op.toString().equals("GET") && s.contains(TopicName.get(topicName).getPersistenceNamingEncoding())) { + return delayTimes.incrementAndGet() == 1; + } + return false; + }); + admin.topics().unload(topicName); + // Verify: at last, "dispatcher.consumers.size" equals "dispatcher.consumerList.size". + Awaitility.await().atMost(Duration.ofSeconds(loadMLDelayMillis * 3)) + .ignoreExceptions().untilAsserted(() -> { + Dispatcher dispatcher = pulsar.getBrokerService() + .getTopic(topicName, false).join().get() + .getSubscription(subName).getDispatcher(); + ObjectSet consumerSet = WhiteboxImpl.getInternalState(dispatcher, "consumerSet"); + List consumerList = WhiteboxImpl.getInternalState(dispatcher, "consumerList"); + log.info("consumerSet_size: {}, consumerList_size: {}", consumerSet.size(), consumerList.size()); + Assert.assertEquals(consumerList.size(), 1); + Assert.assertEquals(consumerSet.size(), 1); + }); + + // Verify: the topic can be deleted. + consumer.close(); + admin.topics().delete(topicName); + // cleanup. + client.close(); + } + + @Test(timeOut = 30 * 1000) + public void testConcurrentlyOfPublishAndSwitchLedger() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final String subscription = "s1"; + admin.topics().createNonPartitionedTopic(topicName); + admin.topics().createSubscription(topicName, subscription, MessageId.earliest); + // Make ledger switches faster. + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).join().get(); + ManagedLedgerConfig config = persistentTopic.getManagedLedger().getConfig(); + config.setMaxEntriesPerLedger(2); + config.setMinimumRolloverTime(0, TimeUnit.MILLISECONDS); + // Inject a delay for switching ledgers, so publishing requests will be push in to the pending queue. + AtomicInteger delayTimes = new AtomicInteger(); + mockZooKeeper.delay(10, (op, s) -> { + if (op.toString().equals("SET") && s.contains(TopicName.get(topicName).getPersistenceNamingEncoding())) { + return delayTimes.incrementAndGet() == 1; + } + return false; + }); + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topicName).enableBatching(false) + .create(); + List> sendRequests = new ArrayList<>(); + List msgsSent = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + String msg = i + ""; + sendRequests.add(producer.sendAsync(i + "")); + msgsSent.add(msg); + } + // Verify: + // - All messages were sent. + // - The order of messages are correct. + Set msgIds = new LinkedHashSet<>(); + MessageIdImpl previousMsgId = null; + for (CompletableFuture msgId : sendRequests) { + Assert.assertNotNull(msgId.join()); + MessageIdImpl messageIdImpl = (MessageIdImpl) msgId.join(); + if (previousMsgId != null) { + Assert.assertTrue(messageIdImpl.compareTo(previousMsgId) > 0); + } + msgIds.add(String.format("%s:%s", messageIdImpl.getLedgerId(), messageIdImpl.getEntryId())); + previousMsgId = messageIdImpl; + } + Assert.assertEquals(msgIds.size(), 100); + log.info("messages were sent: {}", msgIds.toString()); + List msgsReceived = new ArrayList<>(); + Consumer consumer = pulsarClient.newConsumer(Schema.STRING).topic(topicName) + .subscriptionName(subscription).subscribe(); + while (true) { + Message receivedMsg = consumer.receive(2, TimeUnit.SECONDS); + if (receivedMsg == null) { + break; + } + msgsReceived.add(receivedMsg.getValue()); + } + Assert.assertEquals(msgsReceived, msgsSent); + + // cleanup. + consumer.close(); + producer.close(); + admin.topics().delete(topicName); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java index 89e6c684ee17f..5185a3b7e267c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java @@ -46,6 +46,7 @@ import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; @@ -495,9 +496,10 @@ public void testPartitionTopicStats() throws Exception { msg = consumer.receive(5, TimeUnit.SECONDS); String receivedMessage = new String(msg.getData()); log.info("Received message: [{}]", receivedMessage); - String expectedMessage = "my-message-" + i; - testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage); + Assert.assertTrue(messageSet.add(receivedMessage), "Received duplicate message " + receivedMessage); } + Assert.assertEquals(messageSet.size(), numMessages); + // Acknowledge the consumption of all messages at once consumer.acknowledgeCumulative(msg); @@ -545,20 +547,151 @@ public void testMsgRateExpired() throws Exception { admin.topics().expireMessages(topicName, subName, 1); pulsar.getBrokerService().updateRates(); - Awaitility.await().ignoreExceptions().timeout(5, TimeUnit.SECONDS) - .until(() -> admin.topics().getStats(topicName).getSubscriptions().get(subName).getMsgRateExpired() > 0.001); - - Thread.sleep(2000); - pulsar.getBrokerService().updateRates(); + Awaitility.await().ignoreExceptions().timeout(10, TimeUnit.SECONDS) + .until(() -> pulsar.getBrokerService().getTopicStats().get(topicName).getSubscriptions().get(subName).getTotalMsgExpired() > 0); - Awaitility.await().ignoreExceptions().timeout(5, TimeUnit.SECONDS) - .until(() -> admin.topics().getStats(topicName).getSubscriptions().get(subName).getMsgRateExpired() < 0.001); + Awaitility.await().ignoreExceptions().timeout(10, TimeUnit.SECONDS).until(() -> { + pulsar.getBrokerService().updateRates(); + return pulsar.getBrokerService().getTopicStats().get(topicName).getSubscriptions().get(subName).getMsgRateExpired() < 0.001; + }); - assertEquals(admin.topics().getStats(topicName).getSubscriptions().get(subName).getMsgRateExpired(), 0.0, - 0.001); - assertEquals(admin.topics().getStats(topicName).getSubscriptions().get(subName).getTotalMsgExpired(), + assertEquals(pulsar.getBrokerService().getTopicStats().get(topicName).getSubscriptions().get(subName).getMsgRateExpired(), + 0.0, 0.001); + assertEquals(pulsar.getBrokerService().getTopicStats().get(topicName).getSubscriptions().get(subName).getTotalMsgExpired(), numMessages); log.info("-- Exiting {} test --", methodName); } + + @Test + public void testRetryLetterAndDeadLetterStats() throws PulsarClientException, InterruptedException { + final String topicName = "persistent://my-property/my-ns/testRetryLetterAndDeadLetterStats"; + + Consumer consumer = pulsarClient.newConsumer() + .topic(topicName) + .subscriptionType(SubscriptionType.Shared) + .negativeAckRedeliveryDelay(100, TimeUnit.MILLISECONDS) + .enableRetry(true) + .deadLetterPolicy(DeadLetterPolicy.builder() + .maxRedeliverCount(3) + .retryLetterTopic("persistent://my-property/my-ns/retry-topic") + .deadLetterTopic("persistent://my-property/my-ns/dlq-topic") + .build()) + .subscriptionName("sub") + .subscribe(); + + Producer producer = pulsarClient.newProducer() + .topic(topicName) + .create(); + + final int messages = 1; + for (int i = 0; i < messages; i++) { + producer.send(("message-" + i).getBytes()); + } + + for (int i = 0; i < messages * 4; i++) { + // nack and reconsumeLater + Message msg = consumer.receive(1, TimeUnit.SECONDS); + if (msg != null) { + consumer.reconsumeLater(msg, 100, TimeUnit.MILLISECONDS); + } + } + + Awaitility.await().untilAsserted(() -> { + ConsumerStats stats = consumer.getStats(); + ProducerStats retryStats = stats.getRetryLetterProducerStats(); + ProducerStats deadLetterStats = stats.getDeadLetterProducerStats(); + assertNotNull(retryStats); + assertNotNull(deadLetterStats); + assertEquals(retryStats.getTotalMsgsSent(), 3); + assertEquals(deadLetterStats.getTotalMsgsSent(), 1); + }); + } + @Test + public void testDeadLetterStats() throws PulsarClientException, InterruptedException { + final String topicName = "persistent://my-property/my-ns/testDeadLetterStats"; + + Consumer consumer = pulsarClient.newConsumer() + .topic(topicName) + .subscriptionType(SubscriptionType.Shared) + .negativeAckRedeliveryDelay(100, TimeUnit.MILLISECONDS) + .deadLetterPolicy(DeadLetterPolicy.builder() + .maxRedeliverCount(1) + .deadLetterTopic("persistent://my-property/my-ns/dlq-topic") + .build()) + .subscriptionName("sub") + .subscribe(); + + Producer producer = pulsarClient.newProducer() + .topic(topicName) + .create(); + + final int messages = 1; + for (int i = 0; i < messages; i++) { + producer.send(("message-" + i).getBytes()); + } + + for (int i = 0; i < messages * 2; i++) { + // nack and reconsumeLater + Message msg = consumer.receive(1, TimeUnit.SECONDS); + if (msg != null) { + consumer.negativeAcknowledge(msg); + } + } + + Awaitility.await().untilAsserted(() -> { + ConsumerStats stats = consumer.getStats(); + ProducerStats dlqStats = stats.getDeadLetterProducerStats(); + assertNotNull(dlqStats); + assertEquals(dlqStats.getTotalMsgsSent(), 1); + }); + } + + @Test + public void testPartitionedRetryLetterAndDeadLetterStats() + throws PulsarClientException, InterruptedException, PulsarAdminException { + final String topicName = "persistent://my-property/my-ns/testPartitionedRetryLetterAndDeadLetterStats"; + + admin.topics().createPartitionedTopic(topicName, 10); + Consumer consumer = pulsarClient.newConsumer() + .topic(topicName) + .subscriptionType(SubscriptionType.Shared) + .negativeAckRedeliveryDelay(100, TimeUnit.MILLISECONDS) + .enableRetry(true) + .deadLetterPolicy(DeadLetterPolicy.builder() + .maxRedeliverCount(3) + .retryLetterTopic("persistent://my-property/my-ns/retry-topic") + .deadLetterTopic("persistent://my-property/my-ns/dlq-topic") + .build()) + .subscriptionName("sub") + .subscribe(); + + Producer producer = pulsarClient.newProducer() + .topic(topicName) + .messageRoutingMode(MessageRoutingMode.RoundRobinPartition) + .create(); + + final int messages = 30; + for (int i = 0; i < messages; i++) { + producer.send(("message-" + i).getBytes()); + } + + for (int i = 0; i < messages * 4; i++) { + // nack and reconsumeLater + Message msg = consumer.receive(1, TimeUnit.SECONDS); + if (msg != null) { + consumer.reconsumeLater(msg, 100, TimeUnit.MILLISECONDS); + } + } + + Awaitility.await().untilAsserted(() -> { + ConsumerStats stats = consumer.getStats(); + ProducerStats retryStats = stats.getRetryLetterProducerStats(); + ProducerStats deadLetterStats = stats.getDeadLetterProducerStats(); + assertNotNull(retryStats); + assertNotNull(deadLetterStats); + assertEquals(retryStats.getTotalMsgsSent(), 3 * messages); + assertEquals(deadLetterStats.getTotalMsgsSent(), messages); + }); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java index f3a00531eba56..2e71e8cc28c3e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java @@ -39,6 +39,8 @@ import com.google.common.collect.Sets; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelDuplexHandler; +import io.netty.channel.ChannelHandlerContext; import io.netty.util.Timeout; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -46,9 +48,11 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.time.Clock; +import java.time.Duration; import java.time.Instant; import java.time.ZoneId; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -67,6 +71,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -84,22 +89,27 @@ import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.PulsarVersion; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.service.ServerCnx; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.impl.BatchMessageIdImpl; import org.apache.pulsar.client.impl.ClientBuilderImpl; -import org.apache.pulsar.client.impl.ClientCnx; import org.apache.pulsar.client.impl.ConsumerBase; import org.apache.pulsar.client.impl.ConsumerImpl; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.client.impl.MultiTopicsConsumerImpl; import org.apache.pulsar.client.impl.PartitionedProducerImpl; +import org.apache.pulsar.client.impl.ProducerBase; +import org.apache.pulsar.client.impl.ProducerImpl; import org.apache.pulsar.client.impl.TopicMessageImpl; import org.apache.pulsar.client.impl.TypedMessageBuilderImpl; import org.apache.pulsar.client.impl.crypto.MessageCryptoBc; import org.apache.pulsar.client.impl.schema.writer.AvroWriter; +import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.api.EncryptionContext; import org.apache.pulsar.common.api.EncryptionContext.EncryptionKey; import org.apache.pulsar.common.api.proto.MessageMetadata; @@ -107,11 +117,14 @@ import org.apache.pulsar.common.compression.CompressionCodec; import org.apache.pulsar.common.compression.CompressionCodecProvider; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.CursorStats; +import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; import org.apache.pulsar.common.policies.data.PublisherStats; import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.tests.ThreadDumpUtil; import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -138,25 +151,31 @@ protected void setup() throws Exception { } @AfterMethod(alwaysRun = true) - public void rest() throws Exception { - pulsar.getConfiguration().setForceDeleteTenantAllowed(true); - pulsar.getConfiguration().setForceDeleteNamespaceAllowed(true); + public void cleanupAfterMethod() throws Exception { + try { + pulsar.getConfiguration().setForceDeleteTenantAllowed(true); + pulsar.getConfiguration().setForceDeleteNamespaceAllowed(true); - for (String tenant : admin.tenants().getTenants()) { - for (String namespace : admin.namespaces().getNamespaces(tenant)) { - deleteNamespaceWithRetry(namespace, true); + for (String tenant : admin.tenants().getTenants()) { + for (String namespace : admin.namespaces().getNamespaces(tenant)) { + deleteNamespaceWithRetry(namespace, true); + } + admin.tenants().deleteTenant(tenant, true); } - admin.tenants().deleteTenant(tenant, true); - } - - for (String cluster : admin.clusters().getClusters()) { - admin.clusters().deleteCluster(cluster); - } - pulsar.getConfiguration().setForceDeleteTenantAllowed(false); - pulsar.getConfiguration().setForceDeleteNamespaceAllowed(false); + for (String cluster : admin.clusters().getClusters()) { + admin.clusters().deleteCluster(cluster); + } - super.producerBaseSetup(); + pulsar.getConfiguration().setForceDeleteTenantAllowed(false); + pulsar.getConfiguration().setForceDeleteNamespaceAllowed(false); + super.producerBaseSetup(); + } catch (Exception | AssertionError e) { + log.warn("Failed to clean up state. Restarting broker.", e); + log.warn("Thread dump:\n{}", ThreadDumpUtil.buildThreadDiagnosticString()); + cleanup(); + setup(); + } } @DataProvider @@ -2156,7 +2175,7 @@ public void testEnabledChecksumClient() throws Exception { /** * It verifies that redelivery-of-specific messages: that redelivers all those messages even when consumer gets - * blocked due to unacked messsages + * blocked due to unacked messages * * Usecase: produce message with 10ms interval: so, consumer can consume only 10 messages without acking * @@ -2235,7 +2254,7 @@ public void testBlockUnackedConsumerRedeliverySpecificMessagesProduceWithPause() /** * It verifies that redelivery-of-specific messages: that redelivers all those messages even when consumer gets - * blocked due to unacked messsages + * blocked due to unacked messages * * Usecase: Consumer starts consuming only after all messages have been produced. So, consumer consumes total * receiver-queue-size number messages => ask for redelivery and receives all messages again. @@ -2696,12 +2715,17 @@ public EncryptionKeyInfo getPrivateKey(String keyName, Map keyMe Producer cryptoProducer = pulsarClient.newProducer() .topic(topicName).addEncryptionKey("client-ecdsa.pem") + .compressionType(CompressionType.LZ4) .cryptoKeyReader(new EncKeyReader()).create(); for (int i = 0; i < totalMsg; i++) { String message = "my-message-" + i; cryptoProducer.send(message.getBytes()); } + // admin api should be able to fetch compressed and encrypted message + List> msgs = admin.topics().peekMessages(topicName, "my-subscriber-name", 1); + assertNotNull(msgs); + Message msg; msg = normalConsumer.receive(RECEIVE_TIMEOUT_MEDIUM_MILLIS, TimeUnit.MILLISECONDS); @@ -3902,11 +3926,11 @@ public void testReleaseSemaphoreOnFailMessages() throws Exception { .topic("persistent://my-property/my-ns/my-topic2"); @Cleanup - Producer producer = producerBuilder.create(); + ProducerImpl producer = (ProducerImpl)producerBuilder.create(); List> futures = new ArrayList<>(); // Asynchronously produce messages - byte[] message = new byte[ClientCnx.getMaxMessageSize() + 1]; + byte[] message = new byte[producer.getConnectionHandler().getMaxMessageSize() + 1]; for (int i = 0; i < maxPendingMessages + 10; i++) { Future future = producer.sendAsync(message); try { @@ -4109,6 +4133,35 @@ public void testGetStats() throws Exception { consumer.close(); producer.close(); } + @Test(timeOut = 100000) + public void testMessageListenerGetStats() throws Exception { + final String topicName = "persistent://my-property/my-ns/testGetStats" + UUID.randomUUID(); + final String subName = "my-sub"; + final int receiveQueueSize = 100; + @Cleanup + PulsarClient client = newPulsarClient(lookupUrl.toString(), 100); + Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(false).topic(topicName).create(); + ConsumerImpl consumer = (ConsumerImpl) client.newConsumer(Schema.STRING) + .messageListener((MessageListener) (consumer1, msg) -> { + try { + TimeUnit.SECONDS.sleep(10); + } catch (InterruptedException igr) { + } + }) + .topic(topicName).receiverQueueSize(receiveQueueSize).subscriptionName(subName).subscribe(); + Assert.assertNull(consumer.getStats().getMsgNumInSubReceiverQueue()); + Assert.assertEquals(consumer.getStats().getMsgNumInReceiverQueue().intValue(), 0); + + for (int i = 0; i < receiveQueueSize; i++) { + producer.sendAsync("msg" + i); + } + //Give some time to consume + Awaitility.await() + .untilAsserted(() -> Assert.assertEquals(consumer.getStats().getMsgNumInReceiverQueue().intValue(), receiveQueueSize)); + consumer.close(); + producer.close(); + } @Test(timeOut = 100000) public void testGetStatsForPartitionedTopic() throws Exception { @@ -4306,37 +4359,43 @@ public void testAccessAvroSchemaMetadata(Schema schema) throws Exception producer.send(payload); producer.close(); - GenericRecord res = consumer.receive(RECEIVE_TIMEOUT_SECONDS, TimeUnit.SECONDS).getValue(); - consumer.close(); - assertEquals(schema.getSchemaInfo().getType(), res.getSchemaType()); - org.apache.avro.generic.GenericRecord nativeAvroRecord = null; - JsonNode nativeJsonRecord = null; - if (schema.getSchemaInfo().getType() == SchemaType.AVRO) { - nativeAvroRecord = (org.apache.avro.generic.GenericRecord) res.getNativeObject(); - assertNotNull(nativeAvroRecord); - } else { - nativeJsonRecord = (JsonNode) res.getNativeObject(); - assertNotNull(nativeJsonRecord); - } - for (org.apache.pulsar.client.api.schema.Field f : res.getFields()) { - log.info("field {} {}", f.getName(), res.getField(f)); - assertEquals("field", f.getName()); - assertEquals("aaaaaaaaaaaaaaaaaaaaaaaaa", res.getField(f)); - - if (nativeAvroRecord != null) { - // test that the native schema is accessible - org.apache.avro.Schema.Field fieldDetails = nativeAvroRecord.getSchema().getField(f.getName()); - // a nullable string is an UNION - assertEquals(org.apache.avro.Schema.Type.UNION, fieldDetails.schema().getType()); - assertTrue(fieldDetails.schema().getTypes().stream().anyMatch(s -> s.getType() == org.apache.avro.Schema.Type.STRING)); - assertTrue(fieldDetails.schema().getTypes().stream().anyMatch(s -> s.getType() == org.apache.avro.Schema.Type.NULL)); + try { + GenericRecord res = consumer.receive(RECEIVE_TIMEOUT_SECONDS, TimeUnit.SECONDS).getValue(); + consumer.close(); + assertEquals(schema.getSchemaInfo().getType(), res.getSchemaType()); + org.apache.avro.generic.GenericRecord nativeAvroRecord = null; + JsonNode nativeJsonRecord = null; + if (schema.getSchemaInfo().getType() == SchemaType.AVRO) { + nativeAvroRecord = (org.apache.avro.generic.GenericRecord) res.getNativeObject(); + assertNotNull(nativeAvroRecord); } else { - assertEquals(JsonNodeType.STRING, nativeJsonRecord.get("field").getNodeType()); + nativeJsonRecord = (JsonNode) res.getNativeObject(); + assertNotNull(nativeJsonRecord); + } + for (org.apache.pulsar.client.api.schema.Field f : res.getFields()) { + log.info("field {} {}", f.getName(), res.getField(f)); + assertEquals("field", f.getName()); + assertEquals("aaaaaaaaaaaaaaaaaaaaaaaaa", res.getField(f)); + + if (nativeAvroRecord != null) { + // test that the native schema is accessible + org.apache.avro.Schema.Field fieldDetails = nativeAvroRecord.getSchema().getField(f.getName()); + // a nullable string is an UNION + assertEquals(org.apache.avro.Schema.Type.UNION, fieldDetails.schema().getType()); + assertTrue(fieldDetails.schema().getTypes().stream().anyMatch(s -> s.getType() == org.apache.avro.Schema.Type.STRING)); + assertTrue(fieldDetails.schema().getTypes().stream().anyMatch(s -> s.getType() == org.apache.avro.Schema.Type.NULL)); + } else { + assertEquals(JsonNodeType.STRING, nativeJsonRecord.get("field").getNodeType()); + } } + assertEquals(1, res.getFields().size()); + } catch (Exception e) { + fail(); + } finally { + pulsarClient.shutdown(); + pulsarClient = newPulsarClient(lookupUrl.toString(), 0); + admin.schemas().deleteSchema(topic); } - assertEquals(1, res.getFields().size()); - - admin.schemas().deleteSchema(topic); } @Test(timeOut = 100000) @@ -4613,4 +4672,301 @@ public void testClientVersion() throws Exception { producer2.close(); client.close(); } + + @Test + public void testConsumeWhenDeliveryFailedByIOException() throws Exception { + final String topic = BrokerTestUtil.newUniqueName("persistent://my-property/my-ns/tp_"); + final String subscriptionName = "subscription1"; + final int messagesCount = 100; + final int receiverQueueSize = 1; + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).enableBatching(false).create(); + ConsumerImpl consumer = (ConsumerImpl) pulsarClient.newConsumer(Schema.STRING).topic(topic) + .subscriptionName(subscriptionName).receiverQueueSize(receiverQueueSize).subscribe(); + for (int i = 0; i < messagesCount; i++) { + producer.send(i + ""); + } + // Wait incoming queue of the consumer is full. + Awaitility.await().untilAsserted(() -> { + assertEquals(consumer.getIncomingMessageSize(), receiverQueueSize); + }); + + // Mock an io error for sending messages out. + ServerCnx serverCnx = (ServerCnx) pulsar.getBrokerService().getTopic(topic, false).join().get() + .getSubscription(subscriptionName).getDispatcher().getConsumers().iterator().next().cnx(); + serverCnx.ctx().channel().pipeline().addFirst(new ChannelDuplexHandler() { + + @Override + public void flush(ChannelHandlerContext ctx) throws Exception { + throw new IOException("Mocked error"); + } + }); + + // Verify all messages will be consumed. + Set receivedMessages = new HashSet<>(); + while (true) { + Message msg = consumer.receive(2, TimeUnit.SECONDS); + if (msg != null) { + receivedMessages.add(msg.getValue()); + consumer.acknowledge(msg); + } else { + break; + } + } + Assert.assertEquals(receivedMessages.size(), messagesCount); + + producer.close(); + consumer.close(); + admin.topics().delete(topic, false); + } + + @DataProvider(name = "enableBatchSend") + public Object[][] enableBatchSend() { + return new Object[][]{ + {true}, + {false} + }; + } + + @Test(dataProvider = "enableBatchSend") + public void testPublishWithCreateMessageManually(boolean enableBatchSend) throws Exception { + final int messageCount = 10; + final List messageArrayBeforeSend = Collections.synchronizedList(new ArrayList<>()); + final List messageArrayOnSendAcknowledgement = Collections.synchronizedList(new ArrayList<>()); + // Create an interceptor to verify the ref count of Message.payload is as expected. + AtomicBoolean payloadWasReleasedWhenIntercept = new AtomicBoolean(false); + ProducerInterceptor interceptor = new ProducerInterceptor(){ + + @Override + public void close() { + + } + @Override + public Message beforeSend(Producer producer, Message message) { + MessageImpl msgImpl = (MessageImpl) message; + log.info("payload.refCnf before send: {}", msgImpl.getDataBuffer().refCnt()); + if (msgImpl.getDataBuffer().refCnt() < 1) { + payloadWasReleasedWhenIntercept.set(true); + } + messageArrayBeforeSend.add(msgImpl); + return message; + } + + @Override + public void onSendAcknowledgement(Producer producer, Message message, MessageId msgId, + Throwable exception) { + MessageImpl msgImpl = (MessageImpl) message; + log.info("payload.refCnf on send acknowledgement: {}", msgImpl.getDataBuffer().refCnt()); + if (msgImpl.getDataBuffer().refCnt() < 1) { + payloadWasReleasedWhenIntercept.set(true); + } + messageArrayOnSendAcknowledgement.add(msgImpl); + } + }; + + final String topic = BrokerTestUtil.newUniqueName("persistent://my-property/my-ns/tp"); + admin.topics().createNonPartitionedTopic(topic); + ProducerBase producerBase = (ProducerBase) pulsarClient.newProducer().topic(topic).intercept(interceptor) + .enableBatching(enableBatchSend).create(); + + // Publish message. + // Note: "ProducerBase.sendAsync" is not equals to "Producer.sendAsync". + final MessageImpl[] messageArraySent = new MessageImpl[messageCount]; + final ByteBuf[] payloads = new ByteBuf[messageCount]; + List> sendFutureList = new ArrayList<>(); + List releaseFutureList = new ArrayList<>(); + for (int i = 0; i < messageCount; i++) { + // Create message payload, refCnf = 1 now. + ByteBuf payload = PulsarByteBufAllocator.DEFAULT.heapBuffer(1); + payloads[i] = payload; + log.info("payload_{}.refCnf 1st: {}", i, payload.refCnt()); + payload.writeByte(i); + // refCnf = 2 now. + payload.retain(); + log.info("payload_{}.refCnf 2nd: {}", i, payload.refCnt()); + MessageMetadata messageMetadata = new MessageMetadata(); + messageMetadata.setUncompressedSize(1); + MessageImpl message1 = MessageImpl.create(topic, null, messageMetadata, payload, Optional.empty(), + null, Schema.BYTES, 0, true, 0); + messageArraySent[i] = message1; + // Release ByteBuf the first time, refCnf = 1 now. + CompletableFuture future = producerBase.sendAsync(message1); + sendFutureList.add(future); + final int indexForLog = i; + future.whenComplete((v, ex) -> { + message1.release(); + log.info("payload_{}.refCnf 3rd after_complete_refCnf: {}, ex: {}", indexForLog, payload.refCnt(), + ex == null ? "null" : ex.getMessage()); + }); + } + sendFutureList.get(messageCount - 1).join(); + + // Left 2 seconds to wait the code in the finally-block, which is using to avoid this test to be flaky. + Thread.sleep(1000 * 2); + + // Verify: payload's refCnf. + for (int i = 0; i < messageCount; i++) { + log.info("payload_{}.refCnf 4th: {}", i, payloads[i].refCnt()); + assertEquals(payloads[i].refCnt(), 1); + } + + // Verify: the messages has not been released when calling interceptor. + assertFalse(payloadWasReleasedWhenIntercept.get()); + + // Verify: the order of send complete event. + MessageIdImpl messageIdPreviousOne = null; + for (int i = 0; i < messageCount; i++) { + MessageIdImpl messageId = (MessageIdImpl) sendFutureList.get(i).get(); + if (messageIdPreviousOne != null) { + assertTrue(compareMessageIds(messageIdPreviousOne, messageId) > 0); + } + messageIdPreviousOne = messageId; + } + + // Verify: the order of interceptor events. + for (int i = 0; i < messageCount; i++) { + assertTrue(messageArraySent[i] == messageArrayBeforeSend.get(i)); + assertTrue(messageArraySent[i] == messageArrayOnSendAcknowledgement.get(i)); + } + + // cleanup. + for (int i = 0; i < messageCount; i++) { + payloads[i].release(); + } + producerBase.close(); + admin.topics().delete(topic, false); + } + + /** + * It verifies that consumer receives configured number of messages into the batch. + * @throws Exception + */ + @Test + public void testBatchReceiveWithMaxBatchSize() throws Exception { + int maxBatchSize = 100; + final int internalQueueSize = 10; + final int maxBytes = 2000000; + final int timeOutInSeconds = 900; + final String topic = "persistent://my-property/my-ns/testBatchReceive"; + BatchReceivePolicy batchReceivePolicy = BatchReceivePolicy.builder().maxNumBytes(maxBytes) + .maxNumMessages(maxBatchSize).timeout(timeOutInSeconds, TimeUnit.SECONDS).build(); + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.STRING).topic(topic) + .subscriptionName("my-subscriber-name") + .receiverQueueSize(internalQueueSize) + .batchReceivePolicy(batchReceivePolicy).subscribe(); + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topic).enableBatching(false).create(); + + final int numMessages = 100; + for (int i = 0; i < numMessages; i++) { + producer.newMessage().value(("value-" + i).getBytes(UTF_8)).eventTime((i + 1) * 100L).send(); + } + + assertEquals(consumer.batchReceive().size(), maxBatchSize); + } + + /** + * + * This test validates that consumer correctly sends permits for batch message that should be discarded. + * @throws Exception + */ + @Test + public void testEncryptionFailureWithBatchPublish() throws Exception { + log.info("-- Starting {} test --", methodName); + String topicName = "persistent://my-property/my-ns/batchFailureTest-" + System.currentTimeMillis(); + + class EncKeyReader implements CryptoKeyReader { + + final EncryptionKeyInfo keyInfo = new EncryptionKeyInfo(); + + @Override + public EncryptionKeyInfo getPublicKey(String keyName, Map keyMeta) { + String CERT_FILE_PATH = "./src/test/resources/certificate/public-key." + keyName; + if (Files.isReadable(Paths.get(CERT_FILE_PATH))) { + try { + keyInfo.setKey(Files.readAllBytes(Paths.get(CERT_FILE_PATH))); + return keyInfo; + } catch (IOException e) { + Assert.fail("Failed to read certificate from " + CERT_FILE_PATH); + } + } else { + Assert.fail("Certificate file " + CERT_FILE_PATH + " is not present or not readable."); + } + return null; + } + + @Override + public EncryptionKeyInfo getPrivateKey(String keyName, Map keyMeta) { + String CERT_FILE_PATH = "./src/test/resources/certificate/private-key." + keyName; + if (Files.isReadable(Paths.get(CERT_FILE_PATH))) { + try { + keyInfo.setKey(Files.readAllBytes(Paths.get(CERT_FILE_PATH))); + return keyInfo; + } catch (IOException e) { + Assert.fail("Failed to read certificate from " + CERT_FILE_PATH); + } + } else { + Assert.fail("Certificate file " + CERT_FILE_PATH + " is not present or not readable."); + } + return null; + } + } + + final int totalMsg = 2000; + + String subName = "without-cryptoreader"; + @Cleanup + Consumer normalConsumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName) + .messageListener((c, msg) -> { + log.info("Failed to consume message {}", msg.getMessageId()); + c.acknowledgeAsync(msg); + }).cryptoFailureAction(ConsumerCryptoFailureAction.DISCARD).ackTimeout(1, TimeUnit.SECONDS) + .receiverQueueSize(totalMsg / 20).subscribe(); + + @Cleanup + Producer cryptoProducer = pulsarClient.newProducer().topic(topicName) + .addEncryptionKey("client-ecdsa.pem").enableBatching(true).batchingMaxMessages(5) + .batchingMaxPublishDelay(1, TimeUnit.SECONDS).cryptoKeyReader(new EncKeyReader()).create(); + for (int i = 0; i < totalMsg; i++) { + String message = "my-message-" + i; + cryptoProducer.sendAsync(message.getBytes()); + } + cryptoProducer.flush(); + + Awaitility.await().atMost(Duration.ofSeconds(10)).untilAsserted(() -> { + PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(topicName); + CursorStats stats = internalStats.cursors.get(subName); + String readPosition = stats.readPosition; + assertEquals(getMessageId(readPosition, 0, 1), (getMessageId(internalStats.lastConfirmedEntry, 0, 0))); + }); + + log.info("-- Exiting {} test --", methodName); + } + + private MessageId getMessageId(String messageId, long subLedgerId, long subEntryId) { + String[] ids = messageId.split(":"); + return new MessageIdImpl(Long.parseLong(ids[0]) - subLedgerId, Long.parseLong(ids[1]) - subEntryId, -1); + } + + private int compareMessageIds(MessageIdImpl messageId1, MessageIdImpl messageId2) { + if (messageId2.getLedgerId() < messageId1.getLedgerId()) { + return -1; + } + if (messageId2.getLedgerId() > messageId1.getLedgerId()) { + return 1; + } + if (messageId2.getEntryId() < messageId1.getEntryId()) { + return -1; + } + if (messageId2.getEntryId() > messageId1.getEntryId()) { + return 1; + } + if (messageId2 instanceof BatchMessageIdImpl && messageId1 instanceof BatchMessageIdImpl) { + BatchMessageIdImpl batchMessageId1 = (BatchMessageIdImpl) messageId1; + BatchMessageIdImpl batchMessageId2 = (BatchMessageIdImpl) messageId2; + return batchMessageId2.getBatchIndex() - batchMessageId1.getBatchIndex(); + } else { + return 0; + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java index c8c7c3b2ccc38..e006b72fad279 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java @@ -41,6 +41,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema.Parser; import org.apache.avro.reflect.ReflectData; +import org.apache.pulsar.TestNGInstanceOrder; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.PulsarClientException.IncompatibleSchemaException; import org.apache.pulsar.client.api.PulsarClientException.InvalidMessageException; @@ -66,10 +67,12 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Factory; +import org.testng.annotations.Listeners; import org.testng.annotations.Test; @Test(groups = "broker-api") @Slf4j +@Listeners({ TestNGInstanceOrder.class }) public class SimpleSchemaTest extends ProducerConsumerBase { private static final String NAMESPACE = "my-property/my-ns"; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SubscriptionMessageDispatchThrottlingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SubscriptionMessageDispatchThrottlingTest.java index 9036d82d84f01..02de11a2bcc95 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SubscriptionMessageDispatchThrottlingTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SubscriptionMessageDispatchThrottlingTest.java @@ -20,7 +20,7 @@ import static org.awaitility.Awaitility.await; import com.google.common.collect.Sets; -import java.time.Duration; +import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; import org.apache.pulsar.broker.BrokerTestUtil; @@ -47,7 +47,7 @@ public class SubscriptionMessageDispatchThrottlingTest extends MessageDispatchTh * @param subscription * @throws Exception */ - @Test(dataProvider = "subscriptionAndDispatchRateType", timeOut = 5000) + @Test(dataProvider = "subscriptionAndDispatchRateType", timeOut = 30000) public void testMessageRateLimitingNotReceiveAllMessages(SubscriptionType subscription, DispatchRateType dispatchRateType) throws Exception { log.info("-- Starting {} test --", methodName); @@ -143,7 +143,7 @@ public void testMessageRateLimitingNotReceiveAllMessages(SubscriptionType subscr * @param subscription * @throws Exception */ - @Test(dataProvider = "subscriptions", timeOut = 5000) + @Test(dataProvider = "subscriptions", timeOut = 30000) public void testMessageRateLimitingReceiveAllMessagesAfterThrottling(SubscriptionType subscription) throws Exception { log.info("-- Starting {} test --", methodName); @@ -217,7 +217,7 @@ public void testMessageRateLimitingReceiveAllMessagesAfterThrottling(Subscriptio log.info("-- Exiting {} test --", methodName); } - @Test(dataProvider = "subscriptions", timeOut = 30000, invocationCount = 15) + @Test(dataProvider = "subscriptions", timeOut = 30000) private void testMessageNotDuplicated(SubscriptionType subscription) throws Exception { int brokerRate = 1000; int topicRate = 5000; @@ -272,7 +272,7 @@ private void testMessageNotDuplicated(SubscriptionType subscription) throws Exce Assert.fail("Should only have PersistentDispatcher in this test"); } final DispatchRateLimiter subDispatchRateLimiter = subRateLimiter; - Awaitility.await().atMost(Duration.ofMillis(500)).untilAsserted(() -> { + Awaitility.await().untilAsserted(() -> { DispatchRateLimiter brokerDispatchRateLimiter = pulsar.getBrokerService().getBrokerDispatchRateLimiter(); Assert.assertTrue(brokerDispatchRateLimiter != null && brokerDispatchRateLimiter.getDispatchRateOnByte() > 0); @@ -319,7 +319,7 @@ private void testMessageNotDuplicated(SubscriptionType subscription) throws Exce * @param subscription * @throws Exception */ - @Test(dataProvider = "subscriptions", timeOut = 5000) + @Test(dataProvider = "subscriptions", timeOut = 30000) public void testBytesRateLimitingReceiveAllMessagesAfterThrottling(SubscriptionType subscription) throws Exception { log.info("-- Starting {} test --", methodName); @@ -450,7 +450,7 @@ private void testDispatchRate(SubscriptionType subscription, Assert.fail("Should only have PersistentDispatcher in this test"); } final DispatchRateLimiter subDispatchRateLimiter = subRateLimiter; - Awaitility.await().atMost(Duration.ofMillis(500)).untilAsserted(() -> { + Awaitility.await().untilAsserted(() -> { DispatchRateLimiter brokerDispatchRateLimiter = pulsar.getBrokerService().getBrokerDispatchRateLimiter(); Assert.assertTrue(brokerDispatchRateLimiter != null && brokerDispatchRateLimiter.getDispatchRateOnByte() > 0); @@ -525,7 +525,7 @@ public void testMultiLevelDispatch(SubscriptionType subscription) throws Excepti * @param subscription * @throws Exception */ - @Test(dataProvider = "subscriptions", timeOut = 8000) + @Test(dataProvider = "subscriptions", timeOut = 30000) public void testBrokerBytesRateLimitingReceiveAllMessagesAfterThrottling(SubscriptionType subscription) throws Exception { log.info("-- Starting {} test --", methodName); @@ -538,6 +538,11 @@ public void testBrokerBytesRateLimitingReceiveAllMessagesAfterThrottling(Subscri long initBytes = pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInByte(); final int byteRate = 1000; admin.brokers().updateDynamicConfiguration("dispatchThrottlingRateInByte", "" + byteRate); + + Awaitility.await().untilAsserted(() -> { + Assert.assertEquals(pulsar.getConfiguration().getDispatchThrottlingRateInByte(), byteRate); + }); + admin.namespaces().createNamespace(namespace1, Sets.newHashSet("test")); admin.namespaces().createNamespace(namespace2, Sets.newHashSet("test")); @@ -571,7 +576,7 @@ public void testBrokerBytesRateLimitingReceiveAllMessagesAfterThrottling(Subscri Producer producer1 = pulsarClient.newProducer().topic(topicName1).create(); Producer producer2 = pulsarClient.newProducer().topic(topicName2).create(); - Awaitility.await().atMost(Duration.ofMillis(500)).untilAsserted(() -> { + Awaitility.await().untilAsserted(() -> { DispatchRateLimiter rateLimiter = pulsar.getBrokerService().getBrokerDispatchRateLimiter(); Assert.assertTrue(rateLimiter != null && rateLimiter.getDispatchRateOnByte() > 0); @@ -604,7 +609,7 @@ public void testBrokerBytesRateLimitingReceiveAllMessagesAfterThrottling(Subscri * * @throws Exception */ - @Test(timeOut = 5000) + @Test(timeOut = 30000) public void testRateLimitingMultipleConsumers() throws Exception { log.info("-- Starting {} test --", methodName); @@ -690,7 +695,7 @@ public void testRateLimitingMultipleConsumers() throws Exception { } - @Test(dataProvider = "subscriptions", timeOut = 5000) + @Test(dataProvider = "subscriptions", timeOut = 30000) public void testClusterRateLimitingConfiguration(SubscriptionType subscription) throws Exception { log.info("-- Starting {} test --", methodName); @@ -867,7 +872,7 @@ public void testClusterPolicyOverrideConfiguration() throws Exception { log.info("-- Exiting {} test --", methodName); } - @Test(dataProvider = "subscriptions", timeOut = 11000) + @Test(dataProvider = "subscriptions", timeOut = 30000) public void testClosingRateLimiter(SubscriptionType subscription) throws Exception { log.info("-- Starting {} test --", methodName); @@ -908,7 +913,7 @@ public void testClosingRateLimiter(SubscriptionType subscription) throws Excepti producer.close(); consumer.close(); - sub.disconnect().get(); + sub.close(true, Optional.empty()).get(); // Make sure that the rate limiter is closed Assert.assertEquals(dispatchRateLimiter.getDispatchRateOnMsg(), -1); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SubscriptionPauseOnAckStatPersistTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SubscriptionPauseOnAckStatPersistTest.java new file mode 100644 index 0000000000000..36c36735c067e --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SubscriptionPauseOnAckStatPersistTest.java @@ -0,0 +1,602 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.ManagedCursorContainer; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.service.Dispatcher; +import org.apache.pulsar.broker.service.SystemTopicBasedTopicPoliciesService; +import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; +import org.apache.pulsar.broker.service.persistent.PersistentDispatcherSingleActiveConsumer; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.admin.GetStatsOptions; +import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.HierarchyTopicPolicies; +import org.apache.pulsar.common.policies.data.TopicPolicies; +import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker-api") +public class SubscriptionPauseOnAckStatPersistTest extends ProducerConsumerBase { + + private static final int MAX_UNACKED_RANGES_TO_PERSIST = 50; + + @BeforeClass(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + protected void doInitConf() throws Exception { + conf.setManagedLedgerMaxUnackedRangesToPersist(MAX_UNACKED_RANGES_TO_PERSIST); + } + + private void enablePolicyDispatcherPauseOnAckStatePersistent(String tpName) { + TopicPolicies policies = new TopicPolicies(); + policies.setDispatcherPauseOnAckStatePersistentEnabled(true); + policies.setIsGlobal(false); + SystemTopicBasedTopicPoliciesService policiesService = + (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + Map policiesCache = + WhiteboxImpl.getInternalState(policiesService, "policiesCache"); + policiesCache.put(TopicName.get(tpName), policies); + } + + private void cancelPendingRead(String tpName, String cursorName) throws Exception { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(tpName, false).join().get(); + Dispatcher dispatcher = persistentTopic.getSubscription(cursorName).getDispatcher(); + if (dispatcher instanceof PersistentDispatcherMultipleConsumers) { + Method cancelPendingRead = PersistentDispatcherMultipleConsumers.class + .getDeclaredMethod("cancelPendingRead", new Class[]{}); + cancelPendingRead.setAccessible(true); + cancelPendingRead.invoke(dispatcher, new Object[]{}); + } else if (dispatcher instanceof PersistentDispatcherSingleActiveConsumer) { + Method cancelPendingRead = PersistentDispatcherSingleActiveConsumer.class + .getDeclaredMethod("cancelPendingRead", new Class[]{}); + cancelPendingRead.setAccessible(true); + cancelPendingRead.invoke(dispatcher, new Object[]{}); + } + } + + private void triggerNewReadMoreEntries(String tpName, String cursorName) throws Exception { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(tpName, false).join().get(); + Dispatcher dispatcher = persistentTopic.getSubscription(cursorName).getDispatcher(); + if (dispatcher instanceof PersistentDispatcherMultipleConsumers) { + ((PersistentDispatcherMultipleConsumers) dispatcher).readMoreEntries(); + } else if (dispatcher instanceof PersistentDispatcherSingleActiveConsumer) { + PersistentDispatcherSingleActiveConsumer persistentDispatcherSingleActiveConsumer = + ((PersistentDispatcherSingleActiveConsumer) dispatcher); + Method readMoreEntries = PersistentDispatcherSingleActiveConsumer.class.getDeclaredMethod( + "readMoreEntries", new Class[]{org.apache.pulsar.broker.service.Consumer.class}); + readMoreEntries.setAccessible(true); + readMoreEntries.invoke(dispatcher, + new Object[]{persistentDispatcherSingleActiveConsumer.getActiveConsumer()}); + } + } + + @DataProvider(name = "multiConsumerSubscriptionTypes") + private Object[][] multiConsumerSubscriptionTypes() { + return new Object[][]{ + {SubscriptionType.Key_Shared}, + {SubscriptionType.Shared} + }; + } + + @DataProvider(name = "singleConsumerSubscriptionTypes") + private Object[][] singleConsumerSubscriptionTypes() { + return new Object[][]{ + {SubscriptionType.Failover}, + {SubscriptionType.Exclusive} + }; + } + + @DataProvider(name = "skipTypes") + private Object[][] skipTypes() { + return new Object[][]{ + {SkipType.SKIP_ENTRIES}, + {SkipType.CLEAR_BACKLOG}, + {SkipType.SEEK}, + {SkipType.RESET_CURSOR} + }; + } + + private enum SkipType{ + SKIP_ENTRIES, + CLEAR_BACKLOG, + SEEK, + RESET_CURSOR; + } + + private ReceivedMessages ackOddMessagesOnly(Consumer...consumers) throws Exception { + return receiveAndAckMessages((msgId, msgV) -> Integer.valueOf(msgV) % 2 == 1, consumers); + } + + @DataProvider(name = "typesOfSetDispatcherPauseOnAckStatePersistent") + public Object[][] typesOfSetDispatcherPauseOnAckStatePersistent() { + return new Object[][]{ + {TypeOfUpdateTopicConfig.BROKER_CONF}, + {TypeOfUpdateTopicConfig.NAMESPACE_LEVEL_POLICY}, + {TypeOfUpdateTopicConfig.TOPIC_LEVEL_POLICY} + }; + } + + public enum TypeOfUpdateTopicConfig { + BROKER_CONF, + NAMESPACE_LEVEL_POLICY, + TOPIC_LEVEL_POLICY; + } + + private void enableDispatcherPauseOnAckStatePersistentAndCreateTopic(String tpName, TypeOfUpdateTopicConfig type) + throws Exception { + if (type == TypeOfUpdateTopicConfig.BROKER_CONF) { + admin.brokers().updateDynamicConfiguration("dispatcherPauseOnAckStatePersistentEnabled", "true"); + admin.topics().createNonPartitionedTopic(tpName); + } else if (type == TypeOfUpdateTopicConfig.TOPIC_LEVEL_POLICY) { + admin.topics().createNonPartitionedTopic(tpName); + admin.topicPolicies().setDispatcherPauseOnAckStatePersistent(tpName).join(); + } else if (type == TypeOfUpdateTopicConfig.NAMESPACE_LEVEL_POLICY) { + admin.topics().createNonPartitionedTopic(tpName); + admin.namespaces().setDispatcherPauseOnAckStatePersistent(TopicName.get(tpName).getNamespace()); + } + Awaitility.await().untilAsserted(() -> { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(tpName, false).join().get(); + HierarchyTopicPolicies policies = WhiteboxImpl.getInternalState(persistentTopic, "topicPolicies"); + Assert.assertTrue(persistentTopic.isDispatcherPauseOnAckStatePersistentEnabled()); + if (type == TypeOfUpdateTopicConfig.BROKER_CONF) { + Assert.assertTrue(pulsar.getConfig().isDispatcherPauseOnAckStatePersistentEnabled()); + } else if (type == TypeOfUpdateTopicConfig.TOPIC_LEVEL_POLICY){ + Assert.assertTrue(policies.getDispatcherPauseOnAckStatePersistentEnabled().getTopicValue()); + Assert.assertTrue(admin.topicPolicies().getDispatcherPauseOnAckStatePersistent(tpName, false).join()); + } + }); + } + + private void disableDispatcherPauseOnAckStatePersistent(String tpName, TypeOfUpdateTopicConfig type) + throws Exception { + if (type == TypeOfUpdateTopicConfig.BROKER_CONF) { + admin.brokers().updateDynamicConfiguration("dispatcherPauseOnAckStatePersistentEnabled", "false"); + } else if (type == TypeOfUpdateTopicConfig.TOPIC_LEVEL_POLICY) { + admin.topicPolicies().removeDispatcherPauseOnAckStatePersistent(tpName).join(); + } else if (type == TypeOfUpdateTopicConfig.NAMESPACE_LEVEL_POLICY) { + admin.namespaces().removeDispatcherPauseOnAckStatePersistent(TopicName.get(tpName).getNamespace()); + } + Awaitility.await().untilAsserted(() -> { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(tpName, false).join().get(); + HierarchyTopicPolicies policies = WhiteboxImpl.getInternalState(persistentTopic, "topicPolicies"); + Assert.assertFalse(persistentTopic.isDispatcherPauseOnAckStatePersistentEnabled()); + if (type == TypeOfUpdateTopicConfig.BROKER_CONF) { + Assert.assertFalse(pulsar.getConfig().isDispatcherPauseOnAckStatePersistentEnabled()); + } else if (type == TypeOfUpdateTopicConfig.TOPIC_LEVEL_POLICY){ + Assert.assertFalse(policies.getDispatcherPauseOnAckStatePersistentEnabled().getTopicValue()); + Assert.assertFalse(admin.topicPolicies().getDispatcherPauseOnAckStatePersistent(tpName, false).join()); + } + }); + } + + @Test(dataProvider = "typesOfSetDispatcherPauseOnAckStatePersistent") + public void testBrokerDynamicConfig(TypeOfUpdateTopicConfig type) throws Exception { + final String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final String subscription = "s1"; + final int msgSendCount = MAX_UNACKED_RANGES_TO_PERSIST * 4; + final int incomingQueueSize = MAX_UNACKED_RANGES_TO_PERSIST * 10; + + // Enable "dispatcherPauseOnAckStatePersistentEnabled". + enableDispatcherPauseOnAckStatePersistentAndCreateTopic(tpName, type); + admin.topics().createSubscription(tpName, subscription, MessageId.earliest); + + // Send double MAX_UNACKED_RANGES_TO_PERSIST messages. + Producer p1 = pulsarClient.newProducer(Schema.STRING).topic(tpName).enableBatching(false).create(); + ArrayList messageIdsSent = new ArrayList<>(); + for (int i = 0; i < msgSendCount; i++) { + MessageIdImpl messageId = (MessageIdImpl) p1.send(Integer.valueOf(i).toString()); + messageIdsSent.add(messageId); + } + // Make ack holes. + Consumer c1 = pulsarClient.newConsumer(Schema.STRING).topic(tpName).subscriptionName(subscription) + .receiverQueueSize(incomingQueueSize).isAckReceiptEnabled(true) + .subscriptionType(SubscriptionType.Shared).subscribe(); + ackOddMessagesOnly(c1); + verifyAckHolesIsMuchThanLimit(tpName, subscription); + + cancelPendingRead(tpName, subscription); + triggerNewReadMoreEntries(tpName, subscription); + + // Verify: the dispatcher has been paused. + final String specifiedMessage = "9876543210"; + p1.send(specifiedMessage); + Message msg1 = c1.receive(2, TimeUnit.SECONDS); + Assert.assertNull(msg1, msg1 == null ? "null" : msg1.getValue()); + + // Disable "dispatcherPauseOnAckStatePersistentEnabled". + disableDispatcherPauseOnAckStatePersistent(tpName, type); + + // Verify the new message can be received. + Message msg2 = c1.receive(2, TimeUnit.SECONDS); + Assert.assertNotNull(msg2); + Assert.assertEquals(msg2.getValue(), specifiedMessage); + // cleanup. + p1.close(); + c1.close(); + admin.topics().delete(tpName, false); + } + + private void verifyAckHolesIsMuchThanLimit(String tpName, String subscription) { + Awaitility.await().untilAsserted(() -> { + Assert.assertTrue(MAX_UNACKED_RANGES_TO_PERSIST < admin.topics() + .getInternalStats(tpName).cursors.get(subscription).totalNonContiguousDeletedMessagesRange); + }); + } + + @Test(dataProvider = "multiConsumerSubscriptionTypes") + public void testPauseOnAckStatPersist(SubscriptionType subscriptionType) throws Exception { + final String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final String subscription = "s1"; + final int msgSendCount = MAX_UNACKED_RANGES_TO_PERSIST * 4; + final int incomingQueueSize = MAX_UNACKED_RANGES_TO_PERSIST * 10; + + enablePolicyDispatcherPauseOnAckStatePersistent(tpName); + admin.topics().createNonPartitionedTopic(tpName); + admin.topics().createSubscription(tpName, subscription, MessageId.earliest); + + // Send double MAX_UNACKED_RANGES_TO_PERSIST messages. + Producer p1 = pulsarClient.newProducer(Schema.STRING).topic(tpName).enableBatching(false).create(); + ArrayList messageIdsSent = new ArrayList<>(); + for (int i = 0; i < msgSendCount; i++) { + MessageIdImpl messageId = (MessageIdImpl) p1.send(Integer.valueOf(i).toString()); + messageIdsSent.add(messageId); + } + // Make ack holes. + Consumer c1 = pulsarClient.newConsumer(Schema.STRING).topic(tpName).subscriptionName(subscription) + .receiverQueueSize(incomingQueueSize).isAckReceiptEnabled(true).subscriptionType(subscriptionType) + .subscribe(); + ackOddMessagesOnly(c1); + verifyAckHolesIsMuchThanLimit(tpName, subscription); + + cancelPendingRead(tpName, subscription); + triggerNewReadMoreEntries(tpName, subscription); + + // Verify: the dispatcher has been paused. + final String specifiedMessage = "9876543210"; + p1.send(specifiedMessage); + Message msg1 = c1.receive(2, TimeUnit.SECONDS); + Assert.assertNull(msg1); + + // Verify: after ack messages, will unpause the dispatcher. + c1.acknowledge(messageIdsSent); + ReceivedMessages receivedMessagesAfterPause = ackAllMessages(c1); + Assert.assertTrue(receivedMessagesAfterPause.hasReceivedMessage(specifiedMessage)); + Assert.assertTrue(receivedMessagesAfterPause.hasAckedMessage(specifiedMessage)); + + // cleanup. + p1.close(); + c1.close(); + admin.topics().delete(tpName, false); + } + + @Test(dataProvider = "skipTypes") + public void testUnPauseOnSkipEntries(SkipType skipType) throws Exception { + final String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final String subscription = "s1"; + final int msgSendCount = MAX_UNACKED_RANGES_TO_PERSIST * 4; + final int incomingQueueSize = MAX_UNACKED_RANGES_TO_PERSIST * 10; + + enablePolicyDispatcherPauseOnAckStatePersistent(tpName); + admin.topics().createNonPartitionedTopic(tpName); + admin.topics().createSubscription(tpName, subscription, MessageId.earliest); + + // Send double MAX_UNACKED_RANGES_TO_PERSIST messages. + Producer p1 = pulsarClient.newProducer(Schema.STRING).topic(tpName).enableBatching(false).create(); + ArrayList messageIdsSent = new ArrayList<>(); + for (int i = 0; i < msgSendCount; i++) { + MessageIdImpl messageId = (MessageIdImpl) p1.send(Integer.valueOf(i).toString()); + messageIdsSent.add(messageId); + } + // Make ack holes. + Consumer c1 = pulsarClient.newConsumer(Schema.STRING).topic(tpName).subscriptionName(subscription) + .receiverQueueSize(incomingQueueSize).isAckReceiptEnabled(true) + .subscriptionType(SubscriptionType.Shared).subscribe(); + ackOddMessagesOnly(c1); + verifyAckHolesIsMuchThanLimit(tpName, subscription); + + cancelPendingRead(tpName, subscription); + triggerNewReadMoreEntries(tpName, subscription); + + // Verify: the dispatcher has been paused. + final String specifiedMessage1 = "9876543210"; + p1.send(specifiedMessage1); + Message msg1 = c1.receive(2, TimeUnit.SECONDS); + Assert.assertNull(msg1); + + // Verify: after enough messages have been skipped, will unpause the dispatcher. + skipMessages(tpName, subscription, skipType, c1); + // Since the message "specifiedMessage1" might be skipped, we send a new message to verify the result. + final String specifiedMessage2 = "9876543211"; + p1.send(specifiedMessage2); + + ReceivedMessages receivedMessagesAfterPause = ackAllMessages(c1); + Assert.assertTrue(receivedMessagesAfterPause.hasReceivedMessage(specifiedMessage2)); + Assert.assertTrue(receivedMessagesAfterPause.hasAckedMessage(specifiedMessage2)); + + // cleanup. + p1.close(); + c1.close(); + admin.topics().delete(tpName, false); + } + + private void skipMessages(String tpName, String subscription, SkipType skipType, Consumer c) throws Exception { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(tpName, false).join().get(); + Position LAC = persistentTopic.getManagedLedger().getLastConfirmedEntry(); + MessageIdImpl LACMessageId = new MessageIdImpl(LAC.getLedgerId(), LAC.getEntryId(), -1); + if (skipType == SkipType.SKIP_ENTRIES) { + while (true) { + GetStatsOptions getStatsOptions = new GetStatsOptions( + true, /* getPreciseBacklog */ + false, /* subscriptionBacklogSize */ + false, /* getEarliestTimeInBacklog */ + true, /* excludePublishers */ + true /* excludeConsumers */); + org.apache.pulsar.common.policies.data.SubscriptionStats subscriptionStats = + admin.topics().getStats(tpName, getStatsOptions).getSubscriptions().get(subscription); + if (subscriptionStats.getMsgBacklog() < MAX_UNACKED_RANGES_TO_PERSIST) { + break; + } + admin.topics().skipMessages(tpName, subscription, 100); + } + } else if (skipType == SkipType.CLEAR_BACKLOG){ + admin.topics().skipAllMessages(tpName, subscription); + } else if (skipType == SkipType.SEEK) { + c.seek(LACMessageId); + } else if (skipType == SkipType.RESET_CURSOR) { + admin.topics().resetCursor(tpName, subscription, LACMessageId, false); + } + } + + @Test(dataProvider = "singleConsumerSubscriptionTypes") + public void testSingleConsumerDispatcherWillNotPause(SubscriptionType subscriptionType) throws Exception { + final String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final String subscription = "s1"; + final int msgSendCount = MAX_UNACKED_RANGES_TO_PERSIST * 4; + final int incomingQueueSize = MAX_UNACKED_RANGES_TO_PERSIST * 10; + + enablePolicyDispatcherPauseOnAckStatePersistent(tpName); + admin.topics().createNonPartitionedTopic(tpName); + admin.topics().createSubscription(tpName, subscription, MessageId.earliest); + + // Send double MAX_UNACKED_RANGES_TO_PERSIST messages. + Producer p1 = pulsarClient.newProducer(Schema.STRING).topic(tpName).enableBatching(false).create(); + ArrayList messageIdsSent = new ArrayList<>(); + for (int i = 0; i < msgSendCount; i++) { + MessageIdImpl messageId = (MessageIdImpl) p1.send(Integer.valueOf(i).toString()); + messageIdsSent.add(messageId); + } + // Make ack holes. + Consumer c1 = pulsarClient.newConsumer(Schema.STRING).topic(tpName).subscriptionName(subscription) + .receiverQueueSize(incomingQueueSize).isAckReceiptEnabled(true) + .subscriptionType(subscriptionType) + .subscribe(); + ackOddMessagesOnly(c1); + verifyAckHolesIsMuchThanLimit(tpName, subscription); + + cancelPendingRead(tpName, subscription); + triggerNewReadMoreEntries(tpName, subscription); + + // Verify: the dispatcher has been paused. + final String specifiedMessage = "9876543210"; + p1.send(specifiedMessage); + Message msg1 = c1.receive(2, TimeUnit.SECONDS); + Assert.assertNotNull(msg1); + Assert.assertEquals(msg1.getValue(), specifiedMessage); + + // cleanup. + p1.close(); + c1.close(); + admin.topics().delete(tpName, false); + } + + @Test(dataProvider = "multiConsumerSubscriptionTypes") + public void testPauseOnAckStatPersistNotAffectReplayRead(SubscriptionType subscriptionType) throws Exception { + final String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final String subscription = "s1"; + final int msgSendCount = MAX_UNACKED_RANGES_TO_PERSIST * 4; + final int incomingQueueSize = MAX_UNACKED_RANGES_TO_PERSIST * 10; + + enablePolicyDispatcherPauseOnAckStatePersistent(tpName); + admin.topics().createNonPartitionedTopic(tpName); + admin.topics().createSubscription(tpName, subscription, MessageId.earliest); + + // Send double MAX_UNACKED_RANGES_TO_PERSIST messages. + Consumer c1 = pulsarClient.newConsumer(Schema.STRING).topic(tpName).subscriptionName(subscription) + .receiverQueueSize(incomingQueueSize).isAckReceiptEnabled(true).subscriptionType(subscriptionType) + .subscribe(); + Producer p1 = pulsarClient.newProducer(Schema.STRING).topic(tpName).enableBatching(false).create(); + ArrayList messageIdsSent = new ArrayList<>(); + for (int i = 0; i < msgSendCount; i++) { + MessageIdImpl messageId = (MessageIdImpl) p1.send(Integer.valueOf(i).toString()); + messageIdsSent.add(messageId); + } + // Make ack holes. + ReceivedMessages receivedMessagesC1 = ackOddMessagesOnly(c1); + verifyAckHolesIsMuchThanLimit(tpName, subscription); + + cancelPendingRead(tpName, subscription); + triggerNewReadMoreEntries(tpName, subscription); + + // Verify: the dispatcher has been paused. + Consumer c2 = pulsarClient.newConsumer(Schema.STRING).topic(tpName).subscriptionName(subscription) + .receiverQueueSize(incomingQueueSize).isAckReceiptEnabled(true).subscriptionType(subscriptionType) + .subscribe(); + final String specifiedMessage = "9876543210"; + final int specifiedMessageCount = 1; + p1.send(specifiedMessage); + Message msg1 = c1.receive(2, TimeUnit.SECONDS); + Assert.assertNull(msg1); + Message msg2 = c2.receive(2, TimeUnit.SECONDS); + Assert.assertNull(msg2); + + // Verify: close the previous consumer, the new one could receive all messages. + c1.close(); + ReceivedMessages receivedMessagesC2 = ackAllMessages(c2); + int messageCountAckedByC1 = receivedMessagesC1.messagesAcked.size(); + int messageCountAckedByC2 = receivedMessagesC2.messagesAcked.size(); + Assert.assertEquals(messageCountAckedByC2, msgSendCount - messageCountAckedByC1 + specifiedMessageCount); + + // cleanup, c1 has been closed before. + p1.close(); + c2.close(); + admin.topics().delete(tpName, false); + } + + @Test(dataProvider = "multiConsumerSubscriptionTypes") + public void testMultiConsumersPauseOnAckStatPersistNotAffectReplayRead(SubscriptionType subscriptionType) + throws Exception { + final String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final String subscription = "s1"; + final int msgSendCount = MAX_UNACKED_RANGES_TO_PERSIST * 4; + final int incomingQueueSize = MAX_UNACKED_RANGES_TO_PERSIST * 10; + + enablePolicyDispatcherPauseOnAckStatePersistent(tpName); + admin.topics().createNonPartitionedTopic(tpName); + admin.topics().createSubscription(tpName, subscription, MessageId.earliest); + + // Send double MAX_UNACKED_RANGES_TO_PERSIST messages. + Consumer c1 = pulsarClient.newConsumer(Schema.STRING).topic(tpName).subscriptionName(subscription) + .receiverQueueSize(incomingQueueSize).isAckReceiptEnabled(true).subscriptionType(subscriptionType) + .subscribe(); + Consumer c2 = pulsarClient.newConsumer(Schema.STRING).topic(tpName).subscriptionName(subscription) + .receiverQueueSize(incomingQueueSize).isAckReceiptEnabled(true).subscriptionType(subscriptionType) + .subscribe(); + Producer p1 = pulsarClient.newProducer(Schema.STRING).topic(tpName).enableBatching(false).create(); + ArrayList messageIdsSent = new ArrayList<>(); + for (int i = 0; i < msgSendCount; i++) { + MessageIdImpl messageId = (MessageIdImpl) p1.send(Integer.valueOf(i).toString()); + messageIdsSent.add(messageId); + } + // Make ack holes. + ReceivedMessages receivedMessagesC1AndC2 = ackOddMessagesOnly(c1, c2); + verifyAckHolesIsMuchThanLimit(tpName, subscription); + + cancelPendingRead(tpName, subscription); + triggerNewReadMoreEntries(tpName, subscription); + + // Verify: the dispatcher has been paused. + Consumer c3 = pulsarClient.newConsumer(Schema.STRING).topic(tpName).subscriptionName(subscription) + .receiverQueueSize(incomingQueueSize).isAckReceiptEnabled(true).subscriptionType(subscriptionType) + .subscribe(); + Consumer c4 = pulsarClient.newConsumer(Schema.STRING).topic(tpName).subscriptionName(subscription) + .receiverQueueSize(incomingQueueSize).isAckReceiptEnabled(true).subscriptionType(subscriptionType) + .subscribe(); + final String specifiedMessage = "9876543210"; + final int specifiedMessageCount = 1; + p1.send(specifiedMessage); + for (Consumer c : Arrays.asList(c1, c2, c3, c4)) { + Message m = c.receive(2, TimeUnit.SECONDS); + Assert.assertNull(m); + } + + // Verify: close the previous consumer, the new one could receive all messages. + c1.close(); + c2.close(); + ReceivedMessages receivedMessagesC3AndC4 = ackAllMessages(c3, c4); + int messageCountAckedByC1AndC2 = receivedMessagesC1AndC2.messagesAcked.size(); + int messageCountAckedByC3AndC4 = receivedMessagesC3AndC4.messagesAcked.size(); + Assert.assertEquals(messageCountAckedByC3AndC4, + msgSendCount - messageCountAckedByC1AndC2 + specifiedMessageCount); + + // cleanup, c1 has been closed before. + p1.close(); + c3.close(); + c4.close(); + admin.topics().delete(tpName, false); + } + + @Test(dataProvider = "multiConsumerSubscriptionTypes") + public void testNeverCallCursorIsCursorDataFullyPersistableIfDisabledTheFeature(SubscriptionType subscriptionType) + throws Exception { + final String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final String mlName = TopicName.get(tpName).getPersistenceNamingEncoding(); + final String subscription = "s1"; + final int msgSendCount = 100; + // Inject a injection to record the counter of calling "cursor.isCursorDataFullyPersistable". + final ManagedLedgerImpl ml = (ManagedLedgerImpl) pulsar.getBrokerService().getManagedLedgerFactory().open(mlName); + final ManagedCursorImpl cursor = (ManagedCursorImpl) ml.openCursor(subscription); + final ManagedCursorImpl spyCursor = Mockito.spy(cursor); + AtomicInteger callingIsCursorDataFullyPersistableCounter = new AtomicInteger(); + Mockito.doAnswer(invocation -> { + callingIsCursorDataFullyPersistableCounter.incrementAndGet(); + return invocation.callRealMethod(); + }).when(spyCursor).isCursorDataFullyPersistable(); + final ManagedCursorContainer cursors = WhiteboxImpl.getInternalState(ml, "cursors"); + final ManagedCursorContainer activeCursors = WhiteboxImpl.getInternalState(ml, "activeCursors"); + cursors.removeCursor(cursor.getName()); + activeCursors.removeCursor(cursor.getName()); + cursors.add(spyCursor, null); + activeCursors.add(spyCursor, null); + + // Pub & Sub. + Consumer c1 = pulsarClient.newConsumer(Schema.STRING).topic(tpName).subscriptionName(subscription) + .isAckReceiptEnabled(true).subscriptionType(subscriptionType).subscribe(); + Producer p1 = pulsarClient.newProducer(Schema.STRING).topic(tpName).enableBatching(false).create(); + for (int i = 0; i < msgSendCount; i++) { + p1.send(Integer.valueOf(i).toString()); + } + for (int i = 0; i < msgSendCount; i++) { + Message m = c1.receive(2, TimeUnit.SECONDS); + Assert.assertNotNull(m); + c1.acknowledge(m); + } + // Verify: the counter of calling "cursor.isCursorDataFullyPersistable". + // In expected the counter should be "0", to avoid flaky, verify it is less than 5. + Assert.assertTrue(callingIsCursorDataFullyPersistableCounter.get() < 5); + + // cleanup. + p1.close(); + c1.close(); + admin.topics().delete(tpName, false); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TenantTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TenantTest.java index 15c5d55cbea5b..e5b5e211582fd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TenantTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TenantTest.java @@ -34,7 +34,7 @@ public class TenantTest extends MockedPulsarServiceBaseTest { @BeforeMethod @Override protected void setup() throws Exception { - + } @AfterMethod(alwaysRun = true) @@ -66,5 +66,30 @@ public void testMaxTenant() throws Exception { admin.tenants().createTenant("testTenant-unlimited" + i, tenantInfo); } } - + + @Test + public void testBlankAdminRoleTenant() throws Exception { + super.internalSetup(); + admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(brokerUrl.toString()).build()); + TenantInfoImpl blankAdminRoleTenantInfo = + new TenantInfoImpl(Sets.newHashSet(""), Sets.newHashSet("test")); + TenantInfoImpl containsWhitespaceAdminRoleTenantInfo = + new TenantInfoImpl(Sets.newHashSet(" role1 "), Sets.newHashSet("test")); + TenantInfoImpl noneBlankAdminRoleTenantInfo = + new TenantInfoImpl(Sets.newHashSet("role1"), Sets.newHashSet("test")); + admin.tenants().createTenant("testTenant1", noneBlankAdminRoleTenantInfo); + try { + admin.tenants().createTenant("testTenant2", blankAdminRoleTenantInfo); + } catch (PulsarAdminException e) { + Assert.assertEquals(e.getStatusCode(), 412); + Assert.assertEquals(e.getHttpError(), "AdminRoles contains whitespace in the beginning or end."); + } + + try { + admin.tenants().createTenant("testTenant3", containsWhitespaceAdminRoleTenantInfo); + } catch (PulsarAdminException e) { + Assert.assertEquals(e.getStatusCode(), 412); + Assert.assertEquals(e.getHttpError(), "AdminRoles contains whitespace in the beginning or end."); + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsHostVerificationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsHostVerificationTest.java index fff61c5c8c940..c47c446717d69 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsHostVerificationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsHostVerificationTest.java @@ -21,6 +21,7 @@ import java.util.HashMap; import java.util.Map; +import lombok.Cleanup; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -49,6 +50,7 @@ public void testTlsHostVerificationAdminClient() throws Exception { authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8")); Assert.assertTrue(pulsar.getWebServiceAddressTls().startsWith("https://127.0.0.2:"), "Test relies on this address"); + @Cleanup PulsarAdmin adminClientTls = PulsarAdmin.builder() .serviceHttpUrl(pulsar.getWebServiceAddressTls()) .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).allowTlsInsecureConnection(false) @@ -75,6 +77,7 @@ public void testTlsHostVerificationDisabledAdminClient() throws Exception { authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8")); Assert.assertTrue(pulsar.getWebServiceAddressTls().startsWith("https://127.0.0.2:"), "Test relies on this address"); + @Cleanup PulsarAdmin adminClient = PulsarAdmin.builder() .serviceHttpUrl(pulsar.getWebServiceAddressTls()) .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).allowTlsInsecureConnection(false) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerBase.java index 39bab20d97df5..c29d59ede1edd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerBase.java @@ -92,10 +92,7 @@ protected void internalSetUpForNamespace() throws Exception { authParams.put("tlsCertFile", getTlsFileForClient("admin.cert")); authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8")); - if (admin != null) { - admin.close(); - } - + closeAdmin(); admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString()) .tlsTrustCertsFilePath(CA_CERT_FILE_PATH).allowTlsInsecureConnection(false) .authentication(AuthenticationTls.class.getName(), authParams).build()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerTest.java index 879289eb65dc8..44af37ca90f51 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerTest.java @@ -199,6 +199,7 @@ public void testTlsCertsFromDynamicStreamExpiredAndRenewCert() throws Exception log.info("-- Starting {} test --", methodName); ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(pulsar.getBrokerServiceUrlTls()) .enableTls(true).allowTlsInsecureConnection(false) + .autoCertRefreshSeconds(1) .operationTimeout(1000, TimeUnit.MILLISECONDS); AtomicInteger certIndex = new AtomicInteger(1); AtomicInteger keyIndex = new AtomicInteger(0); @@ -223,7 +224,7 @@ public void testTlsCertsFromDynamicStreamExpiredAndRenewCert() throws Exception } catch (PulsarClientException e) { // Ok.. } - + sleepSeconds(2); certIndex.set(0); try { consumer = pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/my-topic1") @@ -232,8 +233,9 @@ public void testTlsCertsFromDynamicStreamExpiredAndRenewCert() throws Exception } catch (PulsarClientException e) { // Ok.. } - + sleepSeconds(2); trustStoreIndex.set(0); + sleepSeconds(2); consumer = pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/my-topic1") .subscriptionName("my-subscriber-name").subscribe(); consumer.close(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenAuthenticatedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenAuthenticatedProducerConsumerTest.java index 87f12e6acdcb2..f8ae0279e08b7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenAuthenticatedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenAuthenticatedProducerConsumerTest.java @@ -37,6 +37,7 @@ import java.util.concurrent.TimeUnit; import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.slf4j.Logger; @@ -92,6 +93,8 @@ protected void setup() throws Exception { Set providers = new HashSet<>(); providers.add(AuthenticationProviderToken.class.getName()); conf.setAuthenticationProviders(providers); + conf.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); + conf.setBrokerClientAuthenticationParameters("token:" + ADMIN_TOKEN); conf.setClusterName("test"); @@ -105,6 +108,7 @@ protected void setup() throws Exception { // setup both admin and pulsar client protected final void clientSetup() throws Exception { + closeAdmin(); admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrl.toString()) .authentication(AuthenticationFactory.token(ADMIN_TOKEN)) .build()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenExpirationProduceConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenExpirationProduceConsumerTest.java index 4fc0d315d2253..d8ed105572033 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenExpirationProduceConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenExpirationProduceConsumerTest.java @@ -18,11 +18,23 @@ */ package org.apache.pulsar.client.api; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; import com.google.common.collect.Sets; import io.jsonwebtoken.Jwts; import io.jsonwebtoken.SignatureAlgorithm; +import java.time.Duration; +import java.util.Base64; +import java.util.Calendar; +import java.util.Date; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import javax.crypto.SecretKey; +import java.nio.charset.StandardCharsets; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; @@ -35,20 +47,9 @@ import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.awaitility.Awaitility; -import org.mockito.Mockito; -import org.mockito.internal.util.MockUtil; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import javax.crypto.SecretKey; -import java.time.Duration; -import java.util.Base64; -import java.util.Calendar; -import java.util.Date; -import java.util.HashSet; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.TimeUnit; @Test(groups = "broker-api") @Slf4j @@ -65,12 +66,7 @@ protected void setup() throws Exception { // Start Broker super.init(); - if (admin != null) { - admin.close(); - if (MockUtil.isMock(admin)) { - Mockito.reset(admin); - } - } + closeAdmin(); admin = getAdmin(ADMIN_TOKEN); admin.clusters().createCluster(configClusterName, ClusterData.builder() @@ -114,6 +110,7 @@ protected void internalSetUpForBroker() { conf.setAuthenticationProviders(Sets.newHashSet(AuthenticationProviderToken.class.getName())); conf.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); conf.setBrokerClientAuthenticationParameters("token:" + ADMIN_TOKEN); + conf.setBrokerClientTlsEnabled(true); conf.getProperties().setProperty("tokenSecretKey", "data:;base64," + Base64.getEncoder().encodeToString(SECRET_KEY.getEncoded())); } @@ -139,6 +136,29 @@ private PulsarAdmin getAdmin(String token) throws Exception { return clientBuilder.build(); } + @Test + public void testNonPersistentTopic() throws Exception { + + @Cleanup + PulsarClient pulsarClient = getClient(ADMIN_TOKEN); + + String topic = "non-persistent://" + namespaceName + "/test-token-non-persistent"; + + @Cleanup + Consumer consumer = pulsarClient.newConsumer().topic(topic) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionName("test").subscribe(); + + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topic).create(); + byte[] msg = "Hello".getBytes(StandardCharsets.UTF_8); + producer.send(msg); + + Message receive = consumer.receive(3, TimeUnit.SECONDS); + assertNotNull(receive); + assertEquals(receive.getData(), msg); + } + @Test public void testTokenExpirationProduceConsumer() throws Exception { Calendar calendar = Calendar.getInstance(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java index 22834b2e0c9c1..c24e192361921 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java @@ -22,15 +22,19 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotEquals; import com.google.common.collect.Sets; +import java.net.MalformedURLException; import java.net.URI; import java.net.URL; import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; +import org.apache.pulsar.broker.auth.MockOIDCIdentityProvider; import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.impl.ProducerImpl; @@ -38,10 +42,13 @@ import org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.util.ObjectMapperFactory; import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -55,25 +62,28 @@ public class TokenOauth2AuthenticatedProducerConsumerTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(TokenOauth2AuthenticatedProducerConsumerTest.class); - // public key in oauth2 server to verify the client passed in token. get from https://jwt.io/ - private final String TOKEN_TEST_PUBLIC_KEY = "data:;base64,MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2tZd/4gJda3U2Pc3tpgRAN7JPGWx/Gn17v/0IiZlNNRbP/Mmf0Vc6G1qsnaRaWNWOR+t6/a6ekFHJMikQ1N2X6yfz4UjMc8/G2FDPRmWjA+GURzARjVhxc/BBEYGoD0Kwvbq/u9CZm2QjlKrYaLfg3AeB09j0btNrDJ8rBsNzU6AuzChRvXj9IdcE/A/4N/UQ+S9cJ4UXP6NJbToLwajQ5km+CnxdGE6nfB7LWHvOFHjn9C2Rb9e37CFlmeKmIVFkagFM0gbmGOb6bnGI8Bp/VNGV0APef4YaBvBTqwoZ1Z4aDHy5eRxXfAMdtBkBupmBXqL6bpd15XRYUbu/7ck9QIDAQAB"; - - private final String ADMIN_ROLE = "Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x@clients"; + private MockOIDCIdentityProvider server; // Credentials File, which contains "client_id" and "client_secret" private final String CREDENTIALS_FILE = "./src/test/resources/authentication/token/credentials_file.json"; - private final String ISSUER_URL = "https://dev-kt-aa9ne.us.auth0.com"; - private final String AUDIENCE = "https://dev-kt-aa9ne.us.auth0.com/api/v2/"; + private final String audience = "my-pulsar-cluster"; + + @BeforeClass(alwaysRun = true) + protected void setupClass() { + String clientSecret = "super-secret-client-secret"; + server = new MockOIDCIdentityProvider(clientSecret, audience, 3000); + } @BeforeMethod(alwaysRun = true) @Override protected void setup() throws Exception { conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(true); - conf.setAuthenticationRefreshCheckSeconds(5); + conf.setAuthenticationRefreshCheckSeconds(1); Set superUserRoles = new HashSet<>(); - superUserRoles.add(ADMIN_ROLE); + // Matches the role in th credentials file + superUserRoles.add("my-admin-role"); conf.setSuperUserRoles(superUserRoles); Set providers = new HashSet<>(); @@ -81,17 +91,18 @@ protected void setup() throws Exception { conf.setAuthenticationProviders(providers); conf.setBrokerClientAuthenticationPlugin(AuthenticationOAuth2.class.getName()); - conf.setBrokerClientAuthenticationParameters("{\n" - + " \"privateKey\": \"" + CREDENTIALS_FILE + "\",\n" - + " \"issuerUrl\": \"" + ISSUER_URL + "\",\n" - + " \"audience\": \"" + AUDIENCE + "\",\n" - + "}\n"); + final Map oauth2Param = new HashMap<>(); + oauth2Param.put("privateKey", CREDENTIALS_FILE); + oauth2Param.put("issuerUrl", server.getIssuer()); + oauth2Param.put("audience", audience); + conf.setBrokerClientAuthenticationParameters(ObjectMapperFactory + .getMapper().getObjectMapper().writeValueAsString(oauth2Param)); conf.setClusterName("test"); // Set provider domain name Properties properties = new Properties(); - properties.setProperty("tokenPublicKey", TOKEN_TEST_PUBLIC_KEY); + properties.setProperty("tokenPublicKey", "data:;base64," + server.getBase64EncodedPublicKey()); conf.setProperties(properties); super.init(); @@ -102,20 +113,22 @@ protected final void clientSetup() throws Exception { Path path = Paths.get(CREDENTIALS_FILE).toAbsolutePath(); log.info("Credentials File path: {}", path.toString()); - // AuthenticationOAuth2 - Authentication authentication = AuthenticationFactoryOAuth2.clientCredentials( - new URL(ISSUER_URL), - path.toUri().toURL(), // key file path - AUDIENCE - ); - + closeAdmin(); admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrl.toString()) - .authentication(authentication) + .authentication(createAuthentication(path)) .build()); replacePulsarClient(PulsarClient.builder().serviceUrl(new URI(pulsar.getBrokerServiceUrl()).toString()) .statsInterval(0, TimeUnit.SECONDS) - .authentication(authentication)); + .authentication(createAuthentication(path))); + } + + private Authentication createAuthentication(Path path) throws MalformedURLException { + return AuthenticationFactoryOAuth2.clientCredentials( + new URL(server.getIssuer()), + path.toUri().toURL(), // key file path + audience + ); } @AfterMethod(alwaysRun = true) @@ -124,6 +137,11 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + @AfterClass(alwaysRun = true) + protected void cleanupAfterClass() { + server.stop(); + } + @DataProvider(name = "batch") public Object[][] codecProvider() { return new Object[][] { { 0 }, { 1000 } }; @@ -210,12 +228,11 @@ public void testOAuth2TokenRefreshedWithoutReconnect() throws Exception { String accessTokenOld = producerImpl.getClientCnx().getAuthenticationDataProvider().getCommandData(); long lastDisconnectTime = producer.getLastDisconnectedTimestamp(); - // the token expire duration is 10 seconds, so we need to wait for the authenticationData refreshed + // the token expire duration is 3 seconds, so we need to wait for the authenticationData refreshed Awaitility.await() - .atLeast(10, TimeUnit.SECONDS) - .atMost(20, TimeUnit.SECONDS) + .atMost(10, TimeUnit.SECONDS) .with() - .pollInterval(Duration.ofSeconds(1)) + .pollInterval(Duration.ofMillis(250)) .untilAsserted(() -> { String accessTokenNew = producerImpl.getClientCnx().getAuthenticationDataProvider().getCommandData(); assertNotEquals(accessTokenNew, accessTokenOld); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicReaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicReaderTest.java index 424081b904c81..e04dde65fa872 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicReaderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicReaderTest.java @@ -996,7 +996,7 @@ public void testMultiReaderMessageAvailableAfterRestart() throws Exception { } // cause broker to drop topic. Will be loaded next time we access it - pulsar.getBrokerService().getTopics().keys().forEach(topicName -> { + pulsar.getBrokerService().getTopics().keySet().forEach(topicName -> { try { pulsar.getBrokerService().getTopicReference(topicName).get().close(false).get(); } catch (Exception e) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/UnloadSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/UnloadSubscriptionTest.java index 93d5bf30ec6b1..22f7a5d6a43e4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/UnloadSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/UnloadSubscriptionTest.java @@ -60,6 +60,7 @@ protected void doInitConf() throws Exception { super.doInitConf(); conf.setSystemTopicEnabled(false); conf.setTransactionCoordinatorEnabled(false); + conf.setAcknowledgmentAtBatchIndexLevelEnabled(true); } @AfterClass(alwaysRun = true) @@ -242,6 +243,7 @@ private Consumer createConsumer(String topicName, String subName, Subscr .subscriptionName(subName) .subscriptionType(subType) .isAckReceiptEnabled(true) + .enableBatchIndexAcknowledgment(true) .subscribe(); return consumer; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java index e126d963a88f5..d3cb1d60d37ed 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java @@ -951,7 +951,7 @@ private void receiveAsync(Consumer consumer, int totalMessage, int curre /** * Verify: Consumer stops receiving msg when reach unack-msg limit and starts receiving once acks messages 1. * Produce X (600) messages 2. Consumer has receive size (10) and receive message without acknowledging 3. Consumer - * will stop receiving message after unAckThreshold = 500 4. Consumer acks messages and starts consuming remanining + * will stop receiving message after unAckThreshold = 500 4. Consumer acks messages and starts consuming remaining * messages This testcase enables checksum sending while producing message and broker verifies the checksum for the * message. * @@ -1574,7 +1574,7 @@ public void testEnabledChecksumClient() throws Exception { /** * It verifies that redelivery-of-specific messages: that redelivers all those messages even when consumer gets - * blocked due to unacked messsages + * blocked due to unacked messages * * Usecase: produce message with 10ms interval: so, consumer can consume only 10 messages without acking * @@ -1659,7 +1659,7 @@ public void testBlockUnackedConsumerRedeliverySpecificMessagesProduceWithPause() /** * It verifies that redelivery-of-specific messages: that redelivers all those messages even when consumer gets - * blocked due to unacked messsages + * blocked due to unacked messages * * Usecase: Consumer starts consuming only after all messages have been produced. So, consumer consumes total * receiver-queue-size number messages => ask for redelivery and receives all messages again. diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/AutoCloseUselessClientConSupports.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/AutoCloseUselessClientConSupports.java index e03b170913751..c9f478969a614 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/AutoCloseUselessClientConSupports.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/AutoCloseUselessClientConSupports.java @@ -18,19 +18,14 @@ */ package org.apache.pulsar.client.impl; -import java.lang.reflect.Field; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.HashSet; -import java.util.List; import java.util.UUID; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import org.apache.pulsar.broker.MultiBrokerBaseTest; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -71,16 +66,8 @@ protected PulsarClient newPulsarClient(String url, int intervalInSecs) throws Pu protected void trigReleaseConnection(PulsarClientImpl pulsarClient) throws InterruptedException, NoSuchFieldException, IllegalAccessException { // Wait for every request has been response. - Field field = ConnectionPool.class.getDeclaredField("pool"); - field.setAccessible(true); - ConcurrentHashMap>> pool = - (ConcurrentHashMap>>) field.get(pulsarClient.getCnxPool()); - final List> clientCnxWrapList = - pool.values().stream().flatMap(c -> c.values().stream()).collect(Collectors.toList()); Awaitility.waitAtMost(Duration.ofSeconds(5)).until(() -> { - for (CompletableFuture clientCnxWrapFuture : clientCnxWrapList){ + for (CompletableFuture clientCnxWrapFuture : pulsarClient.getCnxPool().getConnections()){ if (!clientCnxWrapFuture.isDone()){ continue; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java index 716dd1019f4d8..1e8754a2d675c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java @@ -48,7 +48,6 @@ import java.util.HashSet; import java.util.List; import java.util.NavigableMap; -import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListMap; @@ -67,11 +66,11 @@ import org.apache.bookkeeper.client.PulsarMockBookKeeper; import org.apache.bookkeeper.client.PulsarMockLedgerHandle; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.namespace.OwnershipCache; import org.apache.pulsar.broker.resources.BaseResources; import org.apache.pulsar.broker.service.AbstractDispatcherSingleActiveConsumer; -import org.apache.pulsar.broker.service.ServerCnx; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -101,9 +100,9 @@ import org.apache.pulsar.common.protocol.PulsarHandler; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.awaitility.Awaitility; import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -173,26 +172,15 @@ public void testDisconnectClientWithoutClosingConnection() throws Exception { doAnswer(invocationOnMock -> cons1.getState()).when(consumer1).getState(); doAnswer(invocationOnMock -> cons1.getClientCnx()).when(consumer1).getClientCnx(); doAnswer(invocationOnMock -> cons1.cnx()).when(consumer1).cnx(); - doAnswer(invocationOnMock -> { - cons1.connectionClosed((ClientCnx) invocationOnMock.getArguments()[0]); - return null; - }).when(consumer1).connectionClosed(any()); + doAnswer(InvocationOnMock::callRealMethod).when(consumer1).connectionClosed(any(), any(), any()); ProducerImpl producer1 = spy(prod1); doAnswer(invocationOnMock -> prod1.getState()).when(producer1).getState(); doAnswer(invocationOnMock -> prod1.getClientCnx()).when(producer1).getClientCnx(); doAnswer(invocationOnMock -> prod1.cnx()).when(producer1).cnx(); - doAnswer(invocationOnMock -> { - prod1.connectionClosed((ClientCnx) invocationOnMock.getArguments()[0]); - return null; - }).when(producer1).connectionClosed(any()); ProducerImpl producer2 = spy(prod2); doAnswer(invocationOnMock -> prod2.getState()).when(producer2).getState(); doAnswer(invocationOnMock -> prod2.getClientCnx()).when(producer2).getClientCnx(); doAnswer(invocationOnMock -> prod2.cnx()).when(producer2).cnx(); - doAnswer(invocationOnMock -> { - prod2.connectionClosed((ClientCnx) invocationOnMock.getArguments()[0]); - return null; - }).when(producer2).connectionClosed(any()); ClientCnx clientCnx = producer1.getClientCnx(); @@ -223,11 +211,11 @@ public void testDisconnectClientWithoutClosingConnection() throws Exception { // let server send signal to close-connection and client close the connection Thread.sleep(1000); // [1] Verify: producer1 must get connectionClosed signal - verify(producer1, atLeastOnce()).connectionClosed(any()); + verify(producer1, atLeastOnce()).connectionClosed(any(), any(), any()); // [2] Verify: consumer1 must get connectionClosed signal - verify(consumer1, atLeastOnce()).connectionClosed(any()); + verify(consumer1, atLeastOnce()).connectionClosed(any(), any(), any()); // [3] Verify: producer2 should have not received connectionClosed signal - verify(producer2, never()).connectionClosed(any()); + verify(producer2, never()).connectionClosed(any(), any(), any()); // sleep for sometime to let other disconnected producer and consumer connect again: but they should not get // connected with same broker as that broker is already out from active-broker list @@ -247,7 +235,7 @@ public void testDisconnectClientWithoutClosingConnection() throws Exception { pulsar.getNamespaceService().unloadNamespaceBundle((NamespaceBundle) bundle2).join(); // let producer2 give some time to get disconnect signal and get disconnected Thread.sleep(200); - verify(producer2, atLeastOnce()).connectionClosed(any()); + verify(producer2, atLeastOnce()).connectionClosed(any(), any(), any()); // producer1 must not be able to connect again assertNull(prod1.getClientCnx()); @@ -591,7 +579,7 @@ public void testMaxConcurrentTopicLoading() throws Exception { } /** - * It verifies that client closes the connection on internalSerevrError which is "ServiceNotReady" from Broker-side + * It verifies that client closes the connection on internalServerError which is "ServiceNotReady" from Broker-side * * @throws Exception */ @@ -707,8 +695,7 @@ public void testCleanProducer() throws Exception { @Test(expectedExceptions = PulsarClientException.TimeoutException.class) public void testOperationTimeout() throws PulsarClientException { final String topicName = "persistent://my-property/my-ns/my-topic1"; - ConcurrentOpenHashMap>> topics = pulsar.getBrokerService() - .getTopics(); + final var topics = pulsar.getBrokerService().getTopics(); // non-complete topic future so, create topic should timeout topics.put(topicName, new CompletableFuture<>()); try (PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl.toString()) @@ -1018,49 +1005,61 @@ public void testActiveConsumerCleanup() throws Exception { int numMessages = 100; final CountDownLatch latch = new CountDownLatch(numMessages); - String topic = "persistent://my-property/my-ns/closed-cnx-topic"; + String topic = BrokerTestUtil.newUniqueName("persistent://my-property/my-ns/closed-cnx-topic"); + admin.topics().createNonPartitionedTopic(topic); String sub = "my-subscriber-name"; - + @Cleanup PulsarClient pulsarClient = newPulsarClient(lookupUrl.toString(), 0); - pulsarClient.newConsumer().topic(topic).subscriptionName(sub).messageListener((c1, msg) -> { - Assert.assertNotNull(msg, "Message cannot be null"); - String receivedMessage = new String(msg.getData()); - log.debug("Received message [{}] in the listener", receivedMessage); - c1.acknowledgeAsync(msg); - latch.countDown(); - }).subscribe(); - + ConsumerImpl c = + (ConsumerImpl) pulsarClient.newConsumer().topic(topic).subscriptionName(sub).messageListener((c1, msg) -> { + Assert.assertNotNull(msg, "Message cannot be null"); + String receivedMessage = new String(msg.getData()); + log.debug("Received message [{}] in the listener", receivedMessage); + c1.acknowledgeAsync(msg); + latch.countDown(); + }).subscribe(); PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get(); - AbstractDispatcherSingleActiveConsumer dispatcher = (AbstractDispatcherSingleActiveConsumer) topicRef .getSubscription(sub).getDispatcher(); - ServerCnx cnx = (ServerCnx) dispatcher.getActiveConsumer().cnx(); - Field field = ServerCnx.class.getDeclaredField("isActive"); - field.setAccessible(true); - field.set(cnx, false); - assertNotNull(dispatcher.getActiveConsumer()); - pulsarClient = newPulsarClient(lookupUrl.toString(), 0); + // Inject an blocker to make the "ping & pong" does not work. + CountDownLatch countDownLatch = new CountDownLatch(1); + ConnectionHandler connectionHandler = c.getConnectionHandler(); + ClientCnx clientCnx = connectionHandler.cnx(); + clientCnx.ctx().executor().submit(() -> { + try { + countDownLatch.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + + @Cleanup + PulsarClient pulsarClient2 = newPulsarClient(lookupUrl.toString(), 0); Consumer consumer = null; for (int i = 0; i < 2; i++) { try { - consumer = pulsarClient.newConsumer().topic(topic).subscriptionName(sub).messageListener((c1, msg) -> { + consumer = pulsarClient2.newConsumer().topic(topic).subscriptionName(sub).messageListener((c1, msg) -> { Assert.assertNotNull(msg, "Message cannot be null"); String receivedMessage = new String(msg.getData()); log.debug("Received message [{}] in the listener", receivedMessage); c1.acknowledgeAsync(msg); latch.countDown(); }).subscribe(); - if (i == 0) { - fail("Should failed with ConsumerBusyException!"); - } } catch (PulsarClientException.ConsumerBusyException ignore) { // It's ok. } } assertNotNull(consumer); log.info("-- Exiting {} test --", methodName); + + // cleanup. + countDownLatch.countDown(); + consumer.close(); + pulsarClient.close(); + pulsarClient2.close(); + admin.topics().delete(topic, false); } @Test @@ -1083,4 +1082,41 @@ public void testManagedLedgerLazyCursorLedgerCreation() throws Exception { }); } + @Test + public void testSharedConsumerUnsubscribe() throws Exception { + String topic = "persistent://my-property/my-ns/sharedUnsubscribe"; + String sub = "my-subscriber-name"; + @Cleanup + Consumer consumer1 = pulsarClient.newConsumer().topic(topic).subscriptionType(SubscriptionType.Shared) + .subscriptionName(sub).subscribe(); + @Cleanup + Consumer consumer2 = pulsarClient.newConsumer().topic(topic).subscriptionType(SubscriptionType.Shared) + .subscriptionName(sub).subscribe(); + try { + consumer1.unsubscribe(); + fail("should have failed as consumer-2 is already connected"); + } catch (Exception e) { + // Ok + } + + consumer1.unsubscribe(true); + try { + consumer2.unsubscribe(true); + } catch (PulsarClientException.NotConnectedException e) { + // Ok. consumer-2 is already disconnected with force unsubscription + } + assertFalse(consumer1.isConnected()); + assertFalse(consumer2.isConnected()); + } + + @Test(dataProvider = "subType") + public void testUnsubscribeForce(SubscriptionType type) throws Exception { + String topic = "persistent://my-property/my-ns/sharedUnsubscribe"; + String sub = "my-subscriber-name"; + @Cleanup + Consumer consumer1 = pulsarClient.newConsumer().topic(topic).subscriptionType(type) + .subscriptionName(sub).subscribe(); + consumer1.unsubscribe(true); + assertFalse(consumer1.isConnected()); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java index d2f610ae53f65..e25212e0108f8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java @@ -20,17 +20,25 @@ import com.google.common.collect.Sets; import io.netty.channel.ChannelHandlerContext; +import java.lang.reflect.Field; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.apache.pulsar.PulsarVersion; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.api.proto.ServerError; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.protocol.Commands; import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -43,7 +51,7 @@ public class ClientCnxTest extends MockedPulsarServiceBaseTest { public static final String TENANT = "tnx"; public static final String NAMESPACE = TENANT + "/ns1"; public static String persistentTopic = "persistent://" + NAMESPACE + "/test"; - ExecutorService executorService = Executors.newFixedThreadPool(20); + ExecutorService executorService; @BeforeClass @Override @@ -54,13 +62,14 @@ protected void setup() throws Exception { admin.tenants().createTenant(TENANT, new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); admin.namespaces().createNamespace(NAMESPACE); + executorService = Executors.newFixedThreadPool(20); } @AfterClass(alwaysRun = true) @Override protected void cleanup() throws Exception { super.internalCleanup(); - this.executorService.shutdown(); + this.executorService.shutdownNow(); } @Test @@ -123,4 +132,88 @@ public void testClientVersion() throws Exception { producer.close(); consumer.close(); } + + @Test + public void testCnxReceiveSendError() throws Exception { + final String topicOne = "persistent://" + NAMESPACE + "/testCnxReceiveSendError-one"; + final String topicTwo = "persistent://" + NAMESPACE + "/testCnxReceiveSendError-two"; + + PulsarClient client = PulsarClient.builder().serviceUrl(lookupUrl.toString()).connectionsPerBroker(1).build(); + Producer producerOne = client.newProducer(Schema.STRING) + .topic(topicOne) + .create(); + Producer producerTwo = client.newProducer(Schema.STRING) + .topic(topicTwo) + .create(); + ClientCnx cnxOne = ((ProducerImpl) producerOne).getClientCnx(); + ClientCnx cnxTwo = ((ProducerImpl) producerTwo).getClientCnx(); + + // simulate a sending error + cnxOne.handleSendError(Commands.newSendErrorCommand(((ProducerImpl) producerOne).producerId, + 10, ServerError.PersistenceError, "persistent error").getSendError()); + + // two producer use the same cnx + Assert.assertEquals(cnxOne, cnxTwo); + + // the cnx will not change + try { + Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> + (((ProducerImpl) producerOne).getClientCnx() != null + && !cnxOne.equals(((ProducerImpl) producerOne).getClientCnx())) + || !cnxTwo.equals(((ProducerImpl) producerTwo).getClientCnx())); + Assert.fail(); + } catch (Throwable e) { + Assert.assertTrue(e instanceof ConditionTimeoutException); + } + + // two producer use the same cnx + Assert.assertEquals(((ProducerImpl) producerTwo).getClientCnx(), + ((ProducerImpl) producerOne).getClientCnx()); + + // producer also can send message + producerOne.send("test"); + producerTwo.send("test"); + producerTwo.close(); + producerOne.close(); + client.close(); + } + + public void testSupportsGetPartitionedMetadataWithoutAutoCreation() throws Exception { + final String topic = BrokerTestUtil.newUniqueName( "persistent://" + NAMESPACE + "/tp"); + admin.topics().createNonPartitionedTopic(topic); + PulsarClientImpl clientWitBinaryLookup = (PulsarClientImpl) PulsarClient.builder() + .maxNumberOfRejectedRequestPerConnection(1) + .connectionMaxIdleSeconds(Integer.MAX_VALUE) + .serviceUrl(pulsar.getBrokerServiceUrl()) + .build(); + ProducerImpl producer = (ProducerImpl) clientWitBinaryLookup.newProducer().topic(topic).create(); + + // Verify: the variable "isSupportsGetPartitionedMetadataWithoutAutoCreation" responded from the broker is true. + Awaitility.await().untilAsserted(() -> { + ClientCnx clientCnx = producer.getClientCnx(); + Assert.assertNotNull(clientCnx); + Assert.assertTrue(clientCnx.isSupportsGetPartitionedMetadataWithoutAutoCreation()); + }); + Assert.assertEquals( + clientWitBinaryLookup.getPartitionsForTopic(topic, true).get().size(), 1); + + // Inject a "false" value for the variable "isSupportsGetPartitionedMetadataWithoutAutoCreation". + // Verify: client will get a not support error. + Field field = ClientCnx.class.getDeclaredField("supportsGetPartitionedMetadataWithoutAutoCreation"); + field.setAccessible(true); + for (CompletableFuture clientCnxFuture : clientWitBinaryLookup.getCnxPool().getConnections()) { + field.set(clientCnxFuture.get(), false); + } + try { + clientWitBinaryLookup.getPartitionsForTopic(topic, false).join(); + Assert.fail("Expected an error that the broker version is too old."); + } catch (Exception ex) { + Assert.assertTrue(ex.getMessage().contains("without auto-creation is not supported by the broker")); + } + + // cleanup. + producer.close(); + clientWitBinaryLookup.close(); + admin.topics().delete(topic, false); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionHandlerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionHandlerTest.java new file mode 100644 index 0000000000000..4bc5707946957 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionHandlerTest.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.common.util.Backoff; +import org.apache.pulsar.common.util.BackoffBuilder; +import org.apache.pulsar.common.util.FutureUtil; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker-impl") +public class ConnectionHandlerTest extends ProducerConsumerBase { + + private static final Backoff BACKOFF = new BackoffBuilder().setInitialTime(1, TimeUnit.MILLISECONDS) + .setMandatoryStop(1, TimeUnit.SECONDS) + .setMax(3, TimeUnit.SECONDS).create(); + private ExecutorService executor; + + @BeforeClass(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + executor = Executors.newFixedThreadPool(4); + } + + @AfterClass + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + executor.shutdownNow(); + } + + @Test(timeOut = 30000) + public void testSynchronousGrabCnx() { + for (int i = 0; i < 10; i++) { + final CompletableFuture future = new CompletableFuture<>(); + final int index = i; + final ConnectionHandler handler = new ConnectionHandler( + new MockedHandlerState((PulsarClientImpl) pulsarClient, "my-topic"), BACKOFF, + cnx -> { + future.complete(index); + return CompletableFuture.completedFuture(null); + }); + handler.grabCnx(); + Assert.assertEquals(future.join(), i); + } + } + + @Test + public void testConcurrentGrabCnx() { + final AtomicInteger cnt = new AtomicInteger(0); + final ConnectionHandler handler = new ConnectionHandler( + new MockedHandlerState((PulsarClientImpl) pulsarClient, "my-topic"), BACKOFF, + cnx -> { + cnt.incrementAndGet(); + return CompletableFuture.completedFuture(null); + }); + final int numGrab = 10; + for (int i = 0; i < numGrab; i++) { + handler.grabCnx(); + } + Awaitility.await().atMost(Duration.ofSeconds(3)).until(() -> cnt.get() > 0); + Assert.assertThrows(ConditionTimeoutException.class, + () -> Awaitility.await().atMost(Duration.ofMillis(500)).until(() -> cnt.get() == numGrab)); + Assert.assertEquals(cnt.get(), 1); + } + + @Test + public void testDuringConnectInvokeCount() throws IllegalAccessException { + // 1. connectionOpened completes with null + final AtomicBoolean duringConnect = spy(new AtomicBoolean()); + final ConnectionHandler handler1 = new ConnectionHandler( + new MockedHandlerState((PulsarClientImpl) pulsarClient, "my-topic"), BACKOFF, + cnx -> CompletableFuture.completedFuture(null)); + FieldUtils.writeField(handler1, "duringConnect", duringConnect, true); + handler1.grabCnx(); + Awaitility.await().atMost(Duration.ofSeconds(3)).until(() -> !duringConnect.get()); + verify(duringConnect, times(1)).compareAndSet(false, true); + verify(duringConnect, times(1)).set(false); + + // 2. connectionFailed is called + final ConnectionHandler handler2 = new ConnectionHandler( + new MockedHandlerState((PulsarClientImpl) pulsarClient, null), new MockedBackoff(), + cnx -> CompletableFuture.completedFuture(null)); + FieldUtils.writeField(handler2, "duringConnect", duringConnect, true); + handler2.grabCnx(); + Awaitility.await().atMost(Duration.ofSeconds(3)).until(() -> !duringConnect.get()); + verify(duringConnect, times(2)).compareAndSet(false, true); + verify(duringConnect, times(2)).set(false); + + // 3. connectionOpened completes exceptionally + final ConnectionHandler handler3 = new ConnectionHandler( + new MockedHandlerState((PulsarClientImpl) pulsarClient, "my-topic"), new MockedBackoff(), + cnx -> FutureUtil.failedFuture(new RuntimeException("fail"))); + FieldUtils.writeField(handler3, "duringConnect", duringConnect, true); + handler3.grabCnx(); + Awaitility.await().atMost(Duration.ofSeconds(3)).until(() -> !duringConnect.get()); + verify(duringConnect, times(3)).compareAndSet(false, true); + verify(duringConnect, times(3)).set(false); + } + + private static class MockedHandlerState extends HandlerState { + + public MockedHandlerState(PulsarClientImpl client, String topic) { + super(client, topic); + } + + @Override + String getHandlerName() { + return "mocked"; + } + } + + private static class MockedBackoff extends Backoff { + + // Set a large backoff so that reconnection won't happen in tests + public MockedBackoff() { + super(1, TimeUnit.HOURS, 2, TimeUnit.HOURS, 1, TimeUnit.HOURS); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java index fe0aa4dd4953b..12dc9690115a4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java @@ -28,12 +28,20 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.function.Supplier; import java.util.stream.IntStream; import io.netty.util.concurrent.Promise; +import lombok.Cleanup; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; +import org.apache.pulsar.common.api.proto.CommandCloseProducer; import org.apache.pulsar.common.util.netty.EventLoopUtil; +import org.awaitility.Awaitility; import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterClass; @@ -64,7 +72,12 @@ protected void cleanup() throws Exception { public void testSingleIpAddress() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, new DefaultThreadFactory("test")); - ConnectionPool pool = spyWithClassAndConstructorArgs(ConnectionPool.class, conf, eventLoop); + @Cleanup("shutdownNow") + ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( + new DefaultThreadFactory("test-pulsar-client-scheduled")); + ConnectionPool pool = + spyWithClassAndConstructorArgs(ConnectionPool.class, InstrumentProvider.NOOP, conf, eventLoop, + scheduledExecutorService); conf.setServiceUrl(serviceUrl); PulsarClientImpl client = new PulsarClientImpl(conf, eventLoop, pool); @@ -80,11 +93,45 @@ public void testSingleIpAddress() throws Exception { eventLoop.shutdownGracefully(); } + @Test + public void testSelectConnectionForSameProducer() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://sample/standalone/ns/tp_"); + admin.topics().createNonPartitionedTopic(topicName); + final CommandCloseProducer commandCloseProducer = new CommandCloseProducer(); + // 10 connection per broker. + final PulsarClient clientWith10ConPerBroker = PulsarClient.builder().connectionsPerBroker(10) + .serviceUrl(lookupUrl.toString()).build(); + ProducerImpl producer = (ProducerImpl) clientWith10ConPerBroker.newProducer().topic(topicName).create(); + commandCloseProducer.setProducerId(producer.producerId); + // An error will be reported when the Producer reconnects using a different connection. + // If no error is reported, the same connection was used when reconnecting. + for (int i = 0; i < 20; i++) { + // Trigger reconnect + ClientCnx cnx = producer.getClientCnx(); + if (cnx != null) { + cnx.handleCloseProducer(commandCloseProducer); + Awaitility.await().untilAsserted(() -> + Assert.assertEquals(producer.getState().toString(), HandlerState.State.Ready.toString(), + "The producer uses a different connection when reconnecting") + ); + } + } + + // cleanup. + producer.close(); + clientWith10ConPerBroker.close(); + admin.topics().delete(topicName); + } + @Test public void testDoubleIpAddress() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); + @Cleanup("shutdownNow") + ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( + new DefaultThreadFactory("test-pulsar-client-scheduled")); EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, new DefaultThreadFactory("test")); - ConnectionPool pool = spyWithClassAndConstructorArgs(ConnectionPool.class, conf, eventLoop); + ConnectionPool pool = spyWithClassAndConstructorArgs(ConnectionPool.class, InstrumentProvider.NOOP, conf, + eventLoop, scheduledExecutorService); conf.setServiceUrl(serviceUrl); PulsarClientImpl client = new PulsarClientImpl(conf, eventLoop, pool); @@ -109,7 +156,12 @@ public void testNoConnectionPool() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); conf.setConnectionsPerBroker(0); EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(8, false, new DefaultThreadFactory("test")); - ConnectionPool pool = spyWithClassAndConstructorArgs(ConnectionPool.class, conf, eventLoop); + @Cleanup("shutdownNow") + ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( + new DefaultThreadFactory("test-pulsar-client-scheduled")); + ConnectionPool pool = + spyWithClassAndConstructorArgs(ConnectionPool.class, InstrumentProvider.NOOP, conf, eventLoop, + scheduledExecutorService); InetSocketAddress brokerAddress = InetSocketAddress.createUnresolved("127.0.0.1", brokerPort); @@ -132,7 +184,12 @@ public void testEnableConnectionPool() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); conf.setConnectionsPerBroker(5); EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(8, false, new DefaultThreadFactory("test")); - ConnectionPool pool = spyWithClassAndConstructorArgs(ConnectionPool.class, conf, eventLoop); + @Cleanup("shutdownNow") + ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( + new DefaultThreadFactory("test-pulsar-client-scheduled")); + ConnectionPool pool = + spyWithClassAndConstructorArgs(ConnectionPool.class, InstrumentProvider.NOOP, conf, eventLoop, + scheduledExecutorService); InetSocketAddress brokerAddress = InetSocketAddress.createUnresolved("127.0.0.1", brokerPort); @@ -155,8 +212,9 @@ public void testEnableConnectionPool() throws Exception { public void testSetProxyToTargetBrokerAddress() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); conf.setConnectionsPerBroker(1); - - + @Cleanup("shutdownNow") + ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( + new DefaultThreadFactory("test-pulsar-client-scheduled")); EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(8, false, new DefaultThreadFactory("test")); @@ -199,26 +257,31 @@ protected void doResolveAll(SocketAddress socketAddress, Promise promise) throws } }; - ConnectionPool pool = spyWithClassAndConstructorArgs(ConnectionPool.class, conf, eventLoop, - (Supplier) () -> new ClientCnx(conf, eventLoop), Optional.of(resolver)); + ConnectionPool pool = + spyWithClassAndConstructorArgs(ConnectionPool.class, InstrumentProvider.NOOP, conf, eventLoop, + (Supplier) () -> new ClientCnx(InstrumentProvider.NOOP, conf, eventLoop), + Optional.of(resolver), scheduledExecutorService); ClientCnx cnx = pool.getConnection( InetSocketAddress.createUnresolved("proxy", 9999), - InetSocketAddress.createUnresolved("proxy", 9999)).get(); + InetSocketAddress.createUnresolved("proxy", 9999), + pool.genRandomKeyToSelectCon()).get(); Assert.assertEquals(cnx.remoteHostName, "proxy"); Assert.assertNull(cnx.proxyToTargetBrokerAddress); cnx = pool.getConnection( InetSocketAddress.createUnresolved("broker1", 9999), - InetSocketAddress.createUnresolved("proxy", 9999)).get(); + InetSocketAddress.createUnresolved("proxy", 9999), + pool.genRandomKeyToSelectCon()).get(); Assert.assertEquals(cnx.remoteHostName, "proxy"); Assert.assertEquals(cnx.proxyToTargetBrokerAddress, "broker1:9999"); cnx = pool.getConnection( InetSocketAddress.createUnresolved("broker2", 9999), - InetSocketAddress.createUnresolved("broker2", 9999)).get(); + InetSocketAddress.createUnresolved("broker2", 9999), + pool.genRandomKeyToSelectCon()).get(); Assert.assertEquals(cnx.remoteHostName, "broker2"); Assert.assertNull(cnx.proxyToTargetBrokerAddress); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/HierarchyTopicAutoCreationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/HierarchyTopicAutoCreationTest.java new file mode 100644 index 0000000000000..8ab94e29cfe6e --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/HierarchyTopicAutoCreationTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import lombok.Cleanup; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import java.util.List; +import java.util.UUID; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; +import org.apache.pulsar.common.policies.data.Policies; +import org.apache.pulsar.metadata.api.MetadataCache; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test(groups = "broker-impl") +@Slf4j +public class HierarchyTopicAutoCreationTest extends ProducerConsumerBase { + + @Override + @BeforeMethod + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @Override + @AfterMethod(alwaysRun = true) + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test(invocationCount = 3) + @SneakyThrows + public void testPartitionedTopicAutoCreation() { + // Create namespace + final String namespace = "public/testPartitionedTopicAutoCreation"; + admin.namespaces().createNamespace(namespace); + // Set policies + final AutoTopicCreationOverride expectedPolicies = AutoTopicCreationOverride.builder() + .allowAutoTopicCreation(true) + .topicType("partitioned") + .defaultNumPartitions(1) + .build(); + admin.namespaces().setAutoTopicCreation(namespace, expectedPolicies); + // Double-check the policies + final AutoTopicCreationOverride nsAutoTopicCreationOverride = admin.namespaces() + .getAutoTopicCreation(namespace); + Assert.assertEquals(nsAutoTopicCreationOverride, expectedPolicies); + // Background invalidate cache + final MetadataCache nsCache = pulsar.getPulsarResources().getNamespaceResources().getCache(); + @Cleanup("interrupt") + final Thread t1 = new Thread(() -> { + while (!Thread.currentThread().isInterrupted()) { + nsCache.invalidate("/admin/policies/" + namespace); + } + }); + t1.start(); + + // trigger auto-creation + final String topicName = "persistent://" + namespace + "/test-" + UUID.randomUUID(); + @Cleanup final Producer producer = pulsarClient.newProducer() + .topic(topicName) + .create(); + final List topics = admin.topics().getList(namespace); + Assert.assertEquals(topics.size(), 1); // expect only one topic + Assert.assertEquals(topics.get(0), + TopicName.get(topicName).getPartition(0).toString()); // expect partitioned topic + + // double-check policies + final AutoTopicCreationOverride actualPolicies2 = admin.namespaces().getAutoTopicCreation(namespace); + Assert.assertEquals(actualPolicies2, expectedPolicies); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeySharedSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeySharedSubscriptionTest.java index 6f7d8d9c3f181..7889b19e5b29e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeySharedSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeySharedSubscriptionTest.java @@ -146,15 +146,17 @@ public void testCanRecoverConsumptionWhenLiftMaxUnAckedMessagesRestriction(Subsc // Wait for all consumers to continue receiving messages. Awaitility.await() - .atMost(15, TimeUnit.SECONDS) + .atMost(30, TimeUnit.SECONDS) .pollDelay(5, TimeUnit.SECONDS) .until(() -> (System.currentTimeMillis() - lastActiveTime.get()) > TimeUnit.SECONDS.toMillis(5)); + logTopicStats(topic); + //Determine if all messages have been received. //If the dispatcher is stuck, we can not receive enough messages. - Assert.assertEquals(pubMessages.size(), totalMsg); - Assert.assertEquals(pubMessages.size(), recMessages.size()); + Assert.assertEquals(totalMsg, pubMessages.size()); + Assert.assertEquals(recMessages.size(), pubMessages.size()); Assert.assertTrue(recMessages.containsAll(pubMessages)); // cleanup diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeyStoreTlsProducerConsumerTestWithAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeyStoreTlsProducerConsumerTestWithAuthTest.java index 8e508b6cf2068..ec1cfb3a4c5f4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeyStoreTlsProducerConsumerTestWithAuthTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeyStoreTlsProducerConsumerTestWithAuthTest.java @@ -32,6 +32,7 @@ import io.jsonwebtoken.SignatureAlgorithm; import lombok.Cleanup; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.authentication.AuthenticationProviderTls; import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; @@ -49,6 +50,7 @@ import org.apache.pulsar.client.impl.auth.AuthenticationToken; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.util.ObjectMapperFactory; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -83,6 +85,7 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + @SneakyThrows protected void internalSetUpForBroker() { conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); @@ -114,6 +117,25 @@ protected void internalSetUpForBroker() { conf.setAuthenticationProviders(providers); conf.setNumExecutorThreadPoolSize(5); + Set tlsProtocols = Sets.newConcurrentHashSet(); + tlsProtocols.add("TLSv1.3"); + tlsProtocols.add("TLSv1.2"); + conf.setBrokerClientAuthenticationPlugin(AuthenticationKeyStoreTls.class.getName()); + Map authParams = new HashMap<>(); + authParams.put(AuthenticationKeyStoreTls.KEYSTORE_TYPE, KEYSTORE_TYPE); + authParams.put(AuthenticationKeyStoreTls.KEYSTORE_PATH, CLIENT_KEYSTORE_FILE_PATH); + authParams.put(AuthenticationKeyStoreTls.KEYSTORE_PW, CLIENT_KEYSTORE_PW); + conf.setBrokerClientAuthenticationParameters(ObjectMapperFactory.getMapper() + .getObjectMapper().writeValueAsString(authParams)); + conf.setBrokerClientTlsEnabled(true); + conf.setBrokerClientTlsEnabledWithKeyStore(true); + conf.setBrokerClientTlsTrustStore(BROKER_TRUSTSTORE_FILE_PATH); + conf.setBrokerClientTlsTrustStorePassword(BROKER_TRUSTSTORE_PW); + conf.setBrokerClientTlsKeyStore(CLIENT_KEYSTORE_FILE_PATH); + conf.setBrokerClientTlsKeyStoreType(KEYSTORE_TYPE); + conf.setBrokerClientTlsKeyStorePassword(CLIENT_KEYSTORE_PW); + conf.setBrokerClientTlsProtocols(tlsProtocols); + } protected void internalSetUpForClient(boolean addCertificates, String lookupUrl) throws Exception { @@ -148,10 +170,7 @@ protected void internalSetUpForNamespace() throws Exception { authParams.put(AuthenticationKeyStoreTls.KEYSTORE_PATH, CLIENT_KEYSTORE_FILE_PATH); authParams.put(AuthenticationKeyStoreTls.KEYSTORE_PW, CLIENT_KEYSTORE_PW); - if (admin != null) { - admin.close(); - } - + closeAdmin(); admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString()) .useKeyStoreTls(true) .tlsTrustStorePath(BROKER_TRUSTSTORE_FILE_PATH) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeyStoreTlsProducerConsumerTestWithoutAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeyStoreTlsProducerConsumerTestWithoutAuthTest.java index b5ec18b68f8b3..10ce45d247227 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeyStoreTlsProducerConsumerTestWithoutAuthTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeyStoreTlsProducerConsumerTestWithoutAuthTest.java @@ -118,10 +118,7 @@ protected void internalSetUpForNamespace() throws Exception { authParams.put(AuthenticationKeyStoreTls.KEYSTORE_PATH, CLIENT_KEYSTORE_FILE_PATH); authParams.put(AuthenticationKeyStoreTls.KEYSTORE_PW, CLIENT_KEYSTORE_PW); - if (admin != null) { - admin.close(); - } - + closeAdmin(); admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString()) .useKeyStoreTls(true) .tlsTrustStorePath(BROKER_TRUSTSTORE_FILE_PATH) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/LookupServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/LookupServiceTest.java new file mode 100644 index 0000000000000..59cb7ae03d0e3 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/LookupServiceTest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import static org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace.Mode; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import java.util.Collection; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicName; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Test(groups = "broker-admin") +@Slf4j +public class LookupServiceTest extends ProducerConsumerBase { + + private PulsarClientImpl clientWithHttpLookup; + private PulsarClientImpl clientWitBinaryLookup; + + private boolean enableBrokerSideSubscriptionPatternEvaluation = true; + private int subscriptionPatternMaxLength = 10_000; + + @Override + @BeforeClass + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + clientWithHttpLookup = + (PulsarClientImpl) PulsarClient.builder().serviceUrl(pulsar.getWebServiceAddress()).build(); + clientWitBinaryLookup = + (PulsarClientImpl) PulsarClient.builder().serviceUrl(pulsar.getBrokerServiceUrl()).build(); + } + + @Override + @AfterClass(alwaysRun = true) + protected void cleanup() throws Exception { + super.internalCleanup(); + if (clientWithHttpLookup != null) { + clientWithHttpLookup.close(); + } + if (clientWitBinaryLookup != null) { + clientWitBinaryLookup.close(); + } + } + + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + conf.setEnableBrokerSideSubscriptionPatternEvaluation(enableBrokerSideSubscriptionPatternEvaluation); + conf.setSubscriptionPatternMaxLength(subscriptionPatternMaxLength); + } + + private LookupService getLookupService(boolean isUsingHttpLookup) { + if (isUsingHttpLookup) { + return clientWithHttpLookup.getLookup(); + } else { + return clientWitBinaryLookup.getLookup(); + } + } + + @DataProvider(name = "isUsingHttpLookup") + public Object[][] isUsingHttpLookup() { + return new Object[][]{ + {true}, + {false} + }; + } + + @Test(dataProvider = "isUsingHttpLookup") + public void testGetTopicsOfGetTopicsResult(boolean isUsingHttpLookup) throws Exception { + LookupService lookupService = getLookupService(isUsingHttpLookup); + String nonPartitionedTopic = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createNonPartitionedTopic(nonPartitionedTopic); + String partitionedTopic = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createPartitionedTopic(partitionedTopic, 3); + String nonPersistentTopic = BrokerTestUtil.newUniqueName("non-persistent://public/default/tp"); + + // Verify the new method "GetTopicsResult.getTopics" works as expected. + Collection topics = lookupService.getTopicsUnderNamespace(NamespaceName.get("public/default"), + Mode.PERSISTENT, "public/default/.*", null).join().getTopics(); + assertTrue(topics.contains(nonPartitionedTopic)); + assertTrue(topics.contains(partitionedTopic)); + assertFalse(topics.contains(nonPersistentTopic)); + assertFalse(topics.contains(TopicName.get(partitionedTopic).getPartition(0).toString())); + // Verify the new method "GetTopicsResult.nonPartitionedOrPartitionTopics" works as expected. + Collection nonPartitionedOrPartitionTopics = + lookupService.getTopicsUnderNamespace(NamespaceName.get("public/default"), + Mode.PERSISTENT, "public/default/.*", null).join() + .getNonPartitionedOrPartitionTopics(); + assertTrue(nonPartitionedOrPartitionTopics.contains(nonPartitionedTopic)); + assertFalse(nonPartitionedOrPartitionTopics.contains(partitionedTopic)); + assertFalse(nonPartitionedOrPartitionTopics.contains(nonPersistentTopic)); + assertTrue(nonPartitionedOrPartitionTopics.contains(TopicName.get(partitionedTopic).getPartition(0) + .toString())); + assertTrue(nonPartitionedOrPartitionTopics.contains(TopicName.get(partitionedTopic).getPartition(1) + .toString())); + assertTrue(nonPartitionedOrPartitionTopics.contains(TopicName.get(partitionedTopic).getPartition(2) + .toString())); + + // Cleanup. + admin.topics().deletePartitionedTopic(partitionedTopic, false); + admin.topics().delete(nonPartitionedTopic, false); + } + +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChecksumTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChecksumTest.java index 515b34db8509d..94e763847506b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChecksumTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChecksumTest.java @@ -24,6 +24,8 @@ import static org.testng.Assert.fail; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; + +import java.lang.reflect.Method; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -37,6 +39,7 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.TypedMessageBuilder; import org.apache.pulsar.client.impl.ProducerImpl.OpSendMsg; +import org.apache.pulsar.client.impl.metrics.LatencyHistogram; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.api.proto.ProtocolVersion; import org.apache.pulsar.common.protocol.ByteBufPair; @@ -224,6 +227,9 @@ public void testTamperingMessageIsDetected() throws Exception { .create(); TypedMessageBuilderImpl msgBuilder = (TypedMessageBuilderImpl) producer.newMessage() .value("a message".getBytes()); + Method method = TypedMessageBuilderImpl.class.getDeclaredMethod("beforeSend"); + method.setAccessible(true); + method.invoke(msgBuilder); MessageMetadata msgMetadata = msgBuilder.getMetadataBuilder() .setProducerName("test") .setSequenceId(1) @@ -233,7 +239,7 @@ public void testTamperingMessageIsDetected() throws Exception { // WHEN // protocol message is created with checksum ByteBufPair cmd = Commands.newSend(1, 1, 1, ChecksumType.Crc32c, msgMetadata, payload); - OpSendMsg op = OpSendMsg.create((MessageImpl) msgBuilder.getMessage(), cmd, 1, null); + OpSendMsg op = OpSendMsg.create(LatencyHistogram.NOOP, (MessageImpl) msgBuilder.getMessage(), cmd, 1, null); // THEN // the checksum validation passes diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingDeduplicationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingDeduplicationTest.java new file mode 100644 index 0000000000000..5e590414132a5 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingDeduplicationTest.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import static org.apache.pulsar.client.impl.MessageChunkingSharedTest.sendChunk; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.Schema; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker-impl") +public class MessageChunkingDeduplicationTest extends ProducerConsumerBase { + + @BeforeClass + @Override + protected void setup() throws Exception { + this.conf.setBrokerDeduplicationEnabled(true); + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testSendChunkMessageWithSameSequenceID() throws Exception { + String topicName = "persistent://my-property/my-ns/testSendChunkMessageWithSameSequenceID"; + String producerName = "test-producer"; + @Cleanup + Consumer consumer = pulsarClient + .newConsumer(Schema.STRING) + .subscriptionName("test-sub") + .topic(topicName) + .subscribe(); + @Cleanup + Producer producer = pulsarClient + .newProducer(Schema.STRING) + .producerName(producerName) + .topic(topicName) + .enableChunking(true) + .enableBatching(false) + .create(); + int messageSize = 6000; // payload size in KB + String message = "a".repeat(messageSize * 1000); + producer.newMessage().value(message).sequenceId(10).send(); + Message msg = consumer.receive(10, TimeUnit.SECONDS); + assertNotNull(msg); + assertTrue(msg.getMessageId() instanceof ChunkMessageIdImpl); + assertEquals(msg.getValue(), message); + producer.newMessage().value(message).sequenceId(10).send(); + msg = consumer.receive(3, TimeUnit.SECONDS); + assertNull(msg); + } + + @Test + public void testDeduplicateChunksInSingleChunkMessages() throws Exception { + String topicName = "persistent://my-property/my-ns/testDeduplicateChunksInSingleChunkMessage"; + String producerName = "test-producer"; + @Cleanup + Consumer consumer = pulsarClient + .newConsumer(Schema.STRING) + .subscriptionName("test-sub") + .topic(topicName) + .subscribe(); + final PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService() + .getTopicIfExists(topicName).get().orElse(null); + assertNotNull(persistentTopic); + sendChunk(persistentTopic, producerName, 1, 0, 2); + sendChunk(persistentTopic, producerName, 1, 1, 2); + sendChunk(persistentTopic, producerName, 1, 1, 2); + + Message message = consumer.receive(15, TimeUnit.SECONDS); + assertEquals(message.getData().length, 2); + + sendChunk(persistentTopic, producerName, 2, 0, 3); + sendChunk(persistentTopic, producerName, 2, 1, 3); + sendChunk(persistentTopic, producerName, 2, 1, 3); + sendChunk(persistentTopic, producerName, 2, 2, 3); + message = consumer.receive(20, TimeUnit.SECONDS); + assertEquals(message.getData().length, 3); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingSharedTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingSharedTest.java index 163c42d835b35..3d24d3746d66a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingSharedTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingSharedTest.java @@ -23,6 +23,7 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -34,6 +35,7 @@ import java.util.concurrent.TimeUnit; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.ConsumerBuilder; @@ -45,7 +47,6 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; -import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.protocol.Commands; import org.awaitility.Awaitility; @@ -217,7 +218,7 @@ private static void sendNonChunk(final PersistentTopic persistentTopic, sendChunk(persistentTopic, producerName, sequenceId, null, null); } - private static void sendChunk(final PersistentTopic persistentTopic, + protected static void sendChunk(final PersistentTopic persistentTopic, final String producerName, final long sequenceId, final Integer chunkId, @@ -233,16 +234,33 @@ private static void sendChunk(final PersistentTopic persistentTopic, metadata.setTotalChunkMsgSize(numChunks); } final ByteBuf buf = Commands.serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, metadata, - PulsarByteBufAllocator.DEFAULT.buffer(1)); - persistentTopic.publishMessage(buf, (e, ledgerId, entryId) -> { - String name = producerName + "-" + sequenceId; - if (chunkId != null) { - name += "-" + chunkId + "-" + numChunks; + Unpooled.wrappedBuffer("a".getBytes())); + persistentTopic.publishMessage(buf, new Topic.PublishContext() { + @Override + public boolean isChunked() { + return chunkId != null; } - if (e == null) { - log.info("Sent {} to ({}, {})", name, ledgerId, entryId); - } else { - log.error("Failed to send {}: {}", name, e.getMessage()); + + @Override + public String getProducerName() { + return producerName; + } + + public long getSequenceId() { + return sequenceId; + } + + @Override + public void completed(Exception e, long ledgerId, long entryId) { + String name = producerName + "-" + sequenceId; + if (chunkId != null) { + name += "-" + chunkId + "-" + numChunks; + } + if (e == null) { + log.info("Sent {} to ({}, {})", name, ledgerId, entryId); + } else { + log.error("Failed to send {}: {}", name, e.getMessage()); + } } }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java index 2797d66f7fe21..8df5a38bb461c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java @@ -37,8 +37,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import lombok.Cleanup; +import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.RandomUtils; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.ClientBuilder; @@ -56,12 +56,14 @@ import org.apache.pulsar.client.api.SizeUnit; import org.apache.pulsar.client.impl.MessageImpl.SchemaState; import org.apache.pulsar.client.impl.ProducerImpl.OpSendMsg; +import org.apache.pulsar.client.impl.metrics.LatencyHistogram; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.policies.data.PublisherStats; import org.apache.pulsar.common.protocol.ByteBufPair; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.Commands.ChecksumType; import org.apache.pulsar.common.util.FutureUtil; +import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -162,7 +164,7 @@ public void testLargeMessage(boolean ackReceiptEnabled, boolean clientSizeMaxMes assertTrue(producerStats.getChunkedMessageRate() > 0); ManagedCursorImpl mcursor = (ManagedCursorImpl) topic.getManagedLedger().getCursors().iterator().next(); - PositionImpl readPosition = (PositionImpl) mcursor.getReadPosition(); + Position readPosition = mcursor.getReadPosition(); for (MessageId msgId : msgIds) { consumer.acknowledge(msgId); @@ -268,7 +270,7 @@ public void testLargeMessageAckTimeOut(boolean ackReceiptEnabled) throws Excepti } ManagedCursorImpl mcursor = (ManagedCursorImpl) topic.getManagedLedger().getCursors().iterator().next(); - PositionImpl readPosition = (PositionImpl) mcursor.getReadPosition(); + Position readPosition = mcursor.getReadPosition(); consumer.acknowledgeCumulative(lastMsgId); @@ -319,15 +321,29 @@ private void sendSingleChunk(Producer producer, String uuid, int chunkId msg.send(); } + /** + * This test used to test the consumer configuration of maxPendingChunkedMessage. + * If we set maxPendingChunkedMessage is 1 that means only one incomplete chunk message can be store in this + * consumer. + * For example: + * ChunkMessage1 chunk-1: uuid = 0, chunkId = 0, totalChunk = 2; + * ChunkMessage2 chunk-1: uuid = 1, chunkId = 0, totalChunk = 2; + * ChunkMessage2 chunk-2: uuid = 1, chunkId = 1, totalChunk = 2; + * ChunkMessage1 chunk-2: uuid = 0, chunkId = 1, totalChunk = 2; + * The chunk-1 in the ChunkMessage1 and ChunkMessage2 all is incomplete. + * chunk-1 in the ChunkMessage1 will be discarded and acked when receive the chunk-1 in the ChunkMessage2. + * If ack ChunkMessage2 and redeliver unacknowledged messages, the consumer can not receive any message again. + * @throws Exception + */ @Test public void testMaxPendingChunkMessages() throws Exception { log.info("-- Starting {} test --", methodName); final String topicName = "persistent://my-property/my-ns/maxPending"; - + final String subName = "my-subscriber-name"; @Cleanup Consumer consumer = pulsarClient.newConsumer(Schema.STRING) .topic(topicName) - .subscriptionName("my-subscriber-name") + .subscriptionName(subName) .maxPendingChunkedMessage(1) .autoAckOldestChunkedMessageOnQueueFull(true) .subscribe(); @@ -348,6 +364,8 @@ public void testMaxPendingChunkMessages() throws Exception { assertEquals(receivedMsg.getValue(), "chunk-1-0|chunk-1-1|"); consumer.acknowledge(receivedMsg); + Awaitility.await().untilAsserted(() -> assertEquals(admin.topics().getStats(topicName) + .getSubscriptions().get(subName).getNonContiguousDeletedMessagesRanges(), 0)); consumer.redeliverUnacknowledgedMessages(); sendSingleChunk(producer, "0", 1, 2); @@ -356,6 +374,80 @@ public void testMaxPendingChunkMessages() throws Exception { assertNull(consumer.receive(5, TimeUnit.SECONDS)); } + @Test + public void testResendChunkMessagesWithoutAckHole() throws Exception { + log.info("-- Starting {} test --", methodName); + final String topicName = "persistent://my-property/my-ns/testResendChunkMessagesWithoutAckHole"; + final String subName = "my-subscriber-name"; + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName(subName) + .maxPendingChunkedMessage(10) + .autoAckOldestChunkedMessageOnQueueFull(true) + .subscribe(); + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .chunkMaxMessageSize(100) + .enableChunking(true) + .enableBatching(false) + .create(); + + sendSingleChunk(producer, "0", 0, 2); + + sendSingleChunk(producer, "0", 0, 2); // Resending the first chunk + sendSingleChunk(producer, "0", 1, 2); + + Message receivedMsg = consumer.receive(5, TimeUnit.SECONDS); + assertEquals(receivedMsg.getValue(), "chunk-0-0|chunk-0-1|"); + consumer.acknowledge(receivedMsg); + assertEquals(admin.topics().getStats(topicName).getSubscriptions().get(subName) + .getNonContiguousDeletedMessagesRanges(), 0); + } + + @Test + public void testResendChunkMessages() throws Exception { + log.info("-- Starting {} test --", methodName); + final String topicName = "persistent://my-property/my-ns/testResendChunkMessages"; + + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName("my-subscriber-name") + .maxPendingChunkedMessage(10) + .autoAckOldestChunkedMessageOnQueueFull(true) + .subscribe(); + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .chunkMaxMessageSize(100) + .enableChunking(true) + .enableBatching(false) + .create(); + + sendSingleChunk(producer, "0", 0, 2); + + sendSingleChunk(producer, "0", 0, 2); // Resending the first chunk + sendSingleChunk(producer, "1", 0, 3); // This is for testing the interwoven chunked message + sendSingleChunk(producer, "1", 1, 3); + sendSingleChunk(producer, "1", 0, 3); // Resending the UUID-1 chunked message + + sendSingleChunk(producer, "0", 1, 2); + + Message receivedMsg = consumer.receive(5, TimeUnit.SECONDS); + assertEquals(receivedMsg.getValue(), "chunk-0-0|chunk-0-1|"); + consumer.acknowledge(receivedMsg); + + sendSingleChunk(producer, "1", 1, 3); + sendSingleChunk(producer, "1", 2, 3); + + receivedMsg = consumer.receive(5, TimeUnit.SECONDS); + assertEquals(receivedMsg.getValue(), "chunk-1-0|chunk-1-1|chunk-1-2|"); + consumer.acknowledge(receivedMsg); + Assert.assertEquals(((ConsumerImpl) consumer).getAvailablePermits(), 8); + } + /** * Validate that chunking is not supported with batching and non-persistent topic * @@ -408,7 +500,7 @@ public void testExpireIncompleteChunkMessage() throws Exception{ ByteBufPair cmd = Commands.newSend(producerId, 1, 1, ChecksumType.Crc32c, msgMetadata, payload); MessageImpl msgImpl = ((MessageImpl) msg.getMessage()); msgImpl.setSchemaState(SchemaState.Ready); - OpSendMsg op = OpSendMsg.create(msgImpl, cmd, 1, null); + OpSendMsg op = OpSendMsg.create(LatencyHistogram.NOOP, msgImpl, cmd, 1, null); producer.processOpSendMsg(op); retryStrategically((test) -> { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageParserTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageParserTest.java index 772ddbee4e54d..7b87efa7cae39 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageParserTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageParserTest.java @@ -28,7 +28,7 @@ import lombok.Cleanup; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedCursor; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.CompressionType; @@ -95,7 +95,7 @@ public void testParseMessages(boolean batchEnabled, CompressionType compressionT .create(); ManagedCursor cursor = ((PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get()) - .getManagedLedger().newNonDurableCursor(PositionImpl.EARLIEST); + .getManagedLedger().newNonDurableCursor(PositionFactory.EARLIEST); if (batchEnabled) { for (int i = 0; i < n - 1; i++) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessagePublishThrottlingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessagePublishThrottlingTest.java index ad1955f08d779..1c0ae5547d53b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessagePublishThrottlingTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessagePublishThrottlingTest.java @@ -18,9 +18,6 @@ */ package org.apache.pulsar.client.impl; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotEquals; -import static org.testng.Assert.assertNotSame; import static org.testng.Assert.assertTrue; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -34,14 +31,12 @@ import java.util.concurrent.atomic.AtomicInteger; import lombok.Cleanup; import org.apache.pulsar.broker.service.Producer; -import org.apache.pulsar.broker.service.PublishRateLimiter; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.ProducerConsumerBase; import org.apache.pulsar.common.policies.data.PublishRate; -import org.awaitility.Awaitility; +import org.apache.pulsar.broker.qos.AsyncTokenBucket; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -53,6 +48,7 @@ public class MessagePublishThrottlingTest extends ProducerConsumerBase { @BeforeMethod @Override protected void setup() throws Exception { + AsyncTokenBucket.switchToConsistentTokensView(); this.conf.setClusterName("test"); this.conf.setTopicPublisherThrottlingTickTimeMillis(1); this.conf.setBrokerPublisherThrottlingTickTimeMillis(1); @@ -64,6 +60,7 @@ protected void setup() throws Exception { @Override protected void cleanup() throws Exception { super.internalCleanup(); + AsyncTokenBucket.resetToDefaultEventualConsistentTokensView(); } /** @@ -86,16 +83,9 @@ public void testSimplePublishMessageThrottling() throws Exception { ProducerImpl producer = (ProducerImpl) pulsarClient.newProducer().topic(topicName) .maxPendingMessages(30000).create(); PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicIfExists(topicName).get().get(); - // (1) verify message-rate is -1 initially - Assert.assertEquals(topic.getTopicPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER); // enable throttling admin.namespaces().setPublishRate(namespace, publishMsgRate); - retryStrategically((test) -> - !topic.getTopicPublishRateLimiter().equals(PublishRateLimiter.DISABLED_RATE_LIMITER), - 5, - 200); - Assert.assertNotEquals(topic.getTopicPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER); Producer prod = topic.getProducers().values().iterator().next(); // reset counter @@ -112,11 +102,6 @@ public void testSimplePublishMessageThrottling() throws Exception { // disable throttling publishMsgRate.publishThrottlingRateInMsg = -1; admin.namespaces().setPublishRate(namespace, publishMsgRate); - retryStrategically((test) -> - topic.getTopicPublishRateLimiter().equals(PublishRateLimiter.DISABLED_RATE_LIMITER), - 5, - 200); - Assert.assertEquals(topic.getTopicPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER); // reset counter prod.updateRates(); @@ -150,16 +135,9 @@ public void testSimplePublishByteThrottling() throws Exception { // create producer and topic ProducerImpl producer = (ProducerImpl) pulsarClient.newProducer().topic(topicName).create(); PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get(); - // (1) verify message-rate is -1 initially - Assert.assertEquals(topic.getTopicPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER); // enable throttling admin.namespaces().setPublishRate(namespace, publishMsgRate); - retryStrategically((test) -> - !topic.getTopicPublishRateLimiter().equals(PublishRateLimiter.DISABLED_RATE_LIMITER), - 5, - 200); - Assert.assertNotEquals(topic.getTopicPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER); Producer prod = topic.getProducers().values().iterator().next(); // reset counter @@ -176,9 +154,6 @@ public void testSimplePublishByteThrottling() throws Exception { // disable throttling publishMsgRate.publishThrottlingRateInByte = -1; admin.namespaces().setPublishRate(namespace, publishMsgRate); - retryStrategically((test) -> topic.getTopicPublishRateLimiter().equals(PublishRateLimiter.DISABLED_RATE_LIMITER), 5, - 200); - Assert.assertEquals(topic.getTopicPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER); // reset counter prod.updateRates(); @@ -214,8 +189,6 @@ public void testBrokerPublishMessageThrottling() throws Exception { .enableBatching(false) .maxPendingMessages(30000).create(); PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicIfExists(topicName).get().get(); - // (1) verify message-rate is -1 initially - Assert.assertEquals(topic.getBrokerPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER); // enable throttling admin.brokers(). @@ -223,19 +196,11 @@ public void testBrokerPublishMessageThrottling() throws Exception { "brokerPublisherThrottlingMaxMessageRate", Integer.toString(messageRate)); - retryStrategically( - (test) -> - (topic.getBrokerPublishRateLimiter() != PublishRateLimiter.DISABLED_RATE_LIMITER), - 5, - 200); - log.info("Get broker configuration: brokerTick {}, MaxMessageRate {}, MaxByteRate {}", pulsar.getConfiguration().getBrokerPublisherThrottlingTickTimeMillis(), pulsar.getConfiguration().getBrokerPublisherThrottlingMaxMessageRate(), pulsar.getConfiguration().getBrokerPublisherThrottlingMaxByteRate()); - Assert.assertNotEquals(topic.getBrokerPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER); - Producer prod = topic.getProducers().values().iterator().next(); // reset counter prod.updateRates(); @@ -252,11 +217,6 @@ public void testBrokerPublishMessageThrottling() throws Exception { // disable throttling admin.brokers() .updateDynamicConfiguration("brokerPublisherThrottlingMaxMessageRate", Integer.toString(0)); - retryStrategically((test) -> - topic.getBrokerPublishRateLimiter().equals(PublishRateLimiter.DISABLED_RATE_LIMITER), - 5, - 200); - Assert.assertEquals(topic.getBrokerPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER); // reset counter prod.updateRates(); @@ -293,26 +253,16 @@ public void testBrokerPublishByteThrottling() throws Exception { .enableBatching(false) .maxPendingMessages(30000).create(); PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicIfExists(topicName).get().get(); - // (1) verify byte-rate is -1 disabled - Assert.assertEquals(topic.getBrokerPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER); // enable throttling admin.brokers() .updateDynamicConfiguration("brokerPublisherThrottlingMaxByteRate", Long.toString(byteRate)); - retryStrategically( - (test) -> - (topic.getBrokerPublishRateLimiter() != PublishRateLimiter.DISABLED_RATE_LIMITER), - 5, - 200); - log.info("Get broker configuration after enable: brokerTick {}, MaxMessageRate {}, MaxByteRate {}", pulsar.getConfiguration().getBrokerPublisherThrottlingTickTimeMillis(), pulsar.getConfiguration().getBrokerPublisherThrottlingMaxMessageRate(), pulsar.getConfiguration().getBrokerPublisherThrottlingMaxByteRate()); - Assert.assertNotEquals(topic.getBrokerPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER); - Producer prod = topic.getProducers().values().iterator().next(); // reset counter prod.updateRates(); @@ -331,18 +281,12 @@ public void testBrokerPublishByteThrottling() throws Exception { // disable throttling admin.brokers() .updateDynamicConfiguration("brokerPublisherThrottlingMaxByteRate", Long.toString(0)); - retryStrategically((test) -> - topic.getBrokerPublishRateLimiter().equals(PublishRateLimiter.DISABLED_RATE_LIMITER), - 5, - 200); log.info("Get broker configuration after disable: brokerTick {}, MaxMessageRate {}, MaxByteRate {}", pulsar.getConfiguration().getBrokerPublisherThrottlingTickTimeMillis(), pulsar.getConfiguration().getBrokerPublisherThrottlingMaxMessageRate(), pulsar.getConfiguration().getBrokerPublisherThrottlingMaxByteRate()); - Assert.assertEquals(topic.getBrokerPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER); - // reset counter prod.updateRates(); for (int i = 0; i < numMessage; i++) { @@ -385,21 +329,12 @@ public void testBrokerTopicPublishByteThrottling() throws Exception { .enableBatching(false) .maxPendingMessages(30000).create(); PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicIfExists(topicName).get().get(); - // (1) verify both broker and topic limiter is disabled - Assert.assertEquals(topic.getBrokerPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER); - Assert.assertEquals(topic.getTopicPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER); // enable broker and topic throttling admin.namespaces().setPublishRate(namespace, topicPublishMsgRate); - Awaitility.await().untilAsserted(() -> { - assertNotEquals(topic.getTopicPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER); - }); admin.brokers().updateDynamicConfiguration("brokerPublisherThrottlingMaxByteRate", Long.toString(brokerByteRate)); - Awaitility.await().untilAsserted(() -> { - assertNotSame(topic.getBrokerPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER); - }); log.info("Get broker configuration after enable: brokerTick {}, MaxMessageRate {}, MaxByteRate {}", pulsar.getConfiguration().getBrokerPublisherThrottlingTickTimeMillis(), @@ -440,15 +375,7 @@ public void testBrokerTopicPublishByteThrottling() throws Exception { producers.add(iProducer); topics.add(iTopic); - // verify both broker and topic limiter is enabled - Assert.assertNotEquals(iTopic.getBrokerPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER); - admin.namespaces().setPublishRate(namespace, topicPublishMsgRate); - retryStrategically((test) -> - !iTopic.getTopicPublishRateLimiter().equals(PublishRateLimiter.DISABLED_RATE_LIMITER), - 5, - 200); - Assert.assertNotEquals(iTopic.getTopicPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER); } List> topicRatesCounter = Lists.newArrayListWithExpectedSize(3); @@ -486,10 +413,6 @@ public void testBrokerTopicPublishByteThrottling() throws Exception { topicPublishMsgRate.publishThrottlingRateInByte = -1; admin.namespaces().setPublishRate(namespace, topicPublishMsgRate); - Awaitility.await().untilAsserted(() -> - assertEquals(topic.getTopicPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER) - ); - // reset counter prod.updateRates(); for (int i = 0; i < numMessage; i++) { @@ -505,18 +428,12 @@ public void testBrokerTopicPublishByteThrottling() throws Exception { // disable broker throttling, expected no throttling. admin.brokers() .updateDynamicConfiguration("brokerPublisherThrottlingMaxByteRate", Long.toString(0)); - retryStrategically((test) -> - topic.getBrokerPublishRateLimiter().equals(PublishRateLimiter.DISABLED_RATE_LIMITER), - 5, - 200); log.info("Get broker configuration after disable: brokerTick {}, MaxMessageRate {}, MaxByteRate {}", pulsar.getConfiguration().getBrokerPublisherThrottlingTickTimeMillis(), pulsar.getConfiguration().getBrokerPublisherThrottlingMaxMessageRate(), pulsar.getConfiguration().getBrokerPublisherThrottlingMaxByteRate()); - Assert.assertEquals(topic.getBrokerPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER); - // reset counter prod.updateRates(); for (int i = 0; i < numMessage; i++) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageRedeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageRedeliveryTest.java index 29b06f68b64eb..e2895b1d01e9f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageRedeliveryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageRedeliveryTest.java @@ -18,11 +18,16 @@ */ package org.apache.pulsar.client.impl; -import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import com.google.common.collect.Sets; +import io.netty.util.concurrent.DefaultThreadFactory; +import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -37,6 +42,8 @@ import org.apache.pulsar.client.api.BatchReceivePolicy; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageListener; import org.apache.pulsar.client.api.Messages; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerConsumerBase; @@ -49,8 +56,6 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import com.google.common.collect.Sets; -import io.netty.util.concurrent.DefaultThreadFactory; @Test(groups = "broker-impl") public class MessageRedeliveryTest extends ProducerConsumerBase { @@ -539,4 +544,57 @@ public void testMultiConsumerBatchRedeliveryAddEpoch(boolean enableBatch) throws // can't receive message again assertEquals(consumer.batchReceive().size(), 0); } + + /** + * This test validates that client lib correctly increases permits of individual consumer to retrieve data in case + * of incorrect epoch for partition-topic multi-consumer. + * + * @throws Exception + */ + @Test + public void testRedeliveryWithMultiConsumerAndListenerAddEpoch() throws Exception { + final String topic = "testRedeliveryWithMultiConsumerAndListenerAddEpoch"; + final String subName = "my-sub"; + int totalMessages = 100; + admin.topics().createPartitionedTopic(topic, 2); + + Map ids = new ConcurrentHashMap<>(); + CountDownLatch latch = new CountDownLatch(totalMessages); + MessageListener msgListener = (Consumer consumer, Message msg) -> { + String id = msg.getMessageId().toString(); + consumer.acknowledgeCumulativeAsync(msg); + if (ids.put(msg.getMessageId(), id) == null) { + latch.countDown(); + } + }; + @Cleanup + Consumer newConsumer = pulsarClient.newConsumer(Schema.STRING).topic(topic).subscriptionName(subName) + .messageListener(msgListener).subscriptionType(SubscriptionType.Failover) + .receiverQueueSize(totalMessages / 10).subscribe(); + + MultiTopicsConsumerImpl consumer = (MultiTopicsConsumerImpl) newConsumer; + long epoch = consumer.getConsumerEpoch() + 1; + consumer.setConsumerEpoch(epoch); + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).enableBatching(false) + .create(); + + for (int i = 0; i < totalMessages; i++) { + producer.sendAsync("test" + i); + } + producer.flush(); + + // make sure listener has not received any messages until + // we call redelivery with correct epoch + for (int i = 0; i < 2; i++) { + assertTrue(ids.isEmpty()); + Thread.sleep(1000); + } + // make epoch valid to consume redelivery message again + consumer.setConsumerEpoch(epoch - 1); + consumer.redeliverUnacknowledgedMessages(); + + latch.await(10, TimeUnit.SECONDS); + assertEquals(ids.size(), totalMessages); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MultiTopicsReaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MultiTopicsReaderTest.java index a41aac9bd457f..d9bbc6a9d742a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MultiTopicsReaderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MultiTopicsReaderTest.java @@ -211,7 +211,7 @@ public void testReadMessageWithBatchingWithMessageInclusive() throws Exception { reader.close(); } - @Test(timeOut = 10000) + @Test public void testReaderWithTimeLong() throws Exception { String ns = "my-property/my-ns"; String topic = "persistent://" + ns + "/testReadFromPartition" + UUID.randomUUID(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/NegativeAcksTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/NegativeAcksTest.java index b4d01e263bc7a..a41b7f05a8eb3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/NegativeAcksTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/NegativeAcksTest.java @@ -21,7 +21,6 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; - import java.util.HashSet; import java.util.Set; import java.util.concurrent.CountDownLatch; @@ -39,7 +38,7 @@ import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.TopicMessageId; import org.awaitility.Awaitility; -import org.testcontainers.shaded.org.awaitility.reflect.WhiteboxImpl; +import org.awaitility.reflect.WhiteboxImpl; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -135,7 +134,7 @@ public void testNegativeAcks(boolean batching, boolean usePartitions, Subscripti Set sentMessages = new HashSet<>(); final int N = 10; - for (int i = 0; i < N; i++) { + for (int i = 0; i < N * 2; i++) { String value = "test-" + i; producer.sendAsync(value); sentMessages.add(value); @@ -147,10 +146,18 @@ public void testNegativeAcks(boolean batching, boolean usePartitions, Subscripti consumer.negativeAcknowledge(msg); } + for (int i = 0; i < N; i++) { + Message msg = consumer.receive(); + consumer.negativeAcknowledge(msg.getMessageId()); + } + + assertTrue(consumer instanceof ConsumerBase); + assertEquals(((ConsumerBase) consumer).getUnAckedMessageTracker().size(), 0); + Set receivedMessages = new HashSet<>(); // All the messages should be received again - for (int i = 0; i < N; i++) { + for (int i = 0; i < N * 2; i++) { Message msg = consumer.receive(); receivedMessages.add(msg.getValue()); consumer.acknowledge(msg); @@ -308,9 +315,7 @@ public void testNegativeAcksDeleteFromUnackedTracker() throws Exception { assertEquals(unAckedMessageTracker.size(), 0); negativeAcksTracker.close(); // negative batch message id - unAckedMessageTracker.add(batchMessageId); - unAckedMessageTracker.add(batchMessageId2); - unAckedMessageTracker.add(batchMessageId3); + unAckedMessageTracker.add(messageId); consumer.negativeAcknowledge(batchMessageId); consumer.negativeAcknowledge(batchMessageId2); consumer.negativeAcknowledge(batchMessageId3); @@ -319,7 +324,7 @@ public void testNegativeAcksDeleteFromUnackedTracker() throws Exception { negativeAcksTracker.close(); } - @Test(timeOut = 10000) + @Test public void testNegativeAcksWithBatchAckEnabled() throws Exception { cleanup(); conf.setAcknowledgmentAtBatchIndexLevelEnabled(true); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java index 76936334eb0ba..15cfb2f5654de 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java @@ -30,7 +30,6 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import java.util.stream.IntStream; @@ -63,7 +62,6 @@ import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TenantOperation; import org.apache.pulsar.common.policies.data.TopicOperation; -import org.apache.pulsar.common.util.RestException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -85,6 +83,7 @@ public void setup() throws Exception { // set isTcpLookup = true, to use BinaryProtoLookupService to get topics for a pattern. isTcpLookup = true; + conf.setTopicLevelPoliciesEnabled(false); conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(true); @@ -205,7 +204,7 @@ public void testBinaryProtoToGetTopicsOfNamespace() throws Exception { assertTrue(consumer.getTopic().startsWith(PatternMultiTopicsConsumerImpl.DUMMY_TOPIC_NAME_PREFIX)); // 4. verify consumer - assertSame(pattern, ((PatternMultiTopicsConsumerImpl) consumer).getPattern()); + assertSame(pattern.pattern(), ((PatternMultiTopicsConsumerImpl) consumer).getPattern().pattern()); List topics = ((PatternMultiTopicsConsumerImpl) consumer).getPartitions(); List> consumers = ((PatternMultiTopicsConsumerImpl) consumer).getConsumers(); @@ -331,12 +330,6 @@ public CompletableFuture allowTenantOperationAsync( return CompletableFuture.completedFuture(true); } - @Override - public Boolean allowTenantOperation( - String tenantName, String role, TenantOperation operation, AuthenticationDataSource authData) { - return true; - } - @Override public CompletableFuture allowNamespaceOperationAsync( NamespaceName namespaceName, String role, NamespaceOperation operation, AuthenticationDataSource authData) { @@ -351,16 +344,6 @@ public CompletableFuture allowNamespaceOperationAsync( return isAuthorizedFuture; } - @Override - public Boolean allowNamespaceOperation( - NamespaceName namespaceName, String role, NamespaceOperation operation, AuthenticationDataSource authData) { - try { - return allowNamespaceOperationAsync(namespaceName, role, operation, authData).get(); - } catch (InterruptedException | ExecutionException e) { - throw new RestException(e); - } - } - @Override public CompletableFuture allowTopicOperationAsync( TopicName topic, String role, TopicOperation operation, AuthenticationDataSource authData) { @@ -375,16 +358,6 @@ public CompletableFuture allowTopicOperationAsync( return isAuthorizedFuture; } - @Override - public Boolean allowTopicOperation( - TopicName topicName, String role, TopicOperation operation, AuthenticationDataSource authData) { - try { - return allowTopicOperationAsync(topicName, role, operation, authData).get(); - } catch (InterruptedException | ExecutionException e) { - throw new RestException(e); - } - } - @Override public CompletableFuture allowTopicPolicyOperationAsync(TopicName topic, String role, PolicyName policy, PolicyOperation operation, @@ -399,16 +372,6 @@ public CompletableFuture allowTopicPolicyOperationAsync(TopicName topic return isAuthorizedFuture; } - - @Override - public Boolean allowTopicPolicyOperation(TopicName topicName, String role, PolicyName policy, - PolicyOperation operation, AuthenticationDataSource authData) { - try { - return allowTopicPolicyOperationAsync(topicName, role, policy, operation, authData).get(); - } catch (InterruptedException | ExecutionException e) { - throw new RestException(e); - } - } } public static class ClientAuthentication implements Authentication { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java index 09a9a003f3ec5..4823426c8b83a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java @@ -26,6 +26,9 @@ import com.google.common.collect.Lists; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -33,23 +36,33 @@ import java.util.regex.Pattern; import java.util.stream.IntStream; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.InjectedClientCnxClientBuilder; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.RegexSubscriptionMode; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; +import org.apache.pulsar.common.api.proto.BaseCommand; +import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace; +import org.apache.pulsar.common.api.proto.CommandWatchTopicListSuccess; import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @Test(groups = "broker-impl") @@ -65,6 +78,7 @@ public void setup() throws Exception { isTcpLookup = true; // enabled transaction, to test pattern consumers not subscribe to transaction system topic. conf.setTransactionCoordinatorEnabled(true); + conf.setSubscriptionPatternMaxLength(10000); super.internalSetup(); super.producerBaseSetup(); } @@ -208,8 +222,14 @@ public void testBinaryProtoToGetTopicsOfNamespacePersistent() throws Exception { .subscribe(); assertTrue(consumer.getTopic().startsWith(PatternMultiTopicsConsumerImpl.DUMMY_TOPIC_NAME_PREFIX)); + // Wait topic list watcher creation. + Awaitility.await().untilAsserted(() -> { + CompletableFuture completableFuture = WhiteboxImpl.getInternalState(consumer, "watcherFuture"); + assertTrue(completableFuture.isDone() && !completableFuture.isCompletedExceptionally()); + }); + // 4. verify consumer get methods, to get right number of partitions and topics. - assertSame(pattern, ((PatternMultiTopicsConsumerImpl) consumer).getPattern()); + assertSame(pattern.pattern(), ((PatternMultiTopicsConsumerImpl) consumer).getPattern().pattern()); List topics = ((PatternMultiTopicsConsumerImpl) consumer).getPartitions(); List> consumers = ((PatternMultiTopicsConsumerImpl) consumer).getConsumers(); @@ -285,8 +305,14 @@ public void testBinaryProtoSubscribeAllTopicOfNamespace() throws Exception { .subscribe(); assertTrue(consumer.getTopic().startsWith(PatternMultiTopicsConsumerImpl.DUMMY_TOPIC_NAME_PREFIX)); + // Wait topic list watcher creation. + Awaitility.await().untilAsserted(() -> { + CompletableFuture completableFuture = WhiteboxImpl.getInternalState(consumer, "watcherFuture"); + assertTrue(completableFuture.isDone() && !completableFuture.isCompletedExceptionally()); + }); + // 4. verify consumer get methods, to get right number of partitions and topics. - assertSame(pattern, ((PatternMultiTopicsConsumerImpl) consumer).getPattern()); + assertSame(pattern.pattern(), ((PatternMultiTopicsConsumerImpl) consumer).getPattern().pattern()); List topics = ((PatternMultiTopicsConsumerImpl) consumer).getPartitions(); List> consumers = ((PatternMultiTopicsConsumerImpl) consumer).getConsumers(); @@ -362,8 +388,14 @@ public void testBinaryProtoToGetTopicsOfNamespaceNonPersistent() throws Exceptio .subscriptionTopicsMode(RegexSubscriptionMode.NonPersistentOnly) .subscribe(); + // Wait topic list watcher creation. + Awaitility.await().untilAsserted(() -> { + CompletableFuture completableFuture = WhiteboxImpl.getInternalState(consumer, "watcherFuture"); + assertTrue(completableFuture.isDone() && !completableFuture.isCompletedExceptionally()); + }); + // 4. verify consumer get methods, to get right number of partitions and topics. - assertSame(pattern, ((PatternMultiTopicsConsumerImpl) consumer).getPattern()); + assertSame(pattern.pattern(), ((PatternMultiTopicsConsumerImpl) consumer).getPattern().pattern()); List topics = ((PatternMultiTopicsConsumerImpl) consumer).getPartitions(); List> consumers = ((PatternMultiTopicsConsumerImpl) consumer).getConsumers(); @@ -453,8 +485,14 @@ public void testBinaryProtoToGetTopicsOfNamespaceAll() throws Exception { .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS) .subscribe(); + // Wait topic list watcher creation. + Awaitility.await().untilAsserted(() -> { + CompletableFuture completableFuture = WhiteboxImpl.getInternalState(consumer, "watcherFuture"); + assertTrue(completableFuture.isDone() && !completableFuture.isCompletedExceptionally()); + }); + // 4. verify consumer get methods, to get right number of partitions and topics. - assertSame(pattern, ((PatternMultiTopicsConsumerImpl) consumer).getPattern()); + assertSame(pattern.pattern(), ((PatternMultiTopicsConsumerImpl) consumer).getPattern().pattern()); List topics = ((PatternMultiTopicsConsumerImpl) consumer).getPartitions(); List> consumers = ((PatternMultiTopicsConsumerImpl) consumer).getConsumers(); @@ -514,7 +552,7 @@ public void testStartEmptyPatternConsumer() throws Exception { admin.topics().createPartitionedTopic(topicName2, 2); admin.topics().createPartitionedTopic(topicName3, 3); - // 2. Create consumer, this should success, but with empty sub-consumser internal + // 2. Create consumer, this should success, but with empty sub-consumer internal Consumer consumer = pulsarClient.newConsumer() .topicsPattern(pattern) .patternAutoDiscoveryPeriod(2) @@ -523,9 +561,14 @@ public void testStartEmptyPatternConsumer() throws Exception { .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS) .receiverQueueSize(4) .subscribe(); + // Wait topic list watcher creation. + Awaitility.await().untilAsserted(() -> { + CompletableFuture completableFuture = WhiteboxImpl.getInternalState(consumer, "watcherFuture"); + assertTrue(completableFuture.isDone() && !completableFuture.isCompletedExceptionally()); + }); // 3. verify consumer get methods, to get 5 number of partitions and topics. - assertSame(pattern, ((PatternMultiTopicsConsumerImpl) consumer).getPattern()); + assertSame(pattern.pattern(), ((PatternMultiTopicsConsumerImpl) consumer).getPattern().pattern()); assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitions().size(), 5); assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getConsumers().size(), 5); assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitionedTopics().size(), 2); @@ -554,7 +597,7 @@ public void testStartEmptyPatternConsumer() throws Exception { // 6. verify consumer get methods, to get number of partitions and topics, value 6=1+2+3. Awaitility.await().untilAsserted(() -> { - assertSame(pattern, ((PatternMultiTopicsConsumerImpl) consumer).getPattern()); + assertSame(pattern.pattern(), ((PatternMultiTopicsConsumerImpl) consumer).getPattern().pattern()); assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitions().size(), 6); assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getConsumers().size(), 6); assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitionedTopics().size(), 2); @@ -587,13 +630,28 @@ public void testStartEmptyPatternConsumer() throws Exception { producer3.close(); } - @Test(timeOut = testTimeout) - public void testAutoSubscribePatterConsumerFromBrokerWatcher() throws Exception { - String key = "AutoSubscribePatternConsumer"; - String subscriptionName = "my-ex-subscription-" + key; + @DataProvider(name= "delayTypesOfWatchingTopics") + public Object[][] delayTypesOfWatchingTopics(){ + return new Object[][]{ + {true}, + {false} + }; + } - Pattern pattern = Pattern.compile("persistent://my-property/my-ns/pattern-topic.*"); - Consumer consumer = pulsarClient.newConsumer() + @Test(timeOut = testTimeout, dataProvider = "delayTypesOfWatchingTopics") + public void testAutoSubscribePatterConsumerFromBrokerWatcher(boolean delayWatchingTopics) throws Exception { + final String key = "AutoSubscribePatternConsumer"; + final String subscriptionName = "my-ex-subscription-" + key; + final Pattern pattern = Pattern.compile("persistent://my-property/my-ns/pattern-topic.*"); + + PulsarClient client = null; + if (delayWatchingTopics) { + client = createDelayWatchTopicsClient(); + } else { + client = pulsarClient; + } + + Consumer consumer = client.newConsumer() .topicsPattern(pattern) // Disable automatic discovery. .patternAutoDiscoveryPeriod(1000) @@ -611,14 +669,169 @@ public void testAutoSubscribePatterConsumerFromBrokerWatcher() throws Exception // 2. verify consumer get methods. There is no need to trigger discovery, because the broker will push the // changes to update(CommandWatchTopicUpdate). - assertSame(pattern, ((PatternMultiTopicsConsumerImpl) consumer).getPattern()); + assertSame(pattern.pattern(), ((PatternMultiTopicsConsumerImpl) consumer).getPattern().pattern()); Awaitility.await().untilAsserted(() -> { assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitions().size(), 4); assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getConsumers().size(), 4); assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitionedTopics().size(), 1); }); + // cleanup. + consumer.close(); + admin.topics().deletePartitionedTopic(topicName); + if (delayWatchingTopics) { + client.close(); + } + } + + @DataProvider(name= "regexpConsumerArgs") + public Object[][] regexpConsumerArgs(){ + return new Object[][]{ + {true, true}, + {true, false}, + {false, true}, + {false, false} + }; + } + + private void waitForTopicListWatcherStarted(Consumer consumer) { + Awaitility.await().untilAsserted(() -> { + CompletableFuture completableFuture = WhiteboxImpl.getInternalState(consumer, "watcherFuture"); + log.info("isDone: {}, isCompletedExceptionally: {}", completableFuture.isDone(), + completableFuture.isCompletedExceptionally()); + assertTrue(completableFuture.isDone() && !completableFuture.isCompletedExceptionally()); + }); + } + + @Test(timeOut = testTimeout, dataProvider = "regexpConsumerArgs") + public void testPreciseRegexpSubscribe(boolean partitioned, boolean createTopicAfterWatcherStarted) throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final String subscriptionName = "s1"; + final Pattern pattern = Pattern.compile(String.format("%s$", topicName)); + + Consumer consumer = pulsarClient.newConsumer() + .topicsPattern(pattern) + // Disable automatic discovery. + .patternAutoDiscoveryPeriod(1000) + .subscriptionName(subscriptionName) + .subscriptionType(SubscriptionType.Shared) + .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS) + .receiverQueueSize(4) + .subscribe(); + if (createTopicAfterWatcherStarted) { + waitForTopicListWatcherStarted(consumer); + } + + // 1. create topic. + if (partitioned) { + admin.topics().createPartitionedTopic(topicName, 1); + } else { + admin.topics().createNonPartitionedTopic(topicName); + } + + // 2. verify consumer can subscribe the topic. + assertSame(pattern.pattern(), ((PatternMultiTopicsConsumerImpl) consumer).getPattern().pattern()); + Awaitility.await().untilAsserted(() -> { + assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitions().size(), 1); + assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getConsumers().size(), 1); + if (partitioned) { + assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitionedTopics().size(), 1); + } else { + assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitionedTopics().size(), 0); + } + }); + + // cleanup. consumer.close(); + if (partitioned) { + admin.topics().deletePartitionedTopic(topicName); + } else { + admin.topics().delete(topicName); + } + } + + @DataProvider(name= "partitioned") + public Object[][] partitioned(){ + return new Object[][]{ + {true}, + {true} + }; + } + + @Test(timeOut = 240 * 1000, dataProvider = "partitioned") + public void testPreciseRegexpSubscribeDisabledTopicWatcher(boolean partitioned) throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final String subscriptionName = "s1"; + final Pattern pattern = Pattern.compile(String.format("%s$", topicName)); + + // Close all ServerCnx by close client-side sockets to make the config changes effect. + pulsar.getConfig().setEnableBrokerSideSubscriptionPatternEvaluation(false); + reconnectAllConnections(); + + Consumer consumer = pulsarClient.newConsumer() + .topicsPattern(pattern) + // Disable brokerSideSubscriptionPatternEvaluation will leading disable topic list watcher. + // So set patternAutoDiscoveryPeriod to a little value. + .patternAutoDiscoveryPeriod(1) + .subscriptionName(subscriptionName) + .subscriptionType(SubscriptionType.Shared) + .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS) + .receiverQueueSize(4) + .subscribe(); + + // 1. create topic. + if (partitioned) { + admin.topics().createPartitionedTopic(topicName, 1); + } else { + admin.topics().createNonPartitionedTopic(topicName); + } + + // 2. verify consumer can subscribe the topic. + // Since the minimum value of `patternAutoDiscoveryPeriod` is 60s, we set the test timeout to a triple value. + assertSame(pattern.pattern(), ((PatternMultiTopicsConsumerImpl) consumer).getPattern().pattern()); + Awaitility.await().atMost(Duration.ofMinutes(3)).untilAsserted(() -> { + assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitions().size(), 1); + assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getConsumers().size(), 1); + if (partitioned) { + assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitionedTopics().size(), 1); + } else { + assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitionedTopics().size(), 0); + } + }); + + // cleanup. + consumer.close(); + if (partitioned) { + admin.topics().deletePartitionedTopic(topicName); + } else { + admin.topics().delete(topicName); + } + // Close all ServerCnx by close client-side sockets to make the config changes effect. + pulsar.getConfig().setEnableBrokerSideSubscriptionPatternEvaluation(true); + reconnectAllConnections(); + } + + private PulsarClient createDelayWatchTopicsClient() throws Exception { + ClientBuilderImpl clientBuilder = (ClientBuilderImpl) PulsarClient.builder().serviceUrl(lookupUrl.toString()); + return InjectedClientCnxClientBuilder.create(clientBuilder, + (conf, eventLoopGroup) -> new ClientCnx(InstrumentProvider.NOOP, conf, eventLoopGroup) { + public CompletableFuture newWatchTopicList( + BaseCommand command, long requestId) { + // Inject 2 seconds delay when sending command New Watch Topics. + CompletableFuture res = new CompletableFuture<>(); + new Thread(() -> { + sleepSeconds(2); + super.newWatchTopicList(command, requestId).whenComplete((v, ex) -> { + if (ex != null) { + res.completeExceptionally(ex); + } else { + res.complete(v); + } + }); + }).start(); + return res; + } + }); } // simulate subscribe a pattern which has 3 topics, but then matched topic added in. @@ -663,10 +876,16 @@ public void testAutoSubscribePatternConsumer() throws Exception { .receiverQueueSize(4) .subscribe(); + // Wait topic list watcher creation. + Awaitility.await().untilAsserted(() -> { + CompletableFuture completableFuture = WhiteboxImpl.getInternalState(consumer, "watcherFuture"); + assertTrue(completableFuture.isDone() && !completableFuture.isCompletedExceptionally()); + }); + assertTrue(consumer instanceof PatternMultiTopicsConsumerImpl); // 4. verify consumer get methods, to get 6 number of partitions and topics: 6=1+2+3 - assertSame(pattern, ((PatternMultiTopicsConsumerImpl) consumer).getPattern()); + assertSame(pattern.pattern(), ((PatternMultiTopicsConsumerImpl) consumer).getPattern().pattern()); assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitions().size(), 6); assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getConsumers().size(), 6); assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitionedTopics().size(), 2); @@ -773,10 +992,16 @@ public void testAutoUnsubscribePatternConsumer() throws Exception { .receiverQueueSize(4) .subscribe(); + // Wait topic list watcher creation. + Awaitility.await().untilAsserted(() -> { + CompletableFuture completableFuture = WhiteboxImpl.getInternalState(consumer, "watcherFuture"); + assertTrue(completableFuture.isDone() && !completableFuture.isCompletedExceptionally()); + }); + assertTrue(consumer instanceof PatternMultiTopicsConsumerImpl); // 4. verify consumer get methods, to get 0 number of partitions and topics: 6=1+2+3 - assertSame(pattern, ((PatternMultiTopicsConsumerImpl) consumer).getPattern()); + assertSame(pattern.pattern(), ((PatternMultiTopicsConsumerImpl) consumer).getPattern().pattern()); assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitions().size(), 6); assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getConsumers().size(), 6); assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitionedTopics().size(), 2); @@ -801,15 +1026,17 @@ public void testAutoUnsubscribePatternConsumer() throws Exception { // 6. remove producer 1,3; verify only consumer 2 left // seems no direct way to verify auto-unsubscribe, because this patternConsumer also referenced the topic. - List topicNames = Lists.newArrayList(topicName2); + String tp2p0 = TopicName.get(topicName2).getPartition(0).toString(); + String tp2p1 = TopicName.get(topicName2).getPartition(1).toString(); + List topicNames = Lists.newArrayList(tp2p0, tp2p1); NamespaceService nss = pulsar.getNamespaceService(); doReturn(CompletableFuture.completedFuture(topicNames)).when(nss) .getListOfPersistentTopics(NamespaceName.get("my-property/my-ns")); // 7. call recheckTopics to unsubscribe topic 1,3, verify topics number: 2=6-1-3 log.debug("recheck topics change"); - PatternMultiTopicsConsumerImpl consumer1 = ((PatternMultiTopicsConsumerImpl) consumer); - consumer1.run(consumer1.getRecheckPatternTimeout()); + PatternConsumerUpdateQueue taskQueue = WhiteboxImpl.getInternalState(consumer, "updateTaskQueue"); + taskQueue.appendRecheckOp(); Thread.sleep(100); assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitions().size(), 2); assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getConsumers().size(), 2); @@ -857,11 +1084,17 @@ public void testTopicDeletion() throws Exception { .subscriptionName("sub") .subscribe(); + // Wait topic list watcher creation. + Awaitility.await().untilAsserted(() -> { + CompletableFuture completableFuture = WhiteboxImpl.getInternalState(consumer, "watcherFuture"); + assertTrue(completableFuture.isDone() && !completableFuture.isCompletedExceptionally()); + }); + assertTrue(consumer instanceof PatternMultiTopicsConsumerImpl); PatternMultiTopicsConsumerImpl consumerImpl = (PatternMultiTopicsConsumerImpl) consumer; // 4. verify consumer get methods - assertSame(consumerImpl.getPattern(), pattern); + assertSame(consumerImpl.getPattern().pattern(), pattern.pattern()); assertEquals(consumerImpl.getPartitionedTopics().size(), 0); producer1.send("msg-1"); @@ -884,4 +1117,57 @@ public void testTopicDeletion() throws Exception { assertEquals(pulsar.getBrokerService().getTopicIfExists(baseTopicName + "-1").join(), Optional.empty()); assertTrue(pulsar.getBrokerService().getTopicIfExists(baseTopicName + "-2").join().isPresent()); } + + @Test(dataProvider = "partitioned") + public void testPatternQuote(boolean partitioned) throws Exception { + final NamespaceName namespace = NamespaceName.get("public/default"); + final String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final PulsarClientImpl client = (PulsarClientImpl) pulsarClient; + final LookupService lookup = client.getLookup(); + List expectedRes = new ArrayList<>(); + if (partitioned) { + admin.topics().createPartitionedTopic(topicName, 2); + expectedRes.add(TopicName.get(topicName).getPartition(0).toString()); + expectedRes.add(TopicName.get(topicName).getPartition(1).toString()); + Collections.sort(expectedRes); + } else { + admin.topics().createNonPartitionedTopic(topicName); + expectedRes.add(topicName); + } + + // Verify 1: "java.util.regex.Pattern.quote". + String pattern1 = java.util.regex.Pattern.quote(topicName); + List res1 = lookup.getTopicsUnderNamespace(namespace, CommandGetTopicsOfNamespace.Mode.PERSISTENT, + pattern1, null).join().getNonPartitionedOrPartitionTopics(); + Collections.sort(res1); + assertEquals(res1, expectedRes); + + // Verify 2: "com.google.re2j.Pattern.quote" + String pattern2 = com.google.re2j.Pattern.quote(topicName); + List res2 = lookup.getTopicsUnderNamespace(namespace, CommandGetTopicsOfNamespace.Mode.PERSISTENT, + pattern2, null).join().getNonPartitionedOrPartitionTopics(); + Collections.sort(res2); + assertEquals(res2, expectedRes); + + // Verify 3: "java.util.regex.Pattern.quote" & "^$" + String pattern3 = "^" + java.util.regex.Pattern.quote(topicName) + "$"; + List res3 = lookup.getTopicsUnderNamespace(namespace, CommandGetTopicsOfNamespace.Mode.PERSISTENT, + pattern3, null).join().getNonPartitionedOrPartitionTopics(); + Collections.sort(res3); + assertEquals(res3, expectedRes); + + // Verify 4: "com.google.re2j.Pattern.quote" & "^$" + String pattern4 = "^" + com.google.re2j.Pattern.quote(topicName) + "$"; + List res4 = lookup.getTopicsUnderNamespace(namespace, CommandGetTopicsOfNamespace.Mode.PERSISTENT, + pattern4, null).join().getNonPartitionedOrPartitionTopics(); + Collections.sort(res4); + assertEquals(res4, expectedRes); + + // cleanup. + if (partitioned) { + admin.topics().deletePartitionedTopic(topicName, false); + } else { + admin.topics().delete(topicName, false); + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProduceWithMessageIdTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProduceWithMessageIdTest.java index b8efdeb99696a..45f9a9c52e871 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProduceWithMessageIdTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProduceWithMessageIdTest.java @@ -18,14 +18,18 @@ */ package org.apache.pulsar.client.impl; +import static org.apache.pulsar.client.impl.AbstractBatchMessageContainer.INITIAL_BATCH_BUFFER_SIZE; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MockBrokerService; +import org.apache.pulsar.client.api.ProducerConsumerBase; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.api.proto.MessageMetadata; @@ -38,13 +42,20 @@ @Test(groups = "broker-impl") @Slf4j -public class ProduceWithMessageIdTest { +public class ProduceWithMessageIdTest extends ProducerConsumerBase { MockBrokerService mockBrokerService; @BeforeClass(alwaysRun = true) - public void setup() { + public void setup() throws Exception { mockBrokerService = new MockBrokerService(); mockBrokerService.start(); + super.internalSetup(); + super.producerBaseSetup(); + } + + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); } @AfterClass(alwaysRun = true) @@ -86,7 +97,7 @@ public void testSend() throws Exception { AtomicBoolean result = new AtomicBoolean(false); producer.sendAsync(msg, new SendCallback() { @Override - public void sendComplete(Exception e) { + public void sendComplete(Throwable e, OpSendMsgStats opSendMsgStats) { log.info("sendComplete", e); result.set(e == null); } @@ -115,4 +126,72 @@ public CompletableFuture getFuture() { // the result is true only if broker received right message id. Awaitility.await().untilTrue(result); } + + @Test + public void sendWithCallBack() throws Exception { + + int batchSize = 10; + + String topic = "persistent://public/default/testSendWithCallBack"; + ProducerImpl producer = + (ProducerImpl) pulsarClient.newProducer().topic(topic) + .enableBatching(true) + .batchingMaxMessages(batchSize) + .create(); + + CountDownLatch cdl = new CountDownLatch(1); + AtomicReference sendMsgStats = new AtomicReference<>(); + SendCallback sendComplete = new SendCallback() { + @Override + public void sendComplete(Throwable e, OpSendMsgStats opSendMsgStats) { + log.info("sendComplete", e); + if (e == null){ + cdl.countDown(); + sendMsgStats.set(opSendMsgStats); + } + } + + @Override + public void addCallback(MessageImpl msg, SendCallback scb) { + + } + + @Override + public SendCallback getNextSendCallback() { + return null; + } + + @Override + public MessageImpl getNextMessage() { + return null; + } + + @Override + public CompletableFuture getFuture() { + return null; + } + }; + int totalReadabled = 0; + int totalUncompressedSize = 0; + for (int i = 0; i < batchSize; i++) { + MessageMetadata metadata = new MessageMetadata(); + ByteBuffer buffer = ByteBuffer.wrap("data".getBytes(StandardCharsets.UTF_8)); + MessageImpl msg = MessageImpl.create(metadata, buffer, Schema.BYTES, topic); + msg.getDataBuffer().retain(); + totalReadabled += msg.getDataBuffer().readableBytes(); + totalUncompressedSize += msg.getUncompressedSize(); + producer.sendAsync(msg, sendComplete); + } + + cdl.await(); + OpSendMsgStats opSendMsgStats = sendMsgStats.get(); + Assert.assertEquals(opSendMsgStats.getUncompressedSize(), totalUncompressedSize + INITIAL_BATCH_BUFFER_SIZE); + Assert.assertEquals(opSendMsgStats.getSequenceId(), 0); + Assert.assertEquals(opSendMsgStats.getRetryCount(), 1); + Assert.assertEquals(opSendMsgStats.getBatchSizeByte(), totalReadabled); + Assert.assertEquals(opSendMsgStats.getNumMessagesInBatch(), batchSize); + Assert.assertEquals(opSendMsgStats.getHighestSequenceId(), batchSize-1); + Assert.assertEquals(opSendMsgStats.getTotalChunks(), 0); + Assert.assertEquals(opSendMsgStats.getChunkId(), -1); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerConsumerInternalTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerConsumerInternalTest.java new file mode 100644 index 0000000000000..a06085d3d4626 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerConsumerInternalTest.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.service.ServerCnx; +import org.apache.pulsar.client.api.BatcherBuilder; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.common.api.proto.CommandCloseProducer; +import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +/** + * Different with {@link org.apache.pulsar.client.api.SimpleProducerConsumerTest}, this class can visit the variables + * of {@link ConsumerImpl} or {@link ProducerImpl} which have protected or default access modifiers. + */ +@Slf4j +@Test(groups = "broker-impl") +public class ProducerConsumerInternalTest extends ProducerConsumerBase { + + @BeforeClass(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testSameProducerRegisterTwice() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://my-property/my-ns/tp_"); + admin.topics().createNonPartitionedTopic(topicName); + + // Create producer using default producerName. + ProducerImpl producer = (ProducerImpl) pulsarClient.newProducer().topic(topicName).create(); + ServiceProducer serviceProducer = getServiceProducer(producer, topicName); + + // Remove producer maintained by server cnx. To make it can register the second time. + removeServiceProducerMaintainedByServerCnx(serviceProducer); + + // Trigger the client producer reconnect. + CommandCloseProducer commandCloseProducer = new CommandCloseProducer(); + commandCloseProducer.setProducerId(producer.producerId); + producer.getClientCnx().handleCloseProducer(commandCloseProducer); + + // Verify the reconnection will be success. + Awaitility.await().untilAsserted(() -> { + assertEquals(producer.getState().toString(), "Ready"); + }); + } + + @Test + public void testSameProducerRegisterTwiceWithSpecifiedProducerName() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://my-property/my-ns/tp_"); + final String pName = "p1"; + admin.topics().createNonPartitionedTopic(topicName); + + // Create producer using default producerName. + ProducerImpl producer = (ProducerImpl) pulsarClient.newProducer().producerName(pName).topic(topicName).create(); + ServiceProducer serviceProducer = getServiceProducer(producer, topicName); + + // Remove producer maintained by server cnx. To make it can register the second time. + removeServiceProducerMaintainedByServerCnx(serviceProducer); + + // Trigger the client producer reconnect. + CommandCloseProducer commandCloseProducer = new CommandCloseProducer(); + commandCloseProducer.setProducerId(producer.producerId); + producer.getClientCnx().handleCloseProducer(commandCloseProducer); + + // Verify the reconnection will be success. + Awaitility.await().untilAsserted(() -> { + assertEquals(producer.getState().toString(), "Ready", "The producer registration failed"); + }); + } + + private void removeServiceProducerMaintainedByServerCnx(ServiceProducer serviceProducer) { + ServerCnx serverCnx = (ServerCnx) serviceProducer.getServiceProducer().getCnx(); + serverCnx.removedProducer(serviceProducer.getServiceProducer()); + Awaitility.await().untilAsserted(() -> { + assertFalse(serverCnx.getProducers().containsKey(serviceProducer.getServiceProducer().getProducerId())); + }); + } + + @Test(groups = "flaky") + public void testExclusiveConsumerWillAlwaysRetryEvenIfReceivedConsumerBusyError() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://my-property/my-ns/tp_"); + final String subscriptionName = "subscription1"; + admin.topics().createNonPartitionedTopic(topicName); + + final ConsumerImpl consumer = (ConsumerImpl) pulsarClient.newConsumer().topic(topicName.toString()) + .subscriptionType(SubscriptionType.Exclusive).subscriptionName(subscriptionName).subscribe(); + + ClientCnx clientCnx = consumer.getClientCnx(); + ServerCnx serverCnx = (ServerCnx) pulsar.getBrokerService() + .getTopic(topicName,false).join().get().getSubscription(subscriptionName) + .getDispatcher().getConsumers().get(0).cnx(); + + // Make a disconnect to trigger broker remove the consumer which related this connection. + // Make the second subscribe runs after the broker removing the old consumer, then it will receive + // an error: "Exclusive consumer is already connected" + final CountDownLatch countDownLatch = new CountDownLatch(1); + serverCnx.execute(() -> { + try { + countDownLatch.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + clientCnx.close(); + Thread.sleep(1000); + countDownLatch.countDown(); + + // Verify the consumer will always retry subscribe event received ConsumerBusy error. + Awaitility.await().untilAsserted(() -> { + assertEquals(consumer.getState(), HandlerState.State.Ready); + }); + + // cleanup. + consumer.close(); + admin.topics().delete(topicName, false); + } + + @DataProvider(name = "containerBuilder") + public Object[][] containerBuilderProvider() { + return new Object[][] { + { BatcherBuilder.DEFAULT }, + { BatcherBuilder.KEY_BASED } + }; + } + + @Test(timeOut = 30000, dataProvider = "containerBuilder") + public void testSendTimerCheckForBatchContainer(BatcherBuilder batcherBuilder) throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://my-property/my-ns/tp_"); + @Cleanup Producer producer = pulsarClient.newProducer().topic(topicName) + .batcherBuilder(batcherBuilder) + .sendTimeout(1, TimeUnit.SECONDS) + .batchingMaxPublishDelay(100, TimeUnit.MILLISECONDS) + .batchingMaxMessages(1000) + .create(); + + log.info("Before sendAsync msg-0: {}", System.nanoTime()); + CompletableFuture future = producer.sendAsync("msg-0".getBytes()); + future.thenAccept(msgId -> log.info("msg-0 done: {} (msgId: {})", System.nanoTime(), msgId)); + future.get(); // t: the current time point + + ((ProducerImpl) producer).triggerSendTimer(); // t+1000ms && t+2000ms: run() will be called again + + Thread.sleep(1950); // t+2050ms: the batch timer is expired, which happens after run() is called + log.info("Before sendAsync msg-1: {}", System.nanoTime()); + future = producer.sendAsync("msg-1".getBytes()); + future.thenAccept(msgId -> log.info("msg-1 done: {} (msgId: {})", System.nanoTime(), msgId)); + future.get(); + } + + + @Test + public void testRetentionPolicyByProducingMessages() throws Exception { + // Setup: configure the entries per ledger and retention polices. + final int maxEntriesPerLedger = 10, messagesCount = 10; + final String topicName = BrokerTestUtil.newUniqueName("persistent://my-property/my-ns/tp_"); + pulsar.getConfiguration().setManagedLedgerMaxEntriesPerLedger(maxEntriesPerLedger); + pulsar.getConfiguration().setManagedLedgerMinLedgerRolloverTimeMinutes(0); + pulsar.getConfiguration().setDefaultRetentionTimeInMinutes(0); + pulsar.getConfiguration().setDefaultRetentionSizeInMB(0); + + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topicName) + .sendTimeout(1, TimeUnit.SECONDS) + .enableBatching(false) + .create(); + + @Cleanup + Consumer consumer = pulsarClient.newConsumer().topic(topicName) + .subscriptionName("my-sub") + .subscribe(); + // Act: prepare a full ledger data and ack them. + for (int i = 0; i < messagesCount; i++) { + producer.newMessage().sendAsync(); + } + for (int i = 0; i < messagesCount; i++) { + Message message = consumer.receive(); + assertNotNull(message); + consumer.acknowledge(message); + } + // Verify: a new empty ledger will be created after the current ledger is fulled. + // And the previous consumed ledgers will be deleted + Awaitility.await().untilAsserted(() -> { + admin.topics().trimTopic(topicName); + PersistentTopicInternalStats internalStats = admin.topics().getInternalStatsAsync(topicName).get(); + assertEquals(internalStats.currentLedgerEntries, 0); + assertEquals(internalStats.ledgers.size(), 1); + }); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerMemoryLimitTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerMemoryLimitTest.java index 3ec784e248cba..55a67ae644d36 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerMemoryLimitTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerMemoryLimitTest.java @@ -21,23 +21,25 @@ import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import io.netty.buffer.ByteBufAllocator; import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import lombok.Cleanup; +import org.apache.pulsar.client.api.CompressionType; +import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.ProducerConsumerBase; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SizeUnit; -import org.mockito.MockedStatic; import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.TimeUnit; - @Test(groups = "broker-impl") public class ProducerMemoryLimitTest extends ProducerConsumerBase { @@ -67,10 +69,12 @@ public void testProducerInvalidMessageMemoryRelease() throws Exception { .create(); this.stopBroker(); try { - try (MockedStatic mockedStatic = Mockito.mockStatic(ClientCnx.class)) { - mockedStatic.when(ClientCnx::getMaxMessageSize).thenReturn(8); - producer.send("memory-test".getBytes(StandardCharsets.UTF_8)); - } + ConnectionHandler connectionHandler = Mockito.spy(producer.getConnectionHandler()); + Field field = producer.getClass().getDeclaredField("connectionHandler"); + field.setAccessible(true); + field.set(producer, connectionHandler); + when(connectionHandler.getMaxMessageSize()).thenReturn(8); + producer.send("memory-test".getBytes(StandardCharsets.UTF_8)); throw new IllegalStateException("can not reach here"); } catch (PulsarClientException.InvalidMessageException ex) { PulsarClientImpl clientImpl = (PulsarClientImpl) this.pulsarClient; @@ -191,6 +195,40 @@ public void testProducerCloseMemoryRelease() throws Exception { Assert.assertEquals(memoryLimitController.currentUsage(), 0); } + @Test(timeOut = 10_000) + public void testProducerBlockReserveMemory() throws Exception { + replacePulsarClient(PulsarClient.builder(). + serviceUrl(lookupUrl.toString()) + .memoryLimit(1, SizeUnit.KILO_BYTES)); + @Cleanup + ProducerImpl producer = (ProducerImpl) pulsarClient.newProducer() + .topic("testProducerMemoryLimit") + .sendTimeout(5, TimeUnit.SECONDS) + .compressionType(CompressionType.SNAPPY) + .messageRoutingMode(MessageRoutingMode.RoundRobinPartition) + .maxPendingMessages(0) + .blockIfQueueFull(true) + .enableBatching(true) + .batchingMaxMessages(100) + .batchingMaxBytes(65536) + .batchingMaxPublishDelay(100, TimeUnit.MILLISECONDS) + .create(); + int msgCount = 5; + CountDownLatch cdl = new CountDownLatch(msgCount); + for (int i = 0; i < msgCount; i++) { + producer.sendAsync("memory-test".getBytes(StandardCharsets.UTF_8)).whenComplete(((messageId, throwable) -> { + cdl.countDown(); + })); + } + + cdl.await(); + + producer.close(); + PulsarClientImpl clientImpl = (PulsarClientImpl) this.pulsarClient; + final MemoryLimitController memoryLimitController = clientImpl.getMemoryLimitController(); + Assert.assertEquals(memoryLimitController.currentUsage(), 0); + } + private void initClientWithMemoryLimit() throws PulsarClientException { replacePulsarClient(PulsarClient.builder(). serviceUrl(lookupUrl.toString()) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerSemaphoreTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerSemaphoreTest.java index 2f8cb655401d9..42f431e0b9b53 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerSemaphoreTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerSemaphoreTest.java @@ -22,6 +22,7 @@ import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import io.netty.buffer.ByteBufAllocator; import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; @@ -33,7 +34,6 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.util.FutureUtil; -import org.mockito.MockedStatic; import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -72,13 +72,14 @@ public void testProducerSemaphoreInvalidMessage() throws Exception { .maxPendingMessages(pendingQueueSize) .enableBatching(true) .create(); - this.stopBroker(); try { - try (MockedStatic mockedStatic = Mockito.mockStatic(ClientCnx.class)) { - mockedStatic.when(ClientCnx::getMaxMessageSize).thenReturn(2); - producer.send("semaphore-test".getBytes(StandardCharsets.UTF_8)); - } + ConnectionHandler connectionHandler = Mockito.spy(producer.getConnectionHandler()); + Field field = producer.getClass().getDeclaredField("connectionHandler"); + field.setAccessible(true); + field.set(producer, connectionHandler); + when(connectionHandler.getMaxMessageSize()).thenReturn(2); + producer.send("semaphore-test".getBytes(StandardCharsets.UTF_8)); throw new IllegalStateException("can not reach here"); } catch (PulsarClientException.InvalidMessageException ex) { Assert.assertEquals(producer.getSemaphore().get().availablePermits(), pendingQueueSize); @@ -86,10 +87,7 @@ public void testProducerSemaphoreInvalidMessage() throws Exception { producer.conf.setBatchingEnabled(false); try { - try (MockedStatic mockedStatic = Mockito.mockStatic(ClientCnx.class)) { - mockedStatic.when(ClientCnx::getMaxMessageSize).thenReturn(2); - producer.send("semaphore-test".getBytes(StandardCharsets.UTF_8)); - } + producer.send("semaphore-test".getBytes(StandardCharsets.UTF_8)); throw new IllegalStateException("can not reach here"); } catch (PulsarClientException.InvalidMessageException ex) { Assert.assertEquals(producer.getSemaphore().get().availablePermits(), pendingQueueSize); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PulsarTestClient.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PulsarTestClient.java index 0ba8511c0ca74..f69cd576f9ac2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PulsarTestClient.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PulsarTestClient.java @@ -24,15 +24,18 @@ import java.io.IOException; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Supplier; +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; import org.apache.pulsar.common.util.netty.EventLoopUtil; import org.awaitility.Awaitility; @@ -50,6 +53,7 @@ * called after the message to send out has been added to the pending messages in the client. * */ +@Slf4j public class PulsarTestClient extends PulsarClientImpl { private volatile int overrideRemoteEndpointProtocolVersion; private volatile boolean rejectNewConnections; @@ -73,11 +77,11 @@ public static PulsarTestClient create(ClientBuilder clientBuilder) throws Pulsar // method. EventLoopGroup eventLoopGroup = EventLoopUtil.newEventLoopGroup(clientConfigurationData.getNumIoThreads(), false, - new DefaultThreadFactory("pulsar-client-io", Thread.currentThread().isDaemon())); + new DefaultThreadFactory("pulsar-test-client-io", Thread.currentThread().isDaemon())); AtomicReference> clientCnxSupplierReference = new AtomicReference<>(); - ConnectionPool connectionPool = new ConnectionPool(clientConfigurationData, eventLoopGroup, - () -> clientCnxSupplierReference.get().get()); + ConnectionPool connectionPool = new ConnectionPool(InstrumentProvider.NOOP, clientConfigurationData, eventLoopGroup, + () -> clientCnxSupplierReference.get().get(), null); return new PulsarTestClient(clientConfigurationData, eventLoopGroup, connectionPool, clientCnxSupplierReference); @@ -98,7 +102,7 @@ private PulsarTestClient(ClientConfigurationData conf, EventLoopGroup eventLoopG * @return new ClientCnx instance */ protected ClientCnx createClientCnx() { - return new ClientCnx(conf, eventLoopGroup) { + return new ClientCnx(InstrumentProvider.NOOP, conf, eventLoopGroup) { @Override public int getRemoteEndpointProtocolVersion() { return overrideRemoteEndpointProtocolVersion != 0 @@ -127,7 +131,7 @@ public CompletableFuture getConnection(String topic) { } /** - * Overrides the producer instance with an anonynomous subclass that adds hooks for observing new + * Overrides the producer instance with an anonymous subclass that adds hooks for observing new * OpSendMsg instances being added to pending messages in the client. * It also configures the hook to drop OpSend messages when dropping is enabled. */ @@ -189,7 +193,7 @@ public void disconnectProducerAndRejectReconnecting(ProducerImpl producer) th // make the existing connection between the producer and broker to break by explicitly closing it ClientCnx cnx = producer.cnx(); - producer.connectionClosed(cnx); + producer.connectionClosed(cnx, Optional.empty(), Optional.empty()); cnx.close(); } @@ -217,4 +221,31 @@ public void setPendingMessageCallback( public void dropOpSendMessages() { this.dropOpSendMessages = true; } + + @Override + public CompletableFuture closeAsync() { + return super.closeAsync().handle((__, t) -> { + shutdownCnxPoolAndEventLoopGroup(); + return null; + }); + } + + @Override + public void shutdown() throws PulsarClientException { + super.shutdown(); + shutdownCnxPoolAndEventLoopGroup(); + } + + private void shutdownCnxPoolAndEventLoopGroup() { + try { + getCnxPool().close(); + } catch (Exception e) { + log.warn("Error closing connection pool", e); + } + try { + eventLoopGroup.shutdownGracefully().get(5, TimeUnit.SECONDS); + } catch (Exception e) { + log.warn("Error closing event loop group", e); + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java index 9b8b1e5efb99c..d79a31c07f218 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java @@ -47,7 +47,6 @@ import org.apache.pulsar.compaction.CompactionTest; import org.testng.Assert; import org.testng.annotations.BeforeMethod; -import org.testng.annotations.DataProvider; import org.testng.annotations.Test; public class RawBatchMessageContainerImplTest { @@ -56,8 +55,6 @@ public class RawBatchMessageContainerImplTest { CryptoKeyReader cryptoKeyReader; Map encryptKeys; - int maxBytesInBatch = 5 * 1024 * 1024; - public void setEncryptionAndCompression(boolean encrypt, boolean compress) { if (compress) { compressionType = ZSTD; @@ -107,22 +104,22 @@ public MessageImpl createMessage(String topic, String value, int entryId) { public void setup() throws Exception { setEncryptionAndCompression(false, true); } - @DataProvider(name = "testBatchLimitByMessageCount") - public static Object[][] testBatchLimitByMessageCount() { - return new Object[][] {{true}, {false}}; - } - - @Test(timeOut = 20000, dataProvider = "testBatchLimitByMessageCount") - public void testToByteBufWithBatchLimit(boolean testBatchLimitByMessageCount) throws IOException { - RawBatchMessageContainerImpl container = testBatchLimitByMessageCount ? - new RawBatchMessageContainerImpl(2, Integer.MAX_VALUE) : - new RawBatchMessageContainerImpl(Integer.MAX_VALUE, 5); + @Test(timeOut = 20000) + public void testToByteBufWithBatchLimit()throws IOException { + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(); String topic = "my-topic"; - var full1 = container.add(createMessage(topic, "hi-1", 0), null); - var full2 = container.add(createMessage(topic, "hi-2", 1), null); + MessageImpl message1 = createMessage(topic, "hi-1", 0); + boolean hasEnoughSpase1 = container.haveEnoughSpace(message1); + var full1 = container.add(message1, null); assertFalse(full1); - assertTrue(full2); + assertTrue(hasEnoughSpase1); + MessageImpl message2 = createMessage(topic, "hi-2", 1); + boolean hasEnoughSpase2 = container.haveEnoughSpace(message2); + assertFalse(hasEnoughSpase2); + var full2 = container.add(message2, null); + assertFalse(full2); + ByteBuf buf = container.toByteBuf(); @@ -167,7 +164,7 @@ public void testToByteBufWithBatchLimit(boolean testBatchLimitByMessageCount) th public void testToByteBufWithCompressionAndEncryption() throws IOException { setEncryptionAndCompression(true, true); - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(2, maxBytesInBatch); + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(); container.setCryptoKeyReader(cryptoKeyReader); String topic = "my-topic"; container.add(createMessage(topic, "hi-1", 0), null); @@ -217,7 +214,7 @@ public void testToByteBufWithCompressionAndEncryption() throws IOException { @Test public void testToByteBufWithSingleMessage() throws IOException { - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(2, maxBytesInBatch); + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(); String topic = "my-topic"; container.add(createMessage(topic, "hi-1", 0), null); ByteBuf buf = container.toByteBuf(); @@ -250,25 +247,31 @@ public void testToByteBufWithSingleMessage() throws IOException { } @Test - public void testMaxNumMessagesInBatch() { - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1, maxBytesInBatch); + public void testAddDifferentBatchMessage() { + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(); String topic = "my-topic"; boolean isFull = container.add(createMessage(topic, "hi", 0), null); - Assert.assertTrue(isFull); - Assert.assertTrue(container.isBatchFull()); + Assert.assertFalse(isFull); + Assert.assertFalse(container.isBatchFull()); + MessageImpl message = createMessage(topic, "hi-1", 0); + Assert.assertTrue(container.haveEnoughSpace(message)); + isFull = container.add(message, null); + Assert.assertFalse(isFull); + message = createMessage(topic, "hi-2", 1); + Assert.assertFalse(container.haveEnoughSpace(message)); } @Test(expectedExceptions = UnsupportedOperationException.class) public void testCreateOpSendMsg() { - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1, maxBytesInBatch); + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(); container.createOpSendMsg(); } @Test public void testToByteBufWithEncryptionWithoutCryptoKeyReader() { setEncryptionAndCompression(true, false); - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1, maxBytesInBatch); + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(); String topic = "my-topic"; container.add(createMessage(topic, "hi-1", 0), null); Assert.assertEquals(container.getNumMessagesInBatch(), 1); @@ -286,7 +289,7 @@ public void testToByteBufWithEncryptionWithoutCryptoKeyReader() { @Test public void testToByteBufWithEncryptionWithInvalidEncryptKeys() { setEncryptionAndCompression(true, false); - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1, maxBytesInBatch); + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(); container.setCryptoKeyReader(cryptoKeyReader); encryptKeys = new HashMap<>(); encryptKeys.put(null, null); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java index a201ef104e7b3..d9ddc00b2e863 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java @@ -27,22 +27,31 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.commons.lang3.tuple.ImmutableTriple; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.RawMessage; import org.apache.pulsar.client.api.RawReader; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import org.apache.pulsar.common.api.proto.BrokerEntryMetadata; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.protocol.Commands; import org.awaitility.Awaitility; import org.testng.Assert; @@ -50,7 +59,10 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import static org.apache.pulsar.client.impl.RawReaderImpl.DEFAULT_RECEIVER_QUEUE_SIZE; + @Test(groups = "broker-impl") +@Slf4j public class RawReaderTest extends MockedPulsarServiceBaseTest { private static final String subscription = "foobar-sub"; @@ -62,6 +74,7 @@ public void setup() throws Exception { "org.apache.pulsar.common.intercept.AppendBrokerTimestampMetadataInterceptor", "org.apache.pulsar.common.intercept.AppendIndexMetadataInterceptor" )); + conf.setSystemTopicEnabled(false); conf.setExposingBrokerEntryMetadataToClientEnabled(true); super.internalSetup(); @@ -116,7 +129,7 @@ public static String extractKey(RawMessage m) { @Test public void testHasMessageAvailableWithoutBatch() throws Exception { int numKeys = 10; - String topic = "persistent://my-property/my-ns/my-raw-topic"; + String topic = "persistent://my-property/my-ns/" + BrokerTestUtil.newUniqueName("reader"); Set keys = publishMessages(topic, numKeys); RawReader reader = RawReader.create(pulsarClient, topic, subscription).get(); while (true) { @@ -133,20 +146,18 @@ public void testHasMessageAvailableWithoutBatch() throws Exception { } } Assert.assertTrue(keys.isEmpty()); + reader.closeAsync().get(3, TimeUnit.SECONDS); } @Test public void testHasMessageAvailableWithBatch() throws Exception { int numKeys = 20; - String topic = "persistent://my-property/my-ns/my-raw-topic"; + String topic = "persistent://my-property/my-ns/" + BrokerTestUtil.newUniqueName("reader"); Set keys = publishMessages(topic, numKeys, true); RawReader reader = RawReader.create(pulsarClient, topic, subscription).get(); int messageCount = 0; while (true) { boolean hasMsg = reader.hasMessageAvailableAsync().get(); - if (hasMsg && (messageCount == numKeys)) { - Assert.fail("HasMessageAvailable shows still has message when there is no message"); - } if (hasMsg) { try (RawMessage m = reader.readNextAsync().get()) { MessageMetadata meta = Commands.parseMessageMetadata(m.getHeadersAndPayload()); @@ -163,13 +174,14 @@ public void testHasMessageAvailableWithBatch() throws Exception { } Assert.assertEquals(messageCount, numKeys); Assert.assertTrue(keys.isEmpty()); + reader.closeAsync().get(3, TimeUnit.SECONDS); } @Test public void testRawReader() throws Exception { int numKeys = 10; - String topic = "persistent://my-property/my-ns/my-raw-topic"; + String topic = "persistent://my-property/my-ns/" + BrokerTestUtil.newUniqueName("reader"); Set keys = publishMessages(topic, numKeys); @@ -185,12 +197,43 @@ public void testRawReader() throws Exception { } } Assert.assertTrue(keys.isEmpty()); + reader.closeAsync().get(3, TimeUnit.SECONDS); + } + + @Test + public void testRawReaderWithConfigurationCreation() throws Exception { + int numKeys = 10; + + String topic = "persistent://my-property/my-ns/" + BrokerTestUtil.newUniqueName("reader"); + + Set keys = publishMessages(topic, numKeys); + ConsumerConfigurationData consumerConfiguration = new ConsumerConfigurationData<>(); + consumerConfiguration.getTopicNames().add(topic); + consumerConfiguration.setSubscriptionName(subscription); + consumerConfiguration.setSubscriptionType(SubscriptionType.Exclusive); + consumerConfiguration.setReceiverQueueSize(DEFAULT_RECEIVER_QUEUE_SIZE); + consumerConfiguration.setReadCompacted(true); + consumerConfiguration.setSubscriptionInitialPosition(SubscriptionInitialPosition.Earliest); + consumerConfiguration.setAckReceiptEnabled(true); + RawReader reader = RawReader.create(pulsarClient, consumerConfiguration, true).get(); + + MessageId lastMessageId = reader.getLastMessageIdAsync().get(); + while (true) { + try (RawMessage m = reader.readNextAsync().get()) { + Assert.assertTrue(keys.remove(extractKey(m))); + if (lastMessageId.compareTo(m.getMessageId()) == 0) { + break; + } + } + } + Assert.assertTrue(keys.isEmpty()); + reader.closeAsync().get(3, TimeUnit.SECONDS); } @Test public void testSeekToStart() throws Exception { int numKeys = 10; - String topic = "persistent://my-property/my-ns/my-raw-topic"; + String topic = "persistent://my-property/my-ns/" + BrokerTestUtil.newUniqueName("reader"); publishMessages(topic, numKeys); @@ -219,12 +262,13 @@ public void testSeekToStart() throws Exception { } } Assert.assertTrue(readKeys.isEmpty()); + reader.closeAsync().get(3, TimeUnit.SECONDS); } @Test public void testSeekToMiddle() throws Exception { int numKeys = 10; - String topic = "persistent://my-property/my-ns/my-raw-topic"; + String topic = "persistent://my-property/my-ns/" + BrokerTestUtil.newUniqueName("reader"); publishMessages(topic, numKeys); @@ -262,6 +306,7 @@ public void testSeekToMiddle() throws Exception { } } Assert.assertTrue(readKeys.isEmpty()); + reader.closeAsync().get(3, TimeUnit.SECONDS); } /** @@ -269,8 +314,8 @@ public void testSeekToMiddle() throws Exception { */ @Test public void testFlowControl() throws Exception { - int numMessages = RawReaderImpl.DEFAULT_RECEIVER_QUEUE_SIZE * 5; - String topic = "persistent://my-property/my-ns/my-raw-topic"; + int numMessages = DEFAULT_RECEIVER_QUEUE_SIZE * 5; + String topic = "persistent://my-property/my-ns/" + BrokerTestUtil.newUniqueName("reader"); publishMessages(topic, numMessages); @@ -296,12 +341,13 @@ public void testFlowControl() throws Exception { } Assert.assertEquals(timeouts, 1); Assert.assertEquals(keys.size(), numMessages); + reader.closeAsync().get(3, TimeUnit.SECONDS); } @Test public void testFlowControlBatch() throws Exception { - int numMessages = RawReaderImpl.DEFAULT_RECEIVER_QUEUE_SIZE * 5; - String topic = "persistent://my-property/my-ns/my-raw-topic"; + int numMessages = DEFAULT_RECEIVER_QUEUE_SIZE * 5; + String topic = "persistent://my-property/my-ns/" + BrokerTestUtil.newUniqueName("reader"); publishMessages(topic, numMessages, true); @@ -324,11 +370,12 @@ public void testFlowControlBatch() throws Exception { } } Assert.assertEquals(keys.size(), numMessages); + reader.closeAsync().get(3, TimeUnit.SECONDS); } @Test public void testBatchingExtractKeysAndIds() throws Exception { - String topic = "persistent://my-property/my-ns/my-raw-topic"; + String topic = "persistent://my-property/my-ns/" + BrokerTestUtil.newUniqueName("reader"); try (Producer producer = pulsarClient.newProducer().topic(topic) .maxPendingMessages(3) @@ -363,7 +410,7 @@ public void testBatchingExtractKeysAndIds() throws Exception { @Test public void testBatchingRebatch() throws Exception { - String topic = "persistent://my-property/my-ns/my-raw-topic"; + String topic = "persistent://my-property/my-ns/" + BrokerTestUtil.newUniqueName("reader"); try (Producer producer = pulsarClient.newProducer().topic(topic) .maxPendingMessages(3) @@ -392,7 +439,7 @@ public void testBatchingRebatch() throws Exception { @Test public void testBatchingRebatchWithBrokerEntryMetadata() throws Exception { - String topic = "persistent://my-property/my-ns/my-raw-topic"; + String topic = "persistent://my-property/my-ns/" + BrokerTestUtil.newUniqueName("reader"); try (Producer producer = pulsarClient.newProducer().topic(topic) .maxPendingMessages(3) @@ -428,7 +475,7 @@ public void testBatchingRebatchWithBrokerEntryMetadata() throws Exception { public void testAcknowledgeWithProperties() throws Exception { int numKeys = 10; - String topic = "persistent://my-property/my-ns/my-raw-topic"; + String topic = "persistent://my-property/my-ns/" + BrokerTestUtil.newUniqueName("reader"); Set keys = publishMessages(topic, numKeys); @@ -459,14 +506,14 @@ public void testAcknowledgeWithProperties() throws Exception { Assert.assertEquals( ledger.openCursor(subscription).getProperties().get("foobar"), Long.valueOf(0xdeadbeefdecaL))); - + reader.closeAsync().get(3, TimeUnit.SECONDS); } @Test public void testReadCancellationOnClose() throws Exception { int numKeys = 10; - String topic = "persistent://my-property/my-ns/my-raw-topic"; + String topic = "persistent://my-property/my-ns/" + BrokerTestUtil.newUniqueName("reader"); publishMessages(topic, numKeys/2); RawReader reader = RawReader.create(pulsarClient, topic, subscription).get(); @@ -488,4 +535,23 @@ public void testReadCancellationOnClose() throws Exception { } } } + + @Test + public void testAutoCreateTopic() throws ExecutionException, InterruptedException, PulsarAdminException { + String topic = "persistent://my-property/my-ns/" + BrokerTestUtil.newUniqueName("reader"); + + RawReader reader = RawReader.create(pulsarClient, topic, subscription).get(); + TopicStats stats = admin.topics().getStats(topic); + Assert.assertNotNull(stats); + reader.closeAsync().join(); + + String topic2 = "persistent://my-property/my-ns/" + BrokerTestUtil.newUniqueName("reader"); + try { + reader = RawReader.create(pulsarClient, topic2, subscription, false).get(); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e.getCause() instanceof PulsarClientException.TopicDoesNotExistException); + } + reader.closeAsync().join(); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ReaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ReaderTest.java index a50c92f7ab8f4..12228220b18bd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ReaderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ReaderTest.java @@ -36,6 +36,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; @@ -44,14 +46,17 @@ import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; +import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Range; import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.ReaderBuilder; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.TopicMessageId; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats; @@ -64,8 +69,9 @@ import org.apache.pulsar.schema.Schemas; import org.awaitility.Awaitility; import org.testng.Assert; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @Slf4j @@ -74,7 +80,7 @@ public class ReaderTest extends MockedPulsarServiceBaseTest { private static final String subscription = "reader-sub"; - @BeforeMethod + @BeforeClass(alwaysRun = true) @Override protected void setup() throws Exception { super.internalSetup(); @@ -86,7 +92,7 @@ protected void setup() throws Exception { admin.namespaces().createNamespace("my-property/my-ns", Sets.newHashSet("test")); } - @AfterMethod(alwaysRun = true) + @AfterClass(alwaysRun = true) @Override protected void cleanup() throws Exception { super.internalCleanup(); @@ -139,27 +145,97 @@ public void testReadMessageWithoutBatchingWithMessageInclusive() throws Exceptio Assert.assertFalse(reader.hasMessageAvailable()); } + @Test + public void testReaderGetLastMessageIds() throws Exception { + String topic1 = "persistent://my-property/my-ns/testReaderGetLastMessageIds-1"; + String topic2 = "persistent://my-property/my-ns/testReaderGetLastMessageIds-2"; + List topicList = new ArrayList<>(); + topicList.add(topic1); + topicList.add(topic2); + @Cleanup + Reader reader1 = pulsarClient.newReader() + .topic(topic1) + .startMessageId(MessageId.earliest) + .readerName(subscription) + .create(); + @Cleanup + Reader reader2 = pulsarClient.newReader() + .topics(topicList) + .startMessageId(MessageId.earliest) + .readerName(subscription) + .create(); + + Producer producer1 = pulsarClient.newProducer() + .topic(topic1) + .create(); + Producer producer2 = pulsarClient.newProducer() + .topic(topic2) + .create(); + MessageIdImpl messageId1 = (MessageIdImpl) producer1.newMessage().send(); + MessageIdImpl messageId2 = (MessageIdImpl) producer2.newMessage().send(); + reader1.readNext(); + reader2.readNext(); + reader2.readNext(); + List topicMessageIds1 = reader1.getLastMessageIds(); + assertEquals(topicMessageIds1.size(), 1); + assertEquals(topicMessageIds1.get(0).getOwnerTopic(), topic1); + assertEquals(((MessageIdAdv)topicMessageIds1.get(0)).getEntryId(), messageId1.getEntryId()); + assertEquals(((MessageIdAdv)topicMessageIds1.get(0)).getLedgerId(), messageId1.getLedgerId()); + + List topicMessageIds2 = reader2.getLastMessageIds(); + assertEquals(topicMessageIds2.size(), 2); + for (TopicMessageId topicMessageId: topicMessageIds2) { + if (topicMessageId.getOwnerTopic().equals(topic1)) { + assertEquals(((MessageIdAdv)topicMessageId).getEntryId(), messageId1.getEntryId()); + assertEquals(((MessageIdAdv)topicMessageId).getLedgerId(), messageId1.getLedgerId()); + } else { + assertEquals(((MessageIdAdv)topicMessageId).getEntryId(), messageId2.getEntryId()); + assertEquals(((MessageIdAdv)topicMessageId).getLedgerId(), messageId2.getLedgerId()); + } + } + } + @Test public void testReadMessageWithBatching() throws Exception { String topic = "persistent://my-property/my-ns/my-reader-topic-with-batching"; testReadMessages(topic, true); } - @Test - public void testReadMessageWithBatchingWithMessageInclusive() throws Exception { + @DataProvider + public static Object[][] seekBeforeHasMessageAvailable() { + return new Object[][] { { true }, { false } }; + } + + @Test(timeOut = 20000, dataProvider = "seekBeforeHasMessageAvailable") + public void testReadMessageWithBatchingWithMessageInclusive(boolean seekBeforeHasMessageAvailable) + throws Exception { String topic = "persistent://my-property/my-ns/my-reader-topic-with-batching-inclusive"; Set keys = publishMessages(topic, 10, true); Reader reader = pulsarClient.newReader().topic(topic).startMessageId(MessageId.latest) .startMessageIdInclusive().readerName(subscription).create(); - while (reader.hasMessageAvailable()) { - Assert.assertTrue(keys.remove(reader.readNext().getKey())); + if (seekBeforeHasMessageAvailable) { + reader.seek(0L); // it should seek to the earliest } + + assertTrue(reader.hasMessageAvailable()); + final Message msg = reader.readNext(); + assertTrue(keys.remove(msg.getKey())); // start from latest with start message inclusive should only read the last message in batch assertEquals(keys.size(), 9); - Assert.assertFalse(keys.contains("key9")); - Assert.assertFalse(reader.hasMessageAvailable()); + + final MessageIdAdv msgId = (MessageIdAdv) msg.getMessageId(); + if (seekBeforeHasMessageAvailable) { + assertEquals(msgId.getBatchIndex(), 0); + assertFalse(keys.contains("key0")); + assertTrue(reader.hasMessageAvailable()); + } else { + assertEquals(msgId.getBatchIndex(), 9); + assertFalse(reader.hasMessageAvailable()); + assertFalse(keys.contains("key9")); + assertFalse(reader.hasMessageAvailable()); + } } private void testReadMessages(String topic, boolean enableBatch) throws Exception { @@ -257,7 +333,7 @@ public void testReadFromPartition() throws Exception { @Test public void testReaderWithTimeLong() throws Exception { String ns = "my-property/my-ns"; - String topic = "persistent://" + ns + "/testReadFromPartition"; + String topic = "persistent://" + ns + "/testReaderWithTimeLong"; RetentionPolicies retention = new RetentionPolicies(-1, -1); admin.namespaces().setRetention(ns, retention); @@ -733,4 +809,124 @@ public void testReaderListenerAcknowledgement() admin.topics().deletePartitionedTopic(partitionedTopic); } + @Test + public void testReaderReconnectedFromNextEntry() throws Exception { + final String topic = "persistent://my-property/my-ns/testReaderReconnectedFromNextEntry"; + Reader reader = pulsarClient.newReader(Schema.STRING).topic(topic).receiverQueueSize(1) + .startMessageId(MessageId.earliest).create(); + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + + // Send 3 and consume 1. + producer.send("1"); + producer.send("2"); + producer.send("3"); + Message msg1 = reader.readNext(2, TimeUnit.SECONDS); + assertEquals(msg1.getValue(), "1"); + + // Trigger reader reconnect. + admin.topics().unload(topic); + + // For non-durable we are going to restart from the next entry. + Message msg2 = reader.readNext(2, TimeUnit.SECONDS); + assertEquals(msg2.getValue(), "2"); + Message msg3 = reader.readNext(2, TimeUnit.SECONDS); + assertEquals(msg3.getValue(), "3"); + + // cleanup. + reader.close(); + producer.close(); + admin.topics().delete(topic, false); + } + + @DataProvider + public static Object[][] initializeLastMessageIdInBroker() { + return new Object[][] { { true }, { false } }; + } + + @Test(dataProvider = "initializeLastMessageIdInBroker") + public void testHasMessageAvailableAfterSeek(boolean initializeLastMessageIdInBroker) throws Exception { + final String topic = "persistent://my-property/my-ns/test-has-message-available-after-seek"; + @Cleanup Reader reader = pulsarClient.newReader(Schema.STRING).topic(topic).receiverQueueSize(1) + .startMessageId(MessageId.earliest).create(); + + @Cleanup Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + producer.send("msg"); + + if (initializeLastMessageIdInBroker) { + assertTrue(reader.hasMessageAvailable()); + } // else: lastMessageIdInBroker is earliest + + reader.seek(MessageId.latest); + // lastMessageIdInBroker is the last message ID, while startMessageId is still earliest + assertFalse(reader.hasMessageAvailable()); + + producer.send("msg"); + assertTrue(reader.hasMessageAvailable()); + } + + @Test(dataProvider = "initializeLastMessageIdInBroker") + public void testHasMessageAvailableAfterSeekTimestamp(boolean initializeLastMessageIdInBroker) throws Exception { + final String topic = "persistent://my-property/my-ns/test-has-message-available-after-seek-timestamp"; + + @Cleanup Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + final long timestampBeforeSend = System.currentTimeMillis(); + final MessageId sentMsgId = producer.send("msg"); + + final List messageIds = new ArrayList<>(); + messageIds.add(MessageId.earliest); + messageIds.add(sentMsgId); + messageIds.add(MessageId.latest); + + for (MessageId messageId : messageIds) { + @Cleanup Reader reader = pulsarClient.newReader(Schema.STRING).topic(topic).receiverQueueSize(1) + .startMessageId(messageId).create(); + if (initializeLastMessageIdInBroker) { + if (messageId == MessageId.earliest) { + assertTrue(reader.hasMessageAvailable()); + } else { + assertFalse(reader.hasMessageAvailable()); + } + } // else: lastMessageIdInBroker is earliest + reader.seek(System.currentTimeMillis()); + assertFalse(reader.hasMessageAvailable()); + } + + for (MessageId messageId : messageIds) { + @Cleanup Reader reader = pulsarClient.newReader(Schema.STRING).topic(topic).receiverQueueSize(1) + .startMessageId(messageId).create(); + if (initializeLastMessageIdInBroker) { + if (messageId == MessageId.earliest) { + assertTrue(reader.hasMessageAvailable()); + } else { + assertFalse(reader.hasMessageAvailable()); + } + } // else: lastMessageIdInBroker is earliest + reader.seek(timestampBeforeSend); + assertTrue(reader.hasMessageAvailable()); + } + } + + @Test + public void testReaderBuilderStateOnRetryFailure() throws Exception { + String ns = "my-property/my-ns"; + String topic = "persistent://" + ns + "/testRetryReader"; + RetentionPolicies retention = new RetentionPolicies(-1, -1); + admin.namespaces().setRetention(ns, retention); + String badUrl = "pulsar://bad-host:8080"; + + PulsarClient client = PulsarClient.builder().serviceUrl(badUrl).build(); + + ReaderBuilder readerBuilder = client.newReader().topic(topic).startMessageFromRollbackDuration(100, + TimeUnit.SECONDS); + + for (int i = 0; i < 3; i++) { + try { + readerBuilder.createAsync().get(1, TimeUnit.SECONDS); + } catch (TimeoutException e) { + log.info("It should time out due to invalid url"); + } catch (IllegalArgumentException e) { + fail("It should not fail with corrupt reader state"); + } + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RetryUtilTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RetryUtilTest.java index 604c468b1de45..603378c271f3e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RetryUtilTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RetryUtilTest.java @@ -18,7 +18,10 @@ */ package org.apache.pulsar.client.impl; +import lombok.Cleanup; import org.apache.pulsar.client.util.RetryUtil; +import org.apache.pulsar.common.util.Backoff; +import org.apache.pulsar.common.util.BackoffBuilder; import org.apache.pulsar.common.util.FutureUtil; import org.testng.annotations.Test; @@ -37,6 +40,7 @@ public class RetryUtilTest { @Test public void testFailAndRetry() throws Exception { + @Cleanup("shutdownNow") ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); CompletableFuture callback = new CompletableFuture<>(); AtomicInteger atomicInteger = new AtomicInteger(0); @@ -57,11 +61,11 @@ public void testFailAndRetry() throws Exception { }, backoff, executor, callback); assertTrue(callback.get()); assertEquals(atomicInteger.get(), 5); - executor.shutdownNow(); } @Test public void testFail() throws Exception { + @Cleanup("shutdownNow") ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); CompletableFuture callback = new CompletableFuture<>(); Backoff backoff = new BackoffBuilder() @@ -79,6 +83,5 @@ public void testFail() throws Exception { } long time = System.currentTimeMillis() - start; assertTrue(time >= 5000 - 2000, "Duration:" + time); - executor.shutdownNow(); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/SequenceIdWithErrorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/SequenceIdWithErrorTest.java index 7d330bb82addd..1395424b14123 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/SequenceIdWithErrorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/SequenceIdWithErrorTest.java @@ -21,6 +21,7 @@ import static org.testng.Assert.assertEquals; import io.netty.channel.EventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; +import io.opentelemetry.api.OpenTelemetry; import java.util.Collections; import lombok.Cleanup; import org.apache.bookkeeper.mledger.ManagedLedger; @@ -60,7 +61,7 @@ public void testCheckSequenceId() throws Exception { EventLoopGroup eventLoopGroup = new NioEventLoopGroup(1); ManagedLedgerClientFactory clientFactory = new ManagedLedgerClientFactory(); clientFactory.initialize(pulsar.getConfiguration(), pulsar.getLocalMetadataStore(), - pulsar.getBookKeeperClientFactory(), eventLoopGroup); + pulsar.getBookKeeperClientFactory(), eventLoopGroup, OpenTelemetry.noop()); ManagedLedgerFactory mlFactory = clientFactory.getManagedLedgerFactory(); ManagedLedger ml = mlFactory.open(TopicName.get(topicName).getPersistenceNamingEncoding()); ml.close(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TableViewTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TableViewTest.java index 6c6da5870aed9..5448751160a9c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TableViewTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TableViewTest.java @@ -20,32 +20,42 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.google.common.collect.Sets; +import java.lang.reflect.Method; import java.time.Duration; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomUtils; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.TableView; +import org.apache.pulsar.client.api.TopicMessageId; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; @@ -94,10 +104,11 @@ public static Object[] topicDomain() { } private Set publishMessages(String topic, int count, boolean enableBatch) throws Exception { - return publishMessages(topic, count, enableBatch, false); + return publishMessages(topic, 0, count, enableBatch, false); } - private Set publishMessages(String topic, int count, boolean enableBatch, boolean enableEncryption) throws Exception { + private Set publishMessages(String topic, int keyStartPosition, int count, boolean enableBatch, + boolean enableEncryption) throws Exception { Set keys = new HashSet<>(); ProducerBuilder builder = pulsarClient.newProducer(); builder.messageRoutingMode(MessageRoutingMode.SinglePartition); @@ -117,7 +128,7 @@ private Set publishMessages(String topic, int count, boolean enableBatch } try (Producer producer = builder.create()) { CompletableFuture lastFuture = null; - for (int i = 0; i < count; i++) { + for (int i = keyStartPosition; i < keyStartPosition + count; i++) { String key = "key"+ i; byte[] data = ("my-message-" + i).getBytes(); lastFuture = producer.newMessage().key(key).value(data).sendAsync(); @@ -129,6 +140,129 @@ private Set publishMessages(String topic, int count, boolean enableBatch return keys; } + @DataProvider(name = "partition") + public static Object[][] partition () { + return new Object[][] { + { 3 }, { 0 } + }; + } + + /** + * Case1: + * 1. Slow down the rate of reading messages. + * 2. Send some messages + * 3. Call new `refresh` API, it will wait for reading all the messages completed. + * Case2: + * 1. No new messages. + * 2. Call new `refresh` API, it will be completed immediately. + * Case3: + * 1. multi-partition topic, p1, p2 has new message, p3 has no new messages. + * 2. Call new `refresh` API, it will be completed after read new messages. + */ + @Test(dataProvider = "partition") + public void testRefreshAPI(int partition) throws Exception { + // 1. Prepare resource. + String topic = "persistent://public/default/testRefreshAPI" + RandomUtils.nextLong(); + if (partition == 0) { + admin.topics().createNonPartitionedTopic(topic); + } else { + admin.topics().createPartitionedTopic(topic, partition); + } + + @Cleanup + TableView tv = pulsarClient.newTableView(Schema.BYTES) + .topic(topic) + .create(); + // Verify refresh can handle the case when the topic is empty + tv.refreshAsync().get(3, TimeUnit.SECONDS); + + // 2. Add a listen action to provide the test environment. + // The listen action will be triggered when there are incoming messages every time. + // This is a sync operation, so sleep in the listen action can slow down the reading rate of messages. + tv.listen((k, v) -> { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + // 3. Send 20 messages. After refresh, all the messages should be received. + int count = 20; + Set keys = this.publishMessages(topic, count, false); + // After message sending completely, the table view will take at least 2 seconds to receive all the messages. + // If there is not the refresh operation, all messages will not be received. + tv.refresh(); + // The key of each message is different. + assertEquals(tv.size(), count); + assertEquals(tv.keySet(), keys); + // 4. Test refresh operation can be completed when there is a partition with on new messages + // or no new message for no partition topic. + if (partition > 0) { + publishMessages(topic, partition - 1, count, false, false); + tv.refreshAsync().get(5, TimeUnit.SECONDS); + assertEquals(tv.size(), count + partition - 1); + } else { + tv.refreshAsync().get(5, TimeUnit.SECONDS); + } + } + + /** + * Case1: + * 1. Slow down the read of reading messages. + * 2. Send some messages. + * 3. Call new `refresh` API. + * 4. Close the reader of the tableview. + * 5. The refresh operation will be failed with a `AlreadyClosedException`. + * Case2: + * 1. Close the reader of the tableview. + * 2. Call new `refresh` API. + * 3. The refresh operation will be fail with a `AlreadyClosedException`. + */ + @Test + public void testRefreshTaskCanBeCompletedWhenReaderClosed() throws Exception { + // 1. Prepare resource. + String topic1 = "persistent://public/default/testRefreshTaskCanBeCompletedWhenReaderClosed-1"; + admin.topics().createNonPartitionedTopic(topic1); + String topic2 = "persistent://public/default/testRefreshTaskCanBeCompletedWhenReaderClosed-2"; + admin.topics().createNonPartitionedTopic(topic2); + @Cleanup + TableView tv1 = pulsarClient.newTableView(Schema.BYTES) + .topic(topic1) + .create(); + @Cleanup + TableView tv2 = pulsarClient.newTableView(Schema.BYTES) + .topic(topic1) + .create(); + // 2. Slow down the rate of reading messages. + tv1.listen((k, v) -> { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + publishMessages(topic1, 20, false); + AtomicBoolean completedExceptionally = new AtomicBoolean(false); + // 3. Test failing `refresh` in the reading process. + tv1.refreshAsync().exceptionally(ex -> { + if (ex.getCause() instanceof PulsarClientException.AlreadyClosedException) { + completedExceptionally.set(true); + } + return null; + }); + tv1.close(); + + // 4. Test failing `refresh` when get last message IDs. The topic2 has no available messages. + tv2.close(); + try { + tv2.refresh(); + fail(); + } catch (Exception e) { + assertTrue(e instanceof PulsarClientException.AlreadyClosedException); + } + Awaitility.await().untilAsserted(() -> assertTrue(completedExceptionally.get())); + } + @Test(timeOut = 30 * 1000) public void testTableView() throws Exception { String topic = "persistent://public/default/tableview-test"; @@ -384,7 +518,7 @@ public void testTableViewWithEncryptedMessages() throws Exception { // publish encrypted messages int count = 20; - Set keys = this.publishMessages(topic, count, false, true); + Set keys = this.publishMessages(topic, 0, count, false, true); // TableView can read them using the private key @Cleanup @@ -430,7 +564,7 @@ public void testTableViewTailMessageReadRetry() throws Exception { FieldUtils.writeDeclaredField(reader, "consumer", consumer, true); int msgCnt = 2; - this.publishMessages(topic, msgCnt, false, false); + this.publishMessages(topic, 0, msgCnt, false, false); Awaitility.await() .atMost(5, TimeUnit.SECONDS) .untilAsserted(() -> { @@ -438,4 +572,57 @@ public void testTableViewTailMessageReadRetry() throws Exception { }); verify(consumer, times(msgCnt)).receiveAsync(); } + + @Test + public void testBuildTableViewWithMessagesAlwaysAvailable() throws Exception { + String topic = "persistent://public/default/testBuildTableViewWithMessagesAlwaysAvailable"; + admin.topics().createPartitionedTopic(topic, 10); + @Cleanup + Reader reader = pulsarClient.newReader() + .topic(topic) + .startMessageId(MessageId.earliest) + .create(); + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topic) + .create(); + // Prepare real data to do test. + for (int i = 0; i < 1000; i++) { + producer.newMessage().send(); + } + List lastMessageIds = reader.getLastMessageIds(); + + // Use mock reader to build tableview. In the old implementation, the readAllExistingMessages method + // will not be completed because the `mockReader.hasMessageAvailable()` always return ture. + Reader mockReader = spy(reader); + when(mockReader.hasMessageAvailable()).thenReturn(true); + when(mockReader.getLastMessageIdsAsync()).thenReturn(CompletableFuture.completedFuture(lastMessageIds)); + AtomicInteger index = new AtomicInteger(lastMessageIds.size()); + when(mockReader.readNextAsync()).thenAnswer(invocation -> { + Message message = spy(Message.class); + int localIndex = index.decrementAndGet(); + if (localIndex >= 0) { + when(message.getTopicName()).thenReturn(lastMessageIds.get(localIndex).getOwnerTopic()); + when(message.getMessageId()).thenReturn(lastMessageIds.get(localIndex)); + when(message.hasKey()).thenReturn(false); + doNothing().when(message).release(); + } + return CompletableFuture.completedFuture(message); + }); + @Cleanup + TableViewImpl tableView = (TableViewImpl) pulsarClient.newTableView() + .topic(topic) + .createAsync() + .get(); + TableViewImpl mockTableView = spy(tableView); + Method readAllExistingMessagesMethod = TableViewImpl.class + .getDeclaredMethod("readAllExistingMessages", Reader.class); + readAllExistingMessagesMethod.setAccessible(true); + CompletableFuture> future = + (CompletableFuture>) readAllExistingMessagesMethod.invoke(mockTableView, mockReader); + + // The future will complete after receive all the messages from lastMessageIds. + future.get(3, TimeUnit.SECONDS); + assertTrue(index.get() <= 0); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicPublishThrottlingInitTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicPublishThrottlingInitTest.java index 1b8284c31e67e..fe2e6ec9670dc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicPublishThrottlingInitTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicPublishThrottlingInitTest.java @@ -21,13 +21,10 @@ import static org.testng.Assert.assertTrue; import com.google.common.collect.Sets; import org.apache.pulsar.broker.service.Producer; -import org.apache.pulsar.broker.service.PublishRateLimiter; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.ProducerConsumerBase; -import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -74,16 +71,12 @@ public void testBrokerPublishMessageThrottlingInit() throws Exception { .enableBatching(false) .maxPendingMessages(30000).create(); PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicIfExists(topicName).get().get(); - // (1) verify message-rate is initialized when value configured in broker - Assert.assertNotEquals(topic.getBrokerPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER); log.info("Get broker configuration: brokerTick {}, MaxMessageRate {}, MaxByteRate {}", pulsar.getConfiguration().getBrokerPublisherThrottlingTickTimeMillis(), pulsar.getConfiguration().getBrokerPublisherThrottlingMaxMessageRate(), pulsar.getConfiguration().getBrokerPublisherThrottlingMaxByteRate()); - Assert.assertNotEquals(topic.getBrokerPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER); - Producer prod = topic.getProducers().values().iterator().next(); // reset counter prod.updateRates(); @@ -100,8 +93,6 @@ public void testBrokerPublishMessageThrottlingInit() throws Exception { // disable throttling admin.brokers() .updateDynamicConfiguration("brokerPublisherThrottlingMaxMessageRate", Integer.toString(0)); - Awaitility.await().untilAsserted(() -> - Assert.assertEquals(topic.getBrokerPublishRateLimiter(), PublishRateLimiter.DISABLED_RATE_LIMITER)); // reset counter prod.updateRates(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java index 73fe97996424c..83cb5f2a4400b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import io.netty.util.Timeout; +import java.time.Duration; import lombok.Cleanup; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; @@ -34,6 +35,7 @@ import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageIdAdv; +import org.apache.pulsar.client.api.MessageListener; import org.apache.pulsar.client.api.MessageRouter; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; @@ -57,22 +59,27 @@ import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; - import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; import java.util.Set; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.TreeSet; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -1315,7 +1322,6 @@ public void testPartitionsUpdatesForMultipleTopics() throws Exception { Assert.assertEquals(consumer.allTopicPartitionsNumber.intValue(), 2); admin.topics().updatePartitionedTopic(topicName0, 5); - consumer.getPartitionsAutoUpdateTimeout().task().run(consumer.getPartitionsAutoUpdateTimeout()); Awaitility.await().untilAsserted(() -> { Assert.assertEquals(consumer.getPartitionsOfTheTopicMap(), 5); @@ -1327,6 +1333,7 @@ public void testPartitionsUpdatesForMultipleTopics() throws Exception { assertEquals(admin.topics().getPartitionedTopicMetadata(topicName1).partitions, 3); consumer.getRecheckPatternTimeout().task().run(consumer.getRecheckPatternTimeout()); + Assert.assertTrue(consumer.getRecheckPatternTimeout().isCancelled()); Awaitility.await().untilAsserted(() -> { Assert.assertEquals(consumer.getPartitionsOfTheTopicMap(), 8); @@ -1334,9 +1341,8 @@ public void testPartitionsUpdatesForMultipleTopics() throws Exception { }); admin.topics().updatePartitionedTopic(topicName1, 5); - consumer.getPartitionsAutoUpdateTimeout().task().run(consumer.getPartitionsAutoUpdateTimeout()); - Awaitility.await().untilAsserted(() -> { + Awaitility.await().atMost(Duration.ofSeconds(30)).untilAsserted(() -> { Assert.assertEquals(consumer.getPartitionsOfTheTopicMap(), 10); Assert.assertEquals(consumer.allTopicPartitionsNumber.intValue(), 10); }); @@ -1393,4 +1399,76 @@ public Map getActiveConsumers() { } } + @DataProvider + public static Object[][] seekByFunction() { + return new Object[][] { + { true }, { false } + }; + } + + @Test(timeOut = 30000, dataProvider = "seekByFunction") + public void testSeekToNewerPosition(boolean seekByFunction) throws Exception { + final var topic1 = TopicName.get(newTopicName()).toString() + .replace("my-property", "public").replace("my-ns", "default"); + final var topic2 = TopicName.get(newTopicName()).toString() + .replace("my-property", "public").replace("my-ns", "default"); + @Cleanup final var producer1 = pulsarClient.newProducer(Schema.STRING).topic(topic1).create(); + @Cleanup final var producer2 = pulsarClient.newProducer(Schema.STRING).topic(topic2).create(); + producer1.send("1-0"); + producer2.send("2-0"); + producer1.send("1-1"); + producer2.send("2-1"); + final var consumer1 = pulsarClient.newConsumer(Schema.STRING) + .topics(Arrays.asList(topic1, topic2)).subscriptionName("sub") + .ackTimeout(1, TimeUnit.SECONDS) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe(); + final var timestamps = new ArrayList(); + for (int i = 0; i < 4; i++) { + timestamps.add(consumer1.receive().getPublishTime()); + } + timestamps.sort(Comparator.naturalOrder()); + final var timestamp = timestamps.get(2); + consumer1.close(); + + final Function, CompletableFuture> seekAsync = consumer -> { + final var future = seekByFunction ? consumer.seekAsync(__ -> timestamp) : consumer.seekAsync(timestamp); + assertEquals(((ConsumerBase) consumer).getIncomingMessageSize(), 0L); + assertEquals(((ConsumerBase) consumer).getTotalIncomingMessages(), 0); + assertTrue(((ConsumerBase) consumer).getUnAckedMessageTracker().isEmpty()); + return future; + }; + + @Cleanup final var consumer2 = pulsarClient.newConsumer(Schema.STRING) + .topics(Arrays.asList(topic1, topic2)).subscriptionName("sub-2") + .ackTimeout(1, TimeUnit.SECONDS) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe(); + seekAsync.apply(consumer2).get(); + final var values = new TreeSet(); + for (int i = 0; i < 2; i++) { + values.add(consumer2.receive().getValue()); + } + assertEquals(values, new TreeSet<>(Arrays.asList("1-1", "2-1"))); + + final var valuesInListener = new CopyOnWriteArrayList(); + @Cleanup final var consumer3 = pulsarClient.newConsumer(Schema.STRING) + .topics(Arrays.asList(topic1, topic2)).subscriptionName("sub-3") + .messageListener((MessageListener) (__, msg) -> valuesInListener.add(msg.getValue())) + .ackTimeout(1, TimeUnit.SECONDS) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe(); + seekAsync.apply(consumer3).get(); + if (valuesInListener.isEmpty()) { + Awaitility.await().untilAsserted(() -> assertEquals(valuesInListener.size(), 2)); + assertEquals(valuesInListener.stream().sorted().toList(), Arrays.asList("1-1", "2-1")); + } // else: consumer3 has passed messages to the listener before seek, in this case we cannot assume anything + + @Cleanup final var consumer4 = pulsarClient.newConsumer(Schema.STRING) + .topics(Arrays.asList(topic1, topic2)).subscriptionName("sub-4") + .ackTimeout(1, TimeUnit.SECONDS) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe(); + seekAsync.apply(consumer4).get(); + final var valuesInReceiveAsync = new ArrayList(); + valuesInReceiveAsync.add(consumer4.receiveAsync().get().getValue()); + valuesInReceiveAsync.add(consumer4.receiveAsync().get().getValue()); + assertEquals(valuesInReceiveAsync.stream().sorted().toList(), Arrays.asList("1-1", "2-1")); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java index 34cc3bc1ca526..f490fa70539ea 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java @@ -19,8 +19,8 @@ package org.apache.pulsar.client.impl; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyObject; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.testng.Assert.assertEquals; @@ -46,7 +46,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.TransactionMetadataStoreService; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.Topic; @@ -74,7 +73,6 @@ import org.apache.pulsar.common.api.proto.CommandAck; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; import org.apache.pulsar.transaction.coordinator.TransactionMetadataStore; import org.apache.pulsar.transaction.coordinator.TransactionSubscription; @@ -96,7 +94,9 @@ public class TransactionEndToEndTest extends TransactionTestBase { protected static final String TOPIC_OUTPUT = NAMESPACE1 + "/output"; protected static final String TOPIC_MESSAGE_ACK_TEST = NAMESPACE1 + "/message-ack-test"; protected static final int NUM_PARTITIONS = 16; - @BeforeClass + private static final int waitTimeForCanReceiveMsgInSec = 5; + private static final int waitTimeForCannotReceiveMsgInSec = 5; + @BeforeClass(alwaysRun = true) protected void setup() throws Exception { conf.setAcknowledgmentAtBatchIndexLevelEnabled(true); setUpBase(1, NUM_PARTITIONS, TOPIC_OUTPUT, TOPIC_PARTITION); @@ -173,7 +173,7 @@ private void testIndividualAckAbortFilterAckSetInPendingAckState() throws Except } // can't receive message anymore - assertNull(consumer.receive(2, TimeUnit.SECONDS)); + assertNull(consumer.receive(waitTimeForCannotReceiveMsgInSec, TimeUnit.SECONDS)); } @@ -240,20 +240,79 @@ private void testFilterMsgsInPendingAckStateWhenConsumerDisconnect(boolean enabl .enableBatchIndexAcknowledgment(true) .subscribe(); - Message message = consumer.receive(3, TimeUnit.SECONDS); + Message message = consumer.receive(waitTimeForCannotReceiveMsgInSec, TimeUnit.SECONDS); Assert.assertNull(message); // abort txn1 txn1.abort().get(); // after txn1 aborted, consumer will receive messages txn1 contains int receiveCounter = 0; - while((message = consumer.receive(3, TimeUnit.SECONDS)) != null) { + while((message = consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS)) != null) { Assert.assertEquals(message.getValue().intValue(), receiveCounter); receiveCounter ++; } Assert.assertEquals(receiveCounter, count / 2); } + @Test + private void testMsgsInPendingAckStateWouldNotGetTheConsumerStuck() throws Exception { + final String topicName = NAMESPACE1 + "/testMsgsInPendingAckStateWouldNotGetTheConsumerStuck"; + final String subscription = "test"; + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.INT32) + .topic(topicName) + .create(); + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.INT32) + .topic(topicName) + .subscriptionName(subscription) + .subscriptionType(SubscriptionType.Shared) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + int numStep1Receive = 2, numStep2Receive = 2, numStep3Receive = 2; + int numTotalMessage = numStep1Receive + numStep2Receive + numStep3Receive; + + for (int i = 0; i < numTotalMessage; i++) { + producer.send(i); + } + + Transaction step1Txn = getTxn(); + Transaction step2Txn = getTxn(); + + // Step 1, try to consume some messages but do not commit the transaction + for (int i = 0; i < numStep1Receive; i++) { + consumer.acknowledgeAsync(consumer.receive().getMessageId(), step1Txn).get(); + } + + // Step 2, try to consume some messages and commit the transaction + for (int i = 0; i < numStep2Receive; i++) { + consumer.acknowledgeAsync(consumer.receive().getMessageId(), step2Txn).get(); + } + + // commit step2Txn + step2Txn.commit().get(); + + // close and re-create consumer + consumer.close(); + @Cleanup + Consumer consumer2 = pulsarClient.newConsumer(Schema.INT32) + .topic(topicName) + .receiverQueueSize(numStep3Receive) + .subscriptionName(subscription) + .subscriptionType(SubscriptionType.Shared) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + // Step 3, try to consume the rest messages and should receive all of them + for (int i = 0; i < numStep3Receive; i++) { + // should get the message instead of timeout + Message msg = consumer2.receive(3, TimeUnit.SECONDS); + Assert.assertEquals(msg.getValue(), numStep1Receive + numStep2Receive + i); + } + } + @Test(dataProvider="enableBatch") private void produceCommitTest(boolean enableBatch) throws Exception { @Cleanup @@ -288,7 +347,7 @@ private void produceCommitTest(boolean enableBatch) throws Exception { } // Can't receive transaction messages before commit. - Message message = consumer.receive(300, TimeUnit.MILLISECONDS); + Message message = consumer.receive(waitTimeForCannotReceiveMsgInSec, TimeUnit.SECONDS); Assert.assertNull(message); txn1.commit().get(); @@ -296,13 +355,13 @@ private void produceCommitTest(boolean enableBatch) throws Exception { int receiveCnt = 0; for (int i = 0; i < txnMessageCnt; i++) { - message = consumer.receive(5, TimeUnit.SECONDS); + message = consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS); Assert.assertNotNull(message); receiveCnt ++; } Assert.assertEquals(txnMessageCnt, receiveCnt); - message = consumer.receive(300, TimeUnit.MILLISECONDS); + message = consumer.receive(waitTimeForCannotReceiveMsgInSec, TimeUnit.SECONDS); Assert.assertNull(message); // cleanup. @@ -339,13 +398,13 @@ public void produceAbortTest() throws Exception { Awaitility.await().until(consumer::isConnected); // Can't receive transaction messages before abort. - Message message = consumer.receive(300, TimeUnit.MILLISECONDS); + Message message = consumer.receive(waitTimeForCannotReceiveMsgInSec, TimeUnit.SECONDS); Assert.assertNull(message); txn.abort().get(); // Cant't receive transaction messages after abort. - message = consumer.receive(300, TimeUnit.MILLISECONDS); + message = consumer.receive(waitTimeForCannotReceiveMsgInSec, TimeUnit.SECONDS); Assert.assertNull(message); Awaitility.await().until(() -> { boolean flag = true; @@ -355,11 +414,7 @@ public void produceAbortTest() throws Exception { boolean exist = false; for (int i = 0; i < getPulsarServiceList().size(); i++) { - Field field = BrokerService.class.getDeclaredField("topics"); - field.setAccessible(true); - ConcurrentOpenHashMap>> topics = - (ConcurrentOpenHashMap>>) field - .get(getPulsarServiceList().get(i).getBrokerService()); + final var topics = getPulsarServiceList().get(i).getBrokerService().getTopics(); CompletableFuture> topicFuture = topics.get(topic); if (topicFuture != null) { @@ -378,7 +433,7 @@ public void produceAbortTest() throws Exception { //when delete commit marker operation finish, it can run next delete commit marker operation //so this test may not delete all the position in this manageLedger. Position markerPosition = ((ManagedLedgerImpl) persistentSubscription.getCursor() - .getManagedLedger()).getNextValidPosition((PositionImpl) markDeletePosition); + .getManagedLedger()).getNextValidPosition(markDeletePosition); //marker is the lastConfirmedEntry, after commit the marker will only be write in if (!markerPosition.equals(lastConfirmedEntry)) { log.error("Mark delete position is not commit marker position!"); @@ -433,7 +488,7 @@ private void testAckWithTransactionReduceUnAckMessageCount(boolean enableBatch) Transaction txn = getTxn(); for (int i = 0; i < messageCount / 2; i++) { - Message message = consumer.receive(5, TimeUnit.SECONDS); + Message message = consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS); consumer.acknowledgeAsync(message.getMessageId(), txn).get(); } @@ -513,14 +568,14 @@ protected void txnAckTest(boolean batchEnable, int maxBatchSize, // consume and ack messages with txn for (int i = 0; i < messageCnt; i++) { - Message message = consumer.receive(5, TimeUnit.SECONDS); + Message message = consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS); Assert.assertNotNull(message); log.info("receive msgId: {}, count : {}", message.getMessageId(), i); consumer.acknowledgeAsync(message.getMessageId(), txn).get(); } // the messages are pending ack state and can't be received - Message message = consumer.receive(300, TimeUnit.MILLISECONDS); + Message message = consumer.receive(waitTimeForCannotReceiveMsgInSec, TimeUnit.SECONDS); Assert.assertNull(message); // 1) txn abort @@ -529,7 +584,7 @@ protected void txnAckTest(boolean batchEnable, int maxBatchSize, // after transaction abort, the messages could be received Transaction commitTxn = getTxn(); for (int i = 0; i < messageCnt; i++) { - message = consumer.receive(2, TimeUnit.SECONDS); + message = consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS); Assert.assertNotNull(message); consumer.acknowledgeAsync(message.getMessageId(), commitTxn).get(); log.info("receive msgId: {}, count: {}", message.getMessageId(), i); @@ -539,7 +594,7 @@ protected void txnAckTest(boolean batchEnable, int maxBatchSize, commitTxn.commit().get(); // after transaction commit, the messages can't be received - message = consumer.receive(300, TimeUnit.MILLISECONDS); + message = consumer.receive(waitTimeForCannotReceiveMsgInSec, TimeUnit.SECONDS); Assert.assertNull(message); Field field = TransactionImpl.class.getDeclaredField("state"); @@ -576,7 +631,7 @@ public void testAfterDeleteTopicOtherTopicCanRecover() throws Exception { .topic(topicTwo).subscriptionName(sub).subscribe(); String content = "test"; producer.send(content); - assertEquals(consumer.receive(3, TimeUnit.SECONDS).getValue(), content); + assertEquals(consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS).getValue(), content); // cleanup. producer.close(); @@ -615,7 +670,7 @@ public void txnMessageAckTest() throws Exception { log.info("produce transaction messages finished"); // Can't receive transaction messages before commit. - Message message = consumer.receive(300, TimeUnit.MILLISECONDS); + Message message = consumer.receive(waitTimeForCannotReceiveMsgInSec, TimeUnit.SECONDS); Assert.assertNull(message); log.info("transaction messages can't be received before transaction committed"); @@ -624,7 +679,7 @@ public void txnMessageAckTest() throws Exception { int ackedMessageCount = 0; int receiveCnt = 0; for (int i = 0; i < messageCnt; i++) { - message = consumer.receive(5, TimeUnit.SECONDS); + message = consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS); Assert.assertNotNull(message); receiveCnt ++; if (i % 2 == 0) { @@ -634,7 +689,7 @@ public void txnMessageAckTest() throws Exception { } Assert.assertEquals(messageCnt, receiveCnt); - message = consumer.receive(300, TimeUnit.MILLISECONDS); + message = consumer.receive(waitTimeForCannotReceiveMsgInSec, TimeUnit.SECONDS); Assert.assertNull(message); String checkTopic = TopicName.get(topic).getPartition(0).toString(); @@ -646,14 +701,14 @@ public void txnMessageAckTest() throws Exception { receiveCnt = 0; for (int i = 0; i < messageCnt - ackedMessageCount; i++) { - message = consumer.receive(2, TimeUnit.SECONDS); + message = consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS); Assert.assertNotNull(message); consumer.acknowledge(message); receiveCnt ++; } Assert.assertEquals(messageCnt - ackedMessageCount, receiveCnt); - message = consumer.receive(300, TimeUnit.MILLISECONDS); + message = consumer.receive(waitTimeForCannotReceiveMsgInSec, TimeUnit.SECONDS); Assert.assertNull(message); topic = TopicName.get(topic).getPartition(0).toString(); @@ -662,9 +717,7 @@ public void txnMessageAckTest() throws Exception { Field field = BrokerService.class.getDeclaredField("topics"); field.setAccessible(true); - ConcurrentOpenHashMap>> topics = - (ConcurrentOpenHashMap>>) field - .get(getPulsarServiceList().get(i).getBrokerService()); + final var topics = getPulsarServiceList().get(i).getBrokerService().getTopics(); CompletableFuture> topicFuture = topics.get(topic); if (topicFuture != null) { @@ -683,7 +736,7 @@ public void txnMessageAckTest() throws Exception { //when delete commit marker operation finish, it can run next delete commit marker operation //so this test may not delete all the position in this manageLedger. Position markerPosition = ((ManagedLedgerImpl) persistentSubscription.getCursor() - .getManagedLedger()).getNextValidPosition((PositionImpl) markDeletePosition); + .getManagedLedger()).getNextValidPosition(markDeletePosition); //marker is the lastConfirmedEntry, after commit the marker will only be write in if (!markerPosition.equals(lastConfirmedEntry)) { log.error("Mark delete position is not commit marker position!"); @@ -744,7 +797,7 @@ private void txnCumulativeAckTest(boolean batchEnable, int maxBatchSize, Subscri Message message = null; Thread.sleep(1000L); for (int i = 0; i < messageCnt; i++) { - message = consumer.receive(1, TimeUnit.SECONDS); + message = consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS); Assert.assertNotNull(message); if (i % 3 == 0) { consumer.acknowledgeCumulativeAsync(message.getMessageId(), abortTxn).get(); @@ -769,14 +822,14 @@ private void txnCumulativeAckTest(boolean batchEnable, int maxBatchSize, Subscri } // the messages are pending ack state and can't be received - message = consumer.receive(300, TimeUnit.MILLISECONDS); + message = consumer.receive(waitTimeForCannotReceiveMsgInSec, TimeUnit.SECONDS); Assert.assertNull(message); abortTxn.abort().get(); consumer.redeliverUnacknowledgedMessages(); Transaction commitTxn = getTxn(); for (int i = 0; i < messageCnt; i++) { - message = consumer.receive(1, TimeUnit.SECONDS); + message = consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS); Assert.assertNotNull(message); if (i % 3 == 0) { consumer.acknowledgeCumulativeAsync(message.getMessageId(), commitTxn).get(); @@ -798,7 +851,7 @@ private void txnCumulativeAckTest(boolean batchEnable, int maxBatchSize, Subscri Assert.assertTrue(reCommitError.getCause() instanceof TransactionNotFoundException); } - message = consumer.receive(300, TimeUnit.MILLISECONDS); + message = consumer.receive(waitTimeForCannotReceiveMsgInSec, TimeUnit.SECONDS); Assert.assertNull(message); } @@ -860,7 +913,7 @@ public void txnMetadataHandlerRecoverTest() throws Exception { Awaitility.await().until(consumer::isConnected); for (int i = 0; i < txnCnt * messageCnt; i++) { - Message message = consumer.receive(5, TimeUnit.SECONDS); + Message message = consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS); Assert.assertNotNull(message); } @@ -901,7 +954,7 @@ public void produceTxnMessageOrderTest() throws Exception { txn.commit().get(); for (int i = 0; i < 1000; i++) { - Message message = consumer.receive(5, TimeUnit.SECONDS); + Message message = consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS); Assert.assertNotNull(message); Assert.assertEquals(Integer.valueOf(new String(message.getData())), Integer.valueOf(i)); } @@ -957,7 +1010,7 @@ public void produceAndConsumeCloseStateTxnTest() throws Exception { } - Message message = consumer.receive(5, TimeUnit.SECONDS); + Message message = consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS); consumer.acknowledgeAsync(message.getMessageId(), consumeTxn).get(); consumeTxn.commit().get(); try { @@ -1003,7 +1056,7 @@ public void produceAndConsumeCloseStateTxnTest() throws Exception { constructor.setAccessible(true); TransactionImpl timeoutTxnSkipClientTimeout = constructor.newInstance(pulsarClient, 5, - timeoutTxn.getTxnID().getLeastSigBits(), timeoutTxn.getTxnID().getMostSigBits()); + timeoutTxn.getTxnID().getLeastSigBits(), timeoutTxn.getTxnID().getMostSigBits()); try { timeoutTxnSkipClientTimeout.commit().get(); @@ -1033,7 +1086,7 @@ public void testTxnTimeoutAtTransactionMetadataStore() throws Exception{ .newTransaction(new TransactionCoordinatorID(0), 1, null).get(); Awaitility.await().until(() -> { try { - getPulsarServiceList().get(0).getTransactionMetadataStoreService().getTxnMeta(txnID).get(); + getPulsarServiceList().get(0).getTransactionMetadataStoreService().getTxnMeta(txnID).get(); return false; } catch (Exception e) { return true; @@ -1066,17 +1119,17 @@ public void transactionTimeoutTest() throws Exception { Transaction consumeTimeoutTxn = pulsarClient .newTransaction() - .withTransactionTimeout(3, TimeUnit.SECONDS) + .withTransactionTimeout(7, TimeUnit.SECONDS) .build().get(); - Message message = consumer.receive(5, TimeUnit.SECONDS); + Message message = consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS); consumer.acknowledgeAsync(message.getMessageId(), consumeTimeoutTxn).get(); - Message reReceiveMessage = consumer.receive(300, TimeUnit.MILLISECONDS); + Message reReceiveMessage = consumer.receive(waitTimeForCannotReceiveMsgInSec, TimeUnit.SECONDS); assertNull(reReceiveMessage); - reReceiveMessage = consumer.receive(5, TimeUnit.SECONDS); + reReceiveMessage = consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS); assertEquals(reReceiveMessage.getValue(), message.getValue()); @@ -1123,9 +1176,9 @@ public void txnTransactionRedeliverNullDispatcher(CommandAck.AckType ackType) th } Transaction txn = getTxn(); if (ackType == CommandAck.AckType.Individual) { - consumer.acknowledgeAsync(consumer.receive(5, TimeUnit.SECONDS).getMessageId(), txn); + consumer.acknowledgeAsync(consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS).getMessageId(), txn); } else { - consumer.acknowledgeCumulativeAsync(consumer.receive(5, TimeUnit.SECONDS).getMessageId(), txn); + consumer.acknowledgeCumulativeAsync(consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS).getMessageId(), txn); } topic = TopicName.get(topic).toString(); boolean exist = false; @@ -1133,9 +1186,7 @@ public void txnTransactionRedeliverNullDispatcher(CommandAck.AckType ackType) th Field field = BrokerService.class.getDeclaredField("topics"); field.setAccessible(true); - ConcurrentOpenHashMap>> topics = - (ConcurrentOpenHashMap>>) field - .get(getPulsarServiceList().get(i).getBrokerService()); + final var topics = getPulsarServiceList().get(i).getBrokerService().getTopics(); CompletableFuture> topicFuture = topics.get(topic); if (topicFuture != null) { @@ -1248,7 +1299,7 @@ public void testTxnTimeOutInClient() throws Exception{ .InvalidTxnStatusException); } try { - Message message = consumer.receive(5, TimeUnit.SECONDS); + Message message = consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS); consumer.acknowledgeAsync(message.getMessageId(), transaction).get(); Assert.fail(); } catch (Exception e) { @@ -1290,7 +1341,7 @@ public void testCumulativeAckRedeliverMessages() throws Exception { Message message = null; for (int i = 0; i < transactionCumulativeAck; i++) { - message = consumer.receive(5, TimeUnit.SECONDS); + message = consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS); } // receive transaction in order @@ -1313,7 +1364,7 @@ public void testCumulativeAckRedeliverMessages() throws Exception { // receive the rest of the message for (int i = 0; i < count; i++) { - message = consumer.receive(5, TimeUnit.SECONDS); + message = consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS); } Transaction commitTransaction = getTxn(); @@ -1326,7 +1377,7 @@ public void testCumulativeAckRedeliverMessages() throws Exception { commitTransaction.commit().get(); // then redeliver will not receive any message - message = consumer.receive(300, TimeUnit.MILLISECONDS); + message = consumer.receive(waitTimeForCannotReceiveMsgInSec, TimeUnit.SECONDS); assertNull(message); // cleanup. @@ -1356,7 +1407,7 @@ public void testSendTxnMessageTimeout() throws Exception { CompletableFuture completableFuture = new CompletableFuture<>(); completableFuture.complete(new ProducerResponse("test", 1, "1".getBytes(), Optional.of(30L))); - doReturn(completableFuture).when(cnx).sendRequestWithId(anyObject(), anyLong()); + doReturn(completableFuture).when(cnx).sendRequestWithId(any(), anyLong()); producer.getConnectionHandler().setClientCnx(cnx); @@ -1401,7 +1452,7 @@ public void testAckWithTransactionReduceUnackCountNotInPendingAcks() throws Exce // receive the batch messages add to a list for (int i = 0; i < 5; i++) { - messageIds.add(consumer.receive(5, TimeUnit.SECONDS).getMessageId()); + messageIds.add(consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS).getMessageId()); } MessageIdImpl messageId = (MessageIdImpl) messageIds.get(0); @@ -1461,7 +1512,7 @@ public void testSendTxnAckMessageToDLQ() throws Exception { .build().get(); // consumer receive the message the first time, redeliverCount = 0 - consumer.acknowledgeAsync(consumer.receive(5, TimeUnit.SECONDS).getMessageId(), transaction).get(); + consumer.acknowledgeAsync(consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS).getMessageId(), transaction).get(); transaction.abort().get(); @@ -1469,17 +1520,17 @@ public void testSendTxnAckMessageToDLQ() throws Exception { .build().get(); // consumer receive the message the second time, redeliverCount = 1, also can be received - consumer.acknowledgeAsync(consumer.receive(5, TimeUnit.SECONDS).getMessageId(), transaction).get(); + consumer.acknowledgeAsync(consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS).getMessageId(), transaction).get(); transaction.abort().get(); // consumer receive the message the third time, redeliverCount = 2, // the message will be sent to DLQ, can't receive - assertNull(consumer.receive(300, TimeUnit.MILLISECONDS)); + assertNull(consumer.receive(waitTimeForCannotReceiveMsgInSec, TimeUnit.SECONDS)); assertEquals(((ConsumerImpl) consumer).getAvailablePermits(), 3); - assertEquals(value, new String(deadLetterConsumer.receive(3, TimeUnit.SECONDS).getValue())); + assertEquals(value, new String(deadLetterConsumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS).getValue())); // cleanup. consumer.close(); @@ -1525,7 +1576,7 @@ public void testSendTxnAckBatchMessageToDLQ() throws Exception { Transaction transaction = pulsarClient.newTransaction().withTransactionTimeout(1, TimeUnit.MINUTES) .build().get(); - Message message = consumer.receive(5, TimeUnit.SECONDS); + Message message = consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS); assertEquals(value1, new String(message.getValue())); // consumer receive the batch message one the first time, redeliverCount = 0 consumer.acknowledgeAsync(message.getMessageId(), transaction).get(); @@ -1535,7 +1586,7 @@ public void testSendTxnAckBatchMessageToDLQ() throws Exception { // consumer will receive the batch message two and then receive // the message one and message two again, redeliverCount = 1 for (int i = 0; i < 3; i ++) { - message = consumer.receive(5, TimeUnit.SECONDS); + message = consumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS); } transaction = pulsarClient.newTransaction().withTransactionTimeout(5, TimeUnit.MINUTES) @@ -1549,12 +1600,12 @@ public void testSendTxnAckBatchMessageToDLQ() throws Exception { // consumer receive the batch message the third time, redeliverCount = 2, // the message will be sent to DLQ, can't receive - assertNull(consumer.receive(300, TimeUnit.MILLISECONDS)); + assertNull(consumer.receive(waitTimeForCannotReceiveMsgInSec, TimeUnit.SECONDS)); assertEquals(((ConsumerImpl) consumer).getAvailablePermits(), 6); - assertEquals(value1, new String(deadLetterConsumer.receive(3, TimeUnit.SECONDS).getValue())); - assertEquals(value2, new String(deadLetterConsumer.receive(3, TimeUnit.SECONDS).getValue())); + assertEquals(value1, new String(deadLetterConsumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS).getValue())); + assertEquals(value2, new String(deadLetterConsumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS).getValue())); // cleanup. consumer.close(); @@ -1565,7 +1616,7 @@ public void testSendTxnAckBatchMessageToDLQ() throws Exception { admin.topics().delete(topic, true); } - @Test + @Test(groups = "flaky") public void testDelayedTransactionMessages() throws Exception { String topic = NAMESPACE1 + "/testDelayedTransactionMessages"; @@ -1594,27 +1645,27 @@ public void testDelayedTransactionMessages() throws Exception { for (int i = 0; i < 10; i++) { producer.newMessage(transaction) .value("msg-" + i) - .deliverAfter(5, TimeUnit.SECONDS) + .deliverAfter(7, TimeUnit.SECONDS) .sendAsync(); } producer.flush(); transaction.commit().get(); + Message msg; + for (int i = 0; i < 10; i++) { + msg = failoverConsumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS); + assertEquals(msg.getValue(), "msg-" + i); + } // Failover consumer will receive the messages immediately while // the shared consumer will get them after the delay - Message msg = sharedConsumer.receive(300, TimeUnit.MILLISECONDS); + msg = sharedConsumer.receive(waitTimeForCannotReceiveMsgInSec, TimeUnit.SECONDS); assertNull(msg); - for (int i = 0; i < 10; i++) { - msg = failoverConsumer.receive(100, TimeUnit.MILLISECONDS); - assertEquals(msg.getValue(), "msg-" + i); - } - Set receivedMsgs = new TreeSet<>(); for (int i = 0; i < 10; i++) { - msg = sharedConsumer.receive(10, TimeUnit.SECONDS); + msg = sharedConsumer.receive(waitTimeForCanReceiveMsgInSec, TimeUnit.SECONDS); receivedMsgs.add(msg.getValue()); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndWithoutBatchIndexAckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndWithoutBatchIndexAckTest.java index 52faae2f8ea1f..df4ad32b6a8ae 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndWithoutBatchIndexAckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndWithoutBatchIndexAckTest.java @@ -30,7 +30,7 @@ @Test(groups = "broker-impl") public class TransactionEndToEndWithoutBatchIndexAckTest extends TransactionEndToEndTest { - @BeforeClass + @BeforeClass(alwaysRun = true) protected void setup() throws Exception { conf.setAcknowledgmentAtBatchIndexLevelEnabled(false); setUpBase(1, NUM_PARTITIONS, TOPIC_OUTPUT, TOPIC_PARTITION); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/UnAcknowledgedMessagesTimeoutTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/UnAcknowledgedMessagesTimeoutTest.java index 34e4f6a4d6d42..a0497b294c95f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/UnAcknowledgedMessagesTimeoutTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/UnAcknowledgedMessagesTimeoutTest.java @@ -185,7 +185,7 @@ public void testExclusiveCumulativeAckedNormalTopic(boolean isRedeliveryTracker) } long size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size(); assertEquals(size, totalMessages); - log.info("Comulative Ack sent for " + new String(lastMessage.getData())); + log.info("Cumulative Ack sent for " + new String(lastMessage.getData())); log.info("Message ID details " + lastMessage.getMessageId().toString()); consumer.acknowledgeCumulative(lastMessage); size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/metrics/ClientMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/metrics/ClientMetricsTest.java new file mode 100644 index 0000000000000..31305123c4148 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/metrics/ClientMetricsTest.java @@ -0,0 +1,336 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.metrics; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.fail; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.metrics.data.MetricDataType; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionType; +import org.assertj.core.api.Assertions; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test(groups = "broker-api") +public class ClientMetricsTest extends ProducerConsumerBase { + + InMemoryMetricReader reader; + OpenTelemetry otel; + + @BeforeMethod + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + + this.reader = InMemoryMetricReader.create(); + SdkMeterProvider sdkMeterProvider = SdkMeterProvider.builder() + .registerMetricReader(reader) + .build(); + this.otel = OpenTelemetrySdk.builder().setMeterProvider(sdkMeterProvider).build(); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + private Map collectMetrics() { + Map metrics = new TreeMap<>(); + for (MetricData md : reader.collectAllMetrics()) { + metrics.put(md.getName(), md); + } + return metrics; + } + + private void assertCounterValue(Map metrics, String name, long expectedValue, + Attributes expectedAttributes) { + assertEquals(getCounterValue(metrics, name, expectedAttributes), expectedValue); + } + + private long getCounterValue(Map metrics, String name, + Attributes expectedAttributes) { + MetricData md = metrics.get(name); + assertNotNull(md, "metric not found: " + name); + assertEquals(md.getType(), MetricDataType.LONG_SUM); + + for (var ex : md.getLongSumData().getPoints()) { + if (ex.getAttributes().equals(expectedAttributes)) { + return ex.getValue(); + } + } + + fail("metric attributes not found: " + expectedAttributes); + return -1; + } + + private void assertHistoCountValue(Map metrics, String name, long expectedCount, + Attributes expectedAttributes) { + assertEquals(getHistoCountValue(metrics, name, expectedAttributes), expectedCount); + } + + private long getHistoCountValue(Map metrics, String name, + Attributes expectedAttributes) { + MetricData md = metrics.get(name); + assertNotNull(md, "metric not found: " + name); + assertEquals(md.getType(), MetricDataType.HISTOGRAM); + + for (var ex : md.getHistogramData().getPoints()) { + if (ex.getAttributes().equals(expectedAttributes)) { + return ex.getCount(); + } + } + + fail("metric attributes not found: " + expectedAttributes); + return -1; + } + + @Test + public void testProducerMetrics() throws Exception { + String topic = newTopicName(); + + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsar.getBrokerServiceUrl()) + .openTelemetry(otel) + .build(); + + Producer producer = client.newProducer(Schema.STRING) + .topic(topic) + .create(); + + for (int i = 0; i < 5; i++) { + producer.send("Hello"); + } + + Attributes nsAttrs = Attributes.builder() + .put("pulsar.tenant", "my-property") + .put("pulsar.namespace", "my-property/my-ns") + .build(); + Attributes nsAttrsSuccess = nsAttrs.toBuilder() + .put("pulsar.response.status", "success") + .build(); + + var metrics = collectMetrics(); + + assertCounterValue(metrics, "pulsar.client.connection.opened", 1, Attributes.empty()); + assertCounterValue(metrics, "pulsar.client.producer.message.pending.count", 0, nsAttrs); + assertCounterValue(metrics, "pulsar.client.producer.message.pending.size", 0, nsAttrs); + + assertHistoCountValue(metrics, "pulsar.client.lookup.duration", 1, + Attributes.builder() + .put("pulsar.lookup.transport-type", "binary") + .put("pulsar.lookup.type", "topic") + .put("pulsar.response.status", "success") + .build()); + assertHistoCountValue(metrics, "pulsar.client.lookup.duration", 1, + Attributes.builder() + .put("pulsar.lookup.transport-type", "binary") + .put("pulsar.lookup.type", "metadata") + .put("pulsar.response.status", "success") + .build()); + + assertHistoCountValue(metrics, "pulsar.client.producer.message.send.duration", 5, nsAttrsSuccess); + assertHistoCountValue(metrics, "pulsar.client.producer.rpc.send.duration", 5, nsAttrsSuccess); + assertCounterValue(metrics, "pulsar.client.producer.message.send.size", "hello".length() * 5, nsAttrs); + + + assertCounterValue(metrics, "pulsar.client.producer.opened", 1, nsAttrs); + + producer.close(); + client.close(); + + metrics = collectMetrics(); + assertCounterValue(metrics, "pulsar.client.producer.closed", 1, nsAttrs); + assertCounterValue(metrics, "pulsar.client.connection.closed", 1, Attributes.empty()); + } + + @Test + public void testConnectionsFailedMetrics() throws Exception { + String topic = newTopicName(); + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl("pulsar://invalid-pulsar-address:1234") + .operationTimeout(3, TimeUnit.SECONDS) + .openTelemetry(otel) + .build(); + + Assertions.assertThatThrownBy(() -> { + client.newProducer(Schema.STRING) + .topic(topic) + .create(); + }).isInstanceOf(Exception.class); + + + var metrics = collectMetrics(); + + Assertions.assertThat( + getCounterValue(metrics, "pulsar.client.connection.failed", + Attributes.builder().put("pulsar.failure.type", "tcp-failed").build())) + .isGreaterThanOrEqualTo(1L); + } + + @Test + public void testPublishFailedMetrics() throws Exception { + String topic = newTopicName(); + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(admin.getServiceUrl()) + .operationTimeout(3, TimeUnit.SECONDS) + .openTelemetry(otel) + .build(); + + @Cleanup + Producer producer = client.newProducer(Schema.STRING) + .topic(topic) + .sendTimeout(3, TimeUnit.SECONDS) + .create(); + + // Make the client switch to non-existing broker to make publish fail + client.updateServiceUrl("pulsar://invalid-address:6650"); + + + try { + producer.send("Hello"); + fail("Should have failed to publish"); + } catch (Exception e) { + // expected + } + + var metrics = collectMetrics(); + + Attributes nsAttrs = Attributes.builder() + .put("pulsar.tenant", "my-property") + .put("pulsar.namespace", "my-property/my-ns") + .build(); + Attributes nsAttrsFailure = nsAttrs.toBuilder() + .put("pulsar.response.status", "failed") + .build(); + + assertCounterValue(metrics, "pulsar.client.producer.message.pending.count", 0, nsAttrs); + assertCounterValue(metrics, "pulsar.client.producer.message.pending.size", 0, nsAttrs); + assertHistoCountValue(metrics, "pulsar.client.producer.message.send.duration", 1, nsAttrsFailure); + assertHistoCountValue(metrics, "pulsar.client.producer.rpc.send.duration", 1, nsAttrsFailure); + } + + @Test + public void testConsumerMetrics() throws Exception { + String topic = newTopicName(); + + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsar.getBrokerServiceUrl()) + .openTelemetry(otel) + .build(); + + @Cleanup + Producer producer = client.newProducer(Schema.STRING) + .topic(topic) + .create(); + + Consumer consumer = client.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName("my-sub") + .ackTimeout(1, TimeUnit.SECONDS) + .subscriptionType(SubscriptionType.Shared) + .subscribe(); + + for (int i = 0; i < 10; i++) { + producer.send("Hello"); + } + + Thread.sleep(1000); + + Attributes nsAttrs = Attributes.builder() + .put("pulsar.tenant", "my-property") + .put("pulsar.namespace", "my-property/my-ns") + .put("pulsar.subscription", "my-sub") + .build(); + var metrics = collectMetrics(); + + assertCounterValue(metrics, "pulsar.client.connection.opened", 1, Attributes.empty()); + + assertHistoCountValue(metrics, "pulsar.client.lookup.duration", 2, + Attributes.builder() + .put("pulsar.lookup.transport-type", "binary") + .put("pulsar.lookup.type", "topic") + .put("pulsar.response.status", "success") + .build()); + assertHistoCountValue(metrics, "pulsar.client.lookup.duration", 2, + Attributes.builder() + .put("pulsar.lookup.transport-type", "binary") + .put("pulsar.lookup.type", "metadata") + .put("pulsar.response.status", "success") + .build()); + + assertCounterValue(metrics, "pulsar.client.consumer.receive_queue.count", 10, nsAttrs); + assertCounterValue(metrics, "pulsar.client.consumer.receive_queue.size", "hello".length() * 10, nsAttrs); + assertCounterValue(metrics, "pulsar.client.consumer.opened", 1, nsAttrs); + + Message msg1 = consumer.receive(); + consumer.acknowledge(msg1); + + Message msg2 = consumer.receive(); + consumer.negativeAcknowledge(msg2); + + /* Message msg3 = */ consumer.receive(); + + metrics = collectMetrics(); + assertCounterValue(metrics, "pulsar.client.consumer.receive_queue.count", 7, nsAttrs); + assertCounterValue(metrics, "pulsar.client.consumer.receive_queue.size", "hello".length() * 7, nsAttrs); + assertCounterValue(metrics, "pulsar.client.consumer.message.received.count", 3, nsAttrs); + assertCounterValue(metrics, "pulsar.client.consumer.message.received.size", "hello".length() * 3, nsAttrs); + + + // Let msg3 to reach ack-timeout + Thread.sleep(3000); + + metrics = collectMetrics(); + assertCounterValue(metrics, "pulsar.client.consumer.receive_queue.count", 8, nsAttrs); + assertCounterValue(metrics, "pulsar.client.consumer.receive_queue.size", "hello".length() * 8, nsAttrs); + + assertCounterValue(metrics, "pulsar.client.consumer.message.ack", 1, nsAttrs); + assertCounterValue(metrics, "pulsar.client.consumer.message.nack", 1, nsAttrs); + assertCounterValue(metrics, "pulsar.client.consumer.message.ack.timeout", 1, nsAttrs); + + client.close(); + + metrics = collectMetrics(); + assertCounterValue(metrics, "pulsar.client.consumer.closed", 1, nsAttrs); + assertCounterValue(metrics, "pulsar.client.connection.closed", 1, Attributes.empty()); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/NamespaceBundlesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/NamespaceBundlesTest.java index a8a4610f7f3fe..809ce579ce94b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/NamespaceBundlesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/NamespaceBundlesTest.java @@ -40,6 +40,8 @@ import java.util.concurrent.CompletableFuture; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.resources.LocalPoliciesResources; import org.apache.pulsar.broker.resources.NamespaceResources; import org.apache.pulsar.broker.resources.PulsarResources; @@ -91,6 +93,7 @@ public void testConstructor() throws Exception { private NamespaceBundleFactory getNamespaceBundleFactory() { PulsarService pulsar = mock(PulsarService.class); MetadataStoreExtended store = mock(MetadataStoreExtended.class); + when(pulsar.getConfiguration()).thenReturn(new ServiceConfiguration()); when(pulsar.getLocalMetadataStore()).thenReturn(store); when(pulsar.getConfigurationMetadataStore()).thenReturn(store); @@ -103,7 +106,12 @@ private NamespaceBundleFactory getNamespaceBundleFactory() { when(resources.getNamespaceResources()).thenReturn(mock(NamespaceResources.class)); when(resources.getNamespaceResources().getPoliciesAsync(any())).thenReturn( CompletableFuture.completedFuture(Optional.empty())); - return NamespaceBundleFactory.createFactory(pulsar, Hashing.crc32()); + NamespaceBundleFactory factory1 = NamespaceBundleFactory.createFactory(pulsar, Hashing.crc32()); + NamespaceService namespaceService = mock(NamespaceService.class); + when(namespaceService.getNamespaceBundleFactory()).thenReturn(factory1); + when(pulsar.getNamespaceService()).thenReturn(namespaceService); + return factory1; + } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/ServiceConfigurationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/ServiceConfigurationTest.java index 55971c15adf68..5972c6f724d8c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/ServiceConfigurationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/common/naming/ServiceConfigurationTest.java @@ -35,6 +35,7 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Properties; @@ -73,6 +74,10 @@ public void testInit() throws Exception { assertEquals(config.getManagedLedgerDataReadPriority(), "bookkeeper-first"); assertEquals(config.getBacklogQuotaDefaultLimitGB(), 0.05); assertEquals(config.getHttpMaxRequestHeaderSize(), 1234); + assertEquals(config.isDispatcherPauseOnAckStatePersistentEnabled(), true); + assertEquals(config.getMaxSecondsToClearTopicNameCache(), 1); + assertEquals(config.getTopicNameCacheMaxCapacity(), 200); + assertEquals(config.isCreateTopicToRemoteClusterForReplication(), false); OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.create(config.getProperties()); assertEquals(offloadPolicies.getManagedLedgerOffloadedReadPriority().getValue(), "bookkeeper-first"); } @@ -289,6 +294,8 @@ public void testTransactionBatchConfigurations() throws Exception{ assertEquals(configuration.getTransactionPendingAckBatchedWriteMaxRecords(), 512); assertEquals(configuration.getTransactionPendingAckBatchedWriteMaxSize(), 1024 * 1024 * 4); assertEquals(configuration.getTransactionPendingAckBatchedWriteMaxDelayInMillis(), 1); + assertEquals(configuration.isDispatcherPauseOnAckStatePersistentEnabled(), false); + assertEquals(configuration.isCreateTopicToRemoteClusterForReplication(), true); } // pulsar_broker_test.conf. try (InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(fileName)) { @@ -301,6 +308,8 @@ public void testTransactionBatchConfigurations() throws Exception{ assertEquals(configuration.getTransactionPendingAckBatchedWriteMaxRecords(), 44); assertEquals(configuration.getTransactionPendingAckBatchedWriteMaxSize(), 55); assertEquals(configuration.getTransactionPendingAckBatchedWriteMaxDelayInMillis(), 66); + assertEquals(configuration.isDispatcherPauseOnAckStatePersistentEnabled(), true); + assertEquals(configuration.isCreateTopicToRemoteClusterForReplication(), false); } // string input stream. StringBuilder stringBuilder = new StringBuilder(); @@ -312,6 +321,8 @@ public void testTransactionBatchConfigurations() throws Exception{ stringBuilder.append("transactionPendingAckBatchedWriteMaxRecords=521").append(System.lineSeparator()); stringBuilder.append("transactionPendingAckBatchedWriteMaxSize=1025").append(System.lineSeparator()); stringBuilder.append("transactionPendingAckBatchedWriteMaxDelayInMillis=20").append(System.lineSeparator()); + stringBuilder.append("dispatcherPauseOnAckStatePersistentEnabled=true").append(System.lineSeparator()); + stringBuilder.append("createTopicToRemoteClusterForReplication=false").append(System.lineSeparator()); try(ByteArrayInputStream inputStream = new ByteArrayInputStream(stringBuilder.toString().getBytes(StandardCharsets.UTF_8))){ configuration = PulsarConfigurationLoader.create(inputStream, ServiceConfiguration.class); @@ -323,6 +334,8 @@ public void testTransactionBatchConfigurations() throws Exception{ assertEquals(configuration.getTransactionPendingAckBatchedWriteMaxRecords(), 521); assertEquals(configuration.getTransactionPendingAckBatchedWriteMaxSize(), 1025); assertEquals(configuration.getTransactionPendingAckBatchedWriteMaxDelayInMillis(), 20); + assertEquals(configuration.isDispatcherPauseOnAckStatePersistentEnabled(), true); + assertEquals(configuration.isCreateTopicToRemoteClusterForReplication(), false); } } @@ -370,4 +383,28 @@ public void testAllowAutoTopicCreationType() throws Exception { conf = PulsarConfigurationLoader.create(properties, ServiceConfiguration.class); assertEquals(conf.getAllowAutoTopicCreationType(), TopicType.NON_PARTITIONED); } + + @Test + public void testTopicNameCacheConfiguration() throws Exception { + ServiceConfiguration conf; + final Properties properties = new Properties(); + properties.setProperty("maxSecondsToClearTopicNameCache", "2"); + properties.setProperty("topicNameCacheMaxCapacity", "100"); + conf = PulsarConfigurationLoader.create(properties, ServiceConfiguration.class); + assertEquals(conf.getMaxSecondsToClearTopicNameCache(), 2); + assertEquals(conf.getTopicNameCacheMaxCapacity(), 100); + } + + @Test + public void testLookupProperties() throws Exception { + var confFile = "lookup.key1=value1\nkey=value\nlookup.key2=value2"; + var conf = (ServiceConfiguration) PulsarConfigurationLoader.create( + new ByteArrayInputStream(confFile.getBytes()), ServiceConfiguration.class); + assertEquals(conf.lookupProperties(), Map.of("lookup.key1", "value1", "lookup.key2", "value2")); + + confFile = confFile + "\nlookupPropertyPrefix=lookup.key2"; + conf = PulsarConfigurationLoader.create(new ByteArrayInputStream(confFile.getBytes()), + ServiceConfiguration.class); + assertEquals(conf.lookupProperties(), Map.of("lookup.key2", "value2")); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicImplTest.java index c9933b7ef8f26..9d89207d06fe9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicImplTest.java @@ -18,8 +18,10 @@ */ package org.apache.pulsar.compaction; -import static org.mockito.Mockito.*; -import static org.testng.Assert.*; +import static org.mockito.Mockito.anyLong; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; +import static org.testng.Assert.assertEquals; import com.github.benmanes.caffeine.cache.AsyncLoadingCache; import com.github.benmanes.caffeine.cache.CacheLoader; import com.github.benmanes.caffeine.cache.Caffeine; @@ -29,7 +31,8 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; import java.util.function.Supplier; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.pulsar.common.api.proto.MessageIdData; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; @@ -107,7 +110,7 @@ public void testFindStartPointLoop(long start, long end, long targetMessageId) { AsyncLoadingCache cache = Caffeine.newBuilder() .buildAsync(mockCacheLoader(start, end, targetMessageId, bingoMarker)); // Do test. - PositionImpl targetPosition = PositionImpl.get(DEFAULT_LEDGER_ID, targetMessageId); + Position targetPosition = PositionFactory.create(DEFAULT_LEDGER_ID, targetMessageId); CompletableFuture promise = new CompletableFuture<>(); CompactedTopicImpl.findStartPointLoop(targetPosition, start, end, promise, cache); long result = promise.join(); @@ -137,7 +140,7 @@ public void testRecursionNumberOfFindStartPointLoop() { // executed "findStartPointLoop". Supplier loopCounter = () -> invokeCounterOfCacheGet.get() / 3; // Do test. - PositionImpl targetPosition = PositionImpl.get(DEFAULT_LEDGER_ID, targetMessageId); + Position targetPosition = PositionFactory.create(DEFAULT_LEDGER_ID, targetMessageId); CompletableFuture promise = new CompletableFuture<>(); CompactedTopicImpl.findStartPointLoop(targetPosition, start, end, promise, cacheWithCounter); // Do verify. diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java index 957671b7f8d9c..2692e6fa19698 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java @@ -21,10 +21,8 @@ import static org.apache.pulsar.compaction.Compactor.COMPACTION_SUBSCRIPTION; import com.github.benmanes.caffeine.cache.AsyncLoadingCache; import com.google.common.collect.Sets; - import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; - import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collections; @@ -36,18 +34,16 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.IntStream; - import lombok.Cleanup; - import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Triple; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; -import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.LongRunningProcessStatus; @@ -67,6 +63,7 @@ import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.util.FutureUtil; import org.awaitility.Awaitility; +import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -164,8 +161,9 @@ public void cleanup() throws Exception { @Test public void testEntryLookup() throws Exception { + @Cleanup BookKeeper bk = pulsar.getBookKeeperClientFactory().create( - this.conf, null, null, Optional.empty(), null); + this.conf, null, null, Optional.empty(), null).get(); Triple>, List>> compactedLedgerData = buildCompactedLedger(bk, 500); @@ -183,18 +181,18 @@ public void testEntryLookup() throws Exception { Pair lastPosition = positions.get(positions.size() - 1); // check ids before and after ids in compacted ledger - Assert.assertEquals(CompactedTopicImpl.findStartPoint(new PositionImpl(0, 0), lastEntryId, cache).get(), + Assert.assertEquals(CompactedTopicImpl.findStartPoint(PositionFactory.create(0, 0), lastEntryId, cache).get(), Long.valueOf(0)); - Assert.assertEquals(CompactedTopicImpl.findStartPoint(new PositionImpl(Long.MAX_VALUE, 0), + Assert.assertEquals(CompactedTopicImpl.findStartPoint(PositionFactory.create(Long.MAX_VALUE, 0), lastEntryId, cache).get(), Long.valueOf(CompactedTopicImpl.NEWER_THAN_COMPACTED)); // entry 0 is never in compacted ledger due to how we generate dummy - Assert.assertEquals(CompactedTopicImpl.findStartPoint(new PositionImpl(firstPositionId.getLedgerId(), 0), + Assert.assertEquals(CompactedTopicImpl.findStartPoint(PositionFactory.create(firstPositionId.getLedgerId(), 0), lastEntryId, cache).get(), Long.valueOf(0)); // check next id after last id in compacted ledger - Assert.assertEquals(CompactedTopicImpl.findStartPoint(new PositionImpl(lastPosition.getLeft().getLedgerId(), + Assert.assertEquals(CompactedTopicImpl.findStartPoint(PositionFactory.create(lastPosition.getLeft().getLedgerId(), lastPosition.getLeft().getEntryId() + 1), lastEntryId, cache).get(), Long.valueOf(CompactedTopicImpl.NEWER_THAN_COMPACTED)); @@ -205,22 +203,23 @@ public void testEntryLookup() throws Exception { // Check ids we know are in compacted ledger for (Pair p : positions) { - PositionImpl pos = new PositionImpl(p.getLeft().getLedgerId(), p.getLeft().getEntryId()); + Position pos = PositionFactory.create(p.getLeft().getLedgerId(), p.getLeft().getEntryId()); Long got = CompactedTopicImpl.findStartPoint(pos, lastEntryId, cache).get(); Assert.assertEquals(got, p.getRight()); } // Check ids we know are in the gaps of the compacted ledger for (Pair gap : idsInGaps) { - PositionImpl pos = new PositionImpl(gap.getLeft().getLedgerId(), gap.getLeft().getEntryId()); + Position pos = PositionFactory.create(gap.getLeft().getLedgerId(), gap.getLeft().getEntryId()); Assert.assertEquals(CompactedTopicImpl.findStartPoint(pos, lastEntryId, cache).get(), gap.getRight()); } } @Test public void testCleanupOldCompactedTopicLedger() throws Exception { + @Cleanup BookKeeper bk = pulsar.getBookKeeperClientFactory().create( - this.conf, null, null, Optional.empty(), null); + this.conf, null, null, Optional.empty(), null).get(); LedgerHandle oldCompactedLedger = bk.createLedger(1, 1, Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, @@ -233,7 +232,7 @@ public void testCleanupOldCompactedTopicLedger() throws Exception { // set the compacted topic ledger CompactedTopicImpl compactedTopic = new CompactedTopicImpl(bk); - compactedTopic.newCompactedLedger(new PositionImpl(1,2), oldCompactedLedger.getId()).get(); + compactedTopic.newCompactedLedger(PositionFactory.create(1,2), oldCompactedLedger.getId()).get(); // ensure both ledgers still exist, can be opened bk.openLedger(oldCompactedLedger.getId(), @@ -244,7 +243,7 @@ public void testCleanupOldCompactedTopicLedger() throws Exception { Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD).close(); // update the compacted topic ledger - PositionImpl newHorizon = new PositionImpl(1,3); + Position newHorizon = PositionFactory.create(1,3); compactedTopic.newCompactedLedger(newHorizon, newCompactedLedger.getId()).get(); // Make sure the old compacted ledger still exist after the new compacted ledger created. @@ -665,7 +664,7 @@ public void testReader() throws Exception { producer.newMessage().key("k").value(("value").getBytes()).send(); producer.newMessage().key("k").value(null).send(); - pulsar.getCompactor().compact(topic).get(); + ((PulsarCompactionServiceFactory)pulsar.getCompactionServiceFactory()).getCompactor().compact(topic).get(); Awaitility.await() .pollInterval(3, TimeUnit.SECONDS) @@ -845,4 +844,67 @@ public void testReadCompactedLatestMessageWithInclusive() throws Exception { Assert.assertTrue(reader.hasMessageAvailable()); Assert.assertEquals(reader.readNext().getMessageId(), lastMessage.get()); } + + @Test + public void testCompactWithConcurrentGetCompactionHorizonAndCompactedTopicContext() throws Exception { + @Cleanup + BookKeeper bk = pulsar.getBookKeeperClientFactory().create( + this.conf, null, null, Optional.empty(), null).get(); + + Mockito.doAnswer(invocation -> { + Thread.sleep(1500); + invocation.callRealMethod(); + return null; + }).when(bk).asyncOpenLedger(Mockito.anyLong(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + + LedgerHandle oldCompactedLedger = bk.createLedger(1, 1, + Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, + Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD); + oldCompactedLedger.close(); + LedgerHandle newCompactedLedger = bk.createLedger(1, 1, + Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, + Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD); + newCompactedLedger.close(); + + CompactedTopicImpl compactedTopic = new CompactedTopicImpl(bk); + + Position oldHorizon = PositionFactory.create(1, 2); + var future = CompletableFuture.supplyAsync(() -> { + // set the compacted topic ledger + return compactedTopic.newCompactedLedger(oldHorizon, oldCompactedLedger.getId()); + }); + Thread.sleep(500); + + Optional compactionHorizon = compactedTopic.getCompactionHorizon(); + CompletableFuture compactedTopicContext = + compactedTopic.getCompactedTopicContextFuture(); + + if (compactedTopicContext != null) { + Assert.assertEquals(compactionHorizon.get(), oldHorizon); + Assert.assertNotNull(compactedTopicContext); + Assert.assertEquals(compactedTopicContext.join().ledger.getId(), oldCompactedLedger.getId()); + } else { + Assert.assertTrue(compactionHorizon.isEmpty()); + } + + future.join(); + + Position newHorizon = PositionFactory.create(1, 3); + var future2 = CompletableFuture.supplyAsync(() -> { + // update the compacted topic ledger + return compactedTopic.newCompactedLedger(newHorizon, newCompactedLedger.getId()); + }); + Thread.sleep(500); + + compactionHorizon = compactedTopic.getCompactionHorizon(); + compactedTopicContext = compactedTopic.getCompactedTopicContextFuture(); + + if (compactedTopicContext.join().ledger.getId() == newCompactedLedger.getId()) { + Assert.assertEquals(compactionHorizon.get(), newHorizon); + } else { + Assert.assertEquals(compactionHorizon.get(), oldHorizon); + } + + future2.join(); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicUtilsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicUtilsTest.java new file mode 100644 index 0000000000000..25f42b86b26ba --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicUtilsTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.compaction; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class CompactedTopicUtilsTest { + + @Test + public void testReadCompactedEntriesWithEmptyEntries() throws ExecutionException, InterruptedException { + Position lastCompactedPosition = PositionFactory.create(1, 100); + TopicCompactionService service = Mockito.mock(TopicCompactionService.class); + Mockito.doReturn(CompletableFuture.completedFuture(Collections.emptyList())) + .when(service).readCompactedEntries(Mockito.any(), Mockito.intThat(argument -> argument > 0)); + Mockito.doReturn(CompletableFuture.completedFuture(lastCompactedPosition)).when(service) + .getLastCompactedPosition(); + + + Position initPosition = PositionFactory.create(1, 90); + AtomicReference readPositionRef = new AtomicReference<>(initPosition.getNext()); + ManagedCursorImpl cursor = Mockito.mock(ManagedCursorImpl.class); + Mockito.doReturn(readPositionRef.get()).when(cursor).getReadPosition(); + Mockito.doReturn(1).when(cursor).applyMaxSizeCap(Mockito.anyInt(), Mockito.anyLong()); + Mockito.doAnswer(invocation -> { + readPositionRef.set(invocation.getArgument(0)); + return null; + }).when(cursor).seek(Mockito.any()); + + CompletableFuture> completableFuture = new CompletableFuture<>(); + final AtomicReference throwableRef = new AtomicReference<>(); + AsyncCallbacks.ReadEntriesCallback readEntriesCallback = new AsyncCallbacks.ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + completableFuture.complete(entries); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + completableFuture.completeExceptionally(exception); + throwableRef.set(exception); + } + }; + + CompactedTopicUtils.asyncReadCompactedEntries(service, cursor, 1, 100, + PositionFactory.LATEST, false, readEntriesCallback, false, null); + + List entries = completableFuture.get(); + Assert.assertTrue(entries.isEmpty()); + Assert.assertNull(throwableRef.get()); + Assert.assertEquals(readPositionRef.get(), lastCompactedPosition.getNext()); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionRetentionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionRetentionTest.java index 055c595fbfec8..45dc30d21df64 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionRetentionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionRetentionTest.java @@ -38,6 +38,7 @@ import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; @@ -45,9 +46,13 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.naming.SystemTopicNames; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.awaitility.Awaitility; +import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -57,7 +62,7 @@ public class CompactionRetentionTest extends MockedPulsarServiceBaseTest { protected ScheduledExecutorService compactionScheduler; protected BookKeeper bk; - private TwoPhaseCompactor compactor; + private PublishingOrderCompactor compactor; @BeforeMethod @Override @@ -73,8 +78,8 @@ public void setup() throws Exception { compactionScheduler = Executors.newSingleThreadScheduledExecutor( new ThreadFactoryBuilder().setNameFormat("compaction-%d").setDaemon(true).build()); - bk = pulsar.getBookKeeperClientFactory().create(this.conf, null, null, Optional.empty(), null); - compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); + bk = pulsar.getBookKeeperClientFactory().create(this.conf, null, null, Optional.empty(), null).get(); + compactor = new PublishingOrderCompactor(conf, pulsarClient, bk, compactionScheduler); } @AfterMethod(alwaysRun = true) @@ -212,6 +217,49 @@ public void testCompactionRetentionOnTopicCreationWithTopicPolicies() throws Exc ); } + @Test + public void testRetentionPolicesForSystemTopic() throws Exception { + String namespace = "my-tenant/my-ns"; + String topicPrefix = "persistent://" + namespace + "/"; + admin.namespaces().setRetention(namespace, new RetentionPolicies(-1, -1)); + // Check event topics and transaction internal topics. + for (String eventTopic : SystemTopicNames.EVENTS_TOPIC_NAMES) { + checkSystemTopicRetentionPolicy(topicPrefix + eventTopic); + } + checkSystemTopicRetentionPolicy(topicPrefix + SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN); + checkSystemTopicRetentionPolicy(topicPrefix + SystemTopicNames.TRANSACTION_COORDINATOR_LOG); + checkSystemTopicRetentionPolicy(topicPrefix + SystemTopicNames.PENDING_ACK_STORE_SUFFIX); + + // Check common topics. + checkCommonTopicRetentionPolicy(topicPrefix + "my-topic" + System.nanoTime()); + // Specify retention policies for system topic. + pulsar.getConfiguration().setTopicLevelPoliciesEnabled(true); + pulsar.getConfiguration().setSystemTopicEnabled(true); + admin.topics().createNonPartitionedTopic(topicPrefix + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT); + admin.topicPolicies().setRetention(topicPrefix + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT, + new RetentionPolicies(10, 10)); + Awaitility.await().untilAsserted(() -> { + checkTopicRetentionPolicy(topicPrefix + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT, + new RetentionPolicies(10, 10)); + }); + } + + private void checkSystemTopicRetentionPolicy(String topicName) throws Exception { + checkTopicRetentionPolicy(topicName, new RetentionPolicies(0, 0)); + + } + + private void checkCommonTopicRetentionPolicy(String topicName) throws Exception { + checkTopicRetentionPolicy(topicName, new RetentionPolicies(-1, -1)); + } + + private void checkTopicRetentionPolicy(String topicName, RetentionPolicies retentionPolicies) throws Exception { + ManagedLedgerConfig config = pulsar.getBrokerService() + .getManagedLedgerConfig(TopicName.get(topicName)).get(); + Assert.assertEquals(config.getRetentionSizeInMB(), retentionPolicies.getRetentionSizeInMB()); + Assert.assertEquals(config.getRetentionTimeMillis(),retentionPolicies.getRetentionTimeInMinutes() * 60000L); + } + private void testCompactionCursorRetention(String topic) throws Exception { Set keys = Sets.newHashSet("a", "b", "c"); Set keysToExpire = Sets.newHashSet("x1", "x2"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java index d11a2f87192ff..19f42a7e0570f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java @@ -25,6 +25,7 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; import com.google.common.collect.Sets; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -46,21 +47,29 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; import lombok.Cleanup; import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.api.OpenBuilder; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerInfo; import org.apache.bookkeeper.mledger.Position; +import org.apache.commons.lang3.mutable.MutableLong; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.service.persistent.SystemTopic; +import org.apache.pulsar.client.admin.LongRunningProcessStatus; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.CryptoKeyReader; @@ -70,12 +79,15 @@ import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; +import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Reader; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.impl.BatchMessageIdImpl; -import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.client.impl.ConsumerImpl; +import org.apache.pulsar.common.api.proto.CommandAck; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; @@ -85,6 +97,7 @@ import org.apache.pulsar.common.protocol.Markers; import org.apache.pulsar.common.util.FutureUtil; import org.awaitility.Awaitility; +import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -92,10 +105,11 @@ import org.testng.annotations.Test; @Test(groups = "broker-impl") +@Slf4j public class CompactionTest extends MockedPulsarServiceBaseTest { protected ScheduledExecutorService compactionScheduler; protected BookKeeper bk; - private TwoPhaseCompactor compactor; + private PublishingOrderCompactor compactor; @BeforeMethod @Override @@ -109,8 +123,8 @@ public void setup() throws Exception { compactionScheduler = Executors.newSingleThreadScheduledExecutor( new ThreadFactoryBuilder().setNameFormat("compaction-%d").setDaemon(true).build()); - bk = pulsar.getBookKeeperClientFactory().create(this.conf, null, null, Optional.empty(), null); - compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); + bk = pulsar.getBookKeeperClientFactory().create(this.conf, null, null, Optional.empty(), null).get(); + compactor = new PublishingOrderCompactor(conf, pulsarClient, bk, compactionScheduler); } @AfterMethod(alwaysRun = true) @@ -133,7 +147,7 @@ protected long compact(String topic, CryptoKeyReader cryptoKeyReader) return compactor.compact(topic).get(); } - protected TwoPhaseCompactor getCompactor() { + protected PublishingOrderCompactor getCompactor() { return compactor; } @@ -534,14 +548,8 @@ public void testBatchMessageIdsDontChange() throws Exception { Assert.assertEquals(message2.getKey(), "key2"); Assert.assertEquals(new String(message2.getData()), "my-message-3"); if (getCompactor() instanceof StrategicTwoPhaseCompactor) { - MessageIdImpl id = (MessageIdImpl) messages.get(0).getMessageId(); - MessageIdImpl id1 = new MessageIdImpl( - id.getLedgerId(), id.getEntryId(), id.getPartitionIndex()); - Assert.assertEquals(message1.getMessageId(), id1); - id = (MessageIdImpl) messages.get(2).getMessageId(); - MessageIdImpl id2 = new MessageIdImpl( - id.getLedgerId(), id.getEntryId(), id.getPartitionIndex()); - Assert.assertEquals(message2.getMessageId(), id2); + Assert.assertEquals(message1.getMessageId(), messages.get(0).getMessageId()); + Assert.assertEquals(message2.getMessageId(), messages.get(1).getMessageId()); } else { Assert.assertEquals(message1.getMessageId(), messages.get(0).getMessageId()); Assert.assertEquals(message2.getMessageId(), messages.get(2).getMessageId()); @@ -549,6 +557,60 @@ public void testBatchMessageIdsDontChange() throws Exception { } } + @Test + public void testBatchMessageWithNullValue() throws Exception { + String topic = "persistent://my-property/use/my-ns/my-topic1"; + + pulsarClient.newConsumer().topic(topic).subscriptionName("sub1") + .receiverQueueSize(1).readCompacted(true).subscribe().close(); + + try (Producer producer = pulsarClient.newProducer().topic(topic) + .maxPendingMessages(3) + .enableBatching(true) + .batchingMaxMessages(3) + .batchingMaxPublishDelay(1, TimeUnit.HOURS) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create() + ) { + // batch 1 + producer.newMessage().key("key1").value("my-message-1".getBytes()).sendAsync(); + producer.newMessage().key("key1").value(null).sendAsync(); + producer.newMessage().key("key2").value("my-message-3".getBytes()).send(); + + // batch 2 + producer.newMessage().key("key3").value("my-message-4".getBytes()).sendAsync(); + producer.newMessage().key("key3").value("my-message-5".getBytes()).sendAsync(); + producer.newMessage().key("key3").value("my-message-6".getBytes()).send(); + + // batch 3 + producer.newMessage().key("key4").value("my-message-7".getBytes()).sendAsync(); + producer.newMessage().key("key4").value(null).sendAsync(); + producer.newMessage().key("key5").value("my-message-9".getBytes()).send(); + } + + + // compact the topic + compact(topic); + + // Read messages before compaction to get ids + List> messages = new ArrayList<>(); + try (Consumer consumer = pulsarClient.newConsumer().topic(topic) + .subscriptionName("sub1").receiverQueueSize(1).readCompacted(true).subscribe()) { + while (true) { + Message message = consumer.receive(5, TimeUnit.SECONDS); + if (message == null) { + break; + } + messages.add(message); + } + } + + assertEquals(messages.size(), 3); + assertEquals(messages.get(0).getKey(), "key2"); + assertEquals(messages.get(1).getKey(), "key3"); + assertEquals(messages.get(2).getKey(), "key5"); + } + @Test public void testWholeBatchCompactedOut() throws Exception { String topic = "persistent://my-property/use/my-ns/my-topic1"; @@ -585,8 +647,17 @@ public void testWholeBatchCompactedOut() throws Exception { } } - @Test - public void testKeyLessMessagesPassThrough() throws Exception { + @DataProvider(name = "retainNullKey") + public static Object[][] retainNullKey() { + return new Object[][] {{true}, {false}}; + } + + @Test(dataProvider = "retainNullKey") + public void testKeyLessMessagesPassThrough(boolean retainNullKey) throws Exception { + conf.setTopicCompactionRetainNullKey(retainNullKey); + restartBroker(); + FieldUtils.writeField(compactor, "topicCompactionRetainNullKey", retainNullKey, true); + String topic = "persistent://my-property/use/my-ns/my-topic1"; // subscribe before sending anything, so that we get all messages @@ -627,29 +698,25 @@ public void testKeyLessMessagesPassThrough() throws Exception { Message m = consumer.receive(2, TimeUnit.SECONDS); assertNull(m); } else { - Message message1 = consumer.receive(); - Assert.assertFalse(message1.hasKey()); - Assert.assertEquals(new String(message1.getData()), "my-message-1"); - - Message message2 = consumer.receive(); - Assert.assertFalse(message2.hasKey()); - Assert.assertEquals(new String(message2.getData()), "my-message-2"); - - Message message3 = consumer.receive(); - Assert.assertEquals(message3.getKey(), "key1"); - Assert.assertEquals(new String(message3.getData()), "my-message-4"); - - Message message4 = consumer.receive(); - Assert.assertEquals(message4.getKey(), "key2"); - Assert.assertEquals(new String(message4.getData()), "my-message-6"); - - Message message5 = consumer.receive(); - Assert.assertFalse(message5.hasKey()); - Assert.assertEquals(new String(message5.getData()), "my-message-7"); + List> result = new ArrayList<>(); + while (true) { + Message message = consumer.receive(10, TimeUnit.SECONDS); + if (message == null) { + break; + } + result.add(Pair.of(message.getKey(), message.getData() == null ? null : new String(message.getData()))); + } - Message message6 = consumer.receive(); - Assert.assertFalse(message6.hasKey()); - Assert.assertEquals(new String(message6.getData()), "my-message-8"); + List> expectList; + if (retainNullKey) { + expectList = List.of( + Pair.of(null, "my-message-1"), Pair.of(null, "my-message-2"), + Pair.of("key1", "my-message-4"), Pair.of("key2", "my-message-6"), + Pair.of(null, "my-message-7"), Pair.of(null, "my-message-8")); + } else { + expectList = List.of(Pair.of("key1", "my-message-4"), Pair.of("key2", "my-message-6")); + } + Assert.assertEquals(result, expectList); } } } @@ -1317,7 +1384,7 @@ public void testEmptyPayloadDeletesWhenEncrypted() throws Exception { Message message4 = consumer.receive(); Assert.assertEquals(message4.getKey(), "key2"); - Assert.assertEquals(new String(message4.getData()), ""); + assertNull(message4.getData()); Message message5 = consumer.receive(); Assert.assertEquals(message5.getKey(), "key4"); @@ -1711,9 +1778,9 @@ public void testReadUnCompacted(boolean batchEnabled) throws PulsarClientExcepti @SneakyThrows @Test public void testHealthCheckTopicNotCompacted() { - NamespaceName heartbeatNamespaceV1 = NamespaceService.getHeartbeatNamespace(pulsar.getAdvertisedAddress(), pulsar.getConfiguration()); + NamespaceName heartbeatNamespaceV1 = NamespaceService.getHeartbeatNamespace(pulsar.getBrokerId(), pulsar.getConfiguration()); String topicV1 = "persistent://" + heartbeatNamespaceV1.toString() + "/healthcheck"; - NamespaceName heartbeatNamespaceV2 = NamespaceService.getHeartbeatNamespaceV2(pulsar.getAdvertisedAddress(), pulsar.getConfiguration()); + NamespaceName heartbeatNamespaceV2 = NamespaceService.getHeartbeatNamespaceV2(pulsar.getBrokerId(), pulsar.getConfiguration()); String topicV2 = heartbeatNamespaceV2.toString() + "/healthcheck"; Producer producer1 = pulsarClient.newProducer().topic(topicV1).create(); Producer producer2 = pulsarClient.newProducer().topic(topicV2).create(); @@ -1783,4 +1850,518 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { Assert.assertNotEquals(ledgerId, -1L); }); } + + @Test(timeOut = 100000) + public void testReceiverQueueSize() throws Exception { + final String topicName = "persistent://my-property/use/my-ns/testReceiverQueueSize" + UUID.randomUUID(); + final String subName = "my-sub"; + final int receiveQueueSize = 1; + @Cleanup + PulsarClient client = newPulsarClient(lookupUrl.toString(), 100); + Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(false).topic(topicName).create(); + + for (int i = 0; i < 10; i++) { + producer.newMessage().key(String.valueOf(i % 2)).value(String.valueOf(i)).sendAsync(); + } + producer.flush(); + + admin.topics().triggerCompaction(topicName); + + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS); + }); + + ConsumerImpl consumer = (ConsumerImpl) client.newConsumer(Schema.STRING) + .topic(topicName).readCompacted(true).receiverQueueSize(receiveQueueSize).subscriptionName(subName) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + //Give some time to consume + Awaitility.await() + .untilAsserted(() -> Assert.assertEquals(consumer.getStats().getMsgNumInReceiverQueue().intValue(), + receiveQueueSize)); + consumer.close(); + producer.close(); + } + + @Test + public void testDispatcherMaxReadSizeBytes() throws Exception { + final String topicName = + "persistent://my-property/use/my-ns/testDispatcherMaxReadSizeBytes" + UUID.randomUUID(); + final String subName = "my-sub"; + final int receiveQueueSize = 1; + @Cleanup + PulsarClient client = newPulsarClient(lookupUrl.toString(), 100); + Producer producer = pulsarClient.newProducer(Schema.BYTES) + .topic(topicName).create(); + + for (int i = 0; i < 10; i+=2) { + producer.newMessage().key(UUID.randomUUID().toString()).value(new byte[4*1024*1024]).send(); + } + producer.flush(); + + admin.topics().triggerCompaction(topicName); + + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS); + }); + + admin.topics().unload(topicName); + + + PersistentTopic topic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, true, Map.of()).get().get(); + TopicCompactionService topicCompactionService = Mockito.spy(topic.getTopicCompactionService()); + FieldUtils.writeDeclaredField(topic, "topicCompactionService", topicCompactionService, true); + + ConsumerImpl consumer = (ConsumerImpl) client.newConsumer(Schema.BYTES) + .topic(topicName).readCompacted(true).receiverQueueSize(receiveQueueSize).subscriptionName(subName) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + Awaitility.await().untilAsserted(() -> { + assertEquals(consumer.getStats().getMsgNumInReceiverQueue(), + 1); + }); + + Mockito.verify(topicCompactionService, Mockito.times(1)).readCompactedEntries(Mockito.any(), Mockito.same(1)); + + consumer.close(); + producer.close(); + } + + @Test + public void testCompactionDuplicate() throws Exception { + String topic = "persistent://my-property/use/my-ns/testCompactionDuplicate"; + final int numMessages = 1000; + final int maxKeys = 800; + + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topic) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + + // trigger compaction (create __compaction cursor) + admin.topics().triggerCompaction(topic); + + Map expected = new HashMap<>(); + Random r = new Random(0); + + pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close(); + + for (int j = 0; j < numMessages; j++) { + int keyIndex = r.nextInt(maxKeys); + String key = "key" + keyIndex; + byte[] data = ("my-message-" + key + "-" + j).getBytes(); + producer.newMessage().key(key).value(data).send(); + expected.put(key, data); + } + + producer.flush(); + + // trigger compaction + admin.topics().triggerCompaction(topic); + + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topics().compactionStatus(topic).status, + LongRunningProcessStatus.Status.RUNNING); + }); + + // Wait for phase one to complete + Thread.sleep(500); + + Optional previousTopicRef = pulsar.getBrokerService().getTopicIfExists(topic).get(); + Assert.assertTrue(previousTopicRef.isPresent()); + PersistentTopic previousPersistentTopic = (PersistentTopic) previousTopicRef.get(); + + // Unload topic make reader of compaction reconnect + admin.topics().unload(topic); + + Awaitility.await().untilAsserted(() -> { + LongRunningProcessStatus previousLongRunningProcessStatus = previousPersistentTopic.compactionStatus(); + + Optional currentTopicReference = pulsar.getBrokerService().getTopicReference(topic); + Assert.assertTrue(currentTopicReference.isPresent()); + PersistentTopic currentPersistentTopic = (PersistentTopic) currentTopicReference.get(); + LongRunningProcessStatus currentLongRunningProcessStatus = currentPersistentTopic.compactionStatus(); + + if (previousLongRunningProcessStatus.status == LongRunningProcessStatus.Status.ERROR + && (currentLongRunningProcessStatus.status == LongRunningProcessStatus.Status.NOT_RUN + || currentLongRunningProcessStatus.status == LongRunningProcessStatus.Status.ERROR)) { + // trigger compaction again + admin.topics().triggerCompaction(topic); + Assert.assertEquals(currentLongRunningProcessStatus.status, LongRunningProcessStatus.Status.SUCCESS); + } else if (previousLongRunningProcessStatus.status == LongRunningProcessStatus.Status.RUNNING) { + Assert.assertEquals(previousLongRunningProcessStatus.status, LongRunningProcessStatus.Status.SUCCESS); + } + }); + + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(topic, false); + // Compacted topic ledger should have same number of entry equals to number of unique key. + Assert.assertEquals(internalStats.compactedLedger.entries, expected.size()); + Assert.assertTrue(internalStats.compactedLedger.ledgerId > -1); + Assert.assertFalse(internalStats.compactedLedger.offloaded); + }); + + // consumer with readCompacted enabled only get compacted entries + try (Consumer consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1") + .readCompacted(true).subscribe()) { + while (true) { + Message m = consumer.receive(2, TimeUnit.SECONDS); + Assert.assertEquals(expected.remove(m.getKey()), m.getData()); + if (expected.isEmpty()) { + break; + } + } + } + } + + @Test + public void testDeleteCompactedLedger() throws Exception { + String topicName = "persistent://my-property/use/my-ns/testDeleteCompactedLedger"; + + final String subName = "my-sub"; + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(false).topic(topicName).create(); + + pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).readCompacted(true).subscribe().close(); + + for (int i = 0; i < 10; i++) { + producer.newMessage().key(String.valueOf(i % 2)).value(String.valueOf(i)).sendAsync(); + } + producer.flush(); + + compact(topicName); + + MutableLong compactedLedgerId = new MutableLong(-1); + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats stats = admin.topics().getInternalStats(topicName); + Assert.assertNotEquals(stats.compactedLedger.ledgerId, -1L); + compactedLedgerId.setValue(stats.compactedLedger.ledgerId); + Assert.assertEquals(stats.compactedLedger.entries, 2L); + }); + + // delete compacted ledger + admin.topics().deleteSubscription(topicName, "__compaction"); + + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats stats = admin.topics().getInternalStats(topicName); + Assert.assertEquals(stats.compactedLedger.ledgerId, -1L); + Assert.assertEquals(stats.compactedLedger.entries, -1L); + assertThrows(BKException.BKNoSuchLedgerExistsException.class, () -> pulsarTestContext.getBookKeeperClient() + .openLedger(compactedLedgerId.getValue(), BookKeeper.DigestType.CRC32C, new byte[]{})); + }); + + compact(topicName); + + MutableLong compactedLedgerId2 = new MutableLong(-1); + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats stats = admin.topics().getInternalStats(topicName); + Assert.assertNotEquals(stats.compactedLedger.ledgerId, -1L); + compactedLedgerId2.setValue(stats.compactedLedger.ledgerId); + Assert.assertEquals(stats.compactedLedger.entries, 2L); + }); + + producer.close(); + admin.topics().delete(topicName); + + Awaitility.await().untilAsserted(() -> assertThrows(BKException.BKNoSuchLedgerExistsException.class, + () -> pulsarTestContext.getBookKeeperClient().openLedger( + compactedLedgerId2.getValue(), BookKeeper.DigestType.CRC32, new byte[]{}))); + } + + @Test + public void testDeleteCompactedLedgerWithSlowAck() throws Exception { + // Disable topic level policies, since block ack thread may also block thread of delete topic policies. + conf.setTopicLevelPoliciesEnabled(false); + restartBroker(); + + String topicName = "persistent://my-property/use/my-ns/testDeleteCompactedLedgerWithSlowAck"; + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(false).topic(topicName).create(); + + pulsarClient.newConsumer().topic(topicName).subscriptionType(SubscriptionType.Exclusive) + .subscriptionName(Compactor.COMPACTION_SUBSCRIPTION) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).readCompacted(true).subscribe() + .close(); + + for (int i = 0; i < 10; i++) { + producer.newMessage().key(String.valueOf(i % 2)).value(String.valueOf(i)).sendAsync(); + } + producer.flush(); + + PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get(); + PersistentSubscription subscription = spy(topic.getSubscription(Compactor.COMPACTION_SUBSCRIPTION)); + topic.getSubscriptions().put(Compactor.COMPACTION_SUBSCRIPTION, subscription); + + AtomicLong compactedLedgerId = new AtomicLong(-1); + AtomicBoolean pauseAck = new AtomicBoolean(); + Mockito.doAnswer(invocationOnMock -> { + Map properties = (Map) invocationOnMock.getArguments()[2]; + log.info("acknowledgeMessage properties: {}", properties); + compactedLedgerId.set(properties.get(Compactor.COMPACTED_TOPIC_LEDGER_PROPERTY)); + pauseAck.set(true); + while (pauseAck.get()) { + Thread.sleep(200); + } + return invocationOnMock.callRealMethod(); + }).when(subscription).acknowledgeMessage(Mockito.any(), Mockito.eq( + CommandAck.AckType.Cumulative), Mockito.any()); + + admin.topics().triggerCompaction(topicName); + + while (!pauseAck.get()) { + Thread.sleep(100); + } + + CompletableFuture currentCompaction = + (CompletableFuture) FieldUtils.readDeclaredField(topic, "currentCompaction", true); + CompletableFuture spyCurrentCompaction = spy(currentCompaction); + FieldUtils.writeDeclaredField(topic, "currentCompaction", spyCurrentCompaction, true); + currentCompaction.whenComplete((obj, throwable) -> { + if (throwable != null) { + spyCurrentCompaction.completeExceptionally(throwable); + } else { + spyCurrentCompaction.complete(obj); + } + }); + Mockito.doAnswer(invocationOnMock -> { + pauseAck.set(false); + return invocationOnMock.callRealMethod(); + }).when(spyCurrentCompaction).handle(Mockito.any()); + + admin.topics().delete(topicName, true); + + Awaitility.await().untilAsserted(() -> assertThrows(BKException.BKNoSuchLedgerExistsException.class, + () -> pulsarTestContext.getBookKeeperClient().openLedger( + compactedLedgerId.get(), BookKeeper.DigestType.CRC32, new byte[]{}))); + } + + @Test + public void testCompactionWithTTL() throws Exception { + String topicName = "persistent://my-property/use/my-ns/testCompactionWithTTL"; + String subName = "sub"; + pulsarClient.newConsumer(Schema.STRING).topic(topicName).subscriptionName(subName).readCompacted(true) + .subscribe().close(); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(false).topic(topicName).create(); + + producer.newMessage().key("K1").value("V1").send(); + producer.newMessage().key("K2").value("V2").send(); + + admin.topics().triggerCompaction(topicName); + + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS); + }); + + producer.newMessage().key("K1").value("V3").send(); + producer.newMessage().key("K2").value("V4").send(); + + Thread.sleep(1000); + + // expire messages + admin.topics().expireMessagesForAllSubscriptions(topicName, 1); + + // trim the topic + admin.topics().unload(topicName); + + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(topicName, false); + assertEquals(internalStats.numberOfEntries, 4); + }); + + producer.newMessage().key("K3").value("V5").send(); + + admin.topics().triggerCompaction(topicName); + + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS); + }); + + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.STRING).topic(topicName) + .subscriptionName("sub-2") + .readCompacted(true) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + List result = new ArrayList<>(); + while (true) { + Message receive = consumer.receive(2, TimeUnit.SECONDS); + if (receive == null) { + break; + } + + result.add(receive.getValue()); + } + + Assert.assertEquals(result, List.of("V3", "V4", "V5")); + } + + @Test + public void testAcknowledgeWithReconnection() throws Exception { + final String topicName = "persistent://my-property/use/my-ns/testAcknowledge" + UUID.randomUUID(); + final String subName = "my-sub"; + @Cleanup + PulsarClient client = newPulsarClient(lookupUrl.toString(), 100); + Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(false).topic(topicName).create(); + + List expected = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + producer.newMessage().key(String.valueOf(i)).value(String.valueOf(i)).send(); + expected.add(String.valueOf(i)); + } + producer.flush(); + + admin.topics().triggerCompaction(topicName); + + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS); + }); + + // trim the topic + admin.topics().unload(topicName); + + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(topicName, false); + assertEquals(internalStats.numberOfEntries, 0); + }); + + ConsumerImpl consumer = (ConsumerImpl) client.newConsumer(Schema.STRING) + .topic(topicName).readCompacted(true).receiverQueueSize(1).subscriptionName(subName) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .isAckReceiptEnabled(true) + .subscribe(); + + List results = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + Message message = consumer.receive(3, TimeUnit.SECONDS); + if (message == null) { + break; + } + results.add(message.getValue()); + consumer.acknowledge(message); + } + + Awaitility.await().untilAsserted(() -> + assertEquals(admin.topics().getStats(topicName, true).getSubscriptions().get(subName).getMsgBacklog(), + 5)); + + // Make consumer reconnect to broker + admin.topics().unload(topicName); + + // Wait for consumer to reconnect and clear incomingMessages + consumer.pause(); + Awaitility.await().untilAsserted(() -> { + Assert.assertEquals(consumer.numMessagesInQueue(), 0); + }); + consumer.resume(); + + for (int i = 0; i < 5; i++) { + Message message = consumer.receive(3, TimeUnit.SECONDS); + if (message == null) { + break; + } + results.add(message.getValue()); + consumer.acknowledge(message); + } + + Awaitility.await().untilAsserted(() -> + assertEquals(admin.topics().getStats(topicName, true).getSubscriptions().get(subName).getMsgBacklog(), + 0)); + + Assert.assertEquals(results, expected); + + Message message = consumer.receive(3, TimeUnit.SECONDS); + Assert.assertNull(message); + + // Make consumer reconnect to broker + admin.topics().unload(topicName); + + producer.newMessage().key("K").value("V").send(); + Message message2 = consumer.receive(3, TimeUnit.SECONDS); + Assert.assertEquals(message2.getValue(), "V"); + consumer.acknowledge(message2); + + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(topicName); + Assert.assertEquals(internalStats.lastConfirmedEntry, + internalStats.cursors.get(subName).markDeletePosition); + }); + + consumer.close(); + producer.close(); + } + + @Test + public void testEarliestSubsAfterRollover() throws Exception { + final String topicName = "persistent://my-property/use/my-ns/testEarliestSubsAfterRollover" + UUID.randomUUID(); + final String subName = "my-sub"; + @Cleanup + PulsarClient client = newPulsarClient(lookupUrl.toString(), 100); + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(false).topic(topicName).create(); + + List expected = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + producer.newMessage().key(String.valueOf(i)).value(String.valueOf(i)).send(); + expected.add(String.valueOf(i)); + } + producer.flush(); + + admin.topics().triggerCompaction(topicName); + + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS); + }); + + // trim the topic + admin.topics().unload(topicName); + + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(topicName, false); + assertEquals(internalStats.numberOfEntries, 0); + }); + + // Make ml.getFirstPosition() return new ledger first position + producer.newMessage().key("K").value("V").send(); + expected.add("V"); + + @Cleanup + ConsumerImpl consumer = (ConsumerImpl) client.newConsumer(Schema.STRING) + .topic(topicName).readCompacted(true).receiverQueueSize(1).subscriptionName(subName) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .isAckReceiptEnabled(true) + .subscribe(); + + List results = new ArrayList<>(); + while (true) { + Message message = consumer.receive(3, TimeUnit.SECONDS); + if (message == null) { + break; + } + + results.add(message.getValue()); + consumer.acknowledge(message); + } + + Assert.assertEquals(results, expected); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorMXBeanImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorMXBeanImplTest.java index bbde59d7da8bd..73e7430bd2d08 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorMXBeanImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorMXBeanImplTest.java @@ -59,11 +59,6 @@ public void testSimple() throws Exception { assertTrue(compaction.getCompactionWriteThroughput() > 0L); mxBean.addCompactionLatencyOp(topic, 10, TimeUnit.NANOSECONDS); assertTrue(compaction.getCompactionLatencyBuckets()[0] > 0L); - mxBean.reset(); - assertEquals(compaction.getCompactionRemovedEventCount(), 0, 0); - assertEquals(compaction.getCompactionSucceedCount(), 0, 0); - assertEquals(compaction.getCompactionFailedCount(), 0, 0); - assertEquals(compaction.getCompactionDurationTimeInMills(), 0, 0); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java index e86be6a4db816..5cf7d33200d66 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java @@ -18,14 +18,17 @@ */ package org.apache.pulsar.compaction; +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricDoubleSumValue; +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongSumValue; import static org.apache.pulsar.client.impl.RawReaderTest.extractKey; - +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.util.concurrent.ThreadFactoryBuilder; - import io.netty.buffer.ByteBuf; - +import io.opentelemetry.api.common.Attributes; import java.util.ArrayList; import java.util.Enumeration; import java.util.HashMap; @@ -33,23 +36,41 @@ import java.util.Map; import java.util.Optional; import java.util.Random; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; - +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.LedgerEntry; import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.stats.OpenTelemetryTopicStats; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.apache.pulsar.client.admin.LongRunningProcessStatus; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.RawMessage; +import org.apache.pulsar.client.api.Reader; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.RawMessageImpl; import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; +import org.awaitility.Awaitility; import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -65,7 +86,6 @@ public class CompactorTest extends MockedPulsarServiceBaseTest { protected Compactor compactor; - @BeforeMethod @Override public void setup() throws Exception { @@ -80,8 +100,8 @@ public void setup() throws Exception { compactionScheduler = Executors.newSingleThreadScheduledExecutor( new ThreadFactoryBuilder().setNameFormat("compactor").setDaemon(true).build()); bk = pulsar.getBookKeeperClientFactory().create( - this.conf, null, null, Optional.empty(), null); - compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); + this.conf, null, null, Optional.empty(), null).get(); + compactor = new PublishingOrderCompactor(conf, pulsarClient, bk, compactionScheduler); } @@ -93,6 +113,12 @@ public void cleanup() throws Exception { compactionScheduler.shutdownNow(); } + @Override + protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder pulsarTestContextBuilder) { + super.customizeMainPulsarTestContextBuilder(pulsarTestContextBuilder); + pulsarTestContextBuilder.enableOpenTelemetry(true); + } + protected long compact(String topic) throws ExecutionException, InterruptedException { return compactor.compact(topic).get(); } @@ -101,16 +127,17 @@ protected Compactor getCompactor() { return compactor; } - private List compactAndVerify(String topic, Map expected, boolean checkMetrics) throws Exception { + protected List compactAndVerify(String topic, Map expected, boolean checkMetrics) + throws Exception { long compactedLedgerId = compact(topic); LedgerHandle ledger = bk.openLedger(compactedLedgerId, - Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, - Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD); + Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, + Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD); Assert.assertEquals(ledger.getLastAddConfirmed() + 1, // 0..lac - expected.size(), - "Should have as many entries as there is keys"); + expected.size(), + "Should have as many entries as there is keys"); List keys = new ArrayList<>(); Enumeration entries = ledger.readEntries(0, ledger.getLastAddConfirmed()); @@ -124,7 +151,7 @@ private List compactAndVerify(String topic, Map expected byte[] bytes = new byte[payload.readableBytes()]; payload.readBytes(bytes); Assert.assertEquals(bytes, expected.remove(key), - "Compacted version should match expected version"); + "Compacted version should match expected version"); m.close(); } if (checkMetrics) { @@ -148,17 +175,18 @@ public void testCompaction() throws Exception { final int numMessages = 1000; final int maxKeys = 10; + @Cleanup Producer producer = pulsarClient.newProducer().topic(topic) - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.SinglePartition) - .create(); + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); Map expected = new HashMap<>(); Random r = new Random(0); for (int j = 0; j < numMessages; j++) { int keyIndex = r.nextInt(maxKeys); - String key = "key"+keyIndex; + String key = "key" + keyIndex; byte[] data = ("my-message-" + key + "-" + j).getBytes(); producer.newMessage() .key(key) @@ -169,14 +197,92 @@ public void testCompaction() throws Exception { compactAndVerify(topic, expected, true); } + @Test + public void testAllCompactedOut() throws Exception { + String topicName = BrokerTestUtil.newUniqueName("persistent://my-property/use/my-ns/testAllCompactedOut"); + // set retain null key to true + boolean oldRetainNullKey = pulsar.getConfig().isTopicCompactionRetainNullKey(); + pulsar.getConfig().setTopicCompactionRetainNullKey(true); + this.restartBroker(); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(true).topic(topicName).batchingMaxMessages(3).create(); + + producer.newMessage().key("K1").value("V1").sendAsync(); + producer.newMessage().key("K2").value("V2").sendAsync(); + producer.newMessage().key("K2").value(null).sendAsync(); + producer.flush(); + + admin.topics().triggerCompaction(topicName); + + Awaitility.await().untilAsserted(() -> { + Assert.assertEquals(admin.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS); + }); + + var attributes = Attributes.builder() + .put(OpenTelemetryAttributes.PULSAR_DOMAIN, "persistent") + .put(OpenTelemetryAttributes.PULSAR_TENANT, "my-property") + .put(OpenTelemetryAttributes.PULSAR_NAMESPACE, "my-property/use/my-ns") + .put(OpenTelemetryAttributes.PULSAR_TOPIC, topicName) + .build(); + var metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.COMPACTION_REMOVED_COUNTER, attributes, 1); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.COMPACTION_OPERATION_COUNTER, Attributes.builder() + .putAll(attributes) + .put(OpenTelemetryAttributes.PULSAR_COMPACTION_STATUS, "success") + .build(), + 1); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.COMPACTION_OPERATION_COUNTER, Attributes.builder() + .putAll(attributes) + .put(OpenTelemetryAttributes.PULSAR_COMPACTION_STATUS, "failure") + .build(), + 0); + assertMetricDoubleSumValue(metrics, OpenTelemetryTopicStats.COMPACTION_DURATION_SECONDS, attributes, + actual -> assertThat(actual).isPositive()); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.COMPACTION_BYTES_IN_COUNTER, attributes, + actual -> assertThat(actual).isPositive()); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.COMPACTION_BYTES_OUT_COUNTER, attributes, + actual -> assertThat(actual).isPositive()); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.COMPACTION_ENTRIES_COUNTER, attributes, 1); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.COMPACTION_BYTES_COUNTER, attributes, + actual -> assertThat(actual).isPositive()); + + producer.newMessage().key("K1").value(null).sendAsync(); + producer.flush(); + + admin.topics().triggerCompaction(topicName); + + Awaitility.await().untilAsserted(() -> { + Assert.assertEquals(admin.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS); + }); + + @Cleanup + Reader reader = pulsarClient.newReader(Schema.STRING) + .subscriptionName("reader-test") + .topic(topicName) + .readCompacted(true) + .startMessageId(MessageId.earliest) + .create(); + while (reader.hasMessageAvailable()) { + Message message = reader.readNext(3, TimeUnit.SECONDS); + Assert.assertNotNull(message); + } + // set retain null key back to avoid affecting other tests + pulsar.getConfig().setTopicCompactionRetainNullKey(oldRetainNullKey); + } + @Test public void testCompactAddCompact() throws Exception { String topic = "persistent://my-property/use/my-ns/my-topic1"; + @Cleanup Producer producer = pulsarClient.newProducer().topic(topic) - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.SinglePartition) - .create(); + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); Map expected = new HashMap<>(); @@ -210,10 +316,11 @@ public void testCompactAddCompact() throws Exception { public void testCompactedInOrder() throws Exception { String topic = "persistent://my-property/use/my-ns/my-topic1"; + @Cleanup Producer producer = pulsarClient.newProducer().topic(topic) - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.SinglePartition) - .create(); + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); producer.newMessage() .key("c") @@ -251,11 +358,56 @@ public void testCompactEmptyTopic() throws Exception { public void testPhaseOneLoopTimeConfiguration() { ServiceConfiguration configuration = new ServiceConfiguration(); configuration.setBrokerServiceCompactionPhaseOneLoopTimeInSeconds(60); - TwoPhaseCompactor compactor = new TwoPhaseCompactor(configuration, Mockito.mock(PulsarClientImpl.class), + PulsarClientImpl mockClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(mockClient.getCnxPool()).thenReturn(connectionPool); + PublishingOrderCompactor compactor = new PublishingOrderCompactor(configuration, mockClient, Mockito.mock(BookKeeper.class), compactionScheduler); Assert.assertEquals(compactor.getPhaseOneLoopReadTimeoutInSeconds(), 60); } + @Test + public void testCompactedWithConcurrentSend() throws Exception { + String topic = "persistent://my-property/use/my-ns/testCompactedWithConcurrentSend"; + + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topic) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + + var future = CompletableFuture.runAsync(() -> { + for (int i = 0; i < 100; i++) { + try { + producer.newMessage().key(String.valueOf(i)).value(String.valueOf(i).getBytes()).send(); + } catch (PulsarClientException e) { + throw new RuntimeException(e); + } + } + }); + + PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get(); + PulsarTopicCompactionService topicCompactionService = + (PulsarTopicCompactionService) persistentTopic.getTopicCompactionService(); + + Awaitility.await().untilAsserted(() -> { + long compactedLedgerId = compact(topic); + Thread.sleep(300); + Optional compactedTopicContext = topicCompactionService.getCompactedTopic() + .getCompactedTopicContext(); + Assert.assertTrue(compactedTopicContext.isPresent()); + Assert.assertEquals(compactedTopicContext.get().ledger.getId(), compactedLedgerId); + }); + + Position lastCompactedPosition = topicCompactionService.getLastCompactedPosition().get(); + Entry lastCompactedEntry = topicCompactionService.readLastCompactedEntry().get(); + + Assert.assertTrue(PositionFactory.create(lastCompactedPosition.getLedgerId(), lastCompactedPosition.getEntryId()) + .compareTo(lastCompactedEntry.getLedgerId(), lastCompactedEntry.getEntryId()) >= 0); + + future.join(); + } + public ByteBuf extractPayload(RawMessage m) throws Exception { ByteBuf payloadAndMetadata = m.getHeadersAndPayload(); Commands.skipChecksumIfPresent(payloadAndMetadata); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorToolTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorToolTest.java index b1f653fae012a..101d0a10b4fd1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorToolTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorToolTest.java @@ -22,7 +22,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.testng.Assert.assertTrue; -import com.beust.jcommander.Parameter; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.lang.reflect.Constructor; @@ -35,8 +34,9 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.common.util.CmdGenerateDocs; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.testng.annotations.Test; +import picocli.CommandLine.Option; /** * CompactorTool Tests. @@ -69,9 +69,9 @@ public void testGenerateDocs() throws Exception { Field[] fields = argumentsClass.getDeclaredFields(); for (Field field : fields) { - boolean fieldHasAnno = field.isAnnotationPresent(Parameter.class); + boolean fieldHasAnno = field.isAnnotationPresent(Option.class); if (fieldHasAnno) { - Parameter fieldAnno = field.getAnnotation(Parameter.class); + Option fieldAnno = field.getAnnotation(Option.class); String[] names = fieldAnno.names(); String nameStr = Arrays.asList(names).toString(); nameStr = nameStr.substring(1, nameStr.length() - 1); @@ -98,6 +98,7 @@ public void testUseTlsUrlWithPEM() throws PulsarClientException { verify(serviceConfiguration, times(1)).getBrokerClientKeyFilePath(); verify(serviceConfiguration, times(1)).getBrokerClientTrustCertsFilePath(); verify(serviceConfiguration, times(1)).getBrokerClientCertificateFilePath(); + serviceConfiguration.setBrokerClientTlsTrustStorePassword(MockedPulsarServiceBaseTest.BROKER_KEYSTORE_PW); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/EventTimeOrderCompactorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/EventTimeOrderCompactorTest.java new file mode 100644 index 0000000000000..8fba0983123ee --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/EventTimeOrderCompactorTest.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.compaction; + +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricDoubleSumValue; +import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongSumValue; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.opentelemetry.api.common.Attributes; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.stats.OpenTelemetryTopicStats; +import org.apache.pulsar.client.admin.LongRunningProcessStatus; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageRoutingMode; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.Reader; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.ConnectionPool; +import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; +import org.awaitility.Awaitility; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test(groups = "broker-compaction") +public class EventTimeOrderCompactorTest extends CompactorTest { + + private EventTimeOrderCompactor compactor; + + @BeforeMethod + @Override + public void setup() throws Exception { + super.setup(); + compactor = new EventTimeOrderCompactor(conf, pulsarClient, bk, compactionScheduler); + } + + @Override + protected long compact(String topic) throws ExecutionException, InterruptedException { + return compactor.compact(topic).get(); + } + + @Override + protected Compactor getCompactor() { + return compactor; + } + + @Test + public void testCompactedOutByEventTime() throws Exception { + String topicName = BrokerTestUtil.newUniqueName("persistent://my-property/use/my-ns/testCompactedOutByEventTime"); + this.restartBroker(); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(true).topic(topicName).batchingMaxMessages(3).create(); + + producer.newMessage().key("K1").value("V1").eventTime(1L).sendAsync(); + producer.newMessage().key("K2").value("V2").eventTime(1L).sendAsync(); + producer.newMessage().key("K2").value(null).eventTime(2L).sendAsync(); + producer.flush(); + + admin.topics().triggerCompaction(topicName); + + Awaitility.await().untilAsserted(() -> { + Assert.assertEquals(admin.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS); + }); + + var attributes = Attributes.builder() + .put(OpenTelemetryAttributes.PULSAR_DOMAIN, "persistent") + .put(OpenTelemetryAttributes.PULSAR_TENANT, "my-property") + .put(OpenTelemetryAttributes.PULSAR_NAMESPACE, "my-property/use/my-ns") + .put(OpenTelemetryAttributes.PULSAR_TOPIC, topicName) + .build(); + var metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.COMPACTION_REMOVED_COUNTER, attributes, 1); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.COMPACTION_OPERATION_COUNTER, Attributes.builder() + .putAll(attributes) + .put(OpenTelemetryAttributes.PULSAR_COMPACTION_STATUS, "success") + .build(), + 1); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.COMPACTION_OPERATION_COUNTER, Attributes.builder() + .putAll(attributes) + .put(OpenTelemetryAttributes.PULSAR_COMPACTION_STATUS, "failure") + .build(), + 0); + assertMetricDoubleSumValue(metrics, OpenTelemetryTopicStats.COMPACTION_DURATION_SECONDS, attributes, + actual -> assertThat(actual).isPositive()); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.COMPACTION_BYTES_IN_COUNTER, attributes, + actual -> assertThat(actual).isPositive()); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.COMPACTION_BYTES_OUT_COUNTER, attributes, + actual -> assertThat(actual).isPositive()); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.COMPACTION_ENTRIES_COUNTER, attributes, 1); + assertMetricLongSumValue(metrics, OpenTelemetryTopicStats.COMPACTION_BYTES_COUNTER, attributes, + actual -> assertThat(actual).isPositive()); + + producer.newMessage().key("K1").eventTime(2L).value("V1-2").sendAsync(); + producer.flush(); + + admin.topics().triggerCompaction(topicName); + + Awaitility.await().untilAsserted(() -> { + Assert.assertEquals(admin.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS); + }); + + @Cleanup + Reader reader = pulsarClient.newReader(Schema.STRING) + .subscriptionName("reader-test") + .topic(topicName) + .readCompacted(true) + .startMessageId(MessageId.earliest) + .create(); + while (reader.hasMessageAvailable()) { + Message message = reader.readNext(3, TimeUnit.SECONDS); + Assert.assertEquals(message.getEventTime(), 2L); + } + } + + @Test + public void testCompactWithEventTimeAddCompact() throws Exception { + String topic = "persistent://my-property/use/my-ns/my-topic1"; + + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topic) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + + Map expected = new HashMap<>(); + + producer.newMessage() + .key("a") + .eventTime(1L) + .value("A_1".getBytes()) + .send(); + producer.newMessage() + .key("b") + .eventTime(1L) + .value("B_1".getBytes()) + .send(); + producer.newMessage() + .key("a") + .eventTime(2L) + .value("A_2".getBytes()) + .send(); + expected.put("a", "A_2".getBytes()); + expected.put("b", "B_1".getBytes()); + + compactAndVerify(topic, new HashMap<>(expected), false); + + producer.newMessage() + .key("b") + .eventTime(2L) + .value("B_2".getBytes()) + .send(); + expected.put("b", "B_2".getBytes()); + + compactAndVerify(topic, expected, false); + } + + @Override + @Test + public void testPhaseOneLoopTimeConfiguration() { + ServiceConfiguration configuration = new ServiceConfiguration(); + configuration.setBrokerServiceCompactionPhaseOneLoopTimeInSeconds(60); + PulsarClientImpl mockClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(mockClient.getCnxPool()).thenReturn(connectionPool); + EventTimeOrderCompactor compactor = new EventTimeOrderCompactor(configuration, mockClient, + Mockito.mock(BookKeeper.class), compactionScheduler); + Assert.assertEquals(compactor.getPhaseOneLoopReadTimeoutInSeconds(), 60); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/GetLastMessageIdCompactedTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/GetLastMessageIdCompactedTest.java index 317b1a227e585..a28392fb99d2a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/GetLastMessageIdCompactedTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/GetLastMessageIdCompactedTest.java @@ -20,18 +20,20 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotEquals; import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; @@ -82,8 +84,8 @@ private void triggerCompactionAndWait(String topicName) throws Exception { (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).get().get(); persistentTopic.triggerCompaction(); Awaitility.await().untilAsserted(() -> { - PositionImpl lastConfirmPos = (PositionImpl) persistentTopic.getManagedLedger().getLastConfirmedEntry(); - PositionImpl markDeletePos = (PositionImpl) persistentTopic + Position lastConfirmPos = persistentTopic.getManagedLedger().getLastConfirmedEntry(); + Position markDeletePos = persistentTopic .getSubscription(Compactor.COMPACTION_SUBSCRIPTION).getCursor().getMarkDeletedPosition(); assertEquals(markDeletePos.getLedgerId(), lastConfirmPos.getLedgerId()); assertEquals(markDeletePos.getEntryId(), lastConfirmPos.getEntryId()); @@ -415,4 +417,28 @@ public void testGetLastMessageIdAfterCompactionAllNullMsg(boolean enabledBatch) producer.close(); admin.topics().delete(topicName, false); } + + @Test(dataProvider = "enabledBatch") + public void testReaderStuckWithCompaction(boolean enabledBatch) throws Exception { + String topicName = "persistent://public/default/" + BrokerTestUtil.newUniqueName("tp"); + String subName = "sub"; + Producer producer = createProducer(enabledBatch, topicName); + producer.newMessage().key("k0").value("v0").sendAsync(); + producer.newMessage().key("k0").value("v1").sendAsync(); + producer.flush(); + + triggerCompactionAndWait(topicName); + triggerLedgerSwitch(topicName); + clearAllTheLedgersOutdated(topicName); + + var reader = pulsarClient.newReader(Schema.STRING) + .topic(topicName) + .subscriptionName(subName) + .startMessageId(MessageId.earliest) + .create(); + while (reader.hasMessageAvailable()) { + Message message = reader.readNext(5, TimeUnit.SECONDS); + assertNotEquals(message, null); + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java index e4f0750a981c9..a834fa1fde1e3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java @@ -24,9 +24,10 @@ import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigning; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Releasing; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.StorageType.SystemTopic; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.isValidTransition; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MSG_COMPRESSION_TYPE; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateTableViewImpl.MSG_COMPRESSION_TYPE; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; @@ -54,13 +55,14 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import lombok.Cleanup; import org.apache.bookkeeper.client.BookKeeper; import org.apache.commons.lang.reflect.FieldUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; -import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateCompactionStrategy; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateDataConflictResolver; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; @@ -91,7 +93,7 @@ public class ServiceUnitStateCompactionTest extends MockedPulsarServiceBaseTest private ScheduledExecutorService compactionScheduler; private BookKeeper bk; private Schema schema; - private ServiceUnitStateCompactionStrategy strategy; + private ServiceUnitStateDataConflictResolver strategy; private ServiceUnitState testState = Init; @@ -117,7 +119,7 @@ private ServiceUnitStateData testValue(String broker) { private ServiceUnitState nextValidState(ServiceUnitState from) { List candidates = Arrays.stream(ServiceUnitState.values()) - .filter(to -> isValidTransition(from, to)) + .filter(to -> isValidTransition(from, to, SystemTopic)) .collect(Collectors.toList()); var state= candidates.get(RANDOM.nextInt(candidates.size())); return state; @@ -126,7 +128,7 @@ private ServiceUnitState nextValidState(ServiceUnitState from) { private ServiceUnitState nextValidStateNonSplit(ServiceUnitState from) { List candidates = Arrays.stream(ServiceUnitState.values()) .filter(to -> to != Init && to != Splitting && to != Deleted - && isValidTransition(from, to)) + && isValidTransition(from, to, SystemTopic)) .collect(Collectors.toList()); var state= candidates.get(RANDOM.nextInt(candidates.size())); return state; @@ -134,7 +136,7 @@ && isValidTransition(from, to)) private ServiceUnitState nextInvalidState(ServiceUnitState from) { List candidates = Arrays.stream(ServiceUnitState.values()) - .filter(to -> !isValidTransition(from, to)) + .filter(to -> !isValidTransition(from, to, SystemTopic)) .collect(Collectors.toList()); if (candidates.size() == 0) { return Init; @@ -154,9 +156,9 @@ public void setup() throws Exception { compactionScheduler = Executors.newSingleThreadScheduledExecutor( new ThreadFactoryBuilder().setNameFormat("compaction-%d").setDaemon(true).build()); - bk = pulsar.getBookKeeperClientFactory().create(this.conf, null, null, Optional.empty(), null); + bk = pulsar.getBookKeeperClientFactory().create(this.conf, null, null, Optional.empty(), null).get(); schema = Schema.JSON(ServiceUnitStateData.class); - strategy = new ServiceUnitStateCompactionStrategy(); + strategy = new ServiceUnitStateDataConflictResolver(); strategy.checkBrokers(false); testState = Init; @@ -168,7 +170,7 @@ public void setup() throws Exception { @Override public void cleanup() throws Exception { super.internalCleanup(); - + bk.close(); if (compactionScheduler != null) { compactionScheduler.shutdownNow(); } @@ -328,10 +330,10 @@ public void testCompactionWithTableview() throws Exception { .topic("persistent://my-property/use/my-ns/my-topic1") .loadConf(Map.of( "topicCompactionStrategyClassName", - ServiceUnitStateCompactionStrategy.class.getName())) + ServiceUnitStateDataConflictResolver.class.getName())) .create(); - ((ServiceUnitStateCompactionStrategy) + ((ServiceUnitStateDataConflictResolver) FieldUtils.readDeclaredField(tv, "compactionStrategy", true)) .checkBrokers(false); TestData testData = generateTestData(); @@ -363,7 +365,7 @@ public void testCompactionWithTableview() throws Exception { .topic(topic) .loadConf(Map.of( "topicCompactionStrategyClassName", - ServiceUnitStateCompactionStrategy.class.getName())) + ServiceUnitStateDataConflictResolver.class.getName())) .create(); for(var etr : tableview.entrySet()){ @@ -530,10 +532,11 @@ public void testSlowTableviewAfterCompaction() throws Exception { .subscriptionName("fastTV") .loadConf(Map.of( strategyClassName, - ServiceUnitStateCompactionStrategy.class.getName())) + ServiceUnitStateDataConflictResolver.class.getName())) .create(); var defaultConf = getDefaultConf(); + @Cleanup var additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf); var pulsar2 = additionalPulsarTestContext.getPulsarService(); @@ -542,7 +545,7 @@ public void testSlowTableviewAfterCompaction() throws Exception { .subscriptionName("slowTV") .loadConf(Map.of( strategyClassName, - ServiceUnitStateCompactionStrategy.class.getName())) + ServiceUnitStateDataConflictResolver.class.getName())) .create(); var semaphore = new Semaphore(0); @@ -614,7 +617,7 @@ public void testSlowTableviewAfterCompaction() throws Exception { .topic(topic) .loadConf(Map.of( strategyClassName, - ServiceUnitStateCompactionStrategy.class.getName())) + ServiceUnitStateDataConflictResolver.class.getName())) .create(); Awaitility.await() .pollInterval(200, TimeUnit.MILLISECONDS) @@ -649,7 +652,7 @@ public void testSlowReceiveTableviewAfterCompaction() throws Exception { .subscriptionName("slowTV") .loadConf(Map.of( strategyClassName, - ServiceUnitStateCompactionStrategy.class.getName())) + ServiceUnitStateDataConflictResolver.class.getName())) .create(); // Configure retention to ensue data is retained for reader diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactionRetentionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactionRetentionTest.java index 1cac04c2fa956..e556ec8e0b200 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactionRetentionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactionRetentionTest.java @@ -34,7 +34,7 @@ public class StrategicCompactionRetentionTest extends CompactionRetentionTest { @Override public void setup() throws Exception { super.setup(); - compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler, 1); + compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); strategy = new TopicCompactionStrategyTest.DummyTopicCompactionStrategy(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactionTest.java index 135a839bd54a8..8f67e412267a0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactionTest.java @@ -18,22 +18,33 @@ */ package org.apache.pulsar.compaction; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateTableViewImpl.MSG_COMPRESSION_TYPE; +import static org.testng.Assert.assertEquals; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.CryptoKeyReader; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerBuilder; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.TableView; import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; import org.apache.pulsar.common.topics.TopicCompactionStrategy; +import org.apache.pulsar.common.util.FutureUtil; import org.testng.Assert; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -47,7 +58,7 @@ public class StrategicCompactionTest extends CompactionTest { @Override public void setup() throws Exception { super.setup(); - compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler, 1); + compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); strategy = new TopicCompactionStrategyTest.DummyTopicCompactionStrategy(); } @@ -63,7 +74,7 @@ protected long compact(String topic, CryptoKeyReader cryptoKeyReader) } @Override - protected TwoPhaseCompactor getCompactor() { + protected PublishingOrderCompactor getCompactor() { return compactor; } @@ -148,5 +159,58 @@ public void testNumericOrderCompaction() throws Exception { Assert.assertEquals(tableView.entrySet(), expectedCopy.entrySet()); } + @Test(timeOut = 20000) + public void testSameBatchCompactToSameBatch() throws Exception { + final String topic = + "persistent://my-property/use/my-ns/testSameBatchCompactToSameBatch" + UUID.randomUUID(); + + // Use odd number to make sure the last message is flush by `reader.hasNext() == false`. + final int messages = 11; + + // 1.create producer and publish message to the topic. + ProducerBuilder builder = pulsarClient.newProducer(Schema.INT32) + .compressionType(MSG_COMPRESSION_TYPE).topic(topic); + builder.batchingMaxMessages(2) + .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS); + + Producer producer = builder.create(); + + List> futures = new ArrayList<>(messages); + for (int i = 0; i < messages; i++) { + futures.add(producer.newMessage().key(String.valueOf(i)) + .value(i) + .sendAsync()); + } + FutureUtil.waitForAll(futures).get(); + + // 2.compact the topic. + StrategicTwoPhaseCompactor compactor + = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); + compactor.compact(topic, strategy).get(); + // consumer with readCompacted enabled only get compacted entries + try (Consumer consumer = pulsarClient + .newConsumer(Schema.INT32) + .topic(topic) + .subscriptionName("sub1") + .readCompacted(true) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) { + int received = 0; + while (true) { + Message m = consumer.receive(2, TimeUnit.SECONDS); + if (m == null) { + break; + } + MessageIdAdv messageId = (MessageIdAdv) m.getMessageId(); + if (received < messages - 1) { + assertEquals(messageId.getBatchSize(), 2); + } else { + assertEquals(messageId.getBatchSize(), 0); + } + received++; + } + assertEquals(received, messages); + } + + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactorTest.java index 91dd8a2bd358b..bc65791b323cd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactorTest.java @@ -33,7 +33,7 @@ public class StrategicCompactorTest extends CompactorTest { @Override public void setup() throws Exception { super.setup(); - compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler, 1); + compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); strategy = new TopicCompactionStrategyTest.DummyTopicCompactionStrategy(); } @@ -46,4 +46,4 @@ protected long compact(String topic) throws ExecutionException, InterruptedExcep protected Compactor getCompactor() { return compactor; } -} \ No newline at end of file +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/TopicCompactionServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/TopicCompactionServiceTest.java new file mode 100644 index 0000000000000..9f33479ce4cab --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/TopicCompactionServiceTest.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.compaction; + +import static org.apache.pulsar.compaction.Compactor.COMPACTED_TOPIC_LEDGER_PROPERTY; +import static org.apache.pulsar.compaction.Compactor.COMPACTION_SUBSCRIPTION; +import static org.testng.Assert.assertEquals; +import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.AssertJUnit.fail; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import lombok.Cleanup; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.client.api.MessageRoutingMode; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.impl.MessageImpl; +import org.apache.pulsar.common.api.proto.BrokerEntryMetadata; +import org.apache.pulsar.common.api.proto.MessageMetadata; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.protocol.Commands; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class TopicCompactionServiceTest extends MockedPulsarServiceBaseTest { + + protected ScheduledExecutorService compactionScheduler; + protected BookKeeper bk; + private PublishingOrderCompactor compactor; + + @BeforeMethod + @Override + public void setup() throws Exception { + conf.setExposingBrokerEntryMetadataToClientEnabled(true); + + super.internalSetup(); + + admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); + TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test")); + String defaultTenant = "prop-xyz"; + admin.tenants().createTenant(defaultTenant, tenantInfo); + String defaultNamespace = defaultTenant + "/ns1"; + admin.namespaces().createNamespace(defaultNamespace, Set.of("test")); + + compactionScheduler = Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat("compactor").setDaemon(true).build()); + bk = pulsar.getBookKeeperClientFactory().create( + this.conf, null, null, Optional.empty(), null).get(); + compactor = new PublishingOrderCompactor(conf, pulsarClient, bk, compactionScheduler); + } + + @AfterMethod(alwaysRun = true) + @Override + public void cleanup() throws Exception { + super.internalCleanup(); + bk.close(); + if (compactionScheduler != null) { + compactionScheduler.shutdownNow(); + } + } + + @Test + public void test() throws Exception { + String topic = "persistent://prop-xyz/ns1/my-topic"; + + PulsarTopicCompactionService service = new PulsarTopicCompactionService(topic, bk, () -> compactor); + + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topic) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + + producer.newMessage() + .key("c") + .value("C_0".getBytes()) + .send(); + + conf.setBrokerEntryMetadataInterceptors(org.assertj.core.util.Sets.newTreeSet( + "org.apache.pulsar.common.intercept.AppendIndexMetadataInterceptor" + )); + restartBroker(); + + long startTime = System.currentTimeMillis(); + + producer.newMessage() + .key("a") + .value("A_1".getBytes()) + .send(); + producer.newMessage() + .key("b") + .value("B_1".getBytes()) + .send(); + producer.newMessage() + .key("a") + .value("A_2".getBytes()) + .send(); + producer.newMessage() + .key("b") + .value("B_2".getBytes()) + .send(); + producer.newMessage() + .key("b") + .value("B_3".getBytes()) + .send(); + + producer.flush(); + + service.compact().join(); + + + CompactedTopicImpl compactedTopic = service.getCompactedTopic(); + + Long compactedLedger = admin.topics().getInternalStats(topic).cursors.get(COMPACTION_SUBSCRIPTION).properties.get( + COMPACTED_TOPIC_LEDGER_PROPERTY); + String markDeletePosition = + admin.topics().getInternalStats(topic).cursors.get(COMPACTION_SUBSCRIPTION).markDeletePosition; + String[] split = markDeletePosition.split(":"); + compactedTopic.newCompactedLedger(PositionFactory.create(Long.valueOf(split[0]), Long.valueOf(split[1])), + compactedLedger).join(); + + Position lastCompactedPosition = service.getLastCompactedPosition().join(); + assertEquals(admin.topics().getInternalStats(topic).lastConfirmedEntry, lastCompactedPosition.toString()); + + List entries = service.readCompactedEntries(PositionFactory.EARLIEST, 4).join(); + assertEquals(entries.size(), 3); + entries.stream().map(e -> { + try { + return MessageImpl.deserialize(e.getDataBuffer()); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + }).forEach(message -> { + String data = new String(message.getData()); + if (Objects.equals(message.getKey(), "a")) { + assertEquals(data, "A_2"); + } else if (Objects.equals(message.getKey(), "b")) { + assertEquals(data, "B_3"); + } else if (Objects.equals(message.getKey(), "c")) { + assertEquals(data, "C_0"); + } else { + fail(); + } + }); + + List entries2 = service.readCompactedEntries(PositionFactory.EARLIEST, 1).join(); + assertEquals(entries2.size(), 1); + + Entry entry = service.findEntryByEntryIndex(0).join(); + BrokerEntryMetadata brokerEntryMetadata = Commands.peekBrokerEntryMetadataIfExist(entry.getDataBuffer()); + assertNotNull(brokerEntryMetadata); + assertEquals(brokerEntryMetadata.getIndex(), 2); + MessageMetadata metadata = Commands.parseMessageMetadata(entry.getDataBuffer()); + assertEquals(metadata.getPartitionKey(), "a"); + entry.release(); + + entry = service.findEntryByEntryIndex(3).join(); + brokerEntryMetadata = Commands.peekBrokerEntryMetadataIfExist(entry.getDataBuffer()); + assertNotNull(brokerEntryMetadata); + assertEquals(brokerEntryMetadata.getIndex(), 4); + metadata = Commands.parseMessageMetadata(entry.getDataBuffer()); + assertEquals(metadata.getPartitionKey(), "b"); + entry.release(); + + entry = service.findEntryByPublishTime(startTime).join(); + brokerEntryMetadata = Commands.peekBrokerEntryMetadataIfExist(entry.getDataBuffer()); + assertNotNull(brokerEntryMetadata); + assertEquals(brokerEntryMetadata.getIndex(), 2); + metadata = Commands.parseMessageMetadata(entry.getDataBuffer()); + assertEquals(metadata.getPartitionKey(), "a"); + entry.release(); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/TopicCompactionStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/TopicCompactionStrategyTest.java index 0ecd09606cea7..50d4cdc7b0a71 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/TopicCompactionStrategyTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/TopicCompactionStrategyTest.java @@ -41,13 +41,13 @@ public boolean shouldKeepLeft(byte[] prev, byte[] cur) { @Test(expectedExceptions = IllegalArgumentException.class) public void testLoadInvalidTopicCompactionStrategy() { - TopicCompactionStrategy.load("uknown"); + TopicCompactionStrategy.load("uknown", "uknown"); } @Test public void testNumericOrderCompactionStrategy() { TopicCompactionStrategy strategy = - TopicCompactionStrategy.load(NumericOrderCompactionStrategy.class.getCanonicalName()); + TopicCompactionStrategy.load("numeric", NumericOrderCompactionStrategy.class.getCanonicalName()); Assert.assertFalse(strategy.shouldKeepLeft(1, 2)); Assert.assertTrue(strategy.shouldKeepLeft(2, 1)); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionE2ESecurityTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionE2ESecurityTest.java index 714c9d7269970..e9b3531c7c2e2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionE2ESecurityTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionE2ESecurityTest.java @@ -34,6 +34,7 @@ import java.net.URL; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Properties; @@ -215,11 +216,26 @@ && isNotBlank(workerConfig.getBrokerClientAuthenticationParameters())) { void shutdown() throws Exception { try { log.info("--- Shutting down ---"); - pulsarClient.close(); - superUserAdmin.close(); - functionsWorkerService.stop(); - pulsar.close(); - bkEnsemble.stop(); + if (pulsarClient != null) { + pulsarClient.close(); + pulsarClient = null; + } + if (superUserAdmin != null) { + superUserAdmin.close(); + superUserAdmin = null; + } + if (functionsWorkerService != null) { + functionsWorkerService.stop(); + functionsWorkerService = null; + } + if (pulsar != null) { + pulsar.close(); + pulsar = null; + } + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; + } } finally { if (tempDirectory != null) { tempDirectory.delete(); @@ -267,8 +283,12 @@ private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration conf workerConfig.setAuthorizationEnabled(config.isAuthorizationEnabled()); workerConfig.setAuthorizationProvider(config.getAuthorizationProvider()); + List urlPatterns = + List.of(getPulsarApiExamplesJar().getParentFile().toURI() + ".*", "http://127\\.0\\.0\\.1:.*"); + workerConfig.setAdditionalEnabledConnectorUrlPatterns(urlPatterns); + workerConfig.setAdditionalEnabledFunctionsUrlPatterns(urlPatterns); + PulsarWorkerService workerService = new PulsarWorkerService(); - workerService.init(workerConfig, null, false); return workerService; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java index aa190cd2e0a73..aff13f1a1ca21 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java @@ -538,15 +538,15 @@ private void testE2EPulsarFunctionLocalRun(String jarFilePathUrl, int parallelis totalMsgs); // validate prometheus metrics - String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(metricsPort); + String prometheusMetrics = TestPulsarFunctionUtils.getPrometheusMetrics(metricsPort); log.info("prometheus metrics: {}", prometheusMetrics); - Map metricsMap = new HashMap<>(); + Map metricsMap = new HashMap<>(); Arrays.asList(prometheusMetrics.split("\n")).forEach(line -> { if (line.startsWith("pulsar_function_processed_successfully_total")) { - Map metrics = PulsarFunctionTestUtils.parseMetrics(line); + Map metrics = TestPulsarFunctionUtils.parseMetrics(line); assertFalse(metrics.isEmpty()); - PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_function_processed_successfully_total"); + TestPulsarFunctionUtils.Metric m = metrics.get("pulsar_function_processed_successfully_total"); if (m != null) { metricsMap.put(m.tags.get("instance_id"), m); } @@ -556,7 +556,7 @@ private void testE2EPulsarFunctionLocalRun(String jarFilePathUrl, int parallelis double totalMsgRecv = 0.0; for (int i = 0; i < parallelism; i++) { - PulsarFunctionTestUtils.Metric m = metricsMap.get(String.valueOf(i)); + TestPulsarFunctionUtils.Metric m = metricsMap.get(String.valueOf(i)); Assert.assertNotNull(m); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), String.valueOf(i)); @@ -843,15 +843,15 @@ private void testPulsarSourceLocalRun(String jarFilePathUrl, int parallelism) th assertEquals(admin.topics().getStats(sinkTopic).getPublishers().size(), parallelism); // validate prometheus metrics - String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(metricsPort); + String prometheusMetrics = TestPulsarFunctionUtils.getPrometheusMetrics(metricsPort); log.info("prometheus metrics: {}", prometheusMetrics); - Map metricsMap = new HashMap<>(); + Map metricsMap = new HashMap<>(); Arrays.asList(prometheusMetrics.split("\n")).forEach(line -> { if (line.startsWith("pulsar_source_written_total")) { - Map metrics = PulsarFunctionTestUtils.parseMetrics(line); + Map metrics = TestPulsarFunctionUtils.parseMetrics(line); assertFalse(metrics.isEmpty()); - PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_source_written_total"); + TestPulsarFunctionUtils.Metric m = metrics.get("pulsar_source_written_total"); if (m != null) { metricsMap.put(m.tags.get("instance_id"), m); } @@ -860,7 +860,7 @@ private void testPulsarSourceLocalRun(String jarFilePathUrl, int parallelism) th Assert.assertEquals(metricsMap.size(), parallelism); for (int i = 0; i < parallelism; i++) { - PulsarFunctionTestUtils.Metric m = metricsMap.get(String.valueOf(i)); + TestPulsarFunctionUtils.Metric m = metricsMap.get(String.valueOf(i)); Assert.assertNotNull(m); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), String.valueOf(i)); @@ -1002,22 +1002,22 @@ private void testPulsarSinkLocalRun(String jarFilePathUrl, int parallelism, Stri }, 5, 200)); // validate prometheus metrics - String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(metricsPort); + String prometheusMetrics = TestPulsarFunctionUtils.getPrometheusMetrics(metricsPort); log.info("prometheus metrics: {}", prometheusMetrics); - Map metricsMap = new HashMap<>(); + Map metricsMap = new HashMap<>(); Arrays.asList(prometheusMetrics.split("\n")).forEach(line -> { if (line.startsWith("pulsar_sink_written_total")) { - Map metrics = PulsarFunctionTestUtils.parseMetrics(line); + Map metrics = TestPulsarFunctionUtils.parseMetrics(line); assertFalse(metrics.isEmpty()); - PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_sink_written_total"); + TestPulsarFunctionUtils.Metric m = metrics.get("pulsar_sink_written_total"); if (m != null) { metricsMap.put(m.tags.get("instance_id"), m); } } else if (line.startsWith("pulsar_sink_sink_exceptions_total")) { - Map metrics = PulsarFunctionTestUtils.parseMetrics(line); + Map metrics = TestPulsarFunctionUtils.parseMetrics(line); assertFalse(metrics.isEmpty()); - PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_sink_sink_exceptions_total"); + TestPulsarFunctionUtils.Metric m = metrics.get("pulsar_sink_sink_exceptions_total"); if (m == null) { m = metrics.get("pulsar_sink_sink_exceptions_1min_total"); } @@ -1028,7 +1028,7 @@ private void testPulsarSinkLocalRun(String jarFilePathUrl, int parallelism, Stri double totalNumRecvMsg = 0; for (int i = 0; i < parallelism; i++) { - PulsarFunctionTestUtils.Metric m = metricsMap.get(String.valueOf(i)); + TestPulsarFunctionUtils.Metric m = metricsMap.get(String.valueOf(i)); Assert.assertNotNull(m); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), String.valueOf(i)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java index c820f512a68de..50dc39a3a79d2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java @@ -40,6 +40,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.distributedlog.DistributedLogConfiguration; import org.apache.distributedlog.api.namespace.Namespace; @@ -218,11 +219,26 @@ && isNotBlank(workerConfig.getBrokerClientAuthenticationParameters())) { void shutdown() throws Exception { try { log.info("--- Shutting down ---"); - pulsarClient.close(); - admin.close(); - functionsWorkerService.stop(); - pulsar.close(); - bkEnsemble.stop(); + if (pulsarClient != null) { + pulsarClient.close(); + pulsarClient = null; + } + if (admin != null) { + admin.close(); + admin = null; + } + if (functionsWorkerService != null) { + functionsWorkerService.stop(); + functionsWorkerService = null; + } + if (pulsar != null) { + pulsar.close(); + pulsar = null; + } + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; + } } finally { if (tempDirectory != null) { tempDirectory.delete(); @@ -268,8 +284,11 @@ private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration conf workerConfig.setAuthenticationEnabled(true); workerConfig.setAuthorizationEnabled(true); + List urlPatterns = List.of(getPulsarApiExamplesJar().getParentFile().toURI() + ".*"); + workerConfig.setAdditionalEnabledConnectorUrlPatterns(urlPatterns); + workerConfig.setAdditionalEnabledFunctionsUrlPatterns(urlPatterns); + PulsarWorkerService workerService = new PulsarWorkerService(); - workerService.init(workerConfig, null, false); return workerService; } @@ -405,6 +424,7 @@ public void testMultipleAddress() throws Exception { String secondAddress = pulsar.getWebServiceAddressTls().replace("https://", ""); //set multi webService url + @Cleanup PulsarAdmin pulsarAdmin = PulsarAdmin.builder().serviceHttpUrl(pulsar.getWebServiceAddressTls() + "," + secondAddress) .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) .allowTlsInsecureConnection(true).authentication(authTls) @@ -526,6 +546,7 @@ public void testPulsarFunctionBKCleanup() throws Exception { log.info("dlog url: {}", url); URI dlogUri = URI.create(url); + @Cleanup Namespace dlogNamespace = NamespaceBuilder.newBuilder() .conf(dlogConf) .clientId("function-worker-" + workerConfig.getWorkerId()) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java index 1e8b26beee38a..3be16357d332b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java @@ -20,6 +20,8 @@ import static org.apache.pulsar.common.util.PortManager.nextLockedFreePort; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; @@ -30,9 +32,11 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; @@ -41,6 +45,7 @@ import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.impl.auth.AuthenticationTls; import org.apache.pulsar.common.functions.FunctionConfig; +import org.apache.pulsar.common.functions.WorkerInfo; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.util.ClassLoaderUtils; import org.apache.pulsar.common.util.ObjectMapperFactory; @@ -149,6 +154,12 @@ void setup() throws Exception { workerConfig.setUseTls(true); workerConfig.setTlsEnableHostnameVerification(true); workerConfig.setTlsAllowInsecureConnection(false); + File packagePath = new File( + PulsarSink.class.getProtectionDomain().getCodeSource().getLocation().getPath()).getParentFile(); + List urlPatterns = + List.of(packagePath.toURI() + ".*"); + workerConfig.setAdditionalEnabledConnectorUrlPatterns(urlPatterns); + workerConfig.setAdditionalEnabledFunctionsUrlPatterns(urlPatterns); fnWorkerServices[i] = WorkerServiceLoader.load(workerConfig); configurations[i] = config; @@ -191,11 +202,13 @@ void tearDown() throws Exception { for (int i = 0; i < BROKER_COUNT; i++) { if (pulsarAdmins[i] != null) { pulsarAdmins[i].close(); + pulsarAdmins[i] = null; } } for (int i = 0; i < BROKER_COUNT; i++) { if (fnWorkerServices[i] != null) { fnWorkerServices[i].stop(); + fnWorkerServices[i] = null; } } for (int i = 0; i < BROKER_COUNT; i++) { @@ -210,9 +223,13 @@ void tearDown() throws Exception { getBrokerServicePort().ifPresent(PortManager::releaseLockedPort); pulsarServices[i].getConfiguration() .getWebServicePort().ifPresent(PortManager::releaseLockedPort); + pulsarServices[i] = null; } } - bkEnsemble.stop(); + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; + } } finally { for (int i = 0; i < BROKER_COUNT; i++) { if (tempDirectories[i] != null) { @@ -242,6 +259,18 @@ public void testFunctionsCreation() throws Exception { log.info(" -------- Start test function : {}", functionName); + int finalI = i; + Awaitility.await().atMost(1, TimeUnit.MINUTES).pollInterval(1, TimeUnit.SECONDS).untilAsserted(() -> { + final PulsarWorkerService workerService = ((PulsarWorkerService) fnWorkerServices[finalI]); + final LeaderService leaderService = workerService.getLeaderService(); + assertNotNull(leaderService); + if (leaderService.isLeader()) { + assertTrue(true); + } else { + final WorkerInfo workerInfo = workerService.getMembershipManager().getLeader(); + assertTrue(workerInfo != null && !workerInfo.getWorkerId().equals(workerService.getWorkerConfig().getWorkerId())); + } + }); pulsarAdmins[i].functions().createFunctionWithUrl( functionConfig, jarFilePathUrl ); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarWorkerAssignmentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarWorkerAssignmentTest.java index 0821974bea506..9c137e37095ed 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarWorkerAssignmentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarWorkerAssignmentTest.java @@ -131,11 +131,26 @@ void setup(Method method) throws Exception { void shutdown() { log.info("--- Shutting down ---"); try { - pulsarClient.close(); - admin.close(); - functionsWorkerService.stop(); - pulsar.close(); - bkEnsemble.stop(); + if (pulsarClient != null) { + pulsarClient.close(); + pulsarClient = null; + } + if (admin != null) { + admin.close(); + admin = null; + } + if (functionsWorkerService != null) { + functionsWorkerService.stop(); + functionsWorkerService = null; + } + if (pulsar != null) { + pulsar.close(); + pulsar = null; + } + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; + } } catch (Exception e) { log.warn("Encountered errors at shutting down PulsarWorkerAssignmentTest", e); } finally { @@ -174,7 +189,6 @@ private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration conf workerConfig.setTopicCompactionFrequencySec(1); PulsarWorkerService workerService = new PulsarWorkerService(); - workerService.init(workerConfig, null, false); return workerService; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTestUtils.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/TestPulsarFunctionUtils.java similarity index 98% rename from pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTestUtils.java rename to pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/TestPulsarFunctionUtils.java index 292c571c3bc65..71f10cc2cbc39 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTestUtils.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/TestPulsarFunctionUtils.java @@ -39,7 +39,8 @@ import static com.google.common.base.Preconditions.checkArgument; @Slf4j -public class PulsarFunctionTestUtils { +@Test(groups = "functions-worker") +public class TestPulsarFunctionUtils { public static String getPrometheusMetrics(int metricsPort) throws IOException { StringBuilder result = new StringBuilder(); URL url = new URL(String.format("http://%s:%s/metrics", "localhost", metricsPort)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java index 3a99cc647ed5c..d27e27639048e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java @@ -35,6 +35,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -238,29 +239,35 @@ && isNotBlank(workerConfig.getBrokerClientAuthenticationParameters())) { void shutdown() throws Exception { log.info("--- Shutting down ---"); try { - if (fileServer != null) { - fileServer.stop(); - } + if (fileServer != null) { + fileServer.stop(); + fileServer = null; + } - if (pulsarClient != null) { - pulsarClient.close(); - } + if (pulsarClient != null) { + pulsarClient.close(); + pulsarClient = null; + } - if (admin != null) { - admin.close(); - } + if (admin != null) { + admin.close(); + admin = null; + } - if (functionsWorkerService != null) { - functionsWorkerService.stop(); - } + if (functionsWorkerService != null) { + functionsWorkerService.stop(); + functionsWorkerService = null; + } - if (pulsar != null) { - pulsar.close(); - } + if (pulsar != null) { + pulsar.close(); + pulsar = null; + } - if (bkEnsemble != null) { - bkEnsemble.stop(); - } + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; + } } finally { if (tempDirectory != null) { tempDirectory.delete(); @@ -306,8 +313,12 @@ private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration conf workerConfig.setAuthenticationEnabled(true); workerConfig.setAuthorizationEnabled(true); + List urlPatterns = + List.of(getPulsarApiExamplesJar().getParentFile().toURI() + ".*", "http://127\\.0\\.0\\.1:.*"); + workerConfig.setAdditionalEnabledConnectorUrlPatterns(urlPatterns); + workerConfig.setAdditionalEnabledFunctionsUrlPatterns(urlPatterns); + PulsarWorkerService workerService = new PulsarWorkerService(); - workerService.init(workerConfig, null, false); return workerService; } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarBatchSourceE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarBatchSourceE2ETest.java index 90a1a53750261..3fa4aeb550c92 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarBatchSourceE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarBatchSourceE2ETest.java @@ -35,7 +35,7 @@ import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.functions.utils.FunctionCommon; -import org.apache.pulsar.functions.worker.PulsarFunctionTestUtils; +import org.apache.pulsar.functions.worker.TestPulsarFunctionUtils; import org.apache.pulsar.io.batchdiscovery.ImmediateTriggerer; import org.testng.annotations.Test; @@ -102,11 +102,11 @@ private void testPulsarBatchSourceStats(String jarFilePathUrl) throws Exception }, 50, 150); assertEquals(admin.topics().getStats(sinkTopic2).getPublishers().size(), 1); - String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(pulsar.getListenPortHTTP().get()); + String prometheusMetrics = TestPulsarFunctionUtils.getPrometheusMetrics(pulsar.getListenPortHTTP().get()); log.info("prometheusMetrics: {}", prometheusMetrics); - Map metrics = PulsarFunctionTestUtils.parseMetrics(prometheusMetrics); - PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_source_received_total"); + Map metrics = TestPulsarFunctionUtils.parseMetrics(prometheusMetrics); + TestPulsarFunctionUtils.Metric m = metrics.get("pulsar_source_received_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), sourceName); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionAdminTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionAdminTest.java index d31d0c66bdf93..aafd82d339a1d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionAdminTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionAdminTest.java @@ -172,11 +172,26 @@ && isNotBlank(workerConfig.getBrokerClientAuthenticationParameters())) { @AfterMethod(alwaysRun = true) void shutdown() throws Exception { log.info("--- Shutting down ---"); - pulsarClient.close(); - admin.close(); - functionsWorkerService.stop(); - pulsar.close(); - bkEnsemble.stop(); + if (pulsarClient != null) { + pulsarClient.close(); + pulsarClient = null; + } + if (admin != null) { + admin.close(); + admin = null; + } + if (functionsWorkerService != null) { + functionsWorkerService.stop(); + functionsWorkerService = null; + } + if (pulsar != null) { + pulsar.close(); + pulsar = null; + } + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; + } } private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration config) { @@ -211,7 +226,6 @@ private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration conf workerConfig.setTlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH); PulsarWorkerService workerService = new PulsarWorkerService(); - workerService.init(workerConfig, null, false); return workerService; } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionE2ETest.java index 33ed806350b47..74c2a93b84e9f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionE2ETest.java @@ -62,12 +62,12 @@ import org.apache.pulsar.common.policies.data.SubscriptionStats; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.policies.data.TopicStats; -import org.apache.pulsar.compaction.TwoPhaseCompactor; +import org.apache.pulsar.compaction.PublishingOrderCompactor; import org.apache.pulsar.functions.api.Context; import org.apache.pulsar.functions.instance.InstanceUtils; import org.apache.pulsar.functions.utils.FunctionCommon; import org.apache.pulsar.functions.worker.FunctionRuntimeManager; -import org.apache.pulsar.functions.worker.PulsarFunctionTestUtils; +import org.apache.pulsar.functions.worker.TestPulsarFunctionUtils; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.Test; @@ -259,7 +259,7 @@ public void testReadCompactedFunction() throws Exception { @Cleanup("shutdownNow") ScheduledExecutorService compactionScheduler = Executors.newSingleThreadScheduledExecutor( new ThreadFactoryBuilder().setNameFormat("compactor").setDaemon(true).build()); - TwoPhaseCompactor twoPhaseCompactor = new TwoPhaseCompactor(config, + PublishingOrderCompactor twoPhaseCompactor = new PublishingOrderCompactor(config, pulsarClient, pulsar.getBookKeeperClient(), compactionScheduler); twoPhaseCompactor.compact(sourceTopic).get(); @@ -373,11 +373,11 @@ public void testPulsarFunctionStats() throws Exception { functionStats.getAvgProcessLatency()); // validate prometheus metrics empty - String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(pulsar.getListenPortHTTP().get()); + String prometheusMetrics = TestPulsarFunctionUtils.getPrometheusMetrics(pulsar.getListenPortHTTP().get()); log.info("prometheus metrics: {}", prometheusMetrics); - Map metrics = PulsarFunctionTestUtils.parseMetrics(prometheusMetrics); - PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_function_received_total"); + Map metrics = TestPulsarFunctionUtils.parseMetrics(prometheusMetrics); + TestPulsarFunctionUtils.Metric m = metrics.get("pulsar_function_received_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), functionName); @@ -533,10 +533,10 @@ public void testPulsarFunctionStats() throws Exception { assertEquals(functionInstanceStats, functionStats.instances.get(0).getMetrics()); // validate prometheus metrics - prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(pulsar.getListenPortHTTP().get()); + prometheusMetrics = TestPulsarFunctionUtils.getPrometheusMetrics(pulsar.getListenPortHTTP().get()); log.info("prometheus metrics: {}", prometheusMetrics); - metrics = PulsarFunctionTestUtils.parseMetrics(prometheusMetrics); + metrics = TestPulsarFunctionUtils.parseMetrics(prometheusMetrics); m = metrics.get("pulsar_function_received_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); @@ -736,6 +736,9 @@ public void testAuthorization(boolean validRoleName) throws Exception { FunctionConfig functionConfig = createFunctionConfig(tenant, namespacePortion, functionName, false, "my.*", sinkTopic, subscriptionName); if (!validRoleName) { + if (admin != null) { + admin.close(); + } // create a non-superuser admin to test the api admin = spy( PulsarAdmin.builder().serviceHttpUrl(pulsar.getWebServiceAddressTls()) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionTlsTest.java index 810ac69ac3eb3..da479321b8bc2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionTlsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionTlsTest.java @@ -180,10 +180,22 @@ void setup(Method method) throws Exception { void shutdown() throws Exception { log.info("--- Shutting down ---"); try { - functionAdmin.close(); - functionsWorkerService.stop(); - workerServer.stop(); - bkEnsemble.stop(); + if (functionAdmin != null) { + functionAdmin.close(); + functionAdmin = null; + } + if (functionsWorkerService != null) { + functionsWorkerService.stop(); + functionsWorkerService = null; + } + if (workerServer != null) { + workerServer.stop(); + workerServer = null; + } + if (bkEnsemble != null) { + bkEnsemble.stop(); + bkEnsemble = null; + } } finally { if (tempDirectory != null) { tempDirectory.delete(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java index 7e0dbabb105f9..be2b377a9cff5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java @@ -49,11 +49,11 @@ import org.apache.pulsar.common.policies.data.SinkStatus; import org.apache.pulsar.common.policies.data.SubscriptionStats; import org.apache.pulsar.common.policies.data.TopicStats; -import org.apache.pulsar.compaction.TwoPhaseCompactor; +import org.apache.pulsar.compaction.PublishingOrderCompactor; import org.apache.pulsar.functions.LocalRunner; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.functions.utils.FunctionCommon; -import org.apache.pulsar.functions.worker.PulsarFunctionTestUtils; +import org.apache.pulsar.functions.worker.TestPulsarFunctionUtils; import org.apache.pulsar.io.core.Sink; import org.apache.pulsar.io.core.SinkContext; import org.awaitility.Awaitility; @@ -107,7 +107,7 @@ public void testReadCompactedSink() throws Exception { @Cleanup("shutdownNow") ScheduledExecutorService compactionScheduler = Executors.newSingleThreadScheduledExecutor( new ThreadFactoryBuilder().setNameFormat("compactor").setDaemon(true).build()); - TwoPhaseCompactor twoPhaseCompactor = new TwoPhaseCompactor(config, + PublishingOrderCompactor twoPhaseCompactor = new PublishingOrderCompactor(config, pulsarClient, pulsar.getBookKeeperClient(), compactionScheduler); twoPhaseCompactor.compact(sourceTopic).get(); @@ -122,9 +122,9 @@ public void testReadCompactedSink() throws Exception { // 5 Sink should only read compacted value, so we will only receive compacted messages Awaitility.await().ignoreExceptions().untilAsserted(() -> { - String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(pulsar.getListenPortHTTP().get()); - Map metrics = PulsarFunctionTestUtils.parseMetrics(prometheusMetrics); - PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_sink_received_total"); + String prometheusMetrics = TestPulsarFunctionUtils.getPrometheusMetrics(pulsar.getListenPortHTTP().get()); + Map metrics = TestPulsarFunctionUtils.parseMetrics(prometheusMetrics); + TestPulsarFunctionUtils.Metric m = metrics.get("pulsar_sink_received_total"); assertEquals(m.value, maxKeys); }); } @@ -271,11 +271,11 @@ private void testPulsarSinkStats(String jarFilePathUrl, Function assertEquals(sinkInstanceStatus.status.numSystemExceptions, 0)); // validate prometheus metrics empty - String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(pulsar.getListenPortHTTP().get()); + String prometheusMetrics = TestPulsarFunctionUtils.getPrometheusMetrics(pulsar.getListenPortHTTP().get()); log.info("prometheus metrics: {}", prometheusMetrics); - Map metrics = PulsarFunctionTestUtils.parseMetrics(prometheusMetrics); - PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_sink_received_total"); + Map metrics = TestPulsarFunctionUtils.parseMetrics(prometheusMetrics); + TestPulsarFunctionUtils.Metric m = metrics.get("pulsar_sink_received_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), sinkName); @@ -364,10 +364,10 @@ private void testPulsarSinkStats(String jarFilePathUrl, Function assertEquals(sinkInstanceStatus.status.numSystemExceptions, 0)); // get stats after producing - prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(pulsar.getListenPortHTTP().get()); + prometheusMetrics = TestPulsarFunctionUtils.getPrometheusMetrics(pulsar.getListenPortHTTP().get()); log.info("prometheusMetrics: {}", prometheusMetrics); - metrics = PulsarFunctionTestUtils.parseMetrics(prometheusMetrics); + metrics = TestPulsarFunctionUtils.parseMetrics(prometheusMetrics); m = metrics.get("pulsar_sink_received_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSourceE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSourceE2ETest.java index 99d16447bf894..1b7ffca22832e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSourceE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSourceE2ETest.java @@ -33,7 +33,7 @@ import org.apache.pulsar.common.io.SourceConfig; import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.functions.utils.FunctionCommon; -import org.apache.pulsar.functions.worker.PulsarFunctionTestUtils; +import org.apache.pulsar.functions.worker.TestPulsarFunctionUtils; import org.testng.annotations.Test; import com.google.common.collect.Lists; @@ -106,11 +106,11 @@ private void testPulsarSourceStats(String jarFilePathUrl) throws Exception { }, 50, 150); assertEquals(admin.topics().getStats(sinkTopic2).getPublishers().size(), 1); - String prometheusMetrics = PulsarFunctionTestUtils.getPrometheusMetrics(pulsar.getListenPortHTTP().get()); + String prometheusMetrics = TestPulsarFunctionUtils.getPrometheusMetrics(pulsar.getListenPortHTTP().get()); log.info("prometheusMetrics: {}", prometheusMetrics); - Map metrics = PulsarFunctionTestUtils.parseMetrics(prometheusMetrics); - PulsarFunctionTestUtils.Metric m = metrics.get("pulsar_source_received_total"); + Map metrics = TestPulsarFunctionUtils.parseMetrics(prometheusMetrics); + TestPulsarFunctionUtils.Metric m = metrics.get("pulsar_source_received_total"); assertEquals(m.tags.get("cluster"), config.getClusterName()); assertEquals(m.tags.get("instance_id"), "0"); assertEquals(m.tags.get("name"), sourceName); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java index 7ba4529cdbdf4..ab82f981b5df3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java @@ -23,6 +23,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertThrows; @@ -32,6 +33,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Sets; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.Serializable; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -46,6 +48,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.regex.Pattern; import java.util.stream.Collectors; import lombok.Cleanup; import lombok.EqualsAndHashCode; @@ -57,6 +60,8 @@ import org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage; import org.apache.pulsar.broker.service.schema.SchemaRegistry; import org.apache.pulsar.broker.service.schema.SchemaRegistryServiceImpl; +import org.apache.pulsar.broker.service.schema.SchemaStorageFormat; +import org.apache.pulsar.broker.service.schema.SchemaStorageFormat.SchemaLocator; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -69,6 +74,8 @@ import org.apache.pulsar.client.api.TypedMessageBuilder; import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.schema.SchemaDefinition; +import org.apache.pulsar.client.impl.ConsumerImpl; +import org.apache.pulsar.client.impl.MultiTopicsConsumerImpl; import org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl; import org.apache.pulsar.client.impl.schema.ProtobufSchema; import org.apache.pulsar.client.impl.schema.SchemaInfoImpl; @@ -78,15 +85,20 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; +import org.apache.pulsar.common.policies.data.SchemaMetadata; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.metadata.api.MetadataCache; +import org.apache.pulsar.metadata.api.MetadataSerde; +import org.apache.pulsar.metadata.api.Stat; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @Slf4j @@ -98,6 +110,7 @@ public class SchemaTest extends MockedPulsarServiceBaseTest { @BeforeMethod @Override public void setup() throws Exception { + isTcpLookup = true; super.internalSetup(); // Setup namespaces @@ -106,6 +119,7 @@ public void setup() throws Exception { .allowedClusters(Collections.singleton(CLUSTER_NAME)) .build(); admin.tenants().createTenant(PUBLIC_TENANT, tenantInfo); + admin.namespaces().createNamespace(PUBLIC_TENANT + "/my-ns"); } @AfterMethod(alwaysRun = true) @@ -114,6 +128,11 @@ public void cleanup() throws Exception { super.internalCleanup(); } + @DataProvider(name = "topicDomain") + public static Object[] topicDomain() { + return new Object[] { "persistent://", "non-persistent://" }; + } + @Test public void testGetSchemaWhenCreateAutoProduceBytesProducer() throws Exception{ final String tenant = PUBLIC_TENANT; @@ -130,6 +149,34 @@ public void testGetSchemaWhenCreateAutoProduceBytesProducer() throws Exception{ pulsarClient.newProducer(org.apache.pulsar.client.api.Schema.AUTO_PRODUCE_BYTES()).topic(topic).create(); } + @Test + public void testGetSchemaWithPatternTopic() throws Exception { + final String topicPrefix = "persistent://public/my-ns/test-getSchema"; + + int topicNums = 10; + for (int i = 0; i < topicNums; i++) { + String topic = topicPrefix + "-" + i; + admin.topics().createNonPartitionedTopic(topic); + } + + Pattern pattern = Pattern.compile(topicPrefix + "-.*"); + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) + .topicsPattern(pattern) + .subscriptionName("sub") + .subscriptionType(SubscriptionType.Shared) + .subscribe(); + + List> consumers = + ((MultiTopicsConsumerImpl) consumer).getConsumers(); + Assert.assertEquals(topicNums, consumers.size()); + + for (int i = 0; i < topicNums; i++) { + String topic = topicPrefix + "-" + i; + admin.topics().delete(topic, true); + } + } + @Test public void testMultiTopicSetSchemaProvider() throws Exception { final String tenant = PUBLIC_TENANT; @@ -1290,6 +1337,33 @@ private void testIncompatibleSchema() throws Exception { assertThrows(SchemaSerializationException.class, message2::getValue); } + /** + * This test just ensure that schema check still keeps the original logic: if there has any producer, but no schema + * was registered, the new consumer could not register new schema. + * TODO: I think this design should be improved: if a producer used "AUTO_PRODUCE_BYTES" schema, we should allow + * the new consumer to register new schema. But before we can solve this problem, we need to modify + * "CmdProducer" to let the Broker know that the Producer uses a schema of type "AUTO_PRODUCE_BYTES". + */ + @Test(dataProvider = "topicDomain") + public void testAutoProduceAndSpecifiedConsumer(String domain) throws Exception { + final String namespace = PUBLIC_TENANT + "/ns_" + randomName(16); + admin.namespaces().createNamespace(namespace, Sets.newHashSet(CLUSTER_NAME)); + final String topicName = domain + namespace + "/tp_" + randomName(16); + admin.topics().createNonPartitionedTopic(topicName); + + Producer producer = pulsarClient.newProducer(Schema.AUTO_PRODUCE_BYTES()).topic(topicName).create(); + try { + pulsarClient.newConsumer(Schema.STRING).topic(topicName).subscriptionName("sub1").subscribe(); + fail("Should throw ex: Failed to add schema to an active topic with empty(BYTES) schema"); + } catch (Exception ex){ + assertTrue(ex.getMessage().contains("Failed to add schema to an active topic with empty(BYTES) schema")); + } + + // Cleanup. + producer.close(); + admin.topics().delete(topicName); + } + @Test public void testCreateSchemaInParallel() throws Exception { final String namespace = "test-namespace-" + randomName(16); @@ -1297,6 +1371,7 @@ public void testCreateSchemaInParallel() throws Exception { admin.namespaces().createNamespace(ns, Sets.newHashSet(CLUSTER_NAME)); final String topic = getTopicName(ns, "testCreateSchemaInParallel"); + @Cleanup("shutdownNow") ExecutorService executor = Executors.newFixedThreadPool(16); List>> producers = new ArrayList<>(16); CountDownLatch latch = new CountDownLatch(16); @@ -1338,7 +1413,6 @@ public void testCreateSchemaInParallel() throws Exception { }); producers.clear(); producers2.clear(); - executor.shutdownNow(); } @EqualsAndHashCode @@ -1350,4 +1424,98 @@ public User(String name) { } } + /** + * This test validates that consumer/producers should recover on topic whose + * schema ledgers are not able to open due to non-recoverable error. + * + * @throws Exception + */ + @Test + public void testDeletedSchemaLedgerRecovery() throws Exception { + final String tenant = PUBLIC_TENANT; + final String namespace = "test-namespace-" + randomName(16); + final String topicOne = "test-multi-version-schema-one"; + final String subName = "test"; + final String topicName = TopicName.get(TopicDomain.persistent.value(), tenant, namespace, topicOne).toString(); + + admin.namespaces().createNamespace(tenant + "/" + namespace, Sets.newHashSet(CLUSTER_NAME)); + + // (1) create schema + Producer producer = pulsarClient + .newProducer(Schema.AVRO(SchemaDefinition. builder().withAlwaysAllowNull(false) + .withSupportSchemaVersioning(true).withPojo(Schemas.PersonTwo.class).build())) + .topic(topicName).create(); + + Schemas.PersonTwo personTwo = new Schemas.PersonTwo(); + personTwo.setId(1); + personTwo.setName("Tom"); + + Consumer consumer = pulsarClient + .newConsumer(Schema.AVRO(SchemaDefinition. builder().withAlwaysAllowNull(false) + .withSupportSchemaVersioning(true).withPojo(Schemas.PersonTwo.class).build())) + .subscriptionName(subName).topic(topicName).subscribe(); + + producer.send(personTwo); + producer.close(); + consumer.close(); + + // (2) Delete schema ledger + MetadataCache locatorEntryCache = pulsar.getLocalMetadataStore() + .getMetadataCache(new MetadataSerde() { + @Override + public byte[] serialize(String path, SchemaStorageFormat.SchemaLocator value) { + return value.toByteArray(); + } + + @Override + public SchemaStorageFormat.SchemaLocator deserialize(String path, byte[] content, Stat stat) + throws IOException { + return SchemaStorageFormat.SchemaLocator.parseFrom(content); + } + }); + String path = "/schemas/public/" + namespace + "/test-multi-version-schema-one"; + SchemaLocator schema = locatorEntryCache.get(path).get().get(); + schema = locatorEntryCache.get(path).get().get(); + long ledgerId = schema.getInfo().getPosition().getLedgerId(); + pulsar.getBookKeeperClient().deleteLedger(ledgerId); + + // (3) Topic should recover from deleted schema and should allow to create consumer and producer + consumer = pulsarClient + .newConsumer(Schema.AVRO(SchemaDefinition. builder().withAlwaysAllowNull(false) + .withSupportSchemaVersioning(true).withPojo(Schemas.PersonTwo.class).build())) + .subscriptionName(subName).topic(topicName).subscribe(); + + producer = pulsarClient + .newProducer(Schema.AVRO(SchemaDefinition. builder().withAlwaysAllowNull(false) + .withSupportSchemaVersioning(true).withPojo(Schemas.PersonTwo.class).build())) + .topic(topicName).create(); + assertNotNull(consumer); + assertNotNull(producer); + consumer.close(); + producer.close(); + } + + @Test + public void testTopicSchemaMetadata() throws Exception { + final String tenant = PUBLIC_TENANT; + final String namespace = "test-namespace-" + randomName(16); + final String topicOne = "metadata-topic"; + final String topicName = TopicName.get(TopicDomain.persistent.value(), tenant, namespace, topicOne).toString(); + + admin.namespaces().createNamespace(tenant + "/" + namespace, Sets.newHashSet(CLUSTER_NAME)); + + @Cleanup + Producer producer = pulsarClient + .newProducer(Schema.AVRO(SchemaDefinition. builder().withAlwaysAllowNull(false) + .withSupportSchemaVersioning(true).withPojo(Schemas.PersonTwo.class).build())) + .topic(topicName).create(); + + SchemaMetadata metadata = admin.schemas().getSchemaMetadata(topicName); + + assertNotNull(metadata); + assertNotNull(metadata.info); + assertNotEquals(metadata.info.getLedgerId(), 0); + assertEquals(metadata.info.getEntryId(), 0); + assertEquals(metadata.index.size(), 1); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java index 140dea9e7ebc7..49517a424b936 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java @@ -407,7 +407,7 @@ public void testSchemaComparison() throws Exception { assertEquals(admin.namespaces().getSchemaCompatibilityStrategy(namespaceName.toString()), SchemaCompatibilityStrategy.UNDEFINED); byte[] changeSchemaBytes = (new String(Schema.AVRO(Schemas.PersonOne.class) - .getSchemaInfo().getSchema(), UTF_8) + "/n /n /n").getBytes(); + .getSchemaInfo().getSchema(), UTF_8) + "\n \n \n").getBytes(); SchemaInfo schemaInfo = SchemaInfo.builder().type(SchemaType.AVRO).schema(changeSchemaBytes).build(); admin.schemas().createSchema(fqtn, schemaInfo); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/security/MockedPulsarStandalone.java b/pulsar-broker/src/test/java/org/apache/pulsar/security/MockedPulsarStandalone.java new file mode 100644 index 0000000000000..866018b32fb0c --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/security/MockedPulsarStandalone.java @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.security; + +import static org.apache.pulsar.utils.ResourceUtils.getAbsolutePath; +import io.jsonwebtoken.Jwts; +import io.jsonwebtoken.SignatureAlgorithm; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Sets; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import javax.crypto.SecretKey; +import lombok.Getter; +import lombok.SneakyThrows; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.authentication.AuthenticationProviderTls; +import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; +import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; +import org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.impl.auth.AuthenticationKeyStoreTls; +import org.apache.pulsar.client.impl.auth.AuthenticationTls; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.common.util.ObjectMapperFactory; + + + +public abstract class MockedPulsarStandalone implements AutoCloseable { + + @Getter + private final ServiceConfiguration serviceConfiguration = new ServiceConfiguration(); + private PulsarTestContext pulsarTestContext; + + @Getter + private PulsarService pulsarService; + private PulsarAdmin serviceInternalAdmin; + + + { + serviceConfiguration.setClusterName(TEST_CLUSTER_NAME); + serviceConfiguration.setBrokerShutdownTimeoutMs(0L); + serviceConfiguration.setBrokerServicePort(Optional.of(0)); + serviceConfiguration.setBrokerServicePortTls(Optional.of(0)); + serviceConfiguration.setAdvertisedAddress("localhost"); + serviceConfiguration.setWebServicePort(Optional.of(0)); + serviceConfiguration.setWebServicePortTls(Optional.of(0)); + serviceConfiguration.setNumExecutorThreadPoolSize(5); + serviceConfiguration.setExposeBundlesMetricsInPrometheus(true); + serviceConfiguration.setTlsTrustCertsFilePath(TLS_EC_TRUSTED_CERT_PATH); + serviceConfiguration.setTlsCertificateFilePath(TLS_EC_SERVER_CERT_PATH); + serviceConfiguration.setTlsKeyFilePath(TLS_EC_SERVER_KEY_PATH); + } + + + protected static final SecretKey SECRET_KEY = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); + + private static final String BROKER_INTERNAL_CLIENT_SUBJECT = "broker_internal"; + private static final String BROKER_INTERNAL_CLIENT_TOKEN = Jwts.builder() + .claim("sub", BROKER_INTERNAL_CLIENT_SUBJECT).signWith(SECRET_KEY).compact(); + protected static final String SUPER_USER_SUBJECT = "super-user"; + protected static final String SUPER_USER_TOKEN = Jwts.builder() + .claim("sub", SUPER_USER_SUBJECT).signWith(SECRET_KEY).compact(); + protected static final String NOBODY_SUBJECT = "nobody"; + protected static final String NOBODY_TOKEN = Jwts.builder() + .claim("sub", NOBODY_SUBJECT).signWith(SECRET_KEY).compact(); + + + @SneakyThrows + protected void configureTokenAuthentication() { + serviceConfiguration.setAuthenticationEnabled(true); + serviceConfiguration.setAuthenticationProviders(Set.of(AuthenticationProviderToken.class.getName())); + // internal client + serviceConfiguration.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); + final Map brokerClientAuthParams = new HashMap<>(); + brokerClientAuthParams.put("token", BROKER_INTERNAL_CLIENT_TOKEN); + final String brokerClientAuthParamStr = MAPPER.writeValueAsString(brokerClientAuthParams); + serviceConfiguration.setBrokerClientAuthenticationParameters(brokerClientAuthParamStr); + + Properties properties = serviceConfiguration.getProperties(); + if (properties == null) { + properties = new Properties(); + serviceConfiguration.setProperties(properties); + } + properties.put("tokenSecretKey", AuthTokenUtils.encodeKeyBase64(SECRET_KEY)); + + } + + protected void enableTransaction() { + serviceConfiguration.setTransactionCoordinatorEnabled(true); + } + + protected void configureDefaultAuthorization() { + serviceConfiguration.setAuthorizationEnabled(true); + serviceConfiguration.setAuthorizationProvider(PulsarAuthorizationProvider.class.getName()); + serviceConfiguration.setSuperUserRoles(Set.of(SUPER_USER_SUBJECT, BROKER_INTERNAL_CLIENT_SUBJECT)); + } + + + + @SneakyThrows + protected void loadECTlsCertificateWithFile() { + serviceConfiguration.setTlsEnabled(true); + serviceConfiguration.setBrokerServicePort(Optional.empty()); + serviceConfiguration.setWebServicePort(Optional.empty()); + serviceConfiguration.setTlsTrustCertsFilePath(TLS_EC_TRUSTED_CERT_PATH); + serviceConfiguration.setTlsCertificateFilePath(TLS_EC_SERVER_CERT_PATH); + serviceConfiguration.setTlsKeyFilePath(TLS_EC_SERVER_KEY_PATH); + serviceConfiguration.setBrokerClientTlsEnabled(true); + serviceConfiguration.setBrokerClientTrustCertsFilePath(TLS_EC_TRUSTED_CERT_PATH); + serviceConfiguration.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); + final Map brokerClientAuthParams = new HashMap<>(); + brokerClientAuthParams.put("tlsCertFile", TLS_EC_BROKER_CLIENT_CERT_PATH); + brokerClientAuthParams.put("tlsKeyFile", TLS_EC_BROKER_CLIENT_KEY_PATH); + serviceConfiguration.setBrokerClientAuthenticationParameters(mapper.writeValueAsString(brokerClientAuthParams)); + } + + @SneakyThrows + protected void loadECTlsCertificateWithKeyStore() { + serviceConfiguration.setTlsEnabled(true); + serviceConfiguration.setBrokerServicePort(Optional.empty()); + serviceConfiguration.setWebServicePort(Optional.empty()); + serviceConfiguration.setTlsEnabledWithKeyStore(true); + serviceConfiguration.setTlsKeyStore(TLS_EC_KS_SERVER_STORE); + serviceConfiguration.setTlsKeyStorePassword(TLS_EC_KS_SERVER_PASS); + serviceConfiguration.setTlsTrustStore(TLS_EC_KS_TRUSTED_STORE); + serviceConfiguration.setTlsTrustStorePassword(TLS_EC_KS_TRUSTED_STORE_PASS); + serviceConfiguration.setBrokerClientTlsEnabled(true); + serviceConfiguration.setBrokerClientTlsEnabledWithKeyStore(true); + serviceConfiguration.setBrokerClientTlsKeyStore(TLS_EC_KS_BROKER_CLIENT_STORE); + serviceConfiguration.setBrokerClientTlsKeyStorePassword(TLS_EC_KS_BROKER_CLIENT_PASS); + serviceConfiguration.setBrokerClientTlsTrustStore(TLS_EC_KS_TRUSTED_STORE); + serviceConfiguration.setBrokerClientTlsTrustStorePassword(TLS_EC_KS_TRUSTED_STORE_PASS); + serviceConfiguration.setBrokerClientAuthenticationPlugin(AuthenticationKeyStoreTls.class.getName()); + final Map brokerClientAuthParams = new HashMap<>(); + brokerClientAuthParams.put("keyStorePath", TLS_EC_KS_BROKER_CLIENT_STORE); + brokerClientAuthParams.put("keyStorePassword", TLS_EC_KS_BROKER_CLIENT_PASS); + serviceConfiguration.setBrokerClientAuthenticationParameters(mapper.writeValueAsString(brokerClientAuthParams)); + } + + protected void enableTlsAuthentication() { + serviceConfiguration.setAuthenticationEnabled(true); + serviceConfiguration.setAuthenticationProviders(Sets.newHashSet(AuthenticationProviderTls.class.getName())); + } + + @SneakyThrows + protected void start() { + this.pulsarTestContext = PulsarTestContext.builder() + .spyByDefault() + .config(serviceConfiguration) + .withMockZookeeper(false) + .build(); + this.pulsarService = pulsarTestContext.getPulsarService(); + this.serviceInternalAdmin = pulsarService.getAdminClient(); + setupDefaultTenantAndNamespace(); + } + + private void setupDefaultTenantAndNamespace() throws Exception { + if (!serviceInternalAdmin.clusters().getClusters().contains(TEST_CLUSTER_NAME)) { + serviceInternalAdmin.clusters().createCluster(TEST_CLUSTER_NAME, + ClusterData.builder().serviceUrl(pulsarService.getWebServiceAddress()).build()); + } + if (!serviceInternalAdmin.tenants().getTenants().contains(DEFAULT_TENANT)) { + serviceInternalAdmin.tenants().createTenant(DEFAULT_TENANT, TenantInfo.builder().allowedClusters( + Sets.newHashSet(TEST_CLUSTER_NAME)).build()); + } + if (!serviceInternalAdmin.namespaces().getNamespaces(DEFAULT_TENANT).contains(DEFAULT_NAMESPACE)) { + serviceInternalAdmin.namespaces().createNamespace(DEFAULT_NAMESPACE); + } + } + + + @Override + public void close() throws Exception { + if (pulsarTestContext != null) { + pulsarTestContext.close(); + pulsarTestContext = null; + } + pulsarService = null; + serviceInternalAdmin = null; + } + + // Utils + protected static final ObjectMapper mapper = new ObjectMapper(); + + // Static name + private static final String DEFAULT_TENANT = "public"; + private static final String DEFAULT_NAMESPACE = "public/default"; + private static final String TEST_CLUSTER_NAME = "test-standalone"; + + // EC certificate + protected static final String TLS_EC_TRUSTED_CERT_PATH = + getAbsolutePath("certificate-authority/ec/ca.cert.pem"); + private static final String TLS_EC_SERVER_KEY_PATH = + getAbsolutePath("certificate-authority/ec/server.key-pk8.pem"); + private static final String TLS_EC_SERVER_CERT_PATH = + getAbsolutePath("certificate-authority/ec/server.cert.pem"); + private static final String TLS_EC_BROKER_CLIENT_KEY_PATH = + getAbsolutePath("certificate-authority/ec/broker_client.key-pk8.pem"); + private static final String TLS_EC_BROKER_CLIENT_CERT_PATH = + getAbsolutePath("certificate-authority/ec/broker_client.cert.pem"); + protected static final String TLS_EC_CLIENT_KEY_PATH = + getAbsolutePath("certificate-authority/ec/client.key-pk8.pem"); + protected static final String TLS_EC_CLIENT_CERT_PATH = + getAbsolutePath("certificate-authority/ec/client.cert.pem"); + + // EC KeyStore + private static final String TLS_EC_KS_SERVER_STORE = + getAbsolutePath("certificate-authority/ec/jks/server.keystore.jks"); + private static final String TLS_EC_KS_SERVER_PASS = "serverpw"; + private static final String TLS_EC_KS_BROKER_CLIENT_STORE = + getAbsolutePath("certificate-authority/ec/jks/broker_client.keystore.jks"); + private static final String TLS_EC_KS_BROKER_CLIENT_PASS = "brokerclientpw"; + protected static final String TLS_EC_KS_CLIENT_STORE = + getAbsolutePath("certificate-authority/ec/jks/client.keystore.jks"); + protected static final String TLS_EC_KS_CLIENT_PASS = "clientpw"; + protected static final String TLS_EC_KS_TRUSTED_STORE = + getAbsolutePath("certificate-authority/ec/jks/ca.truststore.jks"); + protected static final String TLS_EC_KS_TRUSTED_STORE_PASS = "rootpw"; + + + private static final ObjectMapper MAPPER = ObjectMapperFactory.getMapper().getObjectMapper(); +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/security/tls/ec/TlsWithECCertificateFileTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/security/tls/ec/TlsWithECCertificateFileTest.java new file mode 100644 index 0000000000000..b02b10f5996bf --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/security/tls/ec/TlsWithECCertificateFileTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.security.tls.ec; + + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import java.nio.charset.StandardCharsets; +import java.util.UUID; +import lombok.Cleanup; +import lombok.SneakyThrows; +import org.apache.pulsar.security.MockedPulsarStandalone; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.impl.auth.AuthenticationTls; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + + +@Test +public class TlsWithECCertificateFileTest extends MockedPulsarStandalone { + + @BeforeClass(alwaysRun = true) + public void suitSetup() { + loadECTlsCertificateWithFile(); + enableTlsAuthentication(); + super.start(); // start standalone service + } + + @SneakyThrows + @AfterClass(alwaysRun = true) + public void suitShutdown() { + super.close(); // close standalone service + } + + + @Test(expectedExceptions = PulsarClientException.class) + @SneakyThrows + public void testConnectionFailWithoutCertificate() { + @Cleanup final PulsarClient client = PulsarClient.builder() + .serviceUrl(getPulsarService().getBrokerServiceUrlTls()) + .build(); + @Cleanup final Producer producer = client.newProducer() + .topic("should_be_failed") + .create(); + } + + + @Test + @SneakyThrows + public void testConnectionSuccessWithCertificate() { + final AuthenticationTls authentication = new AuthenticationTls(TLS_EC_CLIENT_CERT_PATH, TLS_EC_CLIENT_KEY_PATH); + final String topicName = "persistent://public/default/" + UUID.randomUUID(); + final int testMsgNum = 10; + @Cleanup final PulsarAdmin admin = PulsarAdmin.builder() + .authentication(authentication) + .serviceHttpUrl(getPulsarService().getWebServiceAddressTls()) + .tlsTrustCertsFilePath(TLS_EC_TRUSTED_CERT_PATH) + .build(); + admin.topics().createNonPartitionedTopic(topicName); + admin.topics().createSubscription(topicName, "sub-1", MessageId.earliest); + @Cleanup final PulsarClient client = PulsarClient.builder() + .serviceUrl(getPulsarService().getBrokerServiceUrlTls()) + .authentication(authentication) + .tlsTrustCertsFilePath(TLS_EC_TRUSTED_CERT_PATH) + .build(); + @Cleanup final Producer producer = client.newProducer() + .topic(topicName) + .create(); + @Cleanup final Consumer consumer = client.newConsumer() + .topic(topicName) + .subscriptionName("sub-1") + .consumerName("cons-1") + .subscribe(); + for (int i = 0; i < testMsgNum; i++) { + producer.send((i + "").getBytes(StandardCharsets.UTF_8)); + } + + for (int i = 0; i < testMsgNum; i++) { + final Message message = consumer.receive(); + assertNotNull(message); + final byte[] b = message.getValue(); + final String s = new String(b, StandardCharsets.UTF_8); + assertEquals(s, i + ""); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/security/tls/ec/TlsWithECKeyStoreTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/security/tls/ec/TlsWithECKeyStoreTest.java new file mode 100644 index 0000000000000..c6ff16d4cc50d --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/security/tls/ec/TlsWithECKeyStoreTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.security.tls.ec; + + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import lombok.Cleanup; +import lombok.SneakyThrows; +import org.apache.pulsar.security.MockedPulsarStandalone; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.impl.auth.AuthenticationKeyStoreTls; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + + +@Test +public class TlsWithECKeyStoreTest extends MockedPulsarStandalone { + + @BeforeClass(alwaysRun = true) + public void suitSetup() { + loadECTlsCertificateWithKeyStore(); + enableTlsAuthentication(); + super.start(); // start standalone service + } + + @SneakyThrows + @AfterClass(alwaysRun = true) + public void suitShutdown() { + super.close(); // close standalone service + } + + + @Test(expectedExceptions = PulsarClientException.class) + @SneakyThrows + public void testConnectionFailWithoutCertificate() { + @Cleanup final PulsarClient client = PulsarClient.builder() + .serviceUrl(getPulsarService().getBrokerServiceUrlTls()) + .build(); + @Cleanup final Producer producer = client.newProducer() + .topic("should_be_failed") + .create(); + } + + + @Test + @SneakyThrows + public void testConnectionSuccessWithCertificate() { + final String topicName = "persistent://public/default/" + UUID.randomUUID(); + final int testMsgNum = 10; + final Map clientAuthParams = new HashMap<>(); + clientAuthParams.put("keyStorePath", TLS_EC_KS_CLIENT_STORE); + clientAuthParams.put("keyStorePassword", TLS_EC_KS_CLIENT_PASS); + @Cleanup final PulsarAdmin admin = PulsarAdmin.builder() + .useKeyStoreTls(true) + .tlsKeyStorePath(TLS_EC_KS_CLIENT_STORE) + .tlsKeyStorePassword(TLS_EC_KS_CLIENT_PASS) + .tlsTrustStorePath(TLS_EC_KS_TRUSTED_STORE) + .tlsTrustStorePassword(TLS_EC_KS_TRUSTED_STORE_PASS) + .authentication(AuthenticationKeyStoreTls.class.getName(), mapper.writeValueAsString(clientAuthParams)) + .serviceHttpUrl(getPulsarService().getWebServiceAddressTls()) + .build(); + admin.topics().createNonPartitionedTopic(topicName); + admin.topics().createSubscription(topicName, "sub-1", MessageId.earliest); + @Cleanup final PulsarClient client = PulsarClient.builder() + .serviceUrl(getPulsarService().getBrokerServiceUrlTls()) + .useKeyStoreTls(true) + .tlsKeyStorePath(TLS_EC_KS_CLIENT_STORE) + .tlsKeyStorePassword(TLS_EC_KS_CLIENT_PASS) + .tlsTrustStorePath(TLS_EC_KS_TRUSTED_STORE) + .tlsTrustStorePassword(TLS_EC_KS_TRUSTED_STORE_PASS) + .authentication(AuthenticationKeyStoreTls.class.getName(), mapper.writeValueAsString(clientAuthParams)) + .build(); + @Cleanup final Producer producer = client.newProducer() + .topic(topicName) + .create(); + @Cleanup final Consumer consumer = client.newConsumer() + .topic(topicName) + .subscriptionName("sub-1") + .consumerName("cons-1") + .subscribe(); + for (int i = 0; i < testMsgNum; i++) { + producer.send((i + "").getBytes(StandardCharsets.UTF_8)); + } + + for (int i = 0; i < testMsgNum; i++) { + final Message message = consumer.receive(); + assertNotNull(message); + final byte[] b = message.getValue(); + final String s = new String(b, StandardCharsets.UTF_8); + assertEquals(s, i + ""); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/stats/client/PulsarBrokerStatsClientTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/stats/client/PulsarBrokerStatsClientTest.java index b58ed1e40ba3c..53ee254e375d7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/stats/client/PulsarBrokerStatsClientTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/stats/client/PulsarBrokerStatsClientTest.java @@ -28,6 +28,7 @@ import javax.ws.rs.ClientErrorException; import javax.ws.rs.ServerErrorException; +import lombok.Cleanup; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -68,6 +69,7 @@ protected void cleanup() throws Exception { @Test public void testServiceException() throws Exception { URL url = new URL("http://localhost:15000"); + @Cleanup PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(url.toString()).build(); BrokerStatsImpl client = (BrokerStatsImpl) spy(admin.brokerStats()); try { @@ -94,8 +96,6 @@ public void testServiceException() throws Exception { assertTrue(client.getApiException(new ServerErrorException(503)) instanceof PulsarAdminException); log.info("Client: -- {}", client); - - admin.close(); } @Test @@ -126,7 +126,7 @@ public void testTopicInternalStats() throws Exception { PersistentTopicInternalStats internalStats = topic.getInternalStats(true).get(); assertNotNull(internalStats.ledgers.get(0).metadata); // For the mock test, the default ensembles is ["192.0.2.1:1234","192.0.2.2:1234","192.0.2.3:1234"] - // The registed bookie ID is 192.168.1.1:5000 + // The registered bookie ID is 192.168.1.1:5000 assertTrue(internalStats.ledgers.get(0).underReplicated); CursorStats cursor = internalStats.cursors.get(subscriptionName); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/utils/SimpleCacheTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/utils/SimpleCacheTest.java new file mode 100644 index 0000000000000..c590eda171804 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/utils/SimpleCacheTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.utils; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.Test; + +public class SimpleCacheTest { + + private final ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + + @AfterClass + public void shutdown() { + executor.shutdown(); + } + + @Test + public void testConcurrentUpdate() throws Exception { + final var cache = new SimpleCache(executor, 10000L, 10000L); + final var pool = Executors.newFixedThreadPool(2); + final var latch = new CountDownLatch(2); + for (int i = 0; i < 2; i++) { + final var value = i + 100; + pool.execute(() -> { + try { + Thread.sleep(100); + } catch (InterruptedException ignored) { + } + cache.get(0, () -> value, __ -> {}); + latch.countDown(); + }); + } + latch.await(); + final var value = cache.get(0, () -> -1, __ -> {}); + Assert.assertTrue(value == 100 || value == 101); + pool.shutdown(); + } + + @Test + public void testExpire() throws InterruptedException { + final var cache = new SimpleCache(executor, 500L, 5); + final var expiredValues = Collections.synchronizedSet(new HashSet()); + + final var allKeys = IntStream.range(0, 5).boxed().collect(Collectors.toSet()); + allKeys.forEach(key -> cache.get(key, () -> key + 100, expiredValues::add)); + + Thread.sleep(400L); + final var recentAccessedKey = Set.of(1, 2); + recentAccessedKey.forEach(key -> cache.get(key, () -> -1, expiredValues::add)); // access these keys + + Thread.sleep(300L); + recentAccessedKey.forEach(key -> Assert.assertEquals(key + 100, cache.get(key, () -> -1, __ -> {}))); + allKeys.stream().filter(key -> !recentAccessedKey.contains(key)) + .forEach(key -> Assert.assertEquals(-1, cache.get(key, () -> -1, __ -> {}))); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/utils/StatsOutputStreamTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/utils/StatsOutputStreamTest.java index 4e6348d1d4ac3..02bc4132e095e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/utils/StatsOutputStreamTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/utils/StatsOutputStreamTest.java @@ -150,4 +150,26 @@ public String str() { reset(); return s; } + + @Test + public void testBehaviorOfStatsOutputStreamWithDeque() { + // Create a byte buffer for collecting output + ByteBuf buffer = Unpooled.buffer(); + + // Create an instance of StatsOutputStream using Deque + StatsOutputStream output = new StatsOutputStream(buffer); + output.startObject() + .writePair("name", "test") + .startList("items") + .writeItem(true) + .writeItem(123L) + .writeItem("sample") + .endList() + .endObject(); + + // Assert + assertEquals(buffer.toString(java.nio.charset.StandardCharsets.UTF_8), + "{\"name\":\"test\",\"items\":[true,123,\"sample\"]}"); + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtilsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtilsTest.java index a488e4d958429..eec568c64e313 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtilsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtilsTest.java @@ -18,19 +18,120 @@ */ package org.apache.pulsar.utils.auth.tokens; +import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; -import com.beust.jcommander.Parameter; + +import io.jsonwebtoken.Claims; +import io.jsonwebtoken.JwsHeader; +import io.jsonwebtoken.Jwt; +import io.jsonwebtoken.Jwts; +import io.jsonwebtoken.io.Decoders; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.lang.reflect.Field; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Arrays; +import java.util.Date; + +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import picocli.CommandLine.Option; /** * TokensCliUtils Tests. */ public class TokensCliUtilsTest { + @DataProvider(name = "desiredExpireTime") + public Object[][] desiredExpireTime() { + return new Object[][] { + {"600", 600}, //10m + {"5m", 300}, + {"1h", 3600}, + {"1d", 86400}, + {"1w", 604800}, + {"1y", 31536000} + }; + } + + @Test + public void testCreateToken() { + PrintStream oldStream = System.out; + try { + ByteArrayOutputStream baoStream = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baoStream)); + + new TokensCliUtils().execute(new String[]{"create-secret-key", "--base64"}); + String secretKey = baoStream.toString(); + + baoStream.reset(); + + String[] command = {"create", "--secret-key", + "data:;base64," + secretKey, + "--subject", "test", + "--headers", "kid=test", + "--headers", "my-k=my-v" + }; + + new TokensCliUtils().execute(command); + String token = baoStream.toString(); + + Jwt jwt = Jwts.parserBuilder() + .setSigningKey(Decoders.BASE64.decode(secretKey)) + .build() + .parseClaimsJws(token); + + JwsHeader header = (JwsHeader) jwt.getHeader(); + String keyId = header.getKeyId(); + assertEquals(keyId, "test"); + assertEquals(header.get("my-k"), "my-v"); + + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + System.setOut(oldStream); + } + } + + @Test(dataProvider = "desiredExpireTime") + public void commandCreateToken_WhenCreatingATokenWithExpiryTime_ShouldHaveTheDesiredExpireTime(String expireTime, int expireAsSec) throws Exception { + PrintStream oldStream = System.out; + try { + //Arrange + ByteArrayOutputStream baoStream = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baoStream)); + + String[] command = {"create", "--secret-key", + "data:;base64,u+FxaxYWpsTfxeEmMh8fQeS3g2jfXw4+sGIv+PTY+BY=", + "--subject", "test", + "--expiry-time", expireTime, + }; + + new TokensCliUtils().execute(command); + String token = baoStream.toString(); + + Instant start = (new Date().toInstant().plus(expireAsSec - 5, ChronoUnit.SECONDS)); + Instant stop = (new Date().toInstant().plus(expireAsSec + 5, ChronoUnit.SECONDS)); + + //Act + Claims jwt = Jwts.parserBuilder() + .setSigningKey(Decoders.BASE64.decode("u+FxaxYWpsTfxeEmMh8fQeS3g2jfXw4+sGIv+PTY+BY=")) + .build() + .parseClaimsJws(token) + .getBody(); + + //Assert + //Checks if the token expires within +-5 sec. + assertTrue(( ! jwt.getExpiration().toInstant().isBefore( start ) ) && ( jwt.getExpiration().toInstant().isBefore( stop ) )); + + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + System.setOut(oldStream); + } + } + /** * Test tokens generate docs. * @@ -43,7 +144,7 @@ public void testGenerateDocs() throws Exception { ByteArrayOutputStream baoStream = new ByteArrayOutputStream(); System.setOut(new PrintStream(baoStream)); - TokensCliUtils.main(new String[]{"gen-doc"}); + new TokensCliUtils().execute(new String[]{"gen-doc"}); String message = baoStream.toString(); @@ -68,9 +169,9 @@ private void assertInnerClass(String className, String message) throws Exception Class argumentsClass = Class.forName(className); Field[] fields = argumentsClass.getDeclaredFields(); for (Field field : fields) { - boolean fieldHasAnno = field.isAnnotationPresent(Parameter.class); + boolean fieldHasAnno = field.isAnnotationPresent(Option.class); if (fieldHasAnno) { - Parameter fieldAnno = field.getAnnotation(Parameter.class); + Option fieldAnno = field.getAnnotation(Option.class); String[] names = fieldAnno.names(); if (names.length < 1) { continue; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ClientSideEncryptionWssConsumer.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ClientSideEncryptionWssConsumer.java new file mode 100644 index 0000000000000..ab8372cbfb058 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ClientSideEncryptionWssConsumer.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.websocket.proxy; + +import static org.testng.Assert.assertTrue; +import java.io.Closeable; +import java.io.IOException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.client.api.CryptoKeyReader; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.crypto.MessageCryptoBc; +import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.apache.pulsar.websocket.data.ConsumerMessage; +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.WebSocketAdapter; +import org.eclipse.jetty.websocket.api.annotations.WebSocket; +import org.eclipse.jetty.websocket.client.ClientUpgradeRequest; +import org.eclipse.jetty.websocket.client.WebSocketClient; + +@Slf4j +@WebSocket(maxTextMessageSize = 64 * 1024) +public class ClientSideEncryptionWssConsumer extends WebSocketAdapter implements Closeable { + + private Session session; + private final CryptoKeyReader cryptoKeyReader; + private final String topicName; + private final String subscriptionName; + private final SubscriptionType subscriptionType; + private final String webSocketProxyHost; + private final int webSocketProxyPort; + private WebSocketClient wssClient; + private final MessageCryptoBc msgCrypto; + private final LinkedBlockingQueue incomingMessages = new LinkedBlockingQueue<>(); + + public ClientSideEncryptionWssConsumer(String webSocketProxyHost, int webSocketProxyPort, String topicName, + String subscriptionName, SubscriptionType subscriptionType, + CryptoKeyReader cryptoKeyReader) { + this.webSocketProxyHost = webSocketProxyHost; + this.webSocketProxyPort = webSocketProxyPort; + this.topicName = topicName; + this.subscriptionName = subscriptionName; + this.subscriptionType = subscriptionType; + this.msgCrypto = new MessageCryptoBc("[" + topicName + "] [" + subscriptionName + "]", false); + this.cryptoKeyReader = cryptoKeyReader; + } + + public void start() throws Exception { + wssClient = new WebSocketClient(); + wssClient.start(); + session = wssClient.connect(this, buildConnectURL(), new ClientUpgradeRequest()).get(); + assertTrue(session.isOpen()); + } + + private URI buildConnectURL() throws PulsarClientException.CryptoException { + final String protocolAndHostPort = "ws://" + webSocketProxyHost + ":" + webSocketProxyPort; + + // Build the URL for producer. + final StringBuilder consumerUri = new StringBuilder(protocolAndHostPort) + .append("/ws/v2/consumer/persistent/") + .append(topicName) + .append("/") + .append(subscriptionName) + .append("?") + .append("subscriptionType=").append(subscriptionType.toString()) + .append("&").append("cryptoFailureAction=CONSUME"); + return URI.create(consumerUri.toString()); + } + + public synchronized ConsumerMessage receive(int timeout, TimeUnit unit) throws Exception { + ConsumerMessage msg = incomingMessages.poll(timeout, unit); + return msg; + } + + @Override + public void onWebSocketClose(int statusCode, String reason) { + log.info("Connection closed: {} - {}", statusCode, reason); + this.session = null; + } + + @Override + public void onWebSocketConnect(Session session) { + log.info("Got connect: {}", session); + this.session = session; + } + + @Override + public void onWebSocketError(Throwable cause) { + log.error("Received an error", cause); + } + + @Override + public void onWebSocketText(String text) { + + try { + ConsumerMessage msg = + ObjectMapperFactory.getMapper().reader().readValue(text, ConsumerMessage.class); + if (msg.messageId == null) { + log.error("Consumer[{}-{}] Could not extract the response payload: {}", topicName, subscriptionName, + text); + return; + } + // Decrypt. + byte[] decryptedPayload = WssClientSideEncryptUtils.decryptMsgPayload(msg.payload, msg.encryptionContext, + cryptoKeyReader, msgCrypto); + // Un-compression if needed. + byte[] unCompressedPayload = WssClientSideEncryptUtils.unCompressionIfNeeded(decryptedPayload, + msg.encryptionContext); + // Extract batch messages if needed. + if (msg.encryptionContext.getBatchSize().isPresent()) { + List singleMsgs = WssClientSideEncryptUtils.extractBatchMessagesIfNeeded( + unCompressedPayload, msg.encryptionContext); + for (ConsumerMessage singleMsg : singleMsgs) { + incomingMessages.add(singleMsg); + } + } else { + msg.payload = new String(unCompressedPayload, StandardCharsets.UTF_8); + incomingMessages.add(msg); + } + } catch (Exception ex) { + log.error("Consumer[{}-{}] Could not extract the response payload: {}", topicName, subscriptionName, text); + } + } + + @Override + public void close() throws IOException { + try { + wssClient.stop(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ClientSideEncryptionWssProducer.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ClientSideEncryptionWssProducer.java new file mode 100644 index 0000000000000..f9ae6cd4344e0 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ClientSideEncryptionWssProducer.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.websocket.proxy; + +import static org.testng.Assert.assertTrue; +import static org.apache.pulsar.common.api.EncryptionContext.EncryptionKey; +import static org.apache.pulsar.websocket.proxy.WssClientSideEncryptUtils.EncryptedPayloadAndParam; +import java.io.Closeable; +import java.io.IOException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.client.api.CryptoKeyReader; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.impl.crypto.MessageCryptoBc; +import org.apache.pulsar.common.api.proto.CompressionType; +import org.apache.pulsar.common.api.proto.MessageIdData; +import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.apache.pulsar.websocket.data.ProducerMessage; +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.WebSocketAdapter; +import org.eclipse.jetty.websocket.api.annotations.WebSocket; +import org.eclipse.jetty.websocket.client.ClientUpgradeRequest; +import org.eclipse.jetty.websocket.client.WebSocketClient; + +@Slf4j +@WebSocket(maxTextMessageSize = 64 * 1024) +public class ClientSideEncryptionWssProducer extends WebSocketAdapter implements Closeable { + + private Session session; + private volatile CompletableFuture sendFuture; + private final ScheduledExecutorService executor; + private final CryptoKeyReader cryptoKeyReader; + private final String topicName; + private final String producerName; + private final String webSocketProxyHost; + private final int webSocketProxyPort; + private final String keyName; + private WebSocketClient wssClient; + private final MessageCryptoBc msgCrypto; + + public ClientSideEncryptionWssProducer(String webSocketProxyHost, int webSocketProxyPort, String topicName, + String producerName, CryptoKeyReader cryptoKeyReader, String keyName, + ScheduledExecutorService executor) { + this.webSocketProxyHost = webSocketProxyHost; + this.webSocketProxyPort = webSocketProxyPort; + this.topicName = topicName; + this.producerName = producerName; + this.msgCrypto = new MessageCryptoBc("[" + topicName + "] [" + producerName + "]", true); + this.cryptoKeyReader = cryptoKeyReader; + this.keyName = keyName; + this.executor = executor; + } + + public void start() throws Exception { + wssClient = new WebSocketClient(); + wssClient.start(); + session = wssClient.connect(this, buildConnectURL(), new ClientUpgradeRequest()).get(); + assertTrue(session.isOpen()); + } + + private URI buildConnectURL() throws PulsarClientException.CryptoException { + final String protocolAndHostPort = "ws://" + webSocketProxyHost + ":" + webSocketProxyPort; + + // Encode encrypted public key data. + final byte[] keyValue = WssClientSideEncryptUtils.calculateEncryptedKeyValue(msgCrypto, cryptoKeyReader, + keyName); + EncryptionKey encryptionKey = new EncryptionKey(); + encryptionKey.setKeyValue(keyValue); + encryptionKey.setMetadata(cryptoKeyReader.getPublicKey(keyName, Collections.emptyMap()).getMetadata()); + Map encryptionKeyMap = new HashMap<>(); + encryptionKeyMap.put(keyName, encryptionKey); + + final String encryptionKeys = + WssClientSideEncryptUtils.toJSONAndBase64AndUrlEncode(encryptionKeyMap); + + // Build the URL for producer. + final StringBuilder producerUrL = new StringBuilder(protocolAndHostPort) + .append("/ws/v2/producer/persistent/") + .append(topicName) + .append("?") + .append("encryptionKeys=").append(encryptionKeys); + return URI.create(producerUrL.toString()); + } + + public synchronized MessageIdData sendMessage(ProducerMessage msg) throws Exception { + if (sendFuture != null && !sendFuture.isDone() && !sendFuture.isCancelled()) { + throw new IllegalArgumentException("There is a message still in sending."); + } + if (msg.payload == null) { + throw new IllegalArgumentException("Null value message is not supported."); + } + // Compression. + byte[] unCompressedPayload = msg.payload.getBytes(StandardCharsets.UTF_8); + byte[] compressedPayload = WssClientSideEncryptUtils.compressionIfNeeded(msg.compressionType, + unCompressedPayload); + if (msg.compressionType != null && !CompressionType.NONE.equals(msg.compressionType)) { + msg.uncompressedMessageSize = unCompressedPayload.length; + } + // Encrypt. + EncryptedPayloadAndParam encryptedPayloadAndParam = WssClientSideEncryptUtils.encryptPayload( + cryptoKeyReader, msgCrypto, compressedPayload, keyName); + msg.payload = encryptedPayloadAndParam.encryptedPayload; + msg.encryptionParam = encryptedPayloadAndParam.encryptionParam; + // Do send. + sendFuture = new CompletableFuture<>(); + String jsonMsg = ObjectMapperFactory.getMapper().writer().writeValueAsString(msg); + this.session.getRemote().sendString(jsonMsg); + // Wait for response. + executor.schedule(() -> { + synchronized (ClientSideEncryptionWssProducer.this) { + if (!sendFuture.isDone() && !sendFuture.isCancelled()) { + sendFuture.completeExceptionally(new TimeoutException("Send timeout")); + } + } + }, 50, TimeUnit.SECONDS); + return sendFuture.get(); + } + + @Override + public void onWebSocketClose(int statusCode, String reason) { + log.info("Connection closed: {} - {}", statusCode, reason); + this.session = null; + if (!sendFuture.isDone() && !sendFuture.isCancelled()) { + sendFuture.completeExceptionally(new RuntimeException("Connection was closed")); + } + } + + @Override + public void onWebSocketConnect(Session session) { + log.info("Got connect: {}", session); + this.session = session; + } + + @Override + public void onWebSocketError(Throwable cause) { + log.error("Received an error", cause); + } + + @Override + public void onWebSocketText(String text) { + try { + ResponseOfSend responseOfSend = + ObjectMapperFactory.getMapper().reader().readValue(text, ResponseOfSend.class); + if (responseOfSend.getErrorCode() != 0 || responseOfSend.getErrorMsg() != null) { + sendFuture.completeExceptionally(new RuntimeException(text)); + } else { + byte[] bytes = Base64.getDecoder().decode(responseOfSend.getMessageId()); + MessageIdData messageIdData = new MessageIdData(); + messageIdData.parseFrom(bytes); + sendFuture.complete(messageIdData); + } + } catch (Exception ex) { + log.error("Could not extract the response payload: {}", text); + } + } + + @Override + public void close() throws IOException { + try { + wssClient.stop(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Data + public static class ResponseOfSend { + private String result; + private String messageId; + private String errorMsg; + private int errorCode = -1; + private int schemaVersion; + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/CryptoKeyReaderForTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/CryptoKeyReaderForTest.java new file mode 100644 index 0000000000000..ff9cde766dfda --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/CryptoKeyReaderForTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.websocket.proxy; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import org.apache.pulsar.client.api.CryptoKeyReader; +import org.apache.pulsar.client.api.EncryptionKeyInfo; +import org.testng.Assert; + +public class CryptoKeyReaderForTest implements CryptoKeyReader { + + public static final Map RANDOM_METADATA = new HashMap<>(); + + static { + RANDOM_METADATA.put(UUID.randomUUID().toString(), UUID.randomUUID().toString()); + RANDOM_METADATA.put(UUID.randomUUID().toString(), UUID.randomUUID().toString()); + RANDOM_METADATA.put(UUID.randomUUID().toString(), UUID.randomUUID().toString()); + } + + @Override + public EncryptionKeyInfo getPublicKey(String keyName, Map metadata) { + EncryptionKeyInfo keyInfo = new EncryptionKeyInfo(); + String CERT_FILE_PATH = "./src/test/resources/certificate/public-key." + keyName; + if (Files.isReadable(Paths.get(CERT_FILE_PATH))) { + try { + keyInfo.setKey(Files.readAllBytes(Paths.get(CERT_FILE_PATH))); + // The metadata is meaningless, just to test that it can be transferred to the consumer. + keyInfo.setMetadata(RANDOM_METADATA); + return keyInfo; + } catch (IOException e) { + Assert.fail("Failed to read certificate from " + CERT_FILE_PATH); + } + } else { + Assert.fail("Certificate file " + CERT_FILE_PATH + " is not present or not readable."); + } + return null; + } + + @Override + public EncryptionKeyInfo getPrivateKey(String keyName, Map metadata) { + EncryptionKeyInfo keyInfo = new EncryptionKeyInfo(); + String CERT_FILE_PATH = "./src/test/resources/certificate/private-key." + keyName; + if (Files.isReadable(Paths.get(CERT_FILE_PATH))) { + try { + keyInfo.setKey(Files.readAllBytes(Paths.get(CERT_FILE_PATH))); + keyInfo.setMetadata(RANDOM_METADATA); + return keyInfo; + } catch (IOException e) { + Assert.fail("Failed to read certificate from " + CERT_FILE_PATH); + } + } else { + Assert.fail("Certificate file " + CERT_FILE_PATH + " is not present or not readable."); + } + return null; + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthenticationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthenticationTest.java index e4d7b8349ec2d..3f34857f59e8b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthenticationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthenticationTest.java @@ -83,7 +83,7 @@ public void setup() throws Exception { } service = spyWithClassAndConstructorArgs(WebSocketService.class, config); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(service) + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(service) .createConfigMetadataStore(anyString(), anyInt(), anyBoolean()); proxyServer = new ProxyServer(config); WebSocketServiceStarter.start(proxyServer, service); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java index a3b26a4a9d122..2d00e15a13f19 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java @@ -55,6 +55,7 @@ public ProxyAuthorizationTest() { @Override protected void setup() throws Exception { conf.setClusterName(configClusterName); + conf.setForceDeleteNamespaceAllowed(true); internalSetup(); WebSocketProxyConfiguration config = new WebSocketProxyConfiguration(); @@ -65,7 +66,7 @@ protected void setup() throws Exception { config.setWebServicePort(Optional.of(0)); config.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); service = spyWithClassAndConstructorArgs(WebSocketService.class, config); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(service) + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(service) .createConfigMetadataStore(anyString(), anyInt(), anyBoolean()); service.start(); } @@ -99,8 +100,9 @@ public void test() throws Exception { assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); assertTrue(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); - admin.topics().grantPermission("persistent://p1/c1/ns1/ds2", "other-role", - EnumSet.of(AuthAction.consume)); + String topic = "persistent://p1/c1/ns1/ds2"; + admin.topics().createNonPartitionedTopic(topic); + admin.topics().grantPermission(topic, "other-role", EnumSet.of(AuthAction.consume)); waitForChange(); assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds2"), "other-role", null)); @@ -117,7 +119,7 @@ public void test() throws Exception { assertTrue(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null, null)); - admin.namespaces().deleteNamespace("p1/c1/ns1"); + admin.namespaces().deleteNamespace("p1/c1/ns1", true); admin.tenants().deleteTenant("p1"); admin.clusters().deleteCluster("c1"); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyConfigurationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyConfigurationTest.java index 173948ab1be5b..85f512e15670e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyConfigurationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyConfigurationTest.java @@ -68,7 +68,7 @@ public void configTest(int numIoThreads, int connectionsPerBroker) throws Except config.setServiceUrl("http://localhost:8080"); config.getProperties().setProperty("brokerClient_lookupTimeoutMs", "100"); WebSocketService service = spyWithClassAndConstructorArgs(WebSocketService.class, config); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(service) + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(service) .createConfigMetadataStore(anyString(), anyInt(), anyBoolean()); service.start(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyEncryptionPublishConsumeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyEncryptionPublishConsumeTest.java index 041107e5e93d7..5234ca0057875 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyEncryptionPublishConsumeTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyEncryptionPublishConsumeTest.java @@ -73,8 +73,8 @@ public void setup() throws Exception { config.setClusterName("test"); config.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); config.setCryptoKeyReaderFactoryClassName(CryptoKeyReaderFactoryImpl.class.getName()); - WebSocketService service = spy(new WebSocketService(config)); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(service) + service = spy(new WebSocketService(config)); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(service) .createConfigMetadataStore(anyString(), anyInt(), anyBoolean()); proxyServer = new ProxyServer(config); WebSocketServiceStarter.start(proxyServer, service); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyIdleTimeoutTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyIdleTimeoutTest.java index ab5a43b115ab1..6c9c5deb0c3ed 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyIdleTimeoutTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyIdleTimeoutTest.java @@ -65,7 +65,7 @@ public void setup() throws Exception { config.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); config.setWebSocketSessionIdleTimeoutMillis(3 * 1000); service = spyWithClassAndConstructorArgs(WebSocketService.class, config); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(service) + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(service) .createConfigMetadataStore(anyString(), anyInt(), anyBoolean()); proxyServer = new ProxyServer(config); WebSocketServiceStarter.start(proxyServer, service); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPingTest.java index 8ba9283138926..b4ecb84f580b5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPingTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPingTest.java @@ -67,7 +67,7 @@ public void setup() throws Exception { config.setWebSocketSessionIdleTimeoutMillis(3 * 1000); config.setWebSocketPingDurationSeconds(2); service = spyWithClassAndConstructorArgs(WebSocketService.class, config); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(service) + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(service) .createConfigMetadataStore(anyString(), anyInt(), anyBoolean()); proxyServer = new ProxyServer(config); WebSocketServiceStarter.start(proxyServer, service); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeClientSideEncryptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeClientSideEncryptionTest.java new file mode 100644 index 0000000000000..d81c39be28487 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeClientSideEncryptionTest.java @@ -0,0 +1,363 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.websocket.proxy; + +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import static org.testng.Assert.assertEquals; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.CryptoKeyReader; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.common.api.proto.CompressionType; +import org.apache.pulsar.common.api.proto.MessageIdData; +import org.apache.pulsar.metadata.impl.ZKMetadataStore; +import org.apache.pulsar.websocket.WebSocketService; +import org.apache.pulsar.websocket.data.ConsumerMessage; +import org.apache.pulsar.websocket.data.ProducerMessage; +import org.apache.pulsar.websocket.service.ProxyServer; +import org.apache.pulsar.websocket.service.WebSocketProxyConfiguration; +import org.apache.pulsar.websocket.service.WebSocketServiceStarter; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "websocket") +public class ProxyPublishConsumeClientSideEncryptionTest extends ProducerConsumerBase { + private static final int TIME_TO_CHECK_BACKLOG_QUOTA = 5; + private ScheduledExecutorService executor; + private static final Charset charset = Charset.defaultCharset(); + + private ProxyServer proxyServer; + private WebSocketService service; + + @BeforeClass + public void setup() throws Exception { + executor = Executors.newScheduledThreadPool(1); + + conf.setBacklogQuotaCheckIntervalInSeconds(TIME_TO_CHECK_BACKLOG_QUOTA); + + super.internalSetup(); + super.producerBaseSetup(); + + WebSocketProxyConfiguration config = new WebSocketProxyConfiguration(); + config.setWebServicePort(Optional.of(0)); + config.setClusterName("test"); + config.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); + service = spy(new WebSocketService(config)); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(service) + .createConfigMetadataStore(anyString(), anyInt(), anyBoolean()); + proxyServer = new ProxyServer(config); + WebSocketServiceStarter.start(proxyServer, service); + log.info("Proxy Server Started"); + } + + @AfterClass(alwaysRun = true) + protected void cleanup() throws Exception { + super.internalCleanup(); + if (service != null) { + service.close(); + } + if (proxyServer != null) { + proxyServer.stop(); + } + executor.shutdownNow(); + log.info("Finished Cleaning Up Test setup"); + } + + @DataProvider(name = "encryptKeyNames") + public Object[][] encryptKeyNames() { + return new Object[][]{ + {"client-ecdsa.pem"}, + {"client-rsa.pem"} + }; + } + + @Test(dataProvider = "encryptKeyNames") + public void testWssSendAndJavaConsumeWithEncryption(String keyName) throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("public/default/tp_"); + final String subscriptionName = "s1"; + final String producerName = "wss-p1"; + admin.topics().createNonPartitionedTopic(topicName); + admin.topics().createSubscription(topicName, subscriptionName, MessageId.earliest); + + // Create wss producer. + final String webSocketProxyHost = "localhost"; + final int webSocketProxyPort = proxyServer.getListenPortHTTP().get(); + final CryptoKeyReader cryptoKeyReader = new CryptoKeyReaderForTest(); + ClientSideEncryptionWssProducer producer = new ClientSideEncryptionWssProducer(webSocketProxyHost, + webSocketProxyPort, topicName, producerName, cryptoKeyReader, keyName, executor); + producer.start(); + + // Send message. + String msgPayloadBeforeEncrypt = "msg-123"; + ProducerMessage messageSent = new ProducerMessage(); + messageSent.key = "k"; + messageSent.payload = msgPayloadBeforeEncrypt; + MessageIdData messageIdData = producer.sendMessage(messageSent); + log.info("send success: {}", messageIdData.toString()); + + // Consume. + Consumer consumer = pulsarClient.newConsumer().cryptoKeyReader(cryptoKeyReader) + .topic(topicName).subscriptionName(subscriptionName) + .subscribe(); + Message msgReceived = consumer.receive(2, TimeUnit.SECONDS); + assertEquals(new String(msgReceived.getData(), charset), msgPayloadBeforeEncrypt); + + // cleanup. + producer.close(); + consumer.close(); + admin.topics().delete(topicName); + } + + @DataProvider(name = "compressionTypes") + public Object[][] compressionTypes() { + return new Object[][]{ + {CompressionType.NONE}, + {CompressionType.LZ4}, + {CompressionType.ZLIB}, + {CompressionType.SNAPPY}, + {CompressionType.ZSTD} + }; + } + + @Test(dataProvider = "compressionTypes") + public void testWssSendAndJavaConsumeWithEncryptionAndCompression(CompressionType compressionType) + throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("public/default/tp_"); + final String subscriptionName = "s1"; + final String producerName = "wss-p1"; + final String keyName = "client-ecdsa.pem"; + admin.topics().createNonPartitionedTopic(topicName); + admin.topics().createSubscription(topicName, subscriptionName, MessageId.earliest); + + // Create wss producer. + final String webSocketProxyHost = "localhost"; + final int webSocketProxyPort = proxyServer.getListenPortHTTP().get(); + final CryptoKeyReader cryptoKeyReader = new CryptoKeyReaderForTest(); + ClientSideEncryptionWssProducer producer = new ClientSideEncryptionWssProducer(webSocketProxyHost, + webSocketProxyPort, topicName, producerName, cryptoKeyReader, keyName, executor); + producer.start(); + + // Send message. + String originalPayload = "msg-123"; + ProducerMessage messageSent = new ProducerMessage(); + messageSent.key = "k"; + messageSent.payload = originalPayload; + messageSent.compressionType = compressionType; + MessageIdData messageIdData = producer.sendMessage(messageSent); + log.info("send success: {}", messageIdData.toString()); + + // Consume. + Consumer consumer = pulsarClient.newConsumer().cryptoKeyReader(cryptoKeyReader) + .topic(topicName).subscriptionName(subscriptionName) + .subscribe(); + Message msgReceived = consumer.receive(2, TimeUnit.SECONDS); + assertEquals(new String(msgReceived.getData(), charset), originalPayload); + + // cleanup. + producer.close(); + consumer.close(); + admin.topics().delete(topicName); + } + + @Test(dataProvider = "encryptKeyNames") + public void testJavaSendAndWssConsumeWithEncryption(String keyName) throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("public/default/tp_"); + final String subscriptionName = "s1"; + admin.topics().createNonPartitionedTopic(topicName); + admin.topics().createSubscription(topicName, subscriptionName, MessageId.earliest); + final CryptoKeyReader cryptoKeyReader = new CryptoKeyReaderForTest(); + + final String originalPayload = "msg-123"; + Producer producer = pulsarClient.newProducer() + .topic(topicName).addEncryptionKey(keyName) + .cryptoKeyReader(cryptoKeyReader).create(); + producer.send(originalPayload.getBytes(StandardCharsets.UTF_8)); + + // Create wss consumer. + final String webSocketProxyHost = "localhost"; + final int webSocketProxyPort = proxyServer.getListenPortHTTP().get(); + + ClientSideEncryptionWssConsumer consumer = new ClientSideEncryptionWssConsumer(webSocketProxyHost, + webSocketProxyPort, topicName, subscriptionName, SubscriptionType.Shared, cryptoKeyReader); + consumer.start(); + + // Receive message. + ConsumerMessage message = consumer.receive(2, TimeUnit.SECONDS); + assertEquals(message.payload, originalPayload); + + // cleanup. + producer.close(); + consumer.close(); + admin.topics().delete(topicName); + } + + @DataProvider(name = "compressionTypesForJ") + public Object[][] compressionTypesForJ() { + return new Object[][]{ + {org.apache.pulsar.client.api.CompressionType.NONE}, + {org.apache.pulsar.client.api.CompressionType.LZ4}, + {org.apache.pulsar.client.api.CompressionType.ZLIB}, + {org.apache.pulsar.client.api.CompressionType.SNAPPY}, + {org.apache.pulsar.client.api.CompressionType.ZSTD} + }; + } + + @Test(dataProvider = "compressionTypesForJ") + public void testJavaSendAndWssConsumeWithEncryptionAndCompression(org.apache.pulsar.client.api.CompressionType + compressionType) + throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("public/default/tp_"); + final String subscriptionName = "s1"; + admin.topics().createNonPartitionedTopic(topicName); + admin.topics().createSubscription(topicName, subscriptionName, MessageId.earliest); + final CryptoKeyReader cryptoKeyReader = new CryptoKeyReaderForTest(); + final String keyName = "client-ecdsa.pem"; + + final String originalPayload = "msg-123"; + Producer producer = pulsarClient.newProducer() + .topic(topicName).addEncryptionKey(keyName).compressionType(compressionType) + .cryptoKeyReader(cryptoKeyReader).create(); + producer.send(originalPayload.getBytes(StandardCharsets.UTF_8)); + + // Create wss consumer. + final String webSocketProxyHost = "localhost"; + final int webSocketProxyPort = proxyServer.getListenPortHTTP().get(); + + ClientSideEncryptionWssConsumer consumer = new ClientSideEncryptionWssConsumer(webSocketProxyHost, + webSocketProxyPort, topicName, subscriptionName, SubscriptionType.Shared, cryptoKeyReader); + consumer.start(); + + // Receive message. + ConsumerMessage message = consumer.receive(2, TimeUnit.SECONDS); + assertEquals(message.payload, originalPayload); + + // cleanup. + producer.close(); + consumer.close(); + admin.topics().delete(topicName); + } + + @Test + public void testJavaSendAndWssConsumeWithEncryptionAndCompressionAndBatch() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("public/default/tp_"); + final String subscriptionName = "s1"; + admin.topics().createNonPartitionedTopic(topicName); + admin.topics().createSubscription(topicName, subscriptionName, MessageId.earliest); + final CryptoKeyReader cryptoKeyReader = new CryptoKeyReaderForTest(); + final String keyName = "client-ecdsa.pem"; + + final HashSet messagesSent = new HashSet<>(); + Producer producer = pulsarClient.newProducer().enableBatching(true) + .batchingMaxMessages(1000) + .batchingMaxPublishDelay(1, TimeUnit.HOURS) + .topic(topicName).addEncryptionKey(keyName) + .compressionType(org.apache.pulsar.client.api.CompressionType.LZ4) + .cryptoKeyReader(cryptoKeyReader).create(); + for (int i = 0; i < 10; i++) { + String payload = "msg-" + i; + messagesSent.add(payload); + producer.sendAsync(payload.getBytes(StandardCharsets.UTF_8)); + } + producer.flush(); + + // Create wss consumer. + final String webSocketProxyHost = "localhost"; + final int webSocketProxyPort = proxyServer.getListenPortHTTP().get(); + + ClientSideEncryptionWssConsumer consumer = new ClientSideEncryptionWssConsumer(webSocketProxyHost, + webSocketProxyPort, topicName, subscriptionName, SubscriptionType.Shared, cryptoKeyReader); + consumer.start(); + + // Receive message. + final HashSet messagesReceived = new HashSet<>(); + while (true) { + ConsumerMessage message = consumer.receive(2, TimeUnit.SECONDS); + if (message == null) { + break; + } + messagesReceived.add(message.payload); + } + assertEquals(messagesReceived, messagesSent); + + // cleanup. + producer.close(); + consumer.close(); + admin.topics().delete(topicName); + } + + + + @Test(dataProvider = "encryptKeyNames") + public void testWssSendAndWssConsumeWithEncryption(String keyName) throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("public/default/tp_"); + final String subscriptionName = "s1"; + final String producerName = "wss-p1"; + admin.topics().createNonPartitionedTopic(topicName); + admin.topics().createSubscription(topicName, subscriptionName, MessageId.earliest); + + // Create wss producer. + final String webSocketProxyHost = "localhost"; + final int webSocketProxyPort = proxyServer.getListenPortHTTP().get(); + final CryptoKeyReader cryptoKeyReader = new CryptoKeyReaderForTest(); + ClientSideEncryptionWssProducer producer = new ClientSideEncryptionWssProducer(webSocketProxyHost, + webSocketProxyPort, topicName, producerName, cryptoKeyReader, keyName, executor); + producer.start(); + + // Send message. + String msgPayloadBeforeEncrypt = "msg-123"; + ProducerMessage messageSent = new ProducerMessage(); + messageSent.key = "k"; + messageSent.payload = msgPayloadBeforeEncrypt; + MessageIdData messageIdData = producer.sendMessage(messageSent); + log.info("send success: {}", messageIdData.toString()); + + // Consume. + ClientSideEncryptionWssConsumer consumer = new ClientSideEncryptionWssConsumer(webSocketProxyHost, + webSocketProxyPort, topicName, subscriptionName, SubscriptionType.Shared, cryptoKeyReader); + consumer.start(); + ConsumerMessage msgReceived = consumer.receive(2, TimeUnit.SECONDS); + assertEquals(msgReceived.payload, msgPayloadBeforeEncrypt); + assertEquals(msgReceived.encryptionContext.getKeys().get(keyName).getMetadata(), + CryptoKeyReaderForTest.RANDOM_METADATA); + + // cleanup. + producer.close(); + consumer.close(); + admin.topics().delete(topicName); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTest.java index 8c64e40f927c4..9ec6a7daf7234 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTest.java @@ -100,7 +100,7 @@ public void setup() throws Exception { config.setClusterName("test"); config.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); service = spyWithClassAndConstructorArgs(WebSocketService.class, config); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(service) + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(service) .createConfigMetadataStore(anyString(), anyInt(), anyBoolean()); proxyServer = new ProxyServer(config); WebSocketServiceStarter.start(proxyServer, service); @@ -886,6 +886,7 @@ public void ackBatchMessageTest() throws Exception { WebSocketClient consumerClient = new WebSocketClient(); SimpleConsumerSocket consumeSocket = new SimpleConsumerSocket(); + @Cleanup Producer producer = pulsarClient.newProducer() .topic(topic) .batchingMaxPublishDelay(1, TimeUnit.SECONDS) @@ -933,6 +934,7 @@ public void consumeEncryptedMessages() throws Exception { final String rsaPublicKeyData = "data:application/x-pem-file;base64,LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF0S1d3Z3FkblRZck9DditqMU1rVApXZlNIMHdDc0haWmNhOXdBVzNxUDR1dWhsQnZuYjEwSmNGZjVaanpQOUJTWEsrdEhtSTh1b04zNjh2RXY2eWhVClJITTR5dVhxekN4enVBd2tRU28zOXJ6WDhQR0M3cWRqQ043TERKM01ucWlCSXJVc1NhRVAxd3JOc0Ixa0krbzkKRVIxZTVPL3VFUEFvdFA5MzNoSFEwSjJoTUVla0hxTDdzQmxKOThoNk5tc2ljRWFVa2FyZGswVE9YcmxrakMrYwpNZDhaYkdTY1BxSTlNMzhibW4zT0x4RlRuMXZ0aHB2blhMdkNtRzRNKzZ4dFl0RCtucGNWUFp3MWkxUjkwZk1zCjdwcFpuUmJ2OEhjL0RGZE9LVlFJZ2FtNkNEZG5OS2dXN2M3SUJNclAwQUVtMzdIVHUwTFNPalAyT0hYbHZ2bFEKR1FJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg=="; final String rsaPrivateKeyData = "data:application/x-pem-file;base64,LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdEtXd2dxZG5UWXJPQ3YrajFNa1RXZlNIMHdDc0haWmNhOXdBVzNxUDR1dWhsQnZuCmIxMEpjRmY1Wmp6UDlCU1hLK3RIbUk4dW9OMzY4dkV2NnloVVJITTR5dVhxekN4enVBd2tRU28zOXJ6WDhQR0MKN3FkakNON0xESjNNbnFpQklyVXNTYUVQMXdyTnNCMWtJK285RVIxZTVPL3VFUEFvdFA5MzNoSFEwSjJoTUVlawpIcUw3c0JsSjk4aDZObXNpY0VhVWthcmRrMFRPWHJsa2pDK2NNZDhaYkdTY1BxSTlNMzhibW4zT0x4RlRuMXZ0Cmhwdm5YTHZDbUc0TSs2eHRZdEQrbnBjVlBadzFpMVI5MGZNczdwcFpuUmJ2OEhjL0RGZE9LVlFJZ2FtNkNEZG4KTktnVzdjN0lCTXJQMEFFbTM3SFR1MExTT2pQMk9IWGx2dmxRR1FJREFRQUJBb0lCQUFhSkZBaTJDN3UzY05yZgpBc3RZOXZWRExvTEl2SEZabGtCa3RqS1pEWW1WSXNSYitoU0NWaXdWVXJXTEw2N1I2K0l2NGVnNERlVE9BeDAwCjhwbmNYS2daVHcyd0liMS9RalIvWS9SamxhQzhsa2RtUldsaTd1ZE1RQ1pWc3lodVNqVzZQajd2cjhZRTR3b2oKRmhOaWp4RUdjZjl3V3JtTUpyemRuVFdRaVhCeW8rZVR2VVE5QlBnUEdyUmpzTVptVGtMeUFWSmZmMkRmeE81YgpJV0ZEWURKY3lZQU1DSU1RdTd2eXMvSTUwb3U2aWxiMUNPNlFNNlo3S3BQZU9vVkZQd3R6Ymg4Y2Y5eE04VU5TCmo2Si9KbWRXaGdJMzRHUzNOQTY4eFRRNlBWN3pqbmhDYytpY2NtM0pLeXpHWHdhQXBBWitFb2NlLzlqNFdLbXUKNUI0emlSMENnWUVBM2wvOU9IYmwxem15VityUnhXT0lqL2kyclR2SHp3Qm5iblBKeXVlbUw1Vk1GZHBHb2RRMwp2d0h2eVFtY0VDUlZSeG1Yb2pRNFF1UFBIczNxcDZ3RUVGUENXeENoTFNUeGxVYzg1U09GSFdVMk85OWpWN3pJCjcrSk9wREsvTXN0c3g5bkhnWGR1SkYrZ2xURnRBM0xIOE9xeWx6dTJhRlBzcHJ3S3VaZjk0UThDZ1lFQXovWngKYWtFRytQRU10UDVZUzI4Y1g1WGZqc0lYL1YyNkZzNi9zSDE2UWpVSUVkZEU1VDRmQ3Vva3hDalNpd1VjV2htbApwSEVKNVM1eHAzVllSZklTVzNqUlczcXN0SUgxdHBaaXBCNitTMHpUdUptTEpiQTNJaVdFZzJydE10N1gxdUp2CkEvYllPcWUwaE9QVHVYdVpkdFZaMG5NVEtrN0dHOE82VmtCSTdGY0NnWUVBa0RmQ21zY0pnczdKYWhsQldIbVgKekg5cHdlbStTUEtqSWMvNE5CNk4rZGdpa3gyUHAwNWhwUC9WaWhVd1lJdWZ2cy9MTm9nVllOUXJ0SGVwVW5yTgoyK1RtYkhiWmdOU3YxTGR4dDgyVWZCN3kwRnV0S3U2bGhtWEh5TmVjaG8zRmk4c2loMFYwYWlTV21ZdUhmckFICkdhaXNrRVpLbzFpaVp2UVhKSXg5TzJNQ2dZQVRCZjByOWhUWU10eXh0YzZIMy9zZGQwMUM5dGhROGdEeTB5alAKMFRxYzBkTVNKcm9EcW1JV2tvS1lldzkvYmhGQTRMVzVUQ25Xa0NBUGJIbU50RzRmZGZiWXdta0gvaGRuQTJ5MApqS2RscGZwOEdYZVVGQUdIR3gxN0ZBM3NxRnZnS1VoMGVXRWdSSFVMN3ZkUU1WRkJnSlM5M283elFNOTRmTGdQCjZjT0I4d0tCZ0ZjR1Y0R2pJMld3OWNpbGxhQzU1NE12b1NqZjhCLyswNGtYekRPaDhpWUlJek85RVVpbDFqaksKSnZ4cDRobkx6VEtXYnV4M01FV3F1ckxrWWFzNkdwS0JqdytpTk9DYXI2WWRxV0dWcU0zUlV4N1BUVWFad2tLeApVZFA2M0lmWTdpWkNJVC9RYnlIUXZJVWUyTWFpVm5IK3VseGRrSzZZNWU3Z3hjYmNrSUg0Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg=="; + @Cleanup Producer producer = pulsarClient.newProducer() .topic(topic) .enableBatching(false) @@ -1051,5 +1053,71 @@ private void stopWebSocketClient(WebSocketClient... clients) { log.info("proxy clients are stopped successfully"); } + @Test + public void testMultiTopics() throws Exception { + final String subscription1 = "my-sub1"; + final String subscription2 = "my-sub2"; + final String topic1 = "my-property/my-ns/testMultiTopics" + UUID.randomUUID(); + final String topic2 = "my-property/my-ns/testMultiTopics" + UUID.randomUUID(); + final String consumerUri1 = "ws://localhost:" + proxyServer.getListenPortHTTP().get() + + "/ws/v3/consumer/" + subscription1 + "?topics=" + topic1 + "," + topic2; + + final String consumerUri2 = "ws://localhost:" + proxyServer.getListenPortHTTP().get() + + "/ws/v3/consumer/" + subscription2 + "?topicsPattern=my-property/my-ns/testMultiTopics.*"; + + int messages = 10; + WebSocketClient consumerClient1 = new WebSocketClient(); + WebSocketClient consumerClient2 = new WebSocketClient(); + SimpleConsumerSocket consumeSocket1 = new SimpleConsumerSocket(); + SimpleConsumerSocket consumeSocket2 = new SimpleConsumerSocket(); + @Cleanup + Producer producer1 = pulsarClient.newProducer() + .topic(topic1) + .batchingMaxMessages(1) + .create(); + @Cleanup + Producer producer2 = pulsarClient.newProducer() + .topic(topic2) + .batchingMaxMessages(1) + .create(); + + try { + consumerClient1.start(); + consumerClient2.start(); + ClientUpgradeRequest consumerRequest1 = new ClientUpgradeRequest(); + ClientUpgradeRequest consumerRequest2 = new ClientUpgradeRequest(); + Future consumerFuture1 = consumerClient1.connect(consumeSocket1, URI.create(consumerUri1), consumerRequest1); + Future consumerFuture2 = consumerClient2.connect(consumeSocket2, URI.create(consumerUri2), consumerRequest2); + + assertTrue(consumerFuture1.get().isOpen()); + assertTrue(consumerFuture2.get().isOpen()); + assertEquals(consumeSocket1.getReceivedMessagesCount(), 0); + assertEquals(consumeSocket2.getReceivedMessagesCount(), 0); + + for (int i = 1; i <= messages; i ++) { + producer1.sendAsync(String.valueOf(i).getBytes(StandardCharsets.UTF_8)); + producer2.sendAsync(String.valueOf(i).getBytes(StandardCharsets.UTF_8)); + } + producer1.flush(); + producer2.flush(); + + consumeSocket1.sendPermits(2 * messages); + Awaitility.await().untilAsserted(() -> + assertEquals(consumeSocket1.getReceivedMessagesCount(), 2 * messages)); + Awaitility.await().untilAsserted(() -> + assertEquals(admin.topics().getStats(topic1).getSubscriptions() + .get(subscription1).getMsgBacklog(), 0)); + Awaitility.await().untilAsserted(() -> + assertEquals(admin.topics().getStats(topic2).getSubscriptions() + .get(subscription1).getMsgBacklog(), 0)); + + consumeSocket2.sendPermits(2 * messages); + Awaitility.await().untilAsserted(() -> + assertEquals(consumeSocket2.getReceivedMessagesCount(), 2 * messages)); + } finally { + stopWebSocketClient(consumerClient1, consumerClient2); + } + } + private static final Logger log = LoggerFactory.getLogger(ProxyPublishConsumeTest.class); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTlsTest.java index 91cd4fab470d6..dca4964fc987e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTlsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeTlsTest.java @@ -74,7 +74,7 @@ public void setup() throws Exception { config.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); config.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); service = spyWithClassAndConstructorArgs(WebSocketService.class, config); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(service) + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(service) .createConfigMetadataStore(anyString(), anyInt(), anyBoolean()); proxyServer = new ProxyServer(config); WebSocketServiceStarter.start(proxyServer, service); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeWithoutZKTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeWithoutZKTest.java index 0a432406001ad..c3e75bcb4f0ec 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeWithoutZKTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyPublishConsumeWithoutZKTest.java @@ -59,7 +59,7 @@ public void setup() throws Exception { config.setServiceUrl(pulsar.getSafeWebServiceAddress()); config.setServiceUrlTls(pulsar.getWebServiceAddressTls()); service = spyWithClassAndConstructorArgs(WebSocketService.class, config); - doReturn(new ZKMetadataStore(mockZooKeeper)).when(service) + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeper))).when(service) .createConfigMetadataStore(anyString(), anyInt(), anyBoolean()); proxyServer = new ProxyServer(config); WebSocketServiceStarter.start(proxyServer, service); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/WssClientSideEncryptUtils.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/WssClientSideEncryptUtils.java new file mode 100644 index 0000000000000..dbbfb756625a9 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/WssClientSideEncryptUtils.java @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.websocket.proxy; + +import static org.apache.pulsar.client.impl.crypto.MessageCryptoBc.ECDSA; +import static org.apache.pulsar.client.impl.crypto.MessageCryptoBc.ECIES; +import static org.apache.pulsar.client.impl.crypto.MessageCryptoBc.RSA; +import static org.apache.pulsar.client.impl.crypto.MessageCryptoBc.RSA_TRANS; +import com.fasterxml.jackson.core.JsonProcessingException; +import io.netty.buffer.ByteBuf; +import java.io.IOException; +import java.net.URLEncoder; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.security.PublicKey; +import java.security.spec.AlgorithmParameterSpec; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import javax.crypto.Cipher; +import javax.crypto.NoSuchPaddingException; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.client.api.CryptoKeyReader; +import org.apache.pulsar.client.api.EncryptionKeyInfo; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.impl.crypto.MessageCryptoBc; +import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; +import org.apache.pulsar.common.api.EncryptionContext; +import org.apache.pulsar.common.api.proto.CompressionType; +import org.apache.pulsar.common.api.proto.EncryptionKeys; +import org.apache.pulsar.common.api.proto.MessageMetadata; +import org.apache.pulsar.common.api.proto.SingleMessageMetadata; +import org.apache.pulsar.common.compression.CompressionCodec; +import org.apache.pulsar.common.compression.CompressionCodecProvider; +import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.apache.pulsar.websocket.data.ConsumerMessage; +import org.bouncycastle.jce.provider.BouncyCastleProvider; + +@Slf4j +public class WssClientSideEncryptUtils { + + public static Charset UTF8 = StandardCharsets.UTF_8; + + public static String base64AndUrlEncode(String str) { + return base64AndUrlEncode(str.getBytes(UTF8), UTF8); + } + + public static String base64AndUrlEncode(String str, Charset charset) { + return base64AndUrlEncode(str.getBytes(charset), charset); + } + + public static String base64Encode(String str, Charset charset) { + return Base64.getEncoder().encodeToString(str.getBytes(charset)); + } + + public static String base64Encode(String str) { + return Base64.getEncoder().encodeToString(str.getBytes(UTF8)); + } + + public static byte[] base64Decode(String str) { + return Base64.getDecoder().decode(str); + } + + public static String base64Encode(byte[] byteArray) { + return Base64.getEncoder().encodeToString(byteArray); + } + + public static String base64AndUrlEncode(byte[] byteArray) { + String base64Encode = Base64.getEncoder().encodeToString(byteArray); + return URLEncoder.encode(base64Encode, UTF8); + } + + public static String base64AndUrlEncode(byte[] byteArray, Charset charset) { + String base64Encode = Base64.getEncoder().encodeToString(byteArray); + return URLEncoder.encode(base64Encode, charset); + } + + public static String urlEncode(String str) { + return URLEncoder.encode(str, UTF8); + } + + public static String urlEncode(String str, Charset charset) { + return URLEncoder.encode(str, charset); + } + + public static byte[] calculateEncryptedKeyValue(MessageCryptoBc msgCrypto, CryptoKeyReader cryptoKeyReader, + String publicKeyName) + throws PulsarClientException.CryptoException { + EncryptionKeyInfo encryptionKeyInfo = cryptoKeyReader.getPublicKey(publicKeyName, Collections.emptyMap()); + return calculateEncryptedKeyValue(msgCrypto, encryptionKeyInfo.getKey()); + } + + public static String toJSONAndBase64AndUrlEncode(Object obj) + throws PulsarClientException.CryptoException { + try { + String json = ObjectMapperFactory.getMapper().getObjectMapper() + .writeValueAsString(obj); + return urlEncode(base64Encode(json)); + } catch (JsonProcessingException e) { + throw new PulsarClientException.CryptoException(String.format("Serialize object %s failed", obj)); + } + } + + public static byte[] calculateEncryptedKeyValue(MessageCryptoBc msgCrypto, byte[] publicKeyData) + throws PulsarClientException.CryptoException { + try { + PublicKey pubKey = MessageCryptoBc.loadPublicKey(publicKeyData); + Cipher dataKeyCipher = loadAndInitCipher(pubKey); + return dataKeyCipher.doFinal(msgCrypto.getDataKey().getEncoded()); + } catch (Exception e) { + log.error("Failed to encrypt data key. {}", e.getMessage()); + throw new PulsarClientException.CryptoException(e.getMessage()); + } + } + + private static Cipher loadAndInitCipher(PublicKey pubKey) throws PulsarClientException.CryptoException, + NoSuchAlgorithmException, NoSuchProviderException, NoSuchPaddingException, InvalidKeyException, + InvalidAlgorithmParameterException { + Cipher dataKeyCipher = null; + AlgorithmParameterSpec params = null; + // Encrypt data key using public key + if (RSA.equals(pubKey.getAlgorithm())) { + dataKeyCipher = Cipher.getInstance(RSA_TRANS, BouncyCastleProvider.PROVIDER_NAME); + } else if (ECDSA.equals(pubKey.getAlgorithm())) { + dataKeyCipher = Cipher.getInstance(ECIES, BouncyCastleProvider.PROVIDER_NAME); + params = MessageCryptoBc.createIESParameterSpec(); + } else { + String msg = "Unsupported key type " + pubKey.getAlgorithm(); + log.error(msg); + throw new PulsarClientException.CryptoException(msg); + } + if (params != null) { + dataKeyCipher.init(Cipher.ENCRYPT_MODE, pubKey, params); + } else { + dataKeyCipher.init(Cipher.ENCRYPT_MODE, pubKey); + } + return dataKeyCipher; + } + + public static byte[] compressionIfNeeded(CompressionType compressionType, byte[] payload) { + if (compressionType != null && !CompressionType.NONE.equals(compressionType)) { + CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType); + ByteBuf input = PulsarByteBufAllocator.DEFAULT.buffer(payload.length, payload.length); + input.writeBytes(payload); + ByteBuf output = codec.encode(input); + input.release(); + byte[] res = new byte[output.readableBytes()]; + output.readBytes(res); + output.release(); + return res; + } + return payload; + } + + public static EncryptedPayloadAndParam encryptPayload(CryptoKeyReader cryptoKeyReader, MessageCryptoBc msgCrypto, + byte[] payload, String keyName) + throws PulsarClientException { + ByteBuffer unEncryptedMessagePayload = ByteBuffer.wrap(payload); + ByteBuffer encryptedMessagePayload = ByteBuffer.allocate(unEncryptedMessagePayload.remaining() + 512); + MessageMetadata ignoredMessageMetadata = new MessageMetadata(); + msgCrypto.encrypt(Collections.singleton(keyName), cryptoKeyReader, + () -> ignoredMessageMetadata, unEncryptedMessagePayload, encryptedMessagePayload); + byte[] res = new byte[encryptedMessagePayload.remaining()]; + encryptedMessagePayload.get(res); + return new EncryptedPayloadAndParam(WssClientSideEncryptUtils.base64Encode(res), + WssClientSideEncryptUtils.base64Encode(ignoredMessageMetadata.getEncryptionParam())); + } + + @AllArgsConstructor + public static class EncryptedPayloadAndParam { + public final String encryptedPayload; + public final String encryptionParam; + } + + public static byte[] decryptMsgPayload(String payloadString, EncryptionContext encryptionContext, + CryptoKeyReader cryptoKeyReader, MessageCryptoBc msgCrypto) { + byte[] payload = base64Decode(payloadString); + if (encryptionContext == null) { + return payload; + } + + MessageMetadata messageMetadata = new MessageMetadata(); + Map encKeys = encryptionContext.getKeys(); + for (Map.Entry entry : encKeys.entrySet()) { + EncryptionKeys encryptionKeys = messageMetadata.addEncryptionKey() + .setKey(entry.getKey()).setValue(entry.getValue().getKeyValue()); + if (entry.getValue().getMetadata() != null) { + for (Map.Entry prop : entry.getValue().getMetadata().entrySet()) { + encryptionKeys.addMetadata().setKey(prop.getKey()).setValue(prop.getValue()); + } + } + } + messageMetadata.setEncryptionParam(encryptionContext.getParam()); + + // Create input and output. + ByteBuffer input = ByteBuffer.allocate(payload.length); + ByteBuffer output = ByteBuffer.allocate(msgCrypto.getMaxOutputSize(payload.length)); + input.put(payload); + input.flip(); + + // Decrypt. + msgCrypto.decrypt(() -> messageMetadata, input, output, cryptoKeyReader); + byte[] res = new byte[output.limit()]; + output.get(res); + return res; + } + + public static byte[] unCompressionIfNeeded(byte[] payloadBytes, EncryptionContext encryptionContext) throws IOException { + if (encryptionContext.getCompressionType() != null && !org.apache.pulsar.client.api.CompressionType.NONE + .equals(encryptionContext.getCompressionType())) { + CompressionCodec codec = + CompressionCodecProvider.getCompressionCodec(encryptionContext.getCompressionType()); + ByteBuf input = PulsarByteBufAllocator.DEFAULT.buffer(payloadBytes.length, payloadBytes.length); + input.writeBytes(payloadBytes); + ByteBuf output = codec.decode(input, encryptionContext.getUncompressedMessageSize()); + input.release(); + byte[] res = new byte[output.readableBytes()]; + output.readBytes(res); + output.release(); + return res; + } + return payloadBytes; + } + + /** + * Note: this method does not parse the message in its entirety; it only parses the payload of the message. + */ + public static List extractBatchMessagesIfNeeded(byte[] payloadBytes, + EncryptionContext encryptionContext) throws IOException { + ByteBuf payload = PulsarByteBufAllocator.DEFAULT.buffer(payloadBytes.length); + payload.writeBytes(payloadBytes); + if (encryptionContext.getBatchSize().isPresent()) { + List res = new ArrayList<>(); + int batchSize = encryptionContext.getBatchSize().get(); + for (int i = 0; i < batchSize; i++) { + ConsumerMessage msg = new ConsumerMessage(); + SingleMessageMetadata singleMsgMetadata = new SingleMessageMetadata(); + ByteBuf singleMsgPayload = Commands.deSerializeSingleMessageInBatch(payload, singleMsgMetadata, i, + batchSize); + if (singleMsgMetadata.getPayloadSize() < 1) { + msg.payload = null; + } else { + byte[] bs = new byte[singleMsgPayload.readableBytes()]; + singleMsgPayload.readBytes(bs); + msg.payload = new String(bs, UTF8); + } + res.add(msg); + } + return res; + } + ConsumerMessage msg = new ConsumerMessage(); + msg.payload = new String(payloadBytes, UTF8); + return Collections.singletonList(msg); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/v1/V1_ProxyAuthenticationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/v1/V1_ProxyAuthenticationTest.java index 01c851290b621..9767be625a070 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/v1/V1_ProxyAuthenticationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/v1/V1_ProxyAuthenticationTest.java @@ -84,7 +84,7 @@ public void setup() throws Exception { } service = spyWithClassAndConstructorArgs(WebSocketService.class, config); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(service) + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(service) .createConfigMetadataStore(anyString(), anyInt(), anyBoolean()); proxyServer = new ProxyServer(config); WebSocketServiceStarter.start(proxyServer, service); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsembleTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsembleTest.java index 92899feda7371..bfbdf675bd81d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsembleTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsembleTest.java @@ -22,9 +22,7 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; -import java.util.Collections; -import java.util.List; - +import org.apache.bookkeeper.conf.ServerConfiguration; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -39,22 +37,6 @@ void setup() throws Exception { void teardown() throws Exception { } - @Test - public void testAdvertisedAddress() throws Exception { - final int numBk = 1; - - LocalBookkeeperEnsemble ensemble = new LocalBookkeeperEnsemble( - numBk, 0, 0, null, null, true, "127.0.0.2"); - ensemble.startStandalone(); - - List bookies = ensemble.getZkClient().getChildren("/ledgers/available", false); - Collections.sort(bookies); - assertEquals(bookies.size(), 2); - assertTrue(bookies.get(0).startsWith("127.0.0.2:")); - - ensemble.stop(); - } - @Test public void testStartStop() throws Exception { @@ -74,4 +56,18 @@ public void testStartStop() throws Exception { assertFalse(ensemble.getZkClient().getState().isConnected()); assertFalse(ensemble.getBookies()[0].isRunning()); } + + @Test(timeOut = 10_000) + public void testStartWithSpecifiedStreamStoragePort() throws Exception { + LocalBookkeeperEnsemble ensemble = null; + try { + ensemble = + new LocalBookkeeperEnsemble(1, 0, 0, 4182, null, null, true, null); + ensemble.startStandalone(new ServerConfiguration(), true); + } finally { + if (ensemble != null) { + ensemble.stop(); + } + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/zookeeper/ZookeeperServerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/zookeeper/ZookeeperServerTest.java index 013bc23d6630f..355d2a0b1dbe1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/zookeeper/ZookeeperServerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/zookeeper/ZookeeperServerTest.java @@ -68,15 +68,20 @@ public void start() throws IOException { } public void stop() throws IOException { - zks.shutdown(); - serverFactory.shutdown(); - log.info("Stoppend ZK server at {}", hostPort); + if (zks != null) { + zks.shutdown(); + zks = null; + } + if (serverFactory != null) { + serverFactory.shutdown(); + serverFactory = null; + } + log.info("Stopped ZK server at {}", hostPort); } @Override public void close() throws IOException { - zks.shutdown(); - serverFactory.shutdown(); + stop(); zkTmpDir.delete(); } diff --git a/pulsar-broker/src/test/org/apache/pulsar/common/naming/TopicBundleAssignmentStrategyTest.java b/pulsar-broker/src/test/org/apache/pulsar/common/naming/TopicBundleAssignmentStrategyTest.java new file mode 100644 index 0000000000000..b371106cad85a --- /dev/null +++ b/pulsar-broker/src/test/org/apache/pulsar/common/naming/TopicBundleAssignmentStrategyTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.naming; + +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; + +import com.google.common.collect.BoundType; +import com.google.common.collect.Range; + +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.testng.Assert; +import org.testng.annotations.Test; + +@Test(groups = "broker-naming") +public class TopicBundleAssignmentStrategyTest { + @Test + public void testStrategyFactory() { + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setTopicBundleAssignmentStrategy( + "org.apache.pulsar.common.naming.TopicBundleAssignmentStrategyTest$TestStrategy"); + PulsarService pulsarService = mock(PulsarService.class); + doReturn(conf).when(pulsarService).getConfiguration(); + TopicBundleAssignmentStrategy strategy = TopicBundleAssignmentFactory.create(pulsarService); + NamespaceBundle bundle = strategy.findBundle(null, null); + Range keyRange = Range.range(0L, BoundType.CLOSED, 0xffffffffL, BoundType.CLOSED); + String range = String.format("0x%08x_0x%08x", keyRange.lowerEndpoint(), keyRange.upperEndpoint()); + Assert.assertEquals(bundle.getBundleRange(), range); + Assert.assertEquals(bundle.getNamespaceObject(), NamespaceName.get("my/test")); + } + + public static class TestStrategy implements TopicBundleAssignmentStrategy { + @Override + public NamespaceBundle findBundle(TopicName topicName, NamespaceBundles namespaceBundles) { + Range range = Range.range(0L, BoundType.CLOSED, 0xffffffffL, BoundType.CLOSED); + return new NamespaceBundle(NamespaceName.get("my/test"), range, + mock(NamespaceBundleFactory.class)); + } + + @Override + public void init(PulsarService pulsarService) { + + } + } +} diff --git a/pulsar-broker/src/test/resources/authentication/token/credentials_file.json b/pulsar-broker/src/test/resources/authentication/token/credentials_file.json index db1eccd8eb678..d12e786b7cdb5 100644 --- a/pulsar-broker/src/test/resources/authentication/token/credentials_file.json +++ b/pulsar-broker/src/test/resources/authentication/token/credentials_file.json @@ -1,4 +1,4 @@ { - "client_id":"Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x", - "client_secret":"rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N_poGAb" + "client_id":"my-admin-role", + "client_secret":"super-secret-client-secret" } diff --git a/tests/integration/src/test/resources/presto-follow-worker-config.properties b/pulsar-broker/src/test/resources/conf/zk_client_enable_sasl.conf similarity index 83% rename from tests/integration/src/test/resources/presto-follow-worker-config.properties rename to pulsar-broker/src/test/resources/conf/zk_client_enable_sasl.conf index d9849fed71f2c..c59e093450d39 100644 --- a/tests/integration/src/test/resources/presto-follow-worker-config.properties +++ b/pulsar-broker/src/test/resources/conf/zk_client_enable_sasl.conf @@ -17,11 +17,4 @@ # under the License. # -coordinator=false - -node.environment=test -http-server.http.port=8081 -discovery.uri=http://presto-worker:8081 - -query.client.timeout=5m -query.min-expire-age=30m +zookeeper.sasl.client=true diff --git a/pulsar-broker/src/test/resources/configurations/pulsar_broker_test.conf b/pulsar-broker/src/test/resources/configurations/pulsar_broker_test.conf index bfbbfb7487c42..0fdb29e06866f 100644 --- a/pulsar-broker/src/test/resources/configurations/pulsar_broker_test.conf +++ b/pulsar-broker/src/test/resources/configurations/pulsar_broker_test.conf @@ -17,17 +17,17 @@ # under the License. # -applicationName="pulsar_broker" -zookeeperServers="localhost" -configurationStoreServers="localhost" +applicationName=pulsar_broker +zookeeperServers=localhost +configurationStoreServers=localhost brokerServicePort=6650 -brokerServicePortTls=6651 +brokerServicePortTls= webServicePort=8080 -webServicePortTls=4443 +webServicePortTls= httpMaxRequestHeaderSize=1234 bindAddress=0.0.0.0 advertisedAddress= -clusterName="test_cluster" +clusterName=test_cluster brokerShutdownTimeoutMs=3000 backlogQuotaCheckEnabled=true backlogQuotaCheckIntervalInSeconds=60 @@ -42,17 +42,17 @@ clientLibraryVersionCheckEnabled=false clientLibraryVersionCheckAllowUnversioned=true statusFilePath=/tmp/status.html tlsEnabled=false -tlsCertificateFilePath=/usr/local/conf/pulsar/server.crt -tlsKeyFilePath=/home/local/conf/pulsar/server.key +tlsCertificateFilePath= +tlsKeyFilePath= tlsTrustCertsFilePath= tlsAllowInsecureConnection=false authenticationEnabled=false authorizationEnabled=false -superUserRoles="test_user" -brokerClientAuthenticationPlugin="org.apache.pulsar.client.impl.auth.AuthenticationDisabled" +superUserRoles=test_user +brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationDisabled brokerClientAuthenticationParameters= -bookkeeperClientAuthenticationPlugin="test_auth_plugin" -bookkeeperClientAuthenticationAppId="test_auth_id" +bookkeeperClientAuthenticationPlugin= +bookkeeperClientAuthenticationAppId=test_auth_id bookkeeperClientTimeoutInSeconds=30 bookkeeperClientSpeculativeReadTimeoutInMillis=0 bookkeeperClientHealthCheckEnabled=true @@ -64,7 +64,7 @@ bookkeeperClientRegionawarePolicyEnabled=false bookkeeperClientMinNumRacksPerWriteQuorum=2 bookkeeperClientEnforceMinNumRacksPerWriteQuorum=false bookkeeperClientReorderReadSequenceEnabled=false -bookkeeperClientIsolationGroups="test_group" +bookkeeperClientIsolationGroups=test_group managedLedgerDefaultEnsembleSize=3 managedLedgerDefaultWriteQuorum=2 managedLedgerDefaultAckQuorum=2 @@ -93,6 +93,7 @@ brokerDeleteInactiveTopicsMode=delete_when_subscriptions_caught_up supportedNamespaceBundleSplitAlgorithms=[range_equally_divide] defaultNamespaceBundleSplitAlgorithm=topic_count_equally_divide maxMessagePublishBufferSizeInMB=-1 +dispatcherPauseOnAckStatePersistentEnabled=true ### --- Transaction config variables --- ### transactionLogBatchedWriteEnabled=true @@ -103,3 +104,6 @@ transactionPendingAckBatchedWriteEnabled=true transactionPendingAckBatchedWriteMaxRecords=44 transactionPendingAckBatchedWriteMaxSize=55 transactionPendingAckBatchedWriteMaxDelayInMillis=66 +topicNameCacheMaxCapacity=200 +maxSecondsToClearTopicNameCache=1 +createTopicToRemoteClusterForReplication=false diff --git a/pulsar-broker/src/test/resources/configurations/pulsar_broker_test_standalone.conf b/pulsar-broker/src/test/resources/configurations/pulsar_broker_test_standalone.conf index c733409fc0043..d3f9430f29b48 100644 --- a/pulsar-broker/src/test/resources/configurations/pulsar_broker_test_standalone.conf +++ b/pulsar-broker/src/test/resources/configurations/pulsar_broker_test_standalone.conf @@ -17,18 +17,18 @@ # under the License. # -applicationName="pulsar_broker" -metadataStoreUrl="zk:localhost:2181/ledger" -configurationMetadataStoreUrl="zk:localhost:2181" -brokerServicePort=6650 -brokerServicePortTls=6651 -webServicePort=8080 -webServicePortTls=4443 +applicationName=pulsar_broker +metadataStoreUrl=zk:localhost:2181/ledger +configurationMetadataStoreUrl=zk:localhost:2181 +brokerServicePort=0 +brokerServicePortTls= +webServicePort=0 +webServicePortTls= bindAddress=0.0.0.0 advertisedAddress= advertisedListeners=internal:pulsar://192.168.1.11:6660,internal:pulsar+ssl://192.168.1.11:6651 internalListenerName=internal -clusterName="test_cluster" +clusterName=test_cluster brokerShutdownTimeoutMs=3000 backlogQuotaCheckEnabled=true backlogQuotaCheckIntervalInSeconds=60 @@ -49,11 +49,11 @@ tlsTrustCertsFilePath= tlsAllowInsecureConnection=false authenticationEnabled=false authorizationEnabled=false -superUserRoles="test_user" -brokerClientAuthenticationPlugin="org.apache.pulsar.client.impl.auth.AuthenticationDisabled" +superUserRoles=test_user +brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationDisabled brokerClientAuthenticationParameters= -bookkeeperClientAuthenticationPlugin="test_auth_plugin" -bookkeeperClientAuthenticationAppId="test_auth_id" +bookkeeperClientAuthenticationPlugin= +bookkeeperClientAuthenticationAppId= bookkeeperClientTimeoutInSeconds=30 bookkeeperClientSpeculativeReadTimeoutInMillis=0 bookkeeperClientHealthCheckEnabled=true @@ -65,7 +65,7 @@ bookkeeperClientRegionawarePolicyEnabled=false bookkeeperClientMinNumRacksPerWriteQuorum=2 bookkeeperClientEnforceMinNumRacksPerWriteQuorum=false bookkeeperClientReorderReadSequenceEnabled=false -bookkeeperClientIsolationGroups="test_group" +bookkeeperClientIsolationGroups= managedLedgerDefaultEnsembleSize=3 managedLedgerDefaultWriteQuorum=2 managedLedgerDefaultAckQuorum=2 @@ -94,3 +94,7 @@ brokerDeleteInactiveTopicsMode=delete_when_subscriptions_caught_up supportedNamespaceBundleSplitAlgorithms=[range_equally_divide] defaultNamespaceBundleSplitAlgorithm=topic_count_equally_divide maxMessagePublishBufferSizeInMB=-1 +dispatcherPauseOnAckStatePersistentEnabled=true +topicNameCacheMaxCapacity=200 +maxSecondsToClearTopicNameCache=1 +createTopicToRemoteClusterForReplication=true diff --git a/pulsar-broker/src/test/resources/configurations/pulsar_broker_test_standalone_with_rocksdb.conf b/pulsar-broker/src/test/resources/configurations/pulsar_broker_test_standalone_with_rocksdb.conf index d8b26bbbfa99d..46c876686b05b 100644 --- a/pulsar-broker/src/test/resources/configurations/pulsar_broker_test_standalone_with_rocksdb.conf +++ b/pulsar-broker/src/test/resources/configurations/pulsar_broker_test_standalone_with_rocksdb.conf @@ -17,19 +17,19 @@ # under the License. # -applicationName="pulsar_broker" +applicationName=pulsar_broker metadataStoreUrl= configurationMetadataStoreUrl= -brokerServicePort=6650 -brokerServicePortTls=6651 -webServicePort=8080 +brokerServicePort=0 +brokerServicePortTls= +webServicePort=0 allowLoopback=true -webServicePortTls=4443 +webServicePortTls= bindAddress=0.0.0.0 advertisedAddress= advertisedListeners= internalListenerName=internal -clusterName="test_cluster" +clusterName=test_cluster brokerShutdownTimeoutMs=3000 backlogQuotaCheckEnabled=true backlogQuotaCheckIntervalInSeconds=60 @@ -44,17 +44,17 @@ clientLibraryVersionCheckEnabled=false clientLibraryVersionCheckAllowUnversioned=true statusFilePath=/tmp/status.html tlsEnabled=false -tlsCertificateFilePath=/usr/local/conf/pulsar/server.crt -tlsKeyFilePath=/home/local/conf/pulsar/server.key +tlsCertificateFilePath= +tlsKeyFilePath= tlsTrustCertsFilePath= tlsAllowInsecureConnection=false authenticationEnabled=false authorizationEnabled=false -superUserRoles="test_user" -brokerClientAuthenticationPlugin="org.apache.pulsar.client.impl.auth.AuthenticationDisabled" +superUserRoles=test_user +brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationDisabled brokerClientAuthenticationParameters= -bookkeeperClientAuthenticationPlugin="test_auth_plugin" -bookkeeperClientAuthenticationAppId="test_auth_id" +bookkeeperClientAuthenticationPlugin= +bookkeeperClientAuthenticationAppId=test_auth_id bookkeeperClientTimeoutInSeconds=30 bookkeeperClientSpeculativeReadTimeoutInMillis=0 bookkeeperClientHealthCheckEnabled=true @@ -66,7 +66,7 @@ bookkeeperClientRegionawarePolicyEnabled=false bookkeeperClientMinNumRacksPerWriteQuorum=2 bookkeeperClientEnforceMinNumRacksPerWriteQuorum=false bookkeeperClientReorderReadSequenceEnabled=false -bookkeeperClientIsolationGroups="test_group" +bookkeeperClientIsolationGroups=test_group managedLedgerDefaultEnsembleSize=3 managedLedgerDefaultWriteQuorum=2 managedLedgerDefaultAckQuorum=2 diff --git a/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf b/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf index d9411e655ad5b..6f0d82cef17bc 100644 --- a/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf +++ b/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf @@ -17,8 +17,8 @@ # under the License. # -brokerServicePort=6650 -webServicePort=8080 +brokerServicePort=0 +webServicePort=0 allowLoopback=true clusterName=test_cluster superUserRoles=admin @@ -29,4 +29,5 @@ authenticationEnabled=true authenticationProviders=org.apache.pulsar.MockTokenAuthenticationProvider brokerClientAuthenticationPlugin= brokerClientAuthenticationParameters= -loadBalancerOverrideBrokerNicSpeedGbps=2 \ No newline at end of file +loadBalancerOverrideBrokerNicSpeedGbps=2 +topicLevelPoliciesEnabled=false \ No newline at end of file diff --git a/pulsar-broker/src/test/resources/log4j2.xml b/pulsar-broker/src/test/resources/log4j2.xml new file mode 100644 index 0000000000000..09a89702ee2ac --- /dev/null +++ b/pulsar-broker/src/test/resources/log4j2.xml @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + diff --git a/pulsar-broker/src/test/resources/prometheus_metrics_sample.txt b/pulsar-broker/src/test/resources/prometheus_metrics_sample.txt index 2022fbd800000..7cf8d3e7167d7 100644 --- a/pulsar-broker/src/test/resources/prometheus_metrics_sample.txt +++ b/pulsar-broker/src/test/resources/prometheus_metrics_sample.txt @@ -284,8 +284,6 @@ caffeine_cache_load_duration_seconds_count{cluster="use",cache="bookies-racks-da caffeine_cache_load_duration_seconds_sum{cluster="use",cache="bookies-racks-data"} 0.0 caffeine_cache_load_duration_seconds_count{cluster="use",cache="global-zk-exists"} 0.0 caffeine_cache_load_duration_seconds_sum{cluster="use",cache="global-zk-exists"} 0.0 -# TYPE pulsar_broker_throttled_connections_global_limit gauge -pulsar_broker_throttled_connections_global_limit{cluster="use"} 0.0 # TYPE process_cpu_seconds_total counter process_cpu_seconds_total{cluster="use"} 101.68 # TYPE process_start_time_seconds gauge diff --git a/pulsar-cli-utils/pom.xml b/pulsar-cli-utils/pom.xml new file mode 100644 index 0000000000000..8695353e6b40b --- /dev/null +++ b/pulsar-cli-utils/pom.xml @@ -0,0 +1,147 @@ + + + + 4.0.0 + + org.apache.pulsar + pulsar + 4.0.0-SNAPSHOT + + + pulsar-cli-utils + Pulsar CLI Utils + Isolated CLI utility module + + + + info.picocli + picocli + compile + + + org.apache.commons + commons-lang3 + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${pulsar.client.compiler.release} + + + + + org.gaul + modernizer-maven-plugin + + true + 8 + + + + modernizer + verify + + modernizer + + + + + + + pl.project13.maven + git-commit-id-plugin + + + git-info + + revision + + + + + false + true + git + false + false + false + properties + + true + + + + + + org.codehaus.mojo + templating-maven-plugin + 1.0.0 + + + filtering-java-templates + + filter-sources + + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + + spotbugs + verify + + check + + + + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + + diff --git a/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/ValueValidationUtil.java b/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/ValueValidationUtil.java new file mode 100644 index 0000000000000..751f69b20c4dd --- /dev/null +++ b/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/ValueValidationUtil.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.cli; + +import lombok.experimental.UtilityClass; +import org.apache.commons.lang3.StringUtils; + +@UtilityClass +public class ValueValidationUtil { + + public static void maxValueCheck(String paramName, long value, long maxValue) { + if (value > maxValue) { + throw new IllegalArgumentException(paramName + " cannot be bigger than <" + maxValue + ">!"); + } + } + + public static void positiveCheck(String paramName, long value) { + if (value <= 0) { + throw new IllegalArgumentException(paramName + " cannot be less than or equal to <0>!"); + } + } + + public static void positiveCheck(String paramName, int value) { + if (value <= 0) { + throw new IllegalArgumentException(paramName + " cannot be less than or equal to <0>!"); + } + } + + public static void emptyCheck(String paramName, String value) { + if (StringUtils.isEmpty(value)) { + throw new IllegalArgumentException("The value of " + paramName + " can't be empty"); + } + } + + public static void minValueCheck(String name, Long value, long min) { + if (value < min) { + throw new IllegalArgumentException(name + " cannot be less than <" + min + ">!"); + } + } +} diff --git a/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/ByteUnitUtil.java b/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/ByteUnitUtil.java new file mode 100644 index 0000000000000..8b5a0aafcdb6a --- /dev/null +++ b/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/ByteUnitUtil.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.cli.converters; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import lombok.experimental.UtilityClass; + +@UtilityClass +public class ByteUnitUtil { + + private static Set sizeUnit = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList('k', 'K', 'm', 'M', 'g', 'G', 't', 'T'))); + + public static long validateSizeString(String byteStr) { + if (byteStr.isEmpty()) { + throw new IllegalArgumentException("byte string cannot be empty"); + } + + char last = byteStr.charAt(byteStr.length() - 1); + String subStr = byteStr.substring(0, byteStr.length() - 1); + long size; + try { + size = sizeUnit.contains(last) + ? Long.parseLong(subStr) + : Long.parseLong(byteStr); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException(String.format("Invalid size '%s'. Valid formats are: %s", + byteStr, "(4096, 100K, 10M, 16G, 2T)")); + } + switch (last) { + case 'k': + case 'K': + return size * 1024; + + case 'm': + case 'M': + return size * 1024 * 1024; + + case 'g': + case 'G': + return size * 1024 * 1024 * 1024; + + case 't': + case 'T': + return size * 1024 * 1024 * 1024 * 1024; + + default: + return size; + } + } +} diff --git a/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/RelativeTimeUtil.java b/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/RelativeTimeUtil.java new file mode 100644 index 0000000000000..412a6415e3c31 --- /dev/null +++ b/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/RelativeTimeUtil.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.cli.converters; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.concurrent.TimeUnit; +import lombok.experimental.UtilityClass; + +@UtilityClass +public class RelativeTimeUtil { + public static long parseRelativeTimeInSeconds(String relativeTime) { + if (relativeTime.isEmpty()) { + throw new IllegalArgumentException("time cannot be empty"); + } + + int lastIndex = relativeTime.length() - 1; + char lastChar = relativeTime.charAt(lastIndex); + final char timeUnit; + + if (!Character.isAlphabetic(lastChar)) { + // No unit specified, assume seconds + timeUnit = 's'; + lastIndex = relativeTime.length(); + } else { + timeUnit = Character.toLowerCase(lastChar); + } + + long duration = Long.parseLong(relativeTime.substring(0, lastIndex)); + + switch (timeUnit) { + case 's': + return duration; + case 'm': + return TimeUnit.MINUTES.toSeconds(duration); + case 'h': + return TimeUnit.HOURS.toSeconds(duration); + case 'd': + return TimeUnit.DAYS.toSeconds(duration); + case 'w': + return 7 * TimeUnit.DAYS.toSeconds(duration); + // No unit for months + case 'y': + return 365 * TimeUnit.DAYS.toSeconds(duration); + default: + throw new IllegalArgumentException("Invalid time unit '" + lastChar + "'"); + } + } + + /** + * Convert nanoseconds to seconds and keep three decimal places. + * @param ns + * @return seconds + */ + public static double nsToSeconds(long ns) { + double seconds = (double) ns / 1_000_000_000; + BigDecimal bd = new BigDecimal(seconds); + return bd.setScale(3, RoundingMode.HALF_UP).doubleValue(); + } +} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/util/package-info.java b/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/package-info.java similarity index 94% rename from pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/util/package-info.java rename to pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/package-info.java index d163340416364..4204abdef3b31 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/util/package-info.java +++ b/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/package-info.java @@ -16,4 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.sql.presto.util; +package org.apache.pulsar.cli.converters; diff --git a/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/picocli/ByteUnitToIntegerConverter.java b/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/picocli/ByteUnitToIntegerConverter.java new file mode 100644 index 0000000000000..2e5a15c9d9c41 --- /dev/null +++ b/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/picocli/ByteUnitToIntegerConverter.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.cli.converters.picocli; + +import static org.apache.pulsar.cli.converters.ByteUnitUtil.validateSizeString; +import picocli.CommandLine.ITypeConverter; +import picocli.CommandLine.TypeConversionException; + +public class ByteUnitToIntegerConverter implements ITypeConverter { + @Override + public Integer convert(String value) throws Exception { + try { + long l = validateSizeString(value); + return (int) l; + } catch (Exception e) { + throw new TypeConversionException(e.getMessage()); + } + } +} diff --git a/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/picocli/ByteUnitToLongConverter.java b/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/picocli/ByteUnitToLongConverter.java new file mode 100644 index 0000000000000..519cf3dc4c32b --- /dev/null +++ b/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/picocli/ByteUnitToLongConverter.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.cli.converters.picocli; + +import static org.apache.pulsar.cli.converters.ByteUnitUtil.validateSizeString; +import picocli.CommandLine.ITypeConverter; +import picocli.CommandLine.TypeConversionException; + +public class ByteUnitToLongConverter implements ITypeConverter { + @Override + public Long convert(String value) throws Exception { + try { + return validateSizeString(value); + } catch (Exception e) { + throw new TypeConversionException(e.getMessage()); + } + } +} diff --git a/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/picocli/TimeUnitToMillisConverter.java b/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/picocli/TimeUnitToMillisConverter.java new file mode 100644 index 0000000000000..008467a23e6d8 --- /dev/null +++ b/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/picocli/TimeUnitToMillisConverter.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.cli.converters.picocli; + +import java.util.concurrent.TimeUnit; +import org.apache.pulsar.cli.converters.RelativeTimeUtil; +import picocli.CommandLine.ITypeConverter; +import picocli.CommandLine.TypeConversionException; + +public class TimeUnitToMillisConverter implements ITypeConverter { + @Override + public Long convert(String value) throws Exception { + try { + return TimeUnit.SECONDS.toMillis(RelativeTimeUtil.parseRelativeTimeInSeconds(value)); + } catch (Exception e) { + throw new TypeConversionException(e.getMessage()); + } + } +} diff --git a/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/picocli/TimeUnitToSecondsConverter.java b/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/picocli/TimeUnitToSecondsConverter.java new file mode 100644 index 0000000000000..231fa19bdd56c --- /dev/null +++ b/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/picocli/TimeUnitToSecondsConverter.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.cli.converters.picocli; + +import java.util.concurrent.TimeUnit; +import org.apache.pulsar.cli.converters.RelativeTimeUtil; +import picocli.CommandLine.ITypeConverter; +import picocli.CommandLine.TypeConversionException; + +public class TimeUnitToSecondsConverter implements ITypeConverter { + @Override + public Long convert(String value) throws Exception { + try { + return TimeUnit.SECONDS.toSeconds(RelativeTimeUtil.parseRelativeTimeInSeconds(value)); + } catch (Exception e) { + throw new TypeConversionException(e.getMessage()); + } + } +} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/package-info.java b/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/picocli/package-info.java similarity index 87% rename from pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/package-info.java rename to pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/picocli/package-info.java index 44dd00fc9f50a..bdb3e1e85dd19 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/package-info.java +++ b/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/converters/picocli/package-info.java @@ -16,7 +16,4 @@ * specific language governing permissions and limitations * under the License. */ -/** - * This package contains decoder for SchemaType.AVRO. - */ -package org.apache.pulsar.sql.presto.decoder.avro; \ No newline at end of file +package org.apache.pulsar.cli.converters.picocli; diff --git a/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/package-info.java b/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/package-info.java new file mode 100644 index 0000000000000..2b2198c265c64 --- /dev/null +++ b/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.cli; diff --git a/pulsar-sql/presto-pulsar-plugin/src/assembly/assembly.xml b/pulsar-cli-utils/src/main/resources/findbugsExclude.xml similarity index 53% rename from pulsar-sql/presto-pulsar-plugin/src/assembly/assembly.xml rename to pulsar-cli-utils/src/main/resources/findbugsExclude.xml index 6650abfda3fc3..ddde8120ba518 100644 --- a/pulsar-sql/presto-pulsar-plugin/src/assembly/assembly.xml +++ b/pulsar-cli-utils/src/main/resources/findbugsExclude.xml @@ -18,22 +18,5 @@ under the License. --> - - bin - - tar.gz - dir - - - - / - false - runtime - - jakarta.ws.rs:jakarta.ws.rs-api - - - - \ No newline at end of file + + \ No newline at end of file diff --git a/pulsar-cli-utils/src/test/java/org/apache/pulsar/cli/ValueValidationUtilTest.java b/pulsar-cli-utils/src/test/java/org/apache/pulsar/cli/ValueValidationUtilTest.java new file mode 100644 index 0000000000000..06db820819970 --- /dev/null +++ b/pulsar-cli-utils/src/test/java/org/apache/pulsar/cli/ValueValidationUtilTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.cli; + +import static org.testng.Assert.assertThrows; +import org.testng.annotations.Test; + +public class ValueValidationUtilTest { + + @Test + public void testMaxValueCheck() { + assertThrows(IllegalArgumentException.class, () -> ValueValidationUtil.maxValueCheck("param1", 11L, 10L)); + ValueValidationUtil.maxValueCheck("param2", 10L, 10L); + ValueValidationUtil.maxValueCheck("param3", 9L, 10L); + } + + @Test + public void testPositiveCheck() { + // Long + assertThrows(IllegalArgumentException.class, () -> ValueValidationUtil.positiveCheck("param1", 0L)); + assertThrows(IllegalArgumentException.class, () -> ValueValidationUtil.positiveCheck("param2", -1L)); + ValueValidationUtil.positiveCheck("param3", 1L); + + // Integer + assertThrows(IllegalArgumentException.class, () -> ValueValidationUtil.positiveCheck("param4", 0)); + assertThrows(IllegalArgumentException.class, () -> ValueValidationUtil.positiveCheck("param5", -1)); + ValueValidationUtil.positiveCheck("param6", 1); + } + + @Test + public void testEmptyCheck() { + assertThrows(IllegalArgumentException.class, () -> ValueValidationUtil.emptyCheck("param1", "")); + assertThrows(IllegalArgumentException.class, () -> ValueValidationUtil.emptyCheck("param2", null)); + ValueValidationUtil.emptyCheck("param3", "nonEmpty"); + } + + @Test + public void testMinValueCheck() { + assertThrows(IllegalArgumentException.class, () -> ValueValidationUtil.minValueCheck("param1", 9L, 10L)); + ValueValidationUtil.minValueCheck("param2", 10L, 10L); + ValueValidationUtil.minValueCheck("param3", 11L, 10L); + } + + @Test + public void testPositiveCheckInt() { + assertThrows(IllegalArgumentException.class, () -> ValueValidationUtil.positiveCheck("param1", 0)); + assertThrows(IllegalArgumentException.class, () -> ValueValidationUtil.positiveCheck("param2", -1)); + ValueValidationUtil.positiveCheck("param3", 1); + } +} diff --git a/pulsar-cli-utils/src/test/java/org/apache/pulsar/cli/converters/ByteConversionTest.java b/pulsar-cli-utils/src/test/java/org/apache/pulsar/cli/converters/ByteConversionTest.java new file mode 100644 index 0000000000000..6e7a2e6d7e347 --- /dev/null +++ b/pulsar-cli-utils/src/test/java/org/apache/pulsar/cli/converters/ByteConversionTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.cli.converters; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertThrows; +import org.apache.pulsar.cli.converters.picocli.ByteUnitToIntegerConverter; +import org.apache.pulsar.cli.converters.picocli.ByteUnitToLongConverter; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; +import picocli.CommandLine.TypeConversionException; + +public class ByteConversionTest { + + @DataProvider + public static Object[][] successfulByteUnitUtilTestCases() { + return new Object[][] { + {"4096", 4096L}, + {"1000", 1000L}, + {"100K", 102400L}, + {"100k", 102400L}, + {"100M", 104857600L}, + {"100m", 104857600L}, + {"100G", 107374182400L}, + {"100g", 107374182400L}, + {"100T", 109951162777600L}, + {"100t", 109951162777600L}, + }; + } + + @DataProvider + public static Object[][] failingByteUnitUtilTestCases() { + return new Object[][] { + {""}, // Empty string + {"1Z"}, // Invalid size unit + {"1.5K"}, // Non-integer value + {"K"} // Missing size value + }; + } + + @Test(dataProvider = "successfulByteUnitUtilTestCases") + public void testSuccessfulByteUnitUtilConversion(String input, long expected) { + assertEquals(ByteUnitUtil.validateSizeString(input), expected); + } + + @Test(dataProvider = "successfulByteUnitUtilTestCases") + public void testSuccessfulByteUnitToLongConverter(String input, long expected) throws Exception{ + ByteUnitToLongConverter converter = new ByteUnitToLongConverter(); + assertEquals(converter.convert(input), Long.valueOf(expected)); + } + + @Test(dataProvider = "successfulByteUnitUtilTestCases") + public void testSuccessfulByteUnitIntegerConverter(String input, long expected) throws Exception { + ByteUnitToIntegerConverter converter = new ByteUnitToIntegerConverter(); + // Since the converter returns an Integer, we need to cast expected to int + assertEquals(converter.convert(input), Integer.valueOf((int) expected)); + } + + @Test(dataProvider = "failingByteUnitUtilTestCases") + public void testFailedByteUnitUtilConversion(String input) { + assertThrows(IllegalArgumentException.class, () -> ByteUnitUtil.validateSizeString(input)); + } + + @Test(dataProvider = "failingByteUnitUtilTestCases") + public void testFailedByteUnitToLongConverter(String input) { + ByteUnitToLongConverter converter = new ByteUnitToLongConverter(); + assertThrows(TypeConversionException.class, () -> converter.convert(input)); + } + + @Test(dataProvider = "failingByteUnitUtilTestCases") + public void testFailedByteUnitIntegerConverter(String input) { + ByteUnitToIntegerConverter converter = new ByteUnitToIntegerConverter(); + assertThrows(TypeConversionException.class, () -> converter.convert(input)); + } +} + diff --git a/pulsar-cli-utils/src/test/java/org/apache/pulsar/cli/converters/TimeConversionTest.java b/pulsar-cli-utils/src/test/java/org/apache/pulsar/cli/converters/TimeConversionTest.java new file mode 100644 index 0000000000000..451a215bce313 --- /dev/null +++ b/pulsar-cli-utils/src/test/java/org/apache/pulsar/cli/converters/TimeConversionTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.cli.converters; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertThrows; +import java.util.concurrent.TimeUnit; +import org.apache.pulsar.cli.converters.picocli.TimeUnitToMillisConverter; +import org.apache.pulsar.cli.converters.picocli.TimeUnitToSecondsConverter; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class TimeConversionTest { + + @DataProvider + public static Object[][] successfulRelativeTimeUtilTestCases() { + return new Object[][] { + {"-1", -1L}, + {"7", 7L}, + {"100", 100L}, // No time unit, assuming seconds + {"3s", 3L}, + {"3S", 3L}, + {"10s", 10L}, + {"1m", 60L}, + {"5m", TimeUnit.MINUTES.toSeconds(5L)}, + {"5M", TimeUnit.MINUTES.toSeconds(5L)}, + {"7h", TimeUnit.HOURS.toSeconds(7L)}, + {"7H", TimeUnit.HOURS.toSeconds(7L)}, + {"9d", TimeUnit.DAYS.toSeconds(9L)}, + {"9D", TimeUnit.DAYS.toSeconds(9L)}, + {"1w", 604800L}, + {"3W", 7 * TimeUnit.DAYS.toSeconds(3L)}, + {"11y", 365 * TimeUnit.DAYS.toSeconds(11L)}, + {"11Y", 365 * TimeUnit.DAYS.toSeconds(11L)}, + {"-5m", -TimeUnit.MINUTES.toSeconds(5L)} + }; + } + + @Test(dataProvider = "successfulRelativeTimeUtilTestCases") + public void testSuccessfulRelativeTimeUtilParsing(String input, long expected) { + assertEquals(RelativeTimeUtil.parseRelativeTimeInSeconds(input), expected); + } + + @Test(dataProvider = "successfulRelativeTimeUtilTestCases") + public void testSuccessfulTimeUnitToSecondsConverter(String input, long expected) throws Exception { + TimeUnitToSecondsConverter secondsConverter = new TimeUnitToSecondsConverter(); + assertEquals(secondsConverter.convert(input), Long.valueOf(expected)); + } + + @Test(dataProvider = "successfulRelativeTimeUtilTestCases") + public void testSuccessfulTimeUnitToMillisConverter(String input, long expected) throws Exception { + TimeUnitToMillisConverter millisConverter = new TimeUnitToMillisConverter(); + // We multiply the expected by 1000 to convert the seconds into milliseconds + assertEquals(millisConverter.convert(input), Long.valueOf(expected * 1000)); + } + + @Test + public void testFailingParsing() { + assertThrows(IllegalArgumentException.class, () -> RelativeTimeUtil.parseRelativeTimeInSeconds("")); // Empty string + assertThrows(IllegalArgumentException.class, () -> RelativeTimeUtil.parseRelativeTimeInSeconds("s")); // Non-numeric character + assertThrows(IllegalArgumentException.class, () -> RelativeTimeUtil.parseRelativeTimeInSeconds("1z")); // Invalid time unit + assertThrows(IllegalArgumentException.class, () -> RelativeTimeUtil.parseRelativeTimeInSeconds("1.5")); // Floating point number + } + + @Test + public void testNsToSeconds() { + assertEquals(RelativeTimeUtil.nsToSeconds(1_000_000_000), 1.000); + assertEquals(RelativeTimeUtil.nsToSeconds(1_500_000_000), 1.500); + assertEquals(RelativeTimeUtil.nsToSeconds(1_555_555_555), 1.556); + } +} diff --git a/pulsar-client-1x-base/pom.xml b/pulsar-client-1x-base/pom.xml index 41de4d2e06bde..fedbe80cc6261 100644 --- a/pulsar-client-1x-base/pom.xml +++ b/pulsar-client-1x-base/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-client-1x-base diff --git a/pulsar-client-1x-base/pulsar-client-1x/pom.xml b/pulsar-client-1x-base/pulsar-client-1x/pom.xml index 307a1d76f7490..b9c8fa7d3eb04 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-1x/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-client-1x diff --git a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml index 7cf616add030d..2e316e8e5eee3 100644 --- a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-client-2x-shaded diff --git a/pulsar-client-admin-api/pom.xml b/pulsar-client-admin-api/pom.xml index be666085c2c5f..7983c26af44f4 100644 --- a/pulsar-client-admin-api/pom.xml +++ b/pulsar-client-admin-api/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-client-admin-api @@ -43,7 +42,7 @@ org.slf4j slf4j-api - + diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Brokers.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Brokers.java index 464d02121cfc2..eed73f38282ac 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Brokers.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Brokers.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import org.apache.pulsar.client.admin.PulsarAdminException.NotAuthorizedException; import org.apache.pulsar.client.admin.PulsarAdminException.NotFoundException; @@ -35,7 +36,7 @@ public interface Brokers { /** * Get the list of active brokers in the local cluster. *

    - * Get the list of active brokers (web service addresses) in the local cluster. + * Get the list of active brokers (broker ids) in the local cluster. *

    * Response Example: * @@ -44,7 +45,7 @@ public interface Brokers { * * * "prod1-broker3.messaging.use.example.com:8080"] * * - * @return a list of (host:port) + * @return a list of broker ids * @throws NotAuthorizedException * You don't have admin permission to get the list of active brokers in the cluster * @throws PulsarAdminException @@ -55,7 +56,7 @@ public interface Brokers { /** * Get the list of active brokers in the local cluster asynchronously. *

    - * Get the list of active brokers (web service addresses) in the local cluster. + * Get the list of active brokers (broker ids) in the local cluster. *

    * Response Example: * @@ -64,13 +65,13 @@ public interface Brokers { * "prod1-broker3.messaging.use.example.com:8080"] * * - * @return a list of (host:port) + * @return a list of broker ids */ CompletableFuture> getActiveBrokersAsync(); /** * Get the list of active brokers in the cluster. *

    - * Get the list of active brokers (web service addresses) in the cluster. + * Get the list of active brokers (broker ids) in the cluster. *

    * Response Example: * @@ -81,7 +82,7 @@ public interface Brokers { * * @param cluster * Cluster name - * @return a list of (host:port) + * @return a list of broker ids * @throws NotAuthorizedException * You don't have admin permission to get the list of active brokers in the cluster * @throws NotFoundException @@ -94,7 +95,7 @@ public interface Brokers { /** * Get the list of active brokers in the cluster asynchronously. *

    - * Get the list of active brokers (web service addresses) in the cluster. + * Get the list of active brokers (broker ids) in the cluster. *

    * Response Example: * @@ -105,7 +106,7 @@ public interface Brokers { * * @param cluster * Cluster name - * @return a list of (host:port) + * @return a list of broker ids */ CompletableFuture> getActiveBrokersAsync(String cluster); @@ -156,11 +157,11 @@ public interface Brokers { * * * @param cluster - * @param brokerUrl + * @param brokerId * @return * @throws PulsarAdminException */ - Map getOwnedNamespaces(String cluster, String brokerUrl) + Map getOwnedNamespaces(String cluster, String brokerId) throws PulsarAdminException; /** @@ -176,10 +177,10 @@ Map getOwnedNamespaces(String cluster, String * * * @param cluster - * @param brokerUrl + * @param brokerId * @return */ - CompletableFuture> getOwnedNamespacesAsync(String cluster, String brokerUrl); + CompletableFuture> getOwnedNamespacesAsync(String cluster, String brokerId); /** * Update a dynamic configuration value into ZooKeeper. @@ -320,16 +321,26 @@ Map getOwnedNamespaces(String cluster, String */ void healthcheck(TopicVersion topicVersion) throws PulsarAdminException; + /** + * Run a healthcheck on the target broker or on the broker. + * @param brokerId target broker id to check the health. If empty, it checks the health on the connected broker. + * + * @throws PulsarAdminException if the healthcheck fails. + */ + void healthcheck(TopicVersion topicVersion, Optional brokerId) throws PulsarAdminException; + /** * Run a healthcheck on the broker asynchronously. */ - CompletableFuture healthcheckAsync(TopicVersion topicVersion); + CompletableFuture healthcheckAsync(TopicVersion topicVersion, Optional brokerId); + /** - * Shutdown current broker gracefully. - * @param maxConcurrentUnloadPerSec - * @param forcedTerminateTopic - * @return + * Trigger the current broker to graceful-shutdown asynchronously. + * + * @param maxConcurrentUnloadPerSec the maximum number of topics to unload per second. + * This helps control the speed of the unload operation during shutdown. + * @param forcedTerminateTopic if true, topics will be forcefully terminated during the shutdown process. */ CompletableFuture shutDownBrokerGracefully(int maxConcurrentUnloadPerSec, boolean forcedTerminateTopic); diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Clusters.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Clusters.java index 4178bc7483df5..53e6680946566 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Clusters.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Clusters.java @@ -29,7 +29,8 @@ import org.apache.pulsar.client.admin.PulsarAdminException.PreconditionFailedException; import org.apache.pulsar.common.policies.data.BrokerNamespaceIsolationData; import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.ClusterData.ClusterUrl; +import org.apache.pulsar.common.policies.data.ClusterPolicies; +import org.apache.pulsar.common.policies.data.ClusterPolicies.ClusterUrl; import org.apache.pulsar.common.policies.data.FailureDomain; import org.apache.pulsar.common.policies.data.NamespaceIsolationData; @@ -209,6 +210,46 @@ public interface Clusters { */ CompletableFuture updatePeerClusterNamesAsync(String cluster, LinkedHashSet peerClusterNames); + /** + * Get the cluster migration configuration data for the specified cluster. + *

    + * Response Example: + * + * 

    +     * { serviceUrl : "http://my-broker.example.com:8080/" }
+     *
    + * + * @param cluster + * Cluster name + * + * @return the cluster configuration + * + * @throws NotAuthorizedException + * You don't have admin permission to get the configuration of the cluster + * @throws NotFoundException + * Cluster doesn't exist + * @throws PulsarAdminException + * Unexpected error + */ + ClusterPolicies getClusterMigration(String cluster) throws PulsarAdminException; + + /** + * Get the cluster migration configuration data for the specified cluster asynchronously. + *

    + * Response Example: + * + * 

    +     * { serviceUrl : "http://my-broker.example.com:8080/" }
+     *
    + * + * @param cluster + * Cluster name + * + * @return the cluster configuration + * + */ + CompletableFuture getClusterMigrationAsync(String cluster); + /** * Update the configuration for a cluster migration. *

    diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/GetStatsOptions.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/GetStatsOptions.java index 14e99ac014ba8..6ebc365833b27 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/GetStatsOptions.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/GetStatsOptions.java @@ -18,11 +18,13 @@ */ package org.apache.pulsar.client.admin; +import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @Data @Builder +@AllArgsConstructor public class GetStatsOptions { /** * Set to true to get precise backlog, Otherwise get imprecise backlog. @@ -38,4 +40,14 @@ public class GetStatsOptions { * Whether to get the earliest time in backlog. */ private final boolean getEarliestTimeInBacklog; + + /** + * Whether to exclude publishers. + */ + private final boolean excludePublishers; + + /** + * Whether to exclude consumers. + */ + private final boolean excludeConsumers; } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java index 2690df658b7be..65124a6a76a8f 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Namespaces.java @@ -1234,7 +1234,7 @@ CompletableFuture> getAntiAffinityNamespacesAsync( * @param namespace * Namespace name * @param enableDeduplication - * wether to enable or disable deduplication feature + * whether to enable or disable deduplication feature */ CompletableFuture setDeduplicationStatusAsync(String namespace, boolean enableDeduplication); @@ -4623,4 +4623,144 @@ void setIsAllowAutoUpdateSchema(String namespace, boolean isAllowAutoUpdateSchem * @return */ CompletableFuture removeNamespaceEntryFiltersAsync(String namespace); + + /** + * Enable migration for all topics within a namespace. + *

    + * Migrate all topics of a namespace to new broker. + *

    + * Request example: + * + * 

    +     * true
+     *
    + * + * @param namespace + * Namespace name + * @param migrated + * Flag to determine namespace is migrated or not + * @throws NotAuthorizedException + * Don't have admin permission + * @throws NotFoundException + * Namespace does not exist + * @throws PulsarAdminException + * Unexpected error + */ + void updateMigrationState(String namespace, boolean migrated) throws PulsarAdminException; + + /** + * Set DispatcherPauseOnAckStatePersistent for a namespace asynchronously. + */ + CompletableFuture setDispatcherPauseOnAckStatePersistentAsync(String namespace); + + /** + * Remove entry filters of a namespace. + * @param namespace Namespace name + * @throws PulsarAdminException + */ + void setDispatcherPauseOnAckStatePersistent(String namespace) throws PulsarAdminException; + + /** + * Removes the dispatcherPauseOnAckStatePersistentEnabled policy for a given namespace asynchronously. + */ + CompletableFuture removeDispatcherPauseOnAckStatePersistentAsync(String namespace); + + /** + * Removes the dispatcherPauseOnAckStatePersistentEnabled policy for a given namespace. + */ + void removeDispatcherPauseOnAckStatePersistent(String namespace) throws PulsarAdminException; + + /** + * Get the dispatcherPauseOnAckStatePersistentEnabled policy for a given namespace asynchronously. + */ + CompletableFuture getDispatcherPauseOnAckStatePersistentAsync(String namespace); + + /** + * Get the dispatcherPauseOnAckStatePersistentEnabled policy for a given namespace. + */ + boolean getDispatcherPauseOnAckStatePersistent(String namespace) throws PulsarAdminException; + + /** + * Get the allowed clusters for a namespace. + *

    + * Response example: + * + * 

    +     * ["use", "usw", "usc"]
+     *
    + * + * @param namespace + * Namespace name + * @throws NotAuthorizedException + * Don't have admin permission + * @throws NotFoundException + * Namespace does not exist + * @throws PreconditionFailedException + * Namespace is not global + * @throws PulsarAdminException + * Unexpected error + */ + List getNamespaceAllowedClusters(String namespace) throws PulsarAdminException; + + /** + * Get the allowed clusters for a namespace asynchronously. + *

    + * Response example: + * + * 

    +     * ["use", "usw", "usc"]
+     *
    + * + * @param namespace + * Namespace name + */ + CompletableFuture> getNamespaceAllowedClustersAsync(String namespace); + + /** + * Set the allowed clusters for a namespace. + *

    + * Request example: + * + * 

    +     * ["us-west", "us-east", "us-cent"]
+     *
    + * + * @param namespace + * Namespace name + * @param clusterIds + * Pulsar Cluster Ids + * + * @throws ConflictException + * Peer-cluster cannot be part of an allowed-cluster + * @throws NotAuthorizedException + * Don't have admin permission + * @throws NotFoundException + * Namespace does not exist + * @throws PreconditionFailedException + * Namespace is not global + * @throws PreconditionFailedException + * Invalid cluster ids + * @throws PulsarAdminException + * The list of allowed clusters should include all replication clusters. + * @throws PulsarAdminException + * Unexpected error + */ + void setNamespaceAllowedClusters(String namespace, Set clusterIds) throws PulsarAdminException; + + /** + * Set the allowed clusters for a namespace asynchronously. + *

    + * Request example: + * + * 

    +     * ["us-west", "us-east", "us-cent"]
+     *
    + * + * @param namespace + * Namespace name + * @param clusterIds + * Pulsar Cluster Ids + */ + CompletableFuture setNamespaceAllowedClustersAsync(String namespace, Set clusterIds); + } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/PulsarAdminBuilder.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/PulsarAdminBuilder.java index 1260555a7c43f..5c41d98b89dbc 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/PulsarAdminBuilder.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/PulsarAdminBuilder.java @@ -290,6 +290,20 @@ PulsarAdminBuilder authentication(String authPluginClassName, Map tlsProtocols); + /** + * SSL Factory Plugin used to generate the SSL Context and SSLEngine. + * @param sslFactoryPlugin Name of the SSL Factory Class to be used. + * @return PulsarAdminBuilder + */ + PulsarAdminBuilder sslFactoryPlugin(String sslFactoryPlugin); + + /** + * Parameters used by the SSL Factory Plugin class. + * @param sslFactoryPluginParams String parameters to be used by the SSL Factory Class. + * @return + */ + PulsarAdminBuilder sslFactoryPluginParams(String sslFactoryPluginParams); + /** * This sets the connection time out for the pulsar admin client. * @@ -327,4 +341,39 @@ PulsarAdminBuilder authentication(String authPluginClassName, Map + * By default, the connection pool maintains up to 16 connections to a single host. This method allows you to + * modify this default behavior and limit the number of connections. + *

    + * This setting can be useful in scenarios where you want to limit the resources used by the client library, + * or control the level of parallelism for operations so that a single client does not overwhelm + * the Pulsar cluster with too many concurrent connections. + * + * @param maxConnectionsPerHost the maximum number of connections to establish per host. Set to <= 0 to disable + * the limit. + * @return the PulsarAdminBuilder instance, allowing for method chaining + */ + PulsarAdminBuilder maxConnectionsPerHost(int maxConnectionsPerHost); + + /** + * Sets the maximum idle time for a pooled connection. If a connection is idle for more than the specified + * amount of seconds, it will be released back to the connection pool. + * Defaults to 25 seconds. + * + * @param connectionMaxIdleSeconds the maximum idle time, in seconds, for a pooled connection + * @return the PulsarAdminBuilder instance + */ + PulsarAdminBuilder connectionMaxIdleSeconds(int connectionMaxIdleSeconds); +} \ No newline at end of file diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Schemas.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Schemas.java index 9a1eb67d2e53a..ca8bed253702f 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Schemas.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Schemas.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.concurrent.CompletableFuture; +import org.apache.pulsar.common.policies.data.SchemaMetadata; import org.apache.pulsar.common.protocol.schema.IsCompatibilityResponse; import org.apache.pulsar.common.protocol.schema.PostSchemaPayload; import org.apache.pulsar.common.schema.SchemaInfo; @@ -233,4 +234,19 @@ IsCompatibilityResponse testCompatibility(String topic, PostSchemaPayload schema * @param topic topic name, in fully qualified format */ CompletableFuture> getAllSchemasAsync(String topic); + + /** + * Get schema metadata of the topic. + * + * @param topic topic name, in fully qualified format + * @throws PulsarAdminException + */ + SchemaMetadata getSchemaMetadata(String topic) throws PulsarAdminException; + + /** + * Get schema metadata of the topic asynchronously. + * + * @param topic topic name, in fully qualified format + */ + CompletableFuture getSchemaMetadataAsync(String topic); } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/TopicPolicies.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/TopicPolicies.java index f6cd2a5a0ef23..4238842bcfa22 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/TopicPolicies.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/TopicPolicies.java @@ -1913,4 +1913,20 @@ AutoSubscriptionCreationOverride getAutoSubscriptionCreation(String topic, * Topic name */ CompletableFuture removeAutoSubscriptionCreationAsync(String topic); + + /** + * After enabling this feature, Pulsar will stop delivery messages to clients if the cursor metadata is too large to + * # persist, it will help to reduce the duplicates caused by the ack state that can not be fully persistent. + */ + CompletableFuture setDispatcherPauseOnAckStatePersistent(String topic); + + /** + * Removes the dispatcherPauseOnAckStatePersistentEnabled policy for a given topic asynchronously. + */ + CompletableFuture removeDispatcherPauseOnAckStatePersistent(String topic); + + /** + * Get the dispatcherPauseOnAckStatePersistentEnabled policy for a given topic asynchronously. + */ + CompletableFuture getDispatcherPauseOnAckStatePersistent(String topic, boolean applied); } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java index f599e2566bffc..c681bd1a7bca1 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java @@ -31,6 +31,7 @@ import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.TransactionIsolationLevel; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.AuthAction; @@ -1139,23 +1140,26 @@ default CompletableFuture deleteAsync(String topic, boolean force) { default TopicStats getStats(String topic, boolean getPreciseBacklog, boolean subscriptionBacklogSize, boolean getEarliestTimeInBacklog) throws PulsarAdminException { GetStatsOptions getStatsOptions = - new GetStatsOptions(getPreciseBacklog, subscriptionBacklogSize, getEarliestTimeInBacklog); + new GetStatsOptions(getPreciseBacklog, subscriptionBacklogSize, getEarliestTimeInBacklog, false, false); return getStats(topic, getStatsOptions); } default TopicStats getStats(String topic, boolean getPreciseBacklog, boolean subscriptionBacklogSize) throws PulsarAdminException { - GetStatsOptions getStatsOptions = new GetStatsOptions(getPreciseBacklog, subscriptionBacklogSize, false); + GetStatsOptions getStatsOptions = new GetStatsOptions(getPreciseBacklog, subscriptionBacklogSize, false, + false, false); return getStats(topic, getStatsOptions); } default TopicStats getStats(String topic, boolean getPreciseBacklog) throws PulsarAdminException { - GetStatsOptions getStatsOptions = new GetStatsOptions(getPreciseBacklog, false, false); + GetStatsOptions getStatsOptions = new GetStatsOptions(getPreciseBacklog, false, false, + false, false); return getStats(topic, getStatsOptions); } default TopicStats getStats(String topic) throws PulsarAdminException { - return getStats(topic, new GetStatsOptions(false, false, false)); + return getStats(topic, new GetStatsOptions(false, false, false, + false, false)); } /** @@ -1176,6 +1180,8 @@ default TopicStats getStats(String topic) throws PulsarAdminException { CompletableFuture getStatsAsync(String topic, boolean getPreciseBacklog, boolean subscriptionBacklogSize, boolean getEarliestTimeInBacklog); + CompletableFuture getStatsAsync(String topic, GetStatsOptions getStatsOptions); + default CompletableFuture getStatsAsync(String topic) { return getStatsAsync(topic, false, false, false); } @@ -1346,6 +1352,9 @@ PartitionedTopicStats getPartitionedStats(String topic, boolean perPartition, bo boolean subscriptionBacklogSize, boolean getEarliestTimeInBacklog) throws PulsarAdminException; + PartitionedTopicStats getPartitionedStats(String topic, boolean perPartition, GetStatsOptions getStatsOptions) + throws PulsarAdminException; + default PartitionedTopicStats getPartitionedStats(String topic, boolean perPartition) throws PulsarAdminException { return getPartitionedStats(topic, perPartition, false, false, false); } @@ -1361,12 +1370,17 @@ default PartitionedTopicStats getPartitionedStats(String topic, boolean perParti * Set to true to get precise backlog, Otherwise get imprecise backlog. * @param subscriptionBacklogSize * Whether to get backlog size for each subscription. + * @param getEarliestTimeInBacklog + * Whether to get the earliest time in backlog. * @return a future that can be used to track when the partitioned topic statistics are returned */ CompletableFuture getPartitionedStatsAsync( String topic, boolean perPartition, boolean getPreciseBacklog, boolean subscriptionBacklogSize, boolean getEarliestTimeInBacklog); + CompletableFuture getPartitionedStatsAsync( + String topic, boolean perPartition, GetStatsOptions getStatsOptions); + default CompletableFuture getPartitionedStatsAsync(String topic, boolean perPartition) { return getPartitionedStatsAsync(topic, perPartition, false, false, false); } @@ -1640,7 +1654,53 @@ void expireMessagesForAllSubscriptions(String topic, long expireTimeInSeconds) * @throws PulsarAdminException * Unexpected error */ - List> peekMessages(String topic, String subName, int numMessages) throws PulsarAdminException; + default List> peekMessages(String topic, String subName, int numMessages) + throws PulsarAdminException { + return peekMessages(topic, subName, numMessages, false, TransactionIsolationLevel.READ_COMMITTED); + } + + /** + * Peek messages from a topic subscription. + * + * @param topic + * topic name + * @param subName + * Subscription name + * @param numMessages + * Number of messages + * @param showServerMarker + * Enables the display of internal server write markers + * @param transactionIsolationLevel + * Sets the isolation level for peeking messages within transactions. + * - 'READ_COMMITTED' allows peeking only committed transactional messages. + * - 'READ_UNCOMMITTED' allows peeking all messages, + * even transactional messages which have been aborted. + * @return + * @throws NotAuthorizedException + * Don't have admin permission + * @throws NotFoundException + * Topic or subscription does not exist + * @throws PulsarAdminException + * Unexpected error + */ + List> peekMessages(String topic, String subName, int numMessages, + boolean showServerMarker, TransactionIsolationLevel transactionIsolationLevel) + throws PulsarAdminException; + + /** + * Peek messages from a topic subscription asynchronously. + * + * @param topic + * topic name + * @param subName + * Subscription name + * @param numMessages + * Number of messages + * @return a future that can be used to track when the messages are returned + */ + default CompletableFuture>> peekMessagesAsync(String topic, String subName, int numMessages) { + return peekMessagesAsync(topic, subName, numMessages, false, TransactionIsolationLevel.READ_COMMITTED); + } /** * Peek messages from a topic subscription asynchronously. @@ -1651,9 +1711,18 @@ void expireMessagesForAllSubscriptions(String topic, long expireTimeInSeconds) * Subscription name * @param numMessages * Number of messages + * @param showServerMarker + * Enables the display of internal server write markers + @param transactionIsolationLevel + * Sets the isolation level for peeking messages within transactions. + * - 'READ_COMMITTED' allows peeking only committed transactional messages. + * - 'READ_UNCOMMITTED' allows peeking all messages, + * even transactional messages which have been aborted. * @return a future that can be used to track when the messages are returned */ - CompletableFuture>> peekMessagesAsync(String topic, String subName, int numMessages); + CompletableFuture>> peekMessagesAsync( + String topic, String subName, int numMessages, + boolean showServerMarker, TransactionIsolationLevel transactionIsolationLevel); /** * Get a message by its messageId via a topic subscription. @@ -1666,7 +1735,9 @@ void expireMessagesForAllSubscriptions(String topic, long expireTimeInSeconds) * @return the message indexed by the messageId * @throws PulsarAdminException * Unexpected error + * @deprecated Using {@link #getMessagesById(String, long, long)} instead. */ + @Deprecated Message getMessageById(String topic, long ledgerId, long entryId) throws PulsarAdminException; /** @@ -1678,9 +1749,32 @@ void expireMessagesForAllSubscriptions(String topic, long expireTimeInSeconds) * @param entryId * Entry id * @return a future that can be used to track when the message is returned + * @deprecated Using {@link #getMessagesByIdAsync(String, long, long)} instead. */ + @Deprecated CompletableFuture> getMessageByIdAsync(String topic, long ledgerId, long entryId); + /** + * Get the messages by messageId. + * + * @param topic Topic name + * @param ledgerId Ledger id + * @param entryId Entry id + * @return A set of messages. + * @throws PulsarAdminException Unexpected error + */ + List> getMessagesById(String topic, long ledgerId, long entryId) throws PulsarAdminException; + + /** + * Get the messages by messageId asynchronously. + * + * @param topic Topic name + * @param ledgerId Ledger id + * @param entryId Entry id + * @return A future that can be used to track when a set of messages is returned. + */ + CompletableFuture>> getMessagesByIdAsync(String topic, long ledgerId, long entryId); + /** * Get message ID published at or just after this absolute timestamp (in ms). * @param topic diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Transactions.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Transactions.java index 57adf263a574f..8fadabdfba235 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Transactions.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Transactions.java @@ -23,6 +23,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.common.policies.data.TransactionBufferInternalStats; import org.apache.pulsar.common.policies.data.TransactionBufferStats; import org.apache.pulsar.common.policies.data.TransactionCoordinatorInfo; import org.apache.pulsar.common.policies.data.TransactionCoordinatorInternalStats; @@ -139,10 +140,24 @@ TransactionInPendingAckStats getTransactionInPendingAckStats(TxnID txnID, String * Get transaction buffer stats. * * @param topic the topic of getting transaction buffer stats - * @param lowWaterMarks Whether to get information about lowWaterMarks stored in transaction pending ack. + * @param lowWaterMarks Whether to get information about lowWaterMarks stored in transaction pending ack. + * @param segmentStats Whether to get segment statistics. + * @return the future stats of transaction buffer in topic. + */ + CompletableFuture getTransactionBufferStatsAsync(String topic, boolean lowWaterMarks, + boolean segmentStats); + + /** + * Get transaction buffer stats. + * + * @param topic the topic of getting transaction buffer stats + * @param lowWaterMarks Whether to get information about lowWaterMarks stored in transaction pending ack. * @return the future stats of transaction buffer in topic. */ - CompletableFuture getTransactionBufferStatsAsync(String topic, boolean lowWaterMarks); + default CompletableFuture getTransactionBufferStatsAsync(String topic, + boolean lowWaterMarks) { + return getTransactionBufferStatsAsync(topic, lowWaterMarks, false); + } /** * Get transaction buffer stats. @@ -151,17 +166,31 @@ TransactionInPendingAckStats getTransactionInPendingAckStats(TxnID txnID, String * @return the future stats of transaction buffer in topic. */ default CompletableFuture getTransactionBufferStatsAsync(String topic) { - return getTransactionBufferStatsAsync(topic, false); + return getTransactionBufferStatsAsync(topic, false, false); } /** * Get transaction buffer stats. * * @param topic the topic of getting transaction buffer stats - * @param lowWaterMarks Whether to get information about lowWaterMarks stored in transaction buffer. + * @param lowWaterMarks Whether to get information about lowWaterMarks stored in transaction buffer. + * @param segmentStats Whether to get segment statistics. + * @return the stats of transaction buffer in topic. + */ + TransactionBufferStats getTransactionBufferStats(String topic, boolean lowWaterMarks, + boolean segmentStats) throws PulsarAdminException; + + /** + * Get transaction buffer stats. + * + * @param topic the topic of getting transaction buffer stats + * @param lowWaterMarks Whether to get information about lowWaterMarks stored in transaction buffer. * @return the stats of transaction buffer in topic. */ - TransactionBufferStats getTransactionBufferStats(String topic, boolean lowWaterMarks) throws PulsarAdminException; + default TransactionBufferStats getTransactionBufferStats(String topic, + boolean lowWaterMarks) throws PulsarAdminException { + return getTransactionBufferStats(topic, lowWaterMarks, false); + } /** * Get transaction buffer stats. @@ -170,7 +199,7 @@ default CompletableFuture getTransactionBufferStatsAsync * @return the stats of transaction buffer in topic. */ default TransactionBufferStats getTransactionBufferStats(String topic) throws PulsarAdminException { - return getTransactionBufferStats(topic, false); + return getTransactionBufferStats(topic, false, false); } /** @@ -309,6 +338,28 @@ CompletableFuture getPendingAckInternalStats TransactionPendingAckInternalStats getPendingAckInternalStats(String topic, String subName, boolean metadata) throws PulsarAdminException; + /** + * Get transaction buffer internal stats asynchronously. + * + * @param topic the topic to get transaction buffer internal stats from + * @param metadata whether to obtain ledger metadata + * + * @return the future internal stats of transaction buffer + */ + CompletableFuture getTransactionBufferInternalStatsAsync(String topic, + boolean metadata); + + /** + * Get transaction buffer internal stats. + * + * @param topic the topic to get transaction buffer internal stats from + * @param metadata whether to obtain ledger metadata + * + * @return the internal stats of transaction buffer + */ + TransactionBufferInternalStats getTransactionBufferInternalStats(String topic, + boolean metadata) throws PulsarAdminException; + /** * Sets the scale of the transaction coordinators. * And currently, we can only support scale-up. @@ -349,4 +400,18 @@ PositionInPendingAckStats getPositionStatsInPendingAck(String topic, String subN CompletableFuture getPositionStatsInPendingAckAsync(String topic, String subName, Long ledgerId, Long entryId, Integer batchIndex); + + /** + * Abort a transaction. + * + * @param txnID the txnId + */ + void abortTransaction(TxnID txnID) throws PulsarAdminException; + + /** + * Asynchronously abort a transaction. + * + * @param txnID the txnId + */ + CompletableFuture abortTransactionAsync(TxnID txnID); } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/io/SinkConfig.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/io/SinkConfig.java index 09b98249a4df1..57e67c0bcee0d 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/io/SinkConfig.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/io/SinkConfig.java @@ -94,4 +94,5 @@ public class SinkConfig { private String transformFunction; private String transformFunctionClassName; private String transformFunctionConfig; + private String logTopic; } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/io/SourceConfig.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/io/SourceConfig.java index 17b37008127ba..1991957045752 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/io/SourceConfig.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/io/SourceConfig.java @@ -59,6 +59,7 @@ public class SourceConfig { private FunctionConfig.ProcessingGuarantees processingGuarantees; private Resources resources; + private String sourceType; private String archive; // Any flags that you want to pass to the runtime. private String runtimeFlags; @@ -71,4 +72,5 @@ public class SourceConfig { private BatchSourceConfig batchSourceConfig; // batchBuilder provides two types of batch construction methods, DEFAULT and KEY_BASED private String batchBuilder; + private String logTopic; } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/BrokerInfo.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/BrokerInfo.java index 8955fe7a0ac78..19e9ff2d15a2b 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/BrokerInfo.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/BrokerInfo.java @@ -25,9 +25,11 @@ */ public interface BrokerInfo { String getServiceUrl(); + String getBrokerId(); interface Builder { Builder serviceUrl(String serviceUrl); + Builder brokerId(String brokerId); BrokerInfo build(); } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ClusterData.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ClusterData.java index 212a1575f9934..6aeed746db428 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ClusterData.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ClusterData.java @@ -19,9 +19,6 @@ package org.apache.pulsar.common.policies.data; import java.util.LinkedHashSet; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.pulsar.client.admin.utils.ReflectionUtils; import org.apache.pulsar.client.api.ProxyProtocol; @@ -68,11 +65,11 @@ public interface ClusterData { String getBrokerClientTlsKeyStore(); - String getListenerName(); + String getBrokerClientSslFactoryPlugin(); - boolean isMigrated(); + String getBrokerClientSslFactoryPluginParams(); - ClusterUrl getMigratedClusterUrl(); + String getListenerName(); interface Builder { Builder serviceUrl(String serviceUrl); @@ -119,9 +116,9 @@ interface Builder { Builder listenerName(String listenerName); - Builder migrated(boolean migrated); + Builder brokerClientSslFactoryPlugin(String sslFactoryPlugin); - Builder migratedClusterUrl(ClusterUrl migratedClusterUrl); + Builder brokerClientSslFactoryPluginParams(String sslFactoryPluginParams); ClusterData build(); } @@ -131,16 +128,4 @@ interface Builder { static Builder builder() { return ReflectionUtils.newBuilder("org.apache.pulsar.common.policies.data.ClusterDataImpl"); } - - @Data - @NoArgsConstructor - @AllArgsConstructor - class ClusterUrl { - String brokerServiceUrl; - String brokerServiceUrlTls; - - public boolean isEmpty() { - return brokerServiceUrl == null && brokerServiceUrlTls == null; - } - } } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ClusterPolicies.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ClusterPolicies.java new file mode 100644 index 0000000000000..b95f6bb19ce2e --- /dev/null +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ClusterPolicies.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.policies.data; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.apache.pulsar.client.admin.utils.ReflectionUtils; + +public interface ClusterPolicies { + boolean isMigrated(); + + ClusterUrl getMigratedClusterUrl(); + + interface Builder { + Builder migrated(boolean migrated); + + Builder migratedClusterUrl(ClusterUrl migratedClusterUrl); + + ClusterPolicies build(); + } + + Builder clone(); + + static Builder builder() { + return ReflectionUtils.newBuilder("org.apache.pulsar.common.policies.data.ClusterPoliciesImpl"); + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + @EqualsAndHashCode + class ClusterUrl { + String serviceUrl; + String serviceUrlTls; + String brokerServiceUrl; + String brokerServiceUrlTls; + + public boolean isEmpty() { + return serviceUrl != null && serviceUrlTls != null && brokerServiceUrl == null + && brokerServiceUrlTls == null; + } + } +} diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ConsumerStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ConsumerStats.java index 8c9a615d6d01c..5f2cf7b209ee9 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ConsumerStats.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ConsumerStats.java @@ -25,6 +25,9 @@ * Consumer statistics. */ public interface ConsumerStats { + /** the app id. */ + String getAppId(); + /** Total rate of messages delivered to the consumer (msg/s). */ double getMsgRateOut(); @@ -69,8 +72,8 @@ public interface ConsumerStats { /** Flag to verify if consumer is blocked due to reaching threshold of unacked messages. */ boolean isBlockedConsumerOnUnackedMsgs(); - /** The read position of the cursor when the consumer joining. */ - String getReadPositionWhenJoining(); + /** The last sent position of the cursor when the consumer joining. */ + String getLastSentPositionWhenJoining(); /** Address of this consumer. */ String getAddress(); diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/DelayedDeliveryPolicies.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/DelayedDeliveryPolicies.java index 555896ab3e597..f940ecd1b86ea 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/DelayedDeliveryPolicies.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/DelayedDeliveryPolicies.java @@ -26,10 +26,12 @@ public interface DelayedDeliveryPolicies { long getTickTime(); boolean isActive(); + long getMaxDeliveryDelayInMillis(); interface Builder { Builder tickTime(long tickTime); Builder active(boolean active); + Builder maxDeliveryDelayInMillis(long maxDeliveryDelayInMillis); DelayedDeliveryPolicies build(); } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java index 95a45d37d9556..b68b6308c8f3b 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java @@ -20,10 +20,14 @@ import java.util.List; import java.util.Map; +import lombok.AccessLevel; +import lombok.Getter; + /** * ManagedLedger internal statistics. */ +@Getter(AccessLevel.PUBLIC) public class ManagedLedgerInternalStats { /** Messages published since this broker loaded this managedLedger. */ @@ -82,6 +86,7 @@ public static class LedgerInfo { /** * Pulsar cursor statistics. */ + @Getter(AccessLevel.PUBLIC) public static class CursorStats { public String markDeletePosition; public String readPosition; diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/NamespaceIsolationData.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/NamespaceIsolationData.java index aa48e69c14571..4f367f72fda33 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/NamespaceIsolationData.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/NamespaceIsolationData.java @@ -31,6 +31,8 @@ public interface NamespaceIsolationData { AutoFailoverPolicyData getAutoFailoverPolicy(); + NamespaceIsolationPolicyUnloadScope getUnloadScope(); + void validate(); interface Builder { @@ -42,6 +44,8 @@ interface Builder { Builder autoFailoverPolicy(AutoFailoverPolicyData autoFailoverPolicyData); + Builder unloadScope(NamespaceIsolationPolicyUnloadScope unloadScope); + NamespaceIsolationData build(); } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/NamespaceIsolationPolicyUnloadScope.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/NamespaceIsolationPolicyUnloadScope.java new file mode 100644 index 0000000000000..2edeac45630f5 --- /dev/null +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/NamespaceIsolationPolicyUnloadScope.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.policies.data; + +/** + * The type of unload to perform while setting the isolation policy. + */ +public enum NamespaceIsolationPolicyUnloadScope { + all_matching, // unloads all matching namespaces as per new regex + none, // unloads no namespaces + changed; // unloads only the namespaces which are newly added or removed from the regex list + + public static NamespaceIsolationPolicyUnloadScope fromString(String unloadScopeString) { + for (NamespaceIsolationPolicyUnloadScope unloadScope : NamespaceIsolationPolicyUnloadScope.values()) { + if (unloadScope.toString().equalsIgnoreCase(unloadScopeString)) { + return unloadScope; + } + } + return null; + } +} diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/NonPersistentReplicatorStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/NonPersistentReplicatorStats.java index 6c77de9195786..bfeeb6d037a78 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/NonPersistentReplicatorStats.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/NonPersistentReplicatorStats.java @@ -27,4 +27,7 @@ public interface NonPersistentReplicatorStats extends ReplicatorStats { * for non-persistent topic: broker drops msg for replicator if replicator connection is not writable. **/ double getMsgDropRate(); + + /** Total number of messages dropped by the broker for the replicator. */ + long getMsgDropCount(); } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/Policies.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/Policies.java index 066fdf1df4f09..d5e08a1f50cc0 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/Policies.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/Policies.java @@ -36,6 +36,8 @@ public class Policies { public final AuthPolicies auth_policies = AuthPolicies.builder().build(); @SuppressWarnings("checkstyle:MemberName") public Set replication_clusters = new HashSet<>(); + @SuppressWarnings("checkstyle:MemberName") + public Set allowed_clusters = new HashSet<>(); public BundlesData bundles; @SuppressWarnings("checkstyle:MemberName") public Map backlog_quota_map = new HashMap<>(); @@ -126,6 +128,10 @@ public class Policies { @SuppressWarnings("checkstyle:MemberName") public String resource_group_name = null; + public boolean migrated; + + public Boolean dispatcherPauseOnAckStatePersistentEnabled; + public enum BundleType { LARGEST, HOT; } @@ -135,7 +141,7 @@ public enum BundleType { @Override public int hashCode() { - return Objects.hash(auth_policies, replication_clusters, + return Objects.hash(auth_policies, replication_clusters, allowed_clusters, backlog_quota_map, publishMaxMessageRate, clusterDispatchRate, topicDispatchRate, subscriptionDispatchRate, replicatorDispatchRate, clusterSubscribeRate, deduplicationEnabled, autoTopicCreationOverride, @@ -156,7 +162,8 @@ public int hashCode() { offload_policies, subscription_types_enabled, properties, - resource_group_name, entryFilters); + resource_group_name, entryFilters, migrated, + dispatcherPauseOnAckStatePersistentEnabled); } @Override @@ -165,6 +172,7 @@ public boolean equals(Object obj) { Policies other = (Policies) obj; return Objects.equals(auth_policies, other.auth_policies) && Objects.equals(replication_clusters, other.replication_clusters) + && Objects.equals(allowed_clusters, other.allowed_clusters) && Objects.equals(backlog_quota_map, other.backlog_quota_map) && Objects.equals(clusterDispatchRate, other.clusterDispatchRate) && Objects.equals(topicDispatchRate, other.topicDispatchRate) @@ -202,8 +210,11 @@ public boolean equals(Object obj) { && Objects.equals(offload_policies, other.offload_policies) && Objects.equals(subscription_types_enabled, other.subscription_types_enabled) && Objects.equals(properties, other.properties) + && Objects.equals(migrated, other.migrated) && Objects.equals(resource_group_name, other.resource_group_name) - && Objects.equals(entryFilters, other.entryFilters); + && Objects.equals(entryFilters, other.entryFilters) + && Objects.equals(dispatcherPauseOnAckStatePersistentEnabled, + other.dispatcherPauseOnAckStatePersistentEnabled); } return false; diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ReplicatorStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ReplicatorStats.java index 24be2f9380bb7..1790cc35f50c5 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ReplicatorStats.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ReplicatorStats.java @@ -24,20 +24,40 @@ public interface ReplicatorStats { /** Total rate of messages received from the remote cluster (msg/s). */ + @Deprecated double getMsgRateIn(); + /** Total number of messages received from the remote cluster. */ + long getMsgInCount(); + /** Total throughput received from the remote cluster (bytes/s). */ + @Deprecated double getMsgThroughputIn(); + /** Total number of bytes received from the remote cluster. */ + long getBytesInCount(); + /** Total rate of messages delivered to the replication-subscriber (msg/s). */ + @Deprecated double getMsgRateOut(); + /** Total number of messages sent to the remote cluster. */ + long getMsgOutCount(); + /** Total throughput delivered to the replication-subscriber (bytes/s). */ + @Deprecated double getMsgThroughputOut(); + /** Total number of bytes sent to the remote cluster. */ + long getBytesOutCount(); + /** Total rate of messages expired (msg/s). */ + @Deprecated double getMsgRateExpired(); + /** Total number of messages expired. */ + long getMsgExpiredCount(); + /** Number of messages pending to be replicated to remote cluster. */ long getReplicationBacklog(); diff --git a/pulsar-functions/api-java/src/test/java/org/apache/pulsar/functions/api/utils/JavaSerDeTest.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SchemaMetadata.java similarity index 58% rename from pulsar-functions/api-java/src/test/java/org/apache/pulsar/functions/api/utils/JavaSerDeTest.java rename to pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SchemaMetadata.java index 164709869b7ba..ff6ba6e86499e 100644 --- a/pulsar-functions/api-java/src/test/java/org/apache/pulsar/functions/api/utils/JavaSerDeTest.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SchemaMetadata.java @@ -16,36 +16,33 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.functions.api.utils; +package org.apache.pulsar.common.policies.data; -import static org.testng.Assert.assertEquals; -import java.io.Serializable; +import java.util.List; import lombok.AllArgsConstructor; import lombok.Data; -import org.testng.annotations.Test; +import lombok.NoArgsConstructor; /** - * Unit test of {@link JavaSerDe}. + * Schema metadata info. */ -public class JavaSerDeTest { +@Data +public class SchemaMetadata { + + public Entry info; + public List index; @Data @AllArgsConstructor - private static class TestObject implements Serializable { - - private int intField; - private String stringField; - - } - - @Test - public void testSerDe() { - TestObject to = new TestObject(1234, "test-serde-java-object"); - - byte[] data = JavaSerDe.of().serialize(to); - TestObject deserializeTo = (TestObject) JavaSerDe.of().deserialize(data); - - assertEquals(to, deserializeTo); + @NoArgsConstructor + public static class Entry { + private long ledgerId; + private long entryId; + private long version; + + @Override + public String toString() { + return String.format("ledgerId=[%d], entryId=[%d], version=[%d]", ledgerId, entryId, version); + } } - -} +} \ No newline at end of file diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SegmentStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SegmentStats.java new file mode 100644 index 0000000000000..007f4f4d63245 --- /dev/null +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SegmentStats.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.policies.data; + +public class SegmentStats { + public String lastTxnID; + public String persistentPosition; + + public SegmentStats(String lastTxnID, String persistentPosition) { + this.lastTxnID = lastTxnID; + this.persistentPosition = persistentPosition; + } + + public SegmentStats() { + } +} diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SegmentsStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SegmentsStats.java new file mode 100644 index 0000000000000..46422c0b67b58 --- /dev/null +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SegmentsStats.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.policies.data; + +import java.util.List; + +public class SegmentsStats { + // The current number of the snapshot segments. + public long segmentsSize; + + // The capacity of snapshot segment calculated by the current config (transactionBufferSnapshotSegmentSize) + public long currentSegmentCapacity; + + // The latest aborted txn IDs which number less than currentSegmentCapacity + public long unsealedAbortTxnIDSize; + + // A list of individual segment stats + public List segmentStats; + /** The last snapshot segment timestamps of this transaction buffer. */ + public long lastTookSnapshotSegmentTimestamp; +} diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SnapshotSystemTopicInternalStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SnapshotSystemTopicInternalStats.java new file mode 100644 index 0000000000000..7ce95375e7299 --- /dev/null +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SnapshotSystemTopicInternalStats.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.policies.data; + +public class SnapshotSystemTopicInternalStats { + // The managed ledger name for the snapshot segment topic or index topic. + public String managedLedgerName; + + // The managed ledger internal stats for the snapshot segment topic or index topic. + public ManagedLedgerInternalStats managedLedgerInternalStats; +} diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SubscriptionStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SubscriptionStats.java index 9ff94a2952ea3..e307e41862e74 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SubscriptionStats.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SubscriptionStats.java @@ -46,7 +46,7 @@ public interface SubscriptionStats { double getMessageAckRate(); /** Chunked message dispatch rate. */ - int getChunkedMessageRate(); + double getChunkedMessageRate(); /** Number of entries in the subscription backlog. */ long getMsgBacklog(); @@ -66,6 +66,9 @@ public interface SubscriptionStats { /** Number of delayed messages currently being tracked. */ long getMsgDelayed(); + /** Number of messages registered for replay. */ + long getMsgInReplay(); + /** * Number of unacknowledged messages for the subscription, where an unacknowledged message is one that has been * sent to a consumer but not yet acknowledged. Calculated by summing all {@link ConsumerStats#getUnackedMessages()} @@ -118,6 +121,12 @@ public interface SubscriptionStats { /** This is for Key_Shared subscription to get the recentJoinedConsumers in the Key_Shared subscription. */ Map getConsumersAfterMarkDeletePosition(); + /** The last sent position of the cursor. This is for Key_Shared subscription. */ + String getLastSentPosition(); + + /** Set of individually sent ranges. This is for Key_Shared subscription. */ + String getIndividuallySentPositions(); + /** SubscriptionProperties (key/value strings) associated with this subscribe. */ Map getSubscriptionProperties(); diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TopicStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TopicStats.java index 985e42b280eb9..ac50763b7e097 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TopicStats.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TopicStats.java @@ -64,6 +64,31 @@ public interface TopicStats { /** Get the publish time of the earliest message over all the backlogs. */ long getEarliestMsgPublishTimeInBacklogs(); + /** the size in bytes of the topic backlog quota. */ + long getBacklogQuotaLimitSize(); + + /** the topic backlog age quota, in seconds. */ + long getBacklogQuotaLimitTime(); + + /** + * Age of oldest unacknowledged message, as recorded in last backlog quota check interval. + *

    + * The age of the oldest unacknowledged (i.e. backlog) message, measured by the time elapsed from its published + * time, in seconds. This value is recorded every backlog quota check interval, hence it represents the value + * seen in the last check. + *

    + */ + long getOldestBacklogMessageAgeSeconds(); + + /** + * The subscription name containing oldest unacknowledged message as recorded in last backlog quota check. + *

    + * The name of the subscription containing the oldest unacknowledged message. This value is recorded every backlog + * quota check interval, hence it represents the value seen in the last check. + *

    + */ + String getOldestBacklogMessageSubscriptionName(); + /** Space used to store the offloaded messages for the topic/. */ long getOffloadedStorageSize(); diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TransactionBufferInternalStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TransactionBufferInternalStats.java new file mode 100644 index 0000000000000..b4c9e096d0a11 --- /dev/null +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TransactionBufferInternalStats.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.policies.data; + +public class TransactionBufferInternalStats { + // The type of snapshot being used: either "Single" or "Segment" + public String snapshotType; + + // If snapshotType is "Single", this field will provide the statistics of single snapshot log. + public SnapshotSystemTopicInternalStats singleSnapshotSystemTopicInternalStats; + + // If snapshotType is "Segment", this field will provide the statistics of snapshot segment topic. + public SnapshotSystemTopicInternalStats segmentInternalStats; + + // If snapshotType is "Segment", this field will provide the statistics of snapshot segment index topic. + public SnapshotSystemTopicInternalStats segmentIndexInternalStats; +} diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TransactionBufferStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TransactionBufferStats.java index 73d66b8c230bb..1dffa0dd61481 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TransactionBufferStats.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TransactionBufferStats.java @@ -44,4 +44,13 @@ public class TransactionBufferStats { public long recoverStartTime; //End timestamp of transaction buffer recovery. 0L means no startup. public long recoverEndTime; + + // The total number of aborted transactions. + public long totalAbortedTransactions; + + // The type of snapshot being used: either "Single" or "Segment" + public String snapshotType; + + // If snapshotType is "Segment", this field will provide additional segment-related statistics + public SegmentsStats segmentsStats; } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/BrokerInfoImpl.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/BrokerInfoImpl.java index e4d0a68b50ad0..d77f693c7cd70 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/BrokerInfoImpl.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/BrokerInfoImpl.java @@ -31,6 +31,7 @@ @NoArgsConstructor public final class BrokerInfoImpl implements BrokerInfo { private String serviceUrl; + private String brokerId; public static BrokerInfoImplBuilder builder() { return new BrokerInfoImplBuilder(); @@ -38,14 +39,20 @@ public static BrokerInfoImplBuilder builder() { public static class BrokerInfoImplBuilder implements BrokerInfo.Builder { private String serviceUrl; + private String brokerId; public BrokerInfoImplBuilder serviceUrl(String serviceUrl) { this.serviceUrl = serviceUrl; return this; } + public BrokerInfoImplBuilder brokerId(String brokerId) { + this.brokerId = brokerId; + return this; + } + public BrokerInfoImpl build() { - return new BrokerInfoImpl(serviceUrl); + return new BrokerInfoImpl(serviceUrl, brokerId); } } } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/DelayedDeliveryPoliciesImpl.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/DelayedDeliveryPoliciesImpl.java index 408217f363709..580ac6c95fa23 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/DelayedDeliveryPoliciesImpl.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/impl/DelayedDeliveryPoliciesImpl.java @@ -32,6 +32,7 @@ public final class DelayedDeliveryPoliciesImpl implements DelayedDeliveryPolicies { private long tickTime; private boolean active; + private long maxDeliveryDelayInMillis; public static DelayedDeliveryPoliciesImplBuilder builder() { return new DelayedDeliveryPoliciesImplBuilder(); @@ -40,6 +41,7 @@ public static DelayedDeliveryPoliciesImplBuilder builder() { public static class DelayedDeliveryPoliciesImplBuilder implements DelayedDeliveryPolicies.Builder { private long tickTime; private boolean active; + private long maxDeliveryDelayInMillis; public DelayedDeliveryPoliciesImplBuilder tickTime(long tickTime) { this.tickTime = tickTime; @@ -51,8 +53,13 @@ public DelayedDeliveryPoliciesImplBuilder active(boolean active) { return this; } + public DelayedDeliveryPoliciesImplBuilder maxDeliveryDelayInMillis(long maxDeliveryDelayInMillis) { + this.maxDeliveryDelayInMillis = maxDeliveryDelayInMillis; + return this; + } + public DelayedDeliveryPoliciesImpl build() { - return new DelayedDeliveryPoliciesImpl(tickTime, active); + return new DelayedDeliveryPoliciesImpl(tickTime, active, maxDeliveryDelayInMillis); } } } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/stats/AllocatorStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/stats/AllocatorStats.java index 3dbe831053a4b..aa23e2f755379 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/stats/AllocatorStats.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/stats/AllocatorStats.java @@ -29,6 +29,8 @@ public class AllocatorStats { public int numThreadLocalCaches; public int normalCacheSize; public int smallCacheSize; + public long usedDirectMemory; + public long usedHeapMemory; public List directArenas; public List heapArenas; diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LoadManagerReport.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LoadManagerReport.java index bf7371e6dd014..7e170e1d53764 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LoadManagerReport.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LoadManagerReport.java @@ -39,7 +39,7 @@ public interface LoadManagerReport extends ServiceLookupData { Map getBundleStats(); - int getNumTopics(); + long getNumTopics(); int getNumBundles(); diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index 4aaefe3a275b5..304610ad93ab2 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -26,8 +26,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-client-admin @@ -123,6 +122,8 @@ com.google.protobuf:protobuf-java com.google.guava:guava com.google.code.gson:gson + com.google.re2j:re2j + com.spotify:completable-futures com.fasterxml.jackson.*:* io.netty:* io.netty.incubator:* @@ -192,6 +193,10 @@ com.google.protobuf.* + + com.spotify.futures + org.apache.pulsar.shade.com.spotify.futures + com.fasterxml.jackson org.apache.pulsar.shade.com.fasterxml.jackson @@ -295,6 +300,12 @@ org.apache.bookkeeper org.apache.pulsar.shade.org.apache.bookkeeper + + + (META-INF/native/(lib)?)(netty.+\.(so|jnilib|dll))$ + $1org_apache_pulsar_shade_$3 + true + diff --git a/pulsar-client-admin/pom.xml b/pulsar-client-admin/pom.xml index c811236705971..e8b163e0939a4 100644 --- a/pulsar-client-admin/pom.xml +++ b/pulsar-client-admin/pom.xml @@ -26,8 +26,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-client-admin-original @@ -113,6 +112,13 @@ hamcrest test + + + com.github.tomakehurst + wiremock-jre8 + ${wiremock.version} + test + diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BaseResource.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BaseResource.java index 22550666cb698..ea39053c2ceeb 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BaseResource.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BaseResource.java @@ -62,11 +62,11 @@ public abstract class BaseResource { private static final Logger log = LoggerFactory.getLogger(BaseResource.class); protected final Authentication auth; - protected final long readTimeoutMs; + protected final long requestTimeoutMs; - protected BaseResource(Authentication auth, long readTimeoutMs) { + protected BaseResource(Authentication auth, long requestTimeoutMs) { this.auth = auth; - this.readTimeoutMs = readTimeoutMs; + this.requestTimeoutMs = requestTimeoutMs; } public Builder request(final WebTarget target) throws PulsarAdminException { @@ -339,7 +339,7 @@ public static String getReasonFromServer(WebApplicationException e) { protected T sync(Supplier> executor) throws PulsarAdminException { try { - return executor.get().get(this.readTimeoutMs, TimeUnit.MILLISECONDS); + return executor.get().get(this.requestTimeoutMs, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new PulsarAdminException(e); diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BookiesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BookiesImpl.java index 2286fb8c8a381..0bf92e0267791 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BookiesImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BookiesImpl.java @@ -32,8 +32,8 @@ public class BookiesImpl extends BaseResource implements Bookies { private final WebTarget adminBookies; - public BookiesImpl(WebTarget web, Authentication auth, long readTimeoutMs) { - super(auth, readTimeoutMs); + public BookiesImpl(WebTarget web, Authentication auth, long requestTimeoutMs) { + super(auth, requestTimeoutMs); adminBookies = web.path("/admin/v2/bookies"); } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BrokerStatsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BrokerStatsImpl.java index e409d6f4492de..6ddabe9837ef9 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BrokerStatsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BrokerStatsImpl.java @@ -38,8 +38,8 @@ public class BrokerStatsImpl extends BaseResource implements BrokerStats { private final WebTarget adminBrokerStats; private final WebTarget adminV2BrokerStats; - public BrokerStatsImpl(WebTarget target, Authentication auth, long readTimeoutMs) { - super(auth, readTimeoutMs); + public BrokerStatsImpl(WebTarget target, Authentication auth, long requestTimeoutMs) { + super(auth, requestTimeoutMs); adminBrokerStats = target.path("/admin/broker-stats"); adminV2BrokerStats = target.path("/admin/v2/broker-stats"); } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BrokersImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BrokersImpl.java index 0e6296724b3da..35b261b196eee 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BrokersImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/BrokersImpl.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import javax.ws.rs.client.Entity; import javax.ws.rs.client.InvocationCallback; @@ -37,8 +38,8 @@ public class BrokersImpl extends BaseResource implements Brokers { private final WebTarget adminBrokers; - public BrokersImpl(WebTarget web, Authentication auth, long readTimeoutMs) { - super(auth, readTimeoutMs); + public BrokersImpl(WebTarget web, Authentication auth, long requestTimeoutMs) { + super(auth, requestTimeoutMs); adminBrokers = web.path("admin/v2/brokers"); } @@ -75,15 +76,15 @@ public CompletableFuture getLeaderBrokerAsync() { } @Override - public Map getOwnedNamespaces(String cluster, String brokerUrl) + public Map getOwnedNamespaces(String cluster, String brokerId) throws PulsarAdminException { - return sync(() -> getOwnedNamespacesAsync(cluster, brokerUrl)); + return sync(() -> getOwnedNamespacesAsync(cluster, brokerId)); } @Override public CompletableFuture> getOwnedNamespacesAsync( - String cluster, String brokerUrl) { - WebTarget path = adminBrokers.path(cluster).path(brokerUrl).path("ownedNamespaces"); + String cluster, String brokerId) { + WebTarget path = adminBrokers.path(cluster).path(brokerId).path("ownedNamespaces"); return asyncGetRequest(path, new FutureCallback>(){}); } @@ -168,26 +169,35 @@ public CompletableFuture backlogQuotaCheckAsync() { @Override @Deprecated public void healthcheck() throws PulsarAdminException { - healthcheck(TopicVersion.V1); + healthcheck(TopicVersion.V1, Optional.empty()); } @Override @Deprecated public CompletableFuture healthcheckAsync() { - return healthcheckAsync(TopicVersion.V1); + return healthcheckAsync(TopicVersion.V1, Optional.empty()); } + @Override public void healthcheck(TopicVersion topicVersion) throws PulsarAdminException { - sync(() -> healthcheckAsync(topicVersion)); + sync(() -> healthcheckAsync(topicVersion, Optional.empty())); } @Override - public CompletableFuture healthcheckAsync(TopicVersion topicVersion) { + public void healthcheck(TopicVersion topicVersion, Optional brokerId) throws PulsarAdminException { + sync(() -> healthcheckAsync(topicVersion, brokerId)); + } + + @Override + public CompletableFuture healthcheckAsync(TopicVersion topicVersion, Optional brokerId) { WebTarget path = adminBrokers.path("health"); if (topicVersion != null) { path = path.queryParam("topicVersion", topicVersion); } + if (brokerId.isPresent()) { + path = path.queryParam("brokerId", brokerId.get()); + } final CompletableFuture future = new CompletableFuture<>(); asyncGetRequest(path, new InvocationCallback() { diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ClustersImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ClustersImpl.java index 02e44aca62604..24048ea3c0a41 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ClustersImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ClustersImpl.java @@ -34,8 +34,10 @@ import org.apache.pulsar.common.policies.data.BrokerNamespaceIsolationData; import org.apache.pulsar.common.policies.data.BrokerNamespaceIsolationDataImpl; import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.ClusterData.ClusterUrl; import org.apache.pulsar.common.policies.data.ClusterDataImpl; +import org.apache.pulsar.common.policies.data.ClusterPolicies; +import org.apache.pulsar.common.policies.data.ClusterPolicies.ClusterUrl; +import org.apache.pulsar.common.policies.data.ClusterPoliciesImpl; import org.apache.pulsar.common.policies.data.FailureDomain; import org.apache.pulsar.common.policies.data.FailureDomainImpl; import org.apache.pulsar.common.policies.data.NamespaceIsolationData; @@ -45,8 +47,8 @@ public class ClustersImpl extends BaseResource implements Clusters { private final WebTarget adminClusters; - public ClustersImpl(WebTarget web, Authentication auth, long readTimeoutMs) { - super(auth, readTimeoutMs); + public ClustersImpl(WebTarget web, Authentication auth, long requestTimeoutMs) { + super(auth, requestTimeoutMs); adminClusters = web.path("/admin/v2/clusters"); } @@ -107,6 +109,18 @@ public CompletableFuture updatePeerClusterNamesAsync(String cluster, Linke return asyncPostRequest(path, Entity.entity(peerClusterNames, MediaType.APPLICATION_JSON)); } + @Override + public ClusterPolicies getClusterMigration(String cluster) throws PulsarAdminException { + return sync(() -> getClusterMigrationAsync(cluster)); + } + + @Override + public CompletableFuture getClusterMigrationAsync(String cluster) { + WebTarget path = adminClusters.path(cluster).path("migrate"); + return asyncGetRequest(path, new FutureCallback() { + }).thenApply(policies -> policies); + } + @Override public void updateClusterMigration(String cluster, boolean isMigrated, ClusterUrl clusterUrl) throws PulsarAdminException { diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ComponentResource.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ComponentResource.java index 8beecff38975a..0301f0fc2ee2b 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ComponentResource.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ComponentResource.java @@ -37,8 +37,8 @@ */ public class ComponentResource extends BaseResource { - protected ComponentResource(Authentication auth, long readTimeoutMs) { - super(auth, readTimeoutMs); + protected ComponentResource(Authentication auth, long requestTimeoutMs) { + super(auth, requestTimeoutMs); } public RequestBuilder addAuthHeaders(WebTarget target, RequestBuilder requestBuilder) throws PulsarAdminException { diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java index bb4cb0c1ef8ef..bfcc3fe39a444 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java @@ -22,7 +22,6 @@ import static org.asynchttpclient.Dsl.post; import static org.asynchttpclient.Dsl.put; import com.google.gson.Gson; -import io.netty.handler.codec.http.HttpHeaders; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -41,6 +40,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.admin.Functions; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.admin.internal.http.AsyncHttpRequestExecutor; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.functions.FunctionDefinition; @@ -54,10 +54,8 @@ import org.apache.pulsar.common.policies.data.FunctionStats; import org.apache.pulsar.common.policies.data.FunctionStatsImpl; import org.apache.pulsar.common.policies.data.FunctionStatus; -import org.asynchttpclient.AsyncHandler; -import org.asynchttpclient.AsyncHttpClient; +import org.asynchttpclient.AsyncCompletionHandlerBase; import org.asynchttpclient.HttpResponseBodyPart; -import org.asynchttpclient.HttpResponseStatus; import org.asynchttpclient.RequestBuilder; import org.asynchttpclient.request.body.multipart.ByteArrayPart; import org.asynchttpclient.request.body.multipart.FilePart; @@ -70,12 +68,14 @@ public class FunctionsImpl extends ComponentResource implements Functions { private final WebTarget functions; - private final AsyncHttpClient asyncHttpClient; + private final AsyncHttpRequestExecutor asyncHttpRequestExecutor; - public FunctionsImpl(WebTarget web, Authentication auth, AsyncHttpClient asyncHttpClient, long readTimeoutMs) { - super(auth, readTimeoutMs); + public FunctionsImpl(WebTarget web, Authentication auth, + AsyncHttpRequestExecutor asyncHttpRequestExecutor, + long requestTimeoutMs) { + super(auth, requestTimeoutMs); this.functions = web.path("/admin/v3/functions"); - this.asyncHttpClient = asyncHttpClient; + this.asyncHttpRequestExecutor = asyncHttpRequestExecutor; } @Override @@ -171,8 +171,7 @@ public CompletableFuture createFunctionAsync(FunctionConfig functionConfig // If the function code is built in, we don't need to submit here builder.addBodyPart(new FilePart("data", new File(fileName), MediaType.APPLICATION_OCTET_STREAM)); } - asyncHttpClient.executeRequest(addAuthHeaders(functions, builder).build()) - .toCompletableFuture() + asyncHttpRequestExecutor.executeRequest(addAuthHeaders(functions, builder).build()) .thenAccept(response -> { if (response.getStatusCode() < 200 || response.getStatusCode() >= 300) { future.completeExceptionally( @@ -263,8 +262,7 @@ public CompletableFuture updateFunctionAsync( builder.addBodyPart(new FilePart("data", new File(fileName), MediaType.APPLICATION_OCTET_STREAM)); } - asyncHttpClient.executeRequest(addAuthHeaders(functions, builder).build()) - .toCompletableFuture() + asyncHttpRequestExecutor.executeRequest(addAuthHeaders(functions, builder).build()) .thenAccept(response -> { if (response.getStatusCode() < 200 || response.getStatusCode() >= 300) { future.completeExceptionally( @@ -464,7 +462,7 @@ public CompletableFuture uploadFunctionAsync(String sourceFile, String pat .addBodyPart(new FilePart("data", new File(sourceFile), MediaType.APPLICATION_OCTET_STREAM)) .addBodyPart(new StringPart("path", path, MediaType.TEXT_PLAIN)); - asyncHttpClient.executeRequest(addAuthHeaders(functions, builder).build()).toCompletableFuture() + asyncHttpRequestExecutor.executeRequest(addAuthHeaders(functions, builder).build()) .thenAccept(response -> { if (response.getStatusCode() < 200 || response.getStatusCode() >= 300) { future.completeExceptionally( @@ -543,55 +541,31 @@ private CompletableFuture downloadFileAsync(String destinationPath, WebTar RequestBuilder builder = get(target.getUri().toASCIIString()); - CompletableFuture statusFuture = - asyncHttpClient.executeRequest(addAuthHeaders(functions, builder).build(), - new AsyncHandler() { - private HttpResponseStatus status; - - @Override - public State onStatusReceived(HttpResponseStatus responseStatus) throws Exception { - status = responseStatus; - if (status.getStatusCode() != Response.Status.OK.getStatusCode()) { - return State.ABORT; - } - return State.CONTINUE; - } - - @Override - public State onHeadersReceived(HttpHeaders headers) throws Exception { - return State.CONTINUE; - } + CompletableFuture responseFuture = + asyncHttpRequestExecutor.executeRequest(addAuthHeaders(functions, builder).build(), + () -> new AsyncCompletionHandlerBase() { @Override public State onBodyPartReceived(HttpResponseBodyPart bodyPart) throws Exception { os.write(bodyPart.getBodyByteBuffer()); return State.CONTINUE; } + }); - @Override - public HttpResponseStatus onCompleted() throws Exception { - return status; - } - - @Override - public void onThrowable(Throwable t) { - } - }).toCompletableFuture(); - - statusFuture - .whenComplete((status, throwable) -> { + responseFuture + .whenComplete((response, throwable) -> { try { os.close(); } catch (IOException e) { future.completeExceptionally(getApiException(e)); } }) - .thenAccept(status -> { - if (status.getStatusCode() < 200 || status.getStatusCode() >= 300) { + .thenAccept(response -> { + if (response.getStatusCode() < 200 || response.getStatusCode() >= 300) { future.completeExceptionally( getApiException(Response - .status(status.getStatusCode()) - .entity(status.getStatusText()) + .status(response.getStatusCode()) + .entity(response.getStatusText()) .build())); } else { future.complete(null); @@ -700,7 +674,7 @@ public CompletableFuture putFunctionStateAsync( .path("state").path(state.getKey()).getUri().toASCIIString()); builder.addBodyPart(new StringPart("state", objectWriter() .writeValueAsString(state), MediaType.APPLICATION_JSON)); - asyncHttpClient.executeRequest(addAuthHeaders(functions, builder).build()) + asyncHttpRequestExecutor.executeRequest(addAuthHeaders(functions, builder).build()) .toCompletableFuture() .thenAccept(response -> { if (response.getStatusCode() < 200 || response.getStatusCode() >= 300) { @@ -740,7 +714,7 @@ public CompletableFuture updateOnWorkerLeaderAsync(String tenant, String n .addBodyPart(new ByteArrayPart("functionMetaData", functionMetaData)) .addBodyPart(new StringPart("delete", Boolean.toString(delete))); - asyncHttpClient.executeRequest(addAuthHeaders(functions, builder).build()) + asyncHttpRequestExecutor.executeRequest(addAuthHeaders(functions, builder).build()) .toCompletableFuture() .thenAccept(response -> { if (response.getStatusCode() < 200 || response.getStatusCode() >= 300) { diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java index 59f0ef3b34763..7d41c7203d2c7 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.client.admin.internal; -import static com.google.common.base.Preconditions.checkArgument; import java.util.List; import java.util.Map; import java.util.Set; @@ -65,8 +64,8 @@ public class NamespacesImpl extends BaseResource implements Namespaces { private final WebTarget adminNamespaces; private final WebTarget adminV2Namespaces; - public NamespacesImpl(WebTarget web, Authentication auth, long readTimeoutMs) { - super(auth, readTimeoutMs); + public NamespacesImpl(WebTarget web, Authentication auth, long requestTimeoutMs) { + super(auth, requestTimeoutMs); adminNamespaces = web.path("/admin/namespaces"); adminV2Namespaces = web.path("/admin/v2/namespaces"); } @@ -182,9 +181,7 @@ public void createNamespace(String namespace, Policies policies) throws PulsarAd @Override public CompletableFuture createNamespaceAsync(String namespace, Policies policies) { NamespaceName ns = NamespaceName.get(namespace); - checkArgument(ns.isV2(), "Create namespace with policies is only supported on newer namespaces"); - WebTarget path = namespacePath(ns); - // For V2 API we pass full Policy class instance + WebTarget path = ns.isV2() ? namespacePath(ns) : namespacePath(ns, "policy"); return asyncPutRequest(path, Entity.entity(policies, MediaType.APPLICATION_JSON)); } @@ -1883,7 +1880,6 @@ public void removeNamespaceResourceGroup(String namespace) throws PulsarAdminExc @Override public CompletableFuture clearPropertiesAsync(String namespace) { NamespaceName ns = NamespaceName.get(namespace); - final CompletableFuture future = new CompletableFuture<>(); WebTarget path = namespacePath(ns, "properties"); return asyncDeleteRequest(path); } @@ -1900,6 +1896,17 @@ public CompletableFuture removeNamespaceResourceGroupAsync(String namespac return asyncDeleteRequest(path); } + @Override + public void updateMigrationState(String namespace, boolean migrated) throws PulsarAdminException { + sync(() -> updateMigrationStateAsync(namespace, migrated)); + } + + public CompletableFuture updateMigrationStateAsync(String namespace, boolean migrated) { + NamespaceName ns = NamespaceName.get(namespace); + WebTarget path = namespacePath(ns, "migration"); + return asyncPostRequest(path, Entity.entity(migrated, MediaType.APPLICATION_JSON)); + } + private WebTarget namespacePath(NamespaceName namespace, String... parts) { final WebTarget base = namespace.isV2() ? adminV2Namespaces : adminNamespaces; WebTarget namespacePath = base.path(namespace.toString()); @@ -1950,4 +1957,64 @@ public CompletableFuture removeNamespaceEntryFiltersAsync(String namespace WebTarget path = namespacePath(ns, "entryFilters"); return asyncDeleteRequest(path); } + + @Override + public CompletableFuture setDispatcherPauseOnAckStatePersistentAsync(String namespace) { + NamespaceName ns = NamespaceName.get(namespace); + WebTarget path = namespacePath(ns, "dispatcherPauseOnAckStatePersistent"); + return asyncPostRequest(path, Entity.entity("", MediaType.APPLICATION_JSON)); + } + + @Override + public void setDispatcherPauseOnAckStatePersistent(String namespace) throws PulsarAdminException { + sync(() -> setDispatcherPauseOnAckStatePersistentAsync(namespace)); + } + + @Override + public CompletableFuture removeDispatcherPauseOnAckStatePersistentAsync(String namespace) { + NamespaceName ns = NamespaceName.get(namespace); + WebTarget path = namespacePath(ns, "dispatcherPauseOnAckStatePersistent"); + return asyncDeleteRequest(path); + } + + @Override + public void removeDispatcherPauseOnAckStatePersistent(String namespace) throws PulsarAdminException { + sync(() -> removeDispatcherPauseOnAckStatePersistentAsync(namespace)); + } + + @Override + public CompletableFuture getDispatcherPauseOnAckStatePersistentAsync(String namespace) { + NamespaceName ns = NamespaceName.get(namespace); + WebTarget path = namespacePath(ns, "dispatcherPauseOnAckStatePersistent"); + return asyncGetRequest(path, new FutureCallback(){}); + } + + @Override + public boolean getDispatcherPauseOnAckStatePersistent(String namespace) throws PulsarAdminException { + return sync(() -> getDispatcherPauseOnAckStatePersistentAsync(namespace)); + } + + @Override + public List getNamespaceAllowedClusters(String namespace) throws PulsarAdminException { + return sync(() -> getNamespaceAllowedClustersAsync(namespace)); + } + + @Override + public CompletableFuture> getNamespaceAllowedClustersAsync(String namespace) { + return asyncGetNamespaceParts(new FutureCallback>(){}, namespace, "allowedClusters"); + } + + @Override + public void setNamespaceAllowedClusters(String namespace, Set clusterIds) throws PulsarAdminException { + sync(() -> setNamespaceAllowedClustersAsync(namespace, clusterIds)); + } + + @Override + public CompletableFuture setNamespaceAllowedClustersAsync(String namespace, Set clusterIds) { + NamespaceName ns = NamespaceName.get(namespace); + WebTarget path = namespacePath(ns, "allowedClusters"); + return asyncPostRequest(path, Entity.entity(clusterIds, MediaType.APPLICATION_JSON)); + } + + } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NonPersistentTopicsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NonPersistentTopicsImpl.java index 76727cd1e0fc4..e98d44fdc4a69 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NonPersistentTopicsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NonPersistentTopicsImpl.java @@ -38,8 +38,8 @@ public class NonPersistentTopicsImpl extends BaseResource implements NonPersiste private final WebTarget adminNonPersistentTopics; private final WebTarget adminV2NonPersistentTopics; - public NonPersistentTopicsImpl(WebTarget web, Authentication auth, long readTimeoutMs) { - super(auth, readTimeoutMs); + public NonPersistentTopicsImpl(WebTarget web, Authentication auth, long requestTimeoutMs) { + super(auth, requestTimeoutMs); adminNonPersistentTopics = web.path("/admin"); adminV2NonPersistentTopics = web.path("/admin/v2"); } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PackagesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PackagesImpl.java index 694c2160b0f80..2b8efc3b97c8c 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PackagesImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PackagesImpl.java @@ -20,7 +20,6 @@ import static org.asynchttpclient.Dsl.get; import com.google.gson.Gson; -import io.netty.handler.codec.http.HttpHeaders; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -36,15 +35,14 @@ import javax.ws.rs.core.Response; import org.apache.pulsar.client.admin.Packages; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.admin.internal.http.AsyncHttpRequestExecutor; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.packages.management.core.common.PackageMetadata; import org.apache.pulsar.packages.management.core.common.PackageName; -import org.asynchttpclient.AsyncHandler; -import org.asynchttpclient.AsyncHttpClient; +import org.asynchttpclient.AsyncCompletionHandlerBase; import org.asynchttpclient.Dsl; import org.asynchttpclient.HttpResponseBodyPart; -import org.asynchttpclient.HttpResponseStatus; import org.asynchttpclient.RequestBuilder; import org.asynchttpclient.request.body.multipart.FilePart; import org.asynchttpclient.request.body.multipart.StringPart; @@ -55,11 +53,12 @@ public class PackagesImpl extends ComponentResource implements Packages { private final WebTarget packages; - private final AsyncHttpClient httpClient; + private final AsyncHttpRequestExecutor asyncHttpRequestExecutor; - public PackagesImpl(WebTarget webTarget, Authentication auth, AsyncHttpClient client, long readTimeoutMs) { - super(auth, readTimeoutMs); - this.httpClient = client; + public PackagesImpl(WebTarget webTarget, Authentication auth, AsyncHttpRequestExecutor asyncHttpRequestExecutor, + long requestTimeoutMs) { + super(auth, requestTimeoutMs); + this.asyncHttpRequestExecutor = asyncHttpRequestExecutor; this.packages = webTarget.path("/admin/v3/packages"); } @@ -98,7 +97,7 @@ public CompletableFuture uploadAsync(PackageMetadata metadata, String pack .post(packages.path(PackageName.get(packageName).toRestPath()).getUri().toASCIIString()) .addBodyPart(new FilePart("file", new File(path), MediaType.APPLICATION_OCTET_STREAM)) .addBodyPart(new StringPart("metadata", new Gson().toJson(metadata), MediaType.APPLICATION_JSON)); - httpClient.executeRequest(addAuthHeaders(packages, builder).build()) + asyncHttpRequestExecutor.executeRequest(addAuthHeaders(packages, builder).build()) .toCompletableFuture() .thenAccept(response -> { if (response.getStatusCode() < 200 || response.getStatusCode() >= 300) { @@ -138,55 +137,30 @@ public CompletableFuture downloadAsync(String packageName, String path) { FileChannel os = new FileOutputStream(destinyPath.toFile()).getChannel(); RequestBuilder builder = get(webTarget.getUri().toASCIIString()); - CompletableFuture statusFuture = - httpClient.executeRequest(addAuthHeaders(webTarget, builder).build(), - new AsyncHandler() { - private HttpResponseStatus status; + CompletableFuture responseFuture = + asyncHttpRequestExecutor.executeRequest(addAuthHeaders(webTarget, builder).build(), + () -> new AsyncCompletionHandlerBase() { - @Override - public State onStatusReceived(HttpResponseStatus httpResponseStatus) throws Exception { - status = httpResponseStatus; - if (status.getStatusCode() != Response.Status.OK.getStatusCode()) { - return State.ABORT; + @Override + public State onBodyPartReceived(HttpResponseBodyPart bodyPart) throws Exception { + os.write(bodyPart.getBodyByteBuffer()); + return State.CONTINUE; } - return State.CONTINUE; - } - - @Override - public State onHeadersReceived(HttpHeaders httpHeaders) throws Exception { - return State.CONTINUE; - } - - @Override - public State onBodyPartReceived(HttpResponseBodyPart httpResponseBodyPart) throws Exception { - os.write(httpResponseBodyPart.getBodyByteBuffer()); - return State.CONTINUE; - } - - @Override - public void onThrowable(Throwable throwable) { - // we don't need to handle that throwable and use the returned future to handle it. - } - - @Override - public HttpResponseStatus onCompleted() throws Exception { - return status; - } - }).toCompletableFuture(); - statusFuture - .whenComplete((status, throwable) -> { + }); + responseFuture + .whenComplete((response, throwable) -> { try { os.close(); } catch (IOException e) { future.completeExceptionally(getApiException(throwable)); } }) - .thenAccept(status -> { - if (status.getStatusCode() < 200 || status.getStatusCode() >= 300) { + .thenAccept(response -> { + if (response.getStatusCode() < 200 || response.getStatusCode() >= 300) { future.completeExceptionally( getApiException(Response - .status(status.getStatusCode()) - .entity(status.getStatusText()) + .status(response.getStatusCode()) + .entity(response.getStatusText()) .build())); } else { future.complete(null); diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ProxyStatsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ProxyStatsImpl.java index e98d9bf57b31e..7ed07a1a6ad54 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ProxyStatsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ProxyStatsImpl.java @@ -32,8 +32,8 @@ public class ProxyStatsImpl extends BaseResource implements ProxyStats { private final WebTarget adminProxyStats; - public ProxyStatsImpl(WebTarget target, Authentication auth, long readTimeoutMs) { - super(auth, readTimeoutMs); + public ProxyStatsImpl(WebTarget target, Authentication auth, long requestTimeoutMs) { + super(auth, requestTimeoutMs); adminProxyStats = target.path("/proxy-stats"); } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java index 009fa67fbaa29..7f0b3ab9a4218 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java @@ -38,14 +38,16 @@ public class PulsarAdminBuilderImpl implements PulsarAdminBuilder { protected ClientConfigurationData conf; private ClassLoader clientBuilderClassLoader = null; + private boolean acceptGzipCompression = true; @Override public PulsarAdmin build() throws PulsarClientException { - return new PulsarAdminImpl(conf.getServiceUrl(), conf, clientBuilderClassLoader); + return new PulsarAdminImpl(conf.getServiceUrl(), conf, clientBuilderClassLoader, acceptGzipCompression); } public PulsarAdminBuilderImpl() { this.conf = new ClientConfigurationData(); + this.conf.setConnectionsPerBroker(16); } private PulsarAdminBuilderImpl(ClientConfigurationData conf) { @@ -54,13 +56,33 @@ private PulsarAdminBuilderImpl(ClientConfigurationData conf) { @Override public PulsarAdminBuilder clone() { - return new PulsarAdminBuilderImpl(conf.clone()); + PulsarAdminBuilderImpl pulsarAdminBuilder = new PulsarAdminBuilderImpl(conf.clone()); + pulsarAdminBuilder.clientBuilderClassLoader = clientBuilderClassLoader; + pulsarAdminBuilder.acceptGzipCompression = acceptGzipCompression; + return pulsarAdminBuilder; } @Override public PulsarAdminBuilder loadConf(Map config) { conf = ConfigurationDataUtils.loadData(config, conf, ClientConfigurationData.class); setAuthenticationFromPropsIfAvailable(conf); + if (config.containsKey("acceptGzipCompression")) { + Object acceptGzipCompressionObj = config.get("acceptGzipCompression"); + if (acceptGzipCompressionObj instanceof Boolean) { + acceptGzipCompression = (Boolean) acceptGzipCompressionObj; + } else { + acceptGzipCompression = Boolean.parseBoolean(acceptGzipCompressionObj.toString()); + } + } + // in ClientConfigurationData, the maxConnectionsPerHost maps to connectionsPerBroker + if (config.containsKey("maxConnectionsPerHost")) { + Object maxConnectionsPerHostObj = config.get("maxConnectionsPerHost"); + if (maxConnectionsPerHostObj instanceof Integer) { + maxConnectionsPerHost((Integer) maxConnectionsPerHostObj); + } else { + maxConnectionsPerHost(Integer.parseInt(maxConnectionsPerHostObj.toString())); + } + } return this; } @@ -192,6 +214,20 @@ public PulsarAdminBuilder tlsCiphers(Set tlsCiphers) { return this; } + @Override + public PulsarAdminBuilder sslFactoryPlugin(String sslFactoryPlugin) { + if (StringUtils.isNotBlank(sslFactoryPlugin)) { + conf.setSslFactoryPlugin(sslFactoryPlugin); + } + return this; + } + + @Override + public PulsarAdminBuilder sslFactoryPluginParams(String sslFactoryPluginParams) { + conf.setSslFactoryPluginParams(sslFactoryPluginParams); + return this; + } + @Override public PulsarAdminBuilder tlsProtocols(Set tlsProtocols) { conf.setTlsProtocols(tlsProtocols); @@ -227,4 +263,24 @@ public PulsarAdminBuilder setContextClassLoader(ClassLoader clientBuilderClassLo this.clientBuilderClassLoader = clientBuilderClassLoader; return this; } + + @Override + public PulsarAdminBuilder acceptGzipCompression(boolean acceptGzipCompression) { + this.acceptGzipCompression = acceptGzipCompression; + return this; + } + + @Override + public PulsarAdminBuilder maxConnectionsPerHost(int maxConnectionsPerHost) { + // reuse the same configuration as the client, however for the admin client, the connection + // is usually established to a cluster address and not to a broker address + this.conf.setConnectionsPerBroker(maxConnectionsPerHost); + return this; + } + + @Override + public PulsarAdminBuilder connectionMaxIdleSeconds(int connectionMaxIdleSeconds) { + this.conf.setConnectionMaxIdleSeconds(connectionMaxIdleSeconds); + return this; + } } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminImpl.java index 259ca90cc08b7..aaea8a89f8db5 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminImpl.java @@ -106,6 +106,12 @@ public class PulsarAdminImpl implements PulsarAdmin { public PulsarAdminImpl(String serviceUrl, ClientConfigurationData clientConfigData, ClassLoader clientBuilderClassLoader) throws PulsarClientException { + this(serviceUrl, clientConfigData, clientBuilderClassLoader, true); + } + + public PulsarAdminImpl(String serviceUrl, ClientConfigurationData clientConfigData, + ClassLoader clientBuilderClassLoader, boolean acceptGzipCompression) + throws PulsarClientException { checkArgument(StringUtils.isNotBlank(serviceUrl), "Service URL needs to be specified"); this.clientConfigData = clientConfigData; @@ -119,7 +125,7 @@ public PulsarAdminImpl(String serviceUrl, ClientConfigurationData clientConfigDa } AsyncHttpConnectorProvider asyncConnectorProvider = new AsyncHttpConnectorProvider(clientConfigData, - clientConfigData.getAutoCertRefreshSeconds()); + clientConfigData.getAutoCertRefreshSeconds(), acceptGzipCompression); ClientConfig httpConfig = new ClientConfig(); httpConfig.property(ClientProperties.FOLLOW_REDIRECTS, true); @@ -153,29 +159,29 @@ public PulsarAdminImpl(String serviceUrl, ClientConfigurationData clientConfigDa Math.toIntExact(clientConfigData.getRequestTimeoutMs()), clientConfigData.getAutoCertRefreshSeconds()); - long readTimeoutMs = clientConfigData.getReadTimeoutMs(); - this.clusters = new ClustersImpl(root, auth, readTimeoutMs); - this.brokers = new BrokersImpl(root, auth, readTimeoutMs); - this.brokerStats = new BrokerStatsImpl(root, auth, readTimeoutMs); - this.proxyStats = new ProxyStatsImpl(root, auth, readTimeoutMs); - this.tenants = new TenantsImpl(root, auth, readTimeoutMs); - this.resourcegroups = new ResourceGroupsImpl(root, auth, readTimeoutMs); - this.properties = new TenantsImpl(root, auth, readTimeoutMs); - this.namespaces = new NamespacesImpl(root, auth, readTimeoutMs); - this.topics = new TopicsImpl(root, auth, readTimeoutMs); - this.localTopicPolicies = new TopicPoliciesImpl(root, auth, readTimeoutMs, false); - this.globalTopicPolicies = new TopicPoliciesImpl(root, auth, readTimeoutMs, true); - this.nonPersistentTopics = new NonPersistentTopicsImpl(root, auth, readTimeoutMs); - this.resourceQuotas = new ResourceQuotasImpl(root, auth, readTimeoutMs); - this.lookups = new LookupImpl(root, auth, useTls, readTimeoutMs, topics); - this.functions = new FunctionsImpl(root, auth, asyncHttpConnector.getHttpClient(), readTimeoutMs); - this.sources = new SourcesImpl(root, auth, asyncHttpConnector.getHttpClient(), readTimeoutMs); - this.sinks = new SinksImpl(root, auth, asyncHttpConnector.getHttpClient(), readTimeoutMs); - this.worker = new WorkerImpl(root, auth, readTimeoutMs); - this.schemas = new SchemasImpl(root, auth, readTimeoutMs); - this.bookies = new BookiesImpl(root, auth, readTimeoutMs); - this.packages = new PackagesImpl(root, auth, asyncHttpConnector.getHttpClient(), readTimeoutMs); - this.transactions = new TransactionsImpl(root, auth, readTimeoutMs); + long requestTimeoutMs = clientConfigData.getRequestTimeoutMs(); + this.clusters = new ClustersImpl(root, auth, requestTimeoutMs); + this.brokers = new BrokersImpl(root, auth, requestTimeoutMs); + this.brokerStats = new BrokerStatsImpl(root, auth, requestTimeoutMs); + this.proxyStats = new ProxyStatsImpl(root, auth, requestTimeoutMs); + this.tenants = new TenantsImpl(root, auth, requestTimeoutMs); + this.resourcegroups = new ResourceGroupsImpl(root, auth, requestTimeoutMs); + this.properties = new TenantsImpl(root, auth, requestTimeoutMs); + this.namespaces = new NamespacesImpl(root, auth, requestTimeoutMs); + this.topics = new TopicsImpl(root, auth, requestTimeoutMs); + this.localTopicPolicies = new TopicPoliciesImpl(root, auth, requestTimeoutMs, false); + this.globalTopicPolicies = new TopicPoliciesImpl(root, auth, requestTimeoutMs, true); + this.nonPersistentTopics = new NonPersistentTopicsImpl(root, auth, requestTimeoutMs); + this.resourceQuotas = new ResourceQuotasImpl(root, auth, requestTimeoutMs); + this.lookups = new LookupImpl(root, auth, useTls, requestTimeoutMs, topics); + this.functions = new FunctionsImpl(root, auth, asyncHttpConnector, requestTimeoutMs); + this.sources = new SourcesImpl(root, auth, asyncHttpConnector, requestTimeoutMs); + this.sinks = new SinksImpl(root, auth, asyncHttpConnector, requestTimeoutMs); + this.worker = new WorkerImpl(root, auth, requestTimeoutMs); + this.schemas = new SchemasImpl(root, auth, requestTimeoutMs); + this.bookies = new BookiesImpl(root, auth, requestTimeoutMs); + this.packages = new PackagesImpl(root, auth, asyncHttpConnector, requestTimeoutMs); + this.transactions = new TransactionsImpl(root, auth, requestTimeoutMs); if (originalCtxLoader != null) { Thread.currentThread().setContextClassLoader(originalCtxLoader); diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ResourceGroupsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ResourceGroupsImpl.java index a8cef60232fc0..4e7230eebd980 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ResourceGroupsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ResourceGroupsImpl.java @@ -32,8 +32,8 @@ public class ResourceGroupsImpl extends BaseResource implements ResourceGroups { private final WebTarget adminResourceGroups; - public ResourceGroupsImpl(WebTarget web, Authentication auth, long readTimeoutMs) { - super(auth, readTimeoutMs); + public ResourceGroupsImpl(WebTarget web, Authentication auth, long requestTimeoutMs) { + super(auth, requestTimeoutMs); adminResourceGroups = web.path("/admin/v2/resourcegroups"); } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ResourceQuotasImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ResourceQuotasImpl.java index 1e80c9eda94a5..68884d99448dd 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ResourceQuotasImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/ResourceQuotasImpl.java @@ -33,8 +33,8 @@ public class ResourceQuotasImpl extends BaseResource implements ResourceQuotas { private final WebTarget adminQuotas; private final WebTarget adminV2Quotas; - public ResourceQuotasImpl(WebTarget web, Authentication auth, long readTimeoutMs) { - super(auth, readTimeoutMs); + public ResourceQuotasImpl(WebTarget web, Authentication auth, long requestTimeoutMs) { + super(auth, requestTimeoutMs); adminQuotas = web.path("/admin/resource-quotas"); adminV2Quotas = web.path("/admin/v2/resource-quotas"); } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/SchemasImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/SchemasImpl.java index 593eb67fc0dc3..7f2383e1e52ef 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/SchemasImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/SchemasImpl.java @@ -31,6 +31,7 @@ import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.internal.DefaultImplementation; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.SchemaMetadata; import org.apache.pulsar.common.protocol.schema.DeleteSchemaResponse; import org.apache.pulsar.common.protocol.schema.GetAllVersionsSchemaResponse; import org.apache.pulsar.common.protocol.schema.GetSchemaResponse; @@ -46,8 +47,8 @@ public class SchemasImpl extends BaseResource implements Schemas { private final WebTarget adminV2; private final WebTarget adminV1; - public SchemasImpl(WebTarget web, Authentication auth, long readTimeoutMs) { - super(auth, readTimeoutMs); + public SchemasImpl(WebTarget web, Authentication auth, long requestTimeoutMs) { + super(auth, requestTimeoutMs); this.adminV1 = web.path("/admin/schemas"); this.adminV2 = web.path("/admin/v2/schemas"); } @@ -276,6 +277,19 @@ public CompletableFuture> getAllSchemasAsync(String topic) { .collect(Collectors.toList())); } + @Override + public SchemaMetadata getSchemaMetadata(String topic) throws PulsarAdminException { + return sync(() -> getSchemaMetadataAsync(topic)); + } + + @Override + public CompletableFuture getSchemaMetadataAsync(String topic) { + TopicName tn = TopicName.get(topic); + WebTarget path = metadata(tn); + return asyncGetRequest(path, new FutureCallback(){}); + } + + private WebTarget schemaPath(TopicName topicName) { return topicPath(topicName, "schema"); } @@ -292,6 +306,10 @@ private WebTarget compatibilityPath(TopicName topicName) { return topicPath(topicName, "compatibility"); } + private WebTarget metadata(TopicName topicName) { + return topicPath(topicName, "metadata"); + } + private WebTarget topicPath(TopicName topic, String... parts) { final WebTarget base = topic.isV2() ? adminV2 : adminV1; WebTarget topicPath = base.path(topic.getRestPath(false)); diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/SinksImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/SinksImpl.java index c14f75ab36750..bba0289d81254 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/SinksImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/SinksImpl.java @@ -34,13 +34,13 @@ import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.admin.Sink; import org.apache.pulsar.client.admin.Sinks; +import org.apache.pulsar.client.admin.internal.http.AsyncHttpRequestExecutor; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.common.functions.UpdateOptions; import org.apache.pulsar.common.functions.UpdateOptionsImpl; import org.apache.pulsar.common.io.ConnectorDefinition; import org.apache.pulsar.common.io.SinkConfig; import org.apache.pulsar.common.policies.data.SinkStatus; -import org.asynchttpclient.AsyncHttpClient; import org.asynchttpclient.RequestBuilder; import org.asynchttpclient.request.body.multipart.FilePart; import org.asynchttpclient.request.body.multipart.StringPart; @@ -51,12 +51,13 @@ public class SinksImpl extends ComponentResource implements Sinks, Sink { private final WebTarget sink; - private final AsyncHttpClient asyncHttpClient; + private final AsyncHttpRequestExecutor asyncHttpRequestExecutor; - public SinksImpl(WebTarget web, Authentication auth, AsyncHttpClient asyncHttpClient, long readTimeoutMs) { - super(auth, readTimeoutMs); + public SinksImpl(WebTarget web, Authentication auth, AsyncHttpRequestExecutor asyncHttpRequestExecutor, + long requestTimeoutMs) { + super(auth, requestTimeoutMs); this.sink = web.path("/admin/v3/sink"); - this.asyncHttpClient = asyncHttpClient; + this.asyncHttpRequestExecutor = asyncHttpRequestExecutor; } @Override @@ -145,7 +146,7 @@ public CompletableFuture createSinkAsync(SinkConfig sinkConfig, String fil // If the function code is built in, we don't need to submit here builder.addBodyPart(new FilePart("data", new File(fileName), MediaType.APPLICATION_OCTET_STREAM)); } - asyncHttpClient.executeRequest(addAuthHeaders(sink, builder).build()) + asyncHttpRequestExecutor.executeRequest(addAuthHeaders(sink, builder).build()) .toCompletableFuture() .thenAccept(response -> { if (response.getStatusCode() < 200 || response.getStatusCode() >= 300) { @@ -233,7 +234,7 @@ public CompletableFuture updateSinkAsync( // If the function code is built in, we don't need to submit here builder.addBodyPart(new FilePart("data", new File(fileName), MediaType.APPLICATION_OCTET_STREAM)); } - asyncHttpClient.executeRequest(addAuthHeaders(sink, builder).build()) + asyncHttpRequestExecutor.executeRequest(addAuthHeaders(sink, builder).build()) .toCompletableFuture() .thenAccept(response -> { if (response.getStatusCode() < 200 || response.getStatusCode() >= 300) { diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/SourcesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/SourcesImpl.java index 6e5b84c7f0412..56cf7db229b78 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/SourcesImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/SourcesImpl.java @@ -33,13 +33,13 @@ import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.admin.Source; import org.apache.pulsar.client.admin.Sources; +import org.apache.pulsar.client.admin.internal.http.AsyncHttpRequestExecutor; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.common.functions.UpdateOptions; import org.apache.pulsar.common.functions.UpdateOptionsImpl; import org.apache.pulsar.common.io.ConnectorDefinition; import org.apache.pulsar.common.io.SourceConfig; import org.apache.pulsar.common.policies.data.SourceStatus; -import org.asynchttpclient.AsyncHttpClient; import org.asynchttpclient.RequestBuilder; import org.asynchttpclient.request.body.multipart.FilePart; import org.asynchttpclient.request.body.multipart.StringPart; @@ -50,12 +50,13 @@ public class SourcesImpl extends ComponentResource implements Sources, Source { private final WebTarget source; - private final AsyncHttpClient asyncHttpClient; + private final AsyncHttpRequestExecutor asyncHttpRequestExecutor; - public SourcesImpl(WebTarget web, Authentication auth, AsyncHttpClient asyncHttpClient, long readTimeoutMs) { - super(auth, readTimeoutMs); + public SourcesImpl(WebTarget web, Authentication auth, AsyncHttpRequestExecutor asyncHttpRequestExecutor, + long requestTimeoutMs) { + super(auth, requestTimeoutMs); this.source = web.path("/admin/v3/source"); - this.asyncHttpClient = asyncHttpClient; + this.asyncHttpRequestExecutor = asyncHttpRequestExecutor; } @Override @@ -124,7 +125,7 @@ public CompletableFuture createSourceAsync(SourceConfig sourceConfig, Stri // If the function code is built in, we don't need to submit here builder.addBodyPart(new FilePart("data", new File(fileName), MediaType.APPLICATION_OCTET_STREAM)); } - asyncHttpClient.executeRequest(addAuthHeaders(source, builder).build()) + asyncHttpRequestExecutor.executeRequest(addAuthHeaders(source, builder).build()) .toCompletableFuture() .thenAccept(response -> { if (response.getStatusCode() < 200 || response.getStatusCode() >= 300) { @@ -202,7 +203,7 @@ public CompletableFuture updateSourceAsync( // If the function code is built in, we don't need to submit here builder.addBodyPart(new FilePart("data", new File(fileName), MediaType.APPLICATION_OCTET_STREAM)); } - asyncHttpClient.executeRequest(addAuthHeaders(source, builder).build()) + asyncHttpRequestExecutor.executeRequest(addAuthHeaders(source, builder).build()) .toCompletableFuture() .thenAccept(response -> { if (response.getStatusCode() < 200 || response.getStatusCode() >= 300) { diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TenantsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TenantsImpl.java index 9b70e39ec4986..c12f3754b4a92 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TenantsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TenantsImpl.java @@ -34,8 +34,8 @@ public class TenantsImpl extends BaseResource implements Tenants, Properties { private final WebTarget adminTenants; - public TenantsImpl(WebTarget web, Authentication auth, long readTimeoutMs) { - super(auth, readTimeoutMs); + public TenantsImpl(WebTarget web, Authentication auth, long requestTimeoutMs) { + super(auth, requestTimeoutMs); adminTenants = web.path("/admin/v2/tenants"); } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicPoliciesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicPoliciesImpl.java index 915b22a25898a..f58fd86542838 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicPoliciesImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicPoliciesImpl.java @@ -1260,6 +1260,27 @@ public CompletableFuture removeAutoSubscriptionCreationAsync(String topic) return asyncDeleteRequest(path); } + @Override + public CompletableFuture setDispatcherPauseOnAckStatePersistent(String topic) { + TopicName tn = validateTopic(topic); + WebTarget path = topicPath(tn, "dispatcherPauseOnAckStatePersistent"); + return asyncPostRequest(path, Entity.entity("", MediaType.APPLICATION_JSON)); + } + + @Override + public CompletableFuture removeDispatcherPauseOnAckStatePersistent(String topic) { + TopicName tn = validateTopic(topic); + WebTarget path = topicPath(tn, "dispatcherPauseOnAckStatePersistent"); + return asyncDeleteRequest(path); + } + + @Override + public CompletableFuture getDispatcherPauseOnAckStatePersistent(String topic, boolean applied) { + TopicName tn = validateTopic(topic); + WebTarget path = topicPath(tn, "dispatcherPauseOnAckStatePersistent").queryParam("applied", applied); + return asyncGetRequest(path, new FutureCallback(){}); + } + /* * returns topic name with encoded Local Name */ diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java index 33d1cd1785827..9c4a6eef753de 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java @@ -19,6 +19,7 @@ package org.apache.pulsar.client.admin.internal; import static com.google.common.base.Preconditions.checkArgument; +import com.google.gson.Gson; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import java.io.InputStream; @@ -55,6 +56,7 @@ import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.TransactionIsolationLevel; import org.apache.pulsar.client.impl.BatchMessageIdImpl; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.impl.MessageImpl; @@ -129,12 +131,14 @@ public class TopicsImpl extends BaseResource implements Topics { private static final String SCHEMA_VERSION = "X-Pulsar-Base64-schema-version-b64encoded"; private static final String ENCRYPTION_PARAM = "X-Pulsar-Base64-encryption-param"; private static final String ENCRYPTION_KEYS = "X-Pulsar-Base64-encryption-keys"; + public static final String TXN_ABORTED = "X-Pulsar-txn-aborted"; + public static final String TXN_UNCOMMITTED = "X-Pulsar-txn-uncommitted"; // CHECKSTYLE.ON: MemberName public static final String PROPERTY_SHADOW_SOURCE_KEY = "PULSAR.SHADOW_SOURCE"; - public TopicsImpl(WebTarget web, Authentication auth, long readTimeoutMs) { - super(auth, readTimeoutMs); + public TopicsImpl(WebTarget web, Authentication auth, long requestTimeoutMs) { + super(auth, requestTimeoutMs); adminTopics = web.path("/admin"); adminV2Topics = web.path("/admin/v2"); } @@ -590,21 +594,27 @@ public CompletableFuture> getSubscriptionsAsync(String topic) { @Override public TopicStats getStats(String topic, GetStatsOptions getStatsOptions) throws PulsarAdminException { - boolean getPreciseBacklog = getStatsOptions.isGetPreciseBacklog(); - boolean subscriptionBacklogSize = getStatsOptions.isSubscriptionBacklogSize(); - boolean getEarliestTimeInBacklog = getStatsOptions.isGetEarliestTimeInBacklog(); - return sync(() -> getStatsAsync(topic, getPreciseBacklog, subscriptionBacklogSize, getEarliestTimeInBacklog)); + return sync(() -> getStatsAsync(topic, getStatsOptions)); } @Override public CompletableFuture getStatsAsync(String topic, boolean getPreciseBacklog, boolean subscriptionBacklogSize, boolean getEarliestTimeInBacklog) { + GetStatsOptions getStatsOptions = new GetStatsOptions(getPreciseBacklog, subscriptionBacklogSize, + getEarliestTimeInBacklog, false, false); + return getStatsAsync(topic, getStatsOptions); + } + + @Override + public CompletableFuture getStatsAsync(String topic, GetStatsOptions getStatsOptions) { TopicName tn = validateTopic(topic); WebTarget path = topicPath(tn, "stats") - .queryParam("getPreciseBacklog", getPreciseBacklog) - .queryParam("subscriptionBacklogSize", subscriptionBacklogSize) - .queryParam("getEarliestTimeInBacklog", getEarliestTimeInBacklog); + .queryParam("getPreciseBacklog", getStatsOptions.isGetPreciseBacklog()) + .queryParam("subscriptionBacklogSize", getStatsOptions.isSubscriptionBacklogSize()) + .queryParam("getEarliestTimeInBacklog", getStatsOptions.isGetEarliestTimeInBacklog()) + .queryParam("excludePublishers", getStatsOptions.isExcludePublishers()) + .queryParam("excludeConsumers", getStatsOptions.isExcludeConsumers()); final CompletableFuture future = new CompletableFuture<>(); InvocationCallback persistentCB = new InvocationCallback() { @@ -621,16 +631,16 @@ public void failed(Throwable throwable) { InvocationCallback nonpersistentCB = new InvocationCallback() { - @Override - public void completed(NonPersistentTopicStats response) { - future.complete(response); - } + @Override + public void completed(NonPersistentTopicStats response) { + future.complete(response); + } - @Override - public void failed(Throwable throwable) { - future.completeExceptionally(getApiException(throwable.getCause())); - } - }; + @Override + public void failed(Throwable throwable) { + future.completeExceptionally(getApiException(throwable.getCause())); + } + }; if (topic.startsWith(TopicDomain.non_persistent.value())) { asyncGetRequest(path, nonpersistentCB); @@ -684,34 +694,50 @@ public PartitionedTopicStats getPartitionedStats(String topic, boolean perPartit subscriptionBacklogSize, getEarliestTimeInBacklog)); } + @Override + public PartitionedTopicStats getPartitionedStats(String topic, boolean perPartition, + GetStatsOptions getStatsOptions) throws PulsarAdminException { + return sync(()-> getPartitionedStatsAsync(topic, perPartition, getStatsOptions)); + } + @Override public CompletableFuture getPartitionedStatsAsync(String topic, boolean perPartition, boolean getPreciseBacklog, boolean subscriptionBacklogSize, boolean getEarliestTimeInBacklog) { + GetStatsOptions getStatsOptions = new GetStatsOptions(getPreciseBacklog, subscriptionBacklogSize, + getEarliestTimeInBacklog, false, false); + return getPartitionedStatsAsync(topic, perPartition, getStatsOptions); + } + + @Override + public CompletableFuture getPartitionedStatsAsync(String topic, boolean perPartition, + GetStatsOptions getStatsOptions) { TopicName tn = validateTopic(topic); WebTarget path = topicPath(tn, "partitioned-stats"); path = path.queryParam("perPartition", perPartition) - .queryParam("getPreciseBacklog", getPreciseBacklog) - .queryParam("subscriptionBacklogSize", subscriptionBacklogSize) - .queryParam("getEarliestTimeInBacklog", getEarliestTimeInBacklog); + .queryParam("getPreciseBacklog", getStatsOptions.isGetPreciseBacklog()) + .queryParam("subscriptionBacklogSize", getStatsOptions.isSubscriptionBacklogSize()) + .queryParam("getEarliestTimeInBacklog", getStatsOptions.isGetEarliestTimeInBacklog()) + .queryParam("excludePublishers", getStatsOptions.isExcludePublishers()) + .queryParam("excludeConsumers", getStatsOptions.isExcludeConsumers()); final CompletableFuture future = new CompletableFuture<>(); InvocationCallback nonpersistentCB = new InvocationCallback() { - @Override - public void completed(NonPersistentPartitionedTopicStats response) { - if (!perPartition) { - response.getPartitions().clear(); - } - future.complete(response); - } + @Override + public void completed(NonPersistentPartitionedTopicStats response) { + if (!perPartition) { + response.getPartitions().clear(); + } + future.complete(response); + } - @Override - public void failed(Throwable throwable) { - future.completeExceptionally(getApiException(throwable.getCause())); - } - }; + @Override + public void failed(Throwable throwable) { + future.completeExceptionally(getApiException(throwable.getCause())); + } + }; InvocationCallback persistentCB = new InvocationCallback() { @@ -844,7 +870,9 @@ public CompletableFuture expireMessagesForAllSubscriptionsAsync(String top return asyncPostRequest(path, Entity.entity("", MediaType.APPLICATION_JSON)); } - private CompletableFuture>> peekNthMessage(String topic, String subName, int messagePosition) { + private CompletableFuture>> peekNthMessage( + String topic, String subName, int messagePosition, boolean showServerMarker, + TransactionIsolationLevel transactionIsolationLevel) { TopicName tn = validateTopic(topic); String encodedSubName = Codec.encode(subName); WebTarget path = topicPath(tn, "subscription", encodedSubName, @@ -856,7 +884,8 @@ private CompletableFuture>> peekNthMessage(String topic, St @Override public void completed(Response response) { try { - future.complete(getMessagesFromHttpResponse(tn.toString(), response)); + future.complete(getMessagesFromHttpResponse(tn.toString(), response, + showServerMarker, transactionIsolationLevel)); } catch (Exception e) { future.completeExceptionally(getApiException(e)); } @@ -871,28 +900,35 @@ public void failed(Throwable throwable) { } @Override - public List> peekMessages(String topic, String subName, int numMessages) + public List> peekMessages(String topic, String subName, int numMessages, + boolean showServerMarker, + TransactionIsolationLevel transactionIsolationLevel) throws PulsarAdminException { - return sync(() -> peekMessagesAsync(topic, subName, numMessages)); + return sync(() -> peekMessagesAsync(topic, subName, numMessages, showServerMarker, transactionIsolationLevel)); } @Override - public CompletableFuture>> peekMessagesAsync(String topic, String subName, int numMessages) { + public CompletableFuture>> peekMessagesAsync( + String topic, String subName, int numMessages, + boolean showServerMarker, TransactionIsolationLevel transactionIsolationLevel) { checkArgument(numMessages > 0); CompletableFuture>> future = new CompletableFuture>>(); - peekMessagesAsync(topic, subName, numMessages, new ArrayList<>(), future, 1); + peekMessagesAsync(topic, subName, numMessages, new ArrayList<>(), + future, 1, showServerMarker, transactionIsolationLevel); return future; } private void peekMessagesAsync(String topic, String subName, int numMessages, - List> messages, CompletableFuture>> future, int nthMessage) { + List> messages, CompletableFuture>> future, int nthMessage, + boolean showServerMarker, TransactionIsolationLevel transactionIsolationLevel) { if (numMessages <= 0) { future.complete(messages); return; } // if peeking first message succeeds, we know that the topic and subscription exists - peekNthMessage(topic, subName, nthMessage).handle((r, ex) -> { + peekNthMessage(topic, subName, nthMessage, showServerMarker, transactionIsolationLevel) + .handle((r, ex) -> { if (ex != null) { // if we get a not found exception, it means that the position for the message we are trying to get // does not exist. At this point, we can return the already found messages. @@ -907,7 +943,8 @@ private void peekMessagesAsync(String topic, String subName, int numMessages, for (int i = 0; i < Math.min(r.size(), numMessages); i++) { messages.add(r.get(i)); } - peekMessagesAsync(topic, subName, numMessages - r.size(), messages, future, nthMessage + 1); + peekMessagesAsync(topic, subName, numMessages - r.size(), messages, future, + nthMessage + 1, showServerMarker, transactionIsolationLevel); return null; }); } @@ -963,34 +1000,16 @@ public CompletableFuture truncateAsync(String topic) { } @Override - public CompletableFuture> getMessageByIdAsync(String topic, long ledgerId, long entryId) { - CompletableFuture> future = new CompletableFuture<>(); - getRemoteMessageById(topic, ledgerId, entryId).handle((r, ex) -> { - if (ex != null) { - if (ex instanceof NotFoundException) { - log.warn("Exception '{}' occurred while trying to get message.", ex.getMessage()); - future.complete(r); - } else { - future.completeExceptionally(ex); - } - return null; - } - future.complete(r); - return null; - }); - return future; - } - - private CompletableFuture> getRemoteMessageById(String topic, long ledgerId, long entryId) { + public CompletableFuture>> getMessagesByIdAsync(String topic, long ledgerId, long entryId) { TopicName topicName = validateTopic(topic); WebTarget path = topicPath(topicName, "ledger", Long.toString(ledgerId), "entry", Long.toString(entryId)); - final CompletableFuture> future = new CompletableFuture<>(); + final CompletableFuture>> future = new CompletableFuture<>(); asyncGetRequest(path, new InvocationCallback() { @Override public void completed(Response response) { try { - future.complete(getMessagesFromHttpResponse(topicName.toString(), response).get(0)); + future.complete(getMessagesFromHttpResponse(topicName.toString(), response)); } catch (Exception e) { future.completeExceptionally(getApiException(e)); } @@ -1004,6 +1023,19 @@ public void failed(Throwable throwable) { return future; } + @Override + public List> getMessagesById(String topic, long ledgerId, long entryId) + throws PulsarAdminException { + return sync(() -> getMessagesByIdAsync(topic, ledgerId, entryId)); + } + + @Deprecated + @Override + public CompletableFuture> getMessageByIdAsync(String topic, long ledgerId, long entryId) { + return getMessagesByIdAsync(topic, ledgerId, entryId).thenApply(n -> n.get(0)); + } + + @Deprecated @Override public Message getMessageById(String topic, long ledgerId, long entryId) throws PulsarAdminException { @@ -1235,6 +1267,13 @@ private TopicName validateTopic(String topic) { } private List> getMessagesFromHttpResponse(String topic, Response response) throws Exception { + return getMessagesFromHttpResponse(topic, response, true, + TransactionIsolationLevel.READ_UNCOMMITTED); + } + + private List> getMessagesFromHttpResponse( + String topic, Response response, boolean showServerMarker, + TransactionIsolationLevel transactionIsolationLevel) throws Exception { if (response.getStatus() != Status.OK.getStatusCode()) { throw getApiException(response); @@ -1266,7 +1305,32 @@ private List> getMessagesFromHttpResponse(String topic, Response Map properties = new TreeMap<>(); MultivaluedMap headers = response.getHeaders(); - Object tmp = headers.getFirst(PUBLISH_TIME); + Object tmp = headers.getFirst(MARKER_TYPE); + if (tmp != null) { + if (!showServerMarker) { + return new ArrayList<>(); + } else { + messageMetadata.setMarkerType(Integer.parseInt(tmp.toString())); + } + } + + tmp = headers.getFirst(TXN_ABORTED); + if (tmp != null && Boolean.parseBoolean(tmp.toString())) { + properties.put(TXN_ABORTED, tmp.toString()); + if (transactionIsolationLevel == TransactionIsolationLevel.READ_COMMITTED) { + return new ArrayList<>(); + } + } + + tmp = headers.getFirst(TXN_UNCOMMITTED); + if (tmp != null && Boolean.parseBoolean(tmp.toString())) { + properties.put(TXN_UNCOMMITTED, tmp.toString()); + if (transactionIsolationLevel == TransactionIsolationLevel.READ_COMMITTED) { + return new ArrayList<>(); + } + } + + tmp = headers.getFirst(PUBLISH_TIME); if (tmp != null) { messageMetadata.setPublishTime(DateFormatter.parse(tmp.toString())); } @@ -1318,10 +1382,6 @@ private List> getMessagesFromHttpResponse(String topic, Response if (tmp != null) { messageMetadata.setPartitionKeyB64Encoded(Boolean.parseBoolean(tmp.toString())); } - tmp = headers.getFirst(MARKER_TYPE); - if (tmp != null) { - messageMetadata.setMarkerType(Integer.parseInt(tmp.toString())); - } tmp = headers.getFirst(TXNID_LEAST_BITS); if (tmp != null) { messageMetadata.setTxnidLeastBits(Long.parseLong(tmp.toString())); @@ -1391,9 +1451,10 @@ private List> getMessagesFromHttpResponse(String topic, Response for (Entry> entry : headers.entrySet()) { String header = entry.getKey(); - if (header.contains("X-Pulsar-PROPERTY-")) { - String keyName = header.substring("X-Pulsar-PROPERTY-".length()); - properties.put(keyName, (String) entry.getValue().get(0)); + if ("X-Pulsar-PROPERTY".equals(header)) { + Map msgPropsTmp = new Gson().fromJson((String) entry.getValue().get(0), Map.class); + properties.putAll(msgPropsTmp); + break; } } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TransactionsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TransactionsImpl.java index 5693ebc8f60aa..a0b9dd234d920 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TransactionsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TransactionsImpl.java @@ -31,6 +31,7 @@ import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TransactionBufferInternalStats; import org.apache.pulsar.common.policies.data.TransactionBufferStats; import org.apache.pulsar.common.policies.data.TransactionCoordinatorInfo; import org.apache.pulsar.common.policies.data.TransactionCoordinatorInternalStats; @@ -45,8 +46,8 @@ public class TransactionsImpl extends BaseResource implements Transactions { private final WebTarget adminV3Transactions; - public TransactionsImpl(WebTarget web, Authentication auth, long readTimeoutMs) { - super(auth, readTimeoutMs); + public TransactionsImpl(WebTarget web, Authentication auth, long requestTimeoutMs) { + super(auth, requestTimeoutMs); adminV3Transactions = web.path("/admin/v3/transactions"); } @@ -132,17 +133,20 @@ public TransactionMetadata getTransactionMetadata(TxnID txnID) throws PulsarAdmi @Override public CompletableFuture getTransactionBufferStatsAsync(String topic, - boolean lowWaterMarks) { + boolean lowWaterMarks, + boolean segmentStats) { WebTarget path = adminV3Transactions.path("transactionBufferStats"); path = path.path(TopicName.get(topic).getRestPath(false)); path = path.queryParam("lowWaterMarks", lowWaterMarks); + path = path.queryParam("segmentStats", segmentStats); return asyncGetRequest(path, new FutureCallback(){}); } @Override public TransactionBufferStats getTransactionBufferStats(String topic, - boolean lowWaterMarks) throws PulsarAdminException { - return sync(() -> getTransactionBufferStatsAsync(topic, lowWaterMarks)); + boolean lowWaterMarks, + boolean segmentStats) throws PulsarAdminException { + return sync(() -> getTransactionBufferStatsAsync(topic, lowWaterMarks, segmentStats)); } @Override @@ -227,6 +231,22 @@ public TransactionPendingAckInternalStats getPendingAckInternalStats(String topi return sync(() -> getPendingAckInternalStatsAsync(topic, subName, metadata)); } + @Override + public CompletableFuture getTransactionBufferInternalStatsAsync(String topic, + boolean metadata) { + TopicName tn = TopicName.get(topic); + WebTarget path = adminV3Transactions.path("transactionBufferInternalStats"); + path = path.path(tn.getRestPath(false)); + path = path.queryParam("metadata", metadata); + return asyncGetRequest(path, new FutureCallback(){}); + } + + @Override + public TransactionBufferInternalStats getTransactionBufferInternalStats(String topic, boolean metadata) + throws PulsarAdminException { + return sync(() -> getTransactionBufferInternalStatsAsync(topic, metadata)); + } + @Override public void scaleTransactionCoordinators(int replicas) throws PulsarAdminException { sync(() -> scaleTransactionCoordinatorsAsync(replicas)); @@ -263,4 +283,17 @@ public PositionInPendingAckStats getPositionStatsInPendingAck(String topic, Stri throws PulsarAdminException { return sync(() -> getPositionStatsInPendingAckAsync(topic, subName, ledgerId, entryId, batchIndex)); } + + @Override + public CompletableFuture abortTransactionAsync(TxnID txnID) { + WebTarget path = adminV3Transactions.path("abortTransaction"); + path = path.path(String.valueOf(txnID.getMostSigBits())); + path = path.path(String.valueOf(txnID.getLeastSigBits())); + return asyncPostRequest(path, Entity.entity("", MediaType.APPLICATION_JSON)); + } + + @Override + public void abortTransaction(TxnID txnID) throws PulsarAdminException { + sync(() -> abortTransactionAsync(txnID)); + } } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/WorkerImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/WorkerImpl.java index 60b1226d5817e..12a691edb08a2 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/WorkerImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/WorkerImpl.java @@ -40,8 +40,8 @@ public class WorkerImpl extends BaseResource implements Worker { private final WebTarget workerStats; private final WebTarget worker; - public WorkerImpl(WebTarget web, Authentication auth, long readTimeoutMs) { - super(auth, readTimeoutMs); + public WorkerImpl(WebTarget web, Authentication auth, long requestTimeoutMs) { + super(auth, requestTimeoutMs); this.worker = web.path("/admin/v2/worker"); this.workerStats = web.path("/admin/v2/worker-stats"); } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java index e79bacb4156b2..de694534a9e25 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java @@ -18,51 +18,68 @@ */ package org.apache.pulsar.client.admin.internal.http; +import static org.asynchttpclient.util.HttpConstants.Methods.GET; +import static org.asynchttpclient.util.HttpConstants.Methods.HEAD; +import static org.asynchttpclient.util.HttpConstants.Methods.OPTIONS; +import static org.asynchttpclient.util.HttpConstants.ResponseStatusCodes.FOUND_302; +import static org.asynchttpclient.util.HttpConstants.ResponseStatusCodes.MOVED_PERMANENTLY_301; +import static org.asynchttpclient.util.HttpConstants.ResponseStatusCodes.PERMANENT_REDIRECT_308; +import static org.asynchttpclient.util.HttpConstants.ResponseStatusCodes.SEE_OTHER_303; +import static org.asynchttpclient.util.HttpConstants.ResponseStatusCodes.TEMPORARY_REDIRECT_307; +import static org.asynchttpclient.util.MiscUtils.isNonEmpty; +import com.spotify.futures.ConcurrencyReducer; +import io.netty.handler.codec.http.DefaultHttpHeaders; import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponse; -import io.netty.handler.ssl.SslContext; -import io.netty.handler.ssl.SslProvider; import io.netty.util.concurrent.DefaultThreadFactory; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.InetSocketAddress; import java.net.URI; +import java.security.GeneralSecurityException; import java.time.Duration; +import java.util.Map; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.function.Function; import java.util.function.Supplier; -import javax.net.ssl.SSLContext; +import javax.ws.rs.ProcessingException; import javax.ws.rs.client.Client; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response.Status; import lombok.Getter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Validate; import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.client.admin.internal.PulsarAdminImpl; -import org.apache.pulsar.client.api.AuthenticationDataProvider; -import org.apache.pulsar.client.api.KeyStoreParams; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.impl.PulsarServiceNameResolver; +import org.apache.pulsar.client.impl.ServiceNameResolver; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; -import org.apache.pulsar.client.util.WithSNISslEngineFactory; +import org.apache.pulsar.client.util.PulsarHttpAsyncSslEngineFactory; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.SecurityUtility; -import org.apache.pulsar.common.util.keystoretls.KeyStoreSSLContext; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; +import org.asynchttpclient.AsyncCompletionHandlerBase; +import org.asynchttpclient.AsyncHandler; import org.asynchttpclient.AsyncHttpClient; +import org.asynchttpclient.AsyncHttpClientConfig; import org.asynchttpclient.BoundRequestBuilder; import org.asynchttpclient.DefaultAsyncHttpClient; import org.asynchttpclient.DefaultAsyncHttpClientConfig; +import org.asynchttpclient.ListenableFuture; import org.asynchttpclient.Request; import org.asynchttpclient.Response; +import org.asynchttpclient.SslEngineFactory; import org.asynchttpclient.channel.DefaultKeepAliveStrategy; -import org.asynchttpclient.netty.ssl.JsseSslEngineFactory; +import org.asynchttpclient.uri.Uri; import org.glassfish.jersey.client.ClientProperties; import org.glassfish.jersey.client.ClientRequest; import org.glassfish.jersey.client.ClientResponse; @@ -73,32 +90,77 @@ * Customized Jersey client connector with multi-host support. */ @Slf4j -public class AsyncHttpConnector implements Connector { - private static final TimeoutException READ_TIMEOUT_EXCEPTION = - FutureUtil.createTimeoutException("Read timeout", AsyncHttpConnector.class, "retryOrTimeout(...)"); +public class AsyncHttpConnector implements Connector, AsyncHttpRequestExecutor { + private static final TimeoutException REQUEST_TIMEOUT_EXCEPTION = + FutureUtil.createTimeoutException("Request timeout", AsyncHttpConnector.class, "retryOrTimeout(...)"); + private static final int DEFAULT_MAX_QUEUE_SIZE_PER_HOST = 10000; @Getter private final AsyncHttpClient httpClient; - private final Duration readTimeout; + private final Duration requestTimeout; private final int maxRetries; - private final PulsarServiceNameResolver serviceNameResolver; + private final ServiceNameResolver serviceNameResolver; private final ScheduledExecutorService delayer = Executors.newScheduledThreadPool(1, new DefaultThreadFactory("delayer")); + private ScheduledExecutorService sslRefresher; + private final boolean acceptGzipCompression; + private final Map> concurrencyReducers = new ConcurrentHashMap<>(); + private PulsarSslFactory sslFactory; - public AsyncHttpConnector(Client client, ClientConfigurationData conf, int autoCertRefreshTimeSeconds) { + public AsyncHttpConnector(Client client, ClientConfigurationData conf, int autoCertRefreshTimeSeconds, + boolean acceptGzipCompression) { this((int) client.getConfiguration().getProperty(ClientProperties.CONNECT_TIMEOUT), (int) client.getConfiguration().getProperty(ClientProperties.READ_TIMEOUT), PulsarAdminImpl.DEFAULT_REQUEST_TIMEOUT_SECONDS * 1000, autoCertRefreshTimeSeconds, - conf); + conf, acceptGzipCompression); } @SneakyThrows public AsyncHttpConnector(int connectTimeoutMs, int readTimeoutMs, int requestTimeoutMs, - int autoCertRefreshTimeSeconds, ClientConfigurationData conf) { + int autoCertRefreshTimeSeconds, ClientConfigurationData conf, + boolean acceptGzipCompression) { + Validate.notEmpty(conf.getServiceUrl(), "Service URL is not provided"); + serviceNameResolver = new PulsarServiceNameResolver(); + String serviceUrl = conf.getServiceUrl(); + serviceNameResolver.updateServiceUrl(serviceUrl); + this.acceptGzipCompression = acceptGzipCompression; + AsyncHttpClientConfig asyncHttpClientConfig = + createAsyncHttpClientConfig(conf, connectTimeoutMs, readTimeoutMs, requestTimeoutMs, + autoCertRefreshTimeSeconds); + httpClient = createAsyncHttpClient(asyncHttpClientConfig); + this.requestTimeout = requestTimeoutMs > 0 ? Duration.ofMillis(requestTimeoutMs) : null; + this.maxRetries = httpClient.getConfig().getMaxRequestRetry(); + } + + private AsyncHttpClientConfig createAsyncHttpClientConfig(ClientConfigurationData conf, int connectTimeoutMs, + int readTimeoutMs, + int requestTimeoutMs, int autoCertRefreshTimeSeconds) + throws GeneralSecurityException, IOException { DefaultAsyncHttpClientConfig.Builder confBuilder = new DefaultAsyncHttpClientConfig.Builder(); + configureAsyncHttpClientConfig(conf, connectTimeoutMs, readTimeoutMs, requestTimeoutMs, confBuilder); + if (conf.getServiceUrl().startsWith("https://")) { + configureAsyncHttpClientSslEngineFactory(conf, autoCertRefreshTimeSeconds, confBuilder); + } + AsyncHttpClientConfig asyncHttpClientConfig = confBuilder.build(); + return asyncHttpClientConfig; + } + + private void configureAsyncHttpClientConfig(ClientConfigurationData conf, int connectTimeoutMs, int readTimeoutMs, + int requestTimeoutMs, + DefaultAsyncHttpClientConfig.Builder confBuilder) { + if (conf.getConnectionsPerBroker() > 0) { + confBuilder.setMaxConnectionsPerHost(conf.getConnectionsPerBroker()); + // Use the request timeout value for acquireFreeChannelTimeout so that we don't need to add + // yet another configuration property. When the ConcurrencyReducer is in use, it shouldn't be necessary to + // wait for a free channel since the ConcurrencyReducer will queue the requests. + confBuilder.setAcquireFreeChannelTimeout(conf.getRequestTimeoutMs()); + } + if (conf.getConnectionMaxIdleSeconds() > 0) { + confBuilder.setPooledConnectionIdleTimeout(conf.getConnectionMaxIdleSeconds() * 1000); + } confBuilder.setUseProxyProperties(true); - confBuilder.setFollowRedirect(true); + confBuilder.setFollowRedirect(false); confBuilder.setRequestTimeout(conf.getRequestTimeoutMs()); confBuilder.setConnectTimeout(connectTimeoutMs); confBuilder.setReadTimeout(readTimeoutMs); @@ -114,75 +176,35 @@ public boolean keepAlive(InetSocketAddress remoteAddress, Request ahcRequest, && super.keepAlive(remoteAddress, ahcRequest, request, response); } }); + confBuilder.setDisableHttpsEndpointIdentificationAlgorithm(!conf.isTlsHostnameVerificationEnable()); + } - serviceNameResolver = new PulsarServiceNameResolver(); - if (conf != null && StringUtils.isNotBlank(conf.getServiceUrl())) { - serviceNameResolver.updateServiceUrl(conf.getServiceUrl()); - if (conf.getServiceUrl().startsWith("https://")) { - // Set client key and certificate if available - AuthenticationDataProvider authData = conf.getAuthentication().getAuthData(); - - if (conf.isUseKeyStoreTls()) { - KeyStoreParams params = authData.hasDataForTls() ? authData.getTlsKeyStoreParams() : - new KeyStoreParams(conf.getTlsKeyStoreType(), conf.getTlsKeyStorePath(), - conf.getTlsKeyStorePassword()); - - final SSLContext sslCtx = KeyStoreSSLContext.createClientSslContext( - conf.getSslProvider(), - params.getKeyStoreType(), - params.getKeyStorePath(), - params.getKeyStorePassword(), - conf.isTlsAllowInsecureConnection(), - conf.getTlsTrustStoreType(), - conf.getTlsTrustStorePath(), - conf.getTlsTrustStorePassword(), - conf.getTlsCiphers(), - conf.getTlsProtocols()); - - JsseSslEngineFactory sslEngineFactory = new JsseSslEngineFactory(sslCtx); - confBuilder.setSslEngineFactory(sslEngineFactory); - } else { - SslProvider sslProvider = null; - if (conf.getSslProvider() != null) { - sslProvider = SslProvider.valueOf(conf.getSslProvider()); - } - SslContext sslCtx = null; - if (authData.hasDataForTls()) { - sslCtx = authData.getTlsTrustStoreStream() == null - ? SecurityUtility.createAutoRefreshSslContextForClient( - sslProvider, - conf.isTlsAllowInsecureConnection(), - conf.getTlsTrustCertsFilePath(), authData.getTlsCerificateFilePath(), - authData.getTlsPrivateKeyFilePath(), null, autoCertRefreshTimeSeconds, delayer) - : SecurityUtility.createNettySslContextForClient( - sslProvider, - conf.isTlsAllowInsecureConnection(), - authData.getTlsTrustStoreStream(), authData.getTlsCertificates(), - authData.getTlsPrivateKey(), - conf.getTlsCiphers(), - conf.getTlsProtocols()); - } else { - sslCtx = SecurityUtility.createNettySslContextForClient( - sslProvider, - conf.isTlsAllowInsecureConnection(), - conf.getTlsTrustCertsFilePath(), - conf.getTlsCertificateFilePath(), - conf.getTlsKeyFilePath(), - conf.getTlsCiphers(), - conf.getTlsProtocols()); - } - confBuilder.setSslContext(sslCtx); - if (!conf.isTlsHostnameVerificationEnable()) { - confBuilder.setSslEngineFactory(new WithSNISslEngineFactory(serviceNameResolver - .resolveHostUri().getHost())); - } - } - } - confBuilder.setDisableHttpsEndpointIdentificationAlgorithm(!conf.isTlsHostnameVerificationEnable()); + protected AsyncHttpClient createAsyncHttpClient(AsyncHttpClientConfig asyncHttpClientConfig) { + return new DefaultAsyncHttpClient(asyncHttpClientConfig); + } + + @SneakyThrows + private void configureAsyncHttpClientSslEngineFactory(ClientConfigurationData conf, int autoCertRefreshTimeSeconds, + DefaultAsyncHttpClientConfig.Builder confBuilder) + throws GeneralSecurityException, IOException { + // Set client key and certificate if available + sslRefresher = Executors.newScheduledThreadPool(1, + new DefaultThreadFactory("pulsar-admin-ssl-refresher")); + PulsarSslConfiguration sslConfiguration = buildSslConfiguration(conf); + this.sslFactory = (PulsarSslFactory) Class.forName(conf.getSslFactoryPlugin()) + .getConstructor().newInstance(); + this.sslFactory.initialize(sslConfiguration); + this.sslFactory.createInternalSslContext(); + if (conf.getAutoCertRefreshSeconds() > 0) { + this.sslRefresher.scheduleWithFixedDelay(this::refreshSslContext, conf.getAutoCertRefreshSeconds(), + conf.getAutoCertRefreshSeconds(), TimeUnit.SECONDS); } - httpClient = new DefaultAsyncHttpClient(confBuilder.build()); - this.readTimeout = Duration.ofMillis(readTimeoutMs); - this.maxRetries = httpClient.getConfig().getMaxRequestRetry(); + String hostname = conf.isTlsHostnameVerificationEnable() ? null : serviceNameResolver + .resolveHostUri().getHost(); + SslEngineFactory sslEngineFactory = new PulsarHttpAsyncSslEngineFactory(sslFactory, hostname); + confBuilder.setSslEngineFactory(sslEngineFactory); + confBuilder.setUseInsecureTrustManager(conf.isTlsAllowInsecureConnection()); + confBuilder.setDisableHttpsEndpointIdentificationAlgorithm(!conf.isTlsHostnameVerificationEnable()); } @Override @@ -202,9 +224,8 @@ public void failure(Throwable failure) { try { return future.get(); } catch (InterruptedException | ExecutionException e) { - log.error(e.getMessage()); + throw new ProcessingException(e.getCause()); } - return null; } private URI replaceWithNew(InetSocketAddress address, URI uri) { @@ -260,11 +281,14 @@ public String getReasonPhrase() { private CompletableFuture retryOrTimeOut(ClientRequest request) { final CompletableFuture resultFuture = new CompletableFuture<>(); retryOperation(resultFuture, () -> oneShot(serviceNameResolver.resolveHost(), request), maxRetries); - CompletableFuture timeoutAfter = FutureUtil.createFutureWithTimeout(readTimeout, delayer, - () -> READ_TIMEOUT_EXCEPTION); - return resultFuture.applyToEither(timeoutAfter, Function.identity()); + if (requestTimeout != null) { + FutureUtil.addTimeoutHandling(resultFuture, requestTimeout, delayer, () -> REQUEST_TIMEOUT_EXCEPTION); + } + return resultFuture; } + // TODO: There are problems with this solution since AsyncHttpClient already contains logic to retry requests. + // This solution doesn't contain backoff handling. private void retryOperation( final CompletableFuture resultFuture, final Supplier> operation, @@ -276,16 +300,27 @@ private void retryOperation( operationFuture.whenComplete( (t, throwable) -> { if (throwable != null) { + throwable = FutureUtil.unwrapCompletionException(throwable); if (throwable instanceof CancellationException) { resultFuture.completeExceptionally( new RetryException("Operation future was cancelled.", throwable)); + } else if (throwable instanceof MaxRedirectException) { + // don't retry on max redirect + resultFuture.completeExceptionally(throwable); } else { if (retries > 0) { + if (log.isDebugEnabled()) { + log.debug("Retrying operation. Remaining retries: {}", retries); + } retryOperation( resultFuture, operation, retries - 1); } else { + if (log.isDebugEnabled()) { + log.debug("Number of retries has been exhausted. Failing the operation.", + throwable); + } resultFuture.completeExceptionally( new RetryException("Could not complete the operation. Number of retries " + "has been exhausted. Failed reason: " + throwable.getMessage(), @@ -311,7 +346,129 @@ public RetryException(String message, Throwable cause) { } } - private CompletableFuture oneShot(InetSocketAddress host, ClientRequest request) { + public static class MaxRedirectException extends Exception { + public MaxRedirectException(String msg) { + super(msg, null, true, false); + } + } + + protected CompletableFuture oneShot(InetSocketAddress host, ClientRequest request) { + Request preparedRequest; + try { + preparedRequest = prepareRequest(host, request); + } catch (IOException e) { + return FutureUtil.failedFuture(e); + } + return executeRequest(preparedRequest); + } + + public CompletableFuture executeRequest(Request request) { + return executeRequest(request, () -> new AsyncCompletionHandlerBase()); + } + + public CompletableFuture executeRequest(Request request, + Supplier> handlerSupplier) { + return executeRequest(request, handlerSupplier, 0); + } + + private CompletableFuture executeRequest(Request request, + Supplier> handlerSupplier, + int redirectCount) { + int maxRedirects = httpClient.getConfig().getMaxRedirects(); + if (redirectCount > maxRedirects) { + return FutureUtil.failedFuture( + new MaxRedirectException("Maximum redirect reached: " + maxRedirects + " uri:" + request.getUri())); + } + CompletableFuture responseFuture; + if (httpClient.getConfig().getMaxConnectionsPerHost() > 0) { + String hostAndPort = request.getUri().getHost() + ":" + request.getUri().getPort(); + ConcurrencyReducer responseConcurrencyReducer = concurrencyReducers.computeIfAbsent(hostAndPort, + h -> ConcurrencyReducer.create(httpClient.getConfig().getMaxConnectionsPerHost(), + DEFAULT_MAX_QUEUE_SIZE_PER_HOST)); + responseFuture = responseConcurrencyReducer.add(() -> doExecuteRequest(request, handlerSupplier)); + } else { + responseFuture = doExecuteRequest(request, handlerSupplier); + } + CompletableFuture futureWithRedirect = responseFuture.thenCompose(response -> { + if (isRedirectStatusCode(response.getStatusCode())) { + return executeRedirect(request, response, handlerSupplier, redirectCount); + } + return CompletableFuture.completedFuture(response); + }); + futureWithRedirect.whenComplete((response, throwable) -> { + // propagate cancellation or timeout to the original response future + responseFuture.cancel(false); + }); + return futureWithRedirect; + } + + private CompletableFuture executeRedirect(Request request, Response response, + Supplier> handlerSupplier, + int redirectCount) { + String originalMethod = request.getMethod(); + int statusCode = response.getStatusCode(); + boolean switchToGet = !originalMethod.equals(GET) + && !originalMethod.equals(OPTIONS) && !originalMethod.equals(HEAD) && ( + statusCode == MOVED_PERMANENTLY_301 || statusCode == SEE_OTHER_303 || statusCode == FOUND_302); + boolean keepBody = statusCode == TEMPORARY_REDIRECT_307 || statusCode == PERMANENT_REDIRECT_308; + String location = response.getHeader(HttpHeaders.LOCATION); + Uri newUri = Uri.create(request.getUri(), location); + BoundRequestBuilder builder = httpClient.prepareRequest(request); + if (switchToGet) { + builder.setMethod(GET); + } + builder.setUri(newUri); + if (keepBody) { + builder.setCharset(request.getCharset()); + if (isNonEmpty(request.getFormParams())) { + builder.setFormParams(request.getFormParams()); + } else if (request.getStringData() != null) { + builder.setBody(request.getStringData()); + } else if (request.getByteData() != null){ + builder.setBody(request.getByteData()); + } else if (request.getByteBufferData() != null) { + builder.setBody(request.getByteBufferData()); + } else if (request.getBodyGenerator() != null) { + builder.setBody(request.getBodyGenerator()); + } else if (isNonEmpty(request.getBodyParts())) { + builder.setBodyParts(request.getBodyParts()); + } + } else { + builder.resetFormParams(); + builder.resetNonMultipartData(); + builder.resetMultipartData(); + io.netty.handler.codec.http.HttpHeaders headers = new DefaultHttpHeaders(); + headers.add(request.getHeaders()); + headers.remove(HttpHeaders.CONTENT_LENGTH); + headers.remove(HttpHeaders.CONTENT_TYPE); + headers.remove(HttpHeaders.CONTENT_ENCODING); + builder.setHeaders(headers); + } + return executeRequest(builder.build(), handlerSupplier, redirectCount + 1); + } + + private static boolean isRedirectStatusCode(int statusCode) { + return statusCode == MOVED_PERMANENTLY_301 || statusCode == FOUND_302 || statusCode == SEE_OTHER_303 + || statusCode == TEMPORARY_REDIRECT_307 || statusCode == PERMANENT_REDIRECT_308; + } + + private CompletableFuture doExecuteRequest(Request request, + Supplier> handlerSupplier) { + ListenableFuture responseFuture = + httpClient.executeRequest(request, handlerSupplier.get()); + CompletableFuture completableFuture = responseFuture.toCompletableFuture(); + completableFuture.whenComplete((response, throwable) -> { + throwable = FutureUtil.unwrapCompletionException(throwable); + if (throwable != null && (throwable instanceof CancellationException + || throwable instanceof TimeoutException)) { + // abort the request if the future is cancelled or timed out + responseFuture.abort(throwable); + } + }); + return completableFuture; + } + + private Request prepareRequest(InetSocketAddress host, ClientRequest request) throws IOException { ClientRequest currentRequest = new ClientRequest(request); URI newUri = replaceWithNew(host, currentRequest.getUri()); currentRequest.setUri(newUri); @@ -322,14 +479,7 @@ private CompletableFuture oneShot(InetSocketAddress host, ClientReques if (currentRequest.hasEntity()) { ByteArrayOutputStream outStream = new ByteArrayOutputStream(); currentRequest.setStreamProvider(contentLength -> outStream); - try { - currentRequest.writeEntity(); - } catch (IOException e) { - CompletableFuture r = new CompletableFuture<>(); - r.completeExceptionally(e); - return r; - } - + currentRequest.writeEntity(); builder.setBody(outStream.toByteArray()); } @@ -339,7 +489,11 @@ private CompletableFuture oneShot(InetSocketAddress host, ClientReques } }); - return builder.execute().toCompletableFuture(); + if (acceptGzipCompression) { + builder.setHeader(HttpHeaders.ACCEPT_ENCODING, "gzip"); + } + + return builder.build(); } @Override @@ -352,9 +506,45 @@ public void close() { try { httpClient.close(); delayer.shutdownNow(); + if (sslRefresher != null) { + sslRefresher.shutdownNow(); + } } catch (IOException e) { log.warn("Failed to close http client", e); } } + protected PulsarSslConfiguration buildSslConfiguration(ClientConfigurationData conf) + throws PulsarClientException { + return PulsarSslConfiguration.builder() + .tlsProvider(conf.getSslProvider()) + .tlsKeyStoreType(conf.getTlsKeyStoreType()) + .tlsKeyStorePath(conf.getTlsKeyStorePath()) + .tlsKeyStorePassword(conf.getTlsKeyStorePassword()) + .tlsTrustStoreType(conf.getTlsTrustStoreType()) + .tlsTrustStorePath(conf.getTlsTrustStorePath()) + .tlsTrustStorePassword(conf.getTlsTrustStorePassword()) + .tlsCiphers(conf.getTlsCiphers()) + .tlsProtocols(conf.getTlsProtocols()) + .tlsTrustCertsFilePath(conf.getTlsTrustCertsFilePath()) + .tlsCertificateFilePath(conf.getTlsCertificateFilePath()) + .tlsKeyFilePath(conf.getTlsKeyFilePath()) + .allowInsecureConnection(conf.isTlsAllowInsecureConnection()) + .requireTrustedClientCertOnConnect(false) + .tlsEnabledWithKeystore(conf.isUseKeyStoreTls()) + .authData(conf.getAuthentication().getAuthData()) + .tlsCustomParams(conf.getSslFactoryPluginParams()) + .serverMode(false) + .isHttps(true) + .build(); + } + + protected void refreshSslContext() { + try { + this.sslFactory.update(); + } catch (Exception e) { + log.error("Failed to refresh SSL context", e); + } + } + } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnectorProvider.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnectorProvider.java index 4467f77d1f993..d20dc84849458 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnectorProvider.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnectorProvider.java @@ -32,16 +32,19 @@ public class AsyncHttpConnectorProvider implements ConnectorProvider { private final ClientConfigurationData conf; private Connector connector; private final int autoCertRefreshTimeSeconds; + private final boolean acceptGzipCompression; - public AsyncHttpConnectorProvider(ClientConfigurationData conf, int autoCertRefreshTimeSeconds) { + public AsyncHttpConnectorProvider(ClientConfigurationData conf, int autoCertRefreshTimeSeconds, + boolean acceptGzipCompression) { this.conf = conf; this.autoCertRefreshTimeSeconds = autoCertRefreshTimeSeconds; + this.acceptGzipCompression = acceptGzipCompression; } @Override public Connector getConnector(Client client, Configuration runtimeConfig) { if (connector == null) { - connector = new AsyncHttpConnector(client, conf, autoCertRefreshTimeSeconds); + connector = new AsyncHttpConnector(client, conf, autoCertRefreshTimeSeconds, acceptGzipCompression); } return connector; } @@ -50,6 +53,6 @@ public Connector getConnector(Client client, Configuration runtimeConfig) { public AsyncHttpConnector getConnector(int connectTimeoutMs, int readTimeoutMs, int requestTimeoutMs, int autoCertRefreshTimeSeconds) { return new AsyncHttpConnector(connectTimeoutMs, readTimeoutMs, requestTimeoutMs, autoCertRefreshTimeSeconds, - conf); + conf, acceptGzipCompression); } } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpRequestExecutor.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpRequestExecutor.java new file mode 100644 index 0000000000000..d3c7a653b36b4 --- /dev/null +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpRequestExecutor.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.admin.internal.http; + +import java.util.concurrent.CompletableFuture; +import java.util.function.Supplier; +import org.asynchttpclient.AsyncHandler; +import org.asynchttpclient.Request; +import org.asynchttpclient.Response; + +/** + * Interface for executing HTTP requests asynchronously. + * This is used internally in the Pulsar Admin client for executing HTTP requests that by-pass the Jersey client + * and use the AsyncHttpClient API directly. + */ +public interface AsyncHttpRequestExecutor { + /** + * Execute the given HTTP request asynchronously. + * + * @param request the HTTP request to execute + * @return a future that will be completed with the HTTP response + */ + CompletableFuture executeRequest(Request request); + /** + * Execute the given HTTP request asynchronously. + * + * @param request the HTTP request to execute + * @param handlerSupplier a supplier for the async handler to use for the request + * @return a future that will be completed with the HTTP response + */ + CompletableFuture executeRequest(Request request, Supplier> handlerSupplier); +} diff --git a/pulsar-client-admin/src/test/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImplTest.java b/pulsar-client-admin/src/test/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImplTest.java index d3621e729973b..b61b8774b6e2e 100644 --- a/pulsar-client-admin/src/test/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImplTest.java +++ b/pulsar-client-admin/src/test/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImplTest.java @@ -26,6 +26,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import lombok.Cleanup; import lombok.SneakyThrows; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminBuilder; @@ -65,13 +66,16 @@ public void testGetPropertiesFromConf() throws Exception { config.put("autoCertRefreshSeconds", 20); config.put("connectionTimeoutMs", 30); config.put("readTimeoutMs", 40); + config.put("maxConnectionsPerHost", 50); PulsarAdminBuilder adminBuilder = PulsarAdmin.builder().loadConf(config); + @Cleanup PulsarAdminImpl admin = (PulsarAdminImpl) adminBuilder.build(); ClientConfigurationData clientConfigData = admin.getClientConfigData(); Assert.assertEquals(clientConfigData.getRequestTimeoutMs(), 10); Assert.assertEquals(clientConfigData.getAutoCertRefreshSeconds(), 20); Assert.assertEquals(clientConfigData.getConnectionTimeoutMs(), 30); Assert.assertEquals(clientConfigData.getReadTimeoutMs(), 40); + Assert.assertEquals(clientConfigData.getConnectionsPerBroker(), 50); } @Test diff --git a/pulsar-client-admin/src/test/java/org/apache/pulsar/client/admin/internal/PulsarAdminGzipTest.java b/pulsar-client-admin/src/test/java/org/apache/pulsar/client/admin/internal/PulsarAdminGzipTest.java new file mode 100644 index 0000000000000..2bfa382be1096 --- /dev/null +++ b/pulsar-client-admin/src/test/java/org/apache/pulsar/client/admin/internal/PulsarAdminGzipTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.admin.internal; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.absent; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static org.testng.Assert.assertEquals; +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.zip.GZIPOutputStream; +import lombok.Cleanup; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +public class PulsarAdminGzipTest { + WireMockServer server; + + @BeforeClass(alwaysRun = true) + void beforeClass() throws IOException { + server = new WireMockServer(WireMockConfiguration.wireMockConfig() + .port(0)); + server.start(); + } + + @AfterClass(alwaysRun = true) + void afterClass() { + if (server != null) { + server.stop(); + } + } + + static byte[] gzipContent(String content) throws IOException { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + try(GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream)) { + gzipOutputStream.write(content.getBytes(StandardCharsets.UTF_8)); + } + return byteArrayOutputStream.toByteArray(); + } + + @AfterMethod + void resetAllMocks() { + server.resetAll(); + } + + @Test + public void testGzipRequestedGzipResponse() throws Exception { + server.stubFor(get(urlEqualTo("/admin/v2/clusters")) + .withHeader("Accept-Encoding", equalTo("gzip")) + .willReturn(aResponse() + .withHeader("Content-Type", "application/json") + .withHeader("Content-Encoding", "gzip") + .withBody(gzipContent("[\"gzip-test\", \"gzip-test2\"]")))); + + @Cleanup + PulsarAdmin admin = PulsarAdmin.builder() + .serviceHttpUrl("http://localhost:" + server.port()) + .acceptGzipCompression(true) + .build(); + + assertEquals(admin.clusters().getClusters(), Arrays.asList("gzip-test", "gzip-test2")); + } + + @Test + public void testGzipRequestedNoGzipResponse() throws Exception { + server.stubFor(get(urlEqualTo("/admin/v2/clusters")) + .withHeader("Accept-Encoding", equalTo("gzip")) + .willReturn(aResponse() + .withHeader("Content-Type", "application/json") + .withBody("[\"test\", \"test2\"]"))); + + @Cleanup + PulsarAdmin admin = PulsarAdmin.builder() + .serviceHttpUrl("http://localhost:" + server.port()) + .acceptGzipCompression(true) + .build(); + + assertEquals(admin.clusters().getClusters(), Arrays.asList("test", "test2")); + } + + @Test + public void testNoGzipRequestedNoGzipResponse() throws Exception { + server.stubFor(get(urlEqualTo("/admin/v2/clusters")) + .withHeader("Accept-Encoding", absent()) + .willReturn(aResponse() + .withHeader("Content-Type", "application/json") + .withBody("[\"test\", \"test2\"]"))); + + @Cleanup + PulsarAdmin admin = PulsarAdmin.builder() + .serviceHttpUrl("http://localhost:" + server.port()) + .acceptGzipCompression(false) + .build(); + + assertEquals(admin.clusters().getClusters(), Arrays.asList("test", "test2")); + } +} diff --git a/pulsar-client-admin/src/test/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnectorTest.java b/pulsar-client-admin/src/test/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnectorTest.java new file mode 100644 index 0000000000000..f8518b5931034 --- /dev/null +++ b/pulsar-client-admin/src/test/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnectorTest.java @@ -0,0 +1,340 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.admin.internal.http; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.common.FileSource; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import com.github.tomakehurst.wiremock.extension.Parameters; +import com.github.tomakehurst.wiremock.extension.ResponseTransformer; +import com.github.tomakehurst.wiremock.stubbing.Scenario; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.Cleanup; +import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.common.util.FutureUtil; +import org.asynchttpclient.Request; +import org.asynchttpclient.RequestBuilder; +import org.asynchttpclient.Response; +import org.glassfish.jersey.client.ClientConfig; +import org.glassfish.jersey.client.ClientRequest; +import org.glassfish.jersey.client.ClientResponse; +import org.glassfish.jersey.client.JerseyClient; +import org.glassfish.jersey.client.JerseyClientBuilder; +import org.glassfish.jersey.client.spi.AsyncConnectorCallback; +import org.glassfish.jersey.internal.MapPropertiesDelegate; +import org.glassfish.jersey.internal.PropertiesDelegate; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +public class AsyncHttpConnectorTest { + WireMockServer server; + ConcurrencyTestTransformer concurrencyTestTransformer = new ConcurrencyTestTransformer(); + + private static class CopyRequestBodyToResponseBodyTransformer extends ResponseTransformer { + @Override + public com.github.tomakehurst.wiremock.http.Response transform( + com.github.tomakehurst.wiremock.http.Request request, + com.github.tomakehurst.wiremock.http.Response response, FileSource fileSource, Parameters parameters) { + return com.github.tomakehurst.wiremock.http.Response.Builder.like(response) + .body(request.getBodyAsString()) + .build(); + } + + @Override + public String getName() { + return "copy-body"; + } + + @Override + public boolean applyGlobally() { + return false; + } + } + + private static class ConcurrencyTestTransformer extends ResponseTransformer { + private static final long DELAY_MS = 100; + private final AtomicInteger concurrencyCounter = new AtomicInteger(0); + private final AtomicInteger maxConcurrency = new AtomicInteger(0); + + @Override + public com.github.tomakehurst.wiremock.http.Response transform( + com.github.tomakehurst.wiremock.http.Request request, + com.github.tomakehurst.wiremock.http.Response response, FileSource fileSource, Parameters parameters) { + int currentCounter = concurrencyCounter.incrementAndGet(); + maxConcurrency.updateAndGet(v -> Math.max(v, currentCounter)); + try { + try { + Thread.sleep(DELAY_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return com.github.tomakehurst.wiremock.http.Response.Builder.like(response) + .body(String.valueOf(currentCounter)) + .build(); + } finally { + concurrencyCounter.decrementAndGet(); + } + } + + public int getMaxConcurrency() { + return maxConcurrency.get(); + } + + @Override + public String getName() { + return "concurrency-test"; + } + + @Override + public boolean applyGlobally() { + return false; + } + } + + @BeforeClass(alwaysRun = true) + void beforeClass() throws IOException { + server = new WireMockServer(WireMockConfiguration.wireMockConfig() + .extensions(new CopyRequestBodyToResponseBodyTransformer(), concurrencyTestTransformer) + .containerThreads(100) + .port(0)); + server.start(); + } + + @AfterClass(alwaysRun = true) + void afterClass() { + if (server != null) { + server.stop(); + } + } + + static class TestClientRequest extends ClientRequest { + public TestClientRequest(URI uri, ClientConfig clientConfig, PropertiesDelegate propertiesDelegate) { + super(uri, clientConfig, propertiesDelegate); + } + } + + @Test + public void testShouldStopRetriesWhenTimeoutOccurs() throws IOException, ExecutionException, InterruptedException { + server.stubFor(get(urlEqualTo("/admin/v2/clusters")) + .inScenario("once") + .whenScenarioStateIs(Scenario.STARTED) + .willSetStateTo("next") + .willReturn(aResponse() + .withHeader("Content-Type", "application/json") + .withBody("[\"test-cluster\"]"))); + + server.stubFor(get(urlEqualTo("/admin/v2/clusters")) + .inScenario("once") + .whenScenarioStateIs("next") + .willSetStateTo("retried") + .willReturn(aResponse().withStatus(500))); + + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setServiceUrl("http://localhost:" + server.port()); + + int requestTimeout = 500; + + @Cleanup("shutdownNow") + ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(); + Executor delayedExecutor = runnable -> { + scheduledExecutor.schedule(runnable, requestTimeout, TimeUnit.MILLISECONDS); + }; + @Cleanup + AsyncHttpConnector connector = new AsyncHttpConnector(5000, requestTimeout, + requestTimeout, 0, conf, false) { + @Override + protected CompletableFuture oneShot(InetSocketAddress host, ClientRequest request) { + // delay the response to simulate a timeout + return super.oneShot(host, request) + .thenApplyAsync(response -> { + return response; + }, delayedExecutor); + } + }; + + JerseyClient jerseyClient = JerseyClientBuilder.createClient(); + ClientConfig clientConfig = jerseyClient.getConfiguration(); + PropertiesDelegate propertiesDelegate = new MapPropertiesDelegate(); + URI requestUri = URI.create("http://localhost:" + server.port() + "/admin/v2/clusters"); + ClientRequest request = new TestClientRequest(requestUri, clientConfig, propertiesDelegate); + request.setMethod("GET"); + CompletableFuture future = new CompletableFuture<>(); + connector.apply(request, new AsyncConnectorCallback() { + @Override + public void response(ClientResponse response) { + future.complete(response); + } + + @Override + public void failure(Throwable failure) { + future.completeExceptionally(failure); + } + }); + Thread.sleep(2 * requestTimeout); + String scenarioState = + server.getAllScenarios().getScenarios().stream().filter(scenario -> "once".equals(scenario.getName())) + .findFirst().get().getState(); + assertEquals(scenarioState, "next"); + assertTrue(future.isCompletedExceptionally()); + } + + @Test + void testMaxRedirects() { + // Redirect to itself to test max redirects + server.stubFor(get(urlEqualTo("/admin/v2/clusters")) + .willReturn(aResponse() + .withStatus(301) + .withHeader("Location", "http://localhost:" + server.port() + "/admin/v2/clusters"))); + + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setServiceUrl("http://localhost:" + server.port()); + + @Cleanup + AsyncHttpConnector connector = new AsyncHttpConnector(5000, 5000, + 5000, 0, conf, false); + + Request request = new RequestBuilder("GET") + .setUrl("http://localhost:" + server.port() + "/admin/v2/clusters") + .build(); + + try { + connector.executeRequest(request).get(); + fail(); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof AsyncHttpConnector.MaxRedirectException); + } catch (InterruptedException e) { + fail(); + } + } + + @Test + void testRelativeRedirect() throws ExecutionException, InterruptedException { + doTestRedirect("path2"); + } + + @Test + void testAbsoluteRedirect() throws ExecutionException, InterruptedException { + doTestRedirect("/path2"); + } + + @Test + void testUrlRedirect() throws ExecutionException, InterruptedException { + doTestRedirect("http://localhost:" + server.port() + "/path2"); + } + + private void doTestRedirect(String location) throws InterruptedException, ExecutionException { + server.stubFor(get(urlEqualTo("/path1")) + .willReturn(aResponse() + .withStatus(301) + .withHeader("Location", location))); + + server.stubFor(get(urlEqualTo("/path2")) + .willReturn(aResponse() + .withBody("OK"))); + + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setServiceUrl("http://localhost:" + server.port()); + + @Cleanup + AsyncHttpConnector connector = new AsyncHttpConnector(5000, 5000, + 5000, 0, conf, false); + + Request request = new RequestBuilder("GET") + .setUrl("http://localhost:" + server.port() + "/path1") + .build(); + + Response response = connector.executeRequest(request).get(); + assertEquals(response.getResponseBody(), "OK"); + } + + @Test + void testRedirectWithBody() throws ExecutionException, InterruptedException { + server.stubFor(post(urlEqualTo("/path1")) + .willReturn(aResponse() + .withStatus(307) + .withHeader("Location", "/path2"))); + + server.stubFor(post(urlEqualTo("/path2")) + .willReturn(aResponse() + .withTransformers("copy-body"))); + + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setServiceUrl("http://localhost:" + server.port()); + + @Cleanup + AsyncHttpConnector connector = new AsyncHttpConnector(5000, 5000, + 5000, 0, conf, false); + + Request request = new RequestBuilder("POST") + .setUrl("http://localhost:" + server.port() + "/path1") + .setBody("Hello world!") + .build(); + + Response response = connector.executeRequest(request).get(); + assertEquals(response.getResponseBody(), "Hello world!"); + } + + @Test + void testMaxConnections() throws ExecutionException, InterruptedException { + server.stubFor(post(urlEqualTo("/concurrency-test")) + .willReturn(aResponse() + .withTransformers("concurrency-test"))); + + ClientConfigurationData conf = new ClientConfigurationData(); + int maxConnections = 10; + conf.setConnectionsPerBroker(maxConnections); + conf.setServiceUrl("http://localhost:" + server.port()); + + @Cleanup + AsyncHttpConnector connector = new AsyncHttpConnector(5000, 5000, + 5000, 0, conf, false); + + Request request = new RequestBuilder("POST") + .setUrl("http://localhost:" + server.port() + "/concurrency-test") + .build(); + + List> futures = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + futures.add(connector.executeRequest(request)); + } + FutureUtil.waitForAll(futures).get(); + int maxConcurrency = concurrencyTestTransformer.getMaxConcurrency(); + assertTrue(maxConcurrency > maxConnections / 2 && maxConcurrency <= maxConnections, + "concurrency didn't get limited as expected (max: " + maxConcurrency + ")"); + } +} \ No newline at end of file diff --git a/pulsar-client-admin/src/test/resources/log4j2.xml b/pulsar-client-admin/src/test/resources/log4j2.xml new file mode 100644 index 0000000000000..9b57b450ffa43 --- /dev/null +++ b/pulsar-client-admin/src/test/resources/log4j2.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index c893bb6f7e9e3..6f8ab855be372 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-client-all @@ -66,7 +65,7 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl test @@ -167,6 +166,8 @@ com.google.errorprone:* com.google.j2objc:* com.google.code.gson:gson + com.google.re2j:re2j + com.spotify:completable-futures com.fasterxml.jackson.*:* io.netty:netty io.netty:netty-all @@ -243,6 +244,10 @@ com.google.protobuf.* + + com.spotify.futures + org.apache.pulsar.shade.com.spotify.futures + com.fasterxml.jackson org.apache.pulsar.shade.com.fasterxml.jackson @@ -387,6 +392,12 @@ org.tukaani org.apache.pulsar.shade.org.tukaani + + + (META-INF/native/(lib)?)(netty.+\.(so|jnilib|dll))$ + $1org_apache_pulsar_shade_$3 + true + @@ -396,31 +407,6 @@ - - - - exec-maven-plugin - org.codehaus.mojo - - - rename-epoll-library - package - - exec - - - ${project.parent.basedir}/src/${rename.netty.native.libs} - - ${project.artifactId} - - - - - org.apache.maven.plugins maven-enforcer-plugin diff --git a/pulsar-client-api/pom.xml b/pulsar-client-api/pom.xml index 5576aed7e23bf..000a9881cdf05 100644 --- a/pulsar-client-api/pom.xml +++ b/pulsar-client-api/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-client-api @@ -46,6 +45,12 @@ protobuf-java provided + + + io.opentelemetry + opentelemetry-api + provided + diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/AuthenticationDataProvider.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/AuthenticationDataProvider.java index 27c7a1d4edb4f..f10f0884f194e 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/AuthenticationDataProvider.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/AuthenticationDataProvider.java @@ -62,7 +62,7 @@ default Certificate[] getTlsCertificates() { /** * @return a client certificate file path */ - default String getTlsCerificateFilePath() { + default String getTlsCertificateFilePath() { return null; } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java index 8b959690a0363..73ad555165c05 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java @@ -18,9 +18,11 @@ */ package org.apache.pulsar.client.api; +import io.opentelemetry.api.OpenTelemetry; import java.io.Serializable; import java.net.InetSocketAddress; import java.time.Clock; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -128,6 +130,8 @@ public interface ClientBuilder extends Serializable, Cloneable { /** * Release the connection if it is not used for more than {@param connectionMaxIdleSeconds} seconds. + * Defaults to 25 seconds. + * * @return the client builder instance */ ClientBuilder connectionMaxIdleSeconds(int connectionMaxIdleSeconds); @@ -458,7 +462,10 @@ ClientBuilder authentication(String authPluginClassName, Map aut * @param unit * time unit for {@code statsInterval} * @return the client builder instance + * + * @deprecated @see {@link #openTelemetry(OpenTelemetry)} */ + @Deprecated ClientBuilder statsInterval(long statsInterval, TimeUnit unit); /** @@ -553,6 +560,24 @@ ClientBuilder authentication(String authPluginClassName, Map aut */ ClientBuilder enableBusyWait(boolean enableBusyWait); + /** + * Configure OpenTelemetry for Pulsar Client + *

    + * When you pass an OpenTelemetry instance, Pulsar client will emit metrics that can be exported in a variety + * of different methods. + *

    + * Refer to OpenTelemetry Java SDK documentation for + * how to configure OpenTelemetry and the metrics exporter. + *

    + * By default, Pulsar client will use the {@link io.opentelemetry.api.GlobalOpenTelemetry} instance. If an + * OpenTelemetry JVM agent is configured, the metrics will be reported, otherwise the metrics will be + * completely disabled. + * + * @param openTelemetry the OpenTelemetry instance + * @return the client builder instance + */ + ClientBuilder openTelemetry(io.opentelemetry.api.OpenTelemetry openTelemetry); + /** * The clock used by the pulsar client. * @@ -575,7 +600,7 @@ ClientBuilder authentication(String authPluginClassName, Map aut * * @param proxyServiceUrl proxy service url * @param proxyProtocol protocol to decide type of proxy routing eg: SNI-routing - * @return + * @return the client builder instance */ ClientBuilder proxyServiceUrl(String proxyServiceUrl, ProxyProtocol proxyProtocol); @@ -583,7 +608,7 @@ ClientBuilder authentication(String authPluginClassName, Map aut * If enable transaction, start the transactionCoordinatorClient with pulsar client. * * @param enableTransaction whether enable transaction feature - * @return + * @return the client builder instance */ ClientBuilder enableTransaction(boolean enableTransaction); @@ -591,28 +616,68 @@ ClientBuilder authentication(String authPluginClassName, Map aut * Set dns lookup bind address and port. * @param address dnsBindAddress * @param port dnsBindPort - * @return + * @return the client builder instance */ ClientBuilder dnsLookupBind(String address, int port); + /** + * Set dns lookup server addresses. + * @param addresses dnsServerAddresses + * @return the client builder instance + */ + ClientBuilder dnsServerAddresses(List addresses); + /** * Set socks5 proxy address. * @param socks5ProxyAddress - * @return + * @return the client builder instance */ ClientBuilder socks5ProxyAddress(InetSocketAddress socks5ProxyAddress); /** * Set socks5 proxy username. * @param socks5ProxyUsername - * @return + * @return the client builder instance */ ClientBuilder socks5ProxyUsername(String socks5ProxyUsername); /** * Set socks5 proxy password. * @param socks5ProxyPassword - * @return + * @return the client builder instance */ ClientBuilder socks5ProxyPassword(String socks5ProxyPassword); + + /** + * Set the SSL Factory Plugin for custom implementation to create SSL Context and SSLEngine. + * @param sslFactoryPlugin ssl factory class name + * @return the client builder instance + */ + ClientBuilder sslFactoryPlugin(String sslFactoryPlugin); + + /** + * Set the SSL Factory Plugin params for the ssl factory plugin to use. + * @param sslFactoryPluginParams Params in String format that will be inputted to the SSL Factory Plugin + * @return the client builder instance + */ + ClientBuilder sslFactoryPluginParams(String sslFactoryPluginParams); + + /** + * Set Cert Refresh interval in seconds. + * @param autoCertRefreshSeconds + * @return the client builder instance + */ + ClientBuilder autoCertRefreshSeconds(int autoCertRefreshSeconds); + + /** + * Set the properties used for topic lookup. + *

    + * When the broker performs topic lookup, these lookup properties will be taken into consideration in a customized + * load manager. + *

    + * Note: The lookup properties are only used in topic lookup when: + * - The protocol is binary protocol, i.e. the service URL starts with "pulsar://" or "pulsar+ssl://" + * - The `loadManagerClassName` config in broker is a class that implements the `ExtensibleLoadManager` interface + */ + ClientBuilder lookupProperties(Map properties); } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Consumer.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Consumer.java index c67ad08c83631..beba2988e67ef 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Consumer.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Consumer.java @@ -73,6 +73,31 @@ public interface Consumer extends Closeable, MessageAcknowledger { */ CompletableFuture unsubscribeAsync(); + + /** + * Unsubscribe the consumer. + * + *

    This call blocks until the consumer is unsubscribed. + * + *

    Unsubscribing will the subscription to be deleted and all the + * data retained can potentially be deleted as well. + * + *

    The operation will fail when performed on a shared subscription + * where multiple consumers are currently connected. + * + * @param force forcefully unsubscribe by disconnecting connected consumers. + * @throws PulsarClientException if the operation fails + */ + void unsubscribe(boolean force) throws PulsarClientException; + + /** + * Asynchronously unsubscribe the consumer. + * + * @see Consumer#unsubscribe() + * @param force forcefully unsubscribe by disconnecting connected consumers. + * @return {@link CompletableFuture} to track the operation + */ + CompletableFuture unsubscribeAsync(boolean force); /** * Receives a single message. * @@ -474,6 +499,9 @@ CompletableFuture reconsumeLaterCumulativeAsync(Message message, *

  • MessageId.earliest : Reset the subscription on the earliest message available in the topic *
  • MessageId.latest : Reset the subscription on the latest message in the topic *
+ *

+ * This effectively resets the acknowledgement state of the subscription: all messages up to and + * including messageId will be marked as acknowledged and the rest unacknowledged. * *

Note: For multi-topics consumer, if `messageId` is a {@link TopicMessageId}, the seek operation will happen * on the owner topic of the message, which is returned by {@link TopicMessageId#getOwnerTopic()}. Otherwise, you diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java index 14a94cb8286dc..1b2e5cc5a5e51 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java @@ -126,7 +126,7 @@ public interface ConsumerBuilder extends Cloneable { ConsumerBuilder topics(List topicNames); /** - * Specify a pattern for topics that this consumer subscribes to. + * Specify a pattern for topics(not contains the partition suffix) that this consumer subscribes to. * *

The pattern is applied to subscribe to all topics, within a single namespace, that match the * pattern. @@ -134,13 +134,13 @@ public interface ConsumerBuilder extends Cloneable { *

The consumer automatically subscribes to topics created after itself. * * @param topicsPattern - * a regular expression to select a list of topics to subscribe to + * a regular expression to select a list of topics(not contains the partition suffix) to subscribe to * @return the consumer builder instance */ ConsumerBuilder topicsPattern(Pattern topicsPattern); /** - * Specify a pattern for topics that this consumer subscribes to. + * Specify a pattern for topics(not contains the partition suffix) that this consumer subscribes to. * *

It accepts a regular expression that is compiled into a pattern internally. E.g., * "persistent://public/default/pattern-topic-.*" @@ -151,7 +151,7 @@ public interface ConsumerBuilder extends Cloneable { *

The consumer automatically subscribes to topics created after itself. * * @param topicsPattern - * given regular expression for topics pattern + * given regular expression for topics(not contains the partition suffix) pattern * @return the consumer builder instance */ ConsumerBuilder topicsPattern(String topicsPattern); @@ -184,8 +184,6 @@ public interface ConsumerBuilder extends Cloneable { *

By default, the acknowledgment timeout is disabled (set to `0`, which means infinite). * When a consumer with an infinite acknowledgment timeout terminates, any unacknowledged * messages that it receives are re-delivered to another consumer. - *

Since 2.3.0, when a dead letter policy is specified and no ackTimeoutMillis is specified, - * the acknowledgment timeout is set to 30 seconds. * *

When enabling acknowledgment timeout, if a message is not acknowledged within the specified timeout, * it is re-delivered to the consumer (possibly to a different consumer, in the case of @@ -249,6 +247,7 @@ public interface ConsumerBuilder extends Cloneable { *

  • {@link SubscriptionType#Exclusive} (Default)
    • *
  • {@link SubscriptionType#Failover}
    • *
  • {@link SubscriptionType#Shared}
    • + *
  • {@link SubscriptionType#Key_Shared}
    • * * * @param subscriptionType @@ -284,6 +283,21 @@ public interface ConsumerBuilder extends Cloneable { */ ConsumerBuilder messageListener(MessageListener messageListener); + /** + * Set the {@link MessageListenerExecutor} to be used for message listeners of current consumer. + * (default: use executor from PulsarClient, + * {@link org.apache.pulsar.client.impl.PulsarClientImpl#externalExecutorProvider}). + * + *

    The listener thread pool is exclusively owned by current consumer + * that are using a "listener" model to get messages. For a given internal consumer, + * the listener will always be invoked from the same thread, to ensure ordering. + * + *

    The caller need to shut down the thread pool after closing the consumer to avoid leaks. + * @param messageListenerExecutor the executor of the consumer message listener + * @return the consumer builder instance + */ + ConsumerBuilder messageListenerExecutor(MessageListenerExecutor messageListenerExecutor); + /** * Sets a {@link CryptoKeyReader}. * @@ -346,6 +360,10 @@ public interface ConsumerBuilder extends Cloneable { * application calls {@link Consumer#receive()}. Using a higher value can potentially increase consumer * throughput at the expense of bigger memory utilization. * + *

    For the consumer that subscribes to the partitioned topic, the parameter + * {@link ConsumerBuilder#maxTotalReceiverQueueSizeAcrossPartitions} also affects + * the number of messages accumulated in the consumer. + * *

    Setting the consumer queue size as zero *

      *
    • Decreases the throughput of the consumer by disabling pre-fetching of messages. This approach improves the @@ -410,8 +428,13 @@ public interface ConsumerBuilder extends Cloneable { * of messages that a consumer can be pushed at once from a broker, across all * the partitions. * - * @param maxTotalReceiverQueueSizeAcrossPartitions - * max pending messages across all the partitions + *

      This setting is applicable only to consumers subscribing to partitioned topics. In such cases, there will + * be multiple queues for each partition and a single queue for the parent consumer. This setting controls the + * queues of all partitions, not the parent queue. For instance, if a consumer subscribes to a single partitioned + * topic, the total number of messages accumulated in this consumer will be the sum of + * {@link #receiverQueueSize(int)} and maxTotalReceiverQueueSizeAcrossPartitions. + * + * @param maxTotalReceiverQueueSizeAcrossPartitions max pending messages across all the partitions * @return the consumer builder instance */ ConsumerBuilder maxTotalReceiverQueueSizeAcrossPartitions(int maxTotalReceiverQueueSizeAcrossPartitions); @@ -456,7 +479,7 @@ public interface ConsumerBuilder extends Cloneable { ConsumerBuilder readCompacted(boolean readCompacted); /** - * Sets topic's auto-discovery period when using a pattern for topics consumer. + * Sets topic's auto-discovery period when using a pattern for topic's consumer. * The period is in minutes, and the default and minimum values are 1 minute. * * @param periodInMinutes @@ -468,7 +491,8 @@ public interface ConsumerBuilder extends Cloneable { /** - * Sets topic's auto-discovery period when using a pattern for topics consumer. + * Sets topic's auto-discovery period when using a pattern for topic's consumer. + * The default value of period is 1 minute, with a minimum of 1 second. * * @param interval * the amount of delay between checks for @@ -503,9 +527,9 @@ public interface ConsumerBuilder extends Cloneable { * Order in which broker dispatches messages to consumers: C1, C2, C3, C1, C4, C5, C4 * * - *

      Failover subscription - * The broker selects the active consumer for a failover subscription based on consumer's priority-level and - * lexicographical sorting of consumer name. + *

      Failover subscription for partitioned topic + * The broker selects the active consumer for a failover subscription for a partitioned topic + * based on consumer's priority-level and lexicographical sorting of consumer name. * eg: * 

            * 1. Active consumer = C1 : Same priority-level and lexicographical sorting
@@ -522,6 +546,8 @@ public interface ConsumerBuilder extends Cloneable {
      * Broker evenly assigns partitioned topics to highest priority consumers.
      *
      * + *

      Priority level has no effect on failover subscriptions for non-partitioned topics. + * * @param priorityLevel the priority of this consumer * @return the consumer builder instance */ @@ -780,31 +806,66 @@ public interface ConsumerBuilder extends Cloneable { ConsumerBuilder messagePayloadProcessor(MessagePayloadProcessor payloadProcessor); /** - * negativeAckRedeliveryBackoff doesn't work with `consumer.negativeAcknowledge(MessageId messageId)` - * because we are unable to get the redelivery count from the message ID. + * negativeAckRedeliveryBackoff sets the redelivery backoff policy for messages that are negatively acknowledged + * using + * `consumer.negativeAcknowledge(Message message)` but not with `consumer.negativeAcknowledge(MessageId + * messageId)`. + * This setting allows specifying a backoff policy for messages that are negatively acknowledged, + * enabling more flexible control over the delay before such messages are redelivered. * - *

      Example: - * 

      -     * client.newConsumer().negativeAckRedeliveryBackoff(ExponentialRedeliveryBackoff.builder()
-     *              .minNackTimeMs(1000)
-     *              .maxNackTimeMs(60 * 1000)
-     *              .build()).subscribe();
-     *
      + *

      This configuration accepts a {@link RedeliveryBackoff} object that defines the backoff policy. + * The policy can be either a fixed delay or an exponential backoff. An exponential backoff policy + * is beneficial in scenarios where increasing the delay between consecutive redeliveries can help + * mitigate issues like temporary resource constraints or processing bottlenecks. + * + *

      Note: This backoff policy does not apply when using `consumer.negativeAcknowledge(MessageId messageId)` + * because the redelivery count cannot be determined from just the message ID. It is recommended to use + * `consumer.negativeAcknowledge(Message message)` if you want to leverage the redelivery backoff policy. + * + *

      Example usage: + * 

      {@code
+     * client.newConsumer()
+     *       .negativeAckRedeliveryBackoff(ExponentialRedeliveryBackoff.builder()
+     *           .minDelayMs(1000)   // Set minimum delay to 1 second
+     *           .maxDelayMs(60000)  // Set maximum delay to 60 seconds
+     *           .build())
+     *       .subscribe();
+     * }
      + * + * @param negativeAckRedeliveryBackoff the backoff policy to use for negatively acknowledged messages + * @return the consumer builder instance */ ConsumerBuilder negativeAckRedeliveryBackoff(RedeliveryBackoff negativeAckRedeliveryBackoff); + /** - * redeliveryBackoff doesn't work with `consumer.negativeAcknowledge(MessageId messageId)` - * because we are unable to get the redelivery count from the message ID. + * Sets the redelivery backoff policy for messages that are redelivered due to acknowledgement timeout. + * This setting allows you to specify a backoff policy for messages that are not acknowledged within + * the specified ack timeout. By using a backoff policy, you can control the delay before a message + * is redelivered, potentially improving consumer performance by avoiding immediate redelivery of + * messages that might still be processing. * - *

      Example: - * 

      -     * client.newConsumer().ackTimeout(10, TimeUnit.SECOND)
-     *              .ackTimeoutRedeliveryBackoff(ExponentialRedeliveryBackoff.builder()
-     *              .minNackTimeMs(1000)
-     *              .maxNackTimeMs(60 * 1000)
-     *              .build()).subscribe();
-     *
      + *

      This method accepts a {@link RedeliveryBackoff} object that defines the backoff policy to be used. + * You can use either a fixed backoff policy or an exponential backoff policy. The exponential backoff + * policy is particularly useful for scenarios where it may be beneficial to progressively increase the + * delay between redeliveries, reducing the load on the consumer and giving more time to process messages. + * + *

      Example usage: + * 

      {@code
+     * client.newConsumer()
+     *       .ackTimeout(10, TimeUnit.SECONDS)
+     *       .ackTimeoutRedeliveryBackoff(ExponentialRedeliveryBackoff.builder()
+     *           .minDelayMs(1000)   // Set minimum delay to 1 second
+     *           .maxDelayMs(60000)  // Set maximum delay to 60 seconds
+     *           .build())
+     *       .subscribe();
+     * }
      + * + *

      Note: This configuration is effective only if the ack timeout is triggered. It does not apply to + * messages negatively acknowledged using the negative acknowledgment API. + * + * @param ackTimeoutRedeliveryBackoff the backoff policy to use for messages that exceed their ack timeout + * @return the consumer builder instance */ ConsumerBuilder ackTimeoutRedeliveryBackoff(RedeliveryBackoff ackTimeoutRedeliveryBackoff); diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerInterceptor.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerInterceptor.java index be2f9b0f10826..1beea3adba239 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerInterceptor.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerInterceptor.java @@ -41,6 +41,44 @@ public interface ConsumerInterceptor extends AutoCloseable { */ void close(); + /** + * This method is called when a message arrives in the consumer. + * + *

      This method provides visibility into the messages that have been received + * by the consumer but have not yet been processed. This can be useful for + * monitoring the state of the consumer's receiver queue and understanding + * the consumer's processing rate. + * + *

      The method is allowed to modify the message, in which case the modified + * message will be returned. + * + *

      Any exception thrown by this method will be caught by the caller, logged, + * but not propagated to the client. + * + *

      Since the consumer may run multiple interceptors, a particular + * interceptor's onArrival callback will be called in the order + * specified by {@link ConsumerBuilder#intercept(ConsumerInterceptor[])}. The + * first interceptor in the list gets the consumed message, the following + * interceptor will be passed the message returned by the previous interceptor, + * and so on. Since interceptors are allowed to modify the message, interceptors + * may potentially get the messages already modified by other interceptors. + * However, building a pipeline of mutable interceptors that depend on the output + * of the previous interceptor is discouraged, because of potential side-effects + * caused by interceptors potentially failing to modify the message and throwing + * an exception. If one of the interceptors in the list throws an exception from + * onArrival, the exception is caught, logged, and the next interceptor + * is called with the message returned by the last successful interceptor in the + * list, or otherwise the original consumed message. + * + * @param consumer the consumer which contains the interceptor + * @param message the message that has arrived in the receiver queue + * @return the message that is either modified by the interceptor or the same + * message passed into the method + */ + default Message onArrival(Consumer consumer, Message message) { + return message; + } + /** * This is called just before the message is returned by * {@link Consumer#receive()}, {@link MessageListener#received(Consumer, diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerStats.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerStats.java index 529101ecde39c..e488aa81151ce 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerStats.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerStats.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.api; +import io.opentelemetry.api.OpenTelemetry; import java.io.Serializable; import java.util.Collections; import java.util.Map; @@ -29,9 +30,12 @@ * *

      All the stats are relative to the last recording period. The interval of the stats refreshes is configured with * {@link ClientBuilder#statsInterval(long, java.util.concurrent.TimeUnit)} with a default of 1 minute. + * + * @deprecated use {@link ClientBuilder#openTelemetry(OpenTelemetry)} to enable stats */ @InterfaceAudience.Public -@InterfaceStability.Stable +@InterfaceStability.Evolving +@Deprecated public interface ConsumerStats extends Serializable { /** @@ -122,4 +126,14 @@ public interface ConsumerStats extends Serializable { default Map getPartitionStats() { return Collections.emptyMap(); } + + /** + * @return producer stats for deadLetterProducer if available + */ + ProducerStats getDeadLetterProducerStats(); + + /** + * @return producer stats for retryLetterProducer if available + */ + ProducerStats getRetryLetterProducerStats(); } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/DummyCryptoKeyReaderImpl.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/DummyCryptoKeyReaderImpl.java new file mode 100644 index 0000000000000..df9392db2deec --- /dev/null +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/DummyCryptoKeyReaderImpl.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api; + +import java.util.Map; + +/** + * An empty implement. Doesn't provide any public key or private key, and just returns `null`. + */ +public class DummyCryptoKeyReaderImpl implements CryptoKeyReader { + + public static final DummyCryptoKeyReaderImpl INSTANCE = new DummyCryptoKeyReaderImpl(); + + private DummyCryptoKeyReaderImpl(){} + + @Override + public EncryptionKeyInfo getPublicKey(String keyName, Map metadata) { + return null; + } + + @Override + public EncryptionKeyInfo getPrivateKey(String keyName, Map metadata) { + return null; + } +} \ No newline at end of file diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/MessageListenerExecutor.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/MessageListenerExecutor.java new file mode 100644 index 0000000000000..53bb828c05aa8 --- /dev/null +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/MessageListenerExecutor.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api; + +/** + * Interface for providing service to execute message listeners. + */ +public interface MessageListenerExecutor { + + /** + * select a thread by message to execute the runnable! + *

      + * Suggestions: + *

      + * 1. The message listener task will be submitted to this executor for execution, + * so the implementations of this interface should carefully consider execution + * order if sequential consumption is required. + *

      + *

      + * 2. The users should release resources(e.g. threads) of the executor after closing + * the consumer to avoid leaks. + *

      + * @param message the message + * @param runnable the runnable to execute, that is, the message listener task + */ + void execute(Message message, Runnable runnable); +} diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerStats.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerStats.java index a26c20e740d37..9a9ade73669dd 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerStats.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerStats.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.api; +import io.opentelemetry.api.OpenTelemetry; import java.io.Serializable; import java.util.Collections; import java.util.Map; @@ -29,9 +30,12 @@ * *

      All the stats are relative to the last recording period. The interval of the stats refreshes is configured with * {@link ClientBuilder#statsInterval(long, java.util.concurrent.TimeUnit)} with a default of 1 minute. + * + * @deprecated use {@link ClientBuilder#openTelemetry(OpenTelemetry)} to enable stats */ @InterfaceAudience.Public -@InterfaceStability.Stable +@InterfaceStability.Evolving +@Deprecated public interface ProducerStats extends Serializable { /** * @return the number of messages published in the last interval diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClient.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClient.java index 78952fcaed8b3..6c46bce254f6f 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClient.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClient.java @@ -308,14 +308,33 @@ static ClientBuilder builder() { * *

      This can be used to discover the partitions and create {@link Reader}, {@link Consumer} or {@link Producer} * instances directly on a particular partition. - * + * @Deprecated it is not suggested to use now; please use {@link #getPartitionsForTopic(String, boolean)}. * @param topic * the topic name * @return a future that will yield a list of the topic partitions or {@link PulsarClientException} if there was any * error in the operation. + * * @since 2.3.0 */ - CompletableFuture> getPartitionsForTopic(String topic); + @Deprecated + default CompletableFuture> getPartitionsForTopic(String topic) { + return getPartitionsForTopic(topic, true); + } + + /** + * 1. Get the partitions if the topic exists. Return "[{partition-0}, {partition-1}....{partition-n}}]" if a + * partitioned topic exists; return "[{topic}]" if a non-partitioned topic exists. + * 2. When {@param metadataAutoCreationEnabled} is "false", neither the partitioned topic nor non-partitioned + * topic does not exist. You will get an {@link PulsarClientException.NotFoundException} or a + * {@link PulsarClientException.TopicDoesNotExistException}. + * 2-1. You will get a {@link PulsarClientException.NotSupportedException} with metadataAutoCreationEnabled=false + * on an old broker version which does not support getting partitions without partitioned metadata auto-creation. + * 3. When {@param metadataAutoCreationEnabled} is "true," it will trigger an auto-creation for this topic(using + * the default topic auto-creation strategy you set for the broker), and the corresponding result is returned. + * For the result, see case 1. + * @version 3.3.0. + */ + CompletableFuture> getPartitionsForTopic(String topic, boolean metadataAutoCreationEnabled); /** * Close the PulsarClient and release all the resources. diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClientException.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClientException.java index 9409eefe2e0f0..9eb6c612a52a2 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClientException.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClientException.java @@ -19,9 +19,10 @@ package org.apache.pulsar.client.api; import java.io.IOException; -import java.util.Collection; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.Getter; import org.apache.pulsar.common.classification.InterfaceAudience; import org.apache.pulsar.common.classification.InterfaceStability; @@ -33,7 +34,7 @@ @SuppressWarnings("serial") public class PulsarClientException extends IOException { private long sequenceId = -1; - private Collection previous; + private AtomicInteger previousExceptionAttempt; /** * Constructs an {@code PulsarClientException} with the specified detail message. @@ -86,47 +87,16 @@ public PulsarClientException(String msg, Throwable t) { super(msg, t); } - /** - * Add a list of previous exception which occurred for the same operation - * and have been retried. - * - * @param previous A collection of throwables that triggered retries - */ - public void setPreviousExceptions(Collection previous) { - this.previous = previous; - } - - /** - * Get the collection of previous exceptions which have caused retries - * for this operation. - * - * @return a collection of exception, ordered as they occurred - */ - public Collection getPreviousExceptions() { - return this.previous; + public void setPreviousExceptionCount(AtomicInteger previousExceptionCount) { + this.previousExceptionAttempt = previousExceptionCount; } @Override public String toString() { - if (previous == null || previous.isEmpty()) { + if (previousExceptionAttempt == null || previousExceptionAttempt.get() == 0) { return super.toString(); } else { - StringBuilder sb = new StringBuilder(super.toString()); - int i = 0; - boolean first = true; - sb.append("{\"previous\":["); - for (Throwable t : previous) { - if (first) { - first = false; - } else { - sb.append(','); - } - sb.append("{\"attempt\":").append(i++) - .append(",\"error\":\"").append(t.toString().replace("\"", "\\\"")) - .append("\"}"); - } - sb.append("]}"); - return sb.toString(); + return super.toString() + ", previous-attempt: " + previousExceptionAttempt; } } /** @@ -344,6 +314,22 @@ public TopicDoesNotExistException(String msg) { } } + /** + * Not found subscription that cannot be created. + */ + public static class SubscriptionNotFoundException extends PulsarClientException { + /** + * Constructs an {@code SubscriptionNotFoundException} with the specified detail message. + * + * @param msg + * The detail message (which is saved for later retrieval + * by the {@link #getMessage()} method) + */ + public SubscriptionNotFoundException(String msg) { + super(msg); + } + } + /** * Lookup exception thrown by Pulsar client. */ @@ -658,6 +644,10 @@ public NotConnectedException() { public NotConnectedException(long sequenceId) { super("Not connected to broker", sequenceId); } + + public NotConnectedException(String msg) { + super(msg); + } } /** @@ -721,6 +711,30 @@ public NotSupportedException(String msg) { } } + /** + * Not supported exception thrown by Pulsar client. + */ + public static class FeatureNotSupportedException extends NotSupportedException { + + @Getter + private final FailedFeatureCheck failedFeatureCheck; + + public FeatureNotSupportedException(String msg, FailedFeatureCheck failedFeatureCheck) { + super(msg); + this.failedFeatureCheck = failedFeatureCheck; + } + } + + /** + * "supports_auth_refresh" was introduced at "2.6" and is no longer supported, so skip this enum. + * "supports_broker_entry_metadata" was introduced at "2.8" and is no longer supported, so skip this enum. + * "supports_partial_producer" was introduced at "2.10" and is no longer supported, so skip this enum. + * "supports_topic_watchers" was introduced at "2.11" and is no longer supported, so skip this enum. + */ + public enum FailedFeatureCheck { + SupportsGetPartitionedMetadataWithoutAutoCreation; + } + /** * Not allowed exception thrown by Pulsar client. */ @@ -1111,39 +1125,9 @@ public static PulsarClientException unwrap(Throwable t) { newException = new PulsarClientException(t); } - Collection previousExceptions = getPreviousExceptions(t); - if (previousExceptions != null) { - newException.setPreviousExceptions(previousExceptions); - } return newException; } - public static Collection getPreviousExceptions(Throwable t) { - Throwable e = t; - for (int maxDepth = 20; maxDepth > 0 && e != null; maxDepth--) { - if (e instanceof PulsarClientException) { - Collection previous = ((PulsarClientException) e).getPreviousExceptions(); - if (previous != null) { - return previous; - } - } - e = t.getCause(); - } - return null; - } - - public static void setPreviousExceptions(Throwable t, Collection previous) { - Throwable e = t; - for (int maxDepth = 20; maxDepth > 0 && e != null; maxDepth--) { - if (e instanceof PulsarClientException) { - ((PulsarClientException) e).setPreviousExceptions(previous); - return; - } - e = t.getCause(); - } - } - - public long getSequenceId() { return sequenceId; } @@ -1159,6 +1143,7 @@ public static boolean isRetriableError(Throwable t) { || t instanceof NotFoundException || t instanceof IncompatibleSchemaException || t instanceof TopicDoesNotExistException + || t instanceof SubscriptionNotFoundException || t instanceof UnsupportedAuthenticationException || t instanceof InvalidMessageException || t instanceof InvalidTopicNameException @@ -1176,4 +1161,12 @@ public static boolean isRetriableError(Throwable t) { } return true; } + + public static void setPreviousExceptionCount(Throwable e, AtomicInteger previousExceptionCount) { + if (e instanceof PulsarClientException) { + ((PulsarClientException) e).setPreviousExceptionCount(previousExceptionCount); + return; + } + } + } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Range.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Range.java index 4437ffc4ac6a2..488083f484b76 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Range.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Range.java @@ -27,7 +27,7 @@ */ @InterfaceAudience.Public @InterfaceStability.Stable -public class Range { +public class Range implements Comparable { private final int start; private final int end; @@ -84,4 +84,13 @@ public int hashCode() { public String toString() { return "[" + start + ", " + end + "]"; } + + @Override + public int compareTo(Range o) { + int result = Integer.compare(start, o.start); + if (result == 0) { + result = Integer.compare(end, o.end); + } + return result; + } } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Reader.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Reader.java index 419a759f118ba..f151278ebadda 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Reader.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Reader.java @@ -19,6 +19,7 @@ package org.apache.pulsar.client.api; import java.io.Closeable; +import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.function.Function; @@ -217,4 +218,19 @@ public interface Reader extends Closeable { * @return a future to track the completion of the seek operation */ CompletableFuture seekAsync(long timestamp); + + /** + * Get all the last message id of the topics the reader subscribed. + * + * @return the list of TopicMessageId instances of all the topics that the reader subscribed + * @throws PulsarClientException if failed to get last message id. + * @apiNote It's guaranteed that the owner topic of each TopicMessageId in the returned list is different from owner + * topics of other TopicMessageId instances + */ + List getLastMessageIds() throws PulsarClientException; + + /** + * The asynchronous version of {@link Reader#getLastMessageIds()}. + */ + CompletableFuture> getLastMessageIdsAsync(); } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ReaderBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ReaderBuilder.java index 7241bd4cf4ff8..d522c3c4b6e4e 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ReaderBuilder.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ReaderBuilder.java @@ -210,6 +210,15 @@ public interface ReaderBuilder extends Cloneable { */ ReaderBuilder cryptoFailureAction(ConsumerCryptoFailureAction action); + /** + * Sets a {@link MessageCrypto}. + * + *

      Contains methods to encrypt/decrypt message for End to End Encryption. + * + * @param messageCrypto message Crypto Object + * @return ReaderBuilder instance + */ + ReaderBuilder messageCrypto(MessageCrypto messageCrypto); /** * Sets the size of the consumer receive queue. * diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ServiceUrlProvider.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ServiceUrlProvider.java index 5cb22276553ab..e8b513b103f65 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ServiceUrlProvider.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ServiceUrlProvider.java @@ -56,7 +56,7 @@ public interface ServiceUrlProvider extends AutoCloseable { * */ @Override - default void close() { + default void close() throws Exception { // do nothing } } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TableView.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TableView.java index 9e5008c8bd0c8..767b8e1103fa6 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TableView.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TableView.java @@ -110,4 +110,38 @@ public interface TableView extends Closeable { * @return a future that can used to track when the table view has been closed. */ CompletableFuture closeAsync(); + + /** + * Refresh the table view with the latest data in the topic, ensuring that all subsequent reads are based on + * the refreshed data. + * + * Example usage: + * + * table.refreshAsync().thenApply(__ -> table.get(key)); + * + * This function retrieves the last written message in the topic and refreshes the table view accordingly. + * Once the refresh is complete, all subsequent reads will be performed on the refreshed data or a combination of + * the refreshed data and newly published data. The table view remains synchronized with any newly published data + * after the refresh. + * + * |x:0|->|y:0|->|z:0|->|x:1|->|z:1|->|x:2|->|y:1|->|y:2| + * + * If a read occurs after the refresh (at the last published message |y:2|), it ensures that outdated data like x=1 + * is not obtained. However, it does not guarantee that the values will always be x=2, y=2, z=1, + * as the table view may receive updates with newly published data. + * + * |x:0|->|y:0|->|z:0|->|x:1|->|z:1|->|x:2|->|y:1|->|y:2| -> |y:3| + * + * Both y=2 or y=3 are possible. Therefore, different readers may receive different values, + * but all values will be equal to or newer than the data refreshed from the last call to the refresh method. + */ + CompletableFuture refreshAsync(); + + /** + * Refresh the table view with the latest data in the topic, ensuring that all subsequent reads are based on + * the refreshed data. + * + * @throws PulsarClientException if there is any error refreshing the table view. + */ + void refresh() throws PulsarClientException; } diff --git a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/STSAssumeRoleProviderPlugin.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TransactionIsolationLevel.java similarity index 62% rename from pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/STSAssumeRoleProviderPlugin.java rename to pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TransactionIsolationLevel.java index e305c9c9b9fe2..ae385b20232c7 100644 --- a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/STSAssumeRoleProviderPlugin.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TransactionIsolationLevel.java @@ -16,16 +16,16 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.io.kinesis; +package org.apache.pulsar.client.api; -/** - * This is a stub class for backwards compatibility. In new code and configurations, please use the plugins - * from org.apache.pulsar.io.aws - * - * @see org.apache.pulsar.io.aws.STSAssumeRoleProviderPlugin - */ -@Deprecated -public class STSAssumeRoleProviderPlugin extends org.apache.pulsar.io.aws.STSAssumeRoleProviderPlugin - implements AwsCredentialProviderPlugin { -} +import org.apache.pulsar.common.classification.InterfaceAudience; +import org.apache.pulsar.common.classification.InterfaceStability; +@InterfaceAudience.Public +@InterfaceStability.Stable +public enum TransactionIsolationLevel { + // Consumer can only consume all transactional messages which have been committed. + READ_COMMITTED, + // Consumer can consume all messages, even transactional messages which have been aborted. + READ_UNCOMMITTED; +} diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml index e282cdd4e5379..d42d0846d8380 100644 --- a/pulsar-client-auth-athenz/pom.xml +++ b/pulsar-client-auth-athenz/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-client-auth-athenz @@ -52,6 +51,11 @@ athenz-cert-refresher + + org.bouncycastle + bcpkix-jdk18on + + com.google.guava guava diff --git a/pulsar-client-auth-sasl/pom.xml b/pulsar-client-auth-sasl/pom.xml index 2b88386971b8c..172bf32b60514 100644 --- a/pulsar-client-auth-sasl/pom.xml +++ b/pulsar-client-auth-sasl/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-client-auth-sasl diff --git a/pulsar-client-auth-sasl/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationSasl.java b/pulsar-client-auth-sasl/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationSasl.java index f51fb766f03db..f7ec9b964c6df 100644 --- a/pulsar-client-auth-sasl/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationSasl.java +++ b/pulsar-client-auth-sasl/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationSasl.java @@ -165,6 +165,12 @@ public void start() throws PulsarClientException { public void close() throws IOException { if (client != null) { client.close(); + client = null; + } + if (jaasCredentialsContainer != null) { + jaasCredentialsContainer.close(); + jaasCredentialsContainer = null; + initializedJAAS = false; } } diff --git a/pulsar-client-messagecrypto-bc/pom.xml b/pulsar-client-messagecrypto-bc/pom.xml index 4e2604f33169a..d79479db487fc 100644 --- a/pulsar-client-messagecrypto-bc/pom.xml +++ b/pulsar-client-messagecrypto-bc/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-client-messagecrypto-bc diff --git a/pulsar-client-messagecrypto-bc/src/main/java/org/apache/pulsar/client/impl/crypto/MessageCryptoBc.java b/pulsar-client-messagecrypto-bc/src/main/java/org/apache/pulsar/client/impl/crypto/MessageCryptoBc.java index 2d7b779fa7b6c..e41ce633c8824 100644 --- a/pulsar-client-messagecrypto-bc/src/main/java/org/apache/pulsar/client/impl/crypto/MessageCryptoBc.java +++ b/pulsar-client-messagecrypto-bc/src/main/java/org/apache/pulsar/client/impl/crypto/MessageCryptoBc.java @@ -35,6 +35,7 @@ import java.security.PublicKey; import java.security.SecureRandom; import java.security.Security; +import java.security.spec.AlgorithmParameterSpec; import java.security.spec.InvalidKeySpecException; import java.util.HashMap; import java.util.List; @@ -52,6 +53,7 @@ import javax.crypto.ShortBufferException; import javax.crypto.spec.GCMParameterSpec; import javax.crypto.spec.SecretKeySpec; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.CryptoKeyReader; import org.apache.pulsar.client.api.EncryptionKeyInfo; @@ -73,6 +75,7 @@ import org.bouncycastle.jce.spec.ECParameterSpec; import org.bouncycastle.jce.spec.ECPrivateKeySpec; import org.bouncycastle.jce.spec.ECPublicKeySpec; +import org.bouncycastle.jce.spec.IESParameterSpec; import org.bouncycastle.openssl.PEMException; import org.bouncycastle.openssl.PEMKeyPair; import org.bouncycastle.openssl.PEMParser; @@ -81,14 +84,15 @@ @Slf4j public class MessageCryptoBc implements MessageCrypto { - private static final String ECDSA = "ECDSA"; - private static final String RSA = "RSA"; - private static final String ECIES = "ECIES"; + public static final String ECDSA = "ECDSA"; + public static final String RSA = "RSA"; + public static final String ECIES = "ECIES"; // Ideally the transformation should also be part of the message property. This will prevent client // from assuming hardcoded value. However, it will increase the size of the message even further. - private static final String RSA_TRANS = "RSA/NONE/OAEPWithSHA1AndMGF1Padding"; - private static final String AESGCM = "AES/GCM/NoPadding"; + public static final String RSA_TRANS = "RSA/NONE/OAEPWithSHA1AndMGF1Padding"; + public static final String AESGCM = "AES/GCM/NoPadding"; + private static final String AESGCM_PROVIDER_NAME; private static KeyGenerator keyGenerator; private static final int tagLen = 16 * 8; @@ -98,6 +102,7 @@ public class MessageCryptoBc implements MessageCrypto dataKeyCache; @@ -119,6 +124,15 @@ public class MessageCryptoBc implements MessageCrypto Remove the key identified by the keyName from the list of keys.

      * @@ -474,23 +501,27 @@ private boolean decryptDataKey(String keyName, byte[] encryptedDataKey, List org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-client @@ -145,6 +144,8 @@ com.google.errorprone:* com.google.j2objc:* com.google.code.gson:gson + com.google.re2j:re2j + com.spotify:completable-futures com.fasterxml.jackson.*:* io.netty:* io.netty.incubator:* @@ -204,6 +205,10 @@ com.google.protobuf.* + + com.spotify.futures + org.apache.pulsar.shade.com.spotify.futures + com.fasterxml.jackson org.apache.pulsar.shade.com.fasterxml.jackson @@ -300,6 +305,12 @@ org.apache.bookkeeper org.apache.pulsar.shade.org.apache.bookkeeper + + + (META-INF/native/(lib)?)(netty.+\.(so|jnilib|dll))$ + $1org_apache_pulsar_shade_$3 + true + @@ -323,31 +334,6 @@ - - - - exec-maven-plugin - org.codehaus.mojo - - - rename-epoll-library - package - - exec - - - ${project.parent.basedir}/src/${rename.netty.native.libs} - - ${project.artifactId} - - - - - diff --git a/pulsar-client-tools-api/pom.xml b/pulsar-client-tools-api/pom.xml index 19a564bc10f0b..a4cf84d4cbc03 100644 --- a/pulsar-client-tools-api/pom.xml +++ b/pulsar-client-tools-api/pom.xml @@ -24,8 +24,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-client-tools-api diff --git a/pulsar-client-tools-customcommand-example/pom.xml b/pulsar-client-tools-customcommand-example/pom.xml index a3a3de19202c2..ef3bd97403caa 100644 --- a/pulsar-client-tools-customcommand-example/pom.xml +++ b/pulsar-client-tools-customcommand-example/pom.xml @@ -22,8 +22,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT 4.0.0 pulsar-client-tools-customcommand-example diff --git a/pulsar-client-tools-test/pom.xml b/pulsar-client-tools-test/pom.xml index 74278374da518..2f55ca0a23ab0 100644 --- a/pulsar-client-tools-test/pom.xml +++ b/pulsar-client-tools-test/pom.xml @@ -24,8 +24,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-client-tools-test diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java index 39ede3bb7aef1..d3087b7fc873c 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java @@ -29,13 +29,9 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.io.PrintStream; -import java.util.Arrays; -import java.util.List; +import java.io.PrintWriter; +import java.io.StringWriter; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.admin.cli.CmdFunctions.CreateFunction; import org.apache.pulsar.admin.cli.CmdFunctions.DeleteFunction; @@ -55,6 +51,7 @@ import org.apache.pulsar.functions.api.utils.IdentityFunction; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import picocli.CommandLine; /** * Unit test of {@link CmdFunctions}. @@ -168,7 +165,8 @@ public void testCreateFunction() throws Exception { "--className", DummyFunction.class.getName(), "--dead-letter-topic", "test-dead-letter-topic", "--custom-runtime-options", "custom-runtime-options", - "--user-config", "{\"key\": [\"value1\", \"value2\"]}" + "--user-config", "{\"key\": [\"value1\", \"value2\"]}", + "--runtime-flags", "--add-opens java.base/java.lang=ALL-UNNAMED" }); CreateFunction creater = cmd.getCreater(); @@ -178,6 +176,7 @@ public void testCreateFunction() throws Exception { assertEquals(Boolean.FALSE, creater.getAutoAck()); assertEquals("test-dead-letter-topic", creater.getDeadLetterTopic()); assertEquals("custom-runtime-options", creater.getCustomRuntimeOptions()); + assertEquals("--add-opens java.base/java.lang=ALL-UNNAMED", creater.getRuntimeFlags()); verify(functions, times(1)).createFunction(any(FunctionConfig.class), anyString()); @@ -542,11 +541,12 @@ public void testCreateWithoutOutputTopicWithSkipFlag() throws Exception { @Test - public void testCreateWithoutOutputTopic() { - - ConsoleOutputCapturer consoleOutputCapturer = new ConsoleOutputCapturer(); - consoleOutputCapturer.start(); - + public void testCreateWithoutOutputTopic() throws Exception { + @Cleanup + StringWriter stringWriter = new StringWriter(); + @Cleanup + PrintWriter printWriter = new PrintWriter(stringWriter); + cmd.getCommander().setOut(printWriter); cmd.run(new String[] { "create", "--inputs", INPUT_TOPIC_NAME, @@ -557,9 +557,8 @@ public void testCreateWithoutOutputTopic() { }); CreateFunction creater = cmd.getCreater(); - consoleOutputCapturer.stop(); assertNull(creater.getFunctionConfig().getOutput()); - assertTrue(consoleOutputCapturer.getStdout().contains("Created successfully")); + assertTrue(stringWriter.toString().contains("Created successfully")); } @Test @@ -655,17 +654,19 @@ public void testStateGetter() throws Exception { @Test public void testStateGetterWithoutKey() throws Exception { - ConsoleOutputCapturer consoleOutputCapturer = new ConsoleOutputCapturer(); - consoleOutputCapturer.start(); - cmd.run(new String[] { + CommandLine commander = cmd.getCommander(); + @Cleanup + StringWriter stringWriter = new StringWriter(); + @Cleanup + PrintWriter printWriter = new PrintWriter(stringWriter); + commander.setErr(printWriter); + cmd.run(new String[]{ "querystate", "--tenant", TENANT, "--namespace", NAMESPACE, "--name", FN_NAME, }); - consoleOutputCapturer.stop(); - String output = consoleOutputCapturer.getStderr(); - assertTrue(output.replace("\n", "").contains("State key needs to be specified")); + assertTrue(stringWriter.toString().startsWith(("State key needs to be specified"))); StateGetter stateGetter = cmd.getStateGetter(); assertEquals(TENANT, stateGetter.getTenant()); assertEquals(NAMESPACE, stateGetter.getNamespace()); @@ -896,79 +897,4 @@ public void testDownloadTransformFunction() throws Exception { verify(functions, times(1)) .downloadFunction(JAR_NAME, TENANT, NAMESPACE, FN_NAME, true); } - - - public static class ConsoleOutputCapturer { - private ByteArrayOutputStream stdout; - private ByteArrayOutputStream stderr; - private PrintStream previous; - private boolean capturing; - - public void start() { - if (capturing) { - return; - } - - capturing = true; - previous = System.out; - stdout = new ByteArrayOutputStream(); - stderr = new ByteArrayOutputStream(); - - OutputStream outputStreamCombinerstdout = - new OutputStreamCombiner(Arrays.asList(previous, stdout)); - PrintStream stdoutStream = new PrintStream(outputStreamCombinerstdout); - - OutputStream outputStreamCombinerStderr = - new OutputStreamCombiner(Arrays.asList(previous, stderr)); - PrintStream stderrStream = new PrintStream(outputStreamCombinerStderr); - - System.setOut(stdoutStream); - System.setErr(stderrStream); - } - - public void stop() { - if (!capturing) { - return; - } - - System.setOut(previous); - - previous = null; - capturing = false; - } - - public String getStdout() { - return stdout.toString(); - } - - public String getStderr() { - return stderr.toString(); - } - - private static class OutputStreamCombiner extends OutputStream { - private List outputStreams; - - public OutputStreamCombiner(List outputStreams) { - this.outputStreams = outputStreams; - } - - public void write(int b) throws IOException { - for (OutputStream os : outputStreams) { - os.write(b); - } - } - - public void flush() throws IOException { - for (OutputStream os : outputStreams) { - os.flush(); - } - } - - public void close() throws IOException { - for (OutputStream os : outputStreams) { - os.close(); - } - } - } - } } diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/DeprecatedCommanderTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/DeprecatedCommanderTest.java deleted file mode 100644 index 3112344bedcda..0000000000000 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/DeprecatedCommanderTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.admin.cli; - - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotEquals; -import static org.testng.Assert.assertTrue; -import com.beust.jcommander.DefaultUsageFormatter; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.admin.Schemas; -import org.apache.pulsar.client.admin.Topics; -import org.mockito.Mockito; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -public class DeprecatedCommanderTest { - PulsarAdmin admin; - Topics mockTopics; - Schemas mockSchemas; - CmdTopics cmdTopics; - - @BeforeMethod - public void setup() { - admin = Mockito.mock(PulsarAdmin.class); - mockTopics = mock(Topics.class); - when(admin.topics()).thenReturn(mockTopics); - mockSchemas = mock(Schemas.class); - when(admin.schemas()).thenReturn(mockSchemas); - cmdTopics = new CmdTopics(() -> admin); - } - - @Test - public void testDeprecatedCommanderWorks() throws Exception { - - DefaultUsageFormatter defaultUsageFormatter = new DefaultUsageFormatter(cmdTopics.jcommander); - StringBuilder builder = new StringBuilder(); - defaultUsageFormatter.usage(builder); - String defaultOutput = builder.toString(); - - StringBuilder builder2 = new StringBuilder(); - cmdTopics.jcommander.getUsageFormatter().usage(builder2); - String outputWithFiltered = builder2.toString(); - - assertNotEquals(outputWithFiltered, defaultOutput); - assertFalse(outputWithFiltered.contains("enable-deduplication")); - assertTrue(defaultOutput.contains("enable-deduplication")); - assertFalse(outputWithFiltered.contains("get-max-unacked-messages-on-consumer")); - assertTrue(defaultOutput.contains("get-max-unacked-messages-on-consumer")); - assertFalse(outputWithFiltered.contains("get-deduplication")); - assertTrue(defaultOutput.contains("get-deduplication")); - - // annotation was changed to hidden, reset it. - cmdTopics = new CmdTopics(() -> admin); - CmdUsageFormatter formatter = (CmdUsageFormatter)cmdTopics.jcommander.getUsageFormatter(); - formatter.clearDeprecatedCommand(); - StringBuilder builder3 = new StringBuilder(); - cmdTopics.jcommander.getUsageFormatter().usage(builder3); - String outputAfterClean = builder3.toString(); - - assertEquals(outputAfterClean, defaultOutput); - - } - -} diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java index a1d5d695c6398..5f8c9f49d65d1 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.admin.cli; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.longThat; @@ -33,14 +35,13 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import static org.testng.AssertJUnit.assertNotNull; - -import com.beust.jcommander.JCommander; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.PrintStream; +import java.io.PrintWriter; import java.net.URL; import java.net.URLClassLoader; import java.nio.charset.StandardCharsets; @@ -54,6 +55,7 @@ import java.util.Properties; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.admin.cli.extensions.CustomCommandFactory; import org.apache.pulsar.admin.cli.utils.SchemaExtractor; @@ -79,6 +81,7 @@ import org.apache.pulsar.client.admin.internal.OffloadProcessStatusImpl; import org.apache.pulsar.client.admin.internal.PulsarAdminImpl; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.TransactionIsolationLevel; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.api.transaction.TxnID; @@ -121,6 +124,7 @@ import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.Test; +import picocli.CommandLine; @Slf4j public class PulsarAdminToolTest { @@ -166,6 +170,8 @@ public void brokers() throws Exception { brokers.run(split("version")); verify(mockBrokers).getVersion(); + doReturn(CompletableFuture.completedFuture(null)).when(mockBrokers) + .shutDownBrokerGracefully(anyInt(), anyBoolean()); brokers.run(split("shutdown -m 10 -f")); verify(mockBrokers).shutDownBrokerGracefully(10,true); } @@ -178,17 +184,25 @@ public void brokerStats() throws Exception { CmdBrokerStats brokerStats = new CmdBrokerStats(() -> admin); + doReturn("null").when(mockBrokerStats).getTopics(); brokerStats.run(split("topics")); verify(mockBrokerStats).getTopics(); + doReturn(null).when(mockBrokerStats).getLoadReport(); brokerStats.run(split("load-report")); verify(mockBrokerStats).getLoadReport(); + doReturn("null").when(mockBrokerStats).getMBeans(); brokerStats.run(split("mbeans")); verify(mockBrokerStats).getMBeans(); + doReturn("null").when(mockBrokerStats).getMetrics(); brokerStats.run(split("monitoring-metrics")); verify(mockBrokerStats).getMetrics(); + + doReturn(null).when(mockBrokerStats).getAllocatorStats("default"); + brokerStats.run(split("allocator-stats default")); + verify(mockBrokerStats).getAllocatorStats("default"); } @Test @@ -356,6 +370,33 @@ public void tenants() throws Exception { } @Test + public void namespacesSetOffloadPolicies() throws Exception { + PulsarAdmin admin = Mockito.mock(PulsarAdmin.class); + Namespaces mockNamespaces = mock(Namespaces.class); + when(admin.namespaces()).thenReturn(mockNamespaces); + Lookup mockLookup = mock(Lookup.class); + when(admin.lookups()).thenReturn(mockLookup); + + // filesystem offload + CmdNamespaces namespaces = new CmdNamespaces(() -> admin); + namespaces.run(split( + "set-offload-policies myprop/clust/ns2 -d filesystem -oat 100M -oats 1h -oae 1h -orp bookkeeper-first")); + verify(mockNamespaces).setOffloadPolicies("myprop/clust/ns2", + OffloadPoliciesImpl.create("filesystem", null, null, + null, null, null, null, null, 64 * 1024 * 1024, 1024 * 1024, + 100 * 1024 * 1024L, 3600L, 3600 * 1000L, OffloadedReadPriority.BOOKKEEPER_FIRST)); + + // S3 offload + CmdNamespaces namespaces2 = new CmdNamespaces(() -> admin); + namespaces2.run(split( + "set-offload-policies myprop/clust/ns1 -r test-region -d aws-s3 -b test-bucket -e http://test.endpoint -mbs 32M -rbs 5M -oat 10M -oats 100 -oae 10s -orp tiered-storage-first")); + verify(mockNamespaces).setOffloadPolicies("myprop/clust/ns1", + OffloadPoliciesImpl.create("aws-s3", "test-region", "test-bucket", + "http://test.endpoint",null, null, null, null, 32 * 1024 * 1024, 5 * 1024 * 1024, + 10 * 1024 * 1024L, 100L, 10000L, OffloadedReadPriority.TIERED_STORAGE_FIRST)); + } + + @Test public void namespaces() throws Exception { PulsarAdmin admin = Mockito.mock(PulsarAdmin.class); Namespaces mockNamespaces = mock(Namespaces.class); @@ -400,7 +441,15 @@ public void namespaces() throws Exception { namespaces.run(split("get-clusters myprop/clust/ns1")); verify(mockNamespaces).getNamespaceReplicationClusters("myprop/clust/ns1"); - namespaces.run(split("set-subscription-types-enabled myprop/clust/ns1 -t Shared,Failover")); + namespaces.run(split("set-allowed-clusters myprop/clust/ns1 -c use,usw,usc")); + verify(mockNamespaces).setNamespaceAllowedClusters("myprop/clust/ns1", + Sets.newHashSet("use", "usw", "usc")); + + namespaces.run(split("get-allowed-clusters myprop/clust/ns1")); + verify(mockNamespaces).getNamespaceAllowedClusters("myprop/clust/ns1"); + + + namespaces.run(split("set-subscription-types-enabled myprop/clust/ns1 -t Shared,Failover")); verify(mockNamespaces).setSubscriptionTypesEnabled("myprop/clust/ns1", Sets.newHashSet(SubscriptionType.Shared, SubscriptionType.Failover)); @@ -440,6 +489,11 @@ public void namespaces() throws Exception { namespaces.run(split("remove-replicator-dispatch-rate myprop/clust/ns1")); verify(mockNamespaces).removeReplicatorDispatchRate("myprop/clust/ns1"); + + assertFalse(namespaces.run(split("unload myprop/clust/ns1 -d broker"))); + verify(mockNamespaces, times(0)).unload("myprop/clust/ns1"); + + namespaces = new CmdNamespaces(() -> admin); namespaces.run(split("unload myprop/clust/ns1")); verify(mockNamespaces).unload("myprop/clust/ns1"); @@ -456,6 +510,10 @@ public void namespaces() throws Exception { namespaces.run(split("unload myprop/clust/ns1 -b 0x80000000_0xffffffff")); verify(mockNamespaces).unloadNamespaceBundle("myprop/clust/ns1", "0x80000000_0xffffffff", null); + namespaces = new CmdNamespaces(() -> admin); + namespaces.run(split("unload myprop/clust/ns1 -b 0x80000000_0xffffffff -d broker")); + verify(mockNamespaces).unloadNamespaceBundle("myprop/clust/ns1", "0x80000000_0xffffffff", "broker"); + namespaces.run(split("split-bundle myprop/clust/ns1 -b 0x00000000_0xffffffff")); verify(mockNamespaces).splitNamespaceBundle("myprop/clust/ns1", "0x00000000_0xffffffff", false, null); @@ -643,9 +701,9 @@ public void namespaces() throws Exception { namespaces.run(split("remove-retention myprop/clust/ns1")); verify(mockNamespaces).removeRetention("myprop/clust/ns1"); - namespaces.run(split("set-delayed-delivery myprop/clust/ns1 -e -t 1s")); + namespaces.run(split("set-delayed-delivery myprop/clust/ns1 -e -t 1s -md 5s")); verify(mockNamespaces).setDelayedDeliveryMessages("myprop/clust/ns1", - DelayedDeliveryPolicies.builder().tickTime(1000).active(true).build()); + DelayedDeliveryPolicies.builder().tickTime(1000).active(true).maxDeliveryDelayInMillis(5000).build()); namespaces.run(split("get-delayed-delivery myprop/clust/ns1")); verify(mockNamespaces).getDelayedDelivery("myprop/clust/ns1"); @@ -846,6 +904,15 @@ public void namespaces() throws Exception { namespaces.run(split("remove-deduplication-snapshot-interval myprop/clust/ns1")); verify(mockNamespaces).removeDeduplicationSnapshotInterval("myprop/clust/ns1"); + namespaces.run(split("set-dispatcher-pause-on-ack-state-persistent myprop/clust/ns1")); + verify(mockNamespaces).setDispatcherPauseOnAckStatePersistent("myprop/clust/ns1"); + + namespaces.run(split("get-dispatcher-pause-on-ack-state-persistent myprop/clust/ns1")); + verify(mockNamespaces).getDispatcherPauseOnAckStatePersistent("myprop/clust/ns1"); + + namespaces.run(split("remove-dispatcher-pause-on-ack-state-persistent myprop/clust/ns1")); + verify(mockNamespaces).removeDispatcherPauseOnAckStatePersistent("myprop/clust/ns1"); + } @Test @@ -1164,9 +1231,9 @@ public void topicPolicies() throws Exception { cmdTopics.run(split("get-delayed-delivery persistent://myprop/clust/ns1/ds1")); verify(mockTopicsPolicies).getDelayedDeliveryPolicy("persistent://myprop/clust/ns1/ds1", false); - cmdTopics.run(split("set-delayed-delivery persistent://myprop/clust/ns1/ds1 -t 10s --enable")); + cmdTopics.run(split("set-delayed-delivery persistent://myprop/clust/ns1/ds1 -t 10s --enable --maxDelay 5s")); verify(mockTopicsPolicies).setDelayedDeliveryPolicy("persistent://myprop/clust/ns1/ds1", - DelayedDeliveryPolicies.builder().tickTime(10000).active(true).build()); + DelayedDeliveryPolicies.builder().tickTime(10000).active(true).maxDeliveryDelayInMillis(5000).build()); cmdTopics.run(split("remove-delayed-delivery persistent://myprop/clust/ns1/ds1")); verify(mockTopicsPolicies).removeDelayedDeliveryPolicy("persistent://myprop/clust/ns1/ds1") ; @@ -1331,9 +1398,9 @@ public void topicPolicies() throws Exception { cmdTopics.run(split("get-delayed-delivery persistent://myprop/clust/ns1/ds1 -g")); verify(mockGlobalTopicsPolicies).getDelayedDeliveryPolicy("persistent://myprop/clust/ns1/ds1", false); - cmdTopics.run(split("set-delayed-delivery persistent://myprop/clust/ns1/ds1 -t 10s --enable -g")); + cmdTopics.run(split("set-delayed-delivery persistent://myprop/clust/ns1/ds1 -t 10s --enable -md 5s -g")); verify(mockGlobalTopicsPolicies).setDelayedDeliveryPolicy("persistent://myprop/clust/ns1/ds1", - DelayedDeliveryPolicies.builder().tickTime(10000).active(true).build()); + DelayedDeliveryPolicies.builder().tickTime(10000).active(true).maxDeliveryDelayInMillis(5000).build()); cmdTopics.run(split("remove-delayed-delivery persistent://myprop/clust/ns1/ds1 -g")); verify(mockGlobalTopicsPolicies).removeDelayedDeliveryPolicy("persistent://myprop/clust/ns1/ds1") ; @@ -1455,6 +1522,34 @@ public void topicPolicies() throws Exception { verify(mockGlobalTopicsPolicies).removeAutoSubscriptionCreation("persistent://prop/clust/ns1/ds1"); } + @Test + public void topicsSetOffloadPolicies() throws Exception { + PulsarAdmin admin = Mockito.mock(PulsarAdmin.class); + Topics mockTopics = mock(Topics.class); + when(admin.topics()).thenReturn(mockTopics); + Schemas mockSchemas = mock(Schemas.class); + when(admin.schemas()).thenReturn(mockSchemas); + Lookup mockLookup = mock(Lookup.class); + when(admin.lookups()).thenReturn(mockLookup); + + // filesystem offload + CmdTopics cmdTopics = new CmdTopics(() -> admin); + cmdTopics.run(split("set-offload-policies persistent://myprop/clust/ns1/ds1 -d filesystem -oat 100M -oats 1h -oae 1h -orp bookkeeper-first")); + OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.create("filesystem", null, null + , null, null, null, null, null, 64 * 1024 * 1024, 1024 * 1024, + 100 * 1024 * 1024L, 3600L, 3600 * 1000L, OffloadedReadPriority.BOOKKEEPER_FIRST); + verify(mockTopics).setOffloadPolicies("persistent://myprop/clust/ns1/ds1", offloadPolicies); + +// S3 offload + CmdTopics cmdTopics2 = new CmdTopics(() -> admin); + cmdTopics2.run(split("set-offload-policies persistent://myprop/clust/ns1/ds2 -d s3 -r region -b bucket -e endpoint -ts 50 -m 8 -rb 9 -t 10 -orp tiered-storage-first")); + OffloadPoliciesImpl offloadPolicies2 = OffloadPoliciesImpl.create("s3", "region", "bucket" + , "endpoint", null, null, null, null, + 8, 9, 10L, 50L, null, OffloadedReadPriority.TIERED_STORAGE_FIRST); + verify(mockTopics).setOffloadPolicies("persistent://myprop/clust/ns1/ds2", offloadPolicies2); + } + + @Test public void topics() throws Exception { PulsarAdmin admin = Mockito.mock(PulsarAdmin.class); @@ -1495,7 +1590,7 @@ public void topics() throws Exception { verify(mockLookup).lookupPartitionedTopic("persistent://myprop/clust/ns1/ds1"); cmdTopics.run(split("partitioned-lookup persistent://myprop/clust/ns1/ds1 --sort-by-broker")); - verify(mockLookup).lookupPartitionedTopic("persistent://myprop/clust/ns1/ds1"); + verify(mockLookup, times(2)).lookupPartitionedTopic("persistent://myprop/clust/ns1/ds1"); cmdTopics.run(split("bundle-range persistent://myprop/clust/ns1/ds1")); verify(mockLookup).getBundleRange("persistent://myprop/clust/ns1/ds1"); @@ -1662,7 +1757,8 @@ public void topics() throws Exception { verify(mockTopics).deletePartitionedTopic("persistent://myprop/clust/ns1/ds1", true); cmdTopics.run(split("peek-messages persistent://myprop/clust/ns1/ds1 -s sub1 -n 3")); - verify(mockTopics).peekMessages("persistent://myprop/clust/ns1/ds1", "sub1", 3); + verify(mockTopics).peekMessages("persistent://myprop/clust/ns1/ds1", "sub1", 3, + false, TransactionIsolationLevel.READ_COMMITTED); MessageImpl message = mock(MessageImpl.class); when(message.getData()).thenReturn(new byte[]{}); @@ -1732,9 +1828,9 @@ public void topics() throws Exception { cmdTopics.run(split("get-delayed-delivery persistent://myprop/clust/ns1/ds1")); verify(mockTopics).getDelayedDeliveryPolicy("persistent://myprop/clust/ns1/ds1", false); - cmdTopics.run(split("set-delayed-delivery persistent://myprop/clust/ns1/ds1 -t 10s --enable")); + cmdTopics.run(split("set-delayed-delivery persistent://myprop/clust/ns1/ds1 -t 10s -md 5s --enable")); verify(mockTopics).setDelayedDeliveryPolicy("persistent://myprop/clust/ns1/ds1", - DelayedDeliveryPolicies.builder().tickTime(10000).active(true).build()); + DelayedDeliveryPolicies.builder().tickTime(10000).active(true).maxDeliveryDelayInMillis(5000).build()); cmdTopics.run(split("remove-delayed-delivery persistent://myprop/clust/ns1/ds1")); verify(mockTopics).removeDelayedDeliveryPolicy("persistent://myprop/clust/ns1/ds1") ; @@ -2199,11 +2295,30 @@ public void requestTimeout() throws Exception { //Ok } - ClientConfigurationData conf = ((PulsarAdminImpl)tool.getPulsarAdminSupplier().get()).getClientConfigData(); + @Cleanup + PulsarAdminImpl pulsarAdmin = (PulsarAdminImpl) tool.getPulsarAdminSupplier().get(); + ClientConfigurationData conf = pulsarAdmin.getClientConfigData(); assertEquals(1000, conf.getRequestTimeoutMs()); } + @Test + public void testSourceCreateMissingSourceConfigFileFaileWithExitCode1() throws Exception { + Properties properties = new Properties(); + properties.put("webServiceUrl", "http://localhost:2181"); + PulsarAdminTool tool = new PulsarAdminTool(properties); + assertFalse(tool.run("sources create --source-config-file doesnotexist.yaml".split(" "))); + } + + @Test + public void testSourceUpdateMissingSourceConfigFileFaileWithExitCode1() throws Exception { + Properties properties = new Properties(); + properties.put("webServiceUrl", "http://localhost:2181"); + PulsarAdminTool tool = new PulsarAdminTool(properties); + + assertFalse(tool.run("sources update --source-config-file doesnotexist.yaml".split(" "))); + } + @Test public void testAuthTlsWithJsonParam() throws Exception { @@ -2225,8 +2340,9 @@ public void testAuthTlsWithJsonParam() throws Exception { } // validate Authentication-tls has been configured - ClientConfigurationData conf = ((PulsarAdminImpl)tool.getPulsarAdminSupplier().get()) - .getClientConfigData(); + @Cleanup + PulsarAdminImpl pulsarAdmin = (PulsarAdminImpl) tool.getPulsarAdminSupplier().get(); + ClientConfigurationData conf = pulsarAdmin.getClientConfigData(); AuthenticationTls atuh = (AuthenticationTls) conf.getAuthentication(); assertEquals(atuh.getCertFilePath(), certFilePath); assertEquals(atuh.getKeyFilePath(), keyFilePath); @@ -2239,8 +2355,9 @@ public void testAuthTlsWithJsonParam() throws Exception { // Ok } - conf = conf = ((PulsarAdminImpl)tool.getPulsarAdminSupplier().get()) - .getClientConfigData(); + @Cleanup + PulsarAdminImpl pulsarAdmin2 = (PulsarAdminImpl) tool.getPulsarAdminSupplier().get(); + conf = pulsarAdmin2.getClientConfigData(); atuh = (AuthenticationTls) conf.getAuthentication(); assertEquals(atuh.getCertFilePath(), certFilePath); assertEquals(atuh.getKeyFilePath(), keyFilePath); @@ -2304,7 +2421,7 @@ void transactions() throws Exception { cmdTransactions = new CmdTransactions(() -> admin); cmdTransactions.run(split("transaction-buffer-stats -t test -l")); - verify(transactions).getTransactionBufferStats("test", true); + verify(transactions).getTransactionBufferStats("test", true, false); cmdTransactions = new CmdTransactions(() -> admin); cmdTransactions.run(split("pending-ack-stats -t test -s test -l")); @@ -2314,6 +2431,10 @@ void transactions() throws Exception { cmdTransactions.run(split("pending-ack-internal-stats -t test -s test")); verify(transactions).getPendingAckInternalStats("test", "test", false); + cmdTransactions = new CmdTransactions(() -> admin); + cmdTransactions.run(split("buffer-snapshot-internal-stats -t test")); + verify(transactions).getTransactionBufferInternalStats("test", false); + cmdTransactions = new CmdTransactions(() -> admin); cmdTransactions.run(split("scale-transactionCoordinators -r 3")); verify(transactions).scaleTransactionCoordinators(3); @@ -2325,6 +2446,10 @@ void transactions() throws Exception { cmdTransactions = new CmdTransactions(() -> admin); cmdTransactions.run(split("coordinators-list")); verify(transactions).listTransactionCoordinators(); + + cmdTransactions = new CmdTransactions(() -> admin); + cmdTransactions.run(split("abort-transaction -m 1 -l 2")); + verify(transactions).abortTransaction(new TxnID(1, 2)); } @Test @@ -2393,21 +2518,20 @@ public void customCommands() throws Exception { assertTrue(logs.contains("customgroup")); assertTrue(logs.contains("Custom group 1 description")); + // missing subcommand logs = runCustomCommand(new String[]{"customgroup"}); - assertTrue(logs.contains("command1")); + assertTrue(logs.contains("Missing required subcommand")); assertTrue(logs.contains("Command 1 description")); - assertTrue(logs.contains("command2")); assertTrue(logs.contains("Command 2 description")); + // missing required parameter logs = runCustomCommand(new String[]{"customgroup", "command1"}); + assertTrue(logs.contains("Missing required options and parameters")); assertTrue(logs.contains("Command 1 description")); - assertTrue(logs.contains("Usage: command1 [options] Topic")); - // missing required parameter logs = runCustomCommand(new String[]{"customgroup", "command1", "mytopic"}); assertTrue(logs.contains("Command 1 description")); - assertTrue(logs.contains("Usage: command1 [options] Topic")); - assertTrue(logs.contains("The following option is required")); + assertTrue(logs.contains("Missing required option")); // run a comand that uses PulsarAdmin API logs = runCustomCommand(new String[]{"customgroup", "command1", "--type", "stats", "mytopic"}); @@ -2475,39 +2599,26 @@ public void customCommandsFactoryImmutable() throws Exception { } @Test - public void testHelpFlag() { - PulsarAdmin admin = Mockito.mock(PulsarAdmin.class); + public void testHelpFlag() throws Exception { + Properties properties = new Properties(); + properties.put("webServiceUrl", "http://localhost:8080"); + + PulsarAdminTool pulsarAdminTool = new PulsarAdminTool(properties); { - CmdSchemas cmdSchemas = new CmdSchemas(() -> admin); - cmdSchemas.run(split("-h")); - assertTrue(cmdSchemas.isHelp()); + assertTrue(pulsarAdminTool.run(split("schemas -h"))); } { - CmdSchemas cmdSchemas = new CmdSchemas(() -> admin); - cmdSchemas.run(split("--help")); - assertTrue(cmdSchemas.isHelp()); + assertTrue(pulsarAdminTool.run(split("schemas --help"))); } { - CmdSchemas cmdSchemas = new CmdSchemas(() -> admin); - cmdSchemas.run(split("delete --help")); - assertFalse(cmdSchemas.isHelp()); - JCommander commander = cmdSchemas.getJcommander(); - JCommander subCommander = commander.getCommands().get("delete"); - CliCommand subcommand = (CliCommand) subCommander.getObjects().get(0); - assertTrue(subcommand.isHelp()); + assertTrue(pulsarAdminTool.run(split("schemas delete -h"))); } { - CmdSchemas cmdSchemas = new CmdSchemas(() -> admin); - cmdSchemas.run(split("delete -h")); - assertFalse(cmdSchemas.isHelp()); - JCommander commander = cmdSchemas.getJcommander(); - JCommander subCommander = commander.getCommands().get("delete"); - CliCommand subcommand = (CliCommand) subCommander.getObjects().get(0); - assertTrue(subcommand.isHelp()); + assertTrue(pulsarAdminTool.run(split("schemas delete --help"))); } } @@ -2530,11 +2641,9 @@ private static String runCustomCommand(String[] args) throws Exception { properties.put("cliExtensionsDirectory", narFile.getParentFile().getAbsolutePath()); properties.put("customCommandFactories", "dummy"); PulsarAdminTool tool = new PulsarAdminTool(properties); - tool.setPulsarAdminSupplier(new PulsarAdminSupplier(builder, tool.getRootParams())); - - // see the custom command help in the main help + tool.getPulsarAdminSupplier().setAdminBuilder(builder); StringBuilder logs = new StringBuilder(); - try (CaptureStdOut capture = new CaptureStdOut(logs)){ + try (CaptureStdOut capture = new CaptureStdOut(tool.commander, logs)) { tool.run(args); } log.info("Captured out: {}", logs); @@ -2544,13 +2653,18 @@ private static String runCustomCommand(String[] args) throws Exception { private static class CaptureStdOut implements AutoCloseable { final PrintStream currentOut = System.out; final PrintStream currentErr = System.err; - final ByteArrayOutputStream logs = new ByteArrayOutputStream(); - final PrintStream capturedOut = new PrintStream(logs, true); + final ByteArrayOutputStream logs; + final PrintStream capturedOut; final StringBuilder receiver; - public CaptureStdOut(StringBuilder receiver) { + public CaptureStdOut(CommandLine commandLine, StringBuilder receiver) { + logs = new ByteArrayOutputStream(); + capturedOut = new PrintStream(logs, true); this.receiver = receiver; - System.setOut(capturedOut); - System.setErr(capturedOut); + PrintWriter printWriter = new PrintWriter(logs); + commandLine.setErr(printWriter); + commandLine.setOut(printWriter); + System.setOut(new PrintStream(logs)); + System.setErr(new PrintStream(logs)); } public void close() { capturedOut.flush(); diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/utils/NameValueParameterSplitterTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/utils/NameValueParameterSplitterTest.java deleted file mode 100644 index 1bf4f3fedeca4..0000000000000 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/utils/NameValueParameterSplitterTest.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.admin.cli.utils; - -import java.util.Map; - -import org.testng.Assert; -import org.testng.annotations.Test; - -public class NameValueParameterSplitterTest { - @Test(description = "Basic Test") - public void test1() { - NameValueParameterSplitter splitter = new NameValueParameterSplitter(); - Map result = splitter.convert("Name=Sunnyvale"); - Assert.assertEquals(result.get("Name"), "Sunnyvale"); - } - - @Test(description = "Check trimming of values") - public void test2() { - NameValueParameterSplitter splitter = new NameValueParameterSplitter(); - Map result = splitter.convert(" Name = Sunnyvale CA"); - Assert.assertEquals(result.get("Name"), "Sunnyvale CA"); - } - - @Test(description = "Check error on invalid input") - public void test3() { - try { - NameValueParameterSplitter splitter = new NameValueParameterSplitter(); - splitter.convert(" Name Sunnyvale CA"); - // Expecting exception - Assert.fail("' Name Sunnyvale CA' is not a valid name value pair"); - } catch (Exception e) { - // TODO: handle exception - } - } -} diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/DocumentTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/DocumentTest.java index c565fadd2fc9a..84d423c6072b6 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/DocumentTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/DocumentTest.java @@ -20,16 +20,13 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; - -import com.beust.jcommander.JCommander; +import java.util.Map; +import java.util.Properties; import org.apache.pulsar.broker.service.BrokerTestBase; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; - -import java.util.Map; -import java.util.Properties; - +import picocli.CommandLine; public class DocumentTest extends BrokerTestBase { @@ -59,11 +56,12 @@ public void testSpecifyModuleName() { @Test public void testGenerator() { PulsarClientTool pulsarClientTool = new PulsarClientTool(new Properties()); - JCommander commander = pulsarClientTool.jcommander; + CommandLine commander = pulsarClientTool.getCommander(); CmdGenerateDocumentation document = new CmdGenerateDocumentation(); - for (Map.Entry cmd : commander.getCommands().entrySet()) { - String res = document.generateDocument(cmd.getKey(), commander); - assertTrue(res.contains("pulsar-client " + cmd.getKey() + " [options]")); - } + Map subcommands = commander.getSubcommands(); + subcommands.forEach((subcommandName, subCommander) -> { + String res = document.generateDocument(subcommandName, subCommander); + assertTrue(res.contains("pulsar-client " + subcommandName + " [options]")); + }); } } diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolForceBatchNum.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolForceBatchNum.java index e296ab0e0357a..896bee0e030af 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolForceBatchNum.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolForceBatchNum.java @@ -53,11 +53,6 @@ public PulsarClientToolForceBatchNum(Properties properties, String topic, int ba super(properties); this.topic = topic; this.batchNum = batchNum; - } - - @Override - protected void initJCommander() { - super.initJCommander(); produceCommand = new CmdProduce() { @Override public void updateConfig(ClientBuilder newBuilder, Authentication authentication, String serviceURL) { @@ -68,7 +63,7 @@ public void updateConfig(ClientBuilder newBuilder, Authentication authentication } } }; - jcommander.addCommand("produce", produceCommand); + replaceProducerCommand(produceCommand); } private ClientBuilder mockClientBuilder(ClientBuilder newBuilder) throws Exception { diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java index c401f3d0bea64..9edee30d8fee8 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java @@ -20,8 +20,11 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; +import java.io.File; +import java.nio.file.Files; import java.time.Duration; import java.util.Properties; import java.util.UUID; @@ -30,18 +33,24 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import lombok.AllArgsConstructor; import lombok.Cleanup; +import lombok.NoArgsConstructor; import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.ProxyProtocol; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.BatchMessageIdImpl; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.schema.KeyValue; +import org.apache.pulsar.common.util.ObjectMapperFactory; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; public class PulsarClientToolTest extends BrokerTestBase { @@ -64,6 +73,7 @@ public void testInitialization() throws InterruptedException, ExecutionException Properties properties = new Properties(); properties.setProperty("serviceUrl", brokerUrl.toString()); properties.setProperty("useTls", "false"); + properties.setProperty("memoryLimit", "10M"); String tenantName = UUID.randomUUID().toString(); @@ -85,6 +95,7 @@ public void testInitialization() throws InterruptedException, ExecutionException String[] args = { "consume", "-t", "Exclusive", "-s", "sub-name", "-n", Integer.toString(numberOfMessages), "--hex", "-r", "30", topicName }; Assert.assertEquals(pulsarClientToolConsumer.run(args), 0); + Assert.assertEquals(pulsarClientToolConsumer.rootParams.memoryLimit, 10 * 1024 * 1024); future.complete(null); } catch (Throwable t) { future.completeExceptionally(t); @@ -99,6 +110,7 @@ public void testInitialization() throws InterruptedException, ExecutionException String[] args = { "produce", "--messages", "Have a nice day", "-n", Integer.toString(numberOfMessages), "-r", "20", "-p", "key1=value1", "-p", "key2=value2", "-k", "partition_key", topicName }; Assert.assertEquals(pulsarClientToolProducer.run(args), 0); + Assert.assertEquals(pulsarClientToolProducer.rootParams.memoryLimit, 10 * 1024 * 1024); future.get(); } @@ -111,6 +123,7 @@ public void testNonDurableSubscribe() throws Exception { properties.setProperty("useTls", "false"); final String topicName = getTopicWithRandomSuffix("non-durable"); + admin.topics().createNonPartitionedTopic(topicName); int numberOfMessages = 10; @Cleanup("shutdownNow") @@ -202,6 +215,7 @@ public void testRead() throws Exception { properties.setProperty("useTls", "false"); final String topicName = getTopicWithRandomSuffix("reader"); + admin.topics().createNonPartitionedTopic(topicName); int numberOfMessages = 10; @Cleanup("shutdownNow") @@ -251,6 +265,7 @@ public void testEncryption() throws Exception { properties.setProperty("useTls", "false"); final String topicName = getTopicWithRandomSuffix("encryption"); + admin.topics().createNonPartitionedTopic(topicName); final String keyUriBase = "file:../pulsar-broker/src/test/resources/certificate/"; final int numberOfMessages = 10; @@ -333,22 +348,48 @@ public void testArgs() throws Exception { final String message = "test msg"; final int numberOfMessages = 1; final String topicName = getTopicWithRandomSuffix("test-topic"); + final String memoryLimitArg = "10M"; String[] args = {"--url", url, "--auth-plugin", authPlugin, "--auth-params", authParams, "--tlsTrustCertsFilePath", CA_CERT_FILE_PATH, + "--memory-limit", memoryLimitArg, "produce", "-m", message, "-n", Integer.toString(numberOfMessages), topicName}; - pulsarClientTool.jcommander.parse(args); + pulsarClientTool.getCommander().parseArgs(args); assertEquals(pulsarClientTool.rootParams.getTlsTrustCertsFilePath(), CA_CERT_FILE_PATH); assertEquals(pulsarClientTool.rootParams.getAuthParams(), authParams); assertEquals(pulsarClientTool.rootParams.getAuthPluginClassName(), authPlugin); + assertEquals(pulsarClientTool.rootParams.getMemoryLimit(), 10 * 1024 * 1024); assertEquals(pulsarClientTool.rootParams.getServiceURL(), url); assertNull(pulsarClientTool.rootParams.getProxyServiceURL()); assertNull(pulsarClientTool.rootParams.getProxyProtocol()); } + @Test(timeOut = 20000) + public void testMemoryLimitArgShortName() throws Exception { + PulsarClientTool pulsarClientTool = new PulsarClientTool(new Properties()); + final String url = "pulsar+ssl://localhost:6651"; + final String authPlugin = "org.apache.pulsar.client.impl.auth.AuthenticationTls"; + final String authParams = String.format("tlsCertFile:%s,tlsKeyFile:%s", getTlsFileForClient("admin.cert"), + getTlsFileForClient("admin.key-pk8")); + final String message = "test msg"; + final int numberOfMessages = 1; + final String topicName = getTopicWithRandomSuffix("test-topic"); + final String memoryLimitArg = "10M"; + + String[] args = {"--url", url, + "--auth-plugin", authPlugin, + "--auth-params", authParams, + "--tlsTrustCertsFilePath", CA_CERT_FILE_PATH, + "-ml", memoryLimitArg, + "produce", "-m", message, + "-n", Integer.toString(numberOfMessages), topicName}; + pulsarClientTool.getCommander().parseArgs(args); + assertEquals(pulsarClientTool.rootParams.getMemoryLimit(), 10 * 1024 * 1024); + } + @Test public void testParsingProxyServiceUrlAndProxyProtocolFromProperties() throws Exception { Properties properties = new Properties(); @@ -363,7 +404,7 @@ public void testParsingProxyServiceUrlAndProxyProtocolFromProperties() throws Ex String[] args = {"--url", url, "produce", "-m", message, "-n", Integer.toString(numberOfMessages), topicName}; - pulsarClientTool.jcommander.parse(args); + pulsarClientTool.getCommander().parseArgs(args); assertEquals(pulsarClientTool.rootParams.getServiceURL(), url); assertEquals(pulsarClientTool.rootParams.getProxyServiceURL(), "pulsar+ssl://my-proxy-pulsar:4443"); assertEquals(pulsarClientTool.rootParams.getProxyProtocol(), ProxyProtocol.SNI); @@ -395,4 +436,152 @@ private static String getTopicWithRandomSuffix(String localNameBase) { return String.format("persistent://prop/ns-abc/test/%s-%s", localNameBase, UUID.randomUUID().toString()); } + + @Test(timeOut = 20000) + public void testProducePartitioningKey() throws Exception { + + Properties properties = initializeToolProperties(); + + final String topicName = getTopicWithRandomSuffix("key-topic"); + + @Cleanup + Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("sub").subscribe(); + + @Cleanup("shutdownNow") + ExecutorService executor = Executors.newSingleThreadExecutor(); + CompletableFuture future = new CompletableFuture<>(); + executor.execute(() -> { + try { + PulsarClientTool pulsarClientToolConsumer = new PulsarClientTool(properties); + String[] args = {"produce", "-m", "test", "-k", "partition-key1", topicName}; + Assert.assertEquals(pulsarClientToolConsumer.run(args), 0); + future.complete(null); + } catch (Throwable t) { + future.completeExceptionally(t); + } + }); + final Message message = consumer.receive(10, TimeUnit.SECONDS); + assertNotNull(message); + assertTrue(message.hasKey()); + Assert.assertEquals(message.getKey(), "partition-key1"); + } + + @NoArgsConstructor + @AllArgsConstructor + public static class TestKey { + public String key_a; + public int key_b; + + } + + @Test + public void testProduceKeyValueSchemaInlineValue() throws Exception { + + Properties properties = initializeToolProperties(); + + final String topicName = getTopicWithRandomSuffix("key-topic"); + + + @Cleanup + Consumer> consumer = pulsarClient.newConsumer(Schema.KeyValue(Schema.JSON( + TestKey.class), Schema.STRING)).topic(topicName).subscriptionName("sub").subscribe(); + + @Cleanup("shutdownNow") + ExecutorService executor = Executors.newSingleThreadExecutor(); + CompletableFuture future = new CompletableFuture<>(); + final Schema keySchema = Schema.JSON(TestKey.class); + + executor.execute(() -> { + try { + PulsarClientTool pulsarClientToolConsumer = new PulsarClientTool(properties); + String[] args = {"produce", + "-kvet", "inline", + "-ks", String.format("json:%s", keySchema.getSchemaInfo().getSchemaDefinition()), + "-kvk", ObjectMapperFactory.getMapper().writer().writeValueAsString(new TestKey("my-key", Integer.MAX_VALUE)), + "-vs", "string", + "-m", "test", + topicName}; + Assert.assertEquals(pulsarClientToolConsumer.run(args), 0); + future.complete(null); + } catch (Throwable t) { + future.completeExceptionally(t); + } + }); + final Message> message = consumer.receive(10, TimeUnit.SECONDS); + assertNotNull(message); + assertFalse(message.hasKey()); + Assert.assertEquals(message.getValue().getKey().key_a, "my-key"); + Assert.assertEquals(message.getValue().getKey().key_b, Integer.MAX_VALUE); + Assert.assertEquals(message.getValue().getValue(), "test"); + } + + @DataProvider(name = "keyValueKeySchema") + public static Object[][] keyValueKeySchema() { + return new Object[][]{ + {"json"}, + {"avro"} + }; + } + + @Test(dataProvider = "keyValueKeySchema") + public void testProduceKeyValueSchemaFileValue(String schema) throws Exception { + + Properties properties = initializeToolProperties(); + + final String topicName = getTopicWithRandomSuffix("key-topic"); + + + + @Cleanup("shutdownNow") + ExecutorService executor = Executors.newSingleThreadExecutor(); + CompletableFuture future = new CompletableFuture<>(); + File file = Files.createTempFile("", "").toFile(); + final Schema keySchema; + if (schema.equals("json")) { + keySchema = Schema.JSON(TestKey.class); + } else if (schema.equals("avro")) { + keySchema = Schema.AVRO(TestKey.class); + } else { + throw new IllegalStateException(); + } + + + Files.write(file.toPath(), keySchema.encode(new TestKey("my-key", Integer.MAX_VALUE))); + + @Cleanup + Consumer> consumer = pulsarClient.newConsumer(Schema.KeyValue(keySchema, Schema.STRING)) + .topic(topicName).subscriptionName("sub").subscribe(); + + executor.execute(() -> { + try { + PulsarClientTool pulsarClientToolConsumer = new PulsarClientTool(properties); + String[] args = {"produce", + "-k", "partitioning-key", + "-kvet", "inline", + "-ks", String.format("%s:%s", schema, keySchema.getSchemaInfo().getSchemaDefinition()), + "-kvkf", file.getAbsolutePath(), + "-vs", "string", + "-m", "test", + topicName}; + Assert.assertEquals(pulsarClientToolConsumer.run(args), 0); + future.complete(null); + } catch (Throwable t) { + future.completeExceptionally(t); + } + }); + final Message> message = consumer.receive(10, TimeUnit.SECONDS); + assertNotNull(message); + // -k should not be considered + assertFalse(message.hasKey()); + Assert.assertEquals(message.getValue().getKey().key_a, "my-key"); + Assert.assertEquals(message.getValue().getKey().key_b, Integer.MAX_VALUE); + } + + private Properties initializeToolProperties() { + Properties properties = new Properties(); + properties.setProperty("serviceUrl", brokerUrl.toString()); + properties.setProperty("useTls", "false"); + return properties; + } + } diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index 025189b7bbfa5..41b17901262ec 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -24,8 +24,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-client-tools @@ -34,8 +33,13 @@ - com.beust - jcommander + info.picocli + picocli + compile + + + info.picocli + picocli-shell-jline3 compile @@ -53,6 +57,11 @@ pulsar-client-admin-original ${project.version} + + ${project.groupId} + pulsar-cli-utils + ${project.version} + commons-io commons-io @@ -67,6 +76,11 @@ pulsar-client-messagecrypto-bc ${project.version} + + ${project.groupId} + pulsar-cli-utils + ${project.version} + org.asynchttpclient async-http-client @@ -107,6 +121,7 @@ io.swagger swagger-core + provided diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CliCommand.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CliCommand.java index 7a6836eb74762..8a8019cbe8ccc 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CliCommand.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CliCommand.java @@ -18,16 +18,14 @@ */ package org.apache.pulsar.admin.cli; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; -import com.google.common.collect.Sets; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.Callable; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.impl.MessageIdImpl; @@ -36,89 +34,46 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.AuthAction; import org.apache.pulsar.common.util.ObjectMapperFactory; - -public abstract class CliCommand { - - @Parameter(names = { "--help", "-h" }, help = true, hidden = true) - private boolean help = false; - - public boolean isHelp() { - return help; - } - - static String[] validatePropertyCluster(List params) { - return splitParameter(params, 2); +import picocli.CommandLine; +import picocli.CommandLine.Model.CommandSpec; +import picocli.CommandLine.Spec; + +public abstract class CliCommand implements Callable { + @Spec + private CommandSpec commandSpec; + + static String[] validatePropertyCluster(String params) { + String[] parts = params.split("/"); + if (parts.length != 2) { + throw new IllegalArgumentException("Parameter format is incorrect"); + } + return parts; } - static String validateNamespace(List params) { - String namespace = checkArgument(params); + static String validateNamespace(String namespace) { return NamespaceName.get(namespace).toString(); } - static String validateTopicName(List params) { - String topic = checkArgument(params); + static String validateTopicName(String topic) { return TopicName.get(topic).toString(); } - static String validatePersistentTopic(List params) { - String topic = checkArgument(params); + static String validatePersistentTopic(String topic) { TopicName topicName = TopicName.get(topic); if (topicName.getDomain() != TopicDomain.persistent) { - throw new ParameterException("Need to provide a persistent topic name"); + throw new IllegalArgumentException("Need to provide a persistent topic name"); } return topicName.toString(); } - static String validateNonPersistentTopic(List params) { - String topic = checkArgument(params); + static String validateNonPersistentTopic(String topic) { TopicName topicName = TopicName.get(topic); if (topicName.getDomain() != TopicDomain.non_persistent) { - throw new ParameterException("Need to provide a non-persistent topic name"); + throw new IllegalArgumentException("Need to provide a non-persistent topic name"); } return topicName.toString(); } - static void validateLatencySampleRate(int sampleRate) { - if (sampleRate < 0) { - throw new ParameterException( - "Latency sample rate should be positive and non-zero (found " + sampleRate + ")"); - } - } - - static long validateSizeString(String s) { - char last = s.charAt(s.length() - 1); - String subStr = s.substring(0, s.length() - 1); - long size; - try { - size = sizeUnit.contains(last) - ? Long.parseLong(subStr) - : Long.parseLong(s); - } catch (IllegalArgumentException e) { - throw new ParameterException(String.format("Invalid size '%s'. Valid formats are: %s", - s, "(4096, 100K, 10M, 16G, 2T)")); - } - switch (last) { - case 'k': - case 'K': - return size * 1024; - - case 'm': - case 'M': - return size * 1024 * 1024; - - case 'g': - case 'G': - return size * 1024 * 1024 * 1024; - - case 't': - case 'T': - return size * 1024 * 1024 * 1024 * 1024; - - default: - return size; - } - } - static MessageId validateMessageIdString(String resetMessageIdStr) throws PulsarAdminException { return validateMessageIdString(resetMessageIdStr, -1); } @@ -134,54 +89,7 @@ static MessageId validateMessageIdString(String resetMessageIdStr, int partition } } - static String checkArgument(List arguments) { - if (arguments.size() != 1) { - throw new ParameterException("Need to provide just 1 parameter"); - } - - return arguments.get(0); - } - - private static String[] splitParameter(List params, int n) { - if (params.size() != 1) { - throw new ParameterException("Need to provide just 1 parameter"); - } - - String[] parts = params.get(0).split("/"); - if (parts.length != n) { - throw new ParameterException("Parameter format is incorrect"); - } - - return parts; - } - - static String getOneArgument(List params) { - if (params.size() != 1) { - throw new ParameterException("Need to provide just 1 parameter"); - } - - return params.get(0); - } - - /** - * - * @param params - * List of positional arguments - * @param pos - * Positional arguments start with index as 1 - * @param maxArguments - * Validate against max arguments - * @return - */ - static String getOneArgument(List params, int pos, int maxArguments) { - if (params.size() != maxArguments) { - throw new ParameterException(String.format("Need to provide %s parameters", maxArguments)); - } - - return params.get(pos); - } - - static Set getAuthActions(List actions) { + Set getAuthActions(List actions) { Set res = new TreeSet<>(); AuthAction authAction; for (String action : actions) { @@ -212,18 +120,42 @@ void print(Map items) { void print(T item) { try { if (item instanceof String) { - System.out.println(item); + commandSpec.commandLine().getOut().println(item); } else { - System.out.println(writer.writeValueAsString(item)); + prettyPrint(item); } } catch (Exception e) { throw new RuntimeException(e); } } - private static ObjectMapper mapper = ObjectMapperFactory.create(); - private static ObjectWriter writer = mapper.writerWithDefaultPrettyPrinter(); - private static Set sizeUnit = Sets.newHashSet('k', 'K', 'm', 'M', 'g', 'G', 't', 'T'); + void prettyPrint(T item) { + try { + commandSpec.commandLine().getOut().println(WRITER.writeValueAsString(item)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static final ObjectMapper MAPPER = ObjectMapperFactory.create(); + private static final ObjectWriter WRITER = MAPPER.writerWithDefaultPrettyPrinter(); + + // Picocli entrypoint. + @Override + public Integer call() throws Exception { + run(); + return 0; + } abstract void run() throws Exception; + + protected class ParameterException extends CommandLine.ParameterException { + public ParameterException(String msg) { + super(commandSpec.commandLine(), msg); + } + + public ParameterException(String msg, Throwable e) { + super(commandSpec.commandLine(), msg, e); + } + } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBase.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBase.java index ce2c44ec1e827..8ff7f1c31ce2a 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBase.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBase.java @@ -18,114 +18,68 @@ */ package org.apache.pulsar.admin.cli; -import com.beust.jcommander.DefaultUsageFormatter; -import com.beust.jcommander.IUsageFormatter; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; +import static org.apache.pulsar.client.admin.internal.BaseResource.getApiException; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Supplier; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.admin.PulsarAdminException.ConnectException; +import org.apache.pulsar.client.admin.internal.PulsarAdminImpl; +import picocli.CommandLine; public abstract class CmdBase { - protected final JCommander jcommander; + private final CommandLine commander; private final Supplier adminSupplier; - private PulsarAdmin admin; - private IUsageFormatter usageFormatter; - @Parameter(names = { "--help", "-h" }, help = true, hidden = true) - private boolean help = false; - - public boolean isHelp() { - return help; - } + /** + * Default request timeout in milliseconds. + * Used if not found from configuration data in {@link #getRequestTimeoutMs()} + */ + private static final long DEFAULT_REQUEST_TIMEOUT_MILLIS = 60000; public CmdBase(String cmdName, Supplier adminSupplier) { this.adminSupplier = adminSupplier; - jcommander = new JCommander(this); - usageFormatter = new CmdUsageFormatter(jcommander); - jcommander.setProgramName("pulsar-admin " + cmdName); - jcommander.setUsageFormatter(usageFormatter); + commander = new CommandLine(this); + commander.setCommandName(cmdName); } - protected IUsageFormatter getUsageFormatter() { - if (usageFormatter == null) { - usageFormatter = new DefaultUsageFormatter(jcommander); - } - return usageFormatter; + public boolean run(String[] args) { + return commander.execute(args) == 0; } - private void tryShowCommandUsage() { - try { - String chosenCommand = jcommander.getParsedCommand(); - getUsageFormatter().usage(chosenCommand); - } catch (Exception e) { - // it is caused by an invalid command, the invalid command can not be parsed - System.err.println("Invalid command, please use `pulsar-admin --help` to check out how to use"); - } + protected PulsarAdmin getAdmin() { + return adminSupplier.get(); } - public boolean run(String[] args) { - try { - jcommander.parse(args); - } catch (Exception e) { - System.err.println(e.getMessage()); - System.err.println(); - tryShowCommandUsage(); - return false; - } - - String cmd = jcommander.getParsedCommand(); - if (cmd == null) { - jcommander.usage(); - return help; - } - - JCommander obj = jcommander.getCommands().get(cmd); - CliCommand cmdObj = (CliCommand) obj.getObjects().get(0); - - if (cmdObj.isHelp()) { - obj.setProgramName(jcommander.getProgramName() + " " + cmd); - obj.usage(); - return true; + protected long getRequestTimeoutMs() { + PulsarAdmin pulsarAdmin = getAdmin(); + if (pulsarAdmin instanceof PulsarAdminImpl) { + return ((PulsarAdminImpl) pulsarAdmin).getClientConfigData().getRequestTimeoutMs(); } + return DEFAULT_REQUEST_TIMEOUT_MILLIS; + } + protected T sync(Supplier> executor) throws PulsarAdminException { try { - cmdObj.run(); - return true; - } catch (ParameterException e) { - System.err.println(e.getMessage()); - System.err.println(); - return false; - } catch (ConnectException e) { - System.err.println(e.getMessage()); - System.err.println(); - System.err.println("Error connecting to: " + getAdmin().getServiceUrl()); - return false; - } catch (PulsarAdminException e) { - System.err.println(e.getHttpError()); - System.err.println(); - System.err.println("Reason: " + e.getMessage()); - return false; + return executor.get().get(getRequestTimeoutMs(), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new PulsarAdminException(e); + } catch (TimeoutException e) { + throw new PulsarAdminException.TimeoutException(e); + } catch (ExecutionException e) { + throw PulsarAdminException.wrap(getApiException(e.getCause())); } catch (Exception e) { - e.printStackTrace(); - return false; - } - } - - protected PulsarAdmin getAdmin() { - if (admin == null) { - admin = adminSupplier.get(); + throw PulsarAdminException.wrap(getApiException(e)); } - return admin; } - - static Map parseListKeyValueMap(List metadata) { + Map parseListKeyValueMap(List metadata) { Map map = null; if (metadata != null && !metadata.isEmpty()) { map = new HashMap<>(); @@ -141,7 +95,26 @@ static Map parseListKeyValueMap(List metadata) { return map; } - public JCommander getJcommander() { - return jcommander; + // Used to register the subcomand. + protected CommandLine getCommander() { + return commander; + } + + protected void addCommand(String name, Object cmd) { + commander.addSubcommand(name, cmd); + } + + protected void addCommand(String name, Object cmd, String... aliases) { + commander.addSubcommand(name, cmd, aliases); + } + + protected class ParameterException extends CommandLine.ParameterException { + public ParameterException(String msg) { + super(commander, msg); + } + + public ParameterException(String msg, Throwable e) { + super(commander, msg, e); + } } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBookies.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBookies.java index 8c8f0f4e8a2d1..09389d474ff5e 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBookies.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBookies.java @@ -18,31 +18,30 @@ */ package org.apache.pulsar.admin.cli; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; import com.google.common.base.Strings; import java.util.function.Supplier; import lombok.NonNull; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.common.policies.data.BookieInfo; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; -@Parameters(commandDescription = "Operations about bookies rack placement") +@Command(description = "Operations about bookies rack placement") public class CmdBookies extends CmdBase { - @Parameters(commandDescription = "Gets the rack placement information for all the bookies in the cluster") + @Command(description = "Gets the rack placement information for all the bookies in the cluster") private class GetAll extends CliCommand { @Override void run() throws Exception { - print(getAdmin().bookies().getBookiesRackInfo()); + prettyPrint(getAdmin().bookies().getBookiesRackInfo()); } } - @Parameters(commandDescription = "Gets the rack placement information for a specific bookie in the cluster") + @Command(description = "Gets the rack placement information for a specific bookie in the cluster") private class GetBookie extends CliCommand { - @Parameter(names = { "-b", "--bookie" }, + @Option(names = {"-b", "--bookie"}, description = "Bookie address (format: `address:port`)", required = true) private String bookieAddress; @@ -52,7 +51,7 @@ void run() throws Exception { } } - @Parameters(commandDescription = "List bookies") + @Command(description = "List bookies") private class ListBookies extends CliCommand { @Override @@ -61,10 +60,10 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Remove rack placement information for a specific bookie in the cluster") + @Command(description = "Remove rack placement information for a specific bookie in the cluster") private class RemoveBookie extends CliCommand { - @Parameter(names = { "-b", "--bookie" }, + @Option(names = {"-b", "--bookie"}, description = "Bookie address (format: `address:port`)", required = true) private String bookieAddress; @@ -74,19 +73,19 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Updates the rack placement information for a specific bookie in the cluster " + @Command(description = "Updates the rack placement information for a specific bookie in the cluster " + "(note. bookie address format:`address:port`)") private class UpdateBookie extends CliCommand { private static final String PATH_SEPARATOR = "/"; - @Parameter(names = { "-g", "--group" }, description = "Bookie group name", required = false) + @Option(names = {"-g", "--group"}, description = "Bookie group name", required = false) private String group = "default"; - @Parameter(names = { "-b", "--bookie" }, + @Option(names = {"-b", "--bookie"}, description = "Bookie address (format: `address:port`)", required = true) private String bookieAddress; - @Parameter(names = { "-r", "--rack" }, description = "Bookie rack name. " + @Option(names = {"-r", "--rack"}, description = "Bookie rack name. " + "If you set a bookie rack name to slash (/) " + "or an empty string (\"\"): " + "when using Pulsar earlier than 2.7.5, 2.8.3, and 2.9.2, " @@ -104,7 +103,7 @@ private class UpdateBookie extends CliCommand { + "but /region0rack0 and /region0/rack/0 are not allowed.", required = true) private String bookieRack; - @Parameter(names = {"-hn", "--hostname"}, description = "Bookie host name", required = false) + @Option(names = {"-hn", "--hostname"}, description = "Bookie host name", required = false) private String bookieHost; @Override @@ -128,10 +127,10 @@ private void checkArgument(boolean expression, @NonNull Object errorMessage) { public CmdBookies(Supplier admin) { super("bookies", admin); - jcommander.addCommand("racks-placement", new GetAll()); - jcommander.addCommand("list-bookies", new ListBookies()); - jcommander.addCommand("get-bookie-rack", new GetBookie()); - jcommander.addCommand("delete-bookie-rack", new RemoveBookie()); - jcommander.addCommand("set-bookie-rack", new UpdateBookie()); + addCommand("racks-placement", new GetAll()); + addCommand("list-bookies", new ListBookies()); + addCommand("get-bookie-rack", new GetBookie()); + addCommand("delete-bookie-rack", new RemoveBookie()); + addCommand("set-bookie-rack", new UpdateBookie()); } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBrokerStats.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBrokerStats.java index c83beec330f39..b9f7bdabd7c7f 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBrokerStats.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBrokerStats.java @@ -18,8 +18,6 @@ */ package org.apache.pulsar.admin.cli; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.google.gson.Gson; @@ -29,19 +27,21 @@ import java.io.OutputStreamWriter; import java.io.Writer; import java.nio.charset.StandardCharsets; -import java.util.List; import java.util.function.Supplier; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.common.stats.AllocatorStats; import org.apache.pulsar.common.util.ObjectMapperFactory; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; -@Parameters(commandDescription = "Operations to collect broker statistics") +@Command(description = "Operations to collect broker statistics") public class CmdBrokerStats extends CmdBase { private static final String DEFAULT_INDENTATION = " "; - @Parameters(commandDescription = "dump metrics for Monitoring") + @Command(description = "dump metrics for Monitoring") private class CmdMonitoringMetrics extends CliCommand { - @Parameter(names = { "-i", "--indent" }, description = "Indent JSON output", required = false) + @Option(names = {"-i", "--indent"}, description = "Indent JSON output", required = false) private boolean indent = false; @Override @@ -67,9 +67,9 @@ void run() throws Exception { } } - @Parameters(commandDescription = "dump mbean stats") + @Command(description = "dump mbean stats") private class CmdDumpMBeans extends CliCommand { - @Parameter(names = { "-i", "--indent" }, description = "Indent JSON output", required = false) + @Option(names = {"-i", "--indent"}, description = "Indent JSON output", required = false) private boolean indent = false; @Override @@ -88,7 +88,7 @@ void run() throws Exception { } - @Parameters(commandDescription = "dump broker load-report") + @Command(description = "dump broker load-report") private class CmdLoadReport extends CliCommand { @Override @@ -97,9 +97,9 @@ void run() throws Exception { } } - @Parameters(commandDescription = "dump topics stats") + @Command(description = "dump topics stats") private class CmdTopics extends CliCommand { - @Parameter(names = { "-i", "--indent" }, description = "Indent JSON output", required = false) + @Option(names = {"-i", "--indent"}, description = "Indent JSON output", required = false) private boolean indent = false; @Override @@ -118,14 +118,14 @@ void run() throws Exception { } - @Parameters(commandDescription = "dump allocator stats") + @Command(description = "dump allocator stats") private class CmdAllocatorStats extends CliCommand { - @Parameter(description = "allocator-name", required = true) - private List params; + @Parameters(description = "allocator-name", arity = "1") + private String allocatorName; @Override void run() throws Exception { - AllocatorStats stats = getAdmin().brokerStats().getAllocatorStats(params.get(0)); + AllocatorStats stats = getAdmin().brokerStats().getAllocatorStats(allocatorName); ObjectMapper mapper = ObjectMapperFactory.create(); ObjectWriter writer = mapper.writerWithDefaultPrettyPrinter(); try (Writer out = new OutputStreamWriter(System.out, StandardCharsets.UTF_8)) { @@ -138,11 +138,11 @@ void run() throws Exception { public CmdBrokerStats(Supplier admin) { super("broker-stats", admin); - jcommander.addCommand("monitoring-metrics", new CmdMonitoringMetrics()); - jcommander.addCommand("mbeans", new CmdDumpMBeans()); - jcommander.addCommand("topics", new CmdTopics(), "destinations"); - jcommander.addCommand("allocator-stats", new CmdAllocatorStats()); - jcommander.addCommand("load-report", new CmdLoadReport()); + addCommand("monitoring-metrics", new CmdMonitoringMetrics()); + addCommand("mbeans", new CmdDumpMBeans()); + addCommand("topics", new CmdTopics(), "destinations"); + addCommand("allocator-stats", new CmdAllocatorStats()); + addCommand("load-report", new CmdLoadReport()); } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBrokers.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBrokers.java index 1e86edcf59c60..b85a784c3c2b8 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBrokers.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdBrokers.java @@ -18,28 +18,28 @@ */ package org.apache.pulsar.admin.cli; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; import java.util.function.Supplier; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.common.naming.TopicVersion; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; -@Parameters(commandDescription = "Operations about brokers") +@Command(description = "Operations about brokers") public class CmdBrokers extends CmdBase { - @Parameters(commandDescription = "List active brokers of the cluster") + @Command(description = "List active brokers of the cluster") private class List extends CliCommand { - @Parameter(description = "cluster-name") - private java.util.List params; + @Parameters(description = "cluster-name", arity = "1") + private String cluster; @Override void run() throws Exception { - String cluster = params == null ? null : getOneArgument(params); print(getAdmin().brokers().getActiveBrokers(cluster)); } } - @Parameters(commandDescription = "Get the information of the leader broker") + @Command(description = "Get the information of the leader broker") private class LeaderBroker extends CliCommand { @Override @@ -48,25 +48,25 @@ void run() throws Exception { } } - @Parameters(commandDescription = "List namespaces owned by the broker") + @Command(description = "List namespaces owned by the broker") private class Namespaces extends CliCommand { - @Parameter(description = "cluster-name", required = true) - private java.util.List params; - @Parameter(names = {"-u", "--url"}, description = "broker-url", required = true) + @Parameters(description = "cluster-name", arity = "1") + private String cluster; + + @Option(names = {"-u", "--url"}, description = "broker-url", required = true) private String brokerUrl; @Override void run() throws Exception { - String cluster = getOneArgument(params); print(getAdmin().brokers().getOwnedNamespaces(cluster, brokerUrl)); } } - @Parameters(commandDescription = "Update dynamic-serviceConfiguration of broker") + @Command(description = "Update dynamic-serviceConfiguration of broker") private class UpdateConfigurationCmd extends CliCommand { - @Parameter(names = {"-c", "--config"}, description = "service-configuration name", required = true) + @Option(names = {"-c", "--config"}, description = "service-configuration name", required = true) private String configName; - @Parameter(names = {"-v", "--value"}, description = "service-configuration value", required = true) + @Option(names = {"-v", "--value"}, description = "service-configuration value", required = true) private String configValue; @Override @@ -75,9 +75,9 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Delete dynamic-serviceConfiguration of broker") + @Command(description = "Delete dynamic-serviceConfiguration of broker") private class DeleteConfigurationCmd extends CliCommand { - @Parameter(names = {"-c", "--config"}, description = "service-configuration name", required = true) + @Option(names = {"-c", "--config"}, description = "service-configuration name", required = true) private String configName; @Override @@ -86,7 +86,7 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Get all overridden dynamic-configuration values") + @Command(description = "Get all overridden dynamic-configuration values") private class GetAllConfigurationsCmd extends CliCommand { @Override @@ -95,7 +95,7 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Get list of updatable configuration name") + @Command(description = "Get list of updatable configuration name") private class GetUpdatableConfigCmd extends CliCommand { @Override @@ -104,7 +104,7 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Get runtime configuration values") + @Command(description = "Get runtime configuration values") private class GetRuntimeConfigCmd extends CliCommand { @Override @@ -113,7 +113,7 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Get internal configuration information") + @Command(description = "Get internal configuration information") private class GetInternalConfigurationCmd extends CliCommand { @Override @@ -123,10 +123,10 @@ void run() throws Exception { } - @Parameters(commandDescription = "Run a health check against the broker") + @Command(description = "Run a health check against the broker") private class HealthcheckCmd extends CliCommand { - @Parameter(names = {"-tv", "--topic-version"}, description = "topic version V1 is default") + @Option(names = {"-tv", "--topic-version"}, description = "topic version V1 is default") private TopicVersion topicVersion; @Override @@ -137,26 +137,26 @@ void run() throws Exception { } - @Parameters(commandDescription = "Shutdown broker gracefully.") + @Command(description = "Shutdown broker gracefully.") private class ShutDownBrokerGracefully extends CliCommand { - @Parameter(names = {"--max-concurrent-unload-per-sec", "-m"}, + @Option(names = {"--max-concurrent-unload-per-sec", "-m"}, description = "Max concurrent unload per second, " + "if the value absent(value=0) means no concurrent limitation") private int maxConcurrentUnloadPerSec; - @Parameter(names = {"--forced-terminate-topic", "-f"}, description = "Force terminate all topics on Broker") + @Option(names = {"--forced-terminate-topic", "-f"}, description = "Force terminate all topics on Broker") private boolean forcedTerminateTopic; @Override void run() throws Exception { - getAdmin().brokers().shutDownBrokerGracefully(maxConcurrentUnloadPerSec, forcedTerminateTopic); - System.out.println("Successfully trigger broker shutdown gracefully"); + sync(() -> getAdmin().brokers().shutDownBrokerGracefully(maxConcurrentUnloadPerSec, forcedTerminateTopic)); + System.out.println("Successfully shutdown broker gracefully"); } } - @Parameters(commandDescription = "Manually trigger backlogQuotaCheck") + @Command(description = "Manually trigger backlogQuotaCheck") private class BacklogQuotaCheckCmd extends CliCommand { @Override @@ -167,7 +167,7 @@ void run() throws Exception { } - @Parameters(commandDescription = "Get the version of the currently connected broker") + @Command(description = "Get the version of the currently connected broker") private class PulsarVersion extends CliCommand { @Override @@ -178,18 +178,18 @@ void run() throws Exception { public CmdBrokers(Supplier admin) { super("brokers", admin); - jcommander.addCommand("list", new List()); - jcommander.addCommand("leader-broker", new LeaderBroker()); - jcommander.addCommand("namespaces", new Namespaces()); - jcommander.addCommand("update-dynamic-config", new UpdateConfigurationCmd()); - jcommander.addCommand("delete-dynamic-config", new DeleteConfigurationCmd()); - jcommander.addCommand("list-dynamic-config", new GetUpdatableConfigCmd()); - jcommander.addCommand("get-all-dynamic-config", new GetAllConfigurationsCmd()); - jcommander.addCommand("get-internal-config", new GetInternalConfigurationCmd()); - jcommander.addCommand("get-runtime-config", new GetRuntimeConfigCmd()); - jcommander.addCommand("healthcheck", new HealthcheckCmd()); - jcommander.addCommand("backlog-quota-check", new BacklogQuotaCheckCmd()); - jcommander.addCommand("version", new PulsarVersion()); - jcommander.addCommand("shutdown", new ShutDownBrokerGracefully()); + addCommand("list", new List()); + addCommand("leader-broker", new LeaderBroker()); + addCommand("namespaces", new Namespaces()); + addCommand("update-dynamic-config", new UpdateConfigurationCmd()); + addCommand("delete-dynamic-config", new DeleteConfigurationCmd()); + addCommand("list-dynamic-config", new GetUpdatableConfigCmd()); + addCommand("get-all-dynamic-config", new GetAllConfigurationsCmd()); + addCommand("get-internal-config", new GetInternalConfigurationCmd()); + addCommand("get-runtime-config", new GetRuntimeConfigCmd()); + addCommand("healthcheck", new HealthcheckCmd()); + addCommand("backlog-quota-check", new BacklogQuotaCheckCmd()); + addCommand("version", new PulsarVersion()); + addCommand("shutdown", new ShutDownBrokerGracefully()); } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdClusters.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdClusters.java index 173595c9b19a4..9f11b48513867 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdClusters.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdClusters.java @@ -19,56 +19,69 @@ package org.apache.pulsar.admin.cli; import static org.apache.commons.lang3.StringUtils.isNotBlank; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; import com.google.common.collect.Sets; +import java.io.IOException; import java.util.Arrays; import java.util.function.Supplier; -import lombok.Getter; +import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.admin.cli.utils.CmdUtils; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.ProxyProtocol; import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.ClusterData.ClusterUrl; import org.apache.pulsar.common.policies.data.ClusterDataImpl; +import org.apache.pulsar.common.policies.data.ClusterPolicies.ClusterUrl; import org.apache.pulsar.common.policies.data.FailureDomain; import org.apache.pulsar.common.policies.data.FailureDomainImpl; +import picocli.CommandLine.ArgGroup; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; -@Parameters(commandDescription = "Operations about clusters") +@Command(description = "Operations about clusters") public class CmdClusters extends CmdBase { - @Parameters(commandDescription = "List the existing clusters") + @Command(description = "List the existing clusters") private class List extends CliCommand { - void run() throws PulsarAdminException { - print(getAdmin().clusters().getClusters()); + @Option(names = {"-c", "--current"}, + description = "Print the current cluster with (*)", required = false, defaultValue = "false") + private boolean current; + + void run() throws Exception { + java.util.List clusters = getAdmin().clusters().getClusters(); + String clusterName = getAdmin().brokers().getRuntimeConfigurations().get("clusterName"); + final java.util.List result = clusters.stream().map(c -> + c.equals(clusterName) ? (current ? c + "(*)" : c) : c + ).collect(Collectors.toList()); + print(result); } } - @Parameters(commandDescription = "Get the configuration data for the specified cluster") + @Command(description = "Get the configuration data for the specified cluster") private class Get extends CliCommand { - @Parameter(description = "cluster-name", required = true) - private java.util.List params; + @Parameters(description = "cluster-name", arity = "1") + private String cluster; - void run() throws PulsarAdminException { - String cluster = getOneArgument(params); + @Override + void run() throws Exception { print(getAdmin().clusters().getCluster(cluster)); } } - @Parameters(commandDescription = "Provisions a new cluster. This operation requires Pulsar super-user privileges") - private class Create extends ClusterDetailsCommand { + @Command(description = "Provisions a new cluster. This operation requires Pulsar super-user privileges") + private class Create extends CliCommand { + @ArgGroup(exclusive = false) + ClusterDetails clusterDetails = new ClusterDetails(); @Override - void runCmd() throws Exception { - String cluster = getOneArgument(params); - getAdmin().clusters().createCluster(cluster, clusterData); + void run() throws PulsarAdminException, IOException { + getAdmin().clusters().createCluster(clusterDetails.clusterName, clusterDetails.getClusterData()); } } - protected void validateClusterData(ClusterData clusterData) { + protected static void validateClusterData(ClusterData clusterData) { if (clusterData.isBrokerClientTlsEnabled()) { if (clusterData.isBrokerClientTlsEnabledWithKeyStore()) { if (StringUtils.isAnyBlank(clusterData.getBrokerClientTlsTrustStoreType(), @@ -82,29 +95,29 @@ protected void validateClusterData(ClusterData clusterData) { } } - @Parameters(commandDescription = "Update the configuration for a cluster") - private class Update extends ClusterDetailsCommand { + @Command(description = "Update the configuration for a cluster") + private class Update extends CliCommand { + @ArgGroup(exclusive = false) + ClusterDetails clusterDetails = new ClusterDetails(); @Override - void runCmd() throws Exception { - String cluster = getOneArgument(params); - getAdmin().clusters().updateCluster(cluster, clusterData); + void run() throws PulsarAdminException, IOException { + getAdmin().clusters().updateCluster(clusterDetails.clusterName, clusterDetails.getClusterData()); } } - @Parameters(commandDescription = "Deletes an existing cluster") + @Command(description = "Deletes an existing cluster") private class Delete extends CliCommand { - @Parameter(description = "cluster-name", required = true) - private java.util.List params; + @Parameters(description = "cluster-name", index = "0", arity = "1") + private String cluster; - @Parameter(names = { "-a", "--all" }, - description = "Delete all data (tenants) of the cluster", required = false) - private boolean deleteAll = false; + @Option(names = {"-a", "--all"}, + description = "Delete all data (tenants) of the cluster", required = false, defaultValue = "false") + private boolean deleteAll; + @Override void run() throws PulsarAdminException { - String cluster = getOneArgument(params); - if (deleteAll) { for (String tenant : getAdmin().tenants().getTenants()) { for (String namespace : getAdmin().namespaces().getNamespaces(tenant)) { @@ -126,71 +139,88 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Update peer cluster names") + @Command(description = "Update peer cluster names") private class UpdatePeerClusters extends CliCommand { - @Parameter(description = "cluster-name", required = true) - private java.util.List params; + @Parameters(description = "cluster-name", arity = "1") + private String cluster; - @Parameter(names = "--peer-clusters", description = "Comma separated peer-cluster names " + @Option(names = "--peer-clusters", description = "Comma separated peer-cluster names " + "[Pass empty string \"\" to delete list]", required = true) private String peerClusterNames; + @Override void run() throws PulsarAdminException { - String cluster = getOneArgument(params); java.util.LinkedHashSet clusters = StringUtils.isBlank(peerClusterNames) ? null : Sets.newLinkedHashSet(Arrays.asList(peerClusterNames.split(","))); getAdmin().clusters().updatePeerClusterNames(cluster, clusters); } } - @Parameters(commandDescription = "Update cluster migration") + @Command(description = "Get the cluster migration configuration data for the specified cluster") + private class GetClusterMigration extends CliCommand { + @Parameters(description = "cluster-name", arity = "1") + private String cluster; + + @Override + void run() throws PulsarAdminException { + getAdmin().clusters().getClusterMigration(cluster); + } + } + + @Command(description = "Update cluster migration") private class UpdateClusterMigration extends CliCommand { - @Parameter(description = "cluster-name", required = true) - private java.util.List params; + @Parameters(description = "cluster-name", arity = "1") + private String cluster; - @Parameter(names = "--migrated", description = "Is cluster migrated", required = true) + @Option(names = "--migrated", description = "Is cluster migrated") private boolean migrated; - @Parameter(names = "--broker-url", description = "New migrated cluster broker service url", required = false) + @Option(names = "--service-url", description = "New migrated cluster service url") + private String serviceUrl; + + @Option(names = "--service-url-secure", + description = "New migrated cluster service url secure") + private String serviceUrlTls; + + @Option(names = "--broker-url", description = "New migrated cluster broker service url") private String brokerServiceUrl; - @Parameter(names = "--broker-url-secure", description = "New migrated cluster broker service url secure", - required = false) + @Option(names = "--broker-url-secure", description = "New migrated cluster broker service url secure") private String brokerServiceUrlTls; + @Override void run() throws PulsarAdminException { - String cluster = getOneArgument(params); - ClusterUrl clusterUrl = new ClusterUrl(brokerServiceUrl, brokerServiceUrlTls); + ClusterUrl clusterUrl = new ClusterUrl(serviceUrl, serviceUrlTls, brokerServiceUrl, brokerServiceUrlTls); getAdmin().clusters().updateClusterMigration(cluster, migrated, clusterUrl); } } - @Parameters(commandDescription = "Get list of peer-clusters") + @Command(description = "Get list of peer-clusters") private class GetPeerClusters extends CliCommand { - @Parameter(description = "cluster-name", required = true) - private java.util.List params; + @Parameters(description = "cluster-name", arity = "1") + private String cluster; + @Override void run() throws PulsarAdminException { - String cluster = getOneArgument(params); print(getAdmin().clusters().getPeerClusterNames(cluster)); } } - @Parameters(commandDescription = "Create a new failure-domain for a cluster. updates it if already created.") + @Command(description = "Create a new failure-domain for a cluster. updates it if already created.") private class CreateFailureDomain extends CliCommand { - @Parameter(description = "cluster-name", required = true) - private java.util.List params; + @Parameters(description = "cluster-name", arity = "1") + private String cluster; - @Parameter(names = "--domain-name", description = "domain-name", required = true) + @Option(names = "--domain-name", description = "domain-name", required = true) private String domainName; - @Parameter(names = "--broker-list", description = "Comma separated broker list", required = false) + @Option(names = "--broker-list", description = "Comma separated broker list", required = false) private String brokerList; + @Override void run() throws PulsarAdminException { - String cluster = getOneArgument(params); FailureDomain domain = FailureDomainImpl.builder() .brokers((isNotBlank(brokerList) ? Sets.newHashSet(brokerList.split(",")) : null)) .build(); @@ -198,19 +228,19 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Update failure-domain for a cluster. Creates a new one if not exist.") + @Command(description = "Update failure-domain for a cluster. Creates a new one if not exist.") private class UpdateFailureDomain extends CliCommand { - @Parameter(description = "cluster-name", required = true) - private java.util.List params; + @Parameters(description = "cluster-name", arity = "1") + private String cluster; - @Parameter(names = "--domain-name", description = "domain-name", required = true) + @Option(names = "--domain-name", description = "domain-name", required = true) private String domainName; - @Parameter(names = "--broker-list", description = "Comma separated broker list", required = false) + @Option(names = "--broker-list", description = "Comma separated broker list", required = false) private String brokerList; + @Override void run() throws PulsarAdminException { - String cluster = getOneArgument(params); FailureDomain domain = FailureDomainImpl.builder() .brokers((isNotBlank(brokerList) ? Sets.newHashSet(brokerList.split(",")) : null)) .build(); @@ -218,162 +248,139 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Deletes an existing failure-domain") + @Command(description = "Deletes an existing failure-domain") private class DeleteFailureDomain extends CliCommand { - @Parameter(description = "cluster-name", required = true) - private java.util.List params; + @Parameters(description = "cluster-name", arity = "1") + private String cluster; - @Parameter(names = "--domain-name", description = "domain-name", required = true) + @Option(names = "--domain-name", description = "domain-name", required = true) private String domainName; + @Override void run() throws PulsarAdminException { - String cluster = getOneArgument(params); getAdmin().clusters().deleteFailureDomain(cluster, domainName); } } - @Parameters(commandDescription = "List the existing failure-domains for a cluster") + @Command(description = "List the existing failure-domains for a cluster") private class ListFailureDomains extends CliCommand { + @Parameters(description = "cluster-name", arity = "1") + private String cluster; - @Parameter(description = "cluster-name", required = true) - private java.util.List params; - + @Override void run() throws PulsarAdminException { - String cluster = getOneArgument(params); print(getAdmin().clusters().getFailureDomains(cluster)); } } - @Parameters(commandDescription = "Get the configuration brokers of a failure-domain") + @Command(description = "Get the configuration brokers of a failure-domain") private class GetFailureDomain extends CliCommand { - @Parameter(description = "cluster-name", required = true) - private java.util.List params; + @Parameters(description = "cluster-name", arity = "1") + private String cluster; - @Parameter(names = "--domain-name", description = "domain-name", required = true) + @Option(names = "--domain-name", description = "domain-name", required = true) private String domainName; + @Override void run() throws PulsarAdminException { - String cluster = getOneArgument(params); print(getAdmin().clusters().getFailureDomain(cluster, domainName)); } } - /** - * Base command. - */ - @Getter - abstract class BaseCommand extends CliCommand { - @Override - void run() throws Exception { - try { - processArguments(); - } catch (Exception e) { - System.err.println(e.getMessage()); - System.err.println(); - String chosenCommand = jcommander.getParsedCommand(); - getUsageFormatter().usage(chosenCommand); - return; - } - runCmd(); - } + private static class ClusterDetails { + @Parameters(description = "cluster-name", arity = "1") + protected String clusterName; - void processArguments() throws Exception { - } - - abstract void runCmd() throws Exception; - } - - abstract class ClusterDetailsCommand extends BaseCommand { - @Parameter(description = "cluster-name", required = true) - protected java.util.List params; - - @Parameter(names = "--url", description = "service-url", required = false) + @Option(names = "--url", description = "service-url", required = false) protected String serviceUrl; - @Parameter(names = "--url-secure", description = "service-url for secure connection", required = false) + @Option(names = "--url-secure", description = "service-url for secure connection", required = false) protected String serviceUrlTls; - @Parameter(names = "--broker-url", description = "broker-service-url", required = false) + @Option(names = "--broker-url", description = "broker-service-url", required = false) protected String brokerServiceUrl; - @Parameter(names = "--broker-url-secure", + @Option(names = "--broker-url-secure", description = "broker-service-url for secure connection", required = false) protected String brokerServiceUrlTls; - @Parameter(names = "--proxy-url", + @Option(names = "--proxy-url", description = "Proxy-service url when client would like to connect to broker via proxy.") protected String proxyServiceUrl; - @Parameter(names = "--auth-plugin", description = "authentication plugin", required = false) + @Option(names = "--auth-plugin", description = "authentication plugin", required = false) protected String authenticationPlugin; - @Parameter(names = "--auth-parameters", description = "authentication parameters", required = false) + @Option(names = "--auth-parameters", description = "authentication parameters", required = false) protected String authenticationParameters; - @Parameter(names = "--proxy-protocol", + @Option(names = "--proxy-protocol", description = "protocol to decide type of proxy routing eg: SNI", required = false) protected ProxyProtocol proxyProtocol; - @Parameter(names = "--tls-enable", description = "Enable tls connection", required = false) + @Option(names = "--tls-enable", description = "Enable tls connection", required = false) protected Boolean brokerClientTlsEnabled; - @Parameter(names = "--tls-allow-insecure", description = "Allow insecure tls connection", required = false) + @Option(names = "--tls-allow-insecure", description = "Allow insecure tls connection", required = false) protected Boolean tlsAllowInsecureConnection; - @Parameter(names = "--tls-enable-keystore", + @Option(names = "--tls-enable-keystore", description = "Whether use KeyStore type to authenticate", required = false) protected Boolean brokerClientTlsEnabledWithKeyStore; - @Parameter(names = "--tls-trust-store-type", + @Option(names = "--tls-trust-store-type", description = "TLS TrustStore type configuration for internal client eg: JKS", required = false) protected String brokerClientTlsTrustStoreType; - @Parameter(names = "--tls-trust-store", + @Option(names = "--tls-trust-store", description = "TLS TrustStore path for internal client", required = false) protected String brokerClientTlsTrustStore; - @Parameter(names = "--tls-trust-store-pwd", + @Option(names = "--tls-trust-store-pwd", description = "TLS TrustStore password for internal client", required = false) protected String brokerClientTlsTrustStorePassword; - @Parameter(names = "--tls-key-store-type", + @Option(names = "--tls-key-store-type", description = "TLS TrustStore type configuration for internal client eg: JKS", required = false) protected String brokerClientTlsKeyStoreType; - @Parameter(names = "--tls-key-store", + @Option(names = "--tls-key-store", description = "TLS KeyStore path for internal client", required = false) protected String brokerClientTlsKeyStore; - @Parameter(names = "--tls-key-store-pwd", + @Option(names = "--tls-key-store-pwd", description = "TLS KeyStore password for internal client", required = false) protected String brokerClientTlsKeyStorePassword; - @Parameter(names = "--tls-trust-certs-filepath", + @Option(names = "--tls-trust-certs-filepath", description = "path for the trusted TLS certificate file", required = false) protected String brokerClientTrustCertsFilePath; - @Parameter(names = "--tls-key-filepath", + @Option(names = "--tls-key-filepath", description = "path for the TLS private key file", required = false) protected String brokerClientKeyFilePath; - @Parameter(names = "--tls-certs-filepath", + @Option(names = "--tls-certs-filepath", description = "path for the TLS certificate file", required = false) protected String brokerClientCertificateFilePath; - @Parameter(names = "--listener-name", + @Option(names = "--tls-factory-plugin", + description = "TLS Factory Plugin to be used to generate SSL Context and SSL Engine") + protected String brokerClientSslFactoryPlugin; + + @Option(names = "--tls-factory-plugin-params", + description = "Parameters used by the TLS Factory Plugin") + protected String brokerClientSslFactoryPluginParams; + + @Option(names = "--listener-name", description = "listenerName when client would like to connect to cluster", required = false) protected String listenerName; - @Parameter(names = "--cluster-config-file", description = "The path to a YAML config file specifying the " + @Option(names = "--cluster-config-file", description = "The path to a YAML config file specifying the " + "cluster's configuration") protected String clusterConfigFile; - protected ClusterData clusterData; - - @Override - void processArguments() throws Exception { - super.processArguments(); - + protected ClusterData getClusterData() throws IOException { ClusterData.Builder builder; if (null != clusterConfigFile) { builder = CmdUtils.loadConfig(clusterConfigFile, ClusterDataImpl.ClusterDataImplBuilder.class); @@ -441,31 +448,40 @@ void processArguments() throws Exception { if (brokerClientCertificateFilePath != null) { builder.brokerClientCertificateFilePath(brokerClientCertificateFilePath); } + if (StringUtils.isNotBlank(brokerClientSslFactoryPlugin)) { + builder.brokerClientSslFactoryPlugin(brokerClientSslFactoryPlugin); + } + if (StringUtils.isNotBlank(brokerClientSslFactoryPluginParams)) { + builder.brokerClientSslFactoryPluginParams(brokerClientSslFactoryPluginParams); + } if (listenerName != null) { builder.listenerName(listenerName); } - this.clusterData = builder.build(); + ClusterData clusterData = builder.build(); validateClusterData(clusterData); + + return clusterData; } } public CmdClusters(Supplier admin) { super("clusters", admin); - jcommander.addCommand("get", new Get()); - jcommander.addCommand("create", new Create()); - jcommander.addCommand("update", new Update()); - jcommander.addCommand("delete", new Delete()); - jcommander.addCommand("list", new List()); - jcommander.addCommand("update-peer-clusters", new UpdatePeerClusters()); - jcommander.addCommand("update-cluster-migration", new UpdateClusterMigration()); - jcommander.addCommand("get-peer-clusters", new GetPeerClusters()); - jcommander.addCommand("get-failure-domain", new GetFailureDomain()); - jcommander.addCommand("create-failure-domain", new CreateFailureDomain()); - jcommander.addCommand("update-failure-domain", new UpdateFailureDomain()); - jcommander.addCommand("delete-failure-domain", new DeleteFailureDomain()); - jcommander.addCommand("list-failure-domains", new ListFailureDomains()); + addCommand("get", new Get()); + addCommand("create", new Create()); + addCommand("update", new Update()); + addCommand("delete", new Delete()); + addCommand("list", new List()); + addCommand("update-peer-clusters", new UpdatePeerClusters()); + addCommand("get-cluster-migration", new GetClusterMigration()); + addCommand("update-cluster-migration", new UpdateClusterMigration()); + addCommand("get-peer-clusters", new GetPeerClusters()); + addCommand("get-failure-domain", new GetFailureDomain()); + addCommand("create-failure-domain", new CreateFailureDomain()); + addCommand("update-failure-domain", new UpdateFailureDomain()); + addCommand("delete-failure-domain", new DeleteFailureDomain()); + addCommand("list-failure-domains", new ListFailureDomains()); } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctionWorker.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctionWorker.java index 8fa2cbad955ef..cfb7f142d2458 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctionWorker.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctionWorker.java @@ -18,15 +18,15 @@ */ package org.apache.pulsar.admin.cli; -import com.beust.jcommander.Parameters; import java.util.function.Supplier; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.PulsarClientException; +import picocli.CommandLine.Command; @Slf4j -@Parameters(commandDescription = "Operations to collect function-worker statistics") +@Command(description = "Operations to collect function-worker statistics") public class CmdFunctionWorker extends CmdBase { /** @@ -46,7 +46,7 @@ void processArguments() throws Exception { abstract void runCmd() throws Exception; } - @Parameters(commandDescription = "Dump all functions stats running on this broker") + @Command(description = "Dump all functions stats running on this broker") class FunctionsStats extends BaseCommand { @Override @@ -55,7 +55,7 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Dump metrics for Monitoring") + @Command(description = "Dump metrics for Monitoring") class CmdMonitoringMetrics extends BaseCommand { @Override @@ -64,7 +64,7 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Get all workers belonging to this cluster") + @Command(description = "Get all workers belonging to this cluster") class GetCluster extends BaseCommand { @Override @@ -73,7 +73,7 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Get the leader of the worker cluster") + @Command(description = "Get the leader of the worker cluster") class GetClusterLeader extends BaseCommand { @Override @@ -82,7 +82,7 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Get the assignments of the functions across the worker cluster") + @Command(description = "Get the assignments of the functions across the worker cluster") class GetFunctionAssignments extends BaseCommand { @@ -92,7 +92,7 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Triggers a rebalance of functions to workers") + @Command(description = "Triggers a rebalance of functions to workers") class Rebalance extends BaseCommand { @Override @@ -104,12 +104,12 @@ void runCmd() throws Exception { public CmdFunctionWorker(Supplier admin) throws PulsarClientException { super("functions-worker", admin); - jcommander.addCommand("function-stats", new FunctionsStats()); - jcommander.addCommand("monitoring-metrics", new CmdMonitoringMetrics()); - jcommander.addCommand("get-cluster", new GetCluster()); - jcommander.addCommand("get-cluster-leader", new GetClusterLeader()); - jcommander.addCommand("get-function-assignments", new GetFunctionAssignments()); - jcommander.addCommand("rebalance", new Rebalance()); + addCommand("function-stats", new FunctionsStats()); + addCommand("monitoring-metrics", new CmdMonitoringMetrics()); + addCommand("get-cluster", new GetCluster()); + addCommand("get-cluster-leader", new GetClusterLeader()); + addCommand("get-function-assignments", new GetFunctionAssignments()); + addCommand("rebalance", new Rebalance()); } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java index 9b30d59f1679c..4c7e058af6de1 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java @@ -22,10 +22,6 @@ import static org.apache.commons.lang.StringUtils.isNotBlank; import static org.apache.pulsar.common.naming.TopicName.DEFAULT_NAMESPACE; import static org.apache.pulsar.common.naming.TopicName.PUBLIC_TENANT; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; -import com.beust.jcommander.converters.StringConverter; import com.google.common.annotations.VisibleForTesting; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -58,9 +54,11 @@ import org.apache.pulsar.common.functions.Utils; import org.apache.pulsar.common.functions.WindowConfig; import org.apache.pulsar.common.util.ObjectMapperFactory; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; @Slf4j -@Parameters(commandDescription = "Interface for managing Pulsar Functions " +@Command(description = "Interface for managing Pulsar Functions " + "(lightweight, Lambda-style compute processes that work with Pulsar)") public class CmdFunctions extends CmdBase { private final LocalRunner localRunner; @@ -88,15 +86,7 @@ public class CmdFunctions extends CmdBase { abstract class BaseCommand extends CliCommand { @Override void run() throws Exception { - try { - processArguments(); - } catch (Exception e) { - System.err.println(e.getMessage()); - System.err.println(); - String chosenCommand = jcommander.getParsedCommand(); - getUsageFormatter().usage(chosenCommand); - return; - } + processArguments(); runCmd(); } @@ -110,21 +100,11 @@ void processArguments() throws Exception {} */ @Getter abstract class NamespaceCommand extends BaseCommand { - @Parameter(names = "--tenant", description = "The tenant of a Pulsar Function") + @Option(names = "--tenant", description = "The tenant of a Pulsar Function") protected String tenant; - @Parameter(names = "--namespace", description = "The namespace of a Pulsar Function") + @Option(names = "--namespace", description = "The namespace of a Pulsar Function") protected String namespace; - - @Override - public void processArguments() { - if (tenant == null) { - tenant = PUBLIC_TENANT; - } - if (namespace == null) { - namespace = DEFAULT_NAMESPACE; - } - } } /** @@ -132,22 +112,20 @@ public void processArguments() { */ @Getter abstract class FunctionCommand extends BaseCommand { - @Parameter(names = "--fqfn", description = "The Fully Qualified Function Name (FQFN) for the function") + @Option(names = "--fqfn", description = "The Fully Qualified Function Name (FQFN) for the function") protected String fqfn; - @Parameter(names = "--tenant", description = "The tenant of a Pulsar Function") + @Option(names = "--tenant", description = "The tenant of a Pulsar Function") protected String tenant; - @Parameter(names = "--namespace", description = "The namespace of a Pulsar Function") + @Option(names = "--namespace", description = "The namespace of a Pulsar Function") protected String namespace; - @Parameter(names = "--name", description = "The name of a Pulsar Function") + @Option(names = "--name", description = "The name of a Pulsar Function") protected String functionName; @Override void processArguments() throws Exception { - super.processArguments(); - boolean usesSetters = (null != tenant || null != namespace || null != functionName); boolean usesFqfn = (null != fqfn); @@ -185,216 +163,217 @@ void processArguments() throws Exception { */ @Getter abstract class FunctionDetailsCommand extends BaseCommand { - @Parameter(names = "--fqfn", description = "The Fully Qualified Function Name (FQFN) for the function" + @Option(names = "--fqfn", description = "The Fully Qualified Function Name (FQFN) for the function" + " #Java, Python") protected String fqfn; - @Parameter(names = "--tenant", description = "The tenant of a Pulsar Function #Java, Python, Go") + @Option(names = "--tenant", description = "The tenant of a Pulsar Function #Java, Python, Go") protected String tenant; - @Parameter(names = "--namespace", description = "The namespace of a Pulsar Function #Java, Python, Go") + @Option(names = "--namespace", description = "The namespace of a Pulsar Function #Java, Python, Go") protected String namespace; - @Parameter(names = "--name", description = "The name of a Pulsar Function #Java, Python, Go") + @Option(names = "--name", description = "The name of a Pulsar Function #Java, Python, Go") protected String functionName; // for backwards compatibility purposes - @Parameter(names = "--className", description = "The class name of a Pulsar Function", hidden = true) + @Option(names = "--className", description = "The class name of a Pulsar Function", hidden = true) protected String deprecatedClassName; - @Parameter(names = "--classname", description = "The class name of a Pulsar Function #Java, Python") + @Option(names = "--classname", description = "The class name of a Pulsar Function #Java, Python") protected String className; - @Parameter(names = { "-t", "--function-type" }, description = "The built-in Pulsar Function type") + @Option(names = { "-t", "--function-type" }, description = "The built-in Pulsar Function type") protected String functionType; - @Parameter(names = "--cleanup-subscription", description = "Whether delete the subscription " + @Option(names = "--cleanup-subscription", description = "Whether delete the subscription " + "when function is deleted") protected Boolean cleanupSubscription; - @Parameter(names = "--jar", description = "Path to the JAR file for the function " + @Option(names = "--jar", description = "Path to the JAR file for the function " + "(if the function is written in Java). It also supports URL path [http/https/file " + "(file protocol assumes that file already exists on worker host)/function " - + "(package URL from packages management service)] from which worker can download the package. #Java", - listConverter = StringConverter.class) + + "(package URL from packages management service)] from which worker can download the package. #Java") protected String jarFile; - @Parameter(names = "--py", description = "Path to the main Python file/Python Wheel file for the function " + @Option(names = "--py", description = "Path to the main Python file/Python Wheel file for the function " + "(if the function is written in Python). It also supports URL path [http/https/file " + "(file protocol assumes that file already exists on worker host)/function " - + "(package URL from packages management service)] from which worker can download the package. #Python", - listConverter = StringConverter.class) + + "(package URL from packages management service)] from which worker can download the package. #Python") protected String pyFile; - @Parameter(names = "--go", description = "Path to the main Go executable binary for the function " + @Option(names = "--go", description = "Path to the main Go executable binary for the function " + "(if the function is written in Go). It also supports URL path [http/https/file " + "(file protocol assumes that file already exists on worker host)/function " + "(package URL from packages management service)] from which worker can download the package. #Go") protected String goFile; - @Parameter(names = {"-i", "--inputs"}, description = "The input topic or " + @Option(names = {"-i", "--inputs"}, description = "The input topic or " + "topics (multiple topics can be specified as a comma-separated list) of a Pulsar Function" + " #Java, Python, Go") protected String inputs; // for backwards compatibility purposes - @Parameter(names = "--topicsPattern", description = "TopicsPattern to consume from list of topics " + @Option(names = "--topicsPattern", description = "TopicsPattern to consume from list of topics " + "under a namespace that match the pattern. [--input] and [--topic-pattern] are mutually exclusive. " + "Add SerDe class name for a pattern in --custom-serde-inputs (supported for java fun only)", hidden = true) protected String deprecatedTopicsPattern; - @Parameter(names = "--topics-pattern", description = "The topic pattern to consume from a list of topics " + @Option(names = "--topics-pattern", description = "The topic pattern to consume from a list of topics " + "under a namespace that matches the pattern. [--input] and [--topics-pattern] are mutually " + "exclusive. Add SerDe class name for a pattern in --custom-serde-inputs (supported for java " + "functions only) #Java, Python") protected String topicsPattern; - @Parameter(names = {"-o", "--output"}, + @Option(names = {"-o", "--output"}, description = "The output topic of a Pulsar Function (If none is specified, no output is written)" + " #Java, Python, Go") protected String output; - @Parameter(names = "--producer-config", description = "The custom producer configuration (as a JSON string)" + @Option(names = "--producer-config", description = "The custom producer configuration (as a JSON string)" + " #Java") protected String producerConfig; // for backwards compatibility purposes - @Parameter(names = "--logTopic", + @Option(names = "--logTopic", description = "The topic to which the logs of a Pulsar Function are produced", hidden = true) protected String deprecatedLogTopic; - @Parameter(names = "--log-topic", description = "The topic to which the logs of a Pulsar Function are produced" + @Option(names = "--log-topic", description = "The topic to which the logs of a Pulsar Function are produced" + " #Java, Python, Go") protected String logTopic; - @Parameter(names = {"-st", "--schema-type"}, description = "The builtin schema type or " + @Option(names = {"-st", "--schema-type"}, description = "The builtin schema type or " + "custom schema class name to be used for messages output by the function #Java") protected String schemaType = ""; // for backwards compatibility purposes - @Parameter(names = "--customSerdeInputs", + @Option(names = "--customSerdeInputs", description = "The map of input topics to SerDe class names (as a JSON string)", hidden = true) protected String deprecatedCustomSerdeInputString; - @Parameter(names = "--custom-serde-inputs", + @Option(names = "--custom-serde-inputs", description = "The map of input topics to SerDe class names (as a JSON string) #Java, Python") protected String customSerdeInputString; - @Parameter(names = "--custom-schema-inputs", + @Option(names = "--custom-schema-inputs", description = "The map of input topics to Schema properties (as a JSON string) #Java, Python") protected String customSchemaInputString; - @Parameter(names = "--custom-schema-outputs", + @Option(names = "--custom-schema-outputs", description = "The map of input topics to Schema properties (as a JSON string) #Java") protected String customSchemaOutputString; - @Parameter(names = "--input-specs", + @Option(names = "--input-specs", description = "The map of inputs to custom configuration (as a JSON string) #Java, Python, Go") protected String inputSpecs; - @Parameter(names = "--input-type-class-name", + @Option(names = "--input-type-class-name", description = "The class name of input type class #Java, Python, Go") protected String inputTypeClassName; // for backwards compatibility purposes - @Parameter(names = "--outputSerdeClassName", + @Option(names = "--outputSerdeClassName", description = "The SerDe class to be used for messages output by the function", hidden = true) protected String deprecatedOutputSerdeClassName; - @Parameter(names = "--output-serde-classname", + @Option(names = "--output-serde-classname", description = "The SerDe class to be used for messages output by the function #Java, Python") protected String outputSerdeClassName; - @Parameter(names = "--output-type-class-name", + @Option(names = "--output-type-class-name", description = "The class name of output type class #Java, Python, Go") protected String outputTypeClassName; // for backwards compatibility purposes - @Parameter(names = "--functionConfigFile", description = "The path to a YAML config file that specifies " + @Option(names = "--functionConfigFile", description = "The path to a YAML config file that specifies " + "the configuration of a Pulsar Function", hidden = true) protected String deprecatedFnConfigFile; - @Parameter(names = "--function-config-file", + @Option(names = "--function-config-file", description = "The path to a YAML config file that specifies the configuration of a Pulsar Function" + " #Java, Python, Go") protected String fnConfigFile; // for backwards compatibility purposes - @Parameter(names = "--processingGuarantees", description = "The processing guarantees (aka delivery semantics) " + @Option(names = "--processingGuarantees", description = "The processing guarantees (aka delivery semantics) " + "applied to the function", hidden = true) protected FunctionConfig.ProcessingGuarantees deprecatedProcessingGuarantees; - @Parameter(names = "--processing-guarantees", + @Option(names = "--processing-guarantees", description = "The processing guarantees (as known as delivery semantics) applied to the function." + " Available values are: `ATLEAST_ONCE`, `ATMOST_ONCE`, `EFFECTIVELY_ONCE`." + " If it is not specified, the `ATLEAST_ONCE` delivery guarantee is used." + " #Java, Python, Go") protected FunctionConfig.ProcessingGuarantees processingGuarantees; // for backwards compatibility purposes - @Parameter(names = "--userConfig", description = "User-defined config key/values", hidden = true) + @Option(names = "--userConfig", description = "User-defined config key/values", hidden = true) protected String deprecatedUserConfigString; - @Parameter(names = "--user-config", description = "User-defined config key/values #Java, Python, Go") + @Option(names = "--user-config", description = "User-defined config key/values #Java, Python, Go") protected String userConfigString; - @Parameter(names = "--retainOrdering", + @Option(names = "--retainOrdering", description = "Function consumes and processes messages in order", hidden = true) protected Boolean deprecatedRetainOrdering; - @Parameter(names = "--retain-ordering", description = "Function consumes and processes messages in order #Java") + @Option(names = "--retain-ordering", description = "Function consumes and processes messages in order #Java") protected Boolean retainOrdering; - @Parameter(names = "--retain-key-ordering", + @Option(names = "--retain-key-ordering", description = "Function consumes and processes messages in key order #Java") protected Boolean retainKeyOrdering; - @Parameter(names = "--batch-builder", description = "BatcherBuilder provides two types of " + @Option(names = "--batch-builder", description = "BatcherBuilder provides two types of " + "batch construction methods, DEFAULT and KEY_BASED. The default value is: DEFAULT") protected String batchBuilder; - @Parameter(names = "--forward-source-message-property", description = "Forwarding input message's properties " - + "to output topic when processing (use false to disable it) #Java", arity = 1) + @Option(names = "--forward-source-message-property", description = "Forwarding input message's properties " + + "to output topic when processing (use false to disable it) #Java", arity = "1") protected Boolean forwardSourceMessageProperty = true; - @Parameter(names = "--subs-name", description = "Pulsar source subscription name if user wants a specific " + @Option(names = "--subs-name", description = "Pulsar source subscription name if user wants a specific " + "subscription-name for input-topic consumer #Java, Python, Go") protected String subsName; - @Parameter(names = "--subs-position", description = "Pulsar source subscription position if user wants to " + @Option(names = "--subs-position", description = "Pulsar source subscription position if user wants to " + "consume messages from the specified location #Java") protected SubscriptionInitialPosition subsPosition; - @Parameter(names = "--skip-to-latest", description = "Whether or not the consumer skip to latest message " - + "upon function instance restart", arity = 1) + @Option(names = "--skip-to-latest", description = "Whether or not the consumer skip to latest message " + + "upon function instance restart", arity = "1") protected Boolean skipToLatest; - @Parameter(names = "--parallelism", description = "The parallelism factor of a Pulsar Function " + @Option(names = "--parallelism", description = "The parallelism factor of a Pulsar Function " + "(i.e. the number of function instances to run) #Java") protected Integer parallelism; - @Parameter(names = "--cpu", description = "The cpu in cores that need to be allocated " + @Option(names = "--cpu", description = "The cpu in cores that need to be allocated " + "per function instance(applicable only to docker runtime) #Java(Process & K8s),Python(K8s),Go(K8s)") protected Double cpu; - @Parameter(names = "--ram", description = "The ram in bytes that need to be allocated " + @Option(names = "--ram", description = "The ram in bytes that need to be allocated " + "per function instance(applicable only to process/docker runtime)" + " #Java(Process & K8s),Python(K8s),Go(K8s)") protected Long ram; - @Parameter(names = "--disk", description = "The disk in bytes that need to be allocated " + @Option(names = "--disk", description = "The disk in bytes that need to be allocated " + "per function instance(applicable only to docker runtime) #Java(Process & K8s),Python(K8s),Go(K8s)") protected Long disk; // for backwards compatibility purposes - @Parameter(names = "--windowLengthCount", description = "The number of messages per window", hidden = true) + @Option(names = "--windowLengthCount", description = "The number of messages per window", hidden = true) protected Integer deprecatedWindowLengthCount; - @Parameter(names = "--window-length-count", description = "The number of messages per window #Java") + @Option(names = "--window-length-count", description = "The number of messages per window #Java") protected Integer windowLengthCount; // for backwards compatibility purposes - @Parameter(names = "--windowLengthDurationMs", + @Option(names = "--windowLengthDurationMs", description = "The time duration of the window in milliseconds", hidden = true) protected Long deprecatedWindowLengthDurationMs; - @Parameter(names = "--window-length-duration-ms", + @Option(names = "--window-length-duration-ms", description = "The time duration of the window in milliseconds #Java") protected Long windowLengthDurationMs; // for backwards compatibility purposes - @Parameter(names = "--slidingIntervalCount", + @Option(names = "--slidingIntervalCount", description = "The number of messages after which the window slides", hidden = true) protected Integer deprecatedSlidingIntervalCount; - @Parameter(names = "--sliding-interval-count", + @Option(names = "--sliding-interval-count", description = "The number of messages after which the window slides #Java") protected Integer slidingIntervalCount; // for backwards compatibility purposes - @Parameter(names = "--slidingIntervalDurationMs", + @Option(names = "--slidingIntervalDurationMs", description = "The time duration after which the window slides", hidden = true) protected Long deprecatedSlidingIntervalDurationMs; - @Parameter(names = "--sliding-interval-duration-ms", + @Option(names = "--sliding-interval-duration-ms", description = "The time duration after which the window slides #Java") protected Long slidingIntervalDurationMs; // for backwards compatibility purposes - @Parameter(names = "--autoAck", + @Option(names = "--autoAck", description = "Whether or not the framework acknowledges messages automatically", hidden = true) protected Boolean deprecatedAutoAck = null; - @Parameter(names = "--auto-ack", + @Option(names = "--auto-ack", description = "Whether or not the framework acknowledges messages automatically" - + " #Java, Python, Go", arity = 1) + + " #Java, Python, Go", arity = "1") protected Boolean autoAck; // for backwards compatibility purposes - @Parameter(names = "--timeoutMs", description = "The message timeout in milliseconds", hidden = true) + @Option(names = "--timeoutMs", description = "The message timeout in milliseconds", hidden = true) protected Long deprecatedTimeoutMs; - @Parameter(names = "--timeout-ms", description = "The message timeout in milliseconds #Java, Python") + @Option(names = "--timeout-ms", description = "The message timeout in milliseconds #Java, Python") protected Long timeoutMs; - @Parameter(names = "--max-message-retries", + @Option(names = "--max-message-retries", description = "How many times should we try to process a message before giving up #Java") protected Integer maxMessageRetries; - @Parameter(names = "--custom-runtime-options", description = "A string that encodes options to " + @Option(names = "--custom-runtime-options", description = "A string that encodes options to " + "customize the runtime, see docs for configured runtime for details #Java") protected String customRuntimeOptions; - @Parameter(names = "--secrets", description = "The map of secretName to an object that encapsulates " + @Option(names = "--secrets", description = "The map of secretName to an object that encapsulates " + "how the secret is fetched by the underlying secrets provider #Java, Python") protected String secretsString; - @Parameter(names = "--dead-letter-topic", + @Option(names = "--dead-letter-topic", description = "The topic where messages that are not processed successfully are sent to #Java") protected String deadLetterTopic; + @Option(names = "--runtime-flags", description = "Any flags that you want to pass to a runtime" + + " (for process & Kubernetes runtime only).") + protected String runtimeFlags; protected FunctionConfig functionConfig; protected String userCodeFile; @@ -449,7 +428,6 @@ private void mergeArgs() { @Override void processArguments() throws Exception { - super.processArguments(); // merge deprecated args with new args mergeArgs(); @@ -461,7 +439,15 @@ void processArguments() throws Exception { } if (null != fqfn) { - parseFullyQualifiedFunctionName(fqfn, functionConfig); + String[] args = fqfn.split("/"); + if (args.length != 3) { + throw new ParameterException("Fully qualified function names (FQFNs) must " + + "be of the form tenant/namespace/name"); + } else { + functionConfig.setTenant(args[0]); + functionConfig.setNamespace(args[1]); + functionConfig.setName(args[2]); + } } else { if (null != tenant) { functionConfig.setTenant(tenant); @@ -687,6 +673,10 @@ void processArguments() throws Exception { userCodeFile = functionConfig.getGo(); } + if (null != runtimeFlags) { + functionConfig.setRuntimeFlags(runtimeFlags); + } + // check if configs are valid validateFunctionConfigs(functionConfig); } @@ -740,73 +730,73 @@ && isBlank(functionConfig.getGo())) { } } - @Parameters(commandDescription = "Run a Pulsar Function locally, rather than deploy to a Pulsar cluster)") + @Command(description = "Run a Pulsar Function locally, rather than deploy to a Pulsar cluster)") class LocalRunner extends FunctionDetailsCommand { // TODO: this should become BookKeeper URL and it should be fetched from Pulsar client. // for backwards compatibility purposes - @Parameter(names = "--stateStorageServiceUrl", description = "The URL for the state storage service " + @Option(names = "--stateStorageServiceUrl", description = "The URL for the state storage service " + "(the default is Apache BookKeeper)", hidden = true) protected String deprecatedStateStorageServiceUrl; - @Parameter(names = "--state-storage-service-url", description = "The URL for the state storage service " + @Option(names = "--state-storage-service-url", description = "The URL for the state storage service " + "(the default is Apache BookKeeper) #Java, Python") protected String stateStorageServiceUrl; // for backwards compatibility purposes - @Parameter(names = "--brokerServiceUrl", description = "The URL for Pulsar broker", hidden = true) + @Option(names = "--brokerServiceUrl", description = "The URL for Pulsar broker", hidden = true) protected String deprecatedBrokerServiceUrl; - @Parameter(names = "--broker-service-url", description = "The URL for Pulsar broker #Java, Python, Go") + @Option(names = "--broker-service-url", description = "The URL for Pulsar broker #Java, Python, Go") protected String brokerServiceUrl; - @Parameter(names = "--web-service-url", description = "The URL for Pulsar web service #Java, Python") + @Option(names = "--web-service-url", description = "The URL for Pulsar web service #Java, Python") protected String webServiceUrl = null; // for backwards compatibility purposes - @Parameter(names = "--clientAuthPlugin", description = "Client authentication plugin using " + @Option(names = "--clientAuthPlugin", description = "Client authentication plugin using " + "which function-process can connect to broker", hidden = true) protected String deprecatedClientAuthPlugin; - @Parameter(names = "--client-auth-plugin", + @Option(names = "--client-auth-plugin", description = "Client authentication plugin using which function-process can connect to broker" + " #Java, Python") protected String clientAuthPlugin; // for backwards compatibility purposes - @Parameter(names = "--clientAuthParams", description = "Client authentication param", hidden = true) + @Option(names = "--clientAuthParams", description = "Client authentication param", hidden = true) protected String deprecatedClientAuthParams; - @Parameter(names = "--client-auth-params", description = "Client authentication param #Java, Python") + @Option(names = "--client-auth-params", description = "Client authentication param #Java, Python") protected String clientAuthParams; // for backwards compatibility purposes - @Parameter(names = "--use_tls", description = "Use tls connection", hidden = true) + @Option(names = "--use_tls", description = "Use tls connection", hidden = true) protected Boolean deprecatedUseTls = null; - @Parameter(names = "--use-tls", description = "Use tls connection #Java, Python") + @Option(names = "--use-tls", description = "Use tls connection #Java, Python") protected boolean useTls; // for backwards compatibility purposes - @Parameter(names = "--tls_allow_insecure", description = "Allow insecure tls connection", hidden = true) + @Option(names = "--tls_allow_insecure", description = "Allow insecure tls connection", hidden = true) protected Boolean deprecatedTlsAllowInsecureConnection = null; - @Parameter(names = "--tls-allow-insecure", description = "Allow insecure tls connection #Java, Python") + @Option(names = "--tls-allow-insecure", description = "Allow insecure tls connection #Java, Python") protected boolean tlsAllowInsecureConnection; // for backwards compatibility purposes - @Parameter(names = "--hostname_verification_enabled", + @Option(names = "--hostname_verification_enabled", description = "Enable hostname verification", hidden = true) protected Boolean deprecatedTlsHostNameVerificationEnabled = null; - @Parameter(names = "--hostname-verification-enabled", description = "Enable hostname verification" + @Option(names = "--hostname-verification-enabled", description = "Enable hostname verification" + " #Java, Python") protected boolean tlsHostNameVerificationEnabled; // for backwards compatibility purposes - @Parameter(names = "--tls_trust_cert_path", description = "tls trust cert file path", hidden = true) + @Option(names = "--tls_trust_cert_path", description = "tls trust cert file path", hidden = true) protected String deprecatedTlsTrustCertFilePath; - @Parameter(names = "--tls-trust-cert-path", description = "tls trust cert file path #Java, Python") + @Option(names = "--tls-trust-cert-path", description = "tls trust cert file path #Java, Python") protected String tlsTrustCertFilePath; // for backwards compatibility purposes - @Parameter(names = "--instanceIdOffset", description = "Start the instanceIds from this offset", hidden = true) + @Option(names = "--instanceIdOffset", description = "Start the instanceIds from this offset", hidden = true) protected Integer deprecatedInstanceIdOffset = null; - @Parameter(names = "--instance-id-offset", description = "Start the instanceIds from this offset #Java, Python") + @Option(names = "--instance-id-offset", description = "Start the instanceIds from this offset #Java, Python") protected Integer instanceIdOffset = 0; - @Parameter(names = "--runtime", description = "either THREAD or PROCESS. Only applies for Java functions #Java") + @Option(names = "--runtime", description = "either THREAD or PROCESS. Only applies for Java functions #Java") protected String runtime; - @Parameter(names = "--secrets-provider-classname", description = "Whats the classname for secrets provider" + @Option(names = "--secrets-provider-classname", description = "Whats the classname for secrets provider" + " #Java, Python") protected String secretsProviderClassName; - @Parameter(names = "--secrets-provider-config", + @Option(names = "--secrets-provider-config", description = "Config that needs to be passed to secrets provider #Java, Python") protected String secretsProviderConfig; - @Parameter(names = "--metrics-port-start", description = "The starting port range for metrics server" + @Option(names = "--metrics-port-start", description = "The starting port range for metrics server" + " #Java, Python, Go") protected String metricsPortStart; @@ -849,7 +839,7 @@ void runCmd() throws Exception { localRunArgs.add("--functionConfig"); localRunArgs.add(new Gson().toJson(functionConfig)); for (Field field : this.getClass().getDeclaredFields()) { - if (field.getName().startsWith("DEPRECATED")) { + if (field.getName().toUpperCase().startsWith("DEPRECATED")) { continue; } if (field.getName().contains("$")) { @@ -867,7 +857,7 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Create a Pulsar Function in cluster mode (deploy it on a Pulsar cluster)") + @Command(description = "Create a Pulsar Function in cluster mode (deploy it on a Pulsar cluster)") class CreateFunction extends FunctionDetailsCommand { @Override void runCmd() throws Exception { @@ -885,7 +875,7 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Fetch information about a Pulsar Function") + @Command(description = "Fetch information about a Pulsar Function") class GetFunction extends FunctionCommand { @Override void runCmd() throws Exception { @@ -895,10 +885,10 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Check the current status of a Pulsar Function") + @Command(aliases = "getstatus", description = "Check the current status of a Pulsar Function") class GetFunctionStatus extends FunctionCommand { - @Parameter(names = "--instance-id", description = "The function instanceId " + @Option(names = "--instance-id", description = "The function instanceId " + "(Get-status of all instances if instance-id is not provided)") protected String instanceId; @@ -913,10 +903,10 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Get the current stats of a Pulsar Function") + @Command(description = "Get the current stats of a Pulsar Function") class GetFunctionStats extends FunctionCommand { - @Parameter(names = "--instance-id", description = "The function instanceId " + @Option(names = "--instance-id", description = "The function instanceId " + "(Get-stats of all instances if instance-id is not provided)") protected String instanceId; @@ -932,10 +922,10 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Restart function instance") + @Command(description = "Restart function instance") class RestartFunction extends FunctionCommand { - @Parameter(names = "--instance-id", description = "The function instanceId " + @Option(names = "--instance-id", description = "The function instanceId " + "(restart all instances if instance-id is not provided)") protected String instanceId; @@ -955,10 +945,10 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Stops function instance") + @Command(description = "Stops function instance") class StopFunction extends FunctionCommand { - @Parameter(names = "--instance-id", description = "The function instanceId " + @Option(names = "--instance-id", description = "The function instanceId " + "(stop all instances if instance-id is not provided)") protected String instanceId; @@ -977,10 +967,10 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Starts a stopped function instance") + @Command(description = "Starts a stopped function instance") class StartFunction extends FunctionCommand { - @Parameter(names = "--instance-id", description = "The function instanceId " + @Option(names = "--instance-id", description = "The function instanceId " + "(start all instances if instance-id is not provided)") protected String instanceId; @@ -999,7 +989,7 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Delete a Pulsar Function that is running on a Pulsar cluster") + @Command(description = "Delete a Pulsar Function that is running on a Pulsar cluster") class DeleteFunction extends FunctionCommand { @Override void runCmd() throws Exception { @@ -1008,10 +998,10 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Update a Pulsar Function that has been deployed to a Pulsar cluster") + @Command(description = "Update a Pulsar Function that has been deployed to a Pulsar cluster") class UpdateFunction extends FunctionDetailsCommand { - @Parameter(names = "--update-auth-data", description = "Whether or not to update the auth data #Java, Python") + @Option(names = "--update-auth-data", description = "Whether or not to update the auth data #Java, Python") protected boolean updateAuthData; @Override @@ -1048,7 +1038,7 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "List all Pulsar Functions running under a specific tenant and namespace") + @Command(description = "List all Pulsar Functions running under a specific tenant and namespace") class ListFunctions extends NamespaceCommand { @Override void runCmd() throws Exception { @@ -1056,13 +1046,13 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Fetch the current state associated with a Pulsar Function") + @Command(description = "Fetch the current state associated with a Pulsar Function") class StateGetter extends FunctionCommand { - @Parameter(names = { "-k", "--key" }, description = "Key name of State") + @Option(names = {"-k", "--key"}, description = "Key name of State") private String key = null; - @Parameter(names = { "-w", "--watch" }, description = "Watch for changes in the value associated with a key " + @Option(names = {"-w", "--watch"}, description = "Watch for changes in the value associated with a key " + "for a Pulsar Function") private boolean watch = false; @@ -1091,10 +1081,10 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Put the state associated with a Pulsar Function") + @Command(description = "Put the state associated with a Pulsar Function") class StatePutter extends FunctionCommand { - @Parameter(names = { "-s", "--state" }, description = "The FunctionState that needs to be put", required = true) + @Option(names = {"-s", "--state"}, description = "The FunctionState that needs to be put", required = true) private String state = null; @Override @@ -1106,22 +1096,22 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Trigger the specified Pulsar Function with a supplied value") + @Command(description = "Trigger the specified Pulsar Function with a supplied value") class TriggerFunction extends FunctionCommand { // for backward compatibility purposes - @Parameter(names = "--triggerValue", + @Option(names = "--triggerValue", description = "The value with which you want to trigger the function", hidden = true) protected String deprecatedTriggerValue; - @Parameter(names = "--trigger-value", description = "The value with which you want to trigger the function") + @Option(names = "--trigger-value", description = "The value with which you want to trigger the function") protected String triggerValue; // for backward compatibility purposes - @Parameter(names = "--triggerFile", description = "The path to the file that contains the data with which " + @Option(names = "--triggerFile", description = "The path to the file that contains the data with which " + "you want to trigger the function", hidden = true) protected String deprecatedTriggerFile; - @Parameter(names = "--trigger-file", description = "The path to the file that contains the data with which " + @Option(names = "--trigger-file", description = "The path to the file that contains the data with which " + "you want to trigger the function") protected String triggerFile; - @Parameter(names = "--topic", description = "The specific topic name that the function consumes from that" + @Option(names = "--topic", description = "The specific topic name that the function consumes from that" + " you want to inject the data to") protected String topic; @@ -1147,23 +1137,20 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Upload File Data to Pulsar", hidden = true) + @Command(description = "Upload File Data to Pulsar") class UploadFunction extends BaseCommand { // for backward compatibility purposes - @Parameter( + @Option( names = "--sourceFile", - description = "The file whose contents need to be uploaded", - listConverter = StringConverter.class, hidden = true) + description = "The file whose contents need to be uploaded", hidden = true) protected String deprecatedSourceFile; - @Parameter( + @Option( names = "--source-file", - description = "The file whose contents need to be uploaded", - listConverter = StringConverter.class) + description = "The file whose contents need to be uploaded") protected String sourceFile; - @Parameter( + @Option( names = "--path", - description = "Path or functionPkgUrl where the contents need to be stored", - listConverter = StringConverter.class, required = true) + description = "Path or functionPkgUrl where the contents need to be stored", required = true) protected String path; private void mergeArgs() { @@ -1184,25 +1171,22 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Download File Data from Pulsar", hidden = true) + @Command(description = "Download File Data from Pulsar") class DownloadFunction extends FunctionCommand { // for backward compatibility purposes - @Parameter( + @Option( names = "--destinationFile", - description = "The file to store downloaded content", - listConverter = StringConverter.class, hidden = true) + description = "The file to store downloaded content", hidden = true) protected String deprecatedDestinationFile; - @Parameter( + @Option( names = "--destination-file", - description = "The file to store downloaded content", - listConverter = StringConverter.class) + description = "The file to store downloaded content") protected String destinationFile; - @Parameter( + @Option( names = "--path", - description = "Path or functionPkgUrl to store the content", - listConverter = StringConverter.class, required = false, hidden = true) + description = "Path or functionPkgUrl to store the content", required = false, hidden = true) protected String path; - @Parameter( + @Option( names = "--transform-function", description = "Download the transform Function of the connector") protected Boolean transformFunction = false; @@ -1237,7 +1221,7 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Reload the available built-in functions") + @Command(description = "Reload the available built-in functions") public class ReloadBuiltInFunctions extends CmdFunctions.BaseCommand { @Override @@ -1264,25 +1248,25 @@ public CmdFunctions(Supplier admin) throws PulsarClientException { restart = new RestartFunction(); stop = new StopFunction(); start = new StartFunction(); - jcommander.addCommand("localrun", getLocalRunner()); - jcommander.addCommand("create", getCreater()); - jcommander.addCommand("delete", getDeleter()); - jcommander.addCommand("update", getUpdater()); - jcommander.addCommand("get", getGetter()); - jcommander.addCommand("restart", getRestarter()); - jcommander.addCommand("stop", getStopper()); - jcommander.addCommand("start", getStarter()); + addCommand("localrun", getLocalRunner()); + addCommand("create", getCreater()); + addCommand("delete", getDeleter()); + addCommand("update", getUpdater()); + addCommand("get", getGetter()); + addCommand("restart", getRestarter()); + addCommand("stop", getStopper()); + addCommand("start", getStarter()); // TODO depecreate getstatus - jcommander.addCommand("status", getStatuser(), "getstatus"); - jcommander.addCommand("stats", getFunctionStats()); - jcommander.addCommand("list", getLister()); - jcommander.addCommand("querystate", getStateGetter()); - jcommander.addCommand("putstate", getStatePutter()); - jcommander.addCommand("trigger", getTriggerer()); - jcommander.addCommand("upload", getUploader()); - jcommander.addCommand("download", getDownloader()); - jcommander.addCommand("reload", new ReloadBuiltInFunctions()); - jcommander.addCommand("available-functions", new ListBuiltInFunctions()); + addCommand("status", getStatuser(), "getstatus"); + addCommand("stats", getFunctionStats()); + addCommand("list", getLister()); + addCommand("querystate", getStateGetter()); + addCommand("putstate", getStatePutter()); + addCommand("trigger", getTriggerer()); + addCommand("upload", getUploader()); + addCommand("download", getDownloader()); + addCommand("reload", new ReloadBuiltInFunctions()); + addCommand("available-functions", new ListBuiltInFunctions()); } @VisibleForTesting @@ -1360,27 +1344,15 @@ StartFunction getStarter() { return start; } - private void parseFullyQualifiedFunctionName(String fqfn, FunctionConfig functionConfig) { - String[] args = fqfn.split("/"); - if (args.length != 3) { - throw new ParameterException("Fully qualified function names (FQFNs) must " - + "be of the form tenant/namespace/name"); - } else { - functionConfig.setTenant(args[0]); - functionConfig.setNamespace(args[1]); - functionConfig.setName(args[2]); - } - } - - @Parameters(commandDescription = "Get the list of Pulsar Functions supported by Pulsar cluster") + @Command(description = "Get the list of Pulsar Functions supported by Pulsar cluster") public class ListBuiltInFunctions extends BaseCommand { @Override void runCmd() throws Exception { getAdmin().functions().getBuiltInFunctions() .forEach(function -> { - System.out.println(function.getName()); - System.out.println(WordUtils.wrap(function.getDescription(), 80)); - System.out.println("----------------------------------------"); + print(function.getName()); + print(WordUtils.wrap(function.getDescription(), 80)); + print("----------------------------------------"); }); } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdGenerateDocument.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdGenerateDocument.java index 764351a4f5e2e..3f728ca73ea69 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdGenerateDocument.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdGenerateDocument.java @@ -18,64 +18,42 @@ */ package org.apache.pulsar.admin.cli; -import com.beust.jcommander.DefaultUsageFormatter; -import com.beust.jcommander.IUsageFormatter; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterDescription; -import com.beust.jcommander.Parameters; +import static org.apache.pulsar.internal.CommandDescriptionUtil.getArgDescription; +import static org.apache.pulsar.internal.CommandDescriptionUtil.getCommandDescription; import java.util.Arrays; +import java.util.HashSet; import java.util.List; -import java.util.Map; -import java.util.Properties; +import java.util.Set; import java.util.function.Supplier; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Model.ArgSpec; +import picocli.CommandLine.Model.CommandSpec; +import picocli.CommandLine.Model.OptionSpec; +import picocli.CommandLine.Parameters; +import picocli.CommandLine.Spec; @Getter -@Parameters(commandDescription = "Generate documents automatically.") +@Command(description = "Generate documents automatically.") @Slf4j public class CmdGenerateDocument extends CmdBase { - private final JCommander baseJcommander; - private final IUsageFormatter usageFormatter; - - private PulsarAdminTool tool; + @Spec + private CommandSpec pulsarAdminCommandSpec; public CmdGenerateDocument(Supplier admin) { super("documents", admin); - baseJcommander = new JCommander(); - usageFormatter = new DefaultUsageFormatter(baseJcommander); - try { - tool = new PulsarAdminTool(new Properties()); - } catch (Exception e) { - System.err.println(e.getMessage()); - System.err.println(); - baseJcommander.usage(); - return; - } - for (Map.Entry> c : tool.commandMap.entrySet()) { - try { - if (!c.getKey().equals("documents") && c.getValue() != null) { - baseJcommander.addCommand( - c.getKey(), c.getValue().getConstructor(Supplier.class).newInstance(admin)); - } - } catch (Exception e) { - System.err.println(e.getMessage()); - System.err.println(); - baseJcommander.usage(); - return; - } - } - jcommander.addCommand("generate", new GenerateDocument()); + addCommand("generate", new GenerateDocument()); } - @Parameters(commandDescription = "Generate document for modules") + @Command(description = "Generate document for modules") private class GenerateDocument extends CliCommand { - @Parameter(description = "Please specify the module name, if not, documents will be generated for all modules." + @Parameters(description = "Please specify the module name, if not, documents will be generated for all modules." + "Optional modules(clusters, tenants, brokers, broker-stats, namespaces, topics, schemas, bookies," + "functions, ns-isolation-policy, resource-quotas, functions, sources, sinks)") private java.util.List modules; @@ -84,13 +62,17 @@ private class GenerateDocument extends CliCommand { void run() throws PulsarAdminException { StringBuilder sb = new StringBuilder(); if (modules == null || modules.isEmpty()) { - baseJcommander.getCommands().forEach((k, v) -> - this.generateDocument(sb, k, v) + pulsarAdminCommandSpec.parent().subcommands().forEach((k, v) -> + this.generateDocument(sb, k, v) ); } else { - String module = getOneArgument(modules); - JCommander obj = baseJcommander.getCommands().get(module); - this.generateDocument(sb, module, obj); + modules.forEach(module -> { + CommandLine commandLine = pulsarAdminCommandSpec.parent().subcommands().get(module); + if (commandLine == null) { + return; + } + this.generateDocument(sb, module, commandLine); + }); } } @@ -99,21 +81,29 @@ private boolean needsLangSupport(String module, String subK) { return module.equals("functions") && Arrays.asList(langSupport).contains(subK); } - private void generateDocument(StringBuilder sb, String module, JCommander obj) { + private final Set generatedModule = new HashSet<>(); + + private void generateDocument(StringBuilder sb, String module, CommandLine obj) { + // Filter the deprecated command + if (generatedModule.contains(module)) { + return; + } + String commandName = obj.getCommandName(); + generatedModule.add(commandName); + sb.append("# ").append(module).append("\n\n"); - sb.append(usageFormatter.getCommandDescription(module)).append("\n"); + sb.append(getCommandDescription(obj)).append("\n"); sb.append("\n\n```shell\n") .append("$ pulsar-admin ").append(module).append(" subcommand") .append("\n```"); sb.append("\n\n"); - CmdBase cmdObj = (CmdBase) obj.getObjects().get(0); - cmdObj.jcommander.getCommands().forEach((subK, subV) -> { - sb.append("\n\n## ").append(subK).append("\n\n"); - sb.append(cmdObj.getUsageFormatter().getCommandDescription(subK)).append("\n\n"); + obj.getSubcommands().forEach((subK, subV) -> { + sb.append("\n\n## ").append(subK).append("\n\n"); + sb.append(getCommandDescription(subV)).append("\n\n"); sb.append("**Command:**\n\n"); sb.append("```shell\n$ pulsar-admin ").append(module).append(" ") .append(subK).append(" options").append("\n```\n\n"); - List options = cmdObj.jcommander.getCommands().get(subK).getParameters(); + List options = obj.getCommandSpec().args(); if (options.size() > 0) { sb.append("**Options:**\n\n"); sb.append("|Flag|Description|Default|"); @@ -124,22 +114,23 @@ private void generateDocument(StringBuilder sb, String module, JCommander obj) { sb.append("\n|---|---|---|\n"); } } - options.stream().filter( - ele -> ele.getParameterAnnotation() == null - || !ele.getParameterAnnotation().hidden() - ).forEach((option) -> { - String[] descriptions = option.getDescription().replace("\n", " ").split(" #"); - sb.append("| `").append(option.getNames()) - .append("` | ").append(descriptions[0]) - .append("|").append(option.getDefault()).append("|"); - if (needsLangSupport(module, subK) && descriptions.length > 1) { - sb.append(descriptions[1]); - } - sb.append("|\n"); - } - ); + options.forEach(ele -> { + if (ele.hidden() || !(ele instanceof OptionSpec)) { + return; + } + + String argDescription = getArgDescription(ele); + String[] descriptions = argDescription.replace("\n", " ").split(" #"); + sb.append("| `").append(Arrays.toString(((OptionSpec) ele).names())) + .append("` | ").append(descriptions[0]) + .append("|").append(ele.defaultValue()).append("|"); + if (needsLangSupport(module, subK) && descriptions.length > 1) { + sb.append(descriptions[1]); + } + sb.append("|\n"); + }); + System.out.println(sb); }); - System.out.println(sb); } } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java index 8de7ef500eaa0..ef36eb417136d 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java @@ -18,17 +18,12 @@ */ package org.apache.pulsar.admin.cli; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; -import com.beust.jcommander.converters.CommaParameterSplitter; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.admin.cli.utils.NameValueParameterSplitter; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.common.policies.data.AutoFailoverPolicyData; @@ -37,63 +32,73 @@ import org.apache.pulsar.common.policies.data.BrokerNamespaceIsolationDataImpl; import org.apache.pulsar.common.policies.data.NamespaceIsolationData; import org.apache.pulsar.common.policies.data.NamespaceIsolationDataImpl; +import org.apache.pulsar.common.policies.data.NamespaceIsolationPolicyUnloadScope; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; -@Parameters(commandDescription = "Operations about namespace isolation policy") +@Command(description = "Operations about namespace isolation policy") public class CmdNamespaceIsolationPolicy extends CmdBase { - @Parameters(commandDescription = "Create/Update a namespace isolation policy for a cluster. " + @Command(description = "Create/Update a namespace isolation policy for a cluster. " + "This operation requires Pulsar super-user privileges") private class SetPolicy extends CliCommand { - @Parameter(description = "cluster-name policy-name", required = true) - private List params; + @Parameters(description = "cluster-name", index = "0", arity = "1") + private String clusterName; + @Parameters(description = "policy-name", index = "1", arity = "1") + private String policyName; - @Parameter(names = "--namespaces", description = "comma separated namespaces-regex list", - required = true, splitter = CommaParameterSplitter.class) + @Option(names = "--namespaces", description = "comma separated namespaces-regex list", + required = true, split = ",") private List namespaces; - @Parameter(names = "--primary", description = "comma separated primary-broker-regex list. " + @Option(names = "--primary", description = "comma separated primary-broker-regex list. " + "In Pulsar, when namespaces (more specifically, namespace bundles) are assigned dynamically to " + "brokers, the namespace isolation policy limits the set of brokers that can be used for assignment. " + "Before topics are assigned to brokers, you can set the namespace isolation policy with a primary or " + "a secondary regex to select desired brokers. If no broker matches the specified regex, you cannot " + "create a topic. If there are not enough primary brokers, topics are assigned to secondary brokers. " + "If there are not enough secondary brokers, topics are assigned to other brokers which do not have " - + "any isolation policies.", required = true, splitter = CommaParameterSplitter.class) + + "any isolation policies.", required = true, split = ",") private List primary; - @Parameter(names = "--secondary", description = "comma separated secondary-broker-regex list", - required = false, splitter = CommaParameterSplitter.class) + @Option(names = "--secondary", description = "comma separated secondary-broker-regex list", + required = false, split = ",") private List secondary = new ArrayList(); // optional - @Parameter(names = "--auto-failover-policy-type", + @Option(names = "--auto-failover-policy-type", description = "auto failover policy type name ['min_available']", required = true) private String autoFailoverPolicyTypeName; - @Parameter(names = "--auto-failover-policy-params", + @Option(names = "--auto-failover-policy-params", description = "comma separated name=value auto failover policy parameters", - required = true, converter = NameValueParameterSplitter.class) + required = true, split = ",") private Map autoFailoverPolicyParams; - void run() throws PulsarAdminException { - String clusterName = getOneArgument(params, 0, 2); - String policyName = getOneArgument(params, 1, 2); + @Option(names = "--unload-scope", description = "configure the type of unload to do -" + + " ['all_matching', 'none', 'changed'] namespaces. By default, only namespaces whose placement will" + + " actually change would be unloaded and placed again. You can choose to not unload any namespace" + + " while setting this new policy by choosing `none` or choose to unload all namespaces matching" + + " old (if any) and new namespace regex. If you chose 'none', you will need to manually unload the" + + " namespaces for them to be placed correctly, or wait till some namespaces get load balanced" + + " automatically based on load shedding configurations.") + private NamespaceIsolationPolicyUnloadScope unloadScope; + void run() throws PulsarAdminException { // validate and create the POJO NamespaceIsolationData namespaceIsolationData = createNamespaceIsolationData(namespaces, primary, secondary, - autoFailoverPolicyTypeName, autoFailoverPolicyParams); + autoFailoverPolicyTypeName, autoFailoverPolicyParams, unloadScope); getAdmin().clusters().createNamespaceIsolationPolicy(clusterName, policyName, namespaceIsolationData); } } - @Parameters(commandDescription = "List all namespace isolation policies of a cluster. " + @Command(description = "List all namespace isolation policies of a cluster. " + "This operation requires Pulsar super-user privileges") private class GetAllPolicies extends CliCommand { - @Parameter(description = "cluster-name", required = true) - private List params; + @Parameters(description = "cluster-name", arity = "1") + private String clusterName; void run() throws PulsarAdminException { - String clusterName = getOneArgument(params); - Map policyMap = getAdmin().clusters().getNamespaceIsolationPolicies(clusterName); @@ -101,15 +106,13 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "List all brokers with namespace-isolation policies attached to it. " + @Command(description = "List all brokers with namespace-isolation policies attached to it. " + "This operation requires Pulsar super-user privileges") private class GetAllBrokersWithPolicies extends CliCommand { - @Parameter(description = "cluster-name", required = true) - private List params; + @Parameters(description = "cluster-name", arity = "1") + private String clusterName; void run() throws PulsarAdminException { - String clusterName = getOneArgument(params); - List brokers = getAdmin().clusters() .getBrokersWithNamespaceIsolationPolicy(clusterName); List data = new ArrayList<>(); @@ -118,19 +121,16 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get broker with namespace-isolation policies attached to it. " + @Command(description = "Get broker with namespace-isolation policies attached to it. " + "This operation requires Pulsar super-user privileges") private class GetBrokerWithPolicies extends CliCommand { - @Parameter(description = "cluster-name", required = true) - private List params; - - @Parameter(names = "--broker", + @Parameters(description = "cluster-name", arity = "1") + private String clusterName; + @Option(names = "--broker", description = "Broker-name to get namespace-isolation policies attached to it", required = true) private String broker; void run() throws PulsarAdminException { - String clusterName = getOneArgument(params); - BrokerNamespaceIsolationDataImpl brokerData = (BrokerNamespaceIsolationDataImpl) getAdmin().clusters() .getBrokerWithNamespaceIsolationPolicy(clusterName, broker); @@ -138,16 +138,15 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get namespace isolation policy of a cluster. " + @Command(description = "Get namespace isolation policy of a cluster. " + "This operation requires Pulsar super-user privileges") private class GetPolicy extends CliCommand { - @Parameter(description = "cluster-name policy-name", required = true) - private List params; + @Parameters(description = "cluster-name", index = "0", arity = "1") + private String clusterName; + @Parameters(description = "policy-name", index = "1", arity = "1") + private String policyName; void run() throws PulsarAdminException { - String clusterName = getOneArgument(params, 0, 2); - String policyName = getOneArgument(params, 1, 2); - NamespaceIsolationDataImpl nsIsolationData = (NamespaceIsolationDataImpl) getAdmin().clusters() .getNamespaceIsolationPolicy(clusterName, policyName); @@ -155,16 +154,15 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Delete namespace isolation policy of a cluster. " + @Command(description = "Delete namespace isolation policy of a cluster. " + "This operation requires Pulsar super-user privileges") private class DeletePolicy extends CliCommand { - @Parameter(description = "cluster-name policy-name", required = true) - private List params; + @Parameters(description = "cluster-name", index = "0", arity = "1") + private String clusterName; + @Parameters(description = "policy-name", index = "1", arity = "1") + private String policyName; void run() throws PulsarAdminException { - String clusterName = getOneArgument(params, 0, 2); - String policyName = getOneArgument(params, 1, 2); - getAdmin().clusters().deleteNamespaceIsolationPolicy(clusterName, policyName); } } @@ -179,7 +177,8 @@ private NamespaceIsolationData createNamespaceIsolationData(List namespa List primary, List secondary, String autoFailoverPolicyTypeName, - Map autoFailoverPolicyParams) { + Map autoFailoverPolicyParams, + NamespaceIsolationPolicyUnloadScope unload) { // validate namespaces = validateList(namespaces); @@ -246,17 +245,19 @@ private NamespaceIsolationData createNamespaceIsolationData(List namespa throw new ParameterException("Unknown auto failover policy type specified : " + autoFailoverPolicyTypeName); } + nsIsolationDataBuilder.unloadScope(unload); + return nsIsolationDataBuilder.build(); } public CmdNamespaceIsolationPolicy(Supplier admin) { super("ns-isolation-policy", admin); - jcommander.addCommand("set", new SetPolicy()); - jcommander.addCommand("get", new GetPolicy()); - jcommander.addCommand("list", new GetAllPolicies()); - jcommander.addCommand("delete", new DeletePolicy()); - jcommander.addCommand("brokers", new GetAllBrokersWithPolicies()); - jcommander.addCommand("broker", new GetBrokerWithPolicies()); + addCommand("set", new SetPolicy()); + addCommand("get", new GetPolicy()); + addCommand("list", new GetAllPolicies()); + addCommand("delete", new DeletePolicy()); + addCommand("brokers", new GetAllBrokersWithPolicies()); + addCommand("broker", new GetBrokerWithPolicies()); } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 998591f8177d1..e8e644b688029 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -18,14 +18,9 @@ */ package org.apache.pulsar.admin.cli; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; -import com.beust.jcommander.converters.CommaParameterSplitter; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import io.swagger.util.Json; import java.io.IOException; import java.util.Arrays; import java.util.HashSet; @@ -37,6 +32,10 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.admin.cli.utils.IOUtils; +import org.apache.pulsar.cli.converters.picocli.ByteUnitToIntegerConverter; +import org.apache.pulsar.cli.converters.picocli.ByteUnitToLongConverter; +import org.apache.pulsar.cli.converters.picocli.TimeUnitToMillisConverter; +import org.apache.pulsar.cli.converters.picocli.TimeUnitToSecondsConverter; import org.apache.pulsar.client.admin.ListNamespaceTopicsOptions; import org.apache.pulsar.client.admin.Mode; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -66,26 +65,27 @@ import org.apache.pulsar.common.policies.data.SubscribeRate; import org.apache.pulsar.common.policies.data.SubscriptionAuthMode; import org.apache.pulsar.common.policies.data.TopicType; -import org.apache.pulsar.common.util.RelativeTimeUtil; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; -@Parameters(commandDescription = "Operations about namespaces") +@Command(description = "Operations about namespaces") public class CmdNamespaces extends CmdBase { - @Parameters(commandDescription = "Get the namespaces for a tenant") + @Command(description = "Get the namespaces for a tenant") private class GetNamespacesPerProperty extends CliCommand { - @Parameter(description = "tenant-name", required = true) - private java.util.List params; + @Parameters(description = "tenant-name", arity = "1") + private String tenant; @Override void run() throws PulsarAdminException { - String tenant = getOneArgument(params); print(getAdmin().namespaces().getNamespaces(tenant)); } } - @Parameters(commandDescription = "Get the namespaces for a tenant in a cluster", hidden = true) + @Command(description = "Get the namespaces for a tenant in a cluster", hidden = true) private class GetNamespacesPerCluster extends CliCommand { - @Parameter(description = "tenant/cluster", required = true) - private java.util.List params; + @Parameters(description = "tenant/cluster", arity = "1") + private String params; @Override void run() throws PulsarAdminException { @@ -94,22 +94,22 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get the list of topics for a namespace") + @Command(description = "Get the list of topics for a namespace") private class GetTopics extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = {"-m", "--mode"}, + @Option(names = {"-m", "--mode"}, description = "Allowed topic domain mode (persistent, non_persistent, all).") private Mode mode; - @Parameter(names = { "-ist", + @Option(names = { "-ist", "--include-system-topic" }, description = "Include system topic") private boolean includeSystemTopic; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); ListNamespaceTopicsOptions options = ListNamespaceTopicsOptions.builder() .mode(mode) .includeSystemTopic(includeSystemTopic) @@ -118,59 +118,59 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get the list of bundles for a namespace") + @Command(description = "Get the list of bundles for a namespace") private class GetBundles extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getBundles(namespace)); } } - @Parameters(commandDescription = "Get the list of destinations for a namespace", hidden = true) + @Command(description = "Get the list of destinations for a namespace", hidden = true) private class GetDestinations extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getTopics(namespace)); } } - @Parameters(commandDescription = "Get the configuration policies of a namespace") + @Command(description = "Get the configuration policies of a namespace") private class GetPolicies extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getPolicies(namespace)); } } - @Parameters(commandDescription = "Creates a new namespace") + @Command(description = "Creates a new namespace") private class Create extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--clusters", "-c" }, - description = "List of clusters this namespace will be assigned", required = false) + @Option(names = { "--clusters", "-c" }, + description = "List of clusters this namespace will be assigned", required = false, split = ",") private java.util.List clusters; - @Parameter(names = { "--bundles", "-b" }, description = "number of bundles to activate", required = false) + @Option(names = { "--bundles", "-b" }, description = "number of bundles to activate", required = false) private int numBundles = 0; private static final long MAX_BUNDLES = ((long) 1) << 32; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); if (numBundles < 0 || numBundles > MAX_BUNDLES) { throw new ParameterException( "Invalid number of bundles. Number of bundles has to be in the range of (0, 2^32]."); @@ -201,162 +201,162 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Deletes a namespace.") + @Command(description = "Deletes a namespace.") private class Delete extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "-f", + @Option(names = { "-f", "--force" }, description = "Delete namespace forcefully by force deleting all topics under it") private boolean force = false; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().deleteNamespace(namespace, force); } } - @Parameters(commandDescription = "Grant permissions on a namespace") + @Command(description = "Grant permissions on a namespace") private class GrantPermissions extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = "--role", description = "Client role to which grant permissions", required = true) + @Option(names = "--role", description = "Client role to which grant permissions", required = true) private String role; - @Parameter(names = "--actions", description = "Actions to be granted (produce,consume,sources,sinks," - + "functions,packages)", required = true) + @Option(names = "--actions", description = "Actions to be granted (produce,consume,sources,sinks," + + "functions,packages)", required = true, split = ",") private List actions; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().grantPermissionOnNamespace(namespace, role, getAuthActions(actions)); } } - @Parameters(commandDescription = "Revoke permissions on a namespace") + @Command(description = "Revoke permissions on a namespace") private class RevokePermissions extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = "--role", description = "Client role to which revoke permissions", required = true) + @Option(names = "--role", description = "Client role to which revoke permissions", required = true) private String role; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().revokePermissionsOnNamespace(namespace, role); } } - @Parameters(commandDescription = "Get permissions to access subscription admin-api") + @Command(description = "Get permissions to access subscription admin-api") private class SubscriptionPermissions extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getPermissionOnSubscription(namespace)); } } - @Parameters(commandDescription = "Grant permissions to access subscription admin-api") + @Command(description = "Grant permissions to access subscription admin-api") private class GrantSubscriptionPermissions extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = {"-s", "--subscription"}, + @Option(names = {"-s", "--subscription"}, description = "Subscription name for which permission will be granted to roles", required = true) private String subscription; - @Parameter(names = {"-rs", "--roles"}, + @Option(names = {"-rs", "--roles"}, description = "Client roles to which grant permissions (comma separated roles)", - required = true, splitter = CommaParameterSplitter.class) + required = true, split = ",") private List roles; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().grantPermissionOnSubscription(namespace, subscription, Sets.newHashSet(roles)); } } - @Parameters(commandDescription = "Revoke permissions to access subscription admin-api") + @Command(description = "Revoke permissions to access subscription admin-api") private class RevokeSubscriptionPermissions extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = {"-s", "--subscription"}, description = "Subscription name for which permission " + @Option(names = {"-s", "--subscription"}, description = "Subscription name for which permission " + "will be revoked to roles", required = true) private String subscription; - @Parameter(names = {"-r", "--role"}, + @Option(names = {"-r", "--role"}, description = "Client role to which revoke permissions", required = true) private String role; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().revokePermissionOnSubscription(namespace, subscription, role); } } - @Parameters(commandDescription = "Get the permissions on a namespace") + @Command(description = "Get the permissions on a namespace") private class Permissions extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getPermissions(namespace)); } } - @Parameters(commandDescription = "Set replication clusters for a namespace") + @Command(description = "Set replication clusters for a namespace") private class SetReplicationClusters extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--clusters", + @Option(names = { "--clusters", "-c" }, description = "Replication Cluster Ids list (comma separated values)", required = true) private String clusterIds; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); List clusters = Lists.newArrayList(clusterIds.split(",")); getAdmin().namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet(clusters)); } } - @Parameters(commandDescription = "Get replication clusters for a namespace") + @Command(description = "Get replication clusters for a namespace") private class GetReplicationClusters extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getNamespaceReplicationClusters(namespace)); } } - @Parameters(commandDescription = "Set subscription types enabled for a namespace") + @Command(description = "Set subscription types enabled for a namespace") private class SetSubscriptionTypesEnabled extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = {"--types", "-t"}, description = "Subscription types enabled list (comma separated values)." - + " Possible values: (Exclusive, Shared, Failover, Key_Shared).", required = true) + @Option(names = {"--types", "-t"}, description = "Subscription types enabled list (comma separated values)." + + " Possible values: (Exclusive, Shared, Failover, Key_Shared).", required = true, split = ",") private List subTypes; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); Set types = new HashSet<>(); subTypes.forEach(s -> { SubscriptionType subType; @@ -372,177 +372,166 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get subscription types enabled for a namespace") + @Command(description = "Get subscription types enabled for a namespace") private class GetSubscriptionTypesEnabled extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getSubscriptionTypesEnabled(namespace)); } } - @Parameters(commandDescription = "Remove subscription types enabled for a namespace") + @Command(description = "Remove subscription types enabled for a namespace") private class RemoveSubscriptionTypesEnabled extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeSubscriptionTypesEnabled(namespace); } } - @Parameters(commandDescription = "Set Message TTL for a namespace") + @Command(description = "Set Message TTL for a namespace") private class SetMessageTTL extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--messageTTL", "-ttl" }, + @Option(names = {"--messageTTL", "-ttl"}, description = "Message TTL in seconds (or minutes, hours, days, weeks eg: 100m, 3h, 2d, 5w). " - + "When the value is set to `0`, TTL is disabled.", required = true) - private String messageTTLStr; + + "When the value is set to `0`, TTL is disabled.", required = true, + converter = TimeUnitToSecondsConverter.class) + private Long messageTTLInSecond; @Override void run() throws PulsarAdminException { - long messageTTLInSecond; - try { - messageTTLInSecond = RelativeTimeUtil.parseRelativeTimeInSeconds(messageTTLStr); - } catch (IllegalArgumentException e) { - throw new ParameterException(e.getMessage()); - } - - if (messageTTLInSecond < 0 || messageTTLInSecond > Integer.MAX_VALUE) { - throw new ParameterException( - String.format("Message TTL cannot be negative or greater than %d seconds", Integer.MAX_VALUE)); - } - - String namespace = validateNamespace(params); - getAdmin().namespaces().setNamespaceMessageTTL(namespace, (int) messageTTLInSecond); + String namespace = validateNamespace(namespaceName); + getAdmin().namespaces().setNamespaceMessageTTL(namespace, messageTTLInSecond.intValue()); } } - @Parameters(commandDescription = "Remove Message TTL for a namespace") + @Command(description = "Remove Message TTL for a namespace") private class RemoveMessageTTL extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeNamespaceMessageTTL(namespace); } } - @Parameters(commandDescription = "Get max subscriptions per topic for a namespace") + @Command(description = "Get max subscriptions per topic for a namespace") private class GetMaxSubscriptionsPerTopic extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getMaxSubscriptionsPerTopic(namespace)); } } - @Parameters(commandDescription = "Set max subscriptions per topic for a namespace") + @Command(description = "Set max subscriptions per topic for a namespace") private class SetMaxSubscriptionsPerTopic extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--max-subscriptions-per-topic", "-m" }, description = "Max subscriptions per topic", + @Option(names = { "--max-subscriptions-per-topic", "-m" }, description = "Max subscriptions per topic", required = true) private int maxSubscriptionsPerTopic; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setMaxSubscriptionsPerTopic(namespace, maxSubscriptionsPerTopic); } } - @Parameters(commandDescription = "Remove max subscriptions per topic for a namespace") + @Command(description = "Remove max subscriptions per topic for a namespace") private class RemoveMaxSubscriptionsPerTopic extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeMaxSubscriptionsPerTopic(namespace); } } - @Parameters(commandDescription = "Set subscription expiration time for a namespace") + @Command(description = "Set subscription expiration time for a namespace") private class SetSubscriptionExpirationTime extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "-t", "--time" }, description = "Subscription expiration time in minutes", required = true) + @Option(names = { "-t", "--time" }, description = "Subscription expiration time in minutes", required = true) private int expirationTime; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setSubscriptionExpirationTime(namespace, expirationTime); } } - @Parameters(commandDescription = "Remove subscription expiration time for a namespace") + @Command(description = "Remove subscription expiration time for a namespace") private class RemoveSubscriptionExpirationTime extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeSubscriptionExpirationTime(namespace); } } - @Parameters(commandDescription = "Set Anti-affinity group name for a namespace") + @Command(description = "Set Anti-affinity group name for a namespace") private class SetAntiAffinityGroup extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--group", "-g" }, description = "Anti-affinity group name", required = true) + @Option(names = { "--group", "-g" }, description = "Anti-affinity group name", required = true) private String antiAffinityGroup; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setNamespaceAntiAffinityGroup(namespace, antiAffinityGroup); } } - @Parameters(commandDescription = "Get Anti-affinity group name for a namespace") + @Command(description = "Get Anti-affinity group name for a namespace") private class GetAntiAffinityGroup extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getNamespaceAntiAffinityGroup(namespace)); } } - @Parameters(commandDescription = "Get Anti-affinity namespaces grouped with the given anti-affinity group name") + @Command(description = "Get Anti-affinity namespaces grouped with the given anti-affinity group name") private class GetAntiAffinityNamespaces extends CliCommand { - @Parameter(names = { "--tenant", + @Option(names = { "--tenant", "-p" }, description = "tenant is only used for authorization. " + "Client has to be admin of any of the tenant to access this api", required = false) private String tenant; - @Parameter(names = { "--cluster", "-c" }, description = "Cluster name", required = true) + @Option(names = { "--cluster", "-c" }, description = "Cluster name", required = true) private String cluster; - @Parameter(names = { "--group", "-g" }, description = "Anti-affinity group name", required = true) + @Option(names = { "--group", "-g" }, description = "Anti-affinity group name", required = true) private String antiAffinityGroup; @Override @@ -551,56 +540,56 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Remove Anti-affinity group name for a namespace") + @Command(description = "Remove Anti-affinity group name for a namespace") private class DeleteAntiAffinityGroup extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().deleteNamespaceAntiAffinityGroup(namespace); } } - @Parameters(commandDescription = "Get Deduplication for a namespace") + @Command(description = "Get Deduplication for a namespace") private class GetDeduplication extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getDeduplicationStatus(namespace)); } } - @Parameters(commandDescription = "Remove Deduplication for a namespace") + @Command(description = "Remove Deduplication for a namespace") private class RemoveDeduplication extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeDeduplicationStatus(namespace); } } - @Parameters(commandDescription = "Enable or disable deduplication for a namespace") + @Command(description = "Enable or disable deduplication for a namespace") private class SetDeduplication extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--enable", "-e" }, description = "Enable deduplication") + @Option(names = { "--enable", "-e" }, description = "Enable deduplication") private boolean enable = false; - @Parameter(names = { "--disable", "-d" }, description = "Disable deduplication") + @Option(names = { "--disable", "-d" }, description = "Disable deduplication") private boolean disable = false; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); if (enable == disable) { throw new ParameterException("Need to specify either --enable or --disable"); @@ -609,28 +598,28 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Enable or disable autoTopicCreation for a namespace, overriding broker settings") + @Command(description = "Enable or disable autoTopicCreation for a namespace, overriding broker settings") private class SetAutoTopicCreation extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--enable", "-e" }, description = "Enable allowAutoTopicCreation on namespace") + @Option(names = { "--enable", "-e" }, description = "Enable allowAutoTopicCreation on namespace") private boolean enable = false; - @Parameter(names = { "--disable", "-d" }, description = "Disable allowAutoTopicCreation on namespace") + @Option(names = { "--disable", "-d" }, description = "Disable allowAutoTopicCreation on namespace") private boolean disable = false; - @Parameter(names = { "--type", "-t" }, description = "Type of topic to be auto-created. " + @Option(names = { "--type", "-t" }, description = "Type of topic to be auto-created. " + "Possible values: (partitioned, non-partitioned). Default value: non-partitioned") private String type = "non-partitioned"; - @Parameter(names = { "--num-partitions", "-n" }, description = "Default number of partitions of topic to " + @Option(names = { "--num-partitions", "-n" }, description = "Default number of partitions of topic to " + "be auto-created, applicable to partitioned topics only", required = false) private Integer defaultNumPartitions = null; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); type = type.toLowerCase().trim(); if (enable == disable) { @@ -657,42 +646,42 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get autoTopicCreation info for a namespace") + @Command(description = "Get autoTopicCreation info for a namespace") private class GetAutoTopicCreation extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getAutoTopicCreation(namespace)); } } - @Parameters(commandDescription = "Remove override of autoTopicCreation for a namespace") + @Command(description = "Remove override of autoTopicCreation for a namespace") private class RemoveAutoTopicCreation extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeAutoTopicCreation(namespace); } } - @Parameters(commandDescription = "Enable autoSubscriptionCreation for a namespace, overriding broker settings") + @Command(description = "Enable autoSubscriptionCreation for a namespace, overriding broker settings") private class SetAutoSubscriptionCreation extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--enable", "-e" }, description = "Enable allowAutoSubscriptionCreation on namespace") + @Option(names = {"--enable", "-e"}, description = "Enable allowAutoSubscriptionCreation on namespace") private boolean enable = false; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setAutoSubscriptionCreation(namespace, AutoSubscriptionCreationOverride.builder() .allowAutoSubscriptionCreation(enable) @@ -700,112 +689,99 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get the autoSubscriptionCreation for a namespace") + @Command(description = "Get the autoSubscriptionCreation for a namespace") private class GetAutoSubscriptionCreation extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getAutoSubscriptionCreation(namespace)); } } - @Parameters(commandDescription = "Remove override of autoSubscriptionCreation for a namespace") + @Command(description = "Remove override of autoSubscriptionCreation for a namespace") private class RemoveAutoSubscriptionCreation extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeAutoSubscriptionCreation(namespace); } } - @Parameters(commandDescription = "Remove the retention policy for a namespace") + @Command(description = "Remove the retention policy for a namespace") private class RemoveRetention extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeRetention(namespace); } } - @Parameters(commandDescription = "Set the retention policy for a namespace") + @Command(description = "Set the retention policy for a namespace") private class SetRetention extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--time", + @Option(names = { "--time", "-t" }, description = "Retention time with optional time unit suffix. " + "For example, 100m, 3h, 2d, 5w. " + "If the time unit is not specified, the default unit is seconds. For example, " + "-t 120 sets retention to 2 minutes. " - + "0 means no retention and -1 means infinite time retention.", required = true) - private String retentionTimeStr; + + "0 means no retention and -1 means infinite time retention.", required = true, + converter = TimeUnitToSecondsConverter.class) + private Long retentionTimeInSec; - @Parameter(names = { "--size", "-s" }, description = "Retention size limit with optional size unit suffix. " + @Option(names = { "--size", "-s" }, description = "Retention size limit with optional size unit suffix. " + "For example, 4096, 10M, 16G, 3T. The size unit suffix character can be k/K, m/M, g/G, or t/T. " + "If the size unit suffix is not specified, the default unit is bytes. " - + "0 or less than 1MB means no retention and -1 means infinite size retention", required = true) - private String limitStr; + + "0 or less than 1MB means no retention and -1 means infinite size retention", required = true, + converter = ByteUnitToLongConverter.class) + private Long sizeLimit; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); - long sizeLimit = validateSizeString(limitStr); - long retentionTimeInSec; - try { - retentionTimeInSec = RelativeTimeUtil.parseRelativeTimeInSeconds(retentionTimeStr); - } catch (IllegalArgumentException exception) { - throw new ParameterException(exception.getMessage()); - } - - final int retentionTimeInMin; - if (retentionTimeInSec != -1) { - retentionTimeInMin = (int) TimeUnit.SECONDS.toMinutes(retentionTimeInSec); - } else { - retentionTimeInMin = -1; - } - - final int retentionSizeInMB; - if (sizeLimit != -1) { - retentionSizeInMB = (int) (sizeLimit / (1024 * 1024)); - } else { - retentionSizeInMB = -1; - } + String namespace = validateNamespace(namespaceName); + final int retentionTimeInMin = retentionTimeInSec != -1 + ? (int) TimeUnit.SECONDS.toMinutes(retentionTimeInSec) + : retentionTimeInSec.intValue(); + final long retentionSizeInMB = sizeLimit != -1 + ? (sizeLimit / (1024 * 1024)) + : sizeLimit; getAdmin().namespaces() .setRetention(namespace, new RetentionPolicies(retentionTimeInMin, retentionSizeInMB)); } } - @Parameters(commandDescription = "Get the retention policy for a namespace") + @Command(description = "Get the retention policy for a namespace") private class GetRetention extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getRetention(namespace)); } } - @Parameters(commandDescription = "Set the bookie-affinity group name") + @Command(description = "Set the bookie-affinity group name") private class SetBookieAffinityGroup extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--primary-group", + @Option(names = { "--primary-group", "-pg" }, description = "Bookie-affinity primary-groups (comma separated) name " + "where namespace messages should be written", required = true) private String bookieAffinityGroupNamePrimary; - @Parameter(names = { "--secondary-group", + @Option(names = { "--secondary-group", "-sg" }, description = "Bookie-affinity secondary-group (comma separated) name where namespace " + "messages should be written. If you want to verify whether there are enough bookies in groups, " + "use `--secondary-group` flag. Messages in this namespace are stored in secondary groups. " @@ -815,7 +791,7 @@ private class SetBookieAffinityGroup extends CliCommand { @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setBookieAffinityGroup(namespace, BookieAffinityGroupData.builder() .bookkeeperAffinityGroupPrimary(bookieAffinityGroupNamePrimary) @@ -824,70 +800,76 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Set the bookie-affinity group name") + @Command(description = "Set the bookie-affinity group name") private class DeleteBookieAffinityGroup extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().deleteBookieAffinityGroup(namespace); } } - @Parameters(commandDescription = "Get the bookie-affinity group name") + @Command(description = "Get the bookie-affinity group name") private class GetBookieAffinityGroup extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getBookieAffinityGroup(namespace)); } } - @Parameters(commandDescription = "Get message TTL for a namespace") + @Command(description = "Get message TTL for a namespace") private class GetMessageTTL extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getNamespaceMessageTTL(namespace)); } } - @Parameters(commandDescription = "Get subscription expiration time for a namespace") + @Command(description = "Get subscription expiration time for a namespace") private class GetSubscriptionExpirationTime extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getSubscriptionExpirationTime(namespace)); } } - @Parameters(commandDescription = "Unload a namespace from the current serving broker") + @Command(description = "Unload a namespace from the current serving broker") private class Unload extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--bundle", "-b" }, description = "{start-boundary}_{end-boundary}") + @Option(names = { "--bundle", "-b" }, description = "{start-boundary}_{end-boundary}") private String bundle; - @Parameter(names = { "--destinationBroker", "-d" }, - description = "Target brokerWebServiceAddress to which the bundle has to be allocated to") + @Option(names = { "--destinationBroker", "-d" }, + description = "Target brokerWebServiceAddress to which the bundle has to be allocated to. " + + "--destinationBroker cannot be set when --bundle is not specified.") private String destinationBroker; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); + + if (bundle == null) { + if (StringUtils.isNotBlank(destinationBroker)) { + throw new ParameterException("--destinationBroker cannot be set when --bundle is not specified."); + } getAdmin().namespaces().unload(namespace); } else { getAdmin().namespaces().unloadNamespaceBundle(namespace, bundle, destinationBroker); @@ -895,38 +877,38 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Split a namespace-bundle from the current serving broker") + @Command(description = "Split a namespace-bundle from the current serving broker") private class SplitBundle extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--bundle", + @Option(names = { "--bundle", "-b" }, description = "{start-boundary}_{end-boundary} " + "(mutually exclusive with --bundle-type)", required = false) private String bundle; - @Parameter(names = { "--bundle-type", + @Option(names = { "--bundle-type", "-bt" }, description = "bundle type (mutually exclusive with --bundle)", required = false) private BundleType bundleType; - @Parameter(names = { "--unload", + @Option(names = { "--unload", "-u" }, description = "Unload newly split bundles after splitting old bundle", required = false) private boolean unload; - @Parameter(names = { "--split-algorithm-name", "-san" }, description = "Algorithm name for split " + @Option(names = { "--split-algorithm-name", "-san" }, description = "Algorithm name for split " + "namespace bundle. Valid options are: [range_equally_divide, topic_count_equally_divide, " + "specified_positions_divide, flow_or_qps_equally_divide]. Use broker side config if absent" , required = false) private String splitAlgorithmName; - @Parameter(names = { "--split-boundaries", + @Option(names = { "--split-boundaries", "-sb" }, description = "Specified split boundary for bundle split, will split one bundle " + "to multi bundles only works with specified_positions_divide algorithm", required = false) private List splitBoundaries; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); if (StringUtils.isBlank(bundle) && bundleType == null) { throw new ParameterException("Must pass one of the params: --bundle / --bundle-type"); } @@ -944,18 +926,18 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get the positions for one or more topic(s) in a namespace bundle") + @Command(description = "Get the positions for one or more topic(s) in a namespace bundle") private class GetTopicHashPositions extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter( + @Option( names = { "--bundle", "-b" }, description = "{start-boundary}_{end-boundary} format namespace bundle", required = false) private String bundle; - @Parameter( + @Option( names = { "--topic-list", "-tl" }, description = "The list of topics(both non-partitioned topic and partitioned topic) to get positions " + "in this bundle, if none topic provided, will get the positions of all topics in this bundle", @@ -964,7 +946,7 @@ private class GetTopicHashPositions extends CliCommand { @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); if (StringUtils.isBlank(bundle)) { throw new ParameterException("Must pass one of the params: --bundle "); } @@ -972,34 +954,34 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Set message-dispatch-rate for all topics of the namespace") + @Command(description = "Set message-dispatch-rate for all topics of the namespace") private class SetDispatchRate extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--msg-dispatch-rate", + @Option(names = { "--msg-dispatch-rate", "-md" }, description = "message-dispatch-rate " + "(default -1 will be overwrite if not passed)", required = false) private int msgDispatchRate = -1; - @Parameter(names = { "--byte-dispatch-rate", + @Option(names = { "--byte-dispatch-rate", "-bd" }, description = "byte-dispatch-rate " + "(default -1 will be overwrite if not passed)", required = false) private long byteDispatchRate = -1; - @Parameter(names = { "--dispatch-rate-period", + @Option(names = { "--dispatch-rate-period", "-dt" }, description = "dispatch-rate-period in second type " + "(default 1 second will be overwrite if not passed)", required = false) private int dispatchRatePeriodSec = 1; - @Parameter(names = { "--relative-to-publish-rate", + @Option(names = { "--relative-to-publish-rate", "-rp" }, description = "dispatch rate relative to publish-rate (if publish-relative flag is enabled " + "then broker will apply throttling value to (publish-rate + dispatch rate))", required = false) private boolean relativeToPublishRate = false; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setDispatchRate(namespace, DispatchRate.builder() .dispatchThrottlingRateInMsg(msgDispatchRate) @@ -1010,106 +992,106 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Remove configured message-dispatch-rate for all topics of the namespace") + @Command(description = "Remove configured message-dispatch-rate for all topics of the namespace") private class RemoveDispatchRate extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeDispatchRate(namespace); } } - @Parameters(commandDescription = "Get configured message-dispatch-rate for all topics of the namespace " + @Command(description = "Get configured message-dispatch-rate for all topics of the namespace " + "(Disabled if value < 0)") private class GetDispatchRate extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getDispatchRate(namespace)); } } - @Parameters(commandDescription = "Set subscribe-rate per consumer for all topics of the namespace") + @Command(description = "Set subscribe-rate per consumer for all topics of the namespace") private class SetSubscribeRate extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--subscribe-rate", + @Option(names = { "--subscribe-rate", "-sr" }, description = "subscribe-rate (default -1 will be overwrite if not passed)", required = false) private int subscribeRate = -1; - @Parameter(names = { "--subscribe-rate-period", + @Option(names = { "--subscribe-rate-period", "-st" }, description = "subscribe-rate-period in second type " + "(default 30 second will be overwrite if not passed)", required = false) private int subscribeRatePeriodSec = 30; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setSubscribeRate(namespace, new SubscribeRate(subscribeRate, subscribeRatePeriodSec)); } } - @Parameters(commandDescription = "Get configured subscribe-rate per consumer for all topics of the namespace") + @Command(description = "Get configured subscribe-rate per consumer for all topics of the namespace") private class GetSubscribeRate extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getSubscribeRate(namespace)); } } - @Parameters(commandDescription = "Remove configured subscribe-rate per consumer for all topics of the namespace") + @Command(description = "Remove configured subscribe-rate per consumer for all topics of the namespace") private class RemoveSubscribeRate extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeSubscribeRate(namespace); } } - @Parameters(commandDescription = "Set subscription message-dispatch-rate for all subscription of the namespace") + @Command(description = "Set subscription message-dispatch-rate for all subscription of the namespace") private class SetSubscriptionDispatchRate extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--msg-dispatch-rate", + @Option(names = { "--msg-dispatch-rate", "-md" }, description = "message-dispatch-rate " + "(default -1 will be overwrite if not passed)", required = false) private int msgDispatchRate = -1; - @Parameter(names = { "--byte-dispatch-rate", + @Option(names = { "--byte-dispatch-rate", "-bd" }, description = "byte-dispatch-rate (default -1 will be overwrite if not passed)", required = false) private long byteDispatchRate = -1; - @Parameter(names = { "--dispatch-rate-period", + @Option(names = { "--dispatch-rate-period", "-dt" }, description = "dispatch-rate-period in second type " + "(default 1 second will be overwrite if not passed)", required = false) private int dispatchRatePeriodSec = 1; - @Parameter(names = { "--relative-to-publish-rate", + @Option(names = { "--relative-to-publish-rate", "-rp" }, description = "dispatch rate relative to publish-rate (if publish-relative flag is enabled " + "then broker will apply throttling value to (publish-rate + dispatch rate))", required = false) private boolean relativeToPublishRate = false; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setSubscriptionDispatchRate(namespace, DispatchRate.builder() .dispatchThrottlingRateInMsg(msgDispatchRate) @@ -1120,100 +1102,100 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Remove subscription configured message-dispatch-rate " + @Command(description = "Remove subscription configured message-dispatch-rate " + "for all topics of the namespace") private class RemoveSubscriptionDispatchRate extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeSubscriptionDispatchRate(namespace); } } - @Parameters(commandDescription = "Get subscription configured message-dispatch-rate for all topics of " + @Command(description = "Get subscription configured message-dispatch-rate for all topics of " + "the namespace (Disabled if value < 0)") private class GetSubscriptionDispatchRate extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getSubscriptionDispatchRate(namespace)); } } - @Parameters(commandDescription = "Set publish-rate for all topics of the namespace") + @Command(description = "Set publish-rate for all topics of the namespace") private class SetPublishRate extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--msg-publish-rate", + @Option(names = { "--msg-publish-rate", "-m" }, description = "message-publish-rate (default -1 will be overwrite if not passed)", required = false) private int msgPublishRate = -1; - @Parameter(names = { "--byte-publish-rate", + @Option(names = { "--byte-publish-rate", "-b" }, description = "byte-publish-rate (default -1 will be overwrite if not passed)", required = false) private long bytePublishRate = -1; - @Override + @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setPublishRate(namespace, new PublishRate(msgPublishRate, bytePublishRate)); } } - @Parameters(commandDescription = "Remove publish-rate for all topics of the namespace") + @Command(description = "Remove publish-rate for all topics of the namespace") private class RemovePublishRate extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removePublishRate(namespace); } } - @Parameters(commandDescription = "Get configured message-publish-rate for all topics of the namespace " - + "(Disabled if value < 0)") + @Command(name = "get-publish-rate", + description = "Get configured message-publish-rate for all topics of the namespace (Disabled if value < 0)") private class GetPublishRate extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Override + @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getPublishRate(namespace)); } } - @Parameters(commandDescription = "Set replicator message-dispatch-rate for all topics of the namespace") + @Command(description = "Set replicator message-dispatch-rate for all topics of the namespace") private class SetReplicatorDispatchRate extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--msg-dispatch-rate", + @Option(names = { "--msg-dispatch-rate", "-md" }, description = "message-dispatch-rate " + "(default -1 will be overwrite if not passed)", required = false) private int msgDispatchRate = -1; - @Parameter(names = { "--byte-dispatch-rate", + @Option(names = { "--byte-dispatch-rate", "-bd" }, description = "byte-dispatch-rate (default -1 will be overwrite if not passed)", required = false) private long byteDispatchRate = -1; - @Parameter(names = { "--dispatch-rate-period", + @Option(names = { "--dispatch-rate-period", "-dt" }, description = "dispatch-rate-period in second type " + "(default 1 second will be overwrite if not passed)", required = false) private int dispatchRatePeriodSec = 1; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setReplicatorDispatchRate(namespace, DispatchRate.builder() .dispatchThrottlingRateInMsg(msgDispatchRate) @@ -1223,63 +1205,65 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get replicator configured message-dispatch-rate for all topics of the namespace " + @Command(description = "Get replicator configured message-dispatch-rate for all topics of the namespace " + "(Disabled if value < 0)") private class GetReplicatorDispatchRate extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getReplicatorDispatchRate(namespace)); } } - @Parameters(commandDescription = "Remove replicator configured message-dispatch-rate " + @Command(description = "Remove replicator configured message-dispatch-rate " + "for all topics of the namespace") private class RemoveReplicatorDispatchRate extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeReplicatorDispatchRate(namespace); } } - @Parameters(commandDescription = "Get the backlog quota policies for a namespace") + @Command(description = "Get the backlog quota policies for a namespace") private class GetBacklogQuotaMap extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getBacklogQuotaMap(namespace)); } } - @Parameters(commandDescription = "Set a backlog quota policy for a namespace") + @Command(description = "Set a backlog quota policy for a namespace") private class SetBacklogQuota extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "-l", "--limit" }, description = "Size limit (eg: 10M, 16G)") - private String limitStr; + @Option(names = { "-l", "--limit" }, description = "Size limit (eg: 10M, 16G)", + converter = ByteUnitToLongConverter.class) + private Long limit; - @Parameter(names = { "-lt", "--limitTime" }, + @Option(names = { "-lt", "--limitTime" }, description = "Time limit in second (or minutes, hours, days, weeks eg: 100m, 3h, 2d, 5w), " - + "non-positive number for disabling time limit.") - private String limitTimeStr = null; + + "non-positive number for disabling time limit.", + converter = TimeUnitToSecondsConverter.class) + private Long limitTimeInSec; - @Parameter(names = { "-p", "--policy" }, description = "Retention policy to enforce when the limit is reached. " + @Option(names = { "-p", "--policy" }, description = "Retention policy to enforce when the limit is reached. " + "Valid options are: [producer_request_hold, producer_exception, consumer_backlog_eviction]", required = true) private String policyStr; - @Parameter(names = {"-t", "--type"}, description = "Backlog quota type to set. Valid options are: " + @Option(names = {"-t", "--type"}, description = "Backlog quota type to set. Valid options are: " + "destination_storage (default) and message_age. " + "destination_storage limits backlog by size. " + "message_age limits backlog by time, that is, message timestamp (broker or publish timestamp). " @@ -1305,45 +1289,38 @@ void run() throws PulsarAdminException { backlogQuotaTypeStr, Arrays.toString(BacklogQuota.BacklogQuotaType.values()))); } - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); BacklogQuota.Builder builder = BacklogQuota.builder().retentionPolicy(policy); if (backlogQuotaType == BacklogQuota.BacklogQuotaType.destination_storage) { // set quota by storage size - if (limitStr == null) { + if (limit == null) { throw new ParameterException("Quota type of 'destination_storage' needs a size limit"); } - long limit = validateSizeString(limitStr); builder.limitSize(limit); } else { // set quota by time - if (limitTimeStr == null) { + if (limitTimeInSec == null) { throw new ParameterException("Quota type of 'message_age' needs a time limit"); } - long limitTimeInSec; - try { - limitTimeInSec = RelativeTimeUtil.parseRelativeTimeInSeconds(limitTimeStr); - } catch (IllegalArgumentException e) { - throw new ParameterException(e.getMessage()); - } - builder.limitTime((int) limitTimeInSec); + builder.limitTime(limitTimeInSec.intValue()); } getAdmin().namespaces().setBacklogQuota(namespace, builder.build(), backlogQuotaType); } } - @Parameters(commandDescription = "Remove a backlog quota policy from a namespace") + @Command(description = "Remove a backlog quota policy from a namespace") private class RemoveBacklogQuota extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = {"-t", "--type"}, description = "Backlog quota type to remove. Valid options are: " + @Option(names = {"-t", "--type"}, description = "Backlog quota type to remove. Valid options are: " + "destination_storage, message_age") private String backlogQuotaTypeStr = BacklogQuota.BacklogQuotaType.destination_storage.name(); @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); BacklogQuota.BacklogQuotaType backlogQuotaType; try { backlogQuotaType = BacklogQuota.BacklogQuotaType.valueOf(backlogQuotaTypeStr); @@ -1355,56 +1332,56 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get the persistence policies for a namespace") + @Command(description = "Get the persistence policies for a namespace") private class GetPersistence extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getPersistence(namespace)); } } - @Parameters(commandDescription = "Remove the persistence policies for a namespace") + @Command(description = "Remove the persistence policies for a namespace") private class RemovePersistence extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removePersistence(namespace); } } - @Parameters(commandDescription = "Set the persistence policies for a namespace") + @Command(description = "Set the persistence policies for a namespace") private class SetPersistence extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "-e", + @Option(names = { "-e", "--bookkeeper-ensemble" }, description = "Number of bookies to use for a topic") private int bookkeeperEnsemble = 2; - @Parameter(names = { "-w", + @Option(names = { "-w", "--bookkeeper-write-quorum" }, description = "How many writes to make of each entry") private int bookkeeperWriteQuorum = 2; - @Parameter(names = { "-a", + @Option(names = { "-a", "--bookkeeper-ack-quorum" }, description = "Number of acks (guaranteed copies) to wait for each entry") private int bookkeeperAckQuorum = 2; - @Parameter(names = { "-r", + @Option(names = { "-r", "--ml-mark-delete-max-rate" }, description = "Throttling rate of mark-delete operation (0 means no throttle)") private double managedLedgerMaxMarkDeleteRate = 0; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); if (bookkeeperEnsemble <= 0 || bookkeeperWriteQuorum <= 0 || bookkeeperAckQuorum <= 0) { throw new ParameterException("[--bookkeeper-ensemble], [--bookkeeper-write-quorum] " + "and [--bookkeeper-ack-quorum] must greater than 0."); @@ -1417,18 +1394,18 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Clear backlog for a namespace") + @Command(description = "Clear backlog for a namespace") private class ClearBacklog extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--sub", "-s" }, description = "subscription name") + @Option(names = { "--sub", "-s" }, description = "subscription name") private String subscription; - @Parameter(names = { "--bundle", "-b" }, description = "{start-boundary}_{end-boundary}") + @Option(names = { "--bundle", "-b" }, description = "{start-boundary}_{end-boundary}") private String bundle; - @Parameter(names = { "--force", "-force" }, description = "Whether to force clear backlog without prompt") + @Option(names = { "--force", "-force" }, description = "Whether to force clear backlog without prompt") private boolean force; @Override @@ -1440,7 +1417,7 @@ void run() throws PulsarAdminException, IOException { return; } } - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); if (subscription != null && bundle != null) { getAdmin().namespaces().clearNamespaceBundleBacklogForSubscription(namespace, bundle, subscription); } else if (subscription != null) { @@ -1453,20 +1430,20 @@ void run() throws PulsarAdminException, IOException { } } - @Parameters(commandDescription = "Unsubscribe the given subscription on all topics on a namespace") + @Command(description = "Unsubscribe the given subscription on all topics on a namespace") private class Unsubscribe extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--sub", "-s" }, description = "subscription name", required = true) + @Option(names = { "--sub", "-s" }, description = "subscription name", required = true) private String subscription; - @Parameter(names = { "--bundle", "-b" }, description = "{start-boundary}_{end-boundary}") + @Option(names = { "--bundle", "-b" }, description = "{start-boundary}_{end-boundary}") private String bundle; @Override void run() throws Exception { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); if (bundle != null) { getAdmin().namespaces().unsubscribeNamespaceBundle(namespace, bundle, subscription); } else { @@ -1476,20 +1453,20 @@ void run() throws Exception { } - @Parameters(commandDescription = "Enable or disable message encryption required for a namespace") + @Command(description = "Enable or disable message encryption required for a namespace") private class SetEncryptionRequired extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--enable", "-e" }, description = "Enable message encryption required") + @Option(names = { "--enable", "-e" }, description = "Enable message encryption required") private boolean enable = false; - @Parameter(names = { "--disable", "-d" }, description = "Disable message encryption required") + @Option(names = { "--disable", "-d" }, description = "Disable message encryption required") private boolean disable = false; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); if (enable == disable) { throw new ParameterException("Need to specify either --enable or --disable"); @@ -1498,97 +1475,90 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get encryption required for a namespace") + @Command(description = "Get encryption required for a namespace") private class GetEncryptionRequired extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getEncryptionRequiredStatus(namespace)); } } - @Parameters(commandDescription = "Get the delayed delivery policy for a namespace") + @Command(description = "Get the delayed delivery policy for a namespace") private class GetDelayedDelivery extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getDelayedDelivery(namespace)); } } - @Parameters(commandDescription = "Remove delayed delivery policies from a namespace") + @Command(description = "Remove delayed delivery policies from a namespace") private class RemoveDelayedDelivery extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeDelayedDeliveryMessages(namespace); } } - @Parameters(commandDescription = "Get the inactive topic policy for a namespace") + @Command(description = "Get the inactive topic policy for a namespace") private class GetInactiveTopicPolicies extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getInactiveTopicPolicies(namespace)); } } - @Parameters(commandDescription = "Remove inactive topic policies from a namespace") + @Command(description = "Remove inactive topic policies from a namespace") private class RemoveInactiveTopicPolicies extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeInactiveTopicPolicies(namespace); } } - @Parameters(commandDescription = "Set the inactive topic policies on a namespace") + @Command(description = "Set the inactive topic policies on a namespace") private class SetInactiveTopicPolicies extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--enable-delete-while-inactive", "-e" }, description = "Enable delete while inactive") + @Option(names = { "--enable-delete-while-inactive", "-e" }, description = "Enable delete while inactive") private boolean enableDeleteWhileInactive = false; - @Parameter(names = { "--disable-delete-while-inactive", "-d" }, description = "Disable delete while inactive") + @Option(names = { "--disable-delete-while-inactive", "-d" }, description = "Disable delete while inactive") private boolean disableDeleteWhileInactive = false; - @Parameter(names = {"--max-inactive-duration", "-t"}, description = "Max duration of topic inactivity in " + @Option(names = {"--max-inactive-duration", "-t"}, description = "Max duration of topic inactivity in " + "seconds, topics that are inactive for longer than this value will be deleted " - + "(eg: 1s, 10s, 1m, 5h, 3d)", required = true) - private String deleteInactiveTopicsMaxInactiveDuration; + + "(eg: 1s, 10s, 1m, 5h, 3d)", required = true, + converter = TimeUnitToSecondsConverter.class) + private Long maxInactiveDurationInSeconds; - @Parameter(names = { "--delete-mode", "-m" }, description = "Mode of delete inactive topic, Valid options are: " + @Option(names = { "--delete-mode", "-m" }, description = "Mode of delete inactive topic, Valid options are: " + "[delete_when_no_subscriptions, delete_when_subscriptions_caught_up]", required = true) private String inactiveTopicDeleteMode; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); - long maxInactiveDurationInSeconds; - try { - maxInactiveDurationInSeconds = TimeUnit.SECONDS.toSeconds( - RelativeTimeUtil.parseRelativeTimeInSeconds(deleteInactiveTopicsMaxInactiveDuration)); - } catch (IllegalArgumentException exception) { - throw new ParameterException(exception.getMessage()); - } - + String namespace = validateNamespace(namespaceName); if (enableDeleteWhileInactive == disableDeleteWhileInactive) { throw new ParameterException("Need to specify either enable-delete-while-inactive or " + "disable-delete-while-inactive"); @@ -1601,37 +1571,35 @@ void run() throws PulsarAdminException { + "delete_when_subscriptions_caught_up"); } getAdmin().namespaces().setInactiveTopicPolicies(namespace, new InactiveTopicPolicies(deleteMode, - (int) maxInactiveDurationInSeconds, enableDeleteWhileInactive)); + maxInactiveDurationInSeconds.intValue(), enableDeleteWhileInactive)); } } - @Parameters(commandDescription = "Set the delayed delivery policy on a namespace") + @Command(description = "Set the delayed delivery policy on a namespace") private class SetDelayedDelivery extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--enable", "-e" }, description = "Enable delayed delivery messages") + @Option(names = { "--enable", "-e" }, description = "Enable delayed delivery messages") private boolean enable = false; - @Parameter(names = { "--disable", "-d" }, description = "Disable delayed delivery messages") + @Option(names = { "--disable", "-d" }, description = "Disable delayed delivery messages") private boolean disable = false; - @Parameter(names = { "--time", "-t" }, description = "The tick time for when retrying on " + @Option(names = { "--time", "-t" }, description = "The tick time for when retrying on " + "delayed delivery messages, affecting the accuracy of the delivery time compared to " - + "the scheduled time. (eg: 1s, 10s, 1m, 5h, 3d)") - private String delayedDeliveryTimeStr = "1s"; + + "the scheduled time. (eg: 1s, 10s, 1m, 5h, 3d)", + converter = TimeUnitToMillisConverter.class) + private Long delayedDeliveryTimeInMills = 1000L; + + @Option(names = { "--maxDelay", "-md" }, + description = "The max allowed delay for delayed delivery. (eg: 1s, 10s, 1m, 5h, 3d)", + converter = TimeUnitToMillisConverter.class) + private Long delayedDeliveryMaxDelayInMillis = 0L; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); - long delayedDeliveryTimeInMills; - try { - delayedDeliveryTimeInMills = TimeUnit.SECONDS.toMillis( - RelativeTimeUtil.parseRelativeTimeInSeconds(delayedDeliveryTimeStr)); - } catch (IllegalArgumentException exception) { - throw new ParameterException(exception.getMessage()); - } - + String namespace = validateNamespace(namespaceName); if (enable == disable) { throw new ParameterException("Need to specify either --enable or --disable"); } @@ -1639,363 +1607,372 @@ void run() throws PulsarAdminException { getAdmin().namespaces().setDelayedDeliveryMessages(namespace, DelayedDeliveryPolicies.builder() .tickTime(delayedDeliveryTimeInMills) .active(enable) + .maxDeliveryDelayInMillis(delayedDeliveryMaxDelayInMillis) .build()); } } - @Parameters(commandDescription = "Set subscription auth mode on a namespace") + @Command(description = "Set subscription auth mode on a namespace") private class SetSubscriptionAuthMode extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "-m", "--subscription-auth-mode" }, description = "Subscription authorization mode for " + @Option(names = { "-m", "--subscription-auth-mode" }, description = "Subscription authorization mode for " + "Pulsar policies. Valid options are: [None, Prefix]", required = true) private String mode; @Override void run() throws Exception { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setSubscriptionAuthMode(namespace, SubscriptionAuthMode.valueOf(mode)); } } - @Parameters(commandDescription = "Get subscriptionAuthMod for a namespace") + @Command(description = "Get subscriptionAuthMod for a namespace") private class GetSubscriptionAuthMode extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getSubscriptionAuthMode(namespace)); } } - @Parameters(commandDescription = "Get deduplicationSnapshotInterval for a namespace") + @Command(description = "Get deduplicationSnapshotInterval for a namespace") private class GetDeduplicationSnapshotInterval extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getDeduplicationSnapshotInterval(namespace)); } } - @Parameters(commandDescription = "Remove deduplicationSnapshotInterval for a namespace") + @Command(description = "Remove deduplicationSnapshotInterval for a namespace") private class RemoveDeduplicationSnapshotInterval extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeDeduplicationSnapshotInterval(namespace); } } - @Parameters(commandDescription = "Set deduplicationSnapshotInterval for a namespace") + @Command(description = "Set deduplicationSnapshotInterval for a namespace") private class SetDeduplicationSnapshotInterval extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = {"--interval", "-i"} + @Option(names = {"--interval", "-i"} , description = "deduplicationSnapshotInterval for a namespace", required = true) private int interval; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setDeduplicationSnapshotInterval(namespace, interval); } } - @Parameters(commandDescription = "Get maxProducersPerTopic for a namespace") + @Command(description = "Get maxProducersPerTopic for a namespace") private class GetMaxProducersPerTopic extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getMaxProducersPerTopic(namespace)); } } - @Parameters(commandDescription = "Remove max producers per topic for a namespace") + @Command(description = "Remove max producers per topic for a namespace") private class RemoveMaxProducersPerTopic extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeMaxProducersPerTopic(namespace); } } - @Parameters(commandDescription = "Set maxProducersPerTopic for a namespace") + @Command(description = "Set maxProducersPerTopic for a namespace") private class SetMaxProducersPerTopic extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--max-producers-per-topic", "-p" }, + @Option(names = { "--max-producers-per-topic", "-p" }, description = "maxProducersPerTopic for a namespace", required = true) private int maxProducersPerTopic; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setMaxProducersPerTopic(namespace, maxProducersPerTopic); } } - @Parameters(commandDescription = "Get maxConsumersPerTopic for a namespace") + @Command(description = "Get maxConsumersPerTopic for a namespace") private class GetMaxConsumersPerTopic extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getMaxConsumersPerTopic(namespace)); } } - @Parameters(commandDescription = "Set maxConsumersPerTopic for a namespace") + @Command(description = "Set maxConsumersPerTopic for a namespace") private class SetMaxConsumersPerTopic extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--max-consumers-per-topic", "-c" }, + @Option(names = { "--max-consumers-per-topic", "-c" }, description = "maxConsumersPerTopic for a namespace", required = true) private int maxConsumersPerTopic; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setMaxConsumersPerTopic(namespace, maxConsumersPerTopic); } } - @Parameters(commandDescription = "Remove max consumers per topic for a namespace") + @Command(description = "Remove max consumers per topic for a namespace") private class RemoveMaxConsumersPerTopic extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeMaxConsumersPerTopic(namespace); } } - @Parameters(commandDescription = "Get maxConsumersPerSubscription for a namespace") + @Command(description = "Get maxConsumersPerSubscription for a namespace") private class GetMaxConsumersPerSubscription extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getMaxConsumersPerSubscription(namespace)); } } - @Parameters(commandDescription = "Remove maxConsumersPerSubscription for a namespace") + @Command(description = "Remove maxConsumersPerSubscription for a namespace") private class RemoveMaxConsumersPerSubscription extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeMaxConsumersPerSubscription(namespace); } } - @Parameters(commandDescription = "Set maxConsumersPerSubscription for a namespace") + @Command(description = "Set maxConsumersPerSubscription for a namespace") private class SetMaxConsumersPerSubscription extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--max-consumers-per-subscription", "-c" }, + @Option(names = { "--max-consumers-per-subscription", "-c" }, description = "maxConsumersPerSubscription for a namespace", required = true) private int maxConsumersPerSubscription; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setMaxConsumersPerSubscription(namespace, maxConsumersPerSubscription); } } - @Parameters(commandDescription = "Get maxUnackedMessagesPerConsumer for a namespace") + @Command(description = "Get maxUnackedMessagesPerConsumer for a namespace") private class GetMaxUnackedMessagesPerConsumer extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getMaxUnackedMessagesPerConsumer(namespace)); } } - @Parameters(commandDescription = "Set maxUnackedMessagesPerConsumer for a namespace") + @Command(description = "Set maxUnackedMessagesPerConsumer for a namespace") private class SetMaxUnackedMessagesPerConsumer extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--max-unacked-messages-per-topic", "-c" }, + @Option(names = { "--max-unacked-messages-per-topic", "-c" }, description = "maxUnackedMessagesPerConsumer for a namespace", required = true) private int maxUnackedMessagesPerConsumer; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setMaxUnackedMessagesPerConsumer(namespace, maxUnackedMessagesPerConsumer); } } - @Parameters(commandDescription = "Remove maxUnackedMessagesPerConsumer for a namespace") + @Command(description = "Remove maxUnackedMessagesPerConsumer for a namespace") private class RemoveMaxUnackedMessagesPerConsumer extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeMaxUnackedMessagesPerConsumer(namespace); } } - @Parameters(commandDescription = "Get maxUnackedMessagesPerSubscription for a namespace") + @Command(description = "Get maxUnackedMessagesPerSubscription for a namespace") private class GetMaxUnackedMessagesPerSubscription extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getMaxUnackedMessagesPerSubscription(namespace)); } } - @Parameters(commandDescription = "Set maxUnackedMessagesPerSubscription for a namespace") + @Command(description = "Set maxUnackedMessagesPerSubscription for a namespace") private class SetMaxUnackedMessagesPerSubscription extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--max-unacked-messages-per-subscription", "-c" }, + @Option(names = {"--max-unacked-messages-per-subscription", "-c"}, description = "maxUnackedMessagesPerSubscription for a namespace", required = true) private int maxUnackedMessagesPerSubscription; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setMaxUnackedMessagesPerSubscription(namespace, maxUnackedMessagesPerSubscription); } } - @Parameters(commandDescription = "Remove maxUnackedMessagesPerSubscription for a namespace") + @Command(description = "Remove maxUnackedMessagesPerSubscription for a namespace") private class RemoveMaxUnackedMessagesPerSubscription extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeMaxUnackedMessagesPerSubscription(namespace); } } - @Parameters(commandDescription = "Get compactionThreshold for a namespace") + @Command(description = "Get compactionThreshold for a namespace") private class GetCompactionThreshold extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getCompactionThreshold(namespace)); } } - @Parameters(commandDescription = "Remove compactionThreshold for a namespace") + @Command(description = "Remove compactionThreshold for a namespace") private class RemoveCompactionThreshold extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeCompactionThreshold(namespace); } } - @Parameters(commandDescription = "Set compactionThreshold for a namespace") + @Command(description = "Set compactionThreshold for a namespace") private class SetCompactionThreshold extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--threshold", "-t" }, + @Option(names = { "--threshold", "-t" }, description = "Maximum number of bytes in a topic backlog before compaction is triggered " + "(eg: 10M, 16G, 3T). 0 disables automatic compaction", - required = true) - private String thresholdStr = "0"; + required = true, + converter = ByteUnitToLongConverter.class) + private Long threshold = 0L; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); - long threshold = validateSizeString(thresholdStr); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setCompactionThreshold(namespace, threshold); } } - @Parameters(commandDescription = "Get offloadThreshold for a namespace") + @Command(description = "Get offloadThreshold for a namespace") private class GetOffloadThreshold extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); - print(getAdmin().namespaces().getOffloadThreshold(namespace)); + String namespace = validateNamespace(namespaceName); + print("offloadThresholdInBytes: " + getAdmin().namespaces().getOffloadThreshold(namespace)); + print("offloadThresholdInSeconds: " + getAdmin().namespaces().getOffloadThresholdInSeconds(namespace)); } } - @Parameters(commandDescription = "Set offloadThreshold for a namespace") + @Command(description = "Set offloadThreshold for a namespace") private class SetOffloadThreshold extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--size", "-s" }, + @Option(names = { "--size", "-s" }, description = "Maximum number of bytes stored in the pulsar cluster for a topic before data will" + " start being automatically offloaded to longterm storage (eg: 10M, 16G, 3T, 100)." + " -1 falls back to the cluster's namespace default." + " Negative values disable automatic offload." + " 0 triggers offloading as soon as possible.", - required = true) - private String thresholdStr = "-1"; + required = true, + converter = ByteUnitToLongConverter.class) + private Long threshold = -1L; + + @Option(names = {"--time", "-t"}, + description = "Maximum number of seconds stored on the pulsar cluster for a topic" + + " before the broker will start offloading to longterm storage (eg: 10m, 5h, 3d, 2w).", + converter = TimeUnitToSecondsConverter.class) + private Long thresholdInSeconds = -1L; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); - long threshold = validateSizeString(thresholdStr); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setOffloadThreshold(namespace, threshold); + getAdmin().namespaces().setOffloadThresholdInSeconds(namespace, thresholdInSeconds); } } - @Parameters(commandDescription = "Get offloadDeletionLag, in minutes, for a namespace") + @Command(description = "Get offloadDeletionLag, in minutes, for a namespace") private class GetOffloadDeletionLag extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); Long lag = getAdmin().namespaces().getOffloadDeleteLagMs(namespace); if (lag != null) { System.out.println(TimeUnit.MINUTES.convert(lag, TimeUnit.MILLISECONDS) + " minute(s)"); @@ -2005,72 +1982,67 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Set offloadDeletionLag for a namespace") + @Command(description = "Set offloadDeletionLag for a namespace") private class SetOffloadDeletionLag extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--lag", "-l" }, + @Option(names = { "--lag", "-l" }, description = "Duration to wait after offloading a ledger segment, before deleting the copy of that" + " segment from cluster local storage. (eg: 10m, 5h, 3d, 2w).", - required = true) - private String lag = "-1"; + required = true, + converter = TimeUnitToSecondsConverter.class) + private Long lagInSec = -1L; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); - long lagInSec; - try { - lagInSec = RelativeTimeUtil.parseRelativeTimeInSeconds(lag); - } catch (IllegalArgumentException exception) { - throw new ParameterException(exception.getMessage()); - } + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setOffloadDeleteLag(namespace, lagInSec, TimeUnit.SECONDS); } } - @Parameters(commandDescription = "Clear offloadDeletionLag for a namespace") + @Command(description = "Clear offloadDeletionLag for a namespace") private class ClearOffloadDeletionLag extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().clearOffloadDeleteLag(namespace); } } - @Parameters(commandDescription = "Get the schema auto-update strategy for a namespace", hidden = true) + @Command(description = "Get the schema auto-update strategy for a namespace") private class GetSchemaAutoUpdateStrategy extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); System.out.println(getAdmin().namespaces().getSchemaAutoUpdateCompatibilityStrategy(namespace) .toString().toUpperCase()); } } - @Parameters(commandDescription = "Set the schema auto-update strategy for a namespace", hidden = true) + @Command(description = "Set the schema auto-update strategy for a namespace") private class SetSchemaAutoUpdateStrategy extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--compatibility", "-c" }, + @Option(names = { "--compatibility", "-c" }, description = "Compatibility level required for new schemas created via a Producer. " + "Possible values (Full, Backward, Forward).") private String strategyParam = null; - @Parameter(names = { "--disabled", "-d" }, description = "Disable automatic schema updates") + @Option(names = { "--disabled", "-d" }, description = "Disable automatic schema updates") private boolean disabled = false; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); SchemaAutoUpdateCompatibilityStrategy strategy = null; String strategyStr = strategyParam != null ? strategyParam.toUpperCase() : ""; @@ -2091,25 +2063,25 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get the schema compatibility strategy for a namespace") + @Command(description = "Get the schema compatibility strategy for a namespace") private class GetSchemaCompatibilityStrategy extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); System.out.println(getAdmin().namespaces().getSchemaCompatibilityStrategy(namespace) .toString().toUpperCase()); } } - @Parameters(commandDescription = "Set the schema compatibility strategy for a namespace") + @Command(description = "Set the schema compatibility strategy for a namespace") private class SetSchemaCompatibilityStrategy extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--compatibility", "-c" }, + @Option(names = { "--compatibility", "-c" }, description = "Compatibility level required for new schemas created via a Producer. " + "Possible values (FULL, BACKWARD, FORWARD, " + "UNDEFINED, BACKWARD_TRANSITIVE, " @@ -2120,7 +2092,7 @@ private class SetSchemaCompatibilityStrategy extends CliCommand { @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); String strategyStr = strategyParam != null ? strategyParam.toUpperCase() : ""; SchemaCompatibilityStrategy strategy; @@ -2134,33 +2106,33 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get the namespace whether allow auto update schema") + @Command(description = "Get the namespace whether allow auto update schema") private class GetIsAllowAutoUpdateSchema extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); System.out.println(getAdmin().namespaces().getIsAllowAutoUpdateSchema(namespace)); } } - @Parameters(commandDescription = "Set the namespace whether allow auto update schema") + @Command(description = "Set the namespace whether allow auto update schema") private class SetIsAllowAutoUpdateSchema extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--enable", "-e" }, description = "Enable schema validation enforced") + @Option(names = { "--enable", "-e" }, description = "Enable schema validation enforced") private boolean enable = false; - @Parameter(names = { "--disable", "-d" }, description = "Disable schema validation enforced") + @Option(names = { "--disable", "-d" }, description = "Disable schema validation enforced") private boolean disable = false; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); if (enable == disable) { throw new ParameterException("Need to specify either --enable or --disable"); @@ -2169,36 +2141,36 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get the schema validation enforced") + @Command(description = "Get the schema validation enforced") private class GetSchemaValidationEnforced extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the namespace") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the namespace") private boolean applied = false; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); System.out.println(getAdmin().namespaces().getSchemaValidationEnforced(namespace, applied)); } } - @Parameters(commandDescription = "Set the schema whether open schema validation enforced") + @Command(description = "Set the schema whether open schema validation enforced") private class SetSchemaValidationEnforced extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--enable", "-e" }, description = "Enable schema validation enforced") + @Option(names = { "--enable", "-e" }, description = "Enable schema validation enforced") private boolean enable = false; - @Parameter(names = { "--disable", "-d" }, description = "Disable schema validation enforced") + @Option(names = { "--disable", "-d" }, description = "Disable schema validation enforced") private boolean disable = false; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); if (enable == disable) { throw new ParameterException("Need to specify either --enable or --disable"); @@ -2207,94 +2179,100 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Set the offload policies for a namespace") + @Command(description = "Set the offload policies for a namespace") private class SetOffloadPolicies extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter( + @Option( names = {"--driver", "-d"}, description = "Driver to use to offload old data to long term storage, " + "(Possible values: S3, aws-s3, google-cloud-storage, filesystem, azureblob)", required = true) private String driver; - @Parameter( + @Option( names = {"--region", "-r"}, description = "The long term storage region, " - + "default is s3ManagedLedgerOffloadRegion or gcsManagedLedgerOffloadRegion in broker.conf", + + "default is s3ManagedLedgerOffloadRegion or gcsManagedLedgerOffloadRegion in broker.conf", required = false) private String region; - @Parameter( + @Option( names = {"--bucket", "-b"}, description = "Bucket to place offloaded ledger into", - required = true) + required = false) private String bucket; - @Parameter( + @Option( names = {"--endpoint", "-e"}, description = "Alternative endpoint to connect to, " + "s3 default is s3ManagedLedgerOffloadServiceEndpoint in broker.conf", required = false) private String endpoint; - @Parameter( + @Option( names = {"--aws-id", "-i"}, description = "AWS Credential Id to use when using driver S3 or aws-s3", required = false) private String awsId; - @Parameter( + @Option( names = {"--aws-secret", "-s"}, description = "AWS Credential Secret to use when using driver S3 or aws-s3", required = false) private String awsSecret; - @Parameter( + @Option( names = {"--s3-role", "-ro"}, description = "S3 Role used for STSAssumeRoleSessionCredentialsProvider", required = false) private String s3Role; - @Parameter( + @Option( names = {"--s3-role-session-name", "-rsn"}, description = "S3 role session name used for STSAssumeRoleSessionCredentialsProvider", required = false) private String s3RoleSessionName; - @Parameter( + @Option( names = {"--maxBlockSize", "-mbs"}, - description = "Max block size (eg: 32M, 64M), default is 64MB", - required = false) - private String maxBlockSizeStr; + description = "Max block size (eg: 32M, 64M), default is 64MB" + + "s3 and google-cloud-storage requires this parameter", + required = false, + converter = ByteUnitToIntegerConverter.class) + private Integer maxBlockSizeInBytes = OffloadPoliciesImpl.DEFAULT_MAX_BLOCK_SIZE_IN_BYTES; - @Parameter( + @Option( names = {"--readBufferSize", "-rbs"}, description = "Read buffer size (eg: 1M, 5M), default is 1MB", - required = false) - private String readBufferSizeStr; + required = false, + converter = ByteUnitToIntegerConverter.class) + private Integer readBufferSizeInBytes = OffloadPoliciesImpl.DEFAULT_READ_BUFFER_SIZE_IN_BYTES; - @Parameter( + @Option( names = {"--offloadAfterElapsed", "-oae"}, - description = "Offload after elapsed in minutes (or minutes, hours,days,weeks eg: 100m, 3h, 2d, 5w).", - required = false) - private String offloadAfterElapsedStr; + description = "Delay time in Millis for deleting the bookkeeper ledger after offload " + + "(or seconds,minutes,hours,days,weeks eg: 10s, 100m, 3h, 2d, 5w).", + required = false, + converter = TimeUnitToMillisConverter.class) + private Long offloadAfterElapsedInMillis = OffloadPoliciesImpl.DEFAULT_OFFLOAD_DELETION_LAG_IN_MILLIS; - @Parameter( + @Option( names = {"--offloadAfterThreshold", "-oat"}, description = "Offload after threshold size (eg: 1M, 5M)", - required = false) - private String offloadAfterThresholdStr; + required = false, + converter = ByteUnitToLongConverter.class) + private Long offloadAfterThresholdInBytes = OffloadPoliciesImpl.DEFAULT_OFFLOAD_THRESHOLD_IN_BYTES; - @Parameter( + @Option( names = {"--offloadAfterThresholdInSeconds", "-oats"}, - description = "Offload after threshold seconds (eg: 1,5,10)", - required = false - ) - private String offloadAfterThresholdInSecondsStr; + description = "Offload after threshold seconds (or minutes,hours,days,weeks eg: 100m, 3h, 2d, 5w).", + required = false, + converter = TimeUnitToSecondsConverter.class) + private Long offloadThresholdInSeconds = OffloadPoliciesImpl.DEFAULT_OFFLOAD_THRESHOLD_IN_SECONDS; - @Parameter( + @Option( names = {"--offloadedReadPriority", "-orp"}, description = "Read priority for offloaded messages. By default, once messages are offloaded to " + "long-term storage, brokers read messages from long-term storage, but messages can " @@ -2318,23 +2296,9 @@ public boolean isS3Driver(String driver) { return driver.equalsIgnoreCase(driverNames.get(0)) || driver.equalsIgnoreCase(driverNames.get(1)); } - public boolean positiveCheck(String paramName, long value) { - if (value <= 0) { - throw new ParameterException(paramName + " is not be negative or 0!"); - } - return true; - } - - public boolean maxValueCheck(String paramName, long value, long maxValue) { - if (value > maxValue) { - throw new ParameterException(paramName + " is not bigger than " + maxValue + "!"); - } - return true; - } - @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); if (!driverSupported(driver)) { throw new ParameterException("The driver " + driver + " is not supported, " @@ -2347,65 +2311,15 @@ void run() throws PulsarAdminException { + " if s3 offload enabled"); } - int maxBlockSizeInBytes = OffloadPoliciesImpl.DEFAULT_MAX_BLOCK_SIZE_IN_BYTES; - if (StringUtils.isNotEmpty(maxBlockSizeStr)) { - long maxBlockSize = validateSizeString(maxBlockSizeStr); - if (positiveCheck("MaxBlockSize", maxBlockSize) - && maxValueCheck("MaxBlockSize", maxBlockSize, Integer.MAX_VALUE)) { - maxBlockSizeInBytes = Long.valueOf(maxBlockSize).intValue(); - } - } - - int readBufferSizeInBytes = OffloadPoliciesImpl.DEFAULT_READ_BUFFER_SIZE_IN_BYTES; - if (StringUtils.isNotEmpty(readBufferSizeStr)) { - long readBufferSize = validateSizeString(readBufferSizeStr); - if (positiveCheck("ReadBufferSize", readBufferSize) - && maxValueCheck("ReadBufferSize", readBufferSize, Integer.MAX_VALUE)) { - readBufferSizeInBytes = Long.valueOf(readBufferSize).intValue(); - } - } - - Long offloadAfterElapsedInMillis = OffloadPoliciesImpl.DEFAULT_OFFLOAD_DELETION_LAG_IN_MILLIS; - if (StringUtils.isNotEmpty(offloadAfterElapsedStr)) { - Long offloadAfterElapsed; - try { - offloadAfterElapsed = TimeUnit.SECONDS.toMillis( - RelativeTimeUtil.parseRelativeTimeInSeconds(offloadAfterElapsedStr)); - } catch (IllegalArgumentException exception) { - throw new ParameterException(exception.getMessage()); - } - if (positiveCheck("OffloadAfterElapsed", offloadAfterElapsed) - && maxValueCheck("OffloadAfterElapsed", offloadAfterElapsed, Long.MAX_VALUE)) { - offloadAfterElapsedInMillis = offloadAfterElapsed; - } - } - - Long offloadAfterThresholdInBytes = OffloadPoliciesImpl.DEFAULT_OFFLOAD_THRESHOLD_IN_BYTES; - if (StringUtils.isNotEmpty(offloadAfterThresholdStr)) { - long offloadAfterThreshold = validateSizeString(offloadAfterThresholdStr); - if (maxValueCheck("OffloadAfterThreshold", offloadAfterThreshold, Long.MAX_VALUE)) { - offloadAfterThresholdInBytes = offloadAfterThreshold; - } - } - - Long offloadThresholdInSeconds = OffloadPoliciesImpl.DEFAULT_OFFLOAD_THRESHOLD_IN_SECONDS; - if (StringUtils.isNotEmpty(offloadAfterThresholdInSecondsStr)) { - long offloadThresholdInSeconds0 = Long.parseLong(offloadAfterThresholdInSecondsStr.trim()); - if (maxValueCheck("OffloadAfterThresholdInSeconds", offloadThresholdInSeconds0, Long.MAX_VALUE)) { - offloadThresholdInSeconds = offloadThresholdInSeconds0; - } - } - OffloadedReadPriority offloadedReadPriority = OffloadPoliciesImpl.DEFAULT_OFFLOADED_READ_PRIORITY; - if (this.offloadReadPriorityStr != null) { try { offloadedReadPriority = OffloadedReadPriority.fromString(this.offloadReadPriorityStr); } catch (Exception e) { throw new ParameterException("--offloadedReadPriority parameter must be one of " + Arrays.stream(OffloadedReadPriority.values()) - .map(OffloadedReadPriority::toString) - .collect(Collectors.joining(",")) + .map(OffloadedReadPriority::toString) + .collect(Collectors.joining(",")) + " but got: " + this.offloadReadPriorityStr, e); } } @@ -2420,104 +2334,104 @@ && maxValueCheck("OffloadAfterElapsed", offloadAfterElapsed, Long.MAX_VALUE)) { } } - @Parameters(commandDescription = "Remove the offload policies for a namespace") + @Command(description = "Remove the offload policies for a namespace") private class RemoveOffloadPolicies extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeOffloadPolicies(namespace); } } - @Parameters(commandDescription = "Get the offload policies for a namespace") + @Command(description = "Get the offload policies for a namespace") private class GetOffloadPolicies extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getOffloadPolicies(namespace)); } } - @Parameters(commandDescription = "Set max topics per namespace") + @Command(description = "Set max topics per namespace") private class SetMaxTopicsPerNamespace extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = {"--max-topics-per-namespace", "-t"}, + @Option(names = {"--max-topics-per-namespace", "-t"}, description = "max topics per namespace", required = true) private int maxTopicsPerNamespace; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setMaxTopicsPerNamespace(namespace, maxTopicsPerNamespace); } } - @Parameters(commandDescription = "Get max topics per namespace") + @Command(description = "Get max topics per namespace") private class GetMaxTopicsPerNamespace extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getMaxTopicsPerNamespace(namespace)); } } - @Parameters(commandDescription = "Remove max topics per namespace") + @Command(description = "Remove max topics per namespace") private class RemoveMaxTopicsPerNamespace extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeMaxTopicsPerNamespace(namespace); } } - @Parameters(commandDescription = "Set property for a namespace") + @Command(description = "Set property for a namespace") private class SetPropertyForNamespace extends CliCommand { - @Parameter(description = "tenant/namespace\n", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = {"--key", "-k"}, description = "Key of the property", required = true) + @Option(names = {"--key", "-k"}, description = "Key of the property", required = true) private String key; - @Parameter(names = {"--value", "-v"}, description = "Value of the property", required = true) + @Option(names = {"--value", "-v"}, description = "Value of the property", required = true) private String value; @Override void run() throws Exception { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setProperty(namespace, key, value); } } - @Parameters(commandDescription = "Set properties of a namespace") + @Command(description = "Set properties of a namespace") private class SetPropertiesForNamespace extends CliCommand { - @Parameter(description = "tenant/namespace\n", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = {"--properties", "-p"}, description = "key value pair properties(a=a,b=b,c=c)", + @Option(names = {"--properties", "-p"}, description = "key value pair properties(a=a,b=b,c=c)", required = true) private java.util.List properties; @Override void run() throws Exception { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); if (properties.size() == 0) { throw new ParameterException(String.format("Required at least one property for the namespace, " + "but found %d.", properties.size())); @@ -2527,326 +2441,419 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Get property for a namespace") + @Command(description = "Get property for a namespace") private class GetPropertyForNamespace extends CliCommand { - @Parameter(description = "tenant/namespace\n", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = {"--key", "-k"}, description = "Key of the property", required = true) + @Option(names = {"--key", "-k"}, description = "Key of the property", required = true) private String key; @Override void run() throws Exception { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getProperty(namespace, key)); } } - @Parameters(commandDescription = "Get properties of a namespace") + @Command(description = "Get properties of a namespace") private class GetPropertiesForNamespace extends CliCommand { - @Parameter(description = "tenant/namespace\n", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws Exception { - String namespace = validateNamespace(params); - Json.prettyPrint(getAdmin().namespaces().getProperties(namespace)); + final String namespace = validateNamespace(namespaceName); + final Map properties = getAdmin().namespaces().getProperties(namespace); + prettyPrint(properties); } } - @Parameters(commandDescription = "Remove property for a namespace") + @Command(description = "Remove property for a namespace") private class RemovePropertyForNamespace extends CliCommand { - @Parameter(description = "tenant/namespace\n", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = {"--key", "-k"}, description = "Key of the property", required = true) + @Option(names = {"--key", "-k"}, description = "Key of the property", required = true) private String key; @Override void run() throws Exception { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().removeProperty(namespace, key)); } } - @Parameters(commandDescription = "Clear all properties for a namespace") + @Command(description = "Clear all properties for a namespace") private class ClearPropertiesForNamespace extends CliCommand { - @Parameter(description = "tenant/namespace\n", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws Exception { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().clearProperties(namespace); } } - @Parameters(commandDescription = "Get ResourceGroup for a namespace") + @Command(description = "Get ResourceGroup for a namespace") private class GetResourceGroup extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getNamespaceResourceGroup(namespace)); } } - @Parameters(commandDescription = "Set ResourceGroup for a namespace") + @Command(description = "Set ResourceGroup for a namespace") private class SetResourceGroup extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--resource-group-name", "-rgn" }, description = "ResourceGroup name", required = true) + @Option(names = {"--resource-group-name", "-rgn"}, description = "ResourceGroup name", required = true) private String rgName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setNamespaceResourceGroup(namespace, rgName); } } - @Parameters(commandDescription = "Remove ResourceGroup from a namespace") + @Command(description = "Remove ResourceGroup from a namespace") private class RemoveResourceGroup extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeNamespaceResourceGroup(namespace); } } - @Parameters(commandDescription = "Get entry filters for a namespace") + @Command(description = "Update migration state for a namespace") + private class UpdateMigrationState extends CliCommand { + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; + + @Option(names = "--migrated", description = "Is namespace migrated") + private boolean migrated; + + @Override + void run() throws PulsarAdminException { + String namespace = validateNamespace(namespaceName); + getAdmin().namespaces().updateMigrationState(namespace, migrated); + } + } + + @Command(description = "Get entry filters for a namespace") private class GetEntryFiltersPerTopic extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getAdmin().namespaces().getNamespaceEntryFilters(namespace)); } } - @Parameters(commandDescription = "Set entry filters for a namespace") + @Command(description = "Set entry filters for a namespace") private class SetEntryFiltersPerTopic extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "--entry-filters-name", "-efn" }, + @Option(names = { "--entry-filters-name", "-efn" }, description = "The class name for the entry filter.", required = true) - private String entryFiltersName = ""; + private String entryFiltersName = ""; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().setNamespaceEntryFilters(namespace, new EntryFilters(entryFiltersName)); } } - @Parameters(commandDescription = "Remove entry filters for a namespace") + @Command(description = "Remove entry filters for a namespace") private class RemoveEntryFiltersPerTopic extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); getAdmin().namespaces().removeNamespaceEntryFilters(namespace); } } + @Command(description = "Enable dispatcherPauseOnAckStatePersistent for a namespace") + private class SetDispatcherPauseOnAckStatePersistent extends CliCommand { + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; + + @Override + void run() throws PulsarAdminException { + String namespace = validateNamespace(namespaceName); + getAdmin().namespaces().setDispatcherPauseOnAckStatePersistent(namespace); + } + } + + @Command(description = "Get the dispatcherPauseOnAckStatePersistent for a namespace") + private class GetDispatcherPauseOnAckStatePersistent extends CliCommand { + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; + + @Override + void run() throws PulsarAdminException { + String namespace = validateNamespace(namespaceName); + print(getAdmin().namespaces().getDispatcherPauseOnAckStatePersistent(namespace)); + } + } + + @Command(description = "Remove dispatcherPauseOnAckStatePersistent for a namespace") + private class RemoveDispatcherPauseOnAckStatePersistent extends CliCommand { + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; + + @Override + void run() throws PulsarAdminException { + String namespace = validateNamespace(namespaceName); + getAdmin().namespaces().removeDispatcherPauseOnAckStatePersistent(namespace); + } + } + + @Command(description = "Set allowed clusters for a namespace") + private class SetAllowedClusters extends CliCommand { + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; + + @Option(names = { "--clusters", + "-c" }, description = "Replication Cluster Ids list (comma separated values)", required = true) + private String clusterIds; + + @Override + void run() throws PulsarAdminException { + String namespace = validateNamespace(namespaceName); + List clusters = Lists.newArrayList(clusterIds.split(",")); + getAdmin().namespaces().setNamespaceAllowedClusters(namespace, Sets.newHashSet(clusters)); + } + } + + @Command(description = "Get allowed clusters for a namespace") + private class GetAllowedClusters extends CliCommand { + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; + + @Override + void run() throws PulsarAdminException { + String namespace = validateNamespace(namespaceName); + print(getAdmin().namespaces().getNamespaceAllowedClusters(namespace)); + } + } + public CmdNamespaces(Supplier admin) { super("namespaces", admin); - jcommander.addCommand("list", new GetNamespacesPerProperty()); - jcommander.addCommand("list-cluster", new GetNamespacesPerCluster()); + addCommand("list", new GetNamespacesPerProperty()); + addCommand("list-cluster", new GetNamespacesPerCluster()); - jcommander.addCommand("topics", new GetTopics()); - jcommander.addCommand("bundles", new GetBundles()); - jcommander.addCommand("destinations", new GetDestinations()); - jcommander.addCommand("policies", new GetPolicies()); - jcommander.addCommand("create", new Create()); - jcommander.addCommand("delete", new Delete()); + addCommand("topics", new GetTopics()); + addCommand("bundles", new GetBundles()); + addCommand("destinations", new GetDestinations()); + addCommand("policies", new GetPolicies()); + addCommand("create", new Create()); + addCommand("delete", new Delete()); - jcommander.addCommand("permissions", new Permissions()); - jcommander.addCommand("grant-permission", new GrantPermissions()); - jcommander.addCommand("revoke-permission", new RevokePermissions()); + addCommand("permissions", new Permissions()); + addCommand("grant-permission", new GrantPermissions()); + addCommand("revoke-permission", new RevokePermissions()); - jcommander.addCommand("subscription-permission", new SubscriptionPermissions()); - jcommander.addCommand("grant-subscription-permission", new GrantSubscriptionPermissions()); - jcommander.addCommand("revoke-subscription-permission", new RevokeSubscriptionPermissions()); + addCommand("subscription-permission", new SubscriptionPermissions()); + addCommand("grant-subscription-permission", new GrantSubscriptionPermissions()); + addCommand("revoke-subscription-permission", new RevokeSubscriptionPermissions()); - jcommander.addCommand("set-clusters", new SetReplicationClusters()); - jcommander.addCommand("get-clusters", new GetReplicationClusters()); + addCommand("set-clusters", new SetReplicationClusters()); + addCommand("get-clusters", new GetReplicationClusters()); - jcommander.addCommand("set-subscription-types-enabled", new SetSubscriptionTypesEnabled()); - jcommander.addCommand("get-subscription-types-enabled", new GetSubscriptionTypesEnabled()); - jcommander.addCommand("remove-subscription-types-enabled", new RemoveSubscriptionTypesEnabled()); + addCommand("set-subscription-types-enabled", new SetSubscriptionTypesEnabled()); + addCommand("get-subscription-types-enabled", new GetSubscriptionTypesEnabled()); + addCommand("remove-subscription-types-enabled", new RemoveSubscriptionTypesEnabled()); - jcommander.addCommand("get-backlog-quotas", new GetBacklogQuotaMap()); - jcommander.addCommand("set-backlog-quota", new SetBacklogQuota()); - jcommander.addCommand("remove-backlog-quota", new RemoveBacklogQuota()); + addCommand("set-allowed-clusters", new SetAllowedClusters()); + addCommand("get-allowed-clusters", new GetAllowedClusters()); - jcommander.addCommand("get-persistence", new GetPersistence()); - jcommander.addCommand("set-persistence", new SetPersistence()); - jcommander.addCommand("remove-persistence", new RemovePersistence()); + addCommand("get-backlog-quotas", new GetBacklogQuotaMap()); + addCommand("set-backlog-quota", new SetBacklogQuota()); + addCommand("remove-backlog-quota", new RemoveBacklogQuota()); - jcommander.addCommand("get-message-ttl", new GetMessageTTL()); - jcommander.addCommand("set-message-ttl", new SetMessageTTL()); - jcommander.addCommand("remove-message-ttl", new RemoveMessageTTL()); + addCommand("get-persistence", new GetPersistence()); + addCommand("set-persistence", new SetPersistence()); + addCommand("remove-persistence", new RemovePersistence()); - jcommander.addCommand("get-max-subscriptions-per-topic", new GetMaxSubscriptionsPerTopic()); - jcommander.addCommand("set-max-subscriptions-per-topic", new SetMaxSubscriptionsPerTopic()); - jcommander.addCommand("remove-max-subscriptions-per-topic", new RemoveMaxSubscriptionsPerTopic()); + addCommand("get-message-ttl", new GetMessageTTL()); + addCommand("set-message-ttl", new SetMessageTTL()); + addCommand("remove-message-ttl", new RemoveMessageTTL()); - jcommander.addCommand("get-subscription-expiration-time", new GetSubscriptionExpirationTime()); - jcommander.addCommand("set-subscription-expiration-time", new SetSubscriptionExpirationTime()); - jcommander.addCommand("remove-subscription-expiration-time", new RemoveSubscriptionExpirationTime()); + addCommand("get-max-subscriptions-per-topic", new GetMaxSubscriptionsPerTopic()); + addCommand("set-max-subscriptions-per-topic", new SetMaxSubscriptionsPerTopic()); + addCommand("remove-max-subscriptions-per-topic", new RemoveMaxSubscriptionsPerTopic()); - jcommander.addCommand("get-anti-affinity-group", new GetAntiAffinityGroup()); - jcommander.addCommand("set-anti-affinity-group", new SetAntiAffinityGroup()); - jcommander.addCommand("get-anti-affinity-namespaces", new GetAntiAffinityNamespaces()); - jcommander.addCommand("delete-anti-affinity-group", new DeleteAntiAffinityGroup()); + addCommand("get-subscription-expiration-time", new GetSubscriptionExpirationTime()); + addCommand("set-subscription-expiration-time", new SetSubscriptionExpirationTime()); + addCommand("remove-subscription-expiration-time", new RemoveSubscriptionExpirationTime()); - jcommander.addCommand("set-deduplication", new SetDeduplication()); - jcommander.addCommand("get-deduplication", new GetDeduplication()); - jcommander.addCommand("remove-deduplication", new RemoveDeduplication()); + addCommand("get-anti-affinity-group", new GetAntiAffinityGroup()); + addCommand("set-anti-affinity-group", new SetAntiAffinityGroup()); + addCommand("get-anti-affinity-namespaces", new GetAntiAffinityNamespaces()); + addCommand("delete-anti-affinity-group", new DeleteAntiAffinityGroup()); - jcommander.addCommand("set-auto-topic-creation", new SetAutoTopicCreation()); - jcommander.addCommand("get-auto-topic-creation", new GetAutoTopicCreation()); - jcommander.addCommand("remove-auto-topic-creation", new RemoveAutoTopicCreation()); + addCommand("set-deduplication", new SetDeduplication()); + addCommand("get-deduplication", new GetDeduplication()); + addCommand("remove-deduplication", new RemoveDeduplication()); - jcommander.addCommand("set-auto-subscription-creation", new SetAutoSubscriptionCreation()); - jcommander.addCommand("get-auto-subscription-creation", new GetAutoSubscriptionCreation()); - jcommander.addCommand("remove-auto-subscription-creation", new RemoveAutoSubscriptionCreation()); + addCommand("set-auto-topic-creation", new SetAutoTopicCreation()); + addCommand("get-auto-topic-creation", new GetAutoTopicCreation()); + addCommand("remove-auto-topic-creation", new RemoveAutoTopicCreation()); - jcommander.addCommand("get-retention", new GetRetention()); - jcommander.addCommand("set-retention", new SetRetention()); - jcommander.addCommand("remove-retention", new RemoveRetention()); + addCommand("set-auto-subscription-creation", new SetAutoSubscriptionCreation()); + addCommand("get-auto-subscription-creation", new GetAutoSubscriptionCreation()); + addCommand("remove-auto-subscription-creation", new RemoveAutoSubscriptionCreation()); - jcommander.addCommand("set-bookie-affinity-group", new SetBookieAffinityGroup()); - jcommander.addCommand("get-bookie-affinity-group", new GetBookieAffinityGroup()); - jcommander.addCommand("delete-bookie-affinity-group", new DeleteBookieAffinityGroup()); + addCommand("get-retention", new GetRetention()); + addCommand("set-retention", new SetRetention()); + addCommand("remove-retention", new RemoveRetention()); - jcommander.addCommand("unload", new Unload()); + addCommand("set-bookie-affinity-group", new SetBookieAffinityGroup()); + addCommand("get-bookie-affinity-group", new GetBookieAffinityGroup()); + addCommand("delete-bookie-affinity-group", new DeleteBookieAffinityGroup()); - jcommander.addCommand("split-bundle", new SplitBundle()); - jcommander.addCommand("get-topic-positions", new GetTopicHashPositions()); + addCommand("unload", new Unload()); - jcommander.addCommand("set-dispatch-rate", new SetDispatchRate()); - jcommander.addCommand("remove-dispatch-rate", new RemoveDispatchRate()); - jcommander.addCommand("get-dispatch-rate", new GetDispatchRate()); + addCommand("split-bundle", new SplitBundle()); + addCommand("get-topic-positions", new GetTopicHashPositions()); - jcommander.addCommand("set-subscribe-rate", new SetSubscribeRate()); - jcommander.addCommand("get-subscribe-rate", new GetSubscribeRate()); - jcommander.addCommand("remove-subscribe-rate", new RemoveSubscribeRate()); + addCommand("set-dispatch-rate", new SetDispatchRate()); + addCommand("remove-dispatch-rate", new RemoveDispatchRate()); + addCommand("get-dispatch-rate", new GetDispatchRate()); - jcommander.addCommand("set-subscription-dispatch-rate", new SetSubscriptionDispatchRate()); - jcommander.addCommand("get-subscription-dispatch-rate", new GetSubscriptionDispatchRate()); - jcommander.addCommand("remove-subscription-dispatch-rate", new RemoveSubscriptionDispatchRate()); + addCommand("set-subscribe-rate", new SetSubscribeRate()); + addCommand("get-subscribe-rate", new GetSubscribeRate()); + addCommand("remove-subscribe-rate", new RemoveSubscribeRate()); - jcommander.addCommand("set-publish-rate", new SetPublishRate()); - jcommander.addCommand("get-publish-rate", new GetPublishRate()); - jcommander.addCommand("remove-publish-rate", new RemovePublishRate()); + addCommand("set-subscription-dispatch-rate", new SetSubscriptionDispatchRate()); + addCommand("get-subscription-dispatch-rate", new GetSubscriptionDispatchRate()); + addCommand("remove-subscription-dispatch-rate", new RemoveSubscriptionDispatchRate()); - jcommander.addCommand("set-replicator-dispatch-rate", new SetReplicatorDispatchRate()); - jcommander.addCommand("get-replicator-dispatch-rate", new GetReplicatorDispatchRate()); - jcommander.addCommand("remove-replicator-dispatch-rate", new RemoveReplicatorDispatchRate()); + addCommand("set-publish-rate", new SetPublishRate()); + addCommand("get-publish-rate", new GetPublishRate()); + addCommand("remove-publish-rate", new RemovePublishRate()); - jcommander.addCommand("clear-backlog", new ClearBacklog()); + addCommand("set-replicator-dispatch-rate", new SetReplicatorDispatchRate()); + addCommand("get-replicator-dispatch-rate", new GetReplicatorDispatchRate()); + addCommand("remove-replicator-dispatch-rate", new RemoveReplicatorDispatchRate()); - jcommander.addCommand("unsubscribe", new Unsubscribe()); + addCommand("clear-backlog", new ClearBacklog()); - jcommander.addCommand("set-encryption-required", new SetEncryptionRequired()); - jcommander.addCommand("get-encryption-required", new GetEncryptionRequired()); - jcommander.addCommand("set-subscription-auth-mode", new SetSubscriptionAuthMode()); - jcommander.addCommand("get-subscription-auth-mode", new GetSubscriptionAuthMode()); + addCommand("unsubscribe", new Unsubscribe()); - jcommander.addCommand("set-delayed-delivery", new SetDelayedDelivery()); - jcommander.addCommand("get-delayed-delivery", new GetDelayedDelivery()); - jcommander.addCommand("remove-delayed-delivery", new RemoveDelayedDelivery()); + addCommand("set-encryption-required", new SetEncryptionRequired()); + addCommand("get-encryption-required", new GetEncryptionRequired()); + addCommand("set-subscription-auth-mode", new SetSubscriptionAuthMode()); + addCommand("get-subscription-auth-mode", new GetSubscriptionAuthMode()); - jcommander.addCommand("get-inactive-topic-policies", new GetInactiveTopicPolicies()); - jcommander.addCommand("set-inactive-topic-policies", new SetInactiveTopicPolicies()); - jcommander.addCommand("remove-inactive-topic-policies", new RemoveInactiveTopicPolicies()); + addCommand("set-delayed-delivery", new SetDelayedDelivery()); + addCommand("get-delayed-delivery", new GetDelayedDelivery()); + addCommand("remove-delayed-delivery", new RemoveDelayedDelivery()); - jcommander.addCommand("get-max-producers-per-topic", new GetMaxProducersPerTopic()); - jcommander.addCommand("set-max-producers-per-topic", new SetMaxProducersPerTopic()); - jcommander.addCommand("remove-max-producers-per-topic", new RemoveMaxProducersPerTopic()); + addCommand("get-inactive-topic-policies", new GetInactiveTopicPolicies()); + addCommand("set-inactive-topic-policies", new SetInactiveTopicPolicies()); + addCommand("remove-inactive-topic-policies", new RemoveInactiveTopicPolicies()); - jcommander.addCommand("get-max-consumers-per-topic", new GetMaxConsumersPerTopic()); - jcommander.addCommand("set-max-consumers-per-topic", new SetMaxConsumersPerTopic()); - jcommander.addCommand("remove-max-consumers-per-topic", new RemoveMaxConsumersPerTopic()); + addCommand("get-max-producers-per-topic", new GetMaxProducersPerTopic()); + addCommand("set-max-producers-per-topic", new SetMaxProducersPerTopic()); + addCommand("remove-max-producers-per-topic", new RemoveMaxProducersPerTopic()); - jcommander.addCommand("get-max-consumers-per-subscription", new GetMaxConsumersPerSubscription()); - jcommander.addCommand("set-max-consumers-per-subscription", new SetMaxConsumersPerSubscription()); - jcommander.addCommand("remove-max-consumers-per-subscription", new RemoveMaxConsumersPerSubscription()); + addCommand("get-max-consumers-per-topic", new GetMaxConsumersPerTopic()); + addCommand("set-max-consumers-per-topic", new SetMaxConsumersPerTopic()); + addCommand("remove-max-consumers-per-topic", new RemoveMaxConsumersPerTopic()); - jcommander.addCommand("get-max-unacked-messages-per-subscription", new GetMaxUnackedMessagesPerSubscription()); - jcommander.addCommand("set-max-unacked-messages-per-subscription", new SetMaxUnackedMessagesPerSubscription()); - jcommander.addCommand("remove-max-unacked-messages-per-subscription", + addCommand("get-max-consumers-per-subscription", new GetMaxConsumersPerSubscription()); + addCommand("set-max-consumers-per-subscription", new SetMaxConsumersPerSubscription()); + addCommand("remove-max-consumers-per-subscription", new RemoveMaxConsumersPerSubscription()); + + addCommand("get-max-unacked-messages-per-subscription", new GetMaxUnackedMessagesPerSubscription()); + addCommand("set-max-unacked-messages-per-subscription", new SetMaxUnackedMessagesPerSubscription()); + addCommand("remove-max-unacked-messages-per-subscription", new RemoveMaxUnackedMessagesPerSubscription()); - jcommander.addCommand("get-max-unacked-messages-per-consumer", new GetMaxUnackedMessagesPerConsumer()); - jcommander.addCommand("set-max-unacked-messages-per-consumer", new SetMaxUnackedMessagesPerConsumer()); - jcommander.addCommand("remove-max-unacked-messages-per-consumer", new RemoveMaxUnackedMessagesPerConsumer()); + addCommand("get-max-unacked-messages-per-consumer", new GetMaxUnackedMessagesPerConsumer()); + addCommand("set-max-unacked-messages-per-consumer", new SetMaxUnackedMessagesPerConsumer()); + addCommand("remove-max-unacked-messages-per-consumer", new RemoveMaxUnackedMessagesPerConsumer()); + + addCommand("get-compaction-threshold", new GetCompactionThreshold()); + addCommand("set-compaction-threshold", new SetCompactionThreshold()); + addCommand("remove-compaction-threshold", new RemoveCompactionThreshold()); + + addCommand("get-offload-threshold", new GetOffloadThreshold()); + addCommand("set-offload-threshold", new SetOffloadThreshold()); - jcommander.addCommand("get-compaction-threshold", new GetCompactionThreshold()); - jcommander.addCommand("set-compaction-threshold", new SetCompactionThreshold()); - jcommander.addCommand("remove-compaction-threshold", new RemoveCompactionThreshold()); + addCommand("get-offload-deletion-lag", new GetOffloadDeletionLag()); + addCommand("set-offload-deletion-lag", new SetOffloadDeletionLag()); + addCommand("clear-offload-deletion-lag", new ClearOffloadDeletionLag()); - jcommander.addCommand("get-offload-threshold", new GetOffloadThreshold()); - jcommander.addCommand("set-offload-threshold", new SetOffloadThreshold()); + addCommand("get-schema-autoupdate-strategy", new GetSchemaAutoUpdateStrategy()); + addCommand("set-schema-autoupdate-strategy", new SetSchemaAutoUpdateStrategy()); - jcommander.addCommand("get-offload-deletion-lag", new GetOffloadDeletionLag()); - jcommander.addCommand("set-offload-deletion-lag", new SetOffloadDeletionLag()); - jcommander.addCommand("clear-offload-deletion-lag", new ClearOffloadDeletionLag()); + addCommand("get-schema-compatibility-strategy", new GetSchemaCompatibilityStrategy()); + addCommand("set-schema-compatibility-strategy", new SetSchemaCompatibilityStrategy()); - jcommander.addCommand("get-schema-autoupdate-strategy", new GetSchemaAutoUpdateStrategy()); - jcommander.addCommand("set-schema-autoupdate-strategy", new SetSchemaAutoUpdateStrategy()); + addCommand("get-is-allow-auto-update-schema", new GetIsAllowAutoUpdateSchema()); + addCommand("set-is-allow-auto-update-schema", new SetIsAllowAutoUpdateSchema()); - jcommander.addCommand("get-schema-compatibility-strategy", new GetSchemaCompatibilityStrategy()); - jcommander.addCommand("set-schema-compatibility-strategy", new SetSchemaCompatibilityStrategy()); + addCommand("get-schema-validation-enforce", new GetSchemaValidationEnforced()); + addCommand("set-schema-validation-enforce", new SetSchemaValidationEnforced()); - jcommander.addCommand("get-is-allow-auto-update-schema", new GetIsAllowAutoUpdateSchema()); - jcommander.addCommand("set-is-allow-auto-update-schema", new SetIsAllowAutoUpdateSchema()); + addCommand("set-offload-policies", new SetOffloadPolicies()); + addCommand("remove-offload-policies", new RemoveOffloadPolicies()); + addCommand("get-offload-policies", new GetOffloadPolicies()); - jcommander.addCommand("get-schema-validation-enforce", new GetSchemaValidationEnforced()); - jcommander.addCommand("set-schema-validation-enforce", new SetSchemaValidationEnforced()); + addCommand("set-deduplication-snapshot-interval", new SetDeduplicationSnapshotInterval()); + addCommand("get-deduplication-snapshot-interval", new GetDeduplicationSnapshotInterval()); + addCommand("remove-deduplication-snapshot-interval", new RemoveDeduplicationSnapshotInterval()); - jcommander.addCommand("set-offload-policies", new SetOffloadPolicies()); - jcommander.addCommand("remove-offload-policies", new RemoveOffloadPolicies()); - jcommander.addCommand("get-offload-policies", new GetOffloadPolicies()); + addCommand("set-max-topics-per-namespace", new SetMaxTopicsPerNamespace()); + addCommand("get-max-topics-per-namespace", new GetMaxTopicsPerNamespace()); + addCommand("remove-max-topics-per-namespace", new RemoveMaxTopicsPerNamespace()); - jcommander.addCommand("set-deduplication-snapshot-interval", new SetDeduplicationSnapshotInterval()); - jcommander.addCommand("get-deduplication-snapshot-interval", new GetDeduplicationSnapshotInterval()); - jcommander.addCommand("remove-deduplication-snapshot-interval", new RemoveDeduplicationSnapshotInterval()); + addCommand("set-property", new SetPropertyForNamespace()); + addCommand("get-property", new GetPropertyForNamespace()); + addCommand("remove-property", new RemovePropertyForNamespace()); + addCommand("set-properties", new SetPropertiesForNamespace()); + addCommand("get-properties", new GetPropertiesForNamespace()); + addCommand("clear-properties", new ClearPropertiesForNamespace()); - jcommander.addCommand("set-max-topics-per-namespace", new SetMaxTopicsPerNamespace()); - jcommander.addCommand("get-max-topics-per-namespace", new GetMaxTopicsPerNamespace()); - jcommander.addCommand("remove-max-topics-per-namespace", new RemoveMaxTopicsPerNamespace()); + addCommand("get-resource-group", new GetResourceGroup()); + addCommand("set-resource-group", new SetResourceGroup()); + addCommand("remove-resource-group", new RemoveResourceGroup()); - jcommander.addCommand("set-property", new SetPropertyForNamespace()); - jcommander.addCommand("get-property", new GetPropertyForNamespace()); - jcommander.addCommand("remove-property", new RemovePropertyForNamespace()); - jcommander.addCommand("set-properties", new SetPropertiesForNamespace()); - jcommander.addCommand("get-properties", new GetPropertiesForNamespace()); - jcommander.addCommand("clear-properties", new ClearPropertiesForNamespace()); + addCommand("get-entry-filters", new GetEntryFiltersPerTopic()); + addCommand("set-entry-filters", new SetEntryFiltersPerTopic()); + addCommand("remove-entry-filters", new RemoveEntryFiltersPerTopic()); - jcommander.addCommand("get-resource-group", new GetResourceGroup()); - jcommander.addCommand("set-resource-group", new SetResourceGroup()); - jcommander.addCommand("remove-resource-group", new RemoveResourceGroup()); + addCommand("update-migration-state", new UpdateMigrationState()); - jcommander.addCommand("get-entry-filters", new GetEntryFiltersPerTopic()); - jcommander.addCommand("set-entry-filters", new SetEntryFiltersPerTopic()); - jcommander.addCommand("remove-entry-filters", new RemoveEntryFiltersPerTopic()); + addCommand("set-dispatcher-pause-on-ack-state-persistent", + new SetDispatcherPauseOnAckStatePersistent()); + addCommand("get-dispatcher-pause-on-ack-state-persistent", + new GetDispatcherPauseOnAckStatePersistent()); + addCommand("remove-dispatcher-pause-on-ack-state-persistent", + new RemoveDispatcherPauseOnAckStatePersistent()); } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNonPersistentTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNonPersistentTopics.java index c344a76853c1d..cb75ea345787c 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNonPersistentTopics.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNonPersistentTopics.java @@ -18,28 +18,29 @@ */ package org.apache.pulsar.admin.cli; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; import java.util.function.Supplier; import org.apache.pulsar.client.admin.NonPersistentTopics; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; @SuppressWarnings("deprecation") -@Parameters(commandDescription = "Operations on non-persistent topics", hidden = true) +@Command(description = "Operations on non-persistent topics", hidden = true) public class CmdNonPersistentTopics extends CmdBase { private NonPersistentTopics nonPersistentTopics; public CmdNonPersistentTopics(Supplier admin) { super("non-persistent", admin); - jcommander.addCommand("create-partitioned-topic", new CreatePartitionedCmd()); - jcommander.addCommand("lookup", new Lookup()); - jcommander.addCommand("stats", new GetStats()); - jcommander.addCommand("stats-internal", new GetInternalStats()); - jcommander.addCommand("get-partitioned-topic-metadata", new GetPartitionedTopicMetadataCmd()); - jcommander.addCommand("list", new GetList()); - jcommander.addCommand("list-in-bundle", new GetListInBundle()); + addCommand("create-partitioned-topic", new CreatePartitionedCmd()); + addCommand("lookup", new Lookup()); + addCommand("stats", new GetStats()); + addCommand("stats-internal", new GetInternalStats()); + addCommand("get-partitioned-topic-metadata", new GetPartitionedTopicMetadataCmd()); + addCommand("list", new GetList()); + addCommand("list-in-bundle", new GetListInBundle()); } private NonPersistentTopics getNonPersistentTopics() { @@ -49,99 +50,97 @@ private NonPersistentTopics getNonPersistentTopics() { return nonPersistentTopics; } - @Parameters(commandDescription = "Lookup a topic from the current serving broker") + @Command(description = "Lookup a topic from the current serving broker") private class Lookup extends CliCommand { - @Parameter(description = "non-persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "non-persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); print(getAdmin().lookups().lookupTopic(topic)); } } - @Parameters(commandDescription = "Get the stats for the topic and its connected producers and consumers. " + @Command(description = "Get the stats for the topic and its connected producers and consumers. " + "All the rates are computed over a 1 minute window and are relative the last completed 1 minute period.") private class GetStats extends CliCommand { - @Parameter(description = "non-persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "non-persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validateNonPersistentTopic(params); + String persistentTopic = validateNonPersistentTopic(topicName); print(getNonPersistentTopics().getStats(persistentTopic)); } } - @Parameters(commandDescription = "Get the internal stats for the topic") + @Command(description = "Get the internal stats for the topic") private class GetInternalStats extends CliCommand { - @Parameter(description = "non-persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "non-persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validateNonPersistentTopic(params); + String persistentTopic = validateNonPersistentTopic(topicName); print(getNonPersistentTopics().getInternalStats(persistentTopic)); } } - @Parameters(commandDescription = "Create a partitioned topic. " + @Command(description = "Create a partitioned topic. " + "The partitioned topic has to be created before creating a producer on it.") private class CreatePartitionedCmd extends CliCommand { - @Parameter(description = "non-persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "non-persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-p", + @Option(names = { "-p", "--partitions" }, description = "Number of partitions for the topic", required = true) private int numPartitions; @Override void run() throws Exception { - String persistentTopic = validateNonPersistentTopic(params); + String persistentTopic = validateNonPersistentTopic(topicName); getNonPersistentTopics().createPartitionedTopic(persistentTopic, numPartitions); } } - @Parameters(commandDescription = "Get the partitioned topic metadata. " + @Command(description = "Get the partitioned topic metadata. " + "If the topic is not created or is a non-partitioned topic, it returns empty topic with 0 partitions") private class GetPartitionedTopicMetadataCmd extends CliCommand { - @Parameter(description = "non-persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "non-persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws Exception { - String persistentTopic = validateNonPersistentTopic(params); - print(getNonPersistentTopics().getPartitionedTopicMetadata(persistentTopic)); + String nonPersistentTopic = validateNonPersistentTopic(topicName); + print(getNonPersistentTopics().getPartitionedTopicMetadata(nonPersistentTopic)); } } - @Parameters(commandDescription = "Get list of non-persistent topics present under a namespace") + @Command(description = "Get list of non-persistent topics present under a namespace") private class GetList extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespace; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); print(getNonPersistentTopics().getList(namespace)); } } - @Parameters(commandDescription = "Get list of non-persistent topics present under a namespace bundle") + @Command(description = "Get list of non-persistent topics present under a namespace bundle") private class GetListInBundle extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespace; - @Parameter(names = { "-b", + @Option(names = { "-b", "--bundle" }, description = "bundle range", required = true) private String bundleRange; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); print(getNonPersistentTopics().getListInBundle(namespace, bundleRange)); } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPackages.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPackages.java index 24214a89c57c3..68547d233460a 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPackages.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPackages.java @@ -18,20 +18,20 @@ */ package org.apache.pulsar.admin.cli; -import com.beust.jcommander.DynamicParameter; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; import java.util.HashMap; import java.util.Map; import java.util.function.Supplier; import org.apache.pulsar.client.admin.Packages; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.packages.management.core.common.PackageMetadata; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; /** * Commands for administering packages. */ -@Parameters(commandDescription = "Operations about packages") +@Command(description = "Operations about packages") class CmdPackages extends CmdBase { private Packages packages; @@ -40,13 +40,13 @@ public CmdPackages(Supplier admin) { super("packages", admin); - jcommander.addCommand("get-metadata", new GetMetadataCmd()); - jcommander.addCommand("update-metadata", new UpdateMetadataCmd()); - jcommander.addCommand("upload", new UploadCmd()); - jcommander.addCommand("download", new DownloadCmd()); - jcommander.addCommand("list", new ListPackagesCmd()); - jcommander.addCommand("list-versions", new ListPackageVersionsCmd()); - jcommander.addCommand("delete", new DeletePackageCmd()); + addCommand("get-metadata", new GetMetadataCmd()); + addCommand("update-metadata", new UpdateMetadataCmd()); + addCommand("upload", new UploadCmd()); + addCommand("download", new DownloadCmd()); + addCommand("list", new ListPackagesCmd()); + addCommand("list-versions", new ListPackageVersionsCmd()); + addCommand("delete", new DeletePackageCmd()); } private Packages getPackages() { @@ -56,9 +56,9 @@ private Packages getPackages() { return packages; } - @Parameters(commandDescription = "Get a package metadata information.") + @Command(description = "Get a package metadata information.") private class GetMetadataCmd extends CliCommand { - @Parameter(description = "type://tenant/namespace/packageName@version", required = true) + @Parameters(description = "type://tenant/namespace/packageName@version", arity = "1") private String packageName; @Override @@ -67,18 +67,18 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Update a package metadata information.") + @Command(description = "Update a package metadata information.") private class UpdateMetadataCmd extends CliCommand { - @Parameter(description = "type://tenant/namespace/packageName@version", required = true) + @Parameters(description = "type://tenant/namespace/packageName@version", arity = "1") private String packageName; - @Parameter(names = {"-d", "--description"}, description = "descriptions of a package", required = true) + @Option(names = {"-d", "--description"}, description = "descriptions of a package", required = true) private String description; - @Parameter(names = {"-c", "--contact"}, description = "contact info of a package") + @Option(names = {"-c", "--contact"}, description = "contact info of a package") private String contact; - @DynamicParameter(names = {"--properties", "-P"}, description = "external information of a package") + @Option(names = {"--properties", "-P"}, description = "external information of a package") private Map properties = new HashMap<>(); @Override @@ -89,21 +89,21 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Upload a package") + @Command(description = "Upload a package") private class UploadCmd extends CliCommand { - @Parameter(description = "type://tenant/namespace/packageName@version", required = true) + @Parameters(description = "type://tenant/namespace/packageName@version", arity = "1") private String packageName; - @Parameter(names = "--description", description = "descriptions of a package", required = true) + @Option(names = "--description", description = "descriptions of a package", required = true) private String description; - @Parameter(names = "--contact", description = "contact information of a package") + @Option(names = "--contact", description = "contact information of a package") private String contact; - @DynamicParameter(names = {"--properties", "-P"}, description = "external information of a package") + @Option(names = {"--properties", "-P"}, description = "external information of a package") private Map properties = new HashMap<>(); - @Parameter(names = "--path", description = "file path of the package", required = true) + @Option(names = "--path", description = "file path of the package", required = true) private String path; @Override @@ -117,12 +117,12 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Download a package") + @Command(description = "Download a package") private class DownloadCmd extends CliCommand { - @Parameter(description = "type://tenant/namespace/packageName@version", required = true) + @Parameters(description = "type://tenant/namespace/packageName@version", arity = "1") private String packageName; - @Parameter(names = "--path", description = "download destiny path of the package", required = true) + @Option(names = "--path", description = "download destiny path of the package", required = true) private String path; @Override @@ -132,10 +132,10 @@ void run() throws Exception { } } - @Parameters(commandDescription = "List all versions of the given package") + @Command(description = "List all versions of the given package") private class ListPackageVersionsCmd extends CliCommand { - @Parameter(description = "the package name you want to query, don't need to specify the package version. " - + "type://tenant/namespace/packageName", required = true) + @Parameters(description = "the package name you want to query, don't need to specify the package version. " + + "type://tenant/namespace/packageName", arity = "1") private String packageName; @Override @@ -144,12 +144,12 @@ void run() throws Exception { } } - @Parameters(commandDescription = "List all packages with given type in the specified namespace") + @Command(description = "List all packages with given type in the specified namespace") private class ListPackagesCmd extends CliCommand { - @Parameter(names = "--type", description = "type of the package", required = true) + @Option(names = "--type", description = "type of the package", required = true) private String type; - @Parameter(description = "namespace of the package", required = true) + @Parameters(description = "namespace of the package", arity = "1") private String namespace; @Override @@ -158,9 +158,9 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Delete a package") - private class DeletePackageCmd extends CliCommand{ - @Parameter(description = "type://tenant/namespace/packageName@version", required = true) + @Command(description = "Delete a package") + private class DeletePackageCmd extends CliCommand { + @Parameters(description = "type://tenant/namespace/packageName@version", arity = "1") private String packageName; @Override diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java index 3c1662d00a034..3dc0ba7b6f24a 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java @@ -19,9 +19,6 @@ package org.apache.pulsar.admin.cli; import static org.apache.commons.lang3.StringUtils.isNotBlank; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import io.netty.buffer.ByteBuf; @@ -29,21 +26,24 @@ import io.netty.buffer.Unpooled; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import org.apache.pulsar.cli.converters.picocli.TimeUnitToMillisConverter; +import org.apache.pulsar.cli.converters.picocli.TimeUnitToSecondsConverter; import org.apache.pulsar.client.admin.LongRunningProcessStatus; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.admin.Topics; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; -import org.apache.pulsar.client.cli.NoSplitter; import org.apache.pulsar.client.impl.BatchMessageIdImpl; import org.apache.pulsar.client.impl.MessageIdImpl; -import org.apache.pulsar.common.util.RelativeTimeUtil; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; -@Parameters(commandDescription = "Operations on persistent topics. The persistent-topics " +@Command(description = "Operations on persistent topics. The persistent-topics " + "has been deprecated in favor of topics", hidden = true) public class CmdPersistentTopics extends CmdBase { private Topics persistentTopics; @@ -51,39 +51,39 @@ public class CmdPersistentTopics extends CmdBase { public CmdPersistentTopics(Supplier admin) { super("persistent", admin); - jcommander.addCommand("list", new ListCmd()); - jcommander.addCommand("list-partitioned-topics", new PartitionedTopicListCmd()); - jcommander.addCommand("permissions", new Permissions()); - jcommander.addCommand("grant-permission", new GrantPermissions()); - jcommander.addCommand("revoke-permission", new RevokePermissions()); - jcommander.addCommand("lookup", new Lookup()); - jcommander.addCommand("bundle-range", new GetBundleRange()); - jcommander.addCommand("delete", new DeleteCmd()); - jcommander.addCommand("unload", new UnloadCmd()); - jcommander.addCommand("truncate", new TruncateCmd()); - jcommander.addCommand("subscriptions", new ListSubscriptions()); - jcommander.addCommand("unsubscribe", new DeleteSubscription()); - jcommander.addCommand("create-subscription", new CreateSubscription()); - jcommander.addCommand("stats", new GetStats()); - jcommander.addCommand("stats-internal", new GetInternalStats()); - jcommander.addCommand("info-internal", new GetInternalInfo()); - jcommander.addCommand("partitioned-stats", new GetPartitionedStats()); - jcommander.addCommand("partitioned-stats-internal", new GetPartitionedStatsInternal()); - jcommander.addCommand("skip", new Skip()); - jcommander.addCommand("skip-all", new SkipAll()); - jcommander.addCommand("expire-messages", new ExpireMessages()); - jcommander.addCommand("expire-messages-all-subscriptions", new ExpireMessagesForAllSubscriptions()); - jcommander.addCommand("create-partitioned-topic", new CreatePartitionedCmd()); - jcommander.addCommand("update-partitioned-topic", new UpdatePartitionedCmd()); - jcommander.addCommand("get-partitioned-topic-metadata", new GetPartitionedTopicMetadataCmd()); - jcommander.addCommand("delete-partitioned-topic", new DeletePartitionedCmd()); - jcommander.addCommand("peek-messages", new PeekMessages()); - jcommander.addCommand("get-message-by-id", new GetMessageById()); - jcommander.addCommand("last-message-id", new GetLastMessageId()); - jcommander.addCommand("reset-cursor", new ResetCursor()); - jcommander.addCommand("terminate", new Terminate()); - jcommander.addCommand("compact", new Compact()); - jcommander.addCommand("compaction-status", new CompactionStatusCmd()); + addCommand("list", new ListCmd()); + addCommand("list-partitioned-topics", new PartitionedTopicListCmd()); + addCommand("permissions", new Permissions()); + addCommand("grant-permission", new GrantPermissions()); + addCommand("revoke-permission", new RevokePermissions()); + addCommand("lookup", new Lookup()); + addCommand("bundle-range", new GetBundleRange()); + addCommand("delete", new DeleteCmd()); + addCommand("unload", new UnloadCmd()); + addCommand("truncate", new TruncateCmd()); + addCommand("subscriptions", new ListSubscriptions()); + addCommand("unsubscribe", new DeleteSubscription()); + addCommand("create-subscription", new CreateSubscription()); + addCommand("stats", new GetStats()); + addCommand("stats-internal", new GetInternalStats()); + addCommand("info-internal", new GetInternalInfo()); + addCommand("partitioned-stats", new GetPartitionedStats()); + addCommand("partitioned-stats-internal", new GetPartitionedStatsInternal()); + addCommand("skip", new Skip()); + addCommand("skip-all", new SkipAll()); + addCommand("expire-messages", new ExpireMessages()); + addCommand("expire-messages-all-subscriptions", new ExpireMessagesForAllSubscriptions()); + addCommand("create-partitioned-topic", new CreatePartitionedCmd()); + addCommand("update-partitioned-topic", new UpdatePartitionedCmd()); + addCommand("get-partitioned-topic-metadata", new GetPartitionedTopicMetadataCmd()); + addCommand("delete-partitioned-topic", new DeletePartitionedCmd()); + addCommand("peek-messages", new PeekMessages()); + addCommand("get-message-by-id", new GetMessageById()); + addCommand("last-message-id", new GetLastMessageId()); + addCommand("reset-cursor", new ResetCursor()); + addCommand("terminate", new Terminate()); + addCommand("compact", new Compact()); + addCommand("compaction-status", new CompactionStatusCmd()); } private Topics getPersistentTopics() { @@ -93,430 +93,420 @@ private Topics getPersistentTopics() { return persistentTopics; } - @Parameters(commandDescription = "Get the list of topics under a namespace.") + @Command(description = "Get the list of topics under a namespace.") private class ListCmd extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getPersistentTopics().getList(namespace)); } } - @Parameters(commandDescription = "Get the list of partitioned topics under a namespace.") + @Command(description = "Get the list of partitioned topics under a namespace.") private class PartitionedTopicListCmd extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); print(getPersistentTopics().getPartitionedTopicList(namespace)); } } - @Parameters(commandDescription = "Grant a new permission to a client role on a single topic.") + @Command(description = "Grant a new permission to a client role on a single topic.") private class GrantPermissions extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = "--role", description = "Client role to which grant permissions", required = true) + @Option(names = "--role", description = "Client role to which grant permissions", required = true) private String role; - @Parameter(names = "--actions", description = "Actions to be granted (produce,consume,sources,sinks," + @Option(names = "--actions", description = "Actions to be granted (produce,consume,sources,sinks," + "functions,packages)", required = true) private List actions; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); getPersistentTopics().grantPermission(topic, role, getAuthActions(actions)); } } - @Parameters(commandDescription = "Revoke permissions on a topic. " + @Command(description = "Revoke permissions on a topic. " + "Revoke permissions to a client role on a single topic. If the permission " + "was not set at the topic level, but rather at the namespace level, this " + "operation will return an error (HTTP status code 412).") private class RevokePermissions extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = "--role", description = "Client role to which revoke permissions", required = true) + @Option(names = "--role", description = "Client role to which revoke permissions", required = true) private String role; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); getPersistentTopics().revokePermissions(topic, role); } } - @Parameters(commandDescription = "Get the permissions on a topic. " + @Command(description = "Get the permissions on a topic. " + "Retrieve the effective permissions for a topic. These permissions are defined " + "by the permissions set at the namespace level combined (union) with any eventual " + "specific permission set on the topic.") private class Permissions extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); print(getPersistentTopics().getPermissions(topic)); } } - @Parameters(commandDescription = "Lookup a topic from the current serving broker") + @Command(description = "Lookup a topic from the current serving broker") private class Lookup extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); print(getAdmin().lookups().lookupTopic(topic)); } } - @Parameters(commandDescription = "Get Namespace bundle range of a topic") + @Command(description = "Get Namespace bundle range of a topic") private class GetBundleRange extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); print(getAdmin().lookups().getBundleRange(topic)); } } - @Parameters(commandDescription = "Create a partitioned topic. " + @Command(description = "Create a partitioned topic. " + "The partitioned topic has to be created before creating a producer on it.") private class CreatePartitionedCmd extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-p", + @Option(names = { "-p", "--partitions" }, description = "Number of partitions for the topic", required = true) private int numPartitions; @Override void run() throws Exception { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getPersistentTopics().createPartitionedTopic(persistentTopic, numPartitions); } } - @Parameters(commandDescription = "Update existing partitioned topic. " + @Command(description = "Update existing partitioned topic. " + "New updating number of partitions must be greater than existing number of partitions.") private class UpdatePartitionedCmd extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-p", + @Option(names = { "-p", "--partitions" }, description = "Number of partitions for the topic", required = true) private int numPartitions; @Override void run() throws Exception { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getPersistentTopics().updatePartitionedTopic(persistentTopic, numPartitions); } } - @Parameters(commandDescription = "Get the partitioned topic metadata. " + @Command(description = "Get the partitioned topic metadata. " + "If the topic is not created or is a non-partitioned topic, it returns empty topic with 0 partitions") private class GetPartitionedTopicMetadataCmd extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws Exception { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getPersistentTopics().getPartitionedTopicMetadata(persistentTopic)); } } - @Parameters(commandDescription = "Delete a partitioned topic. " + @Command(description = "Delete a partitioned topic. " + "It will also delete all the partitions of the topic if it exists.") private class DeletePartitionedCmd extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = "--force", + @Option(names = "--force", description = "Close all producer/consumer/replicator and delete topic forcefully") private boolean force = false; @Override void run() throws Exception { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getPersistentTopics().deletePartitionedTopic(persistentTopic, force); } } - @Parameters(commandDescription = "Delete a topic. " + @Command(description = "Delete a topic. " + "The topic cannot be deleted if there's any active subscription or producers connected to it.") private class DeleteCmd extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = "--force", + @Option(names = "--force", description = "Close all producer/consumer/replicator and delete topic forcefully") private boolean force = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getPersistentTopics().delete(persistentTopic, force); } } - @Parameters(commandDescription = "Unload a topic.") + @Command(description = "Unload a topic.") private class UnloadCmd extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getPersistentTopics().unload(persistentTopic); } } - @Parameters(commandDescription = "Truncate a topic. \n\t\tThe truncate operation will move all cursors to the end " + @Command(description = "Truncate a topic. \n\t\tThe truncate operation will move all cursors to the end " + "of the topic and delete all inactive ledgers. ") private class TruncateCmd extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic\n", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); getPersistentTopics().truncate(topic); } } - @Parameters(commandDescription = "Get the list of subscriptions on the topic") + @Command(description = "Get the list of subscriptions on the topic") private class ListSubscriptions extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws Exception { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getPersistentTopics().getSubscriptions(persistentTopic)); } } - @Parameters(commandDescription = "Delete a durable subscriber from a topic. " + @Command(description = "Delete a durable subscriber from a topic. " + "The subscription cannot be deleted if there are any active consumers attached to it") private class DeleteSubscription extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-f", + @Option(names = { "-f", "--force" }, description = "Disconnect and close all consumers and delete subscription forcefully") private boolean force = false; - @Parameter(names = { "-s", "--subscription" }, description = "Subscription to be deleted", required = true) + @Option(names = { "-s", "--subscription" }, description = "Subscription to be deleted", required = true) private String subName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getPersistentTopics().deleteSubscription(persistentTopic, subName, force); } } - @Parameters(commandDescription = "Get the stats for the topic and its connected producers and consumers. " + @Command(description = "Get the stats for the topic and its connected producers and consumers. " + "All the rates are computed over a 1 minute window and are relative the last completed 1 minute period.") private class GetStats extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-gpb", "--get-precise-backlog" }, description = "Set true to get precise backlog") + @Option(names = { "-gpb", "--get-precise-backlog" }, description = "Set true to get precise backlog") private boolean getPreciseBacklog = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getPersistentTopics().getStats(persistentTopic, getPreciseBacklog)); } } - @Parameters(commandDescription = "Get the internal stats for the topic") + @Command(description = "Get the internal stats for the topic") private class GetInternalStats extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-m", + @Option(names = { "-m", "--metadata" }, description = "Flag to include ledger metadata") private boolean metadata = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getPersistentTopics().getInternalStats(persistentTopic, metadata)); } } - @Parameters(commandDescription = "Get the internal metadata info for the topic") + @Command(description = "Get the internal metadata info for the topic") private class GetInternalInfo extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); String result = getPersistentTopics().getInternalInfo(persistentTopic); Gson gson = new GsonBuilder().setPrettyPrinting().create(); System.out.println(gson.toJson(result)); } } - @Parameters(commandDescription = "Get the stats for the partitioned topic and " + @Command(description = "Get the stats for the partitioned topic and " + "its connected producers and consumers. All the rates are computed over a 1 minute window and " + "are relative the last completed 1 minute period.") private class GetPartitionedStats extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = "--per-partition", description = "Get per partition stats") + @Option(names = "--per-partition", description = "Get per partition stats") private boolean perPartition = false; @Override void run() throws Exception { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getPersistentTopics().getPartitionedStats(persistentTopic, perPartition)); } } - @Parameters(commandDescription = "Get the stats-internal for the partitioned topic and " + @Command(description = "Get the stats-internal for the partitioned topic and " + "its connected producers and consumers. All the rates are computed over a 1 minute window and " + "are relative the last completed 1 minute period.") private class GetPartitionedStatsInternal extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws Exception { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getPersistentTopics().getPartitionedInternalStats(persistentTopic)); } } - @Parameters(commandDescription = "Skip all the messages for the subscription") + @Command(description = "Skip all the messages for the subscription") private class SkipAll extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-s", "--subscription" }, description = "Subscription to be cleared", required = true) + @Option(names = { "-s", "--subscription" }, description = "Subscription to be cleared", required = true) private String subName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getPersistentTopics().skipAllMessages(persistentTopic, subName); } } - @Parameters(commandDescription = "Skip some messages for the subscription") + @Command(description = "Skip some messages for the subscription") private class Skip extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-s", + @Option(names = { "-s", "--subscription" }, description = "Subscription to be skip messages on", required = true) private String subName; - @Parameter(names = { "-n", "--count" }, description = "Number of messages to skip", required = true) + @Option(names = { "-n", "--count" }, description = "Number of messages to skip", required = true) private long numMessages; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getPersistentTopics().skipMessages(persistentTopic, subName, numMessages); } } - @Parameters(commandDescription = "Expire messages that older than given expiry time (in seconds) " + @Command(description = "Expire messages that older than given expiry time (in seconds) " + "for the subscription") private class ExpireMessages extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-s", + @Option(names = { "-s", "--subscription" }, description = "Subscription to be skip messages on", required = true) private String subName; - @Parameter(names = { "-t", "--expireTime" }, description = "Expire messages older than time in seconds " - + "(or minutes, hours, days, weeks eg: 100m, 3h, 2d, 5w)", required = true) - private String expireTimeStr; + @Option(names = { "-t", "--expireTime" }, description = "Expire messages older than time in seconds " + + "(or minutes, hours, days, weeks eg: 100m, 3h, 2d, 5w)", required = true, + converter = TimeUnitToSecondsConverter.class) + private Long expireTimeInSeconds; @Override void run() throws PulsarAdminException { - long expireTimeInSeconds; - try { - expireTimeInSeconds = RelativeTimeUtil.parseRelativeTimeInSeconds(expireTimeStr); - } catch (IllegalArgumentException e) { - throw new ParameterException(e.getMessage()); - } - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getPersistentTopics().expireMessages(persistentTopic, subName, expireTimeInSeconds); } } - @Parameters(commandDescription = "Expire messages that older than given expiry time (in seconds) " + @Command(description = "Expire messages that older than given expiry time (in seconds) " + "for all subscriptions") private class ExpireMessagesForAllSubscriptions extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-t", "--expireTime" }, description = "Expire messages older than time in seconds " - + "(or minutes, hours, days, weeks eg: 100m, 3h, 2d, 5w)", required = true) - private String expireTimeStr; + @Option(names = {"-t", "--expireTime"}, description = "Expire messages older than time in seconds " + + "(or minutes, hours, days, weeks eg: 100m, 3h, 2d, 5w)", required = true, + converter = TimeUnitToSecondsConverter.class) + private Long expireTimeInSeconds; @Override void run() throws PulsarAdminException { - long expireTimeInSeconds; - try { - expireTimeInSeconds = RelativeTimeUtil.parseRelativeTimeInSeconds(expireTimeStr); - } catch (IllegalArgumentException e) { - throw new ParameterException(e.getMessage()); - } - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getPersistentTopics().expireMessagesForAllSubscriptions(persistentTopic, expireTimeInSeconds); } } - @Parameters(commandDescription = "Create a new subscription on a topic") + @Command(description = "Create a new subscription on a topic") private class CreateSubscription extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-s", + @Option(names = { "-s", "--subscription" }, description = "Subscription name", required = true) private String subscriptionName; - @Parameter(names = { "--messageId", + @Option(names = { "--messageId", "-m" }, description = "messageId where to create the subscription. " + "It can be either 'latest', 'earliest' or (ledgerId:entryId)", required = false) private String messageIdStr = "latest"; - @Parameter(names = {"--property", "-p"}, description = "key value pair properties(-p a=b -p c=d)", - required = false, splitter = NoSplitter.class) - private java.util.List properties; + @Option(names = {"--property", "-p"}, description = "key value pair properties(-p a=b -p c=d)", + required = false) + private Map properties; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); MessageId messageId; if (messageIdStr.equals("latest")) { messageId = MessageId.latest; @@ -525,43 +515,36 @@ void run() throws PulsarAdminException { } else { messageId = validateMessageIdString(messageIdStr); } - Map map = parseListKeyValueMap(properties); - getPersistentTopics().createSubscription(persistentTopic, subscriptionName, messageId, false, map); + getPersistentTopics().createSubscription(persistentTopic, subscriptionName, messageId, false, properties); } } - @Parameters(commandDescription = "Reset position for subscription to position closest to timestamp or messageId") + @Command(description = "Reset position for subscription to position closest to timestamp or messageId") private class ResetCursor extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-s", + @Option(names = { "-s", "--subscription" }, description = "Subscription to reset position on", required = true) private String subName; - @Parameter(names = { "--time", + @Option(names = { "--time", "-t" }, description = "time in minutes to reset back to " - + "(or minutes, hours,days,weeks eg: 100m, 3h, 2d, 5w)", required = false) - private String resetTimeStr; + + "(or minutes, hours,days,weeks eg: 100m, 3h, 2d, 5w)", required = false, + converter = TimeUnitToMillisConverter.class) + private Long resetTimeInMillis = null; - @Parameter(names = { "--messageId", + @Option(names = { "--messageId", "-m" }, description = "messageId to reset back to (ledgerId:entryId)", required = false) private String resetMessageIdStr; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); if (isNotBlank(resetMessageIdStr)) { MessageId messageId = validateMessageIdString(resetMessageIdStr); getPersistentTopics().resetCursor(persistentTopic, subName, messageId); - } else if (isNotBlank(resetTimeStr)) { - long resetTimeInMillis; - try { - resetTimeInMillis = TimeUnit.SECONDS.toMillis( - RelativeTimeUtil.parseRelativeTimeInSeconds(resetTimeStr)); - } catch (IllegalArgumentException exception) { - throw new ParameterException(exception.getMessage()); - } + } else if (Objects.nonNull(resetTimeInMillis)) { // now - go back time long timestamp = System.currentTimeMillis() - resetTimeInMillis; getPersistentTopics().resetCursor(persistentTopic, subName, timestamp); @@ -572,14 +555,14 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Terminate a topic and don't allow any more messages to be published") + @Command(description = "Terminate a topic and don't allow any more messages to be published") private class Terminate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); try { MessageId lastMessageId = getPersistentTopics().terminateTopicAsync(persistentTopic).get(); @@ -590,21 +573,21 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Peek some messages for the subscription") + @Command(description = "Peek some messages for the subscription") private class PeekMessages extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-s", + @Option(names = { "-s", "--subscription" }, description = "Subscription to get messages from", required = true) private String subName; - @Parameter(names = { "-n", "--count" }, description = "Number of messages (default 1)", required = false) + @Option(names = { "-n", "--count" }, description = "Number of messages (default 1)", required = false) private int numMessages = 1; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); List> messages = getPersistentTopics().peekMessages(persistentTopic, subName, numMessages); int position = 0; for (Message msg : messages) { @@ -629,24 +612,24 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get message by its ledgerId and entryId") + @Command(description = "Get message by its ledgerId and entryId") private class GetMessageById extends CliCommand { - @Parameter(description = "persistent://property/cluster/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://property/cluster/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-l", "--ledgerId" }, + @Option(names = { "-l", "--ledgerId" }, description = "ledger id pointing to the desired ledger", required = true) private long ledgerId; - @Parameter(names = { "-e", "--entryId" }, + @Option(names = { "-e", "--entryId" }, description = "entry id pointing to the desired entry", required = true) private long entryId; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); Message message = getPersistentTopics().getMessageById(persistentTopic, ledgerId, entryId); @@ -656,45 +639,45 @@ void run() throws PulsarAdminException { } - @Parameters(commandDescription = "Get last message Id of the topic") + @Command(description = "Get last message Id of the topic") private class GetLastMessageId extends CliCommand { - @Parameter(description = "persistent://property/cluster/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://property/cluster/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); MessageId messageId = getPersistentTopics().getLastMessageId(persistentTopic); print(messageId); } } - @Parameters(commandDescription = "Compact a topic") + @Command(description = "Compact a topic") private class Compact extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getPersistentTopics().triggerCompaction(persistentTopic); System.out.println("Topic compaction requested for " + persistentTopic); } } - @Parameters(commandDescription = "Status of compaction on a topic") + @Command(description = "Status of compaction on a topic") private class CompactionStatusCmd extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-w", "--wait-complete" }, - description = "Wait for compaction to complete", required = false) + @Option(names = {"-w", "--wait-complete"}, + description = "Wait for compaction to complete", required = false) private boolean wait = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); try { LongRunningProcessStatus status = getPersistentTopics().compactionStatus(persistentTopic); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdProxyStats.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdProxyStats.java index a5ec14a3ed725..a8f8205bf8127 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdProxyStats.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdProxyStats.java @@ -18,8 +18,6 @@ */ package org.apache.pulsar.admin.cli; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonArray; @@ -28,13 +26,15 @@ import java.io.IOException; import java.util.function.Supplier; import org.apache.pulsar.client.admin.PulsarAdmin; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; -@Parameters(commandDescription = "Operations to collect Proxy statistics") +@Command(description = "Operations to collect Proxy statistics") public class CmdProxyStats extends CmdBase { - @Parameters(commandDescription = "dump connections metrics for Monitoring") + @Command(description = "dump connections metrics for Monitoring") private class CmdConnectionMetrics extends CliCommand { - @Parameter(names = { "-i", "--indent" }, description = "Indent JSON output", required = false) + @Option(names = {"-i", "--indent"}, description = "Indent JSON output", required = false) private boolean indent = false; @Override @@ -45,9 +45,9 @@ void run() throws Exception { } } - @Parameters(commandDescription = "dump topics metrics for Monitoring") + @Command(description = "dump topics metrics for Monitoring") private class CmdTopicsMetrics extends CliCommand { - @Parameter(names = { "-i", "--indent" }, description = "Indent JSON output", required = false) + @Option(names = {"-i", "--indent"}, description = "Indent JSON output", required = false) private boolean indent = false; @Override @@ -66,7 +66,7 @@ public void printStats(JsonElement json, boolean indent) throws IOException { public CmdProxyStats(Supplier admin) { super("proxy-stats", admin); - jcommander.addCommand("connections", new CmdConnectionMetrics()); - jcommander.addCommand("topics", new CmdTopicsMetrics()); + addCommand("connections", new CmdConnectionMetrics()); + addCommand("topics", new CmdTopicsMetrics()); } } \ No newline at end of file diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceGroups.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceGroups.java index 7ed853be44d9f..6ee8d7a4764a0 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceGroups.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceGroups.java @@ -18,16 +18,17 @@ */ package org.apache.pulsar.admin.cli; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; import java.util.function.Supplier; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.common.policies.data.ResourceGroup; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; -@Parameters(commandDescription = "Operations about ResourceGroups") +@Command(description = "Operations about ResourceGroups") public class CmdResourceGroups extends CmdBase { - @Parameters(commandDescription = "List the existing resourcegroups") + @Command(description = "List the existing resourcegroups") private class List extends CliCommand { @Override void run() throws PulsarAdminException { @@ -35,112 +36,107 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Gets the configuration of a resourcegroup") + @Command(description = "Gets the configuration of a resourcegroup") private class Get extends CliCommand { - @Parameter(description = "resourcegroup-name", required = true) - private java.util.List params; + @Parameters(description = "resourcegroup-name", arity = "1") + private String resourceGroupName; @Override void run() throws PulsarAdminException { - String name = getOneArgument(params); - print(getAdmin().resourcegroups().getResourceGroup(name)); + print(getAdmin().resourcegroups().getResourceGroup(resourceGroupName)); } } - @Parameters(commandDescription = "Creates a new resourcegroup") + + @Command(description = "Creates a new resourcegroup") private class Create extends CliCommand { - @Parameter(description = "resourcegroup-name", required = true) - private java.util.List params; + @Parameters(description = "resourcegroup-name", arity = "1") + private String resourceGroupName; - @Parameter(names = { "--msg-publish-rate", + @Option(names = { "--msg-publish-rate", "-mp" }, description = "message-publish-rate " + "(default -1 will be overwrite if not passed)", required = false) private Integer publishRateInMsgs; - @Parameter(names = { "--byte-publish-rate", + @Option(names = { "--byte-publish-rate", "-bp" }, description = "byte-publish-rate " + "(default -1 will be overwrite if not passed)", required = false) private Long publishRateInBytes; - @Parameter(names = { "--msg-dispatch-rate", + @Option(names = { "--msg-dispatch-rate", "-md" }, description = "message-dispatch-rate " + "(default -1 will be overwrite if not passed)", required = false) private Integer dispatchRateInMsgs; - @Parameter(names = { "--byte-dispatch-rate", + @Option(names = { "--byte-dispatch-rate", "-bd" }, description = "byte-dispatch-rate " + "(default -1 will be overwrite if not passed)", required = false) private Long dispatchRateInBytes; @Override void run() throws PulsarAdminException { - String name = getOneArgument(params); - ResourceGroup resourcegroup = new ResourceGroup(); resourcegroup.setDispatchRateInMsgs(dispatchRateInMsgs); resourcegroup.setDispatchRateInBytes(dispatchRateInBytes); resourcegroup.setPublishRateInMsgs(publishRateInMsgs); resourcegroup.setPublishRateInBytes(publishRateInBytes); - getAdmin().resourcegroups().createResourceGroup(name, resourcegroup); + getAdmin().resourcegroups().createResourceGroup(resourceGroupName, resourcegroup); } } - @Parameters(commandDescription = "Updates a resourcegroup") + @Command(description = "Updates a resourcegroup") private class Update extends CliCommand { - @Parameter(description = "resourcegroup-name", required = true) - private java.util.List params; + @Parameters(description = "resourcegroup-name", arity = "1") + private String resourceGroupName; - @Parameter(names = { "--msg-publish-rate", + @Option(names = { "--msg-publish-rate", "-mp" }, description = "message-publish-rate ", required = false) private Integer publishRateInMsgs; - @Parameter(names = { "--byte-publish-rate", + @Option(names = { "--byte-publish-rate", "-bp" }, description = "byte-publish-rate ", required = false) private Long publishRateInBytes; - @Parameter(names = { "--msg-dispatch-rate", + @Option(names = { "--msg-dispatch-rate", "-md" }, description = "message-dispatch-rate ", required = false) private Integer dispatchRateInMsgs; - @Parameter(names = { "--byte-dispatch-rate", + @Option(names = { "--byte-dispatch-rate", "-bd" }, description = "byte-dispatch-rate ", required = false) private Long dispatchRateInBytes; @Override void run() throws PulsarAdminException { - String name = getOneArgument(params); - ResourceGroup resourcegroup = new ResourceGroup(); resourcegroup.setDispatchRateInMsgs(dispatchRateInMsgs); resourcegroup.setDispatchRateInBytes(dispatchRateInBytes); resourcegroup.setPublishRateInMsgs(publishRateInMsgs); resourcegroup.setPublishRateInBytes(publishRateInBytes); - getAdmin().resourcegroups().updateResourceGroup(name, resourcegroup); + getAdmin().resourcegroups().updateResourceGroup(resourceGroupName, resourcegroup); } } - @Parameters(commandDescription = "Deletes an existing ResourceGroup") + @Command(description = "Deletes an existing ResourceGroup") private class Delete extends CliCommand { - @Parameter(description = "resourcegroup-name", required = true) - private java.util.List params; + @Parameters(description = "resourcegroup-name", arity = "1") + private String resourceGroupName; @Override void run() throws PulsarAdminException { - String name = getOneArgument(params); - getAdmin().resourcegroups().deleteResourceGroup(name); + getAdmin().resourcegroups().deleteResourceGroup(resourceGroupName); } } public CmdResourceGroups(Supplier admin) { super("resourcegroups", admin); - jcommander.addCommand("list", new CmdResourceGroups.List()); - jcommander.addCommand("get", new CmdResourceGroups.Get()); - jcommander.addCommand("create", new CmdResourceGroups.Create()); - jcommander.addCommand("update", new CmdResourceGroups.Update()); - jcommander.addCommand("delete", new CmdResourceGroups.Delete()); + addCommand("list", new CmdResourceGroups.List()); + addCommand("get", new CmdResourceGroups.Get()); + addCommand("create", new CmdResourceGroups.Create()); + addCommand("update", new CmdResourceGroups.Update()); + addCommand("delete", new CmdResourceGroups.Delete()); } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceQuotas.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceQuotas.java index 25940f52a1be3..ad8c432b124cf 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceQuotas.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceQuotas.java @@ -18,35 +18,34 @@ */ package org.apache.pulsar.admin.cli; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; import java.util.function.Supplier; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.common.policies.data.ResourceQuota; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; -@Parameters(commandDescription = "Operations about resource quotas") +@Command(description = "Operations about resource quotas") public class CmdResourceQuotas extends CmdBase { - @Parameters(commandDescription = "Get the resource quota for specified namespace bundle, " + @Command(description = "Get the resource quota for specified namespace bundle, " + "or default quota if no namespace/bundle specified.") private class GetResourceQuota extends CliCommand { - @Parameter(names = { "--namespace", + @Option(names = { "--namespace", "-n" }, description = "tenant/namespace, must be specified together with '--bundle'") - private java.util.List names; + private String namespaceName; - @Parameter(names = { "--bundle", + @Option(names = { "--bundle", "-b" }, description = "{start-boundary}_{end-boundary}, must be specified together with '--namespace'") private String bundle; @Override void run() throws PulsarAdminException, ParameterException { - if (bundle == null && names == null) { + if (bundle == null && namespaceName == null) { print(getAdmin().resourceQuotas().getDefaultResourceQuota()); - } else if (bundle != null && names != null) { - String namespace = validateNamespace(names); + } else if (bundle != null && namespaceName != null) { + String namespace = validateNamespace(namespaceName); print(getAdmin().resourceQuotas().getNamespaceBundleResourceQuota(namespace, bundle)); } else { throw new ParameterException("namespace and bundle must be provided together."); @@ -54,38 +53,38 @@ void run() throws PulsarAdminException, ParameterException { } } - @Parameters(commandDescription = "Set the resource quota for specified namespace bundle, " + @Command(description = "Set the resource quota for specified namespace bundle, " + "or default quota if no namespace/bundle specified.") private class SetResourceQuota extends CliCommand { - @Parameter(names = { "--namespace", + @Option(names = { "--namespace", "-n" }, description = "tenant/namespace, must be specified together with '--bundle'") - private java.util.List names; + private String namespaceName; - @Parameter(names = { "--bundle", + @Option(names = { "--bundle", "-b" }, description = "{start-boundary}_{end-boundary}, must be specified together with '--namespace'") private String bundle; - @Parameter(names = { "--msgRateIn", + @Option(names = { "--msgRateIn", "-mi" }, description = "expected incoming messages per second", required = true) private long msgRateIn = 0; - @Parameter(names = { "--msgRateOut", + @Option(names = { "--msgRateOut", "-mo" }, description = "expected outgoing messages per second", required = true) private long msgRateOut = 0; - @Parameter(names = { "--bandwidthIn", - "-bi" }, description = "expected inbound bandwidth (bytes/second)", required = true) + @Option(names = {"--bandwidthIn", + "-bi"}, description = "expected inbound bandwidth (bytes/second)", required = true) private long bandwidthIn = 0; - @Parameter(names = { "--bandwidthOut", + @Option(names = { "--bandwidthOut", "-bo" }, description = "expected outbound bandwidth (bytes/second)", required = true) private long bandwidthOut = 0; - @Parameter(names = { "--memory", "-mem" }, description = "expected memory usage (Mbytes)", required = true) + @Option(names = { "--memory", "-mem" }, description = "expected memory usage (Mbytes)", required = true) private long memory = 0; - @Parameter(names = { "--dynamic", + @Option(names = { "--dynamic", "-d" }, description = "dynamic (allow to be dynamically re-calculated) or not") private boolean dynamic = false; @@ -99,10 +98,10 @@ void run() throws PulsarAdminException { quota.setMemory(memory); quota.setDynamic(dynamic); - if (bundle == null && names == null) { + if (bundle == null && namespaceName == null) { getAdmin().resourceQuotas().setDefaultResourceQuota(quota); - } else if (bundle != null && names != null) { - String namespace = validateNamespace(names); + } else if (bundle != null && namespaceName != null) { + String namespace = validateNamespace(namespaceName); getAdmin().resourceQuotas().setNamespaceBundleResourceQuota(namespace, bundle, quota); } else { throw new ParameterException("namespace and bundle must be provided together."); @@ -110,26 +109,26 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Reset the specified namespace bundle's resource quota to default value.") + @Command(description = "Reset the specified namespace bundle's resource quota to default value.") private class ResetNamespaceBundleResourceQuota extends CliCommand { - @Parameter(names = { "--namespace", "-n" }, description = "tenant/namespace", required = true) - private java.util.List names; + @Option(names = { "--namespace", "-n" }, description = "tenant/namespace", required = true) + private String namespaceName; - @Parameter(names = { "--bundle", "-b" }, description = "{start-boundary}_{end-boundary}", required = true) + @Option(names = { "--bundle", "-b" }, description = "{start-boundary}_{end-boundary}", required = true) private String bundle; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(names); + String namespace = validateNamespace(namespaceName); getAdmin().resourceQuotas().resetNamespaceBundleResourceQuota(namespace, bundle); } } public CmdResourceQuotas(Supplier admin) { super("resource-quotas", admin); - jcommander.addCommand("get", new GetResourceQuota()); - jcommander.addCommand("set", new SetResourceQuota()); - jcommander.addCommand("reset-namespace-bundle-quota", new ResetNamespaceBundleResourceQuota()); + addCommand("get", new GetResourceQuota()); + addCommand("set", new SetResourceQuota()); + addCommand("reset-namespace-bundle-quota", new ResetNamespaceBundleResourceQuota()); } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java index 44ac143e3507e..9131f11f3d33d 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java @@ -18,47 +18,50 @@ */ package org.apache.pulsar.admin.cli; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; +import java.io.FileNotFoundException; import java.net.URL; import java.net.URLClassLoader; +import java.nio.file.Path; import java.util.function.Supplier; import org.apache.pulsar.admin.cli.utils.SchemaExtractor; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.common.protocol.schema.PostSchemaPayload; import org.apache.pulsar.common.util.ObjectMapperFactory; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; -@Parameters(commandDescription = "Operations about schemas") +@Command(description = "Operations about schemas") public class CmdSchemas extends CmdBase { private static final ObjectMapper MAPPER = ObjectMapperFactory.create(); public CmdSchemas(Supplier admin) { super("schemas", admin); - jcommander.addCommand("get", new GetSchema()); - jcommander.addCommand("delete", new DeleteSchema()); - jcommander.addCommand("upload", new UploadSchema()); - jcommander.addCommand("extract", new ExtractSchema()); - jcommander.addCommand("compatibility", new TestCompatibility()); + addCommand("get", new GetSchema()); + addCommand("delete", new DeleteSchema()); + addCommand("upload", new UploadSchema()); + addCommand("extract", new ExtractSchema()); + addCommand("metadata", new GetSchemaMetadata()); + addCommand("compatibility", new TestCompatibility()); } - @Parameters(commandDescription = "Get the schema for a topic") + @Command(description = "Get the schema for a topic") private class GetSchema extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"-v", "--version"}, description = "version", required = false) + @Option(names = {"-v", "--version"}, description = "version", required = false) private Long version; - @Parameter(names = {"-a", "--all-version"}, description = "all version", required = false) + @Option(names = {"-a", "--all-version"}, description = "all version", required = false) private boolean all = false; @Override void run() throws Exception { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); if (version != null && all) { throw new ParameterException("Only one or neither of --version and --all-version can be specified."); } @@ -75,12 +78,24 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Delete all versions schema of a topic") + @Command(description = "Get the schema for a topic") + private class GetSchemaMetadata extends CliCommand { + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; + + @Override + void run() throws Exception { + String topic = validateTopicName(topicName); + print(getAdmin().schemas().getSchemaMetadata(topic)); + } + } + + @Command(description = "Delete all versions schema of a topic") private class DeleteSchema extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-f", + @Option(names = { "-f", "--force" }, description = "whether to delete schema completely. If true, delete " + "all resources (including metastore and ledger), otherwise only do a mark deletion" + " and not remove any resources indeed") @@ -88,52 +103,64 @@ private class DeleteSchema extends CliCommand { @Override void run() throws Exception { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); getAdmin().schemas().deleteSchema(topic, force); } } - @Parameters(commandDescription = "Update the schema for a topic") + @Command(description = "Update the schema for a topic") private class UploadSchema extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-f", "--filename" }, description = "filename", required = true) + @Option(names = { "-f", "--filename" }, description = "filename", required = true) private String schemaFileName; @Override void run() throws Exception { - String topic = validateTopicName(params); - PostSchemaPayload input = MAPPER.readValue(new File(schemaFileName), PostSchemaPayload.class); + String topic = validateTopicName(topicName); + Path schemaPath = Path.of(schemaFileName); + File schemaFile = schemaPath.toFile(); + if (!schemaFile.exists()) { + final StringBuilder sb = new StringBuilder(); + sb.append("Schema file ").append(schemaPath).append(" is not found."); + if (!schemaPath.isAbsolute()) { + sb.append(" Relative path ").append(schemaPath) + .append(" is resolved to ").append(schemaPath.toAbsolutePath()) + .append(". Try to use absolute path if the relative one resolved wrongly."); + } + throw new FileNotFoundException(sb.toString()); + } + PostSchemaPayload input = MAPPER.readValue(schemaFile, PostSchemaPayload.class); getAdmin().schemas().createSchema(topic, input); } } - @Parameters(commandDescription = "Provide the schema via a topic") + @Command(description = "Provide the schema via a topic") private class ExtractSchema extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-j", "--jar" }, description = "jar filepath", required = true) + @Option(names = { "-j", "--jar" }, description = "jar filepath", required = true) private String jarFilePath; - @Parameter(names = { "-t", "--type" }, description = "type avro or json", required = true) + @Option(names = { "-t", "--type" }, description = "type avro or json", required = true) private String type; - @Parameter(names = { "-c", "--classname" }, description = "class name of pojo", required = true) + @Option(names = { "-c", "--classname" }, description = "class name of pojo", required = true) private String className; - @Parameter(names = {"-a", "--always-allow-null"}, arity = 1, + @Option(names = {"-a", "--always-allow-null"}, arity = "1", description = "set schema whether always allow null or not") private boolean alwaysAllowNull = true; - @Parameter(names = { "-n", "--dry-run"}, + @Option(names = { "-n", "--dry-run"}, description = "dost not apply to schema registry, just prints the post schema payload") private boolean dryRun = false; @Override void run() throws Exception { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); File file = new File(jarFilePath); ClassLoader cl = new URLClassLoader(new URL[]{ file.toURI().toURL() }); @@ -165,17 +192,17 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Test schema compatibility") + @Command(description = "Test schema compatibility") private class TestCompatibility extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-f", "--filename" }, description = "filename", required = true) + @Option(names = { "-f", "--filename" }, description = "filename", required = true) private String schemaFileName; @Override void run() throws Exception { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); PostSchemaPayload input = MAPPER.readValue(new File(schemaFileName), PostSchemaPayload.class); getAdmin().schemas().testCompatibility(topic, input); } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java index 0b27dd8d0a737..a4fb047550dcb 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java @@ -22,13 +22,10 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.pulsar.common.naming.TopicName.DEFAULT_NAMESPACE; import static org.apache.pulsar.common.naming.TopicName.PUBLIC_TENANT; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; -import com.beust.jcommander.converters.StringConverter; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.reflect.TypeToken; @@ -60,9 +57,11 @@ import org.apache.pulsar.common.io.ConnectorDefinition; import org.apache.pulsar.common.io.SinkConfig; import org.apache.pulsar.common.util.ObjectMapperFactory; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; @Getter -@Parameters(commandDescription = "Interface for managing Pulsar IO sinks (egress data from Pulsar)") +@Command(description = "Interface for managing Pulsar IO sinks (egress data from Pulsar)", aliases = "sink") @Slf4j public class CmdSinks extends CmdBase { @@ -90,19 +89,19 @@ public CmdSinks(Supplier admin) { restartSink = new RestartSink(); localSinkRunner = new LocalSinkRunner(); - jcommander.addCommand("create", createSink); - jcommander.addCommand("update", updateSink); - jcommander.addCommand("delete", deleteSink); - jcommander.addCommand("list", listSinks); - jcommander.addCommand("get", getSink); + addCommand("create", createSink); + addCommand("update", updateSink); + addCommand("delete", deleteSink); + addCommand("list", listSinks); + addCommand("get", getSink); // TODO deprecate getstatus - jcommander.addCommand("status", getSinkStatus, "getstatus"); - jcommander.addCommand("stop", stopSink); - jcommander.addCommand("start", startSink); - jcommander.addCommand("restart", restartSink); - jcommander.addCommand("localrun", localSinkRunner); - jcommander.addCommand("available-sinks", new ListBuiltInSinks()); - jcommander.addCommand("reload", new ReloadBuiltInSinks()); + addCommand("status", getSinkStatus, "getstatus"); + addCommand("stop", stopSink); + addCommand("start", startSink); + addCommand("restart", restartSink); + addCommand("localrun", localSinkRunner); + addCommand("available-sinks", new ListBuiltInSinks()); + addCommand("reload", new ReloadBuiltInSinks()); } /** @@ -112,15 +111,7 @@ public CmdSinks(Supplier admin) { abstract class BaseCommand extends CliCommand { @Override void run() throws Exception { - try { - processArguments(); - } catch (Exception e) { - System.err.println(e.getMessage()); - System.err.println(); - String chosenCommand = jcommander.getParsedCommand(); - getUsageFormatter().usage(chosenCommand); - return; - } + processArguments(); runCmd(); } @@ -130,57 +121,57 @@ void processArguments() throws Exception { abstract void runCmd() throws Exception; } - @Parameters(commandDescription = "Run a Pulsar IO sink connector locally " + @Command(description = "Run a Pulsar IO sink connector locally " + "(rather than deploying it to the Pulsar cluster)") protected class LocalSinkRunner extends CreateSink { - @Parameter(names = "--state-storage-service-url", + @Option(names = "--state-storage-service-url", description = "The URL for the state storage service (the default is Apache BookKeeper)") protected String stateStorageServiceUrl; - @Parameter(names = "--brokerServiceUrl", description = "The URL for the Pulsar broker", hidden = true) + @Option(names = "--brokerServiceUrl", description = "The URL for the Pulsar broker", hidden = true) protected String deprecatedBrokerServiceUrl; - @Parameter(names = "--broker-service-url", description = "The URL for the Pulsar broker") + @Option(names = "--broker-service-url", description = "The URL for the Pulsar broker") protected String brokerServiceUrl; - @Parameter(names = "--clientAuthPlugin", description = "Client authentication plugin using " + @Option(names = "--clientAuthPlugin", description = "Client authentication plugin using " + "which function-process can connect to broker", hidden = true) protected String deprecatedClientAuthPlugin; - @Parameter(names = "--client-auth-plugin", + @Option(names = "--client-auth-plugin", description = "Client authentication plugin using which function-process can connect to broker") protected String clientAuthPlugin; - @Parameter(names = "--clientAuthParams", description = "Client authentication param", hidden = true) + @Option(names = "--clientAuthParams", description = "Client authentication param", hidden = true) protected String deprecatedClientAuthParams; - @Parameter(names = "--client-auth-params", description = "Client authentication param") + @Option(names = "--client-auth-params", description = "Client authentication param") protected String clientAuthParams; - @Parameter(names = "--use_tls", description = "Use tls connection", hidden = true) + @Option(names = "--use_tls", description = "Use tls connection", hidden = true) protected Boolean deprecatedUseTls; - @Parameter(names = "--use-tls", description = "Use tls connection") + @Option(names = "--use-tls", description = "Use tls connection") protected boolean useTls; - @Parameter(names = "--tls_allow_insecure", description = "Allow insecure tls connection", hidden = true) + @Option(names = "--tls_allow_insecure", description = "Allow insecure tls connection", hidden = true) protected Boolean deprecatedTlsAllowInsecureConnection; - @Parameter(names = "--tls-allow-insecure", description = "Allow insecure tls connection") + @Option(names = "--tls-allow-insecure", description = "Allow insecure tls connection") protected boolean tlsAllowInsecureConnection; - @Parameter(names = "--hostname_verification_enabled", + @Option(names = "--hostname_verification_enabled", description = "Enable hostname verification", hidden = true) protected Boolean deprecatedTlsHostNameVerificationEnabled; - @Parameter(names = "--hostname-verification-enabled", description = "Enable hostname verification") + @Option(names = "--hostname-verification-enabled", description = "Enable hostname verification") protected boolean tlsHostNameVerificationEnabled; - @Parameter(names = "--tls_trust_cert_path", description = "tls trust cert file path", hidden = true) + @Option(names = "--tls_trust_cert_path", description = "tls trust cert file path", hidden = true) protected String deprecatedTlsTrustCertFilePath; - @Parameter(names = "--tls-trust-cert-path", description = "tls trust cert file path") + @Option(names = "--tls-trust-cert-path", description = "tls trust cert file path") protected String tlsTrustCertFilePath; - @Parameter(names = "--secrets-provider-classname", description = "Whats the classname for secrets provider") + @Option(names = "--secrets-provider-classname", description = "Whats the classname for secrets provider") protected String secretsProviderClassName; - @Parameter(names = "--secrets-provider-config", + @Option(names = "--secrets-provider-config", description = "Config that needs to be passed to secrets provider") protected String secretsProviderConfig; - @Parameter(names = "--metrics-port-start", description = "The starting port range for metrics server") + @Option(names = "--metrics-port-start", description = "The starting port range for metrics server") protected String metricsPortStart; private void mergeArgs() { @@ -207,8 +198,8 @@ private void mergeArgs() { } } - @Override - public void runCmd() throws Exception { + @VisibleForTesting + List getLocalRunArgs() throws Exception { // merge deprecated args with new args mergeArgs(); List localRunArgs = new LinkedList<>(); @@ -216,7 +207,7 @@ public void runCmd() throws Exception { localRunArgs.add("--sinkConfig"); localRunArgs.add(new Gson().toJson(sinkConfig)); for (Field field : this.getClass().getDeclaredFields()) { - if (field.getName().startsWith("DEPRECATED")) { + if (field.getName().toUpperCase().startsWith("DEPRECATED")) { continue; } if (field.getName().contains("$")) { @@ -228,7 +219,12 @@ public void runCmd() throws Exception { localRunArgs.add(value.toString()); } } - ProcessBuilder processBuilder = new ProcessBuilder(localRunArgs).inheritIO(); + return localRunArgs; + } + + @Override + public void runCmd() throws Exception { + ProcessBuilder processBuilder = new ProcessBuilder(getLocalRunArgs()).inheritIO(); Process process = processBuilder.start(); process.waitFor(); } @@ -239,7 +235,7 @@ protected String validateSinkType(String sinkType) { } } - @Parameters(commandDescription = "Submit a Pulsar IO sink connector to run in a Pulsar cluster") + @Command(description = "Submit a Pulsar IO sink connector to run in a Pulsar cluster") protected class CreateSink extends SinkDetailsCommand { @Override void runCmd() throws Exception { @@ -252,10 +248,10 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Update a Pulsar IO sink connector") + @Command(description = "Update a Pulsar IO sink connector") protected class UpdateSink extends SinkDetailsCommand { - @Parameter(names = "--update-auth-data", description = "Whether or not to update the auth data") + @Option(names = "--update-auth-data", description = "Whether or not to update the auth data") protected boolean updateAuthData; @Override @@ -281,137 +277,142 @@ protected void validateSinkConfigs(SinkConfig sinkConfig) { } abstract class SinkDetailsCommand extends BaseCommand { - @Parameter(names = "--tenant", description = "The sink's tenant") + @Option(names = "--tenant", description = "The sink's tenant") protected String tenant; - @Parameter(names = "--namespace", description = "The sink's namespace") + @Option(names = "--namespace", description = "The sink's namespace") protected String namespace; - @Parameter(names = "--name", description = "The sink's name") + @Option(names = "--name", description = "The sink's name") protected String name; - @Parameter(names = { "-t", "--sink-type" }, description = "The sinks's connector provider") + @Option(names = { "-t", "--sink-type" }, description = "The sinks's connector provider") protected String sinkType; - @Parameter(names = "--cleanup-subscription", description = "Whether delete the subscription " + @Option(names = "--cleanup-subscription", description = "Whether delete the subscription " + "when sink is deleted") protected Boolean cleanupSubscription; - @Parameter(names = { "-i", + @Option(names = { "-i", "--inputs" }, description = "The sink's input topic or topics " + "(multiple topics can be specified as a comma-separated list)") protected String inputs; - @Parameter(names = "--topicsPattern", description = "TopicsPattern to consume from list of topics " + @Option(names = "--topicsPattern", description = "TopicsPattern to consume from list of topics " + "under a namespace that match the pattern. [--input] and [--topicsPattern] are mutually exclusive. " + "Add SerDe class name for a pattern in --customSerdeInputs (supported for java fun only)", hidden = true) protected String deprecatedTopicsPattern; - @Parameter(names = "--topics-pattern", description = "The topic pattern to consume from a list of topics " + @Option(names = "--topics-pattern", description = "The topic pattern to consume from a list of topics " + "under a namespace that matches the pattern. [--input] and [--topics-pattern] are mutually " + "exclusive. Add SerDe class name for a pattern in --custom-serde-inputs") protected String topicsPattern; - @Parameter(names = "--subsName", description = "Pulsar source subscription name " + @Option(names = "--subsName", description = "Pulsar source subscription name " + "if user wants a specific subscription-name for input-topic consumer", hidden = true) protected String deprecatedSubsName; - @Parameter(names = "--subs-name", description = "Pulsar source subscription name " + @Option(names = "--subs-name", description = "Pulsar source subscription name " + "if user wants a specific subscription-name for input-topic consumer") protected String subsName; - @Parameter(names = "--subs-position", description = "Pulsar source subscription position " + @Option(names = "--subs-position", description = "Pulsar source subscription position " + "if user wants to consume messages from the specified location") protected SubscriptionInitialPosition subsPosition; - @Parameter(names = "--customSerdeInputs", + @Option(names = "--customSerdeInputs", description = "The map of input topics to SerDe class names (as a JSON string)", hidden = true) protected String deprecatedCustomSerdeInputString; - @Parameter(names = "--custom-serde-inputs", + @Option(names = "--custom-serde-inputs", description = "The map of input topics to SerDe class names (as a JSON string)") protected String customSerdeInputString; - @Parameter(names = "--custom-schema-inputs", + @Option(names = "--custom-schema-inputs", description = "The map of input topics to Schema types or class names (as a JSON string)") protected String customSchemaInputString; - @Parameter(names = "--input-specs", + @Option(names = "--input-specs", description = "The map of inputs to custom configuration (as a JSON string)") protected String inputSpecs; - @Parameter(names = "--max-redeliver-count", description = "Maximum number of times that a message " + @Option(names = "--max-redeliver-count", description = "Maximum number of times that a message " + "will be redelivered before being sent to the dead letter queue") protected Integer maxMessageRetries; - @Parameter(names = "--dead-letter-topic", + @Option(names = "--dead-letter-topic", description = "Name of the dead topic where the failing messages will be sent.") protected String deadLetterTopic; - @Parameter(names = "--processingGuarantees", + @Option(names = "--processingGuarantees", description = "The processing guarantees (aka delivery semantics) applied to the sink", hidden = true) protected FunctionConfig.ProcessingGuarantees deprecatedProcessingGuarantees; - @Parameter(names = "--processing-guarantees", + @Option(names = "--processing-guarantees", description = "The processing guarantees (as known as delivery semantics) applied to the sink." + " The '--processing-guarantees' implementation in Pulsar also relies on sink implementation." + " The available values are `ATLEAST_ONCE`, `ATMOST_ONCE`, `EFFECTIVELY_ONCE`." + " If it is not specified, `ATLEAST_ONCE` delivery guarantee is used.") protected FunctionConfig.ProcessingGuarantees processingGuarantees; - @Parameter(names = "--retainOrdering", description = "Sink consumes and sinks messages in order", hidden = true) + @Option(names = "--retainOrdering", description = "Sink consumes and sinks messages in order", hidden = true) protected Boolean deprecatedRetainOrdering; - @Parameter(names = "--retain-ordering", description = "Sink consumes and sinks messages in order") + @Option(names = "--retain-ordering", description = "Sink consumes and sinks messages in order") protected Boolean retainOrdering; - @Parameter(names = "--parallelism", + @Option(names = "--parallelism", description = "The sink's parallelism factor (i.e. the number of sink instances to run)") protected Integer parallelism; - @Parameter(names = "--retain-key-ordering", + @Option(names = "--retain-key-ordering", description = "Sink consumes and processes messages in key order") protected Boolean retainKeyOrdering; - @Parameter(names = {"-a", "--archive"}, description = "Path to the archive file for the sink. It also supports " + @Option(names = {"-a", "--archive"}, description = "Path to the archive file for the sink. It also supports " + "url-path [http/https/file (file protocol assumes that file already exists on worker host)] from " - + "which worker can download the package.", listConverter = StringConverter.class) + + "which worker can download the package.") protected String archive; - @Parameter(names = "--className", + @Option(names = "--className", description = "The sink's class name if archive is file-url-path (file://)", hidden = true) protected String deprecatedClassName; - @Parameter(names = "--classname", description = "The sink's class name if archive is file-url-path (file://)") + @Option(names = "--classname", description = "The sink's class name if archive is file-url-path (file://)") protected String className; - @Parameter(names = "--sinkConfigFile", description = "The path to a YAML config file specifying the " + @Option(names = "--sinkConfigFile", description = "The path to a YAML config file specifying the " + "sink's configuration", hidden = true) protected String deprecatedSinkConfigFile; - @Parameter(names = "--sink-config-file", description = "The path to a YAML config file specifying the " + @Option(names = "--sink-config-file", description = "The path to a YAML config file specifying the " + "sink's configuration") protected String sinkConfigFile; - @Parameter(names = "--cpu", description = "The CPU (in cores) that needs to be allocated " + @Option(names = "--cpu", description = "The CPU (in cores) that needs to be allocated " + "per sink instance (applicable only to Docker runtime)") protected Double cpu; - @Parameter(names = "--ram", description = "The RAM (in bytes) that need to be allocated " + @Option(names = "--ram", description = "The RAM (in bytes) that need to be allocated " + "per sink instance (applicable only to the process and Docker runtimes)") protected Long ram; - @Parameter(names = "--disk", description = "The disk (in bytes) that need to be allocated " + @Option(names = "--disk", description = "The disk (in bytes) that need to be allocated " + "per sink instance (applicable only to Docker runtime)") protected Long disk; - @Parameter(names = "--sinkConfig", description = "User defined configs key/values", hidden = true) + @Option(names = "--sinkConfig", description = "User defined configs key/values", hidden = true) protected String deprecatedSinkConfigString; - @Parameter(names = "--sink-config", description = "User defined configs key/values") + @Option(names = "--sink-config", description = "User defined configs key/values") protected String sinkConfigString; - @Parameter(names = "--auto-ack", - description = "Whether or not the framework will automatically acknowledge messages", arity = 1) + @Option(names = "--auto-ack", + description = "Whether or not the framework will automatically acknowledge messages", arity = "1") protected Boolean autoAck; - @Parameter(names = "--timeout-ms", description = "The message timeout in milliseconds") + @Option(names = "--timeout-ms", description = "The message timeout in milliseconds") protected Long timeoutMs; - @Parameter(names = "--negative-ack-redelivery-delay-ms", + @Option(names = "--negative-ack-redelivery-delay-ms", description = "The negative ack message redelivery delay in milliseconds") protected Long negativeAckRedeliveryDelayMs; - @Parameter(names = "--custom-runtime-options", description = "A string that encodes options to " + @Option(names = "--custom-runtime-options", description = "A string that encodes options to " + "customize the runtime, see docs for configured runtime for details") protected String customRuntimeOptions; - @Parameter(names = "--secrets", description = "The map of secretName to an object that encapsulates " + @Option(names = "--secrets", description = "The map of secretName to an object that encapsulates " + "how the secret is fetched by the underlying secrets provider") protected String secretsString; - @Parameter(names = "--transform-function", description = "Transform function applied before the Sink") + @Option(names = "--transform-function", description = "Transform function applied before the Sink") protected String transformFunction; - @Parameter(names = "--transform-function-classname", description = "The transform function class name") + @Option(names = "--transform-function-classname", description = "The transform function class name") protected String transformFunctionClassName; - @Parameter(names = "--transform-function-config", description = "Configuration of the transform function " + @Option(names = "--transform-function-config", description = "Configuration of the transform function " + "applied before the Sink") protected String transformFunctionConfig; + @Option(names = "--log-topic", description = "The topic to which the logs of a Pulsar Sink are produced") + protected String logTopic; + @Option(names = "--runtime-flags", description = "Any flags that you want to pass to a runtime" + + " (for process & Kubernetes runtime only).") + protected String runtimeFlags; protected SinkConfig sinkConfig; @@ -526,7 +527,7 @@ void processArguments() throws Exception { sinkConfig.setParallelism(parallelism); } - if (archive != null && sinkType != null) { + if (archive != null && (sinkType != null || sinkConfig.getSinkType() != null)) { throw new ParameterException("Cannot specify both archive and sink-type"); } @@ -570,7 +571,7 @@ void processArguments() throws Exception { sinkConfig.setConfigs(parseConfigs(sinkConfigString)); } } catch (Exception ex) { - throw new ParameterException("Cannot parse sink-config", ex); + throw new IllegalArgumentException("Cannot parse sink-config", ex); } if (autoAck != null) { @@ -607,6 +608,12 @@ void processArguments() throws Exception { if (transformFunctionConfig != null) { sinkConfig.setTransformFunctionConfig(transformFunctionConfig); } + if (null != logTopic) { + sinkConfig.setLogTopic(logTopic); + } + if (null != runtimeFlags) { + sinkConfig.setRuntimeFlags(runtimeFlags); + } // check if configs are valid validateSinkConfigs(sinkConfig); @@ -622,7 +629,7 @@ protected Map parseConfigs(String str) throws JsonProcessingExce protected void validateSinkConfigs(SinkConfig sinkConfig) { if (isBlank(sinkConfig.getArchive())) { - throw new ParameterException("Sink archive not specfied"); + throw new ParameterException("Sink archive not specified"); } org.apache.pulsar.common.functions.Utils.inferMissingArguments(sinkConfig); @@ -660,13 +667,13 @@ protected String validateSinkType(String sinkType) throws IOException { */ @Getter abstract class SinkCommand extends BaseCommand { - @Parameter(names = "--tenant", description = "The sink's tenant") + @Option(names = "--tenant", description = "The sink's tenant") protected String tenant; - @Parameter(names = "--namespace", description = "The sink's namespace") + @Option(names = "--namespace", description = "The sink's namespace") protected String namespace; - @Parameter(names = "--name", description = "The sink's name") + @Option(names = "--name", description = "The sink's name") protected String sinkName; @Override @@ -685,7 +692,7 @@ void processArguments() throws Exception { } } - @Parameters(commandDescription = "Stops a Pulsar IO sink connector") + @Command(description = "Stops a Pulsar IO sink connector") protected class DeleteSink extends SinkCommand { @Override @@ -695,7 +702,7 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Gets the information about a Pulsar IO sink connector") + @Command(description = "Gets the information about a Pulsar IO sink connector") protected class GetSink extends SinkCommand { @Override @@ -709,12 +716,12 @@ void runCmd() throws Exception { /** * List Sources command. */ - @Parameters(commandDescription = "List all running Pulsar IO sink connectors") + @Command(description = "List all running Pulsar IO sink connectors") protected class ListSinks extends BaseCommand { - @Parameter(names = "--tenant", description = "The sink's tenant") + @Option(names = "--tenant", description = "The sink's tenant") protected String tenant; - @Parameter(names = "--namespace", description = "The sink's namespace") + @Option(names = "--namespace", description = "The sink's namespace") protected String namespace; @Override @@ -735,10 +742,10 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Check the current status of a Pulsar Sink") + @Command(description = "Check the current status of a Pulsar Sink") class GetSinkStatus extends SinkCommand { - @Parameter(names = "--instance-id", + @Option(names = "--instance-id", description = "The sink instanceId (Get-status of all instances if instance-id is not provided") protected String instanceId; @@ -752,10 +759,10 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Restart sink instance") + @Command(description = "Restart sink instance") class RestartSink extends SinkCommand { - @Parameter(names = "--instance-id", + @Option(names = "--instance-id", description = "The sink instanceId (restart all instances if instance-id is not provided") protected String instanceId; @@ -774,10 +781,10 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Stops sink instance") + @Command(description = "Stops sink instance") class StopSink extends SinkCommand { - @Parameter(names = "--instance-id", + @Option(names = "--instance-id", description = "The sink instanceId (stop all instances if instance-id is not provided") protected String instanceId; @@ -796,10 +803,10 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Starts sink instance") + @Command(description = "Starts sink instance") class StartSink extends SinkCommand { - @Parameter(names = "--instance-id", + @Option(names = "--instance-id", description = "The sink instanceId (start all instances if instance-id is not provided") protected String instanceId; @@ -818,7 +825,7 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Get the list of Pulsar IO connector sinks supported by Pulsar cluster") + @Command(description = "Get the list of Pulsar IO connector sinks supported by Pulsar cluster") public class ListBuiltInSinks extends BaseCommand { @Override void runCmd() throws Exception { @@ -831,7 +838,7 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Reload the available built-in connectors") + @Command(description = "Reload the available built-in connectors") public class ReloadBuiltInSinks extends BaseCommand { @Override diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java index 3a6b15caf8ea8..c8af7ddd954b1 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java @@ -22,13 +22,10 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.pulsar.common.naming.TopicName.DEFAULT_NAMESPACE; import static org.apache.pulsar.common.naming.TopicName.PUBLIC_TENANT; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; -import com.beust.jcommander.converters.StringConverter; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.reflect.TypeToken; @@ -60,9 +57,11 @@ import org.apache.pulsar.common.io.ConnectorDefinition; import org.apache.pulsar.common.io.SourceConfig; import org.apache.pulsar.common.util.ObjectMapperFactory; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; @Getter -@Parameters(commandDescription = "Interface for managing Pulsar IO Sources (ingress data into Pulsar)") +@Command(description = "Interface for managing Pulsar IO Sources (ingress data into Pulsar)", aliases = "source") @Slf4j public class CmdSources extends CmdBase { @@ -90,19 +89,19 @@ public CmdSources(Supplier admin) { startSource = new StartSource(); localSourceRunner = new LocalSourceRunner(); - jcommander.addCommand("create", createSource); - jcommander.addCommand("update", updateSource); - jcommander.addCommand("delete", deleteSource); - jcommander.addCommand("get", getSource); + addCommand("create", createSource); + addCommand("update", updateSource); + addCommand("delete", deleteSource); + addCommand("get", getSource); // TODO depecreate getstatus - jcommander.addCommand("status", getSourceStatus, "getstatus"); - jcommander.addCommand("list", listSources); - jcommander.addCommand("stop", stopSource); - jcommander.addCommand("start", startSource); - jcommander.addCommand("restart", restartSource); - jcommander.addCommand("localrun", localSourceRunner); - jcommander.addCommand("available-sources", new ListBuiltInSources()); - jcommander.addCommand("reload", new ReloadBuiltInSources()); + addCommand("status", getSourceStatus, "getstatus"); + addCommand("list", listSources); + addCommand("stop", stopSource); + addCommand("start", startSource); + addCommand("restart", restartSource); + addCommand("localrun", localSourceRunner); + addCommand("available-sources", new ListBuiltInSources()); + addCommand("reload", new ReloadBuiltInSources()); } /** @@ -112,15 +111,7 @@ public CmdSources(Supplier admin) { abstract class BaseCommand extends CliCommand { @Override void run() throws Exception { - try { - processArguments(); - } catch (Exception e) { - System.err.println(e.getMessage()); - System.err.println(); - String chosenCommand = jcommander.getParsedCommand(); - getUsageFormatter().usage(chosenCommand); - return; - } + processArguments(); runCmd(); } @@ -130,58 +121,58 @@ void processArguments() throws Exception { abstract void runCmd() throws Exception; } - @Parameters(commandDescription = "Run a Pulsar IO source connector locally " + @Command(description = "Run a Pulsar IO source connector locally " + "(rather than deploying it to the Pulsar cluster)") protected class LocalSourceRunner extends CreateSource { - @Parameter(names = "--state-storage-service-url", + @Option(names = "--state-storage-service-url", description = "The URL for the state storage service (the default is Apache BookKeeper)") protected String stateStorageServiceUrl; - @Parameter(names = "--brokerServiceUrl", description = "The URL for the Pulsar broker", hidden = true) + @Option(names = "--brokerServiceUrl", description = "The URL for the Pulsar broker", hidden = true) protected String deprecatedBrokerServiceUrl; - @Parameter(names = "--broker-service-url", description = "The URL for the Pulsar broker") + @Option(names = "--broker-service-url", description = "The URL for the Pulsar broker") protected String brokerServiceUrl; - @Parameter(names = "--clientAuthPlugin", + @Option(names = "--clientAuthPlugin", description = "Client authentication plugin using which function-process can connect to broker", hidden = true) protected String deprecatedClientAuthPlugin; - @Parameter(names = "--client-auth-plugin", + @Option(names = "--client-auth-plugin", description = "Client authentication plugin using which function-process can connect to broker") protected String clientAuthPlugin; - @Parameter(names = "--clientAuthParams", description = "Client authentication param", hidden = true) + @Option(names = "--clientAuthParams", description = "Client authentication param", hidden = true) protected String deprecatedClientAuthParams; - @Parameter(names = "--client-auth-params", description = "Client authentication param") + @Option(names = "--client-auth-params", description = "Client authentication param") protected String clientAuthParams; - @Parameter(names = "--use_tls", description = "Use tls connection", hidden = true) + @Option(names = "--use_tls", description = "Use tls connection", hidden = true) protected Boolean deprecatedUseTls; - @Parameter(names = "--use-tls", description = "Use tls connection") + @Option(names = "--use-tls", description = "Use tls connection") protected boolean useTls; - @Parameter(names = "--tls_allow_insecure", description = "Allow insecure tls connection", hidden = true) + @Option(names = "--tls_allow_insecure", description = "Allow insecure tls connection", hidden = true) protected Boolean deprecatedTlsAllowInsecureConnection; - @Parameter(names = "--tls-allow-insecure", description = "Allow insecure tls connection") + @Option(names = "--tls-allow-insecure", description = "Allow insecure tls connection") protected boolean tlsAllowInsecureConnection; - @Parameter(names = "--hostname_verification_enabled", + @Option(names = "--hostname_verification_enabled", description = "Enable hostname verification", hidden = true) protected Boolean deprecatedTlsHostNameVerificationEnabled; - @Parameter(names = "--hostname-verification-enabled", description = "Enable hostname verification") + @Option(names = "--hostname-verification-enabled", description = "Enable hostname verification") protected boolean tlsHostNameVerificationEnabled; - @Parameter(names = "--tls_trust_cert_path", description = "tls trust cert file path", hidden = true) + @Option(names = "--tls_trust_cert_path", description = "tls trust cert file path", hidden = true) protected String deprecatedTlsTrustCertFilePath; - @Parameter(names = "--tls-trust-cert-path", description = "tls trust cert file path") + @Option(names = "--tls-trust-cert-path", description = "tls trust cert file path") protected String tlsTrustCertFilePath; - @Parameter(names = "--secrets-provider-classname", description = "Whats the classname for secrets provider") + @Option(names = "--secrets-provider-classname", description = "Whats the classname for secrets provider") protected String secretsProviderClassName; - @Parameter(names = "--secrets-provider-config", + @Option(names = "--secrets-provider-config", description = "Config that needs to be passed to secrets provider") protected String secretsProviderConfig; - @Parameter(names = "--metrics-port-start", description = "The starting port range for metrics server") + @Option(names = "--metrics-port-start", description = "The starting port range for metrics server") protected String metricsPortStart; private void mergeArgs() { @@ -208,17 +199,16 @@ private void mergeArgs() { } } - @Override - public void runCmd() throws Exception { + @VisibleForTesting + List getLocalRunArgs() throws Exception { // merge deprecated args with new args mergeArgs(); - List localRunArgs = new LinkedList<>(); localRunArgs.add(System.getenv("PULSAR_HOME") + "/bin/function-localrunner"); localRunArgs.add("--sourceConfig"); localRunArgs.add(new Gson().toJson(sourceConfig)); for (Field field : this.getClass().getDeclaredFields()) { - if (field.getName().startsWith("DEPRECATED")) { + if (field.getName().toUpperCase().startsWith("DEPRECATED")) { continue; } if (field.getName().contains("$")) { @@ -230,7 +220,12 @@ public void runCmd() throws Exception { localRunArgs.add(value.toString()); } } - ProcessBuilder processBuilder = new ProcessBuilder(localRunArgs).inheritIO(); + return localRunArgs; + } + + @Override + public void runCmd() throws Exception { + ProcessBuilder processBuilder = new ProcessBuilder(getLocalRunArgs()).inheritIO(); Process process = processBuilder.start(); process.waitFor(); } @@ -241,7 +236,7 @@ protected String validateSourceType(String sourceType) { } } - @Parameters(commandDescription = "Submit a Pulsar IO source connector to run in a Pulsar cluster") + @Command(description = "Submit a Pulsar IO source connector to run in a Pulsar cluster") protected class CreateSource extends SourceDetailsCommand { @Override void runCmd() throws Exception { @@ -254,10 +249,10 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Update a Pulsar IO source connector") + @Command(description = "Update a Pulsar IO source connector") protected class UpdateSource extends SourceDetailsCommand { - @Parameter(names = "--update-auth-data", description = "Whether or not to update the auth data") + @Option(names = "--update-auth-data", description = "Whether or not to update the auth data") protected boolean updateAuthData; @Override @@ -283,20 +278,20 @@ protected void validateSourceConfigs(SourceConfig sourceConfig) { } abstract class SourceDetailsCommand extends BaseCommand { - @Parameter(names = "--tenant", description = "The source's tenant") + @Option(names = "--tenant", description = "The source's tenant") protected String tenant; - @Parameter(names = "--namespace", description = "The source's namespace") + @Option(names = "--namespace", description = "The source's namespace") protected String namespace; - @Parameter(names = "--name", description = "The source's name") + @Option(names = "--name", description = "The source's name") protected String name; - @Parameter(names = { "-t", "--source-type" }, description = "The source's connector provider") + @Option(names = {"-t", "--source-type"}, description = "The source's connector provider") protected String sourceType; - @Parameter(names = "--processingGuarantees", + @Option(names = "--processingGuarantees", description = "The processing guarantees (aka delivery semantics) applied to the Source", hidden = true) protected FunctionConfig.ProcessingGuarantees deprecatedProcessingGuarantees; - @Parameter(names = "--processing-guarantees", + @Option(names = "--processing-guarantees", description = "The processing guarantees (as known as delivery semantics) applied to the source." + " A source connector receives messages from external system and writes messages to a Pulsar" + " topic. The '--processing-guarantees' is used to ensure the processing guarantees for writing" @@ -304,69 +299,74 @@ abstract class SourceDetailsCommand extends BaseCommand { + " `EFFECTIVELY_ONCE`. If it is not specified, `ATLEAST_ONCE` delivery guarantee is used.") protected FunctionConfig.ProcessingGuarantees processingGuarantees; - @Parameter(names = { "-o", "--destinationTopicName" }, + @Option(names = { "-o", "--destinationTopicName" }, description = "The Pulsar topic to which data is sent", hidden = true) protected String deprecatedDestinationTopicName; - @Parameter(names = "--destination-topic-name", description = "The Pulsar topic to which data is sent") + @Option(names = "--destination-topic-name", description = "The Pulsar topic to which data is sent") protected String destinationTopicName; - @Parameter(names = "--producer-config", description = "The custom producer configuration (as a JSON string)") + @Option(names = "--producer-config", description = "The custom producer configuration (as a JSON string)") protected String producerConfig; - @Parameter(names = "--batch-builder", description = "BatchBuilder provides two types of " + @Option(names = "--batch-builder", description = "BatchBuilder provides two types of " + "batch construction methods, DEFAULT and KEY_BASED. The default value is: DEFAULT") protected String batchBuilder; - @Parameter(names = "--deserializationClassName", + @Option(names = "--deserializationClassName", description = "The SerDe classname for the source", hidden = true) protected String deprecatedDeserializationClassName; - @Parameter(names = "--deserialization-classname", description = "The SerDe classname for the source") + @Option(names = "--deserialization-classname", description = "The SerDe classname for the source") protected String deserializationClassName; - @Parameter(names = { "-st", + @Option(names = { "-st", "--schema-type" }, description = "The schema type (either a builtin schema like 'avro', 'json', etc.." - + " or custom Schema class name to be used to encode messages emitted from the source") + + " or custom Schema class name to be used to encode messages emitted from the source") protected String schemaType; - @Parameter(names = "--parallelism", + @Option(names = "--parallelism", description = "The source's parallelism factor (i.e. the number of source instances to run)") protected Integer parallelism; - @Parameter(names = { "-a", "--archive" }, + @Option(names = { "-a", "--archive" }, description = "The path to the NAR archive for the Source. It also supports url-path " + "[http/https/file (file protocol assumes that file already exists on worker host)] " - + "from which worker can download the package.", listConverter = StringConverter.class) + + "from which worker can download the package.") protected String archive; - @Parameter(names = "--className", + @Option(names = "--className", description = "The source's class name if archive is file-url-path (file://)", hidden = true) protected String deprecatedClassName; - @Parameter(names = "--classname", description = "The source's class name if archive is file-url-path (file://)") + @Option(names = "--classname", description = "The source's class name if archive is file-url-path (file://)") protected String className; - @Parameter(names = "--sourceConfigFile", description = "The path to a YAML config file specifying the " + @Option(names = "--sourceConfigFile", description = "The path to a YAML config file specifying the " + "source's configuration", hidden = true) protected String deprecatedSourceConfigFile; - @Parameter(names = "--source-config-file", description = "The path to a YAML config file specifying the " + @Option(names = "--source-config-file", description = "The path to a YAML config file specifying the " + "source's configuration") protected String sourceConfigFile; - @Parameter(names = "--cpu", description = "The CPU (in cores) that needs to be allocated " + @Option(names = "--cpu", description = "The CPU (in cores) that needs to be allocated " + "per source instance (applicable only to Docker runtime)") protected Double cpu; - @Parameter(names = "--ram", description = "The RAM (in bytes) that need to be allocated " + @Option(names = "--ram", description = "The RAM (in bytes) that need to be allocated " + "per source instance (applicable only to the process and Docker runtimes)") protected Long ram; - @Parameter(names = "--disk", description = "The disk (in bytes) that need to be allocated " + @Option(names = "--disk", description = "The disk (in bytes) that need to be allocated " + "per source instance (applicable only to Docker runtime)") protected Long disk; - @Parameter(names = "--sourceConfig", description = "Source config key/values", hidden = true) + @Option(names = "--sourceConfig", description = "Source config key/values", hidden = true) protected String deprecatedSourceConfigString; - @Parameter(names = "--source-config", description = "Source config key/values") + @Option(names = "--source-config", description = "Source config key/values") protected String sourceConfigString; - @Parameter(names = "--batch-source-config", description = "Batch source config key/values") + @Option(names = "--batch-source-config", description = "Batch source config key/values") protected String batchSourceConfigString; - @Parameter(names = "--custom-runtime-options", description = "A string that encodes options to " + @Option(names = "--custom-runtime-options", description = "A string that encodes options to " + "customize the runtime, see docs for configured runtime for details") protected String customRuntimeOptions; - @Parameter(names = "--secrets", description = "The map of secretName to an object that encapsulates " + @Option(names = "--secrets", description = "The map of secretName to an object that encapsulates " + "how the secret is fetched by the underlying secrets provider") protected String secretsString; + @Option(names = "--log-topic", description = "The topic to which the logs of a Pulsar Sink are produced") + protected String logTopic; + @Option(names = "--runtime-flags", description = "Any flags that you want to pass to a runtime" + + " (for process & Kubernetes runtime only).") + protected String runtimeFlags; protected SourceConfig sourceConfig; @@ -439,7 +439,7 @@ void processArguments() throws Exception { sourceConfig.setParallelism(parallelism); } - if (archive != null && sourceType != null) { + if (archive != null && (sourceType != null || sourceConfig.getSourceType() != null)) { throw new ParameterException("Cannot specify both archive and source-type"); } @@ -449,6 +449,8 @@ void processArguments() throws Exception { if (sourceType != null) { sourceConfig.setArchive(validateSourceType(sourceType)); + } else if (sourceConfig.getSourceType() != null) { + sourceConfig.setArchive(validateSourceType(sourceConfig.getSourceType())); } Resources resources = sourceConfig.getResources(); @@ -500,6 +502,12 @@ void processArguments() throws Exception { } sourceConfig.setSecrets(secretsMap); } + if (null != logTopic) { + sourceConfig.setLogTopic(logTopic); + } + if (null != runtimeFlags) { + sourceConfig.setRuntimeFlags(runtimeFlags); + } // check if source configs are valid validateSourceConfigs(sourceConfig); @@ -567,13 +575,13 @@ protected String validateSourceType(String sourceType) throws IOException { */ @Getter abstract class SourceCommand extends BaseCommand { - @Parameter(names = "--tenant", description = "The source's tenant") + @Option(names = "--tenant", description = "The source's tenant") protected String tenant; - @Parameter(names = "--namespace", description = "The source's namespace") + @Option(names = "--namespace", description = "The source's namespace") protected String namespace; - @Parameter(names = "--name", description = "The source's name") + @Option(names = "--name", description = "The source's name") protected String sourceName; @Override @@ -592,7 +600,7 @@ void processArguments() throws Exception { } } - @Parameters(commandDescription = "Stops a Pulsar IO source connector") + @Command(description = "Stops a Pulsar IO source connector") protected class DeleteSource extends SourceCommand { @Override @@ -602,7 +610,7 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Gets the information about a Pulsar IO source connector") + @Command(description = "Gets the information about a Pulsar IO source connector") protected class GetSource extends SourceCommand { @Override @@ -616,12 +624,12 @@ void runCmd() throws Exception { /** * List Sources command. */ - @Parameters(commandDescription = "List all running Pulsar IO source connectors") + @Command(description = "List all running Pulsar IO source connectors") protected class ListSources extends BaseCommand { - @Parameter(names = "--tenant", description = "The source's tenant") + @Option(names = "--tenant", description = "The source's tenant") protected String tenant; - @Parameter(names = "--namespace", description = "The source's namespace") + @Option(names = "--namespace", description = "The source's namespace") protected String namespace; @Override @@ -642,10 +650,10 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Check the current status of a Pulsar Source") + @Command(description = "Check the current status of a Pulsar Source") class GetSourceStatus extends SourceCommand { - @Parameter(names = "--instance-id", + @Option(names = "--instance-id", description = "The source instanceId (Get-status of all instances if instance-id is not provided") protected String instanceId; @@ -660,10 +668,10 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Restart source instance") + @Command(description = "Restart source instance") class RestartSource extends SourceCommand { - @Parameter(names = "--instance-id", + @Option(names = "--instance-id", description = "The source instanceId (restart all instances if instance-id is not provided") protected String instanceId; @@ -682,10 +690,10 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Stop source instance") + @Command(description = "Stop source instance") class StopSource extends SourceCommand { - @Parameter(names = "--instance-id", + @Option(names = "--instance-id", description = "The source instanceId (stop all instances if instance-id is not provided") protected String instanceId; @@ -704,10 +712,10 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Start source instance") + @Command(description = "Start source instance") class StartSource extends SourceCommand { - @Parameter(names = "--instance-id", + @Option(names = "--instance-id", description = "The source instanceId (start all instances if instance-id is not provided") protected String instanceId; @@ -726,7 +734,7 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Get the list of Pulsar IO connector sources supported by Pulsar cluster") + @Command(description = "Get the list of Pulsar IO connector sources supported by Pulsar cluster") public class ListBuiltInSources extends BaseCommand { @Override void runCmd() throws Exception { @@ -739,7 +747,7 @@ void runCmd() throws Exception { } } - @Parameters(commandDescription = "Reload the available built-in connectors") + @Command(description = "Reload the available built-in connectors") public class ReloadBuiltInSources extends BaseCommand { @Override diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTenants.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTenants.java index 0d5502e506ceb..324686c3f261b 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTenants.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTenants.java @@ -18,9 +18,6 @@ */ package org.apache.pulsar.admin.cli; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import com.beust.jcommander.converters.CommaParameterSplitter; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -28,10 +25,13 @@ import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; -@Parameters(commandDescription = "Operations about tenants") +@Command(description = "Operations about tenants") public class CmdTenants extends CmdBase { - @Parameters(commandDescription = "List the existing tenants") + @Command(description = "List the existing tenants") private class List extends CliCommand { @Override void run() throws PulsarAdminException { @@ -39,38 +39,35 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Gets the configuration of a tenant") + @Command(description = "Gets the configuration of a tenant") private class Get extends CliCommand { - @Parameter(description = "tenant-name", required = true) - private java.util.List params; + @Parameters(description = "tenant-name", arity = "1") + private String tenant; @Override void run() throws PulsarAdminException { - String tenant = getOneArgument(params); print(getAdmin().tenants().getTenantInfo(tenant)); } } - @Parameters(commandDescription = "Creates a new tenant") + @Command(description = "Creates a new tenant") private class Create extends CliCommand { - @Parameter(description = "tenant-name", required = true) - private java.util.List params; + @Parameters(description = "tenant-name", arity = "1") + private String tenant; - @Parameter(names = { "--admin-roles", + @Option(names = { "--admin-roles", "-r" }, description = "Comma separated list of auth principal allowed to administrate the tenant", - required = false, splitter = CommaParameterSplitter.class) + required = false, split = ",") private java.util.List adminRoles; - @Parameter(names = { "--allowed-clusters", + @Option(names = { "--allowed-clusters", "-c" }, description = "Comma separated allowed clusters. " + "If empty, the tenant will have access to all clusters", - required = false, splitter = CommaParameterSplitter.class) + required = false, split = ",") private java.util.List allowedClusters; @Override void run() throws PulsarAdminException { - String tenant = getOneArgument(params); - if (adminRoles == null) { adminRoles = Collections.emptyList(); } @@ -85,27 +82,25 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Updates the configuration for a tenant") + @Command(description = "Updates the configuration for a tenant") private class Update extends CliCommand { - @Parameter(description = "tenant-name", required = true) - private java.util.List params; + @Parameters(description = "tenant-name", arity = "1") + private String tenant; - @Parameter(names = { "--admin-roles", + @Option(names = { "--admin-roles", "-r" }, description = "Comma separated list of auth principal allowed to administrate the tenant. " + "If empty the current set of roles won't be modified", - required = false, splitter = CommaParameterSplitter.class) + required = false, split = ",") private java.util.List adminRoles; - @Parameter(names = { "--allowed-clusters", + @Option(names = { "--allowed-clusters", "-c" }, description = "Comma separated allowed clusters. " + "If omitted, the current set of clusters will be preserved", - required = false, splitter = CommaParameterSplitter.class) + required = false, split = ",") private java.util.List allowedClusters; @Override void run() throws PulsarAdminException { - String tenant = getOneArgument(params); - if (adminRoles == null) { adminRoles = new ArrayList<>(getAdmin().tenants().getTenantInfo(tenant).getAdminRoles()); } @@ -119,41 +114,34 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Deletes an existing tenant") + @Command(description = "Deletes an existing tenant") private class Delete extends CliCommand { - @Parameter(description = "tenant-name", required = true) - private java.util.List params; + @Parameters(description = "tenant-name", arity = "1") + private String tenant; - @Parameter(names = { "-f", + @Option(names = { "-f", "--force" }, description = "Delete a tenant forcefully by deleting all namespaces under it.") private boolean force = false; @Override void run() throws PulsarAdminException { - String tenant = getOneArgument(params); getAdmin().tenants().deleteTenant(tenant, force); } } public CmdTenants(Supplier admin) { super("tenants", admin); - jcommander.addCommand("list", new List()); - jcommander.addCommand("get", new Get()); - jcommander.addCommand("create", new Create()); - jcommander.addCommand("update", new Update()); - jcommander.addCommand("delete", new Delete()); + addCommand("list", new List()); + addCommand("get", new Get()); + addCommand("create", new Create()); + addCommand("update", new Update()); + addCommand("delete", new Delete()); } - @Parameters(hidden = true) + @Command(hidden = true) static class CmdProperties extends CmdTenants { public CmdProperties(Supplier admin) { super(admin); } - - @Override - public boolean run(String[] args) { - System.err.println("WARN: The properties subcommand is deprecated. Please use tenants instead"); - return super.run(args); - } } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopicPolicies.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopicPolicies.java index d567d0b3671b5..3cc72db2e95f1 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopicPolicies.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopicPolicies.java @@ -18,9 +18,9 @@ */ package org.apache.pulsar.admin.cli; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; +import static org.apache.pulsar.admin.cli.utils.CmdUtils.maxValueCheck; +import static org.apache.pulsar.admin.cli.utils.CmdUtils.positiveCheck; +import com.google.common.base.Strings; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -29,6 +29,10 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.cli.converters.picocli.ByteUnitToIntegerConverter; +import org.apache.pulsar.cli.converters.picocli.ByteUnitToLongConverter; +import org.apache.pulsar.cli.converters.picocli.TimeUnitToMillisConverter; +import org.apache.pulsar.cli.converters.picocli.TimeUnitToSecondsConverter; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.admin.TopicPolicies; @@ -47,364 +51,362 @@ import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.policies.data.SubscribeRate; -import org.apache.pulsar.common.util.RelativeTimeUtil; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; -@Parameters(commandDescription = "Operations on persistent topics") +@Command(description = "Operations on persistent topics") public class CmdTopicPolicies extends CmdBase { public CmdTopicPolicies(Supplier admin) { super("topicPolicies", admin); - jcommander.addCommand("get-message-ttl", new GetMessageTTL()); - jcommander.addCommand("set-message-ttl", new SetMessageTTL()); - jcommander.addCommand("remove-message-ttl", new RemoveMessageTTL()); - - jcommander.addCommand("get-max-unacked-messages-per-consumer", new GetMaxUnackedMessagesPerConsumer()); - jcommander.addCommand("set-max-unacked-messages-per-consumer", new SetMaxUnackedMessagesPerConsumer()); - jcommander.addCommand("remove-max-unacked-messages-per-consumer", new RemoveMaxUnackedMessagesPerConsumer()); - - jcommander.addCommand("get-max-consumers-per-subscription", new GetMaxConsumersPerSubscription()); - jcommander.addCommand("set-max-consumers-per-subscription", new SetMaxConsumersPerSubscription()); - jcommander.addCommand("remove-max-consumers-per-subscription", new RemoveMaxConsumersPerSubscription()); - jcommander.addCommand("set-subscription-types-enabled", new SetSubscriptionTypesEnabled()); - jcommander.addCommand("get-subscription-types-enabled", new GetSubscriptionTypesEnabled()); - jcommander.addCommand("remove-subscription-types-enabled", new RemoveSubscriptionTypesEnabled()); - jcommander.addCommand("get-retention", new GetRetention()); - jcommander.addCommand("set-retention", new SetRetention()); - jcommander.addCommand("remove-retention", new RemoveRetention()); - jcommander.addCommand("get-backlog-quota", new GetBacklogQuotaMap()); - jcommander.addCommand("set-backlog-quota", new SetBacklogQuota()); - jcommander.addCommand("remove-backlog-quota", new RemoveBacklogQuota()); - - jcommander.addCommand("get-max-producers", new GetMaxProducers()); - jcommander.addCommand("set-max-producers", new SetMaxProducers()); - jcommander.addCommand("remove-max-producers", new RemoveMaxProducers()); - - jcommander.addCommand("get-max-message-size", new GetMaxMessageSize()); - jcommander.addCommand("set-max-message-size", new SetMaxMessageSize()); - jcommander.addCommand("remove-max-message-size", new RemoveMaxMessageSize()); - - jcommander.addCommand("set-deduplication", new SetDeduplicationStatus()); - jcommander.addCommand("get-deduplication", new GetDeduplicationStatus()); - jcommander.addCommand("remove-deduplication", new RemoveDeduplicationStatus()); - - jcommander.addCommand("get-deduplication-snapshot-interval", new GetDeduplicationSnapshotInterval()); - jcommander.addCommand("set-deduplication-snapshot-interval", new SetDeduplicationSnapshotInterval()); - jcommander.addCommand("remove-deduplication-snapshot-interval", new RemoveDeduplicationSnapshotInterval()); - - jcommander.addCommand("get-persistence", new GetPersistence()); - jcommander.addCommand("set-persistence", new SetPersistence()); - jcommander.addCommand("remove-persistence", new RemovePersistence()); - - jcommander.addCommand("get-subscription-dispatch-rate", new GetSubscriptionDispatchRate()); - jcommander.addCommand("set-subscription-dispatch-rate", new SetSubscriptionDispatchRate()); - jcommander.addCommand("remove-subscription-dispatch-rate", new RemoveSubscriptionDispatchRate()); - - jcommander.addCommand("get-replicator-dispatch-rate", new GetReplicatorDispatchRate()); - jcommander.addCommand("set-replicator-dispatch-rate", new SetReplicatorDispatchRate()); - jcommander.addCommand("remove-replicator-dispatch-rate", new RemoveReplicatorDispatchRate()); - - jcommander.addCommand("get-publish-rate", new GetPublishRate()); - jcommander.addCommand("set-publish-rate", new SetPublishRate()); - jcommander.addCommand("remove-publish-rate", new RemovePublishRate()); - - jcommander.addCommand("get-compaction-threshold", new GetCompactionThreshold()); - jcommander.addCommand("set-compaction-threshold", new SetCompactionThreshold()); - jcommander.addCommand("remove-compaction-threshold", new RemoveCompactionThreshold()); - - jcommander.addCommand("get-subscribe-rate", new GetSubscribeRate()); - jcommander.addCommand("set-subscribe-rate", new SetSubscribeRate()); - jcommander.addCommand("remove-subscribe-rate", new RemoveSubscribeRate()); - - jcommander.addCommand("get-max-consumers", new GetMaxConsumers()); - jcommander.addCommand("set-max-consumers", new SetMaxConsumers()); - jcommander.addCommand("remove-max-consumers", new RemoveMaxConsumers()); - - jcommander.addCommand("get-delayed-delivery", new GetDelayedDelivery()); - jcommander.addCommand("set-delayed-delivery", new SetDelayedDelivery()); - jcommander.addCommand("remove-delayed-delivery", new RemoveDelayedDelivery()); - - jcommander.addCommand("get-dispatch-rate", new GetDispatchRate()); - jcommander.addCommand("set-dispatch-rate", new SetDispatchRate()); - jcommander.addCommand("remove-dispatch-rate", new RemoveDispatchRate()); - - jcommander.addCommand("get-offload-policies", new GetOffloadPolicies()); - jcommander.addCommand("set-offload-policies", new SetOffloadPolicies()); - jcommander.addCommand("remove-offload-policies", new RemoveOffloadPolicies()); - - jcommander.addCommand("get-max-unacked-messages-per-subscription", new GetMaxUnackedMessagesPerSubscription()); - jcommander.addCommand("set-max-unacked-messages-per-subscription", new SetMaxUnackedMessagesPerSubscription()); - jcommander.addCommand("remove-max-unacked-messages-per-subscription", + addCommand("get-message-ttl", new GetMessageTTL()); + addCommand("set-message-ttl", new SetMessageTTL()); + addCommand("remove-message-ttl", new RemoveMessageTTL()); + + addCommand("get-max-unacked-messages-per-consumer", new GetMaxUnackedMessagesPerConsumer()); + addCommand("set-max-unacked-messages-per-consumer", new SetMaxUnackedMessagesPerConsumer()); + addCommand("remove-max-unacked-messages-per-consumer", new RemoveMaxUnackedMessagesPerConsumer()); + + addCommand("get-max-consumers-per-subscription", new GetMaxConsumersPerSubscription()); + addCommand("set-max-consumers-per-subscription", new SetMaxConsumersPerSubscription()); + addCommand("remove-max-consumers-per-subscription", new RemoveMaxConsumersPerSubscription()); + addCommand("set-subscription-types-enabled", new SetSubscriptionTypesEnabled()); + addCommand("get-subscription-types-enabled", new GetSubscriptionTypesEnabled()); + addCommand("remove-subscription-types-enabled", new RemoveSubscriptionTypesEnabled()); + addCommand("get-retention", new GetRetention()); + addCommand("set-retention", new SetRetention()); + addCommand("remove-retention", new RemoveRetention()); + addCommand("get-backlog-quota", new GetBacklogQuotaMap()); + addCommand("set-backlog-quota", new SetBacklogQuota()); + addCommand("remove-backlog-quota", new RemoveBacklogQuota()); + + addCommand("get-max-producers", new GetMaxProducers()); + addCommand("set-max-producers", new SetMaxProducers()); + addCommand("remove-max-producers", new RemoveMaxProducers()); + + addCommand("get-max-message-size", new GetMaxMessageSize()); + addCommand("set-max-message-size", new SetMaxMessageSize()); + addCommand("remove-max-message-size", new RemoveMaxMessageSize()); + + addCommand("set-deduplication", new SetDeduplicationStatus()); + addCommand("get-deduplication", new GetDeduplicationStatus()); + addCommand("remove-deduplication", new RemoveDeduplicationStatus()); + + addCommand("get-deduplication-snapshot-interval", new GetDeduplicationSnapshotInterval()); + addCommand("set-deduplication-snapshot-interval", new SetDeduplicationSnapshotInterval()); + addCommand("remove-deduplication-snapshot-interval", new RemoveDeduplicationSnapshotInterval()); + + addCommand("get-persistence", new GetPersistence()); + addCommand("set-persistence", new SetPersistence()); + addCommand("remove-persistence", new RemovePersistence()); + + addCommand("get-subscription-dispatch-rate", new GetSubscriptionDispatchRate()); + addCommand("set-subscription-dispatch-rate", new SetSubscriptionDispatchRate()); + addCommand("remove-subscription-dispatch-rate", new RemoveSubscriptionDispatchRate()); + + addCommand("get-replicator-dispatch-rate", new GetReplicatorDispatchRate()); + addCommand("set-replicator-dispatch-rate", new SetReplicatorDispatchRate()); + addCommand("remove-replicator-dispatch-rate", new RemoveReplicatorDispatchRate()); + + addCommand("get-publish-rate", new GetPublishRate()); + addCommand("set-publish-rate", new SetPublishRate()); + addCommand("remove-publish-rate", new RemovePublishRate()); + + addCommand("get-compaction-threshold", new GetCompactionThreshold()); + addCommand("set-compaction-threshold", new SetCompactionThreshold()); + addCommand("remove-compaction-threshold", new RemoveCompactionThreshold()); + + addCommand("get-subscribe-rate", new GetSubscribeRate()); + addCommand("set-subscribe-rate", new SetSubscribeRate()); + addCommand("remove-subscribe-rate", new RemoveSubscribeRate()); + + addCommand("get-max-consumers", new GetMaxConsumers()); + addCommand("set-max-consumers", new SetMaxConsumers()); + addCommand("remove-max-consumers", new RemoveMaxConsumers()); + + addCommand("get-delayed-delivery", new GetDelayedDelivery()); + addCommand("set-delayed-delivery", new SetDelayedDelivery()); + addCommand("remove-delayed-delivery", new RemoveDelayedDelivery()); + + addCommand("get-dispatch-rate", new GetDispatchRate()); + addCommand("set-dispatch-rate", new SetDispatchRate()); + addCommand("remove-dispatch-rate", new RemoveDispatchRate()); + + addCommand("get-offload-policies", new GetOffloadPolicies()); + addCommand("set-offload-policies", new SetOffloadPolicies()); + addCommand("remove-offload-policies", new RemoveOffloadPolicies()); + + addCommand("get-max-unacked-messages-per-subscription", new GetMaxUnackedMessagesPerSubscription()); + addCommand("set-max-unacked-messages-per-subscription", new SetMaxUnackedMessagesPerSubscription()); + addCommand("remove-max-unacked-messages-per-subscription", new RemoveMaxUnackedMessagesPerSubscription()); - jcommander.addCommand("get-inactive-topic-policies", new GetInactiveTopicPolicies()); - jcommander.addCommand("set-inactive-topic-policies", new SetInactiveTopicPolicies()); - jcommander.addCommand("remove-inactive-topic-policies", new RemoveInactiveTopicPolicies()); + addCommand("get-inactive-topic-policies", new GetInactiveTopicPolicies()); + addCommand("set-inactive-topic-policies", new SetInactiveTopicPolicies()); + addCommand("remove-inactive-topic-policies", new RemoveInactiveTopicPolicies()); - jcommander.addCommand("get-max-subscriptions-per-topic", new GetMaxSubscriptionsPerTopic()); - jcommander.addCommand("set-max-subscriptions-per-topic", new SetMaxSubscriptionsPerTopic()); - jcommander.addCommand("remove-max-subscriptions-per-topic", new RemoveMaxSubscriptionsPerTopic()); + addCommand("get-max-subscriptions-per-topic", new GetMaxSubscriptionsPerTopic()); + addCommand("set-max-subscriptions-per-topic", new SetMaxSubscriptionsPerTopic()); + addCommand("remove-max-subscriptions-per-topic", new RemoveMaxSubscriptionsPerTopic()); - jcommander.addCommand("remove-schema-compatibility-strategy", new RemoveSchemaCompatibilityStrategy()); - jcommander.addCommand("set-schema-compatibility-strategy", new SetSchemaCompatibilityStrategy()); - jcommander.addCommand("get-schema-compatibility-strategy", new GetSchemaCompatibilityStrategy()); + addCommand("remove-schema-compatibility-strategy", new RemoveSchemaCompatibilityStrategy()); + addCommand("set-schema-compatibility-strategy", new SetSchemaCompatibilityStrategy()); + addCommand("get-schema-compatibility-strategy", new GetSchemaCompatibilityStrategy()); - jcommander.addCommand("get-entry-filters-per-topic", new GetEntryFiltersPerTopic()); - jcommander.addCommand("set-entry-filters-per-topic", new SetEntryFiltersPerTopic()); - jcommander.addCommand("remove-entry-filters-per-topic", new RemoveEntryFiltersPerTopic()); + addCommand("get-entry-filters-per-topic", new GetEntryFiltersPerTopic()); + addCommand("set-entry-filters-per-topic", new SetEntryFiltersPerTopic()); + addCommand("remove-entry-filters-per-topic", new RemoveEntryFiltersPerTopic()); - jcommander.addCommand("set-auto-subscription-creation", new SetAutoSubscriptionCreation()); - jcommander.addCommand("get-auto-subscription-creation", new GetAutoSubscriptionCreation()); - jcommander.addCommand("remove-auto-subscription-creation", new RemoveAutoSubscriptionCreation()); + addCommand("set-auto-subscription-creation", new SetAutoSubscriptionCreation()); + addCommand("get-auto-subscription-creation", new GetAutoSubscriptionCreation()); + addCommand("remove-auto-subscription-creation", new RemoveAutoSubscriptionCreation()); + + addCommand("set-dispatcher-pause-on-ack-state-persistent", + new SetDispatcherPauseOnAckStatePersistent()); + addCommand("get-dispatcher-pause-on-ack-state-persistent", + new GetDispatcherPauseOnAckStatePersistent()); + addCommand("remove-dispatcher-pause-on-ack-state-persistent", + new RemoveDispatcherPauseOnAckStatePersistent()); } - @Parameters(commandDescription = "Get entry filters for a topic") + @Command(description = "Get entry filters for a topic") private class GetEntryFiltersPerTopic extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getEntryFiltersPerTopic(persistentTopic, applied)); } } - @Parameters(commandDescription = "Set entry filters for a topic") + @Command(description = "Set entry filters for a topic") private class SetEntryFiltersPerTopic extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--entry-filters-name", "-efn" }, + @Option(names = { "--entry-filters-name", "-efn" }, description = "The class name for the entry filter.", required = true) private String entryFiltersName = ""; - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).setEntryFiltersPerTopic(persistentTopic, new EntryFilters(entryFiltersName)); } } - @Parameters(commandDescription = "Remove entry filters for a topic") + @Command(description = "Remove entry filters for a topic") private class RemoveEntryFiltersPerTopic extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).removeEntryFiltersPerTopic(persistentTopic); } } - @Parameters(commandDescription = "Get max consumers per subscription for a topic") + @Command(description = "Get max consumers per subscription for a topic") private class GetMaxConsumersPerSubscription extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getMaxConsumersPerSubscription(persistentTopic)); } } - @Parameters(commandDescription = "Set max consumers per subscription for a topic") + @Command(description = "Set max consumers per subscription for a topic") private class SetMaxConsumersPerSubscription extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--max-consumers-per-subscription", "-c" }, + @Option(names = { "--max-consumers-per-subscription", "-c" }, description = "maxConsumersPerSubscription for a namespace", required = true) private int maxConsumersPerSubscription; - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).setMaxConsumersPerSubscription(persistentTopic, maxConsumersPerSubscription); } } - @Parameters(commandDescription = "Remove max consumers per subscription for a topic") + @Command(description = "Remove max consumers per subscription for a topic") private class RemoveMaxConsumersPerSubscription extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).removeMaxConsumersPerSubscription(persistentTopic); } } - @Parameters(commandDescription = "Get max unacked messages policy per consumer for a topic") + @Command(description = "Get max unacked messages policy per consumer for a topic") private class GetMaxUnackedMessagesPerConsumer extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; - @Parameter(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getMaxUnackedMessagesOnConsumer(persistentTopic, applied)); } } - @Parameters(commandDescription = "Remove max unacked messages policy per consumer for a topic") + @Command(description = "Remove max unacked messages policy per consumer for a topic") private class RemoveMaxUnackedMessagesPerConsumer extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).removeMaxUnackedMessagesOnConsumer(persistentTopic); } } - @Parameters(commandDescription = "Set max unacked messages policy per consumer for a topic") + @Command(description = "Set max unacked messages policy per consumer for a topic") private class SetMaxUnackedMessagesPerConsumer extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"-m", "--maxNum"}, description = "max unacked messages num on consumer", required = true) + @Option(names = {"-m", "--maxNum"}, description = "max unacked messages num on consumer", required = true) private int maxNum; - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).setMaxUnackedMessagesOnConsumer(persistentTopic, maxNum); } } - @Parameters(commandDescription = "Get the message TTL for a topic") + @Command(description = "Get the message TTL for a topic") private class GetMessageTTL extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; - @Parameter(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getMessageTTL(persistentTopic, applied)); } } - @Parameters(commandDescription = "Set message TTL for a topic") + @Command(description = "Set message TTL for a topic") private class SetMessageTTL extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-t", "--ttl" }, + @Option(names = { "-t", "--ttl" }, description = "Message TTL for topic in seconds (or minutes, hours, days, weeks eg: 100m, 3h, 2d, 5w), " - + "allowed range from 1 to Integer.MAX_VALUE", required = true) - private String messageTTLStr; + + "allowed range from 1 to Integer.MAX_VALUE", required = true, + converter = TimeUnitToSecondsConverter.class) + private Long messageTTLInSecond; - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - long messageTTLInSecond; - try { - messageTTLInSecond = RelativeTimeUtil.parseRelativeTimeInSeconds(messageTTLStr); - } catch (IllegalArgumentException e) { - throw new ParameterException(e.getMessage()); - } - - if (messageTTLInSecond < 0 || messageTTLInSecond > Integer.MAX_VALUE) { - throw new ParameterException( - String.format("Message TTL cannot be negative or greater than %d seconds", Integer.MAX_VALUE)); - } - - String persistentTopic = validatePersistentTopic(params); - getTopicPolicies(isGlobal).setMessageTTL(persistentTopic, (int) messageTTLInSecond); + String persistentTopic = validatePersistentTopic(topicName); + getTopicPolicies(isGlobal).setMessageTTL(persistentTopic, messageTTLInSecond.intValue()); } } - @Parameters(commandDescription = "Remove message TTL for a topic") + @Command(description = "Remove message TTL for a topic") private class RemoveMessageTTL extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).removeMessageTTL(persistentTopic); } } - @Parameters(commandDescription = "Set subscription types enabled for a topic") + @Command(description = "Set subscription types enabled for a topic") private class SetSubscriptionTypesEnabled extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--types", "-t"}, description = "Subscription types enabled list (comma separated values)." - + " Possible values: (Exclusive, Shared, Failover, Key_Shared).", required = true) + @Option(names = {"--types", "-t"}, description = "Subscription types enabled list (comma separated values)." + + " Possible values: (Exclusive, Shared, Failover, Key_Shared).", required = true, split = ",") private List subTypes; - @Parameter(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); Set types = new HashSet<>(); subTypes.forEach(s -> { SubscriptionType subType; @@ -420,430 +422,421 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get subscription types enabled for a topic") + @Command(description = "Get subscription types enabled for a topic") private class GetSubscriptionTypesEnabled extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getSubscriptionTypesEnabled(persistentTopic)); } } - @Parameters(commandDescription = "Remove subscription types enabled for a topic") + @Command(description = "Remove subscription types enabled for a topic") private class RemoveSubscriptionTypesEnabled extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + "If set to true, the removing operation will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).removeSubscriptionTypesEnabled(persistentTopic); } } - @Parameters(commandDescription = "Get max number of consumers for a topic") + @Command(description = "Get max number of consumers for a topic") private class GetMaxConsumers extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; - @Parameter(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getMaxConsumers(persistentTopic, applied)); } } - @Parameters(commandDescription = "Set max number of consumers for a topic") + @Command(description = "Set max number of consumers for a topic") private class SetMaxConsumers extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--max-consumers", "-c" }, description = "Max consumers for a topic", required = true) + @Option(names = { "--max-consumers", "-c" }, description = "Max consumers for a topic", required = true) private int maxConsumers; - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).setMaxConsumers(persistentTopic, maxConsumers); } } - @Parameters(commandDescription = "Remove max number of consumers for a topic") + @Command(description = "Remove max number of consumers for a topic") private class RemoveMaxConsumers extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).removeMaxConsumers(persistentTopic); } } - @Parameters(commandDescription = "Get the retention policy for a topic") + @Command(description = "Get the retention policy for a topic") private class GetRetention extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; - @Parameter(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + "If set to true, the broker returns global topic policies" + "If set to false or not set, the broker returns local topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getRetention(persistentTopic, applied)); } } - @Parameters(commandDescription = "Set the retention policy for a topic") + @Command(description = "Set the retention policy for a topic") private class SetRetention extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--time", + @Option(names = { "--time", "-t" }, description = "Retention time with optional time unit suffix. " + "For example, 100m, 3h, 2d, 5w. " + "If the time unit is not specified, the default unit is seconds. For example, " + "-t 120 sets retention to 2 minutes. " - + "0 means no retention and -1 means infinite time retention.", required = true) - private String retentionTimeStr; + + "0 means no retention and -1 means infinite time retention.", required = true, + converter = TimeUnitToSecondsConverter.class) + private Long retentionTimeInSec; - @Parameter(names = { "--size", "-s" }, description = "Retention size limit with optional size unit suffix. " + @Option(names = { "--size", "-s" }, description = "Retention size limit with optional size unit suffix. " + "For example, 4096, 10M, 16G, 3T. The size unit suffix character can be k/K, m/M, g/G, or t/T. " + "If the size unit suffix is not specified, the default unit is bytes. " - + "0 or less than 1MB means no retention and -1 means infinite size retention", required = true) - private String limitStr; + + "0 or less than 1MB means no retention and -1 means infinite size retention", required = true, + converter = ByteUnitToIntegerConverter.class) + private Integer sizeLimit; - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + "If set to true, the policy is replicated to other clusters asynchronously, " + "If set to false or not set, the topic retention policy is replicated to local clusters.") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); - long sizeLimit = validateSizeString(limitStr); - long retentionTimeInSec = RelativeTimeUtil.parseRelativeTimeInSeconds(retentionTimeStr); - - final int retentionTimeInMin; - if (retentionTimeInSec != -1) { - retentionTimeInMin = (int) TimeUnit.SECONDS.toMinutes(retentionTimeInSec); - } else { - retentionTimeInMin = -1; - } - - final int retentionSizeInMB; - if (sizeLimit != -1) { - retentionSizeInMB = (int) (sizeLimit / (1024 * 1024)); - } else { - retentionSizeInMB = -1; - } + String persistentTopic = validatePersistentTopic(topicName); + final int retentionTimeInMin = retentionTimeInSec != -1 + ? (int) TimeUnit.SECONDS.toMinutes(retentionTimeInSec) + : retentionTimeInSec.intValue(); + final int retentionSizeInMB = sizeLimit != -1 + ? (int) (sizeLimit / (1024 * 1024)) + : sizeLimit; getTopicPolicies(isGlobal).setRetention(persistentTopic, new RetentionPolicies(retentionTimeInMin, retentionSizeInMB)); } } - @Parameters(commandDescription = "Remove the retention policy for a topic") + @Command(description = "Remove the retention policy for a topic") private class RemoveRetention extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + "If set to true, the removing operation is replicated to other clusters asynchronously" + "If set to false or not set, the topic retention policy is replicated to local clusters.") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).removeRetention(persistentTopic); } } - @Parameters(commandDescription = "Get max unacked messages policy per subscription for a topic") + @Command(description = "Get max unacked messages policy per subscription for a topic") private class GetMaxUnackedMessagesPerSubscription extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; - @Parameter(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + "If set to true, the removing operation will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getMaxUnackedMessagesOnSubscription(persistentTopic, applied)); } } - @Parameters(commandDescription = "Remove max unacked messages policy per subscription for a topic") + @Command(description = "Remove max unacked messages policy per subscription for a topic") private class RemoveMaxUnackedMessagesPerSubscription extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + "If set to true, the removing operation will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).removeMaxUnackedMessagesOnSubscription(persistentTopic); } } - @Parameters(commandDescription = "Set max unacked messages policy on subscription for a topic") + @Command(description = "Set max unacked messages policy on subscription for a topic") private class SetMaxUnackedMessagesPerSubscription extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"-m", "--maxNum"}, + @Option(names = {"-m", "--maxNum"}, description = "max unacked messages num on subscription", required = true) private int maxNum; - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + "If set to true, the removing operation will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).setMaxUnackedMessagesOnSubscription(persistentTopic, maxNum); } } - @Parameters(commandDescription = "Get max number of producers for a topic") + @Command(description = "Get max number of producers for a topic") private class GetMaxProducers extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; - @Parameter(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getMaxProducers(persistentTopic, applied)); } } - @Parameters(commandDescription = "Set max number of producers for a topic") + @Command(description = "Set max number of producers for a topic") private class SetMaxProducers extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--max-producers", "-p"}, description = "Max producers for a topic", required = true) + @Option(names = {"--max-producers", "-p"}, description = "Max producers for a topic", required = true) private int maxProducers; - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).setMaxProducers(persistentTopic, maxProducers); } } - @Parameters(commandDescription = "Get the delayed delivery policy for a topic") + @Command(description = "Get the delayed delivery policy for a topic") private class GetDelayedDelivery extends CliCommand { - @Parameter(description = "tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; - @Parameter(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal; @Override void run() throws PulsarAdminException { - String topicName = validateTopicName(params); - print(getTopicPolicies(isGlobal).getDelayedDeliveryPolicy(topicName, applied)); + String topic = validateTopicName(topicName); + print(getTopicPolicies(isGlobal).getDelayedDeliveryPolicy(topic, applied)); } } - @Parameters(commandDescription = "Set the delayed delivery policy on a topic") + @Command(description = "Set the delayed delivery policy on a topic") private class SetDelayedDelivery extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--enable", "-e" }, description = "Enable delayed delivery messages") + @Option(names = { "--enable", "-e" }, description = "Enable delayed delivery messages") private boolean enable = false; - @Parameter(names = { "--disable", "-d" }, description = "Disable delayed delivery messages") + @Option(names = { "--disable", "-d" }, description = "Disable delayed delivery messages") private boolean disable = false; - @Parameter(names = { "--time", "-t" }, description = "The tick time for when retrying on " + @Option(names = { "--time", "-t" }, description = "The tick time for when retrying on " + "delayed delivery messages, affecting the accuracy of the delivery time compared to " - + "the scheduled time. (eg: 1s, 10s, 1m, 5h, 3d)") - private String delayedDeliveryTimeStr = "1s"; + + "the scheduled time. (eg: 1s, 10s, 1m, 5h, 3d)", + converter = TimeUnitToMillisConverter.class) + private Long delayedDeliveryTimeInMills = 1000L; - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal; + @Option(names = { "--maxDelay", "-md" }, + description = "The max allowed delay for delayed delivery. (eg: 1s, 10s, 1m, 5h, 3d)", + converter = TimeUnitToMillisConverter.class) + private Long delayedDeliveryMaxDelayInMillis = 0L; + @Override void run() throws PulsarAdminException { - String topicName = validateTopicName(params); - long delayedDeliveryTimeInMills; - try { - delayedDeliveryTimeInMills = TimeUnit.SECONDS.toMillis( - RelativeTimeUtil.parseRelativeTimeInSeconds(delayedDeliveryTimeStr)); - } catch (IllegalArgumentException exception) { - throw new ParameterException(exception.getMessage()); - } - + String topic = validateTopicName(topicName); if (enable == disable) { throw new ParameterException("Need to specify either --enable or --disable"); } - getTopicPolicies(isGlobal).setDelayedDeliveryPolicy(topicName, DelayedDeliveryPolicies.builder() + getTopicPolicies(isGlobal).setDelayedDeliveryPolicy(topic, DelayedDeliveryPolicies.builder() .tickTime(delayedDeliveryTimeInMills) .active(enable) + .maxDeliveryDelayInMillis(delayedDeliveryMaxDelayInMillis) .build()); } } - @Parameters(commandDescription = "Remove the delayed delivery policy on a topic") + @Command(description = "Remove the delayed delivery policy on a topic") private class RemoveDelayedDelivery extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal; @Override void run() throws PulsarAdminException { - String topicName = validateTopicName(params); - getTopicPolicies(isGlobal).removeDelayedDeliveryPolicy(topicName); + String topic = validateTopicName(topicName); + getTopicPolicies(isGlobal).removeDelayedDeliveryPolicy(topic); } } - @Parameters(commandDescription = "Remove max number of producers for a topic") + @Command(description = "Remove max number of producers for a topic") private class RemoveMaxProducers extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + "If set to true, the removing operation will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).removeMaxProducers(persistentTopic); } } - @Parameters(commandDescription = "Get max message size for a topic") + @Command(description = "Get max message size for a topic") private class GetMaxMessageSize extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--global", "-g"}, description = "Whether to get this policy globally. " + @Option(names = {"--global", "-g"}, description = "Whether to get this policy globally. " + "If set to true, broker returns global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getMaxMessageSize(persistentTopic)); } } - @Parameters(commandDescription = "Set max message size for a topic") + @Command(description = "Set max message size for a topic") private class SetMaxMessageSize extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--max-message-size", "-m"}, description = "Max message size for a topic", required = true) + @Option(names = {"--max-message-size", "-m"}, description = "Max message size for a topic", required = true) private int maxMessageSize; - @Parameter(names = {"--global", "-g"}, description = "Whether to set this policy globally.") + @Option(names = {"--global", "-g"}, description = "Whether to set this policy globally.") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).setMaxMessageSize(persistentTopic, maxMessageSize); } } - @Parameters(commandDescription = "Remove max message size for a topic") + @Command(description = "Remove max message size for a topic") private class RemoveMaxMessageSize extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--global", "-g"}, description = "Whether to remove this policy globally. ") + @Option(names = {"--global", "-g"}, description = "Whether to remove this policy globally. ") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).removeMaxMessageSize(persistentTopic); } } - @Parameters(commandDescription = "Enable or disable status for a topic") + @Command(description = "Enable or disable status for a topic") private class SetDeduplicationStatus extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--enable", "-e" }, description = "Enable deduplication") + @Option(names = { "--enable", "-e" }, description = "Enable deduplication") private boolean enable = false; - @Parameter(names = { "--disable", "-d" }, description = "Disable deduplication") + @Option(names = { "--disable", "-d" }, description = "Disable deduplication") private boolean disable = false; - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + "If set to true, the removing operation will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); if (enable == disable) { throw new ParameterException("Need to specify either --enable or --disable"); @@ -852,64 +845,64 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get the deduplication status for a topic") + @Command(description = "Get the deduplication status for a topic") private class GetDeduplicationStatus extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--global", "-g" }, description = "Whether to get this policy globally. ") + @Option(names = { "--global", "-g" }, description = "Whether to get this policy globally. ") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getDeduplicationStatus(persistentTopic)); } } - @Parameters(commandDescription = "Remove the deduplication status for a topic") + @Command(description = "Remove the deduplication status for a topic") private class RemoveDeduplicationStatus extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + "If set to true, the removing operation will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).removeDeduplicationStatus(persistentTopic); } } - @Parameters(commandDescription = "Get deduplication snapshot interval for a topic") + @Command(description = "Get deduplication snapshot interval for a topic") private class GetDeduplicationSnapshotInterval extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--global", "-g"}, description = "Whether to get this policy globally. " + @Option(names = {"--global", "-g"}, description = "Whether to get this policy globally. " + "If set to true, broker returns global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getDeduplicationSnapshotInterval(persistentTopic)); } } - @Parameters(commandDescription = "Set deduplication snapshot interval for a topic") + @Command(description = "Set deduplication snapshot interval for a topic") private class SetDeduplicationSnapshotInterval extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"-i", "--interval"}, description = + @Option(names = {"-i", "--interval"}, description = "Deduplication snapshot interval for topic in second, allowed range from 0 to Integer.MAX_VALUE", required = true) private int interval; - @Parameter(names = {"--global", "-g"}, description = "Whether to set this policy globally.") + @Option(names = {"--global", "-g"}, description = "Whether to set this policy globally.") private boolean isGlobal = false; @Override @@ -918,72 +911,74 @@ void run() throws PulsarAdminException { throw new ParameterException(String.format("Invalid interval '%d'. ", interval)); } - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).setDeduplicationSnapshotInterval(persistentTopic, interval); } } - @Parameters(commandDescription = "Remove deduplication snapshot interval for a topic") + @Command(description = "Remove deduplication snapshot interval for a topic") private class RemoveDeduplicationSnapshotInterval extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--global", "-g"}, description = "Whether to remove this policy globally. ") + @Option(names = {"--global", "-g"}, description = "Whether to remove this policy globally. ") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).removeDeduplicationSnapshotInterval(persistentTopic); } } - @Parameters(commandDescription = "Get the backlog quota policies for a topic") + @Command(description = "Get the backlog quota policies for a topic") private class GetBacklogQuotaMap extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"-ap", "--applied"}, description = "Get the applied policy of the topic") + @Option(names = {"-ap", "--applied"}, description = "Get the applied policy of the topic") private boolean applied = false; - @Parameter(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getBacklogQuotaMap(persistentTopic, applied)); } } - @Parameters(commandDescription = "Set a backlog quota policy for a topic") + @Command(description = "Set a backlog quota policy for a topic") private class SetBacklogQuota extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-l", "--limit" }, description = "Size limit (eg: 10M, 16G)") - private String limitStr = null; + @Option(names = { "-l", "--limit" }, description = "Size limit (eg: 10M, 16G)", + converter = ByteUnitToLongConverter.class) + private Long limit; - @Parameter(names = { "-lt", "--limitTime" }, + @Option(names = { "-lt", "--limitTime" }, description = "Time limit in second (or minutes, hours, days, weeks eg: 100m, 3h, 2d, 5w), " - + "non-positive number for disabling time limit.") - private String limitTimeStr = null; + + "non-positive number for disabling time limit.", + converter = TimeUnitToSecondsConverter.class) + private Long limitTimeInSec; - @Parameter(names = { "-p", "--policy" }, description = "Retention policy to enforce when the limit is reached. " + @Option(names = { "-p", "--policy" }, description = "Retention policy to enforce when the limit is reached. " + "Valid options are: [producer_request_hold, producer_exception, consumer_backlog_eviction]", required = true) private String policyStr; - @Parameter(names = {"-t", "--type"}, description = "Backlog quota type to set. Valid options are: " + @Option(names = {"-t", "--type"}, description = "Backlog quota type to set. Valid options are: " + "destination_storage (default) and message_age. " + "destination_storage limits backlog by size. " + "message_age limits backlog by time, that is, message timestamp (broker or publish timestamp). " + "You can set size or time to control the backlog, or combine them together to control the backlog. ") private String backlogQuotaTypeStr = BacklogQuota.BacklogQuotaType.destination_storage.name(); - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal = false; @@ -1004,32 +999,21 @@ void run() throws PulsarAdminException { throw new ParameterException(String.format("Invalid backlog quota type '%s'. Valid options are: %s", backlogQuotaTypeStr, Arrays.toString(BacklogQuota.BacklogQuotaType.values()))); } - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); BacklogQuota.Builder builder = BacklogQuota.builder().retentionPolicy(policy); if (backlogQuotaType == BacklogQuota.BacklogQuotaType.destination_storage) { // set quota by storage size - if (limitStr == null) { + if (limit == null) { throw new ParameterException("Quota type of 'destination_storage' needs a size limit"); } - long limit = validateSizeString(limitStr); - builder.limitSize((int) limit); + builder.limitSize(limit); } else { // set quota by time - if (limitTimeStr == null) { + if (limitTimeInSec == null) { throw new ParameterException("Quota type of 'message_age' needs a time limit"); } - long limitTimeInSec; - try { - limitTimeInSec = RelativeTimeUtil.parseRelativeTimeInSeconds(limitTimeStr); - } catch (IllegalArgumentException e) { - throw new ParameterException(e.getMessage()); - } - if (limitTimeInSec > Integer.MAX_VALUE) { - throw new ParameterException( - String.format("Time limit cannot be greater than %d seconds", Integer.MAX_VALUE)); - } - builder.limitTime((int) limitTimeInSec); + builder.limitTime(limitTimeInSec.intValue()); } getTopicPolicies(isGlobal).setBacklogQuota(persistentTopic, builder.build(), @@ -1037,185 +1021,185 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Remove a backlog quota policy from a topic") + @Command(description = "Remove a backlog quota policy from a topic") private class RemoveBacklogQuota extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"-t", "--type"}, description = "Backlog quota type to remove") + @Option(names = {"-t", "--type"}, description = "Backlog quota type to remove") private String backlogQuotaType = BacklogQuota.BacklogQuotaType.destination_storage.name(); - @Parameter(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + "If set to true, the removing operation will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal) .removeBacklogQuota(persistentTopic, BacklogQuota.BacklogQuotaType.valueOf(backlogQuotaType)); } } - @Parameters(commandDescription = "Get publish rate for a topic") + @Command(description = "Get publish rate for a topic") private class GetPublishRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--global", "-g"}, description = "Whether to get this policy globally. " + @Option(names = {"--global", "-g"}, description = "Whether to get this policy globally. " + "If set to true, broker returns global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getPublishRate(persistentTopic)); } } - @Parameters(commandDescription = "Set publish rate for a topic") + @Command(description = "Set publish rate for a topic") private class SetPublishRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--msg-publish-rate", "-m"}, description = "message-publish-rate (default -1 will be " + @Option(names = {"--msg-publish-rate", "-m"}, description = "message-publish-rate (default -1 will be " + "overwrite if not passed)", required = false) private int msgPublishRate = -1; - @Parameter(names = {"--byte-publish-rate", "-b"}, description = "byte-publish-rate " + @Option(names = {"--byte-publish-rate", "-b"}, description = "byte-publish-rate " + "(default -1 will be overwrite if not passed)", required = false) private long bytePublishRate = -1; - @Parameter(names = {"--global", "-g"}, description = "Whether to set this policy globally.") + @Option(names = {"--global", "-g"}, description = "Whether to set this policy globally.") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).setPublishRate(persistentTopic, new PublishRate(msgPublishRate, bytePublishRate)); } } - @Parameters(commandDescription = "Remove publish rate for a topic") + @Command(description = "Remove publish rate for a topic") private class RemovePublishRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--global", "-g"}, description = "Whether to remove this policy globally. ") + @Option(names = {"--global", "-g"}, description = "Whether to remove this policy globally. ") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).removePublishRate(persistentTopic); } } - @Parameters(commandDescription = "Get consumer subscribe rate for a topic") + @Command(description = "Get consumer subscribe rate for a topic") private class GetSubscribeRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; - @Parameter(names = {"--global", "-g"}, description = "Whether to get this policy globally. " + @Option(names = {"--global", "-g"}, description = "Whether to get this policy globally. " + "If set to true, broker returns global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getSubscribeRate(persistentTopic, applied)); } } - @Parameters(commandDescription = "Set consumer subscribe rate for a topic") + @Command(description = "Set consumer subscribe rate for a topic") private class SetSubscribeRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--subscribe-rate", + @Option(names = { "--subscribe-rate", "-sr" }, description = "subscribe-rate (default -1 will be overwrite if not passed)", required = false) private int subscribeRate = -1; - @Parameter(names = { "--subscribe-rate-period", + @Option(names = { "--subscribe-rate-period", "-st" }, description = "subscribe-rate-period in second type " + "(default 30 second will be overwrite if not passed)", required = false) private int subscribeRatePeriodSec = 30; - @Parameter(names = {"--global", "-g"}, description = "Whether to set this policy globally.") + @Option(names = {"--global", "-g"}, description = "Whether to set this policy globally.") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).setSubscribeRate(persistentTopic, new SubscribeRate(subscribeRate, subscribeRatePeriodSec)); } } - @Parameters(commandDescription = "Remove consumer subscribe rate for a topic") + @Command(description = "Remove consumer subscribe rate for a topic") private class RemoveSubscribeRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--global", "-g"}, description = "Whether to remove this policy globally. ") + @Option(names = {"--global", "-g"}, description = "Whether to remove this policy globally. ") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).removeSubscribeRate(persistentTopic); } } - @Parameters(commandDescription = "Get the persistence policies for a topic") + @Command(description = "Get the persistence policies for a topic") private class GetPersistence extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--global", "-g" }, description = "Whether to get this policy globally. " - + "If set to true, broker returned global topic policies", arity = 0) + @Option(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + + "If set to true, broker returned global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getPersistence(persistentTopic)); } } - @Parameters(commandDescription = "Set the persistence policies for a topic") + @Command(description = "Set the persistence policies for a topic") private class SetPersistence extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-e", + @Option(names = { "-e", "--bookkeeper-ensemble" }, description = "Number of bookies to use for a topic") private int bookkeeperEnsemble = 2; - @Parameter(names = { "-w", + @Option(names = { "-w", "--bookkeeper-write-quorum" }, description = "How many writes to make of each entry") private int bookkeeperWriteQuorum = 2; - @Parameter(names = { "-a", "--bookkeeper-ack-quorum" }, + @Option(names = { "-a", "--bookkeeper-ack-quorum" }, description = "Number of acks (guaranteed copies) to wait for each entry") private int bookkeeperAckQuorum = 2; - @Parameter(names = { "-r", "--ml-mark-delete-max-rate" }, + @Option(names = { "-r", "--ml-mark-delete-max-rate" }, description = "Throttling rate of mark-delete operation (0 means no throttle)") private double managedLedgerMaxMarkDeleteRate = 0; - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " - + "If set to true, the policy will be replicate to other clusters asynchronously", arity = 0) + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + + "If set to true, the policy will be replicate to other clusters asynchronously", arity = "0") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); if (bookkeeperEnsemble <= 0 || bookkeeperWriteQuorum <= 0 || bookkeeperAckQuorum <= 0) { throw new ParameterException("[--bookkeeper-ensemble], [--bookkeeper-write-quorum] " + "and [--bookkeeper-ack-quorum] must greater than 0."); @@ -1228,128 +1212,129 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Remove the persistence policy for a topic") + @Command(description = "Remove the persistence policy for a topic") private class RemovePersistence extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + "If set to true, the removing operation will be replicate to other clusters asynchronously" - , arity = 0) + , arity = "0") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).removePersistence(persistentTopic); } } - @Parameters(commandDescription = "Get compaction threshold for a topic") + @Command(description = "Get compaction threshold for a topic") private class GetCompactionThreshold extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; - @Parameter(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getCompactionThreshold(persistentTopic, applied)); } } - @Parameters(commandDescription = "Set compaction threshold for a topic") + @Command(description = "Set compaction threshold for a topic") private class SetCompactionThreshold extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--threshold", "-t" }, + @Option(names = { "--threshold", "-t" }, description = "Maximum number of bytes in a topic backlog before compaction is triggered " + "(eg: 10M, 16G, 3T). 0 disables automatic compaction", - required = true) - private String thresholdStr = "0"; - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + required = true, + converter = ByteUnitToLongConverter.class) + private Long threshold = 0L; + + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); - long threshold = validateSizeString(thresholdStr); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).setCompactionThreshold(persistentTopic, threshold); } } - @Parameters(commandDescription = "Remove compaction threshold for a topic") + @Command(description = "Remove compaction threshold for a topic") private class RemoveCompactionThreshold extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; - @Parameter(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; + @Option(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + "If set to true, the removing operation will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).removeCompactionThreshold(persistentTopic); } } - @Parameters(commandDescription = "Get message dispatch rate for a topic") + @Command(description = "Get message dispatch rate for a topic") private class GetDispatchRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; - @Parameter(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getDispatchRate(persistentTopic, applied)); } } - @Parameters(commandDescription = "Set message dispatch rate for a topic") + @Command(description = "Set message dispatch rate for a topic") private class SetDispatchRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--msg-dispatch-rate", + @Option(names = { "--msg-dispatch-rate", "-md" }, description = "message-dispatch-rate " + "(default -1 will be overwrite if not passed)", required = false) private int msgDispatchRate = -1; - @Parameter(names = { "--byte-dispatch-rate", + @Option(names = { "--byte-dispatch-rate", "-bd" }, description = "byte-dispatch-rate " + "(default -1 will be overwrite if not passed)", required = false) private long byteDispatchRate = -1; - @Parameter(names = { "--dispatch-rate-period", + @Option(names = { "--dispatch-rate-period", "-dt" }, description = "dispatch-rate-period in second type " + "(default 1 second will be overwrite if not passed)", required = false) private int dispatchRatePeriodSec = 1; - @Parameter(names = { "--relative-to-publish-rate", + @Option(names = { "--relative-to-publish-rate", "-rp" }, description = "dispatch rate relative to publish-rate (if publish-relative flag is enabled " + "then broker will apply throttling value to (publish-rate + dispatch rate))", required = false) private boolean relativeToPublishRate = false; - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).setDispatchRate(persistentTopic, DispatchRate.builder() .dispatchThrottlingRateInMsg(msgDispatchRate) @@ -1360,76 +1345,69 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Remove message dispatch rate for a topic") + @Command(description = "Remove message dispatch rate for a topic") private class RemoveDispatchRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + "If set to true, the removing operation will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).removeDispatchRate(persistentTopic); } } - @Parameters(commandDescription = "Get the inactive topic policies on a topic") + @Command(description = "Get the inactive topic policies on a topic") private class GetInactiveTopicPolicies extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; - @Parameter(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getInactiveTopicPolicies(persistentTopic, applied)); } } - @Parameters(commandDescription = "Set the inactive topic policies on a topic") + @Command(description = "Set the inactive topic policies on a topic") private class SetInactiveTopicPolicies extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--enable-delete-while-inactive", "-e" }, description = "Enable delete while inactive") + @Option(names = { "--enable-delete-while-inactive", "-e" }, description = "Enable delete while inactive") private boolean enableDeleteWhileInactive = false; - @Parameter(names = { "--disable-delete-while-inactive", "-d" }, description = "Disable delete while inactive") + @Option(names = { "--disable-delete-while-inactive", "-d" }, description = "Disable delete while inactive") private boolean disableDeleteWhileInactive = false; - @Parameter(names = {"--max-inactive-duration", "-t"}, + @Option(names = {"--max-inactive-duration", "-t"}, description = "Max duration of topic inactivity in seconds, topics that are inactive for longer than " - + "this value will be deleted (eg: 1s, 10s, 1m, 5h, 3d)", required = true) - private String deleteInactiveTopicsMaxInactiveDuration; + + "this value will be deleted (eg: 1s, 10s, 1m, 5h, 3d)", required = true, + converter = TimeUnitToSecondsConverter.class) + private Long maxInactiveDurationInSeconds; - @Parameter(names = { "--delete-mode", "-m" }, description = "Mode of delete inactive topic, Valid options are: " + @Option(names = { "--delete-mode", "-m" }, description = "Mode of delete inactive topic, Valid options are: " + "[delete_when_no_subscriptions, delete_when_subscriptions_caught_up]", required = true) private String inactiveTopicDeleteMode; - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); - long maxInactiveDurationInSeconds; - try { - maxInactiveDurationInSeconds = TimeUnit.SECONDS.toSeconds( - RelativeTimeUtil.parseRelativeTimeInSeconds(deleteInactiveTopicsMaxInactiveDuration)); - } catch (IllegalArgumentException exception) { - throw new ParameterException(exception.getMessage()); - } - + String persistentTopic = validatePersistentTopic(topicName); if (enableDeleteWhileInactive == disableDeleteWhileInactive) { throw new ParameterException("Need to specify either enable-delete-while-inactive or " + "disable-delete-while-inactive"); @@ -1442,73 +1420,73 @@ void run() throws PulsarAdminException { + "delete_when_subscriptions_caught_up"); } getTopicPolicies(isGlobal).setInactiveTopicPolicies(persistentTopic, new InactiveTopicPolicies(deleteMode, - (int) maxInactiveDurationInSeconds, enableDeleteWhileInactive)); + maxInactiveDurationInSeconds.intValue(), enableDeleteWhileInactive)); } } - @Parameters(commandDescription = "Remove inactive topic policies from a topic") + @Command(description = "Remove inactive topic policies from a topic") private class RemoveInactiveTopicPolicies extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; - @Parameter(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; + @Option(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + "If set to true, the removing operation will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).removeInactiveTopicPolicies(persistentTopic); } } - @Parameters(commandDescription = "Get replicator message-dispatch-rate for a topic") + @Command(description = "Get replicator message-dispatch-rate for a topic") private class GetReplicatorDispatchRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; - @Parameter(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getReplicatorDispatchRate(persistentTopic, applied)); } } - @Parameters(commandDescription = "Set replicator message-dispatch-rate for a topic") + @Command(description = "Set replicator message-dispatch-rate for a topic") private class SetReplicatorDispatchRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--msg-dispatch-rate", + @Option(names = { "--msg-dispatch-rate", "-md" }, description = "message-dispatch-rate (default -1 will be overwrite if not passed)") private int msgDispatchRate = -1; - @Parameter(names = { "--byte-dispatch-rate", + @Option(names = { "--byte-dispatch-rate", "-bd" }, description = "byte-dispatch-rate (default -1 will be overwrite if not passed)") private long byteDispatchRate = -1; - @Parameter(names = {"--dispatch-rate-period", + @Option(names = {"--dispatch-rate-period", "-dt"}, description = "dispatch-rate-period in second type (default 1 second will be overwrite if not" + " passed)") private int dispatchRatePeriodSec = 1; - @Parameter(names = {"--relative-to-publish-rate", + @Option(names = {"--relative-to-publish-rate", "-rp"}, description = "dispatch rate relative to publish-rate (if publish-relative flag is enabled " + "then broker will apply throttling value to (publish-rate + dispatch rate))") private boolean relativeToPublishRate = false; - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).setReplicatorDispatchRate(persistentTopic, DispatchRate.builder() .dispatchThrottlingRateInMsg(msgDispatchRate) @@ -1519,42 +1497,42 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Remove replicator message-dispatch-rate for a topic") + @Command(description = "Remove replicator message-dispatch-rate for a topic") private class RemoveReplicatorDispatchRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).removeReplicatorDispatchRate(persistentTopic); } } - @Parameters(commandDescription = "Get subscription message-dispatch-rate for a topic") + @Command(description = "Get subscription message-dispatch-rate for a topic") private class GetSubscriptionDispatchRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; - @Parameter(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; - @Parameter(names = {"--subscription", "-s"}, + @Option(names = {"--subscription", "-s"}, description = "Get message-dispatch-rate of a specific subscription") private String subName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); if (StringUtils.isBlank(subName)) { print(getTopicPolicies(isGlobal).getSubscriptionDispatchRate(persistentTopic, applied)); } else { @@ -1563,40 +1541,40 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Set subscription message-dispatch-rate for a topic") + @Command(description = "Set subscription message-dispatch-rate for a topic") private class SetSubscriptionDispatchRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--msg-dispatch-rate", + @Option(names = { "--msg-dispatch-rate", "-md" }, description = "message-dispatch-rate (default -1 will be overwrite if not passed)") private int msgDispatchRate = -1; - @Parameter(names = { "--byte-dispatch-rate", + @Option(names = { "--byte-dispatch-rate", "-bd" }, description = "byte-dispatch-rate (default -1 will be overwrite if not passed)") private long byteDispatchRate = -1; - @Parameter(names = { "--dispatch-rate-period", + @Option(names = { "--dispatch-rate-period", "-dt" }, description = "dispatch-rate-period in second type " + "(default 1 second will be overwrite if not passed)") private int dispatchRatePeriodSec = 1; - @Parameter(names = { "--relative-to-publish-rate", + @Option(names = { "--relative-to-publish-rate", "-rp" }, description = "dispatch rate relative to publish-rate (if publish-relative flag is enabled " + "then broker will apply throttling value to (publish-rate + dispatch rate))") private boolean relativeToPublishRate = false; - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal = false; - @Parameter(names = {"--subscription", "-s"}, + @Option(names = {"--subscription", "-s"}, description = "Set message-dispatch-rate for a specific subscription") private String subName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); DispatchRate rate = DispatchRate.builder() .dispatchThrottlingRateInMsg(msgDispatchRate) .dispatchThrottlingRateInByte(byteDispatchRate) @@ -1611,22 +1589,22 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Remove subscription message-dispatch-rate for a topic") + @Command(description = "Remove subscription message-dispatch-rate for a topic") private class RemoveSubscriptionDispatchRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal = false; - @Parameter(names = {"--subscription", "-s"}, + @Option(names = {"--subscription", "-s"}, description = "Remove message-dispatch-rate for a specific subscription") private String subName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); if (StringUtils.isBlank(subName)) { getTopicPolicies(isGlobal).removeSubscriptionDispatchRate(persistentTopic); } else { @@ -1636,154 +1614,154 @@ void run() throws PulsarAdminException { } - @Parameters(commandDescription = "Get max subscriptions for a topic") + @Command(description = "Get max subscriptions for a topic") private class GetMaxSubscriptionsPerTopic extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--global", "-g"}, description = "Whether to get this policy globally. " + @Option(names = {"--global", "-g"}, description = "Whether to get this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getMaxSubscriptionsPerTopic(persistentTopic)); } } - @Parameters(commandDescription = "Set max subscriptions for a topic") + @Command(description = "Set max subscriptions for a topic") private class SetMaxSubscriptionsPerTopic extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--max-subscriptions-per-topic", + @Option(names = {"--max-subscriptions-per-topic", "-s"}, description = "max subscriptions for a topic (default -1 will be overwrite if not passed)", required = true) private int maxSubscriptionPerTopic; - @Parameter(names = {"--global", "-g"}, description = "Whether to set this policy globally. " + @Option(names = {"--global", "-g"}, description = "Whether to set this policy globally. " + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).setMaxSubscriptionsPerTopic(persistentTopic, maxSubscriptionPerTopic); } } - @Parameters(commandDescription = "Remove max subscriptions for a topic") + @Command(description = "Remove max subscriptions for a topic") private class RemoveMaxSubscriptionsPerTopic extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--global", "-g"}, description = "Whether to remove this policy globally. " + @Option(names = {"--global", "-g"}, description = "Whether to remove this policy globally. " + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).removeMaxSubscriptionsPerTopic(persistentTopic); } } - @Parameters(commandDescription = "Get the offload policies for a topic") + @Command(description = "Get the offload policies for a topic") private class GetOffloadPolicies extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; - @Parameter(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to get this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getOffloadPolicies(persistentTopic, applied)); } } - @Parameters(commandDescription = "Remove the offload policies for a topic") + @Command(description = "Remove the offload policies for a topic") private class RemoveOffloadPolicies extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to remove this policy globally. " + "If set to true, the removing operation will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).removeOffloadPolicies(persistentTopic); } } - @Parameters(commandDescription = "Set the offload policies for a topic") + @Command(description = "Set the offload policies for a topic") private class SetOffloadPolicies extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"-d", "--driver"}, description = "ManagedLedger offload driver", required = true) + @Option(names = {"-d", "--driver"}, description = "ManagedLedger offload driver", required = true) private String driver; - @Parameter(names = {"-r", "--region"} + @Option(names = {"-r", "--region"} , description = "ManagedLedger offload region, s3 and google-cloud-storage requires this parameter") private String region; - @Parameter(names = {"-b", "--bucket"} + @Option(names = {"-b", "--bucket"} , description = "ManagedLedger offload bucket, s3 and google-cloud-storage requires this parameter") private String bucket; - @Parameter(names = {"-e", "--endpoint"} + @Option(names = {"-e", "--endpoint"} , description = "ManagedLedger offload service endpoint, only s3 requires this parameter") private String endpoint; - @Parameter(names = {"-i", "--aws-id"} + @Option(names = {"-i", "--aws-id"} , description = "AWS Credential Id to use when using driver S3 or aws-s3") private String awsId; - @Parameter(names = {"-s", "--aws-secret"} + @Option(names = {"-s", "--aws-secret"} , description = "AWS Credential Secret to use when using driver S3 or aws-s3") private String awsSecret; - @Parameter(names = {"--ro", "--s3-role"} + @Option(names = {"--ro", "--s3-role"} , description = "S3 Role used for STSAssumeRoleSessionCredentialsProvider") private String s3Role; - @Parameter(names = {"--s3-role-session-name", "-rsn"} + @Option(names = {"--s3-role-session-name", "-rsn"} , description = "S3 role session name used for STSAssumeRoleSessionCredentialsProvider") private String s3RoleSessionName; - @Parameter(names = {"-m", "--maxBlockSizeInBytes"}, + @Option(names = {"-m", "--maxBlockSizeInBytes"}, description = "ManagedLedger offload max block Size in bytes," + "s3 and google-cloud-storage requires this parameter") - private int maxBlockSizeInBytes; + private int maxBlockSizeInBytes = OffloadPoliciesImpl.DEFAULT_MAX_BLOCK_SIZE_IN_BYTES; - @Parameter(names = {"-rb", "--readBufferSizeInBytes"}, + @Option(names = {"-rb", "--readBufferSizeInBytes"}, description = "ManagedLedger offload read buffer size in bytes," + "s3 and google-cloud-storage requires this parameter") - private int readBufferSizeInBytes; + private int readBufferSizeInBytes = OffloadPoliciesImpl.DEFAULT_READ_BUFFER_SIZE_IN_BYTES; - @Parameter(names = {"-t", "--offloadThresholdInBytes"} - , description = "ManagedLedger offload threshold in bytes", required = true) - private long offloadThresholdInBytes; + @Option(names = {"-t", "--offloadThresholdInBytes"} + , description = "ManagedLedger offload threshold in bytes") + private Long offloadThresholdInBytes = OffloadPoliciesImpl.DEFAULT_OFFLOAD_THRESHOLD_IN_BYTES; - @Parameter(names = {"-ts", "--offloadThresholdInSeconds"} + @Option(names = {"-ts", "--offloadThresholdInSeconds"} , description = "ManagedLedger offload threshold in seconds") - private Long offloadThresholdInSeconds; + private Long offloadThresholdInSeconds = OffloadPoliciesImpl.DEFAULT_OFFLOAD_THRESHOLD_IN_SECONDS; - @Parameter(names = {"-dl", "--offloadDeletionLagInMillis"} + @Option(names = {"-dl", "--offloadDeletionLagInMillis"} , description = "ManagedLedger offload deletion lag in bytes") - private Long offloadDeletionLagInMillis; + private Long offloadDeletionLagInMillis = OffloadPoliciesImpl.DEFAULT_OFFLOAD_DELETION_LAG_IN_MILLIS; - @Parameter( + @Option( names = {"--offloadedReadPriority", "-orp"}, description = "Read priority for offloaded messages. By default, once messages are offloaded to" + " long-term storage, brokers read messages from long-term storage, but messages can still" @@ -1794,13 +1772,41 @@ private class SetOffloadPolicies extends CliCommand { ) private String offloadReadPriorityStr; - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal = false; + public final List driverNames = OffloadPoliciesImpl.DRIVER_NAMES; + + public boolean driverSupported(String driver) { + return driverNames.stream().anyMatch(d -> d.equalsIgnoreCase(driver)); + } + + public boolean isS3Driver(String driver) { + if (StringUtils.isEmpty(driver)) { + return false; + } + return driver.equalsIgnoreCase(driverNames.get(0)) || driver.equalsIgnoreCase(driverNames.get(1)); + } + @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); + + if (!driverSupported(driver)) { + throw new ParameterException("The driver " + driver + " is not supported, " + + "(Possible values: " + String.join(",", driverNames) + ")."); + } + + if (isS3Driver(driver) && Strings.isNullOrEmpty(region) && Strings.isNullOrEmpty(endpoint)) { + throw new ParameterException( + "Either s3ManagedLedgerOffloadRegion or s3ManagedLedgerOffloadServiceEndpoint must be set" + + " if s3 offload enabled"); + } + positiveCheck("maxBlockSizeInBytes", maxBlockSizeInBytes); + maxValueCheck("maxBlockSizeInBytes", maxBlockSizeInBytes, Integer.MAX_VALUE); + positiveCheck("readBufferSizeInBytes", readBufferSizeInBytes); + maxValueCheck("readBufferSizeInBytes", readBufferSizeInBytes, Integer.MAX_VALUE); OffloadedReadPriority offloadedReadPriority = OffloadPoliciesImpl.DEFAULT_OFFLOADED_READ_PRIORITY; @@ -1828,67 +1834,67 @@ void run() throws PulsarAdminException { } - @Parameters(commandDescription = "Remove schema compatibility strategy on a topic") + @Command(description = "Remove schema compatibility strategy on a topic") private class RemoveSchemaCompatibilityStrategy extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getAdmin().topicPolicies().removeSchemaCompatibilityStrategy(persistentTopic); } } - @Parameters(commandDescription = "Set schema compatibility strategy on a topic") + @Command(description = "Set schema compatibility strategy on a topic") private class SetSchemaCompatibilityStrategy extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--strategy", "-s"}, description = "Schema compatibility strategy: [UNDEFINED, " + @Option(names = {"--strategy", "-s"}, description = "Schema compatibility strategy: [UNDEFINED, " + "ALWAYS_INCOMPATIBLE, ALWAYS_COMPATIBLE, BACKWARD, FORWARD, FULL, BACKWARD_TRANSITIVE, " + "FORWARD_TRANSITIVE, FULL_TRANSITIVE]", required = true) private SchemaCompatibilityStrategy strategy; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getAdmin().topicPolicies().setSchemaCompatibilityStrategy(persistentTopic, strategy); } } - @Parameters(commandDescription = "Get schema compatibility strategy on a topic") + @Command(description = "Get schema compatibility strategy on a topic") private class GetSchemaCompatibilityStrategy extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"-ap", "--applied"}, description = "Get the applied policy of the topic") + @Option(names = {"-ap", "--applied"}, description = "Get the applied policy of the topic") private boolean applied = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); SchemaCompatibilityStrategy strategy = getAdmin().topicPolicies().getSchemaCompatibilityStrategy(persistentTopic, applied); print(strategy == null ? "null" : strategy.name()); } } - @Parameters(commandDescription = "Enable autoSubscriptionCreation for a topic") + @Command(description = "Enable autoSubscriptionCreation for a topic") private class SetAutoSubscriptionCreation extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--enable", "-e"}, description = "Enable allowAutoSubscriptionCreation on topic") + @Option(names = {"--enable", "-e"}, description = "Enable allowAutoSubscriptionCreation on topic") private boolean enable = false; - @Parameter(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).setAutoSubscriptionCreation(persistentTopic, AutoSubscriptionCreationOverride.builder() .allowAutoSubscriptionCreation(enable) @@ -1896,41 +1902,92 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get the autoSubscriptionCreation for a topic") + @Command(description = "Get the autoSubscriptionCreation for a topic") private class GetAutoSubscriptionCreation extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--applied", "-a"}, description = "Get the applied policy of the topic") + @Option(names = {"--applied", "-a"}, description = "Get the applied policy of the topic") private boolean applied = false; - @Parameter(names = {"--global", "-g"}, description = "Whether to get this policy globally. " + @Option(names = {"--global", "-g"}, description = "Whether to get this policy globally. " + "If set to true, broker returned global topic policies") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopicPolicies(isGlobal).getAutoSubscriptionCreation(persistentTopic, applied)); } } - @Parameters(commandDescription = "Remove override of autoSubscriptionCreation for a topic") + @Command(description = "Remove override of autoSubscriptionCreation for a topic") private class RemoveAutoSubscriptionCreation extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--global", "-g"}, description = "Whether to remove this policy globally. " + @Option(names = {"--global", "-g"}, description = "Whether to remove this policy globally. " + "If set to true, the policy will be replicate to other clusters asynchronously") private boolean isGlobal = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopicPolicies(isGlobal).removeAutoSubscriptionCreation(persistentTopic); } } + @Command(description = "Enable dispatcherPauseOnAckStatePersistent for a topic") + private class SetDispatcherPauseOnAckStatePersistent extends CliCommand { + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; + + @Option(names = { "--global", "-g" }, description = "Whether to set this policy globally. " + + "If set to true, the policy will be replicate to other clusters asynchronously") + private boolean isGlobal = false; + + @Override + void run() throws PulsarAdminException { + String persistentTopic = validatePersistentTopic(topicName); + getTopicPolicies(isGlobal).setDispatcherPauseOnAckStatePersistent(persistentTopic); + } + } + + @Command(description = "Get the dispatcherPauseOnAckStatePersistent for a topic") + private class GetDispatcherPauseOnAckStatePersistent extends CliCommand { + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; + + @Option(names = {"--applied", "-a"}, description = "Get the applied policy of the topic") + private boolean applied = false; + + @Option(names = {"--global", "-g"}, description = "Whether to get this policy globally. " + + "If set to true, broker returned global topic policies") + private boolean isGlobal = false; + + @Override + void run() throws PulsarAdminException { + String persistentTopic = validatePersistentTopic(topicName); + print(getTopicPolicies(isGlobal).getDispatcherPauseOnAckStatePersistent(persistentTopic, applied)); + } + } + + @Command(description = "Remove dispatcherPauseOnAckStatePersistent for a topic") + private class RemoveDispatcherPauseOnAckStatePersistent extends CliCommand { + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; + + @Option(names = {"--global", "-g"}, description = "Whether to remove this policy globally. " + + "If set to true, the policy will be replicate to other clusters asynchronously") + private boolean isGlobal = false; + + @Override + void run() throws PulsarAdminException { + String persistentTopic = validatePersistentTopic(topicName); + getTopicPolicies(isGlobal).removeDispatcherPauseOnAckStatePersistent(persistentTopic); + } + } + private TopicPolicies getTopicPolicies(boolean isGlobal) { return getAdmin().topicPolicies(isGlobal); } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java index f96410749aea6..261bd81a5b7bd 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java @@ -19,10 +19,7 @@ package org.apache.pulsar.admin.cli; import static org.apache.commons.lang3.StringUtils.isNotBlank; -import com.beust.jcommander.IUsageFormatter; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -31,6 +28,9 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufUtil; import io.netty.buffer.Unpooled; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -39,6 +39,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -47,6 +48,11 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import lombok.Getter; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.cli.converters.picocli.ByteUnitToIntegerConverter; +import org.apache.pulsar.cli.converters.picocli.ByteUnitToLongConverter; +import org.apache.pulsar.cli.converters.picocli.TimeUnitToMillisConverter; +import org.apache.pulsar.cli.converters.picocli.TimeUnitToSecondsConverter; import org.apache.pulsar.client.admin.ListTopicsOptions; import org.apache.pulsar.client.admin.LongRunningProcessStatus; import org.apache.pulsar.client.admin.OffloadProcessStatus; @@ -56,10 +62,12 @@ import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.SubscriptionType; -import org.apache.pulsar.client.cli.NoSplitter; +import org.apache.pulsar.client.api.TransactionIsolationLevel; import org.apache.pulsar.client.impl.BatchMessageIdImpl; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.impl.MessageImpl; +import org.apache.pulsar.common.api.proto.MarkerType; +import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.BacklogQuota; @@ -67,6 +75,7 @@ import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode; import org.apache.pulsar.common.policies.data.InactiveTopicPolicies; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; import org.apache.pulsar.common.policies.data.OffloadedReadPriority; import org.apache.pulsar.common.policies.data.PersistencePolicies; @@ -76,330 +85,218 @@ import org.apache.pulsar.common.policies.data.SubscribeRate; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.ObjectMapperFactory; -import org.apache.pulsar.common.util.RelativeTimeUtil; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; @Getter -@Parameters(commandDescription = "Operations on persistent topics") +@Command(description = "Operations on persistent topics") public class CmdTopics extends CmdBase { private final CmdTopics.PartitionedLookup partitionedLookup; + private final CmdTopics.DeleteCmd deleteCmd; public CmdTopics(Supplier admin) { super("topics", admin); partitionedLookup = new PartitionedLookup(); - jcommander.addCommand("list", new ListCmd()); - jcommander.addCommand("list-partitioned-topics", new PartitionedTopicListCmd()); - jcommander.addCommand("permissions", new Permissions()); - jcommander.addCommand("grant-permission", new GrantPermissions()); - jcommander.addCommand("revoke-permission", new RevokePermissions()); - jcommander.addCommand("lookup", new Lookup()); - jcommander.addCommand("partitioned-lookup", partitionedLookup); - jcommander.addCommand("bundle-range", new GetBundleRange()); - jcommander.addCommand("delete", new DeleteCmd()); - jcommander.addCommand("truncate", new TruncateCmd()); - jcommander.addCommand("unload", new UnloadCmd()); - jcommander.addCommand("subscriptions", new ListSubscriptions()); - jcommander.addCommand("unsubscribe", new DeleteSubscription()); - jcommander.addCommand("create-subscription", new CreateSubscription()); - jcommander.addCommand("update-subscription-properties", new UpdateSubscriptionProperties()); - jcommander.addCommand("get-subscription-properties", new GetSubscriptionProperties()); - - jcommander.addCommand("stats", new GetStats()); - jcommander.addCommand("stats-internal", new GetInternalStats()); - jcommander.addCommand("info-internal", new GetInternalInfo()); - - jcommander.addCommand("partitioned-stats", new GetPartitionedStats()); - jcommander.addCommand("partitioned-stats-internal", new GetPartitionedStatsInternal()); - - jcommander.addCommand("skip", new Skip()); - jcommander.addCommand("clear-backlog", new ClearBacklog()); - - jcommander.addCommand("expire-messages", new ExpireMessages()); - jcommander.addCommand("expire-messages-all-subscriptions", new ExpireMessagesForAllSubscriptions()); - - jcommander.addCommand("create-partitioned-topic", new CreatePartitionedCmd()); - jcommander.addCommand("create-missed-partitions", new CreateMissedPartitionsCmd()); - jcommander.addCommand("create", new CreateNonPartitionedCmd()); - jcommander.addCommand("update-partitioned-topic", new UpdatePartitionedCmd()); - jcommander.addCommand("get-partitioned-topic-metadata", new GetPartitionedTopicMetadataCmd()); - jcommander.addCommand("get-properties", new GetPropertiesCmd()); - jcommander.addCommand("update-properties", new UpdateProperties()); - jcommander.addCommand("remove-properties", new RemoveProperties()); - - jcommander.addCommand("delete-partitioned-topic", new DeletePartitionedCmd()); - jcommander.addCommand("peek-messages", new PeekMessages()); - jcommander.addCommand("examine-messages", new ExamineMessages()); - jcommander.addCommand("get-message-by-id", new GetMessageById()); - jcommander.addCommand("get-message-id", new GetMessageId()); - jcommander.addCommand("reset-cursor", new ResetCursor()); - jcommander.addCommand("terminate", new Terminate()); - jcommander.addCommand("partitioned-terminate", new PartitionedTerminate()); - jcommander.addCommand("compact", new Compact()); - jcommander.addCommand("compaction-status", new CompactionStatusCmd()); - jcommander.addCommand("offload", new Offload()); - jcommander.addCommand("offload-status", new OffloadStatusCmd()); - jcommander.addCommand("last-message-id", new GetLastMessageId()); - jcommander.addCommand("get-backlog-quotas", new GetBacklogQuotaMap()); - jcommander.addCommand("set-backlog-quota", new SetBacklogQuota()); - jcommander.addCommand("remove-backlog-quota", new RemoveBacklogQuota()); - jcommander.addCommand("get-message-ttl", new GetMessageTTL()); - jcommander.addCommand("set-message-ttl", new SetMessageTTL()); - jcommander.addCommand("remove-message-ttl", new RemoveMessageTTL()); - jcommander.addCommand("get-retention", new GetRetention()); - jcommander.addCommand("set-retention", new SetRetention()); - jcommander.addCommand("remove-retention", new RemoveRetention()); + deleteCmd = new DeleteCmd(); + addCommand("list", new ListCmd()); + addCommand("list-partitioned-topics", new PartitionedTopicListCmd()); + addCommand("permissions", new Permissions()); + addCommand("grant-permission", new GrantPermissions()); + addCommand("revoke-permission", new RevokePermissions()); + addCommand("lookup", new Lookup()); + addCommand("partitioned-lookup", partitionedLookup); + addCommand("bundle-range", new GetBundleRange()); + addCommand("delete", deleteCmd); + addCommand("truncate", new TruncateCmd()); + addCommand("unload", new UnloadCmd()); + addCommand("subscriptions", new ListSubscriptions()); + addCommand("unsubscribe", new DeleteSubscription()); + addCommand("create-subscription", new CreateSubscription()); + addCommand("update-subscription-properties", new UpdateSubscriptionProperties()); + addCommand("get-subscription-properties", new GetSubscriptionProperties()); + + addCommand("stats", new GetStats()); + addCommand("stats-internal", new GetInternalStats()); + addCommand("info-internal", new GetInternalInfo()); + + addCommand("partitioned-stats", new GetPartitionedStats()); + addCommand("partitioned-stats-internal", new GetPartitionedStatsInternal()); + + addCommand("skip", new Skip()); + addCommand("clear-backlog", new ClearBacklog()); + + addCommand("expire-messages", new ExpireMessages()); + addCommand("expire-messages-all-subscriptions", new ExpireMessagesForAllSubscriptions()); + + addCommand("create-partitioned-topic", new CreatePartitionedCmd()); + addCommand("create-missed-partitions", new CreateMissedPartitionsCmd()); + addCommand("create", new CreateNonPartitionedCmd()); + addCommand("update-partitioned-topic", new UpdatePartitionedCmd()); + addCommand("get-partitioned-topic-metadata", new GetPartitionedTopicMetadataCmd()); + addCommand("get-properties", new GetPropertiesCmd()); + addCommand("update-properties", new UpdateProperties()); + addCommand("remove-properties", new RemoveProperties()); + + addCommand("delete-partitioned-topic", new DeletePartitionedCmd()); + addCommand("peek-messages", new PeekMessages()); + addCommand("examine-messages", new ExamineMessages()); + addCommand("get-message-by-id", new GetMessageById()); + addCommand("get-message-id", new GetMessageId()); + addCommand("reset-cursor", new ResetCursor()); + addCommand("terminate", new Terminate()); + addCommand("partitioned-terminate", new PartitionedTerminate()); + addCommand("compact", new Compact()); + addCommand("compaction-status", new CompactionStatusCmd()); + addCommand("offload", new Offload()); + addCommand("offload-status", new OffloadStatusCmd()); + addCommand("last-message-id", new GetLastMessageId()); + addCommand("get-backlog-quotas", new GetBacklogQuotaMap()); + addCommand("set-backlog-quota", new SetBacklogQuota()); + addCommand("remove-backlog-quota", new RemoveBacklogQuota()); + addCommand("get-message-ttl", new GetMessageTTL()); + addCommand("set-message-ttl", new SetMessageTTL()); + addCommand("remove-message-ttl", new RemoveMessageTTL()); + addCommand("get-retention", new GetRetention()); + addCommand("set-retention", new SetRetention()); + addCommand("remove-retention", new RemoveRetention()); //deprecated commands - jcommander.addCommand("enable-deduplication", new EnableDeduplication()); - jcommander.addCommand("disable-deduplication", new DisableDeduplication()); - jcommander.addCommand("get-deduplication-enabled", new GetDeduplicationStatus()); - - jcommander.addCommand("set-deduplication", new SetDeduplicationStatus()); - jcommander.addCommand("get-deduplication", new GetDeduplicationStatus()); - jcommander.addCommand("remove-deduplication", new RemoveDeduplicationStatus()); - - jcommander.addCommand("get-deduplication-snapshot-interval", new GetDeduplicationSnapshotInterval()); - jcommander.addCommand("set-deduplication-snapshot-interval", new SetDeduplicationSnapshotInterval()); - jcommander.addCommand("remove-deduplication-snapshot-interval", new RemoveDeduplicationSnapshotInterval()); - - jcommander.addCommand("get-delayed-delivery", new GetDelayedDelivery()); - jcommander.addCommand("set-delayed-delivery", new SetDelayedDelivery()); - jcommander.addCommand("remove-delayed-delivery", new RemoveDelayedDelivery()); - jcommander.addCommand("get-persistence", new GetPersistence()); - jcommander.addCommand("set-persistence", new SetPersistence()); - jcommander.addCommand("remove-persistence", new RemovePersistence()); - jcommander.addCommand("get-offload-policies", new GetOffloadPolicies()); - jcommander.addCommand("set-offload-policies", new SetOffloadPolicies()); - jcommander.addCommand("remove-offload-policies", new RemoveOffloadPolicies()); - - jcommander.addCommand("get-dispatch-rate", new GetDispatchRate()); - jcommander.addCommand("set-dispatch-rate", new SetDispatchRate()); - jcommander.addCommand("remove-dispatch-rate", new RemoveDispatchRate()); - - jcommander.addCommand("get-subscription-dispatch-rate", new GetSubscriptionDispatchRate()); - jcommander.addCommand("set-subscription-dispatch-rate", new SetSubscriptionDispatchRate()); - jcommander.addCommand("remove-subscription-dispatch-rate", new RemoveSubscriptionDispatchRate()); - - jcommander.addCommand("get-replicator-dispatch-rate", new GetReplicatorDispatchRate()); - jcommander.addCommand("set-replicator-dispatch-rate", new SetReplicatorDispatchRate()); - jcommander.addCommand("remove-replicator-dispatch-rate", new RemoveReplicatorDispatchRate()); - - jcommander.addCommand("get-compaction-threshold", new GetCompactionThreshold()); - jcommander.addCommand("set-compaction-threshold", new SetCompactionThreshold()); - jcommander.addCommand("remove-compaction-threshold", new RemoveCompactionThreshold()); + addCommand("enable-deduplication", new EnableDeduplication()); + addCommand("disable-deduplication", new DisableDeduplication()); + addCommand("get-deduplication-enabled", new GetDeduplicationStatus()); + + addCommand("set-deduplication", new SetDeduplicationStatus()); + addCommand("get-deduplication", new GetDeduplicationStatus()); + addCommand("remove-deduplication", new RemoveDeduplicationStatus()); + + addCommand("get-deduplication-snapshot-interval", new GetDeduplicationSnapshotInterval()); + addCommand("set-deduplication-snapshot-interval", new SetDeduplicationSnapshotInterval()); + addCommand("remove-deduplication-snapshot-interval", new RemoveDeduplicationSnapshotInterval()); + + addCommand("get-delayed-delivery", new GetDelayedDelivery()); + addCommand("set-delayed-delivery", new SetDelayedDelivery()); + addCommand("remove-delayed-delivery", new RemoveDelayedDelivery()); + addCommand("get-persistence", new GetPersistence()); + addCommand("set-persistence", new SetPersistence()); + addCommand("remove-persistence", new RemovePersistence()); + addCommand("get-offload-policies", new GetOffloadPolicies()); + addCommand("set-offload-policies", new SetOffloadPolicies()); + addCommand("remove-offload-policies", new RemoveOffloadPolicies()); + + addCommand("get-dispatch-rate", new GetDispatchRate()); + addCommand("set-dispatch-rate", new SetDispatchRate()); + addCommand("remove-dispatch-rate", new RemoveDispatchRate()); + + addCommand("get-subscription-dispatch-rate", new GetSubscriptionDispatchRate()); + addCommand("set-subscription-dispatch-rate", new SetSubscriptionDispatchRate()); + addCommand("remove-subscription-dispatch-rate", new RemoveSubscriptionDispatchRate()); + + addCommand("get-replicator-dispatch-rate", new GetReplicatorDispatchRate()); + addCommand("set-replicator-dispatch-rate", new SetReplicatorDispatchRate()); + addCommand("remove-replicator-dispatch-rate", new RemoveReplicatorDispatchRate()); + + addCommand("get-compaction-threshold", new GetCompactionThreshold()); + addCommand("set-compaction-threshold", new SetCompactionThreshold()); + addCommand("remove-compaction-threshold", new RemoveCompactionThreshold()); //deprecated commands - jcommander.addCommand("get-max-unacked-messages-on-consumer", new GetMaxUnackedMessagesOnConsumer()); - jcommander.addCommand("set-max-unacked-messages-on-consumer", new SetMaxUnackedMessagesOnConsumer()); - jcommander.addCommand("remove-max-unacked-messages-on-consumer", new RemoveMaxUnackedMessagesOnConsumer()); - jcommander.addCommand("get-max-unacked-messages-on-subscription", new GetMaxUnackedMessagesOnSubscription()); - jcommander.addCommand("set-max-unacked-messages-on-subscription", new SetMaxUnackedMessagesOnSubscription()); - jcommander.addCommand("remove-max-unacked-messages-on-subscription", + addCommand("get-max-unacked-messages-on-consumer", new GetMaxUnackedMessagesOnConsumer()); + addCommand("set-max-unacked-messages-on-consumer", new SetMaxUnackedMessagesOnConsumer()); + addCommand("remove-max-unacked-messages-on-consumer", new RemoveMaxUnackedMessagesOnConsumer()); + addCommand("get-max-unacked-messages-on-subscription", new GetMaxUnackedMessagesOnSubscription()); + addCommand("set-max-unacked-messages-on-subscription", new SetMaxUnackedMessagesOnSubscription()); + addCommand("remove-max-unacked-messages-on-subscription", new RemoveMaxUnackedMessagesOnSubscription()); - jcommander.addCommand("get-max-unacked-messages-per-consumer", new GetMaxUnackedMessagesOnConsumer()); - jcommander.addCommand("set-max-unacked-messages-per-consumer", new SetMaxUnackedMessagesOnConsumer()); - jcommander.addCommand("remove-max-unacked-messages-per-consumer", new RemoveMaxUnackedMessagesOnConsumer()); - jcommander.addCommand("get-max-unacked-messages-per-subscription", new GetMaxUnackedMessagesOnSubscription()); - jcommander.addCommand("set-max-unacked-messages-per-subscription", new SetMaxUnackedMessagesOnSubscription()); - jcommander.addCommand("remove-max-unacked-messages-per-subscription", + addCommand("get-max-unacked-messages-per-consumer", new GetMaxUnackedMessagesOnConsumer()); + addCommand("set-max-unacked-messages-per-consumer", new SetMaxUnackedMessagesOnConsumer()); + addCommand("remove-max-unacked-messages-per-consumer", new RemoveMaxUnackedMessagesOnConsumer()); + addCommand("get-max-unacked-messages-per-subscription", new GetMaxUnackedMessagesOnSubscription()); + addCommand("set-max-unacked-messages-per-subscription", new SetMaxUnackedMessagesOnSubscription()); + addCommand("remove-max-unacked-messages-per-subscription", new RemoveMaxUnackedMessagesOnSubscription()); - jcommander.addCommand("get-publish-rate", new GetPublishRate()); - jcommander.addCommand("set-publish-rate", new SetPublishRate()); - jcommander.addCommand("remove-publish-rate", new RemovePublishRate()); + addCommand("get-publish-rate", new GetPublishRate()); + addCommand("set-publish-rate", new SetPublishRate()); + addCommand("remove-publish-rate", new RemovePublishRate()); - jcommander.addCommand("set-subscription-types-enabled", new SetSubscriptionTypesEnabled()); - jcommander.addCommand("get-subscription-types-enabled", new GetSubscriptionTypesEnabled()); - jcommander.addCommand("remove-subscription-types-enabled", new RemoveSubscriptionTypesEnabled()); + addCommand("set-subscription-types-enabled", new SetSubscriptionTypesEnabled()); + addCommand("get-subscription-types-enabled", new GetSubscriptionTypesEnabled()); + addCommand("remove-subscription-types-enabled", new RemoveSubscriptionTypesEnabled()); //deprecated commands - jcommander.addCommand("get-maxProducers", new GetMaxProducers()); - jcommander.addCommand("set-maxProducers", new SetMaxProducers()); - jcommander.addCommand("remove-maxProducers", new RemoveMaxProducers()); + addCommand("get-maxProducers", new GetMaxProducers()); + addCommand("set-maxProducers", new SetMaxProducers()); + addCommand("remove-maxProducers", new RemoveMaxProducers()); - jcommander.addCommand("get-max-producers", new GetMaxProducers()); - jcommander.addCommand("set-max-producers", new SetMaxProducers()); - jcommander.addCommand("remove-max-producers", new RemoveMaxProducers()); + addCommand("get-max-producers", new GetMaxProducers()); + addCommand("set-max-producers", new SetMaxProducers()); + addCommand("remove-max-producers", new RemoveMaxProducers()); - jcommander.addCommand("get-max-subscriptions", new GetMaxSubscriptionsPerTopic()); - jcommander.addCommand("set-max-subscriptions", new SetMaxSubscriptionsPerTopic()); - jcommander.addCommand("remove-max-subscriptions", new RemoveMaxSubscriptionsPerTopic()); + addCommand("get-max-subscriptions", new GetMaxSubscriptionsPerTopic()); + addCommand("set-max-subscriptions", new SetMaxSubscriptionsPerTopic()); + addCommand("remove-max-subscriptions", new RemoveMaxSubscriptionsPerTopic()); - jcommander.addCommand("get-max-message-size", new GetMaxMessageSize()); - jcommander.addCommand("set-max-message-size", new SetMaxMessageSize()); - jcommander.addCommand("remove-max-message-size", new RemoveMaxMessageSize()); + addCommand("get-max-message-size", new GetMaxMessageSize()); + addCommand("set-max-message-size", new SetMaxMessageSize()); + addCommand("remove-max-message-size", new RemoveMaxMessageSize()); - jcommander.addCommand("get-max-consumers-per-subscription", new GetMaxConsumersPerSubscription()); - jcommander.addCommand("set-max-consumers-per-subscription", new SetMaxConsumersPerSubscription()); - jcommander.addCommand("remove-max-consumers-per-subscription", new RemoveMaxConsumersPerSubscription()); + addCommand("get-max-consumers-per-subscription", new GetMaxConsumersPerSubscription()); + addCommand("set-max-consumers-per-subscription", new SetMaxConsumersPerSubscription()); + addCommand("remove-max-consumers-per-subscription", new RemoveMaxConsumersPerSubscription()); - jcommander.addCommand("get-inactive-topic-policies", new GetInactiveTopicPolicies()); - jcommander.addCommand("set-inactive-topic-policies", new SetInactiveTopicPolicies()); - jcommander.addCommand("remove-inactive-topic-policies", new RemoveInactiveTopicPolicies()); + addCommand("get-inactive-topic-policies", new GetInactiveTopicPolicies()); + addCommand("set-inactive-topic-policies", new SetInactiveTopicPolicies()); + addCommand("remove-inactive-topic-policies", new RemoveInactiveTopicPolicies()); - jcommander.addCommand("get-max-consumers", new GetMaxConsumers()); - jcommander.addCommand("set-max-consumers", new SetMaxConsumers()); - jcommander.addCommand("remove-max-consumers", new RemoveMaxConsumers()); + addCommand("get-max-consumers", new GetMaxConsumers()); + addCommand("set-max-consumers", new SetMaxConsumers()); + addCommand("remove-max-consumers", new RemoveMaxConsumers()); - jcommander.addCommand("get-subscribe-rate", new GetSubscribeRate()); - jcommander.addCommand("set-subscribe-rate", new SetSubscribeRate()); - jcommander.addCommand("remove-subscribe-rate", new RemoveSubscribeRate()); + addCommand("get-subscribe-rate", new GetSubscribeRate()); + addCommand("set-subscribe-rate", new SetSubscribeRate()); + addCommand("remove-subscribe-rate", new RemoveSubscribeRate()); - jcommander.addCommand("set-replicated-subscription-status", new SetReplicatedSubscriptionStatus()); - jcommander.addCommand("get-replicated-subscription-status", new GetReplicatedSubscriptionStatus()); - jcommander.addCommand("get-backlog-size", new GetBacklogSizeByMessageId()); - jcommander.addCommand("analyze-backlog", new AnalyzeBacklog()); + addCommand("set-replicated-subscription-status", new SetReplicatedSubscriptionStatus()); + addCommand("get-replicated-subscription-status", new GetReplicatedSubscriptionStatus()); + addCommand("get-backlog-size", new GetBacklogSizeByMessageId()); + addCommand("analyze-backlog", new AnalyzeBacklog()); - jcommander.addCommand("get-replication-clusters", new GetReplicationClusters()); - jcommander.addCommand("set-replication-clusters", new SetReplicationClusters()); - jcommander.addCommand("remove-replication-clusters", new RemoveReplicationClusters()); + addCommand("get-replication-clusters", new GetReplicationClusters()); + addCommand("set-replication-clusters", new SetReplicationClusters()); + addCommand("remove-replication-clusters", new RemoveReplicationClusters()); - jcommander.addCommand("get-shadow-topics", new GetShadowTopics()); - jcommander.addCommand("set-shadow-topics", new SetShadowTopics()); - jcommander.addCommand("remove-shadow-topics", new RemoveShadowTopics()); - jcommander.addCommand("create-shadow-topic", new CreateShadowTopic()); - jcommander.addCommand("get-shadow-source", new GetShadowSource()); + addCommand("get-shadow-topics", new GetShadowTopics()); + addCommand("set-shadow-topics", new SetShadowTopics()); + addCommand("remove-shadow-topics", new RemoveShadowTopics()); + addCommand("create-shadow-topic", new CreateShadowTopic()); + addCommand("get-shadow-source", new GetShadowSource()); - jcommander.addCommand("get-schema-validation-enforce", new GetSchemaValidationEnforced()); - jcommander.addCommand("set-schema-validation-enforce", new SetSchemaValidationEnforced()); + addCommand("get-schema-validation-enforce", new GetSchemaValidationEnforced()); + addCommand("set-schema-validation-enforce", new SetSchemaValidationEnforced()); - jcommander.addCommand("trim-topic", new TrimTopic()); - - initDeprecatedCommands(); - } - - private void initDeprecatedCommands() { - IUsageFormatter usageFormatter = jcommander.getUsageFormatter(); - if (usageFormatter instanceof CmdUsageFormatter) { - CmdUsageFormatter cmdUsageFormatter = (CmdUsageFormatter) usageFormatter; - cmdUsageFormatter.addDeprecatedCommand("enable-deduplication"); - cmdUsageFormatter.addDeprecatedCommand("disable-deduplication"); - cmdUsageFormatter.addDeprecatedCommand("get-deduplication-enabled"); - - cmdUsageFormatter.addDeprecatedCommand("get-max-consumers"); - cmdUsageFormatter.addDeprecatedCommand("set-max-consumers"); - cmdUsageFormatter.addDeprecatedCommand("remove-max-consumers"); - - cmdUsageFormatter.addDeprecatedCommand("get-max-unacked-messages-per-consumer"); - cmdUsageFormatter.addDeprecatedCommand("set-max-unacked-messages-per-consumer"); - cmdUsageFormatter.addDeprecatedCommand("remove-max-unacked-messages-per-consumer"); - - cmdUsageFormatter.addDeprecatedCommand("get-message-ttl"); - cmdUsageFormatter.addDeprecatedCommand("set-message-ttl"); - cmdUsageFormatter.addDeprecatedCommand("remove-message-ttl"); - - cmdUsageFormatter.addDeprecatedCommand("get-max-consumers-per-subscription"); - cmdUsageFormatter.addDeprecatedCommand("set-max-consumers-per-subscription"); - cmdUsageFormatter.addDeprecatedCommand("remove-max-consumers-per-subscription"); - - cmdUsageFormatter.addDeprecatedCommand("get-max-unacked-messages-on-consumer"); - cmdUsageFormatter.addDeprecatedCommand("remove-max-unacked-messages-on-consumer"); - cmdUsageFormatter.addDeprecatedCommand("set-max-unacked-messages-on-consumer"); - - cmdUsageFormatter.addDeprecatedCommand("get-max-unacked-messages-on-subscription"); - cmdUsageFormatter.addDeprecatedCommand("remove-max-unacked-messages-on-subscription"); - cmdUsageFormatter.addDeprecatedCommand("set-max-unacked-messages-on-subscription"); - - cmdUsageFormatter.addDeprecatedCommand("get-publish-rate"); - cmdUsageFormatter.addDeprecatedCommand("set-publish-rate"); - cmdUsageFormatter.addDeprecatedCommand("remove-publish-rate"); - - cmdUsageFormatter.addDeprecatedCommand("get-subscribe-rate"); - cmdUsageFormatter.addDeprecatedCommand("set-subscribe-rate"); - cmdUsageFormatter.addDeprecatedCommand("remove-subscribe-rate"); - - cmdUsageFormatter.addDeprecatedCommand("get-maxProducers"); - cmdUsageFormatter.addDeprecatedCommand("set-maxProducers"); - cmdUsageFormatter.addDeprecatedCommand("remove-maxProducers"); - - cmdUsageFormatter.addDeprecatedCommand("get-max-message-size"); - cmdUsageFormatter.addDeprecatedCommand("set-max-message-size"); - cmdUsageFormatter.addDeprecatedCommand("remove-max-message-size"); - - cmdUsageFormatter.addDeprecatedCommand("get-retention"); - cmdUsageFormatter.addDeprecatedCommand("set-retention"); - cmdUsageFormatter.addDeprecatedCommand("remove-retention"); - - cmdUsageFormatter.addDeprecatedCommand("get-backlog-quotas"); - cmdUsageFormatter.addDeprecatedCommand("set-backlog-quota"); - cmdUsageFormatter.addDeprecatedCommand("remove-backlog-quota"); - - cmdUsageFormatter.addDeprecatedCommand("get-persistence"); - cmdUsageFormatter.addDeprecatedCommand("set-persistence"); - cmdUsageFormatter.addDeprecatedCommand("remove-persistence"); - - cmdUsageFormatter.addDeprecatedCommand("get-inactive-topic-policies"); - cmdUsageFormatter.addDeprecatedCommand("set-inactive-topic-policies"); - cmdUsageFormatter.addDeprecatedCommand("remove-inactive-topic-policies"); - - cmdUsageFormatter.addDeprecatedCommand("get-compaction-threshold"); - cmdUsageFormatter.addDeprecatedCommand("set-compaction-threshold"); - cmdUsageFormatter.addDeprecatedCommand("remove-compaction-threshold"); - - cmdUsageFormatter.addDeprecatedCommand("get-dispatch-rate"); - cmdUsageFormatter.addDeprecatedCommand("set-dispatch-rate"); - cmdUsageFormatter.addDeprecatedCommand("remove-dispatch-rate"); - - cmdUsageFormatter.addDeprecatedCommand("get-deduplication"); - cmdUsageFormatter.addDeprecatedCommand("set-deduplication"); - cmdUsageFormatter.addDeprecatedCommand("remove-deduplication"); - - cmdUsageFormatter.addDeprecatedCommand("get-deduplication-snapshot-interval"); - cmdUsageFormatter.addDeprecatedCommand("set-deduplication-snapshot-interval"); - cmdUsageFormatter.addDeprecatedCommand("remove-deduplication-snapshot-interval"); - - cmdUsageFormatter.addDeprecatedCommand("get-max-unacked-messages-on-subscription"); - cmdUsageFormatter.addDeprecatedCommand("set-max-unacked-messages-on-subscription"); - cmdUsageFormatter.addDeprecatedCommand("remove-max-unacked-messages-on-subscription"); - - cmdUsageFormatter.addDeprecatedCommand("set-subscription-types-enabled"); - cmdUsageFormatter.addDeprecatedCommand("get-subscription-types-enabled"); - cmdUsageFormatter.addDeprecatedCommand("remove-subscription-types-enabled"); - - cmdUsageFormatter.addDeprecatedCommand("get-delayed-delivery"); - cmdUsageFormatter.addDeprecatedCommand("set-delayed-delivery"); - cmdUsageFormatter.addDeprecatedCommand("remove-delayed-delivery"); - - cmdUsageFormatter.addDeprecatedCommand("get-max-producers"); - cmdUsageFormatter.addDeprecatedCommand("set-max-producers"); - cmdUsageFormatter.addDeprecatedCommand("remove-max-producers"); - - cmdUsageFormatter.addDeprecatedCommand("get-replicator-dispatch-rate"); - cmdUsageFormatter.addDeprecatedCommand("set-replicator-dispatch-rate"); - cmdUsageFormatter.addDeprecatedCommand("remove-replicator-dispatch-rate"); - - cmdUsageFormatter.addDeprecatedCommand("get-subscription-dispatch-rate"); - cmdUsageFormatter.addDeprecatedCommand("set-subscription-dispatch-rate"); - cmdUsageFormatter.addDeprecatedCommand("remove-subscription-dispatch-rate"); - - cmdUsageFormatter.addDeprecatedCommand("get-max-subscriptions-per-topic"); - cmdUsageFormatter.addDeprecatedCommand("set-max-subscriptions-per-topic"); - cmdUsageFormatter.addDeprecatedCommand("remove-max-subscriptions-per-topic"); - - cmdUsageFormatter.addDeprecatedCommand("get-offload-policies"); - cmdUsageFormatter.addDeprecatedCommand("set-offload-policies"); - cmdUsageFormatter.addDeprecatedCommand("remove-offload-policies"); - } + addCommand("trim-topic", new TrimTopic()); } - @Parameters(commandDescription = "Get the list of topics under a namespace.") + @Command(description = "Get the list of topics under a namespace.") private class ListCmd extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = {"-td", "--topic-domain"}, + @Option(names = {"-td", "--topic-domain"}, description = "Allowed topic domain (persistent, non_persistent).") private TopicDomain topicDomain; - @Parameter(names = { "-b", + @Option(names = { "-b", "--bundle" }, description = "Namespace bundle to get list of topics") private String bundle; - @Parameter(names = { "-ist", + @Option(names = { "-ist", "--include-system-topic" }, description = "Include system topic") private boolean includeSystemTopic; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); ListTopicsOptions options = ListTopicsOptions.builder() .bundle(bundle) .includeSystemTopic(includeSystemTopic) @@ -408,428 +305,464 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get the list of partitioned topics under a namespace.") + @Command(description = "Get the list of partitioned topics under a namespace.") private class PartitionedTopicListCmd extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "tenant/namespace", arity = "1") + private String namespaceName; - @Parameter(names = { "-ist", + @Option(names = { "-ist", "--include-system-topic" }, description = "Include system topic") private boolean includeSystemTopic; @Override void run() throws PulsarAdminException { - String namespace = validateNamespace(params); + String namespace = validateNamespace(namespaceName); ListTopicsOptions options = ListTopicsOptions.builder().includeSystemTopic(includeSystemTopic).build(); print(getTopics().getPartitionedTopicList(namespace, options)); } } - @Parameters(commandDescription = "Grant a new permission to a client role on a single topic.") + @Command(description = "Grant a new permission to a client role on a single topic.") private class GrantPermissions extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"-r", "--role"}, description = "Client role to which grant permissions", required = true) + @Option(names = {"-r", "--role"}, description = "Client role to which grant permissions", required = true) private String role; - @Parameter(names = {"-a", "--actions"}, description = "Actions to be granted (produce,consume,sources,sinks," - + "functions,packages)", required = true) + @Option(names = {"-a", "--actions"}, description = "Actions to be granted (produce,consume,sources,sinks," + + "functions,packages)", required = true, split = ",") private List actions; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); getTopics().grantPermission(topic, role, getAuthActions(actions)); } } - @Parameters(commandDescription = "Revoke permissions on a topic. " + @Command(description = "Revoke permissions on a topic. " + "Revoke permissions to a client role on a single topic. If the permission " + "was not set at the topic level, but rather at the namespace level, this " + "operation will return an error (HTTP status code 412).") private class RevokePermissions extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"-r", "--role"}, description = "Client role to which revoke permissions", required = true) + @Option(names = {"-r", "--role"}, description = "Client role to which revoke permissions", required = true) private String role; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); getTopics().revokePermissions(topic, role); } } - @Parameters(commandDescription = "Get the permissions on a topic. " + @Command(description = "Get the permissions on a topic. " + "Retrieve the effective permissions for a topic. These permissions are defined " + "by the permissions set at the namespace level combined (union) with any eventual " + "specific permission set on the topic.") private class Permissions extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); print(getTopics().getPermissions(topic)); } } - @Parameters(commandDescription = "Lookup a topic from the current serving broker") + @Command(description = "Lookup a topic from the current serving broker") private class Lookup extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); print(getAdmin().lookups().lookupTopic(topic)); } } - @Parameters(commandDescription = "Lookup a partitioned topic from the current serving broker") + @Command(description = "Lookup a partitioned topic from the current serving broker") protected class PartitionedLookup extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/partitionedTopic", required = true) - protected java.util.List params; - @Parameter(names = { "-s", - "--sort-by-broker" }, description = "Sort partitioned-topic by Broker Url") + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + protected String topicName; + @Option(names = { "-s", + "--sort-by-broker" }, description = "Sort partitioned-topic by Broker Url") protected boolean sortByBroker = false; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); if (sortByBroker) { - print(lookupPartitionedTopicSortByBroker(topic)); + Map partitionLookup = getAdmin().lookups().lookupPartitionedTopic(topic); + Map> result = new HashMap<>(); + for (Map.Entry entry : partitionLookup.entrySet()) { + List topics = result.getOrDefault(entry.getValue(), new ArrayList()); + topics.add(entry.getKey()); + result.put(entry.getValue(), topics); + } + print(result); } else { print(getAdmin().lookups().lookupPartitionedTopic(topic)); } } } - private Map> lookupPartitionedTopicSortByBroker(String topic) throws PulsarAdminException { - Map partitionLookup = getAdmin().lookups().lookupPartitionedTopic(topic); - Map> result = new HashMap<>(); - for (Map.Entry entry : partitionLookup.entrySet()) { - List topics = result.getOrDefault(entry.getValue(), new ArrayList()); - topics.add(entry.getKey()); - result.put(entry.getValue(), topics); - } - return result; - } - - @Parameters(commandDescription = "Get Namespace bundle range of a topic") + @Command(description = "Get Namespace bundle range of a topic") private class GetBundleRange extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); print(getAdmin().lookups().getBundleRange(topic)); } } - @Parameters(commandDescription = "Create a partitioned topic. " + @Command(description = "Create a partitioned topic. " + "The partitioned topic has to be created before creating a producer on it.") private class CreatePartitionedCmd extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-p", + @Option(names = { "-p", "--partitions" }, description = "Number of partitions for the topic", required = true) private int numPartitions; - @Parameter(names = {"--metadata", "-m"}, description = "key value pair properties(a=a,b=b,c=c)") + @Option(names = {"--metadata", "-m"}, description = "key value pair properties(a=a,b=b,c=c)") private java.util.List metadata; @Override void run() throws Exception { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); Map map = parseListKeyValueMap(metadata); getTopics().createPartitionedTopic(topic, numPartitions, map); } } - @Parameters(commandDescription = "Try to create partitions for partitioned topic. " + @Command(description = "Try to create partitions for partitioned topic. " + "The partitions of partition topic has to be created, can be used by repair partitions when " + "topic auto creation is disabled") private class CreateMissedPartitionsCmd extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws Exception { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); getTopics().createMissedPartitions(topic); } } - @Parameters(commandDescription = "Create a non-partitioned topic.") + @Command(description = "Create a non-partitioned topic.") private class CreateNonPartitionedCmd extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--metadata", "-m"}, description = "key value pair properties(a=a,b=b,c=c)") + @Option(names = {"--metadata", "-m"}, description = "key value pair properties(a=a,b=b,c=c)") private java.util.List metadata; @Override void run() throws Exception { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); Map map = parseListKeyValueMap(metadata); getTopics().createNonPartitionedTopic(topic, map); } } - @Parameters(commandDescription = "Update existing partitioned topic. " + @Command(description = "Update existing partitioned topic. " + "New updating number of partitions must be greater than existing number of partitions.") private class UpdatePartitionedCmd extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-p", + @Option(names = { "-p", "--partitions" }, description = "Number of partitions for the topic", required = true) private int numPartitions; - @Parameter(names = { "-ulo", + @Option(names = { "-ulo", "--update-local-only"}, description = "Update partitions number for topic in local cluster only") private boolean updateLocalOnly = false; - @Parameter(names = { "-f", + @Option(names = { "-f", "--force" }, description = "Update forcefully without validating existing partitioned topic") private boolean force; @Override void run() throws Exception { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); getTopics().updatePartitionedTopic(topic, numPartitions, updateLocalOnly, force); } } - @Parameters(commandDescription = "Get the partitioned topic metadata. " + @Command(description = "Get the partitioned topic metadata. " + "If the topic is not created or is a non-partitioned topic, it returns empty topic with 0 partitions") private class GetPartitionedTopicMetadataCmd extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws Exception { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); print(getTopics().getPartitionedTopicMetadata(topic)); } } - @Parameters(commandDescription = "Get the topic properties.") + @Command(description = "Get the topic properties.") private class GetPropertiesCmd extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws Exception { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); print(getTopics().getProperties(topic)); } } - @Parameters(commandDescription = "Update the properties of on a topic") + @Command(description = "Update the properties of on a topic") private class UpdateProperties extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--property", "-p"}, description = "key value pair properties(-p a=b -p c=d)", - required = false, splitter = NoSplitter.class) - private java.util.List properties; + @Option(names = {"--property", "-p"}, description = "key value pair properties(-p a=b -p c=d)", + required = false) + private Map properties; @Override void run() throws Exception { - String topic = validateTopicName(params); - Map map = parseListKeyValueMap(properties); - if (map == null) { - map = Collections.emptyMap(); + String topic = validateTopicName(topicName); + if (properties == null) { + properties = Collections.emptyMap(); } - getTopics().updateProperties(topic, map); + getTopics().updateProperties(topic, properties); } } - @Parameters(commandDescription = "Remove the key in properties of a topic") + @Command(description = "Remove the key in properties of a topic") private class RemoveProperties extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--key", "-k"}, description = "The key to remove in the properties of topic") + @Option(names = {"--key", "-k"}, description = "The key to remove in the properties of topic") private String key; @Override void run() throws Exception { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); getTopics().removeProperties(topic, key); } } - @Parameters(commandDescription = "Delete a partitioned topic. " + @Command(description = "Delete a partitioned topic. " + "It will also delete all the partitions of the topic if it exists." + "And the application is not able to connect to the topic(delete then re-create with same name) again " + "if the schema auto uploading is disabled. Besides, users should to use the truncate cmd to clean up " + "data of the topic instead of delete cmd if users continue to use this topic later.") private class DeletePartitionedCmd extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-f", + @Option(names = { "-f", "--force" }, description = "Close all producer/consumer/replicator and delete topic forcefully") private boolean force = false; - @Parameter(names = {"-d", "--deleteSchema"}, description = "Delete schema while deleting topic, " + @Option(names = {"-d", "--deleteSchema"}, description = "Delete schema while deleting topic, " + "but the parameter is invalid and the schema is always deleted", hidden = true) private boolean deleteSchema = false; @Override void run() throws Exception { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); getTopics().deletePartitionedTopic(topic, force); } } - @Parameters(commandDescription = "Delete a topic. " + @Command(description = "Delete a topic. " + "The topic cannot be deleted if there's any active subscription or producers connected to it." + "And the application is not able to connect to the topic(delete then re-create with same name) again " + "if the schema auto uploading is disabled. Besides, users should to use the truncate cmd to clean up " + "data of the topic instead of delete cmd if users continue to use this topic later.") - private class DeleteCmd extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + protected class DeleteCmd extends CliCommand { + @Parameters(description = "Provide either a single topic in the format 'persistent://tenant/namespace/topic', " + + "or a path to a file containing a list of topics, e.g., 'path://resources/topics.txt'. " + + "This parameter is required.", arity = "1") + protected String topic; - @Parameter(names = { "-f", + @Option(names = { "-f", "--force" }, description = "Close all producer/consumer/replicator and delete topic forcefully") private boolean force = false; - @Parameter(names = {"-d", "--deleteSchema"}, description = "Delete schema while deleting topic, " + @Option(names = {"-d", "--deleteSchema"}, description = "Delete schema while deleting topic, " + "but the parameter is invalid and the schema is always deleted", hidden = true) private boolean deleteSchema = false; + @Option(names = {"-r", "regex"}, + description = "Use a regex expression to match multiple topics for deletion.") + boolean regex = false; + + @Option(names = {"--from-file"}, description = "Read a list of topics from a file for deletion.") + boolean readFromFile; + + @Override - void run() throws PulsarAdminException { - String topic = validateTopicName(params); - getTopics().delete(topic, force); + void run() throws PulsarAdminException, IOException { + if (readFromFile && regex) { + throw new ParameterException("Could not apply regex when read topics from file."); + } + if (readFromFile) { + List topicsFromFile = Files.readAllLines(Path.of(topic)); + for (String t : topicsFromFile) { + try { + getTopics().delete(t, force); + } catch (Exception e) { + print("Failed to delete topic: " + t + ". Exception: " + e); + } + } + } else { + String topicName = validateTopicName(topic); + if (regex) { + String namespace = TopicName.get(topic).getNamespace(); + List topics = getTopics().getList(namespace); + topics = topics.stream().filter(s -> s.matches(topicName)).toList(); + for (String t : topics) { + try { + getTopics().delete(t, force); + } catch (Exception e) { + print("Failed to delete topic: " + t + ". Exception: " + e); + } + } + } else { + try { + getTopics().delete(topicName, force); + } catch (Exception e) { + print("Failed to delete topic: " + topic + ". Exception: " + e); + } + } + } } } - @Parameters(commandDescription = "Truncate a topic. \n" + @Command(description = "Truncate a topic. \n" + "\t\tThe truncate operation will move all cursors to the end of the topic " + "and delete all inactive ledgers. ") private class TruncateCmd extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic\n", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); getTopics().truncate(topic); } } - @Parameters(commandDescription = "Unload a topic.") + @Command(description = "Unload a topic.") private class UnloadCmd extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); getTopics().unload(topic); } } - @Parameters(commandDescription = "Get the list of subscriptions on the topic") + @Command(description = "Get the list of subscriptions on the topic") private class ListSubscriptions extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws Exception { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); print(getTopics().getSubscriptions(topic)); } } - @Parameters(commandDescription = "Delete a durable subscriber from a topic. " + @Command(description = "Delete a durable subscriber from a topic. " + "The subscription cannot be deleted if there are any active consumers attached to it") private class DeleteSubscription extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-f", + @Option(names = { "-f", "--force" }, description = "Disconnect and close all consumers and delete subscription forcefully") private boolean force = false; - @Parameter(names = { "-s", "--subscription" }, description = "Subscription to be deleted", required = true) + @Option(names = {"-s", "--subscription"}, description = "Subscription to be deleted", required = true) private String subName; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); getTopics().deleteSubscription(topic, subName, force); } } - @Parameters(commandDescription = "Get the stats for the topic and its connected producers and consumers. " + @Command(name = "stats", description = "Get the stats for the topic and its connected producers and consumers. " + "All the rates are computed over a 1 minute window and are relative the last completed 1 minute period.") private class GetStats extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-gpb", + @Option(names = { "-gpb", "--get-precise-backlog" }, description = "Set true to get precise backlog") private boolean getPreciseBacklog = false; - @Parameter(names = { "-sbs", + @Option(names = { "-sbs", "--get-subscription-backlog-size" }, description = "Set true to get backlog size for each subscription" - + ", locking required. If set to false, the attribute 'backlogSize' in the response will be -1") + + ", locking required. If set to false, the attribute 'backlogSize' in the response will be -1") private boolean subscriptionBacklogSize = true; - @Parameter(names = { "-etb", + @Option(names = { "-etb", "--get-earliest-time-in-backlog" }, description = "Set true to get earliest time in backlog") private boolean getEarliestTimeInBacklog = false; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); print(getTopics().getStats(topic, getPreciseBacklog, subscriptionBacklogSize, getEarliestTimeInBacklog)); } } - @Parameters(commandDescription = "Get the internal stats for the topic") + @Command(description = "Get the internal stats for the topic") private class GetInternalStats extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-m", + @Option(names = { "-m", "--metadata" }, description = "Flag to include ledger metadata") private boolean metadata = false; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); print(getTopics().getInternalStats(topic, metadata)); } } - @Parameters(commandDescription = "Get the internal metadata info for the topic") + @Command(description = "Get the internal metadata info for the topic") private class GetInternalInfo extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); String internalInfo = getTopics().getInternalInfo(topic); if (internalInfo == null) { System.out.println("Did not find any internal metadata info"); @@ -841,123 +774,115 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get the stats for the partitioned topic " + @Command(description = "Get the stats for the partitioned topic " + "and its connected producers and consumers. All the rates are computed over a 1 minute window " + "and are relative the last completed 1 minute period.") private class GetPartitionedStats extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = "--per-partition", description = "Get per partition stats") + @Option(names = "--per-partition", description = "Get per partition stats") private boolean perPartition = false; - @Parameter(names = { "-gpb", + @Option(names = { "-gpb", "--get-precise-backlog" }, description = "Set true to get precise backlog") private boolean getPreciseBacklog = false; - @Parameter(names = { "-sbs", + @Option(names = { "-sbs", "--get-subscription-backlog-size" }, description = "Set true to get backlog size for each subscription" + ", locking required.") private boolean subscriptionBacklogSize = true; - @Parameter(names = { "-etb", + @Option(names = { "-etb", "--get-earliest-time-in-backlog" }, description = "Set true to get earliest time in backlog") private boolean getEarliestTimeInBacklog = false; @Override void run() throws Exception { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); print(getTopics().getPartitionedStats(topic, perPartition, getPreciseBacklog, subscriptionBacklogSize, getEarliestTimeInBacklog)); } } - @Parameters(commandDescription = "Get the internal stats for the partitioned topic " + @Command(description = "Get the internal stats for the partitioned topic " + "and its connected producers and consumers. All the rates are computed over a 1 minute window " + "and are relative the last completed 1 minute period.") private class GetPartitionedStatsInternal extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws Exception { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); print(getTopics().getPartitionedInternalStats(topic)); } } - @Parameters(commandDescription = "Skip all the messages for the subscription") + @Command(description = "Skip all the messages for the subscription") private class ClearBacklog extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-s", "--subscription" }, description = "Subscription to be cleared", required = true) + @Option(names = { "-s", "--subscription" }, description = "Subscription to be cleared", required = true) private String subName; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); getTopics().skipAllMessages(topic, subName); } } - @Parameters(commandDescription = "Skip some messages for the subscription") + @Command(description = "Skip some messages for the subscription") private class Skip extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-s", + @Option(names = { "-s", "--subscription" }, description = "Subscription to be skip messages on", required = true) private String subName; - @Parameter(names = { "-n", "--count" }, description = "Number of messages to skip", required = true) + @Option(names = { "-n", "--count" }, description = "Number of messages to skip", required = true) private long numMessages; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); getTopics().skipMessages(topic, subName, numMessages); } } - @Parameters(commandDescription = "Expire messages that older than given expiry time (in seconds) " + @Command(description = "Expire messages that older than given expiry time (in seconds) " + "for the subscription") private class ExpireMessages extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-s", + @Option(names = { "-s", "--subscription" }, description = "Subscription to be skip messages on", required = true) private String subName; - @Parameter(names = { "-t", "--expireTime" }, description = "Expire messages older than time in seconds " - + "(or minutes, hours, days, weeks eg: 100m, 3h, 2d, 5w)") - private String expireTimeStr = null; + @Option(names = { "-t", "--expireTime" }, description = "Expire messages older than time in seconds " + + "(or minutes, hours, days, weeks eg: 100m, 3h, 2d, 5w)", + converter = TimeUnitToSecondsConverter.class) + private Long expireTimeInSeconds = -1L; - @Parameter(names = { "--position", + @Option(names = { "--position", "-p" }, description = "message position to reset back to (ledgerId:entryId)", required = false) private String messagePosition; - @Parameter(names = { "-e", "--exclude-reset-position" }, + @Option(names = { "-e", "--exclude-reset-position" }, description = "Exclude the reset position, start consume messages from the next position.") private boolean excludeResetPosition = false; @Override void run() throws PulsarAdminException { - long expireTimeInSeconds = -1; - if (expireTimeStr != null) { - try { - expireTimeInSeconds = RelativeTimeUtil.parseRelativeTimeInSeconds(expireTimeStr); - } catch (IllegalArgumentException e) { - throw new ParameterException(e.getMessage()); - } - } - if (expireTimeInSeconds >= 0 && isNotBlank(messagePosition)) { throw new ParameterException(String.format("Can't expire message by time and " + "by message position at the same time.")); } - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); if (expireTimeInSeconds >= 0) { getTopics().expireMessages(topic, subName, expireTimeInSeconds); } else if (isNotBlank(messagePosition)) { @@ -972,52 +897,47 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Expire messages that older than given expiry time (in seconds) " + @Command(description = "Expire messages that older than given expiry time (in seconds) " + "for all subscriptions") private class ExpireMessagesForAllSubscriptions extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-t", "--expireTime" }, description = "Expire messages older than time in seconds " - + "(or minutes, hours, days, weeks eg: 100m, 3h, 2d, 5w)", required = true) - private String expireTimeStr; + @Option(names = {"-t", "--expireTime"}, description = "Expire messages older than time in seconds " + + "(or minutes, hours, days, weeks eg: 100m, 3h, 2d, 5w)", required = true, + converter = TimeUnitToSecondsConverter.class) + private Long expireTimeInSeconds; @Override void run() throws PulsarAdminException { - long expireTimeInSeconds; - try { - expireTimeInSeconds = RelativeTimeUtil.parseRelativeTimeInSeconds(expireTimeStr); - } catch (IllegalArgumentException e) { - throw new ParameterException(e.getMessage()); - } - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); getTopics().expireMessagesForAllSubscriptions(topic, expireTimeInSeconds); } } - @Parameters(commandDescription = "Create a new subscription on a topic") + @Command(description = "Create a new subscription on a topic") private class CreateSubscription extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-s", - "--subscription" }, description = "Subscription to reset position on", required = true) + @Option(names = { "-s", + "--subscription" }, description = "Name of subscription to be created", required = true) private String subscriptionName; - @Parameter(names = { "-m" , "--messageId" }, description = "messageId where to create the subscription. " + @Option(names = { "-m" , "--messageId" }, description = "messageId where to create the subscription. " + "It can be either 'latest', 'earliest' or (ledgerId:entryId)", required = false) private String messageIdStr = "latest"; - @Parameter(names = { "-r", "--replicated" }, description = "replicated subscriptions", required = false) + @Option(names = { "-r", "--replicated" }, description = "replicated subscriptions", required = false) private boolean replicated = false; - @Parameter(names = {"--property", "-p"}, description = "key value pair properties(-p a=b -p c=d)", - required = false, splitter = NoSplitter.class) - private java.util.List properties; + @Option(names = {"--property", "-p"}, description = "key value pair properties(-p a=b -p c=d)", + required = false) + private Map properties; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); MessageId messageId; if (messageIdStr.equals("latest")) { messageId = MessageId.latest; @@ -1026,57 +946,55 @@ void run() throws PulsarAdminException { } else { messageId = validateMessageIdString(messageIdStr); } - Map map = parseListKeyValueMap(properties); - getTopics().createSubscription(topic, subscriptionName, messageId, replicated, map); + getTopics().createSubscription(topic, subscriptionName, messageId, replicated, properties); } } - @Parameters(commandDescription = "Update the properties of a subscription on a topic") + @Command(description = "Update the properties of a subscription on a topic") private class UpdateSubscriptionProperties extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-s", + @Option(names = { "-s", "--subscription" }, description = "Subscription to update", required = true) private String subscriptionName; - @Parameter(names = {"--property", "-p"}, description = "key value pair properties(-p a=b -p c=d)", - required = false, splitter = NoSplitter.class) - private java.util.List properties; + @Option(names = {"--property", "-p"}, description = "key value pair properties(-p a=b -p c=d)", + required = false) + private Map properties; - @Parameter(names = {"--clear", "-c"}, description = "Remove all properties", + @Option(names = {"--clear", "-c"}, description = "Remove all properties", required = false) private boolean clear; @Override void run() throws Exception { - String topic = validateTopicName(params); - Map map = parseListKeyValueMap(properties); - if (map == null) { - map = Collections.emptyMap(); + String topic = validateTopicName(topicName); + if (properties == null) { + properties = Collections.emptyMap(); } - if ((map.isEmpty()) && !clear) { - throw new ParameterException("If you want to clear the properties you have to use --clear"); + if ((properties.isEmpty()) && !clear) { + throw new IllegalArgumentException("If you want to clear the properties you have to use --clear"); } - if (clear && !map.isEmpty()) { - throw new ParameterException("If you set --clear then you should not pass any properties"); + if (clear && !properties.isEmpty()) { + throw new IllegalArgumentException("If you set --clear then you should not pass any properties"); } - getTopics().updateSubscriptionProperties(topic, subscriptionName, map); + getTopics().updateSubscriptionProperties(topic, subscriptionName, properties); } } - @Parameters(commandDescription = "Get the properties of a subscription on a topic") + @Command(description = "Get the properties of a subscription on a topic") private class GetSubscriptionProperties extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-s", + @Option(names = { "-s", "--subscription" }, description = "Subscription to describe", required = true) private String subscriptionName; @Override void run() throws Exception { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); Map result = getTopics().getSubscriptionProperties(topic, subscriptionName); // Ensure we are using JSON and not Java toString() System.out.println(ObjectMapperFactory.getMapper().writer().writeValueAsString(result)); @@ -1084,32 +1002,33 @@ void run() throws Exception { } - @Parameters(commandDescription = "Reset position for subscription to a position that is closest to " + @Command(description = "Reset position for subscription to a position that is closest to " + "timestamp or messageId.") private class ResetCursor extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-s", - "--subscription" }, description = "Subscription to reset position on", required = true) + @Option(names = {"-s", + "--subscription"}, description = "Subscription to reset position on", required = true) private String subName; - @Parameter(names = { "--time", + @Option(names = { "--time", "-t" }, description = "time in minutes to reset back to " - + "(or minutes, hours, days, weeks eg: 100m, 3h, 2d, 5w)", required = false) - private String resetTimeStr; + + "(or minutes, hours, days, weeks eg: 100m, 3h, 2d, 5w)", required = false, + converter = TimeUnitToMillisConverter.class) + private Long resetTimeInMillis = null; - @Parameter(names = { "--messageId", + @Option(names = { "--messageId", "-m" }, description = "messageId to reset back to ('latest', 'earliest', or 'ledgerId:entryId')") private String resetMessageIdStr; - @Parameter(names = { "-e", "--exclude-reset-position" }, + @Option(names = { "-e", "--exclude-reset-position" }, description = "Exclude the reset position, start consume messages from the next position.") private boolean excludeResetPosition = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); if (isNotBlank(resetMessageIdStr)) { MessageId messageId; if ("earliest".equals(resetMessageIdStr)) { @@ -1124,14 +1043,7 @@ void run() throws PulsarAdminException { } else { getTopics().resetCursor(persistentTopic, subName, messageId); } - } else if (isNotBlank(resetTimeStr)) { - long resetTimeInMillis; - try { - resetTimeInMillis = TimeUnit.SECONDS.toMillis( - RelativeTimeUtil.parseRelativeTimeInSeconds(resetTimeStr)); - } catch (IllegalArgumentException exception) { - throw new ParameterException(exception.getMessage()); - } + } else if (Objects.nonNull(resetTimeInMillis)) { // now - go back time long timestamp = System.currentTimeMillis() - resetTimeInMillis; getTopics().resetCursor(persistentTopic, subName, timestamp); @@ -1142,14 +1054,14 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Terminate a topic and don't allow any more messages to be published") + @Command(description = "Terminate a topic and don't allow any more messages to be published") private class Terminate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); try { MessageId lastMessageId = getTopics().terminateTopicAsync(persistentTopic).get(); @@ -1160,38 +1072,51 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Terminate a partitioned topic and don't allow any more messages to be published") + @Command(description = "Terminate a partitioned topic and don't allow any more messages to be published") private class PartitionedTerminate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException, TimeoutException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); Map messageIds = getTopics().terminatePartitionedTopic(persistentTopic); - for (Map.Entry entry: messageIds.entrySet()) { + for (Map.Entry entry : messageIds.entrySet()) { String topicName = persistentTopic + "-partition-" + entry.getKey(); - System.out.println("Topic " + topicName + " successfully terminated at " + entry.getValue()); + System.out.println("Topic " + topicName + " successfully terminated at " + entry.getValue()); } } } - @Parameters(commandDescription = "Peek some messages for the subscription") + @Command(description = "Peek some messages for the subscription") private class PeekMessages extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-s", + @Option(names = { "-s", "--subscription" }, description = "Subscription to get messages from", required = true) private String subName; - @Parameter(names = { "-n", "--count" }, description = "Number of messages (default 1)", required = false) + @Option(names = { "-n", "--count" }, description = "Number of messages (default 1)", required = false) private int numMessages = 1; + @Option(names = { "-ssm", "--show-server-marker" }, + description = "Enables the display of internal server write markers.", required = false) + private boolean showServerMarker = false; + + @Option(names = { "-til", "--transaction-isolation-level" }, + description = "Sets the isolation level for peeking messages within transactions. " + + "'READ_COMMITTED' allows peeking only committed transactional messages. " + + "'READ_UNCOMMITTED' allows peeking all messages, " + + "even transactional messages which have been aborted.", + required = false) + private TransactionIsolationLevel transactionIsolationLevel = TransactionIsolationLevel.READ_COMMITTED; + @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); - List> messages = getTopics().peekMessages(persistentTopic, subName, numMessages); + String persistentTopic = validatePersistentTopic(topicName); + List> messages = getTopics().peekMessages(persistentTopic, subName, numMessages, + showServerMarker, transactionIsolationLevel); int position = 0; for (Message msg : messages) { MessageImpl message = (MessageImpl) msg; @@ -1213,6 +1138,10 @@ void run() throws PulsarAdminException { if (message.getDeliverAtTime() != 0) { System.out.println("Deliver at time: " + message.getDeliverAtTime()); } + MessageMetadata msgMetaData = message.getMessageBuilder(); + if (showServerMarker && msgMetaData.hasMarkerType()) { + System.out.println("Marker Type: " + MarkerType.valueOf(msgMetaData.getMarkerType())); + } if (message.getBrokerEntryMetadata() != null) { if (message.getBrokerEntryMetadata().hasBrokerTimestamp()) { @@ -1236,24 +1165,24 @@ void run() throws PulsarAdminException { } - @Parameters(commandDescription = "Examine a specific message on a topic by position relative to the" + @Command(description = "Examine a specific message on a topic by position relative to the" + " earliest or the latest message.") private class ExamineMessages extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-i", "--initialPosition" }, + @Option(names = { "-i", "--initialPosition" }, description = "Relative start position to examine message." + "It can be 'latest' or 'earliest', default is latest") private String initialPosition = "latest"; - @Parameter(names = { "-m", "--messagePosition" }, + @Option(names = { "-m", "--messagePosition" }, description = "The position of messages (default 1)", required = false) private long messagePosition = 1; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); MessageImpl message = (MessageImpl) getTopics().examineMessage(persistentTopic, initialPosition, messagePosition); @@ -1292,24 +1221,24 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get message by its ledgerId and entryId") + @Command(description = "Get message by its ledgerId and entryId") private class GetMessageById extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-l", "--ledgerId" }, + @Option(names = { "-l", "--ledgerId" }, description = "ledger id pointing to the desired ledger", required = true) private long ledgerId; - @Parameter(names = { "-e", "--entryId" }, + @Option(names = { "-e", "--entryId" }, description = "entry id pointing to the desired entry", required = true) private long entryId; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); MessageImpl message = (MessageImpl) getTopics().getMessageById(persistentTopic, ledgerId, entryId); if (message == null) { @@ -1354,12 +1283,12 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get message ID") + @Command(description = "Get message ID") private class GetMessageId extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-d", "--datetime" }, + @Option(names = { "-d", "--datetime" }, description = "datetime at or before this messageId. This datetime is in format of " + "ISO_OFFSET_DATE_TIME, e.g. 2021-06-28T16:53:08Z or 2021-06-28T16:53:08.123456789+08:00", required = true) @@ -1367,7 +1296,7 @@ private class GetMessageId extends CliCommand { @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); long timestamp = DateFormatter.parse(datetime); MessageId messageId = getTopics().getMessageIdByTimestamp(persistentTopic, timestamp); @@ -1379,32 +1308,32 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Compact a topic") + @Command(description = "Compact a topic") private class Compact extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().triggerCompaction(persistentTopic); System.out.println("Topic compaction requested for " + persistentTopic); } } - @Parameters(commandDescription = "Status of compaction on a topic") + @Command(description = "Status of compaction on a topic") private class CompactionStatusCmd extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-w", "--wait-complete" }, + @Option(names = { "-w", "--wait-complete" }, description = "Wait for compaction to complete", required = false) private boolean wait = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); try { LongRunningProcessStatus status = getTopics().compactionStatus(persistentTopic); @@ -1450,20 +1379,20 @@ static MessageId findFirstLedgerWithinThreshold(List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); - long sizeThreshold = validateSizeString(sizeThresholdStr); + String persistentTopic = validatePersistentTopic(topicName); PersistentTopicInternalStats stats = getTopics().getInternalStats(persistentTopic, false); if (stats.ledgers.size() < 1) { @@ -1484,18 +1413,18 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Check the status of data offloading from a topic to long-term storage") + @Command(description = "Check the status of data offloading from a topic to long-term storage") private class OffloadStatusCmd extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-w", "--wait-complete" }, + @Option(names = { "-w", "--wait-complete" }, description = "Wait for offloading to complete", required = false) private boolean wait = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); try { OffloadProcessStatus status = getTopics().offloadStatus(persistentTopic); @@ -1525,52 +1454,54 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "get the last commit message id of topic") + @Command(description = "get the last commit message id of topic") private class GetLastMessageId extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getLastMessageId(persistentTopic)); } } - @Parameters(commandDescription = "Get the backlog quota policies for a topic") + @Command(description = "Get the backlog quota policies for a topic", hidden = true) private class GetBacklogQuotaMap extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"-ap", "--applied"}, description = "Get the applied policy of the topic") + @Option(names = {"-ap", "--applied"}, description = "Get the applied policy of the topic") private boolean applied = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getBacklogQuotaMap(persistentTopic, applied)); } } - @Parameters(commandDescription = "Set a backlog quota policy for a topic") + @Command(description = "Set a backlog quota policy for a topic", hidden = true) private class SetBacklogQuota extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-l", "--limit" }, description = "Size limit (eg: 10M, 16G)") - private String limitStr = "-1"; + @Option(names = { "-l", "--limit" }, description = "Size limit (eg: 10M, 16G)", + converter = ByteUnitToLongConverter.class) + private Long limit = -1L; - @Parameter(names = { "-lt", "--limitTime" }, + @Option(names = { "-lt", "--limitTime" }, description = "Time limit in second (or minutes, hours, days, weeks eg: 100m, 3h, 2d, 5w), " - + "non-positive number for disabling time limit.") - private String limitTimeStr = null; + + "non-positive number for disabling time limit.", + converter = TimeUnitToSecondsConverter.class) + private Long limitTimeInSec = -1L; - @Parameter(names = { "-p", "--policy" }, + @Option(names = { "-p", "--policy" }, description = "Retention policy to enforce when the limit is reached. Valid options are: " + "[producer_request_hold, producer_exception, consumer_backlog_eviction]", required = true) private String policyStr; - @Parameter(names = {"-t", "--type"}, description = "Backlog quota type to set. Valid options are: " + @Option(names = {"-t", "--type"}, description = "Backlog quota type to set. Valid options are: " + "destination_storage and message_age. " + "destination_storage limits backlog by size (in bytes). " + "message_age limits backlog by time, that is, message timestamp (broker or publish timestamp). " @@ -1580,7 +1511,6 @@ private class SetBacklogQuota extends CliCommand { @Override void run() throws PulsarAdminException { BacklogQuota.RetentionPolicy policy; - long limit; BacklogQuota.BacklogQuotaType backlogQuotaType; try { @@ -1590,8 +1520,6 @@ void run() throws PulsarAdminException { policyStr, Arrays.toString(BacklogQuota.RetentionPolicy.values()))); } - limit = validateSizeString(limitStr); - try { backlogQuotaType = BacklogQuota.BacklogQuotaType.valueOf(backlogQuotaTypeStr); } catch (IllegalArgumentException e) { @@ -1599,573 +1527,545 @@ void run() throws PulsarAdminException { backlogQuotaTypeStr, Arrays.toString(BacklogQuota.BacklogQuotaType.values()))); } - long limitTimeInSec = -1; - if (limitTimeStr != null) { - try { - limitTimeInSec = RelativeTimeUtil.parseRelativeTimeInSeconds(limitTimeStr); - } catch (IllegalArgumentException e) { - throw new ParameterException(e.getMessage()); - } - } - if (limitTimeInSec > Integer.MAX_VALUE) { - throw new ParameterException( - String.format("Time limit cannot be greater than %d seconds", Integer.MAX_VALUE)); - } - - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().setBacklogQuota(persistentTopic, BacklogQuota.builder().limitSize(limit) - .limitTime((int) limitTimeInSec) + .limitTime(limitTimeInSec.intValue()) .retentionPolicy(policy) .build(), backlogQuotaType); } } - @Parameters(commandDescription = "Remove a backlog quota policy from a topic") + @Command(description = "Remove a backlog quota policy from a topic", hidden = true) private class RemoveBacklogQuota extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"-t", "--type"}, description = "Backlog quota type to remove") + @Option(names = {"-t", "--type"}, description = "Backlog quota type to remove") private String backlogQuotaType = BacklogQuota.BacklogQuotaType.destination_storage.name(); @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removeBacklogQuota(persistentTopic, BacklogQuota.BacklogQuotaType.valueOf(backlogQuotaType)); } } - @Parameters(commandDescription = "Get the replication clusters for a topic") + @Command(description = "Get the replication clusters for a topic") private class GetReplicationClusters extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getReplicationClusters(persistentTopic, applied)); } } - @Parameters(commandDescription = "Set the replication clusters for a topic") + @Command(description = "Set the replication clusters for a topic") private class SetReplicationClusters extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--clusters", + @Option(names = { "--clusters", "-c" }, description = "Replication Cluster Ids list (comma separated values)", required = true) private String clusterIds; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); List clusters = Lists.newArrayList(clusterIds.split(",")); getTopics().setReplicationClusters(persistentTopic, clusters); } } - @Parameters(commandDescription = "Remove the replication clusters for a topic") + @Command(description = "Remove the replication clusters for a topic") private class RemoveReplicationClusters extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removeReplicationClusters(persistentTopic); } } - @Parameters(commandDescription = "Get the shadow topics for a topic") + @Command(description = "Get the shadow topics for a topic") private class GetShadowTopics extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getShadowTopics(persistentTopic)); } } - @Parameters(commandDescription = "Set the shadow topics for a topic") + @Command(description = "Set the shadow topics for a topic") private class SetShadowTopics extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--topics", + @Option(names = { "--topics", "-t" }, description = "Shadow topic list (comma separated values)", required = true) private String shadowTopics; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); List topics = Lists.newArrayList(shadowTopics.split(",")); getTopics().setShadowTopics(persistentTopic, topics); } } - @Parameters(commandDescription = "Remove the shadow topics for a topic") + @Command(description = "Remove the shadow topics for a topic") private class RemoveShadowTopics extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removeShadowTopics(persistentTopic); } } - @Parameters(commandDescription = "Create a shadow topic for an existing source topic.") + @Command(description = "Create a shadow topic for an existing source topic.") private class CreateShadowTopic extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--source", "-s"}, description = "source topic name", required = true) + @Option(names = {"--source", "-s"}, description = "source topic name", required = true) private String sourceTopic; - @Parameter(names = {"--properties", "-p"}, description = "key value pair properties(eg: a=a b=b c=c)") - private java.util.List propertyList; + @Option(names = {"--properties", "-p"}, description = "key value pair properties(eg: a=a,b=b,c=c)", split = ",") + private Map properties; @Override void run() throws Exception { - String topic = validateTopicName(params); - Map properties = parseListKeyValueMap(propertyList); + String topic = validateTopicName(topicName); getTopics().createShadowTopic(topic, TopicName.get(sourceTopic).toString(), properties); } } - @Parameters(commandDescription = "Get the source topic for a shadow topic") + @Command(description = "Get the source topic for a shadow topic") private class GetShadowSource extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; + @Override void run() throws PulsarAdminException { - String shadowTopic = validatePersistentTopic(params); + String shadowTopic = validatePersistentTopic(topicName); print(getTopics().getShadowSource(shadowTopic)); } } - @Parameters(commandDescription = "Get the delayed delivery policy for a topic") + @Command(description = "Get the delayed delivery policy for a topic", hidden = true) private class GetDelayedDelivery extends CliCommand { - @Parameter(description = "tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; @Override void run() throws PulsarAdminException { - String topicName = validateTopicName(params); - print(getTopics().getDelayedDeliveryPolicy(topicName, applied)); + String topic = validateTopicName(topicName); + print(getTopics().getDelayedDeliveryPolicy(topic, applied)); } } - @Parameters(commandDescription = "Set the delayed delivery policy on a topic") + @Command(description = "Set the delayed delivery policy on a topic", hidden = true) private class SetDelayedDelivery extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--enable", "-e" }, description = "Enable delayed delivery messages") + @Option(names = { "--enable", "-e" }, description = "Enable delayed delivery messages") private boolean enable = false; - @Parameter(names = { "--disable", "-d" }, description = "Disable delayed delivery messages") + @Option(names = { "--disable", "-d" }, description = "Disable delayed delivery messages") private boolean disable = false; - @Parameter(names = { "--time", "-t" }, + @Option(names = { "--time", "-t" }, description = "The tick time for when retrying on delayed delivery messages, affecting the accuracy of " - + "the delivery time compared to the scheduled time. (eg: 1s, 10s, 1m, 5h, 3d)") - private String delayedDeliveryTimeStr = "1s"; + + "the delivery time compared to the scheduled time. (eg: 1s, 10s, 1m, 5h, 3d)", + converter = TimeUnitToMillisConverter.class) + private Long delayedDeliveryTimeInMills = 1_000L; + + @Option(names = {"--maxDelay", "-md"}, + description = "The max allowed delay for delayed delivery. (eg: 1s, 10s, 1m, 5h, 3d)", + converter = TimeUnitToMillisConverter.class) + private Long delayedDeliveryMaxDelayInMillis = 0L; @Override void run() throws PulsarAdminException { - String topicName = validateTopicName(params); - long delayedDeliveryTimeInMills; - try { - delayedDeliveryTimeInMills = TimeUnit.SECONDS.toMillis( - RelativeTimeUtil.parseRelativeTimeInSeconds(delayedDeliveryTimeStr)); - } catch (IllegalArgumentException exception) { - throw new ParameterException(exception.getMessage()); - } - + String topic = validateTopicName(topicName); if (enable == disable) { throw new ParameterException("Need to specify either --enable or --disable"); } - getTopics().setDelayedDeliveryPolicy(topicName, DelayedDeliveryPolicies.builder() + getTopics().setDelayedDeliveryPolicy(topic, DelayedDeliveryPolicies.builder() .tickTime(delayedDeliveryTimeInMills) .active(enable) + .maxDeliveryDelayInMillis(delayedDeliveryMaxDelayInMillis) .build()); } } - @Parameters(commandDescription = "Remove the delayed delivery policy on a topic") + @Command(description = "Remove the delayed delivery policy on a topic", hidden = true) private class RemoveDelayedDelivery extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topic; @Override void run() throws PulsarAdminException { - String topicName = validateTopicName(params); + String topicName = validateTopicName(topic); getTopics().removeDelayedDeliveryPolicy(topicName); } } - @Parameters(commandDescription = "Get the message TTL for a topic") + @Command(description = "Get the message TTL for a topic", hidden = true) private class GetMessageTTL extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getMessageTTL(persistentTopic, applied)); } } - @Parameters(commandDescription = "Set message TTL for a topic") + @Command(description = "Set message TTL for a topic", hidden = true) private class SetMessageTTL extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-t", "--ttl" }, description = "Message TTL for topic in second " + @Option(names = { "-t", "--ttl" }, description = "Message TTL for topic in second " + "(or minutes, hours, days, weeks eg: 100m, 3h, 2d, 5w), " - + "allowed range from 1 to Integer.MAX_VALUE", required = true) - private String messageTTLStr; + + "allowed range from 1 to Integer.MAX_VALUE", required = true, + converter = TimeUnitToSecondsConverter.class) + private Long messageTTLInSecond; @Override void run() throws PulsarAdminException { - long messageTTLInSecond; - try { - messageTTLInSecond = RelativeTimeUtil.parseRelativeTimeInSeconds(messageTTLStr); - } catch (IllegalArgumentException e) { - throw new ParameterException(e.getMessage()); - } - - if (messageTTLInSecond < 0 || messageTTLInSecond > Integer.MAX_VALUE) { - throw new ParameterException( - String.format("Message TTL cannot be negative or greater than %d seconds", Integer.MAX_VALUE)); - } - - String persistentTopic = validatePersistentTopic(params); - getTopics().setMessageTTL(persistentTopic, (int) messageTTLInSecond); + String persistentTopic = validatePersistentTopic(topicName); + getTopics().setMessageTTL(persistentTopic, messageTTLInSecond.intValue()); } } - @Parameters(commandDescription = "Remove message TTL for a topic") + @Command(description = "Remove message TTL for a topic", hidden = true) private class RemoveMessageTTL extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removeMessageTTL(persistentTopic); } } - @Parameters(commandDescription = "Get deduplication snapshot interval for a topic") + @Command(description = "Get deduplication snapshot interval for a topic", hidden = true) private class GetDeduplicationSnapshotInterval extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getDeduplicationSnapshotInterval(persistentTopic)); } } - @Parameters(commandDescription = "Set deduplication snapshot interval for a topic") + @Command(description = "Set deduplication snapshot interval for a topic", hidden = true) private class SetDeduplicationSnapshotInterval extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-i", "--interval" }, description = "Deduplication snapshot interval for topic in second, " + @Option(names = { "-i", "--interval" }, description = "Deduplication snapshot interval for topic in second, " + "allowed range from 0 to Integer.MAX_VALUE", required = true) private int interval; @Override void run() throws PulsarAdminException { if (interval < 0) { - throw new ParameterException(String.format("Invalid interval '%d'. ", interval)); + throw new IllegalArgumentException(String.format("Invalid interval '%d'. ", interval)); } - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().setDeduplicationSnapshotInterval(persistentTopic, interval); } } - @Parameters(commandDescription = "Remove deduplication snapshot interval for a topic") + @Command(description = "Remove deduplication snapshot interval for a topic", hidden = true) private class RemoveDeduplicationSnapshotInterval extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removeDeduplicationSnapshotInterval(persistentTopic); } } - @Parameters(commandDescription = "Get the retention policy for a topic") + @Command(description = "Get the retention policy for a topic") private class GetRetention extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getRetention(persistentTopic, applied)); } } - @Parameters(commandDescription = "Set the retention policy for a topic") + @Command(description = "Set the retention policy for a topic", hidden = true) private class SetRetention extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--time", + @Option(names = { "--time", "-t" }, description = "Retention time with optional time unit suffix. " + "For example, 100m, 3h, 2d, 5w. " + "If the time unit is not specified, the default unit is seconds. For example, " + "-t 120 will set retention to 2 minutes. " - + "0 means no retention and -1 means infinite time retention.", required = true) - private String retentionTimeStr; + + "0 means no retention and -1 means infinite time retention.", required = true, + converter = TimeUnitToSecondsConverter.class) + private Integer retentionTimeInSec; - @Parameter(names = { "--size", "-s" }, description = "Retention size limit with optional size unit suffix. " + @Option(names = { "--size", "-s" }, description = "Retention size limit with optional size unit suffix. " + "For example, 4096, 10M, 16G, 3T. The size unit suffix character can be k/K, m/M, g/G, or t/T. " + "If the size unit suffix is not specified, the default unit is bytes. " - + "0 or less than 1MB means no retention and -1 means infinite size retention", required = true) - private String limitStr; + + "0 or less than 1MB means no retention and -1 means infinite size retention", required = true, + converter = ByteUnitToIntegerConverter.class) + private Integer sizeLimit; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); - long sizeLimit = validateSizeString(limitStr); - long retentionTimeInSec; - try { - retentionTimeInSec = RelativeTimeUtil.parseRelativeTimeInSeconds(retentionTimeStr); - } catch (IllegalArgumentException exception) { - throw new ParameterException(exception.getMessage()); - } - - final int retentionTimeInMin; - if (retentionTimeInSec != -1) { - retentionTimeInMin = (int) TimeUnit.SECONDS.toMinutes(retentionTimeInSec); - } else { - retentionTimeInMin = -1; - } - - final int retentionSizeInMB; - if (sizeLimit != -1) { - retentionSizeInMB = (int) (sizeLimit / (1024 * 1024)); - } else { - retentionSizeInMB = -1; - } + String persistentTopic = validatePersistentTopic(topicName); + final int retentionTimeInMin = retentionTimeInSec != -1 + ? (int) TimeUnit.SECONDS.toMinutes(retentionTimeInSec) + : retentionTimeInSec.intValue(); + final int retentionSizeInMB = sizeLimit != -1 + ? (int) (sizeLimit / (1024 * 1024)) + : sizeLimit; getTopics().setRetention(persistentTopic, new RetentionPolicies(retentionTimeInMin, retentionSizeInMB)); } } @Deprecated - @Parameters(commandDescription = "Enable the deduplication policy for a topic") + @Command(description = "Enable the deduplication policy for a topic", hidden = true) private class EnableDeduplication extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().enableDeduplication(persistentTopic, true); } } @Deprecated - @Parameters(commandDescription = "Disable the deduplication policy for a topic") + @Command(description = "Disable the deduplication policy for a topic", hidden = true) private class DisableDeduplication extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().enableDeduplication(persistentTopic, false); } } - @Parameters(commandDescription = "Enable or disable deduplication for a topic") + @Command(description = "Enable or disable deduplication for a topic", hidden = true) private class SetDeduplicationStatus extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--enable", "-e" }, description = "Enable deduplication") + @Option(names = { "--enable", "-e" }, description = "Enable deduplication") private boolean enable = false; - @Parameter(names = { "--disable", "-d" }, description = "Disable deduplication") + @Option(names = { "--disable", "-d" }, description = "Disable deduplication") private boolean disable = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); if (enable == disable) { - throw new ParameterException("Need to specify either --enable or --disable"); + throw new IllegalArgumentException("Need to specify either --enable or --disable"); } getTopics().setDeduplicationStatus(persistentTopic, enable); } } - @Parameters(commandDescription = "Get the deduplication policy for a topic") + @Command(description = "Get the deduplication policy for a topic", hidden = true) private class GetDeduplicationStatus extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getDeduplicationStatus(persistentTopic)); } } - @Parameters(commandDescription = "Remove the deduplication policy for a topic") + @Command(description = "Remove the deduplication policy for a topic", hidden = true) private class RemoveDeduplicationStatus extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removeDeduplicationStatus(persistentTopic); } } - @Parameters(commandDescription = "Remove the retention policy for a topic") + @Command(description = "Remove the retention policy for a topic", hidden = true) private class RemoveRetention extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removeRetention(persistentTopic); } } - @Parameters(commandDescription = "Get the persistence policies for a topic") + @Command(description = "Get the persistence policies for a topic", hidden = true) private class GetPersistence extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getPersistence(persistentTopic)); } } - @Parameters(commandDescription = "Get the offload policies for a topic") + @Command(description = "Get the offload policies for a topic", hidden = true) private class GetOffloadPolicies extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getOffloadPolicies(persistentTopic, applied)); } } - @Parameters(commandDescription = "Remove the offload policies for a topic") + @Command(description = "Remove the offload policies for a topic", hidden = true) private class RemoveOffloadPolicies extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removeOffloadPolicies(persistentTopic); } } - @Parameters(commandDescription = "Set the offload policies for a topic") + @Command(description = "Set the offload policies for a topic", hidden = true) private class SetOffloadPolicies extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"-d", "--driver"}, description = "ManagedLedger offload driver", required = true) + @Option(names = {"-d", "--driver"}, description = "ManagedLedger offload driver", required = true) private String driver; - @Parameter(names = {"-r", "--region"} + @Option(names = {"-r", "--region"} , description = "ManagedLedger offload region, s3 and google-cloud-storage requires this parameter") private String region; - @Parameter(names = {"-b", "--bucket"} + @Option(names = {"-b", "--bucket"} , description = "ManagedLedger offload bucket, s3 and google-cloud-storage requires this parameter") private String bucket; - @Parameter(names = {"-e", "--endpoint"} + @Option(names = {"-e", "--endpoint"} , description = "ManagedLedger offload service endpoint, only s3 requires this parameter") private String endpoint; - @Parameter(names = {"-i", "--aws-id"} + @Option(names = {"-i", "--aws-id"} , description = "AWS Credential Id to use when using driver S3 or aws-s3") private String awsId; - @Parameter(names = {"-s", "--aws-secret"} + @Option(names = {"-s", "--aws-secret"} , description = "AWS Credential Secret to use when using driver S3 or aws-s3") private String awsSecret; - @Parameter(names = {"--ro", "--s3-role"} + @Option(names = {"--ro", "--s3-role"} , description = "S3 Role used for STSAssumeRoleSessionCredentialsProvider") private String s3Role; - @Parameter(names = {"--s3-role-session-name", "-rsn"} + @Option(names = {"--s3-role-session-name", "-rsn"} , description = "S3 role session name used for STSAssumeRoleSessionCredentialsProvider") private String s3RoleSessionName; - @Parameter(names = {"-m", "--maxBlockSizeInBytes"}, - description = "ManagedLedger offload max block Size in bytes," - + "s3 and google-cloud-storage requires this parameter") - private int maxBlockSizeInBytes; - - @Parameter(names = {"-rb", "--readBufferSizeInBytes"}, - description = "ManagedLedger offload read buffer size in bytes," - + "s3 and google-cloud-storage requires this parameter") - private int readBufferSizeInBytes; - - @Parameter(names = {"-t", "--offloadThresholdInBytes"} - , description = "ManagedLedger offload threshold in bytes", required = true) - private long offloadThresholdInBytes; - - @Parameter(names = {"-ts", "--offloadThresholdInSeconds"} - , description = "ManagedLedger offload threshold in seconds") - private Long offloadThresholdInSeconds; - - @Parameter(names = {"-dl", "--offloadDeletionLagInMillis"} - , description = "ManagedLedger offload deletion lag in bytes") - private Long offloadDeletionLagInMillis; - - @Parameter(names = {"--offloadedReadPriority", "-orp"}, + @Option( + names = {"-m", "--maxBlockSizeInBytes", "--maxBlockSize", "-mbs"}, + description = "Max block size (eg: 32M, 64M), default is 64MB" + + "s3 and google-cloud-storage requires this parameter", + required = false, + converter = ByteUnitToIntegerConverter.class) + private Integer maxBlockSizeInBytes = OffloadPoliciesImpl.DEFAULT_MAX_BLOCK_SIZE_IN_BYTES; + + @Option( + names = {"-rb", "--readBufferSizeInBytes", "--readBufferSize", "-rbs"}, + description = "Read buffer size (eg: 1M, 5M), default is 1MB" + + "s3 and google-cloud-storage requires this parameter", + required = false, + converter = ByteUnitToIntegerConverter.class) + private Integer readBufferSizeInBytes = OffloadPoliciesImpl.DEFAULT_READ_BUFFER_SIZE_IN_BYTES; + + @Option(names = {"-t", "--offloadThresholdInBytes", "--offloadAfterThreshold", "-oat"} + , description = "Offload after threshold size (eg: 1M, 5M)", required = false, + converter = ByteUnitToLongConverter.class) + private Long offloadAfterThresholdInBytes = OffloadPoliciesImpl.DEFAULT_OFFLOAD_THRESHOLD_IN_BYTES; + + @Option(names = {"-ts", "--offloadThresholdInSeconds", "--offloadAfterThresholdInSeconds", "-oats"}, + description = "Offload after threshold seconds (or minutes,hours,days,weeks eg: 100m, 3h, 2d, 5w).", + converter = TimeUnitToSecondsConverter.class) + private Long offloadThresholdInSeconds = OffloadPoliciesImpl.DEFAULT_OFFLOAD_THRESHOLD_IN_SECONDS; + + @Option(names = {"-dl", "--offloadDeletionLagInMillis", "--offloadAfterElapsed", "-oae"} + , description = "Delay time in Millis for deleting the bookkeeper ledger after offload " + + "(or seconds,minutes,hours,days,weeks eg: 10s, 100m, 3h, 2d, 5w).", + converter = TimeUnitToMillisConverter.class) + private Long offloadAfterElapsedInMillis = OffloadPoliciesImpl.DEFAULT_OFFLOAD_DELETION_LAG_IN_MILLIS; + + @Option(names = {"--offloadedReadPriority", "-orp"}, description = "Read priority for offloaded messages. " + "By default, once messages are offloaded to long-term storage, " + "brokers read messages from long-term storage, but messages can still exist in BookKeeper " @@ -2175,12 +2075,35 @@ private class SetOffloadPolicies extends CliCommand { ) private String offloadReadPriorityStr; + public final List driverNames = OffloadPoliciesImpl.DRIVER_NAMES; + + public boolean driverSupported(String driver) { + return driverNames.stream().anyMatch(d -> d.equalsIgnoreCase(driver)); + } + + public boolean isS3Driver(String driver) { + if (StringUtils.isEmpty(driver)) { + return false; + } + return driver.equalsIgnoreCase(driverNames.get(0)) || driver.equalsIgnoreCase(driverNames.get(1)); + } + @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); - OffloadedReadPriority offloadedReadPriority = OffloadPoliciesImpl.DEFAULT_OFFLOADED_READ_PRIORITY; + if (!driverSupported(driver)) { + throw new ParameterException( + "The driver " + driver + " is not supported, " + + "(Possible values: " + String.join(",", driverNames) + ")."); + } + if (isS3Driver(driver) && Strings.isNullOrEmpty(region) && Strings.isNullOrEmpty(endpoint)) { + throw new ParameterException( + "Either s3ManagedLedgerOffloadRegion or s3ManagedLedgerOffloadServiceEndpoint must be set" + + " if s3 offload enabled"); + } + OffloadedReadPriority offloadedReadPriority = OffloadPoliciesImpl.DEFAULT_OFFLOADED_READ_PRIORITY; if (this.offloadReadPriorityStr != null) { try { offloadedReadPriority = OffloadedReadPriority.fromString(this.offloadReadPriorityStr); @@ -2193,42 +2116,41 @@ void run() throws PulsarAdminException { } } - OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.create(driver, region, bucket, endpoint, + OffloadPolicies offloadPolicies = OffloadPoliciesImpl.create(driver, region, bucket, endpoint, s3Role, s3RoleSessionName, awsId, awsSecret, - maxBlockSizeInBytes, - readBufferSizeInBytes, offloadThresholdInBytes, offloadThresholdInSeconds, - offloadDeletionLagInMillis, offloadedReadPriority); + maxBlockSizeInBytes, readBufferSizeInBytes, offloadAfterThresholdInBytes, + offloadThresholdInSeconds, offloadAfterElapsedInMillis, offloadedReadPriority); getTopics().setOffloadPolicies(persistentTopic, offloadPolicies); } } - @Parameters(commandDescription = "Set the persistence policies for a topic") + @Command(description = "Set the persistence policies for a topic", hidden = true) private class SetPersistence extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-e", + @Option(names = { "-e", "--bookkeeper-ensemble" }, description = "Number of bookies to use for a topic") private int bookkeeperEnsemble = 2; - @Parameter(names = { "-w", + @Option(names = { "-w", "--bookkeeper-write-quorum" }, description = "How many writes to make of each entry") private int bookkeeperWriteQuorum = 2; - @Parameter(names = { "-a", + @Option(names = { "-a", "--bookkeeper-ack-quorum" }, description = "Number of acks (guaranteed copies) to wait for each entry") private int bookkeeperAckQuorum = 2; - @Parameter(names = { "-r", + @Option(names = { "-r", "--ml-mark-delete-max-rate" }, description = "Throttling rate of mark-delete operation " + "(0 means no throttle)") private double managedLedgerMaxMarkDeleteRate = 0; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); if (bookkeeperEnsemble <= 0 || bookkeeperWriteQuorum <= 0 || bookkeeperAckQuorum <= 0) { throw new ParameterException("[--bookkeeper-ensemble], [--bookkeeper-write-quorum] " + "and [--bookkeeper-ack-quorum] must greater than 0."); @@ -2241,59 +2163,59 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Remove the persistence policy for a topic") + @Command(description = "Remove the persistence policy for a topic", hidden = true) private class RemovePersistence extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removePersistence(persistentTopic); } } - @Parameters(commandDescription = "Get message dispatch rate for a topic") + @Command(description = "Get message dispatch rate for a topic", hidden = true) private class GetDispatchRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getDispatchRate(persistentTopic, applied)); } } - @Parameters(commandDescription = "Set message dispatch rate for a topic") + @Command(description = "Set message dispatch rate for a topic", hidden = true) private class SetDispatchRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--msg-dispatch-rate", + @Option(names = { "--msg-dispatch-rate", "-md" }, description = "message-dispatch-rate (default -1 will be overwrite if not passed)") private int msgDispatchRate = -1; - @Parameter(names = { "--byte-dispatch-rate", + @Option(names = { "--byte-dispatch-rate", "-bd" }, description = "byte-dispatch-rate (default -1 will be overwrite if not passed)") private long byteDispatchRate = -1; - @Parameter(names = { "--dispatch-rate-period", + @Option(names = { "--dispatch-rate-period", "-dt" }, description = "dispatch-rate-period in second type " + "(default 1 second will be overwrite if not passed)", required = false) private int dispatchRatePeriodSec = 1; - @Parameter(names = { "--relative-to-publish-rate", + @Option(names = { "--relative-to-publish-rate", "-rp" }, description = "dispatch rate relative to publish-rate (if publish-relative flag is enabled " + "then broker will apply throttling value to (publish-rate + dispatch rate))", required = false) private boolean relativeToPublishRate = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().setDispatchRate(persistentTopic, DispatchRate.builder() .dispatchThrottlingRateInMsg(msgDispatchRate) @@ -2304,115 +2226,115 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Remove message dispatch rate for a topic") + @Command(description = "Remove message dispatch rate for a topic", hidden = true) private class RemoveDispatchRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removeDispatchRate(persistentTopic); } } - @Parameters(commandDescription = "Get max unacked messages policy on consumer for a topic") + @Command(description = "Get max unacked messages policy on consumer for a topic", hidden = true) private class GetMaxUnackedMessagesOnConsumer extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getMaxUnackedMessagesOnConsumer(persistentTopic, applied)); } } - @Parameters(commandDescription = "Remove max unacked messages policy on consumer for a topic") + @Command(description = "Remove max unacked messages policy on consumer for a topic", hidden = true) private class RemoveMaxUnackedMessagesOnConsumer extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removeMaxUnackedMessagesOnConsumer(persistentTopic); } } - @Parameters(commandDescription = "Set max unacked messages policy on consumer for a topic") + @Command(description = "Set max unacked messages policy on consumer for a topic", hidden = true) private class SetMaxUnackedMessagesOnConsumer extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"-m", "--maxNum"}, description = "max unacked messages num on consumer", required = true) + @Option(names = {"-m", "--maxNum"}, description = "max unacked messages num on consumer", required = true) private int maxNum; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().setMaxUnackedMessagesOnConsumer(persistentTopic, maxNum); } } - @Parameters(commandDescription = "Get max unacked messages policy on subscription for a topic") + @Command(description = "Get max unacked messages policy on subscription for a topic", hidden = true) private class GetMaxUnackedMessagesOnSubscription extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getMaxUnackedMessagesOnSubscription(persistentTopic, applied)); } } - @Parameters(commandDescription = "Remove max unacked messages policy on subscription for a topic") + @Command(description = "Remove max unacked messages policy on subscription for a topic", hidden = true) private class RemoveMaxUnackedMessagesOnSubscription extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removeMaxUnackedMessagesOnSubscription(persistentTopic); } } - @Parameters(commandDescription = "Set max unacked messages policy on subscription for a topic") + @Command(description = "Set max unacked messages policy on subscription for a topic", hidden = true) private class SetMaxUnackedMessagesOnSubscription extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"-m", "--maxNum"}, + @Option(names = {"-m", "--maxNum"}, description = "max unacked messages num on subscription", required = true) private int maxNum; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().setMaxUnackedMessagesOnSubscription(persistentTopic, maxNum); } } - @Parameters(commandDescription = "Set subscription types enabled for a topic") + @Command(description = "Set subscription types enabled for a topic", hidden = true) private class SetSubscriptionTypesEnabled extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--types", "-t"}, description = "Subscription types enabled list (comma separated values)." - + " Possible values: (Exclusive, Shared, Failover, Key_Shared).", required = true) + @Option(names = {"--types", "-t"}, description = "Subscription types enabled list (comma separated values)." + + " Possible values: (Exclusive, Shared, Failover, Key_Shared).", required = true, split = ",") private List subTypes; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); Set types = new HashSet<>(); subTypes.forEach(s -> { SubscriptionType subType; @@ -2428,162 +2350,162 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get subscription types enabled for a topic") + @Command(description = "Get subscription types enabled for a topic", hidden = true) private class GetSubscriptionTypesEnabled extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getSubscriptionTypesEnabled(persistentTopic)); } } - @Parameters(commandDescription = "Remove subscription types enabled for a topic") + @Command(description = "Remove subscription types enabled for a topic", hidden = true) private class RemoveSubscriptionTypesEnabled extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removeSubscriptionTypesEnabled(persistentTopic); } } - @Parameters(commandDescription = "Get compaction threshold for a topic") + @Command(description = "Get compaction threshold for a topic", hidden = true) private class GetCompactionThreshold extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getCompactionThreshold(persistentTopic, applied)); } } - @Parameters(commandDescription = "Set compaction threshold for a topic") + @Command(description = "Set compaction threshold for a topic", hidden = true) private class SetCompactionThreshold extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--threshold", "-t" }, + @Option(names = { "--threshold", "-t" }, description = "Maximum number of bytes in a topic backlog before compaction is triggered " + "(eg: 10M, 16G, 3T). 0 disables automatic compaction", - required = true) - private String thresholdStr = "0"; + required = true, + converter = ByteUnitToLongConverter.class) + private Long threshold = 0L; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); - long threshold = validateSizeString(thresholdStr); + String persistentTopic = validatePersistentTopic(topicName); getTopics().setCompactionThreshold(persistentTopic, threshold); } } - @Parameters(commandDescription = "Remove compaction threshold for a topic") + @Command(description = "Remove compaction threshold for a topic", hidden = true) private class RemoveCompactionThreshold extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removeCompactionThreshold(persistentTopic); } } - @Parameters(commandDescription = "Get publish rate for a topic") + @Command(description = "Get publish rate for a topic", hidden = true) private class GetPublishRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getPublishRate(persistentTopic)); } } - @Parameters(commandDescription = "Set publish rate for a topic") + @Command(description = "Set publish rate for a topic", hidden = true) private class SetPublishRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--msg-publish-rate", + @Option(names = { "--msg-publish-rate", "-m" }, description = "message-publish-rate (default -1 will be overwrite if not passed)", required = false) private int msgPublishRate = -1; - @Parameter(names = { "--byte-publish-rate", + @Option(names = { "--byte-publish-rate", "-b" }, description = "byte-publish-rate (default -1 will be overwrite if not passed)", required = false) private long bytePublishRate = -1; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().setPublishRate(persistentTopic, new PublishRate(msgPublishRate, bytePublishRate)); } } - @Parameters(commandDescription = "Remove publish rate for a topic") + @Command(description = "Remove publish rate for a topic", hidden = true) private class RemovePublishRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removePublishRate(persistentTopic); } } - @Parameters(commandDescription = "Get subscription message-dispatch-rate for a topic") + @Command(description = "Get subscription message-dispatch-rate for a topic", hidden = true) private class GetSubscriptionDispatchRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getSubscriptionDispatchRate(persistentTopic, applied)); } } - @Parameters(commandDescription = "Set subscription message-dispatch-rate for a topic") + @Command(description = "Set subscription message-dispatch-rate for a topic", hidden = true) private class SetSubscriptionDispatchRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--msg-dispatch-rate", + @Option(names = { "--msg-dispatch-rate", "-md" }, description = "message-dispatch-rate (default -1 will be overwrite if not passed)") private int msgDispatchRate = -1; - @Parameter(names = { "--byte-dispatch-rate", + @Option(names = { "--byte-dispatch-rate", "-bd" }, description = "byte-dispatch-rate (default -1 will be overwrite if not passed)", required = false) private long byteDispatchRate = -1; - @Parameter(names = { "--dispatch-rate-period", + @Option(names = { "--dispatch-rate-period", "-dt" }, description = "dispatch-rate-period in second type" + " (default 1 second will be overwrite if not passed)") private int dispatchRatePeriodSec = 1; - @Parameter(names = { "--relative-to-publish-rate", + @Option(names = { "--relative-to-publish-rate", "-rp" }, description = "dispatch rate relative to publish-rate (if publish-relative flag is enabled " + "then broker will apply throttling value to (publish-rate + dispatch rate))") private boolean relativeToPublishRate = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().setSubscriptionDispatchRate(persistentTopic, DispatchRate.builder() .dispatchThrottlingRateInMsg(msgDispatchRate) @@ -2594,59 +2516,59 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Remove subscription message-dispatch-rate for a topic") + @Command(description = "Remove subscription message-dispatch-rate for a topic", hidden = true) private class RemoveSubscriptionDispatchRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removeSubscriptionDispatchRate(persistentTopic); } } - @Parameters(commandDescription = "Get replicator message-dispatch-rate for a topic") + @Command(description = "Get replicator message-dispatch-rate for a topic", hidden = true) private class GetReplicatorDispatchRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"-ap", "--applied"}, description = "Get the applied policy of the topic") + @Option(names = {"-ap", "--applied"}, description = "Get the applied policy of the topic") private boolean applied = false; @Override void run() throws PulsarAdminException { - String topic = validatePersistentTopic(params); + String topic = validatePersistentTopic(topicName); print(getTopics().getReplicatorDispatchRate(topic, applied)); } } - @Parameters(commandDescription = "Set replicator message-dispatch-rate for a topic") + @Command(description = "Set replicator message-dispatch-rate for a topic", hidden = true) private class SetReplicatorDispatchRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--msg-dispatch-rate", + @Option(names = { "--msg-dispatch-rate", "-md" }, description = "message-dispatch-rate (default -1 will be overwrite if not passed)") private int msgDispatchRate = -1; - @Parameter(names = { "--byte-dispatch-rate", + @Option(names = { "--byte-dispatch-rate", "-bd" }, description = "byte-dispatch-rate (default -1 will be overwrite if not passed)", required = false) private long byteDispatchRate = -1; - @Parameter(names = { "--dispatch-rate-period", + @Option(names = { "--dispatch-rate-period", "-dt" }, description = "dispatch-rate-period in second type " + "(default 1 second will be overwrite if not passed)") private int dispatchRatePeriodSec = 1; - @Parameter(names = { "--relative-to-publish-rate", + @Option(names = { "--relative-to-publish-rate", "-rp" }, description = "dispatch rate relative to publish-rate (if publish-relative flag is enabled " + "then broker will apply throttling value to (publish-rate + dispatch rate))") private boolean relativeToPublishRate = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().setReplicatorDispatchRate(persistentTopic, DispatchRate.builder() .dispatchThrottlingRateInMsg(msgDispatchRate) @@ -2657,227 +2579,220 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Remove replicator message-dispatch-rate for a topic") + @Command(description = "Remove replicator message-dispatch-rate for a topic", hidden = true) private class RemoveReplicatorDispatchRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removeReplicatorDispatchRate(persistentTopic); } } - @Parameters(commandDescription = "Get max number of producers for a topic") + @Command(description = "Get max number of producers for a topic", hidden = true) private class GetMaxProducers extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getMaxProducers(persistentTopic, applied)); } } - @Parameters(commandDescription = "Set max number of producers for a topic") + @Command(description = "Set max number of producers for a topic", hidden = true) private class SetMaxProducers extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--max-producers", "-p"}, description = "Max producers for a topic", required = true) + @Option(names = {"--max-producers", "-p"}, description = "Max producers for a topic", required = true) private int maxProducers; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().setMaxProducers(persistentTopic, maxProducers); } } - @Parameters(commandDescription = "Remove max number of producers for a topic") + @Command(description = "Remove max number of producers for a topic", hidden = true) private class RemoveMaxProducers extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removeMaxProducers(persistentTopic); } } - @Parameters(commandDescription = "Get max number of subscriptions for a topic") + @Command(description = "Get max number of subscriptions for a topic", hidden = true) private class GetMaxSubscriptionsPerTopic extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getMaxSubscriptionsPerTopic(persistentTopic)); } } - @Parameters(commandDescription = "Set max number of subscriptions for a topic") + @Command(description = "Set max number of subscriptions for a topic", hidden = true) private class SetMaxSubscriptionsPerTopic extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--max-subscriptions-per-topic", "-m"}, + @Option(names = {"--max-subscriptions-per-topic", "-m"}, description = "Maximum subscription limit for a topic", required = true) private int maxSubscriptionsPerTopic; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().setMaxSubscriptionsPerTopic(persistentTopic, maxSubscriptionsPerTopic); } } - @Parameters(commandDescription = "Remove max number of subscriptions for a topic") + @Command(description = "Remove max number of subscriptions for a topic", hidden = true) private class RemoveMaxSubscriptionsPerTopic extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removeMaxSubscriptionsPerTopic(persistentTopic); } } - @Parameters(commandDescription = "Get max message size for a topic") + @Command(description = "Get max message size for a topic", hidden = true) private class GetMaxMessageSize extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getMaxMessageSize(persistentTopic)); } } - @Parameters(commandDescription = "Set max message size for a topic") + @Command(description = "Set max message size for a topic", hidden = true) private class SetMaxMessageSize extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"--max-message-size", "-m"}, description = "Max message size for a topic", required = true) + @Option(names = {"--max-message-size", "-m"}, description = "Max message size for a topic", required = true) private int maxMessageSize; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().setMaxMessageSize(persistentTopic, maxMessageSize); } } - @Parameters(commandDescription = "Remove max message size for a topic") + @Command(description = "Remove max message size for a topic", hidden = true) private class RemoveMaxMessageSize extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removeMaxMessageSize(persistentTopic); } } - @Parameters(commandDescription = "Get max consumers per subscription for a topic") + @Command(description = "Get max consumers per subscription for a topic", hidden = true) private class GetMaxConsumersPerSubscription extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getMaxConsumersPerSubscription(persistentTopic)); } } - @Parameters(commandDescription = "Set max consumers per subscription for a topic") + @Command(description = "Set max consumers per subscription for a topic", hidden = true) private class SetMaxConsumersPerSubscription extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--max-consumers-per-subscription", "-c" }, + @Option(names = { "--max-consumers-per-subscription", "-c" }, description = "maxConsumersPerSubscription for a namespace", required = true) private int maxConsumersPerSubscription; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().setMaxConsumersPerSubscription(persistentTopic, maxConsumersPerSubscription); } } - @Parameters(commandDescription = "Remove max consumers per subscription for a topic") + @Command(description = "Remove max consumers per subscription for a topic", hidden = true) private class RemoveMaxConsumersPerSubscription extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removeMaxConsumersPerSubscription(persistentTopic); } } - @Parameters(commandDescription = "Get the inactive topic policies on a topic") + @Command(description = "Get the inactive topic policies on a topic", hidden = true) private class GetInactiveTopicPolicies extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getInactiveTopicPolicies(persistentTopic, applied)); } } - @Parameters(commandDescription = "Set the inactive topic policies on a topic") + @Command(description = "Set the inactive topic policies on a topic", hidden = true) private class SetInactiveTopicPolicies extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--enable-delete-while-inactive", "-e" }, description = "Enable delete while inactive") + @Option(names = { "--enable-delete-while-inactive", "-e" }, description = "Enable delete while inactive") private boolean enableDeleteWhileInactive = false; - @Parameter(names = { "--disable-delete-while-inactive", "-d" }, description = "Disable delete while inactive") + @Option(names = { "--disable-delete-while-inactive", "-d" }, description = "Disable delete while inactive") private boolean disableDeleteWhileInactive = false; - @Parameter(names = {"--max-inactive-duration", "-t"}, description = "Max duration of topic inactivity " + @Option(names = {"--max-inactive-duration", "-t"}, description = "Max duration of topic inactivity " + "in seconds, topics that are inactive for longer than this value will be deleted " - + "(eg: 1s, 10s, 1m, 5h, 3d)", required = true) - private String deleteInactiveTopicsMaxInactiveDuration; + + "(eg: 1s, 10s, 1m, 5h, 3d)", required = true, + converter = TimeUnitToSecondsConverter.class) + private Long maxInactiveDurationInSeconds; - @Parameter(names = { "--delete-mode", "-m" }, description = "Mode of delete inactive topic, Valid options are: " + @Option(names = { "--delete-mode", "-m" }, description = "Mode of delete inactive topic, Valid options are: " + "[delete_when_no_subscriptions, delete_when_subscriptions_caught_up]", required = true) private String inactiveTopicDeleteMode; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); - long maxInactiveDurationInSeconds; - try { - maxInactiveDurationInSeconds = TimeUnit.SECONDS.toSeconds( - RelativeTimeUtil.parseRelativeTimeInSeconds(deleteInactiveTopicsMaxInactiveDuration)); - } catch (IllegalArgumentException exception) { - throw new ParameterException(exception.getMessage()); - } - + String persistentTopic = validatePersistentTopic(topicName); if (enableDeleteWhileInactive == disableDeleteWhileInactive) { - throw new ParameterException("Need to specify either enable-delete-while-inactive " + throw new IllegalArgumentException("Need to specify either enable-delete-while-inactive " + "or disable-delete-while-inactive"); } InactiveTopicDeleteMode deleteMode = null; @@ -2888,150 +2803,150 @@ void run() throws PulsarAdminException { + "or delete_when_subscriptions_caught_up"); } getTopics().setInactiveTopicPolicies(persistentTopic, new InactiveTopicPolicies(deleteMode, - (int) maxInactiveDurationInSeconds, enableDeleteWhileInactive)); + maxInactiveDurationInSeconds.intValue(), enableDeleteWhileInactive)); } } - @Parameters(commandDescription = "Remove inactive topic policies from a topic") + @Command(description = "Remove inactive topic policies from a topic", hidden = true) private class RemoveInactiveTopicPolicies extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removeInactiveTopicPolicies(persistentTopic); } } - @Parameters(commandDescription = "Get max number of consumers for a topic") + @Command(description = "Get max number of consumers for a topic", hidden = true) private class GetMaxConsumers extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getMaxConsumers(persistentTopic, applied)); } } - @Parameters(commandDescription = "Set max number of consumers for a topic") + @Command(description = "Set max number of consumers for a topic", hidden = true) private class SetMaxConsumers extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--max-consumers", "-c" }, description = "Max consumers for a topic", required = true) + @Option(names = { "--max-consumers", "-c" }, description = "Max consumers for a topic", required = true) private int maxConsumers; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().setMaxConsumers(persistentTopic, maxConsumers); } } - @Parameters(commandDescription = "Remove max number of consumers for a topic") + @Command(description = "Remove max number of consumers for a topic", hidden = true) private class RemoveMaxConsumers extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removeMaxConsumers(persistentTopic); } } - @Parameters(commandDescription = "Get consumer subscribe rate for a topic") + @Command(description = "Get consumer subscribe rate for a topic", hidden = true) private class GetSubscribeRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getSubscribeRate(persistentTopic, applied)); } } - @Parameters(commandDescription = "Set consumer subscribe rate for a topic") + @Command(description = "Set consumer subscribe rate for a topic", hidden = true) private class SetSubscribeRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--subscribe-rate", + @Option(names = { "--subscribe-rate", "-sr" }, description = "subscribe-rate (default -1 will be overwrite if not passed)", required = false) private int subscribeRate = -1; - @Parameter(names = { "--subscribe-rate-period", + @Option(names = { "--subscribe-rate-period", "-st" }, description = "subscribe-rate-period in second type " + "(default 30 second will be overwrite if not passed)") private int subscribeRatePeriodSec = 30; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().setSubscribeRate(persistentTopic, new SubscribeRate(subscribeRate, subscribeRatePeriodSec)); } } - @Parameters(commandDescription = "Remove consumer subscribe rate for a topic") + @Command(description = "Remove consumer subscribe rate for a topic", hidden = true) private class RemoveSubscribeRate extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); getTopics().removeSubscribeRate(persistentTopic); } } - @Parameters(commandDescription = "Enable or disable a replicated subscription on a topic") + @Command(description = "Enable or disable a replicated subscription on a topic") private class SetReplicatedSubscriptionStatus extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-s", + @Option(names = { "-s", "--subscription" }, description = "Subscription name to enable or disable replication", required = true) private String subName; - @Parameter(names = { "--enable", "-e" }, description = "Enable replication") + @Option(names = { "--enable", "-e" }, description = "Enable replication") private boolean enable = false; - @Parameter(names = { "--disable", "-d" }, description = "Disable replication") + @Option(names = { "--disable", "-d" }, description = "Disable replication") private boolean disable = false; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); if (enable == disable) { - throw new ParameterException("Need to specify either --enable or --disable"); + throw new IllegalArgumentException("Need to specify either --enable or --disable"); } getTopics().setReplicatedSubscriptionStatus(persistentTopic, subName, enable); } } - @Parameters(commandDescription = "Get replicated subscription status on a topic") + @Command(description = "Get replicated subscription status on a topic") private class GetReplicatedSubscriptionStatus extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = {"-s", + @Option(names = {"-s", "--subscription"}, description = "Subscription name", required = true) private String subName; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); print(getTopics().getReplicatedSubscriptionStatus(persistentTopic, subName)); } } @@ -3040,18 +2955,18 @@ private Topics getTopics() { return getAdmin().topics(); } - @Parameters(commandDescription = "Calculate backlog size by a message ID (in bytes).") + @Command(description = "Calculate backlog size by a message ID (in bytes).") private class GetBacklogSizeByMessageId extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--messageId", + @Option(names = { "--messageId", "-m" }, description = "messageId used to calculate backlog size. It can be (ledgerId:entryId).") private String messagePosition = "-1:-1"; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); MessageId messageId; if ("-1:-1".equals(messagePosition)) { messageId = MessageId.earliest; @@ -3064,21 +2979,21 @@ void run() throws PulsarAdminException { } - @Parameters(commandDescription = "Analyze the backlog of a subscription.") + @Command(description = "Analyze the backlog of a subscription.") private class AnalyzeBacklog extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-s", "--subscription" }, description = "Subscription to be analyzed", required = true) + @Option(names = { "-s", "--subscription" }, description = "Subscription to be analyzed", required = true) private String subName; - @Parameter(names = { "--position", + @Option(names = { "--position", "-p" }, description = "message position to start the scan from (ledgerId:entryId)", required = false) private String messagePosition; @Override void run() throws PulsarAdminException { - String persistentTopic = validatePersistentTopic(params); + String persistentTopic = validatePersistentTopic(topicName); Optional startPosition = Optional.empty(); if (isNotBlank(messagePosition)) { int partitionIndex = TopicName.get(persistentTopic).getPartitionIndex(); @@ -3090,43 +3005,44 @@ void run() throws PulsarAdminException { } } - @Parameters(commandDescription = "Get the schema validation enforced") + @Command(description = "Get the schema validation enforced") private class GetSchemaValidationEnforced extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") + @Option(names = { "-ap", "--applied" }, description = "Get the applied policy of the topic") private boolean applied = false; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); System.out.println(getAdmin().topics().getSchemaValidationEnforced(topic, applied)); } } - @Parameters(commandDescription = "Set the schema whether open schema validation enforced") + @Command(description = "Set the schema whether open schema validation enforced") private class SetSchemaValidationEnforced extends CliCommand { - @Parameter(description = "tenant/namespace", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; - @Parameter(names = { "--enable", "-e" }, description = "Enable schema validation enforced") + @Option(names = { "--enable", "-e" }, description = "Enable schema validation enforced") private boolean enable = false; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); getAdmin().topics().setSchemaValidationEnforced(topic, enable); } } - @Parameters(commandDescription = "Trim a topic") + + @Command(description = "Trim a topic") private class TrimTopic extends CliCommand { - @Parameter(description = "persistent://tenant/namespace/topic", required = true) - private java.util.List params; + @Parameters(description = "persistent://tenant/namespace/topic", arity = "1") + private String topicName; @Override void run() throws PulsarAdminException { - String topic = validateTopicName(params); + String topic = validateTopicName(topicName); getAdmin().topics().trimTopic(topic); } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTransactions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTransactions.java index 08ffba1451f23..63c729263cbe6 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTransactions.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTransactions.java @@ -18,23 +18,22 @@ */ package org.apache.pulsar.admin.cli; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import java.util.stream.Collectors; +import org.apache.pulsar.cli.converters.picocli.TimeUnitToMillisConverter; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.policies.data.TransactionCoordinatorInfo; -import org.apache.pulsar.common.util.RelativeTimeUtil; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; -@Parameters(commandDescription = "Operations on transactions") +@Command(description = "Operations on transactions") public class CmdTransactions extends CmdBase { - @Parameters(commandDescription = "Get transaction coordinator stats") + @Command(description = "Get transaction coordinator stats") private class GetCoordinatorStats extends CliCommand { - @Parameter(names = {"-c", "--coordinator-id"}, description = "The coordinator id", required = false) + @Option(names = {"-c", "--coordinator-id"}, description = "The coordinator id", required = false) private Integer coordinatorId; @Override @@ -47,30 +46,35 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Get transaction buffer stats") + @Command(description = "Get transaction buffer stats") private class GetTransactionBufferStats extends CliCommand { - @Parameter(names = {"-t", "--topic"}, description = "The topic", required = true) + @Option(names = {"-t", "--topic"}, description = "The topic", required = true) private String topic; - @Parameter(names = {"-l", "--low-water-mark"}, + @Option(names = {"-l", "--low-water-mark"}, description = "Whether to get information about lowWaterMarks stored in transaction buffer.") private boolean lowWaterMark; + @Option(names = {"-s", "--segment-stats"}, + description = "Whether to get segment statistics.") + private boolean segmentStats = false; + @Override void run() throws Exception { - print(getAdmin().transactions().getTransactionBufferStats(topic, lowWaterMark)); + // Assuming getTransactionBufferStats method signature has been updated to accept the new parameter + print(getAdmin().transactions().getTransactionBufferStats(topic, lowWaterMark, segmentStats)); } } - @Parameters(commandDescription = "Get transaction pending ack stats") + @Command(description = "Get transaction pending ack stats") private class GetPendingAckStats extends CliCommand { - @Parameter(names = {"-t", "--topic"}, description = "The topic name", required = true) + @Option(names = {"-t", "--topic"}, description = "The topic name", required = true) private String topic; - @Parameter(names = {"-s", "--sub-name"}, description = "The subscription name", required = true) + @Option(names = {"-s", "--sub-name"}, description = "The subscription name", required = true) private String subName; - @Parameter(names = {"-l", "--low-water-mark"}, + @Option(names = {"-l", "--low-water-mark"}, description = "Whether to get information about lowWaterMarks stored in transaction pending ack.") private boolean lowWaterMarks; @@ -80,18 +84,18 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Get transaction in pending ack stats") + @Command(description = "Get transaction in pending ack stats") private class GetTransactionInPendingAckStats extends CliCommand { - @Parameter(names = {"-m", "--most-sig-bits"}, description = "The most sig bits", required = true) + @Option(names = {"-m", "--most-sig-bits"}, description = "The most sig bits", required = true) private int mostSigBits; - @Parameter(names = {"-l", "--least-sig-bits"}, description = "The least sig bits", required = true) + @Option(names = {"-l", "--least-sig-bits"}, description = "The least sig bits", required = true) private long leastSigBits; - @Parameter(names = {"-t", "--topic"}, description = "The topic name", required = true) + @Option(names = {"-t", "--topic"}, description = "The topic name", required = true) private String topic; - @Parameter(names = {"-s", "--sub-name"}, description = "The subscription name", required = true) + @Option(names = {"-s", "--sub-name"}, description = "The subscription name", required = true) private String subName; @Override @@ -102,15 +106,15 @@ void run() throws Exception { } - @Parameters(commandDescription = "Get transaction in buffer stats") + @Command(description = "Get transaction in buffer stats") private class GetTransactionInBufferStats extends CliCommand { - @Parameter(names = {"-m", "--most-sig-bits"}, description = "The most sig bits", required = true) + @Option(names = {"-m", "--most-sig-bits"}, description = "The most sig bits", required = true) private int mostSigBits; - @Parameter(names = {"-l", "--least-sig-bits"}, description = "The least sig bits", required = true) + @Option(names = {"-l", "--least-sig-bits"}, description = "The least sig bits", required = true) private long leastSigBits; - @Parameter(names = {"-t", "--topic"}, description = "The topic name", required = true) + @Option(names = {"-t", "--topic"}, description = "The topic name", required = true) private String topic; @Override @@ -119,12 +123,12 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Get transaction metadata") + @Command(description = "Get transaction metadata") private class GetTransactionMetadata extends CliCommand { - @Parameter(names = {"-m", "--most-sig-bits"}, description = "The most sig bits", required = true) + @Option(names = {"-m", "--most-sig-bits"}, description = "The most sig bits", required = true) private int mostSigBits; - @Parameter(names = {"-l", "--least-sig-bits"}, description = "The least sig bits", required = true) + @Option(names = {"-l", "--least-sig-bits"}, description = "The least sig bits", required = true) private long leastSigBits; @Override @@ -133,38 +137,33 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Get slow transactions.") + @Command(description = "Get slow transactions.") private class GetSlowTransactions extends CliCommand { - @Parameter(names = {"-c", "--coordinator-id"}, description = "The coordinator id", required = false) + @Option(names = {"-c", "--coordinator-id"}, description = "The coordinator id", required = false) private Integer coordinatorId; - @Parameter(names = { "-t", "--time" }, description = "The transaction timeout time. " - + "(eg: 1s, 10s, 1m, 5h, 3d)", required = true) - private String timeoutStr = "1s"; + @Option(names = { "-t", "--time" }, description = "The transaction timeout time. " + + "(eg: 1s, 10s, 1m, 5h, 3d)", required = true, + converter = TimeUnitToMillisConverter.class) + private Long timeoutInMillis = 1L; @Override void run() throws Exception { - long timeout; - try { - timeout = TimeUnit.SECONDS.toMillis(RelativeTimeUtil.parseRelativeTimeInSeconds(timeoutStr)); - } catch (IllegalArgumentException exception) { - throw new ParameterException(exception.getMessage()); - } if (coordinatorId != null) { print(getAdmin().transactions().getSlowTransactionsByCoordinatorId(coordinatorId, - timeout, TimeUnit.MILLISECONDS)); + timeoutInMillis, TimeUnit.MILLISECONDS)); } else { - print(getAdmin().transactions().getSlowTransactions(timeout, TimeUnit.MILLISECONDS)); + print(getAdmin().transactions().getSlowTransactions(timeoutInMillis, TimeUnit.MILLISECONDS)); } } } - @Parameters(commandDescription = "Get transaction coordinator internal stats") + @Command(description = "Get transaction coordinator internal stats") private class GetCoordinatorInternalStats extends CliCommand { - @Parameter(names = {"-c", "--coordinator-id"}, description = "The coordinator id", required = true) + @Option(names = {"-c", "--coordinator-id"}, description = "The coordinator id", required = true) private int coordinatorId; - @Parameter(names = { "-m", "--metadata" }, description = "Flag to include ledger metadata") + @Option(names = { "-m", "--metadata" }, description = "Flag to include ledger metadata") private boolean metadata = false; @Override void run() throws Exception { @@ -172,15 +171,15 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Get pending ack internal stats") + @Command(description = "Get pending ack internal stats") private class GetPendingAckInternalStats extends CliCommand { - @Parameter(names = {"-t", "--topic"}, description = "Topic name", required = true) + @Option(names = {"-t", "--topic"}, description = "Topic name", required = true) private String topic; - @Parameter(names = {"-s", "--subscription-name"}, description = "Subscription name", required = true) + @Option(names = {"-s", "--subscription-name"}, description = "Subscription name", required = true) private String subName; - @Parameter(names = { "-m", "--metadata" }, description = "Flag to include ledger metadata") + @Option(names = { "-m", "--metadata" }, description = "Flag to include ledger metadata") private boolean metadata = false; @Override void run() throws Exception { @@ -188,9 +187,23 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Update the scale of transaction coordinators") + @Command(description = "Get transaction buffer internal stats") + private class GetTransactionBufferInternalStats extends CliCommand { + @Option(names = {"-t", "--topic"}, description = "Topic name", required = true) + private String topic; + + @Option(names = { "-m", "--metadata" }, description = "Flag to include ledger metadata") + private boolean metadata = false; + + @Override + void run() throws Exception { + print(getAdmin().transactions().getTransactionBufferInternalStats(topic, metadata)); + } + } + + @Command(description = "Update the scale of transaction coordinators") private class ScaleTransactionCoordinators extends CliCommand { - @Parameter(names = { "-r", "--replicas" }, description = "The scale of the transaction coordinators") + @Option(names = { "-r", "--replicas" }, description = "The scale of the transaction coordinators") private int replicas; @Override void run() throws Exception { @@ -198,21 +211,21 @@ void run() throws Exception { } } - @Parameters(commandDescription = "Get the position stats in transaction pending ack") + @Command(description = "Get the position stats in transaction pending ack") private class GetPositionStatsInPendingAck extends CliCommand { - @Parameter(names = {"-t", "--topic"}, description = "The topic name", required = true) + @Option(names = {"-t", "--topic"}, description = "The topic name", required = true) private String topic; - @Parameter(names = {"-s", "--subscription-name"}, description = "Subscription name", required = true) + @Option(names = {"-s", "--subscription-name"}, description = "Subscription name", required = true) private String subName; - @Parameter(names = {"-l", "--ledger-id"}, description = "Ledger ID of the position", required = true) + @Option(names = {"-l", "--ledger-id"}, description = "Ledger ID of the position", required = true) private Long ledgerId; - @Parameter(names = {"-e", "--entry-id"}, description = "Entry ID of the position", required = true) + @Option(names = {"-e", "--entry-id"}, description = "Entry ID of the position", required = true) private Long entryId; - @Parameter(names = {"-b", "--batch-index"}, description = "Batch index of the position") + @Option(names = {"-b", "--batch-index"}, description = "Batch index of the position") private Integer batchIndex; @Override @@ -221,7 +234,7 @@ void run() throws Exception { } } - @Parameters(commandDescription = "List transaction coordinators") + @Command(description = "List transaction coordinators") private class ListTransactionCoordinators extends CliCommand { @Override void run() throws Exception { @@ -237,21 +250,36 @@ void run() throws Exception { } } + @Command(description = "Abort transaction") + private class AbortTransaction extends CliCommand { + @Option(names = {"-m", "--most-sig-bits"}, description = "The most sig bits", required = true) + private long mostSigBits; + + @Option(names = {"-l", "--least-sig-bits"}, description = "The least sig bits", required = true) + private long leastSigBits; + + @Override + void run() throws Exception { + getAdmin().transactions().abortTransaction(new TxnID(mostSigBits, leastSigBits)); + } + } public CmdTransactions(Supplier admin) { super("transactions", admin); - jcommander.addCommand("coordinator-internal-stats", new GetCoordinatorInternalStats()); - jcommander.addCommand("pending-ack-internal-stats", new GetPendingAckInternalStats()); - jcommander.addCommand("coordinator-stats", new GetCoordinatorStats()); - jcommander.addCommand("transaction-buffer-stats", new GetTransactionBufferStats()); - jcommander.addCommand("pending-ack-stats", new GetPendingAckStats()); - jcommander.addCommand("transaction-in-buffer-stats", new GetTransactionInBufferStats()); - jcommander.addCommand("transaction-in-pending-ack-stats", new GetTransactionInPendingAckStats()); - jcommander.addCommand("transaction-metadata", new GetTransactionMetadata()); - jcommander.addCommand("slow-transactions", new GetSlowTransactions()); - jcommander.addCommand("scale-transactionCoordinators", new ScaleTransactionCoordinators()); - jcommander.addCommand("position-stats-in-pending-ack", new GetPositionStatsInPendingAck()); - jcommander.addCommand("coordinators-list", new ListTransactionCoordinators()); + addCommand("coordinator-internal-stats", new GetCoordinatorInternalStats()); + addCommand("pending-ack-internal-stats", new GetPendingAckInternalStats()); + addCommand("buffer-snapshot-internal-stats", new GetTransactionBufferInternalStats()); + addCommand("coordinator-stats", new GetCoordinatorStats()); + addCommand("transaction-buffer-stats", new GetTransactionBufferStats()); + addCommand("pending-ack-stats", new GetPendingAckStats()); + addCommand("transaction-in-buffer-stats", new GetTransactionInBufferStats()); + addCommand("transaction-in-pending-ack-stats", new GetTransactionInPendingAckStats()); + addCommand("transaction-metadata", new GetTransactionMetadata()); + addCommand("slow-transactions", new GetSlowTransactions()); + addCommand("scale-transactionCoordinators", new ScaleTransactionCoordinators()); + addCommand("position-stats-in-pending-ack", new GetPositionStatsInPendingAck()); + addCommand("coordinators-list", new ListTransactionCoordinators()); + addCommand("abort-transaction", new AbortTransaction()); } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdUsageFormatter.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdUsageFormatter.java deleted file mode 100644 index a1771a07cc96b..0000000000000 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdUsageFormatter.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.admin.cli; - -import com.beust.jcommander.DefaultUsageFormatter; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameters; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -public class CmdUsageFormatter extends DefaultUsageFormatter { - - /** - * The commands in this set are hidden and not shown to users. - */ - private Set deprecatedCommands = new HashSet<>(); - - private final JCommander commander; - - public CmdUsageFormatter(JCommander commander) { - super(commander); - this.commander = commander; - } - - /** - * This method is copied from DefaultUsageFormatter, - * but the ability to skip deprecated commands is added. - * @param out - * @param indentCount - * @param descriptionIndent - * @param indent - */ - @Override - public void appendCommands(StringBuilder out, int indentCount, int descriptionIndent, String indent) { - out.append(indent + " Commands:\n"); - - for (Map.Entry commands : commander.getRawCommands().entrySet()) { - Object arg = commands.getValue().getObjects().get(0); - Parameters p = arg.getClass().getAnnotation(Parameters.class); - - if (p == null || !p.hidden()) { - JCommander.ProgramName progName = commands.getKey(); - String dispName = progName.getDisplayName(); - //skip the deprecated command - if (deprecatedCommands.contains(dispName)) { - continue; - } - String description = indent + s(4) + dispName + s(6) + getCommandDescription(progName.getName()); - wrapDescription(out, indentCount + descriptionIndent, description); - out.append("\n"); - - JCommander jc = commander.findCommandByAlias(progName.getName()); - jc.getUsageFormatter().usage(out, indent + s(6)); - out.append("\n"); - } - } - } - - public void addDeprecatedCommand(String command) { - this.deprecatedCommands.add(command); - } - - public void removeDeprecatedCommand(String command) { - this.deprecatedCommands.remove(command); - } - - public void clearDeprecatedCommand(){ - this.deprecatedCommands.clear(); - } - -} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CustomCommandsUtils.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CustomCommandsUtils.java index e1968d0349ad2..a49a8c450fa2c 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CustomCommandsUtils.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CustomCommandsUtils.java @@ -18,8 +18,6 @@ */ package org.apache.pulsar.admin.cli; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; import java.lang.reflect.Field; import java.util.HashMap; import java.util.List; @@ -37,7 +35,6 @@ import javassist.bytecode.annotation.Annotation; import javassist.bytecode.annotation.ArrayMemberValue; import javassist.bytecode.annotation.BooleanMemberValue; -import javassist.bytecode.annotation.IntegerMemberValue; import javassist.bytecode.annotation.MemberValue; import javassist.bytecode.annotation.StringMemberValue; import lombok.Setter; @@ -47,6 +44,9 @@ import org.apache.pulsar.admin.cli.extensions.ParameterDescriptor; import org.apache.pulsar.admin.cli.extensions.ParameterType; import org.apache.pulsar.client.admin.PulsarAdmin; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; public final class CustomCommandsUtils { private CustomCommandsUtils() { @@ -68,8 +68,12 @@ public static Object generateCliCommand(CustomCommandGroup group, CommandExecuti ConstPool constpool = classFile.getConstPool(); AnnotationsAttribute annotationsAttribute = new AnnotationsAttribute(constpool, AnnotationsAttribute.visibleTag); - Annotation annotation = new Annotation(Parameters.class.getName(), constpool); - annotation.addMemberValue("commandDescription", new StringMemberValue(description, + Annotation annotation = new Annotation(Command.class.getName(), constpool); + ArrayMemberValue descArrayMemberValue = new ArrayMemberValue(classFile.getConstPool()); + descArrayMemberValue.setValue( + new MemberValue[]{new StringMemberValue(description, classFile.getConstPool())}); + annotation.addMemberValue("description", descArrayMemberValue); + annotation.addMemberValue("name", new StringMemberValue(group.name(), classFile.getConstPool())); annotationsAttribute.setAnnotation(annotation); ctClass.getClassFile().addAttribute(annotationsAttribute); @@ -102,7 +106,7 @@ public CmdBaseAdapter(String cmdName, Supplier adminSupplier, DecoratedCommand commandImpl = generateCustomCommand(cmdName, name, command); commandImpl.setCommand(command); commandImpl.setContext(context); - jcommander.addCommand(name, commandImpl); + addCommand(name, commandImpl); } } } @@ -142,9 +146,12 @@ private static DecoratedCommand generateCustomCommand(String group, String name, AnnotationsAttribute annotationsAttribute = new AnnotationsAttribute(constpool, AnnotationsAttribute.visibleTag); - Annotation annotation = new Annotation(Parameters.class.getName(), constpool); - annotation.addMemberValue("commandDescription", - new StringMemberValue(description, classFile.getConstPool())); + Annotation annotation = new Annotation(Command.class.getName(), constpool); + ArrayMemberValue descArrayMemberValue = new ArrayMemberValue(classFile.getConstPool()); + descArrayMemberValue.setValue( + new MemberValue[]{new StringMemberValue(description, classFile.getConstPool())}); + annotation.addMemberValue("description", descArrayMemberValue); + annotation.addMemberValue("name", new StringMemberValue(name, classFile.getConstPool())); annotationsAttribute.setAnnotation(annotation); ctClass.getClassFile().addAttribute(annotationsAttribute); @@ -183,11 +190,9 @@ private static DecoratedCommand generateCustomCommand(String group, String name, AnnotationsAttribute fieldAnnotationsAttribute = new AnnotationsAttribute(constpool, AnnotationsAttribute.visibleTag); - Annotation fieldAnnotation = new Annotation(Parameter.class.getName(), constpool); - - // in JCommander if you don't set the "names" property then you want to get all the other - // parameters + Annotation fieldAnnotation; if (!parameterDescriptor.isMainParameter()) { + fieldAnnotation = new Annotation(Option.class.getName(), constpool); MemberValue[] memberValues = new MemberValue[parameterNames.size()]; int i = 0; for (String parameterName : parameterNames) { @@ -196,16 +201,23 @@ private static DecoratedCommand generateCustomCommand(String group, String name, ArrayMemberValue arrayMemberValue = new ArrayMemberValue(classFile.getConstPool()); arrayMemberValue.setValue(memberValues); fieldAnnotation.addMemberValue("names", arrayMemberValue); - } - - fieldAnnotation.addMemberValue("description", - new StringMemberValue(parameterDescriptor.getDescription(), classFile.getConstPool())); - fieldAnnotation.addMemberValue("required", - new BooleanMemberValue(parameterDescriptor.isRequired(), classFile.getConstPool())); - if (parameterDescriptor.getType() == ParameterType.BOOLEAN) { + fieldAnnotation.addMemberValue("required", + new BooleanMemberValue(parameterDescriptor.isRequired(), classFile.getConstPool())); + if (parameterDescriptor.getType() == ParameterType.BOOLEAN) { + fieldAnnotation.addMemberValue("arity", + new StringMemberValue("1", classFile.getConstPool())); + } + } else { + fieldAnnotation = new Annotation(Parameters.class.getName(), constpool); + String arityValue = parameterDescriptor.isRequired() ? "1" : "0..1"; fieldAnnotation.addMemberValue("arity", - new IntegerMemberValue(classFile.getConstPool(), 1)); + new StringMemberValue(arityValue, classFile.getConstPool())); } + ArrayMemberValue optionDescArrayMemberValue = new ArrayMemberValue(classFile.getConstPool()); + optionDescArrayMemberValue.setValue( + new MemberValue[]{ + new StringMemberValue(parameterDescriptor.getDescription(), classFile.getConstPool())}); + fieldAnnotation.addMemberValue("description", optionDescArrayMemberValue); fieldAnnotationsAttribute.setAnnotation(fieldAnnotation); field.getFieldInfo().addAttribute(fieldAnnotationsAttribute); field.setModifiers(Modifier.PUBLIC); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminPropertiesProvider.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminPropertiesProvider.java new file mode 100644 index 0000000000000..85d350ce99b7d --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminPropertiesProvider.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli; + +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import java.util.Properties; +import picocli.CommandLine.PropertiesDefaultProvider; + +class PulsarAdminPropertiesProvider extends PropertiesDefaultProvider { + private static final String webServiceUrlKey = "webServiceUrl"; + private final Properties properties; + + private PulsarAdminPropertiesProvider(Properties properties) { + super(properties); + this.properties = properties; + } + + static PulsarAdminPropertiesProvider create(Properties properties) { + Properties clone = (Properties) properties.clone(); + if (isBlank(properties.getProperty(webServiceUrlKey))) { + String serviceUrl = properties.getProperty("serviceUrl"); + if (isNotBlank(serviceUrl)) { + properties.put(webServiceUrlKey, serviceUrl); + } + } + return new PulsarAdminPropertiesProvider(clone); + } + + String getAdminUrl() { + return properties.getProperty(webServiceUrlKey); + } +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminSupplier.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminSupplier.java index 764dc9de5dfdd..3417f2bb2c618 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminSupplier.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminSupplier.java @@ -19,6 +19,7 @@ package org.apache.pulsar.admin.cli; import static org.apache.commons.lang3.StringUtils.isNotBlank; +import com.google.common.annotations.VisibleForTesting; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import lombok.Data; @@ -53,7 +54,7 @@ static RootParamsKey fromRootParams(PulsarAdminTool.RootParams params) { } } - protected final PulsarAdminBuilder adminBuilder; + protected PulsarAdminBuilder adminBuilder; private RootParamsKey currentParamsKey; private PulsarAdmin admin; @@ -108,4 +109,8 @@ private static void applyRootParamsToAdminBuilder(PulsarAdminBuilder adminBuilde } } + @VisibleForTesting + public void setAdminBuilder(PulsarAdminBuilder builder) { + this.adminBuilder = builder; + } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java index c06016be43883..cd79098f0c3e9 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java @@ -19,11 +19,9 @@ package org.apache.pulsar.admin.cli; import static org.apache.commons.lang3.StringUtils.isBlank; -import static org.apache.commons.lang3.StringUtils.isNotBlank; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; import com.google.common.annotations.VisibleForTesting; import java.io.FileInputStream; +import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.util.Arrays; import java.util.HashMap; @@ -32,7 +30,6 @@ import java.util.Properties; import java.util.function.Supplier; import lombok.Getter; -import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.admin.cli.extensions.CommandExecutionContext; import org.apache.pulsar.admin.cli.extensions.CustomCommandFactory; import org.apache.pulsar.admin.cli.extensions.CustomCommandGroup; @@ -40,76 +37,82 @@ import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminBuilder; import org.apache.pulsar.client.admin.internal.PulsarAdminImpl; +import org.apache.pulsar.common.util.DefaultPulsarSslFactory; import org.apache.pulsar.common.util.ShutdownUtil; - -public class PulsarAdminTool { +import org.apache.pulsar.internal.CommandHook; +import org.apache.pulsar.internal.CommanderFactory; +import picocli.CommandLine; +import picocli.CommandLine.ArgGroup; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.ScopeType; + +@Command(name = "pulsar-admin", + scope = ScopeType.INHERIT, + mixinStandardHelpOptions = true, + showDefaultValues = true, + versionProvider = PulsarVersionProvider.class +) +public class PulsarAdminTool implements CommandHook { protected static boolean allowSystemExit = true; private static int lastExitCode = Integer.MIN_VALUE; - protected final List customCommandFactories; + protected List customCommandFactories; protected Map> commandMap; - protected JCommander jcommander; - protected RootParams rootParams; - private final Properties properties; + protected final CommandLine commander; + @ArgGroup(heading = "Options:%n", exclusive = false) + protected RootParams rootParams = new RootParams(); protected PulsarAdminSupplier pulsarAdminSupplier; + private PulsarAdminPropertiesProvider pulsarAdminPropertiesProvider; @Getter public static class RootParams { - @Parameter(names = { "--admin-url" }, description = "Admin Service URL to which to connect.") + @Option(names = { "--admin-url" }, description = "Admin Service URL to which to connect.", + descriptionKey = "webServiceUrl") String serviceUrl = null; - @Parameter(names = { "--auth-plugin" }, description = "Authentication plugin class name.") + @Option(names = { "--auth-plugin" }, description = "Authentication plugin class name.", + descriptionKey = "authPlugin") String authPluginClassName = null; - @Parameter(names = { "--request-timeout" }, description = "Request time out in seconds for " + @Option(names = { "--request-timeout" }, description = "Request time out in seconds for " + "the pulsar admin client for any request") int requestTimeout = PulsarAdminImpl.DEFAULT_REQUEST_TIMEOUT_SECONDS; - @Parameter( - names = { "--auth-params" }, + @Option(names = { "--auth-params" }, descriptionKey = "authParams", description = "Authentication parameters, whose format is determined by the implementation " + "of method `configure` in authentication plugin class, for example \"key1:val1,key2:val2\" " + "or \"{\"key1\":\"val1\",\"key2\":\"val2\"}\".") String authParams = null; - @Parameter(names = { "--tls-allow-insecure" }, description = "Allow TLS insecure connection") + @Option(names = { "--tls-allow-insecure" }, description = "Allow TLS insecure connection") Boolean tlsAllowInsecureConnection; - @Parameter(names = { "--tls-trust-cert-path" }, description = "Allow TLS trust cert file path") + @Option(names = { "--tls-trust-cert-path" }, description = "Allow TLS trust cert file path") String tlsTrustCertsFilePath; - @Parameter(names = { "--tls-enable-hostname-verification" }, + @Option(names = { "--tls-enable-hostname-verification" }, description = "Enable TLS common name verification") Boolean tlsEnableHostnameVerification; - @Parameter(names = {"--tls-provider"}, description = "Set up TLS provider. " + @Option(names = {"--tls-provider"}, description = "Set up TLS provider. " + "When TLS authentication with CACert is used, the valid value is either OPENSSL or JDK. " + "When TLS authentication with KeyStore is used, available options can be SunJSSE, Conscrypt " - + "and so on.") + + "and so on.", descriptionKey = "webserviceTlsProvider") String tlsProvider; - - @Parameter(names = { "-v", "--version" }, description = "Get version of pulsar admin client") - boolean version; - - @Parameter(names = { "-h", "--help", }, help = true, description = "Show this help.") - boolean help; } public PulsarAdminTool(Properties properties) throws Exception { - this.properties = properties; - customCommandFactories = CustomCommandFactoryProvider.createCustomCommandFactories(properties); - rootParams = new RootParams(); - // fallback to previous-version serviceUrl property to maintain backward-compatibility - initRootParamsFromProperties(properties); - final PulsarAdminBuilder baseAdminBuilder = createAdminBuilderFromProperties(properties); - pulsarAdminSupplier = new PulsarAdminSupplier(baseAdminBuilder, rootParams); - initJCommander(); + // Use -v instead -V + System.setProperty("picocli.version.name.0", "-v"); + commander = CommanderFactory.createRootCommanderWithHook(this, pulsarAdminPropertiesProvider); + pulsarAdminSupplier = new PulsarAdminSupplier(createAdminBuilderFromProperties(properties), rootParams); + initCommander(properties); } - private static PulsarAdminBuilder createAdminBuilderFromProperties(Properties properties) { boolean useKeyStoreTls = Boolean .parseBoolean(properties.getProperty("useKeyStoreTls", "false")); @@ -128,6 +131,9 @@ private static PulsarAdminBuilder createAdminBuilderFromProperties(Properties pr boolean tlsEnableHostnameVerification = Boolean.parseBoolean(properties .getProperty("tlsEnableHostnameVerification", "false")); final String tlsTrustCertsFilePath = properties.getProperty("tlsTrustCertsFilePath"); + final String sslFactoryPlugin = properties.getProperty("sslFactoryPlugin", + DefaultPulsarSslFactory.class.getName()); + final String sslFactoryPluginParams = properties.getProperty("sslFactoryPluginParams", ""); return PulsarAdmin.builder().allowTlsInsecureConnection(tlsAllowInsecureConnection) .enableTlsHostnameVerification(tlsEnableHostnameVerification) @@ -140,22 +146,16 @@ private static PulsarAdminBuilder createAdminBuilderFromProperties(Properties pr .tlsKeyStorePath(tlsKeyStorePath) .tlsKeyStorePassword(tlsKeyStorePassword) .tlsKeyFilePath(tlsKeyFilePath) - .tlsCertificateFilePath(tlsCertificateFilePath); + .tlsCertificateFilePath(tlsCertificateFilePath) + .sslFactoryPlugin(sslFactoryPlugin) + .sslFactoryPluginParams(sslFactoryPluginParams); } - protected void initRootParamsFromProperties(Properties properties) { - rootParams.serviceUrl = isNotBlank(properties.getProperty("webServiceUrl")) - ? properties.getProperty("webServiceUrl") - : properties.getProperty("serviceUrl"); - rootParams.authPluginClassName = properties.getProperty("authPlugin"); - rootParams.authParams = properties.getProperty("authParams"); - rootParams.tlsProvider = properties.getProperty("webserviceTlsProvider"); - } - - public void setupCommands() { + private void setupCommands(Properties properties) { try { for (Map.Entry> c : commandMap.entrySet()) { - addCommand(c, pulsarAdminSupplier); + Object o = c.getValue().getConstructor(Supplier.class).newInstance(pulsarAdminSupplier); + addCommand(c.getKey(), o); } CommandExecutionContext context = new CommandExecutionContext() { @@ -174,8 +174,7 @@ public Properties getConfiguration() { List customCommandGroups = factory.commandGroups(context); for (CustomCommandGroup group : customCommandGroups) { Object generated = CustomCommandsUtils.generateCliCommand(group, context, pulsarAdminSupplier); - jcommander.addCommand(group.name(), generated); - commandMap.put(group.name(), null); + addCommand(group.name(), generated); } } } catch (Exception e) { @@ -190,84 +189,16 @@ public Properties getConfiguration() { } } - private void addCommand(Map.Entry> c, Supplier admin) throws Exception { - // To remain backwards compatibility for "source" and "sink" commands - // TODO eventually remove this - if (c.getKey().equals("sources") || c.getKey().equals("source")) { - jcommander.addCommand("sources", c.getValue().getConstructor(Supplier.class).newInstance(admin), "source"); - } else if (c.getKey().equals("sinks") || c.getKey().equals("sink")) { - jcommander.addCommand("sinks", c.getValue().getConstructor(Supplier.class).newInstance(admin), "sink"); - } else if (c.getKey().equals("functions")) { - jcommander.addCommand(c.getKey(), c.getValue().getConstructor(Supplier.class).newInstance(admin)); + private void addCommand(String name, Object o) throws Exception { + if (o instanceof CmdBase) { + commander.addSubcommand(name, ((CmdBase) o).getCommander()); } else { - // Other mode, all components are initialized. - if (c.getValue() != null) { - jcommander.addCommand(c.getKey(), c.getValue().getConstructor(Supplier.class).newInstance(admin)); - } + commander.addSubcommand(o); } } protected boolean run(String[] args) { - setupCommands(); - - if (args.length == 0) { - jcommander.usage(); - return false; - } - - int cmdPos; - for (cmdPos = 0; cmdPos < args.length; cmdPos++) { - if (commandMap.containsKey(args[cmdPos])) { - break; - } - } - - try { - jcommander.parse(Arrays.copyOfRange(args, 0, Math.min(cmdPos, args.length))); - //rootParams are populated by jcommander.parse - pulsarAdminSupplier.rootParamsUpdated(rootParams); - } catch (Exception e) { - System.err.println(e.getMessage()); - System.err.println(); - jcommander.usage(); - return false; - } - - if (isBlank(rootParams.serviceUrl)) { - System.out.println("Can't find any admin url to use"); - jcommander.usage(); - return false; - } - - if (rootParams.version) { - System.out.println("Current version of pulsar admin client is: " + PulsarVersion.getVersion()); - return true; - } - - if (rootParams.help) { - jcommander.usage(); - return true; - } - - if (cmdPos == args.length) { - jcommander.usage(); - return false; - } else { - String cmd = args[cmdPos]; - - // To remain backwards compatibility for "source" and "sink" commands - // TODO eventually remove this - if (cmd.equals("source")) { - cmd = "sources"; - } else if (cmd.equals("sink")) { - cmd = "sinks"; - } - - JCommander obj = jcommander.getCommands().get(cmd); - CmdBase cmdObj = (CmdBase) obj.getObjects().get(0); - - return cmdObj.run(Arrays.copyOfRange(args, cmdPos + 1, args.length)); - } + return commander.execute(args) == 0; } public static void main(String[] args) throws Exception { @@ -306,7 +237,7 @@ private static void exit(int code) { if (allowSystemExit) { // we are using halt and not System.exit, we do not mind about shutdown hooks // they are only slowing down the tool - ShutdownUtil.triggerImmediateForcefulShutdown(code); + ShutdownUtil.triggerImmediateForcefulShutdown(code, false); } else { System.out.println("Exit code is " + code + " (System.exit not called, as we are in test mode)"); } @@ -325,11 +256,20 @@ static void resetLastExitCode() { lastExitCode = Integer.MIN_VALUE; } - protected void initJCommander() { - jcommander = new JCommander(); - jcommander.setProgramName("pulsar-admin"); - jcommander.addObject(rootParams); + @Override + public int preRun() { + if (isBlank(rootParams.serviceUrl)) { + commander.getErr().println("Can't find any admin url to use"); + return 1; + } + pulsarAdminSupplier.rootParamsUpdated(rootParams); + return 0; + } + private void initCommander(Properties properties) throws IOException { + customCommandFactories = CustomCommandFactoryProvider.createCustomCommandFactories(properties); + pulsarAdminPropertiesProvider = PulsarAdminPropertiesProvider.create(properties); + commander.setDefaultValueProvider(pulsarAdminPropertiesProvider); commandMap = new HashMap<>(); commandMap.put("clusters", CmdClusters.class); commandMap.put("ns-isolation-policy", CmdNamespaceIsolationPolicy.class); @@ -361,17 +301,10 @@ protected void initJCommander() { // Automatically generate documents for pulsar-admin commandMap.put("documents", CmdGenerateDocument.class); // To remain backwards compatibility for "source" and "sink" commands - // TODO eventually remove this - commandMap.put("source", CmdSources.class); - commandMap.put("sink", CmdSinks.class); - commandMap.put("packages", CmdPackages.class); commandMap.put("transactions", CmdTransactions.class); - } - @VisibleForTesting - public void setPulsarAdminSupplier(PulsarAdminSupplier pulsarAdminSupplier) { - this.pulsarAdminSupplier = pulsarAdminSupplier; + setupCommands(properties); } @VisibleForTesting @@ -379,8 +312,12 @@ public PulsarAdminSupplier getPulsarAdminSupplier() { return pulsarAdminSupplier; } - @VisibleForTesting - public RootParams getRootParams() { - return rootParams; + // The following methods are used for Pulsar shell. + protected void setCommandName(String name) { + commander.setCommandName(name); + } + + protected String getAdminUrl() { + return pulsarAdminPropertiesProvider.getAdminUrl(); } } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/util/NullCacheSizeAllocator.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarVersionProvider.java similarity index 69% rename from pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/util/NullCacheSizeAllocator.java rename to pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarVersionProvider.java index 4f218950f3564..ff107a0eb1536 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/util/NullCacheSizeAllocator.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarVersionProvider.java @@ -16,25 +16,14 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.sql.presto.util; +package org.apache.pulsar.admin.cli; -/** - * Null cache size allocator. - */ -public class NullCacheSizeAllocator implements CacheSizeAllocator { - - @Override - public long getAvailableCacheSize() { - return -1; - } - - @Override - public void allocate(long size) { - // no op - } +import org.apache.pulsar.PulsarVersion; +import picocli.CommandLine.IVersionProvider; +public class PulsarVersionProvider implements IVersionProvider { @Override - public void release(long size) { - // no op + public String[] getVersion() { + return new String[]{"Current version of pulsar admin client is: " + PulsarVersion.getVersion()}; } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/CmdUtils.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/CmdUtils.java index a8659e066e47a..bfbd78601c4c1 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/CmdUtils.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/CmdUtils.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.admin.cli.utils; -import com.beust.jcommander.ParameterException; import com.fasterxml.jackson.databind.exc.InvalidFormatException; import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException; import java.io.File; @@ -40,7 +39,7 @@ public static T loadConfig(String file, Class clazz) throws IOException { unrecognizedPropertyException.getLocation().getLineNr(), unrecognizedPropertyException.getLocation().getColumnNr(), unrecognizedPropertyException.getKnownPropertyIds()); - throw new ParameterException(exceptionMessage); + throw new IllegalArgumentException(exceptionMessage); } else if (ex instanceof InvalidFormatException) { InvalidFormatException invalidFormatException = (InvalidFormatException) ex; @@ -50,10 +49,24 @@ public static T loadConfig(String file, Class clazz) throws IOException { invalidFormatException.getLocation().getLineNr(), invalidFormatException.getLocation().getColumnNr()); - throw new ParameterException(exceptionMessage); + throw new IllegalArgumentException(exceptionMessage); } else { - throw new ParameterException(ex.getMessage()); + throw new IllegalArgumentException(ex.getMessage()); } } } + + public static boolean positiveCheck(String paramName, long value) { + if (value <= 0) { + throw new IllegalArgumentException(paramName + " cannot be less than or equal to 0!"); + } + return true; + } + + public static boolean maxValueCheck(String paramName, long value, long maxValue) { + if (value > maxValue) { + throw new IllegalArgumentException(paramName + " cannot be greater than " + maxValue + "!"); + } + return true; + } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/NameValueParameterSplitter.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/NameValueParameterSplitter.java deleted file mode 100644 index 011f93e18f1f0..0000000000000 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/NameValueParameterSplitter.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.admin.cli.utils; - -import com.beust.jcommander.IStringConverter; -import com.beust.jcommander.ParameterException; -import java.util.HashMap; -import java.util.Map; - -public class NameValueParameterSplitter implements IStringConverter> { - - @Override - public Map convert(String value) { - boolean error = false; - Map map = new HashMap(); - - String[] nvpairs = value.split(","); - - for (String nvpair : nvpairs) { - error = true; - if (nvpair != null) { - String[] nv = nvpair.split("="); - if (nv != null && nv.length == 2) { - nv[0] = nv[0].trim(); - nv[1] = nv[1].trim(); - if (!nv[0].isEmpty() && !nv[1].isEmpty() && nv[0].charAt(0) != '\'') { - map.put(nv[0], nv[1]); - error = false; - } - } - } - - if (error) { - break; - } - } - - if (error) { - throw new ParameterException("unable to parse bad name=value parameter list: " + value); - } - - return map; - } - -} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/AbstractCmd.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/AbstractCmd.java new file mode 100644 index 0000000000000..10b68648ebbba --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/AbstractCmd.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.cli; + +import java.util.concurrent.Callable; + +public abstract class AbstractCmd implements Callable { + // Picocli entrypoint. + @Override + public Integer call() throws Exception { + return run(); + } + + abstract int run() throws Exception; +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/AbstractCmdConsume.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/AbstractCmdConsume.java index ef0ffbc297340..33df4aca96d2d 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/AbstractCmdConsume.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/AbstractCmdConsume.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.Base64; import java.util.HashMap; import java.util.Map; import java.util.concurrent.BlockingQueue; @@ -40,7 +41,9 @@ import org.apache.pulsar.client.api.schema.Field; import org.apache.pulsar.client.api.schema.GenericObject; import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.common.api.EncryptionContext; import org.apache.pulsar.common.schema.KeyValue; +import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.collections.GrowableArrayBlockingQueue; import org.eclipse.jetty.websocket.api.RemoteEndpoint; import org.eclipse.jetty.websocket.api.Session; @@ -55,7 +58,7 @@ * common part of consume command and read command of pulsar-client. * */ -public abstract class AbstractCmdConsume { +public abstract class AbstractCmdConsume extends AbstractCmd { protected static final Logger LOG = LoggerFactory.getLogger(PulsarClientTool.class); protected static final String MESSAGE_BOUNDARY = "----- got message -----"; @@ -87,7 +90,8 @@ public void updateConfig(ClientBuilder clientBuilder, Authentication authenticat * Whether to display BytesMessages in hexdump style, ignored for simple text messages * @return String representation of the message */ - protected String interpretMessage(Message message, boolean displayHex) throws IOException { + protected String interpretMessage(Message message, boolean displayHex, boolean printMetadata) + throws IOException { StringBuilder sb = new StringBuilder(); String properties = Arrays.toString(message.getProperties().entrySet().toArray()); @@ -108,6 +112,9 @@ protected String interpretMessage(Message message, boolean displayHex) throws data = value.toString(); } + sb.append("publishTime:[").append(message.getPublishTime()).append("], "); + sb.append("eventTime:[").append(message.getEventTime()).append("], "); + String key = null; if (message.hasKey()) { key = message.getKey(); @@ -119,6 +126,45 @@ protected String interpretMessage(Message message, boolean displayHex) throws } sb.append("content:").append(data); + if (printMetadata) { + if (message.getEncryptionCtx().isPresent()) { + EncryptionContext encContext = message.getEncryptionCtx().get(); + if (encContext.getKeys() != null && !encContext.getKeys().isEmpty()) { + sb.append(", "); + sb.append("encryption-keys:").append(", "); + encContext.getKeys().forEach((keyName, keyInfo) -> { + String metadata = Arrays.toString(keyInfo.getMetadata().entrySet().toArray()); + sb.append("name:").append(keyName).append(", ").append("key-value:") + .append(Base64.getEncoder().encode(keyInfo.getKeyValue())).append(", ") + .append("metadata:").append(metadata).append(", "); + + }); + sb.append(", ").append("param:").append(Base64.getEncoder().encode(encContext.getParam())) + .append(", ").append("algorithm:").append(encContext.getAlgorithm()).append(", ") + .append("compression-type:").append(encContext.getCompressionType()).append(", ") + .append("uncompressed-size").append(encContext.getUncompressedMessageSize()).append(", ") + .append("batch-size") + .append(encContext.getBatchSize().isPresent() ? encContext.getBatchSize().get() : 1); + } + } + if (message.hasBrokerPublishTime()) { + sb.append(", ").append("publish-time:").append(DateFormatter.format(message.getPublishTime())); + } + sb.append(", ").append("event-time:").append(DateFormatter.format(message.getEventTime())); + sb.append(", ").append("message-id:").append(message.getMessageId()); + sb.append(", ").append("producer-name:").append(message.getProducerName()); + sb.append(", ").append("sequence-id:").append(message.getSequenceId()); + sb.append(", ").append("replicated-from:").append(message.getReplicatedFrom()); + sb.append(", ").append("redelivery-count:").append(message.getRedeliveryCount()); + sb.append(", ").append("ordering-key:") + .append(message.getOrderingKey() != null ? new String(message.getOrderingKey()) : ""); + sb.append(", ").append("schema-version:") + .append(message.getSchemaVersion() != null ? new String(message.getSchemaVersion()) : ""); + if (message.hasIndex()) { + sb.append(", ").append("index:").append(message.getIndex()); + } + } + return sb.toString(); } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdConsume.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdConsume.java index 0c65604cbe6b8..0f0e2f0a9c813 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdConsume.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdConsume.java @@ -19,16 +19,11 @@ package org.apache.pulsar.client.cli; import static org.apache.commons.lang3.StringUtils.isNotBlank; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.RateLimiter; import java.io.IOException; import java.net.URI; -import java.util.ArrayList; import java.util.Base64; -import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -36,9 +31,9 @@ import org.apache.pulsar.client.api.AuthenticationDataProvider; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.ConsumerBuilder; +import org.apache.pulsar.client.api.ConsumerCryptoFailureAction; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionMode; @@ -47,94 +42,104 @@ import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.websocket.client.ClientUpgradeRequest; import org.eclipse.jetty.websocket.client.WebSocketClient; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Model.CommandSpec; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; +import picocli.CommandLine.Spec; /** * pulsar-client consume command implementation. - * */ -@Parameters(commandDescription = "Consume messages from a specified topic") +@Command(description = "Consume messages from a specified topic") public class CmdConsume extends AbstractCmdConsume { - @Parameter(description = "TopicName", required = true) - private List mainOptions = new ArrayList(); + @Parameters(description = "TopicName", arity = "1") + private String topic; - @Parameter(names = { "-t", "--subscription-type" }, description = "Subscription type.") + @Option(names = { "-t", "--subscription-type" }, description = "Subscription type.") private SubscriptionType subscriptionType = SubscriptionType.Exclusive; - @Parameter(names = { "-m", "--subscription-mode" }, description = "Subscription mode.") + @Option(names = { "-m", "--subscription-mode" }, description = "Subscription mode.") private SubscriptionMode subscriptionMode = SubscriptionMode.Durable; - @Parameter(names = { "-p", "--subscription-position" }, description = "Subscription position.") + @Option(names = { "-p", "--subscription-position" }, description = "Subscription position.") private SubscriptionInitialPosition subscriptionInitialPosition = SubscriptionInitialPosition.Latest; - @Parameter(names = { "-s", "--subscription-name" }, required = true, description = "Subscription name.") + @Option(names = { "-s", "--subscription-name" }, required = true, description = "Subscription name.") private String subscriptionName; - @Parameter(names = { "-n", + @Option(names = { "-n", "--num-messages" }, description = "Number of messages to consume, 0 means to consume forever.") private int numMessagesToConsume = 1; - @Parameter(names = { "--hex" }, description = "Display binary messages in hex.") + @Option(names = { "--hex" }, description = "Display binary messages in hex.") private boolean displayHex = false; - @Parameter(names = { "--hide-content" }, description = "Do not write the message to console.") + @Option(names = { "--hide-content" }, description = "Do not write the message to console.") private boolean hideContent = false; - @Parameter(names = { "-r", "--rate" }, description = "Rate (in msg/sec) at which to consume, " + @Option(names = { "-r", "--rate" }, description = "Rate (in msg/sec) at which to consume, " + "value 0 means to consume messages as fast as possible.") private double consumeRate = 0; - @Parameter(names = { "--regex" }, description = "Indicate the topic name is a regex pattern") + @Option(names = { "--regex" }, description = "Indicate the topic name is a regex pattern") private boolean isRegex = false; - @Parameter(names = {"-q", "--queue-size"}, description = "Consumer receiver queue size.") + @Option(names = {"-q", "--queue-size"}, description = "Consumer receiver queue size.") private int receiverQueueSize = 0; - @Parameter(names = { "-mc", "--max_chunked_msg" }, description = "Max pending chunk messages") + @Option(names = { "-mc", "--max_chunked_msg" }, description = "Max pending chunk messages") private int maxPendingChunkedMessage = 0; - @Parameter(names = { "-ac", + @Option(names = { "-ac", "--auto_ack_chunk_q_full" }, description = "Auto ack for oldest message on queue is full") private boolean autoAckOldestChunkedMessageOnQueueFull = false; - @Parameter(names = { "-ekv", + @Option(names = { "-ekv", "--encryption-key-value" }, description = "The URI of private key to decrypt payload, for example " + "file:///path/to/private.key or data:application/x-pem-file;base64,*****") private String encKeyValue; - @Parameter(names = { "-st", "--schema-type"}, + @Option(names = { "-st", "--schema-type"}, description = "Set a schema type on the consumer, it can be 'bytes' or 'auto_consume'") private String schemaType = "bytes"; - @Parameter(names = { "-pm", "--pool-messages" }, description = "Use the pooled message", arity = 1) + @Option(names = { "-pm", "--pool-messages" }, description = "Use the pooled message", arity = "1") private boolean poolMessages = true; - @Parameter(names = {"-rs", "--replicated" }, description = "Whether the subscription status should be replicated") + @Option(names = {"-rs", "--replicated" }, description = "Whether the subscription status should be replicated") private boolean replicateSubscriptionState = false; + @Option(names = { "-ca", "--crypto-failure-action" }, description = "Crypto Failure Action") + private ConsumerCryptoFailureAction cryptoFailureAction = ConsumerCryptoFailureAction.FAIL; + + @Option(names = { "-mp", "--print-metadata" }, description = "Message metadata") + private boolean printMetadata = false; + public CmdConsume() { // Do nothing super(); } + @Spec + private CommandSpec commandSpec; + /** * Run the consume command. * * @return 0 for success, < 0 otherwise */ - public int run() throws PulsarClientException, IOException { - if (mainOptions.size() != 1) { - throw (new ParameterException("Please provide one and only one topic name.")); - } + public int run() throws IOException { if (this.subscriptionName == null || this.subscriptionName.isEmpty()) { - throw (new ParameterException("Subscription name is not provided.")); + throw new CommandLine.ParameterException(commandSpec.commandLine(), "Subscription name is not provided."); } if (this.numMessagesToConsume < 0) { - throw (new ParameterException("Number of messages should be zero or positive.")); + throw new CommandLine.ParameterException(commandSpec.commandLine(), + "Number of messages should be zero or positive."); } - String topic = this.mainOptions.get(0); - if (this.serviceURL.startsWith("ws")) { return consumeFromWebSocket(topic); } else { @@ -146,7 +151,7 @@ private int consume(String topic) { int numMessagesConsumed = 0; int returnCode = 0; - try (PulsarClient client = clientBuilder.build()){ + try (PulsarClient client = clientBuilder.build()) { ConsumerBuilder builder; Schema schema = poolMessages ? Schema.BYTEBUFFER : Schema.BYTES; if ("auto_consume".equals(schemaType)) { @@ -176,6 +181,7 @@ private int consume(String topic) { } builder.autoAckOldestChunkedMessageOnQueueFull(this.autoAckOldestChunkedMessageOnQueueFull); + builder.cryptoFailureAction(cryptoFailureAction); if (isNotBlank(this.encKeyValue)) { builder.defaultCryptoKeyReader(this.encKeyValue); @@ -196,7 +202,7 @@ private int consume(String topic) { numMessagesConsumed += 1; if (!hideContent) { System.out.println(MESSAGE_BOUNDARY); - String output = this.interpretMessage(msg, displayHex); + String output = this.interpretMessage(msg, displayHex, printMetadata); System.out.println(output); } else if (numMessagesConsumed % 1000 == 0) { System.out.println("Received " + numMessagesConsumed + " messages"); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdGenerateDocumentation.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdGenerateDocumentation.java index cb5c5ef3c5b5a..eb0df56175b9a 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdGenerateDocumentation.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdGenerateDocumentation.java @@ -18,64 +18,81 @@ */ package org.apache.pulsar.client.cli; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterDescription; -import com.beust.jcommander.Parameters; +import static org.apache.pulsar.internal.CommandDescriptionUtil.getArgDescription; +import static org.apache.pulsar.internal.CommandDescriptionUtil.getCommandDescription; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; -import java.util.Map; -import java.util.Properties; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.PulsarClientException; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Model.ArgSpec; +import picocli.CommandLine.Model.CommandSpec; +import picocli.CommandLine.Model.OptionSpec; +import picocli.CommandLine.Option; +import picocli.CommandLine.Spec; @Getter -@Parameters(commandDescription = "Generate documentation automatically.") +@Command(description = "Generate documentation automatically.") @Slf4j -public class CmdGenerateDocumentation { +public class CmdGenerateDocumentation extends AbstractCmd { - @Parameter(names = {"-n", "--command-names"}, description = "List of command names") + @Spec + private CommandSpec pulsarClientCommandSpec; + + @Option(names = {"-n", "--command-names"}, description = "List of command names") private List commandNames = new ArrayList<>(); public int run() throws PulsarClientException { - PulsarClientTool pulsarClientTool = new PulsarClientTool(new Properties()); - JCommander commander = pulsarClientTool.jcommander; - if (commandNames.size() == 0) { - for (Map.Entry cmd : commander.getCommands().entrySet()) { - if (cmd.getKey().equals("generate_documentation")) { - continue; + if (commandNames == null || commandNames.isEmpty()) { + pulsarClientCommandSpec.parent().subcommands().forEach((k, v) -> { + if (k.equals("generate_documentation")) { + return; } - generateDocument(cmd.getKey(), commander); - } + this.generateDocument(k, v); + }); } else { - for (String commandName : commandNames) { - if (commandName.equals("generate_documentation")) { - continue; + commandNames.forEach(module -> { + CommandLine commandLine = pulsarClientCommandSpec.parent().subcommands().get(module); + if (commandLine == null) { + return; } - generateDocument(commandName, commander); - } + if (commandLine.getCommandName().equals("generate_documentation")) { + return; + } + this.generateDocument(module, commandLine); + }); } + return 0; } - protected String generateDocument(String module, JCommander parentCmd) { + protected String generateDocument(String module, CommandLine parentCmd) { StringBuilder sb = new StringBuilder(); - JCommander cmd = parentCmd.getCommands().get(module); sb.append("## ").append(module).append("\n\n"); - sb.append(parentCmd.getUsageFormatter().getCommandDescription(module)).append("\n"); + sb.append(getCommandDescription(parentCmd)).append("\n"); sb.append("\n\n```shell\n") .append("$ pulsar-client ").append(module).append(" [options]") .append("\n```"); sb.append("\n\n"); sb.append("|Flag|Description|Default|\n"); sb.append("|---|---|---|\n"); - List options = cmd.getParameters(); - options.stream().filter(ele -> !ele.getParameterAnnotation().hidden()).forEach((option) -> - sb.append("| `").append(option.getNames()) - .append("` | ").append(option.getDescription().replace("\n", " ")) - .append("|").append(option.getDefault()).append("|\n") - ); + + List options = parentCmd.getCommandSpec().args(); + options.forEach(ele -> { + if (ele.hidden() || !(ele instanceof OptionSpec)) { + return; + } + + String argDescription = getArgDescription(ele); + String descriptions = argDescription.replace("\n", " "); + sb.append("| `").append(Arrays.toString(((OptionSpec) ele).names())) + .append("` | ").append(descriptions) + .append("|").append(ele.defaultValue()).append("|\n"); + sb.append("|\n"); + }); System.out.println(sb.toString()); return sb.toString(); } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdProduce.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdProduce.java index 04d557d2a9302..e5a8836602151 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdProduce.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdProduce.java @@ -19,13 +19,13 @@ package org.apache.pulsar.client.cli; import static org.apache.commons.lang3.StringUtils.isNotBlank; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.RateLimiter; import com.google.gson.JsonParseException; +import java.io.ByteArrayOutputStream; +import java.io.EOFException; +import java.io.IOException; import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -35,10 +35,17 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.Encoder; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.io.JsonDecoder; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.AuthenticationDataProvider; import org.apache.pulsar.client.api.ClientBuilder; @@ -48,6 +55,7 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.TypedMessageBuilder; +import org.apache.pulsar.client.api.schema.KeyValueSchema; import org.apache.pulsar.client.impl.schema.SchemaInfoImpl; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.KeyValue; @@ -66,13 +74,18 @@ import org.eclipse.jetty.websocket.client.WebSocketClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Model.CommandSpec; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; +import picocli.CommandLine.Spec; /** * pulsar-client produce command implementation. - * */ -@Parameters(commandDescription = "Produce messages to a specified topic") -public class CmdProduce { +@Command(description = "Produce messages to a specified topic") +public class CmdProduce extends AbstractCmd { private static final Logger LOG = LoggerFactory.getLogger(PulsarClientTool.class); private static final int MAX_MESSAGES = 1000; @@ -80,66 +93,71 @@ public class CmdProduce { private static final String KEY_VALUE_ENCODING_TYPE_SEPARATED = "separated"; private static final String KEY_VALUE_ENCODING_TYPE_INLINE = "inline"; - @Parameter(description = "TopicName", required = true) - private List mainOptions; + @Parameters(description = "TopicName", arity = "1") + private String topic; - @Parameter(names = { "-m", "--messages" }, - description = "Messages to send, either -m or -f must be specified. Specify -m for each message.", - splitter = NoSplitter.class) + @Option(names = { "-m", "--messages" }, + description = "Messages to send, either -m or -f must be specified. Specify -m for each message.") private List messages = new ArrayList<>(); - @Parameter(names = { "-f", "--files" }, + @Option(names = { "-f", "--files" }, description = "Comma separated file paths to send, either -m or -f must be specified.") private List messageFileNames = new ArrayList<>(); - @Parameter(names = { "-n", "--num-produce" }, + @Option(names = { "-n", "--num-produce" }, description = "Number of times to send message(s), the count of messages/files * num-produce " + "should below than " + MAX_MESSAGES + ".") private int numTimesProduce = 1; - @Parameter(names = { "-r", "--rate" }, + @Option(names = { "-r", "--rate" }, description = "Rate (in msg/sec) at which to produce," + " value 0 means to produce messages as fast as possible.") private double publishRate = 0; - @Parameter(names = { "-db", "--disable-batching" }, description = "Disable batch sending of messages") + @Option(names = { "-db", "--disable-batching" }, description = "Disable batch sending of messages") private boolean disableBatching = false; - @Parameter(names = { "-c", + @Option(names = { "-c", "--chunking" }, description = "Should split the message and publish in chunks if message size is " + "larger than allowed max size") private boolean chunkingAllowed = false; - @Parameter(names = { "-s", "--separator" }, + @Option(names = { "-s", "--separator" }, description = "Character to split messages string on default is comma") private String separator = ","; - @Parameter(names = { "-p", "--properties"}, description = "Properties to add, Comma separated " + @Option(names = { "-p", "--properties"}, description = "Properties to add, Comma separated " + "key=value string, like k1=v1,k2=v2.") private List properties = new ArrayList<>(); - @Parameter(names = { "-k", "--key"}, description = "message key to add ") + @Option(names = { "-k", "--key"}, description = "Partitioning key to add to each message") private String key; - - @Parameter(names = { "-vs", "--value-schema"}, description = "Schema type (can be bytes,avro,json,string...)") + @Option(names = { "-kvk", "--key-value-key"}, description = "Value to add as message key in KeyValue schema") + private String keyValueKey; + @Option(names = {"-kvkf", "--key-value-key-file"}, + description = "Path to file containing the value to add as message key in KeyValue schema. " + + "JSON and AVRO files are supported.") + private String keyValueKeyFile; + + @Option(names = { "-vs", "--value-schema"}, description = "Schema type (can be bytes,avro,json,string...)") private String valueSchema = "bytes"; - @Parameter(names = { "-ks", "--key-schema"}, description = "Schema type (can be bytes,avro,json,string...)") + @Option(names = { "-ks", "--key-schema"}, description = "Schema type (can be bytes,avro,json,string...)") private String keySchema = "string"; - @Parameter(names = { "-kvet", "--key-value-encoding-type"}, + @Option(names = { "-kvet", "--key-value-encoding-type"}, description = "Key Value Encoding Type (it can be separated or inline)") private String keyValueEncodingType = null; - @Parameter(names = { "-ekn", "--encryption-key-name" }, description = "The public key name to encrypt payload") + @Option(names = { "-ekn", "--encryption-key-name" }, description = "The public key name to encrypt payload") private String encKeyName = null; - @Parameter(names = { "-ekv", + @Option(names = { "-ekv", "--encryption-key-value" }, description = "The URI of public key to encrypt payload, for example " + "file:///path/to/public.key or data:application/x-pem-file;base64,*****") private String encKeyValue = null; - @Parameter(names = { "-dr", + @Option(names = { "-dr", "--disable-replication" }, description = "Disable geo-replication for messages.") private boolean disableReplication = false; @@ -153,7 +171,6 @@ public CmdProduce() { /** * Set Pulsar client configuration. - * */ public void updateConfig(ClientBuilder newBuilder, Authentication authentication, String serviceURL) { this.clientBuilder = newBuilder; @@ -170,11 +187,19 @@ public void updateConfig(ClientBuilder newBuilder, Authentication authentication * * @return list of message bodies */ - private List generateMessageBodies(List stringMessages, List messageFileNames) { + static List generateMessageBodies(List stringMessages, List messageFileNames, + Schema schema) { List messageBodies = new ArrayList<>(); for (String m : stringMessages) { - messageBodies.add(m.getBytes()); + if (schema.getSchemaInfo().getType() == SchemaType.AVRO) { + // JSON TO AVRO + org.apache.avro.Schema avroSchema = ((Optional) schema.getNativeSchema()).get(); + byte[] encoded = jsonToAvro(m, avroSchema); + messageBodies.add(encoded); + } else { + messageBodies.add(m.getBytes()); + } } try { @@ -189,6 +214,32 @@ private List generateMessageBodies(List stringMessages, List reader = new GenericDatumReader<>(avroSchema); + JsonDecoder jsonDecoder = DecoderFactory.get().jsonDecoder(avroSchema, m); + GenericDatumWriter writer = new GenericDatumWriter<>(avroSchema); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + Encoder e = EncoderFactory.get().binaryEncoder(out, null); + Object datum = null; + while (true) { + try { + datum = reader.read(datum, jsonDecoder); + } catch (EOFException eofException) { + break; + } + writer.write(datum, e); + e.flush(); + } + return out.toByteArray(); + } catch (IOException e) { + throw new RuntimeException("Cannot convert " + m + " to AVRO " + e.getMessage(), e); + } + } + + @Spec + private CommandSpec commandSpec; + /** * Run the producer. * @@ -196,19 +247,18 @@ private List generateMessageBodies(List stringMessages, List 0){ + if (messages.size() > 0) { messages = messages.stream().map(str -> str.split(separator)).flatMap(Stream::of).toList(); } if (messages.size() == 0 && messageFileNames.size() == 0) { - throw (new ParameterException("Please supply message content with either --messages or --files")); + throw new CommandLine.ParameterException(commandSpec.commandLine(), + "Please supply message content with either --messages or --files"); } if (keyValueEncodingType == null) { @@ -219,7 +269,7 @@ public int run() throws PulsarClientException { case KEY_VALUE_ENCODING_TYPE_INLINE: break; default: - throw (new ParameterException("--key-value-encoding-type " + throw (new IllegalArgumentException("--key-value-encoding-type " + keyValueEncodingType + " is not valid, only 'separated' or 'inline'")); } } @@ -228,11 +278,9 @@ public int run() throws PulsarClientException { if (totalMessages > MAX_MESSAGES) { String msg = "Attempting to send " + totalMessages + " messages. Please do not send more than " + MAX_MESSAGES + " messages"; - throw new ParameterException(msg); + throw new IllegalArgumentException(msg); } - String topic = this.mainOptions.get(0); - if (this.serviceURL.startsWith("ws")) { return publishToWebSocket(topic); } else { @@ -258,8 +306,10 @@ private int publish(String topic) { producerBuilder.defaultCryptoKeyReader(this.encKeyValue); } try (Producer producer = producerBuilder.create();) { - - List messageBodies = generateMessageBodies(this.messages, this.messageFileNames); + Schema schemaForPayload = schema.getSchemaInfo().getType() == SchemaType.KEY_VALUE + ? ((KeyValueSchema) schema).getValueSchema() : schema; + List messageBodies = generateMessageBodies(this.messages, this.messageFileNames, + schemaForPayload); RateLimiter limiter = (this.publishRate > 0) ? RateLimiter.create(this.publishRate) : null; Map kvMap = new HashMap<>(); @@ -268,6 +318,25 @@ private int publish(String topic) { kvMap.put(kv[0], kv[1]); } + final byte[] keyValueKeyBytes; + if (this.keyValueKey != null) { + if (keyValueEncodingType == KEY_VALUE_ENCODING_TYPE_NOT_SET) { + throw new IllegalArgumentException( + "Key value encoding type must be set when using --key-value-key"); + } + keyValueKeyBytes = this.keyValueKey.getBytes(StandardCharsets.UTF_8); + } else if (this.keyValueKeyFile != null) { + if (keyValueEncodingType == KEY_VALUE_ENCODING_TYPE_NOT_SET) { + throw new IllegalArgumentException( + "Key value encoding type must be set when using --key-value-key-file"); + } + keyValueKeyBytes = Files.readAllBytes(Paths.get(this.keyValueKeyFile)); + } else if (this.key != null) { + keyValueKeyBytes = this.key.getBytes(StandardCharsets.UTF_8); + } else { + keyValueKeyBytes = null; + } + for (int i = 0; i < this.numTimesProduce; i++) { for (byte[] content : messageBodies) { if (limiter != null) { @@ -290,8 +359,7 @@ private int publish(String topic) { case KEY_VALUE_ENCODING_TYPE_SEPARATED: case KEY_VALUE_ENCODING_TYPE_INLINE: KeyValue kv = new KeyValue<>( - // TODO: support AVRO encoded key - key != null ? key.getBytes(StandardCharsets.UTF_8) : null, + keyValueKeyBytes, content); message.value(kv); break; @@ -341,7 +409,7 @@ private static Schema buildComponentSchema(String schema) { Schema base; switch (schema) { case "string": - base = Schema.STRING; + base = Schema.STRING; break; case "bytes": // no need for wrappers @@ -432,7 +500,7 @@ private int publishToWebSocket(String topic) { } try { - List messageBodies = generateMessageBodies(this.messages, this.messageFileNames); + List messageBodies = generateMessageBodies(this.messages, this.messageFileNames, Schema.BYTES); RateLimiter limiter = (this.publishRate > 0) ? RateLimiter.create(this.publishRate) : null; for (int i = 0; i < this.numTimesProduce; i++) { int index = i * 10; diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdRead.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdRead.java index 4ad8a5293f6e1..529d1d9c41272 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdRead.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdRead.java @@ -19,22 +19,18 @@ package org.apache.pulsar.client.cli; import static org.apache.commons.lang3.StringUtils.isNotBlank; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.RateLimiter; import java.io.IOException; import java.net.URI; -import java.util.ArrayList; import java.util.Base64; -import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.pulsar.client.api.AuthenticationDataProvider; +import org.apache.pulsar.client.api.ConsumerCryptoFailureAction; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.PulsarClient; @@ -47,63 +43,71 @@ import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.websocket.client.ClientUpgradeRequest; import org.eclipse.jetty.websocket.client.WebSocketClient; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; /** * pulsar-client read command implementation. - * */ -@Parameters(commandDescription = "Read messages from a specified topic") +@Command(description = "Read messages from a specified topic") public class CmdRead extends AbstractCmdConsume { private static final Pattern MSG_ID_PATTERN = Pattern.compile("^(-?[1-9][0-9]*|0):(-?[1-9][0-9]*|0)$"); - @Parameter(description = "TopicName", required = true) - private List mainOptions = new ArrayList(); + @Parameters(description = "TopicName", arity = "1") + private String topic; - @Parameter(names = { "-m", "--start-message-id" }, + @Option(names = { "-m", "--start-message-id" }, description = "Initial reader position, it can be 'latest', 'earliest' or ':'") private String startMessageId = "latest"; - @Parameter(names = { "-i", "--start-message-id-inclusive" }, + @Option(names = { "-i", "--start-message-id-inclusive" }, description = "Whether to include the position specified by -m option.") private boolean startMessageIdInclusive = false; - @Parameter(names = { "-n", + @Option(names = { "-n", "--num-messages" }, description = "Number of messages to read, 0 means to read forever.") private int numMessagesToRead = 1; - @Parameter(names = { "--hex" }, description = "Display binary messages in hex.") + @Option(names = { "--hex" }, description = "Display binary messages in hex.") private boolean displayHex = false; - @Parameter(names = { "--hide-content" }, description = "Do not write the message to console.") + @Option(names = { "--hide-content" }, description = "Do not write the message to console.") private boolean hideContent = false; - @Parameter(names = { "-r", "--rate" }, description = "Rate (in msg/sec) at which to read, " + @Option(names = { "-r", "--rate" }, description = "Rate (in msg/sec) at which to read, " + "value 0 means to read messages as fast as possible.") private double readRate = 0; - @Parameter(names = {"-q", "--queue-size"}, description = "Reader receiver queue size.") + @Option(names = { "-q", "--queue-size" }, description = "Reader receiver queue size.") private int receiverQueueSize = 0; - @Parameter(names = { "-mc", "--max_chunked_msg" }, description = "Max pending chunk messages") + @Option(names = { "-mc", "--max_chunked_msg" }, description = "Max pending chunk messages") private int maxPendingChunkedMessage = 0; - @Parameter(names = { "-ac", + @Option(names = { "-ac", "--auto_ack_chunk_q_full" }, description = "Auto ack for oldest message on queue is full") private boolean autoAckOldestChunkedMessageOnQueueFull = false; - @Parameter(names = { "-ekv", + @Option(names = { "-ekv", "--encryption-key-value" }, description = "The URI of private key to decrypt payload, for example " - + "file:///path/to/private.key or data:application/x-pem-file;base64,*****") + + "file:///path/to/private.key or data:application/x-pem-file;base64,*****") private String encKeyValue; - @Parameter(names = { "-st", "--schema-type"}, + @Option(names = { "-st", "--schema-type" }, description = "Set a schema type on the reader, it can be 'bytes' or 'auto_consume'") private String schemaType = "bytes"; - @Parameter(names = { "-pm", "--pool-messages" }, description = "Use the pooled message", arity = 1) + @Option(names = { "-pm", "--pool-messages" }, description = "Use the pooled message", arity = "1") private boolean poolMessages = true; + @Option(names = { "-ca", "--crypto-failure-action" }, description = "Crypto Failure Action") + private ConsumerCryptoFailureAction cryptoFailureAction = ConsumerCryptoFailureAction.FAIL; + + @Option(names = { "-mp", "--print-metadata" }, description = "Message metadata") + private boolean printMetadata = false; + public CmdRead() { // Do nothing super(); @@ -115,14 +119,10 @@ public CmdRead() { * @return 0 for success, < 0 otherwise */ public int run() throws PulsarClientException, IOException { - if (mainOptions.size() != 1) { - throw (new ParameterException("Please provide one and only one topic name.")); - } if (this.numMessagesToRead < 0) { - throw (new ParameterException("Number of messages should be zero or positive.")); + throw (new IllegalArgumentException("Number of messages should be zero or positive.")); } - String topic = this.mainOptions.get(0); if (this.serviceURL.startsWith("ws")) { return readFromWebSocket(topic); @@ -160,6 +160,7 @@ private int read(String topic) { } builder.autoAckOldestChunkedMessageOnQueueFull(this.autoAckOldestChunkedMessageOnQueueFull); + builder.cryptoFailureAction(cryptoFailureAction); if (isNotBlank(this.encKeyValue)) { builder.defaultCryptoKeyReader(this.encKeyValue); @@ -180,7 +181,7 @@ private int read(String topic) { numMessagesRead += 1; if (!hideContent) { System.out.println(MESSAGE_BOUNDARY); - String output = this.interpretMessage(msg, displayHex); + String output = this.interpretMessage(msg, displayHex, printMetadata); System.out.println(output); } else if (numMessagesRead % 1000 == 0) { System.out.println("Received " + numMessagesRead + " messages"); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/NoSplitter.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/ProxyProtocolConverter.java similarity index 63% rename from pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/NoSplitter.java rename to pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/ProxyProtocolConverter.java index 0429970366504..3aa731db138ba 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/NoSplitter.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/ProxyProtocolConverter.java @@ -18,21 +18,18 @@ */ package org.apache.pulsar.client.cli; -import com.beust.jcommander.converters.IParameterSplitter; -import java.util.LinkedList; -import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.client.api.ProxyProtocol; +import picocli.CommandLine.ITypeConverter; -public class NoSplitter implements IParameterSplitter { +public class ProxyProtocolConverter implements ITypeConverter { - /* - * (non-Javadoc) - * - * @see com.beust.jcommander.converters.IParameterSplitter#split(java.lang.String) - */ @Override - public List split(final String value) { - final List result = new LinkedList<>(); - result.add(value); - return result; + public ProxyProtocol convert(String value) throws Exception { + String proxyProtocolString = StringUtils.trimToNull(value); + if (proxyProtocolString != null) { + return ProxyProtocol.valueOf(proxyProtocolString.toUpperCase()); + } + return null; } -} \ No newline at end of file +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/PulsarClientPropertiesProvider.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/PulsarClientPropertiesProvider.java new file mode 100644 index 0000000000000..accc5df8595d5 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/PulsarClientPropertiesProvider.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.cli; + +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import java.util.Properties; +import picocli.CommandLine.PropertiesDefaultProvider; + +class PulsarClientPropertiesProvider extends PropertiesDefaultProvider { + private static final String brokerServiceUrlKey = "brokerServiceUrl"; + private final Properties properties; + + private PulsarClientPropertiesProvider(Properties properties) { + super(properties); + this.properties = properties; + } + + static PulsarClientPropertiesProvider create(Properties properties) { + Properties clone = (Properties) properties.clone(); + String brokerServiceUrl = clone.getProperty(brokerServiceUrlKey); + if (isBlank(brokerServiceUrl)) { + String serviceUrl = clone.getProperty("webServiceUrl"); + if (isBlank(serviceUrl)) { + // fallback to previous-version serviceUrl property to maintain backward-compatibility + serviceUrl = clone.getProperty("serviceUrl"); + } + if (isNotBlank(serviceUrl)) { + clone.put(brokerServiceUrlKey, serviceUrl); + } + } + return new PulsarClientPropertiesProvider(clone); + } + + String getServiceUrl() { + return properties.getProperty(brokerServiceUrlKey); + } +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/PulsarClientTool.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/PulsarClientTool.java index c64d80f380b9f..98f129441733d 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/PulsarClientTool.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/PulsarClientTool.java @@ -18,20 +18,14 @@ */ package org.apache.pulsar.client.cli; -import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; -import com.beust.jcommander.DefaultUsageFormatter; -import com.beust.jcommander.IUsageFormatter; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; +import com.google.common.annotations.VisibleForTesting; import java.io.FileInputStream; import java.util.Arrays; import java.util.Properties; import lombok.Getter; -import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.PulsarVersion; +import lombok.SneakyThrows; +import org.apache.pulsar.cli.converters.picocli.ByteUnitToLongConverter; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.ClientBuilder; @@ -39,46 +33,69 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException.UnsupportedAuthenticationException; import org.apache.pulsar.client.api.SizeUnit; - - -public class PulsarClientTool { +import org.apache.pulsar.internal.CommandHook; +import org.apache.pulsar.internal.CommanderFactory; +import picocli.CommandLine; +import picocli.CommandLine.ArgGroup; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.ScopeType; + +@Command( + name = "pulsar-client", + mixinStandardHelpOptions = true, + versionProvider = PulsarVersionProvider.class, + scope = ScopeType.INHERIT +) +public class PulsarClientTool implements CommandHook { + + private PulsarClientPropertiesProvider pulsarClientPropertiesProvider; @Getter - @Parameters(commandDescription = "Produce or consume messages on a specified topic") + @Command(description = "Produce or consume messages on a specified topic") public static class RootParams { - @Parameter(names = { "--url" }, description = "Broker URL to which to connect.") + @Option(names = {"--url"}, descriptionKey = "brokerServiceUrl", + description = "Broker URL to which to connect.") String serviceURL = null; - @Parameter(names = { "--proxy-url" }, description = "Proxy-server URL to which to connect.") + @Option(names = {"--proxy-url"}, descriptionKey = "proxyServiceUrl", + description = "Proxy-server URL to which to connect.") String proxyServiceURL = null; - @Parameter(names = { "--proxy-protocol" }, description = "Proxy protocol to select type of routing at proxy.") + @Option(names = {"--proxy-protocol"}, descriptionKey = "proxyProtocol", + description = "Proxy protocol to select type of routing at proxy.", + converter = ProxyProtocolConverter.class) ProxyProtocol proxyProtocol = null; - @Parameter(names = { "--auth-plugin" }, description = "Authentication plugin class name.") + @Option(names = {"--auth-plugin"}, descriptionKey = "authPlugin", + description = "Authentication plugin class name.") String authPluginClassName = null; - @Parameter(names = { "--listener-name" }, description = "Listener name for the broker.") + @Option(names = {"--listener-name"}, description = "Listener name for the broker.") String listenerName = null; - @Parameter( - names = { "--auth-params" }, - description = "Authentication parameters, whose format is determined by the implementation " - + "of method `configure` in authentication plugin class, for example \"key1:val1,key2:val2\" " - + "or \"{\"key1\":\"val1\",\"key2\":\"val2\"}\".") + @Option( + names = {"--auth-params"}, + descriptionKey = "authParams", + description = "Authentication parameters, whose format is determined by the implementation " + + "of method `configure` in authentication plugin class, for example \"key1:val1,key2:val2\" " + + "or \"{\"key1\":\"val1\",\"key2\":\"val2\"}\".") String authParams = null; - @Parameter(names = { "-v", "--version" }, description = "Get version of pulsar client") - boolean version; - - @Parameter(names = { "-h", "--help", }, help = true, description = "Show this help.") - boolean help; - - @Parameter(names = { "--tlsTrustCertsFilePath" }, description = "File path to client trust certificates") + @Option(names = {"--tlsTrustCertsFilePath"}, + descriptionKey = "tlsTrustCertsFilePath", + description = "File path to client trust certificates") String tlsTrustCertsFilePath; + + @Option(names = {"-ml", "--memory-limit"}, description = "Configure the Pulsar client memory limit " + + "(eg: 32M, 64M)", descriptionKey = "memoryLimit", + converter = ByteUnitToLongConverter.class) + long memoryLimit = 0L; } - protected RootParams rootParams; + + @ArgGroup(exclusive = false) + protected RootParams rootParams = new RootParams(); boolean tlsAllowInsecureConnection; boolean tlsEnableHostnameVerification; @@ -94,17 +111,34 @@ public static class RootParams { String tlsKeyStoreType; String tlsKeyStorePath; String tlsKeyStorePassword; + String sslFactoryPlugin; + String sslFactoryPluginParams; - protected JCommander jcommander; - IUsageFormatter usageFormatter; + protected final CommandLine commander; protected CmdProduce produceCommand; protected CmdConsume consumeCommand; protected CmdRead readCommand; CmdGenerateDocumentation generateDocumentation; public PulsarClientTool(Properties properties) { - rootParams = new RootParams(); - initRootParamsFromProperties(properties); + // Use -v instead -V + System.setProperty("picocli.version.name.0", "-v"); + commander = CommanderFactory.createRootCommanderWithHook(this, null); + initCommander(properties); + } + + @Override + @SneakyThrows + public int preRun() { + return updateConfig(); + } + + protected void initCommander(Properties properties) { + produceCommand = new CmdProduce(); + consumeCommand = new CmdConsume(); + readCommand = new CmdRead(); + generateDocumentation = new CmdGenerateDocumentation(); + this.tlsAllowInsecureConnection = Boolean .parseBoolean(properties.getProperty("tlsAllowInsecureConnection", "false")); this.tlsEnableHostnameVerification = Boolean @@ -120,52 +154,24 @@ public PulsarClientTool(Properties properties) { this.tlsKeyStorePassword = properties.getProperty("tlsKeyStorePassword"); this.tlsKeyFilePath = properties.getProperty("tlsKeyFilePath"); this.tlsCertificateFilePath = properties.getProperty("tlsCertificateFilePath"); - - initJCommander(); + this.sslFactoryPlugin = properties.getProperty("sslFactoryPlugin"); + this.sslFactoryPluginParams = properties.getProperty("sslFactoryPluginParams"); + + pulsarClientPropertiesProvider = PulsarClientPropertiesProvider.create(properties); + commander.setDefaultValueProvider(pulsarClientPropertiesProvider); + commander.addSubcommand("produce", produceCommand); + commander.addSubcommand("consume", consumeCommand); + commander.addSubcommand("read", readCommand); + commander.addSubcommand("generate_documentation", generateDocumentation); } - protected void initJCommander() { - produceCommand = new CmdProduce(); - consumeCommand = new CmdConsume(); - readCommand = new CmdRead(); - generateDocumentation = new CmdGenerateDocumentation(); - - this.jcommander = new JCommander(); - this.usageFormatter = new DefaultUsageFormatter(this.jcommander); - jcommander.setProgramName("pulsar-client"); - jcommander.addObject(rootParams); - jcommander.addCommand("produce", produceCommand); - jcommander.addCommand("consume", consumeCommand); - jcommander.addCommand("read", readCommand); - jcommander.addCommand("generate_documentation", generateDocumentation); + protected void addCommand(String name, Object cmd) { + commander.addSubcommand(name, cmd); } - protected void initRootParamsFromProperties(Properties properties) { - this.rootParams.serviceURL = isNotBlank(properties.getProperty("brokerServiceUrl")) - ? properties.getProperty("brokerServiceUrl") : properties.getProperty("webServiceUrl"); - // fallback to previous-version serviceUrl property to maintain backward-compatibility - if (isBlank(this.rootParams.serviceURL)) { - this.rootParams.serviceURL = properties.getProperty("serviceUrl"); - } - this.rootParams.authPluginClassName = properties.getProperty("authPlugin"); - this.rootParams.authParams = properties.getProperty("authParams"); - this.rootParams.tlsTrustCertsFilePath = properties.getProperty("tlsTrustCertsFilePath"); - this.rootParams.proxyServiceURL = StringUtils.trimToNull(properties.getProperty("proxyServiceUrl")); - String proxyProtocolString = StringUtils.trimToNull(properties.getProperty("proxyProtocol")); - if (proxyProtocolString != null) { - try { - this.rootParams.proxyProtocol = ProxyProtocol.valueOf(proxyProtocolString.toUpperCase()); - } catch (IllegalArgumentException e) { - System.out.println("Incorrect proxyProtocol name '" + proxyProtocolString + "'"); - e.printStackTrace(); - System.exit(1); - } - } - } - - private void updateConfig() throws UnsupportedAuthenticationException { + private int updateConfig() throws UnsupportedAuthenticationException { ClientBuilder clientBuilder = PulsarClient.builder() - .memoryLimit(0, SizeUnit.BYTES); + .memoryLimit(rootParams.memoryLimit, SizeUnit.BYTES); Authentication authentication = null; if (isNotBlank(this.rootParams.authPluginClassName)) { authentication = AuthenticationFactory.create(rootParams.authPluginClassName, rootParams.authParams); @@ -190,73 +196,24 @@ private void updateConfig() throws UnsupportedAuthenticationException { .tlsKeyStorePath(tlsKeyStorePath) .tlsKeyStorePassword(tlsKeyStorePassword); + clientBuilder.sslFactoryPlugin(sslFactoryPlugin) + .sslFactoryPluginParams(sslFactoryPluginParams); + if (isNotBlank(rootParams.proxyServiceURL)) { if (rootParams.proxyProtocol == null) { - System.out.println("proxy-protocol must be provided with proxy-url"); - System.exit(1); + commander.getErr().println("proxy-protocol must be provided with proxy-url"); + return 1; } clientBuilder.proxyServiceUrl(rootParams.proxyServiceURL, rootParams.proxyProtocol); } this.produceCommand.updateConfig(clientBuilder, authentication, this.rootParams.serviceURL); this.consumeCommand.updateConfig(clientBuilder, authentication, this.rootParams.serviceURL); this.readCommand.updateConfig(clientBuilder, authentication, this.rootParams.serviceURL); + return 0; } public int run(String[] args) { - try { - jcommander.parse(args); - - if (isBlank(this.rootParams.serviceURL)) { - jcommander.usage(); - return -1; - } - - if (rootParams.version) { - System.out.println("Current version of pulsar client is: " + PulsarVersion.getVersion()); - return 0; - } - - if (rootParams.help) { - jcommander.usage(); - return 0; - } - - try { - this.updateConfig(); // If the --url, --auth-plugin, or --auth-params parameter are not specified, - // it will default to the values passed in by the constructor - } catch (UnsupportedAuthenticationException exp) { - System.out.println("Failed to load an authentication plugin"); - exp.printStackTrace(); - return -1; - } - - String chosenCommand = jcommander.getParsedCommand(); - if ("produce".equals(chosenCommand)) { - return produceCommand.run(); - } else if ("consume".equals(chosenCommand)) { - return consumeCommand.run(); - } else if ("read".equals(chosenCommand)) { - return readCommand.run(); - } else if ("generate_documentation".equals(chosenCommand)) { - return generateDocumentation.run(); - } else { - jcommander.usage(); - return -1; - } - } catch (Exception e) { - System.out.println(e.getMessage()); - String chosenCommand = jcommander.getParsedCommand(); - if (e instanceof ParameterException) { - try { - usageFormatter.usage(chosenCommand); - } catch (ParameterException noCmd) { - e.printStackTrace(); - } - } else { - e.printStackTrace(); - } - return -1; - } + return commander.execute(args); } public static void main(String[] args) throws Exception { @@ -277,6 +234,28 @@ public static void main(String[] args) throws Exception { int exitCode = clientTool.run(Arrays.copyOfRange(args, 1, args.length)); System.exit(exitCode); + } + + @VisibleForTesting + public void replaceProducerCommand(CmdProduce object) { + this.produceCommand = object; + if (commander.getSubcommands().containsKey("produce")) { + commander.getCommandSpec().removeSubcommand("produce"); + } + commander.addSubcommand("produce", this.produceCommand); + } + + @VisibleForTesting + CommandLine getCommander() { + return commander; + } + + // The following methods are used for Pulsar shell. + protected void setCommandName(String name) { + commander.setCommandName(name); + } + protected String getServiceUrl() { + return pulsarClientPropertiesProvider.getServiceUrl(); } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/PulsarVersionProvider.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/PulsarVersionProvider.java new file mode 100644 index 0000000000000..45996b92ff706 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/PulsarVersionProvider.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.cli; + +import org.apache.pulsar.PulsarVersion; +import picocli.CommandLine.IVersionProvider; + +public class PulsarVersionProvider implements IVersionProvider { + @Override + public String[] getVersion() { + return new String[]{"Current version of pulsar client is: " + PulsarVersion.getVersion()}; + } +} diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ClassLoaderSwitcher.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/internal/CommandDescriptionUtil.java similarity index 54% rename from pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ClassLoaderSwitcher.java rename to pulsar-client-tools/src/main/java/org/apache/pulsar/internal/CommandDescriptionUtil.java index 55cb9198da2bc..ae00aebcb1817 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ClassLoaderSwitcher.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/internal/CommandDescriptionUtil.java @@ -16,22 +16,25 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.broker; +package org.apache.pulsar.internal; -/** - * Help to switch the class loader of current thread to the NarClassLoader, and change it back when it's done. - * With the help of try-with-resources statement, the code would be cleaner than using try finally every time. - */ -public class ClassLoaderSwitcher implements AutoCloseable { - private final ClassLoader prevClassLoader; +import picocli.CommandLine; +import picocli.CommandLine.Model.ArgSpec; - public ClassLoaderSwitcher(ClassLoader classLoader) { - prevClassLoader = Thread.currentThread().getContextClassLoader(); - Thread.currentThread().setContextClassLoader(classLoader); +public class CommandDescriptionUtil { + public static String getCommandDescription(CommandLine commandLine) { + String[] description = commandLine.getCommandSpec().usageMessage().description(); + if (description != null && description.length != 0) { + return description[0]; + } + return ""; } - @Override - public void close() { - Thread.currentThread().setContextClassLoader(prevClassLoader); + public static String getArgDescription(ArgSpec argSpec) { + String[] description = argSpec.description(); + if (description != null && description.length != 0) { + return description[0]; + } + return ""; } -} \ No newline at end of file +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/internal/CommandHook.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/internal/CommandHook.java new file mode 100644 index 0000000000000..879c82a75816a --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/internal/CommandHook.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.internal; + +/** + * CommandHook is used for injecting or configuring parameters during different stages of command execution. + */ +public interface CommandHook { + /** + * The preRun hook is used to execute commands before. + * + * @return If the return value is 0, continue to the next stage. + */ + default int preRun() { + return 0; + } +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/internal/CommanderFactory.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/internal/CommanderFactory.java new file mode 100644 index 0000000000000..b9e3fbe90cf22 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/internal/CommanderFactory.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.internal; + +import java.io.PrintWriter; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.admin.PulsarAdminException.ConnectException; +import picocli.CommandLine; +import picocli.CommandLine.ExecutionException; +import picocli.CommandLine.IDefaultValueProvider; +import picocli.CommandLine.IExecutionStrategy; +import picocli.CommandLine.ParameterException; +import picocli.CommandLine.ParseResult; + +public class CommanderFactory { + static class CommandExecutionStrategy implements IExecutionStrategy { + private int preRun(ParseResult parseResult) { + Object userObject = parseResult.commandSpec().userObject(); + if (userObject instanceof CommandHook) { + int exitCode = ((CommandHook) userObject).preRun(); + if (exitCode != 0) { + return exitCode; + } + } + + if (parseResult.hasSubcommand()) { + return preRun(parseResult.subcommand()); + } + + return 0; + } + + + @Override + public int execute(ParseResult parseResult) throws ExecutionException, ParameterException { + int preRunCode = preRun(parseResult); + if (preRunCode != 0) { + return preRunCode; + } + + return new CommandLine.RunLast().execute(parseResult); + } + } + + /** + * createRootCommanderWithHook is used for the root command, which supports the hook feature. + * + * @param object Command class or object. + * @param defaultValueProvider Default value provider of command. + * @return Picocli commander. + */ + public static CommandLine createRootCommanderWithHook(Object object, IDefaultValueProvider defaultValueProvider) { + CommandLine commander = new CommandLine(object); + commander.setExecutionStrategy(new CommandExecutionStrategy()); + commander.setExecutionExceptionHandler((ex, commandLine, parseResult) -> { + PrintWriter errPrinter = commandLine.getErr(); + if (ex instanceof ConnectException) { + errPrinter.println(ex.getMessage()); + errPrinter.println(); + errPrinter.println("Error connecting to Pulsar"); + return 1; + } else if (ex instanceof PulsarAdminException) { + errPrinter.println(((PulsarAdminException) ex).getHttpError()); + errPrinter.println(); + errPrinter.println("Reason: " + ex.getMessage()); + return 1; + } + throw ex; + }); + if (defaultValueProvider != null) { + commander.setDefaultValueProvider(defaultValueProvider); + } + return commander; + } +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/internal/InnerClassFactory.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/internal/InnerClassFactory.java new file mode 100644 index 0000000000000..64c946ba72e3b --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/internal/InnerClassFactory.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.internal; + +import java.lang.reflect.Constructor; +import picocli.CommandLine; +import picocli.CommandLine.IFactory; +import picocli.CommandLine.InitializationException; + +// Copied from https://github.com/remkop/picocli/blob/v4.7.5/src/test/java/picocli/InnerClassFactory.java +// The default Picocli factory doesn't support create non-static inner class. +public class InnerClassFactory implements IFactory { + private final Object outer; + private final IFactory defaultFactory = CommandLine.defaultFactory(); + + public InnerClassFactory(Object outer) { + this.outer = outer; + } + + public K create(final Class cls) throws Exception { + try { + return defaultFactory.create(cls); + } catch (Exception ex0) { + try { + Constructor constructor = cls.getDeclaredConstructor(outer.getClass()); + return constructor.newInstance(outer); + } catch (Exception ex) { + try { + @SuppressWarnings("deprecation") // Class.newInstance is deprecated in Java 9 + K result = cls.newInstance(); + return result; + } catch (Exception ex2) { + try { + Constructor constructor = cls.getDeclaredConstructor(); + return constructor.newInstance(); + } catch (Exception ex3) { + throw new InitializationException("Could not instantiate " + cls.getName() + + " either with or without construction parameter " + outer + ": " + ex, ex); + } + } + } + } + } +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/ShellCommandsProvider.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/internal/ShellCommandsProvider.java similarity index 58% rename from pulsar-client-tools/src/main/java/org/apache/pulsar/shell/ShellCommandsProvider.java rename to pulsar-client-tools/src/main/java/org/apache/pulsar/internal/ShellCommandsProvider.java index 069ee01d4dcd8..caae4eea01547 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/ShellCommandsProvider.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/internal/ShellCommandsProvider.java @@ -16,10 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.shell; +package org.apache.pulsar.internal; -import com.beust.jcommander.JCommander; -import java.util.Properties; +import picocli.CommandLine; /** * Commands provider for Pulsar shell. @@ -47,32 +46,8 @@ public interface ShellCommandsProvider { String getAdminUrl(); /** - * Init state before a command is executed. - * If the implementing class rely on JCommander, it's suggested to not recycle JCommander - * objects because they are meant to single-shot usage. - * @param properties + * Return commander instance. + * @return Non-null. */ - void setupState(Properties properties); - - /** - * Cleanup state after a command is executed. - * If the implementing class rely on JCommander, it's suggested to not recycle JCommander - * objects because they are meant to single-shot usage. - * @param properties - */ - void cleanupState(Properties properties); - - /** - * Return JCommander instance, if exists. - * @return - */ - JCommander getJCommander(); - - /** - * Run command for the passed args. - * - * @param args arguments for the command. Note that the first word of the user command is omitted. - * @throws Exception if any error occurs. The shell session will not be closed. - */ - boolean runCommand(String[] args) throws Exception; + CommandLine getCommander(); } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/internal/package-info.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/internal/package-info.java new file mode 100644 index 0000000000000..99aa7f7e33675 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/internal/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.internal; diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/AdminShell.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/AdminShell.java index e139697860229..35d676eefabfb 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/AdminShell.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/AdminShell.java @@ -18,19 +18,22 @@ */ package org.apache.pulsar.shell; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameters; +import com.google.common.annotations.VisibleForTesting; import java.util.Properties; import org.apache.pulsar.admin.cli.PulsarAdminTool; +import org.apache.pulsar.internal.ShellCommandsProvider; +import picocli.CommandLine; +import picocli.CommandLine.Command; /** * Pulsar Admin tool extension for Pulsar shell. */ -@Parameters(commandDescription = "Admin console") +@Command(description = "Admin console") public class AdminShell extends PulsarAdminTool implements ShellCommandsProvider { public AdminShell(Properties properties) throws Exception { super(properties); + setCommandName(getName()); } @Override @@ -45,30 +48,16 @@ public String getServiceUrl() { @Override public String getAdminUrl() { - return rootParams.getServiceUrl(); + return super.getAdminUrl(); } @Override - public void setupState(Properties properties) { - getJCommander().setProgramName(getName()); - setupCommands(); + public CommandLine getCommander() { + return commander; } - @Override - public JCommander getJCommander() { - return jcommander; - } - - @Override - public void cleanupState(Properties properties) { - rootParams = new RootParams(); - initRootParamsFromProperties(properties); - initJCommander(); - } - - - @Override - public boolean runCommand(String[] args) throws Exception { + @VisibleForTesting + boolean runCommand(String[] args) { return run(args); } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/ClientShell.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/ClientShell.java index 6ba96e046374a..7cc4f825c85a0 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/ClientShell.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/ClientShell.java @@ -18,19 +18,21 @@ */ package org.apache.pulsar.shell; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameters; import java.util.Properties; import org.apache.pulsar.client.cli.PulsarClientTool; +import org.apache.pulsar.internal.ShellCommandsProvider; +import picocli.CommandLine; +import picocli.CommandLine.Command; /** * Pulsar Client tool extension for Pulsar shell. */ -@Parameters(commandDescription = "Produce or consume messages on a specified topic") +@Command(description = "Produce or consume messages on a specified topic") public class ClientShell extends PulsarClientTool implements ShellCommandsProvider { public ClientShell(Properties properties) { super(properties); + setCommandName(getName()); } @Override @@ -40,7 +42,7 @@ public String getName() { @Override public String getServiceUrl() { - return rootParams.getServiceURL(); + return super.getServiceUrl(); } @Override @@ -49,25 +51,7 @@ public String getAdminUrl() { } @Override - public void setupState(Properties properties) { - getJCommander().setProgramName(getName()); - } - - @Override - public void cleanupState(Properties properties) { - rootParams = new RootParams(); - initRootParamsFromProperties(properties); - initJCommander(); - } - - @Override - public JCommander getJCommander() { - return jcommander; - } - - @Override - public boolean runCommand(String[] args) throws Exception { - final int returnCode = run(args); - return returnCode == 0; + public CommandLine getCommander() { + return commander; } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/ConfigShell.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/ConfigShell.java index b957fce70adec..de785c226f3c0 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/ConfigShell.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/ConfigShell.java @@ -19,10 +19,6 @@ package org.apache.pulsar.shell; import static org.apache.pulsar.shell.config.ConfigStore.DEFAULT_CONFIG; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import java.io.ByteArrayOutputStream; @@ -33,22 +29,30 @@ import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.util.ArrayList; import java.util.Base64; -import java.util.HashMap; +import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Properties; +import java.util.concurrent.Callable; import java.util.stream.Collectors; import lombok.Getter; import lombok.SneakyThrows; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.internal.InnerClassFactory; +import org.apache.pulsar.internal.ShellCommandsProvider; import org.apache.pulsar.shell.config.ConfigStore; +import org.apache.pulsar.shell.config.ConfigStore.ConfigEntry; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; /** * Shell commands to manage shell configurations. */ -@Parameters(commandDescription = "Manage Pulsar shell configurations.") +@Command(description = "Manage Pulsar shell configurations.") public class ConfigShell implements ShellCommandsProvider { private static final String LOCAL_FILES_BASE_DIR = System.getProperty("pulsar.shell.working.dir"); @@ -65,25 +69,43 @@ static File resolveLocalFile(String input, String baseDir) { return file; } + private interface RunnableWithResult extends Callable { + boolean run() throws Exception; - @Getter - @Parameters - public static class Params { + // Picocli entrypoint. + @Override + default Integer call() throws Exception { + if (run()) { + return 0; + } + return 1; + } + } - @Parameter(names = {"-h", "--help"}, help = true, description = "Show this help.") - boolean help; + // Must be a public modifier. + public class ConfigNameCompletionCandidates implements Iterable { + @SneakyThrows + @Override + public Iterator iterator() { + return pulsarShell.getConfigStore().listConfigs().stream().map(ConfigEntry::getName).iterator(); + } } - private interface RunnableWithResult { - boolean run() throws Exception; + static class ConfigFileCompletionCandidates implements Iterable { + @Override + public Iterator iterator() { + String path = ConfigShell.resolveLocalFile(".").toPath().toString(); + ArrayList strings = new ArrayList<>(); + strings.add(path); + return strings.iterator(); + } } - private JCommander jcommander; - private Params params; private final PulsarShell pulsarShell; - private final Map commands = new HashMap<>(); private final ConfigStore configStore; private final ObjectMapper writer = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT); + private final CommandLine commander = new CommandLine(this, new InnerClassFactory(this)); + @Getter private String currentConfig; @@ -91,6 +113,15 @@ public ConfigShell(PulsarShell pulsarShell, String currentConfig) { this.configStore = pulsarShell.getConfigStore(); this.pulsarShell = pulsarShell; this.currentConfig = currentConfig; + commander.addSubcommand("list", new CmdConfigList()); + commander.addSubcommand("create", new CmdConfigCreate()); + commander.addSubcommand("clone", new CmdConfigClone()); + commander.addSubcommand("update", new CmdConfigUpdate()); + commander.addSubcommand("delete", new CmdConfigDelete()); + commander.addSubcommand("use", new CmdConfigUse()); + commander.addSubcommand("view", new CmdConfigView()); + commander.addSubcommand("set-property", new CmdConfigSetProperty()); + commander.addSubcommand("get-property", new CmdConfigGetProperty()); } @Override @@ -109,68 +140,11 @@ public String getAdminUrl() { } @Override - public void setupState(Properties properties) { - - this.params = new Params(); - this.jcommander = new JCommander(); - jcommander.addObject(params); - - commands.put("list", new CmdConfigList()); - commands.put("create", new CmdConfigCreate()); - commands.put("clone", new CmdConfigClone()); - commands.put("update", new CmdConfigUpdate()); - commands.put("delete", new CmdConfigDelete()); - commands.put("use", new CmdConfigUse()); - commands.put("view", new CmdConfigView()); - commands.put("set-property", new CmdConfigSetProperty()); - commands.put("get-property", new CmdConfigGetProperty()); - commands.forEach((k, v) -> jcommander.addCommand(k, v)); - } - - @Override - public void cleanupState(Properties properties) { - setupState(properties); - } - - @Override - public JCommander getJCommander() { - return jcommander; + public CommandLine getCommander() { + return commander; } - @Override - public boolean runCommand(String[] args) throws Exception { - try { - jcommander.parse(args); - - if (params.help) { - jcommander.usage(); - return true; - } - - String chosenCommand = jcommander.getParsedCommand(); - final RunnableWithResult command = commands.get(chosenCommand); - if (command == null) { - jcommander.usage(); - return false; - } - return command.run(); - } catch (Throwable e) { - jcommander.getConsole().println(e.getMessage()); - String chosenCommand = jcommander.getParsedCommand(); - if (e instanceof ParameterException) { - try { - jcommander.getUsageFormatter().usage(chosenCommand); - } catch (ParameterException noCmd) { - e.printStackTrace(); - } - } else { - e.printStackTrace(); - } - return false; - } - } - - @Parameters(commandDescription = "List configurations") + @Command(description = "List configurations") private class CmdConfigList implements RunnableWithResult { @Override @@ -194,10 +168,10 @@ private String formatEntry(ConfigStore.ConfigEntry entry) { } } - @Parameters(commandDescription = "Use the configuration for next commands") + @Command(description = "Use the configuration for next commands") private class CmdConfigUse implements RunnableWithResult { - @Parameter(description = "Name of the config", required = true) - @JCommanderCompleter.ParameterCompleter(type = JCommanderCompleter.ParameterCompleter.Type.CONFIGS) + @Parameters(description = "Name of the config", arity = "1", + completionCandidates = ConfigNameCompletionCandidates.class) private String name; @Override @@ -218,10 +192,10 @@ public boolean run() { } } - @Parameters(commandDescription = "View configuration") + @Command(description = "View configuration") private class CmdConfigView implements RunnableWithResult { - @Parameter(description = "Name of the config", required = true) - @JCommanderCompleter.ParameterCompleter(type = JCommanderCompleter.ParameterCompleter.Type.CONFIGS) + @Parameters(description = "Name of the config", arity = "1", + completionCandidates = ConfigNameCompletionCandidates.class) private String name; @Override @@ -237,10 +211,10 @@ public boolean run() { } } - @Parameters(commandDescription = "Delete a configuration") + @Command(description = "Delete a configuration") private class CmdConfigDelete implements RunnableWithResult { - @Parameter(description = "Name of the config", required = true) - @JCommanderCompleter.ParameterCompleter(type = JCommanderCompleter.ParameterCompleter.Type.CONFIGS) + @Parameters(description = "Name of the config", arity = "1", + completionCandidates = ConfigNameCompletionCandidates.class) private String name; @Override @@ -259,10 +233,10 @@ public boolean run() { } } - @Parameters(commandDescription = "Create a new configuration.") + @Command(name = "create", description = "Create a new configuration.") private class CmdConfigCreate extends CmdConfigPut { - @Parameter(description = "Configuration name", required = true) + @Parameters(description = "Configuration name", arity = "1") protected String name; @Override @@ -282,11 +256,11 @@ String name() { } } - @Parameters(commandDescription = "Update an existing configuration.") + @Command(description = "Update an existing configuration.") private class CmdConfigUpdate extends CmdConfigPut { - @Parameter(description = "Configuration name", required = true) - @JCommanderCompleter.ParameterCompleter(type = JCommanderCompleter.ParameterCompleter.Type.CONFIGS) + @Parameters(description = "Name of the config", arity = "1", + completionCandidates = ConfigNameCompletionCandidates.class) protected String name; @Override @@ -312,14 +286,14 @@ String name() { private abstract class CmdConfigPut implements RunnableWithResult { - @Parameter(names = {"--url"}, description = "URL of the config") + @Option(names = {"--url"}, description = "URL of the config") protected String url; - @Parameter(names = {"--file"}, description = "File path of the config") - @JCommanderCompleter.ParameterCompleter(type = JCommanderCompleter.ParameterCompleter.Type.FILES) + @Option(names = {"--file"}, description = "File path of the config", + completionCandidates = ConfigFileCompletionCandidates.class) protected String file; - @Parameter(names = {"--value"}, description = "Inline value of the config") + @Option(names = {"--value"}, description = "Inline value of the config") protected String inlineValue; @Override @@ -372,13 +346,14 @@ public boolean run() { } + @Command private class CmdConfigClone implements RunnableWithResult { - @Parameter(description = "Configuration to clone", required = true) - @JCommanderCompleter.ParameterCompleter(type = JCommanderCompleter.ParameterCompleter.Type.CONFIGS) + @Parameters(description = "Configuration to clone", arity = "1", + completionCandidates = ConfigNameCompletionCandidates.class) protected String cloneFrom; - @Parameter(names = {"--name"}, description = "Name of the new config", required = true) + @Option(names = {"--name"}, description = "Name of the new config", required = true) protected String newName; @Override @@ -410,17 +385,17 @@ private void reloadIfCurrent(ConfigStore.ConfigEntry entry) throws Exception { } - @Parameters(commandDescription = "Set a configuration property by name") + @Command(description = "Set a configuration property by name") private class CmdConfigSetProperty implements RunnableWithResult { - @Parameter(description = "Name of the config", required = true) - @JCommanderCompleter.ParameterCompleter(type = JCommanderCompleter.ParameterCompleter.Type.CONFIGS) + @Parameters(description = "Name of the config", arity = "1", + completionCandidates = ConfigNameCompletionCandidates.class) private String name; - @Parameter(names = {"-p", "--property"}, required = true, description = "Name of the property to update") + @Option(names = {"-p", "--property"}, required = true, description = "Name of the property to update") protected String propertyName; - @Parameter(names = {"-v", "--value"}, description = "New value for the property") + @Option(names = {"-v", "--value"}, description = "New value for the property") protected String propertyValue; @Override @@ -450,14 +425,14 @@ public boolean run() { } } - @Parameters(commandDescription = "Get a configuration property by name") + @Command(description = "Get a configuration property by name") private class CmdConfigGetProperty implements RunnableWithResult { - @Parameter(description = "Name of the config", required = true) - @JCommanderCompleter.ParameterCompleter(type = JCommanderCompleter.ParameterCompleter.Type.CONFIGS) + @Parameters(description = "Name of the config", arity = "1", + completionCandidates = ConfigNameCompletionCandidates.class) private String name; - @Parameter(names = {"-p", "--property"}, required = true, description = "Name of the property") + @Option(names = {"-p", "--property"}, required = true, description = "Name of the property") protected String propertyName; @Override @@ -482,7 +457,6 @@ public boolean run() { } - void print(List items) { for (T item : items) { print(item); @@ -492,9 +466,9 @@ void print(List items) { void print(T item) { try { if (item instanceof String) { - jcommander.getConsole().println((String) item); + commander.getOut().println((String) item); } else { - jcommander.getConsole().println(writer.writeValueAsString(item)); + commander.getOut().println(writer.writeValueAsString(item)); } } catch (Exception e) { throw new RuntimeException(e); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/JCommanderCompleter.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/JCommanderCompleter.java deleted file mode 100644 index 937f9cac04562..0000000000000 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/JCommanderCompleter.java +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.shell; - -import static java.lang.annotation.ElementType.FIELD; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.ParameterDescription; -import com.beust.jcommander.WrappedParameter; -import java.lang.annotation.Retention; -import java.lang.annotation.Target; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.SneakyThrows; -import org.apache.pulsar.admin.cli.CmdBase; -import org.apache.pulsar.shell.config.ConfigStore; -import org.jline.builtins.Completers; -import org.jline.reader.Candidate; -import org.jline.reader.Completer; -import org.jline.reader.LineReader; -import org.jline.reader.ParsedLine; -import org.jline.reader.impl.completer.NullCompleter; -import org.jline.reader.impl.completer.StringsCompleter; - -/** - * Convert JCommander instance to JLine3 completers. - */ -public class JCommanderCompleter { - - @AllArgsConstructor - @Getter - public static class ShellContext { - private final ConfigStore configStore; - } - - private JCommanderCompleter() { - } - - public static List createCompletersForCommand(String program, - JCommander command, - ShellContext shellContext) { - command.setProgramName(program); - return createCompletersForCommand(Collections.emptyList(), - command, - Arrays.asList(NullCompleter.INSTANCE), - shellContext); - } - - private static List createCompletersForCommand(List preCompleters, - JCommander command, - List postCompleters, - ShellContext shellContext) { - List all = new ArrayList<>(); - addCompletersForCommand(preCompleters, postCompleters, all, command, shellContext); - return all; - } - - private static void addCompletersForCommand(List preCompleters, - List postCompleters, - List result, - JCommander command, - ShellContext shellContext) { - final Collection options; - final Map subCommands; - final ParameterDescription mainParameterValue; - - if (command.getObjects().get(0) instanceof CmdBase) { - CmdBase cmdBase = (CmdBase) command.getObjects().get(0); - subCommands = cmdBase.getJcommander().getCommands(); - mainParameterValue = cmdBase.getJcommander().getMainParameter() == null ? null : - cmdBase.getJcommander().getMainParameterValue(); - options = cmdBase.getJcommander().getParameters() - .stream() - .map(option -> createOptionDescriptors(option, shellContext)) - .collect(Collectors.toList()); - } else { - subCommands = command.getCommands(); - mainParameterValue = command.getMainParameter() == null ? null : command.getMainParameterValue(); - options = command.getParameters() - .stream() - .map(option -> createOptionDescriptors(option, shellContext)) - .collect(Collectors.toList()); - } - - final StringsCompleter cmdStringsCompleter = new StringsCompleter(command.getProgramName()); - - for (int i = 0; i < options.size() + 1; i++) { - List completersChain = new ArrayList<>(); - completersChain.addAll(preCompleters); - completersChain.add(cmdStringsCompleter); - for (int j = 0; j < i; j++) { - completersChain.add(new Completers.OptionCompleter(options, preCompleters.size() + 1 + j)); - } - for (Map.Entry subCommand : subCommands.entrySet()) { - addCompletersForCommand(completersChain, postCompleters, result, subCommand.getValue(), shellContext); - } - if (mainParameterValue != null) { - final Completer customCompleter = getCustomCompleter(mainParameterValue, shellContext); - if (customCompleter != null) { - completersChain.add(customCompleter); - } - } - completersChain.addAll(postCompleters); - result.add(new OptionStrictArgumentCompleter(completersChain)); - } - } - - - @SneakyThrows - private static Completers.OptDesc createOptionDescriptors(ParameterDescription param, ShellContext shellContext) { - Completer valueCompleter = getCompleter(param, shellContext); - final WrappedParameter parameter = param.getParameter(); - String shortOption = null; - String longOption = null; - final String[] parameterNames = parameter.names(); - for (String parameterName : parameterNames) { - if (parameterName.startsWith("--")) { - longOption = parameterName; - } else if (parameterName.startsWith("-")) { - shortOption = parameterName; - } - } - return new Completers.OptDesc(shortOption, longOption, param.getDescription(), valueCompleter); - } - - @SneakyThrows - private static Completer getCompleter(ParameterDescription param, ShellContext shellContext) { - - Completer valueCompleter = null; - boolean isBooleanArg = param.getObject() instanceof Boolean || param.getDefault() instanceof Boolean - || param.getObject().getClass().isAssignableFrom(Boolean.class); - if (!isBooleanArg) { - valueCompleter = getCustomCompleter(param, shellContext); - if (valueCompleter == null) { - valueCompleter = Completers.AnyCompleter.INSTANCE; - } - } - return valueCompleter; - } - - @SneakyThrows - private static Completer getCustomCompleter(ParameterDescription param, ShellContext shellContext) { - Completer valueCompleter = null; - final Field reflField = param.getParameterized().getClass().getDeclaredField("field"); - reflField.setAccessible(true); - final Field field = (Field) reflField.get(param.getParameterized()); - final ParameterCompleter parameterCompleter = field.getAnnotation(ParameterCompleter.class); - if (parameterCompleter != null) { - final ParameterCompleter.Type completer = parameterCompleter.type(); - if (completer == ParameterCompleter.Type.FILES) { - valueCompleter = new Completers.FilesCompleter(ConfigShell.resolveLocalFile(".")); - } else if (completer == ParameterCompleter.Type.CONFIGS) { - valueCompleter = new Completer() { - @Override - @SneakyThrows - public void complete(LineReader reader, ParsedLine line, List candidates) { - new StringsCompleter(shellContext.configStore.listConfigs() - .stream().map(ConfigStore.ConfigEntry::getName).collect(Collectors.toList())) - .complete(reader, line, candidates); - } - }; - } - } - return valueCompleter; - } - - @Retention(java.lang.annotation.RetentionPolicy.RUNTIME) - @Target({ FIELD }) - public @interface ParameterCompleter { - - enum Type { - FILES, - CONFIGS; - } - - Type type(); - - } - -} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/OptionStrictArgumentCompleter.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/OptionStrictArgumentCompleter.java deleted file mode 100644 index 3a088d28757d0..0000000000000 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/OptionStrictArgumentCompleter.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.shell; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.LinkedList; -import java.util.List; -import java.util.Objects; -import org.jline.builtins.Completers; -import org.jline.reader.Candidate; -import org.jline.reader.Completer; -import org.jline.reader.LineReader; -import org.jline.reader.ParsedLine; -import org.jline.reader.impl.completer.ArgumentCompleter; - -/** - * Same as {@link ArgumentCompleter} but with more strict validation for options. - */ -public class OptionStrictArgumentCompleter implements Completer { - - private final List completers = new ArrayList<>(); - /** - * Create a new completer. - * - * @param completers The embedded completers - */ - public OptionStrictArgumentCompleter(final Collection completers) { - Objects.requireNonNull(completers); - this.completers.addAll(completers); - } - - @Override - public void complete(LineReader reader, ParsedLine line, List candidates) { - Objects.requireNonNull(line); - Objects.requireNonNull(candidates); - - if (line.wordIndex() < 0) { - return; - } - - Completer completer; - - // if we are beyond the end of the completers, just use the last one - if (line.wordIndex() >= completers.size()) { - completer = completers.get(completers.size() - 1); - } else { - completer = completers.get(line.wordIndex()); - } - - - // ensure that all the previous completers are successful - // before allowing this completer to pass (only if strict). - for (int i = 0; i < line.wordIndex(); i++) { - int idx = i >= completers.size() ? (completers.size() - 1) : i; - Completer sub = completers.get(idx); - - List args = line.words(); - String arg = (args == null || i >= args.size()) ? "" : args.get(i).toString(); - - List subCandidates = new LinkedList<>(); - /* - * This is the part that differs from the original ArgumentCompleter. - * It matches only if there's an actual option. - * The implementation of OptionCompleter will return the same candidate even if it is - * not part of the options set because options are not required. - */ - if (sub instanceof Completers.OptionCompleter) { - if (arg.startsWith("-")) { - sub.complete(reader, new ArgumentCompleter.ArgumentLine(arg, arg.length()), subCandidates); - } else { - return; - } - } else { - sub.complete(reader, new ArgumentCompleter.ArgumentLine(arg, arg.length()), subCandidates); - } - - - boolean found = false; - for (Candidate cand : subCandidates) { - if (cand.value().equals(arg)) { - found = true; - break; - } - } - if (!found) { - return; - } - } - completer.complete(reader, line, candidates); - } - -} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/PulsarShell.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/PulsarShell.java index 257e00fd14242..3cc99126fb8d7 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/PulsarShell.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/shell/PulsarShell.java @@ -18,8 +18,6 @@ */ package org.apache.pulsar.shell; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -28,6 +26,7 @@ import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; @@ -36,27 +35,35 @@ import java.util.Properties; import java.util.Scanner; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; import lombok.AllArgsConstructor; import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringSubstitutor; +import org.apache.pulsar.internal.CommanderFactory; +import org.apache.pulsar.internal.ShellCommandsProvider; import org.apache.pulsar.shell.config.ConfigStore; import org.apache.pulsar.shell.config.FileConfigStore; -import org.jline.reader.Completer; +import org.jline.console.impl.SystemRegistryImpl; import org.jline.reader.LineReader; import org.jline.reader.LineReaderBuilder; import org.jline.reader.impl.DefaultParser; -import org.jline.reader.impl.completer.AggregateCompleter; import org.jline.terminal.Terminal; import org.jline.terminal.TerminalBuilder; import org.jline.utils.AttributedStringBuilder; import org.jline.utils.AttributedStyle; import org.jline.utils.InfoCmp; +import picocli.CommandLine; +import picocli.CommandLine.ArgGroup; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.shell.jline3.PicocliCommands; /** * Main Pulsar shell class invokable from the pulsar-shell script. */ +@Command public class PulsarShell { private static final String EXIT_MESSAGE = "Goodbye!"; @@ -87,37 +94,34 @@ public class PulsarShell { }; private static final String DEFAULT_PULSAR_SHELL_ROOT_DIRECTORY = computeDefaultPulsarShellRootDirectory(); + private SystemRegistryImpl systemRegistry; + private final DefaultParser parser = new DefaultParser().eofOnUnclosedQuote(true); + private Terminal terminal; interface Substitutor { String replace(String str, Map vars); } - static final class ShellOptions { - - @Parameter(names = {"-h", "--help"}, help = true, description = "Show this help.") - boolean help; - } - static final class MainOptions { - @Parameter(names = {"-c", "--config"}, description = "Client configuration file.") + @Option(names = {"-c", "--config"}, description = "Client configuration file.") String configFile; - @Parameter(names = {"-f", "--filename"}, description = "Input filename with a list of commands to be executed." + @Option(names = {"-f", "--filename"}, description = "Input filename with a list of commands to be executed." + " Each command must be separated by a newline.") String filename; - @Parameter(names = {"--fail-on-error"}, description = "If true, the shell will be interrupted " + @Option(names = {"--fail-on-error"}, description = "If true, the shell will be interrupted " + "if a command throws an exception.") boolean failOnError; - @Parameter(names = {"-"}, description = "Read commands from the standard input.") + @Option(names = {"-"}, description = "Read commands from the standard input.") boolean readFromStdin; - @Parameter(names = {"-e", "--execute-command"}, description = "Execute this command and exit.") + @Option(names = {"-e", "--execute-command"}, description = "Execute this command and exit.") String inlineCommand; - @Parameter(names = {"-np", "--no-progress"}, description = "Display raw output of the commands without the " + @Option(names = {"-np", "--no-progress"}, description = "Display raw output of the commands without the " + "fancy progress visualization.") boolean noProgress; } @@ -130,9 +134,10 @@ enum ExecState { @Getter private final ConfigStore configStore; private final File pulsarShellDir; - private final JCommander mainCommander; - private final MainOptions mainOptions; - private JCommander shellCommander; + private final CommandLine mainCommander; + @ArgGroup(exclusive = false) + private final MainOptions mainOptions = new MainOptions(); + private CommandLine shellCommander; private Function, InteractiveLineReader> readerBuilder; private InteractiveLineReader reader; private final ConfigShell configShell; @@ -143,15 +148,14 @@ public PulsarShell(String args[]) throws IOException { } public PulsarShell(String args[], Properties props) throws IOException { properties = props; - mainCommander = new JCommander(); - mainOptions = new MainOptions(); - mainCommander.addObject(mainOptions); + mainCommander = new CommandLine(this); + mainCommander.setCommandName("pulsar-shell"); try { - mainCommander.parse(args); + mainCommander.parseArgs(args); } catch (Exception e) { System.err.println(e.getMessage()); System.err.println(); - mainCommander.usage(); + mainCommander.usage(System.out); exit(1); throw new IllegalArgumentException(e); } @@ -217,7 +221,7 @@ public static void main(String[] args) throws Exception { public void reload(Properties properties) throws Exception { this.properties = properties; - final Map providersMap = registerProviders(shellCommander, properties); + final Map providersMap = registerProviders(properties); reader = readerBuilder.apply(providersMap); } @@ -235,19 +239,9 @@ public void run() throws Exception { }) .build(); run((providersMap) -> { - List completers = new ArrayList<>(); String serviceUrl = ""; String adminUrl = ""; - final JCommanderCompleter.ShellContext shellContext = new JCommanderCompleter.ShellContext(configStore); for (ShellCommandsProvider provider : providersMap.values()) { - provider.setupState(properties); - final JCommander jCommander = provider.getJCommander(); - if (jCommander != null) { - jCommander.createDescriptions(); - completers.addAll(JCommanderCompleter - .createCompletersForCommand(provider.getName(), jCommander, shellContext)); - } - final String providerServiceUrl = provider.getServiceUrl(); if (providerServiceUrl != null) { serviceUrl = providerServiceUrl; @@ -258,12 +252,10 @@ public void run() throws Exception { } } - Completer completer = new AggregateCompleter(completers); - LineReaderBuilder readerBuilder = LineReaderBuilder.builder() .terminal(terminal) - .parser(new DefaultParser().eofOnUnclosedQuote(true)) - .completer(completer) + .parser(parser) + .completer(systemRegistry.completer()) .variable(LineReader.INDENTATION, 2) .option(LineReader.Option.INSERT_BRACKET, true); @@ -301,7 +293,7 @@ public List parseLine(String line) { return reader.getParser().parse(line, 0).words(); } }; - }, (providerMap) -> terminal); + }, () -> terminal); } private void configureHistory(Properties properties, LineReaderBuilder readerBuilder) { @@ -341,19 +333,12 @@ interface InteractiveLineReader { } public void run(Function, InteractiveLineReader> readerBuilder, - Function, Terminal> terminalBuilder) throws Exception { + Supplier terminalBuilder) throws Exception { this.readerBuilder = readerBuilder; - /** - * Options read from the shell session - */ - shellCommander = new JCommander(); - final ShellOptions shellOptions = new ShellOptions(); - shellCommander.addObject(shellOptions); - - final Map providersMap = registerProviders(shellCommander, properties); - + this.terminal = terminalBuilder.get(); + final Map providersMap = registerProviders(properties); reader = readerBuilder.apply(providersMap); - final Terminal terminal = terminalBuilder.apply(providersMap); + final Map variables = System.getenv(); CommandReader commandReader; @@ -437,21 +422,20 @@ public List readCommand() { exit(0); return; } - if (shellOptions.help) { - shellCommander.usage(); + if (isHelp(line)) { + shellCommander.usage(System.out); continue; } final ShellCommandsProvider pulsarShellCommandsProvider = getProviderFromArgs(shellCommander, words); if (pulsarShellCommandsProvider == null) { - shellCommander.usage(); + shellCommander.usage(System.out); continue; } - String[] argv = extractAndConvertArgs(words); boolean commandOk = false; try { printExecutingCommands(terminal, commandsInfo, false); - commandOk = pulsarShellCommandsProvider.runCommand(argv); + systemRegistry.execute(line); } catch (InterruptShellException t) { // no-op } catch (Throwable t) { @@ -463,8 +447,6 @@ public List readCommand() { commandsInfo.executedCommands.add(new CommandsInfo.ExecutedCommandInfo(line, commandOk)); printExecutingCommands(terminal, commandsInfo, true); } - pulsarShellCommandsProvider.cleanupState(properties); - } if (mainOptions.failOnError && !commandOk) { exit(1); @@ -524,13 +506,13 @@ private void printExecutingCommands(Terminal terminal, } } - private static ShellCommandsProvider getProviderFromArgs(JCommander mainCommander, List words) { + private static ShellCommandsProvider getProviderFromArgs(CommandLine mainCommander, List words) { final String providerCmd = words.get(0); - final JCommander commander = mainCommander.getCommands().get(providerCmd); + final CommandLine commander = mainCommander.getSubcommands().get(providerCmd); if (commander == null) { return null; } - return (ShellCommandsProvider) commander.getObjects().get(0); + return commander.getCommand(); } private static String createPrompt(String hostname) { @@ -567,28 +549,24 @@ private static boolean isQuitCommand(String line) { return line.equalsIgnoreCase("quit") || line.equalsIgnoreCase("exit"); } - private static String[] extractAndConvertArgs(List words) { - List parsed = new ArrayList<>(); - for (String s : words.subList(1, words.size())) { - if (s.startsWith("-") && s.contains("=")) { - final String[] split = s.split("=", 2); - parsed.add(split[0]); - parsed.add(split[1]); - } else { - parsed.add(s); - } - } - - String[] argv = parsed.toArray(new String[parsed.size()]); - return argv; + private static boolean isHelp(String line) { + return line.equalsIgnoreCase("help"); } - private Map registerProviders(JCommander commander, Properties properties) + private Map registerProviders(Properties properties) throws Exception { + shellCommander = CommanderFactory.createRootCommanderWithHook(this, null); final Map providerMap = new HashMap<>(); - registerProvider(createAdminShell(properties), commander, providerMap); - registerProvider(createClientShell(properties), commander, providerMap); - registerProvider(configShell, commander, providerMap); + registerProvider(createAdminShell(properties), shellCommander, providerMap); + registerProvider(createClientShell(properties), shellCommander, providerMap); + registerProvider(configShell, shellCommander, providerMap); + + Supplier workDir = () -> Paths.get(DEFAULT_PULSAR_SHELL_ROOT_DIRECTORY); + PicocliCommands picocliCommands = new PicocliCommands(shellCommander); + systemRegistry = new SystemRegistryImpl(parser, terminal, workDir, null); + systemRegistry.setCommandRegistries(picocliCommands); + systemRegistry.register("help", picocliCommands); + return providerMap; } @@ -601,11 +579,11 @@ protected ClientShell createClientShell(Properties properties) { } private static void registerProvider(ShellCommandsProvider provider, - JCommander commander, + CommandLine commander, Map providerMap) { final String name = provider.getName(); - commander.addCommand(name, provider); + commander.addSubcommand(name, provider.getCommander()); providerMap.put(name, provider); } diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdClusters.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdClusters.java index 86faabccc80ea..bdac206cc7f12 100644 --- a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdClusters.java +++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdClusters.java @@ -18,21 +18,29 @@ */ package org.apache.pulsar.admin.cli; -import org.apache.pulsar.client.api.ProxyProtocol; - import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; - +import com.google.common.collect.Lists; import java.io.File; +import java.io.PrintWriter; +import java.io.StringWriter; import java.nio.file.Files; +import java.util.List; +import java.util.Map; +import lombok.Cleanup; import org.apache.pulsar.admin.cli.utils.CmdUtils; -import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.client.admin.Brokers; import org.apache.pulsar.client.admin.Clusters; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.ProxyProtocol; +import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.assertj.core.util.Maps; import org.testng.Assert; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -87,4 +95,44 @@ public ClusterData buildClusterData() { .brokerClientTlsTrustStorePassword("clientpw") .build(); } + + @Test + public void testListCmd() throws Exception { + List clusterList = Lists.newArrayList("us-west", "us-east", "us-cent"); + List clusterResultList = Lists.newArrayList("us-west", "us-east", "us-cent(*)"); + Map configurations = Maps.newHashMap("clusterName", "us-cent"); + + Clusters clusters = mock(Clusters.class); + Brokers brokers = mock(Brokers.class); + PulsarAdmin admin = mock(PulsarAdmin.class); + when(admin.clusters()).thenReturn(clusters); + when(admin.brokers()).thenReturn(brokers); + doReturn(clusterList).when(clusters).getClusters(); + doReturn(configurations).when(brokers).getRuntimeConfigurations(); + + CmdClusters cmd = new CmdClusters(() -> admin); + + @Cleanup + StringWriter stringWriter = new StringWriter(); + @Cleanup + PrintWriter printWriter = new PrintWriter(stringWriter); + cmd.getCommander().setOut(printWriter); + + cmd.run("list".split("\\s+")); + Assert.assertEquals(stringWriter.toString(), String.join("\n", clusterList) + "\n"); + + @Cleanup + StringWriter stringWriter1 = new StringWriter(); + @Cleanup + PrintWriter printWriter1 = new PrintWriter(stringWriter1); + cmd.getCommander().setOut(printWriter1); + cmd.run("list -c".split("\\s+")); + Assert.assertEquals(stringWriter1.toString(), String.join("\n", clusterResultList) + "\n"); + } + + @Test + public void testGetClusterMigration() throws Exception { + cmdClusters.run(new String[]{"get-cluster-migration", "test_cluster"}); + verify(clusters, times(1)).getClusterMigration("test_cluster"); + } } diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdNamespaces.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdNamespaces.java new file mode 100644 index 0000000000000..f9ce84411c6c0 --- /dev/null +++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdNamespaces.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli; + +import org.apache.pulsar.client.admin.Namespaces; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.common.policies.data.RetentionPolicies; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.Test; +import java.io.IOException; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class TestCmdNamespaces { + + @AfterMethod(alwaysRun = true) + public void cleanup() throws IOException { + //NOTHING FOR NOW + } + + + @Test + public void testSetRetentionCmd() throws Exception { + Namespaces namespaces = mock(Namespaces.class); + + PulsarAdmin admin = mock(PulsarAdmin.class); + when(admin.namespaces()).thenReturn(namespaces); + + CmdNamespaces cmd = new CmdNamespaces(() -> admin); + + cmd.run("set-retention public/default -s 2T -t 2h".split("\\s+")); + verify(namespaces, times(1)).setRetention("public/default", new RetentionPolicies(120, 2 * 1024 * 1024)); + } +} diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSchema.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSchema.java new file mode 100644 index 0000000000000..b61ac3b8ef3d5 --- /dev/null +++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSchema.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.Schemas; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class TestCmdSchema { + + private PulsarAdmin pulsarAdmin; + + private CmdSchemas cmdSchemas; + + private Schemas schemas; + + @BeforeMethod + public void setup() throws Exception { + pulsarAdmin = mock(PulsarAdmin.class); + schemas = mock(Schemas.class); + when(pulsarAdmin.schemas()).thenReturn(schemas); + cmdSchemas = spy(new CmdSchemas(() -> pulsarAdmin)); + } + + @Test + public void testCmdClusterConfigFile() throws Exception { + String topic = "persistent://tenant/ns1/t1"; + cmdSchemas.run(new String[]{"metadata", topic}); + verify(schemas).getSchemaMetadata(eq(topic)); + } +} diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSinks.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSinks.java index 84ababa725099..5885b60aef24a 100644 --- a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSinks.java +++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSinks.java @@ -25,7 +25,7 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import com.beust.jcommander.ParameterException; +import static org.testng.Assert.assertFalse; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; import java.io.Closeable; @@ -38,6 +38,7 @@ import java.util.List; import java.util.Map; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.admin.cli.CmdSinks.LocalSinkRunner; import org.apache.pulsar.admin.cli.utils.CmdUtils; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.Sinks; @@ -61,6 +62,7 @@ public class TestCmdSinks { private static final String CLASS_NAME = "SomeRandomClassName"; private static final String INPUTS = "test-src1,test-src2"; private static final List INPUTS_LIST; + static { INPUTS_LIST = new LinkedList<>(); INPUTS_LIST.add("test-src1"); @@ -282,7 +284,8 @@ public void testMissingProcessingGuarantees() throws Exception { ); } - @Test(expectedExceptions = ParameterException.class, expectedExceptionsMessageRegExp = "Sink archive not specfied") + @Test(expectedExceptions = CliCommand.ParameterException.class, + expectedExceptionsMessageRegExp = "Sink archive not specified") public void testMissingArchive() throws Exception { SinkConfig sinkConfig = getSinkConfig(); sinkConfig.setArchive(null); @@ -502,7 +505,7 @@ public void testCmdSinkConfigFileMissingResources() throws Exception { testCmdSinkConfigFile(testSinkConfig, expectedSinkConfig); } - @Test(expectedExceptions = ParameterException.class, expectedExceptionsMessageRegExp = "Sink archive not specfied") + @Test(expectedExceptions = CliCommand.ParameterException.class, expectedExceptionsMessageRegExp = "Sink archive not specified") public void testCmdSinkConfigFileMissingJar() throws Exception { SinkConfig testSinkConfig = getSinkConfig(); testSinkConfig.setArchive(null); @@ -522,8 +525,7 @@ public void testCmdSinkConfigFileInvalidJar() throws Exception { testCmdSinkConfigFile(testSinkConfig, expectedSinkConfig); } - @Test(expectedExceptions = ParameterException.class, expectedExceptionsMessageRegExp = "Invalid sink type 'foo' " + - "-- Available sinks are: \\[\\]") + @Test(expectedExceptions = CliCommand.ParameterException.class, expectedExceptionsMessageRegExp = "Invalid sink type 'foo' -- Available sinks are: \\[\\]") public void testCmdSinkConfigFileInvalidSinkType() throws Exception { SinkConfig testSinkConfig = getSinkConfig(); // sinkType is prior than archive @@ -808,4 +810,14 @@ public void testParseConfigs() throws Exception { Assert.assertEquals(config.get("float_string"), "1000.0"); Assert.assertEquals(config.get("created_at"), "Mon Jul 02 00:33:15 +0000 2018"); } + + @Test + public void testExcludeDeprecatedOptions() throws Exception { + SinkConfig testSinkConfig = getSinkConfig(); + LocalSinkRunner localSinkRunner = spy(new CmdSinks(() -> pulsarAdmin)).getLocalSinkRunner(); + localSinkRunner.sinkConfig = testSinkConfig; + localSinkRunner.deprecatedBrokerServiceUrl = "pulsar://localhost:6650"; + List localRunArgs = localSinkRunner.getLocalRunArgs(); + assertFalse(String.join(",", localRunArgs).contains("--deprecated")); + } } diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSources.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSources.java index ed90748f40f75..576e63310c1fa 100644 --- a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSources.java +++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSources.java @@ -20,21 +20,24 @@ import static org.apache.pulsar.common.naming.TopicName.DEFAULT_NAMESPACE; import static org.apache.pulsar.common.naming.TopicName.PUBLIC_TENANT; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; - -import com.beust.jcommander.ParameterException; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; import java.io.Closeable; import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.util.List; import java.util.Map; - +import java.util.UUID; +import org.apache.pulsar.admin.cli.CmdSources.LocalSourceRunner; import org.apache.pulsar.admin.cli.utils.CmdUtils; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.Sources; @@ -51,8 +54,7 @@ import org.testng.annotations.Test; public class TestCmdSources { - - private static final String TENANT = "test-tenant"; + private static final String TENANT = "test-tenant"; private static final String NAMESPACE = "test-namespace"; private static final String NAME = "test"; private static final String TOPIC_NAME = "src_topic_1"; @@ -82,7 +84,6 @@ public class TestCmdSources { @BeforeMethod public void setup() throws Exception { - pulsarAdmin = mock(PulsarAdmin.class); source = mock(Sources.class); when(pulsarAdmin.sources()).thenReturn(source); @@ -190,7 +191,7 @@ public void testMissingProcessingGuarantees() throws Exception { ); } - @Test(expectedExceptions = ParameterException.class, expectedExceptionsMessageRegExp = "Source archive not specified") + @Test(expectedExceptions = CliCommand.ParameterException.class, expectedExceptionsMessageRegExp = "Source archive not specified") public void testMissingArchive() throws Exception { SourceConfig sourceConfig = getSourceConfig(); sourceConfig.setArchive(null); @@ -368,7 +369,7 @@ public void testCmdSourceConfigFileMissingResources() throws Exception { testCmdSourceConfigFile(testSourceConfig, expectedSourceConfig); } - @Test(expectedExceptions = ParameterException.class, expectedExceptionsMessageRegExp = "Source archive not specified") + @Test(expectedExceptions = CliCommand.ParameterException.class, expectedExceptionsMessageRegExp = "Source archive not specified") public void testCmdSourceConfigFileMissingJar() throws Exception { SourceConfig testSourceConfig = getSourceConfig(); testSourceConfig.setArchive(null); @@ -412,7 +413,15 @@ public void testBatchSourceConfigMissingDiscoveryTriggererClassName() throws Exc expectedSourceConfig.setBatchSourceConfig(batchSourceConfig); testCmdSourceConfigFile(testSourceConfig, expectedSourceConfig); } - + + @Test + public void testCmdSourceConfigFileInvalidSourceType() throws Exception { + SourceConfig sourceConfig = getSourceConfig(); + sourceConfig.setSourceType("foo"); + assertThatThrownBy(() -> testCmdSourceConfigFile(sourceConfig, null)) + .hasMessageContaining("Invalid source type 'foo'"); + } + public void testCmdSourceConfigFile(SourceConfig testSourceConfig, SourceConfig expectedSourceConfig) throws Exception { File file = Files.createTempFile("", "").toFile(); @@ -444,6 +453,19 @@ public void testCmdSourceConfigFile(SourceConfig testSourceConfig, SourceConfig verify(localSourceRunner).validateSourceConfigs(eq(expectedSourceConfig)); } + @Test + public void testCmdSourcesThrowingExceptionOnFailure() throws Exception { + verifyNoSuchFileParameterException(createSource); + verifyNoSuchFileParameterException(updateSource); + verifyNoSuchFileParameterException(localSourceRunner); + } + + private void verifyNoSuchFileParameterException(org.apache.pulsar.admin.cli.CmdSources.SourceDetailsCommand command) { + command.sourceConfigFile = UUID.randomUUID().toString(); + IllegalArgumentException e = Assert.expectThrows(IllegalArgumentException.class, command::processArguments); + assertTrue(e.getMessage().endsWith("(No such file or directory)")); + } + @Test public void testCliOverwriteConfigFile() throws Exception { @@ -661,4 +683,14 @@ public void testParseConfigs() throws Exception { Assert.assertEquals(config.get("float_string"), "1000.0"); Assert.assertEquals(config.get("created_at"), "Mon Jul 02 00:33:15 +0000 2018"); } + + @Test + public void testExcludeDeprecatedOptions() throws Exception { + SourceConfig testSinkConfig = getSourceConfig(); + LocalSourceRunner localSourceRunner = spy(new CmdSources(() -> pulsarAdmin)).getLocalSourceRunner(); + localSourceRunner.sourceConfig = testSinkConfig; + localSourceRunner.deprecatedBrokerServiceUrl = "pulsar://localhost:6650"; + List localRunArgs = localSourceRunner.getLocalRunArgs(); + assertFalse(String.join(",", localRunArgs).contains("--deprecated")); + } } diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdTopics.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdTopics.java index a1c3d6f902cad..fc98b14392c3e 100644 --- a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdTopics.java +++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdTopics.java @@ -20,25 +20,31 @@ import static org.apache.pulsar.common.naming.TopicName.DEFAULT_NAMESPACE; import static org.apache.pulsar.common.naming.TopicName.PUBLIC_TENANT; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import com.google.common.collect.Lists; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.PrintStream; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; +import lombok.Cleanup; import org.apache.pulsar.client.admin.ListTopicsOptions; import org.apache.pulsar.client.admin.Lookup; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.admin.Schemas; import org.apache.pulsar.client.admin.Topics; import org.apache.pulsar.client.impl.MessageIdImpl; @@ -58,19 +64,23 @@ public class TestCmdTopics { private PulsarAdmin pulsarAdmin; private CmdTopics cmdTopics; private Lookup mockLookup; + private Topics mockTopics; private CmdTopics.PartitionedLookup partitionedLookup; + private CmdTopics.DeleteCmd deleteCmd; @BeforeMethod public void setup() throws Exception { pulsarAdmin = Mockito.mock(PulsarAdmin.class); - Topics mockTopics = mock(Topics.class); + mockTopics = mock(Topics.class); when(pulsarAdmin.topics()).thenReturn(mockTopics); Schemas mockSchemas = mock(Schemas.class); when(pulsarAdmin.schemas()).thenReturn(mockSchemas); mockLookup = mock(Lookup.class); when(pulsarAdmin.lookups()).thenReturn(mockLookup); + when(pulsarAdmin.topics()).thenReturn(mockTopics); cmdTopics = spy(new CmdTopics(() -> pulsarAdmin)); partitionedLookup = spy(cmdTopics.getPartitionedLookup()); + deleteCmd = spy(cmdTopics.getDeleteCmd()); } @AfterMethod(alwaysRun = true) @@ -122,19 +132,19 @@ public void testListCmd() throws Exception { assertEquals(admin.topics().getList("test", TopicDomain.persistent), topicList); CmdTopics cmd = new CmdTopics(() -> admin); + @Cleanup + StringWriter stringWriter = new StringWriter(); + @Cleanup + PrintWriter printWriter = new PrintWriter(stringWriter); + cmd.getCommander().setOut(printWriter); + + cmd.run("list public/default".split("\\s+")); + Assert.assertEquals(stringWriter.toString(), String.join("\n", topicList) + "\n"); + } - PrintStream defaultSystemOut = System.out; - try (ByteArrayOutputStream out = new ByteArrayOutputStream(); PrintStream ps = new PrintStream(out)) { - System.setOut(ps); - cmd.run("list public/default".split("\\s+")); - Assert.assertEquals(out.toString(), String.join("\n", topicList) + "\n"); - } finally { - System.setOut(defaultSystemOut); - } - } @Test public void testPartitionedLookup() throws Exception { - partitionedLookup.params = Arrays.asList("persistent://public/default/my-topic"); + partitionedLookup.topicName = "persistent://public/default/my-topic"; partitionedLookup.run(); StringBuilder topic = new StringBuilder(); topic.append(PERSISTENT_TOPIC_URL); @@ -148,7 +158,7 @@ public void testPartitionedLookup() throws Exception { @Test public void testPartitionedLookupSortByBroker() throws Exception { - partitionedLookup.params = Arrays.asList("persistent://public/default/my-topic"); + partitionedLookup.topicName = "persistent://public/default/my-topic"; partitionedLookup.run(); StringBuilder topic = new StringBuilder(); topic.append(PERSISTENT_TOPIC_URL); @@ -160,4 +170,94 @@ public void testPartitionedLookupSortByBroker() throws Exception { partitionedLookup.sortByBroker = true; verify(mockLookup).lookupPartitionedTopic(eq(topic.toString())); } + @Test + public void testRunDeleteSingleTopic() throws PulsarAdminException, IOException { + // Setup: Specify a single topic to delete + deleteCmd.topic = "persistent://tenant/namespace/topic"; + + // Act: Run the delete command + deleteCmd.run(); + + // Assert: Verify that the delete method was called once for the specified topic + verify(mockTopics, times(1)).delete("persistent://tenant/namespace/topic", false); + } + + @Test + public void testRunDeleteMultipleTopics() throws PulsarAdminException, IOException { + // Setup: Specify a regex to delete multiple topics + deleteCmd.topic = "persistent://tenant/namespace/.*"; + deleteCmd.regex = true; + + // Mock: Simulate the return of multiple topics that match the regex + when(mockTopics.getList("tenant/namespace")).thenReturn(List.of( + "persistent://tenant/namespace/topic1", + "persistent://tenant/namespace/topic2")); + + // Act: Run the delete command + deleteCmd.run(); + + // Assert: Verify that the delete method was called once for each of the matching topics + verify(mockTopics, times(1)).getList("tenant/namespace"); + verify(mockTopics, times(1)).delete("persistent://tenant/namespace/topic1", false); + verify(mockTopics, times(1)).delete("persistent://tenant/namespace/topic2", false); + } + + @Test + public void testRunDeleteTopicsFromFile() throws PulsarAdminException, IOException { + // Setup: Create a temporary file and write some topics to it + Path tempFile = Files.createTempFile("topics", ".txt"); + List topics = List.of( + "persistent://tenant/namespace/topic1", + "persistent://tenant/namespace/topic2"); + Files.write(tempFile, topics); + + // Setup: Specify the temporary file as input for the delete command + deleteCmd.topic = tempFile.toString(); + deleteCmd.readFromFile = true; + + // Act: Run the delete command + deleteCmd.run(); + + // Assert: Verify that the delete method was called once for each topic in the file + for (String topic : topics) { + verify(mockTopics, times(1)).delete(topic, false); + } + + // Cleanup: Delete the temporary file + Files.delete(tempFile); + } + + @Test + public void testRunDeleteTopicsFromFileWithException() throws PulsarAdminException, IOException { + // Setup: Create a temporary file and write some topics to it. + // Configure the delete method of mockTopics to throw a PulsarAdminException on any input. + doThrow(new PulsarAdminException("mock fail")).when(mockTopics).delete(anyString(), anyBoolean()); + Path tempFile = Files.createTempFile("topics", ".txt"); + List topics = List.of( + "persistent://tenant/namespace/topic1", + "persistent://tenant/namespace/topic2"); + Files.write(tempFile, topics); + + // Setup: Specify the temporary file as input for the delete command + deleteCmd.topic = tempFile.toString(); + deleteCmd.readFromFile = true; + + // Act: Run the delete command + // Since we have configured the delete method of mockTopics to throw an exception when called, + // an exception should be thrown here. + deleteCmd.run(); + + // Assert: Verify that the delete method was called once for each topic in the file, + // even if one of them threw an exception. + // This proves that the program continues to attempt to delete the other topics + // even if an exception occurred while deleting a topic. + for (String topic : topics) { + verify(mockTopics, times(1)).delete(topic, false); + } + + // Cleanup: Delete the temporary file and recreate the mockTopics. + Files.delete(tempFile); + mockTopics = mock(Topics.class); + } + } diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/client/cli/TestCmdProduce.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/client/cli/TestCmdProduce.java index a9063bfadcc9a..1166eafe4d757 100644 --- a/pulsar-client-tools/src/test/java/org/apache/pulsar/client/cli/TestCmdProduce.java +++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/client/cli/TestCmdProduce.java @@ -20,13 +20,19 @@ import static org.testng.Assert.assertEquals; - +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.DecoderFactory; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.schema.KeyValueSchema; import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.common.schema.SchemaType; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import java.util.Collections; +import java.util.List; + public class TestCmdProduce { @@ -72,4 +78,26 @@ public void testBuildSchema() { assertEquals(SchemaType.JSON, composite2.getKeySchema().getSchemaInfo().getType()); assertEquals(SchemaType.AVRO, composite2.getValueSchema().getSchemaInfo().getType()); } + + @Test + public void generateAvroMessageBodies() throws Exception { + + Schema schema = CmdProduce.buildSchema( + null, + "avro:{\"type\": \"record\",\"namespace\": \"com.example\",\"name\": \"FullName\", \"fields\": [" + + "{ \"name\": \"a\", \"type\": \"string\" }," + + "{ \"name\": \"b\", \"type\": \"int\" }" + + "]}", + ""); + + List bytes = CmdProduce.generateMessageBodies(List.of("{\"a\":\"stringValue\",\"b\":123}"), Collections.emptyList(), schema); + assertEquals(bytes.size(), 1); + + org.apache.avro.Schema avro = (org.apache.avro.Schema) schema.getNativeSchema().get(); + GenericDatumReader reader = new GenericDatumReader<>(avro); + GenericRecord record = reader.read(null, DecoderFactory.get().binaryDecoder(bytes.get(0), null)); + assertEquals("stringValue", record.get("a").toString()); + assertEquals(123, record.get("b")); + + } } diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/shell/AdminShellTest.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/shell/AdminShellTest.java index 2f53546d7bb4a..cb4f63ccf75e5 100644 --- a/pulsar-client-tools/src/test/java/org/apache/pulsar/shell/AdminShellTest.java +++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/shell/AdminShellTest.java @@ -48,13 +48,15 @@ public void test() throws Exception { final PulsarAdmin admin = mock(PulsarAdmin.class); when(builder.build()).thenReturn(admin); when(admin.topics()).thenReturn(mock(Topics.class)); - adminShell.setPulsarAdminSupplier(new PulsarAdminSupplier(builder, adminShell.getRootParams())); + PulsarAdminSupplier pulsarAdminSupplier = adminShell.getPulsarAdminSupplier(); + pulsarAdminSupplier.setAdminBuilder(builder); assertTrue(run(new String[]{"topics", "list", "public/default"})); - verify(builder).build(); + verify(builder, times(1)).build(); assertTrue(run(new String[]{"topics", "list", "public/default"})); - verify(builder).build(); + verify(builder, times(1)).build(); assertTrue(run(new String[]{"--admin-url", "http://localhost:8081", "topics", "list", "public/default"})); + verify(builder, times(2)).build(); assertTrue(run(new String[]{"topics", "list", "public/default"})); verify(builder, times(3)).build(); assertTrue(run(new String[]{"--admin-url", "http://localhost:8080", @@ -62,11 +64,7 @@ public void test() throws Exception { verify(builder, times(3)).build(); } - private boolean run(String[] args) throws Exception { - try { - return adminShell.runCommand(args); - } finally { - adminShell.cleanupState(props); - } + private boolean run(String[] args) { + return adminShell.runCommand(args); } } \ No newline at end of file diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/shell/ConfigShellTest.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/shell/ConfigShellTest.java index f1b761520eb00..fcd856f33695d 100644 --- a/pulsar-client-tools/src/test/java/org/apache/pulsar/shell/ConfigShellTest.java +++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/shell/ConfigShellTest.java @@ -21,7 +21,6 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -29,30 +28,29 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; -import com.beust.jcommander.internal.Console; import java.io.File; +import java.io.PrintWriter; +import java.io.StringWriter; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Properties; import org.apache.pulsar.shell.config.ConfigStore; import org.apache.pulsar.shell.config.FileConfigStore; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import picocli.CommandLine; public class ConfigShellTest { private PulsarShell pulsarShell; private ConfigShell configShell; - private List output; + private String output; + private StringWriter stringWriter; @BeforeMethod(alwaysRun = true) public void before() throws Exception { - pulsarShell = spy(mock(PulsarShell.class)); + pulsarShell = mock(PulsarShell.class); doNothing().when(pulsarShell).reload(any()); final Path tempJson = Files.createTempFile("pulsar-shell", ".json"); @@ -60,54 +58,42 @@ public void before() throws Exception { new FileConfigStore(tempJson.toFile(), new ConfigStore.ConfigEntry(ConfigStore.DEFAULT_CONFIG, "#comment\ndefault-config=true"))); configShell = new ConfigShell(pulsarShell, ConfigStore.DEFAULT_CONFIG); - configShell.setupState(new Properties()); - output = new ArrayList<>(); setConsole(); } private void setConsole() { - configShell.getJCommander().setConsole(new Console() { - @Override - public void print(String msg) { - System.out.print("got: " + msg); - output.add(msg); - } - - @Override - public void println(String msg) { - System.out.println("got: " + msg); - output.add(msg); - } - - @Override - public char[] readPassword(boolean echoInput) { - return new char[0]; - } - }); + CommandLine commander = configShell.getCommander(); + stringWriter = new StringWriter(); + commander.setOut(new PrintWriter(stringWriter)); + } + + private void cleanOutput() { + setConsole(); + output = ""; } @Test public void testDefault() throws Exception { assertTrue(runCommand(new String[]{"list"})); - assertEquals(output, Arrays.asList("default (*)")); - output.clear(); + assertEquals(output, "default (*)\n"); + cleanOutput(); assertTrue(runCommand(new String[]{"view", "default"})); - assertEquals(output.get(0), "default-config=true\n"); - output.clear(); + assertEquals(output, "default-config=true\n\n"); + cleanOutput(); final Path newClientConf = Files.createTempFile("client", ".conf"); assertFalse(runCommand(new String[]{"create", "default", "--file", newClientConf.toFile().getAbsolutePath()})); - assertEquals(output, Arrays.asList("Config 'default' already exists.")); - output.clear(); + assertEquals(output, "Config 'default' already exists.\n"); + cleanOutput(); assertFalse(runCommand(new String[]{"update", "default", "--file", newClientConf.toFile().getAbsolutePath()})); - assertEquals(output, Arrays.asList("'default' can't be updated.")); - output.clear(); + assertEquals(output, "'default' can't be updated.\n"); + cleanOutput(); assertFalse(runCommand(new String[]{"delete", "default"})); - assertEquals(output, Arrays.asList("'default' can't be deleted.")); + assertEquals(output, "'default' can't be deleted.\n"); } @Test @@ -120,25 +106,25 @@ public void test() throws Exception { assertTrue(runCommand(new String[]{"create", "myclient", "--file", newClientConf.toFile().getAbsolutePath()})); assertTrue(output.isEmpty()); - output.clear(); + cleanOutput(); assertNull(pulsarShell.getConfigStore().getLastUsed()); assertTrue(runCommand(new String[]{"use", "myclient"})); assertTrue(output.isEmpty()); - output.clear(); + cleanOutput(); assertEquals(pulsarShell.getConfigStore().getLastUsed(), pulsarShell.getConfigStore() .getConfig("myclient")); verify(pulsarShell).reload(any()); assertTrue(runCommand(new String[]{"list"})); - assertEquals(output, Arrays.asList("default", "myclient (*)")); - output.clear(); + assertEquals(output, "default\nmyclient (*)\n"); + cleanOutput(); assertFalse(runCommand(new String[]{"delete", "myclient"})); - assertEquals(output, Arrays.asList("'myclient' is currently used and it can't be deleted.")); - output.clear(); + assertEquals(output, "'myclient' is currently used and it can't be deleted.\n"); + cleanOutput(); assertTrue(runCommand(new String[]{"update", "myclient", "--file", newClientConf.toFile().getAbsolutePath()})); @@ -151,9 +137,9 @@ public void test() throws Exception { verify(pulsarShell, times(2)).reload(any()); assertTrue(runCommand(new String[]{"view", "myclient-copied"})); - assertEquals(output.get(0), "webServiceUrl=http://localhost:8081/\nbrokerServiceUrl" + - "=pulsar://localhost:6651/\n"); - output.clear(); + assertEquals(output, "webServiceUrl=http://localhost:8081/\nbrokerServiceUrl" + + "=pulsar://localhost:6651/\n\n"); + cleanOutput(); } @Test @@ -166,65 +152,66 @@ public void testSetGetProperty() throws Exception { assertTrue(runCommand(new String[]{"create", "myclient", "--file", newClientConf.toFile().getAbsolutePath()})); assertTrue(output.isEmpty()); - output.clear(); + cleanOutput(); assertTrue(runCommand(new String[]{"use", "myclient"})); assertTrue(output.isEmpty()); - output.clear(); + cleanOutput(); assertTrue(runCommand(new String[]{"get-property", "-p", "webServiceUrl", "myclient"})); - assertEquals(output.get(0), "http://localhost:8081/"); - output.clear(); + assertEquals(output, "http://localhost:8081/\n"); + cleanOutput(); assertTrue(runCommand(new String[]{"set-property", "-p", "newConf", "-v", "myValue", "myclient"})); verify(pulsarShell, times(2)).reload(any()); - output.clear(); + cleanOutput(); assertTrue(runCommand(new String[]{"get-property", "-p", "newConf", "myclient"})); - assertEquals(output.get(0), "myValue"); - output.clear(); + assertEquals(output, "myValue\n"); + cleanOutput(); assertTrue(runCommand(new String[]{"view", "myclient"})); - assertEquals(output.get(0), "webServiceUrl=http://localhost:8081/\nbrokerServiceUrl" + - "=pulsar://localhost:6651/\nnewConf=myValue\n"); - output.clear(); + assertEquals(output, "webServiceUrl=http://localhost:8081/\nbrokerServiceUrl" + + "=pulsar://localhost:6651/\nnewConf=myValue\n\n"); + cleanOutput(); assertTrue(runCommand(new String[]{"set-property", "-p", "newConf", "-v", "myValue2", "myclient"})); verify(pulsarShell, times(3)).reload(any()); - output.clear(); + cleanOutput(); assertTrue(runCommand(new String[]{"get-property", "-p", "newConf", "myclient"})); - assertEquals(output.get(0), "myValue2"); - output.clear(); + assertEquals(output, "myValue2\n"); + cleanOutput(); assertTrue(runCommand(new String[]{"view", "myclient"})); - assertEquals(output.get(0), "webServiceUrl=http://localhost:8081/\nbrokerServiceUrl" + - "=pulsar://localhost:6651/\nnewConf=myValue2\n"); - output.clear(); + assertEquals(output, "webServiceUrl=http://localhost:8081/\nbrokerServiceUrl" + + "=pulsar://localhost:6651/\nnewConf=myValue2\n\n"); + cleanOutput(); assertTrue(runCommand(new String[]{"set-property", "-p", "newConf", "-v", "", "myclient"})); verify(pulsarShell, times(4)).reload(any()); - output.clear(); + cleanOutput(); assertTrue(runCommand(new String[]{"view", "myclient"})); - assertEquals(output.get(0), "webServiceUrl=http://localhost:8081/\nbrokerServiceUrl" + - "=pulsar://localhost:6651/\nnewConf=\n"); - output.clear(); + assertEquals(output, "webServiceUrl=http://localhost:8081/\nbrokerServiceUrl" + + "=pulsar://localhost:6651/\nnewConf=\n\n"); + cleanOutput(); assertTrue(runCommand(new String[]{"get-property", "-p", "newConf", "myclient"})); assertTrue(output.isEmpty()); - output.clear(); + cleanOutput(); } private boolean runCommand(String[] x) throws Exception { try { - return configShell.runCommand(x); + CommandLine commander = configShell.getCommander(); + return commander.execute(x) == 0; } finally { - configShell.setupState(null); + output = stringWriter.toString(); setConsole(); } } diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/shell/JCommanderCompleterTest.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/shell/JCommanderCompleterTest.java deleted file mode 100644 index 09981b100046e..0000000000000 --- a/pulsar-client-tools/src/test/java/org/apache/pulsar/shell/JCommanderCompleterTest.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.shell; - -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertTrue; - -import java.util.List; -import java.util.Properties; -import org.jline.reader.Completer; -import org.testng.annotations.Test; - -public class JCommanderCompleterTest { - - @Test - public void testCompletersAdmin() throws Exception { - final AdminShell shell = new AdminShell(new Properties()); - shell.setupState(new Properties()); - createAndCheckCompleters(shell, "admin"); - } - - @Test - public void testCompletersClient() throws Exception { - final AdminShell shell = new AdminShell(new Properties()); - shell.setupState(new Properties()); - createAndCheckCompleters(shell, "client"); - } - - private void createAndCheckCompleters(AdminShell shell, String mainCommand) { - final List completers = JCommanderCompleter.createCompletersForCommand(mainCommand, - shell.getJCommander(), null); - assertFalse(completers.isEmpty()); - for (Completer completer : completers) { - assertTrue(completer instanceof OptionStrictArgumentCompleter); - } - } -} diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/shell/PulsarShellTest.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/shell/PulsarShellTest.java index 2afb6f35b22c4..165fee923782b 100644 --- a/pulsar-client-tools/src/test/java/org/apache/pulsar/shell/PulsarShellTest.java +++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/shell/PulsarShellTest.java @@ -20,6 +20,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -36,8 +37,8 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicReference; import lombok.SneakyThrows; -import org.apache.pulsar.admin.cli.PulsarAdminSupplier; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminBuilder; import org.apache.pulsar.client.admin.Topics; import org.apache.pulsar.client.cli.CmdProduce; import org.jline.reader.EndOfFileException; @@ -99,9 +100,9 @@ public TestPulsarShell(String[] args, Properties props, PulsarAdmin pulsarAdmin) @Override protected AdminShell createAdminShell(Properties properties) throws Exception { final AdminShell adminShell = new AdminShell(properties); - final PulsarAdminSupplier supplier = mock(PulsarAdminSupplier.class); - when(supplier.get()).thenReturn(pulsarAdmin); - adminShell.setPulsarAdminSupplier(supplier); + PulsarAdminBuilder builder = mock(PulsarAdminBuilder.class); + doReturn(pulsarAdmin).when(builder).build(); + adminShell.getPulsarAdminSupplier().setAdminBuilder(builder); return adminShell; } @@ -109,9 +110,9 @@ protected AdminShell createAdminShell(Properties properties) throws Exception { protected ClientShell createClientShell(Properties properties) { final CmdProduce cmdProduce = mock(CmdProduce.class); cmdProduceHolder.set(cmdProduce); - return new ClientShell(properties) {{ - this.produceCommand = cmdProduce; - }}; + ClientShell clientShell = new ClientShell(properties); + clientShell.replaceProducerCommand(cmdProduce); + return clientShell; } @Override @@ -150,9 +151,9 @@ public void testInteractiveMode() throws Exception { linereader.addCmd("client produce -m msg my-topic"); linereader.addCmd("quit"); final TestPulsarShell testPulsarShell = new TestPulsarShell(new String[]{}, props, pulsarAdmin); - testPulsarShell.run((a) -> linereader, (a) -> terminal); + testPulsarShell.run((a) -> linereader, () -> terminal); verify(topics).createNonPartitionedTopic(eq("persistent://public/default/my-topic"), any(Map.class)); - verify(testPulsarShell.cmdProduceHolder.get()).run(); + verify(testPulsarShell.cmdProduceHolder.get()).call(); assertEquals((int) testPulsarShell.exitCode, 0); } @@ -169,9 +170,9 @@ public void testFileMode() throws Exception { final TestPulsarShell testPulsarShell = new TestPulsarShell(new String[]{"-f", shellFile}, props, pulsarAdmin); - testPulsarShell.run((a) -> linereader, (a) -> terminal); + testPulsarShell.run((a) -> linereader, () -> terminal); verify(topics).createNonPartitionedTopic(eq("persistent://public/default/my-topic"), any(Map.class)); - verify(testPulsarShell.cmdProduceHolder.get()).run(); + verify(testPulsarShell.cmdProduceHolder.get()).call(); } @Test @@ -187,7 +188,7 @@ public void testFileModeExitOnError() throws Exception { final TestPulsarShell testPulsarShell = new TestPulsarShell(new String[]{"-f", shellFile, "--fail-on-error"}, props, pulsarAdmin); try { - testPulsarShell.run((a) -> linereader, (a) -> terminal); + testPulsarShell.run((a) -> linereader, () -> terminal); fail(); } catch (SystemExitCalledException ex) { assertEquals(ex.code, 1); diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index 3386fedfe277a..bc8990fd2028b 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-client-original @@ -52,6 +51,16 @@ pkg + + io.opentelemetry + opentelemetry-api + + + + io.opentelemetry + opentelemetry-api-incubator + + ${project.groupId} @@ -73,6 +82,11 @@ netty-codec-socks + + io.swagger + swagger-annotations + + io.netty netty-resolver-dns diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java index 1827142cdfa2b..3ba7866350a0c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java @@ -25,6 +25,7 @@ import org.apache.pulsar.common.api.proto.CompressionType; import org.apache.pulsar.common.compression.CompressionCodec; import org.apache.pulsar.common.compression.CompressionCodecProvider; +import org.apache.pulsar.common.protocol.Commands; /** * Batch message container framework. @@ -35,7 +36,6 @@ public abstract class AbstractBatchMessageContainer implements BatchMessageConta protected CompressionType compressionType; protected CompressionCodec compressor; protected String topicName; - protected String producerName; protected ProducerImpl producer; protected int maxNumMessagesInBatch; @@ -54,19 +54,24 @@ public abstract class AbstractBatchMessageContainer implements BatchMessageConta // allocate a new buffer that can hold the entire batch without needing costly reallocations protected int maxBatchSize = INITIAL_BATCH_BUFFER_SIZE; protected int maxMessagesNum = INITIAL_MESSAGES_NUM; + private volatile long firstAddedTimestamp = 0L; @Override public boolean haveEnoughSpace(MessageImpl msg) { int messageSize = msg.getDataBuffer().readableBytes(); return ( - (maxBytesInBatch <= 0 && (messageSize + currentBatchSizeBytes) <= ClientCnx.getMaxMessageSize()) + (maxBytesInBatch <= 0 && (messageSize + currentBatchSizeBytes) <= getMaxMessageSize()) || (maxBytesInBatch > 0 && (messageSize + currentBatchSizeBytes) <= maxBytesInBatch) ) && (maxNumMessagesInBatch <= 0 || numMessagesInBatch < maxNumMessagesInBatch); } + protected int getMaxMessageSize() { + return producer != null && producer.getConnectionHandler() != null + ? producer.getConnectionHandler().getMaxMessageSize() : Commands.DEFAULT_MAX_MESSAGE_SIZE; + } protected boolean isBatchFull() { return (maxBytesInBatch > 0 && currentBatchSizeBytes >= maxBytesInBatch) - || (maxBytesInBatch <= 0 && currentBatchSizeBytes >= ClientCnx.getMaxMessageSize()) + || (maxBytesInBatch <= 0 && currentBatchSizeBytes >= getMaxMessageSize()) || (maxNumMessagesInBatch > 0 && numMessagesInBatch >= maxNumMessagesInBatch); } @@ -108,7 +113,6 @@ public ProducerImpl.OpSendMsg createOpSendMsg() throws IOException { public void setProducer(ProducerImpl producer) { this.producer = producer; this.topicName = producer.getTopic(); - this.producerName = producer.getProducerName(); this.compressionType = CompressionCodecProvider .convertToWireProtocol(producer.getConfiguration().getCompressionType()); this.compressor = CompressionCodecProvider.getCompressionCodec(compressionType); @@ -129,4 +133,19 @@ public boolean hasSameTxn(MessageImpl msg) { return currentTxnidMostBits == msg.getMessageBuilder().getTxnidMostBits() && currentTxnidLeastBits == msg.getMessageBuilder().getTxnidLeastBits(); } + + @Override + public long getFirstAddedTimestamp() { + return firstAddedTimestamp; + } + + protected void tryUpdateTimestamp() { + if (numMessagesInBatch == 1) { + firstAddedTimestamp = System.nanoTime(); + } + } + + protected void clearTimestamp() { + firstAddedTimestamp = 0L; + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerBase.java index 8fb4e9f2ce543..ddbe1bc255779 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerBase.java @@ -82,4 +82,11 @@ public interface BatchMessageContainerBase extends BatchMessageContainer { * @return belong to the same txn or not */ boolean hasSameTxn(MessageImpl msg); + + /** + * Get the timestamp in nanoseconds when the 1st message is added into the batch container. + * + * @return the timestamp in nanoseconds or 0L if the batch container is empty + */ + long getFirstAddedTimestamp(); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java index fdbf1f15c296a..44f1fb274655a 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java @@ -89,8 +89,8 @@ public BatchMessageContainerImpl(ProducerImpl producer) { public boolean add(MessageImpl msg, SendCallback callback) { if (log.isDebugEnabled()) { - log.debug("[{}] [{}] add message to batch, num messages in batch so far {}", topicName, producerName, - numMessagesInBatch); + log.debug("[{}] [{}] add message to batch, num messages in batch so far {}", topicName, + producer.getProducerName(), numMessagesInBatch); } if (++numMessagesInBatch == 1) { @@ -101,7 +101,7 @@ public boolean add(MessageImpl msg, SendCallback callback) { lowestSequenceId = Commands.initBatchMessageMetadata(messageMetadata, msg.getMessageBuilder()); this.firstCallback = callback; batchedMessageMetadataAndPayload = allocator.buffer( - Math.min(maxBatchSize, ClientCnx.getMaxMessageSize())); + Math.min(maxBatchSize, getMaxMessageSize())); updateAndReserveBatchAllocatedSize(batchedMessageMetadataAndPayload.capacity()); if (msg.getMessageBuilder().hasTxnidMostBits() && currentTxnidMostBits == -1) { currentTxnidMostBits = msg.getMessageBuilder().getTxnidMostBits(); @@ -127,6 +127,7 @@ public boolean add(MessageImpl msg, SendCallback callback) { previousCallback = callback; currentBatchSizeBytes += msg.getDataBuffer().readableBytes(); messages.add(msg); + tryUpdateTimestamp(); if (lowestSequenceId == -1L) { lowestSequenceId = msg.getSequenceId(); @@ -203,6 +204,7 @@ void updateMaxBatchSize(int uncompressedSize) { @Override public void clear() { + clearTimestamp(); messages = new ArrayList<>(maxMessagesNum); firstCallback = null; previousCallback = null; @@ -227,15 +229,15 @@ public void discard(Exception ex) { try { // Need to protect ourselves from any exception being thrown in the future handler from the application if (firstCallback != null) { - firstCallback.sendComplete(ex); + firstCallback.sendComplete(ex, null); } if (batchedMessageMetadataAndPayload != null) { ReferenceCountUtil.safeRelease(batchedMessageMetadataAndPayload); batchedMessageMetadataAndPayload = null; } } catch (Throwable t) { - log.warn("[{}] [{}] Got exception while completing the callback for msg {}:", topicName, producerName, - lowestSequenceId, t); + log.warn("[{}] [{}] Got exception while completing the callback for msg {}:", topicName, + producer.getProducerName(), lowestSequenceId, t); } clear(); } @@ -261,8 +263,8 @@ public OpSendMsg createOpSendMsg() throws IOException { // Because when invoke `ProducerImpl.processOpSendMsg` on flush, // if `op.msg != null && isBatchMessagingEnabled()` checks true, it will call `batchMessageAndSend` to flush // messageContainers before publishing this one-batch message. - op = OpSendMsg.create(messages, cmd, messageMetadata.getSequenceId(), firstCallback, - batchAllocatedSizeBytes); + op = OpSendMsg.create(producer.rpcLatencyHistogram, messages, cmd, messageMetadata.getSequenceId(), + firstCallback, batchAllocatedSizeBytes); // NumMessagesInBatch and BatchSizeByte will not be serialized to the binary cmd. It's just useful for the // ProducerStats @@ -270,12 +272,12 @@ public OpSendMsg createOpSendMsg() throws IOException { op.setBatchSizeByte(encryptedPayload.readableBytes()); // handle mgs size check as non-batched in `ProducerImpl.isMessageSizeExceeded` - if (op.getMessageHeaderAndPayloadSize() > ClientCnx.getMaxMessageSize()) { + if (op.getMessageHeaderAndPayloadSize() > getMaxMessageSize()) { producer.semaphoreRelease(1); producer.client.getMemoryLimitController().releaseMemory( messages.get(0).getUncompressedSize() + batchAllocatedSizeBytes); discard(new PulsarClientException.InvalidMessageException( - "Message size is bigger than " + ClientCnx.getMaxMessageSize() + " bytes")); + "Message size is bigger than " + getMaxMessageSize() + " bytes")); return null; } lowestSequenceId = -1L; @@ -283,13 +285,13 @@ public OpSendMsg createOpSendMsg() throws IOException { } ByteBuf encryptedPayload = producer.encryptMessage(messageMetadata, getCompressedBatchMetadataAndPayload()); updateAndReserveBatchAllocatedSize(encryptedPayload.capacity()); - if (encryptedPayload.readableBytes() > ClientCnx.getMaxMessageSize()) { + if (encryptedPayload.readableBytes() > getMaxMessageSize()) { producer.semaphoreRelease(messages.size()); messages.forEach(msg -> producer.client.getMemoryLimitController() .releaseMemory(msg.getUncompressedSize())); producer.client.getMemoryLimitController().releaseMemory(batchAllocatedSizeBytes); discard(new PulsarClientException.InvalidMessageException( - "Message size is bigger than " + ClientCnx.getMaxMessageSize() + " bytes")); + "Message size is bigger than " + getMaxMessageSize() + " bytes")); return null; } messageMetadata.setNumMessagesInBatch(numMessagesInBatch); @@ -304,7 +306,15 @@ public OpSendMsg createOpSendMsg() throws IOException { ByteBufPair cmd = producer.sendMessage(producer.producerId, messageMetadata.getSequenceId(), messageMetadata.getHighestSequenceId(), numMessagesInBatch, messageMetadata, encryptedPayload); - OpSendMsg op = OpSendMsg.create(messages, cmd, messageMetadata.getSequenceId(), + if (log.isDebugEnabled()) { + log.debug("[{}] [{}] Build batch msg seq:{}, highest-seq:{}, numMessagesInBatch: {}, uncompressedSize: {}," + + " payloadSize: {}", topicName, producer.getProducerName(), + messageMetadata.getSequenceId(), messageMetadata.getNumMessagesInBatch(), + messageMetadata.getHighestSequenceId(), + messageMetadata.getUncompressedSize(), encryptedPayload.readableBytes()); + } + + OpSendMsg op = OpSendMsg.create(producer.rpcLatencyHistogram, messages, cmd, messageMetadata.getSequenceId(), messageMetadata.getHighestSequenceId(), firstCallback, batchAllocatedSizeBytes); op.setNumMessagesInBatch(numMessagesInBatch); @@ -316,9 +326,11 @@ public OpSendMsg createOpSendMsg() throws IOException { protected void updateAndReserveBatchAllocatedSize(int updatedSizeBytes) { int delta = updatedSizeBytes - batchAllocatedSizeBytes; batchAllocatedSizeBytes = updatedSizeBytes; - if (delta != 0) { - if (producer != null) { + if (producer != null) { + if (delta > 0) { producer.client.getMemoryLimitController().forceReserveMemory(delta); + } else if (delta < 0) { + producer.client.getMemoryLimitController().releaseMemory(-delta); } } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java index 45d683e72b0a3..1592d3cae6cb5 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java @@ -43,8 +43,8 @@ class BatchMessageKeyBasedContainer extends AbstractBatchMessageContainer { @Override public boolean add(MessageImpl msg, SendCallback callback) { if (log.isDebugEnabled()) { - log.debug("[{}] [{}] add message to batch, num messages in batch so far is {}", topicName, producerName, - numMessagesInBatch); + log.debug("[{}] [{}] add message to batch, num messages in batch so far is {}", topicName, + producer.getProducerName(), numMessagesInBatch); } String key = getKey(msg); final BatchMessageContainerImpl batchMessageContainer = batches.computeIfAbsent(key, @@ -57,11 +57,13 @@ public boolean add(MessageImpl msg, SendCallback callback) { numMessagesInBatch++; currentBatchSizeBytes += msg.getDataBuffer().readableBytes(); } + tryUpdateTimestamp(); return isBatchFull(); } @Override public void clear() { + clearTimestamp(); numMessagesInBatch = 0; currentBatchSizeBytes = 0; batches.clear(); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BinaryProtoLookupService.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BinaryProtoLookupService.java index d5ce9213211dd..b45d6e9f6a80a 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BinaryProtoLookupService.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BinaryProtoLookupService.java @@ -19,20 +19,24 @@ package org.apache.pulsar.client.impl; import static java.lang.String.format; +import static org.apache.pulsar.client.api.PulsarClientException.FailedFeatureCheck.SupportsGetPartitionedMetadataWithoutAutoCreation; import io.netty.buffer.ByteBuf; +import io.opentelemetry.api.common.Attributes; import java.net.InetSocketAddress; import java.net.URI; -import java.util.ArrayList; -import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import org.apache.commons.lang3.mutable.MutableObject; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SchemaSerializationException; +import org.apache.pulsar.client.impl.metrics.LatencyHistogram; import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace.Mode; import org.apache.pulsar.common.api.proto.CommandLookupTopicResponse; import org.apache.pulsar.common.api.proto.CommandLookupTopicResponse.LookupType; @@ -43,6 +47,8 @@ import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.schema.BytesSchemaVersion; import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.util.Backoff; +import org.apache.pulsar.common.util.BackoffBuilder; import org.apache.pulsar.common.util.FutureUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,6 +62,17 @@ public class BinaryProtoLookupService implements LookupService { private final String listenerName; private final int maxLookupRedirects; + private final ConcurrentHashMap>, CompletableFuture> + lookupInProgress = new ConcurrentHashMap<>(); + + private final ConcurrentHashMap> + partitionedMetadataInProgress = new ConcurrentHashMap<>(); + + private final LatencyHistogram histoGetBroker; + private final LatencyHistogram histoGetTopicMetadata; + private final LatencyHistogram histoGetSchema; + private final LatencyHistogram histoListTopics; + public BinaryProtoLookupService(PulsarClientImpl client, String serviceUrl, boolean useTls, @@ -77,6 +94,15 @@ public BinaryProtoLookupService(PulsarClientImpl client, this.serviceNameResolver = new PulsarServiceNameResolver(); this.listenerName = listenerName; updateServiceUrl(serviceUrl); + + LatencyHistogram histo = client.instrumentProvider().newLatencyHistogram("pulsar.client.lookup.duration", + "Duration of lookup operations", null, + Attributes.builder().put("pulsar.lookup.transport-type", "binary").build()); + histoGetBroker = histo.withAttributes(Attributes.builder().put("pulsar.lookup.type", "topic").build()); + histoGetTopicMetadata = + histo.withAttributes(Attributes.builder().put("pulsar.lookup.type", "metadata").build()); + histoGetSchema = histo.withAttributes(Attributes.builder().put("pulsar.lookup.type", "schema").build()); + histoListTopics = histo.withAttributes(Attributes.builder().put("pulsar.lookup.type", "list-topics").build()); } @Override @@ -91,21 +117,62 @@ public void updateServiceUrl(String serviceUrl) throws PulsarClientException { * topic-name * @return broker-socket-address that serves given topic */ - public CompletableFuture> getBroker(TopicName topicName) { - return findBroker(serviceNameResolver.resolveHost(), false, topicName, 0); + public CompletableFuture getBroker(TopicName topicName) { + long startTime = System.nanoTime(); + final MutableObject newFutureCreated = new MutableObject<>(); + final Pair> key = Pair.of(topicName, + client.getConfiguration().getLookupProperties()); + try { + return lookupInProgress.computeIfAbsent(key, tpName -> { + CompletableFuture newFuture = findBroker(serviceNameResolver.resolveHost(), false, + topicName, 0, key.getRight()); + newFutureCreated.setValue(newFuture); + + newFuture.thenRun(() -> { + histoGetBroker.recordSuccess(System.nanoTime() - startTime); + }).exceptionally(x -> { + histoGetBroker.recordFailure(System.nanoTime() - startTime); + return null; + }); + return newFuture; + }); + } finally { + if (newFutureCreated.getValue() != null) { + newFutureCreated.getValue().whenComplete((v, ex) -> { + lookupInProgress.remove(key, newFutureCreated.getValue()); + }); + } + } } /** * calls broker binaryProto-lookup api to get metadata of partitioned-topic. * */ - public CompletableFuture getPartitionedTopicMetadata(TopicName topicName) { - return getPartitionedTopicMetadata(serviceNameResolver.resolveHost(), topicName); + @Override + public CompletableFuture getPartitionedTopicMetadata( + TopicName topicName, boolean metadataAutoCreationEnabled, boolean useFallbackForNonPIP344Brokers) { + final MutableObject newFutureCreated = new MutableObject<>(); + try { + return partitionedMetadataInProgress.computeIfAbsent(topicName, tpName -> { + CompletableFuture newFuture = getPartitionedTopicMetadata( + serviceNameResolver.resolveHost(), topicName, metadataAutoCreationEnabled, + useFallbackForNonPIP344Brokers); + newFutureCreated.setValue(newFuture); + return newFuture; + }); + } finally { + if (newFutureCreated.getValue() != null) { + newFutureCreated.getValue().whenComplete((v, ex) -> { + partitionedMetadataInProgress.remove(topicName, newFutureCreated.getValue()); + }); + } + } } - private CompletableFuture> findBroker(InetSocketAddress socketAddress, - boolean authoritative, TopicName topicName, final int redirectCount) { - CompletableFuture> addressFuture = new CompletableFuture<>(); + private CompletableFuture findBroker(InetSocketAddress socketAddress, + boolean authoritative, TopicName topicName, final int redirectCount, Map properties) { + CompletableFuture addressFuture = new CompletableFuture<>(); if (maxLookupRedirects > 0 && redirectCount > maxLookupRedirects) { addressFuture.completeExceptionally( @@ -115,7 +182,8 @@ private CompletableFuture> findBroker client.getCnxPool().getConnection(socketAddress).thenAccept(clientCnx -> { long requestId = client.newRequestId(); - ByteBuf request = Commands.newLookup(topicName.toString(), listenerName, authoritative, requestId); + ByteBuf request = Commands.newLookup(topicName.toString(), listenerName, authoritative, requestId, + properties); clientCnx.newLookup(request, requestId).whenComplete((r, t) -> { if (t != null) { // lookup failed @@ -123,7 +191,6 @@ private CompletableFuture> findBroker if (log.isDebugEnabled()) { log.debug("[{}] Lookup response exception: {}", topicName, t); } - addressFuture.completeExceptionally(t); } else { URI uri = null; @@ -141,7 +208,7 @@ private CompletableFuture> findBroker // (2) redirect to given address if response is: redirect if (r.redirect) { - findBroker(responseBrokerAddress, r.authoritative, topicName, redirectCount + 1) + findBroker(responseBrokerAddress, r.authoritative, topicName, redirectCount + 1, properties) .thenAccept(addressFuture::complete) .exceptionally((lookupException) -> { Throwable cause = FutureUtil.unwrapCompletionException(lookupException); @@ -162,10 +229,12 @@ private CompletableFuture> findBroker // (3) received correct broker to connect if (r.proxyThroughServiceUrl) { // Connect through proxy - addressFuture.complete(Pair.of(responseBrokerAddress, socketAddress)); + addressFuture.complete( + new LookupTopicResult(responseBrokerAddress, socketAddress, true)); } else { // Normal result with direct connection to broker - addressFuture.complete(Pair.of(responseBrokerAddress, responseBrokerAddress)); + addressFuture.complete( + new LookupTopicResult(responseBrokerAddress, responseBrokerAddress, false)); } } @@ -186,20 +255,41 @@ private CompletableFuture> findBroker } private CompletableFuture getPartitionedTopicMetadata(InetSocketAddress socketAddress, - TopicName topicName) { + TopicName topicName, boolean metadataAutoCreationEnabled, boolean useFallbackForNonPIP344Brokers) { + long startTime = System.nanoTime(); CompletableFuture partitionFuture = new CompletableFuture<>(); client.getCnxPool().getConnection(socketAddress).thenAccept(clientCnx -> { + boolean finalAutoCreationEnabled = metadataAutoCreationEnabled; + if (!metadataAutoCreationEnabled && !clientCnx.isSupportsGetPartitionedMetadataWithoutAutoCreation()) { + if (useFallbackForNonPIP344Brokers) { + log.info("[{}] Using original behavior of getPartitionedTopicMetadata(topic) in " + + "getPartitionedTopicMetadata(topic, false) " + + "since the target broker does not support PIP-344 and fallback is enabled.", topicName); + finalAutoCreationEnabled = true; + } else { + partitionFuture.completeExceptionally( + new PulsarClientException.FeatureNotSupportedException("The feature of " + + "getting partitions without auto-creation is not supported by the broker. " + + "Please upgrade the broker to version that supports PIP-344 to resolve this " + + "issue.", + SupportsGetPartitionedMetadataWithoutAutoCreation)); + return; + } + } long requestId = client.newRequestId(); - ByteBuf request = Commands.newPartitionMetadataRequest(topicName.toString(), requestId); + ByteBuf request = Commands.newPartitionMetadataRequest(topicName.toString(), requestId, + finalAutoCreationEnabled); clientCnx.newLookup(request, requestId).whenComplete((r, t) -> { if (t != null) { + histoGetTopicMetadata.recordFailure(System.nanoTime() - startTime); log.warn("[{}] failed to get Partitioned metadata : {}", topicName, t.getMessage(), t); partitionFuture.completeExceptionally(t); } else { try { + histoGetTopicMetadata.recordSuccess(System.nanoTime() - startTime); partitionFuture.complete(new PartitionedTopicMetadata(r.partitions)); } catch (Exception e) { partitionFuture.completeExceptionally(new PulsarClientException.LookupException( @@ -227,6 +317,7 @@ public CompletableFuture> getSchema(TopicName topicName) { @Override public CompletableFuture> getSchema(TopicName topicName, byte[] version) { + long startTime = System.nanoTime(); CompletableFuture> schemaFuture = new CompletableFuture<>(); if (version != null && version.length == 0) { schemaFuture.completeExceptionally(new SchemaSerializationException("Empty schema version")); @@ -239,10 +330,12 @@ public CompletableFuture> getSchema(TopicName topicName, by Optional.ofNullable(BytesSchemaVersion.of(version))); clientCnx.sendGetSchema(request, requestId).whenComplete((r, t) -> { if (t != null) { + histoGetSchema.recordFailure(System.nanoTime() - startTime); log.warn("[{}] failed to get schema : {}", topicName, t.getMessage(), t); schemaFuture.completeExceptionally(t); } else { + histoGetSchema.recordSuccess(System.nanoTime() - startTime); schemaFuture.complete(r); } client.getCnxPool().releaseConnection(clientCnx); @@ -290,6 +383,8 @@ private void getTopicsUnderNamespace(InetSocketAddress socketAddress, Mode mode, String topicsPattern, String topicsHash) { + long startTime = System.nanoTime(); + client.getCnxPool().getConnection(socketAddress).thenAccept(clientCnx -> { long requestId = client.newRequestId(); ByteBuf request = Commands.newGetTopicsOfNamespaceRequest( @@ -297,23 +392,15 @@ private void getTopicsUnderNamespace(InetSocketAddress socketAddress, clientCnx.newGetTopicsOfNamespace(request, requestId).whenComplete((r, t) -> { if (t != null) { + histoListTopics.recordFailure(System.nanoTime() - startTime); getTopicsResultFuture.completeExceptionally(t); } else { + histoListTopics.recordSuccess(System.nanoTime() - startTime); if (log.isDebugEnabled()) { log.debug("[namespace: {}] Success get topics list in request: {}", namespace, requestId); } - // do not keep partition part of topic name - List result = new ArrayList<>(); - r.getTopics().forEach(topic -> { - String filtered = TopicName.get(topic).getPartitionedTopicName(); - if (!result.contains(filtered)) { - result.add(filtered); - } - }); - - getTopicsResultFuture.complete(new GetTopicsResult(result, r.getTopicsHash(), - r.isFiltered(), r.isChanged())); + getTopicsResultFuture.complete(r); } client.getCnxPool().releaseConnection(clientCnx); }); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java index 7677045f0899b..6923218676743 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java @@ -19,8 +19,10 @@ package org.apache.pulsar.client.impl; import static com.google.common.base.Preconditions.checkArgument; +import io.opentelemetry.api.OpenTelemetry; import java.net.InetSocketAddress; import java.time.Clock; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -36,6 +38,8 @@ import org.apache.pulsar.client.api.SizeUnit; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.client.impl.conf.ConfigurationDataUtils; +import org.apache.pulsar.common.tls.InetAddressUtils; +import org.apache.pulsar.common.util.DefaultPulsarSslFactory; public class ClientBuilderImpl implements ClientBuilder { ClientConfigurationData conf; @@ -119,6 +123,12 @@ public ClientBuilder authentication(Authentication authentication) { return this; } + @Override + public ClientBuilder openTelemetry(OpenTelemetry openTelemetry) { + conf.setOpenTelemetry(openTelemetry); + return this; + } + @Override public ClientBuilder authentication(String authPluginClassName, String authParamsString) throws UnsupportedAuthenticationException { @@ -393,6 +403,17 @@ public ClientBuilder dnsLookupBind(String address, int port) { return this; } + @Override + public ClientBuilder dnsServerAddresses(List addresses) { + for (InetSocketAddress address : addresses) { + String ip = address.getHostString(); + checkArgument(InetAddressUtils.isIPv4Address(ip) || InetAddressUtils.isIPv6Address(ip), + "DnsServerAddresses need to be valid IPv4 or IPv6 addresses"); + } + conf.setDnsServerAddresses(addresses); + return this; + } + @Override public ClientBuilder socks5ProxyAddress(InetSocketAddress socks5ProxyAddress) { conf.setSocks5ProxyAddress(socks5ProxyAddress); @@ -411,6 +432,28 @@ public ClientBuilder socks5ProxyPassword(String socks5ProxyPassword) { return this; } + @Override + public ClientBuilder sslFactoryPlugin(String sslFactoryPlugin) { + if (StringUtils.isBlank(sslFactoryPlugin)) { + conf.setSslFactoryPlugin(DefaultPulsarSslFactory.class.getName()); + } else { + conf.setSslFactoryPlugin(sslFactoryPlugin); + } + return this; + } + + @Override + public ClientBuilder sslFactoryPluginParams(String sslFactoryPluginParams) { + conf.setSslFactoryPluginParams(sslFactoryPluginParams); + return this; + } + + @Override + public ClientBuilder autoCertRefreshSeconds(int autoCertRefreshSeconds) { + conf.setAutoCertRefreshSeconds(autoCertRefreshSeconds); + return this; + } + /** * Set the description. * @@ -433,4 +476,10 @@ public ClientBuilder description(String description) { conf.setDescription(description); return this; } + + @Override + public ClientBuilder lookupProperties(Map properties) { + conf.setLookupProperties(properties); + return this; + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index 115c71307c4f2..24163c631ffe9 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -30,9 +30,12 @@ import io.netty.channel.EventLoopGroup; import io.netty.channel.unix.Errors.NativeIoException; import io.netty.handler.codec.LengthFieldBasedFrameDecoder; +import io.netty.handler.ssl.SslHandshakeCompletionEvent; import io.netty.util.concurrent.Promise; +import io.opentelemetry.api.common.Attributes; import java.net.InetSocketAddress; import java.net.SocketAddress; +import java.net.URI; import java.net.URISyntaxException; import java.nio.channels.ClosedChannelException; import java.util.Arrays; @@ -58,6 +61,9 @@ import org.apache.pulsar.client.api.PulsarClientException.TimeoutException; import org.apache.pulsar.client.impl.BinaryProtoLookupService.LookupDataResult; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.impl.metrics.Counter; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; +import org.apache.pulsar.client.impl.metrics.Unit; import org.apache.pulsar.client.impl.schema.SchemaInfoUtil; import org.apache.pulsar.client.impl.transaction.TransactionBufferHandler; import org.apache.pulsar.client.util.TimedCompletableFuture; @@ -166,8 +172,7 @@ public class ClientCnx extends PulsarHandler { private volatile int numberOfRejectRequests = 0; @Getter - private static int maxMessageSize = Commands.DEFAULT_MAX_MESSAGE_SIZE; - + private int maxMessageSize = Commands.DEFAULT_MAX_MESSAGE_SIZE; private final int maxNumberOfRejectedRequestPerConnection; private final int rejectedRequestResetTimeSec = 60; protected final int protocolVersion; @@ -186,6 +191,8 @@ public class ClientCnx extends PulsarHandler { protected AuthenticationDataProvider authenticationDataProvider; private TransactionBufferHandler transactionBufferHandler; private boolean supportsTopicWatchers; + @Getter + private boolean supportsGetPartitionedMetadataWithoutAutoCreation; /** Idle stat. **/ @Getter @@ -200,6 +207,9 @@ protected enum State { None, SentConnectFrame, Ready, Failed, Connecting } + private final Counter connectionsOpenedCounter; + private final Counter connectionsClosedCounter; + private static class RequestTime { private final long creationTimeNanos; final long requestId; @@ -235,12 +245,13 @@ String getDescription() { } } - - public ClientCnx(ClientConfigurationData conf, EventLoopGroup eventLoopGroup) { - this(conf, eventLoopGroup, Commands.getCurrentProtocolVersion()); + public ClientCnx(InstrumentProvider instrumentProvider, + ClientConfigurationData conf, EventLoopGroup eventLoopGroup) { + this(instrumentProvider, conf, eventLoopGroup, Commands.getCurrentProtocolVersion()); } - public ClientCnx(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, int protocolVersion) { + public ClientCnx(InstrumentProvider instrumentProvider, ClientConfigurationData conf, EventLoopGroup eventLoopGroup, + int protocolVersion) { super(conf.getKeepAliveIntervalSeconds(), TimeUnit.SECONDS); checkArgument(conf.getMaxLookupRequest() > conf.getConcurrentLookupRequest()); this.pendingLookupRequestSemaphore = new Semaphore(conf.getConcurrentLookupRequest(), false); @@ -256,11 +267,19 @@ public ClientCnx(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, in this.idleState = new ClientCnxIdleState(this); this.clientVersion = "Pulsar-Java-v" + PulsarVersion.getVersion() + (conf.getDescription() == null ? "" : ("-" + conf.getDescription())); + this.connectionsOpenedCounter = + instrumentProvider.newCounter("pulsar.client.connection.opened", Unit.Connections, + "The number of connections opened", null, Attributes.empty()); + this.connectionsClosedCounter = + instrumentProvider.newCounter("pulsar.client.connection.closed", Unit.Connections, + "The number of connections closed", null, Attributes.empty()); + } @Override public void channelActive(ChannelHandlerContext ctx) throws Exception { super.channelActive(ctx); + connectionsOpenedCounter.increment(); this.localAddress = ctx.channel().localAddress(); this.remoteAddress = ctx.channel().remoteAddress(); @@ -303,6 +322,7 @@ protected ByteBuf newConnectCommand() throws Exception { @Override public void channelInactive(ChannelHandlerContext ctx) throws Exception { super.channelInactive(ctx); + connectionsClosedCounter.increment(); lastDisconnectedTimestamp = System.currentTimeMillis(); log.info("{} Disconnected", ctx.channel()); if (!connectionFuture.isDone()) { @@ -321,8 +341,8 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { waitingLookupRequests.forEach(pair -> pair.getRight().getRight().completeExceptionally(e)); // Notify all attached producers/consumers so they have a chance to reconnect - producers.forEach((id, producer) -> producer.connectionClosed(this)); - consumers.forEach((id, consumer) -> consumer.connectionClosed(this)); + producers.forEach((id, producer) -> producer.connectionClosed(this, Optional.empty(), Optional.empty())); + consumers.forEach((id, consumer) -> consumer.connectionClosed(this, Optional.empty(), Optional.empty())); transactionMetaStoreHandlers.forEach((id, handler) -> handler.connectionClosed(this)); topicListWatchers.forEach((__, watcher) -> watcher.connectionClosed(this)); @@ -382,6 +402,9 @@ protected void handleConnected(CommandConnected connected) { supportsTopicWatchers = connected.hasFeatureFlags() && connected.getFeatureFlags().isSupportsTopicWatchers(); + supportsGetPartitionedMetadataWithoutAutoCreation = + connected.hasFeatureFlags() + && connected.getFeatureFlags().isSupportsGetPartitionedMetadataWithoutAutoCreation(); // set remote protocol version to the correct version before we complete the connection future setRemoteEndpointProtocolVersion(connected.getProtocolVersion()); @@ -760,11 +783,9 @@ protected void handleSendError(CommandSendError sendError) { case NotAllowedError: producers.get(producerId).recoverNotAllowedError(sequenceId, sendError.getMessage()); break; - default: - // By default, for transient error, let the reconnection logic - // to take place and re-establish the produce again - ctx.close(); + // don't close this ctx, otherwise it will close all consumers and producers which use this ctx + producers.get(producerId).connectionClosed(this, Optional.empty(), Optional.empty()); } } @@ -800,28 +821,75 @@ protected void handleError(CommandError error) { @Override protected void handleCloseProducer(CommandCloseProducer closeProducer) { - log.info("[{}] Broker notification of Closed producer: {}", remoteAddress, closeProducer.getProducerId()); final long producerId = closeProducer.getProducerId(); + log.info("[{}] Broker notification of closed producer: {}, assignedBrokerUrl: {}, assignedBrokerUrlTls: {}", + remoteAddress, producerId, + closeProducer.hasAssignedBrokerServiceUrl() ? closeProducer.getAssignedBrokerServiceUrl() : null, + closeProducer.hasAssignedBrokerServiceUrlTls() ? closeProducer.getAssignedBrokerServiceUrlTls() : null); ProducerImpl producer = producers.remove(producerId); if (producer != null) { - producer.connectionClosed(this); + String brokerServiceUrl = getBrokerServiceUrl(closeProducer, producer); + Optional hostUri = parseUri(brokerServiceUrl, + closeProducer.hasRequestId() ? closeProducer.getRequestId() : null); + Optional initialConnectionDelayMs = hostUri.map(__ -> 0L); + producer.connectionClosed(this, initialConnectionDelayMs, hostUri); } else { - log.warn("Producer with id {} not found while closing producer ", producerId); + log.warn("[{}] Producer with id {} not found while closing producer", remoteAddress, producerId); } } + private static String getBrokerServiceUrl(CommandCloseProducer closeProducer, ProducerImpl producer) { + if (producer.getClient().getConfiguration().isUseTls()) { + if (closeProducer.hasAssignedBrokerServiceUrlTls()) { + return closeProducer.getAssignedBrokerServiceUrlTls(); + } + } else if (closeProducer.hasAssignedBrokerServiceUrl()) { + return closeProducer.getAssignedBrokerServiceUrl(); + } + return null; + } + @Override protected void handleCloseConsumer(CommandCloseConsumer closeConsumer) { - log.info("[{}] Broker notification of Closed consumer: {}", remoteAddress, closeConsumer.getConsumerId()); final long consumerId = closeConsumer.getConsumerId(); + log.info("[{}] Broker notification of closed consumer: {}, assignedBrokerUrl: {}, assignedBrokerUrlTls: {}", + remoteAddress, consumerId, + closeConsumer.hasAssignedBrokerServiceUrl() ? closeConsumer.getAssignedBrokerServiceUrl() : null, + closeConsumer.hasAssignedBrokerServiceUrlTls() ? closeConsumer.getAssignedBrokerServiceUrlTls() : null); ConsumerImpl consumer = consumers.remove(consumerId); if (consumer != null) { - consumer.connectionClosed(this); + String brokerServiceUrl = getBrokerServiceUrl(closeConsumer, consumer); + Optional hostUri = parseUri(brokerServiceUrl, + closeConsumer.hasRequestId() ? closeConsumer.getRequestId() : null); + Optional initialConnectionDelayMs = hostUri.map(__ -> 0L); + consumer.connectionClosed(this, initialConnectionDelayMs, hostUri); } else { - log.warn("Consumer with id {} not found while closing consumer ", consumerId); + log.warn("[{}] Consumer with id {} not found while closing consumer", remoteAddress, consumerId); } } + private static String getBrokerServiceUrl(CommandCloseConsumer closeConsumer, ConsumerImpl consumer) { + if (consumer.getClient().getConfiguration().isUseTls()) { + if (closeConsumer.hasAssignedBrokerServiceUrlTls()) { + return closeConsumer.getAssignedBrokerServiceUrlTls(); + } + } else if (closeConsumer.hasAssignedBrokerServiceUrl()) { + return closeConsumer.getAssignedBrokerServiceUrl(); + } + return null; + } + + private Optional parseUri(String url, Long requestId) { + try { + if (url != null) { + return Optional.of(new URI(url)); + } + } catch (URISyntaxException e) { + log.warn("[{}] Invalid redirect URL {}, requestId {}: ", remoteAddress, url, requestId, e); + } + return Optional.empty(); + } + @Override protected boolean isHandshakeCompleted() { return state == State.Ready; @@ -948,10 +1016,6 @@ Channel channel() { return ctx.channel(); } - SocketAddress serverAddrees() { - return remoteAddress; - } - CompletableFuture connectionFuture() { return connectionFuture; } @@ -1291,6 +1355,8 @@ public static PulsarClientException getPulsarClientException(ServerError error, return new PulsarClientException.IncompatibleSchemaException(errorMsg); case TopicNotFound: return new PulsarClientException.TopicDoesNotExistException(errorMsg); + case SubscriptionNotFound: + return new PulsarClientException.SubscriptionNotFoundException(errorMsg); case ConsumerAssignError: return new PulsarClientException.ConsumerAssignException(errorMsg); case NotAllowedError: @@ -1311,6 +1377,18 @@ public void close() { } } + @Override + public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { + if (evt instanceof SslHandshakeCompletionEvent) { + SslHandshakeCompletionEvent sslHandshakeCompletionEvent = (SslHandshakeCompletionEvent) evt; + if (sslHandshakeCompletionEvent.cause() != null) { + log.warn("{} Got ssl handshake exception {}", ctx.channel(), + sslHandshakeCompletionEvent); + } + } + ctx.fireUserEventTriggered(evt); + } + protected void closeWithException(Throwable e) { if (ctx != null) { connectionFuture.completeExceptionally(e); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java index 046cb90643a23..934985949197c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java @@ -19,12 +19,19 @@ package org.apache.pulsar.client.impl; import java.net.InetSocketAddress; +import java.net.URI; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import lombok.Getter; +import lombok.Setter; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.impl.HandlerState.State; +import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.util.Backoff; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +40,10 @@ public class ConnectionHandler { AtomicReferenceFieldUpdater.newUpdater(ConnectionHandler.class, ClientCnx.class, "clientCnx"); @SuppressWarnings("unused") private volatile ClientCnx clientCnx = null; + @Getter + @Setter + // Since the `clientCnx` variable will be set to null at some times, it is necessary to save this value here. + private volatile int maxMessageSize = Commands.DEFAULT_MAX_MESSAGE_SIZE; protected final HandlerState state; protected final Backoff backoff; @@ -41,22 +52,42 @@ public class ConnectionHandler { // Start with -1L because it gets incremented before sending on the first connection private volatile long epoch = -1L; protected volatile long lastConnectionClosedTimestamp = 0L; + private final AtomicBoolean duringConnect = new AtomicBoolean(false); + protected final int randomKeyForSelectConnection; + + private volatile Boolean useProxy; interface Connection { - void connectionFailed(PulsarClientException exception); - void connectionOpened(ClientCnx cnx); + + /** + * @apiNote If the returned future is completed exceptionally, reconnectLater will be called. + */ + CompletableFuture connectionOpened(ClientCnx cnx); + default void connectionFailed(PulsarClientException e) { + } } protected Connection connection; protected ConnectionHandler(HandlerState state, Backoff backoff, Connection connection) { this.state = state; + this.randomKeyForSelectConnection = state.client.getCnxPool().genRandomKeyToSelectCon(); this.connection = connection; this.backoff = backoff; CLIENT_CNX_UPDATER.set(this, null); } protected void grabCnx() { + grabCnx(Optional.empty()); + } + + protected void grabCnx(Optional hostURI) { + if (!duringConnect.compareAndSet(false, true)) { + log.info("[{}] [{}] Skip grabbing the connection since there is a pending connection", + state.topic, state.getHandlerName()); + return; + } + if (CLIENT_CNX_UPDATER.get(this) != null) { log.warn("[{}] [{}] Client cnx already set, ignoring reconnection request", state.topic, state.getHandlerName()); @@ -72,16 +103,36 @@ protected void grabCnx() { try { CompletableFuture cnxFuture; - if (state.redirectedClusterURI != null) { - InetSocketAddress address = InetSocketAddress.createUnresolved(state.redirectedClusterURI.getHost(), - state.redirectedClusterURI.getPort()); - cnxFuture = state.client.getConnection(address, address); + if (hostURI.isPresent() && useProxy != null) { + URI uri = hostURI.get(); + InetSocketAddress address = InetSocketAddress.createUnresolved(uri.getHost(), uri.getPort()); + if (useProxy) { + cnxFuture = state.client.getProxyConnection(address, randomKeyForSelectConnection); + } else { + cnxFuture = state.client.getConnection(address, address, randomKeyForSelectConnection); + } + } else if (state.redirectedClusterURI != null) { + if (state.topic == null) { + InetSocketAddress address = InetSocketAddress.createUnresolved(state.redirectedClusterURI.getHost(), + state.redirectedClusterURI.getPort()); + cnxFuture = state.client.getConnection(address, address, randomKeyForSelectConnection); + } else { + // once, client receives redirection url, client has to perform lookup on migrated + // cluster to find the broker that owns the topic and then create connection. + // below method, performs the lookup for a given topic and then creates connection + cnxFuture = state.client.getConnection(state.topic, (state.redirectedClusterURI.toString())); + } } else if (state.topic == null) { cnxFuture = state.client.getConnectionToServiceUrl(); } else { - cnxFuture = state.client.getConnection(state.topic); // + cnxFuture = state.client.getConnection(state.topic, randomKeyForSelectConnection).thenApply( + connectionResult -> { + useProxy = connectionResult.getRight(); + return connectionResult.getLeft(); + }); } - cnxFuture.thenAccept(cnx -> connection.connectionOpened(cnx)) // + cnxFuture.thenCompose(cnx -> connection.connectionOpened(cnx)) + .thenAccept(__ -> duringConnect.set(false)) .exceptionally(this::handleConnectionError); } catch (Throwable t) { log.warn("[{}] [{}] Exception thrown while getting connection: ", state.topic, state.getHandlerName(), t); @@ -90,26 +141,28 @@ protected void grabCnx() { } private Void handleConnectionError(Throwable exception) { - log.warn("[{}] [{}] Error connecting to broker: {}", - state.topic, state.getHandlerName(), exception.getMessage()); - if (exception instanceof PulsarClientException) { - connection.connectionFailed((PulsarClientException) exception); - } else if (exception.getCause() instanceof PulsarClientException) { - connection.connectionFailed((PulsarClientException) exception.getCause()); - } else { - connection.connectionFailed(new PulsarClientException(exception)); - } - - State state = this.state.getState(); - if (state == State.Uninitialized || state == State.Connecting || state == State.Ready) { - reconnectLater(exception); + try { + log.warn("[{}] [{}] Error connecting to broker: {}", + state.topic, state.getHandlerName(), exception.getMessage()); + if (exception instanceof PulsarClientException) { + connection.connectionFailed((PulsarClientException) exception); + } else if (exception.getCause() instanceof PulsarClientException) { + connection.connectionFailed((PulsarClientException) exception.getCause()); + } else { + connection.connectionFailed(new PulsarClientException(exception)); + } + } catch (Throwable throwable) { + log.error("[{}] [{}] Unexpected exception after the connection", + state.topic, state.getHandlerName(), throwable); } + reconnectLater(exception); return null; } - protected void reconnectLater(Throwable exception) { + void reconnectLater(Throwable exception) { CLIENT_CNX_UPDATER.set(this, null); + duringConnect.set(false); if (!isValidStateForReconnection()) { log.info("[{}] [{}] Ignoring reconnection request (state: {})", state.topic, state.getHandlerName(), state.getState()); @@ -131,7 +184,12 @@ protected void reconnectLater(Throwable exception) { } public void connectionClosed(ClientCnx cnx) { + connectionClosed(cnx, Optional.empty(), Optional.empty()); + } + + public void connectionClosed(ClientCnx cnx, Optional initialConnectionDelayMs, Optional hostUrl) { lastConnectionClosedTimestamp = System.currentTimeMillis(); + duringConnect.set(false); state.client.getCnxPool().releaseConnection(cnx); if (CLIENT_CNX_UPDATER.compareAndSet(this, cnx, null)) { if (!isValidStateForReconnection()) { @@ -139,14 +197,14 @@ public void connectionClosed(ClientCnx cnx) { state.topic, state.getHandlerName(), state.getState()); return; } - long delayMs = backoff.next(); + long delayMs = initialConnectionDelayMs.orElse(backoff.next()); state.setState(State.Connecting); - log.info("[{}] [{}] Closed connection {} -- Will try again in {} s", - state.topic, state.getHandlerName(), cnx.channel(), - delayMs / 1000.0); + log.info("[{}] [{}] Closed connection {} -- Will try again in {} s, hostUrl: {}", + state.topic, state.getHandlerName(), cnx.channel(), delayMs / 1000.0, hostUrl.orElse(null)); state.client.timer().newTimeout(timeout -> { - log.info("[{}] [{}] Reconnecting after timeout", state.topic, state.getHandlerName()); - grabCnx(); + log.info("[{}] [{}] Reconnecting after {} s timeout, hostUrl: {}", + state.topic, state.getHandlerName(), delayMs / 1000.0, hostUrl.orElse(null)); + grabCnx(hostUrl); }, delayMs, TimeUnit.MILLISECONDS); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java index 1420d81c688ee..a6a809af8585b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java @@ -29,8 +29,10 @@ import io.netty.resolver.AddressResolver; import io.netty.resolver.dns.DnsAddressResolverGroup; import io.netty.resolver.dns.DnsNameResolverBuilder; +import io.netty.resolver.dns.SequentialDnsServerAddressStreamProvider; import io.netty.util.concurrent.Future; import io.netty.util.concurrent.ScheduledFuture; +import io.opentelemetry.api.common.Attributes; import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; @@ -45,13 +47,18 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import java.util.stream.Collectors; +import lombok.Value; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.PulsarClientException.InvalidServiceURL; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.impl.metrics.Counter; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; +import org.apache.pulsar.client.impl.metrics.Unit; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.netty.DnsResolverUtil; @@ -61,9 +68,9 @@ public class ConnectionPool implements AutoCloseable { - public static final int IDLE_DETECTION_INTERVAL_SECONDS_MIN = 60; + public static final int IDLE_DETECTION_INTERVAL_SECONDS_MIN = 15; - protected final ConcurrentHashMap>> pool; + protected final ConcurrentMap> pool; private final Bootstrap bootstrap; private final PulsarChannelInitializer channelInitializerHandler; @@ -86,18 +93,36 @@ public class ConnectionPool implements AutoCloseable { /** Async release useless connections task. **/ private ScheduledFuture asyncReleaseUselessConnectionsTask; - public ConnectionPool(ClientConfigurationData conf, EventLoopGroup eventLoopGroup) throws PulsarClientException { - this(conf, eventLoopGroup, () -> new ClientCnx(conf, eventLoopGroup)); + private final Counter connectionsTcpFailureCounter; + private final Counter connectionsHandshakeFailureCounter; + + @Value + private static class Key { + InetSocketAddress logicalAddress; + InetSocketAddress physicalAddress; + int randomKey; + } + + public ConnectionPool(InstrumentProvider instrumentProvider, + ClientConfigurationData conf, EventLoopGroup eventLoopGroup, + ScheduledExecutorService scheduledExecutorService) throws PulsarClientException { + this(instrumentProvider, conf, eventLoopGroup, () -> new ClientCnx(instrumentProvider, conf, eventLoopGroup), + scheduledExecutorService); } - public ConnectionPool(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, - Supplier clientCnxSupplier) throws PulsarClientException { - this(conf, eventLoopGroup, clientCnxSupplier, Optional.empty()); + public ConnectionPool(InstrumentProvider instrumentProvider, + ClientConfigurationData conf, EventLoopGroup eventLoopGroup, + Supplier clientCnxSupplier, + ScheduledExecutorService scheduledExecutorService) throws PulsarClientException { + this(instrumentProvider, conf, eventLoopGroup, clientCnxSupplier, Optional.empty(), + scheduledExecutorService); } - public ConnectionPool(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, + public ConnectionPool(InstrumentProvider instrumentProvider, + ClientConfigurationData conf, EventLoopGroup eventLoopGroup, Supplier clientCnxSupplier, - Optional> addressResolver) + Optional> addressResolver, + ScheduledExecutorService scheduledExecutorService) throws PulsarClientException { this.eventLoopGroup = eventLoopGroup; this.clientConfig = conf; @@ -115,7 +140,8 @@ public ConnectionPool(ClientConfigurationData conf, EventLoopGroup eventLoopGrou bootstrap.option(ChannelOption.ALLOCATOR, PulsarByteBufAllocator.DEFAULT); try { - channelInitializerHandler = new PulsarChannelInitializer(conf, clientCnxSupplier); + channelInitializerHandler = new PulsarChannelInitializer(conf, clientCnxSupplier, + scheduledExecutorService); bootstrap.handler(channelInitializerHandler); } catch (Exception e) { log.error("Failed to create channel initializer"); @@ -145,6 +171,14 @@ public ConnectionPool(ClientConfigurationData conf, EventLoopGroup eventLoopGrou } }, idleDetectionIntervalSeconds, idleDetectionIntervalSeconds, TimeUnit.SECONDS); } + + connectionsTcpFailureCounter = + instrumentProvider.newCounter("pulsar.client.connection.failed", Unit.Connections, + "The number of failed connection attempts", null, + Attributes.builder().put("pulsar.failure.type", "tcp-failed").build()); + connectionsHandshakeFailureCounter = instrumentProvider.newCounter("pulsar.client.connection.failed", + Unit.Connections, "The number of failed connection attempts", null, + Attributes.builder().put("pulsar.failure.type", "handshake").build()); } private static AddressResolver createAddressResolver(ClientConfigurationData conf, @@ -156,6 +190,10 @@ private static AddressResolver createAddressResolver(ClientCo conf.getDnsLookupBindPort()); dnsNameResolverBuilder.localAddress(addr); } + List serverAddresses = conf.getDnsServerAddresses(); + if (serverAddresses != null && !serverAddresses.isEmpty()) { + dnsNameResolverBuilder.nameServerProvider(new SequentialDnsServerAddressStreamProvider(serverAddresses)); + } DnsResolverUtil.applyJdkDnsCacheSettings(dnsNameResolverBuilder); // use DnsAddressResolverGroup to create the AddressResolver since it contains a solution // to prevent cache stampede / thundering herds problem when a DNS entry expires while the system @@ -165,12 +203,22 @@ private static AddressResolver createAddressResolver(ClientCo private static final Random random = new Random(); + public int genRandomKeyToSelectCon() { + if (maxConnectionsPerHosts == 0) { + return -1; + } + return signSafeMod(random.nextInt(), maxConnectionsPerHosts); + } + public CompletableFuture getConnection(final InetSocketAddress address) { - return getConnection(address, address); + if (maxConnectionsPerHosts == 0) { + return getConnection(address, address, -1); + } + return getConnection(address, address, signSafeMod(random.nextInt(), maxConnectionsPerHosts)); } void closeAllConnections() { - pool.values().forEach(map -> map.values().forEach(future -> { + pool.values().forEach(future -> { if (future.isDone()) { if (!future.isCompletedExceptionally()) { // Connection was already created successfully, the join will not throw any exception @@ -183,10 +231,9 @@ void closeAllConnections() { // succeed future.thenAccept(ClientCnx::close); } - })); + }); } - - /** + /** * Get a connection from the pool. *

      * The connection can either be created or be coming from the pool itself. @@ -204,56 +251,47 @@ void closeAllConnections() { * @return a future that will produce the ClientCnx object */ public CompletableFuture getConnection(InetSocketAddress logicalAddress, - InetSocketAddress physicalAddress) { + InetSocketAddress physicalAddress, final int randomKey) { if (maxConnectionsPerHosts == 0) { // Disable pooling - return createConnection(logicalAddress, physicalAddress, -1); + return createConnection(new Key(logicalAddress, physicalAddress, -1)); } - - final int randomKey = signSafeMod(random.nextInt(), maxConnectionsPerHosts); - - final ConcurrentMap> innerPool = - pool.computeIfAbsent(logicalAddress, a -> new ConcurrentHashMap<>()); - CompletableFuture completableFuture = innerPool - .computeIfAbsent(randomKey, k -> createConnection(logicalAddress, physicalAddress, randomKey)); + Key key = new Key(logicalAddress, physicalAddress, randomKey); + CompletableFuture completableFuture = pool.computeIfAbsent(key, k -> createConnection(key)); if (completableFuture.isCompletedExceptionally()) { // we cannot cache a failed connection, so we remove it from the pool // there is a race condition in which // cleanupConnection is called before caching this result // and so the clean up fails - cleanupConnection(logicalAddress, randomKey, completableFuture); + pool.remove(key, completableFuture); return completableFuture; } return completableFuture.thenCompose(clientCnx -> { // If connection already release, create a new one. if (clientCnx.getIdleState().isReleased()) { - cleanupConnection(logicalAddress, randomKey, completableFuture); - return innerPool - .computeIfAbsent(randomKey, k -> createConnection(logicalAddress, physicalAddress, randomKey)); + pool.remove(key, completableFuture); + return pool.computeIfAbsent(key, k -> createConnection(key)); } // Try use exists connection. if (clientCnx.getIdleState().tryMarkUsingAndClearIdleTime()) { return CompletableFuture.completedFuture(clientCnx); } else { // If connection already release, create a new one. - cleanupConnection(logicalAddress, randomKey, completableFuture); - return innerPool - .computeIfAbsent(randomKey, k -> createConnection(logicalAddress, physicalAddress, randomKey)); + pool.remove(key, completableFuture); + return pool.computeIfAbsent(key, k -> createConnection(key)); } }); } - private CompletableFuture createConnection(InetSocketAddress logicalAddress, - InetSocketAddress physicalAddress, int connectionKey) { + private CompletableFuture createConnection(Key key) { if (log.isDebugEnabled()) { - log.debug("Connection for {} not found in cache", logicalAddress); + log.debug("Connection for {} not found in cache", key.logicalAddress); } final CompletableFuture cnxFuture = new CompletableFuture<>(); - // Trigger async connect to broker - createConnection(logicalAddress, physicalAddress).thenAccept(channel -> { + createConnection(key.logicalAddress, key.physicalAddress).thenAccept(channel -> { log.info("[{}] Connected to server", channel); channel.closeFuture().addListener(v -> { @@ -261,7 +299,7 @@ private CompletableFuture createConnection(InetSocketAddress logicalA if (log.isDebugEnabled()) { log.debug("Removing closed connection from pool: {}", v); } - cleanupConnection(logicalAddress, connectionKey, cnxFuture); + pool.remove(key, cnxFuture); }); // We are connected to broker, but need to wait until the connect/connected handshake is @@ -281,20 +319,22 @@ private CompletableFuture createConnection(InetSocketAddress logicalA } cnxFuture.complete(cnx); }).exceptionally(exception -> { + connectionsHandshakeFailureCounter.increment(); log.warn("[{}] Connection handshake failed: {}", cnx.channel(), exception.getMessage()); cnxFuture.completeExceptionally(exception); // this cleanupConnection may happen before that the // CompletableFuture is cached into the "pool" map, // it is not enough to clean it here, we need to clean it // in the "pool" map when the CompletableFuture is cached - cleanupConnection(logicalAddress, connectionKey, cnxFuture); + pool.remove(key, cnxFuture); cnx.ctx().close(); return null; }); }).exceptionally(exception -> { + connectionsTcpFailureCounter.increment(); eventLoopGroup.execute(() -> { - log.warn("Failed to open connection to {} : {}", physicalAddress, exception.getMessage()); - cleanupConnection(logicalAddress, connectionKey, cnxFuture); + log.warn("Failed to open connection to {} : {}", key.physicalAddress, exception.getMessage()); + pool.remove(key, cnxFuture); cnxFuture.completeExceptionally(new PulsarClientException(exception)); }); return null; @@ -426,17 +466,9 @@ public void close() throws Exception { } } - private void cleanupConnection(InetSocketAddress address, int connectionKey, - CompletableFuture connectionFuture) { - ConcurrentMap> map = pool.get(address); - if (map != null) { - map.remove(connectionKey, connectionFuture); - } - } - @VisibleForTesting int getPoolSize() { - return pool.values().stream().mapToInt(Map::size).sum(); + return pool.size(); } private static final Logger log = LoggerFactory.getLogger(ConnectionPool.class); @@ -446,11 +478,8 @@ public void doMarkAndReleaseUselessConnections(){ return; } List releaseIdleConnectionTaskList = new ArrayList<>(); - for (Map.Entry>> entry : - pool.entrySet()){ - ConcurrentMap> innerPool = entry.getValue(); - for (Map.Entry> entry0 : innerPool.entrySet()) { - CompletableFuture future = entry0.getValue(); + for (Map.Entry> entry : pool.entrySet()) { + CompletableFuture future = entry.getValue(); // Ensure connection has been connected. if (!future.isDone()) { continue; @@ -468,18 +497,17 @@ public void doMarkAndReleaseUselessConnections(){ if (clientCnx.getIdleState().isReleasing()) { releaseIdleConnectionTaskList.add(() -> { if (clientCnx.getIdleState().tryMarkReleasedAndCloseConnection()) { - cleanupConnection(entry.getKey(), entry0.getKey(), future); + pool.remove(entry.getKey(), future); } }); } } - } // Do release idle connections. releaseIdleConnectionTaskList.forEach(Runnable::run); } public Set> getConnections() { return Collections.unmodifiableSet( - pool.values().stream().flatMap(n -> n.values().stream()).collect(Collectors.toSet())); + pool.values().stream().collect(Collectors.toSet())); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index 0db2a8e0ab9f5..3073f3a833487 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Queues; import io.netty.util.Timeout; import java.nio.charset.StandardCharsets; @@ -30,6 +31,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -49,6 +51,7 @@ import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.MessageListener; +import org.apache.pulsar.client.api.MessageListenerExecutor; import org.apache.pulsar.client.api.Messages; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; @@ -64,7 +67,7 @@ import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.apache.pulsar.common.util.collections.BitSetRecyclable; import org.apache.pulsar.common.util.collections.GrowableArrayBlockingQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,14 +83,20 @@ public abstract class ConsumerBase extends HandlerState implements Consumer listener; protected final ConsumerEventListener consumerEventListener; protected final ExecutorProvider executorProvider; + protected final MessageListenerExecutor messageListenerExecutor; protected final ExecutorService externalPinnedExecutor; protected final ExecutorService internalPinnedExecutor; protected UnAckedMessageTracker unAckedMessageTracker; final GrowableArrayBlockingQueue> incomingMessages; - protected ConcurrentOpenHashMap unAckedChunkedMessageIdSequenceMap; + protected Map unAckedChunkedMessageIdSequenceMap = new ConcurrentHashMap<>(); protected final ConcurrentLinkedQueue>> pendingReceives; protected final int maxReceiverQueueSize; private volatile int currentReceiverQueueSize; + + protected static final AtomicIntegerFieldUpdater MESSAGE_LISTENER_QUEUE_SIZE_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(ConsumerBase.class, "messageListenerQueueSize"); + protected volatile int messageListenerQueueSize = 0; + protected static final AtomicIntegerFieldUpdater CURRENT_RECEIVER_QUEUE_SIZE_UPDATER = AtomicIntegerFieldUpdater.newUpdater(ConsumerBase.class, "currentReceiverQueueSize"); protected final Schema schema; @@ -129,9 +138,12 @@ protected ConsumerBase(PulsarClientImpl client, String topic, ConsumerConfigurat this.consumerEventListener = conf.getConsumerEventListener(); // Always use growable queue since items can exceed the advertised size this.incomingMessages = new GrowableArrayBlockingQueue<>(); - this.unAckedChunkedMessageIdSequenceMap = - ConcurrentOpenHashMap.newBuilder().build(); this.executorProvider = executorProvider; + this.messageListenerExecutor = conf.getMessageListenerExecutor() == null + ? (conf.getSubscriptionType() == SubscriptionType.Key_Shared + ? this::executeKeySharedMessageListener + : this::executeMessageListener) + : conf.getMessageListenerExecutor(); this.externalPinnedExecutor = executorProvider.getExecutor(); this.internalPinnedExecutor = client.getInternalExecutorService(); this.pendingReceives = Queues.newConcurrentLinkedQueue(); @@ -188,6 +200,10 @@ protected ConsumerBase(PulsarClientImpl client, String topic, ConsumerConfigurat initReceiverQueueSize(); } + protected UnAckedMessageTracker getUnAckedMessageTracker() { + return unAckedMessageTracker; + } + protected void triggerBatchReceiveTimeoutTask() { if (!hasBatchReceiveTimeout() && batchReceivePolicy.getTimeoutMs() > 0) { batchReceiveTimeout = client.timer().newTimeout(this::pendingBatchReceiveTask, @@ -218,7 +234,7 @@ protected void expectMoreIncomingMessages() { } } - // if lister is not null, we will track unAcked msg in callMessageListener + // if listener is not null, we will track unAcked msg in callMessageListener protected void trackUnAckedMsgIfNoListener(MessageId messageId, int redeliveryCount) { if (listener == null) { unAckedMessageTracker.add(messageId, redeliveryCount); @@ -702,8 +718,13 @@ public void negativeAcknowledge(Messages messages) { @Override public void unsubscribe() throws PulsarClientException { + unsubscribe(false); + } + + @Override + public void unsubscribe(boolean force) throws PulsarClientException { try { - unsubscribeAsync().get(); + unsubscribeAsync(force).get(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw PulsarClientException.unwrap(e); @@ -713,7 +734,12 @@ public void unsubscribe() throws PulsarClientException { } @Override - public abstract CompletableFuture unsubscribeAsync(); + public CompletableFuture unsubscribeAsync() { + return unsubscribeAsync(false); + } + + @Override + public abstract CompletableFuture unsubscribeAsync(boolean force); @Override public void close() throws PulsarClientException { @@ -824,6 +850,14 @@ public String toString() { + '}'; } + protected Message onArrival(Message message) { + if (interceptors != null) { + return interceptors.onArrival(this, message); + } else { + return message; + } + } + protected Message beforeConsume(Message message) { if (interceptors != null) { return interceptors.beforeConsume(this, message); @@ -1105,14 +1139,8 @@ private void triggerListener() { // Trigger the notification on the message listener in a separate thread to avoid blocking the // internal pinned executor thread while the message processing happens final Message finalMsg = msg; - if (SubscriptionType.Key_Shared == conf.getSubscriptionType()) { - executorProvider.getExecutor(peekMessageKey(msg)).execute(() -> - callMessageListener(finalMsg)); - } else { - getExternalExecutor(msg).execute(() -> { - callMessageListener(finalMsg); - }); - } + MESSAGE_LISTENER_QUEUE_SIZE_UPDATER.incrementAndGet(this); + messageListenerExecutor.execute(msg, () -> callMessageListener(finalMsg)); } else { if (log.isDebugEnabled()) { log.debug("[{}] [{}] Message has been cleared from the queue", topic, subscription); @@ -1125,6 +1153,14 @@ private void triggerListener() { }); } + private void executeMessageListener(Message message, Runnable runnable) { + getExternalExecutor(message).execute(runnable); + } + + private void executeKeySharedMessageListener(Message message, Runnable runnable) { + executorProvider.getExecutor(peekMessageKey(message)).execute(runnable); + } + protected void callMessageListener(Message msg) { try { if (log.isDebugEnabled()) { @@ -1148,17 +1184,21 @@ protected void callMessageListener(Message msg) { } catch (Throwable t) { log.error("[{}][{}] Message listener error in processing message: {}", topic, subscription, msg.getMessageId(), t); + } finally { + MESSAGE_LISTENER_QUEUE_SIZE_UPDATER.decrementAndGet(this); } } static final byte[] NONE_KEY = "NONE_KEY".getBytes(StandardCharsets.UTF_8); - protected byte[] peekMessageKey(Message msg) { + protected byte[] peekMessageKey(Message msg) { byte[] key = NONE_KEY; - if (msg.hasKey()) { - key = msg.getKeyBytes(); - } if (msg.hasOrderingKey()) { key = msg.getOrderingKey(); + } else if (msg.hasKey()) { + key = msg.getKeyBytes(); + } else if (msg.getProducerName() != null) { + String fallbackKey = msg.getProducerName() + "-" + msg.getSequenceId(); + key = fallbackKey.getBytes(StandardCharsets.UTF_8); } return key; } @@ -1219,7 +1259,7 @@ public int getCurrentReceiverQueueSize() { protected abstract void completeOpBatchReceive(OpBatchReceive op); - private ExecutorService getExternalExecutor(Message msg) { + private ExecutorService getExternalExecutor(Message msg) { ConsumerImpl receivedConsumer = (msg instanceof TopicMessageImpl) ? ((TopicMessageImpl) msg).receivedByconsumer : null; ExecutorService executor = receivedConsumer != null && receivedConsumer.externalPinnedExecutor != null @@ -1254,9 +1294,18 @@ protected boolean isValidConsumerEpoch(MessageImpl message) { return true; } + protected boolean isSingleMessageAcked(BitSetRecyclable ackBitSet, int batchIndex) { + return ackBitSet != null && !ackBitSet.get(batchIndex); + } + public boolean hasBatchReceiveTimeout() { return batchReceiveTimeout != null; } + @VisibleForTesting + CompletableFuture> getSubscribeFuture() { + return subscribeFuture; + } + private static final Logger log = LoggerFactory.getLogger(ConsumerBase.class); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java index f644c6a18398f..351025d426a39 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java @@ -32,6 +32,7 @@ import lombok.Getter; import lombok.NonNull; import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.BatchReceivePolicy; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.ConsumerBuilder; @@ -43,6 +44,7 @@ import org.apache.pulsar.client.api.KeySharedPolicy; import org.apache.pulsar.client.api.MessageCrypto; import org.apache.pulsar.client.api.MessageListener; +import org.apache.pulsar.client.api.MessageListenerExecutor; import org.apache.pulsar.client.api.MessagePayloadProcessor; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.PulsarClientException.InvalidConfigurationException; @@ -58,7 +60,6 @@ import org.apache.pulsar.client.impl.conf.TopicConsumerConfigurationData; import org.apache.pulsar.client.util.RetryMessageUtil; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.util.FutureUtil; @Getter(AccessLevel.PUBLIC) @@ -104,6 +105,31 @@ public Consumer subscribe() throws PulsarClientException { } } + private CompletableFuture checkDlqAlreadyExists(String topic) { + CompletableFuture existsFuture = new CompletableFuture<>(); + client.getPartitionedTopicMetadata(topic, false, true).thenAccept(metadata -> { + TopicName topicName = TopicName.get(topic); + if (topicName.isPersistent()) { + // Either partitioned or non-partitioned, it exists. + existsFuture.complete(true); + } else { + // If it is a non-persistent topic, return true only it is a partitioned topic. + existsFuture.complete(metadata != null && metadata.partitions > 0); + } + }).exceptionally(ex -> { + Throwable actEx = FutureUtil.unwrapCompletionException(ex); + if (actEx instanceof PulsarClientException.NotFoundException + || actEx instanceof PulsarClientException.TopicDoesNotExistException + || actEx instanceof PulsarAdminException.NotFoundException) { + existsFuture.complete(false); + } else { + existsFuture.completeExceptionally(ex); + } + return null; + }); + return existsFuture; + } + @Override public CompletableFuture> subscribeAsync() { if (conf.getTopicNames().isEmpty() && conf.getTopicsPattern() == null) { @@ -120,6 +146,10 @@ public CompletableFuture> subscribeAsync() { return FutureUtil.failedFuture( new InvalidConfigurationException("KeySharedPolicy must set with KeyShared subscription")); } + if (conf.getBatchReceivePolicy() != null) { + conf.setReceiverQueueSize( + Math.max(conf.getBatchReceivePolicy().getMaxNumMessages(), conf.getReceiverQueueSize())); + } CompletableFuture applyDLQConfig; if (conf.isRetryEnable() && conf.getTopicNames().size() > 0) { TopicName topicFirst = TopicName.get(conf.getTopicNames().iterator().next()); @@ -131,20 +161,18 @@ public CompletableFuture> subscribeAsync() { DeadLetterPolicy deadLetterPolicy = conf.getDeadLetterPolicy(); if (deadLetterPolicy == null || StringUtils.isBlank(deadLetterPolicy.getRetryLetterTopic()) || StringUtils.isBlank(deadLetterPolicy.getDeadLetterTopic())) { - CompletableFuture retryLetterTopicMetadata = - client.getPartitionedTopicMetadata(oldRetryLetterTopic); - CompletableFuture deadLetterTopicMetadata = - client.getPartitionedTopicMetadata(oldDeadLetterTopic); + CompletableFuture retryLetterTopicMetadata = checkDlqAlreadyExists(oldRetryLetterTopic); + CompletableFuture deadLetterTopicMetadata = checkDlqAlreadyExists(oldDeadLetterTopic); applyDLQConfig = CompletableFuture.allOf(retryLetterTopicMetadata, deadLetterTopicMetadata) .thenAccept(__ -> { String retryLetterTopic = topicFirst + "-" + conf.getSubscriptionName() + RetryMessageUtil.RETRY_GROUP_TOPIC_SUFFIX; String deadLetterTopic = topicFirst + "-" + conf.getSubscriptionName() + RetryMessageUtil.DLQ_GROUP_TOPIC_SUFFIX; - if (retryLetterTopicMetadata.join().partitions > 0) { + if (retryLetterTopicMetadata.join()) { retryLetterTopic = oldRetryLetterTopic; } - if (deadLetterTopicMetadata.join().partitions > 0) { + if (deadLetterTopicMetadata.join()) { deadLetterTopic = oldDeadLetterTopic; } if (deadLetterPolicy == null) { @@ -272,6 +300,13 @@ public ConsumerBuilder messageListener(@NonNull MessageListener messageLis return this; } + @Override + public ConsumerBuilder messageListenerExecutor(MessageListenerExecutor messageListenerExecutor) { + checkArgument(messageListenerExecutor != null, "messageListenerExecutor needs to be not null"); + conf.setMessageListenerExecutor(messageListenerExecutor); + return this; + } + @Override public ConsumerBuilder consumerEventListener(@NonNull ConsumerEventListener consumerEventListener) { conf.setConsumerEventListener(consumerEventListener); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 4a84e765065f2..b7010a1ddc7b4 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -21,6 +21,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; import static org.apache.pulsar.common.protocol.Commands.hasChecksum; +import static org.apache.pulsar.common.protocol.Commands.serializeWithSize; import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ComparisonChain; @@ -32,9 +33,13 @@ import io.netty.util.Recycler.Handle; import io.netty.util.ReferenceCountUtil; import io.netty.util.Timeout; +import io.netty.util.concurrent.FastThreadLocal; +import io.opentelemetry.api.common.Attributes; import java.io.IOException; +import java.net.URI; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.BitSet; import java.util.Collections; import java.util.HashMap; @@ -49,7 +54,6 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -66,6 +70,7 @@ import lombok.AccessLevel; import lombok.Getter; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Triple; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.ConsumerCryptoFailureAction; import org.apache.pulsar.client.api.DeadLetterPolicy; @@ -86,6 +91,10 @@ import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import org.apache.pulsar.client.impl.crypto.MessageCryptoBc; +import org.apache.pulsar.client.impl.metrics.Counter; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; +import org.apache.pulsar.client.impl.metrics.Unit; +import org.apache.pulsar.client.impl.metrics.UpDownCounter; import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; import org.apache.pulsar.client.impl.transaction.TransactionImpl; import org.apache.pulsar.client.util.ExecutorProvider; @@ -93,7 +102,9 @@ import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.api.EncryptionContext; import org.apache.pulsar.common.api.EncryptionContext.EncryptionKey; +import org.apache.pulsar.common.api.proto.BaseCommand; import org.apache.pulsar.common.api.proto.BrokerEntryMetadata; +import org.apache.pulsar.common.api.proto.CommandAck; import org.apache.pulsar.common.api.proto.CommandAck.AckType; import org.apache.pulsar.common.api.proto.CommandAck.ValidationError; import org.apache.pulsar.common.api.proto.CommandMessage; @@ -111,12 +122,14 @@ import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; +import org.apache.pulsar.common.util.Backoff; +import org.apache.pulsar.common.util.BackoffBuilder; import org.apache.pulsar.common.util.CompletableFutureCancellationHandler; import org.apache.pulsar.common.util.ExceptionHandler; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.SafeCollectionUtils; import org.apache.pulsar.common.util.collections.BitSetRecyclable; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.apache.pulsar.common.util.collections.ConcurrentBitSetRecyclable; import org.apache.pulsar.common.util.collections.GrowableArrayBlockingQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -159,7 +172,9 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle private volatile MessageIdAdv startMessageId; private volatile MessageIdAdv seekMessageId; - private final AtomicBoolean duringSeek; + @VisibleForTesting + final AtomicReference seekStatus; + private volatile CompletableFuture seekFuture; private final MessageIdAdv initialStartMessageId; @@ -186,13 +201,12 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle private volatile CompletableFuture> deadLetterProducer; - private volatile Producer retryLetterProducer; + private volatile CompletableFuture> retryLetterProducer; private final ReadWriteLock createProducerLock = new ReentrantReadWriteLock(); protected volatile boolean paused; - protected ConcurrentOpenHashMap chunkedMessagesMap = - ConcurrentOpenHashMap.newBuilder().build(); + protected Map chunkedMessagesMap = new ConcurrentHashMap<>(); private int pendingChunkedMessageCount = 0; protected long expireTimeOfIncompleteChunkedMessageMillis = 0; private final AtomicBoolean expireChunkMessageTaskScheduled = new AtomicBoolean(false); @@ -206,8 +220,21 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle private final boolean createTopicIfDoesNotExist; private final boolean poolMessages; + private final Counter messagesReceivedCounter; + private final Counter bytesReceivedCounter; + private final UpDownCounter messagesPrefetchedGauge; + private final UpDownCounter bytesPrefetchedGauge; + private final Counter consumersOpenedCounter; + private final Counter consumersClosedCounter; + private final Counter consumerAcksCounter; + private final Counter consumerNacksCounter; + + private final Counter consumerDlqMessagesCounter; + private final AtomicReference clientCnxUsedForConsumerRegistration = new AtomicReference<>(); - private final List previousExceptions = new CopyOnWriteArrayList(); + private final AtomicInteger previousExceptionCount = new AtomicInteger(); + private volatile boolean hasSoughtByTimestamp = false; + static ConsumerImpl newConsumerImpl(PulsarClientImpl client, String topic, ConsumerConfigurationData conf, @@ -251,10 +278,12 @@ static ConsumerImpl newConsumerImpl(PulsarClientImpl client, } protected ConsumerImpl(PulsarClientImpl client, String topic, ConsumerConfigurationData conf, - ExecutorProvider executorProvider, int partitionIndex, boolean hasParentConsumer, - boolean parentConsumerHasListener, CompletableFuture> subscribeFuture, MessageId startMessageId, - long startMessageRollbackDurationInSec, Schema schema, ConsumerInterceptors interceptors, - boolean createTopicIfDoesNotExist) { + ExecutorProvider executorProvider, int partitionIndex, boolean hasParentConsumer, + boolean parentConsumerHasListener, CompletableFuture> subscribeFuture, + MessageId startMessageId, + long startMessageRollbackDurationInSec, Schema schema, + ConsumerInterceptors interceptors, + boolean createTopicIfDoesNotExist) { super(client, topic, conf, conf.getReceiverQueueSize(), executorProvider, subscribeFuture, schema, interceptors); this.consumerId = client.newConsumerId(); @@ -296,7 +325,7 @@ protected ConsumerImpl(PulsarClientImpl client, String topic, ConsumerConfigurat stats = ConsumerStatsDisabled.INSTANCE; } - duringSeek = new AtomicBoolean(false); + seekStatus = new AtomicReference<>(SeekStatus.NOT_STARTED); // Create msgCrypto if not created already if (conf.getCryptoKeyReader() != null) { @@ -326,21 +355,21 @@ protected ConsumerImpl(PulsarClientImpl client, String topic, ConsumerConfigurat } this.connectionHandler = new ConnectionHandler(this, - new BackoffBuilder() - .setInitialTime(client.getConfiguration().getInitialBackoffIntervalNanos(), - TimeUnit.NANOSECONDS) - .setMax(client.getConfiguration().getMaxBackoffIntervalNanos(), TimeUnit.NANOSECONDS) - .setMandatoryStop(0, TimeUnit.MILLISECONDS) - .create(), + new BackoffBuilder() + .setInitialTime(client.getConfiguration().getInitialBackoffIntervalNanos(), + TimeUnit.NANOSECONDS) + .setMax(client.getConfiguration().getMaxBackoffIntervalNanos(), TimeUnit.NANOSECONDS) + .setMandatoryStop(0, TimeUnit.MILLISECONDS) + .create(), this); this.topicName = TopicName.get(topic); if (this.topicName.isPersistent()) { this.acknowledgmentsGroupingTracker = - new PersistentAcknowledgmentsGroupingTracker(this, conf, client.eventLoopGroup()); + new PersistentAcknowledgmentsGroupingTracker(this, conf, client.eventLoopGroup()); } else { this.acknowledgmentsGroupingTracker = - NonPersistentAcknowledgmentGroupingTracker.of(); + NonPersistentAcknowledgmentGroupingTracker.of(); } if (conf.getDeadLetterPolicy() != null) { @@ -378,13 +407,37 @@ protected ConsumerImpl(PulsarClientImpl client, String topic, ConsumerConfigurat topicNameWithoutPartition = topicName.getPartitionedTopicName(); + InstrumentProvider ip = client.instrumentProvider(); + Attributes attrs = Attributes.builder().put("pulsar.subscription", subscription).build(); + consumersOpenedCounter = ip.newCounter("pulsar.client.consumer.opened", Unit.Sessions, + "The number of consumer sessions opened", topic, attrs); + consumersClosedCounter = ip.newCounter("pulsar.client.consumer.closed", Unit.Sessions, + "The number of consumer sessions closed", topic, attrs); + messagesReceivedCounter = ip.newCounter("pulsar.client.consumer.message.received.count", Unit.Messages, + "The number of messages explicitly received by the consumer application", topic, attrs); + bytesReceivedCounter = ip.newCounter("pulsar.client.consumer.message.received.size", Unit.Bytes, + "The number of bytes explicitly received by the consumer application", topic, attrs); + messagesPrefetchedGauge = ip.newUpDownCounter("pulsar.client.consumer.receive_queue.count", Unit.Messages, + "The number of messages currently sitting in the consumer receive queue", topic, attrs); + bytesPrefetchedGauge = ip.newUpDownCounter("pulsar.client.consumer.receive_queue.size", Unit.Bytes, + "The total size in bytes of messages currently sitting in the consumer receive queue", topic, attrs); + + consumerAcksCounter = ip.newCounter("pulsar.client.consumer.message.ack", Unit.Messages, + "The number of acknowledged messages", topic, attrs); + consumerNacksCounter = ip.newCounter("pulsar.client.consumer.message.nack", Unit.Messages, + "The number of negatively acknowledged messages", topic, attrs); + consumerDlqMessagesCounter = ip.newCounter("pulsar.client.consumer.message.dlq", Unit.Messages, + "The number of messages sent to DLQ", topic, attrs); grabCnx(); + + consumersOpenedCounter.increment(); } public ConnectionHandler getConnectionHandler() { return connectionHandler; } + @Override public UnAckedMessageTracker getUnAckedMessageTracker() { return unAckedMessageTracker; } @@ -395,7 +448,7 @@ NegativeAcksTracker getNegativeAcksTracker() { } @Override - public CompletableFuture unsubscribeAsync() { + public CompletableFuture unsubscribeAsync(boolean force) { if (getState() == State.Closing || getState() == State.Closed) { return FutureUtil .failedFuture(new PulsarClientException.AlreadyClosedException("Consumer was already closed")); @@ -404,7 +457,7 @@ public CompletableFuture unsubscribeAsync() { if (isConnected()) { setState(State.Closing); long requestId = client.newRequestId(); - ByteBuf unsubscribe = Commands.newUnsubscribe(consumerId, requestId); + ByteBuf unsubscribe = Commands.newUnsubscribe(consumerId, requestId, force); ClientCnx cnx = cnx(); cnx.sendRequestWithId(unsubscribe, requestId).thenRun(() -> { closeConsumerTasks(); @@ -417,16 +470,16 @@ public CompletableFuture unsubscribeAsync() { log.error("[{}][{}] Failed to unsubscribe: {}", topic, subscription, e.getCause().getMessage()); setState(State.Ready); unsubscribeFuture.completeExceptionally( - PulsarClientException.wrap(e.getCause(), - String.format("Failed to unsubscribe the subscription %s of topic %s", - topicName.toString(), subscription))); + PulsarClientException.wrap(e.getCause(), + String.format("Failed to unsubscribe the subscription %s of topic %s", + subscription, topicName.toString()))); return null; }); } else { unsubscribeFuture.completeExceptionally( - new PulsarClientException( - String.format("The client is not connected to the broker when unsubscribing the " - + "subscription %s of the topic %s", subscription, topicName.toString()))); + new PulsarClientException.NotConnectedException( + String.format("The client is not connected to the broker when unsubscribing the " + + "subscription %s of the topic %s", subscription, topicName.toString()))); } return unsubscribeFuture; } @@ -540,6 +593,8 @@ protected CompletableFuture> internalBatchReceiveAsync() { protected CompletableFuture doAcknowledge(MessageId messageId, AckType ackType, Map properties, TransactionImpl txn) { + consumerAcksCounter.increment(); + if (getState() != State.Ready && getState() != State.Connecting) { stats.incrementNumAcksFailed(); PulsarClientException exception = new PulsarClientException("Consumer not ready. State: " + getState()); @@ -561,6 +616,8 @@ protected CompletableFuture doAcknowledge(MessageId messageId, AckType ack @Override protected CompletableFuture doAcknowledge(List messageIdList, AckType ackType, Map properties, TransactionImpl txn) { + consumerAcksCounter.increment(); + if (getState() != State.Ready && getState() != State.Connecting) { stats.incrementNumAcksFailed(); PulsarClientException exception = new PulsarClientException("Consumer not ready. State: " + getState()); @@ -579,6 +636,18 @@ protected CompletableFuture doAcknowledge(List messageIdList, A } } + private static void copyMessageKeysIfNeeded(Message message, TypedMessageBuilder typedMessageBuilderNew) { + if (message.hasKey()) { + if (message.hasBase64EncodedKey()) { + typedMessageBuilderNew.keyBytes(message.getKeyBytes()); + } else { + typedMessageBuilderNew.key(message.getKey()); + } + } + if (message.hasOrderingKey()) { + typedMessageBuilderNew.orderingKey(message.getOrderingKey()); + } + } @SuppressWarnings("unchecked") @Override @@ -586,6 +655,7 @@ protected CompletableFuture doReconsumeLater(Message message, AckType a Map customProperties, long delayTime, TimeUnit unit) { + MessageId messageId = message.getMessageId(); if (messageId == null) { return FutureUtil.failedFuture(new PulsarClientException @@ -602,27 +672,8 @@ protected CompletableFuture doReconsumeLater(Message message, AckType a } return FutureUtil.failedFuture(exception); } - if (delayTime < 0) { - delayTime = 0; - } - if (retryLetterProducer == null) { - createProducerLock.writeLock().lock(); - try { - if (retryLetterProducer == null) { - retryLetterProducer = client.newProducer(Schema.AUTO_PRODUCE_BYTES(schema)) - .topic(this.deadLetterPolicy.getRetryLetterTopic()) - .enableBatching(false) - .blockIfQueueFull(false) - .create(); - } - } catch (Exception e) { - log.error("Create retry letter producer exception with topic: {}", - deadLetterPolicy.getRetryLetterTopic(), e); - return FutureUtil.failedFuture(e); - } finally { - createProducerLock.writeLock().unlock(); - } - } + + initRetryLetterProducerIfNeeded(); CompletableFuture result = new CompletableFuture<>(); if (retryLetterProducer != null) { try { @@ -642,7 +693,7 @@ protected CompletableFuture doReconsumeLater(Message message, AckType a } propertiesMap.put(RetryMessageUtil.SYSTEM_PROPERTY_RECONSUMETIMES, String.valueOf(reconsumeTimes)); propertiesMap.put(RetryMessageUtil.SYSTEM_PROPERTY_DELAY_TIME, - String.valueOf(unit.toMillis(delayTime))); + String.valueOf(unit.toMillis(delayTime < 0 ? 0 : delayTime))); MessageId finalMessageId = messageId; if (reconsumeTimes > this.deadLetterPolicy.getMaxRedeliverCount() @@ -653,7 +704,10 @@ protected CompletableFuture doReconsumeLater(Message message, AckType a dlqProducer.newMessage(Schema.AUTO_PRODUCE_BYTES(retryMessage.getReaderSchema().get())) .value(retryMessage.getData()) .properties(propertiesMap); + copyMessageKeysIfNeeded(message, typedMessageBuilderNew); typedMessageBuilderNew.sendAsync().thenAccept(msgId -> { + consumerDlqMessagesCounter.increment(); + doAcknowledge(finalMessageId, ackType, Collections.emptyMap(), null).thenAccept(v -> { result.complete(null); }).exceptionally(ex -> { @@ -671,23 +725,27 @@ protected CompletableFuture doReconsumeLater(Message message, AckType a }); } else { assert retryMessage != null; - TypedMessageBuilder typedMessageBuilderNew = retryLetterProducer - .newMessage(Schema.AUTO_PRODUCE_BYTES(message.getReaderSchema().get())) - .value(retryMessage.getData()) - .properties(propertiesMap); - if (delayTime > 0) { - typedMessageBuilderNew.deliverAfter(delayTime, unit); - } - if (message.hasKey()) { - typedMessageBuilderNew.key(message.getKey()); - } - typedMessageBuilderNew.sendAsync() - .thenCompose(__ -> doAcknowledge(finalMessageId, ackType, Collections.emptyMap(), null)) - .thenAccept(v -> result.complete(null)) - .exceptionally(ex -> { - result.completeExceptionally(ex); - return null; - }); + retryLetterProducer.thenAcceptAsync(rtlProducer -> { + TypedMessageBuilder typedMessageBuilderNew = rtlProducer + .newMessage(Schema.AUTO_PRODUCE_BYTES(message.getReaderSchema().get())) + .value(retryMessage.getData()) + .properties(propertiesMap); + if (delayTime > 0) { + typedMessageBuilderNew.deliverAfter(delayTime, unit); + } + copyMessageKeysIfNeeded(message, typedMessageBuilderNew); + typedMessageBuilderNew.sendAsync() + .thenCompose(__ -> doAcknowledge(finalMessageId, ackType, Collections.emptyMap(), null)) + .thenAccept(v -> result.complete(null)) + .exceptionally(ex -> { + result.completeExceptionally(ex); + return null; + }); + }, internalPinnedExecutor).exceptionally(ex -> { + result.completeExceptionally(ex); + retryLetterProducer = null; + return null; + }); } } catch (Exception e) { result.completeExceptionally(e); @@ -696,7 +754,7 @@ protected CompletableFuture doReconsumeLater(Message message, AckType a MessageId finalMessageId = messageId; result.exceptionally(ex -> { log.error("Send to retry letter topic exception with topic: {}, messageId: {}", - retryLetterProducer.getTopic(), finalMessageId, ex); + this.deadLetterPolicy.getRetryLetterTopic(), finalMessageId, ex); Set messageIds = Collections.singleton(finalMessageId); unAckedMessageTracker.remove(finalMessageId); redeliverUnacknowledgedMessages(messageIds); @@ -716,6 +774,7 @@ private SortedMap getPropertiesMap(Message message, //Compatible with the old version, will be deleted in the future propertiesMap.putIfAbsent(RetryMessageUtil.SYSTEM_PROPERTY_ORIGIN_MESSAGE_ID, originMessageIdStr); propertiesMap.putIfAbsent(RetryMessageUtil.PROPERTY_ORIGIN_MESSAGE_ID, originMessageIdStr); + propertiesMap.putIfAbsent(RetryMessageUtil.SYSTEM_PROPERTY_REAL_SUBSCRIPTION, subscription); return propertiesMap; } @@ -745,38 +804,42 @@ private MessageImpl getMessageImpl(Message message) { @Override public void negativeAcknowledge(MessageId messageId) { + consumerNacksCounter.increment(); negativeAcksTracker.add(messageId); // Ensure the message is not redelivered for ack-timeout, since we did receive an "ack" - unAckedMessageTracker.remove(messageId); + unAckedMessageTracker.remove(MessageIdAdvUtils.discardBatch(messageId)); } @Override public void negativeAcknowledge(Message message) { + consumerNacksCounter.increment(); negativeAcksTracker.add(message); // Ensure the message is not redelivered for ack-timeout, since we did receive an "ack" - unAckedMessageTracker.remove(message.getMessageId()); + unAckedMessageTracker.remove(MessageIdAdvUtils.discardBatch(message.getMessageId())); } @Override - public void connectionOpened(final ClientCnx cnx) { - previousExceptions.clear(); + public CompletableFuture connectionOpened(final ClientCnx cnx) { + previousExceptionCount.set(0); + getConnectionHandler().setMaxMessageSize(cnx.getMaxMessageSize()); - if (getState() == State.Closing || getState() == State.Closed) { + final State state = getState(); + if (state == State.Closing || state == State.Closed) { setState(State.Closed); closeConsumerTasks(); deregisterFromClientCnx(); client.cleanupConsumer(this); - clearReceiverQueue(); - return; + clearReceiverQueue(false); + return CompletableFuture.completedFuture(null); } log.info("[{}][{}] Subscribing to topic on cnx {}, consumerId {}", topic, subscription, cnx.ctx().channel(), consumerId); long requestId = client.newRequestId(); - if (duringSeek.get()) { + if (seekStatus.get() != SeekStatus.NOT_STARTED) { acknowledgmentsGroupingTracker.flushAndClean(); } @@ -787,7 +850,8 @@ public void connectionOpened(final ClientCnx cnx) { int currentSize; synchronized (this) { currentSize = incomingMessages.size(); - startMessageId = clearReceiverQueue(); + setClientCnx(cnx); + clearReceiverQueue(true); if (possibleSendToDeadLetterTopicMessages != null) { possibleSendToDeadLetterTopicMessages.clear(); } @@ -823,8 +887,8 @@ public void connectionOpened(final ClientCnx cnx) { && startMessageId.equals(initialStartMessageId)) ? startMessageRollbackDurationInSec : 0; // synchronized this, because redeliverUnAckMessage eliminate the epoch inconsistency between them + final CompletableFuture future = new CompletableFuture<>(); synchronized (this) { - setClientCnx(cnx); ByteBuf request = Commands.newSubscribe(topic, subscription, consumerId, requestId, getSubType(), priorityLevel, consumerName, isDurable, startMessageIdData, metadata, readCompacted, conf.isReplicateSubscriptionState(), @@ -844,6 +908,7 @@ public void connectionOpened(final ClientCnx cnx) { deregisterFromClientCnx(); client.cleanupConsumer(this); cnx.channel().close(); + future.complete(null); return; } } @@ -856,12 +921,14 @@ public void connectionOpened(final ClientCnx cnx) { if (!(firstTimeConnect && hasParentConsumer) && getCurrentReceiverQueueSize() != 0) { increaseAvailablePermits(cnx, getCurrentReceiverQueueSize()); } + future.complete(null); }).exceptionally((e) -> { deregisterFromClientCnx(); if (getState() == State.Closing || getState() == State.Closed) { // Consumer was closed while reconnecting, close the connection to make sure the broker // drops the consumer on its side cnx.channel().close(); + future.complete(null); return null; } log.warn("[{}][{}] Failed to subscribe to topic on {}", topic, @@ -872,14 +939,14 @@ public void connectionOpened(final ClientCnx cnx) { // in case it was indeed created, otherwise it might prevent new create consumer operation, // since we are not necessarily closing the connection. long closeRequestId = client.newRequestId(); - ByteBuf cmd = Commands.newCloseConsumer(consumerId, closeRequestId); + ByteBuf cmd = Commands.newCloseConsumer(consumerId, closeRequestId, null, null); cnx.sendRequestWithId(cmd, closeRequestId); } if (e.getCause() instanceof PulsarClientException && PulsarClientException.isRetriableError(e.getCause()) && System.currentTimeMillis() < SUBSCRIBE_DEADLINE_UPDATER.get(ConsumerImpl.this)) { - reconnectLater(e.getCause()); + future.completeExceptionally(e.getCause()); } else if (!subscribeFuture.isDone()) { // unable to create new consumer, fail operation setState(State.Failed); @@ -903,11 +970,16 @@ public void connectionOpened(final ClientCnx cnx) { topic, subscription, cnx.channel().remoteAddress()); } else { // consumer was subscribed and connected but we got some error, keep trying - reconnectLater(e.getCause()); + future.completeExceptionally(e.getCause()); + } + + if (!future.isDone()) { + future.complete(null); } return null; }); } + return future; } protected void consumerIsReconnectedToBroker(ClientCnx cnx, int currentQueueSize) { @@ -921,15 +993,24 @@ protected void consumerIsReconnectedToBroker(ClientCnx cnx, int currentQueueSize * Clear the internal receiver queue and returns the message id of what was the 1st message in the queue that was * not seen by the application. */ - private MessageIdAdv clearReceiverQueue() { + private void clearReceiverQueue(boolean updateStartMessageId) { List> currentMessageQueue = new ArrayList<>(incomingMessages.size()); incomingMessages.drainTo(currentMessageQueue); resetIncomingMessageSize(); - if (duringSeek.compareAndSet(true, false)) { - return seekMessageId; + CompletableFuture seekFuture = this.seekFuture; + MessageIdAdv seekMessageId = this.seekMessageId; + + if (seekStatus.get() != SeekStatus.NOT_STARTED) { + if (updateStartMessageId) { + startMessageId = seekMessageId; + } + if (seekStatus.compareAndSet(SeekStatus.COMPLETED, SeekStatus.NOT_STARTED)) { + internalPinnedExecutor.execute(() -> seekFuture.complete(null)); + } + return; } else if (subscriptionMode == SubscriptionMode.Durable) { - return startMessageId; + return; } if (!currentMessageQueue.isEmpty()) { @@ -946,15 +1027,14 @@ private MessageIdAdv clearReceiverQueue() { } // release messages if they are pooled messages currentMessageQueue.forEach(Message::release); - return previousMessage; - } else if (!lastDequeuedMessageId.equals(MessageId.earliest)) { + if (updateStartMessageId) { + startMessageId = previousMessage; + } + } else if (updateStartMessageId && !lastDequeuedMessageId.equals(MessageId.earliest)) { // If the queue was empty we need to restart from the message just after the last one that has been dequeued // in the past - return new BatchMessageIdImpl((MessageIdImpl) lastDequeuedMessageId); - } else { - // No message was received or dequeued by this consumer. Next message would still be the startMessageId - return startMessageId; - } + startMessageId = new BatchMessageIdImpl((MessageIdImpl) lastDequeuedMessageId); + } // else: No message was received or dequeued by this consumer. Next message would still be the startMessageId } /** @@ -986,12 +1066,12 @@ public void connectionFailed(PulsarClientException exception) { boolean nonRetriableError = !PulsarClientException.isRetriableError(exception); boolean timeout = System.currentTimeMillis() > lookupDeadline; if (nonRetriableError || timeout) { - exception.setPreviousExceptions(previousExceptions); + exception.setPreviousExceptionCount(previousExceptionCount); if (subscribeFuture.completeExceptionally(exception)) { setState(State.Failed); if (nonRetriableError) { log.info("[{}] Consumer creation failed for consumer {} with unretriableError {}", - topic, consumerId, exception); + topic, consumerId, exception.getMessage()); } else { log.info("[{}] Consumer creation failed for consumer {} after timeout", topic, consumerId); } @@ -1000,12 +1080,12 @@ public void connectionFailed(PulsarClientException exception) { client.cleanupConsumer(this); } } else { - previousExceptions.add(exception); + previousExceptionCount.incrementAndGet(); } } @Override - public CompletableFuture closeAsync() { + public synchronized CompletableFuture closeAsync() { CompletableFuture closeFuture = new CompletableFuture<>(); if (getState() == State.Closing || getState() == State.Closed) { @@ -1014,6 +1094,8 @@ public CompletableFuture closeAsync() { return closeFuture; } + consumersClosedCounter.increment(); + if (!isConnected()) { log.info("[{}] [{}] Closed Consumer (not connected)", topic, subscription); setState(State.Closed); @@ -1036,7 +1118,7 @@ public CompletableFuture closeAsync() { if (null == cnx) { cleanupAtClose(closeFuture, null); } else { - ByteBuf cmd = Commands.newCloseConsumer(consumerId, requestId); + ByteBuf cmd = Commands.newCloseConsumer(consumerId, requestId, null, null); cnx.sendRequestWithId(cmd, requestId).handle((v, exception) -> { final ChannelHandlerContext ctx = cnx.ctx(); boolean ignoreException = ctx == null || !ctx.channel().isActive(); @@ -1051,7 +1133,7 @@ public CompletableFuture closeAsync() { ArrayList> closeFutures = new ArrayList<>(4); closeFutures.add(closeFuture); if (retryLetterProducer != null) { - closeFutures.add(retryLetterProducer.closeAsync().whenComplete((ignore, ex) -> { + closeFutures.add(retryLetterProducer.thenCompose(p -> p.closeAsync()).whenComplete((ignore, ex) -> { if (ex != null) { log.warn("Exception ignored in closing retryLetterProducer of consumer", ex); } @@ -1170,7 +1252,7 @@ protected MessageImpl newSingleMessage(final int index, return null; } - if (ackBitSet != null && !ackBitSet.get(index)) { + if (isSingleMessageAcked(ackBitSet, index)) { return null; } @@ -1206,6 +1288,9 @@ protected MessageImpl newMessage(final MessageIdImpl messageId, } private void executeNotifyCallback(final MessageImpl message) { + messagesPrefetchedGauge.increment(); + bytesPrefetchedGauge.add(message.size()); + // Enqueue the message so that it can be retrieved when application calls receive() // if the conf.getReceiverQueueSize() is 0 then discard message if no one is waiting for it. // if asyncReceive is waiting then notify callback without adding to incomingMessages queue @@ -1214,9 +1299,10 @@ private void executeNotifyCallback(final MessageImpl message) { increaseAvailablePermits(cnx()); return; } + Message interceptMsg = onArrival(message); if (hasNextPendingReceive()) { - notifyPendingReceivedCallback(message, null); - } else if (enqueueMessageAndCheckBatchReceive(message) && hasPendingBatchReceive()) { + notifyPendingReceivedCallback(interceptMsg, null); + } else if (enqueueMessageAndCheckBatchReceive(interceptMsg) && hasPendingBatchReceive()) { notifyPendingBatchReceivedCallBack(); } }); @@ -1408,7 +1494,9 @@ void messageReceived(CommandMessage cmdMessage, ByteBuf headersAndPayload, Clien private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata msgMetadata, MessageIdImpl msgId, MessageIdData messageId, ClientCnx cnx) { - + if (msgMetadata.getChunkId() != (msgMetadata.getNumChunksFromMsg() - 1)) { + increaseAvailablePermits(cnx); + } // Lazy task scheduling to expire incomplete chunk message if (expireTimeOfIncompleteChunkedMessageMillis > 0 && expireChunkMessageTaskScheduled.compareAndSet(false, true)) { @@ -1422,7 +1510,48 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m ChunkedMessageCtx chunkedMsgCtx = chunkedMessagesMap.get(msgMetadata.getUuid()); - if (msgMetadata.getChunkId() == 0 && chunkedMsgCtx == null) { + if (msgMetadata.getChunkId() == 0) { + if (chunkedMsgCtx != null) { + // Handle ack hole case when receive duplicated chunks. + // There are two situation that receives chunks with the same sequence ID and chunk ID. + // Situation 1 - Message redeliver: + // For example: + // Chunk-1 sequence ID: 0, chunk ID: 0, msgID: 1:1 + // Chunk-2 sequence ID: 0, chunk ID: 1, msgID: 1:2 + // Chunk-3 sequence ID: 0, chunk ID: 0, msgID: 1:1 + // Chunk-4 sequence ID: 0, chunk ID: 1, msgID: 1:2 + // Chunk-5 sequence ID: 0, chunk ID: 2, msgID: 1:3 + // In this case, chunk-3 and chunk-4 have the same msgID with chunk-1 and chunk-2. + // This may be caused by message redeliver, we can't ack any chunk in this case here. + // Situation 2 - Corrupted chunk message + // For example: + // Chunk-1 sequence ID: 0, chunk ID: 0, msgID: 1:1 + // Chunk-2 sequence ID: 0, chunk ID: 1, msgID: 1:2 + // Chunk-3 sequence ID: 0, chunk ID: 0, msgID: 1:3 + // Chunk-4 sequence ID: 0, chunk ID: 1, msgID: 1:4 + // Chunk-5 sequence ID: 0, chunk ID: 2, msgID: 1:5 + // In this case, all the chunks with different msgIDs and are persistent in the topic. + // But Chunk-1 and Chunk-2 belong to a corrupted chunk message that must be skipped since + // they will not be delivered to end users. So we should ack them here to avoid ack hole. + boolean isCorruptedChunkMessageDetected = Arrays.stream(chunkedMsgCtx.chunkedMessageIds) + .noneMatch(messageId1 -> messageId1 != null && messageId1.ledgerId == messageId.getLedgerId() + && messageId1.entryId == messageId.getEntryId()); + if (isCorruptedChunkMessageDetected) { + Arrays.stream(chunkedMsgCtx.chunkedMessageIds).forEach(messageId1 -> { + if (messageId1 != null) { + doAcknowledge(messageId1, AckType.Individual, Collections.emptyMap(), null); + } + }); + } + // The first chunk of a new chunked-message received before receiving other chunks of previous + // chunked-message + // so, remove previous chunked-message from map and release buffer + if (chunkedMsgCtx.chunkedMsgBuffer != null) { + ReferenceCountUtil.safeRelease(chunkedMsgCtx.chunkedMsgBuffer); + } + chunkedMsgCtx.recycle(); + chunkedMessagesMap.remove(msgMetadata.getUuid()); + } pendingChunkedMessageCount++; if (maxPendingChunkedMessage > 0 && pendingChunkedMessageCount > maxPendingChunkedMessage) { removeOldestPendingChunkedMessage(); @@ -1438,9 +1567,36 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m // discard message if chunk is out-of-order if (chunkedMsgCtx == null || chunkedMsgCtx.chunkedMsgBuffer == null || msgMetadata.getChunkId() != (chunkedMsgCtx.lastChunkedMessageId + 1)) { + // Filter and ack duplicated chunks instead of discard ctx. + // For example: + // Chunk-1 sequence ID: 0, chunk ID: 0, msgID: 1:1 + // Chunk-2 sequence ID: 0, chunk ID: 1, msgID: 1:2 + // Chunk-3 sequence ID: 0, chunk ID: 2, msgID: 1:3 + // Chunk-4 sequence ID: 0, chunk ID: 1, msgID: 1:4 + // Chunk-5 sequence ID: 0, chunk ID: 2, msgID: 1:5 + // Chunk-6 sequence ID: 0, chunk ID: 3, msgID: 1:6 + // We should filter and ack chunk-4 and chunk-5. + if (chunkedMsgCtx != null && msgMetadata.getChunkId() <= chunkedMsgCtx.lastChunkedMessageId) { + log.warn("[{}] Receive a duplicated chunk message with messageId [{}], last-chunk-Id [{}], " + + "chunkId [{}], sequenceId [{}]", + msgMetadata.getProducerName(), msgId, chunkedMsgCtx.lastChunkedMessageId, + msgMetadata.getChunkId(), msgMetadata.getSequenceId()); + compressedPayload.release(); + // Just like the above logic of receiving the first chunk again. We only ack this chunk in the message + // duplication case. + boolean isDuplicatedChunk = Arrays.stream(chunkedMsgCtx.chunkedMessageIds) + .noneMatch(messageId1 -> messageId1 != null && messageId1.ledgerId == messageId.getLedgerId() + && messageId1.entryId == messageId.getEntryId()); + if (isDuplicatedChunk) { + doAcknowledge(msgId, AckType.Individual, Collections.emptyMap(), null); + } + return null; + } // means we lost the first chunk: should never happen - log.info("Received unexpected chunk messageId {}, last-chunk-id{}, chunkId = {}", msgId, - (chunkedMsgCtx != null ? chunkedMsgCtx.lastChunkedMessageId : null), msgMetadata.getChunkId()); + log.info("[{}] [{}] Received unexpected chunk messageId {}, last-chunk-id = {}, chunkId = {}, uuid = {}", + topic, subscription, msgId, + (chunkedMsgCtx != null ? chunkedMsgCtx.lastChunkedMessageId : null), msgMetadata.getChunkId(), + msgMetadata.getUuid()); if (chunkedMsgCtx != null) { if (chunkedMsgCtx.chunkedMsgBuffer != null) { ReferenceCountUtil.safeRelease(chunkedMsgCtx.chunkedMsgBuffer); @@ -1449,7 +1605,6 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m } chunkedMessagesMap.remove(msgMetadata.getUuid()); compressedPayload.release(); - increaseAvailablePermits(cnx); if (expireTimeOfIncompleteChunkedMessageMillis > 0 && System.currentTimeMillis() > (msgMetadata.getPublishTime() + expireTimeOfIncompleteChunkedMessageMillis)) { @@ -1468,7 +1623,6 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m // if final chunk is not received yet then release payload and return if (msgMetadata.getChunkId() != (msgMetadata.getNumChunksFromMsg() - 1)) { compressedPayload.release(); - increaseAvailablePermits(cnx); return null; } @@ -1553,7 +1707,14 @@ void receiveIndividualMessagesFromBatch(BrokerEntryMetadata brokerEntryMetadata, singleMessageMetadata, uncompressedPayload, batchMessage, schema, true, ackBitSet, ackSetInMessageId, redeliveryCount, consumerEpoch); if (message == null) { - skippedMessages++; + // If it is not in ackBitSet, it means Broker does not want to deliver it to the client, and + // did not decrease the permits in the broker-side. + // So do not acquire more permits for this message. + // Why not skip this single message in the first line of for-loop block? We need call + // "newSingleMessage" to move "payload.readerIndex" to a correct value to get the correct data. + if (!isSingleMessageAcked(ackBitSet, i)) { + skippedMessages++; + } continue; } if (possibleToDeadLetter != null) { @@ -1623,6 +1784,12 @@ protected synchronized void messageProcessed(Message msg) { ClientCnx msgCnx = ((MessageImpl) msg).getCnx(); lastDequeuedMessageId = msg.getMessageId(); + messagesPrefetchedGauge.decrement(); + messagesReceivedCounter.increment(); + + bytesPrefetchedGauge.subtract(msg.size()); + bytesReceivedCounter.add(msg.size()); + if (msgCnx != currentCnx) { // The processed message did belong to the old queue that was cleared after reconnection. } else { @@ -1675,6 +1842,9 @@ protected void increaseAvailablePermits(ClientCnx currentCnx, int delta) { int available = AVAILABLE_PERMITS_UPDATER.addAndGet(this, delta); while (available >= getCurrentReceiverQueueSize() / 2 && !paused) { if (AVAILABLE_PERMITS_UPDATER.compareAndSet(this, available, 0)) { + if (log.isDebugEnabled()) { + log.debug("[{}] Sending permit-cmd to broker with available permits = {}", topic, available); + } sendFlowPermitsToBroker(currentCnx, available); break; } else { @@ -1723,30 +1893,10 @@ private ByteBuf decryptPayloadIfNeeded(MessageIdData messageId, int redeliveryCo if (msgMetadata.getEncryptionKeysCount() == 0) { return payload.retain(); } - + int batchSize = msgMetadata.getNumMessagesInBatch(); // If KeyReader is not configured throw exception based on config param if (conf.getCryptoKeyReader() == null) { - switch (conf.getCryptoFailureAction()) { - case CONSUME: - log.warn("[{}][{}][{}] CryptoKeyReader interface is not implemented. Consuming encrypted message.", - topic, subscription, consumerName); - return payload.retain(); - case DISCARD: - log.warn( - "[{}][{}][{}] Skipping decryption since CryptoKeyReader interface is not implemented and" - + " config is set to discard", - topic, subscription, consumerName); - discardMessage(messageId, currentCnx, ValidationError.DecryptionError); - return null; - case FAIL: - MessageId m = new MessageIdImpl(messageId.getLedgerId(), messageId.getEntryId(), partitionIndex); - log.error( - "[{}][{}][{}][{}] Message delivery failed since CryptoKeyReader interface is not" - + " implemented to consume encrypted message", - topic, subscription, consumerName, m); - unAckedMessageTracker.add(m, redeliveryCount); - return null; - } + return handleCryptoFailure(payload, messageId, currentCnx, redeliveryCount, batchSize, true); } @@ -1760,27 +1910,58 @@ private ByteBuf decryptPayloadIfNeeded(MessageIdData messageId, int redeliveryCo decryptedData.release(); + return handleCryptoFailure(payload, messageId, currentCnx, redeliveryCount, batchSize, false); + } + + private ByteBuf handleCryptoFailure(ByteBuf payload, MessageIdData messageId, ClientCnx currentCnx, + int redeliveryCount, int batchSize, boolean cryptoReaderNotExist) { + switch (conf.getCryptoFailureAction()) { - case CONSUME: + case CONSUME: + if (cryptoReaderNotExist) { + log.warn("[{}][{}][{}] CryptoKeyReader interface is not implemented. Consuming encrypted message.", + topic, subscription, consumerName); + } else { // Note, batch message will fail to consume even if config is set to consume log.warn("[{}][{}][{}][{}] Decryption failed. Consuming encrypted message since config is set to" - + " consume.", - topic, subscription, consumerName, messageId); - return payload.retain(); - case DISCARD: - log.warn("[{}][{}][{}][{}] Discarding message since decryption failed and config is set to discard", - topic, subscription, consumerName, messageId); - discardMessage(messageId, currentCnx, ValidationError.DecryptionError); - return null; - case FAIL: - MessageId m = new MessageIdImpl(messageId.getLedgerId(), messageId.getEntryId(), partitionIndex); + + " consume.", topic, subscription, consumerName, messageId); + } + return payload.retain(); + case DISCARD: + if (cryptoReaderNotExist) { + log.warn( + "[{}][{}][{}] Skipping decryption since CryptoKeyReader interface is not implemented and" + + " config is set to discard message with batch size {}", + topic, subscription, consumerName, batchSize); + } else { + log.warn( + "[{}][{}][{}][{}-{}-{}] Discarding message since decryption failed " + + "and config is set to discard", + topic, subscription, consumerName, messageId.getLedgerId(), messageId.getEntryId(), + messageId.getBatchIndex()); + } + discardMessage(messageId, currentCnx, ValidationError.DecryptionError, batchSize); + return null; + case FAIL: + if (cryptoReaderNotExist) { log.error( - "[{}][{}][{}][{}] Message delivery failed since unable to decrypt incoming message", - topic, subscription, consumerName, m); - unAckedMessageTracker.add(m, redeliveryCount); - return null; + "[{}][{}][{}][{}-{}-{}] Message delivery failed since CryptoKeyReader interface is not" + + " implemented to consume encrypted message", + topic, subscription, consumerName, messageId.getLedgerId(), messageId.getEntryId(), + partitionIndex); + } else { + log.error("[{}][{}][{}][{}-{}-{}] Message delivery failed since unable to decrypt incoming message", + topic, subscription, consumerName, messageId.getLedgerId(), messageId.getEntryId(), + partitionIndex); + } + MessageId m = new MessageIdImpl(messageId.getLedgerId(), messageId.getEntryId(), partitionIndex); + unAckedMessageTracker.add(m, redeliveryCount); + return null; + default: + log.warn("[{}][{}][{}] Invalid crypto failure state found, continue message consumption.", topic, + subscription, consumerName); + return payload.retain(); } - return null; } private ByteBuf uncompressPayloadIfNeeded(MessageIdData messageId, MessageMetadata msgMetadata, ByteBuf payload, @@ -1789,7 +1970,7 @@ private ByteBuf uncompressPayloadIfNeeded(MessageIdData messageId, MessageMetada CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType); int uncompressedSize = msgMetadata.getUncompressedSize(); int payloadSize = payload.readableBytes(); - if (checkMaxMessageSize && payloadSize > ClientCnx.getMaxMessageSize()) { + if (checkMaxMessageSize && payloadSize > getConnectionHandler().getMaxMessageSize()) { // payload size is itself corrupted since it cannot be bigger than the MaxMessageSize log.error("[{}][{}] Got corrupted payload message size {} at {}", topic, subscription, payloadSize, messageId); @@ -1840,14 +2021,15 @@ private void discardCorruptedMessage(MessageIdData messageId, ClientCnx currentC ValidationError validationError) { log.error("[{}][{}] Discarding corrupted message at {}:{}", topic, subscription, messageId.getLedgerId(), messageId.getEntryId()); - discardMessage(messageId, currentCnx, validationError); + discardMessage(messageId, currentCnx, validationError, 1); } - private void discardMessage(MessageIdData messageId, ClientCnx currentCnx, ValidationError validationError) { + private void discardMessage(MessageIdData messageId, ClientCnx currentCnx, ValidationError validationError, + int batchMessages) { ByteBuf cmd = Commands.newAck(consumerId, messageId.getLedgerId(), messageId.getEntryId(), null, AckType.Individual, validationError, Collections.emptyMap(), -1); currentCnx.ctx().writeAndFlush(cmd, currentCnx.ctx().voidPromise()); - increaseAvailablePermits(currentCnx); + increaseAvailablePermits(currentCnx, batchMessages); stats.incrementNumReceiveFailed(); } @@ -2031,9 +2213,7 @@ private CompletableFuture processPossibleToDLQ(MessageIdAdv messageId) producerDLQ.newMessage(Schema.AUTO_PRODUCE_BYTES(message.getReaderSchema().get())) .value(message.getData()) .properties(getPropertiesMap(message, originMessageIdStr, originTopicNameStr)); - if (message.hasKey()) { - typedMessageBuilderNew.key(message.getKey()); - } + copyMessageKeysIfNeeded(message, typedMessageBuilderNew); typedMessageBuilderNew.sendAsync() .thenAccept(messageIdInDLQ -> { possibleSendToDeadLetterTopicMessages.remove(messageId); @@ -2082,8 +2262,37 @@ private void initDeadLetterProducerIfNeeded() { ((ProducerBuilderImpl) client.newProducer(Schema.AUTO_PRODUCE_BYTES(schema))) .initialSubscriptionName(this.deadLetterPolicy.getInitialSubscriptionName()) .topic(this.deadLetterPolicy.getDeadLetterTopic()) + .producerName(String.format("%s-%s-%s-DLQ", this.topicName, this.subscription, + this.consumerName)) .blockIfQueueFull(false) + .enableBatching(false) + .enableChunking(true) .createAsync(); + deadLetterProducer.thenAccept(dlqProducer -> { + stats.setDeadLetterProducerStats(dlqProducer.getStats()); + }); + } + } finally { + createProducerLock.writeLock().unlock(); + } + } + } + + private void initRetryLetterProducerIfNeeded() { + if (retryLetterProducer == null) { + createProducerLock.writeLock().lock(); + try { + if (retryLetterProducer == null) { + retryLetterProducer = client + .newProducer(Schema.AUTO_PRODUCE_BYTES(schema)) + .topic(this.deadLetterPolicy.getRetryLetterTopic()) + .enableBatching(false) + .enableChunking(true) + .blockIfQueueFull(false) + .createAsync(); + retryLetterProducer.thenAccept(rtlProducer -> { + stats.setRetryLetterProducerStats(rtlProducer.getStats()); + }); } } finally { createProducerLock.writeLock().unlock(); @@ -2137,100 +2346,128 @@ public CompletableFuture seekAsync(Function function) { new PulsarClientException("Only support seek by messageId or timestamp")); } - private Optional> seekAsyncCheckState(String seekBy) { - if (getState() == State.Closing || getState() == State.Closed) { - return Optional.of(FutureUtil - .failedFuture(new PulsarClientException.AlreadyClosedException( - String.format("The consumer %s was already closed when seeking the subscription %s of the" - + " topic %s to %s", consumerName, subscription, topicName.toString(), seekBy)))); - } - - if (!isConnected()) { - return Optional.of(FutureUtil.failedFuture(new PulsarClientException( - String.format("The client is not connected to the broker when seeking the subscription %s of the " - + "topic %s to %s", subscription, topicName.toString(), seekBy)))); - } - - return Optional.empty(); - } - - private CompletableFuture seekAsyncInternal(long requestId, ByteBuf seek, MessageId seekId, String seekBy) { - final CompletableFuture seekFuture = new CompletableFuture<>(); - ClientCnx cnx = cnx(); + private CompletableFuture seekAsyncInternal(long requestId, ByteBuf seek, MessageId seekId, + Long seekTimestamp, String seekBy) { + AtomicLong opTimeoutMs = new AtomicLong(client.getConfiguration().getOperationTimeoutMs()); + Backoff backoff = new BackoffBuilder() + .setInitialTime(100, TimeUnit.MILLISECONDS) + .setMax(opTimeoutMs.get() * 2, TimeUnit.MILLISECONDS) + .setMandatoryStop(0, TimeUnit.MILLISECONDS) + .create(); - if (!duringSeek.compareAndSet(false, true)) { + if (!seekStatus.compareAndSet(SeekStatus.NOT_STARTED, SeekStatus.IN_PROGRESS)) { final String message = String.format( "[%s][%s] attempting to seek operation that is already in progress (seek by %s)", topic, subscription, seekBy); log.warn("[{}][{}] Attempting to seek operation that is already in progress, cancelling {}", topic, subscription, seekBy); - seekFuture.completeExceptionally(new IllegalStateException(message)); - return seekFuture; + return FutureUtil.failedFuture(new IllegalStateException(message)); } + seekFuture = new CompletableFuture<>(); + seekAsyncInternal(requestId, seek, seekId, seekTimestamp, seekBy, backoff, opTimeoutMs); + return seekFuture; + } - MessageIdAdv originSeekMessageId = seekMessageId; - seekMessageId = (MessageIdAdv) seekId; - log.info("[{}][{}] Seeking subscription to {}", topic, subscription, seekBy); + private void seekAsyncInternal(long requestId, ByteBuf seek, MessageId seekId, Long seekTimestamp, String seekBy, + final Backoff backoff, final AtomicLong remainingTime) { + ClientCnx cnx = cnx(); + if (isConnected() && cnx != null) { + MessageIdAdv originSeekMessageId = seekMessageId; + seekMessageId = (MessageIdAdv) seekId; + log.info("[{}][{}] Seeking subscription to {}", topic, subscription, seekBy); - cnx.sendRequestWithId(seek, requestId).thenRun(() -> { - log.info("[{}][{}] Successfully reset subscription to {}", topic, subscription, seekBy); - acknowledgmentsGroupingTracker.flushAndClean(); + final boolean originalHasSoughtByTimestamp = hasSoughtByTimestamp; + hasSoughtByTimestamp = (seekTimestamp != null); + cnx.sendRequestWithId(seek, requestId).thenRun(() -> { + log.info("[{}][{}] Successfully reset subscription to {}", topic, subscription, seekBy); + acknowledgmentsGroupingTracker.flushAndClean(); - lastDequeuedMessageId = MessageId.earliest; + lastDequeuedMessageId = MessageId.earliest; + + clearIncomingMessages(); + CompletableFuture future = null; + synchronized (this) { + if (!hasParentConsumer && cnx() == null) { + // It's during reconnection, complete the seek future after connection is established + seekStatus.set(SeekStatus.COMPLETED); + } else { + future = seekFuture; + startMessageId = seekMessageId; + seekStatus.set(SeekStatus.NOT_STARTED); + } + } + if (future != null) { + future.complete(null); + } + }).exceptionally(e -> { + seekMessageId = originSeekMessageId; + hasSoughtByTimestamp = originalHasSoughtByTimestamp; + log.error("[{}][{}] Failed to reset subscription: {}", topic, subscription, e.getCause().getMessage()); + + failSeek( + PulsarClientException.wrap(e.getCause(), + String.format("Failed to seek the subscription %s of the topic %s to %s", + subscription, topicName.toString(), seekBy))); + return null; + }); + } else { + long nextDelay = Math.min(backoff.next(), remainingTime.get()); + if (nextDelay <= 0) { + failSeek( + new PulsarClientException.TimeoutException( + String.format("The subscription %s of the topic %s could not seek " + + "withing configured timeout", subscription, topicName.toString()))); + return; + } - clearIncomingMessages(); - seekFuture.complete(null); - }).exceptionally(e -> { - // re-set duringSeek and seekMessageId if seek failed - seekMessageId = originSeekMessageId; - duringSeek.set(false); - log.error("[{}][{}] Failed to reset subscription: {}", topic, subscription, e.getCause().getMessage()); + ((ScheduledExecutorService) client.getScheduledExecutorProvider().getExecutor()).schedule(() -> { + log.warn("[{}] [{}] Could not get connection while seek -- Will try again in {} ms", + topic, getHandlerName(), nextDelay); + remainingTime.addAndGet(-nextDelay); + seekAsyncInternal(requestId, seek, seekId, seekTimestamp, seekBy, backoff, remainingTime); + }, nextDelay, TimeUnit.MILLISECONDS); + } + } - seekFuture.completeExceptionally( - PulsarClientException.wrap(e.getCause(), - String.format("Failed to seek the subscription %s of the topic %s to %s", - subscription, topicName.toString(), seekBy))); - return null; - }); - return seekFuture; + private void failSeek(Throwable throwable) { + CompletableFuture seekFuture = this.seekFuture; + if (seekStatus.compareAndSet(SeekStatus.IN_PROGRESS, SeekStatus.NOT_STARTED)) { + seekFuture.completeExceptionally(throwable); + } } @Override public CompletableFuture seekAsync(long timestamp) { String seekBy = String.format("the timestamp %d", timestamp); - return seekAsyncCheckState(seekBy).orElseGet(() -> { - long requestId = client.newRequestId(); - return seekAsyncInternal(requestId, Commands.newSeek(consumerId, requestId, timestamp), - MessageId.earliest, seekBy); - }); + long requestId = client.newRequestId(); + return seekAsyncInternal(requestId, Commands.newSeek(consumerId, requestId, timestamp), + MessageId.earliest, timestamp, seekBy); } @Override public CompletableFuture seekAsync(MessageId messageId) { String seekBy = String.format("the message %s", messageId.toString()); - return seekAsyncCheckState(seekBy).orElseGet(() -> { - long requestId = client.newRequestId(); - final MessageIdAdv msgId = (MessageIdAdv) messageId; - final MessageIdAdv firstChunkMsgId = msgId.getFirstChunkMessageId(); - final ByteBuf seek; - if (msgId.getFirstChunkMessageId() != null) { - seek = Commands.newSeek(consumerId, requestId, firstChunkMsgId.getLedgerId(), - firstChunkMsgId.getEntryId(), new long[0]); + long requestId = client.newRequestId(); + final MessageIdAdv msgId = (MessageIdAdv) messageId; + final MessageIdAdv firstChunkMsgId = msgId.getFirstChunkMessageId(); + final ByteBuf seek; + if (msgId.getFirstChunkMessageId() != null) { + seek = Commands.newSeek(consumerId, requestId, firstChunkMsgId.getLedgerId(), + firstChunkMsgId.getEntryId(), new long[0]); + } else { + final long[] ackSetArr; + if (MessageIdAdvUtils.isBatch(msgId)) { + final BitSetRecyclable ackSet = BitSetRecyclable.create(); + ackSet.set(0, msgId.getBatchSize()); + ackSet.clear(0, Math.max(msgId.getBatchIndex(), 0)); + ackSetArr = ackSet.toLongArray(); + ackSet.recycle(); } else { - final long[] ackSetArr; - if (MessageIdAdvUtils.isBatch(msgId)) { - final BitSetRecyclable ackSet = BitSetRecyclable.create(); - ackSet.set(0, msgId.getBatchSize()); - ackSet.clear(0, Math.max(msgId.getBatchIndex(), 0)); - ackSetArr = ackSet.toLongArray(); - ackSet.recycle(); - } else { - ackSetArr = new long[0]; - } - seek = Commands.newSeek(consumerId, requestId, msgId.getLedgerId(), msgId.getEntryId(), ackSetArr); + ackSetArr = new long[0]; } - return seekAsyncInternal(requestId, seek, messageId, seekBy); - }); + seek = Commands.newSeek(consumerId, requestId, msgId.getLedgerId(), msgId.getEntryId(), ackSetArr); + } + return seekAsyncInternal(requestId, seek, messageId, null, seekBy); } public boolean hasMessageAvailable() throws PulsarClientException { @@ -2244,15 +2481,21 @@ public boolean hasMessageAvailable() throws PulsarClientException { public CompletableFuture hasMessageAvailableAsync() { final CompletableFuture booleanFuture = new CompletableFuture<>(); + if (incomingMessages != null && !incomingMessages.isEmpty()) { + return CompletableFuture.completedFuture(true); + } + // we haven't read yet. use startMessageId for comparison if (lastDequeuedMessageId == MessageId.earliest) { + // If the last seek is called with timestamp, startMessageId cannot represent the position to start, so we + // have to get the mark-delete position from the GetLastMessageId response to compare as well. // if we are starting from latest, we should seek to the actual last message first. // allow the last one to be read when read head inclusively. - if (MessageId.latest.equals(startMessageId)) { - + final boolean hasSoughtByTimestamp = this.hasSoughtByTimestamp; + if (MessageId.latest.equals(startMessageId) || hasSoughtByTimestamp) { CompletableFuture future = internalGetLastMessageIdAsync(); // if the consumer is configured to read inclusive then we need to seek to the last message - if (resetIncludeHead) { + if (resetIncludeHead && !hasSoughtByTimestamp) { future = future.thenCompose((lastMessageIdResponse) -> seekAsync(lastMessageIdResponse.lastMessageId) .thenApply((ignore) -> lastMessageIdResponse)); @@ -2443,9 +2686,9 @@ private void internalGetLastMessageIdAsync(final Backoff backoff, return; } + log.warn("[{}] [{}] Could not get connection while getLastMessageId -- Will try again in {} ms", + topic, getHandlerName(), nextDelay); ((ScheduledExecutorService) client.getScheduledExecutorProvider().getExecutor()).schedule(() -> { - log.warn("[{}] [{}] Could not get connection while getLastMessageId -- Will try again in {} ms", - topic, getHandlerName(), nextDelay); remainingTime.addAndGet(-nextDelay); internalGetLastMessageIdAsync(backoff, remainingTime, future); }, nextDelay, TimeUnit.MILLISECONDS); @@ -2562,8 +2805,8 @@ void resetBackoff() { this.connectionHandler.resetBackoff(); } - void connectionClosed(ClientCnx cnx) { - this.connectionHandler.connectionClosed(cnx); + void connectionClosed(ClientCnx cnx, Optional initialConnectionDelayMs, Optional hostUrl) { + this.connectionHandler.connectionClosed(cnx, initialConnectionDelayMs, hostUrl); } public ClientCnx getClientCnx() { @@ -2590,10 +2833,6 @@ void deregisterFromClientCnx() { setClientCnx(null); } - void reconnectLater(Throwable exception) { - this.connectionHandler.reconnectLater(exception); - } - void grabCnx() { this.connectionHandler.grabCnx(); } @@ -2703,7 +2942,7 @@ private CompletableFuture doTransactionAcknowledgeForResponse(MessageId me final MessageIdAdv messageIdAdv = (MessageIdAdv) messageId; final long ledgerId = messageIdAdv.getLedgerId(); final long entryId = messageIdAdv.getEntryId(); - final ByteBuf cmd; + final List cmdList; if (MessageIdAdvUtils.isBatch(messageIdAdv)) { BitSetRecyclable bitSetRecyclable = BitSetRecyclable.create(); bitSetRecyclable.set(0, messageIdAdv.getBatchSize()); @@ -2713,12 +2952,37 @@ private CompletableFuture doTransactionAcknowledgeForResponse(MessageId me } else { bitSetRecyclable.clear(messageIdAdv.getBatchIndex()); } - cmd = Commands.newAck(consumerId, ledgerId, entryId, bitSetRecyclable, ackType, validationError, properties, - txnID.getLeastSigBits(), txnID.getMostSigBits(), requestId, messageIdAdv.getBatchSize()); + cmdList = Collections.singletonList(Commands.newAck(consumerId, ledgerId, entryId, bitSetRecyclable, + ackType, validationError, properties, txnID.getLeastSigBits(), txnID.getMostSigBits(), requestId, + messageIdAdv.getBatchSize())); bitSetRecyclable.recycle(); } else { - cmd = Commands.newAck(consumerId, ledgerId, entryId, null, ackType, validationError, properties, - txnID.getLeastSigBits(), txnID.getMostSigBits(), requestId); + MessageIdImpl[] chunkMsgIds = this.unAckedChunkedMessageIdSequenceMap.remove(messageIdAdv); + // cumulative ack chunk by the last messageId + if (chunkMsgIds == null || ackType == AckType.Cumulative) { + cmdList = Collections.singletonList(Commands.newAck(consumerId, ledgerId, entryId, null, ackType, + validationError, properties, txnID.getLeastSigBits(), txnID.getMostSigBits(), requestId)); + } else { + if (Commands.peerSupportsMultiMessageAcknowledgment( + getClientCnx().getRemoteEndpointProtocolVersion())) { + List> entriesToAck = + new ArrayList<>(chunkMsgIds.length); + for (MessageIdImpl cMsgId : chunkMsgIds) { + if (cMsgId != null && chunkMsgIds.length > 1) { + entriesToAck.add(Triple.of(cMsgId.getLedgerId(), cMsgId.getEntryId(), null)); + } + } + cmdList = Collections.singletonList( + newMultiTransactionMessageAck(consumerId, txnID, entriesToAck, requestId)); + } else { + cmdList = new ArrayList<>(); + for (MessageIdImpl cMsgId : chunkMsgIds) { + cmdList.add(Commands.newAck(consumerId, cMsgId.ledgerId, cMsgId.entryId, null, ackType, + validationError, properties, + txnID.getLeastSigBits(), txnID.getMostSigBits(), requestId)); + } + } + } } if (ackType == AckType.Cumulative) { @@ -2732,8 +2996,55 @@ private CompletableFuture doTransactionAcknowledgeForResponse(MessageId me .ConnectException("Failed to ack message [" + messageId + "] " + "for transaction [" + txnID + "] due to consumer connect fail, consumer state: " + getState())); } else { - return cnx.newAckForReceipt(cmd, requestId); + List> completableFutures = new LinkedList<>(); + cmdList.forEach(cmd -> completableFutures.add(cnx.newAckForReceipt(cmd, requestId))); + return FutureUtil.waitForAll(completableFutures); + } + } + + private ByteBuf newMultiTransactionMessageAck(long consumerId, TxnID txnID, + List> entries, + long requestID) { + BaseCommand cmd = newMultiMessageAckCommon(entries); + cmd.getAck() + .setConsumerId(consumerId) + .setAckType(AckType.Individual) + .setTxnidLeastBits(txnID.getLeastSigBits()) + .setTxnidMostBits(txnID.getMostSigBits()) + .setRequestId(requestID); + return serializeWithSize(cmd); + } + + private static final FastThreadLocal LOCAL_BASE_COMMAND = new FastThreadLocal() { + @Override + protected BaseCommand initialValue() throws Exception { + return new BaseCommand(); + } + }; + + private static BaseCommand newMultiMessageAckCommon(List> entries) { + BaseCommand cmd = LOCAL_BASE_COMMAND.get() + .clear() + .setType(BaseCommand.Type.ACK); + CommandAck ack = cmd.setAck(); + int entriesCount = entries.size(); + for (int i = 0; i < entriesCount; i++) { + long ledgerId = entries.get(i).getLeft(); + long entryId = entries.get(i).getMiddle(); + ConcurrentBitSetRecyclable bitSet = entries.get(i).getRight(); + MessageIdData msgId = ack.addMessageId() + .setLedgerId(ledgerId) + .setEntryId(entryId); + if (bitSet != null) { + long[] ackSet = bitSet.toLongArray(); + for (int j = 0; j < ackSet.length; j++) { + msgId.addAckSet(ackSet[j]); + } + bitSet.recycle(); + } } + + return cmd; } private CompletableFuture doTransactionAcknowledgeForResponse(List messageIds, AckType ackType, @@ -2784,4 +3095,10 @@ boolean isAckReceiptEnabled() { private static final Logger log = LoggerFactory.getLogger(ConsumerImpl.class); + @VisibleForTesting + enum SeekStatus { + NOT_STARTED, + IN_PROGRESS, + COMPLETED + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerInterceptors.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerInterceptors.java index 832dc0bacaee9..dd1e2cec3b3ef 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerInterceptors.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerInterceptors.java @@ -44,6 +44,38 @@ public ConsumerInterceptors(List> interceptors) { this.interceptors = interceptors; } + + /** + * This method is called when a message arrives in the consumer. + *

      + * This method calls {@link ConsumerInterceptor#onArrival(Consumer, Message) method for each + * interceptor. + *

      + * This method does not throw exceptions. If any of the interceptors in the chain throws an exception, it gets + * caught and logged, and next interceptor in int the chain is called with 'messages' returned by the previous + * successful interceptor beforeConsume call. + * + * @param consumer the consumer which contains the interceptors + * @param message message to be consume by the client. + * @return messages that are either modified by interceptors or same as messages passed to this method. + */ + public Message onArrival(Consumer consumer, Message message) { + Message interceptorMessage = message; + for (int i = 0, interceptorsSize = interceptors.size(); i < interceptorsSize; i++) { + try { + interceptorMessage = interceptors.get(i).onArrival(consumer, interceptorMessage); + } catch (Throwable e) { + if (consumer != null) { + log.warn("Error executing interceptor beforeConsume callback topic: {} consumerName: {}", + consumer.getTopic(), consumer.getConsumerName(), e); + } else { + log.warn("Error executing interceptor beforeConsume callback", e); + } + } + } + return interceptorMessage; + } + /** * This is called just before the message is returned by {@link Consumer#receive()}, * {@link MessageListener#received(Consumer, Message)} or the {@link java.util.concurrent.CompletableFuture} diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsDisabled.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsDisabled.java index e8719753befd1..374b88ab3eb7d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsDisabled.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsDisabled.java @@ -23,6 +23,7 @@ import java.util.Optional; import org.apache.pulsar.client.api.ConsumerStats; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.ProducerStats; public class ConsumerStatsDisabled implements ConsumerStatsRecorder { private static final long serialVersionUID = 1L; @@ -124,6 +125,16 @@ public Map getMsgNumInSubReceiverQueue() { return null; } + @Override + public ProducerStats getDeadLetterProducerStats() { + return null; + } + + @Override + public ProducerStats getRetryLetterProducerStats() { + return null; + } + @Override public double getRateMsgsReceived() { return 0; @@ -148,4 +159,14 @@ public void reset() { public void updateCumulativeStats(ConsumerStats stats) { // do nothing } + + @Override + public void setDeadLetterProducerStats(ProducerStats producerStats) { + // do nothing + } + + @Override + public void setRetryLetterProducerStats(ProducerStats producerStats) { + // do nothing + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorder.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorder.java index 1a7de725f31b6..1d0d9e734b38d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorder.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorder.java @@ -22,6 +22,7 @@ import java.util.Optional; import org.apache.pulsar.client.api.ConsumerStats; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.ProducerStats; public interface ConsumerStatsRecorder extends ConsumerStats { void updateNumMsgsReceived(Message message); @@ -39,4 +40,8 @@ public interface ConsumerStatsRecorder extends ConsumerStats { void reset(); void updateCumulativeStats(ConsumerStats stats); + + void setDeadLetterProducerStats(ProducerStats producerStats); + + void setRetryLetterProducerStats(ProducerStats producerStats); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorderImpl.java index 8630bedc65f31..8dfc0af8e1d93 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorderImpl.java @@ -33,6 +33,7 @@ import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.ConsumerStats; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.ProducerStats; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.slf4j.Logger; @@ -63,6 +64,10 @@ public class ConsumerStatsRecorderImpl implements ConsumerStatsRecorder { private volatile double receivedMsgsRate; private volatile double receivedBytesRate; + volatile ProducerStats deadLetterProducerStats; + + volatile ProducerStats retryLetterProducerStats; + private static final DecimalFormat THROUGHPUT_FORMAT = new DecimalFormat("0.00"); public ConsumerStatsRecorderImpl() { @@ -238,7 +243,11 @@ public void updateCumulativeStats(ConsumerStats stats) { @Override public Integer getMsgNumInReceiverQueue() { if (consumer instanceof ConsumerBase) { - return ((ConsumerBase) consumer).incomingMessages.size(); + ConsumerBase consumerBase = (ConsumerBase) consumer; + if (consumerBase.listener != null){ + return ConsumerBase.MESSAGE_LISTENER_QUEUE_SIZE_UPDATER.get(consumerBase); + } + return consumerBase.incomingMessages.size(); } return null; } @@ -255,6 +264,26 @@ public Map getMsgNumInSubReceiverQueue() { return null; } + @Override + public ProducerStats getDeadLetterProducerStats() { + return deadLetterProducerStats; + } + + @Override + public ProducerStats getRetryLetterProducerStats() { + return retryLetterProducerStats; + } + + @Override + public void setDeadLetterProducerStats(ProducerStats producerStats) { + this.deadLetterProducerStats = producerStats; + } + + @Override + public void setRetryLetterProducerStats(ProducerStats producerStats) { + this.retryLetterProducerStats = producerStats; + } + @Override public long getNumMsgsReceived() { return numMsgsReceived.longValue(); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java index 38b8954377957..53796ff7a4bf5 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java @@ -21,40 +21,39 @@ import io.netty.channel.EventLoopGroup; import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponse; -import io.netty.handler.ssl.SslContext; -import io.netty.handler.ssl.SslProvider; import java.io.Closeable; import java.io.IOException; import java.net.HttpURLConnection; import java.net.InetSocketAddress; import java.net.URI; import java.net.URL; -import java.security.GeneralSecurityException; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.CompletableFuture; -import javax.net.ssl.SSLContext; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.AuthenticationDataProvider; -import org.apache.pulsar.client.api.KeyStoreParams; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.PulsarClientException.NotFoundException; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; -import org.apache.pulsar.client.util.WithSNISslEngineFactory; +import org.apache.pulsar.client.util.ExecutorProvider; +import org.apache.pulsar.client.util.PulsarHttpAsyncSslEngineFactory; import org.apache.pulsar.common.util.ObjectMapperFactory; -import org.apache.pulsar.common.util.SecurityUtility; -import org.apache.pulsar.common.util.keystoretls.KeyStoreSSLContext; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.asynchttpclient.AsyncHttpClient; import org.asynchttpclient.AsyncHttpClientConfig; import org.asynchttpclient.BoundRequestBuilder; import org.asynchttpclient.DefaultAsyncHttpClient; import org.asynchttpclient.DefaultAsyncHttpClientConfig; import org.asynchttpclient.Request; +import org.asynchttpclient.SslEngineFactory; import org.asynchttpclient.channel.DefaultKeepAliveStrategy; -import org.asynchttpclient.netty.ssl.JsseSslEngineFactory; @Slf4j @@ -66,6 +65,8 @@ public class HttpClient implements Closeable { protected final AsyncHttpClient httpClient; protected final ServiceNameResolver serviceNameResolver; protected final Authentication authentication; + protected ScheduledExecutorService executorService; + protected PulsarSslFactory sslFactory; protected HttpClient(ClientConfigurationData conf, EventLoopGroup eventLoopGroup) throws PulsarClientException { this.authentication = conf.getAuthentication(); @@ -92,65 +93,28 @@ public boolean keepAlive(InetSocketAddress remoteAddress, Request ahcRequest, if ("https".equals(serviceNameResolver.getServiceUri().getServiceName())) { try { // Set client key and certificate if available - AuthenticationDataProvider authData = authentication.getAuthData(); - - if (conf.isUseKeyStoreTls()) { - SSLContext sslCtx = null; - KeyStoreParams params = authData.hasDataForTls() ? authData.getTlsKeyStoreParams() : - new KeyStoreParams(conf.getTlsKeyStoreType(), conf.getTlsKeyStorePath(), - conf.getTlsKeyStorePassword()); + this.executorService = Executors + .newSingleThreadScheduledExecutor(new ExecutorProvider + .ExtendedThreadFactory("httpclient-ssl-refresh")); + PulsarSslConfiguration sslConfiguration = buildSslConfiguration(conf); + this.sslFactory = (PulsarSslFactory) Class.forName(conf.getSslFactoryPlugin()) + .getConstructor().newInstance(); + this.sslFactory.initialize(sslConfiguration); + this.sslFactory.createInternalSslContext(); + if (conf.getAutoCertRefreshSeconds() > 0) { + this.executorService.scheduleWithFixedDelay(this::refreshSslContext, + conf.getAutoCertRefreshSeconds(), + conf.getAutoCertRefreshSeconds(), TimeUnit.SECONDS); + } + String hostname = conf.isTlsHostnameVerificationEnable() ? null : serviceNameResolver + .resolveHostUri().getHost(); + SslEngineFactory sslEngineFactory = new PulsarHttpAsyncSslEngineFactory(this.sslFactory, hostname); + confBuilder.setSslEngineFactory(sslEngineFactory); - sslCtx = KeyStoreSSLContext.createClientSslContext( - conf.getSslProvider(), - params.getKeyStoreType(), - params.getKeyStorePath(), - params.getKeyStorePassword(), - conf.isTlsAllowInsecureConnection(), - conf.getTlsTrustStoreType(), - conf.getTlsTrustStorePath(), - conf.getTlsTrustStorePassword(), - conf.getTlsCiphers(), - conf.getTlsProtocols()); - JsseSslEngineFactory sslEngineFactory = new JsseSslEngineFactory(sslCtx); - confBuilder.setSslEngineFactory(sslEngineFactory); - } else { - SslProvider sslProvider = null; - if (conf.getSslProvider() != null) { - sslProvider = SslProvider.valueOf(conf.getSslProvider()); - } - SslContext sslCtx = null; - if (authData.hasDataForTls()) { - sslCtx = authData.getTlsTrustStoreStream() == null - ? SecurityUtility.createNettySslContextForClient(sslProvider, - conf.isTlsAllowInsecureConnection(), - conf.getTlsTrustCertsFilePath(), authData.getTlsCertificates(), - authData.getTlsPrivateKey(), conf.getTlsCiphers(), conf.getTlsProtocols()) - : SecurityUtility.createNettySslContextForClient(sslProvider, - conf.isTlsAllowInsecureConnection(), - authData.getTlsTrustStoreStream(), authData.getTlsCertificates(), - authData.getTlsPrivateKey(), conf.getTlsCiphers(), conf.getTlsProtocols()); - } else { - sslCtx = SecurityUtility.createNettySslContextForClient( - sslProvider, - conf.isTlsAllowInsecureConnection(), - conf.getTlsTrustCertsFilePath(), - conf.getTlsCertificateFilePath(), - conf.getTlsKeyFilePath(), - conf.getTlsCiphers(), - conf.getTlsProtocols()); - } - confBuilder.setSslContext(sslCtx); - if (!conf.isTlsHostnameVerificationEnable()) { - confBuilder.setSslEngineFactory(new WithSNISslEngineFactory(serviceNameResolver - .resolveHostUri().getHost())); - } - } confBuilder.setUseInsecureTrustManager(conf.isTlsAllowInsecureConnection()); confBuilder.setDisableHttpsEndpointIdentificationAlgorithm(!conf.isTlsHostnameVerificationEnable()); - } catch (GeneralSecurityException e) { - throw new PulsarClientException.InvalidConfigurationException(e); } catch (Exception e) { throw new PulsarClientException.InvalidConfigurationException(e); } @@ -177,6 +141,9 @@ void setServiceUrl(String serviceUrl) throws PulsarClientException { @Override public void close() throws IOException { httpClient.close(); + if (executorService != null) { + executorService.shutdownNow(); + } } public CompletableFuture get(String path, Class clazz) { @@ -264,4 +231,37 @@ public CompletableFuture get(String path, Class clazz) { return future; } + + protected PulsarSslConfiguration buildSslConfiguration(ClientConfigurationData config) + throws PulsarClientException { + return PulsarSslConfiguration.builder() + .tlsProvider(config.getSslProvider()) + .tlsKeyStoreType(config.getTlsKeyStoreType()) + .tlsKeyStorePath(config.getTlsKeyStorePath()) + .tlsKeyStorePassword(config.getTlsKeyStorePassword()) + .tlsTrustStoreType(config.getTlsTrustStoreType()) + .tlsTrustStorePath(config.getTlsTrustStorePath()) + .tlsTrustStorePassword(config.getTlsTrustStorePassword()) + .tlsCiphers(config.getTlsCiphers()) + .tlsProtocols(config.getTlsProtocols()) + .tlsTrustCertsFilePath(config.getTlsTrustCertsFilePath()) + .tlsCertificateFilePath(config.getTlsCertificateFilePath()) + .tlsKeyFilePath(config.getTlsKeyFilePath()) + .allowInsecureConnection(config.isTlsAllowInsecureConnection()) + .requireTrustedClientCertOnConnect(false) + .tlsEnabledWithKeystore(config.isUseKeyStoreTls()) + .tlsCustomParams(config.getSslFactoryPluginParams()) + .authData(config.getAuthentication().getAuthData()) + .serverMode(false) + .isHttps(true) + .build(); + } + + protected void refreshSslContext() { + try { + this.sslFactory.update(); + } catch (Exception e) { + log.error("Failed to refresh SSL context", e); + } + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpLookupService.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpLookupService.java index 7969ce402363f..4a5557fa869e4 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpLookupService.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpLookupService.java @@ -19,23 +19,21 @@ package org.apache.pulsar.client.impl; import io.netty.channel.EventLoopGroup; -import java.io.IOException; +import io.opentelemetry.api.common.Attributes; import java.net.InetSocketAddress; import java.net.URI; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Base64; -import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.PulsarClientException.NotFoundException; import org.apache.pulsar.client.api.SchemaSerializationException; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; +import org.apache.pulsar.client.impl.metrics.LatencyHistogram; import org.apache.pulsar.client.impl.schema.SchemaInfoUtil; import org.apache.pulsar.client.impl.schema.SchemaUtils; import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace.Mode; @@ -62,11 +60,26 @@ public class HttpLookupService implements LookupService { private static final String BasePathV1 = "lookup/v2/destination/"; private static final String BasePathV2 = "lookup/v2/topic/"; - public HttpLookupService(ClientConfigurationData conf, EventLoopGroup eventLoopGroup) + private final LatencyHistogram histoGetBroker; + private final LatencyHistogram histoGetTopicMetadata; + private final LatencyHistogram histoGetSchema; + private final LatencyHistogram histoListTopics; + + public HttpLookupService(InstrumentProvider instrumentProvider, ClientConfigurationData conf, + EventLoopGroup eventLoopGroup) throws PulsarClientException { this.httpClient = new HttpClient(conf, eventLoopGroup); this.useTls = conf.isUseTls(); this.listenerName = conf.getListenerName(); + + LatencyHistogram histo = instrumentProvider.newLatencyHistogram("pulsar.client.lookup.duration", + "Duration of lookup operations", null, + Attributes.builder().put("pulsar.lookup.transport-type", "http").build()); + histoGetBroker = histo.withAttributes(Attributes.builder().put("pulsar.lookup.type", "topic").build()); + histoGetTopicMetadata = + histo.withAttributes(Attributes.builder().put("pulsar.lookup.type", "metadata").build()); + histoGetSchema = histo.withAttributes(Attributes.builder().put("pulsar.lookup.type", "schema").build()); + histoListTopics = histo.withAttributes(Attributes.builder().put("pulsar.lookup.type", "list-topics").build()); } @Override @@ -82,12 +95,22 @@ public void updateServiceUrl(String serviceUrl) throws PulsarClientException { */ @Override @SuppressWarnings("deprecation") - public CompletableFuture> getBroker(TopicName topicName) { + public CompletableFuture getBroker(TopicName topicName) { String basePath = topicName.isV2() ? BasePathV2 : BasePathV1; String path = basePath + topicName.getLookupName(); path = StringUtils.isBlank(listenerName) ? path : path + "?listenerName=" + Codec.encode(listenerName); - return httpClient.get(path, LookupData.class) - .thenCompose(lookupData -> { + + long startTime = System.nanoTime(); + CompletableFuture httpFuture = httpClient.get(path, LookupData.class); + + httpFuture.thenRun(() -> { + histoGetBroker.recordSuccess(System.nanoTime() - startTime); + }).exceptionally(x -> { + histoGetBroker.recordFailure(System.nanoTime() - startTime); + return null; + }); + + return httpFuture.thenCompose(lookupData -> { // Convert LookupData into as SocketAddress, handling exceptions URI uri = null; try { @@ -102,7 +125,8 @@ public CompletableFuture> getBroker(T } InetSocketAddress brokerAddress = InetSocketAddress.createUnresolved(uri.getHost(), uri.getPort()); - return CompletableFuture.completedFuture(Pair.of(brokerAddress, brokerAddress)); + return CompletableFuture.completedFuture(new LookupTopicResult(brokerAddress, brokerAddress, + false /* HTTP lookups never use the proxy */)); } catch (Exception e) { // Failed to parse url log.warn("[{}] Lookup Failed due to invalid url {}, {}", topicName, uri, e.getMessage()); @@ -111,11 +135,29 @@ public CompletableFuture> getBroker(T }); } + /** + * {@inheritDoc} + * @param useFallbackForNonPIP344Brokers HttpLookupService ignores this parameter + */ @Override - public CompletableFuture getPartitionedTopicMetadata(TopicName topicName) { + public CompletableFuture getPartitionedTopicMetadata( + TopicName topicName, boolean metadataAutoCreationEnabled, boolean useFallbackForNonPIP344Brokers) { + long startTime = System.nanoTime(); + String format = topicName.isV2() ? "admin/v2/%s/partitions" : "admin/%s/partitions"; - return httpClient.get(String.format(format, topicName.getLookupName()) + "?checkAllowAutoCreation=true", + CompletableFuture httpFuture = httpClient.get( + String.format(format, topicName.getLookupName()) + "?checkAllowAutoCreation=" + + metadataAutoCreationEnabled, PartitionedTopicMetadata.class); + + httpFuture.thenRun(() -> { + histoGetTopicMetadata.recordSuccess(System.nanoTime() - startTime); + }).exceptionally(x -> { + histoGetTopicMetadata.recordFailure(System.nanoTime() - startTime); + return null; + }); + + return httpFuture; } @Override @@ -131,6 +173,8 @@ public InetSocketAddress resolveHost() { @Override public CompletableFuture getTopicsUnderNamespace(NamespaceName namespace, Mode mode, String topicsPattern, String topicsHash) { + long startTime = System.nanoTime(); + CompletableFuture future = new CompletableFuture<>(); String format = namespace.isV2() @@ -138,21 +182,21 @@ public CompletableFuture getTopicsUnderNamespace(NamespaceName httpClient .get(String.format(format, namespace, mode.toString()), String[].class) .thenAccept(topics -> { - List result = new ArrayList<>(); - // do not keep partition part of topic name - Arrays.asList(topics).forEach(topic -> { - String filtered = TopicName.get(topic).getPartitionedTopicName(); - if (!result.contains(filtered)) { - result.add(filtered); - } - }); - future.complete(new GetTopicsResult(result, topicsHash, false, true)); + future.complete(new GetTopicsResult(topics)); }).exceptionally(ex -> { Throwable cause = FutureUtil.unwrapCompletionException(ex); log.warn("Failed to getTopicsUnderNamespace namespace {} {}.", namespace, cause.getMessage()); future.completeExceptionally(cause); return null; }); + + future.thenRun(() -> { + histoListTopics.recordSuccess(System.nanoTime() - startTime); + }).exceptionally(x -> { + histoListTopics.recordFailure(System.nanoTime() - startTime); + return null; + }); + return future; } @@ -163,6 +207,7 @@ public CompletableFuture> getSchema(TopicName topicName) { @Override public CompletableFuture> getSchema(TopicName topicName, byte[] version) { + long startTime = System.nanoTime(); CompletableFuture> future = new CompletableFuture<>(); String schemaName = topicName.getSchemaName(); @@ -178,18 +223,14 @@ public CompletableFuture> getSchema(TopicName topicName, by } httpClient.get(path, GetSchemaResponse.class).thenAccept(response -> { if (response.getType() == SchemaType.KEY_VALUE) { - try { - SchemaData data = SchemaData - .builder() - .data(SchemaUtils.convertKeyValueDataStringToSchemaInfoSchema( - response.getData().getBytes(StandardCharsets.UTF_8))) - .type(response.getType()) - .props(response.getProperties()) - .build(); - future.complete(Optional.of(SchemaInfoUtil.newSchemaInfo(schemaName, data))); - } catch (IOException err) { - future.completeExceptionally(err); - } + SchemaData data = SchemaData + .builder() + .data(SchemaUtils.convertKeyValueDataStringToSchemaInfoSchema( + response.getData().getBytes(StandardCharsets.UTF_8))) + .type(response.getType()) + .props(response.getProperties()) + .build(); + future.complete(Optional.of(SchemaInfoUtil.newSchemaInfo(schemaName, data))); } else { future.complete(Optional.of(SchemaInfoUtil.newSchemaInfo(schemaName, response))); } @@ -206,6 +247,13 @@ public CompletableFuture> getSchema(TopicName topicName, by } return null; }); + + future.thenRun(() -> { + histoGetSchema.recordSuccess(System.nanoTime() - startTime); + }).exceptionally(x -> { + histoGetSchema.recordFailure(System.nanoTime() - startTime); + return null; + }); return future; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/LookupService.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/LookupService.java index 48ef67eae2047..3367ae99cb1a2 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/LookupService.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/LookupService.java @@ -21,7 +21,6 @@ import java.net.InetSocketAddress; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace.Mode; import org.apache.pulsar.common.lookup.GetTopicsResult; @@ -54,17 +53,50 @@ public interface LookupService extends AutoCloseable { * * @param topicName * topic-name - * @return a pair of addresses, representing the logical and physical address of the broker that serves given topic + * @return a {@link LookupTopicResult} representing the logical and physical address of the broker that serves the + * given topic, as well as proxying information. */ - CompletableFuture> getBroker(TopicName topicName); + CompletableFuture getBroker(TopicName topicName); /** * Returns {@link PartitionedTopicMetadata} for a given topic. - * - * @param topicName topic-name - * @return + * Note: this method will try to create the topic partitioned metadata if it does not exist. + * @deprecated Please call {{@link #getPartitionedTopicMetadata(TopicName, boolean, boolean)}}. + */ + @Deprecated + default CompletableFuture getPartitionedTopicMetadata(TopicName topicName) { + return getPartitionedTopicMetadata(topicName, true, true); + } + + /** + * See the doc {@link #getPartitionedTopicMetadata(TopicName, boolean, boolean)}. */ - CompletableFuture getPartitionedTopicMetadata(TopicName topicName); + default CompletableFuture getPartitionedTopicMetadata(TopicName topicName, + boolean metadataAutoCreationEnabled) { + return getPartitionedTopicMetadata(topicName, metadataAutoCreationEnabled, false); + } + + /** + * 1.Get the partitions if the topic exists. Return "{partition: n}" if a partitioned topic exists; + * return "{partition: 0}" if a non-partitioned topic exists. + * 2. When {@param metadataAutoCreationEnabled} is "false," neither partitioned topic nor non-partitioned topic + * does not exist. You will get a {@link PulsarClientException.NotFoundException} or + * a {@link PulsarClientException.TopicDoesNotExistException}. + * 2-1. You will get a {@link PulsarClientException.NotSupportedException} with metadataAutoCreationEnabled=false + * on an old broker version which does not support getting partitions without partitioned metadata + * auto-creation. + * 3.When {@param metadataAutoCreationEnabled} is "true," it will trigger an auto-creation for this topic(using + * the default topic auto-creation strategy you set for the broker), and the corresponding result is returned. + * For the result, see case 1. + * @param useFallbackForNonPIP344Brokers

      If true, fallback to the prior behavior of the method + * {@link #getPartitionedTopicMetadata(TopicName)} if the broker does not support the PIP-344 feature + * 'supports_get_partitioned_metadata_without_auto_creation'. This parameter only affects the behavior when + * {@param metadataAutoCreationEnabled} is false.

      + * @version 3.3.0. + */ + CompletableFuture getPartitionedTopicMetadata(TopicName topicName, + boolean metadataAutoCreationEnabled, + boolean useFallbackForNonPIP344Brokers); /** * Returns current SchemaInfo {@link SchemaInfo} for a given topic. @@ -98,12 +130,18 @@ public interface LookupService extends AutoCloseable { InetSocketAddress resolveHost(); /** - * Returns all the topics name for a given namespace. + * Returns all the topics that matches {@param topicPattern} for a given namespace. + * + * Note: {@param topicPattern} it relate to the topic name(without the partition suffix). For example: + * - There is a partitioned topic "tp-a" with two partitions. + * - tp-a-partition-0 + * - tp-a-partition-1 + * - If {@param topicPattern} is "tp-a", the consumer will subscribe to the two partitions. + * - if {@param topicPattern} is "tp-a-partition-0", the consumer will subscribe nothing. * * @param namespace : namespace-name * @return */ CompletableFuture getTopicsUnderNamespace(NamespaceName namespace, Mode mode, String topicPattern, String topicsHash); - } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/LookupTopicResult.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/LookupTopicResult.java new file mode 100644 index 0000000000000..9730b5c1da58a --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/LookupTopicResult.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import java.net.InetSocketAddress; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +@Getter +@Setter +@AllArgsConstructor +@ToString +public class LookupTopicResult { + private final InetSocketAddress logicalAddress; + private final InetSocketAddress physicalAddress; + private final boolean isUseProxy; +} diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MemoryLimitController.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MemoryLimitController.java index 935e3fad2b59d..c15821c054325 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MemoryLimitController.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MemoryLimitController.java @@ -22,7 +22,9 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; +import lombok.extern.slf4j.Slf4j; +@Slf4j public class MemoryLimitController { private final long memoryLimit; @@ -46,11 +48,19 @@ public MemoryLimitController(long memoryLimitBytes, long triggerThreshold, Runna } public void forceReserveMemory(long size) { + checkPositive(size); + if (size == 0) { + return; + } long newUsage = currentUsage.addAndGet(size); checkTrigger(newUsage - size, newUsage); } public boolean tryReserveMemory(long size) { + checkPositive(size); + if (size == 0) { + return true; + } while (true) { long current = currentUsage.get(); long newUsage = current + size; @@ -68,6 +78,15 @@ public boolean tryReserveMemory(long size) { } } + private static void checkPositive(long memorySize) { + if (memorySize < 0) { + String errorMsg = String.format("Try to reserve/release memory failed, the param memorySize" + + " is a negative value: %s", memorySize); + log.error(errorMsg); + throw new IllegalArgumentException(errorMsg); + } + } + private void checkTrigger(long prevUsage, long newUsage) { if (newUsage >= triggerThreshold && prevUsage < triggerThreshold && trigger != null) { if (triggerRunning.compareAndSet(false, true)) { @@ -81,6 +100,10 @@ private void checkTrigger(long prevUsage, long newUsage) { } public void reserveMemory(long size) throws InterruptedException { + checkPositive(size); + if (size == 0) { + return; + } if (!tryReserveMemory(size)) { mutex.lock(); try { @@ -94,6 +117,10 @@ public void reserveMemory(long size) throws InterruptedException { } public void releaseMemory(long size) { + checkPositive(size); + if (size == 0) { + return; + } long newUsage = currentUsage.addAndGet(-size); if (newUsage + size > memoryLimit && newUsage <= memoryLimit) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdAdvUtils.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdAdvUtils.java index c8b18524ec052..f66bb64202115 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdAdvUtils.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdAdvUtils.java @@ -40,6 +40,13 @@ static boolean equals(MessageIdAdv lhs, Object o) { && lhs.getBatchIndex() == rhs.getBatchIndex(); } + /** + * Acknowledge batch message. + * + * @param msgId the message id + * @param individual whether to acknowledge the batch message individually + * @return true if the batch message is fully acknowledged + */ static boolean acknowledge(MessageIdAdv msgId, boolean individual) { if (!isBatch(msgId)) { return true; @@ -51,12 +58,14 @@ static boolean acknowledge(MessageIdAdv msgId, boolean individual) { return false; } int batchIndex = msgId.getBatchIndex(); - if (individual) { - ackSet.clear(batchIndex); - } else { - ackSet.clear(0, batchIndex + 1); + synchronized (ackSet) { + if (individual) { + ackSet.clear(batchIndex); + } else { + ackSet.clear(0, batchIndex + 1); + } + return ackSet.isEmpty(); } - return ackSet.isEmpty(); } static boolean isBatch(MessageIdAdv msgId) { @@ -64,6 +73,9 @@ static boolean isBatch(MessageIdAdv msgId) { } static MessageIdAdv discardBatch(MessageId messageId) { + if (messageId instanceof ChunkMessageIdImpl) { + return (MessageIdAdv) messageId; + } MessageIdAdv msgId = (MessageIdAdv) messageId; return new MessageIdImpl(msgId.getLedgerId(), msgId.getEntryId(), msgId.getPartitionIndex()); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java index d369d639a73a0..72a5fd54e852b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java @@ -306,6 +306,13 @@ public static MessageImpl deserializeSkipBrokerEntryMetaData( return msg; } + public static MessageImpl deserializeMetadataWithEmptyPayload( + ByteBuf headersAndPayloadWithBrokerEntryMetadata) throws IOException { + MessageImpl msg = deserializeSkipBrokerEntryMetaData(headersAndPayloadWithBrokerEntryMetadata); + msg.payload = Unpooled.EMPTY_BUFFER; + return msg; + } + public void setReplicatedFrom(String cluster) { msgMetadata.setReplicatedFrom(cluster); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicConsumerStatsRecorderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicConsumerStatsRecorderImpl.java index 17018be02befc..eb4a339e20b2b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicConsumerStatsRecorderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicConsumerStatsRecorderImpl.java @@ -23,6 +23,7 @@ import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.ConsumerStats; import org.apache.pulsar.client.api.MultiTopicConsumerStats; +import org.apache.pulsar.client.api.ProducerStats; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,6 +33,10 @@ public class MultiTopicConsumerStatsRecorderImpl extends ConsumerStatsRecorderIm private static final long serialVersionUID = 1L; private Map partitionStats = new ConcurrentHashMap<>(); + private PartitionedTopicProducerStatsRecorderImpl deadLetterStats = new PartitionedTopicProducerStatsRecorderImpl(); + private PartitionedTopicProducerStatsRecorderImpl retryLetterStats = + new PartitionedTopicProducerStatsRecorderImpl(); + public MultiTopicConsumerStatsRecorderImpl() { super(); } @@ -55,5 +60,21 @@ public Map getPartitionStats() { return partitionStats; } + @Override + public ProducerStats getDeadLetterProducerStats() { + deadLetterStats.reset(); + partitionStats.forEach((partition, consumerStats) -> deadLetterStats.updateCumulativeStats(partition, + consumerStats.getDeadLetterProducerStats())); + return deadLetterStats; + } + + @Override + public ProducerStats getRetryLetterProducerStats() { + retryLetterStats.reset(); + partitionStats.forEach((partition, consumerStats) -> retryLetterStats.updateCumulativeStats(partition, + consumerStats.getRetryLetterProducerStats())); + return retryLetterStats; + } + private static final Logger log = LoggerFactory.getLogger(MultiTopicConsumerStatsRecorderImpl.class); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index d0607b97c1893..ff293af230838 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -26,6 +26,7 @@ import com.google.common.collect.Lists; import io.netty.util.Timeout; import io.netty.util.TimerTask; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -47,8 +48,11 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; +import java.util.function.Predicate; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.IntStream; +import javax.annotation.Nullable; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.client.api.BatchReceivePolicy; import org.apache.pulsar.client.api.Consumer; @@ -67,6 +71,7 @@ import org.apache.pulsar.client.util.ConsumerName; import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.common.api.proto.CommandAck.AckType; +import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.util.CompletableFutureCancellationHandler; @@ -80,19 +85,19 @@ public class MultiTopicsConsumerImpl extends ConsumerBase { public static final String DUMMY_TOPIC_NAME_PREFIX = "MultiTopicsConsumer-"; // Map , when get do ACK, consumer will by find by topic name - private final ConcurrentHashMap> consumers; + protected final ConcurrentHashMap> consumers; // Map , store partition number for each topic protected final ConcurrentHashMap partitionedTopics; // Queue of partition consumers on which we have stopped calling receiveAsync() because the // shared incoming queue was full - private final ConcurrentLinkedQueue> pausedConsumers; + protected final ConcurrentLinkedQueue> pausedConsumers; // sum of topicPartitions, simple topic has 1, partitioned topic equals to partition number. AtomicInteger allTopicPartitionsNumber; - private boolean paused = false; + private volatile boolean paused = false; private final Object pauseMutex = new Object(); // timeout related to auto check and subscribe partition increasement private volatile Timeout partitionsAutoUpdateTimeout = null; @@ -101,8 +106,10 @@ public class MultiTopicsConsumerImpl extends ConsumerBase { private final MultiTopicConsumerStatsRecorderImpl stats; private final ConsumerConfigurationData internalConfig; - private volatile MessageIdAdv startMessageId; + private final MessageIdAdv startMessageId; + private volatile boolean duringSeek = false; private final long startMessageRollbackDurationInSec; + private final ConsumerInterceptors internalConsumerInterceptors; MultiTopicsConsumerImpl(PulsarClientImpl client, ConsumerConfigurationData conf, ExecutorProvider executorProvider, CompletableFuture> subscribeFuture, Schema schema, ConsumerInterceptors interceptors, boolean createTopicIfDoesNotExist) { @@ -132,6 +139,11 @@ public class MultiTopicsConsumerImpl extends ConsumerBase { long startMessageRollbackDurationInSec) { super(client, singleTopic, conf, Math.max(2, conf.getReceiverQueueSize()), executorProvider, subscribeFuture, schema, interceptors); + if (interceptors != null) { + this.internalConsumerInterceptors = getInternalConsumerInterceptors(interceptors); + } else { + this.internalConsumerInterceptors = null; + } checkArgument(conf.getReceiverQueueSize() > 0, "Receiver queue size needs to be greater than 0 for Topics Consumer"); @@ -163,7 +175,8 @@ public class MultiTopicsConsumerImpl extends ConsumerBase { return; } - checkArgument(topicNamesValid(conf.getTopicNames()), "Topics is invalid."); + checkArgument(topicNamesValid(conf.getTopicNames()), "Subscription topics include duplicate items" + + " or invalid names."); List> futures = conf.getTopicNames().stream() .map(t -> subscribeAsync(t, createTopicIfDoesNotExist)) @@ -200,21 +213,21 @@ private static boolean topicNamesValid(Collection topics) { checkState(topics != null && topics.size() >= 1, "topics should contain more than 1 topic"); - Optional result = topics.stream() - .filter(topic -> !TopicName.isValid(topic)) - .findFirst(); + Set topicNames = new HashSet<>(); - if (result.isPresent()) { - log.warn("Received invalid topic name: {}", result.get()); - return false; + for (String topic : topics) { + if (!TopicName.isValid(topic)) { + log.warn("Received invalid topic name: {}", topic); + return false; + } + topicNames.add(TopicName.get(topic)); } // check topic names are unique - HashSet set = new HashSet<>(topics); - if (set.size() == topics.size()) { + if (topicNames.size() == topics.size()) { return true; } else { - log.warn("Topic names not unique. unique/all : {}/{}", set.size(), topics.size()); + log.warn("Topic names not unique. unique/all : {}/{}", topicNames.size(), topics.size()); return false; } } @@ -235,6 +248,10 @@ private void startReceivingMessages(List> newConsumers) { } private void receiveMessageFromConsumer(ConsumerImpl consumer, boolean batchReceive) { + if (duringSeek) { + log.info("[{}] Pause receiving messages for topic {} due to seek", subscription, consumer.getTopic()); + return; + } CompletableFuture>> messagesFuture; if (batchReceive) { messagesFuture = consumer.batchReceiveAsync().thenApply(msgs -> ((MessagesImpl) msgs).getMessageList()); @@ -252,8 +269,17 @@ private void receiveMessageFromConsumer(ConsumerImpl consumer, boolean batchR } // Process the message, add to the queue and trigger listener or async callback messages.forEach(msg -> { - if (isValidConsumerEpoch((MessageImpl) msg)) { + final boolean skipDueToSeek = duringSeek; + MessageImpl msgImpl = (MessageImpl) msg; + ClientCnx cnx = msgImpl.getCnx(); + boolean isValidEpoch = isValidConsumerEpoch(msgImpl); + if (isValidEpoch && !skipDueToSeek) { messageReceived(consumer, msg); + } else if (!isValidEpoch) { + consumer.increaseAvailablePermits(cnx); + } else if (skipDueToSeek) { + log.info("[{}] [{}] Skip processing message {} received during seek", topic, subscription, + msg.getMessageId()); } }); @@ -302,7 +328,8 @@ private void messageReceived(ConsumerImpl consumer, Message message) { CompletableFuture> receivedFuture = nextPendingReceive(); if (receivedFuture != null) { unAckedMessageTracker.add(topicMessage.getMessageId(), topicMessage.getRedeliveryCount()); - completePendingReceive(receivedFuture, topicMessage); + final Message interceptMessage = beforeConsume(topicMessage); + completePendingReceive(receivedFuture, interceptMessage); } else if (enqueueMessageAndCheckBatchReceive(topicMessage) && hasPendingBatchReceive()) { notifyPendingBatchReceivedCallBack(); } @@ -355,7 +382,7 @@ protected Message internalReceive() throws PulsarClientException { checkState(message instanceof TopicMessageImpl); unAckedMessageTracker.add(message.getMessageId(), message.getRedeliveryCount()); resumeReceivingFromPausedConsumersIfNeeded(); - return message; + return beforeConsume(message); } catch (Exception e) { ExceptionHandler.handleInterruptedException(e); throw PulsarClientException.unwrap(e); @@ -374,6 +401,7 @@ protected Message internalReceive(long timeout, TimeUnit unit) throws PulsarC decreaseIncomingMessageSize(message); checkArgument(message instanceof TopicMessageImpl); trackUnAckedMsgIfNoListener(message.getMessageId(), message.getRedeliveryCount()); + message = beforeConsume(message); } resumeReceivingFromPausedConsumersIfNeeded(); return message; @@ -433,7 +461,7 @@ protected CompletableFuture> internalReceiveAsync() { checkState(message instanceof TopicMessageImpl); unAckedMessageTracker.add(message.getMessageId(), message.getRedeliveryCount()); resumeReceivingFromPausedConsumersIfNeeded(); - result.complete(message); + result.complete(beforeConsume(message)); } }); return result; @@ -546,6 +574,7 @@ public void negativeAcknowledge(MessageId messageId) { checkArgument(messageId instanceof TopicMessageId); ConsumerImpl consumer = consumers.get(((TopicMessageId) messageId).getOwnerTopic()); consumer.negativeAcknowledge(messageId); + unAckedMessageTracker.remove(messageId); } @Override @@ -554,10 +583,11 @@ public void negativeAcknowledge(Message message) { checkArgument(messageId instanceof TopicMessageId); ConsumerImpl consumer = consumers.get(((TopicMessageId) messageId).getOwnerTopic()); consumer.negativeAcknowledge(message); + unAckedMessageTracker.remove(messageId); } @Override - public CompletableFuture unsubscribeAsync() { + public CompletableFuture unsubscribeAsync(boolean force) { if (getState() == State.Closing || getState() == State.Closed) { return FutureUtil.failedFuture( new PulsarClientException.AlreadyClosedException("Topics Consumer was already closed")); @@ -566,7 +596,7 @@ public CompletableFuture unsubscribeAsync() { CompletableFuture unsubscribeFuture = new CompletableFuture<>(); List> futureList = consumers.values().stream() - .map(ConsumerImpl::unsubscribeAsync).collect(Collectors.toList()); + .map(c -> c.unsubscribeAsync(force)).collect(Collectors.toList()); FutureUtil.waitForAll(futureList) .thenComposeAsync((r) -> { @@ -746,17 +776,12 @@ public void seek(Function function) throws PulsarClientException @Override public CompletableFuture seekAsync(Function function) { - List> futures = new ArrayList<>(consumers.size()); - consumers.values().forEach(consumer -> futures.add(consumer.seekAsync(function))); - unAckedMessageTracker.clear(); - incomingMessages.clear(); - resetIncomingMessageSize(); - return FutureUtil.waitForAll(futures); + return seekAllAsync(consumer -> consumer.seekAsync(function)); } @Override public CompletableFuture seekAsync(MessageId messageId) { - final Consumer internalConsumer; + final ConsumerImpl internalConsumer; if (messageId instanceof TopicMessageId) { TopicMessageId topicMessageId = (TopicMessageId) messageId; internalConsumer = consumers.get(topicMessageId.getOwnerTopic()); @@ -773,25 +798,46 @@ public CompletableFuture seekAsync(MessageId messageId) { ); } - final CompletableFuture seekFuture; if (internalConsumer == null) { - List> futures = new ArrayList<>(consumers.size()); - consumers.values().forEach(consumerImpl -> futures.add(consumerImpl.seekAsync(messageId))); - seekFuture = FutureUtil.waitForAll(futures); + return seekAllAsync(consumer -> consumer.seekAsync(messageId)); } else { - seekFuture = internalConsumer.seekAsync(messageId); + return seekAsyncInternal(Collections.singleton(internalConsumer), __ -> __.seekAsync(messageId)); } + } + + @Override + public CompletableFuture seekAsync(long timestamp) { + return seekAllAsync(consumer -> consumer.seekAsync(timestamp)); + } + + private CompletableFuture seekAsyncInternal(Collection> consumers, + Function, CompletableFuture> seekFunc) { + beforeSeek(); + final CompletableFuture future = new CompletableFuture<>(); + FutureUtil.waitForAll(consumers.stream().map(seekFunc).collect(Collectors.toList())) + .whenComplete((__, e) -> afterSeek(future, e)); + return future; + } + + private CompletableFuture seekAllAsync(Function, CompletableFuture> seekFunc) { + return seekAsyncInternal(consumers.values(), seekFunc); + } + private void beforeSeek() { + duringSeek = true; unAckedMessageTracker.clear(); clearIncomingMessages(); - return seekFuture; } - @Override - public CompletableFuture seekAsync(long timestamp) { - List> futures = new ArrayList<>(consumers.size()); - consumers.values().forEach(consumer -> futures.add(consumer.seekAsync(timestamp))); - return FutureUtil.waitForAll(futures); + private void afterSeek(CompletableFuture seekFuture, @Nullable Throwable throwable) { + duringSeek = false; + log.info("[{}] Resume receiving messages for {} since seek is done", subscription, consumers.keySet()); + startReceivingMessages(new ArrayList<>(consumers.values())); + if (throwable == null) { + seekFuture.complete(null); + } else { + seekFuture.completeExceptionally(throwable); + } } @Override @@ -852,6 +898,7 @@ public synchronized ConsumerStats getStats() { return stats; } + @Override public UnAckedMessageTracker getUnAckedMessageTracker() { return unAckedMessageTracker; } @@ -900,7 +947,10 @@ private void removeTopic(String topic) { } } - // subscribe one more given topic + /*** + * Subscribe one more given topic. + * @param topicName topic name without the partition suffix. + */ public CompletableFuture subscribeAsync(String topicName, boolean createTopicIfDoesNotExist) { TopicName topicNameInstance = getTopicName(topicName); if (topicNameInstance == null) { @@ -921,7 +971,7 @@ public CompletableFuture subscribeAsync(String topicName, boolean createTo CompletableFuture subscribeResult = new CompletableFuture<>(); - client.getPartitionedTopicMetadata(topicName) + client.getPartitionedTopicMetadata(topicName, true, false) .thenAccept(metadata -> subscribeTopicPartitions(subscribeResult, fullTopicName, metadata.partitions, createTopicIfDoesNotExist)) .exceptionally(ex1 -> { @@ -976,8 +1026,12 @@ CompletableFuture subscribeAsync(String topicName, int numberPartitions) { new PulsarClientException.AlreadyClosedException("Topic name not valid")); } String fullTopicName = topicNameInstance.toString(); - if (consumers.containsKey(fullTopicName) - || partitionedTopics.containsKey(topicNameInstance.getPartitionedTopicName())) { + if (consumers.containsKey(fullTopicName)) { + return FutureUtil.failedFuture( + new PulsarClientException.AlreadyClosedException("Already subscribed to " + topicName)); + } + if (!topicNameInstance.isPartitioned() + && partitionedTopics.containsKey(topicNameInstance.getPartitionedTopicName())) { return FutureUtil.failedFuture( new PulsarClientException.AlreadyClosedException("Already subscribed to " + topicName)); } @@ -996,13 +1050,13 @@ CompletableFuture subscribeAsync(String topicName, int numberPartitions) { private void subscribeTopicPartitions(CompletableFuture subscribeResult, String topicName, int numPartitions, boolean createIfDoesNotExist) { - client.preProcessSchemaBeforeSubscribe(client, schema, topicName).whenComplete((schema, cause) -> { - if (null == cause) { - doSubscribeTopicPartitions(schema, subscribeResult, topicName, numPartitions, createIfDoesNotExist); - } else { - subscribeResult.completeExceptionally(cause); - } - }); + client.preProcessSchemaBeforeSubscribe(client, schema, topicName) + .thenAccept(schema -> { + doSubscribeTopicPartitions(schema, subscribeResult, topicName, numPartitions, createIfDoesNotExist); + }).exceptionally(cause -> { + subscribeResult.completeExceptionally(cause); + return null; + }); } private void doSubscribeTopicPartitions(Schema schema, @@ -1014,7 +1068,7 @@ private void doSubscribeTopicPartitions(Schema schema, log.debug("Subscribe to topic {} metadata.partitions: {}", topicName, numPartitions); } - List>> futureList; + CompletableFuture subscribeAllPartitionsFuture; if (numPartitions != PartitionedTopicMetadata.NON_PARTITIONED) { // Below condition is true if subscribeAsync() has been invoked second time with same // topicName before the first invocation had reached this point. @@ -1034,59 +1088,77 @@ private void doSubscribeTopicPartitions(Schema schema, ConsumerConfigurationData configurationData = getInternalConsumerConfig(); configurationData.setReceiverQueueSize(receiverQueueSize); - futureList = IntStream - .range(0, numPartitions) - .mapToObj( - partitionIndex -> { - String partitionName = TopicName.get(topicName).getPartition(partitionIndex).toString(); - CompletableFuture> subFuture = new CompletableFuture<>(); - configurationData.setStartPaused(paused); - ConsumerImpl newConsumer = createInternalConsumer(configurationData, partitionName, - partitionIndex, subFuture, createIfDoesNotExist, schema); - synchronized (pauseMutex) { - if (paused) { - newConsumer.pause(); - } else { - newConsumer.resume(); - } - consumers.putIfAbsent(newConsumer.getTopic(), newConsumer); + CompletableFuture> partitionsFuture; + if (createIfDoesNotExist || !TopicName.get(topicName).isPersistent()) { + partitionsFuture = CompletableFuture.completedFuture(IntStream.range(0, numPartitions) + .mapToObj(i -> Integer.valueOf(i)) + .collect(Collectors.toList())); + } else { + partitionsFuture = getExistsPartitions(topicName.toString()); + } + subscribeAllPartitionsFuture = partitionsFuture.thenCompose(partitions -> { + if (partitions.isEmpty()) { + partitionedTopics.remove(topicName, numPartitions); + return CompletableFuture.completedFuture(null); + } + List>> subscribeList = new ArrayList<>(); + for (int partitionIndex : partitions) { + String partitionName = TopicName.get(topicName).getPartition(partitionIndex).toString(); + CompletableFuture> subFuture = new CompletableFuture<>(); + configurationData.setStartPaused(paused); + ConsumerImpl newConsumer = createInternalConsumer(configurationData, partitionName, + partitionIndex, subFuture, createIfDoesNotExist, schema); + synchronized (pauseMutex) { + if (paused) { + newConsumer.pause(); + } else { + newConsumer.resume(); } - return subFuture; - }) - .collect(Collectors.toList()); + Consumer originalValue = consumers.putIfAbsent(newConsumer.getTopic(), newConsumer); + if (originalValue != null) { + newConsumer.closeAsync().exceptionally(ex -> { + log.error("[{}] [{}] Failed to close the orphan consumer", + partitionName, subscription, ex); + return null; + }); + } + } + subscribeList.add(subFuture); + } + return FutureUtil.waitForAll(subscribeList); + }); } else { allTopicPartitionsNumber.incrementAndGet(); - CompletableFuture> subFuture = new CompletableFuture<>(); - - consumers.compute(topicName, (key, existingValue) -> { - if (existingValue != null) { - String errorMessage = String.format("[%s] Failed to subscribe for topic [%s] in topics consumer. " - + "Topic is already being subscribed for in other thread.", topic, topicName); - log.warn(errorMessage); - subscribeResult.completeExceptionally(new PulsarClientException(errorMessage)); - return existingValue; - } else { - internalConfig.setStartPaused(paused); - ConsumerImpl newConsumer = createInternalConsumer(internalConfig, topicName, - -1, subFuture, createIfDoesNotExist, schema); - - synchronized (pauseMutex) { + CompletableFuture> subscribeFuture = new CompletableFuture<>(); + subscribeAllPartitionsFuture = subscribeFuture.thenAccept(__ -> {}); + + synchronized (pauseMutex) { + consumers.compute(topicName, (key, existingValue) -> { + if (existingValue != null) { + String errorMessage = + String.format("[%s] Failed to subscribe for topic [%s] in topics consumer. " + + "Topic is already being subscribed for in other thread.", topic, topicName); + log.warn(errorMessage); + subscribeResult.completeExceptionally(new PulsarClientException(errorMessage)); + return existingValue; + } else { + internalConfig.setStartPaused(paused); + ConsumerImpl newConsumer = createInternalConsumer(internalConfig, topicName, + -1, subscribeFuture, createIfDoesNotExist, schema); if (paused) { newConsumer.pause(); } else { newConsumer.resume(); } + return newConsumer; } - return newConsumer; - } - }); + }); + } - futureList = Collections.singletonList(subFuture); } - FutureUtil.waitForAll(futureList) - .thenAccept(finalFuture -> { + subscribeAllPartitionsFuture.thenAccept(finalFuture -> { if (allTopicPartitionsNumber.get() > getCurrentReceiverQueueSize()) { setCurrentReceiverQueueSize(allTopicPartitionsNumber.get()); } @@ -1107,6 +1179,8 @@ private void doSubscribeTopicPartitions(Schema schema, return; }) .exceptionally(ex -> { + log.warn("[{}] Failed to subscribe for topic [{}] in topics consumer {}", topic, topicName, + ex.getMessage()); handleSubscribeOneTopicError(topicName, ex, subscribeResult); return null; }); @@ -1125,12 +1199,12 @@ private ConsumerImpl createInternalConsumer(ConsumerConfigurationData conf return ConsumerImpl.newConsumerImpl(client, partitionName, configurationData, client.externalExecutorProvider(), partitionIndex, true, listener != null, subFuture, - startMessageId, schema, interceptors, + startMessageId, schema, this.internalConsumerInterceptors, createIfDoesNotExist, startMessageRollbackDurationInSec); } // handling failure during subscribe new topic, unsubscribe success created partitions - private void handleSubscribeOneTopicError(String topicName, + protected void handleSubscribeOneTopicError(String topicName, Throwable error, CompletableFuture subscribeFuture) { log.warn("[{}] Failed to subscribe for topic [{}] in topics consumer {}", topic, topicName, error.getMessage()); @@ -1223,56 +1297,6 @@ public CompletableFuture unsubscribeAsync(String topicName) { return unsubscribeFuture; } - // Remove a consumer for a topic - public CompletableFuture removeConsumerAsync(String topicName) { - checkArgument(TopicName.isValid(topicName), "Invalid topic name:" + topicName); - - if (getState() == State.Closing || getState() == State.Closed) { - return FutureUtil.failedFuture( - new PulsarClientException.AlreadyClosedException("Topics Consumer was already closed")); - } - - CompletableFuture unsubscribeFuture = new CompletableFuture<>(); - String topicPartName = TopicName.get(topicName).getPartitionedTopicName(); - - - List> consumersToClose = consumers.values().stream() - .filter(consumer -> { - String consumerTopicName = consumer.getTopic(); - return TopicName.get(consumerTopicName).getPartitionedTopicName().equals(topicPartName); - }).collect(Collectors.toList()); - - List> futureList = consumersToClose.stream() - .map(ConsumerImpl::closeAsync).collect(Collectors.toList()); - - FutureUtil.waitForAll(futureList) - .whenComplete((r, ex) -> { - if (ex == null) { - consumersToClose.forEach(consumer1 -> { - consumers.remove(consumer1.getTopic()); - pausedConsumers.remove(consumer1); - allTopicPartitionsNumber.decrementAndGet(); - }); - - removeTopic(topicName); - if (unAckedMessageTracker instanceof UnAckedTopicMessageTracker) { - ((UnAckedTopicMessageTracker) unAckedMessageTracker).removeTopicMessages(topicName); - } - - unsubscribeFuture.complete(null); - log.info("[{}] [{}] [{}] Removed Topics Consumer, allTopicPartitionsNumber: {}", - topicName, subscription, consumerName, allTopicPartitionsNumber); - } else { - unsubscribeFuture.completeExceptionally(ex); - setState(State.Failed); - log.error("[{}] [{}] [{}] Could not remove Topics Consumer", - topicName, subscription, consumerName, ex.getCause()); - } - }); - - return unsubscribeFuture; - } - // get topics name public List getPartitionedTopics() { @@ -1409,7 +1433,7 @@ private CompletableFuture subscribeIncreasedTopicPartitions(String topicNa } if (log.isDebugEnabled()) { log.debug("[{}] create consumer {} for partitionName: {}", - topicName, newConsumer.getTopic(), partitionName); + topicName, newConsumer.getTopic(), partitionName); } return subFuture; }) @@ -1538,4 +1562,97 @@ protected void setCurrentReceiverQueueSize(int newSize) { CURRENT_RECEIVER_QUEUE_SIZE_UPDATER.set(this, newSize); resumeReceivingFromPausedConsumersIfNeeded(); } + + /** + * Get the exists partitions of a partitioned topic, the result does not contain the partitions which has not been + * created yet(in other words, the partitions that do not exist in the response of "pulsar-admin topics list"). + * @return sorted partitions list if it is a partitioned topic; @return an empty list if it is a non-partitioned + * topic. + */ + private CompletableFuture> getExistsPartitions(String topic) { + TopicName topicName = TopicName.get(topic); + if (!topicName.isPersistent()) { + return FutureUtil.failedFuture(new IllegalArgumentException("The method getExistsPartitions" + + " does not support non-persistent topic yet.")); + } + return client.getLookup().getTopicsUnderNamespace(topicName.getNamespaceObject(), + CommandGetTopicsOfNamespace.Mode.PERSISTENT, + TopicName.getPattern(topicName.getPartitionedTopicName()), + null).thenApply(getTopicsResult -> { + if (getTopicsResult.getNonPartitionedOrPartitionTopics() == null + || getTopicsResult.getNonPartitionedOrPartitionTopics().isEmpty()) { + return Collections.emptyList(); + } + // If broker version is less than "2.11.x", it does not support broker-side pattern check, so append + // a client-side pattern check. + // If lookup service is typed HttpLookupService, the HTTP API does not support broker-side pattern + // check yet, so append a client-side pattern check. + Predicate clientSideFilter; + if (getTopicsResult.isFiltered()) { + clientSideFilter = __ -> true; + } else { + clientSideFilter = + tp -> Pattern.compile(TopicName.getPartitionPattern(topic)).matcher(tp).matches(); + } + ArrayList list = new ArrayList<>(getTopicsResult.getNonPartitionedOrPartitionTopics().size()); + for (String partition : getTopicsResult.getNonPartitionedOrPartitionTopics()) { + int partitionIndex = TopicName.get(partition).getPartitionIndex(); + if (partitionIndex < 0) { + // It is not a partition. + continue; + } + if (clientSideFilter.test(partition)) { + list.add(partitionIndex); + } + } + Collections.sort(list); + return list; + }); + } + + private ConsumerInterceptors getInternalConsumerInterceptors(ConsumerInterceptors multiTopicInterceptors) { + return new ConsumerInterceptors(new ArrayList<>()) { + + @Override + public Message onArrival(Consumer consumer, Message message) { + return multiTopicInterceptors.onArrival(consumer, message); + } + + @Override + public Message beforeConsume(Consumer consumer, Message message) { + return message; + } + + @Override + public void onAcknowledge(Consumer consumer, MessageId messageId, Throwable exception) { + multiTopicInterceptors.onAcknowledge(consumer, messageId, exception); + } + + @Override + public void onAcknowledgeCumulative(Consumer consumer, + MessageId messageId, Throwable exception) { + multiTopicInterceptors.onAcknowledgeCumulative(consumer, messageId, exception); + } + + @Override + public void onNegativeAcksSend(Consumer consumer, Set set) { + multiTopicInterceptors.onNegativeAcksSend(consumer, set); + } + + @Override + public void onAckTimeoutSend(Consumer consumer, Set set) { + multiTopicInterceptors.onAckTimeoutSend(consumer, set); + } + + @Override + public void onPartitionsChange(String topicName, int partitions) { + multiTopicInterceptors.onPartitionsChange(topicName, partitions); + } + + @Override + public void close() throws IOException { + multiTopicInterceptors.close(); + } + }; + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsReaderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsReaderImpl.java index 0f1a7429f4970..86f3199f2978f 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsReaderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsReaderImpl.java @@ -20,6 +20,7 @@ import java.io.IOException; +import java.util.List; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -38,6 +39,7 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionMode; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.TopicMessageId; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import org.apache.pulsar.client.impl.conf.ReaderConfigurationData; import org.apache.pulsar.client.util.ExecutorProvider; @@ -109,6 +111,10 @@ public void reachedEndOfTopic(Consumer consumer) { if (readerConfiguration.getCryptoKeyReader() != null) { consumerConfiguration.setCryptoKeyReader(readerConfiguration.getCryptoKeyReader()); } + + if (readerConfiguration.getMessageCrypto() != null) { + consumerConfiguration.setMessageCrypto(readerConfiguration.getMessageCrypto()); + } if (readerConfiguration.getKeyHashRanges() != null) { consumerConfiguration.setKeySharedPolicy( KeySharedPolicy @@ -126,7 +132,7 @@ public void reachedEndOfTopic(Consumer consumer) { ReaderInterceptorUtil.convertToConsumerInterceptors( this, readerConfiguration.getReaderInterceptorList()); multiTopicsConsumer = new MultiTopicsConsumerImpl<>(client, consumerConfiguration, executorProvider, - consumerFuture, schema, consumerInterceptors, true, + consumerFuture, schema, consumerInterceptors, true, readerConfiguration.getStartMessageId(), readerConfiguration.getStartMessageFromRollbackDurationInSec()); } @@ -231,4 +237,14 @@ public void close() throws IOException { public MultiTopicsConsumerImpl getMultiTopicsConsumer() { return multiTopicsConsumer; } + + @Override + public List getLastMessageIds() throws PulsarClientException { + return multiTopicsConsumer.getLastMessageIds(); + } + + @Override + public CompletableFuture> getLastMessageIdsAsync() { + return multiTopicsConsumer.getLastMessageIdsAsync(); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/Murmur3Hash32.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/Murmur3Hash32.java index ce76496b42b7f..3d26b9b176942 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/Murmur3Hash32.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/Murmur3Hash32.java @@ -19,7 +19,7 @@ /* * The original MurmurHash3 was written by Austin Appleby, and is placed in the * public domain. This source code, implemented by Licht Takeuchi, is based on - * the orignal MurmurHash3 source code. + * the original MurmurHash3 source code. */ package org.apache.pulsar.client.impl; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java index 37f58a0218091..d6b86e3593dc2 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java @@ -32,8 +32,11 @@ import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.RedeliveryBackoff; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; class NegativeAcksTracker implements Closeable { + private static final Logger log = LoggerFactory.getLogger(NegativeAcksTracker.class); private HashMap nackedMessages = null; @@ -79,9 +82,12 @@ private synchronized void triggerRedelivery(Timeout t) { } }); - messagesToRedeliver.forEach(nackedMessages::remove); - consumer.onNegativeAcksSend(messagesToRedeliver); - consumer.redeliverUnacknowledgedMessages(messagesToRedeliver); + if (!messagesToRedeliver.isEmpty()) { + messagesToRedeliver.forEach(nackedMessages::remove); + consumer.onNegativeAcksSend(messagesToRedeliver); + log.info("[{}] {} messages will be re-delivered", consumer, messagesToRedeliver.size()); + consumer.redeliverUnacknowledgedMessages(messagesToRedeliver); + } this.timeout = timer.newTimeout(this::triggerRedelivery, timerIntervalNanos, TimeUnit.NANOSECONDS); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/OpSendMsgStats.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/OpSendMsgStats.java new file mode 100644 index 0000000000000..dc28df50f2886 --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/OpSendMsgStats.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + + +public interface OpSendMsgStats { + long getUncompressedSize(); + + long getSequenceId(); + + int getRetryCount(); + + long getBatchSizeByte(); + + int getNumMessagesInBatch(); + + long getHighestSequenceId(); + + int getTotalChunks(); + + int getChunkId(); +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiterDisable.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/OpSendMsgStatsImpl.java similarity index 51% rename from pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiterDisable.java rename to pulsar-client/src/main/java/org/apache/pulsar/client/impl/OpSendMsgStatsImpl.java index fdc13ed8f4e31..41bb742776caa 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PublishRateLimiterDisable.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/OpSendMsgStatsImpl.java @@ -16,54 +16,58 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.broker.service; +package org.apache.pulsar.client.impl; -import org.apache.pulsar.common.policies.data.Policies; -import org.apache.pulsar.common.policies.data.PublishRate; +import lombok.Builder; -public class PublishRateLimiterDisable implements PublishRateLimiter { - - public static final PublishRateLimiterDisable DISABLED_RATE_LIMITER = new PublishRateLimiterDisable(); +@Builder +public class OpSendMsgStatsImpl implements OpSendMsgStats { + private long uncompressedSize; + private long sequenceId; + private int retryCount; + private long batchSizeByte; + private int numMessagesInBatch; + private long highestSequenceId; + private int totalChunks; + private int chunkId; @Override - public void checkPublishRate() { - // No-op + public long getUncompressedSize() { + return uncompressedSize; } @Override - public void incrementPublishCount(int numOfMessages, long msgSizeInBytes) { - // No-op + public long getSequenceId() { + return sequenceId; } @Override - public boolean resetPublishCount() { - // No-op - return false; + public int getRetryCount() { + return retryCount; } @Override - public boolean isPublishRateExceeded() { - return false; + public long getBatchSizeByte() { + return batchSizeByte; } @Override - public void update(Policies policies, String clusterName) { - // No-op + public int getNumMessagesInBatch() { + return numMessagesInBatch; } @Override - public void update(PublishRate maxPublishRate) { - // No-op + public long getHighestSequenceId() { + return highestSequenceId; } @Override - public boolean tryAcquire(int numbers, long bytes) { - // Always allow - return true; + public int getTotalChunks() { + return totalChunks; } @Override - public void close() { - // No-op + public int getChunkId() { + return chunkId; } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java index f780edc95c136..2dc826d9e3af3 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java @@ -27,9 +27,11 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -52,7 +54,6 @@ import org.apache.pulsar.client.impl.transaction.TransactionImpl; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,7 +61,7 @@ public class PartitionedProducerImpl extends ProducerBase { private static final Logger log = LoggerFactory.getLogger(PartitionedProducerImpl.class); - private final ConcurrentOpenHashMap> producers; + private final Map> producers = new ConcurrentHashMap<>(); private final MessageRouter routerPolicy; private final PartitionedTopicProducerStatsRecorderImpl stats; private TopicMetadata topicMetadata; @@ -76,8 +77,6 @@ public PartitionedProducerImpl(PulsarClientImpl client, String topic, ProducerCo int numPartitions, CompletableFuture> producerCreatedFuture, Schema schema, ProducerInterceptors interceptors) { super(client, topic, conf, producerCreatedFuture, schema, interceptors); - this.producers = - ConcurrentOpenHashMap.>newBuilder().build(); this.topicMetadata = new TopicMetadataImpl(numPartitions); this.routerPolicy = getMessageRouter(); stats = client.getConfiguration().getStatsIntervalSeconds() > 0 @@ -436,7 +435,7 @@ public CompletableFuture onTopicsExtended(Collection topicsExtende }); // call interceptor with the metadata change onPartitionsChange(topic, currentPartitionNumber); - return null; + return future; } } else { log.error("[{}] not support shrink topic partitions. old: {}, new: {}", diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternConsumerUpdateQueue.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternConsumerUpdateQueue.java new file mode 100644 index 0000000000000..d6eba6463a07d --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternConsumerUpdateQueue.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import com.google.common.annotations.VisibleForTesting; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.util.Collection; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; + +/** + * Used to make all tasks that will modify subscriptions will be executed one by one, and skip the unnecessary updating. + * + * So far, four three scenarios that will modify subscriptions: + * 1. When start pattern consumer. + * 2. After topic list watcher reconnected, it will call {@link PatternMultiTopicsConsumerImpl#recheckTopicsChange()}. + * this scenario only exists in the version >= 2.11 (both client-version and broker version are >= 2.11). + * 3. A scheduled task will call {@link PatternMultiTopicsConsumerImpl#recheckTopicsChange()}, this scenario only + * exists in the version < 2.11. + * 4. The topics change events will trigger a + * {@link PatternMultiTopicsConsumerImpl#topicsChangeListener#onTopicsRemoved(Collection)} or + * {@link PatternMultiTopicsConsumerImpl#topicsChangeListener#onTopicsAdded(Collection)}. + * + * When you are using this client connect to the broker whose version >= 2.11, there are three scenarios: [1, 2, 4]. + * When you are using this client connect to the broker whose version < 2.11, there is only one scenario: [3] and all + * the event will run in the same thread. + */ +@Slf4j +@SuppressFBWarnings("EI_EXPOSE_REP2") +public class PatternConsumerUpdateQueue { + + private static final Pair> RECHECK_OP = + Pair.of(UpdateSubscriptionType.RECHECK, null); + + private final LinkedBlockingQueue>> pendingTasks; + + private final PatternMultiTopicsConsumerImpl patternConsumer; + + private final PatternMultiTopicsConsumerImpl.TopicsChangedListener topicsChangeListener; + + /** + * Whether there is a task is in progress, this variable is used to confirm whether a next-task triggering is + * needed. + */ + private Pair> taskInProgress = null; + + /** + * Whether there is a recheck task in queue. + * - Since recheck task will do all changes, it can be used to compress multiple tasks to one. + * - To avoid skipping the newest changes, once the recheck task is starting to work, this variable will be set + * to "false". + */ + private boolean recheckTaskInQueue = false; + + private volatile long lastRecheckTaskStartingTimestamp = 0; + + private boolean closed; + + public PatternConsumerUpdateQueue(PatternMultiTopicsConsumerImpl patternConsumer) { + this(patternConsumer, patternConsumer.topicsChangeListener); + } + + /** This constructor is only for test. **/ + @VisibleForTesting + public PatternConsumerUpdateQueue(PatternMultiTopicsConsumerImpl patternConsumer, + PatternMultiTopicsConsumerImpl.TopicsChangedListener topicsChangeListener) { + this.patternConsumer = patternConsumer; + this.topicsChangeListener = topicsChangeListener; + this.pendingTasks = new LinkedBlockingQueue<>(); + // To avoid subscribing and topics changed events execute concurrently, let the change events starts after the + // subscribing task. + doAppend(Pair.of(UpdateSubscriptionType.CONSUMER_INIT, null)); + } + + synchronized void appendTopicsAddedOp(Collection topics) { + if (topics == null || topics.isEmpty()) { + return; + } + doAppend(Pair.of(UpdateSubscriptionType.TOPICS_ADDED, topics)); + } + + synchronized void appendTopicsRemovedOp(Collection topics) { + if (topics == null || topics.isEmpty()) { + return; + } + doAppend(Pair.of(UpdateSubscriptionType.TOPICS_REMOVED, topics)); + } + + synchronized void appendRecheckOp() { + doAppend(RECHECK_OP); + } + + synchronized void doAppend(Pair> task) { + if (log.isDebugEnabled()) { + log.debug("Pattern consumer [{}] try to append task. {} {}", patternConsumer.getSubscription(), + task.getLeft(), task.getRight() == null ? "" : task.getRight()); + } + // Once there is a recheck task in queue, it means other tasks can be skipped. + if (recheckTaskInQueue) { + return; + } + + // Once there are too many tasks in queue, compress them as a recheck task. + if (pendingTasks.size() >= 30 && !task.getLeft().equals(UpdateSubscriptionType.RECHECK)) { + appendRecheckOp(); + return; + } + + pendingTasks.add(task); + if (task.getLeft().equals(UpdateSubscriptionType.RECHECK)) { + recheckTaskInQueue = true; + } + + // If no task is in-progress, trigger a task execution. + if (taskInProgress == null) { + triggerNextTask(); + } + } + + synchronized void triggerNextTask() { + if (closed) { + return; + } + + final Pair> task = pendingTasks.poll(); + + // No pending task. + if (task == null) { + taskInProgress = null; + return; + } + + // If there is a recheck task in queue, skip others and only call the recheck task. + if (recheckTaskInQueue && !task.getLeft().equals(UpdateSubscriptionType.RECHECK)) { + triggerNextTask(); + return; + } + + // Execute pending task. + CompletableFuture newTaskFuture = null; + switch (task.getLeft()) { + case CONSUMER_INIT: { + newTaskFuture = patternConsumer.getSubscribeFuture().thenAccept(__ -> {}).exceptionally(ex -> { + // If the subscribe future was failed, the consumer will be closed. + synchronized (PatternConsumerUpdateQueue.this) { + this.closed = true; + patternConsumer.closeAsync().exceptionally(ex2 -> { + log.error("Pattern consumer failed to close, this error may left orphan consumers." + + " Subscription: {}", patternConsumer.getSubscription()); + return null; + }); + } + return null; + }); + break; + } + case TOPICS_ADDED: { + newTaskFuture = topicsChangeListener.onTopicsAdded(task.getRight()); + break; + } + case TOPICS_REMOVED: { + newTaskFuture = topicsChangeListener.onTopicsRemoved(task.getRight()); + break; + } + case RECHECK: { + recheckTaskInQueue = false; + lastRecheckTaskStartingTimestamp = System.currentTimeMillis(); + newTaskFuture = patternConsumer.recheckTopicsChange(); + break; + } + default: { + throw new RuntimeException("Un-support UpdateSubscriptionType"); + } + } + if (log.isDebugEnabled()) { + log.debug("Pattern consumer [{}] starting task. {} {} ", patternConsumer.getSubscription(), + task.getLeft(), task.getRight() == null ? "" : task.getRight()); + } + // Trigger next pending task. + taskInProgress = Pair.of(task.getLeft(), newTaskFuture); + newTaskFuture.thenAccept(ignore -> { + if (log.isDebugEnabled()) { + log.debug("Pattern consumer [{}] task finished. {} {} ", patternConsumer.getSubscription(), + task.getLeft(), task.getRight() == null ? "" : task.getRight()); + } + triggerNextTask(); + }).exceptionally(ex -> { + /** + * Once a updating fails, trigger a delayed new recheck task to guarantee all things is correct. + * - Skip if there is already a recheck task in queue. + * - Skip if the last recheck task has been executed after the current time. + */ + log.error("Pattern consumer [{}] task finished. {} {}. But it failed", patternConsumer.getSubscription(), + task.getLeft(), task.getRight() == null ? "" : task.getRight(), ex); + // Skip if there is already a recheck task in queue. + synchronized (PatternConsumerUpdateQueue.this) { + if (recheckTaskInQueue || PatternConsumerUpdateQueue.this.closed) { + return null; + } + } + // Skip if the last recheck task has been executed after the current time. + long failedTime = System.currentTimeMillis(); + patternConsumer.getClient().timer().newTimeout(timeout -> { + if (lastRecheckTaskStartingTimestamp <= failedTime) { + appendRecheckOp(); + } + }, 10, TimeUnit.SECONDS); + triggerNextTask(); + return null; + }); + } + + public synchronized CompletableFuture cancelAllAndWaitForTheRunningTask() { + this.closed = true; + if (taskInProgress == null) { + return CompletableFuture.completedFuture(null); + } + // If the in-progress task is consumer init task, it means nothing is in-progress. + if (taskInProgress.getLeft().equals(UpdateSubscriptionType.CONSUMER_INIT)) { + return CompletableFuture.completedFuture(null); + } + return taskInProgress.getRight().thenAccept(__ -> {}).exceptionally(ex -> null); + } + + private enum UpdateSubscriptionType { + /** A marker that indicates the consumer's subscribe task.**/ + CONSUMER_INIT, + /** Triggered by {@link PatternMultiTopicsConsumerImpl#topicsChangeListener}.**/ + TOPICS_ADDED, + /** Triggered by {@link PatternMultiTopicsConsumerImpl#topicsChangeListener}.**/ + TOPICS_REMOVED, + /** A fully check for pattern consumer. **/ + RECHECK; + } +} diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java index 12c7e4e4ba3c7..70ba3e33963f4 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java @@ -21,18 +21,19 @@ import static com.google.common.base.Preconditions.checkArgument; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; +import com.google.re2j.Pattern; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import io.netty.util.Timeout; import io.netty.util.TimerTask; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import java.util.regex.Pattern; -import java.util.stream.Collectors; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; @@ -41,20 +42,39 @@ import org.apache.pulsar.common.lookup.GetTopicsResult; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.topics.TopicList; +import org.apache.pulsar.common.util.Backoff; +import org.apache.pulsar.common.util.BackoffBuilder; import org.apache.pulsar.common.util.FutureUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class PatternMultiTopicsConsumerImpl extends MultiTopicsConsumerImpl implements TimerTask { private final Pattern topicsPattern; - private final TopicsChangedListener topicsChangeListener; + final TopicsChangedListener topicsChangeListener; private final Mode subscriptionMode; - private final CompletableFuture watcherFuture; + private final CompletableFuture watcherFuture = new CompletableFuture<>(); protected NamespaceName namespaceName; + + /** + * There is two task to re-check topic changes, the both tasks will not be take affects at the same time. + * 1. {@link #recheckTopicsChangeAfterReconnect}: it will be called after the {@link TopicListWatcher} reconnected + * if you enabled {@link TopicListWatcher}. This backoff used to do a retry if + * {@link #recheckTopicsChangeAfterReconnect} is failed. + * 2. {@link #run} A scheduled task to trigger re-check topic changes, it will be used if you disabled + * {@link TopicListWatcher}. + */ + private final Backoff recheckPatternTaskBackoff; + private final AtomicInteger recheckPatternEpoch = new AtomicInteger(); private volatile Timeout recheckPatternTimeout = null; private volatile String topicsHash; + private PatternConsumerUpdateQueue updateTaskQueue; + + /*** + * @param topicsPattern The regexp for the topic name(not contains partition suffix). + */ public PatternMultiTopicsConsumerImpl(Pattern topicsPattern, String topicsHash, PulsarClientImpl client, @@ -69,6 +89,11 @@ public PatternMultiTopicsConsumerImpl(Pattern topicsPattern, this.topicsPattern = topicsPattern; this.topicsHash = topicsHash; this.subscriptionMode = subscriptionMode; + this.recheckPatternTaskBackoff = new BackoffBuilder() + .setInitialTime(client.getConfiguration().getInitialBackoffIntervalNanos(), TimeUnit.NANOSECONDS) + .setMax(client.getConfiguration().getMaxBackoffIntervalNanos(), TimeUnit.NANOSECONDS) + .setMandatoryStop(0, TimeUnit.SECONDS) + .create(); if (this.namespaceName == null) { this.namespaceName = getNameSpaceFromPattern(topicsPattern); @@ -76,19 +101,23 @@ public PatternMultiTopicsConsumerImpl(Pattern topicsPattern, checkArgument(getNameSpaceFromPattern(topicsPattern).toString().equals(this.namespaceName.toString())); this.topicsChangeListener = new PatternTopicsChangedListener(); + this.updateTaskQueue = new PatternConsumerUpdateQueue(this); this.recheckPatternTimeout = client.timer() .newTimeout(this, Math.max(1, conf.getPatternAutoDiscoveryPeriod()), TimeUnit.SECONDS); - this.watcherFuture = new CompletableFuture<>(); if (subscriptionMode == Mode.PERSISTENT) { long watcherId = client.newTopicListWatcherId(); - new TopicListWatcher(topicsChangeListener, client, topicsPattern, watcherId, - namespaceName, topicsHash, watcherFuture); - watcherFuture.exceptionally(ex -> { - log.debug("Unable to create topic list watcher. Falling back to only polling for new topics", ex); - return null; - }); + new TopicListWatcher(updateTaskQueue, client, topicsPattern, watcherId, + namespaceName, topicsHash, watcherFuture, () -> recheckTopicsChangeAfterReconnect()); + watcherFuture + .thenAccept(__ -> recheckPatternTimeout.cancel()) + .exceptionally(ex -> { + log.warn("Pattern consumer [{}] unable to create topic list watcher. Falling back to only polling" + + " for new topics", conf.getSubscriptionName(), ex); + return null; + }); } else { - log.debug("Not creating topic list watcher for subscription mode {}", subscriptionMode); + log.debug("Pattern consumer [{}] not creating topic list watcher for subscription mode {}", + conf.getSubscriptionName(), subscriptionMode); watcherFuture.complete(null); } } @@ -97,40 +126,53 @@ public static NamespaceName getNameSpaceFromPattern(Pattern pattern) { return TopicName.get(pattern.pattern()).getNamespaceObject(); } + /** + * This method will be called after the {@link TopicListWatcher} reconnected after enabled {@link TopicListWatcher}. + */ + private void recheckTopicsChangeAfterReconnect() { + // Skip if closed or the task has been cancelled. + if (getState() == State.Closing || getState() == State.Closed) { + return; + } + // Do check. + updateTaskQueue.appendRecheckOp(); + } + // TimerTask to recheck topics change, and trigger subscribe/unsubscribe based on the change. @Override public void run(Timeout timeout) throws Exception { if (timeout.isCancelled()) { return; } - client.getLookup().getTopicsUnderNamespace(namespaceName, subscriptionMode, topicsPattern.pattern(), topicsHash) - .thenCompose(getTopicsResult -> { - - if (log.isDebugEnabled()) { - log.debug("Get topics under namespace {}, topics.size: {}, topicsHash: {}, filtered: {}", - namespaceName, getTopicsResult.getTopics().size(), getTopicsResult.getTopicsHash(), - getTopicsResult.isFiltered()); - getTopicsResult.getTopics().forEach(topicName -> - log.debug("Get topics under namespace {}, topic: {}", namespaceName, topicName)); - } + updateTaskQueue.appendRecheckOp(); + } - final List oldTopics = new ArrayList<>(getPartitionedTopics()); - for (String partition : getPartitions()) { - TopicName topicName = TopicName.get(partition); - if (!topicName.isPartitioned() || !oldTopics.contains(topicName.getPartitionedTopicName())) { - oldTopics.add(partition); + CompletableFuture recheckTopicsChange() { + String pattern = topicsPattern.pattern(); + final int epoch = recheckPatternEpoch.incrementAndGet(); + return client.getLookup().getTopicsUnderNamespace(namespaceName, subscriptionMode, pattern, topicsHash) + .thenCompose(getTopicsResult -> { + // If "recheckTopicsChange" has been called more than one times, only make the last one take affects. + // Use "synchronized (recheckPatternTaskBackoff)" instead of + // `synchronized(PatternMultiTopicsConsumerImpl.this)` to avoid locking in a wider range. + synchronized (recheckPatternTaskBackoff) { + if (recheckPatternEpoch.get() > epoch) { + return CompletableFuture.completedFuture(null); + } + if (log.isDebugEnabled()) { + log.debug("Pattern consumer [{}] get topics under namespace {}, topics.size: {}," + + " topicsHash: {}, filtered: {}", + PatternMultiTopicsConsumerImpl.this.getSubscription(), + namespaceName, getTopicsResult.getTopics().size(), getTopicsResult.getTopicsHash(), + getTopicsResult.isFiltered()); + getTopicsResult.getTopics().forEach(topicName -> + log.debug("Get topics under namespace {}, topic: {}", namespaceName, topicName)); } + + final List oldTopics = new ArrayList<>(getPartitions()); + return updateSubscriptions(topicsPattern, this::setTopicsHash, getTopicsResult, + topicsChangeListener, oldTopics, subscription); } - return updateSubscriptions(topicsPattern, this::setTopicsHash, getTopicsResult, - topicsChangeListener, oldTopics); - }).exceptionally(ex -> { - log.warn("[{}] Failed to recheck topics change: {}", topic, ex.getMessage()); - return null; - }).thenAccept(__ -> { - // schedule the next re-check task - this.recheckPatternTimeout = client.timer() - .newTimeout(PatternMultiTopicsConsumerImpl.this, - Math.max(1, conf.getPatternAutoDiscoveryPeriod()), TimeUnit.SECONDS); }); } @@ -138,7 +180,8 @@ static CompletableFuture updateSubscriptions(Pattern topicsPattern, java.util.function.Consumer topicsHashSetter, GetTopicsResult getTopicsResult, TopicsChangedListener topicsChangedListener, - List oldTopics) { + List oldTopics, + String subscriptionForLog) { topicsHashSetter.accept(getTopicsResult.getTopicsHash()); if (!getTopicsResult.isChanged()) { return CompletableFuture.completedFuture(null); @@ -146,14 +189,20 @@ static CompletableFuture updateSubscriptions(Pattern topicsPattern, List newTopics; if (getTopicsResult.isFiltered()) { - newTopics = getTopicsResult.getTopics(); + newTopics = getTopicsResult.getNonPartitionedOrPartitionTopics(); } else { - newTopics = TopicList.filterTopics(getTopicsResult.getTopics(), topicsPattern); + newTopics = getTopicsResult.filterTopics(topicsPattern).getNonPartitionedOrPartitionTopics(); } final List> listenersCallback = new ArrayList<>(2); - listenersCallback.add(topicsChangedListener.onTopicsAdded(TopicList.minus(newTopics, oldTopics))); - listenersCallback.add(topicsChangedListener.onTopicsRemoved(TopicList.minus(oldTopics, newTopics))); + Set topicsAdded = TopicList.minus(newTopics, oldTopics); + Set topicsRemoved = TopicList.minus(oldTopics, newTopics); + if (log.isDebugEnabled()) { + log.debug("Pattern consumer [{}] Recheck pattern consumer's topics. topicsAdded: {}, topicsRemoved: {}", + subscriptionForLog, topicsAdded, topicsRemoved); + } + listenersCallback.add(topicsChangedListener.onTopicsAdded(topicsAdded)); + listenersCallback.add(topicsChangedListener.onTopicsRemoved(topicsRemoved)); return FutureUtil.waitForAll(Collections.unmodifiableList(listenersCallback)); } @@ -167,60 +216,181 @@ void setTopicsHash(String topicsHash) { } interface TopicsChangedListener { - // unsubscribe and delete ConsumerImpl in the `consumers` map in `MultiTopicsConsumerImpl` based on added topics + /*** + * unsubscribe and delete {@link ConsumerImpl} in the {@link MultiTopicsConsumerImpl#consumers} map in + * {@link MultiTopicsConsumerImpl}. + * @param removedTopics topic names removed(contains the partition suffix). + */ CompletableFuture onTopicsRemoved(Collection removedTopics); - // subscribe and create a list of new ConsumerImpl, added them to the `consumers` map in - // `MultiTopicsConsumerImpl`. + + /*** + * subscribe and create a list of new {@link ConsumerImpl}, added them to the + * {@link MultiTopicsConsumerImpl#consumers} map in {@link MultiTopicsConsumerImpl}. + * @param addedTopics topic names added(contains the partition suffix). + */ CompletableFuture onTopicsAdded(Collection addedTopics); } private class PatternTopicsChangedListener implements TopicsChangedListener { + + /** + * {@inheritDoc} + */ @Override public CompletableFuture onTopicsRemoved(Collection removedTopics) { - CompletableFuture removeFuture = new CompletableFuture<>(); - if (removedTopics.isEmpty()) { - removeFuture.complete(null); - return removeFuture; + return CompletableFuture.completedFuture(null); + } + + // Unsubscribe and remove consumers in memory. + List> unsubscribeList = new ArrayList<>(removedTopics.size()); + Set partialRemoved = new HashSet<>(removedTopics.size()); + Set partialRemovedForLog = new HashSet<>(removedTopics.size()); + for (String tp : removedTopics) { + TopicName topicName = TopicName.get(tp); + ConsumerImpl consumer = consumers.get(topicName.toString()); + if (consumer != null) { + CompletableFuture unsubscribeFuture = new CompletableFuture<>(); + consumer.closeAsync().whenComplete((__, ex) -> { + if (ex != null) { + log.error("Pattern consumer [{}] failed to unsubscribe from topics: {}", + PatternMultiTopicsConsumerImpl.this.getSubscription(), topicName.toString(), ex); + unsubscribeFuture.completeExceptionally(ex); + } else { + consumers.remove(topicName.toString(), consumer); + unsubscribeFuture.complete(null); + } + }); + unsubscribeList.add(unsubscribeFuture); + partialRemoved.add(topicName.getPartitionedTopicName()); + partialRemovedForLog.add(topicName.toString()); + } + } + if (log.isDebugEnabled()) { + log.debug("Pattern consumer [{}] remove topics. {}", + PatternMultiTopicsConsumerImpl.this.getSubscription(), + partialRemovedForLog); } - List> futures = Lists.newArrayListWithExpectedSize(partitionedTopics.size()); - removedTopics.stream().forEach(topic -> futures.add(removeConsumerAsync(topic))); - FutureUtil.waitForAll(futures) - .thenAccept(finalFuture -> removeFuture.complete(null)) - .exceptionally(ex -> { - log.warn("[{}] Failed to unsubscribe from topics: {}", topic, ex.getMessage()); - removeFuture.completeExceptionally(ex); + // Remove partitioned topics in memory. + return FutureUtil.waitForAll(unsubscribeList).handle((__, ex) -> { + List removedPartitionedTopicsForLog = new ArrayList<>(); + for (String groupedTopicRemoved : partialRemoved) { + Integer partitions = partitionedTopics.get(groupedTopicRemoved); + if (partitions != null) { + boolean allPartitionsHasBeenRemoved = true; + for (int i = 0; i < partitions; i++) { + if (consumers.containsKey( + TopicName.get(groupedTopicRemoved).getPartition(i).toString())) { + allPartitionsHasBeenRemoved = false; + break; + } + } + if (allPartitionsHasBeenRemoved) { + removedPartitionedTopicsForLog.add(String.format("%s with %s partitions", + groupedTopicRemoved, partitions)); + partitionedTopics.remove(groupedTopicRemoved, partitions); + } + } + } + if (log.isDebugEnabled()) { + log.debug("Pattern consumer [{}] remove partitioned topics because all partitions have been" + + " removed. {}", PatternMultiTopicsConsumerImpl.this.getSubscription(), + removedPartitionedTopicsForLog); + } return null; }); - return removeFuture; } + /** + * {@inheritDoc} + */ @Override public CompletableFuture onTopicsAdded(Collection addedTopics) { - CompletableFuture addFuture = new CompletableFuture<>(); - if (addedTopics.isEmpty()) { - addFuture.complete(null); - return addFuture; + return CompletableFuture.completedFuture(null); } - - Set addTopicPartitionedName = addedTopics.stream() - .map(addTopicName -> TopicName.get(addTopicName).getPartitionedTopicName()) - .collect(Collectors.toSet()); - - List> futures = Lists.newArrayListWithExpectedSize(partitionedTopics.size()); - addTopicPartitionedName.forEach(partitionedTopic -> futures.add( - subscribeAsync(partitionedTopic, - false /* createTopicIfDoesNotExist */))); - FutureUtil.waitForAll(futures) - .thenAccept(finalFuture -> addFuture.complete(null)) - .exceptionally(ex -> { - log.warn("[{}] Failed to subscribe to topics: {}", topic, ex.getMessage()); - addFuture.completeExceptionally(ex); - return null; + List> futures = Lists.newArrayListWithExpectedSize(addedTopics.size()); + /** + * Three normal cases: + * 1. Expand partitions. + * 2. Non-partitioned topic, but has been subscribing. + * 3. Non-partitioned topic or Partitioned topic, but has not been subscribing. + * Two unexpected cases: + * Error-1: Received adding non-partitioned topic event, but has subscribed a partitioned topic with the + * same name. + * Error-2: Received adding partitioned topic event, but has subscribed a non-partitioned topic with the + * same name. + * + * Note: The events that triggered by {@link TopicsPartitionChangedListener} after expanding partitions has + * been disabled through "conf.setAutoUpdatePartitions(false)" when creating + * {@link PatternMultiTopicsConsumerImpl}. + */ + Set groupedTopics = new HashSet<>(); + List expendPartitionsForLog = new ArrayList<>(); + for (String tp : addedTopics) { + TopicName topicName = TopicName.get(tp); + groupedTopics.add(topicName.getPartitionedTopicName()); + } + for (String tp : addedTopics) { + TopicName topicName = TopicName.get(tp); + // Case 1: Expand partitions. + if (partitionedTopics.containsKey(topicName.getPartitionedTopicName())) { + if (consumers.containsKey(topicName.toString())) { + // Already subscribed. + } else if (topicName.getPartitionIndex() < 0) { + // Error-1: Received adding non-partitioned topic event, but has subscribed a partitioned topic + // with the same name. + log.error("Pattern consumer [{}] skip to subscribe to the non-partitioned topic {}, because has" + + "subscribed a partitioned topic with the same name", + PatternMultiTopicsConsumerImpl.this.getSubscription(), topicName.toString()); + } else { + if (topicName.getPartitionIndex() + 1 + > partitionedTopics.get(topicName.getPartitionedTopicName())) { + partitionedTopics.put(topicName.getPartitionedTopicName(), + topicName.getPartitionIndex() + 1); + } + expendPartitionsForLog.add(topicName.toString()); + CompletableFuture consumerFuture = subscribeAsync(topicName.toString(), + PartitionedTopicMetadata.NON_PARTITIONED); + consumerFuture.whenComplete((__, ex) -> { + if (ex != null) { + log.warn("Pattern consumer [{}] Failed to subscribe to topics: {}", + PatternMultiTopicsConsumerImpl.this.getSubscription(), topicName, ex); + } + }); + futures.add(consumerFuture); + } + groupedTopics.remove(topicName.getPartitionedTopicName()); + } else if (consumers.containsKey(topicName.toString())) { + // Case-2: Non-partitioned topic, but has been subscribing. + groupedTopics.remove(topicName.getPartitionedTopicName()); + } else if (consumers.containsKey(topicName.getPartitionedTopicName()) + && topicName.getPartitionIndex() >= 0) { + // Error-2: Received adding partitioned topic event, but has subscribed a non-partitioned topic + // with the same name. + log.error("Pattern consumer [{}] skip to subscribe to the partitioned topic {}, because has" + + "subscribed a non-partitioned topic with the same name", + PatternMultiTopicsConsumerImpl.this.getSubscription(), topicName); + groupedTopics.remove(topicName.getPartitionedTopicName()); + } + } + // Case 3: Non-partitioned topic or Partitioned topic, which has not been subscribed. + for (String partitionedTopic : groupedTopics) { + CompletableFuture consumerFuture = subscribeAsync(partitionedTopic, false); + consumerFuture.whenComplete((__, ex) -> { + if (ex != null) { + log.warn("Pattern consumer [{}] Failed to subscribe to topics: {}", + PatternMultiTopicsConsumerImpl.this.getSubscription(), partitionedTopic, ex); + } }); - return addFuture; + futures.add(consumerFuture); + } + if (log.isDebugEnabled()) { + log.debug("Pattern consumer [{}] add topics. expend partitions {}, new subscribing {}", + PatternMultiTopicsConsumerImpl.this.getSubscription(), expendPartitionsForLog, groupedTopics); + } + return FutureUtil.waitForAll(futures); } } @@ -240,7 +410,7 @@ public CompletableFuture closeAsync() { closeFutures.add(watcher.closeAsync()); } } - closeFutures.add(super.closeAsync()); + closeFutures.add(updateTaskQueue.cancelAllAndWaitForTheRunningTask().thenCompose(__ -> super.closeAsync())); return FutureUtil.waitForAll(closeFutures); } @@ -249,5 +419,11 @@ Timeout getRecheckPatternTimeout() { return recheckPatternTimeout; } + protected void handleSubscribeOneTopicError(String topicName, + Throwable error, + CompletableFuture subscribeFuture) { + subscribeFuture.completeExceptionally(error); + } + private static final Logger log = LoggerFactory.getLogger(PatternMultiTopicsConsumerImpl.class); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java index 9086ccc4ef0e0..c0ee13b346a0b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java @@ -124,7 +124,18 @@ public boolean isDuplicate(MessageId messageId) { // Already included in a cumulative ack return true; } else { - return pendingIndividualAcks.contains(MessageIdAdvUtils.discardBatch(messageIdAdv)); + // If "batchIndexAckEnabled" is false, the batched messages acknowledgment will be traced by + // pendingIndividualAcks. So no matter what type the message ID is, check with "pendingIndividualAcks" + // first. + MessageIdAdv key = MessageIdAdvUtils.discardBatch(messageIdAdv); + if (pendingIndividualAcks.contains(key)) { + return true; + } + if (messageIdAdv.getBatchIndex() >= 0) { + ConcurrentBitSetRecyclable bitSet = pendingIndividualBatchIndexAcks.get(key); + return bitSet != null && !bitSet.get(messageIdAdv.getBatchIndex()); + } + return false; } } @@ -313,8 +324,15 @@ private CompletableFuture doIndividualBatchAckAsync(MessageIdAdv msgId) { MessageIdAdvUtils.discardBatch(msgId), __ -> { final BitSet ackSet = msgId.getAckSet(); final ConcurrentBitSetRecyclable value; - if (ackSet != null && !ackSet.isEmpty()) { - value = ConcurrentBitSetRecyclable.create(ackSet); + if (ackSet != null) { + synchronized (ackSet) { + if (!ackSet.isEmpty()) { + value = ConcurrentBitSetRecyclable.create(ackSet); + } else { + value = ConcurrentBitSetRecyclable.create(); + value.set(0, msgId.getBatchSize()); + } + } } else { value = ConcurrentBitSetRecyclable.create(); value.set(0, msgId.getBatchSize()); @@ -363,8 +381,11 @@ private CompletableFuture doImmediateBatchIndexAck(MessageIdAdv msgId, int .ConnectException("Consumer connect fail! consumer state:" + consumer.getState())); } BitSetRecyclable bitSet; - if (msgId.getAckSet() != null) { - bitSet = BitSetRecyclable.valueOf(msgId.getAckSet().toLongArray()); + BitSet ackSetFromMsgId = msgId.getAckSet(); + if (ackSetFromMsgId != null) { + synchronized (ackSetFromMsgId) { + bitSet = BitSetRecyclable.valueOf(ackSetFromMsgId.toLongArray()); + } } else { bitSet = BitSetRecyclable.create(); bitSet.set(0, batchSize); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBase.java index 7dc5f78398434..12e380fdd510c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBase.java @@ -19,7 +19,9 @@ package org.apache.pulsar.client.impl; import static com.google.common.base.Preconditions.checkArgument; +import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; @@ -32,7 +34,6 @@ import org.apache.pulsar.client.impl.transaction.TransactionImpl; import org.apache.pulsar.common.protocol.schema.SchemaHash; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; public abstract class ProducerBase extends HandlerState implements Producer { @@ -40,7 +41,7 @@ public abstract class ProducerBase extends HandlerState implements Producer schema; protected final ProducerInterceptors interceptors; - protected final ConcurrentOpenHashMap schemaCache; + protected final Map schemaCache = new ConcurrentHashMap<>(); protected volatile MultiSchemaMode multiSchemaMode = MultiSchemaMode.Auto; protected ProducerBase(PulsarClientImpl client, String topic, ProducerConfigurationData conf, @@ -50,8 +51,6 @@ protected ProducerBase(PulsarClientImpl client, String topic, ProducerConfigurat this.conf = conf; this.schema = schema; this.interceptors = interceptors; - this.schemaCache = - ConcurrentOpenHashMap.newBuilder().build(); if (!conf.isMultiSchema()) { multiSchemaMode = MultiSchemaMode.Disabled; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 2192ebfb64e75..b686252b58ade 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -40,7 +40,9 @@ import io.netty.util.Timeout; import io.netty.util.TimerTask; import io.netty.util.concurrent.ScheduledFuture; +import io.opentelemetry.api.common.Attributes; import java.io.IOException; +import java.net.URI; import java.nio.ByteBuffer; import java.util.ArrayDeque; import java.util.ArrayList; @@ -52,7 +54,6 @@ import java.util.Optional; import java.util.Queue; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -75,6 +76,11 @@ import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; import org.apache.pulsar.client.impl.crypto.MessageCryptoBc; +import org.apache.pulsar.client.impl.metrics.Counter; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; +import org.apache.pulsar.client.impl.metrics.LatencyHistogram; +import org.apache.pulsar.client.impl.metrics.Unit; +import org.apache.pulsar.client.impl.metrics.UpDownCounter; import org.apache.pulsar.client.impl.schema.JSONSchema; import org.apache.pulsar.client.impl.transaction.TransactionImpl; import org.apache.pulsar.client.util.MathUtils; @@ -91,6 +97,7 @@ import org.apache.pulsar.common.protocol.schema.SchemaVersion; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; +import org.apache.pulsar.common.util.BackoffBuilder; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.RelativeTimeUtil; @@ -166,10 +173,19 @@ public class ProducerImpl extends ProducerBase implements TimerTask, Conne private long lastBatchSendNanoTime; private Optional topicEpoch = Optional.empty(); - private final List previousExceptions = new CopyOnWriteArrayList(); + private final AtomicInteger previousExceptionCount = new AtomicInteger(); private boolean errorState; + private final LatencyHistogram latencyHistogram; + final LatencyHistogram rpcLatencyHistogram; + private final Counter publishedBytesCounter; + private final UpDownCounter pendingMessagesUpDownCounter; + private final UpDownCounter pendingBytesUpDownCounter; + + private final Counter producersOpenedCounter; + private final Counter producersClosedCounter; + public ProducerImpl(PulsarClientImpl client, String topic, ProducerConfigurationData conf, CompletableFuture> producerCreatedFuture, int partitionIndex, Schema schema, ProducerInterceptors interceptors, Optional overrideProducerName) { @@ -179,9 +195,6 @@ public ProducerImpl(PulsarClientImpl client, String topic, ProducerConfiguration this.userProvidedProducerName = StringUtils.isNotBlank(producerName); this.partitionIndex = partitionIndex; this.pendingMessages = createPendingMessagesQueue(); - this.chunkMaxMessageSize = conf.getChunkMaxMessageSize() > 0 - ? Math.min(conf.getChunkMaxMessageSize(), ClientCnx.getMaxMessageSize()) - : ClientCnx.getMaxMessageSize(); if (conf.getMaxPendingMessages() > 0) { this.semaphore = Optional.of(new Semaphore(conf.getMaxPendingMessages(), true)); } else { @@ -267,6 +280,26 @@ public ProducerImpl(PulsarClientImpl client, String topic, ProducerConfiguration metadata = Collections.unmodifiableMap(new HashMap<>(conf.getProperties())); } + InstrumentProvider ip = client.instrumentProvider(); + latencyHistogram = ip.newLatencyHistogram("pulsar.client.producer.message.send.duration", + "Publish latency experienced by the application, includes client batching time", topic, + Attributes.empty()); + rpcLatencyHistogram = ip.newLatencyHistogram("pulsar.client.producer.rpc.send.duration", + "Publish RPC latency experienced internally by the client when sending data to receiving an ack", topic, + Attributes.empty()); + publishedBytesCounter = ip.newCounter("pulsar.client.producer.message.send.size", + Unit.Bytes, "The number of bytes published", topic, Attributes.empty()); + pendingMessagesUpDownCounter = + ip.newUpDownCounter("pulsar.client.producer.message.pending.count", Unit.Messages, + "The number of messages in the producer internal send queue, waiting to be sent", topic, + Attributes.empty()); + pendingBytesUpDownCounter = ip.newUpDownCounter("pulsar.client.producer.message.pending.size", Unit.Bytes, + "The size of the messages in the producer internal queue, waiting to sent", topic, Attributes.empty()); + producersOpenedCounter = ip.newCounter("pulsar.client.producer.opened", Unit.Sessions, + "The number of producer sessions opened", topic, Attributes.empty()); + producersClosedCounter = ip.newCounter("pulsar.client.producer.closed", Unit.Sessions, + "The number of producer sessions closed", topic, Attributes.empty()); + this.connectionHandler = new ConnectionHandler(this, new BackoffBuilder() .setInitialTime(client.getConfiguration().getInitialBackoffIntervalNanos(), TimeUnit.NANOSECONDS) @@ -274,8 +307,15 @@ public ProducerImpl(PulsarClientImpl client, String topic, ProducerConfiguration .setMandatoryStop(Math.max(100, conf.getSendTimeoutMs() - 100), TimeUnit.MILLISECONDS) .create(), this); - + setChunkMaxMessageSize(); grabCnx(); + producersOpenedCounter.increment(); + } + + private void setChunkMaxMessageSize() { + this.chunkMaxMessageSize = conf.getChunkMaxMessageSize() > 0 + ? Math.min(conf.getChunkMaxMessageSize(), getMaxMessageSize()) + : getMaxMessageSize(); } protected void semaphoreRelease(final int releaseCountRequest) { @@ -333,73 +373,88 @@ CompletableFuture internalSendAsync(Message message) { if (interceptors != null) { interceptorMessage.getProperties(); } - sendAsync(interceptorMessage, new SendCallback() { - SendCallback nextCallback = null; - MessageImpl nextMsg = null; - long createdAt = System.nanoTime(); - @Override - public CompletableFuture getFuture() { - return future; - } + int msgSize = interceptorMessage.getDataBuffer().readableBytes(); + pendingMessagesUpDownCounter.increment(); + pendingBytesUpDownCounter.add(msgSize); - @Override - public SendCallback getNextSendCallback() { - return nextCallback; - } + sendAsync(interceptorMessage, new DefaultSendMessageCallback(future, interceptorMessage, msgSize)); + return future; + } - @Override - public MessageImpl getNextMessage() { - return nextMsg; - } + private class DefaultSendMessageCallback implements SendCallback { - @Override - public void sendComplete(Exception e) { - try { - if (e != null) { - stats.incrementSendFailed(); - onSendAcknowledgement(interceptorMessage, null, e); - future.completeExceptionally(e); - } else { - onSendAcknowledgement(interceptorMessage, interceptorMessage.getMessageId(), null); - future.complete(interceptorMessage.getMessageId()); - stats.incrementNumAcksReceived(System.nanoTime() - createdAt); - } - } finally { - interceptorMessage.getDataBuffer().release(); - } + CompletableFuture sendFuture; + MessageImpl currentMsg; + int msgSize; + long createdAt = System.nanoTime(); + SendCallback nextCallback = null; + MessageImpl nextMsg = null; - while (nextCallback != null) { - SendCallback sendCallback = nextCallback; - MessageImpl msg = nextMsg; - // Retain the buffer used by interceptors callback to get message. Buffer will release after - // complete interceptors. - try { - msg.getDataBuffer().retain(); - if (e != null) { - stats.incrementSendFailed(); - onSendAcknowledgement(msg, null, e); - sendCallback.getFuture().completeExceptionally(e); - } else { - onSendAcknowledgement(msg, msg.getMessageId(), null); - sendCallback.getFuture().complete(msg.getMessageId()); - stats.incrementNumAcksReceived(System.nanoTime() - createdAt); - } - nextMsg = nextCallback.getNextMessage(); - nextCallback = nextCallback.getNextSendCallback(); - } finally { - msg.getDataBuffer().release(); - } - } - } + DefaultSendMessageCallback(CompletableFuture sendFuture, MessageImpl currentMsg, int msgSize) { + this.sendFuture = sendFuture; + this.currentMsg = currentMsg; + this.msgSize = msgSize; + } - @Override - public void addCallback(MessageImpl msg, SendCallback scb) { - nextMsg = msg; - nextCallback = scb; + @Override + public CompletableFuture getFuture() { + return sendFuture; + } + + @Override + public SendCallback getNextSendCallback() { + return nextCallback; + } + + @Override + public MessageImpl getNextMessage() { + return nextMsg; + } + + @Override + public void sendComplete(Throwable e, OpSendMsgStats opSendMsgStats) { + SendCallback loopingCallback = this; + MessageImpl loopingMsg = currentMsg; + while (loopingCallback != null) { + onSendComplete(e, loopingCallback, loopingMsg); + loopingMsg = loopingCallback.getNextMessage(); + loopingCallback = loopingCallback.getNextSendCallback(); + } + } + + private void onSendComplete(Throwable e, SendCallback sendCallback, MessageImpl msg) { + long createdAt = (sendCallback instanceof ProducerImpl.DefaultSendMessageCallback) + ? ((DefaultSendMessageCallback) sendCallback).createdAt : this.createdAt; + long latencyNanos = System.nanoTime() - createdAt; + pendingMessagesUpDownCounter.decrement(); + pendingBytesUpDownCounter.subtract(msgSize); + ByteBuf payload = msg.getDataBuffer(); + if (payload == null) { + log.error("[{}] [{}] Payload is null when calling onSendComplete, which is not expected.", + topic, producerName); + } else { + ReferenceCountUtil.safeRelease(payload); } - }); - return future; + if (e != null) { + latencyHistogram.recordFailure(latencyNanos); + stats.incrementSendFailed(); + onSendAcknowledgement(msg, null, e); + sendCallback.getFuture().completeExceptionally(e); + } else { + latencyHistogram.recordSuccess(latencyNanos); + publishedBytesCounter.add(msgSize); + stats.incrementNumAcksReceived(latencyNanos); + onSendAcknowledgement(msg, msg.getMessageId(), null); + sendCallback.getFuture().complete(msg.getMessageId()); + } + } + + @Override + public void addCallback(MessageImpl msg, SendCallback scb) { + nextMsg = msg; + nextCallback = scb; + } } @Override @@ -409,15 +464,16 @@ CompletableFuture internalSendWithTxnAsync(Message message, Transa } else { CompletableFuture completableFuture = new CompletableFuture<>(); if (!((TransactionImpl) txn).checkIfOpen(completableFuture)) { - return completableFuture; + return completableFuture; } return ((TransactionImpl) txn).registerProducedTopic(topic) - .thenCompose(ignored -> internalSendAsync(message)); + .thenCompose(ignored -> internalSendAsync(message)); } } /** * Compress the payload if compression is configured. + * * @param payload * @return a new payload */ @@ -454,14 +510,14 @@ public void sendAsync(Message message, SendCallback callback) { // validate msg-size (For batching this will be check at the batch completion size) int compressedSize = compressedPayload.readableBytes(); - if (compressedSize > ClientCnx.getMaxMessageSize() && !this.conf.isChunkingEnabled()) { + if (compressedSize > getMaxMessageSize() && !this.conf.isChunkingEnabled()) { compressedPayload.release(); String compressedStr = conf.getCompressionType() != CompressionType.NONE ? "Compressed" : ""; PulsarClientException.InvalidMessageException invalidMessageException = new PulsarClientException.InvalidMessageException( format("The producer %s of the topic %s sends a %s message with %d bytes that exceeds" + " %d bytes", - producerName, topic, compressedStr, compressedSize, ClientCnx.getMaxMessageSize())); + producerName, topic, compressedStr, compressedSize, getMaxMessageSize())); completeCallbackAndReleaseSemaphore(uncompressedSize, callback, invalidMessageException); return; } @@ -469,9 +525,10 @@ public void sendAsync(Message message, SendCallback callback) { if (!msg.isReplicated() && msgMetadata.hasProducerName()) { PulsarClientException.InvalidMessageException invalidMessageException = - new PulsarClientException.InvalidMessageException( - format("The producer %s of the topic %s can not reuse the same message", producerName, topic), - msg.getSequenceId()); + new PulsarClientException.InvalidMessageException( + format("The producer %s of the topic %s can not reuse the same message", producerName, + topic), + msg.getSequenceId()); completeCallbackAndReleaseSemaphore(uncompressedSize, callback, invalidMessageException); compressedPayload.release(); return; @@ -491,19 +548,19 @@ public void sendAsync(Message message, SendCallback callback) { int payloadChunkSize; if (canAddToBatch(msg) || !conf.isChunkingEnabled()) { totalChunks = 1; - payloadChunkSize = ClientCnx.getMaxMessageSize(); + payloadChunkSize = getMaxMessageSize(); } else { // Reserve current metadata size for chunk size to avoid message size overflow. // NOTE: this is not strictly bounded, as metadata will be updated after chunking. // So there is a small chance that the final message size is larger than ClientCnx.getMaxMessageSize(). // But it won't cause produce failure as broker have 10 KB padding space for these cases. - payloadChunkSize = ClientCnx.getMaxMessageSize() - msgMetadata.getSerializedSize(); + payloadChunkSize = getMaxMessageSize() - msgMetadata.getSerializedSize(); if (payloadChunkSize <= 0) { PulsarClientException.InvalidMessageException invalidMessageException = new PulsarClientException.InvalidMessageException( format("The producer %s of the topic %s sends a message with %d bytes metadata that " + "exceeds %d bytes", producerName, topic, - msgMetadata.getSerializedSize(), ClientCnx.getMaxMessageSize())); + msgMetadata.getSerializedSize(), getMaxMessageSize())); completeCallbackAndReleaseSemaphore(uncompressedSize, callback, invalidMessageException); compressedPayload.release(); return; @@ -586,11 +643,14 @@ private void updateMessageMetadata(final MessageMetadata msgMetadata, final int msgMetadata.setProducerName(producerName); - if (conf.getCompressionType() != CompressionType.NONE) { - msgMetadata - .setCompression(CompressionCodecProvider.convertToWireProtocol(conf.getCompressionType())); + // The field "uncompressedSize" is zero means the compression info were not set yet. + if (msgMetadata.getUncompressedSize() <= 0) { + if (conf.getCompressionType() != CompressionType.NONE) { + msgMetadata + .setCompression(CompressionCodecProvider.convertToWireProtocol(conf.getCompressionType())); + } + msgMetadata.setUncompressedSize(uncompressedSize); } - msgMetadata.setUncompressedSize(uncompressedSize); } } @@ -638,8 +698,8 @@ private void serializeAndSendMessage(MessageImpl msg, msgMetadata.setUuid(uuid); } msgMetadata.setChunkId(chunkId) - .setNumChunksFromMsg(totalChunks) - .setTotalChunkMsgSize(compressedPayloadSize); + .setNumChunksFromMsg(totalChunks) + .setTotalChunkMsgSize(compressedPayloadSize); } if (canAddToBatch(msg) && totalChunks <= 1) { @@ -690,11 +750,17 @@ private void serializeAndSendMessage(MessageImpl msg, if (msg.getSchemaState() == MessageImpl.SchemaState.Ready) { ByteBufPair cmd = sendMessage(producerId, sequenceId, numMessages, messageId, msgMetadata, encryptedPayload); - op = OpSendMsg.create(msg, cmd, sequenceId, callback); + op = OpSendMsg.create(rpcLatencyHistogram, msg, cmd, sequenceId, callback); } else { - op = OpSendMsg.create(msg, null, sequenceId, callback); + op = OpSendMsg.create(rpcLatencyHistogram, msg, null, sequenceId, callback); final MessageMetadata finalMsgMetadata = msgMetadata; op.rePopulate = () -> { + if (msgMetadata.hasChunkId()) { + // The message metadata is shared between all chunks in a large message + // We need to reset the chunk id for each call of this method + // It's safe to do that because there is only 1 thread to manipulate this message metadata + finalMsgMetadata.setChunkId(chunkId); + } op.cmd = sendMessage(producerId, sequenceId, numMessages, messageId, finalMsgMetadata, encryptedPayload); }; @@ -767,15 +833,15 @@ private void tryRegisterSchema(ClientCnx cnx, MessageImpl msg, SendCallback call } SchemaInfo schemaInfo = msg.hasReplicateFrom() ? msg.getSchemaInfoForReplicator() : msg.getSchemaInfo(); schemaInfo = Optional.ofNullable(schemaInfo) - .filter(si -> si.getType().getValue() > 0) - .orElse(Schema.BYTES.getSchemaInfo()); + .filter(si -> si.getType().getValue() > 0) + .orElse(Schema.BYTES.getSchemaInfo()); getOrCreateSchemaAsync(cnx, schemaInfo).handle((v, ex) -> { if (ex != null) { Throwable t = FutureUtil.unwrapCompletionException(ex); log.warn("[{}] [{}] GetOrCreateSchema error", topic, producerName, t); if (t instanceof PulsarClientException.IncompatibleSchemaException) { msg.setSchemaState(MessageImpl.SchemaState.Broken); - callback.sendComplete((PulsarClientException.IncompatibleSchemaException) t); + callback.sendComplete(t, null); } } else { log.info("[{}] [{}] GetOrCreateSchema succeed", topic, producerName); @@ -803,10 +869,10 @@ private void tryRegisterSchema(ClientCnx cnx, MessageImpl msg, SendCallback call private CompletableFuture getOrCreateSchemaAsync(ClientCnx cnx, SchemaInfo schemaInfo) { if (!Commands.peerSupportsGetOrCreateSchema(cnx.getRemoteEndpointProtocolVersion())) { return FutureUtil.failedFuture( - new PulsarClientException.NotSupportedException( - format("The command `GetOrCreateSchema` is not supported for the protocol version %d. " - + "The producer is %s, topic is %s", - cnx.getRemoteEndpointProtocolVersion(), producerName, topic))); + new PulsarClientException.NotSupportedException( + format("The command `GetOrCreateSchema` is not supported for the protocol version %d. " + + "The producer is %s, topic is %s", + cnx.getRemoteEndpointProtocolVersion(), producerName, topic))); } long requestId = client.newRequestId(); ByteBuf request = Commands.newGetOrCreateSchema(requestId, topic, schemaInfo); @@ -878,7 +944,7 @@ private boolean canAddToBatch(MessageImpl msg) { private boolean canAddToCurrentBatch(MessageImpl msg) { return batchMessageContainer.haveEnoughSpace(msg) - && (!isMultiSchemaEnabled(false) || batchMessageContainer.hasSameSchema(msg)) + && (!isMultiSchemaEnabled(false) || batchMessageContainer.hasSameSchema(msg)) && batchMessageContainer.hasSameTxn(msg); } @@ -907,30 +973,31 @@ private void doBatchSendAndAdd(MessageImpl msg, SendCallback callback, ByteBu private boolean isValidProducerState(SendCallback callback, long sequenceId) { switch (getState()) { - case Ready: - // OK - case Connecting: - // We are OK to queue the messages on the client, it will be sent to the broker once we get the connection - case RegisteringSchema: - // registering schema - return true; - case Closing: - case Closed: - callback.sendComplete( - new PulsarClientException.AlreadyClosedException("Producer already closed", sequenceId)); - return false; - case ProducerFenced: - callback.sendComplete(new PulsarClientException.ProducerFencedException("Producer was fenced")); - return false; - case Terminated: - callback.sendComplete( - new PulsarClientException.TopicTerminatedException("Topic was terminated", sequenceId)); - return false; - case Failed: - case Uninitialized: - default: - callback.sendComplete(new PulsarClientException.NotConnectedException(sequenceId)); - return false; + case Ready: + // OK + case Connecting: + // We are OK to queue the messages on the client, it will be sent to the broker once we get the + // connection + case RegisteringSchema: + // registering schema + return true; + case Closing: + case Closed: + callback.sendComplete( + new PulsarClientException.AlreadyClosedException("Producer already closed", sequenceId), null); + return false; + case ProducerFenced: + callback.sendComplete(new PulsarClientException.ProducerFencedException("Producer was fenced"), null); + return false; + case Terminated: + callback.sendComplete( + new PulsarClientException.TopicTerminatedException("Topic was terminated", sequenceId), null); + return false; + case Failed: + case Uninitialized: + default: + callback.sendComplete(new PulsarClientException.NotConnectedException(sequenceId), null); + return false; } } @@ -944,20 +1011,20 @@ private boolean canEnqueueRequest(SendCallback callback, long sequenceId, int pa } else { if (!semaphore.map(Semaphore::tryAcquire).orElse(true)) { callback.sendComplete(new PulsarClientException.ProducerQueueIsFullError( - "Producer send queue is full", sequenceId)); + "Producer send queue is full", sequenceId), null); return false; } if (!client.getMemoryLimitController().tryReserveMemory(payloadSize)) { semaphore.ifPresent(Semaphore::release); callback.sendComplete(new PulsarClientException.MemoryBufferIsFullError( - "Client memory buffer is full", sequenceId)); + "Client memory buffer is full", sequenceId), null); return false; } } } catch (InterruptedException e) { Thread.currentThread().interrupt(); - callback.sendComplete(new PulsarClientException(e, sequenceId)); + callback.sendComplete(new PulsarClientException(e, sequenceId), null); return false; } @@ -1030,9 +1097,11 @@ private static final class LastSendFutureWrapper { private LastSendFutureWrapper(CompletableFuture lastSendFuture) { this.lastSendFuture = lastSendFuture; } + static LastSendFutureWrapper create(CompletableFuture lastSendFuture) { return new LastSendFutureWrapper(lastSendFuture); } + public CompletableFuture handleOnce() { return lastSendFuture.handle((ignore, t) -> { if (t != null && THROW_ONCE_UPDATER.compareAndSet(this, FALSE, TRUE)) { @@ -1057,6 +1126,7 @@ public CompletableFuture closeAsync() { return CompletableFuture.completedFuture(null); } + producersClosedCounter.increment(); closeProducerTasks(); ClientCnx cnx = cnx(); @@ -1231,7 +1301,7 @@ private void releaseSemaphoreForSendOp(OpSendMsg op) { private void completeCallbackAndReleaseSemaphore(long payloadSize, SendCallback callback, Exception exception) { semaphore.ifPresent(Semaphore::release); client.getMemoryLimitController().releaseMemory(payloadSize); - callback.sendComplete(exception); + callback.sendComplete(exception, null); } /** @@ -1263,9 +1333,10 @@ protected synchronized void recoverChecksumError(ClientCnx cnx, long sequenceId) releaseSemaphoreForSendOp(op); try { op.sendComplete( - new PulsarClientException.ChecksumException( - format("The checksum of the message which is produced by producer %s to the topic " - + "%s is corrupted", producerName, topic))); + new PulsarClientException.ChecksumException( + format("The checksum of the message which is produced by producer %s to the " + + "topic " + + "%s is corrupted", producerName, topic))); } catch (Throwable t) { log.warn("[{}] [{}] Got exception while completing the callback for msg {}:", topic, producerName, sequenceId, t); @@ -1313,7 +1384,7 @@ protected synchronized void recoverNotAllowedError(long sequenceId, String error * * @param op * @return returns true only if message is not modified and computed-checksum is same as previous checksum else - * return false that means that message is corrupted. Returns true if checksum is not present. + * return false that means that message is corrupted. Returns true if checksum is not present. */ protected boolean verifyLocalBufferIsNotCorrupted(OpSendMsg op) { ByteBufPair msg = op.cmd; @@ -1389,6 +1460,7 @@ public ReferenceCounted touch(Object hint) { } protected static final class OpSendMsg { + LatencyHistogram rpcLatencyHistogram; MessageImpl msg; List> msgs; ByteBufPair cmd; @@ -1408,6 +1480,7 @@ protected static final class OpSendMsg { int chunkId = -1; void initialize() { + rpcLatencyHistogram = null; msg = null; msgs = null; cmd = null; @@ -1427,9 +1500,11 @@ void initialize() { chunkedMessageCtx = null; } - static OpSendMsg create(MessageImpl msg, ByteBufPair cmd, long sequenceId, SendCallback callback) { + static OpSendMsg create(LatencyHistogram rpcLatencyHistogram, MessageImpl msg, ByteBufPair cmd, + long sequenceId, SendCallback callback) { OpSendMsg op = RECYCLER.get(); op.initialize(); + op.rpcLatencyHistogram = rpcLatencyHistogram; op.msg = msg; op.cmd = cmd; op.callback = callback; @@ -1439,10 +1514,11 @@ static OpSendMsg create(MessageImpl msg, ByteBufPair cmd, long sequenceId, Se return op; } - static OpSendMsg create(List> msgs, ByteBufPair cmd, long sequenceId, SendCallback callback, - int batchAllocatedSize) { + static OpSendMsg create(LatencyHistogram rpcLatencyHistogram, List> msgs, ByteBufPair cmd, + long sequenceId, SendCallback callback, int batchAllocatedSize) { OpSendMsg op = RECYCLER.get(); op.initialize(); + op.rpcLatencyHistogram = rpcLatencyHistogram; op.msgs = msgs; op.cmd = cmd; op.callback = callback; @@ -1456,10 +1532,12 @@ static OpSendMsg create(List> msgs, ByteBufPair cmd, long sequenc return op; } - static OpSendMsg create(List> msgs, ByteBufPair cmd, long lowestSequenceId, - long highestSequenceId, SendCallback callback, int batchAllocatedSize) { + static OpSendMsg create(LatencyHistogram rpcLatencyHistogram, List> msgs, ByteBufPair cmd, + long lowestSequenceId, + long highestSequenceId, SendCallback callback, int batchAllocatedSize) { OpSendMsg op = RECYCLER.get(); op.initialize(); + op.rpcLatencyHistogram = rpcLatencyHistogram; op.msgs = msgs; op.cmd = cmd; op.callback = callback; @@ -1484,31 +1562,49 @@ void updateSentTimestamp() { void sendComplete(final Exception e) { SendCallback callback = this.callback; + + long now = System.nanoTime(); if (null != callback) { Exception finalEx = e; if (finalEx instanceof TimeoutException) { TimeoutException te = (TimeoutException) e; long sequenceId = te.getSequenceId(); - long ns = System.nanoTime(); + //firstSentAt and lastSentAt maybe -1, it means that the message didn't flush to channel. String errMsg = String.format( - "%s : createdAt %s seconds ago, firstSentAt %s seconds ago, lastSentAt %s seconds ago, " - + "retryCount %s", - te.getMessage(), - RelativeTimeUtil.nsToSeconds(ns - this.createdAt), - RelativeTimeUtil.nsToSeconds(this.firstSentAt <= 0 - ? this.firstSentAt - : ns - this.firstSentAt), - RelativeTimeUtil.nsToSeconds(this.lastSentAt <= 0 - ? this.lastSentAt - : ns - this.lastSentAt), - retryCount + "%s : createdAt %s seconds ago, firstSentAt %s seconds ago, lastSentAt %s seconds ago, " + + "retryCount %s", + te.getMessage(), + RelativeTimeUtil.nsToSeconds(now - this.createdAt), + RelativeTimeUtil.nsToSeconds(this.firstSentAt <= 0 + ? this.firstSentAt + : now - this.firstSentAt), + RelativeTimeUtil.nsToSeconds(this.lastSentAt <= 0 + ? this.lastSentAt + : now - this.lastSentAt), + retryCount ); finalEx = new TimeoutException(errMsg, sequenceId); } - callback.sendComplete(finalEx); + if (e == null) { + rpcLatencyHistogram.recordSuccess(now - this.lastSentAt); + } else { + rpcLatencyHistogram.recordFailure(now - this.lastSentAt); + } + + OpSendMsgStats opSendMsgStats = OpSendMsgStatsImpl.builder() + .uncompressedSize(uncompressedSize) + .sequenceId(sequenceId) + .retryCount(retryCount) + .batchSizeByte(batchSizeByte) + .numMessagesInBatch(numMessagesInBatch) + .highestSequenceId(highestSequenceId) + .totalChunks(totalChunks) + .chunkId(chunkId) + .build(); + callback.sendComplete(finalEx, opSendMsgStats); } } @@ -1649,17 +1745,19 @@ public Iterator iterator() { } } + @Override - public void connectionOpened(final ClientCnx cnx) { - previousExceptions.clear(); - chunkMaxMessageSize = Math.min(chunkMaxMessageSize, ClientCnx.getMaxMessageSize()); + public CompletableFuture connectionOpened(final ClientCnx cnx) { + previousExceptionCount.set(0); + getConnectionHandler().setMaxMessageSize(cnx.getMaxMessageSize()); + setChunkMaxMessageSize(); final long epoch; synchronized (this) { // Because the state could have been updated while retrieving the connection, we set it back to connecting, // as long as the change from current state to connecting is a valid state change. if (!changeToConnecting()) { - return; + return CompletableFuture.completedFuture(null); } // We set the cnx reference before registering the producer on the cnx, so if the cnx breaks before creating // the producer, it will try to grab a new cnx. We also increment and get the epoch value for the producer. @@ -1672,7 +1770,7 @@ public void connectionOpened(final ClientCnx cnx) { long requestId = client.newRequestId(); PRODUCER_DEADLINE_UPDATER - .compareAndSet(this, 0, System.currentTimeMillis() + client.getConfiguration().getOperationTimeoutMs()); + .compareAndSet(this, 0, System.currentTimeMillis() + client.getConfiguration().getOperationTimeoutMs()); SchemaInfo schemaInfo = null; if (schema != null) { @@ -1683,7 +1781,7 @@ public void connectionOpened(final ClientCnx cnx) { // but now we have standardized on every schema to generate an Avro based schema if (Commands.peerSupportJsonSchemaAvroFormat(cnx.getRemoteEndpointProtocolVersion())) { schemaInfo = schema.getSchemaInfo(); - } else if (schema instanceof JSONSchema){ + } else if (schema instanceof JSONSchema) { JSONSchema jsonSchema = (JSONSchema) schema; schemaInfo = jsonSchema.getBackwardsCompatibleJsonSchemaInfo(); } else { @@ -1699,144 +1797,156 @@ public void connectionOpened(final ClientCnx cnx) { } } + final CompletableFuture future = new CompletableFuture<>(); cnx.sendRequestWithId( Commands.newProducer(topic, producerId, requestId, producerName, conf.isEncryptionEnabled(), metadata, schemaInfo, epoch, userProvidedProducerName, conf.getAccessMode(), topicEpoch, client.conf.isEnableTransaction(), conf.getInitialSubscriptionName()), requestId).thenAccept(response -> { - String producerName = response.getProducerName(); - long lastSequenceId = response.getLastSequenceId(); - schemaVersion = Optional.ofNullable(response.getSchemaVersion()); - schemaVersion.ifPresent(v -> schemaCache.put(SchemaHash.of(schema), v)); - - // We are now reconnected to broker and clear to send messages. Re-send all pending messages and - // set the cnx pointer so that new messages will be sent immediately - synchronized (ProducerImpl.this) { - if (getState() == State.Closing || getState() == State.Closed) { - // Producer was closed while reconnecting, close the connection to make sure the broker - // drops the producer on its side - cnx.removeProducer(producerId); - cnx.channel().close(); - return; - } - resetBackoff(); - - log.info("[{}] [{}] Created producer on cnx {}", topic, producerName, cnx.ctx().channel()); - connectionId = cnx.ctx().channel().toString(); - connectedSince = DateFormatter.now(); - if (conf.getAccessMode() != ProducerAccessMode.Shared && !topicEpoch.isPresent()) { - log.info("[{}] [{}] Producer epoch is {}", topic, producerName, response.getTopicEpoch()); - } - topicEpoch = response.getTopicEpoch(); - - if (this.producerName == null) { - this.producerName = producerName; - } - - if (this.msgIdGenerator == 0 && conf.getInitialSequenceId() == null) { - // Only update sequence id generator if it wasn't already modified. That means we only want - // to update the id generator the first time the producer gets established, and ignore the - // sequence id sent by broker in subsequent producer reconnects - this.lastSequenceIdPublished = lastSequenceId; - this.msgIdGenerator = lastSequenceId + 1; - } - - resendMessages(cnx, epoch); - } - }).exceptionally((e) -> { - Throwable cause = e.getCause(); + String producerName = response.getProducerName(); + long lastSequenceId = response.getLastSequenceId(); + schemaVersion = Optional.ofNullable(response.getSchemaVersion()); + schemaVersion.ifPresent(v -> schemaCache.put(SchemaHash.of(schema), v)); + + // We are now reconnected to broker and clear to send messages. Re-send all pending messages and + // set the cnx pointer so that new messages will be sent immediately + synchronized (ProducerImpl.this) { + State state = getState(); + if (state == State.Closing || state == State.Closed) { + // Producer was closed while reconnecting, close the connection to make sure the broker + // drops the producer on its side cnx.removeProducer(producerId); - if (getState() == State.Closing || getState() == State.Closed) { - // Producer was closed while reconnecting, close the connection to make sure the broker - // drops the producer on its side - cnx.channel().close(); - return null; - } + cnx.channel().close(); + future.complete(null); + return; + } + resetBackoff(); - if (cause instanceof TimeoutException) { - // Creating the producer has timed out. We need to ensure the broker closes the producer - // in case it was indeed created, otherwise it might prevent new create producer operation, - // since we are not necessarily closing the connection. - long closeRequestId = client.newRequestId(); - ByteBuf cmd = Commands.newCloseProducer(producerId, closeRequestId); - cnx.sendRequestWithId(cmd, closeRequestId); - } + log.info("[{}] [{}] Created producer on cnx {}", topic, producerName, cnx.ctx().channel()); + connectionId = cnx.ctx().channel().toString(); + connectedSince = DateFormatter.now(); + if (conf.getAccessMode() != ProducerAccessMode.Shared && !topicEpoch.isPresent()) { + log.info("[{}] [{}] Producer epoch is {}", topic, producerName, response.getTopicEpoch()); + } + topicEpoch = response.getTopicEpoch(); - if (cause instanceof PulsarClientException.ProducerFencedException) { - if (log.isDebugEnabled()) { - log.debug("[{}] [{}] Failed to create producer: {}", - topic, producerName, cause.getMessage()); - } - } else { - log.error("[{}] [{}] Failed to create producer: {}", topic, producerName, cause.getMessage()); - } - // Close the producer since topic does not exist. - if (cause instanceof PulsarClientException.TopicDoesNotExistException) { - closeAsync().whenComplete((v, ex) -> { - if (ex != null) { - log.error("Failed to close producer on TopicDoesNotExistException.", ex); - } - producerCreatedFuture.completeExceptionally(cause); - }); - return null; - } - if (cause instanceof PulsarClientException.ProducerBlockedQuotaExceededException) { - synchronized (this) { - log.warn("[{}] [{}] Topic backlog quota exceeded. Throwing Exception on producer.", topic, - producerName); - - if (log.isDebugEnabled()) { - log.debug("[{}] [{}] Pending messages: {}", topic, producerName, - pendingMessages.messagesCount()); - } - - PulsarClientException bqe = new PulsarClientException.ProducerBlockedQuotaExceededException( - format("The backlog quota of the topic %s that the producer %s produces to is exceeded", - topic, producerName)); - failPendingMessages(cnx(), bqe); - } - } else if (cause instanceof PulsarClientException.ProducerBlockedQuotaExceededError) { - log.warn("[{}] [{}] Producer is blocked on creation because backlog exceeded on topic.", - producerName, topic); + if (this.producerName == null) { + this.producerName = producerName; + } + + if (this.msgIdGenerator == 0 && conf.getInitialSequenceId() == null) { + // Only update sequence id generator if it wasn't already modified. That means we only want + // to update the id generator the first time the producer gets established, and ignore the + // sequence id sent by broker in subsequent producer reconnects + this.lastSequenceIdPublished = lastSequenceId; + this.msgIdGenerator = lastSequenceId + 1; + } + + resendMessages(cnx, epoch); + } + future.complete(null); + }).exceptionally((e) -> { + Throwable cause = e.getCause(); + cnx.removeProducer(producerId); + State state = getState(); + if (state == State.Closing || state == State.Closed) { + // Producer was closed while reconnecting, close the connection to make sure the broker + // drops the producer on its side + cnx.channel().close(); + future.complete(null); + return null; + } + + if (cause instanceof TimeoutException) { + // Creating the producer has timed out. We need to ensure the broker closes the producer + // in case it was indeed created, otherwise it might prevent new create producer operation, + // since we are not necessarily closing the connection. + long closeRequestId = client.newRequestId(); + ByteBuf cmd = Commands.newCloseProducer(producerId, closeRequestId); + cnx.sendRequestWithId(cmd, closeRequestId); + } + + if (cause instanceof PulsarClientException.ProducerFencedException) { + if (log.isDebugEnabled()) { + log.debug("[{}] [{}] Failed to create producer: {}", + topic, producerName, cause.getMessage()); + } + } else { + log.error("[{}] [{}] Failed to create producer: {}", topic, producerName, cause.getMessage()); + } + // Close the producer since topic does not exist. + if (cause instanceof PulsarClientException.TopicDoesNotExistException) { + closeAsync().whenComplete((v, ex) -> { + if (ex != null) { + log.error("Failed to close producer on TopicDoesNotExistException.", ex); } + producerCreatedFuture.completeExceptionally(cause); + }); + future.complete(null); + return null; + } + if (cause instanceof PulsarClientException.ProducerBlockedQuotaExceededException) { + synchronized (this) { + log.warn("[{}] [{}] Topic backlog quota exceeded. Throwing Exception on producer.", topic, + producerName); - if (cause instanceof PulsarClientException.TopicTerminatedException) { - setState(State.Terminated); - synchronized (this) { - failPendingMessages(cnx(), (PulsarClientException) cause); - } - producerCreatedFuture.completeExceptionally(cause); - closeProducerTasks(); - client.cleanupProducer(this); - } else if (cause instanceof PulsarClientException.ProducerFencedException) { - setState(State.ProducerFenced); - synchronized (this) { - failPendingMessages(cnx(), (PulsarClientException) cause); - } - producerCreatedFuture.completeExceptionally(cause); - closeProducerTasks(); - client.cleanupProducer(this); - } else if (producerCreatedFuture.isDone() || // - (cause instanceof PulsarClientException && PulsarClientException.isRetriableError(cause) - && System.currentTimeMillis() < PRODUCER_DEADLINE_UPDATER.get(ProducerImpl.this))) { - // Either we had already created the producer once (producerCreatedFuture.isDone()) or we are - // still within the initial timeout budget and we are dealing with a retriable error - reconnectLater(cause); - } else { - setState(State.Failed); - producerCreatedFuture.completeExceptionally(cause); - closeProducerTasks(); - client.cleanupProducer(this); - Timeout timeout = sendTimeout; - if (timeout != null) { - timeout.cancel(); - sendTimeout = null; - } + if (log.isDebugEnabled()) { + log.debug("[{}] [{}] Pending messages: {}", topic, producerName, + pendingMessages.messagesCount()); } - return null; - }); + PulsarClientException bqe = new PulsarClientException.ProducerBlockedQuotaExceededException( + format("The backlog quota of the topic %s that the producer %s produces to is exceeded", + topic, producerName)); + failPendingMessages(cnx(), bqe); + } + } else if (cause instanceof PulsarClientException.ProducerBlockedQuotaExceededError) { + log.warn("[{}] [{}] Producer is blocked on creation because backlog exceeded on topic.", + producerName, topic); + } + + if (cause instanceof PulsarClientException.TopicTerminatedException) { + setState(State.Terminated); + synchronized (this) { + failPendingMessages(cnx(), (PulsarClientException) cause); + } + producerCreatedFuture.completeExceptionally(cause); + closeProducerTasks(); + client.cleanupProducer(this); + } else if (cause instanceof PulsarClientException.ProducerFencedException) { + setState(State.ProducerFenced); + synchronized (this) { + failPendingMessages(cnx(), (PulsarClientException) cause); + } + producerCreatedFuture.completeExceptionally(cause); + closeProducerTasks(); + client.cleanupProducer(this); + } else if (producerCreatedFuture.isDone() || ( + cause instanceof PulsarClientException + && PulsarClientException.isRetriableError(cause) + && System.currentTimeMillis() < PRODUCER_DEADLINE_UPDATER.get(ProducerImpl.this) + )) { + // Either we had already created the producer once (producerCreatedFuture.isDone()) or we are + // still within the initial timeout budget and we are dealing with a retriable error + future.completeExceptionally(cause); + } else { + setState(State.Failed); + producerCreatedFuture.completeExceptionally(cause); + closeProducerTasks(); + client.cleanupProducer(this); + Timeout timeout = sendTimeout; + if (timeout != null) { + timeout.cancel(); + sendTimeout = null; + } + } + if (!future.isDone()) { + future.complete(null); + } + return null; + }); + return future; } @Override @@ -1844,11 +1954,11 @@ public void connectionFailed(PulsarClientException exception) { boolean nonRetriableError = !PulsarClientException.isRetriableError(exception); boolean timeout = System.currentTimeMillis() > lookupDeadline; if (nonRetriableError || timeout) { - exception.setPreviousExceptions(previousExceptions); + exception.setPreviousExceptionCount(previousExceptionCount); if (producerCreatedFuture.completeExceptionally(exception)) { if (nonRetriableError) { log.info("[{}] Producer creation failed for producer {} with unretriableError = {}", - topic, producerId, exception); + topic, producerId, exception.getMessage()); } else { log.info("[{}] Producer creation failed for producer {} after producerTimeout", topic, producerId); } @@ -1857,7 +1967,7 @@ public void connectionFailed(PulsarClientException exception) { client.cleanupProducer(this); } } else { - previousExceptions.add(exception); + previousExceptionCount.incrementAndGet(); } } @@ -1941,7 +2051,7 @@ private void stripChecksum(OpSendMsg op) { headerFrame.setInt(0, newTotalFrameSizeLength); // rewrite new [total-size] ByteBuf metadata = headerFrame.slice(checksumMark, headerFrameSize - checksumMark); // sliced only - // metadata + // metadata headerFrame.writerIndex(headerSize); // set headerFrame write-index to overwrite metadata over checksum metadata.readBytes(headerFrame, metadata.readableBytes()); headerFrame.capacity(headerFrameSize - checksumSize); // reduce capacity by removed checksum bytes @@ -1962,6 +2072,11 @@ String getHandlerName() { return producerName; } + @VisibleForTesting + void triggerSendTimer() throws Exception { + run(sendTimeout); + } + /** * Process sendTimeout events. */ @@ -1980,7 +2095,8 @@ public void run(Timeout timeout) throws Exception { } OpSendMsg firstMsg = pendingMessages.peek(); - if (firstMsg == null && (batchMessageContainer == null || batchMessageContainer.isEmpty())) { + if (firstMsg == null && (batchMessageContainer == null || batchMessageContainer.isEmpty() + || batchMessageContainer.getFirstAddedTimestamp() == 0L)) { // If there are no pending messages, reset the timeout to the configured value. timeToWaitMs = conf.getSendTimeoutMs(); } else { @@ -1990,7 +2106,7 @@ public void run(Timeout timeout) throws Exception { } else { // Because we don't flush batch messages while disconnected, we consider them "createdAt" when // they would have otherwise been flushed. - createdAt = lastBatchSendNanoTime + createdAt = batchMessageContainer.getFirstAddedTimestamp() + TimeUnit.MICROSECONDS.toNanos(conf.getBatchingMaxPublishDelayMicros()); } // If there is at least one message, calculate the diff between the message timeout and the elapsed @@ -2047,6 +2163,7 @@ private void failPendingMessages(ClientCnx cnx, PulsarClientException ex) { log.warn("[{}] [{}] Got exception while completing the callback for msg {}:", topic, producerName, op.sequenceId, t); } + client.getMemoryLimitController().releaseMemory(op.uncompressedSize); ReferenceCountUtil.safeRelease(op.cmd); op.recycle(); @@ -2071,7 +2188,6 @@ private void failPendingMessages(ClientCnx cnx, PulsarClientException ex) { /** * fail any pending batch messages that were enqueued, however batch was not closed out. - * */ private void failPendingBatchMessages(PulsarClientException ex) { if (batchMessageContainer.isEmpty()) { @@ -2091,7 +2207,7 @@ public CompletableFuture flushAsync() { if (isBatchMessagingEnabled()) { batchMessageAndSend(false); } - CompletableFuture lastSendFuture = this.lastSendFuture; + CompletableFuture lastSendFuture = this.lastSendFuture; if (!(lastSendFuture == this.lastSendFutureWrapper.lastSendFuture)) { this.lastSendFutureWrapper = LastSendFutureWrapper.create(lastSendFuture); } @@ -2210,7 +2326,7 @@ protected void processOpSendMsg(OpSendMsg op) { } else { if (log.isDebugEnabled()) { log.debug("[{}] [{}] Connection is not ready -- sequenceId {}", topic, producerName, - op.sequenceId); + op.sequenceId); } } } catch (Throwable t) { @@ -2226,7 +2342,8 @@ private void recoverProcessOpSendMsgFrom(ClientCnx cnx, MessageImpl from, long e // In this case, the cnx passed to this method is no longer the active connection. This method will get // called again once the new connection registers the producer with the broker. log.info("[{}][{}] Producer epoch mismatch or the current connection is null. Skip re-sending the " - + " {} pending messages since they will deliver using another connection.", topic, producerName, + + " {} pending messages since they will deliver using another connection.", topic, + producerName, pendingMessages.messagesCount()); return; } @@ -2267,7 +2384,7 @@ private void recoverProcessOpSendMsgFrom(ClientCnx cnx, MessageImpl from, long e op.cmd.retain(); if (log.isDebugEnabled()) { log.debug("[{}] [{}] Re-Sending message in cnx {}, sequenceId {}", topic, producerName, - cnx.channel(), op.sequenceId); + cnx.channel(), op.sequenceId); } cnx.ctx().write(op.cmd, cnx.ctx().voidPromise()); op.updateSentTimestamp(); @@ -2291,16 +2408,16 @@ private void recoverProcessOpSendMsgFrom(ClientCnx cnx, MessageImpl from, long e } /** - * Check if final message size for non-batch and non-chunked messages is larger than max message size. + * Check if final message size for non-batch and non-chunked messages is larger than max message size. */ private boolean isMessageSizeExceeded(OpSendMsg op) { if (op.msg != null && !conf.isChunkingEnabled()) { int messageSize = op.getMessageHeaderAndPayloadSize(); - if (messageSize > ClientCnx.getMaxMessageSize()) { + if (messageSize > getMaxMessageSize()) { releaseSemaphoreForSendOp(op); op.sendComplete(new PulsarClientException.InvalidMessageException( format("The producer %s of the topic %s sends a message with %d bytes that exceeds %d bytes", - producerName, topic, messageSize, ClientCnx.getMaxMessageSize()), + producerName, topic, messageSize, getMaxMessageSize()), op.sequenceId)); return true; } @@ -2308,6 +2425,10 @@ private boolean isMessageSizeExceeded(OpSendMsg op) { return false; } + private int getMaxMessageSize() { + return getConnectionHandler().getMaxMessageSize(); + } + public long getDelayInMillis() { OpSendMsg firstMsg = pendingMessages.peek(); if (firstMsg != null) { @@ -2352,8 +2473,8 @@ void resetBackoff() { this.connectionHandler.resetBackoff(); } - void connectionClosed(ClientCnx cnx) { - this.connectionHandler.connectionClosed(cnx); + void connectionClosed(ClientCnx cnx, Optional initialConnectionDelayMs, Optional hostUrl) { + this.connectionHandler.connectionClosed(cnx, initialConnectionDelayMs, hostUrl); } public ClientCnx getClientCnx() { @@ -2364,10 +2485,6 @@ void setClientCnx(ClientCnx clientCnx) { this.connectionHandler.setClientCnx(clientCnx); } - void reconnectLater(Throwable exception) { - this.connectionHandler.reconnectLater(exception); - } - void grabCnx() { this.connectionHandler.grabCnx(); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java index ed34f7d41c130..5097c34e0b2fd 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java @@ -19,30 +19,28 @@ package org.apache.pulsar.client.impl; import io.netty.channel.Channel; +import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelInitializer; import io.netty.channel.socket.SocketChannel; import io.netty.handler.codec.LengthFieldBasedFrameDecoder; import io.netty.handler.flush.FlushConsolidationHandler; import io.netty.handler.proxy.Socks5ProxyHandler; -import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslHandler; -import io.netty.handler.ssl.SslProvider; import java.net.InetSocketAddress; import java.util.Objects; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.client.api.AuthenticationDataProvider; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; -import org.apache.pulsar.client.util.ObjectCache; import org.apache.pulsar.common.protocol.ByteBufPair; import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.apache.pulsar.common.util.SecurityUtility; -import org.apache.pulsar.common.util.keystoretls.NettySSLContextAutoRefreshBuilder; import org.apache.pulsar.common.util.netty.NettyFutureUtil; @Slf4j @@ -54,18 +52,16 @@ public class PulsarChannelInitializer extends ChannelInitializer @Getter private final boolean tlsEnabled; private final boolean tlsHostnameVerificationEnabled; - private final boolean tlsEnabledWithKeyStore; private final InetSocketAddress socks5ProxyAddress; private final String socks5ProxyUsername; private final String socks5ProxyPassword; - private final Supplier sslContextSupplier; - private NettySSLContextAutoRefreshBuilder nettySSLContextAutoRefreshBuilder; + private final PulsarSslFactory pulsarSslFactory; private static final long TLS_CERTIFICATE_CACHE_MILLIS = TimeUnit.MINUTES.toMillis(1); - public PulsarChannelInitializer(ClientConfigurationData conf, Supplier clientCnxSupplier) - throws Exception { + public PulsarChannelInitializer(ClientConfigurationData conf, Supplier clientCnxSupplier, + ScheduledExecutorService scheduledExecutorService) throws Exception { super(); this.clientCnxSupplier = clientCnxSupplier; this.tlsEnabled = conf.isUseTls(); @@ -74,71 +70,25 @@ public PulsarChannelInitializer(ClientConfigurationData conf, Supplier 0) { + scheduledExecutorService.scheduleWithFixedDelay(() -> this.refreshSslContext(conf), + conf.getAutoCertRefreshSeconds(), + conf.getAutoCertRefreshSeconds(), + TimeUnit.SECONDS); } - sslContextSupplier = new ObjectCache(() -> { - try { - SslProvider sslProvider = null; - if (conf.getSslProvider() != null) { - sslProvider = SslProvider.valueOf(conf.getSslProvider()); - } - - // Set client certificate if available - AuthenticationDataProvider authData = conf.getAuthentication().getAuthData(); - if (authData.hasDataForTls()) { - return authData.getTlsTrustStoreStream() == null - ? SecurityUtility.createNettySslContextForClient( - sslProvider, - conf.isTlsAllowInsecureConnection(), - conf.getTlsTrustCertsFilePath(), - authData.getTlsCertificates(), - authData.getTlsPrivateKey(), - conf.getTlsCiphers(), - conf.getTlsProtocols()) - : SecurityUtility.createNettySslContextForClient(sslProvider, - conf.isTlsAllowInsecureConnection(), - authData.getTlsTrustStoreStream(), - authData.getTlsCertificates(), authData.getTlsPrivateKey(), - conf.getTlsCiphers(), - conf.getTlsProtocols()); - } else { - return SecurityUtility.createNettySslContextForClient( - sslProvider, - conf.isTlsAllowInsecureConnection(), - conf.getTlsTrustCertsFilePath(), - conf.getTlsCertificateFilePath(), - conf.getTlsKeyFilePath(), - conf.getTlsCiphers(), - conf.getTlsProtocols()); - } - } catch (Exception e) { - throw new RuntimeException("Failed to create TLS context", e); - } - }, TLS_CERTIFICATE_CACHE_MILLIS, TimeUnit.MILLISECONDS); } else { - sslContextSupplier = null; + pulsarSslFactory = null; } } @@ -147,11 +97,12 @@ public void initChannel(SocketChannel ch) throws Exception { ch.pipeline().addLast("consolidation", new FlushConsolidationHandler(1024, true)); // Setup channel except for the SsHandler for TLS enabled connections - ch.pipeline().addLast("ByteBufPairEncoder", tlsEnabled ? ByteBufPair.COPYING_ENCODER : ByteBufPair.ENCODER); + ch.pipeline().addLast("ByteBufPairEncoder", ByteBufPair.getEncoder(tlsEnabled)); ch.pipeline().addLast("frameDecoder", new LengthFieldBasedFrameDecoder( Commands.DEFAULT_MAX_MESSAGE_SIZE + Commands.MESSAGE_SIZE_FRAME_PADDING, 0, 4, 0, 4)); - ch.pipeline().addLast("handler", clientCnxSupplier.get()); + ChannelHandler clientCnx = clientCnxSupplier.get(); + ch.pipeline().addLast("handler", clientCnx); } /** @@ -172,10 +123,8 @@ CompletableFuture initTls(Channel ch, InetSocketAddress sniHost) { CompletableFuture initTlsFuture = new CompletableFuture<>(); ch.eventLoop().execute(() -> { try { - SslHandler handler = tlsEnabledWithKeyStore - ? new SslHandler(nettySSLContextAutoRefreshBuilder.get() - .createSSLEngine(sniHost.getHostString(), sniHost.getPort())) - : sslContextSupplier.get().newHandler(ch.alloc(), sniHost.getHostString(), sniHost.getPort()); + SslHandler handler = new SslHandler(pulsarSslFactory + .createClientSslEngine(ch.alloc(), sniHost.getHostName(), sniHost.getPort())); if (tlsHostnameVerificationEnabled) { SecurityUtility.configureSSLHandler(handler); @@ -232,5 +181,48 @@ CompletableFuture initializeClientCnx(Channel ch, return ch; })); } + protected PulsarSslConfiguration buildSslConfiguration(ClientConfigurationData config) + throws PulsarClientException { + return PulsarSslConfiguration.builder() + .tlsProvider(config.getSslProvider()) + .tlsKeyStoreType(config.getTlsKeyStoreType()) + .tlsKeyStorePath(config.getTlsKeyStorePath()) + .tlsKeyStorePassword(config.getTlsKeyStorePassword()) + .tlsTrustStoreType(config.getTlsTrustStoreType()) + .tlsTrustStorePath(config.getTlsTrustStorePath()) + .tlsTrustStorePassword(config.getTlsTrustStorePassword()) + .tlsCiphers(config.getTlsCiphers()) + .tlsProtocols(config.getTlsProtocols()) + .tlsTrustCertsFilePath(config.getTlsTrustCertsFilePath()) + .tlsCertificateFilePath(config.getTlsCertificateFilePath()) + .tlsKeyFilePath(config.getTlsKeyFilePath()) + .allowInsecureConnection(config.isTlsAllowInsecureConnection()) + .requireTrustedClientCertOnConnect(false) + .tlsEnabledWithKeystore(config.isUseKeyStoreTls()) + .tlsCustomParams(config.getSslFactoryPluginParams()) + .authData(config.getAuthentication().getAuthData()) + .serverMode(false) + .build(); + } + + protected void refreshSslContext(ClientConfigurationData conf) { + try { + try { + if (conf.isUseKeyStoreTls()) { + this.pulsarSslFactory.getInternalSslContext(); + } else { + this.pulsarSslFactory.getInternalNettySslContext(); + } + } catch (Exception e) { + log.error("SSL Context is not initialized", e); + PulsarSslConfiguration sslConfiguration = buildSslConfiguration(conf); + this.pulsarSslFactory.initialize(sslConfiguration); + } + this.pulsarSslFactory.update(); + } catch (Exception e) { + log.error("Failed to refresh SSL context", e); + } + } + } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java index 6c749a8cf4354..e0d4bf35f8a22 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java @@ -23,6 +23,7 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +import com.google.re2j.Pattern; import io.netty.channel.EventLoopGroup; import io.netty.util.HashedWheelTimer; import io.netty.util.Timer; @@ -32,7 +33,9 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -44,10 +47,14 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import javax.annotation.Nullable; import lombok.Builder; import lombok.Getter; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.ConsumerBuilder; @@ -68,6 +75,7 @@ import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; import org.apache.pulsar.client.impl.conf.ReaderConfigurationData; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; import org.apache.pulsar.client.impl.schema.AutoProduceBytesSchema; import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; @@ -84,6 +92,8 @@ import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.topics.TopicList; +import org.apache.pulsar.common.util.Backoff; +import org.apache.pulsar.common.util.BackoffBuilder; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.netty.EventLoopUtil; import org.slf4j.Logger; @@ -104,6 +114,7 @@ public class PulsarClientImpl implements PulsarClient { private final boolean createdScheduledProviders; private LookupService lookup; + private Map urlLookupMap = new ConcurrentHashMap<>(); private final ConnectionPool cnxPool; @Getter private final Timer timer; @@ -146,6 +157,8 @@ public SchemaInfoProvider load(String topicName) { private final Clock clientClock; + private final InstrumentProvider instrumentProvider; + @Getter private TransactionCoordinatorClientImpl tcClient; @@ -173,6 +186,7 @@ private PulsarClientImpl(ClientConfigurationData conf, EventLoopGroup eventLoopG Timer timer, ExecutorProvider externalExecutorProvider, ExecutorProvider internalExecutorProvider, ScheduledExecutorProvider scheduledExecutorProvider) throws PulsarClientException { + EventLoopGroup eventLoopGroupReference = null; ConnectionPool connectionPoolReference = null; try { @@ -190,19 +204,22 @@ private PulsarClientImpl(ClientConfigurationData conf, EventLoopGroup eventLoopG throw new PulsarClientException.InvalidConfigurationException("Invalid client configuration"); } this.conf = conf; + this.instrumentProvider = new InstrumentProvider(conf.getOpenTelemetry()); clientClock = conf.getClock(); conf.getAuthentication().start(); + this.scheduledExecutorProvider = scheduledExecutorProvider != null ? scheduledExecutorProvider : + new ScheduledExecutorProvider(conf.getNumIoThreads(), "pulsar-client-scheduled"); connectionPoolReference = - connectionPool != null ? connectionPool : new ConnectionPool(conf, this.eventLoopGroup); + connectionPool != null ? connectionPool : + new ConnectionPool(instrumentProvider, conf, this.eventLoopGroup, + (ScheduledExecutorService) this.scheduledExecutorProvider.getExecutor()); this.cnxPool = connectionPoolReference; this.externalExecutorProvider = externalExecutorProvider != null ? externalExecutorProvider : new ExecutorProvider(conf.getNumListenerThreads(), "pulsar-external-listener"); this.internalExecutorProvider = internalExecutorProvider != null ? internalExecutorProvider : new ExecutorProvider(conf.getNumIoThreads(), "pulsar-client-internal"); - this.scheduledExecutorProvider = scheduledExecutorProvider != null ? scheduledExecutorProvider : - new ScheduledExecutorProvider(conf.getNumIoThreads(), "pulsar-client-scheduled"); if (conf.getServiceUrl().startsWith("http")) { - lookup = new HttpLookupService(conf, this.eventLoopGroup); + lookup = new HttpLookupService(instrumentProvider, conf, this.eventLoopGroup); } else { lookup = new BinaryProtoLookupService(this, conf.getServiceUrl(), conf.getListenerName(), conf.isUseTls(), this.scheduledExecutorProvider.getExecutor()); @@ -369,26 +386,55 @@ public CompletableFuture> createProducerAsync(ProducerConfigurat } + private CompletableFuture checkPartitions(String topic, boolean forceNoPartitioned, + @Nullable String producerNameForLog) { + CompletableFuture checkPartitions = new CompletableFuture<>(); + getPartitionedTopicMetadata(topic, !forceNoPartitioned, true).thenAccept(metadata -> { + if (forceNoPartitioned && metadata.partitions > 0) { + String errorMsg = String.format("Can not create the producer[%s] for the topic[%s] that contains %s" + + " partitions b,ut the producer does not support for a partitioned topic.", + producerNameForLog, topic, metadata.partitions); + log.error(errorMsg); + checkPartitions.completeExceptionally( + new PulsarClientException.NotConnectedException(errorMsg)); + } else { + checkPartitions.complete(metadata.partitions); + } + }).exceptionally(ex -> { + Throwable actEx = FutureUtil.unwrapCompletionException(ex); + if (forceNoPartitioned && actEx instanceof PulsarClientException.NotFoundException + || actEx instanceof PulsarClientException.TopicDoesNotExistException + || actEx instanceof PulsarAdminException.NotFoundException) { + checkPartitions.complete(0); + } else { + checkPartitions.completeExceptionally(ex); + } + return null; + }); + return checkPartitions; + } + private CompletableFuture> createProducerAsync(String topic, ProducerConfigurationData conf, Schema schema, ProducerInterceptors interceptors) { CompletableFuture> producerCreatedFuture = new CompletableFuture<>(); - getPartitionedTopicMetadata(topic).thenAccept(metadata -> { + + + checkPartitions(topic, conf.isNonPartitionedTopicExpected(), conf.getProducerName()).thenAccept(partitions -> { if (log.isDebugEnabled()) { - log.debug("[{}] Received topic metadata. partitions: {}", topic, metadata.partitions); + log.debug("[{}] Received topic metadata. partitions: {}", topic, partitions); } ProducerBase producer; - if (metadata.partitions > 0) { + if (partitions > 0) { producer = newPartitionedProducerImpl(topic, conf, schema, interceptors, producerCreatedFuture, - metadata); + partitions); } else { producer = newProducerImpl(topic, -1, conf, schema, interceptors, producerCreatedFuture, Optional.empty()); } - producers.add(producer); }).exceptionally(ex -> { log.warn("[{}] Failed to get partitioned topic metadata: {}", topic, ex.getMessage()); @@ -409,7 +455,6 @@ private CompletableFuture> createProducerAsync(String topic, * @param schema topic schema * @param interceptors producer interceptors * @param producerCreatedFuture future for signaling completion of async producer creation - * @param metadata partitioned topic metadata * @param message type class * @return new PartitionedProducerImpl instance */ @@ -419,8 +464,8 @@ protected PartitionedProducerImpl newPartitionedProducerImpl(String topic ProducerInterceptors interceptors, CompletableFuture> producerCreatedFuture, - PartitionedTopicMetadata metadata) { - return new PartitionedProducerImpl<>(PulsarClientImpl.this, topic, conf, metadata.partitions, + int partitions) { + return new PartitionedProducerImpl<>(PulsarClientImpl.this, topic, conf, partitions, producerCreatedFuture, schema, interceptors); } @@ -517,7 +562,7 @@ private CompletableFuture> doSingleTopicSubscribeAsync(ConsumerC String topic = conf.getSingleTopic(); - getPartitionedTopicMetadata(topic).thenAccept(metadata -> { + getPartitionedTopicMetadata(topic, true, false).thenAccept(metadata -> { if (log.isDebugEnabled()) { log.debug("[{}] Received topic metadata. partitions: {}", topic, metadata.partitions); } @@ -566,25 +611,35 @@ private CompletableFuture> patternTopicSubscribeAsync(ConsumerCo Mode subscriptionMode = convertRegexSubscriptionMode(conf.getRegexSubscriptionMode()); TopicName destination = TopicName.get(regex); NamespaceName namespaceName = destination.getNamespaceObject(); + Pattern pattern = Pattern.compile(conf.getTopicsPattern().pattern()); CompletableFuture> consumerSubscribedFuture = new CompletableFuture<>(); lookup.getTopicsUnderNamespace(namespaceName, subscriptionMode, regex, null) .thenAccept(getTopicsResult -> { if (log.isDebugEnabled()) { - log.debug("Get topics under namespace {}, topics.size: {}," - + " topicsHash: {}, changed: {}, filtered: {}", + log.debug("Pattern consumer [{}] get topics under namespace {}, topics.size: {}," + + " topicsHash: {}, changed: {}, filtered: {}", conf.getSubscriptionName(), namespaceName, getTopicsResult.getTopics().size(), getTopicsResult.getTopicsHash(), getTopicsResult.isChanged(), getTopicsResult.isFiltered()); getTopicsResult.getTopics().forEach(topicName -> - log.debug("Get topics under namespace {}, topic: {}", namespaceName, topicName)); + log.debug("Pattern consumer [{}] get topics under namespace {}, topic: {}", + conf.getSubscriptionName(), namespaceName, topicName)); } List topicsList = getTopicsResult.getTopics(); if (!getTopicsResult.isFiltered()) { - topicsList = TopicList.filterTopics(getTopicsResult.getTopics(), conf.getTopicsPattern()); + topicsList = TopicList.filterTopics(getTopicsResult.getTopics(), pattern); } conf.getTopicNames().addAll(topicsList); - ConsumerBase consumer = new PatternMultiTopicsConsumerImpl<>(conf.getTopicsPattern(), + + if (log.isDebugEnabled()) { + log.debug("Pattern consumer [{}] initialize topics. {}", conf.getSubscriptionName(), + getTopicsResult.getNonPartitionedOrPartitionTopics()); + } + + // Pattern consumer has his unique check mechanism, so do not need the feature "autoUpdatePartitions". + conf.setAutoUpdatePartitions(false); + ConsumerBase consumer = new PatternMultiTopicsConsumerImpl<>(pattern, getTopicsResult.getTopicsHash(), PulsarClientImpl.this, conf, @@ -657,7 +712,7 @@ protected CompletableFuture> createSingleTopicReaderAsync( CompletableFuture> readerFuture = new CompletableFuture<>(); - getPartitionedTopicMetadata(topic).thenAccept(metadata -> { + getPartitionedTopicMetadata(topic, true, false).thenAccept(metadata -> { if (log.isDebugEnabled()) { log.debug("[{}] Received topic metadata. partitions: {}", topic, metadata.partitions); } @@ -731,6 +786,21 @@ public void close() throws PulsarClientException { } } + private void closeUrlLookupMap() { + Map closedUrlLookupServices = new HashMap(urlLookupMap.size()); + urlLookupMap.entrySet().forEach(e -> { + try { + e.getValue().close(); + } catch (Exception ex) { + log.error("Error closing lookup service {}", e.getKey(), ex); + } + closedUrlLookupServices.put(e.getKey(), e.getValue()); + }); + closedUrlLookupServices.entrySet().forEach(e -> { + urlLookupMap.remove(e.getKey(), e.getValue()); + }); + } + @Override public CompletableFuture closeAsync() { log.info("Client closing. URL: {}", lookup.getServiceUrl()); @@ -741,6 +811,8 @@ public CompletableFuture closeAsync() { final CompletableFuture closeFuture = new CompletableFuture<>(); List> futures = new ArrayList<>(); + closeUrlLookupMap(); + producers.forEach(p -> futures.add(p.closeAsync().handle((__, t) -> { if (t != null) { log.error("Error closing producer {}", p, t); @@ -944,10 +1016,42 @@ public void updateTlsTrustStorePathAndPassword(String tlsTrustStorePath, String conf.setTlsTrustStorePassword(tlsTrustStorePassword); } + public CompletableFuture> getConnection(String topic, int randomKeyForSelectConnection) { + CompletableFuture lookupTopicResult = lookup.getBroker(TopicName.get(topic)); + CompletableFuture isUseProxy = lookupTopicResult.thenApply(LookupTopicResult::isUseProxy); + return lookupTopicResult.thenCompose(lookupResult -> getConnection(lookupResult.getLogicalAddress(), + lookupResult.getPhysicalAddress(), randomKeyForSelectConnection)). + thenCombine(isUseProxy, Pair::of); + } + + /** + * Only for test. + */ + @VisibleForTesting public CompletableFuture getConnection(final String topic) { + return getConnection(topic, cnxPool.genRandomKeyToSelectCon()).thenApply(Pair::getLeft); + } + + public CompletableFuture getConnection(final String topic, final String url) { TopicName topicName = TopicName.get(topic); - return lookup.getBroker(topicName) - .thenCompose(pair -> getConnection(pair.getLeft(), pair.getRight())); + return getLookup(url).getBroker(topicName) + .thenCompose(lookupResult -> getConnection(lookupResult.getLogicalAddress(), + lookupResult.getPhysicalAddress(), cnxPool.genRandomKeyToSelectCon())); + } + + public LookupService getLookup(String serviceUrl) { + return urlLookupMap.computeIfAbsent(serviceUrl, url -> { + if (isClosed()) { + throw new IllegalStateException("Pulsar client has been closed, can not build LookupService when" + + " calling get lookup with an url"); + } + try { + return createLookup(serviceUrl); + } catch (PulsarClientException e) { + log.warn("Failed to update url to lookup service {}, {}", url, e.getMessage()); + throw new IllegalStateException("Failed to update url " + url); + } + }); } public CompletableFuture getConnectionToServiceUrl() { @@ -956,12 +1060,22 @@ public CompletableFuture getConnectionToServiceUrl() { "Can't get client connection to HTTP service URL", null)); } InetSocketAddress address = lookup.resolveHost(); - return getConnection(address, address); + return getConnection(address, address, cnxPool.genRandomKeyToSelectCon()); + } + + public CompletableFuture getProxyConnection(final InetSocketAddress logicalAddress, + final int randomKeyForSelectConnection) { + if (!(lookup instanceof BinaryProtoLookupService)) { + return FutureUtil.failedFuture(new PulsarClientException.InvalidServiceURL( + "Cannot proxy connection through HTTP service URL", null)); + } + return getConnection(logicalAddress, lookup.resolveHost(), randomKeyForSelectConnection); } public CompletableFuture getConnection(final InetSocketAddress logicalAddress, - final InetSocketAddress physicalAddress) { - return cnxPool.getConnection(logicalAddress, physicalAddress); + final InetSocketAddress physicalAddress, + final int randomKeyForSelectConnection) { + return cnxPool.getConnection(logicalAddress, physicalAddress, randomKeyForSelectConnection); } /** visible for pulsar-functions. **/ @@ -1007,19 +1121,27 @@ public LookupService getLookup() { } public void reloadLookUp() throws PulsarClientException { - if (conf.getServiceUrl().startsWith("http")) { - lookup = new HttpLookupService(conf, eventLoopGroup); + lookup = createLookup(conf.getServiceUrl()); + } + + public LookupService createLookup(String url) throws PulsarClientException { + if (url.startsWith("http")) { + return new HttpLookupService(instrumentProvider, conf, eventLoopGroup); } else { - lookup = new BinaryProtoLookupService(this, conf.getServiceUrl(), conf.getListenerName(), conf.isUseTls(), + return new BinaryProtoLookupService(this, url, conf.getListenerName(), conf.isUseTls(), externalExecutorProvider.getExecutor()); } } - public CompletableFuture getNumberOfPartitions(String topic) { - return getPartitionedTopicMetadata(topic).thenApply(metadata -> metadata.partitions); - } - - public CompletableFuture getPartitionedTopicMetadata(String topic) { + /** + * @param useFallbackForNonPIP344Brokers

      If true, fallback to the prior behavior of the method + * getPartitionedTopicMetadata if the broker does not support the PIP-344 + * feature 'supports_get_partitioned_metadata_without_auto_creation'. This + * parameter only affects the behavior when + * {@param metadataAutoCreationEnabled} is false.

      + */ + public CompletableFuture getPartitionedTopicMetadata( + String topic, boolean metadataAutoCreationEnabled, boolean useFallbackForNonPIP344Brokers) { CompletableFuture metadataFuture = new CompletableFuture<>(); @@ -1031,8 +1153,9 @@ public CompletableFuture getPartitionedTopicMetadata(S .setMandatoryStop(opTimeoutMs.get() * 2, TimeUnit.MILLISECONDS) .setMax(conf.getMaxBackoffIntervalNanos(), TimeUnit.NANOSECONDS) .create(); - getPartitionedTopicMetadata(topicName, backoff, opTimeoutMs, - metadataFuture, new ArrayList<>()); + getPartitionedTopicMetadata(topicName, backoff, opTimeoutMs, metadataFuture, + new AtomicInteger(0), + metadataAutoCreationEnabled, useFallbackForNonPIP344Brokers); } catch (IllegalArgumentException e) { return FutureUtil.failedFuture(new PulsarClientException.InvalidConfigurationException(e.getMessage())); } @@ -1043,35 +1166,41 @@ private void getPartitionedTopicMetadata(TopicName topicName, Backoff backoff, AtomicLong remainingTime, CompletableFuture future, - List previousExceptions) { + AtomicInteger previousExceptionCount, + boolean metadataAutoCreationEnabled, + boolean useFallbackForNonPIP344Brokers) { long startTime = System.nanoTime(); - lookup.getPartitionedTopicMetadata(topicName).thenAccept(future::complete).exceptionally(e -> { + CompletableFuture queryFuture = lookup.getPartitionedTopicMetadata(topicName, + metadataAutoCreationEnabled, useFallbackForNonPIP344Brokers); + queryFuture.thenAccept(future::complete).exceptionally(e -> { remainingTime.addAndGet(-1 * TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime)); long nextDelay = Math.min(backoff.next(), remainingTime.get()); // skip retry scheduler when set lookup throttle in client or server side which will lead to // `TooManyRequestsException` boolean isLookupThrottling = !PulsarClientException.isRetriableError(e.getCause()) - || e.getCause() instanceof PulsarClientException.AuthenticationException; + || e.getCause() instanceof PulsarClientException.AuthenticationException + || e.getCause() instanceof PulsarClientException.NotFoundException; if (nextDelay <= 0 || isLookupThrottling) { - PulsarClientException.setPreviousExceptions(e, previousExceptions); + PulsarClientException.setPreviousExceptionCount(e, previousExceptionCount); future.completeExceptionally(e); return null; } - previousExceptions.add(e); + previousExceptionCount.getAndIncrement(); ((ScheduledExecutorService) scheduledExecutorProvider.getExecutor()).schedule(() -> { log.warn("[topic: {}] Could not get connection while getPartitionedTopicMetadata -- " + "Will try again in {} ms", topicName, nextDelay); remainingTime.addAndGet(-nextDelay); - getPartitionedTopicMetadata(topicName, backoff, remainingTime, future, previousExceptions); + getPartitionedTopicMetadata(topicName, backoff, remainingTime, future, previousExceptionCount, + metadataAutoCreationEnabled, useFallbackForNonPIP344Brokers); }, nextDelay, TimeUnit.MILLISECONDS); return null; }); } @Override - public CompletableFuture> getPartitionsForTopic(String topic) { - return getPartitionedTopicMetadata(topic).thenApply(metadata -> { + public CompletableFuture> getPartitionsForTopic(String topic, boolean metadataAutoCreationEnabled) { + return getPartitionedTopicMetadata(topic, metadataAutoCreationEnabled, false).thenApply(metadata -> { if (metadata.partitions > 0) { TopicName topicName = TopicName.get(topic); List partitions = new ArrayList<>(metadata.partitions); @@ -1186,6 +1315,11 @@ public ScheduledExecutorProvider getScheduledExecutorProvider() { return scheduledExecutorProvider; } + InstrumentProvider instrumentProvider() { + return instrumentProvider; + } + + // // Transaction related API // diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderBuilderImpl.java index ca2011cf18a42..ef230475be53b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderBuilderImpl.java @@ -32,6 +32,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.ConsumerCryptoFailureAction; import org.apache.pulsar.client.api.CryptoKeyReader; +import org.apache.pulsar.client.api.MessageCrypto; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Range; @@ -85,8 +86,9 @@ public CompletableFuture> createAsync() { .failedFuture(new IllegalArgumentException("Topic name must be set on the reader builder")); } - if (conf.getStartMessageId() != null && conf.getStartMessageFromRollbackDurationInSec() > 0 - || conf.getStartMessageId() == null && conf.getStartMessageFromRollbackDurationInSec() <= 0) { + boolean isStartMsgIdExist = conf.getStartMessageId() != null && conf.getStartMessageId() != MessageId.earliest; + if ((isStartMsgIdExist && conf.getStartMessageFromRollbackDurationInSec() > 0) + || (conf.getStartMessageId() == null && conf.getStartMessageFromRollbackDurationInSec() <= 0)) { return FutureUtil .failedFuture(new IllegalArgumentException( "Start message id or start message from roll back must be specified but they cannot be" @@ -173,6 +175,12 @@ public ReaderBuilder cryptoFailureAction(ConsumerCryptoFailureAction action) return this; } + @Override + public ReaderBuilder messageCrypto(MessageCrypto messageCrypto) { + conf.setMessageCrypto(messageCrypto); + return this; + } + @Override public ReaderBuilder receiverQueueSize(int receiverQueueSize) { checkArgument(receiverQueueSize >= 0, "receiverQueueSize needs to be >= 0"); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderImpl.java index 099098fcfabf4..8760d69447a64 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderImpl.java @@ -19,6 +19,7 @@ package org.apache.pulsar.client.impl; import java.io.IOException; +import java.util.List; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -37,6 +38,7 @@ import org.apache.pulsar.client.api.ReaderListener; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.TopicMessageId; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import org.apache.pulsar.client.impl.conf.ReaderConfigurationData; import org.apache.pulsar.client.util.ExecutorProvider; @@ -121,11 +123,15 @@ public void reachedEndOfTopic(Consumer consumer) { consumerConfiguration.setCryptoKeyReader(readerConfiguration.getCryptoKeyReader()); } + if (readerConfiguration.getMessageCrypto() != null) { + consumerConfiguration.setMessageCrypto(readerConfiguration.getMessageCrypto()); + } + if (readerConfiguration.getKeyHashRanges() != null) { consumerConfiguration.setKeySharedPolicy( - KeySharedPolicy - .stickyHashRange() - .ranges(readerConfiguration.getKeyHashRanges()) + KeySharedPolicy + .stickyHashRange() + .ranges(readerConfiguration.getKeyHashRanges()) ); } @@ -253,4 +259,14 @@ public CompletableFuture seekAsync(MessageId messageId) { public CompletableFuture seekAsync(long timestamp) { return consumer.seekAsync(timestamp); } + + @Override + public List getLastMessageIds() throws PulsarClientException { + return consumer.getLastMessageIds(); + } + + @Override + public CompletableFuture> getLastMessageIdsAsync() { + return consumer.getLastMessageIdsAsync(); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java new file mode 100644 index 0000000000000..4beff4719c895 --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import io.netty.channel.EventLoopGroup; +import io.netty.util.concurrent.ScheduledFuture; +import java.util.Arrays; +import java.util.HashSet; +import java.util.concurrent.TimeUnit; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.ServiceUrlProvider; +import org.apache.pulsar.client.util.ExecutorProvider; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.netty.EventLoopUtil; + +@Slf4j +@SuppressFBWarnings(value = {"EI_EXPOSE_REP2"}) +public class SameAuthParamsLookupAutoClusterFailover implements ServiceUrlProvider { + + private PulsarClientImpl pulsarClient; + private EventLoopGroup executor; + private volatile boolean closed; + private ScheduledFuture scheduledCheckTask; + @Getter + private int failoverThreshold = 5; + @Getter + private int recoverThreshold = 5; + @Getter + private long checkHealthyIntervalMs = 1000; + @Getter + private boolean markTopicNotFoundAsAvailable = true; + @Getter + private String testTopic = "public/default/tp_test"; + + private String[] pulsarServiceUrlArray; + private PulsarServiceState[] pulsarServiceStateArray; + private MutableInt[] checkCounterArray; + @Getter + private volatile int currentPulsarServiceIndex; + + private SameAuthParamsLookupAutoClusterFailover() {} + + @Override + public void initialize(PulsarClient client) { + this.currentPulsarServiceIndex = 0; + this.pulsarClient = (PulsarClientImpl) client; + this.executor = EventLoopUtil.newEventLoopGroup(1, false, + new ExecutorProvider.ExtendedThreadFactory("broker-service-url-check")); + scheduledCheckTask = executor.scheduleAtFixedRate(() -> { + if (closed) { + return; + } + checkPulsarServices(); + int firstHealthyPulsarService = firstHealthyPulsarService(); + if (firstHealthyPulsarService == currentPulsarServiceIndex) { + return; + } + if (firstHealthyPulsarService < 0) { + int failoverTo = findFailoverTo(); + if (failoverTo < 0) { + // No healthy pulsar service to connect. + log.error("Failed to choose a pulsar service to connect, no one pulsar service is healthy. Current" + + " pulsar service: [{}] {}. States: {}, Counters: {}", currentPulsarServiceIndex, + pulsarServiceUrlArray[currentPulsarServiceIndex], Arrays.toString(pulsarServiceStateArray), + Arrays.toString(checkCounterArray)); + } else { + // Failover to low priority pulsar service. + updateServiceUrl(failoverTo); + } + } else { + // Back to high priority pulsar service. + updateServiceUrl(firstHealthyPulsarService); + } + }, checkHealthyIntervalMs, checkHealthyIntervalMs, TimeUnit.MILLISECONDS); + } + + @Override + public String getServiceUrl() { + return pulsarServiceUrlArray[currentPulsarServiceIndex]; + } + + @Override + public void close() throws Exception { + log.info("Closing service url provider. Current pulsar service: [{}] {}", currentPulsarServiceIndex, + pulsarServiceUrlArray[currentPulsarServiceIndex]); + closed = true; + scheduledCheckTask.cancel(false); + executor.shutdownNow(); + } + + private int firstHealthyPulsarService() { + for (int i = 0; i <= currentPulsarServiceIndex; i++) { + if (pulsarServiceStateArray[i] == PulsarServiceState.Healthy + || pulsarServiceStateArray[i] == PulsarServiceState.PreFail) { + return i; + } + } + return -1; + } + + private int findFailoverTo() { + for (int i = currentPulsarServiceIndex + 1; i <= pulsarServiceUrlArray.length; i++) { + if (probeAvailable(i)) { + return i; + } + } + return -1; + } + + private void checkPulsarServices() { + for (int i = 0; i <= currentPulsarServiceIndex; i++) { + if (probeAvailable(i)) { + switch (pulsarServiceStateArray[i]) { + case Healthy: { + break; + } + case PreFail: { + pulsarServiceStateArray[i] = PulsarServiceState.Healthy; + checkCounterArray[i].setValue(0); + break; + } + case Failed: { + pulsarServiceStateArray[i] = PulsarServiceState.PreRecover; + checkCounterArray[i].setValue(1); + break; + } + case PreRecover: { + checkCounterArray[i].setValue(checkCounterArray[i].getValue() + 1); + if (checkCounterArray[i].getValue() >= recoverThreshold) { + pulsarServiceStateArray[i] = PulsarServiceState.Healthy; + checkCounterArray[i].setValue(0); + } + break; + } + } + } else { + switch (pulsarServiceStateArray[i]) { + case Healthy: { + pulsarServiceStateArray[i] = PulsarServiceState.PreFail; + checkCounterArray[i].setValue(1); + break; + } + case PreFail: { + checkCounterArray[i].setValue(checkCounterArray[i].getValue() + 1); + if (checkCounterArray[i].getValue() >= failoverThreshold) { + pulsarServiceStateArray[i] = PulsarServiceState.Failed; + checkCounterArray[i].setValue(0); + } + break; + } + case Failed: { + break; + } + case PreRecover: { + pulsarServiceStateArray[i] = PulsarServiceState.Failed; + checkCounterArray[i].setValue(0); + break; + } + } + } + } + } + + private boolean probeAvailable(int brokerServiceIndex) { + String url = pulsarServiceUrlArray[brokerServiceIndex]; + try { + LookupTopicResult res = pulsarClient.getLookup(url).getBroker(TopicName.get(testTopic)) + .get(3, TimeUnit.SECONDS); + if (log.isDebugEnabled()) { + log.debug("Success to probe available(lookup res: {}), [{}] {}}. States: {}, Counters: {}", + res.toString(), brokerServiceIndex, url, Arrays.toString(pulsarServiceStateArray), + Arrays.toString(checkCounterArray)); + } + return true; + } catch (Exception e) { + Throwable actEx = FutureUtil.unwrapCompletionException(e); + if (actEx instanceof PulsarAdminException.NotFoundException + || actEx instanceof PulsarClientException.NotFoundException + || actEx instanceof PulsarClientException.TopicDoesNotExistException + || actEx instanceof PulsarClientException.LookupException) { + if (markTopicNotFoundAsAvailable) { + if (log.isDebugEnabled()) { + log.debug("Success to probe available(case tenant/namespace/topic not found), [{}] {}." + + " States: {}, Counters: {}", brokerServiceIndex, url, + Arrays.toString(pulsarServiceStateArray), Arrays.toString(checkCounterArray)); + } + return true; + } else { + log.warn("Failed to probe available(error tenant/namespace/topic not found), [{}] {}. States: {}," + + " Counters: {}", brokerServiceIndex, url, Arrays.toString(pulsarServiceStateArray), + Arrays.toString(checkCounterArray)); + return false; + } + } + log.warn("Failed to probe available, [{}] {}. States: {}, Counters: {}", brokerServiceIndex, url, + Arrays.toString(pulsarServiceStateArray), Arrays.toString(checkCounterArray)); + return false; + } + } + + private void updateServiceUrl(int targetIndex) { + String currentUrl = pulsarServiceUrlArray[currentPulsarServiceIndex]; + String targetUrl = pulsarServiceUrlArray[targetIndex]; + String logMsg; + if (targetIndex < currentPulsarServiceIndex) { + logMsg = String.format("Recover to high priority pulsar service [%s] %s --> [%s] %s. States: %s," + + " Counters: %s", currentPulsarServiceIndex, currentUrl, targetIndex, targetUrl, + Arrays.toString(pulsarServiceStateArray), Arrays.toString(checkCounterArray)); + } else { + logMsg = String.format("Failover to low priority pulsar service [%s] %s --> [%s] %s. States: %s," + + " Counters: %s", currentPulsarServiceIndex, currentUrl, targetIndex, targetUrl, + Arrays.toString(pulsarServiceStateArray), Arrays.toString(checkCounterArray)); + } + log.info(logMsg); + try { + pulsarClient.updateServiceUrl(targetUrl); + pulsarClient.reloadLookUp(); + currentPulsarServiceIndex = targetIndex; + } catch (Exception e) { + log.error("Failed to {}", logMsg, e); + } + } + + public enum PulsarServiceState { + Healthy, + PreFail, + Failed, + PreRecover; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + + private SameAuthParamsLookupAutoClusterFailover + sameAuthParamsLookupAutoClusterFailover = new SameAuthParamsLookupAutoClusterFailover(); + + public Builder failoverThreshold(int failoverThreshold) { + if (failoverThreshold < 1) { + throw new IllegalArgumentException("failoverThreshold must be larger than 0"); + } + sameAuthParamsLookupAutoClusterFailover.failoverThreshold = failoverThreshold; + return this; + } + + public Builder recoverThreshold(int recoverThreshold) { + if (recoverThreshold < 1) { + throw new IllegalArgumentException("recoverThreshold must be larger than 0"); + } + sameAuthParamsLookupAutoClusterFailover.recoverThreshold = recoverThreshold; + return this; + } + + public Builder checkHealthyIntervalMs(int checkHealthyIntervalMs) { + if (checkHealthyIntervalMs < 1) { + throw new IllegalArgumentException("checkHealthyIntervalMs must be larger than 0"); + } + sameAuthParamsLookupAutoClusterFailover.checkHealthyIntervalMs = checkHealthyIntervalMs; + return this; + } + + public Builder testTopic(String testTopic) { + if (StringUtils.isBlank(testTopic) && TopicName.get(testTopic) != null) { + throw new IllegalArgumentException("testTopic can not be blank"); + } + sameAuthParamsLookupAutoClusterFailover.testTopic = testTopic; + return this; + } + + public Builder markTopicNotFoundAsAvailable(boolean markTopicNotFoundAsAvailable) { + sameAuthParamsLookupAutoClusterFailover.markTopicNotFoundAsAvailable = markTopicNotFoundAsAvailable; + return this; + } + + public Builder pulsarServiceUrlArray(String[] pulsarServiceUrlArray) { + if (pulsarServiceUrlArray == null || pulsarServiceUrlArray.length == 0) { + throw new IllegalArgumentException("pulsarServiceUrlArray can not be empty"); + } + sameAuthParamsLookupAutoClusterFailover.pulsarServiceUrlArray = pulsarServiceUrlArray; + int pulsarServiceLen = pulsarServiceUrlArray.length; + HashSet uniqueChecker = new HashSet<>(); + for (int i = 0; i < pulsarServiceLen; i++) { + String pulsarService = pulsarServiceUrlArray[i]; + if (StringUtils.isBlank(pulsarService)) { + throw new IllegalArgumentException("pulsarServiceUrlArray contains a blank value at index " + i); + } + if (pulsarService.startsWith("http") || pulsarService.startsWith("HTTP")) { + throw new IllegalArgumentException("SameAuthParamsLookupAutoClusterFailover does not support HTTP" + + " protocol pulsar service url so far."); + } + if (!uniqueChecker.add(pulsarService)) { + throw new IllegalArgumentException("pulsarServiceUrlArray contains duplicated value " + + pulsarServiceUrlArray[i]); + } + } + return this; + } + + public SameAuthParamsLookupAutoClusterFailover build() { + String[] pulsarServiceUrlArray = sameAuthParamsLookupAutoClusterFailover.pulsarServiceUrlArray; + if (pulsarServiceUrlArray == null) { + throw new IllegalArgumentException("pulsarServiceUrlArray can not be empty"); + } + int pulsarServiceLen = pulsarServiceUrlArray.length; + sameAuthParamsLookupAutoClusterFailover.pulsarServiceStateArray = new PulsarServiceState[pulsarServiceLen]; + sameAuthParamsLookupAutoClusterFailover.checkCounterArray = new MutableInt[pulsarServiceLen]; + for (int i = 0; i < pulsarServiceLen; i++) { + sameAuthParamsLookupAutoClusterFailover.pulsarServiceStateArray[i] = PulsarServiceState.Healthy; + sameAuthParamsLookupAutoClusterFailover.checkCounterArray[i] = new MutableInt(0); + } + return sameAuthParamsLookupAutoClusterFailover; + } + } +} + diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SendCallback.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SendCallback.java index 369bb34a29a79..f55d7ae79129c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SendCallback.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SendCallback.java @@ -20,18 +20,21 @@ import java.util.concurrent.CompletableFuture; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.common.classification.InterfaceStability; /** * */ +@InterfaceStability.Evolving public interface SendCallback { /** * invoked when send operation completes. * * @param e + * @param opSendMsgStats stats associated with the send operation */ - void sendComplete(Exception e); + void sendComplete(Throwable e, OpSendMsgStats opSendMsgStats); /** * used to specify a callback to be invoked on completion of a send operation for individual messages sent in a diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java index 77aba7e48cbad..17b49828eeced 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.impl; +import static org.apache.pulsar.common.topics.TopicCompactionStrategy.TABLE_VIEW_TAG; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -35,11 +36,13 @@ import org.apache.pulsar.client.api.CryptoKeyReader; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.ReaderBuilder; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.TableView; +import org.apache.pulsar.client.api.TopicMessageId; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.topics.TopicCompactionStrategy; @@ -58,6 +61,26 @@ public class TableViewImpl implements TableView { private final boolean isPersistentTopic; private TopicCompactionStrategy compactionStrategy; + /** + * Store the refresh tasks. When read to the position recording in the right map, + * then remove the position in the right map. If the right map is empty, complete the future in the left. + * There should be no timeout exception here, because the caller can only retry for TimeoutException. + * It will only be completed exceptionally when no more messages can be read. + */ + private final ConcurrentHashMap, Map> pendingRefreshRequests; + + /** + * This map stored the read position of each partition. It is used for the following case: + *

      + * 1. Get last message ID. + * 2. Receive message p1-1:1, p2-1:1, p2-1:2, p3-1:1 + * 3. Receive response of step1 {|p1-1:1|p2-2:2|p3-3:6|} + * 4. No more messages are written to this topic. + * As a result, the refresh operation will never be completed. + *

      + */ + private final ConcurrentHashMap lastReadPositions; + TableViewImpl(PulsarClientImpl client, Schema schema, TableViewConfigurationData conf) { this.conf = conf; this.isPersistentTopic = conf.getTopicName().startsWith(TopicDomain.persistent.toString()); @@ -65,7 +88,10 @@ public class TableViewImpl implements TableView { this.immutableData = Collections.unmodifiableMap(data); this.listeners = new ArrayList<>(); this.listenersMutex = new ReentrantLock(); - this.compactionStrategy = TopicCompactionStrategy.load(conf.getTopicCompactionStrategyClassName()); + this.compactionStrategy = + TopicCompactionStrategy.load(TABLE_VIEW_TAG, conf.getTopicCompactionStrategyClassName()); + this.pendingRefreshRequests = new ConcurrentHashMap<>(); + this.lastReadPositions = new ConcurrentHashMap<>(); ReaderBuilder readerBuilder = client.newReader(schema) .topic(conf.getTopicName()) .startMessageId(MessageId.earliest) @@ -91,9 +117,10 @@ CompletableFuture> start() { return reader.thenCompose((reader) -> { if (!isPersistentTopic) { readTailMessages(reader); - return CompletableFuture.completedFuture(reader); + return CompletableFuture.completedFuture(null); } - return this.readAllExistingMessages(reader); + return this.readAllExistingMessages(reader) + .thenRun(() -> readTailMessages(reader)); }).thenApply(__ -> this); } @@ -177,6 +204,7 @@ public void close() throws PulsarClientException { } private void handleMessage(Message msg) { + lastReadPositions.put(msg.getTopicName(), msg.getMessageId()); try { if (msg.hasKey()) { String key = msg.getKey(); @@ -198,6 +226,7 @@ private void handleMessage(Message msg) { key, cur, prev); + compactionStrategy.handleSkippedMessage(key, cur); } } @@ -222,34 +251,132 @@ private void handleMessage(Message msg) { } } } + checkAllFreshTask(msg); } finally { msg.release(); } } - private CompletableFuture> readAllExistingMessages(Reader reader) { + @Override + public CompletableFuture refreshAsync() { + CompletableFuture completableFuture = new CompletableFuture<>(); + reader.thenCompose(reader -> getLastMessageIdOfNonEmptyTopics(reader).thenAccept(lastMessageIds -> { + if (lastMessageIds.isEmpty()) { + completableFuture.complete(null); + return; + } + // After get the response of lastMessageIds, put the future and result into `refreshMap` + // and then filter out partitions that has been read to the lastMessageID. + pendingRefreshRequests.put(completableFuture, lastMessageIds); + filterReceivedMessages(lastMessageIds); + // If there is no new messages, the refresh operation could be completed right now. + if (lastMessageIds.isEmpty()) { + pendingRefreshRequests.remove(completableFuture); + completableFuture.complete(null); + } + })).exceptionally(throwable -> { + completableFuture.completeExceptionally(throwable); + pendingRefreshRequests.remove(completableFuture); + return null; + }); + return completableFuture; + } + + @Override + public void refresh() throws PulsarClientException { + try { + refreshAsync().get(); + } catch (Exception e) { + throw PulsarClientException.unwrap(e); + } + } + + private CompletableFuture readAllExistingMessages(Reader reader) { long startTime = System.nanoTime(); AtomicLong messagesRead = new AtomicLong(); - CompletableFuture> future = new CompletableFuture<>(); - readAllExistingMessages(reader, future, startTime, messagesRead); + CompletableFuture future = new CompletableFuture<>(); + getLastMessageIdOfNonEmptyTopics(reader).thenAccept(lastMessageIds -> { + if (lastMessageIds.isEmpty()) { + future.complete(null); + return; + } + readAllExistingMessages(reader, future, startTime, messagesRead, lastMessageIds); + }).exceptionally(ex -> { + future.completeExceptionally(ex); + return null; + }); return future; } - private void readAllExistingMessages(Reader reader, CompletableFuture> future, long startTime, - AtomicLong messagesRead) { + private CompletableFuture> getLastMessageIdOfNonEmptyTopics(Reader reader) { + return reader.getLastMessageIdsAsync().thenApply(lastMessageIds -> { + Map lastMessageIdMap = new ConcurrentHashMap<>(); + lastMessageIds.forEach(topicMessageId -> { + if (((MessageIdAdv) topicMessageId).getEntryId() >= 0) { + lastMessageIdMap.put(topicMessageId.getOwnerTopic(), topicMessageId); + } // else: a negative entry id represents an empty topic so that we don't have to read messages from it + }); + return lastMessageIdMap; + }); + } + + private void filterReceivedMessages(Map lastMessageIds) { + // The `lastMessageIds` and `readPositions` is concurrency-safe data types. + lastMessageIds.forEach((partition, lastMessageId) -> { + MessageId messageId = lastReadPositions.get(partition); + if (messageId != null && lastMessageId.compareTo(messageId) <= 0) { + lastMessageIds.remove(partition); + } + }); + } + + private boolean checkFreshTask(Map maxMessageIds, CompletableFuture future, + MessageId messageId, String topicName) { + // The message received from multi-consumer/multi-reader is processed to TopicMessageImpl. + TopicMessageId maxMessageId = maxMessageIds.get(topicName); + // We need remove the partition from the maxMessageIds map + // once the partition has been read completely. + if (maxMessageId != null && messageId.compareTo(maxMessageId) >= 0) { + maxMessageIds.remove(topicName); + } + if (maxMessageIds.isEmpty()) { + future.complete(null); + return true; + } else { + return false; + } + } + + private void checkAllFreshTask(Message msg) { + pendingRefreshRequests.forEach((future, maxMessageIds) -> { + String topicName = msg.getTopicName(); + MessageId messageId = msg.getMessageId(); + if (checkFreshTask(maxMessageIds, future, messageId, topicName)) { + pendingRefreshRequests.remove(future); + } + }); + } + + private void readAllExistingMessages(Reader reader, CompletableFuture future, long startTime, + AtomicLong messagesRead, Map maxMessageIds) { reader.hasMessageAvailableAsync() .thenAccept(hasMessage -> { if (hasMessage) { reader.readNextAsync() .thenAccept(msg -> { messagesRead.incrementAndGet(); + String topicName = msg.getTopicName(); + MessageId messageId = msg.getMessageId(); handleMessage(msg); - readAllExistingMessages(reader, future, startTime, messagesRead); + if (!checkFreshTask(maxMessageIds, future, messageId, topicName)) { + readAllExistingMessages(reader, future, startTime, + messagesRead, maxMessageIds); + } }).exceptionally(ex -> { if (ex.getCause() instanceof PulsarClientException.AlreadyClosedException) { - log.error("Reader {} was closed while reading existing messages.", - reader.getTopic(), ex); + log.info("Reader {} was closed while reading existing messages.", + reader.getTopic()); } else { log.warn("Reader {} was interrupted while reading existing messages. ", reader.getTopic(), ex); @@ -265,8 +392,7 @@ private void readAllExistingMessages(Reader reader, CompletableFuture reader) { readTailMessages(reader); }).exceptionally(ex -> { if (ex.getCause() instanceof PulsarClientException.AlreadyClosedException) { - log.error("Reader {} was closed while reading tail messages.", - reader.getTopic(), ex); + log.info("Reader {} was closed while reading tail messages.", reader.getTopic()); + // Fail all refresh request when no more messages can be read. + pendingRefreshRequests.keySet().forEach(future -> { + pendingRefreshRequests.remove(future); + future.completeExceptionally(ex); + }); } else { + // Retrying on the other exceptions such as NotConnectedException + try { + Thread.sleep(50); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } log.warn("Reader {} was interrupted while reading tail messages. " - + "Retrying..", reader.getTopic(), ex); + + "Retrying..", reader.getTopic(), ex); readTailMessages(reader); } return null; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java index 384d1b688b8d5..93fa7082f33c6 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java @@ -18,19 +18,19 @@ */ package org.apache.pulsar.client.impl; +import com.google.re2j.Pattern; import io.netty.channel.ChannelHandlerContext; -import java.util.List; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.atomic.AtomicReference; -import java.util.regex.Pattern; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.api.proto.BaseCommand; import org.apache.pulsar.common.api.proto.CommandWatchTopicUpdate; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.util.BackoffBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,7 +42,7 @@ public class TopicListWatcher extends HandlerState implements ConnectionHandler. AtomicLongFieldUpdater .newUpdater(TopicListWatcher.class, "createWatcherDeadline"); - private final PatternMultiTopicsConsumerImpl.TopicsChangedListener topicsChangeListener; + private final PatternConsumerUpdateQueue patternConsumerUpdateQueue; private final String name; private final ConnectionHandler connectionHandler; private final Pattern topicsPattern; @@ -53,16 +53,22 @@ public class TopicListWatcher extends HandlerState implements ConnectionHandler. private String topicsHash; private final CompletableFuture watcherFuture; - private final List previousExceptions = new CopyOnWriteArrayList<>(); + private final AtomicInteger previousExceptionCount = new AtomicInteger(); private final AtomicReference clientCnxUsedForWatcherRegistration = new AtomicReference<>(); + private final Runnable recheckTopicsChangeAfterReconnect; - public TopicListWatcher(PatternMultiTopicsConsumerImpl.TopicsChangedListener topicsChangeListener, + + /*** + * @param topicsPattern The regexp for the topic name(not contains partition suffix). + */ + public TopicListWatcher(PatternConsumerUpdateQueue patternConsumerUpdateQueue, PulsarClientImpl client, Pattern topicsPattern, long watcherId, NamespaceName namespace, String topicsHash, - CompletableFuture watcherFuture) { - super(client, null); - this.topicsChangeListener = topicsChangeListener; + CompletableFuture watcherFuture, + Runnable recheckTopicsChangeAfterReconnect) { + super(client, topicsPattern.pattern()); + this.patternConsumerUpdateQueue = patternConsumerUpdateQueue; this.name = "Watcher(" + topicsPattern + ")"; this.connectionHandler = new ConnectionHandler(this, new BackoffBuilder() @@ -77,6 +83,7 @@ public TopicListWatcher(PatternMultiTopicsConsumerImpl.TopicsChangedListener top this.namespace = namespace; this.topicsHash = topicsHash; this.watcherFuture = watcherFuture; + this.recheckTopicsChangeAfterReconnect = recheckTopicsChangeAfterReconnect; connectionHandler.grabCnx(); } @@ -85,26 +92,27 @@ public TopicListWatcher(PatternMultiTopicsConsumerImpl.TopicsChangedListener top public void connectionFailed(PulsarClientException exception) { boolean nonRetriableError = !PulsarClientException.isRetriableError(exception); if (nonRetriableError) { - exception.setPreviousExceptions(previousExceptions); + exception.setPreviousExceptionCount(previousExceptionCount); if (watcherFuture.completeExceptionally(exception)) { setState(State.Failed); log.info("[{}] Watcher creation failed for {} with non-retriable error {}", - topic, name, exception); + topic, name, exception.getMessage()); deregisterFromClientCnx(); } } else { - previousExceptions.add(exception); + previousExceptionCount.incrementAndGet(); } } @Override - public void connectionOpened(ClientCnx cnx) { - previousExceptions.clear(); + public CompletableFuture connectionOpened(ClientCnx cnx) { + previousExceptionCount.set(0); - if (getState() == State.Closing || getState() == State.Closed) { + State state = getState(); + if (state == State.Closing || state == State.Closed) { setState(State.Closed); deregisterFromClientCnx(); - return; + return CompletableFuture.completedFuture(null); } log.info("[{}][{}] Creating topic list watcher on cnx {}, watcherId {}", @@ -116,6 +124,7 @@ public void connectionOpened(ClientCnx cnx) { .compareAndSet(this, 0L, System.currentTimeMillis() + client.getConfiguration().getOperationTimeoutMs()); + final CompletableFuture future = new CompletableFuture<>(); // synchronized this, because redeliverUnAckMessage eliminate the epoch inconsistency between them synchronized (this) { setClientCnx(cnx); @@ -132,20 +141,22 @@ public void connectionOpened(ClientCnx cnx) { setState(State.Closed); deregisterFromClientCnx(); cnx.channel().close(); + future.complete(null); return; } } - this.connectionHandler.resetBackoff(); + recheckTopicsChangeAfterReconnect.run(); watcherFuture.complete(this); - + future.complete(null); }).exceptionally((e) -> { deregisterFromClientCnx(); if (getState() == State.Closing || getState() == State.Closed) { // Watcher was closed while reconnecting, close the connection to make sure the broker // drops the watcher on its side cnx.channel().close(); + future.complete(null); return null; } log.warn("[{}][{}] Failed to create topic list watcher on {}", @@ -155,7 +166,7 @@ public void connectionOpened(ClientCnx cnx) { && PulsarClientException.isRetriableError(e.getCause()) && System.currentTimeMillis() < CREATE_WATCHER_DEADLINE_UPDATER.get(TopicListWatcher.this)) { - reconnectLater(e.getCause()); + future.completeExceptionally(e.getCause()); } else if (!watcherFuture.isDone()) { // unable to create new watcher, fail operation setState(State.Failed); @@ -164,11 +175,15 @@ public void connectionOpened(ClientCnx cnx) { + "when connecting to the broker", getHandlerName()))); } else { // watcher was subscribed and connected, but we got some error, keep trying - reconnectLater(e.getCause()); + future.completeExceptionally(e.getCause()); + } + if (!future.isDone()) { + future.complete(null); } return null; }); } + return future; } @Override @@ -249,11 +264,6 @@ void deregisterFromClientCnx() { setClientCnx(null); } - void reconnectLater(Throwable exception) { - this.connectionHandler.reconnectLater(exception); - } - - private void cleanupAtClose(CompletableFuture closeFuture, Throwable exception) { log.info("[{}] Closed topic list watcher", getHandlerName()); setState(State.Closed); @@ -266,13 +276,7 @@ private void cleanupAtClose(CompletableFuture closeFuture, Throwable excep } public void handleCommandWatchTopicUpdate(CommandWatchTopicUpdate update) { - List deleted = update.getDeletedTopicsList(); - if (!deleted.isEmpty()) { - topicsChangeListener.onTopicsRemoved(deleted); - } - List added = update.getNewTopicsList(); - if (!added.isEmpty()) { - topicsChangeListener.onTopicsAdded(added); - } + patternConsumerUpdateQueue.appendTopicsRemovedOp(update.getDeletedTopicsList()); + patternConsumerUpdateQueue.appendTopicsAddedOp(update.getNewTopicsList()); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java index 1b6cba2f7234d..1fec08a43f137 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java @@ -47,7 +47,7 @@ public class TopicMessageImpl implements Message { } /** - * Get the topic name without partition part of this message. + * Get the topic name with partition part of this message. * @return the name of the topic on which this message was published */ @Override diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java index 601fa2b8f815a..c8c2fa83f94ae 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java @@ -33,6 +33,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClientException; import org.apache.pulsar.client.api.transaction.TxnID; @@ -46,6 +47,8 @@ import org.apache.pulsar.common.api.proto.Subscription; import org.apache.pulsar.common.api.proto.TxnAction; import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.util.Backoff; +import org.apache.pulsar.common.util.BackoffBuilder; import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -86,6 +89,10 @@ public RequestTime(long creationTime, long requestId) { private Timeout requestTimeout; private final CompletableFuture connectFuture; + private final long lookupDeadline; + private final AtomicInteger previousExceptionCount = new AtomicInteger(); + + public TransactionMetaStoreHandler(long transactionCoordinatorId, PulsarClientImpl pulsarClient, String topic, CompletableFuture connectFuture) { @@ -107,6 +114,7 @@ public TransactionMetaStoreHandler(long transactionCoordinatorId, PulsarClientIm this.connectFuture = connectFuture; this.internalPinnedExecutor = pulsarClient.getInternalExecutorService(); this.timer = pulsarClient.timer(); + this.lookupDeadline = System.currentTimeMillis() + client.getConfiguration().getLookupTimeoutMs(); } public void start() { @@ -115,22 +123,37 @@ public void start() { @Override public void connectionFailed(PulsarClientException exception) { - LOG.error("Transaction meta handler with transaction coordinator id {} connection failed.", - transactionCoordinatorId, exception); - if (!this.connectFuture.isDone()) { - this.connectFuture.completeExceptionally(exception); + boolean nonRetriableError = !PulsarClientException.isRetriableError(exception); + boolean timeout = System.currentTimeMillis() > lookupDeadline; + if (nonRetriableError || timeout) { + exception.setPreviousExceptionCount(previousExceptionCount); + if (connectFuture.completeExceptionally(exception)) { + if (nonRetriableError) { + LOG.error("Transaction meta handler with transaction coordinator id {} connection failed.", + transactionCoordinatorId, exception); + } else { + LOG.error("Transaction meta handler with transaction coordinator id {} connection failed after " + + "timeout", transactionCoordinatorId, exception); + } + setState(State.Failed); + } + } else { + previousExceptionCount.getAndIncrement(); } } @Override - public void connectionOpened(ClientCnx cnx) { + public CompletableFuture connectionOpened(ClientCnx cnx) { + final CompletableFuture future = new CompletableFuture<>(); internalPinnedExecutor.execute(() -> { LOG.info("Transaction meta handler with transaction coordinator id {} connection opened.", transactionCoordinatorId); - if (getState() == State.Closing || getState() == State.Closed) { + State state = getState(); + if (state == State.Closing || state == State.Closed) { setState(State.Closed); failPendingRequest(); + future.complete(null); return; } @@ -146,6 +169,7 @@ public void connectionOpened(ClientCnx cnx) { this.connectionHandler.resetBackoff(); pendingRequests.forEach((requestID, opBase) -> checkStateAndSendRequest(opBase)); } + future.complete(null); }); }).exceptionally((e) -> { internalPinnedExecutor.execute(() -> { @@ -155,16 +179,21 @@ public void connectionOpened(ClientCnx cnx) { || e.getCause() instanceof PulsarClientException.NotAllowedException) { setState(State.Closed); cnx.channel().close(); + future.complete(null); } else { - connectionHandler.reconnectLater(e.getCause()); + future.completeExceptionally(e.getCause()); } }); return null; }); } else { + LOG.warn("Can not connect to the transaction coordinator because the protocol version {} is " + + "lower than 19", cnx.getRemoteEndpointProtocolVersion()); registerToConnection(cnx); + future.complete(null); } }); + return future; } private boolean registerToConnection(ClientCnx cnx) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java index 026f8a1e69e0b..d90c2e8828364 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java @@ -50,6 +50,7 @@ public class TypedMessageBuilderImpl implements TypedMessageBuilder { private final transient Schema schema; private transient ByteBuffer content; private final transient TransactionImpl txn; + private transient T value; public TypedMessageBuilderImpl(ProducerBase producer, Schema schema) { this(producer, schema, null); @@ -65,6 +66,22 @@ public TypedMessageBuilderImpl(ProducerBase producer, } private long beforeSend() { + if (value == null) { + msgMetadata.setNullValue(true); + } else { + getKeyValueSchema().map(keyValueSchema -> { + if (keyValueSchema.getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED) { + setSeparateKeyValue(value, keyValueSchema); + return this; + } else { + return null; + } + }).orElseGet(() -> { + content = ByteBuffer.wrap(schema.encode(value)); + return this; + }); + } + if (txn == null) { return -1L; } @@ -140,22 +157,8 @@ public TypedMessageBuilder orderingKey(byte[] orderingKey) { @Override public TypedMessageBuilder value(T value) { - if (value == null) { - msgMetadata.setNullValue(true); - return this; - } - - return getKeyValueSchema().map(keyValueSchema -> { - if (keyValueSchema.getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED) { - setSeparateKeyValue(value, keyValueSchema); - return this; - } else { - return null; - } - }).orElseGet(() -> { - content = ByteBuffer.wrap(schema.encode(value)); - return this; - }); + this.value = value; + return this; } @Override diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedMessageTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedMessageTracker.java index 534f33350267d..e755b6ba1ee6d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedMessageTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedMessageTracker.java @@ -22,6 +22,7 @@ import io.netty.util.Timeout; import io.netty.util.TimerTask; import io.netty.util.concurrent.FastThreadLocal; +import io.opentelemetry.api.common.Attributes; import java.io.Closeable; import java.util.ArrayDeque; import java.util.Collections; @@ -35,6 +36,9 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; +import org.apache.pulsar.client.impl.metrics.Counter; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; +import org.apache.pulsar.client.impl.metrics.Unit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,6 +56,8 @@ public class UnAckedMessageTracker implements Closeable { protected final long ackTimeoutMillis; protected final long tickDurationInMs; + private final Counter consumerAckTimeoutsCounter; + private static class UnAckedMessageTrackerDisabled extends UnAckedMessageTracker { @Override public void clear() { @@ -89,13 +95,14 @@ public void close() { protected Timeout timeout; - public UnAckedMessageTracker() { + private UnAckedMessageTracker() { readLock = null; writeLock = null; timePartitions = null; messageIdPartitionMap = null; this.ackTimeoutMillis = 0; this.tickDurationInMs = 0; + this.consumerAckTimeoutsCounter = null; } protected static final FastThreadLocal> TL_MESSAGE_IDS_SET = @@ -114,6 +121,14 @@ public UnAckedMessageTracker(PulsarClientImpl client, ConsumerBase consumerBa ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(); this.readLock = readWriteLock.readLock(); this.writeLock = readWriteLock.writeLock(); + + InstrumentProvider ip = client.instrumentProvider(); + consumerAckTimeoutsCounter = ip.newCounter("pulsar.client.consumer.message.ack.timeout", Unit.Messages, + "The number of messages that were not acknowledged in the configured timeout period, hence, were " + + "requested by the client to be redelivered", + consumerBase.getTopic(), + Attributes.builder().put("pulsar.subscription", consumerBase.getSubscription()).build()); + if (conf.getAckTimeoutRedeliveryBackoff() == null) { this.messageIdPartitionMap = new HashMap<>(); this.timePartitions = new ArrayDeque<>(); @@ -136,10 +151,14 @@ public void run(Timeout t) throws Exception { try { HashSet headPartition = timePartitions.removeFirst(); if (!headPartition.isEmpty()) { + consumerAckTimeoutsCounter.add(headPartition.size()); log.info("[{}] {} messages will be re-delivered", consumerBase, headPartition.size()); headPartition.forEach(messageId -> { - addChunkedMessageIdsAndRemoveFromSequenceMap(messageId, messageIds, consumerBase); - messageIds.add(messageId); + if (messageId instanceof ChunkMessageIdImpl) { + addChunkedMessageIdsAndRemoveFromSequenceMap(messageId, messageIds, consumerBase); + } else { + messageIds.add(messageId); + } messageIdPartitionMap.remove(messageId); }); } @@ -189,6 +208,10 @@ public void clear() { } public boolean add(MessageId messageId) { + if (messageId == null) { + return false; + } + writeLock.lock(); try { HashSet partition = timePartitions.peekLast(); @@ -217,6 +240,10 @@ boolean isEmpty() { } public boolean remove(MessageId messageId) { + if (messageId == null) { + return false; + } + writeLock.lock(); try { boolean removed = false; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationDataTls.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationDataTls.java index a16e70f8da7e9..93a2c0b597420 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationDataTls.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationDataTls.java @@ -143,7 +143,7 @@ public InputStream getTlsTrustStoreStream() { } @Override - public String getTlsCerificateFilePath() { + public String getTlsCertificateFilePath() { return certFile != null ? certFile.getFileName() : null; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java index 7d94675ccba7d..c1c2e75925502 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java @@ -19,11 +19,17 @@ package org.apache.pulsar.client.impl.conf; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import io.opentelemetry.api.OpenTelemetry; import io.swagger.annotations.ApiModelProperty; import java.io.Serializable; import java.net.InetSocketAddress; import java.net.URI; import java.time.Clock; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -38,6 +44,8 @@ import org.apache.pulsar.client.api.ServiceUrlProvider; import org.apache.pulsar.client.impl.auth.AuthenticationDisabled; import org.apache.pulsar.client.util.Secret; +import org.apache.pulsar.common.util.DefaultPulsarSslFactory; + /** * This is a simple holder of the client configuration values. @@ -45,6 +53,7 @@ @Data @NoArgsConstructor @AllArgsConstructor +@JsonIgnoreProperties(ignoreUnknown = true) public class ClientConfigurationData implements Serializable, Cloneable { private static final long serialVersionUID = 1L; @@ -130,7 +139,7 @@ public class ClientConfigurationData implements Serializable, Cloneable { value = "Release the connection if it is not used for more than [connectionMaxIdleSeconds] seconds. " + "If [connectionMaxIdleSeconds] < 0, disabled the feature that auto release the idle connections" ) - private int connectionMaxIdleSeconds = 180; + private int connectionMaxIdleSeconds = 25; @ApiModelProperty( name = "useTcpNoDelay", @@ -173,6 +182,18 @@ public class ClientConfigurationData implements Serializable, Cloneable { value = "Whether the hostname is validated when the client creates a TLS connection with brokers." ) private boolean tlsHostnameVerificationEnable = false; + + @ApiModelProperty( + name = "sslFactoryPlugin", + value = "SSL Factory Plugin class to provide SSLEngine and SSLContext objects. The default " + + " class used is DefaultPulsarSslFactory.") + private String sslFactoryPlugin = DefaultPulsarSslFactory.class.getName(); + + @ApiModelProperty( + name = "sslFactoryPluginParams", + value = "SSL Factory plugin configuration parameters.") + private String sslFactoryPluginParams = ""; + @ApiModelProperty( name = "concurrentLookupRequest", value = "The number of concurrent lookup requests that can be sent on each broker connection. " @@ -359,6 +380,13 @@ public class ClientConfigurationData implements Serializable, Cloneable { ) private int dnsLookupBindPort = 0; + @ApiModelProperty( + name = "dnsServerAddresses", + value = "The Pulsar client dns lookup server address" + ) + @SuppressFBWarnings({"EI_EXPOSE_REP2", "EI_EXPOSE_REP"}) + private List dnsServerAddresses = new ArrayList<>(); + // socks5 @ApiModelProperty( name = "socks5ProxyAddress", @@ -385,6 +413,10 @@ public class ClientConfigurationData implements Serializable, Cloneable { ) private String description; + private Map lookupProperties; + + private transient OpenTelemetry openTelemetry; + /** * Gets the authentication settings for the client. * @@ -448,4 +480,12 @@ public String getSocks5ProxyUsername() { public String getSocks5ProxyPassword() { return Objects.nonNull(socks5ProxyPassword) ? socks5ProxyPassword : System.getProperty("socks5Proxy.password"); } + + public void setLookupProperties(Map lookupProperties) { + this.lookupProperties = Collections.unmodifiableMap(lookupProperties); + } + + public Map getLookupProperties() { + return (lookupProperties == null) ? Collections.emptyMap() : Collections.unmodifiableMap(lookupProperties); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/CmdGenerateDocumentation.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/CmdGenerateDocumentation.java deleted file mode 100644 index ae6210b978edc..0000000000000 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/CmdGenerateDocumentation.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.client.impl.conf; - -import com.beust.jcommander.Parameters; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.common.util.BaseGenerateDocumentation; - -@Data -@Parameters(commandDescription = "Generate documentation automatically.") -@Slf4j -public class CmdGenerateDocumentation extends BaseGenerateDocumentation { - - @Override - public String generateDocumentByClassName(String className) throws Exception { - StringBuilder sb = new StringBuilder(); - if (ClientConfigurationData.class.getName().equals(className)) { - return generateDocByApiModelProperty(className, "Client", sb); - } else if (ProducerConfigurationData.class.getName().equals(className)) { - return generateDocByApiModelProperty(className, "Producer", sb); - } else if (ConsumerConfigurationData.class.getName().equals(className)) { - return generateDocByApiModelProperty(className, "Consumer", sb); - } else if (ReaderConfigurationData.class.getName().equals(className)) { - return generateDocByApiModelProperty(className, "Reader", sb); - } - - return "Class [" + className + "] not found"; - } - - public static void main(String[] args) throws Exception { - CmdGenerateDocumentation generateDocumentation = new CmdGenerateDocumentation(); - generateDocumentation.run(args); - } -} diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java index 8760926792cd7..f9ff5913f62da 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java @@ -43,6 +43,7 @@ import org.apache.pulsar.client.api.KeySharedPolicy; import org.apache.pulsar.client.api.MessageCrypto; import org.apache.pulsar.client.api.MessageListener; +import org.apache.pulsar.client.api.MessageListenerExecutor; import org.apache.pulsar.client.api.MessagePayloadProcessor; import org.apache.pulsar.client.api.RedeliveryBackoff; import org.apache.pulsar.client.api.RegexSubscriptionMode; @@ -65,7 +66,7 @@ public class ConsumerConfigurationData implements Serializable, Cloneable { @ApiModelProperty( name = "topicsPattern", - value = "Topic pattern" + value = "The regexp for the topic name(not contains partition suffix)." ) private Pattern topicsPattern; @@ -90,6 +91,8 @@ public class ConsumerConfigurationData implements Serializable, Cloneable { private SubscriptionMode subscriptionMode = SubscriptionMode.Durable; + @JsonIgnore + private transient MessageListenerExecutor messageListenerExecutor; @JsonIgnore private MessageListener messageListener; @@ -310,7 +313,7 @@ public int getMaxPendingChuckedMessage() { name = "patternAutoDiscoveryPeriod", value = "Topic auto discovery period when using a pattern for topic's consumer.\n" + "\n" - + "The default and minimum value is 1 minute." + + "The default value is 1 minute, with a minimum of 1 second." ) private int patternAutoDiscoveryPeriod = 60; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java index 581b3d8a1635e..6ec738bbf4c8d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java @@ -204,6 +204,8 @@ public class ProducerConfigurationData implements Serializable, Cloneable { private SortedMap properties = new TreeMap<>(); + private boolean isNonPartitionedTopicExpected; + @ApiModelProperty( name = "initialSubscriptionName", value = "Use this configuration to automatically create an initial subscription when creating a topic." diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ReaderConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ReaderConfigurationData.java index 86707d2aa2f0e..73d97f1f33607 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ReaderConfigurationData.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ReaderConfigurationData.java @@ -19,6 +19,7 @@ package org.apache.pulsar.client.impl.conf; import com.fasterxml.jackson.annotation.JsonIgnore; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import io.swagger.annotations.ApiModelProperty; import java.io.Serializable; import java.util.HashSet; @@ -28,6 +29,7 @@ import lombok.Data; import org.apache.pulsar.client.api.ConsumerCryptoFailureAction; import org.apache.pulsar.client.api.CryptoKeyReader; +import org.apache.pulsar.client.api.MessageCrypto; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Range; import org.apache.pulsar.client.api.ReaderInterceptor; @@ -113,6 +115,9 @@ public class ReaderConfigurationData implements Serializable, Cloneable { ) private ConsumerCryptoFailureAction cryptoFailureAction = ConsumerCryptoFailureAction.FAIL; + @JsonIgnore + private transient MessageCrypto messageCrypto = null; + @ApiModelProperty( name = "readCompacted", value = "If enabling `readCompacted`, a consumer reads messages from a compacted topic rather than a full " @@ -184,4 +189,14 @@ public ReaderConfigurationData clone() { throw new RuntimeException("Failed to clone ReaderConfigurationData"); } } + + @SuppressFBWarnings({"EI_EXPOSE_REP"}) + public MessageCrypto getMessageCrypto() { + return messageCrypto; + } + + @SuppressFBWarnings({"EI_EXPOSE_REP2"}) + public void setMessageCrypto(MessageCrypto messageCrypto) { + this.messageCrypto = messageCrypto; + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/metrics/Counter.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/metrics/Counter.java new file mode 100644 index 0000000000000..4042ff8e5d66e --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/metrics/Counter.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl.metrics; + +import static org.apache.pulsar.client.impl.metrics.MetricsUtil.getDefaultAggregationLabels; +import static org.apache.pulsar.client.impl.metrics.MetricsUtil.getTopicAttributes; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.incubator.metrics.ExtendedLongCounterBuilder; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.LongCounterBuilder; +import io.opentelemetry.api.metrics.Meter; + +public class Counter { + + private final LongCounter counter; + private final Attributes attributes; + + Counter(Meter meter, String name, Unit unit, String description, String topic, Attributes attributes) { + LongCounterBuilder builder = meter.counterBuilder(name) + .setDescription(description) + .setUnit(unit.toString()); + + if (topic != null) { + if (builder instanceof ExtendedLongCounterBuilder) { + ExtendedLongCounterBuilder eb = (ExtendedLongCounterBuilder) builder; + eb.setAttributesAdvice(getDefaultAggregationLabels(attributes)); + } + + attributes = getTopicAttributes(topic, attributes); + } + + this.counter = builder.build(); + this.attributes = attributes; + } + + public void increment() { + add(1); + } + + public void add(int delta) { + counter.add(delta, attributes); + } + +} diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/metrics/InstrumentProvider.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/metrics/InstrumentProvider.java new file mode 100644 index 0000000000000..1e02af1fd37e1 --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/metrics/InstrumentProvider.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.client.impl.metrics; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.Meter; +import org.apache.pulsar.PulsarVersion; + +public class InstrumentProvider { + + public static final InstrumentProvider NOOP = new InstrumentProvider(OpenTelemetry.noop()); + + private final Meter meter; + + public InstrumentProvider(OpenTelemetry otel) { + if (otel == null) { + // By default, metrics are disabled, unless the OTel java agent is configured. + // This allows to enable metrics without any code change. + otel = GlobalOpenTelemetry.get(); + } + this.meter = otel.getMeterProvider() + .meterBuilder("org.apache.pulsar.client") + .setInstrumentationVersion(PulsarVersion.getVersion()) + .build(); + } + + public Counter newCounter(String name, Unit unit, String description, String topic, Attributes attributes) { + return new Counter(meter, name, unit, description, topic, attributes); + } + + public UpDownCounter newUpDownCounter(String name, Unit unit, String description, String topic, + Attributes attributes) { + return new UpDownCounter(meter, name, unit, description, topic, attributes); + } + + public LatencyHistogram newLatencyHistogram(String name, String description, String topic, Attributes attributes) { + return new LatencyHistogram(meter, name, description, topic, attributes); + } +} diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/metrics/LatencyHistogram.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/metrics/LatencyHistogram.java new file mode 100644 index 0000000000000..fdae0a14d65fc --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/metrics/LatencyHistogram.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.client.impl.metrics; + +import static org.apache.pulsar.client.impl.metrics.MetricsUtil.getDefaultAggregationLabels; +import static org.apache.pulsar.client.impl.metrics.MetricsUtil.getTopicAttributes; +import com.google.common.collect.Lists; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.incubator.metrics.ExtendedDoubleHistogramBuilder; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.DoubleHistogramBuilder; +import io.opentelemetry.api.metrics.Meter; +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class LatencyHistogram { + + // Used for tests + public static final LatencyHistogram NOOP = new LatencyHistogram() { + public void recordSuccess(long latencyNanos) { + } + + public void recordFailure(long latencyNanos) { + } + }; + + private static final List latencyHistogramBuckets = + Lists.newArrayList(.0005, .001, .0025, .005, .01, .025, .05, .1, .25, .5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0); + + private static final double NANOS = TimeUnit.SECONDS.toNanos(1); + + private final Attributes successAttributes; + + private final Attributes failedAttributes; + private final DoubleHistogram histogram; + + private LatencyHistogram() { + successAttributes = null; + failedAttributes = null; + histogram = null; + } + + LatencyHistogram(Meter meter, String name, String description, String topic, Attributes attributes) { + DoubleHistogramBuilder builder = meter.histogramBuilder(name) + .setDescription(description) + .setUnit(Unit.Seconds.toString()) + .setExplicitBucketBoundariesAdvice(latencyHistogramBuckets); + + if (topic != null) { + if (builder instanceof ExtendedDoubleHistogramBuilder) { + ExtendedDoubleHistogramBuilder eb = (ExtendedDoubleHistogramBuilder) builder; + eb.setAttributesAdvice( + getDefaultAggregationLabels( + attributes.toBuilder().put("pulsar.response.status", "success").build())); + } + attributes = getTopicAttributes(topic, attributes); + } + + successAttributes = attributes.toBuilder() + .put("pulsar.response.status", "success") + .build(); + failedAttributes = attributes.toBuilder() + .put("pulsar.response.status", "failed") + .build(); + this.histogram = builder.build(); + } + + private LatencyHistogram(DoubleHistogram histogram, Attributes successAttributes, Attributes failedAttributes) { + this.histogram = histogram; + this.successAttributes = successAttributes; + this.failedAttributes = failedAttributes; + } + + /** + * Create a new histograms that inherits the old histograms attributes and adds new ones. + */ + public LatencyHistogram withAttributes(Attributes attributes) { + return new LatencyHistogram( + histogram, + successAttributes.toBuilder().putAll(attributes).build(), + failedAttributes.toBuilder().putAll(attributes).build() + ); + } + + + public void recordSuccess(long latencyNanos) { + histogram.record(latencyNanos / NANOS, successAttributes); + } + + public void recordFailure(long latencyNanos) { + histogram.record(latencyNanos / NANOS, failedAttributes); + } +} diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/metrics/MetricsUtil.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/metrics/MetricsUtil.java new file mode 100644 index 0000000000000..b9802f4f32b5f --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/metrics/MetricsUtil.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.client.impl.metrics; + +import com.google.common.collect.Lists; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import java.util.ArrayList; +import java.util.List; +import lombok.experimental.UtilityClass; +import org.apache.pulsar.common.naming.TopicName; + +@UtilityClass +public class MetricsUtil { + + // By default, advice to use namespace level aggregation only + private static final List> DEFAULT_AGGREGATION_LABELS = Lists.newArrayList( + AttributeKey.stringKey("pulsar.tenant"), + AttributeKey.stringKey("pulsar.namespace") + ); + + static List> getDefaultAggregationLabels(Attributes attrs) { + List> res = new ArrayList<>(); + res.addAll(DEFAULT_AGGREGATION_LABELS); + res.addAll(attrs.asMap().keySet()); + return res; + } + + static Attributes getTopicAttributes(String topic, Attributes baseAttributes) { + TopicName tn = TopicName.get(topic); + + AttributesBuilder ab = baseAttributes.toBuilder(); + if (tn.isPartitioned()) { + ab.put("pulsar.partition", tn.getPartitionIndex()); + } + ab.put("pulsar.topic", tn.getPartitionedTopicName()); + ab.put("pulsar.namespace", tn.getNamespace()); + ab.put("pulsar.tenant", tn.getTenant()); + return ab.build(); + } +} diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/metrics/Unit.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/metrics/Unit.java new file mode 100644 index 0000000000000..5204cc2f03eae --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/metrics/Unit.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.client.impl.metrics; + +public enum Unit { + Bytes, + + Messages, + + Seconds, + + Connections, + + Sessions, + + None, + + ; + + public String toString() { + switch (this) { + case Bytes: + return "By"; + + case Messages: + return "{message}"; + + case Seconds: + return "s"; + + case Connections: + return "{connection}"; + + case Sessions: + return "{session}"; + + case None: + default: + return "1"; + } + } +} diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/metrics/UpDownCounter.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/metrics/UpDownCounter.java new file mode 100644 index 0000000000000..dc2984268cdb6 --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/metrics/UpDownCounter.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.client.impl.metrics; + +import static org.apache.pulsar.client.impl.metrics.MetricsUtil.getDefaultAggregationLabels; +import static org.apache.pulsar.client.impl.metrics.MetricsUtil.getTopicAttributes; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.incubator.metrics.ExtendedLongUpDownCounterBuilder; +import io.opentelemetry.api.metrics.LongUpDownCounter; +import io.opentelemetry.api.metrics.LongUpDownCounterBuilder; +import io.opentelemetry.api.metrics.Meter; + +public class UpDownCounter { + + private final LongUpDownCounter counter; + private final Attributes attributes; + + UpDownCounter(Meter meter, String name, Unit unit, String description, String topic, Attributes attributes) { + LongUpDownCounterBuilder builder = meter.upDownCounterBuilder(name) + .setDescription(description) + .setUnit(unit.toString()); + + if (topic != null) { + if (builder instanceof ExtendedLongUpDownCounterBuilder) { + ExtendedLongUpDownCounterBuilder eb = (ExtendedLongUpDownCounterBuilder) builder; + eb.setAttributesAdvice(getDefaultAggregationLabels(attributes)); + } + + attributes = getTopicAttributes(topic, attributes); + } + + this.counter = builder.build(); + this.attributes = attributes; + } + + public void increment() { + add(1); + } + + public void decrement() { + add(-1); + } + + public void add(long delta) { + counter.add(delta, attributes); + } + + public void subtract(long diff) { + add(-diff); + } +} diff --git a/pulsar-sql/presto-distribution/src/main/java/org/openjdk/jol/info/package-info.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/metrics/package-info.java similarity index 90% rename from pulsar-sql/presto-distribution/src/main/java/org/openjdk/jol/info/package-info.java rename to pulsar-client/src/main/java/org/apache/pulsar/client/impl/metrics/package-info.java index a1b31fc8e2253..ee99bb3332c26 100644 --- a/pulsar-sql/presto-distribution/src/main/java/org/openjdk/jol/info/package-info.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/metrics/package-info.java @@ -16,7 +16,8 @@ * specific language governing permissions and limitations * under the License. */ + /** - * Implementation of the connector to the Presto engine. + * Pulsar Client OTel metrics utilities */ -package org.openjdk.jol.info; +package org.apache.pulsar.client.impl.metrics; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoProduceBytesSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoProduceBytesSchema.java index f26de02c87494..f5925f65a6047 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoProduceBytesSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoProduceBytesSchema.java @@ -105,7 +105,10 @@ public SchemaInfo getSchemaInfo() { @Override public Optional getNativeSchema() { - return Optional.ofNullable(schema); + return Optional + .ofNullable(schema) + .map(s->s.getNativeSchema()) + .orElse(Optional.empty()); } @Override diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/NativeAvroBytesSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/NativeAvroBytesSchema.java index c97ef0e60e834..b29141a9dcda3 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/NativeAvroBytesSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/NativeAvroBytesSchema.java @@ -18,11 +18,16 @@ */ package org.apache.pulsar.client.impl.schema; -import static com.google.common.base.Preconditions.checkState; +import static org.apache.pulsar.client.impl.schema.SchemaDefinitionBuilderImpl.ALWAYS_ALLOW_NULL; +import static org.apache.pulsar.client.impl.schema.SchemaDefinitionBuilderImpl.JSR310_CONVERSION_ENABLED; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; import java.util.Optional; import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.schema.SchemaType; /** * Schema from a native Apache Avro schema. @@ -33,28 +38,27 @@ * This class also makes it possible for users to bring in their own Avro serialization method. */ public class NativeAvroBytesSchema implements Schema { - - private Schema schema; - private org.apache.avro.Schema nativeSchema; + private final org.apache.avro.Schema nativeSchema; + private final SchemaInfo schemaInfo; public NativeAvroBytesSchema(org.apache.avro.Schema schema) { - setSchema(schema); + Objects.requireNonNull(schema, "Avro schema cannot be null"); + this.nativeSchema = schema; + Map properties = new HashMap<>(); + properties.put(ALWAYS_ALLOW_NULL, "true"); + properties.put(JSR310_CONVERSION_ENABLED, "false"); + this.schemaInfo = SchemaInfo.builder() + .name("") + .schema(schema.toString().getBytes(StandardCharsets.UTF_8)) + .properties(properties) + .type(SchemaType.AVRO) + .build(); } public NativeAvroBytesSchema(Object schema) { this(validateSchema(schema)); } - public void setSchema(org.apache.avro.Schema schema) { - SchemaDefinition schemaDefinition = SchemaDefinition.builder().withJsonDef(schema.toString(false)).build(); - this.nativeSchema = schema; - this.schema = AvroSchema.of(schemaDefinition); - } - - public boolean schemaInitialized() { - return schema != null; - } - private static org.apache.avro.Schema validateSchema (Object schema) { if (!(schema instanceof org.apache.avro.Schema)) { throw new IllegalArgumentException("The input schema is not of type 'org.apache.avro.Schema'."); @@ -62,14 +66,8 @@ private static org.apache.avro.Schema validateSchema (Object schema) { return (org.apache.avro.Schema) schema; } - private void ensureSchemaInitialized() { - checkState(schemaInitialized(), "Schema is not initialized before used"); - } - @Override public byte[] encode(byte[] message) { - ensureSchemaInitialized(); - return message; } @@ -81,9 +79,7 @@ public byte[] decode(byte[] bytes, byte[] schemaVersion) { @Override public SchemaInfo getSchemaInfo() { - ensureSchemaInitialized(); - - return schema.getSchemaInfo(); + return schemaInfo; } @Override diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericProtobufNativeSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericProtobufNativeSchema.java index 4ae2a21929ad1..62a36fee35147 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericProtobufNativeSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericProtobufNativeSchema.java @@ -20,6 +20,7 @@ import static org.apache.pulsar.client.impl.schema.generic.MultiVersionGenericProtobufNativeReader.parseProtobufSchema; import com.google.protobuf.Descriptors; +import java.util.Optional; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.schema.Field; @@ -68,6 +69,11 @@ public Descriptors.Descriptor getProtobufNativeSchema() { return descriptor; } + @Override + public Optional getNativeSchema() { + return Optional.of(descriptor); + } + @Override public boolean supportSchemaVersioning() { return true; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/reader/MultiVersionAvroReader.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/reader/MultiVersionAvroReader.java index 0ca847917eeca..85d4d63a1b136 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/reader/MultiVersionAvroReader.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/reader/MultiVersionAvroReader.java @@ -44,9 +44,11 @@ public MultiVersionAvroReader(Schema readerSchema, ClassLoader pojoClassLoader, protected SchemaReader loadReader(BytesSchemaVersion schemaVersion) { SchemaInfo schemaInfo = getSchemaInfoByVersion(schemaVersion.get()); if (schemaInfo != null) { - LOG.info("Load schema reader for version({}), schema is : {}, schemaInfo: {}", - SchemaUtils.getStringSchemaVersion(schemaVersion.get()), - schemaInfo.getSchemaDefinition(), schemaInfo.toString()); + if (LOG.isDebugEnabled()) { + LOG.debug("Load schema reader for version({}), schema is : {}, schemaInfo: {}", + SchemaUtils.getStringSchemaVersion(schemaVersion.get()), + schemaInfo.getSchemaDefinition(), schemaInfo); + } boolean jsr310ConversionEnabled = getJsr310ConversionEnabledFromSchemaInfo(schemaInfo); return new AvroReader<>(parseAvroSchema(schemaInfo.getSchemaDefinition()), readerSchema, pojoClassLoader, jsr310ConversionEnabled); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBuilderImpl.java index c5e9d4781c56f..0ebfb91e62da7 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBuilderImpl.java @@ -57,7 +57,7 @@ public CompletableFuture build() { new PulsarClientException.InvalidConfigurationException("Transactions are not enabled")); } // talk to TC to begin a transaction - // the builder is responsible for locating the transaction coorindator (TC) + // the builder is responsible for locating the transaction coordinator (TC) // and start the transaction to get the transaction id. // After getting the transaction id, all the operations are handled by the // `TransactionImpl` diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionCoordinatorClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionCoordinatorClientImpl.java index 9e79fc203c225..ce19cbf873eea 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionCoordinatorClientImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionCoordinatorClientImpl.java @@ -79,7 +79,8 @@ public void start() throws TransactionCoordinatorClientException { @Override public CompletableFuture startAsync() { if (STATE_UPDATER.compareAndSet(this, State.NONE, State.STARTING)) { - return pulsarClient.getLookup().getPartitionedTopicMetadata(SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN) + return pulsarClient.getPartitionedTopicMetadata( + SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN.getPartitionedTopicName(), true, false) .thenCompose(partitionMeta -> { List> connectFutureList = new ArrayList<>(); if (LOG.isDebugEnabled()) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java index d1260ba045e6d..a88d65fce3100 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java @@ -106,18 +106,16 @@ public void run(Timeout timeout) throws Exception { public CompletableFuture registerProducedTopic(String topic) { CompletableFuture completableFuture = new CompletableFuture<>(); if (checkIfOpen(completableFuture)) { - synchronized (TransactionImpl.this) { - // we need to issue the request to TC to register the produced topic - return registerPartitionMap.compute(topic, (key, future) -> { - if (future != null) { - return future.thenCompose(ignored -> CompletableFuture.completedFuture(null)); - } else { - return tcClient.addPublishPartitionToTxnAsync( - txnId, Lists.newArrayList(topic)) - .thenCompose(ignored -> CompletableFuture.completedFuture(null)); - } - }); - } + // we need to issue the request to TC to register the produced topic + return registerPartitionMap.compute(topic, (key, future) -> { + if (future != null) { + return future.thenCompose(ignored -> CompletableFuture.completedFuture(null)); + } else { + return tcClient.addPublishPartitionToTxnAsync( + txnId, Lists.newArrayList(topic)) + .thenCompose(ignored -> CompletableFuture.completedFuture(null)); + } + }); } return completableFuture; } @@ -147,18 +145,16 @@ public void registerSendOp(CompletableFuture newSendFuture) { public CompletableFuture registerAckedTopic(String topic, String subscription) { CompletableFuture completableFuture = new CompletableFuture<>(); if (checkIfOpen(completableFuture)) { - synchronized (TransactionImpl.this) { - // we need to issue the request to TC to register the acked topic - return registerSubscriptionMap.compute(Pair.of(topic, subscription), (key, future) -> { - if (future != null) { - return future.thenCompose(ignored -> CompletableFuture.completedFuture(null)); - } else { - return tcClient.addSubscriptionToTxnAsync( - txnId, topic, subscription) - .thenCompose(ignored -> CompletableFuture.completedFuture(null)); - } - }); - } + // we need to issue the request to TC to register the acked topic + return registerSubscriptionMap.compute(Pair.of(topic, subscription), (key, future) -> { + if (future != null) { + return future.thenCompose(ignored -> CompletableFuture.completedFuture(null)); + } else { + return tcClient.addSubscriptionToTxnAsync( + txnId, topic, subscription) + .thenCompose(ignored -> CompletableFuture.completedFuture(null)); + } + }); } return completableFuture; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/util/MessageIdUtils.java b/pulsar-client/src/main/java/org/apache/pulsar/client/util/MessageIdUtils.java index 60cdad8e77200..459a31ee72022 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/util/MessageIdUtils.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/util/MessageIdUtils.java @@ -22,6 +22,7 @@ import org.apache.pulsar.client.impl.MessageIdImpl; public class MessageIdUtils { + @Deprecated public static final long getOffset(MessageId messageId) { MessageIdImpl msgId = (MessageIdImpl) messageId; long ledgerId = msgId.getLedgerId(); @@ -34,6 +35,7 @@ public static final long getOffset(MessageId messageId) { return offset; } + @Deprecated public static final MessageId getMessageId(long offset) { // Demultiplex ledgerId and entryId from offset long ledgerId = offset >>> 28; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/util/WithSNISslEngineFactory.java b/pulsar-client/src/main/java/org/apache/pulsar/client/util/PulsarHttpAsyncSslEngineFactory.java similarity index 53% rename from pulsar-client/src/main/java/org/apache/pulsar/client/util/WithSNISslEngineFactory.java rename to pulsar-client/src/main/java/org/apache/pulsar/client/util/PulsarHttpAsyncSslEngineFactory.java index d950e68271bcd..ddf034bbb098e 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/util/WithSNISslEngineFactory.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/util/PulsarHttpAsyncSslEngineFactory.java @@ -20,23 +20,42 @@ import java.util.Collections; import javax.net.ssl.SNIHostName; +import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLParameters; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.asynchttpclient.AsyncHttpClientConfig; import org.asynchttpclient.netty.ssl.DefaultSslEngineFactory; -public class WithSNISslEngineFactory extends DefaultSslEngineFactory { +public class PulsarHttpAsyncSslEngineFactory extends DefaultSslEngineFactory { + + private final PulsarSslFactory pulsarSslFactory; private final String host; - public WithSNISslEngineFactory(String host) { + public PulsarHttpAsyncSslEngineFactory(PulsarSslFactory pulsarSslFactory, String host) { + this.pulsarSslFactory = pulsarSslFactory; this.host = host; } @Override protected void configureSslEngine(SSLEngine sslEngine, AsyncHttpClientConfig config) { super.configureSslEngine(sslEngine, config); - SSLParameters params = sslEngine.getSSLParameters(); - params.setServerNames(Collections.singletonList(new SNIHostName(host))); - sslEngine.setSSLParameters(params); + if (StringUtils.isNotBlank(host)) { + SSLParameters parameters = sslEngine.getSSLParameters(); + parameters.setServerNames(Collections.singletonList(new SNIHostName(host))); + sslEngine.setSSLParameters(parameters); + } } -} + + @Override + public SSLEngine newSslEngine(AsyncHttpClientConfig config, String peerHost, int peerPort) { + SSLContext sslContext = this.pulsarSslFactory.getInternalSslContext(); + SSLEngine sslEngine = config.isDisableHttpsEndpointIdentificationAlgorithm() + ? sslContext.createSSLEngine() : + sslContext.createSSLEngine(domain(peerHost), peerPort); + configureSslEngine(sslEngine, config); + return sslEngine; + } + +} \ No newline at end of file diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/util/RetryMessageUtil.java b/pulsar-client/src/main/java/org/apache/pulsar/client/util/RetryMessageUtil.java index f73c266877988..e9071f171a29e 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/util/RetryMessageUtil.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/util/RetryMessageUtil.java @@ -23,6 +23,7 @@ public class RetryMessageUtil { public static final String SYSTEM_PROPERTY_RECONSUMETIMES = "RECONSUMETIMES"; public static final String SYSTEM_PROPERTY_DELAY_TIME = "DELAY_TIME"; public static final String SYSTEM_PROPERTY_REAL_TOPIC = "REAL_TOPIC"; + public static final String SYSTEM_PROPERTY_REAL_SUBSCRIPTION = "REAL_SUBSCRIPTION"; public static final String SYSTEM_PROPERTY_RETRY_TOPIC = "RETRY_TOPIC"; @Deprecated public static final String SYSTEM_PROPERTY_ORIGIN_MESSAGE_ID = "ORIGIN_MESSAGE_IDY_TIME"; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/util/RetryUtil.java b/pulsar-client/src/main/java/org/apache/pulsar/client/util/RetryUtil.java index 93501d7b6c18b..912cb7d7c5832 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/util/RetryUtil.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/util/RetryUtil.java @@ -22,7 +22,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; -import org.apache.pulsar.client.impl.Backoff; +import org.apache.pulsar.common.util.Backoff; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/pulsar-client/src/main/resources/findbugsExclude.xml b/pulsar-client/src/main/resources/findbugsExclude.xml index 92ec9e934ee1e..0e05d20cb9bb4 100644 --- a/pulsar-client/src/main/resources/findbugsExclude.xml +++ b/pulsar-client/src/main/resources/findbugsExclude.xml @@ -337,6 +337,11 @@ + + + + + @@ -387,6 +392,11 @@ + + + + + @@ -427,6 +437,11 @@ + + + + + @@ -447,6 +462,11 @@ + + + + + @@ -812,9 +832,10 @@ + - + @@ -1012,4 +1033,14 @@ + + + + + + + + + + diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java index 0418a54c772cc..a62d9e7479852 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java @@ -41,10 +41,10 @@ import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; import org.apache.pulsar.client.util.TimedCompletableFuture; import org.apache.pulsar.common.api.proto.CommandAck.AckType; import org.apache.pulsar.common.util.collections.ConcurrentBitSetRecyclable; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.common.api.proto.ProtocolVersion; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -61,14 +61,15 @@ public class AcknowledgementsGroupingTrackerTest { public void setup() throws NoSuchFieldException, IllegalAccessException { eventLoopGroup = new NioEventLoopGroup(1); consumer = mock(ConsumerImpl.class); - consumer.unAckedChunkedMessageIdSequenceMap = - ConcurrentOpenHashMap.newBuilder().build(); + consumer.unAckedChunkedMessageIdSequenceMap = new ConcurrentHashMap<>(); cnx = spy(new ClientCnxTest(new ClientConfigurationData(), eventLoopGroup)); PulsarClientImpl client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); doReturn(client).when(consumer).getClient(); doReturn(cnx).when(consumer).getClientCnx(); doReturn(new ConsumerStatsRecorderImpl()).when(consumer).getStats(); - doReturn(new UnAckedMessageTracker().UNACKED_MESSAGE_TRACKER_DISABLED) + doReturn(UnAckedMessageTracker.UNACKED_MESSAGE_TRACKER_DISABLED) .when(consumer).getUnAckedMessageTracker(); ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); when(cnx.ctx()).thenReturn(ctx); @@ -421,7 +422,7 @@ public void testDoIndividualBatchAckAsync() throws Exception{ public class ClientCnxTest extends ClientCnx { public ClientCnxTest(ClientConfigurationData conf, EventLoopGroup eventLoopGroup) { - super(conf, eventLoopGroup); + super(InstrumentProvider.NOOP, conf, eventLoopGroup); } @Override diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AutoClusterFailoverTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AutoClusterFailoverTest.java index 36ffa30296bb0..b275ffb6012ca 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AutoClusterFailoverTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AutoClusterFailoverTest.java @@ -19,18 +19,18 @@ package org.apache.pulsar.client.impl; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; -import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.AuthenticationFactory; -import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.ServiceUrlProvider; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.awaitility.Awaitility; @@ -41,12 +41,13 @@ @Slf4j public class AutoClusterFailoverTest { @Test - public void testBuildAutoClusterFailoverInstance() throws PulsarClientException { + public void testBuildAutoClusterFailoverInstance() throws Exception { String primary = "pulsar://localhost:6650"; String secondary = "pulsar://localhost:6651"; long failoverDelay = 30; long switchBackDelay = 60; long checkInterval = 1_000; + @Cleanup ServiceUrlProvider provider = AutoClusterFailover.builder() .primary(primary) .secondary(Collections.singletonList(secondary)) @@ -85,6 +86,7 @@ public void testBuildAutoClusterFailoverInstance() throws PulsarClientException Map secondaryAuthentications = new HashMap<>(); secondaryAuthentications.put(secondary, secondaryAuthentication); + @Cleanup ServiceUrlProvider provider1 = AutoClusterFailover.builder() .primary(primary) .secondary(Collections.singletonList(secondary)) @@ -102,7 +104,7 @@ public void testBuildAutoClusterFailoverInstance() throws PulsarClientException } @Test - public void testInitialize() { + public void testInitialize() throws Exception { String primary = "pulsar://localhost:6650"; String secondary = "pulsar://localhost:6651"; long failoverDelay = 10; @@ -111,6 +113,7 @@ public void testInitialize() { ClientConfigurationData configurationData = new ClientConfigurationData(); + @Cleanup ServiceUrlProvider provider = AutoClusterFailover.builder() .primary(primary) .secondary(Collections.singletonList(secondary)) @@ -121,6 +124,8 @@ public void testInitialize() { AutoClusterFailover autoClusterFailover = Mockito.spy((AutoClusterFailover) provider); PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(pulsarClient.getCnxPool()).thenReturn(connectionPool); Mockito.doReturn(false).when(autoClusterFailover).probeAvailable(primary); Mockito.doReturn(true).when(autoClusterFailover).probeAvailable(secondary); Mockito.doReturn(configurationData).when(pulsarClient).getConfiguration(); @@ -144,7 +149,7 @@ public void testInitialize() { } @Test - public void testAutoClusterFailoverSwitchWithoutAuthentication() { + public void testAutoClusterFailoverSwitchWithoutAuthentication() throws Exception { String primary = "pulsar://localhost:6650"; String secondary = "pulsar://localhost:6651"; long failoverDelay = 1; @@ -153,6 +158,7 @@ public void testAutoClusterFailoverSwitchWithoutAuthentication() { ClientConfigurationData configurationData = new ClientConfigurationData(); + @Cleanup ServiceUrlProvider provider = AutoClusterFailover.builder() .primary(primary) .secondary(Collections.singletonList(secondary)) @@ -163,6 +169,8 @@ public void testAutoClusterFailoverSwitchWithoutAuthentication() { AutoClusterFailover autoClusterFailover = Mockito.spy((AutoClusterFailover) provider); PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(pulsarClient.getCnxPool()).thenReturn(connectionPool); Mockito.doReturn(false).when(autoClusterFailover).probeAvailable(primary); Mockito.doReturn(true).when(autoClusterFailover).probeAvailable(secondary); Mockito.doReturn(configurationData).when(pulsarClient).getConfiguration(); @@ -177,7 +185,7 @@ public void testAutoClusterFailoverSwitchWithoutAuthentication() { } @Test - public void testAutoClusterFailoverSwitchWithAuthentication() throws IOException { + public void testAutoClusterFailoverSwitchWithAuthentication() throws Exception { String primary = "pulsar+ssl://localhost:6651"; String secondary = "pulsar+ssl://localhost:6661"; long failoverDelay = 1; @@ -205,6 +213,7 @@ public void testAutoClusterFailoverSwitchWithAuthentication() throws IOException configurationData.setTlsTrustCertsFilePath(primaryTlsTrustCertsFilePath); configurationData.setAuthentication(primaryAuthentication); + @Cleanup ServiceUrlProvider provider = AutoClusterFailover.builder() .primary(primary) .secondary(Collections.singletonList(secondary)) @@ -217,6 +226,8 @@ public void testAutoClusterFailoverSwitchWithAuthentication() throws IOException AutoClusterFailover autoClusterFailover = Mockito.spy((AutoClusterFailover) provider); PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(pulsarClient.getCnxPool()).thenReturn(connectionPool); Mockito.doReturn(false).when(autoClusterFailover).probeAvailable(primary); Mockito.doReturn(true).when(autoClusterFailover).probeAvailable(secondary); Mockito.doReturn(configurationData).when(pulsarClient).getConfiguration(); @@ -238,7 +249,7 @@ public void testAutoClusterFailoverSwitchWithAuthentication() throws IOException } @Test - public void testAutoClusterFailoverSwitchTlsTrustStore() throws IOException { + public void testAutoClusterFailoverSwitchTlsTrustStore() throws Exception { String primary = "pulsar+ssl://localhost:6651"; String secondary = "pulsar+ssl://localhost:6661"; long failoverDelay = 1; @@ -258,6 +269,7 @@ public void testAutoClusterFailoverSwitchTlsTrustStore() throws IOException { configurationData.setTlsTrustStorePath(primaryTlsTrustStorePath); configurationData.setTlsTrustStorePassword(primaryTlsTrustStorePassword); + @Cleanup ServiceUrlProvider provider = AutoClusterFailover.builder() .primary(primary) .secondary(Collections.singletonList(secondary)) @@ -270,6 +282,8 @@ public void testAutoClusterFailoverSwitchTlsTrustStore() throws IOException { AutoClusterFailover autoClusterFailover = Mockito.spy((AutoClusterFailover) provider); PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(pulsarClient.getCnxPool()).thenReturn(connectionPool); Mockito.doReturn(false).when(autoClusterFailover).probeAvailable(primary); Mockito.doReturn(true).when(autoClusterFailover).probeAvailable(secondary); Mockito.doReturn(configurationData).when(pulsarClient).getConfiguration(); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageContainerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageContainerImplTest.java index 4b80e19c256d7..abb195c9830d0 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageContainerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageContainerImplTest.java @@ -105,6 +105,8 @@ public void recoveryAfterOom() { final ProducerConfigurationData producerConfigurationData = new ProducerConfigurationData(); producerConfigurationData.setCompressionType(CompressionType.NONE); PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(pulsarClient.getCnxPool()).thenReturn(connectionPool); MemoryLimitController memoryLimitController = mock(MemoryLimitController.class); when(pulsarClient.getMemoryLimitController()).thenReturn(memoryLimitController); try { @@ -148,6 +150,8 @@ public void testMessagesSize() throws Exception { final ProducerConfigurationData producerConfigurationData = new ProducerConfigurationData(); producerConfigurationData.setCompressionType(CompressionType.NONE); PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(pulsarClient.getCnxPool()).thenReturn(connectionPool); MemoryLimitController memoryLimitController = mock(MemoryLimitController.class); when(pulsarClient.getMemoryLimitController()).thenReturn(memoryLimitController); try { diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BinaryProtoLookupServiceTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BinaryProtoLookupServiceTest.java index 0254cf8d44cea..f691215b04e08 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BinaryProtoLookupServiceTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BinaryProtoLookupServiceTest.java @@ -27,19 +27,16 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - import io.netty.buffer.ByteBuf; - import java.lang.reflect.Field; import java.net.InetSocketAddress; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; - -import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.client.api.PulsarClientException.LookupException; import org.apache.pulsar.client.impl.BinaryProtoLookupService.LookupDataResult; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; import org.apache.pulsar.common.naming.TopicName; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -69,9 +66,12 @@ public void setup() throws Exception { doReturn(0).when(clientConfig).getMaxLookupRedirects(); PulsarClientImpl client = mock(PulsarClientImpl.class); + doReturn(InstrumentProvider.NOOP).when(client).instrumentProvider(); doReturn(cnxPool).when(client).getCnxPool(); doReturn(clientConfig).when(client).getConfiguration(); doReturn(1L).when(client).newRequestId(); + ClientConfigurationData data = new ClientConfigurationData(); + doReturn(data).when(client).getConfiguration(); lookup = spy( new BinaryProtoLookupService(client, "pulsar://localhost:6650", false, mock(ExecutorService.class))); @@ -80,11 +80,12 @@ public void setup() throws Exception { @Test(invocationTimeOut = 3000) public void maxLookupRedirectsTest1() throws Exception { - Pair addressPair = lookup.getBroker(topicName).get(); - assertEquals(addressPair.getLeft(), InetSocketAddress + LookupTopicResult lookupResult = lookup.getBroker(topicName).get(); + assertEquals(lookupResult.getLogicalAddress(), InetSocketAddress .createUnresolved("broker2.pulsar.apache.org" ,6650)); - assertEquals(addressPair.getRight(), InetSocketAddress + assertEquals(lookupResult.getPhysicalAddress(), InetSocketAddress .createUnresolved("broker2.pulsar.apache.org" ,6650)); + assertEquals(lookupResult.isUseProxy(), false); } @Test(invocationTimeOut = 3000) @@ -93,11 +94,12 @@ public void maxLookupRedirectsTest2() throws Exception { field.setAccessible(true); field.set(lookup, 2); - Pair addressPair = lookup.getBroker(topicName).get(); - assertEquals(addressPair.getLeft(), InetSocketAddress + LookupTopicResult lookupResult = lookup.getBroker(topicName).get(); + assertEquals(lookupResult.getLogicalAddress(), InetSocketAddress .createUnresolved("broker2.pulsar.apache.org" ,6650)); - assertEquals(addressPair.getRight(), InetSocketAddress + assertEquals(lookupResult.getPhysicalAddress(), InetSocketAddress .createUnresolved("broker2.pulsar.apache.org" ,6650)); + assertEquals(lookupResult.isUseProxy(), false); } @Test(invocationTimeOut = 3000) diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BuildersTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BuildersTest.java index 607689e0e2b3b..5f52f86d8b014 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BuildersTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BuildersTest.java @@ -106,7 +106,7 @@ public void readerBuilderLoadConfTest() throws Exception { @Test(expectedExceptions = {PulsarClientException.class}, expectedExceptionsMessageRegExp = ".* must be specified but they cannot be specified at the same time.*") public void shouldNotSetTwoOptAtTheSameTime() throws Exception { PulsarClient client = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build(); - try (Reader reader = client.newReader().topic("abc").startMessageId(MessageId.earliest) + try (Reader reader = client.newReader().topic("abc").startMessageId(MessageId.latest) .startMessageFromRollbackDuration(10, TimeUnit.HOURS).create()) { // no-op } finally { diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientBuilderImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientBuilderImplTest.java index 9a39c906b8ff6..2f9c7536d753d 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientBuilderImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientBuilderImplTest.java @@ -23,6 +23,8 @@ import com.google.gson.Gson; import com.google.gson.JsonObject; import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -101,6 +103,18 @@ public void testClientBuilderWithIllegalLargePort() throws PulsarClientException PulsarClient.builder().dnsLookupBind("localhost", 65536).build(); } + @Test(expectedExceptions = IllegalArgumentException.class) + public void testClientBuilderWithIllegalDNSServerHostname() throws PulsarClientException { + PulsarClient.builder().dnsServerAddresses( + Arrays.asList(new InetSocketAddress("1.2.3.4", 53), new InetSocketAddress("localhost",53))); + } + + @Test() + public void testClientBuilderWithDNSServerIP() throws PulsarClientException { + PulsarClient.builder().dnsServerAddresses( + Arrays.asList(new InetSocketAddress("1.2.3.4", 53))); + } + @Test public void testConnectionMaxIdleSeconds() throws Exception { // test config disabled. @@ -109,7 +123,7 @@ public void testConnectionMaxIdleSeconds() throws Exception { PulsarClient.builder().connectionMaxIdleSeconds(60); // test config not correct. try { - PulsarClient.builder().connectionMaxIdleSeconds(30); + PulsarClient.builder().connectionMaxIdleSeconds(14); fail(); } catch (IllegalArgumentException e){ } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxRequestTimeoutQueueTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxRequestTimeoutQueueTest.java index ca6114d2ed823..d573229fddefa 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxRequestTimeoutQueueTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxRequestTimeoutQueueTest.java @@ -26,6 +26,7 @@ import io.netty.util.concurrent.DefaultThreadFactory; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; import org.apache.pulsar.client.util.TimedCompletableFuture; import org.apache.pulsar.common.util.netty.EventLoopUtil; @@ -60,7 +61,7 @@ void setupClientCnx() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); conf.setKeepAliveIntervalSeconds(0); conf.setOperationTimeoutMs(1); - cnx = new ClientCnx(conf, eventLoop); + cnx = new ClientCnx(InstrumentProvider.NOOP, conf, eventLoop); ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); Channel channel = mock(Channel.class); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java index 22220805814f5..bc1d940c76bbf 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java @@ -33,6 +33,7 @@ import io.netty.channel.EventLoopGroup; import io.netty.util.concurrent.DefaultThreadFactory; import java.lang.reflect.Field; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; @@ -41,6 +42,7 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.PulsarClientException.BrokerMetadataException; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; import org.apache.pulsar.common.api.proto.CommandCloseConsumer; import org.apache.pulsar.common.api.proto.CommandCloseProducer; import org.apache.pulsar.common.api.proto.CommandConnected; @@ -62,7 +64,7 @@ public void testClientCnxTimeout() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); conf.setOperationTimeoutMs(10); conf.setKeepAliveIntervalSeconds(0); - ClientCnx cnx = new ClientCnx(conf, eventLoop); + ClientCnx cnx = new ClientCnx(InstrumentProvider.NOOP, conf, eventLoop); ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); Channel channel = mock(Channel.class); @@ -88,7 +90,7 @@ public void testPendingLookupRequestSemaphore() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); conf.setOperationTimeoutMs(10_000); conf.setKeepAliveIntervalSeconds(0); - ClientCnx cnx = new ClientCnx(conf, eventLoop); + ClientCnx cnx = new ClientCnx(InstrumentProvider.NOOP, conf, eventLoop); ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); Channel channel = mock(Channel.class); @@ -126,7 +128,7 @@ public void testPendingLookupRequestSemaphoreServiceNotReady() throws Exception ClientConfigurationData conf = new ClientConfigurationData(); conf.setOperationTimeoutMs(10_000); conf.setKeepAliveIntervalSeconds(0); - ClientCnx cnx = new ClientCnx(conf, eventLoop); + ClientCnx cnx = new ClientCnx(InstrumentProvider.NOOP, conf, eventLoop); ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); Channel channel = mock(Channel.class); @@ -169,7 +171,7 @@ public void testPendingWaitingLookupRequestSemaphore() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); conf.setOperationTimeoutMs(10_000); conf.setKeepAliveIntervalSeconds(0); - ClientCnx cnx = new ClientCnx(conf, eventLoop); + ClientCnx cnx = new ClientCnx(InstrumentProvider.NOOP, conf, eventLoop); ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); Channel channel = mock(Channel.class); @@ -195,7 +197,7 @@ public void testReceiveErrorAtSendConnectFrameState() throws Exception { EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, threadFactory); ClientConfigurationData conf = new ClientConfigurationData(); conf.setOperationTimeoutMs(10); - ClientCnx cnx = new ClientCnx(conf, eventLoop); + ClientCnx cnx = new ClientCnx(InstrumentProvider.NOOP, conf, eventLoop); ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); Channel channel = mock(Channel.class); @@ -229,7 +231,7 @@ public void testGetLastMessageIdWithError() throws Exception { ThreadFactory threadFactory = new DefaultThreadFactory("testReceiveErrorAtSendConnectFrameState"); EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, threadFactory); ClientConfigurationData conf = new ClientConfigurationData(); - ClientCnx cnx = new ClientCnx(conf, eventLoop); + ClientCnx cnx = new ClientCnx(InstrumentProvider.NOOP, conf, eventLoop); ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); Channel channel = mock(Channel.class); @@ -275,16 +277,22 @@ public void testHandleCloseConsumer() { ThreadFactory threadFactory = new DefaultThreadFactory("testHandleCloseConsumer"); EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, threadFactory); ClientConfigurationData conf = new ClientConfigurationData(); - ClientCnx cnx = new ClientCnx(conf, eventLoop); + ClientCnx cnx = new ClientCnx(InstrumentProvider.NOOP, conf, eventLoop); long consumerId = 1; - cnx.registerConsumer(consumerId, mock(ConsumerImpl.class)); + PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + when(pulsarClient.getConfiguration()).thenReturn(conf); + ConsumerImpl consumer = mock(ConsumerImpl.class); + when(consumer.getClient()).thenReturn(pulsarClient); + cnx.registerConsumer(consumerId, consumer); assertEquals(cnx.consumers.size(), 1); - CommandCloseConsumer closeConsumer = new CommandCloseConsumer().setConsumerId(consumerId); + CommandCloseConsumer closeConsumer = new CommandCloseConsumer().setConsumerId(consumerId).setRequestId(1); cnx.handleCloseConsumer(closeConsumer); assertEquals(cnx.consumers.size(), 0); + verify(consumer).connectionClosed(cnx, Optional.empty(), Optional.empty()); + eventLoop.shutdownGracefully(); } @@ -293,16 +301,22 @@ public void testHandleCloseProducer() { ThreadFactory threadFactory = new DefaultThreadFactory("testHandleCloseProducer"); EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, threadFactory); ClientConfigurationData conf = new ClientConfigurationData(); - ClientCnx cnx = new ClientCnx(conf, eventLoop); + ClientCnx cnx = new ClientCnx(InstrumentProvider.NOOP, conf, eventLoop); long producerId = 1; - cnx.registerProducer(producerId, mock(ProducerImpl.class)); + PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + when(pulsarClient.getConfiguration()).thenReturn(conf); + ProducerImpl producer = mock(ProducerImpl.class); + when(producer.getClient()).thenReturn(pulsarClient); + cnx.registerProducer(producerId, producer); assertEquals(cnx.producers.size(), 1); - CommandCloseProducer closeProducerCmd = new CommandCloseProducer().setProducerId(producerId); + CommandCloseProducer closeProducerCmd = new CommandCloseProducer().setProducerId(producerId).setRequestId(1); cnx.handleCloseProducer(closeProducerCmd); assertEquals(cnx.producers.size(), 0); + verify(producer).connectionClosed(cnx, Optional.empty(), Optional.empty()); + eventLoop.shutdownGracefully(); } @@ -380,7 +394,7 @@ private void withConnection(String testName, Consumer test) { try { ClientConfigurationData conf = new ClientConfigurationData(); - ClientCnx cnx = new ClientCnx(conf, eventLoop); + ClientCnx cnx = new ClientCnx(InstrumentProvider.NOOP, conf, eventLoop); ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); Channel channel = mock(Channel.class); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientInitializationTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientInitializationTest.java index 3b92f362ca188..2682d011cd0c5 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientInitializationTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientInitializationTest.java @@ -19,8 +19,8 @@ package org.apache.pulsar.client.impl; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; import lombok.Cleanup; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.PulsarClient; @@ -40,9 +40,7 @@ public void testInitializeAuthWithTls() throws PulsarClientException { .authentication(auth) .build(); - // Auth should only be started, though we shouldn't have tried to get credentials yet (until we first attempt to - // connect). verify(auth).start(); - verifyNoMoreInteractions(auth); + verify(auth, times(1)).getAuthData(); } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientTestFixtures.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientTestFixtures.java index 4db5dbe877685..915c3dcc05a7e 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientTestFixtures.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientTestFixtures.java @@ -19,9 +19,12 @@ package org.apache.pulsar.client.impl; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.EventLoop; import io.netty.util.Timer; @@ -31,12 +34,18 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.client.util.ExecutorProvider; import org.mockito.Mockito; class ClientTestFixtures { - public static ScheduledExecutorService SCHEDULER = Executors.newSingleThreadScheduledExecutor(); + public static ScheduledExecutorService SCHEDULER = + Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder() + .setNameFormat("ClientTestFixtures-SCHEDULER-%d") + .setDaemon(true) + .build()); // static PulsarClientImpl createPulsarClientMock() { // return createPulsarClientMock(mock(ExecutorService.class)); @@ -72,11 +81,16 @@ static PulsarClientImpl mockClientCnx(PulsarClientImpl clientMock) { .thenReturn(CompletableFuture.completedFuture(mock(ProducerResponse.class))); when(clientCnxMock.channel().remoteAddress()).thenReturn(mock(SocketAddress.class)); when(clientMock.getConnection(any())).thenReturn(CompletableFuture.completedFuture(clientCnxMock)); - when(clientMock.getConnection(any(), any())).thenReturn(CompletableFuture.completedFuture(clientCnxMock)); + when(clientMock.getConnection(anyString())).thenReturn(CompletableFuture.completedFuture(clientCnxMock)); + when(clientMock.getConnection(anyString(), anyInt())) + .thenReturn(CompletableFuture.completedFuture(Pair.of(clientCnxMock, false))); + when(clientMock.getConnection(any(), any(), anyInt())) + .thenReturn(CompletableFuture.completedFuture(clientCnxMock)); ConnectionPool connectionPoolMock = mock(ConnectionPool.class); when(clientMock.getCnxPool()).thenReturn(connectionPoolMock); when(connectionPoolMock.getConnection(any())).thenReturn(CompletableFuture.completedFuture(clientCnxMock)); - when(connectionPoolMock.getConnection(any(), any())).thenReturn(CompletableFuture.completedFuture(clientCnxMock)); + when(connectionPoolMock.getConnection(any(), any(), anyInt())) + .thenReturn(CompletableFuture.completedFuture(clientCnxMock)); return clientMock; } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerBuilderImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerBuilderImplTest.java index 8dbd23f9c29c9..e4b7b4d1ec85e 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerBuilderImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerBuilderImplTest.java @@ -22,8 +22,13 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import com.fasterxml.jackson.annotation.JsonIgnoreType; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.ArrayList; @@ -44,8 +49,11 @@ import org.apache.pulsar.client.api.CryptoKeyReader; import org.apache.pulsar.client.api.DeadLetterPolicy; import org.apache.pulsar.client.api.KeySharedPolicy; +import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageCrypto; import org.apache.pulsar.client.api.MessageListener; +import org.apache.pulsar.client.api.MessagePayload; +import org.apache.pulsar.client.api.MessagePayloadContext; import org.apache.pulsar.client.api.MessagePayloadProcessor; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.RedeliveryBackoff; @@ -60,11 +68,6 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNull; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; - /** * Unit tests of {@link ConsumerBuilderImpl}. */ @@ -76,6 +79,8 @@ public class ConsumerBuilderImplTest { @BeforeMethod(alwaysRun = true) public void setup() { PulsarClientImpl client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); ConsumerConfigurationData consumerConfigurationData = mock(ConsumerConfigurationData.class); when(consumerConfigurationData.getTopicsPattern()).thenReturn(Pattern.compile("\\w+")); when(consumerConfigurationData.getSubscriptionName()).thenReturn("testSubscriptionName"); @@ -104,6 +109,8 @@ public void testConsumerBuilderImpl() throws PulsarClientException { @Test(expectedExceptions = IllegalArgumentException.class) public void testConsumerBuilderImplWhenSchemaIsNull() { PulsarClientImpl client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); ConsumerConfigurationData consumerConfigurationData = mock(ConsumerConfigurationData.class); new ConsumerBuilderImpl(client, consumerConfigurationData, null); } @@ -444,7 +451,7 @@ public void testLoadConf() throws Exception { MessageListener messageListener = (consumer, message) -> {}; conf.put("messageListener", messageListener); - ConsumerEventListener consumerEventListener = mock(ConsumerEventListener.class); + ConsumerEventListener consumerEventListener = createMockConsumerEventListener(); conf.put("consumerEventListener", consumerEventListener); RedeliveryBackoff negativeAckRedeliveryBackoff = MultiplierRedeliveryBackoff.builder().build(); conf.put("negativeAckRedeliveryBackoff", negativeAckRedeliveryBackoff); @@ -458,7 +465,7 @@ public void testLoadConf() throws Exception { conf.put("batchReceivePolicy", batchReceivePolicy); KeySharedPolicy keySharedPolicy = KeySharedPolicy.stickyHashRange(); conf.put("keySharedPolicy", keySharedPolicy); - MessagePayloadProcessor payloadProcessor = mock(MessagePayloadProcessor.class); + MessagePayloadProcessor payloadProcessor = createMockMessagePayloadProcessor(); conf.put("payloadProcessor", payloadProcessor); consumerBuilder.loadConf(conf); @@ -589,7 +596,7 @@ private ConsumerBuilderImpl createConsumerBuilder() { .subscriptionName("subscription") .subscriptionProperties(subscriptionProperties) .messageListener((consumer, message) -> {}) - .consumerEventListener(mock(ConsumerEventListener.class)) + .consumerEventListener(createMockConsumerEventListener()) .negativeAckRedeliveryBackoff(MultiplierRedeliveryBackoff.builder().build()) .ackTimeoutRedeliveryBackoff(MultiplierRedeliveryBackoff.builder().build()) .consumerName("consumer") @@ -599,7 +606,37 @@ private ConsumerBuilderImpl createConsumerBuilder() { .deadLetterPolicy(DeadLetterPolicy.builder().deadLetterTopic("dlq").retryLetterTopic("retry").initialSubscriptionName("dlq-sub").maxRedeliverCount(1).build()) .batchReceivePolicy(BatchReceivePolicy.builder().maxNumBytes(1).build()) .keySharedPolicy(KeySharedPolicy.autoSplitHashRange()) - .messagePayloadProcessor(mock(MessagePayloadProcessor.class)); + .messagePayloadProcessor(createMockMessagePayloadProcessor()); return consumerBuilder; } + + private static ConsumerEventListener createMockConsumerEventListener() { + return new MyConsumerEventListener(); + } + + private static MessagePayloadProcessor createMockMessagePayloadProcessor() { + return new MyMessagePayloadProcessor(); + } + + @JsonIgnoreType + private static class MyMessagePayloadProcessor implements MessagePayloadProcessor { + @Override + public void process(MessagePayload payload, MessagePayloadContext context, Schema schema, + java.util.function.Consumer> messageConsumer) throws Exception { + + } + } + + @JsonIgnoreType + private static class MyConsumerEventListener implements ConsumerEventListener { + @Override + public void becameActive(Consumer consumer, int partitionId) { + + } + + @Override + public void becameInactive(Consumer consumer, int partitionId) { + + } + } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java index 5a223d5da15c0..0c47d17098eb9 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java @@ -21,6 +21,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.any; +import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -34,6 +35,7 @@ import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import lombok.Cleanup; @@ -46,6 +48,8 @@ import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import org.apache.pulsar.client.impl.conf.TopicConsumerConfigurationData; import org.apache.pulsar.client.util.ExecutorProvider; +import org.apache.pulsar.client.util.ScheduledExecutorProvider; +import org.apache.pulsar.common.util.Backoff; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -263,15 +267,24 @@ public void testTopicPriorityLevel() { assertThat(consumer.getPriorityLevel()).isEqualTo(1); } - @Test(invocationTimeOut = 1000) + @Test public void testSeekAsyncInternal() { // given ClientCnx cnx = mock(ClientCnx.class); CompletableFuture clientReq = new CompletableFuture<>(); when(cnx.sendRequestWithId(any(ByteBuf.class), anyLong())).thenReturn(clientReq); + ScheduledExecutorProvider provider = mock(ScheduledExecutorProvider.class); + ScheduledExecutorService scheduledExecutorService = mock(ScheduledExecutorService.class); + when(provider.getExecutor()).thenReturn(scheduledExecutorService); + when(consumer.getClient().getScheduledExecutorProvider()).thenReturn(provider); + + CompletableFuture result = consumer.seekAsync(1L); + verify(scheduledExecutorService, atLeast(1)).schedule(any(Runnable.class), anyLong(), any(TimeUnit.class)); + consumer.setClientCnx(cnx); consumer.setState(HandlerState.State.Ready); + consumer.seekStatus.set(ConsumerImpl.SeekStatus.NOT_STARTED); // when CompletableFuture firstResult = consumer.seekAsync(1L); @@ -279,7 +292,6 @@ public void testSeekAsyncInternal() { clientReq.complete(null); - // then assertTrue(firstResult.isDone()); assertTrue(secondResult.isCompletedExceptionally()); verify(cnx, times(1)).sendRequestWithId(any(ByteBuf.class), anyLong()); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ControlledClusterFailoverTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ControlledClusterFailoverTest.java index 570b139832806..fa7145794e1e2 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ControlledClusterFailoverTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ControlledClusterFailoverTest.java @@ -18,10 +18,10 @@ */ package org.apache.pulsar.client.impl; -import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; +import lombok.Cleanup; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.ServiceUrlProvider; import org.asynchttpclient.Request; @@ -31,11 +31,12 @@ import org.testng.annotations.Test; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; @Test(groups = "broker-impl") public class ControlledClusterFailoverTest { @Test - public void testBuildControlledClusterFailoverInstance() throws IOException { + public void testBuildControlledClusterFailoverInstance() throws Exception { String defaultServiceUrl = "pulsar://localhost:6650"; String urlProvider = "http://localhost:8080/test"; String keyA = "key-a"; @@ -46,6 +47,7 @@ public void testBuildControlledClusterFailoverInstance() throws IOException { Map header = new HashMap<>(); header.put(keyA, valueA); header.put(keyB, valueB); + @Cleanup ServiceUrlProvider provider = ControlledClusterFailover.builder() .defaultServiceUrl(defaultServiceUrl) .urlProvider(urlProvider) @@ -64,7 +66,7 @@ public void testBuildControlledClusterFailoverInstance() throws IOException { } @Test - public void testControlledClusterFailoverSwitch() throws IOException { + public void testControlledClusterFailoverSwitch() throws Exception { String defaultServiceUrl = "pulsar+ssl://localhost:6651"; String backupServiceUrl = "pulsar+ssl://localhost:6661"; String urlProvider = "http://localhost:8080"; @@ -80,6 +82,7 @@ public void testControlledClusterFailoverSwitch() throws IOException { controlledConfiguration.setAuthPluginClassName(authPluginClassName); controlledConfiguration.setAuthParamsString(authParamsString); + @Cleanup ServiceUrlProvider provider = ControlledClusterFailover.builder() .defaultServiceUrl(defaultServiceUrl) .urlProvider(urlProvider) @@ -88,6 +91,8 @@ public void testControlledClusterFailoverSwitch() throws IOException { ControlledClusterFailover controlledClusterFailover = Mockito.spy((ControlledClusterFailover) provider); PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(pulsarClient.getCnxPool()).thenReturn(connectionPool); controlledClusterFailover.initialize(pulsarClient); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MemoryLimitControllerTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MemoryLimitControllerTest.java index 78ffa247f7b6e..1aaf3f77da490 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MemoryLimitControllerTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MemoryLimitControllerTest.java @@ -21,6 +21,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -197,4 +198,39 @@ public void testStepRelease() throws Exception { assertTrue(l3.await(1, TimeUnit.SECONDS)); assertEquals(mlc.currentUsage(), 101); } + + @Test + public void testModifyMemoryFailedDueToNegativeParam() throws Exception { + MemoryLimitController mlc = new MemoryLimitController(100); + + try { + mlc.tryReserveMemory(-1); + fail("The test should fail due to calling tryReserveMemory with a negative value."); + } catch (IllegalArgumentException e) { + // Expected ex. + } + + try { + mlc.reserveMemory(-1); + fail("The test should fail due to calling reserveMemory with a negative value."); + } catch (IllegalArgumentException e) { + // Expected ex. + } + + try { + mlc.forceReserveMemory(-1); + fail("The test should fail due to calling forceReserveMemory with a negative value."); + } catch (IllegalArgumentException e) { + // Expected ex. + } + + try { + mlc.releaseMemory(-1); + fail("The test should fail due to calling releaseMemory with a negative value."); + } catch (IllegalArgumentException e) { + // Expected ex. + } + + assertEquals(mlc.currentUsage(), 0); + } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdAdvUtilsTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdAdvUtilsTest.java new file mode 100644 index 0000000000000..704dfc9cbd77b --- /dev/null +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdAdvUtilsTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import static org.testng.Assert.assertEquals; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.BitSet; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Phaser; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.Cleanup; +import org.apache.pulsar.client.api.MessageIdAdv; +import org.testng.annotations.Test; + +/** + * Unit test for {@link MessageIdAdvUtils}. + */ +public class MessageIdAdvUtilsTest { + + /** + * Call acknowledge concurrently with batch message, and verify that only return true once + * + * @see MessageIdAdvUtils#acknowledge(MessageIdAdv, boolean) + * @see MessageIdAdv#getAckSet() + */ + @Test + public void testAcknowledgeIndividualConcurrently() throws InterruptedException { + ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("pulsar-consumer-%d").build(); + @Cleanup("shutdown") + ExecutorService executorService = Executors.newCachedThreadPool(threadFactory); + for (int i = 0; i < 100; i++) { + int batchSize = 32; + BitSet bitSet = new BitSet(batchSize); + bitSet.set(0, batchSize); + AtomicInteger individualAcked = new AtomicInteger(); + Phaser phaser = new Phaser(1); + CountDownLatch finishLatch = new CountDownLatch(batchSize); + for (int batchIndex = 0; batchIndex < batchSize; batchIndex++) { + phaser.register(); + BatchMessageIdImpl messageId = new BatchMessageIdImpl(1, 0, 0, batchIndex, batchSize, bitSet); + executorService.execute(() -> { + try { + phaser.arriveAndAwaitAdvance(); + if (MessageIdAdvUtils.acknowledge(messageId, true)) { + individualAcked.incrementAndGet(); + } + } finally { + finishLatch.countDown(); + } + }); + } + phaser.arriveAndDeregister(); + finishLatch.await(); + assertEquals(individualAcked.get(), 1); + } + } +} diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java index febec2bff3285..02a4d2ebba8c1 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java @@ -22,6 +22,7 @@ import static org.apache.pulsar.client.impl.ClientTestFixtures.createExceptionFuture; import static org.apache.pulsar.client.impl.ClientTestFixtures.createPulsarClientMockWithMockedClientCnx; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -153,7 +154,8 @@ private MultiTopicsConsumerImpl createMultiTopicsConsumer( int completionDelayMillis = 100; Schema schema = Schema.BYTES; PulsarClientImpl clientMock = createPulsarClientMockWithMockedClientCnx(executorProvider, internalExecutor); - when(clientMock.getPartitionedTopicMetadata(any())).thenAnswer(invocation -> createDelayedCompletedFuture( + when(clientMock.getPartitionedTopicMetadata(any(), anyBoolean(), anyBoolean())) + .thenAnswer(invocation -> createDelayedCompletedFuture( new PartitionedTopicMetadata(), completionDelayMillis)); MultiTopicsConsumerImpl impl = new MultiTopicsConsumerImpl( clientMock, consumerConfData, executorProvider, @@ -201,7 +203,8 @@ public void testConsumerCleanupOnSubscribeFailure() { int completionDelayMillis = 10; Schema schema = Schema.BYTES; PulsarClientImpl clientMock = createPulsarClientMockWithMockedClientCnx(executorProvider, internalExecutor); - when(clientMock.getPartitionedTopicMetadata(any())).thenAnswer(invocation -> createExceptionFuture( + when(clientMock.getPartitionedTopicMetadata(any(), anyBoolean(), anyBoolean())) + .thenAnswer(invocation -> createExceptionFuture( new PulsarClientException.InvalidConfigurationException("a mock exception"), completionDelayMillis)); CompletableFuture> completeFuture = new CompletableFuture<>(); MultiTopicsConsumerImpl impl = new MultiTopicsConsumerImpl(clientMock, consumerConfData, @@ -237,7 +240,8 @@ public void testDontCheckForPartitionsUpdatesOnNonPartitionedTopics() throws Exc // Simulate non partitioned topics PartitionedTopicMetadata metadata = new PartitionedTopicMetadata(0); - when(clientMock.getPartitionedTopicMetadata(any())).thenReturn(CompletableFuture.completedFuture(metadata)); + when(clientMock.getPartitionedTopicMetadata(any(), anyBoolean(), anyBoolean())) + .thenReturn(CompletableFuture.completedFuture(metadata)); CompletableFuture> completeFuture = new CompletableFuture<>(); MultiTopicsConsumerImpl impl = new MultiTopicsConsumerImpl<>( @@ -248,7 +252,7 @@ public void testDontCheckForPartitionsUpdatesOnNonPartitionedTopics() throws Exc // getPartitionedTopicMetadata should have been called only the first time, for each of the 3 topics, // but not anymore since the topics are not partitioned. - verify(clientMock, times(3)).getPartitionedTopicMetadata(any()); + verify(clientMock, times(3)).getPartitionedTopicMetadata(any(), anyBoolean(), anyBoolean()); } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/OpSendMsgQueueTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/OpSendMsgQueueTest.java index 2db23782640eb..efcc06bede3e1 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/OpSendMsgQueueTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/OpSendMsgQueueTest.java @@ -23,6 +23,7 @@ import static org.testng.Assert.assertEquals; import com.google.common.collect.Lists; import java.util.Arrays; +import org.apache.pulsar.client.impl.metrics.LatencyHistogram; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -39,7 +40,7 @@ public void createMockMessage() { } private ProducerImpl.OpSendMsg createDummyOpSendMsg() { - return ProducerImpl.OpSendMsg.create(message, null, 0L, null); + return ProducerImpl.OpSendMsg.create(LatencyHistogram.NOOP, message, null, 0L, null); } @Test diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PartitionedProducerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PartitionedProducerImplTest.java index 223881d85a87b..f96d2e2e0b0e9 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PartitionedProducerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PartitionedProducerImplTest.java @@ -29,10 +29,10 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import com.google.api.client.util.Lists; import io.netty.channel.EventLoopGroup; import io.netty.util.Timer; import io.netty.util.concurrent.DefaultThreadFactory; -import com.google.api.client.util.Lists; import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.util.List; @@ -40,15 +40,14 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ThreadFactory; -import org.apache.pulsar.client.api.Message; -import org.apache.pulsar.client.api.MessageRouter; -import org.apache.pulsar.client.api.MessageRoutingMode; -import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.api.TopicMetadata; +import java.util.concurrent.TimeUnit; + +import lombok.Cleanup; +import org.apache.pulsar.client.api.*; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; import org.apache.pulsar.client.impl.customroute.PartialRoundRobinMessageRouterImpl; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.util.netty.EventLoopUtil; import org.assertj.core.util.Sets; @@ -70,6 +69,8 @@ public class PartitionedProducerImplTest { @BeforeMethod(alwaysRun = true) public void setup() { client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); schema = mock(Schema.class); producerInterceptors = mock(ProducerInterceptors.class); producerCreatedFuture = new CompletableFuture<>(); @@ -78,6 +79,7 @@ public void setup() { producerBuilderImpl = new ProducerBuilderImpl(client, Schema.BYTES); + when(client.instrumentProvider()).thenReturn(InstrumentProvider.NOOP); when(client.getConfiguration()).thenReturn(clientConfigurationData); when(client.timer()).thenReturn(timer); when(client.newProducer()).thenReturn(producerBuilderImpl); @@ -186,8 +188,10 @@ public void testGetStats() throws Exception { conf.setStatsIntervalSeconds(100); ThreadFactory threadFactory = new DefaultThreadFactory("client-test-stats", Thread.currentThread().isDaemon()); + @Cleanup("shutdownGracefully") EventLoopGroup eventLoopGroup = EventLoopUtil.newEventLoopGroup(conf.getNumIoThreads(), false, threadFactory); + @Cleanup PulsarClientImpl clientImpl = new PulsarClientImpl(conf, eventLoopGroup); ProducerConfigurationData producerConfData = new ProducerConfigurationData(); @@ -212,9 +216,11 @@ public void testGetStatsWithoutArriveUpdateInterval() throws Exception { ThreadFactory threadFactory = new DefaultThreadFactory("client-test-stats", Thread.currentThread().isDaemon()); + @Cleanup("shutdownGracefully") EventLoopGroup eventLoopGroup = EventLoopUtil .newEventLoopGroup(conf.getNumIoThreads(), false, threadFactory); + @Cleanup PulsarClientImpl clientImpl = new PulsarClientImpl(conf, eventLoopGroup); ProducerConfigurationData producerConfData = new ProducerConfigurationData(); @@ -244,8 +250,10 @@ public void testGetNumOfPartitions() throws Exception { conf.setStatsIntervalSeconds(100); ThreadFactory threadFactory = new DefaultThreadFactory("client-test-stats", Thread.currentThread().isDaemon()); + @Cleanup("shutdownGracefully") EventLoopGroup eventLoopGroup = EventLoopUtil.newEventLoopGroup(conf.getNumIoThreads(), false, threadFactory); + @Cleanup PulsarClientImpl clientImpl = new PulsarClientImpl(conf, eventLoopGroup); ProducerConfigurationData producerConfData = new ProducerConfigurationData(); @@ -264,4 +272,36 @@ public void testGetNumOfPartitions() throws Exception { assertEquals(producerImpl.getNumOfPartitions(), 0); } + + @Test + public void testOnTopicsExtended() throws Exception { + String topicName = "test-on-topics-extended"; + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setServiceUrl("pulsar://localhost:6650"); + conf.setStatsIntervalSeconds(100); + ThreadFactory threadFactory = new DefaultThreadFactory("client-test-stats", Thread.currentThread().isDaemon()); + @Cleanup("shutdownGracefully") + EventLoopGroup eventLoopGroup = EventLoopUtil.newEventLoopGroup(conf.getNumIoThreads(), false, threadFactory); + + @Cleanup + PulsarClientImpl clientImpl = new PulsarClientImpl(conf, eventLoopGroup); + + ProducerConfigurationData producerConfData = new ProducerConfigurationData(); + producerConfData.setMessageRoutingMode(MessageRoutingMode.CustomPartition); + producerConfData.setCustomMessageRouter(new CustomMessageRouter()); + producerConfData.setAutoUpdatePartitionsIntervalSeconds(1, TimeUnit.MILLISECONDS); + + PartitionedProducerImpl impl = new PartitionedProducerImpl( + clientImpl, topicName, producerConfData, 1, null, null, null); + + impl.setState(HandlerState.State.Ready); + Thread.sleep(1000); + CompletableFuture future = impl.getPartitionsAutoUpdateFuture(); + + // When null is returned in method thenCompose we will encounter an NPE exception. + // Because the returned value will be applied to the next stage. + // We use future instead of null as the return value. + assertNotNull(future); + } + } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PatternConsumerUpdateQueueTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PatternConsumerUpdateQueueTest.java new file mode 100644 index 0000000000000..01f0be6a85ef6 --- /dev/null +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PatternConsumerUpdateQueueTest.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import io.netty.util.HashedWheelTimer; +import java.io.Closeable; +import java.time.Duration; +import java.util.Arrays; +import java.util.Collection; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import lombok.AllArgsConstructor; +import org.apache.pulsar.client.util.ExecutorProvider; +import org.apache.pulsar.common.util.FutureUtil; +import org.awaitility.Awaitility; +import org.testng.annotations.Test; + +@Test(groups = "utils") +public class PatternConsumerUpdateQueueTest { + + private QueueInstance createInstance(CompletableFuture customizedRecheckFuture, + CompletableFuture customizedPartialUpdateFuture, + CompletableFuture customizedConsumerInitFuture) { + return createInstance(customizedRecheckFuture, customizedPartialUpdateFuture, customizedConsumerInitFuture, + null, null); + } + + private QueueInstance createInstance(CompletableFuture customizedRecheckFuture, + CompletableFuture customizedPartialUpdateFuture, + CompletableFuture customizedConsumerInitFuture, + Collection successTopics, + Collection errorTopics) { + HashedWheelTimer timer = new HashedWheelTimer(new ExecutorProvider.ExtendedThreadFactory("timer-x", + Thread.currentThread().isDaemon()), 1, TimeUnit.MILLISECONDS); + PulsarClientImpl client = mock(PulsarClientImpl.class); + when(client.timer()).thenReturn(timer); + + PatternMultiTopicsConsumerImpl patternConsumer = mock(PatternMultiTopicsConsumerImpl.class); + when(patternConsumer.recheckTopicsChange()).thenReturn(customizedRecheckFuture); + when(patternConsumer.getClient()).thenReturn(client); + if (customizedConsumerInitFuture != null) { + when(patternConsumer.getSubscribeFuture()).thenReturn(customizedConsumerInitFuture); + } else { + when(patternConsumer.getSubscribeFuture()).thenReturn(CompletableFuture.completedFuture(null)); + } + + PatternMultiTopicsConsumerImpl.TopicsChangedListener topicsChangeListener = + mock(PatternMultiTopicsConsumerImpl.TopicsChangedListener.class); + if (successTopics == null && errorTopics == null) { + when(topicsChangeListener.onTopicsAdded(anyCollection())).thenReturn(customizedPartialUpdateFuture); + when(topicsChangeListener.onTopicsRemoved(anyCollection())).thenReturn(customizedPartialUpdateFuture); + } else { + CompletableFuture ex = FutureUtil.failedFuture(new RuntimeException("mock error")); + when(topicsChangeListener.onTopicsAdded(successTopics)).thenReturn(customizedPartialUpdateFuture); + when(topicsChangeListener.onTopicsRemoved(successTopics)).thenReturn(customizedPartialUpdateFuture); + when(topicsChangeListener.onTopicsAdded(errorTopics)).thenReturn(ex); + when(topicsChangeListener.onTopicsRemoved(errorTopics)).thenReturn(ex); + } + + PatternConsumerUpdateQueue queue = new PatternConsumerUpdateQueue(patternConsumer, topicsChangeListener); + return new QueueInstance(queue, patternConsumer, topicsChangeListener); + } + + private QueueInstance createInstance() { + CompletableFuture completedFuture = CompletableFuture.completedFuture(null); + return createInstance(completedFuture, completedFuture, completedFuture); + } + + @AllArgsConstructor + private static class QueueInstance implements Closeable { + private PatternConsumerUpdateQueue queue; + private PatternMultiTopicsConsumerImpl mockedConsumer; + private PatternMultiTopicsConsumerImpl.TopicsChangedListener mockedListener; + + @Override + public void close() { + mockedConsumer.getClient().timer().stop(); + } + } + + @Test + public void testTopicsChangedEvents() { + QueueInstance instance = createInstance(); + + Collection topics = Arrays.asList("a"); + for (int i = 0; i < 10; i++) { + instance.queue.appendTopicsAddedOp(topics); + instance.queue.appendTopicsRemovedOp(topics); + } + Awaitility.await().untilAsserted(() -> { + verify(instance.mockedListener, times(10)).onTopicsAdded(topics); + verify(instance.mockedListener, times(10)).onTopicsRemoved(topics); + }); + + // cleanup. + instance.close(); + } + + @Test + public void testRecheckTask() { + QueueInstance instance = createInstance(); + + for (int i = 0; i < 10; i++) { + instance.queue.appendRecheckOp(); + } + + Awaitility.await().untilAsserted(() -> { + verify(instance.mockedConsumer, times(10)).recheckTopicsChange(); + }); + + // cleanup. + instance.close(); + } + + @Test + public void testDelayedRecheckTask() { + CompletableFuture recheckFuture = new CompletableFuture<>(); + CompletableFuture partialUpdateFuture = CompletableFuture.completedFuture(null); + CompletableFuture consumerInitFuture = CompletableFuture.completedFuture(null); + QueueInstance instance = createInstance(recheckFuture, partialUpdateFuture, consumerInitFuture); + + for (int i = 0; i < 10; i++) { + instance.queue.appendRecheckOp(); + } + + recheckFuture.complete(null); + Awaitility.await().untilAsserted(() -> { + // The first task will be running, and never completed until all tasks have been added. + // Since the first was started, the second one will not be skipped. + // The others after the second task will be skipped. + // So the times that called "recheckTopicsChange" will be 2. + verify(instance.mockedConsumer, times(2)).recheckTopicsChange(); + }); + + // cleanup. + instance.close(); + } + + @Test + public void testCompositeTasks() { + CompletableFuture recheckFuture = new CompletableFuture<>(); + CompletableFuture partialUpdateFuture = CompletableFuture.completedFuture(null); + CompletableFuture consumerInitFuture = CompletableFuture.completedFuture(null); + QueueInstance instance = createInstance(recheckFuture, partialUpdateFuture, consumerInitFuture); + + Collection topics = Arrays.asList("a"); + for (int i = 0; i < 10; i++) { + instance.queue.appendRecheckOp(); + instance.queue.appendTopicsAddedOp(topics); + instance.queue.appendTopicsRemovedOp(topics); + } + recheckFuture.complete(null); + Awaitility.await().untilAsserted(() -> { + // The first task will be running, and never completed until all tasks have been added. + // Since the first was started, the second one will not be skipped. + // The others after the second task will be skipped. + // So the times that called "recheckTopicsChange" will be 2. + verify(instance.mockedConsumer, times(2)).recheckTopicsChange(); + // The tasks after the second "recheckTopicsChange" will be skipped due to there is a previous + // "recheckTopicsChange" that has not been executed. + // The tasks between the fist "recheckTopicsChange" and the second "recheckTopicsChange" will be skipped + // due to there is a following "recheckTopicsChange". + verify(instance.mockedListener, times(0)).onTopicsAdded(topics); + verify(instance.mockedListener, times(0)).onTopicsRemoved(topics); + }); + + // cleanup. + instance.close(); + } + + @Test + public void testErrorTask() { + CompletableFuture immediatelyCompleteFuture = CompletableFuture.completedFuture(null); + Collection successTopics = Arrays.asList("a"); + Collection errorTopics = Arrays.asList(UUID.randomUUID().toString()); + QueueInstance instance = createInstance(immediatelyCompleteFuture, immediatelyCompleteFuture, + immediatelyCompleteFuture, successTopics, errorTopics); + + instance.queue.appendTopicsAddedOp(successTopics); + instance.queue.appendTopicsRemovedOp(successTopics); + instance.queue.appendTopicsAddedOp(errorTopics); + instance.queue.appendTopicsAddedOp(successTopics); + instance.queue.appendTopicsRemovedOp(successTopics); + + Awaitility.await().atMost(Duration.ofSeconds(60)).untilAsserted(() -> { + verify(instance.mockedListener, times(2)).onTopicsAdded(successTopics); + verify(instance.mockedListener, times(2)).onTopicsRemoved(successTopics); + verify(instance.mockedListener, times(1)).onTopicsAdded(errorTopics); + // After an error task will push a recheck task to offset. + verify(instance.mockedConsumer, times(1)).recheckTopicsChange(); + }); + + // cleanup. + instance.close(); + } + + @Test + public void testFailedSubscribe() { + CompletableFuture immediatelyCompleteFuture = CompletableFuture.completedFuture(null); + CompletableFuture consumerInitFuture = new CompletableFuture<>(); + Collection successTopics = Arrays.asList("a"); + Collection errorTopics = Arrays.asList(UUID.randomUUID().toString()); + QueueInstance instance = createInstance(immediatelyCompleteFuture, immediatelyCompleteFuture, + consumerInitFuture, successTopics, errorTopics); + + instance.queue.appendTopicsAddedOp(successTopics); + instance.queue.appendTopicsRemovedOp(successTopics); + instance.queue.appendTopicsAddedOp(errorTopics); + instance.queue.appendTopicsAddedOp(successTopics); + instance.queue.appendTopicsRemovedOp(successTopics); + + // Consumer init failed after multi topics changes. + // All the topics changes events should be skipped. + consumerInitFuture.completeExceptionally(new RuntimeException("mocked ex")); + Awaitility.await().untilAsserted(() -> { + verify(instance.mockedListener, times(0)).onTopicsAdded(successTopics); + verify(instance.mockedListener, times(0)).onTopicsRemoved(successTopics); + verify(instance.mockedListener, times(0)).onTopicsAdded(errorTopics); + verify(instance.mockedConsumer, times(0)).recheckTopicsChange(); + }); + + // cleanup. + instance.close(); + } +} diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImplTest.java index 5baca24cf8aa1..3dfb23f31954a 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImplTest.java @@ -24,6 +24,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.google.common.collect.Sets; +import com.google.re2j.Pattern; import org.apache.pulsar.common.lookup.GetTopicsResult; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -32,7 +33,6 @@ import java.util.Collections; import java.util.concurrent.CompletableFuture; import java.util.function.Consumer; -import java.util.regex.Pattern; public class PatternMultiTopicsConsumerImplTest { @@ -61,7 +61,7 @@ public void testChangedUnfilteredResponse() { "persistent://tenant/my-ns/non-matching"), null, false, true), mockListener, - Collections.emptyList()); + Collections.emptyList(), ""); verify(mockListener).onTopicsAdded(Sets.newHashSet( "persistent://tenant/my-ns/name-1", "persistent://tenant/my-ns/name-2")); @@ -80,7 +80,7 @@ public void testChangedFilteredResponse() { "persistent://tenant/my-ns/name-2"), "TOPICS_HASH", true, true), mockListener, - Arrays.asList("persistent://tenant/my-ns/name-0")); + Arrays.asList("persistent://tenant/my-ns/name-0"), ""); verify(mockListener).onTopicsAdded(Sets.newHashSet( "persistent://tenant/my-ns/name-1", "persistent://tenant/my-ns/name-2")); @@ -99,7 +99,7 @@ public void testUnchangedResponse() { "persistent://tenant/my-ns/name-2"), "TOPICS_HASH", true, false), mockListener, - Arrays.asList("persistent://tenant/my-ns/name-0")); + Arrays.asList("persistent://tenant/my-ns/name-0"), ""); verify(mockListener, never()).onTopicsAdded(any()); verify(mockListener, never()).onTopicsRemoved(any()); verify(mockTopicsHashSetter).accept("TOPICS_HASH"); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java index bb3e3fc3accf6..b830d375303bb 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java @@ -52,6 +52,8 @@ public class ProducerBuilderImplTest { public void setup() { Producer producer = mock(Producer.class); client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); producerBuilderImpl = new ProducerBuilderImpl<>(client, Schema.BYTES); when(client.newProducer()).thenReturn(producerBuilderImpl); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerImplTest.java index 6fcedc3f94de7..f9df63759394a 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerImplTest.java @@ -27,6 +27,7 @@ import static org.testng.Assert.assertTrue; import java.nio.ByteBuffer; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.metrics.LatencyHistogram; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.mockito.Mockito; import org.testng.annotations.Test; @@ -42,6 +43,7 @@ public void testChunkedMessageCtxDeallocate() { for (int i = 0; i < totalChunks; i++) { ProducerImpl.OpSendMsg opSendMsg = ProducerImpl.OpSendMsg.create( + LatencyHistogram.NOOP, MessageImpl.create(new MessageMetadata(), ByteBuffer.allocate(0), Schema.STRING, null), null, 0, null); opSendMsg.chunkedMessageCtx = ctx; diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImplTest.java index 32d0eff6e792e..8f648bfd9ffbc 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImplTest.java @@ -26,6 +26,7 @@ import io.netty.util.HashedWheelTimer; import io.netty.util.Timer; import java.util.concurrent.TimeUnit; +import lombok.Cleanup; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; import org.testng.annotations.Test; @@ -40,7 +41,10 @@ public void testIncrementNumAcksReceived() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); conf.setStatsIntervalSeconds(1); PulsarClientImpl client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); when(client.getConfiguration()).thenReturn(conf); + @Cleanup("stop") Timer timer = new HashedWheelTimer(); when(client.timer()).thenReturn(timer); ProducerImpl producer = mock(ProducerImpl.class); @@ -53,6 +57,7 @@ public void testIncrementNumAcksReceived() throws Exception { recorder.incrementNumAcksReceived(latencyNs); Thread.sleep(1200); assertEquals(1000.0, recorder.getSendLatencyMillisMax(), 0.5); + recorder.cancelStatsTimeout(); } @Test @@ -60,7 +65,10 @@ public void testGetStatsAndCancelStatsTimeoutWithoutArriveUpdateInterval() { ClientConfigurationData conf = new ClientConfigurationData(); conf.setStatsIntervalSeconds(60); PulsarClientImpl client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); when(client.getConfiguration()).thenReturn(conf); + @Cleanup("stop") Timer timer = new HashedWheelTimer(); when(client.timer()).thenReturn(timer); ProducerImpl producer = mock(ProducerImpl.class); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PulsarClientImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PulsarClientImplTest.java index 54d13538d7867..4481de9f1e65f 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PulsarClientImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PulsarClientImplTest.java @@ -19,6 +19,8 @@ package org.apache.pulsar.client.impl; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.mock; @@ -36,6 +38,7 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelPromise; import io.netty.channel.EventLoopGroup; +import io.netty.resolver.dns.DefaultDnsServerAddressStreamProvider; import io.netty.util.HashedWheelTimer; import io.netty.util.concurrent.DefaultThreadFactory; import java.lang.reflect.Field; @@ -48,10 +51,10 @@ import java.util.concurrent.ThreadFactory; import java.util.regex.Pattern; import lombok.Cleanup; -import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.client.util.ScheduledExecutorProvider; import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace; @@ -105,11 +108,11 @@ public void testConsumerIsClosed() throws Exception { nullable(String.class))) .thenReturn(CompletableFuture.completedFuture( new GetTopicsResult(Collections.emptyList(), null, false, true))); - when(lookup.getPartitionedTopicMetadata(any(TopicName.class))) + when(lookup.getPartitionedTopicMetadata(any(TopicName.class), anyBoolean(), anyBoolean())) .thenReturn(CompletableFuture.completedFuture(new PartitionedTopicMetadata())); when(lookup.getBroker(any())) - .thenReturn(CompletableFuture.completedFuture( - Pair.of(mock(InetSocketAddress.class), mock(InetSocketAddress.class)))); + .thenReturn(CompletableFuture.completedFuture(new LookupTopicResult( + mock(InetSocketAddress.class), mock(InetSocketAddress.class), false))); ConnectionPool pool = mock(ConnectionPool.class); ClientCnx cnx = mock(ClientCnx.class); ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); @@ -122,7 +125,7 @@ public void testConsumerIsClosed() throws Exception { when(cnx.ctx()).thenReturn(ctx); when(cnx.sendRequestWithId(any(ByteBuf.class), anyLong())) .thenReturn(CompletableFuture.completedFuture(mock(ProducerResponse.class))); - when(pool.getConnection(any(InetSocketAddress.class), any(InetSocketAddress.class))) + when(pool.getConnection(any(InetSocketAddress.class), any(InetSocketAddress.class), anyInt())) .thenReturn(CompletableFuture.completedFuture(cnx)); ClientConfigurationData conf = new ClientConfigurationData(); @@ -177,8 +180,9 @@ public void testInitializeWithoutTimer() throws Exception { @Test public void testInitializeWithTimer() throws PulsarClientException { ClientConfigurationData conf = new ClientConfigurationData(); + @Cleanup("shutdownGracefully") EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, new DefaultThreadFactory("test")); - ConnectionPool pool = Mockito.spy(new ConnectionPool(conf, eventLoop)); + ConnectionPool pool = Mockito.spy(new ConnectionPool(InstrumentProvider.NOOP, conf, eventLoop, null)); conf.setServiceUrl("pulsar://localhost:6650"); HashedWheelTimer timer = new HashedWheelTimer(); @@ -188,12 +192,22 @@ public void testInitializeWithTimer() throws PulsarClientException { client.timer().stop(); } + @Test + public void testInitializeWithDNSServerAddresses() throws Exception { + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setDnsServerAddresses(DefaultDnsServerAddressStreamProvider.defaultAddressList()); + conf.setServiceUrl("pulsar://localhost:6650"); + initializeEventLoopGroup(conf); + PulsarClientImpl client = new PulsarClientImpl(conf, eventLoopGroup); + client.shutdown(); + } + @Test public void testResourceCleanup() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); conf.setServiceUrl(""); initializeEventLoopGroup(conf); - try (ConnectionPool connectionPool = new ConnectionPool(conf, eventLoopGroup)) { + try (ConnectionPool connectionPool = new ConnectionPool(InstrumentProvider.NOOP, conf, eventLoopGroup, null)) { assertThrows(() -> new PulsarClientImpl(conf, eventLoopGroup, connectionPool)); } finally { // Externally passed eventLoopGroup should not be shutdown. diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ReaderImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ReaderImplTest.java index 1bbdf274293a2..0349dbfe2651d 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ReaderImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ReaderImplTest.java @@ -18,23 +18,27 @@ */ package org.apache.pulsar.client.impl; -import static org.testng.AssertJUnit.assertFalse; -import static org.testng.AssertJUnit.assertTrue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.conf.ReaderConfigurationData; +import org.apache.pulsar.client.impl.crypto.MessageCryptoBc; import org.apache.pulsar.client.util.ExecutorProvider; import org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import static org.testng.Assert.assertNotNull; +import static org.testng.AssertJUnit.assertFalse; +import static org.testng.AssertJUnit.assertTrue; public class ReaderImplTest { - private ReaderImpl reader; + private PulsarClientImpl client; private ExecutorProvider executorProvider; private ExecutorService internalExecutor; @@ -42,17 +46,15 @@ public class ReaderImplTest { void setupReader() { executorProvider = new ExecutorProvider(1, "ReaderImplTest"); internalExecutor = Executors.newSingleThreadScheduledExecutor(); - PulsarClientImpl mockedClient = ClientTestFixtures.createPulsarClientMockWithMockedClientCnx( - executorProvider, internalExecutor); - ReaderConfigurationData readerConfiguration = new ReaderConfigurationData<>(); - readerConfiguration.setTopicName("topicName"); - CompletableFuture> consumerFuture = new CompletableFuture<>(); - reader = new ReaderImpl<>(mockedClient, readerConfiguration, ClientTestFixtures.createMockedExecutorProvider(), - consumerFuture, Schema.BYTES); + client = ClientTestFixtures.createPulsarClientMockWithMockedClientCnx(executorProvider, internalExecutor); } @AfterMethod - public void clean() { + public void clean() throws Exception { + if (client != null) { + client.close(); + client = null; + } if (executorProvider != null) { executorProvider.shutdownNow(); executorProvider = null; @@ -65,6 +67,16 @@ public void clean() { @Test void shouldSupportCancellingReadNextAsync() { + ReaderConfigurationData readerConfiguration = new ReaderConfigurationData<>(); + readerConfiguration.setTopicName("topicName"); + CompletableFuture> consumerFuture = new CompletableFuture<>(); + ReaderImpl reader = new ReaderImpl<>( + client, + readerConfiguration, + ClientTestFixtures.createMockedExecutorProvider(), + consumerFuture, + Schema.BYTES); + // given CompletableFuture> future = reader.readNextAsync(); Awaitility.await().untilAsserted(() -> { @@ -77,4 +89,14 @@ void shouldSupportCancellingReadNextAsync() { // then assertFalse(reader.getConsumer().hasNextPendingReceive()); } + + @Test + public void testReaderBuilderWhenMessageCryptoSet() throws PulsarClientException { + ReaderBuilderImpl builder = new ReaderBuilderImpl<>(client, Schema.BYTES); + builder.topic("testTopicName"); + builder.startMessageFromRollbackDuration(1, TimeUnit.SECONDS); + builder.messageCrypto(new MessageCryptoBc("ctx1", true)); + assertNotNull(builder.create()); + assertNotNull(builder.getConf().getMessageCrypto()); + } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewBuilderImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewBuilderImplTest.java index 9959a2038555c..01353e47cd0cb 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewBuilderImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewBuilderImplTest.java @@ -18,6 +18,14 @@ */ package org.apache.pulsar.client.impl; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertNotNull; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import org.apache.pulsar.client.api.ConsumerCryptoFailureAction; import org.apache.pulsar.client.api.CryptoKeyReader; import org.apache.pulsar.client.api.PulsarClientException; @@ -25,33 +33,28 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.TableView; import org.apache.pulsar.client.impl.conf.ReaderConfigurationData; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; - -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertNotNull; - /** - * Unit tests of {@link TablewViewBuilderImpl}. + * Unit tests of {@link TableViewBuilderImpl}. */ public class TableViewBuilderImplTest { private static final String TOPIC_NAME = "testTopicName"; private PulsarClientImpl client; private TableViewBuilderImpl tableViewBuilderImpl; + private CompletableFuture readNextFuture; @BeforeClass(alwaysRun = true) public void setup() { Reader reader = mock(Reader.class); - when(reader.readNextAsync()).thenReturn(CompletableFuture.allOf()); + readNextFuture = new CompletableFuture(); + when(reader.readNextAsync()).thenReturn(readNextFuture); client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); when(client.newReader(any(Schema.class))) .thenReturn(new ReaderBuilderImpl(client, Schema.BYTES)); when(client.createReaderAsync(any(ReaderConfigurationData.class), any(Schema.class))) @@ -59,6 +62,14 @@ public void setup() { tableViewBuilderImpl = new TableViewBuilderImpl(client, Schema.BYTES); } + @AfterClass(alwaysRun = true) + public void cleanup() { + if (readNextFuture != null) { + readNextFuture.completeExceptionally(new PulsarClientException.AlreadyClosedException("Closing test case")); + readNextFuture = null; + } + } + @Test public void testTableViewBuilderImpl() throws PulsarClientException { TableView tableView = tableViewBuilderImpl.topic(TOPIC_NAME) diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewImplTest.java index 68c886bc7211a..6a866034ddbf8 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewImplTest.java @@ -38,6 +38,8 @@ public class TableViewImplTest { @BeforeClass(alwaysRun = true) public void setup() { client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); when(client.newReader(any(Schema.class))) .thenReturn(new ReaderBuilderImpl(client, Schema.BYTES)); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicListWatcherTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicListWatcherTest.java index 0d24d76023c87..7daf316c4c576 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicListWatcherTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicListWatcherTest.java @@ -18,7 +18,11 @@ */ package org.apache.pulsar.client.impl; +import com.google.re2j.Pattern; import io.netty.channel.ChannelHandlerContext; +import io.netty.util.HashedWheelTimer; +import io.netty.util.Timer; +import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.client.impl.PatternMultiTopicsConsumerImpl.TopicsChangedListener; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.common.api.proto.BaseCommand; @@ -26,7 +30,10 @@ import org.apache.pulsar.common.api.proto.CommandWatchTopicUpdate; import org.apache.pulsar.common.naming.NamespaceName; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -36,7 +43,6 @@ import org.testng.annotations.Test; import java.util.Collections; import java.util.concurrent.CompletableFuture; -import java.util.regex.Pattern; public class TopicListWatcherTest { @@ -50,18 +56,37 @@ public class TopicListWatcherTest { public void setup() { listener = mock(TopicsChangedListener.class); client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); + when(connectionPool.genRandomKeyToSelectCon()).thenReturn(0); when(client.getConfiguration()).thenReturn(new ClientConfigurationData()); clientCnxFuture = new CompletableFuture<>(); when(client.getConnectionToServiceUrl()).thenReturn(clientCnxFuture); + Timer timer = new HashedWheelTimer(); + when(client.timer()).thenReturn(timer); + String topic = "persistent://tenant/ns/topic\\d+"; + when(client.getConnection(topic, 0)). + thenReturn(clientCnxFuture.thenApply(clientCnx -> Pair.of(clientCnx, false))); + when(client.getConnection(any(), any(), anyInt())).thenReturn(clientCnxFuture); + when(connectionPool.getConnection(any(), any(), anyInt())).thenReturn(clientCnxFuture); + + CompletableFuture completedFuture = CompletableFuture.completedFuture(null); + PatternMultiTopicsConsumerImpl patternConsumer = mock(PatternMultiTopicsConsumerImpl.class); + when(patternConsumer.getSubscribeFuture()).thenReturn(completedFuture); + when(patternConsumer.recheckTopicsChange()).thenReturn(completedFuture); + when(listener.onTopicsAdded(anyCollection())).thenReturn(completedFuture); + when(listener.onTopicsRemoved(anyCollection())).thenReturn(completedFuture); + PatternConsumerUpdateQueue queue = new PatternConsumerUpdateQueue(patternConsumer, listener); + watcherFuture = new CompletableFuture<>(); - watcher = new TopicListWatcher(listener, client, - Pattern.compile("persistent://tenant/ns/topic\\d+"), 7, - NamespaceName.get("tenant/ns"), null, watcherFuture); + watcher = new TopicListWatcher(queue, client, + Pattern.compile(topic), 7, + NamespaceName.get("tenant/ns"), null, watcherFuture, () -> {}); } @Test public void testWatcherGrabsConnection() { - verify(client).getConnectionToServiceUrl(); + verify(client).getConnection(anyString(), anyInt()); } @Test @@ -106,6 +131,4 @@ public void testWatcherCallsListenerOnUpdate() { watcher.handleCommandWatchTopicUpdate(update); verify(listener).onTopicsAdded(Collections.singletonList("persistent://tenant/ns/topic12")); } - - } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TypedMessageBuilderImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TypedMessageBuilderImplTest.java index 94c683e527177..05db4402a1586 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TypedMessageBuilderImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TypedMessageBuilderImplTest.java @@ -27,6 +27,8 @@ import org.mockito.Mock; import org.testng.annotations.Test; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.util.Base64; @@ -45,7 +47,7 @@ public class TypedMessageBuilderImplTest { protected ProducerBase producerBase; @Test - public void testDefaultValue() { + public void testDefaultValue() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { producerBase = mock(ProducerBase.class); AvroSchema fooSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(SchemaTestUtils.Foo.class).build()); @@ -63,6 +65,9 @@ public void testDefaultValue() { // Check kv.encoding.type default, not set value TypedMessageBuilderImpl typedMessageBuilder = (TypedMessageBuilderImpl)typedMessageBuilderImpl.value(keyValue); + Method method = TypedMessageBuilderImpl.class.getDeclaredMethod("beforeSend"); + method.setAccessible(true); + method.invoke(typedMessageBuilder); ByteBuffer content = typedMessageBuilder.getContent(); byte[] contentByte = new byte[content.remaining()]; content.get(contentByte); @@ -73,7 +78,7 @@ public void testDefaultValue() { } @Test - public void testInlineValue() { + public void testInlineValue() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { producerBase = mock(ProducerBase.class); AvroSchema fooSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(SchemaTestUtils.Foo.class).build()); @@ -91,6 +96,9 @@ public void testInlineValue() { // Check kv.encoding.type INLINE TypedMessageBuilderImpl typedMessageBuilder = (TypedMessageBuilderImpl)typedMessageBuilderImpl.value(keyValue); + Method method = TypedMessageBuilderImpl.class.getDeclaredMethod("beforeSend"); + method.setAccessible(true); + method.invoke(typedMessageBuilder); ByteBuffer content = typedMessageBuilder.getContent(); byte[] contentByte = new byte[content.remaining()]; content.get(contentByte); @@ -101,7 +109,7 @@ public void testInlineValue() { } @Test - public void testSeparatedValue() { + public void testSeparatedValue() throws Exception { producerBase = mock(ProducerBase.class); AvroSchema fooSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(SchemaTestUtils.Foo.class).build()); @@ -119,6 +127,9 @@ public void testSeparatedValue() { // Check kv.encoding.type SEPARATED TypedMessageBuilderImpl typedMessageBuilder = (TypedMessageBuilderImpl)typedMessageBuilderImpl.value(keyValue); + Method method = TypedMessageBuilderImpl.class.getDeclaredMethod("beforeSend"); + method.setAccessible(true); + method.invoke(typedMessageBuilder); ByteBuffer content = typedMessageBuilder.getContent(); byte[] contentByte = new byte[content.remaining()]; content.get(contentByte); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedMessageTrackerTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedMessageTrackerTest.java index 4ccc514e8e7f1..eaac165818a56 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedMessageTrackerTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedMessageTrackerTest.java @@ -29,12 +29,14 @@ import io.netty.util.HashedWheelTimer; import io.netty.util.Timer; import io.netty.util.concurrent.DefaultThreadFactory; - +import java.time.Duration; import java.util.HashSet; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; - import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; +import org.awaitility.Awaitility; import org.testng.annotations.Test; public class UnAckedMessageTrackerTest { @@ -42,6 +44,9 @@ public class UnAckedMessageTrackerTest { @Test public void testAddAndRemove() { PulsarClientImpl client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.instrumentProvider()).thenReturn(InstrumentProvider.NOOP); + when(client.getCnxPool()).thenReturn(connectionPool); Timer timer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-timer", Thread.currentThread().isDaemon()), 1, TimeUnit.MILLISECONDS); when(client.timer()).thenReturn(timer); @@ -77,4 +82,50 @@ public void testAddAndRemove() { timer.stop(); } + @Test + public void testTrackChunkedMessageId() { + PulsarClientImpl client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.instrumentProvider()).thenReturn(InstrumentProvider.NOOP); + when(client.getCnxPool()).thenReturn(connectionPool); + Timer timer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-timer", Thread.currentThread().isDaemon()), + 1, TimeUnit.MILLISECONDS); + when(client.timer()).thenReturn(timer); + + ConsumerBase consumer = mock(ConsumerBase.class); + doNothing().when(consumer).onAckTimeoutSend(any()); + doNothing().when(consumer).redeliverUnacknowledgedMessages(any()); + ConsumerConfigurationData conf = new ConsumerConfigurationData<>(); + conf.setAckTimeoutMillis(1000); + conf.setTickDurationMillis(1000); + UnAckedMessageTracker tracker = new UnAckedMessageTracker(client, consumer, conf); + + assertTrue(tracker.isEmpty()); + assertEquals(tracker.size(), 0); + + // Build chunked message ID + MessageIdImpl[] chunkMsgIds = new MessageIdImpl[5]; + for (int i = 0; i < 5; i++) { + chunkMsgIds[i] = new MessageIdImpl(1L, i, -1); + } + ChunkMessageIdImpl chunkedMessageId = + new ChunkMessageIdImpl(chunkMsgIds[0], chunkMsgIds[chunkMsgIds.length - 1]); + + consumer.unAckedChunkedMessageIdSequenceMap = new ConcurrentHashMap<>(); + consumer.unAckedChunkedMessageIdSequenceMap.put(chunkedMessageId, chunkMsgIds); + + // Redeliver chunked message + tracker.add(chunkedMessageId); + + Awaitility.await() + .pollInterval(Duration.ofMillis(200)) + .atMost(Duration.ofSeconds(3)) + .untilAsserted(() -> assertEquals(tracker.size(), 0)); + + // Assert that all chunk message ID are removed from unAckedChunkedMessageIdSequenceMap + assertEquals(consumer.unAckedChunkedMessageIdSequenceMap.size(), 0); + + timer.stop(); + } + } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/AuthenticationTokenTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/AuthenticationTokenTest.java index 589258eb09efb..a6e529d994031 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/AuthenticationTokenTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/AuthenticationTokenTest.java @@ -28,6 +28,7 @@ import java.nio.charset.StandardCharsets; import java.util.function.Supplier; +import lombok.Cleanup; import org.apache.commons.io.FileUtils; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.AuthenticationDataProvider; @@ -68,6 +69,7 @@ public void testAuthTokenClientConfig() throws Exception { clientConfig.setAuthentication(AuthenticationFactory.create( AuthenticationToken.class.getName(), "token-xyz")); + @Cleanup PulsarClientImpl pulsarClient = new PulsarClientImpl(clientConfig); Authentication authToken = pulsarClient.getConfiguration().getAuthentication(); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ClientConfigurationDataTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ClientConfigurationDataTest.java index 5856395566a67..27f521ef1ff73 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ClientConfigurationDataTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ClientConfigurationDataTest.java @@ -22,7 +22,14 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; +import io.opentelemetry.api.OpenTelemetry; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import lombok.Cleanup; import org.apache.pulsar.client.impl.auth.AuthenticationToken; +import org.testng.Assert; import org.testng.annotations.Test; /** @@ -36,10 +43,35 @@ public void testDoNotPrintSensitiveInfo() throws JsonProcessingException { clientConfigurationData.setTlsTrustStorePassword("xxxx"); clientConfigurationData.setSocks5ProxyPassword("yyyy"); clientConfigurationData.setAuthentication(new AuthenticationToken("zzzz")); + clientConfigurationData.setOpenTelemetry(OpenTelemetry.noop()); ObjectMapper objectMapper = new ObjectMapper(); objectMapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); String serializedConf = objectMapper.writeValueAsString(clientConfigurationData); assertThat(serializedConf).doesNotContain("xxxx", "yyyy", "zzzz"); } + @Test + public void testSerializable() throws Exception { + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setConnectionsPerBroker(3); + conf.setTlsTrustStorePassword("xxxx"); + conf.setOpenTelemetry(OpenTelemetry.noop()); + + @Cleanup + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + @Cleanup + ObjectOutputStream oos = new ObjectOutputStream(bos); + oos.writeObject(conf); + byte[] serialized = bos.toByteArray(); + + // Deserialize + @Cleanup + ByteArrayInputStream bis = new ByteArrayInputStream(serialized); + @Cleanup + ObjectInputStream ois = new ObjectInputStream(bis); + Object object = ois.readObject(); + + Assert.assertEquals(object.getClass(), ClientConfigurationData.class); + Assert.assertEquals(object, conf); + } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ConfigurationDataUtilsTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ConfigurationDataUtilsTest.java index 354d25f5d7fe8..cf4a2ae1813aa 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ConfigurationDataUtilsTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ConfigurationDataUtilsTest.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableSet; +import lombok.Cleanup; import org.testng.Assert; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; @@ -29,6 +30,8 @@ import java.io.IOException; import java.net.InetSocketAddress; +import java.util.Arrays; +import java.util.List; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -62,7 +65,10 @@ public void testLoadClientConfigurationData() { config.put("authParamMap", authParamMap); config.put("dnsLookupBindAddress", "0.0.0.0"); config.put("dnsLookupBindPort", 0); - + List dnsServerAddresses = Arrays.asList(new InetSocketAddress[] { + new InetSocketAddress("1.1.1.1", 53), new InetSocketAddress("2.2.2.2",100) + }); + config.put("dnsServerAddresses", dnsServerAddresses); confData = ConfigurationDataUtils.loadData(config, confData, ClientConfigurationData.class); assertEquals("pulsar://localhost:6650", confData.getServiceUrl()); assertEquals(70000, confData.getMaxLookupRequest()); @@ -73,6 +79,7 @@ public void testLoadClientConfigurationData() { assertEquals("v2", confData.getAuthParamMap().get("k2")); assertEquals("0.0.0.0", confData.getDnsLookupBindAddress()); assertEquals(0, confData.getDnsLookupBindPort()); + assertEquals(dnsServerAddresses, confData.getDnsServerAddresses()); } @Test @@ -148,6 +155,7 @@ public void testConfigBuilder() throws PulsarClientException { clientConfig.setServiceUrl("pulsar://unknown:6650"); clientConfig.setStatsIntervalSeconds(80); + @Cleanup PulsarClientImpl pulsarClient = new PulsarClientImpl(clientConfig); assertNotNull(pulsarClient, "Pulsar client built using config should not be null"); @@ -213,6 +221,7 @@ public void testSocks5() throws PulsarClientException { clientConfig.setSocks5ProxyUsername("test"); clientConfig.setSocks5ProxyPassword("test123"); + @Cleanup PulsarClientImpl pulsarClient = new PulsarClientImpl(clientConfig); assertEquals(pulsarClient.getConfiguration().getSocks5ProxyAddress(), new InetSocketAddress("localhost", 11080)); assertEquals(pulsarClient.getConfiguration().getSocks5ProxyUsername(), "test"); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/ProtobufSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/ProtobufSchemaTest.java index 3fcd6f12b982d..85012276d5af1 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/ProtobufSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/ProtobufSchemaTest.java @@ -41,20 +41,20 @@ public class ProtobufSchemaTest { "\"namespace\":\"org.apache.pulsar.client.schema.proto.Test\"," + "\"fields\":[{\"name\":\"stringField\",\"type\":{\"type\":\"string\"," + "\"avro.java.string\":\"String\"},\"default\":\"\"},{\"name\":\"doubleField\"," + - "\"type\":\"double\",\"default\":0},{\"name\":\"intField\",\"type\":\"int\"," + + "\"type\":\"double\",\"default\":0.0},{\"name\":\"intField\",\"type\":\"int\"," + "\"default\":0},{\"name\":\"testEnum\",\"type\":{\"type\":\"enum\"," + "\"name\":\"TestEnum\",\"symbols\":[\"SHARED\",\"FAILOVER\"]}," + "\"default\":\"SHARED\"},{\"name\":\"nestedField\"," + "\"type\":[\"null\",{\"type\":\"record\",\"name\":\"SubMessage\"," + "\"fields\":[{\"name\":\"foo\",\"type\":{\"type\":\"string\"," + "\"avro.java.string\":\"String\"},\"default\":\"\"}" + - ",{\"name\":\"bar\",\"type\":\"double\",\"default\":0}]}]" + + ",{\"name\":\"bar\",\"type\":\"double\",\"default\":0.0}]}]" + ",\"default\":null},{\"name\":\"repeatedField\",\"type\":{\"type\":\"array\"" + ",\"items\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},\"default\":[]}" + ",{\"name\":\"externalMessage\",\"type\":[\"null\",{\"type\":\"record\"" + ",\"name\":\"ExternalMessage\",\"namespace\":\"org.apache.pulsar.client.schema.proto.ExternalTest\"" + ",\"fields\":[{\"name\":\"stringField\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}," + - "\"default\":\"\"},{\"name\":\"doubleField\",\"type\":\"double\",\"default\":0}]}],\"default\":null}]}"; + "\"default\":\"\"},{\"name\":\"doubleField\",\"type\":\"double\",\"default\":0.0}]}],\"default\":null}]}"; private static final String EXPECTED_PARSING_INFO = "{\"__alwaysAllowNull\":\"true\",\"__jsr310ConversionEnabled\":\"false\"," + "\"__PARSING_INFO__\":\"[{\\\"number\\\":1,\\\"name\\\":\\\"stringField\\\",\\\"type\\\":\\\"STRING\\\"," + diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/GenericProtobufNativeReaderTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/GenericProtobufNativeReaderTest.java index 4cbb325c82f0c..c358f30ccae73 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/GenericProtobufNativeReaderTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/GenericProtobufNativeReaderTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.impl.schema.generic; +import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.schema.GenericRecord; @@ -29,6 +30,7 @@ import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; @Slf4j public class GenericProtobufNativeReaderTest { @@ -79,6 +81,12 @@ public void testGetNativeRecord() { assertEquals(nativeRecord.getField(nativeRecord.getDescriptorForType().findFieldByName("doubleField")), DOUBLE_FIELD_VLUE); } + @Test + public void testGetNativeSchema() { + assertTrue(genericProtobufNativeSchema.getNativeSchema().isPresent()); + assertTrue(genericProtobufNativeSchema.getNativeSchema().get() instanceof Descriptors.Descriptor); + } + private static final String STRING_FIELD_VLUE = "stringFieldValue"; private static final double DOUBLE_FIELD_VLUE = 0.2D; diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/MultiVersionSchemaInfoProviderTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/MultiVersionSchemaInfoProviderTest.java index 8959e67023463..bfd6af37e3ea6 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/MultiVersionSchemaInfoProviderTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/MultiVersionSchemaInfoProviderTest.java @@ -27,6 +27,7 @@ import java.util.concurrent.CompletableFuture; import org.apache.pulsar.client.api.schema.SchemaDefinition; +import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.LookupService; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.schema.AvroSchema; @@ -46,6 +47,8 @@ public class MultiVersionSchemaInfoProviderTest { @BeforeMethod public void setup() { PulsarClientImpl client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); when(client.getLookup()).thenReturn(mock(LookupService.class)); schemaProvider = new MultiVersionSchemaInfoProvider( TopicName.get("persistent://public/default/my-topic"), client); diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index a7d4dcf6beeca..c34b89ff92ca7 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -26,8 +26,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-common @@ -50,6 +49,7 @@ io.swagger swagger-annotations + provided @@ -98,10 +98,15 @@ linux-x86_64 + + io.netty + netty-transport-native-epoll + linux-aarch_64 + + io.netty netty-transport-native-unix-common - linux-x86_64 @@ -194,12 +199,22 @@ provided true - + com.google.protobuf protobuf-java + + com.google.re2j + re2j + + + + com.spotify + completable-futures + + org.bouncycastle @@ -226,14 +241,15 @@ snappy-java test + com.google.code.gson gson - com.beust - jcommander + com.google.re2j + re2j @@ -296,6 +312,7 @@ + false true git false diff --git a/pulsar-common/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java b/pulsar-common/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java index 73ecfed0ad059..76d41a7d3d4fc 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java @@ -75,6 +75,8 @@ default int getBatchSize() { * @implNote The message IDs of a batch should share a BitSet. For example, given 3 messages in the same batch whose * size is 3, all message IDs of them should return "111" (i.e. a BitSet whose size is 3 and all bits are 1). If the * 1st message has been acknowledged, the returned BitSet should become "011" (i.e. the 1st bit become 0). + * If the caller performs any read or write operations on the return value of this method, they should do so with + * lock protection. * * @return null if the message is a non-batched message */ diff --git a/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaUtils.java b/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaUtils.java index 8acbf26559b7b..881ad424669d2 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaUtils.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaUtils.java @@ -359,8 +359,7 @@ private static byte[] getKeyOrValueSchemaBytes(JsonElement jsonElement) { * @param keyValueSchemaInfoDataJsonBytes the key/value schema info data json bytes * @return the key/value schema info data bytes */ - public static byte[] convertKeyValueDataStringToSchemaInfoSchema( - byte[] keyValueSchemaInfoDataJsonBytes) throws IOException { + public static byte[] convertKeyValueDataStringToSchemaInfoSchema(byte[] keyValueSchemaInfoDataJsonBytes) { JsonObject jsonObject = (JsonObject) toJsonElement(new String(keyValueSchemaInfoDataJsonBytes, UTF_8)); byte[] keyBytes = getKeyOrValueSchemaBytes(jsonObject.get("key")); byte[] valueBytes = getKeyOrValueSchemaBytes(jsonObject.get("value")); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/lookup/GetTopicsResult.java b/pulsar-common/src/main/java/org/apache/pulsar/common/lookup/GetTopicsResult.java index 55fe6253ff971..26a295264fcae 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/lookup/GetTopicsResult.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/lookup/GetTopicsResult.java @@ -18,21 +18,129 @@ */ package org.apache.pulsar.common.lookup; +import com.google.re2j.Pattern; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; -import lombok.AllArgsConstructor; +import java.util.Set; import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; import lombok.ToString; +import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace; +import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespaceResponse; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.topics.TopicList; -@Getter -@Setter -@AllArgsConstructor -@NoArgsConstructor +/*** + * A value object. + * - The response of HTTP API "admin/v2/namespaces/{domain}/topics" is a topic(non-partitioned topic or partitions) + * array. It will be wrapped to "topics: {topic array}, topicsHash: null, filtered: false, changed: true". + * - The response of binary API {@link CommandGetTopicsOfNamespace} is a {@link CommandGetTopicsOfNamespaceResponse}, + * it will be transferred to a {@link GetTopicsResult}. + * See more details https://github.com/apache/pulsar/pull/14804. + */ @ToString public class GetTopicsResult { - private List topics; - private String topicsHash; - private boolean filtered; - private boolean changed; + + /** + * Non-partitioned topics, and topic partitions of partitioned topics. + */ + @Getter + private final List nonPartitionedOrPartitionTopics; + + /** + * The topics have been filtered by Broker using a regexp. Otherwise, the client should do a client-side filter. + * There are three cases that brokers will not filter the topics: + * 1. the lookup service is typed HTTP lookup service, the HTTP API has not implemented this feature yet. + * 2. the broker does not support this feature(in other words, its version is lower than "2.11.0"). + * 3. the input param "topicPattern" is too long than the broker config "subscriptionPatternMaxLength". + */ + @Getter + private final boolean filtered; + + /** + * The topics hash that was calculated by {@link TopicList#calculateHash(List)}. The param topics that will be used + * to calculate the hash code is only contains the topics that has been filtered. + * Note: It is always "null" if broker did not filter the topics when calling the API + * "LookupService.getTopicsUnderNamespace"(in other words, {@link #filtered} is false). + */ + @Getter + private final String topicsHash; + + /** + * The topics hash has changed after compare with the input param "topicsHash" when calling + * "LookupService.getTopicsUnderNamespace". + * Note: It is always set "true" if the input param "topicsHash" that used to call + * "LookupService.getTopicsUnderNamespace" is null or the "LookupService" is "HttpLookupService". + */ + @Getter + private final boolean changed; + + /** + * Partitioned topics and non-partitioned topics. + * In other words, there is no topic partitions of partitioned topics in this list. + * Note: it is not a field of the response of "LookupService.getTopicsUnderNamespace", it is generated in + * client-side memory. + */ + private volatile List topics; + + /** + * This constructor is used for binary API. + */ + public GetTopicsResult(List nonPartitionedOrPartitionTopics, String topicsHash, boolean filtered, + boolean changed) { + this.nonPartitionedOrPartitionTopics = nonPartitionedOrPartitionTopics; + this.topicsHash = topicsHash; + this.filtered = filtered; + this.changed = changed; + } + + /** + * This constructor is used for HTTP API. + */ + public GetTopicsResult(String[] nonPartitionedOrPartitionTopics) { + this(Arrays.asList(nonPartitionedOrPartitionTopics), null, false, true); + } + + public List getTopics() { + if (topics != null) { + return topics; + } + synchronized (this) { + if (topics != null) { + return topics; + } + // Group partitioned topics. + List grouped = new ArrayList<>(); + for (String topic : nonPartitionedOrPartitionTopics) { + String partitionedTopic = TopicName.get(topic).getPartitionedTopicName(); + if (!grouped.contains(partitionedTopic)) { + grouped.add(partitionedTopic); + } + } + topics = grouped; + return topics; + } + } + + public GetTopicsResult filterTopics(Pattern topicsPattern) { + List topicsFiltered = TopicList.filterTopics(getTopics(), topicsPattern); + // If nothing changed. + if (topicsFiltered.equals(getTopics())) { + GetTopicsResult newObj = new GetTopicsResult(nonPartitionedOrPartitionTopics, null, true, true); + newObj.topics = topics; + return newObj; + } + // Filtered some topics. + Set topicsFilteredSet = new HashSet<>(topicsFiltered); + List newTps = new ArrayList<>(); + for (String tp: nonPartitionedOrPartitionTopics) { + if (topicsFilteredSet.contains(TopicName.get(tp).getPartitionedTopicName())) { + newTps.add(tp); + } + } + GetTopicsResult newObj = new GetTopicsResult(newTps, null, true, true); + newObj.topics = topicsFiltered; + return newObj; + } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/SystemTopicNames.java b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/SystemTopicNames.java index 8fc7d014b5784..9a3689912c926 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/SystemTopicNames.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/SystemTopicNames.java @@ -51,6 +51,11 @@ public class SystemTopicNames { public static final String PENDING_ACK_STORE_CURSOR_NAME = "__pending_ack_state"; + /** + * Prefix for the system reader for all the system topics. + */ + public static final String SYSTEM_READER_PREFIX = "__system_reader"; + /** * The set of all local topic names declared above. */ @@ -81,7 +86,7 @@ public static boolean isTopicPoliciesSystemTopic(String topic) { if (topic == null) { return false; } - return TopicName.get(topic).getLocalName().equals(NAMESPACE_EVENTS_LOCAL_NAME); + return TopicName.getPartitionedTopicName(topic).getLocalName().equals(NAMESPACE_EVENTS_LOCAL_NAME); } public static boolean isTransactionInternalName(TopicName topicName) { @@ -92,7 +97,7 @@ public static boolean isTransactionInternalName(TopicName topicName) { } public static boolean isSystemTopic(TopicName topicName) { - TopicName nonePartitionedTopicName = TopicName.get(topicName.getPartitionedTopicName()); - return isEventSystemTopic(nonePartitionedTopicName) || isTransactionInternalName(nonePartitionedTopicName); + TopicName nonPartitionedTopicName = TopicName.get(topicName.getPartitionedTopicName()); + return isEventSystemTopic(nonPartitionedTopicName) || isTransactionInternalName(nonPartitionedTopicName); } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java index 79ef64c1ae459..dd24c9a971210 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java @@ -19,15 +19,11 @@ package org.apache.pulsar.common.naming; import com.google.common.base.Splitter; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.util.concurrent.UncheckedExecutionException; +import com.google.re2j.Pattern; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.List; import java.util.Objects; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.common.util.Codec; @@ -53,13 +49,17 @@ public class TopicName implements ServiceUnitId { private final int partitionIndex; - private static final LoadingCache cache = CacheBuilder.newBuilder().maximumSize(100000) - .expireAfterAccess(30, TimeUnit.MINUTES).build(new CacheLoader() { - @Override - public TopicName load(String name) throws Exception { - return new TopicName(name); - } - }); + private static final ConcurrentHashMap cache = new ConcurrentHashMap<>(); + + public static void clearIfReachedMaxCapacity(int maxCapacity) { + if (maxCapacity < 0) { + // Unlimited cache. + return; + } + if (cache.size() > maxCapacity) { + cache.clear(); + } + } public static TopicName get(String domain, NamespaceName namespaceName, String topic) { String name = domain + "://" + namespaceName.toString() + '/' + topic; @@ -78,11 +78,11 @@ public static TopicName get(String domain, String tenant, String cluster, String } public static TopicName get(String topic) { - try { - return cache.get(topic); - } catch (ExecutionException | UncheckedExecutionException e) { - throw (RuntimeException) e.getCause(); + TopicName tp = cache.get(topic); + if (tp != null) { + return tp; } + return cache.computeIfAbsent(topic, k -> new TopicName(k)); } public static TopicName getPartitionedTopicName(String topic) { @@ -102,6 +102,14 @@ public static boolean isValid(String topic) { } } + public static String getPartitionPattern(String topic) { + return "^" + Pattern.quote(get(topic).getPartitionedTopicName().toString()) + "-partition-[0-9]+$"; + } + + public static String getPattern(String topic) { + return "^" + Pattern.quote(get(topic).getPartitionedTopicName().toString()) + "$"; + } + @SuppressFBWarnings("DCN_NULLPOINTER_EXCEPTION") private TopicName(String completeTopicName) { try { @@ -339,6 +347,41 @@ public String getPersistenceNamingEncoding() { } } + /** + * get topic full name from managedLedgerName. + * + * @return the topic full name, format -> domain://tenant/namespace/topic + */ + public static String fromPersistenceNamingEncoding(String mlName) { + // The managedLedgerName convention is: tenant/namespace/domain/topic + // We want to transform to topic full name in the order: domain://tenant/namespace/topic + if (mlName == null || mlName.length() == 0) { + return mlName; + } + List parts = Splitter.on("/").splitToList(mlName); + String tenant; + String cluster; + String namespacePortion; + String domain; + String localName; + if (parts.size() == 4) { + tenant = parts.get(0); + namespacePortion = parts.get(1); + domain = parts.get(2); + localName = Codec.decode(parts.get(3)); + return String.format("%s://%s/%s/%s", domain, tenant, namespacePortion, localName); + } else if (parts.size() == 5) { + tenant = parts.get(0); + cluster = parts.get(1); + namespacePortion = parts.get(2); + domain = parts.get(3); + localName = Codec.decode(parts.get(4)); + return String.format("%s://%s/%s/%s/%s", domain, tenant, cluster, namespacePortion, localName); + } else { + throw new IllegalArgumentException("Invalid managedLedger name: " + mlName); + } + } + /** * Get a string suitable for completeTopicName lookup. * diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoader.java b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoader.java index 620e1156d3555..44cfc2872ef6b 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoader.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoader.java @@ -40,6 +40,7 @@ import java.util.Comparator; import java.util.List; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -135,6 +136,7 @@ public class NarClassLoader extends URLClassLoader { * The NAR for which this ClassLoader is responsible. */ private final File narWorkingDirectory; + private final AtomicBoolean closed = new AtomicBoolean(); private static final String TMP_DIR_PREFIX = "pulsar-nar"; @@ -154,6 +156,11 @@ public NarClassLoader run() { }); } + public static List getClasspathFromArchive(File narPath, String narExtractionDirectory) throws IOException { + File unpacked = NarUnpacker.unpackNar(narPath, getNarExtractionDirectory(narExtractionDirectory)); + return getClassPathEntries(unpacked); + } + private static File getNarExtractionDirectory(String configuredDirectory) { return new File(configuredDirectory + "/" + TMP_DIR_PREFIX); } @@ -164,16 +171,11 @@ private static File getNarExtractionDirectory(String configuredDirectory) { * @param narWorkingDirectory * directory to explode nar contents to * @param parent - * @throws IllegalArgumentException - * if the NAR is missing the Java Services API file for FlowFileProcessor implementations. - * @throws ClassNotFoundException - * if any of the FlowFileProcessor implementations defined by the Java Services API cannot be - * loaded. * @throws IOException * if an error occurs while loading the NAR. */ private NarClassLoader(final File narWorkingDirectory, Set additionalJars, ClassLoader parent) - throws ClassNotFoundException, IOException { + throws IOException { super(new URL[0], parent); this.narWorkingDirectory = narWorkingDirectory; @@ -238,22 +240,31 @@ public List getServiceImplementation(String serviceName) throws IOExcept * if the URL list could not be updated. */ private void updateClasspath(File root) throws IOException { - addURL(root.toURI().toURL()); // for compiled classes, META-INF/, etc. + getClassPathEntries(root).forEach(f -> { + try { + addURL(f.toURI().toURL()); + } catch (IOException e) { + log.error("Failed to add entry to classpath: {}", f, e); + } + }); + } + static List getClassPathEntries(File root) { + List classPathEntries = new ArrayList<>(); + classPathEntries.add(root); File dependencies = new File(root, "META-INF/bundled-dependencies"); if (!dependencies.isDirectory()) { - log.warn("{} does not contain META-INF/bundled-dependencies!", narWorkingDirectory); + log.warn("{} does not contain META-INF/bundled-dependencies!", root); } - addURL(dependencies.toURI().toURL()); + classPathEntries.add(dependencies); if (dependencies.isDirectory()) { final File[] jarFiles = dependencies.listFiles(JAR_FILTER); if (jarFiles != null) { Arrays.sort(jarFiles, Comparator.comparing(File::getName)); - for (File libJar : jarFiles) { - addURL(libJar.toURI().toURL()); - } + classPathEntries.addAll(Arrays.asList(jarFiles)); } } + return classPathEntries; } @Override @@ -283,4 +294,18 @@ protected String findLibrary(final String libname) { public String toString() { return NarClassLoader.class.getName() + "[" + narWorkingDirectory.getPath() + "]"; } + + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + if (closed.get()) { + log.warn("Loading class {} from a closed classloader ({})", name, this); + } + return super.loadClass(name, resolve); + } + + @Override + public void close() throws IOException { + closed.set(true); + super.close(); + } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java index 9bd5bc48df819..ef802674b421a 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java @@ -32,13 +32,16 @@ import java.io.RandomAccessFile; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.Base64; import java.util.Enumeration; import java.util.concurrent.ConcurrentHashMap; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; import lombok.extern.slf4j.Slf4j; /** @@ -85,19 +88,32 @@ static File doUnpackNar(final File nar, final File baseWorkingDirectory, Runnabl try (FileChannel channel = new RandomAccessFile(lockFile, "rw").getChannel(); FileLock lock = channel.lock()) { File narWorkingDirectory = new File(parentDirectory, md5Sum); - if (narWorkingDirectory.mkdir()) { + if (!narWorkingDirectory.exists()) { + File narExtractionTempDirectory = new File(parentDirectory, md5Sum + ".tmp"); + if (narExtractionTempDirectory.exists()) { + FileUtils.deleteFile(narExtractionTempDirectory, true); + } + if (!narExtractionTempDirectory.mkdir()) { + throw new IOException("Cannot create " + narExtractionTempDirectory); + } try { - log.info("Extracting {} to {}", nar, narWorkingDirectory); + log.info("Extracting {} to {}", nar, narExtractionTempDirectory); if (extractCallback != null) { extractCallback.run(); } - unpack(nar, narWorkingDirectory); + unpack(nar, narExtractionTempDirectory); } catch (IOException e) { log.error("There was a problem extracting the nar file. Deleting {} to clean up state.", - narWorkingDirectory, e); - FileUtils.deleteFile(narWorkingDirectory, true); + narExtractionTempDirectory, e); + try { + FileUtils.deleteFile(narExtractionTempDirectory, true); + } catch (IOException e2) { + log.error("Failed to delete temporary directory {}", narExtractionTempDirectory, e2); + } throw e; } + Files.move(narExtractionTempDirectory.toPath(), narWorkingDirectory.toPath(), + StandardCopyOption.ATOMIC_MOVE); } return narWorkingDirectory; } @@ -113,18 +129,24 @@ static File doUnpackNar(final File nar, final File baseWorkingDirectory, Runnabl * if the NAR could not be unpacked. */ private static void unpack(final File nar, final File workingDirectory) throws IOException { - try (JarFile jarFile = new JarFile(nar)) { - Enumeration jarEntries = jarFile.entries(); - while (jarEntries.hasMoreElements()) { - JarEntry jarEntry = jarEntries.nextElement(); - String name = jarEntry.getName(); - File f = new File(workingDirectory, name); - if (jarEntry.isDirectory()) { + Path workingDirectoryPath = workingDirectory.toPath().normalize(); + try (ZipFile zipFile = new ZipFile(nar)) { + Enumeration zipEntries = zipFile.entries(); + while (zipEntries.hasMoreElements()) { + ZipEntry zipEntry = zipEntries.nextElement(); + String name = zipEntry.getName(); + Path targetFilePath = workingDirectoryPath.resolve(name).normalize(); + if (!targetFilePath.startsWith(workingDirectoryPath)) { + log.error("Invalid zip file with entry '{}'", name); + throw new IOException("Invalid zip file. Aborting unpacking."); + } + File f = targetFilePath.toFile(); + if (zipEntry.isDirectory()) { FileUtils.ensureDirectoryExistAndCanReadAndWrite(f); } else { // The directory entry might appear after the file entry FileUtils.ensureDirectoryExistAndCanReadAndWrite(f.getParentFile()); - makeFile(jarFile.getInputStream(jarEntry), f); + makeFile(zipFile.getInputStream(zipEntry), f); } } } @@ -159,8 +181,9 @@ private static void makeFile(final InputStream inputStream, final File file) thr * @throws IOException * if cannot read file */ - private static byte[] calculateMd5sum(final File file) throws IOException { + protected static byte[] calculateMd5sum(final File file) throws IOException { try (final FileInputStream inputStream = new FileInputStream(file)) { + // codeql[java/weak-cryptographic-algorithm] - md5 is sufficient for this use case final MessageDigest md5 = MessageDigest.getInstance("md5"); final byte[] buffer = new byte[1024]; @@ -176,4 +199,4 @@ private static byte[] calculateMd5sum(final File file) throws IOException { throw new IllegalArgumentException(nsae); } } -} +} \ No newline at end of file diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/NamespaceIsolationPolicy.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/NamespaceIsolationPolicy.java index bd28d30d4cee9..52480d91eefa4 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/NamespaceIsolationPolicy.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/NamespaceIsolationPolicy.java @@ -23,6 +23,7 @@ import java.util.SortedSet; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.policies.data.BrokerStatus; +import org.apache.pulsar.common.policies.data.NamespaceIsolationPolicyUnloadScope; /** * Namespace isolation policy. @@ -43,6 +44,11 @@ public interface NamespaceIsolationPolicy { */ List getSecondaryBrokers(); + /** + * Get the unload scope for the policy set call. + */ + NamespaceIsolationPolicyUnloadScope getUnloadScope(); + /** * Get the list of primary brokers for the namespace according to the policy. * diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/ClusterDataImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/ClusterDataImpl.java index 2ca75245a8c22..b887fe0a5861b 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/ClusterDataImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/ClusterDataImpl.java @@ -25,7 +25,10 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.ProxyProtocol; +import org.apache.pulsar.common.util.DefaultPulsarSslFactory; import org.apache.pulsar.common.util.URIPreconditions; /** @@ -38,6 +41,7 @@ @Data @AllArgsConstructor @NoArgsConstructor +@Slf4j public final class ClusterDataImpl implements ClusterData, Cloneable { @ApiModelProperty( name = "serviceUrl", @@ -104,7 +108,7 @@ public final class ClusterDataImpl implements ClusterData, Cloneable { private boolean brokerClientTlsEnabled; @ApiModelProperty( name = "tlsAllowInsecureConnection", - value = "Allow TLS connections to servers whose certificate cannot be" + value = "Allow TLS connections to servers whose certificate cannot" + " be verified to have been signed by a trusted certificate" + " authority." ) @@ -168,22 +172,21 @@ public final class ClusterDataImpl implements ClusterData, Cloneable { ) private String brokerClientCertificateFilePath; @ApiModelProperty( - name = "listenerName", - value = "listenerName when client would like to connect to cluster", - example = "" + name = "brokerClientSslFactoryPlugin", + value = "SSL Factory plugin used by internal client to generate the SSL Context and Engine" ) - private String listenerName; + private String brokerClientSslFactoryPlugin; @ApiModelProperty( - name = "migrated", - value = "flag to check if cluster is migrated to different cluster", - example = "true/false" + name = "brokerClientSslFactoryPluginParams", + value = "Parameters used by the internal client's SSL factory plugin to generate the SSL Context and Engine" ) - private boolean migrated; + private String brokerClientSslFactoryPluginParams; @ApiModelProperty( - name = "migratedClusterUrl", - value = "url of cluster where current cluster is migrated" + name = "listenerName", + value = "listenerName when client would like to connect to cluster", + example = "" ) - private ClusterUrl migratedClusterUrl; + private String listenerName; public static ClusterDataImplBuilder builder() { return new ClusterDataImplBuilder(); @@ -213,9 +216,9 @@ public ClusterDataImplBuilder clone() { .brokerClientTrustCertsFilePath(brokerClientTrustCertsFilePath) .brokerClientCertificateFilePath(brokerClientCertificateFilePath) .brokerClientKeyFilePath(brokerClientKeyFilePath) - .listenerName(listenerName) - .migrated(migrated) - .migratedClusterUrl(migratedClusterUrl); + .brokerClientSslFactoryPlugin(brokerClientSslFactoryPlugin) + .brokerClientSslFactoryPluginParams(brokerClientSslFactoryPluginParams) + .listenerName(listenerName); } @Data @@ -241,9 +244,9 @@ public static class ClusterDataImplBuilder implements ClusterData.Builder { private String brokerClientCertificateFilePath; private String brokerClientKeyFilePath; private String brokerClientTrustCertsFilePath; + private String brokerClientSslFactoryPlugin = DefaultPulsarSslFactory.class.getName(); + private String brokerClientSslFactoryPluginParams; private String listenerName; - private boolean migrated; - private ClusterUrl migratedClusterUrl; ClusterDataImplBuilder() { } @@ -358,19 +361,20 @@ public ClusterDataImplBuilder brokerClientKeyFilePath(String keyFilePath) { return this; } - - public ClusterDataImplBuilder listenerName(String listenerName) { - this.listenerName = listenerName; + @Override + public ClusterDataImplBuilder brokerClientSslFactoryPlugin(String sslFactoryPlugin) { + this.brokerClientSslFactoryPlugin = sslFactoryPlugin; return this; } - public ClusterDataImplBuilder migrated(boolean migrated) { - this.migrated = migrated; + @Override + public ClusterDataImplBuilder brokerClientSslFactoryPluginParams(String sslFactoryPluginParams) { + this.brokerClientSslFactoryPluginParams = sslFactoryPluginParams; return this; } - public ClusterDataImplBuilder migratedClusterUrl(ClusterUrl migratedClusterUrl) { - this.migratedClusterUrl = migratedClusterUrl; + public ClusterDataImplBuilder listenerName(String listenerName) { + this.listenerName = listenerName; return this; } @@ -397,9 +401,9 @@ public ClusterDataImpl build() { brokerClientTrustCertsFilePath, brokerClientKeyFilePath, brokerClientCertificateFilePath, - listenerName, - migrated, - migratedClusterUrl); + brokerClientSslFactoryPlugin, + brokerClientSslFactoryPluginParams, + listenerName); } } @@ -427,5 +431,20 @@ public void checkPropertiesIfPresent() throws IllegalArgumentException { || Objects.equals(uri.getScheme(), "pulsar+ssl"), "Illegal proxy service url, example: pulsar+ssl://ats-proxy.example.com:4443 " + "or pulsar://ats-proxy.example.com:4080"); + + warnIfUrlIsNotPresent(); + } + + private void warnIfUrlIsNotPresent() { + if (StringUtils.isEmpty(getServiceUrl()) && StringUtils.isEmpty(getServiceUrlTls())) { + log.warn("Service url not found, " + + "please provide either service url, example: http://pulsar.example.com:8080 " + + "or service tls url, example: https://pulsar.example.com:8443"); + } + if (StringUtils.isEmpty(getBrokerServiceUrl()) && StringUtils.isEmpty(getBrokerServiceUrlTls())) { + log.warn("Broker service url not found, " + + "please provide either broker service url, example: pulsar://pulsar.example.com:6650 " + + "or broker service tls url, example: pulsar+ssl://pulsar.example.com:6651."); + } } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/ClusterPoliciesImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/ClusterPoliciesImpl.java new file mode 100644 index 0000000000000..c8af2dec3216b --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/ClusterPoliciesImpl.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.policies.data; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * The configuration data for a cluster. + */ +@ApiModel( + value = "ClusterPolicies", + description = "The local cluster policies for a cluster" +) +@Data +@AllArgsConstructor +@NoArgsConstructor +public final class ClusterPoliciesImpl implements ClusterPolicies, Cloneable { + @ApiModelProperty( + name = "migrated", + value = "flag to check if cluster is migrated to different cluster", + example = "true/false" + ) + private boolean migrated; + @ApiModelProperty( + name = "migratedClusterUrl", + value = "url of cluster where current cluster is migrated" + ) + private ClusterUrl migratedClusterUrl; + + public static ClusterPoliciesImplBuilder builder() { + return new ClusterPoliciesImplBuilder(); + } + + @Override + public ClusterPoliciesImplBuilder clone() { + return builder() + .migrated(migrated) + .migratedClusterUrl(migratedClusterUrl); + } + + @Data + public static class ClusterPoliciesImplBuilder implements ClusterPolicies.Builder { + private boolean migrated; + private ClusterUrl migratedClusterUrl; + + ClusterPoliciesImplBuilder() { + } + + public ClusterPoliciesImplBuilder migrated(boolean migrated) { + this.migrated = migrated; + return this; + } + + public ClusterPoliciesImplBuilder migratedClusterUrl(ClusterUrl migratedClusterUrl) { + this.migratedClusterUrl = migratedClusterUrl; + return this; + } + + public ClusterPoliciesImpl build() { + return new ClusterPoliciesImpl( + migrated, + migratedClusterUrl); + } + } +} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java index 0249272b72d84..4edb033498bc0 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java @@ -49,7 +49,9 @@ public class HierarchyTopicPolicies { final PolicyHierarchyValue maxConsumerPerTopic; final PolicyHierarchyValue publishRate; final PolicyHierarchyValue delayedDeliveryEnabled; + final PolicyHierarchyValue dispatcherPauseOnAckStatePersistentEnabled; final PolicyHierarchyValue delayedDeliveryTickTimeMillis; + final PolicyHierarchyValue delayedDeliveryMaxDelayInMillis; final PolicyHierarchyValue replicatorDispatchRate; final PolicyHierarchyValue maxConsumersPerSubscription; final PolicyHierarchyValue subscribeRate; @@ -81,7 +83,9 @@ public HierarchyTopicPolicies() { messageTTLInSeconds = new PolicyHierarchyValue<>(); publishRate = new PolicyHierarchyValue<>(); delayedDeliveryEnabled = new PolicyHierarchyValue<>(); + dispatcherPauseOnAckStatePersistentEnabled = new PolicyHierarchyValue<>(); delayedDeliveryTickTimeMillis = new PolicyHierarchyValue<>(); + delayedDeliveryMaxDelayInMillis = new PolicyHierarchyValue<>(); replicatorDispatchRate = new PolicyHierarchyValue<>(); compactionThreshold = new PolicyHierarchyValue<>(); subscribeRate = new PolicyHierarchyValue<>(); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/LocalPolicies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/LocalPolicies.java index e8a158ace70c5..3b17dbe067ebd 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/LocalPolicies.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/LocalPolicies.java @@ -34,6 +34,7 @@ public class LocalPolicies { public final BookieAffinityGroupData bookieAffinityGroup; // namespace anti-affinity-group public final String namespaceAntiAffinityGroup; + public boolean migrated; public LocalPolicies() { bundles = defaultBundle(); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/NamespaceIsolationDataImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/NamespaceIsolationDataImpl.java index bdb51f63f89ed..85be8090f52a1 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/NamespaceIsolationDataImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/NamespaceIsolationDataImpl.java @@ -75,6 +75,15 @@ public class NamespaceIsolationDataImpl implements NamespaceIsolationData { @JsonProperty("auto_failover_policy") private AutoFailoverPolicyData autoFailoverPolicy; + @ApiModelProperty( + name = "unload_scope", + value = "The type of unload to perform while applying the new isolation policy.", + example = "'changed' (default) for unloading only the namespaces whose placement is actually changing. " + + "'all_matching' for unloading all matching namespaces. 'none' for not unloading any namespaces." + ) + @JsonProperty("unload_scope") + private NamespaceIsolationPolicyUnloadScope unloadScope; + public static NamespaceIsolationDataImplBuilder builder() { return new NamespaceIsolationDataImplBuilder(); } @@ -106,6 +115,7 @@ public static class NamespaceIsolationDataImplBuilder implements NamespaceIsolat private List primary = new ArrayList<>(); private List secondary = new ArrayList<>(); private AutoFailoverPolicyData autoFailoverPolicy; + private NamespaceIsolationPolicyUnloadScope unloadScope; public NamespaceIsolationDataImplBuilder namespaces(List namespaces) { this.namespaces = namespaces; @@ -127,8 +137,13 @@ public NamespaceIsolationDataImplBuilder autoFailoverPolicy(AutoFailoverPolicyDa return this; } + public NamespaceIsolationDataImplBuilder unloadScope(NamespaceIsolationPolicyUnloadScope unloadScope) { + this.unloadScope = unloadScope; + return this; + } + public NamespaceIsolationDataImpl build() { - return new NamespaceIsolationDataImpl(namespaces, primary, secondary, autoFailoverPolicy); + return new NamespaceIsolationDataImpl(namespaces, primary, secondary, autoFailoverPolicy, unloadScope); } } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java index fb33e3198aa60..6c40aa3f2edd0 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java @@ -30,8 +30,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Properties; +import java.util.stream.Collectors; import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -61,16 +64,31 @@ public class OffloadPoliciesImpl implements Serializable, OffloadPolicies { CONFIGURATION_FIELDS = Collections.unmodifiableList(temp); } - public static final int DEFAULT_MAX_BLOCK_SIZE_IN_BYTES = 64 * 1024 * 1024; // 64MB - public static final int DEFAULT_READ_BUFFER_SIZE_IN_BYTES = 1024 * 1024; // 1MB + public static final ImmutableList INTERNAL_SUPPORTED_DRIVER = ImmutableList.of("S3", + "aws-s3", "google-cloud-storage", "filesystem", "azureblob", "aliyun-oss"); + public static final ImmutableList DRIVER_NAMES; + static { + String extraDrivers = System.getProperty("pulsar.extra.offload.drivers", ""); + if (extraDrivers.trim().isEmpty()) { + DRIVER_NAMES = INTERNAL_SUPPORTED_DRIVER; + } else { + DRIVER_NAMES = ImmutableList.builder() + .addAll(INTERNAL_SUPPORTED_DRIVER) + .addAll(Arrays.stream(StringUtils.split(extraDrivers, ',')) + .map(String::trim).collect(Collectors.toSet())).build(); + } + } + + public static final int DEFAULT_MAX_BLOCK_SIZE_IN_BYTES = 64 * 1024 * 1024; // 64MiB + public static final int DEFAULT_GCS_MAX_BLOCK_SIZE_IN_BYTES = 128 * 1024 * 1024; // 128MiB + public static final int DEFAULT_READ_BUFFER_SIZE_IN_BYTES = 1024 * 1024; // 1MiB public static final int DEFAULT_OFFLOAD_MAX_THREADS = 2; public static final int DEFAULT_OFFLOAD_MAX_PREFETCH_ROUNDS = 1; - public static final ImmutableList DRIVER_NAMES = ImmutableList - .of("S3", "aws-s3", "google-cloud-storage", "filesystem", "azureblob", "aliyun-oss"); public static final String DEFAULT_OFFLOADER_DIRECTORY = "./offloaders"; public static final Long DEFAULT_OFFLOAD_THRESHOLD_IN_BYTES = null; public static final Long DEFAULT_OFFLOAD_THRESHOLD_IN_SECONDS = null; public static final Long DEFAULT_OFFLOAD_DELETION_LAG_IN_MILLIS = null; + public static final String EXTRA_CONFIG_PREFIX = "managedLedgerOffloadExtraConfig"; public static final String OFFLOAD_THRESHOLD_NAME_IN_CONF_FILE = "managedLedgerOffloadAutoTriggerSizeThresholdBytes"; @@ -104,7 +122,9 @@ public class OffloadPoliciesImpl implements Serializable, OffloadPolicies { @Configuration @JsonProperty(access = JsonProperty.Access.READ_WRITE) private OffloadedReadPriority managedLedgerOffloadedReadPriority = DEFAULT_OFFLOADED_READ_PRIORITY; - + @Configuration + @JsonProperty(access = JsonProperty.Access.READ_WRITE) + private Map managedLedgerExtraConfigurations = new HashMap<>(); // s3 config, set by service configuration or cli @Configuration @JsonProperty(access = JsonProperty.Access.READ_WRITE) @@ -144,7 +164,7 @@ public class OffloadPoliciesImpl implements Serializable, OffloadPolicies { private String gcsManagedLedgerOffloadBucket = null; @Configuration @JsonProperty(access = JsonProperty.Access.READ_WRITE) - private Integer gcsManagedLedgerOffloadMaxBlockSizeInBytes = DEFAULT_MAX_BLOCK_SIZE_IN_BYTES; + private Integer gcsManagedLedgerOffloadMaxBlockSizeInBytes = DEFAULT_GCS_MAX_BLOCK_SIZE_IN_BYTES; @Configuration @JsonProperty(access = JsonProperty.Access.READ_WRITE) private Integer gcsManagedLedgerOffloadReadBufferSizeInBytes = DEFAULT_READ_BUFFER_SIZE_IN_BYTES; @@ -230,8 +250,7 @@ public static OffloadPoliciesImpl create(String driver, String region, String bu public static OffloadPoliciesImpl create(Properties properties) { OffloadPoliciesImpl data = new OffloadPoliciesImpl(); - Field[] fields = OffloadPoliciesImpl.class.getDeclaredFields(); - Arrays.stream(fields).forEach(f -> { + for (Field f : CONFIGURATION_FIELDS) { if (properties.containsKey(f.getName())) { try { f.setAccessible(true); @@ -242,7 +261,16 @@ public static OffloadPoliciesImpl create(Properties properties) { f.getName(), properties.get(f.getName())), e); } } - }); + } + + Map extraConfigurations = properties.entrySet().stream() + .filter(entry -> entry.getKey().toString().startsWith(EXTRA_CONFIG_PREFIX)) + .collect(Collectors.toMap( + entry -> entry.getKey().toString().replaceFirst(EXTRA_CONFIG_PREFIX, ""), + entry -> entry.getValue().toString())); + + data.getManagedLedgerExtraConfigurations().putAll(extraConfigurations); + data.compatibleWithBrokerConfigFile(properties); return data; } @@ -320,64 +348,21 @@ public boolean bucketValid() { public Properties toProperties() { Properties properties = new Properties(); - setProperty(properties, "managedLedgerOffloadedReadPriority", this.getManagedLedgerOffloadedReadPriority()); - setProperty(properties, "offloadersDirectory", this.getOffloadersDirectory()); - setProperty(properties, "managedLedgerOffloadDriver", this.getManagedLedgerOffloadDriver()); - setProperty(properties, "managedLedgerOffloadMaxThreads", - this.getManagedLedgerOffloadMaxThreads()); - setProperty(properties, "managedLedgerOffloadPrefetchRounds", - this.getManagedLedgerOffloadPrefetchRounds()); - setProperty(properties, "managedLedgerOffloadThresholdInBytes", - this.getManagedLedgerOffloadThresholdInBytes()); - setProperty(properties, "managedLedgerOffloadThresholdInSeconds", - this.getManagedLedgerOffloadThresholdInSeconds()); - setProperty(properties, "managedLedgerOffloadDeletionLagInMillis", - this.getManagedLedgerOffloadDeletionLagInMillis()); - - if (this.isS3Driver()) { - setProperty(properties, "s3ManagedLedgerOffloadRegion", - this.getS3ManagedLedgerOffloadRegion()); - setProperty(properties, "s3ManagedLedgerOffloadBucket", - this.getS3ManagedLedgerOffloadBucket()); - setProperty(properties, "s3ManagedLedgerOffloadServiceEndpoint", - this.getS3ManagedLedgerOffloadServiceEndpoint()); - setProperty(properties, "s3ManagedLedgerOffloadMaxBlockSizeInBytes", - this.getS3ManagedLedgerOffloadMaxBlockSizeInBytes()); - setProperty(properties, "s3ManagedLedgerOffloadCredentialId", - this.getS3ManagedLedgerOffloadCredentialId()); - setProperty(properties, "s3ManagedLedgerOffloadCredentialSecret", - this.getS3ManagedLedgerOffloadCredentialSecret()); - setProperty(properties, "s3ManagedLedgerOffloadRole", - this.getS3ManagedLedgerOffloadRole()); - setProperty(properties, "s3ManagedLedgerOffloadRoleSessionName", - this.getS3ManagedLedgerOffloadRoleSessionName()); - setProperty(properties, "s3ManagedLedgerOffloadReadBufferSizeInBytes", - this.getS3ManagedLedgerOffloadReadBufferSizeInBytes()); - } else if (this.isGcsDriver()) { - setProperty(properties, "gcsManagedLedgerOffloadRegion", - this.getGcsManagedLedgerOffloadRegion()); - setProperty(properties, "gcsManagedLedgerOffloadBucket", - this.getGcsManagedLedgerOffloadBucket()); - setProperty(properties, "gcsManagedLedgerOffloadMaxBlockSizeInBytes", - this.getGcsManagedLedgerOffloadMaxBlockSizeInBytes()); - setProperty(properties, "gcsManagedLedgerOffloadReadBufferSizeInBytes", - this.getGcsManagedLedgerOffloadReadBufferSizeInBytes()); - setProperty(properties, "gcsManagedLedgerOffloadServiceAccountKeyFile", - this.getGcsManagedLedgerOffloadServiceAccountKeyFile()); - } else if (this.isFileSystemDriver()) { - setProperty(properties, "fileSystemProfilePath", this.getFileSystemProfilePath()); - setProperty(properties, "fileSystemURI", this.getFileSystemURI()); - } - - setProperty(properties, "managedLedgerOffloadBucket", this.getManagedLedgerOffloadBucket()); - setProperty(properties, "managedLedgerOffloadRegion", this.getManagedLedgerOffloadRegion()); - setProperty(properties, "managedLedgerOffloadServiceEndpoint", - this.getManagedLedgerOffloadServiceEndpoint()); - setProperty(properties, "managedLedgerOffloadMaxBlockSizeInBytes", - this.getManagedLedgerOffloadMaxBlockSizeInBytes()); - setProperty(properties, "managedLedgerOffloadReadBufferSizeInBytes", - this.getManagedLedgerOffloadReadBufferSizeInBytes()); - + for (Field f : CONFIGURATION_FIELDS) { + try { + f.setAccessible(true); + if ("managedLedgerExtraConfigurations".equals(f.getName())) { + Map extraConfig = (Map) f.get(this); + extraConfig.forEach((key, value) -> { + setProperty(properties, EXTRA_CONFIG_PREFIX + key, value); + }); + } else { + setProperty(properties, f.getName(), f.get(this)); + } + } catch (Exception e) { + throw new IllegalArgumentException("An error occurred while processing the field: " + f.getName(), e); + } + } return properties; } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/PolicyName.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/PolicyName.java index 456d4b9270cd6..86ab545215e99 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/PolicyName.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/PolicyName.java @@ -51,5 +51,7 @@ public enum PolicyName { MAX_TOPICS, RESOURCEGROUP, ENTRY_FILTERS, - SHADOW_TOPIC + SHADOW_TOPIC, + DISPATCHER_PAUSE_ON_ACK_STATE_PERSISTENT, + ALLOW_CLUSTERS } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java index 4a76170d116a3..5403b84a4f7b8 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java @@ -62,6 +62,8 @@ public class TopicPolicies { private Integer maxUnackedMessagesOnSubscription; private Long delayedDeliveryTickTimeMillis; private Boolean delayedDeliveryEnabled; + private Long delayedDeliveryMaxDelayInMillis; + private Boolean dispatcherPauseOnAckStatePersistentEnabled; private OffloadPoliciesImpl offloadPolicies; private InactiveTopicPolicies inactiveTopicPolicies; private DispatchRateImpl dispatchRate; diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ConsumerStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ConsumerStatsImpl.java index ddae2e7135695..b4c5d21e6926e 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ConsumerStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ConsumerStatsImpl.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.common.policies.data.stats; -import com.fasterxml.jackson.annotation.JsonIgnore; import java.util.List; import java.util.Map; import java.util.Objects; @@ -31,6 +30,9 @@ */ @Data public class ConsumerStatsImpl implements ConsumerStats { + /** the app id. */ + public String appId; + /** Total rate of messages delivered to the consumer (msg/s). */ public double msgRateOut; @@ -75,26 +77,15 @@ public class ConsumerStatsImpl implements ConsumerStats { /** Flag to verify if consumer is blocked due to reaching threshold of unacked messages. */ public boolean blockedConsumerOnUnackedMsgs; - /** The read position of the cursor when the consumer joining. */ - public String readPositionWhenJoining; + /** The last sent position of the cursor when the consumer joining. */ + public String lastSentPositionWhenJoining; /** Address of this consumer. */ - @JsonIgnore - private int addressOffset = -1; - @JsonIgnore - private int addressLength; - + private String address; /** Timestamp of connection. */ - @JsonIgnore - private int connectedSinceOffset = -1; - @JsonIgnore - private int connectedSinceLength; - + private String connectedSince; /** Client library version. */ - @JsonIgnore - private int clientVersionOffset = -1; - @JsonIgnore - private int clientVersionLength; + private String clientVersion; // ignore this json field to skip from stats in future release. replaced with readable #getLastAckedTime(). @Deprecated @@ -111,13 +102,6 @@ public class ConsumerStatsImpl implements ConsumerStats { /** Metadata (key/value strings) associated with this consumer. */ public Map metadata; - /** - * In order to prevent multiple string object allocation under stats: create a string-buffer - * that stores data for all string place-holders. - */ - @JsonIgnore - private StringBuilder stringBuffer = new StringBuilder(); - public ConsumerStatsImpl add(ConsumerStatsImpl stats) { Objects.requireNonNull(stats); this.msgRateOut += stats.msgRateOut; @@ -129,56 +113,36 @@ public ConsumerStatsImpl add(ConsumerStatsImpl stats) { this.availablePermits += stats.availablePermits; this.unackedMessages += stats.unackedMessages; this.blockedConsumerOnUnackedMsgs = stats.blockedConsumerOnUnackedMsgs; - this.readPositionWhenJoining = stats.readPositionWhenJoining; + this.lastSentPositionWhenJoining = stats.lastSentPositionWhenJoining; return this; } public String getAddress() { - return addressOffset == -1 ? null : stringBuffer.substring(addressOffset, addressOffset + addressLength); + return address; } public void setAddress(String address) { - if (address == null) { - this.addressOffset = -1; - return; - } - this.addressOffset = this.stringBuffer.length(); - this.addressLength = address.length(); - this.stringBuffer.append(address); + this.address = address; } public String getConnectedSince() { - return connectedSinceOffset == -1 ? null - : stringBuffer.substring(connectedSinceOffset, connectedSinceOffset + connectedSinceLength); + return connectedSince; } public void setConnectedSince(String connectedSince) { - if (connectedSince == null) { - this.connectedSinceOffset = -1; - return; - } - this.connectedSinceOffset = this.stringBuffer.length(); - this.connectedSinceLength = connectedSince.length(); - this.stringBuffer.append(connectedSince); + this.connectedSince = connectedSince; } public String getClientVersion() { - return clientVersionOffset == -1 ? null - : stringBuffer.substring(clientVersionOffset, clientVersionOffset + clientVersionLength); + return clientVersion; } public void setClientVersion(String clientVersion) { - if (clientVersion == null) { - this.clientVersionOffset = -1; - return; - } - this.clientVersionOffset = this.stringBuffer.length(); - this.clientVersionLength = clientVersion.length(); - this.stringBuffer.append(clientVersion); + this.clientVersion = clientVersion; } - public String getReadPositionWhenJoining() { - return readPositionWhenJoining; + public String getLastSentPositionWhenJoining() { + return lastSentPositionWhenJoining; } public String getLastAckedTime() { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/NonPersistentPublisherStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/NonPersistentPublisherStatsImpl.java index adf3f92ae71fc..d62e9b8dbbeae 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/NonPersistentPublisherStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/NonPersistentPublisherStatsImpl.java @@ -18,10 +18,12 @@ */ package org.apache.pulsar.common.policies.data.stats; +import com.fasterxml.jackson.annotation.JsonIgnore; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.Objects; import lombok.Getter; import org.apache.pulsar.common.policies.data.NonPersistentPublisherStats; +import org.apache.pulsar.common.stats.Rate; /** * Non-persistent publisher statistics. @@ -35,10 +37,28 @@ public class NonPersistentPublisherStatsImpl extends PublisherStatsImpl implemen @Getter public double msgDropRate; + @JsonIgnore + private final Rate msgDrop = new Rate(); + public NonPersistentPublisherStatsImpl add(NonPersistentPublisherStatsImpl stats) { Objects.requireNonNull(stats); super.add(stats); this.msgDropRate += stats.msgDropRate; return this; } + + public void calculateRates() { + super.calculateRates(); + msgDrop.calculateRate(); + msgDropRate = msgDrop.getRate(); + } + + public void recordMsgDrop(long numMessages) { + msgDrop.recordEvent(numMessages); + } + + @JsonIgnore + public long getMsgDropCount() { + return msgDrop.getTotalCount(); + } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/NonPersistentReplicatorStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/NonPersistentReplicatorStatsImpl.java index 98f838a94493c..a09d03b21a03a 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/NonPersistentReplicatorStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/NonPersistentReplicatorStatsImpl.java @@ -18,27 +18,43 @@ */ package org.apache.pulsar.common.policies.data.stats; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Objects; -import lombok.Getter; +import java.util.concurrent.atomic.LongAdder; +import lombok.Data; +import lombok.EqualsAndHashCode; import org.apache.pulsar.common.policies.data.NonPersistentReplicatorStats; /** * Statistics for a non-persistent replicator. */ -@SuppressFBWarnings("EQ_DOESNT_OVERRIDE_EQUALS") +@Data +@EqualsAndHashCode(callSuper = true) public class NonPersistentReplicatorStatsImpl extends ReplicatorStatsImpl implements NonPersistentReplicatorStats { /** * for non-persistent topic: broker drops msg for replicator if replicator connection is not writable. **/ - @Getter public double msgDropRate; + @JsonIgnore + private final LongAdder msgDropCount = new LongAdder(); + public NonPersistentReplicatorStatsImpl add(NonPersistentReplicatorStatsImpl stats) { Objects.requireNonNull(stats); super.add(stats); this.msgDropRate += stats.msgDropRate; return this; } + + @Override + @JsonProperty + public long getMsgDropCount() { + return msgDropCount.sum(); + } + + public void incrementMsgDropCount() { + msgDropCount.increment(); + } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/NonPersistentTopicStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/NonPersistentTopicStatsImpl.java index fd643f0db7bf4..7710c27779b9a 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/NonPersistentTopicStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/NonPersistentTopicStatsImpl.java @@ -155,8 +155,9 @@ public NonPersistentTopicStatsImpl add(NonPersistentTopicStats ts) { Objects.requireNonNull(stats); super.add(stats); this.msgDropRate += stats.msgDropRate; - for (int index = 0; index < stats.getNonPersistentPublishers().size(); index++) { - NonPersistentPublisherStats s = stats.getNonPersistentPublishers().get(index); + List publisherStats = stats.getNonPersistentPublishers(); + for (int index = 0; index < publisherStats.size(); index++) { + NonPersistentPublisherStats s = publisherStats.get(index); if (s.isSupportsPartialProducer() && s.getProducerName() != null) { ((NonPersistentPublisherStatsImpl) this.nonPersistentPublishersMap .computeIfAbsent(s.getProducerName(), key -> { @@ -181,46 +182,24 @@ public NonPersistentTopicStatsImpl add(NonPersistentTopicStats ts) { } } - if (this.getNonPersistentSubscriptions().size() != stats.getNonPersistentSubscriptions().size()) { - for (String subscription : stats.getNonPersistentSubscriptions().keySet()) { - NonPersistentSubscriptionStatsImpl subscriptionStats = new NonPersistentSubscriptionStatsImpl(); - this.getNonPersistentSubscriptions().put(subscription, subscriptionStats - .add((NonPersistentSubscriptionStatsImpl) - stats.getNonPersistentSubscriptions().get(subscription))); - } - } else { - for (String subscription : stats.getNonPersistentSubscriptions().keySet()) { - if (this.getNonPersistentSubscriptions().get(subscription) != null) { - ((NonPersistentSubscriptionStatsImpl) this.getNonPersistentSubscriptions().get(subscription)) - .add((NonPersistentSubscriptionStatsImpl) - stats.getNonPersistentSubscriptions().get(subscription)); - } else { - NonPersistentSubscriptionStatsImpl subscriptionStats = new NonPersistentSubscriptionStatsImpl(); - this.getNonPersistentSubscriptions().put(subscription, subscriptionStats - .add((NonPersistentSubscriptionStatsImpl) - stats.getNonPersistentSubscriptions().get(subscription))); - } - } + for (Map.Entry entry : stats.getNonPersistentSubscriptions() + .entrySet()) { + NonPersistentSubscriptionStatsImpl subscriptionStats = + (NonPersistentSubscriptionStatsImpl) this.getNonPersistentSubscriptions() + .computeIfAbsent(entry.getKey(), k -> new NonPersistentSubscriptionStatsImpl()); + subscriptionStats.add( + (NonPersistentSubscriptionStatsImpl) entry.getValue()); } - if (this.getNonPersistentReplicators().size() != stats.getNonPersistentReplicators().size()) { - for (String repl : stats.getNonPersistentReplicators().keySet()) { - NonPersistentReplicatorStatsImpl replStats = new NonPersistentReplicatorStatsImpl(); - this.getNonPersistentReplicators().put(repl, replStats - .add((NonPersistentReplicatorStatsImpl) stats.getNonPersistentReplicators().get(repl))); - } - } else { - for (String repl : stats.getNonPersistentReplicators().keySet()) { - if (this.getNonPersistentReplicators().get(repl) != null) { - ((NonPersistentReplicatorStatsImpl) this.getNonPersistentReplicators().get(repl)) - .add((NonPersistentReplicatorStatsImpl) stats.getNonPersistentReplicators().get(repl)); - } else { - NonPersistentReplicatorStatsImpl replStats = new NonPersistentReplicatorStatsImpl(); - this.getNonPersistentReplicators().put(repl, replStats - .add((NonPersistentReplicatorStatsImpl) stats.getNonPersistentReplicators().get(repl))); - } - } + for (Map.Entry entry : stats.getNonPersistentReplicators().entrySet()) { + NonPersistentReplicatorStatsImpl replStats = (NonPersistentReplicatorStatsImpl) + this.getNonPersistentReplicators().computeIfAbsent(entry.getKey(), k -> { + NonPersistentReplicatorStatsImpl r = new NonPersistentReplicatorStatsImpl(); + return r; + }); + replStats.add((NonPersistentReplicatorStatsImpl) entry.getValue()); } + return this; } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/PublisherStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/PublisherStatsImpl.java index 41407a37e7ca0..3f9067eba0b25 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/PublisherStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/PublisherStatsImpl.java @@ -23,6 +23,7 @@ import lombok.Data; import org.apache.pulsar.client.api.ProducerAccessMode; import org.apache.pulsar.common.policies.data.PublisherStats; +import org.apache.pulsar.common.stats.Rate; /** * Statistics about a publisher. @@ -53,39 +54,22 @@ public class PublisherStatsImpl implements PublisherStats { public boolean supportsPartialProducer; /** Producer name. */ - @JsonIgnore - private int producerNameOffset = -1; - @JsonIgnore - private int producerNameLength; - + private String producerName; /** Address of this publisher. */ - @JsonIgnore - private int addressOffset = -1; - @JsonIgnore - private int addressLength; - + private String address; /** Timestamp of connection. */ - @JsonIgnore - private int connectedSinceOffset = -1; - @JsonIgnore - private int connectedSinceLength; - + private String connectedSince; /** Client library version. */ - @JsonIgnore - private int clientVersionOffset = -1; - @JsonIgnore - private int clientVersionLength; - - /** - * In order to prevent multiple string objects under stats: create a string-buffer that stores data for all string - * place-holders. - */ - @JsonIgnore - private StringBuilder stringBuffer = new StringBuilder(); + private String clientVersion; /** Metadata (key/value strings) associated with this publisher. */ public Map metadata; + @JsonIgnore + private final Rate msgIn = new Rate(); + @JsonIgnore + private final Rate msgChunkIn = new Rate(); + public PublisherStatsImpl add(PublisherStatsImpl stats) { if (stats == null) { throw new IllegalArgumentException("stats can't be null"); @@ -99,61 +83,67 @@ public PublisherStatsImpl add(PublisherStatsImpl stats) { } public String getProducerName() { - return producerNameOffset == -1 ? null - : stringBuffer.substring(producerNameOffset, producerNameOffset + producerNameLength); + return producerName; } public void setProducerName(String producerName) { - if (producerName == null) { - this.producerNameOffset = -1; - return; - } - this.producerNameOffset = this.stringBuffer.length(); - this.producerNameLength = producerName.length(); - this.stringBuffer.append(producerName); + this.producerName = producerName; } public String getAddress() { - return addressOffset == -1 ? null : stringBuffer.substring(addressOffset, addressOffset + addressLength); + return address; } public void setAddress(String address) { - if (address == null) { - this.addressOffset = -1; - return; - } - this.addressOffset = this.stringBuffer.length(); - this.addressLength = address.length(); - this.stringBuffer.append(address); + this.address = address; } public String getConnectedSince() { - return connectedSinceOffset == -1 ? null - : stringBuffer.substring(connectedSinceOffset, connectedSinceOffset + connectedSinceLength); + return connectedSince; } public void setConnectedSince(String connectedSince) { - if (connectedSince == null) { - this.connectedSinceOffset = -1; - return; - } - this.connectedSinceOffset = this.stringBuffer.length(); - this.connectedSinceLength = connectedSince.length(); - this.stringBuffer.append(connectedSince); + this.connectedSince = connectedSince; } public String getClientVersion() { - return clientVersionOffset == -1 ? null - : stringBuffer.substring(clientVersionOffset, clientVersionOffset + clientVersionLength); + return clientVersion; } public void setClientVersion(String clientVersion) { - if (clientVersion == null) { - this.clientVersionOffset = -1; - return; - } - this.clientVersionOffset = this.stringBuffer.length(); - this.clientVersionLength = clientVersion.length(); - this.stringBuffer.append(clientVersion); + this.clientVersion = clientVersion; + } + + public void calculateRates() { + msgIn.calculateRate(); + msgChunkIn.calculateRate(); + + msgRateIn = msgIn.getRate(); + msgThroughputIn = msgIn.getValueRate(); + averageMsgSize = msgIn.getAverageValue(); + chunkedMessageRate = msgChunkIn.getRate(); + } + + public void recordMsgIn(long messageCount, long byteCount) { + msgIn.recordMultipleEvents(messageCount, byteCount); + } + + @JsonIgnore + public long getMsgInCounter() { + return msgIn.getTotalCount(); + } + + @JsonIgnore + public long getBytesInCounter() { + return msgIn.getTotalValue(); + } + + public void recordChunkedMsgIn() { + msgChunkIn.recordEvent(); + } + + @JsonIgnore + public long getChunkedMsgInCounter() { + return msgChunkIn.getTotalCount(); } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java index 6933f5cc7ed76..c19169cbee57f 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java @@ -18,7 +18,10 @@ */ package org.apache.pulsar.common.policies.data.stats; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Objects; +import java.util.concurrent.atomic.LongAdder; import lombok.Data; import org.apache.pulsar.common.policies.data.ReplicatorStats; @@ -31,15 +34,27 @@ public class ReplicatorStatsImpl implements ReplicatorStats { /** Total rate of messages received from the remote cluster (msg/s). */ public double msgRateIn; + @JsonIgnore + private final LongAdder msgInCount = new LongAdder(); + /** Total throughput received from the remote cluster (bytes/s). */ public double msgThroughputIn; + @JsonIgnore + private final LongAdder bytesInCount = new LongAdder(); + /** Total rate of messages delivered to the replication-subscriber (msg/s). */ public double msgRateOut; + @JsonIgnore + private final LongAdder msgOutCount = new LongAdder(); + /** Total throughput delivered to the replication-subscriber (bytes/s). */ public double msgThroughputOut; + @JsonIgnore + private final LongAdder bytesOutCount = new LongAdder(); + /** Total rate of messages expired (msg/s). */ public double msgRateExpired; @@ -72,10 +87,51 @@ public ReplicatorStatsImpl add(ReplicatorStatsImpl stats) { this.msgThroughputOut += stats.msgThroughputOut; this.msgRateExpired += stats.msgRateExpired; this.replicationBacklog += stats.replicationBacklog; - if (this.connected) { - this.connected &= stats.connected; - } + this.connected &= stats.connected; this.replicationDelayInSeconds = Math.max(this.replicationDelayInSeconds, stats.replicationDelayInSeconds); return this; } + + @Override + @JsonProperty + public long getMsgInCount() { + return msgInCount.sum(); + } + + @Override + @JsonProperty + public long getBytesInCount() { + return bytesInCount.sum(); + } + + public void incrementPublishCount(int numOfMessages, long msgSizeInBytes) { + msgInCount.add(numOfMessages); + bytesInCount.add(msgSizeInBytes); + } + + @Override + @JsonProperty + public long getMsgOutCount() { + return msgOutCount.sum(); + } + + public void incrementMsgOutCounter() { + msgOutCount.increment(); + } + + @Override + @JsonProperty + public long getBytesOutCount() { + return bytesOutCount.sum(); + } + + public void incrementBytesOutCounter(long bytes) { + bytesOutCount.add(bytes); + } + + @Override + @JsonProperty + public long getMsgExpiredCount() { + return 0; + } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java index ea7639a8cd2c6..977ed28e86814 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.common.policies.data.stats; +import com.fasterxml.jackson.annotation.JsonIgnore; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; @@ -53,7 +54,7 @@ public class SubscriptionStatsImpl implements SubscriptionStats { public double messageAckRate; /** Chunked message dispatch rate. */ - public int chunkedMessageRate; + public double chunkedMessageRate; /** Number of entries in the subscription backlog. */ public long msgBacklog; @@ -73,6 +74,9 @@ public class SubscriptionStatsImpl implements SubscriptionStats { /** Number of delayed messages currently being tracked. */ public long msgDelayed; + /** Number of messages registered for replay. */ + public long msgInReplay; + /** * Number of unacknowledged messages for the subscription, where an unacknowledged message is one that has been * sent to a consumer but not yet acknowledged. Calculated by summing all {@link ConsumerStatsImpl#unackedMessages} @@ -125,15 +129,22 @@ public class SubscriptionStatsImpl implements SubscriptionStats { /** This is for Key_Shared subscription to get the recentJoinedConsumers in the Key_Shared subscription. */ public Map consumersAfterMarkDeletePosition; + /** The last sent position of the cursor. This is for Key_Shared subscription. */ + public String lastSentPosition; + + /** Set of individually sent ranges. This is for Key_Shared subscription. */ + public String individuallySentPositions; + /** The number of non-contiguous deleted messages ranges. */ public int nonContiguousDeletedMessagesRanges; /** The serialized size of non-contiguous deleted messages ranges. */ public int nonContiguousDeletedMessagesRangesSerializedSize; - /** The size of InMemoryDelayedDeliveryTracer memory usage. */ + /** The size of DelayedDeliveryTracer memory usage. */ public long delayedMessageIndexSizeInBytes; + @JsonIgnore public Map bucketDelayedIndexStats; /** SubscriptionProperties (key/value strings) associated with this subscribe. */ @@ -165,6 +176,8 @@ public void reset() { msgBacklog = 0; backlogSize = 0; msgBacklogNoDelayed = 0; + msgDelayed = 0; + msgInReplay = 0; unackedMessages = 0; type = null; msgRateExpired = 0; @@ -175,6 +188,7 @@ public void reset() { consumersAfterMarkDeletePosition.clear(); nonContiguousDeletedMessagesRanges = 0; nonContiguousDeletedMessagesRangesSerializedSize = 0; + earliestMsgPublishTimeInBacklog = 0L; delayedMessageIndexSizeInBytes = 0; subscriptionProperties.clear(); filterProcessedMsgCount = 0; @@ -199,6 +213,7 @@ public SubscriptionStatsImpl add(SubscriptionStatsImpl stats) { this.backlogSize += stats.backlogSize; this.msgBacklogNoDelayed += stats.msgBacklogNoDelayed; this.msgDelayed += stats.msgDelayed; + this.msgInReplay += stats.msgInReplay; this.unackedMessages += stats.unackedMessages; this.type = stats.type; this.msgRateExpired += stats.msgRateExpired; @@ -219,6 +234,17 @@ public SubscriptionStatsImpl add(SubscriptionStatsImpl stats) { this.consumersAfterMarkDeletePosition.putAll(stats.consumersAfterMarkDeletePosition); this.nonContiguousDeletedMessagesRanges += stats.nonContiguousDeletedMessagesRanges; this.nonContiguousDeletedMessagesRangesSerializedSize += stats.nonContiguousDeletedMessagesRangesSerializedSize; + if (this.earliestMsgPublishTimeInBacklog != 0 && stats.earliestMsgPublishTimeInBacklog != 0) { + this.earliestMsgPublishTimeInBacklog = Math.min( + this.earliestMsgPublishTimeInBacklog, + stats.earliestMsgPublishTimeInBacklog + ); + } else { + this.earliestMsgPublishTimeInBacklog = Math.max( + this.earliestMsgPublishTimeInBacklog, + stats.earliestMsgPublishTimeInBacklog + ); + } this.delayedMessageIndexSizeInBytes += stats.delayedMessageIndexSizeInBytes; this.subscriptionProperties.putAll(stats.subscriptionProperties); this.filterProcessedMsgCount += stats.filterProcessedMsgCount; diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java index c9c4739b904f6..022fffd3a7e59 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java @@ -66,12 +66,18 @@ public class TopicStatsImpl implements TopicStats { /** Total messages published to the topic (msg). */ public long msgInCounter; + /** Total bytes published to the system topic (bytes). */ + public long systemTopicBytesInCounter; + /** Total bytes delivered to consumer (bytes). */ public long bytesOutCounter; /** Total messages delivered to consumer (msg). */ public long msgOutCounter; + /** Total bytes delivered to internal cursors. */ + public long bytesOutInternalCounter; + /** Average size of published messages (bytes). */ public double averageMsgSize; @@ -84,6 +90,31 @@ public class TopicStatsImpl implements TopicStats { /** Get estimated total unconsumed or backlog size in bytes. */ public long backlogSize; + /** the size in bytes of the topic backlog quota. */ + public long backlogQuotaLimitSize; + + /** the topic backlog age quota, in seconds. */ + public long backlogQuotaLimitTime; + + /** + * Age of oldest unacknowledged message, as recorded in last backlog quota check interval. + *

      + * The age of the oldest unacknowledged (i.e. backlog) message, measured by the time elapsed from its published + * time, in seconds. This value is recorded every backlog quota check interval, hence it represents the value + * seen in the last check. + *

      + */ + public long oldestBacklogMessageAgeSeconds; + + /** + * The subscription name containing oldest unacknowledged message as recorded in last backlog quota check. + *

      + * The name of the subscription containing the oldest unacknowledged message. This value is recorded every backlog + * quota check interval, hence it represents the value seen in the last check. + *

      + */ + public String oldestBacklogMessageSubscriptionName; + /** The number of times the publishing rate limit was triggered. */ public long publishRateLimitedTimes; @@ -136,10 +167,11 @@ public class TopicStatsImpl implements TopicStats { /** The serialized size of non-contiguous deleted messages ranges. */ public int nonContiguousDeletedMessagesRangesSerializedSize; - /** The size of InMemoryDelayedDeliveryTracer memory usage. */ + /** The size of DelayedDeliveryTracer memory usage. */ public long delayedMessageIndexSizeInBytes; /** Map of bucket delayed index statistics. */ + @JsonIgnore public Map bucketDelayedIndexStats; /** The compaction stats. */ @@ -215,10 +247,15 @@ public void reset() { this.lastOffloadFailureTimeStamp = 0; this.lastOffloadSuccessTimeStamp = 0; this.publishRateLimitedTimes = 0L; + this.earliestMsgPublishTimeInBacklogs = 0L; this.delayedMessageIndexSizeInBytes = 0; this.compaction.reset(); this.ownerBroker = null; this.bucketDelayedIndexStats.clear(); + this.backlogQuotaLimitSize = 0; + this.backlogQuotaLimitTime = 0; + this.oldestBacklogMessageAgeSeconds = -1; + this.oldestBacklogMessageSubscriptionName = null; } // if the stats are added for the 1st time, we will need to make a copy of these stats and add it to the current @@ -248,6 +285,12 @@ public TopicStatsImpl add(TopicStats ts) { this.ongoingTxnCount = stats.ongoingTxnCount; this.abortedTxnCount = stats.abortedTxnCount; this.committedTxnCount = stats.committedTxnCount; + this.backlogQuotaLimitTime = stats.backlogQuotaLimitTime; + this.backlogQuotaLimitSize = stats.backlogQuotaLimitSize; + if (stats.oldestBacklogMessageAgeSeconds > this.oldestBacklogMessageAgeSeconds) { + this.oldestBacklogMessageAgeSeconds = stats.oldestBacklogMessageAgeSeconds; + this.oldestBacklogMessageSubscriptionName = stats.oldestBacklogMessageSubscriptionName; + } stats.bucketDelayedIndexStats.forEach((k, v) -> { TopicMetricBean topicMetricBean = @@ -257,8 +300,9 @@ public TopicStatsImpl add(TopicStats ts) { topicMetricBean.value += v.value; }); - for (int index = 0; index < stats.getPublishers().size(); index++) { - PublisherStats s = stats.getPublishers().get(index); + List publisherStats = stats.getPublishers(); + for (int index = 0; index < publisherStats.size(); index++) { + PublisherStats s = publisherStats.get(index); if (s.isSupportsPartialProducer() && s.getProducerName() != null) { this.publishersMap.computeIfAbsent(s.getProducerName(), key -> { final PublisherStatsImpl newStats = new PublisherStatsImpl(); @@ -282,37 +326,32 @@ public TopicStatsImpl add(TopicStats ts) { } } - if (this.subscriptions.size() != stats.subscriptions.size()) { - for (String subscription : stats.subscriptions.keySet()) { - SubscriptionStatsImpl subscriptionStats = new SubscriptionStatsImpl(); - this.subscriptions.put(subscription, subscriptionStats.add(stats.subscriptions.get(subscription))); - } - } else { - for (String subscription : stats.subscriptions.keySet()) { - if (this.subscriptions.get(subscription) != null) { - this.subscriptions.get(subscription).add(stats.subscriptions.get(subscription)); - } else { - SubscriptionStatsImpl subscriptionStats = new SubscriptionStatsImpl(); - this.subscriptions.put(subscription, subscriptionStats.add(stats.subscriptions.get(subscription))); - } - } + for (Map.Entry entry : stats.subscriptions.entrySet()) { + SubscriptionStatsImpl subscriptionStats = + this.subscriptions.computeIfAbsent(entry.getKey(), k -> new SubscriptionStatsImpl()); + subscriptionStats.add(entry.getValue()); + } + + for (Map.Entry entry : stats.replication.entrySet()) { + ReplicatorStatsImpl replStats = + this.replication.computeIfAbsent(entry.getKey(), k -> { + ReplicatorStatsImpl r = new ReplicatorStatsImpl(); + r.setConnected(true); + return r; + }); + replStats.add(entry.getValue()); } - if (this.replication.size() != stats.replication.size()) { - for (String repl : stats.replication.keySet()) { - ReplicatorStatsImpl replStats = new ReplicatorStatsImpl(); - replStats.setConnected(true); - this.replication.put(repl, replStats.add(stats.replication.get(repl))); - } + + if (earliestMsgPublishTimeInBacklogs != 0 && ((TopicStatsImpl) ts).earliestMsgPublishTimeInBacklogs != 0) { + earliestMsgPublishTimeInBacklogs = Math.min( + earliestMsgPublishTimeInBacklogs, + ((TopicStatsImpl) ts).earliestMsgPublishTimeInBacklogs + ); } else { - for (String repl : stats.replication.keySet()) { - if (this.replication.get(repl) != null) { - this.replication.get(repl).add(stats.replication.get(repl)); - } else { - ReplicatorStatsImpl replStats = new ReplicatorStatsImpl(); - replStats.setConnected(true); - this.replication.put(repl, replStats.add(stats.replication.get(repl))); - } - } + earliestMsgPublishTimeInBacklogs = Math.max( + earliestMsgPublishTimeInBacklogs, + ((TopicStatsImpl) ts).earliestMsgPublishTimeInBacklogs + ); } return this; } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/impl/NamespaceIsolationPolicyImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/impl/NamespaceIsolationPolicyImpl.java index af3663869fa02..440282f29cb36 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/impl/NamespaceIsolationPolicyImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/impl/NamespaceIsolationPolicyImpl.java @@ -29,6 +29,7 @@ import org.apache.pulsar.common.policies.NamespaceIsolationPolicy; import org.apache.pulsar.common.policies.data.BrokerStatus; import org.apache.pulsar.common.policies.data.NamespaceIsolationData; +import org.apache.pulsar.common.policies.data.NamespaceIsolationPolicyUnloadScope; /** * Implementation of the namespace isolation policy. @@ -39,6 +40,7 @@ public class NamespaceIsolationPolicyImpl implements NamespaceIsolationPolicy { private List primary; private List secondary; private AutoFailoverPolicy autoFailoverPolicy; + private NamespaceIsolationPolicyUnloadScope unloadScope; private boolean matchNamespaces(String fqnn) { for (String nsRegex : namespaces) { @@ -64,6 +66,7 @@ public NamespaceIsolationPolicyImpl(NamespaceIsolationData policyData) { this.primary = policyData.getPrimary(); this.secondary = policyData.getSecondary(); this.autoFailoverPolicy = AutoFailoverPolicyFactory.create(policyData.getAutoFailoverPolicy()); + this.unloadScope = policyData.getUnloadScope(); } @Override @@ -76,6 +79,11 @@ public List getSecondaryBrokers() { return this.secondary; } + @Override + public NamespaceIsolationPolicyUnloadScope getUnloadScope() { + return this.unloadScope; + } + @Override public List findPrimaryBrokers(List availableBrokers, NamespaceName namespace) { if (!this.matchNamespaces(namespace.toString())) { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/ByteBufPair.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/ByteBufPair.java index cfd89d3bb28ab..6c4f42fcf88b9 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/ByteBufPair.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/ByteBufPair.java @@ -107,9 +107,39 @@ public ReferenceCounted touch(Object hint) { return this; } + /** + * Encoder that writes a {@link ByteBufPair} to the socket. + * Use {@link #getEncoder(boolean)} to get the appropriate encoder instead of referencing this. + */ + @Deprecated public static final Encoder ENCODER = new Encoder(); + + private static final boolean COPY_ENCODER_REQUIRED_FOR_TLS; + static { + boolean copyEncoderRequiredForTls = false; + try { + // io.netty.handler.ssl.SslHandlerCoalescingBufferQueue is only available in netty 4.1.111 and later + // when the class is available, there's no need to use the CopyingEncoder when TLS is enabled + ByteBuf.class.getClassLoader().loadClass("io.netty.handler.ssl.SslHandlerCoalescingBufferQueue"); + } catch (ClassNotFoundException e) { + copyEncoderRequiredForTls = true; + } + COPY_ENCODER_REQUIRED_FOR_TLS = copyEncoderRequiredForTls; + } + + /** + * Encoder that makes a copy of the ByteBufs before writing them to the socket. + * This is needed with Netty <4.1.111.Final when TLS is enabled, because the SslHandler will modify the input + * ByteBufs. + * Use {@link #getEncoder(boolean)} to get the appropriate encoder instead of referencing this. + */ + @Deprecated public static final CopyingEncoder COPYING_ENCODER = new CopyingEncoder(); + public static ChannelOutboundHandlerAdapter getEncoder(boolean tlsEnabled) { + return tlsEnabled && COPY_ENCODER_REQUIRED_FOR_TLS ? COPYING_ENCODER : ENCODER; + } + @Sharable @SuppressWarnings("checkstyle:JavadocType") public static class Encoder extends ChannelOutboundHandlerAdapter { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java index cf0cd820a6d10..15b5676094ec1 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java @@ -60,6 +60,8 @@ import org.apache.pulsar.common.api.proto.CommandAddSubscriptionToTxn; import org.apache.pulsar.common.api.proto.CommandAddSubscriptionToTxnResponse; import org.apache.pulsar.common.api.proto.CommandAuthChallenge; +import org.apache.pulsar.common.api.proto.CommandCloseConsumer; +import org.apache.pulsar.common.api.proto.CommandCloseProducer; import org.apache.pulsar.common.api.proto.CommandConnect; import org.apache.pulsar.common.api.proto.CommandConnected; import org.apache.pulsar.common.api.proto.CommandEndTxnOnPartitionResponse; @@ -188,6 +190,7 @@ private static void setFeatureFlags(FeatureFlags flags) { flags.setSupportsAuthRefresh(true); flags.setSupportsBrokerEntryMetadata(true); flags.setSupportsPartialProducer(true); + flags.setSupportsGetPartitionedMetadataWithoutAutoCreation(true); } public static ByteBuf newConnect(String authMethodName, String authData, int protocolVersion, String libVersion, @@ -298,6 +301,7 @@ public static BaseCommand newConnectedCommand(int clientProtocolVersion, int max connected.setProtocolVersion(versionToAdvertise); connected.setFeatureFlags().setSupportsTopicWatchers(supportsTopicWatchers); + connected.setFeatureFlags().setSupportsGetPartitionedMetadataWithoutAutoCreation(true); return cmd; } @@ -696,11 +700,12 @@ private static KeySharedMode convertKeySharedMode(org.apache.pulsar.client.api.K } } - public static ByteBuf newUnsubscribe(long consumerId, long requestId) { + public static ByteBuf newUnsubscribe(long consumerId, long requestId, boolean force) { BaseCommand cmd = localCmd(Type.UNSUBSCRIBE); cmd.setUnsubscribe() .setConsumerId(consumerId) - .setRequestId(requestId); + .setRequestId(requestId) + .setForce(force); return serializeWithSize(cmd); } @@ -736,11 +741,21 @@ public static ByteBuf newSeek(long consumerId, long requestId, long timestamp) { return serializeWithSize(cmd); } - public static ByteBuf newCloseConsumer(long consumerId, long requestId) { + public static ByteBuf newCloseConsumer( + long consumerId, long requestId, String assignedBrokerUrl, String assignedBrokerUrlTls) { BaseCommand cmd = localCmd(Type.CLOSE_CONSUMER); - cmd.setCloseConsumer() + CommandCloseConsumer commandCloseConsumer = cmd.setCloseConsumer() .setConsumerId(consumerId) .setRequestId(requestId); + + if (assignedBrokerUrl != null) { + commandCloseConsumer.setAssignedBrokerServiceUrl(assignedBrokerUrl); + } + + if (assignedBrokerUrlTls != null) { + commandCloseConsumer.setAssignedBrokerServiceUrlTls(assignedBrokerUrlTls); + } + return serializeWithSize(cmd); } @@ -761,11 +776,28 @@ public static ByteBuf newTopicMigrated(ResourceType type, long resourceId, Strin return serializeWithSize(cmd); } - public static ByteBuf newCloseProducer(long producerId, long requestId) { + public static ByteBuf newCloseProducer( + long producerId, long requestId) { + return newCloseProducer(producerId, requestId, null, null); + } + + public static ByteBuf newCloseProducer( + long producerId, long requestId, String assignedBrokerUrl, String assignedBrokerUrlTls) { BaseCommand cmd = localCmd(Type.CLOSE_PRODUCER); - cmd.setCloseProducer() - .setProducerId(producerId) - .setRequestId(requestId); + CommandCloseProducer commandCloseProducer = cmd.setCloseProducer() + .setProducerId(producerId) + .setRequestId(requestId); + + if (assignedBrokerUrl != null) { + commandCloseProducer + .setAssignedBrokerServiceUrl(assignedBrokerUrl); + } + + if (assignedBrokerUrlTls != null){ + commandCloseProducer + .setAssignedBrokerServiceUrlTls(assignedBrokerUrlTls); + } + return serializeWithSize(cmd); } @@ -880,11 +912,13 @@ public static ByteBuf newPartitionMetadataResponse(ServerError error, String err return serializeWithSize(newPartitionMetadataResponseCommand(error, errorMsg, requestId)); } - public static ByteBuf newPartitionMetadataRequest(String topic, long requestId) { + public static ByteBuf newPartitionMetadataRequest(String topic, long requestId, + boolean metadataAutoCreationEnabled) { BaseCommand cmd = localCmd(Type.PARTITIONED_METADATA); cmd.setPartitionMetadata() .setTopic(topic) - .setRequestId(requestId); + .setRequestId(requestId) + .setMetadataAutoCreationEnabled(metadataAutoCreationEnabled); return serializeWithSize(cmd); } @@ -902,10 +936,11 @@ public static ByteBuf newPartitionMetadataResponse(int partitions, long requestI } public static ByteBuf newLookup(String topic, boolean authoritative, long requestId) { - return newLookup(topic, null, authoritative, requestId); + return newLookup(topic, null, authoritative, requestId, null); } - public static ByteBuf newLookup(String topic, String listenerName, boolean authoritative, long requestId) { + public static ByteBuf newLookup(String topic, String listenerName, boolean authoritative, long requestId, + Map properties) { BaseCommand cmd = localCmd(Type.LOOKUP); CommandLookupTopic lookup = cmd.setLookupTopic() .setTopic(topic) @@ -914,6 +949,9 @@ public static ByteBuf newLookup(String topic, String listenerName, boolean autho if (StringUtils.isNotBlank(listenerName)) { lookup.setAdvertisedListenerName(listenerName); } + if (properties != null) { + properties.forEach((key, value) -> lookup.addProperty().setKey(key).setValue(value)); + } return serializeWithSize(cmd); } @@ -1555,6 +1593,9 @@ public static BaseCommand newWatchTopicList( return cmd; } + /*** + * @param topics topic names which are matching, the topic name contains the partition suffix. + */ public static BaseCommand newWatchTopicListSuccess(long requestId, long watcherId, String topicsHash, List topics) { BaseCommand cmd = localCmd(Type.WATCH_TOPIC_LIST_SUCCESS); @@ -1570,6 +1611,10 @@ public static BaseCommand newWatchTopicListSuccess(long requestId, long watcherI return cmd; } + /** + * @param deletedTopics topic names deleted(contains the partition suffix). + * @param newTopics topics names added(contains the partition suffix). + */ public static BaseCommand newWatchTopicUpdate(long watcherId, List newTopics, List deletedTopics, String topicsHash) { BaseCommand cmd = localCmd(Type.WATCH_TOPIC_UPDATE); @@ -1668,6 +1713,7 @@ public static ByteBuf addBrokerEntryMetadata(ByteBuf headerAndPayload, // | 2 bytes | 4 bytes | BROKER_ENTRY_METADATA_SIZE bytes | BrokerEntryMetadata brokerEntryMetadata = BROKER_ENTRY_METADATA.get(); + brokerEntryMetadata.clear(); for (BrokerEntryMetadataInterceptor interceptor : brokerInterceptors) { interceptor.intercept(brokerEntryMetadata); if (numberOfMessages >= 0) { @@ -1937,6 +1983,9 @@ public static byte[] peekStickyKey(ByteBuf metadataAndPayload, String topic, Str return Base64.getDecoder().decode(metadata.getPartitionKey()); } return metadata.getPartitionKey().getBytes(StandardCharsets.UTF_8); + } else if (metadata.hasProducerName() && metadata.hasSequenceId()) { + String fallbackKey = metadata.getProducerName() + "-" + metadata.getSequenceId(); + return fallbackKey.getBytes(StandardCharsets.UTF_8); } } catch (Throwable t) { log.error("[{}] [{}] Failed to peek sticky key from the message metadata", topic, subscription, t); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Markers.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Markers.java index 50b036f99ee4b..2291aee781f60 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Markers.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Markers.java @@ -19,13 +19,13 @@ package org.apache.pulsar.common.protocol; import io.netty.buffer.ByteBuf; -import io.netty.buffer.PooledByteBufAllocator; import io.netty.util.concurrent.FastThreadLocal; import java.io.IOException; import java.util.Map; import java.util.Optional; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; +import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.api.proto.MarkerType; import org.apache.pulsar.common.api.proto.MarkersMessageIdData; import org.apache.pulsar.common.api.proto.MessageMetadata; @@ -109,7 +109,7 @@ public static ByteBuf newReplicatedSubscriptionsSnapshotRequest(String snapshotI .clear() .setSnapshotId(snapshotId) .setSourceCluster(sourceCluster); - ByteBuf payload = PooledByteBufAllocator.DEFAULT.buffer(req.getSerializedSize()); + ByteBuf payload = PulsarByteBufAllocator.DEFAULT.buffer(req.getSerializedSize()); try { req.writeTo(payload); @@ -138,7 +138,7 @@ public static ByteBuf newReplicatedSubscriptionsSnapshotResponse(String snapshot .setLedgerId(ledgerId) .setEntryId(entryId); - ByteBuf payload = PooledByteBufAllocator.DEFAULT.buffer(response.getSerializedSize()); + ByteBuf payload = PulsarByteBufAllocator.DEFAULT.buffer(response.getSerializedSize()); try { response.writeTo(payload); return newMessage(MarkerType.REPLICATED_SUBSCRIPTION_SNAPSHOT_RESPONSE, Optional.of(replyToCluster), @@ -172,7 +172,7 @@ public static ByteBuf newReplicatedSubscriptionsSnapshot(String snapshotId, Stri }); int size = snapshot.getSerializedSize(); - ByteBuf payload = PooledByteBufAllocator.DEFAULT.buffer(size); + ByteBuf payload = PulsarByteBufAllocator.DEFAULT.buffer(size); try { snapshot.writeTo(payload); return newMessage(MarkerType.REPLICATED_SUBSCRIPTION_SNAPSHOT, Optional.of(sourceCluster), payload); @@ -201,7 +201,7 @@ public static ByteBuf newReplicatedSubscriptionsUpdate(String subscriptionName, .setMessageId().copyFrom(msgId); }); - ByteBuf payload = PooledByteBufAllocator.DEFAULT.buffer(update.getSerializedSize()); + ByteBuf payload = PulsarByteBufAllocator.DEFAULT.buffer(update.getSerializedSize()); try { update.writeTo(payload); @@ -258,7 +258,7 @@ private static ByteBuf newTxnMarker(MarkerType markerType, long sequenceId, long .setTxnidMostBits(txnMostBits) .setTxnidLeastBits(txnLeastBits); - ByteBuf payload = PooledByteBufAllocator.DEFAULT.buffer(0); + ByteBuf payload = PulsarByteBufAllocator.DEFAULT.buffer(0); try { return Commands.serializeMetadataAndPayload(ChecksumType.Crc32c, diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/OptionalProxyProtocolDecoder.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/OptionalProxyProtocolDecoder.java index 2f0a7884dde35..b4e15f8cd1d75 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/OptionalProxyProtocolDecoder.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/OptionalProxyProtocolDecoder.java @@ -19,36 +19,63 @@ package org.apache.pulsar.common.protocol; import io.netty.buffer.ByteBuf; +import io.netty.buffer.CompositeByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.handler.codec.ProtocolDetectionResult; import io.netty.handler.codec.ProtocolDetectionState; import io.netty.handler.codec.haproxy.HAProxyMessageDecoder; import io.netty.handler.codec.haproxy.HAProxyProtocolVersion; +import lombok.extern.slf4j.Slf4j; /** * Decoder that added whether a new connection is prefixed with the ProxyProtocol. * More about the ProxyProtocol see: http://www.haproxy.org/download/1.8/doc/proxy-protocol.txt. */ +@Slf4j public class OptionalProxyProtocolDecoder extends ChannelInboundHandlerAdapter { public static final String NAME = "optional-proxy-protocol-decoder"; + public static final int MIN_BYTES_SIZE_TO_DETECT_PROTOCOL = 12; + + private CompositeByteBuf cumulatedByteBuf; + @Override public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { if (msg instanceof ByteBuf) { - ProtocolDetectionResult result = - HAProxyMessageDecoder.detectProtocol((ByteBuf) msg); - // should accumulate data if need more data to detect the protocol + // Combine cumulated buffers. + ByteBuf buf = (ByteBuf) msg; + if (cumulatedByteBuf != null) { + buf = cumulatedByteBuf.addComponent(true, buf); + } + + ProtocolDetectionResult result = HAProxyMessageDecoder.detectProtocol(buf); if (result.state() == ProtocolDetectionState.NEEDS_MORE_DATA) { + // Accumulate data if need more data to detect the protocol. + if (cumulatedByteBuf == null) { + cumulatedByteBuf = new CompositeByteBuf(ctx.alloc(), false, MIN_BYTES_SIZE_TO_DETECT_PROTOCOL, buf); + } return; } + cumulatedByteBuf = null; if (result.state() == ProtocolDetectionState.DETECTED) { ctx.pipeline().addAfter(NAME, null, new HAProxyMessageDecoder()); - ctx.pipeline().remove(this); } + ctx.pipeline().remove(this); + super.channelRead(ctx, buf); + } else { + super.channelRead(ctx, msg); + } + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + super.channelInactive(ctx); + if (cumulatedByteBuf != null) { + log.info("Release cumulated byte buffer when channel inactive."); + cumulatedByteBuf = null; } - super.channelRead(ctx, msg); } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarDecoder.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarDecoder.java index 496652fed0b6b..c05b1d796dfdd 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarDecoder.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarDecoder.java @@ -122,7 +122,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception cmd.parseFrom(buffer, cmdSize); if (log.isDebugEnabled()) { - log.debug("[{}] Received cmd {}", ctx.channel().remoteAddress(), cmd.getType()); + log.debug("[{}] Received cmd {}", ctx.channel(), cmd.getType()); } messageReceived(); @@ -291,6 +291,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception case REDELIVER_UNACKNOWLEDGED_MESSAGES: checkArgument(cmd.hasRedeliverUnacknowledgedMessages()); + safeInterceptCommand(cmd); handleRedeliverUnacknowledged(cmd.getRedeliverUnacknowledgedMessages()); break; diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java index 51cd61afd6362..d5c741be01e22 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/PulsarHandler.java @@ -67,7 +67,7 @@ public void channelActive(ChannelHandlerContext ctx) throws Exception { this.ctx = ctx; if (log.isDebugEnabled()) { - log.debug("[{}] Scheduling keep-alive task every {} s", ctx.channel(), keepAliveIntervalSeconds); + log.debug("[{}] Scheduling keep-alive task every {} s", this.toString(), keepAliveIntervalSeconds); } if (keepAliveIntervalSeconds > 0) { this.keepAliveTask = ctx.executor() @@ -85,13 +85,13 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { protected final void handlePing(CommandPing ping) { // Immediately reply success to ping requests if (log.isDebugEnabled()) { - log.debug("[{}] Replying back to ping message", ctx.channel()); + log.debug("[{}] Replying back to ping message", this.toString()); } ctx.writeAndFlush(Commands.newPong()) .addListener(future -> { if (!future.isSuccess()) { log.warn("[{}] Forcing connection to close since cannot send a pong message.", - ctx.channel(), future.cause()); + toString(), future.cause()); ctx.close(); } }); @@ -107,24 +107,24 @@ private void handleKeepAliveTimeout() { } if (!isHandshakeCompleted()) { - log.warn("[{}] Pulsar Handshake was not completed within timeout, closing connection", ctx.channel()); + log.warn("[{}] Pulsar Handshake was not completed within timeout, closing connection", this.toString()); ctx.close(); } else if (waitingForPingResponse && ctx.channel().config().isAutoRead()) { // We were waiting for a response and another keep-alive just completed. // If auto-read was disabled, it means we stopped reading from the connection, so we might receive the Ping // response later and thus not enforce the strict timeout here. - log.warn("[{}] Forcing connection to close after keep-alive timeout", ctx.channel()); + log.warn("[{}] Forcing connection to close after keep-alive timeout", this.toString()); ctx.close(); } else if (getRemoteEndpointProtocolVersion() >= ProtocolVersion.v1.getValue()) { // Send keep alive probe to peer only if it supports the ping/pong commands, added in v1 if (log.isDebugEnabled()) { - log.debug("[{}] Sending ping message", ctx.channel()); + log.debug("[{}] Sending ping message", this.toString()); } waitingForPingResponse = true; sendPing(); } else { if (log.isDebugEnabled()) { - log.debug("[{}] Peer doesn't support keep-alive", ctx.channel()); + log.debug("[{}] Peer doesn't support keep-alive", this.toString()); } } } @@ -134,7 +134,7 @@ protected ChannelFuture sendPing() { .addListener(future -> { if (!future.isSuccess()) { log.warn("[{}] Forcing connection to close since cannot send a ping message.", - ctx.channel(), future.cause()); + this.toString(), future.cause()); ctx.close(); } }); @@ -152,5 +152,20 @@ public void cancelKeepAliveTask() { */ protected abstract boolean isHandshakeCompleted(); + /** + * Demo: [id: 0x2561bcd1, L:/10.0.136.103:6650 ! R:/240.240.0.5:58038]. + * L: local Address. + * R: remote address. + */ + @Override + public String toString() { + ChannelHandlerContext ctx = this.ctx; + if (ctx == null) { + return "[ctx: null]"; + } else { + return ctx.channel().toString(); + } + } + private static final Logger log = LoggerFactory.getLogger(PulsarHandler.class); } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/sasl/TGTRefreshThread.java b/pulsar-common/src/main/java/org/apache/pulsar/common/sasl/TGTRefreshThread.java index aa4c8de4a9c14..6a0a7448f75d5 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/sasl/TGTRefreshThread.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/sasl/TGTRefreshThread.java @@ -97,7 +97,7 @@ private long getRefreshTime(KerberosTicket tgt) { @Override public void run() { log.info("TGT refresh thread started."); - while (true) { + while (!Thread.currentThread().isInterrupted()) { // renewal thread's main loop. if it exits from here, thread will exit. KerberosTicket tgt = getTGT(); long now = System.currentTimeMillis(); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/stats/JvmMetrics.java b/pulsar-common/src/main/java/org/apache/pulsar/common/stats/JvmMetrics.java index 1f15beb8a0b92..8a8da0bb1ac93 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/stats/JvmMetrics.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/stats/JvmMetrics.java @@ -99,6 +99,9 @@ public List generate() { Runtime r = Runtime.getRuntime(); + RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean(); + + m.put("jvm_start_time", runtimeMXBean.getStartTime()); m.put("jvm_heap_used", r.totalMemory() - r.freeMemory()); m.put("jvm_max_memory", r.maxMemory()); m.put("jvm_total_memory", r.totalMemory()); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/stats/MetricsUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/stats/MetricsUtil.java new file mode 100644 index 0000000000000..f13abb6645e86 --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/stats/MetricsUtil.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.stats; + +import java.util.concurrent.TimeUnit; + +/** + * Utility class for metrics. + */ +public class MetricsUtil { + + private static final double NANOS_IN_SECOND = TimeUnit.SECONDS.toNanos(1); + + /** + * Convert a duration to seconds. Unlike {@link TimeUnit#toSeconds(long)}, this method preserves fractional + * precision. + * + * @param duration the duration + * @param timeUnit the time unit + * @return the duration in seconds + */ + public static double convertToSeconds(long duration, TimeUnit timeUnit) { + return timeUnit.toNanos(duration) / NANOS_IN_SECOND; + } + +} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/stats/Rate.java b/pulsar-common/src/main/java/org/apache/pulsar/common/stats/Rate.java index 886e31ab71216..936962d8ee544 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/stats/Rate.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/stats/Rate.java @@ -28,6 +28,7 @@ public class Rate { private final LongAdder valueAdder = new LongAdder(); private final LongAdder countAdder = new LongAdder(); private final LongAdder totalCountAdder = new LongAdder(); + private final LongAdder totalValueAdder = new LongAdder(); // Computed stats private long count = 0L; @@ -43,12 +44,14 @@ public void recordEvent() { public void recordEvent(long value) { valueAdder.add(value); + totalValueAdder.add(value); countAdder.increment(); totalCountAdder.increment(); } public void recordMultipleEvents(long events, long totalValue) { valueAdder.add(totalValue); + totalValueAdder.add(totalValue); countAdder.add(events); totalCountAdder.add(events); } @@ -88,4 +91,8 @@ public double getValueRate() { public long getTotalCount() { return this.totalCountAdder.longValue(); } + + public long getTotalValue() { + return this.totalValueAdder.sum(); + } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/tls/InetAddressUtils.java b/pulsar-common/src/main/java/org/apache/pulsar/common/tls/InetAddressUtils.java index a8bf837ef5666..d0f3c81a074a1 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/tls/InetAddressUtils.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/tls/InetAddressUtils.java @@ -35,9 +35,12 @@ private InetAddressUtils() { } private static final String IPV4_BASIC_PATTERN_STRING = - "(([1-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){1}" + // initial first field, 1-255 - "(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){2}" + // following 2 fields, 0-255 followed by . - "([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])"; // final field, 0-255 + // initial first field, 1-255 + "(([1-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){1}" + // following 2 fields, 0-255 followed by . + + "(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){2}" + // final field, 0-255 + + "([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])"; private static final Pattern IPV4_PATTERN = Pattern.compile("^" + IPV4_BASIC_PATTERN_STRING + "$"); @@ -50,10 +53,9 @@ private InetAddressUtils() { "^[0-9a-fA-F]{1,4}(:[0-9a-fA-F]{1,4}){7}$"); private static final Pattern IPV6_HEX_COMPRESSED_PATTERN = - Pattern.compile( - "^(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4}){0,5})?)" + // 0-6 hex fields - "::" + // concat - "(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4}){0,5})?)$"); // 0-6 hex fields + Pattern.compile("^(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4}){0,5})?)" // 0-6 hex fields + + "::" // concat + + "(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4}){0,5})?)$"); // 0-6 hex fields /* * The above pattern is not totally rigorous as it allows for more than 7 hex fields in total diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicCompactionStrategy.java b/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicCompactionStrategy.java index f06374b234fc7..39bfa6d71bc96 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicCompactionStrategy.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicCompactionStrategy.java @@ -18,7 +18,11 @@ */ package org.apache.pulsar.common.topics; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.classification.InterfaceAudience; +import org.apache.pulsar.common.classification.InterfaceStability; /** * Defines a custom strategy to compact messages in a topic. @@ -43,8 +47,13 @@ * "topicCompactionStrategyClassName", strategy.getClass().getCanonicalName())) * .create(); */ +@InterfaceAudience.Private +@InterfaceStability.Unstable public interface TopicCompactionStrategy { + String TABLE_VIEW_TAG = "table-view"; + Map INSTANCES = new ConcurrentHashMap<>(); + /** * Returns the schema object for this strategy. * @return @@ -60,17 +69,27 @@ public interface TopicCompactionStrategy { */ boolean shouldKeepLeft(T prev, T cur); - static TopicCompactionStrategy load(String topicCompactionStrategyClassName) { + default void handleSkippedMessage(String key, T cur) { + } + + + static TopicCompactionStrategy load(String tag, String topicCompactionStrategyClassName) { if (topicCompactionStrategyClassName == null) { return null; } + try { Class clazz = Class.forName(topicCompactionStrategyClassName); - Object instance = clazz.getDeclaredConstructor().newInstance(); - return (TopicCompactionStrategy) instance; + TopicCompactionStrategy instance = (TopicCompactionStrategy) clazz.getDeclaredConstructor().newInstance(); + INSTANCES.put(tag, instance); + return instance; } catch (Exception e) { throw new IllegalArgumentException( "Error when loading topic compaction strategy: " + topicCompactionStrategyClassName, e); } } + + static TopicCompactionStrategy getInstance(String tag) { + return INSTANCES.get(tag); + } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicList.java b/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicList.java index 250cea217ee5f..9e24483df8239 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicList.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicList.java @@ -18,16 +18,18 @@ */ package org.apache.pulsar.common.topics; +import com.google.common.annotations.VisibleForTesting; import com.google.common.hash.Hashing; +import com.google.re2j.Pattern; import java.nio.charset.StandardCharsets; import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.regex.Pattern; import java.util.stream.Collectors; import lombok.experimental.UtilityClass; import org.apache.pulsar.common.naming.SystemTopicNames; +import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; @UtilityClass @@ -47,13 +49,17 @@ public static List filterTopics(List original, String regex) { } public static List filterTopics(List original, Pattern topicsPattern) { - final Pattern shortenedTopicsPattern = topicsPattern.toString().contains(SCHEME_SEPARATOR) - ? Pattern.compile(SCHEME_SEPARATOR_PATTERN.split(topicsPattern.toString())[1]) : topicsPattern; + + final Pattern shortenedTopicsPattern = Pattern.compile(removeTopicDomainScheme(topicsPattern.toString())); return original.stream() .map(TopicName::get) + .filter(topicName -> { + String partitionedTopicName = topicName.getPartitionedTopicName(); + String removedScheme = SCHEME_SEPARATOR_PATTERN.split(partitionedTopicName)[1]; + return shortenedTopicsPattern.matcher(removedScheme).matches(); + }) .map(TopicName::toString) - .filter(topic -> shortenedTopicsPattern.matcher(SCHEME_SEPARATOR_PATTERN.split(topic)[1]).matches()) .collect(Collectors.toList()); } @@ -78,4 +84,24 @@ public static Set minus(Collection list1, Collection lis s1.removeAll(list2); return s1; } + + @VisibleForTesting + static String removeTopicDomainScheme(String originalRegexp) { + if (!originalRegexp.toString().contains(SCHEME_SEPARATOR)) { + return originalRegexp; + } + String[] parts = SCHEME_SEPARATOR_PATTERN.split(originalRegexp.toString()); + String prefix = parts[0]; + String removedTopicDomain = parts[1]; + if (prefix.equals(TopicDomain.persistent.value()) || prefix.equals(TopicDomain.non_persistent.value())) { + prefix = ""; + } else if (prefix.endsWith(TopicDomain.non_persistent.value())) { + prefix = prefix.substring(0, prefix.length() - TopicDomain.non_persistent.value().length()); + } else if (prefix.endsWith(TopicDomain.persistent.value())){ + prefix = prefix.substring(0, prefix.length() - TopicDomain.persistent.value().length()); + } else { + throw new IllegalArgumentException("Does not support topic domain: " + prefix); + } + return String.format("%s%s", prefix, removedTopicDomain); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/Backoff.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/Backoff.java similarity index 99% rename from pulsar-client/src/main/java/org/apache/pulsar/client/impl/Backoff.java rename to pulsar-common/src/main/java/org/apache/pulsar/common/util/Backoff.java index daaf349940035..4eab85f3c41be 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/Backoff.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/Backoff.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.client.impl; +package org.apache.pulsar.common.util; import com.google.common.annotations.VisibleForTesting; import java.time.Clock; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BackoffBuilder.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/BackoffBuilder.java similarity index 91% rename from pulsar-client/src/main/java/org/apache/pulsar/client/impl/BackoffBuilder.java rename to pulsar-common/src/main/java/org/apache/pulsar/common/util/BackoffBuilder.java index 9913393fa9aa9..69b390300815b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BackoffBuilder.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/BackoffBuilder.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.client.impl; +package org.apache.pulsar.common.util; import java.time.Clock; import java.util.concurrent.TimeUnit; @@ -32,8 +32,11 @@ public class BackoffBuilder { public BackoffBuilder() { this.initial = 0; + this.unitInitial = TimeUnit.MILLISECONDS; this.max = 0; + this.unitMax = TimeUnit.MILLISECONDS; this.mandatoryStop = 0; + this.unitMandatoryStop = TimeUnit.MILLISECONDS; this.clock = Clock.systemDefaultZone(); } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/CmdGenerateDocs.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/CmdGenerateDocs.java deleted file mode 100644 index a4deb4444601e..0000000000000 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/CmdGenerateDocs.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util; - -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterDescription; -import com.beust.jcommander.Parameters; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import lombok.Getter; -import lombok.Setter; - -@Getter -@Setter -@Parameters(commandDescription = "Generate documentation automatically.") -public class CmdGenerateDocs { - - @Parameter( - names = {"-h", "--help"}, - description = "Display help information" - ) - public boolean help; - - @Parameter( - names = {"-n", "--command-names"}, - description = "List of command names" - ) - private List commandNames = new ArrayList<>(); - - private static final String name = "gen-doc"; - private final JCommander jcommander; - - public CmdGenerateDocs(String cmdName) { - jcommander = new JCommander(this); - jcommander.setProgramName(cmdName); - } - - public CmdGenerateDocs addCommand(String name, Object command) { - jcommander.addCommand(name, command); - return this; - } - - public boolean run(String[] args) { - JCommander tmpCmd = new JCommander(this); - tmpCmd.setProgramName(jcommander.getProgramName() + " " + name); - try { - if (args == null) { - args = new String[]{}; - } - tmpCmd.parse(args); - } catch (Exception e) { - System.err.println(e.getMessage()); - System.err.println(); - tmpCmd.usage(); - return false; - } - if (help) { - tmpCmd.usage(); - return true; - } - - if (commandNames.size() == 0) { - for (Map.Entry cmd : jcommander.getCommands().entrySet()) { - if (cmd.getKey().equals(name)) { - continue; - } - System.out.println(generateDocument(cmd.getKey(), jcommander)); - } - } else { - for (String commandName : commandNames) { - if (commandName.equals(name)) { - continue; - } - if (!jcommander.getCommands().keySet().contains(commandName)) { - continue; - } - System.out.println(generateDocument(commandName, jcommander)); - } - } - return true; - } - - private String generateDocument(String module, JCommander commander) { - JCommander cmd = commander.getCommands().get(module); - StringBuilder sb = new StringBuilder(); - sb.append("# ").append(module).append("\n\n"); - String desc = commander.getUsageFormatter().getCommandDescription(module); - if (null != desc && !desc.isEmpty()) { - sb.append(desc).append("\n"); - } - sb.append("\n\n```shell\n") - .append("$ "); - if (null != jcommander.getProgramName() && !jcommander.getProgramName().isEmpty()) { - sb.append(jcommander.getProgramName()).append(" "); - } - sb.append(module); - if (cmd.getObjects().size() > 0 - && cmd.getObjects().get(0).getClass().getName().equals("com.beust.jcommander.JCommander")) { - JCommander cmdObj = (JCommander) cmd.getObjects().get(0); - sb.append(" subcommand").append("\n```").append("\n\n"); - cmdObj.getCommands().forEach((subK, subV) -> { - if (!subK.equals(name)) { - sb.append("\n\n## ").append(subK).append("\n\n"); - String subDesc = cmdObj.getUsageFormatter().getCommandDescription(subK); - if (null != subDesc && !subDesc.isEmpty()) { - sb.append(subDesc).append("\n"); - } - sb.append("```shell\n$ "); - if (null != jcommander.getProgramName() && !jcommander.getProgramName().isEmpty()) { - sb.append(jcommander.getProgramName()).append(" "); - } - sb.append(module).append(" ").append(subK).append(" options").append("\n```\n\n"); - List options = cmdObj.getCommands().get(subK).getParameters(); - if (options.size() > 0) { - sb.append("|Flag|Description|Default|\n"); - sb.append("|---|---|---|\n"); - } - options.forEach((option) -> - sb.append("| `").append(option.getNames()) - .append("` | ").append(option.getDescription().replace("\n", " ")) - .append("|").append(option.getDefault()).append("|\n") - ); - } - }); - } else { - sb.append(" options").append("\n```").append("\n\n"); - sb.append("|Flag|Description|Default|\n"); - sb.append("|---|---|---|\n"); - List options = cmd.getParameters(); - options.forEach((option) -> - sb.append("| `").append(option.getNames()) - .append("` | ").append(option.getDescription().replace("\n", " ")) - .append("|").append(option.getDefault()).append("|\n") - ); - } - return sb.toString(); - } -} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/DefaultPulsarSslFactory.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/DefaultPulsarSslFactory.java new file mode 100644 index 0000000000000..9be16f835b28b --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/DefaultPulsarSslFactory.java @@ -0,0 +1,366 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util; + +import io.netty.buffer.ByteBufAllocator; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslProvider; +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.util.concurrent.atomic.AtomicReference; +import javax.annotation.concurrent.NotThreadSafe; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; +import javax.net.ssl.SSLParameters; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.client.api.AuthenticationDataProvider; +import org.apache.pulsar.client.api.KeyStoreParams; +import org.apache.pulsar.common.util.keystoretls.KeyStoreSSLContext; + +/** + * Default Implementation of {@link PulsarSslFactory}. This factory loads file based certificates to create SSLContext + * and SSL Engines. This class is not thread safe. It has been integrated into the pulsar code base as a single writer, + * multiple readers pattern. + */ +@NotThreadSafe +public class DefaultPulsarSslFactory implements PulsarSslFactory { + + private PulsarSslConfiguration config; + private final AtomicReference internalSslContext = new AtomicReference<>(); + private final AtomicReference internalNettySslContext = new AtomicReference<>(); + + protected FileModifiedTimeUpdater tlsKeyStore; + protected FileModifiedTimeUpdater tlsTrustStore; + protected FileModifiedTimeUpdater tlsTrustCertsFilePath; + protected FileModifiedTimeUpdater tlsCertificateFilePath; + protected FileModifiedTimeUpdater tlsKeyFilePath; + protected AuthenticationDataProvider authData; + protected boolean isTlsTrustStoreStreamProvided; + protected final String[] defaultSslEnabledProtocols = {"TLSv1.3", "TLSv1.2"}; + protected String tlsKeystoreType; + protected String tlsKeystorePath; + protected String tlsKeystorePassword; + + /** + * Initializes the DefaultPulsarSslFactory. + * + * @param config {@link PulsarSslConfiguration} object required for initialization. + * + */ + @Override + public void initialize(PulsarSslConfiguration config) { + this.config = config; + AuthenticationDataProvider authData = this.config.getAuthData(); + if (this.config.isTlsEnabledWithKeystore()) { + if (authData != null && authData.hasDataForTls()) { + KeyStoreParams authParams = authData.getTlsKeyStoreParams(); + if (authParams != null) { + this.tlsKeystoreType = authParams.getKeyStoreType(); + this.tlsKeystorePath = authParams.getKeyStorePath(); + this.tlsKeystorePassword = authParams.getKeyStorePassword(); + } + } + if (this.tlsKeystoreType == null) { + this.tlsKeystoreType = this.config.getTlsKeyStoreType(); + } + if (this.tlsKeystorePath == null) { + this.tlsKeystorePath = this.config.getTlsKeyStorePath(); + } + if (this.tlsKeystorePassword == null) { + this.tlsKeystorePassword = this.config.getTlsKeyStorePassword(); + } + this.tlsKeyStore = new FileModifiedTimeUpdater(this.tlsKeystorePath); + this.tlsTrustStore = new FileModifiedTimeUpdater(this.config.getTlsTrustStorePath()); + } else { + if (authData != null && authData.hasDataForTls()) { + if (authData.getTlsTrustStoreStream() != null) { + this.isTlsTrustStoreStreamProvided = true; + } else { + this.tlsTrustCertsFilePath = new FileModifiedTimeUpdater(this.config.getTlsTrustCertsFilePath()); + } + this.authData = authData; + } else { + this.tlsCertificateFilePath = new FileModifiedTimeUpdater(this.config.getTlsCertificateFilePath()); + this.tlsTrustCertsFilePath = new FileModifiedTimeUpdater(this.config.getTlsTrustCertsFilePath()); + this.tlsKeyFilePath = new FileModifiedTimeUpdater(this.config.getTlsKeyFilePath()); + } + } + } + + /** + * Creates a Client {@link SSLEngine} utilizing the peer hostname, peer port and {@link PulsarSslConfiguration} + * object provided during initialization. + * + * @param peerHost the name of the peer host + * @param peerPort the port number of the peer + * @return {@link SSLEngine} + */ + @Override + public SSLEngine createClientSslEngine(ByteBufAllocator buf, String peerHost, int peerPort) { + return createSSLEngine(buf, peerHost, peerPort, NetworkMode.CLIENT); + } + + /** + * Creates a Server {@link SSLEngine} utilizing the {@link PulsarSslConfiguration} object provided during + * initialization. + * + * @return {@link SSLEngine} + */ + @Override + public SSLEngine createServerSslEngine(ByteBufAllocator buf) { + return createSSLEngine(buf, "", 0, NetworkMode.SERVER); + } + + /** + * Returns a boolean value based on if the underlying certificate files have been modified since it was last read. + * + * @return {@code true} if the underlying certificates have been modified indicating that + * the SSL Context should be refreshed. + */ + @Override + public boolean needsUpdate() { + if (this.config.isTlsEnabledWithKeystore()) { + return (this.tlsKeyStore != null && this.tlsKeyStore.checkAndRefresh()) + || (this.tlsTrustStore != null && this.tlsTrustStore.checkAndRefresh()); + } else { + if (this.authData != null && this.authData.hasDataForTls()) { + return true; + } else { + return this.tlsTrustCertsFilePath.checkAndRefresh() || this.tlsCertificateFilePath.checkAndRefresh() + || this.tlsKeyFilePath.checkAndRefresh(); + } + } + } + + /** + * Creates a {@link SSLContext} object and saves it internally. + * + * @throws Exception If there were any issues generating the {@link SSLContext} + */ + @Override + public void createInternalSslContext() throws Exception { + if (this.config.isTlsEnabledWithKeystore()) { + this.internalSslContext.set(buildKeystoreSslContext(this.config.isServerMode())); + } else { + if (this.config.isHttps()) { + this.internalSslContext.set(buildSslContext()); + } else { + this.internalNettySslContext.set(buildNettySslContext()); + } + } + } + + + /** + * Get the internally stored {@link SSLContext}. + * + * @return {@link SSLContext} + * @throws RuntimeException if the {@link SSLContext} object has not yet been initialized. + */ + @Override + public SSLContext getInternalSslContext() { + if (this.internalSslContext.get() == null) { + throw new RuntimeException("Internal SSL context is not initialized. " + + "Please call createInternalSslContext() first."); + } + return this.internalSslContext.get(); + } + + /** + * Get the internally stored {@link SslContext}. + * + * @return {@link SslContext} + * @throws RuntimeException if the {@link SslContext} object has not yet been initialized. + */ + public SslContext getInternalNettySslContext() { + if (this.internalNettySslContext.get() == null) { + throw new RuntimeException("Internal SSL context is not initialized. " + + "Please call createInternalSslContext() first."); + } + return this.internalNettySslContext.get(); + } + + private SSLContext buildKeystoreSslContext(boolean isServerMode) throws GeneralSecurityException, IOException { + KeyStoreSSLContext keyStoreSSLContext; + if (isServerMode) { + keyStoreSSLContext = KeyStoreSSLContext.createServerKeyStoreSslContext(this.config.getTlsProvider(), + this.tlsKeystoreType, this.tlsKeyStore.getFileName(), + this.tlsKeystorePassword, this.config.isAllowInsecureConnection(), + this.config.getTlsTrustStoreType(), this.tlsTrustStore.getFileName(), + this.config.getTlsTrustStorePassword(), this.config.isRequireTrustedClientCertOnConnect(), + this.config.getTlsCiphers(), this.config.getTlsProtocols()); + } else { + keyStoreSSLContext = KeyStoreSSLContext.createClientKeyStoreSslContext(this.config.getTlsProvider(), + this.tlsKeystoreType, this.tlsKeyStore.getFileName(), + this.tlsKeystorePassword, this.config.isAllowInsecureConnection(), + this.config.getTlsTrustStoreType(), this.tlsTrustStore.getFileName(), + this.config.getTlsTrustStorePassword(), this.config.getTlsCiphers(), + this.config.getTlsProtocols()); + } + return keyStoreSSLContext.createSSLContext(); + } + + private SSLContext buildSslContext() throws GeneralSecurityException { + if (this.authData != null && this.authData.hasDataForTls()) { + if (this.isTlsTrustStoreStreamProvided) { + return SecurityUtility.createSslContext(this.config.isAllowInsecureConnection(), + SecurityUtility.loadCertificatesFromPemStream(this.authData.getTlsTrustStoreStream()), + this.authData.getTlsCertificates(), + this.authData.getTlsPrivateKey(), + this.config.getTlsProvider()); + } else { + if (this.authData.getTlsCertificates() != null) { + return SecurityUtility.createSslContext(this.config.isAllowInsecureConnection(), + SecurityUtility.loadCertificatesFromPemFile(this.tlsTrustCertsFilePath.getFileName()), + this.authData.getTlsCertificates(), + this.authData.getTlsPrivateKey(), + this.config.getTlsProvider()); + } else { + return SecurityUtility.createSslContext(this.config.isAllowInsecureConnection(), + this.tlsTrustCertsFilePath.getFileName(), + this.authData.getTlsCertificateFilePath(), + this.authData.getTlsPrivateKeyFilePath(), + this.config.getTlsProvider() + ); + } + } + } else { + return SecurityUtility.createSslContext(this.config.isAllowInsecureConnection(), + this.tlsTrustCertsFilePath.getFileName(), + this.tlsCertificateFilePath.getFileName(), + this.tlsKeyFilePath.getFileName(), + this.config.getTlsProvider()); + } + } + + private SslContext buildNettySslContext() throws GeneralSecurityException, IOException { + SslProvider sslProvider = null; + if (StringUtils.isNotBlank(this.config.getTlsProvider())) { + sslProvider = SslProvider.valueOf(this.config.getTlsProvider()); + } + if (this.authData != null && this.authData.hasDataForTls()) { + if (this.isTlsTrustStoreStreamProvided) { + return SecurityUtility.createNettySslContextForClient(sslProvider, + this.config.isAllowInsecureConnection(), + this.authData.getTlsTrustStoreStream(), + this.authData.getTlsCertificates(), + this.authData.getTlsPrivateKey(), + this.config.getTlsCiphers(), + this.config.getTlsProtocols()); + } else { + if (this.authData.getTlsCertificates() != null) { + return SecurityUtility.createNettySslContextForClient(sslProvider, + this.config.isAllowInsecureConnection(), + this.tlsTrustCertsFilePath.getFileName(), + this.authData.getTlsCertificates(), + this.authData.getTlsPrivateKey(), + this.config.getTlsCiphers(), + this.config.getTlsProtocols()); + } else { + return SecurityUtility.createNettySslContextForClient(sslProvider, + this.config.isAllowInsecureConnection(), + this.tlsTrustCertsFilePath.getFileName(), + this.authData.getTlsCertificateFilePath(), + this.authData.getTlsPrivateKeyFilePath(), + this.config.getTlsCiphers(), + this.config.getTlsProtocols()); + } + } + } else { + if (this.config.isServerMode()) { + return SecurityUtility.createNettySslContextForServer(sslProvider, + this.config.isAllowInsecureConnection(), + this.tlsTrustCertsFilePath.getFileName(), + this.tlsCertificateFilePath.getFileName(), + this.tlsKeyFilePath.getFileName(), + this.config.getTlsCiphers(), + this.config.getTlsProtocols(), + this.config.isRequireTrustedClientCertOnConnect()); + } else { + return SecurityUtility.createNettySslContextForClient(sslProvider, + this.config.isAllowInsecureConnection(), + this.tlsTrustCertsFilePath.getFileName(), + this.tlsCertificateFilePath.getFileName(), + this.tlsKeyFilePath.getFileName(), + this.config.getTlsCiphers(), + this.config.getTlsProtocols()); + } + } + } + + private SSLEngine createSSLEngine(ByteBufAllocator buf, String peerHost, int peerPort, NetworkMode mode) { + SSLEngine sslEngine; + SSLParameters sslParams; + SSLContext sslContext = this.internalSslContext.get(); + SslContext nettySslContext = this.internalNettySslContext.get(); + validateSslContext(sslContext, nettySslContext); + if (mode == NetworkMode.CLIENT) { + if (sslContext != null) { + sslEngine = sslContext.createSSLEngine(peerHost, peerPort); + } else { + sslEngine = nettySslContext.newEngine(buf, peerHost, peerPort); + } + sslEngine.setUseClientMode(true); + sslParams = sslEngine.getSSLParameters(); + } else { + if (sslContext != null) { + sslEngine = sslContext.createSSLEngine(); + } else { + sslEngine = nettySslContext.newEngine(buf); + } + sslEngine.setUseClientMode(false); + sslParams = sslEngine.getSSLParameters(); + if (this.config.isRequireTrustedClientCertOnConnect()) { + sslParams.setNeedClientAuth(true); + } else { + sslParams.setWantClientAuth(true); + } + } + if (this.config.getTlsProtocols() != null && !this.config.getTlsProtocols().isEmpty()) { + sslParams.setProtocols(this.config.getTlsProtocols().toArray(new String[0])); + } else { + sslParams.setProtocols(defaultSslEnabledProtocols); + } + if (this.config.getTlsCiphers() != null && !this.config.getTlsCiphers().isEmpty()) { + sslParams.setCipherSuites(this.config.getTlsCiphers().toArray(new String[0])); + } + sslEngine.setSSLParameters(sslParams); + return sslEngine; + } + + private void validateSslContext(SSLContext sslContext, SslContext nettySslContext) { + if (sslContext == null && nettySslContext == null) { + throw new RuntimeException("Internal SSL context is not initialized. " + + "Please call createInternalSslContext() first."); + } + } + + /** + * Clean any resources that may have been created. + * @throws Exception if any resources failed to be cleaned. + */ + @Override + public void close() throws Exception { + // noop + } + + private enum NetworkMode { + CLIENT, SERVER + } +} \ No newline at end of file diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/DefaultSslContextBuilder.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/DefaultSslContextBuilder.java deleted file mode 100644 index ab5f41c6bbf8d..0000000000000 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/DefaultSslContextBuilder.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util; - -import java.security.GeneralSecurityException; -import javax.net.ssl.SSLContext; - -@SuppressWarnings("checkstyle:JavadocType") -public class DefaultSslContextBuilder extends SslContextAutoRefreshBuilder { - private volatile SSLContext sslContext; - - protected final boolean tlsAllowInsecureConnection; - protected final FileModifiedTimeUpdater tlsTrustCertsFilePath, tlsCertificateFilePath, tlsKeyFilePath; - protected final boolean tlsRequireTrustedClientCertOnConnect; - private final String providerName; - - public DefaultSslContextBuilder(boolean allowInsecure, String trustCertsFilePath, String certificateFilePath, - String keyFilePath, boolean requireTrustedClientCertOnConnect, - long certRefreshInSec) { - super(certRefreshInSec); - this.tlsAllowInsecureConnection = allowInsecure; - this.tlsTrustCertsFilePath = new FileModifiedTimeUpdater(trustCertsFilePath); - this.tlsCertificateFilePath = new FileModifiedTimeUpdater(certificateFilePath); - this.tlsKeyFilePath = new FileModifiedTimeUpdater(keyFilePath); - this.tlsRequireTrustedClientCertOnConnect = requireTrustedClientCertOnConnect; - this.providerName = null; - } - - public DefaultSslContextBuilder(boolean allowInsecure, String trustCertsFilePath, String certificateFilePath, - String keyFilePath, boolean requireTrustedClientCertOnConnect, - long certRefreshInSec, String providerName) { - super(certRefreshInSec); - this.tlsAllowInsecureConnection = allowInsecure; - this.tlsTrustCertsFilePath = new FileModifiedTimeUpdater(trustCertsFilePath); - this.tlsCertificateFilePath = new FileModifiedTimeUpdater(certificateFilePath); - this.tlsKeyFilePath = new FileModifiedTimeUpdater(keyFilePath); - this.tlsRequireTrustedClientCertOnConnect = requireTrustedClientCertOnConnect; - this.providerName = providerName; - } - - @Override - public synchronized SSLContext update() throws GeneralSecurityException { - this.sslContext = SecurityUtility.createSslContext(tlsAllowInsecureConnection, - tlsTrustCertsFilePath.getFileName(), tlsCertificateFilePath.getFileName(), - tlsKeyFilePath.getFileName(), this.providerName); - return this.sslContext; - } - - @Override - public SSLContext getSslContext() { - return this.sslContext; - } - - @Override - public boolean needUpdate() { - return tlsTrustCertsFilePath.checkAndRefresh() - || tlsCertificateFilePath.checkAndRefresh() - || tlsKeyFilePath.checkAndRefresh(); - } -} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java index 626a14b92eedd..10c1951ab208b 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java @@ -21,8 +21,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static java.lang.String.format; import static java.util.Objects.requireNonNull; -import com.fasterxml.jackson.databind.AnnotationIntrospector; -import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector; import com.fasterxml.jackson.databind.util.EnumResolver; import java.lang.reflect.Field; import java.lang.reflect.Method; @@ -58,8 +56,6 @@ public final class FieldParser { private static final Map CONVERTERS = new HashMap<>(); private static final Map, Class> WRAPPER_TYPES = new HashMap<>(); - private static final AnnotationIntrospector ANNOTATION_INTROSPECTOR = new JacksonAnnotationIntrospector(); - static { // Preload converters and wrapperTypes. initConverters(); @@ -100,7 +96,8 @@ public static T convert(Object from, Class to) { if (to.isEnum()) { // Converting string to enum - EnumResolver r = EnumResolver.constructUsingToString((Class>) to, ANNOTATION_INTROSPECTOR); + EnumResolver r = EnumResolver.constructUsingToString( + ObjectMapperFactory.getMapper().getObjectMapper().getDeserializationConfig(), to); T value = (T) r.findEnum((String) from); if (value == null) { throw new RuntimeException("Invalid value '" + from + "' for enum " + to); @@ -314,6 +311,9 @@ public static Float stringToFloat(String val) { * @return The converted list with type {@code }. */ public static List stringToList(String val, Class type) { + if (val == null) { + return null; + } String[] tokens = trim(val).split(","); return Arrays.stream(tokens).map(t -> { return convert(trim(t), type); @@ -330,6 +330,9 @@ public static List stringToList(String val, Class type) { * @return The converted set with type {@code }. */ public static Set stringToSet(String val, Class type) { + if (val == null) { + return null; + } String[] tokens = trim(val).split(","); return Arrays.stream(tokens).map(t -> { return convert(trim(t), type); @@ -337,6 +340,9 @@ public static Set stringToSet(String val, Class type) { } private static Map stringToMap(String strValue, Class keyType, Class valueType) { + if (strValue == null) { + return null; + } String[] tokens = trim(strValue).split(","); Map map = new HashMap<>(); for (String token : tokens) { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java index 2b082b4a7899b..454eee0f966c5 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java @@ -18,8 +18,10 @@ */ package org.apache.pulsar.common.util; +import com.google.common.util.concurrent.MoreExecutors; import java.time.Duration; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Objects; @@ -52,9 +54,17 @@ public class FutureUtil { * @return a new CompletableFuture that is completed when all of the given CompletableFutures complete */ public static CompletableFuture waitForAll(Collection> futures) { + if (futures == null || futures.isEmpty()) { + return CompletableFuture.completedFuture(null); + } return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); } + public static CompletableFuture runWithCurrentThread(Runnable runnable) { + return CompletableFuture.runAsync( + () -> runnable.run(), MoreExecutors.directExecutor()); + } + public static CompletableFuture> waitForAll(Stream>> futures) { return futures.reduce(CompletableFuture.completedFuture(new ArrayList<>()), (pre, curr) -> pre.thenCompose(preV -> curr.thenApply(currV -> { @@ -63,6 +73,36 @@ public static CompletableFuture> waitForAll(Stream void completeAfter(final CompletableFuture dest, CompletableFuture src) { + src.whenComplete((v, ex) -> { + if (ex != null) { + dest.completeExceptionally(ex); + } else { + dest.complete(v); + } + }); + } + + /** + * Make the dest future complete after others. {@param dest} is will be completed with a {@link Void} value + * if all the futures of {@param src} is completed, or be completed exceptionally with the same error as the first + * one completed exceptionally future of {@param src}. + */ + public static void completeAfterAll(final CompletableFuture dest, + CompletableFuture... src) { + FutureUtil.waitForAll(Arrays.asList(src)).whenComplete((ignore, ex) -> { + if (ex != null) { + dest.completeExceptionally(ex); + } else { + dest.complete(null); + } + }); + } + /** * Return a future that represents the completion of any future in the provided Collection. * @@ -125,7 +165,7 @@ public static CompletableFuture> waitForAny(Collection waitForAllAndSupportCancel( - Collection> futures) { + Collection> futures) { CompletableFuture[] futuresArray = futures.toArray(new CompletableFuture[0]); CompletableFuture combinedFuture = CompletableFuture.allOf(futuresArray); whenCancelledOrTimedOut(combinedFuture, () -> { @@ -162,9 +202,9 @@ public static CompletableFuture failedFuture(Throwable t) { public static Throwable unwrapCompletionException(Throwable ex) { if (ex instanceof CompletionException) { - return ex.getCause(); + return unwrapCompletionException(ex.getCause()); } else if (ex instanceof ExecutionException) { - return ex.getCause(); + return unwrapCompletionException(ex.getCause()); } else { return ex; } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/LazyLoadableValue.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/LazyLoadableValue.java new file mode 100644 index 0000000000000..063d434a64fc0 --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/LazyLoadableValue.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util; + +import java.util.function.Supplier; + +/*** + * Used to lazy load a value, only calculate it when used. Not thread-safety. + */ +public class LazyLoadableValue { + + private Supplier loader; + + private T value; + + public LazyLoadableValue(Supplier loader) { + this.loader = loader; + } + + public T getValue() { + if (value == null) { + value = loader.get(); + } + return value; + } +} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyClientSslContextRefresher.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyClientSslContextRefresher.java deleted file mode 100644 index 6f1690310c104..0000000000000 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyClientSslContextRefresher.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util; - -import io.netty.handler.ssl.SslContext; -import io.netty.handler.ssl.SslProvider; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.security.GeneralSecurityException; -import java.security.cert.X509Certificate; -import java.util.Set; -import javax.net.ssl.SSLException; -import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.client.api.AuthenticationDataProvider; - -/** - * SSL context builder for Netty Client side. - */ -@Slf4j -public class NettyClientSslContextRefresher extends SslContextAutoRefreshBuilder { - private volatile SslContext sslNettyContext; - private final boolean tlsAllowInsecureConnection; - protected final FileModifiedTimeUpdater tlsTrustCertsFilePath; - protected final FileModifiedTimeUpdater tlsCertsFilePath; - protected final FileModifiedTimeUpdater tlsPrivateKeyFilePath; - private final AuthenticationDataProvider authData; - private final SslProvider sslProvider; - private final Set ciphers; - private final Set protocols; - - public NettyClientSslContextRefresher(SslProvider sslProvider, boolean allowInsecure, - String trustCertsFilePath, - AuthenticationDataProvider authData, - Set ciphers, - Set protocols, - long delayInSeconds) { - super(delayInSeconds); - this.tlsAllowInsecureConnection = allowInsecure; - this.tlsTrustCertsFilePath = new FileModifiedTimeUpdater(trustCertsFilePath); - this.authData = authData; - this.tlsCertsFilePath = new FileModifiedTimeUpdater( - authData != null ? authData.getTlsCerificateFilePath() : null); - this.tlsPrivateKeyFilePath = new FileModifiedTimeUpdater( - authData != null ? authData.getTlsPrivateKeyFilePath() : null); - this.sslProvider = sslProvider; - this.ciphers = ciphers; - this.protocols = protocols; - } - - @Override - public synchronized SslContext update() - throws SSLException, FileNotFoundException, GeneralSecurityException, IOException { - if (authData != null && authData.hasDataForTls()) { - this.sslNettyContext = authData.getTlsTrustStoreStream() == null - ? SecurityUtility.createNettySslContextForClient(this.sslProvider, this.tlsAllowInsecureConnection, - tlsTrustCertsFilePath.getFileName(), (X509Certificate[]) authData.getTlsCertificates(), - authData.getTlsPrivateKey(), this.ciphers, this.protocols) - : SecurityUtility.createNettySslContextForClient(this.sslProvider, this.tlsAllowInsecureConnection, - authData.getTlsTrustStoreStream(), (X509Certificate[]) authData.getTlsCertificates(), - authData.getTlsPrivateKey(), this.ciphers, this.protocols); - } else { - this.sslNettyContext = - SecurityUtility.createNettySslContextForClient(this.sslProvider, this.tlsAllowInsecureConnection, - this.tlsTrustCertsFilePath.getFileName(), this.ciphers, this.protocols); - } - return this.sslNettyContext; - } - - @Override - public SslContext getSslContext() { - return this.sslNettyContext; - } - - @Override - public boolean needUpdate() { - return tlsTrustCertsFilePath.checkAndRefresh() || tlsCertsFilePath.checkAndRefresh() - || tlsPrivateKeyFilePath.checkAndRefresh(); - - } -} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyServerSslContextBuilder.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyServerSslContextBuilder.java deleted file mode 100644 index eda61be3f87c2..0000000000000 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyServerSslContextBuilder.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util; - -import io.netty.handler.ssl.SslContext; -import io.netty.handler.ssl.SslProvider; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.security.GeneralSecurityException; -import java.util.Set; -import javax.net.ssl.SSLException; - -/** - * SSL context builder for Netty Server side. - */ -public class NettyServerSslContextBuilder extends SslContextAutoRefreshBuilder { - private volatile SslContext sslNettyContext; - - protected final boolean tlsAllowInsecureConnection; - protected final FileModifiedTimeUpdater tlsTrustCertsFilePath, tlsCertificateFilePath, tlsKeyFilePath; - protected final Set tlsCiphers; - protected final Set tlsProtocols; - protected final boolean tlsRequireTrustedClientCertOnConnect; - protected final SslProvider sslProvider; - - public NettyServerSslContextBuilder(boolean allowInsecure, String trustCertsFilePath, - String certificateFilePath, - String keyFilePath, Set ciphers, Set protocols, - boolean requireTrustedClientCertOnConnect, - long delayInSeconds) { - this(null, allowInsecure, trustCertsFilePath, certificateFilePath, keyFilePath, ciphers, protocols, - requireTrustedClientCertOnConnect, delayInSeconds); - } - - public NettyServerSslContextBuilder(SslProvider sslProvider, boolean allowInsecure, String trustCertsFilePath, - String certificateFilePath, - String keyFilePath, Set ciphers, Set protocols, - boolean requireTrustedClientCertOnConnect, - long delayInSeconds) { - super(delayInSeconds); - this.tlsAllowInsecureConnection = allowInsecure; - this.tlsTrustCertsFilePath = new FileModifiedTimeUpdater(trustCertsFilePath); - this.tlsCertificateFilePath = new FileModifiedTimeUpdater(certificateFilePath); - this.tlsKeyFilePath = new FileModifiedTimeUpdater(keyFilePath); - this.tlsCiphers = ciphers; - this.tlsProtocols = protocols; - this.tlsRequireTrustedClientCertOnConnect = requireTrustedClientCertOnConnect; - this.sslProvider = sslProvider; - } - - @Override - public synchronized SslContext update() - throws SSLException, FileNotFoundException, GeneralSecurityException, IOException { - this.sslNettyContext = - SecurityUtility.createNettySslContextForServer(this.sslProvider, tlsAllowInsecureConnection, - tlsTrustCertsFilePath.getFileName(), tlsCertificateFilePath.getFileName(), - tlsKeyFilePath.getFileName(), - tlsCiphers, tlsProtocols, tlsRequireTrustedClientCertOnConnect); - return this.sslNettyContext; - } - - @Override - public SslContext getSslContext() { - return this.sslNettyContext; - } - - @Override - public boolean needUpdate() { - return tlsTrustCertsFilePath.checkAndRefresh() - || tlsCertificateFilePath.checkAndRefresh() - || tlsKeyFilePath.checkAndRefresh(); - } -} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/PulsarSslConfiguration.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/PulsarSslConfiguration.java new file mode 100644 index 0000000000000..f71888009bf4c --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/PulsarSslConfiguration.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util; + +import io.swagger.annotations.ApiModelProperty; +import java.io.Serializable; +import java.util.Set; +import lombok.Builder; +import lombok.Getter; +import lombok.ToString; +import org.apache.pulsar.client.api.AuthenticationDataProvider; + +/** + * Pulsar SSL Configuration Object to be used by all Pulsar Server and Client Components. + */ +@Builder +@Getter +@ToString +public class PulsarSslConfiguration implements Serializable, Cloneable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty( + name = "tlsCiphers", + value = "TLS ciphers to be used", + required = true + ) + private Set tlsCiphers; + + @ApiModelProperty( + name = "tlsProtocols", + value = "TLS protocols to be used", + required = true + ) + private Set tlsProtocols; + + @ApiModelProperty( + name = "allowInsecureConnection", + value = "Insecure Connections are allowed", + required = true + ) + private boolean allowInsecureConnection; + + @ApiModelProperty( + name = "requireTrustedClientCertOnConnect", + value = "Require trusted client certificate on connect", + required = true + ) + private boolean requireTrustedClientCertOnConnect; + + @ApiModelProperty( + name = "authData", + value = "Authentication Data Provider utilized by the Client for identification" + ) + private AuthenticationDataProvider authData; + + @ApiModelProperty( + name = "tlsCustomParams", + value = "Custom Parameters required by Pulsar SSL factory plugins" + ) + private String tlsCustomParams; + + @ApiModelProperty( + name = "tlsProvider", + value = "TLS Provider to be used" + ) + private String tlsProvider; + + @ApiModelProperty( + name = "tlsTrustStoreType", + value = "TLS Trust Store Type to be used" + ) + private String tlsTrustStoreType; + + @ApiModelProperty( + name = "tlsTrustStorePath", + value = "TLS Trust Store Path" + ) + private String tlsTrustStorePath; + + @ApiModelProperty( + name = "tlsTrustStorePassword", + value = "TLS Trust Store Password" + ) + private String tlsTrustStorePassword; + + @ApiModelProperty( + name = "tlsTrustCertsFilePath", + value = " TLS Trust certificates file path" + ) + private String tlsTrustCertsFilePath; + + @ApiModelProperty( + name = "tlsCertificateFilePath", + value = "Path for the TLS Certificate file" + ) + private String tlsCertificateFilePath; + + @ApiModelProperty( + name = "tlsKeyFilePath", + value = "Path for TLS Private key file" + ) + private String tlsKeyFilePath; + + @ApiModelProperty( + name = "tlsKeyStoreType", + value = "TLS Key Store Type to be used" + ) + private String tlsKeyStoreType; + + @ApiModelProperty( + name = "tlsKeyStorePath", + value = "TLS Key Store Path" + ) + private String tlsKeyStorePath; + + @ApiModelProperty( + name = "tlsKeyStorePassword", + value = "TLS Key Store Password" + ) + private String tlsKeyStorePassword; + + @ApiModelProperty( + name = "isTlsEnabledWithKeystore", + value = "TLS configuration enabled with key store configs" + ) + private boolean tlsEnabledWithKeystore; + + @ApiModelProperty( + name = "isServerMode", + value = "Is the SSL Configuration for a Server or Client", + required = true + ) + private boolean serverMode; + + @ApiModelProperty( + name = "isHttps", + value = "Is the SSL Configuration for a Http client or Server" + ) + private boolean isHttps; + + @Override + public PulsarSslConfiguration clone() { + try { + return (PulsarSslConfiguration) super.clone(); + } catch (CloneNotSupportedException e) { + throw new RuntimeException("Failed to clone PulsarSslConfiguration", e); + } + } + +} \ No newline at end of file diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/PulsarSslFactory.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/PulsarSslFactory.java new file mode 100644 index 0000000000000..bccbbbe5b2516 --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/PulsarSslFactory.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util; + +import io.netty.buffer.ByteBufAllocator; +import io.netty.handler.ssl.SslContext; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; + +/** + * Factory for generating SSL Context and SSL Engine using {@link PulsarSslConfiguration}. + */ +public interface PulsarSslFactory extends AutoCloseable { + + /** + * Initializes the PulsarSslFactory. + * @param config {@link PulsarSslConfiguration} object required for initialization + */ + void initialize(PulsarSslConfiguration config); + + /** + * Creates a Client {@link SSLEngine} utilizing {@link ByteBufAllocator} object, the peer hostname, peer port and + * {@link PulsarSslConfiguration} object provided during initialization. + * + * @param buf The ByteBufAllocator required for netty connections. This can be passed as {@code null} if utilized + * for web connections. + * @param peerHost the name of the peer host + * @param peerPort the port number of the peer + * @return {@link SSLEngine} + */ + SSLEngine createClientSslEngine(ByteBufAllocator buf, String peerHost, int peerPort); + + /** + * Creates a Server {@link SSLEngine} utilizing the {@link ByteBufAllocator} object and + * {@link PulsarSslConfiguration} object provided during initialization. + * + * @param buf The ByteBufAllocator required for netty connections. This can be passed as {@code null} if utilized + * for web connections. + * @return {@link SSLEngine} + */ + SSLEngine createServerSslEngine(ByteBufAllocator buf); + + /** + * Returns a boolean value indicating {@link SSLContext} or {@link SslContext} should be refreshed. + * + * @return {@code true} if {@link SSLContext} or {@link SslContext} should be refreshed. + */ + boolean needsUpdate(); + + /** + * Update the internal {@link SSLContext} or {@link SslContext}. + * @throws Exception if there are any issues generating the new {@link SSLContext} or {@link SslContext} + */ + default void update() throws Exception { + if (this.needsUpdate()) { + this.createInternalSslContext(); + } + } + + /** + * Creates the following: + * 1. {@link SslContext} if netty connections are being created for Non-Keystore based TLS configurations. + * 2. {@link SSLContext} if netty connections are being created for Keystore based TLS configurations. It will + * also create it for all web connections irrespective of it being Keystore or Non-Keystore based TLS + * configurations. + * + * @throws Exception if there are any issues creating the new {@link SSLContext} or {@link SslContext} + */ + void createInternalSslContext() throws Exception; + + /** + * Get the internally stored {@link SSLContext}. It will be used in the following scenarios: + * 1. Netty connection creations for keystore based TLS configurations + * 2. All Web connections + * + * @return {@link SSLContext} + * @throws RuntimeException if the {@link SSLContext} object has not yet been initialized. + */ + SSLContext getInternalSslContext(); + + /** + * Get the internally stored {@link SslContext}. It will be used to create Netty Connections for non-keystore based + * tls configurations. + * + * @return {@link SslContext} + * @throws RuntimeException if the {@link SslContext} object has not yet been initialized. + */ + SslContext getInternalNettySslContext(); + +} \ No newline at end of file diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/RateLimiter.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/RateLimiter.java deleted file mode 100644 index 4ecb29b2462cc..0000000000000 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/RateLimiter.java +++ /dev/null @@ -1,286 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util; - -import static com.google.common.base.Preconditions.checkArgument; -import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; -import com.google.common.base.MoreObjects; -import io.netty.util.concurrent.DefaultThreadFactory; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; -import lombok.Builder; - -/** - * A Rate Limiter that distributes permits at a configurable rate. Each {@link #acquire()} blocks if necessary until a - * permit is available, and then takes it. Each {@link #tryAcquire()} tries to acquire permits from available permits, - * it returns true if it succeed else returns false. Rate limiter release configured permits at every configured rate - * time, so, on next ticket new fresh permits will be available. - * - *

      For example: if RateLimiter is configured to release 10 permits at every 1 second then RateLimiter will allow to - * acquire 10 permits at any time with in that 1 second. - * - *

      Comparison with other RateLimiter such as {@link com.google.common.util.concurrent.RateLimiter} - *

        - *
      • Per second rate-limiting: Per second rate-limiting not satisfied by Guava-RateLimiter
        • - *
      • Guava RateLimiter: For X permits: it releases X/1000 permits every msec. therefore, - * for permits=2/sec => it release 1st permit on first 500msec and 2nd permit on next 500ms. therefore, - * if 2 request comes with in 500msec duration then 2nd request fails to acquire permit - * though we have configured 2 permits/second.
        • - *
      • RateLimiter: it releases X permits every second. so, in above usecase: - * if 2 requests comes at the same time then both will acquire the permit.
        • - *
      • Faster: RateLimiter is light-weight and faster than Guava-RateLimiter
        • - *
      - */ -public class RateLimiter implements AutoCloseable{ - private final ScheduledExecutorService executorService; - private long rateTime; - private TimeUnit timeUnit; - private final boolean externalExecutor; - private ScheduledFuture renewTask; - private volatile long permits; - private volatile long acquiredPermits; - private boolean isClosed; - // permitUpdate helps to update permit-rate at runtime - private Supplier permitUpdater; - private RateLimitFunction rateLimitFunction; - private boolean isDispatchOrPrecisePublishRateLimiter; - - @Builder - RateLimiter(final ScheduledExecutorService scheduledExecutorService, final long permits, final long rateTime, - final TimeUnit timeUnit, Supplier permitUpdater, boolean isDispatchOrPrecisePublishRateLimiter, - RateLimitFunction rateLimitFunction) { - checkArgument(permits > 0, "rate must be > 0"); - checkArgument(rateTime > 0, "Renew permit time must be > 0"); - - this.rateTime = rateTime; - this.timeUnit = timeUnit; - this.permits = permits; - this.permitUpdater = permitUpdater; - this.isDispatchOrPrecisePublishRateLimiter = isDispatchOrPrecisePublishRateLimiter; - - if (scheduledExecutorService != null) { - this.executorService = scheduledExecutorService; - this.externalExecutor = true; - } else { - final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1, - new DefaultThreadFactory("pulsar-rate-limiter")); - executor.setContinueExistingPeriodicTasksAfterShutdownPolicy(false); - executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); - this.executorService = executor; - this.externalExecutor = false; - } - - this.rateLimitFunction = rateLimitFunction; - - } - - // default values for Lombok generated builder class - public static class RateLimiterBuilder { - private long rateTime = 1; - private TimeUnit timeUnit = TimeUnit.SECONDS; - } - - @Override - public synchronized void close() { - if (!isClosed) { - if (!externalExecutor) { - executorService.shutdownNow(); - } - if (renewTask != null) { - renewTask.cancel(false); - } - isClosed = true; - // If there is a ratelimit function registered, invoke it to unblock. - if (rateLimitFunction != null) { - rateLimitFunction.apply(); - } - } - } - - public synchronized boolean isClosed() { - return isClosed; - } - - /** - * Acquires the given number of permits from this {@code RateLimiter}, blocking until the request be granted. - * - *

      This method is equivalent to {@code acquire(1)}. - */ - public synchronized void acquire() throws InterruptedException { - acquire(1); - } - - /** - * Acquires the given number of permits from this {@code RateLimiter}, blocking until the request be granted. - * - * @param acquirePermit - * the number of permits to acquire - */ - public synchronized void acquire(long acquirePermit) throws InterruptedException { - checkArgument(!isClosed(), "Rate limiter is already shutdown"); - checkArgument(acquirePermit <= this.permits, - "acquiring permits must be less or equal than initialized rate =" + this.permits); - - // lazy init and start task only once application start using it - if (renewTask == null) { - renewTask = createTask(); - } - - boolean canAcquire = false; - do { - canAcquire = acquirePermit < 0 || acquiredPermits < this.permits; - if (!canAcquire) { - wait(); - } else { - acquiredPermits += acquirePermit; - } - } while (!canAcquire); - } - - /** - * Acquires permits from this {@link RateLimiter} if it can be acquired immediately without delay. - * - *

      This method is equivalent to {@code tryAcquire(1)}. - * - * @return {@code true} if the permits were acquired, {@code false} otherwise - */ - public synchronized boolean tryAcquire() { - return tryAcquire(1); - } - - /** - * Acquires permits from this {@link RateLimiter} if it can be acquired immediately without delay. - * - * @param acquirePermit - * the number of permits to acquire - * @return {@code true} if the permits were acquired, {@code false} otherwise - */ - public synchronized boolean tryAcquire(long acquirePermit) { - checkArgument(!isClosed(), "Rate limiter is already shutdown"); - // lazy init and start task only once application start using it - if (renewTask == null) { - renewTask = createTask(); - } - - boolean canAcquire = acquirePermit < 0 || acquiredPermits < this.permits; - if (isDispatchOrPrecisePublishRateLimiter) { - // for dispatch rate limiter just add acquirePermit - acquiredPermits += acquirePermit; - - // we want to back-pressure from the current state of the rateLimiter therefore we should check if there - // are any available premits again - canAcquire = acquirePermit < 0 || acquiredPermits < this.permits; - } else { - // acquired-permits can't be larger than the rate - if (acquirePermit + acquiredPermits > this.permits) { - return false; - } - - if (canAcquire) { - acquiredPermits += acquirePermit; - } - } - - return canAcquire; - } - - /** - * Return available permits for this {@link RateLimiter}. - * - * @return returns 0 if permits is not available - */ - public long getAvailablePermits() { - return Math.max(0, this.permits - this.acquiredPermits); - } - - /** - * Resets new rate by configuring new value for permits per configured rate-period. - * - * @param permits - */ - public synchronized void setRate(long permits) { - this.permits = permits; - } - - /** - * Resets new rate with new permits and rate-time. - * - * @param permits - * @param rateTime - * @param timeUnit - * @param permitUpdaterByte - */ - public synchronized void setRate(long permits, long rateTime, TimeUnit timeUnit, Supplier permitUpdaterByte) { - if (renewTask != null) { - renewTask.cancel(false); - } - this.permits = permits; - this.rateTime = rateTime; - this.timeUnit = timeUnit; - this.permitUpdater = permitUpdaterByte; - this.renewTask = createTask(); - } - - /** - * Returns configured permit rate per pre-configured rate-period. - * - * @return rate - */ - public synchronized long getRate() { - return this.permits; - } - - public synchronized long getRateTime() { - return this.rateTime; - } - - public synchronized TimeUnit getRateTimeUnit() { - return this.timeUnit; - } - - protected ScheduledFuture createTask() { - return executorService.scheduleAtFixedRate(catchingAndLoggingThrowables(this::renew), this.rateTime, - this.rateTime, this.timeUnit); - } - - synchronized void renew() { - acquiredPermits = isDispatchOrPrecisePublishRateLimiter ? Math.max(0, acquiredPermits - permits) : 0; - if (permitUpdater != null) { - long newPermitRate = permitUpdater.get(); - if (newPermitRate > 0) { - setRate(newPermitRate); - } - } - // release the back-pressure by applying the rateLimitFunction only when there are available permits - if (rateLimitFunction != null && this.getAvailablePermits() > 0) { - rateLimitFunction.apply(); - } - notifyAll(); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this).add("rateTime", rateTime).add("permits", permits) - .add("acquiredPermits", acquiredPermits).toString(); - } - -} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java index 12ab9ae0b0bc9..2b7b1a984634f 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java @@ -48,10 +48,14 @@ import java.security.cert.CertificateException; import java.security.cert.CertificateFactory; import java.security.cert.X509Certificate; +import java.security.spec.InvalidKeySpecException; import java.security.spec.KeySpec; import java.security.spec.PKCS8EncodedKeySpec; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Base64; import java.util.Collection; +import java.util.List; import java.util.Set; import java.util.concurrent.ScheduledExecutorService; import javax.net.ssl.HostnameVerifier; @@ -79,6 +83,10 @@ public class SecurityUtility { public static final String BC_NON_FIPS_PROVIDER_CLASS = "org.bouncycastle.jce.provider.BouncyCastleProvider"; public static final String CONSCRYPT_PROVIDER_CLASS = "org.conscrypt.OpenSSLProvider"; public static final Provider CONSCRYPT_PROVIDER = loadConscryptProvider(); + private static final List KEY_FACTORIES = Arrays.asList( + createKeyFactory("RSA"), + createKeyFactory("EC") + ); // Security.getProvider("BC") / Security.getProvider("BCFIPS"). // also used to get Factories. e.g. CertificateFactory.getInstance("X.509", "BCFIPS") @@ -124,11 +132,14 @@ private static Provider loadConscryptProvider() { conscryptClazz = Class.forName("org.conscrypt.Conscrypt"); conscryptClazz.getMethod("checkAvailability").invoke(null); } catch (Throwable e) { - if (e.getCause() instanceof UnsatisfiedLinkError) { - log.warn("Conscrypt isn't available for {} {}. Using JDK default security provider.", + if (e instanceof ClassNotFoundException) { + log.debug("Conscrypt isn't available in the classpath. Using JDK default security provider."); + } else if (e.getCause() instanceof UnsatisfiedLinkError) { + log.debug("Conscrypt isn't available for {} {}. Using JDK default security provider.", System.getProperty("os.name"), System.getProperty("os.arch")); } else { - log.warn("Conscrypt isn't available. Using JDK default security provider.", e); + log.debug("Conscrypt isn't available. Using JDK default security provider." + + " Cause : {}, Reason : {}", e.getCause(), e.getMessage()); } return null; } @@ -137,7 +148,7 @@ private static Provider loadConscryptProvider() { try { provider = (Provider) Class.forName(CONSCRYPT_PROVIDER_CLASS).getDeclaredConstructor().newInstance(); } catch (ReflectiveOperationException e) { - log.warn("Unable to get security provider for class {}", CONSCRYPT_PROVIDER_CLASS, e); + log.debug("Unable to get security provider for class {}", CONSCRYPT_PROVIDER_CLASS, e); return null; } @@ -509,15 +520,21 @@ public static PrivateKey loadPrivateKeyFromPemStream(InputStream inStream) throw while ((currentLine = reader.readLine()) != null && !currentLine.startsWith("-----END")) { sb.append(currentLine); } - - KeyFactory kf = KeyFactory.getInstance("RSA"); - KeySpec keySpec = new PKCS8EncodedKeySpec(Base64.getDecoder().decode(sb.toString())); - privateKey = kf.generatePrivate(keySpec); - } catch (GeneralSecurityException | IOException e) { + final KeySpec keySpec = new PKCS8EncodedKeySpec(Base64.getDecoder().decode(sb.toString())); + final List failedAlgorithm = new ArrayList<>(KEY_FACTORIES.size()); + for (KeyFactory kf : KEY_FACTORIES) { + try { + return kf.generatePrivate(keySpec); + } catch (InvalidKeySpecException ex) { + failedAlgorithm.add(kf.getAlgorithm()); + } + } + throw new KeyManagementException("The private key algorithm is not supported. attempted: " + + StringUtils.join(failedAlgorithm, ",")); + } catch (IOException e) { throw new KeyManagementException("Private key loading error", e); } - return privateKey; } private static void setupTrustCerts(SslContextBuilder builder, boolean allowInsecureConnection, @@ -578,4 +595,12 @@ public static Provider resolveProvider(String providerName) throws NoSuchAlgorit return provider; } + + private static KeyFactory createKeyFactory(String algorithm) { + try { + return KeyFactory.getInstance(algorithm); + } catch (Exception e) { + throw new IllegalArgumentException(String.format("Illegal key factory algorithm " + algorithm), e); + } + } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/ShutdownUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/ShutdownUtil.java index b461cfab0d1bc..ff12484707123 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/ShutdownUtil.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/ShutdownUtil.java @@ -47,8 +47,11 @@ public class ShutdownUtil { * @see Runtime#halt(int) */ public static void triggerImmediateForcefulShutdown(int status) { + triggerImmediateForcefulShutdown(status, true); + } + public static void triggerImmediateForcefulShutdown(int status, boolean logging) { try { - if (status != 0) { + if (status != 0 && logging) { log.warn("Triggering immediate shutdown of current process with status {}", status, new Exception("Stacktrace for immediate shutdown")); } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SimpleTextOutputStream.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SimpleTextOutputStream.java index c8c639606aa3e..d3f319bd958ba 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SimpleTextOutputStream.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SimpleTextOutputStream.java @@ -20,6 +20,7 @@ import io.netty.buffer.ByteBuf; import io.netty.util.CharsetUtil; +import java.nio.CharBuffer; /** * Format strings and numbers into a ByteBuf without any memory allocation. @@ -28,6 +29,7 @@ public class SimpleTextOutputStream { private final ByteBuf buffer; private static final char[] hexChars = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; + private final CharBuffer singleCharBuffer = CharBuffer.allocate(1); public SimpleTextOutputStream(ByteBuf buffer) { this.buffer = buffer; @@ -44,7 +46,13 @@ public SimpleTextOutputStream write(byte[] a, int offset, int len) { } public SimpleTextOutputStream write(char c) { - write(String.valueOf(c)); + // In UTF-8, any character from U+0000 to U+007F is encoded in one byte + if (c <= '\u007F') { + buffer.writeByte((byte) c); + return this; + } + singleCharBuffer.put(0, c); + buffer.writeCharSequence(singleCharBuffer, CharsetUtil.UTF_8); return this; } @@ -57,6 +65,15 @@ public SimpleTextOutputStream write(String s) { return this; } + public SimpleTextOutputStream write(CharSequence s) { + if (s == null) { + return this; + } + + buffer.writeCharSequence(s, CharsetUtil.UTF_8); + return this; + } + public SimpleTextOutputStream write(Number n) { if (n instanceof Integer) { return write(n.intValue()); @@ -136,4 +153,8 @@ public void write(ByteBuf byteBuf) { public ByteBuf getBuffer() { return buffer; } + + public void writeByte(int b) { + buffer.writeByte(b); + } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SslContextAutoRefreshBuilder.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SslContextAutoRefreshBuilder.java deleted file mode 100644 index 8c8f580046448..0000000000000 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SslContextAutoRefreshBuilder.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util; - -import java.io.IOException; -import java.security.GeneralSecurityException; -import java.util.concurrent.TimeUnit; -import lombok.extern.slf4j.Slf4j; - -/** - * Auto refresher and builder of SSLContext. - * - * @param - * type of SSLContext - */ -@Slf4j -public abstract class SslContextAutoRefreshBuilder { - protected final long refreshTime; - protected long lastRefreshTime; - - public SslContextAutoRefreshBuilder( - long certRefreshInSec) { - this.refreshTime = TimeUnit.SECONDS.toMillis(certRefreshInSec); - this.lastRefreshTime = -1; - - if (log.isDebugEnabled()) { - log.debug("Certs will be refreshed every {} seconds", certRefreshInSec); - } - } - - /** - * updates and returns cached SSLContext. - * - * @return - * @throws GeneralSecurityException - * @throws IOException - */ - protected abstract T update() throws GeneralSecurityException, IOException; - - /** - * Returns cached SSLContext. - * - * @return - */ - protected abstract T getSslContext(); - - /** - * Returns whether the key files modified after a refresh time, and context need update. - * - * @return true if files modified - */ - protected abstract boolean needUpdate(); - - /** - * It updates SSLContext at every configured refresh time and returns updated SSLContext. - * - * @return - */ - public T get() { - T ctx = getSslContext(); - if (ctx == null) { - try { - update(); - lastRefreshTime = System.currentTimeMillis(); - return getSslContext(); - } catch (GeneralSecurityException | IOException e) { - log.error("Exception while trying to refresh ssl Context {}", e.getMessage(), e); - } - } else { - long now = System.currentTimeMillis(); - if (refreshTime <= 0 || now > (lastRefreshTime + refreshTime)) { - if (needUpdate()) { - try { - ctx = update(); - lastRefreshTime = now; - } catch (GeneralSecurityException | IOException e) { - log.error("Exception while trying to refresh ssl Context {} ", e.getMessage(), e); - } - } - } - } - return ctx; - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorId.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/StringInterner.java similarity index 54% rename from pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorId.java rename to pulsar-common/src/main/java/org/apache/pulsar/common/util/StringInterner.java index c74b8f650f3eb..3f6b1c453cdbc 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorId.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/StringInterner.java @@ -16,41 +16,31 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.sql.presto; +package org.apache.pulsar.common.util; -import static java.util.Objects.requireNonNull; +import com.google.common.collect.Interner; +import com.google.common.collect.Interners; /** - * Unique identifier of a connector. + * Deduplicates String instances by interning them using Guava's Interner + * which is more efficient than String.intern(). */ -public class PulsarConnectorId { - private final String id; +public class StringInterner { + private static final StringInterner INSTANCE = new StringInterner(); + private final Interner interner; - public PulsarConnectorId(String id) { - this.id = requireNonNull(id, "id is null"); + public static String intern(String sample) { + return INSTANCE.doIntern(sample); } - @Override - public String toString() { - return id; + private StringInterner() { + this.interner = Interners.newWeakInterner(); } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; + String doIntern(String sample) { + if (sample == null) { + return null; } - if (o == null || getClass() != o.getClass()) { - return false; - } - - PulsarConnectorId that = (PulsarConnectorId) o; - - return id.equals(that.id); - } - - @Override - public int hashCode() { - return id.hashCode(); + return interner.intern(sample); } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/ThreadDumpUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/ThreadDumpUtil.java index 2916269209d86..ca9ca0794e05f 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/ThreadDumpUtil.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/ThreadDumpUtil.java @@ -25,7 +25,7 @@ import java.lang.management.MonitorInfo; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; -import java.time.LocalDateTime; +import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.Map; import javax.management.JMException; @@ -65,7 +65,7 @@ static String buildThreadDump() { // fallback to using JMX for creating the thread dump StringBuilder dump = new StringBuilder(); - dump.append(String.format("Timestamp: %s", DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(LocalDateTime.now()))); + dump.append(String.format("Timestamp: %s", DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now()))); dump.append("\n\n"); Map stackTraces = Thread.getAllStackTraces(); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/BitSetRecyclable.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/BitSetRecyclable.java index 12ce7eb74c72b..b801d5f2b05a1 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/BitSetRecyclable.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/BitSetRecyclable.java @@ -216,6 +216,14 @@ public static BitSetRecyclable valueOf(byte[] bytes) { return BitSetRecyclable.valueOf(ByteBuffer.wrap(bytes)); } + /** + * Copy a BitSetRecyclable. + */ + public static BitSetRecyclable valueOf(BitSetRecyclable src) { + // The internal implementation will do the array-copy. + return valueOf(src.words); + } + /** * Returns a new bit set containing all the bits in the given byte * buffer between its position and limit. diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentBitSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentBitSet.java index 23842fe5b556c..a37628cb300b8 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentBitSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentBitSet.java @@ -20,12 +20,13 @@ import java.util.BitSet; import java.util.concurrent.locks.StampedLock; -import lombok.EqualsAndHashCode; +import java.util.stream.IntStream; /** - * Safe multithreaded version of {@code BitSet}. + * A {@code BitSet} that is protected by a {@code StampedLock} to provide thread-safe access. + * The {@link #length()} method is not thread safe and is not overridden because StampedLock is not reentrant. + * Use the {@link #safeLength()} method to get the length of the bit set in a thread-safe manner. */ -@EqualsAndHashCode(callSuper = true) public class ConcurrentBitSet extends BitSet { private static final long serialVersionUID = 1L; @@ -39,10 +40,8 @@ public ConcurrentBitSet() { * Creates a bit set whose initial size is large enough to explicitly represent bits with indices in the range * {@code 0} through {@code nbits-1}. All bits are initially {@code false}. * - * @param nbits - * the initial size of the bit set - * @throws NegativeArraySizeException - * if the specified initial size is negative + * @param nbits the initial size of the bit set + * @throws NegativeArraySizeException if the specified initial size is negative */ public ConcurrentBitSet(int nbits) { super(nbits); @@ -65,105 +64,405 @@ public boolean get(int bitIndex) { @Override public void set(int bitIndex) { + long stamp = rwLock.writeLock(); + try { + super.set(bitIndex); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public void clear(int bitIndex) { + long stamp = rwLock.writeLock(); + try { + super.clear(bitIndex); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public void set(int fromIndex, int toIndex) { + long stamp = rwLock.writeLock(); + try { + super.set(fromIndex, toIndex); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public void clear(int fromIndex, int toIndex) { + long stamp = rwLock.writeLock(); + try { + super.clear(fromIndex, toIndex); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public void clear() { + long stamp = rwLock.writeLock(); + try { + super.clear(); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public int nextSetBit(int fromIndex) { long stamp = rwLock.tryOptimisticRead(); - super.set(bitIndex); + int nextSetBit = super.nextSetBit(fromIndex); if (!rwLock.validate(stamp)) { + // Fallback to read lock stamp = rwLock.readLock(); try { - super.set(bitIndex); + nextSetBit = super.nextSetBit(fromIndex); } finally { rwLock.unlockRead(stamp); } } + return nextSetBit; } @Override - public void set(int fromIndex, int toIndex) { + public int nextClearBit(int fromIndex) { long stamp = rwLock.tryOptimisticRead(); - super.set(fromIndex, toIndex); + int nextClearBit = super.nextClearBit(fromIndex); if (!rwLock.validate(stamp)) { + // Fallback to read lock stamp = rwLock.readLock(); try { - super.set(fromIndex, toIndex); + nextClearBit = super.nextClearBit(fromIndex); } finally { rwLock.unlockRead(stamp); } } + return nextClearBit; } @Override - public int nextSetBit(int fromIndex) { + public int previousSetBit(int fromIndex) { long stamp = rwLock.tryOptimisticRead(); - int bit = super.nextSetBit(fromIndex); + int previousSetBit = super.previousSetBit(fromIndex); if (!rwLock.validate(stamp)) { + // Fallback to read lock stamp = rwLock.readLock(); try { - bit = super.nextSetBit(fromIndex); + previousSetBit = super.previousSetBit(fromIndex); } finally { rwLock.unlockRead(stamp); } } - return bit; + return previousSetBit; } @Override - public int nextClearBit(int fromIndex) { + public int previousClearBit(int fromIndex) { long stamp = rwLock.tryOptimisticRead(); - int bit = super.nextClearBit(fromIndex); + int previousClearBit = super.previousClearBit(fromIndex); if (!rwLock.validate(stamp)) { + // Fallback to read lock stamp = rwLock.readLock(); try { - bit = super.nextClearBit(fromIndex); + previousClearBit = super.previousClearBit(fromIndex); } finally { rwLock.unlockRead(stamp); } } - return bit; + return previousClearBit; } @Override - public int previousSetBit(int fromIndex) { + public boolean isEmpty() { long stamp = rwLock.tryOptimisticRead(); - int bit = super.previousSetBit(fromIndex); + boolean isEmpty = super.isEmpty(); if (!rwLock.validate(stamp)) { + // Fallback to read lock stamp = rwLock.readLock(); try { - bit = super.previousSetBit(fromIndex); + isEmpty = super.isEmpty(); } finally { rwLock.unlockRead(stamp); } } - return bit; + return isEmpty; } @Override - public int previousClearBit(int fromIndex) { + public int cardinality() { long stamp = rwLock.tryOptimisticRead(); - int bit = super.previousClearBit(fromIndex); + int cardinality = super.cardinality(); if (!rwLock.validate(stamp)) { + // Fallback to read lock stamp = rwLock.readLock(); try { - bit = super.previousClearBit(fromIndex); + cardinality = super.cardinality(); } finally { rwLock.unlockRead(stamp); } } - return bit; + return cardinality; } @Override - public boolean isEmpty() { + public int size() { long stamp = rwLock.tryOptimisticRead(); - boolean isEmpty = super.isEmpty(); + int size = super.size(); if (!rwLock.validate(stamp)) { // Fallback to read lock stamp = rwLock.readLock(); try { - isEmpty = super.isEmpty(); + size = super.size(); } finally { rwLock.unlockRead(stamp); } } - return isEmpty; + return size; + } + + @Override + public byte[] toByteArray() { + long stamp = rwLock.tryOptimisticRead(); + byte[] byteArray = super.toByteArray(); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + byteArray = super.toByteArray(); + } finally { + rwLock.unlockRead(stamp); + } + } + return byteArray; + } + + @Override + public long[] toLongArray() { + long stamp = rwLock.tryOptimisticRead(); + long[] longArray = super.toLongArray(); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + longArray = super.toLongArray(); + } finally { + rwLock.unlockRead(stamp); + } + } + return longArray; + } + + @Override + public void flip(int bitIndex) { + long stamp = rwLock.writeLock(); + try { + super.flip(bitIndex); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public void flip(int fromIndex, int toIndex) { + long stamp = rwLock.writeLock(); + try { + super.flip(fromIndex, toIndex); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public void set(int bitIndex, boolean value) { + long stamp = rwLock.writeLock(); + try { + super.set(bitIndex, value); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public void set(int fromIndex, int toIndex, boolean value) { + long stamp = rwLock.writeLock(); + try { + super.set(fromIndex, toIndex, value); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public BitSet get(int fromIndex, int toIndex) { + long stamp = rwLock.tryOptimisticRead(); + BitSet bitSet = super.get(fromIndex, toIndex); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + bitSet = super.get(fromIndex, toIndex); + } finally { + rwLock.unlockRead(stamp); + } + } + return bitSet; + } + + /** + * Thread-safe version of {@code length()}. + * StampedLock is not reentrant and that's why the length() method is not overridden. Overriding length() method + * would require to use a reentrant lock which would be less performant. + * + * @return length of the bit set + */ + public int safeLength() { + long stamp = rwLock.tryOptimisticRead(); + int length = super.length(); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + length = super.length(); + } finally { + rwLock.unlockRead(stamp); + } + } + return length; + } + + @Override + public boolean intersects(BitSet set) { + long stamp = rwLock.writeLock(); + try { + return super.intersects(set); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public void and(BitSet set) { + long stamp = rwLock.writeLock(); + try { + super.and(set); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public void or(BitSet set) { + long stamp = rwLock.writeLock(); + try { + super.or(set); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public void xor(BitSet set) { + long stamp = rwLock.writeLock(); + try { + super.xor(set); + } finally { + rwLock.unlockWrite(stamp); + } + } + + @Override + public void andNot(BitSet set) { + long stamp = rwLock.writeLock(); + try { + super.andNot(set); + } finally { + rwLock.unlockWrite(stamp); + } + } + + /** + * Returns the clone of the internal wrapped {@code BitSet}. + * This won't be a clone of the {@code ConcurrentBitSet} object. + * + * @return a clone of the internal wrapped {@code BitSet} + */ + @Override + public Object clone() { + long stamp = rwLock.tryOptimisticRead(); + BitSet clonedBitSet = (BitSet) super.clone(); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + clonedBitSet = (BitSet) super.clone(); + } finally { + rwLock.unlockRead(stamp); + } + } + return clonedBitSet; + } + + @Override + public String toString() { + long stamp = rwLock.tryOptimisticRead(); + String str = super.toString(); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + str = super.toString(); + } finally { + rwLock.unlockRead(stamp); + } + } + return str; + } + + /** + * This operation is not supported on {@code ConcurrentBitSet}. + */ + @Override + public IntStream stream() { + throw new UnsupportedOperationException("stream is not supported"); + } + + public boolean equals(final Object o) { + if (o == this) { + return true; + } + if (!(o instanceof ConcurrentBitSet)) { + return false; + } + long stamp = rwLock.tryOptimisticRead(); + boolean isEqual = super.equals(o); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + isEqual = super.equals(o); + } finally { + rwLock.unlockRead(stamp); + } + } + return isEqual; + } + + public int hashCode() { + long stamp = rwLock.tryOptimisticRead(); + int hashCode = super.hashCode(); + if (!rwLock.validate(stamp)) { + // Fallback to read lock + stamp = rwLock.readLock(); + try { + hashCode = super.hashCode(); + } finally { + rwLock.unlockRead(stamp); + } + } + return hashCode; } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java index 31b4cb7cbf152..b6408ee98192f 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java @@ -36,6 +36,19 @@ *

    • Open hash map with linear probing, no node allocations to store the values * * + * WARN: method forEach do not guarantee thread safety, nor do the keys and values method. + *
      + * The forEach method is specifically designed for single-threaded usage. When iterating over a map + * with concurrent writes, it becomes possible for new values to be either observed or not observed. + * There is no guarantee that if we write value1 and value2, and are able to see value2, then we will also see value1. + * In some cases, it is even possible to encounter two mappings with the same key, + * leading the keys method to return a List containing two identical keys. + * + *
      + * It is crucial to understand that the results obtained from aggregate status methods such as keys and values + * are typically reliable only when the map is not undergoing concurrent updates from other threads. + * When concurrent updates are involved, the results of these methods reflect transient states + * that may be suitable for monitoring or estimation purposes, but not for program control. * @param */ @SuppressWarnings("unchecked") @@ -237,6 +250,12 @@ public void clear() { } } + /** + * Iterate over all the entries in the map and apply the processor function to each of them. + *

      + * Warning: Do Not Guarantee Thread-Safety. + * @param processor the processor to apply to each entry + */ public void forEach(EntryProcessor processor) { for (int i = 0; i < sections.length; i++) { sections[i].forEach(processor); @@ -306,16 +325,17 @@ private static final class Section extends StampedLock { } V get(long key, int keyHash) { - int bucket = keyHash; - long stamp = tryOptimisticRead(); boolean acquiredLock = false; + // add local variable here, so OutOfBound won't happen + long[] keys = this.keys; + V[] values = this.values; + // calculate table.length as capacity to avoid rehash changing capacity + int bucket = signSafeMod(keyHash, values.length); + try { while (true) { - int capacity = this.capacity; - bucket = signSafeMod(bucket, capacity); - // First try optimistic locking long storedKey = keys[bucket]; V storedValue = values[bucket]; @@ -333,16 +353,15 @@ V get(long key, int keyHash) { if (!acquiredLock) { stamp = readLock(); acquiredLock = true; + + // update local variable + keys = this.keys; + values = this.values; + bucket = signSafeMod(keyHash, values.length); storedKey = keys[bucket]; storedValue = values[bucket]; } - if (capacity != this.capacity) { - // There has been a rehashing. We need to restart the search - bucket = keyHash; - continue; - } - if (storedKey == key) { return storedValue != DeletedValue ? storedValue : null; } else if (storedValue == EmptyValue) { @@ -350,8 +369,7 @@ V get(long key, int keyHash) { return null; } } - - ++bucket; + bucket = (bucket + 1) & (values.length - 1); } } finally { if (acquiredLock) { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMap.java index c0ccad9b73d5b..cfa414278ccbb 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMap.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMap.java @@ -36,6 +36,20 @@ * no node allocations are required to store the keys and values, and no boxing is required. * *

      Keys MUST be >= 0. + *
      + * WARN: method forEach do not guarantee thread safety, nor do the keys, values and asMap method. + *
      + * The forEach method is specifically designed for single-threaded usage. When iterating over a map + * with concurrent writes, it becomes possible for new values to be either observed or not observed. + * There is no guarantee that if we write value1 and value2, and are able to see value2, then we will also see value1. + * In some cases, it is even possible to encounter two mappings with the same key, + * leading the keys method to return a List containing two identical keys. + * + *
      + * It is crucial to understand that the results obtained from aggregate status methods such as keys, values, and asMap + * are typically reliable only when the map is not undergoing concurrent updates from other threads. + * When concurrent updates are involved, the results of these methods reflect transient states + * that may be suitable for monitoring or estimation purposes, but not for program control. */ public class ConcurrentLongLongPairHashMap { @@ -254,6 +268,12 @@ public void clear() { } } + /** + * Iterate over all the entries in the map and apply the processor function to each of them. + *

      + * Warning: Do Not Guarantee Thread-Safety. + * @param processor the processor to process the elements. + */ public void forEach(BiConsumerLongPair processor) { for (Section s : sections) { s.forEach(processor); @@ -284,6 +304,9 @@ public Map asMap() { // A section is a portion of the hash map that is covered by a single @SuppressWarnings("serial") private static final class Section extends StampedLock { + // Each item take up 4 continuous array space. + private static final int ITEM_SIZE = 4; + // Keys and values are stored interleaved in the table array private volatile long[] table; @@ -306,7 +329,7 @@ private static final class Section extends StampedLock { float expandFactor, float shrinkFactor) { this.capacity = alignToPowerOfTwo(capacity); this.initCapacity = this.capacity; - this.table = new long[4 * this.capacity]; + this.table = new long[ITEM_SIZE * this.capacity]; this.size = 0; this.usedBuckets = 0; this.autoShrink = autoShrink; @@ -322,7 +345,10 @@ private static final class Section extends StampedLock { LongPair get(long key1, long key2, int keyHash) { long stamp = tryOptimisticRead(); boolean acquiredLock = false; - int bucket = signSafeMod(keyHash, capacity); + // add local variable here, so OutOfBound won't happen + long[] table = this.table; + // calculate table.length / 4 as capacity to avoid rehash changing capacity + int bucket = signSafeMod(keyHash, table.length / ITEM_SIZE); try { while (true) { @@ -345,8 +371,9 @@ LongPair get(long key1, long key2, int keyHash) { if (!acquiredLock) { stamp = readLock(); acquiredLock = true; - - bucket = signSafeMod(keyHash, capacity); + // update local variable + table = this.table; + bucket = signSafeMod(keyHash, table.length / ITEM_SIZE); storedKey1 = table[bucket]; storedKey2 = table[bucket + 1]; storedValue1 = table[bucket + 2]; @@ -361,7 +388,7 @@ LongPair get(long key1, long key2, int keyHash) { } } - bucket = (bucket + 4) & (table.length - 1); + bucket = (bucket + ITEM_SIZE) & (table.length - 1); } } finally { if (acquiredLock) { @@ -413,7 +440,7 @@ boolean put(long key1, long key2, long value1, long value2, int keyHash, boolean } } - bucket = (bucket + 4) & (table.length - 1); + bucket = (bucket + ITEM_SIZE) & (table.length - 1); } } finally { if (usedBuckets > resizeThresholdUp) { @@ -454,7 +481,7 @@ private boolean remove(long key1, long key2, long value1, long value2, int keyHa return false; } - bucket = (bucket + 4) & (table.length - 1); + bucket = (bucket + ITEM_SIZE) & (table.length - 1); } } finally { @@ -480,7 +507,7 @@ private boolean remove(long key1, long key2, long value1, long value2, int keyHa } private void cleanBucket(int bucket) { - int nextInArray = (bucket + 4) & (table.length - 1); + int nextInArray = (bucket + ITEM_SIZE) & (table.length - 1); if (table[nextInArray] == EmptyKey) { table[bucket] = EmptyKey; table[bucket + 1] = EmptyKey; @@ -489,7 +516,7 @@ private void cleanBucket(int bucket) { --usedBuckets; // Cleanup all the buckets that were in `DeletedKey` state, so that we can reduce unnecessary expansions - bucket = (bucket - 4) & (table.length - 1); + bucket = (bucket - ITEM_SIZE) & (table.length - 1); while (table[bucket] == DeletedKey) { table[bucket] = EmptyKey; table[bucket + 1] = EmptyKey; @@ -497,7 +524,7 @@ private void cleanBucket(int bucket) { table[bucket + 3] = ValueNotFound; --usedBuckets; - bucket = (bucket - 4) & (table.length - 1); + bucket = (bucket - ITEM_SIZE) & (table.length - 1); } } else { table[bucket] = DeletedKey; @@ -540,7 +567,7 @@ public void forEach(BiConsumerLongPair processor) { } // Go through all the buckets for this section - for (int bucket = 0; bucket < table.length; bucket += 4) { + for (int bucket = 0; bucket < table.length; bucket += ITEM_SIZE) { long storedKey1 = table[bucket]; long storedKey2 = table[bucket + 1]; long storedValue1 = table[bucket + 2]; @@ -569,11 +596,11 @@ public void forEach(BiConsumerLongPair processor) { } private void rehash(int newCapacity) { - long[] newTable = new long[4 * newCapacity]; + long[] newTable = new long[ITEM_SIZE * newCapacity]; Arrays.fill(newTable, EmptyKey); // Re-hash table - for (int i = 0; i < table.length; i += 4) { + for (int i = 0; i < table.length; i += ITEM_SIZE) { long storedKey1 = table[i]; long storedKey2 = table[i + 1]; long storedValue1 = table[i + 2]; @@ -593,7 +620,7 @@ private void rehash(int newCapacity) { } private void shrinkToInitCapacity() { - long[] newTable = new long[4 * initCapacity]; + long[] newTable = new long[ITEM_SIZE * initCapacity]; Arrays.fill(newTable, EmptyKey); table = newTable; @@ -622,7 +649,7 @@ private static void insertKeyValueNoLock(long[] table, int capacity, long key1, return; } - bucket = (bucket + 4) & (table.length - 1); + bucket = (bucket + ITEM_SIZE) & (table.length - 1); } } } @@ -641,6 +668,8 @@ static final long hash(long key1, long key2) { } static final int signSafeMod(long n, int max) { + // as the ITEM_SIZE of Section is 4, so the index is the multiple of 4 + // that is to left shift 2 bits return (int) (n & (max - 1)) << 2; } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java index 2a1090503857c..389279c5b395b 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java @@ -34,6 +34,18 @@ * no node allocations are required to store the keys and values, and no boxing is required. * *

      Values MUST be >= 0. + *
      + * WARN: method forEach do not guarantee thread safety, nor does the items method. + *
      + * The forEach method is specifically designed for single-threaded usage. When iterating over a set + * with concurrent writes, it becomes possible for new values to be either observed or not observed. + * There is no guarantee that if we write value1 and value2, and are able to see value2, then we will also see value1. + * + *
      + * It is crucial to understand that the results obtained from aggregate status methods such as items + * are typically reliable only when the map is not undergoing concurrent updates from other threads. + * When concurrent updates are involved, the results of these methods reflect transient states + * that may be suitable for monitoring or estimation purposes, but not for program control. */ public class ConcurrentLongPairSet implements LongPairSet { @@ -237,6 +249,12 @@ public void clear() { } } + /** + * Iterate over all the elements in the set and apply the provided function. + *

      + * Warning: Do Not Guarantee Thread-Safety. + * @param processor the processor to process the elements + */ public void forEach(LongPairConsumer processor) { for (int i = 0; i < sections.length; i++) { sections[i].forEach(processor); @@ -260,7 +278,7 @@ public int removeIf(LongPairPredicate filter) { } /** - * @return a new list of all keys (makes a copy) + * @return a new set of all keys (makes a copy) */ public Set items() { Set items = new HashSet<>(); @@ -294,6 +312,9 @@ public Set items(int numberOfItems, LongPairFunction longPairConverter // A section is a portion of the hash map that is covered by a single @SuppressWarnings("serial") private static final class Section extends StampedLock { + // Each item take up 2 continuous array space. + private static final int ITEM_SIZE = 2; + // Keys and values are stored interleaved in the table array private volatile long[] table; @@ -315,7 +336,7 @@ private static final class Section extends StampedLock { float expandFactor, float shrinkFactor) { this.capacity = alignToPowerOfTwo(capacity); this.initCapacity = this.capacity; - this.table = new long[2 * this.capacity]; + this.table = new long[ITEM_SIZE * this.capacity]; this.size = 0; this.usedBuckets = 0; this.autoShrink = autoShrink; @@ -331,7 +352,11 @@ private static final class Section extends StampedLock { boolean contains(long item1, long item2, int hash) { long stamp = tryOptimisticRead(); boolean acquiredLock = false; - int bucket = signSafeMod(hash, capacity); + + // add local variable here, so OutOfBound won't happen + long[] table = this.table; + // calculate table.length / 2 as capacity to avoid rehash changing capacity + int bucket = signSafeMod(hash, table.length / ITEM_SIZE); try { while (true) { @@ -353,7 +378,9 @@ boolean contains(long item1, long item2, int hash) { stamp = readLock(); acquiredLock = true; - bucket = signSafeMod(hash, capacity); + // update local variable + table = this.table; + bucket = signSafeMod(hash, table.length / ITEM_SIZE); storedItem1 = table[bucket]; storedItem2 = table[bucket + 1]; } @@ -366,7 +393,7 @@ boolean contains(long item1, long item2, int hash) { } } - bucket = (bucket + 2) & (table.length - 1); + bucket = (bucket + ITEM_SIZE) & (table.length - 1); } } finally { if (acquiredLock) { @@ -410,7 +437,7 @@ boolean add(long item1, long item2, long hash) { } } - bucket = (bucket + 2) & (table.length - 1); + bucket = (bucket + ITEM_SIZE) & (table.length - 1); } } finally { if (usedBuckets > resizeThresholdUp) { @@ -445,7 +472,7 @@ private boolean remove(long item1, long item2, int hash) { return false; } - bucket = (bucket + 2) & (table.length - 1); + bucket = (bucket + ITEM_SIZE) & (table.length - 1); } } finally { tryShrinkThenUnlock(stamp); @@ -459,7 +486,7 @@ private int removeIf(LongPairPredicate filter) { // Go through all the buckets for this section long stamp = writeLock(); try { - for (int bucket = 0; bucket < table.length; bucket += 2) { + for (int bucket = 0; bucket < table.length; bucket += ITEM_SIZE) { long storedItem1 = table[bucket]; long storedItem2 = table[bucket + 1]; if (storedItem1 != DeletedItem && storedItem1 != EmptyItem) { @@ -498,7 +525,7 @@ private void tryShrinkThenUnlock(long stamp) { } private void cleanBucket(int bucket) { - int nextInArray = (bucket + 2) & (table.length - 1); + int nextInArray = (bucket + ITEM_SIZE) & (table.length - 1); if (table[nextInArray] == EmptyItem) { table[bucket] = EmptyItem; table[bucket + 1] = EmptyItem; @@ -506,13 +533,13 @@ private void cleanBucket(int bucket) { // Cleanup all the buckets that were in `DeletedItem` state, // so that we can reduce unnecessary expansions - int lastBucket = (bucket - 2) & (table.length - 1); + int lastBucket = (bucket - ITEM_SIZE) & (table.length - 1); while (table[lastBucket] == DeletedItem) { table[lastBucket] = EmptyItem; table[lastBucket + 1] = EmptyItem; --usedBuckets; - lastBucket = (lastBucket - 2) & (table.length - 1); + lastBucket = (lastBucket - ITEM_SIZE) & (table.length - 1); } } else { table[bucket] = DeletedItem; @@ -542,7 +569,7 @@ public void forEach(LongPairConsumer processor) { // Go through all the buckets for this section. We try to renew the stamp only after a validation // error, otherwise we keep going with the same. long stamp = 0; - for (int bucket = 0; bucket < table.length; bucket += 2) { + for (int bucket = 0; bucket < table.length; bucket += ITEM_SIZE) { if (stamp == 0) { stamp = tryOptimisticRead(); } @@ -572,11 +599,11 @@ public void forEach(LongPairConsumer processor) { private void rehash(int newCapacity) { // Expand the hashmap - long[] newTable = new long[2 * newCapacity]; + long[] newTable = new long[ITEM_SIZE * newCapacity]; Arrays.fill(newTable, EmptyItem); // Re-hash table - for (int i = 0; i < table.length; i += 2) { + for (int i = 0; i < table.length; i += ITEM_SIZE) { long storedItem1 = table[i]; long storedItem2 = table[i + 1]; if (storedItem1 != EmptyItem && storedItem1 != DeletedItem) { @@ -595,7 +622,7 @@ private void rehash(int newCapacity) { private void shrinkToInitCapacity() { // Expand the hashmap - long[] newTable = new long[2 * initCapacity]; + long[] newTable = new long[ITEM_SIZE * initCapacity]; Arrays.fill(newTable, EmptyItem); table = newTable; @@ -621,7 +648,7 @@ private static void insertKeyValueNoLock(long[] table, int capacity, long item1, return; } - bucket = (bucket + 2) & (table.length - 1); + bucket = (bucket + ITEM_SIZE) & (table.length - 1); } } } @@ -640,6 +667,8 @@ static final long hash(long key1, long key2) { } static final int signSafeMod(long n, int max) { + // as the ITEM_SIZE of Section is 2, so the index is the multiple of 2 + // that is to left shift 1 bit return (int) (n & (max - 1)) << 1; } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMap.java deleted file mode 100644 index ea2e01768ac7e..0000000000000 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMap.java +++ /dev/null @@ -1,627 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util.collections; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; -import java.util.concurrent.locks.StampedLock; -import java.util.function.BiConsumer; -import java.util.function.Function; - -/** - * Concurrent hash map. - * - *

      Provides similar methods as a {@code ConcurrentMap} but since it's an open hash map with linear probing, - * no node allocations are required to store the values. - * - * @param - */ -@SuppressWarnings("unchecked") -public class ConcurrentOpenHashMap { - - private static final Object EmptyKey = null; - private static final Object DeletedKey = new Object(); - private static final ConcurrentOpenHashMap EmptyMap = new ConcurrentOpenHashMap<>(1, 1); - - /** - * This object is used to delete empty value in this map. - * EmptyValue.equals(null) = true. - */ - private static final Object EmptyValue = new Object() { - - @SuppressFBWarnings - @Override - public boolean equals(Object obj) { - return obj == null; - } - - /** - * This is just for avoiding spotbugs errors - */ - @Override - public int hashCode() { - return super.hashCode(); - } - }; - - private static final int DefaultExpectedItems = 256; - private static final int DefaultConcurrencyLevel = 16; - - private static final float DefaultMapFillFactor = 0.66f; - private static final float DefaultMapIdleFactor = 0.15f; - - private static final float DefaultExpandFactor = 2; - private static final float DefaultShrinkFactor = 2; - - private static final boolean DefaultAutoShrink = false; - - private final Section[] sections; - - public static Builder newBuilder() { - return new Builder<>(); - } - - /** - * Builder of ConcurrentOpenHashMap. - */ - public static class Builder { - int expectedItems = DefaultExpectedItems; - int concurrencyLevel = DefaultConcurrencyLevel; - float mapFillFactor = DefaultMapFillFactor; - float mapIdleFactor = DefaultMapIdleFactor; - float expandFactor = DefaultExpandFactor; - float shrinkFactor = DefaultShrinkFactor; - boolean autoShrink = DefaultAutoShrink; - - public Builder expectedItems(int expectedItems) { - this.expectedItems = expectedItems; - return this; - } - - public Builder concurrencyLevel(int concurrencyLevel) { - this.concurrencyLevel = concurrencyLevel; - return this; - } - - public Builder mapFillFactor(float mapFillFactor) { - this.mapFillFactor = mapFillFactor; - return this; - } - - public Builder mapIdleFactor(float mapIdleFactor) { - this.mapIdleFactor = mapIdleFactor; - return this; - } - - public Builder expandFactor(float expandFactor) { - this.expandFactor = expandFactor; - return this; - } - - public Builder shrinkFactor(float shrinkFactor) { - this.shrinkFactor = shrinkFactor; - return this; - } - - public Builder autoShrink(boolean autoShrink) { - this.autoShrink = autoShrink; - return this; - } - - public ConcurrentOpenHashMap build() { - return new ConcurrentOpenHashMap<>(expectedItems, concurrencyLevel, - mapFillFactor, mapIdleFactor, autoShrink, expandFactor, shrinkFactor); - } - } - - @Deprecated - public ConcurrentOpenHashMap() { - this(DefaultExpectedItems); - } - - @Deprecated - public ConcurrentOpenHashMap(int expectedItems) { - this(expectedItems, DefaultConcurrencyLevel); - } - - @Deprecated - public ConcurrentOpenHashMap(int expectedItems, int concurrencyLevel) { - this(expectedItems, concurrencyLevel, DefaultMapFillFactor, DefaultMapIdleFactor, - DefaultAutoShrink, DefaultExpandFactor, DefaultShrinkFactor); - } - - public ConcurrentOpenHashMap(int expectedItems, int concurrencyLevel, - float mapFillFactor, float mapIdleFactor, - boolean autoShrink, float expandFactor, float shrinkFactor) { - checkArgument(expectedItems > 0); - checkArgument(concurrencyLevel > 0); - checkArgument(expectedItems >= concurrencyLevel); - checkArgument(mapFillFactor > 0 && mapFillFactor < 1); - checkArgument(mapIdleFactor > 0 && mapIdleFactor < 1); - checkArgument(mapFillFactor > mapIdleFactor); - checkArgument(expandFactor > 1); - checkArgument(shrinkFactor > 1); - - int numSections = concurrencyLevel; - int perSectionExpectedItems = expectedItems / numSections; - int perSectionCapacity = (int) (perSectionExpectedItems / mapFillFactor); - this.sections = (Section[]) new Section[numSections]; - - for (int i = 0; i < numSections; i++) { - sections[i] = new Section<>(perSectionCapacity, mapFillFactor, mapIdleFactor, - autoShrink, expandFactor, shrinkFactor); - } - } - - public static ConcurrentOpenHashMap emptyMap() { - return (ConcurrentOpenHashMap) EmptyMap; - } - - long getUsedBucketCount() { - long usedBucketCount = 0; - for (Section s : sections) { - usedBucketCount += s.usedBuckets; - } - return usedBucketCount; - } - - public long size() { - long size = 0; - for (Section s : sections) { - size += s.size; - } - return size; - } - - public long capacity() { - long capacity = 0; - for (Section s : sections) { - capacity += s.capacity; - } - return capacity; - } - - public boolean isEmpty() { - for (Section s : sections) { - if (s.size != 0) { - return false; - } - } - - return true; - } - - public V get(K key) { - requireNonNull(key); - long h = hash(key); - return getSection(h).get(key, (int) h); - } - - public boolean containsKey(K key) { - return get(key) != null; - } - - public V put(K key, V value) { - requireNonNull(key); - requireNonNull(value); - long h = hash(key); - return getSection(h).put(key, value, (int) h, false, null); - } - - public V putIfAbsent(K key, V value) { - requireNonNull(key); - requireNonNull(value); - long h = hash(key); - return getSection(h).put(key, value, (int) h, true, null); - } - - public V computeIfAbsent(K key, Function provider) { - requireNonNull(key); - requireNonNull(provider); - long h = hash(key); - return getSection(h).put(key, null, (int) h, true, provider); - } - - public V remove(K key) { - requireNonNull(key); - long h = hash(key); - return getSection(h).remove(key, null, (int) h); - } - - public boolean remove(K key, Object value) { - requireNonNull(key); - requireNonNull(value); - long h = hash(key); - return getSection(h).remove(key, value, (int) h) != null; - } - - public void removeNullValue(K key) { - remove(key, EmptyValue); - } - - private Section getSection(long hash) { - // Use 32 msb out of long to get the section - final int sectionIdx = (int) (hash >>> 32) & (sections.length - 1); - return sections[sectionIdx]; - } - - public void clear() { - for (int i = 0; i < sections.length; i++) { - sections[i].clear(); - } - } - - public void forEach(BiConsumer processor) { - for (int i = 0; i < sections.length; i++) { - sections[i].forEach(processor); - } - } - - /** - * @return a new list of all keys (makes a copy) - */ - public List keys() { - List keys = new ArrayList<>((int) size()); - forEach((key, value) -> keys.add(key)); - return keys; - } - - public List values() { - List values = new ArrayList<>((int) size()); - forEach((key, value) -> values.add(value)); - return values; - } - - // A section is a portion of the hash map that is covered by a single - @SuppressWarnings("serial") - private static final class Section extends StampedLock { - // Keys and values are stored interleaved in the table array - private volatile Object[] table; - - private volatile int capacity; - private final int initCapacity; - private static final AtomicIntegerFieldUpdater

      SIZE_UPDATER = - AtomicIntegerFieldUpdater.newUpdater(Section.class, "size"); - private volatile int size; - private int usedBuckets; - private int resizeThresholdUp; - private int resizeThresholdBelow; - private final float mapFillFactor; - private final float mapIdleFactor; - private final float expandFactor; - private final float shrinkFactor; - private final boolean autoShrink; - - Section(int capacity, float mapFillFactor, float mapIdleFactor, boolean autoShrink, - float expandFactor, float shrinkFactor) { - this.capacity = alignToPowerOfTwo(capacity); - this.initCapacity = this.capacity; - this.table = new Object[2 * this.capacity]; - this.size = 0; - this.usedBuckets = 0; - this.autoShrink = autoShrink; - this.mapFillFactor = mapFillFactor; - this.mapIdleFactor = mapIdleFactor; - this.expandFactor = expandFactor; - this.shrinkFactor = shrinkFactor; - this.resizeThresholdUp = (int) (this.capacity * mapFillFactor); - this.resizeThresholdBelow = (int) (this.capacity * mapIdleFactor); - } - - V get(K key, int keyHash) { - long stamp = tryOptimisticRead(); - boolean acquiredLock = false; - int bucket = signSafeMod(keyHash, capacity); - - try { - while (true) { - // First try optimistic locking - K storedKey = (K) table[bucket]; - V storedValue = (V) table[bucket + 1]; - - if (!acquiredLock && validate(stamp)) { - // The values we have read are consistent - if (key.equals(storedKey)) { - return storedValue; - } else if (storedKey == EmptyKey) { - // Not found - return null; - } - } else { - // Fallback to acquiring read lock - if (!acquiredLock) { - stamp = readLock(); - acquiredLock = true; - - bucket = signSafeMod(keyHash, capacity); - storedKey = (K) table[bucket]; - storedValue = (V) table[bucket + 1]; - } - - if (key.equals(storedKey)) { - return storedValue; - } else if (storedKey == EmptyKey) { - // Not found - return null; - } - } - - bucket = (bucket + 2) & (table.length - 1); - } - } finally { - if (acquiredLock) { - unlockRead(stamp); - } - } - } - - V put(K key, V value, int keyHash, boolean onlyIfAbsent, Function valueProvider) { - long stamp = writeLock(); - int bucket = signSafeMod(keyHash, capacity); - - // Remember where we find the first available spot - int firstDeletedKey = -1; - - try { - while (true) { - K storedKey = (K) table[bucket]; - V storedValue = (V) table[bucket + 1]; - - if (key.equals(storedKey)) { - if (!onlyIfAbsent) { - // Over written an old value for same key - table[bucket + 1] = value; - return storedValue; - } else { - return storedValue; - } - } else if (storedKey == EmptyKey) { - // Found an empty bucket. This means the key is not in the map. If we've already seen a deleted - // key, we should write at that position - if (firstDeletedKey != -1) { - bucket = firstDeletedKey; - } else { - ++usedBuckets; - } - - if (value == null) { - value = valueProvider.apply(key); - } - - table[bucket] = key; - table[bucket + 1] = value; - SIZE_UPDATER.incrementAndGet(this); - return valueProvider != null ? value : null; - } else if (storedKey == DeletedKey) { - // The bucket contained a different deleted key - if (firstDeletedKey == -1) { - firstDeletedKey = bucket; - } - } - - bucket = (bucket + 2) & (table.length - 1); - } - } finally { - if (usedBuckets > resizeThresholdUp) { - try { - // Expand the hashmap - int newCapacity = alignToPowerOfTwo((int) (capacity * expandFactor)); - rehash(newCapacity); - } finally { - unlockWrite(stamp); - } - } else { - unlockWrite(stamp); - } - } - } - - private V remove(K key, Object value, int keyHash) { - long stamp = writeLock(); - int bucket = signSafeMod(keyHash, capacity); - - try { - while (true) { - K storedKey = (K) table[bucket]; - V storedValue = (V) table[bucket + 1]; - if (key.equals(storedKey)) { - if (value == null || value.equals(storedValue)) { - SIZE_UPDATER.decrementAndGet(this); - - int nextInArray = (bucket + 2) & (table.length - 1); - if (table[nextInArray] == EmptyKey) { - table[bucket] = EmptyKey; - table[bucket + 1] = null; - --usedBuckets; - - // Cleanup all the buckets that were in `DeletedKey` state, - // so that we can reduce unnecessary expansions - int lastBucket = (bucket - 2) & (table.length - 1); - while (table[lastBucket] == DeletedKey) { - table[lastBucket] = EmptyKey; - table[lastBucket + 1] = null; - --usedBuckets; - - lastBucket = (lastBucket - 2) & (table.length - 1); - } - } else { - table[bucket] = DeletedKey; - table[bucket + 1] = null; - } - - return storedValue; - } else { - return null; - } - } else if (storedKey == EmptyKey) { - // Key wasn't found - return null; - } - - bucket = (bucket + 2) & (table.length - 1); - } - - } finally { - if (autoShrink && size < resizeThresholdBelow) { - try { - // Shrinking must at least ensure initCapacity, - // so as to avoid frequent shrinking and expansion near initCapacity, - // frequent shrinking and expansion, - // additionally opened arrays will consume more memory and affect GC - int newCapacity = Math.max(alignToPowerOfTwo((int) (capacity / shrinkFactor)), initCapacity); - int newResizeThresholdUp = (int) (newCapacity * mapFillFactor); - if (newCapacity < capacity && newResizeThresholdUp > size) { - // shrink the hashmap - rehash(newCapacity); - } - } finally { - unlockWrite(stamp); - } - } else { - unlockWrite(stamp); - } - } - } - - void clear() { - long stamp = writeLock(); - - try { - if (autoShrink && capacity > initCapacity) { - shrinkToInitCapacity(); - } else { - Arrays.fill(table, EmptyKey); - this.size = 0; - this.usedBuckets = 0; - } - } finally { - unlockWrite(stamp); - } - } - - public void forEach(BiConsumer processor) { - // Take a reference to the data table, if there is a rehashing event, we'll be - // simply iterating over a snapshot of the data. - Object[] table = this.table; - - // Go through all the buckets for this section. We try to renew the stamp only after a validation - // error, otherwise we keep going with the same. - long stamp = 0; - for (int bucket = 0; bucket < table.length; bucket += 2) { - if (stamp == 0) { - stamp = tryOptimisticRead(); - } - - K storedKey = (K) table[bucket]; - V storedValue = (V) table[bucket + 1]; - - if (!validate(stamp)) { - // Fallback to acquiring read lock - stamp = readLock(); - - try { - storedKey = (K) table[bucket]; - storedValue = (V) table[bucket + 1]; - } finally { - unlockRead(stamp); - } - - stamp = 0; - } - - if (storedKey != DeletedKey && storedKey != EmptyKey) { - processor.accept(storedKey, storedValue); - } - } - } - - private void rehash(int newCapacity) { - // Expand the hashmap - Object[] newTable = new Object[2 * newCapacity]; - - // Re-hash table - for (int i = 0; i < table.length; i += 2) { - K storedKey = (K) table[i]; - V storedValue = (V) table[i + 1]; - if (storedKey != EmptyKey && storedKey != DeletedKey) { - insertKeyValueNoLock(newTable, newCapacity, storedKey, storedValue); - } - } - - table = newTable; - capacity = newCapacity; - usedBuckets = size; - resizeThresholdUp = (int) (capacity * mapFillFactor); - resizeThresholdBelow = (int) (capacity * mapIdleFactor); - } - - private void shrinkToInitCapacity() { - Object[] newTable = new Object[2 * initCapacity]; - - table = newTable; - size = 0; - usedBuckets = 0; - // Capacity needs to be updated after the values, so that we won't see - // a capacity value bigger than the actual array size - capacity = initCapacity; - resizeThresholdUp = (int) (capacity * mapFillFactor); - resizeThresholdBelow = (int) (capacity * mapIdleFactor); - } - - private static void insertKeyValueNoLock(Object[] table, int capacity, K key, V value) { - int bucket = signSafeMod(hash(key), capacity); - - while (true) { - K storedKey = (K) table[bucket]; - - if (storedKey == EmptyKey) { - // The bucket is empty, so we can use it - table[bucket] = key; - table[bucket + 1] = value; - return; - } - - bucket = (bucket + 2) & (table.length - 1); - } - } - } - - private static final long HashMixer = 0xc6a4a7935bd1e995L; - private static final int R = 47; - - static final long hash(K key) { - long hash = key.hashCode() * HashMixer; - hash ^= hash >>> R; - hash *= HashMixer; - return hash; - } - - static final int signSafeMod(long n, int max) { - return (int) (n & (max - 1)) << 1; - } - - private static int alignToPowerOfTwo(int n) { - return (int) Math.pow(2, 32 - Integer.numberOfLeadingZeros(n - 1)); - } -} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSet.java deleted file mode 100644 index cc8bc07b43095..0000000000000 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSet.java +++ /dev/null @@ -1,608 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util.collections; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; -import java.util.concurrent.locks.StampedLock; -import java.util.function.Consumer; -import java.util.function.Predicate; - -/** - * Concurrent hash set. - * - *

      Provides similar methods as a {@code ConcurrentMap} but since it's an open hash map with linear probing, - * no node allocations are required to store the values. - * - * @param - */ -@SuppressWarnings("unchecked") -public class ConcurrentOpenHashSet { - - private static final Object EmptyValue = null; - private static final Object DeletedValue = new Object(); - - private static final int DefaultExpectedItems = 256; - private static final int DefaultConcurrencyLevel = 16; - - private static final float DefaultMapFillFactor = 0.66f; - private static final float DefaultMapIdleFactor = 0.15f; - - private static final float DefaultExpandFactor = 2; - private static final float DefaultShrinkFactor = 2; - - private static final boolean DefaultAutoShrink = false; - - private final Section[] sections; - - public static Builder newBuilder() { - return new Builder<>(); - } - - /** - * Builder of ConcurrentOpenHashSet. - */ - public static class Builder { - int expectedItems = DefaultExpectedItems; - int concurrencyLevel = DefaultConcurrencyLevel; - float mapFillFactor = DefaultMapFillFactor; - float mapIdleFactor = DefaultMapIdleFactor; - float expandFactor = DefaultExpandFactor; - float shrinkFactor = DefaultShrinkFactor; - boolean autoShrink = DefaultAutoShrink; - - public Builder expectedItems(int expectedItems) { - this.expectedItems = expectedItems; - return this; - } - - public Builder concurrencyLevel(int concurrencyLevel) { - this.concurrencyLevel = concurrencyLevel; - return this; - } - - public Builder mapFillFactor(float mapFillFactor) { - this.mapFillFactor = mapFillFactor; - return this; - } - - public Builder mapIdleFactor(float mapIdleFactor) { - this.mapIdleFactor = mapIdleFactor; - return this; - } - - public Builder expandFactor(float expandFactor) { - this.expandFactor = expandFactor; - return this; - } - - public Builder shrinkFactor(float shrinkFactor) { - this.shrinkFactor = shrinkFactor; - return this; - } - - public Builder autoShrink(boolean autoShrink) { - this.autoShrink = autoShrink; - return this; - } - - public ConcurrentOpenHashSet build() { - return new ConcurrentOpenHashSet<>(expectedItems, concurrencyLevel, - mapFillFactor, mapIdleFactor, autoShrink, expandFactor, shrinkFactor); - } - } - - @Deprecated - public ConcurrentOpenHashSet() { - this(DefaultExpectedItems); - } - - @Deprecated - public ConcurrentOpenHashSet(int expectedItems) { - this(expectedItems, DefaultConcurrencyLevel); - } - - @Deprecated - public ConcurrentOpenHashSet(int expectedItems, int concurrencyLevel) { - this(expectedItems, concurrencyLevel, DefaultMapFillFactor, DefaultMapIdleFactor, - DefaultAutoShrink, DefaultExpandFactor, DefaultShrinkFactor); - } - - public ConcurrentOpenHashSet(int expectedItems, int concurrencyLevel, - float mapFillFactor, float mapIdleFactor, - boolean autoShrink, float expandFactor, float shrinkFactor) { - checkArgument(expectedItems > 0); - checkArgument(concurrencyLevel > 0); - checkArgument(expectedItems >= concurrencyLevel); - checkArgument(mapFillFactor > 0 && mapFillFactor < 1); - checkArgument(mapIdleFactor > 0 && mapIdleFactor < 1); - checkArgument(mapFillFactor > mapIdleFactor); - checkArgument(expandFactor > 1); - checkArgument(shrinkFactor > 1); - - int numSections = concurrencyLevel; - int perSectionExpectedItems = expectedItems / numSections; - int perSectionCapacity = (int) (perSectionExpectedItems / mapFillFactor); - this.sections = (Section[]) new Section[numSections]; - - for (int i = 0; i < numSections; i++) { - sections[i] = new Section<>(perSectionCapacity, mapFillFactor, mapIdleFactor, - autoShrink, expandFactor, shrinkFactor); - } - } - - long getUsedBucketCount() { - long usedBucketCount = 0; - for (Section s : sections) { - usedBucketCount += s.usedBuckets; - } - return usedBucketCount; - } - - public long size() { - long size = 0; - for (int i = 0; i < sections.length; i++) { - size += sections[i].size; - } - return size; - } - - public long capacity() { - long capacity = 0; - for (int i = 0; i < sections.length; i++) { - capacity += sections[i].capacity; - } - return capacity; - } - - public boolean isEmpty() { - for (int i = 0; i < sections.length; i++) { - if (sections[i].size != 0) { - return false; - } - } - - return true; - } - - public boolean contains(V value) { - requireNonNull(value); - long h = hash(value); - return getSection(h).contains(value, (int) h); - } - - public boolean add(V value) { - requireNonNull(value); - long h = hash(value); - return getSection(h).add(value, (int) h); - } - - public boolean remove(V value) { - requireNonNull(value); - long h = hash(value); - return getSection(h).remove(value, (int) h); - } - - private Section getSection(long hash) { - // Use 32 msb out of long to get the section - final int sectionIdx = (int) (hash >>> 32) & (sections.length - 1); - return sections[sectionIdx]; - } - - public void clear() { - for (int i = 0; i < sections.length; i++) { - sections[i].clear(); - } - } - - public void forEach(Consumer processor) { - for (int i = 0; i < sections.length; i++) { - sections[i].forEach(processor); - } - } - - public int removeIf(Predicate filter) { - requireNonNull(filter); - - int removedCount = 0; - for (int i = 0; i < sections.length; i++) { - removedCount += sections[i].removeIf(filter); - } - - return removedCount; - } - - /** - * @return a new list of all values (makes a copy) - */ - public List values() { - List values = new ArrayList<>(); - forEach(value -> values.add(value)); - return values; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append('{'); - final AtomicBoolean first = new AtomicBoolean(true); - forEach(value -> { - if (!first.getAndSet(false)) { - sb.append(", "); - } - - sb.append(value.toString()); - }); - sb.append('}'); - return sb.toString(); - } - - // A section is a portion of the hash map that is covered by a single - @SuppressWarnings("serial") - private static final class Section extends StampedLock { - private volatile V[] values; - - private volatile int capacity; - private final int initCapacity; - private static final AtomicIntegerFieldUpdater

      SIZE_UPDATER = - AtomicIntegerFieldUpdater.newUpdater(Section.class, "size"); - private volatile int size; - private int usedBuckets; - private int resizeThresholdUp; - private int resizeThresholdBelow; - private final float mapFillFactor; - private final float mapIdleFactor; - private final float expandFactor; - private final float shrinkFactor; - private final boolean autoShrink; - - Section(int capacity, float mapFillFactor, float mapIdleFactor, boolean autoShrink, - float expandFactor, float shrinkFactor) { - this.capacity = alignToPowerOfTwo(capacity); - this.initCapacity = this.capacity; - this.values = (V[]) new Object[this.capacity]; - this.size = 0; - this.usedBuckets = 0; - this.autoShrink = autoShrink; - this.mapFillFactor = mapFillFactor; - this.mapIdleFactor = mapIdleFactor; - this.expandFactor = expandFactor; - this.shrinkFactor = shrinkFactor; - this.resizeThresholdUp = (int) (this.capacity * mapFillFactor); - this.resizeThresholdBelow = (int) (this.capacity * mapIdleFactor); - } - - boolean contains(V value, int keyHash) { - int bucket = keyHash; - - long stamp = tryOptimisticRead(); - boolean acquiredLock = false; - - try { - while (true) { - int capacity = this.capacity; - bucket = signSafeMod(bucket, capacity); - - // First try optimistic locking - V storedValue = values[bucket]; - - if (!acquiredLock && validate(stamp)) { - // The values we have read are consistent - if (value.equals(storedValue)) { - return true; - } else if (storedValue == EmptyValue) { - // Not found - return false; - } - } else { - // Fallback to acquiring read lock - if (!acquiredLock) { - stamp = readLock(); - acquiredLock = true; - - storedValue = values[bucket]; - } - - if (capacity != this.capacity) { - // There has been a rehashing. We need to restart the search - bucket = keyHash; - continue; - } - - if (value.equals(storedValue)) { - return true; - } else if (storedValue == EmptyValue) { - // Not found - return false; - } - } - - ++bucket; - } - } finally { - if (acquiredLock) { - unlockRead(stamp); - } - } - } - - boolean add(V value, int keyHash) { - int bucket = keyHash; - - long stamp = writeLock(); - int capacity = this.capacity; - - // Remember where we find the first available spot - int firstDeletedValue = -1; - - try { - while (true) { - bucket = signSafeMod(bucket, capacity); - - V storedValue = values[bucket]; - - if (value.equals(storedValue)) { - return false; - } else if (storedValue == EmptyValue) { - // Found an empty bucket. This means the value is not in the set. If we've already seen a - // deleted value, we should write at that position - if (firstDeletedValue != -1) { - bucket = firstDeletedValue; - } else { - ++usedBuckets; - } - - values[bucket] = value; - SIZE_UPDATER.incrementAndGet(this); - return true; - } else if (storedValue == DeletedValue) { - // The bucket contained a different deleted key - if (firstDeletedValue == -1) { - firstDeletedValue = bucket; - } - } - - ++bucket; - } - } finally { - if (usedBuckets > resizeThresholdUp) { - try { - // Expand the hashmap - int newCapacity = alignToPowerOfTwo((int) (capacity * expandFactor)); - rehash(newCapacity); - } finally { - unlockWrite(stamp); - } - } else { - unlockWrite(stamp); - } - } - } - - private boolean remove(V value, int keyHash) { - int bucket = keyHash; - long stamp = writeLock(); - - try { - while (true) { - int capacity = this.capacity; - bucket = signSafeMod(bucket, capacity); - - V storedValue = values[bucket]; - if (value.equals(storedValue)) { - SIZE_UPDATER.decrementAndGet(this); - cleanBucket(bucket); - return true; - } else if (storedValue == EmptyValue) { - // Value wasn't found - return false; - } - - ++bucket; - } - - } finally { - if (autoShrink && size < resizeThresholdBelow) { - try { - // Shrinking must at least ensure initCapacity, - // so as to avoid frequent shrinking and expansion near initCapacity, - // frequent shrinking and expansion, - // additionally opened arrays will consume more memory and affect GC - int newCapacity = Math.max(alignToPowerOfTwo((int) (capacity / shrinkFactor)), initCapacity); - int newResizeThresholdUp = (int) (newCapacity * mapFillFactor); - if (newCapacity < capacity && newResizeThresholdUp > size) { - // shrink the hashmap - rehash(newCapacity); - } - } finally { - unlockWrite(stamp); - } - } else { - unlockWrite(stamp); - } - } - } - - void clear() { - long stamp = writeLock(); - - try { - if (autoShrink && capacity > initCapacity) { - shrinkToInitCapacity(); - } else { - Arrays.fill(values, EmptyValue); - this.size = 0; - this.usedBuckets = 0; - } - } finally { - unlockWrite(stamp); - } - } - - int removeIf(Predicate filter) { - long stamp = writeLock(); - - int removedCount = 0; - try { - // Go through all the buckets for this section - for (int bucket = capacity - 1; bucket >= 0; bucket--) { - V storedValue = values[bucket]; - - if (storedValue != DeletedValue && storedValue != EmptyValue) { - if (filter.test(storedValue)) { - // Removing item - SIZE_UPDATER.decrementAndGet(this); - ++removedCount; - cleanBucket(bucket); - } - } - } - - return removedCount; - } finally { - unlockWrite(stamp); - } - } - - private void cleanBucket(int bucket) { - int nextInArray = signSafeMod(bucket + 1, capacity); - if (values[nextInArray] == EmptyValue) { - values[bucket] = (V) EmptyValue; - --usedBuckets; - - // Cleanup all the buckets that were in `DeletedValue` state, - // so that we can reduce unnecessary expansions - int lastBucket = signSafeMod(bucket - 1, capacity); - while (values[lastBucket] == DeletedValue) { - values[lastBucket] = (V) EmptyValue; - --usedBuckets; - - lastBucket = signSafeMod(lastBucket - 1, capacity); - } - } else { - values[bucket] = (V) DeletedValue; - } - } - - public void forEach(Consumer processor) { - V[] values = this.values; - - // Go through all the buckets for this section. We try to renew the stamp only after a validation - // error, otherwise we keep going with the same. - long stamp = 0; - for (int bucket = 0; bucket < capacity; bucket++) { - if (stamp == 0) { - stamp = tryOptimisticRead(); - } - - V storedValue = values[bucket]; - - if (!validate(stamp)) { - // Fallback to acquiring read lock - stamp = readLock(); - - try { - storedValue = values[bucket]; - } finally { - unlockRead(stamp); - } - - stamp = 0; - } - - if (storedValue != DeletedValue && storedValue != EmptyValue) { - processor.accept(storedValue); - } - } - } - - private void rehash(int newCapacity) { - // Expand the hashmap - V[] newValues = (V[]) new Object[newCapacity]; - - // Re-hash table - for (int i = 0; i < values.length; i++) { - V storedValue = values[i]; - if (storedValue != EmptyValue && storedValue != DeletedValue) { - insertValueNoLock(newValues, storedValue); - } - } - - values = newValues; - capacity = newCapacity; - usedBuckets = size; - resizeThresholdUp = (int) (capacity * mapFillFactor); - resizeThresholdBelow = (int) (capacity * mapIdleFactor); - } - - private void shrinkToInitCapacity() { - V[] newValues = (V[]) new Object[initCapacity]; - - values = newValues; - size = 0; - usedBuckets = 0; - // Capacity needs to be updated after the values, so that we won't see - // a capacity value bigger than the actual array size - capacity = initCapacity; - resizeThresholdUp = (int) (capacity * mapFillFactor); - resizeThresholdBelow = (int) (capacity * mapIdleFactor); - } - - private static void insertValueNoLock(V[] values, V value) { - int bucket = (int) hash(value); - - while (true) { - bucket = signSafeMod(bucket, values.length); - - V storedValue = values[bucket]; - - if (storedValue == EmptyValue) { - // The bucket is empty, so we can use it - values[bucket] = value; - return; - } - - ++bucket; - } - } - } - - private static final long HashMixer = 0xc6a4a7935bd1e995L; - private static final int R = 47; - - static final long hash(K key) { - long hash = key.hashCode() * HashMixer; - hash ^= hash >>> R; - hash *= HashMixer; - return hash; - } - - static final int signSafeMod(long n, int max) { - return (int) n & (max - 1); - } - - private static int alignToPowerOfTwo(int n) { - return (int) Math.pow(2, 32 - Integer.numberOfLeadingZeros(n - 1)); - } -} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java index 72215d7296cc3..51f4a9ac51c90 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java @@ -18,16 +18,21 @@ */ package org.apache.pulsar.common.util.collections; +import static java.util.BitSet.valueOf; import static java.util.Objects.requireNonNull; import com.google.common.collect.BoundType; import com.google.common.collect.Range; import java.util.ArrayList; import java.util.BitSet; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Map.Entry; import java.util.NavigableMap; +import java.util.Objects; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.lang.mutable.MutableInt; /** @@ -253,6 +258,42 @@ public Range lastRange() { return Range.openClosed(consumer.apply(lastSet.getKey(), lower), consumer.apply(lastSet.getKey(), upper)); } + @Override + public Map toRanges(int maxRanges) { + Map internalBitSetMap = new HashMap<>(); + AtomicInteger rangeCount = new AtomicInteger(); + rangeBitSetMap.forEach((id, bmap) -> { + if (rangeCount.getAndAdd(bmap.cardinality()) > maxRanges) { + return; + } + internalBitSetMap.put(id, bmap.toLongArray()); + }); + return internalBitSetMap; + } + + @Override + public void build(Map internalRange) { + internalRange.forEach((id, ranges) -> rangeBitSetMap.put(id, valueOf(ranges))); + } + + @Override + public int hashCode() { + return Objects.hashCode(rangeBitSetMap); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ConcurrentOpenLongPairRangeSet)) { + return false; + } + if (this == obj) { + return true; + } + @SuppressWarnings("rawtypes") + ConcurrentOpenLongPairRangeSet set = (ConcurrentOpenLongPairRangeSet) obj; + return this.rangeBitSetMap.equals(set.rangeBitSetMap); + } + @Override public int cardinality(long lowerKey, long lowerValue, long upperKey, long upperValue) { NavigableMap subMap = rangeBitSetMap.subMap(lowerKey, true, upperKey, true); @@ -417,4 +458,4 @@ private BitSet createNewBitSet() { return this.threadSafe ? new ConcurrentBitSet(bitSetSize) : new BitSet(bitSetSize); } -} +} \ No newline at end of file diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSet.java deleted file mode 100644 index 0718a4f81a61f..0000000000000 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSet.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util.collections; - -import java.util.NavigableMap; -import java.util.NavigableSet; -import java.util.Set; -import java.util.TreeSet; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.commons.lang.mutable.MutableInt; -import org.apache.commons.lang.mutable.MutableLong; -import org.apache.pulsar.common.util.collections.ConcurrentLongPairSet.LongPair; -import org.apache.pulsar.common.util.collections.ConcurrentLongPairSet.LongPairConsumer; - -/** - * Sorted concurrent {@link LongPairSet} which is not fully accurate in sorting. - * - * {@link ConcurrentSortedLongPairSet} creates separate {@link ConcurrentLongPairSet} for unique first-key of - * inserted item. So, it can iterate over all items by sorting on item's first key. However, item's second key will not - * be sorted. eg: - * - * 
      - *  insert: (1,2), (1,4), (2,1), (1,5), (2,6)
- *  while iterating set will first read all the entries for items whose first-key=1 and then first-key=2.
- *  output: (1,4), (1,5), (1,2), (2,6), (2,1)
- *
      - * - *

      This map can be expensive and not recommended if set has to store large number of unique item.first's key - * because set has to create that many {@link ConcurrentLongPairSet} objects. - */ -public class ConcurrentSortedLongPairSet implements LongPairSet { - - protected final NavigableMap longPairSets = new ConcurrentSkipListMap<>(); - private final int expectedItems; - private final int concurrencyLevel; - /** - * If {@link #longPairSets} adds and removes the item-set frequently then it allocates and removes - * {@link ConcurrentLongPairSet} for the same item multiple times which can lead to gc-puases. To avoid such - * situation, avoid removing empty LogPairSet until it reaches max limit. - */ - private final int maxAllowedSetOnRemove; - private final boolean autoShrink; - private static final int DEFAULT_MAX_ALLOWED_SET_ON_REMOVE = 10; - - public ConcurrentSortedLongPairSet() { - this(16, 1, DEFAULT_MAX_ALLOWED_SET_ON_REMOVE); - } - - public ConcurrentSortedLongPairSet(int expectedItems) { - this(expectedItems, 1, DEFAULT_MAX_ALLOWED_SET_ON_REMOVE); - } - - public ConcurrentSortedLongPairSet(int expectedItems, int concurrencyLevel) { - this(expectedItems, concurrencyLevel, DEFAULT_MAX_ALLOWED_SET_ON_REMOVE); - } - - public ConcurrentSortedLongPairSet(int expectedItems, int concurrencyLevel, boolean autoShrink) { - this(expectedItems, concurrencyLevel, DEFAULT_MAX_ALLOWED_SET_ON_REMOVE, autoShrink); - } - - public ConcurrentSortedLongPairSet(int expectedItems, int concurrencyLevel, int maxAllowedSetOnRemove) { - this(expectedItems, concurrencyLevel, maxAllowedSetOnRemove, false); - } - - public ConcurrentSortedLongPairSet(int expectedItems, int concurrencyLevel, int maxAllowedSetOnRemove, - boolean autoShrink) { - this.expectedItems = expectedItems; - this.concurrencyLevel = concurrencyLevel; - this.maxAllowedSetOnRemove = maxAllowedSetOnRemove; - this.autoShrink = autoShrink; - } - - @Override - public boolean add(long item1, long item2) { - ConcurrentLongPairSet messagesToReplay = longPairSets.computeIfAbsent(item1, - (key) -> ConcurrentLongPairSet.newBuilder() - .expectedItems(expectedItems) - .concurrencyLevel(concurrencyLevel) - .autoShrink(autoShrink) - .build()); - return messagesToReplay.add(item1, item2); - } - - @Override - public boolean remove(long item1, long item2) { - ConcurrentLongPairSet messagesToReplay = longPairSets.get(item1); - if (messagesToReplay != null) { - boolean removed = messagesToReplay.remove(item1, item2); - if (messagesToReplay.isEmpty() && longPairSets.size() > maxAllowedSetOnRemove) { - longPairSets.remove(item1, messagesToReplay); - } - return removed; - } - return false; - } - - @Override - public int removeIf(LongPairPredicate filter) { - MutableInt removedValues = new MutableInt(0); - longPairSets.forEach((item1, longPairSet) -> { - removedValues.add(longPairSet.removeIf(filter)); - if (longPairSet.isEmpty() && longPairSets.size() > maxAllowedSetOnRemove) { - longPairSets.remove(item1, longPairSet); - } - }); - return removedValues.intValue(); - } - - @Override - public Set items() { - return items((int) this.size()); - } - - @Override - public void forEach(LongPairConsumer processor) { - longPairSets.forEach((__, longPairSet) -> longPairSet.forEach(processor)); - } - - @Override - public Set items(int numberOfItems) { - return items(numberOfItems, (item1, item2) -> new LongPair(item1, item2)); - } - - @Override - public Set items(int numberOfItems, LongPairFunction longPairConverter) { - NavigableSet items = new TreeSet<>(); - forEach((i1, i2) -> { - items.add(longPairConverter.apply(i1, i2)); - if (items.size() > numberOfItems) { - items.pollLast(); - } - }); - return items; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append('{'); - final AtomicBoolean first = new AtomicBoolean(true); - longPairSets.forEach((key, longPairSet) -> { - longPairSet.forEach((item1, item2) -> { - if (!first.getAndSet(false)) { - sb.append(", "); - } - sb.append('['); - sb.append(item1); - sb.append(':'); - sb.append(item2); - sb.append(']'); - }); - }); - sb.append('}'); - return sb.toString(); - } - - @Override - public boolean isEmpty() { - if (longPairSets.isEmpty()) { - return true; - } - for (ConcurrentLongPairSet subSet : longPairSets.values()) { - if (!subSet.isEmpty()) { - return false; - } - } - return true; - } - - @Override - public void clear() { - longPairSets.clear(); - } - - @Override - public long size() { - MutableLong size = new MutableLong(0); - longPairSets.forEach((__, longPairSet) -> size.add(longPairSet.size())); - return size.longValue(); - } - - @Override - public long capacity() { - MutableLong capacity = new MutableLong(0); - longPairSets.forEach((__, longPairSet) -> capacity.add(longPairSet.capacity())); - return capacity.longValue(); - } - - @Override - public boolean contains(long item1, long item2) { - ConcurrentLongPairSet longPairSet = longPairSets.get(item1); - if (longPairSet != null) { - return longPairSet.contains(item1, item2); - } - return false; - } - -} \ No newline at end of file diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongPairRangeSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongPairRangeSet.java index 8aad5587dfd38..df74857245bb3 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongPairRangeSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongPairRangeSet.java @@ -25,6 +25,7 @@ import java.util.Collection; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; import lombok.EqualsAndHashCode; @@ -136,6 +137,19 @@ public interface LongPairRangeSet> { */ Range lastRange(); + default Map toRanges(int maxRanges) { + throw new UnsupportedOperationException(); + } + + /** + * Build {@link LongPairRangeSet} using internal ranges returned by {@link #toRanges(int)} . + * + * @param ranges + */ + default void build(Map ranges) { + throw new UnsupportedOperationException(); + } + /** * Return the number bit sets to true from lower (inclusive) to upper (inclusive). */ diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongPairSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongPairSet.java index 3750d8c22020f..e699d01b9c21c 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongPairSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongPairSet.java @@ -96,7 +96,7 @@ public interface LongPairSet { /** * Predicate to checks for a key-value pair where both of them have long types. */ - public interface LongPairPredicate { + interface LongPairPredicate { boolean test(long v1, long v2); } @@ -132,7 +132,7 @@ public interface LongPairPredicate { * */ @FunctionalInterface - public interface LongPairFunction { + interface LongPairFunction { /** * Applies this function to the given arguments. diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/OpenLongPairRangeSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/OpenLongPairRangeSet.java new file mode 100644 index 0000000000000..3076c6c5c5fa1 --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/OpenLongPairRangeSet.java @@ -0,0 +1,463 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util.collections; + +import static java.util.BitSet.valueOf; +import static java.util.Objects.requireNonNull; +import com.google.common.collect.BoundType; +import com.google.common.collect.Range; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.NavigableMap; +import java.util.Objects; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; +import javax.annotation.concurrent.NotThreadSafe; +import org.apache.commons.lang.mutable.MutableInt; + +/** + * A Concurrent set comprising zero or more ranges of type {@link LongPair}. This can be alternative of + * {@link com.google.common.collect.RangeSet} and can be used if {@code range} type is {@link LongPair} + * + * 

      + * Usage:
+ * a. This can be used if one doesn't want to create object for every new inserted {@code range}
+ * b. It creates {@link BitSet} for every unique first-key of the range.
+ * So, this rangeSet is not suitable for large number of unique keys.
+ *
      + */ +@NotThreadSafe +public class OpenLongPairRangeSet> implements LongPairRangeSet { + + protected final NavigableMap rangeBitSetMap = new ConcurrentSkipListMap<>(); + private final LongPairConsumer consumer; + private final Supplier bitSetSupplier; + + // caching place-holder for cpu-optimization to avoid calculating ranges again + private volatile int cachedSize = 0; + private volatile String cachedToString = "[]"; + private volatile boolean updatedAfterCachedForSize = true; + private volatile boolean updatedAfterCachedForToString = true; + + public OpenLongPairRangeSet(LongPairConsumer consumer) { + this(consumer, BitSet::new); + } + + public OpenLongPairRangeSet(LongPairConsumer consumer, Supplier bitSetSupplier) { + this.consumer = consumer; + this.bitSetSupplier = bitSetSupplier; + } + + /** + * Adds the specified range to this {@code RangeSet} (optional operation). That is, for equal range sets a and b, + * the result of {@code a.add(range)} is that {@code a} will be the minimal range set for which both + * {@code a.enclosesAll(b)} and {@code a.encloses(range)}. + * + *

      Note that {@code range} will merge given {@code range} with any ranges in the range set that are + * {@linkplain Range#isConnected(Range) connected} with it. Moreover, if {@code range} is empty, this is a no-op. + */ + @Override + public void addOpenClosed(long lowerKey, long lowerValueOpen, long upperKey, long upperValue) { + long lowerValue = lowerValueOpen + 1; + if (lowerKey != upperKey) { + // (1) set lower to last in lowerRange.getKey() + if (isValid(lowerKey, lowerValue)) { + BitSet rangeBitSet = rangeBitSetMap.get(lowerKey); + // if lower and upper has different key/ledger then set ranges for lower-key only if + // a. bitSet already exist and given value is not the last value in the bitset. + // it will prevent setting up values which are not actually expected to set + // eg: (2:10..4:10] in this case, don't set any value for 2:10 and set [4:0..4:10] + if (rangeBitSet != null && (rangeBitSet.previousSetBit(rangeBitSet.size()) > lowerValueOpen)) { + int lastValue = rangeBitSet.previousSetBit(rangeBitSet.size()); + rangeBitSet.set((int) lowerValue, (int) Math.max(lastValue, lowerValue) + 1); + } + } + // (2) set 0th-index to upper-index in upperRange.getKey() + if (isValid(upperKey, upperValue)) { + BitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(upperKey, (key) -> createNewBitSet()); + if (rangeBitSet != null) { + rangeBitSet.set(0, (int) upperValue + 1); + } + } + // No-op if values are not valid eg: if lower == LongPair.earliest or upper == LongPair.latest then nothing + // to set + } else { + long key = lowerKey; + BitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(key, (k) -> createNewBitSet()); + rangeBitSet.set((int) lowerValue, (int) upperValue + 1); + } + updatedAfterCachedForSize = true; + updatedAfterCachedForToString = true; + } + + private boolean isValid(long key, long value) { + return key != LongPair.earliest.getKey() && value != LongPair.earliest.getValue() + && key != LongPair.latest.getKey() && value != LongPair.latest.getValue(); + } + + @Override + public boolean contains(long key, long value) { + + BitSet rangeBitSet = rangeBitSetMap.get(key); + if (rangeBitSet != null) { + return rangeBitSet.get(getSafeEntry(value)); + } + return false; + } + + @Override + public Range rangeContaining(long key, long value) { + BitSet rangeBitSet = rangeBitSetMap.get(key); + if (rangeBitSet != null) { + if (!rangeBitSet.get(getSafeEntry(value))) { + // if position is not part of any range then return null + return null; + } + int lowerValue = rangeBitSet.previousClearBit(getSafeEntry(value)) + 1; + final T lower = consumer.apply(key, lowerValue); + final T upper = consumer.apply(key, + Math.max(rangeBitSet.nextClearBit(getSafeEntry(value)) - 1, lowerValue)); + return Range.closed(lower, upper); + } + return null; + } + + @Override + public void removeAtMost(long key, long value) { + this.remove(Range.atMost(new LongPair(key, value))); + } + + @Override + public boolean isEmpty() { + if (rangeBitSetMap.isEmpty()) { + return true; + } + for (BitSet rangeBitSet : rangeBitSetMap.values()) { + if (!rangeBitSet.isEmpty()) { + return false; + } + } + return true; + } + + @Override + public void clear() { + rangeBitSetMap.clear(); + updatedAfterCachedForSize = true; + updatedAfterCachedForToString = true; + } + + @Override + public Range span() { + if (rangeBitSetMap.isEmpty()) { + return null; + } + Entry firstSet = rangeBitSetMap.firstEntry(); + Entry lastSet = rangeBitSetMap.lastEntry(); + int first = firstSet.getValue().nextSetBit(0); + int last = lastSet.getValue().previousSetBit(lastSet.getValue().size()); + return Range.openClosed(consumer.apply(firstSet.getKey(), first - 1), consumer.apply(lastSet.getKey(), last)); + } + + @Override + public List> asRanges() { + List> ranges = new ArrayList<>(); + forEach((range) -> { + ranges.add(range); + return true; + }); + return ranges; + } + + @Override + public void forEach(RangeProcessor action) { + forEach(action, consumer); + } + + @Override + public void forEach(RangeProcessor action, LongPairConsumer consumerParam) { + forEachRawRange((lowerKey, lowerValue, upperKey, upperValue) -> { + Range range = Range.openClosed( + consumerParam.apply(lowerKey, lowerValue), + consumerParam.apply(upperKey, upperValue) + ); + return action.process(range); + }); + } + + @Override + public void forEachRawRange(RawRangeProcessor processor) { + AtomicBoolean completed = new AtomicBoolean(false); + rangeBitSetMap.forEach((key, set) -> { + if (completed.get()) { + return; + } + if (set.isEmpty()) { + return; + } + int first = set.nextSetBit(0); + int last = set.previousSetBit(set.size()); + int currentClosedMark = first; + while (currentClosedMark != -1 && currentClosedMark <= last) { + int nextOpenMark = set.nextClearBit(currentClosedMark); + if (!processor.processRawRange(key, currentClosedMark - 1, + key, nextOpenMark - 1)) { + completed.set(true); + break; + } + currentClosedMark = set.nextSetBit(nextOpenMark); + } + }); + } + + + @Override + public Range firstRange() { + if (rangeBitSetMap.isEmpty()) { + return null; + } + Entry firstSet = rangeBitSetMap.firstEntry(); + int lower = firstSet.getValue().nextSetBit(0); + int upper = Math.max(lower, firstSet.getValue().nextClearBit(lower) - 1); + return Range.openClosed(consumer.apply(firstSet.getKey(), lower - 1), consumer.apply(firstSet.getKey(), upper)); + } + + @Override + public Range lastRange() { + if (rangeBitSetMap.isEmpty()) { + return null; + } + Entry lastSet = rangeBitSetMap.lastEntry(); + int upper = lastSet.getValue().previousSetBit(lastSet.getValue().size()); + int lower = Math.min(lastSet.getValue().previousClearBit(upper), upper); + return Range.openClosed(consumer.apply(lastSet.getKey(), lower), consumer.apply(lastSet.getKey(), upper)); + } + + @Override + public Map toRanges(int maxRanges) { + Map internalBitSetMap = new HashMap<>(); + AtomicInteger rangeCount = new AtomicInteger(); + rangeBitSetMap.forEach((id, bmap) -> { + if (rangeCount.getAndAdd(bmap.cardinality()) > maxRanges) { + return; + } + internalBitSetMap.put(id, bmap.toLongArray()); + }); + return internalBitSetMap; + } + + @Override + public void build(Map internalRange) { + internalRange.forEach((id, ranges) -> { + BitSet bitset = createNewBitSet(); + bitset.or(valueOf(ranges)); + rangeBitSetMap.put(id, bitset); + }); + } + + + @Override + public int hashCode() { + return Objects.hashCode(rangeBitSetMap); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof OpenLongPairRangeSet)) { + return false; + } + if (this == obj) { + return true; + } + @SuppressWarnings("rawtypes") + OpenLongPairRangeSet set = (OpenLongPairRangeSet) obj; + return this.rangeBitSetMap.equals(set.rangeBitSetMap); + } + + @Override + public int cardinality(long lowerKey, long lowerValue, long upperKey, long upperValue) { + NavigableMap subMap = rangeBitSetMap.subMap(lowerKey, true, upperKey, true); + MutableInt v = new MutableInt(0); + subMap.forEach((key, bitset) -> { + if (key == lowerKey || key == upperKey) { + BitSet temp = (BitSet) bitset.clone(); + // Trim the bitset index which < lowerValue + if (key == lowerKey) { + temp.clear(0, (int) Math.max(0, lowerValue)); + } + // Trim the bitset index which > upperValue + if (key == upperKey) { + temp.clear((int) Math.min(upperValue + 1, temp.length()), temp.length()); + } + v.add(temp.cardinality()); + } else { + v.add(bitset.cardinality()); + } + }); + return v.intValue(); + } + + @Override + public int size() { + if (updatedAfterCachedForSize) { + MutableInt size = new MutableInt(0); + + // ignore result because we just want to count + forEachRawRange((lowerKey, lowerValue, upperKey, upperValue) -> { + size.increment(); + return true; + }); + + cachedSize = size.intValue(); + updatedAfterCachedForSize = false; + } + return cachedSize; + } + + @Override + public String toString() { + if (updatedAfterCachedForToString) { + StringBuilder toString = new StringBuilder(); + AtomicBoolean first = new AtomicBoolean(true); + if (toString != null) { + toString.append("["); + } + forEach((range) -> { + if (!first.get()) { + toString.append(","); + } + toString.append(range); + first.set(false); + return true; + }); + toString.append("]"); + cachedToString = toString.toString(); + updatedAfterCachedForToString = false; + } + return cachedToString; + } + + /** + * Adds the specified range to this {@code RangeSet} (optional operation). That is, for equal range sets a and b, + * the result of {@code a.add(range)} is that {@code a} will be the minimal range set for which both + * {@code a.enclosesAll(b)} and {@code a.encloses(range)}. + * + *

      Note that {@code range} will merge given {@code range} with any ranges in the range set that are + * {@linkplain Range#isConnected(Range) connected} with it. Moreover, if {@code range} is empty/invalid, this is a + * no-op. + */ + public void add(Range range) { + LongPair lowerEndpoint = range.hasLowerBound() ? range.lowerEndpoint() : LongPair.earliest; + LongPair upperEndpoint = range.hasUpperBound() ? range.upperEndpoint() : LongPair.latest; + + long lowerValueOpen = (range.hasLowerBound() && range.lowerBoundType().equals(BoundType.CLOSED)) + ? getSafeEntry(lowerEndpoint) - 1 + : getSafeEntry(lowerEndpoint); + long upperValueClosed = (range.hasUpperBound() && range.upperBoundType().equals(BoundType.CLOSED)) + ? getSafeEntry(upperEndpoint) + : getSafeEntry(upperEndpoint) + 1; + + // #addOpenClosed doesn't create bitSet for lower-key because it avoids setting up values for non-exist items + // into the key-ledger. so, create bitSet and initialize so, it can't be ignored at #addOpenClosed + rangeBitSetMap.computeIfAbsent(lowerEndpoint.getKey(), (key) -> createNewBitSet()) + .set((int) lowerValueOpen + 1); + this.addOpenClosed(lowerEndpoint.getKey(), lowerValueOpen, upperEndpoint.getKey(), upperValueClosed); + } + + public boolean contains(LongPair position) { + requireNonNull(position, "argument can't be null"); + return contains(position.getKey(), position.getValue()); + } + + public void remove(Range range) { + LongPair lowerEndpoint = range.hasLowerBound() ? range.lowerEndpoint() : LongPair.earliest; + LongPair upperEndpoint = range.hasUpperBound() ? range.upperEndpoint() : LongPair.latest; + + long lower = (range.hasLowerBound() && range.lowerBoundType().equals(BoundType.CLOSED)) + ? getSafeEntry(lowerEndpoint) + : getSafeEntry(lowerEndpoint) + 1; + long upper = (range.hasUpperBound() && range.upperBoundType().equals(BoundType.CLOSED)) + ? getSafeEntry(upperEndpoint) + : getSafeEntry(upperEndpoint) - 1; + + // if lower-bound is not set then remove all the keys less than given upper-bound range + if (lowerEndpoint.equals(LongPair.earliest)) { + // remove all keys with + rangeBitSetMap.forEach((key, set) -> { + if (key < upperEndpoint.getKey()) { + rangeBitSetMap.remove(key); + } + }); + } + + // if upper-bound is not set then remove all the keys greater than given lower-bound range + if (upperEndpoint.equals(LongPair.latest)) { + // remove all keys with + rangeBitSetMap.forEach((key, set) -> { + if (key > lowerEndpoint.getKey()) { + rangeBitSetMap.remove(key); + } + }); + } + + // remove all the keys between two endpoint keys + rangeBitSetMap.forEach((key, set) -> { + if (lowerEndpoint.getKey() == upperEndpoint.getKey() && key == upperEndpoint.getKey()) { + set.clear((int) lower, (int) upper + 1); + } else { + // eg: remove-range: [(3,5) - (5,5)] -> Delete all items from 3,6->3,N,4.*,5,0->5,5 + if (key == lowerEndpoint.getKey()) { + // remove all entries from given position to last position + set.clear((int) lower, set.previousSetBit(set.size())); + } else if (key == upperEndpoint.getKey()) { + // remove all entries from 0 to given position + set.clear(0, (int) upper + 1); + } else if (key > lowerEndpoint.getKey() && key < upperEndpoint.getKey()) { + rangeBitSetMap.remove(key); + } + } + // remove bit-set if set is empty + if (set.isEmpty()) { + rangeBitSetMap.remove(key); + } + }); + + updatedAfterCachedForSize = true; + updatedAfterCachedForToString = true; + } + + private int getSafeEntry(LongPair position) { + return (int) Math.max(position.getValue(), -1); + } + + private int getSafeEntry(long value) { + return (int) Math.max(value, -1); + } + + private BitSet createNewBitSet() { + return bitSetSupplier.get(); + } + +} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/SegmentedLongArray.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/SegmentedLongArray.java index 0b3520983b2ba..c551895c51a92 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/SegmentedLongArray.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/SegmentedLongArray.java @@ -19,11 +19,11 @@ package org.apache.pulsar.common.util.collections; import io.netty.buffer.ByteBuf; -import io.netty.buffer.PooledByteBufAllocator; import java.util.ArrayList; import java.util.List; import javax.annotation.concurrent.NotThreadSafe; import lombok.Getter; +import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; @NotThreadSafe public class SegmentedLongArray implements AutoCloseable { @@ -44,14 +44,14 @@ public SegmentedLongArray(long initialCapacity) { // Add first segment int sizeToAdd = (int) Math.min(remainingToAdd, MAX_SEGMENT_SIZE); - ByteBuf buffer = PooledByteBufAllocator.DEFAULT.directBuffer(sizeToAdd * SIZE_OF_LONG); + ByteBuf buffer = PulsarByteBufAllocator.DEFAULT.directBuffer(sizeToAdd * SIZE_OF_LONG); buffer.writerIndex(sizeToAdd * SIZE_OF_LONG); buffers.add(buffer); remainingToAdd -= sizeToAdd; // Add the remaining segments, all at full segment size, if necessary while (remainingToAdd > 0) { - buffer = PooledByteBufAllocator.DEFAULT.directBuffer(MAX_SEGMENT_SIZE * SIZE_OF_LONG); + buffer = PulsarByteBufAllocator.DEFAULT.directBuffer(MAX_SEGMENT_SIZE * SIZE_OF_LONG); buffer.writerIndex(MAX_SEGMENT_SIZE * SIZE_OF_LONG); buffers.add(buffer); remainingToAdd -= MAX_SEGMENT_SIZE; @@ -83,7 +83,7 @@ public void increaseCapacity() { } else { // Let's add 1 mode buffer to the list int bufferSize = MAX_SEGMENT_SIZE * SIZE_OF_LONG; - ByteBuf buffer = PooledByteBufAllocator.DEFAULT.directBuffer(bufferSize, bufferSize); + ByteBuf buffer = PulsarByteBufAllocator.DEFAULT.directBuffer(bufferSize, bufferSize); buffer.writerIndex(bufferSize); buffers.add(buffer); capacity += MAX_SEGMENT_SIZE; @@ -107,7 +107,7 @@ public void shrink(long newCapacity) { // We should also reduce the capacity of the first buffer capacity -= sizeToReduce; ByteBuf oldBuffer = buffers.get(0); - ByteBuf newBuffer = PooledByteBufAllocator.DEFAULT.directBuffer((int) capacity * SIZE_OF_LONG); + ByteBuf newBuffer = PulsarByteBufAllocator.DEFAULT.directBuffer((int) capacity * SIZE_OF_LONG); oldBuffer.getBytes(0, newBuffer, (int) capacity * SIZE_OF_LONG); oldBuffer.release(); buffers.set(0, newBuffer); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java index c717127d085db..a70857bdf3b5f 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java @@ -201,7 +201,11 @@ private SSLEngine configureSSLEngine(SSLEngine sslEngine) { } if (this.mode == Mode.SERVER) { - sslEngine.setNeedClientAuth(this.needClientAuth); + if (needClientAuth) { + sslEngine.setNeedClientAuth(true); + } else { + sslEngine.setWantClientAuth(true); + } sslEngine.setUseClientMode(false); } else { sslEngine.setUseClientMode(true); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/NetSslContextBuilder.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/NetSslContextBuilder.java deleted file mode 100644 index 3d4d4e72546ea..0000000000000 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/NetSslContextBuilder.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util.keystoretls; - -import java.io.IOException; -import java.security.GeneralSecurityException; -import javax.net.ssl.SSLContext; -import org.apache.pulsar.common.util.FileModifiedTimeUpdater; -import org.apache.pulsar.common.util.SslContextAutoRefreshBuilder; - -/** - * Similar to `DefaultSslContextBuilder`, which build `javax.net.ssl.SSLContext` for web service. - */ -public class NetSslContextBuilder extends SslContextAutoRefreshBuilder { - private volatile SSLContext sslContext; - - protected final boolean tlsAllowInsecureConnection; - protected final boolean tlsRequireTrustedClientCertOnConnect; - - protected final String tlsProvider; - protected final String tlsKeyStoreType; - protected final String tlsKeyStorePassword; - protected final FileModifiedTimeUpdater tlsKeyStore; - protected final String tlsTrustStoreType; - protected final String tlsTrustStorePassword; - protected final FileModifiedTimeUpdater tlsTrustStore; - - public NetSslContextBuilder(String sslProviderString, - String keyStoreTypeString, - String keyStore, - String keyStorePasswordPath, - boolean allowInsecureConnection, - String trustStoreTypeString, - String trustStore, - String trustStorePasswordPath, - boolean requireTrustedClientCertOnConnect, - long certRefreshInSec) { - super(certRefreshInSec); - - this.tlsAllowInsecureConnection = allowInsecureConnection; - this.tlsProvider = sslProviderString; - this.tlsKeyStoreType = keyStoreTypeString; - this.tlsKeyStore = new FileModifiedTimeUpdater(keyStore); - this.tlsKeyStorePassword = keyStorePasswordPath; - - this.tlsTrustStoreType = trustStoreTypeString; - this.tlsTrustStore = new FileModifiedTimeUpdater(trustStore); - this.tlsTrustStorePassword = trustStorePasswordPath; - - this.tlsRequireTrustedClientCertOnConnect = requireTrustedClientCertOnConnect; - } - - @Override - public synchronized SSLContext update() - throws GeneralSecurityException, IOException { - this.sslContext = KeyStoreSSLContext.createServerSslContext(tlsProvider, - tlsKeyStoreType, tlsKeyStore.getFileName(), tlsKeyStorePassword, - tlsAllowInsecureConnection, - tlsTrustStoreType, tlsTrustStore.getFileName(), tlsTrustStorePassword, - tlsRequireTrustedClientCertOnConnect); - return this.sslContext; - } - - @Override - public SSLContext getSslContext() { - return this.sslContext; - } - - @Override - public boolean needUpdate() { - return tlsKeyStore.checkAndRefresh() - || tlsTrustStore.checkAndRefresh(); - } -} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/NettySSLContextAutoRefreshBuilder.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/NettySSLContextAutoRefreshBuilder.java deleted file mode 100644 index 6d0cfb108bd0e..0000000000000 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/NettySSLContextAutoRefreshBuilder.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util.keystoretls; - -import java.io.IOException; -import java.security.GeneralSecurityException; -import java.util.Set; -import org.apache.pulsar.client.api.AuthenticationDataProvider; -import org.apache.pulsar.client.api.KeyStoreParams; -import org.apache.pulsar.common.util.FileModifiedTimeUpdater; -import org.apache.pulsar.common.util.SslContextAutoRefreshBuilder; - -/** - * SSL context builder for Netty. - */ -public class NettySSLContextAutoRefreshBuilder extends SslContextAutoRefreshBuilder { - private volatile KeyStoreSSLContext keyStoreSSLContext; - - protected final boolean tlsAllowInsecureConnection; - protected final Set tlsCiphers; - protected final Set tlsProtocols; - protected boolean tlsRequireTrustedClientCertOnConnect; - - protected final String tlsProvider; - protected final String tlsTrustStoreType; - protected final String tlsTrustStorePassword; - protected final FileModifiedTimeUpdater tlsTrustStore; - - // client context not need keystore at start time, keyStore is passed in by authData. - protected String tlsKeyStoreType; - protected String tlsKeyStorePassword; - protected FileModifiedTimeUpdater tlsKeyStore; - - protected final boolean isServer; - - // for server - public NettySSLContextAutoRefreshBuilder(String sslProviderString, - String keyStoreTypeString, - String keyStore, - String keyStorePassword, - boolean allowInsecureConnection, - String trustStoreTypeString, - String trustStore, - String trustStorePassword, - boolean requireTrustedClientCertOnConnect, - Set ciphers, - Set protocols, - long certRefreshInSec) { - super(certRefreshInSec); - - this.tlsAllowInsecureConnection = allowInsecureConnection; - this.tlsProvider = sslProviderString; - - this.tlsKeyStoreType = keyStoreTypeString; - this.tlsKeyStore = new FileModifiedTimeUpdater(keyStore); - this.tlsKeyStorePassword = keyStorePassword; - - this.tlsTrustStoreType = trustStoreTypeString; - this.tlsTrustStore = new FileModifiedTimeUpdater(trustStore); - this.tlsTrustStorePassword = trustStorePassword; - - this.tlsRequireTrustedClientCertOnConnect = requireTrustedClientCertOnConnect; - this.tlsCiphers = ciphers; - this.tlsProtocols = protocols; - - this.isServer = true; - } - - // for client - public NettySSLContextAutoRefreshBuilder(String sslProviderString, - boolean allowInsecureConnection, - String trustStoreTypeString, - String trustStore, - String trustStorePassword, - String keyStoreTypeString, - String keyStore, - String keyStorePassword, - Set ciphers, - Set protocols, - long certRefreshInSec, - AuthenticationDataProvider authData) { - super(certRefreshInSec); - - this.tlsAllowInsecureConnection = allowInsecureConnection; - this.tlsProvider = sslProviderString; - - if (authData != null) { - KeyStoreParams authParams = authData.getTlsKeyStoreParams(); - if (authParams != null) { - keyStoreTypeString = authParams.getKeyStoreType(); - keyStore = authParams.getKeyStorePath(); - keyStorePassword = authParams.getKeyStorePassword(); - } - } - this.tlsKeyStoreType = keyStoreTypeString; - this.tlsKeyStore = new FileModifiedTimeUpdater(keyStore); - this.tlsKeyStorePassword = keyStorePassword; - - this.tlsTrustStoreType = trustStoreTypeString; - this.tlsTrustStore = new FileModifiedTimeUpdater(trustStore); - this.tlsTrustStorePassword = trustStorePassword; - - this.tlsCiphers = ciphers; - this.tlsProtocols = protocols; - - this.isServer = false; - } - - @Override - public synchronized KeyStoreSSLContext update() throws GeneralSecurityException, IOException { - if (isServer) { - this.keyStoreSSLContext = KeyStoreSSLContext.createServerKeyStoreSslContext(tlsProvider, - tlsKeyStoreType, tlsKeyStore.getFileName(), tlsKeyStorePassword, - tlsAllowInsecureConnection, - tlsTrustStoreType, tlsTrustStore.getFileName(), tlsTrustStorePassword, - tlsRequireTrustedClientCertOnConnect, tlsCiphers, tlsProtocols); - } else { - this.keyStoreSSLContext = KeyStoreSSLContext.createClientKeyStoreSslContext(tlsProvider, - tlsKeyStoreType, - tlsKeyStore.getFileName(), - tlsKeyStorePassword, - tlsAllowInsecureConnection, - tlsTrustStoreType, tlsTrustStore.getFileName(), tlsTrustStorePassword, - tlsCiphers, tlsProtocols); - } - return this.keyStoreSSLContext; - } - - @Override - public KeyStoreSSLContext getSslContext() { - return this.keyStoreSSLContext; - } - - @Override - public boolean needUpdate() { - return (tlsKeyStore != null && tlsKeyStore.checkAndRefresh()) - || (tlsTrustStore != null && tlsTrustStore.checkAndRefresh()); - } -} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/netty/DnsResolverUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/netty/DnsResolverUtil.java index f49a6453c72b3..bcff83acd949f 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/netty/DnsResolverUtil.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/netty/DnsResolverUtil.java @@ -19,12 +19,20 @@ package org.apache.pulsar.common.util.netty; import io.netty.resolver.dns.DnsNameResolverBuilder; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; +import java.security.Security; +import java.util.Optional; import lombok.extern.slf4j.Slf4j; @Slf4j public class DnsResolverUtil { + + private static final String CACHE_POLICY_PROP = "networkaddress.cache.ttl"; + private static final String CACHE_POLICY_PROP_FALLBACK = "sun.net.inetaddr.ttl"; + private static final String NEGATIVE_CACHE_POLICY_PROP = "networkaddress.cache.negative.ttl"; + private static final String NEGATIVE_CACHE_POLICY_PROP_FALLBACK = "sun.net.inetaddr.negative.ttl"; + /* default ttl value from sun.net.InetAddressCachePolicy.DEFAULT_POSITIVE， which is used when no security manager + is used */ + private static final int JDK_DEFAULT_TTL = 30; private static final int MIN_TTL = 0; private static final int TTL; private static final int NEGATIVE_TTL; @@ -39,19 +47,35 @@ public class DnsResolverUtil { int ttl = DEFAULT_TTL; int negativeTtl = DEFAULT_NEGATIVE_TTL; try { - // use reflection to call sun.net.InetAddressCachePolicy's get and getNegative methods for getting - // effective JDK settings for DNS caching - Class inetAddressCachePolicyClass = Class.forName("sun.net.InetAddressCachePolicy"); - Method getTTLMethod = inetAddressCachePolicyClass.getMethod("get"); - ttl = (Integer) getTTLMethod.invoke(null); - Method getNegativeTTLMethod = inetAddressCachePolicyClass.getMethod("getNegative"); - negativeTtl = (Integer) getNegativeTTLMethod.invoke(null); - } catch (NoSuchMethodException | ClassNotFoundException | InvocationTargetException - | IllegalAccessException e) { - log.warn("Cannot get DNS TTL settings from sun.net.InetAddressCachePolicy class", e); + String ttlStr = Security.getProperty(CACHE_POLICY_PROP); + if (ttlStr == null) { + // Compatible with sun.net.inetaddr.ttl settings + ttlStr = System.getProperty(CACHE_POLICY_PROP_FALLBACK); + } + String negativeTtlStr = Security.getProperty(NEGATIVE_CACHE_POLICY_PROP); + if (negativeTtlStr == null) { + // Compatible with sun.net.inetaddr.negative.ttl settings + negativeTtlStr = System.getProperty(NEGATIVE_CACHE_POLICY_PROP_FALLBACK); + } + ttl = Optional.ofNullable(ttlStr) + .map(Integer::decode) + .filter(i -> i > 0) + .orElseGet(() -> { + if (System.getSecurityManager() == null) { + return JDK_DEFAULT_TTL; + } + return DEFAULT_TTL; + }); + + negativeTtl = Optional.ofNullable(negativeTtlStr) + .map(Integer::decode) + .filter(i -> i >= 0) + .orElse(DEFAULT_NEGATIVE_TTL); + } catch (NumberFormatException e) { + log.warn("Cannot get DNS TTL settings", e); } - TTL = ttl <= 0 ? DEFAULT_TTL : ttl; - NEGATIVE_TTL = negativeTtl < 0 ? DEFAULT_NEGATIVE_TTL : negativeTtl; + TTL = ttl; + NEGATIVE_TTL = negativeTtl; } private DnsResolverUtil() { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/BundleData.java b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/BundleData.java index e5e32046e4970..3c03b7b79bc07 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/BundleData.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/BundleData.java @@ -18,10 +18,13 @@ */ package org.apache.pulsar.policies.data.loadbalancer; +import lombok.EqualsAndHashCode; + /** * Data class comprising the short term and long term historical data for this bundle. */ -public class BundleData { +@EqualsAndHashCode +public class BundleData implements Comparable { // Short term data for this bundle. The time frame of this data is // determined by the number of short term samples // and the bundle update period. @@ -103,4 +106,13 @@ public int getTopics() { public void setTopics(int topics) { this.topics = topics; } + + @Override + public int compareTo(BundleData o) { + int result = this.shortTermData.compareTo(o.shortTermData); + if (result == 0) { + result = this.longTermData.compareTo(o.longTermData); + } + return result; + } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LoadReport.java b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LoadReport.java index 6e519a3f0735f..e6459e051bfc2 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LoadReport.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LoadReport.java @@ -50,7 +50,7 @@ public class LoadReport implements LoadManagerReport { private long timestamp; private double msgRateIn; private double msgRateOut; - private int numTopics; + private long numTopics; private int numConsumers; private int numProducers; private int numBundles; @@ -205,7 +205,7 @@ public String getLoadReportType() { } @Override - public int getNumTopics() { + public long getNumTopics() { numTopics = 0; if (this.bundleStats != null) { this.bundleStats.forEach((bundle, stats) -> { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java index df85a4d989f99..7fd0140bab22f 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java @@ -66,7 +66,7 @@ public class LocalBrokerData implements LoadManagerReport { // The stats given in the most recent invocation of update. private Map lastStats; - private int numTopics; + private long numTopics; private int numBundles; private int numConsumers; private int numProducers; @@ -202,7 +202,7 @@ private void updateBundleData(final Map bundleStat msgRateOut = 0; msgThroughputIn = 0; msgThroughputOut = 0; - int totalNumTopics = 0; + long totalNumTopics = 0; int totalNumBundles = 0; int totalNumConsumers = 0; int totalNumProducers = 0; @@ -253,14 +253,7 @@ public String printResourceUsage() { cpu.percentUsage(), memory.percentUsage(), directMemory.percentUsage(), bandwidthIn.percentUsage(), bandwidthOut.percentUsage()); } - @Deprecated - public double getMaxResourceUsageWithWeight(final double cpuWeight, final double memoryWeight, - final double directMemoryWeight, final double bandwidthInWeight, - final double bandwidthOutWeight) { - return max(cpu.percentUsage() * cpuWeight, memory.percentUsage() * memoryWeight, - directMemory.percentUsage() * directMemoryWeight, bandwidthIn.percentUsage() * bandwidthInWeight, - bandwidthOut.percentUsage() * bandwidthOutWeight) / 100; - } + public double getMaxResourceUsageWithWeight(final double cpuWeight, final double directMemoryWeight, final double bandwidthInWeight, final double bandwidthOutWeight) { @@ -389,7 +382,7 @@ public void setLastStats(Map lastStats) { } @Override - public int getNumTopics() { + public long getNumTopics() { return numTopics; } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/TimeAverageMessageData.java b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/TimeAverageMessageData.java index 777a6684ce81e..b9c7a43c3a7a0 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/TimeAverageMessageData.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/TimeAverageMessageData.java @@ -18,10 +18,13 @@ */ package org.apache.pulsar.policies.data.loadbalancer; +import lombok.EqualsAndHashCode; + /** * Data class comprising the average message data over a fixed period of time. */ -public class TimeAverageMessageData { +@EqualsAndHashCode +public class TimeAverageMessageData implements Comparable { // The maximum number of samples this data will consider. private int maxSamples; @@ -41,6 +44,11 @@ public class TimeAverageMessageData { // The average message rate out per second. private double msgRateOut; + // Consider the throughput equal if difference is less than 100 KB/s + private static final double throughputDifferenceThreshold = 1e5; + // Consider the msgRate equal if the difference is less than 100 + private static final double msgRateDifferenceThreshold = 100; + // For JSON only. public TimeAverageMessageData() { } @@ -177,4 +185,40 @@ public double totalMsgRate() { public double totalMsgThroughput() { return msgThroughputIn + msgThroughputOut; } + + @Override + public int compareTo(TimeAverageMessageData other) { + int result = this.compareByBandwidthIn(other); + + if (result == 0) { + result = this.compareByBandwidthOut(other); + } + if (result == 0) { + result = this.compareByMsgRate(other); + } + return result; + } + + public int compareByMsgRate(TimeAverageMessageData other) { + double thisMsgRate = this.msgRateIn + this.msgRateOut; + double otherMsgRate = other.msgRateIn + other.msgRateOut; + if (Math.abs(thisMsgRate - otherMsgRate) > msgRateDifferenceThreshold) { + return Double.compare(thisMsgRate, otherMsgRate); + } + return 0; + } + + public int compareByBandwidthIn(TimeAverageMessageData other) { + if (Math.abs(this.msgThroughputIn - other.msgThroughputIn) > throughputDifferenceThreshold) { + return Double.compare(this.msgThroughputIn, other.msgThroughputIn); + } + return 0; + } + + public int compareByBandwidthOut(TimeAverageMessageData other) { + if (Math.abs(this.msgThroughputOut - other.msgThroughputOut) > throughputDifferenceThreshold) { + return Double.compare(this.msgThroughputOut, other.msgThroughputOut); + } + return 0; + } } diff --git a/pulsar-common/src/main/proto/PulsarApi.proto b/pulsar-common/src/main/proto/PulsarApi.proto index afe193eeb7e9d..19658c5e57ff9 100644 --- a/pulsar-common/src/main/proto/PulsarApi.proto +++ b/pulsar-common/src/main/proto/PulsarApi.proto @@ -295,11 +295,13 @@ message CommandConnect { optional string proxy_version = 11; // Version of the proxy. Should only be forwarded by a proxy. } +// Please also add a new enum for the class "PulsarClientException.FailedFeatureCheck" when adding a new feature flag. message FeatureFlags { optional bool supports_auth_refresh = 1 [default = false]; optional bool supports_broker_entry_metadata = 2 [default = false]; optional bool supports_partial_producer = 3 [default = false]; optional bool supports_topic_watchers = 4 [default = false]; + optional bool supports_get_partitioned_metadata_without_auto_creation = 5 [default = false]; } message CommandConnected { @@ -413,6 +415,7 @@ message CommandPartitionedTopicMetadata { // to the proxy. optional string original_auth_data = 4; optional string original_auth_method = 5; + optional bool metadata_auto_creation_enabled = 6 [default = true]; } message CommandPartitionedTopicMetadataResponse { @@ -443,6 +446,8 @@ message CommandLookupTopic { optional string original_auth_method = 6; // optional string advertised_listener_name = 7; + // The properties used for topic lookup + repeated KeyValue properties = 8; } message CommandLookupTopicResponse { @@ -607,6 +612,7 @@ message CommandFlow { message CommandUnsubscribe { required uint64 consumer_id = 1; required uint64 request_id = 2; + optional bool force = 3 [default = false]; } // Reset an existing consumer to a particular message id @@ -641,11 +647,15 @@ message CommandTopicMigrated { message CommandCloseProducer { required uint64 producer_id = 1; required uint64 request_id = 2; + optional string assignedBrokerServiceUrl = 3; + optional string assignedBrokerServiceUrlTls = 4; } message CommandCloseConsumer { required uint64 consumer_id = 1; required uint64 request_id = 2; + optional string assignedBrokerServiceUrl = 3; + optional string assignedBrokerServiceUrlTls = 4; } message CommandRedeliverUnacknowledgedMessages { diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/compression/CommandsTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/compression/CommandsTest.java index 42f1a58100283..a1f79b7ae7faf 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/compression/CommandsTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/compression/CommandsTest.java @@ -98,9 +98,11 @@ private int computeChecksum(MessageMetadata msgMetadata, ByteBuf compressedPaylo public void testPeekStickyKey() { String message = "msg-1"; String partitionedKey = "key1"; + String producerName = "testProducer"; + int sequenceId = 1; MessageMetadata messageMetadata2 = new MessageMetadata() - .setSequenceId(1) - .setProducerName("testProducer") + .setSequenceId(sequenceId) + .setProducerName(producerName) .setPartitionKey(partitionedKey) .setPartitionKeyB64Encoded(false) .setPublishTime(System.currentTimeMillis()); @@ -113,16 +115,28 @@ public void testPeekStickyKey() { // test 64 encoded String partitionedKey2 = Base64.getEncoder().encodeToString("key2".getBytes(UTF_8)); MessageMetadata messageMetadata = new MessageMetadata() - .setSequenceId(1) - .setProducerName("testProducer") + .setSequenceId(sequenceId) + .setProducerName(producerName) .setPartitionKey(partitionedKey2) .setPartitionKeyB64Encoded(true) .setPublishTime(System.currentTimeMillis()); ByteBuf byteBuf2 = serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, messageMetadata, Unpooled.copiedBuffer(message.getBytes(UTF_8))); byte[] bytes2 = Commands.peekStickyKey(byteBuf2, "topic-2", "sub-2"); - String key2 = Base64.getEncoder().encodeToString(bytes2);; + String key2 = Base64.getEncoder().encodeToString(bytes2); Assert.assertEquals(partitionedKey2, key2); ReferenceCountUtil.safeRelease(byteBuf2); + // test fallback key if no key given in message metadata + String fallbackPartitionedKey = producerName + "-" + sequenceId; + MessageMetadata messageMetadataWithoutKey = new MessageMetadata() + .setSequenceId(sequenceId) + .setProducerName(producerName) + .setPublishTime(System.currentTimeMillis()); + ByteBuf byteBuf3 = serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, messageMetadataWithoutKey, + Unpooled.copiedBuffer(message.getBytes(UTF_8))); + byte[] bytes3 = Commands.peekStickyKey(byteBuf3, "topic-3", "sub-3"); + String key3 = new String(bytes3); + Assert.assertEquals(fallbackPartitionedKey, key3); + ReferenceCountUtil.safeRelease(byteBuf3); } } diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/naming/SystemTopicNamesTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/naming/SystemTopicNamesTest.java new file mode 100644 index 0000000000000..92d93021973b1 --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/naming/SystemTopicNamesTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.naming; + +import static org.testng.AssertJUnit.assertEquals; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Test +public class SystemTopicNamesTest { + + @DataProvider(name = "topicPoliciesSystemTopicNames") + public static Object[][] topicPoliciesSystemTopicNames() { + return new Object[][] { + {"persistent://public/default/__change_events", true}, + {"persistent://public/default/__change_events-partition-0", true}, + {"persistent://random-tenant/random-ns/__change_events", true}, + {"persistent://random-tenant/random-ns/__change_events-partition-1", true}, + {"persistent://public/default/not_really__change_events", false}, + {"persistent://public/default/__change_events-diff-suffix", false}, + {"persistent://a/b/not_really__change_events", false}, + }; + } + + @Test(dataProvider = "topicPoliciesSystemTopicNames") + public void testIsTopicPoliciesSystemTopic(String topicName, boolean expectedResult) { + assertEquals(expectedResult, SystemTopicNames.isTopicPoliciesSystemTopic(topicName)); + assertEquals(expectedResult, SystemTopicNames.isSystemTopic(TopicName.get(topicName))); + assertEquals(expectedResult, SystemTopicNames.isEventSystemTopic(TopicName.get(topicName))); + } +} diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java index 8e32fbe3d33c0..485bea3f1addb 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java @@ -236,6 +236,46 @@ public void testDecodeEncode() throws Exception { assertEquals(name.getPersistenceNamingEncoding(), "prop/colo/ns/persistent/" + encodedName); } + @Test + public void testFromPersistenceNamingEncoding() { + // case1: V2 + String mlName1 = "public_tenant/default_namespace/persistent/test_topic"; + String expectedTopicName1 = "persistent://public_tenant/default_namespace/test_topic"; + + TopicName name1 = TopicName.get(expectedTopicName1); + assertEquals(name1.getPersistenceNamingEncoding(), mlName1); + assertEquals(TopicName.fromPersistenceNamingEncoding(mlName1), expectedTopicName1); + + // case2: V1 + String mlName2 = "public_tenant/my_cluster/default_namespace/persistent/test_topic"; + String expectedTopicName2 = "persistent://public_tenant/my_cluster/default_namespace/test_topic"; + + TopicName name2 = TopicName.get(expectedTopicName2); + assertEquals(name2.getPersistenceNamingEncoding(), mlName2); + assertEquals(TopicName.fromPersistenceNamingEncoding(mlName2), expectedTopicName2); + + // case3: null + String mlName3 = ""; + String expectedTopicName3 = ""; + assertEquals(expectedTopicName3, TopicName.fromPersistenceNamingEncoding(mlName3)); + + // case4: Invalid name + try { + String mlName4 = "public_tenant/my_cluster/default_namespace/persistent/test_topic/sub_topic"; + TopicName.fromPersistenceNamingEncoding(mlName4); + fail("Should have raised exception"); + } catch (IllegalArgumentException e) { + // Exception is expected. + } + + // case5: local name with special characters e.g. a:b:c + String topicName = "persistent://tenant/namespace/a:b:c"; + String persistentNamingEncoding = "tenant/namespace/persistent/a%3Ab%3Ac"; + assertEquals(TopicName.get(topicName).getPersistenceNamingEncoding(), persistentNamingEncoding); + assertEquals(TopicName.fromPersistenceNamingEncoding(persistentNamingEncoding), topicName); + } + + @SuppressWarnings("deprecation") @Test public void testTopicNameWithoutCluster() throws Exception { diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/nar/NarUnpackerTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/nar/NarUnpackerTest.java index c6c5ee180f69a..1c3a2c276537b 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/nar/NarUnpackerTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/nar/NarUnpackerTest.java @@ -38,6 +38,7 @@ import org.testng.annotations.Test; @Slf4j +@Test public class NarUnpackerTest { File sampleZipFile; File extractDirectory; @@ -46,7 +47,7 @@ public class NarUnpackerTest { public void createSampleZipFile() throws IOException { sampleZipFile = Files.createTempFile("sample", ".zip").toFile(); try (ZipOutputStream out = new ZipOutputStream(new FileOutputStream(sampleZipFile))) { - for (int i = 0; i < 10000; i++) { + for (int i = 0; i < 5000; i++) { ZipEntry e = new ZipEntry("hello" + i + ".txt"); out.putNextEntry(e); byte[] msg = "hello world!".getBytes(StandardCharsets.UTF_8); @@ -58,12 +59,20 @@ public void createSampleZipFile() throws IOException { } @AfterMethod(alwaysRun = true) - void deleteSampleZipFile() throws IOException { - if (sampleZipFile != null) { - sampleZipFile.delete(); + void deleteSampleZipFile() { + if (sampleZipFile != null && sampleZipFile.exists()) { + try { + sampleZipFile.delete(); + } catch (Exception e) { + log.warn("Failed to delete file {}", sampleZipFile, e); + } } - if (extractDirectory != null) { - FileUtils.deleteFile(extractDirectory, true); + if (extractDirectory != null && extractDirectory.exists()) { + try { + FileUtils.deleteFile(extractDirectory, true); + } catch (IOException e) { + log.warn("Failed to delete directory {}", extractDirectory, e); + } } } @@ -109,9 +118,20 @@ public static void main(String[] args) { } } + @Test + void shouldReExtractWhenUnpackedDirectoryIsMissing() throws IOException { + AtomicInteger extractCounter = new AtomicInteger(); + + File narWorkingDirectory = NarUnpacker.doUnpackNar(sampleZipFile, extractDirectory, extractCounter::incrementAndGet); + FileUtils.deleteFile(narWorkingDirectory, true); + NarUnpacker.doUnpackNar(sampleZipFile, extractDirectory, extractCounter::incrementAndGet); + + assertEquals(extractCounter.get(), 2); + } + @Test void shouldExtractFilesOnceInDifferentProcess() throws InterruptedException { - int processes = 10; + int processes = 5; String javaExePath = findJavaExe().getAbsolutePath(); CountDownLatch countDownLatch = new CountDownLatch(processes); AtomicInteger exceptionCounter = new AtomicInteger(); @@ -122,7 +142,9 @@ void shouldExtractFilesOnceInDifferentProcess() throws InterruptedException { // fork a new process with the same classpath Process process = new ProcessBuilder() .command(javaExePath, - "-Xmx64m", + "-Xmx96m", + "-XX:TieredStopAtLevel=1", + "-Dlog4j2.disable.jmx=true", "-cp", System.getProperty("java.class.path"), // use NarUnpackerWorker as the main class @@ -130,6 +152,7 @@ void shouldExtractFilesOnceInDifferentProcess() throws InterruptedException { // pass arguments to use for testing sampleZipFile.getAbsolutePath(), extractDirectory.getAbsolutePath()) + .redirectErrorStream(true) .start(); String output = IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8); int retval = process.waitFor(); @@ -147,7 +170,7 @@ void shouldExtractFilesOnceInDifferentProcess() throws InterruptedException { } }).start(); } - assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); + assertTrue(countDownLatch.await(30, TimeUnit.SECONDS), "All processes should finish before timeout"); assertEquals(exceptionCounter.get(), 0); assertEquals(extractCounter.get(), 1); } diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/ClusterDataImplTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/ClusterDataImplTest.java index ca4cba2cf9749..0bf1616653107 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/ClusterDataImplTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/ClusterDataImplTest.java @@ -53,8 +53,6 @@ public void verifyClone() { .brokerClientKeyFilePath("/my/key/file") .brokerClientCertificateFilePath("/my/cert/file") .listenerName("a-listener") - .migrated(true) - .migratedClusterUrl(new ClusterData.ClusterUrl("pulsar://remote", "pulsar+ssl://remote")) .build(); ClusterDataImpl clone = originalData.clone().build(); diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/OffloadPoliciesTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/OffloadPoliciesTest.java index 88036b1688437..bbede4e982044 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/OffloadPoliciesTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/OffloadPoliciesTest.java @@ -18,6 +18,18 @@ */ package org.apache.pulsar.common.policies.data; +import static org.apache.pulsar.common.policies.data.OffloadPoliciesImpl.EXTRA_CONFIG_PREFIX; +import static org.testng.Assert.assertEquals; +import java.io.DataInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Method; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; import org.testng.Assert; import org.testng.annotations.Test; @@ -365,4 +377,99 @@ public void brokerPropertyCompatibleTest() { Assert.assertEquals(offloadPolicies.getManagedLedgerOffloadDeletionLagInMillis(), brokerDeletionLag); Assert.assertEquals(offloadPolicies.getManagedLedgerOffloadedReadPriority().toString(), brokerReadPriority); } + + @Test + public void testSupportExtraOffloadDrivers() throws Exception { + System.setProperty("pulsar.extra.offload.drivers", "driverA, driverB"); + // using the custom classloader to reload the offload policies class to read the + // system property correctly. + TestClassLoader loader = new TestClassLoader(); + Class clazz = loader.loadClass("org.apache.pulsar.common.policies.data.OffloadPoliciesImpl"); + Object o = clazz.getDeclaredConstructor().newInstance(); + clazz.getDeclaredMethod("setManagedLedgerOffloadDriver", String.class).invoke(o, "driverA"); + Method method = clazz.getDeclaredMethod("driverSupported"); + Assert.assertEquals(method.invoke(o), true); + clazz.getDeclaredMethod("setManagedLedgerOffloadDriver", String.class).invoke(o, "driverB"); + Assert.assertEquals(method.invoke(o), true); + clazz.getDeclaredMethod("setManagedLedgerOffloadDriver", String.class).invoke(o, "driverC"); + Assert.assertEquals(method.invoke(o), false); + clazz.getDeclaredMethod("setManagedLedgerOffloadDriver", String.class).invoke(o, "aws-s3"); + Assert.assertEquals(method.invoke(o), true); + } + + // this is used for the testSupportExtraOffloadDrivers. Because we need to change the system property, + // we need to reload the class to read the system property. + static class TestClassLoader extends ClassLoader { + @Override + public Class loadClass(String name) throws ClassNotFoundException { + if (name.contains("OffloadPoliciesImpl")) { + return getClass(name); + } + return super.loadClass(name); + } + + private Class getClass(String name) { + String file = name.replace('.', File.separatorChar) + ".class"; + Path targetPath = Paths.get(getClass().getClassLoader().getResource(".").getPath()).getParent(); + file = Paths.get(targetPath.toString(), "classes", file).toString(); + byte[] byteArr = null; + try { + byteArr = loadClassData(file); + Class c = defineClass(name, byteArr, 0, byteArr.length); + resolveClass(c); + return c; + } catch (Exception e) { + e.printStackTrace(); + return null; + } + } + + private byte[] loadClassData(String name) throws IOException { + InputStream stream = Files.newInputStream(Paths.get(name)); + int size = stream.available(); + byte buff[] = new byte[size]; + DataInputStream in = new DataInputStream(stream); + // Reading the binary data + in.readFully(buff); + in.close(); + return buff; + } + } + + @Test + public void testCreateOffloadPoliciesWithExtraConfiguration() { + Properties properties = new Properties(); + properties.put(EXTRA_CONFIG_PREFIX + "Key1", "value1"); + properties.put(EXTRA_CONFIG_PREFIX + "Key2", "value2"); + OffloadPoliciesImpl policies = OffloadPoliciesImpl.create(properties); + + Map extraConfigurations = policies.getManagedLedgerExtraConfigurations(); + Assert.assertEquals(extraConfigurations.size(), 2); + Assert.assertEquals(extraConfigurations.get("Key1"), "value1"); + Assert.assertEquals(extraConfigurations.get("Key2"), "value2"); + } + + /** + * Test toProperties as well as create from properties. + * @throws Exception + */ + @Test + public void testToProperties() throws Exception { + // Base information convert. + OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.create("aws-s3", "test-region", "test-bucket", + "http://test.endpoint", null, null, null, null, 32 * 1024 * 1024, 5 * 1024 * 1024, + 10 * 1024 * 1024L, 100L, 10000L, OffloadedReadPriority.TIERED_STORAGE_FIRST); + assertEquals(offloadPolicies, OffloadPoliciesImpl.create(offloadPolicies.toProperties())); + + // Set useless config to offload policies. Make sure convert conversion result is the same. + offloadPolicies.setFileSystemProfilePath("/test/file"); + assertEquals(offloadPolicies, OffloadPoliciesImpl.create(offloadPolicies.toProperties())); + + // Set extra config to offload policies. Make sure convert conversion result is the same. + Map extraConfiguration = new HashMap<>(); + extraConfiguration.put("key1", "value1"); + extraConfiguration.put("key2", "value2"); + offloadPolicies.setManagedLedgerExtraConfigurations(extraConfiguration); + assertEquals(offloadPolicies, OffloadPoliciesImpl.create(offloadPolicies.toProperties())); + } } diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImplTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImplTest.java new file mode 100644 index 0000000000000..8a4b5da9edd20 --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImplTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.policies.data.stats; + +import static org.testng.Assert.assertEquals; +import org.testng.annotations.Test; + +public class SubscriptionStatsImplTest { + + @Test + public void testReset() { + SubscriptionStatsImpl stats = new SubscriptionStatsImpl(); + stats.earliestMsgPublishTimeInBacklog = 1L; + stats.reset(); + assertEquals(stats.earliestMsgPublishTimeInBacklog, 0L); + + } + + @Test + public void testAdd_EarliestMsgPublishTimeInBacklogs_Earliest() { + SubscriptionStatsImpl stats1 = new SubscriptionStatsImpl(); + stats1.earliestMsgPublishTimeInBacklog = 10L; + + SubscriptionStatsImpl stats2 = new SubscriptionStatsImpl(); + stats2.earliestMsgPublishTimeInBacklog = 20L; + + SubscriptionStatsImpl aggregate = stats1.add(stats2); + assertEquals(aggregate.earliestMsgPublishTimeInBacklog, 10L); + } + + @Test + public void testAdd_EarliestMsgPublishTimeInBacklogs_First0() { + SubscriptionStatsImpl stats1 = new SubscriptionStatsImpl(); + stats1.earliestMsgPublishTimeInBacklog = 0L; + + SubscriptionStatsImpl stats2 = new SubscriptionStatsImpl(); + stats2.earliestMsgPublishTimeInBacklog = 20L; + + SubscriptionStatsImpl aggregate = stats1.add(stats2); + assertEquals(aggregate.earliestMsgPublishTimeInBacklog, 20L); + } + + @Test + public void testAdd_EarliestMsgPublishTimeInBacklogs_Second0() { + SubscriptionStatsImpl stats1 = new SubscriptionStatsImpl(); + stats1.earliestMsgPublishTimeInBacklog = 10L; + + SubscriptionStatsImpl stats2 = new SubscriptionStatsImpl(); + stats2.earliestMsgPublishTimeInBacklog = 0L; + + SubscriptionStatsImpl aggregate = stats1.add(stats2); + assertEquals(aggregate.earliestMsgPublishTimeInBacklog, 10L); + } + + @Test + public void testAdd_EarliestMsgPublishTimeInBacklogs_Zero() { + SubscriptionStatsImpl stats1 = new SubscriptionStatsImpl(); + stats1.earliestMsgPublishTimeInBacklog = 0L; + + SubscriptionStatsImpl stats2 = new SubscriptionStatsImpl(); + stats2.earliestMsgPublishTimeInBacklog = 0L; + + SubscriptionStatsImpl aggregate = stats1.add(stats2); + assertEquals(aggregate.earliestMsgPublishTimeInBacklog, 0L); + } +} \ No newline at end of file diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImplTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImplTest.java new file mode 100644 index 0000000000000..09cef4c4d0f82 --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImplTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.policies.data.stats; + +import static org.testng.Assert.assertEquals; +import org.testng.annotations.Test; + +public class TopicStatsImplTest { + + @Test + public void testReset() { + TopicStatsImpl stats = new TopicStatsImpl(); + stats.earliestMsgPublishTimeInBacklogs = 1L; + stats.reset(); + assertEquals(stats.earliestMsgPublishTimeInBacklogs, 0L); + } + + @Test + public void testAdd_EarliestMsgPublishTimeInBacklogs_Earliest() { + TopicStatsImpl stats1 = new TopicStatsImpl(); + stats1.earliestMsgPublishTimeInBacklogs = 10L; + + TopicStatsImpl stats2 = new TopicStatsImpl(); + stats2.earliestMsgPublishTimeInBacklogs = 20L; + + TopicStatsImpl aggregate = stats1.add(stats2); + assertEquals(aggregate.earliestMsgPublishTimeInBacklogs, 10L); + } + + @Test + public void testAdd_EarliestMsgPublishTimeInBacklogs_First0() { + TopicStatsImpl stats1 = new TopicStatsImpl(); + stats1.earliestMsgPublishTimeInBacklogs = 0L; + + TopicStatsImpl stats2 = new TopicStatsImpl(); + stats2.earliestMsgPublishTimeInBacklogs = 20L; + + TopicStatsImpl aggregate = stats1.add(stats2); + assertEquals(aggregate.earliestMsgPublishTimeInBacklogs, 20L); + } + + @Test + public void testAdd_EarliestMsgPublishTimeInBacklogs_Second0() { + TopicStatsImpl stats1 = new TopicStatsImpl(); + stats1.earliestMsgPublishTimeInBacklogs = 10L; + + TopicStatsImpl stats2 = new TopicStatsImpl(); + stats2.earliestMsgPublishTimeInBacklogs = 0L; + + TopicStatsImpl aggregate = stats1.add(stats2); + assertEquals(aggregate.earliestMsgPublishTimeInBacklogs, 10L); + } + + @Test + public void testAdd_EarliestMsgPublishTimeInBacklogs_Zero() { + TopicStatsImpl stats1 = new TopicStatsImpl(); + stats1.earliestMsgPublishTimeInBacklogs = 0L; + + TopicStatsImpl stats2 = new TopicStatsImpl(); + stats2.earliestMsgPublishTimeInBacklogs = 0L; + + TopicStatsImpl aggregate = stats1.add(stats2); + assertEquals(aggregate.earliestMsgPublishTimeInBacklogs, 0L); + } +} \ No newline at end of file diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/stats/MetricsUtilTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/stats/MetricsUtilTest.java new file mode 100644 index 0000000000000..51bb31c4370e7 --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/stats/MetricsUtilTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.stats; + +import static org.assertj.core.api.Assertions.assertThat; +import java.util.concurrent.TimeUnit; +import org.testng.annotations.Test; + +public class MetricsUtilTest { + + @Test + public void testConvertToSeconds() { + assertThat(MetricsUtil.convertToSeconds(1, TimeUnit.HOURS)).isEqualTo(3600.0); + assertThat(MetricsUtil.convertToSeconds(1, TimeUnit.MINUTES)).isEqualTo(60.0); + assertThat(MetricsUtil.convertToSeconds(1, TimeUnit.SECONDS)).isEqualTo(1.0); + assertThat(MetricsUtil.convertToSeconds(1, TimeUnit.MILLISECONDS)).isEqualTo(0.001); + assertThat(MetricsUtil.convertToSeconds(1, TimeUnit.MICROSECONDS)).isEqualTo(0.000_001); + assertThat(MetricsUtil.convertToSeconds(1, TimeUnit.NANOSECONDS)).isEqualTo(0.000_000_001); + } + +} diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/topics/TopicListTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/topics/TopicListTest.java index 9069dd6dcc7b9..7bcdacb2e9b20 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/topics/TopicListTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/topics/TopicListTest.java @@ -19,17 +19,18 @@ package org.apache.pulsar.common.topics; import com.google.common.collect.Lists; +import com.google.re2j.Pattern; import org.testng.annotations.Test; import java.util.Arrays; import java.util.List; import java.util.Set; -import java.util.regex.Pattern; import java.util.stream.Stream; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; public class TopicListTest { @@ -107,5 +108,60 @@ public void testCalculateHash() { } - + @Test + public void testRemoveTopicDomainScheme() { + // persistent. + final String tpName1 = "persistent://public/default/tp"; + String res1 = TopicList.removeTopicDomainScheme(tpName1); + assertEquals(res1, "public/default/tp"); + + // non-persistent + final String tpName2 = "non-persistent://public/default/tp"; + String res2 = TopicList.removeTopicDomainScheme(tpName2); + assertEquals(res2, "public/default/tp"); + + // without topic domain. + final String tpName3 = "public/default/tp"; + String res3 = TopicList.removeTopicDomainScheme(tpName3); + assertEquals(res3, "public/default/tp"); + + // persistent & "java.util.regex.Pattern.quote". + final String tpName4 = java.util.regex.Pattern.quote(tpName1); + String res4 = TopicList.removeTopicDomainScheme(tpName4); + assertEquals(res4, java.util.regex.Pattern.quote("public/default/tp")); + + // persistent & "java.util.regex.Pattern.quote" & "^$". + final String tpName5 = "^" + java.util.regex.Pattern.quote(tpName1) + "$"; + String res5 = TopicList.removeTopicDomainScheme(tpName5); + assertEquals(res5, "^" + java.util.regex.Pattern.quote("public/default/tp") + "$"); + + // persistent & "com.google.re2j.Pattern.quote". + final String tpName6 = Pattern.quote(tpName1); + String res6 = TopicList.removeTopicDomainScheme(tpName6); + assertEquals(res6, Pattern.quote("public/default/tp")); + + // non-persistent & "java.util.regex.Pattern.quote". + final String tpName7 = java.util.regex.Pattern.quote(tpName2); + String res7 = TopicList.removeTopicDomainScheme(tpName7); + assertEquals(res7, java.util.regex.Pattern.quote("public/default/tp")); + + // non-persistent & "com.google.re2j.Pattern.quote". + final String tpName8 = Pattern.quote(tpName2); + String res8 = TopicList.removeTopicDomainScheme(tpName8); + assertEquals(res8, Pattern.quote("public/default/tp")); + + // non-persistent & "com.google.re2j.Pattern.quote" & "^$". + final String tpName9 = "^" + Pattern.quote(tpName2) + "$"; + String res9 = TopicList.removeTopicDomainScheme(tpName9); + assertEquals(res9, "^" + Pattern.quote("public/default/tp") + "$"); + + // wrong topic domain. + final String tpName10 = "xx://public/default/tp"; + try { + TopicList.removeTopicDomainScheme(tpName10); + fail("Does not support the topic domain xx"); + } catch (Exception ex) { + // expected error. + } + } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BackoffTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/BackoffTest.java similarity index 99% rename from pulsar-client/src/test/java/org/apache/pulsar/client/impl/BackoffTest.java rename to pulsar-common/src/test/java/org/apache/pulsar/common/util/BackoffTest.java index 7f13acb769492..b3786236a70ef 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BackoffTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/BackoffTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.client.impl; +package org.apache.pulsar.common.util; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/CmdGenerateDocsTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/CmdGenerateDocsTest.java deleted file mode 100644 index 68d8a02f48673..0000000000000 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/CmdGenerateDocsTest.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util; - -import static org.testng.Assert.assertEquals; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; -import org.testng.annotations.Test; - -public class CmdGenerateDocsTest { - - @Parameters(commandDescription = "Options") - public class Arguments { - @Parameter(names = {"-h", "--help"}, description = "Show this help message") - private boolean help = false; - - @Parameter(names = {"-n", "--name"}, description = "Name") - private String name; - } - - @Test - public void testHelp() { - PrintStream oldStream = System.out; - try { - ByteArrayOutputStream baoStream = new ByteArrayOutputStream(2048); - PrintStream cacheStream = new PrintStream(baoStream); - System.setOut(cacheStream); - - CmdGenerateDocs cmd = new CmdGenerateDocs("pulsar"); - cmd.addCommand("test", new Arguments()); - cmd.run(new String[]{"-h"}); - - String message = baoStream.toString(); - String rightMsg = "Usage: pulsar gen-doc [options]\n" - + " Options:\n" - + " -n, --command-names\n" - + " List of command names\n" - + " Default: []\n" - + " -h, --help\n" - + " Display help information\n" - + " Default: false\n" - + System.lineSeparator(); - assertEquals(rightMsg, message); - } finally { - System.setOut(oldStream); - } - } - - @Test - public void testGenerateDocs() { - PrintStream oldStream = System.out; - try { - ByteArrayOutputStream baoStream = new ByteArrayOutputStream(2048); - PrintStream cacheStream = new PrintStream(baoStream); - System.setOut(cacheStream); - - CmdGenerateDocs cmd = new CmdGenerateDocs("pulsar"); - cmd.addCommand("test", new Arguments()); - cmd.run(null); - - String message = baoStream.toString(); - String rightMsg = "# test\n\n" - + "Options\n\n" - + "\n" - + "```shell\n" - + "$ pulsar test options\n" - + "```\n" - + "\n" - + "|Flag|Description|Default|\n" - + "|---|---|---|\n" - + "| `-n, --name` | Name|null|\n" - + "| `-h, --help` | Show this help message|false|\n" - + System.lineSeparator(); - assertEquals(rightMsg, message); - } finally { - System.setOut(oldStream); - } - } -} diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/DefaultPulsarSslFactoryTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/DefaultPulsarSslFactoryTest.java new file mode 100644 index 0000000000000..34cf3a97ce803 --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/DefaultPulsarSslFactoryTest.java @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertThrows; +import com.google.common.io.Resources; +import io.netty.buffer.ByteBufAllocator; +import io.netty.handler.ssl.OpenSslEngine; +import java.io.File; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import javax.net.ssl.SSLEngine; +import org.testng.annotations.Test; + +public class DefaultPulsarSslFactoryTest { + + public final static String KEYSTORE_FILE_PATH = + getAbsolutePath("certificate-authority/jks/broker.keystore.jks"); + public final static String TRUSTSTORE_FILE_PATH = + getAbsolutePath("certificate-authority/jks/broker.truststore.jks"); + public final static String TRUSTSTORE_NO_PASSWORD_FILE_PATH = + getAbsolutePath("certificate-authority/jks/broker.truststore.nopassword.jks"); + public final static String KEYSTORE_PW = "111111"; + public final static String TRUSTSTORE_PW = "111111"; + public final static String KEYSTORE_TYPE = "JKS"; + + public final static String CA_CERT_FILE_PATH = + getAbsolutePath("certificate-authority/certs/ca.cert.pem"); + public final static String CERT_FILE_PATH = + getAbsolutePath("certificate-authority/server-keys/broker.cert.pem"); + public final static String KEY_FILE_PATH = + getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem"); + + @Test + public void sslContextCreationUsingKeystoreTest() throws Exception { + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsEnabledWithKeystore(true) + .tlsKeyStoreType(KEYSTORE_TYPE) + .tlsKeyStorePath(KEYSTORE_FILE_PATH) + .tlsKeyStorePassword(KEYSTORE_PW) + .tlsTrustStorePath(TRUSTSTORE_FILE_PATH) + .tlsTrustStorePassword(TRUSTSTORE_PW) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalSslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalNettySslContext); + } + + @Test + public void sslContextCreationUsingPasswordLessTruststoreTest() throws Exception { + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsEnabledWithKeystore(true) + .tlsKeyStoreType(KEYSTORE_TYPE) + .tlsKeyStorePath(KEYSTORE_FILE_PATH) + .tlsKeyStorePassword(KEYSTORE_PW) + .tlsTrustStorePath(TRUSTSTORE_NO_PASSWORD_FILE_PATH) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalSslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalNettySslContext); + } + + @Test + public void sslContextCreationUsingTlsCertsTest() throws Exception { + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsCertificateFilePath(CERT_FILE_PATH) + .tlsKeyFilePath(KEY_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalNettySslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalSslContext); + } + + @Test + public void sslContextCreationUsingOnlyCACertsTest() throws Exception { + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalNettySslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalSslContext); + } + + @Test + public void sslContextCreationForWebClientConnections() throws Exception { + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsCertificateFilePath(CERT_FILE_PATH) + .tlsKeyFilePath(KEY_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) + .isHttps(true) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalSslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalNettySslContext); + } + + @Test + public void sslContextCreationForWebServerConnectionsTest() throws Exception { + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsCertificateFilePath(CERT_FILE_PATH) + .tlsKeyFilePath(KEY_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) + .isHttps(true) + .serverMode(true) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalSslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalNettySslContext); + } + + @Test + public void sslEngineCreationWithEnabledProtocolsAndCiphersForOpenSSLTest() throws Exception { + Set ciphers = new HashSet<>(); + ciphers.add("TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"); + ciphers.add("TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"); + Set protocols = new HashSet<>(); + protocols.add("TLSv1.2"); + protocols.add("TLSv1"); + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsCertificateFilePath(CERT_FILE_PATH) + .tlsKeyFilePath(KEY_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) + .tlsCiphers(ciphers) + .tlsProtocols(protocols) + .serverMode(true) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalNettySslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalSslContext); + SSLEngine sslEngine = pulsarSslFactory.createServerSslEngine(ByteBufAllocator.DEFAULT); + /* Adding SSLv2Hello protocol only during expected checks as Netty adds it as part of the + ReferenceCountedOpenSslEngine's setEnabledProtocols method. The reasoning is that OpenSSL currently has no + way to disable this protocol. + */ + protocols.add("SSLv2Hello"); + assertEquals(new HashSet<>(Arrays.asList(sslEngine.getEnabledProtocols())), protocols); + assertEquals(new HashSet<>(Arrays.asList(sslEngine.getEnabledCipherSuites())), ciphers); + assert(!sslEngine.getUseClientMode()); + } + + @Test + public void sslEngineCreationWithEnabledProtocolsAndCiphersForWebTest() throws Exception { + Set ciphers = new HashSet<>(); + ciphers.add("TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"); + ciphers.add("TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"); + Set protocols = new HashSet<>(); + protocols.add("TLSv1.2"); + protocols.add("TLSv1"); + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsCertificateFilePath(CERT_FILE_PATH) + .tlsKeyFilePath(KEY_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) + .tlsCiphers(ciphers) + .tlsProtocols(protocols) + .isHttps(true) + .serverMode(true) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalSslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalNettySslContext); + SSLEngine sslEngine = pulsarSslFactory.createServerSslEngine(ByteBufAllocator.DEFAULT); + assertEquals(new HashSet<>(Arrays.asList(sslEngine.getEnabledProtocols())), protocols); + assertEquals(new HashSet<>(Arrays.asList(sslEngine.getEnabledCipherSuites())), ciphers); + assert(!sslEngine.getUseClientMode()); + } + + @Test + public void sslContextCreationAsOpenSslTlsProvider() throws Exception { + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsProvider("OPENSSL") + .tlsCertificateFilePath(CERT_FILE_PATH) + .tlsKeyFilePath(KEY_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) + .serverMode(true) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalNettySslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalSslContext); + SSLEngine sslEngine = pulsarSslFactory.createServerSslEngine(ByteBufAllocator.DEFAULT); + assert(sslEngine instanceof OpenSslEngine); + } + + @Test + public void sslContextCreationAsJDKTlsProvider() throws Exception { + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsProvider("JDK") + .tlsCertificateFilePath(CERT_FILE_PATH) + .tlsKeyFilePath(KEY_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) + .serverMode(true) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalNettySslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalSslContext); + SSLEngine sslEngine = pulsarSslFactory.createServerSslEngine(ByteBufAllocator.DEFAULT); + assert (!(sslEngine instanceof OpenSslEngine)); + } + + @Test + public void sslEngineMutualAuthEnabledTest() throws Exception { + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsProvider("JDK") + .tlsCertificateFilePath(CERT_FILE_PATH) + .tlsKeyFilePath(KEY_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) + .requireTrustedClientCertOnConnect(true) + .serverMode(true) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalNettySslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalSslContext); + SSLEngine sslEngine = pulsarSslFactory.createServerSslEngine(ByteBufAllocator.DEFAULT); + assert(sslEngine.getNeedClientAuth()); + } + + @Test + public void sslEngineSniClientTest() throws Exception { + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsCertificateFilePath(CERT_FILE_PATH) + .tlsKeyFilePath(KEY_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalNettySslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalSslContext); + SSLEngine sslEngine = pulsarSslFactory.createClientSslEngine(ByteBufAllocator.DEFAULT, "localhost", + 1234); + assertEquals(sslEngine.getPeerHost(), "localhost"); + assertEquals(sslEngine.getPeerPort(), 1234); + } + + + + private static String getAbsolutePath(String resourceName) { + return new File(Resources.getResource(resourceName).getPath()).getAbsolutePath(); + } + +} diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FieldParserTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FieldParserTest.java index e90b6cbc4a13a..b24e9ae40822a 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FieldParserTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FieldParserTest.java @@ -19,12 +19,15 @@ package org.apache.pulsar.common.util; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; +import java.lang.reflect.Field; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import org.testng.annotations.Test; @@ -94,4 +97,46 @@ public static class MyConfig { public Set stringSet; } + @Test + public void testNullStrValue() throws Exception { + class TestMap { + public List list; + public Set set; + public Map map; + public Optional optional; + } + + Field listField = TestMap.class.getField("list"); + Object listValue = FieldParser.value(null, listField); + assertNull(listValue); + + listValue = FieldParser.value("null", listField); + assertTrue(listValue instanceof List); + assertEquals(((List) listValue).size(), 1); + assertEquals(((List) listValue).get(0), "null"); + + + Field setField = TestMap.class.getField("set"); + Object setValue = FieldParser.value(null, setField); + assertNull(setValue); + + setValue = FieldParser.value("null", setField); + assertTrue(setValue instanceof Set); + assertEquals(((Set) setValue).size(), 1); + assertEquals(((Set) setValue).iterator().next(), "null"); + + Field mapField = TestMap.class.getField("map"); + Object mapValue = FieldParser.value(null, mapField); + assertNull(mapValue); + + try { + FieldParser.value("null", mapField); + } catch (IllegalArgumentException iae) { + assertTrue(iae.getMessage().contains("null map-value is not in correct format key1=value,key2=value2")); + } + + Field optionalField = TestMap.class.getField("optional"); + Object optionalValue = FieldParser.value(null, optionalField); + assertEquals(optionalValue, Optional.empty()); + } } diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FileModifiedTimeUpdaterTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FileModifiedTimeUpdaterTest.java index 9c75a5cdb0a82..a41c9ceb2cbb7 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FileModifiedTimeUpdaterTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FileModifiedTimeUpdaterTest.java @@ -48,6 +48,11 @@ public BasicAuthenticationData(String authParam) { this.authParam = authParam; } + @Override + public boolean hasDataForTls() { + return true; + } + public boolean hasDataFromCommand() { return true; } @@ -60,7 +65,7 @@ public boolean hasDataForHttp() { return true; } - public String getTlsCerificateFilePath() { + public String getTlsCertificateFilePath() { return certFilePath; } @@ -107,14 +112,17 @@ public void testNettyClientSslContextRefresher() throws Exception { createFile(Paths.get(certFile)); provider.certFilePath = certFile; provider.keyFilePath = certFile; - NettyClientSslContextRefresher refresher = new NettyClientSslContextRefresher(null, false, certFile, - provider, null, null, 1); - Thread.sleep(5000); - Paths.get(certFile).toFile().delete(); - // update the file - createFile(Paths.get(certFile)); - Awaitility.await().atMost(30, TimeUnit.SECONDS).until(()-> refresher.needUpdate()); - assertTrue(refresher.needUpdate()); + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .allowInsecureConnection(false).tlsTrustCertsFilePath(certFile).authData(provider).build(); + try (PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory()) { + pulsarSslFactory.initialize(pulsarSslConfiguration); + Thread.sleep(5000); + Paths.get(certFile).toFile().delete(); + // update the file + createFile(Paths.get(certFile)); + Awaitility.await().atMost(30, TimeUnit.SECONDS).until(pulsarSslFactory::needsUpdate); + assertTrue(pulsarSslFactory.needsUpdate()); + } } } diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java index 6df4494edf886..09ce9f9f137ca 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java @@ -181,9 +181,9 @@ public void testWaitForAny() { } } - @Test public void testSequencer() { int concurrentNum = 1000; + @Cleanup("shutdownNow") final ScheduledExecutorService executor = Executors.newScheduledThreadPool(concurrentNum); final FutureUtil.Sequencer sequencer = FutureUtil.Sequencer.create(); // normal case -- allowExceptionBreakChain=false diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/RateLimiterTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/RateLimiterTest.java deleted file mode 100644 index 5cbd024556593..0000000000000 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/RateLimiterTest.java +++ /dev/null @@ -1,245 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; - -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Supplier; -import org.testng.annotations.Test; - -public class RateLimiterTest { - - @Test - public void testInvalidRenewTime() { - try { - RateLimiter.builder().permits(0).rateTime(100).timeUnit(TimeUnit.SECONDS).build(); - fail("should have thrown exception: invalid rate, must be > 0"); - } catch (IllegalArgumentException ie) { - // Ok - } - - try { - RateLimiter.builder().permits(10).rateTime(0).timeUnit(TimeUnit.SECONDS).build(); - fail("should have thrown exception: invalid rateTime, must be > 0"); - } catch (IllegalArgumentException ie) { - // Ok - } - } - - @Test - public void testClose() throws Exception { - RateLimiter rate = RateLimiter.builder().permits(1).rateTime(1000).timeUnit(TimeUnit.MILLISECONDS).build(); - assertFalse(rate.isClosed()); - rate.close(); - assertTrue(rate.isClosed()); - try { - rate.acquire(); - fail("should have failed, executor is already closed"); - } catch (IllegalArgumentException e) { - // ok - } - } - - @Test - public void testAcquireBlock() throws Exception { - final long rateTimeMSec = 1000; - RateLimiter rate = RateLimiter.builder().permits(1).rateTime(rateTimeMSec).timeUnit(TimeUnit.MILLISECONDS) - .build(); - rate.acquire(); - assertEquals(rate.getAvailablePermits(), 0); - long start = System.currentTimeMillis(); - rate.acquire(); - long end = System.currentTimeMillis(); - // no permits are available: need to wait on acquire - assertTrue((end - start) > rateTimeMSec / 2); - rate.close(); - } - - @Test - public void testAcquire() throws Exception { - final long rateTimeMSec = 1000; - final int permits = 100; - RateLimiter rate = RateLimiter.builder().permits(permits).rateTime(rateTimeMSec).timeUnit(TimeUnit.MILLISECONDS) - .build(); - long start = System.currentTimeMillis(); - for (int i = 0; i < permits; i++) { - rate.acquire(); - } - long end = System.currentTimeMillis(); - assertTrue((end - start) < rateTimeMSec); - assertEquals(rate.getAvailablePermits(), 0); - rate.close(); - } - - @Test - public void testMultipleAcquire() throws Exception { - final long rateTimeMSec = 1000; - final int permits = 100; - final int acquirePermits = 50; - RateLimiter rate = RateLimiter.builder().permits(permits).rateTime(rateTimeMSec).timeUnit(TimeUnit.MILLISECONDS) - .build(); - long start = System.currentTimeMillis(); - for (int i = 0; i < permits / acquirePermits; i++) { - rate.acquire(acquirePermits); - } - long end = System.currentTimeMillis(); - assertTrue((end - start) < rateTimeMSec); - assertEquals(rate.getAvailablePermits(), 0); - rate.close(); - } - - @Test - public void testTryAcquireNoPermits() { - final long rateTimeMSec = 1000; - RateLimiter rate = RateLimiter.builder().permits(1).rateTime(rateTimeMSec).timeUnit(TimeUnit.MILLISECONDS) - .build(); - assertTrue(rate.tryAcquire()); - assertFalse(rate.tryAcquire()); - assertEquals(rate.getAvailablePermits(), 0); - rate.close(); - } - - @Test - public void testTryAcquire() { - final long rateTimeMSec = 1000; - final int permits = 100; - RateLimiter rate = RateLimiter.builder().permits(permits).rateTime(rateTimeMSec).timeUnit(TimeUnit.MILLISECONDS) - .build(); - for (int i = 0; i < permits; i++) { - rate.tryAcquire(); - } - assertEquals(rate.getAvailablePermits(), 0); - rate.close(); - } - - @Test - public void testTryAcquireMoreThanPermits() { - final long rateTimeMSec = 1000; - RateLimiter rate = RateLimiter.builder().permits(3).rateTime(rateTimeMSec).timeUnit(TimeUnit.MILLISECONDS) - .build(); - assertTrue(rate.tryAcquire(2)); - assertEquals(rate.getAvailablePermits(), 1); - - //try to acquire failed, not decrease availablePermits. - assertFalse(rate.tryAcquire(2)); - assertEquals(rate.getAvailablePermits(), 1); - - assertTrue(rate.tryAcquire(1)); - assertEquals(rate.getAvailablePermits(), 0); - - rate.close(); - } - - @Test - public void testMultipleTryAcquire() { - final long rateTimeMSec = 1000; - final int permits = 100; - final int acquirePermits = 50; - RateLimiter rate = RateLimiter.builder().permits(permits).rateTime(rateTimeMSec).timeUnit(TimeUnit.MILLISECONDS) - .build(); - for (int i = 0; i < permits / acquirePermits; i++) { - rate.tryAcquire(acquirePermits); - } - assertEquals(rate.getAvailablePermits(), 0); - rate.close(); - } - - @Test - public void testResetRate() throws Exception { - final long rateTimeMSec = 1000; - final int permits = 100; - RateLimiter rate = RateLimiter.builder().permits(permits).rateTime(rateTimeMSec).timeUnit(TimeUnit.MILLISECONDS) - .build(); - rate.tryAcquire(permits); - assertEquals(rate.getAvailablePermits(), 0); - // check after a rate-time: permits must be renewed - Thread.sleep(rateTimeMSec * 2); - assertEquals(rate.getAvailablePermits(), permits); - - // change rate-time from 1sec to 5sec - rate.setRate(permits, 5 * rateTimeMSec, TimeUnit.MILLISECONDS, null); - assertEquals(rate.getAvailablePermits(), 100); - assertTrue(rate.tryAcquire(permits)); - assertEquals(rate.getAvailablePermits(), 0); - // check after a rate-time: permits can't be renewed - Thread.sleep(rateTimeMSec); - assertEquals(rate.getAvailablePermits(), 0); - - rate.close(); - } - - @Test - public void testDispatchRate() throws Exception { - final long rateTimeMSec = 1000; - final int permits = 100; - RateLimiter rate = RateLimiter.builder().permits(permits).rateTime(rateTimeMSec).timeUnit(TimeUnit.MILLISECONDS) - .isDispatchOrPrecisePublishRateLimiter(true) - .build(); - rate.tryAcquire(100); - rate.tryAcquire(100); - rate.tryAcquire(100); - assertEquals(rate.getAvailablePermits(), 0); - - Thread.sleep(rateTimeMSec * 2); - // check after two rate-time: acquiredPermits is 100 - assertEquals(rate.getAvailablePermits(), 0); - - Thread.sleep(rateTimeMSec); - // check after three rate-time: acquiredPermits is 0 - assertTrue(rate.getAvailablePermits() > 0); - - rate.close(); - } - - @Test - public void testRateLimiterWithPermitUpdater() throws Exception { - long permits = 10; - long rateTime = 1; - long newUpdatedRateLimit = 100L; - Supplier permitUpdater = () -> newUpdatedRateLimit; - RateLimiter limiter = RateLimiter.builder().permits(permits).rateTime(1).timeUnit(TimeUnit.SECONDS) - .permitUpdater(permitUpdater) - .build(); - limiter.acquire(); - Thread.sleep(rateTime * 3 * 1000); - assertEquals(limiter.getAvailablePermits(), newUpdatedRateLimit); - } - - @Test - public void testRateLimiterWithFunction() { - final AtomicInteger atomicInteger = new AtomicInteger(0); - long permits = 10; - long rateTime = 1; - int reNewTime = 3; - RateLimitFunction rateLimitFunction = atomicInteger::incrementAndGet; - RateLimiter rateLimiter = RateLimiter.builder().permits(permits).rateTime(rateTime).timeUnit(TimeUnit.SECONDS) - .rateLimitFunction(rateLimitFunction) - .build(); - for (int i = 0; i < reNewTime; i++) { - rateLimiter.renew(); - } - assertEquals(reNewTime, atomicInteger.get()); - } - -} diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/TrustManagerProxyTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/TrustManagerProxyTest.java index 8114f9b9356f3..ab31740bd5f11 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/TrustManagerProxyTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/TrustManagerProxyTest.java @@ -25,6 +25,7 @@ import java.util.Arrays; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import lombok.Cleanup; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -41,15 +42,12 @@ public static Object[][] caDataProvider() { public void testLoadCA(String path, int count) { String caPath = Resources.getResource(path).getPath(); + @Cleanup("shutdownNow") ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(); - try { - TrustManagerProxy trustManagerProxy = - new TrustManagerProxy(caPath, 120, scheduledExecutor); - X509Certificate[] x509Certificates = trustManagerProxy.getAcceptedIssuers(); - assertNotNull(x509Certificates); - assertEquals(Arrays.stream(x509Certificates).count(), count); - } finally { - scheduledExecutor.shutdown(); - } + TrustManagerProxy trustManagerProxy = + new TrustManagerProxy(caPath, 120, scheduledExecutor); + X509Certificate[] x509Certificates = trustManagerProxy.getAcceptedIssuers(); + assertNotNull(x509Certificates); + assertEquals(Arrays.stream(x509Certificates).count(), count); } } diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMapTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMapTest.java index a317fa63c0986..60bfbd31c2868 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMapTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMapTest.java @@ -38,6 +38,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.LongFunction; import lombok.Cleanup; @@ -213,6 +214,67 @@ public void testExpandShrinkAndClear() { assertTrue(map.capacity() == initCapacity); } + @Test + public void testConcurrentExpandAndShrinkAndGet() throws Throwable { + ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() + .expectedItems(2) + .concurrencyLevel(1) + .autoShrink(true) + .mapIdleFactor(0.25f) + .build(); + assertEquals(map.capacity(), 4); + + @Cleanup("shutdownNow") + ExecutorService executor = Executors.newCachedThreadPool(); + final int readThreads = 16; + final int writeThreads = 1; + final int n = 1_000; + CyclicBarrier barrier = new CyclicBarrier(writeThreads + readThreads); + Future future = null; + AtomicReference ex = new AtomicReference<>(); + + for (int i = 0; i < readThreads; i++) { + executor.submit(() -> { + try { + barrier.await(); + } catch (Exception e) { + throw new RuntimeException(e); + } + try { + map.get(1); + } catch (Exception e) { + ex.set(e); + } + }); + } + + assertNull(map.put(1,"v1")); + future = executor.submit(() -> { + try { + barrier.await(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + for (int i = 0; i < n; i++) { + // expand hashmap + assertNull(map.put(2, "v2")); + assertNull(map.put(3, "v3")); + assertEquals(map.capacity(), 8); + + // shrink hashmap + assertTrue(map.remove(2, "v2")); + assertTrue(map.remove(3, "v3")); + assertEquals(map.capacity(), 4); + } + }); + + future.get(); + assertTrue(ex.get() == null); + // shut down pool + executor.shutdown(); + } + @Test public void testRemove() { ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() @@ -361,7 +423,6 @@ public void concurrentInsertionsAndReads() throws Throwable { assertEquals(map.size(), N * nThreads); } - @Test public void stressConcurrentInsertionsAndReads() throws Throwable { ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() .expectedItems(4) diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMapTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMapTest.java index 8e74d285ffb9b..7b7255bacc854 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMapTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMapTest.java @@ -31,9 +31,13 @@ import java.util.List; import java.util.Map; import java.util.Random; +import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicReference; + +import lombok.Cleanup; import org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap.LongPair; import org.testng.annotations.Test; @@ -173,6 +177,69 @@ public void testExpandAndShrink() { assertEquals(map.capacity(), 8); } + @Test + public void testConcurrentExpandAndShrinkAndGet() throws Throwable { + ConcurrentLongLongPairHashMap map = ConcurrentLongLongPairHashMap.newBuilder() + .expectedItems(2) + .concurrencyLevel(1) + .autoShrink(true) + .mapIdleFactor(0.25f) + .build(); + assertEquals(map.capacity(), 4); + + @Cleanup("shutdownNow") + ExecutorService executor = Executors.newCachedThreadPool(); + final int readThreads = 16; + final int writeThreads = 1; + final int n = 1_000; + CyclicBarrier barrier = new CyclicBarrier(writeThreads + readThreads); + Future future = null; + AtomicReference ex = new AtomicReference<>(); + + for (int i = 0; i < readThreads; i++) { + executor.submit(() -> { + try { + barrier.await(); + } catch (Exception e) { + throw new RuntimeException(e); + } + while (!Thread.currentThread().isInterrupted()) { + try { + map.get(1, 1); + } catch (Exception e) { + ex.set(e); + } + } + }); + } + + assertTrue(map.put(1, 1, 11, 11)); + future = executor.submit(() -> { + try { + barrier.await(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + for (int i = 0; i < n; i++) { + // expand hashmap + assertTrue(map.put(2, 2, 22, 22)); + assertTrue(map.put(3, 3, 33, 33)); + assertEquals(map.capacity(), 8); + + // shrink hashmap + assertTrue(map.remove(2, 2, 22, 22)); + assertTrue(map.remove(3, 3, 33, 33)); + assertEquals(map.capacity(), 4); + } + }); + + future.get(); + assertTrue(ex.get() == null); + // shut down pool + executor.shutdown(); + } + @Test public void testExpandShrinkAndClear() { ConcurrentLongLongPairHashMap map = ConcurrentLongLongPairHashMap.newBuilder() @@ -270,6 +337,7 @@ public void testRehashingWithDeletes() { public void concurrentInsertions() throws Throwable { ConcurrentLongLongPairHashMap map = ConcurrentLongLongPairHashMap.newBuilder() .build(); + @Cleanup("shutdownNow") ExecutorService executor = Executors.newCachedThreadPool(); final int nThreads = 16; @@ -310,6 +378,7 @@ public void concurrentInsertions() throws Throwable { public void concurrentInsertionsAndReads() throws Throwable { ConcurrentLongLongPairHashMap map = ConcurrentLongLongPairHashMap.newBuilder() .build(); + @Cleanup("shutdownNow") ExecutorService executor = Executors.newCachedThreadPool(); final int nThreads = 16; diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSetTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSetTest.java index 7e947ae6e6aa3..0287e986b004d 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSetTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSetTest.java @@ -30,9 +30,11 @@ import java.util.List; import java.util.Random; import java.util.Set; +import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicReference; import lombok.Cleanup; import org.apache.pulsar.common.util.collections.ConcurrentLongPairSet.LongPair; @@ -211,6 +213,69 @@ public void testExpandShrinkAndClear() { assertTrue(map.capacity() == initCapacity); } + @Test + public void testConcurrentExpandAndShrinkAndGet() throws Throwable { + ConcurrentLongPairSet set = ConcurrentLongPairSet.newBuilder() + .expectedItems(2) + .concurrencyLevel(1) + .autoShrink(true) + .mapIdleFactor(0.25f) + .build(); + assertEquals(set.capacity(), 4); + + @Cleanup("shutdownNow") + ExecutorService executor = Executors.newCachedThreadPool(); + final int readThreads = 16; + final int writeThreads = 1; + final int n = 1_000; + CyclicBarrier barrier = new CyclicBarrier(writeThreads + readThreads); + Future future = null; + AtomicReference ex = new AtomicReference<>(); + + for (int i = 0; i < readThreads; i++) { + executor.submit(() -> { + try { + barrier.await(); + } catch (Exception e) { + throw new RuntimeException(e); + } + while (!Thread.currentThread().isInterrupted()) { + try { + set.contains(1, 1); + } catch (Exception e) { + ex.set(e); + } + } + }); + } + + assertTrue(set.add(1, 1)); + future = executor.submit(() -> { + try { + barrier.await(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + for (int i = 0; i < n; i++) { + // expand hashmap + assertTrue(set.add(2, 2)); + assertTrue(set.add(3, 3)); + assertEquals(set.capacity(), 8); + + // shrink hashmap + assertTrue(set.remove(2, 2)); + assertTrue(set.remove(3, 3)); + assertEquals(set.capacity(), 4); + } + }); + + future.get(); + assertTrue(ex.get() == null); + // shut down pool + executor.shutdown(); + } + @Test public void testRemove() { diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMapTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMapTest.java deleted file mode 100644 index 198a3f4c5c38b..0000000000000 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMapTest.java +++ /dev/null @@ -1,635 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util.collections; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotEquals; -import static org.testng.Assert.assertNull; -import static org.testng.Assert.assertThrows; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Random; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Function; - -import lombok.Cleanup; -import org.testng.annotations.Test; - -import com.google.common.collect.Lists; - -public class ConcurrentOpenHashMapTest { - - @Test - public void testConstructor() { - try { - ConcurrentOpenHashMap.newBuilder() - .expectedItems(0) - .build(); - fail("should have thrown exception"); - } catch (IllegalArgumentException e) { - // ok - } - - try { - ConcurrentOpenHashMap.newBuilder() - .expectedItems(16) - .concurrencyLevel(0) - .build(); - fail("should have thrown exception"); - } catch (IllegalArgumentException e) { - // ok - } - - try { - ConcurrentOpenHashMap.newBuilder() - .expectedItems(4) - .concurrencyLevel(8) - .build(); - fail("should have thrown exception"); - } catch (IllegalArgumentException e) { - // ok - } - } - - @Test - public void simpleInsertions() { - ConcurrentOpenHashMap map = - ConcurrentOpenHashMap.newBuilder() - .expectedItems(16) - .build(); - - assertTrue(map.isEmpty()); - assertNull(map.put("1", "one")); - assertFalse(map.isEmpty()); - - assertNull(map.put("2", "two")); - assertNull(map.put("3", "three")); - - assertEquals(map.size(), 3); - - assertEquals(map.get("1"), "one"); - assertEquals(map.size(), 3); - - assertEquals(map.remove("1"), "one"); - assertEquals(map.size(), 2); - assertNull(map.get("1")); - assertNull(map.get("5")); - assertEquals(map.size(), 2); - - assertNull(map.put("1", "one")); - assertEquals(map.size(), 3); - assertEquals(map.put("1", "uno"), "one"); - assertEquals(map.size(), 3); - } - - @Test - public void testReduceUnnecessaryExpansions() { - ConcurrentOpenHashMap map = ConcurrentOpenHashMap.newBuilder() - .expectedItems(2) - .concurrencyLevel(1) - .build(); - assertNull(map.put("1", "1")); - assertNull(map.put("2", "2")); - assertNull(map.put("3", "3")); - assertNull(map.put("4", "4")); - - assertEquals(map.remove("1"), "1"); - assertEquals(map.remove("2"), "2"); - assertEquals(map.remove("3"), "3"); - assertEquals(map.remove("4"), "4"); - - assertEquals(0, map.getUsedBucketCount()); - } - - @Test - public void testClear() { - ConcurrentOpenHashMap map = ConcurrentOpenHashMap.newBuilder() - .expectedItems(2) - .concurrencyLevel(1) - .autoShrink(true) - .mapIdleFactor(0.25f) - .build(); - assertTrue(map.capacity() == 4); - - assertNull(map.put("k1", "v1")); - assertNull(map.put("k2", "v2")); - assertNull(map.put("k3", "v3")); - - assertTrue(map.capacity() == 8); - map.clear(); - assertTrue(map.capacity() == 4); - } - - @Test - public void testExpandAndShrink() { - ConcurrentOpenHashMap map = ConcurrentOpenHashMap.newBuilder() - .expectedItems(2) - .concurrencyLevel(1) - .autoShrink(true) - .mapIdleFactor(0.25f) - .build(); - assertTrue(map.capacity() == 4); - - assertNull(map.put("k1", "v1")); - assertNull(map.put("k2", "v2")); - assertNull(map.put("k3", "v3")); - - // expand hashmap - assertTrue(map.capacity() == 8); - - assertTrue(map.remove("k1", "v1")); - // not shrink - assertTrue(map.capacity() == 8); - assertTrue(map.remove("k2", "v2")); - // shrink hashmap - assertTrue(map.capacity() == 4); - - // expand hashmap - assertNull(map.put("k4", "v4")); - assertNull(map.put("k5", "v5")); - assertTrue(map.capacity() == 8); - - //verify that the map does not keep shrinking at every remove() operation - assertNull(map.put("k6", "v6")); - assertTrue(map.remove("k6", "v6")); - assertTrue(map.capacity() == 8); - } - - @Test - public void testExpandShrinkAndClear() { - ConcurrentOpenHashMap map = ConcurrentOpenHashMap.newBuilder() - .expectedItems(2) - .concurrencyLevel(1) - .autoShrink(true) - .mapIdleFactor(0.25f) - .build(); - final long initCapacity = map.capacity(); - assertTrue(map.capacity() == 4); - assertNull(map.put("k1", "v1")); - assertNull(map.put("k2", "v2")); - assertNull(map.put("k3", "v3")); - - // expand hashmap - assertTrue(map.capacity() == 8); - - assertTrue(map.remove("k1", "v1")); - // not shrink - assertTrue(map.capacity() == 8); - assertTrue(map.remove("k2", "v2")); - // shrink hashmap - assertTrue(map.capacity() == 4); - - assertTrue(map.remove("k3", "v3")); - // Will not shrink the hashmap again because shrink capacity is less than initCapacity - // current capacity is equal than the initial capacity - assertTrue(map.capacity() == initCapacity); - map.clear(); - // after clear, because current capacity is equal than the initial capacity, so not shrinkToInitCapacity - assertTrue(map.capacity() == initCapacity); - } - - @Test - public void testRemove() { - ConcurrentOpenHashMap map = - ConcurrentOpenHashMap.newBuilder().build(); - - assertTrue(map.isEmpty()); - assertNull(map.put("1", "one")); - assertFalse(map.isEmpty()); - - assertFalse(map.remove("0", "zero")); - assertFalse(map.remove("1", "uno")); - - assertFalse(map.isEmpty()); - assertTrue(map.remove("1", "one")); - assertTrue(map.isEmpty()); - } - - @Test - public void testRehashing() { - int n = 16; - ConcurrentOpenHashMap map = ConcurrentOpenHashMap.newBuilder() - .expectedItems(n / 2) - .concurrencyLevel(1) - .build(); - assertEquals(map.capacity(), n); - assertEquals(map.size(), 0); - - for (int i = 0; i < n; i++) { - map.put(Integer.toString(i), i); - } - - assertEquals(map.capacity(), 2 * n); - assertEquals(map.size(), n); - } - - @Test - public void testRehashingWithDeletes() { - int n = 16; - ConcurrentOpenHashMap map = - ConcurrentOpenHashMap.newBuilder() - .expectedItems(n / 2) - .concurrencyLevel(1) - .build(); - assertEquals(map.capacity(), n); - assertEquals(map.size(), 0); - - for (int i = 0; i < n / 2; i++) { - map.put(i, i); - } - - for (int i = 0; i < n / 2; i++) { - map.remove(i); - } - - for (int i = n; i < (2 * n); i++) { - map.put(i, i); - } - - assertEquals(map.capacity(), 2 * n); - assertEquals(map.size(), n); - } - - @Test - public void concurrentInsertions() throws Throwable { - ConcurrentOpenHashMap map = ConcurrentOpenHashMap.newBuilder() - .expectedItems(16) - .concurrencyLevel(1) - .build(); - @Cleanup("shutdownNow") - ExecutorService executor = Executors.newCachedThreadPool(); - - final int nThreads = 16; - final int N = 100_000; - String value = "value"; - - List> futures = new ArrayList<>(); - for (int i = 0; i < nThreads; i++) { - final int threadIdx = i; - - futures.add(executor.submit(() -> { - Random random = new Random(); - - for (int j = 0; j < N; j++) { - long key = random.nextLong(); - // Ensure keys are uniques - key -= key % (threadIdx + 1); - - map.put(key, value); - } - })); - } - - for (Future future : futures) { - future.get(); - } - - assertEquals(map.size(), N * nThreads); - } - - @Test - public void concurrentInsertionsAndReads() throws Throwable { - ConcurrentOpenHashMap map = - ConcurrentOpenHashMap.newBuilder().build(); - @Cleanup("shutdownNow") - ExecutorService executor = Executors.newCachedThreadPool(); - - final int nThreads = 16; - final int N = 100_000; - String value = "value"; - - List> futures = new ArrayList<>(); - for (int i = 0; i < nThreads; i++) { - final int threadIdx = i; - - futures.add(executor.submit(() -> { - Random random = new Random(); - - for (int j = 0; j < N; j++) { - long key = random.nextLong(); - // Ensure keys are uniques - key -= key % (threadIdx + 1); - - map.put(key, value); - } - })); - } - - for (Future future : futures) { - future.get(); - } - - assertEquals(map.size(), N * nThreads); - } - - @Test - public void testIteration() { - ConcurrentOpenHashMap map = - ConcurrentOpenHashMap.newBuilder().build(); - - assertEquals(map.keys(), Collections.emptyList()); - assertEquals(map.values(), Collections.emptyList()); - - map.put(0l, "zero"); - - assertEquals(map.keys(), Lists.newArrayList(0l)); - assertEquals(map.values(), Lists.newArrayList("zero")); - - map.remove(0l); - - assertEquals(map.keys(), Collections.emptyList()); - assertEquals(map.values(), Collections.emptyList()); - - map.put(0l, "zero"); - map.put(1l, "one"); - map.put(2l, "two"); - - List keys = map.keys(); - keys.sort(null); - assertEquals(keys, Lists.newArrayList(0l, 1l, 2l)); - - List values = map.values(); - values.sort(null); - assertEquals(values, Lists.newArrayList("one", "two", "zero")); - - map.put(1l, "uno"); - - keys = map.keys(); - keys.sort(null); - assertEquals(keys, Lists.newArrayList(0l, 1l, 2l)); - - values = map.values(); - values.sort(null); - assertEquals(values, Lists.newArrayList("two", "uno", "zero")); - - map.clear(); - assertTrue(map.isEmpty()); - } - - @Test - public void testHashConflictWithDeletion() { - final int Buckets = 16; - ConcurrentOpenHashMap map = ConcurrentOpenHashMap.newBuilder() - .expectedItems(Buckets) - .concurrencyLevel(1) - .build(); - - // Pick 2 keys that fall into the same bucket - long key1 = 1; - long key2 = 27; - - int bucket1 = ConcurrentOpenHashMap.signSafeMod(ConcurrentOpenHashMap.hash(key1), Buckets); - int bucket2 = ConcurrentOpenHashMap.signSafeMod(ConcurrentOpenHashMap.hash(key2), Buckets); - assertEquals(bucket1, bucket2); - - assertNull(map.put(key1, "value-1")); - assertNull(map.put(key2, "value-2")); - assertEquals(map.size(), 2); - - assertEquals(map.remove(key1), "value-1"); - assertEquals(map.size(), 1); - - assertNull(map.put(key1, "value-1-overwrite")); - assertEquals(map.size(), 2); - - assertEquals(map.remove(key1), "value-1-overwrite"); - assertEquals(map.size(), 1); - - assertEquals(map.put(key2, "value-2-overwrite"), "value-2"); - assertEquals(map.get(key2), "value-2-overwrite"); - - assertEquals(map.size(), 1); - assertEquals(map.remove(key2), "value-2-overwrite"); - assertTrue(map.isEmpty()); - } - - @Test - public void testPutIfAbsent() { - ConcurrentOpenHashMap map = - ConcurrentOpenHashMap.newBuilder().build(); - assertNull(map.putIfAbsent(1l, "one")); - assertEquals(map.get(1l), "one"); - - assertEquals(map.putIfAbsent(1l, "uno"), "one"); - assertEquals(map.get(1l), "one"); - } - - @Test - public void testComputeIfAbsent() { - ConcurrentOpenHashMap map = ConcurrentOpenHashMap.newBuilder() - .expectedItems(16) - .concurrencyLevel(1) - .build(); - AtomicInteger counter = new AtomicInteger(); - Function provider = key -> counter.getAndIncrement(); - - assertEquals(map.computeIfAbsent(0, provider).intValue(), 0); - assertEquals(map.get(0).intValue(), 0); - - assertEquals(map.computeIfAbsent(1, provider).intValue(), 1); - assertEquals(map.get(1).intValue(), 1); - - assertEquals(map.computeIfAbsent(1, provider).intValue(), 1); - assertEquals(map.get(1).intValue(), 1); - - assertEquals(map.computeIfAbsent(2, provider).intValue(), 2); - assertEquals(map.get(2).intValue(), 2); - } - - @Test - public void testEqualsKeys() { - class T { - int value; - - T(int value) { - this.value = value; - } - - @Override - public int hashCode() { - return Integer.hashCode(value); - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof T) { - return value == ((T) obj).value; - } - - return false; - } - } - - ConcurrentOpenHashMap map = - ConcurrentOpenHashMap.newBuilder().build(); - - T t1 = new T(1); - T t1_b = new T(1); - T t2 = new T(2); - - assertEquals(t1, t1_b); - assertNotEquals(t2, t1); - assertNotEquals(t2, t1_b); - - assertNull(map.put(t1, "t1")); - assertEquals(map.get(t1), "t1"); - assertEquals(map.get(t1_b), "t1"); - assertNull(map.get(t2)); - - assertEquals(map.remove(t1_b), "t1"); - assertNull(map.get(t1)); - assertNull(map.get(t1_b)); - } - - @Test - public void testNullValue() { - ConcurrentOpenHashMap map = - ConcurrentOpenHashMap.newBuilder() - .expectedItems(16) - .concurrencyLevel(1) - .build(); - String key = "a"; - assertThrows(NullPointerException.class, () -> map.put(key, null)); - - //put a null value. - assertNull(map.computeIfAbsent(key, k -> null)); - assertEquals(1, map.size()); - assertEquals(1, map.keys().size()); - assertEquals(1, map.values().size()); - assertNull(map.get(key)); - assertFalse(map.containsKey(key)); - - //test remove null value - map.removeNullValue(key); - assertTrue(map.isEmpty()); - assertEquals(0, map.keys().size()); - assertEquals(0, map.values().size()); - assertNull(map.get(key)); - assertFalse(map.containsKey(key)); - - - //test not remove non-null value - map.put(key, "V"); - assertEquals(1, map.size()); - map.removeNullValue(key); - assertEquals(1, map.size()); - - } - - static final int Iterations = 1; - static final int ReadIterations = 1000; - static final int N = 1_000_000; - - public void benchConcurrentOpenHashMap() throws Exception { - ConcurrentOpenHashMap map = ConcurrentOpenHashMap.newBuilder() - .expectedItems(N) - .concurrencyLevel(1) - .build(); - - for (long i = 0; i < Iterations; i++) { - for (int j = 0; j < N; j++) { - map.put(i, "value"); - } - - for (long h = 0; h < ReadIterations; h++) { - for (int j = 0; j < N; j++) { - map.get(i); - } - } - - for (long j = 0; j < N; j++) { - map.remove(i); - } - } - } - - public void benchConcurrentHashMap() throws Exception { - ConcurrentHashMap map = new ConcurrentHashMap(N, 0.66f, 1); - - for (long i = 0; i < Iterations; i++) { - for (int j = 0; j < N; j++) { - map.put(i, "value"); - } - - for (long h = 0; h < ReadIterations; h++) { - for (int j = 0; j < N; j++) { - map.get(i); - } - } - - for (int j = 0; j < N; j++) { - map.remove(i); - } - } - } - - void benchHashMap() { - HashMap map = new HashMap<>(N, 0.66f); - - for (long i = 0; i < Iterations; i++) { - for (int j = 0; j < N; j++) { - map.put(i, "value"); - } - - for (long h = 0; h < ReadIterations; h++) { - for (int j = 0; j < N; j++) { - map.get(i); - } - } - - for (int j = 0; j < N; j++) { - map.remove(i); - } - } - } - - public static void main(String[] args) throws Exception { - ConcurrentOpenHashMapTest t = new ConcurrentOpenHashMapTest(); - - long start = System.nanoTime(); - t.benchHashMap(); - long end = System.nanoTime(); - - System.out.println("HM: " + TimeUnit.NANOSECONDS.toMillis(end - start) + " ms"); - - start = System.nanoTime(); - t.benchConcurrentHashMap(); - end = System.nanoTime(); - - System.out.println("CHM: " + TimeUnit.NANOSECONDS.toMillis(end - start) + " ms"); - - start = System.nanoTime(); - t.benchConcurrentOpenHashMap(); - end = System.nanoTime(); - - System.out.println("CLHM: " + TimeUnit.NANOSECONDS.toMillis(end - start) + " ms"); - - } -} diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSetTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSetTest.java deleted file mode 100644 index 27c18abb8b347..0000000000000 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSetTest.java +++ /dev/null @@ -1,438 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util.collections; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotEquals; -import static org.testng.Assert.assertThrows; -import static org.testng.Assert.assertTrue; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Random; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -import lombok.Cleanup; -import org.testng.annotations.Test; - -import com.google.common.collect.Lists; - -// Deprecation warning suppressed as this test targets deprecated class -@SuppressWarnings("deprecation") -public class ConcurrentOpenHashSetTest { - - @Test - public void testConstructor() { - assertThrows(IllegalArgumentException.class, () -> new ConcurrentOpenHashSet(0)); - assertThrows(IllegalArgumentException.class, () -> new ConcurrentOpenHashSet(16, 0)); - assertThrows(IllegalArgumentException.class, () -> new ConcurrentOpenHashSet(4, 8)); - } - - @Test - public void simpleInsertions() { - ConcurrentOpenHashSet set = new ConcurrentOpenHashSet<>(16); - - assertTrue(set.isEmpty()); - assertTrue(set.add("1")); - assertFalse(set.isEmpty()); - - assertTrue(set.add("2")); - assertTrue(set.add("3")); - - assertEquals(set.size(), 3); - - assertTrue(set.contains("1")); - assertEquals(set.size(), 3); - - assertTrue(set.remove("1")); - assertEquals(set.size(), 2); - assertFalse(set.contains("1")); - assertFalse(set.contains("5")); - assertEquals(set.size(), 2); - - assertTrue(set.add("1")); - assertEquals(set.size(), 3); - assertFalse(set.add("1")); - assertEquals(set.size(), 3); - } - - @Test - public void testReduceUnnecessaryExpansions() { - ConcurrentOpenHashSet set = - ConcurrentOpenHashSet.newBuilder() - .expectedItems(2) - .concurrencyLevel(1) - .build(); - - assertTrue(set.add("1")); - assertTrue(set.add("2")); - assertTrue(set.add("3")); - assertTrue(set.add("4")); - - assertTrue(set.remove("1")); - assertTrue(set.remove("2")); - assertTrue(set.remove("3")); - assertTrue(set.remove("4")); - assertEquals(0, set.getUsedBucketCount()); - } - - @Test - public void testClear() { - ConcurrentOpenHashSet set = - ConcurrentOpenHashSet.newBuilder() - .expectedItems(2) - .concurrencyLevel(1) - .autoShrink(true) - .mapIdleFactor(0.25f) - .build(); - assertEquals(set.capacity(), 4); - - assertTrue(set.add("k1")); - assertTrue(set.add("k2")); - assertTrue(set.add("k3")); - - assertEquals(set.capacity(), 8); - set.clear(); - assertEquals(set.capacity(), 4); - } - - @Test - public void testExpandAndShrink() { - ConcurrentOpenHashSet map = - ConcurrentOpenHashSet.newBuilder() - .expectedItems(2) - .concurrencyLevel(1) - .autoShrink(true) - .mapIdleFactor(0.25f) - .build(); - assertEquals(map.capacity(), 4); - - assertTrue(map.add("k1")); - assertTrue(map.add("k2")); - assertTrue(map.add("k3")); - - // expand hashmap - assertEquals(map.capacity(), 8); - - assertTrue(map.remove("k1")); - // not shrink - assertEquals(map.capacity(), 8); - assertTrue(map.remove("k2")); - // shrink hashmap - assertEquals(map.capacity(), 4); - - // expand hashmap - assertTrue(map.add("k4")); - assertTrue(map.add("k5")); - assertEquals(map.capacity(), 8); - - //verify that the map does not keep shrinking at every remove() operation - assertTrue(map.add("k6")); - assertTrue(map.remove("k6")); - assertEquals(map.capacity(), 8); - } - - @Test - public void testExpandShrinkAndClear() { - ConcurrentOpenHashSet map = ConcurrentOpenHashSet.newBuilder() - .expectedItems(2) - .concurrencyLevel(1) - .autoShrink(true) - .mapIdleFactor(0.25f) - .build(); - final long initCapacity = map.capacity(); - assertTrue(map.capacity() == 4); - - assertTrue(map.add("k1")); - assertTrue(map.add("k2")); - assertTrue(map.add("k3")); - - // expand hashmap - assertTrue(map.capacity() == 8); - - assertTrue(map.remove("k1")); - // not shrink - assertTrue(map.capacity() == 8); - assertTrue(map.remove("k2")); - // shrink hashmap - assertTrue(map.capacity() == 4); - - assertTrue(map.remove("k3")); - // Will not shrink the hashmap again because shrink capacity is less than initCapacity - // current capacity is equal than the initial capacity - assertTrue(map.capacity() == initCapacity); - map.clear(); - // after clear, because current capacity is equal than the initial capacity, so not shrinkToInitCapacity - assertTrue(map.capacity() == initCapacity); - } - - @Test - public void testRemove() { - ConcurrentOpenHashSet set = - ConcurrentOpenHashSet.newBuilder().build(); - - assertTrue(set.isEmpty()); - assertTrue(set.add("1")); - assertFalse(set.isEmpty()); - - assertFalse(set.remove("0")); - assertFalse(set.isEmpty()); - assertTrue(set.remove("1")); - assertTrue(set.isEmpty()); - } - - @Test - public void testRehashing() { - int n = 16; - ConcurrentOpenHashSet set = new ConcurrentOpenHashSet<>(n / 2, 1); - assertEquals(set.capacity(), n); - assertEquals(set.size(), 0); - - for (int i = 0; i < n; i++) { - set.add(i); - } - - assertEquals(set.capacity(), 2 * n); - assertEquals(set.size(), n); - } - - @Test - public void testRehashingWithDeletes() { - int n = 16; - ConcurrentOpenHashSet set = new ConcurrentOpenHashSet<>(n / 2, 1); - assertEquals(set.capacity(), n); - assertEquals(set.size(), 0); - - for (int i = 0; i < n / 2; i++) { - set.add(i); - } - - for (int i = 0; i < n / 2; i++) { - set.remove(i); - } - - for (int i = n; i < (2 * n); i++) { - set.add(i); - } - - assertEquals(set.capacity(), 2 * n); - assertEquals(set.size(), n); - } - - @Test - public void concurrentInsertions() throws Throwable { - ConcurrentOpenHashSet set = - ConcurrentOpenHashSet.newBuilder().build(); - @Cleanup("shutdownNow") - ExecutorService executor = Executors.newCachedThreadPool(); - - final int nThreads = 16; - final int N = 100_000; - - List> futures = new ArrayList<>(); - for (int i = 0; i < nThreads; i++) { - final int threadIdx = i; - - futures.add(executor.submit(() -> { - Random random = new Random(); - - for (int j = 0; j < N; j++) { - long key = random.nextLong(); - // Ensure keys are unique - key -= key % (threadIdx + 1); - - set.add(key); - } - })); - } - - for (Future future : futures) { - future.get(); - } - - assertEquals(set.size(), N * nThreads); - } - - @Test - public void concurrentInsertionsAndReads() throws Throwable { - ConcurrentOpenHashSet map = - ConcurrentOpenHashSet.newBuilder().build(); - @Cleanup("shutdownNow") - ExecutorService executor = Executors.newCachedThreadPool(); - - final int nThreads = 16; - final int N = 100_000; - - List> futures = new ArrayList<>(); - for (int i = 0; i < nThreads; i++) { - final int threadIdx = i; - - futures.add(executor.submit(() -> { - Random random = new Random(); - - for (int j = 0; j < N; j++) { - long key = random.nextLong(); - // Ensure keys are unique - key -= key % (threadIdx + 1); - - map.add(key); - } - })); - } - - for (Future future : futures) { - future.get(); - } - - assertEquals(map.size(), N * nThreads); - } - - @Test - public void testIteration() { - ConcurrentOpenHashSet set = ConcurrentOpenHashSet.newBuilder().build(); - - assertEquals(set.values(), Collections.emptyList()); - - set.add(0l); - - assertEquals(set.values(), Lists.newArrayList(0l)); - - set.remove(0l); - - assertEquals(set.values(), Collections.emptyList()); - - set.add(0l); - set.add(1l); - set.add(2l); - - List values = set.values(); - values.sort(null); - assertEquals(values, Lists.newArrayList(0l, 1l, 2l)); - - set.clear(); - assertTrue(set.isEmpty()); - } - - @Test - public void testRemoval() { - ConcurrentOpenHashSet set = - ConcurrentOpenHashSet.newBuilder().build(); - - set.add(0); - set.add(1); - set.add(3); - set.add(6); - set.add(7); - - List values = set.values(); - values.sort(null); - assertEquals(values, Lists.newArrayList(0, 1, 3, 6, 7)); - - int numOfItemsDeleted = set.removeIf(i -> i < 5); - assertEquals(numOfItemsDeleted, 3); - assertEquals(set.size(), values.size() - numOfItemsDeleted); - values = set.values(); - values.sort(null); - assertEquals(values, Lists.newArrayList(6, 7)); - } - - @Test - public void testHashConflictWithDeletion() { - final int Buckets = 16; - ConcurrentOpenHashSet set = new ConcurrentOpenHashSet<>(Buckets, 1); - - // Pick 2 keys that fall into the same bucket - long key1 = 1; - long key2 = 27; - - int bucket1 = ConcurrentOpenHashSet.signSafeMod(ConcurrentOpenHashSet.hash(key1), Buckets); - int bucket2 = ConcurrentOpenHashSet.signSafeMod(ConcurrentOpenHashSet.hash(key2), Buckets); - assertEquals(bucket1, bucket2); - - assertTrue(set.add(key1)); - assertTrue(set.add(key2)); - assertEquals(set.size(), 2); - - assertTrue(set.remove(key1)); - assertEquals(set.size(), 1); - - assertTrue(set.add(key1)); - assertEquals(set.size(), 2); - - assertTrue(set.remove(key1)); - assertEquals(set.size(), 1); - - assertFalse(set.add(key2)); - assertTrue(set.contains(key2)); - - assertEquals(set.size(), 1); - assertTrue(set.remove(key2)); - assertTrue(set.isEmpty()); - } - - @Test - public void testEqualsObjects() { - class T { - int value; - - T(int value) { - this.value = value; - } - - @Override - public int hashCode() { - return Integer.hashCode(value); - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof T) { - return value == ((T) obj).value; - } - - return false; - } - } - - ConcurrentOpenHashSet set = - ConcurrentOpenHashSet.newBuilder().build(); - - T t1 = new T(1); - T t1_b = new T(1); - T t2 = new T(2); - - assertEquals(t1, t1_b); - assertNotEquals(t2, t1); - assertNotEquals(t2, t1_b); - - set.add(t1); - assertTrue(set.contains(t1)); - assertTrue(set.contains(t1_b)); - assertFalse(set.contains(t2)); - - assertTrue(set.remove(t1_b)); - assertFalse(set.contains(t1)); - assertFalse(set.contains(t1_b)); - } - -} diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSetTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSetTest.java deleted file mode 100644 index eff49883215d7..0000000000000 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentSortedLongPairSetTest.java +++ /dev/null @@ -1,291 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util.collections; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotEquals; -import static org.testng.Assert.assertTrue; -import com.google.common.collect.ComparisonChain; -import com.google.common.collect.Lists; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import lombok.Cleanup; -import org.apache.pulsar.common.util.collections.ConcurrentLongPairSet.LongPair; -import org.testng.annotations.Test; - -public class ConcurrentSortedLongPairSetTest { - - @Test - public void simpleInsertions() { - LongPairSet set = new ConcurrentSortedLongPairSet(16); - - assertTrue(set.isEmpty()); - assertTrue(set.add(1, 1)); - assertFalse(set.isEmpty()); - - assertTrue(set.add(2, 2)); - assertTrue(set.add(3, 3)); - - assertEquals(set.size(), 3); - - assertTrue(set.contains(1, 1)); - assertEquals(set.size(), 3); - - assertTrue(set.remove(1, 1)); - assertEquals(set.size(), 2); - assertFalse(set.contains(1, 1)); - assertFalse(set.contains(5, 5)); - assertEquals(set.size(), 2); - - assertTrue(set.add(1, 1)); - assertEquals(set.size(), 3); - assertFalse(set.add(1, 1)); - assertEquals(set.size(), 3); - } - - @Test - public void testRemove() { - LongPairSet set = new ConcurrentSortedLongPairSet(16); - - assertTrue(set.isEmpty()); - assertTrue(set.add(1, 1)); - assertFalse(set.isEmpty()); - - assertFalse(set.remove(1, 0)); - assertFalse(set.isEmpty()); - assertTrue(set.remove(1, 1)); - assertTrue(set.isEmpty()); - } - - @Test - public void concurrentInsertions() throws Throwable { - LongPairSet set = new ConcurrentSortedLongPairSet(16); - @Cleanup("shutdownNow") - ExecutorService executor = Executors.newCachedThreadPool(); - - final int nThreads = 8; - final int N = 1000; - - List> futures = new ArrayList<>(); - for (int i = 0; i < nThreads; i++) { - final int threadIdx = i; - - futures.add(executor.submit(() -> { - Random random = new Random(); - - for (int j = 0; j < N; j++) { - long key = random.nextLong(); - // Ensure keys are unique - key -= key % (threadIdx + 1); - key = Math.abs(key); - set.add(key, key); - } - })); - } - - for (Future future : futures) { - future.get(); - } - - assertEquals(set.size(), N * nThreads); - } - - @Test - public void testIteration() { - LongPairSet set = new ConcurrentSortedLongPairSet(16); - - for (int i = 0; i < 10; i++) { - for (int j = 0; j < 10; j++) { - set.add(i, j); - } - } - - for (int i = 0; i < 10; i++) { - final int firstKey = i; - Set longSetResult = set.items(10); - assertEquals(longSetResult.size(), 10); - longSetResult.forEach(longPair -> { - assertEquals(firstKey, longPair.first); - }); - set.removeIf((item1, item2) -> item1 == firstKey); - } - - } - - @Test - public void testRemoval() { - LongPairSet set = new ConcurrentSortedLongPairSet(16); - - set.add(0, 0); - set.add(1, 1); - set.add(3, 3); - set.add(6, 6); - set.add(7, 7); - - List values = new ArrayList<>(set.items()); - values.sort(null); - assertEquals(values, Lists.newArrayList(new LongPair(0, 0), new LongPair(1, 1), new LongPair(3, 3), - new LongPair(6, 6), new LongPair(7, 7))); - - set.forEach((first, second) -> { - if (first < 5) { - set.remove(first, second); - } - }); - assertEquals(set.size(), values.size() - 3); - values = new ArrayList<>(set.items()); - values.sort(null); - assertEquals(values, Lists.newArrayList(new LongPair(6, 6), new LongPair(7, 7))); - } - - @Test - public void testIfRemoval() { - LongPairSet set = new ConcurrentSortedLongPairSet(16, 1, 1); - - set.add(0, 0); - set.add(1, 1); - set.add(3, 3); - set.add(6, 6); - set.add(7, 7); - - List values = new ArrayList<>(set.items()); - values.sort(null); - assertEquals(values, Lists.newArrayList(new LongPair(0, 0), new LongPair(1, 1), new LongPair(3, 3), - new LongPair(6, 6), new LongPair(7, 7))); - - int removeItems = set.removeIf((first, second) -> first < 5); - - assertEquals(3, removeItems); - assertEquals(set.size(), values.size() - 3); - values = new ArrayList<>(set.items()); - values.sort(null); - assertEquals(values, Lists.newArrayList(new LongPair(6, 6), new LongPair(7, 7))); - - set = new ConcurrentSortedLongPairSet(128, 2, true); - set.add(2, 2); - set.add(1, 3); - set.add(3, 1); - set.add(2, 1); - set.add(3, 2); - set.add(1, 2); - set.add(1, 1); - removeItems = set.removeIf((ledgerId, entryId) -> { - return ComparisonChain.start().compare(ledgerId, 1).compare(entryId, 3) - .result() <= 0; - }); - assertEquals(removeItems, 3); - } - - @Test - public void testItems() { - LongPairSet set = new ConcurrentSortedLongPairSet(16); - - int n = 100; - int limit = 10; - for (int i = 0; i < n; i++) { - set.add(i, i); - } - - Set items = set.items(); - Set limitItems = set.items(limit); - assertEquals(items.size(), n); - assertEquals(limitItems.size(), limit); - - int totalRemovedItems = set.removeIf((first, second) -> limitItems.contains((new LongPair(first, second)))); - assertEquals(limitItems.size(), totalRemovedItems); - assertEquals(set.size(), n - limit); - } - - @Test - public void testEqualsObjects() { - - LongPairSet set = new ConcurrentSortedLongPairSet(16); - - long t1 = 1; - long t2 = 2; - long t1_b = 1; - assertEquals(t1, t1_b); - assertNotEquals(t2, t1); - assertNotEquals(t2, t1_b); - - set.add(t1, t1); - assertTrue(set.contains(t1, t1)); - assertTrue(set.contains(t1_b, t1_b)); - assertFalse(set.contains(t2, t2)); - - assertTrue(set.remove(t1_b, t1_b)); - assertFalse(set.contains(t1, t1)); - assertFalse(set.contains(t1_b, t1_b)); - } - - @Test - public void testToString() { - - LongPairSet set = new ConcurrentSortedLongPairSet(16); - - set.add(0, 0); - set.add(1, 1); - set.add(3, 3); - final String toString = "{[0:0], [1:1], [3:3]}"; - System.out.println(set.toString()); - assertEquals(set.toString(), toString); - } - - @Test - public void testIsEmpty() { - LongPairSet set = new ConcurrentSortedLongPairSet(); - assertTrue(set.isEmpty()); - set.add(1, 1); - assertFalse(set.isEmpty()); - } - - @Test - public void testShrink() { - LongPairSet set = new ConcurrentSortedLongPairSet(2, 1, true); - set.add(0, 0); - assertTrue(set.capacity() == 4); - set.add(0, 1); - assertTrue(set.capacity() == 4); - set.add(1, 1); - assertTrue(set.capacity() == 8); - set.add(1, 2); - assertTrue(set.capacity() == 8); - set.add(1, 3); - set.add(1, 4); - set.add(1, 5); - assertTrue(set.capacity() == 12); - set.remove(1, 5); - // not shrink - assertTrue(set.capacity() == 12); - set.remove(1, 4); - // the internal map does not keep shrinking at every remove() operation - assertTrue(set.capacity() == 12); - set.remove(1, 3); - set.remove(1, 2); - set.remove(1, 1); - // shrink - assertTrue(set.capacity() == 8); - } -} diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/DefaultRangeSetTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/DefaultRangeSetTest.java index f6103061a420c..730f4b4ceca22 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/DefaultRangeSetTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/DefaultRangeSetTest.java @@ -34,8 +34,8 @@ public class DefaultRangeSetTest { public void testBehavior() { LongPairRangeSet.DefaultRangeSet set = new LongPairRangeSet.DefaultRangeSet<>(consumer, reverseConsumer); - ConcurrentOpenLongPairRangeSet rangeSet = - new ConcurrentOpenLongPairRangeSet<>(consumer); + OpenLongPairRangeSet rangeSet = + new OpenLongPairRangeSet<>(consumer); assertNull(set.firstRange()); assertNull(set.lastRange()); diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSetTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/OpenLongPairRangeSetTest.java similarity index 92% rename from pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSetTest.java rename to pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/OpenLongPairRangeSetTest.java index 40bb337935742..4dd0f5551f1f9 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSetTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/OpenLongPairRangeSetTest.java @@ -37,14 +37,14 @@ import com.google.common.collect.Range; import com.google.common.collect.TreeRangeSet; -public class ConcurrentOpenLongPairRangeSetTest { +public class OpenLongPairRangeSetTest { static final LongPairConsumer consumer = LongPair::new; static final RangeBoundConsumer reverseConsumer = pair -> pair; @Test public void testIsEmpty() { - ConcurrentOpenLongPairRangeSet set = new ConcurrentOpenLongPairRangeSet<>(consumer); + OpenLongPairRangeSet set = new OpenLongPairRangeSet<>(consumer); assertTrue(set.isEmpty()); // lowerValueOpen and upperValue are both -1 so that an empty set will be added set.addOpenClosed(0, -1, 0, -1); @@ -55,7 +55,7 @@ public void testIsEmpty() { @Test public void testAddForSameKey() { - ConcurrentOpenLongPairRangeSet set = new ConcurrentOpenLongPairRangeSet<>(consumer); + OpenLongPairRangeSet set = new OpenLongPairRangeSet<>(consumer); // add 0 to 5 set.add(Range.closed(new LongPair(0, 0), new LongPair(0, 5))); // add 8,9,10 @@ -76,7 +76,7 @@ public void testAddForSameKey() { @Test public void testAddForDifferentKey() { - ConcurrentOpenLongPairRangeSet set = new ConcurrentOpenLongPairRangeSet<>(consumer); + OpenLongPairRangeSet set = new OpenLongPairRangeSet<>(consumer); // [98,100],[(1,5),(1,5)],[(1,10,1,15)],[(1,20),(1,20)],[(2,0),(2,10)] set.addOpenClosed(0, 98, 0, 99); set.addOpenClosed(0, 100, 1, 5); @@ -93,7 +93,7 @@ public void testAddForDifferentKey() { @Test public void testAddCompareCompareWithGuava() { - ConcurrentOpenLongPairRangeSet set = new ConcurrentOpenLongPairRangeSet<>(consumer); + OpenLongPairRangeSet set = new OpenLongPairRangeSet<>(consumer); com.google.common.collect.RangeSet gSet = TreeRangeSet.create(); // add 10K values for key 0 @@ -132,14 +132,14 @@ public void testAddCompareCompareWithGuava() { @Test public void testNPE() { - ConcurrentOpenLongPairRangeSet set = new ConcurrentOpenLongPairRangeSet<>(consumer); + OpenLongPairRangeSet set = new OpenLongPairRangeSet<>(consumer); assertNull(set.span()); } @Test public void testDeleteCompareWithGuava() { - ConcurrentOpenLongPairRangeSet set = new ConcurrentOpenLongPairRangeSet<>(consumer); + OpenLongPairRangeSet set = new OpenLongPairRangeSet<>(consumer); com.google.common.collect.RangeSet gSet = TreeRangeSet.create(); // add 10K values for key 0 @@ -193,7 +193,7 @@ public void testDeleteCompareWithGuava() { @Test public void testRemoveRangeInSameKey() { - ConcurrentOpenLongPairRangeSet set = new ConcurrentOpenLongPairRangeSet<>(consumer); + OpenLongPairRangeSet set = new OpenLongPairRangeSet<>(consumer); set.addOpenClosed(0, 1, 0, 50); set.addOpenClosed(0, 97, 0, 99); set.addOpenClosed(0, 99, 1, 5); @@ -217,7 +217,7 @@ public void testRemoveRangeInSameKey() { @Test public void testSpanWithGuava() { - ConcurrentOpenLongPairRangeSet set = new ConcurrentOpenLongPairRangeSet<>(consumer); + OpenLongPairRangeSet set = new OpenLongPairRangeSet<>(consumer); com.google.common.collect.RangeSet gSet = TreeRangeSet.create(); set.add(Range.openClosed(new LongPair(0, 97), new LongPair(0, 99))); gSet.add(Range.openClosed(new LongPair(0, 97), new LongPair(0, 99))); @@ -242,7 +242,7 @@ public void testSpanWithGuava() { @Test public void testFirstRange() { - ConcurrentOpenLongPairRangeSet set = new ConcurrentOpenLongPairRangeSet<>(consumer); + OpenLongPairRangeSet set = new OpenLongPairRangeSet<>(consumer); assertNull(set.firstRange()); Range range = Range.openClosed(new LongPair(0, 97), new LongPair(0, 99)); set.add(range); @@ -260,7 +260,7 @@ public void testFirstRange() { @Test public void testLastRange() { - ConcurrentOpenLongPairRangeSet set = new ConcurrentOpenLongPairRangeSet<>(consumer); + OpenLongPairRangeSet set = new OpenLongPairRangeSet<>(consumer); assertNull(set.lastRange()); Range range = Range.openClosed(new LongPair(0, 97), new LongPair(0, 99)); set.add(range); @@ -282,7 +282,7 @@ public void testLastRange() { @Test public void testToString() { - ConcurrentOpenLongPairRangeSet set = new ConcurrentOpenLongPairRangeSet<>(consumer); + OpenLongPairRangeSet set = new OpenLongPairRangeSet<>(consumer); Range range = Range.openClosed(new LongPair(0, 97), new LongPair(0, 99)); set.add(range); assertEquals(set.toString(), "[(0:97..0:99]]"); @@ -296,7 +296,7 @@ public void testToString() { @Test public void testDeleteForDifferentKey() { - ConcurrentOpenLongPairRangeSet set = new ConcurrentOpenLongPairRangeSet<>(consumer); + OpenLongPairRangeSet set = new OpenLongPairRangeSet<>(consumer); set.addOpenClosed(0, 97, 0, 99); set.addOpenClosed(0, 99, 1, 5); set.addOpenClosed(1, 9, 1, 15); @@ -327,7 +327,7 @@ public void testDeleteForDifferentKey() { @Test public void testDeleteWithAtMost() { - ConcurrentOpenLongPairRangeSet set = new ConcurrentOpenLongPairRangeSet<>(consumer); + OpenLongPairRangeSet set = new OpenLongPairRangeSet<>(consumer); set.add(Range.closed(new LongPair(0, 98), new LongPair(0, 99))); set.add(Range.closed(new LongPair(0, 100), new LongPair(1, 5))); set.add(Range.closed(new LongPair(1, 10), new LongPair(1, 15))); @@ -353,7 +353,7 @@ public void testDeleteWithAtMost() { @Test public void testDeleteWithLeastMost() { - ConcurrentOpenLongPairRangeSet set = new ConcurrentOpenLongPairRangeSet<>(consumer); + OpenLongPairRangeSet set = new OpenLongPairRangeSet<>(consumer); set.add(Range.closed(new LongPair(0, 98), new LongPair(0, 99))); set.add(Range.closed(new LongPair(0, 100), new LongPair(1, 5))); set.add(Range.closed(new LongPair(1, 10), new LongPair(1, 15))); @@ -382,7 +382,7 @@ public void testDeleteWithLeastMost() { @Test public void testRangeContaining() { - ConcurrentOpenLongPairRangeSet set = new ConcurrentOpenLongPairRangeSet<>(consumer); + OpenLongPairRangeSet set = new OpenLongPairRangeSet<>(consumer); set.add(Range.closed(new LongPair(0, 98), new LongPair(0, 99))); set.add(Range.closed(new LongPair(0, 100), new LongPair(1, 5))); com.google.common.collect.RangeSet gSet = TreeRangeSet.create(); @@ -423,7 +423,7 @@ public void testRangeContaining() { */ @Test public void testCacheFlagConflict() { - ConcurrentOpenLongPairRangeSet set = new ConcurrentOpenLongPairRangeSet<>(consumer); + OpenLongPairRangeSet set = new OpenLongPairRangeSet<>(consumer); set.add(Range.openClosed(new LongPair(0, 1), new LongPair(0, 2))); set.add(Range.openClosed(new LongPair(0, 3), new LongPair(0, 4))); assertEquals(set.toString(), "[(0:1..0:2],(0:3..0:4]]"); @@ -466,7 +466,7 @@ private List> getConnectedRange(Set> gRanges) { @Test public void testCardinality() { - ConcurrentOpenLongPairRangeSet set = new ConcurrentOpenLongPairRangeSet<>(consumer); + OpenLongPairRangeSet set = new OpenLongPairRangeSet<>(consumer); int v = set.cardinality(0, 0, Integer.MAX_VALUE, Integer.MAX_VALUE); assertEquals(v, 0 ); set.addOpenClosed(1, 0, 1, 20); @@ -486,8 +486,8 @@ public void testCardinality() { @Test public void testForEachResultTheSameAsForEachWithRangeBoundMapper() { - ConcurrentOpenLongPairRangeSet set = - new ConcurrentOpenLongPairRangeSet<>(consumer); + OpenLongPairRangeSet set = + new OpenLongPairRangeSet<>(consumer); LongPairRangeSet.DefaultRangeSet defaultRangeSet = new LongPairRangeSet.DefaultRangeSet<>(consumer, reverseConsumer); diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/netty/DnsResolverTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/netty/DnsResolverTest.java index 0ccb960e79887..46599cc45a090 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/netty/DnsResolverTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/netty/DnsResolverTest.java @@ -18,13 +18,57 @@ */ package org.apache.pulsar.common.util.netty; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.testng.Assert.assertEquals; import io.netty.channel.EventLoop; import io.netty.resolver.dns.DnsNameResolverBuilder; +import java.security.Security; +import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; public class DnsResolverTest { + private static final int MIN_TTL = 0; + private static final int TTL = 101; + private static final int NEGATIVE_TTL = 121; + private static final String CACHE_POLICY_PROP = "networkaddress.cache.ttl"; + private static final String NEGATIVE_CACHE_POLICY_PROP = "networkaddress.cache.negative.ttl"; + + private String originalCachePolicy; + private String originalNegativeCachePolicy; + + @BeforeClass(alwaysRun = true) + public void beforeClass() { + originalCachePolicy = Security.getProperty(CACHE_POLICY_PROP); + originalNegativeCachePolicy = Security.getProperty(NEGATIVE_CACHE_POLICY_PROP); + Security.setProperty(CACHE_POLICY_PROP, Integer.toString(TTL)); + Security.setProperty(NEGATIVE_CACHE_POLICY_PROP, Integer.toString(NEGATIVE_TTL)); + } + + @AfterClass(alwaysRun = true) + public void afterClass() { + Security.setProperty(CACHE_POLICY_PROP, originalCachePolicy != null ? originalCachePolicy : "-1"); + Security.setProperty(NEGATIVE_CACHE_POLICY_PROP, + originalNegativeCachePolicy != null ? originalNegativeCachePolicy : "0"); + } + + @Test + public void testTTl() { + final DnsNameResolverBuilder builder = mock(DnsNameResolverBuilder.class); + ArgumentCaptor minTtlCaptor = ArgumentCaptor.forClass(Integer.class); + ArgumentCaptor maxTtlCaptor = ArgumentCaptor.forClass(Integer.class); + ArgumentCaptor negativeTtlCaptor = ArgumentCaptor.forClass(Integer.class); + DnsResolverUtil.applyJdkDnsCacheSettings(builder); + verify(builder).ttl(minTtlCaptor.capture(), maxTtlCaptor.capture()); + verify(builder).negativeTtl(negativeTtlCaptor.capture()); + assertEquals(minTtlCaptor.getValue(), MIN_TTL); + assertEquals(maxTtlCaptor.getValue(), TTL); + assertEquals(negativeTtlCaptor.getValue(), NEGATIVE_TTL); + } @Test public void testMaxTtl() { diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/netty/SslContextTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/netty/SslContextTest.java index 303df5a003278..120fee9319db7 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/netty/SslContextTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/netty/SslContextTest.java @@ -21,16 +21,14 @@ import static org.testng.Assert.assertThrows; import com.google.common.io.Resources; import io.netty.handler.ssl.SslProvider; -import java.io.IOException; -import java.security.GeneralSecurityException; import java.util.HashSet; import java.util.Set; import javax.net.ssl.SSLException; import org.apache.pulsar.client.api.AuthenticationDataProvider; import org.apache.pulsar.client.api.KeyStoreParams; -import org.apache.pulsar.common.util.NettyClientSslContextRefresher; -import org.apache.pulsar.common.util.NettyServerSslContextBuilder; -import org.apache.pulsar.common.util.keystoretls.NettySSLContextAutoRefreshBuilder; +import org.apache.pulsar.common.util.DefaultPulsarSslFactory; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -84,13 +82,22 @@ public static Object[] getCipher() { @Test(dataProvider = "cipherDataProvider") public void testServerKeyStoreSSLContext(Set cipher) throws Exception { - NettySSLContextAutoRefreshBuilder contextAutoRefreshBuilder = new NettySSLContextAutoRefreshBuilder( - null, - keyStoreType, brokerKeyStorePath, keyStorePassword, false, - keyStoreType, brokerTrustStorePath, keyStorePassword, - true, cipher, - null, 600); - contextAutoRefreshBuilder.update(); + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsEnabledWithKeystore(true) + .tlsKeyStoreType(keyStoreType) + .tlsKeyStorePath(brokerKeyStorePath) + .tlsKeyStorePassword(keyStorePassword) + .allowInsecureConnection(false) + .tlsTrustStoreType(keyStoreType) + .tlsTrustStorePath(brokerTrustStorePath) + .tlsTrustStorePassword(keyStorePassword) + .requireTrustedClientCertOnConnect(true) + .tlsCiphers(cipher) + .build(); + try (PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory()) { + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + } } private static class ClientAuthenticationData implements AuthenticationDataProvider { @@ -102,45 +109,67 @@ public KeyStoreParams getTlsKeyStoreParams() { @Test(dataProvider = "cipherDataProvider") public void testClientKeyStoreSSLContext(Set cipher) throws Exception { - NettySSLContextAutoRefreshBuilder contextAutoRefreshBuilder = new NettySSLContextAutoRefreshBuilder( - null, - false, - keyStoreType, brokerTrustStorePath, keyStorePassword, - null, null, null, - cipher, null, 0, new ClientAuthenticationData()); - contextAutoRefreshBuilder.update(); + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .allowInsecureConnection(false) + .tlsEnabledWithKeystore(true) + .tlsTrustStoreType(keyStoreType) + .tlsTrustStorePath(brokerTrustStorePath) + .tlsTrustStorePassword(keyStorePassword) + .tlsCiphers(cipher) + .authData(new ClientAuthenticationData()) + .build(); + try (PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory()) { + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + } } @Test(dataProvider = "caCertSslContextDataProvider") public void testServerCaCertSslContextWithSslProvider(SslProvider sslProvider, Set ciphers) - throws GeneralSecurityException, IOException { - NettyServerSslContextBuilder sslContext = new NettyServerSslContextBuilder(sslProvider, - true, - caCertPath, brokerCertPath, brokerKeyPath, - ciphers, - null, - true, 60); - if (ciphers != null) { - if (sslProvider == null || sslProvider == SslProvider.OPENSSL) { - assertThrows(SSLException.class, sslContext::update); - return; + throws Exception { + try (PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory()) { + PulsarSslConfiguration.PulsarSslConfigurationBuilder builder = PulsarSslConfiguration.builder() + .tlsTrustCertsFilePath(caCertPath) + .tlsCertificateFilePath(brokerCertPath) + .tlsKeyFilePath(brokerKeyPath) + .tlsCiphers(ciphers) + .requireTrustedClientCertOnConnect(true); + if (sslProvider != null) { + builder.tlsProvider(sslProvider.name()); } + PulsarSslConfiguration pulsarSslConfiguration = builder.build(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + + if (ciphers != null) { + if (sslProvider == null || sslProvider == SslProvider.OPENSSL) { + assertThrows(SSLException.class, pulsarSslFactory::createInternalSslContext); + return; + } + } + pulsarSslFactory.createInternalSslContext(); } - sslContext.update(); } @Test(dataProvider = "caCertSslContextDataProvider") public void testClientCaCertSslContextWithSslProvider(SslProvider sslProvider, Set ciphers) - throws GeneralSecurityException, IOException { - NettyClientSslContextRefresher sslContext = new NettyClientSslContextRefresher(sslProvider, - true, caCertPath, - null, ciphers, null, 0); - if (ciphers != null) { - if (sslProvider == null || sslProvider == SslProvider.OPENSSL) { - assertThrows(SSLException.class, sslContext::update); - return; + throws Exception { + try (PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory()) { + PulsarSslConfiguration.PulsarSslConfigurationBuilder builder = PulsarSslConfiguration.builder() + .allowInsecureConnection(true) + .tlsTrustCertsFilePath(caCertPath) + .tlsCiphers(ciphers); + if (sslProvider != null) { + builder.tlsProvider(sslProvider.name()); + } + PulsarSslConfiguration pulsarSslConfiguration = builder.build(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + if (ciphers != null) { + if (sslProvider == null || sslProvider == SslProvider.OPENSSL) { + assertThrows(SSLException.class, pulsarSslFactory::createInternalSslContext); + return; + } } + pulsarSslFactory.createInternalSslContext(); } - sslContext.update(); } } diff --git a/pulsar-common/src/test/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerDataTest.java b/pulsar-common/src/test/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerDataTest.java index db55ecfe5035a..b5d7e3c355a50 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerDataTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerDataTest.java @@ -20,7 +20,8 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.gson.Gson; +import com.fasterxml.jackson.databind.ObjectReader; +import org.apache.pulsar.common.util.ObjectMapperFactory; import org.testng.Assert; import org.testng.annotations.Test; @@ -30,14 +31,26 @@ public class LocalBrokerDataTest { @Test - public void testLocalBrokerDataDeserialization() { + public void testLocalBrokerDataDeserialization() throws JsonProcessingException { + ObjectReader LOAD_REPORT_READER = ObjectMapperFactory.getMapper().reader() + .forType(LoadManagerReport.class); String data = "{\"webServiceUrl\":\"http://10.244.2.23:8080\",\"webServiceUrlTls\":\"https://10.244.2.23:8081\",\"pulsarServiceUrlTls\":\"pulsar+ssl://10.244.2.23:6651\",\"persistentTopicsEnabled\":true,\"nonPersistentTopicsEnabled\":false,\"cpu\":{\"usage\":3.1577712104798255,\"limit\":100.0},\"memory\":{\"usage\":614.0,\"limit\":1228.0},\"directMemory\":{\"usage\":32.0,\"limit\":1228.0},\"bandwidthIn\":{\"usage\":0.0,\"limit\":0.0},\"bandwidthOut\":{\"usage\":0.0,\"limit\":0.0},\"msgThroughputIn\":0.0,\"msgThroughputOut\":0.0,\"msgRateIn\":0.0,\"msgRateOut\":0.0,\"lastUpdate\":1650886425227,\"lastStats\":{\"pulsar/pulsar/10.244.2.23:8080/0x00000000_0xffffffff\":{\"msgRateIn\":0.0,\"msgThroughputIn\":0.0,\"msgRateOut\":0.0,\"msgThroughputOut\":0.0,\"consumerCount\":0,\"producerCount\":0,\"topics\":1,\"cacheSize\":0}},\"numTopics\":1,\"numBundles\":1,\"numConsumers\":0,\"numProducers\":0,\"bundles\":[\"pulsar/pulsar/10.244.2.23:8080/0x00000000_0xffffffff\"],\"lastBundleGains\":[],\"lastBundleLosses\":[],\"brokerVersionString\":\"2.11.0-hw-0.0.4-SNAPSHOT\",\"protocols\":{},\"advertisedListeners\":{},\"bundleStats\":{\"pulsar/pulsar/10.244.2.23:8080/0x00000000_0xffffffff\":{\"msgRateIn\":0.0,\"msgThroughputIn\":0.0,\"msgRateOut\":0.0,\"msgThroughputOut\":0.0,\"consumerCount\":0,\"producerCount\":0,\"topics\":1,\"cacheSize\":0}},\"maxResourceUsage\":0.49645519256591797,\"loadReportType\":\"LocalBrokerData\"}"; - Gson gson = new Gson(); - LocalBrokerData localBrokerData = gson.fromJson(data, LocalBrokerData.class); + LoadManagerReport localBrokerData = LOAD_REPORT_READER.readValue(data); Assert.assertEquals(localBrokerData.getMemory().limit, 1228.0d, 0.0001f); Assert.assertEquals(localBrokerData.getMemory().usage, 614.0d, 0.0001f); Assert.assertEquals(localBrokerData.getMemory().percentUsage(), ((float) localBrokerData.getMemory().usage) / ((float) localBrokerData.getMemory().limit) * 100, 0.0001f); } + @Test + public void testTimeAverageBrokerDataDataDeserialization() throws JsonProcessingException { + ObjectReader TIME_AVERAGE_READER = ObjectMapperFactory.getMapper().reader() + .forType(TimeAverageBrokerData.class); + String data = "{\"shortTermMsgThroughputIn\":100,\"shortTermMsgThroughputOut\":200,\"shortTermMsgRateIn\":300,\"shortTermMsgRateOut\":400,\"longTermMsgThroughputIn\":567.891,\"longTermMsgThroughputOut\":678.912,\"longTermMsgRateIn\":789.123,\"longTermMsgRateOut\":890.123}"; + TimeAverageBrokerData timeAverageBrokerData = TIME_AVERAGE_READER.readValue(data); + assertEquals(timeAverageBrokerData.getShortTermMsgThroughputIn(), 100.00); + assertEquals(timeAverageBrokerData.getShortTermMsgThroughputOut(), 200.00); + assertEquals(timeAverageBrokerData.getShortTermMsgRateIn(), 300.00); + assertEquals(timeAverageBrokerData.getShortTermMsgRateOut(), 400.00); + } @Test public void testMaxResourceUsage() { diff --git a/pulsar-config-validation/pom.xml b/pulsar-config-validation/pom.xml index 9c7ab1061b70b..5b418f1a53560 100644 --- a/pulsar-config-validation/pom.xml +++ b/pulsar-config-validation/pom.xml @@ -26,8 +26,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-config-validation @@ -59,7 +58,7 @@ - + com.github.spotbugs spotbugs-maven-plugin diff --git a/pulsar-config-validation/src/main/java/org/apache/pulsar/config/validation/Validator.java b/pulsar-config-validation/src/main/java/org/apache/pulsar/config/validation/Validator.java index ea3332d886c9e..d850b2b654568 100644 --- a/pulsar-config-validation/src/main/java/org/apache/pulsar/config/validation/Validator.java +++ b/pulsar-config-validation/src/main/java/org/apache/pulsar/config/validation/Validator.java @@ -32,7 +32,7 @@ public Validator() { } /** - * validate the field value o that belogs to the field which is named name + * validate the field value o that belongs to the field which is named name * This method should throw IllegalArgumentException in case o doesn't * validate per this validator's implementation. */ diff --git a/pulsar-docs-tools/pom.xml b/pulsar-docs-tools/pom.xml new file mode 100644 index 0000000000000..63b74f2c63ee6 --- /dev/null +++ b/pulsar-docs-tools/pom.xml @@ -0,0 +1,51 @@ + + + + 4.0.0 + + + org.apache.pulsar + pulsar + 4.0.0-SNAPSHOT + + + pulsar-docs-tools + Pulsar Documentation Generators + + + + io.swagger + swagger-annotations + + + io.swagger + swagger-core + + + info.picocli + picocli + + + + diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/BaseGenerateDocumentation.java b/pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/BaseGenerateDocumentation.java similarity index 88% rename from pulsar-common/src/main/java/org/apache/pulsar/common/util/BaseGenerateDocumentation.java rename to pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/BaseGenerateDocumentation.java index b88ed197e8f2e..ff474d98edc1a 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/BaseGenerateDocumentation.java +++ b/pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/BaseGenerateDocumentation.java @@ -16,11 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.common.util; +package org.apache.pulsar.docs.tools; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; import io.swagger.annotations.ApiModelProperty; import java.io.Serializable; import java.lang.annotation.Annotation; @@ -29,59 +26,52 @@ import java.util.Arrays; import java.util.Comparator; import java.util.List; +import java.util.concurrent.Callable; import java.util.function.Predicate; import java.util.stream.Collectors; -import lombok.Data; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.reflect.MethodUtils; import org.apache.commons.lang3.tuple.Pair; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.ScopeType; -@Data -@Parameters(commandDescription = "Generate documentation automatically.") @Slf4j -public abstract class BaseGenerateDocumentation { +@Command(name = "gen-doc", showDefaultValues = true, scope = ScopeType.INHERIT) +public abstract class BaseGenerateDocumentation implements Callable { - JCommander jcommander; + CommandLine commander; - @Parameter(names = {"-c", "--class-names"}, description = + @Option(names = {"-c", "--class-names"}, description = "List of class names, generate documentation based on the annotations in the Class") private List classNames = new ArrayList<>(); - @Parameter(names = {"-h", "--help"}, help = true, description = "Show this help.") + @Option(names = {"-h", "--help"}, usageHelp = true, description = "Show this help.") boolean help; public BaseGenerateDocumentation() { - jcommander = new JCommander(); - jcommander.setProgramName("pulsar-generateDocumentation"); - jcommander.addObject(this); + commander = new CommandLine(this); } - public boolean run(String[] args) throws Exception { - if (args.length == 0) { - jcommander.usage(); - return false; - } - - if (help) { - jcommander.usage(); - return true; - } - - try { - jcommander.parse(Arrays.copyOfRange(args, 0, args.length)); - } catch (Exception e) { - System.err.println(e.getMessage()); - jcommander.usage(); - return false; - } + @Override + public Integer call() throws Exception { if (classNames != null) { for (String className : classNames) { System.out.println(generateDocumentByClassName(className)); } } - return true; + return 0; + } + + public boolean run(String[] args) throws Exception { + if (args.length == 0) { + commander.usage(commander.getOut()); + return false; + } + return commander.execute(args) == 0; } protected abstract String generateDocumentByClassName(String className) throws Exception; @@ -186,7 +176,9 @@ public int compare(Pair o1, Pair clazz = Class.forName(className); Object obj = clazz.getDeclaredConstructor().newInstance(); Field[] fields = clazz.getDeclaredFields(); @@ -218,7 +210,9 @@ protected String generateDocByFieldContext(String className, String type, String return sb.toString(); } - protected String generateDocByApiModelProperty(String className, String type, StringBuilder sb) throws Exception { + protected String generateDocByApiModelProperty(String className, String type) throws Exception { + final StringBuilder sb = new StringBuilder(); + Class clazz = Class.forName(className); Object obj = clazz.getDeclaredConstructor().newInstance(); Field[] fields = clazz.getDeclaredFields(); diff --git a/pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/CmdGenerateDocs.java b/pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/CmdGenerateDocs.java new file mode 100644 index 0000000000000..a66da9fd6c650 --- /dev/null +++ b/pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/CmdGenerateDocs.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.docs.tools; + +import com.google.common.annotations.VisibleForTesting; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import lombok.Getter; +import lombok.Setter; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Model.ArgSpec; +import picocli.CommandLine.Model.OptionSpec; +import picocli.CommandLine.Option; +import picocli.CommandLine.ScopeType; + +@Getter +@Setter +@Command(showDefaultValues = true, scope = ScopeType.INHERIT) +public class CmdGenerateDocs implements Callable { + + @Option( + names = {"-h", "--help"}, + description = "Display help information", + usageHelp = true + ) + public boolean help; + + @Option( + names = {"-n", "--command-names"}, + description = "List of command names" + ) + private List commandNames = new ArrayList<>(); + + private static final String name = "gen-doc"; + private final CommandLine commander; + + public CmdGenerateDocs(String cmdName) { + commander = new CommandLine(this); + commander.setCommandName(cmdName); + } + + public CmdGenerateDocs addCommand(String name, Object command) { + commander.addSubcommand(name, command); + return this; + } + + public boolean run(String[] args) { + if (args == null) { + args = new String[]{}; + } + return commander.execute(args) == 0; + } + + private static String getCommandDescription(CommandLine commandLine) { + String[] description = commandLine.getCommandSpec().usageMessage().description(); + if (description != null && description.length != 0) { + return description[0]; + } + return ""; + } + + private static String getArgDescription(ArgSpec argSpec) { + String[] description = argSpec.description(); + if (description != null && description.length != 0) { + return description[0]; + } + return ""; + } + + private String generateDocument(String module, CommandLine commander) { + StringBuilder sb = new StringBuilder(); + sb.append("# ").append(module).append("\n\n"); + String desc = getCommandDescription(commander); + if (null != desc && !desc.isEmpty()) { + sb.append(desc).append("\n"); + } + sb.append("\n\n```shell\n") + .append("$ "); + String commandName = commander.getCommandName(); + sb.append(this.commander.getCommandName() + " " + commandName); + if (!commander.getSubcommands().isEmpty()) { + sb.append(" subcommand").append("\n```").append("\n\n"); + commander.getSubcommands().forEach((subK, subV) -> { + if (!subK.equals(name)) { + sb.append("\n\n## ").append(subK).append("\n\n"); + String subDesc = getCommandDescription(subV); + if (null != subDesc && !subDesc.isEmpty()) { + sb.append(subDesc).append("\n"); + } + sb.append("```shell\n$ "); + sb.append(this.commander.getCommandName()).append(" "); + sb.append(module).append(" ").append(subK).append(" options").append("\n```\n\n"); + List argSpecs = subV.getCommandSpec().args(); + if (argSpecs.size() > 0) { + sb.append("|Flag|Description|Default|\n"); + sb.append("|---|---|---|\n"); + } + + argSpecs.forEach(option -> { + if (option.hidden() || !(option instanceof OptionSpec)) { + return; + } + sb.append("| `").append(String.join(", ", ((OptionSpec) option).names())) + .append("` | ").append(getArgDescription(option).replace("\n", " ")) + .append("|").append(option.defaultValueString()).append("|\n"); + }); + } + }); + } else { + sb.append(" options").append("\n```").append("\n\n"); + sb.append("|Flag|Description|Default|\n"); + sb.append("|---|---|---|\n"); + List argSpecs = commander.getCommandSpec().args(); + argSpecs.forEach(option -> { + if (option.hidden() || !(option instanceof OptionSpec)) { + return; + } + sb.append("| `") + .append(String.join(", ", ((OptionSpec) option).names())) + .append("` | ") + .append(getArgDescription(option).replace("\n", " ")) + .append("|") + .append(option.defaultValueString()).append("|\n"); + }); + } + return sb.toString(); + } + + @Override + public Integer call() throws Exception { + if (commandNames.size() == 0) { + commander.getSubcommands().forEach((name, cmd) -> { + commander.getOut().println(generateDocument(name, cmd)); + }); + } else { + for (String commandName : commandNames) { + if (commandName.equals(name)) { + continue; + } + CommandLine cmd = commander.getSubcommands().get(commandName); + if (cmd == null) { + continue; + } + commander.getOut().println(generateDocument(commandName, cmd)); + } + } + return 0; + } + + @VisibleForTesting + CommandLine getCommander() { + return commander; + } +} diff --git a/pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/package-info.java b/pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/package-info.java new file mode 100644 index 0000000000000..d971fc8ee6b11 --- /dev/null +++ b/pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.docs.tools; diff --git a/pulsar-docs-tools/src/test/java/org/apache/pulsar/docs/tools/CmdGenerateDocsTest.java b/pulsar-docs-tools/src/test/java/org/apache/pulsar/docs/tools/CmdGenerateDocsTest.java new file mode 100644 index 0000000000000..0f0f96f80ecb0 --- /dev/null +++ b/pulsar-docs-tools/src/test/java/org/apache/pulsar/docs/tools/CmdGenerateDocsTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.docs.tools; + +import static org.testng.Assert.assertEquals; +import java.io.PrintWriter; +import java.io.StringWriter; +import org.testng.annotations.Test; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +public class CmdGenerateDocsTest { + + @Command + public class Arguments { + @Option(names = {"-h", "--help"}, description = "Show this help message") + private boolean help = false; + + @Option(names = {"-n", "--name"}, description = "Name") + private String name; + } + + @Test + public void testHelp() { + CmdGenerateDocs cmd = new CmdGenerateDocs("pulsar"); + cmd.addCommand("test", new Arguments()); + StringWriter stringWriter = new StringWriter(); + cmd.getCommander().setOut(new PrintWriter(stringWriter)); + cmd.run(new String[]{"-h"}); + + String message = stringWriter.toString(); + String rightMsg = "Usage: pulsar [-h] [-n=]... [COMMAND]\n" + + " -h, --help Display help information\n" + + " -n, --command-names=\n" + + " List of command names\n" + + " Default: []\n" + + "Commands:\n" + + " test\n"; + assertEquals(message, rightMsg); + } + + @Test + public void testGenerateDocs() { + CmdGenerateDocs cmd = new CmdGenerateDocs("pulsar"); + cmd.addCommand("test", new Arguments()); + StringWriter stringWriter = new StringWriter(); + cmd.getCommander().setOut(new PrintWriter(stringWriter)); + cmd.run(null); + + String message = stringWriter.toString(); + String rightMsg = "# test\n\n" + + "\n" + + "\n" + + "```shell\n" + + "$ pulsar test options\n" + + "```\n" + + "\n" + + "|Flag|Description|Default|\n" + + "|---|---|---|\n" + + "| `-h, --help` | Show this help message|false|\n" + + "| `-n, --name` | Name|null|\n" + + System.lineSeparator(); + assertEquals(message, rightMsg); + } +} diff --git a/pulsar-function-go/conf/conf.go b/pulsar-function-go/conf/conf.go index d52b886b540f9..1442a0f865f4a 100644 --- a/pulsar-function-go/conf/conf.go +++ b/pulsar-function-go/conf/conf.go @@ -33,14 +33,16 @@ import ( const ConfigPath = "conf/conf.yaml" type Conf struct { - PulsarServiceURL string `json:"pulsarServiceURL" yaml:"pulsarServiceURL"` - InstanceID int `json:"instanceID" yaml:"instanceID"` - FuncID string `json:"funcID" yaml:"funcID"` - FuncVersion string `json:"funcVersion" yaml:"funcVersion"` - MaxBufTuples int `json:"maxBufTuples" yaml:"maxBufTuples"` - Port int `json:"port" yaml:"port"` - ClusterName string `json:"clusterName" yaml:"clusterName"` - KillAfterIdleMs time.Duration `json:"killAfterIdleMs" yaml:"killAfterIdleMs"` + PulsarServiceURL string `json:"pulsarServiceURL" yaml:"pulsarServiceURL"` + StateStorageServiceURL string `json:"stateStorageServiceUrl" yaml:"stateStorageServiceUrl"` + PulsarWebServiceURL string `json:"pulsarWebServiceUrl" yaml:"pulsarWebServiceUrl"` + InstanceID int `json:"instanceID" yaml:"instanceID"` + FuncID string `json:"funcID" yaml:"funcID"` + FuncVersion string `json:"funcVersion" yaml:"funcVersion"` + MaxBufTuples int `json:"maxBufTuples" yaml:"maxBufTuples"` + Port int `json:"port" yaml:"port"` + ClusterName string `json:"clusterName" yaml:"clusterName"` + KillAfterIdleMs time.Duration `json:"killAfterIdleMs" yaml:"killAfterIdleMs"` // function details config Tenant string `json:"tenant" yaml:"tenant"` NameSpace string `json:"nameSpace" yaml:"nameSpace"` @@ -49,7 +51,13 @@ type Conf struct { ProcessingGuarantees int32 `json:"processingGuarantees" yaml:"processingGuarantees"` SecretsMap string `json:"secretsMap" yaml:"secretsMap"` Runtime int32 `json:"runtime" yaml:"runtime"` - //Deprecated + // Authentication + ClientAuthenticationPlugin string `json:"clientAuthenticationPlugin" yaml:"clientAuthenticationPlugin"` + ClientAuthenticationParameters string `json:"clientAuthenticationParameters" yaml:"clientAuthenticationParameters"` + TLSTrustCertsFilePath string `json:"tlsTrustCertsFilePath" yaml:"tlsTrustCertsFilePath"` + TLSAllowInsecureConnection bool `json:"tlsAllowInsecureConnection" yaml:"tlsAllowInsecureConnection"` + TLSHostnameVerificationEnable bool `json:"tlsHostnameVerificationEnable" yaml:"tlsHostnameVerificationEnable"` + // Deprecated AutoACK bool `json:"autoAck" yaml:"autoAck"` Parallelism int32 `json:"parallelism" yaml:"parallelism"` //source config @@ -83,6 +91,8 @@ type Conf struct { UserConfig string `json:"userConfig" yaml:"userConfig"` //metrics config MetricsPort int `json:"metricsPort" yaml:"metricsPort"` + // FunctionDetails + FunctionDetails string `json:"functionDetails" yaml:"functionDetails"` } var ( diff --git a/pulsar-function-go/examples/go.mod b/pulsar-function-go/examples/go.mod index 074b50d2e66e9..0c2c6235b0fb6 100644 --- a/pulsar-function-go/examples/go.mod +++ b/pulsar-function-go/examples/go.mod @@ -1,12 +1,59 @@ module github.com/apache/pulsar/pulsar-function-go/examples -go 1.13 +go 1.21 require ( github.com/apache/pulsar-client-go v0.8.1 github.com/apache/pulsar/pulsar-function-go v0.0.0 - github.com/datadog/zstd v1.4.6-0.20200617134701-89f69fb7df32 // indirect - github.com/yahoo/athenz v1.8.55 // indirect +) + +require ( + github.com/99designs/keyring v1.1.6 // indirect + github.com/AthenZ/athenz v1.10.39 // indirect + github.com/DataDog/zstd v1.5.0 // indirect + github.com/apache/pulsar-client-go/oauth2 v0.0.0-20220120090717-25e59572242e // indirect + github.com/ardielle/ardielle-go v1.5.2 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/danieljoos/wincred v1.0.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dvsekhvalnov/jose2go v1.6.0 // indirect + github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/snappy v0.0.1 // indirect + github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect + github.com/keybase/go-keychain v0.0.0-20190712205309-48d3d31d256d // indirect + github.com/klauspost/compress v1.10.8 // indirect + github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect + github.com/linkedin/goavro/v2 v2.9.8 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mtibben/percent v0.2.1 // indirect + github.com/pierrec/lz4 v2.0.5+incompatible // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.15.1 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect + github.com/sirupsen/logrus v1.6.0 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/stretchr/testify v1.8.4 // indirect + go.uber.org/atomic v1.7.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/oauth2 v0.13.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect + google.golang.org/grpc v1.60.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/apache/pulsar/pulsar-function-go => ../ diff --git a/pulsar-function-go/examples/go.sum b/pulsar-function-go/examples/go.sum index d167adb92fe41..37c84e71c8b26 100644 --- a/pulsar-function-go/examples/go.sum +++ b/pulsar-function-go/examples/go.sum @@ -37,18 +37,13 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/99designs/keyring v1.1.5 h1:wLv7QyzYpFIyMSwOADq1CLTF9KbjbBfcnfmOGJ64aO4= -github.com/99designs/keyring v1.1.5/go.mod h1:7hsVvt2qXgtadGevGJ4ujg+u8m6SpJ5TpHqTozIPqf0= +github.com/99designs/keyring v1.1.6 h1:kVDC2uCgVwecxCk+9zoCt2uEL6dt+dfVzMvGgnVcIuM= github.com/99designs/keyring v1.1.6/go.mod h1:16e0ds7LGQQcT59QqkTg72Hh5ShM51Byv5PEmW6uoRU= -github.com/AthenZ/athenz v1.10.15 h1:8Bc2W313k/ev/SGokuthNbzpwfg9W3frg3PKq1r943I= -github.com/AthenZ/athenz v1.10.15/go.mod h1:7KMpEuJ9E4+vMCMI3UQJxwWs0RZtQq7YXZ1IteUjdsc= +github.com/AthenZ/athenz v1.10.39 h1:mtwHTF/v62ewY2Z5KWhuZgVXftBej1/Tn80zx4DcawY= github.com/AthenZ/athenz v1.10.39/go.mod h1:3Tg8HLsiQZp81BJY58JBeU2BR6B/H4/0MQGfCwhHNEA= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/zstd v1.4.6-0.20200617134701-89f69fb7df32 h1:/gZKpgSMydtrih81nvUhlkXpZIUfthKShSCVbRzBt9Y= -github.com/DataDog/zstd v1.4.6-0.20200617134701-89f69fb7df32/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 h1:++HGU87uq9UsSTlFeiOV9uZR3NpYkndUXeYyLv2DTc8= -github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -56,24 +51,10 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apache/pulsar-client-go v0.0.0-20200113085434-9b739cf9d098/go.mod h1:G+CQVHnh2EPfNEQXOuisIDAyPMiKnzz4Vim/kjtj4U4= -github.com/apache/pulsar-client-go v0.0.0-20200116214305-4d788d9935ed h1:Lp7eU5ym84jPmIXoonoaJWVN6psyB90Olookp61LCeA= -github.com/apache/pulsar-client-go v0.0.0-20200116214305-4d788d9935ed/go.mod h1:G+CQVHnh2EPfNEQXOuisIDAyPMiKnzz4Vim/kjtj4U4= -github.com/apache/pulsar-client-go v0.1.0/go.mod h1:G+CQVHnh2EPfNEQXOuisIDAyPMiKnzz4Vim/kjtj4U4= -github.com/apache/pulsar-client-go v0.2.0 h1:7teu0FaXzzKPjDdUNjA7dVYKFjCy6OVX5as6nUww4qk= -github.com/apache/pulsar-client-go v0.2.0/go.mod h1:POSPPmXv1RuoM7FzHaS3NurCSOopwin2ekGK2PcOgVM= -github.com/apache/pulsar-client-go v0.3.1-0.20201201083639-154bff0bb825 h1:RfvcnGzo67yEHA+eDjoeAEwx5ZxWDgIoMGHJ/Z6Zq9A= -github.com/apache/pulsar-client-go v0.3.1-0.20201201083639-154bff0bb825/go.mod h1:pTmScVVHRhbB8wh0J+m5ZzHI0Lyfe0TwfPEbYEh+JUw= -github.com/apache/pulsar-client-go v0.7.0 h1:sZBkjJPHC7akM8n8DuzkLdwioKPSzyub3efCJ1Ltw9Y= -github.com/apache/pulsar-client-go v0.7.0/go.mod h1:EauTUv9sTmP9QRznRgK9hxnzCsIVfS8fyhTfGcuJBrE= github.com/apache/pulsar-client-go v0.8.1 h1:UZINLbH3I5YtNzqkju7g9vrl4CKrEgYSx2rbpvGufrE= github.com/apache/pulsar-client-go v0.8.1/go.mod h1:yJNcvn/IurarFDxwmoZvb2Ieylg630ifxeO/iXpk27I= -github.com/apache/pulsar-client-go/oauth2 v0.0.0-20200715083626-b9f8c5cedefb h1:E1P0FudxDdj2RhbveZC9i3PwukLCA/4XQSkBS/dw6/I= -github.com/apache/pulsar-client-go/oauth2 v0.0.0-20200715083626-b9f8c5cedefb/go.mod h1:0UtvvETGDdvXNDCHa8ZQpxl+w3HbdFtfYZvDHLgWGTY= -github.com/apache/pulsar-client-go/oauth2 v0.0.0-20201120111947-b8bd55bc02bd h1:P5kM7jcXJ7TaftX0/EMKiSJgvQc/ct+Fw0KMvcH3WuY= -github.com/apache/pulsar-client-go/oauth2 v0.0.0-20201120111947-b8bd55bc02bd/go.mod h1:0UtvvETGDdvXNDCHa8ZQpxl+w3HbdFtfYZvDHLgWGTY= +github.com/apache/pulsar-client-go/oauth2 v0.0.0-20220120090717-25e59572242e h1:EqiJ0Xil8NmcXyupNqXV9oYDBeWntEIegxLahrTr8DY= github.com/apache/pulsar-client-go/oauth2 v0.0.0-20220120090717-25e59572242e/go.mod h1:Xee4tgYLFpYcPMcTfBYWE1uKRzeciodGTSEDMzsR6i8= -github.com/apache/pulsar/pulsar-function-go v0.0.0-20200124033432-ec122ed9562c/go.mod h1:2a3PacwSg4KPcGxO3bjH29xsoKSuSkq2mG0sjKtxsP4= github.com/ardielle/ardielle-go v1.5.2 h1:TilHTpHIQJ27R1Tl/iITBzMwiUGSlVfiVhwDNGM3Zj4= github.com/ardielle/ardielle-go v1.5.2/go.mod h1:I4hy1n795cUhaVt/ojz83SNVCYIGsAFAONtv2Dr7HUI= github.com/ardielle/ardielle-tools v1.5.4/go.mod h1:oZN+JRMnqGiIhrzkRN9l26Cej9dEx4jeNG6A+AdkShk= @@ -81,7 +62,6 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.32.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA= github.com/beefsack/go-rate v0.0.0-20220214233405-116f4ca011a0/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -90,10 +70,10 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= -github.com/boynton/repl v0.0.0-20170116235056-348863958e3e/go.mod h1:Crc/GCZ3NXDVCio7Yr0o+SSrytpcFhLmVCIzi0s49t4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -104,17 +84,16 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/danieljoos/wincred v1.0.2 h1:zf4bhty2iLuwgjgpraD2E9UbvO+fe54XXGJbOwe23fU= github.com/danieljoos/wincred v1.0.2/go.mod h1:SnuYRW9lp1oJrZX/dXJqr0cPK5gYXqx3EJbmjhLdK9U= -github.com/datadog/zstd v1.4.6-0.20200617134701-89f69fb7df32/go.mod h1:inRp+etsHuvVqMPNTXaFlpf/Tj7wqviBtdJoPVrPEFQ= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4wg/adLLz5xh5CmpiCA= github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0= -github.com/dvsekhvalnov/jose2go v0.0.0-20180829124132-7f401d37b68a h1:mq+R6XEM6lJX5VlLyZIrUSP8tSuJp82xTK89hvBwJbU= -github.com/dvsekhvalnov/jose2go v0.0.0-20180829124132-7f401d37b68a/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM= github.com/dvsekhvalnov/jose2go v0.0.0-20200901110807-248326c1351b/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM= +github.com/dvsekhvalnov/jose2go v1.6.0 h1:Y9gnSnP4qEI0+/uQkHvFXeD2PLPJeXEL+ySMEA2EjTY= +github.com/dvsekhvalnov/jose2go v1.6.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -124,6 +103,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -137,14 +117,14 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -160,7 +140,6 @@ github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71 github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= @@ -171,13 +150,13 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -193,6 +172,8 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -210,13 +191,14 @@ github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= @@ -243,12 +225,10 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jawher/mow.cli v1.0.4/go.mod h1:5hQj2V8g+qYmLUVWqu4Wuja1pI57M83EChYLVZ0sMKk= -github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg= github.com/jawher/mow.cli v1.2.0/go.mod h1:y+pcA3jBAdo/GIZx/0rFjw/K2bVEODP9rfZOfaiq8Ko= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -258,28 +238,30 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/keybase/go-keychain v0.0.0-20190712205309-48d3d31d256d h1:Z+RDyXzjKE0i2sTjZ/b1uxiGtPhFy34Ou/Tk0qwN0kM= github.com/keybase/go-keychain v0.0.0-20190712205309-48d3d31d256d/go.mod h1:JJNrCn9otv/2QP4D7SMJBgaleKpOf66PnW6F5WGNRIc= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.9.2 h1:LfVyl+ZlLlLDeQ/d2AqfGIIH4qEDu0Ed2S5GyhCWIWY= -github.com/klauspost/compress v1.9.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.10.8 h1:eLeJ3dr/Y9+XRfJT4l+8ZjmtB5RPJhucH2HeCV5+IZY= github.com/klauspost/compress v1.10.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/linkedin/goavro/v2 v2.9.8 h1:jN50elxBsGBDGVDEKqUlDuU1cFwJ11K/yrJCBMe/7Wg= github.com/linkedin/goavro/v2 v2.9.8/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -299,11 +281,15 @@ github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -311,7 +297,6 @@ github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -321,37 +306,38 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= -github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= +github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= -github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= @@ -359,30 +345,30 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/yahoo/athenz v1.8.55 h1:xGhxN3yLq334APyn0Zvcc+aqu78Q7BBhYJevM3EtTW0= -github.com/yahoo/athenz v1.8.55/go.mod h1:G7LLFUH7Z/r4QAB7FfudfuA7Am/eCzO1GlzBhDL6Kv0= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= @@ -402,12 +388,13 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -443,6 +430,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -451,7 +439,6 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -471,14 +458,11 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -488,11 +472,13 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= @@ -501,6 +487,8 @@ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= +golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -512,6 +500,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -521,7 +510,6 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -529,13 +517,11 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -551,12 +537,10 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -572,23 +556,31 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -601,7 +593,6 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190808195139-e713427fea3f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -640,9 +631,11 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -673,13 +666,14 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= @@ -699,7 +693,6 @@ google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -716,14 +709,14 @@ google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 h1:6GQBEOdGkX6MMTLT9V+TjtIRZCw9VPD5Z+yHY9wMgS0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97/go.mod h1:v7nGkzlmW8P3n/bKmWBn2WpBjpOEx8Q6gMueudAmKfY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= @@ -738,43 +731,49 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.60.0 h1:6FQAR0kM31P6MRdeluor2w2gPaS4SVNrD/DNTxrQ15k= +google.golang.org/grpc v1.60.0/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pulsar-function-go/go.mod b/pulsar-function-go/go.mod index 8483d802a86e8..8dd3f4ef55473 100644 --- a/pulsar-function-go/go.mod +++ b/pulsar-function-go/go.mod @@ -1,19 +1,61 @@ module github.com/apache/pulsar/pulsar-function-go -go 1.13 +go 1.21 require ( - github.com/apache/pulsar-client-go v0.8.1 - github.com/golang/protobuf v1.5.2 - github.com/prometheus/client_golang v1.11.1 - github.com/prometheus/client_model v0.2.0 + github.com/apache/pulsar-client-go v0.8.0 + github.com/golang/protobuf v1.5.3 + github.com/prometheus/client_golang v1.15.1 + github.com/prometheus/client_model v0.4.0 github.com/sirupsen/logrus v1.6.0 - github.com/stretchr/testify v1.7.0 - google.golang.org/grpc v1.38.0 - google.golang.org/protobuf v1.26.0 + github.com/stretchr/testify v1.8.4 + google.golang.org/grpc v1.60.0 + google.golang.org/protobuf v1.34.1 gopkg.in/yaml.v2 v2.4.0 ) +require ( + github.com/99designs/keyring v1.1.6 // indirect + github.com/AthenZ/athenz v1.10.39 // indirect + github.com/DataDog/zstd v1.5.0 // indirect + github.com/apache/pulsar-client-go/oauth2 v0.0.0-20220120090717-25e59572242e // indirect + github.com/ardielle/ardielle-go v1.5.2 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/danieljoos/wincred v1.0.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dvsekhvalnov/jose2go v1.6.0 // indirect + github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/golang/snappy v0.0.1 // indirect + github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect + github.com/keybase/go-keychain v0.0.0-20190712205309-48d3d31d256d // indirect + github.com/klauspost/compress v1.10.8 // indirect + github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect + github.com/linkedin/goavro/v2 v2.9.8 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mtibben/percent v0.2.1 // indirect + github.com/pierrec/lz4 v2.0.5+incompatible // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + go.uber.org/atomic v1.7.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/oauth2 v0.13.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + replace github.com/apache/pulsar/pulsar-function-go/pf => ./pf replace github.com/apache/pulsar/pulsar-function-go/logutil => ./logutil diff --git a/pulsar-function-go/go.sum b/pulsar-function-go/go.sum index 01ae249a5b456..0acd26248a8fd 100644 --- a/pulsar-function-go/go.sum +++ b/pulsar-function-go/go.sum @@ -37,19 +37,12 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/99designs/keyring v1.1.5 h1:wLv7QyzYpFIyMSwOADq1CLTF9KbjbBfcnfmOGJ64aO4= -github.com/99designs/keyring v1.1.5/go.mod h1:7hsVvt2qXgtadGevGJ4ujg+u8m6SpJ5TpHqTozIPqf0= github.com/99designs/keyring v1.1.6 h1:kVDC2uCgVwecxCk+9zoCt2uEL6dt+dfVzMvGgnVcIuM= github.com/99designs/keyring v1.1.6/go.mod h1:16e0ds7LGQQcT59QqkTg72Hh5ShM51Byv5PEmW6uoRU= -github.com/AthenZ/athenz v1.10.15 h1:8Bc2W313k/ev/SGokuthNbzpwfg9W3frg3PKq1r943I= -github.com/AthenZ/athenz v1.10.15/go.mod h1:7KMpEuJ9E4+vMCMI3UQJxwWs0RZtQq7YXZ1IteUjdsc= github.com/AthenZ/athenz v1.10.39 h1:mtwHTF/v62ewY2Z5KWhuZgVXftBej1/Tn80zx4DcawY= github.com/AthenZ/athenz v1.10.39/go.mod h1:3Tg8HLsiQZp81BJY58JBeU2BR6B/H4/0MQGfCwhHNEA= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 h1:++HGU87uq9UsSTlFeiOV9uZR3NpYkndUXeYyLv2DTc8= -github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -58,14 +51,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apache/pulsar-client-go v0.7.0 h1:sZBkjJPHC7akM8n8DuzkLdwioKPSzyub3efCJ1Ltw9Y= -github.com/apache/pulsar-client-go v0.7.0/go.mod h1:EauTUv9sTmP9QRznRgK9hxnzCsIVfS8fyhTfGcuJBrE= github.com/apache/pulsar-client-go v0.8.0 h1:0xDqMF7I8Fvyxvw7dLHyJUZcpTw+0WDyj4U5iOT/jCQ= github.com/apache/pulsar-client-go v0.8.0/go.mod h1:kpFNN3AqZWQEGcRnhu0rNhWLI91+6RDYqPbmNEiIsWs= -github.com/apache/pulsar-client-go v0.8.1 h1:UZINLbH3I5YtNzqkju7g9vrl4CKrEgYSx2rbpvGufrE= -github.com/apache/pulsar-client-go v0.8.1/go.mod h1:yJNcvn/IurarFDxwmoZvb2Ieylg630ifxeO/iXpk27I= -github.com/apache/pulsar-client-go/oauth2 v0.0.0-20201120111947-b8bd55bc02bd h1:P5kM7jcXJ7TaftX0/EMKiSJgvQc/ct+Fw0KMvcH3WuY= -github.com/apache/pulsar-client-go/oauth2 v0.0.0-20201120111947-b8bd55bc02bd/go.mod h1:0UtvvETGDdvXNDCHa8ZQpxl+w3HbdFtfYZvDHLgWGTY= github.com/apache/pulsar-client-go/oauth2 v0.0.0-20220120090717-25e59572242e h1:EqiJ0Xil8NmcXyupNqXV9oYDBeWntEIegxLahrTr8DY= github.com/apache/pulsar-client-go/oauth2 v0.0.0-20220120090717-25e59572242e/go.mod h1:Xee4tgYLFpYcPMcTfBYWE1uKRzeciodGTSEDMzsR6i8= github.com/ardielle/ardielle-go v1.5.2 h1:TilHTpHIQJ27R1Tl/iITBzMwiUGSlVfiVhwDNGM3Zj4= @@ -76,7 +63,6 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.32.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA= -github.com/beefsack/go-rate v0.0.0-20220214233405-116f4ca011a0/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -85,8 +71,9 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -102,14 +89,11 @@ github.com/danieljoos/wincred v1.0.2/go.mod h1:SnuYRW9lp1oJrZX/dXJqr0cPK5gYXqx3E github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4wg/adLLz5xh5CmpiCA= github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0= -github.com/dvsekhvalnov/jose2go v0.0.0-20180829124132-7f401d37b68a h1:mq+R6XEM6lJX5VlLyZIrUSP8tSuJp82xTK89hvBwJbU= -github.com/dvsekhvalnov/jose2go v0.0.0-20180829124132-7f401d37b68a/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM= -github.com/dvsekhvalnov/jose2go v0.0.0-20200901110807-248326c1351b h1:HBah4D48ypg3J7Np4N+HY/ZR76fx3HEUGxDU6Uk39oQ= github.com/dvsekhvalnov/jose2go v0.0.0-20200901110807-248326c1351b/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM= +github.com/dvsekhvalnov/jose2go v1.6.0 h1:Y9gnSnP4qEI0+/uQkHvFXeD2PLPJeXEL+ySMEA2EjTY= +github.com/dvsekhvalnov/jose2go v1.6.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -142,12 +126,10 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -158,7 +140,6 @@ github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71 github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= @@ -169,34 +150,30 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -213,8 +190,9 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -264,14 +242,15 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.8 h1:eLeJ3dr/Y9+XRfJT4l+8ZjmtB5RPJhucH2HeCV5+IZY= github.com/klauspost/compress v1.10.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -280,8 +259,9 @@ github.com/linkedin/goavro/v2 v2.9.8/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8 github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -301,6 +281,7 @@ github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -316,7 +297,6 @@ github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -326,37 +306,36 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= -github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= +github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -366,32 +345,30 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= @@ -408,16 +385,16 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -453,6 +430,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -461,14 +439,12 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -482,14 +458,11 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -498,13 +471,14 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985 h1:4CSI6oo7cOjJKajidEljs9h+uP0rRZBPPPhcCbj5mw8= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= @@ -512,9 +486,9 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 h1:0Ja1LBD+yisY6RWM/BH7TJVXWsSjs2VwBSmvSX4HdBc= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= +golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= +golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -526,18 +500,16 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -565,12 +537,10 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -585,21 +555,28 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -612,7 +589,6 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -655,9 +631,9 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -684,20 +660,20 @@ google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBz google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= @@ -717,7 +693,6 @@ google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -733,15 +708,15 @@ google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 h1:6GQBEOdGkX6MMTLT9V+TjtIRZCw9VPD5Z+yHY9wMgS0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97/go.mod h1:v7nGkzlmW8P3n/bKmWBn2WpBjpOEx8Q6gMueudAmKfY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= @@ -755,29 +730,30 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.60.0 h1:6FQAR0kM31P6MRdeluor2w2gPaS4SVNrD/DNTxrQ15k= +google.golang.org/grpc v1.60.0/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= @@ -786,19 +762,18 @@ gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pulsar-function-go/pf/context.go b/pulsar-function-go/pf/context.go index fd3da9b59b656..0b269a568ffec 100644 --- a/pulsar-function-go/pf/context.go +++ b/pulsar-function-go/pf/context.go @@ -54,14 +54,14 @@ func NewFuncContext() *FunctionContext { return fc } -//GetInstanceID returns the id of the instance that invokes the running pulsar -//function. +// GetInstanceID returns the id of the instance that invokes the running pulsar +// function. func (c *FunctionContext) GetInstanceID() int { return c.instanceConf.instanceID } -//GetInputTopics returns a list of all input topics the pulsar function has been -//invoked on +// GetInputTopics returns a list of all input topics the pulsar function has been +// invoked on func (c *FunctionContext) GetInputTopics() []string { inputMap := c.instanceConf.funcDetails.GetSource().InputSpecs inputTopics := make([]string, len(inputMap)) @@ -73,84 +73,84 @@ func (c *FunctionContext) GetInputTopics() []string { return inputTopics } -//GetOutputTopic returns the output topic the pulsar function was invoked on +// GetOutputTopic returns the output topic the pulsar function was invoked on func (c *FunctionContext) GetOutputTopic() string { return c.instanceConf.funcDetails.GetSink().Topic } -//GetTenantAndNamespace returns the tenant and namespace the pulsar function -//belongs to in the format of `/` +// GetTenantAndNamespace returns the tenant and namespace the pulsar function +// belongs to in the format of `/` func (c *FunctionContext) GetTenantAndNamespace() string { return c.GetFuncTenant() + "/" + c.GetFuncNamespace() } -//GetTenantAndNamespaceAndName returns the full name of the pulsar function in -//the format of `//` +// GetTenantAndNamespaceAndName returns the full name of the pulsar function in +// the format of `//` func (c *FunctionContext) GetTenantAndNamespaceAndName() string { return c.GetFuncTenant() + "/" + c.GetFuncNamespace() + "/" + c.GetFuncName() } -//GetFuncTenant returns the tenant the pulsar function belongs to +// GetFuncTenant returns the tenant the pulsar function belongs to func (c *FunctionContext) GetFuncTenant() string { return c.instanceConf.funcDetails.Tenant } -//GetFuncName returns the name given to the pulsar function +// GetFuncName returns the name given to the pulsar function func (c *FunctionContext) GetFuncName() string { return c.instanceConf.funcDetails.Name } -//GetFuncNamespace returns the namespace the pulsar function belongs to +// GetFuncNamespace returns the namespace the pulsar function belongs to func (c *FunctionContext) GetFuncNamespace() string { return c.instanceConf.funcDetails.Namespace } -//GetFuncID returns the id of the pulsar function +// GetFuncID returns the id of the pulsar function func (c *FunctionContext) GetFuncID() string { return c.instanceConf.funcID } -//GetPort returns the port the pulsar function communicates on +// GetPort returns the port the pulsar function communicates on func (c *FunctionContext) GetPort() int { return c.instanceConf.port } -//GetClusterName returns the name of the cluster the pulsar function is running -//in +// GetClusterName returns the name of the cluster the pulsar function is running +// in func (c *FunctionContext) GetClusterName() string { return c.instanceConf.clusterName } -//GetExpectedHealthCheckInterval returns the expected time between health checks -//in seconds +// GetExpectedHealthCheckInterval returns the expected time between health checks +// in seconds func (c *FunctionContext) GetExpectedHealthCheckInterval() int32 { return c.instanceConf.expectedHealthCheckInterval } -//GetExpectedHealthCheckIntervalAsDuration returns the expected time between -//health checks in seconds as a time.Duration +// GetExpectedHealthCheckIntervalAsDuration returns the expected time between +// health checks in seconds as a time.Duration func (c *FunctionContext) GetExpectedHealthCheckIntervalAsDuration() time.Duration { return time.Duration(c.instanceConf.expectedHealthCheckInterval) } -//GetMaxIdleTime returns the amount of time the pulsar function has to respond -//to the most recent health check before it is considered to be failing. +// GetMaxIdleTime returns the amount of time the pulsar function has to respond +// to the most recent health check before it is considered to be failing. func (c *FunctionContext) GetMaxIdleTime() int64 { return int64(c.GetExpectedHealthCheckIntervalAsDuration() * 3 * time.Second) } -//GetFuncVersion returns the version of the pulsar function +// GetFuncVersion returns the version of the pulsar function func (c *FunctionContext) GetFuncVersion() string { return c.instanceConf.funcVersion } -//GetUserConfValue returns the value of a key from the pulsar function's user -//configuration map +// GetUserConfValue returns the value of a key from the pulsar function's user +// configuration map func (c *FunctionContext) GetUserConfValue(key string) interface{} { return c.userConfigs[key] } -//GetUserConfMap returns the pulsar function's user configuration map +// GetUserConfMap returns the pulsar function's user configuration map func (c *FunctionContext) GetUserConfMap() map[string]interface{} { return c.userConfigs } @@ -172,12 +172,12 @@ func (c *FunctionContext) GetCurrentRecord() pulsar.Message { return c.record } -//GetMetricsPort returns the port the pulsar function metrics listen on +// GetMetricsPort returns the port the pulsar function metrics listen on func (c *FunctionContext) GetMetricsPort() int { return c.instanceConf.metricsPort } -//RecordMetric records an observation to the user_metric summary with the provided value +// RecordMetric records an observation to the user_metric summary with the provided value func (c *FunctionContext) RecordMetric(metricName string, metricValue float64) { v, ok := c.userMetrics.Load(metricName) if !ok { diff --git a/pulsar-function-go/pf/function.go b/pulsar-function-go/pf/function.go index 4efbb2b4cec1c..f6e46ff7ac83b 100644 --- a/pulsar-function-go/pf/function.go +++ b/pulsar-function-go/pf/function.go @@ -143,25 +143,25 @@ func newFunction(inputFunc interface{}) function { } // Rules: -// -// * handler must be a function -// * handler may take between 0 and two arguments. -// * if there are two arguments, the first argument must satisfy the "context.Context" interface. -// * handler may return between 0 and two arguments. -// * if there are two return values, the second argument must be an error. -// * if there is one return value it must be an error. + +// - handler must be a function +// - handler may take between 0 and two arguments. +// - if there are two arguments, the first argument must satisfy the "context.Context" interface. +// - handler may return between 0 and two arguments. +// - if there are two return values, the second argument must be an error. +// - if there is one return value it must be an error. // // Valid function signatures: // -// func () -// func () error -// func (input) error -// func () (output, error) -// func (input) (output, error) -// func (context.Context) error -// func (context.Context, input) error -// func (context.Context) (output, error) -// func (context.Context, input) (output, error) +// func () +// func () error +// func (input) error +// func () (output, error) +// func (input) (output, error) +// func (context.Context) error +// func (context.Context, input) error +// func (context.Context) (output, error) +// func (context.Context, input) (output, error) // // Where "input" and "output" are types compatible with the "encoding/json" standard library. // See https://golang.org/pkg/encoding/json/#Unmarshal for how deserialization behaves diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go index 138489444d160..1064aece46fe8 100644 --- a/pulsar-function-go/pf/instance.go +++ b/pulsar-function-go/pf/instance.go @@ -21,8 +21,10 @@ package pf import ( "context" + "fmt" "math" "strconv" + "strings" "time" "github.com/golang/protobuf/ptypes/empty" @@ -149,7 +151,6 @@ func (gi *goInstance) startFunction(function function) error { defer metricsServicer.close() CLOSE: for { - idleTimer.Reset(idleDuration) select { case cm := <-channel: msgInput := cm.Message @@ -181,6 +182,11 @@ CLOSE: close(channel) break CLOSE } + // reset the idle timer and drain if appropriate before the next loop + if !idleTimer.Stop() { + <-idleTimer.C + } + idleTimer.Reset(idleDuration) } gi.closeLogTopic() @@ -188,11 +194,40 @@ CLOSE: return nil } +const ( + authPluginToken = "org.apache.pulsar.client.impl.auth.AuthenticationToken" + authPluginNone = "" +) + func (gi *goInstance) setupClient() error { - client, err := pulsar.NewClient(pulsar.ClientOptions{ + ic := gi.context.instanceConf + + clientOpts := pulsar.ClientOptions{ + URL: ic.pulsarServiceURL, + TLSTrustCertsFilePath: ic.tlsTrustCertsPath, + TLSAllowInsecureConnection: ic.tlsAllowInsecure, + TLSValidateHostname: ic.tlsHostnameVerification, + } + + switch ic.authPlugin { + case authPluginToken: + switch { + case strings.HasPrefix(ic.authParams, "file://"): + clientOpts.Authentication = pulsar.NewAuthenticationTokenFromFile(ic.authParams[7:]) + case strings.HasPrefix(ic.authParams, "token:"): + clientOpts.Authentication = pulsar.NewAuthenticationToken(ic.authParams[6:]) + case ic.authParams == "": + return fmt.Errorf("auth plugin %s given, but authParams is empty", authPluginToken) + default: + return fmt.Errorf(`unknown token format - expecting "file://" or "token:" prefix`) + } + case authPluginNone: + clientOpts.Authentication, _ = pulsar.NewAuthentication("", "") // ret: auth.NewAuthDisabled() + default: + return fmt.Errorf("unknown auth provider: %s", ic.authPlugin) + } - URL: gi.context.instanceConf.pulsarServiceURL, - }) + client, err := pulsar.NewClient(clientOpts) if err != nil { log.Errorf("create client error:%v", err) gi.stats.incrTotalSysExceptions(err) @@ -404,11 +439,25 @@ func (gi *goInstance) processResult(msgInput pulsar.Message, output []byte) { // ackInputMessage doesn't produce any result, or the user doesn't want the result. func (gi *goInstance) ackInputMessage(inputMessage pulsar.Message) { log.Debugf("ack input message topic name is: %s", inputMessage.Topic()) - gi.consumers[inputMessage.Topic()].Ack(inputMessage) + gi.respondMessage(inputMessage, true) } func (gi *goInstance) nackInputMessage(inputMessage pulsar.Message) { - gi.consumers[inputMessage.Topic()].Nack(inputMessage) + gi.respondMessage(inputMessage, false) +} + +func (gi *goInstance) respondMessage(inputMessage pulsar.Message, ack bool) { + topicName, err := ParseTopicName(inputMessage.Topic()) + if err != nil { + log.Errorf("unable respond to message ID %s - invalid topic: %v", messageIDStr(inputMessage), err) + return + } + // consumers are indexed by topic name only (no partition) + if ack { + gi.consumers[topicName.NameWithoutPartition()].Ack(inputMessage) + return + } + gi.consumers[topicName.NameWithoutPartition()].Nack(inputMessage) } func getIdleTimeout(timeoutMilliSecond time.Duration) time.Duration { diff --git a/pulsar-function-go/pf/instanceConf.go b/pulsar-function-go/pf/instanceConf.go index 9d4cabfae5a9b..844a2bc9b89a3 100644 --- a/pulsar-function-go/pf/instanceConf.go +++ b/pulsar-function-go/pf/instanceConf.go @@ -25,7 +25,9 @@ import ( "time" "github.com/apache/pulsar/pulsar-function-go/conf" + log "github.com/apache/pulsar/pulsar-function-go/logutil" pb "github.com/apache/pulsar/pulsar-function-go/pb" + "google.golang.org/protobuf/encoding/protojson" ) // This is the config passed to the Golang Instance. Contains all the information @@ -39,9 +41,16 @@ type instanceConf struct { port int clusterName string pulsarServiceURL string + stateServiceURL string + pulsarWebServiceURL string killAfterIdle time.Duration expectedHealthCheckInterval int32 metricsPort int + authPlugin string + authParams string + tlsTrustCertsPath string + tlsAllowInsecure bool + tlsHostnameVerification bool } func newInstanceConfWithConf(cfg *conf.Conf) *instanceConf { @@ -71,6 +80,8 @@ func newInstanceConfWithConf(cfg *conf.Conf) *instanceConf { port: cfg.Port, clusterName: cfg.ClusterName, pulsarServiceURL: cfg.PulsarServiceURL, + stateServiceURL: cfg.StateStorageServiceURL, + pulsarWebServiceURL: cfg.PulsarWebServiceURL, killAfterIdle: cfg.KillAfterIdleMs, expectedHealthCheckInterval: cfg.ExpectedHealthCheckInterval, metricsPort: cfg.MetricsPort, @@ -107,6 +118,20 @@ func newInstanceConfWithConf(cfg *conf.Conf) *instanceConf { }, UserConfig: cfg.UserConfig, }, + authPlugin: cfg.ClientAuthenticationPlugin, + authParams: cfg.ClientAuthenticationParameters, + tlsTrustCertsPath: cfg.TLSTrustCertsFilePath, + tlsAllowInsecure: cfg.TLSAllowInsecureConnection, + tlsHostnameVerification: cfg.TLSHostnameVerificationEnable, + } + // parse the raw function details and ignore the unmarshal error(fallback to original way) + if cfg.FunctionDetails != "" { + functionDetails := pb.FunctionDetails{} + if err := protojson.Unmarshal([]byte(cfg.FunctionDetails), &functionDetails); err != nil { + log.Errorf("Failed to unmarshal function details: %v", err) + } else { + instanceConf.funcDetails = functionDetails + } } if instanceConf.funcDetails.ProcessingGuarantees == pb.ProcessingGuarantees_EFFECTIVELY_ONCE { diff --git a/pulsar-function-go/pf/instanceConf_test.go b/pulsar-function-go/pf/instanceConf_test.go index 02aef913ebc97..cc5f46e2fe12b 100644 --- a/pulsar-function-go/pf/instanceConf_test.go +++ b/pulsar-function-go/pf/instanceConf_test.go @@ -20,6 +20,7 @@ package pf import ( + "fmt" "testing" cfg "github.com/apache/pulsar/pulsar-function-go/conf" @@ -113,3 +114,209 @@ func TestInstanceConf_Fail(t *testing.T) { newInstanceConfWithConf(&cfg.Conf{ProcessingGuarantees: 3}) }, "Should have a panic") } + +func TestInstanceConf_WithDetails(t *testing.T) { + cfg := &cfg.Conf{ + FunctionDetails: `{"tenant":"public","namespace":"default","name":"test-function","className":"process", +"logTopic":"test-logs","userConfig":"{\"key1\":\"value1\"}","runtime":"GO","autoAck":true,"parallelism":1, +"source":{"configs":"{\"username\":\"admin\"}","typeClassName":"string","timeoutMs":"15000", +"subscriptionName":"test-subscription","inputSpecs":{"input":{"schemaType":"avro","receiverQueueSize":{"value":1000}, +"schemaProperties":{"schema_prop1":"schema1"},"consumerProperties":{"consumer_prop1":"consumer1"},"cryptoSpec": +{"cryptoKeyReaderClassName":"key-reader","producerCryptoFailureAction":"SEND","consumerCryptoFailureAction":"CONSUME"}}} +,"negativeAckRedeliveryDelayMs":"15000"},"sink":{"configs":"{\"password\":\"admin\"}","topic":"test-output", +"typeClassName":"string","schemaType":"avro","producerSpec":{"maxPendingMessages":2000,"useThreadLocalProducers":true, +"cryptoSpec":{"cryptoKeyReaderClassName":"key-reader","producerCryptoFailureAction":"DISCARD"}, +"batchBuilder":"DEFAULT"}},"resources":{"cpu":2.0,"ram":"1024","disk":"1024"},"packageUrl":"/path/to/package", +"retryDetails":{"maxMessageRetries":3,"deadLetterTopic":"test-dead-letter-topic"},"secretsMap": +"{\"secret1\":\"secret-value1\"}","runtimeFlags":"flags","componentType":"FUNCTION","customRuntimeOptions":"options", +"retainOrdering":true,"retainKeyOrdering":true,"subscriptionPosition":"EARLIEST"}`, + } + instanceConf := newInstanceConfWithConf(cfg) + assert.Equal(t, "public", instanceConf.funcDetails.Tenant) + assert.Equal(t, "default", instanceConf.funcDetails.Namespace) + assert.Equal(t, "test-function", instanceConf.funcDetails.Name) + assert.Equal(t, "process", instanceConf.funcDetails.ClassName) + assert.Equal(t, "test-logs", instanceConf.funcDetails.LogTopic) + assert.Equal(t, pb.ProcessingGuarantees_ATLEAST_ONCE, instanceConf.funcDetails.ProcessingGuarantees) + assert.Equal(t, `{"key1":"value1"}`, instanceConf.funcDetails.UserConfig) + assert.Equal(t, `{"secret1":"secret-value1"}`, instanceConf.funcDetails.SecretsMap) + assert.Equal(t, pb.FunctionDetails_GO, instanceConf.funcDetails.Runtime) + + assert.Equal(t, true, instanceConf.funcDetails.AutoAck) + assert.Equal(t, int32(1), instanceConf.funcDetails.Parallelism) + + sourceSpec := pb.SourceSpec{ + TypeClassName: "string", + TimeoutMs: 15000, + Configs: `{"username":"admin"}`, + SubscriptionName: "test-subscription", + SubscriptionType: pb.SubscriptionType_SHARED, + NegativeAckRedeliveryDelayMs: 15000, + InputSpecs: map[string]*pb.ConsumerSpec{ + "input": { + SchemaType: "avro", + SchemaProperties: map[string]string{ + "schema_prop1": "schema1", + }, + ConsumerProperties: map[string]string{ + "consumer_prop1": "consumer1", + }, + ReceiverQueueSize: &pb.ConsumerSpec_ReceiverQueueSize{ + Value: 1000, + }, + CryptoSpec: &pb.CryptoSpec{ + CryptoKeyReaderClassName: "key-reader", + ProducerCryptoFailureAction: pb.CryptoSpec_SEND, + ConsumerCryptoFailureAction: pb.CryptoSpec_CONSUME, + }, + }, + }, + } + assert.Equal(t, sourceSpec.String(), instanceConf.funcDetails.Source.String()) + + sinkSpec := pb.SinkSpec{ + TypeClassName: "string", + Topic: "test-output", + Configs: `{"password":"admin"}`, + SchemaType: "avro", + ProducerSpec: &pb.ProducerSpec{ + MaxPendingMessages: 2000, + UseThreadLocalProducers: true, + CryptoSpec: &pb.CryptoSpec{ + CryptoKeyReaderClassName: "key-reader", + ProducerCryptoFailureAction: pb.CryptoSpec_DISCARD, + ConsumerCryptoFailureAction: pb.CryptoSpec_FAIL, + }, + BatchBuilder: "DEFAULT", + }, + } + assert.Equal(t, sinkSpec.String(), instanceConf.funcDetails.Sink.String()) + + resource := pb.Resources{ + Cpu: 2.0, + Ram: 1024, + Disk: 1024, + } + assert.Equal(t, resource.String(), instanceConf.funcDetails.Resources.String()) + assert.Equal(t, "/path/to/package", instanceConf.funcDetails.PackageUrl) + + retryDetails := pb.RetryDetails{ + MaxMessageRetries: 3, + DeadLetterTopic: "test-dead-letter-topic", + } + assert.Equal(t, retryDetails.String(), instanceConf.funcDetails.RetryDetails.String()) + + assert.Equal(t, "flags", instanceConf.funcDetails.RuntimeFlags) + assert.Equal(t, pb.FunctionDetails_FUNCTION, instanceConf.funcDetails.ComponentType) + assert.Equal(t, "options", instanceConf.funcDetails.CustomRuntimeOptions) + assert.Equal(t, "", instanceConf.funcDetails.Builtin) + assert.Equal(t, true, instanceConf.funcDetails.RetainOrdering) + assert.Equal(t, true, instanceConf.funcDetails.RetainKeyOrdering) + assert.Equal(t, pb.SubscriptionPosition_EARLIEST, instanceConf.funcDetails.SubscriptionPosition) +} + +func TestInstanceConf_WithEmptyOrInvalidDetails(t *testing.T) { + testCases := []struct { + name string + details string + }{ + { + name: "empty details", + details: "", + }, + { + name: "invalid details", + details: "error", + }, + } + + for i, testCase := range testCases { + + t.Run(fmt.Sprintf("testCase[%d] %s", i, testCase.name), func(t *testing.T) { + cfg := &cfg.Conf{ + FunctionDetails: testCase.details, + Tenant: "public", + NameSpace: "default", + Name: "test-function", + LogTopic: "test-logs", + ProcessingGuarantees: 0, + UserConfig: `{"key1":"value1"}`, + SecretsMap: `{"secret1":"secret-value1"}`, + Runtime: 3, + AutoACK: true, + Parallelism: 1, + SubscriptionType: 1, + TimeoutMs: 15000, + SubscriptionName: "test-subscription", + CleanupSubscription: false, + SubscriptionPosition: 0, + SinkSpecTopic: "test-output", + SinkSchemaType: "avro", + Cpu: 2.0, + Ram: 1024, + Disk: 1024, + MaxMessageRetries: 3, + DeadLetterTopic: "test-dead-letter-topic", + SourceInputSpecs: map[string]string{ + "input": `{"schemaType":"avro","receiverQueueSize":{"value":1000},"schemaProperties": +{"schema_prop1":"schema1"},"consumerProperties":{"consumer_prop1":"consumer1"}}`, + }, + } + instanceConf := newInstanceConfWithConf(cfg) + + assert.Equal(t, "public", instanceConf.funcDetails.Tenant) + assert.Equal(t, "default", instanceConf.funcDetails.Namespace) + assert.Equal(t, "test-function", instanceConf.funcDetails.Name) + assert.Equal(t, "test-logs", instanceConf.funcDetails.LogTopic) + assert.Equal(t, pb.ProcessingGuarantees_ATLEAST_ONCE, instanceConf.funcDetails.ProcessingGuarantees) + assert.Equal(t, `{"key1":"value1"}`, instanceConf.funcDetails.UserConfig) + assert.Equal(t, `{"secret1":"secret-value1"}`, instanceConf.funcDetails.SecretsMap) + assert.Equal(t, pb.FunctionDetails_GO, instanceConf.funcDetails.Runtime) + + assert.Equal(t, true, instanceConf.funcDetails.AutoAck) + assert.Equal(t, int32(1), instanceConf.funcDetails.Parallelism) + + sourceSpec := pb.SourceSpec{ + SubscriptionType: pb.SubscriptionType_FAILOVER, + TimeoutMs: 15000, + SubscriptionName: "test-subscription", + CleanupSubscription: false, + SubscriptionPosition: pb.SubscriptionPosition_LATEST, + InputSpecs: map[string]*pb.ConsumerSpec{ + "input": { + SchemaType: "avro", + SchemaProperties: map[string]string{ + "schema_prop1": "schema1", + }, + ConsumerProperties: map[string]string{ + "consumer_prop1": "consumer1", + }, + ReceiverQueueSize: &pb.ConsumerSpec_ReceiverQueueSize{ + Value: 1000, + }, + }, + }, + } + assert.Equal(t, sourceSpec.String(), instanceConf.funcDetails.Source.String()) + + sinkSpec := pb.SinkSpec{ + Topic: "test-output", + SchemaType: "avro", + } + assert.Equal(t, sinkSpec.String(), instanceConf.funcDetails.Sink.String()) + + resource := pb.Resources{ + Cpu: 2.0, + Ram: 1024, + Disk: 1024, + } + assert.Equal(t, resource.String(), instanceConf.funcDetails.Resources.String()) + + retryDetails := pb.RetryDetails{ + MaxMessageRetries: 3, + DeadLetterTopic: "test-dead-letter-topic", + } + assert.Equal(t, retryDetails.String(), instanceConf.funcDetails.RetryDetails.String()) + }) + } +} diff --git a/pulsar-function-go/pf/instanceControlServicer_test.go b/pulsar-function-go/pf/instanceControlServicer_test.go index 9344d0a591547..30021adfb926c 100644 --- a/pulsar-function-go/pf/instanceControlServicer_test.go +++ b/pulsar-function-go/pf/instanceControlServicer_test.go @@ -30,6 +30,7 @@ import ( "github.com/golang/protobuf/ptypes/empty" "github.com/stretchr/testify/assert" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/test/bufconn" ) @@ -62,7 +63,8 @@ func TestInstanceControlServicer_serve_creates_valid_instance(t *testing.T) { // Now we can setup the client: ctx := context.Background() - conn, err := grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(getBufDialer(lis)), grpc.WithInsecure()) + conn, err := grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(getBufDialer(lis)), + grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { t.Fatalf("Failed to dial bufnet: %v", err) } @@ -86,7 +88,7 @@ func instanceCommunicationClient(t *testing.T, instance *goInstance) pb.Instance } var ( - ctx context.Context = context.Background() + ctx = context.Background() cf context.CancelFunc ) @@ -119,7 +121,8 @@ func instanceCommunicationClient(t *testing.T, instance *goInstance) pb.Instance }() // Now we can setup the client: - conn, err := grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(getBufDialer(lis)), grpc.WithInsecure()) + conn, err := grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(getBufDialer(lis)), + grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { t.Fatalf("Failed to dial bufnet: %v", err) } diff --git a/pulsar-function-go/pf/stats.go b/pulsar-function-go/pf/stats.go index 85a7fff8a38bb..d25424b37047a 100644 --- a/pulsar-function-go/pf/stats.go +++ b/pulsar-function-go/pf/stats.go @@ -282,7 +282,7 @@ func (stat *StatWithLabelValues) addUserException(err error) { stat.reportUserExceptionPrometheus(err) } -//@limits(calls=5, period=60) +// @limits(calls=5, period=60) func (stat *StatWithLabelValues) reportUserExceptionPrometheus(exception error) { errorTS := []string{exception.Error()} exceptionMetricLabels := append(stat.metricsLabels, errorTS...) @@ -312,7 +312,7 @@ func (stat *StatWithLabelValues) addSysException(exception error) { stat.reportSystemExceptionPrometheus(exception) } -//@limits(calls=5, period=60) +// @limits(calls=5, period=60) func (stat *StatWithLabelValues) reportSystemExceptionPrometheus(exception error) { errorTS := []string{exception.Error()} exceptionMetricLabels := append(stat.metricsLabels, errorTS...) diff --git a/pulsar-function-go/pf/stats_test.go b/pulsar-function-go/pf/stats_test.go index 7b415ef5eff0b..138dc91cd9cd3 100644 --- a/pulsar-function-go/pf/stats_test.go +++ b/pulsar-function-go/pf/stats_test.go @@ -22,16 +22,16 @@ package pf import ( "context" "fmt" - "io/ioutil" + "io" "math" "net/http" "testing" "time" - "github.com/golang/protobuf/proto" "github.com/golang/protobuf/ptypes/empty" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/encoding/prototext" prometheus_client "github.com/prometheus/client_model/go" ) @@ -73,77 +73,9 @@ func TestExampleSummaryVec(t *testing.T) { if len(filteredMetricFamilies) > 1 { t.Fatal("Too many metric families") } - // Then, we need to filter the metrics in the family to one that matches our label. - expectedValue := "name: \"pond_temperature_celsius\"\n" + - "help: \"The temperature of the frog pond.\"\n" + - "type: SUMMARY\n" + - "metric: <\n" + - " label: <\n" + - " name: \"species\"\n" + - " value: \"leiopelma-hochstetteri\"\n" + - " >\n" + - " summary: <\n" + - " sample_count: 0\n" + - " sample_sum: 0\n" + - " quantile: <\n" + - " quantile: 0.5\n" + - " value: nan\n" + - " >\n" + - " quantile: <\n" + - " quantile: 0.9\n" + - " value: nan\n" + - " >\n" + - " quantile: <\n" + - " quantile: 0.99\n" + - " value: nan\n" + - " >\n" + - " >\n" + - ">\n" + - "metric: <\n" + - " label: <\n" + - " name: \"species\"\n" + - " value: \"lithobates-catesbeianus\"\n" + - " >\n" + - " summary: <\n" + - " sample_count: 1000\n" + - " sample_sum: 31956.100000000017\n" + - " quantile: <\n" + - " quantile: 0.5\n" + - " value: 32.4\n" + - " >\n" + - " quantile: <\n" + - " quantile: 0.9\n" + - " value: 41.4\n" + - " >\n" + - " quantile: <\n" + - " quantile: 0.99\n" + - " value: 41.9\n" + - " >\n" + - " >\n" + - ">\n" + - "metric: <\n" + - " label: <\n" + - " name: \"species\"\n" + - " value: \"litoria-caerulea\"\n" + - " >\n" + - " summary: <\n" + - " sample_count: 1000\n" + - " sample_sum: 29969.50000000001\n" + - " quantile: <\n" + - " quantile: 0.5\n" + - " value: 31.1\n" + - " >\n" + - " quantile: <\n" + - " quantile: 0.9\n" + - " value: 41.3\n" + - " >\n" + - " quantile: <\n" + - " quantile: 0.99\n" + - " value: 41.9\n" + - " >\n" + - " >\n" + - ">\n" - assert.Equal(t, expectedValue, proto.MarshalTextString(metricFamilies[0])) + + _, err = prototext.MarshalOptions{Indent: " "}.Marshal(metricFamilies[0]) + assert.NoError(t, err) } func TestExampleSummaryVec_Pulsar(t *testing.T) { _statProcessLatencyMs1 := prometheus.NewSummaryVec( @@ -202,7 +134,7 @@ func TestMetricsServer(t *testing.T) { assert.Equal(t, nil, err) assert.NotEqual(t, nil, resp) assert.Equal(t, 200, resp.StatusCode) - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) assert.Equal(t, nil, err) assert.NotEmpty(t, body) resp.Body.Close() @@ -211,7 +143,7 @@ func TestMetricsServer(t *testing.T) { assert.Equal(t, nil, err) assert.NotEqual(t, nil, resp) assert.Equal(t, 200, resp.StatusCode) - body, err = ioutil.ReadAll(resp.Body) + body, err = io.ReadAll(resp.Body) assert.Equal(t, nil, err) assert.NotEmpty(t, body) resp.Body.Close() @@ -229,7 +161,7 @@ func TestUserMetrics(t *testing.T) { assert.Equal(t, nil, err) assert.NotEqual(t, nil, resp) assert.Equal(t, 200, resp.StatusCode) - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) assert.Equal(t, nil, err) assert.NotEmpty(t, body) assert.NotContainsf(t, string(body), "pulsar_function_user_metric", "user metric should not appear yet") @@ -245,7 +177,7 @@ func TestUserMetrics(t *testing.T) { assert.Equal(t, nil, err) assert.NotEqual(t, nil, resp) assert.Equal(t, 200, resp.StatusCode) - body, err = ioutil.ReadAll(resp.Body) + body, err = io.ReadAll(resp.Body) assert.Equal(t, nil, err) assert.NotEmpty(t, body) diff --git a/pulsar-function-go/pf/util.go b/pulsar-function-go/pf/util.go index d5b32da841121..1d1aa1cab939f 100644 --- a/pulsar-function-go/pf/util.go +++ b/pulsar-function-go/pf/util.go @@ -21,6 +21,8 @@ package pf import ( "fmt" + + "github.com/apache/pulsar-client-go/pulsar" ) func getProperties(fullyQualifiedName string, instanceID int) map[string]string { @@ -39,3 +41,12 @@ func getDefaultSubscriptionName(tenant, namespace, name string) string { func getFullyQualifiedInstanceID(tenant, namespace, name string, instanceID int) string { return fmt.Sprintf("%s/%s/%s:%d", tenant, namespace, name, instanceID) } + +func messageIDStr(msg pulsar.Message) string { + // ::: + return fmt.Sprintf("%d:%d:%d:%d", + msg.ID().LedgerID(), + msg.ID().EntryID(), + msg.ID().PartitionIdx(), + msg.ID().BatchIdx()) +} diff --git a/pulsar-functions/api-java/pom.xml b/pulsar-functions/api-java/pom.xml index 7735a0b142745..279d8eed8473e 100644 --- a/pulsar-functions/api-java/pom.xml +++ b/pulsar-functions/api-java/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-functions-api diff --git a/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/BaseContext.java b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/BaseContext.java index 25874c595d9d4..185031fa29d88 100644 --- a/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/BaseContext.java +++ b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/BaseContext.java @@ -217,4 +217,10 @@ default ClientBuilder getPulsarClientBuilder() { throw new UnsupportedOperationException("not implemented"); } + /** + * Terminate the function instance with a fatal exception. + * + * @param t the fatal exception to be raised + */ + void fatal(Throwable t); } diff --git a/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/Record.java b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/Record.java index ea6987e5f0391..0487b3d02b3a1 100644 --- a/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/Record.java +++ b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/Record.java @@ -27,7 +27,7 @@ import org.apache.pulsar.common.classification.InterfaceStability; /** - * Pulsar Connect's Record interface. Record encapsulates the information about a record being read from a Source. + * Pulsar IO's Record interface. Record encapsulates the information about a record being read from a Source. */ @InterfaceAudience.Public @InterfaceStability.Stable diff --git a/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/state/ByteBufferStateStore.java b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/state/ByteBufferStateStore.java index 8dbd7b322a5ee..d938fe0c82b7d 100644 --- a/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/state/ByteBufferStateStore.java +++ b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/state/ByteBufferStateStore.java @@ -73,4 +73,31 @@ public interface ByteBufferStateStore extends StateStore { */ CompletableFuture getAsync(String key); + /** + * Retrieve the StateValue for the key. + * + * @param key name of the key + * @return the StateValue. + */ + default StateValue getStateValue(String key) { + return getStateValueAsync(key).join(); + } + + /** + * Retrieve the StateValue for the key, but don't wait for the operation to be completed. + * + * @param key name of the key + * @return the StateValue. + */ + default CompletableFuture getStateValueAsync(String key) { + return getAsync(key).thenApply(val -> { + if (val != null && val.remaining() >= 0) { + byte[] data = new byte[val.remaining()]; + val.get(data); + return new StateValue(data, null, null); + } else { + return null; + } + }); + } } diff --git a/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/state/StateValue.java b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/state/StateValue.java new file mode 100644 index 0000000000000..ce06b54a6e490 --- /dev/null +++ b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/state/StateValue.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.api.state; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public class StateValue { + private final byte[] value; + private final Long version; + private final Boolean isNumber; +} \ No newline at end of file diff --git a/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/utils/JavaSerDe.java b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/utils/JavaSerDe.java deleted file mode 100644 index c145179abb42b..0000000000000 --- a/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/utils/JavaSerDe.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.functions.api.utils; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.ObjectInputStream; -import java.io.ObjectOutput; -import java.io.ObjectOutputStream; -import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.common.classification.InterfaceAudience; -import org.apache.pulsar.common.classification.InterfaceStability; -import org.apache.pulsar.functions.api.SerDe; - -/** - * Java Serialization based SerDe. - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -@Slf4j -public class JavaSerDe implements SerDe { - - private static final JavaSerDe INSTANCE = new JavaSerDe(); - - public static JavaSerDe of() { - return INSTANCE; - } - - @Override - public byte[] serialize(Object resultValue) { - try (ByteArrayOutputStream bos = new ByteArrayOutputStream(); - ObjectOutput out = new ObjectOutputStream(bos)) { - out.writeObject(resultValue); - out.flush(); - return bos.toByteArray(); - } catch (Exception ex) { - log.info("Exception during serialization", ex); - } - return null; - } - - @Override - public Object deserialize(byte[] data) { - Object obj = null; - try (ByteArrayInputStream bis = new ByteArrayInputStream(data); - ObjectInputStream ois = new ObjectInputStream(bis)) { - obj = ois.readObject(); - } catch (Exception ex) { - log.info("Exception during deserialization", ex); - } - return obj; - } -} diff --git a/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml b/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml index 9638cfcca8da9..d593536d4679b 100644 --- a/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml @@ -29,6 +29,11 @@ + + + + + @@ -39,4 +44,8 @@ + + + + \ No newline at end of file diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index c0d68cad72e27..89b52aa8120cb 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-functions-instance @@ -36,7 +36,7 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl org.apache.logging.log4j @@ -101,7 +101,7 @@ io.grpc - grpc-all + * com.google.protobuf @@ -110,6 +110,11 @@ + + io.grpc + grpc-netty-shaded + + io.grpc grpc-stub @@ -153,8 +158,13 @@ - com.beust - jcommander + com.github.ben-manes.caffeine + caffeine + + + + info.picocli + picocli @@ -215,7 +225,7 @@ - + @@ -269,6 +279,15 @@ + + org.apache.maven.plugins + maven-surefire-plugin + + + some-configuration + + + diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java index 5cbbcad24c7a2..f613f749bd0fe 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java @@ -27,27 +27,24 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import lombok.ToString; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.api.BatcherBuilder; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.ConsumerBuilder; -import org.apache.pulsar.client.api.HashingScheme; import org.apache.pulsar.client.api.MessageId; -import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; @@ -55,7 +52,7 @@ import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.TypedMessageBuilder; import org.apache.pulsar.client.impl.MultiTopicsConsumerImpl; -import org.apache.pulsar.client.impl.ProducerBuilderImpl; +import org.apache.pulsar.common.functions.ProducerConfig; import org.apache.pulsar.common.io.SinkConfig; import org.apache.pulsar.common.io.SourceConfig; import org.apache.pulsar.common.naming.TopicName; @@ -77,6 +74,7 @@ import org.apache.pulsar.functions.source.PulsarFunctionRecord; import org.apache.pulsar.functions.source.TopicSchema; import org.apache.pulsar.functions.utils.FunctionCommon; +import org.apache.pulsar.functions.utils.FunctionConfigUtils; import org.apache.pulsar.functions.utils.SinkConfigUtils; import org.apache.pulsar.functions.utils.SourceConfigUtils; import org.apache.pulsar.io.core.SinkContext; @@ -86,8 +84,11 @@ /** * This class implements the Context interface exposed to the user. */ +@Slf4j @ToString(exclude = {"pulsarAdmin"}) class ContextImpl implements Context, SinkContext, SourceContext, AutoCloseable { + private final ProducerBuilderFactory producerBuilderFactory; + private final Map producerProperties; private InstanceConfig config; private Logger logger; @@ -97,9 +98,6 @@ class ContextImpl implements Context, SinkContext, SourceContext, AutoCloseable private final ClientBuilder clientBuilder; private final PulsarClient client; private final PulsarAdmin pulsarAdmin; - private Map> publishProducers; - private ThreadLocal>> tlPublishProducers; - private ProducerBuilderImpl producerBuilder; private final TopicSchema topicSchema; @@ -137,12 +135,17 @@ class ContextImpl implements Context, SinkContext, SourceContext, AutoCloseable private final Function.FunctionDetails.ComponentType componentType; + private final java.util.function.Consumer fatalHandler; + + private final ProducerCache producerCache; + private final boolean useThreadLocalProducers; + public ContextImpl(InstanceConfig config, Logger logger, PulsarClient client, SecretsProvider secretsProvider, FunctionCollectorRegistry collectorRegistry, String[] metricsLabels, Function.FunctionDetails.ComponentType componentType, ComponentStatsManager statsManager, - StateManager stateManager, PulsarAdmin pulsarAdmin, ClientBuilder clientBuilder) - throws PulsarClientException { + StateManager stateManager, PulsarAdmin pulsarAdmin, ClientBuilder clientBuilder, + java.util.function.Consumer fatalHandler, ProducerCache producerCache) { this.config = config; this.logger = logger; this.clientBuilder = clientBuilder; @@ -150,33 +153,31 @@ public ContextImpl(InstanceConfig config, Logger logger, PulsarClient client, this.pulsarAdmin = pulsarAdmin; this.topicSchema = new TopicSchema(client, Thread.currentThread().getContextClassLoader()); this.statsManager = statsManager; + this.fatalHandler = fatalHandler; + + this.producerCache = producerCache; - this.producerBuilder = (ProducerBuilderImpl) client.newProducer().blockIfQueueFull(true).enableBatching(true) - .batchingMaxPublishDelay(1, TimeUnit.MILLISECONDS); - boolean useThreadLocalProducers = false; Function.ProducerSpec producerSpec = config.getFunctionDetails().getSink().getProducerSpec(); + ProducerConfig producerConfig = null; if (producerSpec != null) { - if (producerSpec.getMaxPendingMessages() != 0) { - this.producerBuilder.maxPendingMessages(producerSpec.getMaxPendingMessages()); - } - if (producerSpec.getMaxPendingMessagesAcrossPartitions() != 0) { - this.producerBuilder - .maxPendingMessagesAcrossPartitions(producerSpec.getMaxPendingMessagesAcrossPartitions()); - } - if (producerSpec.getBatchBuilder() != null) { - if (producerSpec.getBatchBuilder().equals("KEY_BASED")) { - this.producerBuilder.batcherBuilder(BatcherBuilder.KEY_BASED); - } else { - this.producerBuilder.batcherBuilder(BatcherBuilder.DEFAULT); - } - } + producerConfig = FunctionConfigUtils.convertProducerSpecToProducerConfig(producerSpec); useThreadLocalProducers = producerSpec.getUseThreadLocalProducers(); - } - if (useThreadLocalProducers) { - tlPublishProducers = new ThreadLocal<>(); } else { - publishProducers = new ConcurrentHashMap<>(); - } + useThreadLocalProducers = false; + } + + producerBuilderFactory = new ProducerBuilderFactory(client, producerConfig, + Thread.currentThread().getContextClassLoader(), + // This is for backwards compatibility. The PR https://github.com/apache/pulsar/pull/19470 removed + // the default and made it configurable for the producers created in PulsarSink, but not in ContextImpl. + // This is to keep the default unchanged for the producers created in ContextImpl. + producerBuilder -> producerBuilder.compressionType(CompressionType.LZ4)); + producerProperties = Collections.unmodifiableMap(InstanceUtils.getProperties(componentType, + FunctionCommon.getFullyQualifiedName( + this.config.getFunctionDetails().getTenant(), + this.config.getFunctionDetails().getNamespace(), + this.config.getFunctionDetails().getName()), + this.config.getInstanceId())); if (config.getFunctionDetails().getUserConfig().isEmpty()) { userConfigs = new HashMap<>(); @@ -534,57 +535,21 @@ public ClientBuilder getPulsarClientBuilder() { return clientBuilder; } + @Override + public void fatal(Throwable t) { + fatalHandler.accept(t); + } + private Producer getProducer(String topicName, Schema schema) throws PulsarClientException { - Producer producer; - if (tlPublishProducers != null) { - Map> producerMap = tlPublishProducers.get(); - if (producerMap == null) { - producerMap = new HashMap<>(); - tlPublishProducers.set(producerMap); - } - producer = (Producer) producerMap.get(topicName); - } else { - producer = (Producer) publishProducers.get(topicName); - } - - if (producer == null) { - - Producer newProducer = ((ProducerBuilderImpl) producerBuilder.clone()) - .schema(schema) - .blockIfQueueFull(true) - .enableBatching(true) - .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS) - .compressionType(CompressionType.LZ4) - .hashingScheme(HashingScheme.Murmur3_32Hash) // - .messageRoutingMode(MessageRoutingMode.CustomPartition) - .messageRouter(FunctionResultRouter.of()) - // set send timeout to be infinity to prevent potential deadlock with consumer - // that might happen when consumer is blocked due to unacked messages - .sendTimeout(0, TimeUnit.SECONDS) - .topic(topicName) - .properties(InstanceUtils.getProperties(componentType, - FunctionCommon.getFullyQualifiedName( - this.config.getFunctionDetails().getTenant(), - this.config.getFunctionDetails().getNamespace(), - this.config.getFunctionDetails().getName()), - this.config.getInstanceId())) - .create(); - - if (tlPublishProducers != null) { - tlPublishProducers.get().put(topicName, newProducer); - } else { - Producer existingProducer = (Producer) publishProducers.putIfAbsent(topicName, newProducer); - - if (existingProducer != null) { - // The value in the map was not updated after the concurrent put - newProducer.close(); - producer = existingProducer; - } else { - producer = newProducer; - } - } - } - return producer; + Long additionalCacheKey = useThreadLocalProducers ? Thread.currentThread().getId() : null; + return producerCache.getOrCreateProducer(ProducerCache.CacheArea.CONTEXT_CACHE, + topicName, additionalCacheKey, () -> { + log.info("Initializing producer on topic {} with schema {}", topicName, schema); + return producerBuilderFactory + .createProducerBuilder(topicName, schema, null) + .properties(producerProperties) + .create(); + }); } public Map getAndResetMetrics() { @@ -603,12 +568,13 @@ public Map getMetrics() { String metricName = userMetricsLabelsEntry.getKey(); String[] labels = userMetricsLabelsEntry.getValue(); Summary.Child.Value summary = userMetricsSummary.labels(labels).get(); - metricsMap.put(String.format("%s%s_sum", USER_METRIC_PREFIX, metricName), summary.sum); - metricsMap.put(String.format("%s%s_count", USER_METRIC_PREFIX, metricName), summary.count); + String prefix = USER_METRIC_PREFIX + metricName + "_"; + metricsMap.put(prefix + "sum", summary.sum); + metricsMap.put(prefix + "count", summary.count); for (Map.Entry entry : summary.quantiles.entrySet()) { Double quantile = entry.getKey(); Double value = entry.getValue(); - metricsMap.put(String.format("%s%s_%s", USER_METRIC_PREFIX, metricName, quantile), value); + metricsMap.put(prefix + quantile, value); } } return metricsMap; @@ -722,29 +688,9 @@ public void setUnderlyingBuilder(TypedMessageBuilder underlyingBuilder) { @Override public void close() { - List futures = new LinkedList<>(); - - if (publishProducers != null) { - for (Producer producer : publishProducers.values()) { - futures.add(producer.closeAsync()); - } - } - - if (tlPublishProducers != null) { - for (Producer producer : tlPublishProducers.get().values()) { - futures.add(producer.closeAsync()); - } - } - if (pulsarAdmin != null) { pulsarAdmin.close(); } - - try { - CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).get(); - } catch (InterruptedException | ExecutionException e) { - logger.warn("Failed to close producers", e); - } } @Override diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java index c3f36f754daca..baf0c5f7400ec 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java @@ -132,7 +132,7 @@ public class JavaInstanceRunnable implements AutoCloseable, Runnable { private JavaInstance javaInstance; @Getter - private Throwable deathException; + private volatile Throwable deathException; // function stats private ComponentStatsManager stats; @@ -168,6 +168,8 @@ public class JavaInstanceRunnable implements AutoCloseable, Runnable { private final AtomicReference> sinkSchema = new AtomicReference<>(); private SinkSchemaInfoProvider sinkSchemaInfoProvider = null; + private final ProducerCache producerCache = new ProducerCache(); + public JavaInstanceRunnable(InstanceConfig instanceConfig, ClientBuilder clientBuilder, PulsarClient pulsarClient, @@ -282,9 +284,20 @@ private synchronized void setup() throws Exception { ContextImpl setupContext() throws PulsarClientException { Logger instanceLog = LoggerFactory.getILoggerFactory().getLogger( "function-" + instanceConfig.getFunctionDetails().getName()); - return new ContextImpl(instanceConfig, instanceLog, client, secretsProvider, + Thread currentThread = Thread.currentThread(); + ClassLoader clsLoader = currentThread.getContextClassLoader(); + Consumer fatalHandler = throwable -> { + this.deathException = throwable; + currentThread.interrupt(); + }; + try { + Thread.currentThread().setContextClassLoader(functionClassLoader); + return new ContextImpl(instanceConfig, instanceLog, client, secretsProvider, collectorRegistry, metricsLabels, this.componentType, this.stats, stateManager, - pulsarAdmin, clientBuilder); + pulsarAdmin, clientBuilder, fatalHandler, producerCache); + } finally { + Thread.currentThread().setContextClassLoader(clsLoader); + } } public interface AsyncResultConsumer { @@ -340,16 +353,35 @@ public void run() { // process the synchronous results handleResult(currentRecord, result); } + + if (deathException != null) { + // Ideally the current java instance thread will be interrupted when the deathException is set. + // But if the CompletableFuture returned by the Pulsar Function is completed exceptionally(the + // function has invoked the fatal method) before being put into the JavaInstance + // .pendingAsyncRequests, the interrupted exception may be thrown when putting this future to + // JavaInstance.pendingAsyncRequests. The interrupted exception would be caught by the JavaInstance + // and be skipped. + // Therefore, we need to handle this case by checking the deathException here and rethrow it. + throw deathException; + } } } catch (Throwable t) { - log.error("[{}] Uncaught exception in Java Instance", FunctionCommon.getFullyQualifiedInstanceId( - instanceConfig.getFunctionDetails().getTenant(), - instanceConfig.getFunctionDetails().getNamespace(), - instanceConfig.getFunctionDetails().getName(), - instanceConfig.getInstanceId()), t); - deathException = t; + if (deathException != null) { + log.error("[{}] Fatal exception occurred in the instance", FunctionCommon.getFullyQualifiedInstanceId( + instanceConfig.getFunctionDetails().getTenant(), + instanceConfig.getFunctionDetails().getNamespace(), + instanceConfig.getFunctionDetails().getName(), + instanceConfig.getInstanceId()), deathException); + } else { + log.error("[{}] Uncaught exception in Java Instance", FunctionCommon.getFullyQualifiedInstanceId( + instanceConfig.getFunctionDetails().getTenant(), + instanceConfig.getFunctionDetails().getNamespace(), + instanceConfig.getFunctionDetails().getName(), + instanceConfig.getInstanceId()), t); + deathException = t; + } if (stats != null) { - stats.incrSysExceptions(t); + stats.incrSysExceptions(deathException); } } finally { log.info("Closing instance"); @@ -366,7 +398,7 @@ private void setupStateStore() throws Exception { stateStoreProvider = getStateStoreProvider(); Map stateStoreProviderConfig = new HashMap<>(); stateStoreProviderConfig.put(BKStateStoreProviderImpl.STATE_STORAGE_SERVICE_URL, stateStorageServiceUrl); - stateStoreProvider.init(stateStoreProviderConfig, instanceConfig.getFunctionDetails()); + stateStoreProvider.init(stateStoreProviderConfig); StateStore store = stateStoreProvider.getStateStore( instanceConfig.getFunctionDetails().getTenant(), @@ -577,6 +609,8 @@ public synchronized void close() { instanceCache = null; + producerCache.close(); + if (logAppender != null) { removeLogTopicAppender(LoggerContext.getContext()); removeLogTopicAppender(LoggerContext.getContext(false)); @@ -862,11 +896,7 @@ private void setupInput(ContextImpl contextImpl) throws Exception { Thread.currentThread().setContextClassLoader(this.componentClassLoader); } try { - if (sourceSpec.getConfigs().isEmpty()) { - this.source.open(new HashMap<>(), contextImpl); - } else { - this.source.open(parseComponentConfig(sourceSpec.getConfigs()), contextImpl); - } + this.source.open(augmentAndFilterConnectorConfig(sourceSpec.getConfigs()), contextImpl); if (this.source instanceof PulsarSource) { contextImpl.setInputConsumers(((PulsarSource) this.source).getInputConsumers()); } @@ -877,31 +907,60 @@ private void setupInput(ContextImpl contextImpl) throws Exception { Thread.currentThread().setContextClassLoader(this.instanceClassLoader); } } - private Map parseComponentConfig(String connectorConfigs) throws IOException { - return parseComponentConfig(connectorConfigs, instanceConfig, componentClassLoader, componentType); + + /** + * Recursively interpolate configured secrets into the config map by calling + * {@link SecretsProvider#interpolateSecretForValue(String)}. + * @param secretsProvider - the secrets provider that will convert secret's values into config values. + * @param configs - the connector configuration map, which will be mutated. + */ + private static void interpolateSecretsIntoConfigs(SecretsProvider secretsProvider, + Map configs) { + for (Map.Entry entry : configs.entrySet()) { + Object value = entry.getValue(); + if (value instanceof String) { + String updatedValue = secretsProvider.interpolateSecretForValue((String) value); + if (updatedValue != null) { + entry.setValue(updatedValue); + } + } else if (value instanceof Map) { + interpolateSecretsIntoConfigs(secretsProvider, (Map) value); + } + } + } + + private Map augmentAndFilterConnectorConfig(String connectorConfigs) throws IOException { + return augmentAndFilterConnectorConfig(connectorConfigs, instanceConfig, secretsProvider, + componentClassLoader, componentType); } - static Map parseComponentConfig(String connectorConfigs, - InstanceConfig instanceConfig, - ClassLoader componentClassLoader, - org.apache.pulsar.functions.proto.Function + static Map augmentAndFilterConnectorConfig(String connectorConfigs, + InstanceConfig instanceConfig, + SecretsProvider secretsProvider, + ClassLoader componentClassLoader, + org.apache.pulsar.functions.proto.Function .FunctionDetails.ComponentType componentType) throws IOException { - final Map config = ObjectMapperFactory + final Map config = connectorConfigs.isEmpty() ? new HashMap<>() : ObjectMapperFactory .getMapper() .reader() .forType(new TypeReference>() {}) .readValue(connectorConfigs); + if (componentType != org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SINK + && componentType != org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SOURCE) { + return config; + } + + interpolateSecretsIntoConfigs(secretsProvider, config); + if (instanceConfig.isIgnoreUnknownConfigFields() && componentClassLoader instanceof NarClassLoader) { final String configClassName; if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SOURCE) { configClassName = ConnectorUtils .getConnectorDefinition((NarClassLoader) componentClassLoader).getSourceConfigClass(); - } else if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SINK) { + } else { configClassName = ConnectorUtils .getConnectorDefinition((NarClassLoader) componentClassLoader).getSinkConfigClass(); - } else { - return config; } if (configClassName != null) { @@ -995,7 +1054,7 @@ private void setupOutput(ContextImpl contextImpl) throws Exception { } object = new PulsarSink(this.client, pulsarSinkConfig, this.properties, this.stats, - this.functionClassLoader); + this.functionClassLoader, this.producerCache); } } else { object = Reflections.createInstance( @@ -1014,19 +1073,11 @@ private void setupOutput(ContextImpl contextImpl) throws Exception { Thread.currentThread().setContextClassLoader(this.componentClassLoader); } try { - if (sinkSpec.getConfigs().isEmpty()) { - if (log.isDebugEnabled()) { - log.debug("Opening Sink with empty hashmap with contextImpl: {} ", contextImpl.toString()); - } - this.sink.open(new HashMap<>(), contextImpl); - } else { - if (log.isDebugEnabled()) { - log.debug("Opening Sink with SinkSpec {} and contextImpl: {} ", sinkSpec, - contextImpl.toString()); - } - final Map config = parseComponentConfig(sinkSpec.getConfigs()); - this.sink.open(config, contextImpl); + if (log.isDebugEnabled()) { + log.debug("Opening Sink with SinkSpec {} and contextImpl: {} ", sinkSpec.getConfigs(), + contextImpl.toString()); } + this.sink.open(augmentAndFilterConnectorConfig(sinkSpec.getConfigs()), contextImpl); } catch (Exception e) { log.error("Sink open produced uncaught exception: ", e); throw e; diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ProducerBuilderFactory.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ProducerBuilderFactory.java new file mode 100644 index 0000000000000..b08f7f3f2cb0f --- /dev/null +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ProducerBuilderFactory.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.instance; + +import static org.apache.commons.lang.StringUtils.isEmpty; +import com.google.common.annotations.VisibleForTesting; +import java.security.Security; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import lombok.Builder; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.client.api.BatcherBuilder; +import org.apache.pulsar.client.api.CompressionType; +import org.apache.pulsar.client.api.CryptoKeyReader; +import org.apache.pulsar.client.api.HashingScheme; +import org.apache.pulsar.client.api.MessageRoutingMode; +import org.apache.pulsar.client.api.ProducerBuilder; +import org.apache.pulsar.client.api.ProducerCryptoFailureAction; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.functions.CryptoConfig; +import org.apache.pulsar.common.functions.ProducerConfig; +import org.apache.pulsar.functions.utils.CryptoUtils; +import org.bouncycastle.jce.provider.BouncyCastleProvider; + +/** + * This class is responsible for creating ProducerBuilders with the appropriate configurations to + * match the ProducerConfig provided. Producers are created in 2 locations in Pulsar Functions and Connectors + * and this class is used to unify the configuration of the producers without duplicating code. + */ +@Slf4j +public class ProducerBuilderFactory { + + private final PulsarClient client; + private final ProducerConfig producerConfig; + private final Consumer> defaultConfigurer; + private final Crypto crypto; + + public ProducerBuilderFactory(PulsarClient client, ProducerConfig producerConfig, ClassLoader functionClassLoader, + Consumer> defaultConfigurer) { + this.client = client; + this.producerConfig = producerConfig; + this.defaultConfigurer = defaultConfigurer; + try { + this.crypto = initializeCrypto(functionClassLoader); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Unable to initialize crypto config " + producerConfig.getCryptoConfig(), e); + } + if (crypto == null) { + log.info("crypto key reader is not provided, not enabling end to end encryption"); + } + } + + public ProducerBuilder createProducerBuilder(String topic, Schema schema, String producerName) { + ProducerBuilder builder = client.newProducer(schema); + if (defaultConfigurer != null) { + defaultConfigurer.accept(builder); + } + builder.blockIfQueueFull(true) + .enableBatching(true) + .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS) + .hashingScheme(HashingScheme.Murmur3_32Hash) // + .messageRoutingMode(MessageRoutingMode.CustomPartition) + .messageRouter(FunctionResultRouter.of()) + // set send timeout to be infinity to prevent potential deadlock with consumer + // that might happen when consumer is blocked due to unacked messages + .sendTimeout(0, TimeUnit.SECONDS) + .topic(topic); + if (producerName != null) { + builder.producerName(producerName); + } + if (producerConfig != null) { + if (producerConfig.getCompressionType() != null) { + builder.compressionType(producerConfig.getCompressionType()); + } else { + // TODO: address this inconsistency. + // PR https://github.com/apache/pulsar/pull/19470 removed the default compression type of LZ4 + // from the top level. This default is only used if producer config is provided. + builder.compressionType(CompressionType.LZ4); + } + if (producerConfig.getMaxPendingMessages() != null && producerConfig.getMaxPendingMessages() != 0) { + builder.maxPendingMessages(producerConfig.getMaxPendingMessages()); + } + if (producerConfig.getMaxPendingMessagesAcrossPartitions() != null + && producerConfig.getMaxPendingMessagesAcrossPartitions() != 0) { + builder.maxPendingMessagesAcrossPartitions(producerConfig.getMaxPendingMessagesAcrossPartitions()); + } + if (producerConfig.getCryptoConfig() != null) { + builder.cryptoKeyReader(crypto.keyReader); + builder.cryptoFailureAction(crypto.failureAction); + for (String encryptionKeyName : crypto.getEncryptionKeys()) { + builder.addEncryptionKey(encryptionKeyName); + } + } + if (producerConfig.getBatchBuilder() != null) { + if (producerConfig.getBatchBuilder().equals("KEY_BASED")) { + builder.batcherBuilder(BatcherBuilder.KEY_BASED); + } else { + builder.batcherBuilder(BatcherBuilder.DEFAULT); + } + } + } + return builder; + } + + + @SuppressWarnings("unchecked") + @VisibleForTesting + Crypto initializeCrypto(ClassLoader functionClassLoader) throws ClassNotFoundException { + if (producerConfig == null + || producerConfig.getCryptoConfig() == null + || isEmpty(producerConfig.getCryptoConfig().getCryptoKeyReaderClassName())) { + return null; + } + + CryptoConfig cryptoConfig = producerConfig.getCryptoConfig(); + + // add provider only if it's not in the JVM + if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { + Security.addProvider(new BouncyCastleProvider()); + } + + final String[] encryptionKeys = cryptoConfig.getEncryptionKeys(); + Crypto.CryptoBuilder bldr = Crypto.builder() + .failureAction(cryptoConfig.getProducerCryptoFailureAction()) + .encryptionKeys(encryptionKeys); + + bldr.keyReader(CryptoUtils.getCryptoKeyReaderInstance( + cryptoConfig.getCryptoKeyReaderClassName(), cryptoConfig.getCryptoKeyReaderConfig(), + functionClassLoader)); + + return bldr.build(); + } + + @Data + @Builder + private static class Crypto { + private CryptoKeyReader keyReader; + private ProducerCryptoFailureAction failureAction; + private String[] encryptionKeys; + } +} diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ProducerCache.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ProducerCache.java new file mode 100644 index 0000000000000..f68c4e9589558 --- /dev/null +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ProducerCache.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.instance; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.Scheduler; +import com.google.common.annotations.VisibleForTesting; +import java.io.Closeable; +import java.time.Duration; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.common.util.FutureUtil; + +@Slf4j +public class ProducerCache implements Closeable { + // allow tuning the cache timeout with PRODUCER_CACHE_TIMEOUT_SECONDS env variable + private static final int PRODUCER_CACHE_TIMEOUT_SECONDS = + Integer.parseInt(System.getenv().getOrDefault("PRODUCER_CACHE_TIMEOUT_SECONDS", "300")); + // allow tuning the cache size with PRODUCER_CACHE_MAX_SIZE env variable + private static final int PRODUCER_CACHE_MAX_SIZE = + Integer.parseInt(System.getenv().getOrDefault("PRODUCER_CACHE_MAX_SIZE", "10000")); + private static final int FLUSH_OR_CLOSE_TIMEOUT_SECONDS = 60; + + // prevents the different producers created in different code locations from mixing up + public enum CacheArea { + // producers created by calling Context, SinkContext, SourceContext methods + CONTEXT_CACHE, + // producers created in Pulsar Sources, multiple topics are possible by returning destination topics + // by SinkRecord.getDestinationTopic call + SINK_RECORD_CACHE, + } + + record ProducerCacheKey(CacheArea cacheArea, String topic, Object additionalKey) { + } + + private final Cache> cache; + private final AtomicBoolean closed = new AtomicBoolean(false); + private final CopyOnWriteArrayList> closeFutures = new CopyOnWriteArrayList<>(); + + public ProducerCache() { + Caffeine builder = Caffeine.newBuilder() + .scheduler(Scheduler.systemScheduler()) + .removalListener((key, producer, cause) -> { + log.info("Closing producer for topic {}, cause {}", key.topic(), cause); + CompletableFuture closeFuture = + producer.flushAsync() + .orTimeout(FLUSH_OR_CLOSE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .exceptionally(ex -> { + log.error("Error flushing producer for topic {}", key.topic(), ex); + return null; + }).thenCompose(__ -> + producer.closeAsync().orTimeout(FLUSH_OR_CLOSE_TIMEOUT_SECONDS, + TimeUnit.SECONDS) + .exceptionally(ex -> { + log.error("Error closing producer for topic {}", key.topic(), + ex); + return null; + })); + if (closed.get()) { + closeFutures.add(closeFuture); + } + }) + .weigher((key, producer) -> Math.max(producer.getNumOfPartitions(), 1)) + .maximumWeight(PRODUCER_CACHE_MAX_SIZE); + if (PRODUCER_CACHE_TIMEOUT_SECONDS > 0) { + builder.expireAfterAccess(Duration.ofSeconds(PRODUCER_CACHE_TIMEOUT_SECONDS)); + } + cache = builder.build(); + } + + public Producer getOrCreateProducer(CacheArea cacheArea, String topicName, Object additionalCacheKey, + Callable> supplier) { + if (closed.get()) { + throw new IllegalStateException("ProducerCache is already closed"); + } + return (Producer) cache.get(new ProducerCacheKey(cacheArea, topicName, additionalCacheKey), key -> { + try { + return supplier.call(); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Unable to create producer for topic '" + topicName + "'", e); + } + }); + } + + public void close() { + if (closed.compareAndSet(false, true)) { + cache.invalidateAll(); + try { + FutureUtil.waitForAll(closeFutures).get(); + } catch (InterruptedException | ExecutionException e) { + log.warn("Failed to close producers", e); + } + } + } + + @VisibleForTesting + public boolean containsKey(CacheArea cacheArea, String topic) { + return containsKey(cacheArea, topic, null); + } + + @VisibleForTesting + public boolean containsKey(CacheArea cacheArea, String topic, Object additionalCacheKey) { + return cache.getIfPresent(new ProducerCacheKey(cacheArea, topic, additionalCacheKey)) != null; + } +} diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java index 67fe2a41d553d..467ec74921330 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java @@ -27,6 +27,8 @@ @Getter public class GoInstanceConfig { private String pulsarServiceURL = ""; + private String stateStorageServiceUrl = ""; + private String pulsarWebServiceUrl = ""; private int instanceID; private String funcID = ""; private String funcVersion = ""; @@ -44,6 +46,13 @@ public class GoInstanceConfig { private int processingGuarantees; private String secretsMap = ""; private String userConfig = ""; + + private String clientAuthenticationPlugin = ""; + private String clientAuthenticationParameters = ""; + private String tlsTrustCertsFilePath = ""; + private boolean tlsHostnameVerificationEnable = false; + private boolean tlsAllowInsecureConnection = false; + private int runtime; private boolean autoAck; private int parallelism; @@ -74,4 +83,6 @@ public class GoInstanceConfig { private String deadLetterTopic = ""; private int metricsPort; + + private String functionDetails = ""; } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreImpl.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreImpl.java index bf43f18b175e7..d85e4afd762ca 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreImpl.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreImpl.java @@ -28,6 +28,7 @@ import org.apache.bookkeeper.api.kv.Table; import org.apache.bookkeeper.api.kv.options.Options; import org.apache.pulsar.functions.api.StateStoreContext; +import org.apache.pulsar.functions.api.state.StateValue; import org.apache.pulsar.functions.utils.FunctionCommon; /** @@ -190,4 +191,33 @@ public ByteBuffer get(String key) { throw new RuntimeException("Failed to retrieve the state value for key '" + key + "'", e); } } + + @Override + public StateValue getStateValue(String key) { + try { + return result(getStateValueAsync(key)); + } catch (Exception e) { + throw new RuntimeException("Failed to retrieve the state value for key '" + key + "'", e); + } + } + + @Override + public CompletableFuture getStateValueAsync(String key) { + return table.getKv(Unpooled.wrappedBuffer(key.getBytes(UTF_8))).thenApply( + data -> { + try { + if (data != null && data.value() != null && data.value().readableBytes() >= 0) { + byte[] result = new byte[data.value().readableBytes()]; + data.value().readBytes(result); + return new StateValue(result, data.version(), data.isNumber()); + } + return null; + } finally { + if (data != null) { + ReferenceCountUtil.safeRelease(data); + } + } + } + ); + } } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreProviderImpl.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreProviderImpl.java index dbd0c8d2a0254..5faab27b341d3 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreProviderImpl.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreProviderImpl.java @@ -45,7 +45,6 @@ import org.apache.bookkeeper.stream.proto.StorageType; import org.apache.bookkeeper.stream.proto.StreamConfiguration; import org.apache.pulsar.functions.api.StateStore; -import org.apache.pulsar.functions.proto.Function.FunctionDetails; import org.apache.pulsar.functions.utils.FunctionCommon; /** @@ -58,7 +57,7 @@ public class BKStateStoreProviderImpl implements StateStoreProvider { private Map clients; @Override - public void init(Map config, FunctionDetails functionDetails) throws Exception { + public void init(Map config) throws Exception { stateStorageServiceUrl = (String) config.get(STATE_STORAGE_SERVICE_URL); clients = new HashMap<>(); } @@ -190,6 +189,29 @@ public T getStateStore(String tenant, String namespace, S return (T) new BKStateStoreImpl(tenant, namespace, name, table); } + @Override + public void cleanUp(String tenant, String namespace, String name) throws Exception { + StorageAdminClient storageAdminClient = new SimpleStorageAdminClientImpl( + StorageClientSettings.newBuilder().serviceUri(stateStorageServiceUrl).build(), + ClientResources.create().scheduler()); + String tableNs = FunctionCommon.getStateNamespace(tenant, namespace); + storageAdminClient.deleteStream(tableNs, name).whenComplete((res, throwable) -> { + if ((throwable == null && res) + || ((throwable instanceof NamespaceNotFoundException + || throwable instanceof StreamNotFoundException))) { + log.info("{}/{} table deleted successfully", tableNs, name); + } else { + if (throwable != null) { + log.error("{}/{} table deletion failed {} but moving on", tableNs, name, throwable); + } else { + log.error("{}/{} table deletion failed but moving on", tableNs, name); + } + } + }); + storageAdminClient.close(); + } + + @Override public void close() { clients.forEach((name, client) -> client.closeAsync() diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreImpl.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreImpl.java index 50541c40ae973..bba3cea0d8f38 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreImpl.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreImpl.java @@ -22,6 +22,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import org.apache.pulsar.functions.api.StateStoreContext; +import org.apache.pulsar.functions.api.state.StateValue; import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.MetadataStore; @@ -111,6 +112,20 @@ public CompletableFuture getAsync(String key) { .orElse(null)); } + @Override + public StateValue getStateValue(String key) { + return getStateValueAsync(key).join(); + } + + @Override + public CompletableFuture getStateValueAsync(String key) { + return store.get(getPath(key)) + .thenApply(optRes -> + optRes.map(x -> + new StateValue(x.getValue(), x.getStat().getVersion(), null)) + .orElse(null)); + } + @Override public void incrCounter(String key, long amount) { incrCounterAsync(key, amount).join(); diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreProviderImpl.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreProviderImpl.java index 0674398d1acda..7b9807b2a7816 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreProviderImpl.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreProviderImpl.java @@ -20,7 +20,6 @@ import java.util.Map; import lombok.SneakyThrows; -import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.MetadataStoreFactory; @@ -38,7 +37,7 @@ public class PulsarMetadataStateStoreProviderImpl implements StateStoreProvider private boolean shouldCloseStore; @Override - public void init(Map config, Function.FunctionDetails functionDetails) throws Exception { + public void init(Map config) throws Exception { prefix = (String) config.getOrDefault(METADATA_PREFIX, METADATA_DEFAULT_PREFIX); @@ -58,6 +57,13 @@ public DefaultStateStore getStateStore(String tenant, String namespace, String n return new PulsarMetadataStateStoreImpl(store, prefix, tenant, namespace, name); } + @Override + public void cleanUp(String tenant, String namespace, String name) throws Exception { + String fqsn = tenant + '/' + namespace + '/' + name; + String prefixPath = prefix + '/' + fqsn + '/'; + store.deleteRecursive(prefixPath); + } + @SneakyThrows @Override public void close() { diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/StateStoreProvider.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/StateStoreProvider.java index 1602d8f5ba367..4088888e4a5d4 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/StateStoreProvider.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/StateStoreProvider.java @@ -51,8 +51,17 @@ public void close() { * @param functionDetails the function details. * @throws Exception when failed to init the state store provider. */ + @Deprecated default void init(Map config, FunctionDetails functionDetails) throws Exception {} + /** + * Initialize the state store provider. + * + * @param config the config to init the state store provider. + * @throws Exception when failed to init the state store provider. + */ + default void init(Map config) throws Exception {} + /** * Get the state store with the provided store name. * @@ -67,6 +76,16 @@ default void init(Map config, FunctionDetails functionDetails) t */ T getStateStore(String tenant, String namespace, String name) throws Exception; + /** + * Clean up the state store with the provided store name. + * + * @param tenant the tenant that owns this state store + * @param namespace the namespace that owns this state store + * @param name the state store name + * @throws Exception when failed to clean up the state store provider. + */ + default void cleanUp(String tenant, String namespace, String name) throws Exception {} + @Override void close(); } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/ComponentStatsManager.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/ComponentStatsManager.java index 85d68531b5b9e..6da3c082f78f4 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/ComponentStatsManager.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/ComponentStatsManager.java @@ -149,7 +149,7 @@ public String getStatsAsString() throws IOException { protected InstanceCommunication.FunctionStatus.ExceptionInformation getExceptionInfo(Throwable th, long ts) { InstanceCommunication.FunctionStatus.ExceptionInformation.Builder exceptionInfoBuilder = InstanceCommunication.FunctionStatus.ExceptionInformation.newBuilder().setMsSinceEpoch(ts); - String msg = th.getMessage(); + String msg = String.format("[%s]: %s", th.getClass().getName(), th.getMessage()); if (msg != null) { exceptionInfoBuilder.setExceptionString(msg); } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/FunctionStatsManager.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/FunctionStatsManager.java index 1bb46da947224..8737c8a4fa913 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/FunctionStatsManager.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/FunctionStatsManager.java @@ -19,16 +19,15 @@ package org.apache.pulsar.functions.instance.stats; import com.google.common.collect.EvictingQueue; +import com.google.common.util.concurrent.RateLimiter; import io.prometheus.client.Counter; import io.prometheus.client.Gauge; import io.prometheus.client.Summary; import java.util.Arrays; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.common.util.RateLimiter; import org.apache.pulsar.functions.proto.InstanceCommunication; /** @@ -262,18 +261,8 @@ public FunctionStatsManager(FunctionCollectorRegistry collectorRegistry, .help("Exception from sink.") .create()); - userExceptionRateLimiter = RateLimiter.builder() - .scheduledExecutorService(scheduledExecutorService) - .permits(5) - .rateTime(1) - .timeUnit(TimeUnit.MINUTES) - .build(); - sysExceptionRateLimiter = RateLimiter.builder() - .scheduledExecutorService(scheduledExecutorService) - .permits(5) - .rateTime(1) - .timeUnit(TimeUnit.MINUTES) - .build(); + userExceptionRateLimiter = RateLimiter.create(5.0d / 60.0d); + sysExceptionRateLimiter = RateLimiter.create(5.0d / 60.0d); } public void addUserException(Throwable ex) { diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/SinkStatsManager.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/SinkStatsManager.java index 779a56cdf61be..c515ce6bc872c 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/SinkStatsManager.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/SinkStatsManager.java @@ -19,13 +19,12 @@ package org.apache.pulsar.functions.instance.stats; import com.google.common.collect.EvictingQueue; +import com.google.common.util.concurrent.RateLimiter; import io.prometheus.client.Counter; import io.prometheus.client.Gauge; import java.util.Arrays; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import lombok.Getter; -import org.apache.pulsar.common.util.RateLimiter; import org.apache.pulsar.functions.proto.InstanceCommunication; public class SinkStatsManager extends ComponentStatsManager { @@ -196,18 +195,8 @@ public SinkStatsManager(FunctionCollectorRegistry collectorRegistry, String[] me .help("Exception from sink.") .create()); - sysExceptionRateLimiter = RateLimiter.builder() - .scheduledExecutorService(scheduledExecutorService) - .permits(5) - .rateTime(1) - .timeUnit(TimeUnit.MINUTES) - .build(); - sinkExceptionRateLimiter = RateLimiter.builder() - .scheduledExecutorService(scheduledExecutorService) - .permits(5) - .rateTime(1) - .timeUnit(TimeUnit.MINUTES) - .build(); + sysExceptionRateLimiter = RateLimiter.create(5.0d / 60.0d); + sinkExceptionRateLimiter = RateLimiter.create(5.0d / 60.0d); } @Override diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/SourceStatsManager.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/SourceStatsManager.java index 4470310c6c3f7..1f7e159c4dcb5 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/SourceStatsManager.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/stats/SourceStatsManager.java @@ -19,13 +19,12 @@ package org.apache.pulsar.functions.instance.stats; import com.google.common.collect.EvictingQueue; +import com.google.common.util.concurrent.RateLimiter; import io.prometheus.client.Counter; import io.prometheus.client.Gauge; import java.util.Arrays; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import lombok.Getter; -import org.apache.pulsar.common.util.RateLimiter; import org.apache.pulsar.functions.proto.InstanceCommunication; public class SourceStatsManager extends ComponentStatsManager { @@ -196,18 +195,8 @@ public SourceStatsManager(FunctionCollectorRegistry collectorRegistry, String[] .help("Exception from source.") .create()); - sysExceptionRateLimiter = RateLimiter.builder() - .scheduledExecutorService(scheduledExecutorService) - .permits(5) - .rateTime(1) - .timeUnit(TimeUnit.MINUTES) - .build(); - sourceExceptionRateLimiter = RateLimiter.builder() - .scheduledExecutorService(scheduledExecutorService) - .permits(5) - .rateTime(1) - .timeUnit(TimeUnit.MINUTES) - .build(); + sysExceptionRateLimiter = RateLimiter.create(5.0d / 60.0d); + sourceExceptionRateLimiter = RateLimiter.create(5.0d / 60.0d); } @Override diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java index 97a0ad0a2ce17..da6b8006eb987 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java @@ -18,32 +18,17 @@ */ package org.apache.pulsar.functions.sink; -import static org.apache.commons.lang.StringUtils.isEmpty; import com.google.common.annotations.VisibleForTesting; import java.nio.charset.StandardCharsets; -import java.security.Security; -import java.util.ArrayList; import java.util.Base64; -import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; import java.util.function.Function; -import lombok.Builder; -import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.client.api.BatcherBuilder; -import org.apache.pulsar.client.api.CompressionType; -import org.apache.pulsar.client.api.CryptoKeyReader; -import org.apache.pulsar.client.api.HashingScheme; import org.apache.pulsar.client.api.MessageId; -import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.ProducerBuilder; -import org.apache.pulsar.client.api.ProducerCryptoFailureAction; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; @@ -52,22 +37,19 @@ import org.apache.pulsar.client.api.schema.KeyValueSchema; import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; import org.apache.pulsar.common.functions.ConsumerConfig; -import org.apache.pulsar.common.functions.CryptoConfig; import org.apache.pulsar.common.functions.FunctionConfig; -import org.apache.pulsar.common.functions.ProducerConfig; import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.Reflections; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.functions.instance.AbstractSinkRecord; -import org.apache.pulsar.functions.instance.FunctionResultRouter; +import org.apache.pulsar.functions.instance.ProducerBuilderFactory; +import org.apache.pulsar.functions.instance.ProducerCache; import org.apache.pulsar.functions.instance.stats.ComponentStatsManager; import org.apache.pulsar.functions.source.PulsarRecord; import org.apache.pulsar.functions.source.TopicSchema; -import org.apache.pulsar.functions.utils.CryptoUtils; import org.apache.pulsar.io.core.Sink; import org.apache.pulsar.io.core.SinkContext; -import org.bouncycastle.jce.provider.BouncyCastleProvider; @Slf4j public class PulsarSink implements Sink { @@ -77,11 +59,14 @@ public class PulsarSink implements Sink { private final Map properties; private final ClassLoader functionClassLoader; private ComponentStatsManager stats; + private final ProducerCache producerCache; @VisibleForTesting PulsarSinkProcessor pulsarSinkProcessor; private final TopicSchema topicSchema; + private Schema schema; + private ProducerBuilderFactory producerBuilderFactory; private interface PulsarSinkProcessor { @@ -93,98 +78,25 @@ private interface PulsarSinkProcessor { } abstract class PulsarSinkProcessorBase implements PulsarSinkProcessor { - protected Map> publishProducers = new ConcurrentHashMap<>(); - protected Schema schema; - protected Crypto crypto; - - protected PulsarSinkProcessorBase(Schema schema, Crypto crypto) { - this.schema = schema; - this.crypto = crypto; - } - - public Producer createProducer(PulsarClient client, String topic, String producerName, Schema schema) - throws PulsarClientException { - ProducerBuilder builder = client.newProducer(schema) - .blockIfQueueFull(true) - .enableBatching(true) - .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS) - .hashingScheme(HashingScheme.Murmur3_32Hash) // - .messageRoutingMode(MessageRoutingMode.CustomPartition) - .messageRouter(FunctionResultRouter.of()) - // set send timeout to be infinity to prevent potential deadlock with consumer - // that might happen when consumer is blocked due to unacked messages - .sendTimeout(0, TimeUnit.SECONDS) - .topic(topic); - if (producerName != null) { - builder.producerName(producerName); - } - if (pulsarSinkConfig.getProducerConfig() != null) { - ProducerConfig producerConfig = pulsarSinkConfig.getProducerConfig(); - if (producerConfig.getCompressionType() != null) { - builder.compressionType(producerConfig.getCompressionType()); - } else { - builder.compressionType(CompressionType.LZ4); - } - if (producerConfig.getMaxPendingMessages() != 0) { - builder.maxPendingMessages(producerConfig.getMaxPendingMessages()); - } - if (producerConfig.getMaxPendingMessagesAcrossPartitions() != 0) { - builder.maxPendingMessagesAcrossPartitions(producerConfig.getMaxPendingMessagesAcrossPartitions()); - } - if (producerConfig.getCryptoConfig() != null) { - builder.cryptoKeyReader(crypto.keyReader); - builder.cryptoFailureAction(crypto.failureAction); - for (String encryptionKeyName : crypto.getEncryptionKeys()) { - builder.addEncryptionKey(encryptionKeyName); - } - } - if (producerConfig.getBatchBuilder() != null) { - if (producerConfig.getBatchBuilder().equals("KEY_BASED")) { - builder.batcherBuilder(BatcherBuilder.KEY_BASED); - } else { - builder.batcherBuilder(BatcherBuilder.DEFAULT); - } - } - } - return builder.properties(properties).create(); - } - protected Producer getProducer(String destinationTopic, Schema schema) { - return getProducer(destinationTopic, null, destinationTopic, schema); + return getProducer(destinationTopic, schema, null, null); } - protected Producer getProducer(String producerId, String producerName, String topicName, Schema schema) { - return publishProducers.computeIfAbsent(producerId, s -> { - try { - log.info("Initializing producer {} on topic {} with schema {}", - producerName, topicName, schema); - Producer producer = createProducer( - client, - topicName, - producerName, - schema != null ? schema : this.schema); - log.info("Initialized producer {} on topic {} with schema {}: {} -> {}", - producerName, topicName, schema, producerId, producer); - return producer; - } catch (PulsarClientException e) { - log.error("Failed to create Producer while doing user publish", e); - throw new RuntimeException(e); - } - }); + protected Producer getProducer(String topicName, Schema schema, String producerName, String partitionId) { + return producerCache.getOrCreateProducer(ProducerCache.CacheArea.SINK_RECORD_CACHE, topicName, partitionId, + () -> { + Producer producer = createProducer(topicName, schema, producerName); + log.info( + "Initialized producer with name '{}' on topic '{}' with schema {} partitionId {} " + + "-> {}", + producerName, topicName, schema, partitionId, producer); + return producer; + }); } @Override public void close() throws Exception { - List> closeFutures = new ArrayList<>(publishProducers.size()); - for (Map.Entry> entry : publishProducers.entrySet()) { - Producer producer = entry.getValue(); - closeFutures.add(producer.closeAsync()); - } - try { - org.apache.pulsar.common.util.FutureUtil.waitForAll(closeFutures); - } catch (Exception e) { - log.warn("Failed to close all the producers", e); - } + // no op } public Function getPublishErrorHandler(AbstractSinkRecord record, boolean failSource) { @@ -218,17 +130,10 @@ public Function getPublishErrorHandler(AbstractSinkRecord re @VisibleForTesting class PulsarSinkAtMostOnceProcessor extends PulsarSinkProcessorBase { - public PulsarSinkAtMostOnceProcessor(Schema schema, Crypto crypto) { - super(schema, crypto); + public PulsarSinkAtMostOnceProcessor() { if (!(schema instanceof AutoConsumeSchema)) { // initialize default topic - try { - publishProducers.put(pulsarSinkConfig.getTopic(), - createProducer(client, pulsarSinkConfig.getTopic(), null, schema)); - } catch (PulsarClientException e) { - log.error("Failed to create Producer while doing user publish", e); - throw new RuntimeException(e); - } + getProducer(pulsarSinkConfig.getTopic(), schema); } else { if (log.isDebugEnabled()) { log.debug("The Pulsar producer is not initialized until the first record is" @@ -270,10 +175,6 @@ public void sendOutputMessage(TypedMessageBuilder msg, AbstractSinkRecord @VisibleForTesting class PulsarSinkAtLeastOnceProcessor extends PulsarSinkAtMostOnceProcessor { - public PulsarSinkAtLeastOnceProcessor(Schema schema, Crypto crypto) { - super(schema, crypto); - } - @Override public void sendOutputMessage(TypedMessageBuilder msg, AbstractSinkRecord record) { msg.sendAsync() @@ -284,11 +185,6 @@ public void sendOutputMessage(TypedMessageBuilder msg, AbstractSinkRecord @VisibleForTesting class PulsarSinkManualProcessor extends PulsarSinkAtMostOnceProcessor { - - public PulsarSinkManualProcessor(Schema schema, Crypto crypto) { - super(schema, crypto); - } - @Override public void sendOutputMessage(TypedMessageBuilder msg, AbstractSinkRecord record) { super.sendOutputMessage(msg, record); @@ -297,11 +193,6 @@ public void sendOutputMessage(TypedMessageBuilder msg, AbstractSinkRecord @VisibleForTesting class PulsarSinkEffectivelyOnceProcessor extends PulsarSinkProcessorBase { - - public PulsarSinkEffectivelyOnceProcessor(Schema schema, Crypto crypto) { - super(schema, crypto); - } - @Override public TypedMessageBuilder newMessage(AbstractSinkRecord record) { if (!record.getPartitionId().isPresent()) { @@ -315,13 +206,10 @@ public TypedMessageBuilder newMessage(AbstractSinkRecord record) { // we must use the destination topic schema schemaToWrite = schema; } - Producer producer = getProducer( - String.format("%s-%s", record.getDestinationTopic().orElse(pulsarSinkConfig.getTopic()), - record.getPartitionId().get()), - record.getPartitionId().get(), - record.getDestinationTopic().orElse(pulsarSinkConfig.getTopic()), - schemaToWrite - ); + String topicName = record.getDestinationTopic().orElse(pulsarSinkConfig.getTopic()); + String partitionId = record.getPartitionId().get(); + String producerName = partitionId; + Producer producer = getProducer(topicName, schemaToWrite, producerName, partitionId); if (schemaToWrite != null) { return producer.newMessage(schemaToWrite); } else { @@ -346,43 +234,41 @@ public void sendOutputMessage(TypedMessageBuilder msg, AbstractSinkRecord } public PulsarSink(PulsarClient client, PulsarSinkConfig pulsarSinkConfig, Map properties, - ComponentStatsManager stats, ClassLoader functionClassLoader) { + ComponentStatsManager stats, ClassLoader functionClassLoader, ProducerCache producerCache) { this.client = client; this.pulsarSinkConfig = pulsarSinkConfig; this.topicSchema = new TopicSchema(client, functionClassLoader); this.properties = properties; this.stats = stats; this.functionClassLoader = functionClassLoader; + this.producerCache = producerCache; } @Override public void open(Map config, SinkContext sinkContext) throws Exception { log.info("Opening pulsar sink with config: {}", pulsarSinkConfig); - Schema schema = initializeSchema(); + schema = initializeSchema(); if (schema == null) { log.info("Since output type is null, not creating any real sink"); return; } - - Crypto crypto = initializeCrypto(); - if (crypto == null) { - log.info("crypto key reader is not provided, not enabling end to end encryption"); - } + producerBuilderFactory = + new ProducerBuilderFactory(client, pulsarSinkConfig.getProducerConfig(), functionClassLoader, null); FunctionConfig.ProcessingGuarantees processingGuarantees = this.pulsarSinkConfig.getProcessingGuarantees(); switch (processingGuarantees) { case ATMOST_ONCE: - this.pulsarSinkProcessor = new PulsarSinkAtMostOnceProcessor(schema, crypto); + this.pulsarSinkProcessor = new PulsarSinkAtMostOnceProcessor(); break; case ATLEAST_ONCE: - this.pulsarSinkProcessor = new PulsarSinkAtLeastOnceProcessor(schema, crypto); + this.pulsarSinkProcessor = new PulsarSinkAtLeastOnceProcessor(); break; case EFFECTIVELY_ONCE: - this.pulsarSinkProcessor = new PulsarSinkEffectivelyOnceProcessor(schema, crypto); + this.pulsarSinkProcessor = new PulsarSinkEffectivelyOnceProcessor(); break; case MANUAL: - this.pulsarSinkProcessor = new PulsarSinkManualProcessor(schema, crypto); + this.pulsarSinkProcessor = new PulsarSinkManualProcessor(); break; } } @@ -427,6 +313,19 @@ public void close() throws Exception { } } + Producer createProducer(String topicName, Schema schema, String producerName) { + Schema schemaToUse = schema != null ? schema : this.schema; + try { + log.info("Initializing producer {} on topic {} with schema {}", producerName, topicName, schemaToUse); + return producerBuilderFactory.createProducerBuilder(topicName, schemaToUse, producerName) + .properties(properties) + .create(); + } catch (PulsarClientException e) { + throw new RuntimeException("Failed to create Producer for topic " + topicName + + " producerName " + producerName + " schema " + schemaToUse, e); + } + } + @SuppressWarnings("unchecked") @VisibleForTesting Schema initializeSchema() throws ClassNotFoundException { @@ -461,39 +360,5 @@ Schema initializeSchema() throws ClassNotFoundException { } } - @SuppressWarnings("unchecked") - @VisibleForTesting - Crypto initializeCrypto() throws ClassNotFoundException { - if (pulsarSinkConfig.getProducerConfig() == null - || pulsarSinkConfig.getProducerConfig().getCryptoConfig() == null - || isEmpty(pulsarSinkConfig.getProducerConfig().getCryptoConfig().getCryptoKeyReaderClassName())) { - return null; - } - - CryptoConfig cryptoConfig = pulsarSinkConfig.getProducerConfig().getCryptoConfig(); - - // add provider only if it's not in the JVM - if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { - Security.addProvider(new BouncyCastleProvider()); - } - final String[] encryptionKeys = cryptoConfig.getEncryptionKeys(); - Crypto.CryptoBuilder bldr = Crypto.builder() - .failureAction(cryptoConfig.getProducerCryptoFailureAction()) - .encryptionKeys(encryptionKeys); - - bldr.keyReader(CryptoUtils.getCryptoKeyReaderInstance( - cryptoConfig.getCryptoKeyReaderClassName(), cryptoConfig.getCryptoKeyReaderConfig(), - functionClassLoader)); - - return bldr.build(); - } - - @Data - @Builder - private static class Crypto { - private CryptoKeyReader keyReader; - private ProducerCryptoFailureAction failureAction; - private String[] encryptionKeys; - } } diff --git a/pulsar-functions/instance/src/main/python/python_instance.py b/pulsar-functions/instance/src/main/python/python_instance.py index 57edbf954b376..2ab3ccc46171c 100755 --- a/pulsar-functions/instance/src/main/python/python_instance.py +++ b/pulsar-functions/instance/src/main/python/python_instance.py @@ -147,6 +147,8 @@ def run(self): if self.instance_config.function_details.retainOrdering or \ self.instance_config.function_details.processingGuarantees == Function_pb2.ProcessingGuarantees.Value("EFFECTIVELY_ONCE"): mode = pulsar._pulsar.ConsumerType.Failover + elif self.instance_config.function_details.retainKeyOrdering: + mode = pulsar._pulsar.ConsumerType.KeyShared position = pulsar._pulsar.InitialPosition.Latest if self.instance_config.function_details.source.subscriptionPosition == Function_pb2.SubscriptionPosition.Value("EARLIEST"): diff --git a/pulsar-functions/instance/src/main/python/python_instance_main.py b/pulsar-functions/instance/src/main/python/python_instance_main.py index 9a923c7e3a18b..943e1c1c245f5 100755 --- a/pulsar-functions/instance/src/main/python/python_instance_main.py +++ b/pulsar-functions/instance/src/main/python/python_instance_main.py @@ -207,6 +207,15 @@ def main(): zpfile.extractall(os.path.dirname(str(args.py))) basename = os.path.basename(os.path.splitext(str(args.py))[0]) + requirements_file = os.path.join(os.path.dirname(str(args.py)), basename, "requirements.txt") + if os.path.isfile(requirements_file): + cmd = "pip install -r %s" % requirements_file + Log.debug("Install python dependencies via cmd: %s" % cmd) + retval = os.system(cmd) + if retval != 0: + print("Could not install user depedencies specified by the requirements.txt file") + sys.exit(1) + deps_dir = os.path.join(os.path.dirname(str(args.py)), basename, "deps") if os.path.isdir(deps_dir) and os.listdir(deps_dir): diff --git a/pulsar-functions/instance/src/main/resources/findbugsExclude.xml b/pulsar-functions/instance/src/main/resources/findbugsExclude.xml index 7fe247d2ab20a..40e3e91112328 100644 --- a/pulsar-functions/instance/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/instance/src/main/resources/findbugsExclude.xml @@ -49,7 +49,12 @@ - + + + + + + diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/ContextImplTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/ContextImplTest.java index e0ebb52da7490..cb4c93f153fd9 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/ContextImplTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/ContextImplTest.java @@ -38,6 +38,7 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.Consumer; @@ -47,6 +48,7 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.TypedMessageBuilder; +import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.ConsumerImpl; import org.apache.pulsar.client.impl.MultiTopicsConsumerImpl; import org.apache.pulsar.client.impl.ProducerBase; @@ -70,6 +72,7 @@ import org.mockito.Mockito; import org.slf4j.Logger; import org.testng.Assert; +import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -85,6 +88,7 @@ public class ContextImplTest { private PulsarAdmin pulsarAdmin; private ContextImpl context; private Producer producer; + private ProducerCache producerCache; @BeforeMethod(alwaysRun = true) public void setup() throws PulsarClientException { @@ -99,7 +103,11 @@ public void setup() throws PulsarClientException { producer = mock(Producer.class); client = mock(PulsarClientImpl.class); - when(client.newProducer()).thenReturn(new ProducerBuilderImpl(client, Schema.BYTES)); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); + when(client.newProducer()).thenAnswer(invocation -> new ProducerBuilderImpl(client, Schema.BYTES)); + when(client.newProducer(any())).thenAnswer( + invocation -> new ProducerBuilderImpl(client, invocation.getArgument(0))); when(client.createProducerAsync(any(ProducerConfigurationData.class), any(), any())) .thenReturn(CompletableFuture.completedFuture(producer)); when(client.getSchema(anyString())).thenReturn(CompletableFuture.completedFuture(Optional.empty())); @@ -111,16 +119,24 @@ public void setup() throws PulsarClientException { TypedMessageBuilder messageBuilder = spy(new TypedMessageBuilderImpl(mock(ProducerBase.class), Schema.STRING)); doReturn(new CompletableFuture<>()).when(messageBuilder).sendAsync(); when(producer.newMessage()).thenReturn(messageBuilder); + doReturn(CompletableFuture.completedFuture(null)).when(producer).flushAsync(); + producerCache = new ProducerCache(); context = new ContextImpl( config, logger, client, new EnvironmentBasedSecretsProvider(), FunctionCollectorRegistry.getDefaultImplementation(), new String[0], FunctionDetails.ComponentType.FUNCTION, null, new InstanceStateManager(), - pulsarAdmin, clientBuilder); + pulsarAdmin, clientBuilder, t -> {}, producerCache); context.setCurrentMessageContext((Record) () -> null); } + @AfterMethod(alwaysRun = true) + public void tearDown() { + producerCache.close(); + producerCache = null; + } + @Test(expectedExceptions = IllegalStateException.class) public void testIncrCounterStateDisabled() { context.incrCounter("test-key", 10); @@ -231,7 +247,7 @@ public void testGetPulsarAdminWithExposePulsarAdminDisabled() throws PulsarClien new EnvironmentBasedSecretsProvider(), FunctionCollectorRegistry.getDefaultImplementation(), new String[0], FunctionDetails.ComponentType.FUNCTION, null, new InstanceStateManager(), - pulsarAdmin, clientBuilder); + pulsarAdmin, clientBuilder, t -> {}, producerCache); context.getPulsarAdmin(); } @@ -245,7 +261,7 @@ public void testUnsupportedExtendedSinkContext() throws PulsarClientException { new EnvironmentBasedSecretsProvider(), FunctionCollectorRegistry.getDefaultImplementation(), new String[0], FunctionDetails.ComponentType.FUNCTION, null, new InstanceStateManager(), - pulsarAdmin, clientBuilder); + pulsarAdmin, clientBuilder, t -> {}, producerCache); try { context.seek("z", 0, Mockito.mock(MessageId.class)); Assert.fail("Expected exception"); @@ -276,7 +292,7 @@ public void testExtendedSinkContext() throws PulsarClientException { new EnvironmentBasedSecretsProvider(), FunctionCollectorRegistry.getDefaultImplementation(), new String[0], FunctionDetails.ComponentType.FUNCTION, null, new InstanceStateManager(), - pulsarAdmin, clientBuilder); + pulsarAdmin, clientBuilder, t -> {}, producerCache); Consumer mockConsumer = Mockito.mock(Consumer.class); when(mockConsumer.getTopic()).thenReturn(TopicName.get("z").toString()); context.setInputConsumers(Lists.newArrayList(mockConsumer)); @@ -308,7 +324,7 @@ public void testGetConsumer() throws PulsarClientException { new EnvironmentBasedSecretsProvider(), FunctionCollectorRegistry.getDefaultImplementation(), new String[0], FunctionDetails.ComponentType.FUNCTION, null, new InstanceStateManager(), - pulsarAdmin, clientBuilder); + pulsarAdmin, clientBuilder, t -> {}, producerCache); Consumer mockConsumer = Mockito.mock(Consumer.class); when(mockConsumer.getTopic()).thenReturn(TopicName.get("z").toString()); context.setInputConsumers(Lists.newArrayList(mockConsumer)); @@ -332,7 +348,7 @@ public void testGetConsumerMultiTopic() throws PulsarClientException { new EnvironmentBasedSecretsProvider(), FunctionCollectorRegistry.getDefaultImplementation(), new String[0], FunctionDetails.ComponentType.FUNCTION, null, new InstanceStateManager(), - pulsarAdmin, clientBuilder); + pulsarAdmin, clientBuilder, t -> {}, producerCache); ConsumerImpl consumer1 = Mockito.mock(ConsumerImpl.class); when(consumer1.getTopic()).thenReturn(TopicName.get("first").toString()); ConsumerImpl consumer2 = Mockito.mock(ConsumerImpl.class); @@ -435,4 +451,23 @@ public Map getProperties() { assertEquals(record.getProperties().get("prop-key"), "prop-value"); assertNull(record.getValue()); } + + @Test + public void testFatal() { + Throwable fatalException = new Exception("test-fatal-exception"); + AtomicBoolean fatalInvoked = new AtomicBoolean(false); + context = new ContextImpl( + config, + logger, + client, + new EnvironmentBasedSecretsProvider(), FunctionCollectorRegistry.getDefaultImplementation(), + new String[0], + FunctionDetails.ComponentType.FUNCTION, null, new InstanceStateManager(), + pulsarAdmin, clientBuilder, t -> { + assertEquals(t, fatalException); + fatalInvoked.set(true); + }, producerCache); + context.fatal(fatalException); + assertTrue(fatalInvoked.get()); + } } diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java index 5fea8bcc9fde9..c83648132d488 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java @@ -24,18 +24,24 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; - +import com.fasterxml.jackson.annotation.JsonIgnore; import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.TreeSet; - -import com.fasterxml.jackson.annotation.JsonIgnore; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicReference; import lombok.Getter; import lombok.Setter; +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.ClientBuilder; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.io.ConnectorDefinition; import org.apache.pulsar.common.nar.NarClassLoader; import org.apache.pulsar.common.util.ObjectMapperFactory; @@ -46,13 +52,24 @@ import org.apache.pulsar.functions.instance.stats.ComponentStatsManager; import org.apache.pulsar.functions.proto.Function.FunctionDetails; import org.apache.pulsar.functions.proto.Function.SinkSpec; +import org.apache.pulsar.functions.proto.Function.SourceSpec; import org.apache.pulsar.functions.proto.InstanceCommunication; +import org.apache.pulsar.functions.secretsprovider.EnvironmentBasedSecretsProvider; +import org.apache.pulsar.io.core.Sink; +import org.apache.pulsar.io.core.SinkContext; +import org.apache.pulsar.io.core.Source; +import org.apache.pulsar.io.core.SourceContext; +import org.awaitility.Awaitility; import org.jetbrains.annotations.NotNull; import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +@Slf4j public class JavaInstanceRunnableTest { + private final List closeables = new ArrayList<>(); static class IntegerSerDe implements SerDe { @Override @@ -90,6 +107,25 @@ private JavaInstanceRunnable createRunnable(FunctionDetails functionDetails) thr return javaInstanceRunnable; } + private JavaInstanceRunnable createRunnable(SourceSpec sourceSpec, + String functionClassName, SinkSpec sinkSpec) + throws PulsarClientException { + ClientBuilder clientBuilder = mock(ClientBuilder.class); + when(clientBuilder.build()).thenReturn(null); + FunctionDetails functionDetails = FunctionDetails.newBuilder() + .setSource(sourceSpec) + .setClassName(functionClassName) + .setSink(sinkSpec) + .build(); + InstanceConfig config = createInstanceConfig(functionDetails); + config.setClusterName("test-cluster"); + PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("pulsar://test-cluster:6650").build(); + registerCloseable(pulsarClient); + return new JavaInstanceRunnable(config, clientBuilder, + pulsarClient, null, null, null, null, null, + Thread.currentThread().getContextClassLoader(), null); + } + private Method makeAccessible(JavaInstanceRunnable javaInstanceRunnable) throws Exception { Method method = javaInstanceRunnable.getClass().getDeclaredMethod("setupSerDe", Class[].class, ClassLoader.class); @@ -191,9 +227,10 @@ public void testStatsManagerNull() throws Exception { @Test public void testSinkConfigParsingPreservesOriginalType() throws Exception { - final Map parsedConfig = JavaInstanceRunnable.parseComponentConfig( + final Map parsedConfig = JavaInstanceRunnable.augmentAndFilterConnectorConfig( "{\"ttl\": 9223372036854775807}", new InstanceConfig(), + new EnvironmentBasedSecretsProvider(), null, FunctionDetails.ComponentType.SINK ); @@ -203,9 +240,10 @@ public void testSinkConfigParsingPreservesOriginalType() throws Exception { @Test public void testSourceConfigParsingPreservesOriginalType() throws Exception { - final Map parsedConfig = JavaInstanceRunnable.parseComponentConfig( + final Map parsedConfig = JavaInstanceRunnable.augmentAndFilterConnectorConfig( "{\"ttl\": 9223372036854775807}", new InstanceConfig(), + new EnvironmentBasedSecretsProvider(), null, FunctionDetails.ComponentType.SOURCE ); @@ -213,6 +251,58 @@ public void testSourceConfigParsingPreservesOriginalType() throws Exception { Assert.assertEquals(parsedConfig.get("ttl"), Long.MAX_VALUE); } + @DataProvider(name = "component") + public Object[][] component() { + return new Object[][]{ + // Schema: component type, whether to map in secrets + { FunctionDetails.ComponentType.SINK }, + { FunctionDetails.ComponentType.SOURCE }, + { FunctionDetails.ComponentType.FUNCTION }, + { FunctionDetails.ComponentType.UNKNOWN }, + }; + } + + @Test(dataProvider = "component") + public void testEmptyStringInput(FunctionDetails.ComponentType componentType) throws Exception { + final Map parsedConfig = JavaInstanceRunnable.augmentAndFilterConnectorConfig( + "", + new InstanceConfig(), + new EnvironmentBasedSecretsProvider(), + null, + componentType + ); + Assert.assertEquals(parsedConfig.size(), 0); + } + + // Environment variables are set in the pom.xml file + @Test(dataProvider = "component") + public void testInterpolatingEnvironmentVariables(FunctionDetails.ComponentType componentType) throws Exception { + final Map parsedConfig = JavaInstanceRunnable.augmentAndFilterConnectorConfig( + """ + { + "key": { + "key1": "${TEST_JAVA_INSTANCE_PARSE_ENV_VAR}", + "key2": "${unset-env-var}" + }, + "key3": "${TEST_JAVA_INSTANCE_PARSE_ENV_VAR}" + } + """, + new InstanceConfig(), + new EnvironmentBasedSecretsProvider(), + null, + componentType + ); + if ((componentType == FunctionDetails.ComponentType.SOURCE + || componentType == FunctionDetails.ComponentType.SINK)) { + Assert.assertEquals(((Map) parsedConfig.get("key")).get("key1"), "some-configuration"); + Assert.assertEquals(((Map) parsedConfig.get("key")).get("key2"), "${unset-env-var}"); + Assert.assertEquals(parsedConfig.get("key3"), "some-configuration"); + } else { + Assert.assertEquals(((Map) parsedConfig.get("key")).get("key1"), "${TEST_JAVA_INSTANCE_PARSE_ENV_VAR}"); + Assert.assertEquals(((Map) parsedConfig.get("key")).get("key2"), "${unset-env-var}"); + Assert.assertEquals(parsedConfig.get("key3"), "${TEST_JAVA_INSTANCE_PARSE_ENV_VAR}"); + } + } public static class ConnectorTestConfig1 { public String field1; @@ -243,9 +333,10 @@ public void testSinkConfigIgnoreUnknownFields(boolean ignoreUnknownConfigFields, final InstanceConfig instanceConfig = new InstanceConfig(); instanceConfig.setIgnoreUnknownConfigFields(ignoreUnknownConfigFields); - final Map parsedConfig = JavaInstanceRunnable.parseComponentConfig( + final Map parsedConfig = JavaInstanceRunnable.augmentAndFilterConnectorConfig( "{\"field1\": \"value\", \"field2\": \"value2\"}", instanceConfig, + new EnvironmentBasedSecretsProvider(), narClassLoader, type ); @@ -277,4 +368,162 @@ public void testBeanPropertiesReader() throws Exception { .getBeanProperties(ConnectorTestConfig2.class); Assert.assertEquals(new TreeSet<>(beanProperties), new TreeSet<>(Arrays.asList("field1", "withGetter"))); } + + public static class TestSourceConnector implements Source { + + private LinkedBlockingQueue> queue; + private SourceContext context; + + public void pushRecord(Record record) throws Exception { + queue.put(record); + } + + @Override + public void open(Map config, SourceContext sourceContext) throws Exception { + context = sourceContext; + queue = new LinkedBlockingQueue<>(); + } + + @Override + public Record read() throws Exception { + return queue.take(); + } + + @Override + public void close() throws Exception { + + } + + public void fatalConnector() { + context.fatal(new Exception(FailComponentType.FAIL_SOURCE.toString())); + } + } + + public static class TestFunction implements Function> { + @Override + public CompletableFuture process(String input, Context context) throws Exception { + CompletableFuture future = new CompletableFuture<>(); + new Thread(() -> { + if (FailComponentType.FAIL_FUNC.toString().equals(input)) { + context.fatal(new Exception(FailComponentType.FAIL_FUNC.toString())); + } else { + future.complete(input); + } + }).start(); + return future; + } + } + + public static class TestSinkConnector implements Sink { + SinkContext context; + + @Override + public void open(Map config, SinkContext sinkContext) throws Exception { + this.context = sinkContext; + } + + @Override + public void write(Record record) throws Exception { + new Thread(() -> { + if (FailComponentType.FAIL_SINK.toString().equals(record.getValue())) { + context.fatal(new Exception(FailComponentType.FAIL_SINK.toString())); + } + }).start(); + } + + @Override + public void close() throws Exception { + + } + } + + private Object getPrivateField(JavaInstanceRunnable javaInstanceRunnable, String fieldName) + throws NoSuchFieldException, IllegalAccessException { + Field field = JavaInstanceRunnable.class.getDeclaredField(fieldName); + field.setAccessible(true); + return field.get(javaInstanceRunnable); + } + + public enum FailComponentType { + FAIL_SOURCE, + FAIL_FUNC, + FAIL_SINK + } + + @DataProvider(name = "failComponentType") + public Object[][] failType() { + return new Object[][]{{FailComponentType.FAIL_SOURCE}, {FailComponentType.FAIL_FUNC}, + {FailComponentType.FAIL_SINK}}; + } + + @Test(dataProvider = "failComponentType") + public void testFatalTheInstance(FailComponentType failComponentType) throws Exception { + JavaInstanceRunnable javaInstanceRunnable = createRunnable( + SourceSpec.newBuilder() + .setClassName(TestSourceConnector.class.getName()).build(), + TestFunction.class.getName(), + SinkSpec.newBuilder().setClassName(TestSinkConnector.class.getName()).build() + ); + + Thread fnThread = new Thread(javaInstanceRunnable); + fnThread.start(); + + // Wait for the setup to complete + AtomicReference source = new AtomicReference<>(); + Awaitility.await() + .pollInterval(Duration.ofMillis(200)) + .atMost(Duration.ofSeconds(10)) + .ignoreExceptions().untilAsserted(() -> { + TestSourceConnector sourceConnector = (TestSourceConnector) getPrivateField(javaInstanceRunnable, + "source"); + Assert.assertNotNull(sourceConnector); + source.set(sourceConnector); + }); + + // Fail the connector or function + if (failComponentType == FailComponentType.FAIL_SOURCE) { + source.get().fatalConnector(); + } else { + source.get().pushRecord(failComponentType::toString); + } + + // Assert that the instance is terminated with the fatal exception + Awaitility.await() + .pollInterval(Duration.ofMillis(200)) + .atMost(Duration.ofSeconds(10)) + .ignoreExceptions().untilAsserted(() -> { + Assert.assertNotNull(javaInstanceRunnable.getDeathException()); + Assert.assertEquals(javaInstanceRunnable.getDeathException().getMessage(), + failComponentType.toString()); + + // Assert the java instance is closed + Assert.assertFalse(fnThread.isAlive()); + Assert.assertFalse((boolean) getPrivateField(javaInstanceRunnable, "isInitialized")); + }); + } + + @AfterClass + public void cleanupInstanceCache() { + InstanceCache.shutdown(); + } + + @AfterMethod(alwaysRun = true) + public void cleanupCloseables() { + callCloseables(closeables); + } + + protected T registerCloseable(T closeable) { + closeables.add(closeable); + return closeable; + } + + private static void callCloseables(List closeables) { + for (int i = closeables.size() - 1; i >= 0; i--) { + try { + closeables.get(i).close(); + } catch (Exception e) { + log.error("Failure in calling close method", e); + } + } + } } diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceTest.java index efe80922dfa8c..5a3332042938d 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceTest.java @@ -24,6 +24,7 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertSame; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import lombok.Cleanup; @@ -185,6 +186,7 @@ public void testUserExceptionThrowingAsyncFunction() throws Exception { @Test public void testAsyncFunctionMaxPending() throws Exception { + CountDownLatch count = new CountDownLatch(1); InstanceConfig instanceConfig = new InstanceConfig(); int pendingQueueSize = 3; instanceConfig.setMaxPendingAsyncRequests(pendingQueueSize); @@ -196,7 +198,7 @@ public void testAsyncFunctionMaxPending() throws Exception { CompletableFuture result = new CompletableFuture<>(); executor.submit(() -> { try { - Thread.sleep(500); + count.await(); result.complete(String.format("%s-lambda", input)); } catch (Exception e) { result.completeExceptionally(e); @@ -222,8 +224,13 @@ public void testAsyncFunctionMaxPending() throws Exception { // no space left assertEquals(0, instance.getPendingAsyncRequests().remainingCapacity()); + AsyncFuncRequest[] asyncFuncRequests = new AsyncFuncRequest[3]; for (int i = 0; i < 3; i++) { - AsyncFuncRequest request = instance.getPendingAsyncRequests().poll(); + asyncFuncRequests[i] = instance.getPendingAsyncRequests().poll(); + } + + count.countDown(); + for (AsyncFuncRequest request : asyncFuncRequests) { Assert.assertEquals(request.getProcessResult().get(), testString + "-lambda"); } diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/ProducerBuilderFactoryTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/ProducerBuilderFactoryTest.java new file mode 100644 index 0000000000000..42940f7e2dae3 --- /dev/null +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/ProducerBuilderFactoryTest.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.instance; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.apache.pulsar.client.api.BatcherBuilder; +import org.apache.pulsar.client.api.CompressionType; +import org.apache.pulsar.client.api.CryptoKeyReader; +import org.apache.pulsar.client.api.EncryptionKeyInfo; +import org.apache.pulsar.client.api.HashingScheme; +import org.apache.pulsar.client.api.MessageRoutingMode; +import org.apache.pulsar.client.api.ProducerBuilder; +import org.apache.pulsar.client.api.ProducerCryptoFailureAction; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.functions.CryptoConfig; +import org.apache.pulsar.common.functions.ProducerConfig; +import org.mockito.internal.util.MockUtil; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class ProducerBuilderFactoryTest { + private PulsarClient pulsarClient; + private ProducerBuilder producerBuilder; + + @BeforeMethod + public void setup() { + pulsarClient = mock(PulsarClient.class); + + producerBuilder = mock(ProducerBuilder.class); + doReturn(producerBuilder).when(producerBuilder).blockIfQueueFull(anyBoolean()); + doReturn(producerBuilder).when(producerBuilder).enableBatching(anyBoolean()); + doReturn(producerBuilder).when(producerBuilder).batchingMaxPublishDelay(anyLong(), any()); + doReturn(producerBuilder).when(producerBuilder).compressionType(any()); + doReturn(producerBuilder).when(producerBuilder).hashingScheme(any()); + doReturn(producerBuilder).when(producerBuilder).messageRoutingMode(any()); + doReturn(producerBuilder).when(producerBuilder).messageRouter(any()); + doReturn(producerBuilder).when(producerBuilder).topic(anyString()); + doReturn(producerBuilder).when(producerBuilder).producerName(anyString()); + doReturn(producerBuilder).when(producerBuilder).property(anyString(), anyString()); + doReturn(producerBuilder).when(producerBuilder).properties(any()); + doReturn(producerBuilder).when(producerBuilder).sendTimeout(anyInt(), any()); + + doReturn(producerBuilder).when(pulsarClient).newProducer(); + doReturn(producerBuilder).when(pulsarClient).newProducer(any()); + } + + @AfterMethod + public void tearDown() { + MockUtil.resetMock(pulsarClient); + pulsarClient = null; + MockUtil.resetMock(producerBuilder); + producerBuilder = null; + TestCryptoKeyReader.LAST_INSTANCE = null; + } + + @Test + public void testCreateProducerBuilder() { + ProducerBuilderFactory builderFactory = new ProducerBuilderFactory(pulsarClient, null, null, null); + builderFactory.createProducerBuilder("topic", Schema.STRING, "producerName"); + verifyCommon(); + verifyNoMoreInteractions(producerBuilder); + } + + private void verifyCommon() { + verify(pulsarClient).newProducer(Schema.STRING); + verify(producerBuilder).blockIfQueueFull(true); + verify(producerBuilder).enableBatching(true); + verify(producerBuilder).batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS); + verify(producerBuilder).hashingScheme(HashingScheme.Murmur3_32Hash); + verify(producerBuilder).messageRoutingMode(MessageRoutingMode.CustomPartition); + verify(producerBuilder).messageRouter(FunctionResultRouter.of()); + verify(producerBuilder).sendTimeout(0, TimeUnit.SECONDS); + verify(producerBuilder).topic("topic"); + verify(producerBuilder).producerName("producerName"); + } + + @Test + public void testCreateProducerBuilderWithDefaultConfigurer() { + ProducerBuilderFactory builderFactory = new ProducerBuilderFactory(pulsarClient, null, null, + builder -> builder.property("key", "value")); + builderFactory.createProducerBuilder("topic", Schema.STRING, "producerName"); + verifyCommon(); + verify(producerBuilder).property("key", "value"); + verifyNoMoreInteractions(producerBuilder); + } + + @Test + public void testCreateProducerBuilderWithSimpleProducerConfig() { + ProducerConfig producerConfig = new ProducerConfig(); + producerConfig.setBatchBuilder("KEY_BASED"); + ProducerBuilderFactory builderFactory = new ProducerBuilderFactory(pulsarClient, producerConfig, null, null); + builderFactory.createProducerBuilder("topic", Schema.STRING, "producerName"); + verifyCommon(); + verify(producerBuilder).compressionType(CompressionType.LZ4); + verify(producerBuilder).batcherBuilder(BatcherBuilder.KEY_BASED); + verifyNoMoreInteractions(producerBuilder); + } + + @Test + public void testCreateProducerBuilderWithAdvancedProducerConfig() { + ProducerConfig producerConfig = new ProducerConfig(); + producerConfig.setBatchBuilder("KEY_BASED"); + producerConfig.setCompressionType(CompressionType.SNAPPY); + producerConfig.setMaxPendingMessages(5000); + producerConfig.setMaxPendingMessagesAcrossPartitions(50000); + CryptoConfig cryptoConfig = new CryptoConfig(); + cryptoConfig.setProducerCryptoFailureAction(ProducerCryptoFailureAction.FAIL); + cryptoConfig.setEncryptionKeys(new String[]{"key1", "key2"}); + cryptoConfig.setCryptoKeyReaderConfig(Map.of("key", "value")); + cryptoConfig.setCryptoKeyReaderClassName(TestCryptoKeyReader.class.getName()); + producerConfig.setCryptoConfig(cryptoConfig); + ProducerBuilderFactory builderFactory = new ProducerBuilderFactory(pulsarClient, producerConfig, null, null); + builderFactory.createProducerBuilder("topic", Schema.STRING, "producerName"); + verifyCommon(); + verify(producerBuilder).compressionType(CompressionType.SNAPPY); + verify(producerBuilder).batcherBuilder(BatcherBuilder.KEY_BASED); + verify(producerBuilder).maxPendingMessages(5000); + verify(producerBuilder).maxPendingMessagesAcrossPartitions(50000); + TestCryptoKeyReader lastInstance = TestCryptoKeyReader.LAST_INSTANCE; + assertNotNull(lastInstance); + assertEquals(lastInstance.configs, cryptoConfig.getCryptoKeyReaderConfig()); + verify(producerBuilder).cryptoKeyReader(lastInstance); + verify(producerBuilder).cryptoFailureAction(ProducerCryptoFailureAction.FAIL); + verify(producerBuilder).addEncryptionKey("key1"); + verify(producerBuilder).addEncryptionKey("key2"); + verifyNoMoreInteractions(producerBuilder); + } + + public static class TestCryptoKeyReader implements CryptoKeyReader { + static TestCryptoKeyReader LAST_INSTANCE; + Map configs; + public TestCryptoKeyReader(Map configs) { + this.configs = configs; + assert LAST_INSTANCE == null; + LAST_INSTANCE = this; + } + + @Override + public EncryptionKeyInfo getPublicKey(String keyName, Map metadata) { + throw new UnsupportedOperationException(); + } + + @Override + public EncryptionKeyInfo getPrivateKey(String keyName, Map metadata) { + throw new UnsupportedOperationException(); + } + } +} \ No newline at end of file diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/state/BKStateStoreImplTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/state/BKStateStoreImplTest.java index 1d35f3dfe5be1..7696c71d5d1e6 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/state/BKStateStoreImplTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/state/BKStateStoreImplTest.java @@ -35,7 +35,9 @@ import org.apache.bookkeeper.api.kv.Table; import org.apache.bookkeeper.api.kv.options.Options; import org.apache.bookkeeper.api.kv.result.DeleteResult; +import org.apache.bookkeeper.api.kv.result.KeyValue; import org.apache.bookkeeper.common.concurrent.FutureUtils; +import org.apache.pulsar.functions.api.state.StateValue; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -114,6 +116,24 @@ public void testGetValue() throws Exception { ); } + @Test + public void testGetStateValue() throws Exception { + KeyValue returnedKeyValue = mock(KeyValue.class); + ByteBuf returnedValue = Unpooled.copiedBuffer("test-value", UTF_8); + when(returnedKeyValue.value()).thenReturn(returnedValue); + when(returnedKeyValue.version()).thenReturn(1l); + when(returnedKeyValue.isNumber()).thenReturn(false); + when(mockTable.getKv(any(ByteBuf.class))) + .thenReturn(FutureUtils.value(returnedKeyValue)); + StateValue result = stateContext.getStateValue("test-key"); + assertEquals("test-value", new String(result.getValue(), UTF_8)); + assertEquals(1l, result.getVersion().longValue()); + assertEquals(false, result.getIsNumber().booleanValue()); + verify(mockTable, times(1)).getKv( + eq(Unpooled.copiedBuffer("test-key", UTF_8)) + ); + } + @Test public void testGetAmount() throws Exception { when(mockTable.getNumber(any(ByteBuf.class))) @@ -132,6 +152,12 @@ public void testGetKeyNotPresent() throws Exception { assertTrue(result != null); assertEquals(result.get(), null); + when(mockTable.getKv(any(ByteBuf.class))) + .thenReturn(FutureUtils.value(null)); + CompletableFuture stateValueResult = stateContext.getStateValueAsync("test-key"); + assertTrue(stateValueResult != null); + assertEquals(stateValueResult.get(), null); + } } diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreImplTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreImplTest.java index 3b8cb02c3bb26..4d1a1f73fe6d2 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreImplTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/state/PulsarMetadataStateStoreImplTest.java @@ -24,6 +24,7 @@ import static org.testng.Assert.assertTrue; import java.nio.ByteBuffer; import java.util.concurrent.CompletableFuture; +import org.apache.pulsar.functions.api.state.StateValue; import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.MetadataStoreConfig; @@ -101,6 +102,10 @@ public void testGetKeyNotPresent() throws Exception { CompletableFuture result = stateContext.getAsync("test-key"); assertTrue(result != null); assertEquals(result.get(), null); + + CompletableFuture stateValueResult = stateContext.getStateValueAsync("test-key"); + assertTrue(stateValueResult != null); + assertEquals(stateValueResult.get(), null); } } diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/sink/PulsarSinkTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/sink/PulsarSinkTest.java index fdac39512cc24..8a946a3f7571b 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/sink/PulsarSinkTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/sink/PulsarSinkTest.java @@ -18,17 +18,18 @@ */ package org.apache.pulsar.functions.sink; +import static org.apache.pulsar.functions.instance.ProducerCache.CacheArea.SINK_RECORD_CACHE; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNull; @@ -36,7 +37,6 @@ import static org.testng.Assert.fail; import java.io.IOException; import java.util.HashMap; -import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; import lombok.Getter; @@ -56,6 +56,7 @@ import org.apache.pulsar.client.api.schema.GenericSchema; import org.apache.pulsar.client.api.schema.RecordSchemaBuilder; import org.apache.pulsar.client.api.schema.SchemaBuilder; +import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; import org.apache.pulsar.common.functions.FunctionConfig; @@ -63,12 +64,14 @@ import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.functions.api.SerDe; +import org.apache.pulsar.functions.instance.ProducerCache; import org.apache.pulsar.functions.instance.SinkRecord; import org.apache.pulsar.functions.instance.stats.ComponentStatsManager; import org.apache.pulsar.functions.sink.PulsarSink.PulsarSinkProcessorBase; import org.apache.pulsar.functions.source.TopicSchema; import org.apache.pulsar.io.core.SinkContext; import org.testng.Assert; +import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -95,6 +98,8 @@ public byte[] serialize(String input) { */ private static PulsarClientImpl getPulsarClient() throws PulsarClientException { PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(pulsarClient.getCnxPool()).thenReturn(connectionPool); ConsumerBuilder consumerBuilder = mock(ConsumerBuilder.class); doReturn(consumerBuilder).when(consumerBuilder).topics(anyList()); doReturn(consumerBuilder).when(consumerBuilder).subscriptionName(anyString()); @@ -128,6 +133,7 @@ private static PulsarClientImpl getPulsarClient() throws PulsarClientException { doReturn(producer).when(producerBuilder).create(); doReturn(typedMessageBuilder).when(producer).newMessage(); doReturn(typedMessageBuilder).when(producer).newMessage(any(Schema.class)); + doReturn(CompletableFuture.completedFuture(null)).when(producer).flushAsync(); doReturn(producerBuilder).when(pulsarClient).newProducer(); doReturn(producerBuilder).when(pulsarClient).newProducer(any()); @@ -135,9 +141,17 @@ private static PulsarClientImpl getPulsarClient() throws PulsarClientException { return pulsarClient; } - @BeforeMethod + ProducerCache producerCache; + + @BeforeMethod(alwaysRun = true) public void setup() { + producerCache = new ProducerCache(); + } + @AfterMethod(alwaysRun = true) + public void tearDown() { + producerCache.close(); + producerCache = null; } private static PulsarSinkConfig getPulsarConfigs() { @@ -178,7 +192,7 @@ public void testVoidOutputClasses() throws Exception { pulsarConfig.setTypeClassName(Void.class.getName()); PulsarSink pulsarSink = new PulsarSink(getPulsarClient(), pulsarConfig, new HashMap<>(), mock(ComponentStatsManager.class), - Thread.currentThread().getContextClassLoader()); + Thread.currentThread().getContextClassLoader(), producerCache); try { Schema schema = pulsarSink.initializeSchema(); @@ -198,7 +212,7 @@ public void testInconsistentOutputType() throws IOException { pulsarConfig.setSerdeClassName(TestSerDe.class.getName()); PulsarSink pulsarSink = new PulsarSink(getPulsarClient(), pulsarConfig, new HashMap<>(), mock(ComponentStatsManager.class), - Thread.currentThread().getContextClassLoader()); + Thread.currentThread().getContextClassLoader(), producerCache); try { pulsarSink.initializeSchema(); fail("Should fail constructing java instance if function type is inconsistent with serde type"); @@ -223,7 +237,7 @@ public void testDefaultSerDe() throws PulsarClientException { pulsarConfig.setTypeClassName(String.class.getName()); PulsarSink pulsarSink = new PulsarSink(getPulsarClient(), pulsarConfig, new HashMap<>(), mock(ComponentStatsManager.class), - Thread.currentThread().getContextClassLoader()); + Thread.currentThread().getContextClassLoader(), producerCache); try { pulsarSink.initializeSchema(); @@ -244,7 +258,7 @@ public void testExplicitDefaultSerDe() throws PulsarClientException { pulsarConfig.setSerdeClassName(TopicSchema.DEFAULT_SERDE); PulsarSink pulsarSink = new PulsarSink(getPulsarClient(), pulsarConfig, new HashMap<>(), mock(ComponentStatsManager.class), - Thread.currentThread().getContextClassLoader()); + Thread.currentThread().getContextClassLoader(), producerCache); try { pulsarSink.initializeSchema(); @@ -262,7 +276,7 @@ public void testComplexOuputType() throws PulsarClientException { pulsarConfig.setSerdeClassName(ComplexSerDe.class.getName()); PulsarSink pulsarSink = new PulsarSink(getPulsarClient(), pulsarConfig, new HashMap<>(), mock(ComponentStatsManager.class), - Thread.currentThread().getContextClassLoader()); + Thread.currentThread().getContextClassLoader(), producerCache); try { pulsarSink.initializeSchema(); @@ -282,7 +296,7 @@ public void testInitializeSchema() throws Exception { pulsarSinkConfig.setTypeClassName(GenericRecord.class.getName()); PulsarSink sink = new PulsarSink( pulsarClient, pulsarSinkConfig, new HashMap<>(), mock(ComponentStatsManager.class), - Thread.currentThread().getContextClassLoader()); + Thread.currentThread().getContextClassLoader(), producerCache); Schema schema = sink.initializeSchema(); assertTrue(schema instanceof AutoConsumeSchema); @@ -291,7 +305,7 @@ public void testInitializeSchema() throws Exception { pulsarSinkConfig.setTypeClassName(GenericRecord.class.getName()); sink = new PulsarSink( pulsarClient, pulsarSinkConfig, new HashMap<>(), mock(ComponentStatsManager.class), - Thread.currentThread().getContextClassLoader()); + Thread.currentThread().getContextClassLoader(), producerCache); schema = sink.initializeSchema(); assertTrue(schema instanceof AutoConsumeSchema); @@ -302,7 +316,7 @@ public void testInitializeSchema() throws Exception { pulsarSinkConfig.setTypeClassName(GenericRecord.class.getName()); sink = new PulsarSink( pulsarClient, pulsarSinkConfig, new HashMap<>(), mock(ComponentStatsManager.class), - Thread.currentThread().getContextClassLoader()); + Thread.currentThread().getContextClassLoader(), producerCache); schema = sink.initializeSchema(); assertTrue(schema instanceof AutoConsumeSchema); @@ -313,7 +327,7 @@ public void testInitializeSchema() throws Exception { pulsarSinkConfig.setTypeClassName(GenericRecord.class.getName()); sink = new PulsarSink( pulsarClient, pulsarSinkConfig, new HashMap<>(), mock(ComponentStatsManager.class), - Thread.currentThread().getContextClassLoader()); + Thread.currentThread().getContextClassLoader(), producerCache); schema = sink.initializeSchema(); assertTrue(schema instanceof AutoConsumeSchema); @@ -323,7 +337,7 @@ public void testInitializeSchema() throws Exception { pulsarSinkConfig.setTypeClassName(GenericRecord.class.getName()); sink = new PulsarSink( pulsarClient, pulsarSinkConfig, new HashMap<>(), mock(ComponentStatsManager.class), - Thread.currentThread().getContextClassLoader()); + Thread.currentThread().getContextClassLoader(), producerCache); schema = sink.initializeSchema(); assertTrue(schema instanceof AutoConsumeSchema); } @@ -340,9 +354,12 @@ public void testSinkAndMessageRouting() throws Exception { /** test MANUAL **/ pulsarClient = getPulsarClient(); pulsarConfig.setProcessingGuarantees(ProcessingGuarantees.MANUAL); - PulsarSink pulsarSink = new PulsarSink(pulsarClient, pulsarConfig, new HashMap<>(), mock(ComponentStatsManager.class), Thread.currentThread().getContextClassLoader()); + PulsarSink pulsarSink = + new PulsarSink(pulsarClient, pulsarConfig, new HashMap<>(), mock(ComponentStatsManager.class), + Thread.currentThread().getContextClassLoader(), producerCache); pulsarSink.open(new HashMap<>(), mock(SinkContext.class)); + verify(pulsarClient.newProducer(), times(1)).topic(defaultTopic); for (String topic : topics) { @@ -366,23 +383,19 @@ public Optional getDestinationTopic() { PulsarSink.PulsarSinkManualProcessor pulsarSinkManualProcessor = (PulsarSink.PulsarSinkManualProcessor) pulsarSink.pulsarSinkProcessor; if (topic != null) { - Assert.assertTrue(pulsarSinkManualProcessor.publishProducers.containsKey(topic)); + Assert.assertTrue(producerCache.containsKey(SINK_RECORD_CACHE, topic)); } else { - Assert.assertTrue(pulsarSinkManualProcessor.publishProducers.containsKey(defaultTopic)); + Assert.assertTrue(producerCache.containsKey(SINK_RECORD_CACHE, defaultTopic)); } - verify(pulsarClient.newProducer(), times(1)).topic(argThat(otherTopic -> { - if (topic != null) { - return topic.equals(otherTopic); - } else { - return defaultTopic.equals(otherTopic); - } - })); + String actualTopic = topic != null ? topic : defaultTopic; + verify(pulsarClient.newProducer(), times(1)).topic(actualTopic); } /** test At-least-once **/ pulsarClient = getPulsarClient(); pulsarConfig.setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE); - pulsarSink = new PulsarSink(pulsarClient, pulsarConfig, new HashMap<>(), mock(ComponentStatsManager.class), Thread.currentThread().getContextClassLoader()); + pulsarSink = new PulsarSink(pulsarClient, pulsarConfig, new HashMap<>(), mock(ComponentStatsManager.class), + Thread.currentThread().getContextClassLoader(), producerCache); pulsarSink.open(new HashMap<>(), mock(SinkContext.class)); @@ -406,24 +419,17 @@ public Optional getDestinationTopic() { PulsarSink.PulsarSinkAtLeastOnceProcessor pulsarSinkAtLeastOnceProcessor = (PulsarSink.PulsarSinkAtLeastOnceProcessor) pulsarSink.pulsarSinkProcessor; if (topic != null) { - Assert.assertTrue(pulsarSinkAtLeastOnceProcessor.publishProducers.containsKey(topic)); + Assert.assertTrue(producerCache.containsKey(SINK_RECORD_CACHE, topic)); } else { - Assert.assertTrue(pulsarSinkAtLeastOnceProcessor.publishProducers.containsKey(defaultTopic)); + Assert.assertTrue(producerCache.containsKey(SINK_RECORD_CACHE, defaultTopic)); } - verify(pulsarClient.newProducer(), times(1)).topic(argThat(otherTopic -> { - if (topic != null) { - return topic.equals(otherTopic); - } else { - return defaultTopic.equals(otherTopic); - } - })); } /** test At-most-once **/ pulsarClient = getPulsarClient(); pulsarConfig.setProcessingGuarantees(ProcessingGuarantees.ATMOST_ONCE); pulsarSink = new PulsarSink(pulsarClient, pulsarConfig, new HashMap<>(), mock(ComponentStatsManager.class), - Thread.currentThread().getContextClassLoader()); + Thread.currentThread().getContextClassLoader(), producerCache); pulsarSink.open(new HashMap<>(), mock(SinkContext.class)); @@ -453,20 +459,17 @@ public Optional getDestinationTopic() { PulsarSink.PulsarSinkAtMostOnceProcessor pulsarSinkAtLeastOnceProcessor = (PulsarSink.PulsarSinkAtMostOnceProcessor) pulsarSink.pulsarSinkProcessor; if (topic != null) { - Assert.assertTrue(pulsarSinkAtLeastOnceProcessor.publishProducers.containsKey(topic)); + Assert.assertTrue(producerCache.containsKey(SINK_RECORD_CACHE, topic)); } else { - Assert.assertTrue(pulsarSinkAtLeastOnceProcessor.publishProducers.containsKey(defaultTopic)); + Assert.assertTrue(producerCache.containsKey(SINK_RECORD_CACHE, defaultTopic)); } - verify(pulsarClient.newProducer(), times(1)).topic(argThat(o -> { - return getTopicEquals(o, topic, defaultTopic); - })); } /** test Effectively-once **/ pulsarClient = getPulsarClient(); pulsarConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE); pulsarSink = new PulsarSink(pulsarClient, pulsarConfig, new HashMap<>(), mock(ComponentStatsManager.class), - Thread.currentThread().getContextClassLoader()); + Thread.currentThread().getContextClassLoader(), producerCache); pulsarSink.open(new HashMap<>(), mock(SinkContext.class)); @@ -516,23 +519,19 @@ public Optional getRecordSequence() { PulsarSink.PulsarSinkEffectivelyOnceProcessor pulsarSinkEffectivelyOnceProcessor = (PulsarSink.PulsarSinkEffectivelyOnceProcessor) pulsarSink.pulsarSinkProcessor; if (topic != null) { - Assert.assertTrue(pulsarSinkEffectivelyOnceProcessor.publishProducers - .containsKey(String.format("%s-%s-id-1", topic, topic))); + Assert.assertTrue(producerCache + .containsKey(SINK_RECORD_CACHE, topic, String.format("%s-id-1", topic))); } else { - Assert.assertTrue(pulsarSinkEffectivelyOnceProcessor.publishProducers - .containsKey(String.format("%s-%s-id-1", defaultTopic, defaultTopic))); + Assert.assertTrue(producerCache + .containsKey(SINK_RECORD_CACHE, + defaultTopic, String.format("%s-id-1", defaultTopic) + )); } - verify(pulsarClient.newProducer(), times(1)).topic(argThat(o -> { - return getTopicEquals(o, topic, defaultTopic); - })); - verify(pulsarClient.newProducer(), times(1)).producerName(argThat(o -> { - if (topic != null) { - return String.format("%s-id-1", topic).equals(o); - } else { - return String.format("%s-id-1", defaultTopic).equals(o); - } - })); + String expectedTopicName = topic != null ? topic : defaultTopic; + verify(pulsarClient.newProducer(), times(1)).topic(expectedTopicName); + String expectedProducerName = String.format("%s-id-1", expectedTopicName); + verify(pulsarClient.newProducer(), times(1)).producerName(expectedProducerName); } } @@ -562,7 +561,7 @@ private void testWriteGenericRecords(ProcessingGuarantees guarantees) throws Exc PulsarClient client = getPulsarClient(); PulsarSink pulsarSink = new PulsarSink( client, sinkConfig, new HashMap<>(), mock(ComponentStatsManager.class), - Thread.currentThread().getContextClassLoader()); + Thread.currentThread().getContextClassLoader(), producerCache); pulsarSink.open(new HashMap<>(), mock(SinkContext.class)); @@ -574,7 +573,7 @@ private void testWriteGenericRecords(ProcessingGuarantees guarantees) throws Exc assertTrue(pulsarSink.pulsarSinkProcessor instanceof PulsarSink.PulsarSinkEffectivelyOnceProcessor); } PulsarSinkProcessorBase processor = (PulsarSinkProcessorBase) pulsarSink.pulsarSinkProcessor; - assertFalse(processor.publishProducers.containsKey(defaultTopic)); + assertFalse(producerCache.containsKey(SINK_RECORD_CACHE, defaultTopic)); String[] topics = {"topic-1", "topic-2", "topic-3"}; for (String topic : topics) { @@ -621,17 +620,15 @@ public Optional getRecordSequence() { pulsarSink.write(record); if (ProcessingGuarantees.EFFECTIVELY_ONCE == guarantees) { - assertTrue(processor.publishProducers.containsKey(String.format("%s-%s-id-1", topic, topic))); + assertTrue(producerCache.containsKey(SINK_RECORD_CACHE, + topic, String.format("%s-id-1", topic) + )); } else { - assertTrue(processor.publishProducers.containsKey(topic)); + assertTrue(producerCache.containsKey(SINK_RECORD_CACHE, topic)); } - verify(client.newProducer(), times(1)) - .topic(argThat( - otherTopic -> topic != null ? topic.equals(otherTopic) : defaultTopic.equals(otherTopic))); - - verify(client, times(1)) - .newProducer(argThat( - otherSchema -> Objects.equals(otherSchema, schema))); + String expectedTopicName = topic != null ? topic : defaultTopic; + verify(client.newProducer(), times(1)).topic(expectedTopicName); + verify(client, times(1)).newProducer(schema); } } @@ -642,13 +639,4 @@ private Optional getTopicOptional(String topic) { return Optional.empty(); } } - - private boolean getTopicEquals(Object o, String topic, String defaultTopic) { - if (topic != null) { - return topic.equals(o); - } else { - return defaultTopic.equals(o); - } - } - } diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/PulsarSourceTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/PulsarSourceTest.java index 91e4c06fe5b49..5d6e4a3dc75e7 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/PulsarSourceTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/PulsarSourceTest.java @@ -21,6 +21,8 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.testng.Assert.assertSame; import static org.testng.AssertJUnit.assertEquals; import static org.testng.AssertJUnit.assertTrue; @@ -44,6 +46,7 @@ import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.functions.ConsumerConfig; @@ -105,6 +108,8 @@ public static Object[] getPulsarSourceImpls() { */ private static PulsarClientImpl getPulsarClient() throws PulsarClientException { PulsarClientImpl pulsarClient = Mockito.mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(pulsarClient.getCnxPool()).thenReturn(connectionPool); ConsumerBuilder goodConsumerBuilder = Mockito.mock(ConsumerBuilder.class); ConsumerBuilder badConsumerBuilder = Mockito.mock(ConsumerBuilder.class); Mockito.doReturn(goodConsumerBuilder).when(goodConsumerBuilder) diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/windowing/WaterMarkEventGeneratorTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/windowing/WaterMarkEventGeneratorTest.java index ce3109b852ec2..162c9ad51ced9 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/windowing/WaterMarkEventGeneratorTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/windowing/WaterMarkEventGeneratorTest.java @@ -63,7 +63,7 @@ public void add(Event event) { @AfterMethod(alwaysRun = true) public void tearDown() { -// waterMarkEventGenerator.shutdown(); + waterMarkEventGenerator.shutdown(); eventList.clear(); } diff --git a/pulsar-functions/java-examples-builtin/pom.xml b/pulsar-functions/java-examples-builtin/pom.xml index 273fe3a0d7c4a..dbb8ebcc6ced3 100644 --- a/pulsar-functions/java-examples-builtin/pom.xml +++ b/pulsar-functions/java-examples-builtin/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-functions-api-examples-builtin diff --git a/pulsar-functions/java-examples/pom.xml b/pulsar-functions/java-examples/pom.xml index b383295423f40..e16655feab6a1 100644 --- a/pulsar-functions/java-examples/pom.xml +++ b/pulsar-functions/java-examples/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-functions-api-examples diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml index 37e9d40249d49..770493ef306db 100644 --- a/pulsar-functions/localrun-shaded/pom.xml +++ b/pulsar-functions/localrun-shaded/pom.xml @@ -26,8 +26,7 @@ org.apache.pulsar pulsar-functions - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-functions-local-runner @@ -133,7 +132,7 @@ org.rocksdb:* org.eclipse.jetty*:* org.apache.avro:avro - com.beust:* + info.picocli:* net.jodah:* io.airlift:* com.yahoo.datasketches:* @@ -385,8 +384,8 @@ org.apache.pulsar.shaded.com.yahoo.sketches - com.beust - org.apache.pulsar.functions.runtime.shaded.com.beust + info.picocli + org.apache.pulsar.functions.runtime.shaded.info.picocli diff --git a/pulsar-functions/localrun/pom.xml b/pulsar-functions/localrun/pom.xml index 5779695eae8e4..921c80106b077 100644 --- a/pulsar-functions/localrun/pom.xml +++ b/pulsar-functions/localrun/pom.xml @@ -26,8 +26,7 @@ org.apache.pulsar pulsar-functions - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-functions-local-runner-original @@ -43,7 +42,7 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl org.apache.logging.log4j diff --git a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java index ed9b0af3b43d8..3b1c86a68c285 100644 --- a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java +++ b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java @@ -20,9 +20,6 @@ import static org.apache.commons.lang3.StringUtils.isNotEmpty; import static org.apache.pulsar.common.functions.Utils.inferMissingArguments; -import com.beust.jcommander.IStringConverter; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonParser; @@ -52,7 +49,9 @@ import lombok.Value; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.common.functions.FunctionConfig; +import org.apache.pulsar.common.functions.FunctionDefinition; import org.apache.pulsar.common.functions.Utils; +import org.apache.pulsar.common.io.ConnectorDefinition; import org.apache.pulsar.common.io.SinkConfig; import org.apache.pulsar.common.io.SourceConfig; import org.apache.pulsar.common.nar.FileUtils; @@ -75,13 +74,20 @@ import org.apache.pulsar.functions.secretsproviderconfigurator.SecretsProviderConfigurator; import org.apache.pulsar.functions.utils.FunctionCommon; import org.apache.pulsar.functions.utils.FunctionConfigUtils; +import org.apache.pulsar.functions.utils.FunctionRuntimeCommon; +import org.apache.pulsar.functions.utils.LoadedFunctionPackage; import org.apache.pulsar.functions.utils.SinkConfigUtils; import org.apache.pulsar.functions.utils.SourceConfigUtils; +import org.apache.pulsar.functions.utils.ValidatableFunctionPackage; import org.apache.pulsar.functions.utils.functioncache.FunctionCacheEntry; import org.apache.pulsar.functions.utils.functions.FunctionArchive; import org.apache.pulsar.functions.utils.functions.FunctionUtils; import org.apache.pulsar.functions.utils.io.Connector; import org.apache.pulsar.functions.utils.io.ConnectorUtils; +import picocli.CommandLine; +import picocli.CommandLine.ITypeConverter; +import picocli.CommandLine.Option; +import picocli.CommandLine.TypeConversionException; @Slf4j public class LocalRunner implements AutoCloseable { @@ -110,95 +116,95 @@ private static class UserCodeClassLoader { boolean classLoaderCreated; } - public static class FunctionConfigConverter implements IStringConverter { + public static class FunctionConfigConverter implements ITypeConverter { @Override public FunctionConfig convert(String value) { try { return ObjectMapperFactory.getMapper().reader().readValue(value, FunctionConfig.class); } catch (IOException e) { - throw new RuntimeException("Failed to parse function config:", e); + throw new TypeConversionException(e.getMessage()); } } } - public static class SourceConfigConverter implements IStringConverter { + public static class SourceConfigConverter implements ITypeConverter { @Override public SourceConfig convert(String value) { try { return ObjectMapperFactory.getMapper().reader().readValue(value, SourceConfig.class); } catch (IOException e) { - throw new RuntimeException("Failed to parse source config:", e); + throw new TypeConversionException(e.getMessage()); } } } - public static class SinkConfigConverter implements IStringConverter { + public static class SinkConfigConverter implements ITypeConverter { @Override public SinkConfig convert(String value) { try { return ObjectMapperFactory.getMapper().reader().readValue(value, SinkConfig.class); } catch (IOException e) { - throw new RuntimeException("Failed to parse sink config:", e); + throw new TypeConversionException(e.getMessage()); } } } - public static class RuntimeConverter implements IStringConverter { + public static class RuntimeConverter implements ITypeConverter { @Override public RuntimeEnv convert(String value) { return RuntimeEnv.valueOf(value); } } - @Parameter(names = "--functionConfig", description = "The json representation of FunctionConfig", + @Option(names = "--functionConfig", description = "The json representation of FunctionConfig", hidden = true, converter = FunctionConfigConverter.class) protected FunctionConfig functionConfig; - @Parameter(names = "--sourceConfig", description = "The json representation of SourceConfig", + @Option(names = "--sourceConfig", description = "The json representation of SourceConfig", hidden = true, converter = SourceConfigConverter.class) protected SourceConfig sourceConfig; - @Parameter(names = "--sinkConfig", description = "The json representation of SinkConfig", + @Option(names = "--sinkConfig", description = "The json representation of SinkConfig", hidden = true, converter = SinkConfigConverter.class) protected SinkConfig sinkConfig; - @Parameter(names = "--stateStorageImplClass", description = "The implemenatation class " + @Option(names = "--stateStorageImplClass", description = "The implemenatation class " + "state storage service (by default Apache BookKeeper)", hidden = true, required = false) protected String stateStorageImplClass; - @Parameter(names = "--stateStorageServiceUrl", description = "The URL for the state storage service " + @Option(names = "--stateStorageServiceUrl", description = "The URL for the state storage service " + "(by default Apache BookKeeper)", hidden = true) protected String stateStorageServiceUrl; - @Parameter(names = "--brokerServiceUrl", description = "The URL for the Pulsar broker", hidden = true) + @Option(names = "--brokerServiceUrl", description = "The URL for the Pulsar broker", hidden = true) protected String brokerServiceUrl; - @Parameter(names = "--webServiceUrl", description = "The URL for the Pulsar web service", hidden = true) + @Option(names = "--webServiceUrl", description = "The URL for the Pulsar web service", hidden = true) protected String webServiceUrl = null; - @Parameter(names = "--clientAuthPlugin", description = "Client authentication plugin using which " + @Option(names = "--clientAuthPlugin", description = "Client authentication plugin using which " + "function-process can connect to broker", hidden = true) protected String clientAuthPlugin; - @Parameter(names = "--clientAuthParams", description = "Client authentication param", hidden = true) + @Option(names = "--clientAuthParams", description = "Client authentication param", hidden = true) protected String clientAuthParams; - @Parameter(names = "--useTls", description = "Use tls connection\n", hidden = true, arity = 1) + @Option(names = "--useTls", description = "Use tls connection\n", hidden = true, arity = "1") protected boolean useTls; - @Parameter(names = "--tlsAllowInsecureConnection", description = "Allow insecure tls connection\n", - hidden = true, arity = 1) + @Option(names = "--tlsAllowInsecureConnection", description = "Allow insecure tls connection\n", + hidden = true, arity = "1") protected boolean tlsAllowInsecureConnection; - @Parameter(names = "--tlsHostNameVerificationEnabled", description = "Enable hostname verification", hidden = true - , arity = 1) + @Option(names = "--tlsHostNameVerificationEnabled", description = "Enable hostname verification", hidden = true + , arity = "1") protected boolean tlsHostNameVerificationEnabled; - @Parameter(names = "--tlsTrustCertFilePath", description = "tls trust cert file path", hidden = true) + @Option(names = "--tlsTrustCertFilePath", description = "tls trust cert file path", hidden = true) protected String tlsTrustCertFilePath; - @Parameter(names = "--instanceIdOffset", description = "Start the instanceIds from this offset", hidden = true) + @Option(names = "--instanceIdOffset", description = "Start the instanceIds from this offset", hidden = true) protected int instanceIdOffset = 0; - @Parameter(names = "--runtime", description = "Function runtime to use (Thread/Process)", hidden = true, + @Option(names = "--runtime", description = "Function runtime to use (Thread/Process)", hidden = true, converter = RuntimeConverter.class) protected RuntimeEnv runtimeEnv; - @Parameter(names = "--secretsProviderClassName", + @Option(names = "--secretsProviderClassName", description = "Whats the classname of secrets provider", hidden = true) protected String secretsProviderClassName; - @Parameter(names = "--secretsProviderConfig", + @Option(names = "--secretsProviderConfig", description = "Whats the config for the secrets provider", hidden = true) protected String secretsProviderConfig; - @Parameter(names = "--metricsPortStart", description = "The starting port range for metrics server. When running " + @Option(names = "--metricsPortStart", description = "The starting port range for metrics server. When running " + "instances as threads, one metrics server is used to host the stats for all instances.", hidden = true) protected Integer metricsPortStart; - @Parameter(names = "--exitOnError", description = "The starting port range for metrics server. When running " + @Option(names = "--exitOnError", description = "The starting port range for metrics server. When running " + "instances as threads, one metrics server is used to host the stats for all instances.", hidden = true) protected boolean exitOnError; @@ -207,11 +213,10 @@ public RuntimeEnv convert(String value) { public static void main(String[] args) throws Exception { LocalRunner localRunner = LocalRunner.builder().build(); - JCommander jcommander = new JCommander(localRunner); - jcommander.setProgramName("LocalRunner"); + CommandLine jcommander = new CommandLine(localRunner); + jcommander.setCommandName("LocalRunner"); - // parse args by JCommander - jcommander.parse(args); + jcommander.parseArgs(args); try { localRunner.start(true); } catch (Exception e) { @@ -357,9 +362,12 @@ public void start(boolean blocking) throws Exception { userCodeFile = functionConfig.getJar(); userCodeClassLoader = extractClassLoader( userCodeFile, ComponentType.FUNCTION, functionConfig.getClassName()); + ValidatableFunctionPackage validatableFunctionPackage = + new LoadedFunctionPackage(getCurrentOrUserCodeClassLoader(), + FunctionDefinition.class); functionDetails = FunctionConfigUtils.convert( functionConfig, - FunctionConfigUtils.validateJavaFunction(functionConfig, getCurrentOrUserCodeClassLoader())); + FunctionConfigUtils.validateJavaFunction(functionConfig, validatableFunctionPackage)); } else if (functionConfig.getRuntime() == FunctionConfig.Runtime.GO) { userCodeFile = functionConfig.getGo(); } else if (functionConfig.getRuntime() == FunctionConfig.Runtime.PYTHON) { @@ -369,7 +377,10 @@ public void start(boolean blocking) throws Exception { } if (functionDetails == null) { - functionDetails = FunctionConfigUtils.convert(functionConfig, getCurrentOrUserCodeClassLoader()); + ValidatableFunctionPackage validatableFunctionPackage = + new LoadedFunctionPackage(getCurrentOrUserCodeClassLoader(), + FunctionDefinition.class); + functionDetails = FunctionConfigUtils.convert(functionConfig, validatableFunctionPackage); } } else if (sourceConfig != null) { inferMissingArguments(sourceConfig); @@ -377,9 +388,10 @@ public void start(boolean blocking) throws Exception { parallelism = sourceConfig.getParallelism(); userCodeClassLoader = extractClassLoader( userCodeFile, ComponentType.SOURCE, sourceConfig.getClassName()); - functionDetails = SourceConfigUtils.convert( - sourceConfig, - SourceConfigUtils.validateAndExtractDetails(sourceConfig, getCurrentOrUserCodeClassLoader(), true)); + ValidatableFunctionPackage validatableFunctionPackage = + new LoadedFunctionPackage(getCurrentOrUserCodeClassLoader(), ConnectorDefinition.class); + functionDetails = SourceConfigUtils.convert(sourceConfig, + SourceConfigUtils.validateAndExtractDetails(sourceConfig, validatableFunctionPackage, true)); } else if (sinkConfig != null) { inferMissingArguments(sinkConfig); userCodeFile = sinkConfig.getArchive(); @@ -387,6 +399,8 @@ public void start(boolean blocking) throws Exception { parallelism = sinkConfig.getParallelism(); userCodeClassLoader = extractClassLoader( userCodeFile, ComponentType.SINK, sinkConfig.getClassName()); + ValidatableFunctionPackage validatableFunctionPackage = + new LoadedFunctionPackage(getCurrentOrUserCodeClassLoader(), ConnectorDefinition.class); if (isNotEmpty(sinkConfig.getTransformFunction())) { transformFunctionCodeClassLoader = extractClassLoader( sinkConfig.getTransformFunction(), @@ -395,16 +409,19 @@ public void start(boolean blocking) throws Exception { } ClassLoader functionClassLoader = null; + ValidatableFunctionPackage validatableTransformFunction = null; if (transformFunctionCodeClassLoader != null) { functionClassLoader = transformFunctionCodeClassLoader.getClassLoader() == null ? Thread.currentThread().getContextClassLoader() : transformFunctionCodeClassLoader.getClassLoader(); + validatableTransformFunction = + new LoadedFunctionPackage(functionClassLoader, FunctionDefinition.class); } functionDetails = SinkConfigUtils.convert( sinkConfig, - SinkConfigUtils.validateAndExtractDetails(sinkConfig, getCurrentOrUserCodeClassLoader(), - functionClassLoader, true)); + SinkConfigUtils.validateAndExtractDetails(sinkConfig, validatableFunctionPackage, + validatableTransformFunction, true)); } else { throw new IllegalArgumentException("Must specify Function, Source or Sink config"); } @@ -472,7 +489,7 @@ private UserCodeClassLoader extractClassLoader(String userCodeFile, ComponentTyp if (classLoader == null) { if (userCodeFile != null && Utils.isFunctionPackageUrlSupported(userCodeFile)) { File file = FunctionCommon.extractFileFromPkgURL(userCodeFile); - classLoader = FunctionCommon.getClassLoaderFromPackage( + classLoader = FunctionRuntimeCommon.getClassLoaderFromPackage( componentType, className, file, narExtractionDirectory); classLoaderCreated = true; } else if (userCodeFile != null) { @@ -494,7 +511,7 @@ private UserCodeClassLoader extractClassLoader(String userCodeFile, ComponentTyp } throw new RuntimeException(errorMsg + " (" + userCodeFile + ") does not exist"); } - classLoader = FunctionCommon.getClassLoaderFromPackage( + classLoader = FunctionRuntimeCommon.getClassLoaderFromPackage( componentType, className, file, narExtractionDirectory); classLoaderCreated = true; } else { @@ -713,7 +730,7 @@ private ClassLoader isBuiltInFunction(String functionType) throws IOException { FunctionArchive function = functions.get(functionName); if (function != null && function.getFunctionDefinition().getFunctionClass() != null) { // Function type is a valid built-in type. - return function.getClassLoader(); + return function.getFunctionPackage().getClassLoader(); } else { return null; } @@ -727,7 +744,7 @@ private ClassLoader isBuiltInSource(String sourceType) throws IOException { Connector connector = connectors.get(source); if (connector != null && connector.getConnectorDefinition().getSourceClass() != null) { // Source type is a valid built-in connector type. - return connector.getClassLoader(); + return connector.getConnectorFunctionPackage().getClassLoader(); } else { return null; } @@ -741,18 +758,18 @@ private ClassLoader isBuiltInSink(String sinkType) throws IOException { Connector connector = connectors.get(sink); if (connector != null && connector.getConnectorDefinition().getSinkClass() != null) { // Sink type is a valid built-in connector type - return connector.getClassLoader(); + return connector.getConnectorFunctionPackage().getClassLoader(); } else { return null; } } private TreeMap getFunctions() throws IOException { - return FunctionUtils.searchForFunctions(functionsDir); + return FunctionUtils.searchForFunctions(functionsDir, narExtractionDirectory, true); } private TreeMap getConnectors() throws IOException { - return ConnectorUtils.searchForConnectors(connectorsDir, narExtractionDirectory); + return ConnectorUtils.searchForConnectors(connectorsDir, narExtractionDirectory, true); } private SecretsProviderConfigurator getSecretsProviderConfigurator() { diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index 1d9973a4d41a8..af7fe9ad4694d 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-functions diff --git a/pulsar-functions/proto/pom.xml b/pulsar-functions/proto/pom.xml index b84b2d9181a38..12b6f5ef8dc0d 100644 --- a/pulsar-functions/proto/pom.xml +++ b/pulsar-functions/proto/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-functions - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-functions-proto diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml index 55e4009db077e..df40548a0f13e 100644 --- a/pulsar-functions/runtime-all/pom.xml +++ b/pulsar-functions/runtime-all/pom.xml @@ -26,8 +26,7 @@ org.apache.pulsar pulsar-functions - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT - - io.grpc - grpc-netty - ${grpc.version} - - - - io.dropwizard.metrics - metrics-jvm - ${metrics.version} - - + + + + io.netty + netty-bom + ${netty.version} + pom + import + + + io.grpc + grpc-bom + ${grpc.version} + pom + import + + + io.dropwizard.metrics + metrics-jvm + ${metrics.version} + + + + diff --git a/pulsar-io/alluxio/src/main/java/org/apache/pulsar/io/alluxio/sink/AlluxioSink.java b/pulsar-io/alluxio/src/main/java/org/apache/pulsar/io/alluxio/sink/AlluxioSink.java index 413f05e0e17c5..3b72dc9666b78 100644 --- a/pulsar-io/alluxio/src/main/java/org/apache/pulsar/io/alluxio/sink/AlluxioSink.java +++ b/pulsar-io/alluxio/src/main/java/org/apache/pulsar/io/alluxio/sink/AlluxioSink.java @@ -22,12 +22,13 @@ import alluxio.client.WriteType; import alluxio.client.file.FileOutStream; import alluxio.client.file.FileSystem; +import alluxio.conf.Configuration; import alluxio.conf.InstancedConfiguration; import alluxio.conf.PropertyKey; import alluxio.exception.AlluxioException; import alluxio.grpc.CreateFilePOptions; import alluxio.grpc.WritePType; -import alluxio.util.FileSystemOptions; +import alluxio.util.FileSystemOptionsUtils; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -78,7 +79,7 @@ public class AlluxioSink implements Sink { private AlluxioSinkConfig alluxioSinkConfig; private AlluxioState alluxioState; - private InstancedConfiguration configuration = InstancedConfiguration.defaults(); + private InstancedConfiguration configuration = Configuration.modifiableGlobal(); private ObjectMapper objectMapper = new ObjectMapper(); @@ -205,7 +206,7 @@ private void writeToAlluxio(Record record) throws AlluxioExceptio private void createTmpFile() throws AlluxioException, IOException { CreateFilePOptions.Builder optionsBuilder = - FileSystemOptions.createFileDefaults(configuration).toBuilder(); + FileSystemOptionsUtils.createFileDefaults(configuration).toBuilder(); UUID id = UUID.randomUUID(); String fileExtension = alluxioSinkConfig.getFileExtension(); tmpFilePath = tmpFileDirPath + "/" + id.toString() + "_tmp" + fileExtension; diff --git a/pulsar-io/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java b/pulsar-io/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java index 9325a2255ab0a..bf40581aae155 100644 --- a/pulsar-io/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java +++ b/pulsar-io/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java @@ -22,8 +22,8 @@ import alluxio.client.WriteType; import alluxio.client.file.FileSystem; import alluxio.client.file.URIStatus; +import alluxio.conf.Configuration; import alluxio.conf.PropertyKey; -import alluxio.conf.ServerConfiguration; import alluxio.master.LocalAlluxioCluster; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FilenameUtils; @@ -237,8 +237,8 @@ public Object getNativeObject() { private LocalAlluxioCluster setupSingleMasterCluster() throws Exception { // Setup and start the local alluxio cluster LocalAlluxioCluster cluster = new LocalAlluxioCluster(); - cluster.initConfiguration(getTestName(getClass().getSimpleName(), LocalAlluxioCluster.DEFAULT_TEST_NAME)); - ServerConfiguration.set(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.MUST_CACHE); + cluster.initConfiguration(getTestName(getClass().getSimpleName(), "test")); + Configuration.set(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.MUST_CACHE); cluster.start(); return cluster; } diff --git a/pulsar-io/aws/pom.xml b/pulsar-io/aws/pom.xml index 74e74f6e25851..2687e4baa04de 100644 --- a/pulsar-io/aws/pom.xml +++ b/pulsar-io/aws/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-aws diff --git a/pulsar-sql/presto-distribution/src/main/resources/conf/config.properties b/pulsar-io/azure-data-explorer/docker-compose.yml similarity index 52% rename from pulsar-sql/presto-distribution/src/main/resources/conf/config.properties rename to pulsar-io/azure-data-explorer/docker-compose.yml index 8915a677d3a92..4825d92edeba6 100644 --- a/pulsar-sql/presto-distribution/src/main/resources/conf/config.properties +++ b/pulsar-io/azure-data-explorer/docker-compose.yml @@ -17,24 +17,31 @@ # under the License. # -node.id=ffffffff-ffff-ffff-ffff-ffffffffffff -node.environment=test -http-server.http.port=8081 -discovery-server.enabled=true -discovery.uri=http://localhost:8081 +version: '3.0' -exchange.http-client.max-connections=1000 -exchange.http-client.max-connections-per-server=1000 -exchange.http-client.connect-timeout=1m -exchange.http-client.idle-timeout=1m +services: + pulsar-server: + command: bin/pulsar standalone + image: apachepulsar/pulsar-all:latest + container_name: pulsar-server + hostname: pulsar-server + volumes: + - ./target/pulsar-io-azuredataexplorer-3.2.0-SNAPSHOT.nar:/pulsar/connectors/pulsar-io-azuredataexplorer-3.2.0-SNAPSHOT.nar + ports: + - 8080:8080 + - 6650:6650 + networks: + - custom-sink-connector + healthcheck: + interval: 10s + retries: 20 + test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:8080/admin/v2/clusters/standalone -scheduler.http-client.max-connections=1000 -scheduler.http-client.max-connections-per-server=1000 -scheduler.http-client.connect-timeout=1m -scheduler.http-client.idle-timeout=1m +volumes: + custom-sink-connector: + driver: local -query.client.timeout=5m -query.min-expire-age=30m - -node-scheduler.include-coordinator=true +networks: + custom-sink-connector: + driver: bridge \ No newline at end of file diff --git a/pulsar-io/azure-data-explorer/pom.xml b/pulsar-io/azure-data-explorer/pom.xml new file mode 100644 index 0000000000000..3281b6663dc2c --- /dev/null +++ b/pulsar-io/azure-data-explorer/pom.xml @@ -0,0 +1,83 @@ + + + 4.0.0 + + org.apache.pulsar + pulsar-io + 4.0.0-SNAPSHOT + + + pulsar-io-azuredataexplorer + Pulsar IO :: AzureDataExplorer + + + 5.0.4 + + + + ${project.groupId} + pulsar-io-common + ${project.version} + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + org.apache.pulsar + pulsar-functions-instance + ${project.version} + test + + + + com.microsoft.azure.kusto + kusto-data + ${kusto.sdk.version} + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + + com.microsoft.azure.kusto + kusto-ingest + ${kusto.sdk.version} + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + + \ No newline at end of file diff --git a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/AwsCredentialProviderPlugin.java b/pulsar-io/azure-data-explorer/src/main/java/org/apache/pulsar/io/azuredataexplorer/ADXPulsarEvent.java similarity index 68% rename from pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/AwsCredentialProviderPlugin.java rename to pulsar-io/azure-data-explorer/src/main/java/org/apache/pulsar/io/azuredataexplorer/ADXPulsarEvent.java index e88a952293b4f..3c64fc7be0698 100644 --- a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/AwsCredentialProviderPlugin.java +++ b/pulsar-io/azure-data-explorer/src/main/java/org/apache/pulsar/io/azuredataexplorer/ADXPulsarEvent.java @@ -16,14 +16,19 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.io.kinesis; +package org.apache.pulsar.io.azuredataexplorer; -/** - * This is a stub class for backwards compatibility. In new code and configurations, please use the plugins - * from org.apache.pulsar.io.aws - * - * @see org.apache.pulsar.io.aws.AwsCredentialProviderPlugin - */ -@Deprecated -public interface AwsCredentialProviderPlugin extends org.apache.pulsar.io.aws.AwsCredentialProviderPlugin { +import java.time.Instant; +import lombok.Data; +import lombok.experimental.Accessors; + +@Data +@Accessors(chain = true) +public class ADXPulsarEvent { + private String key; + private String value; + private String properties; + private String producerName; + private long sequenceId; + private Instant eventTime; } diff --git a/pulsar-io/azure-data-explorer/src/main/java/org/apache/pulsar/io/azuredataexplorer/ADXSink.java b/pulsar-io/azure-data-explorer/src/main/java/org/apache/pulsar/io/azuredataexplorer/ADXSink.java new file mode 100644 index 0000000000000..07eb372833bcf --- /dev/null +++ b/pulsar-io/azure-data-explorer/src/main/java/org/apache/pulsar/io/azuredataexplorer/ADXSink.java @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.azuredataexplorer; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.microsoft.azure.kusto.data.StringUtils; +import com.microsoft.azure.kusto.data.auth.ConnectionStringBuilder; +import com.microsoft.azure.kusto.data.exceptions.KustoDataExceptionBase; +import com.microsoft.azure.kusto.ingest.IngestClient; +import com.microsoft.azure.kusto.ingest.IngestClientFactory; +import com.microsoft.azure.kusto.ingest.IngestionMapping; +import com.microsoft.azure.kusto.ingest.IngestionProperties; +import com.microsoft.azure.kusto.ingest.ManagedStreamingIngestClient; +import com.microsoft.azure.kusto.ingest.exceptions.IngestionClientException; +import com.microsoft.azure.kusto.ingest.exceptions.IngestionServiceException; +import com.microsoft.azure.kusto.ingest.result.IngestionResult; +import com.microsoft.azure.kusto.ingest.result.IngestionStatus; +import com.microsoft.azure.kusto.ingest.result.TableReportIngestionResult; +import com.microsoft.azure.kusto.ingest.source.StreamSourceInfo; +import java.io.ByteArrayInputStream; +import java.net.ConnectException; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.functions.api.Record; +import org.apache.pulsar.io.core.Sink; +import org.apache.pulsar.io.core.SinkContext; +import org.apache.pulsar.io.core.annotations.Connector; +import org.apache.pulsar.io.core.annotations.IOType; +import org.jetbrains.annotations.NotNull; + +@Connector( + name = "adx", + type = IOType.SINK, + help = "The ADXSink is used for moving messages from Pulsar to ADX.", + configClass = ADXSinkConfig.class +) +@Slf4j +public class ADXSink implements Sink { + private final ObjectMapper mapper = new ObjectMapper().registerModule(new JavaTimeModule()); + IngestionProperties ingestionProperties; + private IngestClient ingestClient; + private List> incomingRecordsList; + private int batchSize; + private long batchTimeMs; + private ScheduledExecutorService adxSinkExecutor; + private int maxRetryAttempts; + private long retryBackOffTime; + + @Override + public void open(Map config, SinkContext sinkContext) throws Exception { + log.info("Open ADX Sink"); + // Azure data explorer, initializations + ADXSinkConfig adxConfig = ADXSinkConfig.load(config, sinkContext); + adxConfig.validate(); + ConnectionStringBuilder kcsb = getConnectionStringBuilder(adxConfig); + if (kcsb == null) { + throw new Exception("Kusto Connection String NULL"); + } + log.debug("ConnectionString created: {}.", kcsb); + ingestClient = adxConfig.getManagedIdentityId() != null + ? IngestClientFactory.createManagedStreamingIngestClient(kcsb) : + IngestClientFactory.createClient(kcsb); + ingestionProperties = new IngestionProperties(adxConfig.getDatabase(), adxConfig.getTable()); + ingestionProperties.setIngestionMapping(adxConfig.getMappingRefName(), + getParseMappingRefType(adxConfig.getMappingRefType())); + ingestionProperties.setReportLevel(IngestionProperties.IngestionReportLevel.FAILURES_AND_SUCCESSES); + ingestionProperties.setReportMethod(IngestionProperties.IngestionReportMethod.TABLE); + ingestionProperties.setFlushImmediately(adxConfig.isFlushImmediately()); + ingestionProperties.setDataFormat(IngestionProperties.DataFormat.MULTIJSON); + log.debug("Ingestion Properties: {}", ingestionProperties.toString()); + + maxRetryAttempts = adxConfig.getMaxRetryAttempts() + 1; + retryBackOffTime = adxConfig.getRetryBackOffTime(); + /*incoming records list will hold incoming messages, + flushExecutor executes the flushData according to batch time */ + batchSize = adxConfig.getBatchSize(); + batchTimeMs = adxConfig.getBatchTimeMs(); + incomingRecordsList = new ArrayList<>(); + adxSinkExecutor = Executors.newScheduledThreadPool(1); + adxSinkExecutor.scheduleAtFixedRate(this::sinkData, batchTimeMs, batchTimeMs, TimeUnit.MILLISECONDS); + } + + @Override + public void write(Record record) { + int runningSize; + synchronized (this) { + incomingRecordsList.add(record); + runningSize = incomingRecordsList.size(); + } + if (runningSize == batchSize) { + adxSinkExecutor.execute(this::sinkData); + } + } + + private void sinkData() { + final List> recordsToSink; + synchronized (this) { + if (incomingRecordsList.isEmpty()) { + return; + } + recordsToSink = incomingRecordsList; + incomingRecordsList = new ArrayList<>(); + } + + List eventsToSink = new LinkedList<>(); + for (Record record : recordsToSink) { + try { + eventsToSink.add(getADXPulsarEvent(record)); + } catch (Exception ex) { + record.fail(); + log.error("Failed to collect the record for ADX cluster.", ex); + } + } + try { + for (int retryAttempts = 0; true; retryAttempts++) { + try { + StreamSourceInfo streamSourceInfo = + new StreamSourceInfo(new ByteArrayInputStream(mapper.writeValueAsBytes(eventsToSink))); + IngestionResult ingestionResult = + ingestClient.ingestFromStream(streamSourceInfo, ingestionProperties); + if (ingestionResult instanceof TableReportIngestionResult) { + // If TableReportIngestionResult returned then the ingestion status is from streaming ingest + IngestionStatus ingestionStatus = ingestionResult.getIngestionStatusCollection().get(0); + if (!hasStreamingSucceeded(ingestionStatus)) { + retryAttempts += ManagedStreamingIngestClient.ATTEMPT_COUNT; + backOffForRemainingAttempts(retryAttempts, null, recordsToSink); + continue; + } + recordsToSink.forEach(Record::ack); + } + return; + } catch (IngestionServiceException exception) { + Throwable innerException = exception.getCause(); + if (innerException instanceof KustoDataExceptionBase + && ((KustoDataExceptionBase) innerException).isPermanent()) { + recordsToSink.forEach(Record::fail); + throw new ConnectException(exception.getMessage()); + } + // retrying transient exceptions + backOffForRemainingAttempts(retryAttempts, exception, recordsToSink); + } catch (IngestionClientException | URISyntaxException exception) { + recordsToSink.forEach(Record::fail); + throw new ConnectException(exception.getMessage()); + } + } + + } catch (Exception ex) { + log.error("Failed to publish the message to ADX cluster", ex); + } + } + + private boolean hasStreamingSucceeded(@NotNull IngestionStatus status) { + switch (status.status) { + case Succeeded: + case Queued: + case Pending: + return true; + case Skipped: + case PartiallySucceeded: + String failureStatus = status.getFailureStatus(); + String details = status.getDetails(); + UUID ingestionSourceId = status.getIngestionSourceId(); + log.warn("A batch of streaming records has {} ingestion: table:{}, database:{}, operationId: {}," + + "ingestionSourceId: {}{}{}.\n" + + "Status is final and therefore ingestion won't be retried and data won't reach dlq", + status.getStatus(), + status.getTable(), + status.getDatabase(), + status.getOperationId(), + ingestionSourceId, + (StringUtils.isNotEmpty(failureStatus) ? (", failure: " + failureStatus) : ""), + (StringUtils.isNotEmpty(details) ? (", details: " + details) : "")); + return true; + case Failed: + } + return false; + } + + private void backOffForRemainingAttempts(int retryAttempts, Exception exception, List> records) + throws PulsarClientException.ConnectException { + if (retryAttempts < maxRetryAttempts) { + long sleepTimeMs = retryBackOffTime; + log.error( + "Failed to ingest records into Kusto, backing off and retrying ingesting records " + + "after {} milliseconds.", + sleepTimeMs); + try { + TimeUnit.MILLISECONDS.sleep(sleepTimeMs); + throw new InterruptedException(); + } catch (InterruptedException interruptedErr) { + records.forEach(Record::fail); + throw new PulsarClientException.ConnectException(String.format( + "Retrying ingesting records into KustoDB was interrupted after retryAttempts=%s", + retryAttempts + 1) + ); + } + } else { + records.forEach(Record::fail); + throw new PulsarClientException.ConnectException( + String.format("Retry attempts exhausted, failed to ingest records into KustoDB. Exception: %s", + exception.getMessage())); + } + } + + private @NotNull ADXPulsarEvent getADXPulsarEvent(@NotNull Record record) throws Exception { + ADXPulsarEvent event = new ADXPulsarEvent(); + record.getEventTime().ifPresent(time -> event.setEventTime(Instant.ofEpochMilli(time))); + record.getKey().ifPresent(event::setKey); + record.getMessage().ifPresent(message -> event.setProducerName(message.getProducerName())); + record.getMessage().ifPresent(message -> event.setSequenceId(message.getSequenceId())); + event.setValue(new String(record.getValue(), StandardCharsets.UTF_8)); + event.setProperties(new ObjectMapper().writeValueAsString(record.getProperties())); + return event; + } + + private IngestionMapping.IngestionMappingKind getParseMappingRefType(String mappingRefType) { + if (mappingRefType == null || mappingRefType.isEmpty()) { + return null; + } + return switch (mappingRefType) { + case "CSV" -> IngestionMapping.IngestionMappingKind.CSV; + case "AVRO" -> IngestionMapping.IngestionMappingKind.AVRO; + case "JSON" -> IngestionMapping.IngestionMappingKind.JSON; + case "PARQUET" -> IngestionMapping.IngestionMappingKind.PARQUET; + default -> null; + }; + } + + private ConnectionStringBuilder getConnectionStringBuilder(@NotNull ADXSinkConfig adxConfig) { + + if (adxConfig.getManagedIdentityId() != null) { + if ("system".equalsIgnoreCase(adxConfig.getManagedIdentityId())) { + return ConnectionStringBuilder.createWithAadManagedIdentity(adxConfig.getClusterUrl()); + } + ConnectionStringBuilder.createWithAadManagedIdentity(adxConfig.getClusterUrl(), + adxConfig.getManagedIdentityId()); + } + return ConnectionStringBuilder.createWithAadApplicationCredentials(adxConfig.getClusterUrl(), + adxConfig.getAppId(), adxConfig.getAppKey(), adxConfig.getTenantId()); + } + + @Override + public void close() throws Exception { + ingestClient.close(); + adxSinkExecutor.shutdown(); + try { + if (!adxSinkExecutor.awaitTermination(2 * batchTimeMs, TimeUnit.MILLISECONDS)) { + adxSinkExecutor.shutdownNow(); + } + } catch (InterruptedException ie) { + adxSinkExecutor.shutdownNow(); + Thread.currentThread().interrupt(); + } + log.info("Kusto ingest client closed."); + } +} \ No newline at end of file diff --git a/pulsar-io/azure-data-explorer/src/main/java/org/apache/pulsar/io/azuredataexplorer/ADXSinkConfig.java b/pulsar-io/azure-data-explorer/src/main/java/org/apache/pulsar/io/azuredataexplorer/ADXSinkConfig.java new file mode 100644 index 0000000000000..42c583bd3cce9 --- /dev/null +++ b/pulsar-io/azure-data-explorer/src/main/java/org/apache/pulsar/io/azuredataexplorer/ADXSinkConfig.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.azuredataexplorer; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.util.Map; +import java.util.Objects; +import lombok.Data; +import lombok.experimental.Accessors; +import org.apache.pulsar.io.common.IOConfigUtils; +import org.apache.pulsar.io.core.SinkContext; +import org.apache.pulsar.io.core.annotations.FieldDoc; + + +@Data +@Accessors(chain = true) +public class ADXSinkConfig implements Serializable { + + @FieldDoc(required = true, defaultValue = "", help = "The ADX cluster URL") + private String clusterUrl; + + @FieldDoc(required = true, defaultValue = "", help = "The database name to which data need to be ingested") + private String database; + + @FieldDoc(required = true, defaultValue = "", help = "Table name to which pulsar data need to be ingested.") + private String table; + + @FieldDoc(defaultValue = "", help = "The AAD app Id for authentication", sensitive = true) + private String appId; + + @FieldDoc(defaultValue = "", help = "The AAD app secret for authentication", sensitive = true) + private String appKey; + + @FieldDoc(defaultValue = "", help = "The tenant Id for authentication") + private String tenantId; + + @FieldDoc(defaultValue = "", help = "The Managed Identity credential for authentication." + + " Set this with clientId in case of User assigned MI." + + " and 'system' in case of System assigned managed identity") + private String managedIdentityId; + + @FieldDoc(defaultValue = "", help = "The mapping reference for ingestion") + private String mappingRefName; + + @FieldDoc(defaultValue = "", help = "The type of mapping reference provided") + private String mappingRefType; + + @FieldDoc(defaultValue = "false", help = "Denotes if flush should happen immediately without aggregation. " + + "Not recommended to enable flushImmediately for production workloads") + private boolean flushImmediately = false; + + @FieldDoc(defaultValue = "100", help = "For batching, this defines the number of " + + "records to hold for batching, to sink data to adx") + private int batchSize = 100; + + @FieldDoc(defaultValue = "10000", help = "For batching, this defines the time to hold" + + " records before sink to adx") + private long batchTimeMs = 10000; + + @FieldDoc(defaultValue = "1", help = "Max retry attempts, In case of transient ingestion errors") + private int maxRetryAttempts = 1; + + @FieldDoc(defaultValue = "10", help = "Period of time in milliseconds to backoff" + + " before retry for transient errors") + private long retryBackOffTime = 10; + + + public static ADXSinkConfig load(String yamlFile) throws IOException { + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + return mapper.readValue(new File(yamlFile), ADXSinkConfig.class); + } + + protected static ADXSinkConfig load(Map config, SinkContext sinkContext) { + return IOConfigUtils.loadWithSecrets(config, ADXSinkConfig.class, sinkContext); + } + + public void validate() throws Exception { + Objects.requireNonNull(clusterUrl, "clusterUrl property not set."); + Objects.requireNonNull(database, "database property not set."); + Objects.requireNonNull(table, "table property not set."); + if (managedIdentityId == null && (appId == null || appKey == null || tenantId == null)) { + throw new Exception("Auth credentials not valid"); + } + } +} diff --git a/pulsar-io/azure-data-explorer/src/main/java/org/apache/pulsar/io/azuredataexplorer/ADXSinkUtils.java b/pulsar-io/azure-data-explorer/src/main/java/org/apache/pulsar/io/azuredataexplorer/ADXSinkUtils.java new file mode 100644 index 0000000000000..4a073f8bf7aa1 --- /dev/null +++ b/pulsar-io/azure-data-explorer/src/main/java/org/apache/pulsar/io/azuredataexplorer/ADXSinkUtils.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.azuredataexplorer; + +public class ADXSinkUtils { + + static final String INGEST_PREFIX = "ingest-"; + static final String PROTOCOL_SUFFIX = "://"; + + static String getIngestionEndpoint(String clusterUrl) { + if (clusterUrl.contains(INGEST_PREFIX)) { + return clusterUrl; + } else { + return clusterUrl.replaceFirst(PROTOCOL_SUFFIX, PROTOCOL_SUFFIX + INGEST_PREFIX); + } + } + + static String getQueryEndpoint(String clusterUrl) { + if (clusterUrl.contains(INGEST_PREFIX)) { + return clusterUrl.replaceFirst(INGEST_PREFIX, ""); + } else { + return clusterUrl; + } + } +} diff --git a/pulsar-io/azure-data-explorer/src/main/java/org/apache/pulsar/io/azuredataexplorer/package-info.java b/pulsar-io/azure-data-explorer/src/main/java/org/apache/pulsar/io/azuredataexplorer/package-info.java new file mode 100644 index 0000000000000..9a9bd7d4db687 --- /dev/null +++ b/pulsar-io/azure-data-explorer/src/main/java/org/apache/pulsar/io/azuredataexplorer/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.azuredataexplorer; \ No newline at end of file diff --git a/pulsar-sql/presto-distribution/src/main/resources/conf/jvm.config b/pulsar-io/azure-data-explorer/src/main/resources/META-INF/services/pulsar-io.yaml similarity index 78% rename from pulsar-sql/presto-distribution/src/main/resources/conf/jvm.config rename to pulsar-io/azure-data-explorer/src/main/resources/META-INF/services/pulsar-io.yaml index 86c9d0613b233..753bfb3ff053f 100644 --- a/pulsar-sql/presto-distribution/src/main/resources/conf/jvm.config +++ b/pulsar-io/azure-data-explorer/src/main/resources/META-INF/services/pulsar-io.yaml @@ -17,12 +17,7 @@ # under the License. # --server --Xmx16G --XX:+UseZGC --XX:+UseGCOverheadLimit --XX:+ExplicitGCInvokesConcurrent --XX:+HeapDumpOnOutOfMemoryError --XX:+ExitOnOutOfMemoryError --Dpresto-temporarily-allow-java8=true --Djdk.attach.allowAttachSelf=true +name: azure-data-explorer +description: Azure Data Explorer sink +sinkClass: org.apache.pulsar.io.azuredataexplorer.ADXSink +sinkConfigClass: org.apache.pulsar.io.azuredataexplorer.ADXSinkConfig diff --git a/pulsar-io/azure-data-explorer/src/test/java/org/apache/pulsar/io/azuredataexplorer/ADXSinkConfigTest.java b/pulsar-io/azure-data-explorer/src/test/java/org/apache/pulsar/io/azuredataexplorer/ADXSinkConfigTest.java new file mode 100644 index 0000000000000..c05e3b159d803 --- /dev/null +++ b/pulsar-io/azure-data-explorer/src/test/java/org/apache/pulsar/io/azuredataexplorer/ADXSinkConfigTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.azuredataexplorer; + +import org.jetbrains.annotations.NotNull; +import org.testng.annotations.Test; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import org.apache.pulsar.io.core.SinkContext; +import org.mockito.Mockito; + +import static org.testng.Assert.*; + +public class ADXSinkConfigTest { + @Test + public final void loadFromYamlFileTest() throws IOException { + File sinkConfig = readSinkConfig("sinkConfig.yaml"); + String path = sinkConfig.getAbsolutePath(); + ADXSinkConfig config = ADXSinkConfig.load(path); + assertNotNull(config); + assertEquals(config.getClusterUrl(), "https://somecluster.eastus.kusto.windows.net"); + assertEquals(config.getDatabase(), "somedb"); + assertEquals(config.getTable(), "tableName"); + assertEquals(config.getAppId(), "xxxx-xxxx-xxxx-xxxx"); + assertEquals(config.getAppKey(), "xxxx-xxxx-xxxx-xxxx"); + assertEquals(config.getTenantId(), "xxxx-xxxx-xxxx-xxxx"); + assertEquals(config.getManagedIdentityId(), "xxxx-some-id-xxxx OR empty string"); + assertEquals(config.getMappingRefName(), "mapping ref name"); + assertEquals(config.getMappingRefType(), "CSV"); + assertFalse(config.isFlushImmediately()); + assertEquals(config.getBatchSize(), 100); + assertEquals(config.getBatchTimeMs(), 10000); + } + + @Test + public final void validateConfigTest() throws Exception { + File yamlFile = readSinkConfig("sinkConfigValid.yaml"); + String path = yamlFile.getAbsolutePath(); + ADXSinkConfig config = ADXSinkConfig.load(path); + config.validate(); + } + + @Test(expectedExceptions = Exception.class) + public final void validateInvalidConfigTest() throws Exception { + File yamlFile = readSinkConfig("sinkConfigInvalid.yaml"); + String path = yamlFile.getAbsolutePath(); + ADXSinkConfig config = ADXSinkConfig.load(path); + config.validate(); + } + + @Test + public final void loadFromMapTest() throws IOException { + Map map = getConfig(); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + ADXSinkConfig config = ADXSinkConfig.load(map, sinkContext); + assertNotNull(config); + assertEquals(config.getClusterUrl(), "https://somecluster.eastus.kusto.windows.net"); + assertEquals(config.getDatabase(), "somedb"); + assertEquals(config.getTable(), "tableName"); + assertEquals(config.getAppId(), "xxxx-xxxx-xxxx-xxxx"); + assertEquals(config.getAppKey(), "xxxx-xxxx-xxxx-xxxx"); + assertEquals(config.getTenantId(), "xxxx-xxxx-xxxx-xxxx"); + assertEquals(config.getManagedIdentityId(), "xxxx-some-id-xxxx OR empty string"); + assertEquals(config.getMappingRefName(), "mapping ref name"); + assertEquals(config.getMappingRefType(), "CSV"); + assertFalse(config.isFlushImmediately()); + assertEquals(config.getBatchSize(), 100); + assertEquals(config.getBatchTimeMs(), 10000); + } + + @NotNull + private static Map getConfig() { + Map map = new HashMap<>(); + map.put("clusterUrl", "https://somecluster.eastus.kusto.windows.net"); + map.put("database", "somedb"); + map.put("table", "tableName"); + map.put("appId", "xxxx-xxxx-xxxx-xxxx"); + map.put("appKey", "xxxx-xxxx-xxxx-xxxx"); + map.put("tenantId", "xxxx-xxxx-xxxx-xxxx"); + map.put("managedIdentityId", "xxxx-some-id-xxxx OR empty string"); + map.put("mappingRefName", "mapping ref name"); + map.put("mappingRefType", "CSV"); + map.put("flushImmediately", false); + map.put("batchSize", 100); + map.put("batchTimeMs", 10000); + return map; + } + + private @NotNull File readSinkConfig(String name) { + ClassLoader classLoader = getClass().getClassLoader(); + return new File(Objects.requireNonNull(classLoader.getResource(name)).getFile()); + } +} diff --git a/pulsar-io/azure-data-explorer/src/test/java/org/apache/pulsar/io/azuredataexplorer/ADXSinkE2ETest.java b/pulsar-io/azure-data-explorer/src/test/java/org/apache/pulsar/io/azuredataexplorer/ADXSinkE2ETest.java new file mode 100644 index 0000000000000..b04c20dad89b9 --- /dev/null +++ b/pulsar-io/azure-data-explorer/src/test/java/org/apache/pulsar/io/azuredataexplorer/ADXSinkE2ETest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.azuredataexplorer; + +import com.microsoft.azure.kusto.data.Client; +import com.microsoft.azure.kusto.data.ClientFactory; +import com.microsoft.azure.kusto.data.KustoOperationResult; +import com.microsoft.azure.kusto.data.KustoResultSetTable; +import com.microsoft.azure.kusto.data.auth.ConnectionStringBuilder; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.functions.api.Record; +import org.apache.pulsar.functions.instance.SinkRecord; +import org.apache.pulsar.io.core.SinkContext; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.SkipException; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ThreadLocalRandom; + +@Slf4j +public class ADXSinkE2ETest { + + String database; + String cluster; + String authorityId; + String appId; + String appKey; + private final String table = "ADXPulsarTest_" + ThreadLocalRandom.current().nextInt(0, 100); + private Client kustoAdminClient = null; + Map configs; + + @BeforeMethod + public void setUp() throws Exception { + cluster = System.getenv("kustoCluster"); + database = System.getenv("kustoDatabase"); + authorityId = System.getenv("kustoAadAuthorityID"); + appId = System.getenv("kustoAadAppId"); + appKey = System.getenv("kustoAadAppSecret"); + + if(cluster == null){ + throw new SkipException("Skipping tests because environment vars was not accessible."); + } + + configs = new HashMap<>(); + configs.put("clusterUrl", cluster); + configs.put("database", database); + configs.put("table", table); + configs.put("batchTimeMs", 1000); + configs.put("flushImmediately", true); + configs.put("appId", appId); + configs.put("appKey", appKey); + configs.put("tenantId", authorityId); + configs.put("maxRetryAttempts", 3); + configs.put("retryBackOffTime", 100); + + ConnectionStringBuilder engineKcsb = + ConnectionStringBuilder.createWithAadApplicationCredentials(ADXSinkUtils.getQueryEndpoint(cluster), + appId, appKey, authorityId); + kustoAdminClient = ClientFactory.createClient(engineKcsb); + String createTableCommand = ".create table " + table + + " ( key:string , value:string, eventTime:datetime , producerName:string , sequenceId:long ,properties:dynamic )"; + log.info("Creating test table {} ", table); + kustoAdminClient.execute(database, createTableCommand); + kustoAdminClient.execute(database, generateAlterIngestionBatchingPolicyCommand(database, + "{\"MaximumBatchingTimeSpan\":\"00:00:10\", \"MaximumNumberOfItems\": 500, \"MaximumRawDataSizeMB\": 1024}")); + log.info("Ingestion policy on table {} altered",table); + } + + private String generateAlterIngestionBatchingPolicyCommand(String entityName, String targetBatchingPolicy) { + return ".alter database " + entityName + " policy ingestionbatching ```" + targetBatchingPolicy + "```"; + } + + @AfterMethod(alwaysRun = true) + public void tearDown() { + try { + log.warn("Dropping test table {} ", table); + kustoAdminClient.execute(".drop table " + table + " ifexists"); + } catch (Exception ignore) { + } + } + + @Test + public void testOpenAndWriteSink() throws Exception { + + ADXSink sink = new ADXSink(); + sink.open(configs, Mockito.mock(SinkContext.class)); + int writeCount = 50; + + for (int i = 0; i < writeCount; i++) { + Record record = build("key_" + i, "test data from ADX Pulsar Sink_" + i); + sink.write(record); + } + Thread.sleep(40000); + KustoOperationResult result = kustoAdminClient.execute(database, table + " | count"); + KustoResultSetTable mainTableResult = result.getPrimaryResults(); + mainTableResult.next(); + int actualRowsCount = mainTableResult.getInt(0); + Assert.assertEquals(actualRowsCount, writeCount); + kustoAdminClient.execute(database, ".clear table " + table + " data"); + sink.close(); + } + + @Test + public void testOpenAndWriteSinkWithTimeouts() throws Exception { + ADXSink sink = new ADXSink(); + sink.open(configs, Mockito.mock(SinkContext.class)); + int writeCount = 9; + + for (int i = 0; i < writeCount; i++) { + Record record = build("key_" + i, "test data from ADX Pulsar Sink_" + i); + sink.write(record); + } + Thread.sleep(40000); + KustoOperationResult result = kustoAdminClient.execute(database, table + " | count"); + KustoResultSetTable mainTableResult = result.getPrimaryResults(); + mainTableResult.next(); + int actualRowsCount = mainTableResult.getInt(0); + Assert.assertEquals(actualRowsCount, writeCount); + + sink.close(); + } + + private Record build(String key, String value) { + return new SinkRecord(new Record<>() { + + @Override + public byte[] getValue() { + return value.getBytes(StandardCharsets.UTF_8); + } + + @Override + public Optional getDestinationTopic() { + return Optional.of("destination-topic"); + } + + @Override + public Optional getEventTime() { + return Optional.of(System.currentTimeMillis()); + } + + @Override + public Optional getKey() { + return Optional.of("key-" + key); + } + + @Override + public Map getProperties() { + return new HashMap<>(); + } + }, value.getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/pulsar-io/azure-data-explorer/src/test/resources/sinkConfig.yaml b/pulsar-io/azure-data-explorer/src/test/resources/sinkConfig.yaml new file mode 100644 index 0000000000000..32086a158f059 --- /dev/null +++ b/pulsar-io/azure-data-explorer/src/test/resources/sinkConfig.yaml @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{ + "clusterUrl":"https://somecluster.eastus.kusto.windows.net", + "database":"somedb", + "table": "tableName", + "appId": "xxxx-xxxx-xxxx-xxxx", + "appKey": "xxxx-xxxx-xxxx-xxxx", + "tenantId": "xxxx-xxxx-xxxx-xxxx", + "managedIdentityId": "xxxx-some-id-xxxx OR empty string", + "mappingRefName": "mapping ref name", + "mappingRefType":"CSV", + "flushImmediately":false, + "batchSize":100, + "batchTimeMs":10000, +} diff --git a/pulsar-io/azure-data-explorer/src/test/resources/sinkConfigInvalid.yaml b/pulsar-io/azure-data-explorer/src/test/resources/sinkConfigInvalid.yaml new file mode 100644 index 0000000000000..55537a90af80e --- /dev/null +++ b/pulsar-io/azure-data-explorer/src/test/resources/sinkConfigInvalid.yaml @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{ + "clusterUrl":"https://somecluster.eastus.kusto.windows.net", + "database":"somedb", + "table": "tableName", + "appId": "xxxx-xxxx-xxxx-xxxx", + "appKey": "xxxx-xxxx-xxxx-xxxx", +} diff --git a/pulsar-io/azure-data-explorer/src/test/resources/sinkConfigValid.yaml b/pulsar-io/azure-data-explorer/src/test/resources/sinkConfigValid.yaml new file mode 100644 index 0000000000000..32086a158f059 --- /dev/null +++ b/pulsar-io/azure-data-explorer/src/test/resources/sinkConfigValid.yaml @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +{ + "clusterUrl":"https://somecluster.eastus.kusto.windows.net", + "database":"somedb", + "table": "tableName", + "appId": "xxxx-xxxx-xxxx-xxxx", + "appKey": "xxxx-xxxx-xxxx-xxxx", + "tenantId": "xxxx-xxxx-xxxx-xxxx", + "managedIdentityId": "xxxx-some-id-xxxx OR empty string", + "mappingRefName": "mapping ref name", + "mappingRefType":"CSV", + "flushImmediately":false, + "batchSize":100, + "batchTimeMs":10000, +} diff --git a/pulsar-io/batch-data-generator/pom.xml b/pulsar-io/batch-data-generator/pom.xml index 1a6bffd531ffb..a15c956cbe18e 100644 --- a/pulsar-io/batch-data-generator/pom.xml +++ b/pulsar-io/batch-data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-batch-data-generator diff --git a/pulsar-io/batch-discovery-triggerers/pom.xml b/pulsar-io/batch-discovery-triggerers/pom.xml index ddec093bccade..0a36b337b01dc 100644 --- a/pulsar-io/batch-discovery-triggerers/pom.xml +++ b/pulsar-io/batch-discovery-triggerers/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-batch-discovery-triggerers diff --git a/pulsar-io/canal/pom.xml b/pulsar-io/canal/pom.xml index ecab67eba26b6..cb2f64584e0cc 100644 --- a/pulsar-io/canal/pom.xml +++ b/pulsar-io/canal/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT 4.0.0 @@ -37,6 +37,11 @@ + + ${project.groupId} + pulsar-io-common + ${project.version} + ${project.groupId} pulsar-io-core diff --git a/pulsar-io/canal/src/main/java/org/apache/pulsar/io/canal/CanalAbstractSource.java b/pulsar-io/canal/src/main/java/org/apache/pulsar/io/canal/CanalAbstractSource.java index 06c8788d5aea1..7d0cd0305a49e 100644 --- a/pulsar-io/canal/src/main/java/org/apache/pulsar/io/canal/CanalAbstractSource.java +++ b/pulsar-io/canal/src/main/java/org/apache/pulsar/io/canal/CanalAbstractSource.java @@ -57,7 +57,7 @@ public abstract class CanalAbstractSource extends PushSource { @Override public void open(Map config, SourceContext sourceContext) throws Exception { - canalSourceConfig = CanalSourceConfig.load(config); + canalSourceConfig = CanalSourceConfig.load(config, sourceContext); if (canalSourceConfig.getCluster()) { connector = CanalConnectors.newClusterConnector(canalSourceConfig.getZkServers(), canalSourceConfig.getDestination(), canalSourceConfig.getUsername(), diff --git a/pulsar-io/canal/src/main/java/org/apache/pulsar/io/canal/CanalSourceConfig.java b/pulsar-io/canal/src/main/java/org/apache/pulsar/io/canal/CanalSourceConfig.java index a0408e60e5f76..5a754988ffdc1 100644 --- a/pulsar-io/canal/src/main/java/org/apache/pulsar/io/canal/CanalSourceConfig.java +++ b/pulsar-io/canal/src/main/java/org/apache/pulsar/io/canal/CanalSourceConfig.java @@ -26,6 +26,8 @@ import java.util.Map; import lombok.Data; import lombok.experimental.Accessors; +import org.apache.pulsar.io.common.IOConfigUtils; +import org.apache.pulsar.io.core.SourceContext; import org.apache.pulsar.io.core.annotations.FieldDoc; @@ -86,8 +88,7 @@ public static CanalSourceConfig load(String yamlFile) throws IOException { } - public static CanalSourceConfig load(Map map) throws IOException { - ObjectMapper mapper = new ObjectMapper(); - return mapper.readValue(mapper.writeValueAsString(map), CanalSourceConfig.class); + public static CanalSourceConfig load(Map map, SourceContext sourceContext) throws IOException { + return IOConfigUtils.loadWithSecrets(map, CanalSourceConfig.class, sourceContext); } } diff --git a/pulsar-io/cassandra/pom.xml b/pulsar-io/cassandra/pom.xml index 3859bd1029e86..9fb56347cd725 100644 --- a/pulsar-io/cassandra/pom.xml +++ b/pulsar-io/cassandra/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-cassandra diff --git a/pulsar-io/common/pom.xml b/pulsar-io/common/pom.xml index 14a134bc42858..28feb5253a2ca 100644 --- a/pulsar-io/common/pom.xml +++ b/pulsar-io/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-common diff --git a/pulsar-io/common/src/main/java/org/apache/pulsar/io/common/IOConfigUtils.java b/pulsar-io/common/src/main/java/org/apache/pulsar/io/common/IOConfigUtils.java index d15986a897caa..69d981bf68728 100644 --- a/pulsar-io/common/src/main/java/org/apache/pulsar/io/common/IOConfigUtils.java +++ b/pulsar-io/common/src/main/java/org/apache/pulsar/io/common/IOConfigUtils.java @@ -77,13 +77,14 @@ private static T loadWithSecrets(Map map, Class clazz, } } configs.computeIfAbsent(field.getName(), key -> { - if (fieldDoc.required()) { - throw new IllegalArgumentException(field.getName() + " cannot be null"); - } + // Use default value if it is not null before checking required String value = fieldDoc.defaultValue(); if (value != null && !value.isEmpty()) { return value; } + if (fieldDoc.required()) { + throw new IllegalArgumentException(field.getName() + " cannot be null"); + } return null; }); } diff --git a/pulsar-io/common/src/test/java/org/apache/pulsar/io/common/IOConfigUtilsTest.java b/pulsar-io/common/src/test/java/org/apache/pulsar/io/common/IOConfigUtilsTest.java index 52afac1a5ac0c..cd31a1a4f066a 100644 --- a/pulsar-io/common/src/test/java/org/apache/pulsar/io/common/IOConfigUtilsTest.java +++ b/pulsar-io/common/src/test/java/org/apache/pulsar/io/common/IOConfigUtilsTest.java @@ -54,6 +54,14 @@ static class TestDefaultConfig { ) protected String testRequired; + @FieldDoc( + required = true, + defaultValue = "defaultRequired", + sensitive = true, + help = "testRequired" + ) + protected String testDefaultRequired; + @FieldDoc( required = false, defaultValue = "defaultStr", @@ -259,12 +267,12 @@ public ByteBuffer getState(String key) { public CompletableFuture getStateAsync(String key) { return null; } - + @Override public void deleteState(String key) { - + } - + @Override public CompletableFuture deleteStateAsync(String key) { return null; @@ -284,6 +292,11 @@ public ConsumerBuilder newConsumerBuilder(Schema schema) throws Pulsar public PulsarClient getPulsarClient() { return null; } + + @Override + public void fatal(Throwable t) { + + } } @Test @@ -299,6 +312,9 @@ public void testDefaultValue() { configMap.put("testRequired", "test"); TestDefaultConfig testDefaultConfig = IOConfigUtils.loadWithSecrets(configMap, TestDefaultConfig.class, new TestSinkContext()); + // if there is default value for a required field and no value provided when load config, + // it should not throw exception but use the default value. + Assert.assertEquals(testDefaultConfig.getTestDefaultRequired(), "defaultRequired"); Assert.assertEquals(testDefaultConfig.getDefaultStr(), "defaultStr"); Assert.assertEquals(testDefaultConfig.isDefaultBool(), true); Assert.assertEquals(testDefaultConfig.getDefaultInt(), 100); @@ -449,12 +465,12 @@ public ByteBuffer getState(String key) { public CompletableFuture getStateAsync(String key) { return null; } - + @Override public void deleteState(String key) { - + } - + @Override public CompletableFuture deleteStateAsync(String key) { return null; @@ -464,6 +480,11 @@ public CompletableFuture deleteStateAsync(String key) { public PulsarClient getPulsarClient() { return null; } + + @Override + public void fatal(Throwable t) { + + } } @Test diff --git a/pulsar-io/core/pom.xml b/pulsar-io/core/pom.xml index 6eaa7a1a70d8a..60145cd087c90 100644 --- a/pulsar-io/core/pom.xml +++ b/pulsar-io/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-core diff --git a/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/AbstractPushSource.java b/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/AbstractPushSource.java new file mode 100644 index 0000000000000..185d1cebfbc3d --- /dev/null +++ b/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/AbstractPushSource.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.core; + +import java.util.concurrent.LinkedBlockingQueue; +import org.apache.pulsar.functions.api.Record; + +/** + * Pulsar's Push Source Abstract. + */ +public abstract class AbstractPushSource { + + private static class NullRecord implements Record { + @Override + public Object getValue() { + return null; + } + } + + private static class ErrorNotifierRecord implements Record { + private final Exception e; + public ErrorNotifierRecord(Exception e) { + this.e = e; + } + @Override + public Object getValue() { + return null; + } + + public Exception getException() { + return e; + } + } + + private LinkedBlockingQueue> queue; + private static final int DEFAULT_QUEUE_LENGTH = 1000; + private final NullRecord nullRecord = new NullRecord(); + + public AbstractPushSource() { + this.queue = new LinkedBlockingQueue<>(this.getQueueLength()); + } + + protected Record readNext() throws Exception { + Record record = queue.take(); + if (record instanceof ErrorNotifierRecord) { + throw ((ErrorNotifierRecord) record).getException(); + } + if (record instanceof NullRecord) { + return null; + } else { + return record; + } + } + + /** + * Send this message to be written to Pulsar. + * Pass null if you you are done with this task + * @param record next message from source which should be sent to a Pulsar topic + */ + public void consume(Record record) { + try { + if (record != null) { + queue.put(record); + } else { + queue.put(nullRecord); + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + /** + * Get length of the queue that records are push onto. + * Users can override this method to customize the queue length + * @return queue length + */ + public int getQueueLength() { + return DEFAULT_QUEUE_LENGTH; + } + + /** + * Allows the source to notify errors asynchronously. + * @param ex + */ + public void notifyError(Exception ex) { + consume(new ErrorNotifierRecord(ex)); + } +} diff --git a/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/BatchPushSource.java b/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/BatchPushSource.java index edf7b2756dd99..6a145b66ff03d 100644 --- a/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/BatchPushSource.java +++ b/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/BatchPushSource.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.io.core; -import java.util.concurrent.LinkedBlockingQueue; import org.apache.pulsar.common.classification.InterfaceAudience; import org.apache.pulsar.common.classification.InterfaceStability; import org.apache.pulsar.functions.api.Record; @@ -31,82 +30,10 @@ */ @InterfaceAudience.Public @InterfaceStability.Evolving -public abstract class BatchPushSource implements BatchSource { - - private static class NullRecord implements Record { - @Override - public Object getValue() { - return null; - } - } - - private static class ErrorNotifierRecord implements Record { - private Exception e; - public ErrorNotifierRecord(Exception e) { - this.e = e; - } - @Override - public Object getValue() { - return null; - } - - public Exception getException() { - return e; - } - } - - private LinkedBlockingQueue> queue; - private static final int DEFAULT_QUEUE_LENGTH = 1000; - private final NullRecord nullRecord = new NullRecord(); - - public BatchPushSource() { - this.queue = new LinkedBlockingQueue<>(this.getQueueLength()); - } +public abstract class BatchPushSource extends AbstractPushSource implements BatchSource { @Override public Record readNext() throws Exception { - Record record = queue.take(); - if (record instanceof ErrorNotifierRecord) { - throw ((ErrorNotifierRecord) record).getException(); - } - if (record instanceof NullRecord) { - return null; - } else { - return record; - } - } - - /** - * Send this message to be written to Pulsar. - * Pass null if you you are done with this task - * @param record next message from source which should be sent to a Pulsar topic - */ - public void consume(Record record) { - try { - if (record != null) { - queue.put(record); - } else { - queue.put(nullRecord); - } - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - /** - * Get length of the queue that records are push onto. - * Users can override this method to customize the queue length - * @return queue length - */ - public int getQueueLength() { - return DEFAULT_QUEUE_LENGTH; - } - - /** - * Allows the source to notify errors asynchronously. - * @param ex - */ - public void notifyError(Exception ex) { - consume(new ErrorNotifierRecord(ex)); + return super.readNext(); } -} \ No newline at end of file +} diff --git a/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/PushSource.java b/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/PushSource.java index dbd4dc4e1e9ff..6acccdda121db 100644 --- a/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/PushSource.java +++ b/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/PushSource.java @@ -18,8 +18,6 @@ */ package org.apache.pulsar.io.core; -import java.util.Map; -import java.util.concurrent.LinkedBlockingQueue; import org.apache.pulsar.common.classification.InterfaceAudience; import org.apache.pulsar.common.classification.InterfaceStability; import org.apache.pulsar.functions.api.Record; @@ -38,49 +36,10 @@ */ @InterfaceAudience.Public @InterfaceStability.Stable -public abstract class PushSource implements Source { - - private LinkedBlockingQueue> queue; - private static final int DEFAULT_QUEUE_LENGTH = 1000; - - public PushSource() { - this.queue = new LinkedBlockingQueue<>(this.getQueueLength()); - } +public abstract class PushSource extends AbstractPushSource implements Source { @Override public Record read() throws Exception { - return queue.take(); - } - - /** - * Open connector with configuration. - * - * @param config initialization config - * @param sourceContext environment where the source connector is running - * @throws Exception IO type exceptions when opening a connector - */ - public abstract void open(Map config, SourceContext sourceContext) throws Exception; - - /** - * Attach a consumer function to this Source. This is invoked by the implementation - * to pass messages whenever there is data to be pushed to Pulsar. - * - * @param record next message from source which should be sent to a Pulsar topic - */ - public void consume(Record record) { - try { - queue.put(record); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - /** - * Get length of the queue that records are push onto. - * Users can override this method to customize the queue length - * @return queue length - */ - public int getQueueLength() { - return DEFAULT_QUEUE_LENGTH; + return super.readNext(); } } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionImplRecyclableTest.java b/pulsar-io/core/src/test/java/org/apache/pulsar/io/core/PushSourceTest.java similarity index 60% rename from managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionImplRecyclableTest.java rename to pulsar-io/core/src/test/java/org/apache/pulsar/io/core/PushSourceTest.java index f46e3ec36b24c..3c23e6401f0dc 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionImplRecyclableTest.java +++ b/pulsar-io/core/src/test/java/org/apache/pulsar/io/core/PushSourceTest.java @@ -16,19 +16,28 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.bookkeeper.mledger.impl; +package org.apache.pulsar.io.core; -import static org.testng.Assert.assertNull; +import java.util.Map; import org.testng.annotations.Test; -public class PositionImplRecyclableTest { +public class PushSourceTest { + + PushSource testBatchSource = new PushSource() { + @Override + public void open(Map config, SourceContext context) throws Exception { - @Test - void shouldNotCarryStateInAckSetWhenRecycled() { - PositionImplRecyclable position = PositionImplRecyclable.create(); - position.ackSet = new long[]{1L, 2L, 3L}; - position.recycle(); - PositionImplRecyclable position2 = PositionImplRecyclable.create(); - assertNull(position2.ackSet); } -} \ No newline at end of file + + @Override + public void close() throws Exception { + + } + }; + + @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "test exception") + public void testNotifyErrors() throws Exception { + testBatchSource.notifyError(new RuntimeException("test exception")); + testBatchSource.readNext(); + } +} diff --git a/pulsar-io/data-generator/pom.xml b/pulsar-io/data-generator/pom.xml index 9e6e1f9ea6ebf..c78b3867b0fd0 100644 --- a/pulsar-io/data-generator/pom.xml +++ b/pulsar-io/data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-data-generator diff --git a/pulsar-io/debezium/core/pom.xml b/pulsar-io/debezium/core/pom.xml index 2e5a6f5c24da9..0b38982b8d36d 100644 --- a/pulsar-io/debezium/core/pom.xml +++ b/pulsar-io/debezium/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-debezium-core @@ -71,6 +71,10 @@ org.apache.kafka kafka-log4j-appender + + jose4j + org.bitbucket.b_c + diff --git a/pulsar-io/debezium/core/src/test/java/org/apache/pulsar/io/debezium/PulsarDatabaseHistoryTest.java b/pulsar-io/debezium/core/src/test/java/org/apache/pulsar/io/debezium/PulsarDatabaseHistoryTest.java index 1c5863e557e9a..cf7290f53d186 100644 --- a/pulsar-io/debezium/core/src/test/java/org/apache/pulsar/io/debezium/PulsarDatabaseHistoryTest.java +++ b/pulsar-io/debezium/core/src/test/java/org/apache/pulsar/io/debezium/PulsarDatabaseHistoryTest.java @@ -75,6 +75,7 @@ protected void setup() throws Exception { @Override protected void cleanup() throws Exception { super.internalCleanup(); + history.stop(); } private void testHistoryTopicContent(boolean skipUnparseableDDL, boolean testWithClientBuilder, boolean testWithReaderConfig) throws Exception { diff --git a/pulsar-io/debezium/mongodb/pom.xml b/pulsar-io/debezium/mongodb/pom.xml index ad57e0de02b77..b02a9074ea76f 100644 --- a/pulsar-io/debezium/mongodb/pom.xml +++ b/pulsar-io/debezium/mongodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-debezium-mongodb diff --git a/pulsar-io/debezium/mssql/pom.xml b/pulsar-io/debezium/mssql/pom.xml index 4db8477f185bc..071eed5a66198 100644 --- a/pulsar-io/debezium/mssql/pom.xml +++ b/pulsar-io/debezium/mssql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-debezium-mssql diff --git a/pulsar-io/debezium/mysql/pom.xml b/pulsar-io/debezium/mysql/pom.xml index 8c443f2f6b6df..4ee5dbbb8fb02 100644 --- a/pulsar-io/debezium/mysql/pom.xml +++ b/pulsar-io/debezium/mysql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-debezium-mysql diff --git a/pulsar-io/debezium/oracle/pom.xml b/pulsar-io/debezium/oracle/pom.xml index 9d6bffd38f2d2..423ea7d21dd55 100644 --- a/pulsar-io/debezium/oracle/pom.xml +++ b/pulsar-io/debezium/oracle/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-debezium-oracle diff --git a/pulsar-io/debezium/pom.xml b/pulsar-io/debezium/pom.xml index ba64915466710..d9feb840bd620 100644 --- a/pulsar-io/debezium/pom.xml +++ b/pulsar-io/debezium/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-debezium diff --git a/pulsar-io/debezium/postgres/pom.xml b/pulsar-io/debezium/postgres/pom.xml index 86fba2d3d1f34..672a51ecc0689 100644 --- a/pulsar-io/debezium/postgres/pom.xml +++ b/pulsar-io/debezium/postgres/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-debezium-postgres diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index 0c772104dbbfc..ac4ae9496d1bb 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-docs @@ -42,8 +42,8 @@ reflections - com.beust - jcommander + info.picocli + picocli @@ -57,6 +57,11 @@ pulsar-io-alluxio ${project.version} + + ${project.groupId} + pulsar-io-azuredataexplorer + ${project.version} + ${project.groupId} pulsar-io-canal @@ -253,7 +258,6 @@ org.owasp dependency-check-maven - ${dependency-check-maven.version} diff --git a/pulsar-io/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java b/pulsar-io/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java index fec7b12087977..2e9d6a9f27acc 100644 --- a/pulsar-io/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java +++ b/pulsar-io/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java @@ -18,8 +18,6 @@ */ package org.apache.pulsar.io.docs; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; import com.google.common.base.Strings; import java.io.File; import java.io.FileOutputStream; @@ -34,14 +32,19 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.concurrent.Callable; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.io.core.annotations.Connector; import org.apache.pulsar.io.core.annotations.FieldDoc; import org.reflections.Reflections; import org.reflections.util.ConfigurationBuilder; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; @Slf4j -public class ConnectorDocGenerator { +@Command(name = "connector-doc-gen") +public class ConnectorDocGenerator implements Callable { private static final String INDENT = " "; @@ -118,41 +121,25 @@ private void generatorConnectorYamlFiles(String outputDir) throws IOException { } } - /** - * Args for stats generator. - */ - private static class MainArgs { - @Parameter( - names = {"-o", "--output-dir"}, - description = "The output dir to dump connector docs", - required = true) - String outputDir = null; - - @Parameter(names = {"-h", "--help"}, description = "Show this help message") - boolean help = false; - } + @Option( + names = {"-o", "--output-dir"}, + description = "The output dir to dump connector docs", + required = true) + String outputDir = null; - public static void main(String[] args) throws Exception { - MainArgs mainArgs = new MainArgs(); - - JCommander commander = new JCommander(); - try { - commander.setProgramName("connector-doc-gen"); - commander.addObject(mainArgs); - commander.parse(args); - if (mainArgs.help) { - commander.usage(); - Runtime.getRuntime().exit(0); - return; - } - } catch (Exception e) { - commander.usage(); - Runtime.getRuntime().exit(1); - return; - } + @Option(names = {"-h", "--help"}, usageHelp = true, description = "Show this help message") + boolean help = false; + @Override + public Integer call() throws Exception { ConnectorDocGenerator docGen = new ConnectorDocGenerator(); - docGen.generatorConnectorYamlFiles(mainArgs.outputDir); + docGen.generatorConnectorYamlFiles(outputDir); + return 0; + } + + public static void main(String[] args) throws Exception { + CommandLine commander = new CommandLine(new ConnectorDocGenerator()); + Runtime.getRuntime().exit(commander.execute(args)); } } diff --git a/pulsar-io/dynamodb/pom.xml b/pulsar-io/dynamodb/pom.xml index 46a5154ab8021..cd58da5c9a4fb 100644 --- a/pulsar-io/dynamodb/pom.xml +++ b/pulsar-io/dynamodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-dynamodb @@ -32,6 +32,12 @@ + + ${project.groupId} + pulsar-io-common + ${project.version} + + ${project.groupId} pulsar-io-core diff --git a/pulsar-io/dynamodb/src/main/java/org/apache/pulsar/io/dynamodb/DynamoDBSource.java b/pulsar-io/dynamodb/src/main/java/org/apache/pulsar/io/dynamodb/DynamoDBSource.java index d67c4e21154ee..2193cf39c17a5 100644 --- a/pulsar-io/dynamodb/src/main/java/org/apache/pulsar/io/dynamodb/DynamoDBSource.java +++ b/pulsar-io/dynamodb/src/main/java/org/apache/pulsar/io/dynamodb/DynamoDBSource.java @@ -65,7 +65,7 @@ public void close() throws Exception { @Override public void open(Map config, SourceContext sourceContext) throws Exception { - this.dynamodbSourceConfig = DynamoDBSourceConfig.load(config); + this.dynamodbSourceConfig = DynamoDBSourceConfig.load(config, sourceContext); checkArgument(isNotBlank(dynamodbSourceConfig.getAwsDynamodbStreamArn()), "empty dynamo-stream arn"); // Even if the endpoint is set, it seems to require a region to go with it diff --git a/pulsar-io/dynamodb/src/main/java/org/apache/pulsar/io/dynamodb/DynamoDBSourceConfig.java b/pulsar-io/dynamodb/src/main/java/org/apache/pulsar/io/dynamodb/DynamoDBSourceConfig.java index b734dd5741155..0547ff8f863e0 100644 --- a/pulsar-io/dynamodb/src/main/java/org/apache/pulsar/io/dynamodb/DynamoDBSourceConfig.java +++ b/pulsar-io/dynamodb/src/main/java/org/apache/pulsar/io/dynamodb/DynamoDBSourceConfig.java @@ -35,6 +35,8 @@ import java.util.Map; import lombok.Data; import org.apache.pulsar.io.aws.AwsCredentialProviderPlugin; +import org.apache.pulsar.io.common.IOConfigUtils; +import org.apache.pulsar.io.core.SourceContext; import org.apache.pulsar.io.core.annotations.FieldDoc; import software.amazon.awssdk.regions.Region; @@ -77,6 +79,7 @@ public class DynamoDBSourceConfig implements Serializable { @FieldDoc( required = false, defaultValue = "", + sensitive = true, help = "json-parameters to initialize `AwsCredentialsProviderPlugin`") private String awsCredentialPluginParam = ""; @@ -170,9 +173,8 @@ public static DynamoDBSourceConfig load(String yamlFile) throws IOException { return mapper.readValue(new File(yamlFile), DynamoDBSourceConfig.class); } - public static DynamoDBSourceConfig load(Map map) throws IOException { - ObjectMapper mapper = new ObjectMapper(); - return mapper.readValue(mapper.writeValueAsString(map), DynamoDBSourceConfig.class); + public static DynamoDBSourceConfig load(Map map, SourceContext sourceContext) throws IOException { + return IOConfigUtils.loadWithSecrets(map, DynamoDBSourceConfig.class, sourceContext); } protected Region regionAsV2Region() { diff --git a/pulsar-io/dynamodb/src/test/java/org/apache/pulsar/io/dynamodb/DynamoDBSourceConfigTests.java b/pulsar-io/dynamodb/src/test/java/org/apache/pulsar/io/dynamodb/DynamoDBSourceConfigTests.java index f84cb785896e6..bdccaa2e5846e 100644 --- a/pulsar-io/dynamodb/src/test/java/org/apache/pulsar/io/dynamodb/DynamoDBSourceConfigTests.java +++ b/pulsar-io/dynamodb/src/test/java/org/apache/pulsar/io/dynamodb/DynamoDBSourceConfigTests.java @@ -31,6 +31,8 @@ import java.util.Map; import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream; +import org.apache.pulsar.io.core.SourceContext; +import org.mockito.Mockito; import org.testng.annotations.Test; @@ -90,7 +92,8 @@ public final void loadFromMapTest() throws IOException { map.put("initialPositionInStream", InitialPositionInStream.TRIM_HORIZON); map.put("startAtTime", DAY); - DynamoDBSourceConfig config = DynamoDBSourceConfig.load(map); + SourceContext sourceContext = Mockito.mock(SourceContext.class); + DynamoDBSourceConfig config = DynamoDBSourceConfig.load(map, sourceContext); assertNotNull(config); assertEquals(config.getAwsEndpoint(), "https://some.endpoint.aws"); @@ -111,7 +114,46 @@ public final void loadFromMapTest() throws IOException { ZonedDateTime expected = ZonedDateTime.ofInstant(DAY.toInstant(), ZoneOffset.UTC); assertEquals(actual, expected); } - + + @Test + public final void loadFromMapCredentialFromSecretTest() throws IOException { + Map map = new HashMap (); + map.put("awsEndpoint", "https://some.endpoint.aws"); + map.put("awsRegion", "us-east-1"); + map.put("awsDynamodbStreamArn", "arn:aws:dynamodb:us-west-2:111122223333:table/TestTable/stream/2015-05-11T21:21:33.291"); + map.put("checkpointInterval", "30000"); + map.put("backoffTime", "4000"); + map.put("numRetries", "3"); + map.put("receiveQueueSize", 2000); + map.put("applicationName", "My test application"); + map.put("initialPositionInStream", InitialPositionInStream.TRIM_HORIZON); + map.put("startAtTime", DAY); + + SourceContext sourceContext = Mockito.mock(SourceContext.class); + Mockito.when(sourceContext.getSecret("awsCredentialPluginParam")) + .thenReturn("{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}"); + DynamoDBSourceConfig config = DynamoDBSourceConfig.load(map, sourceContext); + + assertNotNull(config); + assertEquals(config.getAwsEndpoint(), "https://some.endpoint.aws"); + assertEquals(config.getAwsRegion(), "us-east-1"); + assertEquals(config.getAwsDynamodbStreamArn(), "arn:aws:dynamodb:us-west-2:111122223333:table/TestTable/stream/2015-05-11T21:21:33.291"); + assertEquals(config.getAwsCredentialPluginParam(), + "{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}"); + assertEquals(config.getApplicationName(), "My test application"); + assertEquals(config.getCheckpointInterval(), 30000); + assertEquals(config.getBackoffTime(), 4000); + assertEquals(config.getNumRetries(), 3); + assertEquals(config.getReceiveQueueSize(), 2000); + assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.TRIM_HORIZON); + + Calendar cal = Calendar.getInstance(); + cal.setTime(config.getStartAtTime()); + ZonedDateTime actual = ZonedDateTime.ofInstant(cal.toInstant(), ZoneOffset.UTC); + ZonedDateTime expected = ZonedDateTime.ofInstant(DAY.toInstant(), ZoneOffset.UTC); + assertEquals(actual, expected); + } + @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "empty aws-credential param") public final void missingCredentialsTest() throws Exception { @@ -121,7 +163,8 @@ public final void missingCredentialsTest() throws Exception { map.put("awsDynamodbStreamArn", "arn:aws:dynamodb:us-west-2:111122223333:table/TestTable/stream/2015-05-11T21:21:33.291"); DynamoDBSource source = new DynamoDBSource(); - source.open(map, null); + SourceContext sourceContext = Mockito.mock(SourceContext.class); + source.open(map, sourceContext); } @Test(expectedExceptions = IllegalArgumentException.class, @@ -136,7 +179,8 @@ public final void missingStartTimeTest() throws Exception { map.put("initialPositionInStream", InitialPositionInStream.AT_TIMESTAMP); DynamoDBSource source = new DynamoDBSource(); - source.open(map, null); + SourceContext sourceContext = Mockito.mock(SourceContext.class); + source.open(map, sourceContext); } private File getFile(String name) { diff --git a/pulsar-io/elastic-search/pom.xml b/pulsar-io/elastic-search/pom.xml index 24c0bdc052841..8507d9e4efea5 100644 --- a/pulsar-io/elastic-search/pom.xml +++ b/pulsar-io/elastic-search/pom.xml @@ -23,16 +23,12 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-elastic-search Pulsar IO :: ElasticSearch - - false 1 diff --git a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClient.java b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClient.java index d6fb5bb705d86..3b2359f16e8e2 100644 --- a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClient.java +++ b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClient.java @@ -33,6 +33,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.client.api.schema.GenericObject; import org.apache.pulsar.functions.api.Record; +import org.apache.pulsar.io.core.SinkContext; import org.apache.pulsar.io.elasticsearch.client.BulkProcessor; import org.apache.pulsar.io.elasticsearch.client.RestClient; import org.apache.pulsar.io.elasticsearch.client.RestClientFactory; @@ -53,11 +54,19 @@ public class ElasticSearchClient implements AutoCloseable { final Set indexCache = new HashSet<>(); final Map topicToIndexCache = new HashMap<>(); - final AtomicReference irrecoverableError = new AtomicReference<>(); + final AtomicReference state = new AtomicReference<>(State.Open); + private final IndexNameFormatter indexNameFormatter; - public ElasticSearchClient(ElasticSearchConfig elasticSearchConfig) { + enum State { + Open, Failed, Closed + } + + final SinkContext sinkContext; + + public ElasticSearchClient(ElasticSearchConfig elasticSearchConfig, SinkContext sinkContext) { this.config = elasticSearchConfig; + this.sinkContext = sinkContext; if (this.config.getIndexName() != null) { this.indexNameFormatter = new IndexNameFormatter(this.config.getIndexName()); } else { @@ -94,18 +103,15 @@ public void afterBulk(long executionId, List }; this.backoffRetry = new RandomExponentialRetry(elasticSearchConfig.getMaxRetryTimeInSec()); this.client = retry(() -> RestClientFactory.createClient(config, bulkListener), -1, "client creation"); + state.set(State.Open); } void failed(Exception e) { - if (irrecoverableError.compareAndSet(null, e)) { - log.error("Irrecoverable error:", e); + if (state.compareAndSet(State.Open, State.Failed)) { + sinkContext.fatal(e); } } - boolean isFailed() { - return irrecoverableError.get() != null; - } - void checkForIrrecoverableError(Record record, BulkProcessor.BulkOperationResult result) { if (!result.isError()) { return; @@ -145,7 +151,7 @@ void checkForIrrecoverableError(Record record, BulkProcessor.BulkOperationRes public void bulkIndex(Record record, Pair idAndDoc) throws Exception { try { - checkNotFailed(); + checkState(); checkIndexExists(record); final String indexName = indexName(record); final String documentId = idAndDoc.getLeft(); @@ -174,7 +180,7 @@ public void bulkIndex(Record record, Pair idAndDoc) throws Excep */ public boolean indexDocument(Record record, Pair idAndDoc) throws Exception { try { - checkNotFailed(); + checkState(); checkIndexExists(record); final String indexName = indexName(record); @@ -197,7 +203,7 @@ public boolean indexDocument(Record record, Pair public void bulkDelete(Record record, String id) throws Exception { try { - checkNotFailed(); + checkState(); checkIndexExists(record); final String indexName = indexName(record); @@ -224,7 +230,7 @@ public void bulkDelete(Record record, String id) throws Exception */ public boolean deleteDocument(Record record, String id) throws Exception { try { - checkNotFailed(); + checkState(); checkIndexExists(record); final String indexName = indexName(record); final boolean deleted = client.deleteDocument(indexName, id); @@ -254,6 +260,7 @@ public void close() { client.close(); client = null; } + state.compareAndSet(State.Open, State.Closed); } @VisibleForTesting @@ -261,9 +268,9 @@ void setClient(RestClient client) { this.client = client; } - private void checkNotFailed() throws Exception { - if (irrecoverableError.get() != null) { - throw irrecoverableError.get(); + private void checkState() { + if (state.get() != State.Open) { + throw new IllegalStateException(String.format("Elasticsearch client is in %s state", state.get().name())); } } diff --git a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java index 9f42dbda7be1b..33c2d34a1c992 100644 --- a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java +++ b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java @@ -59,10 +59,12 @@ public class ElasticSearchConfig implements Serializable { ) private String indexName; + @Deprecated @FieldDoc( required = false, defaultValue = "_doc", - help = "The type name that the connector writes messages to, with the default value set to _doc." + help = "No longer in use in OpenSearch 2+. " + + "The type name that the connector writes messages to, with the default value set to _doc." + " This value should be set explicitly to a valid type name other than _doc for Elasticsearch version before 6.2," + " and left to the default value otherwise." ) diff --git a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSink.java b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSink.java index e2566d2063872..6d38775bd0524 100644 --- a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSink.java +++ b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSink.java @@ -29,6 +29,7 @@ import com.google.common.base.Strings; import com.google.common.hash.Hasher; import com.google.common.hash.Hashing; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -75,7 +76,7 @@ public class ElasticSearchSink implements Sink { public void open(Map config, SinkContext sinkContext) throws Exception { elasticSearchConfig = ElasticSearchConfig.load(config, sinkContext); elasticSearchConfig.validate(); - elasticsearchClient = new ElasticSearchClient(elasticSearchConfig); + elasticsearchClient = new ElasticSearchClient(elasticSearchConfig, sinkContext); if (!Strings.isNullOrEmpty(elasticSearchConfig.getPrimaryFields())) { primaryFields = Arrays.asList(elasticSearchConfig.getPrimaryFields().split(",")); } @@ -109,60 +110,55 @@ void setElasticsearchClient(ElasticSearchClient elasticsearchClient) { @Override public void write(Record record) throws Exception { - if (!elasticsearchClient.isFailed()) { - Pair idAndDoc = extractIdAndDocument(record); - try { - if (log.isDebugEnabled()) { - log.debug("index doc {} {}", idAndDoc.getLeft(), idAndDoc.getRight()); - } - if (idAndDoc.getRight() == null) { - switch (elasticSearchConfig.getNullValueAction()) { - case DELETE: - if (idAndDoc.getLeft() != null) { - if (elasticSearchConfig.isBulkEnabled()) { - elasticsearchClient.bulkDelete(record, idAndDoc.getLeft()); - } else { - elasticsearchClient.deleteDocument(record, idAndDoc.getLeft()); - } + Pair idAndDoc = extractIdAndDocument(record); + try { + if (log.isDebugEnabled()) { + log.debug("index doc {} {}", idAndDoc.getLeft(), idAndDoc.getRight()); + } + if (idAndDoc.getRight() == null) { + switch (elasticSearchConfig.getNullValueAction()) { + case DELETE: + if (idAndDoc.getLeft() != null) { + if (elasticSearchConfig.isBulkEnabled()) { + elasticsearchClient.bulkDelete(record, idAndDoc.getLeft()); + } else { + elasticsearchClient.deleteDocument(record, idAndDoc.getLeft()); } - break; - case IGNORE: - break; - case FAIL: - elasticsearchClient.failed( - new PulsarClientException.InvalidMessageException("Unexpected null message value")); - throw elasticsearchClient.irrecoverableError.get(); - } - } else { - if (elasticSearchConfig.isBulkEnabled()) { - elasticsearchClient.bulkIndex(record, idAndDoc); - } else { - elasticsearchClient.indexDocument(record, idAndDoc); - } - } - } catch (JsonProcessingException jsonProcessingException) { - switch (elasticSearchConfig.getMalformedDocAction()) { + } + break; case IGNORE: break; - case WARN: - log.warn("Ignoring malformed document messageId={}", - record.getMessage().map(Message::getMessageId).orElse(null), - jsonProcessingException); - elasticsearchClient.failed(jsonProcessingException); - throw jsonProcessingException; case FAIL: - log.error("Malformed document messageId={}", - record.getMessage().map(Message::getMessageId).orElse(null), - jsonProcessingException); - elasticsearchClient.failed(jsonProcessingException); - throw jsonProcessingException; + elasticsearchClient.failed( + new PulsarClientException.InvalidMessageException("Unexpected null message value")); + } + } else { + if (elasticSearchConfig.isBulkEnabled()) { + elasticsearchClient.bulkIndex(record, idAndDoc); + } else { + elasticsearchClient.indexDocument(record, idAndDoc); } - } catch (Exception e) { - log.error("write error for {} {}:", idAndDoc.getLeft(), idAndDoc.getRight(), e); - throw e; } - } else { - throw new IllegalStateException("Elasticsearch client is in FAILED status"); + } catch (JsonProcessingException jsonProcessingException) { + switch (elasticSearchConfig.getMalformedDocAction()) { + case IGNORE: + break; + case WARN: + log.warn("Ignoring malformed document messageId={}", + record.getMessage().map(Message::getMessageId).orElse(null), + jsonProcessingException); + elasticsearchClient.failed(jsonProcessingException); + break; + case FAIL: + log.error("Malformed document messageId={}", + record.getMessage().map(Message::getMessageId).orElse(null), + jsonProcessingException); + elasticsearchClient.failed(jsonProcessingException); + break; + } + } catch (Exception e) { + log.error("write error for {} {}:", idAndDoc.getLeft(), idAndDoc.getRight(), e); + throw e; } } @@ -194,12 +190,16 @@ public Pair extractIdAndDocument(Record record) t } else { key = record.getKey().orElse(null); valueSchema = record.getSchema(); - value = record.getValue(); + value = getGenericObjectFromRecord(record); } String id = null; - if (!elasticSearchConfig.isKeyIgnore() && key != null && keySchema != null) { - id = stringifyKey(keySchema, key); + if (!elasticSearchConfig.isKeyIgnore() && key != null) { + if (keySchema != null){ + id = stringifyKey(keySchema, key); + } else { + id = key.toString(); + } } String doc = null; @@ -278,16 +278,42 @@ public Pair extractIdAndDocument(Record record) t doc = sanitizeValue(doc); return Pair.of(id, doc); } else { - final byte[] data = record - .getMessage() - .orElseThrow(() -> new IllegalArgumentException("Record does not carry message information")) - .getData(); - String doc = new String(data, StandardCharsets.UTF_8); - doc = sanitizeValue(doc); - return Pair.of(null, doc); + Message message = record.getMessage().orElse(null); + final String rawData; + if (message != null) { + rawData = new String(message.getData(), StandardCharsets.UTF_8); + } else { + GenericObject recordObject = getGenericObjectFromRecord(record); + rawData = stringifyValue(record.getSchema(), recordObject); + } + if (rawData == null || rawData.length() == 0){ + throw new IllegalArgumentException("Record does not carry message information."); + } + String key = elasticSearchConfig.isKeyIgnore() ? null : record.getKey().map(Object::toString).orElse(null); + return Pair.of(key, sanitizeValue(rawData)); } } + private GenericObject getGenericObjectFromRecord(Record record){ + if (record.getValue() == null) { + return null; + } + if (record.getValue() instanceof GenericObject){ + return (GenericObject) record.getValue(); + } + return new GenericObject() { + @Override + public SchemaType getSchemaType() { + return record.getSchema().getSchemaInfo().getType(); + } + + @Override + public Object getNativeObject() { + return record.getValue(); + } + }; + } + private String sanitizeValue(String value) { if (value == null || !elasticSearchConfig.isStripNonPrintableCharacters()) { return value; @@ -373,17 +399,38 @@ public static JsonNode stripNullNodes(JsonNode node) { return node; } - public static JsonNode extractJsonNode(Schema schema, Object val) { + public JsonNode extractJsonNode(Schema schema, Object val) throws JsonProcessingException { if (val == null) { return null; } switch (schema.getSchemaInfo().getType()) { case JSON: - return (JsonNode) ((GenericRecord) val).getNativeObject(); + Object nativeObject = ((GenericRecord) val).getNativeObject(); + if (nativeObject instanceof String) { + try { + return objectMapper.readTree((String) nativeObject); + } catch (JsonProcessingException e) { + log.error("Failed to read JSON string: {}", nativeObject, e); + throw e; + } + } + return (JsonNode) nativeObject; case AVRO: org.apache.avro.generic.GenericRecord node = (org.apache.avro.generic.GenericRecord) ((GenericRecord) val).getNativeObject(); return JsonConverter.toJson(node); + case STRING: + try { + return objectMapper.readTree((String) ((GenericObject) val).getNativeObject()); + } catch (JsonProcessingException e) { + throw new RuntimeException("Error parsing string as JSON.", e); + } + case BYTES: + try { + return objectMapper.readTree((byte[]) ((GenericObject) val).getNativeObject()); + } catch (IOException e) { + throw new RuntimeException("Error parsing byte[] as JSON.", e); + } default: throw new UnsupportedOperationException("Unsupported value schemaType=" + schema.getSchemaInfo().getType()); diff --git a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/client/elastic/ElasticSearchJavaRestClient.java b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/client/elastic/ElasticSearchJavaRestClient.java index 4749ea2e2d383..133daa8cd6a68 100644 --- a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/client/elastic/ElasticSearchJavaRestClient.java +++ b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/client/elastic/ElasticSearchJavaRestClient.java @@ -84,6 +84,7 @@ public ElasticSearchJavaRestClient(ElasticSearchConfig elasticSearchConfig, .setConnectionRequestTimeout(config.getConnectionRequestTimeoutInMs()) .setConnectTimeout(config.getConnectTimeoutInMs()) .setSocketTimeout(config.getSocketTimeoutInMs())) + .setCompressionEnabled(config.isCompressionEnabled()) .setHttpClientConfigCallback(this.configCallback) .setFailureListener(new org.elasticsearch.client.RestClient.FailureListener() { public void onFailure(Node node) { @@ -143,7 +144,7 @@ public boolean deleteIndex(String index) throws IOException { public boolean deleteDocument(String index, String documentId) throws IOException { final DeleteRequest req = new DeleteRequest.Builder() - .index(config.getIndexName()) + .index(index) .id(documentId) .build(); @@ -155,7 +156,7 @@ public boolean deleteDocument(String index, String documentId) throws IOExceptio public boolean indexDocument(String index, String documentId, String documentSource) throws IOException { final Map mapped = objectMapper.readValue(documentSource, Map.class); final IndexRequest indexRequest = new IndexRequest.Builder<>() - .index(config.getIndexName()) + .index(index) .document(mapped) .id(documentId) .build(); diff --git a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/client/opensearch/OpenSearchHighLevelRestClient.java b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/client/opensearch/OpenSearchHighLevelRestClient.java index 7b7041967026e..87c4913529f04 100644 --- a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/client/opensearch/OpenSearchHighLevelRestClient.java +++ b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/client/opensearch/OpenSearchHighLevelRestClient.java @@ -49,12 +49,12 @@ import org.opensearch.client.indices.CreateIndexRequest; import org.opensearch.client.indices.CreateIndexResponse; import org.opensearch.client.indices.GetIndexRequest; -import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.Strings; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.builder.SearchSourceBuilder; @@ -112,6 +112,7 @@ public OpenSearchHighLevelRestClient(ElasticSearchConfig elasticSearchConfig, .setConnectionRequestTimeout(config.getConnectionRequestTimeoutInMs()) .setConnectTimeout(config.getConnectTimeoutInMs()) .setSocketTimeout(config.getSocketTimeoutInMs())) + .setCompressionEnabled(config.isCompressionEnabled()) .setHttpClientConfigCallback(this.configCallback) .setFailureListener(new org.opensearch.client.RestClient.FailureListener() { @Override @@ -228,7 +229,6 @@ public boolean indexDocument(String index, String documentId, String documentSou if (!Strings.isNullOrEmpty(documentId)) { indexRequest.id(documentId); } - indexRequest.type(config.getTypeName()); indexRequest.source(documentSource, XContentType.JSON); IndexResponse indexResponse = client.index(indexRequest, RequestOptions.DEFAULT); @@ -244,7 +244,6 @@ public boolean indexDocument(String index, String documentId, String documentSou public boolean deleteDocument(String index, String documentId) throws IOException { DeleteRequest deleteRequest = Requests.deleteRequest(index); deleteRequest.id(documentId); - deleteRequest.type(config.getTypeName()); DeleteResponse deleteResponse = client.delete(deleteRequest, RequestOptions.DEFAULT); if (log.isDebugEnabled()) { log.debug("delete result {}", deleteResponse.getResult()); @@ -300,7 +299,6 @@ public void appendIndexRequest(BulkProcessor.BulkIndexRequest request) throws IO if (!Strings.isNullOrEmpty(request.getDocumentId())) { indexRequest.id(request.getDocumentId()); } - indexRequest.type(config.getTypeName()); indexRequest.source(request.getDocumentSource(), XContentType.JSON); internalBulkProcessor.add(indexRequest); } @@ -309,7 +307,6 @@ public void appendIndexRequest(BulkProcessor.BulkIndexRequest request) throws IO public void appendDeleteRequest(BulkProcessor.BulkDeleteRequest request) throws IOException { DeleteRequest deleteRequest = new DeleteRequestWithPulsarRecord(request.getIndex(), request.getRecord()); deleteRequest.id(request.getDocumentId()); - deleteRequest.type(config.getTypeName()); internalBulkProcessor.add(deleteRequest); } diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchAuthTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchAuthTests.java index 7c56bfc23c96d..db899caaa3931 100644 --- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchAuthTests.java +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchAuthTests.java @@ -18,39 +18,34 @@ */ package org.apache.pulsar.io.elasticsearch; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.functions.api.Record; +import org.apache.pulsar.io.core.SinkContext; import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import java.io.IOException; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.expectThrows; - @Slf4j public abstract class ElasticSearchAuthTests extends ElasticSearchTestBase { public static final String ELASTICPWD = "elastic"; - static ElasticsearchContainer container; + private ElasticsearchContainer container; public ElasticSearchAuthTests(String elasticImageName) { super(elasticImageName); } - @BeforeMethod(alwaysRun = true) - public void initBeforeClass() throws IOException { - if (container != null) { - return; - } + @BeforeClass(alwaysRun = true) + public void initBeforeClass() { container = createElasticsearchContainer() .withEnv("xpack.security.enabled", "true") .withEnv("xpack.security.authc.token.enabled", "true") @@ -61,7 +56,7 @@ public void initBeforeClass() throws IOException { } @AfterClass(alwaysRun = true) - public static void closeAfterClass() { + public void closeAfterClass() { if (container != null) { container.close(); } @@ -79,7 +74,7 @@ public void testBasicAuth() throws Exception { config.setMaxRetries(1); config.setBulkEnabled(true); // ensure auth is needed - try (ElasticSearchClient client = new ElasticSearchClient(config);) { + try (ElasticSearchClient client = new ElasticSearchClient(config, mock(SinkContext.class));) { expectThrows(ElasticSearchConnectionException.class, () -> { client.createIndexIfNeeded(indexName); }); @@ -87,7 +82,7 @@ public void testBasicAuth() throws Exception { config.setPassword(ELASTICPWD); - try (ElasticSearchClient client = new ElasticSearchClient(config);) { + try (ElasticSearchClient client = new ElasticSearchClient(config, mock(SinkContext.class));) { ensureCalls(client, indexName); } } @@ -106,7 +101,7 @@ public void testTokenAuth() throws Exception { config.setPassword(ELASTICPWD); String token; - try (ElasticSearchClient client = new ElasticSearchClient(config);) { + try (ElasticSearchClient client = new ElasticSearchClient(config, mock(SinkContext.class));) { token = createAuthToken(client, "elastic", ELASTICPWD); } @@ -114,14 +109,14 @@ public void testTokenAuth() throws Exception { config.setPassword(null); // ensure auth is needed - try (ElasticSearchClient client = new ElasticSearchClient(config);) { + try (ElasticSearchClient client = new ElasticSearchClient(config, mock(SinkContext.class));) { expectThrows(ElasticSearchConnectionException.class, () -> { client.createIndexIfNeeded(indexName); }); } config.setToken(token); - try (ElasticSearchClient client = new ElasticSearchClient(config);) { + try (ElasticSearchClient client = new ElasticSearchClient(config, mock(SinkContext.class));) { ensureCalls(client, indexName); } } @@ -139,7 +134,7 @@ public void testApiKey() throws Exception { config.setPassword(ELASTICPWD); String apiKey; - try (ElasticSearchClient client = new ElasticSearchClient(config);) { + try (ElasticSearchClient client = new ElasticSearchClient(config, mock(SinkContext.class));) { apiKey = createApiKey(client); } @@ -147,14 +142,14 @@ public void testApiKey() throws Exception { config.setPassword(null); // ensure auth is needed - try (ElasticSearchClient client = new ElasticSearchClient(config);) { + try (ElasticSearchClient client = new ElasticSearchClient(config, mock(SinkContext.class));) { expectThrows(ElasticSearchConnectionException.class, () -> { client.createIndexIfNeeded(indexName); }); } config.setApiKey(apiKey); - try (ElasticSearchClient client = new ElasticSearchClient(config);) { + try (ElasticSearchClient client = new ElasticSearchClient(config, mock(SinkContext.class));) { ensureCalls(client, indexName); } } diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientSslTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientSslTests.java index 5598a88d410a9..5e0b2a029b88c 100644 --- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientSslTests.java +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientSslTests.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.io.elasticsearch; +import org.apache.pulsar.io.core.SinkContext; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testcontainers.utility.MountableFile; @@ -26,6 +27,7 @@ import java.io.IOException; import java.time.Duration; +import static org.mockito.Mockito.mock; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -183,7 +185,7 @@ public void testSslDisableCertificateValidation() throws IOException { } private void testClientWithConfig(ElasticSearchConfig config) throws IOException { - try (ElasticSearchClient client = new ElasticSearchClient(config);) { + try (ElasticSearchClient client = new ElasticSearchClient(config, mock(SinkContext.class));) { testIndexExists(client); } } diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java index 6d9928c042697..468d78d989cf1 100644 --- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java @@ -20,6 +20,9 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; @@ -27,50 +30,52 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; +import co.elastic.clients.transport.rest_client.RestClientTransport; import eu.rekawek.toxiproxy.model.ToxicDirection; import java.io.IOException; +import java.lang.reflect.Field; import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.LongAdder; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.client.api.schema.GenericObject; import org.apache.pulsar.functions.api.Record; +import org.apache.pulsar.io.core.SinkContext; import org.apache.pulsar.io.elasticsearch.client.elastic.ElasticSearchJavaRestClient; import org.apache.pulsar.io.elasticsearch.client.opensearch.OpenSearchHighLevelRestClient; import org.apache.pulsar.io.elasticsearch.testcontainers.ElasticToxiproxiContainer; import org.awaitility.Awaitility; -import org.mockito.Mockito; +import org.opensearch.client.RestClient; +import org.opensearch.client.RestHighLevelClient; import org.testcontainers.containers.Network; import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @Slf4j public abstract class ElasticSearchClientTests extends ElasticSearchTestBase { public final static String INDEX = "myindex"; - static ElasticsearchContainer container; - static Network network; + private ElasticsearchContainer container; + private Network network; public ElasticSearchClientTests(String elasticImageName) { super(elasticImageName); } - @BeforeMethod(alwaysRun = true) + @BeforeClass(alwaysRun = true) public void initBeforeClass() throws IOException { - if (container != null) { - return; - } network = Network.newNetwork(); container = createElasticsearchContainer().withNetwork(network); container.start(); } @AfterClass(alwaysRun = true) - public static void closeAfterClass() { + public void closeAfterClass() { container.close(); container = null; network.close(); @@ -109,11 +114,41 @@ public void fail() { public void testClientInstance() throws Exception { try (ElasticSearchClient client = new ElasticSearchClient(new ElasticSearchConfig() .setElasticSearchUrl("http://" + container.getHttpHostAddress()) - .setIndexName(INDEX))) { + .setCompressionEnabled(true) + .setIndexName(INDEX), mock(SinkContext.class));) { if (elasticImageName.equals(OPENSEARCH) || elasticImageName.equals(ELASTICSEARCH_7)) { assertTrue(client.getRestClient() instanceof OpenSearchHighLevelRestClient); + OpenSearchHighLevelRestClient osRestHighLevelClient = (OpenSearchHighLevelRestClient) client.getRestClient(); + RestHighLevelClient restHighLevelClient = osRestHighLevelClient.getClient(); + assertNotNull(restHighLevelClient); + + Field field = RestHighLevelClient.class.getDeclaredField("client"); + field.setAccessible(true); + RestClient restClient = (RestClient) field.get(restHighLevelClient); + assertNotNull(restClient); + + Field compressionEnabledFiled = RestClient.class.getDeclaredField("compressionEnabled"); + compressionEnabledFiled.setAccessible(true); + boolean compressionEnabled = (boolean) compressionEnabledFiled.get(restClient); + assertTrue(compressionEnabled); } else { assertTrue(client.getRestClient() instanceof ElasticSearchJavaRestClient); + ElasticSearchJavaRestClient javaRestClient = (ElasticSearchJavaRestClient) client.getRestClient(); + + Field field = ElasticSearchJavaRestClient.class.getDeclaredField("transport"); + field.setAccessible(true); + RestClientTransport transport = (RestClientTransport) field.get(javaRestClient); + assertNotNull(transport); + + Field restClientFiled = RestClientTransport.class.getDeclaredField("restClient"); + restClientFiled.setAccessible(true); + org.elasticsearch.client.RestClient restClient = (org.elasticsearch.client.RestClient) restClientFiled.get(transport); + assertNotNull(restClient); + + Field compressionEnabledFiled = org.elasticsearch.client.RestClient.class.getDeclaredField("compressionEnabled"); + compressionEnabledFiled.setAccessible(true); + boolean compressionEnabled = (boolean) compressionEnabledFiled.get(restClient); + assertTrue(compressionEnabled); } } } @@ -121,23 +156,23 @@ public void testClientInstance() throws Exception { @Test public void testIndexName() throws Exception { String index = "myindex-" + UUID.randomUUID(); - Record record = Mockito.mock(Record.class); + Record record = mock(Record.class); String topicName = "topic-" + UUID.randomUUID(); when(record.getTopicName()).thenReturn(Optional.of(topicName)); try (ElasticSearchClient client = new ElasticSearchClient(new ElasticSearchConfig() .setElasticSearchUrl("http://" + container.getHttpHostAddress()) - .setIndexName(index))) { + .setIndexName(index), mock(SinkContext.class))) { assertEquals(client.indexName(record), index); } try (ElasticSearchClient client = new ElasticSearchClient(new ElasticSearchConfig() - .setElasticSearchUrl("http://" + container.getHttpHostAddress()))) { + .setElasticSearchUrl("http://" + container.getHttpHostAddress()), mock(SinkContext.class))) { assertEquals(client.indexName(record), topicName); } String indexBase = "myindex-" + UUID.randomUUID(); index = indexBase + "-%{+yyyy-MM-dd}"; try (ElasticSearchClient client = new ElasticSearchClient(new ElasticSearchConfig() .setElasticSearchUrl("http://" + container.getHttpHostAddress()) - .setIndexName(index))) { + .setIndexName(index), mock(SinkContext.class))) { assertThrows(IllegalStateException.class, () -> { client.indexName(record); }); @@ -145,7 +180,7 @@ public void testIndexName() throws Exception { when(record.getEventTime()).thenReturn(Optional.of(1645182000000L)); try (ElasticSearchClient client = new ElasticSearchClient(new ElasticSearchConfig() .setElasticSearchUrl("http://" + container.getHttpHostAddress()) - .setIndexName(index))) { + .setIndexName(index), mock(SinkContext.class))) { assertEquals(client.indexName(record), indexBase + "-2022-02-18"); } } @@ -155,7 +190,7 @@ public void testIndexDelete() throws Exception { String index = "myindex-" + UUID.randomUUID(); try (ElasticSearchClient client = new ElasticSearchClient(new ElasticSearchConfig() .setElasticSearchUrl("http://" + container.getHttpHostAddress()) - .setIndexName(index));) { + .setIndexName(index), mock(SinkContext.class));) { assertTrue(client.createIndexIfNeeded(index)); try { MockRecord mockRecord = new MockRecord<>(); @@ -179,7 +214,7 @@ public void testIndexExists() throws IOException { String index = "mynewindex-" + UUID.randomUUID(); try (ElasticSearchClient client = new ElasticSearchClient(new ElasticSearchConfig() .setElasticSearchUrl("http://" + container.getHttpHostAddress()) - .setIndexName(index));) { + .setIndexName(index), mock(SinkContext.class));) { assertFalse(client.indexExists(index)); assertTrue(client.createIndexIfNeeded(index)); try { @@ -194,7 +229,7 @@ public void testIndexExists() throws IOException { @Test public void testTopicToIndexName() throws IOException { try (ElasticSearchClient client = new ElasticSearchClient(new ElasticSearchConfig() - .setElasticSearchUrl("http://" + container.getHttpHostAddress()));) { + .setElasticSearchUrl("http://" + container.getHttpHostAddress()), mock(SinkContext.class));) { assertEquals(client.topicToIndexName("data-ks1.table1"), "data-ks1.table1"); assertEquals(client.topicToIndexName("persistent://public/default/testesjson"), "testesjson"); assertEquals(client.topicToIndexName("default/testesjson"), "testesjson"); @@ -217,13 +252,19 @@ public void testMalformedDocFails() throws Exception { .setBulkEnabled(true) .setBulkFlushIntervalInMs(-1L) .setMalformedDocAction(ElasticSearchConfig.MalformedDocAction.FAIL); - try (ElasticSearchClient client = new ElasticSearchClient(config);) { + SinkContext sinkContext = mock(SinkContext.class); + AtomicReference irrecoverableError = new AtomicReference<>(); + doAnswer(invocation -> { + irrecoverableError.compareAndSet(null, invocation.getArgument(0)); + return null; + }).when(sinkContext).fatal(any(Throwable.class)); + try (ElasticSearchClient client = new ElasticSearchClient(config, sinkContext);) { MockRecord mockRecord = new MockRecord<>(); client.bulkIndex(mockRecord, Pair.of("1", "{\"a\":1}")); client.bulkIndex(mockRecord, Pair.of("2", "{\"a\":\"toto\"}")); client.flush(); - assertNotNull(client.irrecoverableError.get()); - assertTrue(client.irrecoverableError.get().getMessage().contains("mapper_parsing_exception")); + assertNotNull(irrecoverableError.get()); + assertTrue(irrecoverableError.get().getMessage().contains("mapper_parsing_exception")); assertEquals(mockRecord.getAcked(), 1); assertEquals(mockRecord.getFailed(), 1); assertThrows(Exception.class, () -> client.bulkIndex(mockRecord, Pair.of("3", "{\"a\":3}"))); @@ -241,12 +282,18 @@ public void testMalformedDocIgnore() throws Exception { .setBulkEnabled(true) .setBulkFlushIntervalInMs(-1) .setMalformedDocAction(ElasticSearchConfig.MalformedDocAction.IGNORE); - try (ElasticSearchClient client = new ElasticSearchClient(config);) { + SinkContext sinkContext = mock(SinkContext.class); + AtomicReference irrecoverableError = new AtomicReference<>(); + doAnswer(invocation -> { + irrecoverableError.set(invocation.getArgument(0)); + return null; + }).when(sinkContext).fatal(any(Throwable.class)); + try (ElasticSearchClient client = new ElasticSearchClient(config, mock(SinkContext.class));) { MockRecord mockRecord = new MockRecord<>(); client.bulkIndex(mockRecord, Pair.of("1", "{\"a\":1}")); client.bulkIndex(mockRecord, Pair.of("2", "{\"a\":\"toto\"}")); client.flush(); - assertNull(client.irrecoverableError.get()); + assertNull(irrecoverableError.get()); assertEquals(mockRecord.getAcked(), 1); assertEquals(mockRecord.getFailed(), 1); } @@ -268,7 +315,7 @@ public void testBulkRetry() throws Exception { // disabled, we want to have full control over flush() method .setBulkFlushIntervalInMs(-1); - try (ElasticSearchClient client = new ElasticSearchClient(config);) { + try (ElasticSearchClient client = new ElasticSearchClient(config, mock(SinkContext.class));) { try { assertTrue(client.createIndexIfNeeded(index)); MockRecord mockRecord = new MockRecord<>(); @@ -314,7 +361,7 @@ public void testBulkBlocking() throws Exception { .setBulkConcurrentRequests(2) .setRetryBackoffInMs(100) .setBulkFlushIntervalInMs(10000); - try (ElasticSearchClient client = new ElasticSearchClient(config);) { + try (ElasticSearchClient client = new ElasticSearchClient(config, mock(SinkContext.class));) { assertTrue(client.createIndexIfNeeded(index)); try { @@ -373,7 +420,7 @@ public void testBulkIndexAndDelete() throws Exception { .setBulkActions(10) .setBulkFlushIntervalInMs(-1L); - try (ElasticSearchClient client = new ElasticSearchClient(config)) { + try (ElasticSearchClient client = new ElasticSearchClient(config, mock(SinkContext.class))) { assertTrue(client.createIndexIfNeeded(index)); MockRecord mockRecord = new MockRecord<>(); for (int i = 0; i < 5; i++) { @@ -397,7 +444,7 @@ public void testIndexKeepNulls() throws Exception { .setElasticSearchUrl("http://" + container.getHttpHostAddress()) .setIndexName(index); - try (ElasticSearchClient client = new ElasticSearchClient(config)) { + try (ElasticSearchClient client = new ElasticSearchClient(config, mock(SinkContext.class))) { MockRecord mockRecord = new MockRecord<>(); client.indexDocument(mockRecord, Pair.of("key0", "{\"a\":1,\"b\":null}")); final Map sourceAsMap; diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfigTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfigTests.java index 85e30e766f030..506df31923378 100644 --- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfigTests.java +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfigTests.java @@ -44,7 +44,6 @@ public final void loadFromYamlFileTest() throws IOException { assertNotNull(config); assertEquals(config.getElasticSearchUrl(), "http://localhost:90902"); assertEquals(config.getIndexName(), "myIndex"); - assertEquals(config.getTypeName(), "doc"); assertEquals(config.getUsername(), "scooby"); assertEquals(config.getPassword(), "doobie"); assertEquals(config.getPrimaryFields(), "id,a"); @@ -64,7 +63,6 @@ public final void loadFromMapTest() throws IOException { assertNotNull(config); assertEquals(config.getElasticSearchUrl(), "http://localhost:90902"); assertEquals(config.getIndexName(), "myIndex"); - assertEquals(config.getTypeName(), "doc"); assertEquals(config.getUsername(), "racerX"); assertEquals(config.getPassword(), "go-speedie-go"); assertEquals(config.getPrimaryFields(), "x"); @@ -75,7 +73,6 @@ public final void defaultValueTest() throws IOException { Map requiredConfig = Map.of("elasticSearchUrl", "http://localhost:90902"); ElasticSearchConfig config = ElasticSearchConfig.load(requiredConfig, mockContext); assertNull(config.getIndexName()); - assertEquals(config.getTypeName(), "_doc"); assertNull(config.getUsername()); assertNull(config.getPassword()); assertNull(config.getToken()); @@ -336,7 +333,6 @@ public final void loadConfigFromSecretsTest() throws IOException { assertNotNull(config); assertEquals(config.getElasticSearchUrl(), "http://localhost:90902"); assertEquals(config.getIndexName(), "myIndex"); - assertEquals(config.getTypeName(), "doc"); assertEquals(config.getPrimaryFields(), "x"); assertEquals(config.getUsername(), "secretUser"); assertEquals(config.getPassword(), "$ecret123"); diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java index cbc3de908c68f..f5a2e36aef44d 100644 --- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java @@ -101,6 +101,7 @@ public GenericObject getValue() { Pair pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord); assertEquals(pair.getLeft(), "1"); assertEquals(pair.getRight(), "{\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0,\"i\":1,\"l\":10}}"); + elasticSearchSink.close(); // two fields PK ElasticSearchSink elasticSearchSink2 = new ElasticSearchSink(); @@ -113,6 +114,7 @@ public GenericObject getValue() { Pair pair2 = elasticSearchSink2.extractIdAndDocument(genericObjectRecord); assertEquals(pair2.getLeft(), "[\"1\",1]"); assertEquals(pair2.getRight(), "{\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0,\"i\":1,\"l\":10}}"); + elasticSearchSink2.close(); // default config with null PK => indexed with auto generated _id ElasticSearchSink elasticSearchSink3 = new ElasticSearchSink(); @@ -122,6 +124,7 @@ public GenericObject getValue() { Pair pair3 = elasticSearchSink3.extractIdAndDocument(genericObjectRecord); assertNull(pair3.getLeft()); assertEquals(pair3.getRight(), "{\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0,\"i\":1,\"l\":10}}"); + elasticSearchSink3.close(); // default config with null PK + null value ElasticSearchSink elasticSearchSink4 = new ElasticSearchSink(); @@ -146,6 +149,7 @@ public GenericObject getValue() { }); assertNull(pair4.getLeft()); assertNull(pair4.getRight()); + elasticSearchSink4.close(); } @Test(dataProvider = "schemaType") @@ -225,6 +229,7 @@ public GenericObject getValue() { Pair pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord); assertEquals(pair.getLeft(), "[\"1\",1]"); assertEquals(pair.getRight(), "{\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0,\"i\":1,\"l\":10}}"); + elasticSearchSink.close(); elasticSearchSink = new ElasticSearchSink(); elasticSearchSink.open(ImmutableMap.of( @@ -236,6 +241,7 @@ public GenericObject getValue() { pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord); assertEquals(pair.getLeft(), "[\"1\",1]"); assertEquals(pair.getRight(), "{\"a\":\"1\",\"b\":1,\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0,\"i\":1,\"l\":10}}"); + elasticSearchSink.close(); elasticSearchSink = new ElasticSearchSink(); elasticSearchSink.open(ImmutableMap.of( @@ -246,6 +252,7 @@ public GenericObject getValue() { pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord); assertNull(pair.getLeft()); assertEquals(pair.getRight(), "{\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0,\"i\":1,\"l\":10}}"); + elasticSearchSink.close(); elasticSearchSink = new ElasticSearchSink(); elasticSearchSink.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", @@ -255,6 +262,7 @@ public GenericObject getValue() { pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord); assertNull(pair.getLeft()); assertEquals(pair.getRight(), "{\"a\":\"1\",\"b\":1,\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0,\"i\":1,\"l\":10}}"); + elasticSearchSink.close(); // test null value elasticSearchSink = new ElasticSearchSink(); @@ -291,6 +299,7 @@ public Object getNativeObject() { }); assertEquals(pair.getLeft(), "[\"1\",1]"); assertNull(pair.getRight()); + elasticSearchSink.close(); } @Test(dataProvider = "schemaType") @@ -326,6 +335,7 @@ public void testSortKeysSingle(SchemaType schemaType) throws Exception { "keyIgnore", "false"), null); Pair pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord); assertEquals(pair.getKey(), "{\"b_inside_inner\":\"0b_value_from_inner\",\"a_inside_inner\":\"a_value_from_inner\"}"); + elasticSearchSink.close(); elasticSearchSink = new ElasticSearchSink(); elasticSearchSink.open(ImmutableMap.of( @@ -336,6 +346,7 @@ public void testSortKeysSingle(SchemaType schemaType) throws Exception { "keyIgnore", "false"), null); pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord); assertEquals(pair.getKey(), "{\"a_inside_inner\":\"a_value_from_inner\",\"b_inside_inner\":\"0b_value_from_inner\"}"); + elasticSearchSink.close(); } @@ -378,6 +389,7 @@ public void testSortKeysMulti(SchemaType schemaType) throws Exception { "keyIgnore", "false"), null); Pair pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord); assertEquals(pair.getKey(), "[\"a_key\",\"0b_key\",\"c_key\",{\"b_inside_inner\":\"0b_value_from_inner\",\"a_inside_inner\":\"a_value_from_inner\"}]"); + elasticSearchSink.close(); elasticSearchSink = new ElasticSearchSink(); elasticSearchSink.open(ImmutableMap.of( @@ -388,6 +400,7 @@ public void testSortKeysMulti(SchemaType schemaType) throws Exception { "keyIgnore", "false"), null); pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord); assertEquals(pair.getKey(), "[\"a_key\",\"0b_key\",\"c_key\",{\"a_inside_inner\":\"a_value_from_inner\",\"b_inside_inner\":\"0b_value_from_inner\"}]"); + elasticSearchSink.close(); } private Record getKeyValueGenericObject(SchemaType schemaType, GenericSchema keySchema, GenericRecord keyGenericRecord) { diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchRawRecordTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchRawRecordTests.java new file mode 100644 index 0000000000000..5060261053265 --- /dev/null +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchRawRecordTests.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.elasticsearch; + +import com.google.common.collect.ImmutableMap; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.functions.api.Record; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; + +public class ElasticSearchRawRecordTests { + String json = "{\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0,\"i\":1,\"l\":10}}"; + + @DataProvider(name = "rawRecordSchema") + public Object[] rawRecordSchema() { + return new Object[]{ + createRecord(json, Schema.STRING), + createRecord(json.getBytes(StandardCharsets.UTF_8), Schema.BYTES), + createRecord(json, "12345", Schema.STRING), + createRecord(json.getBytes(StandardCharsets.UTF_8), "abcd", Schema.BYTES) + }; + } + + @Test(dataProvider = "rawRecordSchema") + public final void testRawRecord(Record record) throws Exception { + ElasticSearchSink elasticSearchSink = new ElasticSearchSink(); + elasticSearchSink.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", + "schemaEnable", "true", + "keyIgnore", "false", + "compatibilityMode", "ELASTICSEARCH"), + null); + Pair pair = elasticSearchSink.extractIdAndDocument(record); + String key = (String) record.getKey().orElse(null); + assertEquals(pair.getKey(), key); + assertEquals(pair.getValue(), json); + } + + @Test(dataProvider = "rawRecordSchema") + public final void testRawRecordKeyIgnore(Record record) throws Exception { + ElasticSearchSink elasticSearchSink = new ElasticSearchSink(); + elasticSearchSink.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", + "schemaEnable", "true", + "keyIgnore", "true", + "compatibilityMode", "ELASTICSEARCH"), + null); + Pair pair = elasticSearchSink.extractIdAndDocument(record); + assertNull(pair.getKey()); + assertEquals(pair.getValue(), json); + } + + @Test(dataProvider = "rawRecordSchema") + public final void testRawRecordSchemaNotEnabled(Record record) throws Exception { + ElasticSearchSink elasticSearchSink = new ElasticSearchSink(); + elasticSearchSink.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", + "schemaEnable", "false", + "keyIgnore", "false", + "compatibilityMode", "ELASTICSEARCH"), + null); + Pair pair = elasticSearchSink.extractIdAndDocument(record); + String key = (String) record.getKey().orElse(null); + assertEquals(pair.getKey(), key); + assertEquals(pair.getValue(), json); + } + + @Test(dataProvider = "rawRecordSchema") + public final void testRawRecordSchemaNotEnabledKeyIgnore(Record record) throws Exception { + ElasticSearchSink elasticSearchSink = new ElasticSearchSink(); + elasticSearchSink.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", + "schemaEnable", "false", + "keyIgnore", "true", + "compatibilityMode", "ELASTICSEARCH"), + null); + Pair pair = elasticSearchSink.extractIdAndDocument(record); + assertNull(pair.getKey()); + assertEquals(pair.getValue(), json); + } + + private Record createRecord(T value, Schema schema){ + return new Record() { + @Override + public Optional getTopicName() { + return Optional.of("topic-name"); + } + + @Override + public Schema getSchema() { + return schema; + } + + @Override + public T getValue() { + return value; + } + }; + } + + private Record createRecord(T value, String key, Schema schema){ + return new Record() { + @Override + public Optional getTopicName() { + return Optional.of("topic-name"); + } + + @Override + public Schema getSchema() { + return schema; + } + + @Override + public T getValue() { + return value; + } + + @Override + public Optional getKey() { + return Optional.of(key); + } + }; + } +} diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkRawDataTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkRawDataTests.java index 7749dc5ecca2f..7c6d8197eb88b 100644 --- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkRawDataTests.java +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkRawDataTests.java @@ -18,6 +18,15 @@ */ package org.apache.pulsar.io.elasticsearch; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.Assert.fail; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; import lombok.AllArgsConstructor; import lombok.Data; import org.apache.pulsar.client.api.Message; @@ -31,24 +40,14 @@ import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static org.testng.Assert.fail; - public abstract class ElasticSearchSinkRawDataTests extends ElasticSearchTestBase { - private static ElasticsearchContainer container; + private ElasticsearchContainer container; public ElasticSearchSinkRawDataTests(String elasticImageName) { super(elasticImageName); @@ -67,18 +66,15 @@ public ElasticSearchSinkRawDataTests(String elasticImageName) { static Schema schema; - @BeforeMethod(alwaysRun = true) + @BeforeClass(alwaysRun = true) public final void initBeforeClass() { - if (container != null) { - return; - } container = createElasticsearchContainer(); container.start(); schema = Schema.BYTES; } @AfterClass(alwaysRun = true) - public static void closeAfterClass() { + public void closeAfterClass() { container.close(); container = null; } diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkTests.java index 9fad03c357975..f1da6fd0c7e15 100644 --- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkTests.java +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkTests.java @@ -18,38 +18,44 @@ */ package org.apache.pulsar.io.elasticsearch; -import co.elastic.clients.transport.ElasticsearchTransport; -import org.apache.pulsar.client.api.Message; -import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.api.schema.GenericObject; -import org.apache.pulsar.client.api.schema.GenericRecord; -import org.apache.pulsar.client.api.schema.GenericSchema; -import org.apache.pulsar.client.api.schema.RecordSchemaBuilder; -import org.apache.pulsar.client.api.schema.SchemaBuilder; -import org.apache.pulsar.client.impl.MessageImpl; -import org.apache.pulsar.common.schema.KeyValue; -import org.apache.pulsar.common.schema.KeyValueEncodingType; -import org.apache.pulsar.common.schema.SchemaType; - +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; - +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import co.elastic.clients.transport.ElasticsearchTransport; +import com.fasterxml.jackson.core.JsonParseException; +import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.HashMap; import java.util.List; import java.util.Locale; -import java.io.IOException; -import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.schema.GenericObject; +import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.api.schema.GenericSchema; +import org.apache.pulsar.client.api.schema.RecordSchemaBuilder; +import org.apache.pulsar.client.api.schema.SchemaBuilder; +import org.apache.pulsar.client.impl.MessageImpl; +import org.apache.pulsar.common.schema.KeyValue; +import org.apache.pulsar.common.schema.KeyValueEncodingType; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.io.core.SinkContext; import org.apache.pulsar.io.elasticsearch.client.BulkProcessor; @@ -67,17 +73,14 @@ import org.testng.SkipException; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; - -import static org.testng.Assert.assertNull; -import static org.testng.Assert.fail; - public abstract class ElasticSearchSinkTests extends ElasticSearchTestBase { - private static ElasticsearchContainer container; + private ElasticsearchContainer container; public ElasticSearchSinkTests(String elasticImageName) { super(elasticImageName); @@ -88,6 +91,7 @@ public ElasticSearchSinkTests(String elasticImageName) { @Mock protected SinkContext mockSinkContext; + AtomicReference irrecoverableError = new AtomicReference<>(); protected Map map; protected ElasticSearchSink sink; @@ -97,17 +101,14 @@ public ElasticSearchSinkTests(String elasticImageName) { GenericRecord userProfile; String recordKey; - @BeforeMethod(alwaysRun = true) + @BeforeClass(alwaysRun = true) public final void initBeforeClass() { - if (container != null) { - return; - } container = createElasticsearchContainer(); container.start(); } @AfterClass(alwaysRun = true) - public static void closeAfterClass() { + public void closeAfterClass() { container.close(); container = null; } @@ -134,6 +135,11 @@ public final void setUp() throws Exception { mockRecord = mock(Record.class); mockSinkContext = mock(SinkContext.class); + irrecoverableError.set(null); + doAnswer(invocation -> { + irrecoverableError.set(invocation.getArgument(0)); + return null; + }).when(mockSinkContext).fatal(any(Throwable.class)); when(mockRecord.getValue()).thenAnswer((Answer) invocation -> new GenericObject() { @Override @@ -148,6 +154,7 @@ public Object getNativeObject() { }); when(mockRecord.getSchema()).thenAnswer((Answer>>) invocation -> kvSchema); + when(mockRecord.getEventTime()).thenAnswer(invocation -> Optional.of(System.currentTimeMillis())); } @AfterMethod(alwaysRun = true) @@ -205,6 +212,16 @@ public final void send100Test() throws Exception { verify(mockRecord, times(100)).ack(); } + @Test + public final void send1WithFormattedIndexTest() throws Exception { + map.put("indexName", "test-formatted-index-%{+yyyy-MM-dd}"); + sink.open(map, mockSinkContext); + send(1); + verify(mockRecord, times(1)).ack(); + String value = getHitIdAtIndex("test-formatted-index-*", 0); + assertTrue(StringUtils.isNotBlank(value)); + } + @Test public final void sendNoSchemaTest() throws Exception { @@ -219,7 +236,7 @@ public Optional> answer(InvocationOnMock invocation) throws Thro when(mockRecord.getKey()).thenAnswer(new Answer>() { public Optional answer(InvocationOnMock invocation) throws Throwable { - return null; + return Optional.empty(); } }); @@ -243,6 +260,85 @@ public Schema answer(InvocationOnMock invocation) throws Throwable { verify(mockRecord, times(1)).ack(); } + @Test + public final void sendJsonStringSchemaTest() throws Exception { + + when(mockRecord.getMessage()).thenAnswer(new Answer>>() { + @Override + public Optional> answer(InvocationOnMock invocation) throws Throwable { + final MessageImpl mock = mock(MessageImpl.class); + when(mock.getData()).thenReturn("{\"a\":1}".getBytes(StandardCharsets.UTF_8)); + return Optional.of(mock); + } + }); + + when(mockRecord.getKey()).thenAnswer(new Answer>() { + public Optional answer(InvocationOnMock invocation) throws Throwable { + return Optional.empty(); + } + }); + + GenericRecord genericRecord = mock(GenericRecord.class); + when(genericRecord.getNativeObject()).thenReturn("{\"a\":1}"); + when(genericRecord.getSchemaType()).thenReturn(SchemaType.JSON); + when(mockRecord.getValue()).thenAnswer(new Answer() { + public GenericRecord answer(InvocationOnMock invocation) throws Throwable { + return genericRecord; + } + }); + + when(mockRecord.getSchema()).thenAnswer(new Answer() { + public Schema answer(InvocationOnMock invocation) throws Throwable { + return Schema.JSON(String.class); + } + }); + + map.put("indexName", "test-index"); + map.put("schemaEnable", "true"); + sink.open(map, mockSinkContext); + sink.write(mockRecord); + verify(mockRecord, times(1)).ack(); + } + + @Test(expectedExceptions = JsonParseException.class) + public final void sendJsonStringSchemaErrorTest() throws Exception { + + when(mockRecord.getMessage()).thenAnswer(new Answer>>() { + @Override + public Optional> answer(InvocationOnMock invocation) throws Throwable { + final MessageImpl mock = mock(MessageImpl.class); + when(mock.getData()).thenReturn("no-json-format".getBytes(StandardCharsets.UTF_8)); + return Optional.of(mock); + } + }); + + when(mockRecord.getKey()).thenAnswer(new Answer>() { + public Optional answer(InvocationOnMock invocation) throws Throwable { + return Optional.empty(); + } + }); + + GenericRecord genericRecord = mock(GenericRecord.class); + when(genericRecord.getNativeObject()).thenReturn("no-json-format"); + when(genericRecord.getSchemaType()).thenReturn(SchemaType.JSON); + when(mockRecord.getValue()).thenAnswer(new Answer() { + public GenericRecord answer(InvocationOnMock invocation) throws Throwable { + return genericRecord; + } + }); + + when(mockRecord.getSchema()).thenAnswer(new Answer() { + public Schema answer(InvocationOnMock invocation) throws Throwable { + return Schema.JSON(String.class); + } + }); + + map.put("indexName", "test-index"); + map.put("schemaEnable", "true"); + sink.open(map, mockSinkContext); + sink.write(mockRecord); + } + @Test(enabled = true) public final void sendKeyIgnoreSingleField() throws Exception { final String index = "testkeyignore"; @@ -342,7 +438,7 @@ public void testKeepNullNodes() throws Exception { assertEquals(json, "{\"name\":null,\"userName\":\"boby\",\"email\":null}"); } - @Test(expectedExceptions = PulsarClientException.InvalidMessageException.class) + @Test public void testNullValueFailure() throws Exception { String index = "testnullvaluefail"; map.put("indexName", index); @@ -351,6 +447,7 @@ public void testNullValueFailure() throws Exception { sink.open(map, mockSinkContext); MockRecordNullValue mockRecordNullValue = new MockRecordNullValue(); sink.write(mockRecordNullValue); + assertNotNull(irrecoverableError.get()); } @Test @@ -400,7 +497,7 @@ public Object getNativeObject() { assertEquals(sink.getElasticsearchClient().getRestClient().totalHits(index), 1L); sink.write(new MockRecordNullValue()); assertEquals(sink.getElasticsearchClient().getRestClient().totalHits(index), action.equals(ElasticSearchConfig.NullValueAction.DELETE) ? 0L : 1L); - assertNull(sink.getElasticsearchClient().irrecoverableError.get()); + assertNull(irrecoverableError.get()); } @Test @@ -437,7 +534,7 @@ public void testCloseClient() throws Exception { sink.close(); verify(restHighLevelClient).close(); - verify(internalBulkProcessor).awaitClose(Mockito.anyLong(), Mockito.any(TimeUnit.class)); + verify(internalBulkProcessor).awaitClose(Mockito.anyLong(), any(TimeUnit.class)); verify(client).close(); verify(restClient).close(); } else { diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchTestBase.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchTestBase.java index 4c6fd020fa338..8c5868f27689b 100644 --- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchTestBase.java +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchTestBase.java @@ -18,10 +18,6 @@ */ package org.apache.pulsar.io.elasticsearch; -import java.io.IOException; -import java.util.Map; -import java.util.Optional; - import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch.security.CreateApiKeyRequest; import co.elastic.clients.elasticsearch.security.CreateApiKeyResponse; @@ -29,6 +25,10 @@ import co.elastic.clients.elasticsearch.security.GetTokenResponse; import co.elastic.clients.elasticsearch.security.get_token.AccessTokenGrantType; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.util.Map; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.io.elasticsearch.client.elastic.ElasticSearchJavaRestClient; import org.apache.pulsar.io.elasticsearch.client.opensearch.OpenSearchHighLevelRestClient; import org.opensearch.client.Request; @@ -36,16 +36,17 @@ import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testcontainers.utility.DockerImageName; +@Slf4j public abstract class ElasticSearchTestBase { public static final String ELASTICSEARCH_8 = Optional.ofNullable(System.getenv("ELASTICSEARCH_IMAGE_V8")) - .orElse("docker.elastic.co/elasticsearch/elasticsearch:8.5.1"); + .orElse("docker.elastic.co/elasticsearch/elasticsearch:8.5.3"); public static final String ELASTICSEARCH_7 = Optional.ofNullable(System.getenv("ELASTICSEARCH_IMAGE_V7")) .orElse("docker.elastic.co/elasticsearch/elasticsearch:7.17.7"); public static final String OPENSEARCH = Optional.ofNullable(System.getenv("OPENSEARCH_IMAGE")) - .orElse("opensearchproject/opensearch:1.2.4"); + .orElse("opensearchproject/opensearch:2.16.0"); protected final String elasticImageName; @@ -54,17 +55,29 @@ public ElasticSearchTestBase(String elasticImageName) { } protected ElasticsearchContainer createElasticsearchContainer() { + ElasticsearchContainer elasticsearchContainer; if (elasticImageName.equals(OPENSEARCH)) { DockerImageName dockerImageName = DockerImageName.parse(OPENSEARCH).asCompatibleSubstituteFor("docker.elastic.co/elasticsearch/elasticsearch"); - return new ElasticsearchContainer(dockerImageName) + elasticsearchContainer = new ElasticsearchContainer(dockerImageName) + .withEnv("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "0pEn7earch!") .withEnv("OPENSEARCH_JAVA_OPTS", "-Xms128m -Xmx256m") .withEnv("bootstrap.memory_lock", "true") .withEnv("plugins.security.disabled", "true"); + } else { + elasticsearchContainer = new ElasticsearchContainer(elasticImageName) + .withEnv("ES_JAVA_OPTS", "-Xms128m -Xmx256m") + .withEnv("xpack.security.enabled", "false") + .withEnv("xpack.security.http.ssl.enabled", "false"); + } + configureElasticContainer(elasticsearchContainer); + return elasticsearchContainer; + } + + protected void configureElasticContainer(ElasticsearchContainer elasticContainer) { + if (getCompatibilityMode() != ElasticSearchConfig.CompatibilityMode.OPENSEARCH) { + elasticContainer.withEnv("ingest.geoip.downloader.enabled", "false"); } - return new ElasticsearchContainer(elasticImageName) - .withEnv("ES_JAVA_OPTS", "-Xms128m -Xmx256m") - .withEnv("xpack.security.enabled", "false") - .withEnv("xpack.security.http.ssl.enabled", "false"); + elasticContainer.withLogConsumer(o -> log.info("elastic> {}", o.getUtf8String())); } protected ElasticSearchConfig.CompatibilityMode getCompatibilityMode() { diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/opensearch/OpenSearchClientSslTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/opensearch/OpenSearchClientSslTests.java index de6946136855a..0b78506491657 100644 --- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/opensearch/OpenSearchClientSslTests.java +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/opensearch/OpenSearchClientSslTests.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.io.elasticsearch.opensearch; +import org.apache.pulsar.io.core.SinkContext; import org.apache.pulsar.io.elasticsearch.ElasticSearchClient; import org.apache.pulsar.io.elasticsearch.ElasticSearchConfig; import org.apache.pulsar.io.elasticsearch.ElasticSearchSslConfig; @@ -32,6 +33,7 @@ import java.util.HashMap; import java.util.Map; +import static org.mockito.Mockito.mock; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -76,12 +78,12 @@ public void testSslBasic() throws IOException { .setElasticSearchUrl("https://" + container.getHttpHostAddress()) .setIndexName(INDEX) .setUsername("admin") - .setPassword("admin") + .setPassword("0pEn7earch!") .setSsl(new ElasticSearchSslConfig() .setEnabled(true) .setTruststorePath(sslResourceDir + "/truststore.jks") .setTruststorePassword("changeit")); - ElasticSearchClient client = new ElasticSearchClient(config); + ElasticSearchClient client = new ElasticSearchClient(config, mock(SinkContext.class)); testIndexExists(client); } } @@ -100,14 +102,14 @@ public void testSslWithHostnameVerification() throws IOException { .setElasticSearchUrl("https://" + container.getHttpHostAddress()) .setIndexName(INDEX) .setUsername("admin") - .setPassword("admin") + .setPassword("0pEn7earch!") .setSsl(new ElasticSearchSslConfig() .setEnabled(true) .setProtocols("TLSv1.2") .setHostnameVerification(true) .setTruststorePath(sslResourceDir + "/truststore.jks") .setTruststorePassword("changeit")); - ElasticSearchClient client = new ElasticSearchClient(config); + ElasticSearchClient client = new ElasticSearchClient(config, mock(SinkContext.class)); testIndexExists(client); } } @@ -125,7 +127,7 @@ public void testSslWithClientAuth() throws IOException { .setElasticSearchUrl("https://" + container.getHttpHostAddress()) .setIndexName(INDEX) .setUsername("admin") - .setPassword("admin") + .setPassword("0pEn7earch!") .setSsl(new ElasticSearchSslConfig() .setEnabled(true) .setHostnameVerification(true) @@ -133,7 +135,7 @@ public void testSslWithClientAuth() throws IOException { .setTruststorePassword("changeit") .setKeystorePath(sslResourceDir + "/keystore.jks") .setKeystorePassword("changeit")); - ElasticSearchClient client = new ElasticSearchClient(config); + ElasticSearchClient client = new ElasticSearchClient(config, mock(SinkContext.class)); testIndexExists(client); } } diff --git a/pulsar-io/elastic-search/src/test/resources/ssl/README.md b/pulsar-io/elastic-search/src/test/resources/ssl/README.md new file mode 100644 index 0000000000000..0a1e91a9e47c7 --- /dev/null +++ b/pulsar-io/elastic-search/src/test/resources/ssl/README.md @@ -0,0 +1,43 @@ +# SSL Cert Provenance + +The files were generated with the following steps. They are not in a script because a script likely won't +work the next time these files need to be updated. These files were copied out of convenience. + +One important assumption is that all certs and keystores share the `cacert.pem` as a root CA. + +[cacert.pem](./cacert.pem) was copied from the tests/certificate-authority/certs/ca.cert.pem file. +```shell +cp ../../../../../../tests/certificate-authority/certs/ca.cert.pem cacert.pem +``` + +[cacert.crt](./cacert.crt) was generated using the following command: +```shell +openssl x509 -in cacert.pem -inform pem -out cacert.crt -outform der +``` + +The [truststore.jks](./truststore.jks) file was generated using the following command: +```shell +keytool -importcert -alias rootca -keystore truststore.jks -storepass changeit -file cacert.crt -noprompt +``` + +The [keystore.jks](./keystore.jks) file was generated using the following commands: +```shell +cat ../../../../../../tests/certificate-authority/client-keys/admin.cert.pem > client.pem +cat ../../../../../../tests/certificate-authority/client-keys/admin.key.pem >> client.pem +openssl pkcs12 -export -in client.pem -out client.p12 +``` + +Manually enter `123456` password. + +```shell +keytool -importkeystore -srckeystore client.p12 -srcstoretype pkcs12 -srcstorepass 123456 -destkeystore keystore.jks -deststorepass changeit -noprompt +rm client.pem client.p12 +``` + +The [elasticsearch.crt](./elasticsearch.crt), [elasticsearch.key](./elasticsearch.key), [elasticsearch.pem](./elasticsearch.pem) files were all copied from broker certs. + +```shell +cp ../../../../../../tests/certificate-authority/server-keys/broker.cert.pem elasticsearch.crt +cp ../../../../../../tests/certificate-authority/server-keys/broker.key.pem elasticsearch.key +cp ../../../../../../tests/certificate-authority/server-keys/broker.key-pk8.pem elasticsearch.pem +``` \ No newline at end of file diff --git a/pulsar-io/elastic-search/src/test/resources/ssl/cacert.crt b/pulsar-io/elastic-search/src/test/resources/ssl/cacert.crt index 070eaa67e21df..87d462dcb4a89 100644 Binary files a/pulsar-io/elastic-search/src/test/resources/ssl/cacert.crt and b/pulsar-io/elastic-search/src/test/resources/ssl/cacert.crt differ diff --git a/pulsar-io/elastic-search/src/test/resources/ssl/cacert.pem b/pulsar-io/elastic-search/src/test/resources/ssl/cacert.pem index ed6f0fffdad9b..0446700135d39 100644 --- a/pulsar-io/elastic-search/src/test/resources/ssl/cacert.pem +++ b/pulsar-io/elastic-search/src/test/resources/ssl/cacert.pem @@ -1,22 +1,29 @@ -----BEGIN CERTIFICATE----- -MIIDqzCCApOgAwIBAgIJAIM4kKJTNpVtMA0GCSqGSIb3DQEBCwUAMEAxCzAJBgNV -BAYTAkZSMREwDwYDVQQKDAhEYXRhc3RheDENMAsGA1UECwwEVGVzdDEPMA0GA1UE -AwwGcm9vdGNhMCAXDTIxMDUxMTE3MjUwMVoYDzIxMjEwNDE3MTcyNTAxWjBAMQsw -CQYDVQQGEwJGUjERMA8GA1UECgwIRGF0YXN0YXgxDTALBgNVBAsMBFRlc3QxDzAN -BgNVBAMMBnJvb3RjYTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANST -1q4mSWgVpUFqmtkgF/SgPUEDpFwzv9oOBFvw/71gy1+THmcw7VAM729O3ZrsJywJ -3iMEo1IwzUIV+tnLFPPFonT8HBuySTQ0rTmGC7+rdOmM26TclBZmOm0pYNwVg3td -rGGceN/eOLG4oIlaJM6SjlNLY8NbjVtB45V5G7I7IxZN44+PiYaH8b1OJld4Z9dE -3bXheLodPRyDIZwOnoTOtOZmPjICk80N1OiEZQa2OIfyhGECqggu8vN+HYoVqhS2 -DFmbeXqQH8piJHsf/gwy9o1EwUnUTcg3XKRu/qAywYwqo43j/+fon+qxRsEzhbLU -9UCBqQakpuyEu6RLTd8CAwEAAaOBpTCBojAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud -DgQWBBTweKQ5biriOHaVWywXTyKv451lDDBwBgNVHSMEaTBngBTweKQ5biriOHaV -WywXTyKv451lDKFEpEIwQDELMAkGA1UEBhMCRlIxETAPBgNVBAoMCERhdGFzdGF4 -MQ0wCwYDVQQLDARUZXN0MQ8wDQYDVQQDDAZyb290Y2GCCQCDOJCiUzaVbTANBgkq -hkiG9w0BAQsFAAOCAQEAaW4cDHLmhgLJUOctuengm47YPpledOmXzvquFb5iyULU -xRyYaYm5D4OcZEgs8E84upB9mo66uc1gg48m3PkOLTY8+0gpS23wJ4128MCDkYG1 -s8N3OOXPfZxySPY37Ii162cDjknQ19E4j00zk7jRUdOI4cNDDP7AZ65G96bA18Vs -HLpuJY2y2bJ+W9LwT0oZvxCrJztCXUS2rNqwECdustSf9zolvJKmgt+iKCUrQtho -xI0Qsc4KvGT1CvKmEkwlB6z0JNO0HXEdxIvF7NLNNR/URt4+zLx0ieCajCKKKr7x -6Bkyax1iPBTlcSX80e3RKORs/mKgT+DCvESh6kKkBA== +MIIFCDCCAvCgAwIBAgIJANfih0+geeIMMA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBmZvb2JhcjAeFw0xODA2MjIwODQ2MjFaFw0zODA2MTcwODQ2MjFaMBExDzAN +BgNVBAMMBmZvb2JhcjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOVU +UpTPeXCeyfUiQS824l9s9krZd4R6TA4D97eQ9EWm2D7ppV4gPApHO8j5f+joo/b6 +Iso4aFlHpJ8VV2a5Ol7rjQw43MJHaBgwDxB1XWgsNdfoI7ebtp/BWg2nM3r8wm+Z +gKenf9d1/1Ol+6yFUehkLkIXUvldiVegmmje8FnwhcDNE1eTrh66XqSJXEXqgBKu +NqsoYcVak72OyOO1/N8CESoSdyBkbSiH5vJyo0AUCjn7tULga7fxojmqBZDog9Pg +e5Fi/hbCrdinbxBrMgIxQ7wqXw2sw6iOWu4FU8Ih/CuF4xaQy2YP7MEk4Ff0LCY0 +KMhFMWU7550r/fz/C2l7fKhREyCQPa/bVE+dfxgZ/gCZ+p7vQ154hCCjpd+5bECv +SN1bcVIPG6ngQu4vMXa7QRBi/Od40jSVGVJXYY6kXvrYatad7035w2GGGGkvMsQm +y53yh4tqQfH7ulHqB0J5LebTQRp6nRizWigVCLjNkxJYI+Dj51qvT1zdyWEegKr1 +CthBfYzXlfjeH3xri1f0UABeC12n24Wkacd9af7zs7S3rYntEK444w/3fB0F62Lh +SESfMLAmUH0dF5plRShrFUXz23nUeS8EYgWmnGkpf/HDzB67vdfAK0tfJEtmmY78 +q06OSgMr+AOOqaomh4Ez2ZQG592bS71G8MrE7r2/AgMBAAGjYzBhMB0GA1UdDgQW +BBRXC+nLI+i/Rz5Qej9FfqEYQ50VJzAfBgNVHSMEGDAWgBRXC+nLI+i/Rz5Qej9F +fqEYQ50VJzAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG +9w0BAQsFAAOCAgEAYd2PxdV+YOaWcmMG1fK7CGwSzDOGsgC7hi4gWPiNsVbz6fwQ +m5Ac7Zw76dzin8gzOPKST7B8WIoc7ZWrMnyh3G6A3u29Ec8iWahqGa91NPA3bOIl +0ldXnXfa416+JL/Q5utpiV6W2XDaB53v9GqpMk4rOTS9kCFOiuH5ZU8P69jp9mq6 +7pI/+hWFr+21ibmXH6ANxRLd/5+AqojRUYowAu2997Z+xmbpwx/2Svciq3LNY/Vz +s9DudUHCBHj/DPgNxsEUt8QNohjQkRbFTY0a1aXodJ/pm0Ehk2kf9KwYYYduR7ak +6UmPIPrZg6FePNahxwMZ0RtgX7EXmpiiIH1q9BsulddWkrFQclevsWO3ONQVrDs2 +gwY0HQuCRCJ+xgS2cyGiGohW5MkIsg1aI0i0j5GIUSppCIYgirAGCairARbCjhcx +pbMe8RTuBhCqO3R2wZ0wXu7P7/ArI/Ltm1dU6IeHUAUmeneVj5ie0SdA19mHTS2o +lG77N0jy6eq2zyEwJE6tuS/tyP1xrxdzXCYY7f6X9aNfsuPVQTcnrFajvDv8R6uD +YnRStVCdS6fZEP0JzsLrqp9bgLIRRsiqsVVBCgJdK1I/X59qk2EyCLXWSgk8T9XZ +iux8LlPpskt30YYt1KhlWB9zVz7k0uYAwits5foU6RfCRDPAyOa1q/QOXk0= -----END CERTIFICATE----- diff --git a/pulsar-io/elastic-search/src/test/resources/ssl/elasticsearch.crt b/pulsar-io/elastic-search/src/test/resources/ssl/elasticsearch.crt index c8dab8a816352..4237719f20ebd 100644 --- a/pulsar-io/elastic-search/src/test/resources/ssl/elasticsearch.crt +++ b/pulsar-io/elastic-search/src/test/resources/ssl/elasticsearch.crt @@ -1,21 +1,111 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 4103 (0x1007) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=foobar + Validity + Not Before: May 10 15:50:18 2023 GMT + Not After : Feb 22 15:50:18 2297 GMT + Subject: CN=broker-localhost-SAN + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:de:d1:da:bb:91:b3:16:c4:b2:e8:89:30:9e:c1: + 5e:0b:cf:db:c4:c3:d9:b1:af:40:a5:0b:38:36:1b: + 14:fe:0f:22:9c:e6:59:6a:15:5b:db:f6:f7:f3:a5: + 02:29:94:7a:d2:0c:67:ad:aa:63:62:7e:fc:58:11: + 29:48:b8:3c:91:b2:73:7e:12:6b:f2:ea:36:77:0f: + 15:9b:46:95:ce:73:15:8d:c8:d9:97:57:03:90:33: + 2d:7d:f3:ee:e5:01:6d:d8:c6:da:ab:07:b9:dd:1c: + e0:4b:ce:6a:de:a8:d2:e3:c1:52:6d:83:3a:0a:f0: + ed:cf:f7:56:6a:87:0e:73:e3:12:82:2b:65:ab:d8: + a9:44:5b:4a:2f:a5:92:94:32:f1:a1:e4:af:18:0f: + 0f:18:60:cd:f7:d0:9d:03:9f:d7:e9:a8:60:54:bb: + 3b:9a:05:db:fd:38:04:3c:b4:23:41:16:6c:7c:3b: + d9:b6:e0:2f:bd:cb:62:55:1b:e8:d0:8f:43:76:ef: + 55:86:cf:25:c3:bc:ae:e3:46:50:89:f7:71:ad:06: + 5e:28:e6:f6:f0:76:27:ea:7e:1b:67:53:39:26:20: + 19:18:82:b1:11:5f:ea:91:c2:e3:d3:f6:5a:c7:fd: + 61:a2:92:de:7d:7c:da:6d:e8:bf:39:52:10:31:60: + 4b:e1 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + Netscape Cert Type: + SSL Server + Netscape Comment: + OpenSSL Generated Server Certificate + X509v3 Subject Key Identifier: + 17:07:3B:AA:85:83:B5:04:83:EC:B2:6C:1E:3A:F0:F5:59:AA:61:28 + X509v3 Subject Alternative Name: + DNS:localhost, DNS:unresolvable-broker-address, IP Address:127.0.0.1 + X509v3 Authority Key Identifier: + keyid:57:0B:E9:CB:23:E8:BF:47:3E:50:7A:3F:45:7E:A1:18:43:9D:15:27 + DirName:/CN=foobar + serial:D7:E2:87:4F:A0:79:E2:0C + + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication + Signature Algorithm: sha256WithRSAEncryption + e4:27:61:e2:0f:b6:a0:ca:9f:ce:e3:53:0b:44:ab:86:a1:e2: + 4d:88:e1:7d:2e:b0:aa:32:96:2b:3d:da:60:70:6a:c3:62:c5: + 76:f2:8f:0d:16:31:f2:ad:e5:2f:43:f3:cb:e4:fa:95:6c:20: + 81:33:1a:c7:5a:55:57:c9:ab:ca:66:45:30:58:00:db:e8:51: + c9:2c:a9:72:c1:18:f5:01:87:9f:73:20:85:6c:e5:6c:3f:c9: + 67:b4:f0:20:e5:ed:e2:4a:08:0b:af:68:43:e5:a9:c7:e1:39: + e8:b5:49:cb:47:4a:6d:e5:16:ae:88:92:13:85:8e:42:1e:0a: + eb:59:ed:a7:c1:9b:bc:4b:7b:99:f8:1d:f0:d7:1d:90:c9:cf: + 86:6a:d3:10:d0:36:e4:f5:b9:33:79:c7:a2:68:31:f7:bb:8d: + 1e:d6:33:79:bd:e7:0e:4f:4d:e9:2e:15:04:4f:6b:4b:2e:93: + 28:72:d1:0e:aa:ee:e6:ef:68:be:58:2b:cc:56:01:27:16:f9: + 34:8e:66:86:27:0a:b0:fb:32:56:a9:8a:d9:6f:b1:86:bd:ba: + fd:50:6c:d5:b2:54:e7:4e:c6:2d:19:88:a9:89:2c:ef:be:08: + 0d:2b:49:91:0b:09:42:64:06:a3:9d:d7:94:ed:e8:74:74:48: + 43:57:41:6f:e5:06:98:46:1d:c5:60:9c:69:f8:fb:fe:a6:01: + 4a:35:be:21:36:c2:a3:44:c8:c4:2c:21:09:f4:28:9a:ad:a0: + 97:1e:00:29:cc:0f:26:fa:59:21:25:c0:9e:fa:22:53:67:6d: + ab:a6:56:08:fd:37:1d:69:fe:ef:6f:29:89:1a:66:7b:c7:ff: + b1:34:f1:d6:be:21:81:e3:bc:4f:13:02:a7:4b:9d:13:05:46: + 40:88:4a:aa:db:fb:64:f8:6b:fb:5d:a0:b1:0c:1a:b8:4c:ab: + 6f:69:fe:0b:55:4e:b3:38:1f:91:0b:71:77:1e:11:39:54:9a: + 62:51:ea:6d:a8:5e:0d:4a:91:fb:d8:be:5d:93:e8:43:f3:4a: + 11:fb:31:cf:14:1a:1c:8d:31:1b:99:31:e0:2b:81:01:91:6f: + da:ba:cb:1f:51:21:55:29:3f:4c:71:e3:d0:29:41:de:a0:00: + da:07:ed:5e:c9:af:32:61:6d:55:f8:f5:2d:46:03:34:33:fb: + 2e:1e:aa:7c:fe:d2:30:4d:40:cc:ed:76:ec:f6:bd:ed:35:c8: + d8:b3:46:56:aa:2c:53:84:56:45:b0:a3:f6:35:66:93:da:8c: + 17:39:c1:29:7c:99:c5:0b:73:c1:f9:16:d0:57:fc:57:59:06: + af:39:9f:a9:51:35:0b:c7 -----BEGIN CERTIFICATE----- -MIIDgzCCAmugAwIBAgIIJDCWmNVVXt8wDQYJKoZIhvcNAQELBQAwQDELMAkGA1UE -BhMCRlIxETAPBgNVBAoMCERhdGFzdGF4MQ0wCwYDVQQLDARUZXN0MQ8wDQYDVQQD -DAZyb290Y2EwHhcNMjEwNTExMTcyNjU1WhcNMjMwODE0MTcyNjU1WjAYMRYwFAYD -VQQDDA1lbGFzdGljc2VhcmNoMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC -AQEAhlMnjkAdRlD9rZ7yMjTZK+GOfXjNiMZORrtIRSmILuOHw9GxEtSaWXFvDPD4 -a+JloIYK/N3CV5LvE/3zcB5jv6/X/TOAaz+eeQOMj9QIDjtX1ia4YLOe3FqVf3vB -2m9paIM6ug7sgDWIxYmpL3HYTfL4B7sSInsQzpErDDIH1IquvCo2pHRggKPbAgJ9 -8pOaCLgvY2JRLe4oj2EMB0fYpEzRRg7mQgkal3w0CySmatHvGqDw2dghHjkNdNFl -1hnikuE2HRvcx+MmA5ADSBfQU6IZKAetouQOMo3Fom2eSgoGLiT7+dlSoBAaxUnh -oDxHI6WkNdaTSjqC3AlF7xIGzQIDAQABo4GoMIGlMAkGA1UdEwQCMAAwDgYDVR0P -AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAdBgNVHQ4E -FgQUCxGbyzMIwC84X57K+LX16fgFrvEwHwYDVR0jBBgwFoAU8HikOW4q4jh2lVss -F08ir+OdZQwwKQYDVR0RBCIwIIINZWxhc3RpY3NlYXJjaIIJbG9jYWxob3N0hwR/ -AAABMA0GCSqGSIb3DQEBCwUAA4IBAQDHJiJHs9qg9UavG+dJxAtqleLObNem7cpH -06LaH0+Z2BExmw/GJdAsRIKIIL4xQLmbNaY5vpmTi5JSPNC/ZxvS1RLla0RbJv3s -dfTEhPfar37XJCiKB917wck47LdPu3FUdwbNKZY/tpUBaasRQ7nJdnJgJwb6zkr/ -ifN9NVDa8LpKK31qgA4bE7iJw0HY/4LkPdglQebjJtEHFCzi4AbPrHMX6xdxnCAM -G1DGldOjEbv7kTVACr1WCRZXu3vynJs+umLKX+twebvTsv6fpUnGj85AUt/p961v -6BblUYD3tJxCKkID9Lrs82C1sMxGqBkSDqwtNensaj1ba0v9HiYe +MIIExzCCAq+gAwIBAgICEAcwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v +YmFyMCAXDTIzMDUxMDE1NTAxOFoYDzIyOTcwMjIyMTU1MDE4WjAfMR0wGwYDVQQD +DBRicm9rZXItbG9jYWxob3N0LVNBTjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBAN7R2ruRsxbEsuiJMJ7BXgvP28TD2bGvQKULODYbFP4PIpzmWWoVW9v2 +9/OlAimUetIMZ62qY2J+/FgRKUi4PJGyc34Sa/LqNncPFZtGlc5zFY3I2ZdXA5Az +LX3z7uUBbdjG2qsHud0c4EvOat6o0uPBUm2DOgrw7c/3VmqHDnPjEoIrZavYqURb +Si+lkpQy8aHkrxgPDxhgzffQnQOf1+moYFS7O5oF2/04BDy0I0EWbHw72bbgL73L +YlUb6NCPQ3bvVYbPJcO8ruNGUIn3ca0GXijm9vB2J+p+G2dTOSYgGRiCsRFf6pHC +49P2Wsf9YaKS3n182m3ovzlSEDFgS+ECAwEAAaOCARcwggETMAkGA1UdEwQCMAAw +EQYJYIZIAYb4QgEBBAQDAgZAMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVy +YXRlZCBTZXJ2ZXIgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFBcHO6qFg7UEg+yybB46 +8PVZqmEoMDcGA1UdEQQwMC6CCWxvY2FsaG9zdIIbdW5yZXNvbHZhYmxlLWJyb2tl +ci1hZGRyZXNzhwR/AAABMEEGA1UdIwQ6MDiAFFcL6csj6L9HPlB6P0V+oRhDnRUn +oRWkEzARMQ8wDQYDVQQDDAZmb29iYXKCCQDX4odPoHniDDAOBgNVHQ8BAf8EBAMC +BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAOQnYeIP +tqDKn87jUwtEq4ah4k2I4X0usKoylis92mBwasNixXbyjw0WMfKt5S9D88vk+pVs +IIEzGsdaVVfJq8pmRTBYANvoUcksqXLBGPUBh59zIIVs5Ww/yWe08CDl7eJKCAuv +aEPlqcfhOei1SctHSm3lFq6IkhOFjkIeCutZ7afBm7xLe5n4HfDXHZDJz4Zq0xDQ +NuT1uTN5x6JoMfe7jR7WM3m95w5PTekuFQRPa0sukyhy0Q6q7ubvaL5YK8xWAScW ++TSOZoYnCrD7MlapitlvsYa9uv1QbNWyVOdOxi0ZiKmJLO++CA0rSZELCUJkBqOd +15Tt6HR0SENXQW/lBphGHcVgnGn4+/6mAUo1viE2wqNEyMQsIQn0KJqtoJceACnM +Dyb6WSElwJ76IlNnbaumVgj9Nx1p/u9vKYkaZnvH/7E08da+IYHjvE8TAqdLnRMF +RkCISqrb+2T4a/tdoLEMGrhMq29p/gtVTrM4H5ELcXceETlUmmJR6m2oXg1KkfvY +vl2T6EPzShH7Mc8UGhyNMRuZMeArgQGRb9q6yx9RIVUpP0xx49ApQd6gANoH7V7J +rzJhbVX49S1GAzQz+y4eqnz+0jBNQMztduz2ve01yNizRlaqLFOEVkWwo/Y1ZpPa +jBc5wSl8mcULc8H5FtBX/FdZBq85n6lRNQvH -----END CERTIFICATE----- diff --git a/pulsar-io/elastic-search/src/test/resources/ssl/elasticsearch.jks b/pulsar-io/elastic-search/src/test/resources/ssl/elasticsearch.jks deleted file mode 100644 index 0001e656925ae..0000000000000 Binary files a/pulsar-io/elastic-search/src/test/resources/ssl/elasticsearch.jks and /dev/null differ diff --git a/pulsar-io/elastic-search/src/test/resources/ssl/elasticsearch.key b/pulsar-io/elastic-search/src/test/resources/ssl/elasticsearch.key index da60f25cb4fc1..5c20238c7b9c9 100644 --- a/pulsar-io/elastic-search/src/test/resources/ssl/elasticsearch.key +++ b/pulsar-io/elastic-search/src/test/resources/ssl/elasticsearch.key @@ -1,27 +1,27 @@ -----BEGIN RSA PRIVATE KEY----- -MIIEpQIBAAKCAQEAhlMnjkAdRlD9rZ7yMjTZK+GOfXjNiMZORrtIRSmILuOHw9Gx -EtSaWXFvDPD4a+JloIYK/N3CV5LvE/3zcB5jv6/X/TOAaz+eeQOMj9QIDjtX1ia4 -YLOe3FqVf3vB2m9paIM6ug7sgDWIxYmpL3HYTfL4B7sSInsQzpErDDIH1IquvCo2 -pHRggKPbAgJ98pOaCLgvY2JRLe4oj2EMB0fYpEzRRg7mQgkal3w0CySmatHvGqDw -2dghHjkNdNFl1hnikuE2HRvcx+MmA5ADSBfQU6IZKAetouQOMo3Fom2eSgoGLiT7 -+dlSoBAaxUnhoDxHI6WkNdaTSjqC3AlF7xIGzQIDAQABAoIBABkgzTGLRO62j/4+ -0cgaotXWqDVIuApyLoqE1ow5rMQ4xFkZjSqcoxNAaSnemlp0S9avvLZ5NbX0Qs1W -oIfE55wgZSN00v09NYQl6dGuNKOU7mWdcLiGYZ2PRJBIsocOeIWYpw/PYubJHQD3 -M7MwdOqAHW40zLuukgJSnd72LQjRczPjUKJf5dGo6i7Md3JHaLIbFiY1NCxPj2q2 -Sf4TbU73tsdM/s0DvrhWDXpuSGLQWRkwovNWvu3aVWlpurGUYoXNmUw/gWCLeVQu -TXOxi3C5EVxigWhXFKw9W0ffQnwclletNI09M20Iw6C2vxEStFRByVPwyB+DS2Tc -CTufhYECgYEAx8oeKKkHvYdIjfgXe3uG+4xtB+gBpibGErd4108HBZre1On3Rd9f -8IPZhtgnSEsRYa58a3RcGl+Glan14DowyaX6Sazj0lODCYLn4EEEIqDZQ4JQlILD -+ynTG1dd0FhxHKYxkZd1zrTdsjReBcld+jo5xXYJDaXuMuXJBB4MSfkCgYEArB3t -jr2FsI5ORkeAzd8PZZMEPMDh/xrKS0BGqA7rz2UzZjRsW/ADYB1kPxrbOzAw5MGN -FvqJ7Wy76+VE18+MIiadRkVwIoIjmeD3ngQsq5AckLXBKsXo2BShc3ndjrq93Km0 -LvKuVq+OmtEElhnRPdDSPK1iMVw6tjEQRVTj+HUCgYEAnvU6H6G1Dd/II2Sj0lSL -JjMpQKQgQ8EP0S53D9/Jt3TsHDz2x3odTDCrbvUl9Af+EVkRlzEiYr1kSEnM3hWO -YbIUPHA5Z0uYs4Wa1bsE/kQ5+NODJ1YPdhNl6pgNhUoI3QGB6NO4ILmYtkOiFzqK -8h9HfcsXEuvVZFudHxCFOIkCgYEAm2n2vOIjGpaU1V7xTnPxKi3DMLyWFMzzB1oF -svCuOzpNXCdQBQBHe/1uLJEqjlpoe9bNeIdIDgwV7964+AnfpmKptu8yXuaWEPeT -f6qTT2M0d/K2dtH3+009r9DFn4C9uce+/HmDtKCXKvI7qvGq/59UhxekR5/ZH/RT -ldcedpUCgYEAmuwIZQJakt90HQ9GY4NlBRXJAailPzJq5c5jXscifJWbPrQFoKYn -uBfGuCWIKqUrLcKbuNl0H05MjCv/2qN/eVQW294ax8FUIzW7tU6af2YuIUeXUM6f -R08agqN+uEZMCEil8hVJsEWkp4pCpg2SbO+fkBnRkgkqYCpWqU9BfEc= +MIIEogIBAAKCAQEA3tHau5GzFsSy6IkwnsFeC8/bxMPZsa9ApQs4NhsU/g8inOZZ +ahVb2/b386UCKZR60gxnrapjYn78WBEpSLg8kbJzfhJr8uo2dw8Vm0aVznMVjcjZ +l1cDkDMtffPu5QFt2Mbaqwe53RzgS85q3qjS48FSbYM6CvDtz/dWaocOc+MSgitl +q9ipRFtKL6WSlDLxoeSvGA8PGGDN99CdA5/X6ahgVLs7mgXb/TgEPLQjQRZsfDvZ +tuAvvctiVRvo0I9Ddu9Vhs8lw7yu40ZQifdxrQZeKOb28HYn6n4bZ1M5JiAZGIKx +EV/qkcLj0/Zax/1hopLefXzabei/OVIQMWBL4QIDAQABAoIBAG0OMQxUx16Bbz84 +xj8tTSZi2aF4aff5Wp5s21o/7wpZxgsdAu5U/dyvB7SMMn6/WU2tHKF3H6V5mXBR +Fe+tnJeCy9wMkCEYWQf0rhKNfYzJ7uayQy07PVc6dS2MUoRrKqRKz4OeCG4cT/tK +UCwiXPV0DS/kZmgse3iqCfWhnIVC2AXWwkXLWIR2qvwAtqjWGReNWLPc3TTdP9jZ +0an4GmgI/YVM7ty5WiV9U1h24IpC6EHZkTZkzDMXy5dpMqMHgxAujHUigm3HQ8jO +OlWt8mNyk7gHYz+sdQC8wGqwQ4+s22tFae4PNDnCo5i2LfKxSJo8G+w/RGpzsthp +CtgsIQECgYEA/L+IDk3jWAuF6OkbYqbL/kArh/IYi3XnuChXc6rYZRCw34Z3Ayl3 +8HGSNzzBP2gVr7wJ/q6JJf1HwpuZ0H8F1tzGMjaZpNByY1gEHHsbHRwjPSNAoBKf +LiJ5/vBBapWFYdpbaEC4RriMDqqj5N3EFfCJlvgpqplmM20mcPezqqkCgYEA4a+/ +qDn/rJctDFX+VosR01ESQFOqdlck7WjbDYrizf1t1WsfUJfRxJt0T8dksRRCMZZT +3MG9VZrPzjZr09bd8Vhqe0yDl/9P5roTSBtvwXK+OzYFo4YleMxP032sJHD+/zLo +1MZdv85u0Ry61Nm5ovK1fjxqpVRNML+4io+G0nkCgYB5CN+GuhXc2+fMmZTbsTSP +FPvDplPKtTO6JNd8NJxcCZop8KKdiQY+xqRWf2mri5SXC11d8QcMgjYTI6CND8ck +FaVz9mGtY4Tjvgp5+RoK3qahaHhSL5i65xe01ij9eYzeR0ruqc+VTlsDywOhXfHA +7+dzvHN3lu4yQlreTkBjwQKBgC8GyBDtxPDZr3Famdy+rcAyrHLq/CIcln1B1CNG +RWxW2oQWBjhs6jDlk57sFXcwmXprsJ3XYPRthc+aV22U4DpiCg4XK3SIpsWVEF9+ +GBNfLXR5FUO1uCkrqZaQIUaWpQvYN9veWbqV8VFxgxzHIX9qw5bCUBaTMmJuEK4O +qjFBAoGAYQRv4i7xOv2daIenpMullA8e5y/hQw7tyWzQFrOkTA7PXxPu4gxI/DMW +S9D1J1TLEL3+NxMNW6MiBsJTbXUWeXS2ihQICfGG6iKbZxXxkP/LyoSvseBVsxHd +5Lkc0GT1UVVeTqem0adSm/DTDwkF2+9qon2FGiO7oq1CzUorfcI= -----END RSA PRIVATE KEY----- diff --git a/pulsar-io/elastic-search/src/test/resources/ssl/elasticsearch.pem b/pulsar-io/elastic-search/src/test/resources/ssl/elasticsearch.pem index 25b2a0f56cd60..dd9fa523e8ede 100644 --- a/pulsar-io/elastic-search/src/test/resources/ssl/elasticsearch.pem +++ b/pulsar-io/elastic-search/src/test/resources/ssl/elasticsearch.pem @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCGUyeOQB1GUP2t -nvIyNNkr4Y59eM2Ixk5Gu0hFKYgu44fD0bES1JpZcW8M8Phr4mWghgr83cJXku8T -/fNwHmO/r9f9M4BrP555A4yP1AgOO1fWJrhgs57cWpV/e8Hab2logzq6DuyANYjF -iakvcdhN8vgHuxIiexDOkSsMMgfUiq68KjakdGCAo9sCAn3yk5oIuC9jYlEt7iiP -YQwHR9ikTNFGDuZCCRqXfDQLJKZq0e8aoPDZ2CEeOQ100WXWGeKS4TYdG9zH4yYD -kANIF9BTohkoB62i5A4yjcWibZ5KCgYuJPv52VKgEBrFSeGgPEcjpaQ11pNKOoLc -CUXvEgbNAgMBAAECggEAGSDNMYtE7raP/j7RyBqi1daoNUi4CnIuioTWjDmsxDjE -WRmNKpyjE0BpKd6aWnRL1q+8tnk1tfRCzVagh8TnnCBlI3TS/T01hCXp0a40o5Tu -ZZ1wuIZhnY9EkEiyhw54hZinD89i5skdAPczszB06oAdbjTMu66SAlKd3vYtCNFz -M+NQol/l0ajqLsx3ckdoshsWJjU0LE+ParZJ/hNtTve2x0z+zQO+uFYNem5IYtBZ -GTCi81a+7dpVaWm6sZRihc2ZTD+BYIt5VC5Nc7GLcLkRXGKBaFcUrD1bR99CfByW -V600jT0zbQjDoLa/ERK0VEHJU/DIH4NLZNwJO5+FgQKBgQDHyh4oqQe9h0iN+Bd7 -e4b7jG0H6AGmJsYSt3jXTwcFmt7U6fdF31/wg9mG2CdISxFhrnxrdFwaX4aVqfXg -OjDJpfpJrOPSU4MJgufgQQQioNlDglCUgsP7KdMbV13QWHEcpjGRl3XOtN2yNF4F -yV36OjnFdgkNpe4y5ckEHgxJ+QKBgQCsHe2OvYWwjk5GR4DN3w9lkwQ8wOH/GspL -QEaoDuvPZTNmNGxb8ANgHWQ/Gts7MDDkwY0W+ontbLvr5UTXz4wiJp1GRXAigiOZ -4PeeBCyrkByQtcEqxejYFKFzed2Our3cqbQu8q5Wr46a0QSWGdE90NI8rWIxXDq2 -MRBFVOP4dQKBgQCe9TofobUN38gjZKPSVIsmMylApCBDwQ/RLncP38m3dOwcPPbH -eh1MMKtu9SX0B/4RWRGXMSJivWRISczeFY5hshQ8cDlnS5izhZrVuwT+RDn404Mn -Vg92E2XqmA2FSgjdAYHo07gguZi2Q6IXOoryH0d9yxcS69VkW50fEIU4iQKBgQCb -afa84iMalpTVXvFOc/EqLcMwvJYUzPMHWgWy8K47Ok1cJ1AFAEd7/W4skSqOWmh7 -1s14h0gODBXv3rj4Cd+mYqm27zJe5pYQ95N/qpNPYzR38rZ20ff7TT2v0MWfgL25 -x778eYO0oJcq8juq8ar/n1SHF6RHn9kf9FOV1x52lQKBgQCa7AhlAlqS33QdD0Zj -g2UFFckBqKU/MmrlzmNexyJ8lZs+tAWgpie4F8a4JYgqpSstwpu42XQfTkyMK//a -o395VBbb3hrHwVQjNbu1Tpp/Zi4hR5dQzp9HTxqCo364RkwISKXyFUmwRaSnikKm -DZJs75+QGdGSCSpgKlapT0F8Rw== +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDe0dq7kbMWxLLo +iTCewV4Lz9vEw9mxr0ClCzg2GxT+DyKc5llqFVvb9vfzpQIplHrSDGetqmNifvxY +ESlIuDyRsnN+Emvy6jZ3DxWbRpXOcxWNyNmXVwOQMy198+7lAW3YxtqrB7ndHOBL +zmreqNLjwVJtgzoK8O3P91Zqhw5z4xKCK2Wr2KlEW0ovpZKUMvGh5K8YDw8YYM33 +0J0Dn9fpqGBUuzuaBdv9OAQ8tCNBFmx8O9m24C+9y2JVG+jQj0N271WGzyXDvK7j +RlCJ93GtBl4o5vbwdifqfhtnUzkmIBkYgrERX+qRwuPT9lrH/WGikt59fNpt6L85 +UhAxYEvhAgMBAAECggEAbQ4xDFTHXoFvPzjGPy1NJmLZoXhp9/lanmzbWj/vClnG +Cx0C7lT93K8HtIwyfr9ZTa0coXcfpXmZcFEV762cl4LL3AyQIRhZB/SuEo19jMnu +5rJDLTs9Vzp1LYxShGsqpErPg54IbhxP+0pQLCJc9XQNL+RmaCx7eKoJ9aGchULY +BdbCRctYhHaq/AC2qNYZF41Ys9zdNN0/2NnRqfgaaAj9hUzu3LlaJX1TWHbgikLo +QdmRNmTMMxfLl2kyoweDEC6MdSKCbcdDyM46Va3yY3KTuAdjP6x1ALzAarBDj6zb +a0Vp7g80OcKjmLYt8rFImjwb7D9EanOy2GkK2CwhAQKBgQD8v4gOTeNYC4Xo6Rti +psv+QCuH8hiLdee4KFdzqthlELDfhncDKXfwcZI3PME/aBWvvAn+rokl/UfCm5nQ +fwXW3MYyNpmk0HJjWAQcexsdHCM9I0CgEp8uInn+8EFqlYVh2ltoQLhGuIwOqqPk +3cQV8ImW+CmqmWYzbSZw97OqqQKBgQDhr7+oOf+sly0MVf5WixHTURJAU6p2VyTt +aNsNiuLN/W3Vax9Ql9HEm3RPx2SxFEIxllPcwb1Vms/ONmvT1t3xWGp7TIOX/0/m +uhNIG2/Bcr47NgWjhiV4zE/TfawkcP7/MujUxl2/zm7RHLrU2bmi8rV+PGqlVE0w +v7iKj4bSeQKBgHkI34a6Fdzb58yZlNuxNI8U+8OmU8q1M7ok13w0nFwJminwop2J +Bj7GpFZ/aauLlJcLXV3xBwyCNhMjoI0PxyQVpXP2Ya1jhOO+Cnn5GgrepqFoeFIv +mLrnF7TWKP15jN5HSu6pz5VOWwPLA6Fd8cDv53O8c3eW7jJCWt5OQGPBAoGALwbI +EO3E8NmvcVqZ3L6twDKscur8IhyWfUHUI0ZFbFbahBYGOGzqMOWTnuwVdzCZemuw +nddg9G2Fz5pXbZTgOmIKDhcrdIimxZUQX34YE18tdHkVQ7W4KSuplpAhRpalC9g3 +295ZupXxUXGDHMchf2rDlsJQFpMyYm4Qrg6qMUECgYBhBG/iLvE6/Z1oh6eky6WU +Dx7nL+FDDu3JbNAWs6RMDs9fE+7iDEj8MxZL0PUnVMsQvf43Ew1boyIGwlNtdRZ5 +dLaKFAgJ8YbqIptnFfGQ/8vKhK+x4FWzEd3kuRzQZPVRVV5Op6bRp1Kb8NMPCQXb +72qifYUaI7uirULNSit9wg== -----END PRIVATE KEY----- diff --git a/pulsar-io/elastic-search/src/test/resources/ssl/keystore.jks b/pulsar-io/elastic-search/src/test/resources/ssl/keystore.jks index 37a3d3b0c90d5..f1e4176d0cf11 100644 Binary files a/pulsar-io/elastic-search/src/test/resources/ssl/keystore.jks and b/pulsar-io/elastic-search/src/test/resources/ssl/keystore.jks differ diff --git a/pulsar-io/elastic-search/src/test/resources/ssl/truststore.jks b/pulsar-io/elastic-search/src/test/resources/ssl/truststore.jks index 165ba17fa8057..0099d0ab956d0 100644 Binary files a/pulsar-io/elastic-search/src/test/resources/ssl/truststore.jks and b/pulsar-io/elastic-search/src/test/resources/ssl/truststore.jks differ diff --git a/pulsar-io/file/pom.xml b/pulsar-io/file/pom.xml index b8685972856a3..53d8b332fdf5e 100644 --- a/pulsar-io/file/pom.xml +++ b/pulsar-io/file/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-file diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml index e68ef45dbfd76..96fa665e51c09 100644 --- a/pulsar-io/flume/pom.xml +++ b/pulsar-io/flume/pom.xml @@ -25,16 +25,12 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-flume Pulsar IO :: Flume - - 1.8.2 - - ${project.groupId} @@ -54,7 +50,7 @@ org.apache.flume flume-ng-node - 1.9.0 + 1.11.0 pom @@ -145,7 +141,6 @@ org.owasp dependency-check-maven - ${dependency-check-maven.version} diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml index ae2845a791dcd..2d9b55e0a9204 100644 --- a/pulsar-io/hbase/pom.xml +++ b/pulsar-io/hbase/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-hbase Pulsar IO :: Hbase @@ -108,7 +108,6 @@ org.owasp dependency-check-maven - ${dependency-check-maven.version} diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml index 7b2f11c7b7c21..d5fb33c170db1 100644 --- a/pulsar-io/hdfs2/pom.xml +++ b/pulsar-io/hdfs2/pom.xml @@ -23,18 +23,18 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-hdfs2 Pulsar IO :: Hdfs2 - + ${project.groupId} pulsar-io-core ${project.version} - + com.fasterxml.jackson.core jackson-databind @@ -55,14 +55,18 @@ hadoop-client ${hadoop2.version} - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - + + log4j + log4j + + + org.slf4j + * + + + org.apache.avro + avro + @@ -70,7 +74,7 @@ commons-lang3 - + @@ -109,7 +113,6 @@ org.owasp dependency-check-maven - ${dependency-check-maven.version} diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml index 30f7368102df4..f5a87879db942 100644 --- a/pulsar-io/hdfs3/pom.xml +++ b/pulsar-io/hdfs3/pom.xml @@ -23,18 +23,18 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-hdfs3 Pulsar IO :: Hdfs3 - + ${project.groupId} pulsar-io-core ${project.version} - + com.fasterxml.jackson.core jackson-databind @@ -49,7 +49,7 @@ org.apache.commons commons-collections4 - + org.apache.hadoop hadoop-client @@ -65,7 +65,11 @@ org.slf4j - slf4j-log4j12 + * + + + org.apache.avro + avro @@ -76,7 +80,7 @@ - + @@ -115,7 +119,6 @@ org.owasp dependency-check-maven - ${dependency-check-maven.version} diff --git a/pulsar-io/http/pom.xml b/pulsar-io/http/pom.xml index 47e48648e570e..f78575ddf79ce 100644 --- a/pulsar-io/http/pom.xml +++ b/pulsar-io/http/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-http diff --git a/pulsar-io/influxdb/pom.xml b/pulsar-io/influxdb/pom.xml index cad7956ccfcce..22fe51cb87459 100644 --- a/pulsar-io/influxdb/pom.xml +++ b/pulsar-io/influxdb/pom.xml @@ -25,13 +25,18 @@ pulsar-io org.apache.pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-influxdb Pulsar IO :: InfluxDB + + ${project.groupId} + pulsar-io-common + ${project.version} + ${project.groupId} pulsar-io-core diff --git a/pulsar-io/influxdb/src/main/java/org/apache/pulsar/io/influxdb/InfluxDBGenericRecordSink.java b/pulsar-io/influxdb/src/main/java/org/apache/pulsar/io/influxdb/InfluxDBGenericRecordSink.java index 5b51461fc7b8e..0d431f84c52f2 100644 --- a/pulsar-io/influxdb/src/main/java/org/apache/pulsar/io/influxdb/InfluxDBGenericRecordSink.java +++ b/pulsar-io/influxdb/src/main/java/org/apache/pulsar/io/influxdb/InfluxDBGenericRecordSink.java @@ -46,12 +46,12 @@ public class InfluxDBGenericRecordSink implements Sink { @Override public void open(Map map, SinkContext sinkContext) throws Exception { try { - val configV2 = InfluxDBSinkConfig.load(map); + val configV2 = InfluxDBSinkConfig.load(map, sinkContext); configV2.validate(); sink = new InfluxDBSink(); } catch (Exception e) { try { - val configV1 = org.apache.pulsar.io.influxdb.v1.InfluxDBSinkConfig.load(map); + val configV1 = org.apache.pulsar.io.influxdb.v1.InfluxDBSinkConfig.load(map, sinkContext); configV1.validate(); sink = new org.apache.pulsar.io.influxdb.v1.InfluxDBGenericRecordSink(); } catch (Exception e1) { diff --git a/pulsar-io/influxdb/src/main/java/org/apache/pulsar/io/influxdb/v1/InfluxDBAbstractSink.java b/pulsar-io/influxdb/src/main/java/org/apache/pulsar/io/influxdb/v1/InfluxDBAbstractSink.java index 06856bad80edc..217c5304b24f7 100644 --- a/pulsar-io/influxdb/src/main/java/org/apache/pulsar/io/influxdb/v1/InfluxDBAbstractSink.java +++ b/pulsar-io/influxdb/src/main/java/org/apache/pulsar/io/influxdb/v1/InfluxDBAbstractSink.java @@ -43,7 +43,7 @@ public abstract class InfluxDBAbstractSink extends BatchSink { @Override public void open(Map config, SinkContext sinkContext) throws Exception { - InfluxDBSinkConfig influxDBSinkConfig = InfluxDBSinkConfig.load(config); + InfluxDBSinkConfig influxDBSinkConfig = InfluxDBSinkConfig.load(config, sinkContext); influxDBSinkConfig.validate(); super.init(influxDBSinkConfig.getBatchTimeMs(), influxDBSinkConfig.getBatchSize()); diff --git a/pulsar-io/influxdb/src/main/java/org/apache/pulsar/io/influxdb/v1/InfluxDBSinkConfig.java b/pulsar-io/influxdb/src/main/java/org/apache/pulsar/io/influxdb/v1/InfluxDBSinkConfig.java index 9b7d8e1ce905d..4ae2cf1e4a3a1 100644 --- a/pulsar-io/influxdb/src/main/java/org/apache/pulsar/io/influxdb/v1/InfluxDBSinkConfig.java +++ b/pulsar-io/influxdb/src/main/java/org/apache/pulsar/io/influxdb/v1/InfluxDBSinkConfig.java @@ -27,6 +27,8 @@ import java.util.Map; import lombok.Data; import lombok.experimental.Accessors; +import org.apache.pulsar.io.common.IOConfigUtils; +import org.apache.pulsar.io.core.SinkContext; import org.apache.pulsar.io.core.annotations.FieldDoc; /** @@ -94,7 +96,7 @@ public class InfluxDBSinkConfig implements Serializable { @FieldDoc( required = false, - defaultValue = "1000L", + defaultValue = "1000", help = "The InfluxDB operation time in milliseconds") private long batchTimeMs = 1000L; @@ -110,14 +112,11 @@ public static InfluxDBSinkConfig load(String yamlFile) throws IOException { return mapper.readValue(new File(yamlFile), InfluxDBSinkConfig.class); } - public static InfluxDBSinkConfig load(Map map) throws IOException { - ObjectMapper mapper = new ObjectMapper(); - return mapper.readValue(mapper.writeValueAsString(map), InfluxDBSinkConfig.class); + public static InfluxDBSinkConfig load(Map map, SinkContext sinkContext) throws IOException { + return IOConfigUtils.loadWithSecrets(map, InfluxDBSinkConfig.class, sinkContext); } public void validate() { - Preconditions.checkNotNull(influxdbUrl, "influxdbUrl property not set."); - Preconditions.checkNotNull(database, "database property not set."); Preconditions.checkArgument(batchSize > 0, "batchSize must be a positive integer."); Preconditions.checkArgument(batchTimeMs > 0, "batchTimeMs must be a positive long."); } diff --git a/pulsar-io/influxdb/src/main/java/org/apache/pulsar/io/influxdb/v2/InfluxDBSink.java b/pulsar-io/influxdb/src/main/java/org/apache/pulsar/io/influxdb/v2/InfluxDBSink.java index 08f1ab2339992..0aa43570596af 100644 --- a/pulsar-io/influxdb/src/main/java/org/apache/pulsar/io/influxdb/v2/InfluxDBSink.java +++ b/pulsar-io/influxdb/src/main/java/org/apache/pulsar/io/influxdb/v2/InfluxDBSink.java @@ -49,7 +49,7 @@ public class InfluxDBSink extends BatchSink { @Override public void open(Map config, SinkContext sinkContext) throws Exception { - InfluxDBSinkConfig influxDBSinkConfig = InfluxDBSinkConfig.load(config); + InfluxDBSinkConfig influxDBSinkConfig = InfluxDBSinkConfig.load(config, sinkContext); influxDBSinkConfig.validate(); super.init(influxDBSinkConfig.getBatchTimeMs(), influxDBSinkConfig.getBatchSize()); diff --git a/pulsar-io/influxdb/src/main/java/org/apache/pulsar/io/influxdb/v2/InfluxDBSinkConfig.java b/pulsar-io/influxdb/src/main/java/org/apache/pulsar/io/influxdb/v2/InfluxDBSinkConfig.java index 899b00c002155..ea87ee66b90a3 100644 --- a/pulsar-io/influxdb/src/main/java/org/apache/pulsar/io/influxdb/v2/InfluxDBSinkConfig.java +++ b/pulsar-io/influxdb/src/main/java/org/apache/pulsar/io/influxdb/v2/InfluxDBSinkConfig.java @@ -27,6 +27,8 @@ import java.util.Map; import lombok.Data; import lombok.experimental.Accessors; +import org.apache.pulsar.io.common.IOConfigUtils; +import org.apache.pulsar.io.core.SinkContext; import org.apache.pulsar.io.core.annotations.FieldDoc; /** @@ -87,7 +89,7 @@ public class InfluxDBSinkConfig implements Serializable { @FieldDoc( required = false, - defaultValue = "1000L", + defaultValue = "1000", help = "The InfluxDB operation time in milliseconds") private long batchTimeMs = 1000; @@ -103,17 +105,11 @@ public static InfluxDBSinkConfig load(String yamlFile) throws IOException { return mapper.readValue(new File(yamlFile), InfluxDBSinkConfig.class); } - public static InfluxDBSinkConfig load(Map map) throws IOException { - ObjectMapper mapper = new ObjectMapper(); - return mapper.readValue(mapper.writeValueAsString(map), InfluxDBSinkConfig.class); + public static InfluxDBSinkConfig load(Map map, SinkContext sinkContext) throws IOException { + return IOConfigUtils.loadWithSecrets(map, InfluxDBSinkConfig.class, sinkContext); } public void validate() { - Preconditions.checkNotNull(influxdbUrl, "influxdbUrl property not set."); - Preconditions.checkNotNull(token, "token property not set."); - Preconditions.checkNotNull(organization, "organization property not set."); - Preconditions.checkNotNull(bucket, "bucket property not set."); - Preconditions.checkArgument(batchSize > 0, "batchSize must be a positive integer."); Preconditions.checkArgument(batchTimeMs > 0, "batchTimeMs must be a positive long."); } diff --git a/pulsar-io/influxdb/src/test/java/org/apache/pulsar/io/influxdb/v1/InfluxDBSinkConfigTest.java b/pulsar-io/influxdb/src/test/java/org/apache/pulsar/io/influxdb/v1/InfluxDBSinkConfigTest.java index 4493dcfb24854..10b1bfb624f49 100644 --- a/pulsar-io/influxdb/src/test/java/org/apache/pulsar/io/influxdb/v1/InfluxDBSinkConfigTest.java +++ b/pulsar-io/influxdb/src/test/java/org/apache/pulsar/io/influxdb/v1/InfluxDBSinkConfigTest.java @@ -18,7 +18,9 @@ */ package org.apache.pulsar.io.influxdb.v1; +import org.apache.pulsar.io.core.SinkContext; import org.influxdb.InfluxDB; +import org.mockito.Mockito; import org.testng.annotations.Test; import java.io.File; @@ -60,8 +62,11 @@ public final void loadFromMapTest() throws IOException { map.put("gzipEnable", "false"); map.put("batchTimeMs", "1000"); map.put("batchSize", "100"); + map.put("username", "admin"); + map.put("password", "admin"); - InfluxDBSinkConfig config = InfluxDBSinkConfig.load(map); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + InfluxDBSinkConfig config = InfluxDBSinkConfig.load(map, sinkContext); assertNotNull(config); assertEquals("http://localhost:8086", config.getInfluxdbUrl()); assertEquals("test_db", config.getDatabase()); @@ -71,6 +76,39 @@ public final void loadFromMapTest() throws IOException { assertEquals(Boolean.parseBoolean("false"), config.isGzipEnable()); assertEquals(Long.parseLong("1000"), config.getBatchTimeMs()); assertEquals(Integer.parseInt("100"), config.getBatchSize()); + assertEquals("admin", config.getUsername()); + assertEquals("admin", config.getPassword()); + } + + @Test + public final void loadFromMapCredentialFromSecretTest() throws IOException { + Map map = new HashMap<>(); + map.put("influxdbUrl", "http://localhost:8086"); + map.put("database", "test_db"); + map.put("consistencyLevel", "ONE"); + map.put("logLevel", "NONE"); + map.put("retentionPolicy", "autogen"); + map.put("gzipEnable", "false"); + map.put("batchTimeMs", "1000"); + map.put("batchSize", "100"); + + SinkContext sinkContext = Mockito.mock(SinkContext.class); + Mockito.when(sinkContext.getSecret("username")) + .thenReturn("admin"); + Mockito.when(sinkContext.getSecret("password")) + .thenReturn("admin"); + InfluxDBSinkConfig config = InfluxDBSinkConfig.load(map, sinkContext); + assertNotNull(config); + assertEquals("http://localhost:8086", config.getInfluxdbUrl()); + assertEquals("test_db", config.getDatabase()); + assertEquals("ONE", config.getConsistencyLevel()); + assertEquals("NONE", config.getLogLevel()); + assertEquals("autogen", config.getRetentionPolicy()); + assertEquals(Boolean.parseBoolean("false"), config.isGzipEnable()); + assertEquals(Long.parseLong("1000"), config.getBatchTimeMs()); + assertEquals(Integer.parseInt("100"), config.getBatchSize()); + assertEquals("admin", config.getUsername()); + assertEquals("admin", config.getPassword()); } @Test @@ -85,12 +123,13 @@ public final void validValidateTest() throws IOException { map.put("batchTimeMs", "1000"); map.put("batchSize", "100"); - InfluxDBSinkConfig config = InfluxDBSinkConfig.load(map); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + InfluxDBSinkConfig config = InfluxDBSinkConfig.load(map, sinkContext); config.validate(); } - @Test(expectedExceptions = NullPointerException.class, - expectedExceptionsMessageRegExp = "influxdbUrl property not set.") + @Test(expectedExceptions = IllegalArgumentException.class, + expectedExceptionsMessageRegExp = "influxdbUrl cannot be null") public final void missingInfluxdbUrlValidateTest() throws IOException { Map map = new HashMap<>(); map.put("database", "test_db"); @@ -101,7 +140,8 @@ public final void missingInfluxdbUrlValidateTest() throws IOException { map.put("batchTimeMs", "1000"); map.put("batchSize", "100"); - InfluxDBSinkConfig config = InfluxDBSinkConfig.load(map); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + InfluxDBSinkConfig config = InfluxDBSinkConfig.load(map, sinkContext); config.validate(); } @@ -118,7 +158,8 @@ public final void invalidBatchSizeTest() throws IOException { map.put("batchTimeMs", "1000"); map.put("batchSize", "-100"); - InfluxDBSinkConfig config = InfluxDBSinkConfig.load(map); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + InfluxDBSinkConfig config = InfluxDBSinkConfig.load(map, sinkContext); config.validate(); } @@ -135,7 +176,8 @@ public final void invalidConsistencyLevelTest() throws IOException { map.put("batchTimeMs", "1000"); map.put("batchSize", "100"); - InfluxDBSinkConfig config = InfluxDBSinkConfig.load(map); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + InfluxDBSinkConfig config = InfluxDBSinkConfig.load(map, sinkContext); config.validate(); InfluxDB.ConsistencyLevel.valueOf(config.getConsistencyLevel().toUpperCase()); diff --git a/pulsar-io/influxdb/src/test/java/org/apache/pulsar/io/influxdb/v2/InfluxDBSinkConfigTest.java b/pulsar-io/influxdb/src/test/java/org/apache/pulsar/io/influxdb/v2/InfluxDBSinkConfigTest.java index df1f7fd29a637..d6cee1e308d2b 100644 --- a/pulsar-io/influxdb/src/test/java/org/apache/pulsar/io/influxdb/v2/InfluxDBSinkConfigTest.java +++ b/pulsar-io/influxdb/src/test/java/org/apache/pulsar/io/influxdb/v2/InfluxDBSinkConfigTest.java @@ -24,6 +24,8 @@ import java.io.File; import java.util.HashMap; import java.util.Map; +import org.apache.pulsar.io.core.SinkContext; +import org.mockito.Mockito; import org.testng.annotations.Test; public class InfluxDBSinkConfigTest { @@ -58,18 +60,34 @@ private Map buildValidConfigMap() { public final void testLoadFromMap() throws Exception { Map map = buildValidConfigMap(); - InfluxDBSinkConfig config = InfluxDBSinkConfig.load(map); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + InfluxDBSinkConfig config = InfluxDBSinkConfig.load(map, sinkContext); assertNotNull(config); config.validate(); verifyValues(config); } - @Test(expectedExceptions = NullPointerException.class, - expectedExceptionsMessageRegExp = "influxdbUrl property not set.") + @Test + public final void testLoadFromMapCredentialFromSecret() throws Exception { + Map map = buildValidConfigMap(); + map.remove("token"); + + SinkContext sinkContext = Mockito.mock(SinkContext.class); + Mockito.when(sinkContext.getSecret("token")) + .thenReturn("xxxx"); + InfluxDBSinkConfig config = InfluxDBSinkConfig.load(map, sinkContext); + assertNotNull(config); + config.validate(); + verifyValues(config); + } + + @Test(expectedExceptions = IllegalArgumentException.class, + expectedExceptionsMessageRegExp = "influxdbUrl cannot be null") public void testRequiredConfigMissing() throws Exception { Map map = buildValidConfigMap(); map.remove("influxdbUrl"); - InfluxDBSinkConfig config = InfluxDBSinkConfig.load(map); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + InfluxDBSinkConfig config = InfluxDBSinkConfig.load(map, sinkContext); config.validate(); } @@ -78,7 +96,8 @@ public void testRequiredConfigMissing() throws Exception { public void testBatchConfig() throws Exception { Map map = buildValidConfigMap(); map.put("batchSize", -1); - InfluxDBSinkConfig config = InfluxDBSinkConfig.load(map); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + InfluxDBSinkConfig config = InfluxDBSinkConfig.load(map, sinkContext); config.validate(); } diff --git a/pulsar-io/jdbc/clickhouse/pom.xml b/pulsar-io/jdbc/clickhouse/pom.xml index 14dc030a753e4..069667efca855 100644 --- a/pulsar-io/jdbc/clickhouse/pom.xml +++ b/pulsar-io/jdbc/clickhouse/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/core/pom.xml b/pulsar-io/jdbc/core/pom.xml index 5c2e97c7fdc29..68c0e194b3692 100644 --- a/pulsar-io/jdbc/core/pom.xml +++ b/pulsar-io/jdbc/core/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT 4.0.0 @@ -32,12 +32,24 @@ Pulsar IO :: Jdbc :: Core + + ${project.groupId} + pulsar-io-common + ${project.version} + + ${project.groupId} pulsar-io-core ${project.version} + + ${project.groupId} + pulsar-client-original + ${project.version} + + org.apache.avro avro diff --git a/pulsar-io/jdbc/core/src/main/java/org/apache/pulsar/io/jdbc/BaseJdbcAutoSchemaSink.java b/pulsar-io/jdbc/core/src/main/java/org/apache/pulsar/io/jdbc/BaseJdbcAutoSchemaSink.java index 36c3674091932..c1f44cf37efdf 100644 --- a/pulsar-io/jdbc/core/src/main/java/org/apache/pulsar/io/jdbc/BaseJdbcAutoSchemaSink.java +++ b/pulsar-io/jdbc/core/src/main/java/org/apache/pulsar/io/jdbc/BaseJdbcAutoSchemaSink.java @@ -33,7 +33,9 @@ import org.apache.pulsar.client.api.schema.GenericObject; import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.schema.KeyValueSchema; +import org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord; import org.apache.pulsar.common.schema.KeyValue; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.io.jdbc.JdbcUtils.ColumnId; @@ -137,6 +139,10 @@ public Mutation createMutation(Record message) { } recordValueGetter = (k) -> data.get(k); } else { + SchemaType schemaType = message.getSchema().getSchemaInfo().getType(); + if (schemaType.isPrimitive()) { + throw new UnsupportedOperationException("Primitive schema is not supported: " + schemaType); + } recordValueGetter = (key) -> ((GenericRecord) record).getField(key); } String action = message.getProperties().get(ACTION_PROPERTY); @@ -168,7 +174,7 @@ private static void setColumnNull(PreparedStatement statement, int index, int ty } - private static void setColumnValue(PreparedStatement statement, int index, Object value) throws Exception { + protected void setColumnValue(PreparedStatement statement, int index, Object value) throws Exception { log.debug("Setting column value, statement: {}, index: {}, value: {}", statement, index, value); @@ -188,6 +194,8 @@ private static void setColumnValue(PreparedStatement statement, int index, Objec statement.setShort(index, (Short) value); } else if (value instanceof ByteString) { statement.setBytes(index, ((ByteString) value).toByteArray()); + } else if (value instanceof GenericJsonRecord) { + statement.setString(index, ((GenericJsonRecord) value).getJsonNode().toString()); } else { throw new Exception("Not supported value type, need to add it. " + value.getClass()); } diff --git a/pulsar-io/jdbc/core/src/main/java/org/apache/pulsar/io/jdbc/JdbcAbstractSink.java b/pulsar-io/jdbc/core/src/main/java/org/apache/pulsar/io/jdbc/JdbcAbstractSink.java index 4586fcebcf167..ca33b3cfdaba9 100644 --- a/pulsar-io/jdbc/core/src/main/java/org/apache/pulsar/io/jdbc/JdbcAbstractSink.java +++ b/pulsar-io/jdbc/core/src/main/java/org/apache/pulsar/io/jdbc/JdbcAbstractSink.java @@ -76,7 +76,7 @@ public abstract class JdbcAbstractSink implements Sink { @Override public void open(Map config, SinkContext sinkContext) throws Exception { - jdbcSinkConfig = JdbcSinkConfig.load(config); + jdbcSinkConfig = JdbcSinkConfig.load(config, sinkContext); jdbcSinkConfig.validate(); jdbcUrl = jdbcSinkConfig.getJdbcUrl(); diff --git a/pulsar-io/jdbc/core/src/main/java/org/apache/pulsar/io/jdbc/JdbcSinkConfig.java b/pulsar-io/jdbc/core/src/main/java/org/apache/pulsar/io/jdbc/JdbcSinkConfig.java index f798d94f7c35e..854d68381312c 100644 --- a/pulsar-io/jdbc/core/src/main/java/org/apache/pulsar/io/jdbc/JdbcSinkConfig.java +++ b/pulsar-io/jdbc/core/src/main/java/org/apache/pulsar/io/jdbc/JdbcSinkConfig.java @@ -26,6 +26,8 @@ import java.util.Map; import lombok.Data; import lombok.experimental.Accessors; +import org.apache.pulsar.io.common.IOConfigUtils; +import org.apache.pulsar.io.core.SinkContext; import org.apache.pulsar.io.core.annotations.FieldDoc; @Data @@ -145,9 +147,8 @@ public static JdbcSinkConfig load(String yamlFile) throws IOException { return mapper.readValue(new File(yamlFile), JdbcSinkConfig.class); } - public static JdbcSinkConfig load(Map map) throws IOException { - ObjectMapper mapper = new ObjectMapper(); - return mapper.readValue(mapper.writeValueAsString(map), JdbcSinkConfig.class); + public static JdbcSinkConfig load(Map map, SinkContext sinkContext) throws IOException { + return IOConfigUtils.loadWithSecrets(map, JdbcSinkConfig.class, sinkContext); } public void validate() { diff --git a/pulsar-io/jdbc/core/src/test/java/org/apache/pulsar/io/jdbc/BaseJdbcAutoSchemaSinkTest.java b/pulsar-io/jdbc/core/src/test/java/org/apache/pulsar/io/jdbc/BaseJdbcAutoSchemaSinkTest.java index b15eb832242c7..8cb6219deb89e 100644 --- a/pulsar-io/jdbc/core/src/test/java/org/apache/pulsar/io/jdbc/BaseJdbcAutoSchemaSinkTest.java +++ b/pulsar-io/jdbc/core/src/test/java/org/apache/pulsar/io/jdbc/BaseJdbcAutoSchemaSinkTest.java @@ -18,10 +18,24 @@ */ package org.apache.pulsar.io.jdbc; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import java.lang.reflect.Field; +import java.sql.PreparedStatement; +import java.util.Arrays; +import java.util.List; import java.util.function.Function; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; import org.apache.avro.util.Utf8; +import org.apache.pulsar.client.api.schema.GenericObject; +import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; +import org.apache.pulsar.client.impl.schema.generic.GenericJsonSchema; +import org.apache.pulsar.functions.api.Record; import org.testng.Assert; import org.testng.annotations.Test; @@ -143,5 +157,84 @@ private Schema createFieldAndGetSchema(Function record = new Record() { + @Override + public org.apache.pulsar.client.api.Schema getSchema() { + return autoConsumeSchema; + } + + @Override + public GenericRecord getValue() { + return null; + } + }; + baseJdbcAutoSchemaSink.createMutation((Record) record); + } + + + @Test + @SuppressWarnings("unchecked") + public void testSubFieldJsonArray() throws Exception { + BaseJdbcAutoSchemaSink baseJdbcAutoSchemaSink = new BaseJdbcAutoSchemaSink() {}; + + Field field = JdbcAbstractSink.class.getDeclaredField("jdbcSinkConfig"); + field.setAccessible(true); + JdbcSinkConfig jdbcSinkConfig = new JdbcSinkConfig(); + jdbcSinkConfig.setNullValueAction(JdbcSinkConfig.NullValueAction.FAIL); + field.set(baseJdbcAutoSchemaSink, jdbcSinkConfig); + + TStates tStates = new TStates("tstats", Arrays.asList( + new PC("brand1", "model1"), + new PC("brand2", "model2") + )); + org.apache.pulsar.client.api.Schema jsonSchema = org.apache.pulsar.client.api.Schema.JSON(TStates.class); + GenericJsonSchema genericJsonSchema = new GenericJsonSchema(jsonSchema.getSchemaInfo()); + byte[] encode = jsonSchema.encode(tStates); + GenericRecord genericRecord = genericJsonSchema.decode(encode); + + AutoConsumeSchema autoConsumeSchema = new AutoConsumeSchema(); + autoConsumeSchema.setSchema(org.apache.pulsar.client.api.Schema.JSON(TStates.class)); + Record record = new Record() { + @Override + public org.apache.pulsar.client.api.Schema getSchema() { + return genericJsonSchema; + } + + @Override + public GenericRecord getValue() { + return genericRecord; + } + }; + JdbcAbstractSink.Mutation mutation = baseJdbcAutoSchemaSink.createMutation((Record) record); + PreparedStatement mockPreparedStatement = mock(PreparedStatement.class); + baseJdbcAutoSchemaSink.setColumnValue(mockPreparedStatement, 0, mutation.getValues().apply("state")); + baseJdbcAutoSchemaSink.setColumnValue(mockPreparedStatement, 1, mutation.getValues().apply("pcList")); + verify(mockPreparedStatement).setString(0, "tstats"); + verify(mockPreparedStatement).setString(1, "[{\"brand\":\"brand1\",\"model\":\"model1\"},{\"brand\":\"brand2\",\"model\":\"model2\"}]"); + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + private static class TStates { + public String state; + public List pcList; + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + private static class PC { + public String brand; + public String model; + } + } \ No newline at end of file diff --git a/pulsar-io/jdbc/mariadb/pom.xml b/pulsar-io/jdbc/mariadb/pom.xml index 943406a8b900f..d83448f75e174 100644 --- a/pulsar-io/jdbc/mariadb/pom.xml +++ b/pulsar-io/jdbc/mariadb/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/openmldb/pom.xml b/pulsar-io/jdbc/openmldb/pom.xml index 92376583d25ba..23fbe7fab726d 100644 --- a/pulsar-io/jdbc/openmldb/pom.xml +++ b/pulsar-io/jdbc/openmldb/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/pom.xml b/pulsar-io/jdbc/pom.xml index 5a82e163c554b..43da97320501d 100644 --- a/pulsar-io/jdbc/pom.xml +++ b/pulsar-io/jdbc/pom.xml @@ -33,7 +33,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-jdbc diff --git a/pulsar-io/jdbc/postgres/pom.xml b/pulsar-io/jdbc/postgres/pom.xml index e7177cb2e19aa..526c4d86d049a 100644 --- a/pulsar-io/jdbc/postgres/pom.xml +++ b/pulsar-io/jdbc/postgres/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/sqlite/pom.xml b/pulsar-io/jdbc/sqlite/pom.xml index 5cf8833e6684a..aff60760c6ae9 100644 --- a/pulsar-io/jdbc/sqlite/pom.xml +++ b/pulsar-io/jdbc/sqlite/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT 4.0.0 pulsar-io-jdbc-sqlite diff --git a/pulsar-io/jdbc/sqlite/src/test/java/org/apache/pulsar/io/jdbc/SqliteJdbcSinkTest.java b/pulsar-io/jdbc/sqlite/src/test/java/org/apache/pulsar/io/jdbc/SqliteJdbcSinkTest.java index d9ed4cbd442bf..ca01615bef193 100644 --- a/pulsar-io/jdbc/sqlite/src/test/java/org/apache/pulsar/io/jdbc/SqliteJdbcSinkTest.java +++ b/pulsar-io/jdbc/sqlite/src/test/java/org/apache/pulsar/io/jdbc/SqliteJdbcSinkTest.java @@ -48,6 +48,7 @@ import org.apache.pulsar.client.api.schema.RecordSchemaBuilder; import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.MessageImpl; +import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; import org.apache.pulsar.client.impl.schema.AvroSchema; import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; import org.apache.pulsar.common.schema.KeyValue; @@ -282,9 +283,12 @@ public void TestUnknownAction() throws Exception { } @Test + @SuppressWarnings("unchecked") public void TestUpdateAction() throws Exception { AvroSchema schema = AvroSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); + AutoConsumeSchema autoConsumeSchema = new AutoConsumeSchema(); + autoConsumeSchema.setSchema(schema); Foo updateObj = new Foo(); updateObj.setField1("ValueOfField3"); @@ -292,10 +296,11 @@ public void TestUpdateAction() throws Exception { updateObj.setField3(4); byte[] updateBytes = schema.encode(updateObj); - Message updateMessage = mock(MessageImpl.class); + Message updateMessage = mock(MessageImpl.class); CompletableFuture future = new CompletableFuture<>(); - Record updateRecord = PulsarRecord.builder() + Record updateRecord = PulsarRecord.builder() .message(updateMessage) + .schema(autoConsumeSchema) .topicName("fake_topic_name") .ackFunction(() -> future.complete(null)) .build(); @@ -312,7 +317,7 @@ public void TestUpdateAction() throws Exception { updateMessage.getValue().toString(), updateRecord.getValue().toString()); - jdbcSink.write(updateRecord); + jdbcSink.write((Record) updateRecord); future.get(1, TimeUnit.SECONDS); // value has been written to db, read it out and verify. @@ -325,18 +330,22 @@ public void TestUpdateAction() throws Exception { } @Test + @SuppressWarnings("unchecked") public void TestDeleteAction() throws Exception { AvroSchema schema = AvroSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); + AutoConsumeSchema autoConsumeSchema = new AutoConsumeSchema(); + autoConsumeSchema.setSchema(schema); Foo deleteObj = new Foo(); deleteObj.setField3(5); byte[] deleteBytes = schema.encode(deleteObj); - Message deleteMessage = mock(MessageImpl.class); + Message deleteMessage = mock(MessageImpl.class); CompletableFuture future = new CompletableFuture<>(); - Record deleteRecord = PulsarRecord.builder() + Record deleteRecord = PulsarRecord.builder() .message(deleteMessage) + .schema(autoConsumeSchema) .topicName("fake_topic_name") .ackFunction(() -> future.complete(null)) .build(); @@ -352,7 +361,7 @@ public void TestDeleteAction() throws Exception { deleteMessage.getValue().toString(), deleteRecord.getValue().toString()); - jdbcSink.write(deleteRecord); + jdbcSink.write((Record) deleteRecord); future.get(1, TimeUnit.SECONDS); // value has been written to db, read it out and verify. @@ -848,17 +857,21 @@ public void testNullValueAction(NullValueActionTestConfig config) throws Excepti } } + @SuppressWarnings("unchecked") private Record createMockFooRecord(Foo record, Map actionProperties, CompletableFuture future) { - Message insertMessage = mock(MessageImpl.class); + Message insertMessage = mock(MessageImpl.class); GenericSchema genericAvroSchema; AvroSchema schema = AvroSchema.of(SchemaDefinition.builder().withPojo(Foo.class).withAlwaysAllowNull(true).build()); + AutoConsumeSchema autoConsumeSchema = new AutoConsumeSchema(); + autoConsumeSchema.setSchema(schema); byte[] insertBytes = schema.encode(record); - Record insertRecord = PulsarRecord.builder() + Record insertRecord = PulsarRecord.builder() .message(insertMessage) .topicName("fake_topic_name") + .schema(autoConsumeSchema) .ackFunction(() -> future.complete(true)) .failFunction(() -> future.complete(false)) .build(); @@ -866,7 +879,7 @@ private Record createMockFooRecord(Foo record, Map) insertRecord; } } diff --git a/pulsar-io/kafka-connect-adaptor-nar/pom.xml b/pulsar-io/kafka-connect-adaptor-nar/pom.xml index b3d547d333127..6b4518205f00c 100644 --- a/pulsar-io/kafka-connect-adaptor-nar/pom.xml +++ b/pulsar-io/kafka-connect-adaptor-nar/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-kafka-connect-adaptor-nar diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index 71a3cee11dcd0..bffdeb5ebf108 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-kafka-connect-adaptor @@ -46,6 +46,12 @@ compile + + org.apache.pulsar + pulsar-functions-utils + ${project.version} + + com.fasterxml.jackson.core jackson-databind @@ -73,6 +79,10 @@ org.eclipse.jetty * + + jose4j + org.bitbucket.b_c + @@ -80,12 +90,24 @@ org.apache.kafka connect-json ${kafka-client.version} + + + jose4j + org.bitbucket.b_c + + org.apache.kafka connect-api ${kafka-client.version} + + + jose4j + org.bitbucket.b_c + + @@ -116,6 +138,12 @@ io.confluent kafka-connect-avro-converter ${confluent.version} + + + org.apache.avro + avro + + @@ -130,6 +158,12 @@ connect-file ${kafka-client.version} test + + + jose4j + org.bitbucket.b_c + + diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaSinkTaskContext.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaSinkTaskContext.java index 7a908b553a89a..760799e0daa29 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaSinkTaskContext.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaSinkTaskContext.java @@ -40,7 +40,7 @@ import org.apache.kafka.connect.sink.SinkTaskContext; import org.apache.kafka.connect.storage.OffsetBackingStore; import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.client.util.MessageIdUtils; +import org.apache.pulsar.functions.utils.FunctionCommon; import org.apache.pulsar.io.core.SinkContext; @Slf4j @@ -150,7 +150,7 @@ private void seekAndUpdateOffset(TopicPartition topicPartition, long offset) { try { ctx.seek(desanitizeTopicName.apply(topicPartition.topic()), topicPartition.partition(), - MessageIdUtils.getMessageId(offset)); + FunctionCommon.getMessageId(offset)); } catch (PulsarClientException e) { log.error("Failed to seek topic {} partition {} offset {}", topicPartition.topic(), topicPartition.partition(), offset, e); diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java index 1100b13b425b4..1bcd244200199 100644 --- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java +++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java @@ -64,12 +64,12 @@ import org.apache.pulsar.client.impl.schema.SchemaInfoImpl; import org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord; import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; -import org.apache.pulsar.client.util.MessageIdUtils; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.functions.source.PulsarRecord; +import org.apache.pulsar.functions.utils.FunctionCommon; import org.apache.pulsar.io.core.SinkContext; import org.apache.pulsar.io.kafka.connect.schema.KafkaConnectData; import org.apache.pulsar.io.kafka.connect.schema.PulsarSchemaToKafkaSchema; @@ -303,8 +303,8 @@ public void seekPauseResumeTest() throws Exception { assertEquals(status.get(), 1); final TopicPartition tp = new TopicPartition("fake-topic", 0); - assertNotEquals(MessageIdUtils.getOffset(msgId), 0); - assertEquals(sink.currentOffset(tp.topic(), tp.partition()), MessageIdUtils.getOffset(msgId)); + assertNotEquals(FunctionCommon.getSequenceId(msgId), 0); + assertEquals(sink.currentOffset(tp.topic(), tp.partition()), FunctionCommon.getSequenceId(msgId)); sink.taskContext.offset(tp, 0); verify(context, times(1)).seek(Mockito.anyString(), Mockito.anyInt(), any()); @@ -347,12 +347,12 @@ public void seekPauseResumeWithSanitizeTest() throws Exception { assertEquals(status.get(), 1); final TopicPartition tp = new TopicPartition(sink.sanitizeNameIfNeeded(pulsarTopicName, true), 0); - assertNotEquals(MessageIdUtils.getOffset(msgId), 0); - assertEquals(sink.currentOffset(tp.topic(), tp.partition()), MessageIdUtils.getOffset(msgId)); + assertNotEquals(FunctionCommon.getSequenceId(msgId), 0); + assertEquals(sink.currentOffset(tp.topic(), tp.partition()), FunctionCommon.getSequenceId(msgId)); sink.taskContext.offset(tp, 0); verify(context, times(1)).seek(pulsarTopicName, - tp.partition(), MessageIdUtils.getMessageId(0)); + tp.partition(), FunctionCommon.getMessageId(0)); assertEquals(sink.currentOffset(tp.topic(), tp.partition()), 0); sink.taskContext.pause(tp); diff --git a/pulsar-io/kafka/pom.xml b/pulsar-io/kafka/pom.xml index 608a3c21591a8..d2031d064b843 100644 --- a/pulsar-io/kafka/pom.xml +++ b/pulsar-io/kafka/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-kafka @@ -46,6 +46,11 @@ + + ${project.groupId} + pulsar-io-common + ${project.version} + ${project.groupId} @@ -79,6 +84,12 @@ org.apache.kafka kafka-clients ${kafka-client.version} + + + jose4j + org.bitbucket.b_c + + @@ -109,6 +120,12 @@ test + + org.awaitility + awaitility + test + + diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/AvroSchemaCache.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/AvroSchemaCache.java index f0ad79549eb13..4e3abe245beb9 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/AvroSchemaCache.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/AvroSchemaCache.java @@ -38,9 +38,9 @@ final class AvroSchemaCache { private final LoadingCache> cache = CacheBuilder .newBuilder() .maximumSize(100) - .build(new CacheLoader>() { + .build(new CacheLoader<>() { @Override - public Schema load(Integer schemaId) throws Exception { + public Schema load(Integer schemaId) { return fetchSchema(schemaId); } }); diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/ByteBufferSchemaWrapper.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/ByteBufferSchemaWrapper.java index fb4683e511dd6..aefe62e38570c 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/ByteBufferSchemaWrapper.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/ByteBufferSchemaWrapper.java @@ -29,7 +29,7 @@ class ByteBufferSchemaWrapper implements Schema { private final Supplier original; - public ByteBufferSchemaWrapper(Schema original) { + public ByteBufferSchemaWrapper(Schema original) { this(original::getSchemaInfo); } diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/BytesWithKafkaSchema.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/BytesWithKafkaSchema.java index f0429c92eedfd..585cd927c30cd 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/BytesWithKafkaSchema.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/BytesWithKafkaSchema.java @@ -26,6 +26,6 @@ */ @Value public class BytesWithKafkaSchema { - private final ByteBuffer value; - private final int schemaId; + ByteBuffer value; + int schemaId; } diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSink.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSink.java index 8fbd6c8186110..2bedba928b756 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSink.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSink.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.util.Map; -import java.util.Objects; import java.util.Properties; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -44,7 +43,7 @@ public abstract class KafkaAbstractSink implements Sink { private Producer producer; - private Properties props = new Properties(); + private final Properties props = new Properties(); private KafkaSinkConfig kafkaSinkConfig; @Override @@ -79,10 +78,7 @@ protected Properties beforeCreateProducer(Properties props) { @Override public void open(Map config, SinkContext sinkContext) throws Exception { - kafkaSinkConfig = KafkaSinkConfig.load(config); - Objects.requireNonNull(kafkaSinkConfig.getTopic(), "Kafka topic is not set"); - Objects.requireNonNull(kafkaSinkConfig.getBootstrapServers(), "Kafka bootstrapServers is not set"); - Objects.requireNonNull(kafkaSinkConfig.getAcks(), "Kafka acks mode is not set"); + kafkaSinkConfig = KafkaSinkConfig.load(config, sinkContext); if (kafkaSinkConfig.getBatchSize() <= 0) { throw new IllegalArgumentException("Invalid Kafka Producer batchSize : " + kafkaSinkConfig.getBatchSize()); diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java index 565c36047474b..7eba7438b2b1d 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java @@ -27,7 +27,7 @@ import java.util.Optional; import java.util.Properties; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -64,10 +64,11 @@ public abstract class KafkaAbstractSource extends PushSource { private volatile boolean running = false; private KafkaSourceConfig kafkaSourceConfig; private Thread runnerThread; + private long maxPollIntervalMs; @Override public void open(Map config, SourceContext sourceContext) throws Exception { - kafkaSourceConfig = KafkaSourceConfig.load(config); + kafkaSourceConfig = KafkaSourceConfig.load(config, sourceContext); Objects.requireNonNull(kafkaSourceConfig.getTopic(), "Kafka topic is not set"); Objects.requireNonNull(kafkaSourceConfig.getBootstrapServers(), "Kafka bootstrapServers is not set"); Objects.requireNonNull(kafkaSourceConfig.getGroupId(), "Kafka consumer group id is not set"); @@ -127,13 +128,19 @@ public void open(Map config, SourceContext sourceContext) throws props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, kafkaSourceConfig.getAutoOffsetReset()); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, kafkaSourceConfig.getKeyDeserializationClass()); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, kafkaSourceConfig.getValueDeserializationClass()); + if (props.containsKey(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG)) { + maxPollIntervalMs = Long.parseLong(props.get(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG).toString()); + } else { + maxPollIntervalMs = Long.parseLong( + ConsumerConfig.configDef().defaultValues().get(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG) + .toString()); + } try { consumer = new KafkaConsumer<>(beforeCreateConsumer(props)); } catch (Exception ex) { throw new IllegalArgumentException("Unable to instantiate Kafka consumer", ex); } this.start(); - running = true; } protected Properties beforeCreateConsumer(Properties props) { @@ -158,43 +165,43 @@ public void close() throws InterruptedException { @SuppressWarnings("unchecked") public void start() { + LOG.info("Starting subscribe kafka source on {}", kafkaSourceConfig.getTopic()); + consumer.subscribe(Collections.singletonList(kafkaSourceConfig.getTopic())); runnerThread = new Thread(() -> { - LOG.info("Starting kafka source on {}", kafkaSourceConfig.getTopic()); - consumer.subscribe(Collections.singletonList(kafkaSourceConfig.getTopic())); LOG.info("Kafka source started."); while (running) { - ConsumerRecords consumerRecords = consumer.poll(Duration.ofSeconds(1L)); - CompletableFuture[] futures = new CompletableFuture[consumerRecords.count()]; - int index = 0; - for (ConsumerRecord consumerRecord : consumerRecords) { - KafkaRecord record = buildRecord(consumerRecord); - if (LOG.isDebugEnabled()) { - LOG.debug("Write record {} {} {}", record.getKey(), record.getValue(), record.getSchema()); + try { + ConsumerRecords consumerRecords = consumer.poll(Duration.ofSeconds(1L)); + CompletableFuture[] futures = new CompletableFuture[consumerRecords.count()]; + int index = 0; + for (ConsumerRecord consumerRecord : consumerRecords) { + KafkaRecord record = buildRecord(consumerRecord); + if (LOG.isDebugEnabled()) { + LOG.debug("Write record {} {} {}", record.getKey(), record.getValue(), record.getSchema()); + } + consume(record); + futures[index] = record.getCompletableFuture(); + index++; } - consume(record); - futures[index] = record.getCompletableFuture(); - index++; - } - if (!kafkaSourceConfig.isAutoCommitEnabled()) { - try { - CompletableFuture.allOf(futures).get(); + if (!kafkaSourceConfig.isAutoCommitEnabled()) { + // Wait about 2/3 of the time of maxPollIntervalMs. + // so as to avoid waiting for the timeout to be kicked out of the consumer group. + CompletableFuture.allOf(futures).get(maxPollIntervalMs * 2 / 3, TimeUnit.MILLISECONDS); consumer.commitSync(); - } catch (InterruptedException ex) { - break; - } catch (ExecutionException ex) { - LOG.error("Error while processing records", ex); - break; } + } catch (Exception e) { + LOG.error("Error while processing records", e); + notifyError(e); + break; } } }); - runnerThread.setUncaughtExceptionHandler( - (t, e) -> LOG.error("[{}] Error while consuming records", t.getName(), e)); + running = true; runnerThread.setName("Kafka Source Thread"); runnerThread.start(); } - public abstract KafkaRecord buildRecord(ConsumerRecord consumerRecord); + public abstract KafkaRecord buildRecord(ConsumerRecord consumerRecord); protected Map copyKafkaHeaders(ConsumerRecord consumerRecord) { if (!kafkaSourceConfig.isCopyHeadersEnabled()) { @@ -212,7 +219,7 @@ protected Map copyKafkaHeaders(ConsumerRecord co @Slf4j protected static class KafkaRecord implements Record { - private final ConsumerRecord record; + private final ConsumerRecord record; private final V value; private final Schema schema; private final Map properties; @@ -220,7 +227,7 @@ protected static class KafkaRecord implements Record { @Getter private final CompletableFuture completableFuture = new CompletableFuture<>(); - public KafkaRecord(ConsumerRecord record, V value, Schema schema, + public KafkaRecord(ConsumerRecord record, V value, Schema schema, Map properties) { this.record = record; this.value = value; @@ -244,7 +251,7 @@ public Optional getRecordSequence() { @Override public Optional getKey() { - return Optional.ofNullable(record.key()); + return Optional.ofNullable(record.key() instanceof String ? (String) record.key() : null); } @Override @@ -257,6 +264,21 @@ public void ack() { completableFuture.complete(null); } + @Override + public void fail() { + completableFuture.completeExceptionally( + new RuntimeException( + String.format( + "Failed to process record with kafka topic: %s partition: %d offset: %d key: %s", + record.topic(), + record.partition(), + record.offset(), + getKey() + ) + ) + ); + } + @Override public Schema getSchema() { return schema; @@ -267,13 +289,14 @@ public Map getProperties(){ return properties; } } - protected static class KeyValueKafkaRecord extends KafkaRecord implements KVRecord { - private final Schema keySchema; - private final Schema valueSchema; + protected static class KeyValueKafkaRecord extends KafkaRecord implements KVRecord { + + private final Schema keySchema; + private final Schema valueSchema; - public KeyValueKafkaRecord(ConsumerRecord record, KeyValue value, - Schema keySchema, Schema valueSchema, + public KeyValueKafkaRecord(ConsumerRecord record, KeyValue value, + Schema keySchema, Schema valueSchema, Map properties) { super(record, value, null, properties); this.keySchema = keySchema; @@ -281,12 +304,12 @@ public KeyValueKafkaRecord(ConsumerRecord record, KeyValue value, } @Override - public Schema getKeySchema() { + public Schema getKeySchema() { return keySchema; } @Override - public Schema getValueSchema() { + public Schema getValueSchema() { return valueSchema; } diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSink.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSink.java index 2a200531b67cb..0ed2385699511 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSink.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSink.java @@ -29,7 +29,7 @@ import org.apache.pulsar.io.core.annotations.IOType; /** - * Kafka sink should treats incoming messages as pure bytes. So we don't + * Kafka sink should treat incoming messages as pure bytes. So we don't * apply schema into it. */ @Connector( diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSource.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSource.java index 4e35d98e0bb71..51408f0451977 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSource.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaBytesSource.java @@ -70,8 +70,8 @@ public class KafkaBytesSource extends KafkaAbstractSource { private AvroSchemaCache schemaCache; - private Schema keySchema; - private Schema valueSchema; + private Schema keySchema; + private Schema valueSchema; private boolean produceKeyValue; @Override @@ -93,11 +93,11 @@ protected Properties beforeCreateConsumer(Properties props) { } if (keySchema.getSchemaInfo().getType() != SchemaType.STRING) { - // if the Key is a String we can use native Pulsar Key - // otherwise we use KeyValue schema - // that allows you to set a schema for the Key and a schema for the Value. - // using SEPARATED encoding the key is saved into the binary key - // so it is used for routing and for compaction + // If the Key is a String we can use native Pulsar Key. + // Otherwise, we use KeyValue schema. + // That allows you to set a schema for the Key and a schema for the Value. + // Using SEPARATED encoding the key is saved into the binary key, + // so it is used for routing and for compaction. produceKeyValue = true; } @@ -114,13 +114,13 @@ private void initSchemaCache(Properties props) { } @Override - public KafkaRecord buildRecord(ConsumerRecord consumerRecord) { + public KafkaRecord buildRecord(ConsumerRecord consumerRecord) { if (produceKeyValue) { - Object key = extractSimpleValue(consumerRecord.key()); - Object value = extractSimpleValue(consumerRecord.value()); - Schema currentKeySchema = getSchemaFromObject(consumerRecord.key(), keySchema); - Schema currentValueSchema = getSchemaFromObject(consumerRecord.value(), valueSchema); - return new KeyValueKafkaRecord(consumerRecord, + ByteBuffer key = extractSimpleValue(consumerRecord.key()); + ByteBuffer value = extractSimpleValue(consumerRecord.value()); + Schema currentKeySchema = getSchemaFromObject(consumerRecord.key(), keySchema); + Schema currentValueSchema = getSchemaFromObject(consumerRecord.value(), valueSchema); + return new KeyValueKafkaRecord(consumerRecord, new KeyValue<>(key, value), currentKeySchema, currentValueSchema, @@ -128,7 +128,7 @@ public KafkaRecord buildRecord(ConsumerRecord consumerRecord) { } else { Object value = consumerRecord.value(); - return new KafkaRecord(consumerRecord, + return new KafkaRecord<>(consumerRecord, extractSimpleValue(value), getSchemaFromObject(value, valueSchema), copyKafkaHeaders(consumerRecord)); @@ -152,7 +152,7 @@ private static ByteBuffer extractSimpleValue(Object value) { } } - private Schema getSchemaFromObject(Object value, Schema fallback) { + private Schema getSchemaFromObject(Object value, Schema fallback) { if (value instanceof BytesWithKafkaSchema) { // this is a Struct with schema downloaded by the schema registry // the schema may be different from record to record @@ -179,7 +179,7 @@ private static Schema getSchemaFromDeserializerAndAdaptConfiguration result = Schema.BYTEBUFFER; } else if (StringDeserializer.class.getName().equals(kafkaDeserializerClass)) { if (isKey) { - // for the key we use the String value and we want StringDeserializer + // for the key we use the String value, and we want StringDeserializer props.put(key, kafkaDeserializerClass); } result = Schema.STRING; @@ -206,11 +206,11 @@ private static Schema getSchemaFromDeserializerAndAdaptConfiguration return new ByteBufferSchemaWrapper(result); } - Schema getKeySchema() { + Schema getKeySchema() { return keySchema; } - Schema getValueSchema() { + Schema getValueSchema() { return valueSchema; } diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaSinkConfig.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaSinkConfig.java index dbbf1a7b5e2a0..b63e3756693bb 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaSinkConfig.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaSinkConfig.java @@ -26,6 +26,8 @@ import java.util.Map; import lombok.Data; import lombok.experimental.Accessors; +import org.apache.pulsar.io.common.IOConfigUtils; +import org.apache.pulsar.io.core.SinkContext; import org.apache.pulsar.io.core.annotations.FieldDoc; @Data @@ -43,44 +45,38 @@ public class KafkaSinkConfig implements Serializable { private String bootstrapServers; @FieldDoc( - required = false, defaultValue = "", help = "Protocol used to communicate with Kafka brokers.") private String securityProtocol; @FieldDoc( - required = false, defaultValue = "", help = "SASL mechanism used for Kafka client connections.") private String saslMechanism; @FieldDoc( - required = false, defaultValue = "", help = "JAAS login context parameters for SASL connections in the format used by JAAS configuration files.") private String saslJaasConfig; @FieldDoc( - required = false, defaultValue = "", help = "The list of protocols enabled for SSL connections.") private String sslEnabledProtocols; @FieldDoc( - required = false, defaultValue = "", help = "The endpoint identification algorithm to validate server hostname using server certificate.") private String sslEndpointIdentificationAlgorithm; @FieldDoc( - required = false, defaultValue = "", help = "The location of the trust store file.") private String sslTruststoreLocation; @FieldDoc( - required = false, defaultValue = "", + sensitive = true, help = "The password for the trust store file.") private String sslTruststorePassword; @@ -91,12 +87,12 @@ public class KafkaSinkConfig implements Serializable { + " before considering a request complete. This controls the durability of records that are sent.") private String acks; @FieldDoc( - defaultValue = "16384L", + defaultValue = "16384", help = "The batch size that Kafka producer will attempt to batch records together" + " before sending them to brokers.") private long batchSize = 16384L; @FieldDoc( - defaultValue = "1048576L", + defaultValue = "1048576", help = "The maximum size of a Kafka request in bytes.") private long maxRequestSize = 1048576L; @@ -129,8 +125,7 @@ public static KafkaSinkConfig load(String yamlFile) throws IOException { return mapper.readValue(new File(yamlFile), KafkaSinkConfig.class); } - public static KafkaSinkConfig load(Map map) throws IOException { - ObjectMapper mapper = new ObjectMapper(); - return mapper.readValue(mapper.writeValueAsString(map), KafkaSinkConfig.class); + public static KafkaSinkConfig load(Map map, SinkContext sinkContext) throws IOException { + return IOConfigUtils.loadWithSecrets(map, KafkaSinkConfig.class, sinkContext); } -} \ No newline at end of file +} diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaSourceConfig.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaSourceConfig.java index ad2e121d26abf..bc278ce22c64c 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaSourceConfig.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaSourceConfig.java @@ -27,6 +27,7 @@ import java.util.Map; import lombok.Data; import lombok.experimental.Accessors; +import org.apache.pulsar.io.core.SourceContext; import org.apache.pulsar.io.core.annotations.FieldDoc; @Data @@ -44,44 +45,38 @@ public class KafkaSourceConfig implements Serializable { private String bootstrapServers; @FieldDoc( - required = false, defaultValue = "", help = "Protocol used to communicate with Kafka brokers.") private String securityProtocol; @FieldDoc( - required = false, defaultValue = "", help = "SASL mechanism used for Kafka client connections.") private String saslMechanism; @FieldDoc( - required = false, defaultValue = "", help = "JAAS login context parameters for SASL connections in the format used by JAAS configuration files.") private String saslJaasConfig; @FieldDoc( - required = false, defaultValue = "", help = "The list of protocols enabled for SSL connections.") private String sslEnabledProtocols; @FieldDoc( - required = false, defaultValue = "", help = "The endpoint identification algorithm to validate server hostname using server certificate.") private String sslEndpointIdentificationAlgorithm; @FieldDoc( - required = false, defaultValue = "", help = "The location of the trust store file.") private String sslTruststoreLocation; @FieldDoc( - required = false, defaultValue = "", + sensitive = true, help = "The password for the trust store file.") private String sslTruststorePassword; @@ -158,9 +153,15 @@ public static KafkaSourceConfig load(String yamlFile) throws IOException { return mapper.readValue(new File(yamlFile), KafkaSourceConfig.class); } - public static KafkaSourceConfig load(Map map) throws IOException { + public static KafkaSourceConfig load(Map map, SourceContext sourceContext) throws IOException { ObjectMapper mapper = new ObjectMapper(); + // since the KafkaSourceConfig requires the ACCEPT_EMPTY_STRING_AS_NULL_OBJECT feature + // We manually set the sensitive fields here instead of calling `IOConfigUtils.loadWithSecrets` + String sslTruststorePassword = sourceContext.getSecret("sslTruststorePassword"); + if (sslTruststorePassword != null) { + map.put("sslTruststorePassword", sslTruststorePassword); + } mapper.enable(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT); return mapper.readValue(mapper.writeValueAsString(map), KafkaSourceConfig.class); } -} \ No newline at end of file +} diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaStringSource.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaStringSource.java index 58df1838b2eb1..0c96528cc9349 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaStringSource.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaStringSource.java @@ -27,14 +27,12 @@ */ public class KafkaStringSource extends KafkaAbstractSource { - @Override - public KafkaRecord buildRecord(ConsumerRecord consumerRecord) { - KafkaRecord record = new KafkaRecord(consumerRecord, + public KafkaRecord buildRecord(ConsumerRecord consumerRecord) { + return new KafkaRecord<>(consumerRecord, new String((byte[]) consumerRecord.value(), StandardCharsets.UTF_8), Schema.STRING, copyKafkaHeaders(consumerRecord)); - return record; } } diff --git a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/ByteBufferSchemaWrapperTest.java b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/ByteBufferSchemaWrapperTest.java index e7f108bf60146..eff4b294851c3 100644 --- a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/ByteBufferSchemaWrapperTest.java +++ b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/ByteBufferSchemaWrapperTest.java @@ -32,7 +32,7 @@ public class ByteBufferSchemaWrapperTest { @Test - public void testGetBytesNoCopy() throws Exception { + public void testGetBytesNoCopy() { byte[] originalArray = {1, 2, 3}; ByteBuffer wrapped = ByteBuffer.wrap(originalArray); assertEquals(0, wrapped.arrayOffset()); @@ -41,7 +41,7 @@ public void testGetBytesNoCopy() throws Exception { } @Test - public void testGetBytesOffsetZeroDifferentLen() throws Exception { + public void testGetBytesOffsetZeroDifferentLen() { byte[] originalArray = {1, 2, 3}; ByteBuffer wrapped = ByteBuffer.wrap(originalArray, 1, 2); assertEquals(0, wrapped.arrayOffset()); @@ -52,7 +52,7 @@ public void testGetBytesOffsetZeroDifferentLen() throws Exception { } @Test - public void testGetBytesOffsetNonZero() throws Exception { + public void testGetBytesOffsetNonZero() { byte[] originalArray = {1, 2, 3}; ByteBuffer wrapped = ByteBuffer.wrap(originalArray); wrapped.position(1); @@ -66,7 +66,7 @@ public void testGetBytesOffsetNonZero() throws Exception { } @Test - public void testGetBytesOffsetZero() throws Exception { + public void testGetBytesOffsetZero() { byte[] originalArray = {1, 2, 3}; ByteBuffer wrapped = ByteBuffer.wrap(originalArray, 0, 2); assertEquals(0, wrapped.arrayOffset()); diff --git a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/KafkaBytesSourceTest.java b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/KafkaBytesSourceTest.java index d370d51cc23c1..401bd64c36f84 100644 --- a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/KafkaBytesSourceTest.java +++ b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/KafkaBytesSourceTest.java @@ -44,6 +44,8 @@ import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.io.core.SourceContext; +import org.apache.pulsar.io.kafka.KafkaAbstractSource.KafkaRecord; +import org.apache.pulsar.io.kafka.KafkaAbstractSource.KeyValueKafkaRecord; import org.bouncycastle.util.encoders.Base64; import org.mockito.Mockito; import org.testng.annotations.Test; @@ -88,24 +90,25 @@ public void testNoKeyValueSchema() throws Exception { } - private void validateSchemaNoKeyValue(String keyDeserializationClass, Schema expectedKeySchema, - String valueDeserializationClass, Schema expectedValueSchema) throws Exception { - KafkaBytesSource source = new KafkaBytesSource(); - Map config = new HashMap<>(); - config.put("topic","test"); - config.put("bootstrapServers","localhost:9092"); - config.put("groupId", "test"); - config.put("valueDeserializationClass", valueDeserializationClass); - config.put("keyDeserializationClass", keyDeserializationClass); - config.put("consumerConfigProperties", ImmutableMap.builder() - .put("schema.registry.url", "http://localhost:8081") - .build()); - source.open(config, Mockito.mock(SourceContext.class)); - assertFalse(source.isProduceKeyValue()); - Schema keySchema = source.getKeySchema(); - Schema valueSchema = source.getValueSchema(); - assertEquals(keySchema.getSchemaInfo().getType(), expectedKeySchema.getSchemaInfo().getType()); - assertEquals(valueSchema.getSchemaInfo().getType(), expectedValueSchema.getSchemaInfo().getType()); + private void validateSchemaNoKeyValue(String keyDeserializationClass, Schema expectedKeySchema, + String valueDeserializationClass, Schema expectedValueSchema) throws Exception { + try (KafkaBytesSource source = new KafkaBytesSource()) { + Map config = new HashMap<>(); + config.put("topic", "test"); + config.put("bootstrapServers", "localhost:9092"); + config.put("groupId", "test"); + config.put("valueDeserializationClass", valueDeserializationClass); + config.put("keyDeserializationClass", keyDeserializationClass); + config.put("consumerConfigProperties", ImmutableMap.builder() + .put("schema.registry.url", "http://localhost:8081") + .build()); + source.open(config, Mockito.mock(SourceContext.class)); + assertFalse(source.isProduceKeyValue()); + Schema keySchema = source.getKeySchema(); + Schema valueSchema = source.getValueSchema(); + assertEquals(keySchema.getSchemaInfo().getType(), expectedKeySchema.getSchemaInfo().getType()); + assertEquals(valueSchema.getSchemaInfo().getType(), expectedValueSchema.getSchemaInfo().getType()); + } } @Test @@ -120,96 +123,98 @@ public void testKeyValueSchema() throws Exception { public void testCopyKafkaHeadersEnabled() throws Exception { ByteBuffer key = ByteBuffer.wrap(new IntegerSerializer().serialize("test", 10)); ByteBuffer value = ByteBuffer.wrap(new StringSerializer().serialize("test", "test")); - KafkaBytesSource source = new KafkaBytesSource(); - Map config = new HashMap<>(); - config.put("copyHeadersEnabled", true); - config.put("topic","test"); - config.put("bootstrapServers","localhost:9092"); - config.put("groupId", "test"); - config.put("valueDeserializationClass", IntegerDeserializer.class.getName()); - config.put("keyDeserializationClass", StringDeserializer.class.getName()); - config.put("consumerConfigProperties", ImmutableMap.builder() - .put("schema.registry.url", "http://localhost:8081") - .build()); - source.open(config, Mockito.mock(SourceContext.class)); - ConsumerRecord record = new ConsumerRecord("test", 88, 99, key, value); - record.headers().add("k1", "v1".getBytes(StandardCharsets.UTF_8)); - record.headers().add("k2", new byte[]{0xF}); - - Map props = source.copyKafkaHeaders(record); - assertEquals(props.size(), 5); - assertTrue(props.containsKey("__kafka_topic")); - assertTrue(props.containsKey("__kafka_partition")); - assertTrue(props.containsKey("__kafka_offset")); - assertTrue(props.containsKey("k1")); - assertTrue(props.containsKey("k2")); - - assertEquals(props.get("__kafka_topic"), "test"); - assertEquals(props.get("__kafka_partition"), "88"); - assertEquals(props.get("__kafka_offset"), "99"); - assertEquals(Base64.decode(props.get("k1")), "v1".getBytes(StandardCharsets.UTF_8)); - assertEquals(Base64.decode(props.get("k2")), new byte[]{0xF}); + try (KafkaBytesSource source = new KafkaBytesSource()) { + Map config = new HashMap<>(); + config.put("copyHeadersEnabled", true); + config.put("topic", "test"); + config.put("bootstrapServers", "localhost:9092"); + config.put("groupId", "test"); + config.put("valueDeserializationClass", IntegerDeserializer.class.getName()); + config.put("keyDeserializationClass", StringDeserializer.class.getName()); + config.put("consumerConfigProperties", ImmutableMap.builder() + .put("schema.registry.url", "http://localhost:8081") + .build()); + source.open(config, Mockito.mock(SourceContext.class)); + ConsumerRecord record = new ConsumerRecord<>("test", 88, 99, key, value); + record.headers().add("k1", "v1".getBytes(StandardCharsets.UTF_8)); + record.headers().add("k2", new byte[]{0xF}); + + Map props = source.copyKafkaHeaders(record); + assertEquals(props.size(), 5); + assertTrue(props.containsKey("__kafka_topic")); + assertTrue(props.containsKey("__kafka_partition")); + assertTrue(props.containsKey("__kafka_offset")); + assertTrue(props.containsKey("k1")); + assertTrue(props.containsKey("k2")); + + assertEquals(props.get("__kafka_topic"), "test"); + assertEquals(props.get("__kafka_partition"), "88"); + assertEquals(props.get("__kafka_offset"), "99"); + assertEquals(Base64.decode(props.get("k1")), "v1".getBytes(StandardCharsets.UTF_8)); + assertEquals(Base64.decode(props.get("k2")), new byte[]{0xF}); + } } @Test public void testCopyKafkaHeadersDisabled() throws Exception { ByteBuffer key = ByteBuffer.wrap(new IntegerSerializer().serialize("test", 10)); ByteBuffer value = ByteBuffer.wrap(new StringSerializer().serialize("test", "test")); - KafkaBytesSource source = new KafkaBytesSource(); - Map config = new HashMap<>(); - config.put("topic","test"); - config.put("bootstrapServers","localhost:9092"); - config.put("groupId", "test"); - config.put("valueDeserializationClass", IntegerDeserializer.class.getName()); - config.put("keyDeserializationClass", StringDeserializer.class.getName()); - config.put("consumerConfigProperties", ImmutableMap.builder() - .put("schema.registry.url", "http://localhost:8081") - .build()); - source.open(config, Mockito.mock(SourceContext.class)); - ConsumerRecord record = new ConsumerRecord("test", 88, 99, key, value); - record.headers().add("k1", "v1".getBytes(StandardCharsets.UTF_8)); - record.headers().add("k2", new byte[]{0xF}); - - Map props = source.copyKafkaHeaders(record); - assertTrue(props.isEmpty()); + try (KafkaBytesSource source = new KafkaBytesSource()) { + Map config = new HashMap<>(); + config.put("topic", "test"); + config.put("bootstrapServers", "localhost:9092"); + config.put("groupId", "test"); + config.put("valueDeserializationClass", IntegerDeserializer.class.getName()); + config.put("keyDeserializationClass", StringDeserializer.class.getName()); + config.put("consumerConfigProperties", ImmutableMap.builder() + .put("schema.registry.url", "http://localhost:8081") + .build()); + source.open(config, Mockito.mock(SourceContext.class)); + ConsumerRecord record = new ConsumerRecord<>("test", 88, 99, key, value); + record.headers().add("k1", "v1".getBytes(StandardCharsets.UTF_8)); + record.headers().add("k2", new byte[]{0xF}); + + Map props = source.copyKafkaHeaders(record); + assertTrue(props.isEmpty()); + } } - private void validateSchemaKeyValue(String keyDeserializationClass, Schema expectedKeySchema, - String valueDeserializationClass, Schema expectedValueSchema, + private void validateSchemaKeyValue(String keyDeserializationClass, Schema expectedKeySchema, + String valueDeserializationClass, Schema expectedValueSchema, ByteBuffer key, ByteBuffer value) throws Exception { - KafkaBytesSource source = new KafkaBytesSource(); - Map config = new HashMap<>(); - config.put("topic","test"); - config.put("bootstrapServers","localhost:9092"); - config.put("groupId", "test"); - config.put("valueDeserializationClass", valueDeserializationClass); - config.put("keyDeserializationClass", keyDeserializationClass); - config.put("consumerConfigProperties", ImmutableMap.builder() - .put("schema.registry.url", "http://localhost:8081") - .build()); - source.open(config, Mockito.mock(SourceContext.class)); - assertTrue(source.isProduceKeyValue()); - Schema keySchema = source.getKeySchema(); - Schema valueSchema = source.getValueSchema(); - assertEquals(keySchema.getSchemaInfo().getType(), expectedKeySchema.getSchemaInfo().getType()); - assertEquals(valueSchema.getSchemaInfo().getType(), expectedValueSchema.getSchemaInfo().getType()); - - KafkaAbstractSource.KafkaRecord record = source.buildRecord(new ConsumerRecord("test", 0, 0, key, value)); - assertThat(record, instanceOf(KafkaAbstractSource.KeyValueKafkaRecord.class)); - KafkaAbstractSource.KeyValueKafkaRecord kvRecord = (KafkaAbstractSource.KeyValueKafkaRecord) record; - assertSame(keySchema, kvRecord.getKeySchema()); - assertSame(valueSchema, kvRecord.getValueSchema()); - assertEquals(KeyValueEncodingType.SEPARATED, kvRecord.getKeyValueEncodingType()); - KeyValue kvValue = (KeyValue) kvRecord.getValue(); - log.info("key {}", Arrays.toString(toArray(key))); - log.info("value {}", Arrays.toString(toArray(value))); - - log.info("key {}", Arrays.toString(toArray((ByteBuffer) kvValue.getKey()))); - log.info("value {}", Arrays.toString(toArray((ByteBuffer) kvValue.getValue()))); - - assertEquals(ByteBuffer.wrap(toArray(key)).compareTo((ByteBuffer) kvValue.getKey()), 0); - assertEquals(ByteBuffer.wrap(toArray(value)).compareTo((ByteBuffer) kvValue.getValue()), 0); + try (KafkaBytesSource source = new KafkaBytesSource()) { + Map config = new HashMap<>(); + config.put("topic", "test"); + config.put("bootstrapServers", "localhost:9092"); + config.put("groupId", "test"); + config.put("valueDeserializationClass", valueDeserializationClass); + config.put("keyDeserializationClass", keyDeserializationClass); + config.put("consumerConfigProperties", ImmutableMap.builder() + .put("schema.registry.url", "http://localhost:8081") + .build()); + source.open(config, Mockito.mock(SourceContext.class)); + assertTrue(source.isProduceKeyValue()); + Schema keySchema = source.getKeySchema(); + Schema valueSchema = source.getValueSchema(); + assertEquals(keySchema.getSchemaInfo().getType(), expectedKeySchema.getSchemaInfo().getType()); + assertEquals(valueSchema.getSchemaInfo().getType(), expectedValueSchema.getSchemaInfo().getType()); + + KafkaRecord record = source.buildRecord(new ConsumerRecord<>("test", 0, 0, key, value)); + assertThat(record, instanceOf(KeyValueKafkaRecord.class)); + KeyValueKafkaRecord kvRecord = (KeyValueKafkaRecord) record; + assertSame(keySchema, kvRecord.getKeySchema()); + assertSame(valueSchema, kvRecord.getValueSchema()); + assertEquals(KeyValueEncodingType.SEPARATED, kvRecord.getKeyValueEncodingType()); + KeyValue kvValue = (KeyValue) kvRecord.getValue(); + log.info("key {}", Arrays.toString(toArray(key))); + log.info("value {}", Arrays.toString(toArray(value))); + log.info("key {}", Arrays.toString(toArray(kvValue.getKey()))); + log.info("value {}", Arrays.toString(toArray(kvValue.getValue()))); + + assertEquals(ByteBuffer.wrap(toArray(key)).compareTo(kvValue.getKey()), 0); + assertEquals(ByteBuffer.wrap(toArray(value)).compareTo(kvValue.getValue()), 0); + } } private static byte[] toArray(ByteBuffer b) { diff --git a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/sink/KafkaAbstractSinkTest.java b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/sink/KafkaAbstractSinkTest.java index ec9ee4a957d78..0f6920690d6cf 100644 --- a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/sink/KafkaAbstractSinkTest.java +++ b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/sink/KafkaAbstractSinkTest.java @@ -47,7 +47,7 @@ public class KafkaAbstractSinkTest { private static class DummySink extends KafkaAbstractSink { @Override - public KeyValue extractKeyValue(Record record) { + public KeyValue extractKeyValue(Record record) { return new KeyValue<>(record.getKey().orElse(null), record.getValue()); } } @@ -74,7 +74,7 @@ private static void expectThrows(Class expectedType, St @Test public void testInvalidConfigWillThrownException() throws Exception { - KafkaAbstractSink sink = new DummySink(); + KafkaAbstractSink sink = new DummySink(); Map config = new HashMap<>(); SinkContext sc = new SinkContext() { @Override @@ -164,12 +164,12 @@ public ByteBuffer getState(String key) { public CompletableFuture getStateAsync(String key) { return null; } - + @Override public void deleteState(String key) { - + } - + @Override public CompletableFuture deleteStateAsync(String key) { return null; @@ -179,6 +179,11 @@ public CompletableFuture deleteStateAsync(String key) { public PulsarClient getPulsarClient() { return null; } + + @Override + public void fatal(Throwable t) { + + } }; ThrowingRunnable openAndClose = ()->{ try { @@ -188,12 +193,12 @@ public PulsarClient getPulsarClient() { sink.close(); } }; - expectThrows(NullPointerException.class, "Kafka topic is not set", openAndClose); - config.put("topic", "topic_2"); - expectThrows(NullPointerException.class, "Kafka bootstrapServers is not set", openAndClose); + expectThrows(IllegalArgumentException.class, "bootstrapServers cannot be null", openAndClose); config.put("bootstrapServers", "localhost:6667"); - expectThrows(NullPointerException.class, "Kafka acks mode is not set", openAndClose); + expectThrows(IllegalArgumentException.class, "acks cannot be null", openAndClose); config.put("acks", "1"); + expectThrows(IllegalArgumentException.class, "topic cannot be null", openAndClose); + config.put("topic", "topic_2"); config.put("batchSize", "-1"); expectThrows(IllegalArgumentException.class, "Invalid Kafka Producer batchSize : -1", openAndClose); config.put("batchSize", "16384"); diff --git a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java index 612cf0bc6d2b1..6b4719709a178 100644 --- a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java +++ b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java @@ -18,16 +18,25 @@ */ package org.apache.pulsar.io.kafka.source; - import com.google.common.collect.ImmutableMap; +import java.time.Duration; import java.util.Collections; +import java.util.Arrays; +import java.lang.reflect.Field; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; +import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.io.core.SourceContext; import org.apache.pulsar.io.kafka.KafkaAbstractSource; import org.apache.pulsar.io.kafka.KafkaSourceConfig; +import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.Test; @@ -43,6 +52,7 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; import static org.testng.Assert.expectThrows; import static org.testng.Assert.fail; @@ -52,18 +62,17 @@ public class KafkaAbstractSourceTest { private static class DummySource extends KafkaAbstractSource { @Override - public KafkaRecord buildRecord(ConsumerRecord consumerRecord) { - KafkaRecord record = new KafkaRecord(consumerRecord, + public KafkaRecord buildRecord(ConsumerRecord consumerRecord) { + return new KafkaRecord<>(consumerRecord, new String((byte[]) consumerRecord.value(), StandardCharsets.UTF_8), Schema.STRING, Collections.emptyMap()); - return record; } } @Test public void testInvalidConfigWillThrownException() throws Exception { - KafkaAbstractSource source = new DummySource(); + KafkaAbstractSource source = new DummySource(); SourceContext ctx = mock(SourceContext.class); Map config = new HashMap<>(); Assert.ThrowingRunnable openAndClose = ()->{ @@ -106,19 +115,39 @@ public void testInvalidConfigWillThrownException() throws Exception { public void loadConsumerConfigPropertiesFromMapTest() throws Exception { Map config = new HashMap<>(); config.put("consumerConfigProperties", ""); - KafkaSourceConfig kafkaSourceConfig = KafkaSourceConfig.load(config); + config.put("bootstrapServers", "localhost:8080"); + config.put("groupId", "test-group"); + config.put("topic", "test-topic"); + SourceContext sourceContext = Mockito.mock(SourceContext.class); + KafkaSourceConfig kafkaSourceConfig = KafkaSourceConfig.load(config, sourceContext); assertNotNull(kafkaSourceConfig); assertNull(kafkaSourceConfig.getConsumerConfigProperties()); config.put("consumerConfigProperties", null); - kafkaSourceConfig = KafkaSourceConfig.load(config); + kafkaSourceConfig = KafkaSourceConfig.load(config, sourceContext); assertNull(kafkaSourceConfig.getConsumerConfigProperties()); config.put("consumerConfigProperties", ImmutableMap.of("foo", "bar")); - kafkaSourceConfig = KafkaSourceConfig.load(config); + kafkaSourceConfig = KafkaSourceConfig.load(config, sourceContext); assertEquals(kafkaSourceConfig.getConsumerConfigProperties(), ImmutableMap.of("foo", "bar")); } + @Test + public void loadSensitiveFieldsFromSecretTest() throws Exception { + Map config = new HashMap<>(); + config.put("consumerConfigProperties", ""); + config.put("bootstrapServers", "localhost:8080"); + config.put("groupId", "test-group"); + config.put("topic", "test-topic"); + SourceContext sourceContext = Mockito.mock(SourceContext.class); + Mockito.when(sourceContext.getSecret("sslTruststorePassword")) + .thenReturn("xxxx"); + KafkaSourceConfig kafkaSourceConfig = KafkaSourceConfig.load(config, sourceContext); + assertNotNull(kafkaSourceConfig); + assertNull(kafkaSourceConfig.getConsumerConfigProperties()); + assertEquals("xxxx", kafkaSourceConfig.getSslTruststorePassword()); + } + @Test public final void loadFromYamlFileTest() throws IOException { File yamlFile = getFile("kafkaSourceConfig.yaml"); @@ -153,6 +182,131 @@ public final void loadFromSaslYamlFileTest() throws IOException { assertEquals(config.getSslTruststorePassword(), "cert_pwd"); } + @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "Subscribe exception") + public final void throwExceptionBySubscribe() throws Exception { + KafkaAbstractSource source = new DummySource(); + + KafkaSourceConfig kafkaSourceConfig = new KafkaSourceConfig(); + kafkaSourceConfig.setTopic("test-topic"); + Field kafkaSourceConfigField = KafkaAbstractSource.class.getDeclaredField("kafkaSourceConfig"); + kafkaSourceConfigField.setAccessible(true); + kafkaSourceConfigField.set(source, kafkaSourceConfig); + + Consumer consumer = mock(Consumer.class); + Mockito.doThrow(new RuntimeException("Subscribe exception")).when(consumer) + .subscribe(Mockito.anyCollection()); + + Field consumerField = KafkaAbstractSource.class.getDeclaredField("consumer"); + consumerField.setAccessible(true); + consumerField.set(source, consumer); + // will throw RuntimeException. + source.start(); + } + + @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "Pool exception") + public final void throwExceptionByPoll() throws Exception { + KafkaAbstractSource source = new DummySource(); + + KafkaSourceConfig kafkaSourceConfig = new KafkaSourceConfig(); + kafkaSourceConfig.setTopic("test-topic"); + Field kafkaSourceConfigField = KafkaAbstractSource.class.getDeclaredField("kafkaSourceConfig"); + kafkaSourceConfigField.setAccessible(true); + kafkaSourceConfigField.set(source, kafkaSourceConfig); + + Consumer consumer = mock(Consumer.class); + Mockito.doThrow(new RuntimeException("Pool exception")).when(consumer) + .poll(Mockito.any(Duration.class)); + + Field consumerField = KafkaAbstractSource.class.getDeclaredField("consumer"); + consumerField.setAccessible(true); + consumerField.set(source, consumer); + source.start(); + // will throw RuntimeException. + source.read(); + } + + @Test + public final void throwExceptionBySendFail() throws Exception { + KafkaAbstractSource source = new DummySource(); + + KafkaSourceConfig kafkaSourceConfig = new KafkaSourceConfig(); + kafkaSourceConfig.setTopic("test-topic"); + kafkaSourceConfig.setAutoCommitEnabled(false); + Field kafkaSourceConfigField = KafkaAbstractSource.class.getDeclaredField("kafkaSourceConfig"); + kafkaSourceConfigField.setAccessible(true); + kafkaSourceConfigField.set(source, kafkaSourceConfig); + + Field defaultMaxPollIntervalMsField = KafkaAbstractSource.class.getDeclaredField("maxPollIntervalMs"); + defaultMaxPollIntervalMsField.setAccessible(true); + defaultMaxPollIntervalMsField.set(source, 300000); + + Consumer consumer = mock(Consumer.class); + ConsumerRecord consumerRecord = new ConsumerRecord<>("topic", 0, 0, + "t-key", "t-value".getBytes(StandardCharsets.UTF_8)); + ConsumerRecords consumerRecords = new ConsumerRecords<>(Collections.singletonMap( + new TopicPartition("topic", 0), + Arrays.asList(consumerRecord))); + Mockito.doReturn(consumerRecords).when(consumer).poll(Mockito.any(Duration.class)); + + Field consumerField = KafkaAbstractSource.class.getDeclaredField("consumer"); + consumerField.setAccessible(true); + consumerField.set(source, consumer); + source.start(); + + // Mock send message fail + Record record = source.read(); + record.fail(); + + // read again will throw RuntimeException. + try { + source.read(); + fail("Should throw exception"); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof RuntimeException); + assertTrue(e.getCause().getMessage().contains("Failed to process record with kafka topic")); + } + } + + @Test + public final void throwExceptionBySendTimeOut() throws Exception { + KafkaAbstractSource source = new DummySource(); + + KafkaSourceConfig kafkaSourceConfig = new KafkaSourceConfig(); + kafkaSourceConfig.setTopic("test-topic"); + kafkaSourceConfig.setAutoCommitEnabled(false); + Field kafkaSourceConfigField = KafkaAbstractSource.class.getDeclaredField("kafkaSourceConfig"); + kafkaSourceConfigField.setAccessible(true); + kafkaSourceConfigField.set(source, kafkaSourceConfig); + + Field defaultMaxPollIntervalMsField = KafkaAbstractSource.class.getDeclaredField("maxPollIntervalMs"); + defaultMaxPollIntervalMsField.setAccessible(true); + defaultMaxPollIntervalMsField.set(source, 1); + + Consumer consumer = mock(Consumer.class); + ConsumerRecord consumerRecord = new ConsumerRecord<>("topic", 0, 0, + "t-key", "t-value".getBytes(StandardCharsets.UTF_8)); + ConsumerRecords consumerRecords = new ConsumerRecords<>(Collections.singletonMap( + new TopicPartition("topic", 0), + Arrays.asList(consumerRecord))); + Mockito.doReturn(consumerRecords).when(consumer).poll(Mockito.any(Duration.class)); + + Field consumerField = KafkaAbstractSource.class.getDeclaredField("consumer"); + consumerField.setAccessible(true); + consumerField.set(source, consumer); + source.start(); + + // Mock send message fail, just read do noting. + source.read(); + + // read again will throw TimeOutException. + try { + source.read(); + fail("Should throw exception"); + } catch (Exception e) { + assertTrue(e instanceof TimeoutException); + } + } + private File getFile(String name) { ClassLoader classLoader = getClass().getClassLoader(); return new File(classLoader.getResource(name).getFile()); diff --git a/pulsar-io/kinesis/pom.xml b/pulsar-io/kinesis/pom.xml index 06bb479a5472e..9a84c088067a8 100644 --- a/pulsar-io/kinesis/pom.xml +++ b/pulsar-io/kinesis/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-kinesis @@ -32,7 +32,7 @@ 2.2.8 - 0.14.0 + 0.14.13 0.13.0 1.9.0 2.3.0 diff --git a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/AwsDefaultProviderChainPlugin.java b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/AwsDefaultProviderChainPlugin.java deleted file mode 100644 index 75952a71a29a0..0000000000000 --- a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/AwsDefaultProviderChainPlugin.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.io.kinesis; - -/** - * This is a stub class for backwards compatibility. In new code and configurations, please use the plugins - * from org.apache.pulsar.io.aws - * - * @see org.apache.pulsar.io.aws.AwsDefaultProviderChainPlugin - */ -@Deprecated -public class AwsDefaultProviderChainPlugin extends org.apache.pulsar.io.aws.AwsDefaultProviderChainPlugin - implements AwsCredentialProviderPlugin { -} diff --git a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/BaseKinesisConfig.java b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/BaseKinesisConfig.java index c9c951ae2b70e..7bd95b0d6e3ab 100644 --- a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/BaseKinesisConfig.java +++ b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/BaseKinesisConfig.java @@ -35,6 +35,13 @@ public abstract class BaseKinesisConfig implements Serializable { ) private String awsEndpoint = ""; + @FieldDoc( + required = false, + defaultValue = "", + help = "Cloudwatch end-point url. It can be found at " + + "https://docs.aws.amazon.com/general/latest/gr/rande.html" + ) + private String cloudwatchEndpoint = ""; @FieldDoc( required = false, diff --git a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java index fb8eedff82f0a..d8e4e4bab85e5 100644 --- a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java +++ b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.io.kinesis; -import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.util.concurrent.Futures.addCallback; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static org.apache.commons.lang3.StringUtils.isNotBlank; @@ -49,7 +48,6 @@ import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.io.aws.AbstractAwsConnector; import org.apache.pulsar.io.aws.AwsCredentialProviderPlugin; -import org.apache.pulsar.io.common.IOConfigUtils; import org.apache.pulsar.io.core.Sink; import org.apache.pulsar.io.core.SinkContext; import org.apache.pulsar.io.core.annotations.Connector; @@ -155,17 +153,16 @@ public void close() { @Override public void open(Map config, SinkContext sinkContext) { scheduledExecutor = Executors.newSingleThreadScheduledExecutor(); - kinesisSinkConfig = IOConfigUtils.loadWithSecrets(config, KinesisSinkConfig.class, sinkContext); + kinesisSinkConfig = KinesisSinkConfig.load(config, sinkContext); this.sinkContext = sinkContext; - checkArgument(isNotBlank(kinesisSinkConfig.getAwsKinesisStreamName()), "empty kinesis-stream name"); - checkArgument(isNotBlank(kinesisSinkConfig.getAwsEndpoint()) - || isNotBlank(kinesisSinkConfig.getAwsRegion()), - "Either the aws-end-point or aws-region must be set"); - checkArgument(isNotBlank(kinesisSinkConfig.getAwsCredentialPluginParam()), "empty aws-credential param"); - KinesisProducerConfiguration kinesisConfig = new KinesisProducerConfiguration(); - kinesisConfig.setKinesisEndpoint(kinesisSinkConfig.getAwsEndpoint()); + if (isNotBlank(kinesisSinkConfig.getAwsEndpoint())) { + kinesisConfig.setKinesisEndpoint(kinesisSinkConfig.getAwsEndpoint()); + } + if (isNotBlank(kinesisSinkConfig.getCloudwatchEndpoint())) { + kinesisConfig.setCloudwatchEndpoint(kinesisSinkConfig.getCloudwatchEndpoint()); + } if (kinesisSinkConfig.getAwsEndpointPort() != null) { kinesisConfig.setKinesisPort(kinesisSinkConfig.getAwsEndpointPort()); } diff --git a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSinkConfig.java b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSinkConfig.java index c5b26a26d0cf2..f81fd32134be2 100644 --- a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSinkConfig.java +++ b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSinkConfig.java @@ -18,13 +18,14 @@ */ package org.apache.pulsar.io.kinesis; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import java.io.File; -import java.io.IOException; +import static com.google.common.base.Preconditions.checkArgument; +import static org.apache.commons.lang3.StringUtils.isNotBlank; import java.io.Serializable; +import java.util.Map; import lombok.Data; import lombok.EqualsAndHashCode; +import org.apache.pulsar.io.common.IOConfigUtils; +import org.apache.pulsar.io.core.SinkContext; import org.apache.pulsar.io.core.annotations.FieldDoc; @Data @@ -103,9 +104,12 @@ public class KinesisSinkConfig extends BaseKinesisConfig implements Serializable help = "The maximum delay(in milliseconds) between retries.") private long retryMaxDelayInMillis = 60000; - public static KinesisSinkConfig load(String yamlFile) throws IOException { - ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); - return mapper.readValue(new File(yamlFile), KinesisSinkConfig.class); + public static KinesisSinkConfig load(Map config, SinkContext sinkContext) { + KinesisSinkConfig kinesisSinkConfig = IOConfigUtils.loadWithSecrets(config, KinesisSinkConfig.class, sinkContext); + checkArgument(isNotBlank(kinesisSinkConfig.getAwsRegion()) + || (isNotBlank(kinesisSinkConfig.getAwsEndpoint()) && isNotBlank(kinesisSinkConfig.getCloudwatchEndpoint())), + "Either \"awsRegion\" must be set OR all of [\"awsEndpoint\", \"cloudwatchEndpoint\"] must be set."); + return kinesisSinkConfig; } public enum MessageFormat { diff --git a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSource.java b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSource.java index 2412244e1b5dc..279368db2a028 100644 --- a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSource.java +++ b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSource.java @@ -18,8 +18,6 @@ */ package org.apache.pulsar.io.kinesis; -import static com.google.common.base.Preconditions.checkArgument; -import static org.apache.commons.lang3.StringUtils.isNotBlank; import java.net.InetAddress; import java.util.Map; import java.util.UUID; @@ -27,14 +25,12 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.io.aws.AbstractAwsConnector; import org.apache.pulsar.io.aws.AwsCredentialProviderPlugin; -import org.apache.pulsar.io.common.IOConfigUtils; import org.apache.pulsar.io.core.Source; import org.apache.pulsar.io.core.SourceContext; import org.apache.pulsar.io.core.annotations.Connector; import org.apache.pulsar.io.core.annotations.IOType; import software.amazon.awssdk.services.kinesis.KinesisAsyncClient; import software.amazon.kinesis.common.ConfigsBuilder; -import software.amazon.kinesis.common.InitialPositionInStream; import software.amazon.kinesis.coordinator.Scheduler; import software.amazon.kinesis.processor.ShardRecordProcessorFactory; import software.amazon.kinesis.retrieval.RetrievalConfig; @@ -68,18 +64,7 @@ public void close() throws Exception { @Override public void open(Map config, SourceContext sourceContext) throws Exception { - this.kinesisSourceConfig = IOConfigUtils.loadWithSecrets(config, KinesisSourceConfig.class, sourceContext); - - checkArgument(isNotBlank(kinesisSourceConfig.getAwsKinesisStreamName()), "empty kinesis-stream name"); - checkArgument(isNotBlank(kinesisSourceConfig.getAwsEndpoint()) - || isNotBlank(kinesisSourceConfig.getAwsRegion()), - "Either the aws-end-point or aws-region must be set"); - checkArgument(isNotBlank(kinesisSourceConfig.getAwsCredentialPluginParam()), "empty aws-credential param"); - - if (kinesisSourceConfig.getInitialPositionInStream() == InitialPositionInStream.AT_TIMESTAMP) { - checkArgument((kinesisSourceConfig.getStartAtTime() != null), "Timestamp must be specified"); - } - + this.kinesisSourceConfig = KinesisSourceConfig.load(config, sourceContext); queue = new LinkedBlockingQueue<>(kinesisSourceConfig.getReceiveQueueSize()); workerId = InetAddress.getLocalHost().getCanonicalHostName() + ":" + UUID.randomUUID(); diff --git a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSourceConfig.java b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSourceConfig.java index f0bf7cfc9781d..0dd9bfce9e0c2 100644 --- a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSourceConfig.java +++ b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSourceConfig.java @@ -18,16 +18,17 @@ */ package org.apache.pulsar.io.kinesis; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import java.io.File; -import java.io.IOException; +import static com.google.common.base.Preconditions.checkArgument; +import static org.apache.commons.lang3.StringUtils.isNotBlank; import java.io.Serializable; import java.net.URI; import java.util.Date; +import java.util.Map; import lombok.Data; import lombok.EqualsAndHashCode; import org.apache.pulsar.io.aws.AwsCredentialProviderPlugin; +import org.apache.pulsar.io.common.IOConfigUtils; +import org.apache.pulsar.io.core.SourceContext; import org.apache.pulsar.io.core.annotations.FieldDoc; import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient; import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClientBuilder; @@ -76,7 +77,7 @@ public class KinesisSourceConfig extends BaseKinesisConfig implements Serializab @FieldDoc( required = false, - defaultValue = "Apache Pulsar IO Connector", + defaultValue = "pulsar-kinesis", help = "Name of the Amazon Kinesis application. By default the application name is included " + "in the user agent string used to make AWS requests. This can assist with troubleshooting " + "(e.g. distinguish requests made by separate connectors instances)." @@ -122,13 +123,6 @@ public class KinesisSourceConfig extends BaseKinesisConfig implements Serializab ) private String dynamoEndpoint = ""; - @FieldDoc( - required = false, - defaultValue = "", - help = "Cloudwatch end-point url. It can be found at https://docs.aws.amazon.com/general/latest/gr/rande.html" - ) - private String cloudwatchEndpoint = ""; - @FieldDoc( required = false, defaultValue = "true", @@ -136,10 +130,20 @@ public class KinesisSourceConfig extends BaseKinesisConfig implements Serializab ) private boolean useEnhancedFanOut = true; - - public static KinesisSourceConfig load(String yamlFile) throws IOException { - ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); - return mapper.readValue(new File(yamlFile), KinesisSourceConfig.class); + public static KinesisSourceConfig load(Map config, SourceContext sourceContext) { + KinesisSourceConfig kinesisSourceConfig = IOConfigUtils.loadWithSecrets(config, + KinesisSourceConfig.class, sourceContext); + boolean isNotBlankEndpoint = isNotBlank(kinesisSourceConfig.getAwsEndpoint()) + && isNotBlank(kinesisSourceConfig.getCloudwatchEndpoint()) + && isNotBlank(kinesisSourceConfig.getDynamoEndpoint()); + checkArgument(isNotBlank(kinesisSourceConfig.getAwsRegion()) || isNotBlankEndpoint, + "Either \"awsRegion\" must be set OR all of " + + "[ \"awsEndpoint\", \"cloudwatchEndpoint\", and \"dynamoEndpoint\" ] must be set."); + if (kinesisSourceConfig.getInitialPositionInStream() == InitialPositionInStream.AT_TIMESTAMP) { + checkArgument((kinesisSourceConfig.getStartAtTime() != null), + "When initialPositionInStream is AT_TIMESTAMP, startAtTime must be specified"); + } + return kinesisSourceConfig; } public KinesisAsyncClient buildKinesisAsyncClient(AwsCredentialProviderPlugin credPlugin) { diff --git a/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSinkConfigTests.java b/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSinkConfigTests.java index 6f76d9e69a211..a5051927ace54 100644 --- a/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSinkConfigTests.java +++ b/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSinkConfigTests.java @@ -21,34 +21,16 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; -import java.io.File; import java.io.IOException; import java.util.HashMap; import java.util.Map; -import org.apache.pulsar.io.common.IOConfigUtils; import org.apache.pulsar.io.core.SinkContext; -import org.apache.pulsar.io.kinesis.KinesisSinkConfig.MessageFormat; import org.mockito.Mockito; import org.testng.annotations.Test; public class KinesisSinkConfigTests { - @Test - public final void loadFromYamlFileTest() throws IOException { - File yamlFile = getFile("sinkConfig.yaml"); - KinesisSinkConfig config = KinesisSinkConfig.load(yamlFile.getAbsolutePath()); - - assertNotNull(config); - assertEquals(config.getAwsEndpoint(), "https://some.endpoint.aws"); - assertEquals(config.getAwsRegion(), "us-east-1"); - assertEquals(config.getAwsKinesisStreamName(), "my-stream"); - assertEquals(config.getAwsCredentialPluginParam(), - "{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}"); - assertEquals(config.getMessageFormat(), MessageFormat.ONLY_RAW_PAYLOAD); - assertEquals(true, config.isRetainOrdering()); - } - @Test public final void loadFromMapTest() throws IOException { Map map = new HashMap (); @@ -58,7 +40,7 @@ public final void loadFromMapTest() throws IOException { map.put("awsCredentialPluginParam", "{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}"); SinkContext sinkContext = Mockito.mock(SinkContext.class); - KinesisSinkConfig config = IOConfigUtils.loadWithSecrets(map, KinesisSinkConfig.class, sinkContext); + KinesisSinkConfig config = KinesisSinkConfig.load(map, sinkContext); assertNotNull(config); assertEquals(config.getAwsEndpoint(), "https://some.endpoint.aws"); @@ -78,7 +60,7 @@ public final void loadFromMapCredentialFromSecretTest() throws IOException { SinkContext sinkContext = Mockito.mock(SinkContext.class); Mockito.when(sinkContext.getSecret("awsCredentialPluginParam")) .thenReturn("{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}"); - KinesisSinkConfig config = IOConfigUtils.loadWithSecrets(map, KinesisSinkConfig.class, sinkContext); + KinesisSinkConfig config = KinesisSinkConfig.load(map, sinkContext); assertNotNull(config); assertEquals(config.getAwsEndpoint(), "https://some.endpoint.aws"); @@ -88,8 +70,13 @@ public final void loadFromMapCredentialFromSecretTest() throws IOException { "{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}"); } - private File getFile(String name) { - ClassLoader classLoader = getClass().getClassLoader(); - return new File(classLoader.getResource(name).getFile()); + @Test(expectedExceptions = IllegalArgumentException.class) + public final void missCloudWatchEndpointTest() { + Map map = new HashMap (); + map.put("awsEndpoint", "https://some.endpoint.aws"); + map.put("awsKinesisStreamName", "my-stream"); + map.put("awsCredentialPluginParam", "{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}"); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + KinesisSinkConfig.load(map, sinkContext); } } diff --git a/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSourceConfigTests.java b/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSourceConfigTests.java index f6b0666d34ba3..4ba3593b1d9b8 100644 --- a/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSourceConfigTests.java +++ b/pulsar-io/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisSourceConfigTests.java @@ -21,7 +21,6 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; -import java.io.File; import java.io.IOException; import java.time.ZoneOffset; import java.time.ZonedDateTime; @@ -30,7 +29,6 @@ import java.util.HashMap; import java.util.Map; -import org.apache.pulsar.io.common.IOConfigUtils; import org.apache.pulsar.io.core.SourceContext; import org.mockito.Mockito; import org.testng.annotations.Test; @@ -54,30 +52,6 @@ public class KinesisSourceConfigTests { DAY = then.getTime(); } - @Test - public final void loadFromYamlFileTest() throws IOException { - File yamlFile = getFile("sourceConfig.yaml"); - KinesisSourceConfig config = KinesisSourceConfig.load(yamlFile.getAbsolutePath()); - assertNotNull(config); - assertEquals(config.getAwsEndpoint(), "https://some.endpoint.aws"); - assertEquals(config.getAwsRegion(), "us-east-1"); - assertEquals(config.getAwsKinesisStreamName(), "my-stream"); - assertEquals(config.getAwsCredentialPluginParam(), - "{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}"); - assertEquals(config.getApplicationName(), "My test application"); - assertEquals(config.getCheckpointInterval(), 30000); - assertEquals(config.getBackoffTime(), 4000); - assertEquals(config.getNumRetries(), 3); - assertEquals(config.getReceiveQueueSize(), 2000); - assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.TRIM_HORIZON); - - Calendar cal = Calendar.getInstance(); - cal.setTime(config.getStartAtTime()); - ZonedDateTime actual = ZonedDateTime.ofInstant(cal.toInstant(), ZoneOffset.UTC); - ZonedDateTime expected = ZonedDateTime.ofInstant(DAY.toInstant(), ZoneOffset.UTC); - assertEquals(actual, expected); - } - @Test public final void loadFromMapTest() throws IOException { Map map = new HashMap (); @@ -89,12 +63,11 @@ public final void loadFromMapTest() throws IOException { map.put("backoffTime", "4000"); map.put("numRetries", "3"); map.put("receiveQueueSize", 2000); - map.put("applicationName", "My test application"); map.put("initialPositionInStream", InitialPositionInStream.TRIM_HORIZON); map.put("startAtTime", DAY); SourceContext sourceContext = Mockito.mock(SourceContext.class); - KinesisSourceConfig config = IOConfigUtils.loadWithSecrets(map, KinesisSourceConfig.class, sourceContext); + KinesisSourceConfig config = KinesisSourceConfig.load(map, sourceContext); assertNotNull(config); assertEquals(config.getAwsEndpoint(), "https://some.endpoint.aws"); @@ -102,7 +75,7 @@ public final void loadFromMapTest() throws IOException { assertEquals(config.getAwsKinesisStreamName(), "my-stream"); assertEquals(config.getAwsCredentialPluginParam(), "{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}"); - assertEquals(config.getApplicationName(), "My test application"); + assertEquals(config.getApplicationName(), "pulsar-kinesis"); assertEquals(config.getCheckpointInterval(), 30000); assertEquals(config.getBackoffTime(), 4000); assertEquals(config.getNumRetries(), 3); @@ -133,7 +106,7 @@ public final void loadFromMapCredentialFromSecretTest() throws IOException { SourceContext sourceContext = Mockito.mock(SourceContext.class); Mockito.when(sourceContext.getSecret("awsCredentialPluginParam")) .thenReturn("{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}"); - KinesisSourceConfig config = IOConfigUtils.loadWithSecrets(map, KinesisSourceConfig.class, sourceContext); + KinesisSourceConfig config = KinesisSourceConfig.load(map, sourceContext); assertNotNull(config); assertEquals(config.getAwsEndpoint(), "https://some.endpoint.aws"); @@ -156,19 +129,17 @@ public final void loadFromMapCredentialFromSecretTest() throws IOException { } @Test(expectedExceptions = IllegalArgumentException.class, - expectedExceptionsMessageRegExp = "empty aws-credential param") + expectedExceptionsMessageRegExp = "awsCredentialPluginParam cannot be null") public final void missingCredentialsTest() throws Exception { Map map = new HashMap (); map.put("awsEndpoint", "https://some.endpoint.aws"); map.put("awsRegion", "us-east-1"); map.put("awsKinesisStreamName", "my-stream"); - - KinesisSource source = new KinesisSource(); - source.open(map, null); + KinesisSourceConfig.load(map, Mockito.mock(SourceContext.class)); } @Test(expectedExceptions = IllegalArgumentException.class, - expectedExceptionsMessageRegExp = "Timestamp must be specified") + expectedExceptionsMessageRegExp = "When initialPositionInStream is AT_TIMESTAMP, startAtTime must be specified") public final void missingStartTimeTest() throws Exception { Map map = new HashMap (); map.put("awsEndpoint", "https://some.endpoint.aws"); @@ -177,13 +148,16 @@ public final void missingStartTimeTest() throws Exception { map.put("awsCredentialPluginParam", "{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}"); map.put("initialPositionInStream", InitialPositionInStream.AT_TIMESTAMP); - - KinesisSource source = new KinesisSource(); - source.open(map, null); + KinesisSourceConfig.load(map, Mockito.mock(SourceContext.class)); } - private File getFile(String name) { - ClassLoader classLoader = getClass().getClassLoader(); - return new File(classLoader.getResource(name).getFile()); + @Test(expectedExceptions = IllegalArgumentException.class) + public final void missCloudWatchEndpointTest() { + Map map = new HashMap (); + map.put("awsEndpoint", "https://some.endpoint.aws"); + map.put("awsKinesisStreamName", "my-stream"); + map.put("awsCredentialPluginParam", + "{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}"); + KinesisSourceConfig.load(map, Mockito.mock(SourceContext.class)); } } diff --git a/pulsar-io/kinesis/src/test/resources/sourceConfig.yaml b/pulsar-io/kinesis/src/test/resources/sourceConfig.yaml deleted file mode 100644 index 64b564486c18c..0000000000000 --- a/pulsar-io/kinesis/src/test/resources/sourceConfig.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -{ - "awsEndpoint" : "https://some.endpoint.aws", - "awsRegion": "us-east-1", - "awsKinesisStreamName": "my-stream", - "awsCredentialPluginParam": "{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}", - "applicationName": "My test application", - "checkpointInterval": "30000", - "backoffTime":"4000", - "numRetries":"3", - "receiveQueueSize": 2000, - "initialPositionInStream": "TRIM_HORIZON", - "startAtTime": "2019-03-05T19:28:58.000Z" -} \ No newline at end of file diff --git a/pulsar-io/mongo/pom.xml b/pulsar-io/mongo/pom.xml index 214e76bad7927..80d39cacab14a 100644 --- a/pulsar-io/mongo/pom.xml +++ b/pulsar-io/mongo/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-mongo @@ -37,6 +37,11 @@ + + ${project.groupId} + pulsar-io-common + ${project.version} + ${project.parent.groupId} pulsar-io-core diff --git a/pulsar-io/mongo/src/main/java/org/apache/pulsar/io/mongodb/MongoAbstractConfig.java b/pulsar-io/mongo/src/main/java/org/apache/pulsar/io/mongodb/MongoAbstractConfig.java index 35c327ed82b99..74f077da62036 100644 --- a/pulsar-io/mongo/src/main/java/org/apache/pulsar/io/mongodb/MongoAbstractConfig.java +++ b/pulsar-io/mongo/src/main/java/org/apache/pulsar/io/mongodb/MongoAbstractConfig.java @@ -24,7 +24,6 @@ import java.io.Serializable; import lombok.Data; import lombok.experimental.Accessors; -import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.io.core.annotations.FieldDoc; /** @@ -42,6 +41,7 @@ public abstract class MongoAbstractConfig implements Serializable { @FieldDoc( required = true, + sensitive = true, // it may contain password defaultValue = "", help = "The URI of MongoDB that the connector connects to " + "(see: https://docs.mongodb.com/manual/reference/connection-string/)" @@ -95,7 +95,6 @@ public MongoAbstractConfig( } public void validate() { - checkArgument(!StringUtils.isEmpty(getMongoUri()), "Required MongoDB URI is not set."); checkArgument(getBatchSize() > 0, "batchSize must be a positive integer."); checkArgument(getBatchTimeMs() > 0, "batchTimeMs must be a positive long."); } diff --git a/pulsar-io/mongo/src/main/java/org/apache/pulsar/io/mongodb/MongoSink.java b/pulsar-io/mongo/src/main/java/org/apache/pulsar/io/mongodb/MongoSink.java index 2206d232eaf97..61d5aeb697e01 100644 --- a/pulsar-io/mongo/src/main/java/org/apache/pulsar/io/mongodb/MongoSink.java +++ b/pulsar-io/mongo/src/main/java/org/apache/pulsar/io/mongodb/MongoSink.java @@ -86,7 +86,7 @@ public MongoSink(Supplier clientProvider) { public void open(Map config, SinkContext sinkContext) throws Exception { log.info("Open MongoDB Sink"); - mongoSinkConfig = MongoSinkConfig.load(config); + mongoSinkConfig = MongoSinkConfig.load(config, sinkContext); mongoSinkConfig.validate(); if (clientProvider != null) { diff --git a/pulsar-io/mongo/src/main/java/org/apache/pulsar/io/mongodb/MongoSinkConfig.java b/pulsar-io/mongo/src/main/java/org/apache/pulsar/io/mongodb/MongoSinkConfig.java index 285f3c97bef1a..9431fe4910800 100644 --- a/pulsar-io/mongo/src/main/java/org/apache/pulsar/io/mongodb/MongoSinkConfig.java +++ b/pulsar-io/mongo/src/main/java/org/apache/pulsar/io/mongodb/MongoSinkConfig.java @@ -30,6 +30,8 @@ import lombok.EqualsAndHashCode; import lombok.experimental.Accessors; import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.io.common.IOConfigUtils; +import org.apache.pulsar.io.core.SinkContext; /** * Configuration class for the MongoDB Sink Connectors. @@ -59,11 +61,8 @@ public static MongoSinkConfig load(String yamlFile) throws IOException { return cfg; } - public static MongoSinkConfig load(Map map) throws IOException { - final ObjectMapper mapper = new ObjectMapper(); - final MongoSinkConfig cfg = mapper.readValue(mapper.writeValueAsString(map), MongoSinkConfig.class); - - return cfg; + public static MongoSinkConfig load(Map map, SinkContext sinkContext) throws IOException { + return IOConfigUtils.loadWithSecrets(map, MongoSinkConfig.class, sinkContext); } @Override diff --git a/pulsar-io/mongo/src/main/java/org/apache/pulsar/io/mongodb/MongoSource.java b/pulsar-io/mongo/src/main/java/org/apache/pulsar/io/mongodb/MongoSource.java index 6ee95fc4cd4b5..68a31b461a51c 100644 --- a/pulsar-io/mongo/src/main/java/org/apache/pulsar/io/mongodb/MongoSource.java +++ b/pulsar-io/mongo/src/main/java/org/apache/pulsar/io/mongodb/MongoSource.java @@ -79,7 +79,7 @@ public MongoSource(Supplier clientProvider) { public void open(Map config, SourceContext sourceContext) throws Exception { log.info("Open MongoDB Source"); - mongoSourceConfig = MongoSourceConfig.load(config); + mongoSourceConfig = MongoSourceConfig.load(config, sourceContext); mongoSourceConfig.validate(); if (clientProvider != null) { diff --git a/pulsar-io/mongo/src/main/java/org/apache/pulsar/io/mongodb/MongoSourceConfig.java b/pulsar-io/mongo/src/main/java/org/apache/pulsar/io/mongodb/MongoSourceConfig.java index cf887a93bf3c3..1c0c7f4b3657a 100644 --- a/pulsar-io/mongo/src/main/java/org/apache/pulsar/io/mongodb/MongoSourceConfig.java +++ b/pulsar-io/mongo/src/main/java/org/apache/pulsar/io/mongodb/MongoSourceConfig.java @@ -29,6 +29,8 @@ import lombok.EqualsAndHashCode; import lombok.experimental.Accessors; import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.io.common.IOConfigUtils; +import org.apache.pulsar.io.core.SourceContext; import org.apache.pulsar.io.core.annotations.FieldDoc; /** @@ -75,12 +77,8 @@ public static MongoSourceConfig load(String yamlFile) throws IOException { return cfg; } - public static MongoSourceConfig load(Map map) throws IOException { - final ObjectMapper mapper = new ObjectMapper(); - final MongoSourceConfig cfg = - mapper.readValue(mapper.writeValueAsString(map), MongoSourceConfig.class); - - return cfg; + public static MongoSourceConfig load(Map map, SourceContext sourceContext) throws IOException { + return IOConfigUtils.loadWithSecrets(map, MongoSourceConfig.class, sourceContext); } /** diff --git a/pulsar-io/mongo/src/test/java/org/apache/pulsar/io/mongodb/MongoSinkConfigTest.java b/pulsar-io/mongo/src/test/java/org/apache/pulsar/io/mongodb/MongoSinkConfigTest.java index b1166eac5722a..c86e45feb2348 100644 --- a/pulsar-io/mongo/src/test/java/org/apache/pulsar/io/mongodb/MongoSinkConfigTest.java +++ b/pulsar-io/mongo/src/test/java/org/apache/pulsar/io/mongodb/MongoSinkConfigTest.java @@ -19,6 +19,8 @@ package org.apache.pulsar.io.mongodb; import java.util.Map; +import org.apache.pulsar.io.core.SinkContext; +import org.mockito.Mockito; import org.testng.annotations.Test; import java.io.File; @@ -34,7 +36,27 @@ public void testLoadMapConfig() throws IOException { commonConfigMap.put("batchSize", TestHelper.BATCH_SIZE); commonConfigMap.put("batchTimeMs", TestHelper.BATCH_TIME); - final MongoSinkConfig cfg = MongoSinkConfig.load(commonConfigMap); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + final MongoSinkConfig cfg = MongoSinkConfig.load(commonConfigMap, sinkContext); + + assertEquals(cfg.getMongoUri(), TestHelper.URI); + assertEquals(cfg.getDatabase(), TestHelper.DB); + assertEquals(cfg.getCollection(), TestHelper.COLL); + assertEquals(cfg.getBatchSize(), TestHelper.BATCH_SIZE); + assertEquals(cfg.getBatchTimeMs(), TestHelper.BATCH_TIME); + } + + @Test + public void testLoadMapConfigUrlFromSecret() throws IOException { + final Map commonConfigMap = TestHelper.createCommonConfigMap(); + commonConfigMap.put("batchSize", TestHelper.BATCH_SIZE); + commonConfigMap.put("batchTimeMs", TestHelper.BATCH_TIME); + commonConfigMap.remove("mongoUri"); + + SinkContext sinkContext = Mockito.mock(SinkContext.class); + Mockito.when(sinkContext.getSecret("mongoUri")) + .thenReturn(TestHelper.URI); + final MongoSinkConfig cfg = MongoSinkConfig.load(commonConfigMap, sinkContext); assertEquals(cfg.getMongoUri(), TestHelper.URI); assertEquals(cfg.getDatabase(), TestHelper.DB); @@ -44,12 +66,13 @@ public void testLoadMapConfig() throws IOException { } @Test(expectedExceptions = IllegalArgumentException.class, - expectedExceptionsMessageRegExp = "Required MongoDB URI is not set.") + expectedExceptionsMessageRegExp = "mongoUri cannot be null") public void testBadMongoUri() throws IOException { final Map configMap = TestHelper.createCommonConfigMap(); TestHelper.removeMongoUri(configMap); - final MongoSinkConfig cfg = MongoSinkConfig.load(configMap); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + final MongoSinkConfig cfg = MongoSinkConfig.load(configMap, sinkContext); cfg.validate(); } @@ -60,7 +83,8 @@ public void testBadDatabase() throws IOException { final Map configMap = TestHelper.createCommonConfigMap(); TestHelper.removeDatabase(configMap); - final MongoSinkConfig cfg = MongoSinkConfig.load(configMap); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + final MongoSinkConfig cfg = MongoSinkConfig.load(configMap, sinkContext); cfg.validate(); } @@ -71,7 +95,8 @@ public void testBadCollection() throws IOException { final Map configMap = TestHelper.createCommonConfigMap(); TestHelper.removeCollection(configMap); - final MongoSinkConfig cfg = MongoSinkConfig.load(configMap); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + final MongoSinkConfig cfg = MongoSinkConfig.load(configMap, sinkContext); cfg.validate(); } @@ -82,7 +107,8 @@ public void testBadBatchSize() throws IOException { final Map configMap = TestHelper.createCommonConfigMap(); TestHelper.putBatchSize(configMap, 0); - final MongoSinkConfig cfg = MongoSinkConfig.load(configMap); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + final MongoSinkConfig cfg = MongoSinkConfig.load(configMap, sinkContext); cfg.validate(); } @@ -93,7 +119,8 @@ public void testBadBatchTime() throws IOException { final Map configMap = TestHelper.createCommonConfigMap(); TestHelper.putBatchTime(configMap, 0L); - final MongoSinkConfig cfg = MongoSinkConfig.load(configMap); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + final MongoSinkConfig cfg = MongoSinkConfig.load(configMap, sinkContext); cfg.validate(); } diff --git a/pulsar-io/mongo/src/test/java/org/apache/pulsar/io/mongodb/MongoSourceConfigTest.java b/pulsar-io/mongo/src/test/java/org/apache/pulsar/io/mongodb/MongoSourceConfigTest.java index e7fd01549b033..528cd0237ef16 100644 --- a/pulsar-io/mongo/src/test/java/org/apache/pulsar/io/mongodb/MongoSourceConfigTest.java +++ b/pulsar-io/mongo/src/test/java/org/apache/pulsar/io/mongodb/MongoSourceConfigTest.java @@ -23,6 +23,8 @@ import java.io.File; import java.io.IOException; import java.util.Map; +import org.apache.pulsar.io.core.SourceContext; +import org.mockito.Mockito; import org.testng.annotations.Test; public class MongoSourceConfigTest { @@ -32,7 +34,27 @@ public void testLoadMapConfig() throws IOException { final Map configMap = TestHelper.createCommonConfigMap(); TestHelper.putSyncType(configMap, TestHelper.SYNC_TYPE); - final MongoSourceConfig cfg = MongoSourceConfig.load(configMap); + SourceContext sourceContext = Mockito.mock(SourceContext.class); + final MongoSourceConfig cfg = MongoSourceConfig.load(configMap, sourceContext); + + assertEquals(cfg.getMongoUri(), TestHelper.URI); + assertEquals(cfg.getDatabase(), TestHelper.DB); + assertEquals(cfg.getCollection(), TestHelper.COLL); + assertEquals(cfg.getSyncType(), TestHelper.SYNC_TYPE); + assertEquals(cfg.getBatchSize(), TestHelper.BATCH_SIZE); + assertEquals(cfg.getBatchTimeMs(), TestHelper.BATCH_TIME); + } + + @Test + public void testLoadMapConfigUriFromSecret() throws IOException { + final Map configMap = TestHelper.createCommonConfigMap(); + TestHelper.putSyncType(configMap, TestHelper.SYNC_TYPE); + configMap.remove("mongoUri"); + + SourceContext sourceContext = Mockito.mock(SourceContext.class); + Mockito.when(sourceContext.getSecret("mongoUri")) + .thenReturn(TestHelper.URI); + final MongoSourceConfig cfg = MongoSourceConfig.load(configMap, sourceContext); assertEquals(cfg.getMongoUri(), TestHelper.URI); assertEquals(cfg.getDatabase(), TestHelper.DB); @@ -43,12 +65,13 @@ public void testLoadMapConfig() throws IOException { } @Test(expectedExceptions = IllegalArgumentException.class, - expectedExceptionsMessageRegExp = "Required MongoDB URI is not set.") + expectedExceptionsMessageRegExp = "mongoUri cannot be null") public void testBadMongoUri() throws IOException { final Map configMap = TestHelper.createCommonConfigMap(); TestHelper.removeMongoUri(configMap); - final MongoSourceConfig cfg = MongoSourceConfig.load(configMap); + SourceContext sourceContext = Mockito.mock(SourceContext.class); + final MongoSourceConfig cfg = MongoSourceConfig.load(configMap, sourceContext); cfg.validate(); } @@ -61,7 +84,8 @@ public void testBadSyncType() throws IOException { final Map configMap = TestHelper.createCommonConfigMap(); TestHelper.putSyncType(configMap, "wrong_sync_type_str"); - final MongoSourceConfig cfg = MongoSourceConfig.load(configMap); + SourceContext sourceContext = Mockito.mock(SourceContext.class); + final MongoSourceConfig cfg = MongoSourceConfig.load(configMap, sourceContext); cfg.validate(); } @@ -72,7 +96,8 @@ public void testBadBatchSize() throws IOException { final Map configMap = TestHelper.createCommonConfigMap(); TestHelper.putBatchSize(configMap, 0); - final MongoSourceConfig cfg = MongoSourceConfig.load(configMap); + SourceContext sourceContext = Mockito.mock(SourceContext.class); + final MongoSourceConfig cfg = MongoSourceConfig.load(configMap, sourceContext); cfg.validate(); } @@ -83,7 +108,8 @@ public void testBadBatchTime() throws IOException { final Map configMap = TestHelper.createCommonConfigMap(); TestHelper.putBatchTime(configMap, 0L); - final MongoSourceConfig cfg = MongoSourceConfig.load(configMap); + SourceContext sourceContext = Mockito.mock(SourceContext.class); + final MongoSourceConfig cfg = MongoSourceConfig.load(configMap, sourceContext); cfg.validate(); } diff --git a/pulsar-io/netty/pom.xml b/pulsar-io/netty/pom.xml index dcbc025510b67..2748982870de3 100644 --- a/pulsar-io/netty/pom.xml +++ b/pulsar-io/netty/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-netty diff --git a/pulsar-io/nsq/pom.xml b/pulsar-io/nsq/pom.xml index f6bed0c7a74b4..cbacdee18d0c2 100644 --- a/pulsar-io/nsq/pom.xml +++ b/pulsar-io/nsq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-nsq diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml index 53079cdfbc807..46e946f10f804 100644 --- a/pulsar-io/pom.xml +++ b/pulsar-io/pom.xml @@ -22,10 +22,10 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 pom - + org.apache.pulsar pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io @@ -42,6 +42,7 @@ + azure-data-explorer core batch-discovery-triggerers batch-data-generator @@ -81,26 +82,21 @@ pulsar-io-tests + azure-data-explorer core batch-discovery-triggerers batch-data-generator common - docs aws twitter cassandra aerospike http - kafka rabbitmq kinesis hdfs3 jdbc data-generator - elastic-search - kafka-connect-adaptor - kafka-connect-adaptor-nar - debezium hdfs2 canal file @@ -117,6 +113,27 @@ + + pulsar-io-elastic-tests + + core + common + elastic-search + + + + + pulsar-io-kafka-connect-tests + + core + common + kafka + kafka-connect-adaptor + kafka-connect-adaptor-nar + debezium + + + core-modules diff --git a/pulsar-io/rabbitmq/pom.xml b/pulsar-io/rabbitmq/pom.xml index 3cdcb681b3fa8..418842a55290f 100644 --- a/pulsar-io/rabbitmq/pom.xml +++ b/pulsar-io/rabbitmq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-rabbitmq @@ -32,6 +32,11 @@ + + ${project.groupId} + pulsar-io-common + ${project.version} + ${project.groupId} pulsar-io-core @@ -44,7 +49,7 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl @@ -78,8 +83,22 @@ org.apache.qpid qpid-broker - 6.1.6 + 9.2.0 test + + + org.apache.qpid + qpid-bdbstore + + + org.apache.qpid + qpid-broker-plugins-amqp-1-0-protocol-bdb-link-store + + + org.apache.qpid + qpid-broker-plugins-derby-store + + org.awaitility diff --git a/pulsar-io/rabbitmq/src/main/java/org/apache/pulsar/io/rabbitmq/RabbitMQSink.java b/pulsar-io/rabbitmq/src/main/java/org/apache/pulsar/io/rabbitmq/RabbitMQSink.java index f317a35734e69..89192c42346e8 100644 --- a/pulsar-io/rabbitmq/src/main/java/org/apache/pulsar/io/rabbitmq/RabbitMQSink.java +++ b/pulsar-io/rabbitmq/src/main/java/org/apache/pulsar/io/rabbitmq/RabbitMQSink.java @@ -53,7 +53,7 @@ public class RabbitMQSink implements Sink { @Override public void open(Map config, SinkContext sinkContext) throws Exception { - rabbitMQSinkConfig = RabbitMQSinkConfig.load(config); + rabbitMQSinkConfig = RabbitMQSinkConfig.load(config, sinkContext); rabbitMQSinkConfig.validate(); ConnectionFactory connectionFactory = rabbitMQSinkConfig.createConnectionFactory(); diff --git a/pulsar-io/rabbitmq/src/main/java/org/apache/pulsar/io/rabbitmq/RabbitMQSinkConfig.java b/pulsar-io/rabbitmq/src/main/java/org/apache/pulsar/io/rabbitmq/RabbitMQSinkConfig.java index c1f8d6b8ad3d3..39f97e5e460c8 100644 --- a/pulsar-io/rabbitmq/src/main/java/org/apache/pulsar/io/rabbitmq/RabbitMQSinkConfig.java +++ b/pulsar-io/rabbitmq/src/main/java/org/apache/pulsar/io/rabbitmq/RabbitMQSinkConfig.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.google.common.base.Preconditions; import java.io.File; import java.io.IOException; import java.io.Serializable; @@ -28,6 +27,8 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.experimental.Accessors; +import org.apache.pulsar.io.common.IOConfigUtils; +import org.apache.pulsar.io.core.SinkContext; import org.apache.pulsar.io.core.annotations.FieldDoc; @Data @@ -60,14 +61,12 @@ public static RabbitMQSinkConfig load(String yamlFile) throws IOException { return mapper.readValue(new File(yamlFile), RabbitMQSinkConfig.class); } - public static RabbitMQSinkConfig load(Map map) throws IOException { - ObjectMapper mapper = new ObjectMapper(); - return mapper.readValue(mapper.writeValueAsString(map), RabbitMQSinkConfig.class); + public static RabbitMQSinkConfig load(Map map, SinkContext sinkContext) throws IOException { + return IOConfigUtils.loadWithSecrets(map, RabbitMQSinkConfig.class, sinkContext); } @Override public void validate() { super.validate(); - Preconditions.checkNotNull(exchangeName, "exchangeName property not set."); } } diff --git a/pulsar-io/rabbitmq/src/main/java/org/apache/pulsar/io/rabbitmq/RabbitMQSource.java b/pulsar-io/rabbitmq/src/main/java/org/apache/pulsar/io/rabbitmq/RabbitMQSource.java index d15108c4d8288..b0b7ef31b08de 100644 --- a/pulsar-io/rabbitmq/src/main/java/org/apache/pulsar/io/rabbitmq/RabbitMQSource.java +++ b/pulsar-io/rabbitmq/src/main/java/org/apache/pulsar/io/rabbitmq/RabbitMQSource.java @@ -54,7 +54,7 @@ public class RabbitMQSource extends PushSource { @Override public void open(Map config, SourceContext sourceContext) throws Exception { - rabbitMQSourceConfig = RabbitMQSourceConfig.load(config); + rabbitMQSourceConfig = RabbitMQSourceConfig.load(config, sourceContext); rabbitMQSourceConfig.validate(); ConnectionFactory connectionFactory = rabbitMQSourceConfig.createConnectionFactory(); diff --git a/pulsar-io/rabbitmq/src/main/java/org/apache/pulsar/io/rabbitmq/RabbitMQSourceConfig.java b/pulsar-io/rabbitmq/src/main/java/org/apache/pulsar/io/rabbitmq/RabbitMQSourceConfig.java index f24018e70da13..01e23a7146080 100644 --- a/pulsar-io/rabbitmq/src/main/java/org/apache/pulsar/io/rabbitmq/RabbitMQSourceConfig.java +++ b/pulsar-io/rabbitmq/src/main/java/org/apache/pulsar/io/rabbitmq/RabbitMQSourceConfig.java @@ -28,6 +28,8 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.experimental.Accessors; +import org.apache.pulsar.io.common.IOConfigUtils; +import org.apache.pulsar.io.core.SourceContext; import org.apache.pulsar.io.core.annotations.FieldDoc; @Data @@ -66,9 +68,8 @@ public static RabbitMQSourceConfig load(String yamlFile) throws IOException { return mapper.readValue(new File(yamlFile), RabbitMQSourceConfig.class); } - public static RabbitMQSourceConfig load(Map map) throws IOException { - ObjectMapper mapper = new ObjectMapper(); - return mapper.readValue(mapper.writeValueAsString(map), RabbitMQSourceConfig.class); + public static RabbitMQSourceConfig load(Map map, SourceContext sourceContext) throws IOException { + return IOConfigUtils.loadWithSecrets(map, RabbitMQSourceConfig.class, sourceContext); } @Override diff --git a/pulsar-io/rabbitmq/src/test/java/org/apache/pulsar/io/rabbitmq/RabbitMQBrokerManager.java b/pulsar-io/rabbitmq/src/test/java/org/apache/pulsar/io/rabbitmq/RabbitMQBrokerManager.java index 507313c86fd7f..4ff8c61e4f401 100644 --- a/pulsar-io/rabbitmq/src/test/java/org/apache/pulsar/io/rabbitmq/RabbitMQBrokerManager.java +++ b/pulsar-io/rabbitmq/src/test/java/org/apache/pulsar/io/rabbitmq/RabbitMQBrokerManager.java @@ -18,49 +18,36 @@ */ package org.apache.pulsar.io.rabbitmq; -import org.apache.qpid.server.Broker; -import org.apache.qpid.server.BrokerOptions; - -import java.io.File; -import java.io.FileOutputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import org.apache.qpid.server.SystemLauncher; +import org.apache.qpid.server.model.SystemConfig; public class RabbitMQBrokerManager { - private final Broker broker = new Broker(); + private final SystemLauncher systemLauncher = new SystemLauncher(); public void startBroker(String port) throws Exception { - BrokerOptions brokerOptions = getBrokerOptions(port); - broker.startup(brokerOptions); + Map brokerOptions = getBrokerOptions(port); + systemLauncher.startup(brokerOptions); } public void stopBroker() { - broker.shutdown(); + systemLauncher.shutdown(); } - BrokerOptions getBrokerOptions(String port) throws Exception { + Map getBrokerOptions(String port) throws Exception { Path tmpFolder = Files.createTempDirectory("qpidWork"); - Path homeFolder = Files.createTempDirectory("qpidHome"); - File etc = new File(homeFolder.toFile(), "etc"); - etc.mkdir(); - FileOutputStream fos = new FileOutputStream(new File(etc, "passwd")); - fos.write("guest:guest\n".getBytes()); - fos.close(); - - BrokerOptions brokerOptions = new BrokerOptions(); - - brokerOptions.setConfigProperty("qpid.work_dir", tmpFolder.toAbsolutePath().toString()); - brokerOptions.setConfigProperty("qpid.amqp_port", port); - brokerOptions.setConfigProperty("qpid.home_dir", homeFolder.toAbsolutePath().toString()); - String configPath = getFile("qpid.json").getAbsolutePath(); - brokerOptions.setInitialConfigurationLocation(configPath); - - return brokerOptions; - } - - private File getFile(String name) { - ClassLoader classLoader = getClass().getClassLoader(); - return new File(classLoader.getResource(name).getFile()); + Map config = new HashMap<>(); + config.put("qpid.work_dir", tmpFolder.toAbsolutePath().toString()); + config.put("qpid.amqp_port", port); + + Map context = new HashMap<>(); + context.put(SystemConfig.INITIAL_CONFIGURATION_LOCATION, "classpath:qpid.json"); + context.put(SystemConfig.TYPE, "Memory"); + context.put(SystemConfig.CONTEXT, config); + return context; } } diff --git a/pulsar-io/rabbitmq/src/test/java/org/apache/pulsar/io/rabbitmq/sink/RabbitMQSinkConfigTest.java b/pulsar-io/rabbitmq/src/test/java/org/apache/pulsar/io/rabbitmq/sink/RabbitMQSinkConfigTest.java index 3d4fd6f46e16f..8706cb567524f 100644 --- a/pulsar-io/rabbitmq/src/test/java/org/apache/pulsar/io/rabbitmq/sink/RabbitMQSinkConfigTest.java +++ b/pulsar-io/rabbitmq/src/test/java/org/apache/pulsar/io/rabbitmq/sink/RabbitMQSinkConfigTest.java @@ -18,7 +18,9 @@ */ package org.apache.pulsar.io.rabbitmq.sink; +import org.apache.pulsar.io.core.SinkContext; import org.apache.pulsar.io.rabbitmq.RabbitMQSinkConfig; +import org.mockito.Mockito; import org.testng.annotations.Test; import java.io.File; @@ -71,7 +73,45 @@ public final void loadFromMapTest() throws IOException { map.put("exchangeName", "test-exchange"); map.put("exchangeType", "test-exchange-type"); - RabbitMQSinkConfig config = RabbitMQSinkConfig.load(map); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + RabbitMQSinkConfig config = RabbitMQSinkConfig.load(map, sinkContext); + assertNotNull(config); + assertEquals(config.getHost(), "localhost"); + assertEquals(config.getPort(), Integer.parseInt("5673")); + assertEquals(config.getVirtualHost(), "/"); + assertEquals(config.getUsername(), "guest"); + assertEquals(config.getPassword(), "guest"); + assertEquals(config.getConnectionName(), "test-connection"); + assertEquals(config.getRequestedChannelMax(), Integer.parseInt("0")); + assertEquals(config.getRequestedFrameMax(), Integer.parseInt("0")); + assertEquals(config.getConnectionTimeout(), Integer.parseInt("60000")); + assertEquals(config.getHandshakeTimeout(), Integer.parseInt("10000")); + assertEquals(config.getRequestedHeartbeat(), Integer.parseInt("60")); + assertEquals(config.getExchangeName(), "test-exchange"); + assertEquals(config.getExchangeType(), "test-exchange-type"); + } + + @Test + public final void loadFromMapCredentialsFromSecretTest() throws IOException { + Map map = new HashMap<>(); + map.put("host", "localhost"); + map.put("port", "5673"); + map.put("virtualHost", "/"); + map.put("connectionName", "test-connection"); + map.put("requestedChannelMax", "0"); + map.put("requestedFrameMax", "0"); + map.put("connectionTimeout", "60000"); + map.put("handshakeTimeout", "10000"); + map.put("requestedHeartbeat", "60"); + map.put("exchangeName", "test-exchange"); + map.put("exchangeType", "test-exchange-type"); + + SinkContext sinkContext = Mockito.mock(SinkContext.class); + Mockito.when(sinkContext.getSecret("username")) + .thenReturn("guest"); + Mockito.when(sinkContext.getSecret("password")) + .thenReturn("guest"); + RabbitMQSinkConfig config = RabbitMQSinkConfig.load(map, sinkContext); assertNotNull(config); assertEquals(config.getHost(), "localhost"); assertEquals(config.getPort(), Integer.parseInt("5673")); @@ -105,12 +145,13 @@ public final void validValidateTest() throws IOException { map.put("exchangeName", "test-exchange"); map.put("exchangeType", "test-exchange-type"); - RabbitMQSinkConfig config = RabbitMQSinkConfig.load(map); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + RabbitMQSinkConfig config = RabbitMQSinkConfig.load(map, sinkContext); config.validate(); } - @Test(expectedExceptions = NullPointerException.class, - expectedExceptionsMessageRegExp = "exchangeName property not set.") + @Test(expectedExceptions = IllegalArgumentException.class, + expectedExceptionsMessageRegExp = "exchangeName cannot be null") public final void missingExchangeValidateTest() throws IOException { Map map = new HashMap<>(); map.put("host", "localhost"); @@ -126,7 +167,8 @@ public final void missingExchangeValidateTest() throws IOException { map.put("requestedHeartbeat", "60"); map.put("exchangeType", "test-exchange-type"); - RabbitMQSinkConfig config = RabbitMQSinkConfig.load(map); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + RabbitMQSinkConfig config = RabbitMQSinkConfig.load(map, sinkContext); config.validate(); } diff --git a/pulsar-io/rabbitmq/src/test/java/org/apache/pulsar/io/rabbitmq/sink/RabbitMQSinkTest.java b/pulsar-io/rabbitmq/src/test/java/org/apache/pulsar/io/rabbitmq/sink/RabbitMQSinkTest.java index 3b20c61f82636..f03a36ce11485 100644 --- a/pulsar-io/rabbitmq/src/test/java/org/apache/pulsar/io/rabbitmq/sink/RabbitMQSinkTest.java +++ b/pulsar-io/rabbitmq/src/test/java/org/apache/pulsar/io/rabbitmq/sink/RabbitMQSinkTest.java @@ -18,12 +18,15 @@ */ package org.apache.pulsar.io.rabbitmq.sink; +import static org.mockito.Mockito.mock; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.HashMap; import java.util.Map; import java.util.Optional; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.functions.instance.SinkRecord; +import org.apache.pulsar.io.core.SinkContext; import org.apache.pulsar.io.rabbitmq.RabbitMQBrokerManager; import org.apache.pulsar.io.rabbitmq.RabbitMQSink; import org.awaitility.Awaitility; @@ -46,7 +49,7 @@ public void tearDown() { } @Test - public void TestOpenAndWriteSink() throws Exception { + public void testOpenAndWriteSink() throws Exception { Map configs = new HashMap<>(); configs.put("host", "localhost"); configs.put("port", "5673"); @@ -66,7 +69,9 @@ public void TestOpenAndWriteSink() throws Exception { // open should success // rabbitmq service may need time to initialize - Awaitility.await().ignoreExceptions().untilAsserted(() -> sink.open(configs, null)); + SinkContext sinkContext = mock(SinkContext.class); + Awaitility.await().ignoreExceptions().pollDelay(Duration.ofSeconds(1)) + .untilAsserted(() -> sink.open(configs, sinkContext)); // write should success Record record = build("test-topic", "fakeKey", "fakeValue", "fakeRoutingKey"); diff --git a/pulsar-io/rabbitmq/src/test/java/org/apache/pulsar/io/rabbitmq/source/RabbitMQSourceConfigTest.java b/pulsar-io/rabbitmq/src/test/java/org/apache/pulsar/io/rabbitmq/source/RabbitMQSourceConfigTest.java index c33e0070c6fd0..43a90062fa453 100644 --- a/pulsar-io/rabbitmq/src/test/java/org/apache/pulsar/io/rabbitmq/source/RabbitMQSourceConfigTest.java +++ b/pulsar-io/rabbitmq/src/test/java/org/apache/pulsar/io/rabbitmq/source/RabbitMQSourceConfigTest.java @@ -18,7 +18,9 @@ */ package org.apache.pulsar.io.rabbitmq.source; +import org.apache.pulsar.io.core.SourceContext; import org.apache.pulsar.io.rabbitmq.RabbitMQSourceConfig; +import org.mockito.Mockito; import org.testng.annotations.Test; import java.io.File; @@ -76,7 +78,50 @@ public final void loadFromMapTest() throws IOException { map.put("prefetchGlobal", "false"); map.put("passive", "true"); - RabbitMQSourceConfig config = RabbitMQSourceConfig.load(map); + SourceContext sourceContext = Mockito.mock(SourceContext.class); + RabbitMQSourceConfig config = RabbitMQSourceConfig.load(map, sourceContext); + assertNotNull(config); + assertEquals("localhost", config.getHost()); + assertEquals(Integer.parseInt("5672"), config.getPort()); + assertEquals("/", config.getVirtualHost()); + assertEquals("guest", config.getUsername()); + assertEquals("guest", config.getPassword()); + assertEquals("test-queue", config.getQueueName()); + assertEquals("test-connection", config.getConnectionName()); + assertEquals(Integer.parseInt("0"), config.getRequestedChannelMax()); + assertEquals(Integer.parseInt("0"), config.getRequestedFrameMax()); + assertEquals(Integer.parseInt("60000"), config.getConnectionTimeout()); + assertEquals(Integer.parseInt("10000"), config.getHandshakeTimeout()); + assertEquals(Integer.parseInt("60"), config.getRequestedHeartbeat()); + assertEquals(Integer.parseInt("0"), config.getPrefetchCount()); + assertEquals(false, config.isPrefetchGlobal()); + assertEquals(false, config.isPrefetchGlobal()); + assertEquals(true, config.isPassive()); + } + + @Test + public final void loadFromMapCredentialsFromSecretTest() throws IOException { + Map map = new HashMap<>(); + map.put("host", "localhost"); + map.put("port", "5672"); + map.put("virtualHost", "/"); + map.put("queueName", "test-queue"); + map.put("connectionName", "test-connection"); + map.put("requestedChannelMax", "0"); + map.put("requestedFrameMax", "0"); + map.put("connectionTimeout", "60000"); + map.put("handshakeTimeout", "10000"); + map.put("requestedHeartbeat", "60"); + map.put("prefetchCount", "0"); + map.put("prefetchGlobal", "false"); + map.put("passive", "true"); + + SourceContext sourceContext = Mockito.mock(SourceContext.class); + Mockito.when(sourceContext.getSecret("username")) + .thenReturn("guest"); + Mockito.when(sourceContext.getSecret("password")) + .thenReturn("guest"); + RabbitMQSourceConfig config = RabbitMQSourceConfig.load(map, sourceContext); assertNotNull(config); assertEquals("localhost", config.getHost()); assertEquals(Integer.parseInt("5672"), config.getPort()); @@ -115,12 +160,13 @@ public final void validValidateTest() throws IOException { map.put("prefetchGlobal", "false"); map.put("passive", "false"); - RabbitMQSourceConfig config = RabbitMQSourceConfig.load(map); + SourceContext sourceContext = Mockito.mock(SourceContext.class); + RabbitMQSourceConfig config = RabbitMQSourceConfig.load(map, sourceContext); config.validate(); } - @Test(expectedExceptions = NullPointerException.class, - expectedExceptionsMessageRegExp = "host property not set.") + @Test(expectedExceptions = IllegalArgumentException.class, + expectedExceptionsMessageRegExp = "host cannot be null") public final void missingHostValidateTest() throws IOException { Map map = new HashMap<>(); map.put("port", "5672"); @@ -138,7 +184,8 @@ public final void missingHostValidateTest() throws IOException { map.put("prefetchGlobal", "false"); map.put("passive", "false"); - RabbitMQSourceConfig config = RabbitMQSourceConfig.load(map); + SourceContext sourceContext = Mockito.mock(SourceContext.class); + RabbitMQSourceConfig config = RabbitMQSourceConfig.load(map, sourceContext); config.validate(); } @@ -162,7 +209,8 @@ public final void invalidPrefetchCountTest() throws IOException { map.put("prefetchGlobal", "false"); map.put("passive", "false"); - RabbitMQSourceConfig config = RabbitMQSourceConfig.load(map); + SourceContext sourceContext = Mockito.mock(SourceContext.class); + RabbitMQSourceConfig config = RabbitMQSourceConfig.load(map, sourceContext); config.validate(); } diff --git a/pulsar-io/rabbitmq/src/test/java/org/apache/pulsar/io/rabbitmq/source/RabbitMQSourceTest.java b/pulsar-io/rabbitmq/src/test/java/org/apache/pulsar/io/rabbitmq/source/RabbitMQSourceTest.java index abff93a363298..08869e018c625 100644 --- a/pulsar-io/rabbitmq/src/test/java/org/apache/pulsar/io/rabbitmq/source/RabbitMQSourceTest.java +++ b/pulsar-io/rabbitmq/src/test/java/org/apache/pulsar/io/rabbitmq/source/RabbitMQSourceTest.java @@ -18,6 +18,9 @@ */ package org.apache.pulsar.io.rabbitmq.source; +import static org.mockito.Mockito.mock; +import java.time.Duration; +import org.apache.pulsar.io.core.SourceContext; import org.apache.pulsar.io.rabbitmq.RabbitMQBrokerManager; import org.apache.pulsar.io.rabbitmq.RabbitMQSource; import org.awaitility.Awaitility; @@ -44,7 +47,7 @@ public void tearDown() { } @Test - public void TestOpenAndWriteSink() { + public void testOpenAndWriteSink() throws Exception { Map configs = new HashMap<>(); configs.put("host", "localhost"); configs.put("port", "5672"); @@ -66,7 +69,11 @@ public void TestOpenAndWriteSink() { // open should success // rabbitmq service may need time to initialize - Awaitility.await().ignoreExceptions().untilAsserted(() -> source.open(configs, null)); + SourceContext sourceContext = mock(SourceContext.class); + Awaitility.await().ignoreExceptions().pollDelay(Duration.ofSeconds(1)) + .untilAsserted(() -> source.open(configs, sourceContext)); + source.close(); } + } diff --git a/pulsar-io/rabbitmq/src/test/resources/qpid.json b/pulsar-io/rabbitmq/src/test/resources/qpid.json index 6a0381f6ddc2c..419e9cc1e4a55 100644 --- a/pulsar-io/rabbitmq/src/test/resources/qpid.json +++ b/pulsar-io/rabbitmq/src/test/resources/qpid.json @@ -1,25 +1,57 @@ { - "name": "EmbeddedBroker", + "name": "${broker.name}", "modelVersion": "2.0", - "storeVersion": 1, "authenticationproviders": [ { - "name": "noPassword", - "type": "Anonymous", - "secureOnlyMechanisms": [] - }, + "name": "plain", + "type": "Plain", + "secureOnlyMechanisms": [], + "users": [ + { + "name": "guest", + "password": "guest", + "type": "managed" + } + ] + } + ], + "brokerloggers": [ { - "name": "passwordFile", - "type": "PlainPasswordFile", - "path": "${qpid.home_dir}${file.separator}etc${file.separator}passwd", - "secureOnlyMechanisms": [] + "name": "console", + "type": "Console", + "brokerloginclusionrules": [ + { + "name": "Root", + "type": "NameAndLevel", + "level": "WARN", + "loggerName": "ROOT" + }, + { + "name": "Qpid", + "type": "NameAndLevel", + "level": "INFO", + "loggerName": "org.apache.qpid.*" + }, + { + "name": "Operational", + "type": "NameAndLevel", + "level": "INFO", + "loggerName": "qpid.message.*" + }, + { + "name": "Statistics", + "type": "NameAndLevel", + "level": "INFO", + "loggerName": "qpid.statistics.*" + } + ] } ], "ports": [ { "name": "AMQP", "port": "${qpid.amqp_port}", - "authenticationProvider": "passwordFile", + "authenticationProvider": "plain", "protocols": [ "AMQP_0_9_1" ] @@ -28,10 +60,9 @@ "virtualhostnodes": [ { "name": "default", - "type": "JSON", + "type": "Memory", "defaultVirtualHostNode": "true", - "virtualHostInitialConfiguration": "${qpid.initial_config_virtualhost_config}", - "storeType": "DERBY" + "virtualHostInitialConfiguration": "{\"type\": \"Memory\"}" } ] } \ No newline at end of file diff --git a/pulsar-io/redis/pom.xml b/pulsar-io/redis/pom.xml index 351ae205e667f..376e867b2cf48 100644 --- a/pulsar-io/redis/pom.xml +++ b/pulsar-io/redis/pom.xml @@ -25,13 +25,18 @@ pulsar-io org.apache.pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-redis Pulsar IO :: Redis + + ${project.groupId} + pulsar-io-common + ${project.version} + ${project.groupId} pulsar-io-core diff --git a/pulsar-io/redis/src/main/java/org/apache/pulsar/io/redis/RedisAbstractConfig.java b/pulsar-io/redis/src/main/java/org/apache/pulsar/io/redis/RedisAbstractConfig.java index 978e7de31a51c..89ec684dded72 100644 --- a/pulsar-io/redis/src/main/java/org/apache/pulsar/io/redis/RedisAbstractConfig.java +++ b/pulsar-io/redis/src/main/java/org/apache/pulsar/io/redis/RedisAbstractConfig.java @@ -88,13 +88,11 @@ public class RedisAbstractConfig implements Serializable { @FieldDoc( required = false, - defaultValue = "10000L", + defaultValue = "10000", help = "The amount of time in milliseconds to wait before timing out when connecting") private long connectTimeout = 10000L; public void validate() { - Preconditions.checkNotNull(redisHosts, "redisHosts property not set."); - Preconditions.checkNotNull(redisDatabase, "redisDatabase property not set."); Preconditions.checkNotNull(clientMode, "clientMode property not set."); } @@ -105,7 +103,6 @@ public enum ClientMode { public List getHostAndPorts() { List hostAndPorts = Lists.newArrayList(); - Preconditions.checkNotNull(redisHosts, "redisHosts property not set."); String[] hosts = StringUtils.split(redisHosts, ","); for (String host : hosts) { HostAndPort hostAndPort = HostAndPort.fromString(host); diff --git a/pulsar-io/redis/src/main/java/org/apache/pulsar/io/redis/sink/RedisSink.java b/pulsar-io/redis/src/main/java/org/apache/pulsar/io/redis/sink/RedisSink.java index bff0a5c2da592..ebd6e9dbab272 100644 --- a/pulsar-io/redis/src/main/java/org/apache/pulsar/io/redis/sink/RedisSink.java +++ b/pulsar-io/redis/src/main/java/org/apache/pulsar/io/redis/sink/RedisSink.java @@ -68,7 +68,7 @@ public class RedisSink implements Sink { public void open(Map config, SinkContext sinkContext) throws Exception { log.info("Open Redis Sink"); - redisSinkConfig = RedisSinkConfig.load(config); + redisSinkConfig = RedisSinkConfig.load(config, sinkContext); redisSinkConfig.validate(); redisSession = RedisSession.create(redisSinkConfig); diff --git a/pulsar-io/redis/src/main/java/org/apache/pulsar/io/redis/sink/RedisSinkConfig.java b/pulsar-io/redis/src/main/java/org/apache/pulsar/io/redis/sink/RedisSinkConfig.java index a9db66812a475..f7a70cb65a826 100644 --- a/pulsar-io/redis/src/main/java/org/apache/pulsar/io/redis/sink/RedisSinkConfig.java +++ b/pulsar-io/redis/src/main/java/org/apache/pulsar/io/redis/sink/RedisSinkConfig.java @@ -28,6 +28,8 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.experimental.Accessors; +import org.apache.pulsar.io.common.IOConfigUtils; +import org.apache.pulsar.io.core.SinkContext; import org.apache.pulsar.io.core.annotations.FieldDoc; import org.apache.pulsar.io.redis.RedisAbstractConfig; @@ -40,13 +42,13 @@ public class RedisSinkConfig extends RedisAbstractConfig implements Serializable @FieldDoc( required = false, - defaultValue = "10000L", + defaultValue = "10000", help = "The amount of time in milliseconds before an operation is marked as timed out") private long operationTimeout = 10000L; @FieldDoc( required = false, - defaultValue = "1000L", + defaultValue = "1000", help = "The Redis operation time in milliseconds") private long batchTimeMs = 1000L; @@ -62,9 +64,8 @@ public static RedisSinkConfig load(String yamlFile) throws IOException { return mapper.readValue(new File(yamlFile), RedisSinkConfig.class); } - public static RedisSinkConfig load(Map map) throws IOException { - ObjectMapper mapper = new ObjectMapper(); - return mapper.readValue(mapper.writeValueAsString(map), RedisSinkConfig.class); + public static RedisSinkConfig load(Map map, SinkContext sinkContext) throws IOException { + return IOConfigUtils.loadWithSecrets(map, RedisSinkConfig.class, sinkContext); } @Override diff --git a/pulsar-io/redis/src/test/java/org/apache/pulsar/io/redis/sink/RedisSinkConfigTest.java b/pulsar-io/redis/src/test/java/org/apache/pulsar/io/redis/sink/RedisSinkConfigTest.java index 1316d0994a1cd..39fc6e540c242 100644 --- a/pulsar-io/redis/src/test/java/org/apache/pulsar/io/redis/sink/RedisSinkConfigTest.java +++ b/pulsar-io/redis/src/test/java/org/apache/pulsar/io/redis/sink/RedisSinkConfigTest.java @@ -18,7 +18,9 @@ */ package org.apache.pulsar.io.redis.sink; +import org.apache.pulsar.io.core.SinkContext; import org.apache.pulsar.io.redis.RedisAbstractConfig; +import org.mockito.Mockito; import org.testng.annotations.Test; import java.io.File; @@ -62,7 +64,34 @@ public final void loadFromMapTest() throws IOException { map.put("batchTimeMs", "1000"); map.put("connectTimeout", "3000"); - RedisSinkConfig config = RedisSinkConfig.load(map); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + RedisSinkConfig config = RedisSinkConfig.load(map, sinkContext); + assertNotNull(config); + assertEquals(config.getRedisHosts(), "localhost:6379"); + assertEquals(config.getRedisPassword(), "fake@123"); + assertEquals(config.getRedisDatabase(), Integer.parseInt("1")); + assertEquals(config.getClientMode(), "Standalone"); + assertEquals(config.getOperationTimeout(), Long.parseLong("2000")); + assertEquals(config.getBatchSize(), Integer.parseInt("100")); + assertEquals(config.getBatchTimeMs(), Long.parseLong("1000")); + assertEquals(config.getConnectTimeout(), Long.parseLong("3000")); + } + + @Test + public final void loadFromMapCredentialsFromSecretTest() throws IOException { + Map map = new HashMap(); + map.put("redisHosts", "localhost:6379"); + map.put("redisDatabase", "1"); + map.put("clientMode", "Standalone"); + map.put("operationTimeout", "2000"); + map.put("batchSize", "100"); + map.put("batchTimeMs", "1000"); + map.put("connectTimeout", "3000"); + + SinkContext sinkContext = Mockito.mock(SinkContext.class); + Mockito.when(sinkContext.getSecret("redisPassword")) + .thenReturn("fake@123"); + RedisSinkConfig config = RedisSinkConfig.load(map, sinkContext); assertNotNull(config); assertEquals(config.getRedisHosts(), "localhost:6379"); assertEquals(config.getRedisPassword(), "fake@123"); @@ -86,12 +115,13 @@ public final void validValidateTest() throws IOException { map.put("batchTimeMs", "1000"); map.put("connectTimeout", "3000"); - RedisSinkConfig config = RedisSinkConfig.load(map); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + RedisSinkConfig config = RedisSinkConfig.load(map, sinkContext); config.validate(); } - @Test(expectedExceptions = NullPointerException.class, - expectedExceptionsMessageRegExp = "redisHosts property not set.") + @Test(expectedExceptions = IllegalArgumentException.class, + expectedExceptionsMessageRegExp = "redisHosts cannot be null") public final void missingValidValidateTableNameTest() throws IOException { Map map = new HashMap(); map.put("redisPassword", "fake@123"); @@ -102,7 +132,8 @@ public final void missingValidValidateTableNameTest() throws IOException { map.put("batchTimeMs", "1000"); map.put("connectTimeout", "3000"); - RedisSinkConfig config = RedisSinkConfig.load(map); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + RedisSinkConfig config = RedisSinkConfig.load(map, sinkContext); config.validate(); } @@ -119,7 +150,8 @@ public final void invalidBatchTimeMsTest() throws IOException { map.put("batchTimeMs", "-100"); map.put("connectTimeout", "3000"); - RedisSinkConfig config = RedisSinkConfig.load(map); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + RedisSinkConfig config = RedisSinkConfig.load(map, sinkContext); config.validate(); } @@ -136,7 +168,8 @@ public final void invalidClientModeTest() throws IOException { map.put("batchTimeMs", "1000"); map.put("connectTimeout", "3000"); - RedisSinkConfig config = RedisSinkConfig.load(map); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + RedisSinkConfig config = RedisSinkConfig.load(map, sinkContext); config.validate(); RedisAbstractConfig.ClientMode.valueOf(config.getClientMode().toUpperCase()); diff --git a/pulsar-io/redis/src/test/java/org/apache/pulsar/io/redis/sink/RedisSinkTest.java b/pulsar-io/redis/src/test/java/org/apache/pulsar/io/redis/sink/RedisSinkTest.java index 214151345b42c..2b407fafa5e04 100644 --- a/pulsar-io/redis/src/test/java/org/apache/pulsar/io/redis/sink/RedisSinkTest.java +++ b/pulsar-io/redis/src/test/java/org/apache/pulsar/io/redis/sink/RedisSinkTest.java @@ -21,7 +21,9 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.functions.instance.SinkRecord; +import org.apache.pulsar.io.core.SinkContext; import org.apache.pulsar.io.redis.EmbeddedRedisUtils; +import org.mockito.Mockito; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -66,7 +68,8 @@ public void TestOpenAndWriteSink() throws Exception { Record record = build("fakeTopic", "fakeKey", "fakeValue"); // open should success - sink.open(configs, null); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + sink.open(configs, sinkContext); // write should success. sink.write(record); diff --git a/pulsar-io/solr/pom.xml b/pulsar-io/solr/pom.xml index 741c6d135c509..f81476f15d245 100644 --- a/pulsar-io/solr/pom.xml +++ b/pulsar-io/solr/pom.xml @@ -25,17 +25,22 @@ pulsar-io org.apache.pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT - 8.11.1 + 8.11.3 pulsar-io-solr Pulsar IO :: Solr + + ${project.groupId} + pulsar-io-common + ${project.version} + ${project.parent.groupId} pulsar-io-core @@ -60,6 +65,16 @@ org.apache.solr solr-core ${solr.version} + + + jose4j + org.bitbucket.b_c + + + log4j-slf4j-impl + org.apache.logging.log4j + + test diff --git a/pulsar-io/solr/src/main/java/org/apache/pulsar/io/solr/SolrAbstractSink.java b/pulsar-io/solr/src/main/java/org/apache/pulsar/io/solr/SolrAbstractSink.java index de9cdb4a9d82a..202c782c14c49 100644 --- a/pulsar-io/solr/src/main/java/org/apache/pulsar/io/solr/SolrAbstractSink.java +++ b/pulsar-io/solr/src/main/java/org/apache/pulsar/io/solr/SolrAbstractSink.java @@ -48,7 +48,7 @@ public abstract class SolrAbstractSink implements Sink { @Override public void open(Map config, SinkContext sinkContext) throws Exception { - solrSinkConfig = SolrSinkConfig.load(config); + solrSinkConfig = SolrSinkConfig.load(config, sinkContext); solrSinkConfig.validate(); enableBasicAuth = !Strings.isNullOrEmpty(solrSinkConfig.getUsername()); diff --git a/pulsar-io/solr/src/main/java/org/apache/pulsar/io/solr/SolrSinkConfig.java b/pulsar-io/solr/src/main/java/org/apache/pulsar/io/solr/SolrSinkConfig.java index 02733d230bdcb..daa93a366b110 100644 --- a/pulsar-io/solr/src/main/java/org/apache/pulsar/io/solr/SolrSinkConfig.java +++ b/pulsar-io/solr/src/main/java/org/apache/pulsar/io/solr/SolrSinkConfig.java @@ -27,6 +27,8 @@ import java.util.Map; import lombok.Data; import lombok.experimental.Accessors; +import org.apache.pulsar.io.common.IOConfigUtils; +import org.apache.pulsar.io.core.SinkContext; import org.apache.pulsar.io.core.annotations.FieldDoc; /** @@ -84,9 +86,8 @@ public static SolrSinkConfig load(String yamlFile) throws IOException { return mapper.readValue(new File(yamlFile), SolrSinkConfig.class); } - public static SolrSinkConfig load(Map map) throws IOException { - ObjectMapper mapper = new ObjectMapper(); - return mapper.readValue(mapper.writeValueAsString(map), SolrSinkConfig.class); + public static SolrSinkConfig load(Map map, SinkContext sinkContext) throws IOException { + return IOConfigUtils.loadWithSecrets(map, SolrSinkConfig.class, sinkContext); } public void validate() { diff --git a/pulsar-io/solr/src/test/java/org/apache/pulsar/io/solr/SolrSinkConfigTest.java b/pulsar-io/solr/src/test/java/org/apache/pulsar/io/solr/SolrSinkConfigTest.java index 42d2121dbfcbd..2c2447a637d35 100644 --- a/pulsar-io/solr/src/test/java/org/apache/pulsar/io/solr/SolrSinkConfigTest.java +++ b/pulsar-io/solr/src/test/java/org/apache/pulsar/io/solr/SolrSinkConfigTest.java @@ -19,6 +19,8 @@ package org.apache.pulsar.io.solr; import com.google.common.collect.Lists; +import org.apache.pulsar.io.core.SinkContext; +import org.mockito.Mockito; import org.testng.annotations.Test; import java.io.File; @@ -61,7 +63,31 @@ public final void loadFromMapTest() throws IOException { map.put("username", "fakeuser"); map.put("password", "fake@123"); - SolrSinkConfig config = SolrSinkConfig.load(map); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + SolrSinkConfig config = SolrSinkConfig.load(map, sinkContext); + assertNotNull(config); + assertEquals(config.getSolrUrl(), "localhost:2181,localhost:2182/chroot"); + assertEquals(config.getSolrMode(), "SolrCloud"); + assertEquals(config.getSolrCollection(), "techproducts"); + assertEquals(config.getSolrCommitWithinMs(), Integer.parseInt("100")); + assertEquals(config.getUsername(), "fakeuser"); + assertEquals(config.getPassword(), "fake@123"); + } + + @Test + public final void loadFromMapCredentialsFromSecretTest() throws IOException { + Map map = new HashMap<>(); + map.put("solrUrl", "localhost:2181,localhost:2182/chroot"); + map.put("solrMode", "SolrCloud"); + map.put("solrCollection", "techproducts"); + map.put("solrCommitWithinMs", "100"); + + SinkContext sinkContext = Mockito.mock(SinkContext.class); + Mockito.when(sinkContext.getSecret("username")) + .thenReturn("fakeuser"); + Mockito.when(sinkContext.getSecret("password")) + .thenReturn("fake@123"); + SolrSinkConfig config = SolrSinkConfig.load(map, sinkContext); assertNotNull(config); assertEquals(config.getSolrUrl(), "localhost:2181,localhost:2182/chroot"); assertEquals(config.getSolrMode(), "SolrCloud"); @@ -81,12 +107,13 @@ public final void validValidateTest() throws IOException { map.put("username", "fakeuser"); map.put("password", "fake@123"); - SolrSinkConfig config = SolrSinkConfig.load(map); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + SolrSinkConfig config = SolrSinkConfig.load(map, sinkContext); config.validate(); } - @Test(expectedExceptions = NullPointerException.class, - expectedExceptionsMessageRegExp = "solrUrl property not set.") + @Test(expectedExceptions = IllegalArgumentException.class, + expectedExceptionsMessageRegExp = "solrUrl cannot be null") public final void missingValidValidateSolrModeTest() throws IOException { Map map = new HashMap<>(); map.put("solrMode", "SolrCloud"); @@ -95,7 +122,8 @@ public final void missingValidValidateSolrModeTest() throws IOException { map.put("username", "fakeuser"); map.put("password", "fake@123"); - SolrSinkConfig config = SolrSinkConfig.load(map); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + SolrSinkConfig config = SolrSinkConfig.load(map, sinkContext); config.validate(); } @@ -110,7 +138,8 @@ public final void invalidBatchTimeMsTest() throws IOException { map.put("username", "fakeuser"); map.put("password", "fake@123"); - SolrSinkConfig config = SolrSinkConfig.load(map); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + SolrSinkConfig config = SolrSinkConfig.load(map, sinkContext); config.validate(); } @@ -125,7 +154,8 @@ public final void invalidClientModeTest() throws IOException { map.put("username", "fakeuser"); map.put("password", "fake@123"); - SolrSinkConfig config = SolrSinkConfig.load(map); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + SolrSinkConfig config = SolrSinkConfig.load(map, sinkContext); config.validate(); SolrAbstractSink.SolrMode.valueOf(config.getSolrMode().toUpperCase()); @@ -141,7 +171,8 @@ public final void validZkChrootTest() throws IOException { map.put("username", "fakeuser"); map.put("password", "fake@123"); - SolrSinkConfig config = SolrSinkConfig.load(map); + SinkContext sinkContext = Mockito.mock(SinkContext.class); + SolrSinkConfig config = SolrSinkConfig.load(map, sinkContext); config.validate(); String url = config.getSolrUrl(); diff --git a/pulsar-io/twitter/pom.xml b/pulsar-io/twitter/pom.xml index 03b9394c44109..7109a035b36e2 100644 --- a/pulsar-io/twitter/pom.xml +++ b/pulsar-io/twitter/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-io-twitter diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index a494fa1c35f8f..f33dfe65bd005 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -25,10 +25,14 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT + + + 2 + + pulsar-metadata Pulsar Metadata @@ -57,14 +61,49 @@ - + + io.streamnative.oxia + oxia-client + + + + io.streamnative.oxia + oxia-testcontainers + test + + io.dropwizard.metrics metrics-core test - + + org.apache.zookeeper + zookeeper + tests + test + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + io.netty + netty-tcnative + + + + + ${project.groupId} + testmocks + ${project.version} + test + org.xerial.snappy snappy-java @@ -83,10 +122,15 @@ - io.etcd - jetcd-core + ${project.groupId} + jetcd-core-shaded + ${project.version} + shaded + + + io.grpc + grpc-netty-shaded - io.etcd @@ -125,7 +169,7 @@ - + org.apache.maven.plugins maven-jar-plugin @@ -170,4 +214,20 @@ + + + skipTestsForUnitGroupOther + + + + org.apache.maven.plugins + maven-surefire-plugin + + true + + + + + + diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCache.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCache.java index 94da382b74dcf..4af712d33571e 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCache.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCache.java @@ -18,12 +18,14 @@ */ package org.apache.pulsar.metadata.api; +import java.util.EnumSet; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.function.Function; import org.apache.pulsar.metadata.api.MetadataStoreException.AlreadyExistsException; import org.apache.pulsar.metadata.api.MetadataStoreException.NotFoundException; +import org.apache.pulsar.metadata.api.extended.CreateOption; /** * Represent the caching layer access for a specific type of objects. @@ -57,7 +59,7 @@ public interface MetadataCache { * * @param path * the path of the object in the metadata store - * @return the cached object or an empty {@link Optional} is the cache doesn't have the object + * @return the cached object or an empty {@link Optional} is the cache does not have the object */ Optional getIfCached(String path); @@ -128,6 +130,24 @@ public interface MetadataCache { */ CompletableFuture create(String path, T value); + /** + * Create or update the value of the given path in the metadata store without version comparison. + *

      + * This method is equivalent to + * {@link org.apache.pulsar.metadata.api.extended.MetadataStoreExtended#put(String, byte[], Optional, EnumSet)} or + * {@link MetadataStore#put(String, byte[], Optional)} if the metadata store does not support this extended API, + * with `Optional.empty()` as the 3rd argument. It means if the path does not exist, it will be created. If the path + * already exists, the new value will override the old value. + *

      + * @param path the path of the object in the metadata store + * @param value the object to put in the metadata store + * @param options the create options if the path does not in the metadata store + * @return the future that indicates if this operation failed, it could fail with + * {@link java.io.IOException} if the value failed to be serialized + * {@link MetadataStoreException} if the metadata store operation failed + */ + CompletableFuture put(String path, T value, EnumSet options); + /** * Delete an object from the metadata store. *

      @@ -148,6 +168,11 @@ public interface MetadataCache { */ void invalidate(String path); + /** + * Force the invalidation of all object in the metadata cache. + */ + void invalidateAll(); + /** * Invalidate and reload an object in the metadata cache. * diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCacheConfig.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCacheConfig.java index 55b159071fda4..2bc042aebb308 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCacheConfig.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCacheConfig.java @@ -18,7 +18,9 @@ */ package org.apache.pulsar.metadata.api; +import java.util.Optional; import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; import lombok.Builder; import lombok.Getter; import lombok.ToString; @@ -29,7 +31,7 @@ @Builder @Getter @ToString -public class MetadataCacheConfig { +public class MetadataCacheConfig { private static final long DEFAULT_CACHE_REFRESH_TIME_MILLIS = TimeUnit.MINUTES.toMillis(5); /** @@ -47,4 +49,12 @@ public class MetadataCacheConfig { */ @Builder.Default private final long expireAfterWriteMillis = 2 * DEFAULT_CACHE_REFRESH_TIME_MILLIS; + + /** + * Specifies cache reload consumer behavior when the cache is refreshed automatically at refreshAfterWriteMillis + * frequency. + */ + @Builder.Default + private final BiConsumer>> asyncReloadConsumer = null; + } diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataEventSynchronizer.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataEventSynchronizer.java index 9a735e0f15ab8..cababd0324627 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataEventSynchronizer.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataEventSynchronizer.java @@ -49,5 +49,5 @@ public interface MetadataEventSynchronizer { /** * close synchronizer resources. */ - void close(); + CompletableFuture closeAsync(); } diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStore.java index 33942c19520a3..89b0e7a6fe1c0 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStore.java @@ -23,9 +23,12 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.function.Consumer; import org.apache.pulsar.metadata.api.MetadataStoreException.BadVersionException; import org.apache.pulsar.metadata.api.MetadataStoreException.NotFoundException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Metadata store client interface. @@ -36,6 +39,8 @@ @Beta public interface MetadataStore extends AutoCloseable { + Logger LOGGER = LoggerFactory.getLogger(MetadataStore.class); + /** * Read the value of one key, identified by the path * @@ -121,6 +126,23 @@ default CompletableFuture sync(String path) { */ CompletableFuture delete(String path, Optional expectedVersion); + default CompletableFuture deleteIfExists(String path, Optional expectedVersion) { + return delete(path, expectedVersion) + .exceptionally(e -> { + if (e.getCause() instanceof NotFoundException) { + LOGGER.info("Path {} not found while deleting (this is not a problem)", path); + return null; + } else { + if (expectedVersion.isEmpty()) { + LOGGER.info("Failed to delete path {}", path, e); + } else { + LOGGER.info("Failed to delete path {} with expected version {}", path, expectedVersion, e); + } + throw new CompletionException(e); + } + }); + } + /** * Delete a key-value pair and all the children nodes. * diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStoreConfig.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStoreConfig.java index 5ddfe33c3912a..be29f843eea18 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStoreConfig.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStoreConfig.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.metadata.api; +import io.opentelemetry.api.OpenTelemetry; import lombok.Builder; import lombok.Getter; import lombok.ToString; @@ -92,4 +93,10 @@ public class MetadataStoreConfig { * separate clusters. */ private MetadataEventSynchronizer synchronizer; + + /** + * OpenTelemetry instance to monitor metadata store operations. + */ + @Builder.Default + private OpenTelemetry openTelemetry = OpenTelemetry.noop(); } diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStoreTableView.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStoreTableView.java new file mode 100644 index 0000000000000..64de22890a0f1 --- /dev/null +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStoreTableView.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.metadata.api; + +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +/** + * Defines metadata store tableview. + * MetadataStoreTableView initially fills existing items to its local tableview and eventually + * synchronize remote updates to its local tableview from the remote metadata store. + * This abstraction can help replicate metadata in memory from metadata store. + */ +public interface MetadataStoreTableView { + + class ConflictException extends RuntimeException { + public ConflictException(String msg) { + super(msg); + } + } + + /** + * Starts the tableview by filling existing items to its local tableview from the remote metadata store. + */ + void start() throws MetadataStoreException; + + /** + * Gets one item from the local tableview. + *

      + * If the key is not found, return null. + * + * @param key the key to check + * @return value if exists. Otherwise, null. + */ + T get(String key); + + /** + * Tries to put the item in the persistent store. + * All peer tableviews (including the local one) will be notified and be eventually consistent with this put value. + *

      + * This operation can fail if the input value conflicts with the existing one. + * + * @param key the key to check on the tableview + * @return a future to track the completion of the operation + * @throws MetadataStoreTableView.ConflictException + * if the input value conflicts with the existing one. + */ + CompletableFuture put(String key, T value); + + /** + * Tries to delete the item from the persistent store. + * All peer tableviews (including the local one) will be notified and be eventually consistent with this deletion. + *

      + * This can fail if the item is not present in the metadata store. + * + * @param key the key to check on the tableview + * @return a future to track the completion of the operation + * @throws MetadataStoreException.NotFoundException + * if the key is not present in the metadata store. + */ + CompletableFuture delete(String key); + + /** + * Returns the entry set of the items in the local tableview. + * @return entry set + */ + Set> entrySet(); +} + diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/extended/MetadataStoreExtended.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/extended/MetadataStoreExtended.java index e565ba30d3dfb..182c14ef601a4 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/extended/MetadataStoreExtended.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/extended/MetadataStoreExtended.java @@ -84,6 +84,8 @@ default Optional getMetadataEventSynchronizer() { return Optional.empty(); } + default void updateMetadataEventSynchronizer(MetadataEventSynchronizer synchronizer) {} + /** * Handles a metadata synchronizer event. * diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/AbstractHierarchicalLedgerManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/AbstractHierarchicalLedgerManager.java index a33c8761cba9e..4db7f4798c309 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/AbstractHierarchicalLedgerManager.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/AbstractHierarchicalLedgerManager.java @@ -206,10 +206,11 @@ protected void asyncProcessLedgersInSingleNode( mcb = new BookkeeperInternalCallbacks.MultiCallback(activeLedgers.size(), finalCb, ctx, successRc, failureRc); // start loop over all ledgers - for (Long ledger : activeLedgers) { - processor.process(ledger, mcb); - } - + scheduler.submit(() -> { + for (Long ledger : activeLedgers) { + processor.process(ledger, mcb); + } + }); }).exceptionally(ex -> { finalCb.processResult(failureRc, null, ctx); return null; diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/AbstractMetadataDriver.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/AbstractMetadataDriver.java index cc5f759c73fe5..435f94b05dc2b 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/AbstractMetadataDriver.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/AbstractMetadataDriver.java @@ -21,6 +21,7 @@ import java.io.Closeable; import java.io.IOException; import java.net.URI; +import java.util.concurrent.TimeUnit; import lombok.SneakyThrows; import org.apache.bookkeeper.conf.AbstractConfiguration; import org.apache.bookkeeper.discover.RegistrationClient; @@ -40,6 +41,7 @@ public abstract class AbstractMetadataDriver implements Closeable { public static final String METADATA_STORE_SCHEME = "metadata-store"; public static final String METADATA_STORE_INSTANCE = "metadata-store-instance"; + public static final long BLOCKING_CALL_TIMEOUT = TimeUnit.SECONDS.toMillis(30); protected MetadataStoreExtended store; private boolean storeInstanceIsOwned; diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/BKCluster.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/BKCluster.java index c2f3f72ec21c0..fe2b981ffe995 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/BKCluster.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/BKCluster.java @@ -30,6 +30,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Matcher; import java.util.regex.Pattern; import lombok.Getter; @@ -49,8 +50,8 @@ import org.apache.bookkeeper.replication.AutoRecoveryMain; import org.apache.bookkeeper.server.conf.BookieConfiguration; import org.apache.bookkeeper.util.IOUtils; -import org.apache.bookkeeper.util.PortManager; import org.apache.commons.io.FileUtils; +import org.apache.pulsar.common.util.PortManager; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; @@ -74,6 +75,9 @@ public class BKCluster implements AutoCloseable { protected final ServerConfiguration baseConf; protected final ClientConfiguration baseClientConf; + private final List lockedPorts = new ArrayList<>(); + private final AtomicBoolean closed = new AtomicBoolean(false); + public static class BKClusterConf { private ServerConfiguration baseServerConfiguration; @@ -148,20 +152,24 @@ private BKCluster(BKClusterConf bkClusterConf) throws Exception { @Override public void close() throws Exception { - // stop bookkeeper service - try { - stopBKCluster(); - } catch (Exception e) { - log.error("Got Exception while trying to stop BKCluster", e); - } - // cleanup temp dirs - try { - cleanupTempDirs(); - } catch (Exception e) { - log.error("Got Exception while trying to cleanupTempDirs", e); - } + if (closed.compareAndSet(false, true)) { + // stop bookkeeper service + try { + stopBKCluster(); + } catch (Exception e) { + log.error("Got Exception while trying to stop BKCluster", e); + } + lockedPorts.forEach(PortManager::releaseLockedPort); + lockedPorts.clear(); + // cleanup temp dirs + try { + cleanupTempDirs(); + } catch (Exception e) { + log.error("Got Exception while trying to cleanupTempDirs", e); + } - this.store.close(); + this.store.close(); + } } private File createTempDir(String prefix, String suffix) throws IOException { @@ -224,12 +232,14 @@ private ServerConfiguration newServerConfiguration(int index) throws Exception { } if (clusterConf.clearOldData && dataDir.exists()) { + log.info("Wiping Bookie data directory at {}", dataDir.getAbsolutePath()); cleanDirectory(dataDir); } int port; if (baseConf.isEnableLocalTransport() || !baseConf.getAllowEphemeralPorts() || clusterConf.bkPort == 0) { - port = PortManager.nextFreePort(); + port = PortManager.nextLockedFreePort(); + lockedPorts.add(port); } else { // bk 4.15 cookie validation finds the same ip:port in case of port 0 // and 2nd bookie's cookie validation fails @@ -399,4 +409,8 @@ private static ServerConfiguration setLoopbackInterfaceAndAllowLoopback(ServerCo serverConf.setAllowLoopback(true); return serverConf; } + + public boolean isClosed() { + return closed.get(); + } } diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/LegacyHierarchicalLedgerRangeIterator.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/LegacyHierarchicalLedgerRangeIterator.java index 15b1d561f901c..37e6dc836f254 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/LegacyHierarchicalLedgerRangeIterator.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/LegacyHierarchicalLedgerRangeIterator.java @@ -18,17 +18,21 @@ */ package org.apache.pulsar.metadata.bookkeeper; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.apache.pulsar.metadata.bookkeeper.AbstractMetadataDriver.BLOCKING_CALL_TIMEOUT; import java.io.IOException; import java.util.Iterator; import java.util.List; import java.util.NavigableSet; import java.util.NoSuchElementException; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.meta.LedgerManager; import org.apache.bookkeeper.util.StringUtils; import org.apache.pulsar.metadata.api.MetadataStore; + /** * Hierarchical Ledger Manager which manages ledger meta in zookeeper using 2-level hierarchical znodes. * @@ -67,7 +71,7 @@ public LegacyHierarchicalLedgerRangeIterator(MetadataStore store, String ledgers * @return false if have visited all level1 nodes * @throws InterruptedException/KeeperException if error occurs reading zookeeper children */ - private boolean nextL1Node() throws ExecutionException, InterruptedException { + private boolean nextL1Node() throws ExecutionException, InterruptedException, TimeoutException { l2NodesIter = null; while (l2NodesIter == null) { if (l1NodesIter.hasNext()) { @@ -79,7 +83,8 @@ private boolean nextL1Node() throws ExecutionException, InterruptedException { if (!isLedgerParentNode(curL1Nodes)) { continue; } - List l2Nodes = store.getChildren(ledgersRoot + "/" + curL1Nodes).get(); + List l2Nodes = store.getChildren(ledgersRoot + "/" + curL1Nodes) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); l2NodesIter = l2Nodes.iterator(); if (!l2NodesIter.hasNext()) { l2NodesIter = null; @@ -94,7 +99,8 @@ private synchronized void preload() throws IOException { boolean hasMoreElements = false; try { if (l1NodesIter == null) { - List l1Nodes = store.getChildren(ledgersRoot).get(); + List l1Nodes = store.getChildren(ledgersRoot) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); l1NodesIter = l1Nodes.iterator(); hasMoreElements = nextL1Node(); } else if (l2NodesIter == null || !l2NodesIter.hasNext()) { @@ -102,7 +108,7 @@ private synchronized void preload() throws IOException { } else { hasMoreElements = true; } - } catch (ExecutionException ke) { + } catch (ExecutionException | TimeoutException ke) { throw new IOException("Error preloading next range", ke); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); @@ -156,8 +162,8 @@ LedgerManager.LedgerRange getLedgerRangeByLevel(final String level1, final Strin String nodePath = nodeBuilder.toString(); List ledgerNodes = null; try { - ledgerNodes = store.getChildren(nodePath).get(); - } catch (ExecutionException e) { + ledgerNodes = store.getChildren(nodePath).get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); + } catch (ExecutionException | TimeoutException e) { throw new IOException("Error when get child nodes from zk", e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/LongHierarchicalLedgerRangeIterator.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/LongHierarchicalLedgerRangeIterator.java index 9a36ac53b8991..3b32916e6e7a9 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/LongHierarchicalLedgerRangeIterator.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/LongHierarchicalLedgerRangeIterator.java @@ -24,6 +24,8 @@ import java.util.NoSuchElementException; import java.util.Set; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.meta.LedgerManager; import org.apache.bookkeeper.util.StringUtils; @@ -57,8 +59,9 @@ class LongHierarchicalLedgerRangeIterator implements LedgerManager.LedgerRangeIt */ List getChildrenAt(String path) throws IOException { try { - return store.getChildren(path).get(); - } catch (ExecutionException e) { + return store.getChildren(path) + .get(AbstractMetadataDriver.BLOCKING_CALL_TIMEOUT, TimeUnit.MILLISECONDS); + } catch (ExecutionException | TimeoutException e) { if (log.isDebugEnabled()) { log.debug("Failed to get children at {}", path); } diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLayoutManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLayoutManager.java index a4336b876398a..54675a2f649fe 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLayoutManager.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLayoutManager.java @@ -18,9 +18,12 @@ */ package org.apache.pulsar.metadata.bookkeeper; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.apache.pulsar.metadata.bookkeeper.AbstractMetadataDriver.BLOCKING_CALL_TIMEOUT; import java.io.IOException; import java.util.Optional; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; import lombok.AccessLevel; import lombok.Getter; import org.apache.bookkeeper.bookie.BookieException; @@ -30,7 +33,8 @@ import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; -class PulsarLayoutManager implements LayoutManager { + +public class PulsarLayoutManager implements LayoutManager { @Getter(AccessLevel.PACKAGE) private final MetadataStoreExtended store; @@ -40,7 +44,7 @@ class PulsarLayoutManager implements LayoutManager { private final String layoutPath; - PulsarLayoutManager(MetadataStoreExtended store, String ledgersRootPath) { + public PulsarLayoutManager(MetadataStoreExtended store, String ledgersRootPath) { this.ledgersRootPath = ledgersRootPath; this.store = store; this.layoutPath = ledgersRootPath + "/" + BookKeeperConstants.LAYOUT_ZNODE; @@ -49,14 +53,14 @@ class PulsarLayoutManager implements LayoutManager { @Override public LedgerLayout readLedgerLayout() throws IOException { try { - byte[] layoutData = store.get(layoutPath).get() + byte[] layoutData = store.get(layoutPath).get(BLOCKING_CALL_TIMEOUT, MILLISECONDS) .orElseThrow(() -> new BookieException.MetadataStoreException("Layout node not found")) .getValue(); return LedgerLayout.parseLayout(layoutData); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new IOException(e); - } catch (BookieException | ExecutionException e) { + } catch (BookieException | ExecutionException | TimeoutException e) { throw new IOException(e); } } @@ -66,10 +70,13 @@ public void storeLedgerLayout(LedgerLayout ledgerLayout) throws IOException { try { byte[] layoutData = ledgerLayout.serialize(); - store.put(layoutPath, layoutData, Optional.of(-1L)).get(); + store.put(layoutPath, layoutData, Optional.of(-1L)) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new IOException(e); + } catch (TimeoutException e) { + throw new IOException(e); } catch (ExecutionException e) { if (e.getCause() instanceof MetadataStoreException.BadVersionException) { throw new LedgerLayoutExistsException(e); @@ -82,11 +89,12 @@ public void storeLedgerLayout(LedgerLayout ledgerLayout) throws IOException { @Override public void deleteLedgerLayout() throws IOException { try { - store.delete(layoutPath, Optional.empty()).get(); + store.delete(layoutPath, Optional.empty()) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new IOException(e); - } catch (ExecutionException e) { + } catch (ExecutionException | TimeoutException e) { throw new IOException(e); } } diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerAuditorManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerAuditorManager.java index bc35380fec19c..44870ed47f05b 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerAuditorManager.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerAuditorManager.java @@ -27,17 +27,19 @@ import org.apache.pulsar.metadata.api.coordination.LeaderElection; import org.apache.pulsar.metadata.api.coordination.LeaderElectionState; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.apache.pulsar.metadata.api.extended.SessionEvent; import org.apache.pulsar.metadata.coordination.impl.CoordinationServiceImpl; @Slf4j -class PulsarLedgerAuditorManager implements LedgerAuditorManager { +public class PulsarLedgerAuditorManager implements LedgerAuditorManager { - private static final String ELECTION_PATH = "leader"; + public static final String ELECTION_PATH = "leader"; private final CoordinationService coordinationService; private final LeaderElection leaderElection; private LeaderElectionState leaderElectionState; private String bookieId; + private boolean sessionExpired = false; PulsarLedgerAuditorManager(MetadataStoreExtended store, String ledgersRoot) { this.coordinationService = new CoordinationServiceImpl(store); @@ -47,6 +49,14 @@ class PulsarLedgerAuditorManager implements LedgerAuditorManager { this.leaderElection = coordinationService.getLeaderElection(String.class, electionPath, this::handleStateChanges); this.leaderElectionState = LeaderElectionState.NoLeader; + store.registerSessionListener(event -> { + if (SessionEvent.SessionLost == event) { + synchronized (this) { + sessionExpired = true; + notifyAll(); + } + } + }); } private void handleStateChanges(LeaderElectionState state) { @@ -71,6 +81,9 @@ public void tryToBecomeAuditor(String bookieId, Consumer listener) while (true) { try { synchronized (this) { + if (sessionExpired) { + throw new IllegalStateException("Zookeeper session expired, give up to become auditor."); + } if (leaderElectionState == LeaderElectionState.Leading) { return; } else { diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerManager.java index 59452a3d54db3..b003c656353c0 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerManager.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerManager.java @@ -377,7 +377,7 @@ public void close() throws IOException { } } - private String getLedgerPath(long ledgerId) { + public String getLedgerPath(long ledgerId) { return this.ledgerRootPath + StringUtils.getHybridHierarchicalLedgerPath(ledgerId); } diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerManagerFactory.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerManagerFactory.java index 1b229757c9c30..bfcbf0b22d924 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerManagerFactory.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerManagerFactory.java @@ -19,8 +19,12 @@ package org.apache.pulsar.metadata.bookkeeper; import static com.google.common.base.Preconditions.checkArgument; +import static org.apache.pulsar.metadata.bookkeeper.AbstractMetadataDriver.BLOCKING_CALL_TIMEOUT; import java.io.IOException; import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.conf.AbstractConfiguration; @@ -110,7 +114,13 @@ public boolean validateAndNukeExistingCluster(AbstractConfiguration conf, * before proceeding with nuking existing cluster, make sure there * are no unexpected nodes under ledgersRootPath */ - List ledgersRootPathChildrenList = store.getChildren(ledgerRootPath).join(); + final List ledgersRootPathChildrenList; + try { + ledgersRootPathChildrenList = store.getChildren(ledgerRootPath) + .get(BLOCKING_CALL_TIMEOUT, TimeUnit.MILLISECONDS); + } catch (ExecutionException | TimeoutException e) { + throw new IOException(e); + } for (String ledgersRootPathChildren : ledgersRootPathChildrenList) { if ((!AbstractZkLedgerManager.isSpecialZnode(ledgersRootPathChildren)) && (!ledgerManager.isLedgerParentNode(ledgersRootPathChildren))) { @@ -124,18 +134,34 @@ public boolean validateAndNukeExistingCluster(AbstractConfiguration conf, format(conf, layoutManager); // now delete all the special nodes recursively - for (String ledgersRootPathChildren : store.getChildren(ledgerRootPath).join()) { - if (AbstractZkLedgerManager.isSpecialZnode(ledgersRootPathChildren)) { - store.deleteRecursive(ledgerRootPath + "/" + ledgersRootPathChildren).join(); + final List ledgersRootPathChildren; + try { + ledgersRootPathChildren = store.getChildren(ledgerRootPath) + .get(BLOCKING_CALL_TIMEOUT, TimeUnit.MILLISECONDS); + } catch (ExecutionException | TimeoutException e) { + throw new IOException(e); + } + for (String ledgersRootPathChild :ledgersRootPathChildren) { + if (AbstractZkLedgerManager.isSpecialZnode(ledgersRootPathChild)) { + try { + store.deleteRecursive(ledgerRootPath + "/" + ledgersRootPathChild) + .get(BLOCKING_CALL_TIMEOUT, TimeUnit.MILLISECONDS); + } catch (ExecutionException | TimeoutException e) { + throw new IOException(e); + } } else { log.error("Found unexpected node : {} under ledgersRootPath : {} so exiting nuke operation", - ledgersRootPathChildren, ledgerRootPath); + ledgersRootPathChild, ledgerRootPath); return false; } } // finally deleting the ledgers rootpath - store.deleteRecursive(ledgerRootPath).join(); + try { + store.deleteRecursive(ledgerRootPath).get(BLOCKING_CALL_TIMEOUT, TimeUnit.MILLISECONDS); + } catch (ExecutionException | TimeoutException e) { + throw new IOException(e); + } log.info("Successfully nuked existing cluster"); return true; diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java index cf8ff208c94b6..2673328b81139 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java @@ -19,16 +19,20 @@ package org.apache.pulsar.metadata.bookkeeper; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.bookkeeper.proto.DataFormats.CheckAllLedgersFormat; import static org.apache.bookkeeper.proto.DataFormats.LedgerRereplicationLayoutFormat; import static org.apache.bookkeeper.proto.DataFormats.LockDataFormat; import static org.apache.bookkeeper.proto.DataFormats.PlacementPolicyCheckFormat; import static org.apache.bookkeeper.proto.DataFormats.ReplicasCheckFormat; import static org.apache.bookkeeper.proto.DataFormats.UnderreplicatedLedgerFormat; +import static org.apache.pulsar.metadata.bookkeeper.AbstractMetadataDriver.BLOCKING_CALL_TIMEOUT; +import com.google.common.base.Joiner; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.TextFormat; import java.net.UnknownHostException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; @@ -41,6 +45,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -61,6 +66,8 @@ import org.apache.pulsar.metadata.api.NotificationType; import org.apache.pulsar.metadata.api.extended.CreateOption; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.apache.pulsar.metadata.impl.ZKMetadataStore; +import org.apache.zookeeper.KeeperException; @Slf4j public class PulsarLedgerUnderreplicationManager implements LedgerUnderreplicationManager { @@ -98,14 +105,17 @@ long getLedgerNodeVersion() { private final String urLockPath; private final String layoutPath; private final String lostBookieRecoveryDelayPath; + private final String replicationDisablePath; private final String checkAllLedgersCtimePath; private final String placementPolicyCheckCtimePath; private final String replicasCheckCtimePath; private final MetadataStoreExtended store; - private BookkeeperInternalCallbacks.GenericCallback replicationEnabledListener; - private BookkeeperInternalCallbacks.GenericCallback lostBookieRecoveryDelayListener; + private final List> replicationEnabledCallbacks = + new ArrayList<>(); + private final List> lostBookieRecoveryDelayCallbacks = + new ArrayList<>(); private static class PulsarUnderreplicatedLedger extends UnderreplicatedLedger { PulsarUnderreplicatedLedger(long ledgerId) { @@ -132,6 +142,7 @@ public PulsarLedgerUnderreplicationManager(AbstractConfiguration conf, Metada urLedgerPath = basePath + BookKeeperConstants.DEFAULT_ZK_LEDGERS_ROOT_PATH; urLockPath = basePath + '/' + BookKeeperConstants.UNDER_REPLICATION_LOCK; lostBookieRecoveryDelayPath = basePath + '/' + BookKeeperConstants.LOSTBOOKIERECOVERYDELAY_NODE; + replicationDisablePath = basePath + '/' + BookKeeperConstants.DISABLE_NODE; checkAllLedgersCtimePath = basePath + '/' + BookKeeperConstants.CHECK_ALL_LEDGERS_CTIME; placementPolicyCheckCtimePath = basePath + '/' + BookKeeperConstants.PLACEMENT_POLICY_CHECK_CTIME; replicasCheckCtimePath = basePath + '/' + BookKeeperConstants.REPLICAS_CHECK_CTIME; @@ -225,17 +236,34 @@ private void handleNotification(Notification n) { synchronized (this) { // Notify that there were some changes on the under-replicated z-nodes notifyAll(); - - if (n.getType() == NotificationType.Deleted) { - if (n.getPath().equals(basePath + '/' + BookKeeperConstants.DISABLE_NODE)) { - log.info("LedgerReplication is enabled externally through MetadataStore, " - + "since DISABLE_NODE ZNode is deleted"); - if (replicationEnabledListener != null) { - replicationEnabledListener.operationComplete(0, null); + if (lostBookieRecoveryDelayPath.equals(n.getPath())) { + final List> callbackList; + synchronized (lostBookieRecoveryDelayCallbacks) { + callbackList = new ArrayList<>(lostBookieRecoveryDelayCallbacks); + lostBookieRecoveryDelayCallbacks.clear(); + } + for (BookkeeperInternalCallbacks.GenericCallback callback : callbackList) { + try { + callback.operationComplete(0, null); + } catch (Exception e) { + log.warn("lostBookieRecoveryDelayCallbacks handle error", e); } - } else if (n.getPath().equals(lostBookieRecoveryDelayPath)) { - if (lostBookieRecoveryDelayListener != null) { - lostBookieRecoveryDelayListener.operationComplete(0, null); + } + return; + } + if (replicationDisablePath.equals(n.getPath()) && n.getType() == NotificationType.Deleted) { + log.info("LedgerReplication is enabled externally through MetadataStore, " + + "since DISABLE_NODE ZNode is deleted"); + final List> callbackList; + synchronized (replicationEnabledCallbacks) { + callbackList = new ArrayList<>(replicationEnabledCallbacks); + replicationEnabledCallbacks.clear(); + } + for (BookkeeperInternalCallbacks.GenericCallback callback : callbackList) { + try { + callback.operationComplete(0, null); + } catch (Exception e) { + log.warn("replicationEnabledCallbacks handle error", e); } } } @@ -249,7 +277,7 @@ public UnderreplicatedLedger getLedgerUnreplicationInfo(long ledgerId) try { String path = getUrLedgerPath(ledgerId); - Optional optRes = store.get(path).get(); + Optional optRes = store.get(path).get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); if (!optRes.isPresent()) { if (log.isDebugEnabled()) { log.debug("Ledger: {} is not marked underreplicated", ledgerId); @@ -270,7 +298,7 @@ public UnderreplicatedLedger getLedgerUnreplicationInfo(long ledgerId) underreplicatedLedger.setCtime(ctime); underreplicatedLedger.setReplicaList(replicaList); return underreplicatedLedger; - } catch (ExecutionException ee) { + } catch (ExecutionException | TimeoutException ee) { throw new ReplicationException.UnavailableException("Error contacting with metadata store", ee); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); @@ -374,14 +402,16 @@ private void handleLedgerUnderreplicatedAlreadyMarked(final String path, public void acquireUnderreplicatedLedger(long ledgerId) throws ReplicationException { try { internalAcquireUnderreplicatedLedger(ledgerId); - } catch (ExecutionException | InterruptedException e) { + } catch (ExecutionException | TimeoutException | InterruptedException e) { throw new ReplicationException.UnavailableException("Failed to acuire under-replicated ledger", e); } } - private void internalAcquireUnderreplicatedLedger(long ledgerId) throws ExecutionException, InterruptedException { + private void internalAcquireUnderreplicatedLedger(long ledgerId) throws ExecutionException, + InterruptedException, TimeoutException { String lockPath = getUrLedgerLockPath(urLockPath, ledgerId); - store.put(lockPath, LOCK_DATA, Optional.of(-1L), EnumSet.of(CreateOption.Ephemeral)).get(); + store.put(lockPath, LOCK_DATA, Optional.of(-1L), EnumSet.of(CreateOption.Ephemeral)) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); } @Override @@ -392,7 +422,34 @@ public void markLedgerReplicated(long ledgerId) throws ReplicationException.Unav try { Lock l = heldLocks.get(ledgerId); if (l != null) { - store.delete(getUrLedgerPath(ledgerId), Optional.of(l.getLedgerNodeVersion())).get(); + store.delete(getUrLedgerPath(ledgerId), Optional.of(l.getLedgerNodeVersion())) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); + if (store instanceof ZKMetadataStore) { + try { + // clean up the hierarchy + String[] parts = getUrLedgerPath(ledgerId).split("/"); + for (int i = 1; i <= 4; i++) { + String[] p = Arrays.copyOf(parts, parts.length - i); + String path = Joiner.on("/").join(p); + Optional getResult = store.get(path).get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); + if (getResult.isPresent()) { + store.delete(path, Optional.of(getResult.get().getStat().getVersion())) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); + } + } + } catch (ExecutionException ee) { + // This can happen when cleaning up the hierarchy. + // It's safe to ignore, it simply means another + // ledger in the same hierarchy has been marked as + // underreplicated. + if (ee.getCause() instanceof MetadataStoreException && ee.getCause().getCause() + instanceof KeeperException.NotEmptyException) { + //do nothing. + } else { + log.warn("Error deleting underrepcalited ledger parent node", ee); + } + } + } } } catch (ExecutionException ee) { if (ee.getCause() instanceof MetadataStoreException.NotFoundException) { @@ -405,6 +462,8 @@ public void markLedgerReplicated(long ledgerId) throws ReplicationException.Unav log.error("Error deleting underreplicated ledger node", ee); throw new ReplicationException.UnavailableException("Error contacting metadata store", ee); } + } catch (TimeoutException ex) { + throw new ReplicationException.UnavailableException("Error contacting metadata store", ex); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); throw new ReplicationException.UnavailableException("Interrupted while contacting metadata store", ie); @@ -445,7 +504,7 @@ public boolean hasNext() { while (queue.size() > 0 && curBatch.size() == 0) { String parent = queue.remove(); try { - for (String c : store.getChildren(parent).get()) { + for (String c : store.getChildren(parent).get(BLOCKING_CALL_TIMEOUT, MILLISECONDS)) { String child = parent + "/" + c; if (c.startsWith("urL")) { long ledgerId = getLedgerId(child); @@ -479,21 +538,23 @@ public UnderreplicatedLedger next() { } private long getLedgerToRereplicateFromHierarchy(String parent, long depth) - throws ExecutionException, InterruptedException { + throws ExecutionException, InterruptedException, TimeoutException { if (depth == 4) { - List children = new ArrayList<>(store.getChildren(parent).get()); + List children = new ArrayList<>(store.getChildren(parent) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS)); Collections.shuffle(children); while (!children.isEmpty()) { String tryChild = children.get(0); try { - List locks = store.getChildren(urLockPath).get(); + List locks = store.getChildren(urLockPath).get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); if (locks.contains(tryChild)) { children.remove(tryChild); continue; } - Optional optRes = store.get(parent + "/" + tryChild).get(); + Optional optRes = store.get(parent + "/" + tryChild) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); if (!optRes.isPresent()) { if (log.isDebugEnabled()) { log.debug("{}/{} doesn't exist", parent, tryChild); @@ -522,7 +583,7 @@ private long getLedgerToRereplicateFromHierarchy(String parent, long depth) return -1; } - List children = new ArrayList<>(store.getChildren(parent).join()); + List children = new ArrayList<>(store.getChildren(parent).get(BLOCKING_CALL_TIMEOUT, MILLISECONDS)); Collections.shuffle(children); while (children.size() > 0) { @@ -545,7 +606,7 @@ public long pollLedgerToRereplicate() throws ReplicationException.UnavailableExc } try { return getLedgerToRereplicateFromHierarchy(urLedgerPath, 0); - } catch (ExecutionException ee) { + } catch (ExecutionException | TimeoutException ee) { throw new ReplicationException.UnavailableException("Error contacting metadata store", ee); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); @@ -571,7 +632,7 @@ public long getLedgerToRereplicate() throws ReplicationException.UnavailableExce // nothing found, wait for a watcher to trigger this.wait(1000); } - } catch (ExecutionException ee) { + } catch (ExecutionException | TimeoutException ee) { throw new ReplicationException.UnavailableException("Error contacting metadata store", ee); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); @@ -597,7 +658,8 @@ public void releaseUnderreplicatedLedger(long ledgerId) throws ReplicationExcept try { Lock l = heldLocks.get(ledgerId); if (l != null) { - store.delete(l.getLockPath(), Optional.empty()).get(); + store.delete(l.getLockPath(), Optional.empty()) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); } } catch (ExecutionException ee) { if (ee.getCause() instanceof MetadataStoreException.NotFoundException) { @@ -606,6 +668,8 @@ public void releaseUnderreplicatedLedger(long ledgerId) throws ReplicationExcept log.error("Error deleting underreplicated ledger lock", ee); throw new ReplicationException.UnavailableException("Error contacting metadata store", ee); } + } catch (TimeoutException ex) { + throw new ReplicationException.UnavailableException("Error contacting metadata store", ex); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); throw new ReplicationException.UnavailableException("Interrupted while connecting metadata store", ie); @@ -620,7 +684,8 @@ public void close() throws ReplicationException.UnavailableException { } try { for (Map.Entry e : heldLocks.entrySet()) { - store.delete(e.getValue().getLockPath(), Optional.empty()).get(); + store.delete(e.getValue().getLockPath(), Optional.empty()) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); } } catch (ExecutionException ee) { if (ee.getCause() instanceof MetadataStoreException.NotFoundException) { @@ -629,6 +694,8 @@ public void close() throws ReplicationException.UnavailableException { log.error("Error deleting underreplicated ledger lock", ee); throw new ReplicationException.UnavailableException("Error contacting metadata store", ee); } + } catch (TimeoutException ex) { + throw new ReplicationException.UnavailableException("Error contacting metadata store", ex); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); throw new ReplicationException.UnavailableException("Interrupted while connecting metadata store", ie); @@ -642,10 +709,10 @@ public void disableLedgerReplication() log.debug("disableLedegerReplication()"); } try { - String path = basePath + '/' + BookKeeperConstants.DISABLE_NODE; - store.put(path, "".getBytes(UTF_8), Optional.of(-1L)).get(); + store.put(replicationDisablePath, "".getBytes(UTF_8), Optional.of(-1L)) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); log.info("Auto ledger re-replication is disabled!"); - } catch (ExecutionException ee) { + } catch (ExecutionException | TimeoutException ee) { log.error("Exception while stopping auto ledger re-replication", ee); throw new ReplicationException.UnavailableException( "Exception while stopping auto ledger re-replication", ee); @@ -663,9 +730,10 @@ public void enableLedgerReplication() log.debug("enableLedegerReplication()"); } try { - store.delete(basePath + '/' + BookKeeperConstants.DISABLE_NODE, Optional.empty()).get(); + store.delete(replicationDisablePath, Optional.empty()) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); log.info("Resuming automatic ledger re-replication"); - } catch (ExecutionException ee) { + } catch (ExecutionException | TimeoutException ee) { log.error("Exception while resuming ledger replication", ee); throw new ReplicationException.UnavailableException( "Exception while resuming auto ledger re-replication", ee); @@ -683,8 +751,9 @@ public boolean isLedgerReplicationEnabled() log.debug("isLedgerReplicationEnabled()"); } try { - return !store.exists(basePath + '/' + BookKeeperConstants.DISABLE_NODE).get(); - } catch (ExecutionException ee) { + return !store.exists(replicationDisablePath) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); + } catch (ExecutionException | TimeoutException ee) { log.error("Error while checking the state of " + "ledger re-replication", ee); throw new ReplicationException.UnavailableException( @@ -702,19 +771,18 @@ public void notifyLedgerReplicationEnabled(final BookkeeperInternalCallbacks.Gen if (log.isDebugEnabled()) { log.debug("notifyLedgerReplicationEnabled()"); } - - synchronized (this) { - replicationEnabledListener = cb; + synchronized (replicationEnabledCallbacks) { + replicationEnabledCallbacks.add(cb); } - try { - if (!store.exists(basePath + '/' + BookKeeperConstants.DISABLE_NODE).get()) { + if (!store.exists(replicationDisablePath) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS)) { log.info("LedgerReplication is enabled externally through metadata store, " + "since DISABLE_NODE node is deleted"); cb.operationComplete(0, null); return; } - } catch (ExecutionException ee) { + } catch (ExecutionException | TimeoutException ee) { log.error("Error while checking the state of " + "ledger re-replication", ee); throw new ReplicationException.UnavailableException( @@ -732,7 +800,7 @@ public void notifyLedgerReplicationEnabled(final BookkeeperInternalCallbacks.Gen @Override public boolean isLedgerBeingReplicated(long ledgerId) throws ReplicationException { try { - return store.exists(getUrLedgerLockPath(urLockPath, ledgerId)).get(); + return store.exists(getUrLedgerLockPath(urLockPath, ledgerId)).get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); } catch (Exception e) { throw new ReplicationException.UnavailableException("Failed to check if ledger is beinge replicated", e); } @@ -744,7 +812,7 @@ public boolean initializeLostBookieRecoveryDelay(int lostBookieRecoveryDelay) th log.debug("initializeLostBookieRecoveryDelay()"); try { store.put(lostBookieRecoveryDelayPath, Integer.toString(lostBookieRecoveryDelay).getBytes(UTF_8), - Optional.of(-1L)).get(); + Optional.of(-1L)).get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); } catch (ExecutionException ee) { if (ee.getCause() instanceof MetadataStoreException.BadVersionException) { log.info("lostBookieRecoveryDelay node is already present, so using " @@ -754,6 +822,9 @@ public boolean initializeLostBookieRecoveryDelay(int lostBookieRecoveryDelay) th log.error("Error while initializing LostBookieRecoveryDelay", ee); throw new ReplicationException.UnavailableException("Error contacting zookeeper", ee); } + } catch (TimeoutException ex) { + log.error("Error while initializing LostBookieRecoveryDelay", ex); + throw new ReplicationException.UnavailableException("Error contacting zookeeper", ex); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); throw new ReplicationException.UnavailableException("Interrupted while contacting zookeeper", ie); @@ -767,9 +838,9 @@ public void setLostBookieRecoveryDelay(int lostBookieRecoveryDelay) throws log.debug("setLostBookieRecoveryDelay()"); try { store.put(lostBookieRecoveryDelayPath, Integer.toString(lostBookieRecoveryDelay).getBytes(UTF_8), - Optional.empty()).get(); + Optional.empty()).get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); - } catch (ExecutionException ee) { + } catch (ExecutionException | TimeoutException ee) { log.error("Error while setting LostBookieRecoveryDelay ", ee); throw new ReplicationException.UnavailableException("Error contacting zookeeper", ee); } catch (InterruptedException ie) { @@ -782,9 +853,10 @@ public void setLostBookieRecoveryDelay(int lostBookieRecoveryDelay) throws public int getLostBookieRecoveryDelay() throws ReplicationException.UnavailableException { log.debug("getLostBookieRecoveryDelay()"); try { - byte[] data = store.get(lostBookieRecoveryDelayPath).get().get().getValue(); + byte[] data = store.get(lostBookieRecoveryDelayPath).get(BLOCKING_CALL_TIMEOUT, MILLISECONDS) + .get().getValue(); return Integer.parseInt(new String(data, UTF_8)); - } catch (ExecutionException ee) { + } catch (ExecutionException | TimeoutException ee) { log.error("Error while getting LostBookieRecoveryDelay ", ee); throw new ReplicationException.UnavailableException("Error contacting zookeeper", ee); } catch (InterruptedException ie) { @@ -797,16 +869,16 @@ public int getLostBookieRecoveryDelay() throws ReplicationException.UnavailableE public void notifyLostBookieRecoveryDelayChanged(BookkeeperInternalCallbacks.GenericCallback cb) throws ReplicationException.UnavailableException { log.debug("notifyLostBookieRecoveryDelayChanged()"); - synchronized (this) { - lostBookieRecoveryDelayListener = cb; + synchronized (lostBookieRecoveryDelayCallbacks) { + lostBookieRecoveryDelayCallbacks.add(cb); } try { - if (!store.exists(lostBookieRecoveryDelayPath).get()) { + if (!store.exists(lostBookieRecoveryDelayPath).get(BLOCKING_CALL_TIMEOUT, MILLISECONDS)) { cb.operationComplete(0, null); return; } - } catch (ExecutionException ee) { + } catch (ExecutionException | TimeoutException ee) { log.error("Error while checking the state of lostBookieRecoveryDelay", ee); throw new ReplicationException.UnavailableException("Error contacting zookeeper", ee); } catch (InterruptedException ie) { @@ -820,7 +892,8 @@ public String getReplicationWorkerIdRereplicatingLedger(long ledgerId) throws ReplicationException.UnavailableException { try { - Optional optRes = store.get(getUrLedgerLockPath(urLockPath, ledgerId)).get(); + Optional optRes = store.get(getUrLedgerLockPath(urLockPath, ledgerId)) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); if (!optRes.isPresent()) { // this is ok. return null; @@ -831,7 +904,7 @@ public String getReplicationWorkerIdRereplicatingLedger(long ledgerId) TextFormat.merge(new String(lockData, UTF_8), lockDataBuilder); LockDataFormat lock = lockDataBuilder.build(); return lock.getBookieId(); - } catch (ExecutionException e) { + } catch (ExecutionException | TimeoutException e) { log.error("Error while getting ReplicationWorkerId rereplicating Ledger", e); throw new ReplicationException.UnavailableException( "Error while getting ReplicationWorkerId rereplicating Ledger", e); @@ -855,8 +928,9 @@ public void setCheckAllLedgersCTime(long checkAllLedgersCTime) throws Replicatio builder.setCheckAllLedgersCTime(checkAllLedgersCTime); byte[] checkAllLedgersFormatByteArray = builder.build().toByteArray(); - store.put(checkAllLedgersCtimePath, checkAllLedgersFormatByteArray, Optional.empty()).get(); - } catch (ExecutionException ee) { + store.put(checkAllLedgersCtimePath, checkAllLedgersFormatByteArray, Optional.empty()) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); + } catch (ExecutionException | TimeoutException ee) { throw new ReplicationException.UnavailableException("Error contacting zookeeper", ee); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); @@ -870,7 +944,7 @@ public long getCheckAllLedgersCTime() throws ReplicationException.UnavailableExc log.debug("setCheckAllLedgersCTime"); } try { - Optional optRes = store.get(checkAllLedgersCtimePath).get(); + Optional optRes = store.get(checkAllLedgersCtimePath).get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); if (!optRes.isPresent()) { log.warn("checkAllLedgersCtimeZnode is not yet available"); return -1; @@ -879,7 +953,7 @@ public long getCheckAllLedgersCTime() throws ReplicationException.UnavailableExc CheckAllLedgersFormat checkAllLedgersFormat = CheckAllLedgersFormat.parseFrom(data); return checkAllLedgersFormat.hasCheckAllLedgersCTime() ? checkAllLedgersFormat.getCheckAllLedgersCTime() : -1; - } catch (ExecutionException ee) { + } catch (ExecutionException | TimeoutException ee) { throw new ReplicationException.UnavailableException("Error contacting zookeeper", ee); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); @@ -899,8 +973,9 @@ public void setPlacementPolicyCheckCTime(long placementPolicyCheckCTime) throws PlacementPolicyCheckFormat.Builder builder = PlacementPolicyCheckFormat.newBuilder(); builder.setPlacementPolicyCheckCTime(placementPolicyCheckCTime); byte[] placementPolicyCheckFormatByteArray = builder.build().toByteArray(); - store.put(placementPolicyCheckCtimePath, placementPolicyCheckFormatByteArray, Optional.empty()).get(); - } catch (ExecutionException ke) { + store.put(placementPolicyCheckCtimePath, placementPolicyCheckFormatByteArray, Optional.empty()) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); + } catch (ExecutionException | TimeoutException ke) { throw new ReplicationException.UnavailableException("Error contacting zookeeper", ke); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); @@ -914,7 +989,8 @@ public long getPlacementPolicyCheckCTime() throws ReplicationException.Unavailab log.debug("getPlacementPolicyCheckCTime"); } try { - Optional optRes = store.get(placementPolicyCheckCtimePath).get(); + Optional optRes = store.get(placementPolicyCheckCtimePath) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); if (!optRes.isPresent()) { log.warn("placementPolicyCheckCtimeZnode is not yet available"); return -1; @@ -923,7 +999,7 @@ public long getPlacementPolicyCheckCTime() throws ReplicationException.Unavailab PlacementPolicyCheckFormat placementPolicyCheckFormat = PlacementPolicyCheckFormat.parseFrom(data); return placementPolicyCheckFormat.hasPlacementPolicyCheckCTime() ? placementPolicyCheckFormat.getPlacementPolicyCheckCTime() : -1; - } catch (ExecutionException ee) { + } catch (ExecutionException | TimeoutException ee) { throw new ReplicationException.UnavailableException("Error contacting zookeeper", ee); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); @@ -939,11 +1015,12 @@ public void setReplicasCheckCTime(long replicasCheckCTime) throws ReplicationExc ReplicasCheckFormat.Builder builder = ReplicasCheckFormat.newBuilder(); builder.setReplicasCheckCTime(replicasCheckCTime); byte[] replicasCheckFormatByteArray = builder.build().toByteArray(); - store.put(replicasCheckCtimePath, replicasCheckFormatByteArray, Optional.empty()).get(); + store.put(replicasCheckCtimePath, replicasCheckFormatByteArray, Optional.empty()) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); if (log.isDebugEnabled()) { log.debug("setReplicasCheckCTime completed successfully"); } - } catch (ExecutionException ke) { + } catch (ExecutionException | TimeoutException ke) { throw new ReplicationException.UnavailableException("Error contacting zookeeper", ke); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); @@ -954,7 +1031,8 @@ public void setReplicasCheckCTime(long replicasCheckCTime) throws ReplicationExc @Override public long getReplicasCheckCTime() throws ReplicationException.UnavailableException { try { - Optional optRes = store.get(replicasCheckCtimePath).get(); + Optional optRes = store.get(replicasCheckCtimePath) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); if (!optRes.isPresent()) { log.warn("placementPolicyCheckCtimeZnode is not yet available"); return -1; @@ -965,7 +1043,7 @@ public long getReplicasCheckCTime() throws ReplicationException.UnavailableExcep log.debug("getReplicasCheckCTime completed successfully"); } return replicasCheckFormat.hasReplicasCheckCTime() ? replicasCheckFormat.getReplicasCheckCTime() : -1; - } catch (ExecutionException ee) { + } catch (ExecutionException | TimeoutException ee) { throw new ReplicationException.UnavailableException("Error contacting zookeeper", ee); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); @@ -978,11 +1056,13 @@ public long getReplicasCheckCTime() throws ReplicationException.UnavailableExcep @Override public void notifyUnderReplicationLedgerChanged(BookkeeperInternalCallbacks.GenericCallback cb) throws ReplicationException.UnavailableException { - log.debug("notifyUnderReplicationLedgerChanged()"); - store.registerListener(e -> { - if (e.getType() == NotificationType.Deleted && ID_EXTRACTION_PATTERN.matcher(e.getPath()).find()) { - cb.operationComplete(0, null); - } - }); + //The store listener callback executor is metadata-store executor, + //in cb.operationComplete(0, null), it will get all underreplication ledgers from metadata-store, it's sync + //operation. So it's a deadlock. +// store.registerListener(e -> { +// if (e.getType() == NotificationType.Deleted && ID_EXTRACTION_PATTERN.matcher(e.getPath()).find()) { +// cb.operationComplete(0, null); +// } +// }); } } diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java index a32625926e72c..be945d988fb88 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java @@ -18,13 +18,18 @@ */ package org.apache.pulsar.metadata.bookkeeper; +import static java.util.concurrent.CompletableFuture.completedFuture; +import static java.util.concurrent.CompletableFuture.failedFuture; import static org.apache.bookkeeper.util.BookKeeperConstants.AVAILABLE_NODE; import static org.apache.bookkeeper.util.BookKeeperConstants.COOKIE_NODE; import static org.apache.bookkeeper.util.BookKeeperConstants.READONLY; +import static org.apache.pulsar.common.util.FutureUtil.Sequencer; +import static org.apache.pulsar.common.util.FutureUtil.waitForAll; import io.netty.util.concurrent.DefaultThreadFactory; import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -42,34 +47,39 @@ import org.apache.bookkeeper.versioning.Version; import org.apache.bookkeeper.versioning.Versioned; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.metadata.api.CacheGetResult; import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.Notification; -import org.apache.pulsar.metadata.api.NotificationType; +import org.apache.pulsar.metadata.api.extended.SessionEvent; +import org.apache.pulsar.metadata.impl.AbstractMetadataStore; @Slf4j public class PulsarRegistrationClient implements RegistrationClient { - private final MetadataStore store; + private final AbstractMetadataStore store; private final String ledgersRootPath; // registration paths private final String bookieRegistrationPath; private final String bookieAllRegistrationPath; private final String bookieReadonlyRegistrationPath; - - private final ConcurrentHashMap> bookieServiceInfoCache = - new ConcurrentHashMap(); private final Set writableBookiesWatchers = new CopyOnWriteArraySet<>(); private final Set readOnlyBookiesWatchers = new CopyOnWriteArraySet<>(); private final MetadataCache bookieServiceInfoMetadataCache; private final ScheduledExecutorService executor; + private final Map> writableBookieInfo; + private final Map> readOnlyBookieInfo; + private final FutureUtil.Sequencer sequencer; + private SessionEvent lastMetadataSessionEvent; public PulsarRegistrationClient(MetadataStore store, String ledgersRootPath) { - this.store = store; + this.store = (AbstractMetadataStore) store; this.ledgersRootPath = ledgersRootPath; this.bookieServiceInfoMetadataCache = store.getMetadataCache(BookieServiceInfoSerde.INSTANCE); - + this.sequencer = Sequencer.create(); + this.writableBookieInfo = new ConcurrentHashMap<>(); + this.readOnlyBookieInfo = new ConcurrentHashMap<>(); // Following Bookie Network Address Changes is an expensive operation // as it requires additional ZooKeeper watches // we can disable this feature, in case the BK cluster has only @@ -77,11 +87,11 @@ public PulsarRegistrationClient(MetadataStore store, this.bookieRegistrationPath = ledgersRootPath + "/" + AVAILABLE_NODE; this.bookieAllRegistrationPath = ledgersRootPath + "/" + COOKIE_NODE; this.bookieReadonlyRegistrationPath = this.bookieRegistrationPath + "/" + READONLY; - this.executor = Executors .newSingleThreadScheduledExecutor(new DefaultThreadFactory("pulsar-registration-client")); store.registerListener(this::updatedBookies); + this.store.registerSessionListener(this::refreshBookies); } @Override @@ -89,40 +99,79 @@ public void close() { executor.shutdownNow(); } + private void refreshBookies(SessionEvent sessionEvent) { + lastMetadataSessionEvent = sessionEvent; + if (!SessionEvent.Reconnected.equals(sessionEvent) && !SessionEvent.SessionReestablished.equals(sessionEvent)){ + return; + } + // Clean caches. + store.invalidateCaches(bookieRegistrationPath, bookieAllRegistrationPath, bookieReadonlyRegistrationPath); + bookieServiceInfoMetadataCache.invalidateAll(); + // Refresh caches of the listeners. + getReadOnlyBookies().thenAccept(bookies -> + readOnlyBookiesWatchers.forEach(w -> executor.execute(() -> w.onBookiesChanged(bookies)))); + getWritableBookies().thenAccept(bookies -> + writableBookiesWatchers.forEach(w -> executor.execute(() -> w.onBookiesChanged(bookies)))); + } + @Override public CompletableFuture>> getWritableBookies() { - return getChildren(bookieRegistrationPath); + return getBookiesThenFreshCache(bookieRegistrationPath); } @Override public CompletableFuture>> getAllBookies() { // this method is meant to return all the known bookies, even the bookies // that are not in a running state - return getChildren(bookieAllRegistrationPath); + return getBookiesThenFreshCache(bookieAllRegistrationPath); } @Override public CompletableFuture>> getReadOnlyBookies() { - return getChildren(bookieReadonlyRegistrationPath); + return getBookiesThenFreshCache(bookieReadonlyRegistrationPath); } - private CompletableFuture>> getChildren(String path) { + /** + * @throws IllegalArgumentException if parameter path is null or empty. + */ + private CompletableFuture>> getBookiesThenFreshCache(String path) { + if (path == null || path.isEmpty()) { + return failedFuture( + new IllegalArgumentException("parameter [path] can not be null or empty.")); + } return store.getChildren(path) .thenComposeAsync(children -> { - Set bookieIds = PulsarRegistrationClient.convertToBookieAddresses(children); - List> bookieInfoUpdated = - new ArrayList<>(bookieIds.size()); + final Set bookieIds = PulsarRegistrationClient.convertToBookieAddresses(children); + final List> bookieInfoUpdated = new ArrayList<>(bookieIds.size()); for (BookieId id : bookieIds) { // update the cache for new bookies - if (!bookieServiceInfoCache.containsKey(id)) { - bookieInfoUpdated.add(readBookieServiceInfoAsync(id)); + if (path.equals(bookieReadonlyRegistrationPath) && readOnlyBookieInfo.get(id) == null) { + bookieInfoUpdated.add(readBookieInfoAsReadonlyBookie(id)); + continue; + } + if (path.equals(bookieRegistrationPath) && writableBookieInfo.get(id) == null) { + bookieInfoUpdated.add(readBookieInfoAsWritableBookie(id)); + continue; + } + if (path.equals(bookieAllRegistrationPath)) { + if (writableBookieInfo.get(id) != null || readOnlyBookieInfo.get(id) != null) { + // jump to next bookie id + continue; + } + // check writable first + final CompletableFuture revalidateAllBookiesFuture = readBookieInfoAsWritableBookie(id) + .thenCompose(writableBookieInfo -> writableBookieInfo + .>>>map( + bookieServiceInfo -> completedFuture(null)) + // check read-only then + .orElseGet(() -> readBookieInfoAsReadonlyBookie(id))); + bookieInfoUpdated.add(revalidateAllBookiesFuture); } } if (bookieInfoUpdated.isEmpty()) { - return CompletableFuture.completedFuture(bookieIds); + return completedFuture(bookieIds); } else { - return FutureUtil - .waitForAll(bookieInfoUpdated) + return waitForAll(bookieInfoUpdated) .thenApply(___ -> bookieIds); } }) @@ -153,42 +202,67 @@ public void unwatchReadOnlyBookies(RegistrationListener registrationListener) { readOnlyBookiesWatchers.remove(registrationListener); } - private void handleDeletedBookieNode(Notification n) { - if (n.getType() == NotificationType.Deleted) { - BookieId bookieId = stripBookieIdFromPath(n.getPath()); - if (bookieId != null) { - log.info("Bookie {} disappeared", bookieId); - bookieServiceInfoCache.remove(bookieId); - } + /** + * This method will receive metadata store notifications and then update the + * local cache in background sequentially. + */ + private void updatedBookies(Notification n) { + // make the notification callback run sequential in background. + final String path = n.getPath(); + if (!path.startsWith(bookieReadonlyRegistrationPath) && !path.startsWith(bookieRegistrationPath)) { + // ignore unknown path + return; } - } - - private void handleUpdatedBookieNode(Notification n) { - BookieId bookieId = stripBookieIdFromPath(n.getPath()); - if (bookieId != null) { - log.info("Bookie {} info updated", bookieId); - readBookieServiceInfoAsync(bookieId); + if (path.equals(bookieReadonlyRegistrationPath) || path.equals(bookieRegistrationPath)) { + // ignore root path + return; } - } - - private void updatedBookies(Notification n) { - if (n.getType() == NotificationType.Created || n.getType() == NotificationType.Deleted) { - if (n.getPath().startsWith(bookieReadonlyRegistrationPath)) { - getReadOnlyBookies().thenAccept(bookies -> { - readOnlyBookiesWatchers.forEach(w -> executor.execute(() -> w.onBookiesChanged(bookies))); - }); - handleDeletedBookieNode(n); - } else if (n.getPath().startsWith(bookieRegistrationPath)) { - getWritableBookies().thenAccept(bookies -> - writableBookiesWatchers.forEach(w -> executor.execute(() -> w.onBookiesChanged(bookies)))); - handleDeletedBookieNode(n); - } - } else if (n.getType() == NotificationType.Modified) { - if (n.getPath().startsWith(bookieReadonlyRegistrationPath) - || n.getPath().startsWith(bookieRegistrationPath)) { - handleUpdatedBookieNode(n); + final BookieId bookieId = stripBookieIdFromPath(n.getPath()); + sequencer.sequential(() -> { + switch (n.getType()) { + case Created: + log.info("Bookie {} created. path: {}", bookieId, n.getPath()); + if (path.startsWith(bookieReadonlyRegistrationPath)) { + return getReadOnlyBookies().thenAccept(bookies -> + readOnlyBookiesWatchers.forEach(w -> + executor.execute(() -> w.onBookiesChanged(bookies)))); + } + return getWritableBookies().thenAccept(bookies -> + writableBookiesWatchers.forEach(w -> + executor.execute(() -> w.onBookiesChanged(bookies)))); + case Modified: + if (bookieId == null) { + return completedFuture(null); + } + log.info("Bookie {} modified. path: {}", bookieId, n.getPath()); + if (path.startsWith(bookieReadonlyRegistrationPath)) { + return readBookieInfoAsReadonlyBookie(bookieId).thenApply(__ -> null); + } + return readBookieInfoAsWritableBookie(bookieId).thenApply(__ -> null); + case Deleted: + if (bookieId == null) { + return completedFuture(null); + } + log.info("Bookie {} deleted. path: {}", bookieId, n.getPath()); + if (path.startsWith(bookieReadonlyRegistrationPath)) { + readOnlyBookieInfo.remove(bookieId); + return getReadOnlyBookies().thenAccept(bookies -> { + readOnlyBookiesWatchers.forEach(w -> + executor.execute(() -> w.onBookiesChanged(bookies))); + }); + } + if (path.startsWith(bookieRegistrationPath)) { + writableBookieInfo.remove(bookieId); + return getWritableBookies().thenAccept(bookies -> { + writableBookiesWatchers.forEach(w -> + executor.execute(() -> w.onBookiesChanged(bookies))); + }); + } + return completedFuture(null); + default: + return completedFuture(null); } - } + }); } private static BookieId stripBookieIdFromPath(String path) { @@ -200,7 +274,7 @@ private static BookieId stripBookieIdFromPath(String path) { try { return BookieId.parse(path.substring(slash + 1)); } catch (IllegalArgumentException e) { - log.warn("Cannot decode bookieId from {}", path, e); + log.warn("Cannot decode bookieId from {}, error: {}", path, e.getMessage()); } } return null; @@ -227,46 +301,48 @@ public CompletableFuture> getBookieServiceInfo(Book // this is because there are a few cases in which some operations on the main thread // wait for the result. This is due to the fact that resolving the address of a bookie // is needed in many code paths. - Versioned resultFromCache = bookieServiceInfoCache.get(bookieId); + Versioned info; + if ((info = writableBookieInfo.get(bookieId)) == null) { + info = readOnlyBookieInfo.get(bookieId); + } if (log.isDebugEnabled()) { - log.debug("getBookieServiceInfo {} -> {}", bookieId, resultFromCache); + log.debug("getBookieServiceInfo {} -> {}", bookieId, info); } - if (resultFromCache != null) { - return CompletableFuture.completedFuture(resultFromCache); + if (info != null) { + return completedFuture(info); } else { return FutureUtils.exception(new BKException.BKBookieHandleNotAvailableException()); } } - public CompletableFuture readBookieServiceInfoAsync(BookieId bookieId) { - String asWritable = bookieRegistrationPath + "/" + bookieId; - return bookieServiceInfoMetadataCache.get(asWritable) - .thenCompose((Optional getResult) -> { - if (getResult.isPresent()) { - Versioned res = - new Versioned<>(getResult.get(), new LongVersion(-1)); - log.info("Update BookieInfoCache (writable bookie) {} -> {}", bookieId, getResult.get()); - bookieServiceInfoCache.put(bookieId, res); - return CompletableFuture.completedFuture(null); - } else { - return readBookieInfoAsReadonlyBookie(bookieId); - } - } - ); + public CompletableFuture>> readBookieInfoAsWritableBookie( + BookieId bookieId) { + final String asWritable = bookieRegistrationPath + "/" + bookieId; + return bookieServiceInfoMetadataCache.getWithStats(asWritable) + .thenApply((Optional> bkInfoWithStats) -> { + if (bkInfoWithStats.isPresent()) { + final CacheGetResult r = bkInfoWithStats.get(); + log.info("Update BookieInfoCache (writable bookie) {} -> {}", bookieId, r.getValue()); + writableBookieInfo.put(bookieId, + new Versioned<>(r.getValue(), new LongVersion(r.getStat().getVersion()))); + } + return bkInfoWithStats; + } + ); } - final CompletableFuture readBookieInfoAsReadonlyBookie(BookieId bookieId) { - String asReadonly = bookieReadonlyRegistrationPath + "/" + bookieId; - return bookieServiceInfoMetadataCache.get(asReadonly) - .thenApply((Optional getResultAsReadOnly) -> { - if (getResultAsReadOnly.isPresent()) { - Versioned res = - new Versioned<>(getResultAsReadOnly.get(), new LongVersion(-1)); - log.info("Update BookieInfoCache (readonly bookie) {} -> {}", bookieId, - getResultAsReadOnly.get()); - bookieServiceInfoCache.put(bookieId, res); + final CompletableFuture>> readBookieInfoAsReadonlyBookie( + BookieId bookieId) { + final String asReadonly = bookieReadonlyRegistrationPath + "/" + bookieId; + return bookieServiceInfoMetadataCache.getWithStats(asReadonly) + .thenApply((Optional> bkInfoWithStats) -> { + if (bkInfoWithStats.isPresent()) { + final CacheGetResult r = bkInfoWithStats.get(); + log.info("Update BookieInfoCache (readonly bookie) {} -> {}", bookieId, r.getValue()); + readOnlyBookieInfo.put(bookieId, + new Versioned<>(r.getValue(), new LongVersion(r.getStat().getVersion()))); } - return null; + return bkInfoWithStats; }); } } diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationManager.java index 32ec89e717e0e..c6aba6b7d93d0 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationManager.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationManager.java @@ -19,10 +19,12 @@ package org.apache.pulsar.metadata.bookkeeper; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.bookkeeper.util.BookKeeperConstants.AVAILABLE_NODE; import static org.apache.bookkeeper.util.BookKeeperConstants.COOKIE_NODE; import static org.apache.bookkeeper.util.BookKeeperConstants.INSTANCEID; import static org.apache.bookkeeper.util.BookKeeperConstants.READONLY; +import static org.apache.pulsar.metadata.bookkeeper.AbstractMetadataDriver.BLOCKING_CALL_TIMEOUT; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; @@ -32,8 +34,8 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; import lombok.Cleanup; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.bookie.BookieException; import org.apache.bookkeeper.conf.AbstractConfiguration; @@ -85,12 +87,11 @@ public class PulsarRegistrationManager implements RegistrationManager { } @Override - @SneakyThrows public void close() { for (ResourceLock rwBookie : bookieRegistration.values()) { try { - rwBookie.release().get(); - } catch (ExecutionException ignore) { + rwBookie.release().get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); + } catch (ExecutionException | TimeoutException ignore) { log.error("Cannot release correctly {}", rwBookie, ignore.getCause()); } catch (InterruptedException ignore) { log.error("Cannot release correctly {}", rwBookie, ignore); @@ -100,26 +101,30 @@ public void close() { for (ResourceLock roBookie : bookieRegistrationReadOnly.values()) { try { - roBookie.release().get(); - } catch (ExecutionException ignore) { + roBookie.release().get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); + } catch (ExecutionException | TimeoutException ignore) { log.error("Cannot release correctly {}", roBookie, ignore.getCause()); } catch (InterruptedException ignore) { log.error("Cannot release correctly {}", roBookie, ignore); Thread.currentThread().interrupt(); } } - coordinationService.close(); + try { + coordinationService.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } } @Override public String getClusterInstanceId() throws BookieException { try { return store.get(ledgersRootPath + "/" + INSTANCEID) - .get() + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS) .map(res -> new String(res.getValue(), UTF_8)) .orElseThrow( () -> new BookieException.MetadataStoreException("BookKeeper cluster not initialized")); - } catch (ExecutionException | InterruptedException e) { + } catch (ExecutionException | InterruptedException | TimeoutException e) { throw new BookieException.MetadataStoreException("Failed to get cluster instance id", e); } } @@ -135,23 +140,25 @@ public void registerBookie(BookieId bookieId, boolean readOnly, BookieServiceInf if (readOnly) { ResourceLock rwRegistration = bookieRegistration.remove(bookieId); if (rwRegistration != null) { - log.info("Bookie {} was already registered as writable, unregistering"); - rwRegistration.release().get(); + log.info("Bookie {} was already registered as writable, unregistering", bookieId); + rwRegistration.release().get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); } bookieRegistrationReadOnly.put(bookieId, - lockManager.acquireLock(regPathReadOnly, bookieServiceInfo).get()); + lockManager.acquireLock(regPathReadOnly, bookieServiceInfo) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS)); } else { ResourceLock roRegistration = bookieRegistrationReadOnly.remove(bookieId); if (roRegistration != null) { - log.info("Bookie {} was already registered as read-only, unregistering"); - roRegistration.release().get(); + log.info("Bookie {} was already registered as read-only, unregistering", bookieId); + roRegistration.release().get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); } bookieRegistration.put(bookieId, - lockManager.acquireLock(regPath, bookieServiceInfo).get()); + lockManager.acquireLock(regPath, bookieServiceInfo) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS)); } - } catch (ExecutionException ee) { + } catch (ExecutionException | TimeoutException ee) { log.error("Exception registering ephemeral node for Bookie!", ee); // Throw an IOException back up. This will cause the Bookie // constructor to error out. Alternatively, we could do a System @@ -173,18 +180,18 @@ public void unregisterBookie(BookieId bookieId, boolean readOnly) throws BookieE if (readOnly) { ResourceLock roRegistration = bookieRegistrationReadOnly.get(bookieId); if (roRegistration != null) { - roRegistration.release().get(); + roRegistration.release().get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); } } else { ResourceLock rwRegistration = bookieRegistration.get(bookieId); if (rwRegistration != null) { - rwRegistration.release().get(); + rwRegistration.release().get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); } } } catch (InterruptedException ie) { Thread.currentThread().interrupt(); throw new BookieException.MetadataStoreException(ie); - } catch (ExecutionException e) { + } catch (ExecutionException | TimeoutException e) { throw new BookieException.MetadataStoreException(e); } } @@ -195,8 +202,9 @@ public boolean isBookieRegistered(BookieId bookieId) throws BookieException { String readonlyRegPath = bookieReadonlyRegistrationPath + "/" + bookieId; try { - return (store.exists(regPath).get() || store.exists(readonlyRegPath).get()); - } catch (ExecutionException e) { + return (store.exists(regPath).get(BLOCKING_CALL_TIMEOUT, MILLISECONDS) + || store.exists(readonlyRegPath).get(BLOCKING_CALL_TIMEOUT, MILLISECONDS)); + } catch (ExecutionException | TimeoutException e) { log.error("Exception while checking registration ephemeral nodes for BookieId: {}", bookieId, e); throw new BookieException.MetadataStoreException(e); } catch (InterruptedException e) { @@ -222,7 +230,8 @@ public void writeCookie(BookieId bookieId, Versioned cookieData) throws version = ((LongVersion) cookieData.getVersion()).getLongVersion(); } - store.put(path, cookieData.getValue(), Optional.of(version)).get(); + store.put(path, cookieData.getValue(), Optional.of(version)) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); throw new BookieException.MetadataStoreException("Interrupted writing cookie for bookie " + bookieId, ie); @@ -232,6 +241,8 @@ public void writeCookie(BookieId bookieId, Versioned cookieData) throws } else { throw new BookieException.MetadataStoreException("Failed to write cookie for bookie " + bookieId); } + } catch (TimeoutException ex) { + throw new BookieException.MetadataStoreException("Failed to write cookie for bookie " + bookieId, ex); } } @@ -239,7 +250,7 @@ public void writeCookie(BookieId bookieId, Versioned cookieData) throws public Versioned readCookie(BookieId bookieId) throws BookieException { String path = this.cookiePath + "/" + bookieId; try { - Optional res = store.get(path).get(); + Optional res = store.get(path).get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); if (!res.isPresent()) { throw new BookieException.CookieNotFoundException(bookieId.toString()); } @@ -250,7 +261,7 @@ public Versioned readCookie(BookieId bookieId) throws BookieException { } catch (InterruptedException ie) { Thread.currentThread().interrupt(); throw new BookieException.MetadataStoreException(ie); - } catch (ExecutionException e) { + } catch (ExecutionException | TimeoutException e) { throw new BookieException.MetadataStoreException(e); } } @@ -259,7 +270,8 @@ public Versioned readCookie(BookieId bookieId) throws BookieException { public void removeCookie(BookieId bookieId, Version version) throws BookieException { String path = this.cookiePath + "/" + bookieId; try { - store.delete(path, Optional.of(((LongVersion) version).getLongVersion())).get(); + store.delete(path, Optional.of(((LongVersion) version).getLongVersion())) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new BookieException.MetadataStoreException("Interrupted deleting cookie for bookie " + bookieId, e); @@ -269,6 +281,8 @@ public void removeCookie(BookieId bookieId, Version version) throws BookieExcept } else { throw new BookieException.MetadataStoreException("Failed to delete cookie for bookie " + bookieId); } + } catch (TimeoutException ex) { + throw new BookieException.MetadataStoreException("Failed to delete cookie for bookie " + bookieId); } log.info("Removed cookie from {} for bookie {}.", cookiePath, bookieId); @@ -276,20 +290,23 @@ public void removeCookie(BookieId bookieId, Version version) throws BookieExcept @Override public boolean prepareFormat() throws Exception { - boolean ledgerRootExists = store.exists(ledgersRootPath).get(); - boolean availableNodeExists = store.exists(bookieRegistrationPath).get(); + boolean ledgerRootExists = store.exists(ledgersRootPath).get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); + boolean availableNodeExists = store.exists(bookieRegistrationPath).get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); // Create ledgers root node if not exists if (!ledgerRootExists) { - store.put(ledgersRootPath, new byte[0], Optional.empty()).get(); + store.put(ledgersRootPath, new byte[0], Optional.empty()) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); } // create available bookies node if not exists if (!availableNodeExists) { - store.put(bookieRegistrationPath, new byte[0], Optional.empty()).get(); + store.put(bookieRegistrationPath, new byte[0], Optional.empty()) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); } // create readonly bookies node if not exists - if (!store.exists(bookieReadonlyRegistrationPath).get()) { - store.put(bookieReadonlyRegistrationPath, new byte[0], Optional.empty()).get(); + if (!store.exists(bookieReadonlyRegistrationPath).get(BLOCKING_CALL_TIMEOUT, MILLISECONDS)) { + store.put(bookieReadonlyRegistrationPath, new byte[0], Optional.empty()) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); } return ledgerRootExists; @@ -301,16 +318,18 @@ public boolean initNewCluster() throws Exception { log.info("Initializing metadata for new cluster, ledger root path: {}", ledgersRootPath); - if (store.exists(instanceIdPath).get()) { + if (store.exists(instanceIdPath).get(BLOCKING_CALL_TIMEOUT, MILLISECONDS)) { log.error("Ledger root path: {} already exists", ledgersRootPath); return false; } - store.put(ledgersRootPath, new byte[0], Optional.empty()).get(); + store.put(ledgersRootPath, new byte[0], Optional.empty()) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); // create INSTANCEID String instanceId = UUID.randomUUID().toString(); - store.put(instanceIdPath, instanceId.getBytes(UTF_8), Optional.of(-1L)).join(); + store.put(instanceIdPath, instanceId.getBytes(UTF_8), Optional.of(-1L)) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); log.info("Successfully initiated cluster. ledger root path: {} instanceId: {}", ledgersRootPath, instanceId); @@ -321,23 +340,28 @@ public boolean initNewCluster() throws Exception { public boolean format() throws Exception { // Clear underreplicated ledgers store.deleteRecursive(PulsarLedgerUnderreplicationManager.getBasePath(ledgersRootPath) - + BookKeeperConstants.DEFAULT_ZK_LEDGERS_ROOT_PATH).get(); + + BookKeeperConstants.DEFAULT_ZK_LEDGERS_ROOT_PATH) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); // Clear underreplicatedledger locks - store.deleteRecursive(PulsarLedgerUnderreplicationManager.getUrLockPath(ledgersRootPath)).get(); + store.deleteRecursive(PulsarLedgerUnderreplicationManager.getUrLockPath(ledgersRootPath)) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); // Clear the cookies - store.deleteRecursive(cookiePath).get(); + store.deleteRecursive(cookiePath).get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); // Clear the INSTANCEID - if (store.exists(ledgersRootPath + "/" + BookKeeperConstants.INSTANCEID).get()) { - store.delete(ledgersRootPath + "/" + BookKeeperConstants.INSTANCEID, Optional.empty()).get(); + if (store.exists(ledgersRootPath + "/" + BookKeeperConstants.INSTANCEID) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS)) { + store.delete(ledgersRootPath + "/" + BookKeeperConstants.INSTANCEID, Optional.empty()) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); } // create INSTANCEID String instanceId = UUID.randomUUID().toString(); store.put(ledgersRootPath + "/" + BookKeeperConstants.INSTANCEID, - instanceId.getBytes(StandardCharsets.UTF_8), Optional.of(-1L)).get(); + instanceId.getBytes(StandardCharsets.UTF_8), Optional.of(-1L)) + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); log.info("Successfully formatted BookKeeper metadata"); return true; @@ -347,7 +371,7 @@ public boolean format() throws Exception { public boolean nukeExistingCluster() throws Exception { log.info("Nuking metadata of existing cluster, ledger root path: {}", ledgersRootPath); - if (!store.exists(ledgersRootPath + "/" + INSTANCEID).join()) { + if (!store.exists(ledgersRootPath + "/" + INSTANCEID).get(BLOCKING_CALL_TIMEOUT, MILLISECONDS)) { log.info("There is no existing cluster with ledgersRootPath: {}, so exiting nuke operation", ledgersRootPath); return true; @@ -356,17 +380,19 @@ public boolean nukeExistingCluster() throws Exception { @Cleanup RegistrationClient registrationClient = new PulsarRegistrationClient(store, ledgersRootPath); - Collection rwBookies = registrationClient.getWritableBookies().join().getValue(); + Collection rwBookies = registrationClient.getWritableBookies() + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS).getValue(); if (rwBookies != null && !rwBookies.isEmpty()) { log.error("Bookies are still up and connected to this cluster, " - + "stop all bookies before nuking the cluster"); + + "stop all bookies before nuking the cluster"); return false; } - Collection roBookies = registrationClient.getReadOnlyBookies().join().getValue(); + Collection roBookies = registrationClient.getReadOnlyBookies() + .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS).getValue(); if (roBookies != null && !roBookies.isEmpty()) { log.error("Readonly Bookies are still up and connected to this cluster, " - + "stop all bookies before nuking the cluster"); + + "stop all bookies before nuking the cluster"); return false; } diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java index b9051a7dc7df4..4c7f34aa5c16e 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java @@ -25,6 +25,7 @@ import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.annotations.VisibleForTesting; import java.io.IOException; +import java.util.EnumSet; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -47,26 +48,34 @@ import org.apache.pulsar.metadata.api.MetadataStoreException.ContentDeserializationException; import org.apache.pulsar.metadata.api.MetadataStoreException.NotFoundException; import org.apache.pulsar.metadata.api.Notification; +import org.apache.pulsar.metadata.api.extended.CreateOption; +import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.metadata.impl.AbstractMetadataStore; @Slf4j public class MetadataCacheImpl implements MetadataCache, Consumer { @Getter private final MetadataStore store; + private final MetadataStoreExtended storeExtended; private final MetadataSerde serde; private final AsyncLoadingCache>> objCache; - public MetadataCacheImpl(MetadataStore store, TypeReference typeRef, MetadataCacheConfig cacheConfig) { + public MetadataCacheImpl(MetadataStore store, TypeReference typeRef, MetadataCacheConfig cacheConfig) { this(store, new JSONMetadataSerdeTypeRef<>(typeRef), cacheConfig); } - public MetadataCacheImpl(MetadataStore store, JavaType type, MetadataCacheConfig cacheConfig) { + public MetadataCacheImpl(MetadataStore store, JavaType type, MetadataCacheConfig cacheConfig) { this(store, new JSONMetadataSerdeSimpleType<>(type), cacheConfig); } - public MetadataCacheImpl(MetadataStore store, MetadataSerde serde, MetadataCacheConfig cacheConfig) { + public MetadataCacheImpl(MetadataStore store, MetadataSerde serde, MetadataCacheConfig cacheConfig) { this.store = store; + if (store instanceof MetadataStoreExtended) { + this.storeExtended = (MetadataStoreExtended) store; + } else { + this.storeExtended = null; + } this.serde = serde; Caffeine cacheBuilder = Caffeine.newBuilder(); @@ -89,7 +98,12 @@ public CompletableFuture>> asyncReload( Optional> oldValue, Executor executor) { if (store instanceof AbstractMetadataStore && ((AbstractMetadataStore) store).isConnected()) { - return readValueFromStore(key); + return readValueFromStore(key).thenApply(val -> { + if (cacheConfig.getAsyncReloadConsumer() != null) { + cacheConfig.getAsyncReloadConsumer().accept(key, val); + } + return val; + }); } else { // Do not try to refresh the cache item if we know that we're not connected to the // metadata store @@ -243,6 +257,21 @@ public CompletableFuture create(String path, T value) { return future; } + @Override + public CompletableFuture put(String path, T value, EnumSet options) { + final byte[] bytes; + try { + bytes = serde.serialize(path, value); + } catch (IOException e) { + return CompletableFuture.failedFuture(e); + } + if (storeExtended != null) { + return storeExtended.put(path, bytes, Optional.empty(), options).thenAccept(__ -> refresh(path)); + } else { + return store.put(path, bytes, Optional.empty()).thenAccept(__ -> refresh(path)); + } + } + @Override public CompletableFuture delete(String path) { return store.delete(path, Optional.empty()); diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java index 9e6a9b94c42a3..ab35eb7040c10 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java @@ -129,19 +129,29 @@ private synchronized CompletableFuture handleExistingLeader return FutureUtils.exception(t); } - if (existingValue.equals(proposedValue.orElse(null))) { + T value = proposedValue.orElse(null); + if (existingValue.equals(value)) { // If the value is the same as our proposed value, it means this instance was the leader at some // point before. The existing value can either be for this same session or for a previous one. if (res.getStat().isCreatedBySelf()) { + log.info("Keeping the existing value {} for {} as it's from the same session stat={}", existingValue, + path, res.getStat()); // The value is still valid because it was created in the same session changeState(LeaderElectionState.Leading); + return CompletableFuture.completedFuture(LeaderElectionState.Leading); } else { + log.info("Conditionally deleting existing equals value {} for {} because it's not created in the " + + "current session. stat={}", existingValue, path, res.getStat()); // Since the value was created in a different session, it might be expiring. We need to delete it // and try the election again. return store.delete(path, Optional.of(res.getStat().getVersion())) .thenCompose(__ -> tryToBecomeLeader()); } } else if (res.getStat().isCreatedBySelf()) { + log.warn("Conditionally deleting existing value {} for {} because it's different from the proposed value " + + "({}). This is unexpected since it was created within the same session. " + + "In tests this could happen because of an invalid shared session id when using mocks.", + existingValue, path, value); // The existing value is different but was created from the same session return store.delete(path, Optional.of(res.getStat().getVersion())) .thenCompose(__ -> tryToBecomeLeader()); @@ -165,9 +175,10 @@ private synchronized void changeState(LeaderElectionState les) { } private synchronized CompletableFuture tryToBecomeLeader() { + T value = proposedValue.get(); byte[] payload; try { - payload = serde.serialize(path, proposedValue.get()); + payload = serde.serialize(path, value); } catch (Throwable t) { return FutureUtils.exception(t); } @@ -181,7 +192,7 @@ private synchronized CompletableFuture tryToBecomeLeader() cache.get(path) .thenRun(() -> { synchronized (LeaderElectionImpl.this) { - log.info("Acquired leadership on {}", path); + log.info("Acquired leadership on {} with {}", path, value); internalState = InternalState.LeaderIsPresent; if (leaderElectionState != LeaderElectionState.Leading) { leaderElectionState = LeaderElectionState.Leading; @@ -196,6 +207,8 @@ private synchronized CompletableFuture tryToBecomeLeader() }).exceptionally(ex -> { // We fail to do the get(), so clean up the leader election fail the whole // operation + log.warn("Failed to get the current state after acquiring leadership on {}. " + + " Conditionally deleting current entry.", path, ex); store.delete(path, Optional.of(stat.getVersion())) .thenRun(() -> result.completeExceptionally(ex)) .exceptionally(ex2 -> { @@ -205,6 +218,8 @@ private synchronized CompletableFuture tryToBecomeLeader() return null; }); } else { + log.info("Leadership on {} with value {} was lost. " + + "Conditionally deleting entry with stat={}.", path, value, stat); // LeaderElection was closed in between. Release the lock asynchronously store.delete(path, Optional.of(stat.getVersion())) .thenRun(() -> result.completeExceptionally( @@ -219,7 +234,9 @@ private synchronized CompletableFuture tryToBecomeLeader() if (ex.getCause() instanceof BadVersionException) { // There was a conflict between 2 participants trying to become leaders at same time. Retry // to fetch info on new leader. - + log.info("There was a conflict between 2 participants trying to become leaders at the same " + + "time on {}. Attempted with value {}. Retrying.", + path, value); elect() .thenAccept(lse -> result.complete(lse)) .exceptionally(ex2 -> { @@ -257,7 +274,13 @@ public synchronized CompletableFuture asyncClose() { return CompletableFuture.completedFuture(null); } - return store.delete(path, version); + return store.delete(path, version) + .thenAccept(__ -> { + synchronized (LeaderElectionImpl.this) { + leaderElectionState = LeaderElectionState.NoLeader; + } + } + ); } @Override @@ -278,8 +301,8 @@ public Optional getLeaderValueIfPresent() { private void handleSessionNotification(SessionEvent event) { // Ensure we're only processing one session event at a time. sequencer.sequential(() -> FutureUtil.composeAsync(() -> { - if (event == SessionEvent.SessionReestablished) { - log.info("Revalidating leadership for {}", path); + if (event == SessionEvent.Reconnected || event == SessionEvent.SessionReestablished) { + log.info("Revalidating leadership for {}, event:{}", path, event); return elect().thenAccept(leaderState -> { log.info("Resynced leadership for {} - State: {}", path, leaderState); }).exceptionally(ex -> { diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java index 4da6b7998a0c4..b6b5c57ccea39 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java @@ -27,7 +27,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.common.util.FutureUtil; @@ -52,7 +52,7 @@ class LockManagerImpl implements LockManager { private final MetadataCache cache; private final MetadataSerde serde; private final FutureUtil.Sequencer sequencer; - private final ExecutorService executor; + private final ScheduledExecutorService executor; private enum State { Ready, Closed @@ -60,13 +60,13 @@ private enum State { private State state = State.Ready; - LockManagerImpl(MetadataStoreExtended store, Class clazz, ExecutorService executor) { + LockManagerImpl(MetadataStoreExtended store, Class clazz, ScheduledExecutorService executor) { this(store, new JSONMetadataSerdeSimpleType<>( TypeFactory.defaultInstance().constructSimpleType(clazz, null)), executor); } - LockManagerImpl(MetadataStoreExtended store, MetadataSerde serde, ExecutorService executor) { + LockManagerImpl(MetadataStoreExtended store, MetadataSerde serde, ScheduledExecutorService executor) { this.store = store; this.cache = store.getMetadataCache(serde); this.serde = serde; @@ -83,7 +83,7 @@ public CompletableFuture> readLock(String path) { @Override public CompletableFuture> acquireLock(String path, T value) { - ResourceLockImpl lock = new ResourceLockImpl<>(store, serde, path); + ResourceLockImpl lock = new ResourceLockImpl<>(store, serde, path, executor); CompletableFuture> result = new CompletableFuture<>(); lock.acquire(value).thenRun(() -> { diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java index 93c994b2436b9..692f224594cae 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java @@ -21,8 +21,13 @@ import java.util.EnumSet; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.common.concurrent.FutureUtils; +import org.apache.pulsar.common.util.Backoff; +import org.apache.pulsar.common.util.BackoffBuilder; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.GetResult; import org.apache.pulsar.metadata.api.MetadataSerde; @@ -44,7 +49,10 @@ public class ResourceLockImpl implements ResourceLock { private long version; private final CompletableFuture expiredFuture; private boolean revalidateAfterReconnection = false; + private final Backoff backoff; private final FutureUtil.Sequencer sequencer; + private final ScheduledExecutorService executor; + private ScheduledFuture revalidateTask; private enum State { Init, @@ -55,7 +63,8 @@ private enum State { private State state; - public ResourceLockImpl(MetadataStoreExtended store, MetadataSerde serde, String path) { + ResourceLockImpl(MetadataStoreExtended store, MetadataSerde serde, String path, + ScheduledExecutorService executor) { this.store = store; this.serde = serde; this.path = path; @@ -63,6 +72,11 @@ public ResourceLockImpl(MetadataStoreExtended store, MetadataSerde serde, Str this.expiredFuture = new CompletableFuture<>(); this.sequencer = FutureUtil.Sequencer.create(); this.state = State.Init; + this.executor = executor; + this.backoff = new BackoffBuilder() + .setInitialTime(100, TimeUnit.MILLISECONDS) + .setMax(60, TimeUnit.SECONDS) + .create(); } @Override @@ -93,6 +107,10 @@ public synchronized CompletableFuture release() { } state = State.Releasing; + if (revalidateTask != null) { + revalidateTask.cancel(true); + } + CompletableFuture result = new CompletableFuture<>(); store.delete(path, Optional.of(version)) @@ -210,8 +228,15 @@ synchronized CompletableFuture revalidateIfNeededAfterReconnection() { * This method is thread-safe and it will perform multiple re-validation operations in turn. */ synchronized CompletableFuture silentRevalidateOnce() { + if (state != State.Valid) { + return CompletableFuture.completedFuture(null); + } + return sequencer.sequential(() -> revalidate(value)) - .thenRun(() -> log.info("Successfully revalidated the lock on {}", path)) + .thenRun(() -> { + log.info("Successfully revalidated the lock on {}", path); + backoff.reset(); + }) .exceptionally(ex -> { synchronized (ResourceLockImpl.this) { Throwable realCause = FutureUtil.unwrapCompletionException(ex); @@ -225,8 +250,12 @@ synchronized CompletableFuture silentRevalidateOnce() { // Continue assuming we hold the lock, until we can revalidate it, either // on Reconnected or SessionReestablished events. revalidateAfterReconnection = true; - log.warn("Failed to revalidate the lock at {}. Retrying later on reconnection {}", path, - realCause.getMessage()); + + long delayMillis = backoff.next(); + log.warn("Failed to revalidate the lock at {}: {} - Retrying in {} seconds", path, + realCause.getMessage(), delayMillis / 1000.0); + revalidateTask = + executor.schedule(this::silentRevalidateOnce, delayMillis, TimeUnit.MILLISECONDS); } } return null; diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java index 072d513cca962..c458d0da2146a 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java @@ -23,8 +23,10 @@ import com.github.benmanes.caffeine.cache.AsyncCacheLoader; import com.github.benmanes.caffeine.cache.AsyncLoadingCache; import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; import com.google.common.annotations.VisibleForTesting; import io.netty.util.concurrent.DefaultThreadFactory; +import io.opentelemetry.api.OpenTelemetry; import java.time.Instant; import java.util.Collections; import java.util.EnumSet; @@ -41,6 +43,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; +import java.util.function.Supplier; import java.util.stream.Collectors; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -86,8 +89,10 @@ public abstract class AbstractMetadataStore implements MetadataStoreExtended, Co protected abstract CompletableFuture existsFromStore(String path); - protected AbstractMetadataStore(String metadataStoreName) { - this.executor = new ScheduledThreadPoolExecutor(1, new DefaultThreadFactory(metadataStoreName)); + protected AbstractMetadataStore(String metadataStoreName, OpenTelemetry openTelemetry) { + this.executor = new ScheduledThreadPoolExecutor(1, + new DefaultThreadFactory( + StringUtils.isNotBlank(metadataStoreName) ? metadataStoreName : getClass().getSimpleName())); registerListener(this); this.childrenCache = Caffeine.newBuilder() @@ -133,7 +138,7 @@ public CompletableFuture asyncReload(String key, Boolean oldValue, }); this.metadataStoreName = metadataStoreName; - this.metadataStoreStats = new MetadataStoreStats(metadataStoreName); + this.metadataStoreStats = new MetadataStoreStats(metadataStoreName, openTelemetry); } @Override @@ -173,7 +178,15 @@ public CompletableFuture handleMetadataEvent(MetadataEvent event) { return result; } + /** + * @deprecated Use {@link #registerSyncListener(Optional)} instead. + */ + @Deprecated protected void registerSyncLister(Optional synchronizer) { + this.registerSyncListener(synchronizer); + } + + protected void registerSyncListener(Optional synchronizer) { synchronizer.ifPresent(s -> s.registerSyncListener(this::handleMetadataEvent)); } @@ -245,8 +258,7 @@ public MetadataCache getMetadataCache(MetadataSerde serde, MetadataCac @Override public CompletableFuture> get(String path) { if (isClosed()) { - return FutureUtil.failedFuture( - new MetadataStoreException.AlreadyClosedException()); + return alreadyClosedFailedFuture(); } long start = System.currentTimeMillis(); if (!isValidPath(path)) { @@ -274,8 +286,7 @@ public CompletableFuture put(String path, byte[] value, Optional exp @Override public final CompletableFuture> getChildren(String path) { if (isClosed()) { - return FutureUtil.failedFuture( - new MetadataStoreException.AlreadyClosedException()); + return alreadyClosedFailedFuture(); } if (!isValidPath(path)) { return FutureUtil.failedFuture(new MetadataStoreException.InvalidPathException(path)); @@ -286,8 +297,7 @@ public final CompletableFuture> getChildren(String path) { @Override public final CompletableFuture exists(String path) { if (isClosed()) { - return FutureUtil.failedFuture( - new MetadataStoreException.AlreadyClosedException()); + return alreadyClosedFailedFuture(); } if (!isValidPath(path)) { return FutureUtil.failedFuture(new MetadataStoreException.InvalidPathException(path)); @@ -328,6 +338,7 @@ public void accept(Notification n) { if (type == NotificationType.Created || type == NotificationType.Deleted) { existsCache.synchronous().invalidate(path); + childrenCache.synchronous().invalidate(path); String parent = parent(path); if (parent != null) { childrenCache.synchronous().invalidate(parent); @@ -347,9 +358,9 @@ public void accept(Notification n) { @Override public final CompletableFuture delete(String path, Optional expectedVersion) { + log.info("Deleting path: {} (v. {})", path, expectedVersion); if (isClosed()) { - return FutureUtil.failedFuture( - new MetadataStoreException.AlreadyClosedException()); + return alreadyClosedFailedFuture(); } long start = System.currentTimeMillis(); if (!isValidPath(path)) { @@ -385,33 +396,31 @@ private CompletableFuture deleteInternal(String path, Optional expec // Ensure caches are invalidated before the operation is confirmed return storeDelete(path, expectedVersion).thenRun(() -> { existsCache.synchronous().invalidate(path); + childrenCache.synchronous().invalidate(path); String parent = parent(path); if (parent != null) { childrenCache.synchronous().invalidate(parent); } metadataCaches.forEach(c -> c.invalidate(path)); + log.info("Deleted path: {} (v. {})", path, expectedVersion); }); } @Override public CompletableFuture deleteRecursive(String path) { + log.info("Deleting recursively path: {}", path); if (isClosed()) { - return FutureUtil.failedFuture( - new MetadataStoreException.AlreadyClosedException()); + return alreadyClosedFailedFuture(); } return getChildren(path) .thenCompose(children -> FutureUtil.waitForAll( children.stream() .map(child -> deleteRecursive(path + "/" + child)) .collect(Collectors.toList()))) - .thenCompose(__ -> exists(path)) - .thenCompose(exists -> { - if (exists) { - return delete(path, Optional.empty()); - } else { - return CompletableFuture.completedFuture(null); - } + .thenCompose(__ -> { + log.info("After deleting all children, now deleting path: {}", path); + return deleteIfExists(path, Optional.empty()); }); } @@ -422,8 +431,7 @@ protected abstract CompletableFuture storePut(String path, byte[] data, Op public final CompletableFuture put(String path, byte[] data, Optional optExpectedVersion, EnumSet options) { if (isClosed()) { - return FutureUtil.failedFuture( - new MetadataStoreException.AlreadyClosedException()); + return alreadyClosedFailedFuture(); } long start = System.currentTimeMillis(); if (!isValidPath(path)) { @@ -488,6 +496,16 @@ public void registerSessionListener(Consumer listener) { protected void receivedSessionEvent(SessionEvent event) { isConnected = event.isConnected(); + + // Clear cache after session expired. + if (event == SessionEvent.SessionReestablished || event == SessionEvent.Reconnected) { + for (MetadataCacheImpl metadataCache : metadataCaches) { + metadataCache.invalidateAll(); + } + invalidateAll(); + } + + // Notice listeners. try { executor.execute(() -> { sessionListeners.forEach(l -> { @@ -503,10 +521,15 @@ protected void receivedSessionEvent(SessionEvent event) { } } - private boolean isClosed() { + protected boolean isClosed() { return isClosed.get(); } + protected static CompletableFuture alreadyClosedFailedFuture() { + return FutureUtil.failedFuture( + new MetadataStoreException.AlreadyClosedException()); + } + @Override public void close() throws Exception { executor.shutdownNow(); @@ -520,6 +543,13 @@ public void invalidateAll() { existsCache.synchronous().invalidateAll(); } + public void invalidateCaches(String...paths) { + LoadingCache> loadingCache = childrenCache.synchronous(); + for (String path : paths) { + loadingCache.invalidate(path); + } + } + /** * Run the task in the executor thread and fail the future if the executor is shutting down. */ @@ -532,6 +562,18 @@ public void execute(Runnable task, CompletableFuture future) { } } + /** + * Run the task in the executor thread and fail the future if the executor is shutting down. + */ + @VisibleForTesting + public void execute(Runnable task, Supplier>> futures) { + try { + executor.execute(task); + } catch (final Throwable t) { + futures.get().forEach(f -> f.completeExceptionally(t)); + } + } + protected static String parent(String path) { int idx = path.lastIndexOf('/'); if (idx <= 0) { diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/EtcdMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/EtcdMetadataStore.java index a7fb7192cb5fe..3937fd712dc9f 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/EtcdMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/EtcdMetadataStore.java @@ -43,10 +43,10 @@ import io.etcd.jetcd.watch.WatchResponse; import io.grpc.Status; import io.grpc.StatusRuntimeException; -import io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslProvider; import io.grpc.stub.StreamObserver; -import io.netty.handler.ssl.SslContext; -import io.netty.handler.ssl.SslProvider; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -109,9 +109,9 @@ public EtcdMetadataStore(String metadataURL, MetadataStoreConfig conf, boolean e try { this.client = newEtcdClient(metadataURL, conf); this.kv = client.getKVClient(); - this.client.getWatchClient().watch(ByteSequence.from("\0", StandardCharsets.UTF_8), + this.client.getWatchClient().watch(ByteSequence.from("/", StandardCharsets.UTF_8), WatchOption.newBuilder() - .withPrefix(ByteSequence.from("/", StandardCharsets.UTF_8)) + .isPrefix(true) .build(), this::handleWatchResponse); if (enableSessionWatcher) { this.sessionWatcher = @@ -285,7 +285,7 @@ protected void batchOperation(List ops) { .withKeysOnly(true) .withSortField(GetOption.SortTarget.KEY) .withSortOrder(GetOption.SortOrder.ASCEND) - .withPrefix(prefix) + .isPrefix(true) .build())); break; } diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/LocalMemoryMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/LocalMemoryMetadataStore.java index 94d9a1f8937f7..e95f1947740c8 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/LocalMemoryMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/LocalMemoryMetadataStore.java @@ -78,12 +78,11 @@ private static class Value { public LocalMemoryMetadataStore(String metadataURL, MetadataStoreConfig metadataStoreConfig) throws MetadataStoreException { - super(metadataStoreConfig.getMetadataStoreName()); + super(metadataStoreConfig.getMetadataStoreName(), metadataStoreConfig.getOpenTelemetry()); String name = metadataURL.substring(MEMORY_SCHEME_IDENTIFIER.length()); // Local means a private data set // update synchronizer and register sync listener - synchronizer = metadataStoreConfig.getSynchronizer(); - registerSyncLister(Optional.ofNullable(synchronizer)); + updateMetadataEventSynchronizer(metadataStoreConfig.getSynchronizer()); if ("local".equals(name)) { map = new TreeMap<>(); sequentialIdGenerator = new AtomicLong(); @@ -233,6 +232,12 @@ public Optional getMetadataEventSynchronizer() { return Optional.ofNullable(synchronizer); } + @Override + public void updateMetadataEventSynchronizer(MetadataEventSynchronizer synchronizer) { + this.synchronizer = synchronizer; + registerSyncListener(Optional.ofNullable(synchronizer)); + } + @Override public void close() throws Exception { if (isClosed.compareAndSet(false, true)) { diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImpl.java index dd4df69fc430b..cb7bea718e4be 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImpl.java @@ -22,6 +22,7 @@ import static org.apache.pulsar.metadata.impl.LocalMemoryMetadataStore.MEMORY_SCHEME_IDENTIFIER; import static org.apache.pulsar.metadata.impl.RocksdbMetadataStore.ROCKSDB_SCHEME_IDENTIFIER; import static org.apache.pulsar.metadata.impl.ZKMetadataStore.ZK_SCHEME_IDENTIFIER; +import static org.apache.pulsar.metadata.impl.oxia.OxiaMetadataStoreProvider.OXIA_SCHEME_IDENTIFIER; import com.google.common.base.Splitter; import java.util.HashMap; import java.util.Map; @@ -31,6 +32,7 @@ import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.MetadataStoreProvider; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.apache.pulsar.metadata.impl.oxia.OxiaMetadataStoreProvider; @Slf4j public class MetadataStoreFactoryImpl { @@ -66,6 +68,7 @@ static Map loadProviders() { providers.put(MEMORY_SCHEME_IDENTIFIER, new MemoryMetadataStoreProvider()); providers.put(ROCKSDB_SCHEME_IDENTIFIER, new RocksdbMetadataStoreProvider()); providers.put(ETCD_SCHEME_IDENTIFIER, new EtcdMetadataStoreProvider()); + providers.put(OXIA_SCHEME_IDENTIFIER, new OxiaMetadataStoreProvider()); providers.put(ZK_SCHEME_IDENTIFIER, new ZkMetadataStoreProvider()); String factoryClasses = System.getProperty(METADATASTORE_PROVIDERS_PROPERTY, ""); diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/PulsarZooKeeperClient.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/PulsarZooKeeperClient.java index cc29b615c1107..e8bfb39395a0e 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/PulsarZooKeeperClient.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/PulsarZooKeeperClient.java @@ -61,8 +61,10 @@ import org.apache.zookeeper.Watcher.Event.EventType; import org.apache.zookeeper.Watcher.Event.KeeperState; import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.client.ZKClientConfig; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Stat; +import org.apache.zookeeper.server.quorum.QuorumPeerConfig; /** * Provide a zookeeper client to handle session expire. @@ -92,6 +94,9 @@ public class PulsarZooKeeperClient extends ZooKeeper implements Watcher, AutoClo private final RetryPolicy connectRetryPolicy; private final RetryPolicy operationRetryPolicy; + // Zookeeper config path + private final String configPath; + // Stats Logger private final OpStatsLogger createStats; private final OpStatsLogger getStats; @@ -120,8 +125,9 @@ public ZooKeeper call() throws KeeperException, InterruptedException { ZooKeeper newZk; try { newZk = createZooKeeper(); - } catch (IOException ie) { - log.error("Failed to create zookeeper instance to " + connectString, ie); + } catch (IOException | QuorumPeerConfig.ConfigException e) { + log.error("Failed to create zookeeper instance to {} with config path {}", + connectString, configPath, e); throw KeeperException.create(KeeperException.Code.CONNECTIONLOSS); } waitForConnection(); @@ -149,7 +155,7 @@ public String toString() { static PulsarZooKeeperClient createConnectedZooKeeperClient( String connectString, int sessionTimeoutMs, Set childWatchers, RetryPolicy operationRetryPolicy) - throws KeeperException, InterruptedException, IOException { + throws KeeperException, InterruptedException, IOException, QuorumPeerConfig.ConfigException { return PulsarZooKeeperClient.newBuilder() .connectString(connectString) .sessionTimeoutMs(sessionTimeoutMs) @@ -171,6 +177,7 @@ public static class Builder { int retryExecThreadCount = DEFAULT_RETRY_EXECUTOR_THREAD_COUNT; double requestRateLimit = 0; boolean allowReadOnlyMode = false; + String configPath = null; private Builder() {} @@ -219,7 +226,15 @@ public Builder allowReadOnlyMode(boolean allowReadOnlyMode) { return this; } - public PulsarZooKeeperClient build() throws IOException, KeeperException, InterruptedException { + public Builder configPath(String configPath) { + this.configPath = configPath; + return this; + } + + public PulsarZooKeeperClient build() throws IOException, + KeeperException, + InterruptedException, + QuorumPeerConfig.ConfigException { requireNonNull(connectString); checkArgument(sessionTimeoutMs > 0); requireNonNull(statsLogger); @@ -251,7 +266,8 @@ public PulsarZooKeeperClient build() throws IOException, KeeperException, Interr statsLogger, retryExecThreadCount, requestRateLimit, - allowReadOnlyMode + allowReadOnlyMode, + configPath ); // Wait for connection to be established. try { @@ -273,16 +289,19 @@ public static Builder newBuilder() { } protected PulsarZooKeeperClient(String connectString, - int sessionTimeoutMs, - ZooKeeperWatcherBase watcherManager, - RetryPolicy connectRetryPolicy, - RetryPolicy operationRetryPolicy, - StatsLogger statsLogger, - int retryExecThreadCount, - double rate, - boolean allowReadOnlyMode) throws IOException { - super(connectString, sessionTimeoutMs, watcherManager, allowReadOnlyMode); + int sessionTimeoutMs, + ZooKeeperWatcherBase watcherManager, + RetryPolicy connectRetryPolicy, + RetryPolicy operationRetryPolicy, + StatsLogger statsLogger, + int retryExecThreadCount, + double rate, + boolean allowReadOnlyMode, + String configPath) throws IOException, QuorumPeerConfig.ConfigException { + super(connectString, sessionTimeoutMs, watcherManager, allowReadOnlyMode, + configPath == null ? null : new ZKClientConfig(configPath)); this.connectString = connectString; + this.configPath = configPath; this.sessionTimeoutMs = sessionTimeoutMs; this.allowReadOnlyMode = allowReadOnlyMode; this.watcherManager = watcherManager; @@ -334,7 +353,11 @@ public void waitForConnection() throws KeeperException, InterruptedException { watcherManager.waitForConnection(); } - protected ZooKeeper createZooKeeper() throws IOException { + protected ZooKeeper createZooKeeper() throws IOException, QuorumPeerConfig.ConfigException { + if (null != configPath) { + return new ZooKeeper(connectString, sessionTimeoutMs, watcherManager, allowReadOnlyMode, + new ZKClientConfig(configPath)); + } return new ZooKeeper(connectString, sessionTimeoutMs, watcherManager, allowReadOnlyMode); } diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/RocksdbMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/RocksdbMetadataStore.java index fcec2fcf9c1ef..20e3c4c2b27b2 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/RocksdbMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/RocksdbMetadataStore.java @@ -112,8 +112,7 @@ public static RocksdbMetadataStore get(String metadataStoreUri, MetadataStoreCon // Create a new store instance store = new RocksdbMetadataStore(metadataStoreUri, conf); // update synchronizer and register sync listener - store.synchronizer = conf.getSynchronizer(); - store.registerSyncLister(Optional.ofNullable(store.synchronizer)); + store.updateMetadataEventSynchronizer(conf.getSynchronizer()); instancesCache.put(metadataStoreUri, store); return store; } @@ -210,7 +209,7 @@ static long toLong(byte[] bytes) { */ private RocksdbMetadataStore(String metadataURL, MetadataStoreConfig metadataStoreConfig) throws MetadataStoreException { - super(metadataStoreConfig.getMetadataStoreName()); + super(metadataStoreConfig.getMetadataStoreName(), metadataStoreConfig.getOpenTelemetry()); this.metadataUrl = metadataURL; try { RocksDB.loadLibrary(); @@ -376,6 +375,9 @@ public CompletableFuture> storeGet(String path) { } try { dbStateLock.readLock().lock(); + if (isClosed()) { + return alreadyClosedFailedFuture(); + } byte[] value = db.get(optionCache, toBytes(path)); if (value == null) { return CompletableFuture.completedFuture(Optional.empty()); @@ -408,6 +410,9 @@ protected CompletableFuture> getChildrenFromStore(String path) { } try { dbStateLock.readLock().lock(); + if (isClosed()) { + return alreadyClosedFailedFuture(); + } try (RocksIterator iterator = db.newIterator(optionDontCache)) { Set result = new HashSet<>(); String firstKey = path.equals("/") ? path : path + "/"; @@ -450,6 +455,9 @@ protected CompletableFuture existsFromStore(String path) { } try { dbStateLock.readLock().lock(); + if (isClosed()) { + return alreadyClosedFailedFuture(); + } byte[] value = db.get(optionDontCache, toBytes(path)); if (log.isDebugEnabled()) { if (value != null) { @@ -472,6 +480,9 @@ protected CompletableFuture storeDelete(String path, Optional expect } try { dbStateLock.readLock().lock(); + if (isClosed()) { + return alreadyClosedFailedFuture(); + } try (Transaction transaction = db.beginTransaction(writeOptions)) { byte[] pathBytes = toBytes(path); byte[] oldValueData = transaction.getForUpdate(optionDontCache, pathBytes, true); @@ -508,6 +519,9 @@ protected CompletableFuture storePut(String path, byte[] data, Optional storePut(String path, byte[] data, Optional getMetadataEventSynchronizer() { return Optional.ofNullable(synchronizer); } + + @Override + public void updateMetadataEventSynchronizer(MetadataEventSynchronizer synchronizer) { + this.synchronizer = synchronizer; + registerSyncListener(Optional.ofNullable(synchronizer)); + } } class RocksdbMetadataStoreProvider implements MetadataStoreProvider { diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java index a6d8eb8344c96..603a4503dc8bb 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java @@ -100,6 +100,7 @@ public ZKMetadataStore(String metadataURL, MetadataStoreConfig metadataStoreConf .allowReadOnlyMode(metadataStoreConfig.isAllowReadOnlyOperations()) .sessionTimeoutMs(metadataStoreConfig.getSessionTimeoutMillis()) .watchers(Collections.singleton(this::processSessionWatcher)) + .configPath(metadataStoreConfig.getConfigFilePath()) .build(); if (enableSessionWatcher) { sessionWatcher = new ZKSessionWatcher(zkc, this::receivedSessionEvent); @@ -192,7 +193,20 @@ protected void batchOperation(List ops) { Code code = Code.get(rc); if (code == Code.CONNECTIONLOSS) { // There is the chance that we caused a connection reset by sending or requesting a batch - // that passed the max ZK limit. Retry with the individual operations + // that passed the max ZK limit. + + // Build the log warning message + // summarize the operations by type + String countsByType = ops.stream().collect( + Collectors.groupingBy(MetadataOp::getType, Collectors.summingInt(op -> 1))) + .entrySet().stream().map(e -> e.getValue() + " " + e.getKey().name() + " entries") + .collect(Collectors.joining(", ")); + Long totalSize = ops.stream().collect(Collectors.summingLong(MetadataOp::size)); + log.warn("Connection loss while executing batch operation of {} " + + "of total data size of {}. " + + "Retrying individual operations one-by-one.", countsByType, totalSize); + + // Retry with the individual operations executor.schedule(() -> { ops.forEach(o -> batchOperation(Collections.singletonList(o))); }, 100, TimeUnit.MILLISECONDS); @@ -204,29 +218,29 @@ protected void batchOperation(List ops) { } // Trigger all the futures in the batch - for (int i = 0; i < ops.size(); i++) { - OpResult opr = results.get(i); - MetadataOp op = ops.get(i); - - switch (op.getType()) { - case PUT: - handlePutResult(op.asPut(), opr); - break; - case DELETE: - handleDeleteResult(op.asDelete(), opr); - break; - case GET: - handleGetResult(op.asGet(), opr); - break; - case GET_CHILDREN: - handleGetChildrenResult(op.asGetChildren(), opr); - break; - - default: - op.getFuture().completeExceptionally(new MetadataStoreException( - "Operation type not supported in multi: " + op.getType())); - } - } + execute(() -> { + for (int i = 0; i < ops.size(); i++) { + OpResult opr = results.get(i); + MetadataOp op = ops.get(i); + switch (op.getType()) { + case PUT: + handlePutResult(op.asPut(), opr); + break; + case DELETE: + handleDeleteResult(op.asDelete(), opr); + break; + case GET: + handleGetResult(op.asGet(), opr); + break; + case GET_CHILDREN: + handleGetChildrenResult(op.asGetChildren(), opr); + break; + default: + op.getFuture().completeExceptionally(new MetadataStoreException( + "Operation type not supported in multi: " + op.getType())); + } + } + }, () -> ops.stream().map(MetadataOp::getFuture).collect(Collectors.toList())); }, null); } catch (Throwable t) { ops.forEach(o -> o.getFuture().completeExceptionally(new MetadataStoreException(t))); @@ -564,6 +578,7 @@ public CompletableFuture initializeCluster() { .connectRetryPolicy( new BoundExponentialBackoffRetryPolicy(metadataStoreConfig.getSessionTimeoutMillis(), metadataStoreConfig.getSessionTimeoutMillis(), 0)) + .configPath(metadataStoreConfig.getConfigFilePath()) .build()) { if (chrootZk.exists(chrootPath, false) == null) { createFullPathOptimistic(chrootZk, chrootPath, new byte[0], CreateMode.PERSISTENT); diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/batching/AbstractBatchedMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/batching/AbstractBatchedMetadataStore.java index 93aeb28c39bf1..4275920d7f954 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/batching/AbstractBatchedMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/batching/AbstractBatchedMetadataStore.java @@ -52,11 +52,11 @@ public abstract class AbstractBatchedMetadataStore extends AbstractMetadataStore private final int maxDelayMillis; private final int maxOperations; private final int maxSize; - private final MetadataEventSynchronizer synchronizer; + private MetadataEventSynchronizer synchronizer; private final BatchMetadataStoreStats batchMetadataStoreStats; protected AbstractBatchedMetadataStore(MetadataStoreConfig conf) { - super(conf.getMetadataStoreName()); + super(conf.getMetadataStoreName(), conf.getOpenTelemetry()); this.enabled = conf.isBatchingEnabled(); this.maxDelayMillis = conf.getBatchingMaxDelayMillis(); @@ -75,10 +75,9 @@ protected AbstractBatchedMetadataStore(MetadataStoreConfig conf) { } // update synchronizer and register sync listener - synchronizer = conf.getSynchronizer(); - registerSyncLister(Optional.ofNullable(synchronizer)); + updateMetadataEventSynchronizer(conf.getSynchronizer()); this.batchMetadataStoreStats = - new BatchMetadataStoreStats(metadataStoreName, executor); + new BatchMetadataStoreStats(metadataStoreName, executor, conf.getOpenTelemetry()); } @Override @@ -87,9 +86,13 @@ public void close() throws Exception { // Fail all the pending items MetadataStoreException ex = new MetadataStoreException.AlreadyClosedException("Metadata store is getting closed"); - readOps.drain(op -> op.getFuture().completeExceptionally(ex)); - writeOps.drain(op -> op.getFuture().completeExceptionally(ex)); - + MetadataOp op; + while ((op = readOps.poll()) != null) { + op.getFuture().completeExceptionally(ex); + } + while ((op = writeOps.poll()) != null) { + op.getFuture().completeExceptionally(ex); + } scheduledTask.cancel(true); } super.close(); @@ -99,7 +102,13 @@ public void close() throws Exception { private void flush() { while (!readOps.isEmpty()) { List ops = new ArrayList<>(); - readOps.drain(ops::add, maxOperations); + for (int i = 0; i < maxOperations; i++) { + MetadataOp op = readOps.poll(); + if (op == null) { + break; + } + ops.add(op); + } internalBatchOperation(ops); } @@ -161,7 +170,18 @@ public Optional getMetadataEventSynchronizer() { return Optional.ofNullable(synchronizer); } + @Override + public void updateMetadataEventSynchronizer(MetadataEventSynchronizer synchronizer) { + this.synchronizer = synchronizer; + registerSyncListener(Optional.ofNullable(synchronizer)); + } + private void enqueue(MessagePassingQueue queue, MetadataOp op) { + if (isClosed()) { + MetadataStoreException ex = new MetadataStoreException.AlreadyClosedException(); + op.getFuture().completeExceptionally(ex); + return; + } if (enabled) { if (!queue.offer(op)) { // Execute individually if we're failing to enqueue @@ -177,6 +197,12 @@ private void enqueue(MessagePassingQueue queue, MetadataOp op) { } private void internalBatchOperation(List ops) { + if (isClosed()) { + MetadataStoreException ex = + new MetadataStoreException.AlreadyClosedException(); + ops.forEach(op -> op.getFuture().completeExceptionally(ex)); + return; + } long now = System.currentTimeMillis(); for (MetadataOp op : ops) { this.batchMetadataStoreStats.recordOpWaiting(now - op.created()); diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/OxiaMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/OxiaMetadataStore.java new file mode 100644 index 0000000000000..27cd4a2d2f60b --- /dev/null +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/OxiaMetadataStore.java @@ -0,0 +1,317 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.metadata.impl.oxia; + +import io.opentelemetry.api.OpenTelemetry; +import io.streamnative.oxia.client.api.AsyncOxiaClient; +import io.streamnative.oxia.client.api.DeleteOption; +import io.streamnative.oxia.client.api.Notification; +import io.streamnative.oxia.client.api.OxiaClientBuilder; +import io.streamnative.oxia.client.api.PutOption; +import io.streamnative.oxia.client.api.PutResult; +import io.streamnative.oxia.client.api.Version; +import io.streamnative.oxia.client.api.exceptions.KeyAlreadyExistsException; +import io.streamnative.oxia.client.api.exceptions.UnexpectedVersionIdException; +import java.time.Duration; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.metadata.api.GetResult; +import org.apache.pulsar.metadata.api.MetadataEventSynchronizer; +import org.apache.pulsar.metadata.api.MetadataStoreConfig; +import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.metadata.api.NotificationType; +import org.apache.pulsar.metadata.api.Stat; +import org.apache.pulsar.metadata.api.extended.CreateOption; +import org.apache.pulsar.metadata.impl.AbstractMetadataStore; + +@Slf4j +public class OxiaMetadataStore extends AbstractMetadataStore { + + private final AsyncOxiaClient client; + + private final String identity; + private Optional synchronizer; + + public OxiaMetadataStore(AsyncOxiaClient oxia, String identity) { + super("oxia-metadata", OpenTelemetry.noop()); + this.client = oxia; + this.identity = identity; + this.synchronizer = Optional.empty(); + init(); + } + + public OxiaMetadataStore( + @NonNull String serviceAddress, + @NonNull String namespace, + MetadataStoreConfig metadataStoreConfig, + boolean enableSessionWatcher) + throws Exception { + super("oxia-metadata", Objects.requireNonNull(metadataStoreConfig).getOpenTelemetry()); + + var linger = metadataStoreConfig.getBatchingMaxDelayMillis(); + if (!metadataStoreConfig.isBatchingEnabled()) { + linger = 0; + } + synchronizer = Optional.ofNullable(metadataStoreConfig.getSynchronizer()); + identity = UUID.randomUUID().toString(); + OxiaClientBuilder oxiaClientBuilder = OxiaClientBuilder + .create(serviceAddress) + .clientIdentifier(identity) + .namespace(namespace) + .sessionTimeout(Duration.ofMillis(metadataStoreConfig.getSessionTimeoutMillis())) + .batchLinger(Duration.ofMillis(linger)) + .maxRequestsPerBatch(metadataStoreConfig.getBatchingMaxOperations()); + if (StringUtils.isNotBlank(metadataStoreConfig.getConfigFilePath())) { + oxiaClientBuilder.loadConfig(metadataStoreConfig.getConfigFilePath()); + } + client = oxiaClientBuilder.asyncClient().get(); + init(); + } + + private void init() { + updateMetadataEventSynchronizer(synchronizer.orElse(null)); + + client.notifications(this::notificationCallback); + super.registerSyncListener(synchronizer); + } + + private void notificationCallback(Notification notification) { + if (notification instanceof Notification.KeyCreated keyCreated) { + receivedNotification( + new org.apache.pulsar.metadata.api.Notification( + NotificationType.Created, keyCreated.key())); + notifyParentChildrenChanged(keyCreated.key()); + + } else if (notification instanceof Notification.KeyModified keyModified) { + receivedNotification( + new org.apache.pulsar.metadata.api.Notification( + NotificationType.Modified, keyModified.key())); + } else if (notification instanceof Notification.KeyDeleted keyDeleted) { + receivedNotification( + new org.apache.pulsar.metadata.api.Notification( + NotificationType.Deleted, keyDeleted.key())); + notifyParentChildrenChanged(keyDeleted.key()); + } else { + log.error("Unknown notification type {}", notification); + } + } + + Optional convertGetResult( + String path, io.streamnative.oxia.client.api.GetResult result) { + if (result == null) { + return Optional.empty(); + } + return Optional.of(result) + .map( + oxiaResult -> + new GetResult(oxiaResult.getValue(), convertStat(path, oxiaResult.getVersion()))); + } + + Stat convertStat(String path, Version version) { + return new Stat( + path, + version.versionId(), + version.createdTimestamp(), + version.modifiedTimestamp(), + version.sessionId().isPresent(), + version.clientIdentifier().stream().anyMatch(identity::equals), + version.modificationsCount() == 0); + } + + @Override + protected CompletableFuture> getChildrenFromStore(String path) { + var pathWithSlash = path + "/"; + + return client + .list(pathWithSlash, pathWithSlash + "/") + .thenApply( + children -> + children.stream().map(child -> child.substring(pathWithSlash.length())).toList()) + .exceptionallyCompose(this::convertException); + } + + @Override + protected CompletableFuture existsFromStore(String path) { + return client.get(path).thenApply(Objects::nonNull) + .exceptionallyCompose(this::convertException); + } + + @Override + protected CompletableFuture> storeGet(String path) { + return client.get(path).thenApply(res -> convertGetResult(path, res)) + .exceptionallyCompose(this::convertException); + } + + @Override + protected CompletableFuture storeDelete(String path, Optional expectedVersion) { + return getChildrenFromStore(path) + .thenCompose( + children -> { + if (!children.isEmpty()) { + return CompletableFuture.failedFuture( + new MetadataStoreException("Key '" + path + "' has children")); + } else { + Set delOption = + expectedVersion + .map(v -> Collections.singleton(DeleteOption.IfVersionIdEquals(v))) + .orElse(Collections.emptySet()); + CompletableFuture result = client.delete(path, delOption); + return result + .thenCompose( + exists -> { + if (!exists) { + return CompletableFuture.failedFuture( + new MetadataStoreException.NotFoundException( + "Key '" + path + "' does not exist")); + } + return CompletableFuture.completedFuture((Void) null); + }) + .exceptionallyCompose(this::convertException); + } + }); + } + + @Override + protected CompletableFuture storePut( + String path, byte[] data, Optional optExpectedVersion, EnumSet options) { + CompletableFuture parentsCreated = createParents(path); + return parentsCreated.thenCompose( + __ -> { + var expectedVersion = optExpectedVersion; + if (expectedVersion.isPresent() + && expectedVersion.get() != -1L + && options.contains(CreateOption.Sequential)) { + return CompletableFuture.failedFuture( + new MetadataStoreException( + "Can't have expectedVersion and Sequential at the same time")); + } + CompletableFuture actualPath; + if (options.contains(CreateOption.Sequential)) { + var parent = parent(path); + var parentPath = parent == null ? "/" : parent; + + actualPath = + client + .put(parentPath, new byte[] {}) + .thenApply( + r -> String.format("%s%010d", path, r.version().modificationsCount())); + expectedVersion = Optional.of(-1L); + } else { + actualPath = CompletableFuture.completedFuture(path); + } + Set putOptions = new HashSet<>(); + expectedVersion + .map( + ver -> { + if (ver == -1) { + return PutOption.IfRecordDoesNotExist; + } + return PutOption.IfVersionIdEquals(ver); + }) + .ifPresent(putOptions::add); + + if (options.contains(CreateOption.Ephemeral)) { + putOptions.add(PutOption.AsEphemeralRecord); + } + return actualPath + .thenCompose( + aPath -> + client + .put(aPath, data, putOptions) + .thenApply(res -> new PathWithPutResult(aPath, res))) + .thenApply(res -> convertStat(res.path(), res.result().version())) + .exceptionallyCompose(this::convertException); + }); + } + + private CompletionStage convertException(Throwable ex) { + Throwable actEx = FutureUtil.unwrapCompletionException(ex); + if (actEx instanceof UnexpectedVersionIdException || actEx instanceof KeyAlreadyExistsException) { + return CompletableFuture.failedFuture( + new MetadataStoreException.BadVersionException(actEx)); + } else if (actEx instanceof IllegalStateException) { + return CompletableFuture.failedFuture(new MetadataStoreException.AlreadyClosedException(actEx)); + } else if (actEx instanceof MetadataStoreException) { + return CompletableFuture.failedFuture(actEx); + } else { + return CompletableFuture.failedFuture(new MetadataStoreException(actEx)); + } + } + + private static final byte[] EMPTY_VALUE = new byte[0]; + private static final Set IF_RECORD_DOES_NOT_EXIST = + Collections.singleton(PutOption.IfRecordDoesNotExist); + + private CompletableFuture createParents(String path) { + var parent = parent(path); + if (parent == null || parent.isEmpty()) { + return CompletableFuture.completedFuture(null); + } + return exists(parent) + .thenCompose( + exists -> { + if (exists) { + return CompletableFuture.completedFuture(null); + } else { + return client + .put(parent, EMPTY_VALUE, IF_RECORD_DOES_NOT_EXIST) + .thenCompose(__ -> createParents(parent)); + } + }) + .exceptionallyCompose( + ex -> { + if (ex.getCause() instanceof KeyAlreadyExistsException) { + return CompletableFuture.completedFuture(null); + } + return CompletableFuture.failedFuture(ex.getCause()); + }); + } + + @Override + public void close() throws Exception { + if (client != null) { + client.close(); + } + super.close(); + } + + public Optional getMetadataEventSynchronizer() { + return synchronizer; + } + + @Override + public void updateMetadataEventSynchronizer(MetadataEventSynchronizer synchronizer) { + this.synchronizer = Optional.ofNullable(synchronizer); + registerSyncListener(this.synchronizer); + } + + private record PathWithPutResult(String path, PutResult result) {} +} diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/OxiaMetadataStoreProvider.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/OxiaMetadataStoreProvider.java new file mode 100644 index 0000000000000..a4c52134a8a75 --- /dev/null +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/OxiaMetadataStoreProvider.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.metadata.impl.oxia; + +import lombok.NonNull; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.metadata.api.MetadataStore; +import org.apache.pulsar.metadata.api.MetadataStoreConfig; +import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.metadata.api.MetadataStoreProvider; + +public class OxiaMetadataStoreProvider implements MetadataStoreProvider { + // declare the specific namespace to avoid any changes in the future. + public static final String DefaultNamespace = "default"; + + public static final String OXIA_SCHEME = "oxia"; + public static final String OXIA_SCHEME_IDENTIFIER = OXIA_SCHEME + ":"; + + @Override + public String urlScheme() { + return OXIA_SCHEME; + } + + @Override + public @NonNull MetadataStore create( + String metadataURL, MetadataStoreConfig metadataStoreConfig, boolean enableSessionWatcher) + throws MetadataStoreException { + var serviceAddress = getServiceAddressAndNamespace(metadataURL); + try { + return new OxiaMetadataStore( + serviceAddress.getLeft(), + serviceAddress.getRight(), + metadataStoreConfig, + enableSessionWatcher); + } catch (Exception e) { + throw new MetadataStoreException(e); + } + } + + @NonNull + Pair getServiceAddressAndNamespace(String metadataURL) + throws MetadataStoreException { + if (metadataURL == null || !metadataURL.startsWith(urlScheme() + "://")) { + throw new MetadataStoreException("Invalid metadata URL. Must start with 'oxia://'."); + } + final var addressWithNamespace = metadataURL.substring("oxia://".length()); + final var split = addressWithNamespace.split("/"); + if (split.length > 2) { + throw new MetadataStoreException( + "Invalid metadata URL." + + " the oxia metadata format should be 'oxia://host:port/[namespace]'."); + } + if (split.length == 1) { + // Use default namespace + return Pair.of(split[0], DefaultNamespace); + } + return Pair.of(split[0], split[1]); + } +} diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/package-info.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/package-info.java new file mode 100644 index 0000000000000..d63afa5b0a8f0 --- /dev/null +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.metadata.impl.oxia; diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/stats/BatchMetadataStoreStats.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/stats/BatchMetadataStoreStats.java index f87155b9259be..9549a8df8f9f1 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/stats/BatchMetadataStoreStats.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/stats/BatchMetadataStoreStats.java @@ -18,6 +18,9 @@ */ package org.apache.pulsar.metadata.impl.stats; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.ObservableLongUpDownCounter; import io.prometheus.client.Gauge; import io.prometheus.client.Histogram; import java.util.concurrent.ExecutorService; @@ -58,7 +61,10 @@ public final class BatchMetadataStoreStats implements AutoCloseable { private final Histogram.Child batchExecuteTimeChild; private final Histogram.Child opsPerBatchChild; - public BatchMetadataStoreStats(String metadataStoreName, ExecutorService executor) { + public static final String EXECUTOR_QUEUE_SIZE_METRIC_NAME = "pulsar.broker.metadata.store.executor.queue.size"; + private final ObservableLongUpDownCounter batchMetadataStoreSizeCounter; + + public BatchMetadataStoreStats(String metadataStoreName, ExecutorService executor, OpenTelemetry openTelemetry) { if (executor instanceof ThreadPoolExecutor tx) { this.executor = tx; } else { @@ -69,8 +75,7 @@ public BatchMetadataStoreStats(String metadataStoreName, ExecutorService executo EXECUTOR_QUEUE_SIZE.setChild(new Gauge.Child() { @Override public double get() { - return BatchMetadataStoreStats.this.executor == null ? 0 : - BatchMetadataStoreStats.this.executor.getQueue().size(); + return getQueueSize(); } }, metadataStoreName); @@ -78,6 +83,17 @@ public double get() { this.batchExecuteTimeChild = BATCH_EXECUTE_TIME.labels(metadataStoreName); this.opsPerBatchChild = OPS_PER_BATCH.labels(metadataStoreName); + var meter = openTelemetry.getMeter("org.apache.pulsar"); + var attributes = Attributes.of(MetadataStoreStats.METADATA_STORE_NAME, metadataStoreName); + this.batchMetadataStoreSizeCounter = meter + .upDownCounterBuilder(EXECUTOR_QUEUE_SIZE_METRIC_NAME) + .setDescription("The number of batch operations in the metadata store executor queue") + .setUnit("{operation}") + .buildWithCallback(measurement -> measurement.record(getQueueSize(), attributes)); + } + + private int getQueueSize() { + return executor == null ? 0 : executor.getQueue().size(); } public void recordOpWaiting(long millis) { @@ -99,6 +115,7 @@ public void close() throws Exception { OPS_WAITING.remove(this.metadataStoreName); BATCH_EXECUTE_TIME.remove(this.metadataStoreName); OPS_PER_BATCH.remove(metadataStoreName); + batchMetadataStoreSizeCounter.close(); } } } diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/stats/MetadataStoreStats.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/stats/MetadataStoreStats.java index 45024a68383bd..5f0383f9520a7 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/stats/MetadataStoreStats.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/stats/MetadataStoreStats.java @@ -18,6 +18,10 @@ */ package org.apache.pulsar.metadata.impl.stats; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.LongCounter; import io.prometheus.client.Counter; import io.prometheus.client.Histogram; import java.util.concurrent.atomic.AtomicBoolean; @@ -48,6 +52,12 @@ public final class MetadataStoreStats implements AutoCloseable { .labelNames(METADATA_STORE_LABEL_NAME) .register(); + public static final AttributeKey METADATA_STORE_NAME = AttributeKey.stringKey("pulsar.metadata.store.name"); + public static final String METADATA_STORE_PUT_BYTES_COUNTER_METRIC_NAME = + "pulsar.broker.metadata.store.outgoing.size"; + private final Attributes attributes; + private final LongCounter putBytesCounter; + private final Histogram.Child getOpsSucceedChild; private final Histogram.Child delOpsSucceedChild; private final Histogram.Child putOpsSucceedChild; @@ -58,7 +68,7 @@ public final class MetadataStoreStats implements AutoCloseable { private final String metadataStoreName; private final AtomicBoolean closed = new AtomicBoolean(false); - public MetadataStoreStats(String metadataStoreName) { + public MetadataStoreStats(String metadataStoreName, OpenTelemetry openTelemetry) { this.metadataStoreName = metadataStoreName; this.getOpsSucceedChild = OPS_LATENCY.labels(metadataStoreName, OPS_TYPE_GET, STATUS_SUCCESS); @@ -68,6 +78,13 @@ public MetadataStoreStats(String metadataStoreName) { this.delOpsFailedChild = OPS_LATENCY.labels(metadataStoreName, OPS_TYPE_DEL, STATUS_FAIL); this.putOpsFailedChild = OPS_LATENCY.labels(metadataStoreName, OPS_TYPE_PUT, STATUS_FAIL); this.putBytesChild = PUT_BYTES.labels(metadataStoreName); + + attributes = Attributes.of(METADATA_STORE_NAME, metadataStoreName); + putBytesCounter = openTelemetry.getMeter("org.apache.pulsar") + .counterBuilder(METADATA_STORE_PUT_BYTES_COUNTER_METRIC_NAME) + .setDescription("The total amount of data written to the metadata store") + .setUnit("{By}") + .build(); } public void recordGetOpsSucceeded(long millis) { @@ -81,6 +98,7 @@ public void recordDelOpsSucceeded(long millis) { public void recordPutOpsSucceeded(long millis, int bytes) { this.putOpsSucceedChild.observe(millis); this.putBytesChild.inc(bytes); + this.putBytesCounter.add(bytes, attributes); } public void recordGetOpsFailed(long millis) { diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/tableview/impl/MetadataStoreTableViewImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/tableview/impl/MetadataStoreTableViewImpl.java new file mode 100644 index 0000000000000..4f9aad0ba658b --- /dev/null +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/tableview/impl/MetadataStoreTableViewImpl.java @@ -0,0 +1,342 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.metadata.tableview.impl; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.LongAdder; +import java.util.function.BiConsumer; +import java.util.function.BiPredicate; +import java.util.function.Predicate; +import lombok.Builder; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.mutable.MutableBoolean; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.metadata.api.CacheGetResult; +import org.apache.pulsar.metadata.api.MetadataCache; +import org.apache.pulsar.metadata.api.MetadataCacheConfig; +import org.apache.pulsar.metadata.api.MetadataStore; +import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.metadata.api.MetadataStoreTableView; +import org.apache.pulsar.metadata.api.NotificationType; + +@Slf4j +public class MetadataStoreTableViewImpl implements MetadataStoreTableView { + + private static final int FILL_TIMEOUT_IN_MILLIS = 300_000; + private static final int MAX_CONCURRENT_METADATA_OPS_DURING_FILL = 50; + private static final long CACHE_REFRESH_FREQUENCY_IN_MILLIS = 600_000; + private final ConcurrentMap data; + private final Map immutableData; + private final String name; + private final MetadataStore store; + private final MetadataCache cache; + private final Predicate listenPathValidator; + private final BiPredicate conflictResolver; + private final List> tailItemListeners; + private final List> existingItemListeners; + private final long timeoutInMillis; + private final String pathPrefix; + + /** + * Construct MetadataStoreTableViewImpl. + * + * @param clazz clazz of the value type + * @param name metadata store tableview name + * @param store metadata store + * @param pathPrefix metadata store path prefix + * @param listenPathValidator path validator to listen + * @param conflictResolver resolve conflicts for concurrent puts + * @param tailItemListeners listener for tail item(recently updated) notifications + * @param existingItemListeners listener for existing items in metadata store + * @param timeoutInMillis timeout duration for each sync operation. + * @throws MetadataStoreException if init fails. + */ + @Builder + public MetadataStoreTableViewImpl(@NonNull Class clazz, + @NonNull String name, + @NonNull MetadataStore store, + @NonNull String pathPrefix, + @NonNull BiPredicate conflictResolver, + Predicate listenPathValidator, + List> tailItemListeners, + List> existingItemListeners, + long timeoutInMillis) { + this.name = name; + this.data = new ConcurrentHashMap<>(); + this.immutableData = Collections.unmodifiableMap(data); + this.pathPrefix = pathPrefix; + this.conflictResolver = conflictResolver; + this.listenPathValidator = listenPathValidator; + this.tailItemListeners = new ArrayList<>(); + if (tailItemListeners != null) { + this.tailItemListeners.addAll(tailItemListeners); + } + this.existingItemListeners = new ArrayList<>(); + if (existingItemListeners != null) { + this.existingItemListeners.addAll(existingItemListeners); + } + this.timeoutInMillis = timeoutInMillis; + this.store = store; + this.cache = store.getMetadataCache(clazz, + MetadataCacheConfig.builder() + .expireAfterWriteMillis(-1) + .refreshAfterWriteMillis(CACHE_REFRESH_FREQUENCY_IN_MILLIS) + .asyncReloadConsumer(this::consumeAsyncReload) + .build()); + store.registerListener(this::handleNotification); + } + + public void start() throws MetadataStoreException { + fill(); + } + + private void consumeAsyncReload(String path, Optional> cached) { + if (!isValidPath(path)) { + return; + } + String key = getKey(path); + var val = getValue(cached); + handleTailItem(key, val); + } + + private boolean isValidPath(String path) { + if (listenPathValidator != null && !listenPathValidator.test(path)) { + return false; + } + return true; + } + + private T getValue(Optional> cached) { + return cached.map(CacheGetResult::getValue).orElse(null); + } + + boolean updateData(String key, T cur) { + MutableBoolean updated = new MutableBoolean(); + data.compute(key, (k, prev) -> { + if (Objects.equals(prev, cur)) { + if (log.isDebugEnabled()) { + log.debug("{} skipped item key={} value={} prev={}", + name, key, cur, prev); + } + updated.setValue(false); + return prev; + } else { + updated.setValue(true); + return cur; + } + }); + return updated.booleanValue(); + } + + private void handleTailItem(String key, T val) { + if (updateData(key, val)) { + if (log.isDebugEnabled()) { + log.debug("{} applying item key={} value={}", + name, + key, + val); + } + for (var listener : tailItemListeners) { + try { + listener.accept(key, val); + } catch (Throwable e) { + log.error("{} failed to listen tail item key:{}, val:{}", + name, + key, val, e); + } + } + } + + } + + private CompletableFuture doHandleNotification(String path) { + if (!isValidPath(path)) { + return CompletableFuture.completedFuture(null); + } + return cache.get(path).thenAccept(valOpt -> { + String key = getKey(path); + var val = valOpt.orElse(null); + handleTailItem(key, val); + }).exceptionally(e -> { + log.error("{} failed to handle notification for path:{}", name, path, e); + return null; + }); + } + + private void handleNotification(org.apache.pulsar.metadata.api.Notification notification) { + + if (notification.getType() == NotificationType.ChildrenChanged) { + return; + } + + String path = notification.getPath(); + + doHandleNotification(path); + } + + + private CompletableFuture handleExisting(String path) { + if (!isValidPath(path)) { + return CompletableFuture.completedFuture(null); + } + return cache.get(path) + .thenAccept(valOpt -> { + valOpt.ifPresent(val -> { + String key = getKey(path); + updateData(key, val); + if (log.isDebugEnabled()) { + log.debug("{} applying existing item key={} value={}", + name, + key, + val); + } + for (var listener : existingItemListeners) { + try { + listener.accept(key, val); + } catch (Throwable e) { + log.error("{} failed to listen existing item key:{}, val:{}", name, key, val, + e); + throw e; + } + } + }); + }); + } + + private void fill() throws MetadataStoreException { + final var deadline = System.currentTimeMillis() + FILL_TIMEOUT_IN_MILLIS; + log.info("{} start filling existing items under the pathPrefix:{}", name, pathPrefix); + ConcurrentLinkedDeque q = new ConcurrentLinkedDeque<>(); + List> futures = new ArrayList<>(); + q.add(pathPrefix); + LongAdder count = new LongAdder(); + while (!q.isEmpty()) { + var now = System.currentTimeMillis(); + if (now >= deadline) { + String err = name + " failed to fill existing items in " + + TimeUnit.MILLISECONDS.toSeconds(FILL_TIMEOUT_IN_MILLIS) + " secs. Filled count:" + + count.sum(); + log.error(err); + throw new MetadataStoreException(err); + } + int size = Math.min(MAX_CONCURRENT_METADATA_OPS_DURING_FILL, q.size()); + for (int i = 0; i < size; i++) { + String path = q.poll(); + futures.add(store.getChildren(path) + .thenCompose(children -> { + // The path is leaf + if (children.isEmpty()) { + count.increment(); + return handleExisting(path); + } else { + for (var child : children) { + q.add(path + "/" + child); + } + return CompletableFuture.completedFuture(null); + } + })); + } + try { + FutureUtil.waitForAll(futures).get( + Math.min(timeoutInMillis, deadline - now), + TimeUnit.MILLISECONDS); + } catch (Throwable e) { + Throwable c = FutureUtil.unwrapCompletionException(e); + log.error("{} failed to fill existing items", name, c); + throw new MetadataStoreException(c); + } + futures.clear(); + } + log.info("{} completed filling existing items with size:{}", name, count.sum()); + } + + + private String getPath(String key) { + return pathPrefix + "/" + key; + } + + private String getKey(String path) { + return path.replaceFirst(pathPrefix + "/", ""); + } + + public boolean exists(String key) { + return immutableData.containsKey(key); + } + + public T get(String key) { + return data.get(key); + } + + public CompletableFuture put(String key, T value) { + String path = getPath(key); + return cache.readModifyUpdateOrCreate(path, (old) -> { + if (conflictResolver.test(old.orElse(null), value)) { + return value; + } else { + throw new ConflictException( + String.format("Failed to update from old:%s to value:%s", old, value)); + } + }).thenCompose(__ -> doHandleNotification(path)); // immediately notify local tableview + } + + public CompletableFuture delete(String key) { + String path = getPath(key); + return cache.delete(path) + .thenCompose(__ -> doHandleNotification(path)); // immediately notify local tableview + } + + public int size() { + return immutableData.size(); + } + + public boolean isEmpty() { + return immutableData.isEmpty(); + } + + public Set> entrySet() { + return immutableData.entrySet(); + } + + public Set keySet() { + return immutableData.keySet(); + } + + public Collection values() { + return immutableData.values(); + } + + public void forEach(BiConsumer action) { + immutableData.forEach(action); + } + +} diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/tableview/impl/package-info.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/tableview/impl/package-info.java new file mode 100644 index 0000000000000..2c47770610b05 --- /dev/null +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/tableview/impl/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.metadata.tableview.impl; diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieCheckTaskTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieCheckTaskTest.java new file mode 100644 index 0000000000000..d151226992f3c --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieCheckTaskTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.AssertJUnit.assertTrue; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeperAdmin; +import org.apache.bookkeeper.client.api.LedgerMetadata; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.stats.OpStatsLogger; +import org.apache.bookkeeper.test.TestStatsProvider; +import org.apache.bookkeeper.versioning.LongVersion; +import org.apache.bookkeeper.versioning.Versioned; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Unit test {@link AuditorBookieCheckTask}. + */ +public class AuditorBookieCheckTaskTest { + + private AuditorStats auditorStats; + private BookKeeperAdmin admin; + private LedgerManager ledgerManager; + private LedgerUnderreplicationManager underreplicationManager; + private BookieLedgerIndexer ledgerIndexer; + private AuditorBookieCheckTask bookieCheckTask; + private final AtomicBoolean shutdownCompleted = new AtomicBoolean(false); + private final AuditorTask.ShutdownTaskHandler shutdownTaskHandler = () -> shutdownCompleted.set(true); + private long startLedgerId = 0; + + @BeforeMethod + public void setup() { + ServerConfiguration conf = mock(ServerConfiguration.class); + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsProvider.TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + final AuditorStats auditorStats = new AuditorStats(statsLogger); + this.auditorStats = spy(auditorStats); + admin = mock(BookKeeperAdmin.class); + ledgerManager = mock(LedgerManager.class); + underreplicationManager = mock(LedgerUnderreplicationManager.class); + ledgerIndexer = mock(BookieLedgerIndexer.class); + AuditorBookieCheckTask bookieCheckTask1 = new AuditorBookieCheckTask( + conf, this.auditorStats, admin, ledgerManager, underreplicationManager, + shutdownTaskHandler, ledgerIndexer, null, null); + bookieCheckTask = spy(bookieCheckTask1); + } + + @Test + public void testShutdownAuditBookiesException() + throws BKException, ReplicationException.BKAuditException, InterruptedException { + doThrow(new ReplicationException.BKAuditException("test failed")) + .when(bookieCheckTask) + .auditBookies(); + bookieCheckTask.startAudit(true); + + assertTrue("shutdownTaskHandler should be execute.", shutdownCompleted.get()); + } + + @Test + public void testAuditBookies() + throws ReplicationException.UnavailableException, ReplicationException.BKAuditException, BKException { + final String bookieId1 = "127.0.0.1:1000"; + final String bookieId2 = "127.0.0.1:1001"; + final long bookie1LedgersCount = 10; + final long bookie2LedgersCount = 20; + + final Map> bookiesAndLedgers = new HashMap<>(); + bookiesAndLedgers.put(bookieId1, getLedgers(bookie1LedgersCount)); + bookiesAndLedgers.put(bookieId2, getLedgers(bookie2LedgersCount)); + when(ledgerIndexer.getBookieToLedgerIndex()).thenReturn(bookiesAndLedgers); + when(underreplicationManager.isLedgerReplicationEnabled()).thenReturn(true); + + CompletableFuture> metaPromise = new CompletableFuture<>(); + final LongVersion version = mock(LongVersion.class); + final LedgerMetadata metadata = mock(LedgerMetadata.class); + metaPromise.complete(new Versioned<>(metadata, version)); + when(ledgerManager.readLedgerMetadata(anyLong())).thenReturn(metaPromise); + + CompletableFuture markPromise = new CompletableFuture<>(); + markPromise.complete(null); + when(underreplicationManager.markLedgerUnderreplicatedAsync(anyLong(), anyCollection())) + .thenReturn(markPromise); + + OpStatsLogger numUnderReplicatedLedgerStats = mock(OpStatsLogger.class); + when(auditorStats.getNumUnderReplicatedLedger()).thenReturn(numUnderReplicatedLedgerStats); + + final List availableBookies = new ArrayList<>(); + final List readOnlyBookies = new ArrayList<>(); + // test bookie1 lost + availableBookies.add(BookieId.parse(bookieId2)); + when(admin.getAvailableBookies()).thenReturn(availableBookies); + when(admin.getReadOnlyBookies()).thenReturn(readOnlyBookies); + bookieCheckTask.startAudit(true); + verify(numUnderReplicatedLedgerStats, times(1)) + .registerSuccessfulValue(eq(bookie1LedgersCount)); + + // test bookie2 lost + numUnderReplicatedLedgerStats = mock(OpStatsLogger.class); + when(auditorStats.getNumUnderReplicatedLedger()).thenReturn(numUnderReplicatedLedgerStats); + availableBookies.clear(); + availableBookies.add(BookieId.parse(bookieId1)); + bookieCheckTask.startAudit(true); + verify(numUnderReplicatedLedgerStats, times(1)) + .registerSuccessfulValue(eq(bookie2LedgersCount)); + + } + + private Set getLedgers(long count) { + final Set ledgers = new HashSet<>(); + for (int i = 0; i < count; i++) { + ledgers.add(i + startLedgerId++); + } + return ledgers; + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieTest.java new file mode 100644 index 0000000000000..14cdf3e1fc29c --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieTest.java @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertFalse; +import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.AssertJUnit.assertNotSame; +import static org.testng.AssertJUnit.assertSame; +import static org.testng.AssertJUnit.assertTrue; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.proto.BookieServer; +import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerAuditorManager; +import org.apache.zookeeper.ZooKeeper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * This test verifies the auditor bookie scenarios which will be monitoring the + * bookie failures. + */ +public class AuditorBookieTest extends BookKeeperClusterTestCase { + // Depending on the taste, select the amount of logging + // by decommenting one of the two lines below + // private static final Logger LOG = Logger.getRootLogger(); + private static final Logger LOG = LoggerFactory + .getLogger(AuditorBookieTest.class); + private String electionPath; + private HashMap auditorElectors = new HashMap(); + private List zkClients = new LinkedList(); + + public AuditorBookieTest() throws Exception { + super(6); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + electionPath = ZKMetadataDriverBase.resolveZkLedgersRootPath(baseConf) + + "/underreplication/" + PulsarLedgerAuditorManager.ELECTION_PATH; + baseConf.setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + startAuditorElectors(); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + stopAuditorElectors(); + for (ZooKeeper zk : zkClients) { + zk.close(); + } + zkClients.clear(); + super.tearDown(); + } + + /** + * Test should ensure only one should act as Auditor. Starting/shutdown + * other than auditor bookie shouldn't initiate re-election and multiple + * auditors. + */ + @Test + public void testEnsureOnlySingleAuditor() throws Exception { + BookieServer auditor = verifyAuditor(); + + // shutdown bookie which is not an auditor + int indexOf = indexOfServer(auditor); + int bkIndexDownBookie; + if (indexOf < lastBookieIndex()) { + bkIndexDownBookie = indexOf + 1; + } else { + bkIndexDownBookie = indexOf - 1; + } + shutdownBookie(serverByIndex(bkIndexDownBookie)); + + startNewBookie(); + startNewBookie(); + // grace period for the auditor re-election if any + BookieServer newAuditor = waitForNewAuditor(auditor); + assertSame( + "Auditor re-election is not happened for auditor failure!", + auditor, newAuditor); + } + + /** + * Test Auditor crashes should trigger re-election and another bookie should + * take over the auditor ship. + */ + @Test + public void testSuccessiveAuditorCrashes() throws Exception { + BookieServer auditor = verifyAuditor(); + shutdownBookie(auditor); + + BookieServer newAuditor1 = waitForNewAuditor(auditor); + shutdownBookie(newAuditor1); + BookieServer newAuditor2 = waitForNewAuditor(newAuditor1); + assertNotSame( + "Auditor re-election is not happened for auditor failure!", + auditor, newAuditor2); + } + + /** + * Test restarting the entire bookie cluster. It shouldn't create multiple + * bookie auditors. + */ + @Test + public void testBookieClusterRestart() throws Exception { + BookieServer auditor = verifyAuditor(); + for (AuditorElector auditorElector : auditorElectors.values()) { + assertTrue("Auditor elector is not running!", auditorElector + .isRunning()); + } + stopBKCluster(); + stopAuditorElectors(); + + startBKCluster(zkUtil.getMetadataServiceUri()); + //startBKCluster(zkUtil.getMetadataServiceUri()) override the base conf metadataServiceUri + baseConf.setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + startAuditorElectors(); + BookieServer newAuditor = waitForNewAuditor(auditor); + assertNotSame( + "Auditor re-election is not happened for auditor failure!", + auditor, newAuditor); + } + + /** + * Test the vote is deleting from the ZooKeeper during shutdown. + */ + @Test + public void testShutdown() throws Exception { + BookieServer auditor = verifyAuditor(); + shutdownBookie(auditor); + + // waiting for new auditor + BookieServer newAuditor = waitForNewAuditor(auditor); + assertNotSame( + "Auditor re-election is not happened for auditor failure!", + auditor, newAuditor); + + List children = zkc.getChildren(electionPath, false); + for (String child : children) { + byte[] data = zkc.getData(electionPath + '/' + child, false, null); + String bookieIP = new String(data); + String addr = auditor.getBookieId().toString(); + assertFalse("AuditorElection cleanup fails", bookieIP + .contains(addr)); + } + } + + /** + * Test restart of the previous Auditor bookie shouldn't initiate + * re-election and should create new vote after restarting. + */ + @Test + public void testRestartAuditorBookieAfterCrashing() throws Exception { + BookieServer auditor = verifyAuditor(); + + String addr = auditor.getBookieId().toString(); + + // restarting Bookie with same configurations. + ServerConfiguration serverConfiguration = shutdownBookie(auditor); + + auditorElectors.remove(addr); + startBookie(serverConfiguration); + // starting corresponding auditor elector + + if (LOG.isDebugEnabled()) { + LOG.debug("Performing Auditor Election:" + addr); + } + startAuditorElector(addr); + + // waiting for new auditor to come + BookieServer newAuditor = waitForNewAuditor(auditor); + assertNotSame( + "Auditor re-election is not happened for auditor failure!", + auditor, newAuditor); + assertFalse("No relection after old auditor rejoins", auditor + .getBookieId().equals(newAuditor.getBookieId())); + } + + private void startAuditorElector(String addr) throws Exception { + AuditorElector auditorElector = new AuditorElector(addr, + baseConf); + auditorElectors.put(addr, auditorElector); + auditorElector.start(); + if (LOG.isDebugEnabled()) { + LOG.debug("Starting Auditor Elector"); + } + } + + private void startAuditorElectors() throws Exception { + for (BookieId addr : bookieAddresses()) { + startAuditorElector(addr.toString()); + } + } + + private void stopAuditorElectors() throws Exception { + for (AuditorElector auditorElector : auditorElectors.values()) { + auditorElector.shutdown(); + if (LOG.isDebugEnabled()) { + LOG.debug("Stopping Auditor Elector!"); + } + } + } + + private BookieServer verifyAuditor() throws Exception { + List auditors = getAuditorBookie(); + assertEquals("Multiple Bookies acting as Auditor!", 1, auditors + .size()); + if (LOG.isDebugEnabled()) { + LOG.debug("Bookie running as Auditor:" + auditors.get(0)); + } + return auditors.get(0); + } + + private List getAuditorBookie() throws Exception { + List auditors = new LinkedList(); + byte[] data = zkc.getData(electionPath, false, null); + assertNotNull("Auditor election failed", data); + for (int i = 0; i < bookieCount(); i++) { + BookieServer bks = serverByIndex(i); + if (new String(data).contains(bks.getBookieId() + "")) { + auditors.add(bks); + } + } + return auditors; + } + + private ServerConfiguration shutdownBookie(BookieServer bkServer) throws Exception { + int index = indexOfServer(bkServer); + String addr = addressByIndex(index).toString(); + if (LOG.isDebugEnabled()) { + LOG.debug("Shutting down bookie:" + addr); + } + + // shutdown bookie which is an auditor + ServerConfiguration conf = killBookie(index); + + // stopping corresponding auditor elector + auditorElectors.get(addr).shutdown(); + return conf; + } + + private BookieServer waitForNewAuditor(BookieServer auditor) + throws Exception { + BookieServer newAuditor = null; + int retryCount = 8; + while (retryCount > 0) { + try { + List auditors = getAuditorBookie(); + if (auditors.size() > 0) { + newAuditor = auditors.get(0); + if (auditor != newAuditor) { + break; + } + } + } catch (Exception ignore) { + } + + Thread.sleep(500); + retryCount--; + } + assertNotNull( + "New Auditor is not reelected after auditor crashes", + newAuditor); + verifyAuditor(); + return newAuditor; + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorCheckAllLedgersTaskTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorCheckAllLedgersTaskTest.java new file mode 100644 index 0000000000000..795e91c7572e1 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorCheckAllLedgersTaskTest.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE; +import static org.testng.AssertJUnit.assertEquals; +import java.util.LinkedList; +import java.util.List; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.BookKeeperAdmin; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.meta.LayoutManager; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.bookkeeper.test.TestStatsProvider; +import org.apache.pulsar.metadata.api.MetadataStoreConfig; +import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.apache.pulsar.metadata.bookkeeper.PulsarLayoutManager; +import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Unit test {@link AuditorCheckAllLedgersTask}. + */ +public class AuditorCheckAllLedgersTaskTest extends BookKeeperClusterTestCase { + private static final Logger LOG = LoggerFactory + .getLogger(AuditorCheckAllLedgersTaskTest.class); + + private static final int maxNumberOfConcurrentOpenLedgerOperations = 500; + private static final int acquireConcurrentOpenLedgerOperationsTimeoutMSec = 120000; + + private BookKeeperAdmin admin; + private LedgerManager ledgerManager; + private LedgerUnderreplicationManager ledgerUnderreplicationManager; + + public AuditorCheckAllLedgersTaskTest() { + super(3); + baseConf.setPageLimit(1); + baseConf.setAutoRecoveryDaemonEnabled(false); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + final BookKeeper bookKeeper = registerCloseable(new BookKeeper(baseClientConf)); + admin = new BookKeeperAdmin(bookKeeper, NullStatsLogger.INSTANCE, new ClientConfiguration(baseClientConf)); + + String ledgersRoot = "/ledgers"; + String storeUri = metadataServiceUri.replaceAll("zk://", "").replaceAll("/ledgers", ""); + MetadataStoreExtended store = registerCloseable(MetadataStoreExtended.create(storeUri, + MetadataStoreConfig.builder().fsyncEnable(false).build())); + LayoutManager layoutManager = new PulsarLayoutManager(store, ledgersRoot); + PulsarLedgerManagerFactory ledgerManagerFactory = registerCloseable(new PulsarLedgerManagerFactory()); + + ClientConfiguration conf = new ClientConfiguration(); + conf.setZkLedgersRootPath(ledgersRoot); + ledgerManagerFactory.initialize(conf, layoutManager, 1); + ledgerUnderreplicationManager = ledgerManagerFactory.newLedgerUnderreplicationManager(); + ledgerManager = ledgerManagerFactory.newLedgerManager(); + + baseConf.setAuditorMaxNumberOfConcurrentOpenLedgerOperations(maxNumberOfConcurrentOpenLedgerOperations); + baseConf.setAuditorAcquireConcurrentOpenLedgerOperationsTimeoutMSec( + acquireConcurrentOpenLedgerOperationsTimeoutMSec); + } + + @AfterMethod(alwaysRun = true) + @Override + public void tearDown() throws Exception { + if (ledgerManager != null) { + ledgerManager.close(); + } + if (ledgerUnderreplicationManager != null) { + ledgerUnderreplicationManager.close(); + } + if (admin != null) { + admin.close(); + } + super.tearDown(); + } + + @Test + public void testCheckAllLedgers() throws Exception { + // 1. create ledgers + final int numLedgers = 10; + List ids = new LinkedList(); + for (int i = 0; i < numLedgers; i++) { + LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, "passwd".getBytes()); + ids.add(lh.getId()); + for (int j = 0; j < 2; j++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + } + // 2. init CheckAllLedgersTask + final TestStatsProvider statsProvider = new TestStatsProvider(); + final TestStatsProvider.TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + final AuditorStats auditorStats = new AuditorStats(statsLogger); + + AuditorCheckAllLedgersTask auditorCheckAllLedgersTask = new AuditorCheckAllLedgersTask( + baseConf, auditorStats, admin, ledgerManager, + ledgerUnderreplicationManager, null, (flag, throwable) -> flag.set(false)); + + // 3. checkAllLedgers + auditorCheckAllLedgersTask.runTask(); + + // 4. verify + assertEquals("CHECK_ALL_LEDGERS_TIME", 1, ((TestStatsProvider.TestOpStatsLogger) statsLogger + .getOpStatsLogger(ReplicationStats.CHECK_ALL_LEDGERS_TIME)).getSuccessCount()); + assertEquals("NUM_LEDGERS_CHECKED", numLedgers, + (long) statsLogger.getCounter(ReplicationStats.NUM_LEDGERS_CHECKED).get()); + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java new file mode 100644 index 0000000000000..220b2ed179972 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java @@ -0,0 +1,1138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertFalse; +import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.AssertJUnit.assertNotSame; +import static org.testng.AssertJUnit.assertTrue; +import static org.testng.AssertJUnit.fail; +import java.net.URI; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import lombok.Cleanup; +import org.apache.bookkeeper.bookie.BookieImpl; +import org.apache.bookkeeper.client.AsyncCallback.AddCallback; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeper.DigestType; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.client.LedgerMetadataBuilder; +import org.apache.bookkeeper.client.api.LedgerMetadata; +import org.apache.bookkeeper.common.util.OrderedScheduler; +import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.meta.LayoutManager; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.meta.MetadataClientDriver; +import org.apache.bookkeeper.meta.MetadataDrivers; +import org.apache.bookkeeper.meta.UnderreplicatedLedger; +import org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager; +import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.net.BookieSocketAddress; +import org.apache.bookkeeper.proto.BookieServer; +import org.apache.bookkeeper.replication.ReplicationException.UnavailableException; +import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.pulsar.metadata.api.MetadataStoreConfig; +import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.apache.pulsar.metadata.bookkeeper.PulsarLayoutManager; +import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerAuditorManager; +import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Tests publishing of under replicated ledgers by the Auditor bookie node when + * corresponding bookies identifes as not running. + */ +public class AuditorLedgerCheckerTest extends BookKeeperClusterTestCase { + + // Depending on the taste, select the amount of logging + // by decommenting one of the two lines below + // private static final Logger LOG = Logger.getRootLogger(); + private static final Logger LOG = LoggerFactory + .getLogger(AuditorLedgerCheckerTest.class); + + private static final byte[] ledgerPassword = "aaa".getBytes(); + private Random rng; // Random Number Generator + + private DigestType digestType; + + private String underreplicatedPath; + private Map auditorElectors = new ConcurrentHashMap<>(); + private LedgerUnderreplicationManager urLedgerMgr; + + private Set urLedgerList; + private String electionPath; + + private List ledgerList; + + public AuditorLedgerCheckerTest() + throws Exception { + this("org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory"); + } + + AuditorLedgerCheckerTest(String ledgerManagerFactoryClass) + throws Exception { + super(3); + LOG.info("Running test case using ledger manager : " + + ledgerManagerFactoryClass); + this.digestType = DigestType.CRC32; + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); // set ledger manager name + baseConf.setLedgerManagerFactoryClassName(ledgerManagerFactoryClass); + baseClientConf + .setLedgerManagerFactoryClassName(ledgerManagerFactoryClass); + } + + @BeforeMethod + public void setUp() throws Exception { + super.setUp(); + underreplicatedPath = ZKMetadataDriverBase.resolveZkLedgersRootPath(baseClientConf) + + "/underreplication/ledgers"; + electionPath = ZKMetadataDriverBase.resolveZkLedgersRootPath(baseConf) + + "/underreplication/" + PulsarLedgerAuditorManager.ELECTION_PATH; + + String ledgersRoot = "/ledgers"; + String storeUri = metadataServiceUri.replaceAll("zk://", "").replaceAll("/ledgers", ""); + MetadataStoreExtended store = registerCloseable(MetadataStoreExtended.create(storeUri, + MetadataStoreConfig.builder().fsyncEnable(false).build())); + LayoutManager layoutManager = new PulsarLayoutManager(store, ledgersRoot); + PulsarLedgerManagerFactory ledgerManagerFactory = registerCloseable(new PulsarLedgerManagerFactory()); + ClientConfiguration conf = new ClientConfiguration(); + conf.setZkLedgersRootPath(ledgersRoot); + ledgerManagerFactory.initialize(conf, layoutManager, 1); + urLedgerMgr = registerCloseable(ledgerManagerFactory.newLedgerUnderreplicationManager()); + urLedgerMgr.setCheckAllLedgersCTime(System.currentTimeMillis()); + + baseClientConf.setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + baseConf.setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + startAuditorElectors(); + rng = new Random(System.currentTimeMillis()); // Initialize the Random + urLedgerList = new HashSet(); + ledgerList = new ArrayList(2); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + stopAuditorElectors(); + super.tearDown(); + } + + private void startAuditorElectors() throws Exception { + for (String addr : bookieAddresses().stream().map(Object::toString) + .collect(Collectors.toList())) { + AuditorElector auditorElector = new AuditorElector(addr, baseConf); + auditorElectors.put(addr, auditorElector); + auditorElector.start(); + if (LOG.isDebugEnabled()) { + LOG.debug("Starting Auditor Elector"); + } + } + } + + private void stopAuditorElectors() throws Exception { + for (AuditorElector auditorElector : auditorElectors.values()) { + auditorElector.shutdown(); + if (LOG.isDebugEnabled()) { + LOG.debug("Stopping Auditor Elector!"); + } + } + } + + /** + * Test publishing of under replicated ledgers by the auditor bookie. + */ + @Test + public void testSimpleLedger() throws Exception { + LedgerHandle lh1 = createAndAddEntriesToLedger(); + Long ledgerId = lh1.getId(); + if (LOG.isDebugEnabled()) { + LOG.debug("Created ledger : " + ledgerId); + } + ledgerList.add(ledgerId); + lh1.close(); + + final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList + .size()); + + int bkShutdownIndex = lastBookieIndex(); + String shutdownBookie = shutdownBookie(bkShutdownIndex); + + // grace period for publishing the bk-ledger + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for ledgers to be marked as under replicated"); + } + waitForAuditToComplete(); + underReplicaLatch.await(5, TimeUnit.SECONDS); + Map urLedgerData = getUrLedgerData(urLedgerList); + assertEquals("Missed identifying under replicated ledgers", 1, + urLedgerList.size()); + + /* + * Sample data format present in the under replicated ledger path + * + * {4=replica: "10.18.89.153:5002"} + */ + assertTrue("Ledger is not marked as underreplicated:" + ledgerId, + urLedgerList.contains(ledgerId)); + String data = urLedgerData.get(ledgerId); + assertTrue("Bookie " + shutdownBookie + + "is not listed in the ledger as missing replica :" + data, + data.contains(shutdownBookie)); + } + + /** + * Test once published under replicated ledger should exists even after + * restarting respective bookie. + */ + @Test + public void testRestartBookie() throws Exception { + LedgerHandle lh1 = createAndAddEntriesToLedger(); + LedgerHandle lh2 = createAndAddEntriesToLedger(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Created following ledgers : {}, {}", lh1, lh2); + } + + int bkShutdownIndex = lastBookieIndex(); + ServerConfiguration bookieConf1 = confByIndex(bkShutdownIndex); + String shutdownBookie = shutdownBookie(bkShutdownIndex); + + // restart the failed bookie + startAndAddBookie(bookieConf1); + + waitForLedgerMissingReplicas(lh1.getId(), 10, shutdownBookie); + waitForLedgerMissingReplicas(lh2.getId(), 10, shutdownBookie); + } + + /** + * Test publishing of under replicated ledgers when multiple bookie failures + * one after another. + */ + @Test + public void testMultipleBookieFailures() throws Exception { + LedgerHandle lh1 = createAndAddEntriesToLedger(); + + // failing first bookie + shutdownBookie(lastBookieIndex()); + + // simulate re-replication + doLedgerRereplication(lh1.getId()); + + // failing another bookie + String shutdownBookie = shutdownBookie(lastBookieIndex()); + + // grace period for publishing the bk-ledger + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for ledgers to be marked as under replicated"); + } + assertTrue("Ledger should be missing second replica", + waitForLedgerMissingReplicas(lh1.getId(), 10, shutdownBookie)); + } + + @Test + public void testToggleLedgerReplication() throws Exception { + LedgerHandle lh1 = createAndAddEntriesToLedger(); + ledgerList.add(lh1.getId()); + if (LOG.isDebugEnabled()) { + LOG.debug("Created following ledgers : " + ledgerList); + } + + // failing another bookie + CountDownLatch urReplicaLatch = registerUrLedgerWatcher(ledgerList + .size()); + + // disabling ledger replication + urLedgerMgr.disableLedgerReplication(); + ArrayList shutdownBookieList = new ArrayList(); + shutdownBookieList.add(shutdownBookie(lastBookieIndex())); + shutdownBookieList.add(shutdownBookie(lastBookieIndex())); + + assertFalse("Ledger replication is not disabled!", urReplicaLatch + .await(1, TimeUnit.SECONDS)); + + // enabling ledger replication + urLedgerMgr.enableLedgerReplication(); + assertTrue("Ledger replication is not enabled!", urReplicaLatch.await( + 5, TimeUnit.SECONDS)); + } + + @Test + public void testDuplicateEnDisableAutoRecovery() throws Exception { + urLedgerMgr.disableLedgerReplication(); + try { + urLedgerMgr.disableLedgerReplication(); + fail("Must throw exception, since AutoRecovery is already disabled"); + } catch (UnavailableException e) { + assertTrue("AutoRecovery is not disabled previously!", + e.getCause().getCause() instanceof MetadataStoreException.BadVersionException); + } + urLedgerMgr.enableLedgerReplication(); + try { + urLedgerMgr.enableLedgerReplication(); + fail("Must throw exception, since AutoRecovery is already enabled"); + } catch (UnavailableException e) { + assertTrue("AutoRecovery is not enabled previously!", + e.getCause().getCause() instanceof MetadataStoreException.NotFoundException); + } + } + + /** + * Test Auditor should consider Readonly bookie as available bookie. Should not publish ur ledgers for + * readonly bookies. + */ + @Test + public void testReadOnlyBookieExclusionFromURLedgersCheck() throws Exception { + LedgerHandle lh = createAndAddEntriesToLedger(); + ledgerList.add(lh.getId()); + if (LOG.isDebugEnabled()) { + LOG.debug("Created following ledgers : " + ledgerList); + } + + int count = ledgerList.size(); + final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(count); + + final int bkIndex = 2; + ServerConfiguration bookieConf = confByIndex(bkIndex); + BookieServer bk = serverByIndex(bkIndex); + bookieConf.setReadOnlyModeEnabled(true); + + ((BookieImpl) bk.getBookie()).getStateManager().doTransitionToReadOnlyMode(); + bkc.waitForReadOnlyBookie(BookieImpl.getBookieId(confByIndex(bkIndex))) + .get(30, TimeUnit.SECONDS); + + // grace period for publishing the bk-ledger + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for Auditor to finish ledger check."); + } + waitForAuditToComplete(); + assertFalse("latch should not have completed", underReplicaLatch.await(5, TimeUnit.SECONDS)); + } + + /** + * Test Auditor should consider Readonly bookie fail and publish ur ledgers for readonly bookies. + */ + @Test + public void testReadOnlyBookieShutdown() throws Exception { + LedgerHandle lh = createAndAddEntriesToLedger(); + long ledgerId = lh.getId(); + ledgerList.add(ledgerId); + if (LOG.isDebugEnabled()) { + LOG.debug("Created following ledgers : " + ledgerList); + } + + int count = ledgerList.size(); + final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(count); + + int bkIndex = lastBookieIndex(); + if (LOG.isDebugEnabled()) { + LOG.debug("Moving bookie {} {} to read only...", bkIndex, serverByIndex(bkIndex)); + } + ServerConfiguration bookieConf = confByIndex(bkIndex); + BookieServer bk = serverByIndex(bkIndex); + bookieConf.setReadOnlyModeEnabled(true); + + ((BookieImpl) bk.getBookie()).getStateManager().doTransitionToReadOnlyMode(); + bkc.waitForReadOnlyBookie(BookieImpl.getBookieId(confByIndex(bkIndex))) + .get(30, TimeUnit.SECONDS); + + // grace period for publishing the bk-ledger + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for Auditor to finish ledger check."); + } + waitForAuditToComplete(); + assertFalse("latch should not have completed", underReplicaLatch.await(1, TimeUnit.SECONDS)); + + String shutdownBookie = shutdownBookie(bkIndex); + + // grace period for publishing the bk-ledger + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for ledgers to be marked as under replicated"); + } + waitForAuditToComplete(); + underReplicaLatch.await(5, TimeUnit.SECONDS); + Map urLedgerData = getUrLedgerData(urLedgerList); + assertEquals("Missed identifying under replicated ledgers", 1, urLedgerList.size()); + + /* + * Sample data format present in the under replicated ledger path + * + * {4=replica: "10.18.89.153:5002"} + */ + assertTrue("Ledger is not marked as underreplicated:" + ledgerId, urLedgerList.contains(ledgerId)); + String data = urLedgerData.get(ledgerId); + assertTrue("Bookie " + shutdownBookie + "is not listed in the ledger as missing replica :" + data, + data.contains(shutdownBookie)); + } + + public void testInnerDelayedAuditOfLostBookies() throws Exception { + LedgerHandle lh1 = createAndAddEntriesToLedger(); + Long ledgerId = lh1.getId(); + if (LOG.isDebugEnabled()) { + LOG.debug("Created ledger : " + ledgerId); + } + ledgerList.add(ledgerId); + lh1.close(); + + final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList + .size()); + + // wait for 5 seconds before starting the recovery work when a bookie fails + urLedgerMgr.setLostBookieRecoveryDelay(5); + + AtomicReference shutdownBookieRef = new AtomicReference<>(); + CountDownLatch shutdownLatch = new CountDownLatch(1); + new Thread(() -> { + try { + String shutdownBookie = shutDownNonAuditorBookie(); + shutdownBookieRef.set(shutdownBookie); + shutdownLatch.countDown(); + } catch (Exception ignore) { + } + }).start(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for ledgers to be marked as under replicated"); + } + assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(4, TimeUnit.SECONDS)); + assertEquals("under replicated ledgers identified when it was not expected", 0, + urLedgerList.size()); + + // wait for another 5 seconds for the ledger to get reported as under replicated + assertTrue("audit of lost bookie isn't delayed", underReplicaLatch.await(2, TimeUnit.SECONDS)); + + assertTrue("Ledger is not marked as underreplicated:" + ledgerId, + urLedgerList.contains(ledgerId)); + Map urLedgerData = getUrLedgerData(urLedgerList); + String data = urLedgerData.get(ledgerId); + shutdownLatch.await(); + assertTrue("Bookie " + shutdownBookieRef.get() + + "is not listed in the ledger as missing replica :" + data, + data.contains(shutdownBookieRef.get())); + } + + /** + * Test publishing of under replicated ledgers by the auditor + * bookie is delayed if LostBookieRecoveryDelay option is set. + */ + @Test + public void testDelayedAuditOfLostBookies() throws Exception { + // wait for a second so that the initial periodic check finishes + Thread.sleep(1000); + + testInnerDelayedAuditOfLostBookies(); + } + + /** + * Test publishing of under replicated ledgers by the auditor + * bookie is delayed if LostBookieRecoveryDelay option is set + * and it continues to be delayed even when periodic bookie check + * is set to run every 2 secs. I.e. periodic bookie check doesn't + * override the delay + */ + @Test + public void testDelayedAuditWithPeriodicBookieCheck() throws Exception { + // enable periodic bookie check on a cadence of every 2 seconds. + // this requires us to stop the auditor/auditorElectors, set the + // periodic check interval and restart the auditorElectors + stopAuditorElectors(); + baseConf.setAuditorPeriodicBookieCheckInterval(2); + startAuditorElectors(); + + // wait for a second so that the initial periodic check finishes + Thread.sleep(1000); + + // the delaying of audit should just work despite the fact + // we have enabled periodic bookie check + testInnerDelayedAuditOfLostBookies(); + } + + @Test + public void testRescheduleOfDelayedAuditOfLostBookiesToStartImmediately() throws Exception { + // wait for a second so that the initial periodic check finishes + Thread.sleep(1000); + + LedgerHandle lh1 = createAndAddEntriesToLedger(); + Long ledgerId = lh1.getId(); + if (LOG.isDebugEnabled()) { + LOG.debug("Created ledger : " + ledgerId); + } + ledgerList.add(ledgerId); + lh1.close(); + + final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList + .size()); + + // wait for 50 seconds before starting the recovery work when a bookie fails + urLedgerMgr.setLostBookieRecoveryDelay(50); + + // shutdown a non auditor bookie; choosing non-auditor to avoid another election + AtomicReference shutdownBookieRef = new AtomicReference<>(); + CountDownLatch shutdownLatch = new CountDownLatch(1); + new Thread(() -> { + try { + String shutdownBookie = shutDownNonAuditorBookie(); + shutdownBookieRef.set(shutdownBookie); + shutdownLatch.countDown(); + } catch (Exception ignore) { + } + }).start(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for ledgers to be marked as under replicated"); + } + assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(4, TimeUnit.SECONDS)); + assertEquals("under replicated ledgers identified when it was not expected", 0, + urLedgerList.size()); + + // set lostBookieRecoveryDelay to 0, so that it triggers AuditTask immediately + urLedgerMgr.setLostBookieRecoveryDelay(0); + + // wait for 1 second for the ledger to get reported as under replicated + assertTrue("audit of lost bookie isn't delayed", underReplicaLatch.await(1, TimeUnit.SECONDS)); + + assertTrue("Ledger is not marked as underreplicated:" + ledgerId, + urLedgerList.contains(ledgerId)); + Map urLedgerData = getUrLedgerData(urLedgerList); + String data = urLedgerData.get(ledgerId); + shutdownLatch.await(); + assertTrue("Bookie " + shutdownBookieRef.get() + + "is not listed in the ledger as missing replica :" + data, + data.contains(shutdownBookieRef.get())); + } + + @Test + public void testRescheduleOfDelayedAuditOfLostBookiesToStartLater() throws Exception { + // wait for a second so that the initial periodic check finishes + Thread.sleep(1000); + + LedgerHandle lh1 = createAndAddEntriesToLedger(); + Long ledgerId = lh1.getId(); + if (LOG.isDebugEnabled()) { + LOG.debug("Created ledger : " + ledgerId); + } + ledgerList.add(ledgerId); + lh1.close(); + + final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList + .size()); + + // wait for 3 seconds before starting the recovery work when a bookie fails + urLedgerMgr.setLostBookieRecoveryDelay(3); + + // shutdown a non auditor bookie; choosing non-auditor to avoid another election + AtomicReference shutdownBookieRef = new AtomicReference<>(); + CountDownLatch shutdownLatch = new CountDownLatch(1); + new Thread(() -> { + try { + String shutdownBookie = shutDownNonAuditorBookie(); + shutdownBookieRef.set(shutdownBookie); + shutdownLatch.countDown(); + } catch (Exception ignore) { + } + }).start(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for ledgers to be marked as under replicated"); + } + assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(2, TimeUnit.SECONDS)); + assertEquals("under replicated ledgers identified when it was not expected", 0, + urLedgerList.size()); + + // set lostBookieRecoveryDelay to 4, so the pending AuditTask is resheduled + urLedgerMgr.setLostBookieRecoveryDelay(4); + + // since we changed the BookieRecoveryDelay period to 4, the audittask shouldn't have been executed + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for ledgers to be marked as under replicated"); + } + assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(2, TimeUnit.SECONDS)); + assertEquals("under replicated ledgers identified when it was not expected", 0, + urLedgerList.size()); + + // wait for 3 seconds (since we already waited for 2 secs) for the ledger to get reported as under replicated + assertTrue("audit of lost bookie isn't delayed", underReplicaLatch.await(3, TimeUnit.SECONDS)); + assertTrue("Ledger is not marked as underreplicated:" + ledgerId, + urLedgerList.contains(ledgerId)); + Map urLedgerData = getUrLedgerData(urLedgerList); + String data = urLedgerData.get(ledgerId); + shutdownLatch.await(); + assertTrue("Bookie " + shutdownBookieRef.get() + + "is not listed in the ledger as missing replica :" + data, + data.contains(shutdownBookieRef.get())); + } + + @Test + public void testTriggerAuditorWithNoPendingAuditTask() throws Exception { + // wait for a second so that the initial periodic check finishes + Thread.sleep(1000); + int lostBookieRecoveryDelayConfValue = baseConf.getLostBookieRecoveryDelay(); + Auditor auditorBookiesAuditor = getAuditorBookiesAuditor(); + Future auditTask = auditorBookiesAuditor.getAuditTask(); + int lostBookieRecoveryDelayBeforeChange = auditorBookiesAuditor.getLostBookieRecoveryDelayBeforeChange(); + assertEquals("auditTask is supposed to be null", null, auditTask); + assertEquals( + "lostBookieRecoveryDelayBeforeChange of Auditor should be equal to BaseConf's lostBookieRecoveryDelay", + lostBookieRecoveryDelayConfValue, lostBookieRecoveryDelayBeforeChange); + + @Cleanup("shutdown") OrderedScheduler scheduler = OrderedScheduler.newSchedulerBuilder() + .name("test-scheduler") + .numThreads(1) + .build(); + @Cleanup MetadataClientDriver driver = + MetadataDrivers.getClientDriver(URI.create(baseClientConf.getMetadataServiceUri())); + driver.initialize(baseClientConf, scheduler, NullStatsLogger.INSTANCE, Optional.of(zkc)); + + // there is no easy way to validate if the Auditor has executed Audit process (Auditor.startAudit), + // without shuttingdown Bookie. To test if by resetting LostBookieRecoveryDelay it does Auditing + // even when there is no pending AuditTask, following approach is needed. + + // Here we are creating few ledgers ledgermetadata with non-existing bookies as its ensemble. + // When Auditor does audit it recognizes these ledgers as underreplicated and mark them as + // under-replicated, since these bookies are not available. + int numofledgers = 5; + Random rand = new Random(); + for (int i = 0; i < numofledgers; i++) { + ArrayList ensemble = new ArrayList(); + ensemble.add(new BookieSocketAddress("99.99.99.99:9999").toBookieId()); + ensemble.add(new BookieSocketAddress("11.11.11.11:1111").toBookieId()); + ensemble.add(new BookieSocketAddress("88.88.88.88:8888").toBookieId()); + + long ledgerId = (Math.abs(rand.nextLong())) % 100000000; + + LedgerMetadata metadata = LedgerMetadataBuilder.create() + .withId(ledgerId) + .withEnsembleSize(3).withWriteQuorumSize(2).withAckQuorumSize(2) + .withPassword("passwd".getBytes()) + .withDigestType(DigestType.CRC32.toApiDigestType()) + .newEnsembleEntry(0L, ensemble).build(); + + try (LedgerManager lm = driver.getLedgerManagerFactory().newLedgerManager()) { + lm.createLedgerMetadata(ledgerId, metadata).get(2000, TimeUnit.MILLISECONDS); + } + ledgerList.add(ledgerId); + } + + final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList.size()); + urLedgerMgr.setLostBookieRecoveryDelay(lostBookieRecoveryDelayBeforeChange); + assertTrue("Audit should be triggered and created ledgers should be marked as underreplicated", + underReplicaLatch.await(2, TimeUnit.SECONDS)); + assertEquals("All the ledgers should be marked as underreplicated", ledgerList.size(), urLedgerList.size()); + + auditTask = auditorBookiesAuditor.getAuditTask(); + assertEquals("auditTask is supposed to be null", null, auditTask); + assertEquals( + "lostBookieRecoveryDelayBeforeChange of Auditor should be equal to BaseConf's lostBookieRecoveryDelay", + lostBookieRecoveryDelayBeforeChange, auditorBookiesAuditor.getLostBookieRecoveryDelayBeforeChange()); + } + + @Test + public void testTriggerAuditorWithPendingAuditTask() throws Exception { + // wait for a second so that the initial periodic check finishes + Thread.sleep(1000); + + Auditor auditorBookiesAuditor = getAuditorBookiesAuditor(); + LedgerHandle lh1 = createAndAddEntriesToLedger(); + Long ledgerId = lh1.getId(); + if (LOG.isDebugEnabled()) { + LOG.debug("Created ledger : " + ledgerId); + } + ledgerList.add(ledgerId); + lh1.close(); + + final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList + .size()); + + int lostBookieRecoveryDelay = 5; + // wait for 5 seconds before starting the recovery work when a bookie fails + urLedgerMgr.setLostBookieRecoveryDelay(lostBookieRecoveryDelay); + + // shutdown a non auditor bookie; choosing non-auditor to avoid another election + new Thread(() -> { + try { + shutDownNonAuditorBookie(); + } catch (Exception ignore) { + } + }).start(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for ledgers to be marked as under replicated"); + } + assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(2, TimeUnit.SECONDS)); + assertEquals("under replicated ledgers identified when it was not expected", 0, + urLedgerList.size()); + + Future auditTask = auditorBookiesAuditor.getAuditTask(); + assertNotSame("auditTask is not supposed to be null", null, auditTask); + assertEquals( + "lostBookieRecoveryDelayBeforeChange of Auditor should be equal to what we set", + lostBookieRecoveryDelay, auditorBookiesAuditor.getLostBookieRecoveryDelayBeforeChange()); + + // set lostBookieRecoveryDelay to 5 (previous value), so that Auditor is triggered immediately + urLedgerMgr.setLostBookieRecoveryDelay(lostBookieRecoveryDelay); + assertTrue("audit of lost bookie shouldn't be delayed", underReplicaLatch.await(2, TimeUnit.SECONDS)); + assertEquals("all under replicated ledgers should be identified", ledgerList.size(), + urLedgerList.size()); + + Thread.sleep(100); + auditTask = auditorBookiesAuditor.getAuditTask(); + assertEquals("auditTask is supposed to be null", null, auditTask); + assertEquals( + "lostBookieRecoveryDelayBeforeChange of Auditor should be equal to previously set value", + lostBookieRecoveryDelay, auditorBookiesAuditor.getLostBookieRecoveryDelayBeforeChange()); + } + + @Test + public void testTriggerAuditorBySettingDelayToZeroWithPendingAuditTask() throws Exception { + // wait for a second so that the initial periodic check finishes + Thread.sleep(1000); + + Auditor auditorBookiesAuditor = getAuditorBookiesAuditor(); + LedgerHandle lh1 = createAndAddEntriesToLedger(); + Long ledgerId = lh1.getId(); + if (LOG.isDebugEnabled()) { + LOG.debug("Created ledger : " + ledgerId); + } + ledgerList.add(ledgerId); + lh1.close(); + + final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList + .size()); + + int lostBookieRecoveryDelay = 5; + // wait for 5 seconds before starting the recovery work when a bookie fails + urLedgerMgr.setLostBookieRecoveryDelay(lostBookieRecoveryDelay); + + // shutdown a non auditor bookie; choosing non-auditor to avoid another election + new Thread(() -> { + try { + shutDownNonAuditorBookie(); + } catch (Exception ignore) { + } + }).start(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for ledgers to be marked as under replicated"); + } + assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(2, TimeUnit.SECONDS)); + assertEquals("under replicated ledgers identified when it was not expected", 0, + urLedgerList.size()); + + Future auditTask = auditorBookiesAuditor.getAuditTask(); + assertNotSame("auditTask is not supposed to be null", null, auditTask); + assertEquals( + "lostBookieRecoveryDelayBeforeChange of Auditor should be equal to what we set", + lostBookieRecoveryDelay, auditorBookiesAuditor.getLostBookieRecoveryDelayBeforeChange()); + + // set lostBookieRecoveryDelay to 0, so that Auditor is triggered immediately + urLedgerMgr.setLostBookieRecoveryDelay(0); + assertTrue("audit of lost bookie shouldn't be delayed", underReplicaLatch.await(1, TimeUnit.SECONDS)); + assertEquals("all under replicated ledgers should be identified", ledgerList.size(), + urLedgerList.size()); + + Thread.sleep(100); + auditTask = auditorBookiesAuditor.getAuditTask(); + assertEquals("auditTask is supposed to be null", null, auditTask); + assertEquals( + "lostBookieRecoveryDelayBeforeChange of Auditor should be equal to previously set value", + 0, auditorBookiesAuditor.getLostBookieRecoveryDelayBeforeChange()); + } + + /** + * Test audit of bookies is delayed when one bookie is down. But when + * another one goes down, the audit is started immediately. + */ + @Test + public void testDelayedAuditWithMultipleBookieFailures() throws Exception { + // wait for the periodic bookie check to finish + Thread.sleep(1000); + + // create a ledger with a bunch of entries + LedgerHandle lh1 = createAndAddEntriesToLedger(); + Long ledgerId = lh1.getId(); + if (LOG.isDebugEnabled()) { + LOG.debug("Created ledger : " + ledgerId); + } + ledgerList.add(ledgerId); + lh1.close(); + + CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList.size()); + + // wait for 10 seconds before starting the recovery work when a bookie fails + urLedgerMgr.setLostBookieRecoveryDelay(10); + + // shutdown a non auditor bookie; choosing non-auditor to avoid another election + AtomicReference shutdownBookieRef1 = new AtomicReference<>(); + CountDownLatch shutdownLatch1 = new CountDownLatch(1); + new Thread(() -> { + try { + String shutdownBookie1 = shutDownNonAuditorBookie(); + shutdownBookieRef1.set(shutdownBookie1); + shutdownLatch1.countDown(); + } catch (Exception ignore) { + } + }).start(); + + // wait for 3 seconds and there shouldn't be any under replicated ledgers + // because we have delayed the start of audit by 10 seconds + assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(3, TimeUnit.SECONDS)); + assertEquals("under replicated ledgers identified when it was not expected", 0, + urLedgerList.size()); + + // Now shutdown the second non auditor bookie; We want to make sure that + // the history about having delayed recovery remains. Hence we make sure + // we bring down a non auditor bookie. This should cause the audit to take + // place immediately and not wait for the remaining 7 seconds to elapse + AtomicReference shutdownBookieRef2 = new AtomicReference<>(); + CountDownLatch shutdownLatch2 = new CountDownLatch(1); + new Thread(() -> { + try { + String shutdownBookie2 = shutDownNonAuditorBookie(); + shutdownBookieRef2.set(shutdownBookie2); + shutdownLatch2.countDown(); + } catch (Exception ignore) { + } + }).start(); + + // 2 second grace period for the ledgers to get reported as under replicated + Thread.sleep(2000); + + // If the following checks pass, it means that audit happened + // within 2 seconds of second bookie going down and it didn't + // wait for 7 more seconds. Hence the second bookie failure doesn't + // delay the audit + assertTrue("Ledger is not marked as underreplicated:" + ledgerId, + urLedgerList.contains(ledgerId)); + Map urLedgerData = getUrLedgerData(urLedgerList); + String data = urLedgerData.get(ledgerId); + shutdownLatch1.await(); + shutdownLatch2.await(); + assertTrue("Bookie " + shutdownBookieRef1.get() + shutdownBookieRef2.get() + + " are not listed in the ledger as missing replicas :" + data, + data.contains(shutdownBookieRef1.get()) && data.contains(shutdownBookieRef2.get())); + } + + /** + * Test audit of bookies is delayed during rolling upgrade scenario: + * a bookies goes down and comes up, the next bookie go down and up and so on. + * At any time only one bookie is down. + */ + @Test + public void testDelayedAuditWithRollingUpgrade() throws Exception { + // wait for the periodic bookie check to finish + Thread.sleep(1000); + + // create a ledger with a bunch of entries + LedgerHandle lh1 = createAndAddEntriesToLedger(); + Long ledgerId = lh1.getId(); + if (LOG.isDebugEnabled()) { + LOG.debug("Created ledger : " + ledgerId); + } + ledgerList.add(ledgerId); + lh1.close(); + + CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList.size()); + + // wait for 5 seconds before starting the recovery work when a bookie fails + urLedgerMgr.setLostBookieRecoveryDelay(5); + + // shutdown a non auditor bookie to avoid an election + int idx1 = getShutDownNonAuditorBookieIdx(""); + ServerConfiguration conf1 = confByIndex(idx1); + AtomicReference shutdownBookieRef1 = new AtomicReference<>(); + CountDownLatch shutdownLatch1 = new CountDownLatch(1); + new Thread(() -> { + try { + String shutdownBookie1 = shutdownBookie(idx1); + shutdownBookieRef1.set(shutdownBookie1); + shutdownLatch1.countDown(); + } catch (Exception ignore) { + } + }).start(); + + // wait for 2 seconds and there shouldn't be any under replicated ledgers + // because we have delayed the start of audit by 5 seconds + assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(2, TimeUnit.SECONDS)); + assertEquals("under replicated ledgers identified when it was not expected", 0, + urLedgerList.size()); + + // restart the bookie we shut down above + startAndAddBookie(conf1); + + // Now to simulate the rolling upgrade, bring down a bookie different from + // the one we brought down/up above. + // shutdown a non auditor bookie; choosing non-auditor to avoid another election + AtomicReference shutdownBookieRef2 = new AtomicReference<>(); + CountDownLatch shutdownLatch2 = new CountDownLatch(1); + new Thread(() -> { + try { + String shutdownBookie2 = shutDownNonAuditorBookie(); + shutdownBookieRef2.set(shutdownBookie2); + shutdownLatch2.countDown(); + } catch (Exception ignore) { + } + }).start(); + + // since the first bookie that was brought down/up has come up, there is only + // one bookie down at this time. Hence the lost bookie check shouldn't start + // immediately; it will start 5 seconds after the second bookie went down + assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(2, TimeUnit.SECONDS)); + assertEquals("under replicated ledgers identified when it was not expected", 0, + urLedgerList.size()); + + // wait for a total of 6 seconds(2+4) for the ledgers to get reported as under replicated + Thread.sleep(4000); + + // If the following checks pass, it means that auditing happened + // after lostBookieRecoveryDelay during rolling upgrade as expected + assertTrue("Ledger is not marked as underreplicated:" + ledgerId, + urLedgerList.contains(ledgerId)); + Map urLedgerData = getUrLedgerData(urLedgerList); + String data = urLedgerData.get(ledgerId); + shutdownLatch1.await(); + shutdownLatch2.await(); + assertTrue("Bookie " + shutdownBookieRef1.get() + "wrongly listed as missing the ledger: " + data, + !data.contains(shutdownBookieRef1.get())); + assertTrue("Bookie " + shutdownBookieRef2.get() + + " is not listed in the ledger as missing replicas :" + data, + data.contains(shutdownBookieRef2.get())); + LOG.info("*****************Test Complete"); + } + + private void waitForAuditToComplete() throws Exception { + long endTime = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < endTime) { + Auditor auditor = getAuditorBookiesAuditor(); + if (auditor != null) { + Future task = auditor.submitAuditTask(); + task.get(5, TimeUnit.SECONDS); + return; + } + Thread.sleep(100); + } + throw new TimeoutException("Could not find an audit within 5 seconds"); + } + + /** + * Wait for ledger to be underreplicated, and to be missing all replicas specified. + */ + private boolean waitForLedgerMissingReplicas(Long ledgerId, long secondsToWait, String... replicas) + throws Exception { + for (int i = 0; i < secondsToWait; i++) { + try { + UnderreplicatedLedger data = urLedgerMgr.getLedgerUnreplicationInfo(ledgerId); + boolean all = true; + for (String r : replicas) { + all = all && data.getReplicaList().contains(r); + } + if (all) { + return true; + } + } catch (Exception e) { + // may not find node + } + Thread.sleep(1000); + } + return false; + } + + private CountDownLatch registerUrLedgerWatcher(int count) + throws KeeperException, InterruptedException { + final CountDownLatch underReplicaLatch = new CountDownLatch(count); + for (Long ledgerId : ledgerList) { + Watcher urLedgerWatcher = new ChildWatcher(underReplicaLatch); + String znode = ZkLedgerUnderreplicationManager.getUrLedgerZnode(underreplicatedPath, + ledgerId); + zkc.exists(znode, urLedgerWatcher); + } + return underReplicaLatch; + } + + private void doLedgerRereplication(Long... ledgerIds) + throws UnavailableException { + for (int i = 0; i < ledgerIds.length; i++) { + long lid = urLedgerMgr.getLedgerToRereplicate(); + assertTrue("Received unexpected ledgerid", Arrays.asList(ledgerIds).contains(lid)); + urLedgerMgr.markLedgerReplicated(lid); + urLedgerMgr.releaseUnderreplicatedLedger(lid); + } + } + + private String shutdownBookie(int bkShutdownIndex) throws Exception { + BookieServer bkServer = serverByIndex(bkShutdownIndex); + String bookieAddr = bkServer.getBookieId().toString(); + if (LOG.isInfoEnabled()) { + LOG.info("Shutting down bookie:" + bookieAddr); + } + killBookie(bkShutdownIndex); + auditorElectors.get(bookieAddr).shutdown(); + auditorElectors.remove(bookieAddr); + return bookieAddr; + } + + private LedgerHandle createAndAddEntriesToLedger() throws BKException, + InterruptedException { + int numEntriesToWrite = 100; + // Create a ledger + LedgerHandle lh = bkc.createLedger(digestType, ledgerPassword); + LOG.info("Ledger ID: " + lh.getId()); + addEntry(numEntriesToWrite, lh); + return lh; + } + + private void addEntry(int numEntriesToWrite, LedgerHandle lh) + throws InterruptedException, BKException { + final CountDownLatch completeLatch = new CountDownLatch(numEntriesToWrite); + final AtomicInteger rc = new AtomicInteger(BKException.Code.OK); + + for (int i = 0; i < numEntriesToWrite; i++) { + ByteBuffer entry = ByteBuffer.allocate(4); + entry.putInt(rng.nextInt(Integer.MAX_VALUE)); + entry.position(0); + lh.asyncAddEntry(entry.array(), new AddCallback() { + public void addComplete(int rc2, LedgerHandle lh, long entryId, Object ctx) { + rc.compareAndSet(BKException.Code.OK, rc2); + completeLatch.countDown(); + } + }, null); + } + completeLatch.await(); + if (rc.get() != BKException.Code.OK) { + throw BKException.create(rc.get()); + } + + } + + private Map getUrLedgerData(Set urLedgerList) + throws KeeperException, InterruptedException { + Map urLedgerData = new HashMap(); + for (Long ledgerId : urLedgerList) { + String znode = ZkLedgerUnderreplicationManager.getUrLedgerZnode(underreplicatedPath, + ledgerId); + byte[] data = zkc.getData(znode, false, null); + urLedgerData.put(ledgerId, new String(data)); + } + return urLedgerData; + } + + private class ChildWatcher implements Watcher { + private final CountDownLatch underReplicaLatch; + + public ChildWatcher(CountDownLatch underReplicaLatch) { + this.underReplicaLatch = underReplicaLatch; + } + + @Override + public void process(WatchedEvent event) { + LOG.info("Received notification for the ledger path : " + + event.getPath()); + for (Long ledgerId : ledgerList) { + if (event.getPath().contains(ledgerId + "")) { + urLedgerList.add(ledgerId); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("Count down and waiting for next notification"); + } + // count down and waiting for next notification + underReplicaLatch.countDown(); + } + } + + private BookieServer getAuditorBookie() throws Exception { + List auditors = new LinkedList(); + byte[] data = zkc.getData(electionPath, false, null); + assertNotNull("Auditor election failed", data); + for (int i = 0; i < bookieCount(); i++) { + BookieId bookieId = addressByIndex(i); + if (new String(data).contains(bookieId + "")) { + auditors.add(serverByIndex(i)); + } + } + assertEquals("Multiple Bookies acting as Auditor!", 1, auditors + .size()); + return auditors.get(0); + } + + private Auditor getAuditorBookiesAuditor() throws Exception { + BookieServer auditorBookieServer = getAuditorBookie(); + String bookieAddr = auditorBookieServer.getBookieId().toString(); + return auditorElectors.get(bookieAddr).auditor; + } + + private String shutDownNonAuditorBookie() throws Exception { + // shutdown bookie which is not an auditor + int indexOf = indexOfServer(getAuditorBookie()); + int bkIndexDownBookie; + if (indexOf < lastBookieIndex()) { + bkIndexDownBookie = indexOf + 1; + } else { + bkIndexDownBookie = indexOf - 1; + } + return shutdownBookie(bkIndexDownBookie); + } + + private int getShutDownNonAuditorBookieIdx(String exclude) throws Exception { + // shutdown bookie which is not an auditor + int indexOf = indexOfServer(getAuditorBookie()); + int bkIndexDownBookie = 0; + for (int i = 0; i <= lastBookieIndex(); i++) { + if (i == indexOf || addressByIndex(i).toString().equals(exclude)) { + continue; + } + bkIndexDownBookie = i; + break; + } + return bkIndexDownBookie; + } + + private String shutDownNonAuditorBookie(String exclude) throws Exception { + return shutdownBookie(getShutDownNonAuditorBookieIdx(exclude)); + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java new file mode 100644 index 0000000000000..9e8c5a54a5d91 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.apache.bookkeeper.meta.MetadataDrivers.runFunctionWithLedgerManagerFactory; +import static org.testng.AssertJUnit.assertEquals; +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.UncheckedExecutionException; +import lombok.Cleanup; +import org.apache.bookkeeper.client.ClientUtil; +import org.apache.bookkeeper.client.LedgerMetadataBuilder; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.conf.TestBKConfiguration; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.net.BookieSocketAddress; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * This test verifies that the period check on the auditor + * will pick up on missing data in the client. + */ +public class AuditorPeriodicBookieCheckTest extends BookKeeperClusterTestCase { + private static final Logger LOG = LoggerFactory + .getLogger(AuditorPeriodicBookieCheckTest.class); + + private AuditorElector auditorElector = null; + + private static final int CHECK_INTERVAL = 1; // run every second + + public AuditorPeriodicBookieCheckTest() throws Exception { + super(3); + baseConf.setPageLimit(1); // to make it easy to push ledger out of cache + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + + ServerConfiguration conf = TestBKConfiguration.newServerConfiguration(); + conf.setAuditorPeriodicBookieCheckInterval(CHECK_INTERVAL); + + conf.setMetadataServiceUri( + metadataServiceUri.replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + conf.setProperty("clientConnectTimeoutMillis", 500); + String addr = addressByIndex(0).toString(); + + auditorElector = new AuditorElector(addr, conf); + auditorElector.start(); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + auditorElector.shutdown(); + super.tearDown(); + } + + /** + * Test that the periodic bookie checker works. + */ + @Test + public void testPeriodicBookieCheckInterval() throws Exception { + confByIndex(0).setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + runFunctionWithLedgerManagerFactory(confByIndex(0), mFactory -> { + try (LedgerManager ledgerManager = mFactory.newLedgerManager()) { + @Cleanup final LedgerUnderreplicationManager underReplicationManager = + mFactory.newLedgerUnderreplicationManager(); + long ledgerId = 12345L; + ClientUtil.setupLedger(bkc.getLedgerManager(), ledgerId, + LedgerMetadataBuilder.create().withEnsembleSize(3) + .withWriteQuorumSize(3).withAckQuorumSize(3) + .newEnsembleEntry(0L, Lists.newArrayList( + new BookieSocketAddress("192.0.2.1", 1000).toBookieId(), + getBookie(0), + getBookie(1)))); + long underReplicatedLedger = -1; + for (int i = 0; i < 10; i++) { + underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate(); + if (underReplicatedLedger != -1) { + break; + } + Thread.sleep(CHECK_INTERVAL * 1000); + } + assertEquals("Ledger should be under replicated", ledgerId, underReplicatedLedger); + } catch (Exception e) { + throw new UncheckedExecutionException(e.getMessage(), e); + } + return null; + }); + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java new file mode 100644 index 0000000000000..9c5805dc536d6 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java @@ -0,0 +1,1070 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE; +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertFalse; +import static org.testng.AssertJUnit.assertNotSame; +import static org.testng.AssertJUnit.assertTrue; +import static org.testng.AssertJUnit.fail; +import io.netty.buffer.ByteBuf; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FilenameFilter; +import java.io.IOException; +import java.net.URI; +import java.net.UnknownHostException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.bookkeeper.bookie.Bookie; +import org.apache.bookkeeper.bookie.BookieAccessor; +import org.apache.bookkeeper.bookie.BookieException; +import org.apache.bookkeeper.bookie.BookieImpl; +import org.apache.bookkeeper.bookie.IndexPersistenceMgr; +import org.apache.bookkeeper.bookie.TestBookieImpl; +import org.apache.bookkeeper.client.AsyncCallback.AddCallback; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.BookKeeper.DigestType; +import org.apache.bookkeeper.client.BookKeeperAdmin; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.meta.LedgerManagerFactory; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.meta.MetadataBookieDriver; +import org.apache.bookkeeper.meta.MetadataDrivers; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.WriteCallback; +import org.apache.bookkeeper.replication.ReplicationException.UnavailableException; +import org.apache.bookkeeper.stats.Counter; +import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.bookkeeper.stats.StatsLogger; +import org.apache.bookkeeper.test.TestStatsProvider; +import org.apache.bookkeeper.test.TestStatsProvider.TestOpStatsLogger; +import org.apache.bookkeeper.test.TestStatsProvider.TestStatsLogger; +import org.awaitility.Awaitility; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * This test verifies that the period check on the auditor + * will pick up on missing data in the client. + */ +public class AuditorPeriodicCheckTest extends BookKeeperClusterTestCase { + private static final Logger LOG = LoggerFactory + .getLogger(AuditorPeriodicCheckTest.class); + + private MetadataBookieDriver driver; + private HashMap auditorElectors = new HashMap(); + + private static final int CHECK_INTERVAL = 1; // run every second + + public AuditorPeriodicCheckTest() throws Exception { + super(3); + baseConf.setPageLimit(1); // to make it easy to push ledger out of cache + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + + for (int i = 0; i < numBookies; i++) { + ServerConfiguration conf = new ServerConfiguration(confByIndex(i)); + conf.setAuditorPeriodicCheckInterval(CHECK_INTERVAL); + conf.setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + + String addr = addressByIndex(i).toString(); + + AuditorElector auditorElector = new AuditorElector(addr, conf); + auditorElectors.put(addr, auditorElector); + auditorElector.start(); + if (LOG.isDebugEnabled()) { + LOG.debug("Starting Auditor Elector"); + } + } + + URI uri = URI.create(confByIndex(0).getMetadataServiceUri().replaceAll("zk://", "metadata-store:") + .replaceAll("/ledgers", "")); + driver = MetadataDrivers.getBookieDriver(uri); + ServerConfiguration serverConfiguration = new ServerConfiguration(confByIndex(0)); + serverConfiguration.setMetadataServiceUri( + serverConfiguration.getMetadataServiceUri().replaceAll("zk://", "metadata-store:") + .replaceAll("/ledgers", "")); + driver.initialize(serverConfiguration, NullStatsLogger.INSTANCE); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + if (null != driver) { + driver.close(); + } + + for (AuditorElector e : auditorElectors.values()) { + e.shutdown(); + } + super.tearDown(); + } + + /** + * test that the periodic checking will detect corruptions in + * the bookie entry log. + */ + @Test + public void testEntryLogCorruption() throws Exception { + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager(); + underReplicationManager.disableLedgerReplication(); + + LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + long ledgerId = lh.getId(); + for (int i = 0; i < 100; i++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + + BookieAccessor.forceFlush((BookieImpl) serverByIndex(0).getBookie()); + + + File ledgerDir = confByIndex(0).getLedgerDirs()[0]; + ledgerDir = BookieImpl.getCurrentDirectory(ledgerDir); + // corrupt of entryLogs + File[] entryLogs = ledgerDir.listFiles(new FilenameFilter() { + public boolean accept(File dir, String name) { + return name.endsWith(".log"); + } + }); + ByteBuffer junk = ByteBuffer.allocate(1024 * 1024); + for (File f : entryLogs) { + FileOutputStream out = new FileOutputStream(f); + out.getChannel().write(junk); + out.close(); + } + restartBookies(); // restart to clear read buffers + + underReplicationManager.enableLedgerReplication(); + long underReplicatedLedger = -1; + for (int i = 0; i < 10; i++) { + underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate(); + if (underReplicatedLedger != -1) { + break; + } + Thread.sleep(CHECK_INTERVAL * 1000); + } + assertEquals("Ledger should be under replicated", ledgerId, underReplicatedLedger); + underReplicationManager.close(); + } + + /** + * test that the period checker will detect corruptions in + * the bookie index files. + */ + @Test + public void testIndexCorruption() throws Exception { + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + + LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager(); + + LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + long ledgerToCorrupt = lh.getId(); + for (int i = 0; i < 100; i++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + + // push ledgerToCorrupt out of page cache (bookie is configured to only use 1 page) + lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + for (int i = 0; i < 100; i++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + + BookieAccessor.forceFlush((BookieImpl) serverByIndex(0).getBookie()); + + File ledgerDir = confByIndex(0).getLedgerDirs()[0]; + ledgerDir = BookieImpl.getCurrentDirectory(ledgerDir); + + // corrupt of entryLogs + File index = new File(ledgerDir, IndexPersistenceMgr.getLedgerName(ledgerToCorrupt)); + LOG.info("file to corrupt{}", index); + ByteBuffer junk = ByteBuffer.allocate(1024 * 1024); + FileOutputStream out = new FileOutputStream(index); + out.getChannel().write(junk); + out.close(); + + long underReplicatedLedger = -1; + for (int i = 0; i < 15; i++) { + underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate(); + if (underReplicatedLedger != -1) { + break; + } + Thread.sleep(CHECK_INTERVAL * 1000); + } + assertEquals("Ledger should be under replicated", ledgerToCorrupt, underReplicatedLedger); + underReplicationManager.close(); + } + + /** + * Test that the period checker will not run when auto replication has been disabled. + */ + @Test + public void testPeriodicCheckWhenDisabled() throws Exception { + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + final LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager(); + final int numLedgers = 10; + final int numMsgs = 2; + final CountDownLatch completeLatch = new CountDownLatch(numMsgs * numLedgers); + final AtomicInteger rc = new AtomicInteger(BKException.Code.OK); + + List lhs = new ArrayList(); + for (int i = 0; i < numLedgers; i++) { + LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + lhs.add(lh); + for (int j = 0; j < 2; j++) { + lh.asyncAddEntry("testdata".getBytes(), new AddCallback() { + public void addComplete(int rc2, LedgerHandle lh, long entryId, Object ctx) { + if (rc.compareAndSet(BKException.Code.OK, rc2)) { + LOG.info("Failed to add entry : {}", BKException.getMessage(rc2)); + } + completeLatch.countDown(); + } + }, null); + } + } + completeLatch.await(); + if (rc.get() != BKException.Code.OK) { + throw BKException.create(rc.get()); + } + + for (LedgerHandle lh : lhs) { + lh.close(); + } + + underReplicationManager.disableLedgerReplication(); + + final AtomicInteger numReads = new AtomicInteger(0); + ServerConfiguration conf = killBookie(0); + + Bookie deadBookie = new TestBookieImpl(conf) { + @Override + public ByteBuf readEntry(long ledgerId, long entryId) + throws IOException, NoLedgerException { + // we want to disable during checking + numReads.incrementAndGet(); + throw new IOException("Fake I/O exception"); + } + }; + startAndAddBookie(conf, deadBookie); + + Thread.sleep(CHECK_INTERVAL * 2000); + assertEquals("Nothing should have tried to read", 0, numReads.get()); + underReplicationManager.enableLedgerReplication(); + Thread.sleep(CHECK_INTERVAL * 2000); // give it time to run + + underReplicationManager.disableLedgerReplication(); + // give it time to stop, from this point nothing new should be marked + Thread.sleep(CHECK_INTERVAL * 2000); + + int numUnderreplicated = 0; + long underReplicatedLedger = -1; + do { + underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate(); + if (underReplicatedLedger == -1) { + break; + } + numUnderreplicated++; + + underReplicationManager.markLedgerReplicated(underReplicatedLedger); + } while (underReplicatedLedger != -1); + + Thread.sleep(CHECK_INTERVAL * 2000); // give a chance to run again (it shouldn't, it's disabled) + + // ensure that nothing is marked as underreplicated + underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate(); + assertEquals("There should be no underreplicated ledgers", -1, underReplicatedLedger); + + LOG.info("{} of {} ledgers underreplicated", numUnderreplicated, numUnderreplicated); + assertTrue("All should be underreplicated", + numUnderreplicated <= numLedgers && numUnderreplicated > 0); + } + + /** + * Test that the period check will succeed if a ledger is deleted midway. + */ + @Test + public void testPeriodicCheckWhenLedgerDeleted() throws Exception { + for (AuditorElector e : auditorElectors.values()) { + e.shutdown(); + } + + final int numLedgers = 10; + List ids = new LinkedList(); + for (int i = 0; i < numLedgers; i++) { + LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + ids.add(lh.getId()); + for (int j = 0; j < 2; j++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + } + + try (final Auditor auditor = new Auditor( + BookieImpl.getBookieId(confByIndex(0)).toString(), + confByIndex(0), NullStatsLogger.INSTANCE)) { + final AtomicBoolean exceptionCaught = new AtomicBoolean(false); + final CountDownLatch latch = new CountDownLatch(1); + Thread t = new Thread() { + public void run() { + try { + latch.countDown(); + for (int i = 0; i < numLedgers; i++) { + ((AuditorCheckAllLedgersTask) auditor.auditorCheckAllLedgersTask).checkAllLedgers(); + } + } catch (Exception e) { + LOG.error("Caught exception while checking all ledgers", e); + exceptionCaught.set(true); + } + } + }; + t.start(); + latch.await(); + for (Long id : ids) { + bkc.deleteLedger(id); + } + t.join(); + assertFalse("Shouldn't have thrown exception", exceptionCaught.get()); + } + } + + @Test + public void testGetLedgerFromZookeeperThrottled() throws Exception { + final int numberLedgers = 30; + + // write ledgers into bookkeeper cluster + try { + for (AuditorElector e : auditorElectors.values()) { + e.shutdown(); + } + + for (int i = 0; i < numberLedgers; ++i) { + LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + for (int j = 0; j < 5; j++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + } + } catch (InterruptedException | BKException e) { + LOG.error("Failed to shutdown auditor elector or write data to ledgers ", e); + fail(); + } + + // create auditor and call `checkAllLedgers` + ServerConfiguration configuration = confByIndex(0); + configuration.setAuditorMaxNumberOfConcurrentOpenLedgerOperations(10); + + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + Counter numLedgersChecked = statsLogger + .getCounter(ReplicationStats.NUM_LEDGERS_CHECKED); + Auditor auditor = new Auditor(BookieImpl.getBookieId(configuration).toString(), + configuration, statsLogger); + + try { + ((AuditorCheckAllLedgersTask) auditor.auditorCheckAllLedgersTask).checkAllLedgers(); + assertEquals("NUM_LEDGERS_CHECKED", numberLedgers, (long) numLedgersChecked.get()); + } catch (Exception e) { + LOG.error("Caught exception while checking all ledgers ", e); + fail(); + } + } + + @Test + public void testInitialDelayOfCheckAllLedgers() throws Exception { + for (AuditorElector e : auditorElectors.values()) { + e.shutdown(); + } + + final int numLedgers = 10; + List ids = new LinkedList(); + for (int i = 0; i < numLedgers; i++) { + LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + ids.add(lh.getId()); + for (int j = 0; j < 2; j++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + } + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager(); + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + validateInitialDelayOfCheckAllLedgers(urm, -1, 1000, servConf, bkc); + validateInitialDelayOfCheckAllLedgers(urm, 999, 1000, servConf, bkc); + validateInitialDelayOfCheckAllLedgers(urm, 1001, 1000, servConf, bkc); + } + + void validateInitialDelayOfCheckAllLedgers(LedgerUnderreplicationManager urm, long timeSinceLastExecutedInSecs, + long auditorPeriodicCheckInterval, ServerConfiguration servConf, + BookKeeper bkc) + throws UnavailableException, UnknownHostException, InterruptedException { + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + TestOpStatsLogger checkAllLedgersStatsLogger = (TestOpStatsLogger) statsLogger + .getOpStatsLogger(ReplicationStats.CHECK_ALL_LEDGERS_TIME); + servConf.setAuditorPeriodicCheckInterval(auditorPeriodicCheckInterval); + servConf.setAuditorPeriodicPlacementPolicyCheckInterval(0); + servConf.setAuditorPeriodicBookieCheckInterval(0); + + final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, bkc, false, + statsLogger, null); + CountDownLatch latch = auditor.getLatch(); + assertEquals("CHECK_ALL_LEDGERS_TIME SuccessCount", 0, checkAllLedgersStatsLogger.getSuccessCount()); + long curTimeBeforeStart = System.currentTimeMillis(); + long checkAllLedgersCTime = -1; + long initialDelayInMsecs = -1; + long nextExpectedCheckAllLedgersExecutionTime = -1; + long bufferTimeInMsecs = 12000L; + if (timeSinceLastExecutedInSecs == -1) { + /* + * if we are setting checkAllLedgersCTime to -1, it means that + * checkAllLedgers hasn't run before. So initialDelay for + * checkAllLedgers should be 0. + */ + checkAllLedgersCTime = -1; + initialDelayInMsecs = 0; + } else { + checkAllLedgersCTime = curTimeBeforeStart - timeSinceLastExecutedInSecs * 1000L; + initialDelayInMsecs = timeSinceLastExecutedInSecs > auditorPeriodicCheckInterval ? 0 + : (auditorPeriodicCheckInterval - timeSinceLastExecutedInSecs) * 1000L; + } + /* + * next checkAllLedgers should happen atleast after + * nextExpectedCheckAllLedgersExecutionTime. + */ + nextExpectedCheckAllLedgersExecutionTime = curTimeBeforeStart + initialDelayInMsecs; + + urm.setCheckAllLedgersCTime(checkAllLedgersCTime); + auditor.start(); + /* + * since auditorPeriodicCheckInterval are higher values (in the order of + * 100s of seconds), its ok bufferTimeInMsecs to be ` 10 secs. + */ + assertTrue("checkAllLedgers should have executed with initialDelay " + initialDelayInMsecs, + latch.await(initialDelayInMsecs + bufferTimeInMsecs, TimeUnit.MILLISECONDS)); + for (int i = 0; i < 10; i++) { + Thread.sleep(100); + if (checkAllLedgersStatsLogger.getSuccessCount() >= 1) { + break; + } + } + assertEquals("CHECK_ALL_LEDGERS_TIME SuccessCount", 1, checkAllLedgersStatsLogger.getSuccessCount()); + long currentCheckAllLedgersCTime = urm.getCheckAllLedgersCTime(); + assertTrue( + "currentCheckAllLedgersCTime: " + currentCheckAllLedgersCTime + + " should be greater than nextExpectedCheckAllLedgersExecutionTime: " + + nextExpectedCheckAllLedgersExecutionTime, + currentCheckAllLedgersCTime > nextExpectedCheckAllLedgersExecutionTime); + assertTrue( + "currentCheckAllLedgersCTime: " + currentCheckAllLedgersCTime + + " should be lesser than nextExpectedCheckAllLedgersExecutionTime+bufferTimeInMsecs: " + + (nextExpectedCheckAllLedgersExecutionTime + bufferTimeInMsecs), + currentCheckAllLedgersCTime < (nextExpectedCheckAllLedgersExecutionTime + bufferTimeInMsecs)); + auditor.close(); + } + + @Test + public void testInitialDelayOfPlacementPolicyCheck() throws Exception { + for (AuditorElector e : auditorElectors.values()) { + e.shutdown(); + } + + final int numLedgers = 10; + List ids = new LinkedList(); + for (int i = 0; i < numLedgers; i++) { + LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + ids.add(lh.getId()); + for (int j = 0; j < 2; j++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + } + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager(); + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + validateInitialDelayOfPlacementPolicyCheck(urm, -1, 1000, servConf, bkc); + validateInitialDelayOfPlacementPolicyCheck(urm, 999, 1000, servConf, bkc); + validateInitialDelayOfPlacementPolicyCheck(urm, 1001, 1000, servConf, bkc); + } + + void validateInitialDelayOfPlacementPolicyCheck(LedgerUnderreplicationManager urm, long timeSinceLastExecutedInSecs, + long auditorPeriodicPlacementPolicyCheckInterval, + ServerConfiguration servConf, BookKeeper bkc) + throws UnavailableException, UnknownHostException, InterruptedException { + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + TestOpStatsLogger placementPolicyCheckStatsLogger = (TestOpStatsLogger) statsLogger + .getOpStatsLogger(ReplicationStats.PLACEMENT_POLICY_CHECK_TIME); + servConf.setAuditorPeriodicPlacementPolicyCheckInterval(auditorPeriodicPlacementPolicyCheckInterval); + servConf.setAuditorPeriodicCheckInterval(0); + servConf.setAuditorPeriodicBookieCheckInterval(0); + + final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, bkc, false, + statsLogger, null); + CountDownLatch latch = auditor.getLatch(); + assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 0, placementPolicyCheckStatsLogger.getSuccessCount()); + long curTimeBeforeStart = System.currentTimeMillis(); + long placementPolicyCheckCTime = -1; + long initialDelayInMsecs = -1; + long nextExpectedPlacementPolicyCheckExecutionTime = -1; + long bufferTimeInMsecs = 20000L; + if (timeSinceLastExecutedInSecs == -1) { + /* + * if we are setting placementPolicyCheckCTime to -1, it means that + * placementPolicyCheck hasn't run before. So initialDelay for + * placementPolicyCheck should be 0. + */ + placementPolicyCheckCTime = -1; + initialDelayInMsecs = 0; + } else { + placementPolicyCheckCTime = curTimeBeforeStart - timeSinceLastExecutedInSecs * 1000L; + initialDelayInMsecs = timeSinceLastExecutedInSecs > auditorPeriodicPlacementPolicyCheckInterval ? 0 + : (auditorPeriodicPlacementPolicyCheckInterval - timeSinceLastExecutedInSecs) * 1000L; + } + /* + * next placementPolicyCheck should happen atleast after + * nextExpectedPlacementPolicyCheckExecutionTime. + */ + nextExpectedPlacementPolicyCheckExecutionTime = curTimeBeforeStart + initialDelayInMsecs; + + urm.setPlacementPolicyCheckCTime(placementPolicyCheckCTime); + auditor.start(); + /* + * since auditorPeriodicPlacementPolicyCheckInterval are higher values (in the + * order of 100s of seconds), its ok bufferTimeInMsecs to be ` 20 secs. + */ + assertTrue("placementPolicyCheck should have executed with initialDelay " + initialDelayInMsecs, + latch.await(initialDelayInMsecs + bufferTimeInMsecs, TimeUnit.MILLISECONDS)); + for (int i = 0; i < 20; i++) { + Thread.sleep(100); + if (placementPolicyCheckStatsLogger.getSuccessCount() >= 1) { + break; + } + } + assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 1, placementPolicyCheckStatsLogger.getSuccessCount()); + long currentPlacementPolicyCheckCTime = urm.getPlacementPolicyCheckCTime(); + assertTrue( + "currentPlacementPolicyCheckCTime: " + currentPlacementPolicyCheckCTime + + " should be greater than nextExpectedPlacementPolicyCheckExecutionTime: " + + nextExpectedPlacementPolicyCheckExecutionTime, + currentPlacementPolicyCheckCTime > nextExpectedPlacementPolicyCheckExecutionTime); + assertTrue( + "currentPlacementPolicyCheckCTime: " + currentPlacementPolicyCheckCTime + + " should be lesser than nextExpectedPlacementPolicyCheckExecutionTime+bufferTimeInMsecs: " + + (nextExpectedPlacementPolicyCheckExecutionTime + bufferTimeInMsecs), + currentPlacementPolicyCheckCTime < (nextExpectedPlacementPolicyCheckExecutionTime + bufferTimeInMsecs)); + auditor.close(); + } + + @Test + public void testInitialDelayOfReplicasCheck() throws Exception { + for (AuditorElector e : auditorElectors.values()) { + e.shutdown(); + } + + LedgerHandle lh = bkc.createLedger(3, 2, DigestType.CRC32, "passwd".getBytes()); + for (int j = 0; j < 5; j++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + + long ledgerId = 100000L; + lh = bkc.createLedgerAdv(ledgerId, 3, 2, 2, DigestType.CRC32, "passwd".getBytes(), null); + lh.close(); + + ledgerId = 100001234L; + lh = bkc.createLedgerAdv(ledgerId, 3, 3, 2, DigestType.CRC32, "passwd".getBytes(), null); + for (int j = 0; j < 4; j++) { + lh.addEntry(j, "testdata".getBytes()); + } + lh.close(); + + ledgerId = 991234L; + lh = bkc.createLedgerAdv(ledgerId, 3, 2, 2, DigestType.CRC32, "passwd".getBytes(), null); + lh.addEntry(0, "testdata".getBytes()); + lh.close(); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager(); + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + validateInitialDelayOfReplicasCheck(urm, -1, 1000, servConf, bkc); + validateInitialDelayOfReplicasCheck(urm, 999, 1000, servConf, bkc); + validateInitialDelayOfReplicasCheck(urm, 1001, 1000, servConf, bkc); + } + + void validateInitialDelayOfReplicasCheck(LedgerUnderreplicationManager urm, long timeSinceLastExecutedInSecs, + long auditorPeriodicReplicasCheckInterval, ServerConfiguration servConf, + BookKeeper bkc) + throws UnavailableException, UnknownHostException, InterruptedException { + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + TestOpStatsLogger replicasCheckStatsLogger = (TestOpStatsLogger) statsLogger + .getOpStatsLogger(ReplicationStats.REPLICAS_CHECK_TIME); + servConf.setAuditorPeriodicReplicasCheckInterval(auditorPeriodicReplicasCheckInterval); + servConf.setAuditorPeriodicCheckInterval(0); + servConf.setAuditorPeriodicBookieCheckInterval(0); + final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, bkc, false, + statsLogger, null); + CountDownLatch latch = auditor.getLatch(); + assertEquals("REPLICAS_CHECK_TIME SuccessCount", 0, replicasCheckStatsLogger.getSuccessCount()); + long curTimeBeforeStart = System.currentTimeMillis(); + long replicasCheckCTime = -1; + long initialDelayInMsecs = -1; + long nextExpectedReplicasCheckExecutionTime = -1; + long bufferTimeInMsecs = 20000L; + if (timeSinceLastExecutedInSecs == -1) { + /* + * if we are setting replicasCheckCTime to -1, it means that + * replicasCheck hasn't run before. So initialDelay for + * replicasCheck should be 0. + */ + replicasCheckCTime = -1; + initialDelayInMsecs = 0; + } else { + replicasCheckCTime = curTimeBeforeStart - timeSinceLastExecutedInSecs * 1000L; + initialDelayInMsecs = timeSinceLastExecutedInSecs > auditorPeriodicReplicasCheckInterval ? 0 + : (auditorPeriodicReplicasCheckInterval - timeSinceLastExecutedInSecs) * 1000L; + } + /* + * next replicasCheck should happen atleast after + * nextExpectedReplicasCheckExecutionTime. + */ + nextExpectedReplicasCheckExecutionTime = curTimeBeforeStart + initialDelayInMsecs; + + urm.setReplicasCheckCTime(replicasCheckCTime); + auditor.start(); + /* + * since auditorPeriodicReplicasCheckInterval are higher values (in the + * order of 100s of seconds), its ok bufferTimeInMsecs to be ` 20 secs. + */ + assertTrue("replicasCheck should have executed with initialDelay " + initialDelayInMsecs, + latch.await(initialDelayInMsecs + bufferTimeInMsecs, TimeUnit.MILLISECONDS)); + for (int i = 0; i < 20; i++) { + Thread.sleep(100); + if (replicasCheckStatsLogger.getSuccessCount() >= 1) { + break; + } + } + assertEquals("REPLICAS_CHECK_TIME SuccessCount", 1, replicasCheckStatsLogger.getSuccessCount()); + long currentReplicasCheckCTime = urm.getReplicasCheckCTime(); + assertTrue( + "currentReplicasCheckCTime: " + currentReplicasCheckCTime + + " should be greater than nextExpectedReplicasCheckExecutionTime: " + + nextExpectedReplicasCheckExecutionTime, + currentReplicasCheckCTime > nextExpectedReplicasCheckExecutionTime); + assertTrue( + "currentReplicasCheckCTime: " + currentReplicasCheckCTime + + " should be lesser than nextExpectedReplicasCheckExecutionTime+bufferTimeInMsecs: " + + (nextExpectedReplicasCheckExecutionTime + bufferTimeInMsecs), + currentReplicasCheckCTime < (nextExpectedReplicasCheckExecutionTime + bufferTimeInMsecs)); + auditor.close(); + } + + @Test + public void testDelayBookieAuditOfCheckAllLedgers() throws Exception { + for (AuditorElector e : auditorElectors.values()) { + e.shutdown(); + } + + final int numLedgers = 10; + List ids = new LinkedList(); + for (int i = 0; i < numLedgers; i++) { + LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + ids.add(lh.getId()); + for (int j = 0; j < 2; j++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + } + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager(); + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + Counter numBookieAuditsDelayed = + statsLogger.getCounter(ReplicationStats.NUM_BOOKIE_AUDITS_DELAYED); + TestOpStatsLogger underReplicatedLedgerTotalSizeStatsLogger = (TestOpStatsLogger) statsLogger + .getOpStatsLogger(ReplicationStats.UNDER_REPLICATED_LEDGERS_TOTAL_SIZE); + + servConf.setAuditorPeriodicCheckInterval(1); + servConf.setAuditorPeriodicPlacementPolicyCheckInterval(0); + servConf.setAuditorPeriodicBookieCheckInterval(Long.MAX_VALUE); + + urm.setLostBookieRecoveryDelay(Integer.MAX_VALUE); + + AtomicBoolean canRun = new AtomicBoolean(false); + + final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, bkc, + false, statsLogger, canRun); + final CountDownLatch latch = auditor.getLatch(); + + auditor.start(); + + killBookie(addressByIndex(0)); + + Awaitility.await().untilAsserted(() -> assertEquals(1, (long) numBookieAuditsDelayed.get())); + final Future auditTask = auditor.auditTask; + assertTrue(auditTask != null && !auditTask.isDone()); + + canRun.set(true); + + assertTrue(latch.await(10, TimeUnit.SECONDS)); + assertTrue(auditor.auditTask.equals(auditTask) + && auditor.auditTask != null && !auditor.auditTask.isDone()); + // wrong num is numLedgers, right num is 0 + assertEquals("UNDER_REPLICATED_LEDGERS_TOTAL_SIZE", + 0, + underReplicatedLedgerTotalSizeStatsLogger.getSuccessCount()); + + auditor.close(); + } + + @Test + public void testDelayBookieAuditOfPlacementPolicy() throws Exception { + for (AuditorElector e : auditorElectors.values()) { + e.shutdown(); + } + + final int numLedgers = 10; + List ids = new LinkedList(); + for (int i = 0; i < numLedgers; i++) { + LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + ids.add(lh.getId()); + for (int j = 0; j < 2; j++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + } + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager(); + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + Counter numBookieAuditsDelayed = + statsLogger.getCounter(ReplicationStats.NUM_BOOKIE_AUDITS_DELAYED); + TestOpStatsLogger placementPolicyCheckTime = (TestOpStatsLogger) statsLogger + .getOpStatsLogger(ReplicationStats.PLACEMENT_POLICY_CHECK_TIME); + + servConf.setAuditorPeriodicCheckInterval(0); + servConf.setAuditorPeriodicPlacementPolicyCheckInterval(1); + servConf.setAuditorPeriodicBookieCheckInterval(Long.MAX_VALUE); + + urm.setLostBookieRecoveryDelay(Integer.MAX_VALUE); + + AtomicBoolean canRun = new AtomicBoolean(false); + + final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, bkc, + false, statsLogger, canRun); + final CountDownLatch latch = auditor.getLatch(); + + auditor.start(); + + killBookie(addressByIndex(0)); + + Awaitility.await().untilAsserted(() -> assertEquals(1, (long) numBookieAuditsDelayed.get())); + final Future auditTask = auditor.auditTask; + assertTrue(auditTask != null && !auditTask.isDone()); + assertEquals("PLACEMENT_POLICY_CHECK_TIME", 0, placementPolicyCheckTime.getSuccessCount()); + + canRun.set(true); + + assertTrue(latch.await(10, TimeUnit.SECONDS)); + assertTrue(auditor.auditTask.equals(auditTask) + && auditor.auditTask != null && !auditor.auditTask.isDone()); + // wrong successCount is > 0, right successCount is = 0 + assertEquals("PLACEMENT_POLICY_CHECK_TIME", 0, placementPolicyCheckTime.getSuccessCount()); + + auditor.close(); + } + + @Test + public void testDelayBookieAuditOfReplicasCheck() throws Exception { + for (AuditorElector e : auditorElectors.values()) { + e.shutdown(); + } + + final int numLedgers = 10; + List ids = new LinkedList(); + for (int i = 0; i < numLedgers; i++) { + LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + ids.add(lh.getId()); + for (int j = 0; j < 2; j++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + } + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager(); + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + Counter numBookieAuditsDelayed = + statsLogger.getCounter(ReplicationStats.NUM_BOOKIE_AUDITS_DELAYED); + TestOpStatsLogger replicasCheckTime = (TestOpStatsLogger) statsLogger + .getOpStatsLogger(ReplicationStats.REPLICAS_CHECK_TIME); + + servConf.setAuditorPeriodicCheckInterval(0); + servConf.setAuditorPeriodicPlacementPolicyCheckInterval(0); + servConf.setAuditorPeriodicBookieCheckInterval(Long.MAX_VALUE); + servConf.setAuditorPeriodicReplicasCheckInterval(1); + + urm.setLostBookieRecoveryDelay(Integer.MAX_VALUE); + + AtomicBoolean canRun = new AtomicBoolean(false); + + final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, bkc, + false, statsLogger, canRun); + final CountDownLatch latch = auditor.getLatch(); + + auditor.start(); + + killBookie(addressByIndex(0)); + + Awaitility.await().untilAsserted(() -> assertEquals(1, (long) numBookieAuditsDelayed.get())); + final Future auditTask = auditor.auditTask; + assertTrue(auditTask != null && !auditTask.isDone()); + assertEquals("REPLICAS_CHECK_TIME", 0, replicasCheckTime.getSuccessCount()); + + canRun.set(true); + + assertTrue(latch.await(10, TimeUnit.SECONDS)); + assertTrue(auditor.auditTask.equals(auditTask) + && auditor.auditTask != null && !auditor.auditTask.isDone()); + // wrong successCount is > 0, right successCount is = 0 + assertEquals("REPLICAS_CHECK_TIME", 0, replicasCheckTime.getSuccessCount()); + + auditor.close(); + } + + static class TestAuditor extends Auditor { + + final AtomicReference latchRef = new AtomicReference(new CountDownLatch(1)); + + public TestAuditor(String bookieIdentifier, ServerConfiguration conf, BookKeeper bkc, boolean ownBkc, + StatsLogger statsLogger, AtomicBoolean exceptedRun) throws UnavailableException { + super(bookieIdentifier, conf, bkc, ownBkc, statsLogger); + renewAuditorTestWrapperTask(exceptedRun); + } + + public TestAuditor(String bookieIdentifier, ServerConfiguration conf, BookKeeper bkc, boolean ownBkc, + BookKeeperAdmin bkadmin, boolean ownadmin, StatsLogger statsLogger, + AtomicBoolean exceptedRun) throws UnavailableException { + super(bookieIdentifier, conf, bkc, ownBkc, bkadmin, ownadmin, statsLogger); + renewAuditorTestWrapperTask(exceptedRun); + } + + public TestAuditor(final String bookieIdentifier, ServerConfiguration conf, StatsLogger statsLogger, + AtomicBoolean exceptedRun) + throws UnavailableException { + super(bookieIdentifier, conf, statsLogger); + renewAuditorTestWrapperTask(exceptedRun); + } + + private void renewAuditorTestWrapperTask(AtomicBoolean exceptedRun) { + super.auditorCheckAllLedgersTask = + new AuditorTestWrapperTask(super.auditorCheckAllLedgersTask, latchRef, exceptedRun); + super.auditorPlacementPolicyCheckTask = + new AuditorTestWrapperTask(super.auditorPlacementPolicyCheckTask, latchRef, exceptedRun); + super.auditorReplicasCheckTask = + new AuditorTestWrapperTask(super.auditorReplicasCheckTask, latchRef, exceptedRun); + } + + CountDownLatch getLatch() { + return latchRef.get(); + } + + void setLatch(CountDownLatch latch) { + latchRef.set(latch); + } + + private static class AuditorTestWrapperTask extends AuditorTask { + private final AuditorTask innerTask; + private final AtomicReference latchRef; + private final AtomicBoolean exceptedRun; + + AuditorTestWrapperTask(AuditorTask innerTask, + AtomicReference latchRef, + AtomicBoolean exceptedRun) { + super(null, null, null, null, null, + null, null); + this.innerTask = innerTask; + this.latchRef = latchRef; + this.exceptedRun = exceptedRun; + } + + @Override + protected void runTask() { + if (exceptedRun == null || exceptedRun.get()) { + innerTask.runTask(); + latchRef.get().countDown(); + } + } + + @Override + public void shutdown() { + innerTask.shutdown(); + } + } + } + + private BookieId replaceBookieWithWriteFailingBookie(LedgerHandle lh) throws Exception { + int bookieIdx = -1; + Long entryId = lh.getLedgerMetadata().getAllEnsembles().firstKey(); + List curEnsemble = lh.getLedgerMetadata().getAllEnsembles().get(entryId); + + // Identify a bookie in the current ledger ensemble to be replaced + BookieId replacedBookie = null; + for (int i = 0; i < numBookies; i++) { + if (curEnsemble.contains(addressByIndex(i))) { + bookieIdx = i; + replacedBookie = addressByIndex(i); + break; + } + } + assertNotSame("Couldn't find ensemble bookie in bookie list", -1, bookieIdx); + + LOG.info("Killing bookie " + addressByIndex(bookieIdx)); + ServerConfiguration conf = killBookie(bookieIdx); + Bookie writeFailingBookie = new TestBookieImpl(conf) { + @Override + public void addEntry(ByteBuf entry, boolean ackBeforeSync, WriteCallback cb, + Object ctx, byte[] masterKey) + throws IOException, BookieException { + try { + LOG.info("Failing write to entry "); + // sleep a bit so that writes to other bookies succeed before + // the client hears about the failure on this bookie. If the + // client gets ack-quorum number of acks first, it won't care + // about any failures and won't reform the ensemble. + Thread.sleep(100); + throw new IOException(); + } catch (InterruptedException ie) { + // ignore, only interrupted if shutting down, + // and an exception would spam the logs + Thread.currentThread().interrupt(); + } + } + }; + startAndAddBookie(conf, writeFailingBookie); + return replacedBookie; + } + + /* + * Validates that the periodic ledger check will fix entries with a failed write. + */ + @Test + public void testFailedWriteRecovery() throws Exception { + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager(); + underReplicationManager.disableLedgerReplication(); + + LedgerHandle lh = bkc.createLedger(2, 2, 1, DigestType.CRC32, "passwd".getBytes()); + + // kill one of the bookies and replace it with one that rejects write; + // This way we get into the under replication state + BookieId replacedBookie = replaceBookieWithWriteFailingBookie(lh); + + // Write a few entries; this should cause under replication + byte[] data = "foobar".getBytes(); + data = "foobar".getBytes(); + lh.addEntry(data); + lh.addEntry(data); + lh.addEntry(data); + + lh.close(); + + // enable under replication detection and wait for it to report + // under replicated ledger + underReplicationManager.enableLedgerReplication(); + long underReplicatedLedger = -1; + for (int i = 0; i < 5; i++) { + underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate(); + if (underReplicatedLedger != -1) { + break; + } + Thread.sleep(CHECK_INTERVAL * 1000); + } + assertEquals("Ledger should be under replicated", lh.getId(), underReplicatedLedger); + + // now start the replication workers + List l = new ArrayList(); + for (int i = 0; i < numBookies; i++) { + ReplicationWorker rw = new ReplicationWorker(confByIndex(i), NullStatsLogger.INSTANCE); + rw.start(); + l.add(rw); + } + underReplicationManager.close(); + + // Wait for ensemble to change after replication + Thread.sleep(3000); + for (ReplicationWorker rw : l) { + rw.shutdown(); + } + + // check that ensemble has changed and the bookie that rejected writes has + // been replaced in the ensemble + LedgerHandle newLh = bkc.openLedger(lh.getId(), DigestType.CRC32, "passwd".getBytes()); + for (Map.Entry> e : + newLh.getLedgerMetadata().getAllEnsembles().entrySet()) { + List ensemble = e.getValue(); + assertFalse("Ensemble hasn't been updated", ensemble.contains(replacedBookie)); + } + newLh.close(); + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTaskTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTaskTest.java new file mode 100644 index 0000000000000..f5d6576229e2b --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTaskTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE; +import static org.testng.AssertJUnit.assertEquals; +import java.util.LinkedList; +import java.util.List; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.BookKeeperAdmin; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.meta.LedgerManagerFactory; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.bookkeeper.test.TestStatsProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Unit test {@link AuditorPlacementPolicyCheckTask}. + */ +public class AuditorPlacementPolicyCheckTaskTest extends BookKeeperClusterTestCase { + private static final Logger LOG = LoggerFactory + .getLogger(AuditorPlacementPolicyCheckTaskTest.class); + + private BookKeeperAdmin admin; + private LedgerManager ledgerManager; + private LedgerUnderreplicationManager ledgerUnderreplicationManager; + + public AuditorPlacementPolicyCheckTaskTest() throws Exception { + super(3); + baseConf.setPageLimit(1); + baseConf.setAutoRecoveryDaemonEnabled(false); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + baseClientConf.setMetadataServiceUri( + metadataServiceUri.replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + final BookKeeper bookKeeper = registerCloseable(new BookKeeper(baseClientConf)); + admin = new BookKeeperAdmin(bookKeeper, NullStatsLogger.INSTANCE, new ClientConfiguration(baseClientConf)); + LedgerManagerFactory ledgerManagerFactory = registerCloseable(bookKeeper.getLedgerManagerFactory()); + ledgerManager = ledgerManagerFactory.newLedgerManager(); + ledgerUnderreplicationManager = ledgerManagerFactory.newLedgerUnderreplicationManager(); + } + + @AfterMethod(alwaysRun = true) + @Override + public void tearDown() throws Exception { + if (admin != null) { + admin.close(); + } + if (ledgerManager != null) { + ledgerManager.close(); + } + if (ledgerUnderreplicationManager != null) { + ledgerUnderreplicationManager.close(); + } + super.tearDown(); + } + + @Test + public void testPlacementPolicyCheck() throws BKException, InterruptedException { + + // 1. create ledgers + final int numLedgers = 10; + List ids = new LinkedList(); + for (int i = 0; i < numLedgers; i++) { + LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, "passwd".getBytes()); + ids.add(lh.getId()); + for (int j = 0; j < 2; j++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + } + + // 2. init auditorPlacementPolicyCheckTask + final TestStatsProvider statsProvider = new TestStatsProvider(); + final TestStatsProvider.TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + final AuditorStats auditorStats = new AuditorStats(statsLogger); + + AuditorPlacementPolicyCheckTask auditorPlacementPolicyCheckTask = new AuditorPlacementPolicyCheckTask( + baseConf, auditorStats, admin, ledgerManager, + ledgerUnderreplicationManager, null, (flag, throwable) -> flag.set(false)); + + // 3. placementPolicyCheck + auditorPlacementPolicyCheckTask.runTask(); + + // 4. verify + assertEquals("PLACEMENT_POLICY_CHECK_TIME", 1, ((TestStatsProvider.TestOpStatsLogger) + statsLogger.getOpStatsLogger(ReplicationStats.PLACEMENT_POLICY_CHECK_TIME)).getSuccessCount()); + assertEquals("numOfClosedLedgersAuditedInPlacementPolicyCheck", + numLedgers, + auditorPlacementPolicyCheckTask.getNumOfClosedLedgersAuditedInPlacementPolicyCheck().get()); + assertEquals("numOfLedgersFoundNotAdheringInPlacementPolicyCheck", + numLedgers, + auditorPlacementPolicyCheckTask.getNumOfLedgersFoundNotAdheringInPlacementPolicyCheck().get()); + } + +} \ No newline at end of file diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTest.java new file mode 100644 index 0000000000000..5637819a9275b --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTest.java @@ -0,0 +1,860 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicyImpl.REPP_DNS_RESOLVER_CLASS; +import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE; +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertTrue; +import java.net.URI; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.apache.bookkeeper.bookie.BookieImpl; +import org.apache.bookkeeper.client.LedgerMetadataBuilder; +import org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy; +import org.apache.bookkeeper.client.ZoneawareEnsemblePlacementPolicy; +import org.apache.bookkeeper.client.api.DigestType; +import org.apache.bookkeeper.client.api.LedgerMetadata; +import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.discover.BookieServiceInfo; +import org.apache.bookkeeper.discover.RegistrationManager; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.meta.LedgerManagerFactory; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.meta.MetadataBookieDriver; +import org.apache.bookkeeper.meta.MetadataDrivers; +import org.apache.bookkeeper.meta.exceptions.MetadataException; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.net.BookieSocketAddress; +import org.apache.bookkeeper.replication.AuditorPeriodicCheckTest.TestAuditor; +import org.apache.bookkeeper.replication.ReplicationException.CompatibilityException; +import org.apache.bookkeeper.replication.ReplicationException.UnavailableException; +import org.apache.bookkeeper.stats.Gauge; +import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.bookkeeper.test.TestStatsProvider; +import org.apache.bookkeeper.test.TestStatsProvider.TestOpStatsLogger; +import org.apache.bookkeeper.test.TestStatsProvider.TestStatsLogger; +import org.apache.bookkeeper.util.StaticDNSResolver; +import org.apache.commons.lang3.mutable.MutableObject; +import org.apache.zookeeper.KeeperException; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Tests the logic of Auditor's PlacementPolicyCheck. + */ +public class AuditorPlacementPolicyCheckTest extends BookKeeperClusterTestCase { + private MetadataBookieDriver driver; + + public AuditorPlacementPolicyCheckTest() throws Exception { + super(1); + baseConf.setPageLimit(1); // to make it easy to push ledger out of cache + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + StaticDNSResolver.reset(); + + URI uri = URI.create(confByIndex(0).getMetadataServiceUri().replaceAll("zk://", "metadata-store:") + .replaceAll("/ledgers", "")); + driver = MetadataDrivers.getBookieDriver(uri); + ServerConfiguration serverConfiguration = new ServerConfiguration(confByIndex(0)); + serverConfiguration.setMetadataServiceUri( + serverConfiguration.getMetadataServiceUri().replaceAll("zk://", "metadata-store:") + .replaceAll("/ledgers", "")); + driver.initialize(serverConfiguration, NullStatsLogger.INSTANCE); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + if (null != driver) { + driver.close(); + } + super.tearDown(); + } + + @Test + public void testPlacementPolicyCheckWithBookiesFromDifferentRacks() throws Exception { + int numOfBookies = 5; + List bookieAddresses = new ArrayList<>(); + BookieSocketAddress bookieAddress; + RegistrationManager regManager = driver.createRegistrationManager(); + // all the numOfBookies (5) are going to be in different racks + for (int i = 0; i < numOfBookies; i++) { + bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181); + StaticDNSResolver.addNodeToRack(bookieAddress.getHostName(), "/rack" + (i)); + bookieAddresses.add(bookieAddress.toBookieId()); + regManager.registerBookie(bookieAddress.toBookieId(), false, BookieServiceInfo.EMPTY); + } + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + int ensembleSize = 5; + int writeQuorumSize = 4; + int ackQuorumSize = 2; + int minNumRacksPerWriteQuorumConfValue = 4; + Collections.shuffle(bookieAddresses); + + // closed ledger + LedgerMetadata initMeta = LedgerMetadataBuilder.create() + .withId(1L) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(1L, initMeta).get(); + + Collections.shuffle(bookieAddresses); + ensembleSize = 4; + // closed ledger with multiple segments + initMeta = LedgerMetadataBuilder.create() + .withId(2L) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses.subList(0, 4)) + .newEnsembleEntry(20L, bookieAddresses.subList(1, 5)) + .newEnsembleEntry(60L, bookieAddresses.subList(0, 4)) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(2L, initMeta).get(); + + Collections.shuffle(bookieAddresses); + // non-closed ledger + initMeta = LedgerMetadataBuilder.create() + .withId(3L) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses.subList(0, 4)) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(3L, initMeta).get(); + + Collections.shuffle(bookieAddresses); + // non-closed ledger with multiple segments + initMeta = LedgerMetadataBuilder.create() + .withId(4L) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses.subList(0, 4)) + .newEnsembleEntry(20L, bookieAddresses.subList(1, 5)) + .newEnsembleEntry(60L, bookieAddresses.subList(0, 4)) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(4L, initMeta).get(); + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue); + setServerConfigPropertiesForRackPlacement(servConf); + MutableObject auditorRef = new MutableObject(); + try { + TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef); + Gauge ledgersNotAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY); + Gauge ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY); + /* + * since all of the bookies are in different racks, there shouldn't be any ledger not adhering + * to placement policy. + */ + assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY guage value", 0, + ledgersNotAdheringToPlacementPolicyGuage.getSample()); + /* + * since all of the bookies are in different racks, there shouldn't be any ledger softly adhering + * to placement policy. + */ + assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY guage value", 0, + ledgersSoftlyAdheringToPlacementPolicyGuage.getSample()); + } finally { + Auditor auditor = auditorRef.getValue(); + if (auditor != null) { + auditor.close(); + } + regManager.close(); + } + } + + @Test + public void testPlacementPolicyCheckWithLedgersNotAdheringToPlacementPolicy() throws Exception { + int numOfBookies = 5; + int numOfLedgersNotAdheringToPlacementPolicy = 0; + List bookieAddresses = new ArrayList<>(); + RegistrationManager regManager = driver.createRegistrationManager(); + for (int i = 0; i < numOfBookies; i++) { + BookieId bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181).toBookieId(); + bookieAddresses.add(bookieAddress); + regManager.registerBookie(bookieAddress, false, BookieServiceInfo.EMPTY); + } + + // only three racks + StaticDNSResolver.addNodeToRack("98.98.98.0", "/rack1"); + StaticDNSResolver.addNodeToRack("98.98.98.1", "/rack2"); + StaticDNSResolver.addNodeToRack("98.98.98.2", "/rack3"); + StaticDNSResolver.addNodeToRack("98.98.98.3", "/rack1"); + StaticDNSResolver.addNodeToRack("98.98.98.4", "/rack2"); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + int ensembleSize = 5; + int writeQuorumSize = 3; + int ackQuorumSize = 2; + int minNumRacksPerWriteQuorumConfValue = 3; + + /* + * this closed ledger doesn't adhere to placement policy because there are only + * 3 racks, and the ensembleSize is 5. + */ + LedgerMetadata initMeta = LedgerMetadataBuilder.create() + .withId(1L) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(1L, initMeta).get(); + numOfLedgersNotAdheringToPlacementPolicy++; + + /* + * this is non-closed ledger, so it shouldn't count as ledger not + * adhering to placement policy + */ + initMeta = LedgerMetadataBuilder.create() + .withId(2L) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(2L, initMeta).get(); + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue); + setServerConfigPropertiesForRackPlacement(servConf); + MutableObject auditorRef = new MutableObject(); + try { + TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef); + Gauge ledgersNotAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY guage value", + numOfLedgersNotAdheringToPlacementPolicy, ledgersNotAdheringToPlacementPolicyGuage.getSample()); + Gauge ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY guage value", + 0, ledgersSoftlyAdheringToPlacementPolicyGuage.getSample()); + } finally { + Auditor auditor = auditorRef.getValue(); + if (auditor != null) { + auditor.close(); + } + regManager.close(); + } + } + + @Test + public void testPlacementPolicyCheckWithLedgersNotAdheringToPlacementPolicyAndNotMarkToUnderreplication() + throws Exception { + int numOfBookies = 5; + int numOfLedgersNotAdheringToPlacementPolicy = 0; + List bookieAddresses = new ArrayList<>(); + RegistrationManager regManager = driver.createRegistrationManager(); + for (int i = 0; i < numOfBookies; i++) { + BookieId bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181).toBookieId(); + bookieAddresses.add(bookieAddress); + regManager.registerBookie(bookieAddress, false, BookieServiceInfo.EMPTY); + } + + // only three racks + StaticDNSResolver.addNodeToRack("98.98.98.0", "/rack1"); + StaticDNSResolver.addNodeToRack("98.98.98.1", "/rack2"); + StaticDNSResolver.addNodeToRack("98.98.98.2", "/rack3"); + StaticDNSResolver.addNodeToRack("98.98.98.3", "/rack1"); + StaticDNSResolver.addNodeToRack("98.98.98.4", "/rack2"); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + int ensembleSize = 5; + int writeQuorumSize = 3; + int ackQuorumSize = 2; + int minNumRacksPerWriteQuorumConfValue = 3; + + /* + * this closed ledger doesn't adhere to placement policy because there are only + * 3 racks, and the ensembleSize is 5. + */ + LedgerMetadata initMeta = LedgerMetadataBuilder.create() + .withId(1L) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(1L, initMeta).get(); + numOfLedgersNotAdheringToPlacementPolicy++; + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue); + setServerConfigPropertiesForRackPlacement(servConf); + MutableObject auditorRef = new MutableObject(); + try { + TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef); + Gauge ledgersNotAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY guage value", + numOfLedgersNotAdheringToPlacementPolicy, ledgersNotAdheringToPlacementPolicyGuage.getSample()); + Gauge ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY guage value", + 0, ledgersSoftlyAdheringToPlacementPolicyGuage.getSample()); + } finally { + Auditor auditor = auditorRef.getValue(); + if (auditor != null) { + auditor.close(); + } + regManager.close(); + } + LedgerUnderreplicationManager underreplicationManager = mFactory.newLedgerUnderreplicationManager(); + long unnderReplicateLedgerId = underreplicationManager.pollLedgerToRereplicate(); + assertEquals(unnderReplicateLedgerId, -1); + } + + @Test + public void testPlacementPolicyCheckWithLedgersNotAdheringToPlacementPolicyAndMarkToUnderreplication() + throws Exception { + int numOfBookies = 5; + int numOfLedgersNotAdheringToPlacementPolicy = 0; + List bookieAddresses = new ArrayList<>(); + RegistrationManager regManager = driver.createRegistrationManager(); + for (int i = 0; i < numOfBookies; i++) { + BookieId bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181).toBookieId(); + bookieAddresses.add(bookieAddress); + regManager.registerBookie(bookieAddress, false, BookieServiceInfo.EMPTY); + } + + // only three racks + StaticDNSResolver.addNodeToRack("98.98.98.0", "/rack1"); + StaticDNSResolver.addNodeToRack("98.98.98.1", "/rack2"); + StaticDNSResolver.addNodeToRack("98.98.98.2", "/rack3"); + StaticDNSResolver.addNodeToRack("98.98.98.3", "/rack1"); + StaticDNSResolver.addNodeToRack("98.98.98.4", "/rack2"); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + int ensembleSize = 5; + int writeQuorumSize = 3; + int ackQuorumSize = 2; + int minNumRacksPerWriteQuorumConfValue = 3; + + /* + * this closed ledger doesn't adhere to placement policy because there are only + * 3 racks, and the ensembleSize is 5. + */ + LedgerMetadata initMeta = LedgerMetadataBuilder.create() + .withId(1L) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(1L, initMeta).get(); + numOfLedgersNotAdheringToPlacementPolicy++; + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue); + servConf.setRepairedPlacementPolicyNotAdheringBookieEnable(true); + setServerConfigPropertiesForRackPlacement(servConf); + MutableObject auditorRef = new MutableObject(); + try { + TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef); + Gauge ledgersNotAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY guage value", + numOfLedgersNotAdheringToPlacementPolicy, ledgersNotAdheringToPlacementPolicyGuage.getSample()); + Gauge ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY guage value", + 0, ledgersSoftlyAdheringToPlacementPolicyGuage.getSample()); + } finally { + Auditor auditor = auditorRef.getValue(); + if (auditor != null) { + auditor.close(); + } + regManager.close(); + } + LedgerUnderreplicationManager underreplicationManager = mFactory.newLedgerUnderreplicationManager(); + long unnderReplicateLedgerId = underreplicationManager.pollLedgerToRereplicate(); + assertEquals(unnderReplicateLedgerId, 1L); + } + + @Test + public void testPlacementPolicyCheckForURLedgersElapsedRecoveryGracePeriod() throws Exception { + testPlacementPolicyCheckWithURLedgers(true); + } + + @Test + public void testPlacementPolicyCheckForURLedgersNotElapsedRecoveryGracePeriod() throws Exception { + testPlacementPolicyCheckWithURLedgers(false); + } + + public void testPlacementPolicyCheckWithURLedgers(boolean timeElapsed) throws Exception { + int numOfBookies = 4; + /* + * in timeElapsed=true scenario, set some low value, otherwise set some + * highValue. + */ + int underreplicatedLedgerRecoveryGracePeriod = timeElapsed ? 1 : 1000; + int numOfURLedgersElapsedRecoveryGracePeriod = 0; + List bookieAddresses = new ArrayList(); + RegistrationManager regManager = driver.createRegistrationManager(); + for (int i = 0; i < numOfBookies; i++) { + BookieId bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181).toBookieId(); + bookieAddresses.add(bookieAddress); + regManager.registerBookie(bookieAddress, false, BookieServiceInfo.EMPTY); + } + + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + LedgerUnderreplicationManager underreplicationManager = mFactory.newLedgerUnderreplicationManager(); + int ensembleSize = 4; + int writeQuorumSize = 3; + int ackQuorumSize = 2; + + long ledgerId1 = 1L; + LedgerMetadata initMeta = LedgerMetadataBuilder.create() + .withId(ledgerId1) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(ledgerId1, initMeta).get(); + underreplicationManager.markLedgerUnderreplicated(ledgerId1, bookieAddresses.get(0).toString()); + if (timeElapsed) { + numOfURLedgersElapsedRecoveryGracePeriod++; + } + + /* + * this is non-closed ledger, it should also be reported as + * URLedgersElapsedRecoveryGracePeriod + */ + ensembleSize = 3; + long ledgerId2 = 21234561L; + initMeta = LedgerMetadataBuilder.create() + .withId(ledgerId2) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, + Arrays.asList(bookieAddresses.get(0), bookieAddresses.get(1), bookieAddresses.get(2))) + .newEnsembleEntry(100L, + Arrays.asList(bookieAddresses.get(3), bookieAddresses.get(1), bookieAddresses.get(2))) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(ledgerId2, initMeta).get(); + underreplicationManager.markLedgerUnderreplicated(ledgerId2, bookieAddresses.get(0).toString()); + if (timeElapsed) { + numOfURLedgersElapsedRecoveryGracePeriod++; + } + + /* + * this ledger is not marked underreplicated. + */ + long ledgerId3 = 31234561L; + initMeta = LedgerMetadataBuilder.create() + .withId(ledgerId3) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, + Arrays.asList(bookieAddresses.get(1), bookieAddresses.get(2), bookieAddresses.get(3))) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(ledgerId3, initMeta).get(); + + if (timeElapsed) { + /* + * in timeelapsed scenario, by waiting for + * underreplicatedLedgerRecoveryGracePeriod, recovery time must be + * elapsed. + */ + Thread.sleep((underreplicatedLedgerRecoveryGracePeriod + 1) * 1000); + } else { + /* + * in timeElapsed=false scenario, since + * underreplicatedLedgerRecoveryGracePeriod is set to some high + * value, there is no value in waiting. So just wait for some time + * and make sure urledgers are not reported as recoverytime elapsed + * urledgers. + */ + Thread.sleep(5000); + } + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + servConf.setUnderreplicatedLedgerRecoveryGracePeriod(underreplicatedLedgerRecoveryGracePeriod); + setServerConfigPropertiesForRackPlacement(servConf); + MutableObject auditorRef = new MutableObject(); + try { + TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef); + Gauge underreplicatedLedgersElapsedRecoveryGracePeriodGuage = statsLogger + .getGauge(ReplicationStats.NUM_UNDERREPLICATED_LEDGERS_ELAPSED_RECOVERY_GRACE_PERIOD); + assertEquals("NUM_UNDERREPLICATED_LEDGERS_ELAPSED_RECOVERY_GRACE_PERIOD guage value", + numOfURLedgersElapsedRecoveryGracePeriod, + underreplicatedLedgersElapsedRecoveryGracePeriodGuage.getSample()); + } finally { + Auditor auditor = auditorRef.getValue(); + if (auditor != null) { + auditor.close(); + } + regManager.close(); + } + } + + @Test + public void testPlacementPolicyCheckWithLedgersNotAdheringToPolicyWithMultipleSegments() throws Exception { + int numOfBookies = 7; + int numOfLedgersNotAdheringToPlacementPolicy = 0; + List bookieAddresses = new ArrayList<>(); + RegistrationManager regManager = driver.createRegistrationManager(); + for (int i = 0; i < numOfBookies; i++) { + BookieId bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181).toBookieId(); + bookieAddresses.add(bookieAddress); + regManager.registerBookie(bookieAddress, false, BookieServiceInfo.EMPTY); + } + + // only three racks + StaticDNSResolver.addNodeToRack("98.98.98.0", "/rack1"); + StaticDNSResolver.addNodeToRack("98.98.98.1", "/rack2"); + StaticDNSResolver.addNodeToRack("98.98.98.2", "/rack3"); + StaticDNSResolver.addNodeToRack("98.98.98.3", "/rack4"); + StaticDNSResolver.addNodeToRack("98.98.98.4", "/rack1"); + StaticDNSResolver.addNodeToRack("98.98.98.5", "/rack2"); + StaticDNSResolver.addNodeToRack("98.98.98.6", "/rack3"); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + int ensembleSize = 5; + int writeQuorumSize = 5; + int ackQuorumSize = 2; + int minNumRacksPerWriteQuorumConfValue = 4; + + /* + * this closed ledger in each writeQuorumSize (5), there would be + * atleast minNumRacksPerWriteQuorumConfValue (4) racks. So it wont be + * counted as ledgers not adhering to placement policy. + */ + LedgerMetadata initMeta = LedgerMetadataBuilder.create() + .withId(1L) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses.subList(0, 5)) + .newEnsembleEntry(20L, bookieAddresses.subList(1, 6)) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(1L, initMeta).get(); + + /* + * for the second segment bookies are from /rack1, /rack2 and /rack3, + * which is < minNumRacksPerWriteQuorumConfValue (4). So it is not + * adhering to placement policy. + * + * also for the third segment are from /rack1, /rack2 and /rack3, which + * is < minNumRacksPerWriteQuorumConfValue (4). So it is not adhering to + * placement policy. + * + * Though there are multiple segments are not adhering to placement + * policy, it should be counted as single ledger. + */ + initMeta = LedgerMetadataBuilder.create() + .withId(2L) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses.subList(0, 5)) + .newEnsembleEntry(20L, + Arrays.asList(bookieAddresses.get(0), bookieAddresses.get(1), bookieAddresses.get(2), + bookieAddresses.get(4), bookieAddresses.get(5))) + .newEnsembleEntry(40L, + Arrays.asList(bookieAddresses.get(0), bookieAddresses.get(1), bookieAddresses.get(2), + bookieAddresses.get(4), bookieAddresses.get(6))) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(2L, initMeta).get(); + numOfLedgersNotAdheringToPlacementPolicy++; + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue); + setServerConfigPropertiesForRackPlacement(servConf); + MutableObject auditorRef = new MutableObject(); + try { + TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef); + Gauge ledgersNotAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY gauge value", + numOfLedgersNotAdheringToPlacementPolicy, ledgersNotAdheringToPlacementPolicyGuage.getSample()); + Gauge ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY gauge value", + 0, ledgersSoftlyAdheringToPlacementPolicyGuage.getSample()); + } finally { + Auditor auditor = auditorRef.getValue(); + if (auditor != null) { + auditor.close(); + } + regManager.close(); + } + } + + @Test + public void testZoneawarePlacementPolicyCheck() throws Exception { + int numOfBookies = 6; + int numOfLedgersNotAdheringToPlacementPolicy = 0; + int numOfLedgersSoftlyAdheringToPlacementPolicy = 0; + List bookieAddresses = new ArrayList(); + RegistrationManager regManager = driver.createRegistrationManager(); + /* + * 6 bookies - 3 zones and 2 uds + */ + for (int i = 0; i < numOfBookies; i++) { + BookieSocketAddress bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181); + bookieAddresses.add(bookieAddress.toBookieId()); + regManager.registerBookie(bookieAddress.toBookieId(), false, BookieServiceInfo.EMPTY); + String zone = "/zone" + (i % 3); + String upgradeDomain = "/ud" + (i % 2); + String networkLocation = zone + upgradeDomain; + StaticDNSResolver.addNodeToRack(bookieAddress.getHostName(), networkLocation); + } + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + servConf.setDesiredNumZonesPerWriteQuorum(3); + servConf.setMinNumZonesPerWriteQuorum(2); + setServerConfigPropertiesForZonePlacement(servConf); + + /* + * this closed ledger adheres to ZoneAwarePlacementPolicy, since + * ensemble is spread across 3 zones and 2 UDs + */ + LedgerMetadata initMeta = LedgerMetadataBuilder.create() + .withId(1L) + .withEnsembleSize(6) + .withWriteQuorumSize(6) + .withAckQuorumSize(2) + .newEnsembleEntry(0L, bookieAddresses) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(1L, initMeta).get(); + + /* + * this is non-closed ledger, so though ensemble is not adhering to + * placement policy (since ensemble is not multiple of writeQuorum), + * this shouldn't be reported + */ + initMeta = LedgerMetadataBuilder.create() + .withId(2L) + .withEnsembleSize(6) + .withWriteQuorumSize(5) + .withAckQuorumSize(2) + .newEnsembleEntry(0L, bookieAddresses) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(2L, initMeta).get(); + + /* + * this is closed ledger, since ensemble is not multiple of writeQuorum, + * this ledger is not adhering to placement policy. + */ + initMeta = LedgerMetadataBuilder.create() + .withId(3L) + .withEnsembleSize(6) + .withWriteQuorumSize(5) + .withAckQuorumSize(2) + .newEnsembleEntry(0L, bookieAddresses) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(3L, initMeta).get(); + numOfLedgersNotAdheringToPlacementPolicy++; + + /* + * this closed ledger adheres softly to ZoneAwarePlacementPolicy, since + * ensemble/writeQuorum of size 4 has spread across just + * minNumZonesPerWriteQuorum (2). + */ + List newEnsemble = new ArrayList(); + newEnsemble.add(bookieAddresses.get(0)); + newEnsemble.add(bookieAddresses.get(1)); + newEnsemble.add(bookieAddresses.get(3)); + newEnsemble.add(bookieAddresses.get(4)); + initMeta = LedgerMetadataBuilder.create() + .withId(4L) + .withEnsembleSize(4) + .withWriteQuorumSize(4) + .withAckQuorumSize(2) + .newEnsembleEntry(0L, newEnsemble) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(4L, initMeta).get(); + numOfLedgersSoftlyAdheringToPlacementPolicy++; + + MutableObject auditorRef = new MutableObject(); + try { + TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef); + Gauge ledgersNotAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY guage value", + numOfLedgersNotAdheringToPlacementPolicy, ledgersNotAdheringToPlacementPolicyGuage.getSample()); + Gauge ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY guage value", + numOfLedgersSoftlyAdheringToPlacementPolicy, + ledgersSoftlyAdheringToPlacementPolicyGuage.getSample()); + } finally { + Auditor auditor = auditorRef.getValue(); + if (auditor != null) { + auditor.close(); + } + regManager.close(); + } + } + + private void setServerConfigPropertiesForRackPlacement(ServerConfiguration servConf) { + setServerConfigProperties(servConf, RackawareEnsemblePlacementPolicy.class.getName()); + } + + private void setServerConfigPropertiesForZonePlacement(ServerConfiguration servConf) { + setServerConfigProperties(servConf, ZoneawareEnsemblePlacementPolicy.class.getName()); + } + + private void setServerConfigProperties(ServerConfiguration servConf, String ensemblePlacementPolicyClass) { + servConf.setProperty(REPP_DNS_RESOLVER_CLASS, StaticDNSResolver.class.getName()); + servConf.setProperty(ClientConfiguration.ENSEMBLE_PLACEMENT_POLICY, ensemblePlacementPolicyClass); + servConf.setAuditorPeriodicCheckInterval(0); + servConf.setAuditorPeriodicBookieCheckInterval(0); + servConf.setAuditorPeriodicReplicasCheckInterval(0); + servConf.setAuditorPeriodicPlacementPolicyCheckInterval(1000); + } + + private TestStatsLogger startAuditorAndWaitForPlacementPolicyCheck(ServerConfiguration servConf, + MutableObject auditorRef) throws MetadataException, CompatibilityException, KeeperException, + InterruptedException, UnavailableException, UnknownHostException { + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager(); + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + TestOpStatsLogger placementPolicyCheckStatsLogger = (TestOpStatsLogger) statsLogger + .getOpStatsLogger(ReplicationStats.PLACEMENT_POLICY_CHECK_TIME); + + final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, + statsLogger, null); + auditorRef.setValue(auditor); + CountDownLatch latch = auditor.getLatch(); + assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 0, placementPolicyCheckStatsLogger.getSuccessCount()); + urm.setPlacementPolicyCheckCTime(-1); + auditor.start(); + /* + * since placementPolicyCheckCTime is set to -1, placementPolicyCheck should be + * scheduled to run with no initialdelay + */ + assertTrue("placementPolicyCheck should have executed", latch.await(20, TimeUnit.SECONDS)); + for (int i = 0; i < 20; i++) { + Thread.sleep(100); + if (placementPolicyCheckStatsLogger.getSuccessCount() >= 1) { + break; + } + } + assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 1, placementPolicyCheckStatsLogger.getSuccessCount()); + return statsLogger; + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTaskTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTaskTest.java new file mode 100644 index 0000000000000..b48498639e7e2 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTaskTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE; +import static org.testng.AssertJUnit.assertEquals; +import java.util.LinkedList; +import java.util.List; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.BookKeeperAdmin; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.meta.LedgerManagerFactory; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.bookkeeper.test.TestStatsProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Unit test {@link AuditorReplicasCheckTask}. + */ +public class AuditorReplicasCheckTaskTest extends BookKeeperClusterTestCase { + private static final Logger LOG = LoggerFactory + .getLogger(AuditorReplicasCheckTaskTest.class); + + private BookKeeperAdmin admin; + private LedgerManager ledgerManager; + private LedgerUnderreplicationManager ledgerUnderreplicationManager; + + public AuditorReplicasCheckTaskTest() throws Exception { + super(3); + baseConf.setPageLimit(1); + baseConf.setAutoRecoveryDaemonEnabled(false); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + baseClientConf.setMetadataServiceUri( + metadataServiceUri.replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + final BookKeeper bookKeeper = registerCloseable(new BookKeeper(baseClientConf)); + admin = new BookKeeperAdmin(bookKeeper, NullStatsLogger.INSTANCE, new ClientConfiguration(baseClientConf)); + LedgerManagerFactory ledgerManagerFactory = registerCloseable(bookKeeper.getLedgerManagerFactory()); + ledgerManager = ledgerManagerFactory.newLedgerManager(); + ledgerUnderreplicationManager = ledgerManagerFactory.newLedgerUnderreplicationManager(); + } + + @AfterMethod(alwaysRun = true) + @Override + public void tearDown() throws Exception { + if (ledgerManager != null) { + ledgerManager.close(); + } + if (ledgerUnderreplicationManager != null) { + ledgerUnderreplicationManager.close(); + } + if (admin != null) { + admin.close(); + } + super.tearDown(); + } + + @Test + public void testReplicasCheck() throws BKException, InterruptedException { + + // 1. create ledgers + final int numLedgers = 10; + List ids = new LinkedList(); + for (int i = 0; i < numLedgers; i++) { + LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, "passwd".getBytes()); + ids.add(lh.getId()); + for (int j = 0; j < 2; j++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + } + + // 2. init auditorReplicasCheckTask + final TestStatsProvider statsProvider = new TestStatsProvider(); + final TestStatsProvider.TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + final AuditorStats auditorStats = new AuditorStats(statsLogger); + AuditorReplicasCheckTask auditorReplicasCheckTask = new AuditorReplicasCheckTask( + baseConf, auditorStats, admin, ledgerManager, + ledgerUnderreplicationManager, null, (flag, throwable) -> flag.set(false)); + + // 3. replicasCheck + auditorReplicasCheckTask.runTask(); + + // 4. verify + assertEquals("REPLICAS_CHECK_TIME", 1, ((TestStatsProvider.TestOpStatsLogger) + statsLogger.getOpStatsLogger(ReplicationStats.REPLICAS_CHECK_TIME)).getSuccessCount()); + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTest.java new file mode 100644 index 0000000000000..2e9dbc158597d --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTest.java @@ -0,0 +1,936 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE; +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertTrue; +import java.net.URI; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import org.apache.bookkeeper.bookie.BookieException; +import org.apache.bookkeeper.bookie.BookieImpl; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.BookKeeperAdmin; +import org.apache.bookkeeper.client.LedgerMetadataBuilder; +import org.apache.bookkeeper.client.api.DigestType; +import org.apache.bookkeeper.client.api.LedgerMetadata; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.discover.BookieServiceInfo; +import org.apache.bookkeeper.discover.RegistrationManager; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.meta.LedgerManagerFactory; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.meta.MetadataBookieDriver; +import org.apache.bookkeeper.meta.MetadataDrivers; +import org.apache.bookkeeper.meta.exceptions.MetadataException; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.net.BookieSocketAddress; +import org.apache.bookkeeper.replication.AuditorPeriodicCheckTest.TestAuditor; +import org.apache.bookkeeper.replication.ReplicationException.CompatibilityException; +import org.apache.bookkeeper.replication.ReplicationException.UnavailableException; +import org.apache.bookkeeper.stats.Gauge; +import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.bookkeeper.stats.StatsLogger; +import org.apache.bookkeeper.test.TestStatsProvider; +import org.apache.bookkeeper.test.TestStatsProvider.TestOpStatsLogger; +import org.apache.bookkeeper.test.TestStatsProvider.TestStatsLogger; +import org.apache.bookkeeper.util.AvailabilityOfEntriesOfLedger; +import org.apache.bookkeeper.util.StaticDNSResolver; +import org.apache.commons.collections4.map.MultiKeyMap; +import org.apache.commons.lang3.mutable.MutableObject; +import org.apache.zookeeper.KeeperException; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Tests the logic of Auditor's ReplicasCheck. + */ +public class AuditorReplicasCheckTest extends BookKeeperClusterTestCase { + private MetadataBookieDriver driver; + private RegistrationManager regManager; + + public AuditorReplicasCheckTest() throws Exception { + super(1); + baseConf.setPageLimit(1); // to make it easy to push ledger out of cache + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + StaticDNSResolver.reset(); + + URI uri = URI.create(confByIndex(0).getMetadataServiceUri().replaceAll("zk://", "metadata-store:") + .replaceAll("/ledgers", "")); + driver = MetadataDrivers.getBookieDriver(uri); + ServerConfiguration serverConfiguration = new ServerConfiguration(confByIndex(0)); + serverConfiguration.setMetadataServiceUri( + serverConfiguration.getMetadataServiceUri().replaceAll("zk://", "metadata-store:") + .replaceAll("/ledgers", "")); + driver.initialize(serverConfiguration, NullStatsLogger.INSTANCE); + regManager = driver.createRegistrationManager(); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + if (null != regManager) { + regManager.close(); + } + if (null != driver) { + driver.close(); + } + super.tearDown(); + } + + private class TestBookKeeperAdmin extends BookKeeperAdmin { + + private final MultiKeyMap returnAvailabilityOfEntriesOfLedger; + private final MultiKeyMap errorReturnValueForGetAvailabilityOfEntriesOfLedger; + + public TestBookKeeperAdmin(BookKeeper bkc, StatsLogger statsLogger, + MultiKeyMap returnAvailabilityOfEntriesOfLedger, + MultiKeyMap errorReturnValueForGetAvailabilityOfEntriesOfLedger) { + super(bkc, statsLogger, baseClientConf); + this.returnAvailabilityOfEntriesOfLedger = returnAvailabilityOfEntriesOfLedger; + this.errorReturnValueForGetAvailabilityOfEntriesOfLedger = + errorReturnValueForGetAvailabilityOfEntriesOfLedger; + } + + @Override + public CompletableFuture asyncGetListOfEntriesOfLedger( + BookieId address, long ledgerId) { + CompletableFuture futureResult = + new CompletableFuture(); + Integer errorReturnValue = errorReturnValueForGetAvailabilityOfEntriesOfLedger.get(address.toString(), + Long.toString(ledgerId)); + if (errorReturnValue != null) { + futureResult.completeExceptionally(BKException.create(errorReturnValue).fillInStackTrace()); + } else { + AvailabilityOfEntriesOfLedger availabilityOfEntriesOfLedger = returnAvailabilityOfEntriesOfLedger + .get(address.toString(), Long.toString(ledgerId)); + futureResult.complete(availabilityOfEntriesOfLedger); + } + return futureResult; + } + } + + private TestStatsLogger startAuditorAndWaitForReplicasCheck(ServerConfiguration servConf, + MutableObject auditorRef, + MultiKeyMap expectedReturnAvailabilityOfEntriesOfLedger, + MultiKeyMap errorReturnValueForGetAvailabilityOfEntriesOfLedger) + throws MetadataException, CompatibilityException, KeeperException, InterruptedException, + UnavailableException, UnknownHostException { + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager(); + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + TestOpStatsLogger replicasCheckStatsLogger = (TestOpStatsLogger) statsLogger + .getOpStatsLogger(ReplicationStats.REPLICAS_CHECK_TIME); + + final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, bkc, true, + new TestBookKeeperAdmin(bkc, statsLogger, expectedReturnAvailabilityOfEntriesOfLedger, + errorReturnValueForGetAvailabilityOfEntriesOfLedger), + true, statsLogger, null); + auditorRef.setValue(auditor); + CountDownLatch latch = auditor.getLatch(); + assertEquals("REPLICAS_CHECK_TIME SuccessCount", 0, replicasCheckStatsLogger.getSuccessCount()); + urm.setReplicasCheckCTime(-1); + auditor.start(); + /* + * since replicasCheckCTime is set to -1, replicasCheck should be + * scheduled to run with no initialdelay + */ + assertTrue("replicasCheck should have executed", latch.await(20, TimeUnit.SECONDS)); + for (int i = 0; i < 200; i++) { + Thread.sleep(100); + if (replicasCheckStatsLogger.getSuccessCount() >= 1) { + break; + } + } + assertEquals("REPLICAS_CHECK_TIME SuccessCount", 1, replicasCheckStatsLogger.getSuccessCount()); + return statsLogger; + } + + private void setServerConfigProperties(ServerConfiguration servConf) { + servConf.setAuditorPeriodicCheckInterval(0); + servConf.setAuditorPeriodicBookieCheckInterval(0); + servConf.setAuditorPeriodicPlacementPolicyCheckInterval(0); + servConf.setAuditorPeriodicReplicasCheckInterval(1000); + } + + List addAndRegisterBookies(int numOfBookies) + throws BookieException { + BookieId bookieAddress; + List bookieAddresses = new ArrayList(); + for (int i = 0; i < numOfBookies; i++) { + bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181).toBookieId(); + bookieAddresses.add(bookieAddress); + regManager.registerBookie(bookieAddress, false, BookieServiceInfo.EMPTY); + } + return bookieAddresses; + } + + private void createClosedLedgerMetadata(LedgerManager lm, long ledgerId, int ensembleSize, int writeQuorumSize, + int ackQuorumSize, Map> segmentEnsembles, long lastEntryId, int length, + DigestType digestType, byte[] password) throws InterruptedException, ExecutionException { + LedgerMetadataBuilder ledgerMetadataBuilder = LedgerMetadataBuilder.create(); + ledgerMetadataBuilder.withId(ledgerId).withEnsembleSize(ensembleSize).withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize).withClosedState().withLastEntryId(lastEntryId).withLength(length) + .withDigestType(digestType).withPassword(password); + for (Map.Entry> mapEntry : segmentEnsembles.entrySet()) { + ledgerMetadataBuilder.newEnsembleEntry(mapEntry.getKey(), mapEntry.getValue()); + } + LedgerMetadata initMeta = ledgerMetadataBuilder.build(); + lm.createLedgerMetadata(ledgerId, initMeta).get(); + } + + private void createNonClosedLedgerMetadata(LedgerManager lm, long ledgerId, int ensembleSize, int writeQuorumSize, + int ackQuorumSize, Map> segmentEnsembles, DigestType digestType, + byte[] password) throws InterruptedException, ExecutionException { + LedgerMetadataBuilder ledgerMetadataBuilder = LedgerMetadataBuilder.create(); + ledgerMetadataBuilder.withId(ledgerId).withEnsembleSize(ensembleSize).withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize).withDigestType(digestType).withPassword(password); + for (Map.Entry> mapEntry : segmentEnsembles.entrySet()) { + ledgerMetadataBuilder.newEnsembleEntry(mapEntry.getKey(), mapEntry.getValue()); + } + LedgerMetadata initMeta = ledgerMetadataBuilder.build(); + lm.createLedgerMetadata(ledgerId, initMeta).get(); + } + + private void runTestScenario(MultiKeyMap returnAvailabilityOfEntriesOfLedger, + MultiKeyMap errorReturnValueForGetAvailabilityOfEntriesOfLedger, + int expectedNumLedgersFoundHavingNoReplicaOfAnEntry, + int expectedNumLedgersHavingLessThanAQReplicasOfAnEntry, + int expectedNumLedgersHavingLessThanWQReplicasOfAnEntry) throws Exception { + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + setServerConfigProperties(servConf); + MutableObject auditorRef = new MutableObject(); + try { + TestStatsLogger statsLogger = startAuditorAndWaitForReplicasCheck(servConf, auditorRef, + returnAvailabilityOfEntriesOfLedger, errorReturnValueForGetAvailabilityOfEntriesOfLedger); + checkReplicasCheckStats(statsLogger, expectedNumLedgersFoundHavingNoReplicaOfAnEntry, + expectedNumLedgersHavingLessThanAQReplicasOfAnEntry, + expectedNumLedgersHavingLessThanWQReplicasOfAnEntry); + } finally { + Auditor auditor = auditorRef.getValue(); + if (auditor != null) { + auditor.close(); + } + } + } + + private void checkReplicasCheckStats(TestStatsLogger statsLogger, + int expectedNumLedgersFoundHavingNoReplicaOfAnEntry, + int expectedNumLedgersHavingLessThanAQReplicasOfAnEntry, + int expectedNumLedgersHavingLessThanWQReplicasOfAnEntry) { + Gauge numLedgersFoundHavingNoReplicaOfAnEntryGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_HAVING_NO_REPLICA_OF_AN_ENTRY); + Gauge numLedgersHavingLessThanAQReplicasOfAnEntryGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_HAVING_LESS_THAN_AQ_REPLICAS_OF_AN_ENTRY); + Gauge numLedgersHavingLessThanWQReplicasOfAnEntryGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_HAVING_LESS_THAN_WQ_REPLICAS_OF_AN_ENTRY); + + assertEquals("NUM_LEDGERS_HAVING_NO_REPLICA_OF_AN_ENTRY guage value", + expectedNumLedgersFoundHavingNoReplicaOfAnEntry, + numLedgersFoundHavingNoReplicaOfAnEntryGuage.getSample()); + assertEquals("NUM_LEDGERS_HAVING_LESS_THAN_AQ_REPLICAS_OF_AN_ENTRY guage value", + expectedNumLedgersHavingLessThanAQReplicasOfAnEntry, + numLedgersHavingLessThanAQReplicasOfAnEntryGuage.getSample()); + assertEquals("NUM_LEDGERS_HAVING_LESS_THAN_WQ_REPLICAS_OF_AN_ENTRY guage value", + expectedNumLedgersHavingLessThanWQReplicasOfAnEntry, + numLedgersHavingLessThanWQReplicasOfAnEntryGuage.getSample()); + } + + /* + * For all the ledgers and for all the bookies, + * asyncGetListOfEntriesOfLedger would return + * BookieHandleNotAvailableException, so these ledgers wouldn't be counted + * against expectedNumLedgersFoundHavingNoReplicaOfAnEntry / + * LessThanAQReplicasOfAnEntry / LessThanWQReplicasOfAnEntry. + */ + @Test + public void testReplicasCheckForBookieHandleNotAvailable() throws Exception { + int numOfBookies = 5; + MultiKeyMap returnAvailabilityOfEntriesOfLedger = + new MultiKeyMap(); + MultiKeyMap errorReturnValueForGetAvailabilityOfEntriesOfLedger = + new MultiKeyMap(); + List bookieAddresses = addAndRegisterBookies(numOfBookies); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + int ensembleSize = 5; + int writeQuorumSize = 4; + int ackQuorumSize = 2; + long lastEntryId = 100; + int length = 10000; + DigestType digestType = DigestType.DUMMY; + byte[] password = new byte[0]; + Collections.shuffle(bookieAddresses); + + /* + * closed ledger + * + * for this ledger, for all the bookies we are setting + * errorReturnValueForGetAvailabilityOfEntriesOfLedger to + * BookieHandleNotAvailableException so asyncGetListOfEntriesOfLedger will + * return BookieHandleNotAvailableException. + */ + Map> segmentEnsembles = new LinkedHashMap>(); + segmentEnsembles.put(0L, bookieAddresses); + long ledgerId = 1L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + for (BookieId bookieSocketAddress : bookieAddresses) { + errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(), + Long.toString(ledgerId), BKException.Code.BookieHandleNotAvailableException); + } + + ensembleSize = 4; + /* + * closed ledger with multiple segments + * + * for this ledger, for all the bookies we are setting + * errorReturnValueForGetAvailabilityOfEntriesOfLedger to + * BookieHandleNotAvailableException so asyncGetListOfEntriesOfLedger will + * return BookieHandleNotAvailableException. + */ + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put(20L, bookieAddresses.subList(1, 5)); + segmentEnsembles.put(60L, bookieAddresses.subList(0, 4)); + ledgerId = 2L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + for (BookieId bookieSocketAddress : bookieAddresses) { + errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(), + Long.toString(ledgerId), BKException.Code.BookieHandleNotAvailableException); + } + + /* + * non-closed ledger + */ + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + ledgerId = 3L; + createNonClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + digestType, password); + for (BookieId bookieSocketAddress : bookieAddresses) { + errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(), + Long.toString(ledgerId), BKException.Code.BookieHandleNotAvailableException); + } + + /* + * non-closed ledger with multiple segments + * + */ + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put(20L, bookieAddresses.subList(1, 5)); + segmentEnsembles.put(60L, bookieAddresses.subList(0, 4)); + ledgerId = 4L; + createNonClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + digestType, password); + for (BookieId bookieSocketAddress : bookieAddresses) { + errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(), + Long.toString(ledgerId), BKException.Code.BookieHandleNotAvailableException); + } + + runTestScenario(returnAvailabilityOfEntriesOfLedger, errorReturnValueForGetAvailabilityOfEntriesOfLedger, 0, 0, + 0); + } + + /* + * In this testscenario all the ledgers have a missing entry. So all closed + * ledgers should be counted towards + * numLedgersFoundHavingNoReplicaOfAnEntry. + */ + @Test + public void testReplicasCheckForLedgersFoundHavingNoReplica() throws Exception { + int numOfBookies = 5; + MultiKeyMap returnAvailabilityOfEntriesOfLedger = + new MultiKeyMap(); + MultiKeyMap errorReturnValueForGetAvailabilityOfEntriesOfLedger = + new MultiKeyMap(); + List bookieAddresses = addAndRegisterBookies(numOfBookies); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + int ensembleSize = 5; + int writeQuorumSize = 4; + int ackQuorumSize = 2; + long lastEntryId = 100; + int length = 10000; + DigestType digestType = DigestType.DUMMY; + byte[] password = new byte[0]; + Collections.shuffle(bookieAddresses); + + int numLedgersFoundHavingNoReplicaOfAnEntry = 0; + + /* + * closed ledger + * + * for this ledger we are setting returnAvailabilityOfEntriesOfLedger to + * Empty one for all of the bookies, so this ledger would be counted in + * ledgersFoundHavingNoReplicaOfAnEntry . + */ + Map> segmentEnsembles = new LinkedHashMap>(); + segmentEnsembles.put(0L, bookieAddresses); + long ledgerId = 1L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + for (BookieId bookieSocketAddress : bookieAddresses) { + returnAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(), Long.toString(ledgerId), + AvailabilityOfEntriesOfLedger.EMPTY_AVAILABILITYOFENTRIESOFLEDGER); + } + numLedgersFoundHavingNoReplicaOfAnEntry++; + + ensembleSize = 4; + /* + * closed ledger with multiple segments + * + * for this ledger we are setting + * errorReturnValueForGetAvailabilityOfEntriesOfLedger to + * NoSuchLedgerExistsException. This is equivalent to + * EMPTY_AVAILABILITYOFENTRIESOFLEDGER. So this ledger would be counted + * in ledgersFoundHavingNoReplicaOfAnEntry + */ + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put(20L, bookieAddresses.subList(1, 5)); + segmentEnsembles.put(60L, bookieAddresses.subList(0, 4)); + ledgerId = 2L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + for (BookieId bookieSocketAddress : bookieAddresses) { + errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(), + Long.toString(ledgerId), BKException.Code.NoSuchLedgerExistsException); + } + numLedgersFoundHavingNoReplicaOfAnEntry++; + + /* + * non-closed ledger + * + * since this is non-closed ledger, it should not be counted in + * ledgersFoundHavingNoReplicaOfAnEntry + */ + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + ledgerId = 3L; + createNonClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + digestType, password); + for (BookieId bookieSocketAddress : bookieAddresses) { + returnAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(), Long.toString(ledgerId), + AvailabilityOfEntriesOfLedger.EMPTY_AVAILABILITYOFENTRIESOFLEDGER); + } + + ensembleSize = 3; + writeQuorumSize = 3; + ackQuorumSize = 2; + lastEntryId = 1; + length = 1000; + /* + * closed ledger + * + * for this ledger we are setting returnAvailabilityOfEntriesOfLedger to + * just {0l} for all of the bookies and entry 1l is missing for all of + * the bookies, so this ledger would be counted in + * ledgersFoundHavingNoReplicaOfAnEntry + */ + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 3)); + ledgerId = 4L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + for (BookieId bookieSocketAddress : bookieAddresses) { + returnAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0L })); + } + numLedgersFoundHavingNoReplicaOfAnEntry++; + + /* + * For this closed ledger, entry 1 is missing. So it should be counted + * towards numLedgersFoundHavingNoReplicaOfAnEntry. + */ + ensembleSize = 4; + writeQuorumSize = 3; + ackQuorumSize = 2; + lastEntryId = 3; + length = 10000; + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + ledgerId = 5L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 2, 3 })); + numLedgersFoundHavingNoReplicaOfAnEntry++; + + runTestScenario(returnAvailabilityOfEntriesOfLedger, errorReturnValueForGetAvailabilityOfEntriesOfLedger, + numLedgersFoundHavingNoReplicaOfAnEntry, 0, 0); + } + + /* + * In this testscenario all the ledgers have an entry with less than AQ + * number of copies. So all closed ledgers should be counted towards + * numLedgersFoundHavingLessThanAQReplicasOfAnEntry. + */ + @Test + public void testReplicasCheckForLedgersFoundHavingLessThanAQReplicasOfAnEntry() throws Exception { + int numOfBookies = 5; + MultiKeyMap returnAvailabilityOfEntriesOfLedger = + new MultiKeyMap(); + MultiKeyMap errorReturnValueForGetAvailabilityOfEntriesOfLedger = + new MultiKeyMap(); + List bookieAddresses = addAndRegisterBookies(numOfBookies); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + DigestType digestType = DigestType.DUMMY; + byte[] password = new byte[0]; + Collections.shuffle(bookieAddresses); + + int numLedgersFoundHavingLessThanAQReplicasOfAnEntry = 0; + + /* + * closed ledger + * + * for this ledger there is only one copy of entry 2, so this ledger + * would be counted towards + * ledgersFoundHavingLessThanAQReplicasOfAnEntry. + */ + Map> segmentEnsembles = new LinkedHashMap>(); + int ensembleSize = 4; + int writeQuorumSize = 3; + int ackQuorumSize = 2; + long lastEntryId = 3; + int length = 10000; + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + long ledgerId = 1L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 1, 2, 3 })); + numLedgersFoundHavingLessThanAQReplicasOfAnEntry++; + + /* + * closed ledger with multiple segments. + * + * for this ledger there is only one copy of entry 2, so this ledger + * would be counted towards + * ledgersFoundHavingLessThanAQReplicasOfAnEntry. + * + */ + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put(2L, bookieAddresses.subList(1, 5)); + ledgerId = 2L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] {})); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 1 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(4).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 3 })); + numLedgersFoundHavingLessThanAQReplicasOfAnEntry++; + + /* + * closed ledger with multiple segments + * + * for this ledger entry 2 is overrreplicated, but it has only one copy + * in the set of bookies it is supposed to be. So it should be counted + * towards ledgersFoundHavingLessThanAQReplicasOfAnEntry. + */ + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put(2L, bookieAddresses.subList(1, 5)); + ledgerId = 3L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 2 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 1 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(4).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 3 })); + numLedgersFoundHavingLessThanAQReplicasOfAnEntry++; + + /* + * non-closed ledger + * + * since this is non-closed ledger, it should not be counted towards + * ledgersFoundHavingLessThanAQReplicasOfAnEntry + */ + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put(2L, bookieAddresses.subList(1, 5)); + ledgerId = 4L; + createNonClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] {})); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 1 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(4).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 3 })); + + /* + * this is closed ledger. + * + * For third bookie, asyncGetListOfEntriesOfLedger will fail with + * BookieHandleNotAvailableException, so this should not be counted + * against missing copies of an entry. Other than that, for both entries + * 0 and 1, two copies are missing. Hence this should be counted towards + * numLedgersFoundHavingLessThanAQReplicasOfAnEntry. + */ + ensembleSize = 3; + writeQuorumSize = 3; + ackQuorumSize = 2; + lastEntryId = 1; + length = 1000; + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 3)); + ledgerId = 5L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + AvailabilityOfEntriesOfLedger.EMPTY_AVAILABILITYOFENTRIESOFLEDGER); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + AvailabilityOfEntriesOfLedger.EMPTY_AVAILABILITYOFENTRIESOFLEDGER); + errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), + Long.toString(ledgerId), BKException.Code.BookieHandleNotAvailableException); + numLedgersFoundHavingLessThanAQReplicasOfAnEntry++; + + runTestScenario(returnAvailabilityOfEntriesOfLedger, errorReturnValueForGetAvailabilityOfEntriesOfLedger, 0, + numLedgersFoundHavingLessThanAQReplicasOfAnEntry, 0); + } + + /* + * In this testscenario all the ledgers have an entry with less than WQ + * number of copies but greater than AQ. So all closed ledgers should be + * counted towards numLedgersFoundHavingLessThanWQReplicasOfAnEntry. + */ + @Test + public void testReplicasCheckForLedgersFoundHavingLessThanWQReplicasOfAnEntry() throws Exception { + int numOfBookies = 5; + MultiKeyMap returnAvailabilityOfEntriesOfLedger = + new MultiKeyMap(); + MultiKeyMap errorReturnValueForGetAvailabilityOfEntriesOfLedger = + new MultiKeyMap(); + List bookieAddresses = addAndRegisterBookies(numOfBookies); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + DigestType digestType = DigestType.DUMMY; + byte[] password = new byte[0]; + Collections.shuffle(bookieAddresses); + + int numLedgersFoundHavingLessThanWQReplicasOfAnEntry = 0; + + /* + * closed ledger + * + * for this ledger a copy of entry 3, so this ledger would be counted + * towards ledgersFoundHavingLessThanWQReplicasOfAnEntry. + */ + Map> segmentEnsembles = new LinkedHashMap>(); + int ensembleSize = 4; + int writeQuorumSize = 3; + int ackQuorumSize = 2; + long lastEntryId = 3; + int length = 10000; + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + long ledgerId = 1L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 2 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 1, 2, 3 })); + numLedgersFoundHavingLessThanWQReplicasOfAnEntry++; + + /* + * closed ledger with multiple segments + * + * for this ledger a copy of entry 0 and entry 2 are missing, so this + * ledger would be counted towards + * ledgersFoundHavingLessThanWQReplicasOfAnEntry. + */ + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put(2L, bookieAddresses.subList(1, 5)); + ledgerId = 2L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] {})); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 1 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(4).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 2, 3 })); + numLedgersFoundHavingLessThanWQReplicasOfAnEntry++; + + /* + * non-closed ledger with multiple segments + * + * since this is non-closed ledger, it should not be counted towards + * ledgersFoundHavingLessThanWQReplicasOfAnEntry + */ + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put(2L, bookieAddresses.subList(1, 5)); + ledgerId = 3L; + createNonClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + digestType, password); + errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), + Long.toString(ledgerId), BKException.Code.NoSuchLedgerExistsException); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 1 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(4).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 2, 3 })); + + /* + * closed ledger. + * + * for this ledger entry 0 is overrreplicated, but a copy is missing in + * the set of bookies it is supposed to be. So it should be counted + * towards ledgersFoundHavingLessThanWQReplicasOfAnEntry. + */ + ensembleSize = 4; + writeQuorumSize = 3; + ackQuorumSize = 2; + lastEntryId = 1; + length = 1000; + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + ledgerId = 4L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 1, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0 })); + numLedgersFoundHavingLessThanWQReplicasOfAnEntry++; + + /* + * this is closed ledger. + * + * For third bookie, asyncGetListOfEntriesOfLedger will fail with + * BookieHandleNotAvailableException, so this should not be counted + * against missing copies of an entry. Other than that, for both entries + * 0 and 1, a copy is missing. Hence this should be counted towards + * numLedgersFoundHavingLessThanWQReplicasOfAnEntry. + */ + ensembleSize = 3; + writeQuorumSize = 3; + ackQuorumSize = 2; + lastEntryId = 1; + length = 1000; + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 3)); + ledgerId = 5L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + AvailabilityOfEntriesOfLedger.EMPTY_AVAILABILITYOFENTRIESOFLEDGER); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1 })); + errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), + Long.toString(ledgerId), BKException.Code.BookieHandleNotAvailableException); + numLedgersFoundHavingLessThanWQReplicasOfAnEntry++; + + runTestScenario(returnAvailabilityOfEntriesOfLedger, errorReturnValueForGetAvailabilityOfEntriesOfLedger, 0, 0, + numLedgersFoundHavingLessThanWQReplicasOfAnEntry); + } + + /* + * In this testscenario all the ledgers have empty segments. + */ + @Test + public void testReplicasCheckForLedgersWithEmptySegments() throws Exception { + int numOfBookies = 5; + MultiKeyMap returnAvailabilityOfEntriesOfLedger = + new MultiKeyMap(); + MultiKeyMap errorReturnValueForGetAvailabilityOfEntriesOfLedger = + new MultiKeyMap(); + List bookieAddresses = addAndRegisterBookies(numOfBookies); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + DigestType digestType = DigestType.DUMMY; + byte[] password = new byte[0]; + Collections.shuffle(bookieAddresses); + + int numLedgersFoundHavingNoReplicaOfAnEntry = 0; + int numLedgersFoundHavingLessThanAQReplicasOfAnEntry = 0; + int numLedgersFoundHavingLessThanWQReplicasOfAnEntry = 0; + + /* + * closed ledger. + * + * This closed Ledger has no entry. So it should not be counted towards + * numLedgersFoundHavingNoReplicaOfAnEntry/LessThanAQReplicasOfAnEntry + * /WQReplicasOfAnEntry. + */ + Map> segmentEnsembles = new LinkedHashMap>(); + int ensembleSize = 4; + int writeQuorumSize = 3; + int ackQuorumSize = 2; + long lastEntryId = -1L; + int length = 0; + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + long ledgerId = 1L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + + /* + * closed ledger with multiple segments. + * + * This ledger has empty last segment, but all the entries have + * writeQuorumSize number of copies, So it should not be counted towards + * numLedgersFoundHavingNoReplicaOfAnEntry/LessThanAQReplicasOfAnEntry/ + * WQReplicasOfAnEntry. + */ + lastEntryId = 2; + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put((lastEntryId + 1), bookieAddresses.subList(1, 5)); + ledgerId = 2L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 2 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 1, 2 })); + + /* + * Closed ledger with multiple segments. + * + * Segment0, Segment1, Segment3, Segment5 and Segment6 are empty. + * Entries from entryid 3 are missing. So it should be counted towards + * numLedgersFoundHavingNoReplicaOfAnEntry. + */ + lastEntryId = 5; + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(1, 5)); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put(4L, bookieAddresses.subList(1, 5)); + segmentEnsembles.put(4L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put((lastEntryId + 1), bookieAddresses.subList(1, 5)); + segmentEnsembles.put((lastEntryId + 1), bookieAddresses.subList(0, 4)); + ledgerId = 3L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 2 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 1, 2 })); + numLedgersFoundHavingNoReplicaOfAnEntry++; + + /* + * non-closed ledger with multiple segments + * + * since this is non-closed ledger, it should not be counted towards + * ledgersFoundHavingLessThanWQReplicasOfAnEntry + */ + lastEntryId = 2; + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put(0L, bookieAddresses.subList(1, 5)); + segmentEnsembles.put((lastEntryId + 1), bookieAddresses.subList(1, 5)); + ledgerId = 4L; + createNonClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 2 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 1, 2 })); + + runTestScenario(returnAvailabilityOfEntriesOfLedger, errorReturnValueForGetAvailabilityOfEntriesOfLedger, + numLedgersFoundHavingNoReplicaOfAnEntry, numLedgersFoundHavingLessThanAQReplicasOfAnEntry, + numLedgersFoundHavingLessThanWQReplicasOfAnEntry); + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorRollingRestartTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorRollingRestartTest.java new file mode 100644 index 0000000000000..3e5081ed0ef9d --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorRollingRestartTest.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.apache.bookkeeper.meta.MetadataDrivers.runFunctionWithLedgerManagerFactory; +import static org.testng.AssertJUnit.assertEquals; +import com.google.common.util.concurrent.UncheckedExecutionException; +import lombok.Cleanup; +import org.apache.bookkeeper.client.BookKeeper.DigestType; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.meta.LedgerAuditorManager; +import org.apache.bookkeeper.meta.LedgerManagerFactory; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.test.TestCallbacks; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Test auditor behaviours during a rolling restart. + */ +public class AuditorRollingRestartTest extends BookKeeperClusterTestCase { + + public AuditorRollingRestartTest() throws Exception { + super(3, 600); + // run the daemon within the bookie + setAutoRecoveryEnabled(true); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + super.tearDown(); + } + + @Override + protected void startBKCluster(String metadataServiceUri) throws Exception { + super.startBKCluster(metadataServiceUri.replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + } + + /** + * Test no auditing during restart if disabled. + */ + @Test + public void testAuditingDuringRollingRestart() throws Exception { + confByIndex(0).setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + runFunctionWithLedgerManagerFactory( + confByIndex(0), + mFactory -> { + try { + testAuditingDuringRollingRestart(mFactory); + } catch (Exception e) { + throw new UncheckedExecutionException(e.getMessage(), e); + } + return null; + } + ); + } + + private void testAuditingDuringRollingRestart(LedgerManagerFactory mFactory) throws Exception { + final LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager(); + + LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + for (int i = 0; i < 10; i++) { + lh.asyncAddEntry("foobar".getBytes(), new TestCallbacks.AddCallbackFuture(i), null); + } + lh.addEntry("foobar".getBytes()); + lh.close(); + + assertEquals("shouldn't be anything under replicated", + underReplicationManager.pollLedgerToRereplicate(), -1); + underReplicationManager.disableLedgerReplication(); + + @Cleanup + LedgerAuditorManager lam = mFactory.newLedgerAuditorManager(); + BookieId auditor = lam.getCurrentAuditor(); + ServerConfiguration conf = killBookie(auditor); + Thread.sleep(2000); + startBookie(conf); + Thread.sleep(2000); // give it time to run + assertEquals("shouldn't be anything under replicated", -1, + underReplicationManager.pollLedgerToRereplicate()); + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuthAutoRecoveryTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuthAutoRecoveryTest.java new file mode 100644 index 0000000000000..41e159b77714f --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuthAutoRecoveryTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertTrue; +import org.apache.bookkeeper.auth.AuthCallbacks; +import org.apache.bookkeeper.auth.AuthToken; +import org.apache.bookkeeper.auth.ClientAuthProvider; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.proto.ClientConnectionPeer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * This test verifies the auditor bookie scenarios from the auth point-of-view. + */ +public class AuthAutoRecoveryTest extends BookKeeperClusterTestCase { + + private static final Logger LOG = LoggerFactory + .getLogger(AuthAutoRecoveryTest.class); + + public static final String TEST_AUTH_PROVIDER_PLUGIN_NAME = "TestAuthProviderPlugin"; + + private static String clientSideRole; + + private static class AuditorClientAuthInterceptorFactory + implements ClientAuthProvider.Factory { + + @Override + public String getPluginName() { + return TEST_AUTH_PROVIDER_PLUGIN_NAME; + } + + @Override + public void init(ClientConfiguration conf) { + clientSideRole = conf.getClientRole(); + } + + @Override + public ClientAuthProvider newProvider(ClientConnectionPeer addr, + final AuthCallbacks.GenericCallback completeCb) { + return new ClientAuthProvider() { + public void init(AuthCallbacks.GenericCallback cb) { + completeCb.operationComplete(BKException.Code.OK, null); + } + + public void process(AuthToken m, AuthCallbacks.GenericCallback cb) { + } + }; + } + } + + protected ServerConfiguration newServerConfiguration() throws Exception { + ServerConfiguration conf = super.newServerConfiguration(); + conf.setClientAuthProviderFactoryClass(AuditorClientAuthInterceptorFactory.class.getName()); + return conf; + } + + public AuthAutoRecoveryTest() { + super(6); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + super.tearDown(); + } + + /* + * test the client role of the auditor + */ + @Test + public void testAuthClientRole() throws Exception { + ServerConfiguration config = confByIndex(0); + assertEquals(AuditorClientAuthInterceptorFactory.class.getName(), config.getClientAuthProviderFactoryClass()); + AutoRecoveryMain main = new AutoRecoveryMain(config); + try { + main.start(); + Thread.sleep(500); + assertTrue("AuditorElector should be running", + main.auditorElector.isRunning()); + assertTrue("Replication worker should be running", + main.replicationWorker.isRunning()); + } finally { + main.shutdown(); + } + assertEquals(ClientConfiguration.CLIENT_ROLE_SYSTEM, clientSideRole); + } + +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java new file mode 100644 index 0000000000000..1d741c551ddb9 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertFalse; +import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.AssertJUnit.assertTrue; +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.concurrent.TimeUnit; +import org.apache.bookkeeper.bookie.BookieImpl; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.util.TestUtils; +import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory; +import org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver; +import org.apache.pulsar.metadata.impl.ZKMetadataStore; +import org.apache.zookeeper.ZooKeeper; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Test the AuditorPeer. + */ +public class AutoRecoveryMainTest extends BookKeeperClusterTestCase { + + public AutoRecoveryMainTest() throws Exception { + super(3); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + super.tearDown(); + } + + /** + * Test the startup of the auditorElector and RW. + */ + @Test + public void testStartup() throws Exception { + confByIndex(0).setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + AutoRecoveryMain main = new AutoRecoveryMain(confByIndex(0)); + try { + main.start(); + Thread.sleep(500); + assertTrue("AuditorElector should be running", + main.auditorElector.isRunning()); + assertTrue("Replication worker should be running", + main.replicationWorker.isRunning()); + } finally { + main.shutdown(); + } + } + + /* + * Test the shutdown of all daemons + */ + @Test + public void testShutdown() throws Exception { + confByIndex(0).setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + AutoRecoveryMain main = new AutoRecoveryMain(confByIndex(0)); + main.start(); + Thread.sleep(500); + assertTrue("AuditorElector should be running", + main.auditorElector.isRunning()); + assertTrue("Replication worker should be running", + main.replicationWorker.isRunning()); + + main.shutdown(); + assertFalse("AuditorElector should not be running", + main.auditorElector.isRunning()); + assertFalse("Replication worker should not be running", + main.replicationWorker.isRunning()); + } + + /** + * Test that, if an autorecovery looses its ZK connection/session it will + * shutdown. + */ + @Test + public void testAutoRecoverySessionLoss() throws Exception { + confByIndex(0).setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + confByIndex(1).setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + confByIndex(2).setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + /* + * initialize three AutoRecovery instances. + */ + AutoRecoveryMain main1 = new AutoRecoveryMain(confByIndex(0)); + AutoRecoveryMain main2 = new AutoRecoveryMain(confByIndex(1)); + AutoRecoveryMain main3 = new AutoRecoveryMain(confByIndex(2)); + + /* + * start main1, make sure all the components are started and main1 is + * the current Auditor + */ + PulsarMetadataClientDriver pulsarMetadataClientDriver1 = startAutoRecoveryMain(main1); + ZooKeeper zk1 = getZk(pulsarMetadataClientDriver1); + + // Wait until auditor gets elected + for (int i = 0; i < 10; i++) { + try { + if (main1.auditorElector.getCurrentAuditor() != null) { + break; + } else { + Thread.sleep(1000); + } + } catch (IOException e) { + Thread.sleep(1000); + } + } + BookieId currentAuditor = main1.auditorElector.getCurrentAuditor(); + assertNotNull(currentAuditor); + Auditor auditor1 = main1.auditorElector.getAuditor(); + assertEquals("Current Auditor should be AR1", currentAuditor, BookieImpl.getBookieId(confByIndex(0))); + Awaitility.waitAtMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + assertNotNull(auditor1); + assertTrue("Auditor of AR1 should be running", auditor1.isRunning()); + }); + + + /* + * start main2 and main3 + */ + PulsarMetadataClientDriver pulsarMetadataClientDriver2 = startAutoRecoveryMain(main2); + ZooKeeper zk2 = getZk(pulsarMetadataClientDriver2); + + PulsarMetadataClientDriver pulsarMetadataClientDriver3 = startAutoRecoveryMain(main3); + ZooKeeper zk3 = getZk(pulsarMetadataClientDriver3); + + + /* + * make sure AR1 is still the current Auditor and AR2's and AR3's + * auditors are not running. + */ + assertEquals("Current Auditor should still be AR1", currentAuditor, BookieImpl.getBookieId(confByIndex(0))); + Awaitility.await().untilAsserted(() -> { + assertTrue("AR2's Auditor should not be running", (main2.auditorElector.getAuditor() == null + || !main2.auditorElector.getAuditor().isRunning())); + assertTrue("AR3's Auditor should not be running", (main3.auditorElector.getAuditor() == null + || !main3.auditorElector.getAuditor().isRunning())); + }); + + + /* + * expire zk2 and zk1 sessions. + */ + zkUtil.expireSession(zk2); + zkUtil.expireSession(zk1); + + /* + * wait for some time for all the components of AR1 and AR2 are + * shutdown. + */ + for (int i = 0; i < 10; i++) { + if (!main1.auditorElector.isRunning() && !main1.replicationWorker.isRunning() + && !main1.isAutoRecoveryRunning() && !main2.auditorElector.isRunning() + && !main2.replicationWorker.isRunning() && !main2.isAutoRecoveryRunning()) { + break; + } + Thread.sleep(1000); + } + + /* + * the AR3 should be current auditor. + */ + currentAuditor = main3.auditorElector.getCurrentAuditor(); + assertEquals("Current Auditor should be AR3", currentAuditor, BookieImpl.getBookieId(confByIndex(2))); + Awaitility.await().untilAsserted(() -> { + assertNotNull(main3.auditorElector.getAuditor()); + assertTrue("Auditor of AR3 should be running", main3.auditorElector.getAuditor().isRunning()); + }); + + Awaitility.waitAtMost(100, TimeUnit.SECONDS).untilAsserted(() -> { + /* + * since AR3 is current auditor, AR1's auditor should not be running + * anymore. + */ + assertFalse("AR1's auditor should not be running", auditor1.isRunning()); + + /* + * components of AR2 and AR3 should not be running since zk1 and zk2 + * sessions are expired. + */ + assertFalse("Elector1 should have shutdown", main1.auditorElector.isRunning()); + assertFalse("RW1 should have shutdown", main1.replicationWorker.isRunning()); + assertFalse("AR1 should have shutdown", main1.isAutoRecoveryRunning()); + assertFalse("Elector2 should have shutdown", main2.auditorElector.isRunning()); + assertFalse("RW2 should have shutdown", main2.replicationWorker.isRunning()); + assertFalse("AR2 should have shutdown", main2.isAutoRecoveryRunning()); + }); + + } + + /* + * start autoRecoveryMain and make sure all its components are running and + * myVote node is existing + */ + PulsarMetadataClientDriver startAutoRecoveryMain(AutoRecoveryMain autoRecoveryMain) throws Exception { + autoRecoveryMain.start(); + PulsarMetadataClientDriver pulsarMetadataClientDriver = (PulsarMetadataClientDriver) autoRecoveryMain.bkc + .getMetadataClientDriver(); + TestUtils.assertEventuallyTrue("autoRecoveryMain components should be running", + () -> autoRecoveryMain.auditorElector.isRunning() + && autoRecoveryMain.replicationWorker.isRunning() && autoRecoveryMain.isAutoRecoveryRunning()); + return pulsarMetadataClientDriver; + } + + private ZooKeeper getZk(PulsarMetadataClientDriver pulsarMetadataClientDriver) throws Exception { + PulsarLedgerManagerFactory pulsarLedgerManagerFactory = + (PulsarLedgerManagerFactory) pulsarMetadataClientDriver.getLedgerManagerFactory(); + Field field = pulsarLedgerManagerFactory.getClass().getDeclaredField("store"); + field.setAccessible(true); + ZKMetadataStore zkMetadataStore = (ZKMetadataStore) field.get(pulsarLedgerManagerFactory); + return zkMetadataStore.getZkClient(); + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookKeeperClusterTestCase.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookKeeperClusterTestCase.java new file mode 100644 index 0000000000000..ccbdb8cef64c5 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookKeeperClusterTestCase.java @@ -0,0 +1,871 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * This file is derived from BookKeeperClusterTestCase from Apache BookKeeper + * http://bookkeeper.apache.org + */ + +package org.apache.bookkeeper.replication; + +import static org.apache.bookkeeper.util.BookKeeperConstants.AVAILABLE_NODE; +import static org.apache.pulsar.common.util.PortManager.nextLockedFreePort; +import static org.testng.Assert.assertFalse; +import com.google.common.base.Stopwatch; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Method; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.bookkeeper.bookie.Bookie; +import org.apache.bookkeeper.bookie.BookieException; +import org.apache.bookkeeper.client.BookKeeperTestClient; +import org.apache.bookkeeper.client.TestStatsProvider; +import org.apache.bookkeeper.common.allocator.PoolingPolicy; +import org.apache.bookkeeper.conf.AbstractConfiguration; +import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.conf.TestBKConfiguration; +import org.apache.bookkeeper.metastore.InMemoryMetaStore; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.net.BookieSocketAddress; +import org.apache.bookkeeper.proto.BookieServer; +import org.apache.bookkeeper.test.ServerTester; +import org.apache.bookkeeper.test.TmpDirs; +import org.apache.bookkeeper.test.ZooKeeperCluster; +import org.apache.bookkeeper.test.ZooKeeperClusterUtil; +import org.apache.pulsar.common.util.PortManager; +import org.apache.pulsar.metadata.api.MetadataStoreConfig; +import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.apache.pulsar.metadata.impl.FaultInjectionMetadataStore; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooKeeper; +import org.awaitility.Awaitility; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterTest; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.BeforeTest; + +/** + * A class runs several bookie servers for testing. + */ +public abstract class BookKeeperClusterTestCase { + + static final Logger LOG = LoggerFactory.getLogger(BookKeeperClusterTestCase.class); + + protected String testName; + + @BeforeMethod + public void handleTestMethodName(Method method) { + testName = method.getName(); + } + + // Metadata service related variables + protected final ZooKeeperCluster zkUtil; + protected ZooKeeper zkc; + protected String metadataServiceUri; + protected FaultInjectionMetadataStore metadataStore; + + // BookKeeper related variables + protected final TmpDirs tmpDirs = new TmpDirs(); + protected final List servers = new LinkedList<>(); + + protected int numBookies; + protected BookKeeperTestClient bkc; + protected boolean useUUIDasBookieId = true; + + /* + * Loopback interface is set as the listening interface and allowloopback is + * set to true in this server config. So bookies in this test process would + * bind to loopback address. + */ + protected final ServerConfiguration baseConf = TestBKConfiguration.newServerConfiguration(); + protected final ClientConfiguration baseClientConf = TestBKConfiguration.newClientConfiguration(); + + private boolean isAutoRecoveryEnabled; + protected ExecutorService executor; + private final List bookiePorts = new ArrayList<>(); + + private final List closeables = new ArrayList<>(); + + SynchronousQueue asyncExceptions = new SynchronousQueue<>(); + protected void captureThrowable(Runnable c) { + try { + c.run(); + } catch (Throwable e) { + LOG.error("Captured error: ", e); + asyncExceptions.add(e); + } + } + + public BookKeeperClusterTestCase(int numBookies) { + this(numBookies, 120); + } + + public BookKeeperClusterTestCase(int numBookies, int testTimeoutSecs) { + this(numBookies, 1, testTimeoutSecs); + } + + public BookKeeperClusterTestCase(int numBookies, int numOfZKNodes, int testTimeoutSecs) { + this.numBookies = numBookies; + if (numOfZKNodes == 1) { + zkUtil = new ZooKeeperUtil(getLedgersRootPath()); + } else { + try { + zkUtil = new ZooKeeperClusterUtil(numOfZKNodes); + } catch (IOException | KeeperException | InterruptedException e) { + throw new RuntimeException(e); + } + } + } + + @BeforeTest + public void setUp() throws Exception { + setUp(getLedgersRootPath()); + } + + protected void setUp(String ledgersRootPath) throws Exception { + LOG.info("Setting up test {}", getClass()); + InMemoryMetaStore.reset(); + setMetastoreImplClass(baseConf); + setMetastoreImplClass(baseClientConf); + executor = Executors.newCachedThreadPool(); + + Stopwatch sw = Stopwatch.createStarted(); + try { + // start zookeeper service + startZKCluster(); + // start bookkeeper service + this.metadataServiceUri = getMetadataServiceUri(ledgersRootPath); + startBKCluster(metadataServiceUri); + LOG.info("Setup testcase {} @ metadata service {} in {} ms.", + testName, metadataServiceUri, sw.elapsed(TimeUnit.MILLISECONDS)); + } catch (Exception e) { + LOG.error("Error setting up", e); + throw e; + } + } + + protected String getMetadataServiceUri(String ledgersRootPath) { + return zkUtil.getMetadataServiceUri(ledgersRootPath); + } + + private String getLedgersRootPath() { + return changeLedgerPath() + "/ledgers"; + } + + protected String changeLedgerPath() { + return ""; + } + + @AfterTest(alwaysRun = true) + public void tearDown() throws Exception { + callCloseables(closeables); + closeables.clear(); + + boolean failed = false; + for (Throwable e : asyncExceptions) { + LOG.error("Got async exception: ", e); + failed = true; + } + assertFalse(failed, "Async failure"); + Stopwatch sw = Stopwatch.createStarted(); + LOG.info("TearDown"); + Exception tearDownException = null; + // stop bookkeeper service + try { + stopBKCluster(); + } catch (Exception e) { + LOG.error("Got Exception while trying to stop BKCluster", e); + tearDownException = e; + } + // stop zookeeper service + try { + // cleanup for metrics. + metadataStore.close(); + stopZKCluster(); + } catch (Exception e) { + LOG.error("Got Exception while trying to stop ZKCluster", e); + tearDownException = e; + } + // cleanup temp dirs + try { + tmpDirs.cleanup(); + } catch (Exception e) { + LOG.error("Got Exception while trying to cleanupTempDirs", e); + tearDownException = e; + } + + executor.shutdownNow(); + + LOG.info("Tearing down test {} in {} ms.", testName, sw.elapsed(TimeUnit.MILLISECONDS)); + if (tearDownException != null) { + throw tearDownException; + } + } + + protected T registerCloseable(T closeable) { + closeables.add(closeable); + return closeable; + } + + private static void callCloseables(List closeables) { + for (int i = closeables.size() - 1; i >= 0; i--) { + try { + closeables.get(i).close(); + } catch (Exception e) { + LOG.error("Failure in calling close method", e); + } + } + } + + /** + * Start zookeeper cluster. + * + * @throws Exception + */ + protected void startZKCluster() throws Exception { + zkUtil.startCluster(); + zkc = zkUtil.getZooKeeperClient(); + metadataStore = new FaultInjectionMetadataStore( + MetadataStoreExtended.create(zkUtil.getZooKeeperConnectString(), + MetadataStoreConfig.builder().metadataStoreName("metastore-" + getClass().getSimpleName()).build())); + } + + /** + * Stop zookeeper cluster. + * + * @throws Exception + */ + protected void stopZKCluster() throws Exception { + zkUtil.killCluster(); + } + + /** + * Start cluster. Also, starts the auto recovery process for each bookie, if + * isAutoRecoveryEnabled is true. + * + * @throws Exception + */ + protected void startBKCluster(String metadataServiceUri) throws Exception { + baseConf.setMetadataServiceUri(metadataServiceUri); + baseClientConf.setMetadataServiceUri(metadataServiceUri); + baseClientConf.setAllocatorPoolingPolicy(PoolingPolicy.UnpooledHeap); + + if (numBookies > 0) { + bkc = new BookKeeperTestClient(baseClientConf, new TestStatsProvider()); + } + + // Create Bookie Servers (B1, B2, B3) + for (int i = 0; i < numBookies; i++) { + bookiePorts.add(startNewBookie()); + } + } + + /** + * Stop cluster. Also, stops all the auto recovery processes for the bookie + * cluster, if isAutoRecoveryEnabled is true. + * + * @throws Exception + */ + protected void stopBKCluster() throws Exception { + if (bkc != null) { + bkc.close(); + } + + for (ServerTester t : servers) { + t.shutdown(); + } + servers.clear(); + bookiePorts.removeIf(PortManager::releaseLockedPort); + } + + protected ServerConfiguration newServerConfiguration() throws Exception { + File f = tmpDirs.createNew("bookie", "test"); + + int port; + if (baseConf.isEnableLocalTransport() || !baseConf.getAllowEphemeralPorts()) { + port = nextLockedFreePort(); + } else { + port = 0; + } + return newServerConfiguration(port, f, new File[] { f }); + } + + protected ClientConfiguration newClientConfiguration() { + return new ClientConfiguration(baseConf); + } + + protected ServerConfiguration newServerConfiguration(int port, File journalDir, File[] ledgerDirs) { + ServerConfiguration conf = new ServerConfiguration(baseConf); + conf.setBookiePort(port); + conf.setJournalDirName(journalDir.getPath()); + String[] ledgerDirNames = new String[ledgerDirs.length]; + for (int i = 0; i < ledgerDirs.length; i++) { + ledgerDirNames[i] = ledgerDirs[i].getPath(); + } + conf.setLedgerDirNames(ledgerDirNames); + conf.setEnableTaskExecutionStats(true); + conf.setAllocatorPoolingPolicy(PoolingPolicy.UnpooledHeap); + return conf; + } + + protected void stopAllBookies() throws Exception { + stopAllBookies(true); + } + + protected void stopAllBookies(boolean shutdownClient) throws Exception { + for (ServerTester t : servers) { + t.shutdown(); + } + servers.clear(); + if (shutdownClient && bkc != null) { + bkc.close(); + bkc = null; + } + } + + protected String newMetadataServiceUri(String ledgersRootPath) { + return zkUtil.getMetadataServiceUri(ledgersRootPath); + } + + protected String newMetadataServiceUri(String ledgersRootPath, String type) { + return zkUtil.getMetadataServiceUri(ledgersRootPath, type); + } + + /** + * Get bookie address for bookie at index. + */ + public BookieId getBookie(int index) throws Exception { + return servers.get(index).getServer().getBookieId(); + } + + protected List bookieAddresses() throws Exception { + List bookieIds = new ArrayList<>(); + for (ServerTester a : servers) { + bookieIds.add(a.getServer().getBookieId()); + } + return bookieIds; + } + + protected List bookieLedgerDirs() throws Exception { + return servers.stream() + .flatMap(t -> Arrays.stream(t.getConfiguration().getLedgerDirs())) + .collect(Collectors.toList()); + } + + protected List bookieJournalDirs() throws Exception { + return servers.stream() + .flatMap(t -> Arrays.stream(t.getConfiguration().getJournalDirs())) + .collect(Collectors.toList()); + } + + protected BookieId addressByIndex(int index) throws Exception { + return servers.get(index).getServer().getBookieId(); + } + + protected BookieServer serverByIndex(int index) throws Exception { + return servers.get(index).getServer(); + } + + protected ServerConfiguration confByIndex(int index) throws Exception { + return servers.get(index).getConfiguration(); + } + + private Optional byAddress(BookieId addr) throws UnknownHostException { + for (ServerTester s : servers) { + if (s.getServer().getBookieId().equals(addr)) { + return Optional.of(s); + } + } + return Optional.empty(); + } + + protected int indexOfServer(BookieServer b) throws Exception { + for (int i = 0; i < servers.size(); i++) { + if (servers.get(i).getServer().equals(b)) { + return i; + } + } + return -1; + } + + protected int lastBookieIndex() { + return servers.size() - 1; + } + + protected int bookieCount() { + return servers.size(); + } + + private OptionalInt indexByAddress(BookieId addr) throws UnknownHostException { + for (int i = 0; i < servers.size(); i++) { + if (addr.equals(servers.get(i).getServer().getBookieId())) { + return OptionalInt.of(i); + } + } + return OptionalInt.empty(); + } + + /** + * Get bookie configuration for bookie. + */ + public ServerConfiguration getBkConf(BookieId addr) throws Exception { + return byAddress(addr).get().getConfiguration(); + } + + /** + * Kill a bookie by its socket address. Also, stops the autorecovery process + * for the corresponding bookie server, if isAutoRecoveryEnabled is true. + * + * @param addr + * Socket Address + * @return the configuration of killed bookie + * @throws InterruptedException + */ + public ServerConfiguration killBookie(BookieId addr) throws Exception { + Optional tester = byAddress(addr); + if (tester.isPresent()) { + if (tester.get().autoRecovery != null + && tester.get().autoRecovery.getAuditor() != null + && tester.get().autoRecovery.getAuditor().isRunning()) { + LOG.warn("Killing bookie {} who is the current Auditor", addr); + } + servers.remove(tester.get()); + tester.get().shutdown(); + return tester.get().getConfiguration(); + } + return null; + } + + /** + * Set the bookie identified by its socket address to readonly. + * + * @param addr + * Socket Address + * @throws InterruptedException + */ + public void setBookieToReadOnly(BookieId addr) throws Exception { + Optional tester = byAddress(addr); + if (tester.isPresent()) { + tester.get().getServer().getBookie().getStateManager().transitionToReadOnlyMode().get(); + } + } + + /** + * Kill a bookie by index. Also, stops the respective auto recovery process + * for this bookie, if isAutoRecoveryEnabled is true. + * + * @param index + * Bookie Index + * @return the configuration of killed bookie + * @throws InterruptedException + * @throws IOException + */ + public ServerConfiguration killBookie(int index) throws Exception { + ServerTester tester = servers.remove(index); + tester.shutdown(); + return tester.getConfiguration(); + } + + /** + * Kill bookie by index and verify that it's stopped. + * + * @param index index of bookie to kill + * + * @return configuration of killed bookie + */ + public ServerConfiguration killBookieAndWaitForZK(int index) throws Exception { + ServerTester tester = servers.get(index); // IKTODO: this method is awful + ServerConfiguration ret = killBookie(index); + while (zkc.exists("/ledgers/" + AVAILABLE_NODE + "/" + + tester.getServer().getBookieId().toString(), false) != null) { + Thread.sleep(500); + } + return ret; + } + + /** + * Sleep a bookie. + * + * @param addr + * Socket Address + * @param seconds + * Sleep seconds + * @return Count Down latch which will be counted down just after sleep begins + * @throws InterruptedException + * @throws IOException + */ + public CountDownLatch sleepBookie(BookieId addr, final int seconds) + throws Exception { + Optional tester = byAddress(addr); + if (tester.isPresent()) { + CountDownLatch latch = new CountDownLatch(1); + Thread sleeper = new Thread() { + @Override + public void run() { + try { + tester.get().getServer().suspendProcessing(); + LOG.info("bookie {} is asleep", tester.get().getAddress()); + latch.countDown(); + Thread.sleep(seconds * 1000); + tester.get().getServer().resumeProcessing(); + LOG.info("bookie {} is awake", tester.get().getAddress()); + } catch (Exception e) { + LOG.error("Error suspending bookie", e); + } + } + }; + sleeper.start(); + return latch; + } else { + throw new IOException("Bookie not found"); + } + } + + /** + * Sleep a bookie until I count down the latch. + * + * @param addr + * Socket Address + * @param l + * Latch to wait on + * @throws InterruptedException + * @throws IOException + */ + public void sleepBookie(BookieId addr, final CountDownLatch l) + throws InterruptedException, IOException { + final CountDownLatch suspendLatch = new CountDownLatch(1); + sleepBookie(addr, l, suspendLatch); + suspendLatch.await(); + } + + public void sleepBookie(BookieId addr, final CountDownLatch l, final CountDownLatch suspendLatch) + throws InterruptedException, IOException { + Optional tester = byAddress(addr); + if (tester.isPresent()) { + BookieServer bookie = tester.get().getServer(); + LOG.info("Sleep bookie {}.", addr); + Thread sleeper = new Thread() { + @Override + public void run() { + try { + bookie.suspendProcessing(); + if (null != suspendLatch) { + suspendLatch.countDown(); + } + l.await(); + bookie.resumeProcessing(); + } catch (Exception e) { + LOG.error("Error suspending bookie", e); + } + } + }; + sleeper.start(); + } else { + throw new IOException("Bookie not found"); + } + } + + /** + * Restart bookie servers. Also restarts all the respective auto recovery + * process, if isAutoRecoveryEnabled is true. + * + * @throws InterruptedException + * @throws IOException + * @throws KeeperException + * @throws BookieException + */ + public void restartBookies() + throws Exception { + restartBookies(c -> c); + } + + /** + * Restart a bookie. Also restart the respective auto recovery process, + * if isAutoRecoveryEnabled is true. + * + * @param addr + * @throws InterruptedException + * @throws IOException + * @throws KeeperException + * @throws BookieException + */ + public void restartBookie(BookieId addr) throws Exception { + OptionalInt toRemove = indexByAddress(addr); + if (toRemove.isPresent()) { + ServerConfiguration newConfig = killBookie(toRemove.getAsInt()); + Thread.sleep(1000); + startAndAddBookie(newConfig); + } else { + throw new IOException("Bookie not found"); + } + } + + public void restartBookies(Function reconfFunction) + throws Exception { + // shut down bookie server + List confs = new ArrayList<>(); + for (ServerTester server : servers) { + server.shutdown(); + confs.add(server.getConfiguration()); + } + servers.clear(); + Thread.sleep(1000); + // restart them to ensure we can't + for (ServerConfiguration conf : confs) { + // ensure the bookie port is loaded correctly + startAndAddBookie(reconfFunction.apply(conf)); + } + } + + /** + * Helper method to startup a new bookie server with the indicated port + * number. Also, starts the auto recovery process, if the + * isAutoRecoveryEnabled is set true. + * + * @throws IOException + */ + public int startNewBookie() + throws Exception { + return startNewBookieAndReturnAddress().getPort(); + } + + public BookieSocketAddress startNewBookieAndReturnAddress() + throws Exception { + ServerConfiguration conf = newServerConfiguration(); + LOG.info("Starting new bookie on port: {}", conf.getBookiePort()); + return startAndAddBookie(conf).getServer().getLocalAddress(); + } + + public BookieId startNewBookieAndReturnBookieId() + throws Exception { + ServerConfiguration conf = newServerConfiguration(); + LOG.info("Starting new bookie on port: {}", conf.getBookiePort()); + return startAndAddBookie(conf).getServer().getBookieId(); + } + + protected ServerTester startAndAddBookie(ServerConfiguration conf) throws Exception { + ServerTester server = startBookie(conf); + servers.add(server); + return server; + } + + protected ServerTester startAndAddBookie(ServerConfiguration conf, Bookie b) throws Exception { + ServerTester server = startBookie(conf, b); + servers.add(server); + return server; + } + /** + * Helper method to startup a bookie server using a configuration object. + * Also, starts the auto recovery process if isAutoRecoveryEnabled is true. + * + * @param conf + * Server Configuration Object + * + */ + protected ServerTester startBookie(ServerConfiguration conf) + throws Exception { + ServerTester tester = new ServerTester(conf); + + if (bkc == null) { + bkc = new BookKeeperTestClient(baseClientConf, new TestStatsProvider()); + } + + BookieId address = tester.getServer().getBookieId(); + Future waitForBookie = conf.isForceReadOnlyBookie() + ? bkc.waitForReadOnlyBookie(address) + : bkc.waitForWritableBookie(address); + + tester.getServer().start(); + + waitForBookie.get(30, TimeUnit.SECONDS); + LOG.info("New bookie '{}' has been created.", address); + + if (isAutoRecoveryEnabled()) { + tester.startAutoRecovery(); + } + + int port = conf.getBookiePort(); + + Awaitility.await().atMost(30, TimeUnit.SECONDS).until(() -> { + while (zkc.exists("/ledgers/" + AVAILABLE_NODE + "/" + + tester.getServer().getBookieId().toString(), false) == null) { + Thread.sleep(100); + } + return true; + }); + bkc.readBookiesBlocking(); + + LOG.info("New bookie on port " + port + " has been created."); + + return tester; + } + + /** + * Start a bookie with the given bookie instance. Also, starts the auto + * recovery for this bookie, if isAutoRecoveryEnabled is true. + */ + protected ServerTester startBookie(ServerConfiguration conf, final Bookie b) + throws Exception { + ServerTester tester = new ServerTester(conf, b); + if (bkc == null) { + bkc = new BookKeeperTestClient(baseClientConf, new TestStatsProvider()); + } + BookieId address = tester.getServer().getBookieId(); + Future waitForBookie = conf.isForceReadOnlyBookie() + ? bkc.waitForReadOnlyBookie(address) + : bkc.waitForWritableBookie(address); + + tester.getServer().start(); + + waitForBookie.get(30, TimeUnit.SECONDS); + + if (isAutoRecoveryEnabled()) { + tester.startAutoRecovery(); + } + + int port = conf.getBookiePort(); + Awaitility.await().atMost(30, TimeUnit.SECONDS).until(() -> + metadataStore.exists( + getLedgersRootPath() + "/available/" + address).join() + ); + bkc.readBookiesBlocking(); + + LOG.info("New bookie '{}' has been created.", address); + return tester; + } + + public void setMetastoreImplClass(AbstractConfiguration conf) { + conf.setMetastoreImplClass(InMemoryMetaStore.class.getName()); + } + + /** + * Flags used to enable/disable the auto recovery process. If it is enabled, + * starting the bookie server will starts the auto recovery process for that + * bookie. Also, stopping bookie will stops the respective auto recovery + * process. + * + * @param isAutoRecoveryEnabled + * Value true will enable the auto recovery process. Value false + * will disable the auto recovery process + */ + public void setAutoRecoveryEnabled(boolean isAutoRecoveryEnabled) { + this.isAutoRecoveryEnabled = isAutoRecoveryEnabled; + } + + /** + * Flag used to check whether auto recovery process is enabled/disabled. By + * default the flag is false. + * + * @return true, if the auto recovery is enabled. Otherwise return false. + */ + public boolean isAutoRecoveryEnabled() { + return isAutoRecoveryEnabled; + } + + /** + * Will starts the auto recovery process for the bookie servers. One auto + * recovery process per each bookie server, if isAutoRecoveryEnabled is + * enabled. + */ + public void startReplicationService() throws Exception { + for (ServerTester t : servers) { + t.startAutoRecovery(); + } + } + + /** + * Will stops all the auto recovery processes for the bookie cluster, if + * isAutoRecoveryEnabled is true. + */ + public void stopReplicationService() throws Exception{ + for (ServerTester t : servers) { + t.stopAutoRecovery(); + } + } + + public Auditor getAuditor(int timeout, TimeUnit unit) throws Exception { + final long timeoutAt = System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeout, unit); + while (System.nanoTime() < timeoutAt) { + for (ServerTester t : servers) { + Auditor a = t.getAuditor(); + ReplicationWorker replicationWorker = t.getReplicationWorker(); + + // found a candidate Auditor + ReplicationWorker + if (a != null && a.isRunning() + && replicationWorker != null && replicationWorker.isRunning()) { + int deathWatchInterval = t.getConfiguration().getDeathWatchInterval(); + Thread.sleep(deathWatchInterval + 1000); + } + + // double check, because in the meantime AutoRecoveryDeathWatcher may have killed the + // AutoRecovery daemon + if (a != null && a.isRunning() + && replicationWorker != null && replicationWorker.isRunning()) { + LOG.info("Found Auditor Bookie {}", t.getServer().getBookieId()); + return a; + } + } + Thread.sleep(100); + } + throw new Exception("No auditor found"); + } + + /** + * Check whether the InetSocketAddress was created using a hostname or an IP + * address. Represent as 'hostname/IPaddress' if the InetSocketAddress was + * created using hostname. Represent as '/IPaddress' if the + * InetSocketAddress was created using an IPaddress + * + * @param bookieId id + * @return true if the address was created using an IP address, false if the + * address was created using a hostname + */ + public boolean isCreatedFromIp(BookieId bookieId) { + BookieSocketAddress addr = bkc.getBookieAddressResolver().resolve(bookieId); + return addr.getSocketAddress().toString().startsWith("/"); + } + + public void resetBookieOpLoggers() { + servers.forEach(t -> t.getStatsProvider().clear()); + } + + public TestStatsProvider getStatsProvider(BookieId addr) throws UnknownHostException { + return byAddress(addr).get().getStatsProvider(); + } + + public TestStatsProvider getStatsProvider(int index) throws Exception { + return servers.get(index).getStatsProvider(); + } + +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieAutoRecoveryTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieAutoRecoveryTest.java new file mode 100644 index 0000000000000..888303d3e665c --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieAutoRecoveryTest.java @@ -0,0 +1,651 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.AssertJUnit.assertNull; +import static org.testng.AssertJUnit.assertTrue; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.SortedMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeper.DigestType; +import org.apache.bookkeeper.client.BookKeeperTestClient; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.common.util.OrderedScheduler; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.meta.LedgerManagerFactory; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.meta.MetadataClientDriver; +import org.apache.bookkeeper.meta.MetadataDrivers; +import org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.proto.BookieServer; +import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.Watcher.Event.EventType; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Integration tests verifies the complete functionality of the + * Auditor-rereplication process: Auditor will publish the bookie failures, + * consequently ReplicationWorker will get the notifications and act on it. + */ +public class BookieAutoRecoveryTest extends BookKeeperClusterTestCase { + private static final Logger LOG = LoggerFactory + .getLogger(BookieAutoRecoveryTest.class); + private static final byte[] PASSWD = "admin".getBytes(); + private static final byte[] data = "TESTDATA".getBytes(); + private static final String openLedgerRereplicationGracePeriod = "3000"; // milliseconds + + private DigestType digestType; + private MetadataClientDriver metadataClientDriver; + private LedgerManagerFactory mFactory; + private LedgerUnderreplicationManager underReplicationManager; + private LedgerManager ledgerManager; + private OrderedScheduler scheduler; + + private final String underreplicatedPath = "/ledgers/underreplication/ledgers"; + + public BookieAutoRecoveryTest() throws Exception { + super(3); + + baseConf.setLedgerManagerFactoryClassName( + "org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory"); + baseConf.setOpenLedgerRereplicationGracePeriod(openLedgerRereplicationGracePeriod); + baseConf.setRwRereplicateBackoffMs(500); + baseClientConf.setLedgerManagerFactoryClassName( + "org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory"); + this.digestType = DigestType.MAC; + setAutoRecoveryEnabled(true); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + baseConf.setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + baseClientConf.setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + + scheduler = OrderedScheduler.newSchedulerBuilder() + .name("test-scheduler") + .numThreads(1) + .build(); + + metadataClientDriver = MetadataDrivers.getClientDriver( + URI.create(baseClientConf.getMetadataServiceUri())); + metadataClientDriver.initialize( + baseClientConf, + scheduler, + NullStatsLogger.INSTANCE, + Optional.empty()); + + // initialize urReplicationManager + mFactory = metadataClientDriver.getLedgerManagerFactory(); + underReplicationManager = mFactory.newLedgerUnderreplicationManager(); + ledgerManager = mFactory.newLedgerManager(); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + super.tearDown(); + + if (null != underReplicationManager) { + underReplicationManager.close(); + underReplicationManager = null; + } + if (null != ledgerManager) { + ledgerManager.close(); + ledgerManager = null; + } + if (null != metadataClientDriver) { + metadataClientDriver.close(); + metadataClientDriver = null; + } + if (null != scheduler) { + scheduler.shutdown(); + } + } + + /** + * Test verifies publish urLedger by Auditor and replication worker is + * picking up the entries and finishing the rereplication of open ledger. + */ + @Test + public void testOpenLedgers() throws Exception { + List listOfLedgerHandle = createLedgersAndAddEntries(1, 5); + LedgerHandle lh = listOfLedgerHandle.get(0); + int ledgerReplicaIndex = 0; + BookieId replicaToKillAddr = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + + final String urLedgerZNode = getUrLedgerZNode(lh); + ledgerReplicaIndex = getReplicaIndexInLedger(lh, replicaToKillAddr); + + CountDownLatch latch = new CountDownLatch(1); + assertNull("UrLedger already exists!", + watchUrLedgerNode(urLedgerZNode, latch)); + + LOG.info("Killing Bookie :" + replicaToKillAddr); + killBookie(replicaToKillAddr); + + // waiting to publish urLedger znode by Auditor + latch.await(); + latch = new CountDownLatch(1); + LOG.info("Watching on urLedgerPath:" + urLedgerZNode + + " to know the status of rereplication process"); + assertNotNull("UrLedger doesn't exists!", + watchUrLedgerNode(urLedgerZNode, latch)); + + // starting the replication service, so that he will be able to act as + // target bookie + startNewBookie(); + int newBookieIndex = lastBookieIndex(); + BookieServer newBookieServer = serverByIndex(newBookieIndex); + + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting to finish the replication of failed bookie : " + + replicaToKillAddr); + } + latch.await(); + + // grace period to update the urledger metadata in zookeeper + LOG.info("Waiting to update the urledger metadata in zookeeper"); + + verifyLedgerEnsembleMetadataAfterReplication(newBookieServer, + listOfLedgerHandle.get(0), ledgerReplicaIndex); + } + + /** + * Test verifies publish urLedger by Auditor and replication worker is + * picking up the entries and finishing the rereplication of closed ledgers. + */ + @Test + public void testClosedLedgers() throws Exception { + List listOfReplicaIndex = new ArrayList(); + List listOfLedgerHandle = createLedgersAndAddEntries(1, 5); + closeLedgers(listOfLedgerHandle); + LedgerHandle lhandle = listOfLedgerHandle.get(0); + int ledgerReplicaIndex = 0; + BookieId replicaToKillAddr = lhandle.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + + CountDownLatch latch = new CountDownLatch(listOfLedgerHandle.size()); + for (LedgerHandle lh : listOfLedgerHandle) { + ledgerReplicaIndex = getReplicaIndexInLedger(lh, replicaToKillAddr); + listOfReplicaIndex.add(ledgerReplicaIndex); + assertNull("UrLedger already exists!", + watchUrLedgerNode(getUrLedgerZNode(lh), latch)); + } + + LOG.info("Killing Bookie :" + replicaToKillAddr); + killBookie(replicaToKillAddr); + + // waiting to publish urLedger znode by Auditor + latch.await(); + + // Again watching the urLedger znode to know the replication status + latch = new CountDownLatch(listOfLedgerHandle.size()); + for (LedgerHandle lh : listOfLedgerHandle) { + String urLedgerZNode = getUrLedgerZNode(lh); + LOG.info("Watching on urLedgerPath:" + urLedgerZNode + + " to know the status of rereplication process"); + assertNotNull("UrLedger doesn't exists!", + watchUrLedgerNode(urLedgerZNode, latch)); + } + + // starting the replication service, so that he will be able to act as + // target bookie + startNewBookie(); + int newBookieIndex = lastBookieIndex(); + BookieServer newBookieServer = serverByIndex(newBookieIndex); + + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting to finish the replication of failed bookie : " + + replicaToKillAddr); + } + + // waiting to finish replication + latch.await(); + + // grace period to update the urledger metadata in zookeeper + LOG.info("Waiting to update the urledger metadata in zookeeper"); + + for (int index = 0; index < listOfLedgerHandle.size(); index++) { + verifyLedgerEnsembleMetadataAfterReplication(newBookieServer, + listOfLedgerHandle.get(index), + listOfReplicaIndex.get(index)); + } + } + + /** + * Test stopping replica service while replication in progress. Considering + * when there is an exception will shutdown Auditor and RW processes. After + * restarting should be able to finish the re-replication activities + */ + @Test + public void testStopWhileReplicationInProgress() throws Exception { + int numberOfLedgers = 2; + List listOfReplicaIndex = new ArrayList(); + List listOfLedgerHandle = createLedgersAndAddEntries( + numberOfLedgers, 5); + closeLedgers(listOfLedgerHandle); + LedgerHandle handle = listOfLedgerHandle.get(0); + BookieId replicaToKillAddr = handle.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + LOG.info("Killing Bookie:" + replicaToKillAddr); + + // Each ledger, there will be two events : create urLedger and after + // rereplication delete urLedger + CountDownLatch latch = new CountDownLatch(listOfLedgerHandle.size()); + for (int i = 0; i < listOfLedgerHandle.size(); i++) { + final String urLedgerZNode = getUrLedgerZNode(listOfLedgerHandle + .get(i)); + assertNull("UrLedger already exists!", + watchUrLedgerNode(urLedgerZNode, latch)); + int replicaIndexInLedger = getReplicaIndexInLedger( + listOfLedgerHandle.get(i), replicaToKillAddr); + listOfReplicaIndex.add(replicaIndexInLedger); + } + + LOG.info("Killing Bookie :" + replicaToKillAddr); + killBookie(replicaToKillAddr); + + // waiting to publish urLedger znode by Auditor + latch.await(); + + // Again watching the urLedger znode to know the replication status + latch = new CountDownLatch(listOfLedgerHandle.size()); + for (LedgerHandle lh : listOfLedgerHandle) { + String urLedgerZNode = getUrLedgerZNode(lh); + LOG.info("Watching on urLedgerPath:" + urLedgerZNode + + " to know the status of rereplication process"); + assertNotNull("UrLedger doesn't exists!", + watchUrLedgerNode(urLedgerZNode, latch)); + } + + // starting the replication service, so that he will be able to act as + // target bookie + startNewBookie(); + int newBookieIndex = lastBookieIndex(); + BookieServer newBookieServer = serverByIndex(newBookieIndex); + + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting to finish the replication of failed bookie : " + + replicaToKillAddr); + } + while (true) { + if (latch.getCount() < numberOfLedgers || latch.getCount() <= 0) { + stopReplicationService(); + LOG.info("Latch Count is:" + latch.getCount()); + break; + } + // grace period to take breath + Thread.sleep(1000); + } + + startReplicationService(); + + LOG.info("Waiting to finish rereplication processes"); + latch.await(); + + // grace period to update the urledger metadata in zookeeper + LOG.info("Waiting to update the urledger metadata in zookeeper"); + + for (int index = 0; index < listOfLedgerHandle.size(); index++) { + verifyLedgerEnsembleMetadataAfterReplication(newBookieServer, + listOfLedgerHandle.get(index), + listOfReplicaIndex.get(index)); + } + } + + /** + * Verify the published urledgers of deleted ledgers(those ledgers where + * deleted after publishing as urledgers by Auditor) should be cleared off + * by the newly selected replica bookie. + */ + @Test + public void testNoSuchLedgerExists() throws Exception { + List listOfLedgerHandle = createLedgersAndAddEntries(2, 5); + CountDownLatch latch = new CountDownLatch(listOfLedgerHandle.size()); + for (LedgerHandle lh : listOfLedgerHandle) { + assertNull("UrLedger already exists!", + watchUrLedgerNode(getUrLedgerZNode(lh), latch)); + } + BookieId replicaToKillAddr = listOfLedgerHandle.get(0) + .getLedgerMetadata().getAllEnsembles() + .get(0L).get(0); + killBookie(replicaToKillAddr); + replicaToKillAddr = listOfLedgerHandle.get(0) + .getLedgerMetadata().getAllEnsembles() + .get(0L).get(0); + killBookie(replicaToKillAddr); + // waiting to publish urLedger znode by Auditor + latch.await(); + + latch = new CountDownLatch(listOfLedgerHandle.size()); + for (LedgerHandle lh : listOfLedgerHandle) { + assertNotNull("UrLedger doesn't exists!", + watchUrLedgerNode(getUrLedgerZNode(lh), latch)); + } + + // delete ledgers + for (LedgerHandle lh : listOfLedgerHandle) { + bkc.deleteLedger(lh.getId()); + } + startNewBookie(); + + // waiting to delete published urledgers, since it doesn't exists + latch.await(); + + for (LedgerHandle lh : listOfLedgerHandle) { + assertNull("UrLedger still exists after rereplication", + watchUrLedgerNode(getUrLedgerZNode(lh), latch)); + } + } + + /** + * Test that if a empty ledger loses the bookie not in the quorum for entry 0, it will + * still be openable when it loses enough bookies to lose a whole quorum. + */ + @Test + public void testEmptyLedgerLosesQuorumEventually() throws Exception { + LedgerHandle lh = bkc.createLedger(3, 2, 2, DigestType.CRC32, PASSWD); + CountDownLatch latch = new CountDownLatch(1); + String urZNode = getUrLedgerZNode(lh); + watchUrLedgerNode(urZNode, latch); + + BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(2); + LOG.info("Killing last bookie, {}, in ensemble {}", replicaToKill, + lh.getLedgerMetadata().getAllEnsembles().get(0L)); + killBookie(replicaToKill); + startNewBookie(); + + getAuditor(10, TimeUnit.SECONDS).submitAuditTask().get(); // ensure auditor runs + + assertTrue("Should be marked as underreplicated", latch.await(5, TimeUnit.SECONDS)); + latch = new CountDownLatch(1); + Stat s = watchUrLedgerNode(urZNode, latch); // should be marked as replicated + if (s != null) { + assertTrue("Should be marked as replicated", latch.await(15, TimeUnit.SECONDS)); + } + + replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(1); + LOG.info("Killing second bookie, {}, in ensemble {}", replicaToKill, + lh.getLedgerMetadata().getAllEnsembles().get(0L)); + killBookie(replicaToKill); + + getAuditor(10, TimeUnit.SECONDS).submitAuditTask().get(); // ensure auditor runs + + assertTrue("Should be marked as underreplicated", latch.await(5, TimeUnit.SECONDS)); + latch = new CountDownLatch(1); + s = watchUrLedgerNode(urZNode, latch); // should be marked as replicated + + startNewBookie(); + getAuditor(10, TimeUnit.SECONDS).submitAuditTask().get(); // ensure auditor runs + + if (s != null) { + assertTrue("Should be marked as replicated", latch.await(20, TimeUnit.SECONDS)); + } + + // should be able to open ledger without issue + bkc.openLedger(lh.getId(), DigestType.CRC32, PASSWD); + } + + /** + * Test verifies bookie recovery, the host (recorded via ipaddress in + * ledgermetadata). + */ + @Test + public void testLedgerMetadataContainsIpAddressAsBookieID() + throws Exception { + stopBKCluster(); + bkc = new BookKeeperTestClient(baseClientConf); + // start bookie with useHostNameAsBookieID=false, as old bookie + ServerConfiguration serverConf1 = newServerConfiguration(); + // start 2 more bookies with useHostNameAsBookieID=true + ServerConfiguration serverConf2 = newServerConfiguration(); + serverConf2.setUseHostNameAsBookieID(true); + ServerConfiguration serverConf3 = newServerConfiguration(); + serverConf3.setUseHostNameAsBookieID(true); + startAndAddBookie(serverConf1); + startAndAddBookie(serverConf2); + startAndAddBookie(serverConf3); + + List listOfLedgerHandle = createLedgersAndAddEntries(1, 5); + LedgerHandle lh = listOfLedgerHandle.get(0); + int ledgerReplicaIndex = 0; + final SortedMap> ensembles = lh.getLedgerMetadata().getAllEnsembles(); + final List bkAddresses = ensembles.get(0L); + BookieId replicaToKillAddr = bkAddresses.get(0); + for (BookieId bookieSocketAddress : bkAddresses) { + if (!isCreatedFromIp(bookieSocketAddress)) { + replicaToKillAddr = bookieSocketAddress; + LOG.info("Kill bookie which has registered using hostname"); + break; + } + } + + final String urLedgerZNode = getUrLedgerZNode(lh); + ledgerReplicaIndex = getReplicaIndexInLedger(lh, replicaToKillAddr); + + CountDownLatch latch = new CountDownLatch(1); + assertNull("UrLedger already exists!", + watchUrLedgerNode(urLedgerZNode, latch)); + + LOG.info("Killing Bookie :" + replicaToKillAddr); + killBookie(replicaToKillAddr); + + // waiting to publish urLedger znode by Auditor + latch.await(); + latch = new CountDownLatch(1); + LOG.info("Watching on urLedgerPath:" + urLedgerZNode + + " to know the status of rereplication process"); + assertNotNull("UrLedger doesn't exists!", + watchUrLedgerNode(urLedgerZNode, latch)); + + // starting the replication service, so that he will be able to act as + // target bookie + ServerConfiguration serverConf = newServerConfiguration(); + serverConf.setUseHostNameAsBookieID(false); + startAndAddBookie(serverConf); + + int newBookieIndex = lastBookieIndex(); + BookieServer newBookieServer = serverByIndex(newBookieIndex); + + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting to finish the replication of failed bookie : " + + replicaToKillAddr); + } + latch.await(); + + // grace period to update the urledger metadata in zookeeper + LOG.info("Waiting to update the urledger metadata in zookeeper"); + + verifyLedgerEnsembleMetadataAfterReplication(newBookieServer, + listOfLedgerHandle.get(0), ledgerReplicaIndex); + + } + + /** + * Test verifies bookie recovery, the host (recorded via useHostName in + * ledgermetadata). + */ + @Test + public void testLedgerMetadataContainsHostNameAsBookieID() + throws Exception { + stopBKCluster(); + + bkc = new BookKeeperTestClient(baseClientConf); + // start bookie with useHostNameAsBookieID=false, as old bookie + ServerConfiguration serverConf1 = newServerConfiguration(); + // start 2 more bookies with useHostNameAsBookieID=true + ServerConfiguration serverConf2 = newServerConfiguration(); + serverConf2.setUseHostNameAsBookieID(true); + ServerConfiguration serverConf3 = newServerConfiguration(); + serverConf3.setUseHostNameAsBookieID(true); + startAndAddBookie(serverConf1); + startAndAddBookie(serverConf2); + startAndAddBookie(serverConf3); + + List listOfLedgerHandle = createLedgersAndAddEntries(1, 5); + LedgerHandle lh = listOfLedgerHandle.get(0); + int ledgerReplicaIndex = 0; + final SortedMap> ensembles = lh.getLedgerMetadata().getAllEnsembles(); + final List bkAddresses = ensembles.get(0L); + BookieId replicaToKillAddr = bkAddresses.get(0); + for (BookieId bookieSocketAddress : bkAddresses) { + if (isCreatedFromIp(bookieSocketAddress)) { + replicaToKillAddr = bookieSocketAddress; + LOG.info("Kill bookie which has registered using ipaddress"); + break; + } + } + + final String urLedgerZNode = getUrLedgerZNode(lh); + ledgerReplicaIndex = getReplicaIndexInLedger(lh, replicaToKillAddr); + + CountDownLatch latch = new CountDownLatch(1); + assertNull("UrLedger already exists!", + watchUrLedgerNode(urLedgerZNode, latch)); + + LOG.info("Killing Bookie :" + replicaToKillAddr); + killBookie(replicaToKillAddr); + + // waiting to publish urLedger znode by Auditor + latch.await(); + latch = new CountDownLatch(1); + LOG.info("Watching on urLedgerPath:" + urLedgerZNode + + " to know the status of rereplication process"); + assertNotNull("UrLedger doesn't exists!", + watchUrLedgerNode(urLedgerZNode, latch)); + + // creates new bkclient + bkc = new BookKeeperTestClient(baseClientConf); + // starting the replication service, so that he will be able to act as + // target bookie + ServerConfiguration serverConf = newServerConfiguration(); + serverConf.setUseHostNameAsBookieID(true); + startAndAddBookie(serverConf); + + int newBookieIndex = lastBookieIndex(); + BookieServer newBookieServer = serverByIndex(newBookieIndex); + + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting to finish the replication of failed bookie : " + + replicaToKillAddr); + } + latch.await(); + + // grace period to update the urledger metadata in zookeeper + LOG.info("Waiting to update the urledger metadata in zookeeper"); + + verifyLedgerEnsembleMetadataAfterReplication(newBookieServer, + listOfLedgerHandle.get(0), ledgerReplicaIndex); + + } + + private int getReplicaIndexInLedger(LedgerHandle lh, BookieId replicaToKill) { + SortedMap> ensembles = lh.getLedgerMetadata().getAllEnsembles(); + int ledgerReplicaIndex = -1; + for (BookieId addr : ensembles.get(0L)) { + ++ledgerReplicaIndex; + if (addr.equals(replicaToKill)) { + break; + } + } + return ledgerReplicaIndex; + } + + private void verifyLedgerEnsembleMetadataAfterReplication( + BookieServer newBookieServer, LedgerHandle lh, + int ledgerReplicaIndex) throws Exception { + LedgerHandle openLedger = bkc + .openLedger(lh.getId(), digestType, PASSWD); + + BookieId inetSocketAddress = openLedger.getLedgerMetadata().getAllEnsembles().get(0L) + .get(ledgerReplicaIndex); + assertEquals("Rereplication has been failed and ledgerReplicaIndex :" + + ledgerReplicaIndex, newBookieServer.getBookieId(), + inetSocketAddress); + openLedger.close(); + } + + private void closeLedgers(List listOfLedgerHandle) + throws InterruptedException, BKException { + for (LedgerHandle lh : listOfLedgerHandle) { + lh.close(); + } + } + + private List createLedgersAndAddEntries(int numberOfLedgers, + int numberOfEntries) + throws InterruptedException, BKException { + List listOfLedgerHandle = new ArrayList( + numberOfLedgers); + for (int index = 0; index < numberOfLedgers; index++) { + LedgerHandle lh = bkc.createLedger(3, 3, digestType, PASSWD); + listOfLedgerHandle.add(lh); + for (int i = 0; i < numberOfEntries; i++) { + lh.addEntry(data); + } + } + return listOfLedgerHandle; + } + + private String getUrLedgerZNode(LedgerHandle lh) { + return ZkLedgerUnderreplicationManager.getUrLedgerZnode( + underreplicatedPath, lh.getId()); + } + + private Stat watchUrLedgerNode(final String znode, + final CountDownLatch latch) throws KeeperException, + InterruptedException { + return zkc.exists(znode, new Watcher() { + @Override + public void process(WatchedEvent event) { + if (event.getType() == EventType.NodeDeleted) { + LOG.info("Received Ledger rereplication completion event :" + + event.getType()); + latch.countDown(); + } + if (event.getType() == EventType.NodeCreated) { + LOG.info("Received urLedger publishing event :" + + event.getType()); + latch.countDown(); + } + } + }); + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieLedgerIndexTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieLedgerIndexTest.java new file mode 100644 index 0000000000000..425f485e1ad2b --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieLedgerIndexTest.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertTrue; +import static org.testng.AssertJUnit.fail; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeper.DigestType; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.meta.LayoutManager; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.meta.LedgerManagerFactory; +import org.apache.pulsar.metadata.api.MetadataStoreConfig; +import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.apache.pulsar.metadata.bookkeeper.PulsarLayoutManager; +import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Tests verifies bookie vs ledger mapping generating by the BookieLedgerIndexer. + */ +public class BookieLedgerIndexTest extends BookKeeperClusterTestCase { + + // Depending on the taste, select the amount of logging + // by decommenting one of the two lines below + // private final static Logger LOG = Logger.getRootLogger(); + private static final Logger LOG = LoggerFactory + .getLogger(BookieLedgerIndexTest.class); + + private Random rng; // Random Number Generator + private ArrayList entries; // generated entries + private final DigestType digestType = DigestType.CRC32; + private int numberOfLedgers = 3; + private List ledgerList; + private LedgerManagerFactory newLedgerManagerFactory; + private LedgerManager ledgerManager; + + public BookieLedgerIndexTest() throws Exception { + this("org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory"); + } + + BookieLedgerIndexTest(String ledgerManagerFactory) throws Exception { + super(3); + LOG.info("Running test case using ledger manager : " + + ledgerManagerFactory); + // set ledger manager name + baseConf.setLedgerManagerFactoryClassName(ledgerManagerFactory); + baseClientConf.setLedgerManagerFactoryClassName(ledgerManagerFactory); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + } + + @BeforeMethod + public void setUp() throws Exception { + super.setUp(); + baseConf.setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + rng = new Random(System.currentTimeMillis()); // Initialize the Random + // Number Generator + entries = new ArrayList(); // initialize the entries list + ledgerList = new ArrayList(3); + + String ledgersRoot = "/ledgers"; + String storeUri = metadataServiceUri.replaceAll("zk://", "").replaceAll("/ledgers", ""); + MetadataStoreExtended store = registerCloseable(MetadataStoreExtended.create(storeUri, + MetadataStoreConfig.builder().fsyncEnable(false).build())); + LayoutManager layoutManager = new PulsarLayoutManager(store, ledgersRoot); + newLedgerManagerFactory = new PulsarLedgerManagerFactory(); + + ClientConfiguration conf = new ClientConfiguration(); + conf.setZkLedgersRootPath(ledgersRoot); + newLedgerManagerFactory.initialize(conf, layoutManager, 1); + ledgerManager = newLedgerManagerFactory.newLedgerManager(); + } + + @AfterMethod + public void tearDown() throws Exception { + super.tearDown(); + if (null != newLedgerManagerFactory) { + newLedgerManagerFactory.close(); + newLedgerManagerFactory = null; + } + if (null != ledgerManager) { + ledgerManager.close(); + ledgerManager = null; + } + } + + /** + * Verify the bookie-ledger mapping with minimum number of bookies and few + * ledgers. + */ + @Test + public void testSimpleBookieLedgerMapping() throws Exception { + + for (int i = 0; i < numberOfLedgers; i++) { + createAndAddEntriesToLedger().close(); + } + + BookieLedgerIndexer bookieLedgerIndex = new BookieLedgerIndexer( + ledgerManager); + + Map> bookieToLedgerIndex = bookieLedgerIndex + .getBookieToLedgerIndex(); + + assertEquals("Missed few bookies in the bookie-ledger mapping!", 3, + bookieToLedgerIndex.size()); + Collection> bk2ledgerEntry = bookieToLedgerIndex.values(); + for (Set ledgers : bk2ledgerEntry) { + assertEquals("Missed few ledgers in the bookie-ledger mapping!", 3, + ledgers.size()); + for (Long ledgerId : ledgers) { + assertTrue("Unknown ledger-bookie mapping", ledgerList + .contains(ledgerId)); + } + } + } + + /** + * Verify ledger index with failed bookies and throws exception. + */ + @SuppressWarnings("deprecation") +// @Test +// public void testWithoutZookeeper() throws Exception { +// // This test case is for ledger metadata that stored in ZooKeeper. As +// // far as MSLedgerManagerFactory, ledger metadata are stored in other +// // storage. So this test is not suitable for MSLedgerManagerFactory. +// if (newLedgerManagerFactory instanceof org.apache.bookkeeper.meta.MSLedgerManagerFactory) { +// return; +// } +// +// for (int i = 0; i < numberOfLedgers; i++) { +// createAndAddEntriesToLedger().close(); +// } +// +// BookieLedgerIndexer bookieLedgerIndex = new BookieLedgerIndexer( +// ledgerManager); +// stopZKCluster(); +// try { +// bookieLedgerIndex.getBookieToLedgerIndex(); +// fail("Must throw exception as zookeeper are not running!"); +// } catch (BKAuditException bkAuditException) { +// // expected behaviour +// } +// } + + /** + * Verify indexing with multiple ensemble reformation. + */ + @Test + public void testEnsembleReformation() throws Exception { + try { + LedgerHandle lh1 = createAndAddEntriesToLedger(); + LedgerHandle lh2 = createAndAddEntriesToLedger(); + + startNewBookie(); + shutdownBookie(lastBookieIndex() - 1); + + // add few more entries after ensemble reformation + for (int i = 0; i < 10; i++) { + ByteBuffer entry = ByteBuffer.allocate(4); + entry.putInt(rng.nextInt(Integer.MAX_VALUE)); + entry.position(0); + + entries.add(entry.array()); + lh1.addEntry(entry.array()); + lh2.addEntry(entry.array()); + } + + BookieLedgerIndexer bookieLedgerIndex = new BookieLedgerIndexer( + ledgerManager); + + Map> bookieToLedgerIndex = bookieLedgerIndex + .getBookieToLedgerIndex(); + assertEquals("Missed few bookies in the bookie-ledger mapping!", 4, + bookieToLedgerIndex.size()); + Collection> bk2ledgerEntry = bookieToLedgerIndex.values(); + for (Set ledgers : bk2ledgerEntry) { + assertEquals( + "Missed few ledgers in the bookie-ledger mapping!", 2, + ledgers.size()); + for (Long ledgerNode : ledgers) { + assertTrue("Unknown ledger-bookie mapping", ledgerList + .contains(ledgerNode)); + } + } + } catch (BKException e) { + LOG.error("Test failed", e); + fail("Test failed due to BookKeeper exception"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.error("Test failed", e); + fail("Test failed due to interruption"); + } + } + + private void shutdownBookie(int bkShutdownIndex) throws Exception { + killBookie(bkShutdownIndex); + } + + private LedgerHandle createAndAddEntriesToLedger() throws BKException, + InterruptedException { + int numEntriesToWrite = 20; + // Create a ledger + LedgerHandle lh = bkc.createLedger(digestType, "admin".getBytes()); + LOG.info("Ledger ID: " + lh.getId()); + for (int i = 0; i < numEntriesToWrite; i++) { + ByteBuffer entry = ByteBuffer.allocate(4); + entry.putInt(rng.nextInt(Integer.MAX_VALUE)); + entry.position(0); + + entries.add(entry.array()); + lh.addEntry(entry.array()); + } + ledgerList.add(lh.getId()); + return lh; + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/ReplicationTestUtil.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/ReplicationTestUtil.java new file mode 100644 index 0000000000000..4360bf3254675 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/ReplicationTestUtil.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import java.util.List; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooKeeper; + +/** + * Utility class for replication tests. + */ +public class ReplicationTestUtil { + + /** + * Checks whether ledger is in under-replication. + */ + public static boolean isLedgerInUnderReplication(ZooKeeper zkc, long id, + String basePath) throws KeeperException, InterruptedException { + List children; + try { + children = zkc.getChildren(basePath, true); + } catch (KeeperException.NoNodeException nne) { + return false; + } + + boolean isMatched = false; + for (String child : children) { + if (child.startsWith("urL") && child.contains(String.valueOf(id))) { + isMatched = true; + break; + } else { + String path = basePath + '/' + child; + try { + if (zkc.getChildren(path, false).size() > 0) { + isMatched = isLedgerInUnderReplication(zkc, id, path); + } + } catch (KeeperException.NoNodeException nne) { + return false; + } + } + + } + return isMatched; + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestAutoRecoveryAlongWithBookieServers.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestAutoRecoveryAlongWithBookieServers.java new file mode 100644 index 0000000000000..11797c8373715 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestAutoRecoveryAlongWithBookieServers.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertTrue; +import java.util.Enumeration; +import java.util.List; +import java.util.Map.Entry; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.LedgerEntry; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.util.BookKeeperConstants; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Test auto recovery. + */ +public class TestAutoRecoveryAlongWithBookieServers extends + BookKeeperClusterTestCase { + + private String basePath = ""; + + public TestAutoRecoveryAlongWithBookieServers() throws Exception { + super(3); + setAutoRecoveryEnabled(true); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + basePath = BookKeeperConstants.DEFAULT_ZK_LEDGERS_ROOT_PATH + '/' + + BookKeeperConstants.UNDER_REPLICATION_NODE + + BookKeeperConstants.DEFAULT_ZK_LEDGERS_ROOT_PATH; + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + super.tearDown(); + } + + @Override + protected void startBKCluster(String metadataServiceUri) throws Exception { + super.startBKCluster(metadataServiceUri.replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + } + + /** + * Tests that the auto recovery service along with Bookie servers itself. + */ + @Test + public void testAutoRecoveryAlongWithBookieServers() throws Exception { + LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, + "testpasswd".getBytes()); + byte[] testData = "testBuiltAutoRecovery".getBytes(); + + for (int i = 0; i < 10; i++) { + lh.addEntry(testData); + } + lh.close(); + BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + + killBookie(replicaToKill); + + BookieId newBkAddr = startNewBookieAndReturnBookieId(); + + while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), + basePath)) { + Thread.sleep(100); + } + + // Killing all bookies except newly replicated bookie + for (Entry> entry : + lh.getLedgerMetadata().getAllEnsembles().entrySet()) { + List bookies = entry.getValue(); + for (BookieId bookie : bookies) { + if (bookie.equals(newBkAddr)) { + continue; + } + killBookie(bookie); + } + } + + // Should be able to read the entries from 0-9 + LedgerHandle lhs = bkc.openLedgerNoRecovery(lh.getId(), + BookKeeper.DigestType.CRC32, "testpasswd".getBytes()); + Enumeration entries = lhs.readEntries(0, 9); + assertTrue("Should have the elements", entries.hasMoreElements()); + while (entries.hasMoreElements()) { + LedgerEntry entry = entries.nextElement(); + assertEquals("testBuiltAutoRecovery", new String(entry.getEntry())); + } + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java new file mode 100644 index 0000000000000..7938feaba19fe --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java @@ -0,0 +1,1248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE; +import static org.apache.bookkeeper.replication.ReplicationStats.NUM_ENTRIES_UNABLE_TO_READ_FOR_REPLICATION; +import static org.apache.bookkeeper.replication.ReplicationStats.REPLICATION_SCOPE; +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertFalse; +import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.AssertJUnit.assertNull; +import static org.testng.AssertJUnit.assertTrue; +import static org.testng.AssertJUnit.fail; +import io.netty.util.HashedWheelTimer; +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.net.URI; +import java.net.UnknownHostException; +import java.util.Enumeration; +import java.util.List; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; +import java.util.TimerTask; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; +import lombok.Cleanup; +import org.apache.bookkeeper.bookie.BookieImpl; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.BookKeeperTestClient; +import org.apache.bookkeeper.client.ClientUtil; +import org.apache.bookkeeper.client.EnsemblePlacementPolicy; +import org.apache.bookkeeper.client.LedgerEntry; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy; +import org.apache.bookkeeper.client.ZoneawareEnsemblePlacementPolicy; +import org.apache.bookkeeper.client.api.LedgerMetadata; +import org.apache.bookkeeper.common.util.OrderedScheduler; +import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.feature.FeatureProvider; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.meta.LedgerManagerFactory; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.meta.MetadataBookieDriver; +import org.apache.bookkeeper.meta.MetadataClientDriver; +import org.apache.bookkeeper.meta.MetadataDrivers; +import org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager; +import org.apache.bookkeeper.meta.exceptions.MetadataException; +import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.net.DNSToSwitchMapping; +import org.apache.bookkeeper.proto.BookieAddressResolver; +import org.apache.bookkeeper.replication.ReplicationException.CompatibilityException; +import org.apache.bookkeeper.stats.Counter; +import org.apache.bookkeeper.stats.Gauge; +import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.bookkeeper.stats.StatsLogger; +import org.apache.bookkeeper.test.TestStatsProvider; +import org.apache.bookkeeper.test.TestStatsProvider.TestStatsLogger; +import org.apache.bookkeeper.util.BookKeeperConstants; +import org.apache.bookkeeper.util.StaticDNSResolver; +import org.apache.bookkeeper.zookeeper.ZooKeeperClient; +import org.apache.commons.lang3.mutable.MutableObject; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory; +import org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver; +import org.apache.pulsar.metadata.impl.ZKMetadataStore; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.data.Stat; +import org.awaitility.Awaitility; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Test the ReplicationWroker, where it has to replicate the fragments from + * failed Bookies to given target Bookie. + */ +public class TestReplicationWorker extends BookKeeperClusterTestCase { + + private static final byte[] TESTPASSWD = "testpasswd".getBytes(); + private static final Logger LOG = LoggerFactory + .getLogger(TestReplicationWorker.class); + private String basePath = ""; + private String baseLockPath = ""; + private MetadataBookieDriver driver; + private LedgerManagerFactory mFactory; + private LedgerUnderreplicationManager underReplicationManager; + private LedgerManager ledgerManager; + private static byte[] data = "TestReplicationWorker".getBytes(); + private OrderedScheduler scheduler; + private String zkLedgersRootPath; + + public TestReplicationWorker() throws Exception { + this("org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory"); + } + + TestReplicationWorker(String ledgerManagerFactory) throws Exception { + super(3, 300); + LOG.info("Running test case using ledger manager : " + + ledgerManagerFactory); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + // set ledger manager name + baseConf.setLedgerManagerFactoryClassName(ledgerManagerFactory); + baseClientConf.setLedgerManagerFactoryClassName(ledgerManagerFactory); + baseConf.setRereplicationEntryBatchSize(3); + baseConf.setZkTimeout(7000); + baseConf.setZkRetryBackoffMaxMs(500); + baseConf.setZkRetryBackoffStartMs(10); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + zkLedgersRootPath = ZKMetadataDriverBase.resolveZkLedgersRootPath(baseClientConf); + basePath = zkLedgersRootPath + '/' + + BookKeeperConstants.UNDER_REPLICATION_NODE + + BookKeeperConstants.DEFAULT_ZK_LEDGERS_ROOT_PATH; + baseLockPath = zkLedgersRootPath + '/' + + BookKeeperConstants.UNDER_REPLICATION_NODE + + "/locks"; + baseClientConf.setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + baseConf.setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + this.scheduler = OrderedScheduler.newSchedulerBuilder() + .name("test-scheduler") + .numThreads(1) + .build(); + + this.driver = MetadataDrivers.getBookieDriver( + URI.create(baseConf.getMetadataServiceUri())); + this.driver.initialize( + baseConf, + NullStatsLogger.INSTANCE); + // initialize urReplicationManager + mFactory = driver.getLedgerManagerFactory(); + ledgerManager = mFactory.newLedgerManager(); + underReplicationManager = mFactory.newLedgerUnderreplicationManager(); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + super.tearDown(); + if (null != ledgerManager) { + ledgerManager.close(); + ledgerManager = null; + } + if (null != underReplicationManager) { + underReplicationManager.close(); + underReplicationManager = null; + } + if (null != driver) { + driver.close(); + } + if (null != scheduler) { + scheduler.shutdown(); + scheduler = null; + } + if (null != mFactory) { + mFactory.close(); + } + } + + /** + * Tests that replication worker should replicate the failed bookie + * fragments to target bookie given to the worker. + */ + @Test + public void testRWShouldReplicateFragmentsToTargetBookie() throws Exception { + LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, + TESTPASSWD); + + for (int i = 0; i < 10; i++) { + lh.addEntry(data); + } + BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + + LOG.info("Killing Bookie : {}", replicaToKill); + killBookie(replicaToKill); + + BookieId newBkAddr = startNewBookieAndReturnBookieId(); + LOG.info("New Bookie addr : {}", newBkAddr); + + for (int i = 0; i < 10; i++) { + lh.addEntry(data); + } + + ReplicationWorker rw = new ReplicationWorker(baseConf); + + rw.start(); + try { + + underReplicationManager.markLedgerUnderreplicated(lh.getId(), + replicaToKill.toString()); + + while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh + .getId(), basePath)) { + Thread.sleep(100); + } + + killAllBookies(lh, newBkAddr); + + // Should be able to read the entries from 0-9 + verifyRecoveredLedgers(lh, 0, 9); + } finally { + rw.shutdown(); + } + } + + /** + * Tests that replication worker should retry for replication until enough + * bookies available for replication. + */ + @Test + public void testRWShouldRetryUntilThereAreEnoughBksAvailableForReplication() + throws Exception { + LedgerHandle lh = bkc.createLedger(1, 1, BookKeeper.DigestType.CRC32, + TESTPASSWD); + + for (int i = 0; i < 10; i++) { + lh.addEntry(data); + } + lh.close(); + BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + LOG.info("Killing Bookie : {}", replicaToKill); + ServerConfiguration killedBookieConfig = killBookie(replicaToKill); + + BookieId newBkAddr = startNewBookieAndReturnBookieId(); + LOG.info("New Bookie addr :" + newBkAddr); + + killAllBookies(lh, newBkAddr); + ReplicationWorker rw = new ReplicationWorker(baseConf); + + rw.start(); + try { + underReplicationManager.markLedgerUnderreplicated(lh.getId(), + replicaToKill.toString()); + int counter = 30; + while (counter-- > 0) { + assertTrue("Expecting that replication should not complete", + ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh + .getId(), basePath)); + Thread.sleep(100); + } + // restart killed bookie + startAndAddBookie(killedBookieConfig); + while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh + .getId(), basePath)) { + Thread.sleep(100); + } + // Should be able to read the entries from 0-9 + verifyRecoveredLedgers(lh, 0, 9); + } finally { + rw.shutdown(); + } + } + + /** + * Tests that replication worker1 should take one fragment replication and + * other replication worker also should compete for the replication. + */ + @Test + public void test2RWsShouldCompeteForReplicationOf2FragmentsAndCompleteReplication() + throws Exception { + LedgerHandle lh = bkc.createLedger(2, 2, BookKeeper.DigestType.CRC32, + TESTPASSWD); + + for (int i = 0; i < 10; i++) { + lh.addEntry(data); + } + lh.close(); + BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + LOG.info("Killing Bookie : {}", replicaToKill); + ServerConfiguration killedBookieConfig = killBookie(replicaToKill); + + killAllBookies(lh, null); + // Starte RW1 + BookieId newBkAddr1 = startNewBookieAndReturnBookieId(); + LOG.info("New Bookie addr : {}", newBkAddr1); + ReplicationWorker rw1 = new ReplicationWorker(baseConf); + + // Starte RW2 + BookieId newBkAddr2 = startNewBookieAndReturnBookieId(); + LOG.info("New Bookie addr : {}", newBkAddr2); + ReplicationWorker rw2 = new ReplicationWorker(baseConf); + rw1.start(); + rw2.start(); + + try { + underReplicationManager.markLedgerUnderreplicated(lh.getId(), + replicaToKill.toString()); + int counter = 10; + while (counter-- > 0) { + assertTrue("Expecting that replication should not complete", + ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh + .getId(), basePath)); + Thread.sleep(100); + } + // restart killed bookie + startAndAddBookie(killedBookieConfig); + while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh + .getId(), basePath)) { + Thread.sleep(100); + } + // Should be able to read the entries from 0-9 + verifyRecoveredLedgers(lh, 0, 9); + } finally { + rw1.shutdown(); + rw2.shutdown(); + } + } + + /** + * Tests that Replication worker should clean the leadger under replication + * node of the ledger already deleted. + */ + @Test + public void testRWShouldCleanTheLedgerFromUnderReplicationIfLedgerAlreadyDeleted() + throws Exception { + LedgerHandle lh = bkc.createLedger(2, 2, BookKeeper.DigestType.CRC32, + TESTPASSWD); + + for (int i = 0; i < 10; i++) { + lh.addEntry(data); + } + lh.close(); + BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + LOG.info("Killing Bookie : {}", replicaToKill); + killBookie(replicaToKill); + + BookieId newBkAddr = startNewBookieAndReturnBookieId(); + LOG.info("New Bookie addr : {}", newBkAddr); + ReplicationWorker rw = new ReplicationWorker(baseConf); + rw.start(); + + try { + bkc.deleteLedger(lh.getId()); // Deleting the ledger + // Also mark ledger as in UnderReplication + underReplicationManager.markLedgerUnderreplicated(lh.getId(), + replicaToKill.toString()); + while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh + .getId(), basePath)) { + Thread.sleep(100); + } + } finally { + rw.shutdown(); + } + + } + + @Test + public void testMultipleLedgerReplicationWithReplicationWorker() + throws Exception { + // Ledger1 + LedgerHandle lh1 = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, + TESTPASSWD); + + for (int i = 0; i < 10; i++) { + lh1.addEntry(data); + } + BookieId replicaToKillFromFirstLedger = lh1.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + + LOG.info("Killing Bookie : {}", replicaToKillFromFirstLedger); + + // Ledger2 + LedgerHandle lh2 = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, + TESTPASSWD); + + for (int i = 0; i < 10; i++) { + lh2.addEntry(data); + } + BookieId replicaToKillFromSecondLedger = lh2.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + + LOG.info("Killing Bookie : {}", replicaToKillFromSecondLedger); + + // Kill ledger1 + killBookie(replicaToKillFromFirstLedger); + lh1.close(); + // Kill ledger2 + killBookie(replicaToKillFromFirstLedger); + lh2.close(); + + BookieId newBkAddr = startNewBookieAndReturnBookieId(); + LOG.info("New Bookie addr : {}", newBkAddr); + + ReplicationWorker rw = new ReplicationWorker(baseConf); + + rw.start(); + try { + + // Mark ledger1 and 2 as underreplicated + underReplicationManager.markLedgerUnderreplicated(lh1.getId(), + replicaToKillFromFirstLedger.toString()); + underReplicationManager.markLedgerUnderreplicated(lh2.getId(), + replicaToKillFromSecondLedger.toString()); + + while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh1 + .getId(), basePath)) { + Thread.sleep(100); + } + + while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh2 + .getId(), basePath)) { + Thread.sleep(100); + } + + killAllBookies(lh1, newBkAddr); + + // Should be able to read the entries from 0-9 + verifyRecoveredLedgers(lh1, 0, 9); + verifyRecoveredLedgers(lh2, 0, 9); + } finally { + rw.shutdown(); + } + + } + + /** + * Tests that ReplicationWorker should fence the ledger and release ledger + * lock after timeout. Then replication should happen normally. + */ + @Test + public void testRWShouldReplicateTheLedgersAfterTimeoutIfLastFragmentIsUR() + throws Exception { + LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, + TESTPASSWD); + + for (int i = 0; i < 10; i++) { + lh.addEntry(data); + } + BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + + LOG.info("Killing Bookie : {}", replicaToKill); + killBookie(replicaToKill); + + BookieId newBkAddr = startNewBookieAndReturnBookieId(); + LOG.info("New Bookie addr : {}", newBkAddr); + + // set to 3s instead of default 30s + baseConf.setOpenLedgerRereplicationGracePeriod("3000"); + ReplicationWorker rw = new ReplicationWorker(baseConf); + + @Cleanup MetadataClientDriver clientDriver = MetadataDrivers.getClientDriver( + URI.create(baseClientConf.getMetadataServiceUri())); + clientDriver.initialize(baseClientConf, scheduler, NullStatsLogger.INSTANCE, Optional.empty()); + + LedgerManagerFactory mFactory = clientDriver.getLedgerManagerFactory(); + + LedgerUnderreplicationManager underReplicationManager = mFactory + .newLedgerUnderreplicationManager(); + rw.start(); + try { + + underReplicationManager.markLedgerUnderreplicated(lh.getId(), + replicaToKill.toString()); + while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh + .getId(), basePath)) { + Thread.sleep(100); + } + killAllBookies(lh, newBkAddr); + // Should be able to read the entries from 0-9 + verifyRecoveredLedgers(lh, 0, 9); + lh = bkc.openLedgerNoRecovery(lh.getId(), + BookKeeper.DigestType.CRC32, TESTPASSWD); + assertFalse("Ledger must have been closed by RW", ClientUtil + .isLedgerOpen(lh)); + } finally { + rw.shutdown(); + underReplicationManager.close(); + } + + } + + @Test + public void testBookiesNotAvailableScenarioForReplicationWorker() throws Exception { + int ensembleSize = 3; + LedgerHandle lh = bkc.createLedger(ensembleSize, ensembleSize, BookKeeper.DigestType.CRC32, TESTPASSWD); + + int numOfEntries = 7; + for (int i = 0; i < numOfEntries; i++) { + lh.addEntry(data); + } + lh.close(); + + BookieId[] bookiesKilled = new BookieId[ensembleSize]; + ServerConfiguration[] killedBookiesConfig = new ServerConfiguration[ensembleSize]; + + // kill all bookies + for (int i = 0; i < ensembleSize; i++) { + bookiesKilled[i] = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(i); + killedBookiesConfig[i] = getBkConf(bookiesKilled[i]); + LOG.info("Killing Bookie : {}", bookiesKilled[i]); + killBookie(bookiesKilled[i]); + } + + // start new bookiesToKill number of bookies + for (int i = 0; i < ensembleSize; i++) { + BookieId newBkAddr = startNewBookieAndReturnBookieId(); + } + + // create couple of replicationworkers + ServerConfiguration newRWConf = new ServerConfiguration(baseConf); + newRWConf.setLockReleaseOfFailedLedgerGracePeriod("64"); + ReplicationWorker rw1 = new ReplicationWorker(newRWConf); + ReplicationWorker rw2 = new ReplicationWorker(newRWConf); + + @Cleanup + MetadataClientDriver clientDriver = MetadataDrivers + .getClientDriver(URI.create(baseClientConf.getMetadataServiceUri())); + clientDriver.initialize(baseClientConf, scheduler, NullStatsLogger.INSTANCE, Optional.empty()); + + LedgerManagerFactory mFactory = clientDriver.getLedgerManagerFactory(); + + LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager(); + try { + //mark ledger underreplicated + for (int i = 0; i < bookiesKilled.length; i++) { + underReplicationManager.markLedgerUnderreplicated(lh.getId(), bookiesKilled[i].toString()); + } + while (!ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath)) { + Thread.sleep(100); + } + rw1.start(); + rw2.start(); + + AtomicBoolean isBookieRestarted = new AtomicBoolean(false); + + (new Thread(new Runnable() { + @Override + public void run() { + try { + Thread.sleep(3000); + isBookieRestarted.set(true); + /* + * after sleeping for 3000 msecs, restart one of the + * bookie, so that replication can succeed. + */ + startBookie(killedBookiesConfig[0]); + } catch (Exception e) { + e.printStackTrace(); + } + } + })).start(); + + int rw1PrevFailedAttemptsCount = 0; + int rw2PrevFailedAttemptsCount = 0; + while (!isBookieRestarted.get()) { + /* + * since all the bookies containing the ledger entries are down + * replication wouldnt have succeeded. + */ + assertTrue("Ledger: " + lh.getId() + " should be underreplicated", + ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath)); + + // the number of failed attempts should have increased. + int rw1CurFailedAttemptsCount = rw1.replicationFailedLedgers.get(lh.getId()).get(); + assertTrue( + "The current number of failed attempts: " + rw1CurFailedAttemptsCount + + " should be greater than or equal to previous value: " + rw1PrevFailedAttemptsCount, + rw1CurFailedAttemptsCount >= rw1PrevFailedAttemptsCount); + rw1PrevFailedAttemptsCount = rw1CurFailedAttemptsCount; + + int rw2CurFailedAttemptsCount = rw2.replicationFailedLedgers.get(lh.getId()).get(); + assertTrue( + "The current number of failed attempts: " + rw2CurFailedAttemptsCount + + " should be greater than or equal to previous value: " + rw2PrevFailedAttemptsCount, + rw2CurFailedAttemptsCount >= rw2PrevFailedAttemptsCount); + rw2PrevFailedAttemptsCount = rw2CurFailedAttemptsCount; + + Thread.sleep(50); + } + + /** + * since one of the killed bookie is restarted, replicationworker + * should succeed in replicating this under replicated ledger and it + * shouldn't be under replicated anymore. + */ + int timeToWaitForReplicationToComplete = 20000; + int timeWaited = 0; + while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath)) { + Thread.sleep(100); + timeWaited += 100; + if (timeWaited == timeToWaitForReplicationToComplete) { + fail("Ledger should be replicated by now"); + } + } + + rw1PrevFailedAttemptsCount = rw1.replicationFailedLedgers.get(lh.getId()).get(); + rw2PrevFailedAttemptsCount = rw2.replicationFailedLedgers.get(lh.getId()).get(); + Thread.sleep(2000); + // now since the ledger is replicated, number of failed attempts + // counter shouldn't be increased even after sleeping for sometime. + assertEquals("rw1 failedattempts", rw1PrevFailedAttemptsCount, + rw1.replicationFailedLedgers.get(lh.getId()).get()); + assertEquals("rw2 failed attempts ", rw2PrevFailedAttemptsCount, + rw2.replicationFailedLedgers.get(lh.getId()).get()); + + /* + * Since these entries are eventually available, and replication has + * eventually succeeded, in one of the RW + * unableToReadEntriesForReplication should be 0. + */ + int rw1UnableToReadEntriesForReplication = rw1.unableToReadEntriesForReplication.get(lh.getId()).size(); + int rw2UnableToReadEntriesForReplication = rw2.unableToReadEntriesForReplication.get(lh.getId()).size(); + assertTrue( + "unableToReadEntriesForReplication in RW1: " + rw1UnableToReadEntriesForReplication + + " in RW2: " + + rw2UnableToReadEntriesForReplication, + (rw1UnableToReadEntriesForReplication == 0) + || (rw2UnableToReadEntriesForReplication == 0)); + } finally { + rw1.shutdown(); + rw2.shutdown(); + underReplicationManager.close(); + } + } + + class InjectedReplicationWorker extends ReplicationWorker { + CopyOnWriteArrayList delayReplicationPeriods; + + public InjectedReplicationWorker(ServerConfiguration conf, StatsLogger statsLogger, + CopyOnWriteArrayList delayReplicationPeriods) + throws CompatibilityException, ReplicationException.UnavailableException, + InterruptedException, IOException { + super(conf, statsLogger); + this.delayReplicationPeriods = delayReplicationPeriods; + } + + @Override + protected void scheduleTaskWithDelay(TimerTask timerTask, long delayPeriod) { + delayReplicationPeriods.add(delayPeriod); + super.scheduleTaskWithDelay(timerTask, delayPeriod); + } + } + + @Test + public void testDeferLedgerLockReleaseForReplicationWorker() throws Exception { + int ensembleSize = 3; + LedgerHandle lh = bkc.createLedger(ensembleSize, ensembleSize, BookKeeper.DigestType.CRC32, TESTPASSWD); + int numOfEntries = 7; + for (int i = 0; i < numOfEntries; i++) { + lh.addEntry(data); + } + lh.close(); + + BookieId[] bookiesKilled = new BookieId[ensembleSize]; + ServerConfiguration[] killedBookiesConfig = new ServerConfiguration[ensembleSize]; + + // kill all bookies + for (int i = 0; i < ensembleSize; i++) { + bookiesKilled[i] = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(i); + killedBookiesConfig[i] = getBkConf(bookiesKilled[i]); + LOG.info("Killing Bookie : {}", bookiesKilled[i]); + killBookie(bookiesKilled[i]); + } + + // start new bookiesToKill number of bookies + for (int i = 0; i < ensembleSize; i++) { + startNewBookieAndReturnBookieId(); + } + + // create couple of replicationworkers + long lockReleaseOfFailedLedgerGracePeriod = 64L; + long baseBackoffForLockReleaseOfFailedLedger = lockReleaseOfFailedLedgerGracePeriod + / (int) Math.pow(2, ReplicationWorker.NUM_OF_EXPONENTIAL_BACKOFF_RETRIALS); + ServerConfiguration newRWConf = new ServerConfiguration(baseConf); + newRWConf.setLockReleaseOfFailedLedgerGracePeriod(Long.toString(lockReleaseOfFailedLedgerGracePeriod)); + newRWConf.setRereplicationEntryBatchSize(1000); + CopyOnWriteArrayList rw1DelayReplicationPeriods = new CopyOnWriteArrayList(); + CopyOnWriteArrayList rw2DelayReplicationPeriods = new CopyOnWriteArrayList(); + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger1 = statsProvider.getStatsLogger("rw1"); + TestStatsLogger statsLogger2 = statsProvider.getStatsLogger("rw2"); + ReplicationWorker rw1 = new InjectedReplicationWorker(newRWConf, statsLogger1, rw1DelayReplicationPeriods); + ReplicationWorker rw2 = new InjectedReplicationWorker(newRWConf, statsLogger2, rw2DelayReplicationPeriods); + + Counter numEntriesUnableToReadForReplication1 = statsLogger1 + .getCounter(NUM_ENTRIES_UNABLE_TO_READ_FOR_REPLICATION); + Counter numEntriesUnableToReadForReplication2 = statsLogger2 + .getCounter(NUM_ENTRIES_UNABLE_TO_READ_FOR_REPLICATION); + @Cleanup + MetadataClientDriver clientDriver = MetadataDrivers + .getClientDriver(URI.create(baseClientConf.getMetadataServiceUri())); + clientDriver.initialize(baseClientConf, scheduler, NullStatsLogger.INSTANCE, Optional.empty()); + + LedgerManagerFactory mFactory = clientDriver.getLedgerManagerFactory(); + + LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager(); + try { + // mark ledger underreplicated + for (int i = 0; i < bookiesKilled.length; i++) { + underReplicationManager.markLedgerUnderreplicated(lh.getId(), bookiesKilled[i].toString()); + } + while (!ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath)) { + Thread.sleep(100); + } + rw1.start(); + rw2.start(); + + // wait for RWs to complete 'numOfAttemptsToWaitFor' failed attempts + int numOfAttemptsToWaitFor = 10; + while ((rw1.replicationFailedLedgers.get(lh.getId()).get() < numOfAttemptsToWaitFor) + || rw2.replicationFailedLedgers.get(lh.getId()).get() < numOfAttemptsToWaitFor) { + Thread.sleep(500); + } + + /* + * since all the bookies containing the ledger entries are down + * replication wouldn't have succeeded. + */ + assertTrue("Ledger: " + lh.getId() + " should be underreplicated", + ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath)); + + /* + * since RW failed 'numOfAttemptsToWaitFor' number of times, we + * should have atleast (numOfAttemptsToWaitFor - 1) + * delayReplicationPeriods and their value should be + * (lockReleaseOfFailedLedgerGracePeriod/16) , 2 * previous value,.. + * with max : lockReleaseOfFailedLedgerGracePeriod + */ + for (int i = 0; i < ((numOfAttemptsToWaitFor - 1)); i++) { + long expectedDelayValue = Math.min(lockReleaseOfFailedLedgerGracePeriod, + baseBackoffForLockReleaseOfFailedLedger * (1 << i)); + assertEquals("RW1 delayperiod", (Long) expectedDelayValue, rw1DelayReplicationPeriods.get(i)); + assertEquals("RW2 delayperiod", (Long) expectedDelayValue, rw2DelayReplicationPeriods.get(i)); + } + + /* + * RW wont try to replicate until and unless RW succeed in reading + * those failed entries before proceeding with replication of under + * replicated fragment, so the numEntriesUnableToReadForReplication + * should be just 'numOfEntries', though RW failed to replicate + * multiple times. + */ + assertEquals("numEntriesUnableToReadForReplication for RW1", Long.valueOf((long) numOfEntries), + numEntriesUnableToReadForReplication1.get()); + assertEquals("numEntriesUnableToReadForReplication for RW2", Long.valueOf((long) numOfEntries), + numEntriesUnableToReadForReplication2.get()); + + /* + * Since these entries are unavailable, + * unableToReadEntriesForReplication should be of size numOfEntries. + */ + assertEquals("RW1 unabletoreadentries", numOfEntries, + rw1.unableToReadEntriesForReplication.get(lh.getId()).size()); + assertEquals("RW2 unabletoreadentries", numOfEntries, + rw2.unableToReadEntriesForReplication.get(lh.getId()).size()); + } finally { + rw1.shutdown(); + rw2.shutdown(); + underReplicationManager.close(); + } + } + + /** + * Tests that ReplicationWorker should not have identified for postponing + * the replication if ledger is in open state and lastFragment is not in + * underReplication state. Note that RW should not fence such ledgers. + */ + @Test + public void testRWShouldReplicateTheLedgersAfterTimeoutIfLastFragmentIsNotUR() + throws Exception { + LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, + TESTPASSWD); + + for (int i = 0; i < 10; i++) { + lh.addEntry(data); + } + BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + + LOG.info("Killing Bookie : {}", replicaToKill); + killBookie(replicaToKill); + + BookieId newBkAddr = startNewBookieAndReturnBookieId(); + LOG.info("New Bookie addr : {}", newBkAddr); + + // Reform ensemble...Making sure that last fragment is not in + // under-replication + for (int i = 0; i < 10; i++) { + lh.addEntry(data); + } + + ReplicationWorker rw = new ReplicationWorker(baseConf); + + baseClientConf.setMetadataServiceUri(zkUtil.getMetadataServiceUri()); + + @Cleanup MetadataClientDriver driver = MetadataDrivers.getClientDriver( + URI.create(baseClientConf.getMetadataServiceUri())); + driver.initialize(baseClientConf, scheduler, NullStatsLogger.INSTANCE, Optional.empty()); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + + LedgerUnderreplicationManager underReplicationManager = mFactory + .newLedgerUnderreplicationManager(); + + rw.start(); + try { + + underReplicationManager.markLedgerUnderreplicated(lh.getId(), + replicaToKill.toString()); + while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh + .getId(), basePath)) { + Thread.sleep(100); + } + + killAllBookies(lh, newBkAddr); + + // Should be able to read the entries from 0-9 + verifyRecoveredLedgers(lh, 0, 9); + lh = bkc.openLedgerNoRecovery(lh.getId(), + BookKeeper.DigestType.CRC32, TESTPASSWD); + + // Ledger should be still in open state + assertTrue("Ledger must have been closed by RW", ClientUtil + .isLedgerOpen(lh)); + } finally { + rw.shutdown(); + underReplicationManager.close(); + } + + } + + /** + * Test that the replication worker will not shutdown on a simple ZK disconnection. + */ + @Test + public void testRWZKConnectionLost() throws Exception { + try (ZooKeeperClient zk = ZooKeeperClient.newBuilder() + .connectString(zkUtil.getZooKeeperConnectString()) + .sessionTimeoutMs(10000) + .build()) { + + ReplicationWorker rw = new ReplicationWorker(baseConf); + rw.start(); + for (int i = 0; i < 10; i++) { + if (rw.isRunning()) { + break; + } + Thread.sleep(1000); + } + assertTrue("Replication worker should be running", rw.isRunning()); + + stopZKCluster(); + // ZK is down for shorter period than reconnect timeout + Thread.sleep(1000); + startZKCluster(); + + assertTrue("Replication worker should not shutdown", rw.isRunning()); + } + } + + /** + * Test that the replication worker shuts down on non-recoverable ZK connection loss. + */ + @Test + public void testRWZKConnectionLostOnNonRecoverableZkError() throws Exception { + for (int j = 0; j < 3; j++) { + LedgerHandle lh = bkc.createLedger(1, 1, 1, + BookKeeper.DigestType.CRC32, TESTPASSWD, + null); + final long createdLedgerId = lh.getId(); + for (int i = 0; i < 10; i++) { + lh.addEntry(data); + } + lh.close(); + } + + killBookie(2); + killBookie(1); + startNewBookie(); + startNewBookie(); + + servers.get(0).getConfiguration().setRwRereplicateBackoffMs(100); + servers.get(0).startAutoRecovery(); + + Auditor auditor = getAuditor(10, TimeUnit.SECONDS); + ReplicationWorker rw = servers.get(0).getReplicationWorker(); + + ZkLedgerUnderreplicationManager ledgerUnderreplicationManager = + (ZkLedgerUnderreplicationManager) FieldUtils.readField(auditor, + "ledgerUnderreplicationManager", true); + + ZooKeeper zkc = (ZooKeeper) FieldUtils.readField(ledgerUnderreplicationManager, "zkc", true); + auditor.submitAuditTask().get(); + + assertTrue(zkc.getState().isConnected()); + zkc.close(); + assertFalse(zkc.getState().isConnected()); + + auditor.submitAuditTask(); + rw.run(); + + for (int i = 0; i < 10; i++) { + if (!rw.isRunning() && !auditor.isRunning()) { + break; + } + Thread.sleep(1000); + } + assertFalse("Replication worker should NOT be running", rw.isRunning()); + assertFalse("Auditor should NOT be running", auditor.isRunning()); + } + + private void killAllBookies(LedgerHandle lh, BookieId excludeBK) + throws Exception { + // Killing all bookies except newly replicated bookie + for (Entry> entry : + lh.getLedgerMetadata().getAllEnsembles().entrySet()) { + List bookies = entry.getValue(); + for (BookieId bookie : bookies) { + if (bookie.equals(excludeBK)) { + continue; + } + killBookie(bookie); + } + } + } + + private void verifyRecoveredLedgers(LedgerHandle lh, long startEntryId, + long endEntryId) throws BKException, InterruptedException { + LedgerHandle lhs = bkc.openLedgerNoRecovery(lh.getId(), + BookKeeper.DigestType.CRC32, TESTPASSWD); + Enumeration entries = lhs.readEntries(startEntryId, + endEntryId); + assertTrue("Should have the elements", entries.hasMoreElements()); + while (entries.hasMoreElements()) { + LedgerEntry entry = entries.nextElement(); + assertEquals("TestReplicationWorker", new String(entry.getEntry())); + } + } + + @Test + public void testReplicateEmptyOpenStateLedger() throws Exception { + LedgerHandle lh = bkc.createLedger(3, 3, 2, BookKeeper.DigestType.CRC32, TESTPASSWD); + assertFalse(lh.getLedgerMetadata().isClosed()); + + List firstEnsemble = lh.getLedgerMetadata().getAllEnsembles().firstEntry().getValue(); + List ensemble = lh.getLedgerMetadata().getAllEnsembles().entrySet().iterator().next().getValue(); + killBookie(ensemble.get(1)); + + startNewBookie(); + baseConf.setOpenLedgerRereplicationGracePeriod(String.valueOf(30)); + ReplicationWorker replicationWorker = new ReplicationWorker(baseConf); + replicationWorker.start(); + + try { + underReplicationManager.markLedgerUnderreplicated(lh.getId(), ensemble.get(1).toString()); + Awaitility.waitAtMost(60, TimeUnit.SECONDS).untilAsserted(() -> + assertFalse(ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath)) + ); + + LedgerHandle lh1 = bkc.openLedgerNoRecovery(lh.getId(), BookKeeper.DigestType.CRC32, TESTPASSWD); + assertTrue(lh1.getLedgerMetadata().isClosed()); + } finally { + replicationWorker.shutdown(); + } + } + + @Test + public void testRepairedNotAdheringPlacementPolicyLedgerFragmentsOnRack() throws Exception { + testRepairedNotAdheringPlacementPolicyLedgerFragments(RackawareEnsemblePlacementPolicy.class, null); + } + + @Test + public void testReplicationStats() throws Exception { + BiConsumer checkReplicationStats = (first, rw) -> { + try { + final Method rereplicate = rw.getClass().getDeclaredMethod("rereplicate"); + rereplicate.setAccessible(true); + final Object result = rereplicate.invoke(rw); + final Field statsLoggerField = rw.getClass().getDeclaredField("statsLogger"); + statsLoggerField.setAccessible(true); + final TestStatsLogger statsLogger = (TestStatsLogger) statsLoggerField.get(rw); + + final Counter numDeferLedgerLockReleaseOfFailedLedgerCounter = + statsLogger.getCounter(ReplicationStats.NUM_DEFER_LEDGER_LOCK_RELEASE_OF_FAILED_LEDGER); + final Counter numLedgersReplicatedCounter = + statsLogger.getCounter(ReplicationStats.NUM_FULL_OR_PARTIAL_LEDGERS_REPLICATED); + final Counter numNotAdheringPlacementLedgersCounter = statsLogger + .getCounter(ReplicationStats.NUM_NOT_ADHERING_PLACEMENT_LEDGERS_REPLICATED); + + assertEquals("NUM_DEFER_LEDGER_LOCK_RELEASE_OF_FAILED_LEDGER", + 1, numDeferLedgerLockReleaseOfFailedLedgerCounter.get().longValue()); + + if (first) { + assertFalse((boolean) result); + assertEquals("NUM_FULL_OR_PARTIAL_LEDGERS_REPLICATED", + 0, numLedgersReplicatedCounter.get().longValue()); + assertEquals("NUM_NOT_ADHERING_PLACEMENT_LEDGERS_REPLICATED", + 0, numNotAdheringPlacementLedgersCounter.get().longValue()); + + } else { + assertTrue((boolean) result); + assertEquals("NUM_FULL_OR_PARTIAL_LEDGERS_REPLICATED", + 1, numLedgersReplicatedCounter.get().longValue()); + assertEquals("NUM_NOT_ADHERING_PLACEMENT_LEDGERS_REPLICATED", + 1, numNotAdheringPlacementLedgersCounter.get().longValue()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }; + testRepairedNotAdheringPlacementPolicyLedgerFragments( + RackawareEnsemblePlacementPolicy.class, checkReplicationStats); + } + + private void testRepairedNotAdheringPlacementPolicyLedgerFragments( + Class placementPolicyClass, + BiConsumer checkReplicationStats) throws Exception { + List firstThreeBookies = servers.stream().map(ele -> { + try { + return ele.getServer().getBookieId(); + } catch (UnknownHostException e) { + return null; + } + }).filter(Objects::nonNull).collect(Collectors.toList()); + + baseClientConf.setProperty("reppDnsResolverClass", StaticDNSResolver.class.getName()); + baseClientConf.setProperty("enforceStrictZoneawarePlacement", false); + bkc.close(); + bkc = new BookKeeperTestClient(baseClientConf) { + @Override + protected EnsemblePlacementPolicy initializeEnsemblePlacementPolicy(ClientConfiguration conf, + DNSToSwitchMapping dnsResolver, + HashedWheelTimer timer, + FeatureProvider featureProvider, + StatsLogger statsLogger, + BookieAddressResolver bookieAddressResolver) + throws IOException { + EnsemblePlacementPolicy ensemblePlacementPolicy = null; + if (ZoneawareEnsemblePlacementPolicy.class == placementPolicyClass) { + ensemblePlacementPolicy = buildZoneAwareEnsemblePlacementPolicy(firstThreeBookies); + } else if (RackawareEnsemblePlacementPolicy.class == placementPolicyClass) { + ensemblePlacementPolicy = buildRackAwareEnsemblePlacementPolicy(firstThreeBookies); + } + ensemblePlacementPolicy.initialize(conf, Optional.ofNullable(dnsResolver), timer, + featureProvider, statsLogger, bookieAddressResolver); + return ensemblePlacementPolicy; + } + }; + + //This ledger not adhering placement policy, the combine(0,1,2) rack is 1. + LedgerHandle lh = bkc.createLedger(3, 3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD); + + int entrySize = 10; + for (int i = 0; i < entrySize; i++) { + lh.addEntry(data); + } + lh.close(); + + int minNumRacksPerWriteQuorumConfValue = 2; + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue); + servConf.setProperty("reppDnsResolverClass", StaticDNSResolver.class.getName()); + servConf.setAuditorPeriodicPlacementPolicyCheckInterval(1000); + servConf.setRepairedPlacementPolicyNotAdheringBookieEnable(true); + + MutableObject auditorRef = new MutableObject(); + try { + TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef); + Gauge ledgersNotAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY guage value", + 1, ledgersNotAdheringToPlacementPolicyGuage.getSample()); + Gauge ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY guage value", + 0, ledgersSoftlyAdheringToPlacementPolicyGuage.getSample()); + } finally { + Auditor auditor = auditorRef.getValue(); + if (auditor != null) { + auditor.close(); + } + } + + ZooKeeper zk = getZk((PulsarMetadataClientDriver) bkc.getMetadataClientDriver()); + + + Stat stat = zk.exists("/ledgers/underreplication/ledgers/0000/0000/0000/0000/urL0000000000", false); + assertNotNull(stat); + + baseConf.setRepairedPlacementPolicyNotAdheringBookieEnable(true); + BookKeeper bookKeeper = new BookKeeperTestClient(baseClientConf) { + @Override + protected EnsemblePlacementPolicy initializeEnsemblePlacementPolicy(ClientConfiguration conf, + DNSToSwitchMapping dnsResolver, + HashedWheelTimer timer, + FeatureProvider featureProvider, + StatsLogger statsLogger, + BookieAddressResolver bookieAddressResolver) + throws IOException { + EnsemblePlacementPolicy ensemblePlacementPolicy = null; + if (ZoneawareEnsemblePlacementPolicy.class == placementPolicyClass) { + ensemblePlacementPolicy = buildZoneAwareEnsemblePlacementPolicy(firstThreeBookies); + } else if (RackawareEnsemblePlacementPolicy.class == placementPolicyClass) { + ensemblePlacementPolicy = buildRackAwareEnsemblePlacementPolicy(firstThreeBookies); + } + ensemblePlacementPolicy.initialize(conf, Optional.ofNullable(dnsResolver), timer, + featureProvider, statsLogger, bookieAddressResolver); + return ensemblePlacementPolicy; + } + }; + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(REPLICATION_SCOPE); + ReplicationWorker rw = new ReplicationWorker(baseConf, bookKeeper, false, statsLogger); + + if (checkReplicationStats != null) { + checkReplicationStats.accept(true, rw); + } else { + rw.start(); + } + + //start new bookie, the rack is /rack2 + BookieId newBookieId = startNewBookieAndReturnBookieId(); + + if (checkReplicationStats != null) { + checkReplicationStats.accept(false, rw); + } + + Awaitility.await().untilAsserted(() -> { + LedgerMetadata metadata = bkc.getLedgerManager().readLedgerMetadata(lh.getId()).get().getValue(); + List newBookies = metadata.getAllEnsembles().get(0L); + assertTrue(newBookies.contains(newBookieId)); + }); + + Awaitility.await().untilAsserted(() -> { + Stat stat1 = zk.exists("/ledgers/underreplication/ledgers/0000/0000/0000/0000/urL0000000000", false); + assertNull(stat1); + }); + + for (BookieId rack1Book : firstThreeBookies) { + killBookie(rack1Book); + } + + verifyRecoveredLedgers(lh, 0, entrySize - 1); + + if (checkReplicationStats == null) { + rw.shutdown(); + } + baseConf.setRepairedPlacementPolicyNotAdheringBookieEnable(false); + bookKeeper.close(); + } + + private EnsemblePlacementPolicy buildRackAwareEnsemblePlacementPolicy(List bookieIds) { + return new RackawareEnsemblePlacementPolicy() { + @Override + public String resolveNetworkLocation(BookieId addr) { + if (bookieIds.contains(addr)) { + return "/rack1"; + } + //The other bookie is /rack2 + return "/rack2"; + } + }; + } + + private EnsemblePlacementPolicy buildZoneAwareEnsemblePlacementPolicy(List firstThreeBookies) { + return new ZoneawareEnsemblePlacementPolicy() { + @Override + protected String resolveNetworkLocation(BookieId addr) { + //The first three bookie 1 is /zone1/ud1 + //The first three bookie 2,3 is /zone1/ud2 + if (firstThreeBookies.get(0).equals(addr)) { + return "/zone1/ud1"; + } else if (firstThreeBookies.contains(addr)) { + return "/zone1/ud2"; + } + //The other bookie is /zone2/ud1 + return "/zone2/ud1"; + } + }; + } + + private TestStatsLogger startAuditorAndWaitForPlacementPolicyCheck(ServerConfiguration servConf, + MutableObject auditorRef) + throws MetadataException, CompatibilityException, KeeperException, + InterruptedException, ReplicationException.UnavailableException, UnknownHostException { + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager(); + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + TestStatsProvider.TestOpStatsLogger placementPolicyCheckStatsLogger = + (TestStatsProvider.TestOpStatsLogger) statsLogger + .getOpStatsLogger(ReplicationStats.PLACEMENT_POLICY_CHECK_TIME); + + final AuditorPeriodicCheckTest.TestAuditor auditor = new AuditorPeriodicCheckTest.TestAuditor( + BookieImpl.getBookieId(servConf).toString(), servConf, bkc, false, statsLogger, null); + auditorRef.setValue(auditor); + CountDownLatch latch = auditor.getLatch(); + assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 0, + placementPolicyCheckStatsLogger.getSuccessCount()); + urm.setPlacementPolicyCheckCTime(-1); + auditor.start(); + /* + * since placementPolicyCheckCTime is set to -1, placementPolicyCheck should be + * scheduled to run with no initialdelay + */ + assertTrue("placementPolicyCheck should have executed", latch.await(20, TimeUnit.SECONDS)); + for (int i = 0; i < 20; i++) { + Thread.sleep(100); + if (placementPolicyCheckStatsLogger.getSuccessCount() >= 1) { + break; + } + } + assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 1, + placementPolicyCheckStatsLogger.getSuccessCount()); + return statsLogger; + } + + private ZooKeeper getZk(PulsarMetadataClientDriver pulsarMetadataClientDriver) throws Exception { + PulsarLedgerManagerFactory pulsarLedgerManagerFactory = + (PulsarLedgerManagerFactory) pulsarMetadataClientDriver.getLedgerManagerFactory(); + Field field = pulsarLedgerManagerFactory.getClass().getDeclaredField("store"); + field.setAccessible(true); + ZKMetadataStore zkMetadataStore = (ZKMetadataStore) field.get(pulsarLedgerManagerFactory); + return zkMetadataStore.getZkClient(); + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/ZooKeeperUtil.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/ZooKeeperUtil.java new file mode 100644 index 0000000000000..5113edb72c49a --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/ZooKeeperUtil.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * This file is derived from ZooKeeperUtil from Apache BookKeeper + * http://bookkeeper.apache.org + */ + +package org.apache.bookkeeper.replication; + +import static org.testng.Assert.assertTrue; +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.apache.bookkeeper.test.ZooKeeperCluster; +import org.apache.bookkeeper.util.IOUtils; +import org.apache.bookkeeper.util.ZkUtils; +import org.apache.bookkeeper.zookeeper.ZooKeeperClient; +import org.apache.commons.io.FileUtils; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.server.NIOServerCnxnFactory; +import org.apache.zookeeper.server.ZooKeeperServer; +import org.apache.zookeeper.test.ClientBase; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Test the zookeeper utilities. + */ +public class ZooKeeperUtil implements ZooKeeperCluster { + + static { + // org.apache.zookeeper.test.ClientBase uses FourLetterWordMain, from 3.5.3 four letter words + // are disabled by default due to security reasons + System.setProperty("zookeeper.4lw.commands.whitelist", "*"); + } + static final Logger LOG = LoggerFactory.getLogger(ZooKeeperUtil.class); + + // ZooKeeper related variables + protected Integer zooKeeperPort = 0; + private InetSocketAddress zkaddr; + + protected ZooKeeperServer zks; + protected ZooKeeper zkc; // zookeeper client + protected NIOServerCnxnFactory serverFactory; + protected File zkTmpDir; + private String connectString; + private String ledgersRootPath; + + public ZooKeeperUtil(String ledgersRootPath) { + this.ledgersRootPath = ledgersRootPath; + String loopbackIPAddr = InetAddress.getLoopbackAddress().getHostAddress(); + zkaddr = new InetSocketAddress(loopbackIPAddr, 0); + connectString = loopbackIPAddr + ":" + zooKeeperPort; + } + + @Override + public ZooKeeper getZooKeeperClient() { + return zkc; + } + + @Override + public String getZooKeeperConnectString() { + return connectString; + } + + @Override + public String getMetadataServiceUri() { + return getMetadataServiceUri("/ledgers"); + } + + @Override + public String getMetadataServiceUri(String zkLedgersRootPath) { + return "zk://" + connectString + zkLedgersRootPath; + } + + @Override + public String getMetadataServiceUri(String zkLedgersRootPath, String type) { + return "zk+" + type + "://" + connectString + zkLedgersRootPath; + } + + @Override + public void startCluster() throws Exception { + // create a ZooKeeper server(dataDir, dataLogDir, port) + LOG.debug("Running ZK server"); + ClientBase.setupTestEnv(); + zkTmpDir = IOUtils.createTempDir("zookeeper", "test"); + + // start the server and client. + restartCluster(); + + // create default bk ensemble + createBKEnsemble(ledgersRootPath); + } + + @Override + public void createBKEnsemble(String ledgersPath) throws KeeperException, InterruptedException { + int last = ledgersPath.lastIndexOf('/'); + if (last > 0) { + String pathToCreate = ledgersPath.substring(0, last); + CompletableFuture future = new CompletableFuture<>(); + if (zkc.exists(pathToCreate, false) == null) { + ZkUtils.asyncCreateFullPathOptimistic(zkc, + pathToCreate, + new byte[0], + ZooDefs.Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT, (i, s, o, s1) -> { + future.complete(null); + }, null); + } + future.join(); + } + + ZooKeeperCluster.super.createBKEnsemble(ledgersPath); + } + @Override + public void restartCluster() throws Exception { + zks = new ZooKeeperServer(zkTmpDir, zkTmpDir, + ZooKeeperServer.DEFAULT_TICK_TIME); + serverFactory = new NIOServerCnxnFactory(); + serverFactory.configure(zkaddr, 100); + serverFactory.startup(zks); + + if (0 == zooKeeperPort) { + zooKeeperPort = serverFactory.getLocalPort(); + zkaddr = new InetSocketAddress(zkaddr.getAddress().getHostAddress(), zooKeeperPort); + connectString = zkaddr.getAddress().getHostAddress() + ":" + zooKeeperPort; + } + + boolean b = ClientBase.waitForServerUp(getZooKeeperConnectString(), + ClientBase.CONNECTION_TIMEOUT); + LOG.debug("Server up: " + b); + + // create a zookeeper client + LOG.debug("Instantiate ZK Client"); + zkc = ZooKeeperClient.newBuilder() + .connectString(getZooKeeperConnectString()) + .sessionTimeoutMs(10000) + .build(); + } + + @Override + public void sleepCluster(final int time, + final TimeUnit timeUnit, + final CountDownLatch l) + throws InterruptedException, IOException { + Thread[] allthreads = new Thread[Thread.activeCount()]; + Thread.enumerate(allthreads); + for (final Thread t : allthreads) { + if (t.getName().contains("SyncThread:0")) { + Thread sleeper = new Thread() { + @SuppressWarnings("deprecation") + public void run() { + try { + t.suspend(); + l.countDown(); + timeUnit.sleep(time); + t.resume(); + } catch (Exception e) { + LOG.error("Error suspending thread", e); + } + } + }; + sleeper.start(); + return; + } + } + throw new IOException("ZooKeeper thread not found"); + } + + @Override + public void stopCluster() throws Exception { + if (zkc != null) { + zkc.close(); + } + + // shutdown ZK server + if (serverFactory != null) { + serverFactory.shutdown(); + assertTrue(ClientBase.waitForServerDown(getZooKeeperConnectString(), ClientBase.CONNECTION_TIMEOUT), + "waiting for server down"); + } + if (zks != null) { + zks.getTxnLogFactory().close(); + } + } + + @Override + public void killCluster() throws Exception { + stopCluster(); + FileUtils.deleteDirectory(zkTmpDir); + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java index ec6e6e03eae71..d0265e3ca44ee 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java @@ -22,6 +22,7 @@ import static org.testng.Assert.assertTrue; import io.etcd.jetcd.launcher.EtcdCluster; import io.etcd.jetcd.test.EtcdClusterExtension; +import io.streamnative.oxia.testcontainers.OxiaContainer; import java.io.File; import java.net.URI; import java.util.UUID; @@ -39,6 +40,8 @@ public abstract class BaseMetadataStoreTest extends TestRetrySupport { protected TestZKServer zks; protected EtcdCluster etcdCluster; + protected OxiaContainer oxiaServer; + @BeforeClass(alwaysRun = true) @Override public void setup() throws Exception { @@ -59,6 +62,11 @@ public void cleanup() throws Exception { etcdCluster.close(); etcdCluster = null; } + + if (oxiaServer != null) { + oxiaServer.close(); + oxiaServer = null; + } } private static String createTempFolder() { @@ -79,9 +87,27 @@ public Object[][] implementations() { {"Memory", stringSupplier(() -> "memory:" + UUID.randomUUID())}, {"RocksDB", stringSupplier(() -> "rocksdb:" + createTempFolder())}, {"Etcd", stringSupplier(() -> "etcd:" + getEtcdClusterConnectString())}, + {"Oxia", stringSupplier(() -> "oxia://" + getOxiaServerConnectString())}, + }; + } + + @DataProvider(name = "distributedImpl") + public Object[][] distributedImplementations() { + return new Object[][]{ + {"ZooKeeper", stringSupplier(() -> zks.getConnectionString())}, + {"Etcd", stringSupplier(() -> "etcd:" + getEtcdClusterConnectString())}, + {"Oxia", stringSupplier(() -> "oxia://" + getOxiaServerConnectString())}, }; } + protected synchronized String getOxiaServerConnectString() { + if (oxiaServer == null) { + oxiaServer = new OxiaContainer(OxiaContainer.DEFAULT_IMAGE_NAME); + oxiaServer.start(); + } + return oxiaServer.getServiceAddress(); + } + private synchronized String getEtcdClusterConnectString() { if (etcdCluster == null) { etcdCluster = EtcdClusterExtension.builder().withClusterName("test").withNodes(1).withSsl(false).build() @@ -119,10 +145,11 @@ public static void assertEqualsAndRetry(Supplier actual, int retryCount, long intSleepTimeInMillis) throws Exception { assertTrue(retryStrategically((__) -> { - if (actual.get().equals(expectedAndRetry)) { + Object actualObject = actual.get(); + if (actualObject.equals(expectedAndRetry)) { return false; } - assertEquals(actual.get(), expected); + assertEquals(actualObject, expected); return true; }, retryCount, intSleepTimeInMillis)); } diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LeaderElectionTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LeaderElectionTest.java index 6b4f74a30b563..4b48f3c20b02b 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LeaderElectionTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LeaderElectionTest.java @@ -69,6 +69,8 @@ public void basicTest(String provider, Supplier urlSupplier) throws Exce leaderElection.close(); + assertEquals(leaderElection.getState(), LeaderElectionState.NoLeader); + assertEquals(cache.get("/my/leader-election").join(), Optional.empty()); } diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LockManagerTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LockManagerTest.java index 05e6d4a3845e2..ebd60bad5507d 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LockManagerTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LockManagerTest.java @@ -35,6 +35,7 @@ import java.util.function.Supplier; import lombok.Cleanup; import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.apache.pulsar.metadata.api.GetResult; import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.MetadataStoreException.LockBusyException; @@ -352,4 +353,34 @@ public void testCleanUpStateWhenRevalidationGotLockBusy(String provider, Supplie } }); } + + @Test(dataProvider = "impl") + public void lockDeletedAndReacquired(String provider, Supplier urlSupplier) throws Exception { + @Cleanup + MetadataStoreExtended store = MetadataStoreExtended.create(urlSupplier.get(), + MetadataStoreConfig.builder().fsyncEnable(false).build()); + + MetadataCache cache = store.getMetadataCache(String.class); + + @Cleanup + CoordinationService coordinationService = new CoordinationServiceImpl(store); + + @Cleanup + LockManager lockManager = coordinationService.getLockManager(String.class); + + String key = newKey(); + ResourceLock lock = lockManager.acquireLock(key, "lock").join(); + assertEquals(lock.getValue(), "lock"); + var res = cache.get(key).join(); + assertTrue(res.isPresent()); + assertEquals(res.get(), "lock"); + + store.delete(key, Optional.empty()).join(); + + Awaitility.await().untilAsserted(() -> { + Optional val = store.get(key).join(); + assertTrue(val.isPresent()); + assertFalse(lock.getLockExpiredFuture().isDone()); + }); + } } diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataBenchmark.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataBenchmark.java index 227c0e2c9dc35..b3b95ddc58076 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataBenchmark.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataBenchmark.java @@ -34,7 +34,7 @@ import org.testng.annotations.Test; @Slf4j -public class MetadataBenchmark extends MetadataStoreTest { +public class MetadataBenchmark extends BaseMetadataStoreTest { @Test(dataProvider = "impl", enabled = false) public void testGet(String provider, Supplier urlSupplier) throws Exception { diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java index 0c30b238049c0..6992c69b7252e 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -46,6 +47,7 @@ import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.metadata.api.CacheGetResult; import org.apache.pulsar.metadata.api.MetadataCache; +import org.apache.pulsar.metadata.api.MetadataCacheConfig; import org.apache.pulsar.metadata.api.MetadataSerde; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.MetadataStoreConfig; @@ -55,6 +57,7 @@ import org.apache.pulsar.metadata.api.MetadataStoreFactory; import org.apache.pulsar.metadata.api.NotificationType; import org.apache.pulsar.metadata.api.Stat; +import org.apache.pulsar.metadata.api.extended.CreateOption; import org.apache.pulsar.metadata.cache.impl.MetadataCacheImpl; import org.awaitility.Awaitility; import org.testng.annotations.DataProvider; @@ -486,6 +489,7 @@ public void readModifyUpdateBadVersionRetry() throws Exception { String url = zks.getConnectionString(); @Cleanup MetadataStore sourceStore1 = MetadataStoreFactory.create(url, MetadataStoreConfig.builder().build()); + @Cleanup MetadataStore sourceStore2 = MetadataStoreFactory.create(url, MetadataStoreConfig.builder().build()); MetadataCache objCache1 = sourceStore1.getMetadataCache(MyClass.class); @@ -596,4 +600,51 @@ public CustomClass deserialize(String path, byte[] content, Stat stat) throws IO assertEquals(res.getValue().b, 2); assertEquals(res.getValue().path, key1); } + + @Test(dataProvider = "distributedImpl") + public void testPut(String provider, Supplier urlSupplier) throws Exception { + @Cleanup final var store1 = MetadataStoreFactory.create(urlSupplier.get(), MetadataStoreConfig.builder() + .build()); + final var cache1 = store1.getMetadataCache(Integer.class); + @Cleanup final var store2 = MetadataStoreFactory.create(urlSupplier.get(), MetadataStoreConfig.builder() + .build()); + final var cache2 = store2.getMetadataCache(Integer.class); + final var key = "/testPut"; + + cache1.put(key, 1, EnumSet.of(CreateOption.Ephemeral)); // create + Awaitility.await().untilAsserted(() -> { + assertEquals(cache1.get(key).get().orElse(-1), 1); + assertEquals(cache2.get(key).get().orElse(-1), 1); + }); + + cache2.put(key, 2, EnumSet.of(CreateOption.Ephemeral)); // update + Awaitility.await().untilAsserted(() -> { + assertEquals(cache1.get(key).get().orElse(-1), 2); + assertEquals(cache2.get(key).get().orElse(-1), 2); + }); + } + + @Test(dataProvider = "impl") + public void testAsyncReloadConsumer(String provider, Supplier urlSupplier) throws Exception { + @Cleanup + MetadataStore store = MetadataStoreFactory.create(urlSupplier.get(), MetadataStoreConfig.builder().build()); + + List refreshed = new ArrayList<>(); + MetadataCache objCache = store.getMetadataCache(MyClass.class, + MetadataCacheConfig.builder().refreshAfterWriteMillis(100) + .asyncReloadConsumer((k, v) -> v.map(vv -> refreshed.add(vv.getValue()))).build()); + + String key1 = newKey(); + + MyClass value1 = new MyClass("a", 1); + objCache.create(key1, value1); + + MyClass value2 = new MyClass("a", 2); + store.put(key1, ObjectMapperFactory.getMapper().writer().writeValueAsBytes(value2), Optional.empty()) + .join(); + + Awaitility.await().untilAsserted(() -> { + refreshed.contains(value2); + }); + } } diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTableViewTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTableViewTest.java new file mode 100644 index 0000000000000..5a2ea32890dbd --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTableViewTest.java @@ -0,0 +1,499 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.metadata; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import com.fasterxml.jackson.databind.type.TypeFactory; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import lombok.Cleanup; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.metadata.api.MetadataStore; +import org.apache.pulsar.metadata.api.MetadataStoreConfig; +import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.metadata.api.MetadataStoreException.NotFoundException; +import org.apache.pulsar.metadata.cache.impl.JSONMetadataSerdeSimpleType; +import org.apache.pulsar.metadata.impl.MetadataStoreFactoryImpl; +import org.apache.pulsar.metadata.tableview.impl.MetadataStoreTableViewImpl; +import org.awaitility.Awaitility; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Slf4j +public class MetadataStoreTableViewTest extends BaseMetadataStoreTest { + + LinkedBlockingDeque> tails; + LinkedBlockingDeque> existings; + + @BeforeMethod + void init(){ + tails = new LinkedBlockingDeque<>(); + existings = new LinkedBlockingDeque<>(); + } + + private void tailListener(String k, Integer v){ + tails.add(Pair.of(k, v)); + } + + private void existingListener(String k, Integer v){ + existings.add(Pair.of(k, v)); + } + + MetadataStoreTableViewImpl createTestTableView(MetadataStore store, String prefix, + Supplier urlSupplier) + throws Exception { + var tv = MetadataStoreTableViewImpl.builder() + .name("test") + .clazz(Integer.class) + .store(store) + .pathPrefix(prefix) + .conflictResolver((old, cur) -> { + if (old == null || cur == null) { + return true; + } + return old < cur; + }) + .listenPathValidator((path) -> path.startsWith(prefix) && path.contains("my")) + .tailItemListeners(List.of(this::tailListener)) + .existingItemListeners(List.of(this::existingListener)) + .timeoutInMillis(5_000) + .build(); + tv.start(); + return tv; + } + + private void assertGet(MetadataStoreTableViewImpl tv, String path, Integer expected) { + assertEquals(tv.get(path), expected); + //Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> assertEquals(tv.get(path), expected)); + } + + + @Test(dataProvider = "impl") + public void emptyTableViewTest(String provider, Supplier urlSupplier) throws Exception { + String prefix = newKey(); + @Cleanup + MetadataStore store = MetadataStoreFactoryImpl.create(urlSupplier.get(), MetadataStoreConfig.builder().build()); + MetadataStoreTableViewImpl tv = createTestTableView(store, prefix, urlSupplier); + + assertFalse(tv.exists("non-existing-key")); + assertFalse(tv.exists("non-existing-key/child")); + assertNull(tv.get("non-existing-key")); + assertNull(tv.get("non-existing-key/child")); + + try { + tv.delete("non-existing-key").join(); + fail("Should have failed"); + } catch (CompletionException e) { + assertException(e, NotFoundException.class); + } + + } + + @Test(dataProvider = "impl") + public void concurrentPutTest(String provider, Supplier urlSupplier) throws Exception { + String prefix = newKey(); + @Cleanup + MetadataStore store = MetadataStoreFactoryImpl.create(urlSupplier.get(), MetadataStoreConfig.builder().build()); + MetadataStoreTableViewImpl tv = createTestTableView(store, prefix, urlSupplier); + + int data = 1; + String path = "my"; + int concurrent = 50; + List> futureList = new ArrayList<>(); + for (int i = 0; i < concurrent; i++) { + futureList.add(tv.put(path, data).exceptionally(ex -> { + if (!(ex.getCause() instanceof MetadataStoreTableViewImpl.ConflictException)) { + fail("fail to execute concurrent put", ex); + } + return null; + })); + } + FutureUtil.waitForAll(futureList).join(); + + assertGet(tv, path, data); + } + + @Test(dataProvider = "impl") + public void conflictResolverTest(String provider, Supplier urlSupplier) throws Exception { + String prefix = newKey(); + @Cleanup + MetadataStore store = MetadataStoreFactoryImpl.create(urlSupplier.get(), MetadataStoreConfig.builder().build()); + MetadataStoreTableViewImpl tv = createTestTableView(store, prefix, urlSupplier); + + String key1 = "my"; + + tv.put(key1, 0).join(); + tv.put(key1, 0).exceptionally(ex -> { + if (!(ex.getCause() instanceof MetadataStoreTableViewImpl.ConflictException)) { + fail("fail to execute concurrent put", ex); + } + return null; + }).join(); + assertGet(tv, key1, 0); + tv.put(key1, 1).join(); + assertGet(tv, key1, 1); + tv.put(key1, 0).exceptionally(ex -> { + if (!(ex.getCause() instanceof MetadataStoreTableViewImpl.ConflictException)) { + fail("fail to execute concurrent put", ex); + } + return null; + }).join(); + assertGet(tv, key1, 1); + } + + @Test(dataProvider = "impl") + public void deleteTest(String provider, Supplier urlSupplier) throws Exception { + String prefix = newKey(); + @Cleanup + MetadataStore store = MetadataStoreFactoryImpl.create(urlSupplier.get(), MetadataStoreConfig.builder().build()); + MetadataStoreTableViewImpl tv = createTestTableView(store, prefix, urlSupplier); + + String key1 = "key"; + tv.put(key1, 0).join(); + tv.delete(key1).join(); + assertNull(tv.get(key1)); + } + + @Test(dataProvider = "impl") + public void mapApiTest(String provider, Supplier urlSupplier) throws Exception { + String prefix = newKey(); + @Cleanup + MetadataStore store = MetadataStoreFactoryImpl.create(urlSupplier.get(), MetadataStoreConfig.builder().build()); + MetadataStoreTableViewImpl tv = createTestTableView(store, prefix, urlSupplier); + + assertTrue(tv.isEmpty()); + assertEquals(tv.size(), 0); + + String key1 = "my1"; + String key2 = "my2"; + + int val1 = 1; + int val2 = 2; + + tv.put(key1, val1).join(); + tv.put(key2, val2).join(); + assertGet(tv, key1, 1); + assertGet(tv, key2, 2); + + assertFalse(tv.isEmpty()); + assertEquals(tv.size(), 2); + + List actual = new ArrayList<>(); + tv.forEach((k, v) -> { + actual.add(k + "," + v); + }); + assertEquals(actual, List.of(key1 + "," + val1, key2 + "," + val2)); + + var values = tv.values(); + assertEquals(values.size(), 2); + assertTrue(values.containsAll(List.of(val1, val2))); + + var keys = tv.keySet(); + assertEquals(keys.size(), 2); + assertTrue(keys.containsAll(List.of(key1, key2))); + + var entries = tv.entrySet(); + assertEquals(entries.size(), 2); + assertTrue(entries.containsAll(Map.of(key1, val1, key2, val2).entrySet())); + } + + @Test(dataProvider = "impl") + public void notificationListeners(String provider, Supplier urlSupplier) throws Exception { + String prefix = newKey(); + @Cleanup + MetadataStore store = MetadataStoreFactoryImpl.create(urlSupplier.get(), MetadataStoreConfig.builder().build()); + MetadataStoreTableViewImpl tv = createTestTableView(store, prefix, urlSupplier); + + String keyPrefix = "tenant/ns"; + String key1 = keyPrefix + "/my-1"; + int val1 = 1; + + assertGet(tv, key1, null); + + // listen on put + tv.put(key1, val1).join(); + var kv = tails.poll(3, TimeUnit.SECONDS); + assertEquals(kv, Pair.of(key1, val1)); + assertEquals(tv.get(key1), val1); + + // listen on modified + int val2 = 2; + tv.put(key1, val2).join(); + kv = tails.poll(3, TimeUnit.SECONDS); + assertEquals(kv, Pair.of(key1, val2)); + assertEquals(tv.get(key1), val2); + + // no listen on the parent + int val0 = 0; + String childKey = key1 + "/my-child-1"; + tv.put(childKey, val0).join(); + kv = tails.poll(3, TimeUnit.SECONDS); + assertEquals(kv, Pair.of(childKey, val0)); + kv = tails.poll(3, TimeUnit.SECONDS); + assertNull(kv); + assertEquals(tv.get(key1), val2); + assertEquals(tv.get(childKey), val0); + + tv.put(childKey, val1).join(); + kv = tails.poll(3, TimeUnit.SECONDS); + assertEquals(kv, Pair.of(childKey, val1)); + kv = tails.poll(3, TimeUnit.SECONDS); + assertNull(kv); + assertEquals(tv.get(key1), val2); + assertEquals(tv.get(childKey), val1); + + tv.delete(childKey).join(); + kv = tails.poll(3, TimeUnit.SECONDS); + assertEquals(kv, Pair.of(childKey, null)); + kv = tails.poll(3, TimeUnit.SECONDS); + assertNull(kv); + assertEquals(tv.get(key1), val2); + assertNull(tv.get(childKey)); + + // No listen on the filtered key + String noListenKey = keyPrefix + "/to-be-filtered"; + tv.put(noListenKey, val0).join(); + kv = tails.poll(3, TimeUnit.SECONDS); + assertNull(kv); + assertEquals(tv.get(key1), val2); + assertNull(tv.get(noListenKey)); + + // Trigger deleted notification + tv.delete(key1).join(); + kv = tails.poll(3, TimeUnit.SECONDS); + assertEquals(kv, Pair.of(key1, null)); + assertNull(tv.get(key1)); + } + + @Test(dataProvider = "impl") + public void testConcurrentPutGetOneKey(String provider, Supplier urlSupplier) throws Exception { + String prefix = newKey(); + @Cleanup + MetadataStore store = MetadataStoreFactoryImpl.create(urlSupplier.get(), MetadataStoreConfig.builder().build()); + MetadataStoreTableViewImpl tv = createTestTableView(store, prefix, urlSupplier); + + String key = "my"; + int val = 0; + int maxValue = 50; + tv.put(key, val).join(); + + AtomicInteger successWrites = new AtomicInteger(0); + Runnable task = new Runnable() { + @SneakyThrows + @Override + public void run() { + for (int k = 0; k < 1000; k++) { + var kv = tails.poll(3, TimeUnit.SECONDS); + if (kv == null) { + break; + } + Integer val = kv.getRight() + 1; + if (val <= maxValue) { + CompletableFuture putResult = + tv.put(key, val).thenRun(successWrites::incrementAndGet); + try { + putResult.get(); + } catch (Exception ignore) { + } + log.info("Put value {} success:{}. ", val, !putResult.isCompletedExceptionally()); + } else { + break; + } + } + } + }; + CompletableFuture t1 = CompletableFuture.completedFuture(null).thenRunAsync(task); + CompletableFuture t2 = CompletableFuture.completedFuture(null).thenRunAsync(task); + task.run(); + t1.join(); + t2.join(); + assertFalse(t1.isCompletedExceptionally()); + assertFalse(t2.isCompletedExceptionally()); + + assertEquals(successWrites.get(), maxValue); + assertEquals(tv.get(key), maxValue); + } + + @Test(dataProvider = "impl") + public void testConcurrentPut(String provider, Supplier urlSupplier) throws Exception { + String prefix = newKey(); + @Cleanup + MetadataStore store = MetadataStoreFactoryImpl.create(urlSupplier.get(), MetadataStoreConfig.builder().build()); + MetadataStoreTableViewImpl tv = createTestTableView(store, prefix, urlSupplier); + + String k = "my"; + int v = 0; + CompletableFuture f1 = + CompletableFuture.runAsync(() -> tv.put(k, v).join()); + CompletableFuture f2 = + CompletableFuture.runAsync(() -> tv.put(k, v).join()); + Awaitility.await().until(() -> f1.isDone() && f2.isDone()); + assertTrue(f1.isCompletedExceptionally() && !f2.isCompletedExceptionally() || + ! f1.isCompletedExceptionally() && f2.isCompletedExceptionally()); + } + + @Test(dataProvider = "impl") + public void testConcurrentDelete(String provider, Supplier urlSupplier) throws Exception { + String prefix = newKey(); + @Cleanup + MetadataStore store = MetadataStoreFactoryImpl.create(urlSupplier.get(), MetadataStoreConfig.builder().build()); + MetadataStoreTableViewImpl tv = createTestTableView(store, prefix, urlSupplier); + + String k = "my"; + tv.put(k, 0).join(); + CompletableFuture f1 = + CompletableFuture.runAsync(() -> tv.delete(k).join()); + CompletableFuture f2 = + CompletableFuture.runAsync(() -> tv.delete(k).join()); + Awaitility.await().until(() -> f1.isDone() && f2.isDone()); + assertTrue(f1.isCompletedExceptionally() && !f2.isCompletedExceptionally() || + ! f1.isCompletedExceptionally() && f2.isCompletedExceptionally()); + } + + @Test(dataProvider = "impl") + public void testClosedMetadataStore(String provider, Supplier urlSupplier) throws Exception { + String prefix = newKey(); + String k = "my"; + @Cleanup + MetadataStore store = MetadataStoreFactoryImpl.create(urlSupplier.get(), MetadataStoreConfig.builder().build()); + MetadataStoreTableViewImpl tv = createTestTableView(store, prefix, urlSupplier); + store.close(); + try { + tv.put(k, 0).get(); + fail(); + } catch (Exception e) { + assertTrue(e.getCause() instanceof MetadataStoreException.AlreadyClosedException); + } + try { + tv.delete(k).get(); + fail(); + } catch (Exception e) { + assertTrue(e.getCause() instanceof MetadataStoreException.AlreadyClosedException); + } + + } + + + @Test(dataProvider = "distributedImpl") + public void testGetIfCachedDistributed(String provider, Supplier urlSupplier) throws Exception { + + String prefix = newKey(); + String k = "my"; + @Cleanup + MetadataStore store1 = MetadataStoreFactoryImpl.create(urlSupplier.get(), MetadataStoreConfig.builder().build()); + MetadataStoreTableViewImpl tv1 = createTestTableView(store1, prefix, urlSupplier); + @Cleanup + MetadataStore store2 = MetadataStoreFactoryImpl.create(urlSupplier.get(), MetadataStoreConfig.builder().build()); + MetadataStoreTableViewImpl tv2 = createTestTableView(store2, prefix, urlSupplier); + + + assertNull(tv1.get(k)); + assertNull(tv2.get(k)); + + tv1.put(k, 0).join(); + assertGet(tv1, k, 0); + Awaitility.await() + .untilAsserted(() -> assertEquals(tv2.get(k), 0)); + + tv2.put(k, 1).join(); + assertGet(tv2, k, 1); + Awaitility.await() + .untilAsserted(() -> assertEquals(tv1.get(k), 1)); + + tv1.delete(k).join(); + assertGet(tv1, k, null); + Awaitility.await() + .untilAsserted(() -> assertNull(tv2.get(k))); + } + + @Test(dataProvider = "distributedImpl") + public void testInitialFill(String provider, Supplier urlSupplier) throws Exception { + + String prefix = newKey(); + String k1 = "tenant-1/ns-1/my-1"; + String k2 = "tenant-1/ns-1/my-2"; + String k3 = "tenant-1/ns-2/my-3"; + String k4 = "tenant-2/ns-3/my-4"; + String k5 = "tenant-2/ns-3/your-1"; + @Cleanup + MetadataStore store = MetadataStoreFactoryImpl.create(urlSupplier.get(), MetadataStoreConfig.builder().build()); + MetadataStoreTableViewImpl btv = createTestTableView(store, prefix, urlSupplier); + + assertFalse(btv.exists(k1)); + + var serde = new JSONMetadataSerdeSimpleType<>( + TypeFactory.defaultInstance().constructSimpleType(Integer.class, null)); + store.put(prefix + "/" + k1, serde.serialize(prefix + "/" + k1, 0), Optional.empty()).join(); + store.put(prefix + "/" + k2, serde.serialize(prefix + "/" + k2, 1), Optional.empty()).join(); + store.put(prefix + "/" + k3, serde.serialize(prefix + "/" + k3, 2), Optional.empty()).join(); + store.put(prefix + "/" + k4, serde.serialize(prefix + "/" + k4, 3), Optional.empty()).join(); + store.put(prefix + "/" + k5, serde.serialize(prefix + "/" + k5, 4), Optional.empty()).join(); + + var expected = new HashSet<>(Set.of(Pair.of(k1, 0), Pair.of(k2, 1), Pair.of(k3, 2), Pair.of(k4, 3))); + var tailExpected = new HashSet<>(expected); + + for (int i = 0; i < 4; i++) { + var kv = tails.poll(3, TimeUnit.SECONDS); + assertTrue(tailExpected.remove(kv)); + } + assertNull(tails.poll(3, TimeUnit.SECONDS)); + assertTrue(tailExpected.isEmpty()); + + @Cleanup + MetadataStore store2 = MetadataStoreFactoryImpl.create(urlSupplier.get(), MetadataStoreConfig.builder().build()); + MetadataStoreTableViewImpl tv = createTestTableView(store2, prefix, urlSupplier); + + var existingExpected = new HashSet<>(Set.of(Pair.of(k1, 0), Pair.of(k2, 1), Pair.of(k3, 2), Pair.of(k4, 3))); + var entrySetExpected = expected.stream().collect(Collectors.toMap(Pair::getLeft, Pair::getRight)).entrySet(); + + + for (int i = 0; i < 4; i++) { + var kv = existings.poll(3, TimeUnit.SECONDS); + assertTrue(existingExpected.remove(kv)); + } + assertNull(existings.poll(3, TimeUnit.SECONDS)); + assertTrue(existingExpected.isEmpty()); + + assertEquals(tv.get(k1), 0); + assertEquals(tv.get(k2), 1); + assertEquals(tv.get(k3), 2); + assertEquals(tv.get(k4), 3); + assertNull(tv.get(k5)); + + assertEquals(tv.entrySet(), entrySetExpected); + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java index 949b4a9b2bacb..2c589dfd48222 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java @@ -24,18 +24,27 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; + +import io.streamnative.oxia.client.ClientConfig; +import io.streamnative.oxia.client.api.AsyncOxiaClient; +import io.streamnative.oxia.client.session.SessionFactory; +import io.streamnative.oxia.client.session.SessionManager; import lombok.Cleanup; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -50,8 +59,16 @@ import org.apache.pulsar.metadata.api.Notification; import org.apache.pulsar.metadata.api.NotificationType; import org.apache.pulsar.metadata.api.Stat; +import org.apache.pulsar.metadata.impl.PulsarZooKeeperClient; +import org.apache.pulsar.metadata.impl.ZKMetadataStore; +import org.apache.pulsar.metadata.impl.oxia.OxiaMetadataStore; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooKeeper; import org.assertj.core.util.Lists; import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @Slf4j @@ -59,27 +76,28 @@ public class MetadataStoreTest extends BaseMetadataStoreTest { @Test(dataProvider = "impl") public void emptyStoreTest(String provider, Supplier urlSupplier) throws Exception { + String prefix = newKey(); @Cleanup MetadataStore store = MetadataStoreFactory.create(urlSupplier.get(), MetadataStoreConfig.builder().fsyncEnable(false).build()); - assertFalse(store.exists("/non-existing-key").join()); - assertFalse(store.exists("/non-existing-key/child").join()); - assertFalse(store.get("/non-existing-key").join().isPresent()); - assertFalse(store.get("/non-existing-key/child").join().isPresent()); + assertFalse(store.exists(prefix + "/non-existing-key").join()); + assertFalse(store.exists(prefix + "/non-existing-key/child").join()); + assertFalse(store.get(prefix + "/non-existing-key").join().isPresent()); + assertFalse(store.get(prefix + "/non-existing-key/child").join().isPresent()); - assertEquals(store.getChildren("/non-existing-key").join(), Collections.emptyList()); - assertEquals(store.getChildren("/non-existing-key/child").join(), Collections.emptyList()); + assertEquals(store.getChildren(prefix + "/non-existing-key").join(), Collections.emptyList()); + assertEquals(store.getChildren(prefix + "/non-existing-key/child").join(), Collections.emptyList()); try { - store.delete("/non-existing-key", Optional.empty()).join(); + store.delete(prefix + "/non-existing-key", Optional.empty()).join(); fail("Should have failed"); } catch (CompletionException e) { assertException(e, NotFoundException.class); } try { - store.delete("/non-existing-key", Optional.of(1L)).join(); + store.delete(prefix + "/non-existing-key", Optional.of(1L)).join(); fail("Should have failed"); } catch (CompletionException e) { assertTrue(NotFoundException.class.isInstance(e.getCause()) || BadVersionException.class.isInstance( @@ -397,6 +415,10 @@ public void testDeleteRecursive(String provider, Supplier urlSupplier) t @Test(dataProvider = "impl") public void testDeleteUnusedDirectories(String provider, Supplier urlSupplier) throws Exception { + if (provider.equals("Oxia")) { + return; + } + @Cleanup MetadataStore store = MetadataStoreFactory.create(urlSupplier.get(), MetadataStoreConfig.builder().fsyncEnable(false).build()); @@ -425,6 +447,117 @@ public void testDeleteUnusedDirectories(String provider, Supplier urlSup assertFalse(store.exists(prefix).join()); } + @DataProvider(name = "conditionOfSwitchThread") + public Object[][] conditionOfSwitchThread(){ + return new Object[][]{ + {false, false}, + {false, true}, + {true, false}, + {true, true} + }; + } + + @Test(dataProvider = "conditionOfSwitchThread") + public void testThreadSwitchOfZkMetadataStore(boolean hasSynchronizer, boolean enabledBatch) throws Exception { + final String prefix = newKey(); + final String metadataStoreName = UUID.randomUUID().toString().replaceAll("-", ""); + MetadataStoreConfig.MetadataStoreConfigBuilder builder = + MetadataStoreConfig.builder().metadataStoreName(metadataStoreName); + builder.fsyncEnable(false); + builder.batchingEnabled(enabledBatch); + if (!hasSynchronizer) { + builder.synchronizer(null); + } + MetadataStoreConfig config = builder.build(); + @Cleanup + ZKMetadataStore store = (ZKMetadataStore) MetadataStoreFactory.create(zks.getConnectionString(), config); + ZooKeeper zkClient = store.getZkClient(); + assertTrue(zkClient.getClientConfig().isSaslClientEnabled()); + final Runnable verify = () -> { + String currentThreadName = Thread.currentThread().getName(); + String errorMessage = String.format("Expect to switch to thread %s, but currently it is thread %s", + metadataStoreName, currentThreadName); + assertTrue(Thread.currentThread().getName().startsWith(metadataStoreName), errorMessage); + }; + + // put with node which has parent(but the parent node is not exists). + store.put(prefix + "/a1/b1/c1", "value".getBytes(), Optional.of(-1L)).thenApply((ignore) -> { + verify.run(); + return null; + }).join(); + // put. + store.put(prefix + "/b1", "value".getBytes(), Optional.of(-1L)).thenApply((ignore) -> { + verify.run(); + return null; + }).join(); + // get. + store.get(prefix + "/b1").thenApply((ignore) -> { + verify.run(); + return null; + }).join(); + // get the node which is not exists. + store.get(prefix + "/non").thenApply((ignore) -> { + verify.run(); + return null; + }).join(); + // delete. + store.delete(prefix + "/b1", Optional.empty()).thenApply((ignore) -> { + verify.run(); + return null; + }).join(); + // delete the node which is not exists. + store.delete(prefix + "/non", Optional.empty()).thenApply((ignore) -> { + verify.run(); + return null; + }).exceptionally(ex -> { + verify.run(); + return null; + }).join(); + } + + @Test + public void testZkLoadConfigFromFile() throws Exception { + final String metadataStoreName = UUID.randomUUID().toString().replaceAll("-", ""); + MetadataStoreConfig.MetadataStoreConfigBuilder builder = + MetadataStoreConfig.builder().metadataStoreName(metadataStoreName); + builder.fsyncEnable(false); + builder.batchingEnabled(true); + builder.configFilePath("src/test/resources/zk_client_disabled_sasl.conf"); + MetadataStoreConfig config = builder.build(); + @Cleanup + ZKMetadataStore store = (ZKMetadataStore) MetadataStoreFactory.create(zks.getConnectionString(), config); + + PulsarZooKeeperClient zkClient = (PulsarZooKeeperClient) store.getZkClient(); + assertFalse(zkClient.getClientConfig().isSaslClientEnabled()); + + zkClient.process(new WatchedEvent(Watcher.Event.EventType.None, Watcher.Event.KeeperState.Expired, null)); + + var zooKeeperRef = (AtomicReference) WhiteboxImpl.getInternalState(zkClient, "zk"); + var zooKeeper = Awaitility.await().until(zooKeeperRef::get, Objects::nonNull); + assertFalse(zooKeeper.getClientConfig().isSaslClientEnabled()); + } + + @Test + public void testOxiaLoadConfigFromFile() throws Exception { + final String metadataStoreName = UUID.randomUUID().toString().replaceAll("-", ""); + String oxia = "oxia://" + getOxiaServerConnectString(); + MetadataStoreConfig.MetadataStoreConfigBuilder builder = + MetadataStoreConfig.builder().metadataStoreName(metadataStoreName); + builder.fsyncEnable(false); + builder.batchingEnabled(true); + builder.sessionTimeoutMillis(30000); + builder.configFilePath("src/test/resources/oxia_client.conf"); + MetadataStoreConfig config = builder.build(); + + OxiaMetadataStore store = (OxiaMetadataStore) MetadataStoreFactory.create(oxia, config); + var client = (AsyncOxiaClient) WhiteboxImpl.getInternalState(store, "client"); + var sessionManager = (SessionManager) WhiteboxImpl.getInternalState(client, "sessionManager"); + var sessionFactory = (SessionFactory) WhiteboxImpl.getInternalState(sessionManager, "factory"); + var clientConfig = (ClientConfig) WhiteboxImpl.getInternalState(sessionFactory, "config"); + var sessionTimeout = clientConfig.sessionTimeout(); + assertEquals(sessionTimeout, Duration.ofSeconds(60)); + } + @Test(dataProvider = "impl") public void testPersistent(String provider, Supplier urlSupplier) throws Exception { String metadataUrl = urlSupplier.get(); @@ -445,6 +578,7 @@ public void testPersistent(String provider, Supplier urlSupplier) throws @Test(dataProvider = "impl") public void testConcurrentPutGetOneKey(String provider, Supplier urlSupplier) throws Exception { + @Cleanup MetadataStore store = MetadataStoreFactory.create(urlSupplier.get(), MetadataStoreConfig.builder().fsyncEnable(false).build()); byte[] data = new byte[]{0}; @@ -593,4 +727,57 @@ public void testClosedMetadataStore(String provider, Supplier urlSupplie assertTrue(e.getCause() instanceof MetadataStoreException.AlreadyClosedException); } } + + @Test(dataProvider = "distributedImpl") + public void testGetChildrenDistributed(String provider, Supplier urlSupplier) throws Exception { + @Cleanup + MetadataStore store1 = MetadataStoreFactory.create(urlSupplier.get(), + MetadataStoreConfig.builder().fsyncEnable(false).build()); + @Cleanup + MetadataStore store2 = MetadataStoreFactory.create(urlSupplier.get(), + MetadataStoreConfig.builder().fsyncEnable(false).build()); + + String parent = newKey(); + byte[] value = "value1".getBytes(StandardCharsets.UTF_8); + store1.put(parent, value, Optional.empty()).get(); + store1.put(parent + "/a", value, Optional.empty()).get(); + assertEquals(store1.getChildren(parent).get(), List.of("a")); + store1.delete(parent + "/a", Optional.empty()).get(); + assertEquals(store1.getChildren(parent).get(), Collections.emptyList()); + store1.delete(parent, Optional.empty()).get(); + assertEquals(store1.getChildren(parent).get(), Collections.emptyList()); + store2.put(parent + "/b", value, Optional.empty()).get(); + // There is a chance watcher event is not triggered before the store1.getChildren() call. + Awaitility.await().atMost(3, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .untilAsserted(() -> assertEquals(store1.getChildren(parent).get(), List.of("b"))); + store2.put(parent + "/c", value, Optional.empty()).get(); + Awaitility.await().atMost(3, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .untilAsserted(() -> assertEquals(store1.getChildren(parent).get(), List.of("b", "c"))); + } + + @Test(dataProvider = "distributedImpl") + public void testExistsDistributed(String provider, Supplier urlSupplier) throws Exception { + @Cleanup + MetadataStore store1 = MetadataStoreFactory.create(urlSupplier.get(), + MetadataStoreConfig.builder().fsyncEnable(false).build()); + @Cleanup + MetadataStore store2 = MetadataStoreFactory.create(urlSupplier.get(), + MetadataStoreConfig.builder().fsyncEnable(false).build()); + + String parent = newKey(); + byte[] value = "value1".getBytes(StandardCharsets.UTF_8); + assertFalse(store1.exists(parent).get()); + store1.put(parent, value, Optional.empty()).get(); + assertTrue(store1.exists(parent).get()); + assertFalse(store1.exists(parent + "/a").get()); + store2.put(parent + "/a", value, Optional.empty()).get(); + + Awaitility.await() + .untilAsserted(() -> assertTrue(store1.exists(parent + "/a").get())); + + // There is a chance watcher event is not triggered before the store1.exists() call. + assertFalse(store1.exists(parent + "/b").get()); + } } diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/OxiaMetadataStoreErrorTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/OxiaMetadataStoreErrorTest.java new file mode 100644 index 0000000000000..b45afe5413fc1 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/OxiaMetadataStoreErrorTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.metadata; + +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.metadata.api.MetadataStore; +import org.apache.pulsar.metadata.api.MetadataStoreConfig; +import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.metadata.api.MetadataStoreFactory; +import org.testng.annotations.Test; + +@Slf4j +public class OxiaMetadataStoreErrorTest extends BaseMetadataStoreTest { + + @Test + public void emptyStoreTest() throws Exception { + String metadataStoreUrl = "oxia://" + getOxiaServerConnectString(); + String prefix = newKey(); + @Cleanup + MetadataStore store = MetadataStoreFactory.create(metadataStoreUrl, + MetadataStoreConfig.builder().fsyncEnable(false).build()); + oxiaServer.close(); + try { + store.exists(prefix + "/non-existing-key").join(); + fail("Expected an exception because the metadata store server has been closed."); + } catch (Exception ex) { + Throwable actEx = FutureUtil.unwrapCompletionException(ex); + assertTrue(actEx instanceof MetadataStoreException); + } + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/TestZKServer.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/TestZKServer.java index 726f5ae312d19..0d01d9c56abc8 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/TestZKServer.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/TestZKServer.java @@ -19,39 +19,33 @@ package org.apache.pulsar.metadata; import static org.testng.Assert.assertTrue; - import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStream; -import java.net.InetSocketAddress; +import java.lang.reflect.Field; import java.net.Socket; - -import java.nio.charset.StandardCharsets; - +import java.util.Properties; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; - import org.apache.commons.io.FileUtils; -import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.server.ContainerManager; -import org.apache.zookeeper.server.NIOServerCnxnFactory; -import org.apache.zookeeper.server.Request; -import org.apache.zookeeper.server.RequestProcessor; import org.apache.zookeeper.server.ServerCnxnFactory; -import org.apache.zookeeper.server.SessionTracker; import org.apache.zookeeper.server.ZooKeeperServer; +import org.apache.zookeeper.server.ZooKeeperServerMain; +import org.apache.zookeeper.server.embedded.ExitHandler; +import org.apache.zookeeper.server.embedded.ZooKeeperServerEmbedded; import org.assertj.core.util.Files; @Slf4j public class TestZKServer implements AutoCloseable { + public static final int TICK_TIME = 1000; - protected ZooKeeperServer zks; - private final File zkDataDir; - private ServerCnxnFactory serverFactory; - private ContainerManager containerManager; - private int zkPort = 0; + private final File zkDataDir; + private int zkPort; // initially this is zero + private ZooKeeperServerEmbedded zooKeeperServerEmbedded; public TestZKServer() throws Exception { this.zkDataDir = Files.newTemporaryFolder(); @@ -64,86 +58,87 @@ public TestZKServer() throws Exception { } public void start() throws Exception { - this.zks = new ZooKeeperServer(zkDataDir, zkDataDir, TICK_TIME); - this.zks.setMaxSessionTimeout(300_000); - this.serverFactory = new NIOServerCnxnFactory(); - this.serverFactory.configure(new InetSocketAddress(zkPort), 1000); - this.serverFactory.startup(zks, true); - - this.zkPort = serverFactory.getLocalPort(); - log.info("Started test ZK server on port {}", zkPort); + final Properties configZookeeper = new Properties(); + configZookeeper.put("clientPort", zkPort + ""); + configZookeeper.put("host", "127.0.0.1"); + configZookeeper.put("ticktime", TICK_TIME + ""); + zooKeeperServerEmbedded = ZooKeeperServerEmbedded + .builder() + .baseDir(zkDataDir.toPath()) + .configuration(configZookeeper) + .exitHandler(ExitHandler.LOG_ONLY) + .build(); + + zooKeeperServerEmbedded.start(60_000); + log.info("Started test ZK server on at {}", zooKeeperServerEmbedded.getConnectionString()); + + ZooKeeperServerMain zooKeeperServerMain = getZooKeeperServerMain(zooKeeperServerEmbedded); + ServerCnxnFactory serverCnxnFactory = getServerCnxnFactory(zooKeeperServerMain); + // save the port, in order to allow restarting on the same port + zkPort = serverCnxnFactory.getLocalPort(); boolean zkServerReady = waitForServerUp(this.getConnectionString(), 30_000); assertTrue(zkServerReady); + } - this.containerManager = new ContainerManager(zks.getZKDatabase(), new RequestProcessor() { - @Override - public void processRequest(Request request) throws RequestProcessorException { - String path = StandardCharsets.UTF_8.decode(request.request).toString(); - try { - zks.getZKDatabase().getDataTree().deleteNode(path, -1); - } catch (KeeperException.NoNodeException e) { - // Ok - } - } + @SneakyThrows + private static ZooKeeperServerMain getZooKeeperServerMain(ZooKeeperServerEmbedded zooKeeperServerEmbedded) { + ZooKeeperServerMain zooKeeperServerMain = readField(zooKeeperServerEmbedded.getClass(), + "mainsingle", zooKeeperServerEmbedded); + return zooKeeperServerMain; + } - @Override - public void shutdown() { + @SneakyThrows + private static ContainerManager getContainerManager(ZooKeeperServerMain zooKeeperServerMain) { + ContainerManager containerManager = readField(ZooKeeperServerMain.class, "containerManager", zooKeeperServerMain); + return containerManager; + } - } - }, 10, 10000, 0L); + @SneakyThrows + private static ZooKeeperServer getZooKeeperServer(ZooKeeperServerMain zooKeeperServerMain) { + ServerCnxnFactory serverCnxnFactory = getServerCnxnFactory(zooKeeperServerMain); + ZooKeeperServer zkServer = readField(ServerCnxnFactory.class, "zkServer", serverCnxnFactory); + return zkServer; + } + + @SneakyThrows + private static T readField(Class clazz, String field, Object object) { + Field declaredField = clazz.getDeclaredField(field); + boolean accessible = declaredField.isAccessible(); + if (!accessible) { + declaredField.setAccessible(true); + } + try { + return (T) declaredField.get(object); + } finally { + declaredField.setAccessible(accessible); + } + } + + private static ServerCnxnFactory getServerCnxnFactory(ZooKeeperServerMain zooKeeperServerMain) throws Exception { + ServerCnxnFactory serverCnxnFactory = readField(ZooKeeperServerMain.class, "cnxnFactory", zooKeeperServerMain); + return serverCnxnFactory; } public void checkContainers() throws Exception { // Make sure the container nodes are actually deleted Thread.sleep(1000); + ContainerManager containerManager = getContainerManager(getZooKeeperServerMain(zooKeeperServerEmbedded)); containerManager.checkContainers(); } public void stop() throws Exception { - if (containerManager != null) { - containerManager.stop(); - containerManager = null; - } - - if (serverFactory != null) { - serverFactory.shutdown(); - serverFactory = null; - } - - if (zks != null) { - SessionTracker sessionTracker = zks.getSessionTracker(); - zks.shutdown(); - zks.getZKDatabase().close(); - if (sessionTracker instanceof Thread) { - Thread sessionTrackerThread = (Thread) sessionTracker; - sessionTrackerThread.interrupt(); - sessionTrackerThread.join(); - } - zks = null; + if (zooKeeperServerEmbedded != null) { + zooKeeperServerEmbedded.close(); + zooKeeperServerEmbedded = null; } - log.info("Stopped test ZK server"); } public void expireSession(long sessionId) { - zks.expire(new SessionTracker.Session() { - @Override - public long getSessionId() { - return sessionId; - } - - @Override - public int getTimeout() { - return 10_000; - } - - @Override - public boolean isClosing() { - return false; - } - }); + getZooKeeperServer(getZooKeeperServerMain(zooKeeperServerEmbedded)) + .expire(sessionId); } @Override @@ -152,12 +147,9 @@ public void close() throws Exception { FileUtils.deleteDirectory(zkDataDir); } - public int getPort() { - return zkPort; - } - + @SneakyThrows public String getConnectionString() { - return "127.0.0.1:" + getPort(); + return zooKeeperServerEmbedded.getConnectionString(); } public static boolean waitForServerUp(String hp, long timeout) { diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/ZKSessionTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/ZKSessionTest.java index 36cb0f132ba58..02d65fd21ed5c 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/ZKSessionTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/ZKSessionTest.java @@ -27,6 +27,7 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import lombok.Cleanup; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.coordination.CoordinationService; import org.apache.pulsar.metadata.api.coordination.LeaderElection; @@ -180,4 +181,58 @@ public void testReacquireLeadershipAfterSessionLost() throws Exception { .untilAsserted(()-> assertEquals(le1.getState(),LeaderElectionState.Leading)); assertTrue(store.get(path).join().isPresent()); } + + + @Test + public void testElectAfterReconnected() throws Exception { + // --- init + @Cleanup + MetadataStoreExtended store = MetadataStoreExtended.create(zks.getConnectionString(), + MetadataStoreConfig.builder() + .sessionTimeoutMillis(2_000) + .build()); + + + BlockingQueue sessionEvents = new LinkedBlockingQueue<>(); + store.registerSessionListener(sessionEvents::add); + BlockingQueue leaderElectionEvents = new LinkedBlockingQueue<>(); + String path = newKey(); + + @Cleanup + CoordinationService coordinationService = new CoordinationServiceImpl(store); + @Cleanup + LeaderElection le1 = coordinationService.getLeaderElection(String.class, path, + leaderElectionEvents::add); + + // --- test manual elect + String proposed = "value-1"; + le1.elect(proposed).join(); + assertEquals(le1.getState(), LeaderElectionState.Leading); + LeaderElectionState les = leaderElectionEvents.poll(5, TimeUnit.SECONDS); + assertEquals(les, LeaderElectionState.Leading); + + + // simulate no leader state + FieldUtils.writeDeclaredField(le1, "leaderElectionState", LeaderElectionState.NoLeader, true); + + // reconnect + zks.stop(); + + SessionEvent e = sessionEvents.poll(5, TimeUnit.SECONDS); + assertEquals(e, SessionEvent.ConnectionLost); + + zks.start(); + + + // --- test le1 can be leader + e = sessionEvents.poll(10, TimeUnit.SECONDS); + assertEquals(e, SessionEvent.Reconnected); + Awaitility.await().atMost(Duration.ofSeconds(15)) + .untilAsserted(()-> { + assertEquals(le1.getState(),LeaderElectionState.Leading); + }); // reacquire leadership + + + assertTrue(store.get(path).join().isPresent()); + } } diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/FaultInjectableZKRegistrationManager.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/FaultInjectableZKRegistrationManager.java new file mode 100644 index 0000000000000..bcbf41addbae3 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/FaultInjectableZKRegistrationManager.java @@ -0,0 +1,630 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.metadata.bookkeeper; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.bookkeeper.util.BookKeeperConstants.AVAILABLE_NODE; +import static org.apache.bookkeeper.util.BookKeeperConstants.COOKIE_NODE; +import static org.apache.bookkeeper.util.BookKeeperConstants.EMPTY_BYTE_ARRAY; +import static org.apache.bookkeeper.util.BookKeeperConstants.INSTANCEID; +import static org.apache.bookkeeper.util.BookKeeperConstants.READONLY; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.bookie.BookieException; +import org.apache.bookkeeper.bookie.BookieException.BookieIllegalOpException; +import org.apache.bookkeeper.bookie.BookieException.CookieExistException; +import org.apache.bookkeeper.bookie.BookieException.CookieNotFoundException; +import org.apache.bookkeeper.bookie.BookieException.MetadataStoreException; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BKException.BKInterruptedException; +import org.apache.bookkeeper.client.BKException.MetaStoreException; +import org.apache.bookkeeper.common.concurrent.FutureUtils; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.discover.BookieServiceInfo; +import org.apache.bookkeeper.discover.RegistrationClient; +import org.apache.bookkeeper.discover.RegistrationManager; +import org.apache.bookkeeper.discover.ZKRegistrationClient; +import org.apache.bookkeeper.meta.AbstractZkLedgerManagerFactory; +import org.apache.bookkeeper.meta.LayoutManager; +import org.apache.bookkeeper.meta.LedgerManagerFactory; +import org.apache.bookkeeper.meta.ZkLayoutManager; +import org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager; +import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.proto.DataFormats.BookieServiceInfoFormat; +import org.apache.bookkeeper.util.BookKeeperConstants; +import org.apache.bookkeeper.util.ZkUtils; +import org.apache.bookkeeper.versioning.LongVersion; +import org.apache.bookkeeper.versioning.Version; +import org.apache.bookkeeper.versioning.Versioned; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.KeeperException.NoNodeException; +import org.apache.zookeeper.KeeperException.NodeExistsException; +import org.apache.zookeeper.Op; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.Watcher.Event.EventType; +import org.apache.zookeeper.Watcher.Event.KeeperState; +import org.apache.zookeeper.ZKUtil; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; + +/** + * Fault injectable ZK registration manager. + * Copy from #{@link org.apache.bookkeeper.discover.ZKRegistrationManager}. + */ +@Slf4j +public class FaultInjectableZKRegistrationManager implements RegistrationManager { + + private static final Function EXCEPTION_FUNC = cause -> { + if (cause instanceof BKException) { + log.error("Failed to get bookie list : ", cause); + return (BKException) cause; + } else if (cause instanceof InterruptedException) { + log.error("Interrupted reading bookie list : ", cause); + return new BKInterruptedException(); + } else { + return new MetaStoreException(); + } + }; + + private final ServerConfiguration conf; + private final ZooKeeper zk; + private final List zkAcls; + private final LayoutManager layoutManager; + + private volatile boolean zkRegManagerInitialized = false; + + // ledgers root path + private final String ledgersRootPath; + // cookie path + private final String cookiePath; + // registration paths + protected final String bookieRegistrationPath; + protected final String bookieReadonlyRegistrationPath; + // session timeout in milliseconds + private final int zkTimeoutMs; + private final List listeners = new ArrayList<>(); + private Function hookOnRegisterReadOnly; + + public FaultInjectableZKRegistrationManager(ServerConfiguration conf, + ZooKeeper zk) { + this(conf, zk, ZKMetadataDriverBase.resolveZkLedgersRootPath(conf)); + } + + public FaultInjectableZKRegistrationManager(ServerConfiguration conf, + ZooKeeper zk, + String ledgersRootPath) { + this.conf = conf; + this.zk = zk; + this.zkAcls = ZkUtils.getACLs(conf); + this.ledgersRootPath = ledgersRootPath; + this.cookiePath = ledgersRootPath + "/" + COOKIE_NODE; + this.bookieRegistrationPath = ledgersRootPath + "/" + AVAILABLE_NODE; + this.bookieReadonlyRegistrationPath = this.bookieRegistrationPath + "/" + READONLY; + this.zkTimeoutMs = conf.getZkTimeout(); + + this.layoutManager = new ZkLayoutManager( + zk, + ledgersRootPath, + zkAcls); + + this.zk.register(event -> { + if (!zkRegManagerInitialized) { + // do nothing until first registration + return; + } + // Check for expired connection. + if (event.getType().equals(EventType.None) + && event.getState().equals(KeeperState.Expired)) { + listeners.forEach(RegistrationListener::onRegistrationExpired); + } + }); + } + + @Override + public void close() { + // no-op + } + + /** + * Returns the CookiePath of the bookie in the ZooKeeper. + * + * @param bookieId bookie id + * @return + */ + public String getCookiePath(BookieId bookieId) { + return this.cookiePath + "/" + bookieId; + } + + // + // Registration Management + // + + /** + * Check existence of regPath and wait it expired if possible. + * + * @param regPath reg node path. + * @return true if regPath exists, otherwise return false + * @throws IOException if can't create reg path + */ + protected boolean checkRegNodeAndWaitExpired(String regPath) throws IOException { + final CountDownLatch prevNodeLatch = new CountDownLatch(1); + Watcher zkPrevRegNodewatcher = new Watcher() { + @Override + public void process(WatchedEvent event) { + // Check for prev znode deletion. Connection expiration is + // not handling, since bookie has logic to shutdown. + if (EventType.NodeDeleted == event.getType()) { + prevNodeLatch.countDown(); + } + } + }; + try { + Stat stat = zk.exists(regPath, zkPrevRegNodewatcher); + if (null != stat) { + // if the ephemeral owner isn't current zookeeper client + // wait for it to be expired. + if (stat.getEphemeralOwner() != zk.getSessionId()) { + log.info("Previous bookie registration znode: {} exists, so waiting zk sessiontimeout:" + + " {} ms for znode deletion", regPath, zkTimeoutMs); + // waiting for the previous bookie reg znode deletion + if (!prevNodeLatch.await(zkTimeoutMs, TimeUnit.MILLISECONDS)) { + throw new NodeExistsException(regPath); + } else { + return false; + } + } + return true; + } else { + return false; + } + } catch (KeeperException ke) { + log.error("ZK exception checking and wait ephemeral znode {} expired : ", regPath, ke); + throw new IOException("ZK exception checking and wait ephemeral znode " + + regPath + " expired", ke); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + log.error("Interrupted checking and wait ephemeral znode {} expired : ", regPath, ie); + throw new IOException("Interrupted checking and wait ephemeral znode " + + regPath + " expired", ie); + } + } + + @Override + public void registerBookie(BookieId bookieId, boolean readOnly, + BookieServiceInfo bookieServiceInfo) throws BookieException { + if (!readOnly) { + String regPath = bookieRegistrationPath + "/" + bookieId; + doRegisterBookie(regPath, bookieServiceInfo); + } else { + doRegisterReadOnlyBookie(bookieId, bookieServiceInfo); + } + } + + @VisibleForTesting + static byte[] serializeBookieServiceInfo(BookieServiceInfo bookieServiceInfo) { + if (log.isDebugEnabled()) { + log.debug("serialize BookieServiceInfo {}", bookieServiceInfo); + } + try (ByteArrayOutputStream os = new ByteArrayOutputStream()) { + BookieServiceInfoFormat.Builder builder = BookieServiceInfoFormat.newBuilder(); + List bsiEndpoints = bookieServiceInfo.getEndpoints().stream() + .map(e -> { + return BookieServiceInfoFormat.Endpoint.newBuilder() + .setId(e.getId()) + .setPort(e.getPort()) + .setHost(e.getHost()) + .setProtocol(e.getProtocol()) + .addAllAuth(e.getAuth()) + .addAllExtensions(e.getExtensions()) + .build(); + }) + .collect(Collectors.toList()); + + builder.addAllEndpoints(bsiEndpoints); + builder.putAllProperties(bookieServiceInfo.getProperties()); + + builder.build().writeTo(os); + return os.toByteArray(); + } catch (IOException err) { + log.error("Cannot serialize bookieServiceInfo from " + bookieServiceInfo); + throw new RuntimeException(err); + } + } + + private void doRegisterBookie(String regPath, BookieServiceInfo bookieServiceInfo) throws BookieException { + // ZK ephemeral node for this Bookie. + try { + if (!checkRegNodeAndWaitExpired(regPath)) { + // Create the ZK ephemeral node for this Bookie. + zk.create(regPath, serializeBookieServiceInfo(bookieServiceInfo), zkAcls, CreateMode.EPHEMERAL); + zkRegManagerInitialized = true; + } + } catch (KeeperException ke) { + log.error("ZK exception registering ephemeral Znode for Bookie!", ke); + // Throw an IOException back up. This will cause the Bookie + // constructor to error out. Alternatively, we could do a System + // exit here as this is a fatal error. + throw new MetadataStoreException(ke); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + log.error("Interrupted exception registering ephemeral Znode for Bookie!", ie); + // Throw an IOException back up. This will cause the Bookie + // constructor to error out. Alternatively, we could do a System + // exit here as this is a fatal error. + throw new MetadataStoreException(ie); + } catch (IOException e) { + throw new MetadataStoreException(e); + } + } + + private void doRegisterReadOnlyBookie(BookieId bookieId, BookieServiceInfo bookieServiceInfo) + throws BookieException { + try { + if (null == zk.exists(this.bookieReadonlyRegistrationPath, false)) { + try { + zk.create(this.bookieReadonlyRegistrationPath, serializeBookieServiceInfo(bookieServiceInfo), + zkAcls, CreateMode.PERSISTENT); + } catch (NodeExistsException e) { + // this node is just now created by someone. + } + } + String regPath = bookieReadonlyRegistrationPath + "/" + bookieId; + doRegisterBookie(regPath, bookieServiceInfo); + // clear the write state + regPath = bookieRegistrationPath + "/" + bookieId; + try { + if (hookOnRegisterReadOnly != null) { + hookOnRegisterReadOnly.apply(null); + } + // Clear the current registered node + zk.delete(regPath, -1); + } catch (KeeperException.NoNodeException nne) { + log.warn("No writable bookie registered node {} when transitioning to readonly", + regPath, nne); + } + } catch (KeeperException | InterruptedException e) { + throw new MetadataStoreException(e); + } + } + + @Override + public void unregisterBookie(BookieId bookieId, boolean readOnly) throws BookieException { + String regPath; + if (!readOnly) { + regPath = bookieRegistrationPath + "/" + bookieId; + } else { + regPath = bookieReadonlyRegistrationPath + "/" + bookieId; + } + doUnregisterBookie(regPath); + } + + private void doUnregisterBookie(String regPath) throws BookieException { + try { + zk.delete(regPath, -1); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new MetadataStoreException(ie); + } catch (KeeperException e) { + throw new MetadataStoreException(e); + } + } + + // + // Cookie Management + // + + @Override + public void writeCookie(BookieId bookieId, + Versioned cookieData) throws BookieException { + String zkPath = getCookiePath(bookieId); + try { + if (Version.NEW == cookieData.getVersion()) { + if (zk.exists(cookiePath, false) == null) { + try { + zk.create(cookiePath, new byte[0], zkAcls, CreateMode.PERSISTENT); + } catch (NodeExistsException nne) { + log.info("More than one bookie tried to create {} at once. Safe to ignore.", + cookiePath); + } + } + zk.create(zkPath, cookieData.getValue(), zkAcls, CreateMode.PERSISTENT); + } else { + if (!(cookieData.getVersion() instanceof LongVersion)) { + throw new BookieIllegalOpException("Invalid version type, expected it to be LongVersion"); + } + zk.setData( + zkPath, + cookieData.getValue(), + (int) ((LongVersion) cookieData.getVersion()).getLongVersion()); + } + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new MetadataStoreException("Interrupted writing cookie for bookie " + bookieId, ie); + } catch (NoNodeException nne) { + throw new CookieNotFoundException(bookieId.toString()); + } catch (NodeExistsException nee) { + throw new CookieExistException(bookieId.toString()); + } catch (KeeperException e) { + throw new MetadataStoreException("Failed to write cookie for bookie " + bookieId); + } + } + + @Override + public Versioned readCookie(BookieId bookieId) throws BookieException { + String zkPath = getCookiePath(bookieId); + try { + Stat stat = zk.exists(zkPath, false); + byte[] data = zk.getData(zkPath, false, stat); + // sets stat version from ZooKeeper + LongVersion version = new LongVersion(stat.getVersion()); + return new Versioned<>(data, version); + } catch (NoNodeException nne) { + throw new CookieNotFoundException(bookieId.toString()); + } catch (KeeperException | InterruptedException e) { + throw new MetadataStoreException("Failed to read cookie for bookie " + bookieId); + } + } + + @Override + public void removeCookie(BookieId bookieId, Version version) throws BookieException { + String zkPath = getCookiePath(bookieId); + try { + zk.delete(zkPath, (int) ((LongVersion) version).getLongVersion()); + } catch (NoNodeException e) { + throw new CookieNotFoundException(bookieId.toString()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new MetadataStoreException("Interrupted deleting cookie for bookie " + bookieId, e); + } catch (KeeperException e) { + throw new MetadataStoreException("Failed to delete cookie for bookie " + bookieId); + } + + log.info("Removed cookie from {} for bookie {}.", cookiePath, bookieId); + } + + + @Override + public String getClusterInstanceId() throws BookieException { + String instanceId = null; + try { + if (zk.exists(ledgersRootPath, null) == null) { + log.error("BookKeeper metadata doesn't exist in zookeeper. " + + "Has the cluster been initialized? " + + "Try running bin/bookkeeper shell metaformat"); + throw new KeeperException.NoNodeException("BookKeeper metadata"); + } + try { + byte[] data = zk.getData(ledgersRootPath + "/" + + INSTANCEID, false, null); + instanceId = new String(data, UTF_8); + } catch (KeeperException.NoNodeException e) { + log.info("INSTANCEID not exists in zookeeper. Not considering it for data verification"); + } + } catch (KeeperException | InterruptedException e) { + throw new MetadataStoreException("Failed to get cluster instance id", e); + } + return instanceId; + } + + @Override + public boolean prepareFormat() throws Exception { + boolean ledgerRootExists = null != zk.exists(ledgersRootPath, false); + boolean availableNodeExists = null != zk.exists(bookieRegistrationPath, false); + // Create ledgers root node if not exists + if (!ledgerRootExists) { + ZkUtils.createFullPathOptimistic(zk, ledgersRootPath, "".getBytes(StandardCharsets.UTF_8), zkAcls, + CreateMode.PERSISTENT); + } + // create available bookies node if not exists + if (!availableNodeExists) { + zk.create(bookieRegistrationPath, "".getBytes(StandardCharsets.UTF_8), zkAcls, CreateMode.PERSISTENT); + } + + // create readonly bookies node if not exists + if (null == zk.exists(bookieReadonlyRegistrationPath, false)) { + zk.create(bookieReadonlyRegistrationPath, new byte[0], zkAcls, CreateMode.PERSISTENT); + } + + return ledgerRootExists; + } + + @Override + public boolean initNewCluster() throws Exception { + String zkServers = ZKMetadataDriverBase.resolveZkServers(conf); + String instanceIdPath = ledgersRootPath + "/" + INSTANCEID; + log.info("Initializing ZooKeeper metadata for new cluster, ZKServers: {} ledger root path: {}", zkServers, + ledgersRootPath); + + boolean ledgerRootExists = null != zk.exists(ledgersRootPath, false); + + if (ledgerRootExists) { + log.error("Ledger root path: {} already exists", ledgersRootPath); + return false; + } + + List multiOps = Lists.newArrayListWithExpectedSize(4); + + // Create ledgers root node + multiOps.add(Op.create(ledgersRootPath, EMPTY_BYTE_ARRAY, zkAcls, CreateMode.PERSISTENT)); + + // create available bookies node + multiOps.add(Op.create(bookieRegistrationPath, EMPTY_BYTE_ARRAY, zkAcls, CreateMode.PERSISTENT)); + + // create readonly bookies node + multiOps.add(Op.create( + bookieReadonlyRegistrationPath, + EMPTY_BYTE_ARRAY, + zkAcls, + CreateMode.PERSISTENT)); + + // create INSTANCEID + String instanceId = UUID.randomUUID().toString(); + multiOps.add(Op.create(instanceIdPath, instanceId.getBytes(UTF_8), + zkAcls, CreateMode.PERSISTENT)); + + // execute the multi ops + zk.multi(multiOps); + + // creates the new layout and stores in zookeeper + AbstractZkLedgerManagerFactory.newLedgerManagerFactory(conf, layoutManager); + + log.info("Successfully initiated cluster. ZKServers: {} ledger root path: {} instanceId: {}", zkServers, + ledgersRootPath, instanceId); + return true; + } + + @Override + public boolean nukeExistingCluster() throws Exception { + String zkServers = ZKMetadataDriverBase.resolveZkServers(conf); + log.info("Nuking ZooKeeper metadata of existing cluster, ZKServers: {} ledger root path: {}", + zkServers, ledgersRootPath); + + boolean ledgerRootExists = null != zk.exists(ledgersRootPath, false); + if (!ledgerRootExists) { + log.info("There is no existing cluster with ledgersRootPath: {} in ZKServers: {}, " + + "so exiting nuke operation", ledgersRootPath, zkServers); + return true; + } + + boolean availableNodeExists = null != zk.exists(bookieRegistrationPath, false); + try (RegistrationClient regClient = new ZKRegistrationClient( + zk, + ledgersRootPath, + null, + false + )) { + if (availableNodeExists) { + Collection rwBookies = FutureUtils + .result(regClient.getWritableBookies(), EXCEPTION_FUNC).getValue(); + if (rwBookies != null && !rwBookies.isEmpty()) { + log.error("Bookies are still up and connected to this cluster, " + + "stop all bookies before nuking the cluster"); + return false; + } + + boolean readonlyNodeExists = null != zk.exists(bookieReadonlyRegistrationPath, false); + if (readonlyNodeExists) { + Collection roBookies = FutureUtils + .result(regClient.getReadOnlyBookies(), EXCEPTION_FUNC).getValue(); + if (roBookies != null && !roBookies.isEmpty()) { + log.error("Readonly Bookies are still up and connected to this cluster, " + + "stop all bookies before nuking the cluster"); + return false; + } + } + } + } + + LedgerManagerFactory ledgerManagerFactory = + AbstractZkLedgerManagerFactory.newLedgerManagerFactory(conf, layoutManager); + return ledgerManagerFactory.validateAndNukeExistingCluster(conf, layoutManager); + } + + @Override + public boolean format() throws Exception { + // Clear underreplicated ledgers + try { + ZKUtil.deleteRecursive(zk, ZkLedgerUnderreplicationManager.getBasePath(ledgersRootPath) + + BookKeeperConstants.DEFAULT_ZK_LEDGERS_ROOT_PATH); + } catch (KeeperException.NoNodeException e) { + if (log.isDebugEnabled()) { + log.debug("underreplicated ledgers root path node not exists in zookeeper to delete"); + } + } + + // Clear underreplicatedledger locks + try { + ZKUtil.deleteRecursive(zk, ZkLedgerUnderreplicationManager.getBasePath(ledgersRootPath) + '/' + + BookKeeperConstants.UNDER_REPLICATION_LOCK); + } catch (KeeperException.NoNodeException e) { + if (log.isDebugEnabled()) { + log.debug("underreplicatedledger locks node not exists in zookeeper to delete"); + } + } + + // Clear the cookies + try { + ZKUtil.deleteRecursive(zk, cookiePath); + } catch (KeeperException.NoNodeException e) { + if (log.isDebugEnabled()) { + log.debug("cookies node not exists in zookeeper to delete"); + } + } + + // Clear the INSTANCEID + try { + zk.delete(ledgersRootPath + "/" + BookKeeperConstants.INSTANCEID, -1); + } catch (KeeperException.NoNodeException e) { + if (log.isDebugEnabled()) { + log.debug("INSTANCEID not exists in zookeeper to delete"); + } + } + + // create INSTANCEID + String instanceId = UUID.randomUUID().toString(); + zk.create(ledgersRootPath + "/" + BookKeeperConstants.INSTANCEID, + instanceId.getBytes(StandardCharsets.UTF_8), zkAcls, CreateMode.PERSISTENT); + + log.info("Successfully formatted BookKeeper metadata"); + return true; + } + + @Override + public boolean isBookieRegistered(BookieId bookieId) throws BookieException { + String regPath = bookieRegistrationPath + "/" + bookieId; + String readonlyRegPath = bookieReadonlyRegistrationPath + "/" + bookieId; + try { + return ((null != zk.exists(regPath, false)) || (null != zk.exists(readonlyRegPath, false))); + } catch (KeeperException e) { + log.error("ZK exception while checking registration ephemeral znodes for BookieId: {}", bookieId, e); + throw new MetadataStoreException(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.error("InterruptedException while checking registration ephemeral znodes for BookieId: {}", bookieId, + e); + throw new MetadataStoreException(e); + } + } + + @Override + public void addRegistrationListener(RegistrationListener listener) { + listeners.add(listener); + } + + public void betweenRegisterReadOnlyBookie(Function fn) { + hookOnRegisterReadOnly = fn; + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerManagerIteratorTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerManagerIteratorTest.java index b64cc964a999c..f8a51602686ed 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerManagerIteratorTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerManagerIteratorTest.java @@ -403,6 +403,7 @@ public void checkConcurrentModifications(String provider, Supplier urlSu final long start = MathUtils.nowInNano(); final CountDownLatch latch = new CountDownLatch(1); ArrayList> futures = new ArrayList<>(); + @Cleanup("shutdownNow") ExecutorService executor = Executors.newCachedThreadPool(); final ConcurrentSkipListSet createdLedgers = new ConcurrentSkipListSet<>(); for (int i = 0; i < numWriters; ++i) { diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java index 0df325b3c57a0..0e9c781fb9143 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java @@ -23,12 +23,14 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.google.protobuf.TextFormat; +import java.lang.reflect.Field; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -38,6 +40,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; import lombok.Cleanup; @@ -54,6 +57,7 @@ import org.apache.bookkeeper.util.BookKeeperConstants; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.metadata.BaseMetadataStoreTest; +import org.apache.pulsar.metadata.api.GetResult; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.NotificationType; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; @@ -296,6 +300,69 @@ public void testMarkingAsReplicated(String provider, Supplier urlSupplie assertEquals(l, lB.get(), "Should be the ledger I marked"); } + + @Test(timeOut = 10000) + public void testZkMetasStoreMarkReplicatedDeleteEmptyParentNodes() throws Exception { + methodSetup(stringSupplier(() -> zks.getConnectionString())); + + String missingReplica = "localhost:3181"; + + @Cleanup + LedgerUnderreplicationManager m1 = lmf.newLedgerUnderreplicationManager(); + + Long ledgerA = 0xfeadeefdacL; + m1.markLedgerUnderreplicated(ledgerA, missingReplica); + + Field storeField = m1.getClass().getDeclaredField("store"); + storeField.setAccessible(true); + MetadataStoreExtended metadataStore = (MetadataStoreExtended) storeField.get(m1); + + String fiveLevelPath = PulsarLedgerUnderreplicationManager.getUrLedgerPath(urLedgerPath, ledgerA); + Optional getResult = metadataStore.get(fiveLevelPath).get(1, TimeUnit.SECONDS); + assertTrue(getResult.isPresent()); + + String fourLevelPath = fiveLevelPath.substring(0, fiveLevelPath.lastIndexOf("/")); + getResult = metadataStore.get(fourLevelPath).get(1, TimeUnit.SECONDS); + assertTrue(getResult.isPresent()); + + String threeLevelPath = fourLevelPath.substring(0, fourLevelPath.lastIndexOf("/")); + getResult = metadataStore.get(threeLevelPath).get(1, TimeUnit.SECONDS); + assertTrue(getResult.isPresent()); + + String twoLevelPath = fourLevelPath.substring(0, threeLevelPath.lastIndexOf("/")); + getResult = metadataStore.get(twoLevelPath).get(1, TimeUnit.SECONDS); + assertTrue(getResult.isPresent()); + + String oneLevelPath = fourLevelPath.substring(0, twoLevelPath.lastIndexOf("/")); + getResult = metadataStore.get(oneLevelPath).get(1, TimeUnit.SECONDS); + assertTrue(getResult.isPresent()); + + getResult = metadataStore.get(urLedgerPath).get(1, TimeUnit.SECONDS); + assertTrue(getResult.isPresent()); + + long ledgerToRereplicate = m1.getLedgerToRereplicate(); + assertEquals(ledgerToRereplicate, ledgerA); + m1.markLedgerReplicated(ledgerA); + + getResult = metadataStore.get(fiveLevelPath).get(1, TimeUnit.SECONDS); + assertFalse(getResult.isPresent()); + + getResult = metadataStore.get(fourLevelPath).get(1, TimeUnit.SECONDS); + assertFalse(getResult.isPresent()); + + getResult = metadataStore.get(threeLevelPath).get(1, TimeUnit.SECONDS); + assertFalse(getResult.isPresent()); + + getResult = metadataStore.get(twoLevelPath).get(1, TimeUnit.SECONDS); + assertFalse(getResult.isPresent()); + + getResult = metadataStore.get(oneLevelPath).get(1, TimeUnit.SECONDS); + assertFalse(getResult.isPresent()); + + getResult = metadataStore.get(urLedgerPath).get(1, TimeUnit.SECONDS); + assertTrue(getResult.isPresent()); + } + /** * Test releasing of a ledger * A ledger is released when a client decides it does not want @@ -548,6 +615,8 @@ public void testDisableLedgerReplication(String provider, Supplier urlSu final String missingReplica = "localhost:3181"; // disabling replication + AtomicInteger callbackCount = new AtomicInteger(); + lum.notifyLedgerReplicationEnabled((rc, result) -> callbackCount.incrementAndGet()); lum.disableLedgerReplication(); log.info("Disabled Ledeger Replication"); @@ -565,6 +634,7 @@ public void testDisableLedgerReplication(String provider, Supplier urlSu } catch (TimeoutException te) { // expected behaviour, as the replication is disabled } + assertEquals(callbackCount.get(), 1, "Notify callback times mismatch"); } /** @@ -585,7 +655,8 @@ public void testEnableLedgerReplication(String provider, Supplier urlSup log.debug("Unexpected exception while marking urLedger", e); fail("Unexpected exception while marking urLedger" + e.getMessage()); } - + AtomicInteger callbackCount = new AtomicInteger(); + lum.notifyLedgerReplicationEnabled((rc, result) -> callbackCount.incrementAndGet()); // disabling replication lum.disableLedgerReplication(); log.debug("Disabled Ledeger Replication"); @@ -622,6 +693,7 @@ public void testEnableLedgerReplication(String provider, Supplier urlSup znodeLatch.await(5, TimeUnit.SECONDS); log.debug("Enabled Ledeger Replication"); assertEquals(znodeLatch.getCount(), 0, "Failed to disable ledger replication!"); + assertEquals(callbackCount.get(), 2, "Notify callback times mismatch"); } finally { thread1.interrupt(); } @@ -683,6 +755,17 @@ public void testReplicasCheckCTime(String provider, Supplier urlSupplier assertEquals(underReplicaMgr1.getReplicasCheckCTime(), curTime); } + @Test(timeOut = 60000, dataProvider = "impl") + public void testLostBookieRecoveryDelay(String provider, Supplier urlSupplier) throws Exception { + methodSetup(urlSupplier); + + AtomicInteger callbackCount = new AtomicInteger(); + lum.notifyLostBookieRecoveryDelayChanged((rc, result) -> callbackCount.incrementAndGet()); + // disabling replication + lum.setLostBookieRecoveryDelay(10); + Awaitility.await().until(() -> callbackCount.get() == 2); + } + private void verifyMarkLedgerUnderreplicated(Collection missingReplica) throws Exception { Long ledgerA = 0xfeadeefdacL; String znodeA = getUrLedgerZnode(ledgerA); diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerAuditorManagerTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerAuditorManagerTest.java index b5232f49bf44a..bb3d157b3c5c8 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerAuditorManagerTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerAuditorManagerTest.java @@ -77,6 +77,7 @@ public void testSimple(String provider, Supplier urlSupplier) throws Exc methodSetup(urlSupplier); + @Cleanup LedgerAuditorManager lam1 = new PulsarLedgerAuditorManager(store1, ledgersRootPath); assertNull(lam1.getCurrentAuditor()); @@ -89,6 +90,7 @@ public void testSimple(String provider, Supplier urlSupplier) throws Exc @Cleanup("shutdownNow") ExecutorService executor = Executors.newCachedThreadPool(); + @Cleanup LedgerAuditorManager lam2 = new PulsarLedgerAuditorManager(store2, ledgersRootPath); assertEquals(lam2.getCurrentAuditor(), BookieId.parse("bookie-1:3181")); diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClientTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClientTest.java index f599451c00710..5660b3518f1aa 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClientTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClientTest.java @@ -42,9 +42,8 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.conf.AbstractConfiguration; -import org.apache.bookkeeper.discover.BookieServiceInfo; -import org.apache.bookkeeper.discover.RegistrationClient; -import org.apache.bookkeeper.discover.RegistrationManager; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.discover.*; import org.apache.bookkeeper.net.BookieId; import org.apache.bookkeeper.net.BookieSocketAddress; import org.apache.bookkeeper.versioning.Version; @@ -52,6 +51,7 @@ import org.apache.pulsar.metadata.BaseMetadataStoreTest; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.apache.zookeeper.ZooKeeper; import org.awaitility.Awaitility; import org.testng.annotations.Test; @@ -126,7 +126,7 @@ public void testGetReadonlyBookies(String provider, Supplier urlSupplier public void testGetBookieServiceInfo(String provider, Supplier urlSupplier) throws Exception { @Cleanup MetadataStoreExtended store = MetadataStoreExtended.create(urlSupplier.get(), - MetadataStoreConfig.builder().fsyncEnable(false).build()); + MetadataStoreConfig.builder().fsyncEnable(false).build()); String ledgersRoot = "/test/ledgers-" + UUID.randomUUID(); @@ -168,10 +168,10 @@ public void testGetBookieServiceInfo(String provider, Supplier urlSuppli getAndVerifyAllBookies(rc, addresses); Awaitility.await().untilAsserted(() -> { - for (BookieId address : addresses) { - BookieServiceInfo bookieServiceInfo = rc.getBookieServiceInfo(address).get().getValue(); - compareBookieServiceInfo(bookieServiceInfo, bookieServiceInfos.get(address)); - }}); + for (BookieId address : addresses) { + BookieServiceInfo bookieServiceInfo = rc.getBookieServiceInfo(address).get().getValue(); + compareBookieServiceInfo(bookieServiceInfo, bookieServiceInfos.get(address)); + }}); // shutdown the bookies (but keep the cookie) for (BookieId address : addresses) { @@ -184,12 +184,12 @@ public void testGetBookieServiceInfo(String provider, Supplier urlSuppli // getBookieServiceInfo should fail with BKBookieHandleNotAvailableException Awaitility.await().untilAsserted(() -> { - for (BookieId address : addresses) { - assertTrue( - expectThrows(ExecutionException.class, () -> { - rc.getBookieServiceInfo(address).get(); - }).getCause() instanceof BKException.BKBookieHandleNotAvailableException); - }}); + for (BookieId address : addresses) { + assertTrue( + expectThrows(ExecutionException.class, () -> { + rc.getBookieServiceInfo(address).get(); + }).getCause() instanceof BKException.BKBookieHandleNotAvailableException); + }}); // restart the bookies, all writable @@ -241,12 +241,12 @@ public void testGetBookieServiceInfo(String provider, Supplier urlSuppli .await() .ignoreExceptionsMatching(e -> e.getCause() instanceof BKException.BKBookieHandleNotAvailableException) .untilAsserted(() -> { - // verify that infos are updated - for (BookieId address : addresses) { - BookieServiceInfo bookieServiceInfo = rc.getBookieServiceInfo(address).get().getValue(); - compareBookieServiceInfo(bookieServiceInfo, bookieServiceInfos.get(address)); - } - }); + // verify that infos are updated + for (BookieId address : addresses) { + BookieServiceInfo bookieServiceInfo = rc.getBookieServiceInfo(address).get().getValue(); + compareBookieServiceInfo(bookieServiceInfo, bookieServiceInfos.get(address)); + } + }); } @@ -318,7 +318,7 @@ private void testWatchBookiesSuccess(String provider, Supplier urlSuppli @Cleanup MetadataStoreExtended store = MetadataStoreExtended.create(urlSupplier.get(), - MetadataStoreConfig.builder().fsyncEnable(false).build()); + MetadataStoreConfig.builder().fsyncEnable(false).build()); String ledgersRoot = "/test/ledgers-" + UUID.randomUUID(); @@ -357,4 +357,89 @@ private void testWatchBookiesSuccess(String provider, Supplier urlSuppli }); } + + @Test + public void testNetworkDelayWithBkZkManager() throws Throwable { + final String zksConnectionString = zks.getConnectionString(); + final String ledgersRoot = "/test/ledgers-" + UUID.randomUUID(); + // prepare registration manager + @Cleanup + ZooKeeper zk = new ZooKeeper(zksConnectionString, 5000, null); + final ServerConfiguration serverConfiguration = new ServerConfiguration(); + serverConfiguration.setZkLedgersRootPath(ledgersRoot); + final FaultInjectableZKRegistrationManager rm = new FaultInjectableZKRegistrationManager(serverConfiguration, zk); + rm.prepareFormat(); + // prepare registration client + @Cleanup + MetadataStoreExtended store = MetadataStoreExtended.create(zksConnectionString, + MetadataStoreConfig.builder().fsyncEnable(false).build()); + @Cleanup + RegistrationClient rc1 = new PulsarRegistrationClient(store, ledgersRoot); + @Cleanup + RegistrationClient rc2 = new PulsarRegistrationClient(store, ledgersRoot); + + final List addresses = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + addresses.add(BookieId.parse("BOOKIE-" + i)); + } + final Map bookieServiceInfos = new HashMap<>(); + + int port = 223; + for (BookieId address : addresses) { + BookieServiceInfo info = new BookieServiceInfo(); + BookieServiceInfo.Endpoint endpoint = new BookieServiceInfo.Endpoint(); + endpoint.setAuth(Collections.emptyList()); + endpoint.setExtensions(Collections.emptyList()); + endpoint.setId("id"); + endpoint.setHost("localhost"); + endpoint.setPort(port++); + endpoint.setProtocol("bookie-rpc"); + info.setEndpoints(Arrays.asList(endpoint)); + bookieServiceInfos.put(address, info); + rm.registerBookie(address, false, info); + // write the cookie + rm.writeCookie(address, new Versioned<>(new byte[0], Version.NEW)); + } + + // trigger loading the BookieServiceInfo in the local cache + getAndVerifyAllBookies(rc1, addresses); + getAndVerifyAllBookies(rc2, addresses); + + Awaitility.await().untilAsserted(() -> { + for (BookieId address : addresses) { + compareBookieServiceInfo(rc1.getBookieServiceInfo(address).get().getValue(), + bookieServiceInfos.get(address)); + compareBookieServiceInfo(rc2.getBookieServiceInfo(address).get().getValue(), + bookieServiceInfos.get(address)); + } + }); + + // verified the init status. + + + // mock network delay + rm.betweenRegisterReadOnlyBookie(__ -> { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return null; + }); + + for (int i = 0; i < addresses.size() / 2; i++) { + final BookieId bkId = addresses.get(i); + // turn some bookies to be read only. + rm.registerBookie(bkId, true, bookieServiceInfos.get(bkId)); + } + + Awaitility.await().untilAsserted(() -> { + for (BookieId address : addresses) { + compareBookieServiceInfo(rc1.getBookieServiceInfo(address).get().getValue(), + bookieServiceInfos.get(address)); + compareBookieServiceInfo(rc2.getBookieServiceInfo(address).get().getValue(), + bookieServiceInfos.get(address)); + } + }); + } } diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/LocalMemoryMetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/LocalMemoryMetadataStoreTest.java index 3fabe9647eb34..caca16ff538a4 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/LocalMemoryMetadataStoreTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/LocalMemoryMetadataStoreTest.java @@ -206,8 +206,8 @@ public String getClusterName() { } @Override - public void close() { - // No-op + public CompletableFuture closeAsync() { + return CompletableFuture.completedFuture(null); } } diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImplTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImplTest.java index c0159be4303bc..6ede02b67136e 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImplTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImplTest.java @@ -20,6 +20,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import io.opentelemetry.api.OpenTelemetry; import lombok.Cleanup; import org.apache.pulsar.metadata.api.GetResult; import org.apache.pulsar.metadata.api.MetadataStore; @@ -91,7 +92,7 @@ public MetadataStore create(String metadataURL, MetadataStoreConfig metadataStor public static class MyMetadataStore extends AbstractMetadataStore { protected MyMetadataStore() { - super("custom"); + super("custom", OpenTelemetry.noop()); } @Override diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/RocksdbMetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/RocksdbMetadataStoreTest.java index 7700fb3654d7e..e0f509cbce9d7 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/RocksdbMetadataStoreTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/RocksdbMetadataStoreTest.java @@ -132,6 +132,7 @@ public void testMultipleInstances() throws Exception { store1.close(); store2.put("/test-2", new byte[0], Optional.empty()).join(); Assert.assertTrue(store2.exists("/test-2").join()); + store2.close(); FileUtils.deleteQuietly(tempDir.toFile()); } diff --git a/pulsar-metadata/src/test/resources/oxia_client.conf b/pulsar-metadata/src/test/resources/oxia_client.conf new file mode 100644 index 0000000000000..3e92f05a34019 --- /dev/null +++ b/pulsar-metadata/src/test/resources/oxia_client.conf @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +sessionTimeout=60000 diff --git a/pulsar-metadata/src/test/resources/zk_client_disabled_sasl.conf b/pulsar-metadata/src/test/resources/zk_client_disabled_sasl.conf new file mode 100644 index 0000000000000..9e0f6e8fd0fd2 --- /dev/null +++ b/pulsar-metadata/src/test/resources/zk_client_disabled_sasl.conf @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +zookeeper.sasl.client=false diff --git a/pulsar-opentelemetry/pom.xml b/pulsar-opentelemetry/pom.xml new file mode 100644 index 0000000000000..4e43b95813c61 --- /dev/null +++ b/pulsar-opentelemetry/pom.xml @@ -0,0 +1,149 @@ + + + + 4.0.0 + + org.apache.pulsar + pulsar + 4.0.0-SNAPSHOT + + + pulsar-opentelemetry + OpenTelemetry Integration + + + + + io.opentelemetry + opentelemetry-exporter-otlp + + + io.opentelemetry + opentelemetry-exporter-prometheus + + + io.opentelemetry + opentelemetry-sdk + + + io.opentelemetry + opentelemetry-sdk-extension-autoconfigure + + + io.opentelemetry.instrumentation + opentelemetry-resources + + + io.opentelemetry.semconv + opentelemetry-semconv + + + io.opentelemetry.instrumentation + opentelemetry-runtime-telemetry-java17 + + + + com.google.guava + guava + + + + org.apache.commons + commons-lang3 + + + + + org.apache.pulsar + pulsar-broker-common + ${project.version} + test-jar + test + + + + io.rest-assured + rest-assured + test + + + + org.awaitility + awaitility + test + + + + io.opentelemetry + opentelemetry-sdk-testing + test + + + + + + + org.gaul + modernizer-maven-plugin + + true + 8 + + + + modernizer + verify + + modernizer + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + jvm + + + + + + diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarTransactionHandle.java b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/Constants.java similarity index 79% rename from pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarTransactionHandle.java rename to pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/Constants.java index c1e3236da4c06..6d61cafb5a01a 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarTransactionHandle.java +++ b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/Constants.java @@ -16,13 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.sql.presto; - -import io.trino.spi.connector.ConnectorTransactionHandle; +package org.apache.pulsar.opentelemetry; /** - * A handle for transactions. + * Common OpenTelemetry constants to be used by Pulsar components. */ -public enum PulsarTransactionHandle implements ConnectorTransactionHandle { - INSTANCE +public interface Constants { + + String BROKER_INSTRUMENTATION_SCOPE_NAME = "org.apache.pulsar.broker"; + } diff --git a/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryAttributes.java b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryAttributes.java new file mode 100644 index 0000000000000..6eb84e94bc61b --- /dev/null +++ b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryAttributes.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.opentelemetry; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import java.util.List; + +/** + * Common OpenTelemetry attributes to be used by Pulsar components. + */ +public interface OpenTelemetryAttributes { + /** + * The name of the Pulsar cluster. This attribute is automatically added to all signals by + * {@link OpenTelemetryService}. + */ + AttributeKey PULSAR_CLUSTER = AttributeKey.stringKey("pulsar.cluster"); + + /** + * The name of the Pulsar namespace. + */ + AttributeKey PULSAR_NAMESPACE = AttributeKey.stringKey("pulsar.namespace"); + + /** + * The name of the Pulsar tenant. + */ + AttributeKey PULSAR_TENANT = AttributeKey.stringKey("pulsar.tenant"); + + /** + * The Pulsar topic domain. + */ + AttributeKey PULSAR_DOMAIN = AttributeKey.stringKey("pulsar.domain"); + + /** + * The name of the Pulsar topic. + */ + AttributeKey PULSAR_TOPIC = AttributeKey.stringKey("pulsar.topic"); + + /** + * The partition index of a Pulsar topic. + */ + AttributeKey PULSAR_PARTITION_INDEX = AttributeKey.longKey("pulsar.partition.index"); + + /** + * The name of the Pulsar subscription. + */ + AttributeKey PULSAR_SUBSCRIPTION_NAME = AttributeKey.stringKey("pulsar.subscription.name"); + + /** + * The type of the Pulsar subscription. + */ + AttributeKey PULSAR_SUBSCRIPTION_TYPE = AttributeKey.stringKey("pulsar.subscription.type"); + + /** + * The name of the Pulsar consumer. + */ + AttributeKey PULSAR_CONSUMER_NAME = AttributeKey.stringKey("pulsar.consumer.name"); + + /** + * The ID of the Pulsar consumer. + */ + AttributeKey PULSAR_CONSUMER_ID = AttributeKey.longKey("pulsar.consumer.id"); + + /** + * The consumer metadata properties, as a list of "key:value" pairs. + */ + AttributeKey> PULSAR_CONSUMER_METADATA = AttributeKey.stringArrayKey("pulsar.consumer.metadata"); + + /** + * The UTC timestamp of the Pulsar consumer creation. + */ + AttributeKey PULSAR_CONSUMER_CONNECTED_SINCE = AttributeKey.longKey("pulsar.consumer.connected_since"); + + /** + * The name of the Pulsar producer. + */ + AttributeKey PULSAR_PRODUCER_NAME = AttributeKey.stringKey("pulsar.producer.name"); + + /** + * The ID of the Pulsar producer. + */ + AttributeKey PULSAR_PRODUCER_ID = AttributeKey.longKey("pulsar.producer.id"); + + /** + * The access mode of the Pulsar producer. + */ + AttributeKey PULSAR_PRODUCER_ACCESS_MODE = AttributeKey.stringKey("pulsar.producer.access_mode"); + + /** + * The address of the Pulsar client. + */ + AttributeKey PULSAR_CLIENT_ADDRESS = AttributeKey.stringKey("pulsar.client.address"); + + /** + * The version of the Pulsar client. + */ + AttributeKey PULSAR_CLIENT_VERSION = AttributeKey.stringKey("pulsar.client.version"); + + AttributeKey PULSAR_CONNECTION_RATE_LIMIT_OPERATION_NAME = + AttributeKey.stringKey("pulsar.connection.rate_limit.operation.name"); + enum ConnectionRateLimitOperationName { + PAUSED, + RESUMED, + THROTTLED, + UNTHROTTLED; + public final Attributes attributes = + Attributes.of(PULSAR_CONNECTION_RATE_LIMIT_OPERATION_NAME, name().toLowerCase()); + } + + /** + * The status of the Pulsar transaction. + */ + AttributeKey PULSAR_TRANSACTION_STATUS = AttributeKey.stringKey("pulsar.transaction.status"); + + enum TransactionStatus { + ABORTED, + ACTIVE, + COMMITTED, + CREATED, + TIMEOUT; + public final Attributes attributes = Attributes.of(PULSAR_TRANSACTION_STATUS, name().toLowerCase()); + } + + /** + * The status of the Pulsar transaction ack store operation. + */ + AttributeKey PULSAR_TRANSACTION_ACK_STORE_OPERATION_STATUS = + AttributeKey.stringKey("pulsar.transaction.pending.ack.store.operation.status"); + enum TransactionPendingAckOperationStatus { + SUCCESS, + FAILURE; + public final Attributes attributes = + Attributes.of(PULSAR_TRANSACTION_ACK_STORE_OPERATION_STATUS, name().toLowerCase()); + } + + /** + * The ID of the Pulsar transaction coordinator. + */ + AttributeKey PULSAR_TRANSACTION_COORDINATOR_ID = AttributeKey.longKey("pulsar.transaction.coordinator.id"); + + /** + * The status of the Pulsar transaction buffer client operation. + */ + AttributeKey PULSAR_TRANSACTION_BUFFER_CLIENT_OPERATION_STATUS = + AttributeKey.stringKey("pulsar.transaction.buffer.client.operation.status"); + enum TransactionBufferClientOperationStatus { + SUCCESS, + FAILURE; + public final Attributes attributes = + Attributes.of(PULSAR_TRANSACTION_BUFFER_CLIENT_OPERATION_STATUS, name().toLowerCase()); + } + + /** + * The status of the Pulsar compaction operation. + */ + AttributeKey PULSAR_COMPACTION_STATUS = AttributeKey.stringKey("pulsar.compaction.status"); + enum CompactionStatus { + SUCCESS, + FAILURE; + public final Attributes attributes = Attributes.of(PULSAR_COMPACTION_STATUS, name().toLowerCase()); + } + + /** + * The type of the backlog quota. + */ + AttributeKey PULSAR_BACKLOG_QUOTA_TYPE = AttributeKey.stringKey("pulsar.backlog.quota.type"); + enum BacklogQuotaType { + SIZE, + TIME; + public final Attributes attributes = Attributes.of(PULSAR_BACKLOG_QUOTA_TYPE, name().toLowerCase()); + } + + // Managed Ledger Attributes + /** + * The name of the managed ledger. + */ + AttributeKey ML_LEDGER_NAME = AttributeKey.stringKey("pulsar.managed_ledger.name"); + + /** + * The name of the managed cursor. + */ + AttributeKey ML_CURSOR_NAME = AttributeKey.stringKey("pulsar.managed_ledger.cursor.name"); + + /** + * The status of the managed cursor operation. + */ + AttributeKey ML_CURSOR_OPERATION_STATUS = + AttributeKey.stringKey("pulsar.managed_ledger.cursor.operation.status"); + enum ManagedCursorOperationStatus { + SUCCESS, + FAILURE; + public final Attributes attributes = Attributes.of(ML_CURSOR_OPERATION_STATUS, name().toLowerCase()); + } + + AttributeKey MANAGED_LEDGER_READ_INFLIGHT_USAGE = + AttributeKey.stringKey("pulsar.managed_ledger.inflight.read.usage.state"); + enum InflightReadLimiterUtilization { + USED, + FREE; + public final Attributes attributes = Attributes.of(MANAGED_LEDGER_READ_INFLIGHT_USAGE, name().toLowerCase()); + } + + /** + * The name of the remote cluster for a Pulsar replicator. + */ + AttributeKey PULSAR_REPLICATION_REMOTE_CLUSTER_NAME = + AttributeKey.stringKey("pulsar.replication.remote.cluster.name"); + + AttributeKey PULSAR_CONNECTION_STATUS = AttributeKey.stringKey("pulsar.connection.status"); + enum ConnectionStatus { + ACTIVE, + OPEN, + CLOSE; + public final Attributes attributes = Attributes.of(PULSAR_CONNECTION_STATUS, name().toLowerCase()); + } + + AttributeKey PULSAR_CONNECTION_CREATE_STATUS = + AttributeKey.stringKey("pulsar.connection.create.operation.status"); + enum ConnectionCreateStatus { + SUCCESS, + FAILURE; + public final Attributes attributes = Attributes.of(PULSAR_CONNECTION_CREATE_STATUS, name().toLowerCase()); + } + + // Managed Ledger Attributes + + /** + * The name of the managed ledger. + */ + AttributeKey ML_NAME = AttributeKey.stringKey("pulsar.managed_ledger.name"); + + /** + * The status of the managed ledger operation. + */ + AttributeKey ML_OPERATION_STATUS = AttributeKey.stringKey("pulsar.managed_ledger.operation.status"); + enum ManagedLedgerOperationStatus { + SUCCESS, + FAILURE; + public final Attributes attributes = Attributes.of(ML_OPERATION_STATUS, name().toLowerCase()); + }; + + /** + * The type of the pool arena. + */ + AttributeKey ML_POOL_ARENA_TYPE = AttributeKey.stringKey("pulsar.managed_ledger.pool.arena.type"); + enum PoolArenaType { + SMALL, + NORMAL, + HUGE; + public final Attributes attributes = Attributes.of(ML_POOL_ARENA_TYPE, name().toLowerCase()); + } + + /** + * The type of the pool chunk allocation. + */ + AttributeKey ML_POOL_CHUNK_ALLOCATION_TYPE = + AttributeKey.stringKey("pulsar.managed_ledger.pool.chunk.allocation.type"); + enum PoolChunkAllocationType { + ALLOCATED, + USED; + public final Attributes attributes = Attributes.of(ML_POOL_CHUNK_ALLOCATION_TYPE, name().toLowerCase()); + } + + /** + * The status of the cache entry. + */ + AttributeKey ML_CACHE_ENTRY_STATUS = AttributeKey.stringKey("pulsar.managed_ledger.cache.entry.status"); + enum CacheEntryStatus { + ACTIVE, + EVICTED, + INSERTED; + public final Attributes attributes = Attributes.of(ML_CACHE_ENTRY_STATUS, name().toLowerCase()); + } + + /** + * The result of the cache operation. + */ + AttributeKey ML_CACHE_OPERATION_STATUS = + AttributeKey.stringKey("pulsar.managed_ledger.cache.operation.status"); + enum CacheOperationStatus { + HIT, + MISS; + public final Attributes attributes = Attributes.of(ML_CACHE_OPERATION_STATUS, name().toLowerCase()); + } +} diff --git a/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java new file mode 100644 index 0000000000000..e6c6d95273e0e --- /dev/null +++ b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.opentelemetry; + +import static com.google.common.base.Preconditions.checkArgument; +import com.google.common.annotations.VisibleForTesting; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.exporter.prometheus.PrometheusHttpServer; +import io.opentelemetry.instrumentation.runtimemetrics.java17.RuntimeMetrics; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdkBuilder; +import io.opentelemetry.sdk.common.export.MemoryMode; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.semconv.ResourceAttributes; +import java.io.Closeable; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import lombok.Builder; +import org.apache.commons.lang3.StringUtils; + +/** + * Provides a common OpenTelemetry service for Pulsar components to use. Responsible for instantiating the OpenTelemetry + * SDK with a set of override properties. Once initialized, furnishes access to OpenTelemetry. + */ +public class OpenTelemetryService implements Closeable { + + public static final String OTEL_SDK_DISABLED_KEY = "otel.sdk.disabled"; + static final int MAX_CARDINALITY_LIMIT = 10000; + + private final AtomicReference openTelemetrySdkReference = new AtomicReference<>(); + + private final AtomicReference runtimeMetricsReference = new AtomicReference<>(); + + /** + * Instantiates the OpenTelemetry SDK. All attributes are overridden by system properties or environment + * variables. + * + * @param clusterName + * The name of the Pulsar cluster. Cannot be null or blank. + * @param serviceName + * The name of the service. Optional. + * @param serviceVersion + * The version of the service. Optional. + * @param builderCustomizer + * Allows customizing the SDK builder; for testing purposes only. + */ + @Builder + public OpenTelemetryService(String clusterName, + String serviceName, + String serviceVersion, + @VisibleForTesting Consumer builderCustomizer) { + checkArgument(StringUtils.isNotBlank(clusterName), "Cluster name cannot be empty"); + var sdkBuilder = AutoConfiguredOpenTelemetrySdk.builder(); + + sdkBuilder.addPropertiesSupplier(() -> Map.of( + OTEL_SDK_DISABLED_KEY, "true", + // Cardinality limit includes the overflow attribute set, so we need to add 1. + "otel.experimental.metrics.cardinality.limit", Integer.toString(MAX_CARDINALITY_LIMIT + 1), + // Reduce number of allocations by using reusable data mode. + "otel.java.experimental.exporter.memory_mode", MemoryMode.REUSABLE_DATA.name() + )); + + sdkBuilder.addResourceCustomizer( + (resource, __) -> { + var resourceBuilder = Resource.builder(); + // Do not override attributes if already set (via system properties or environment variables). + if (resource.getAttribute(OpenTelemetryAttributes.PULSAR_CLUSTER) == null) { + resourceBuilder.put(OpenTelemetryAttributes.PULSAR_CLUSTER, clusterName); + } + if (StringUtils.isNotBlank(serviceName) + && Objects.equals(Resource.getDefault().getAttribute(ResourceAttributes.SERVICE_NAME), + resource.getAttribute(ResourceAttributes.SERVICE_NAME))) { + resourceBuilder.put(ResourceAttributes.SERVICE_NAME, serviceName); + } + if (StringUtils.isNotBlank(serviceVersion) + && resource.getAttribute(ResourceAttributes.SERVICE_VERSION) == null) { + resourceBuilder.put(ResourceAttributes.SERVICE_VERSION, serviceVersion); + } + return resource.merge(resourceBuilder.build()); + }); + + sdkBuilder.addMetricReaderCustomizer((metricReader, configProperties) -> { + if (metricReader instanceof PrometheusHttpServer prometheusHttpServer) { + // At this point, the server is already started. We need to close it and create a new one with the + // correct resource attributes filter. + prometheusHttpServer.close(); + + // Allow all resource attributes to be exposed. + return prometheusHttpServer.toBuilder() + .setAllowedResourceAttributesFilter(s -> true) + .build(); + } + return metricReader; + }); + + if (builderCustomizer != null) { + builderCustomizer.accept(sdkBuilder); + } + + openTelemetrySdkReference.set(sdkBuilder.build().getOpenTelemetrySdk()); + + // For a list of exposed metrics, see https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/ + runtimeMetricsReference.set(RuntimeMetrics.builder(openTelemetrySdkReference.get()) + // disable JFR based telemetry and use only JMX telemetry + .disableAllFeatures() + // enable experimental JMX telemetry in addition + .enableExperimentalJmxTelemetry() + .build()); + } + + public OpenTelemetry getOpenTelemetry() { + return openTelemetrySdkReference.get(); + } + + @Override + public void close() { + RuntimeMetrics runtimeMetrics = runtimeMetricsReference.getAndSet(null); + if (runtimeMetrics != null) { + runtimeMetrics.close(); + } + OpenTelemetrySdk openTelemetrySdk = openTelemetrySdkReference.getAndSet(null); + if (openTelemetrySdk != null) { + openTelemetrySdk.close(); + } + } +} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoder.java b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/annotations/PulsarDeprecatedMetric.java similarity index 62% rename from pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoder.java rename to pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/annotations/PulsarDeprecatedMetric.java index d45c5ab07875e..52dbe5fa68160 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoder.java +++ b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/annotations/PulsarDeprecatedMetric.java @@ -16,25 +16,17 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.sql.presto; +package org.apache.pulsar.opentelemetry.annotations; -import io.netty.buffer.ByteBuf; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.decoder.FieldValueProvider; -import java.util.Map; -import java.util.Optional; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; /** - * RowDecoder interface for Pulsar. + * Marks a metric as deprecated and provides information about the new metric name. */ -public interface PulsarRowDecoder { - - /** - * decode byteBuf to Column FieldValueProvider. - * - * @param byteBuf - * @return - */ - Optional> decodeRow(ByteBuf byteBuf); - +@Retention(java.lang.annotation.RetentionPolicy.SOURCE) +@Target({java.lang.annotation.ElementType.TYPE, ElementType.LOCAL_VARIABLE, ElementType.FIELD}) +public @interface PulsarDeprecatedMetric { + String newMetricName() default ""; } diff --git a/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/annotations/package-info.java b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/annotations/package-info.java new file mode 100644 index 0000000000000..711884c7f610f --- /dev/null +++ b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/annotations/package-info.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Provides OpenTelemetry related annotations for Pulsar components. + * @since 3.3.0 + */ +package org.apache.pulsar.opentelemetry.annotations; \ No newline at end of file diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/protobufnative/package-info.java b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/package-info.java similarity index 85% rename from pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/protobufnative/package-info.java rename to pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/package-info.java index b49610dd8c903..9a7426aa0471d 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/protobufnative/package-info.java +++ b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/package-info.java @@ -16,7 +16,9 @@ * specific language governing permissions and limitations * under the License. */ + /** - * This package contains decoder for SchemaType.AVRO. + * Provides a wrapper layer for the OpenTelemetry API to be used in Pulsar. + * @since 3.3.0 */ -package org.apache.pulsar.sql.presto.decoder.protobufnative; \ No newline at end of file +package org.apache.pulsar.opentelemetry; \ No newline at end of file diff --git a/pulsar-opentelemetry/src/test/java/org/apache/pulsar/opentelemetry/OpenTelemetryServiceTest.java b/pulsar-opentelemetry/src/test/java/org/apache/pulsar/opentelemetry/OpenTelemetryServiceTest.java new file mode 100644 index 0000000000000..31a6c60f83afe --- /dev/null +++ b/pulsar-opentelemetry/src/test/java/org/apache/pulsar/opentelemetry/OpenTelemetryServiceTest.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.opentelemetry; + +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat; +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.satisfies; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.LongCounterBuilder; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.instrumentation.resources.JarServiceNameDetector; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdkBuilder; +import io.opentelemetry.sdk.common.InstrumentationScopeInfo; +import io.opentelemetry.sdk.metrics.export.MetricReader; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; +import io.opentelemetry.semconv.ResourceAttributes; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import lombok.Cleanup; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient; +import org.assertj.core.api.AbstractCharSequenceAssert; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class OpenTelemetryServiceTest { + + private OpenTelemetryService openTelemetryService; + private InMemoryMetricReader reader; + private Meter meter; + + @BeforeMethod + public void setup() throws Exception { + reader = InMemoryMetricReader.create(); + openTelemetryService = OpenTelemetryService.builder() + .builderCustomizer( + getBuilderCustomizer(reader, Map.of(OpenTelemetryService.OTEL_SDK_DISABLED_KEY, "false"))) + .clusterName("openTelemetryServiceTestCluster") + .build(); + meter = openTelemetryService.getOpenTelemetry().getMeter("openTelemetryServiceTestInstrument"); + } + + @AfterMethod + public void teardown() throws Exception { + openTelemetryService.close(); + reader.close(); + } + + // Customizes the SDK builder to include the MetricReader and extra properties for testing purposes. + private static Consumer getBuilderCustomizer(MetricReader extraReader, + Map extraProperties) { + return autoConfigurationCustomizer -> { + if (extraReader != null) { + autoConfigurationCustomizer.addMeterProviderCustomizer( + (sdkMeterProviderBuilder, __) -> sdkMeterProviderBuilder.registerMetricReader(extraReader)); + } + autoConfigurationCustomizer.addPropertiesSupplier(() -> extraProperties); + }; + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testClusterNameCannotBeNull() { + @Cleanup + var ots = OpenTelemetryService.builder().build(); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testClusterNameCannotBeEmpty() { + @Cleanup + var ots = OpenTelemetryService.builder().clusterName(StringUtils.EMPTY).build(); + } + + @Test + public void testResourceAttributesAreSet() throws Exception { + @Cleanup + var reader = InMemoryMetricReader.create(); + + @Cleanup + var ots = OpenTelemetryService.builder() + .builderCustomizer(getBuilderCustomizer(reader, + Map.of(OpenTelemetryService.OTEL_SDK_DISABLED_KEY, "false", + "otel.java.disabled.resource.providers", JarServiceNameDetector.class.getName()))) + .clusterName("testServiceNameAndVersion") + .serviceName("openTelemetryServiceTestService") + .serviceVersion("1.0.0") + .build(); + + assertThat(reader.collectAllMetrics()) + .allSatisfy(metric -> assertThat(metric) + .hasResourceSatisfying(resource -> resource + .hasAttribute(OpenTelemetryAttributes.PULSAR_CLUSTER, "testServiceNameAndVersion") + .hasAttribute(ResourceAttributes.SERVICE_NAME, "openTelemetryServiceTestService") + .hasAttribute(ResourceAttributes.SERVICE_VERSION, "1.0.0") + .hasAttribute(satisfies(ResourceAttributes.HOST_NAME, AbstractCharSequenceAssert::isNotBlank)))); + } + + @Test + public void testIsInstrumentationNameSetOnMeter() { + var meter = openTelemetryService.getOpenTelemetry().getMeter("testInstrumentationScope"); + meter.counterBuilder("dummyCounter").build().add(1); + assertThat(reader.collectAllMetrics()) + .anySatisfy(metricData -> assertThat(metricData) + .hasInstrumentationScope(InstrumentationScopeInfo.create("testInstrumentationScope"))); + } + + @Test + public void testMetricCardinalityIsSet() { + var prometheusExporterPort = 9464; + @Cleanup + var ots = OpenTelemetryService.builder() + .builderCustomizer(getBuilderCustomizer(null, + Map.of(OpenTelemetryService.OTEL_SDK_DISABLED_KEY, "false", + "otel.metrics.exporter", "prometheus", + "otel.exporter.prometheus.port", Integer.toString(prometheusExporterPort)))) + .clusterName("openTelemetryServiceCardinalityTestCluster") + .build(); + var meter = ots.getOpenTelemetry().getMeter("openTelemetryMetricCardinalityTest"); + var counter = meter.counterBuilder("dummyCounter").build(); + for (int i = 0; i < OpenTelemetryService.MAX_CARDINALITY_LIMIT + 100; i++) { + counter.add(1, Attributes.of(AttributeKey.stringKey("attribute"), "value" + i)); + } + + Awaitility.waitAtMost(30, TimeUnit.SECONDS).ignoreExceptions().until(() -> { + var client = new PrometheusMetricsClient("localhost", prometheusExporterPort); + var allMetrics = client.getMetrics(); + var actualMetrics = allMetrics.findByNameAndLabels("dummyCounter_total"); + var overflowMetric = allMetrics.findByNameAndLabels("dummyCounter_total", "otel_metric_overflow", "true"); + return actualMetrics.size() == OpenTelemetryService.MAX_CARDINALITY_LIMIT + 1 && overflowMetric.size() == 1; + }); + } + + @Test + public void testLongCounter() { + var longCounter = meter.counterBuilder("dummyLongCounter").build(); + var attributes = Attributes.of(AttributeKey.stringKey("dummyAttr"), "dummyValue"); + longCounter.add(1, attributes); + longCounter.add(2, attributes); + + assertThat(reader.collectAllMetrics()) + .anySatisfy(metric -> assertThat(metric) + .hasName("dummyLongCounter") + .hasLongSumSatisfying(sum -> sum + .hasPointsSatisfying(point -> point + .hasAttributes(attributes) + .hasValue(3)))); + } + + @Test + public void testServiceIsDisabledByDefault() throws Exception { + @Cleanup + var metricReader = InMemoryMetricReader.create(); + + @Cleanup + var ots = OpenTelemetryService.builder() + .builderCustomizer(getBuilderCustomizer(metricReader, Map.of())) + .clusterName("openTelemetryServiceTestCluster") + .build(); + var meter = ots.getOpenTelemetry().getMeter("openTelemetryServiceTestInstrument"); + + var builders = List.of( + meter.counterBuilder("dummyCounterA"), + meter.counterBuilder("dummyCounterB").setDescription("desc"), + meter.counterBuilder("dummyCounterC").setDescription("desc").setUnit("unit"), + meter.counterBuilder("dummyCounterD").setUnit("unit") + ); + + var callback = new AtomicBoolean(); + // Validate that no matter how the counters are being built, they are all backed by the same underlying object. + // This ensures we conserve memory when the SDK is disabled. + assertThat(builders.stream().map(LongCounterBuilder::build).distinct()).hasSize(1); + assertThat(builders.stream().map(LongCounterBuilder::buildObserver).distinct()).hasSize(1); + assertThat(builders.stream().map(b -> b.buildWithCallback(__ -> callback.set(true))).distinct()).hasSize(1); + + // Validate that no metrics are being emitted at all. + assertThat(metricReader.collectAllMetrics()).isEmpty(); + + // Validate that the callback has not being called. + assertThat(callback).isFalse(); + } + + @Test + public void testJvmRuntimeMetrics() { + // Attempt collection of GC metrics. The metrics should be populated regardless if GC is triggered or not. + Runtime.getRuntime().gc(); + + var metrics = reader.collectAllMetrics(); + + // Process Metrics + // Replaces process_cpu_seconds_total + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.cpu.time")); + + // Memory Metrics + // Replaces jvm_memory_bytes_used + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.memory.used")); + // Replaces jvm_memory_bytes_committed + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.memory.committed")); + // Replaces jvm_memory_bytes_max + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.memory.limit")); + // Replaces jvm_memory_bytes_init + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.memory.init")); + // Replaces jvm_memory_pool_allocated_bytes_total + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.memory.used_after_last_gc")); + + // Buffer Pool Metrics + // Replaces jvm_buffer_pool_used_bytes + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.buffer.memory.usage")); + // Replaces jvm_buffer_pool_capacity_bytes + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.buffer.memory.limit")); + // Replaces jvm_buffer_pool_used_buffers + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.buffer.count")); + + // Garbage Collector Metrics + // Replaces jvm_gc_collection_seconds + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.gc.duration")); + + // Thread Metrics + // Replaces jvm_threads_state, jvm_threads_current and jvm_threads_daemon + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.thread.count")); + + // Class Loading Metrics + // Replaces jvm_classes_currently_loaded + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.class.count")); + // Replaces jvm_classes_loaded_total + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.class.loaded")); + // Replaces jvm_classes_unloaded_total + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.class.unloaded")); + } +} diff --git a/pulsar-package-management/bookkeeper-storage/pom.xml b/pulsar-package-management/bookkeeper-storage/pom.xml index eda8edbf93048..0a83ba6cc6400 100644 --- a/pulsar-package-management/bookkeeper-storage/pom.xml +++ b/pulsar-package-management/bookkeeper-storage/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT 4.0.0 @@ -71,6 +71,12 @@ + + org.hamcrest + hamcrest + test + + io.dropwizard.metrics diff --git a/pulsar-package-management/bookkeeper-storage/src/main/java/org/apache/pulsar/packages/management/storage/bookkeeper/DLOutputStream.java b/pulsar-package-management/bookkeeper-storage/src/main/java/org/apache/pulsar/packages/management/storage/bookkeeper/DLOutputStream.java index 222987aa49d43..67345ebd47e31 100644 --- a/pulsar-package-management/bookkeeper-storage/src/main/java/org/apache/pulsar/packages/management/storage/bookkeeper/DLOutputStream.java +++ b/pulsar-package-management/bookkeeper-storage/src/main/java/org/apache/pulsar/packages/management/storage/bookkeeper/DLOutputStream.java @@ -22,8 +22,6 @@ import io.netty.buffer.Unpooled; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.distributedlog.LogRecord; @@ -38,6 +36,7 @@ class DLOutputStream { private final DistributedLogManager distributedLogManager; private final AsyncLogWriter writer; + private final byte[] readBuffer = new byte[8192]; private long offset = 0L; private DLOutputStream(DistributedLogManager distributedLogManager, AsyncLogWriter writer) { @@ -50,42 +49,38 @@ static CompletableFuture openWriterAsync(DistributedLogManager d return distributedLogManager.openAsyncLogWriter().thenApply(w -> new DLOutputStream(distributedLogManager, w)); } - private CompletableFuture> getRecords(InputStream inputStream) { - CompletableFuture> future = new CompletableFuture<>(); - CompletableFuture.runAsync(() -> { - byte[] readBuffer = new byte[8192]; - List records = new ArrayList<>(); - try { - int read = 0; - while ((read = inputStream.read(readBuffer)) != -1) { - log.info("write something into the ledgers offset: {}, length: {}", offset, read); - ByteBuf writeBuf = Unpooled.copiedBuffer(readBuffer, 0, read); - offset += writeBuf.readableBytes(); - LogRecord record = new LogRecord(offset, writeBuf); - records.add(record); - } - future.complete(records); - } catch (IOException e) { - log.error("Failed to get all records from the input stream", e); - future.completeExceptionally(e); + private void writeAsyncHelper(InputStream is, CompletableFuture result) { + try { + int read = is.read(readBuffer); + if (read != -1) { + log.info("write something into the ledgers offset: {}, length: {}", offset, read); + final ByteBuf writeBuf = Unpooled.wrappedBuffer(readBuffer, 0, read); + offset += writeBuf.readableBytes(); + final LogRecord record = new LogRecord(offset, writeBuf); + writer.write(record).thenAccept(v -> writeAsyncHelper(is, result)) + .exceptionally(e -> { + result.completeExceptionally(e); + return null; + }); + } else { + result.complete(this); } - }); - return future; + } catch (IOException e) { + log.error("Failed to get all records from the input stream", e); + result.completeExceptionally(e); + } } /** * Write all input stream data to the distribute log. * * @param inputStream the data we need to write - * @return + * @return CompletableFuture */ CompletableFuture writeAsync(InputStream inputStream) { - return getRecords(inputStream) - .thenCompose(this::writeAsync); - } - - private CompletableFuture writeAsync(List records) { - return writer.writeBulk(records).thenApply(ignore -> this); + CompletableFuture result = new CompletableFuture<>(); + writeAsyncHelper(inputStream, result); + return result; } /** diff --git a/pulsar-package-management/bookkeeper-storage/src/test/java/org/apache/pulsar/packages/management/storage/bookkeeper/DLOutputStreamTest.java b/pulsar-package-management/bookkeeper-storage/src/test/java/org/apache/pulsar/packages/management/storage/bookkeeper/DLOutputStreamTest.java index 63fcf5e46ebe1..b55e0e0d34a4f 100644 --- a/pulsar-package-management/bookkeeper-storage/src/test/java/org/apache/pulsar/packages/management/storage/bookkeeper/DLOutputStreamTest.java +++ b/pulsar-package-management/bookkeeper-storage/src/test/java/org/apache/pulsar/packages/management/storage/bookkeeper/DLOutputStreamTest.java @@ -21,17 +21,18 @@ import java.io.ByteArrayInputStream; import java.io.IOException; -import java.util.Collections; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import org.apache.distributedlog.DLSN; +import org.apache.distributedlog.LogRecord; import org.apache.distributedlog.api.AsyncLogWriter; import org.apache.distributedlog.api.DistributedLogManager; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.mockito.Mockito.anyList; @@ -53,9 +54,8 @@ public void setup() { when(dlm.asyncClose()).thenReturn(CompletableFuture.completedFuture(null)); when(writer.markEndOfStream()).thenReturn(CompletableFuture.completedFuture(null)); when(writer.asyncClose()).thenReturn(CompletableFuture.completedFuture(null)); - when(writer.writeBulk(anyList())) - .thenReturn(CompletableFuture.completedFuture( - Collections.singletonList(CompletableFuture.completedFuture(DLSN.InitialDLSN)))); + when(writer.write(any(LogRecord.class))) + .thenReturn(CompletableFuture.completedFuture(DLSN.InitialDLSN)); } @AfterMethod(alwaysRun = true) @@ -75,7 +75,7 @@ public void writeInputStreamData() throws ExecutionException, InterruptedExcepti .thenCompose(w -> w.writeAsync(new ByteArrayInputStream(data)) .thenCompose(DLOutputStream::closeAsync)).get(); - verify(writer, times(1)).writeBulk(anyList()); + verify(writer, times(1)).write(any(LogRecord.class)); verify(writer, times(1)).markEndOfStream(); verify(writer, times(1)).asyncClose(); verify(dlm, times(1)).asyncClose(); @@ -91,7 +91,7 @@ public void writeBytesArrayData() throws ExecutionException, InterruptedExceptio .thenCompose(w -> w.writeAsync(new ByteArrayInputStream(data)) .thenCompose(DLOutputStream::closeAsync)).get(); - verify(writer, times(1)).writeBulk(anyList()); + verify(writer, times(1)).write(any(LogRecord.class)); verify(writer, times(1)).markEndOfStream(); verify(writer, times(1)).asyncClose(); verify(dlm, times(1)).asyncClose(); @@ -104,7 +104,7 @@ public void writeLongBytesArrayData() throws ExecutionException, InterruptedExce .thenCompose(w -> w.writeAsync(new ByteArrayInputStream(data)) .thenCompose(DLOutputStream::closeAsync)).get(); - verify(writer, times(1)).writeBulk(anyList()); + verify(writer, times(4)).write(any(LogRecord.class)); verify(writer, times(1)).markEndOfStream(); verify(writer, times(1)).asyncClose(); verify(dlm, times(1)).asyncClose(); diff --git a/pulsar-package-management/bookkeeper-storage/src/test/java/org/apache/pulsar/packages/management/storage/bookkeeper/bookkeeper/test/BookKeeperClusterTestCase.java b/pulsar-package-management/bookkeeper-storage/src/test/java/org/apache/pulsar/packages/management/storage/bookkeeper/bookkeeper/test/BookKeeperClusterTestCase.java index 40c2041d4e6c4..43db5ad4ba845 100644 --- a/pulsar-package-management/bookkeeper-storage/src/test/java/org/apache/pulsar/packages/management/storage/bookkeeper/bookkeeper/test/BookKeeperClusterTestCase.java +++ b/pulsar-package-management/bookkeeper-storage/src/test/java/org/apache/pulsar/packages/management/storage/bookkeeper/bookkeeper/test/BookKeeperClusterTestCase.java @@ -70,7 +70,6 @@ import org.apache.bookkeeper.meta.LedgerManager; import org.apache.bookkeeper.meta.LedgerManagerFactory; import org.apache.bookkeeper.meta.MetadataBookieDriver; -import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase; import org.apache.bookkeeper.metastore.InMemoryMetaStore; import org.apache.bookkeeper.net.BookieId; import org.apache.bookkeeper.net.BookieSocketAddress; @@ -486,7 +485,7 @@ public ServerConfiguration killBookie(int index) throws Exception { public ServerConfiguration killBookieAndWaitForZK(int index) throws Exception { ServerTester tester = servers.get(index); // IKTODO: this method is awful ServerConfiguration ret = killBookie(index); - while (zkc.exists(ZKMetadataDriverBase.resolveZkLedgersRootPath(baseConf) + "/" + AVAILABLE_NODE + "/" + while (zkc.exists("/ledgers/" + AVAILABLE_NODE + "/" + tester.getServer().getBookieId().toString(), false) != null) { Thread.sleep(500); } diff --git a/pulsar-package-management/core/pom.xml b/pulsar-package-management/core/pom.xml index 605c4c13e61fa..00b967791bb03 100644 --- a/pulsar-package-management/core/pom.xml +++ b/pulsar-package-management/core/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT 4.0.0 diff --git a/pulsar-package-management/core/src/main/java/org/apache/pulsar/packages/management/core/impl/PackagesManagementImpl.java b/pulsar-package-management/core/src/main/java/org/apache/pulsar/packages/management/core/impl/PackagesManagementImpl.java index 449d1268f3fd5..7861d07c775ce 100644 --- a/pulsar-package-management/core/src/main/java/org/apache/pulsar/packages/management/core/impl/PackagesManagementImpl.java +++ b/pulsar-package-management/core/src/main/java/org/apache/pulsar/packages/management/core/impl/PackagesManagementImpl.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.file.FileAlreadyExistsException; import java.util.List; import java.util.concurrent.CompletableFuture; import org.apache.pulsar.packages.management.core.PackagesManagement; @@ -187,8 +188,8 @@ private CompletableFuture checkMetadataExistsAndThrowException(PackageName if (!exist) { future.complete(null); } else { - future.completeExceptionally( - new NotFoundException(String.format("Package '%s' metadata already exists", packageName))); + future.completeExceptionally(new FileAlreadyExistsException( + String.format("Package '%s' metadata already exists", packageName))); } }); return future; diff --git a/pulsar-package-management/filesystem-storage/pom.xml b/pulsar-package-management/filesystem-storage/pom.xml index 8084ccc67925e..d7606b9a98b43 100644 --- a/pulsar-package-management/filesystem-storage/pom.xml +++ b/pulsar-package-management/filesystem-storage/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT 4.0.0 diff --git a/pulsar-package-management/filesystem-storage/src/main/java/org/apache/pulsar/packages/management/storage/filesystem/FileSystemPackagesStorage.java b/pulsar-package-management/filesystem-storage/src/main/java/org/apache/pulsar/packages/management/storage/filesystem/FileSystemPackagesStorage.java index 47d825ea928f4..2bb43bb207203 100644 --- a/pulsar-package-management/filesystem-storage/src/main/java/org/apache/pulsar/packages/management/storage/filesystem/FileSystemPackagesStorage.java +++ b/pulsar-package-management/filesystem-storage/src/main/java/org/apache/pulsar/packages/management/storage/filesystem/FileSystemPackagesStorage.java @@ -58,7 +58,11 @@ public class FileSystemPackagesStorage implements PackagesStorage { } } - private File getPath(String path) { + private File getPath(String path) throws IOException { + if (path.contains("..")) { + throw new IOException("Invalid path: " + path); + } + File f = Paths.get(storagePath.toString(), path).toFile(); if (!f.getParentFile().exists()) { if (!f.getParentFile().mkdirs()) { @@ -119,28 +123,40 @@ public CompletableFuture readAsync(String path, OutputStream outputStream) @Override public CompletableFuture deleteAsync(String path) { - if (getPath(path).delete()) { - return CompletableFuture.completedFuture(null); - } else { - CompletableFuture f = new CompletableFuture<>(); - f.completeExceptionally(new IOException("Failed to delete file at " + path)); - return f; + try { + if (getPath(path).delete()) { + return CompletableFuture.completedFuture(null); + } else { + CompletableFuture f = new CompletableFuture<>(); + f.completeExceptionally(new IOException("Failed to delete file at " + path)); + return f; + } + } catch (IOException e) { + return CompletableFuture.failedFuture(e); } } @Override public CompletableFuture> listAsync(String path) { - String[] files = getPath(path).list(); - if (files == null) { - return CompletableFuture.completedFuture(Collections.emptyList()); - } else { - return CompletableFuture.completedFuture(Arrays.asList(files)); + try { + String[] files = getPath(path).list(); + if (files == null) { + return CompletableFuture.completedFuture(Collections.emptyList()); + } else { + return CompletableFuture.completedFuture(Arrays.asList(files)); + } + } catch (IOException e) { + return CompletableFuture.failedFuture(e); } } @Override public CompletableFuture existAsync(String path) { - return CompletableFuture.completedFuture(getPath(path).exists()); + try { + return CompletableFuture.completedFuture(getPath(path).exists()); + } catch (IOException e) { + return CompletableFuture.failedFuture(e); + } } @Override diff --git a/pulsar-package-management/pom.xml b/pulsar-package-management/pom.xml index 3a61f659d3109..94577a70226a0 100644 --- a/pulsar-package-management/pom.xml +++ b/pulsar-package-management/pom.xml @@ -25,8 +25,7 @@ pulsar org.apache.pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT 4.0.0 diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index 286c0ac0912a0..f0e52036e55b4 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-proxy @@ -49,6 +49,18 @@ ${project.version} + + ${project.groupId} + pulsar-opentelemetry + ${project.version} + + + + ${project.groupId} + pulsar-docs-tools + ${project.version} + + ${project.groupId} pulsar-websocket @@ -60,6 +72,12 @@ commons-lang3 + + io.swagger + swagger-annotations + provided + + org.eclipse.jetty jetty-server @@ -166,8 +184,8 @@ - com.beust - jcommander + info.picocli + picocli @@ -185,6 +203,18 @@ testcontainers test + + com.github.tomakehurst + wiremock-jre8 + ${wiremock.version} + test + + + io.github.hakky54 + consolecaptor + ${consolecaptor.version} + test + diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/extensions/ProxyExtensions.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/extensions/ProxyExtensions.java index 75059fc0d2551..95a3bf032fe21 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/extensions/ProxyExtensions.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/extensions/ProxyExtensions.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketAddress; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -49,6 +50,9 @@ public class ProxyExtensions implements AutoCloseable { * @return the collection of extensions */ public static ProxyExtensions load(ProxyConfiguration conf) throws IOException { + if (conf.getProxyExtensions().isEmpty()) { + return new ProxyExtensions(Collections.emptyMap()); + } ExtensionsDefinitions definitions = ProxyExtensionsUtils.searchForExtensions( conf.getProxyExtensionsDirectory(), conf.getNarExtractionDirectory()); diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java index d9dda9823ea89..54b6db5198c57 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java @@ -24,14 +24,15 @@ import java.io.InputStream; import java.net.URI; import java.nio.ByteBuffer; -import java.security.cert.X509Certificate; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.Iterator; -import java.util.Objects; import java.util.Set; import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import javax.net.ssl.SSLContext; import javax.servlet.ServletConfig; import javax.servlet.ServletException; @@ -40,9 +41,10 @@ import org.apache.pulsar.broker.web.AuthenticationFilter; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.AuthenticationDataProvider; -import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.common.util.SecurityUtility; +import org.apache.pulsar.client.util.ExecutorProvider; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.HttpRequest; @@ -85,17 +87,31 @@ class AdminProxyHandler extends ProxyServlet { private final ProxyConfiguration config; private final BrokerDiscoveryProvider discoveryProvider; + private final Authentication proxyClientAuthentication; private final String brokerWebServiceUrl; private final String functionWorkerWebServiceUrl; + private PulsarSslFactory pulsarSslFactory; + private ScheduledExecutorService sslContextRefresher; - AdminProxyHandler(ProxyConfiguration config, BrokerDiscoveryProvider discoveryProvider) { + AdminProxyHandler(ProxyConfiguration config, BrokerDiscoveryProvider discoveryProvider, + Authentication proxyClientAuthentication) { this.config = config; this.discoveryProvider = discoveryProvider; + this.proxyClientAuthentication = proxyClientAuthentication; this.brokerWebServiceUrl = config.isTlsEnabledWithBroker() ? config.getBrokerWebServiceURLTLS() : config.getBrokerWebServiceURL(); this.functionWorkerWebServiceUrl = config.isTlsEnabledWithBroker() ? config.getFunctionWorkerWebServiceURLTLS() : config.getFunctionWorkerWebServiceURL(); - + if (config.isTlsEnabledWithBroker()) { + this.pulsarSslFactory = createPulsarSslFactory(); + this.sslContextRefresher = Executors.newSingleThreadScheduledExecutor( + new ExecutorProvider.ExtendedThreadFactory("pulsar-proxy-admin-handler-ssl-refresh")); + if (config.getTlsCertRefreshCheckDurationSec() > 0) { + this.sslContextRefresher.scheduleWithFixedDelay(this::refreshSslContext, + config.getTlsCertRefreshCheckDurationSec(), config.getTlsCertRefreshCheckDurationSec(), + TimeUnit.SECONDS); + } + } super.setTimeout(config.getHttpProxyTimeout()); } @@ -156,6 +172,7 @@ protected HttpClient createHttpClient() throws ServletException { client.start(); // Content must not be decoded, otherwise the client gets confused. + // Allow encoded content, such as "Content-Encoding: gzip", to pass through without decoding it. client.getContentDecoderFactories().clear(); // Pass traffic to the client, only intercept what's necessary. @@ -253,51 +270,15 @@ protected ContentProvider proxyRequestContent(HttpServletRequest request, @Override protected HttpClient newHttpClient() { try { - Authentication auth = AuthenticationFactory.create( - config.getBrokerClientAuthenticationPlugin(), - config.getBrokerClientAuthenticationParameters() - ); - - Objects.requireNonNull(auth, "No supported auth found for proxy"); - - auth.start(); - if (config.isTlsEnabledWithBroker()) { try { - X509Certificate[] trustCertificates = SecurityUtility - .loadCertificatesFromPemFile(config.getBrokerClientTrustCertsFilePath()); - - SSLContext sslCtx; - AuthenticationDataProvider authData = auth.getAuthData(); - if (authData.hasDataForTls()) { - sslCtx = SecurityUtility.createSslContext( - config.isTlsAllowInsecureConnection(), - trustCertificates, - authData.getTlsCertificates(), - authData.getTlsPrivateKey(), - config.getBrokerClientSslProvider() - ); - } else { - sslCtx = SecurityUtility.createSslContext( - config.isTlsAllowInsecureConnection(), - trustCertificates, - config.getBrokerClientSslProvider() - ); - } - - SslContextFactory contextFactory = new SslContextFactory.Client(); - contextFactory.setSslContext(sslCtx); + SslContextFactory contextFactory = new Client(this.pulsarSslFactory); if (!config.isTlsHostnameVerificationEnabled()) { contextFactory.setEndpointIdentificationAlgorithm(null); } return new JettyHttpClient(contextFactory); } catch (Exception e) { LOG.error("new jetty http client exception ", e); - try { - auth.close(); - } catch (IOException ioe) { - LOG.error("Failed to close the authentication service", ioe); - } throw new PulsarClientException.InvalidConfigurationException(e.getMessage()); } } @@ -374,4 +355,79 @@ protected void addProxyHeaders(HttpServletRequest clientRequest, Request proxyRe proxyRequest.header(ORIGINAL_PRINCIPAL_HEADER, user); } } + + private static class Client extends SslContextFactory.Client { + + private final PulsarSslFactory sslFactory; + + public Client(PulsarSslFactory sslFactory) { + super(); + this.sslFactory = sslFactory; + } + + @Override + public SSLContext getSslContext() { + return this.sslFactory.getInternalSslContext(); + } + } + + protected PulsarSslConfiguration buildSslConfiguration(AuthenticationDataProvider authData) { + return PulsarSslConfiguration.builder() + .tlsProvider(config.getBrokerClientSslProvider()) + .tlsKeyStoreType(config.getBrokerClientTlsKeyStoreType()) + .tlsKeyStorePath(config.getBrokerClientTlsKeyStore()) + .tlsKeyStorePassword(config.getBrokerClientTlsKeyStorePassword()) + .tlsTrustStoreType(config.getBrokerClientTlsTrustStoreType()) + .tlsTrustStorePath(config.getBrokerClientTlsTrustStore()) + .tlsTrustStorePassword(config.getBrokerClientTlsTrustStorePassword()) + .tlsCiphers(config.getBrokerClientTlsCiphers()) + .tlsProtocols(config.getBrokerClientTlsProtocols()) + .tlsTrustCertsFilePath(config.getBrokerClientTrustCertsFilePath()) + .tlsCertificateFilePath(config.getBrokerClientCertificateFilePath()) + .tlsKeyFilePath(config.getBrokerClientKeyFilePath()) + .allowInsecureConnection(config.isTlsAllowInsecureConnection()) + .requireTrustedClientCertOnConnect(false) + .tlsEnabledWithKeystore(config.isBrokerClientTlsEnabledWithKeyStore()) + .tlsCustomParams(config.getBrokerClientSslFactoryPluginParams()) + .authData(authData) + .serverMode(false) + .isHttps(true) + .build(); + } + + protected PulsarSslFactory createPulsarSslFactory() { + try { + try { + AuthenticationDataProvider authData = proxyClientAuthentication.getAuthData(); + PulsarSslConfiguration pulsarSslConfiguration = buildSslConfiguration(authData); + PulsarSslFactory sslFactory = + (PulsarSslFactory) Class.forName(config.getBrokerClientSslFactoryPlugin()) + .getConstructor().newInstance(); + sslFactory.initialize(pulsarSslConfiguration); + sslFactory.createInternalSslContext(); + return sslFactory; + } catch (Exception e) { + LOG.error("Failed to create Pulsar SSLFactory ", e); + throw new PulsarClientException.InvalidConfigurationException(e.getMessage()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + protected void refreshSslContext() { + try { + this.pulsarSslFactory.update(); + } catch (Exception e) { + LOG.error("Failed to refresh SSL context", e); + } + } + + @Override + public void destroy() { + super.destroy(); + if (this.sslContextRefresher != null) { + this.sslContextRefresher.shutdownNow(); + } + } } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java index d63b04b6734de..407c93074a0fc 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java @@ -40,19 +40,17 @@ import io.netty.handler.codec.haproxy.HAProxyProtocolVersion; import io.netty.handler.codec.haproxy.HAProxyProxiedProtocol; import io.netty.handler.flush.FlushConsolidationHandler; -import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslHandler; -import io.netty.handler.ssl.SslProvider; import io.netty.handler.timeout.ReadTimeoutHandler; import io.netty.util.CharsetUtil; import java.net.InetSocketAddress; import java.util.Arrays; import java.util.concurrent.TimeUnit; import lombok.Getter; +import lombok.SneakyThrows; import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.AuthenticationDataProvider; -import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.api.AuthData; @@ -61,10 +59,9 @@ import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.PulsarDecoder; import org.apache.pulsar.common.stats.Rate; -import org.apache.pulsar.common.util.NettyClientSslContextRefresher; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.apache.pulsar.common.util.SecurityUtility; -import org.apache.pulsar.common.util.SslContextAutoRefreshBuilder; -import org.apache.pulsar.common.util.keystoretls.NettySSLContextAutoRefreshBuilder; import org.apache.pulsar.common.util.netty.NettyChannelUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -89,11 +86,10 @@ public class DirectProxyHandler { private final ProxyService service; private final Runnable onHandshakeCompleteAction; private final boolean tlsHostnameVerificationEnabled; - private final boolean tlsEnabledWithKeyStore; final boolean tlsEnabledWithBroker; - private final SslContextAutoRefreshBuilder clientSslCtxRefresher; - private final NettySSLContextAutoRefreshBuilder clientSSLContextAutoRefreshBuilder; + private PulsarSslFactory sslFactory; + @SneakyThrows public DirectProxyHandler(ProxyService service, ProxyConnection proxyConnection) { this.service = service; this.authentication = proxyConnection.getClientAuthentication(); @@ -105,7 +101,6 @@ public DirectProxyHandler(ProxyService service, ProxyConnection proxyConnection) this.clientAuthMethod = proxyConnection.clientAuthMethod; this.tlsEnabledWithBroker = service.getConfiguration().isTlsEnabledWithBroker(); this.tlsHostnameVerificationEnabled = service.getConfiguration().isTlsHostnameVerificationEnabled(); - this.tlsEnabledWithKeyStore = service.getConfiguration().isTlsEnabledWithKeyStore(); this.onHandshakeCompleteAction = proxyConnection::cancelKeepAliveTask; ProxyConfiguration config = service.getConfiguration(); @@ -114,47 +109,16 @@ public DirectProxyHandler(ProxyService service, ProxyConnection proxyConnection) if (!isEmpty(config.getBrokerClientAuthenticationPlugin())) { try { - authData = AuthenticationFactory.create(config.getBrokerClientAuthenticationPlugin(), - config.getBrokerClientAuthenticationParameters()).getAuthData(); + authData = authentication.getAuthData(); } catch (PulsarClientException e) { throw new RuntimeException(e); } } - - if (tlsEnabledWithKeyStore) { - clientSSLContextAutoRefreshBuilder = new NettySSLContextAutoRefreshBuilder( - config.getBrokerClientSslProvider(), - config.isTlsAllowInsecureConnection(), - config.getBrokerClientTlsTrustStoreType(), - config.getBrokerClientTlsTrustStore(), - config.getBrokerClientTlsTrustStorePassword(), - config.getBrokerClientTlsKeyStoreType(), - config.getBrokerClientTlsKeyStore(), - config.getBrokerClientTlsKeyStorePassword(), - config.getBrokerClientTlsCiphers(), - config.getBrokerClientTlsProtocols(), - config.getTlsCertRefreshCheckDurationSec(), - authData); - clientSslCtxRefresher = null; - } else { - SslProvider sslProvider = null; - if (config.getBrokerClientSslProvider() != null) { - sslProvider = SslProvider.valueOf(config.getBrokerClientSslProvider()); - } - clientSslCtxRefresher = new NettyClientSslContextRefresher( - sslProvider, - config.isTlsAllowInsecureConnection(), - config.getBrokerClientTrustCertsFilePath(), - authData, - config.getBrokerClientTlsCiphers(), - config.getBrokerClientTlsProtocols(), - config.getTlsCertRefreshCheckDurationSec() - ); - clientSSLContextAutoRefreshBuilder = null; - } - } else { - clientSSLContextAutoRefreshBuilder = null; - clientSslCtxRefresher = null; + PulsarSslConfiguration sslConfiguration = buildSslConfiguration(config, authData); + this.sslFactory = (PulsarSslFactory) Class.forName(config.getSslFactoryPlugin()) + .getConstructor().newInstance(); + this.sslFactory.initialize(sslConfiguration); + this.sslFactory.createInternalSslContext(); } } @@ -195,9 +159,7 @@ protected void initChannel(SocketChannel ch) { if (tlsEnabledWithBroker) { String host = targetBrokerAddress.getHostString(); int port = targetBrokerAddress.getPort(); - SslHandler handler = tlsEnabledWithKeyStore - ? new SslHandler(clientSSLContextAutoRefreshBuilder.get().createSSLEngine(host, port)) - : clientSslCtxRefresher.get().newHandler(ch.alloc(), host, port); + SslHandler handler = new SslHandler(sslFactory.createClientSslEngine(ch.alloc(), host, port)); if (tlsHostnameVerificationEnabled) { SecurityUtility.configureSSLHandler(handler); } @@ -501,5 +463,29 @@ private void writeAndFlush(ByteBuf cmd) { NettyChannelUtil.writeAndFlushWithVoidPromise(outboundChannel, cmd); } + protected PulsarSslConfiguration buildSslConfiguration(ProxyConfiguration config, + AuthenticationDataProvider authData) { + return PulsarSslConfiguration.builder() + .tlsProvider(config.getBrokerClientSslProvider()) + .tlsKeyStoreType(config.getBrokerClientTlsKeyStoreType()) + .tlsKeyStorePath(config.getBrokerClientTlsKeyStore()) + .tlsKeyStorePassword(config.getBrokerClientTlsKeyStorePassword()) + .tlsTrustStoreType(config.getBrokerClientTlsTrustStoreType()) + .tlsTrustStorePath(config.getBrokerClientTlsTrustStore()) + .tlsTrustStorePassword(config.getBrokerClientTlsTrustStorePassword()) + .tlsCiphers(config.getBrokerClientTlsCiphers()) + .tlsProtocols(config.getBrokerClientTlsProtocols()) + .tlsTrustCertsFilePath(config.getBrokerClientTrustCertsFilePath()) + .tlsCertificateFilePath(config.getBrokerClientCertificateFilePath()) + .tlsKeyFilePath(config.getBrokerClientKeyFilePath()) + .allowInsecureConnection(config.isTlsAllowInsecureConnection()) + .requireTrustedClientCertOnConnect(false) + .tlsEnabledWithKeystore(config.isBrokerClientTlsEnabledWithKeyStore()) + .tlsCustomParams(config.getBrokerClientSslFactoryPluginParams()) + .authData(authData) + .serverMode(false) + .build(); + } + private static final Logger log = LoggerFactory.getLogger(DirectProxyHandler.class); } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java index b62b3bacf0114..03975e153acb6 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java @@ -116,7 +116,7 @@ public void handleLookup(CommandLookupTopic lookup) { log.debug("Lookup Request ID {} from {} rejected - {}.", clientRequestId, clientAddress, throttlingErrorMessage); } - writeAndFlush(Commands.newLookupErrorResponse(ServerError.ServiceNotReady, + writeAndFlush(Commands.newLookupErrorResponse(ServerError.TooManyRequests, throttlingErrorMessage, clientRequestId)); } @@ -241,7 +241,8 @@ private void handlePartitionMetadataResponse(CommandPartitionedTopicMetadata par // Connected to backend broker long requestId = proxyConnection.newRequestId(); ByteBuf command; - command = Commands.newPartitionMetadataRequest(topicName.toString(), requestId); + command = Commands.newPartitionMetadataRequest(topicName.toString(), requestId, + partitionMetadata.isMetadataAutoCreationEnabled()); clientCnx.newLookup(command, requestId).whenComplete((r, t) -> { if (t != null) { log.warn("[{}] failed to get Partitioned metadata : {}", topicName.toString(), diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java index 782454022b1ed..d15d48b9209d0 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java @@ -26,6 +26,7 @@ import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.client.impl.ClientCnx; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; import org.apache.pulsar.common.api.AuthData; import org.apache.pulsar.common.api.proto.CommandAuthChallenge; import org.apache.pulsar.common.protocol.Commands; @@ -47,7 +48,7 @@ public class ProxyClientCnx extends ClientCnx { public ProxyClientCnx(ClientConfigurationData conf, EventLoopGroup eventLoopGroup, String clientAuthRole, String clientAuthMethod, int protocolVersion, boolean forwardClientAuthData, ProxyConnection proxyConnection) { - super(conf, eventLoopGroup, protocolVersion); + super(InstrumentProvider.NOOP, conf, eventLoopGroup, protocolVersion); this.clientAuthRole = clientAuthRole; this.clientAuthMethod = clientAuthMethod; this.forwardClientAuthData = forwardClientAuthData; diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java index 3ecd670cbbf7a..b9360e403f6f4 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java @@ -42,6 +42,8 @@ import org.apache.pulsar.common.nar.NarClassLoader; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.sasl.SaslConstants; +import org.apache.pulsar.common.util.DefaultPulsarSslFactory; + @Getter @Setter @@ -173,35 +175,43 @@ public class ProxyConfiguration implements PulsarConfiguration { @FieldContext( category = CATEGORY_BROKER_DISCOVERY, - doc = "The service url points to the broker cluster. URL must have the pulsar:// prefix." + doc = "If does not set metadataStoreUrl or configurationMetadataStoreUrl, this url should point to the" + + " discovery service provider." + + " URL must have the pulsar:// prefix. And does not support multi url yet." ) private String brokerServiceURL; @FieldContext( category = CATEGORY_BROKER_DISCOVERY, - doc = "The tls service url points to the broker cluster. URL must have the pulsar+ssl:// prefix." + doc = "If does not set metadataStoreUrl or configurationMetadataStoreUrl, this url should point to the" + + " discovery service provider." + + " URL must have the pulsar+ssl:// prefix. And does not support multi url yet." ) private String brokerServiceURLTLS; @FieldContext( category = CATEGORY_BROKER_DISCOVERY, - doc = "The web service url points to the broker cluster" + doc = "The web service url points to the discovery service provider of the broker cluster, and does not support" + + " multi url yet." ) private String brokerWebServiceURL; @FieldContext( category = CATEGORY_BROKER_DISCOVERY, - doc = "The tls web service url points to the broker cluster" + doc = "The tls web service url points to the discovery service provider of the broker cluster, and does not" + + " support multi url yet." ) private String brokerWebServiceURLTLS; @FieldContext( category = CATEGORY_BROKER_DISCOVERY, - doc = "The web service url points to the function worker cluster." + doc = "The web service url points to the discovery service provider of the function worker cluster, and does" + + " not support multi url yet." + " Only configure it when you setup function workers in a separate cluster" ) private String functionWorkerWebServiceURL; @FieldContext( category = CATEGORY_BROKER_DISCOVERY, - doc = "The tls web service url points to the function worker cluster." + doc = "The tls web service url points to the discovery service provider of the function worker cluster, and" + + " does not support multi url yet." + " Only configure it when you setup function workers in a separate cluster" ) private String functionWorkerWebServiceURLTLS; @@ -260,6 +270,22 @@ public class ProxyConfiguration implements PulsarConfiguration { doc = "Enable or disable the proxy protocol.") private boolean haProxyProtocolEnabled; + @FieldContext(category = CATEGORY_SERVER, + doc = "Enable or disable the use of HA proxy protocol for resolving the client IP for http/https " + + "requests. Default is false.") + private boolean webServiceHaProxyProtocolEnabled = false; + + @FieldContext(category = CATEGORY_SERVER, doc = + "Trust X-Forwarded-For header for resolving the client IP for http/https requests.\n" + + "Default is false.") + private boolean webServiceTrustXForwardedFor = false; + + @FieldContext(category = CATEGORY_SERVER, doc = + "Add detailed client/remote and server/local addresses and ports to http/https request logging.\n" + + "Defaults to true when either webServiceHaProxyProtocolEnabled or webServiceTrustXForwardedFor " + + "is enabled.") + private Boolean webServiceLogDetailedAddresses; + @FieldContext(category = CATEGORY_SERVER, doc = "Enables zero-copy transport of data across network interfaces using the spice. " + "Zero copy mode cannot be used when TLS is enabled or when proxyLogLevel is > 0.") @@ -371,6 +397,12 @@ public class ProxyConfiguration implements PulsarConfiguration { ) private int authenticationRefreshCheckSeconds = 60; + @FieldContext( + category = CATEGORY_HTTP, + doc = "Whether to enable the proxy's /metrics and /proxy-stats http endpoints" + ) + private boolean enableProxyStatsEndpoints = true; + @FieldContext( category = CATEGORY_AUTHENTICATION, doc = "Whether the '/metrics' endpoint requires authentication. Defaults to true." @@ -378,6 +410,12 @@ public class ProxyConfiguration implements PulsarConfiguration { ) private boolean authenticateMetricsEndpoint = true; + @FieldContext( + category = CATEGORY_HTTP, + doc = "Time in milliseconds that metrics endpoint would time out. Default is 30s.\n" + + " Set it to 0 to disable timeout." + ) + private long metricsServletTimeoutMs = 30000; @FieldContext( category = CATEGORY_SASL_AUTH, @@ -578,6 +616,16 @@ public class ProxyConfiguration implements PulsarConfiguration { ) private String tlsTrustStorePassword = null; + @FieldContext( + category = CATEGORY_TLS, + doc = "SSL Factory Plugin class to provide SSLEngine and SSLContext objects. The default " + + " class used is DefaultSslFactory.") + private String sslFactoryPlugin = DefaultPulsarSslFactory.class.getName(); + @FieldContext( + category = CATEGORY_TLS, + doc = "SSL Factory plugin configuration parameters.") + private String sslFactoryPluginParams = ""; + /** * KeyStore TLS config variables used for proxy to auth with broker. */ @@ -647,6 +695,16 @@ public class ProxyConfiguration implements PulsarConfiguration { ) private Set brokerClientTlsProtocols = new TreeSet<>(); + @FieldContext( + category = CATEGORY_TLS, + doc = "SSL Factory Plugin class used by internal client to provide SSLEngine and SSLContext objects. " + + "The default class used is DefaultSslFactory.") + private String brokerClientSslFactoryPlugin = DefaultPulsarSslFactory.class.getName(); + @FieldContext( + category = CATEGORY_TLS, + doc = "SSL Factory plugin configuration parameters used by internal client.") + private String brokerClientSslFactoryPluginParams = ""; + // HTTP @FieldContext( diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java index ba9247a085dff..f8b5d0844509e 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java @@ -59,6 +59,7 @@ import org.apache.pulsar.client.impl.PulsarChannelInitializer; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.client.impl.conf.ConfigurationDataUtils; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; import org.apache.pulsar.client.internal.PropertiesUtils; import org.apache.pulsar.common.api.AuthData; import org.apache.pulsar.common.api.proto.CommandAuthResponse; @@ -168,6 +169,10 @@ public void channelRegistered(ChannelHandlerContext ctx) throws Exception { ProxyService.ACTIVE_CONNECTIONS.inc(); SocketAddress rmAddress = ctx.channel().remoteAddress(); ConnectionController.State state = connectionController.increaseConnection(rmAddress); + if (LOG.isDebugEnabled()) { + LOG.debug("Active connection count={} for cnx {} with state {}", ProxyService.ACTIVE_CONNECTIONS.get(), + rmAddress, state); + } if (!state.equals(ConnectionController.State.OK)) { ctx.writeAndFlush(Commands.newError(-1, ServerError.NotAllowedError, state.equals(ConnectionController.State.REACH_MAX_CONNECTION) @@ -183,6 +188,9 @@ public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { super.channelUnregistered(ctx); connectionController.decreaseConnection(ctx.channel().remoteAddress()); ProxyService.ACTIVE_CONNECTIONS.dec(); + if (LOG.isDebugEnabled()) { + LOG.debug("Decreasing active connection count={} ", ProxyService.ACTIVE_CONNECTIONS.get()); + } } @Override @@ -383,13 +391,14 @@ private synchronized void completeConnect() throws PulsarClientException { service.getConfiguration().isForwardAuthorizationCredentials(), this); } else { clientCnxSupplier = - () -> new ClientCnx(clientConf, service.getWorkerGroup(), protocolVersionToAdvertise); + () -> new ClientCnx(InstrumentProvider.NOOP, clientConf, service.getWorkerGroup(), + protocolVersionToAdvertise); } if (this.connectionPool == null) { - this.connectionPool = new ConnectionPool(clientConf, service.getWorkerGroup(), + this.connectionPool = new ConnectionPool(InstrumentProvider.NOOP, clientConf, service.getWorkerGroup(), clientCnxSupplier, - Optional.of(dnsAddressResolverGroup.getResolver(service.getWorkerGroup().next()))); + Optional.of(dnsAddressResolverGroup.getResolver(service.getWorkerGroup().next())), null); } else { LOG.error("BUG! Connection Pool has already been created for proxy connection to {} state {} role {}", remoteAddress, state, clientAuthRole); diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java index a934b8b078426..4ee15fd7124a6 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyService.java @@ -34,6 +34,7 @@ import io.netty.resolver.dns.DnsAddressResolverGroup; import io.netty.resolver.dns.DnsNameResolverBuilder; import io.netty.util.concurrent.DefaultThreadFactory; +import io.netty.util.concurrent.Future; import io.prometheus.client.Counter; import io.prometheus.client.Gauge; import java.io.Closeable; @@ -63,8 +64,6 @@ import org.apache.pulsar.broker.stats.prometheus.PrometheusRawMetricsProvider; import org.apache.pulsar.broker.web.plugin.servlet.AdditionalServlets; import org.apache.pulsar.client.api.Authentication; -import org.apache.pulsar.client.api.AuthenticationFactory; -import org.apache.pulsar.client.impl.auth.AuthenticationDisabled; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.common.util.netty.DnsResolverUtil; @@ -72,6 +71,7 @@ import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.proxy.extensions.ProxyExtensions; +import org.apache.pulsar.proxy.stats.PulsarProxyOpenTelemetry; import org.apache.pulsar.proxy.stats.TopicStats; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -119,6 +119,7 @@ public class ProxyService implements Closeable { protected boolean proxyZeroCopyModeEnabled; private final ScheduledExecutorService statsExecutor; + private ScheduledExecutorService sslContextRefresher; static final Gauge ACTIVE_CONNECTIONS = Gauge .build("pulsar_proxy_active_connections", "Number of connections currently active in the proxy").create() @@ -147,12 +148,17 @@ public class ProxyService implements Closeable { private PrometheusMetricsServlet metricsServlet; private List pendingMetricsProviders; + @Getter + private PulsarProxyOpenTelemetry openTelemetry; @Getter private final ConnectionController connectionController; + private boolean gracefulShutdown = true; + public ProxyService(ProxyConfiguration proxyConfig, - AuthenticationService authenticationService) throws Exception { + AuthenticationService authenticationService, + Authentication proxyClientAuthentication) throws Exception { requireNonNull(proxyConfig); this.proxyConfig = proxyConfig; this.clientCnxs = Sets.newConcurrentHashSet(); @@ -201,12 +207,7 @@ public ProxyService(ProxyConfiguration proxyConfig, }); }, 60, TimeUnit.SECONDS); this.proxyAdditionalServlets = AdditionalServlets.load(proxyConfig); - if (proxyConfig.getBrokerClientAuthenticationPlugin() != null) { - proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), - proxyConfig.getBrokerClientAuthenticationParameters()); - } else { - proxyClientAuthentication = AuthenticationDisabled.INSTANCE; - } + this.proxyClientAuthentication = proxyClientAuthentication; this.connectionController = new ConnectionController.DefaultConnectionController( proxyConfig.getMaxConcurrentInboundConnections(), proxyConfig.getMaxConcurrentInboundConnectionsPerIp()); @@ -245,7 +246,7 @@ public void start() throws Exception { proxyZeroCopyModeEnabled = true; } - bootstrap.childHandler(new ServiceChannelInitializer(this, proxyConfig, false)); + bootstrap.childHandler(new ServiceChannelInitializer(this, proxyConfig, false, null)); // Bind and start to accept incoming connections. if (proxyConfig.getServicePort().isPresent()) { try { @@ -258,8 +259,12 @@ public void start() throws Exception { } if (proxyConfig.getServicePortTls().isPresent()) { + this.sslContextRefresher = Executors + .newSingleThreadScheduledExecutor( + new DefaultThreadFactory("proxy-ssl-context-refresher")); ServerBootstrap tlsBootstrap = bootstrap.clone(); - tlsBootstrap.childHandler(new ServiceChannelInitializer(this, proxyConfig, true)); + tlsBootstrap.childHandler(new ServiceChannelInitializer(this, proxyConfig, true, + sslContextRefresher)); listenChannelTls = tlsBootstrap.bind(proxyConfig.getBindAddress(), proxyConfig.getServicePortTls().get()).sync().channel(); LOG.info("Started Pulsar TLS Proxy on {}", listenChannelTls.localAddress()); @@ -281,6 +286,7 @@ public void start() throws Exception { } createMetricsServlet(); + openTelemetry = new PulsarProxyOpenTelemetry(proxyConfig); // Initialize the message protocol handlers. // start the protocol handlers only after the broker is ready, @@ -292,7 +298,8 @@ public void start() throws Exception { } private synchronized void createMetricsServlet() { - this.metricsServlet = new PrometheusMetricsServlet(-1L, proxyConfig.getClusterName()); + this.metricsServlet = + new PrometheusMetricsServlet(proxyConfig.getMetricsServletTimeoutMs(), proxyConfig.getClusterName()); if (pendingMetricsProviders != null) { pendingMetricsProviders.forEach(provider -> metricsServlet.addRawMetricsProvider(provider)); this.pendingMetricsProviders = null; @@ -373,7 +380,7 @@ public void close() throws IOException { // Don't accept any new connections try { - acceptorGroup.shutdownGracefully().sync(); + shutdownEventLoop(acceptorGroup).sync(); } catch (InterruptedException e) { LOG.info("Shutdown of acceptorGroup interrupted"); Thread.currentThread().interrupt(); @@ -387,8 +394,12 @@ public void close() throws IOException { discoveryProvider.close(); } + if (this.sslContextRefresher != null) { + this.sslContextRefresher.shutdownNow(); + } + if (statsExecutor != null) { - statsExecutor.shutdown(); + statsExecutor.shutdownNow(); } if (proxyAdditionalServlets != null) { @@ -396,6 +407,9 @@ public void close() throws IOException { proxyAdditionalServlets = null; } + if (openTelemetry != null) { + openTelemetry.close(); + } resetMetricsServlet(); if (localMetadataStore != null) { @@ -413,14 +427,14 @@ public void close() throws IOException { } } try { - workerGroup.shutdownGracefully().sync(); + shutdownEventLoop(workerGroup).sync(); } catch (InterruptedException e) { LOG.info("Shutdown of workerGroup interrupted"); Thread.currentThread().interrupt(); } for (EventLoopGroup group : extensionsWorkerGroups) { try { - group.shutdownGracefully().sync(); + shutdownEventLoop(group).sync(); } catch (InterruptedException e) { LOG.info("Shutdown of {} interrupted", group); Thread.currentThread().interrupt(); @@ -532,4 +546,24 @@ public synchronized void addPrometheusRawMetricsProvider(PrometheusRawMetricsPro protected LookupProxyHandler newLookupProxyHandler(ProxyConnection proxyConnection) { return new LookupProxyHandler(this, proxyConnection); } + + // Shutdown the event loop. + // If graceful is true, will wait for the current requests to be completed, up to 15 seconds. + // Graceful shutdown can be disabled by setting the gracefulShutdown flag to false. This is used in tests + // to speed up the shutdown process. + private Future shutdownEventLoop(EventLoopGroup eventLoop) { + if (gracefulShutdown) { + return eventLoop.shutdownGracefully(); + } else { + return eventLoop.shutdownGracefully(0, 0, TimeUnit.SECONDS); + } + } + + public boolean isGracefulShutdown() { + return gracefulShutdown; + } + + public void setGracefulShutdown(boolean gracefulShutdown) { + this.gracefulShutdown = gracefulShutdown; + } } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java index beee9f1a4f763..a5504cac100a4 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java @@ -23,18 +23,21 @@ import static org.apache.commons.lang3.StringUtils.isEmpty; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.pulsar.common.stats.JvmMetrics.getJvmDirectMemoryUsed; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; import com.google.common.annotations.VisibleForTesting; +import io.prometheus.client.Collector; import io.prometheus.client.CollectorRegistry; import io.prometheus.client.Gauge; import io.prometheus.client.Gauge.Child; import io.prometheus.client.hotspot.DefaultExports; +import java.io.IOException; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Collection; import java.util.Collections; import java.util.Date; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import lombok.Getter; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.util.datetime.FixedDateFormat; @@ -43,15 +46,19 @@ import org.apache.pulsar.broker.authentication.AuthenticationService; import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsServlet; import org.apache.pulsar.broker.web.plugin.servlet.AdditionalServletWithClassLoader; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.impl.auth.AuthenticationDisabled; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.common.configuration.VipStatus; import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.DirectMemoryUtils; import org.apache.pulsar.common.util.ShutdownUtil; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.apache.pulsar.proxy.stats.ProxyStats; import org.apache.pulsar.websocket.WebSocketConsumerServlet; -import org.apache.pulsar.websocket.WebSocketPingPongServlet; +import org.apache.pulsar.websocket.WebSocketMultiTopicConsumerServlet; import org.apache.pulsar.websocket.WebSocketProducerServlet; import org.apache.pulsar.websocket.WebSocketReaderServlet; import org.apache.pulsar.websocket.WebSocketService; @@ -60,53 +67,67 @@ import org.eclipse.jetty.websocket.servlet.WebSocketServlet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.ScopeType; /** * Starts an instance of the Pulsar ProxyService. - * */ +@Command(name = "proxy", showDefaultValues = true, scope = ScopeType.INHERIT) public class ProxyServiceStarter { - @Parameter(names = { "-c", "--config" }, description = "Configuration file path", required = true) + @Option(names = { "-c", "--config" }, description = "Configuration file path", required = true) private String configFile; @Deprecated - @Parameter(names = { "-zk", "--zookeeper-servers" }, + @Option(names = { "-zk", "--zookeeper-servers" }, description = "Local zookeeper connection string, please use --metadata-store instead") private String zookeeperServers = ""; - @Parameter(names = { "-md", "--metadata-store" }, description = "Metadata Store service url. eg: zk:my-zk:2181") + @Option(names = { "-md", "--metadata-store" }, description = "Metadata Store service url. eg: zk:my-zk:2181") private String metadataStoreUrl = ""; @Deprecated - @Parameter(names = { "-gzk", "--global-zookeeper-servers" }, + @Option(names = { "-gzk", "--global-zookeeper-servers" }, description = "Global zookeeper connection string, please use --configuration-metadata-store instead") private String globalZookeeperServers = ""; @Deprecated - @Parameter(names = { "-cs", "--configuration-store-servers" }, + @Option(names = { "-cs", "--configuration-store-servers" }, description = "Configuration store connection string, " + "please use --configuration-metadata-store instead") private String configurationStoreServers = ""; - @Parameter(names = { "-cms", "--configuration-metadata-store" }, + @Option(names = { "-cms", "--configuration-metadata-store" }, description = "The metadata store URL for the configuration data") private String configurationMetadataStoreUrl = ""; - @Parameter(names = { "-h", "--help" }, description = "Show this help message") + @Option(names = { "-h", "--help" }, description = "Show this help message") private boolean help = false; - @Parameter(names = {"-g", "--generate-docs"}, description = "Generate docs") + @Option(names = {"-g", "--generate-docs"}, description = "Generate docs") private boolean generateDocs = false; private ProxyConfiguration config; + @Getter + private Authentication proxyClientAuthentication; + @Getter private ProxyService proxyService; private WebServer server; + private WebSocketService webSocketService; private static boolean metricsInitialized; + private boolean embeddedMode; public ProxyServiceStarter(String[] args) throws Exception { + this(args, null, false); + } + + public ProxyServiceStarter(String[] args, Consumer proxyConfigurationCustomizer, + boolean embeddedMode) throws Exception { + this.embeddedMode = embeddedMode; try { DateFormat dateFormat = new SimpleDateFormat( FixedDateFormat.FixedFormat.ISO8601_OFFSET_DATE_TIME_HHMM.getPattern()); @@ -116,28 +137,38 @@ public ProxyServiceStarter(String[] args) throws Exception { exception.printStackTrace(System.out); }); - JCommander jcommander = new JCommander(); + CommandLine commander = new CommandLine(this); try { - jcommander.addObject(this); - jcommander.parse(args); + commander.parseArgs(args); if (help || isBlank(configFile)) { - jcommander.usage(); + commander.usage(commander.getOut()); return; } if (this.generateDocs) { CmdGenerateDocs cmd = new CmdGenerateDocs("pulsar"); - cmd.addCommand("proxy", this); + cmd.addCommand("proxy", commander); cmd.run(null); - System.exit(0); + if (embeddedMode) { + return; + } else { + System.exit(0); + } } } catch (Exception e) { - jcommander.usage(); - System.exit(1); + commander.getErr().println(e); + if (embeddedMode) { + return; + } else { + System.exit(1); + } } // load config file config = PulsarConfigurationLoader.create(configFile, ProxyConfiguration.class); + if (proxyConfigurationCustomizer != null) { + proxyConfigurationCustomizer.accept(config); + } if (!isBlank(zookeeperServers)) { // Use zookeeperServers from command line @@ -162,11 +193,28 @@ public ProxyServiceStarter(String[] args) throws Exception { if (isNotBlank(config.getBrokerServiceURL())) { checkArgument(config.getBrokerServiceURL().startsWith("pulsar://"), "brokerServiceURL must start with pulsar://"); + ensureUrlNotContainsComma("brokerServiceURL", config.getBrokerServiceURL()); } - if (isNotBlank(config.getBrokerServiceURLTLS())) { checkArgument(config.getBrokerServiceURLTLS().startsWith("pulsar+ssl://"), "brokerServiceURLTLS must start with pulsar+ssl://"); + ensureUrlNotContainsComma("brokerServiceURLTLS", config.getBrokerServiceURLTLS()); + } + + if (isNotBlank(config.getBrokerWebServiceURL())) { + ensureUrlNotContainsComma("brokerWebServiceURL", config.getBrokerWebServiceURL()); + } + if (isNotBlank(config.getBrokerWebServiceURLTLS())) { + ensureUrlNotContainsComma("brokerWebServiceURLTLS", config.getBrokerWebServiceURLTLS()); + } + + if (isNotBlank(config.getFunctionWorkerWebServiceURL())) { + ensureUrlNotContainsComma("functionWorkerWebServiceURLTLS", + config.getFunctionWorkerWebServiceURL()); + } + if (isNotBlank(config.getFunctionWorkerWebServiceURLTLS())) { + ensureUrlNotContainsComma("functionWorkerWebServiceURLTLS", + config.getFunctionWorkerWebServiceURLTLS()); } if ((isBlank(config.getBrokerServiceURL()) && isBlank(config.getBrokerServiceURLTLS())) @@ -187,6 +235,11 @@ public ProxyServiceStarter(String[] args) throws Exception { } } + private void ensureUrlNotContainsComma(String paramName, String paramValue) { + checkArgument(!paramValue.contains(","), paramName + " does not support multi urls yet," + + " it should point to the discovery service provider."); + } + public static void main(String[] args) throws Exception { ProxyServiceStarter serviceStarter = new ProxyServiceStarter(args); try { @@ -200,38 +253,77 @@ public static void main(String[] args) throws Exception { public void start() throws Exception { AuthenticationService authenticationService = new AuthenticationService( PulsarConfigurationLoader.convertFrom(config)); + + if (config.getBrokerClientAuthenticationPlugin() != null) { + proxyClientAuthentication = AuthenticationFactory.create(config.getBrokerClientAuthenticationPlugin(), + config.getBrokerClientAuthenticationParameters()); + Objects.requireNonNull(proxyClientAuthentication, "No supported auth found for proxy"); + try { + proxyClientAuthentication.start(); + } catch (Exception e) { + try { + proxyClientAuthentication.close(); + } catch (IOException ioe) { + log.error("Failed to close the authentication service", ioe); + } + throw new PulsarClientException.InvalidConfigurationException(e.getMessage()); + } + } else { + proxyClientAuthentication = AuthenticationDisabled.INSTANCE; + } + // create proxy service - proxyService = new ProxyService(config, authenticationService); + proxyService = new ProxyService(config, authenticationService, proxyClientAuthentication); // create a web-service server = new WebServer(config, authenticationService); - Runtime.getRuntime().addShutdownHook(new Thread(this::close)); + if (!embeddedMode) { + Runtime.getRuntime().addShutdownHook(new Thread(this::close)); + } proxyService.start(); if (!metricsInitialized) { // Setup metrics DefaultExports.initialize(); + CollectorRegistry registry = CollectorRegistry.defaultRegistry; // Report direct memory from Netty counters - Gauge.build("jvm_memory_direct_bytes_used", "-").create().setChild(new Child() { - @Override - public double get() { - return getJvmDirectMemoryUsed(); - } - }).register(CollectorRegistry.defaultRegistry); + Collector jvmMemoryDirectBytesUsed = + Gauge.build("jvm_memory_direct_bytes_used", "-").create().setChild(new Child() { + @Override + public double get() { + return getJvmDirectMemoryUsed(); + } + }); + try { + registry.register(jvmMemoryDirectBytesUsed); + } catch (IllegalArgumentException e) { + // workaround issue in tests where the metric is already registered + log.debug("Failed to register jvm_memory_direct_bytes_used metric: {}", e.getMessage()); + } - Gauge.build("jvm_memory_direct_bytes_max", "-").create().setChild(new Child() { - @Override - public double get() { - return DirectMemoryUtils.jvmMaxDirectMemory(); - } - }).register(CollectorRegistry.defaultRegistry); + Collector jvmMemoryDirectBytesMax = + Gauge.build("jvm_memory_direct_bytes_max", "-").create().setChild(new Child() { + @Override + public double get() { + return DirectMemoryUtils.jvmMaxDirectMemory(); + } + }); + try { + registry.register(jvmMemoryDirectBytesMax); + } catch (IllegalArgumentException e) { + // workaround issue in tests where the metric is already registered + log.debug("Failed to register jvm_memory_direct_bytes_max metric: {}", e.getMessage()); + } metricsInitialized = true; } - addWebServerHandlers(server, config, proxyService, proxyService.getDiscoveryProvider()); + AtomicReference webSocketServiceRef = new AtomicReference<>(); + addWebServerHandlers(server, config, proxyService, proxyService.getDiscoveryProvider(), webSocketServiceRef, + proxyClientAuthentication); + webSocketService = webSocketServiceRef.get(); // start web-service server.start(); @@ -245,28 +337,53 @@ public void close() { if (server != null) { server.stop(); } + if (webSocketService != null) { + webSocketService.close(); + } + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } } catch (Exception e) { log.warn("server couldn't stop gracefully {}", e.getMessage(), e); } finally { - LogManager.shutdown(); + if (!embeddedMode) { + LogManager.shutdown(); + } } } public static void addWebServerHandlers(WebServer server, - ProxyConfiguration config, - ProxyService service, - BrokerDiscoveryProvider discoveryProvider) throws Exception { - if (service != null) { - PrometheusMetricsServlet metricsServlet = service.getMetricsServlet(); - if (metricsServlet != null) { - server.addServlet("/metrics", new ServletHolder(metricsServlet), - Collections.emptyList(), config.isAuthenticateMetricsEndpoint()); + ProxyConfiguration config, + ProxyService service, + BrokerDiscoveryProvider discoveryProvider, + Authentication proxyClientAuthentication) throws Exception { + addWebServerHandlers(server, config, service, discoveryProvider, null, proxyClientAuthentication); + } + + public static void addWebServerHandlers(WebServer server, + ProxyConfiguration config, + ProxyService service, + BrokerDiscoveryProvider discoveryProvider, + AtomicReference webSocketServiceRef, + Authentication proxyClientAuthentication) throws Exception { + // We can make 'status.html' publicly accessible without authentication since + // it does not contain any sensitive data. + server.addRestResource("/", VipStatus.ATTRIBUTE_STATUS_FILE_PATH, config.getStatusFilePath(), + VipStatus.class, false); + if (config.isEnableProxyStatsEndpoints()) { + server.addRestResource("/proxy-stats", ProxyStats.ATTRIBUTE_PULSAR_PROXY_NAME, service, + ProxyStats.class); + if (service != null) { + PrometheusMetricsServlet metricsServlet = service.getMetricsServlet(); + if (metricsServlet != null) { + server.addServlet("/metrics", new ServletHolder(metricsServlet), + Collections.emptyList(), config.isAuthenticateMetricsEndpoint()); + } } } - server.addRestResource("/", VipStatus.ATTRIBUTE_STATUS_FILE_PATH, config.getStatusFilePath(), VipStatus.class); - server.addRestResource("/proxy-stats", ProxyStats.ATTRIBUTE_PULSAR_PROXY_NAME, service, ProxyStats.class); - AdminProxyHandler adminProxyHandler = new AdminProxyHandler(config, discoveryProvider); + AdminProxyHandler adminProxyHandler = new AdminProxyHandler(config, discoveryProvider, + proxyClientAuthentication); ServletHolder servletHolder = new ServletHolder(adminProxyHandler); server.addServlet("/admin", servletHolder); server.addServlet("/lookup", servletHolder); @@ -298,6 +415,9 @@ public static void addWebServerHandlers(WebServer server, serviceConfiguration.setBrokerClientTlsEnabled(config.isTlsEnabledWithBroker()); WebSocketService webSocketService = new WebSocketService(createClusterData(config), serviceConfiguration); webSocketService.start(); + if (webSocketServiceRef != null) { + webSocketServiceRef.set(webSocketService); + } final WebSocketServlet producerWebSocketServlet = new WebSocketProducerServlet(webSocketService); server.addServlet(WebSocketProducerServlet.SERVLET_PATH, new ServletHolder(producerWebSocketServlet)); @@ -316,11 +436,10 @@ public static void addWebServerHandlers(WebServer server, server.addServlet(WebSocketReaderServlet.SERVLET_PATH_V2, new ServletHolder(readerWebSocketServlet)); - final WebSocketServlet pingPongWebSocketServlet = new WebSocketPingPongServlet(webSocketService); - server.addServlet(WebSocketPingPongServlet.SERVLET_PATH, - new ServletHolder(pingPongWebSocketServlet)); - server.addServlet(WebSocketPingPongServlet.SERVLET_PATH_V2, - new ServletHolder(pingPongWebSocketServlet)); + final WebSocketMultiTopicConsumerServlet multiTopicConsumerWebSocketServlet = + new WebSocketMultiTopicConsumerServlet(webSocketService); + server.addServlet(WebSocketMultiTopicConsumerServlet.SERVLET_PATH, + new ServletHolder(multiTopicConsumerWebSocketServlet)); } } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java index 19f4002ad52ce..728d27c815ff3 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java @@ -22,22 +22,23 @@ import io.netty.channel.socket.SocketChannel; import io.netty.handler.codec.LengthFieldBasedFrameDecoder; import io.netty.handler.flush.FlushConsolidationHandler; -import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslHandler; -import io.netty.handler.ssl.SslProvider; import io.netty.handler.timeout.ReadTimeoutHandler; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.OptionalProxyProtocolDecoder; -import org.apache.pulsar.common.util.NettyServerSslContextBuilder; -import org.apache.pulsar.common.util.SslContextAutoRefreshBuilder; -import org.apache.pulsar.common.util.keystoretls.NettySSLContextAutoRefreshBuilder; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Initialize service channel handlers. * */ public class ServiceChannelInitializer extends ChannelInitializer { + private static final Logger log = LoggerFactory.getLogger(ServiceChannelInitializer.class); public static final String TLS_HANDLER = "tls"; private final ProxyService proxyService; @@ -46,10 +47,10 @@ public class ServiceChannelInitializer extends ChannelInitializer private final int brokerProxyReadTimeoutMs; private final int maxMessageSize; - private SslContextAutoRefreshBuilder serverSslCtxRefresher; - private NettySSLContextAutoRefreshBuilder serverSSLContextAutoRefreshBuilder; + private PulsarSslFactory sslFactory; - public ServiceChannelInitializer(ProxyService proxyService, ProxyConfiguration serviceConfig, boolean enableTls) + public ServiceChannelInitializer(ProxyService proxyService, ProxyConfiguration serviceConfig, + boolean enableTls, ScheduledExecutorService sslContextRefresher) throws Exception { super(); this.proxyService = proxyService; @@ -59,36 +60,16 @@ public ServiceChannelInitializer(ProxyService proxyService, ProxyConfiguration s this.maxMessageSize = serviceConfig.getMaxMessageSize(); if (enableTls) { - if (tlsEnabledWithKeyStore) { - serverSSLContextAutoRefreshBuilder = new NettySSLContextAutoRefreshBuilder( - serviceConfig.getTlsProvider(), - serviceConfig.getTlsKeyStoreType(), - serviceConfig.getTlsKeyStore(), - serviceConfig.getTlsKeyStorePassword(), - serviceConfig.isTlsAllowInsecureConnection(), - serviceConfig.getTlsTrustStoreType(), - serviceConfig.getTlsTrustStore(), - serviceConfig.getTlsTrustStorePassword(), - serviceConfig.isTlsRequireTrustedClientCertOnConnect(), - serviceConfig.getTlsCiphers(), - serviceConfig.getTlsProtocols(), - serviceConfig.getTlsCertRefreshCheckDurationSec()); - } else { - SslProvider sslProvider = null; - if (serviceConfig.getTlsProvider() != null) { - sslProvider = SslProvider.valueOf(serviceConfig.getTlsProvider()); - } - serverSslCtxRefresher = new NettyServerSslContextBuilder( - sslProvider, - serviceConfig.isTlsAllowInsecureConnection(), - serviceConfig.getTlsTrustCertsFilePath(), serviceConfig.getTlsCertificateFilePath(), - serviceConfig.getTlsKeyFilePath(), serviceConfig.getTlsCiphers(), - serviceConfig.getTlsProtocols(), - serviceConfig.isTlsRequireTrustedClientCertOnConnect(), - serviceConfig.getTlsCertRefreshCheckDurationSec()); + PulsarSslConfiguration sslConfiguration = buildSslConfiguration(serviceConfig); + this.sslFactory = (PulsarSslFactory) Class.forName(serviceConfig.getSslFactoryPlugin()) + .getConstructor().newInstance(); + this.sslFactory.initialize(sslConfiguration); + this.sslFactory.createInternalSslContext(); + if (serviceConfig.getTlsCertRefreshCheckDurationSec() > 0) { + sslContextRefresher.scheduleWithFixedDelay(this::refreshSslContext, + serviceConfig.getTlsCertRefreshCheckDurationSec(), + serviceConfig.getTlsCertRefreshCheckDurationSec(), TimeUnit.SECONDS); } - } else { - this.serverSslCtxRefresher = null; } } @@ -96,14 +77,8 @@ public ServiceChannelInitializer(ProxyService proxyService, ProxyConfiguration s protected void initChannel(SocketChannel ch) throws Exception { ch.pipeline().addLast("consolidation", new FlushConsolidationHandler(1024, true)); - if (serverSslCtxRefresher != null && this.enableTls) { - SslContext sslContext = serverSslCtxRefresher.get(); - if (sslContext != null) { - ch.pipeline().addLast(TLS_HANDLER, sslContext.newHandler(ch.alloc())); - } - } else if (this.tlsEnabledWithKeyStore && serverSSLContextAutoRefreshBuilder != null) { - ch.pipeline().addLast(TLS_HANDLER, - new SslHandler(serverSSLContextAutoRefreshBuilder.get().createSSLEngine())); + if (this.enableTls) { + ch.pipeline().addLast(TLS_HANDLER, new SslHandler(this.sslFactory.createServerSslEngine(ch.alloc()))); } if (brokerProxyReadTimeoutMs > 0) { ch.pipeline().addLast("readTimeoutHandler", @@ -117,4 +92,35 @@ protected void initChannel(SocketChannel ch) throws Exception { ch.pipeline().addLast("handler", new ProxyConnection(proxyService, proxyService.getDnsAddressResolverGroup())); } + + protected PulsarSslConfiguration buildSslConfiguration(ProxyConfiguration config) { + return PulsarSslConfiguration.builder() + .tlsProvider(config.getTlsProvider()) + .tlsKeyStoreType(config.getTlsKeyStoreType()) + .tlsKeyStorePath(config.getTlsKeyStore()) + .tlsKeyStorePassword(config.getTlsKeyStorePassword()) + .tlsTrustStoreType(config.getTlsTrustStoreType()) + .tlsTrustStorePath(config.getTlsTrustStore()) + .tlsTrustStorePassword(config.getTlsTrustStorePassword()) + .tlsCiphers(config.getTlsCiphers()) + .tlsProtocols(config.getTlsProtocols()) + .tlsTrustCertsFilePath(config.getTlsTrustCertsFilePath()) + .tlsCertificateFilePath(config.getTlsCertificateFilePath()) + .tlsKeyFilePath(config.getTlsKeyFilePath()) + .allowInsecureConnection(config.isTlsAllowInsecureConnection()) + .requireTrustedClientCertOnConnect(config.isTlsRequireTrustedClientCertOnConnect()) + .tlsEnabledWithKeystore(config.isTlsEnabledWithKeyStore()) + .tlsCustomParams(config.getSslFactoryPluginParams()) + .authData(null) + .serverMode(true) + .build(); + } + + protected void refreshSslContext() { + try { + this.sslFactory.update(); + } catch (Exception e) { + log.error("Failed to refresh SSL context", e); + } + } } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java index 1ca8dc93ebf9e..3c472135bdfb0 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java @@ -19,6 +19,7 @@ package org.apache.pulsar.proxy.server; import static org.apache.pulsar.proxy.server.AdminProxyHandler.INIT_PARAM_REQUEST_BUFFER_SIZE; +import io.opentelemetry.api.OpenTelemetry; import io.prometheus.client.jetty.JettyStatisticsCollector; import java.io.IOException; import java.net.URI; @@ -28,6 +29,9 @@ import java.util.EnumSet; import java.util.List; import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import javax.servlet.DispatcherType; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.authentication.AuthenticationService; @@ -36,14 +40,23 @@ import org.apache.pulsar.broker.web.JsonMapperProvider; import org.apache.pulsar.broker.web.RateLimitingFilter; import org.apache.pulsar.broker.web.WebExecutorThreadPool; +import org.apache.pulsar.client.util.ExecutorProvider; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.apache.pulsar.jetty.tls.JettySslContextFactory; +import org.apache.pulsar.proxy.stats.PulsarProxyOpenTelemetry; +import org.eclipse.jetty.server.ConnectionFactory; import org.eclipse.jetty.server.ConnectionLimit; import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.ForwardedRequestCustomizer; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.ProxyConnectionFactory; +import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.server.handler.ContextHandlerCollection; import org.eclipse.jetty.server.handler.DefaultHandler; import org.eclipse.jetty.server.handler.HandlerCollection; @@ -78,6 +91,9 @@ public class WebServer { private ServerConnector connector; private ServerConnector connectorTls; + private ScheduledExecutorService sslRefreshScheduledExecutor; + private PulsarSslFactory sslFactory; + private final FilterInitializer filterInitializer; public WebServer(ProxyConfiguration config, AuthenticationService authenticationService) { @@ -93,47 +109,55 @@ public WebServer(ProxyConfiguration config, AuthenticationService authentication List connectors = new ArrayList<>(); HttpConfiguration httpConfig = new HttpConfiguration(); + if (config.isWebServiceTrustXForwardedFor()) { + httpConfig.addCustomizer(new ForwardedRequestCustomizer()); + } httpConfig.setOutputBufferSize(config.getHttpOutputBufferSize()); httpConfig.setRequestHeaderSize(config.getHttpMaxRequestHeaderSize()); + HttpConnectionFactory httpConnectionFactory = new HttpConnectionFactory(httpConfig); if (config.getWebServicePort().isPresent()) { this.externalServicePort = config.getWebServicePort().get(); - connector = new ServerConnector(server, new HttpConnectionFactory(httpConfig)); + List connectionFactories = new ArrayList<>(); + if (config.isWebServiceHaProxyProtocolEnabled()) { + connectionFactories.add(new ProxyConnectionFactory()); + } + connectionFactories.add(httpConnectionFactory); + connector = new ServerConnector(server, connectionFactories.toArray(new ConnectionFactory[0])); connector.setHost(config.getBindAddress()); connector.setPort(externalServicePort); connectors.add(connector); } if (config.getWebServicePortTls().isPresent()) { try { - SslContextFactory sslCtxFactory; - if (config.isTlsEnabledWithKeyStore()) { - sslCtxFactory = JettySslContextFactory.createServerSslContextWithKeystore( - config.getWebServiceTlsProvider(), - config.getTlsKeyStoreType(), - config.getTlsKeyStore(), - config.getTlsKeyStorePassword(), - config.isTlsAllowInsecureConnection(), - config.getTlsTrustStoreType(), - config.getTlsTrustStore(), - config.getTlsTrustStorePassword(), - config.isTlsRequireTrustedClientCertOnConnect(), - config.getWebServiceTlsCiphers(), - config.getWebServiceTlsProtocols(), - config.getTlsCertRefreshCheckDurationSec() - ); - } else { - sslCtxFactory = JettySslContextFactory.createServerSslContext( - config.getWebServiceTlsProvider(), - config.isTlsAllowInsecureConnection(), - config.getTlsTrustCertsFilePath(), - config.getTlsCertificateFilePath(), - config.getTlsKeyFilePath(), - config.isTlsRequireTrustedClientCertOnConnect(), - config.getWebServiceTlsCiphers(), - config.getWebServiceTlsProtocols(), - config.getTlsCertRefreshCheckDurationSec()); + this.sslRefreshScheduledExecutor = Executors.newSingleThreadScheduledExecutor( + new ExecutorProvider.ExtendedThreadFactory("pulsar-proxy-web-server-tls-refresh")); + PulsarSslConfiguration sslConfiguration = buildSslConfiguration(config); + this.sslFactory = (PulsarSslFactory) Class.forName(config.getSslFactoryPlugin()) + .getConstructor().newInstance(); + this.sslFactory.initialize(sslConfiguration); + this.sslFactory.createInternalSslContext(); + if (config.getTlsCertRefreshCheckDurationSec() > 0) { + sslRefreshScheduledExecutor.scheduleWithFixedDelay(this::refreshSslContext, + config.getTlsCertRefreshCheckDurationSec(), + config.getTlsCertRefreshCheckDurationSec(), TimeUnit.SECONDS); } - connectorTls = new ServerConnector(server, sslCtxFactory, new HttpConnectionFactory(httpConfig)); + SslContextFactory sslCtxFactory = + JettySslContextFactory.createSslContextFactory(config.getTlsProvider(), + sslFactory, config.isTlsRequireTrustedClientCertOnConnect(), + config.getWebServiceTlsCiphers(), config.getWebServiceTlsProtocols()); + List connectionFactories = new ArrayList<>(); + if (config.isWebServiceHaProxyProtocolEnabled()) { + connectionFactories.add(new ProxyConnectionFactory()); + } + connectionFactories.add(new SslConnectionFactory(sslCtxFactory, httpConnectionFactory.getProtocol())); + connectionFactories.add(httpConnectionFactory); + // org.eclipse.jetty.server.AbstractConnectionFactory.getFactories contains similar logic + // this is needed for TLS authentication + if (httpConfig.getCustomizer(SecureRequestCustomizer.class) == null) { + httpConfig.addCustomizer(new SecureRequestCustomizer()); + } + connectorTls = new ServerConnector(server, connectionFactories.toArray(new ConnectionFactory[0])); connectorTls.setPort(config.getWebServicePortTls().get()); connectorTls.setHost(config.getBindAddress()); connectors.add(connectorTls); @@ -166,7 +190,8 @@ private static class FilterInitializer { if (config.isHttpRequestsLimitEnabled()) { filterHolders.add(new FilterHolder( - new RateLimitingFilter(config.getHttpRequestsMaxPerSecond()))); + new RateLimitingFilter(config.getHttpRequestsMaxPerSecond(), + OpenTelemetry.noop().getMeter(PulsarProxyOpenTelemetry.INSTRUMENTATION_SCOPE_NAME)))); } if (config.isAuthenticationEnabled()) { @@ -197,12 +222,20 @@ public void addServlet(String basePath, ServletHolder servletHolder, List> attributes, boolean requireAuthentication) { + addServlet(basePath, servletHolder, attributes, requireAuthentication, true); + } + + private void addServlet(String basePath, ServletHolder servletHolder, + List> attributes, boolean requireAuthentication, boolean checkForExistingPaths) { popularServletParams(servletHolder, config); - Optional existingPath = servletPaths.stream().filter(p -> p.startsWith(basePath)).findFirst(); - if (existingPath.isPresent()) { - throw new IllegalArgumentException( - String.format("Cannot add servlet at %s, path %s already exists", basePath, existingPath.get())); + if (checkForExistingPaths) { + Optional existingPath = servletPaths.stream().filter(p -> p.startsWith(basePath)).findFirst(); + if (existingPath.isPresent()) { + throw new IllegalArgumentException( + String.format("Cannot add servlet at %s, path %s already exists", basePath, + existingPath.get())); + } } servletPaths.add(basePath); @@ -231,17 +264,40 @@ private static void popularServletParams(ServletHolder servletHolder, ProxyConfi } } + /** + * Add a REST resource to the servlet context with authentication coverage. + * + * @see WebServer#addRestResource(String, String, Object, Class, boolean) + * + * @param basePath The base path for the resource. + * @param attribute An attribute associated with the resource. + * @param attributeValue The value of the attribute. + * @param resourceClass The class representing the resource. + */ public void addRestResource(String basePath, String attribute, Object attributeValue, Class resourceClass) { + addRestResource(basePath, attribute, attributeValue, resourceClass, true); + } + + /** + * Add a REST resource to the servlet context. + * + * @param basePath The base path for the resource. + * @param attribute An attribute associated with the resource. + * @param attributeValue The value of the attribute. + * @param resourceClass The class representing the resource. + * @param requireAuthentication A boolean indicating whether authentication is required for this resource. + */ + public void addRestResource(String basePath, String attribute, Object attributeValue, + Class resourceClass, boolean requireAuthentication) { ResourceConfig config = new ResourceConfig(); config.register(resourceClass); config.register(JsonMapperProvider.class); ServletHolder servletHolder = new ServletHolder(new ServletContainer(config)); servletHolder.setAsyncSupported(true); - ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS); - context.setContextPath(basePath); - context.addServlet(servletHolder, MATCH_ALL); - context.setAttribute(attribute, attributeValue); - handlers.add(context); + // This method has not historically checked for existing paths, so we don't check here either. The + // method call is added to reduce code duplication. + addServlet(basePath, servletHolder, Collections.singletonList(Pair.of(attribute, attributeValue)), + requireAuthentication, false); } public int getExternalServicePort() { @@ -250,7 +306,10 @@ public int getExternalServicePort() { public void start() throws Exception { RequestLogHandler requestLogHandler = new RequestLogHandler(); - requestLogHandler.setRequestLog(JettyRequestLogFactory.createRequestLogger()); + boolean showDetailedAddresses = config.getWebServiceLogDetailedAddresses() != null + ? config.getWebServiceLogDetailedAddresses() : + (config.isWebServiceHaProxyProtocolEnabled() || config.isWebServiceTrustXForwardedFor()); + requestLogHandler.setRequestLog(JettyRequestLogFactory.createRequestLogger(showDetailedAddresses, server)); handlers.add(0, new ContextHandlerCollection()); handlers.add(requestLogHandler); @@ -301,6 +360,9 @@ public void start() throws Exception { } public void stop() throws Exception { + if (this.sslRefreshScheduledExecutor != null) { + this.sslRefreshScheduledExecutor.shutdownNow(); + } server.stop(); webServiceExecutor.stop(); log.info("Server stopped successfully"); @@ -326,5 +388,37 @@ public Optional getListenPortHTTPS() { } } + protected PulsarSslConfiguration buildSslConfiguration(ProxyConfiguration config) { + return PulsarSslConfiguration.builder() + .tlsProvider(config.getTlsProvider()) + .tlsKeyStoreType(config.getTlsKeyStoreType()) + .tlsKeyStorePath(config.getTlsKeyStore()) + .tlsKeyStorePassword(config.getTlsKeyStorePassword()) + .tlsTrustStoreType(config.getTlsTrustStoreType()) + .tlsTrustStorePath(config.getTlsTrustStore()) + .tlsTrustStorePassword(config.getTlsTrustStorePassword()) + .tlsCiphers(config.getTlsCiphers()) + .tlsProtocols(config.getTlsProtocols()) + .tlsTrustCertsFilePath(config.getTlsTrustCertsFilePath()) + .tlsCertificateFilePath(config.getTlsCertificateFilePath()) + .tlsKeyFilePath(config.getTlsKeyFilePath()) + .allowInsecureConnection(config.isTlsAllowInsecureConnection()) + .requireTrustedClientCertOnConnect(config.isTlsRequireTrustedClientCertOnConnect()) + .tlsEnabledWithKeystore(config.isTlsEnabledWithKeyStore()) + .tlsCustomParams(config.getSslFactoryPluginParams()) + .authData(null) + .serverMode(true) + .isHttps(true) + .build(); + } + + protected void refreshSslContext() { + try { + this.sslFactory.update(); + } catch (Exception e) { + log.error("Failed to refresh SSL context", e); + } + } + private static final Logger log = LoggerFactory.getLogger(WebServer.class); } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/ProxyStats.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/ProxyStats.java index 27e61c90e9e26..67fe30db1613f 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/ProxyStats.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/ProxyStats.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.proxy.stats; +import static java.util.concurrent.TimeUnit.SECONDS; import io.netty.channel.Channel; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -27,7 +28,10 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; import javax.servlet.ServletContext; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; @@ -36,21 +40,27 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response.Status; +import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; +import org.apache.pulsar.broker.web.AuthenticationFilter; import org.apache.pulsar.proxy.server.ProxyService; - - +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Path("/") @Api(value = "/proxy-stats", description = "Stats for proxy", tags = "proxy-stats", hidden = true) @Produces(MediaType.APPLICATION_JSON) public class ProxyStats { + private static final Logger log = LoggerFactory.getLogger(ProxyStats.class); public static final String ATTRIBUTE_PULSAR_PROXY_NAME = "pulsar-proxy"; private ProxyService service; @Context protected ServletContext servletContext; + @Context + protected HttpServletRequest httpRequest; @GET @Path("/connections") @@ -58,6 +68,7 @@ public class ProxyStats { response = List.class, responseContainer = "List") @ApiResponses(value = { @ApiResponse(code = 503, message = "Proxy service is not initialized") }) public List metrics() { + throwIfNotSuperUser("metrics"); List stats = new ArrayList<>(); proxyService().getClientCnxs().forEach(cnx -> { if (cnx.getDirectProxyHandler() == null) { @@ -78,7 +89,7 @@ public List metrics() { @ApiResponses(value = { @ApiResponse(code = 412, message = "Proxy logging should be > 2 to capture topic stats"), @ApiResponse(code = 503, message = "Proxy service is not initialized") }) public Map topics() { - + throwIfNotSuperUser("topics"); Optional logLevel = proxyService().getConfiguration().getProxyLogLevel(); if (!logLevel.isPresent() || logLevel.get() < 2) { throw new RestException(Status.PRECONDITION_FAILED, "Proxy doesn't have logging level 2"); @@ -92,6 +103,7 @@ public Map topics() { notes = "It only changes log-level in memory, change it config file to persist the change") @ApiResponses(value = { @ApiResponse(code = 412, message = "Proxy log level can be [0-2]"), }) public void updateProxyLogLevel(@PathParam("logLevel") int logLevel) { + throwIfNotSuperUser("updateProxyLogLevel"); if (logLevel < 0 || logLevel > 2) { throw new RestException(Status.PRECONDITION_FAILED, "Proxy log level can be only [0-2]"); } @@ -102,6 +114,7 @@ public void updateProxyLogLevel(@PathParam("logLevel") int logLevel) { @Path("/logging") @ApiOperation(hidden = true, value = "Get proxy logging") public int getProxyLogLevel(@PathParam("logLevel") int logLevel) { + throwIfNotSuperUser("getProxyLogLevel"); return proxyService().getProxyLogLevel(); } @@ -114,4 +127,26 @@ protected ProxyService proxyService() { } return service; } + + private void throwIfNotSuperUser(String action) { + if (proxyService().getConfiguration().isAuthorizationEnabled()) { + AuthenticationParameters authParams = AuthenticationParameters.builder() + .clientRole((String) httpRequest.getAttribute(AuthenticationFilter.AuthenticatedRoleAttributeName)) + .clientAuthenticationDataSource((AuthenticationDataSource) + httpRequest.getAttribute(AuthenticationFilter.AuthenticatedDataAttributeName)) + .build(); + try { + if (authParams.getClientRole() == null + || !proxyService().getAuthorizationService().isSuperUser(authParams).get(30, SECONDS)) { + log.error("Client with role [{}] is not authorized to {}", authParams.getClientRole(), action); + throw new org.apache.pulsar.common.util.RestException(Status.UNAUTHORIZED, + "Client is not authorized to perform operation"); + } + } catch (ExecutionException | TimeoutException | InterruptedException e) { + log.warn("Time-out {} sec while checking the role {} is a super user role ", 30, + authParams.getClientRole()); + throw new org.apache.pulsar.common.util.RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); + } + } + } } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/PulsarProxyOpenTelemetry.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/PulsarProxyOpenTelemetry.java new file mode 100644 index 0000000000000..2748e2c3df5b0 --- /dev/null +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/PulsarProxyOpenTelemetry.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.proxy.stats; + +import io.opentelemetry.api.metrics.Meter; +import java.io.Closeable; +import lombok.Getter; +import org.apache.pulsar.PulsarVersion; +import org.apache.pulsar.opentelemetry.OpenTelemetryService; +import org.apache.pulsar.proxy.server.ProxyConfiguration; + +public class PulsarProxyOpenTelemetry implements Closeable { + + public static final String SERVICE_NAME = "pulsar-proxy"; + public static final String INSTRUMENTATION_SCOPE_NAME = "org.apache.pulsar.proxy"; + + private final OpenTelemetryService openTelemetryService; + + @Getter + private final Meter meter; + + public PulsarProxyOpenTelemetry(ProxyConfiguration config) { + openTelemetryService = OpenTelemetryService.builder() + .clusterName(config.getClusterName()) + .serviceName(SERVICE_NAME) + .serviceVersion(PulsarVersion.getVersion()) + .build(); + meter = openTelemetryService.getOpenTelemetry().getMeter(INSTRUMENTATION_SCOPE_NAME); + } + + @Override + public void close() { + openTelemetryService.close(); + } +} diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentation.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentation.java index a1c7e30a25ed6..a9164b6d95393 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentation.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentation.java @@ -16,26 +16,41 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.pulsar.proxy.util; -import com.beust.jcommander.Parameters; -import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.common.util.BaseGenerateDocumentation; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; +import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; +import org.apache.pulsar.client.impl.conf.ReaderConfigurationData; +import org.apache.pulsar.docs.tools.BaseGenerateDocumentation; import org.apache.pulsar.proxy.server.ProxyConfiguration; +import org.apache.pulsar.websocket.service.WebSocketProxyConfiguration; -@Data -@Parameters(commandDescription = "Generate documentation automatically.") @Slf4j public class CmdGenerateDocumentation extends BaseGenerateDocumentation { @Override public String generateDocumentByClassName(String className) throws Exception { - StringBuilder sb = new StringBuilder(); if (ProxyConfiguration.class.getName().equals(className)) { - return generateDocByFieldContext(className, "Pulsar proxy", sb); + return generateDocByFieldContext(className, "Pulsar proxy"); + } else if (ServiceConfiguration.class.getName().equals(className)) { + return generateDocByFieldContext(className, "Broker"); + } else if (ClientConfigurationData.class.getName().equals(className)) { + return generateDocByApiModelProperty(className, "Client"); + } else if (WebSocketProxyConfiguration.class.getName().equals(className)) { + return generateDocByFieldContext(className, "WebSocket"); + } else if (ProducerConfigurationData.class.getName().equals(className)) { + return generateDocByApiModelProperty(className, "Producer"); + } else if (ConsumerConfigurationData.class.getName().equals(className)) { + return generateDocByApiModelProperty(className, "Consumer"); + } else if (ReaderConfigurationData.class.getName().equals(className)) { + return generateDocByApiModelProperty(className, "Reader"); + } else { + return "Class [" + className + "] not found"; } - return "Class [" + className + "] not found"; } public static void main(String[] args) throws Exception { diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/package-info.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/package-info.java index 1501a16f55813..85284c0dd229a 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/package-info.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/package-info.java @@ -16,4 +16,5 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.proxy.util; \ No newline at end of file + +package org.apache.pulsar.proxy.util; diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/extensions/SimpleProxyExtensionTestBase.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/extensions/SimpleProxyExtensionTestBase.java index 79662097c3b2f..050199acc496d 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/extensions/SimpleProxyExtensionTestBase.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/extensions/SimpleProxyExtensionTestBase.java @@ -26,6 +26,8 @@ import org.apache.commons.io.IOUtils; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.common.util.PortManager; import org.apache.pulsar.metadata.impl.ZKMetadataStore; @@ -121,6 +123,7 @@ public void close() { private ProxyService proxyService; private boolean useSeparateThreadPoolForProxyExtensions; private ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; public SimpleProxyExtensionTestBase(boolean useSeparateThreadPoolForProxyExtensions) { this.useSeparateThreadPoolForProxyExtensions = useSeparateThreadPoolForProxyExtensions; @@ -140,11 +143,17 @@ protected void setup() throws Exception { proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setMetadataStoreUrl(DUMMY_VALUE); proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); + proxyConfig.setClusterName(configClusterName); + + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService( - PulsarConfigurationLoader.convertFrom(proxyConfig)))); - doReturn(new ZKMetadataStore(mockZooKeeper)).when(proxyService).createLocalMetadataStore(); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(proxyService).createConfigurationMetadataStore(); + PulsarConfigurationLoader.convertFrom(proxyConfig)), proxyClientAuthentication)); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeper))).when(proxyService).createLocalMetadataStore(); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(proxyService) + .createConfigurationMetadataStore(); proxyService.start(); } @@ -172,6 +181,9 @@ public void testBootstrapProtocolHandler() throws Exception { protected void cleanup() throws Exception { super.internalCleanup(); proxyService.close(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } if (tempDirectory != null) { FileUtils.deleteDirectory(tempDirectory); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AdminProxyHandlerKeystoreTLSTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AdminProxyHandlerKeystoreTLSTest.java new file mode 100644 index 0000000000000..5995d11b33b21 --- /dev/null +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AdminProxyHandlerKeystoreTLSTest.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.proxy.server; + +import lombok.Cleanup; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.authentication.AuthenticationProviderTls; +import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.broker.resources.PulsarResources; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; +import org.apache.pulsar.client.impl.auth.AuthenticationKeyStoreTls; +import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.metadata.impl.ZKMetadataStore; +import org.apache.pulsar.policies.data.loadbalancer.LoadManagerReport; +import org.apache.pulsar.policies.data.loadbalancer.LoadReport; +import org.eclipse.jetty.servlet.ServletHolder; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; + +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; + +public class AdminProxyHandlerKeystoreTLSTest extends MockedPulsarServiceBaseTest { + + + private final ProxyConfiguration proxyConfig = new ProxyConfiguration(); + + private Authentication proxyClientAuthentication; + + private WebServer webServer; + + private BrokerDiscoveryProvider discoveryProvider; + + private PulsarResources resource; + + @BeforeMethod + @Override + protected void setup() throws Exception { + + conf.setAuthenticationEnabled(true); + conf.setAuthorizationEnabled(false); + conf.setWebServicePortTls(Optional.of(0)); + conf.setBrokerServicePortTls(Optional.of(0)); + conf.setTlsEnabledWithKeyStore(true); + conf.setTlsAllowInsecureConnection(false); + conf.setTlsKeyStoreType(KEYSTORE_TYPE); + conf.setTlsKeyStore(BROKER_KEYSTORE_FILE_PATH); + conf.setTlsKeyStorePassword(BROKER_KEYSTORE_PW); + conf.setTlsTrustStoreType(KEYSTORE_TYPE); + conf.setTlsTrustStore(CLIENT_TRUSTSTORE_FILE_PATH); + conf.setTlsTrustStorePassword(CLIENT_TRUSTSTORE_PW); + + super.internalSetup(); + + proxyConfig.setWebServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); + proxyConfig.setServicePortTls(Optional.of(0)); + proxyConfig.setWebServicePortTls(Optional.of(0)); + proxyConfig.setTlsEnabledWithBroker(true); + proxyConfig.setTlsEnabledWithKeyStore(true); + + proxyConfig.setTlsKeyStoreType(KEYSTORE_TYPE); + proxyConfig.setTlsKeyStore(BROKER_KEYSTORE_FILE_PATH); + proxyConfig.setTlsKeyStorePassword(BROKER_KEYSTORE_PW); + proxyConfig.setTlsTrustStoreType(KEYSTORE_TYPE); + proxyConfig.setTlsTrustStore(CLIENT_TRUSTSTORE_FILE_PATH); + proxyConfig.setTlsTrustStorePassword(CLIENT_TRUSTSTORE_PW); + + proxyConfig.setMetadataStoreUrl(DUMMY_VALUE); + proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); + proxyConfig.setBrokerClientTlsEnabledWithKeyStore(true); + proxyConfig.setBrokerClientTlsKeyStoreType(KEYSTORE_TYPE); + proxyConfig.setBrokerClientTlsKeyStore(BROKER_KEYSTORE_FILE_PATH); + proxyConfig.setBrokerClientTlsKeyStorePassword(BROKER_KEYSTORE_PW); + proxyConfig.setBrokerClientTlsTrustStoreType(KEYSTORE_TYPE); + proxyConfig.setBrokerClientTlsTrustStore(BROKER_TRUSTSTORE_FILE_PATH); + proxyConfig.setBrokerClientTlsTrustStorePassword(BROKER_TRUSTSTORE_PW); + Set providers = new HashSet<>(); + providers.add(AuthenticationProviderTls.class.getName()); + proxyConfig.setAuthenticationProviders(providers); + proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationKeyStoreTls.class.getName()); + proxyConfig.setBrokerClientAuthenticationParameters(String.format("keyStoreType:%s,keyStorePath:%s,keyStorePassword:%s", + KEYSTORE_TYPE, BROKER_KEYSTORE_FILE_PATH, BROKER_KEYSTORE_PW)); + proxyConfig.setClusterName(configClusterName); + + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); + + resource = new PulsarResources(registerCloseable(new ZKMetadataStore(mockZooKeeper)), + registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))); + webServer = new WebServer(proxyConfig, new AuthenticationService( + PulsarConfigurationLoader.convertFrom(proxyConfig))); + discoveryProvider = spy(registerCloseable(new BrokerDiscoveryProvider(proxyConfig, resource))); + LoadManagerReport report = new LoadReport(brokerUrl.toString(), brokerUrlTls.toString(), null, null); + doReturn(report).when(discoveryProvider).nextBroker(); + ServletHolder servletHolder = new ServletHolder(new AdminProxyHandler(proxyConfig, discoveryProvider, proxyClientAuthentication)); + webServer.addServlet("/admin", servletHolder); + webServer.addServlet("/lookup", servletHolder); + webServer.start(); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + webServer.stop(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } + super.internalCleanup(); + } + + PulsarAdmin getAdminClient() throws Exception { + return PulsarAdmin.builder() + .serviceHttpUrl("https://localhost:" + webServer.getListenPortHTTPS().get()) + .useKeyStoreTls(true) + .allowTlsInsecureConnection(false) + .tlsTrustStorePath(BROKER_TRUSTSTORE_FILE_PATH) + .tlsTrustStorePassword(BROKER_TRUSTSTORE_PW) + .authentication(AuthenticationKeyStoreTls.class.getName(), + String.format("keyStoreType:%s,keyStorePath:%s,keyStorePassword:%s", + KEYSTORE_TYPE, BROKER_KEYSTORE_FILE_PATH, BROKER_KEYSTORE_PW)) + .build(); + } + + @Test + public void testAdmin() throws Exception { + @Cleanup + PulsarAdmin admin = getAdminClient(); + admin.clusters().createCluster(configClusterName, ClusterData.builder().serviceUrl(brokerUrl.toString()).build()); + } + +} diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AdminProxyHandlerTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AdminProxyHandlerTest.java index becebe0059e56..fdf9242c9f3d8 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AdminProxyHandlerTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AdminProxyHandlerTest.java @@ -32,6 +32,7 @@ import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.pulsar.client.api.Authentication; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.api.Request; import org.testng.Assert; @@ -42,11 +43,11 @@ public class AdminProxyHandlerTest { private AdminProxyHandler adminProxyHandler; @BeforeClass - public void setupMocks() throws ServletException { + public void setupMocks() throws Exception { // given HttpClient httpClient = mock(HttpClient.class); adminProxyHandler = new AdminProxyHandler(mock(ProxyConfiguration.class), - mock(BrokerDiscoveryProvider.class)) { + mock(BrokerDiscoveryProvider.class), mock(Authentication.class)) { @Override protected HttpClient createHttpClient() throws ServletException { return httpClient; diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java index af70276aed95e..97bb91d924cf8 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AuthedAdminProxyHandlerTest.java @@ -32,6 +32,8 @@ import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.impl.auth.AuthenticationTls; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.common.policies.data.ClusterData; @@ -51,6 +53,7 @@ public class AuthedAdminProxyHandlerTest extends MockedPulsarServiceBaseTest { private static final Logger LOG = LoggerFactory.getLogger(AuthedAdminProxyHandlerTest.class); private ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; private WebServer webServer; private BrokerDiscoveryProvider discoveryProvider; private PulsarResources resource; @@ -85,6 +88,7 @@ protected void setup() throws Exception { proxyConfig.setWebServicePortTls(Optional.of(0)); proxyConfig.setTlsEnabledWithBroker(true); proxyConfig.setHttpMaxRequestHeaderSize(20000); + proxyConfig.setClusterName(configClusterName); // enable tls and auth&auth at proxy proxyConfig.setTlsCertificateFilePath(PROXY_CERT_FILE_PATH); @@ -98,15 +102,19 @@ protected void setup() throws Exception { proxyConfig.setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH); proxyConfig.setAuthenticationProviders(ImmutableSet.of(AuthenticationProviderTls.class.getName())); - resource = new PulsarResources(new ZKMetadataStore(mockZooKeeper), - new ZKMetadataStore(mockZooKeeperGlobal)); + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); + + resource = new PulsarResources(registerCloseable(new ZKMetadataStore(mockZooKeeper)), + registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))); webServer = new WebServer(proxyConfig, new AuthenticationService( PulsarConfigurationLoader.convertFrom(proxyConfig))); - discoveryProvider = spy(new BrokerDiscoveryProvider(proxyConfig, resource)); + discoveryProvider = spy(registerCloseable(new BrokerDiscoveryProvider(proxyConfig, resource))); LoadManagerReport report = new LoadReport(brokerUrl.toString(), brokerUrlTls.toString(), null, null); doReturn(report).when(discoveryProvider).nextBroker(); - ServletHolder servletHolder = new ServletHolder(new AdminProxyHandler(proxyConfig, discoveryProvider)); + ServletHolder servletHolder = new ServletHolder(new AdminProxyHandler(proxyConfig, discoveryProvider, proxyClientAuthentication)); webServer.addServlet("/admin", servletHolder); webServer.addServlet("/lookup", servletHolder); @@ -118,6 +126,9 @@ protected void setup() throws Exception { @Override protected void cleanup() throws Exception { webServer.stop(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } super.internalCleanup(); } @@ -186,27 +197,28 @@ public void testAuthenticatedProxyAsNonAdmin() throws Exception { @Test public void testAuthenticatedRequestWithLongUri() throws Exception { - PulsarAdmin user1Admin = getAdminClient("user1"); - PulsarAdmin brokerAdmin = getDirectToBrokerAdminClient("admin"); - StringBuilder longTenant = new StringBuilder("tenant"); - for (int i = 10 * 1024; i > 0; i = i - 4){ - longTenant.append("_abc"); - } - try { - brokerAdmin.namespaces().getNamespaces(longTenant.toString()); - Assert.fail("expect error: Tenant not found"); - } catch (Exception ex){ - Assert.assertTrue(ex instanceof PulsarAdminException); - PulsarAdminException pulsarAdminException = (PulsarAdminException) ex; - Assert.assertEquals(pulsarAdminException.getStatusCode(), 404); - } - try { - user1Admin.namespaces().getNamespaces(longTenant.toString()); - Assert.fail("expect error: Tenant not found"); - } catch (Exception ex){ - Assert.assertTrue(ex instanceof PulsarAdminException); - PulsarAdminException pulsarAdminException = (PulsarAdminException) ex; - Assert.assertEquals(pulsarAdminException.getStatusCode(), 404); + try (PulsarAdmin user1Admin = getAdminClient("user1"); + PulsarAdmin brokerAdmin = getDirectToBrokerAdminClient("admin")) { + StringBuilder longTenant = new StringBuilder("tenant"); + for (int i = 10 * 1024; i > 0; i = i - 4) { + longTenant.append("_abc"); + } + try { + brokerAdmin.namespaces().getNamespaces(longTenant.toString()); + Assert.fail("expect error: Tenant not found"); + } catch (Exception ex) { + Assert.assertTrue(ex instanceof PulsarAdminException); + PulsarAdminException pulsarAdminException = (PulsarAdminException) ex; + Assert.assertEquals(pulsarAdminException.getStatusCode(), 404); + } + try { + user1Admin.namespaces().getNamespaces(longTenant.toString()); + Assert.fail("expect error: Tenant not found"); + } catch (Exception ex) { + Assert.assertTrue(ex instanceof PulsarAdminException); + PulsarAdminException pulsarAdminException = (PulsarAdminException) ex; + Assert.assertEquals(pulsarAdminException.getStatusCode(), 404); + } } } } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/FunctionWorkerRoutingTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/FunctionWorkerRoutingTest.java index db5e9e12bd2db..a07a0f082d39a 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/FunctionWorkerRoutingTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/FunctionWorkerRoutingTest.java @@ -18,6 +18,9 @@ */ package org.apache.pulsar.proxy.server; +import lombok.Cleanup; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.testng.Assert; import org.testng.annotations.Test; @@ -37,8 +40,13 @@ public void testFunctionWorkerRedirect() throws Exception { proxyConfig.setBrokerWebServiceURL(brokerUrl); proxyConfig.setFunctionWorkerWebServiceURL(functionWorkerUrl); + @Cleanup + final Authentication proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); + BrokerDiscoveryProvider discoveryProvider = mock(BrokerDiscoveryProvider.class); - AdminProxyHandler handler = new AdminProxyHandler(proxyConfig, discoveryProvider); + AdminProxyHandler handler = new AdminProxyHandler(proxyConfig, discoveryProvider, proxyClientAuthentication); String funcUrl = handler.rewriteTarget(buildRequest("/admin/v3/functions/test/test")); Assert.assertEquals(funcUrl, String.format("%s/admin/v3/functions/%s/%s", diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/InvalidProxyConfigForAuthorizationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/InvalidProxyConfigForAuthorizationTest.java index c29bfaa964812..b7ef0855e383c 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/InvalidProxyConfigForAuthorizationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/InvalidProxyConfigForAuthorizationTest.java @@ -22,6 +22,7 @@ import static org.testng.Assert.fail; import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.client.api.Authentication; import org.mockito.Mockito; import org.testng.annotations.Test; @@ -33,7 +34,7 @@ void startupShouldFailWhenAuthorizationIsEnabledWithoutAuthentication() throws E proxyConfiguration.setAuthorizationEnabled(true); proxyConfiguration.setAuthenticationEnabled(false); try (ProxyService proxyService = new ProxyService(proxyConfiguration, - Mockito.mock(AuthenticationService.class))) { + Mockito.mock(AuthenticationService.class), Mockito.mock(Authentication.class))) { proxyService.start(); fail("An exception should have been thrown"); } catch (Exception e) { diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAdditionalServletTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAdditionalServletTest.java index 17cd3c33e799d..e12224da37199 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAdditionalServletTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAdditionalServletTest.java @@ -25,6 +25,8 @@ import org.apache.commons.lang3.RandomUtils; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.metadata.impl.ZKMetadataStore; import org.apache.pulsar.broker.web.plugin.servlet.AdditionalServletWithClassLoader; @@ -65,6 +67,7 @@ public class ProxyAdditionalServletTest extends MockedPulsarServiceBaseTest { private ProxyService proxyService; private WebServer proxyWebServer; private ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; @Override @BeforeClass @@ -78,14 +81,21 @@ protected void setup() throws Exception { proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); // enable full parsing feature proxyConfig.setProxyLogLevel(Optional.of(2)); + proxyConfig.setClusterName(configClusterName); // this is for nar package test // addServletNar(); + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); + proxyService = Mockito.spy(new ProxyService(proxyConfig, - new AuthenticationService(PulsarConfigurationLoader.convertFrom(proxyConfig)))); - doReturn(new ZKMetadataStore(mockZooKeeper)).when(proxyService).createLocalMetadataStore(); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(proxyService).createConfigurationMetadataStore(); + new AuthenticationService(PulsarConfigurationLoader.convertFrom(proxyConfig)), + proxyClientAuthentication)); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeper))).when(proxyService).createLocalMetadataStore(); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(proxyService) + .createConfigurationMetadataStore(); Optional proxyLogLevel = Optional.of(2); assertEquals(proxyLogLevel, proxyService.getConfiguration().getProxyLogLevel()); @@ -97,7 +107,7 @@ protected void setup() throws Exception { mockAdditionalServlet(); proxyWebServer = new WebServer(proxyConfig, authService); - ProxyServiceStarter.addWebServerHandlers(proxyWebServer, proxyConfig, proxyService, null); + ProxyServiceStarter.addWebServerHandlers(proxyWebServer, proxyConfig, proxyService, null, proxyClientAuthentication); proxyWebServer.start(); } @@ -177,6 +187,10 @@ protected void cleanup() throws Exception { internalCleanup(); proxyService.close(); + proxyWebServer.stop(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } } @Test diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticatedProducerConsumerTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticatedProducerConsumerTest.java index bfe86f86976ee..2a9a9f15b4568 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticatedProducerConsumerTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticatedProducerConsumerTest.java @@ -35,6 +35,7 @@ import org.apache.pulsar.broker.authentication.AuthenticationService; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Producer; @@ -74,6 +75,7 @@ public class ProxyAuthenticatedProducerConsumerTest extends ProducerConsumerBase private ProxyService proxyService; private ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; private final String configClusterName = "test"; @BeforeMethod @@ -137,11 +139,17 @@ protected void setup() throws Exception { proxyConfig.setMetadataStoreUrl(DUMMY_VALUE); proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); + proxyConfig.setClusterName(configClusterName); + + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService( - PulsarConfigurationLoader.convertFrom(proxyConfig)))); - doReturn(new ZKMetadataStore(mockZooKeeper)).when(proxyService).createLocalMetadataStore(); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(proxyService).createConfigurationMetadataStore(); + PulsarConfigurationLoader.convertFrom(proxyConfig)), proxyClientAuthentication)); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeper))).when(proxyService).createLocalMetadataStore(); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(proxyService) + .createConfigurationMetadataStore(); proxyService.start(); } @@ -150,6 +158,9 @@ protected void setup() throws Exception { protected void cleanup() throws Exception { super.internalCleanup(); proxyService.close(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } } /** @@ -220,6 +231,7 @@ public void testTlsSyncProducerAndConsumer() throws Exception { } protected final PulsarClient createPulsarClient(Authentication auth, String lookupUrl) throws Exception { + closeAdmin(); admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString()) .tlsTrustCertsFilePath(TLS_BROKER_TRUST_CERT_FILE_PATH) .enableTlsHostnameVerification(true).authentication(auth).build()); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java index 8229d929ee5e3..7d3cf57d594df 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java @@ -43,6 +43,7 @@ import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.AuthenticationDataProvider; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.ProducerConsumerBase; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; @@ -58,6 +59,7 @@ public class ProxyAuthenticationTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(ProxyAuthenticationTest.class); + private static final String CLUSTER_NAME = "test"; public static class BasicAuthenticationData implements AuthenticationDataProvider { private final String authParam; @@ -168,7 +170,7 @@ protected void setup() throws Exception { conf.setBrokerClientAuthenticationPlugin(BasicAuthentication.class.getName()); // Expires after an hour conf.setBrokerClientAuthenticationParameters( - "entityType:broker,expiryTime:" + (System.currentTimeMillis() + 3600 * 1000)); + "entityType:admin,expiryTime:" + (System.currentTimeMillis() + 3600 * 1000)); Set superUserRoles = new HashSet<>(); superUserRoles.add("admin"); @@ -178,7 +180,7 @@ protected void setup() throws Exception { providers.add(BasicAuthenticationProvider.class.getName()); conf.setAuthenticationProviders(providers); - conf.setClusterName("test"); + conf.setClusterName(CLUSTER_NAME); Set proxyRoles = new HashSet<>(); proxyRoles.add("proxy"); conf.setProxyRoles(proxyRoles); @@ -222,6 +224,7 @@ void testAuthentication() throws Exception { proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl()); + proxyConfig.setClusterName(CLUSTER_NAME); proxyConfig.setBrokerClientAuthenticationPlugin(BasicAuthentication.class.getName()); proxyConfig.setBrokerClientAuthenticationParameters(proxyAuthParams); @@ -233,7 +236,11 @@ void testAuthentication() throws Exception { AuthenticationService authenticationService = new AuthenticationService( PulsarConfigurationLoader.convertFrom(proxyConfig)); @Cleanup - ProxyService proxyService = new ProxyService(proxyConfig, authenticationService); + final Authentication proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); + @Cleanup + ProxyService proxyService = new ProxyService(proxyConfig, authenticationService, proxyClientAuthentication); proxyService.start(); final String proxyServiceUrl = proxyService.getServiceUrl(); @@ -255,6 +262,7 @@ void testAuthentication() throws Exception { private void updateAdminClient() throws PulsarClientException { // Expires after an hour String adminAuthParams = "entityType:admin,expiryTime:" + (System.currentTimeMillis() + 3600 * 1000); + closeAdmin(); admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrl.toString()) .authentication(BasicAuthentication.class.getName(), adminAuthParams).build()); } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConfigurationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConfigurationTest.java index 97a73c20b60d0..a9a562e04c899 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConfigurationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConfigurationTest.java @@ -20,6 +20,8 @@ import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.testng.annotations.Test; import java.beans.Introspector; @@ -36,6 +38,8 @@ import java.util.Properties; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; @Test(groups = "broker") public class ProxyConfigurationTest { @@ -134,4 +138,119 @@ public void testConvert() throws IOException { } } + @Test + public void testBrokerUrlCheck() throws IOException { + ProxyConfiguration configuration = new ProxyConfiguration(); + // brokerServiceURL must start with pulsar:// + configuration.setBrokerServiceURL("127.0.0.1:6650"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("brokerServiceURL must start with pulsar://"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("brokerServiceURL must start with pulsar://")); + } + } + configuration.setBrokerServiceURL("pulsar://127.0.0.1:6650"); + + // brokerServiceURLTLS must start with pulsar+ssl:// + configuration.setBrokerServiceURLTLS("pulsar://127.0.0.1:6650"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("brokerServiceURLTLS must start with pulsar+ssl://"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("brokerServiceURLTLS must start with pulsar+ssl://")); + } + } + + // brokerServiceURL did not support multi urls yet. + configuration.setBrokerServiceURL("pulsar://127.0.0.1:6650,pulsar://127.0.0.2:6650"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("brokerServiceURL does not support multi urls yet"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("does not support multi urls yet")); + } + } + configuration.setBrokerServiceURL("pulsar://127.0.0.1:6650"); + + // brokerServiceURLTLS did not support multi urls yet. + configuration.setBrokerServiceURLTLS("pulsar+ssl://127.0.0.1:6650,pulsar+ssl:127.0.0.2:6650"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("brokerServiceURLTLS does not support multi urls yet"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("does not support multi urls yet")); + } + } + configuration.setBrokerServiceURLTLS("pulsar+ssl://127.0.0.1:6650"); + + // brokerWebServiceURL did not support multi urls yet. + configuration.setBrokerWebServiceURL("http://127.0.0.1:8080,http://127.0.0.2:8080"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("brokerWebServiceURL does not support multi urls yet"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("does not support multi urls yet")); + } + } + configuration.setBrokerWebServiceURL("http://127.0.0.1:8080"); + + // brokerWebServiceURLTLS did not support multi urls yet. + configuration.setBrokerWebServiceURLTLS("https://127.0.0.1:443,https://127.0.0.2:443"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("brokerWebServiceURLTLS does not support multi urls yet"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("does not support multi urls yet")); + } + } + configuration.setBrokerWebServiceURLTLS("https://127.0.0.1:443"); + + // functionWorkerWebServiceURL did not support multi urls yet. + configuration.setFunctionWorkerWebServiceURL("http://127.0.0.1:8080,http://127.0.0.2:8080"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("functionWorkerWebServiceURL does not support multi urls yet"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("does not support multi urls yet")); + } + } + configuration.setFunctionWorkerWebServiceURL("http://127.0.0.1:8080"); + + // functionWorkerWebServiceURLTLS did not support multi urls yet. + configuration.setFunctionWorkerWebServiceURLTLS("http://127.0.0.1:443,http://127.0.0.2:443"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("functionWorkerWebServiceURLTLS does not support multi urls yet"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("does not support multi urls yet")); + } + } + configuration.setFunctionWorkerWebServiceURLTLS("http://127.0.0.1:443"); + } + } \ No newline at end of file diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConnectionThrottlingTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConnectionThrottlingTest.java index 336f11ae19da6..671e68e5c3fb7 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConnectionThrottlingTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConnectionThrottlingTest.java @@ -27,6 +27,8 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationService; import org.apache.pulsar.broker.limiter.ConnectionController; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Schema; @@ -46,6 +48,7 @@ public class ProxyConnectionThrottlingTest extends MockedPulsarServiceBaseTest { private final int NUM_CONCURRENT_INBOUND_CONNECTION = 4; private ProxyService proxyService; private ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; @Override @BeforeClass @@ -59,10 +62,15 @@ protected void setup() throws Exception { proxyConfig.setMaxConcurrentLookupRequests(NUM_CONCURRENT_LOOKUP); proxyConfig.setMaxConcurrentInboundConnections(NUM_CONCURRENT_INBOUND_CONNECTION); proxyConfig.setMaxConcurrentInboundConnectionsPerIp(NUM_CONCURRENT_INBOUND_CONNECTION); + proxyConfig.setClusterName(configClusterName); + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService( - PulsarConfigurationLoader.convertFrom(proxyConfig)))); - doReturn(new ZKMetadataStore(mockZooKeeper)).when(proxyService).createLocalMetadataStore(); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(proxyService).createConfigurationMetadataStore(); + PulsarConfigurationLoader.convertFrom(proxyConfig)), proxyClientAuthentication)); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeper))).when(proxyService).createLocalMetadataStore(); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(proxyService) + .createConfigurationMetadataStore(); proxyService.start(); } @@ -72,6 +80,9 @@ protected void setup() throws Exception { protected void cleanup() throws Exception { internalCleanup(); proxyService.close(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } } @Test diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyDisableZeroCopyTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyDisableZeroCopyTest.java index 3aa71413d540b..6a3992c550fd3 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyDisableZeroCopyTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyDisableZeroCopyTest.java @@ -18,32 +18,11 @@ */ package org.apache.pulsar.proxy.server; -import static org.mockito.Mockito.doReturn; -import java.util.Optional; -import org.apache.pulsar.broker.authentication.AuthenticationService; -import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; -import org.apache.pulsar.metadata.impl.ZKMetadataStore; -import org.mockito.Mockito; -import org.testng.annotations.BeforeClass; - public class ProxyDisableZeroCopyTest extends ProxyTest { @Override - @BeforeClass - protected void setup() throws Exception { - internalSetup(); - - proxyConfig.setServicePort(Optional.ofNullable(0)); - proxyConfig.setBrokerProxyAllowedTargetPorts("*"); - proxyConfig.setMetadataStoreUrl(DUMMY_VALUE); - proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); + protected void initializeProxyConfig() throws Exception { + super.initializeProxyConfig(); proxyConfig.setProxyZeroCopyModeEnabled(false); - - proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService( - PulsarConfigurationLoader.convertFrom(proxyConfig)))); - doReturn(new ZKMetadataStore(mockZooKeeper)).when(proxyService).createLocalMetadataStore(); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(proxyService).createConfigurationMetadataStore(); - - proxyService.start(); } } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyEnableHAProxyProtocolTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyEnableHAProxyProtocolTest.java index 8b3092c6f5170..40aa8f5040556 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyEnableHAProxyProtocolTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyEnableHAProxyProtocolTest.java @@ -22,6 +22,8 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationService; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.impl.ConsumerImpl; @@ -48,6 +50,7 @@ public class ProxyEnableHAProxyProtocolTest extends MockedPulsarServiceBaseTest private ProxyService proxyService; private ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; @Override @BeforeClass @@ -60,11 +63,17 @@ protected void setup() throws Exception { proxyConfig.setMetadataStoreUrl(DUMMY_VALUE); proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); proxyConfig.setHaProxyProtocolEnabled(true); + proxyConfig.setClusterName(configClusterName); + + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService( - PulsarConfigurationLoader.convertFrom(proxyConfig)))); - doReturn(new ZKMetadataStore(mockZooKeeper)).when(proxyService).createLocalMetadataStore(); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(proxyService).createConfigurationMetadataStore(); + PulsarConfigurationLoader.convertFrom(proxyConfig)), proxyClientAuthentication)); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeper))).when(proxyService).createLocalMetadataStore(); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(proxyService) + .createConfigurationMetadataStore(); proxyService.start(); } @@ -75,6 +84,9 @@ protected void cleanup() throws Exception { internalCleanup(); proxyService.close(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } } @Test diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyForwardAuthDataTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyForwardAuthDataTest.java index 99af3b1cf6abe..9c3a69b5f4451 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyForwardAuthDataTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyForwardAuthDataTest.java @@ -30,6 +30,8 @@ import org.apache.pulsar.broker.authentication.AuthenticationService; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.ProducerConsumerBase; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; @@ -46,6 +48,7 @@ public class ProxyForwardAuthDataTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(ProxyForwardAuthDataTest.class); + private static final String CLUSTER_NAME = "test"; @BeforeMethod @Override @@ -53,7 +56,7 @@ protected void setup() throws Exception { conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(true); conf.setBrokerClientAuthenticationPlugin(BasicAuthentication.class.getName()); - conf.setBrokerClientAuthenticationParameters("authParam:broker"); + conf.setBrokerClientAuthenticationParameters("authParam:admin"); conf.setAuthenticateOriginalAuthData(true); Set superUserRoles = new HashSet(); @@ -64,7 +67,7 @@ protected void setup() throws Exception { providers.add(BasicAuthenticationProvider.class.getName()); conf.setAuthenticationProviders(providers); - conf.setClusterName("test"); + conf.setClusterName(CLUSTER_NAME); Set proxyRoles = new HashSet(); proxyRoles.add("proxy"); conf.setProxyRoles(proxyRoles); @@ -109,6 +112,7 @@ public void testForwardAuthData() throws Exception { proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl()); proxyConfig.setBrokerClientAuthenticationPlugin(BasicAuthentication.class.getName()); proxyConfig.setBrokerClientAuthenticationParameters(proxyAuthParams); + proxyConfig.setClusterName(CLUSTER_NAME); Set providers = new HashSet<>(); providers.add(BasicAuthenticationProvider.class.getName()); @@ -116,7 +120,11 @@ public void testForwardAuthData() throws Exception { AuthenticationService authenticationService = new AuthenticationService( PulsarConfigurationLoader.convertFrom(proxyConfig)); - try (ProxyService proxyService = new ProxyService(proxyConfig, authenticationService)) { + @Cleanup + final Authentication proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); + try (ProxyService proxyService = new ProxyService(proxyConfig, authenticationService, proxyClientAuthentication)) { proxyService.start(); try (PulsarClient proxyClient = createPulsarClient(proxyService.getServiceUrl(), clientAuthParams)) { proxyClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscribe(); @@ -132,7 +140,7 @@ public void testForwardAuthData() throws Exception { PulsarConfigurationLoader.convertFrom(proxyConfig)); @Cleanup - ProxyService proxyService = new ProxyService(proxyConfig, authenticationService); + ProxyService proxyService = new ProxyService(proxyConfig, authenticationService, proxyClientAuthentication); proxyService.start(); @Cleanup @@ -142,6 +150,7 @@ public void testForwardAuthData() throws Exception { private void createAdminClient() throws PulsarClientException { String adminAuthParams = "authParam:admin"; + closeAdmin(); admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrl.toString()) .authentication(BasicAuthentication.class.getName(), adminAuthParams).build()); } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyIsAHttpProxyTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyIsAHttpProxyTest.java index 246dd9f85e319..cf587015544b7 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyIsAHttpProxyTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyIsAHttpProxyTest.java @@ -19,27 +19,26 @@ package org.apache.pulsar.proxy.server; import static java.nio.charset.StandardCharsets.UTF_8; - import java.io.IOException; import java.util.Properties; import java.util.Queue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.LinkedBlockingQueue; import java.util.function.BooleanSupplier; - import javax.servlet.AsyncContext; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; - import javax.ws.rs.client.Client; import javax.ws.rs.client.ClientBuilder; import javax.ws.rs.core.Response; - +import lombok.Cleanup; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationService; import org.apache.pulsar.broker.resources.PulsarResources; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.metadata.impl.ZKMetadataStore; import org.eclipse.jetty.client.HttpClient; @@ -56,7 +55,6 @@ import org.eclipse.jetty.util.ProcessorUtils; import org.glassfish.jersey.client.ClientConfig; import org.glassfish.jersey.logging.LoggingFeature; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -81,8 +79,8 @@ protected void setup() throws Exception { // Set number of CPU's to two for unit tests for running in resource constrained env. ProcessorUtils.setAvailableProcessors(2); - resource = new PulsarResources(new ZKMetadataStore(mockZooKeeper), - new ZKMetadataStore(mockZooKeeperGlobal)); + resource = new PulsarResources(registerCloseable(new ZKMetadataStore(mockZooKeeper)), + registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))); backingServer1 = new Server(0); backingServer1.setHandler(newHandler("server1")); backingServer1.start(); @@ -164,6 +162,7 @@ protected void cleanup() throws Exception { backingServer1.stop(); backingServer2.stop(); + backingServer3.stop(); client.close(); } @@ -201,10 +200,14 @@ public void testSingleRedirect() throws Exception { ProxyConfiguration proxyConfig = PulsarConfigurationLoader.create(props, ProxyConfiguration.class); AuthenticationService authService = new AuthenticationService( PulsarConfigurationLoader.convertFrom(proxyConfig)); + @Cleanup + final Authentication proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); WebServer webServer = new WebServer(proxyConfig, authService); ProxyServiceStarter.addWebServerHandlers(webServer, proxyConfig, null, - new BrokerDiscoveryProvider(proxyConfig, resource)); + registerCloseable(new BrokerDiscoveryProvider(proxyConfig, resource)), proxyClientAuthentication); webServer.start(); try { Response r = client.target(webServer.getServiceUri()).path("/ui/foobar").request().get(); @@ -230,10 +233,14 @@ public void testMultipleRedirect() throws Exception { ProxyConfiguration proxyConfig = PulsarConfigurationLoader.create(props, ProxyConfiguration.class); AuthenticationService authService = new AuthenticationService( PulsarConfigurationLoader.convertFrom(proxyConfig)); + @Cleanup + final Authentication proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); WebServer webServer = new WebServer(proxyConfig, authService); ProxyServiceStarter.addWebServerHandlers(webServer, proxyConfig, null, - new BrokerDiscoveryProvider(proxyConfig, resource)); + registerCloseable(new BrokerDiscoveryProvider(proxyConfig, resource)), proxyClientAuthentication); webServer.start(); try { Response r1 = client.target(webServer.getServiceUri()).path("/server1/foobar").request().get(); @@ -261,10 +268,14 @@ public void testTryingToUseExistingPath() throws Exception { ProxyConfiguration proxyConfig = PulsarConfigurationLoader.create(props, ProxyConfiguration.class); AuthenticationService authService = new AuthenticationService( PulsarConfigurationLoader.convertFrom(proxyConfig)); + @Cleanup + final Authentication proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); WebServer webServer = new WebServer(proxyConfig, authService); ProxyServiceStarter.addWebServerHandlers(webServer, proxyConfig, null, - new BrokerDiscoveryProvider(proxyConfig, resource)); + registerCloseable(new BrokerDiscoveryProvider(proxyConfig, resource)), proxyClientAuthentication); } @@ -280,10 +291,14 @@ public void testLongPathInProxyTo() throws Exception { ProxyConfiguration proxyConfig = PulsarConfigurationLoader.create(props, ProxyConfiguration.class); AuthenticationService authService = new AuthenticationService( PulsarConfigurationLoader.convertFrom(proxyConfig)); + @Cleanup + final Authentication proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); WebServer webServer = new WebServer(proxyConfig, authService); ProxyServiceStarter.addWebServerHandlers(webServer, proxyConfig, null, - new BrokerDiscoveryProvider(proxyConfig, resource)); + registerCloseable(new BrokerDiscoveryProvider(proxyConfig, resource)), proxyClientAuthentication); webServer.start(); try { Response r = client.target(webServer.getServiceUri()).path("/ui/foobar").request().get(); @@ -307,10 +322,14 @@ public void testProxyToEndsInSlash() throws Exception { ProxyConfiguration proxyConfig = PulsarConfigurationLoader.create(props, ProxyConfiguration.class); AuthenticationService authService = new AuthenticationService( PulsarConfigurationLoader.convertFrom(proxyConfig)); + @Cleanup + final Authentication proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); WebServer webServer = new WebServer(proxyConfig, authService); ProxyServiceStarter.addWebServerHandlers(webServer, proxyConfig, null, - new BrokerDiscoveryProvider(proxyConfig, resource)); + registerCloseable(new BrokerDiscoveryProvider(proxyConfig, resource)), proxyClientAuthentication); webServer.start(); try { Response r = client.target(webServer.getServiceUri()).path("/ui/foobar").request().get(); @@ -333,10 +352,14 @@ public void testLongPath() throws Exception { ProxyConfiguration proxyConfig = PulsarConfigurationLoader.create(props, ProxyConfiguration.class); AuthenticationService authService = new AuthenticationService( PulsarConfigurationLoader.convertFrom(proxyConfig)); + @Cleanup + final Authentication proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); WebServer webServer = new WebServer(proxyConfig, authService); ProxyServiceStarter.addWebServerHandlers(webServer, proxyConfig, null, - new BrokerDiscoveryProvider(proxyConfig, resource)); + registerCloseable(new BrokerDiscoveryProvider(proxyConfig, resource)), proxyClientAuthentication); webServer.start(); try { Response r = client.target(webServer.getServiceUri()).path("/foo/bar/blah/foobar").request().get(); @@ -358,6 +381,10 @@ public void testLongUri() throws Exception { ProxyConfiguration proxyConfig = PulsarConfigurationLoader.create(props, ProxyConfiguration.class); AuthenticationService authService = new AuthenticationService( PulsarConfigurationLoader.convertFrom(proxyConfig)); + @Cleanup + final Authentication proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); StringBuilder longUri = new StringBuilder("/service3/tp"); for (int i = 10 * 1024; i > 0; i = i - 11){ @@ -366,7 +393,7 @@ public void testLongUri() throws Exception { WebServer webServerMaxUriLen8k = new WebServer(proxyConfig, authService); ProxyServiceStarter.addWebServerHandlers(webServerMaxUriLen8k, proxyConfig, null, - new BrokerDiscoveryProvider(proxyConfig, resource)); + registerCloseable(new BrokerDiscoveryProvider(proxyConfig, resource)), proxyClientAuthentication); webServerMaxUriLen8k.start(); try { Response r = client.target(webServerMaxUriLen8k.getServiceUri()).path(longUri.toString()).request().get(); @@ -378,7 +405,7 @@ public void testLongUri() throws Exception { proxyConfig.setHttpMaxRequestHeaderSize(12 * 1024); WebServer webServerMaxUriLen12k = new WebServer(proxyConfig, authService); ProxyServiceStarter.addWebServerHandlers(webServerMaxUriLen12k, proxyConfig, null, - new BrokerDiscoveryProvider(proxyConfig, resource)); + registerCloseable(new BrokerDiscoveryProvider(proxyConfig, resource)), proxyClientAuthentication); webServerMaxUriLen12k.start(); try { Response r = client.target(webServerMaxUriLen12k.getServiceUri()).path(longUri.toString()).request().get(); @@ -399,10 +426,14 @@ public void testPathEndsInSlash() throws Exception { ProxyConfiguration proxyConfig = PulsarConfigurationLoader.create(props, ProxyConfiguration.class); AuthenticationService authService = new AuthenticationService( PulsarConfigurationLoader.convertFrom(proxyConfig)); + @Cleanup + final Authentication proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); WebServer webServer = new WebServer(proxyConfig, authService); ProxyServiceStarter.addWebServerHandlers(webServer, proxyConfig, null, - new BrokerDiscoveryProvider(proxyConfig, resource)); + registerCloseable(new BrokerDiscoveryProvider(proxyConfig, resource)), proxyClientAuthentication); webServer.start(); try { Response r = client.target(webServer.getServiceUri()).path("/ui/foobar").request().get(); @@ -431,10 +462,14 @@ public void testStreaming() throws Exception { ProxyConfiguration proxyConfig = PulsarConfigurationLoader.create(props, ProxyConfiguration.class); AuthenticationService authService = new AuthenticationService( PulsarConfigurationLoader.convertFrom(proxyConfig)); + @Cleanup + final Authentication proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); WebServer webServer = new WebServer(proxyConfig, authService); ProxyServiceStarter.addWebServerHandlers(webServer, proxyConfig, null, - new BrokerDiscoveryProvider(proxyConfig, resource)); + registerCloseable(new BrokerDiscoveryProvider(proxyConfig, resource)), proxyClientAuthentication); webServer.start(); HttpClient httpClient = new HttpClient(); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTransportTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTransportTest.java index 5c4e40ed65a70..8aa5581a0fe46 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTransportTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTransportTest.java @@ -24,6 +24,8 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; @@ -40,6 +42,7 @@ public class ProxyKeyStoreTlsTransportTest extends MockedPulsarServiceBaseTest { private ProxyService proxyService; private ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; @Override @BeforeMethod @@ -77,6 +80,7 @@ protected void setup() throws Exception { proxyConfig.setMetadataStoreUrl(DUMMY_VALUE); proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); + proxyConfig.setClusterName(configClusterName); proxyConfig.setTlsRequireTrustedClientCertOnConnect(false); @@ -86,11 +90,16 @@ protected void setup() throws Exception { proxyConfig.setBrokerClientTlsTrustStore(BROKER_TRUSTSTORE_FILE_PATH); proxyConfig.setBrokerClientTlsTrustStorePassword(BROKER_TRUSTSTORE_PW); + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); + proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService( - PulsarConfigurationLoader.convertFrom(proxyConfig)))); - doReturn(new ZKMetadataStore(mockZooKeeper)).when(proxyService).createLocalMetadataStore(); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(proxyService).createConfigurationMetadataStore(); + PulsarConfigurationLoader.convertFrom(proxyConfig)), proxyClientAuthentication)); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeper))).when(proxyService).createLocalMetadataStore(); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(proxyService) + .createConfigurationMetadataStore(); proxyService.start(); } @@ -101,6 +110,9 @@ protected void cleanup() throws Exception { internalCleanup(); proxyService.close(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } } protected PulsarClient newClient() throws Exception { diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTestWithAuth.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsWithAuthTest.java similarity index 88% rename from pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTestWithAuth.java rename to pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsWithAuthTest.java index 88e7b269d6eeb..2c6d080bf2c0f 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTestWithAuth.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsWithAuthTest.java @@ -33,6 +33,8 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationProviderTls; import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -51,9 +53,10 @@ import org.testng.annotations.Test; @Slf4j -public class ProxyKeyStoreTlsTestWithAuth extends MockedPulsarServiceBaseTest { +public class ProxyKeyStoreTlsWithAuthTest extends MockedPulsarServiceBaseTest { private ProxyService proxyService; private ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; @Override @BeforeMethod @@ -77,6 +80,7 @@ protected void setup() throws Exception { proxyConfig.setMetadataStoreUrl(DUMMY_VALUE); proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); + proxyConfig.setClusterName(configClusterName); // config for authentication and authorization. proxyConfig.setTlsRequireTrustedClientCertOnConnect(true); @@ -87,11 +91,16 @@ protected void setup() throws Exception { providers.add(AuthenticationProviderTls.class.getName()); proxyConfig.setAuthenticationProviders(providers); + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); + proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService( - PulsarConfigurationLoader.convertFrom(proxyConfig)))); - doReturn(new ZKMetadataStore(mockZooKeeper)).when(proxyService).createLocalMetadataStore(); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(proxyService).createConfigurationMetadataStore(); + PulsarConfigurationLoader.convertFrom(proxyConfig)), proxyClientAuthentication)); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeper))).when(proxyService).createLocalMetadataStore(); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(proxyService) + .createConfigurationMetadataStore(); proxyService.start(); } @@ -102,6 +111,9 @@ protected void cleanup() throws Exception { internalCleanup(); proxyService.close(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } } protected PulsarClient internalSetUpForClient(boolean addCertificates, String lookupUrl) throws Exception { diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTestWithoutAuth.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsWithoutAuthTest.java similarity index 87% rename from pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTestWithoutAuth.java rename to pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsWithoutAuthTest.java index 5feef74e3b94b..3a20273b8c067 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTestWithoutAuth.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsWithoutAuthTest.java @@ -29,6 +29,8 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -47,9 +49,10 @@ import org.testng.annotations.Test; @Slf4j -public class ProxyKeyStoreTlsTestWithoutAuth extends MockedPulsarServiceBaseTest { +public class ProxyKeyStoreTlsWithoutAuthTest extends MockedPulsarServiceBaseTest { private ProxyService proxyService; private ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; @Override @BeforeMethod @@ -74,11 +77,17 @@ protected void setup() throws Exception { proxyConfig.setMetadataStoreUrl(DUMMY_VALUE); proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); + proxyConfig.setClusterName(configClusterName); + + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService( - PulsarConfigurationLoader.convertFrom(proxyConfig)))); - doReturn(new ZKMetadataStore(mockZooKeeper)).when(proxyService).createLocalMetadataStore(); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(proxyService).createConfigurationMetadataStore(); + PulsarConfigurationLoader.convertFrom(proxyConfig)), proxyClientAuthentication)); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeper))).when(proxyService).createLocalMetadataStore(); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(proxyService) + .createConfigurationMetadataStore(); proxyService.start(); } @@ -107,6 +116,9 @@ protected PulsarClient internalSetUpForClient(boolean addCertificates, String lo protected void cleanup() throws Exception { internalCleanup(); proxyService.close(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } } @Test diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyLookupThrottlingTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyLookupThrottlingTest.java index 4861117ef6ff5..4d12fdd77e763 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyLookupThrottlingTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyLookupThrottlingTest.java @@ -20,18 +20,28 @@ import static org.mockito.Mockito.doReturn; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import java.util.Optional; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import lombok.Cleanup; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.BinaryProtoLookupService; +import org.apache.pulsar.client.impl.ClientCnx; +import org.apache.pulsar.client.impl.LookupService; +import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.metadata.impl.ZKMetadataStore; import org.mockito.Mockito; import org.testng.Assert; @@ -45,6 +55,7 @@ public class ProxyLookupThrottlingTest extends MockedPulsarServiceBaseTest { private final int NUM_CONCURRENT_INBOUND_CONNECTION = 5; private ProxyService proxyService; private ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; @Override @BeforeMethod(alwaysRun = true) @@ -57,12 +68,17 @@ protected void setup() throws Exception { proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); proxyConfig.setMaxConcurrentLookupRequests(NUM_CONCURRENT_LOOKUP); proxyConfig.setMaxConcurrentInboundConnections(NUM_CONCURRENT_INBOUND_CONNECTION); + proxyConfig.setClusterName(configClusterName); AuthenticationService authenticationService = new AuthenticationService( PulsarConfigurationLoader.convertFrom(proxyConfig)); - proxyService = Mockito.spy(new ProxyService(proxyConfig, authenticationService)); - doReturn(new ZKMetadataStore(mockZooKeeper)).when(proxyService).createLocalMetadataStore(); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(proxyService).createConfigurationMetadataStore(); + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); + proxyService = Mockito.spy(new ProxyService(proxyConfig, authenticationService, proxyClientAuthentication)); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeper))).when(proxyService).createLocalMetadataStore(); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(proxyService) + .createConfigurationMetadataStore(); proxyService.start(); } @@ -74,6 +90,9 @@ protected void cleanup() throws Exception { if (proxyService != null) { proxyService.close(); } + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } } @Test(groups = "quarantine") @@ -112,4 +131,32 @@ public void testLookup() throws Exception { Assert.assertEquals(LookupProxyHandler.REJECTED_PARTITIONS_METADATA_REQUESTS.get(), 5.0d); } + + @Test + public void testLookupThrottling() throws Exception { + PulsarClientImpl client = (PulsarClientImpl) PulsarClient.builder() + .serviceUrl(proxyService.getServiceUrl()).build(); + String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + LookupService lookupService = client.getLookup(); + assertTrue(lookupService instanceof BinaryProtoLookupService); + ClientCnx lookupConnection = client.getCnxPool().getConnection(lookupService.resolveHost()).join(); + + // Make no permits to lookup. + Semaphore lookupSemaphore = proxyService.getLookupRequestSemaphore(); + int availablePermits = lookupSemaphore.availablePermits(); + lookupSemaphore.acquire(availablePermits); + + // Verify will receive too many request exception, and the socket will not be closed. + try { + lookupService.getBroker(TopicName.get(tpName)).get(); + fail("Expected too many request error."); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("Too many")); + } + assertTrue(lookupConnection.ctx().channel().isActive()); + + // cleanup. + lookupSemaphore.release(availablePermits); + client.close(); + } } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyMutualTlsTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyMutualTlsTest.java index ad237c2539700..ab428c31b7fd9 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyMutualTlsTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyMutualTlsTest.java @@ -26,6 +26,8 @@ import lombok.Cleanup; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; @@ -48,6 +50,7 @@ public class ProxyMutualTlsTest extends MockedPulsarServiceBaseTest { private ProxyService proxyService; private ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; @Override @BeforeClass @@ -66,11 +69,17 @@ protected void setup() throws Exception { proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); proxyConfig.setTlsRequireTrustedClientCertOnConnect(true); proxyConfig.setTlsAllowInsecureConnection(false); + proxyConfig.setClusterName(configClusterName); + + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService( - PulsarConfigurationLoader.convertFrom(proxyConfig)))); - doReturn(new ZKMetadataStore(mockZooKeeper)).when(proxyService).createLocalMetadataStore(); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(proxyService).createConfigurationMetadataStore(); + PulsarConfigurationLoader.convertFrom(proxyConfig)), proxyClientAuthentication)); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeper))).when(proxyService).createLocalMetadataStore(); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(proxyService) + .createConfigurationMetadataStore(); proxyService.start(); } @@ -81,6 +90,9 @@ protected void cleanup() throws Exception { internalCleanup(); proxyService.close(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } } @Test diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyOriginalClientIPTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyOriginalClientIPTest.java new file mode 100644 index 0000000000000..b267439d47113 --- /dev/null +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyOriginalClientIPTest.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.proxy.server; + +import static org.testng.Assert.assertTrue; +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import nl.altindag.console.ConsoleCaptor; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.assertj.core.api.ThrowingConsumer; +import org.awaitility.Awaitility; +import org.eclipse.jetty.client.HttpClient; +import org.eclipse.jetty.client.ProxyProtocolClientConnectionFactory.V2; +import org.eclipse.jetty.client.api.ContentResponse; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class ProxyOriginalClientIPTest extends MockedPulsarServiceBaseTest { + static final String[] ARGS = new String[]{"-c", "./src/test/resources/proxy.conf"}; + HttpClient httpClient; + ProxyServiceStarter serviceStarter; + String webServiceUrl; + String webServiceUrlTls; + + @Override + @BeforeClass + protected void setup() throws Exception { + internalSetup(); + serviceStarter = new ProxyServiceStarter(ARGS, proxyConfig -> { + proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl()); + proxyConfig.setBrokerWebServiceURL(pulsar.getWebServiceAddress()); + proxyConfig.setWebServicePort(Optional.of(0)); + proxyConfig.setWebServicePortTls(Optional.of(0)); + proxyConfig.setTlsEnabledWithBroker(false); + proxyConfig.setTlsCertificateFilePath(PROXY_CERT_FILE_PATH); + proxyConfig.setTlsKeyFilePath(PROXY_KEY_FILE_PATH); + proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setWebSocketServiceEnabled(true); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); + proxyConfig.setClusterName(configClusterName); + proxyConfig.setWebServiceTrustXForwardedFor(true); + proxyConfig.setWebServiceHaProxyProtocolEnabled(true); + }, true); + serviceStarter.start(); + webServiceUrl = "http://localhost:" + serviceStarter.getServer().getListenPortHTTP().get(); + webServiceUrlTls = "https://localhost:" + serviceStarter.getServer().getListenPortHTTPS().get(); + httpClient = new HttpClient(new SslContextFactory(true)); + httpClient.start(); + } + + @Override + @AfterClass(alwaysRun = true) + protected void cleanup() throws Exception { + internalCleanup(); + if (serviceStarter != null) { + serviceStarter.close(); + } + if (httpClient != null) { + httpClient.stop(); + } + } + + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + conf.setWebServiceTrustXForwardedFor(true); + } + + @DataProvider(name = "tlsEnabled") + public Object[][] tlsEnabled() { + return new Object[][] { { true }, { false } }; + } + + @Test(dataProvider = "tlsEnabled") + public void testClientIPIsPickedFromXForwardedForHeaderAndLogged(boolean tlsEnabled) throws Exception { + String url = (tlsEnabled ? webServiceUrlTls : webServiceUrl) + "/admin/v2/brokers/leaderBroker"; + performLoggingTest(consoleCaptor -> { + // Send a GET request to the metrics URL + ContentResponse response = httpClient.newRequest(url) + .header("X-Forwarded-For", "11.22.33.44") + .send(); + + // Validate the response + assertTrue(response.getContentAsString().contains("\"brokerId\":\"" + pulsar.getBrokerId() + "\"")); + + // Validate that the client IP passed in X-Forwarded-For is logged + assertTrue(consoleCaptor.getStandardOutput().stream() + .anyMatch(line -> line.contains("pulsar-external-web-") && line.contains("RequestLog") + && line.contains("R:11.22.33.44")), "Expected to find client IP in proxy logs"); + assertTrue(consoleCaptor.getStandardOutput().stream() + .anyMatch(line -> line.contains("pulsar-web-") && line.contains("RequestLog") + && line.contains("R:11.22.33.44")), "Expected to find client IP in broker logs"); + }); + } + + @Test(dataProvider = "tlsEnabled") + public void testClientIPIsPickedFromHAProxyProtocolAndLogged(boolean tlsEnabled) throws Exception { + String url = (tlsEnabled ? webServiceUrlTls : webServiceUrl) + "/admin/v2/brokers/leaderBroker"; + performLoggingTest(consoleCaptor -> { + // Send a GET request to the metrics URL + ContentResponse response = httpClient.newRequest(url) + // Jetty client will add HA Proxy protocol header with the given IP to the request + .tag(new V2.Tag("99.22.33.44", 1234)) + .send(); + + // Validate the response + assertTrue(response.getContentAsString().contains("\"brokerId\":\"" + pulsar.getBrokerId() + "\"")); + + // Validate that the client IP passed in HA proxy protocol is logged + assertTrue(consoleCaptor.getStandardOutput().stream() + .anyMatch(line -> line.contains("pulsar-external-web-") && line.contains("RequestLog") + && line.contains("R:99.22.33.44")), "Expected to find client IP in proxy logs"); + assertTrue(consoleCaptor.getStandardOutput().stream() + .anyMatch(line -> line.contains("pulsar-web-") && line.contains("RequestLog") + && line.contains("R:99.22.33.44")), "Expected to find client IP in broker logs"); + }); + } + + void performLoggingTest(ThrowingConsumer testFunction) { + ConsoleCaptor consoleCaptor = new ConsoleCaptor(); + try { + Awaitility.await().atMost(Duration.of(2, ChronoUnit.SECONDS)).untilAsserted(() -> { + consoleCaptor.clearOutput(); + testFunction.accept(consoleCaptor); + }); + } finally { + consoleCaptor.close(); + System.out.println("--- Captured console output:"); + consoleCaptor.getStandardOutput().forEach(System.out::println); + consoleCaptor.getErrorOutput().forEach(System.err::println); + System.out.println("--- End of captured console output"); + } + } +} diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyParserTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyParserTest.java index 82cd702aa7f0a..ee0f8010b7d79 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyParserTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyParserTest.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.pulsar.proxy.server; import static com.google.common.base.Preconditions.checkArgument; @@ -23,17 +24,15 @@ import static java.util.Objects.requireNonNull; import static org.mockito.Mockito.doReturn; import static org.testng.Assert.assertEquals; - import io.netty.channel.EventLoopGroup; import io.netty.util.concurrent.DefaultThreadFactory; - import java.util.Optional; -import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; - import lombok.Cleanup; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageRoutingMode; @@ -45,6 +44,7 @@ import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; import org.apache.pulsar.common.api.proto.CommandActiveConsumerChange; import org.apache.pulsar.common.api.proto.ProtocolVersion; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; @@ -64,6 +64,7 @@ public class ProxyParserTest extends MockedPulsarServiceBaseTest { private ProxyService proxyService; private ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; @Override @BeforeClass @@ -74,16 +75,21 @@ protected void setup() throws Exception { proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setMetadataStoreUrl(DUMMY_VALUE); proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); + proxyConfig.setClusterName(configClusterName); //enable full parsing feature proxyConfig.setProxyLogLevel(Optional.ofNullable(2)); + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService( - PulsarConfigurationLoader.convertFrom(proxyConfig)))); - doReturn(new ZKMetadataStore(mockZooKeeper)).when(proxyService).createLocalMetadataStore(); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(proxyService).createConfigurationMetadataStore(); + PulsarConfigurationLoader.convertFrom(proxyConfig)), proxyClientAuthentication)); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeper))).when(proxyService).createLocalMetadataStore(); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(proxyService) + .createConfigurationMetadataStore(); Optional proxyLogLevel = Optional.of(2); - assertEquals( proxyLogLevel, proxyService.getConfiguration().getProxyLogLevel()); + assertEquals(proxyLogLevel, proxyService.getConfiguration().getProxyLogLevel()); proxyService.start(); } @@ -93,6 +99,9 @@ protected void cleanup() throws Exception { internalCleanup(); proxyService.close(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } } @Test @@ -100,8 +109,9 @@ public void testProducer() throws Exception { @Cleanup PulsarClient client = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl()) .build(); - Producer producer = client.newProducer(Schema.BYTES).topic("persistent://sample/test/local/producer-topic") - .create(); + Producer producer = + client.newProducer(Schema.BYTES).topic("persistent://sample/test/local/producer-topic") + .create(); for (int i = 0; i < 10; i++) { producer.send("test".getBytes()); @@ -114,10 +124,10 @@ public void testProducerConsumer() throws Exception { PulsarClient client = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl()) .build(); Producer producer = client.newProducer(Schema.BYTES) - .topic("persistent://sample/test/local/producer-consumer-topic") - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.SinglePartition) - .create(); + .topic("persistent://sample/test/local/producer-consumer-topic") + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); // Create a consumer directly attached to broker Consumer consumer = pulsarClient.newConsumer() @@ -149,9 +159,9 @@ public void testPartitions() throws Exception { admin.topics().createPartitionedTopic("persistent://sample/test/local/partitioned-topic", 2); Producer producer = client.newProducer(Schema.BYTES) - .topic("persistent://sample/test/local/partitioned-topic") - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create(); + .topic("persistent://sample/test/local/partitioned-topic") + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create(); // Create a consumer directly attached to broker Consumer consumer = pulsarClient.newConsumer().topic("persistent://sample/test/local/partitioned-topic") @@ -171,18 +181,18 @@ public void testPartitions() throws Exception { public void testRegexSubscription() throws Exception { @Cleanup PulsarClient client = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl()) - .connectionsPerBroker(5).ioThreads(5).build(); + .connectionsPerBroker(5).ioThreads(5).build(); // create two topics by subscribing to a topic and closing it try (Consumer ignored = client.newConsumer() - .topic("persistent://sample/test/local/topic1") - .subscriptionName("ignored") - .subscribe()) { + .topic("persistent://sample/test/local/topic1") + .subscriptionName("ignored") + .subscribe()) { } try (Consumer ignored = client.newConsumer() - .topic("persistent://sample/test/local/topic2") - .subscriptionName("ignored") - .subscribe()) { + .topic("persistent://sample/test/local/topic2") + .subscriptionName("ignored") + .subscribe()) { } String subName = "regex-sub-proxy-parser-test-" + System.currentTimeMillis(); @@ -190,16 +200,16 @@ public void testRegexSubscription() throws Exception { String regexSubscriptionPattern = "persistent://sample/test/local/topic.*"; log.info("Regex subscribe to topics {}", regexSubscriptionPattern); try (Consumer consumer = client.newConsumer() - .topicsPattern(regexSubscriptionPattern) - .subscriptionName(subName) - .subscribe()) { + .topicsPattern(regexSubscriptionPattern) + .subscriptionName(subName) + .subscribe()) { log.info("Successfully subscribe to topics using regex {}", regexSubscriptionPattern); final int numMessages = 20; try (Producer producer = client.newProducer(Schema.BYTES) - .topic("persistent://sample/test/local/topic1") - .create()) { + .topic("persistent://sample/test/local/topic1") + .create()) { for (int i = 0; i < numMessages; i++) { producer.send(("message-" + i).getBytes(UTF_8)); } @@ -219,8 +229,12 @@ public void testProtocolVersionAdvertisement() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); conf.setServiceUrl(proxyService.getServiceUrl()); + @Cleanup("shutdownNow") + EventLoopGroup eventLoopGroup = EventLoopUtil.newEventLoopGroup(conf.getNumIoThreads(), false, + new DefaultThreadFactory("pulsar-client-io", Thread.currentThread().isDaemon())); @Cleanup - PulsarClient client = getClientActiveConsumerChangeNotSupported(conf); + PulsarClient client = getClientActiveConsumerChangeNotSupported(conf, + eventLoopGroup); Producer producer = client.newProducer().topic(topic).create(); Consumer consumer = client.newConsumer().topic(topic).subscriptionName(sub) @@ -243,19 +257,18 @@ public void testProtocolVersionAdvertisement() throws Exception { ((PulsarClientImpl) client).getCnxPool().close(); } - private static PulsarClient getClientActiveConsumerChangeNotSupported(ClientConfigurationData conf) + private static PulsarClient getClientActiveConsumerChangeNotSupported(ClientConfigurationData conf, + final EventLoopGroup eventLoopGroup) throws Exception { - ThreadFactory threadFactory = new DefaultThreadFactory("pulsar-client-io", Thread.currentThread().isDaemon()); - EventLoopGroup eventLoopGroup = EventLoopUtil.newEventLoopGroup(conf.getNumIoThreads(), false, threadFactory); - ConnectionPool cnxPool = new ConnectionPool(conf, eventLoopGroup, () -> { - return new ClientCnx(conf, eventLoopGroup, ProtocolVersion.v11_VALUE) { + ConnectionPool cnxPool = new ConnectionPool(InstrumentProvider.NOOP, conf, eventLoopGroup, () -> { + return new ClientCnx(InstrumentProvider.NOOP, conf, eventLoopGroup, ProtocolVersion.v11_VALUE) { @Override protected void handleActiveConsumerChange(CommandActiveConsumerChange change) { throw new UnsupportedOperationException(); } }; - }); + }, null); return new PulsarClientImpl(conf, eventLoopGroup, cnxPool); } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyPrometheusMetricsTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyPrometheusMetricsTest.java index 6948996ad4636..4dd7bc981e59b 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyPrometheusMetricsTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyPrometheusMetricsTest.java @@ -42,6 +42,8 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.metadata.impl.ZKMetadataStore; import org.awaitility.Awaitility; @@ -59,6 +61,7 @@ public class ProxyPrometheusMetricsTest extends MockedPulsarServiceBaseTest { private ProxyService proxyService; private WebServer proxyWebServer; private final ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; @Override @BeforeClass @@ -72,10 +75,16 @@ protected void setup() throws Exception { proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); proxyConfig.setClusterName(TEST_CLUSTER); + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); + proxyService = Mockito.spy(new ProxyService(proxyConfig, - new AuthenticationService(PulsarConfigurationLoader.convertFrom(proxyConfig)))); - doReturn(new ZKMetadataStore(mockZooKeeper)).when(proxyService).createLocalMetadataStore(); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(proxyService).createConfigurationMetadataStore(); + new AuthenticationService(PulsarConfigurationLoader.convertFrom(proxyConfig)), + proxyClientAuthentication)); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeper))).when(proxyService).createLocalMetadataStore(); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(proxyService) + .createConfigurationMetadataStore(); proxyService.start(); @@ -85,7 +94,7 @@ protected void setup() throws Exception { PulsarConfigurationLoader.convertFrom(proxyConfig)); proxyWebServer = new WebServer(proxyConfig, authService); - ProxyServiceStarter.addWebServerHandlers(proxyWebServer, proxyConfig, proxyService, null); + ProxyServiceStarter.addWebServerHandlers(proxyWebServer, proxyConfig, proxyService, null, proxyClientAuthentication); proxyWebServer.start(); } @@ -108,6 +117,9 @@ protected void cleanup() throws Exception { if (proxyService != null) { proxyService.close(); } + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } } /** diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRefreshAuthTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRefreshAuthTest.java index bde989fc432f9..bdabfecaa439d 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRefreshAuthTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRefreshAuthTest.java @@ -35,6 +35,8 @@ import org.apache.pulsar.broker.authentication.AuthenticationService; import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.ProducerConsumerBase; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.impl.ClientCnx; @@ -52,10 +54,12 @@ @Slf4j public class ProxyRefreshAuthTest extends ProducerConsumerBase { + private static final String CLUSTER_NAME = "proxy-authorization"; private final SecretKey SECRET_KEY = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); private ProxyService proxyService; private final ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; @Override protected void doInitConf() throws Exception { @@ -69,6 +73,7 @@ protected void doInitConf() throws Exception { conf.setAdvertisedAddress(null); conf.setAuthenticateOriginalAuthData(true); conf.setBrokerServicePort(Optional.of(0)); + conf.setWebServicePortTls(Optional.of(0)); conf.setWebServicePort(Optional.of(0)); Set superUserRoles = new HashSet<>(); @@ -83,7 +88,7 @@ protected void doInitConf() throws Exception { properties.setProperty("tokenAllowedClockSkewSeconds", "2"); conf.setProperties(properties); - conf.setClusterName("proxy-authorization"); + conf.setClusterName(CLUSTER_NAME); conf.setNumExecutorThreadPoolSize(5); conf.setAuthenticationRefreshCheckSeconds(1); @@ -93,7 +98,7 @@ protected void doInitConf() throws Exception { @Override protected void setup() throws Exception { super.init(); - + closeAdmin(); admin = PulsarAdmin.builder().serviceHttpUrl(pulsar.getWebServiceAddress()) .authentication(new AuthenticationToken( () -> AuthTokenUtils.createToken(SECRET_KEY, "client", Optional.empty()))).build(); @@ -115,6 +120,7 @@ protected void setup() throws Exception { proxyConfig.setServicePort(Optional.of(0)); proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setWebServicePort(Optional.of(0)); + proxyConfig.setClusterName(CLUSTER_NAME); proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); proxyConfig.setBrokerClientAuthenticationParameters( @@ -124,9 +130,13 @@ protected void setup() throws Exception { properties.setProperty("tokenSecretKey", AuthTokenUtils.encodeKeyBase64(SECRET_KEY)); proxyConfig.setProperties(properties); + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); + proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService( - PulsarConfigurationLoader.convertFrom(proxyConfig)))); + PulsarConfigurationLoader.convertFrom(proxyConfig)), proxyClientAuthentication)); } @AfterClass(alwaysRun = true) @@ -134,6 +144,9 @@ protected void setup() throws Exception { protected void cleanup() throws Exception { super.internalCleanup(); proxyService.close(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } } private void startProxy(boolean forwardAuthData) throws Exception { diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRolesEnforcementTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRolesEnforcementTest.java index 2c8c382b6a5ef..883b725e15dd2 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRolesEnforcementTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRolesEnforcementTest.java @@ -28,6 +28,7 @@ import java.util.Optional; import java.util.Set; import javax.naming.AuthenticationException; +import lombok.Cleanup; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.authentication.AuthenticationProvider; @@ -35,6 +36,7 @@ import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.AuthenticationDataProvider; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.ProducerConsumerBase; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; @@ -49,6 +51,7 @@ public class ProxyRolesEnforcementTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(ProxyRolesEnforcementTest.class); + private static final String CLUSTER_NAME = "test"; public static class BasicAuthenticationData implements AuthenticationDataProvider { private final String authParam; @@ -144,7 +147,7 @@ protected void setup() throws Exception { conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(true); conf.setBrokerClientAuthenticationPlugin(BasicAuthentication.class.getName()); - conf.setBrokerClientAuthenticationParameters("authParam:broker"); + conf.setBrokerClientAuthenticationParameters("authParam:admin"); Set superUserRoles = new HashSet<>(); superUserRoles.add("admin"); @@ -154,7 +157,7 @@ protected void setup() throws Exception { providers.add(BasicAuthenticationProvider.class.getName()); conf.setAuthenticationProviders(providers); - conf.setClusterName("test"); + conf.setClusterName(CLUSTER_NAME); Set proxyRoles = new HashSet<>(); proxyRoles.add("proxy"); conf.setProxyRoles(proxyRoles); @@ -209,6 +212,7 @@ public void testIncorrectRoles() throws Exception { proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl()); + proxyConfig.setClusterName(CLUSTER_NAME); proxyConfig.setBrokerClientAuthenticationPlugin(BasicAuthentication.class.getName()); proxyConfig.setBrokerClientAuthenticationParameters(proxyAuthParams); @@ -217,9 +221,14 @@ public void testIncorrectRoles() throws Exception { providers.add(BasicAuthenticationProvider.class.getName()); proxyConfig.setAuthenticationProviders(providers); + @Cleanup + final Authentication proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); + try (ProxyService proxyService = new ProxyService(proxyConfig, new AuthenticationService( - PulsarConfigurationLoader.convertFrom(proxyConfig)))) { + PulsarConfigurationLoader.convertFrom(proxyConfig)), proxyClientAuthentication)) { proxyService.start(); @@ -243,6 +252,7 @@ public void testIncorrectRoles() throws Exception { private void createAdminClient() throws PulsarClientException { String adminAuthParams = "authParam:admin"; + closeAdmin(); admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrl.toString()) .authentication(BasicAuthentication.class.getName(), adminAuthParams).build()); } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterDisableZeroCopyTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterDisableZeroCopyTest.java index 0c9fa5c7ac322..937526629acf0 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterDisableZeroCopyTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterDisableZeroCopyTest.java @@ -21,13 +21,13 @@ import java.util.Optional; import org.testng.annotations.BeforeClass; -public class ProxyServiceStarterDisableZeroCopyTest extends ProxyServiceStarterTest{ +public class ProxyServiceStarterDisableZeroCopyTest extends ProxyServiceStarterTest { @Override @BeforeClass protected void setup() throws Exception { internalSetup(); - serviceStarter = new ProxyServiceStarter(ARGS); + serviceStarter = new ProxyServiceStarter(ARGS, null, true); serviceStarter.getConfig().setBrokerServiceURL(pulsar.getBrokerServiceUrl()); serviceStarter.getConfig().setBrokerWebServiceURL(pulsar.getWebServiceAddress()); serviceStarter.getConfig().setWebServicePort(Optional.of(0)); @@ -35,6 +35,7 @@ protected void setup() throws Exception { serviceStarter.getConfig().setWebSocketServiceEnabled(true); serviceStarter.getConfig().setBrokerProxyAllowedTargetPorts("*"); serviceStarter.getConfig().setProxyZeroCopyModeEnabled(false); + serviceStarter.getConfig().setClusterName(configClusterName); serviceStarter.start(); serviceUrl = serviceStarter.getProxyService().getServiceUrl(); } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java index 4dcfc17096448..d96d2cd1f6e9c 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java @@ -20,16 +20,22 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import java.io.IOException; import java.net.URI; import java.nio.ByteBuffer; import java.util.Base64; +import java.util.Map; import java.util.Optional; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.Future; +import java.util.function.Consumer; import lombok.Cleanup; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.websocket.data.ProducerMessage; import org.eclipse.jetty.client.HttpClient; @@ -45,7 +51,7 @@ public class ProxyServiceStarterTest extends MockedPulsarServiceBaseTest { - static final String[] ARGS = new String[]{"-c", "./src/test/resources/proxy.conf"}; + public static final String[] ARGS = new String[]{"-c", "./src/test/resources/proxy.conf"}; protected ProxyServiceStarter serviceStarter; protected String serviceUrl; @@ -54,13 +60,14 @@ public class ProxyServiceStarterTest extends MockedPulsarServiceBaseTest { @BeforeClass protected void setup() throws Exception { internalSetup(); - serviceStarter = new ProxyServiceStarter(ARGS); + serviceStarter = new ProxyServiceStarter(ARGS, null, true); serviceStarter.getConfig().setBrokerServiceURL(pulsar.getBrokerServiceUrl()); serviceStarter.getConfig().setBrokerWebServiceURL(pulsar.getWebServiceAddress()); serviceStarter.getConfig().setWebServicePort(Optional.of(0)); serviceStarter.getConfig().setServicePort(Optional.of(0)); serviceStarter.getConfig().setWebSocketServiceEnabled(true); serviceStarter.getConfig().setBrokerProxyAllowedTargetPorts("*"); + serviceStarter.getConfig().setClusterName(configClusterName); serviceStarter.start(); serviceUrl = serviceStarter.getProxyService().getServiceUrl(); } @@ -76,18 +83,6 @@ private String computeWsBasePath() { return String.format("ws://localhost:%d/ws", serviceStarter.getServer().getListenPortHTTP().get()); } - @Test - public void testEnableWebSocketServer() throws Exception { - HttpClient httpClient = new HttpClient(); - WebSocketClient webSocketClient = new WebSocketClient(httpClient); - webSocketClient.start(); - MyWebSocket myWebSocket = new MyWebSocket(); - String webSocketUri = computeWsBasePath() + "/pingpong"; - Future sessionFuture = webSocketClient.connect(myWebSocket, URI.create(webSocketUri)); - System.out.println("uri" + webSocketUri); - sessionFuture.get().getRemote().sendPing(ByteBuffer.wrap("ping".getBytes())); - assertTrue(myWebSocket.getResponse().contains("ping")); - } @Test public void testProducer() throws Exception { @@ -107,7 +102,9 @@ public void testProducer() throws Exception { @Test public void testProduceAndConsumeMessageWithWebsocket() throws Exception { + @Cleanup("stop") HttpClient producerClient = new HttpClient(); + @Cleanup("stop") WebSocketClient producerWebSocketClient = new WebSocketClient(producerClient); producerWebSocketClient.start(); MyWebSocket producerSocket = new MyWebSocket(); @@ -118,7 +115,9 @@ public void testProduceAndConsumeMessageWithWebsocket() throws Exception { produceRequest.setContext("context"); produceRequest.setPayload(Base64.getEncoder().encodeToString("my payload".getBytes())); + @Cleanup("stop") HttpClient consumerClient = new HttpClient(); + @Cleanup("stop") WebSocketClient consumerWebSocketClient = new WebSocketClient(consumerClient); consumerWebSocketClient.start(); MyWebSocket consumerSocket = new MyWebSocket(); @@ -167,4 +166,89 @@ public String getResponse() throws InterruptedException { } } + @Test + public void testProxyClientAuthentication() throws Exception { + final Consumer initConfig = (proxyConfig) -> { + proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl()); + proxyConfig.setBrokerWebServiceURL(pulsar.getWebServiceAddress()); + proxyConfig.setWebServicePort(Optional.of(0)); + proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setWebSocketServiceEnabled(true); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); + proxyConfig.setClusterName(configClusterName); + }; + + + + ProxyServiceStarter serviceStarter = new ProxyServiceStarter(ARGS, null, true); + initConfig.accept(serviceStarter.getConfig()); + // ProxyServiceStarter will throw an exception when Authentication#start is failed + serviceStarter.getConfig().setBrokerClientAuthenticationPlugin(ExceptionAuthentication1.class.getName()); + try { + serviceStarter.start(); + fail("ProxyServiceStarter should throw an exception when Authentication#start is failed"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("ExceptionAuthentication1#start")); + assertTrue(serviceStarter.getProxyClientAuthentication() instanceof ExceptionAuthentication1); + } + + serviceStarter = new ProxyServiceStarter(ARGS, null, true); + initConfig.accept(serviceStarter.getConfig()); + // ProxyServiceStarter will throw an exception when Authentication#start and Authentication#close are failed + serviceStarter.getConfig().setBrokerClientAuthenticationPlugin(ExceptionAuthentication2.class.getName()); + try { + serviceStarter.start(); + fail("ProxyServiceStarter should throw an exception when Authentication#start and Authentication#close are failed"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("ExceptionAuthentication2#start")); + assertTrue(serviceStarter.getProxyClientAuthentication() instanceof ExceptionAuthentication2); + } + } + + public static class ExceptionAuthentication1 implements Authentication { + + @Override + public String getAuthMethodName() { + return "org.apache.pulsar.proxy.server.ProxyConfigurationTest.ExceptionAuthentication1"; + } + + @Override + public void configure(Map authParams) { + // no-op + } + + @Override + public void start() throws PulsarClientException { + throw new PulsarClientException("ExceptionAuthentication1#start"); + } + + @Override + public void close() throws IOException { + // no-op + } + } + + public static class ExceptionAuthentication2 implements Authentication { + + @Override + public String getAuthMethodName() { + return "org.apache.pulsar.proxy.server.ProxyConfigurationTest.ExceptionAuthentication2"; + } + + @Override + public void configure(Map authParams) { + // no-op + } + + @Override + public void start() throws PulsarClientException { + throw new PulsarClientException("ExceptionAuthentication2#start"); + } + + @Override + public void close() throws IOException { + throw new IOException("ExceptionAuthentication2#close"); + } + } + } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceTlsStarterTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceTlsStarterTest.java index 01c06fbf52f4e..1148234be624c 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceTlsStarterTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceTlsStarterTest.java @@ -55,11 +55,13 @@ public class ProxyServiceTlsStarterTest extends MockedPulsarServiceBaseTest { @BeforeClass protected void setup() throws Exception { internalSetup(); - serviceStarter = new ProxyServiceStarter(ARGS); + serviceStarter = new ProxyServiceStarter(ARGS, null, true); serviceStarter.getConfig().setBrokerServiceURL(pulsar.getBrokerServiceUrl()); serviceStarter.getConfig().setBrokerServiceURLTLS(pulsar.getBrokerServiceUrlTls()); serviceStarter.getConfig().setBrokerWebServiceURL(pulsar.getWebServiceAddress()); serviceStarter.getConfig().setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH); + serviceStarter.getConfig().setBrokerClientCertificateFilePath(BROKER_CERT_FILE_PATH); + serviceStarter.getConfig().setBrokerClientKeyFilePath(BROKER_KEY_FILE_PATH); serviceStarter.getConfig().setServicePort(Optional.empty()); serviceStarter.getConfig().setServicePortTls(Optional.of(0)); serviceStarter.getConfig().setWebServicePort(Optional.of(0)); @@ -68,6 +70,7 @@ protected void setup() throws Exception { serviceStarter.getConfig().setTlsCertificateFilePath(PROXY_CERT_FILE_PATH); serviceStarter.getConfig().setTlsKeyFilePath(PROXY_KEY_FILE_PATH); serviceStarter.getConfig().setBrokerProxyAllowedTargetPorts("*"); + serviceStarter.getConfig().setClusterName(configClusterName); serviceStarter.start(); serviceUrl = serviceStarter.getProxyService().getServiceUrlTls(); webPort = serviceStarter.getServer().getListenPortHTTP().get(); @@ -75,6 +78,7 @@ protected void setup() throws Exception { protected void doInitConf() throws Exception { super.doInitConf(); + this.conf.setBrokerServicePortTls(Optional.of(0)); this.conf.setTlsCertificateFilePath(PROXY_CERT_FILE_PATH); this.conf.setTlsKeyFilePath(PROXY_KEY_FILE_PATH); } @@ -105,7 +109,9 @@ public void testProducer() throws Exception { @Test public void testProduceAndConsumeMessageWithWebsocket() throws Exception { + @Cleanup("stop") HttpClient producerClient = new HttpClient(); + @Cleanup("stop") WebSocketClient producerWebSocketClient = new WebSocketClient(producerClient); producerWebSocketClient.start(); MyWebSocket producerSocket = new MyWebSocket(); @@ -116,7 +122,9 @@ public void testProduceAndConsumeMessageWithWebsocket() throws Exception { produceRequest.setContext("context"); produceRequest.setPayload(Base64.getEncoder().encodeToString("my payload".getBytes())); + @Cleanup("stop") HttpClient consumerClient = new HttpClient(); + @Cleanup("stop") WebSocketClient consumerWebSocketClient = new WebSocketClient(consumerClient); consumerWebSocketClient.start(); MyWebSocket consumerSocket = new MyWebSocket(); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStatsTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStatsTest.java index 140af88aae71b..86d572702f3b1 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStatsTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStatsTest.java @@ -38,6 +38,8 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageRoutingMode; @@ -61,6 +63,7 @@ public class ProxyStatsTest extends MockedPulsarServiceBaseTest { private ProxyService proxyService; private WebServer proxyWebServer; private final ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; @Override @BeforeClass @@ -72,13 +75,19 @@ protected void setup() throws Exception { proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setMetadataStoreUrl(DUMMY_VALUE); proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); + proxyConfig.setClusterName(configClusterName); // enable full parsing feature proxyConfig.setProxyLogLevel(Optional.of(2)); + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); + proxyService = Mockito.spy(new ProxyService(proxyConfig, - new AuthenticationService(PulsarConfigurationLoader.convertFrom(proxyConfig)))); - doReturn(new ZKMetadataStore(mockZooKeeper)).when(proxyService).createLocalMetadataStore(); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(proxyService).createConfigurationMetadataStore(); + new AuthenticationService(PulsarConfigurationLoader.convertFrom(proxyConfig)), proxyClientAuthentication)); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeper))).when(proxyService).createLocalMetadataStore(); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(proxyService) + .createConfigurationMetadataStore(); Optional proxyLogLevel = Optional.of(2); assertEquals(proxyLogLevel, proxyService.getConfiguration().getProxyLogLevel()); @@ -88,7 +97,7 @@ protected void setup() throws Exception { PulsarConfigurationLoader.convertFrom(proxyConfig)); proxyWebServer = new WebServer(proxyConfig, authService); - ProxyServiceStarter.addWebServerHandlers(proxyWebServer, proxyConfig, proxyService, null); + ProxyServiceStarter.addWebServerHandlers(proxyWebServer, proxyConfig, proxyService, null, proxyClientAuthentication); proxyWebServer.start(); } @@ -106,6 +115,10 @@ protected ServiceConfiguration getDefaultConf() { protected void cleanup() throws Exception { internalCleanup(); proxyService.close(); + proxyWebServer.stop(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } } /** diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStuckConnectionTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStuckConnectionTest.java index 97279659af626..30c6e45654ba0 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStuckConnectionTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyStuckConnectionTest.java @@ -28,6 +28,8 @@ import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.KeySharedPolicy; import org.apache.pulsar.client.api.Message; @@ -56,6 +58,7 @@ public class ProxyStuckConnectionTest extends MockedPulsarServiceBaseTest { private ProxyService proxyService; private ProxyConfiguration proxyConfig; + private Authentication proxyClientAuthentication; private SocatContainer socatContainer; private String brokerServiceUriSocat; @@ -79,6 +82,11 @@ protected void setup() throws Exception { proxyConfig.setServicePort(Optional.ofNullable(0)); proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl()); + proxyConfig.setClusterName(configClusterName); + + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); startProxyService(); // use the same port for subsequent restarts @@ -87,14 +95,15 @@ protected void setup() throws Exception { private void startProxyService() throws Exception { proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService( - PulsarConfigurationLoader.convertFrom(proxyConfig))) { + PulsarConfigurationLoader.convertFrom(proxyConfig)), proxyClientAuthentication) { @Override protected LookupProxyHandler newLookupProxyHandler(ProxyConnection proxyConnection) { return new TestLookupProxyHandler(this, proxyConnection); } }); - doReturn(new ZKMetadataStore(mockZooKeeper)).when(proxyService).createLocalMetadataStore(); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(proxyService).createConfigurationMetadataStore(); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeper))).when(proxyService).createLocalMetadataStore(); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(proxyService) + .createConfigurationMetadataStore(); proxyService.start(); } @@ -105,6 +114,9 @@ protected void cleanup() throws Exception { if (proxyService != null) { proxyService.close(); } + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } if (socatContainer != null) { socatContainer.close(); } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTest.java index 237cf5a48112c..4c0cd49d74fa7 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTest.java @@ -38,6 +38,8 @@ import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageRoutingMode; @@ -50,6 +52,7 @@ import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; import org.apache.pulsar.common.api.proto.CommandActiveConsumerChange; import org.apache.pulsar.common.api.proto.ProtocolVersion; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; @@ -73,6 +76,7 @@ public class ProxyTest extends MockedPulsarServiceBaseTest { protected ProxyService proxyService; protected ProxyConfiguration proxyConfig = new ProxyConfiguration(); + protected Authentication proxyClientAuthentication; @Data @ToString @@ -90,17 +94,27 @@ public static class Foo { protected void setup() throws Exception { internalSetup(); + initializeProxyConfig(); + + proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService( + PulsarConfigurationLoader.convertFrom(proxyConfig)), proxyClientAuthentication)); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeper))).when(proxyService).createLocalMetadataStore(); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(proxyService) + .createConfigurationMetadataStore(); + + proxyService.start(); + } + + protected void initializeProxyConfig() throws Exception { proxyConfig.setServicePort(Optional.ofNullable(0)); proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setMetadataStoreUrl(DUMMY_VALUE); proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); + proxyConfig.setClusterName(configClusterName); - proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService( - PulsarConfigurationLoader.convertFrom(proxyConfig)))); - doReturn(new ZKMetadataStore(mockZooKeeper)).when(proxyService).createLocalMetadataStore(); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(proxyService).createConfigurationMetadataStore(); - - proxyService.start(); + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); } @Override @@ -109,6 +123,9 @@ protected void cleanup() throws Exception { internalCleanup(); proxyService.close(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } } @Test @@ -259,6 +276,34 @@ public void testRegexSubscription() throws Exception { } } + @Test(timeOut = 60_000) + public void testRegexSubscriptionWithTopicDiscovery() throws Exception { + @Cleanup + PulsarClient client = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl()).build(); + String subName = "regex-proxy-test-" + System.currentTimeMillis(); + String regexSubscriptionPattern = "persistent://sample/test/local/regex-topic-.*"; + try (Consumer consumer = client.newConsumer() + .topicsPattern(regexSubscriptionPattern) + .subscriptionName(subName) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .patternAutoDiscoveryPeriod(10, TimeUnit.MINUTES) + .subscribe()) { + final int topics = 10; + final String topicPrefix = "persistent://sample/test/local/regex-topic-"; + for (int i = 0; i < topics; i++) { + Producer producer = client.newProducer(Schema.BYTES) + .topic(topicPrefix + i) + .create(); + producer.send(("" + i).getBytes(UTF_8)); + producer.close(); + } + for (int i = 0; i < topics; i++) { + Message msg = consumer.receive(); + assertEquals(topicPrefix + new String(msg.getValue(), UTF_8), msg.getTopicName()); + } + } + } + @Test public void testGetSchema() throws Exception { @Cleanup @@ -334,19 +379,21 @@ public void testGetClientVersion() throws Exception { .get(0).getClientVersion(), String.format("Pulsar-Java-v%s", PulsarVersion.getVersion())); } - private static PulsarClient getClientActiveConsumerChangeNotSupported(ClientConfigurationData conf) + private PulsarClient getClientActiveConsumerChangeNotSupported(ClientConfigurationData conf) throws Exception { ThreadFactory threadFactory = new DefaultThreadFactory("pulsar-client-io", Thread.currentThread().isDaemon()); EventLoopGroup eventLoopGroup = EventLoopUtil.newEventLoopGroup(conf.getNumIoThreads(), false, threadFactory); + registerCloseable(() -> eventLoopGroup.shutdownNow()); - ConnectionPool cnxPool = new ConnectionPool(conf, eventLoopGroup, () -> { - return new ClientCnx(conf, eventLoopGroup, ProtocolVersion.v11_VALUE) { + ConnectionPool cnxPool = new ConnectionPool(InstrumentProvider.NOOP, conf, eventLoopGroup, () -> { + return new ClientCnx(InstrumentProvider.NOOP, conf, eventLoopGroup, ProtocolVersion.v11_VALUE) { @Override protected void handleActiveConsumerChange(CommandActiveConsumerChange change) { throw new UnsupportedOperationException(); } }; - }); + }, null); + registerCloseable(cnxPool); return new PulsarClientImpl(conf, eventLoopGroup, cnxPool); } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTest.java index 64b0cd6b1a610..0f0dc30b62096 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTest.java @@ -27,6 +27,8 @@ import lombok.Cleanup; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageRoutingMode; @@ -45,6 +47,7 @@ public class ProxyTlsTest extends MockedPulsarServiceBaseTest { private ProxyService proxyService; private ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; @Override @BeforeClass @@ -61,11 +64,17 @@ protected void setup() throws Exception { proxyConfig.setTlsKeyFilePath(PROXY_KEY_FILE_PATH); proxyConfig.setMetadataStoreUrl(DUMMY_VALUE); proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); + proxyConfig.setClusterName(configClusterName); + + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService( - PulsarConfigurationLoader.convertFrom(proxyConfig)))); - doReturn(new ZKMetadataStore(mockZooKeeper)).when(proxyService).createLocalMetadataStore(); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(proxyService).createConfigurationMetadataStore(); + PulsarConfigurationLoader.convertFrom(proxyConfig)), proxyClientAuthentication)); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeper))).when(proxyService).createLocalMetadataStore(); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(proxyService) + .createConfigurationMetadataStore(); proxyService.start(); } @@ -76,6 +85,9 @@ protected void cleanup() throws Exception { internalCleanup(); proxyService.close(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } } @Test diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsWithAuthTest.java similarity index 68% rename from pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java rename to pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsWithAuthTest.java index f77c0eeb2d41c..42b5ae178d3b0 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsWithAuthTest.java @@ -24,8 +24,11 @@ import java.io.FileWriter; import java.util.Optional; +import org.apache.pulsar.broker.auth.MockOIDCIdentityProvider; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.metadata.impl.ZKMetadataStore; import org.mockito.Mockito; @@ -33,22 +36,27 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -public class ProxyTlsTestWithAuth extends MockedPulsarServiceBaseTest { +public class ProxyTlsWithAuthTest extends MockedPulsarServiceBaseTest { private ProxyService proxyService; private ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; + + private MockOIDCIdentityProvider server; @Override @BeforeClass protected void setup() throws Exception { internalSetup(); + String clientSecret = "super-secret-client-secret"; + server = new MockOIDCIdentityProvider(clientSecret, "an-audience", 3000); File tempFile = File.createTempFile("oauth2", ".tmp"); tempFile.deleteOnExit(); FileWriter writer = new FileWriter(tempFile); writer.write("{\n" + - " \"client_id\":\"Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x\",\n" + - " \"client_secret\":\"rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N_poGAb\"\n" + + " \"client_id\":\"my-user\",\n" + + " \"client_secret\":\"" + clientSecret + "\"\n" + "}"); writer.flush(); writer.close(); @@ -65,14 +73,20 @@ protected void setup() throws Exception { proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); proxyConfig.setBrokerClientAuthenticationPlugin("org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2"); proxyConfig.setBrokerClientAuthenticationParameters("{\"grant_type\":\"client_credentials\"," + - " \"issuerUrl\":\"https://dev-kt-aa9ne.us.auth0.com\"," + - " \"audience\": \"https://dev-kt-aa9ne.us.auth0.com/api/v2/\"," + + " \"issuerUrl\":\"" + server.getIssuer() + "\"," + + " \"audience\": \"an-audience\"," + " \"privateKey\":\"file://" + tempFile.getAbsolutePath() + "\"}"); + proxyConfig.setClusterName(configClusterName); + + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService( - PulsarConfigurationLoader.convertFrom(proxyConfig)))); - doReturn(new ZKMetadataStore(mockZooKeeper)).when(proxyService).createLocalMetadataStore(); - doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(proxyService).createConfigurationMetadataStore(); + PulsarConfigurationLoader.convertFrom(proxyConfig)), proxyClientAuthentication)); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeper))).when(proxyService).createLocalMetadataStore(); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(proxyService) + .createConfigurationMetadataStore(); proxyService.start(); } @@ -81,8 +95,11 @@ protected void setup() throws Exception { @AfterClass(alwaysRun = true) protected void cleanup() throws Exception { internalCleanup(); - proxyService.close(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } + server.stop(); } @Test diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java index e8bb128c8c190..92a54aa12fda2 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java @@ -34,6 +34,7 @@ import org.apache.pulsar.broker.authentication.AuthenticationService; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Producer; @@ -57,6 +58,7 @@ public class ProxyWithAuthorizationNegTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(ProxyWithAuthorizationNegTest.class); + private static final String CLUSTER_NAME = "proxy-authorization-neg"; private final String TLS_PROXY_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cacert.pem"; private final String TLS_PROXY_CERT_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cert.pem"; @@ -72,12 +74,15 @@ public class ProxyWithAuthorizationNegTest extends ProducerConsumerBase { private ProxyService proxyService; private ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; @BeforeMethod @Override protected void setup() throws Exception { // enable tls and auth&auth at broker + conf.setTopicLevelPoliciesEnabled(false); + conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(true); @@ -102,7 +107,7 @@ protected void setup() throws Exception { providers.add(AuthenticationProviderTls.class.getName()); conf.setAuthenticationProviders(providers); - conf.setClusterName("proxy-authorization-neg"); + conf.setClusterName(CLUSTER_NAME); conf.setNumExecutorThreadPoolSize(5); super.init(); @@ -119,6 +124,7 @@ protected void setup() throws Exception { proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setWebServicePortTls(Optional.of(0)); proxyConfig.setTlsEnabledWithBroker(true); + proxyConfig.setClusterName(CLUSTER_NAME); // enable tls and auth&auth at proxy proxyConfig.setTlsCertificateFilePath(TLS_PROXY_CERT_FILE_PATH); @@ -134,7 +140,10 @@ protected void setup() throws Exception { AuthenticationService authenticationService = new AuthenticationService( PulsarConfigurationLoader.convertFrom(proxyConfig)); - proxyService = Mockito.spy(new ProxyService(proxyConfig, authenticationService)); + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); + proxyService = Mockito.spy(new ProxyService(proxyConfig, authenticationService, proxyClientAuthentication)); proxyService.start(); } @@ -144,6 +153,9 @@ protected void setup() throws Exception { protected void cleanup() throws Exception { super.internalCleanup(); proxyService.close(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } } /** @@ -233,7 +245,7 @@ protected final void createAdminClient() throws Exception { Map authParams = Maps.newHashMap(); authParams.put("tlsCertFile", TLS_SUPERUSER_CLIENT_CERT_FILE_PATH); authParams.put("tlsKeyFile", TLS_SUPERUSER_CLIENT_KEY_FILE_PATH); - + closeAdmin(); admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString()) .tlsTrustCertsFilePath(TLS_BROKER_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(true) .authentication(AuthenticationTls.class.getName(), authParams).build()); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java index 31757cc036720..51f42ea077165 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java @@ -38,6 +38,7 @@ import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -64,6 +65,7 @@ public class ProxyWithAuthorizationTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(ProxyWithAuthorizationTest.class); + private static final String CLUSTER_NAME = "proxy-authorization"; private final SecretKey SECRET_KEY = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); private final String CLIENT_TOKEN = AuthTokenUtils.createToken(SECRET_KEY, "Client", Optional.empty()); @@ -86,6 +88,7 @@ public class ProxyWithAuthorizationTest extends ProducerConsumerBase { private ProxyService proxyService; private WebServer webServer; private final ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; @DataProvider(name = "hostnameVerification") public Object[][] hostnameVerificationCodecProvider() { @@ -189,7 +192,7 @@ protected void doInitConf() throws Exception { properties.setProperty("tokenSecretKey", AuthTokenUtils.encodeKeyBase64(SECRET_KEY)); conf.setProperties(properties); - conf.setClusterName("proxy-authorization"); + conf.setClusterName(CLUSTER_NAME); conf.setNumExecutorThreadPoolSize(5); } @@ -206,6 +209,7 @@ protected void setup() throws Exception { proxyConfig.setBrokerServiceURLTLS(pulsar.getBrokerServiceUrlTls()); proxyConfig.setBrokerWebServiceURLTLS(pulsar.getWebServiceAddressTls()); proxyConfig.setAdvertisedAddress(null); + proxyConfig.setClusterName(CLUSTER_NAME); proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setServicePortTls(Optional.of(0)); @@ -228,7 +232,11 @@ protected void setup() throws Exception { AuthenticationService authService = new AuthenticationService(PulsarConfigurationLoader.convertFrom(proxyConfig)); - proxyService = Mockito.spy(new ProxyService(proxyConfig, authService)); + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); + proxyService = Mockito.spy(new ProxyService(proxyConfig, authService, proxyClientAuthentication)); + proxyService.setGracefulShutdown(false); webServer = new WebServer(proxyConfig, authService); } @@ -238,11 +246,14 @@ protected void cleanup() throws Exception { super.internalCleanup(); proxyService.close(); webServer.stop(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } } private void startProxy() throws Exception { proxyService.start(); - ProxyServiceStarter.addWebServerHandlers(webServer, proxyConfig, proxyService, null); + ProxyServiceStarter.addWebServerHandlers(webServer, proxyConfig, proxyService, null, proxyClientAuthentication); webServer.start(); } @@ -431,6 +442,7 @@ public void tlsCiphersAndProtocols(Set tlsCiphers, Set tlsProtoc proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl()); proxyConfig.setBrokerServiceURLTLS(pulsar.getBrokerServiceUrlTls()); proxyConfig.setAdvertisedAddress(null); + proxyConfig.setClusterName(CLUSTER_NAME); proxyConfig.setServicePort(Optional.of(0)); proxyConfig.setBrokerProxyAllowedTargetPorts("*"); @@ -455,9 +467,16 @@ public void tlsCiphersAndProtocols(Set tlsCiphers, Set tlsProtoc proxyConfig.setTlsProtocols(tlsProtocols); proxyConfig.setTlsCiphers(tlsCiphers); + @Cleanup + final Authentication proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); + + @Cleanup ProxyService proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService( - PulsarConfigurationLoader.convertFrom(proxyConfig)))); + PulsarConfigurationLoader.convertFrom(proxyConfig)), proxyClientAuthentication)); + proxyService.setGracefulShutdown(false); try { proxyService.start(); } catch (Exception ex) { @@ -575,7 +594,7 @@ private void createProxyAdminClient(boolean enableTlsHostnameVerification) throw Map authParams = Maps.newHashMap(); authParams.put("tlsCertFile", TLS_SUPERUSER_CLIENT_CERT_FILE_PATH); authParams.put("tlsKeyFile", TLS_SUPERUSER_CLIENT_KEY_FILE_PATH); - + closeAdmin(); admin = spy(PulsarAdmin.builder().serviceHttpUrl("https://localhost:" + webServer.getListenPortHTTPS().get()) .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) .enableTlsHostnameVerification(enableTlsHostnameVerification) @@ -586,7 +605,7 @@ private void createBrokerAdminClient() throws Exception { Map authParams = Maps.newHashMap(); authParams.put("tlsCertFile", TLS_SUPERUSER_CLIENT_CERT_FILE_PATH); authParams.put("tlsKeyFile", TLS_SUPERUSER_CLIENT_KEY_FILE_PATH); - + closeAdmin(); admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString()) .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) .authentication(AuthenticationTls.class.getName(), authParams).build()); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithExtensibleLoadManagerTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithExtensibleLoadManagerTest.java new file mode 100644 index 0000000000000..3567c8264f1a3 --- /dev/null +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithExtensibleLoadManagerTest.java @@ -0,0 +1,349 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.proxy.server; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import java.net.InetSocketAddress; +import java.util.Collections; +import java.util.HashSet; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.MultiBrokerBaseTest; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; +import org.apache.pulsar.client.impl.ConsumerImpl; +import org.apache.pulsar.client.impl.LookupService; +import org.apache.pulsar.client.impl.ProducerImpl; +import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.client.impl.ServiceNameResolver; +import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicDomain; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.net.ServiceURI; +import org.apache.pulsar.metadata.impl.ZKMetadataStore; +import org.jetbrains.annotations.NotNull; +import org.mockito.Mockito; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Slf4j +public class ProxyWithExtensibleLoadManagerTest extends MultiBrokerBaseTest { + + private static final int TEST_TIMEOUT_MS = 30_000; + + private Authentication proxyClientAuthentication; + private ProxyService proxyService; + + @Override + public int numberOfAdditionalBrokers() { + return 1; + } + + @Override + public void doInitConf() throws Exception { + super.doInitConf(); + configureExtensibleLoadManager(conf); + } + + @Override + protected ServiceConfiguration createConfForAdditionalBroker(int additionalBrokerIndex) { + return configureExtensibleLoadManager(getDefaultConf()); + } + + private ServiceConfiguration configureExtensibleLoadManager(ServiceConfiguration config) { + config.setNumIOThreads(8); + config.setLoadBalancerInFlightServiceUnitStateWaitingTimeInMillis(5 * 1000); + config.setLoadBalancerServiceUnitStateMonitorIntervalInSeconds(1); + config.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); + config.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); + config.setLoadBalancerSheddingEnabled(false); + return config; + } + + private ProxyConfiguration initializeProxyConfig() { + var proxyConfig = new ProxyConfiguration(); + proxyConfig.setNumIOThreads(8); + proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); + proxyConfig.setMetadataStoreUrl(DUMMY_VALUE); + proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); + proxyConfig.setClusterName(configClusterName); + return proxyConfig; + } + + private T spyField(Object target, String fieldName) throws IllegalAccessException { + T t = (T) FieldUtils.readDeclaredField(target, fieldName, true); + var fieldSpy = spy(t); + FieldUtils.writeDeclaredField(target, fieldName, fieldSpy, true); + return fieldSpy; + } + + private PulsarClientImpl createClient(ProxyService proxyService) { + try { + return Mockito.spy((PulsarClientImpl) PulsarClient.builder(). + serviceUrl(proxyService.getServiceUrl()). + build()); + } catch (PulsarClientException e) { + throw new CompletionException(e); + } + } + + @NotNull + private InetSocketAddress getSourceBrokerInetAddress(TopicName topicName) throws PulsarAdminException { + var srcBrokerUrl = admin.lookups().lookupTopic(topicName.toString()); + var serviceUri = ServiceURI.create(srcBrokerUrl); + var uri = serviceUri.getUri(); + return InetSocketAddress.createUnresolved(uri.getHost(), uri.getPort()); + } + + private String getDstBrokerLookupUrl(TopicName topicName) throws Exception { + var srcBrokerUrl = admin.lookups().lookupTopic(topicName.toString()); + return getAllBrokers().stream(). + filter(pulsarService -> !Objects.equals(srcBrokerUrl, pulsarService.getBrokerServiceUrl())). + map(PulsarService::getBrokerId). + findAny().orElseThrow(() -> new Exception("Could not determine destination broker lookup URL")); + } + + @BeforeMethod(alwaysRun = true) + public void proxySetup() throws Exception { + var proxyConfig = initializeProxyConfig(); + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); + proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService( + PulsarConfigurationLoader.convertFrom(proxyConfig)), proxyClientAuthentication)); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeper))).when(proxyService).createLocalMetadataStore(); + doReturn(registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))).when(proxyService) + .createConfigurationMetadataStore(); + proxyService.start(); + } + + @AfterMethod(alwaysRun = true) + public void proxyCleanup() throws Exception { + if (proxyService != null) { + proxyService.close(); + } + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } + } + + @Test(timeOut = TEST_TIMEOUT_MS) + public void testProxyProduceConsume() throws Exception { + var namespaceName = NamespaceName.get("public", "default"); + var topicName = TopicName.get(TopicDomain.persistent.toString(), namespaceName, + BrokerTestUtil.newUniqueName("testProxyProduceConsume")); + + @Cleanup("shutdownNow") + var threadPool = Executors.newCachedThreadPool(); + + var producerClientFuture = CompletableFuture.supplyAsync(() -> createClient(proxyService), threadPool); + var consumerClientFuture = CompletableFuture.supplyAsync(() -> createClient(proxyService), threadPool); + + @Cleanup + var producerClient = producerClientFuture.get(); + @Cleanup + var producer = producerClient.newProducer(Schema.INT32).topic(topicName.toString()).create(); + LookupService producerLookupServiceSpy = spyField(producerClient, "lookup"); + + @Cleanup + var consumerClient = consumerClientFuture.get(); + @Cleanup + var consumer = consumerClient.newConsumer(Schema.INT32).topic(topicName.toString()). + subscriptionInitialPosition(SubscriptionInitialPosition.Earliest). + subscriptionName(BrokerTestUtil.newUniqueName("my-sub")). + ackTimeout(1000, TimeUnit.MILLISECONDS). + subscribe(); + LookupService consumerLookupServiceSpy = spyField(consumerClient, "lookup"); + + var bundleRange = admin.lookups().getBundleRange(topicName.toString()); + + var semSend = new Semaphore(0); + var messagesBeforeUnload = 100; + var messagesAfterUnload = 100; + + var pendingMessageIds = Collections.synchronizedSet(new HashSet()); + var producerFuture = CompletableFuture.runAsync(() -> { + try { + for (int i = 0; i < messagesBeforeUnload + messagesAfterUnload; i++) { + semSend.acquire(); + pendingMessageIds.add(i); + producer.send(i); + } + } catch (Exception e) { + throw new CompletionException(e); + } + }, threadPool).orTimeout(TEST_TIMEOUT_MS, TimeUnit.MILLISECONDS); + + var consumerFuture = CompletableFuture.runAsync(() -> { + while (!producerFuture.isDone() || !pendingMessageIds.isEmpty()) { + try { + var recvMessage = consumer.receive(1_500, TimeUnit.MILLISECONDS); + if (recvMessage != null) { + consumer.acknowledge(recvMessage); + pendingMessageIds.remove(recvMessage.getValue()); + } + } catch (PulsarClientException e) { + // Retry + } + } + }, threadPool).orTimeout(TEST_TIMEOUT_MS, TimeUnit.MILLISECONDS); + + var dstBrokerLookupUrl = getDstBrokerLookupUrl(topicName); + semSend.release(messagesBeforeUnload); + admin.namespaces().unloadNamespaceBundle(namespaceName.toString(), bundleRange, dstBrokerLookupUrl); + semSend.release(messagesAfterUnload); + + // Verify all futures completed successfully. + producerFuture.get(); + consumerFuture.get(); + + verify(producerClient, times(1)).getProxyConnection(any(), anyInt()); + verify(producerLookupServiceSpy, never()).getBroker(topicName); + + verify(consumerClient, times(1)).getProxyConnection(any(), anyInt()); + verify(consumerLookupServiceSpy, never()).getBroker(topicName); + } + + @Test(timeOut = TEST_TIMEOUT_MS) + public void testClientReconnectsToBrokerOnProxyClosing() throws Exception { + var namespaceName = NamespaceName.get("public", "default"); + var topicName = TopicName.get(TopicDomain.persistent.toString(), namespaceName, + BrokerTestUtil.newUniqueName("testClientReconnectsToBrokerOnProxyClosing")); + + @Cleanup("shutdownNow") + var threadPool = Executors.newCachedThreadPool(); + + var producerClientFuture = CompletableFuture.supplyAsync(() -> createClient(proxyService), threadPool); + var consumerClientFuture = CompletableFuture.supplyAsync(() -> createClient(proxyService), threadPool); + + @Cleanup + var producerClient = producerClientFuture.get(); + @Cleanup + var producer = (ProducerImpl) producerClient.newProducer(Schema.INT32).topic(topicName.toString()). + create(); + LookupService producerLookupServiceSpy = spyField(producerClient, "lookup"); + when(((ServiceNameResolver) spyField(producerLookupServiceSpy, "serviceNameResolver")).resolveHost()). + thenCallRealMethod().then(invocation -> getSourceBrokerInetAddress(topicName)); + + @Cleanup + var consumerClient = consumerClientFuture.get(); + @Cleanup + var consumer = (ConsumerImpl) consumerClient.newConsumer(Schema.INT32).topic(topicName.toString()). + subscriptionInitialPosition(SubscriptionInitialPosition.Earliest). + subscriptionName(BrokerTestUtil.newUniqueName("my-sub")). + ackTimeout(1000, TimeUnit.MILLISECONDS). + subscribe(); + LookupService consumerLookupServiceSpy = spyField(consumerClient, "lookup"); + when(((ServiceNameResolver) spyField(consumerLookupServiceSpy, "serviceNameResolver")).resolveHost()). + thenCallRealMethod().then(invocation -> getSourceBrokerInetAddress(topicName)); + + var bundleRange = admin.lookups().getBundleRange(topicName.toString()); + + var semSend = new Semaphore(0); + var messagesPerPhase = 100; + var phases = 4; + var totalMessages = messagesPerPhase * phases; + var cdlSentMessages = new CountDownLatch(messagesPerPhase * 2); + + var pendingMessageIds = Collections.synchronizedSet(new HashSet()); + var producerFuture = CompletableFuture.runAsync(() -> { + try { + for (int i = 0; i < totalMessages; i++) { + semSend.acquire(); + pendingMessageIds.add(i); + producer.send(i); + cdlSentMessages.countDown(); + } + } catch (Exception e) { + throw new CompletionException(e); + } + }, threadPool).orTimeout(TEST_TIMEOUT_MS, TimeUnit.MILLISECONDS); + + var consumerFuture = CompletableFuture.runAsync(() -> { + while (!producerFuture.isDone() || !pendingMessageIds.isEmpty()) { + try { + var recvMessage = consumer.receive(1_500, TimeUnit.MILLISECONDS); + if (recvMessage != null) { + consumer.acknowledge(recvMessage); + pendingMessageIds.remove(recvMessage.getValue()); + } + } catch (PulsarClientException e) { + // Retry + } + } + }, threadPool).orTimeout(TEST_TIMEOUT_MS, TimeUnit.MILLISECONDS); + + var dstBrokerLookupUrl = getDstBrokerLookupUrl(topicName); + semSend.release(messagesPerPhase); + admin.namespaces().unloadNamespaceBundle(namespaceName.toString(), bundleRange, dstBrokerLookupUrl); + semSend.release(messagesPerPhase); + + cdlSentMessages.await(); + assertEquals(FieldUtils.readDeclaredField(producer.getConnectionHandler(), "useProxy", true), Boolean.TRUE); + assertEquals(FieldUtils.readDeclaredField(consumer.getConnectionHandler(), "useProxy", true), Boolean.TRUE); + semSend.release(messagesPerPhase); + proxyService.close(); + proxyService = null; + semSend.release(messagesPerPhase); + + // Verify produce/consume futures completed successfully. + producerFuture.get(); + consumerFuture.get(); + + assertEquals(FieldUtils.readDeclaredField(producer.getConnectionHandler(), "useProxy", true), Boolean.FALSE); + assertEquals(FieldUtils.readDeclaredField(consumer.getConnectionHandler(), "useProxy", true), Boolean.FALSE); + + verify(producerClient, times(1)).getProxyConnection(any(), anyInt()); + verify(producerLookupServiceSpy, times(1)).getBroker(topicName); + + verify(consumerClient, times(1)).getProxyConnection(any(), anyInt()); + verify(consumerLookupServiceSpy, times(1)).getBroker(topicName); + } +} diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java index e912006faa022..63929ee72e446 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java @@ -39,7 +39,16 @@ import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.api.*; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; +import org.apache.pulsar.client.api.ClientBuilder; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.auth.AuthenticationToken; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.common.policies.data.AuthAction; @@ -59,6 +68,7 @@ public class ProxyWithJwtAuthorizationTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(ProxyWithJwtAuthorizationTest.class); + private static final String CLUSTER_NAME = "proxy-authorization"; private final String ADMIN_ROLE = "admin"; private final String PROXY_ROLE = "proxy"; @@ -74,6 +84,7 @@ public class ProxyWithJwtAuthorizationTest extends ProducerConsumerBase { private ProxyService proxyService; private WebServer webServer; private final ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; @BeforeMethod @Override @@ -96,7 +107,7 @@ protected void setup() throws Exception { providers.add(AuthenticationProviderToken.class.getName()); conf.setAuthenticationProviders(providers); - conf.setClusterName("proxy-authorization"); + conf.setClusterName(CLUSTER_NAME); conf.setNumExecutorThreadPoolSize(5); super.init(); @@ -111,15 +122,20 @@ protected void setup() throws Exception { proxyConfig.setServicePort(Optional.of(0)); proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setWebServicePort(Optional.of(0)); + proxyConfig.setClusterName(CLUSTER_NAME); // enable auth&auth and use JWT at proxy proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); proxyConfig.setBrokerClientAuthenticationParameters(PROXY_TOKEN); proxyConfig.setAuthenticationProviders(providers); + proxyConfig.setStatusFilePath("./src/test/resources/vip_status.html"); AuthenticationService authService = new AuthenticationService(PulsarConfigurationLoader.convertFrom(proxyConfig)); - proxyService = Mockito.spy(new ProxyService(proxyConfig, authService)); + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); + proxyService = Mockito.spy(new ProxyService(proxyConfig, authService, proxyClientAuthentication)); webServer = new WebServer(proxyConfig, authService); } @@ -129,11 +145,14 @@ protected void cleanup() throws Exception { super.internalCleanup(); proxyService.close(); webServer.stop(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } } private void startProxy() throws Exception { proxyService.start(); - ProxyServiceStarter.addWebServerHandlers(webServer, proxyConfig, proxyService, null); + ProxyServiceStarter.addWebServerHandlers(webServer, proxyConfig, proxyService, null, proxyClientAuthentication); webServer.start(); } @@ -405,18 +424,41 @@ public void testProxyAuthorizationWithPrefixSubscriptionAuthMode() throws Except log.info("-- Exiting {} test --", methodName); } + @Test + void testGetStatus() throws Exception { + log.info("-- Starting {} test --", methodName); + final PulsarResources resource = new PulsarResources(registerCloseable(new ZKMetadataStore(mockZooKeeper)), + registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))); + final AuthenticationService authService = new AuthenticationService( + PulsarConfigurationLoader.convertFrom(proxyConfig)); + final WebServer webServer = new WebServer(proxyConfig, authService); + ProxyServiceStarter.addWebServerHandlers(webServer, proxyConfig, proxyService, + registerCloseable(new BrokerDiscoveryProvider(proxyConfig, resource)), proxyClientAuthentication); + webServer.start(); + @Cleanup + final Client client = javax.ws.rs.client.ClientBuilder + .newClient(new ClientConfig().register(LoggingFeature.class)); + try { + final Response r = client.target(webServer.getServiceUri()).path("/status.html").request().get(); + Assert.assertEquals(r.getStatus(), Response.Status.OK.getStatusCode()); + } finally { + webServer.stop(); + } + log.info("-- Exiting {} test --", methodName); + } + @Test void testGetMetrics() throws Exception { log.info("-- Starting {} test --", methodName); startProxy(); - PulsarResources resource = new PulsarResources(new ZKMetadataStore(mockZooKeeper), - new ZKMetadataStore(mockZooKeeperGlobal)); + PulsarResources resource = new PulsarResources(registerCloseable(new ZKMetadataStore(mockZooKeeper)), + registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))); AuthenticationService authService = new AuthenticationService( PulsarConfigurationLoader.convertFrom(proxyConfig)); proxyConfig.setAuthenticateMetricsEndpoint(false); WebServer webServer = new WebServer(proxyConfig, authService); ProxyServiceStarter.addWebServerHandlers(webServer, proxyConfig, proxyService, - new BrokerDiscoveryProvider(proxyConfig, resource)); + registerCloseable(new BrokerDiscoveryProvider(proxyConfig, resource)), proxyClientAuthentication); webServer.start(); @Cleanup Client client = javax.ws.rs.client.ClientBuilder.newClient(new ClientConfig().register(LoggingFeature.class)); @@ -429,7 +471,7 @@ void testGetMetrics() throws Exception { proxyConfig.setAuthenticateMetricsEndpoint(true); webServer = new WebServer(proxyConfig, authService); ProxyServiceStarter.addWebServerHandlers(webServer, proxyConfig, proxyService, - new BrokerDiscoveryProvider(proxyConfig, resource)); + registerCloseable(new BrokerDiscoveryProvider(proxyConfig, resource)), proxyClientAuthentication); webServer.start(); try { Response r = client.target(webServer.getServiceUri()).path("/metrics").request().get(); @@ -441,6 +483,7 @@ void testGetMetrics() throws Exception { } private void createAdminClient() throws Exception { + closeAdmin(); admin = spy(PulsarAdmin.builder().serviceHttpUrl(webServer.getServiceUri().toString()) .authentication(AuthenticationFactory.token(ADMIN_TOKEN)).build()); } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithoutServiceDiscoveryTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithoutServiceDiscoveryTest.java index 9c8e2ba33c9e8..885064b8e7404 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithoutServiceDiscoveryTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithoutServiceDiscoveryTest.java @@ -33,6 +33,7 @@ import org.apache.pulsar.broker.authentication.AuthenticationService; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Producer; @@ -54,8 +55,11 @@ public class ProxyWithoutServiceDiscoveryTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(ProxyWithoutServiceDiscoveryTest.class); + private static final String CLUSTER_NAME = "without-service-discovery"; private ProxyService proxyService; private ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; + @BeforeMethod @Override @@ -89,7 +93,7 @@ protected void setup() throws Exception { providers.add(AuthenticationProviderTls.class.getName()); conf.setAuthenticationProviders(providers); - conf.setClusterName("without-service-discovery"); + conf.setClusterName(CLUSTER_NAME); conf.setNumExecutorThreadPoolSize(5); super.init(); @@ -106,6 +110,7 @@ protected void setup() throws Exception { proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setWebServicePortTls(Optional.of(0)); proxyConfig.setTlsEnabledWithBroker(true); + proxyConfig.setClusterName(CLUSTER_NAME); // enable tls and auth&auth at proxy proxyConfig.setTlsCertificateFilePath(PROXY_CERT_FILE_PATH); @@ -119,9 +124,13 @@ protected void setup() throws Exception { proxyConfig.setAuthenticationProviders(providers); + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); + proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService( - PulsarConfigurationLoader.convertFrom(proxyConfig)))); + PulsarConfigurationLoader.convertFrom(proxyConfig)), proxyClientAuthentication)); proxyService.start(); } @@ -131,6 +140,9 @@ protected void setup() throws Exception { protected void cleanup() throws Exception { super.internalCleanup(); proxyService.close(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } } /** @@ -196,6 +208,7 @@ public void testDiscoveryService() throws Exception { } protected final PulsarClient createPulsarClient(Authentication auth, String lookupUrl) throws Exception { + closeAdmin(); admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString()).tlsTrustCertsFilePath(CA_CERT_FILE_PATH) .authentication(auth).build()); return PulsarClient.builder().serviceUrl(lookupUrl).statsInterval(0, TimeUnit.SECONDS) diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/SuperUserAuthedAdminProxyHandlerTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/SuperUserAuthedAdminProxyHandlerTest.java index d3291c8fb910d..71025ed484f7c 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/SuperUserAuthedAdminProxyHandlerTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/SuperUserAuthedAdminProxyHandlerTest.java @@ -32,6 +32,8 @@ import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.impl.auth.AuthenticationTls; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.common.policies.data.ClusterData; @@ -47,6 +49,7 @@ public class SuperUserAuthedAdminProxyHandlerTest extends MockedPulsarServiceBaseTest { private ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; private WebServer webServer; private BrokerDiscoveryProvider discoveryProvider; private PulsarResources resource; @@ -80,6 +83,7 @@ protected void setup() throws Exception { proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setWebServicePortTls(Optional.of(0)); proxyConfig.setTlsEnabledWithBroker(true); + proxyConfig.setClusterName(configClusterName); // enable tls and auth&auth at proxy proxyConfig.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); @@ -93,15 +97,19 @@ protected void setup() throws Exception { proxyConfig.setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH); proxyConfig.setAuthenticationProviders(ImmutableSet.of(AuthenticationProviderTls.class.getName())); - resource = new PulsarResources(new ZKMetadataStore(mockZooKeeper), - new ZKMetadataStore(mockZooKeeperGlobal)); + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); + + resource = new PulsarResources(registerCloseable(new ZKMetadataStore(mockZooKeeper)), + registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))); webServer = new WebServer(proxyConfig, new AuthenticationService( PulsarConfigurationLoader.convertFrom(proxyConfig))); - discoveryProvider = spy(new BrokerDiscoveryProvider(proxyConfig, resource)); + discoveryProvider = spy(registerCloseable(new BrokerDiscoveryProvider(proxyConfig, resource))); LoadManagerReport report = new LoadReport(brokerUrl.toString(), brokerUrlTls.toString(), null, null); doReturn(report).when(discoveryProvider).nextBroker(); - ServletHolder servletHolder = new ServletHolder(new AdminProxyHandler(proxyConfig, discoveryProvider)); + ServletHolder servletHolder = new ServletHolder(new AdminProxyHandler(proxyConfig, discoveryProvider, proxyClientAuthentication)); webServer.addServlet("/admin", servletHolder); webServer.addServlet("/lookup", servletHolder); @@ -113,6 +121,9 @@ protected void setup() throws Exception { @Override protected void cleanup() throws Exception { webServer.stop(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } super.internalCleanup(); } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/UnauthedAdminProxyHandlerTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/UnauthedAdminProxyHandlerTest.java index aa4aeaa2ea887..0b597b933544a 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/UnauthedAdminProxyHandlerTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/UnauthedAdminProxyHandlerTest.java @@ -35,6 +35,8 @@ import org.apache.pulsar.broker.authentication.AuthenticationService; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.common.configuration.VipStatus; import org.apache.pulsar.metadata.impl.ZKMetadataStore; @@ -49,6 +51,7 @@ public class UnauthedAdminProxyHandlerTest extends MockedPulsarServiceBaseTest { private final String STATUS_FILE_PATH = "./src/test/resources/vip_status.html"; private ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private Authentication proxyClientAuthentication; private WebServer webServer; private BrokerDiscoveryProvider discoveryProvider; private AdminProxyWrapper adminProxyHandler; @@ -75,14 +78,19 @@ protected void setup() throws Exception { proxyConfig.setStatusFilePath(STATUS_FILE_PATH); proxyConfig.setMetadataStoreUrl(DUMMY_VALUE); proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); + proxyConfig.setClusterName(configClusterName); + + proxyClientAuthentication = AuthenticationFactory.create(proxyConfig.getBrokerClientAuthenticationPlugin(), + proxyConfig.getBrokerClientAuthenticationParameters()); + proxyClientAuthentication.start(); webServer = new WebServer(proxyConfig, new AuthenticationService( PulsarConfigurationLoader.convertFrom(proxyConfig))); - resource = new PulsarResources(new ZKMetadataStore(mockZooKeeper), - new ZKMetadataStore(mockZooKeeperGlobal)); - discoveryProvider = spy(new BrokerDiscoveryProvider(proxyConfig, resource)); - adminProxyHandler = new AdminProxyWrapper(proxyConfig, discoveryProvider); + resource = new PulsarResources(registerCloseable(new ZKMetadataStore(mockZooKeeper)), + registerCloseable(new ZKMetadataStore(mockZooKeeperGlobal))); + discoveryProvider = spy(registerCloseable(new BrokerDiscoveryProvider(proxyConfig, resource))); + adminProxyHandler = new AdminProxyWrapper(proxyConfig, discoveryProvider, proxyClientAuthentication); ServletHolder servletHolder = new ServletHolder(adminProxyHandler); webServer.addServlet("/admin", servletHolder); webServer.addServlet("/lookup", servletHolder); @@ -100,10 +108,14 @@ protected void setup() throws Exception { protected void cleanup() throws Exception { internalCleanup(); webServer.stop(); + if (proxyClientAuthentication != null) { + proxyClientAuthentication.close(); + } } @Test public void testUnauthenticatedProxy() throws Exception { + @Cleanup PulsarAdmin admin = PulsarAdmin.builder() .serviceHttpUrl("http://127.0.0.1:" + webServer.getListenPortHTTP().get()) .build(); @@ -126,8 +138,8 @@ public void testVipStatus() throws Exception { static class AdminProxyWrapper extends AdminProxyHandler { String rewrittenUrl; - AdminProxyWrapper(ProxyConfiguration config, BrokerDiscoveryProvider discoveryProvider) { - super(config, discoveryProvider); + AdminProxyWrapper(ProxyConfiguration config, BrokerDiscoveryProvider discoveryProvider, Authentication proxyClientAuthentication) { + super(config, discoveryProvider, proxyClientAuthentication); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/utils/CmdTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentationTest.java similarity index 83% rename from pulsar-broker/src/test/java/org/apache/pulsar/utils/CmdTest.java rename to pulsar-proxy/src/test/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentationTest.java index a571a1cd0c92d..b76463e862f50 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/utils/CmdTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentationTest.java @@ -16,17 +16,22 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.utils; -import static org.testng.Assert.assertTrue; +package org.apache.pulsar.proxy.util; + import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.lang.reflect.Field; import org.apache.pulsar.common.configuration.FieldContext; import org.testng.annotations.Test; +import static org.testng.Assert.assertTrue; -@Test(groups = "utils") -public class CmdTest { +public class CmdGenerateDocumentationTest { + @Test + public void cmdParserProxyConfigurationTest() throws Exception { + String value = generateDoc("org.apache.pulsar.proxy.server.ProxyConfiguration"); + assertTrue(value.contains("Pulsar proxy")); + } @Test public void cmdParserTest() throws Exception { @@ -43,14 +48,14 @@ public void cmdParserClientTest() throws Exception { generateDoc("org.apache.pulsar.client.impl.conf.ClientConfigurationData"); } - private void generateDoc(String clazz) throws Exception { + private String generateDoc(String clazz) throws Exception { PrintStream oldStream = System.out; try (ByteArrayOutputStream baoStream = new ByteArrayOutputStream(2048); PrintStream cacheStream = new PrintStream(baoStream);) { System.setOut(cacheStream); CmdGenerateDocumentation.main(("-c " + clazz).split(" ")); String message = baoStream.toString(); - Class cls = Class.forName(clazz); + Class cls = Class.forName(clazz); Field[] fields = cls.getDeclaredFields(); for (Field field : fields) { field.setAccessible(true); @@ -60,6 +65,7 @@ private void generateDoc(String clazz) throws Exception { } assertTrue(message.indexOf(field.getName()) > 0); } + return message; } finally { System.setOut(oldStream); } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/util/CmdTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/util/CmdTest.java deleted file mode 100644 index 59e1a43310d67..0000000000000 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/util/CmdTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.proxy.util; - -import static org.testng.Assert.assertTrue; -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; -import java.lang.reflect.Field; -import org.apache.pulsar.common.configuration.FieldContext; -import org.testng.annotations.Test; - -public class CmdTest { - - @Test - public void cmdParserProxyConfigurationTest() throws Exception { - String value = generateDoc("org.apache.pulsar.proxy.server.ProxyConfiguration"); - assertTrue(value.contains("Pulsar proxy")); - } - - private String generateDoc(String clazz) throws Exception { - PrintStream oldStream = System.out; - try (ByteArrayOutputStream baoStream = new ByteArrayOutputStream(2048); - PrintStream cacheStream = new PrintStream(baoStream);) { - System.setOut(cacheStream); - CmdGenerateDocumentation.main(("-c " + clazz).split(" ")); - String message = baoStream.toString(); - Class cls = Class.forName(clazz); - Field[] fields = cls.getDeclaredFields(); - for (Field field : fields) { - field.setAccessible(true); - FieldContext fieldContext = field.getAnnotation(FieldContext.class); - if (fieldContext == null) { - continue; - } - assertTrue(message.indexOf(field.getName()) > 0); - } - return message; - } finally { - System.setOut(oldStream); - } - } -} diff --git a/pulsar-proxy/src/test/resources/log4j2.xml b/pulsar-proxy/src/test/resources/log4j2.xml new file mode 100644 index 0000000000000..261bd2edf6980 --- /dev/null +++ b/pulsar-proxy/src/test/resources/log4j2.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml deleted file mode 100644 index d027e1a56c578..0000000000000 --- a/pulsar-sql/pom.xml +++ /dev/null @@ -1,175 +0,0 @@ - - - 4.0.0 - pom - - org.apache.pulsar - pulsar - 3.1.0-SNAPSHOT - - - pulsar-sql - Pulsar SQL :: Parent - - - - 3.14.9 - - 1.17.2 - 213 - - - - - - - com.squareup.okhttp3 - okhttp - ${okhttp3.version} - - - com.squareup.okhttp3 - okhttp-urlconnection - ${okhttp3.version} - - - com.squareup.okhttp3 - logging-interceptor - ${okhttp3.version} - - - com.squareup.okio - okio - ${okio.version} - - - - - org.jline - jline-reader - ${jline3.version} - - - org.jline - jline-terminal - ${jline3.version} - - - org.jline - jline-terminal-jna - ${jline3.version} - - - - - org.slf4j - log4j-over-slf4j - ${slf4j.version} - - - org.slf4j - slf4j-jdk14 - ${slf4j.version} - - - - io.airlift - bom - ${airlift.version} - pom - import - - - - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - - - - - main - - - disableSqlMainProfile - - !true - - - - presto-pulsar - presto-pulsar-plugin - presto-distribution - - - - pulsar-sql-tests - - presto-pulsar - presto-pulsar-plugin - presto-distribution - - - - - owasp-dependency-check - - - - org.owasp - dependency-check-maven - ${dependency-check-maven.version} - - - - aggregate - - none - - - - - - - - - diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE deleted file mode 100644 index a85d1bc363c1b..0000000000000 --- a/pulsar-sql/presto-distribution/LICENSE +++ /dev/null @@ -1,598 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - ----------------------------------------------------------------------------------------------------- - -This projects includes binary packages with the following licenses: - -The Apache Software License, Version 2.0 - - * Jackson - - jackson-annotations-2.14.2.jar - - jackson-core-2.14.2.jar - - jackson-databind-2.14.2.jar - - jackson-dataformat-smile-2.14.2.jar - - jackson-datatype-guava-2.14.2.jar - - jackson-datatype-jdk8-2.14.2.jar - - jackson-datatype-joda-2.14.2.jar - - jackson-datatype-jsr310-2.14.2.jar - - jackson-dataformat-yaml-2.14.2.jar - - jackson-jaxrs-base-2.14.2.jar - - jackson-jaxrs-json-provider-2.14.2.jar - - jackson-module-jaxb-annotations-2.14.2.jar - - jackson-module-jsonSchema-2.14.2.jar - * Guava - - guava-31.0.1-jre.jar - - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar - - failureaccess-1.0.1.jar - * Google Guice - - guice-5.1.0.jar - * Apache Commons - - commons-math3-3.6.1.jar - - commons-compress-1.21.jar - - commons-lang3-3.11.jar - * Netty - - netty-buffer-4.1.89.Final.jar - - netty-codec-4.1.89.Final.jar - - netty-codec-dns-4.1.89.Final.jar - - netty-codec-http-4.1.89.Final.jar - - netty-codec-haproxy-4.1.89.Final.jar - - netty-codec-socks-4.1.89.Final.jar - - netty-handler-proxy-4.1.89.Final.jar - - netty-common-4.1.89.Final.jar - - netty-handler-4.1.89.Final.jar - - netty-reactive-streams-2.0.6.jar - - netty-resolver-4.1.89.Final.jar - - netty-resolver-dns-4.1.89.Final.jar - - netty-resolver-dns-classes-macos-4.1.89.Final.jar - - netty-resolver-dns-native-macos-4.1.89.Final-osx-aarch_64.jar - - netty-resolver-dns-native-macos-4.1.89.Final-osx-x86_64.jar - - netty-tcnative-boringssl-static-2.0.56.Final.jar - - netty-tcnative-boringssl-static-2.0.56.Final-linux-aarch_64.jar - - netty-tcnative-boringssl-static-2.0.56.Final-linux-x86_64.jar - - netty-tcnative-boringssl-static-2.0.56.Final-osx-aarch_64.jar - - netty-tcnative-boringssl-static-2.0.56.Final-osx-x86_64.jar - - netty-tcnative-boringssl-static-2.0.56.Final-windows-x86_64.jar - - netty-tcnative-classes-2.0.56.Final.jar - - netty-transport-4.1.89.Final.jar - - netty-transport-classes-epoll-4.1.89.Final.jar - - netty-transport-native-epoll-4.1.89.Final-linux-x86_64.jar - - netty-transport-native-unix-common-4.1.89.Final.jar - - netty-transport-native-unix-common-4.1.89.Final-linux-x86_64.jar - - netty-codec-http2-4.1.89.Final.jar - - netty-incubator-transport-classes-io_uring-0.0.18.Final.jar - - netty-incubator-transport-native-io_uring-0.0.18.Final-linux-x86_64.jar - - netty-incubator-transport-native-io_uring-0.0.18.Final-linux-aarch_64.jar - * GRPC - - grpc-api-1.45.1.jar - - grpc-context-1.45.1.jar - - grpc-core-1.45.1.jar - - grpc-grpclb-1.45.1.jar - - grpc-netty-1.45.1.jar - - grpc-protobuf-1.45.1.jar - - grpc-protobuf-lite-1.45.1.jar - - grpc-stub-1.45.1.jar - * JEtcd - - jetcd-api-0.7.5.jar - - jetcd-common-0.7.5.jar - - jetcd-core-0.7.5.jar - - jetcd-grpc-0.7.5.jar - * Vertx - - vertx-core-4.3.8.jar - - vertx-grpc-4.3.5.jar - * Joda Time - - joda-time-2.10.10.jar - - failsafe-2.4.4.jar - * Jetty - - http2-client-9.4.48.v20220622.jar - - http2-common-9.4.48.v20220622.jar - - http2-hpack-9.4.48.v20220622.jar - - http2-http-client-transport-9.4.48.v20220622.jar - - jetty-alpn-client-9.4.48.v20220622.jar - - http2-server-9.4.48.v20220622.jar - - jetty-alpn-java-client-9.4.48.v20220622.jar - - jetty-client-9.4.48.v20220622.jar - - jetty-http-9.4.48.v20220622.jar - - jetty-io-9.4.48.v20220622.jar - - jetty-jmx-9.4.48.v20220622.jar - - jetty-security-9.4.48.v20220622.jar - - jetty-server-9.4.48.v20220622.jar - - jetty-servlet-9.4.48.v20220622.jar - - jetty-util-9.4.48.v20220622.jar - - jetty-util-ajax-9.4.48.v20220622.jar - * Byte Buddy - - byte-buddy-1.11.13.jar - * Apache BVal - - bval-jsr-2.0.5.jar - * Bytecode - - bytecode-1.2.jar - * Airlift - - aircompressor-0.20.jar - - bootstrap-213.jar - - concurrent-213.jar - - configuration-213.jar - - discovery-213.jar - - discovery-server-1.30.jar - - event-213.jar - - event-http-213.jar - - http-client-213.jar - - http-server-213.jar - - jmx-213.jar - - jmx-http-213.jar - - jmx-http-rpc-213.jar - - joni-2.1.5.3.jar - - json-213.jar - - log-213.jar - - log-manager-213.jar - - node-213.jar - - parameternames-1.4.jar - - security-213.jar - - slice-0.41.jar - - stats-213.jar - - trace-token-213.jar - - units-1.6.jar - * Apache HTTP Client - - httpclient-4.5.13.jar - - httpcore-4.4.15.jar - * Error Prone Annotations - - error_prone_annotations-2.5.1.jar - * Esri Geometry API For Java - - esri-geometry-api-2.2.4.jar - * Failsafe - - failsafe-2.4.0.jar - * Fastutil - - fastutil-8.3.0.jar - * J2ObjC Annotations - - j2objc-annotations-1.3.jar - * JSON Web Token Support For The JVM - - jjwt-api-0.11.1.jar - - jjwt-impl-0.11.1.jar - - jjwt-jackson-0.11.1.jar - * Jmxutils - - jmxutils-1.21.jar - * LevelDB - - leveldb-0.12.jar - - leveldb-api-0.12.jar - * Log4j - - log4j-api-2.18.0.jar - - log4j-core-2.18.0.jar - - log4j-slf4j-impl-2.18.0.jar - * Log4j implemented over SLF4J - - log4j-over-slf4j-1.7.32.jar - * Lucene Common Analyzers - - lucene-analyzers-common-8.4.1.jar - - lucene-core-8.4.1.jar - * PicoCLI - - picocli-4.6.1.jar - * RxJava - - rxjava-3.0.1.jar - * OkHttp - - logging-interceptor-3.14.9.jar - - okhttp-3.14.9.jar - - okhttp-urlconnection-3.14.9.jar - * OpenCSV - - opencsv-2.3.jar - * Avro - - avro-1.10.2.jar - - avro-protobuf-1.10.2.jar - * Caffeine - - caffeine-2.9.1.jar - * Javax - - javax.inject-1.jar - - javax.servlet-api-3.1.0.jar - - javax.servlet-api-4.0.1.jar - - javax.ws.rs-api-2.1.jar - * JCommander - - jcommander-1.82.jar - * FindBugs JSR305 - - jsr305-3.0.2.jar - * Objenesis - - objenesis-2.6.jar - * Okio - - okio-1.17.2.jar - * Trino - - trino-array-368.jar - - trino-cli-368.jar - - trino-client-368.jar - - trino-geospatial-toolkit-368.jar - - trino-main-368.jar - - trino-matching-368.jar - - trino-memory-context-368.jar - - trino-parser-368.jar - - trino-plugin-toolkit-368.jar - - trino-server-main-368.jar - - trino-spi-368.jar - - trino-record-decoder-368.jar - * RocksDB JNI - - rocksdbjni-7.9.2.jar - * SnakeYAML - - snakeyaml-2.0.jar - * Bean Validation API - - validation-api-2.0.1.Final.jar - * Objectsize - - objectsize-0.0.12.jar - * Dropwizard Metrics - - metrics-core-4.1.12.1.jar - - metrics-graphite-4.1.12.1.jar - - metrics-jvm-4.1.12.1.jar - - metrics-jmx-4.1.12.1.jar - * Prometheus - - simpleclient-0.16.0.jar - - simpleclient_common-0.16.0.jar - - simpleclient_hotspot-0.16.0.jar - - simpleclient_servlet-0.16.0.jar - - simpleclient_servlet_common-0.16.0.jar - - simpleclient_tracer_common-0.16.0.jar - - simpleclient_tracer_otel-0.16.0.jar - - simpleclient_tracer_otel_agent-0.16.0.jar - * JCTools - - jctools-core-2.1.2.jar - * Asynchronous Http Client - - async-http-client-2.12.1.jar - - async-http-client-netty-utils-2.12.1.jar - * Apache Bookkeeper - - bookkeeper-common-4.16.1.jar - - bookkeeper-common-allocator-4.16.1.jar - - bookkeeper-proto-4.16.1.jar - - bookkeeper-server-4.16.1.jar - - bookkeeper-stats-api-4.16.1.jar - - bookkeeper-tools-framework-4.16.1.jar - - circe-checksum-4.16.1.jar - - codahale-metrics-provider-4.16.1.jar - - cpu-affinity-4.16.1.jar - - http-server-4.16.1.jar - - prometheus-metrics-provider-4.16.1.jar - - codahale-metrics-provider-4.16.1.jar - - bookkeeper-slogger-api-4.16.1.jar - - bookkeeper-slogger-slf4j-4.16.1.jar - - native-io-4.16.1.jar - * Apache Commons - - commons-cli-1.5.0.jar - - commons-codec-1.15.jar - - commons-collections4-4.4.jar - - commons-configuration-1.10.jar - - commons-io-2.8.0.jar - - commons-lang-2.6.jar - - commons-logging-1.2.jar - * GSON - - gson-2.8.9.jar - * JSON Simple - - json-simple-1.1.1.jar - * Snappy - - snappy-java-1.1.8.4.jar - * Jackson - - jackson-module-parameter-names-2.14.2.jar - * Java Assist - - javassist-3.25.0-GA.jar - * Java Native Access - - jna-5.12.1.jar - - jna-platform-5.10.0.jar - * Java Object Layout: Core - - jol-core-0.2.jar - * Yahoo Datasketches - - memory-0.8.3.jar - - sketches-core-0.8.3.jar - * Apache Zookeeper - - zookeeper-3.8.1.jar - - zookeeper-jute-3.8.1.jar - * Apache Yetus Audience Annotations - - audience-annotations-0.12.0.jar - * Swagger - - swagger-annotations-1.6.2.jar - * Perfmark - - perfmark-api-0.19.0.jar - * RabbitMQ Java Client - - amqp-client-5.5.3.jar - * Stream Lib - - stream-2.9.5.jar - * High Performance Primitive Collections for Java - - hppc-0.9.1.jar - - -Protocol Buffers License - * Protocol Buffers - - protobuf-java-3.19.6.jar - - protobuf-java-util-3.19.6.jar - - proto-google-common-protos-2.0.1.jar - -BSD 3-clause "New" or "Revised" License - * RE2J TD -- re2j-td-1.4.jar - * DSL Platform JSON - - dsl-json-1.8.4.jar - -BSD License - * ANTLR 4 Runtime - - antlr4-runtime-4.9.2.jar - * ASM, a very small and fast Java bytecode manipulation framework - - asm-9.1.jar - - asm-analysis-6.2.1.jar - - asm-tree-6.2.1.jar - - asm-util-6.2.1.jar - * JLine - - jline-reader-3.21.0.jar - - jline-terminal-3.21.0.jar - - jline-terminal-jna-3.21.0.jar - -MIT License - * PCollections - - pcollections-2.1.2.jar - * SLF4J - - slf4j-api-1.7.32.jar - - slf4j-jdk14-1.7.32.jar - * JCL 1.2 Implemented Over SLF4J - - jcl-over-slf4j-1.7.32.jar - * Checker Qual - - checker-qual-3.12.0.jar - * ScribeJava - - scribejava-apis-6.9.0.jar - - scribejava-core-6.9.0.jar - * OSHI - - oshi-core-5.8.5.jar - -CDDL - 1.0 - * OSGi Resource Locator - - osgi-resource-locator-1.0.3.jar - -CDDL-1.1 -- licenses/LICENSE-CDDL-1.1.txt - * Java Annotations API - - javax.activation-1.2.0.jar - - javax.activation-api-1.2.0.jar - * HK2 - Dependency Injection Kernel - - hk2-api-2.6.1.jar - - hk2-locator-2.6.1.jar - - hk2-utils-2.6.1.jar - - aopalliance-repackaged-2.6.1.jar - * Jersey - - jaxrs-213.jar - - jersey-client-2.34.jar - - jersey-common-2.34.jar - - jersey-container-servlet-2.34.jar - - jersey-container-servlet-core-2.34.jar - - jersey-entity-filtering-2.34.jar - - jersey-hk2-2.34.jar - - jersey-media-json-jackson-2.34.jar - - jersey-media-multipart-2.34.jar - - jersey-server-2.34.jar - * JAXB - - jaxb-api-2.3.1.jar - - jaxb-runtime-2.3.4.jar - - txw2-2.3.4.jar - Eclipse Distribution License 1.0 -- licenses/LICENSE-EDL-1.0.txt - * istack-commons-runtime-3.0.12.jar - * jts-io-common-1.16.1.jar - - Eclipse Public License 1.0 -- licenses/LICENSE-EPL-1.0.txt - * JTS Core - - jts-core-1.16.1.jar - * JGraphT Core - - jgrapht-core-0.9.0.jar - * Logback Core Module - - logback-core-1.2.3.jar - * MIME Streaming Extension - - mimepull-1.9.13.jar - -Eclipse Public License - v2.0 -- licenses/LICENSE-EPL-2.0.txt - * jakarta.annotation-api-1.3.5.jar - * jakarta.inject-2.6.1.jar - * jakarta.validation-api-2.0.2.jar - * jakarta.ws.rs-api-2.1.6.jar - * jakarta.activation-api-1.2.2.jar - * jakarta.xml.bind-api-2.3.3.jar - -Public Domain (CC0) -- licenses/LICENSE-CC0.txt - * HdrHistogram - - HdrHistogram-2.1.9.jar - * AOP Alliance - - aopalliance-1.0.jar - * Reactive Streams - - reactive-streams-1.0.3.jar - -Creative Commons Attribution License - * Jcip -- licenses/LICENSE-jcip.txt - - jcip-annotations-1.0.jar - -Bouncy Castle License - * Bouncy Castle -- licenses/LICENSE-bouncycastle.txt - - bcpkix-jdk15on-1.69.jar - - bcprov-ext-jdk15on-1.69.jar - - bcprov-jdk15on-1.69.jar - - bcutil-jdk15on-1.69.jar diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml deleted file mode 100644 index e33a5733bbefb..0000000000000 --- a/pulsar-sql/presto-distribution/pom.xml +++ /dev/null @@ -1,384 +0,0 @@ - - - 4.0.0 - - - org.apache.pulsar - pulsar-sql - 3.1.0-SNAPSHOT - - - pulsar-presto-distribution - Pulsar SQL :: Pulsar Presto Distribution - - - false - 2.34 - 2.6 - 0.0.12 - 3.0.5 - 31.0.1-jre - 2.12.1 - 2.5.1 - 4.0.1 - - - - - org.glassfish.jersey.core - jersey-common - ${jersey.version} - - - org.glassfish.jersey.core - jersey-server - ${jersey.version} - - - org.glassfish.jersey.containers - jersey-container-servlet-core - ${jersey.version} - - - org.glassfish.jersey.containers - jersey-container-servlet - ${jersey.version} - - - org.glassfish.jersey.inject - jersey-hk2 - ${jersey.version} - - - org.glassfish.jersey.core - jersey-client - ${jersey.version} - - - - io.trino - trino-server-main - ${trino.version} - - - - org.openjdk.jol - jol-core - - - com.sun - tools - - - javax.activation - activation - - - com.google.inject.extensions - guice-multibindings - - - org.apache.logging.log4j - log4j-to-slf4j - - - - - - io.trino - trino-cli - ${trino.version} - - - - io.airlift - launcher - ${airlift.version} - tar.gz - bin - - - - io.airlift - launcher - ${airlift.version} - tar.gz - properties - - - - org.objenesis - objenesis - ${objenesis.version} - - - - com.twitter.common - objectsize - ${objectsize.version} - - - jsr305 - com.google.code.findbugs - - - - - - - - com.fasterxml.jackson.core - jackson-core - - - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.core - jackson-annotations - - - - com.fasterxml.jackson.datatype - jackson-datatype-joda - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - com.fasterxml.jackson.datatype - jackson-datatype-guava - - - - com.fasterxml.jackson.datatype - jackson-datatype-jdk8 - - - - com.fasterxml.jackson.datatype - jackson-datatype-jsr310 - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-smile - - - - - - - - org.asynchttpclient - async-http-client - ${asynchttpclient.version} - - - io.netty - netty - 3.10.6.Final - - - - org.apache.maven - maven-core - ${maven.version} - - - org.apache.maven - maven-model - ${maven.version} - - - org.apache.maven - maven-artifact - ${maven.version} - - - org.apache.maven - maven-aether-provider - ${maven.version} - - - org.apache.maven - maven-embedder - ${maven.version} - - - com.google.guava - guava - ${guava.version} - - - com.google.errorprone - error_prone_annotations - ${errorprone.version} - - - com.fasterxml.jackson - jackson-bom - ${jackson.version} - pom - import - - - org.eclipse.jetty - jetty-http - ${jetty.version} - - - org.eclipse.jetty - jetty-client - ${jetty.version} - - - org.eclipse.jetty - jetty-io - ${jetty.version} - - - org.eclipse.jetty - jetty-security - ${jetty.version} - - - org.eclipse.jetty - jetty-jmx - ${jetty.version} - - - org.eclipse.jetty.http2 - http2-client - ${jetty.version} - - - org.eclipse.jetty.http2 - http2-http-client-transport - ${jetty.version} - - - org.eclipse.jetty.http2 - http2-server - ${jetty.version} - - - javax.servlet - javax.servlet-api - ${javax.servlet-api} - - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - ${skipBuildDistribution} - - - - - org.apache.maven.plugins - maven-assembly-plugin - ${maven-assembly-plugin.version} - - false - true - posix - - src/assembly/assembly.xml - - ${project.artifactId} - - - - package - package - - single - - - - - - - com.mycila - license-maven-plugin - ${license-maven-plugin.version} - - - -
      ../../src/license-header.txt
      -       -       - - SLASHSTAR_STYLE - -       -       -       - - - org.apache.maven.wagon - wagon-ssh-external - 3.4.3 - - -       - - - - skipBuildDistributionDisabled - - - skipBuildDistribution - !true - - - - - ${project.groupId} - pulsar-presto-connector - ${project.version} - tar.gz - provided - - - * - * - - - - - - -       diff --git a/pulsar-sql/presto-distribution/src/assembly/assembly.xml b/pulsar-sql/presto-distribution/src/assembly/assembly.xml deleted file mode 100644 index 96c0421c71515..0000000000000 --- a/pulsar-sql/presto-distribution/src/assembly/assembly.xml +++ /dev/null @@ -1,81 +0,0 @@ - - - bin - - tar.gz - dir - - false - - - ${basedir}/LICENSE - LICENSE - . - 644 - - - ${basedir}/src/main/resources/launcher.properties - launcher.properties - bin/ - 644 - - - - - ${basedir}/../presto-pulsar-plugin/target/pulsar-presto-connector/ - plugin/ - - - ${basedir}/src/main/resources/conf/ - conf/ - - - - - lib/ - true - runtime - - io.airlift:launcher:tar.gz:bin:${airlift.version} - io.airlift:launcher:tar.gz:properties:${airlift.version} - *:tar.gz - - - - - - io.airlift:launcher:tar.gz:bin:${airlift.version} - - true - 755 - - - - - io.airlift:launcher:tar.gz:properties:${airlift.version} - - true - - - \ No newline at end of file diff --git a/pulsar-sql/presto-distribution/src/main/java/org/openjdk/jol/info/ClassLayout.java b/pulsar-sql/presto-distribution/src/main/java/org/openjdk/jol/info/ClassLayout.java deleted file mode 100644 index 2d0ed3b7b98b4..0000000000000 --- a/pulsar-sql/presto-distribution/src/main/java/org/openjdk/jol/info/ClassLayout.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.openjdk.jol.info; - -import com.twitter.common.objectsize.ObjectSizeCalculator; -import io.airlift.log.Logger; -import org.objenesis.ObjenesisStd; - -/** - * Mock class avoid a dependency on OpenJDK JOL, - * which is incompatible with the Apache License. - */ -public class ClassLayout { - - private static final Logger log = Logger.get(ClassLayout.class); - - private int size; - private static final int DEFAULT_SIZE = 64; - - private ClassLayout(int size) { - this.size = size; - } - - public static ClassLayout parseClass(Class clazz) { - long size = DEFAULT_SIZE; - try { - size = ObjectSizeCalculator.getObjectSize(new ObjenesisStd().newInstance(clazz)); - } catch (Throwable th) { - log.info("Error estimating size of class %s", clazz, th); - } - return new ClassLayout(Math.toIntExact(size)); - } - - public int instanceSize() { - return size; - } -} \ No newline at end of file diff --git a/pulsar-sql/presto-distribution/src/main/resources/conf/catalog/pulsar.properties b/pulsar-sql/presto-distribution/src/main/resources/conf/catalog/pulsar.properties deleted file mode 100644 index f15cd2657e0b5..0000000000000 --- a/pulsar-sql/presto-distribution/src/main/resources/conf/catalog/pulsar.properties +++ /dev/null @@ -1,127 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# name of the connector to be displayed in the catalog -connector.name=pulsar -# the url of Pulsar broker service -# DEPRECATED -pulsar.broker-service-url=http://localhost:8080 -# the url of Pulsar broker web service -pulsar.web-service-url=http://localhost:8080 -# the url of Pulsar broker binary service -pulsar.broker-binary-service-url=pulsar://localhost:6650 -# the url of metadata store -pulsar.metadata-url=zk:127.0.0.1:2181 -# minimum number of entries to read at a single time -pulsar.max-entry-read-batch-size=100 -# default number of splits to use per query -pulsar.target-num-splits=2 -# max message queue size -pulsar.max-split-message-queue-size=10000 -# max entry queue size -pulsar.max-split-entry-queue-size=1000 -# half of this value is used as max entry queue size bytes and the left is used as max message queue size bytes, -# the queue size bytes shouldn't exceed this value, but it's not strict, the default value -1 indicate no limit. -pulsar.max-split-queue-cache-size=-1 -# Rewrite namespace delimiter -# Warn: avoid using symbols allowed by Namespace (a-zA-Z_0-9 -=:%) -# to prevent erroneous rewriting -pulsar.namespace-delimiter-rewrite-enable=false -pulsar.rewrite-namespace-delimiter=/ -# max size of one batch message (default value is 5MB) -# pulsar.max-message-size=5242880 - -####### TIERED STORAGE OFFLOADER CONFIGS ####### - -## Driver to use to offload old data to long term storage -#pulsar.managed-ledger-offload-driver = aws-s3 - -## The directory to locate offloaders -#pulsar.offloaders-directory = /pulsar/offloaders - -## Maximum number of thread pool threads for ledger offloading -#pulsar.managed-ledger-offload-max-threads = 2 - -## Properties and configurations related to specific offloader implementation -#pulsar.offloader-properties = \ -# {"s3ManagedLedgerOffloadBucket": "offload-bucket", \ -# "s3ManagedLedgerOffloadRegion": "us-west-2", \ -# "s3ManagedLedgerOffloadServiceEndpoint": "http://s3.amazonaws.com"} - - -####### AUTHENTICATION CONFIGS ####### - -## the authentication plugin to be used to authenticate to Pulsar cluster -#pulsar.auth-plugin= - -## the authentication parameter to be used to authenticate to Pulsar cluster -#pulsar.auth-params= - -## Accept untrusted TLS certificate -#pulsar.tls-allow-insecure-connection = - -## Whether to enable hostname verification on TLS connections -#pulsar.tls-hostname-verification-enable = - -## Path for the trusted TLS certificate file -#pulsar.tls-trust-cert-file-path = - -####### PULSAR AUTHORIZATION CONFIGS ####### - -## Whether to enable pulsar authorization -pulsar.authorization-enabled=false - -####### BOOKKEEPER CONFIGS ####### - -# Entries read count throttling-limit per seconds, 0 is represents disable the throttle, default is 0. -pulsar.bookkeeper-throttle-value = 0 - -# The number of threads used by Netty to handle TCP connections, -# default is 2 * Runtime.getRuntime().availableProcessors(). -# pulsar.bookkeeper-num-io-threads = - -# The number of worker threads used by bookkeeper client to submit operations, -# default is Runtime.getRuntime().availableProcessors(). -# pulsar.bookkeeper-num-worker-threads = - -# Whether the bookkeeper client use v2 protocol or v3 protocol. -# Default is the v2 protocol which the LAC is piggy back lac. Otherwise the client -# will use v3 protocol and use explicit lac. -pulsar.bookkeeper-use-v2-protocol=true -pulsar.bookkeeper-explicit-interval=0 - -####### MANAGED LEDGER CONFIGS ####### - -# Amount of memory to use for caching data payload in managed ledger. This memory -# is allocated from JVM direct memory and it's shared across all the managed ledgers -# running in same sql worker. 0 is represents disable the cache, default is 0. -pulsar.managed-ledger-cache-size-MB = 0 - -# Number of threads to be used for managed ledger tasks dispatching, -# default is Runtime.getRuntime().availableProcessors(). -# pulsar.managed-ledger-num-worker-threads = - -# Number of threads to be used for managed ledger scheduled tasks, -# default is Runtime.getRuntime().availableProcessors(). -# pulsar.managed-ledger-num-scheduler-threads = - -####### PROMETHEUS CONFIGS ####### - -# pulsar.stats-provider=org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider -# pulsar.stats-provider-configs={"httpServerEnabled":"false", "prometheusStatsHttpPort":"9092", "prometheusStatsHttpEnable":"true"} diff --git a/pulsar-sql/presto-pulsar-plugin/pom.xml b/pulsar-sql/presto-pulsar-plugin/pom.xml deleted file mode 100644 index 5e88a24bba530..0000000000000 --- a/pulsar-sql/presto-pulsar-plugin/pom.xml +++ /dev/null @@ -1,78 +0,0 @@ - - - 4.0.0 - - - org.apache.pulsar - pulsar-sql - 3.1.0-SNAPSHOT - - - pulsar-presto-connector - Pulsar SQL :: Pulsar Presto Connector - - - - - ${project.groupId} - pulsar-presto-connector-original - ${project.version} - - - - ${project.groupId} - bouncy-castle-bc - ${project.version} - pkg - true - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - 3.3.0 - - false - true - posix - - src/assembly/assembly.xml - - - - - package - package - - single - - - - - - - - diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml deleted file mode 100644 index 3ff073f62e459..0000000000000 --- a/pulsar-sql/presto-pulsar/pom.xml +++ /dev/null @@ -1,260 +0,0 @@ - - - 4.0.0 - - - org.apache.pulsar - pulsar-sql - 3.1.0-SNAPSHOT - - - pulsar-presto-connector-original - Pulsar SQL - Pulsar Presto Connector - Pulsar SQL :: Pulsar Presto Connector Packaging - - - 2.1.2 - 1.8.4 - - - - - io.airlift - bootstrap - - - org.apache.logging.log4j - log4j-to-slf4j - - - - - io.airlift - json - - - - org.apache.avro - avro - ${avro.version} - - - - ${project.groupId} - pulsar-client-admin-original - ${project.version} - - - - ${project.groupId} - managed-ledger - ${project.version} - - - - org.jctools - jctools-core - ${jctools.version} - - - - com.dslplatform - dsl-json - ${dslJson.verson} - - - - io.trino - trino-plugin-toolkit - ${trino.version} - - - - - io.trino - trino-spi - ${trino.version} - provided - - - - joda-time - joda-time - ${joda.version} - - - - io.trino - trino-record-decoder - ${trino.version} - - - - javax.annotation - javax.annotation-api - ${javax.annotation-api.version} - - - - io.jsonwebtoken - jjwt-impl - ${jsonwebtoken.version} - test - - - - io.trino - trino-main - ${trino.version} - test - - - - io.trino - trino-testing - ${trino.version} - test - - - - org.apache.pulsar - pulsar-broker - ${project.version} - test - - - - org.apache.pulsar - testmocks - ${project.version} - test - - - - org.eclipse.jetty - jetty-http - ${jetty.version} - test - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - ${shadePluginPhase} - - shade - - - true - true - - - - org.apache.pulsar:pulsar-client-original - org.apache.pulsar:pulsar-client-admin-original - org.apache.pulsar:managed-ledger - org.apache.pulsar:pulsar-metadata - - org.glassfish.jersey*:* - javax.ws.rs:* - javax.annotation:* - org.glassfish.hk2*:* - - org.eclipse.jetty:* - - - - - - org.apache.pulsar:pulsar-client-original - - ** - - - - org/bouncycastle/** - - - - - - org.glassfish - org.apache.pulsar.shade.org.glassfish - - - javax.ws - org.apache.pulsar.shade.javax.ws - - - javax.annotation - org.apache.pulsar.shade.javax.annotation - - - jersey - org.apache.pulsar.shade.jersey - - - org.eclipse.jetty - org.apache.pulsar.shade.org.eclipse.jetty - - - - - - - - - - - - - - - - - - test-jar-dependencies - - - maven.test.skip - !true - - - - - ${project.groupId} - pulsar-broker - ${project.version} - test-jar - test - - - - - diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarAuth.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarAuth.java deleted file mode 100644 index 3307faf9c2186..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarAuth.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import static io.trino.spi.StandardErrorCode.PERMISSION_DENIED; -import static io.trino.spi.StandardErrorCode.QUERY_REJECTED; -import com.google.common.annotations.VisibleForTesting; -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ConnectorSession; -import java.io.IOException; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import lombok.Cleanup; -import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.client.api.SubscriptionMode; -import org.apache.pulsar.client.api.SubscriptionType; - -/** - * This class implements the authentication and authorization integration between the Pulsar SQL worker and the - * Pulsar broker. - * - * It will check permissions against the session-topic pair by trying to subscribe to a topic using the Pulsar Reader - * to check the consumption privilege. The same topic will only be checked once during the same session. - */ -public class PulsarAuth { - - private static final Logger log = Logger.get(PulsarAuth.class); - - private final PulsarConnectorConfig pulsarConnectorConfig; - private static final String CREDENTIALS_AUTH_PLUGIN = "auth-plugin"; - private static final String CREDENTIALS_AUTH_PARAMS = "auth-params"; - @VisibleForTesting - final Map> authorizedQueryTopicsMap = new ConcurrentHashMap<>(); - - @Inject - public PulsarAuth(PulsarConnectorConfig pulsarConnectorConfig) { - this.pulsarConnectorConfig = pulsarConnectorConfig; - if (pulsarConnectorConfig.getAuthorizationEnabled() && StringUtils.isEmpty( - pulsarConnectorConfig.getBrokerBinaryServiceUrl())) { - throw new IllegalArgumentException( - "pulsar.broker-binary-service-url must be present when the pulsar.authorization-enable is true."); - } - } - - /** - * Check if the session has read access to the topic. - * It will try to subscribe to that topic using the Pulsar Reader to check the consumption privilege. - * The same topic will only be checked once during the same session. - */ - public void checkTopicAuth(ConnectorSession session, String topic) { - Set authorizedTopics = - authorizedQueryTopicsMap.computeIfAbsent(session.getQueryId(), query -> new HashSet<>()); - if (authorizedTopics.contains(topic)) { - if (log.isDebugEnabled()) { - log.debug("The topic %s is already authorized.", topic); - } - return; - } - if (log.isDebugEnabled()) { - log.debug("Checking the authorization for the topic: %s", topic); - } - Map extraCredentials = session.getIdentity().getExtraCredentials(); - if (extraCredentials.isEmpty()) { // the extraCredentials won't be null - throw new TrinoException(QUERY_REJECTED, - String.format( - "Failed to check the authorization for topic %s: The credential information is empty.", - topic)); - } - String authMethod = extraCredentials.get(CREDENTIALS_AUTH_PLUGIN); - String authParams = extraCredentials.get(CREDENTIALS_AUTH_PARAMS); - if (StringUtils.isEmpty(authMethod) || StringUtils.isEmpty(authParams)) { - throw new TrinoException(QUERY_REJECTED, - String.format( - "Failed to check the authorization for topic %s: Required credential parameters are " - + "missing. Please specify the auth-method and auth-params in the extra " - + "credentials.", - topic)); - } - try { - @Cleanup - PulsarClient client = PulsarClient.builder() - .serviceUrl(pulsarConnectorConfig.getBrokerBinaryServiceUrl()) - .authentication(authMethod, authParams) - .build(); - client.newConsumer().topic(topic) - .subscriptionName("pulsar-sql-auth" + session.getQueryId()) - .subscriptionType(SubscriptionType.Exclusive) - .subscriptionMode(SubscriptionMode.NonDurable) - .startPaused(true) - .subscribe() - .close(); - authorizedQueryTopicsMap.computeIfPresent(session.getQueryId(), (query, topics) -> { - topics.add(topic); - return topics; - }); - if (log.isDebugEnabled()) { - log.debug("Check the authorization for the topic %s successfully.", topic); - } - } catch (PulsarClientException.AuthenticationException | PulsarClientException.AuthorizationException e) { - throw new TrinoException(PERMISSION_DENIED, - String.format("Failed to access topic %s: %s", topic, e.getLocalizedMessage())); - } catch (IOException e) { - throw new TrinoException(QUERY_REJECTED, - String.format("Failed to check authorization for topic %s: %s", topic, e.getLocalizedMessage())); - } - } - - /** - * When the session is closed, this method needs to be called to clear the session's auth verification status. - */ - public void cleanSession(ConnectorSession session) { - authorizedQueryTopicsMap.remove(session.getQueryId()); - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnHandle.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnHandle.java deleted file mode 100644 index 979d62e430284..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnHandle.java +++ /dev/null @@ -1,247 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import static java.util.Objects.requireNonNull; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.type.Type; -import java.util.Objects; - -/** - * This class represents the basic information about a presto column. - */ -public class PulsarColumnHandle implements DecoderColumnHandle { - - private final String connectorId; - - /** - * Column Name. - */ - private final String name; - - /** - * Column type. - */ - private final Type type; - - /** - * True if the column should be hidden. - */ - private final boolean hidden; - - /** - * True if the column is internal to the connector and not defined by a topic definition. - */ - private final boolean internal; - - - private HandleKeyValueType handleKeyValueType; - - /** - * {@link org.apache.pulsar.sql.presto.PulsarColumnMetadata.DecoderExtraInfo#mapping}. - */ - private String mapping; - /** - * {@link org.apache.pulsar.sql.presto.PulsarColumnMetadata.DecoderExtraInfo#dataFormat}. - */ - private String dataFormat; - - /** - * {@link org.apache.pulsar.sql.presto.PulsarColumnMetadata.DecoderExtraInfo#formatHint}. - */ - private String formatHint; - - /** - * Column Handle keyValue type, used for keyValue schema. - */ - public enum HandleKeyValueType { - /** - * The handle not for keyValue schema. - */ - NONE, - /** - * The key schema handle for keyValue schema. - */ - KEY, - /** - * The value schema handle for keyValue schema. - */ - VALUE - } - - @JsonCreator - public PulsarColumnHandle( - @JsonProperty("connectorId") String connectorId, - @JsonProperty("name") String name, - @JsonProperty("type") Type type, - @JsonProperty("hidden") boolean hidden, - @JsonProperty("internal") boolean internal, - @JsonProperty("mapping") String mapping, - @JsonProperty("dataFormat") String dataFormat, - @JsonProperty("formatHint") String formatHint, - @JsonProperty("handleKeyValueType") HandleKeyValueType handleKeyValueType) { - this.connectorId = requireNonNull(connectorId, "connectorId is null"); - this.name = requireNonNull(name, "name is null"); - this.type = requireNonNull(type, "type is null"); - this.hidden = hidden; - this.internal = internal; - this.mapping = mapping; - this.dataFormat = dataFormat; - this.formatHint = formatHint; - if (handleKeyValueType == null) { - this.handleKeyValueType = HandleKeyValueType.NONE; - } else { - this.handleKeyValueType = handleKeyValueType; - } - } - - @JsonProperty - public String getConnectorId() { - return connectorId; - } - - @JsonProperty - public String getName() { - return name; - } - - @JsonProperty - public String getMapping() { - return mapping; - } - - @JsonProperty - public String getDataFormat() { - return dataFormat; - } - - @JsonProperty - public Type getType() { - return type; - } - - @JsonProperty - public boolean isHidden() { - return hidden; - } - - @JsonProperty - public boolean isInternal() { - return internal; - } - - @JsonProperty - public String getFormatHint() { - return formatHint; - } - - @JsonProperty - public HandleKeyValueType getHandleKeyValueType() { - return handleKeyValueType; - } - - @JsonIgnore - public boolean isKey() { - return Objects.equals(handleKeyValueType, HandleKeyValueType.KEY); - } - - @JsonIgnore - public boolean isValue() { - return Objects.equals(handleKeyValueType, HandleKeyValueType.VALUE); - } - - ColumnMetadata getColumnMetadata() { - return new PulsarColumnMetadata(name, type, null, null, hidden, - internal, handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo( - mapping, dataFormat, formatHint)); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - PulsarColumnHandle that = (PulsarColumnHandle) o; - - if (hidden != that.hidden) { - return false; - } - if (internal != that.internal) { - return false; - } - if (connectorId != null ? !connectorId.equals(that.connectorId) : that.connectorId != null) { - return false; - } - if (name != null ? !name.equals(that.name) : that.name != null) { - return false; - } - if (type != null ? !type.equals(that.type) : that.type != null) { - return false; - } - if (mapping != null ? !mapping.equals(that.mapping) : that.mapping != null) { - return false; - } - if (dataFormat != null ? !dataFormat.equals(that.dataFormat) : that.dataFormat != null) { - return false; - } - - if (formatHint != null ? !formatHint.equals(that.formatHint) : that.formatHint != null) { - return false; - } - - return Objects.equals(handleKeyValueType, that.handleKeyValueType); - } - - @Override - public int hashCode() { - int result = connectorId != null ? connectorId.hashCode() : 0; - result = 31 * result + (name != null ? name.hashCode() : 0); - result = 31 * result + (type != null ? type.hashCode() : 0); - result = 31 * result + (hidden ? 1 : 0); - result = 31 * result + (internal ? 1 : 0); - result = 31 * result + (mapping != null ? mapping.hashCode() : 0); - result = 31 * result + (dataFormat != null ? dataFormat.hashCode() : 0); - result = 31 * result + (formatHint != null ? formatHint.hashCode() : 0); - result = 31 * result + (handleKeyValueType != null ? handleKeyValueType.hashCode() : 0); - return result; - } - - @Override - public String toString() { - return "PulsarColumnHandle{" - + "connectorId='" + connectorId + '\'' - + ", name='" + name + '\'' - + ", type=" + type - + ", hidden=" + hidden - + ", internal=" + internal - + ", mapping=" + mapping - + ", dataFormat=" + dataFormat - + ", formatHint=" + formatHint - + ", handleKeyValueType=" + handleKeyValueType - + '}'; - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java deleted file mode 100644 index e545f5d129ee1..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.type.Type; -import java.util.Objects; - -/** - * Description of the column metadata. - */ -public class PulsarColumnMetadata extends ColumnMetadata { - - private boolean isInternal; - // need this because presto ColumnMetadata saves name in lowercase - private String nameWithCase; - private PulsarColumnHandle.HandleKeyValueType handleKeyValueType; - public static final String KEY_SCHEMA_COLUMN_PREFIX = "__key."; - - private DecoderExtraInfo decoderExtraInfo; - - public PulsarColumnMetadata(String name, Type type, String comment, String extraInfo, - boolean hidden, boolean isInternal, - PulsarColumnHandle.HandleKeyValueType handleKeyValueType, - DecoderExtraInfo decoderExtraInfo) { - super(name, type, comment, extraInfo, hidden); - this.nameWithCase = name; - this.isInternal = isInternal; - this.handleKeyValueType = handleKeyValueType; - this.decoderExtraInfo = decoderExtraInfo; - } - - public DecoderExtraInfo getDecoderExtraInfo() { - return decoderExtraInfo; - } - - - public String getNameWithCase() { - return nameWithCase; - } - - public boolean isInternal() { - return isInternal; - } - - - public PulsarColumnHandle.HandleKeyValueType getHandleKeyValueType() { - return handleKeyValueType; - } - - public boolean isKey() { - return Objects.equals(handleKeyValueType, PulsarColumnHandle.HandleKeyValueType.KEY); - } - - public boolean isValue() { - return Objects.equals(handleKeyValueType, PulsarColumnHandle.HandleKeyValueType.VALUE); - } - - public static String getColumnName(PulsarColumnHandle.HandleKeyValueType handleKeyValueType, String name) { - if (Objects.equals(PulsarColumnHandle.HandleKeyValueType.KEY, handleKeyValueType)) { - return KEY_SCHEMA_COLUMN_PREFIX + name; - } - return name; - } - - @Override - public String toString() { - return "PulsarColumnMetadata{" - + "isInternal=" + isInternal - + ", nameWithCase='" + nameWithCase + '\'' - + ", handleKeyValueType=" + handleKeyValueType - + ", decoderExtraInfo=" + decoderExtraInfo.toString() - + '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - - PulsarColumnMetadata that = (PulsarColumnMetadata) o; - - if (isInternal != that.isInternal) { - return false; - } - if (!Objects.equals(nameWithCase, that.nameWithCase)) { - return false; - } - if (!Objects.equals(decoderExtraInfo, that.decoderExtraInfo)) { - return false; - } - return Objects.equals(handleKeyValueType, that.handleKeyValueType); - } - - @Override - public int hashCode() { - int result = super.hashCode(); - result = 31 * result + (isInternal ? 1 : 0); - result = 31 * result + (nameWithCase != null ? nameWithCase.hashCode() : 0); - result = 31 * result + (decoderExtraInfo != null ? decoderExtraInfo.hashCode() : 0); - result = 31 * result + (handleKeyValueType != null ? handleKeyValueType.hashCode() : 0); - return result; - } - - - /** - * Decoder extra info for {@link org.apache.pulsar.sql.presto.PulsarColumnHandle} - * used by {@link io.trino.decoder.RowDecoder}. - */ - public static class DecoderExtraInfo { - - public DecoderExtraInfo(String mapping, String dataFormat, String formatHint) { - this.mapping = mapping; - this.dataFormat = dataFormat; - this.formatHint = formatHint; - } - - public DecoderExtraInfo() {} - - //equals ColumnName in general, may used as alias or embedded field in future. - private String mapping; - //reserved dataFormat used by RowDecoder. - private String dataFormat; - //reserved formatHint used by RowDecoder. - private String formatHint; - - public String getMapping() { - return mapping; - } - - public void setMapping(String mapping) { - this.mapping = mapping; - } - - public String getDataFormat() { - return dataFormat; - } - - public void setDataFormat(String dataFormat) { - this.dataFormat = dataFormat; - } - - public String getFormatHint() { - return formatHint; - } - - public void setFormatHint(String formatHint) { - this.formatHint = formatHint; - } - - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - - DecoderExtraInfo that = (DecoderExtraInfo) o; - - if (!Objects.equals(mapping, that.mapping)) { - return false; - } - if (!Objects.equals(dataFormat, that.dataFormat)) { - return false; - } - return Objects.equals(formatHint, that.formatHint); - } - - @Override - public String toString() { - return "DecoderExtraInfo{" - + "mapping=" + mapping - + ", dataFormat=" + dataFormat - + ", formatHint=" + formatHint - + '}'; - } - - @Override - public int hashCode() { - int result = super.hashCode(); - result = 31 * result + (mapping != null ? mapping.hashCode() : 0); - result = 31 * result + (dataFormat != null ? dataFormat.hashCode() : 0); - result = 31 * result + (formatHint != null ? formatHint.hashCode() : 0); - return result; - } - - } - - -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnector.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnector.java deleted file mode 100644 index f696afedf04a6..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnector.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import static io.trino.spi.transaction.IsolationLevel.READ_COMMITTED; -import static io.trino.spi.transaction.IsolationLevel.checkConnectorSupports; -import static java.util.Objects.requireNonNull; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.log.Logger; -import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorMetadata; -import io.trino.spi.connector.Connector; -import io.trino.spi.connector.ConnectorMetadata; -import io.trino.spi.connector.ConnectorRecordSetProvider; -import io.trino.spi.connector.ConnectorSplitManager; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.transaction.IsolationLevel; -import javax.inject.Inject; - -/** - * This file contains implementation of the connector to the Presto engine. - */ -public class PulsarConnector implements Connector { - - private static final Logger log = Logger.get(PulsarConnector.class); - - private final LifeCycleManager lifeCycleManager; - private final PulsarMetadata metadata; - private final PulsarSplitManager splitManager; - private final PulsarRecordSetProvider recordSetProvider; - private final PulsarConnectorConfig pulsarConnectorConfig; - - @Inject - public PulsarConnector( - LifeCycleManager lifeCycleManager, - PulsarMetadata metadata, - PulsarSplitManager splitManager, - PulsarRecordSetProvider recordSetProvider, - PulsarConnectorConfig pulsarConnectorConfig - ) { - this.lifeCycleManager = requireNonNull(lifeCycleManager, "lifeCycleManager is null"); - this.metadata = requireNonNull(metadata, "metadata is null"); - this.splitManager = requireNonNull(splitManager, "splitManager is null"); - this.recordSetProvider = requireNonNull(recordSetProvider, "recordSetProvider is null"); - this.pulsarConnectorConfig = requireNonNull(pulsarConnectorConfig, "pulsarConnectorConfig is null"); - } - - @Override - public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly) { - checkConnectorSupports(READ_COMMITTED, isolationLevel); - return PulsarTransactionHandle.INSTANCE; - } - - @Override - public ConnectorMetadata getMetadata(ConnectorTransactionHandle transactionHandle) { - return new ClassLoaderSafeConnectorMetadata(metadata, getClass().getClassLoader()); - } - - @Override - public ConnectorSplitManager getSplitManager() { - return splitManager; - } - - @Override - public ConnectorRecordSetProvider getRecordSetProvider() { - return recordSetProvider; - } - - public void initConnectorCache() throws Exception { - PulsarConnectorCache.getConnectorCache(pulsarConnectorConfig); - } - - @Override - public final void shutdown() { - try { - this.pulsarConnectorConfig.close(); - } catch (Exception e) { - log.error(e, "Failed to close pulsar connector"); - } - try { - lifeCycleManager.stop(); - } catch (Exception e) { - log.error(e, "Error shutting down connector"); - } - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java deleted file mode 100644 index 20b00b59e5ab8..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import static com.google.common.base.Preconditions.checkNotNull; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; -import io.airlift.log.Logger; -import java.io.IOException; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import org.apache.bookkeeper.common.util.OrderedScheduler; -import org.apache.bookkeeper.conf.ClientConfiguration; -import org.apache.bookkeeper.mledger.LedgerOffloader; -import org.apache.bookkeeper.mledger.LedgerOffloaderFactory; -import org.apache.bookkeeper.mledger.LedgerOffloaderStats; -import org.apache.bookkeeper.mledger.ManagedLedgerConfig; -import org.apache.bookkeeper.mledger.ManagedLedgerFactory; -import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; -import org.apache.bookkeeper.mledger.impl.NullLedgerOffloader; -import org.apache.bookkeeper.mledger.offload.Offloaders; -import org.apache.bookkeeper.mledger.offload.OffloadersCache; -import org.apache.bookkeeper.stats.StatsProvider; -import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.PulsarVersion; -import org.apache.pulsar.common.naming.NamespaceName; -import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; -import org.apache.pulsar.common.protocol.Commands; -import org.apache.pulsar.metadata.api.MetadataStoreConfig; -import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; -import org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver; - -/** - * Implementation of a cache for the Pulsar connector. - */ -public class PulsarConnectorCache { - - private static final Logger log = Logger.get(PulsarConnectorCache.class); - - @VisibleForTesting - static PulsarConnectorCache instance; - - private final MetadataStoreExtended metadataStore; - private final ManagedLedgerFactory managedLedgerFactory; - - private final StatsProvider statsProvider; - private OrderedScheduler offloaderScheduler; - private final LedgerOffloaderStats offloaderStats; - private OffloadersCache offloadersCache = new OffloadersCache(); - private LedgerOffloader defaultOffloader; - private Map offloaderMap = new ConcurrentHashMap<>(); - - private static final String OFFLOADERS_DIRECTOR = "offloadersDirectory"; - private static final String MANAGED_LEDGER_OFFLOAD_DRIVER = "managedLedgerOffloadDriver"; - private static final String MANAGED_LEDGER_OFFLOAD_MAX_THREADS = "managedLedgerOffloadMaxThreads"; - - - private PulsarConnectorCache(PulsarConnectorConfig pulsarConnectorConfig) throws Exception { - this.metadataStore = MetadataStoreExtended.create(pulsarConnectorConfig.getMetadataUrl(), - MetadataStoreConfig.builder().metadataStoreName(MetadataStoreConfig.METADATA_STORE).build()); - this.managedLedgerFactory = initManagedLedgerFactory(pulsarConnectorConfig); - this.statsProvider = PulsarConnectorUtils.createInstance(pulsarConnectorConfig.getStatsProvider(), - StatsProvider.class, getClass().getClassLoader()); - - // start stats provider - ClientConfiguration clientConfiguration = new ClientConfiguration(); - - pulsarConnectorConfig.getStatsProviderConfigs().forEach(clientConfiguration::setProperty); - - this.statsProvider.start(clientConfiguration); - - this.initOffloaderScheduler(pulsarConnectorConfig.getOffloadPolices()); - - int period = pulsarConnectorConfig.getManagedLedgerStatsPeriodSeconds(); - boolean exposeTopicLevelMetrics = pulsarConnectorConfig.isExposeTopicLevelMetricsInPrometheus(); - this.offloaderStats = - LedgerOffloaderStats.create(pulsarConnectorConfig.isExposeManagedLedgerMetricsInPrometheus(), - exposeTopicLevelMetrics, offloaderScheduler, period); - - this.defaultOffloader = initManagedLedgerOffloader( - pulsarConnectorConfig.getOffloadPolices(), pulsarConnectorConfig); - } - - public static PulsarConnectorCache getConnectorCache(PulsarConnectorConfig pulsarConnectorConfig) throws Exception { - synchronized (PulsarConnectorCache.class) { - if (instance == null) { - instance = new PulsarConnectorCache(pulsarConnectorConfig); - } - } - return instance; - } - - private ManagedLedgerFactory initManagedLedgerFactory(PulsarConnectorConfig pulsarConnectorConfig) - throws Exception { - PulsarMetadataClientDriver.init(); - - ClientConfiguration bkClientConfiguration = new ClientConfiguration() - .setMetadataServiceUri("metadata-store:" + pulsarConnectorConfig.getMetadataUrl()) - .setClientTcpNoDelay(false) - .setUseV2WireProtocol(pulsarConnectorConfig.getBookkeeperUseV2Protocol()) - .setExplictLacInterval(pulsarConnectorConfig.getBookkeeperExplicitInterval()) - .setStickyReadsEnabled(false) - .setReadEntryTimeout(60) - .setThrottleValue(pulsarConnectorConfig.getBookkeeperThrottleValue()) - .setNumIOThreads(pulsarConnectorConfig.getBookkeeperNumIOThreads()) - .setNumWorkerThreads(pulsarConnectorConfig.getBookkeeperNumWorkerThreads()) - .setNettyMaxFrameSizeBytes(pulsarConnectorConfig.getMaxMessageSize() + Commands.MESSAGE_SIZE_FRAME_PADDING); - - ManagedLedgerFactoryConfig managedLedgerFactoryConfig = new ManagedLedgerFactoryConfig(); - managedLedgerFactoryConfig.setMaxCacheSize(pulsarConnectorConfig.getManagedLedgerCacheSizeMB()); - managedLedgerFactoryConfig.setNumManagedLedgerSchedulerThreads( - pulsarConnectorConfig.getManagedLedgerNumSchedulerThreads()); - return new ManagedLedgerFactoryImpl(metadataStore, bkClientConfiguration, managedLedgerFactoryConfig); - } - - public ManagedLedgerConfig getManagedLedgerConfig(NamespaceName namespaceName, OffloadPoliciesImpl offloadPolicies, - PulsarConnectorConfig pulsarConnectorConfig) { - ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); - if (offloadPolicies == null) { - managedLedgerConfig.setLedgerOffloader(this.defaultOffloader); - } else { - LedgerOffloader ledgerOffloader = offloaderMap.compute(namespaceName, - (ns, offloader) -> { - if (offloader != null && Objects.equals(offloader.getOffloadPolicies(), offloadPolicies)) { - return offloader; - } else { - if (offloader != null) { - offloader.close(); - } - return initManagedLedgerOffloader(offloadPolicies, pulsarConnectorConfig); - } - }); - managedLedgerConfig.setLedgerOffloader(ledgerOffloader); - } - return managedLedgerConfig; - } - - private void initOffloaderScheduler(OffloadPoliciesImpl offloadPolicies) { - this.offloaderScheduler = OrderedScheduler.newSchedulerBuilder() - .numThreads(offloadPolicies.getManagedLedgerOffloadMaxThreads()) - .name("pulsar-offloader").build(); - } - - private LedgerOffloader initManagedLedgerOffloader(OffloadPoliciesImpl offloadPolicies, - PulsarConnectorConfig pulsarConnectorConfig) { - - try { - if (StringUtils.isNotBlank(offloadPolicies.getManagedLedgerOffloadDriver())) { - checkNotNull(offloadPolicies.getOffloadersDirectory(), - "Offloader driver is configured to be '%s' but no offloaders directory is configured.", - offloadPolicies.getManagedLedgerOffloadDriver()); - Offloaders offloaders = offloadersCache.getOrLoadOffloaders(offloadPolicies.getOffloadersDirectory(), - pulsarConnectorConfig.getNarExtractionDirectory()); - LedgerOffloaderFactory offloaderFactory = offloaders.getOffloaderFactory( - offloadPolicies.getManagedLedgerOffloadDriver()); - - try { - return offloaderFactory.create( - offloadPolicies, - ImmutableMap.of( - LedgerOffloader.METADATA_SOFTWARE_VERSION_KEY.toLowerCase(), PulsarVersion.getVersion(), - LedgerOffloader.METADATA_SOFTWARE_GITSHA_KEY.toLowerCase(), PulsarVersion.getGitSha() - ), - this.offloaderScheduler, this.offloaderStats); - } catch (IOException ioe) { - log.error("Failed to create offloader: ", ioe); - throw new RuntimeException(ioe.getMessage(), ioe.getCause()); - } - } else { - log.info("No ledger offloader configured, using NULL instance"); - return NullLedgerOffloader.INSTANCE; - } - } catch (Throwable t) { - throw new RuntimeException(t); - } - } - - public ManagedLedgerFactory getManagedLedgerFactory() { - return managedLedgerFactory; - } - - public StatsProvider getStatsProvider() { - return statsProvider; - } - - public static void shutdown() throws Exception { - synchronized (PulsarConnectorCache.class) { - if (instance != null) { - instance.statsProvider.stop(); - instance.managedLedgerFactory.shutdown(); - instance.metadataStore.close(); - instance.offloaderScheduler.shutdown(); - instance.offloadersCache.close(); - } - } - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorConfig.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorConfig.java deleted file mode 100644 index f6907e3cba37d..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorConfig.java +++ /dev/null @@ -1,552 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import com.fasterxml.jackson.databind.ObjectMapper; -import io.airlift.configuration.Config; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; -import java.util.regex.Matcher; -import javax.validation.constraints.NotNull; -import javax.ws.rs.client.ClientBuilder; -import org.apache.bookkeeper.stats.NullStatsProvider; -import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.admin.PulsarAdminBuilder; -import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.common.naming.NamedEntity; -import org.apache.pulsar.common.nar.NarClassLoader; -import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; -import org.apache.pulsar.common.protocol.Commands; - -/** - * This object handles configuration of the Pulsar connector for the Presto engine. - */ -public class PulsarConnectorConfig implements AutoCloseable { - - private boolean hasMetadataUrl = false; - - private String brokerServiceUrl = "http://localhost:8080"; - private String brokerBinaryServiceUrl = "pulsar://localhost:6650/"; - private String webServiceUrl = ""; //leave empty - private String metadataUrl = "zk:localhost:2181"; - private int entryReadBatchSize = 100; - private int targetNumSplits = 2; - private int maxSplitMessageQueueSize = 10000; - private int maxSplitEntryQueueSize = 1000; - private long maxSplitQueueSizeBytes = -1; - private int maxMessageSize = Commands.DEFAULT_MAX_MESSAGE_SIZE; - private String statsProvider = NullStatsProvider.class.getName(); - - private Map statsProviderConfigs = new HashMap<>(); - private String authPluginClassName; - private String authParams; - private String tlsTrustCertsFilePath; - private Boolean tlsAllowInsecureConnection; - private Boolean tlsHostnameVerificationEnable; - - private boolean namespaceDelimiterRewriteEnable = false; - private String rewriteNamespaceDelimiter = "/"; - - private boolean authorizationEnabled = false; - - // --- Ledger Offloading --- - private String managedLedgerOffloadDriver = null; - private int managedLedgerOffloadMaxThreads = 2; - private String offloadersDirectory = "./offloaders"; - private Map offloaderProperties = new HashMap<>(); - - //--- Ledger metrics --- - private boolean exposeTopicLevelMetricsInPrometheus = false; - private boolean exposeManagedLedgerMetricsInPrometheus = false; - private int managedLedgerStatsPeriodSeconds = 60; - - private PulsarAdmin pulsarAdmin; - - // --- Bookkeeper - private int bookkeeperThrottleValue = 0; - private int bookkeeperNumIOThreads = 2 * Runtime.getRuntime().availableProcessors(); - private int bookkeeperNumWorkerThreads = Runtime.getRuntime().availableProcessors(); - private boolean bookkeeperUseV2Protocol = true; - private int bookkeeperExplicitInterval = 0; - - // --- ManagedLedger - private long managedLedgerCacheSizeMB = 0L; - private int managedLedgerNumSchedulerThreads = Runtime.getRuntime().availableProcessors(); - - // --- Nar extraction - private String narExtractionDirectory = NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR; - - @NotNull - public String getBrokerServiceUrl() { - if (StringUtils.isEmpty(webServiceUrl)){ - return brokerServiceUrl; - } else { - return getWebServiceUrl(); - } - } - @Config("pulsar.broker-service-url") - public PulsarConnectorConfig setBrokerServiceUrl(String brokerServiceUrl) { - this.brokerServiceUrl = brokerServiceUrl; - return this; - } - public String getBrokerBinaryServiceUrl() { - return this.brokerBinaryServiceUrl; - } - @Config("pulsar.broker-binary-service-url") - public PulsarConnectorConfig setBrokerBinaryServiceUrl(String brokerBinaryServiceUrl) { - this.brokerBinaryServiceUrl = brokerBinaryServiceUrl; - return this; - } - @Config("pulsar.web-service-url") - public PulsarConnectorConfig setWebServiceUrl(String webServiceUrl) { - this.webServiceUrl = webServiceUrl; - return this; - } - - public String getWebServiceUrl() { - return webServiceUrl; - } - - @Config("pulsar.max-message-size") - public PulsarConnectorConfig setMaxMessageSize(int maxMessageSize) { - this.maxMessageSize = maxMessageSize; - return this; - } - - public int getMaxMessageSize() { - return this.maxMessageSize; - } - - /** - * @deprecated use {@link #getMetadataUrl()} - */ - @Deprecated - @NotNull - public String getZookeeperUri() { - return getMetadataUrl(); - } - - /** - * @deprecated use {@link #setMetadataUrl(String)} - */ - @Deprecated - @Config("pulsar.zookeeper-uri") - public PulsarConnectorConfig setZookeeperUri(String zookeeperUri) { - if (hasMetadataUrl) { - return this; - } - this.metadataUrl = zookeeperUri; - return this; - } - - @NotNull - public String getMetadataUrl() { - return this.metadataUrl; - } - - @Config("pulsar.metadata-url") - public PulsarConnectorConfig setMetadataUrl(String metadataUrl) { - this.hasMetadataUrl = true; - this.metadataUrl = metadataUrl; - return this; - } - - @NotNull - public int getMaxEntryReadBatchSize() { - return this.entryReadBatchSize; - } - - @Config("pulsar.max-entry-read-batch-size") - public PulsarConnectorConfig setMaxEntryReadBatchSize(int batchSize) { - this.entryReadBatchSize = batchSize; - return this; - } - - @NotNull - public int getTargetNumSplits() { - return this.targetNumSplits; - } - - @Config("pulsar.target-num-splits") - public PulsarConnectorConfig setTargetNumSplits(int targetNumSplits) { - this.targetNumSplits = targetNumSplits; - return this; - } - - @NotNull - public int getMaxSplitMessageQueueSize() { - return this.maxSplitMessageQueueSize; - } - - @Config("pulsar.max-split-message-queue-size") - public PulsarConnectorConfig setMaxSplitMessageQueueSize(int maxSplitMessageQueueSize) { - this.maxSplitMessageQueueSize = maxSplitMessageQueueSize; - return this; - } - - @NotNull - public int getMaxSplitEntryQueueSize() { - return this.maxSplitEntryQueueSize; - } - - @Config("pulsar.max-split-entry-queue-size") - public PulsarConnectorConfig setMaxSplitEntryQueueSize(int maxSplitEntryQueueSize) { - this.maxSplitEntryQueueSize = maxSplitEntryQueueSize; - return this; - } - - @NotNull - public long getMaxSplitQueueSizeBytes() { - return this.maxSplitQueueSizeBytes; - } - - @Config("pulsar.max-split-queue-cache-size") - public PulsarConnectorConfig setMaxSplitQueueSizeBytes(long maxSplitQueueSizeBytes) { - this.maxSplitQueueSizeBytes = maxSplitQueueSizeBytes; - return this; - } - - @NotNull - public String getStatsProvider() { - return statsProvider; - } - - @Config("pulsar.stats-provider") - public PulsarConnectorConfig setStatsProvider(String statsProvider) { - this.statsProvider = statsProvider; - return this; - } - - @NotNull - public Map getStatsProviderConfigs() { - return statsProviderConfigs; - } - - @Config("pulsar.stats-provider-configs") - public PulsarConnectorConfig setStatsProviderConfigs(String statsProviderConfigs) throws IOException { - this.statsProviderConfigs = new ObjectMapper().readValue(statsProviderConfigs, Map.class); - return this; - } - - public String getRewriteNamespaceDelimiter() { - return rewriteNamespaceDelimiter; - } - - @Config("pulsar.rewrite-namespace-delimiter") - public PulsarConnectorConfig setRewriteNamespaceDelimiter(String rewriteNamespaceDelimiter) { - Matcher m = NamedEntity.NAMED_ENTITY_PATTERN.matcher(rewriteNamespaceDelimiter); - if (m.matches()) { - throw new IllegalArgumentException( - "Can't use " + rewriteNamespaceDelimiter + "as delimiter, " - + "because delimiter must contain characters which name of namespace not allowed" - ); - } - this.rewriteNamespaceDelimiter = rewriteNamespaceDelimiter; - return this; - } - - public boolean getNamespaceDelimiterRewriteEnable() { - return namespaceDelimiterRewriteEnable; - } - - @Config("pulsar.namespace-delimiter-rewrite-enable") - public PulsarConnectorConfig setNamespaceDelimiterRewriteEnable(boolean namespaceDelimiterRewriteEnable) { - this.namespaceDelimiterRewriteEnable = namespaceDelimiterRewriteEnable; - return this; - } - - public boolean getAuthorizationEnabled() { - return authorizationEnabled; - } - - @Config("pulsar.authorization-enabled") - public PulsarConnectorConfig setAuthorizationEnabled(boolean authorizationEnabled) { - this.authorizationEnabled = authorizationEnabled; - return this; - } - - // --- Ledger Offloading --- - - public int getManagedLedgerOffloadMaxThreads() { - return this.managedLedgerOffloadMaxThreads; - } - - @Config("pulsar.managed-ledger-offload-max-threads") - public PulsarConnectorConfig setManagedLedgerOffloadMaxThreads(int managedLedgerOffloadMaxThreads) - throws IOException { - this.managedLedgerOffloadMaxThreads = managedLedgerOffloadMaxThreads; - return this; - } - - public String getManagedLedgerOffloadDriver() { - return this.managedLedgerOffloadDriver; - } - - @Config("pulsar.managed-ledger-offload-driver") - public PulsarConnectorConfig setManagedLedgerOffloadDriver(String managedLedgerOffloadDriver) throws IOException { - this.managedLedgerOffloadDriver = managedLedgerOffloadDriver; - return this; - } - - public String getOffloadersDirectory() { - return this.offloadersDirectory; - } - - - @Config("pulsar.offloaders-directory") - public PulsarConnectorConfig setOffloadersDirectory(String offloadersDirectory) throws IOException { - this.offloadersDirectory = offloadersDirectory; - return this; - } - - public Map getOffloaderProperties() { - return this.offloaderProperties; - } - - @Config("pulsar.offloader-properties") - public PulsarConnectorConfig setOffloaderProperties(String offloaderProperties) throws IOException { - this.offloaderProperties = new ObjectMapper().readValue(offloaderProperties, Map.class); - return this; - } - - @Config("pulsar.expose-topic-level-metrics-in-prometheus") - public PulsarConnectorConfig setExposeTopicLevelMetricsInPrometheus(boolean exposeTopicLevelMetricsInPrometheus) { - this.exposeTopicLevelMetricsInPrometheus = exposeTopicLevelMetricsInPrometheus; - return this; - } - - public boolean isExposeTopicLevelMetricsInPrometheus() { - return exposeTopicLevelMetricsInPrometheus; - } - - @Config("pulsar.expose-managed-ledger-metrics-in-prometheus") - public PulsarConnectorConfig setExposeManagedLedgerMetricsInPrometheus( - boolean exposeManagedLedgerMetricsInPrometheus) { - this.exposeManagedLedgerMetricsInPrometheus = exposeManagedLedgerMetricsInPrometheus; - return this; - } - - public boolean isExposeManagedLedgerMetricsInPrometheus() { - return exposeManagedLedgerMetricsInPrometheus; - } - - @Config("pulsar.managed-ledger-stats-period-seconds") - public PulsarConnectorConfig setManagedLedgerStatsPeriodSeconds(int managedLedgerStatsPeriodSeconds) { - this.managedLedgerStatsPeriodSeconds = managedLedgerStatsPeriodSeconds; - return this; - } - - public int getManagedLedgerStatsPeriodSeconds() { - return managedLedgerStatsPeriodSeconds; - } - - // --- Authentication --- - - public String getAuthPlugin() { - return this.authPluginClassName; - } - - @Config("pulsar.auth-plugin") - public PulsarConnectorConfig setAuthPlugin(String authPluginClassName) throws IOException { - this.authPluginClassName = authPluginClassName; - return this; - } - - public String getAuthParams() { - return this.authParams; - } - - @Config("pulsar.auth-params") - public PulsarConnectorConfig setAuthParams(String authParams) throws IOException { - this.authParams = authParams; - return this; - } - - public Boolean isTlsAllowInsecureConnection() { - return tlsAllowInsecureConnection; - } - - @Config("pulsar.tls-allow-insecure-connection") - public PulsarConnectorConfig setTlsAllowInsecureConnection(boolean tlsAllowInsecureConnection) { - this.tlsAllowInsecureConnection = tlsAllowInsecureConnection; - return this; - } - - public Boolean isTlsHostnameVerificationEnable() { - return tlsHostnameVerificationEnable; - } - - @Config("pulsar.tls-hostname-verification-enable") - public PulsarConnectorConfig setTlsHostnameVerificationEnable(boolean tlsHostnameVerificationEnable) { - this.tlsHostnameVerificationEnable = tlsHostnameVerificationEnable; - return this; - } - - public String getTlsTrustCertsFilePath() { - return tlsTrustCertsFilePath; - } - - @Config("pulsar.tls-trust-cert-file-path") - public PulsarConnectorConfig setTlsTrustCertsFilePath(String tlsTrustCertsFilePath) { - this.tlsTrustCertsFilePath = tlsTrustCertsFilePath; - return this; - } - - // --- Bookkeeper Config --- - - public int getBookkeeperThrottleValue() { - return bookkeeperThrottleValue; - } - - @Config("pulsar.bookkeeper-throttle-value") - public PulsarConnectorConfig setBookkeeperThrottleValue(int bookkeeperThrottleValue) { - this.bookkeeperThrottleValue = bookkeeperThrottleValue; - return this; - } - - public int getBookkeeperNumIOThreads() { - return bookkeeperNumIOThreads; - } - - @Config("pulsar.bookkeeper-num-io-threads") - public PulsarConnectorConfig setBookkeeperNumIOThreads(int bookkeeperNumIOThreads) { - this.bookkeeperNumIOThreads = bookkeeperNumIOThreads; - return this; - } - - public int getBookkeeperNumWorkerThreads() { - return bookkeeperNumWorkerThreads; - } - - @Config("pulsar.bookkeeper-num-worker-threads") - public PulsarConnectorConfig setBookkeeperNumWorkerThreads(int bookkeeperNumWorkerThreads) { - this.bookkeeperNumWorkerThreads = bookkeeperNumWorkerThreads; - return this; - } - - public boolean getBookkeeperUseV2Protocol() { - return bookkeeperUseV2Protocol; - } - - @Config("pulsar.bookkeeper-use-v2-protocol") - public PulsarConnectorConfig setBookkeeperUseV2Protocol(boolean bookkeeperUseV2Protocol) { - this.bookkeeperUseV2Protocol = bookkeeperUseV2Protocol; - return this; - } - - public int getBookkeeperExplicitInterval() { - return bookkeeperExplicitInterval; - } - - @Config("pulsar.bookkeeper-explicit-interval") - public PulsarConnectorConfig setBookkeeperExplicitInterval(int bookkeeperExplicitInterval) { - this.bookkeeperExplicitInterval = bookkeeperExplicitInterval; - return this; - } - - // --- ManagedLedger - public long getManagedLedgerCacheSizeMB() { - return managedLedgerCacheSizeMB; - } - - @Config("pulsar.managed-ledger-cache-size-MB") - public PulsarConnectorConfig setManagedLedgerCacheSizeMB(int managedLedgerCacheSizeMB) { - this.managedLedgerCacheSizeMB = managedLedgerCacheSizeMB * 1024 * 1024; - return this; - } - - public int getManagedLedgerNumSchedulerThreads() { - return managedLedgerNumSchedulerThreads; - } - - @Config("pulsar.managed-ledger-num-scheduler-threads") - public PulsarConnectorConfig setManagedLedgerNumSchedulerThreads(int managedLedgerNumSchedulerThreads) { - this.managedLedgerNumSchedulerThreads = managedLedgerNumSchedulerThreads; - return this; - } - - // --- Nar extraction config - public String getNarExtractionDirectory() { - return narExtractionDirectory; - } - - @Config("pulsar.nar-extraction-directory") - public PulsarConnectorConfig setNarExtractionDirectory(String narExtractionDirectory) { - this.narExtractionDirectory = narExtractionDirectory; - return this; - } - - @NotNull - public PulsarAdmin getPulsarAdmin() throws PulsarClientException { - if (this.pulsarAdmin == null) { - PulsarAdminBuilder builder = PulsarAdmin.builder(); - - if (getAuthPlugin() != null) { - builder.authentication(getAuthPlugin(), getAuthParams()); - } - - if (isTlsAllowInsecureConnection() != null) { - builder.allowTlsInsecureConnection(isTlsAllowInsecureConnection()); - } - - if (isTlsHostnameVerificationEnable() != null) { - builder.enableTlsHostnameVerification(isTlsHostnameVerificationEnable()); - } - - if (getTlsTrustCertsFilePath() != null) { - builder.tlsTrustCertsFilePath(getTlsTrustCertsFilePath()); - } - - builder.setContextClassLoader(ClientBuilder.class.getClassLoader()); - this.pulsarAdmin = builder.serviceHttpUrl(getBrokerServiceUrl()).build(); - } - return this.pulsarAdmin; - } - - public OffloadPoliciesImpl getOffloadPolices() { - Properties offloadProperties = new Properties(); - offloadProperties.putAll(getOffloaderProperties()); - OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.create(offloadProperties); - offloadPolicies.setManagedLedgerOffloadDriver(getManagedLedgerOffloadDriver()); - offloadPolicies.setManagedLedgerOffloadMaxThreads(getManagedLedgerOffloadMaxThreads()); - offloadPolicies.setOffloadersDirectory(getOffloadersDirectory()); - return offloadPolicies; - } - - @Override - public void close() throws Exception { - this.pulsarAdmin.close(); - } - - @Override - public String toString() { - if (StringUtils.isEmpty(webServiceUrl)){ - return "PulsarConnectorConfig{" - + "brokerServiceUrl='" + brokerServiceUrl + '\'' - + '}'; - } else { - return "PulsarConnectorConfig{" - + "brokerServiceUrl='" + webServiceUrl + '\'' - + '}'; - } - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorFactory.java deleted file mode 100644 index fb80b30de04e5..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorFactory.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import static com.google.common.base.Throwables.throwIfUnchecked; -import static java.util.Objects.requireNonNull; -import com.google.inject.Injector; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonModule; -import io.airlift.log.Logger; -import io.trino.spi.connector.Connector; -import io.trino.spi.connector.ConnectorContext; -import io.trino.spi.connector.ConnectorFactory; -import io.trino.spi.connector.ConnectorHandleResolver; -import java.util.Map; - -/** - * The factory class which helps to build the presto connector. - */ -public class PulsarConnectorFactory implements ConnectorFactory { - - private static final Logger log = Logger.get(PulsarConnectorFactory.class); - - @Override - public String getName() { - return "pulsar"; - } - - @Override - public ConnectorHandleResolver getHandleResolver() { - return new PulsarHandleResolver(); - } - - @Override - public Connector create(String connectorId, Map config, ConnectorContext context) { - requireNonNull(config, "requiredConfig is null"); - if (log.isDebugEnabled()) { - log.debug("Creating Pulsar connector with configs: %s", config); - } - try { - // A plugin is not required to use Guice; it is just very convenient - Bootstrap app = new Bootstrap( - new JsonModule(), - new PulsarConnectorModule(connectorId, context.getTypeManager()) - ); - - Injector injector = app - .strictConfig() - .doNotInitializeLogging() - .setRequiredConfigurationProperties(config) - .initialize(); - - PulsarConnector connector = injector.getInstance(PulsarConnector.class); - connector.initConnectorCache(); - return connector; - } catch (Exception e) { - throwIfUnchecked(e); - throw new RuntimeException(e); - } - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorMetricsTracker.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorMetricsTracker.java deleted file mode 100644 index 12ee2da463c40..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorMetricsTracker.java +++ /dev/null @@ -1,356 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import java.util.concurrent.TimeUnit; -import org.apache.bookkeeper.stats.Counter; -import org.apache.bookkeeper.stats.NullStatsProvider; -import org.apache.bookkeeper.stats.OpStatsLogger; -import org.apache.bookkeeper.stats.StatsLogger; -import org.apache.bookkeeper.stats.StatsProvider; - -/** - * This class helps to track metrics related to the connector. - */ -public class PulsarConnectorMetricsTracker implements AutoCloseable{ - - private final StatsLogger statsLogger; - - private static final String SCOPE = "split"; - - // metric names - - // time spend on waiting to get entry from entry queue because it is empty - private static final String ENTRY_QUEUE_DEQUEUE_WAIT_TIME = "entry-queue-dequeue-wait-time"; - - // total time spent on waiting to get entry from entry queue per query - private static final String ENTRY_QUEUE_DEQUEUE_WAIT_TIME_PER_QUERY = "entry-queue-dequeue-wait-time-per-query"; - - // number of bytes read from BookKeeper - private static final String BYTES_READ = "bytes-read"; - - // total number of bytes read per query - private static final String BYTES_READ_PER_QUERY = "bytes-read-per-query"; - - // time spent on derserializing entries - private static final String ENTRY_DESERIALIZE_TIME = "entry-deserialize-time"; - - // time spent on derserializing entries per query - private static final String ENTRY_DESERIALIZE_TIME_PER_QUERY = "entry-deserialize-time_per_query"; - - // time spent on waiting for message queue enqueue because the message queue is full - private static final String MESSAGE_QUEUE_ENQUEUE_WAIT_TIME = "message-queue-enqueue-wait-time"; - - // time spent on waiting for message queue enqueue because message queue is full per query - private static final String MESSAGE_QUEUE_ENQUEUE_WAIT_TIME_PER_QUERY = "message-queue-enqueue-wait-time-per-query"; - - private static final String NUM_MESSAGES_DERSERIALIZED = "num-messages-deserialized"; - - // number of messages deserialized - public static final String NUM_MESSAGES_DERSERIALIZED_PER_ENTRY = "num-messages-deserialized-per-entry"; - - // number of messages deserialized per query - public static final String NUM_MESSAGES_DERSERIALIZED_PER_QUERY = "num-messages-deserialized-per-query"; - - // number of read attempts (fail if queues are full) - public static final String READ_ATTEMPTS = "read-attempts"; - - // number of read attempts per query - public static final String READ_ATTEMTPS_PER_QUERY = "read-attempts-per-query"; - - // latency of reads per batch - public static final String READ_LATENCY_PER_BATCH = "read-latency-per-batch"; - - // total read latency per query - public static final String READ_LATENCY_PER_QUERY = "read-latency-per-query"; - - // number of entries per batch - public static final String NUM_ENTRIES_PER_BATCH = "num-entries-per-batch"; - - // number of entries per query - public static final String NUM_ENTRIES_PER_QUERY = "num-entries-per-query"; - - // time spent on waiting to dequeue from message queue because it is empty per query - public static final String MESSAGE_QUEUE_DEQUEUE_WAIT_TIME_PER_QUERY = "message-queue-dequeue-wait-time-per-query"; - - // time spent on deserializing message to record. For example, Avro, JSON, and so on - public static final String RECORD_DESERIALIZE_TIME = "record-deserialize-time"; - - // time spent on deserializing message to record per query - private static final String RECORD_DESERIALIZE_TIME_PER_QUERY = "record-deserialize-time-per-query"; - - // Number of records deserialized - private static final String NUM_RECORD_DESERIALIZED = "num-record-deserialized"; - - private static final String TOTAL_EXECUTION_TIME = "total-execution-time"; - - // stats loggers - - private final OpStatsLogger statsLoggerEntryQueueDequeueWaitTime; - private final Counter statsLoggerBytesRead; - private final OpStatsLogger statsLoggerEntryDeserializeTime; - private final OpStatsLogger statsLoggerMessageQueueEnqueueWaitTime; - private final Counter statsLoggerNumMessagesDeserialized; - private final OpStatsLogger statsLoggerNumMessagesDeserializedPerEntry; - private final OpStatsLogger statsLoggerReadAttempts; - private final OpStatsLogger statsLoggerReadLatencyPerBatch; - private final OpStatsLogger statsLoggerNumEntriesPerBatch; - private final OpStatsLogger statsLoggerRecordDeserializeTime; - private final Counter statsLoggerNumRecordDeserialized; - private final OpStatsLogger statsLoggerTotalExecutionTime; - - // internal tracking variables - private long entryQueueDequeueWaitTimeStartTime; - private long entryQueueDequeueWaitTimeSum = 0L; - private long bytesReadSum = 0L; - private long entryDeserializeTimeStartTime; - private long entryDeserializeTimeSum = 0L; - private long messageQueueEnqueueWaitTimeStartTime; - private long messageQueueEnqueueWaitTimeSum = 0L; - private long numMessagesDerserializedSum = 0L; - private long numMessagedDerserializedPerBatch = 0L; - private long readAttemptsSuccessSum = 0L; - private long readAttemptsFailSum = 0L; - private long readLatencySuccessSum = 0L; - private long readLatencyFailSum = 0L; - private long numEntriesPerBatchSum = 0L; - private long messageQueueDequeueWaitTimeSum = 0L; - private long recordDeserializeTimeStartTime; - private long recordDeserializeTimeSum = 0L; - - public PulsarConnectorMetricsTracker(StatsProvider statsProvider) { - this.statsLogger = statsProvider instanceof NullStatsProvider - ? null : statsProvider.getStatsLogger(SCOPE); - - if (this.statsLogger != null) { - statsLoggerEntryQueueDequeueWaitTime = statsLogger.getOpStatsLogger(ENTRY_QUEUE_DEQUEUE_WAIT_TIME); - statsLoggerBytesRead = statsLogger.getCounter(BYTES_READ); - statsLoggerEntryDeserializeTime = statsLogger.getOpStatsLogger(ENTRY_DESERIALIZE_TIME); - statsLoggerMessageQueueEnqueueWaitTime = statsLogger.getOpStatsLogger(MESSAGE_QUEUE_ENQUEUE_WAIT_TIME); - statsLoggerNumMessagesDeserialized = statsLogger.getCounter(NUM_MESSAGES_DERSERIALIZED); - statsLoggerNumMessagesDeserializedPerEntry = statsLogger - .getOpStatsLogger(NUM_MESSAGES_DERSERIALIZED_PER_ENTRY); - statsLoggerReadAttempts = statsLogger.getOpStatsLogger(READ_ATTEMPTS); - statsLoggerReadLatencyPerBatch = statsLogger.getOpStatsLogger(READ_LATENCY_PER_BATCH); - statsLoggerNumEntriesPerBatch = statsLogger.getOpStatsLogger(NUM_ENTRIES_PER_BATCH); - statsLoggerRecordDeserializeTime = statsLogger.getOpStatsLogger(RECORD_DESERIALIZE_TIME); - statsLoggerNumRecordDeserialized = statsLogger.getCounter(NUM_RECORD_DESERIALIZED); - statsLoggerTotalExecutionTime = statsLogger.getOpStatsLogger(TOTAL_EXECUTION_TIME); - } else { - statsLoggerEntryQueueDequeueWaitTime = null; - statsLoggerBytesRead = null; - statsLoggerEntryDeserializeTime = null; - statsLoggerMessageQueueEnqueueWaitTime = null; - statsLoggerNumMessagesDeserialized = null; - statsLoggerNumMessagesDeserializedPerEntry = null; - statsLoggerReadAttempts = null; - statsLoggerReadLatencyPerBatch = null; - statsLoggerNumEntriesPerBatch = null; - statsLoggerRecordDeserializeTime = null; - statsLoggerNumRecordDeserialized = null; - statsLoggerTotalExecutionTime = null; - } - } - - public void start_ENTRY_QUEUE_DEQUEUE_WAIT_TIME() { - if (statsLogger != null) { - entryQueueDequeueWaitTimeStartTime = System.nanoTime(); - } - } - - public void end_ENTRY_QUEUE_DEQUEUE_WAIT_TIME() { - if (statsLogger != null) { - long time = System.nanoTime() - entryQueueDequeueWaitTimeStartTime; - entryQueueDequeueWaitTimeSum += time; - statsLoggerEntryQueueDequeueWaitTime.registerSuccessfulEvent(time, TimeUnit.NANOSECONDS); - } - } - - public void register_BYTES_READ(long bytes) { - if (statsLogger != null) { - bytesReadSum += bytes; - statsLoggerBytesRead.addCount(bytes); - } - } - - public void start_ENTRY_DESERIALIZE_TIME() { - if (statsLogger != null) { - entryDeserializeTimeStartTime = System.nanoTime(); - } - } - - public void end_ENTRY_DESERIALIZE_TIME() { - if (statsLogger != null) { - long time = System.nanoTime() - entryDeserializeTimeStartTime; - entryDeserializeTimeSum += time; - statsLoggerEntryDeserializeTime.registerSuccessfulEvent(time, TimeUnit.NANOSECONDS); - } - } - - public void start_MESSAGE_QUEUE_ENQUEUE_WAIT_TIME() { - if (statsLogger != null) { - messageQueueEnqueueWaitTimeStartTime = System.nanoTime(); - } - } - - public void end_MESSAGE_QUEUE_ENQUEUE_WAIT_TIME() { - if (statsLogger != null) { - long time = System.nanoTime() - messageQueueEnqueueWaitTimeStartTime; - messageQueueEnqueueWaitTimeSum += time; - statsLoggerMessageQueueEnqueueWaitTime.registerSuccessfulEvent(time, TimeUnit.NANOSECONDS); - } - } - - public void incr_NUM_MESSAGES_DESERIALIZED_PER_ENTRY() { - if (statsLogger != null) { - numMessagedDerserializedPerBatch++; - statsLoggerNumMessagesDeserialized.addCount(1); - } - } - - public void end_NUM_MESSAGES_DESERIALIZED_PER_ENTRY() { - if (statsLogger != null) { - numMessagesDerserializedSum += numMessagedDerserializedPerBatch; - statsLoggerNumMessagesDeserializedPerEntry.registerSuccessfulValue(numMessagedDerserializedPerBatch); - numMessagedDerserializedPerBatch = 0L; - } - } - - public void incr_READ_ATTEMPTS_SUCCESS() { - if (statsLogger != null) { - readAttemptsSuccessSum++; - statsLoggerReadAttempts.registerSuccessfulValue(1L); - } - } - - public void incr_READ_ATTEMPTS_FAIL() { - if (statsLogger != null) { - readAttemptsFailSum++; - statsLoggerReadAttempts.registerFailedValue(1L); - } - } - - public void register_READ_LATENCY_PER_BATCH_SUCCESS(long latency) { - if (statsLogger != null) { - readLatencySuccessSum += latency; - statsLoggerReadLatencyPerBatch.registerSuccessfulEvent(latency, TimeUnit.NANOSECONDS); - } - } - - public void register_READ_LATENCY_PER_BATCH_FAIL(long latency) { - if (statsLogger != null) { - readLatencyFailSum += latency; - statsLoggerReadLatencyPerBatch.registerFailedEvent(latency, TimeUnit.NANOSECONDS); - } - } - - public void incr_NUM_ENTRIES_PER_BATCH_SUCCESS(long delta) { - if (statsLogger != null) { - numEntriesPerBatchSum += delta; - statsLoggerNumEntriesPerBatch.registerSuccessfulValue(delta); - } - } - - public void incr_NUM_ENTRIES_PER_BATCH_FAIL(long delta) { - if (statsLogger != null) { - statsLoggerNumEntriesPerBatch.registerFailedValue(delta); - } - } - - public void register_MESSAGE_QUEUE_DEQUEUE_WAIT_TIME(long latency) { - if (statsLogger != null) { - messageQueueDequeueWaitTimeSum += latency; - } - } - - public void start_RECORD_DESERIALIZE_TIME() { - if (statsLogger != null) { - recordDeserializeTimeStartTime = System.nanoTime(); - } - } - - public void end_RECORD_DESERIALIZE_TIME() { - if (statsLogger != null) { - long time = System.nanoTime() - recordDeserializeTimeStartTime; - recordDeserializeTimeSum += time; - statsLoggerRecordDeserializeTime.registerSuccessfulEvent(time, TimeUnit.NANOSECONDS); - } - } - - public void incr_NUM_RECORD_DESERIALIZED() { - if (statsLogger != null) { - statsLoggerNumRecordDeserialized.addCount(1); - } - } - - public void register_TOTAL_EXECUTION_TIME(long latency) { - if (statsLogger != null) { - statsLoggerTotalExecutionTime.registerSuccessfulEvent(latency, TimeUnit.NANOSECONDS); - } - } - - @Override - public void close() { - if (statsLogger != null) { - // register total entry dequeue wait time for query - statsLogger.getOpStatsLogger(ENTRY_QUEUE_DEQUEUE_WAIT_TIME_PER_QUERY) - .registerSuccessfulEvent(entryQueueDequeueWaitTimeSum, TimeUnit.NANOSECONDS); - - //register bytes read per query - statsLogger.getOpStatsLogger(BYTES_READ_PER_QUERY) - .registerSuccessfulValue(bytesReadSum); - - // register total time spent deserializing entries for query - statsLogger.getOpStatsLogger(ENTRY_DESERIALIZE_TIME_PER_QUERY) - .registerSuccessfulEvent(entryDeserializeTimeSum, TimeUnit.NANOSECONDS); - - // register time spent waiting for message queue enqueue because message queue is full per query - statsLogger.getOpStatsLogger(MESSAGE_QUEUE_ENQUEUE_WAIT_TIME_PER_QUERY) - .registerSuccessfulEvent(messageQueueEnqueueWaitTimeSum, TimeUnit.NANOSECONDS); - - // register number of messages deserialized per query - statsLogger.getOpStatsLogger(NUM_MESSAGES_DERSERIALIZED_PER_QUERY) - .registerSuccessfulValue(numMessagesDerserializedSum); - - // register number of read attempts per query - statsLogger.getOpStatsLogger(READ_ATTEMTPS_PER_QUERY) - .registerSuccessfulValue(readAttemptsSuccessSum); - statsLogger.getOpStatsLogger(READ_ATTEMTPS_PER_QUERY) - .registerFailedValue(readAttemptsFailSum); - - // register total read latency for query - statsLogger.getOpStatsLogger(READ_LATENCY_PER_QUERY) - .registerSuccessfulEvent(readLatencySuccessSum, TimeUnit.NANOSECONDS); - statsLogger.getOpStatsLogger(READ_LATENCY_PER_QUERY) - .registerFailedEvent(readLatencyFailSum, TimeUnit.NANOSECONDS); - - // register number of entries per query - statsLogger.getOpStatsLogger(NUM_ENTRIES_PER_QUERY) - .registerSuccessfulValue(numEntriesPerBatchSum); - - // register time spent waiting to read for message queue per query - statsLogger.getOpStatsLogger(MESSAGE_QUEUE_DEQUEUE_WAIT_TIME_PER_QUERY) - .registerSuccessfulEvent(messageQueueDequeueWaitTimeSum, TimeUnit.MILLISECONDS); - - // register time spent deserializing records per query - statsLogger.getOpStatsLogger(RECORD_DESERIALIZE_TIME_PER_QUERY) - .registerSuccessfulEvent(recordDeserializeTimeSum, TimeUnit.NANOSECONDS); - } - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorModule.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorModule.java deleted file mode 100644 index ea0799ef7a695..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorModule.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.json.JsonBinder.jsonBinder; -import static java.util.Objects.requireNonNull; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer; -import com.google.inject.Binder; -import com.google.inject.Module; -import com.google.inject.Scopes; -import io.trino.decoder.DecoderModule; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeId; -import io.trino.spi.type.TypeManager; -import javax.inject.Inject; - -/** - * This class defines binding of classes in the Presto connector. - */ -public class PulsarConnectorModule implements Module { - - private final String connectorId; - private final TypeManager typeManager; - - public PulsarConnectorModule(String connectorId, TypeManager typeManager) { - this.connectorId = requireNonNull(connectorId, "connector id is null"); - this.typeManager = requireNonNull(typeManager, "typeManager is null"); - } - - @Override - public void configure(Binder binder) { - binder.bind(TypeManager.class).toInstance(typeManager); - - binder.bind(PulsarConnector.class).in(Scopes.SINGLETON); - binder.bind(PulsarConnectorId.class).toInstance(new PulsarConnectorId(connectorId)); - - binder.bind(PulsarMetadata.class).in(Scopes.SINGLETON); - binder.bind(PulsarSplitManager.class).in(Scopes.SINGLETON); - binder.bind(PulsarRecordSetProvider.class).in(Scopes.SINGLETON); - binder.bind(PulsarAuth.class).in(Scopes.SINGLETON); - - binder.bind(PulsarDispatchingRowDecoderFactory.class).in(Scopes.SINGLETON); - - configBinder(binder).bindConfig(PulsarConnectorConfig.class); - - jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class); - - binder.install(new DecoderModule()); - - } - - /** - * A wrapper to deserialize the Presto types. - */ - public static final class TypeDeserializer - extends FromStringDeserializer { - private static final long serialVersionUID = 1L; - - private final TypeManager typeManager; - - @Inject - public TypeDeserializer(TypeManager typeManager) { - super(Type.class); - this.typeManager = requireNonNull(typeManager, "typeManager is null"); - } - - @Override - protected Type _deserialize(String value, DeserializationContext context) { - return typeManager.getType(TypeId.of(value)); - } - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorUtils.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorUtils.java deleted file mode 100644 index 8401306462fbc..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorUtils.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.util.Map; -import java.util.Properties; -import org.apache.avro.Schema; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.common.naming.TopicName; - -/** - * A helper class containing repeatable logic used in the other classes. - */ -public class PulsarConnectorUtils { - - public static Schema parseSchema(String schemaJson) { - Schema.Parser parser = new Schema.Parser(); - parser.setValidateDefaults(false); - return parser.parse(schemaJson); - } - - public static boolean isPartitionedTopic(TopicName topicName, PulsarAdmin pulsarAdmin) throws PulsarAdminException { - return pulsarAdmin.topics().getPartitionedTopicMetadata(topicName.toString()).partitions > 0; - } - - /** - * Create an instance of userClassName using provided classLoader. - * This instance should implement the provided interface xface. - * - * @param userClassName user class name - * @param xface the interface that the reflected instance should implement - * @param classLoader class loader to load the class. - * @return the instance - */ - public static T createInstance(String userClassName, - Class xface, - ClassLoader classLoader) { - Class theCls; - try { - theCls = Class.forName(userClassName, true, classLoader); - } catch (ClassNotFoundException | NoClassDefFoundError cnfe) { - throw new RuntimeException("User class must be in class path", cnfe); - } - if (!xface.isAssignableFrom(theCls)) { - throw new RuntimeException(userClassName + " not " + xface.getName()); - } - Class tCls = (Class) theCls.asSubclass(xface); - try { - Constructor meth = tCls.getDeclaredConstructor(); - return meth.newInstance(); - } catch (InstantiationException ie) { - throw new RuntimeException("User class must be concrete", ie); - } catch (NoSuchMethodException e) { - throw new RuntimeException("User class must have a no-arg constructor", e); - } catch (IllegalAccessException e) { - throw new RuntimeException("User class must a public constructor", e); - } catch (InvocationTargetException e) { - throw new RuntimeException("User class constructor throws exception", e); - } - } - - public static Properties getProperties(Map configMap) { - Properties properties = new Properties(); - for (Map.Entry entry : configMap.entrySet()) { - properties.setProperty(entry.getKey(), entry.getValue()); - } - return properties; - } - - - public static String rewriteNamespaceDelimiterIfNeeded(String namespace, PulsarConnectorConfig config) { - return config.getNamespaceDelimiterRewriteEnable() - ? namespace.replace("/", config.getRewriteNamespaceDelimiter()) - : namespace; - } - - public static String restoreNamespaceDelimiterIfNeeded(String namespace, PulsarConnectorConfig config) { - return config.getNamespaceDelimiterRewriteEnable() - ? namespace.replace(config.getRewriteNamespaceDelimiter(), "/") - : namespace; - } - -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java deleted file mode 100644 index 3247249d0775c..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import static java.lang.String.format; -import com.google.inject.Inject; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.type.TypeManager; -import java.util.List; -import java.util.Set; -import java.util.function.Function; -import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.common.schema.SchemaType; -import org.apache.pulsar.sql.presto.decoder.avro.PulsarAvroRowDecoderFactory; -import org.apache.pulsar.sql.presto.decoder.json.PulsarJsonRowDecoderFactory; -import org.apache.pulsar.sql.presto.decoder.primitive.PulsarPrimitiveRowDecoderFactory; -import org.apache.pulsar.sql.presto.decoder.protobufnative.PulsarProtobufNativeRowDecoderFactory; - -/** - * dispatcher RowDecoderFactory for {@link org.apache.pulsar.common.schema.SchemaType}. - */ -@Slf4j -public class PulsarDispatchingRowDecoderFactory { - private final Function decoderFactories; - private final TypeManager typeManager; - - @Inject - public PulsarDispatchingRowDecoderFactory(TypeManager typeManager) { - this.typeManager = typeManager; - - final PulsarRowDecoderFactory avro = new PulsarAvroRowDecoderFactory(typeManager); - final PulsarRowDecoderFactory json = new PulsarJsonRowDecoderFactory(typeManager); - final PulsarRowDecoderFactory protobufNative = new PulsarProtobufNativeRowDecoderFactory(typeManager); - final PulsarRowDecoderFactory primitive = new PulsarPrimitiveRowDecoderFactory(); - this.decoderFactories = (schema) -> { - if (SchemaType.AVRO.equals(schema)) { - return avro; - } else if (SchemaType.JSON.equals(schema)) { - return json; - } else if (SchemaType.PROTOBUF_NATIVE.equals(schema)) { - return protobufNative; - } else if (schema.isPrimitive()) { - return primitive; - } else { - return null; - } - }; - } - - public PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, - Set columns) { - PulsarRowDecoderFactory rowDecoderFactory = createDecoderFactory(schemaInfo); - return rowDecoderFactory.createRowDecoder(topicName, schemaInfo, columns); - } - - public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, - PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { - PulsarRowDecoderFactory rowDecoderFactory = createDecoderFactory(schemaInfo); - return rowDecoderFactory.extractColumnMetadata(topicName, schemaInfo, handleKeyValueType); - } - - private PulsarRowDecoderFactory createDecoderFactory(SchemaInfo schemaInfo) { - PulsarRowDecoderFactory decoderFactory = decoderFactories.apply(schemaInfo.getType()); - if (decoderFactory == null) { - throw new RuntimeException(format("'%s' is unsupported type '%s'", - schemaInfo.getName(), schemaInfo.getType())); - } - return decoderFactory; - } - - public TypeManager getTypeManager() { - return typeManager; - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarFieldValueProviders.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarFieldValueProviders.java deleted file mode 100644 index b4a1409a86acf..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarFieldValueProviders.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import io.trino.decoder.FieldValueProvider; -import io.trino.spi.type.Timestamps; - -/** - * custom FieldValueProvider for Pulsar. - */ -public class PulsarFieldValueProviders { - - public static FieldValueProvider doubleValueProvider(double value) { - return new FieldValueProvider() { - @Override - public double getDouble() { - return value; - } - - @Override - public boolean isNull() { - return false; - } - }; - } - - /** - * FieldValueProvider for Time (Data, Timestamp etc.) with indicate Null instead of longValueProvider. - */ - public static FieldValueProvider timeValueProvider(long millis, boolean isNull) { - return new FieldValueProvider() { - @Override - public long getLong() { - return millis * Timestamps.MICROSECONDS_PER_MILLISECOND; - } - - @Override - public boolean isNull() { - return isNull; - } - }; - } - -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarHandleResolver.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarHandleResolver.java deleted file mode 100644 index 1f2f0d9455627..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarHandleResolver.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ConnectorHandleResolver; -import io.trino.spi.connector.ConnectorSplit; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.ConnectorTableLayoutHandle; -import io.trino.spi.connector.ConnectorTransactionHandle; - -/** - * This class helps to resolve classes for the Presto connector. - */ -public class PulsarHandleResolver implements ConnectorHandleResolver { - @Override - public Class getTableHandleClass() { - return PulsarTableHandle.class; - } - - @Override - public Class getTableLayoutHandleClass() { - return PulsarTableLayoutHandle.class; - } - - @Override - public Class getColumnHandleClass() { - return PulsarColumnHandle.class; - } - - @Override - public Class getSplitClass() { - return PulsarSplit.class; - } - - static PulsarTableHandle convertTableHandle(ConnectorTableHandle tableHandle) { - requireNonNull(tableHandle, "tableHandle is null"); - checkArgument(tableHandle instanceof PulsarTableHandle, "tableHandle is not an instance of PulsarTableHandle"); - return (PulsarTableHandle) tableHandle; - } - - static PulsarColumnHandle convertColumnHandle(ColumnHandle columnHandle) { - requireNonNull(columnHandle, "columnHandle is null"); - checkArgument(columnHandle instanceof PulsarColumnHandle, "columnHandle is not an instance of " - + "PulsarColumnHandle"); - return (PulsarColumnHandle) columnHandle; - } - - static PulsarSplit convertSplit(ConnectorSplit split) { - requireNonNull(split, "split is null"); - checkArgument(split instanceof PulsarSplit, "split is not an instance of PulsarSplit"); - return (PulsarSplit) split; - } - - static PulsarTableLayoutHandle convertLayout(ConnectorTableLayoutHandle layout) { - requireNonNull(layout, "layout is null"); - checkArgument(layout instanceof PulsarTableLayoutHandle, "layout is not an instance of " - + "PulsarTableLayoutHandle"); - return (PulsarTableLayoutHandle) layout; - } - - @Override - public Class getTransactionHandleClass() { - return PulsarTransactionHandle.class; - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java deleted file mode 100644 index a0812085f04e1..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Strings.isNullOrEmpty; -import static java.util.Objects.requireNonNull; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import io.trino.spi.type.BigintType; -import io.trino.spi.type.IntegerType; -import io.trino.spi.type.TimestampType; -import io.trino.spi.type.Type; -import io.trino.spi.type.VarcharType; -import java.util.Map; -import java.util.Set; -import java.util.function.Consumer; - -/** - * This abstract class represents internal columns. - */ -public class PulsarInternalColumn { - - - public static final PulsarInternalColumn PARTITION = new PulsarInternalColumn("__partition__", - IntegerType.INTEGER, "The partition number which the message belongs to"); - - public static final PulsarInternalColumn EVENT_TIME = new PulsarInternalColumn("__event_time__", - TimestampType.TIMESTAMP, "Application defined timestamp in milliseconds of when the event occurred"); - - public static final PulsarInternalColumn PUBLISH_TIME = new PulsarInternalColumn("__publish_time__", - TimestampType.TIMESTAMP, "The timestamp in milliseconds of when event as published"); - - public static final PulsarInternalColumn MESSAGE_ID = new PulsarInternalColumn("__message_id__", - VarcharType.VARCHAR, "The message ID of the message used to generate this row"); - - public static final PulsarInternalColumn SEQUENCE_ID = new PulsarInternalColumn("__sequence_id__", - BigintType.BIGINT, "The sequence ID of the message used to generate this row"); - - public static final PulsarInternalColumn PRODUCER_NAME = new PulsarInternalColumn("__producer_name__", - VarcharType.VARCHAR, "The name of the producer that publish the message used to generate this row"); - - public static final PulsarInternalColumn KEY = new PulsarInternalColumn("__key__", - VarcharType.VARCHAR, "The partition key for the topic"); - - public static final PulsarInternalColumn PROPERTIES = new PulsarInternalColumn("__properties__", - VarcharType.VARCHAR, "User defined properties"); - - private static Set internalFields = ImmutableSet.of(PARTITION, EVENT_TIME, PUBLISH_TIME, - MESSAGE_ID, SEQUENCE_ID, PRODUCER_NAME, KEY, PROPERTIES); - - - private final String name; - private final Type type; - private final String comment; - - PulsarInternalColumn( - String name, - Type type, - String comment) { - checkArgument(!isNullOrEmpty(name), "name is null or is empty"); - this.name = name; - this.type = requireNonNull(type, "type is null"); - this.comment = requireNonNull(comment, "comment is null"); - } - - public String getName() { - return name; - } - - public Type getType() { - return type; - } - - PulsarColumnHandle getColumnHandle(String connectorId, boolean hidden) { - return new PulsarColumnHandle(connectorId, - getName(), - getType(), - hidden, - true, getName(), null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - } - - PulsarColumnMetadata getColumnMetadata(boolean hidden) { - return new PulsarColumnMetadata(name, type, comment, null, hidden, true, - PulsarColumnHandle.HandleKeyValueType.NONE, new PulsarColumnMetadata.DecoderExtraInfo()); - } - - public static Set getInternalFields() { - return internalFields; - } - - public static Map getInternalFieldsMap() { - ImmutableMap.Builder builder = ImmutableMap.builder(); - getInternalFields().forEach(new Consumer() { - @Override - public void accept(PulsarInternalColumn pulsarInternalColumn) { - builder.put(pulsarInternalColumn.getName(), pulsarInternalColumn); - } - }); - return builder.build(); - } - -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java deleted file mode 100644 index 24ff2b295e9fa..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java +++ /dev/null @@ -1,434 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import static io.trino.spi.StandardErrorCode.NOT_FOUND; -import static io.trino.spi.StandardErrorCode.QUERY_REJECTED; -import static java.util.Objects.requireNonNull; -import static org.apache.pulsar.sql.presto.PulsarConnectorUtils.restoreNamespaceDelimiterIfNeeded; -import static org.apache.pulsar.sql.presto.PulsarConnectorUtils.rewriteNamespaceDelimiterIfNeeded; -import static org.apache.pulsar.sql.presto.PulsarHandleResolver.convertColumnHandle; -import static org.apache.pulsar.sql.presto.PulsarHandleResolver.convertTableHandle; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import io.airlift.log.Logger; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorMetadata; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.ConnectorTableLayout; -import io.trino.spi.connector.ConnectorTableLayoutHandle; -import io.trino.spi.connector.ConnectorTableLayoutResult; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.Constraint; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.connector.SchemaTablePrefix; -import io.trino.spi.connector.TableNotFoundException; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import javax.inject.Inject; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.client.impl.schema.KeyValueSchemaInfo; -import org.apache.pulsar.common.naming.TopicDomain; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.schema.KeyValue; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.common.schema.SchemaType; - -/** - * This connector helps to work with metadata. - */ -public class PulsarMetadata implements ConnectorMetadata { - - private final String connectorId; - private final PulsarAdmin pulsarAdmin; - private final PulsarConnectorConfig pulsarConnectorConfig; - - private final PulsarDispatchingRowDecoderFactory decoderFactory; - private final PulsarAuth pulsarAuth; - - private static final String INFORMATION_SCHEMA = "information_schema"; - - private static final Logger log = Logger.get(PulsarMetadata.class); - - private final LoadingCache tableNameTopicNameCache = - CacheBuilder.newBuilder() - // use a short live cache to make sure one query not get matched the topic many times and - // prevent get the wrong cache due to the topic changes in the Pulsar. - .expireAfterWrite(30, TimeUnit.SECONDS) - .build(new CacheLoader() { - @Override - public TopicName load(SchemaTableName schemaTableName) throws Exception { - return getMatchedPulsarTopic(schemaTableName); - } - }); - - @Inject - public PulsarMetadata(PulsarConnectorId connectorId, PulsarConnectorConfig pulsarConnectorConfig, - PulsarDispatchingRowDecoderFactory decoderFactory, PulsarAuth pulsarAuth) { - this.decoderFactory = decoderFactory; - this.connectorId = requireNonNull(connectorId, "connectorId is null").toString(); - this.pulsarConnectorConfig = pulsarConnectorConfig; - this.pulsarAuth = pulsarAuth; - try { - this.pulsarAdmin = pulsarConnectorConfig.getPulsarAdmin(); - } catch (PulsarClientException e) { - throw new RuntimeException(e); - } - } - - @Override - public List listSchemaNames(ConnectorSession session) { - List prestoSchemas = new LinkedList<>(); - try { - List tenants = pulsarAdmin.tenants().getTenants(); - for (String tenant : tenants) { - prestoSchemas.addAll(pulsarAdmin.namespaces().getNamespaces(tenant).stream().map(namespace -> - rewriteNamespaceDelimiterIfNeeded(namespace, pulsarConnectorConfig)).collect(Collectors.toList())); - } - } catch (PulsarAdminException e) { - if (e.getStatusCode() == 401) { - throw new TrinoException(QUERY_REJECTED, "Failed to get schemas from pulsar: Unauthorized"); - } - throw new RuntimeException("Failed to get schemas from pulsar: " - + ExceptionUtils.getRootCause(e).getLocalizedMessage(), e); - } - return prestoSchemas; - } - - @Override - public ConnectorTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName) { - TopicName topicName = getMatchedTopicName(tableName); - checkTopicAuthorization(session, topicName.toString()); - return new PulsarTableHandle( - this.connectorId, - tableName.getSchemaName(), - tableName.getTableName(), - topicName.getLocalName()); - } - - @Override - public List getTableLayouts(ConnectorSession session, ConnectorTableHandle table, - Constraint constraint, - Optional> desiredColumns) { - - PulsarTableHandle handle = convertTableHandle(table); - ConnectorTableLayout layout = new ConnectorTableLayout( - new PulsarTableLayoutHandle(handle, constraint.getSummary())); - return ImmutableList.of(new ConnectorTableLayoutResult(layout, constraint.getSummary())); - } - - @Override - public ConnectorTableLayout getTableLayout(ConnectorSession session, ConnectorTableLayoutHandle handle) { - return new ConnectorTableLayout(handle); - } - - @Override - public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle table) { - ConnectorTableMetadata connectorTableMetadata; - SchemaTableName schemaTableName = convertTableHandle(table).toSchemaTableName(); - connectorTableMetadata = getTableMetadata(session, schemaTableName, true); - if (connectorTableMetadata == null) { - ImmutableList.Builder builder = ImmutableList.builder(); - connectorTableMetadata = new ConnectorTableMetadata(schemaTableName, builder.build()); - } - return connectorTableMetadata; - } - - @Override - public List listTables(ConnectorSession session, Optional schemaName) { - ImmutableList.Builder builder = ImmutableList.builder(); - - if (schemaName.isPresent()) { - String schemaNameOrNull = schemaName.get(); - - if (schemaNameOrNull.equals(INFORMATION_SCHEMA)) { - // no-op for now but add pulsar connector specific system tables here - } else { - List pulsarTopicList = null; - try { - pulsarTopicList = this.pulsarAdmin.topics() - .getList(restoreNamespaceDelimiterIfNeeded(schemaNameOrNull, pulsarConnectorConfig), - TopicDomain.persistent); - } catch (PulsarAdminException e) { - if (e.getStatusCode() == 404) { - log.warn("Schema " + schemaNameOrNull + " does not exsit"); - return builder.build(); - } else if (e.getStatusCode() == 401) { - throw new TrinoException(QUERY_REJECTED, - String.format("Failed to get tables/topics in %s: Unauthorized", schemaNameOrNull)); - } - throw new RuntimeException("Failed to get tables/topics in " + schemaNameOrNull + ": " - + ExceptionUtils.getRootCause(e).getLocalizedMessage(), e); - } - if (pulsarTopicList != null) { - pulsarTopicList.stream() - .map(topic -> TopicName.get(topic).getPartitionedTopicName()) - .distinct() - .forEach(topic -> builder.add(new SchemaTableName(schemaNameOrNull, - TopicName.get(topic).getLocalName()))); - } - } - } - return builder.build(); - } - - @Override - public Map getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle) { - PulsarTableHandle pulsarTableHandle = convertTableHandle(tableHandle); - - ConnectorTableMetadata tableMetaData = getTableMetadata(session, pulsarTableHandle.toSchemaTableName(), false); - if (tableMetaData == null) { - return new HashMap<>(); - } - - ImmutableMap.Builder columnHandles = ImmutableMap.builder(); - - tableMetaData.getColumns().forEach(columnMetadata -> { - - PulsarColumnMetadata pulsarColumnMetadata = (PulsarColumnMetadata) columnMetadata; - - PulsarColumnHandle pulsarColumnHandle = new PulsarColumnHandle( - connectorId, - pulsarColumnMetadata.getNameWithCase(), - pulsarColumnMetadata.getType(), - pulsarColumnMetadata.isHidden(), - pulsarColumnMetadata.isInternal(), - pulsarColumnMetadata.getDecoderExtraInfo().getMapping(), - pulsarColumnMetadata.getDecoderExtraInfo().getDataFormat(), - pulsarColumnMetadata.getDecoderExtraInfo().getFormatHint(), - pulsarColumnMetadata.getHandleKeyValueType()); - - columnHandles.put( - columnMetadata.getName(), - pulsarColumnHandle); - }); - - PulsarInternalColumn.getInternalFields().forEach(pulsarInternalColumn -> { - PulsarColumnHandle pulsarColumnHandle = pulsarInternalColumn.getColumnHandle(connectorId, false); - columnHandles.put(pulsarColumnHandle.getName(), pulsarColumnHandle); - }); - - return columnHandles.build(); - } - - @Override - public ColumnMetadata getColumnMetadata(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle - columnHandle) { - PulsarTableHandle handle = convertTableHandle(tableHandle); - return convertColumnHandle(columnHandle).getColumnMetadata(); - } - - @Override - public Map> listTableColumns(ConnectorSession session, SchemaTablePrefix - prefix) { - - requireNonNull(prefix, "prefix is null"); - - ImmutableMap.Builder> columns = ImmutableMap.builder(); - - List tableNames; - if (!prefix.getTable().isPresent()) { - tableNames = listTables(session, prefix.getSchema()); - } else { - tableNames = ImmutableList.of(new SchemaTableName(prefix.getSchema().get(), prefix.getTable().get())); - } - - for (SchemaTableName tableName : tableNames) { - ConnectorTableMetadata connectorTableMetadata = getTableMetadata(session, tableName, true); - if (connectorTableMetadata != null) { - columns.put(tableName, connectorTableMetadata.getColumns()); - } - } - - return columns.build(); - } - - @Override - public void cleanupQuery(ConnectorSession session) { - if (pulsarConnectorConfig.getAuthorizationEnabled()) { - pulsarAuth.cleanSession(session); - } - } - - private ConnectorTableMetadata getTableMetadata(ConnectorSession session, SchemaTableName schemaTableName, - boolean withInternalColumns) { - - if (schemaTableName.getSchemaName().equals(INFORMATION_SCHEMA)) { - return null; - } - - TopicName topicName = getMatchedTopicName(schemaTableName); - - checkTopicAuthorization(session, topicName.toString()); - - SchemaInfo schemaInfo; - try { - schemaInfo = this.pulsarAdmin.schemas().getSchemaInfo(topicName.getSchemaName()); - } catch (PulsarAdminException e) { - if (e.getStatusCode() == 404) { - // use default schema because there is no schema - schemaInfo = PulsarSqlSchemaInfoProvider.defaultSchema(); - - } else if (e.getStatusCode() == 401) { - throw new TrinoException(QUERY_REJECTED, - String.format("Failed to get pulsar topic schema information for topic %s: Unauthorized", - topicName)); - } else { - throw new RuntimeException("Failed to get pulsar topic schema information for topic " - + topicName + ": " + ExceptionUtils.getRootCause(e).getLocalizedMessage(), e); - } - } - List handles = getPulsarColumns( - topicName, schemaInfo, withInternalColumns, PulsarColumnHandle.HandleKeyValueType.NONE - ); - - - return new ConnectorTableMetadata(schemaTableName, handles); - } - - /** - * Convert pulsar schema into presto table metadata. - */ - @VisibleForTesting - public List getPulsarColumns(TopicName topicName, - SchemaInfo schemaInfo, - boolean withInternalColumns, - PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { - SchemaType schemaType = schemaInfo.getType(); - if (schemaType.isStruct() || schemaType.isPrimitive()) { - return getPulsarColumnsFromSchema(topicName, schemaInfo, withInternalColumns, handleKeyValueType); - } else if (schemaType.equals(SchemaType.KEY_VALUE)) { - return getPulsarColumnsFromKeyValueSchema(topicName, schemaInfo, withInternalColumns); - } else { - throw new IllegalArgumentException("Unsupported schema : " + schemaInfo); - } - } - - List getPulsarColumnsFromSchema(TopicName topicName, - SchemaInfo schemaInfo, - boolean withInternalColumns, - PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { - ImmutableList.Builder builder = ImmutableList.builder(); - builder.addAll(decoderFactory.extractColumnMetadata(topicName, schemaInfo, handleKeyValueType)); - if (withInternalColumns) { - PulsarInternalColumn.getInternalFields() - .stream() - .forEach(pulsarInternalColumn -> builder.add(pulsarInternalColumn.getColumnMetadata(false))); - } - return builder.build(); - } - - List getPulsarColumnsFromKeyValueSchema(TopicName topicName, - SchemaInfo schemaInfo, - boolean withInternalColumns) { - ImmutableList.Builder builder = ImmutableList.builder(); - KeyValue kvSchemaInfo = KeyValueSchemaInfo.decodeKeyValueSchemaInfo(schemaInfo); - SchemaInfo keySchemaInfo = kvSchemaInfo.getKey(); - List keyColumnMetadataList = getPulsarColumns(topicName, keySchemaInfo, false, - PulsarColumnHandle.HandleKeyValueType.KEY); - builder.addAll(keyColumnMetadataList); - - SchemaInfo valueSchemaInfo = kvSchemaInfo.getValue(); - List valueColumnMetadataList = getPulsarColumns(topicName, valueSchemaInfo, false, - PulsarColumnHandle.HandleKeyValueType.VALUE); - builder.addAll(valueColumnMetadataList); - - if (withInternalColumns) { - PulsarInternalColumn.getInternalFields() - .forEach(pulsarInternalColumn -> builder.add(pulsarInternalColumn.getColumnMetadata(false))); - } - return builder.build(); - } - - private TopicName getMatchedTopicName(SchemaTableName schemaTableName) { - TopicName topicName; - try { - topicName = tableNameTopicNameCache.get(schemaTableName); - } catch (Exception e) { - log.error(e, "Failed to get table handler for tableName " + schemaTableName); - if (e.getCause() != null && e.getCause() instanceof RuntimeException) { - throw (RuntimeException) e.getCause(); - } - throw new TableNotFoundException(schemaTableName); - } - return topicName; - } - - private TopicName getMatchedPulsarTopic(SchemaTableName schemaTableName) { - String namespace = restoreNamespaceDelimiterIfNeeded(schemaTableName.getSchemaName(), pulsarConnectorConfig); - - Set topicsSetWithoutPartition; - try { - List allTopics = this.pulsarAdmin.topics().getList(namespace, TopicDomain.persistent); - topicsSetWithoutPartition = allTopics.stream() - .map(t -> t.split(TopicName.PARTITIONED_TOPIC_SUFFIX)[0]) - .collect(Collectors.toSet()); - } catch (PulsarAdminException e) { - if (e.getStatusCode() == 404) { - throw new TrinoException(NOT_FOUND, "Schema " + namespace + " does not exist"); - } else if (e.getStatusCode() == 401) { - throw new TrinoException(QUERY_REJECTED, - String.format("Failed to get topics in schema %s: Unauthorized", namespace)); - } - throw new RuntimeException("Failed to get topics in schema " + namespace - + ": " + ExceptionUtils.getRootCause(e).getLocalizedMessage(), e); - } - - List matchedTopics = topicsSetWithoutPartition.stream() - .filter(t -> TopicName.get(t).getLocalName().equalsIgnoreCase(schemaTableName.getTableName())) - .collect(Collectors.toList()); - - if (matchedTopics.size() == 0) { - log.error("Table %s not found", String.format("%s/%s", namespace, schemaTableName.getTableName())); - throw new TableNotFoundException(schemaTableName); - } else if (matchedTopics.size() != 1) { - String errMsg = String.format("There are multiple topics %s matched the table name %s", - matchedTopics.toString(), - String.format("%s/%s", namespace, schemaTableName.getTableName())); - log.error(errMsg); - throw new TableNotFoundException(schemaTableName, errMsg); - } - log.info("matched topic %s for table %s ", matchedTopics.get(0), schemaTableName); - return TopicName.get(matchedTopics.get(0)); - } - - void checkTopicAuthorization(ConnectorSession session, String topic) { - if (!pulsarConnectorConfig.getAuthorizationEnabled()) { - return; - } - pulsarAuth.checkTopicAuth(session, topic); - } - -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java deleted file mode 100644 index 858624b156dbb..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java +++ /dev/null @@ -1,879 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static io.trino.decoder.FieldValueProviders.bytesValueProvider; -import static io.trino.decoder.FieldValueProviders.longValueProvider; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.annotations.VisibleForTesting; -import io.airlift.log.Logger; -import io.airlift.slice.Slice; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import io.netty.util.Recycler; -import io.netty.util.ReferenceCountUtil; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.decoder.FieldValueProvider; -import io.trino.spi.block.Block; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.RecordCursor; -import io.trino.spi.type.Type; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import org.apache.bookkeeper.mledger.AsyncCallbacks; -import org.apache.bookkeeper.mledger.Entry; -import org.apache.bookkeeper.mledger.ManagedLedgerConfig; -import org.apache.bookkeeper.mledger.ManagedLedgerException; -import org.apache.bookkeeper.mledger.ManagedLedgerFactory; -import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.ReadOnlyCursor; -import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.bookkeeper.mledger.impl.ReadOnlyCursorImpl; -import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.impl.schema.KeyValueSchemaInfo; -import org.apache.pulsar.common.api.raw.MessageParser; -import org.apache.pulsar.common.api.raw.RawMessage; -import org.apache.pulsar.common.api.raw.RawMessageIdImpl; -import org.apache.pulsar.common.api.raw.RawMessageImpl; -import org.apache.pulsar.common.naming.NamespaceName; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; -import org.apache.pulsar.common.protocol.schema.BytesSchemaVersion; -import org.apache.pulsar.common.schema.KeyValue; -import org.apache.pulsar.common.schema.KeyValueEncodingType; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.common.schema.SchemaType; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; -import org.apache.pulsar.sql.presto.util.CacheSizeAllocator; -import org.apache.pulsar.sql.presto.util.NoStrictCacheSizeAllocator; -import org.apache.pulsar.sql.presto.util.NullCacheSizeAllocator; -import org.jctools.queues.MessagePassingQueue; -import org.jctools.queues.SpscArrayQueue; - - -/** - * Implementation of a cursor to read records. - */ -public class PulsarRecordCursor implements RecordCursor { - - private List columnHandles; - private PulsarSplit pulsarSplit; - private PulsarConnectorConfig pulsarConnectorConfig; - private ReadOnlyCursor cursor; - private SpscArrayQueue messageQueue; - private CacheSizeAllocator messageQueueCacheSizeAllocator; - private SpscArrayQueue entryQueue; - private CacheSizeAllocator entryQueueCacheSizeAllocator; - private RawMessage currentMessage; - private int maxBatchSize; - private long completedBytes = 0; - private ReadEntries readEntries; - private DeserializeEntries deserializeEntries; - private TopicName topicName; - private PulsarConnectorMetricsTracker metricsTracker; - private boolean readOffloaded; - - // Stats total execution time of split - private long startTime; - - // Used to make sure we don't finish before all entries are processed since entries that have been dequeued - // but not been deserialized and added messages to the message queue can be missed if we just check if the queues - // are empty or not - private final long splitSize; - private long entriesProcessed = 0; - private int partition = -1; - private volatile Throwable deserializingError; - - private PulsarSqlSchemaInfoProvider schemaInfoProvider; - - private FieldValueProvider[] currentRowValues = null; - - PulsarDispatchingRowDecoderFactory decoderFactory; - - protected ConcurrentOpenHashMap chunkedMessagesMap = - ConcurrentOpenHashMap.newBuilder().build(); - - private static final Logger log = Logger.get(PulsarRecordCursor.class); - - public PulsarRecordCursor(List columnHandles, PulsarSplit pulsarSplit, - PulsarConnectorConfig pulsarConnectorConfig, - PulsarDispatchingRowDecoderFactory decoderFactory) { - this.splitSize = pulsarSplit.getSplitSize(); - // Set start time for split - this.startTime = System.nanoTime(); - PulsarConnectorCache pulsarConnectorCache; - try { - pulsarConnectorCache = PulsarConnectorCache.getConnectorCache(pulsarConnectorConfig); - } catch (Exception e) { - log.error(e, "Failed to initialize Pulsar connector cache"); - close(); - throw new RuntimeException(e); - } - - OffloadPoliciesImpl offloadPolicies = pulsarSplit.getOffloadPolicies(); - if (offloadPolicies != null) { - offloadPolicies.setOffloadersDirectory(pulsarConnectorConfig.getOffloadersDirectory()); - offloadPolicies.setManagedLedgerOffloadMaxThreads( - pulsarConnectorConfig.getManagedLedgerOffloadMaxThreads()); - } - initialize(columnHandles, pulsarSplit, pulsarConnectorConfig, - pulsarConnectorCache.getManagedLedgerFactory(), - pulsarConnectorCache.getManagedLedgerConfig( - TopicName.get("persistent", NamespaceName.get(pulsarSplit.getSchemaName()), - pulsarSplit.getTableName()).getNamespaceObject(), offloadPolicies, - pulsarConnectorConfig), - new PulsarConnectorMetricsTracker(pulsarConnectorCache.getStatsProvider())); - this.decoderFactory = decoderFactory; - initEntryCacheSizeAllocator(pulsarConnectorConfig); - } - - // Exposed for testing purposes - PulsarRecordCursor(List columnHandles, PulsarSplit pulsarSplit, PulsarConnectorConfig - pulsarConnectorConfig, ManagedLedgerFactory managedLedgerFactory, ManagedLedgerConfig managedLedgerConfig, - PulsarConnectorMetricsTracker pulsarConnectorMetricsTracker, - PulsarDispatchingRowDecoderFactory decoderFactory) { - this.splitSize = pulsarSplit.getSplitSize(); - initialize(columnHandles, pulsarSplit, pulsarConnectorConfig, managedLedgerFactory, managedLedgerConfig, - pulsarConnectorMetricsTracker); - this.decoderFactory = decoderFactory; - } - - private void initialize(List columnHandles, PulsarSplit pulsarSplit, PulsarConnectorConfig - pulsarConnectorConfig, ManagedLedgerFactory managedLedgerFactory, ManagedLedgerConfig managedLedgerConfig, - PulsarConnectorMetricsTracker pulsarConnectorMetricsTracker) { - this.columnHandles = columnHandles; - this.currentRowValues = new FieldValueProvider[columnHandles.size()]; - this.pulsarSplit = pulsarSplit; - this.partition = TopicName.getPartitionIndex(pulsarSplit.getTableName()); - this.pulsarConnectorConfig = pulsarConnectorConfig; - this.maxBatchSize = pulsarConnectorConfig.getMaxEntryReadBatchSize(); - this.messageQueue = new SpscArrayQueue<>(pulsarConnectorConfig.getMaxSplitMessageQueueSize()); - this.entryQueue = new SpscArrayQueue<>(pulsarConnectorConfig.getMaxSplitEntryQueueSize()); - this.topicName = TopicName.get("persistent", - NamespaceName.get(pulsarSplit.getSchemaName()), - pulsarSplit.getTableName()); - this.metricsTracker = pulsarConnectorMetricsTracker; - this.readOffloaded = pulsarConnectorConfig.getManagedLedgerOffloadDriver() != null; - this.pulsarConnectorConfig = pulsarConnectorConfig; - initEntryCacheSizeAllocator(pulsarConnectorConfig); - - try { - this.schemaInfoProvider = new PulsarSqlSchemaInfoProvider(this.topicName, - pulsarConnectorConfig.getPulsarAdmin()); - } catch (PulsarClientException e) { - log.error(e, "Failed to init Pulsar SchemaInfo Provider"); - throw new RuntimeException(e); - - } - log.info("Initializing split with parameters: %s", pulsarSplit); - - try { - this.cursor = getCursor(TopicName.get("persistent", NamespaceName.get(pulsarSplit.getSchemaName()), - pulsarSplit.getTableName()), pulsarSplit.getStartPosition(), managedLedgerFactory, managedLedgerConfig); - } catch (ManagedLedgerException | InterruptedException e) { - log.error(e, "Failed to get read only cursor"); - close(); - throw new RuntimeException(e); - } - } - - private ReadOnlyCursor getCursor(TopicName topicName, Position startPosition, ManagedLedgerFactory - managedLedgerFactory, ManagedLedgerConfig managedLedgerConfig) - throws ManagedLedgerException, InterruptedException { - - ReadOnlyCursor cursor = managedLedgerFactory.openReadOnlyCursor(topicName.getPersistenceNamingEncoding(), - startPosition, managedLedgerConfig); - - return cursor; - } - - @Override - public long getCompletedBytes() { - return this.completedBytes; - } - - @Override - public long getReadTimeNanos() { - return 0; - } - - @Override - public Type getType(int field) { - checkArgument(field < columnHandles.size(), "Invalid field index"); - return columnHandles.get(field).getType(); - } - - @VisibleForTesting - public void setPulsarSqlSchemaInfoProvider(PulsarSqlSchemaInfoProvider schemaInfoProvider) { - this.schemaInfoProvider = schemaInfoProvider; - } - - @VisibleForTesting - class DeserializeEntries extends Thread { - - private final AtomicBoolean isRunning; - - private final CompletableFuture closeHandle; - - public DeserializeEntries() { - super("deserialize-thread-split-" + pulsarSplit.getSplitId()); - this.isRunning = new AtomicBoolean(false); - this.closeHandle = new CompletableFuture<>(); - } - - @Override - public void start() { - if (isRunning.compareAndSet(false, true)) { - super.start(); - } - } - - public CompletableFuture close() { - if (isRunning.compareAndSet(true, false)) { - super.interrupt(); - } - return closeHandle; - } - - @Override - public void run() { - try { - while (isRunning.get()) { - int read = entryQueue.drain(new MessagePassingQueue.Consumer() { - @Override - public void accept(Entry entry) { - - try { - entryQueueCacheSizeAllocator.release(entry.getLength()); - - long bytes = entry.getDataBuffer().readableBytes(); - completedBytes += bytes; - // register stats for bytes read - metricsTracker.register_BYTES_READ(bytes); - - // check if we have processed all entries in this split - // and no incomplete chunked messages exist - if (entryExceedSplitEndPosition(entry) && chunkedMessagesMap.isEmpty()) { - return; - } - - // set start time for time deserializing entries for stats - metricsTracker.start_ENTRY_DESERIALIZE_TIME(); - - try { - MessageParser.parseMessage(topicName, entry.getLedgerId(), entry.getEntryId(), - entry.getDataBuffer(), (message) -> { - try { - // start time for message queue read - metricsTracker.start_MESSAGE_QUEUE_ENQUEUE_WAIT_TIME(); - - if (message.getNumChunksFromMsg() > 1) { - message = processChunkedMessages(message); - } else if (entryExceedSplitEndPosition(entry)) { - // skip no chunk or no multi chunk message - // that exceed split end position - message.release(); - message = null; - } - if (message != null) { - while (true) { - if (!haveAvailableCacheSize( - messageQueueCacheSizeAllocator, messageQueue) - || !messageQueue.offer(message)) { - Thread.sleep(1); - } else { - messageQueueCacheSizeAllocator.allocate( - message.getData().readableBytes()); - break; - } - } - } - - // stats for how long a read from message queue took - metricsTracker.end_MESSAGE_QUEUE_ENQUEUE_WAIT_TIME(); - // stats for number of messages read - metricsTracker.incr_NUM_MESSAGES_DESERIALIZED_PER_ENTRY(); - - } catch (InterruptedException e) { - //no-op - } - }, pulsarConnectorConfig.getMaxMessageSize()); - } catch (IOException e) { - log.error(e, "Failed to parse message from pulsar topic %s", topicName.toString()); - throw new RuntimeException(e); - } - // stats for time spend deserializing entries - metricsTracker.end_ENTRY_DESERIALIZE_TIME(); - - // stats for num messages per entry - metricsTracker.end_NUM_MESSAGES_DESERIALIZED_PER_ENTRY(); - - } finally { - entriesProcessed++; - entry.release(); - } - } - }); - - if (read <= 0) { - try { - Thread.sleep(1); - } catch (InterruptedException e) { - return; - } - } - } - closeHandle.complete(null); - } catch (Throwable ex) { - log.error(ex, "Stop running DeserializeEntries"); - closeHandle.completeExceptionally(ex); - throw ex; - } - } - } - - private boolean entryExceedSplitEndPosition(Entry entry) { - return ((PositionImpl) entry.getPosition()).compareTo(pulsarSplit.getEndPosition()) >= 0; - } - - @VisibleForTesting - class ReadEntries implements AsyncCallbacks.ReadEntriesCallback { - - // indicate whether there are any additional entries left to read - private boolean isDone = false; - - //num of outstanding read requests - // set to 1 because we can only read one batch a time - private final AtomicLong outstandingReadsRequests = new AtomicLong(1); - - public void run() { - - if (outstandingReadsRequests.get() > 0) { - if (!cursor.hasMoreEntries() - || (((PositionImpl) cursor.getReadPosition()).compareTo(pulsarSplit.getEndPosition()) >= 0 - && chunkedMessagesMap.isEmpty())) { - isDone = true; - - } else { - int batchSize = Math.min(maxBatchSize, entryQueue.capacity() - entryQueue.size()); - - if (batchSize > 0) { - - ReadOnlyCursorImpl readOnlyCursorImpl = ((ReadOnlyCursorImpl) cursor); - // check if ledger is offloaded - if (!readOffloaded && readOnlyCursorImpl.getCurrentLedgerInfo().hasOffloadContext()) { - log.warn( - "Ledger %s is offloaded for topic %s. Ignoring it because offloader is not configured", - readOnlyCursorImpl.getCurrentLedgerInfo().getLedgerId(), pulsarSplit.getTableName()); - - long numEntries = readOnlyCursorImpl.getCurrentLedgerInfo().getEntries(); - long entriesToSkip = - (numEntries - ((PositionImpl) cursor.getReadPosition()).getEntryId()) + 1; - cursor.skipEntries(Math.toIntExact((entriesToSkip))); - - entriesProcessed += entriesToSkip; - } else { - if (!haveAvailableCacheSize(entryQueueCacheSizeAllocator, entryQueue)) { - metricsTracker.incr_READ_ATTEMPTS_FAIL(); - return; - } - // if the available size is invalid and the entry queue size is 0, read one entry - outstandingReadsRequests.decrementAndGet(); - cursor.asyncReadEntries(batchSize, entryQueueCacheSizeAllocator.getAvailableCacheSize(), - this, System.nanoTime(), PositionImpl.LATEST); - } - - // stats for successful read request - metricsTracker.incr_READ_ATTEMPTS_SUCCESS(); - } else { - // stats for failed read request because entry queue is full - metricsTracker.incr_READ_ATTEMPTS_FAIL(); - } - } - } - } - - @Override - public void readEntriesComplete(List entries, Object ctx) { - - entryQueue.fill(new MessagePassingQueue.Supplier() { - private int i = 0; - @Override - public Entry get() { - Entry entry = entries.get(i); - i++; - entryQueueCacheSizeAllocator.allocate(entry.getLength()); - return entry; - } - }, entries.size()); - - outstandingReadsRequests.incrementAndGet(); - - //set read latency stats for success - metricsTracker.register_READ_LATENCY_PER_BATCH_SUCCESS(System.nanoTime() - (long) ctx); - //stats for number of entries read - metricsTracker.incr_NUM_ENTRIES_PER_BATCH_SUCCESS(entries.size()); - } - - public boolean hasFinished() { - return messageQueue.isEmpty() && isDone && outstandingReadsRequests.get() >= 1 - && splitSize <= entriesProcessed && chunkedMessagesMap.isEmpty(); - } - - @Override - public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { - if (log.isDebugEnabled()) { - log.debug(exception, "Failed to read entries from topic %s", topicName.toString()); - } - outstandingReadsRequests.incrementAndGet(); - - //set read latency stats for failed - metricsTracker.register_READ_LATENCY_PER_BATCH_FAIL(System.nanoTime() - (long) ctx); - //stats for number of entries read failed - metricsTracker.incr_NUM_ENTRIES_PER_BATCH_FAIL(maxBatchSize); - } - } - - /** - * Check the queue has available cache size quota or not. - * 1. If the CacheSizeAllocator is NullCacheSizeAllocator, return true. - * 2. If the available cache size > 0, return true. - * 3. If the available cache size is invalid and the queue size == 0, return true, ensure not block the query. - */ - private boolean haveAvailableCacheSize(CacheSizeAllocator cacheSizeAllocator, SpscArrayQueue queue) { - if (cacheSizeAllocator instanceof NullCacheSizeAllocator) { - return true; - } - return cacheSizeAllocator.getAvailableCacheSize() > 0 || queue.size() == 0; - } - - @Override - public boolean advanceNextPosition() { - - if (readEntries == null) { - // start deserialize thread - deserializeEntries = new DeserializeEntries(); - deserializeEntries.setUncaughtExceptionHandler((t, ex) -> { - deserializingError = ex; - }); - deserializeEntries.start(); - - readEntries = new ReadEntries(); - readEntries.run(); - } - - if (currentMessage != null) { - currentMessage.release(); - currentMessage = null; - } - - while (true) { - if (readEntries.hasFinished()) { - return false; - } - - if ((messageQueue.capacity() - messageQueue.size()) > 0) { - readEntries.run(); - } - - currentMessage = messageQueue.poll(); - if (currentMessage != null) { - messageQueueCacheSizeAllocator.release(currentMessage.getData().readableBytes()); - break; - } else if (deserializingError != null) { - throw new RuntimeException(deserializingError); - } else { - try { - Thread.sleep(1); - // stats for time spent wait to read from message queue because its empty - metricsTracker.register_MESSAGE_QUEUE_DEQUEUE_WAIT_TIME(1); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - } - - //start time for deserializing record - metricsTracker.start_RECORD_DESERIALIZE_TIME(); - - SchemaInfo schemaInfo = getSchemaInfo(pulsarSplit); - - Map currentRowValuesMap = new HashMap<>(); - - if (schemaInfo.getType().equals(SchemaType.KEY_VALUE)) { - ByteBuf keyByteBuf; - ByteBuf valueByteBuf; - - KeyValueEncodingType keyValueEncodingType = KeyValueSchemaInfo.decodeKeyValueEncodingType(schemaInfo); - if (Objects.equals(keyValueEncodingType, KeyValueEncodingType.INLINE)) { - ByteBuf dataPayload = this.currentMessage.getData(); - int keyLength = dataPayload.readInt(); - keyByteBuf = dataPayload.readSlice(keyLength); - int valueLength = dataPayload.readInt(); - valueByteBuf = dataPayload.readSlice(valueLength); - } else { - keyByteBuf = this.currentMessage.getKeyBytes().get(); - valueByteBuf = this.currentMessage.getData(); - } - - KeyValue kvSchemaInfo = KeyValueSchemaInfo.decodeKeyValueSchemaInfo(schemaInfo); - Set keyColumnHandles = columnHandles.stream() - .filter(col -> !col.isInternal()) - .filter(col -> PulsarColumnHandle.HandleKeyValueType.KEY - .equals(col.getHandleKeyValueType())) - .collect(toImmutableSet()); - PulsarRowDecoder keyDecoder = null; - if (keyColumnHandles.size() > 0) { - keyDecoder = decoderFactory.createRowDecoder(topicName, - kvSchemaInfo.getKey(), keyColumnHandles - ); - } - - Set valueColumnHandles = columnHandles.stream() - .filter(col -> !col.isInternal()) - .filter(col -> PulsarColumnHandle.HandleKeyValueType.VALUE - .equals(col.getHandleKeyValueType())) - .collect(toImmutableSet()); - PulsarRowDecoder valueDecoder = null; - if (valueColumnHandles.size() > 0) { - valueDecoder = decoderFactory.createRowDecoder(topicName, - kvSchemaInfo.getValue(), - valueColumnHandles); - } - - Optional> decodedKey; - if (keyColumnHandles.size() > 0) { - decodedKey = keyDecoder.decodeRow(keyByteBuf); - decodedKey.ifPresent(currentRowValuesMap::putAll); - } - if (valueColumnHandles.size() > 0) { - Optional> decodedValue = - valueDecoder.decodeRow(valueByteBuf); - decodedValue.ifPresent(currentRowValuesMap::putAll); - } - } else { - PulsarRowDecoder messageDecoder = decoderFactory.createRowDecoder(topicName, - schemaInfo, - columnHandles.stream() - .filter(col -> !col.isInternal()) - .filter(col -> PulsarColumnHandle.HandleKeyValueType.NONE - .equals(col.getHandleKeyValueType())) - .collect(toImmutableSet())); - - Optional> decodedValue = - messageDecoder.decodeRow(this.currentMessage.getData()); - decodedValue.ifPresent(currentRowValuesMap::putAll); - } - - for (DecoderColumnHandle columnHandle : columnHandles) { - if (columnHandle.isInternal()) { - switch (columnHandle.getName()) { - case "__partition__": - currentRowValuesMap.put(columnHandle, longValueProvider(this.partition)); - break; - case "__event_time__": - currentRowValuesMap.put(columnHandle, PulsarFieldValueProviders.timeValueProvider( - this.currentMessage.getEventTime(), this.currentMessage.getEventTime() == 0)); - break; - case "__publish_time__": - currentRowValuesMap.put(columnHandle, PulsarFieldValueProviders.timeValueProvider( - this.currentMessage.getPublishTime(), this.currentMessage.getPublishTime() == 0)); - break; - case "__message_id__": - currentRowValuesMap.put(columnHandle, bytesValueProvider( - this.currentMessage.getMessageId().toString().getBytes())); - break; - case "__sequence_id__": - currentRowValuesMap.put(columnHandle, longValueProvider(this.currentMessage.getSequenceId())); - break; - case "__producer_name__": - currentRowValuesMap.put(columnHandle, - bytesValueProvider(this.currentMessage.getProducerName().getBytes())); - break; - case "__key__": - String key = this.currentMessage.getKey().orElse(null); - currentRowValuesMap.put(columnHandle, bytesValueProvider(key == null ? null : key.getBytes())); - break; - case "__properties__": - try { - currentRowValuesMap.put(columnHandle, bytesValueProvider( - new ObjectMapper().writeValueAsBytes(this.currentMessage.getProperties()))); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - break; - default: - throw new IllegalArgumentException("unknown internal field " + columnHandle.getName()); - } - } - } - for (int i = 0; i < columnHandles.size(); i++) { - ColumnHandle columnHandle = columnHandles.get(i); - currentRowValues[i] = currentRowValuesMap.get(columnHandle); - } - - metricsTracker.incr_NUM_RECORD_DESERIALIZED(); - - // stats for time spend deserializing - metricsTracker.end_RECORD_DESERIALIZE_TIME(); - - return true; - } - - /** - * Get the schemaInfo of the message. - * - * 1. If the schema type of pulsarSplit is NONE or BYTES, use the BYTES schema. - * 2. If the schema type of pulsarSplit is BYTEBUFFER, use the BYTEBUFFER schema. - * 3. If the schema version of the message is null, use the schema info of pulsarSplit. - * 4. If the schema version of the message is not null, get the specific version schema by PulsarAdmin. - * 5. If the final schema is null throw a runtime exception. - */ - private SchemaInfo getSchemaInfo(PulsarSplit pulsarSplit) { - SchemaInfo schemaInfo = getBytesSchemaInfo(pulsarSplit.getSchemaType(), pulsarSplit.getSchemaName()); - if (schemaInfo != null) { - return schemaInfo; - } - try { - if (this.currentMessage.getSchemaVersion() == null || this.currentMessage.getSchemaVersion().length == 0) { - schemaInfo = pulsarSplit.getSchemaInfo(); - } else { - schemaInfo = schemaInfoProvider.getSchemaByVersion(this.currentMessage.getSchemaVersion()).get(); - } - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); - } - if (schemaInfo == null) { - String schemaVersion = this.currentMessage.getSchemaVersion() == null - ? "null" : BytesSchemaVersion.of(this.currentMessage.getSchemaVersion()).toString(); - throw new RuntimeException("The specific version (" + schemaVersion + ") schema of the table " - + pulsarSplit.getTableName() + " is null"); - } - return schemaInfo; - } - - private SchemaInfo getBytesSchemaInfo(SchemaType schemaType, String schemaName) { - if (!schemaType.equals(SchemaType.BYTES) && !schemaType.equals(SchemaType.NONE)) { - return null; - } - if (schemaName.equals(Schema.BYTES.getSchemaInfo().getName())) { - return Schema.BYTES.getSchemaInfo(); - } else if (schemaName.equals(Schema.BYTEBUFFER.getSchemaInfo().getName())) { - return Schema.BYTEBUFFER.getSchemaInfo(); - } else { - return Schema.BYTES.getSchemaInfo(); - } - } - - @Override - public boolean getBoolean(int field) { - return getFieldValueProvider(field, boolean.class).getBoolean(); - } - - @Override - public long getLong(int field) { - return getFieldValueProvider(field, long.class).getLong(); - } - - @Override - public double getDouble(int field) { - return getFieldValueProvider(field, double.class).getDouble(); - } - - @Override - public Slice getSlice(int field) { - return getFieldValueProvider(field, Slice.class).getSlice(); - } - - private FieldValueProvider getFieldValueProvider(int fieldIndex, Class expectedType) { - checkArgument(fieldIndex < columnHandles.size(), "Invalid field index"); - checkFieldType(fieldIndex, expectedType); - return currentRowValues[fieldIndex]; - } - - @Override - public Object getObject(int field) { - return getFieldValueProvider(field, Block.class).getBlock(); - } - - @Override - public boolean isNull(int field) { - FieldValueProvider provider = currentRowValues[field]; - return provider == null || provider.isNull(); - } - - @Override - public void close() { - log.info("Closing cursor record"); - - if (deserializeEntries != null) { - deserializeEntries.close().whenComplete((r, t) -> { - if (entryQueue != null) { - entryQueue.drain(Entry::release); - } - if (messageQueue != null) { - messageQueue.drain(RawMessage::release); - } - if (currentMessage != null) { - currentMessage.release(); - } - }); - } - - if (this.cursor != null) { - try { - this.cursor.close(); - } catch (Exception e) { - log.error(e); - } - } - - // set stat for total execution time of split - if (this.metricsTracker != null) { - this.metricsTracker.register_TOTAL_EXECUTION_TIME(System.nanoTime() - startTime); - this.metricsTracker.close(); - } - - } - - private void checkFieldType(int field, Class expected) { - Class actual = getType(field).getJavaType(); - checkArgument(actual == expected, "Expected field %s to be type %s but is %s", field, expected, actual); - } - - private void initEntryCacheSizeAllocator(PulsarConnectorConfig connectorConfig) { - if (connectorConfig.getMaxSplitQueueSizeBytes() >= 0) { - this.entryQueueCacheSizeAllocator = new NoStrictCacheSizeAllocator( - connectorConfig.getMaxSplitQueueSizeBytes() / 2); - this.messageQueueCacheSizeAllocator = new NoStrictCacheSizeAllocator( - connectorConfig.getMaxSplitQueueSizeBytes() / 2); - log.info("Init cacheSizeAllocator with maxSplitEntryQueueSizeBytes {}.", - connectorConfig.getMaxSplitQueueSizeBytes()); - } else { - this.entryQueueCacheSizeAllocator = new NullCacheSizeAllocator(); - this.messageQueueCacheSizeAllocator = new NullCacheSizeAllocator(); - log.info("Init cacheSizeAllocator with NullCacheSizeAllocator."); - } - } - - private RawMessage processChunkedMessages(RawMessage message) { - final String uuid = message.getUUID(); - final int chunkId = message.getChunkId(); - final int totalChunkMsgSize = message.getTotalChunkMsgSize(); - final int numChunks = message.getNumChunksFromMsg(); - - RawMessageIdImpl rawMessageId = (RawMessageIdImpl) message.getMessageId(); - if (rawMessageId.getLedgerId() > pulsarSplit.getEndPositionLedgerId() - && !chunkedMessagesMap.containsKey(uuid)) { - // If the message is out of the split range, we only care about the incomplete chunked messages. - message.release(); - return null; - } - if (chunkId == 0) { - ByteBuf chunkedMsgBuffer = Unpooled.directBuffer(totalChunkMsgSize, totalChunkMsgSize); - chunkedMessagesMap.computeIfAbsent(uuid, (key) -> ChunkedMessageCtx.get(numChunks, chunkedMsgBuffer)); - } - - ChunkedMessageCtx chunkedMsgCtx = chunkedMessagesMap.get(uuid); - if (chunkedMsgCtx == null || chunkedMsgCtx.chunkedMsgBuffer == null - || chunkId != (chunkedMsgCtx.lastChunkedMessageId + 1) || chunkId >= numChunks) { - // Means we lost the first chunk, it will happen when the beginning chunk didn't belong to this split. - log.info("Received unexpected chunk. messageId: %s, last-chunk-id: %s chunkId: %s, totalChunks: %s", - message.getMessageId(), - (chunkedMsgCtx != null ? chunkedMsgCtx.lastChunkedMessageId : null), chunkId, - numChunks); - if (chunkedMsgCtx != null) { - if (chunkedMsgCtx.chunkedMsgBuffer != null) { - ReferenceCountUtil.safeRelease(chunkedMsgCtx.chunkedMsgBuffer); - } - chunkedMsgCtx.recycle(); - } - chunkedMessagesMap.remove(uuid); - message.release(); - return null; - } - - // append the chunked payload and update lastChunkedMessage-id - chunkedMsgCtx.chunkedMsgBuffer.writeBytes(message.getData()); - chunkedMsgCtx.lastChunkedMessageId = chunkId; - - // if final chunk is not received yet then release payload and return - if (chunkId != (numChunks - 1)) { - message.release(); - return null; - } - - if (log.isDebugEnabled()) { - log.debug("Chunked message completed. chunkId: %s, totalChunks: %s, msgId: %s, sequenceId: %s", - chunkId, numChunks, rawMessageId, message.getSequenceId()); - } - chunkedMessagesMap.remove(uuid); - ByteBuf unCompressedPayload = chunkedMsgCtx.chunkedMsgBuffer; - chunkedMsgCtx.recycle(); - // The chunked message complete, we use the entire payload to instead of the last chunk payload. - return ((RawMessageImpl) message).updatePayloadForChunkedMessage(unCompressedPayload); - } - - static class ChunkedMessageCtx { - - protected int totalChunks = -1; - protected ByteBuf chunkedMsgBuffer; - protected int lastChunkedMessageId = -1; - - static ChunkedMessageCtx get(int numChunksFromMsg, ByteBuf chunkedMsgBuffer) { - ChunkedMessageCtx ctx = RECYCLER.get(); - ctx.totalChunks = numChunksFromMsg; - ctx.chunkedMsgBuffer = chunkedMsgBuffer; - return ctx; - } - - private final Recycler.Handle recyclerHandle; - - private ChunkedMessageCtx(Recycler.Handle recyclerHandle) { - this.recyclerHandle = recyclerHandle; - } - - private static final Recycler RECYCLER = new Recycler() { - protected ChunkedMessageCtx newObject(Recycler.Handle handle) { - return new ChunkedMessageCtx(handle); - } - }; - - public void recycle() { - this.totalChunks = -1; - this.chunkedMsgBuffer = null; - this.lastChunkedMessageId = -1; - recyclerHandle.recycle(this); - } - } - -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordSet.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordSet.java deleted file mode 100644 index 5d44c07ea7544..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordSet.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import static java.util.Objects.requireNonNull; -import com.google.common.collect.ImmutableList; -import io.trino.spi.connector.RecordCursor; -import io.trino.spi.connector.RecordSet; -import io.trino.spi.type.Type; -import java.util.List; - -/** - * Implementation of a record set. - */ -public class PulsarRecordSet implements RecordSet { - - private final List columnHandles; - private final List columnTypes; - private final PulsarSplit pulsarSplit; - private final PulsarConnectorConfig pulsarConnectorConfig; - - private PulsarDispatchingRowDecoderFactory decoderFactory; - - public PulsarRecordSet(PulsarSplit split, List columnHandles, PulsarConnectorConfig - pulsarConnectorConfig, PulsarDispatchingRowDecoderFactory decoderFactory) { - requireNonNull(split, "split is null"); - this.columnHandles = requireNonNull(columnHandles, "column handles is null"); - ImmutableList.Builder types = ImmutableList.builder(); - for (PulsarColumnHandle column : columnHandles) { - types.add(column.getType()); - } - this.columnTypes = types.build(); - - this.pulsarSplit = split; - - this.pulsarConnectorConfig = pulsarConnectorConfig; - - this.decoderFactory = decoderFactory; - } - - - @Override - public List getColumnTypes() { - return this.columnTypes; - } - - @Override - public RecordCursor cursor() { - return new PulsarRecordCursor(this.columnHandles, this.pulsarSplit, - this.pulsarConnectorConfig, this.decoderFactory); - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordSetProvider.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordSetProvider.java deleted file mode 100644 index b3de23f9fe158..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordSetProvider.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import static java.util.Objects.requireNonNull; -import com.google.common.collect.ImmutableList; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ConnectorRecordSetProvider; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorSplit; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.RecordSet; -import java.util.List; -import javax.inject.Inject; - -/** - * Implementation of the provider for record sets. - */ -public class PulsarRecordSetProvider implements ConnectorRecordSetProvider { - - private final PulsarConnectorConfig pulsarConnectorConfig; - - private final PulsarDispatchingRowDecoderFactory decoderFactory; - - @Inject - public PulsarRecordSetProvider(PulsarConnectorConfig pulsarConnectorConfig, - PulsarDispatchingRowDecoderFactory decoderFactory) { - this.decoderFactory = requireNonNull(decoderFactory, "decoderFactory is null"); - this.pulsarConnectorConfig = requireNonNull(pulsarConnectorConfig, "pulsarConnectorConfig is null"); - } - - @Override - public RecordSet getRecordSet(ConnectorTransactionHandle transactionHandle, ConnectorSession session, - ConnectorSplit split, List columns) { - - requireNonNull(split, "Connector split is null"); - PulsarSplit pulsarSplit = (PulsarSplit) split; - - ImmutableList.Builder handles = ImmutableList.builder(); - for (ColumnHandle handle : columns) { - handles.add((PulsarColumnHandle) handle); - } - - return new PulsarRecordSet(pulsarSplit, handles.build(), this.pulsarConnectorConfig, decoderFactory); - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoderFactory.java deleted file mode 100644 index 04cc61261d4d0..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoderFactory.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import io.trino.decoder.DecoderColumnHandle; -import io.trino.spi.connector.ColumnMetadata; -import java.util.List; -import java.util.Set; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.schema.SchemaInfo; - -/** - * Pulsar customized RowDecoderFactory interface. - */ -public interface PulsarRowDecoderFactory { - - /** - * extract ColumnMetadata from pulsar SchemaInfo and HandleKeyValueType. - * @param schemaInfo - * @param handleKeyValueType - * @return - */ - List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, - PulsarColumnHandle.HandleKeyValueType handleKeyValueType); - - /** - * createRowDecoder RowDecoder by pulsar SchemaInfo and column DecoderColumnHandles. - * @param schemaInfo - * @param columns - * @return - */ - PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, - Set columns); - -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplit.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplit.java deleted file mode 100644 index 1967ec5e436b6..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplit.java +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import static java.util.Objects.requireNonNull; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableList; -import io.airlift.log.Logger; -import io.trino.spi.HostAddress; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ConnectorSplit; -import io.trino.spi.predicate.TupleDomain; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.common.schema.SchemaType; - -/** - * This class represents information for a split. - */ -public class PulsarSplit implements ConnectorSplit { - - private static final Logger log = Logger.get(PulsarSplit.class); - - private final long splitId; - private final String connectorId; - private final String schemaName; - private final String originSchemaName; - private final String tableName; - private final long splitSize; - private final String schema; - private final SchemaType schemaType; - private final long startPositionEntryId; - private final long endPositionEntryId; - private final long startPositionLedgerId; - private final long endPositionLedgerId; - private final TupleDomain tupleDomain; - private final SchemaInfo schemaInfo; - - private final PositionImpl startPosition; - private final PositionImpl endPosition; - private final String schemaInfoProperties; - - private final OffloadPoliciesImpl offloadPolicies; - - @JsonCreator - public PulsarSplit( - @JsonProperty("splitId") long splitId, - @JsonProperty("connectorId") String connectorId, - @JsonProperty("schemaName") String schemaName, - @JsonProperty("originSchemaName") String originSchemaName, - @JsonProperty("tableName") String tableName, - @JsonProperty("splitSize") long splitSize, - @JsonProperty("schema") String schema, - @JsonProperty("schemaType") SchemaType schemaType, - @JsonProperty("startPositionEntryId") long startPositionEntryId, - @JsonProperty("endPositionEntryId") long endPositionEntryId, - @JsonProperty("startPositionLedgerId") long startPositionLedgerId, - @JsonProperty("endPositionLedgerId") long endPositionLedgerId, - @JsonProperty("tupleDomain") TupleDomain tupleDomain, - @JsonProperty("schemaInfoProperties") String schemaInfoProperties, - @JsonProperty("offloadPolicies") OffloadPoliciesImpl offloadPolicies) throws IOException { - this.splitId = splitId; - requireNonNull(schemaName, "schema name is null"); - this.originSchemaName = originSchemaName; - this.schemaName = requireNonNull(schemaName, "schema name is null"); - this.connectorId = requireNonNull(connectorId, "connector id is null"); - this.tableName = requireNonNull(tableName, "table name is null"); - this.splitSize = splitSize; - this.schema = schema; - this.schemaType = schemaType; - this.startPositionEntryId = startPositionEntryId; - this.endPositionEntryId = endPositionEntryId; - this.startPositionLedgerId = startPositionLedgerId; - this.endPositionLedgerId = endPositionLedgerId; - this.tupleDomain = requireNonNull(tupleDomain, "tupleDomain is null"); - this.startPosition = PositionImpl.get(startPositionLedgerId, startPositionEntryId); - this.endPosition = PositionImpl.get(endPositionLedgerId, endPositionEntryId); - this.schemaInfoProperties = schemaInfoProperties; - this.offloadPolicies = offloadPolicies; - - ObjectMapper objectMapper = new ObjectMapper(); - this.schemaInfo = SchemaInfo.builder() - .name(originSchemaName) - .type(schemaType) - .schema(schema.getBytes("ISO8859-1")) - .properties(objectMapper.readValue(schemaInfoProperties, Map.class)) - .build(); - } - - @JsonProperty - public long getSplitId() { - return splitId; - } - - @JsonProperty - public String getConnectorId() { - return connectorId; - } - - @JsonProperty - public String getSchemaName() { - return schemaName; - } - - @JsonProperty - public SchemaType getSchemaType() { - return schemaType; - } - - @JsonProperty - public String getTableName() { - return tableName; - } - - @JsonProperty - public long getSplitSize() { - return splitSize; - } - - @JsonProperty - public String getOriginSchemaName() { - return originSchemaName; - } - - @JsonProperty - public String getSchema() { - return schema; - } - - @JsonProperty - public long getStartPositionEntryId() { - return startPositionEntryId; - } - - @JsonProperty - public long getEndPositionEntryId() { - return endPositionEntryId; - } - - @JsonProperty - public long getStartPositionLedgerId() { - return startPositionLedgerId; - } - - @JsonProperty - public long getEndPositionLedgerId() { - return endPositionLedgerId; - } - - @JsonProperty - public TupleDomain getTupleDomain() { - return tupleDomain; - } - - public PositionImpl getStartPosition() { - return startPosition; - } - - public PositionImpl getEndPosition() { - return endPosition; - } - - @JsonProperty - public String getSchemaInfoProperties() { - return schemaInfoProperties; - } - - @JsonProperty - public OffloadPoliciesImpl getOffloadPolicies() { - return offloadPolicies; - } - - @Override - public boolean isRemotelyAccessible() { - return true; - } - - @Override - public List getAddresses() { - return ImmutableList.of(HostAddress.fromParts("localhost", 12345)); - } - - @Override - public Object getInfo() { - return this; - } - - @Override - public String toString() { - return "PulsarSplit{" - + "splitId=" + splitId - + ", connectorId='" + connectorId + '\'' - + ", originSchemaName='" + originSchemaName + '\'' - + ", schemaName='" + schemaName + '\'' - + ", tableName='" + tableName + '\'' - + ", splitSize=" + splitSize - + ", schema='" + schema + '\'' - + ", schemaType=" + schemaType - + ", startPositionEntryId=" + startPositionEntryId - + ", endPositionEntryId=" + endPositionEntryId - + ", startPositionLedgerId=" + startPositionLedgerId - + ", endPositionLedgerId=" + endPositionLedgerId - + ", schemaInfoProperties=" + schemaInfoProperties - + (offloadPolicies == null ? "" : offloadPolicies.toString()) - + '}'; - } - - public SchemaInfo getSchemaInfo() { - return schemaInfo; - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java deleted file mode 100644 index 464e70b18dddd..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java +++ /dev/null @@ -1,457 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import static com.google.common.base.Preconditions.checkArgument; -import static io.trino.spi.StandardErrorCode.QUERY_REJECTED; -import static java.util.Objects.requireNonNull; -import static org.apache.bookkeeper.mledger.ManagedCursor.FindPositionConstraint.SearchAllAvailableEntries; -import static org.apache.pulsar.sql.presto.PulsarConnectorUtils.restoreNamespaceDelimiterIfNeeded; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.annotations.VisibleForTesting; -import io.airlift.log.Logger; -import io.trino.spi.TrinoException; -import io.trino.spi.block.Block; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorSplitManager; -import io.trino.spi.connector.ConnectorSplitSource; -import io.trino.spi.connector.ConnectorTableLayoutHandle; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.FixedSplitSource; -import io.trino.spi.predicate.Domain; -import io.trino.spi.predicate.Range; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.predicate.Utils; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import javax.inject.Inject; -import lombok.Data; -import org.apache.bookkeeper.mledger.ManagedLedgerConfig; -import org.apache.bookkeeper.mledger.ManagedLedgerException; -import org.apache.bookkeeper.mledger.ManagedLedgerFactory; -import org.apache.bookkeeper.mledger.ReadOnlyCursor; -import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.common.naming.NamespaceName; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; -import org.apache.pulsar.common.protocol.Commands; -import org.apache.pulsar.common.schema.SchemaInfo; - -/** - * The class helping to manage splits. - */ -public class PulsarSplitManager implements ConnectorSplitManager { - - private final String connectorId; - - private final PulsarConnectorConfig pulsarConnectorConfig; - - private final PulsarAdmin pulsarAdmin; - - private static final Logger log = Logger.get(PulsarSplitManager.class); - - private final ObjectMapper objectMapper = new ObjectMapper(); - - @Inject - public PulsarSplitManager(PulsarConnectorId connectorId, PulsarConnectorConfig pulsarConnectorConfig) { - this.connectorId = requireNonNull(connectorId, "connectorId is null").toString(); - this.pulsarConnectorConfig = requireNonNull(pulsarConnectorConfig, "pulsarConnectorConfig is null"); - try { - this.pulsarAdmin = pulsarConnectorConfig.getPulsarAdmin(); - } catch (PulsarClientException e) { - log.error(e); - throw new RuntimeException(e); - } - } - - @Override - public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, - ConnectorTableLayoutHandle layout, - ConnectorSplitManager.SplitSchedulingStrategy splitSchedulingStrategy) { - - int numSplits = this.pulsarConnectorConfig.getTargetNumSplits(); - - PulsarTableLayoutHandle layoutHandle = (PulsarTableLayoutHandle) layout; - PulsarTableHandle tableHandle = layoutHandle.getTable(); - TupleDomain tupleDomain = layoutHandle.getTupleDomain(); - - String namespace = restoreNamespaceDelimiterIfNeeded(tableHandle.getSchemaName(), pulsarConnectorConfig); - TopicName topicName = TopicName.get("persistent", NamespaceName.get(namespace), tableHandle.getTopicName()); - - SchemaInfo schemaInfo; - - try { - schemaInfo = this.pulsarAdmin.schemas().getSchemaInfo( - String.format("%s/%s", namespace, tableHandle.getTopicName())); - } catch (PulsarAdminException e) { - if (e.getStatusCode() == 401) { - throw new TrinoException(QUERY_REJECTED, - String.format("Failed to get pulsar topic schema for topic %s/%s: Unauthorized", - namespace, tableHandle.getTopicName())); - } else if (e.getStatusCode() == 404) { - schemaInfo = PulsarSqlSchemaInfoProvider.defaultSchema(); - } else { - throw new RuntimeException("Failed to get pulsar topic schema for topic " - + String.format("%s/%s", namespace, tableHandle.getTopicName()) - + ": " + ExceptionUtils.getRootCause(e).getLocalizedMessage(), e); - } - } - - Collection splits; - try { - OffloadPoliciesImpl offloadPolicies = (OffloadPoliciesImpl) this.pulsarAdmin.namespaces() - .getOffloadPolicies(topicName.getNamespace()); - if (offloadPolicies != null) { - offloadPolicies.setOffloadersDirectory(pulsarConnectorConfig.getOffloadersDirectory()); - offloadPolicies.setManagedLedgerOffloadMaxThreads( - pulsarConnectorConfig.getManagedLedgerOffloadMaxThreads()); - } - if (!PulsarConnectorUtils.isPartitionedTopic(topicName, this.pulsarAdmin)) { - splits = getSplitsNonPartitionedTopic( - numSplits, topicName, tableHandle, schemaInfo, tupleDomain, offloadPolicies); - log.debug("Splits for non-partitioned topic %s: %s", topicName, splits); - } else { - splits = getSplitsPartitionedTopic( - numSplits, topicName, tableHandle, schemaInfo, tupleDomain, offloadPolicies); - log.debug("Splits for partitioned topic %s: %s", topicName, splits); - } - } catch (Exception e) { - log.error(e, "Failed to get splits"); - throw new RuntimeException(e); - } - return new FixedSplitSource(splits); - } - - @VisibleForTesting - Collection getSplitsPartitionedTopic(int numSplits, TopicName topicName, PulsarTableHandle - tableHandle, SchemaInfo schemaInfo, TupleDomain tupleDomain, - OffloadPoliciesImpl offloadPolicies) throws Exception { - - List predicatedPartitions = getPredicatedPartitions(topicName, tupleDomain); - if (log.isDebugEnabled()) { - log.debug("Partition filter result %s", predicatedPartitions); - } - - int actualNumSplits = Math.max(predicatedPartitions.size(), numSplits); - - int splitsPerPartition = actualNumSplits / predicatedPartitions.size(); - - int splitRemainder = actualNumSplits % predicatedPartitions.size(); - - PulsarConnectorCache pulsarConnectorCache = PulsarConnectorCache.getConnectorCache(pulsarConnectorConfig); - ManagedLedgerFactory managedLedgerFactory = pulsarConnectorCache.getManagedLedgerFactory(); - ManagedLedgerConfig managedLedgerConfig = pulsarConnectorCache.getManagedLedgerConfig( - topicName.getNamespaceObject(), offloadPolicies, pulsarConnectorConfig); - - List splits = new LinkedList<>(); - for (int i = 0; i < predicatedPartitions.size(); i++) { - int splitsForThisPartition = (splitRemainder > i) ? splitsPerPartition + 1 : splitsPerPartition; - splits.addAll( - getSplitsForTopic( - topicName.getPartition(predicatedPartitions.get(i)).getPersistenceNamingEncoding(), - managedLedgerFactory, - managedLedgerConfig, - splitsForThisPartition, - tableHandle, - schemaInfo, - topicName.getPartition(predicatedPartitions.get(i)).getLocalName(), - tupleDomain, - offloadPolicies)); - } - return splits; - } - - private List getPredicatedPartitions(TopicName topicName, TupleDomain tupleDomain) { - int numPartitions; - try { - numPartitions = (this.pulsarAdmin.topics().getPartitionedTopicMetadata(topicName.toString())).partitions; - } catch (PulsarAdminException e) { - if (e.getStatusCode() == 401) { - throw new TrinoException(QUERY_REJECTED, - String.format("Failed to get metadata for partitioned topic %s: Unauthorized", topicName)); - } - - throw new RuntimeException("Failed to get metadata for partitioned topic " - + topicName + ": " + ExceptionUtils.getRootCause(e).getLocalizedMessage(), e); - } - List predicatePartitions = new ArrayList<>(); - if (tupleDomain.getDomains().isPresent()) { - Domain domain = tupleDomain.getDomains().get().get(PulsarInternalColumn.PARTITION - .getColumnHandle(connectorId, false)); - if (domain != null) { - domain.getValues().getValuesProcessor().consume( - ranges -> domain.getValues().getRanges().getOrderedRanges().forEach(range -> { - int low = 0; - int high = numPartitions; - if (range.getLowValue().isPresent()) { - Block block = Utils.nativeValueToBlock(range.getType(), range.getLowBoundedValue()); - low = block.getInt(0, 0); - } - if (range.getHighValue().isPresent()) { - Block block = Utils.nativeValueToBlock(range.getType(), range.getHighBoundedValue()); - high = block.getInt(0, 0); - } - for (int i = low; i <= high; i++) { - predicatePartitions.add(i); - } - }), - discreteValues -> {}, - allOrNone -> {}); - } else { - for (int i = 0; i < numPartitions; i++) { - predicatePartitions.add(i); - } - } - } else { - for (int i = 0; i < numPartitions; i++) { - predicatePartitions.add(i); - } - } - return predicatePartitions; - } - - @VisibleForTesting - Collection getSplitsNonPartitionedTopic(int numSplits, TopicName topicName, - PulsarTableHandle tableHandle, SchemaInfo schemaInfo, TupleDomain tupleDomain, - OffloadPoliciesImpl offloadPolicies) throws Exception { - PulsarConnectorCache pulsarConnectorCache = PulsarConnectorCache.getConnectorCache(pulsarConnectorConfig); - ManagedLedgerFactory managedLedgerFactory = pulsarConnectorCache.getManagedLedgerFactory(); - ManagedLedgerConfig managedLedgerConfig = pulsarConnectorCache.getManagedLedgerConfig( - topicName.getNamespaceObject(), offloadPolicies, pulsarConnectorConfig); - - return getSplitsForTopic( - topicName.getPersistenceNamingEncoding(), - managedLedgerFactory, - managedLedgerConfig, - numSplits, - tableHandle, - schemaInfo, - topicName.getLocalName(), - tupleDomain, - offloadPolicies); - } - - @VisibleForTesting - Collection getSplitsForTopic(String topicNamePersistenceEncoding, - ManagedLedgerFactory managedLedgerFactory, - ManagedLedgerConfig managedLedgerConfig, - int numSplits, - PulsarTableHandle tableHandle, - SchemaInfo schemaInfo, String tableName, - TupleDomain tupleDomain, - OffloadPoliciesImpl offloadPolicies) - throws ManagedLedgerException, InterruptedException, IOException { - - ReadOnlyCursor readOnlyCursor = null; - try { - readOnlyCursor = managedLedgerFactory.openReadOnlyCursor( - topicNamePersistenceEncoding, - PositionImpl.EARLIEST, managedLedgerConfig); - - long numEntries = readOnlyCursor.getNumberOfEntries(); - if (numEntries <= 0) { - return Collections.emptyList(); - } - - PredicatePushdownInfo predicatePushdownInfo = PredicatePushdownInfo.getPredicatePushdownInfo( - this.connectorId, - tupleDomain, - managedLedgerFactory, - managedLedgerConfig, - topicNamePersistenceEncoding, - numEntries); - - PositionImpl initialStartPosition; - if (predicatePushdownInfo != null) { - numEntries = predicatePushdownInfo.getNumOfEntries(); - initialStartPosition = predicatePushdownInfo.getStartPosition(); - } else { - initialStartPosition = (PositionImpl) readOnlyCursor.getReadPosition(); - } - - - readOnlyCursor.close(); - readOnlyCursor = managedLedgerFactory.openReadOnlyCursor( - topicNamePersistenceEncoding, - initialStartPosition, new ManagedLedgerConfig()); - - long remainder = numEntries % numSplits; - - long avgEntriesPerSplit = numEntries / numSplits; - - List splits = new LinkedList<>(); - for (int i = 0; i < numSplits; i++) { - long entriesForSplit = (remainder > i) ? avgEntriesPerSplit + 1 : avgEntriesPerSplit; - PositionImpl startPosition = (PositionImpl) readOnlyCursor.getReadPosition(); - readOnlyCursor.skipEntries(Math.toIntExact(entriesForSplit)); - PositionImpl endPosition = (PositionImpl) readOnlyCursor.getReadPosition(); - - PulsarSplit pulsarSplit = new PulsarSplit(i, this.connectorId, - restoreNamespaceDelimiterIfNeeded(tableHandle.getSchemaName(), pulsarConnectorConfig), - schemaInfo.getName(), - tableName, - entriesForSplit, - new String(schemaInfo.getSchema(), "ISO8859-1"), - schemaInfo.getType(), - startPosition.getEntryId(), - endPosition.getEntryId(), - startPosition.getLedgerId(), - endPosition.getLedgerId(), - tupleDomain, - objectMapper.writeValueAsString(schemaInfo.getProperties()), - offloadPolicies); - splits.add(pulsarSplit); - } - return splits; - } finally { - if (readOnlyCursor != null) { - try { - readOnlyCursor.close(); - } catch (Exception e) { - log.error(e); - } - } - } - } - - @Data - private static class PredicatePushdownInfo { - private PositionImpl startPosition; - private PositionImpl endPosition; - private long numOfEntries; - - private PredicatePushdownInfo(PositionImpl startPosition, PositionImpl endPosition, long numOfEntries) { - this.startPosition = startPosition; - this.endPosition = endPosition; - this.numOfEntries = numOfEntries; - } - - public static PredicatePushdownInfo getPredicatePushdownInfo(String connectorId, - TupleDomain tupleDomain, - ManagedLedgerFactory managedLedgerFactory, - ManagedLedgerConfig managedLedgerConfig, - String topicNamePersistenceEncoding, - long totalNumEntries) throws - ManagedLedgerException, InterruptedException { - - ReadOnlyCursor readOnlyCursor = null; - try { - readOnlyCursor = managedLedgerFactory.openReadOnlyCursor( - topicNamePersistenceEncoding, - PositionImpl.EARLIEST, managedLedgerConfig); - - if (tupleDomain.getDomains().isPresent()) { - Domain domain = tupleDomain.getDomains().get().get(PulsarInternalColumn.PUBLISH_TIME - .getColumnHandle(connectorId, false)); - if (domain != null) { - // TODO support arbitrary number of ranges - // only worry about one range for now - if (domain.getValues().getRanges().getRangeCount() == 1) { - - checkArgument(domain.getType().isOrderable(), "Domain type must be orderable"); - - Long upperBoundTs = null; - Long lowerBoundTs = null; - - Range range = domain.getValues().getRanges().getOrderedRanges().get(0); - - if (!range.isHighUnbounded()) { - Block block = Utils.nativeValueToBlock(range.getType(), range.getHighBoundedValue()); - upperBoundTs = block.getLong(0, 0) / 1000; - } - - if (!range.isLowUnbounded()) { - Block block = Utils.nativeValueToBlock(range.getType(), range.getLowBoundedValue()); - lowerBoundTs = block.getLong(0, 0) / 1000; - } - - PositionImpl overallStartPos; - if (lowerBoundTs == null) { - overallStartPos = (PositionImpl) readOnlyCursor.getReadPosition(); - } else { - overallStartPos = findPosition(readOnlyCursor, lowerBoundTs); - if (overallStartPos == null) { - overallStartPos = (PositionImpl) readOnlyCursor.getReadPosition(); - } - } - - PositionImpl overallEndPos; - if (upperBoundTs == null) { - readOnlyCursor.skipEntries(Math.toIntExact(totalNumEntries)); - overallEndPos = (PositionImpl) readOnlyCursor.getReadPosition(); - } else { - overallEndPos = findPosition(readOnlyCursor, upperBoundTs); - if (overallEndPos == null) { - overallEndPos = overallStartPos; - } - } - - // Just use a close bound since presto can always filter out the extra entries even if - // the bound - // should be open or a mixture of open and closed - com.google.common.collect.Range posRange = - com.google.common.collect.Range.range(overallStartPos, - com.google.common.collect.BoundType.CLOSED, - overallEndPos, com.google.common.collect.BoundType.CLOSED); - - long numOfEntries = readOnlyCursor.getNumberOfEntries(posRange) - 1; - - PredicatePushdownInfo predicatePushdownInfo = - new PredicatePushdownInfo(overallStartPos, overallEndPos, numOfEntries); - log.debug("Predicate pushdown optimization calculated: %s", predicatePushdownInfo); - return predicatePushdownInfo; - } - } - } - } finally { - if (readOnlyCursor != null) { - readOnlyCursor.close(); - } - } - return null; - } - } - - private static PositionImpl findPosition(ReadOnlyCursor readOnlyCursor, long timestamp) throws - ManagedLedgerException, - InterruptedException { - return (PositionImpl) readOnlyCursor.findNewestMatching( - SearchAllAvailableEntries, - entry -> { - try { - long entryTimestamp = Commands.getEntryTimestamp(entry.getDataBuffer()); - return entryTimestamp <= timestamp; - } catch (Exception e) { - log.error(e, "Failed To deserialize message when finding position with error: %s", e); - } finally { - entry.release(); - } - return false; - }); - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java deleted file mode 100644 index e2d030d2d7f1b..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import static java.util.concurrent.CompletableFuture.completedFuture; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import javax.annotation.Nonnull; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.api.schema.SchemaInfoProvider; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.protocol.schema.BytesSchemaVersion; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.common.util.FutureUtil; -import org.glassfish.jersey.internal.inject.InjectionManagerFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Multi version schema info provider for Pulsar SQL leverage guava cache. - */ -public class PulsarSqlSchemaInfoProvider implements SchemaInfoProvider { - - private static final Logger LOG = LoggerFactory.getLogger(PulsarSqlSchemaInfoProvider.class); - - private final TopicName topicName; - - private final PulsarAdmin pulsarAdmin; - - private final LoadingCache> cache = CacheBuilder.newBuilder() - .maximumSize(100000) - .expireAfterAccess(30, TimeUnit.MINUTES) - .build(new CacheLoader<>() { - @Nonnull - @Override - public CompletableFuture load(@Nonnull BytesSchemaVersion schemaVersion) { - return loadSchema(schemaVersion); - } - }); - - public PulsarSqlSchemaInfoProvider(TopicName topicName, PulsarAdmin pulsarAdmin) { - this.topicName = topicName; - this.pulsarAdmin = pulsarAdmin; - } - - @Override - public CompletableFuture getSchemaByVersion(byte[] schemaVersion) { - try { - if (null == schemaVersion) { - return completedFuture(null); - } - return cache.get(BytesSchemaVersion.of(schemaVersion)); - } catch (ExecutionException e) { - LOG.error("Can't get generic schema for topic {} schema version {}", - topicName.toString(), new String(schemaVersion, StandardCharsets.UTF_8), e); - return FutureUtil.failedFuture(e.getCause()); - } - } - - @Override - public CompletableFuture getLatestSchema() { - return pulsarAdmin.schemas().getSchemaInfoAsync(topicName.toString()); - } - - @Override - public String getTopicName() { - return topicName.getLocalName(); - } - - private CompletableFuture loadSchema(BytesSchemaVersion bytesSchemaVersion) { - ClassLoader originalContextLoader = Thread.currentThread().getContextClassLoader(); - try { - Thread.currentThread().setContextClassLoader(InjectionManagerFactory.class.getClassLoader()); - long version = ByteBuffer.wrap(bytesSchemaVersion.get()).getLong(); - return pulsarAdmin.schemas().getSchemaInfoAsync(topicName.toString(), version); - } finally { - Thread.currentThread().setContextClassLoader(originalContextLoader); - } - } - - - public static SchemaInfo defaultSchema() { - return Schema.BYTES.getSchemaInfo(); - } - -} \ No newline at end of file diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarTableHandle.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarTableHandle.java deleted file mode 100644 index 167496faedfcf..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarTableHandle.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.SchemaTableName; -import java.util.Objects; - -/** - * Description of basic metadata of a table. - */ -public class PulsarTableHandle implements ConnectorTableHandle { - - /** - * Connector id. - */ - private final String connectorId; - - /** - * The schema name for this table. - */ - private final String schemaName; - - /** - * The table name used by presto. - */ - private final String tableName; - - /** - * The topic name that is read from Pulsar. - */ - private final String topicName; - - @JsonCreator - public PulsarTableHandle( - @JsonProperty("connectorId") String connectorId, - @JsonProperty("schemaName") String schemaName, - @JsonProperty("tableName") String tableName, - @JsonProperty("topicName") String topicName - ) { - this.connectorId = requireNonNull(connectorId, "connectorId is null"); - this.schemaName = requireNonNull(schemaName, "schemaName is null"); - this.tableName = requireNonNull(tableName, "tableName is null"); - this.topicName = requireNonNull(topicName, "topicName is null"); - } - - @JsonProperty - public String getConnectorId() { - return connectorId; - } - - @JsonProperty - public String getSchemaName() { - return schemaName; - } - - @JsonProperty - public String getTableName() { - return tableName; - } - - @JsonProperty - public String getTopicName() { - return topicName; - } - - public SchemaTableName toSchemaTableName() { - return new SchemaTableName(schemaName, tableName); - } - - @Override - public int hashCode() { - return Objects.hash(connectorId, schemaName, tableName, topicName); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - - PulsarTableHandle other = (PulsarTableHandle) obj; - return Objects.equals(this.connectorId, other.connectorId) - && Objects.equals(this.schemaName, other.schemaName) - && Objects.equals(this.tableName, other.tableName) - && Objects.equals(this.topicName, other.topicName); - } - - @Override - public String toString() { - return toStringHelper(this) - .add("connectorId", connectorId) - .add("schemaName", schemaName) - .add("tableName", tableName) - .add("topicName", topicName) - .toString(); - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarTableLayoutHandle.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarTableLayoutHandle.java deleted file mode 100644 index 015c6a65ee05b..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarTableLayoutHandle.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import static java.util.Objects.requireNonNull; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ConnectorTableLayoutHandle; -import io.trino.spi.predicate.TupleDomain; -import java.util.Objects; - -/** - * This class handles the table layout. - */ -public class PulsarTableLayoutHandle implements ConnectorTableLayoutHandle { - private final PulsarTableHandle table; - private final TupleDomain tupleDomain; - - @JsonCreator - public PulsarTableLayoutHandle(@JsonProperty("table") PulsarTableHandle table, - @JsonProperty("tupleDomain") TupleDomain domain) { - - this.table = requireNonNull(table, "table is null"); - this.tupleDomain = requireNonNull(domain, "tupleDomain is null"); - } - - @JsonProperty - public PulsarTableHandle getTable() { - return table; - } - - @JsonProperty - public TupleDomain getTupleDomain() { - return tupleDomain; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - PulsarTableLayoutHandle that = (PulsarTableLayoutHandle) o; - return Objects.equals(table, that.table) - && Objects.equals(tupleDomain, that.tupleDomain); - } - - @Override - public int hashCode() { - return Objects.hash(table, tupleDomain); - } - - @Override - public String toString() { - return table.toString(); - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarTopicDescription.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarTopicDescription.java deleted file mode 100644 index 8a2558e104cce..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarTopicDescription.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Strings.isNullOrEmpty; -import static java.util.Objects.requireNonNull; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * Represents the basic information about a pulsar topic. - */ -public class PulsarTopicDescription { - private final String tableName; - private final String topicName; - private final String schemaName; - - @JsonCreator - public PulsarTopicDescription( - @JsonProperty("tableName") String tableName, - @JsonProperty("schemaName") String schemaName, - @JsonProperty("topicName") String topicName) { - checkArgument(!isNullOrEmpty(tableName), "tableName is null or is empty"); - this.tableName = tableName; - this.topicName = requireNonNull(topicName, "topicName is null"); - this.schemaName = schemaName; - } - - @JsonProperty - public String getTableName() { - return tableName; - } - - @JsonProperty - public String getTopicName() { - return topicName; - } - - @JsonProperty - public String getSchemaName() { - return schemaName; - } - - @Override - public String toString() { - return toStringHelper(this) - .add("tableName", tableName) - .add("topicName", topicName) - .add("schemaName", schemaName) - .toString(); - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java deleted file mode 100644 index 73081f8948a51..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java +++ /dev/null @@ -1,427 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.decoder.avro; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.decoder.DecoderErrorCode.DECODER_CONVERSION_NOT_SUPPORTED; -import static io.trino.spi.StandardErrorCode.GENERIC_USER_ERROR; -import static io.trino.spi.type.Varchars.truncateToLength; -import static java.lang.Float.floatToIntBits; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableSet; -import io.airlift.slice.Slice; -import io.airlift.slice.Slices; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.decoder.FieldValueProvider; -import io.trino.spi.TrinoException; -import io.trino.spi.block.Block; -import io.trino.spi.block.BlockBuilder; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.BigintType; -import io.trino.spi.type.BooleanType; -import io.trino.spi.type.DateType; -import io.trino.spi.type.DecimalType; -import io.trino.spi.type.DoubleType; -import io.trino.spi.type.Int128; -import io.trino.spi.type.IntegerType; -import io.trino.spi.type.MapType; -import io.trino.spi.type.RealType; -import io.trino.spi.type.RowType; -import io.trino.spi.type.RowType.Field; -import io.trino.spi.type.SmallintType; -import io.trino.spi.type.TimeType; -import io.trino.spi.type.TimestampType; -import io.trino.spi.type.Timestamps; -import io.trino.spi.type.TinyintType; -import io.trino.spi.type.Type; -import io.trino.spi.type.VarbinaryType; -import io.trino.spi.type.VarcharType; -import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Map; -import java.util.Set; -import org.apache.avro.generic.GenericEnumSymbol; -import org.apache.avro.generic.GenericFixed; -import org.apache.avro.generic.GenericRecord; - -/** - * Copy from {@link io.trino.decoder.avro.AvroColumnDecoder} - * with A little pulsar's extensions. - * 1) support date and time types. - * {@link io.trino.spi.type.TimestampType} - * {@link io.trino.spi.type.DateType} - * {@link io.trino.spi.type.TimeType} - * 2) support {@link io.trino.spi.type.RealType}. - * 3) support {@link io.trino.spi.type.DecimalType}. - */ -public class PulsarAvroColumnDecoder { - private static final Set SUPPORTED_PRIMITIVE_TYPES = ImmutableSet.of( - BooleanType.BOOLEAN, - TinyintType.TINYINT, - SmallintType.SMALLINT, - IntegerType.INTEGER, - BigintType.BIGINT, - RealType.REAL, - DoubleType.DOUBLE, - TimestampType.TIMESTAMP_MILLIS, - DateType.DATE, - TimeType.TIME_MILLIS, - VarbinaryType.VARBINARY); - - private final Type columnType; - private final String columnMapping; - private final String columnName; - - public PulsarAvroColumnDecoder(DecoderColumnHandle columnHandle) { - try { - requireNonNull(columnHandle, "columnHandle is null"); - this.columnType = columnHandle.getType(); - this.columnMapping = columnHandle.getMapping(); - this.columnName = columnHandle.getName(); - checkArgument(!columnHandle.isInternal(), - "unexpected internal column '%s'", columnName); - checkArgument(columnHandle.getFormatHint() == null, - "unexpected format hint '%s' defined for column '%s'", columnHandle.getFormatHint(), columnName); - checkArgument(columnHandle.getDataFormat() == null, - "unexpected data format '%s' defined for column '%s'", columnHandle.getDataFormat(), columnName); - checkArgument(columnHandle.getMapping() != null, - "mapping not defined for column '%s'", columnName); - checkArgument(isSupportedType(columnType), - "Unsupported column type '%s' for column '%s'", columnType, columnName); - } catch (IllegalArgumentException e) { - throw new TrinoException(GENERIC_USER_ERROR, e); - } - } - - private boolean isSupportedType(Type type) { - if (isSupportedPrimitive(type)) { - return true; - } - - if (type instanceof ArrayType) { - checkArgument(type.getTypeParameters().size() == 1, - "expecting exactly one type parameter for array"); - return isSupportedType(type.getTypeParameters().get(0)); - } - - if (type instanceof MapType) { - List typeParameters = type.getTypeParameters(); - checkArgument(typeParameters.size() == 2, - "expecting exactly two type parameters for map"); - checkArgument(typeParameters.get(0) instanceof VarcharType, - "Unsupported column type '%s' for map key", typeParameters.get(0)); - return isSupportedType(type.getTypeParameters().get(1)); - } - - if (type instanceof RowType) { - for (Type fieldType : type.getTypeParameters()) { - if (!isSupportedType(fieldType)) { - return false; - } - } - return true; - } - return false; - } - - private boolean isSupportedPrimitive(Type type) { - return type instanceof VarcharType || type instanceof DecimalType || SUPPORTED_PRIMITIVE_TYPES.contains(type); - } - - public FieldValueProvider decodeField(GenericRecord avroRecord) { - Object avroColumnValue = locateNode(avroRecord, columnMapping); - return new ObjectValueProvider(avroColumnValue, columnType, columnName); - } - - private static Object locateNode(GenericRecord element, String columnMapping) { - Object value = element; - for (String pathElement : Splitter.on('/').omitEmptyStrings().split(columnMapping)) { - if (value == null) { - return null; - } - value = ((GenericRecord) value).get(pathElement); - } - return value; - } - - private static class ObjectValueProvider - extends FieldValueProvider { - private final Object value; - private final Type columnType; - private final String columnName; - - public ObjectValueProvider(Object value, Type columnType, String columnName) { - this.value = value; - this.columnType = columnType; - this.columnName = columnName; - } - - @Override - public boolean isNull() { - return value == null; - } - - @Override - public double getDouble() { - if (value instanceof Double || value instanceof Float) { - return ((Number) value).doubleValue(); - } - throw new TrinoException(DECODER_CONVERSION_NOT_SUPPORTED, - format("cannot decode object of '%s' as '%s' for column '%s'", - value.getClass(), columnType, columnName)); - } - - @Override - public boolean getBoolean() { - if (value instanceof Boolean) { - return (Boolean) value; - } - throw new TrinoException(DECODER_CONVERSION_NOT_SUPPORTED, - format("cannot decode object of '%s' as '%s' for column '%s'", - value.getClass(), columnType, columnName)); - } - - @Override - public long getLong() { - if (value instanceof Long || value instanceof Integer) { - final long payload = ((Number) value).longValue(); - if (TimestampType.TIMESTAMP_MILLIS.equals(columnType)) { - return payload * Timestamps.MICROSECONDS_PER_MILLISECOND; - } - if (TimeType.TIME_MILLIS.equals(columnType)) { - return payload * Timestamps.PICOSECONDS_PER_MILLISECOND; - } - return payload; - } - - if (columnType instanceof RealType) { - return floatToIntBits((Float) value); - } - - if (columnType instanceof DecimalType) { - ByteBuffer buffer = (ByteBuffer) value; - byte[] bytes = new byte[buffer.remaining()]; - buffer.get(bytes); - return new BigInteger(bytes).longValue(); - } - - throw new TrinoException(DECODER_CONVERSION_NOT_SUPPORTED, - format("cannot decode object of '%s' as '%s' for column '%s'", - value.getClass(), columnType, columnName)); - } - - @Override - public Slice getSlice() { - return PulsarAvroColumnDecoder.getSlice(value, columnType, columnName); - } - - @Override - public Block getBlock() { - return serializeObject(null, value, columnType, columnName); - } - } - - private static Slice getSlice(Object value, Type type, String columnName) { - if (type instanceof VarcharType && (value instanceof CharSequence || value instanceof GenericEnumSymbol)) { - return truncateToLength(utf8Slice(value.toString()), type); - } - - if (type instanceof VarbinaryType) { - if (value instanceof ByteBuffer) { - return Slices.wrappedBuffer((ByteBuffer) value); - } else if (value instanceof GenericFixed) { - return Slices.wrappedBuffer(((GenericFixed) value).bytes()); - } - } - - throw new TrinoException(DECODER_CONVERSION_NOT_SUPPORTED, - format("cannot decode object of '%s' as '%s' for column '%s'", - value.getClass(), type, columnName)); - } - - private static Block serializeObject(BlockBuilder builder, Object value, Type type, String columnName) { - if (type instanceof ArrayType) { - return serializeList(builder, value, type, columnName); - } - if (type instanceof MapType) { - return serializeMap(builder, value, type, columnName); - } - if (type instanceof RowType) { - return serializeRow(builder, value, type, columnName); - } - if (type instanceof DecimalType && !((DecimalType) type).isShort()) { - return serializeLongDecimal(builder, value, type, columnName); - } - serializePrimitive(builder, value, type, columnName); - return null; - } - - private static Block serializeList(BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) { - if (value == null) { - checkState(parentBlockBuilder != null, "parentBlockBuilder is null"); - parentBlockBuilder.appendNull(); - return null; - } - List list = (List) value; - List typeParameters = type.getTypeParameters(); - Type elementType = typeParameters.get(0); - - BlockBuilder blockBuilder = elementType.createBlockBuilder(null, list.size()); - for (Object element : list) { - serializeObject(blockBuilder, element, elementType, columnName); - } - if (parentBlockBuilder != null) { - type.writeObject(parentBlockBuilder, blockBuilder.build()); - return null; - } - return blockBuilder.build(); - } - - private static Block serializeLongDecimal( - BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) { - final BlockBuilder blockBuilder; - if (parentBlockBuilder != null) { - blockBuilder = parentBlockBuilder; - } else { - blockBuilder = type.createBlockBuilder(null, 1); - } - final ByteBuffer buffer = (ByteBuffer) value; - type.writeObject(blockBuilder, Int128.fromBigEndian(buffer.array())); - if (parentBlockBuilder == null) { - return blockBuilder.getSingleValueBlock(0); - } - return null; - } - - private static void serializePrimitive(BlockBuilder blockBuilder, Object value, Type type, String columnName) { - requireNonNull(blockBuilder, "parent blockBuilder is null"); - - if (value == null) { - blockBuilder.appendNull(); - return; - } - - if (type instanceof BooleanType) { - type.writeBoolean(blockBuilder, (Boolean) value); - return; - } - - if (value instanceof Integer || value instanceof Long) { - final long payload = ((Number) value).longValue(); - if (type instanceof BigintType || type instanceof IntegerType - || type instanceof SmallintType || type instanceof TinyintType) { - type.writeLong(blockBuilder, payload); - return; - } - if (TimestampType.TIMESTAMP_MILLIS.equals(type)) { - type.writeLong(blockBuilder, payload * Timestamps.MICROSECONDS_PER_MILLISECOND); - return; - } - if (TimeType.TIME_MILLIS.equals(type)) { - type.writeLong(blockBuilder, payload * Timestamps.PICOSECONDS_PER_MILLISECOND); - return; - } - } - - if (type instanceof DoubleType) { - type.writeDouble(blockBuilder, (Double) value); - return; - } - - if (type instanceof RealType) { - type.writeLong(blockBuilder, floatToIntBits((Float) value)); - return; - } - - if (type instanceof VarcharType || type instanceof VarbinaryType) { - type.writeSlice(blockBuilder, getSlice(value, type, columnName)); - return; - } - - throw new TrinoException(DECODER_CONVERSION_NOT_SUPPORTED, - format("cannot decode object of '%s' as '%s' for column '%s'", - value.getClass(), type, columnName)); - } - - private static Block serializeMap(BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) { - if (value == null) { - checkState(parentBlockBuilder != null, "parentBlockBuilder is null"); - parentBlockBuilder.appendNull(); - return null; - } - - Map map = (Map) value; - List typeParameters = type.getTypeParameters(); - Type keyType = typeParameters.get(0); - Type valueType = typeParameters.get(1); - - BlockBuilder blockBuilder; - if (parentBlockBuilder != null) { - blockBuilder = parentBlockBuilder; - } else { - blockBuilder = type.createBlockBuilder(null, 1); - } - - BlockBuilder entryBuilder = blockBuilder.beginBlockEntry(); - for (Map.Entry entry : map.entrySet()) { - if (entry.getKey() != null) { - keyType.writeSlice(entryBuilder, truncateToLength(utf8Slice(entry.getKey().toString()), keyType)); - serializeObject(entryBuilder, entry.getValue(), valueType, columnName); - } - } - blockBuilder.closeEntry(); - - if (parentBlockBuilder == null) { - return blockBuilder.getObject(0, Block.class); - } - return null; - } - - private static Block serializeRow(BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) { - if (value == null) { - checkState(parentBlockBuilder != null, "parent block builder is null"); - parentBlockBuilder.appendNull(); - return null; - } - - BlockBuilder blockBuilder; - if (parentBlockBuilder != null) { - blockBuilder = parentBlockBuilder; - } else { - blockBuilder = type.createBlockBuilder(null, 1); - } - BlockBuilder singleRowBuilder = blockBuilder.beginBlockEntry(); - GenericRecord record = (GenericRecord) value; - List fields = ((RowType) type).getFields(); - for (Field field : fields) { - checkState(field.getName().isPresent(), "field name not found"); - serializeObject(singleRowBuilder, record.get(field.getName().get()), field.getType(), columnName); - } - blockBuilder.closeEntry(); - if (parentBlockBuilder == null) { - return blockBuilder.getObject(0, Block.class); - } - return null; - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java deleted file mode 100644 index 97d0bb3377227..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.decoder.avro; - -import static com.google.common.base.Functions.identity; -import static com.google.common.collect.ImmutableMap.toImmutableMap; -import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; -import static java.util.Objects.requireNonNull; -import io.netty.buffer.ByteBuf; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.decoder.FieldValueProvider; -import io.trino.spi.TrinoException; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import org.apache.avro.generic.GenericRecord; -import org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord; -import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; -import org.apache.pulsar.sql.presto.PulsarRowDecoder; - -/** - * Refer to {@link io.trino.decoder.avro.AvroRowDecoderFactory}. - */ -public class PulsarAvroRowDecoder implements PulsarRowDecoder { - - private final GenericAvroSchema genericAvroSchema; - private final Map columnDecoders; - - public PulsarAvroRowDecoder(GenericAvroSchema genericAvroSchema, Set columns) { - this.genericAvroSchema = requireNonNull(genericAvroSchema, "genericAvroSchema is null"); - columnDecoders = columns.stream() - .collect(toImmutableMap(identity(), this::createColumnDecoder)); - } - - private PulsarAvroColumnDecoder createColumnDecoder(DecoderColumnHandle columnHandle) { - return new PulsarAvroColumnDecoder(columnHandle); - } - - /** - * decode ByteBuf by {@link org.apache.pulsar.client.api.schema.GenericSchema}. - * @param byteBuf - * @return - */ - @Override - public Optional> decodeRow(ByteBuf byteBuf) { - GenericRecord avroRecord; - try { - GenericAvroRecord record = (GenericAvroRecord) genericAvroSchema.decode(byteBuf); - avroRecord = record.getAvroRecord(); - } catch (Exception e) { - e.printStackTrace(); - throw new TrinoException(GENERIC_INTERNAL_ERROR, "Decoding avro record failed.", e); - } - return Optional.of(columnDecoders.entrySet().stream() - .collect(toImmutableMap( - Map.Entry::getKey, - entry -> entry.getValue().decodeField(avroRecord)))); - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java deleted file mode 100644 index 3072bf9441b2c..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.decoder.avro; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; -import static io.trino.spi.type.VarcharType.createUnboundedVarcharType; -import static java.lang.String.format; -import static java.util.stream.Collectors.toList; -import com.google.common.collect.ImmutableList; -import io.airlift.log.Logger; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.BigintType; -import io.trino.spi.type.BooleanType; -import io.trino.spi.type.DateType; -import io.trino.spi.type.DecimalType; -import io.trino.spi.type.DoubleType; -import io.trino.spi.type.IntegerType; -import io.trino.spi.type.RealType; -import io.trino.spi.type.RowType; -import io.trino.spi.type.StandardTypes; -import io.trino.spi.type.TimeType; -import io.trino.spi.type.TimestampType; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeManager; -import io.trino.spi.type.TypeSignature; -import io.trino.spi.type.TypeSignatureParameter; -import io.trino.spi.type.VarbinaryType; -import io.trino.spi.type.VarcharType; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import org.apache.avro.LogicalType; -import org.apache.avro.LogicalTypes; -import org.apache.avro.Schema; -import org.apache.avro.SchemaParseException; -import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; -import org.apache.pulsar.client.impl.schema.generic.GenericJsonSchema; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.sql.presto.PulsarColumnHandle; -import org.apache.pulsar.sql.presto.PulsarColumnMetadata; -import org.apache.pulsar.sql.presto.PulsarRowDecoder; -import org.apache.pulsar.sql.presto.PulsarRowDecoderFactory; - -/** - * PulsarRowDecoderFactory for {@link org.apache.pulsar.common.schema.SchemaType#AVRO}. - */ -public class PulsarAvroRowDecoderFactory implements PulsarRowDecoderFactory { - - private final TypeManager typeManager; - - public PulsarAvroRowDecoderFactory(TypeManager typeManager) { - this.typeManager = typeManager; - } - - @Override - public PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, - Set columns) { - return new PulsarAvroRowDecoder((GenericAvroSchema) GenericAvroSchema.of(schemaInfo), columns); - } - - @Override - public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, - PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { - List columnMetadata; - String schemaJson = new String(schemaInfo.getSchema()); - if (StringUtils.isBlank(schemaJson)) { - throw new TrinoException(NOT_SUPPORTED, "Topic " - + topicName.toString() + " does not have a valid schema"); - } - Schema schema; - try { - schema = GenericJsonSchema.of(schemaInfo).getAvroSchema(); - } catch (SchemaParseException ex) { - throw new TrinoException(NOT_SUPPORTED, "Topic " - + topicName.toString() + " does not have a valid schema"); - } - - try { - columnMetadata = schema.getFields().stream() - .map(field -> - new PulsarColumnMetadata(PulsarColumnMetadata.getColumnName(handleKeyValueType, - field.name()), parseAvroPrestoType(field.name(), field.schema()), - field.schema().toString(), null, false, false, - handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo(field.name(), - null, null)) - - ).collect(toList()); - } catch (StackOverflowError e){ - log.warn(e, "Topic " - + topicName.toString() + " extractColumnMetadata failed."); - throw new TrinoException(NOT_SUPPORTED, "Topic " - + topicName.toString() + " schema may contains cyclic definitions.", e); - } - return columnMetadata; - } - - private Type parseAvroPrestoType(String fieldName, Schema schema) { - Schema.Type type = schema.getType(); - LogicalType logicalType = schema.getLogicalType(); - switch (type) { - case STRING: - case ENUM: - return createUnboundedVarcharType(); - case NULL: - throw new UnsupportedOperationException( - format("field '%s' NULL type code should not be reached," - + "please check the schema or report the bug.", fieldName)); - case FIXED: - case BYTES: - // When the precision <= 0, throw Exception. - // When the precision > 0 and <= 18, use ShortDecimalType. and mapping Long - // When the precision > 18 and <= 36, use LongDecimalType. and mapping Slice - // When the precision > 36, throw Exception. - if (logicalType instanceof LogicalTypes.Decimal) { - LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType; - return DecimalType.createDecimalType(decimal.getPrecision(), decimal.getScale()); - } - return VarbinaryType.VARBINARY; - case INT: - if (logicalType == LogicalTypes.timeMillis()) { - return TimeType.TIME_MILLIS; - } else if (logicalType == LogicalTypes.date()) { - return DateType.DATE; - } - return IntegerType.INTEGER; - case LONG: - if (logicalType == LogicalTypes.timestampMillis()) { - return TimestampType.TIMESTAMP_MILLIS; - } - return BigintType.BIGINT; - case FLOAT: - return RealType.REAL; - case DOUBLE: - return DoubleType.DOUBLE; - case BOOLEAN: - return BooleanType.BOOLEAN; - case ARRAY: - return new ArrayType(parseAvroPrestoType(fieldName, schema.getElementType())); - case MAP: - //The key for an avro map must be string - TypeSignature valueType = parseAvroPrestoType(fieldName, schema.getValueType()).getTypeSignature(); - return typeManager.getParameterizedType(StandardTypes.MAP, - ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), - TypeSignatureParameter.typeParameter(valueType))); - case RECORD: - if (schema.getFields().size() > 0) { - return RowType.from(schema.getFields().stream() - .map(field -> new RowType.Field(Optional.of(field.name()), - parseAvroPrestoType(field.name(), field.schema()))) - .collect(toImmutableList())); - } else { - throw new UnsupportedOperationException(format( - "field '%s' of record type has no fields, " - + "please check schema definition. ", fieldName)); - } - case UNION: - for (Schema nestType : schema.getTypes()) { - if (nestType.getType() != Schema.Type.NULL) { - return parseAvroPrestoType(fieldName, nestType); - } - } - throw new UnsupportedOperationException(format( - "field '%s' of UNION type must contains not NULL type.", fieldName)); - default: - throw new UnsupportedOperationException(format( - "Can't convert from schema type '%s' (%s) to presto type.", - schema.getType(), schema.getFullName())); - } - } - - private static final Logger log = Logger.get(PulsarAvroRowDecoderFactory.class); - -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java deleted file mode 100644 index 905e3bd6becb4..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java +++ /dev/null @@ -1,469 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.decoder.json; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.decoder.DecoderErrorCode.DECODER_CONVERSION_NOT_SUPPORTED; -import static io.trino.spi.type.Varchars.truncateToLength; -import static java.lang.Double.parseDouble; -import static java.lang.Float.floatToIntBits; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.DecimalNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableList; -import io.airlift.log.Logger; -import io.airlift.slice.Slice; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.decoder.FieldValueProvider; -import io.trino.decoder.json.JsonFieldDecoder; -import io.trino.decoder.json.JsonRowDecoderFactory; -import io.trino.spi.TrinoException; -import io.trino.spi.block.Block; -import io.trino.spi.block.BlockBuilder; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.BigintType; -import io.trino.spi.type.BooleanType; -import io.trino.spi.type.DateType; -import io.trino.spi.type.DecimalType; -import io.trino.spi.type.DoubleType; -import io.trino.spi.type.Int128; -import io.trino.spi.type.IntegerType; -import io.trino.spi.type.MapType; -import io.trino.spi.type.RealType; -import io.trino.spi.type.RowType; -import io.trino.spi.type.SmallintType; -import io.trino.spi.type.TimeType; -import io.trino.spi.type.TimestampType; -import io.trino.spi.type.Timestamps; -import io.trino.spi.type.TinyintType; -import io.trino.spi.type.Type; -import io.trino.spi.type.VarbinaryType; -import io.trino.spi.type.VarcharType; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import org.apache.commons.lang3.tuple.Pair; - -/** - * Copy from {@link io.trino.decoder.json.DefaultJsonFieldDecoder} with some pulsar's extensions. - * 1) support {@link io.trino.spi.type.ArrayType}. - * 2) support {@link io.trino.spi.type.MapType}. - * 3) support {@link io.trino.spi.type.RowType}. - * 4) support date and time types. - * {@link io.trino.spi.type.TimestampType} - * {@link io.trino.spi.type.DateType} - * {@link io.trino.spi.type.TimeType} - * 5) support {@link io.trino.spi.type.RealType}. - * 6) support {@link io.trino.spi.type.DecimalType}. - */ -public class PulsarJsonFieldDecoder - implements JsonFieldDecoder { - - private final DecoderColumnHandle columnHandle; - private final long minValue; - private final long maxValue; - - public PulsarJsonFieldDecoder(DecoderColumnHandle columnHandle) { - this.columnHandle = requireNonNull(columnHandle, "columnHandle is null"); - if (!isSupportedType(columnHandle.getType())) { - JsonRowDecoderFactory.throwUnsupportedColumnType(columnHandle); - } - Pair range = getNumRangeByType(columnHandle.getType()); - minValue = range.getKey(); - maxValue = range.getValue(); - } - - private static Pair getNumRangeByType(Type type) { - if (type == TinyintType.TINYINT) { - return Pair.of((long) Byte.MIN_VALUE, (long) Byte.MAX_VALUE); - } else if (type == SmallintType.SMALLINT) { - return Pair.of((long) Short.MIN_VALUE, (long) Short.MAX_VALUE); - } else if (type == IntegerType.INTEGER) { - return Pair.of((long) Integer.MIN_VALUE, (long) Integer.MAX_VALUE); - } else if (type == BigintType.BIGINT) { - return Pair.of(Long.MIN_VALUE, Long.MAX_VALUE); - } else { - // those values will not be used if column type is not one of mentioned above - return Pair.of(Long.MIN_VALUE, Long.MAX_VALUE); - } - } - - private boolean isSupportedType(Type type) { - if (type instanceof DecimalType) { - return true; - } - if (type instanceof VarcharType) { - return true; - } - if (ImmutableList.of( - BigintType.BIGINT, - IntegerType.INTEGER, - SmallintType.SMALLINT, - TinyintType.TINYINT, - BooleanType.BOOLEAN, - DoubleType.DOUBLE, - TimestampType.TIMESTAMP_MILLIS, - DateType.DATE, - TimeType.TIME_MILLIS, - RealType.REAL - ).contains(type)) { - return true; - } - - if (type instanceof ArrayType) { - checkArgument(type.getTypeParameters().size() == 1, "expecting exactly one type parameter for array"); - return isSupportedType(type.getTypeParameters().get(0)); - } - if (type instanceof MapType) { - List typeParameters = type.getTypeParameters(); - checkArgument(typeParameters.size() == 2, "expecting exactly two type parameters for map"); - return isSupportedType(type.getTypeParameters().get(0)) && isSupportedType(type.getTypeParameters().get(1)); - } - - if (type instanceof RowType) { - for (Type fieldType : type.getTypeParameters()) { - if (!isSupportedType(fieldType)) { - return false; - } - } - return true; - } - - return false; - } - - @Override - public FieldValueProvider decode(JsonNode value) { - return new JsonValueProvider(value, columnHandle, minValue, maxValue); - } - - /** - * JsonValueProvider. - */ - public static class JsonValueProvider - extends FieldValueProvider { - private final JsonNode value; - private final DecoderColumnHandle columnHandle; - private final long minValue; - private final long maxValue; - - public JsonValueProvider(JsonNode value, DecoderColumnHandle columnHandle, long minValue, long maxValue) { - this.value = value; - this.columnHandle = columnHandle; - this.minValue = minValue; - this.maxValue = maxValue; - } - - @Override - public final boolean isNull() { - return value.isMissingNode() || value.isNull(); - } - - @Override - public boolean getBoolean() { - return getBoolean(value, columnHandle.getType(), columnHandle.getName()); - } - - @Override - public long getLong() { - return getLong(value, columnHandle.getType(), columnHandle.getName(), minValue, maxValue); - } - - @Override - public double getDouble() { - return getDouble(value, columnHandle.getType(), columnHandle.getName()); - } - - @Override - public Slice getSlice() { - return getSlice(value, columnHandle.getType(), columnHandle.getName()); - } - - @Override - public Block getBlock() { - return serializeObject(null, value, columnHandle.getType(), columnHandle.getName()); - } - - - public static boolean getBoolean(JsonNode value, Type type, String columnName) { - if (value.isValueNode()) { - return value.asBoolean(); - } - throw new TrinoException( - DECODER_CONVERSION_NOT_SUPPORTED, - format("could not parse non-value node as '%s' for column '%s'", type, columnName)); - } - - public static long getLong(JsonNode value, Type type, String columnName, long minValue, long maxValue) { - try { - if (type instanceof RealType) { - return floatToIntBits(Float.parseFloat(value.asText())); - } - - // If it is decimalType, need to eliminate the decimal point, - // and give it to trino to set the decimal point - if (type instanceof DecimalType) { - String decimalLong = value.asText().replace(".", ""); - return Long.parseLong(decimalLong); - } - - Long longValue; - if (value.isIntegralNumber() && !value.isBigInteger()) { - longValue = value.longValue(); - } else if (value.isValueNode()) { - longValue = Long.parseLong(value.asText()); - } else { - longValue = null; - } - - if (longValue != null && longValue >= minValue && longValue <= maxValue) { - if (TimestampType.TIMESTAMP_MILLIS.equals(type)) { - return longValue * Timestamps.MICROSECONDS_PER_MILLISECOND; - } - if (TimeType.TIME_MILLIS.equals(type)) { - return longValue * Timestamps.PICOSECONDS_PER_MILLISECOND; - } - return longValue; - } - } catch (NumberFormatException ignore) { - // ignore - } - throw new TrinoException( - DECODER_CONVERSION_NOT_SUPPORTED, - format("could not parse value '%s' as '%s' for column '%s'", value.asText(), type, columnName)); - } - - public static double getDouble(JsonNode value, Type type, String columnName) { - try { - if (value.isNumber()) { - return value.doubleValue(); - } - if (value.isValueNode()) { - return parseDouble(value.asText()); - } - } catch (NumberFormatException ignore) { - // ignore - } - throw new TrinoException( - DECODER_CONVERSION_NOT_SUPPORTED, - format("could not parse value '%s' as '%s' for column '%s'", value.asText(), type, columnName)); - - } - - private static Slice getSlice(JsonNode value, Type type, String columnName) { - String textValue = value.isValueNode() ? value.asText() : value.toString(); - - Slice slice = utf8Slice(textValue); - if (type instanceof VarcharType) { - slice = truncateToLength(slice, type); - } - return slice; - } - - private Block serializeObject(BlockBuilder builder, Object value, Type type, String columnName) { - if (type instanceof ArrayType) { - return serializeList(builder, value, type, columnName); - } - if (type instanceof MapType) { - return serializeMap(builder, value, type, columnName); - } - if (type instanceof RowType) { - return serializeRow(builder, value, type, columnName); - } - if (type instanceof DecimalType && !((DecimalType) type).isShort()) { - return serializeLongDecimal(builder, value, type, columnName); - } - serializePrimitive(builder, value, type, columnName); - return null; - } - - private Block serializeList(BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) { - if (value == null) { - checkState(parentBlockBuilder != null, "parentBlockBuilder is null"); - parentBlockBuilder.appendNull(); - return null; - } - - checkState(value instanceof ArrayNode, "Json array node must is ArrayNode type"); - - Iterator jsonNodeIterator = ((ArrayNode) value).elements(); - - List typeParameters = type.getTypeParameters(); - Type elementType = typeParameters.get(0); - - BlockBuilder blockBuilder = elementType.createBlockBuilder(null, ((ArrayNode) value).size()); - - while (jsonNodeIterator.hasNext()) { - Object element = jsonNodeIterator.next(); - serializeObject(blockBuilder, element, elementType, columnName); - } - - if (parentBlockBuilder != null) { - type.writeObject(parentBlockBuilder, blockBuilder.build()); - return null; - } - return blockBuilder.build(); - } - - private static Block serializeLongDecimal( - BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) { - final BlockBuilder blockBuilder; - if (parentBlockBuilder != null) { - blockBuilder = parentBlockBuilder; - } else { - blockBuilder = type.createBlockBuilder(null, 1); - } - - assert value instanceof DecimalNode; - final DecimalNode node = (DecimalNode) value; - // For decimalType, need to eliminate the decimal point, - // and give it to trino to set the decimal point - type.writeObject(blockBuilder, Int128.valueOf(node.asText().replace(".", ""))); - - if (parentBlockBuilder == null) { - return blockBuilder.getSingleValueBlock(0); - } - return null; - } - - private void serializePrimitive(BlockBuilder blockBuilder, Object node, Type type, String columnName) { - requireNonNull(blockBuilder, "parent blockBuilder is null"); - - JsonNode value; - if (node == null) { - blockBuilder.appendNull(); - return; - } - - if (node instanceof JsonNode) { - value = (JsonNode) node; - } else { - throw new TrinoException(DECODER_CONVERSION_NOT_SUPPORTED, - format("primitive object of '%s' as '%s' for column '%s' cann't convert to JsonNode", - node.getClass(), type, columnName)); - } - - if (type instanceof BooleanType) { - type.writeBoolean(blockBuilder, getBoolean(value, type, columnName)); - return; - } - - if (type instanceof RealType || type instanceof BigintType - || type instanceof IntegerType || type instanceof SmallintType - || type instanceof TinyintType || type instanceof TimestampType - || type instanceof TimeType || type instanceof DateType) { - Pair numRange = getNumRangeByType(type); - type.writeLong(blockBuilder, getLong(value, type, columnName, numRange.getKey(), numRange.getValue())); - return; - } - - if (type instanceof DoubleType) { - type.writeDouble(blockBuilder, getDouble(value, type, columnName)); - return; - } - - if (type instanceof VarcharType || type instanceof VarbinaryType) { - type.writeSlice(blockBuilder, getSlice(value, type, columnName)); - return; - } - - throw new TrinoException(DECODER_CONVERSION_NOT_SUPPORTED, - format("cannot decode object of '%s' as '%s' for column '%s'", value.getClass(), type, columnName)); - } - - private Block serializeMap(BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) { - if (value == null) { - checkState(parentBlockBuilder != null, "parentBlockBuilder is null"); - parentBlockBuilder.appendNull(); - return null; - } - checkState(value instanceof ObjectNode, "Json map node must is ObjectNode type"); - - List typeParameters = type.getTypeParameters(); - Type keyType = typeParameters.get(0); - Type valueType = typeParameters.get(1); - - BlockBuilder blockBuilder; - if (parentBlockBuilder != null) { - blockBuilder = parentBlockBuilder; - } else { - blockBuilder = type.createBlockBuilder(null, 1); - } - - BlockBuilder entryBuilder = blockBuilder.beginBlockEntry(); - - Iterator> fields = ((ObjectNode) value).fields(); - while (fields.hasNext()) { - Map.Entry entry = fields.next(); - if (entry.getKey() != null) { - keyType.writeSlice(entryBuilder, truncateToLength(utf8Slice(entry.getKey().toString()), keyType)); - serializeObject(entryBuilder, entry.getValue(), valueType, columnName); - } - } - - blockBuilder.closeEntry(); - - if (parentBlockBuilder == null) { - return blockBuilder.getObject(0, Block.class); - } - return null; - } - - - private Block serializeRow(BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) { - if (value == null) { - checkState(parentBlockBuilder != null, "parent block builder is null"); - parentBlockBuilder.appendNull(); - return null; - } - - BlockBuilder blockBuilder; - if (parentBlockBuilder != null) { - blockBuilder = parentBlockBuilder; - } else { - blockBuilder = type.createBlockBuilder(null, 1); - } - BlockBuilder singleRowBuilder = blockBuilder.beginBlockEntry(); - - List fields = ((RowType) type).getFields(); - - checkState(value instanceof ObjectNode, "Json row node must be ObjectNode type"); - - for (RowType.Field field : fields) { - checkState(field.getName().isPresent(), "field name not found"); - serializeObject(singleRowBuilder, ((ObjectNode) value).get(field.getName().get()), - field.getType(), columnName); - } - blockBuilder.closeEntry(); - if (parentBlockBuilder == null) { - return blockBuilder.getObject(0, Block.class); - } - return null; - } - - } - - private static final Logger log = Logger.get(PulsarJsonFieldDecoder.class); - -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java deleted file mode 100644 index 9e8e059ea3a75..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.decoder.json; - -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.collect.ImmutableMap.toImmutableMap; -import static java.util.Objects.requireNonNull; -import static java.util.function.Function.identity; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.MissingNode; -import com.google.common.base.Splitter; -import io.airlift.log.Logger; -import io.netty.buffer.ByteBuf; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.decoder.FieldValueProvider; -import io.trino.decoder.json.JsonFieldDecoder; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord; -import org.apache.pulsar.client.impl.schema.generic.GenericJsonSchema; -import org.apache.pulsar.sql.presto.PulsarRowDecoder; - -/** - * Json PulsarRowDecoder. - */ -public class PulsarJsonRowDecoder implements PulsarRowDecoder { - - private final Map fieldDecoders; - - private final GenericJsonSchema genericJsonSchema; - - public PulsarJsonRowDecoder(GenericJsonSchema genericJsonSchema, Set columns) { - this.genericJsonSchema = requireNonNull(genericJsonSchema, "genericJsonSchema is null"); - this.fieldDecoders = columns.stream().collect(toImmutableMap(identity(), PulsarJsonFieldDecoder::new)); - } - - private static JsonNode locateNode(JsonNode tree, DecoderColumnHandle columnHandle) { - String mapping = columnHandle.getMapping(); - checkState(mapping != null, "No mapping for %s", columnHandle.getName()); - JsonNode currentNode = tree; - for (String pathElement : Splitter.on('/').omitEmptyStrings().split(mapping)) { - if (!currentNode.has(pathElement)) { - return MissingNode.getInstance(); - } - currentNode = currentNode.path(pathElement); - } - return currentNode; - } - - /** - * decode ByteBuf by {@link org.apache.pulsar.client.api.schema.GenericSchema}. - * @param byteBuf - * @return - */ - @Override - public Optional> decodeRow(ByteBuf byteBuf) { - GenericJsonRecord record = (GenericJsonRecord) genericJsonSchema.decode(byteBuf); - JsonNode tree = record.getJsonNode(); - Map decodedRow = new HashMap<>(); - for (Map.Entry entry : fieldDecoders.entrySet()) { - DecoderColumnHandle columnHandle = entry.getKey(); - JsonFieldDecoder decoder = entry.getValue(); - JsonNode node = locateNode(tree, columnHandle); - decodedRow.put(columnHandle, decoder.decode(node)); - } - return Optional.of(decodedRow); - } - - private static final Logger log = Logger.get(PulsarJsonRowDecoderFactory.class); -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java deleted file mode 100644 index 0d5cc2d262dfe..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.decoder.json; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; -import static io.trino.spi.type.VarcharType.createUnboundedVarcharType; -import static java.lang.String.format; -import static java.util.stream.Collectors.toList; -import com.google.common.collect.ImmutableList; -import io.airlift.log.Logger; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.BigintType; -import io.trino.spi.type.BooleanType; -import io.trino.spi.type.DateType; -import io.trino.spi.type.DecimalType; -import io.trino.spi.type.DoubleType; -import io.trino.spi.type.IntegerType; -import io.trino.spi.type.RealType; -import io.trino.spi.type.RowType; -import io.trino.spi.type.StandardTypes; -import io.trino.spi.type.TimeType; -import io.trino.spi.type.TimestampType; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeManager; -import io.trino.spi.type.TypeSignature; -import io.trino.spi.type.TypeSignatureParameter; -import io.trino.spi.type.VarbinaryType; -import io.trino.spi.type.VarcharType; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import org.apache.avro.LogicalType; -import org.apache.avro.LogicalTypes; -import org.apache.avro.Schema; -import org.apache.avro.SchemaParseException; -import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.client.impl.schema.generic.GenericJsonSchema; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.sql.presto.PulsarColumnHandle; -import org.apache.pulsar.sql.presto.PulsarColumnMetadata; -import org.apache.pulsar.sql.presto.PulsarRowDecoderFactory; - -/** - * PulsarRowDecoderFactory for {@link org.apache.pulsar.common.schema.SchemaType#JSON}. - */ -public class PulsarJsonRowDecoderFactory implements PulsarRowDecoderFactory { - - private final TypeManager typeManager; - - public PulsarJsonRowDecoderFactory(TypeManager typeManager) { - this.typeManager = typeManager; - } - - @Override - public PulsarJsonRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, - Set columns) { - return new PulsarJsonRowDecoder((GenericJsonSchema) GenericJsonSchema.of(schemaInfo), columns); - } - - @Override - public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, - PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { - List columnMetadata; - String schemaJson = new String(schemaInfo.getSchema()); - if (StringUtils.isBlank(schemaJson)) { - throw new TrinoException(NOT_SUPPORTED, "Topic " - + topicName.toString() + " does not have a valid schema"); - } - - Schema schema; - try { - schema = GenericJsonSchema.of(schemaInfo).getAvroSchema(); - } catch (SchemaParseException ex) { - throw new TrinoException(NOT_SUPPORTED, "Topic " - + topicName.toString() + " does not have a valid schema"); - } - - try { - columnMetadata = schema.getFields().stream() - .map(field -> - new PulsarColumnMetadata(PulsarColumnMetadata.getColumnName(handleKeyValueType, - field.name()), parseJsonPrestoType(field.name(), field.schema()), - field.schema().toString(), null, false, false, - handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo( - field.name(), null, null)) - - ).collect(toList()); - } catch (StackOverflowError e) { - log.warn(e, "Topic " - + topicName.toString() + " extractColumnMetadata failed."); - throw new TrinoException(NOT_SUPPORTED, "Topic " - + topicName.toString() + " schema may contains cyclic definitions.", e); - } - return columnMetadata; - } - - - private Type parseJsonPrestoType(String fieldName, Schema schema) { - Schema.Type type = schema.getType(); - LogicalType logicalType = schema.getLogicalType(); - switch (type) { - case STRING: - case ENUM: - return createUnboundedVarcharType(); - case NULL: - throw new UnsupportedOperationException(format( - "field '%s' NULL type code should not be reached , " - + "please check the schema or report the bug.", fieldName)); - case FIXED: - case BYTES: - // When the precision <= 0, throw Exception. - // When the precision > 0 and <= 18, use ShortDecimalType. and mapping Long - // When the precision > 18 and <= 36, use LongDecimalType. and mapping Slice - // When the precision > 36, throw Exception. - if (logicalType instanceof LogicalTypes.Decimal) { - LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType; - return DecimalType.createDecimalType(decimal.getPrecision(), decimal.getScale()); - } - return VarbinaryType.VARBINARY; - case INT: - if (logicalType == LogicalTypes.timeMillis()) { - return TimeType.TIME_MILLIS; - } else if (logicalType == LogicalTypes.date()) { - return DateType.DATE; - } - return IntegerType.INTEGER; - case LONG: - if (logicalType == LogicalTypes.timestampMillis()) { - return TimestampType.TIMESTAMP_MILLIS; - } - return BigintType.BIGINT; - case FLOAT: - return RealType.REAL; - case DOUBLE: - return DoubleType.DOUBLE; - case BOOLEAN: - return BooleanType.BOOLEAN; - case ARRAY: - return new ArrayType(parseJsonPrestoType(fieldName, schema.getElementType())); - case MAP: - //The key for an avro map must be string. - TypeSignature valueType = parseJsonPrestoType(fieldName, schema.getValueType()).getTypeSignature(); - return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter. - typeParameter(VarcharType.VARCHAR.getTypeSignature()), - TypeSignatureParameter.typeParameter(valueType))); - case RECORD: - if (schema.getFields().size() > 0) { - return RowType.from(schema.getFields().stream() - .map(field -> new RowType.Field(Optional.of(field.name()), - parseJsonPrestoType(field.name(), field.schema()))) - .collect(toImmutableList())); - } else { - throw new UnsupportedOperationException(format( - "field '%s' of record type has no fields, " - + "please check schema definition. ", fieldName)); - } - case UNION: - for (Schema nestType : schema.getTypes()) { - if (nestType.getType() != Schema.Type.NULL) { - return parseJsonPrestoType(fieldName, nestType); - } - } - throw new UnsupportedOperationException(format( - "field '%s' of UNION type must contains not NULL type.", fieldName)); - default: - throw new UnsupportedOperationException(format( - "Can't convert from schema type '%s' (%s) to presto type.", - schema.getType(), schema.getFullName())); - } - } - - private static final Logger log = Logger.get(PulsarJsonRowDecoderFactory.class); - -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java deleted file mode 100644 index 6a6e495e030bd..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.decoder.primitive; - -import static io.trino.decoder.FieldValueProviders.booleanValueProvider; -import static io.trino.decoder.FieldValueProviders.bytesValueProvider; -import static io.trino.decoder.FieldValueProviders.longValueProvider; -import static org.apache.pulsar.sql.presto.PulsarFieldValueProviders.doubleValueProvider; -import io.netty.buffer.ByteBuf; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.decoder.FieldValueProvider; -import io.trino.decoder.FieldValueProviders; -import io.trino.spi.type.BigintType; -import io.trino.spi.type.BooleanType; -import io.trino.spi.type.DateType; -import io.trino.spi.type.DoubleType; -import io.trino.spi.type.IntegerType; -import io.trino.spi.type.RealType; -import io.trino.spi.type.SmallintType; -import io.trino.spi.type.TimeType; -import io.trino.spi.type.TimestampType; -import io.trino.spi.type.Timestamps; -import io.trino.spi.type.TinyintType; -import io.trino.spi.type.Type; -import io.trino.spi.type.VarbinaryType; -import io.trino.spi.type.VarcharType; -import java.sql.Time; -import java.sql.Timestamp; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import org.apache.pulsar.client.impl.schema.AbstractSchema; -import org.apache.pulsar.sql.presto.PulsarRowDecoder; - -/** - * Primitive Schema PulsarRowDecoder. - */ -public class PulsarPrimitiveRowDecoder implements PulsarRowDecoder { - - private final DecoderColumnHandle columnHandle; - private AbstractSchema schema; - - public PulsarPrimitiveRowDecoder(AbstractSchema schema, DecoderColumnHandle column) { - this.columnHandle = column; - this.schema = schema; - } - - @Override - public Optional> decodeRow(ByteBuf byteBuf) { - if (columnHandle == null) { - return Optional.empty(); - } - Object value = schema.decode(byteBuf); - Map primitiveColumn = new HashMap<>(); - if (value == null) { - primitiveColumn.put(columnHandle, FieldValueProviders.nullValueProvider()); - } else { - Type type = columnHandle.getType(); - if (type instanceof BooleanType) { - primitiveColumn.put(columnHandle, booleanValueProvider(Boolean.valueOf((Boolean) value))); - } else if (type instanceof TinyintType || type instanceof SmallintType || type instanceof IntegerType - || type instanceof BigintType) { - primitiveColumn.put(columnHandle, longValueProvider(Long.parseLong(value.toString()))); - } else if (type instanceof DoubleType) { - primitiveColumn.put(columnHandle, doubleValueProvider(Double.parseDouble(value.toString()))); - } else if (type instanceof RealType) { - primitiveColumn.put(columnHandle, longValueProvider( - Float.floatToIntBits((Float.parseFloat(value.toString()))))); - } else if (type instanceof VarbinaryType) { - primitiveColumn.put(columnHandle, bytesValueProvider((byte[]) value)); - } else if (type instanceof VarcharType) { - primitiveColumn.put(columnHandle, bytesValueProvider(value.toString().getBytes())); - } else if (type instanceof DateType) { - primitiveColumn.put(columnHandle, longValueProvider(((Date) value).getTime())); - } else if (type instanceof TimeType) { - final long millis = ((Time) value).getTime(); - final long picos = millis * Timestamps.PICOSECONDS_PER_MILLISECOND; - primitiveColumn.put(columnHandle, longValueProvider(picos)); - } else if (type instanceof TimestampType) { - final long millis = ((Timestamp) value).getTime(); - final long micros = millis * Timestamps.MICROSECONDS_PER_MILLISECOND; - primitiveColumn.put(columnHandle, longValueProvider(micros)); - } else { - primitiveColumn.put(columnHandle, bytesValueProvider(value.toString().getBytes())); - } - } - return Optional.of(primitiveColumn); - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java deleted file mode 100644 index d6c8b3ca99e51..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.decoder.primitive; - -import io.airlift.log.Logger; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.type.BigintType; -import io.trino.spi.type.BooleanType; -import io.trino.spi.type.DateType; -import io.trino.spi.type.DoubleType; -import io.trino.spi.type.IntegerType; -import io.trino.spi.type.RealType; -import io.trino.spi.type.SmallintType; -import io.trino.spi.type.TimeType; -import io.trino.spi.type.TimestampType; -import io.trino.spi.type.TinyintType; -import io.trino.spi.type.Type; -import io.trino.spi.type.VarbinaryType; -import io.trino.spi.type.VarcharType; -import java.util.Arrays; -import java.util.List; -import java.util.Set; -import org.apache.pulsar.client.impl.schema.AbstractSchema; -import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.common.schema.SchemaType; -import org.apache.pulsar.sql.presto.PulsarColumnHandle; -import org.apache.pulsar.sql.presto.PulsarColumnMetadata; -import org.apache.pulsar.sql.presto.PulsarRowDecoder; -import org.apache.pulsar.sql.presto.PulsarRowDecoderFactory; - -/** - * Primitive Schema PulsarRowDecoderFactory. - */ -public class PulsarPrimitiveRowDecoderFactory implements PulsarRowDecoderFactory { - - private static final Logger log = Logger.get(PulsarPrimitiveRowDecoderFactory.class); - - public static final String PRIMITIVE_COLUMN_NAME = "__value__"; - - @Override - public PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, - Set columns) { - if (columns.size() == 1) { - return new PulsarPrimitiveRowDecoder((AbstractSchema) AutoConsumeSchema.getSchema(schemaInfo), - columns.iterator().next()); - } else { - return new PulsarPrimitiveRowDecoder((AbstractSchema) AutoConsumeSchema.getSchema(schemaInfo), - null); - } - } - - @Override - public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, - PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { - ColumnMetadata valueColumn = new PulsarColumnMetadata( - PulsarColumnMetadata.getColumnName(handleKeyValueType, PRIMITIVE_COLUMN_NAME), - parsePrimitivePrestoType(PRIMITIVE_COLUMN_NAME, schemaInfo.getType()), - "The value of the message with primitive type schema", null, false, false, - handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo(PRIMITIVE_COLUMN_NAME, - null, null)); - return Arrays.asList(valueColumn); - } - - private Type parsePrimitivePrestoType(String fieldName, SchemaType pulsarType) { - switch (pulsarType) { - case BOOLEAN: - return BooleanType.BOOLEAN; - case INT8: - return TinyintType.TINYINT; - case INT16: - return SmallintType.SMALLINT; - case INT32: - return IntegerType.INTEGER; - case INT64: - return BigintType.BIGINT; - case FLOAT: - return RealType.REAL; - case DOUBLE: - return DoubleType.DOUBLE; - case NONE: - case BYTES: - return VarbinaryType.VARBINARY; - case STRING: - return VarcharType.VARCHAR; - case DATE: - return DateType.DATE; - case TIME: - return TimeType.TIME_MILLIS; - case TIMESTAMP: - return TimestampType.TIMESTAMP_MILLIS; - default: - log.error("Can't convert type: %s for %s", pulsarType, fieldName); - return null; - } - - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/package-info.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/package-info.java deleted file mode 100644 index 9eccb9571b5e9..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/** - * This package contains decoder for SchemaType of SchemaType.isPrimitive() return true. - */ -package org.apache.pulsar.sql.presto.decoder.primitive; \ No newline at end of file diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/protobufnative/PulsarProtobufNativeColumnDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/protobufnative/PulsarProtobufNativeColumnDecoder.java deleted file mode 100644 index d0174974e1628..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/protobufnative/PulsarProtobufNativeColumnDecoder.java +++ /dev/null @@ -1,398 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.decoder.protobufnative; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.decoder.DecoderErrorCode.DECODER_CONVERSION_NOT_SUPPORTED; -import static io.trino.spi.StandardErrorCode.GENERIC_USER_ERROR; -import static io.trino.spi.type.Varchars.truncateToLength; -import static java.lang.Float.floatToIntBits; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableSet; -import com.google.protobuf.ByteString; -import com.google.protobuf.DynamicMessage; -import com.google.protobuf.EnumValue; -import io.airlift.slice.Slice; -import io.airlift.slice.Slices; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.decoder.FieldValueProvider; -import io.trino.spi.TrinoException; -import io.trino.spi.block.Block; -import io.trino.spi.block.BlockBuilder; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.BigintType; -import io.trino.spi.type.BooleanType; -import io.trino.spi.type.DoubleType; -import io.trino.spi.type.IntegerType; -import io.trino.spi.type.MapType; -import io.trino.spi.type.RealType; -import io.trino.spi.type.RowType; -import io.trino.spi.type.RowType.Field; -import io.trino.spi.type.SmallintType; -import io.trino.spi.type.TimestampType; -import io.trino.spi.type.Timestamps; -import io.trino.spi.type.TinyintType; -import io.trino.spi.type.Type; -import io.trino.spi.type.VarbinaryType; -import io.trino.spi.type.VarcharType; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * Pulsar {@link org.apache.pulsar.common.schema.SchemaType#PROTOBUF_NATIVE} ColumnDecoder. - */ -public class PulsarProtobufNativeColumnDecoder { - private static final Set SUPPORTED_PRIMITIVE_TYPES = ImmutableSet.of( - BooleanType.BOOLEAN, - IntegerType.INTEGER, - BigintType.BIGINT, - RealType.REAL, - DoubleType.DOUBLE, - VarbinaryType.VARBINARY, - TimestampType.TIMESTAMP_MILLIS); - - private final Type columnType; - private final String columnMapping; - private final String columnName; - - public PulsarProtobufNativeColumnDecoder(DecoderColumnHandle columnHandle) { - try { - requireNonNull(columnHandle, "columnHandle is null"); - this.columnType = columnHandle.getType(); - this.columnMapping = columnHandle.getMapping(); - this.columnName = columnHandle.getName(); - checkArgument(!columnHandle.isInternal(), - "unexpected internal column '%s'", columnName); - checkArgument(columnHandle.getFormatHint() == null, - "unexpected format hint '%s' defined for column '%s'", columnHandle.getFormatHint(), columnName); - checkArgument(columnHandle.getDataFormat() == null, - "unexpected data format '%s' defined for column '%s'", columnHandle.getDataFormat(), columnName); - checkArgument(columnHandle.getMapping() != null, - "mapping not defined for column '%s'", columnName); - checkArgument(isSupportedType(columnType), - "Unsupported column type '%s' for column '%s'", columnType, columnName); - } catch (IllegalArgumentException e) { - throw new TrinoException(GENERIC_USER_ERROR, e); - } - } - - private static boolean isSupportedType(Type type) { - if (isSupportedPrimitive(type)) { - return true; - } - - if (type instanceof ArrayType) { - checkArgument(type.getTypeParameters().size() == 1, - "expecting exactly one type parameter for array"); - return isSupportedType(type.getTypeParameters().get(0)); - } - - if (type instanceof MapType) { - List typeParameters = type.getTypeParameters(); - checkArgument(typeParameters.size() == 2, - "expecting exactly two type parameters for map"); - return isSupportedType(typeParameters.get(1)) && isSupportedType(typeParameters.get(0)); - } - - if (type instanceof RowType) { - for (Type fieldType : type.getTypeParameters()) { - if (!isSupportedType(fieldType)) { - return false; - } - } - return true; - } - return false; - } - - private static boolean isSupportedPrimitive(Type type) { - return type instanceof VarcharType || SUPPORTED_PRIMITIVE_TYPES.contains(type); - } - - public FieldValueProvider decodeField(DynamicMessage dynamicMessage) { - Object columnValue = locateNode(dynamicMessage, columnMapping); - return new ObjectValueProvider(columnValue, columnType, columnName); - } - - private static Object locateNode(DynamicMessage element, String columnMapping) { - Object value = element; - for (String pathElement : Splitter.on('/').omitEmptyStrings().split(columnMapping)) { - if (value == null) { - return null; - } - value = ((DynamicMessage) value).getField(((DynamicMessage) value).getDescriptorForType() - .findFieldByName(pathElement)); - } - return value; - } - - private static class ObjectValueProvider - extends FieldValueProvider { - private final Object value; - private final Type columnType; - private final String columnName; - - public ObjectValueProvider(Object value, Type columnType, String columnName) { - this.value = value; - this.columnType = columnType; - this.columnName = columnName; - } - - @Override - public boolean isNull() { - return value == null; - } - - @Override - public double getDouble() { - if (value instanceof Double || value instanceof Float) { - return ((Number) value).doubleValue(); - } - throw new TrinoException(DECODER_CONVERSION_NOT_SUPPORTED, - format("cannot decode object of '%s' as '%s' for column '%s'", - value.getClass(), columnType, columnName)); - } - - @Override - public boolean getBoolean() { - if (value instanceof Boolean) { - return (Boolean) value; - } - throw new TrinoException(DECODER_CONVERSION_NOT_SUPPORTED, - format("cannot decode object of '%s' as '%s' for column '%s'", - value.getClass(), columnType, columnName)); - } - - @Override - public long getLong() { - if (value instanceof Long || value instanceof Integer) { - return ((Number) value).longValue(); - } - - if (columnType instanceof RealType) { - return floatToIntBits((Float) value); - } - - //return millisecond which parsed from protobuf/timestamp - if (TimestampType.TIMESTAMP_MILLIS.equals(columnType) && value instanceof DynamicMessage) { - DynamicMessage message = (DynamicMessage) value; - int nanos = (int) message.getField(message.getDescriptorForType().findFieldByName("nanos")); - long seconds = (long) message.getField(message.getDescriptorForType().findFieldByName("seconds")); - //maybe an exception here, but seems will never happen in hundred years. - long millis = seconds * MILLIS_PER_SECOND + nanos / NANOS_PER_MILLISECOND; - return millis * Timestamps.MICROSECONDS_PER_MILLISECOND; - } - - throw new TrinoException(DECODER_CONVERSION_NOT_SUPPORTED, - format("cannot decode object of '%s' as '%s' for column '%s'", - value.getClass(), columnType, columnName)); - } - - @Override - public Slice getSlice() { - return PulsarProtobufNativeColumnDecoder.getSlice(value, columnType, columnName); - } - - @Override - public Block getBlock() { - return serializeObject(null, value, columnType, columnName); - } - } - - private static Slice getSlice(Object value, Type type, String columnName) { - - if (value instanceof ByteString) { - return Slices.wrappedBuffer(((ByteString) value).toByteArray()); - } else if (value instanceof EnumValue) { //enum - return truncateToLength(utf8Slice(((EnumValue) value).getName()), type); - } else if (value instanceof byte[]) { - return Slices.wrappedBuffer((byte[]) value); - } - - if (type instanceof VarcharType) { - return truncateToLength(utf8Slice(value.toString()), type); - } - - throw new TrinoException(DECODER_CONVERSION_NOT_SUPPORTED, - format("cannot decode object of '%s' as '%s' for column '%s'", - value.getClass(), type, columnName)); - } - - private static Block serializeObject(BlockBuilder builder, Object value, Type type, String columnName) { - if (type instanceof ArrayType) { - return serializeList(builder, value, type, columnName); - } - if (type instanceof MapType) { - return serializeMap(builder, value, type, columnName); - } - if (type instanceof RowType) { - return serializeRow(builder, value, type, columnName); - } - serializePrimitive(builder, value, type, columnName); - return null; - } - - private static Block serializeList(BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) { - if (value == null) { - checkState(parentBlockBuilder != null, "parentBlockBuilder is null"); - parentBlockBuilder.appendNull(); - return null; - } - List list = (List) value; - List typeParameters = type.getTypeParameters(); - Type elementType = typeParameters.get(0); - - BlockBuilder blockBuilder = elementType.createBlockBuilder(null, list.size()); - for (Object element : list) { - serializeObject(blockBuilder, element, elementType, columnName); - } - if (parentBlockBuilder != null) { - type.writeObject(parentBlockBuilder, blockBuilder.build()); - return null; - } - return blockBuilder.build(); - } - - private static void serializePrimitive(BlockBuilder blockBuilder, Object value, Type type, String columnName) { - requireNonNull(blockBuilder, "parent blockBuilder is null"); - - if (value == null) { - blockBuilder.appendNull(); - return; - } - - if (type instanceof BooleanType) { - type.writeBoolean(blockBuilder, (Boolean) value); - return; - } - - if ((value instanceof Integer || value instanceof Long) - && (type instanceof BigintType || type instanceof IntegerType - || type instanceof SmallintType || type instanceof TinyintType)) { - type.writeLong(blockBuilder, ((Number) value).longValue()); - return; - } - - if (type instanceof DoubleType) { - type.writeDouble(blockBuilder, (Double) value); - return; - } - - if (type instanceof RealType) { - type.writeLong(blockBuilder, floatToIntBits((Float) value)); - return; - } - - if (type instanceof VarcharType || type instanceof VarbinaryType) { - type.writeSlice(blockBuilder, getSlice(value, type, columnName)); - return; - } - - throw new TrinoException(DECODER_CONVERSION_NOT_SUPPORTED, - format("cannot decode object of '%s' as '%s' for column '%s'", - value.getClass(), type, columnName)); - } - - private static Block serializeMap(BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) { - if (value == null) { - checkState(parentBlockBuilder != null, "parentBlockBuilder is null"); - parentBlockBuilder.appendNull(); - return null; - } - - Map map = parseProtobufMap(value); - - List typeParameters = type.getTypeParameters(); - Type keyType = typeParameters.get(0); - Type valueType = typeParameters.get(1); - - BlockBuilder blockBuilder; - if (parentBlockBuilder != null) { - blockBuilder = parentBlockBuilder; - } else { - blockBuilder = type.createBlockBuilder(null, 1); - } - - BlockBuilder entryBuilder = blockBuilder.beginBlockEntry(); - for (Map.Entry entry : map.entrySet()) { - if (entry.getKey() != null) { - serializeObject(entryBuilder, entry.getKey(), keyType, columnName); - serializeObject(entryBuilder, entry.getValue(), valueType, columnName); - } - } - blockBuilder.closeEntry(); - - if (parentBlockBuilder == null) { - return blockBuilder.getObject(0, Block.class); - } - return null; - } - - protected static Map parseProtobufMap(Object value) { - Map map = new HashMap(); - for (Object mapMsg : ((List) value)) { - map.put(((DynamicMessage) mapMsg).getField(((DynamicMessage) mapMsg).getDescriptorForType() - .findFieldByName(PROTOBUF_MAP_KEY_NAME)), ((DynamicMessage) mapMsg) - .getField(((DynamicMessage) mapMsg).getDescriptorForType() - .findFieldByName(PROTOBUF_MAP_VALUE_NAME))); - } - return map; - } - - private static Block serializeRow(BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) { - if (value == null) { - checkState(parentBlockBuilder != null, "parent block builder is null"); - parentBlockBuilder.appendNull(); - return null; - } - - BlockBuilder blockBuilder; - if (parentBlockBuilder != null) { - blockBuilder = parentBlockBuilder; - } else { - blockBuilder = type.createBlockBuilder(null, 1); - } - BlockBuilder singleRowBuilder = blockBuilder.beginBlockEntry(); - checkState(value instanceof DynamicMessage, "Row Field value should be DynamicMessage type."); - DynamicMessage record = (DynamicMessage) value; - List fields = ((RowType) type).getFields(); - for (Field field : fields) { - checkState(field.getName().isPresent(), "field name not found"); - serializeObject(singleRowBuilder, record.getField(((DynamicMessage) value).getDescriptorForType() - .findFieldByName(field.getName().get())), field.getType(), columnName); - - } - blockBuilder.closeEntry(); - if (parentBlockBuilder == null) { - return blockBuilder.getObject(0, Block.class); - } - return null; - } - - protected static final String PROTOBUF_MAP_KEY_NAME = "key"; - protected static final String PROTOBUF_MAP_VALUE_NAME = "value"; - private static final long MILLIS_PER_SECOND = 1000; - private static final long NANOS_PER_MILLISECOND = 1000000; -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/protobufnative/PulsarProtobufNativeRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/protobufnative/PulsarProtobufNativeRowDecoder.java deleted file mode 100644 index dcdf3e54c46f6..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/protobufnative/PulsarProtobufNativeRowDecoder.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.decoder.protobufnative; - -import static com.google.common.base.Functions.identity; -import static com.google.common.collect.ImmutableMap.toImmutableMap; -import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; -import static java.util.Objects.requireNonNull; -import com.google.protobuf.DynamicMessage; -import io.airlift.log.Logger; -import io.netty.buffer.ByteBuf; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.decoder.FieldValueProvider; -import io.trino.spi.TrinoException; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import org.apache.pulsar.client.impl.schema.generic.GenericProtobufNativeRecord; -import org.apache.pulsar.client.impl.schema.generic.GenericProtobufNativeSchema; -import org.apache.pulsar.sql.presto.PulsarRowDecoder; - -/** - * Pulsar {@link org.apache.pulsar.common.schema.SchemaType#PROTOBUF_NATIVE} RowDecoder. - */ -public class PulsarProtobufNativeRowDecoder implements PulsarRowDecoder { - - private final GenericProtobufNativeSchema genericProtobufNativeSchema; - private final Map columnDecoders; - - public PulsarProtobufNativeRowDecoder(GenericProtobufNativeSchema genericProtobufNativeSchema, - Set columns) { - this.genericProtobufNativeSchema = requireNonNull(genericProtobufNativeSchema, - "genericProtobufNativeSchema is null"); - columnDecoders = columns.stream() - .collect(toImmutableMap(identity(), this::createColumnDecoder)); - } - - private PulsarProtobufNativeColumnDecoder createColumnDecoder(DecoderColumnHandle columnHandle) { - return new PulsarProtobufNativeColumnDecoder(columnHandle); - } - - /** - * Decode ByteBuf by {@link org.apache.pulsar.client.api.schema.GenericSchema}. - * @param byteBuf - * @return - */ - @Override - public Optional> decodeRow(ByteBuf byteBuf) { - DynamicMessage dynamicMessage; - try { - GenericProtobufNativeRecord record = (GenericProtobufNativeRecord) genericProtobufNativeSchema - .decode(byteBuf); - dynamicMessage = record.getProtobufRecord(); - } catch (Exception e) { - log.error(e); - throw new TrinoException(GENERIC_INTERNAL_ERROR, "Decoding protobuf record failed.", e); - } - return Optional.of(columnDecoders.entrySet().stream() - .collect(toImmutableMap( - Map.Entry::getKey, - entry -> entry.getValue().decodeField(dynamicMessage)))); - } - - private static final Logger log = Logger.get(PulsarProtobufNativeRowDecoder.class); -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/protobufnative/PulsarProtobufNativeRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/protobufnative/PulsarProtobufNativeRowDecoderFactory.java deleted file mode 100644 index e16c42142677e..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/protobufnative/PulsarProtobufNativeRowDecoderFactory.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.decoder.protobufnative; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; -import static io.trino.spi.type.VarcharType.createUnboundedVarcharType; -import static java.util.stream.Collectors.toList; -import com.google.common.collect.ImmutableList; -import com.google.protobuf.Descriptors; -import com.google.protobuf.TimestampProto; -import io.airlift.log.Logger; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.BigintType; -import io.trino.spi.type.BooleanType; -import io.trino.spi.type.DoubleType; -import io.trino.spi.type.IntegerType; -import io.trino.spi.type.RealType; -import io.trino.spi.type.RowType; -import io.trino.spi.type.StandardTypes; -import io.trino.spi.type.TimestampType; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeManager; -import io.trino.spi.type.TypeSignature; -import io.trino.spi.type.TypeSignatureParameter; -import io.trino.spi.type.VarbinaryType; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.client.impl.schema.generic.GenericProtobufNativeSchema; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.sql.presto.PulsarColumnHandle; -import org.apache.pulsar.sql.presto.PulsarColumnMetadata; -import org.apache.pulsar.sql.presto.PulsarRowDecoder; -import org.apache.pulsar.sql.presto.PulsarRowDecoderFactory; - -/** - * PulsarRowDecoderFactory for {@link org.apache.pulsar.common.schema.SchemaType#PROTOBUF_NATIVE}. - */ -public class PulsarProtobufNativeRowDecoderFactory implements PulsarRowDecoderFactory { - - private final TypeManager typeManager; - - public PulsarProtobufNativeRowDecoderFactory(TypeManager typeManager) { - this.typeManager = typeManager; - } - - @Override - public PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, - Set columns) { - return new PulsarProtobufNativeRowDecoder((GenericProtobufNativeSchema) - GenericProtobufNativeSchema.of(schemaInfo), columns); - } - - @Override - public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, - PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { - List columnMetadata; - String schemaJson = new String(schemaInfo.getSchema()); - if (StringUtils.isBlank(schemaJson)) { - throw new TrinoException(NOT_SUPPORTED, "Topic " - + topicName.toString() + " does not have a valid schema"); - } - Descriptors.Descriptor schema; - try { - schema = - ((GenericProtobufNativeSchema) GenericProtobufNativeSchema.of(schemaInfo)) - .getProtobufNativeSchema(); - } catch (Exception ex) { - log.error(ex); - throw new TrinoException(NOT_SUPPORTED, "Topic " - + topicName.toString() + " does not have a valid schema"); - } - - //Protobuf have not yet supported Cyclic Objects. - columnMetadata = schema.getFields().stream() - .map(field -> - new PulsarColumnMetadata(PulsarColumnMetadata.getColumnName(handleKeyValueType, - field.getName()), parseProtobufPrestoType(field), field.getType().toString(), null, - false, false, handleKeyValueType, - new PulsarColumnMetadata.DecoderExtraInfo(field.getName(), null, null)) - - ).collect(toList()); - - return columnMetadata; - } - - private Type parseProtobufPrestoType(Descriptors.FieldDescriptor field) { - //parse by proto JavaType - Descriptors.FieldDescriptor.JavaType type = field.getJavaType(); - Type dataType; - switch (type) { - case BOOLEAN: - dataType = BooleanType.BOOLEAN; - break; - case BYTE_STRING: - dataType = VarbinaryType.VARBINARY; - break; - case DOUBLE: - dataType = DoubleType.DOUBLE; - break; - case ENUM: - case STRING: - dataType = createUnboundedVarcharType(); - break; - case FLOAT: - dataType = RealType.REAL; - break; - case INT: - dataType = IntegerType.INTEGER; - break; - case LONG: - dataType = BigintType.BIGINT; - break; - case MESSAGE: - Descriptors.Descriptor msg = field.getMessageType(); - if (field.isMapField()) { - //map - TypeSignature keyType = - parseProtobufPrestoType(msg.findFieldByName(PulsarProtobufNativeColumnDecoder - .PROTOBUF_MAP_KEY_NAME)).getTypeSignature(); - TypeSignature valueType = - parseProtobufPrestoType(msg.findFieldByName(PulsarProtobufNativeColumnDecoder - .PROTOBUF_MAP_VALUE_NAME)).getTypeSignature(); - return typeManager.getParameterizedType(StandardTypes.MAP, - ImmutableList.of(TypeSignatureParameter.typeParameter(keyType), - TypeSignatureParameter.typeParameter(valueType))); - } else { - if (TimestampProto.getDescriptor().toProto().getName().equals(msg.getFile().toProto().getName())) { - //if msg type is protobuf/timestamp - dataType = TimestampType.TIMESTAMP_MILLIS; - } else { - //row - dataType = RowType.from(msg.getFields().stream() - .map(rowField -> new RowType.Field(Optional.of(rowField.getName()), - parseProtobufPrestoType(rowField))) - .collect(toImmutableList())); - } - } - break; - default: - throw new RuntimeException("Unknown type: " + type.toString() + " for FieldDescriptor: " - + field.getName()); - } - //list - if (field.isRepeated() && !field.isMapField()) { - dataType = new ArrayType(dataType); - } - - return dataType; - } - - private static final Logger log = Logger.get(PulsarProtobufNativeRowDecoderFactory.class); - -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/util/NoStrictCacheSizeAllocator.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/util/NoStrictCacheSizeAllocator.java deleted file mode 100644 index 01fdb0886ebea..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/util/NoStrictCacheSizeAllocator.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.util; - -import java.util.concurrent.atomic.LongAdder; -import java.util.concurrent.locks.ReentrantLock; - -/** - * Cache size allocator. - */ -public class NoStrictCacheSizeAllocator implements CacheSizeAllocator { - - private final long maxCacheSize; - private final LongAdder availableCacheSize; - private final ReentrantLock lock; - - public NoStrictCacheSizeAllocator(long maxCacheSize) { - this.maxCacheSize = maxCacheSize; - this.availableCacheSize = new LongAdder(); - this.availableCacheSize.add(maxCacheSize); - this.lock = new ReentrantLock(); - } - - public long getAvailableCacheSize() { - if (availableCacheSize.longValue() < 0) { - return 0; - } - return availableCacheSize.longValue(); - } - - /** - * This operation will cost available cache size. - * if the request size exceed the available size, it's should be allowed, - * because maybe one entry size exceed the size and - * the query must be finished, the available size will become invalid. - * - * @param size allocate size - */ - public void allocate(long size) { - lock.lock(); - try { - availableCacheSize.add(-size); - } finally { - lock.unlock(); - } - } - - /** - * This method used to release used cache size and add available cache size. - * in normal case, the available size shouldn't exceed max cache size. - * - * @param size release size - */ - public void release(long size) { - lock.lock(); - try { - availableCacheSize.add(size); - if (availableCacheSize.longValue() > maxCacheSize) { - availableCacheSize.reset(); - availableCacheSize.add(maxCacheSize); - } - } finally { - lock.unlock(); - } - } - -} diff --git a/pulsar-sql/presto-pulsar/src/main/resources/META-INF/services/io.trino.spi.Plugin b/pulsar-sql/presto-pulsar/src/main/resources/META-INF/services/io.trino.spi.Plugin deleted file mode 100644 index a2e9401c32249..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/resources/META-INF/services/io.trino.spi.Plugin +++ /dev/null @@ -1 +0,0 @@ -org.apache.pulsar.sql.presto.PulsarPlugin \ No newline at end of file diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestCacheSizeAllocator.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestCacheSizeAllocator.java deleted file mode 100644 index 716cd2cf9d663..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestCacheSizeAllocator.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Sets; -import io.trino.spi.connector.ConnectorContext; -import io.trino.spi.predicate.TupleDomain; -import io.trino.testing.TestingConnectorContext; -import java.lang.reflect.Field; -import java.util.HashMap; -import java.util.List; -import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.mledger.Entry; -import org.apache.bookkeeper.mledger.ManagedLedgerConfig; -import org.apache.bookkeeper.mledger.ReadOnlyCursor; -import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.bookkeeper.stats.NullStatsProvider; -import org.apache.commons.lang3.RandomUtils; -import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; -import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.impl.MessageIdImpl; -import org.apache.pulsar.common.api.raw.RawMessageImpl; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; -import org.jctools.queues.SpscArrayQueue; -import org.testng.Assert; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - -/** - * Test cache size allocator. - */ -@Slf4j -public class TestCacheSizeAllocator extends MockedPulsarServiceBaseTest { - - private final int singleEntrySize = 500; - - @BeforeClass - @Override - public void setup() throws Exception { - super.internalSetup(); - - admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(brokerUrl.toString()).build()); - - // so that clients can test short names - admin.tenants().createTenant("public", - new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("test"))); - admin.namespaces().createNamespace("public/default"); - admin.namespaces().setNamespaceReplicationClusters("public/default", Sets.newHashSet("test")); - } - - @AfterClass - @Override - public void cleanup() throws Exception { - super.internalCleanup(); - } - - @DataProvider(name = "cacheSizeProvider") - public Object[][] dataProvider() { - return new Object[][] { - {-1}, {0}, {2000}, {5000} - }; - } - - @Test(dataProvider = "cacheSizeProvider", timeOut = 1000 * 20) - public void cacheSizeAllocatorTest(long entryQueueSizeBytes) throws Exception { - TopicName topicName = TopicName.get( - "public/default/cache-size-" + entryQueueSizeBytes + "test_" + + RandomUtils.nextInt()) ; - int totalMsgCnt = 1000; - MessageIdImpl firstMessageId = prepareData(topicName, totalMsgCnt); - - ReadOnlyCursor readOnlyCursor = pulsar.getManagedLedgerFactory().openReadOnlyCursor( - topicName.getPersistenceNamingEncoding(), - PositionImpl.get(firstMessageId.getLedgerId(), firstMessageId.getEntryId()), - new ManagedLedgerConfig()); - readOnlyCursor.skipEntries(totalMsgCnt); - PositionImpl lastPosition = (PositionImpl) readOnlyCursor.getReadPosition(); - - ObjectMapper objectMapper = new ObjectMapper(); - - PulsarSplit pulsarSplit = new PulsarSplit( - 0, - "connector-id", - topicName.getNamespace(), - topicName.getLocalName(), - topicName.getLocalName(), - totalMsgCnt, - new String(Schema.BYTES.getSchemaInfo().getSchema()), - Schema.BYTES.getSchemaInfo().getType(), - firstMessageId.getEntryId(), - lastPosition.getEntryId(), - firstMessageId.getLedgerId(), - lastPosition.getLedgerId(), - TupleDomain.all(), - objectMapper.writeValueAsString(new HashMap<>()), - null); - - List pulsarColumnHandles = TestPulsarConnector.getColumnColumnHandles( - topicName, Schema.BYTES.getSchemaInfo(), PulsarColumnHandle.HandleKeyValueType.NONE, true); - - PulsarConnectorConfig connectorConfig = new PulsarConnectorConfig(); - connectorConfig.setMaxSplitQueueSizeBytes(entryQueueSizeBytes); - - ConnectorContext prestoConnectorContext = new TestingConnectorContext(); - PulsarRecordCursor pulsarRecordCursor = new PulsarRecordCursor( - pulsarColumnHandles, pulsarSplit, connectorConfig, pulsar.getManagedLedgerFactory(), - new ManagedLedgerConfig(), new PulsarConnectorMetricsTracker(new NullStatsProvider()), - new PulsarDispatchingRowDecoderFactory(prestoConnectorContext.getTypeManager())); - - Class recordCursorClass = PulsarRecordCursor.class; - Field entryQueueField = recordCursorClass.getDeclaredField("entryQueue"); - entryQueueField.setAccessible(true); - SpscArrayQueue entryQueue = (SpscArrayQueue) entryQueueField.get(pulsarRecordCursor); - - Field messageQueueField = recordCursorClass.getDeclaredField("messageQueue"); - messageQueueField.setAccessible(true); - SpscArrayQueue messageQueue = - (SpscArrayQueue) messageQueueField.get(pulsarRecordCursor); - - long maxQueueSize = 0; - if (entryQueueSizeBytes == -1) { - maxQueueSize = Long.MAX_VALUE; - } else if (entryQueueSizeBytes == 0) { - maxQueueSize = 1; - } else if (entryQueueSizeBytes > 0) { - maxQueueSize = entryQueueSizeBytes / 2 / singleEntrySize + 1; - } - - int receiveCnt = 0; - while (receiveCnt != totalMsgCnt) { - if (pulsarRecordCursor.advanceNextPosition()) { - receiveCnt ++; - } - Assert.assertTrue(entryQueue.size() <= maxQueueSize); - Assert.assertTrue(messageQueue.size() <= maxQueueSize); - } - } - - private MessageIdImpl prepareData(TopicName topicName, int messageNum) throws Exception { - Producer producer = pulsarClient.newProducer() - .topic(topicName.toString()) - .create(); - - MessageIdImpl firstMessageId = null; - for (int i = 0; i < messageNum; i++) { - MessageIdImpl messageId = (MessageIdImpl) producer.send(new byte[singleEntrySize]); - if (firstMessageId == null) { - firstMessageId = messageId; - } - } - return firstMessageId; - } - -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestNoStrictCacheSizeAllocator.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestNoStrictCacheSizeAllocator.java deleted file mode 100644 index 7f29e5d251191..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestNoStrictCacheSizeAllocator.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import org.apache.pulsar.sql.presto.util.NoStrictCacheSizeAllocator; -import org.testng.Assert; -import org.testng.annotations.Test; - -/** - * Cache size allocator test. - */ -public class TestNoStrictCacheSizeAllocator { - - @Test - public void allocatorTest() { - NoStrictCacheSizeAllocator noStrictCacheSizeAllocator = new NoStrictCacheSizeAllocator(1000); - Assert.assertEquals(noStrictCacheSizeAllocator.getAvailableCacheSize(), 1000); - - noStrictCacheSizeAllocator.allocate(500); - Assert.assertEquals(noStrictCacheSizeAllocator.getAvailableCacheSize(), 1000 - 500); - - noStrictCacheSizeAllocator.allocate(600); - Assert.assertEquals(noStrictCacheSizeAllocator.getAvailableCacheSize(), 0); - - noStrictCacheSizeAllocator.release(500 + 600); - Assert.assertEquals(noStrictCacheSizeAllocator.getAvailableCacheSize(), 1000); - - noStrictCacheSizeAllocator.release(100); - Assert.assertEquals(noStrictCacheSizeAllocator.getAvailableCacheSize(), 1000); - } - -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarAuth.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarAuth.java deleted file mode 100644 index 9119ffed4e28f..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarAuth.java +++ /dev/null @@ -1,250 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import static io.trino.spi.StandardErrorCode.PERMISSION_DENIED; -import static io.trino.spi.StandardErrorCode.QUERY_REJECTED; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import com.google.common.collect.Sets; -import io.jsonwebtoken.SignatureAlgorithm; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.security.ConnectorIdentity; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.Optional; -import java.util.Properties; -import javax.crypto.SecretKey; -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; -import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; -import org.apache.pulsar.client.admin.PulsarAdminBuilder; -import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.api.AuthenticationFactory; -import org.apache.pulsar.common.policies.data.AuthAction; -import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; -import org.testng.Assert; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -public class TestPulsarAuth extends MockedPulsarServiceBaseTest { - private SecretKey secretKey; - private final String SUPER_USER_ROLE = "admin"; - - @BeforeClass - @Override - public void setup() throws Exception { - conf.setAuthenticationEnabled(true); - conf.setAuthenticationProviders( - Sets.newHashSet("org.apache.pulsar.broker.authentication.AuthenticationProviderToken")); - conf.setAuthorizationEnabled(true); - secretKey = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); - Properties properties = new Properties(); - properties.setProperty("tokenSecretKey", AuthTokenUtils.encodeKeyBase64(secretKey)); - conf.setProperties(properties); - conf.setSuperUserRoles(Sets.newHashSet(SUPER_USER_ROLE)); - conf.setClusterName("c1"); - internalSetup(); - - admin.clusters().createCluster("c1", ClusterData.builder().build()); - admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet(SUPER_USER_ROLE), Sets.newHashSet("c1"))); - waitForChange(); - admin.namespaces().createNamespace("p1/c1/ns1"); - waitForChange(); - } - - @Override - protected void customizeNewPulsarAdminBuilder(PulsarAdminBuilder pulsarAdminBuilder) { - pulsarAdminBuilder.authentication( - AuthenticationFactory.token(AuthTokenUtils.createToken(secretKey, SUPER_USER_ROLE, Optional.empty()))); - } - - @AfterClass - @Override - public void cleanup() throws Exception { - internalCleanup(); - } - - @Test(expectedExceptions = IllegalArgumentException.class) - public void testConfigCheck() { - PulsarConnectorConfig pulsarConnectorConfig = new PulsarConnectorConfig(); - pulsarConnectorConfig.setAuthorizationEnabled(true); - pulsarConnectorConfig.setBrokerBinaryServiceUrl(""); - - new PulsarAuth(pulsarConnectorConfig); - } - - @Test - public void testEmptyExtraCredentials() { - PulsarConnectorConfig pulsarConnectorConfig = mock(PulsarConnectorConfig.class); - - doReturn(true).when(pulsarConnectorConfig).getAuthorizationEnabled(); - doReturn(pulsar.getBrokerServiceUrl()).when(pulsarConnectorConfig).getBrokerBinaryServiceUrl(); - - PulsarAuth pulsarAuth = new PulsarAuth(pulsarConnectorConfig); - - ConnectorSession session = mock(ConnectorSession.class); - ConnectorIdentity identity = mock(ConnectorIdentity.class); - doReturn("query-1").when(session).getQueryId(); - doReturn(identity).when(session).getIdentity(); - - // Test empty extra credentials map - doReturn(new HashMap()).when(identity).getExtraCredentials(); - try { - pulsarAuth.checkTopicAuth(session, "test"); - Assert.fail(); // should fail - } catch (TrinoException e) { - Assert.assertEquals(QUERY_REJECTED.toErrorCode(), e.getErrorCode()); - Assert.assertTrue(e.getMessage().contains("The credential information is empty")); - } - - // Test empty extra credentials parameters - doReturn(new HashMap() {{ - put("auth-plugin", "org.apache.pulsar.client.impl.auth.AuthenticationToken"); - }}).when(identity).getExtraCredentials(); - try { - pulsarAuth.checkTopicAuth(session, "test"); - Assert.fail(); // should fail - } catch (TrinoException e) { - Assert.assertEquals(QUERY_REJECTED.toErrorCode(), e.getErrorCode()); - Assert.assertTrue(e.getMessage().contains("Please specify the auth-method and auth-params")); - } - - doReturn(new HashMap() {{ - put("auth-params", "test-token"); - }}).when(identity).getExtraCredentials(); - try { - pulsarAuth.checkTopicAuth(session, "test"); - Assert.fail(); // should fail - } catch (TrinoException e) { - Assert.assertEquals(QUERY_REJECTED.toErrorCode(), e.getErrorCode()); - Assert.assertTrue(e.getMessage().contains("Please specify the auth-method and auth-params")); - } - } - - @Test - public void testPulsarSqlAuth() throws PulsarAdminException { - String passRole = RandomStringUtils.randomAlphabetic(4) + "-pass"; - String deniedRole = RandomStringUtils.randomAlphabetic(4) + "-denied"; - String topic = "persistent://p1/c1/ns1/" + RandomStringUtils.randomAlphabetic(4); - String otherTopic = "persistent://p1/c1/ns1/" + RandomStringUtils.randomAlphabetic(4) + "-other"; - String partitionedTopic = "persistent://p1/c1/ns1/" + RandomStringUtils.randomAlphabetic(4); - String passToken = AuthTokenUtils.createToken(secretKey, passRole, Optional.empty()); - String deniedToken = AuthTokenUtils.createToken(secretKey, deniedRole, Optional.empty()); - - admin.topics().grantPermission(topic, passRole, EnumSet.of(AuthAction.consume)); - admin.topics().createPartitionedTopic(partitionedTopic, 2); - admin.topics().grantPermission(partitionedTopic, passRole, EnumSet.of(AuthAction.consume)); - waitForChange(); - - ConnectorSession session = mock(ConnectorSession.class); - ConnectorIdentity identity = mock(ConnectorIdentity.class); - PulsarConnectorConfig pulsarConnectorConfig = mock(PulsarConnectorConfig.class); - - doReturn(true).when(pulsarConnectorConfig).getAuthorizationEnabled(); - doReturn(pulsar.getBrokerServiceUrl()).when(pulsarConnectorConfig).getBrokerBinaryServiceUrl(); - - doReturn("query-1").when(session).getQueryId(); - doReturn(identity).when(session).getIdentity(); - - doReturn(new HashMap() {{ - put("auth-plugin", "org.apache.pulsar.client.impl.auth.AuthenticationToken"); - put("auth-params", passToken); - }}).when(identity).getExtraCredentials(); - - PulsarAuth pulsarAuth = new PulsarAuth(pulsarConnectorConfig); - - pulsarAuth.checkTopicAuth(session, topic); // should pass - - // authorizedQueryTopicPairs should contain the authorized query and topic. - Assert.assertTrue( - pulsarAuth.authorizedQueryTopicsMap.containsKey(session.getQueryId())); - Assert.assertTrue(pulsarAuth.authorizedQueryTopicsMap.get(session.getQueryId()).contains(topic)); - - // Using the authorized query but not authorized topic should fail. - // This part of the test case is for the case where a query accesses multiple topics but only some of them - // have permission. - try { - pulsarAuth.checkTopicAuth(session, otherTopic); - Assert.fail(); // should fail - } catch (TrinoException e){ - Assert.assertEquals(PERMISSION_DENIED.toErrorCode(), e.getErrorCode()); - Assert.assertTrue(e.getMessage().contains("not authorized")); - } - - // test clean session - pulsarAuth.cleanSession(session); - - Assert.assertFalse(pulsarAuth.authorizedQueryTopicsMap.containsKey(session.getQueryId())); - - doReturn("test-fail").when(session).getQueryId(); - - doReturn("query-2").when(session).getQueryId(); - - try{ - doReturn(new HashMap() {{ - put("auth-plugin", "org.apache.pulsar.client.impl.auth.AuthenticationToken"); - put("auth-params", "invalid-token"); - }}).when(identity).getExtraCredentials(); - pulsarAuth.checkTopicAuth(session, topic); - Assert.fail(); // should fail - } catch (TrinoException e){ - Assert.assertEquals(PERMISSION_DENIED.toErrorCode(), e.getErrorCode()); - Assert.assertTrue(e.getMessage().contains("Failed to authenticate")); - } - - pulsarAuth.cleanSession(session); - Assert.assertTrue(pulsarAuth.authorizedQueryTopicsMap.isEmpty()); - - doReturn("query-3").when(session).getQueryId(); - - try{ - doReturn(new HashMap() {{ - put("auth-plugin", "org.apache.pulsar.client.impl.auth.AuthenticationToken"); - put("auth-params", deniedToken); - }}).when(identity).getExtraCredentials(); - pulsarAuth.checkTopicAuth(session, topic); - Assert.fail(); // should fail - } catch (TrinoException e){ - Assert.assertEquals(PERMISSION_DENIED.toErrorCode(), e.getErrorCode()); - Assert.assertTrue(e.getMessage().contains("not authorized")); - } - - pulsarAuth.cleanSession(session); - - doReturn(new HashMap() {{ - put("auth-plugin", "org.apache.pulsar.client.impl.auth.AuthenticationToken"); - put("auth-params", passToken); - }}).when(identity).getExtraCredentials(); - pulsarAuth.checkTopicAuth(session, topic); // should pass for the partitioned topic case - - pulsarAuth.cleanSession(session); - Assert.assertTrue(pulsarAuth.authorizedQueryTopicsMap.isEmpty()); - } - - private static void waitForChange() { - try { - Thread.sleep(100); - } catch (InterruptedException ignored) { - } - } -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java deleted file mode 100644 index 61f6edd38530c..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java +++ /dev/null @@ -1,737 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import static org.apache.pulsar.common.protocol.Commands.serializeMetadataAndPayload; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.anyInt; -import static org.mockito.Mockito.anyString; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertNotNull; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.airlift.log.Logger; -import io.netty.buffer.ByteBuf; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorContext; -import io.trino.spi.predicate.TupleDomain; -import io.trino.testing.TestingConnectorContext; -import java.math.BigDecimal; -import java.time.LocalDate; -import java.time.LocalTime; -import java.time.ZoneId; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import javax.ws.rs.ClientErrorException; -import javax.ws.rs.core.Response; -import org.apache.bookkeeper.mledger.AsyncCallbacks; -import org.apache.bookkeeper.mledger.Entry; -import org.apache.bookkeeper.mledger.ManagedLedgerConfig; -import org.apache.bookkeeper.mledger.ManagedLedgerFactory; -import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.ReadOnlyCursor; -import org.apache.bookkeeper.mledger.impl.EntryImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.bookkeeper.mledger.impl.ReadOnlyCursorImpl; -import org.apache.bookkeeper.mledger.proto.MLDataFormats; -import org.apache.bookkeeper.stats.NullStatsProvider; -import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.client.admin.Namespaces; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.admin.Schemas; -import org.apache.pulsar.client.admin.Tenants; -import org.apache.pulsar.client.admin.Topics; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.api.schema.SchemaDefinition; -import org.apache.pulsar.client.impl.schema.AvroSchema; -import org.apache.pulsar.client.impl.schema.JSONSchema; -import org.apache.pulsar.common.api.proto.MessageMetadata; -import org.apache.pulsar.common.naming.NamespaceName; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.partition.PartitionedTopicMetadata; -import org.apache.pulsar.common.protocol.Commands; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.common.schema.SchemaType; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.DataProvider; - -public abstract class TestPulsarConnector { - - protected static final long currentTimeMicros = 1534806330000000L; - - protected PulsarConnectorConfig pulsarConnectorConfig; - - protected PulsarMetadata pulsarMetadata; - - protected PulsarAuth pulsarAuth; - - protected PulsarAdmin pulsarAdmin; - - protected Schemas schemas; - - protected PulsarSplitManager pulsarSplitManager; - - protected Map pulsarRecordCursors = new HashMap<>(); - - protected static PulsarDispatchingRowDecoderFactory dispatchingRowDecoderFactory; - - protected static final PulsarConnectorId pulsarConnectorId = new PulsarConnectorId("test-connector"); - - protected static List topicNames; - protected static List partitionedTopicNames; - protected static Map partitionedTopicsToPartitions; - protected static Map topicsToSchemas; - protected static Map topicsToNumEntries; - - private static final ObjectMapper objectMapper = new ObjectMapper(); - - protected static List fooFieldNames = new ArrayList<>(); - - protected static final NamespaceName NAMESPACE_NAME_1 = NamespaceName.get("tenant-1", "ns-1"); - protected static final NamespaceName NAMESPACE_NAME_2 = NamespaceName.get("tenant-1", "ns-2"); - protected static final NamespaceName NAMESPACE_NAME_3 = NamespaceName.get("tenant-2", "ns-1"); - protected static final NamespaceName NAMESPACE_NAME_4 = NamespaceName.get("tenant-2", "ns-2"); - - protected static final TopicName TOPIC_1 = TopicName.get("persistent", NAMESPACE_NAME_1, "topic-1"); - protected static final TopicName TOPIC_2 = TopicName.get("persistent", NAMESPACE_NAME_1, "topic-2"); - protected static final TopicName TOPIC_3 = TopicName.get("persistent", NAMESPACE_NAME_2, "topic-1"); - protected static final TopicName TOPIC_4 = TopicName.get("persistent", NAMESPACE_NAME_3, "topic-1"); - protected static final TopicName TOPIC_5 = TopicName.get("persistent", NAMESPACE_NAME_4, "topic-1"); - protected static final TopicName TOPIC_6 = TopicName.get("persistent", NAMESPACE_NAME_4, "topic-2"); - protected static final TopicName NON_SCHEMA_TOPIC = TopicName.get( - "persistent", NAMESPACE_NAME_2, "non-schema-topic"); - - - protected static final TopicName PARTITIONED_TOPIC_1 = TopicName.get("persistent", NAMESPACE_NAME_1, - "partitioned-topic-1"); - protected static final TopicName PARTITIONED_TOPIC_2 = TopicName.get("persistent", NAMESPACE_NAME_1, - "partitioned-topic-2"); - protected static final TopicName PARTITIONED_TOPIC_3 = TopicName.get("persistent", NAMESPACE_NAME_2, - "partitioned-topic-1"); - protected static final TopicName PARTITIONED_TOPIC_4 = TopicName.get("persistent", NAMESPACE_NAME_3, - "partitioned-topic-1"); - protected static final TopicName PARTITIONED_TOPIC_5 = TopicName.get("persistent", NAMESPACE_NAME_4, - "partitioned-topic-1"); - protected static final TopicName PARTITIONED_TOPIC_6 = TopicName.get("persistent", NAMESPACE_NAME_4, - "partitioned-topic-2"); - - - public static class Foo { - public enum TestEnum { - TEST_ENUM_1, - TEST_ENUM_2, - TEST_ENUM_3 - } - - public int field1; - public String field2; - public float field3; - public double field4; - public boolean field5; - public long field6; - @org.apache.avro.reflect.AvroSchema("{ \"type\": \"long\", \"logicalType\": \"timestamp-millis\" }") - public long timestamp; - @org.apache.avro.reflect.AvroSchema("{ \"type\": \"int\", \"logicalType\": \"time-millis\" }") - public int time; - @org.apache.avro.reflect.AvroSchema("{ \"type\": \"int\", \"logicalType\": \"date\" }") - public int date; - @org.apache.avro.reflect.AvroSchema("{ \"type\": \"bytes\", \"logicalType\": \"decimal\", \"precision\": 4, \"scale\": 2 }") - public BigDecimal decimal; - public TestPulsarConnector.Bar bar; - public TestEnum field7; - } - - public static class Bar { - public Integer field1; - public String field2; - public float field3; - } - - - protected static Map> topicsToColumnHandles = new HashMap<>(); - - protected static Map splits; - protected static Map> fooFunctions; - - static { - try { - topicNames = new LinkedList<>(); - topicNames.add(TOPIC_1); - topicNames.add(TOPIC_2); - topicNames.add(TOPIC_3); - topicNames.add(TOPIC_4); - topicNames.add(TOPIC_5); - topicNames.add(TOPIC_6); - topicNames.add(NON_SCHEMA_TOPIC); - - partitionedTopicNames = new LinkedList<>(); - partitionedTopicNames.add(PARTITIONED_TOPIC_1); - partitionedTopicNames.add(PARTITIONED_TOPIC_2); - partitionedTopicNames.add(PARTITIONED_TOPIC_3); - partitionedTopicNames.add(PARTITIONED_TOPIC_4); - partitionedTopicNames.add(PARTITIONED_TOPIC_5); - partitionedTopicNames.add(PARTITIONED_TOPIC_6); - - - partitionedTopicsToPartitions = new HashMap<>(); - partitionedTopicsToPartitions.put(PARTITIONED_TOPIC_1.toString(), 2); - partitionedTopicsToPartitions.put(PARTITIONED_TOPIC_2.toString(), 3); - partitionedTopicsToPartitions.put(PARTITIONED_TOPIC_3.toString(), 4); - partitionedTopicsToPartitions.put(PARTITIONED_TOPIC_4.toString(), 5); - partitionedTopicsToPartitions.put(PARTITIONED_TOPIC_5.toString(), 6); - partitionedTopicsToPartitions.put(PARTITIONED_TOPIC_6.toString(), 7); - - topicsToSchemas = new HashMap<>(); - topicsToSchemas.put(TOPIC_1.getSchemaName(), Schema.AVRO(TestPulsarMetadata.Foo.class).getSchemaInfo()); - topicsToSchemas.put(TOPIC_2.getSchemaName(), Schema.AVRO(TestPulsarMetadata.Foo.class).getSchemaInfo()); - topicsToSchemas.put(TOPIC_3.getSchemaName(), Schema.AVRO(TestPulsarMetadata.Foo.class).getSchemaInfo()); - topicsToSchemas.put(TOPIC_4.getSchemaName(), Schema.JSON(TestPulsarMetadata.Foo.class).getSchemaInfo()); - topicsToSchemas.put(TOPIC_5.getSchemaName(), Schema.JSON(TestPulsarMetadata.Foo.class).getSchemaInfo()); - topicsToSchemas.put(TOPIC_6.getSchemaName(), Schema.JSON(TestPulsarMetadata.Foo.class).getSchemaInfo()); - - topicsToSchemas.put(PARTITIONED_TOPIC_1.getSchemaName(), Schema.AVRO(TestPulsarMetadata.Foo.class).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_2.getSchemaName(), Schema.AVRO(TestPulsarMetadata.Foo.class).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_3.getSchemaName(), Schema.AVRO(TestPulsarMetadata.Foo.class).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_4.getSchemaName(), Schema.JSON(TestPulsarMetadata.Foo.class).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_5.getSchemaName(), Schema.JSON(TestPulsarMetadata.Foo.class).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_6.getSchemaName(), Schema.JSON(TestPulsarMetadata.Foo.class).getSchemaInfo()); - - topicsToNumEntries = new HashMap<>(); - topicsToNumEntries.put(TOPIC_1.getSchemaName(), 1233L); - topicsToNumEntries.put(TOPIC_2.getSchemaName(), 0L); - topicsToNumEntries.put(TOPIC_3.getSchemaName(), 100L); - topicsToNumEntries.put(TOPIC_4.getSchemaName(), 12345L); - topicsToNumEntries.put(TOPIC_5.getSchemaName(), 8000L); - topicsToNumEntries.put(TOPIC_6.getSchemaName(), 1L); - - topicsToNumEntries.put(NON_SCHEMA_TOPIC.getSchemaName(), 8000L); - topicsToNumEntries.put(PARTITIONED_TOPIC_1.getSchemaName(), 1233L); - topicsToNumEntries.put(PARTITIONED_TOPIC_2.getSchemaName(), 8000L); - topicsToNumEntries.put(PARTITIONED_TOPIC_3.getSchemaName(), 100L); - topicsToNumEntries.put(PARTITIONED_TOPIC_4.getSchemaName(), 0L); - topicsToNumEntries.put(PARTITIONED_TOPIC_5.getSchemaName(), 800L); - topicsToNumEntries.put(PARTITIONED_TOPIC_6.getSchemaName(), 1L); - - - fooFieldNames.add("field1"); - fooFieldNames.add("field2"); - fooFieldNames.add("field3"); - fooFieldNames.add("field4"); - fooFieldNames.add("field5"); - fooFieldNames.add("field6"); - fooFieldNames.add("timestamp"); - fooFieldNames.add("time"); - fooFieldNames.add("date"); - fooFieldNames.add("bar"); - fooFieldNames.add("field7"); - fooFieldNames.add("decimal"); - - - ConnectorContext prestoConnectorContext = new TestingConnectorContext(); - dispatchingRowDecoderFactory = new PulsarDispatchingRowDecoderFactory(prestoConnectorContext.getTypeManager()); - - topicsToColumnHandles.put(PARTITIONED_TOPIC_1, getColumnColumnHandles(PARTITIONED_TOPIC_1,topicsToSchemas.get(PARTITIONED_TOPIC_1.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); - topicsToColumnHandles.put(PARTITIONED_TOPIC_2, getColumnColumnHandles(PARTITIONED_TOPIC_2,topicsToSchemas.get(PARTITIONED_TOPIC_2.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); - topicsToColumnHandles.put(PARTITIONED_TOPIC_3, getColumnColumnHandles(PARTITIONED_TOPIC_3,topicsToSchemas.get(PARTITIONED_TOPIC_3.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); - topicsToColumnHandles.put(PARTITIONED_TOPIC_4, getColumnColumnHandles(PARTITIONED_TOPIC_4,topicsToSchemas.get(PARTITIONED_TOPIC_4.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); - topicsToColumnHandles.put(PARTITIONED_TOPIC_5, getColumnColumnHandles(PARTITIONED_TOPIC_5,topicsToSchemas.get(PARTITIONED_TOPIC_5.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); - topicsToColumnHandles.put(PARTITIONED_TOPIC_6, getColumnColumnHandles(PARTITIONED_TOPIC_6,topicsToSchemas.get(PARTITIONED_TOPIC_6.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); - - - topicsToColumnHandles.put(TOPIC_1, getColumnColumnHandles(TOPIC_1,topicsToSchemas.get(TOPIC_1.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); - topicsToColumnHandles.put(TOPIC_2, getColumnColumnHandles(TOPIC_2,topicsToSchemas.get(TOPIC_2.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); - topicsToColumnHandles.put(TOPIC_3, getColumnColumnHandles(TOPIC_3,topicsToSchemas.get(TOPIC_3.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); - topicsToColumnHandles.put(TOPIC_4, getColumnColumnHandles(TOPIC_4,topicsToSchemas.get(TOPIC_4.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); - topicsToColumnHandles.put(TOPIC_5, getColumnColumnHandles(TOPIC_5,topicsToSchemas.get(TOPIC_5.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); - topicsToColumnHandles.put(TOPIC_6, getColumnColumnHandles(TOPIC_6,topicsToSchemas.get(TOPIC_6.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); - - - splits = new HashMap<>(); - - List allTopics = new LinkedList<>(); - allTopics.addAll(topicNames); - allTopics.addAll(partitionedTopicNames); - - - for (TopicName topicName : allTopics) { - if (topicsToSchemas.containsKey(topicName.getSchemaName())) { - splits.put(topicName, new PulsarSplit(0, pulsarConnectorId.toString(), - topicName.getNamespace(), topicName.getLocalName(), topicName.getLocalName(), - topicsToNumEntries.get(topicName.getSchemaName()), - new String(topicsToSchemas.get(topicName.getSchemaName()).getSchema()), - topicsToSchemas.get(topicName.getSchemaName()).getType(), - 0, topicsToNumEntries.get(topicName.getSchemaName()), - 0, 0, TupleDomain.all(), - objectMapper.writeValueAsString( - topicsToSchemas.get(topicName.getSchemaName()).getProperties()), null)); - } - } - - fooFunctions = new HashMap<>(); - - fooFunctions.put("field1", integer -> integer); - fooFunctions.put("field2", String::valueOf); - fooFunctions.put("field3", Integer::floatValue); - fooFunctions.put("field4", Integer::doubleValue); - fooFunctions.put("field5", integer -> integer % 2 == 0); - fooFunctions.put("field6", Integer::longValue); - fooFunctions.put("timestamp", integer -> System.currentTimeMillis()); - fooFunctions.put("time", integer -> { - LocalTime now = LocalTime.now(ZoneId.systemDefault()); - return now.toSecondOfDay() * 1000; - }); - fooFunctions.put("date", integer -> { - LocalDate localDate = LocalDate.now(); - LocalDate epoch = LocalDate.ofEpochDay(0); - return Math.toIntExact(ChronoUnit.DAYS.between(epoch, localDate)); - }); - fooFunctions.put("decimal", integer -> BigDecimal.valueOf(1234, 2)); - fooFunctions.put("bar.field1", integer -> integer % 3 == 0 ? null : integer + 1); - fooFunctions.put("bar.field2", integer -> integer % 2 == 0 ? null : String.valueOf(integer + 2)); - fooFunctions.put("bar.field3", integer -> integer + 3.0f); - - fooFunctions.put("field7", integer -> Foo.TestEnum.values()[integer % Foo.TestEnum.values().length]); - - } catch (Throwable e) { - System.out.println("Error: " + e); - System.out.println("Stacktrace: " + Arrays.asList(e.getStackTrace())); - } - } - - - /** - * Parse PulsarColumnMetadata to PulsarColumnHandle Util - * @param schemaInfo - * @param handleKeyValueType - * @param includeInternalColumn - * @return - */ - protected static List getColumnColumnHandles(TopicName topicName, SchemaInfo schemaInfo, - PulsarColumnHandle.HandleKeyValueType handleKeyValueType, boolean includeInternalColumn) { - List columnHandles = new ArrayList<>(); - List columnMetadata = mockColumnMetadata().getPulsarColumns(topicName, schemaInfo, - includeInternalColumn, handleKeyValueType); - columnMetadata.forEach(column -> { - PulsarColumnMetadata pulsarColumnMetadata = (PulsarColumnMetadata) column; - columnHandles.add(new PulsarColumnHandle( - pulsarConnectorId.toString(), - pulsarColumnMetadata.getNameWithCase(), - pulsarColumnMetadata.getType(), - pulsarColumnMetadata.isHidden(), - pulsarColumnMetadata.isInternal(), - pulsarColumnMetadata.getDecoderExtraInfo().getMapping(), - pulsarColumnMetadata.getDecoderExtraInfo().getDataFormat(), pulsarColumnMetadata.getDecoderExtraInfo().getFormatHint(), - pulsarColumnMetadata.getHandleKeyValueType())); - - }); - return columnHandles; - } - - public static PulsarMetadata mockColumnMetadata() { - ConnectorContext prestoConnectorContext = new TestingConnectorContext(); - PulsarConnectorConfig pulsarConnectorConfig = spy(PulsarConnectorConfig.class); - pulsarConnectorConfig.setMaxEntryReadBatchSize(1); - pulsarConnectorConfig.setMaxSplitEntryQueueSize(10); - pulsarConnectorConfig.setMaxSplitMessageQueueSize(100); - PulsarDispatchingRowDecoderFactory dispatchingRowDecoderFactory = - new PulsarDispatchingRowDecoderFactory(prestoConnectorContext.getTypeManager()); - PulsarAuth pulsarAuth = new PulsarAuth(pulsarConnectorConfig); - PulsarMetadata pulsarMetadata = - new PulsarMetadata(pulsarConnectorId, pulsarConnectorConfig, dispatchingRowDecoderFactory, pulsarAuth); - return pulsarMetadata; - } - - public static PulsarConnectorId getPulsarConnectorId() { - assertNotNull(pulsarConnectorId); - return pulsarConnectorId; - } - - private static List getTopicEntries(String topicSchemaName) { - List entries = new LinkedList<>(); - - long count = topicsToNumEntries.get(topicSchemaName); - for (int i=0 ; i < count; i++) { - - Foo foo = new Foo(); - foo.field1 = (int) count; - foo.field2 = String.valueOf(count); - foo.field3 = count; - foo.field4 = count; - foo.field5 = count % 2 == 0; - foo.field6 = count; - foo.timestamp = System.currentTimeMillis(); - - LocalTime now = LocalTime.now(ZoneId.systemDefault()); - foo.time = now.toSecondOfDay() * 1000; - - LocalDate localDate = LocalDate.now(); - LocalDate epoch = LocalDate.ofEpochDay(0); - foo.date = Math.toIntExact(ChronoUnit.DAYS.between(epoch, localDate)); - foo.decimal= BigDecimal.valueOf(count, 2); - - MessageMetadata messageMetadata = new MessageMetadata() - .setProducerName("test-producer").setSequenceId(i) - .setPublishTime(currentTimeMicros / 1000 + i); - - Schema schema = topicsToSchemas.get(topicSchemaName).getType() == SchemaType.AVRO ? AvroSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()) : JSONSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); - - ByteBuf payload = io.netty.buffer.Unpooled - .copiedBuffer(schema.encode(foo)); - - ByteBuf byteBuf = serializeMetadataAndPayload( - Commands.ChecksumType.Crc32c, messageMetadata, payload); - - Entry entry = EntryImpl.create(0, i, byteBuf); - log.info("create entry: %s", entry.getEntryId()); - entries.add(entry); - } - return entries; - } - - public long completedBytes = 0L; - - private static final Logger log = Logger.get(TestPulsarConnector.class); - - protected static List getNamespace(String tenant) { - return topicNames.stream() - .filter(topicName -> topicName.getTenant().equals(tenant)) - .map(TopicName::getNamespace) - .distinct() - .collect(Collectors.toCollection(LinkedList::new)); - } - - protected static List getTopics(String ns) { - List topics = new ArrayList<>(topicNames.stream() - .filter(topicName -> topicName.getNamespace().equals(ns)) - .map(TopicName::toString).collect(Collectors.toList())); - partitionedTopicNames.stream().filter(topicName -> topicName.getNamespace().equals(ns)).forEach(topicName -> { - for (Integer i = 0; i < partitionedTopicsToPartitions.get(topicName.toString()); i++) { - topics.add(TopicName.get(topicName + "-partition-" + i).toString()); - } - }); - return topics; - } - - protected static List getPartitionedTopics(String ns) { - return partitionedTopicNames.stream() - .filter(topicName -> topicName.getNamespace().equals(ns)) - .map(TopicName::toString) - .collect(Collectors.toList()); - } - - @BeforeMethod - public void setup() throws Exception { - this.pulsarConnectorConfig = spy(PulsarConnectorConfig.class); - this.pulsarConnectorConfig.setMaxEntryReadBatchSize(1); - this.pulsarConnectorConfig.setMaxSplitEntryQueueSize(10); - this.pulsarConnectorConfig.setMaxSplitMessageQueueSize(100); - - Tenants tenants = mock(Tenants.class); - doReturn(new LinkedList<>(topicNames.stream() - .map(TopicName::getTenant) - .collect(Collectors.toSet()))).when(tenants).getTenants(); - - Namespaces namespaces = mock(Namespaces.class); - - when(namespaces.getNamespaces(anyString())).thenAnswer(new Answer>() { - @Override - public List answer(InvocationOnMock invocation) throws Throwable { - Object[] args = invocation.getArguments(); - String tenant = (String) args[0]; - List ns = getNamespace(tenant); - if (ns.isEmpty()) { - ClientErrorException cee = new ClientErrorException(Response.status(404).build()); - throw new PulsarAdminException(cee, cee.getMessage(), cee.getResponse().getStatus()); - } - return ns; - } - }); - - Topics topics = mock(Topics.class); - when(topics.getList(anyString(), any())).thenAnswer(new Answer>() { - @Override - public List answer(InvocationOnMock invocationOnMock) throws Throwable { - Object[] args = invocationOnMock.getArguments(); - String ns = (String) args[0]; - List topics = getTopics(ns); - if (topics.isEmpty()) { - ClientErrorException cee = new ClientErrorException(Response.status(404).build()); - throw new PulsarAdminException(cee, cee.getMessage(), cee.getResponse().getStatus()); - } - return topics; - } - }); - - when(topics.getPartitionedTopicList(anyString())).thenAnswer(new Answer>() { - @Override - public List answer(InvocationOnMock invocationOnMock) throws Throwable { - Object[] args = invocationOnMock.getArguments(); - String ns = (String) args[0]; - List topics = getPartitionedTopics(ns); - if (topics.isEmpty()) { - ClientErrorException cee = new ClientErrorException(Response.status(404).build()); - throw new PulsarAdminException(cee, cee.getMessage(), cee.getResponse().getStatus()); - } - return topics; - } - }); - - when(topics.getPartitionedTopicMetadata(anyString())).thenAnswer(new Answer() { - @Override - public PartitionedTopicMetadata answer(InvocationOnMock invocationOnMock) throws Throwable { - Object[] args = invocationOnMock.getArguments(); - String topic = (String) args[0]; - int partitions = partitionedTopicsToPartitions.get(topic) == null - ? 0 : partitionedTopicsToPartitions.get(topic); - return new PartitionedTopicMetadata(partitions); - } - }); - - schemas = mock(Schemas.class); - when(schemas.getSchemaInfo(anyString())).thenAnswer(new Answer() { - @Override - public SchemaInfo answer(InvocationOnMock invocationOnMock) throws Throwable { - Object[] args = invocationOnMock.getArguments(); - String topic = (String) args[0]; - if (topicsToSchemas.get(topic) != null) { - return topicsToSchemas.get(topic); - } else { - ClientErrorException cee = new ClientErrorException(Response.status(404).build()); - throw new PulsarAdminException(cee, cee.getMessage(), cee.getResponse().getStatus()); - } - } - }); - - pulsarAdmin = mock(PulsarAdmin.class); - doReturn(tenants).when(pulsarAdmin).tenants(); - doReturn(namespaces).when(pulsarAdmin).namespaces(); - doReturn(topics).when(pulsarAdmin).topics(); - doReturn(schemas).when(pulsarAdmin).schemas(); - doReturn(pulsarAdmin).when(this.pulsarConnectorConfig).getPulsarAdmin(); - - this.pulsarAuth = mock(PulsarAuth.class); - - this.pulsarMetadata = - new PulsarMetadata(pulsarConnectorId, this.pulsarConnectorConfig, dispatchingRowDecoderFactory, - this.pulsarAuth); - this.pulsarSplitManager = Mockito.spy(new PulsarSplitManager(pulsarConnectorId, this.pulsarConnectorConfig)); - - ManagedLedgerFactory managedLedgerFactory = mock(ManagedLedgerFactory.class); - when(managedLedgerFactory.openReadOnlyCursor(any(), any(), any())).then(new Answer() { - - private Map positions = new HashMap<>(); - - private int count = 0; - @Override - public ReadOnlyCursor answer(InvocationOnMock invocationOnMock) throws Throwable { - Object[] args = invocationOnMock.getArguments(); - String topic = (String) args[0]; - PositionImpl positionImpl = (PositionImpl) args[1]; - - int position = positionImpl.getEntryId() == -1 ? 0 : (int) positionImpl.getEntryId(); - - positions.put(topic, position); - String schemaName = TopicName.get( - TopicName.get( - topic.replaceAll("/persistent", "")) - .getPartitionedTopicName()).getSchemaName(); - long entries = topicsToNumEntries.get(schemaName); - - - ReadOnlyCursorImpl readOnlyCursor = mock(ReadOnlyCursorImpl.class); - doReturn(entries).when(readOnlyCursor).getNumberOfEntries(); - - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock invocation) throws Throwable { - Object[] args = invocation.getArguments(); - Integer skipEntries = (Integer) args[0]; - positions.put(topic, positions.get(topic) + skipEntries); - return null; - } - }).when(readOnlyCursor).skipEntries(anyInt()); - - when(readOnlyCursor.getReadPosition()).thenAnswer(new Answer() { - @Override - public PositionImpl answer(InvocationOnMock invocationOnMock) throws Throwable { - return PositionImpl.get(0, positions.get(topic)); - } - }); - - doAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock invocationOnMock) throws Throwable { - Object[] args = invocationOnMock.getArguments(); - Integer readEntries = (Integer) args[0]; - AsyncCallbacks.ReadEntriesCallback callback = (AsyncCallbacks.ReadEntriesCallback) args[2]; - Object ctx = args[3]; - - new Thread(new Runnable() { - @Override - public void run() { - List entries = new LinkedList<>(); - for (int i = 0; i < readEntries; i++) { - - TestPulsarConnector.Bar bar = new TestPulsarConnector.Bar(); - bar.field1 = fooFunctions.get("bar.field1").apply(count) == null ? null : (int) fooFunctions.get("bar.field1").apply(count); - bar.field2 = fooFunctions.get("bar.field2").apply(count) == null ? null : (String) fooFunctions.get("bar.field2").apply(count); - bar.field3 = (float) fooFunctions.get("bar.field3").apply(count); - - - Foo foo = new Foo(); - foo.field1 = (int) fooFunctions.get("field1").apply(count); - foo.field2 = (String) fooFunctions.get("field2").apply(count); - foo.field3 = (float) fooFunctions.get("field3").apply(count); - foo.field4 = (double) fooFunctions.get("field4").apply(count); - foo.field5 = (boolean) fooFunctions.get("field5").apply(count); - foo.field6 = (long) fooFunctions.get("field6").apply(count); - foo.timestamp = (long) fooFunctions.get("timestamp").apply(count); - foo.time = (int) fooFunctions.get("time").apply(count); - foo.date = (int) fooFunctions.get("date").apply(count); - foo.decimal = (BigDecimal) fooFunctions.get("decimal").apply(count); - foo.bar = bar; - foo.field7 = (Foo.TestEnum) fooFunctions.get("field7").apply(count); - - MessageMetadata messageMetadata = new MessageMetadata() - .setProducerName("test-producer").setSequenceId(positions.get(topic)) - .setPublishTime(System.currentTimeMillis()); - - Schema schema = topicsToSchemas.get(schemaName).getType() == SchemaType.AVRO ? AvroSchema.of(Foo.class) : JSONSchema.of(Foo.class); - - ByteBuf payload = io.netty.buffer.Unpooled - .copiedBuffer(schema.encode(foo)); - - ByteBuf byteBuf = serializeMetadataAndPayload( - Commands.ChecksumType.Crc32c, messageMetadata, payload); - - completedBytes += byteBuf.readableBytes(); - - entries.add(EntryImpl.create(0, positions.get(topic), byteBuf)); - positions.put(topic, positions.get(topic) + 1); - count++; - } - - callback.readEntriesComplete(entries, ctx); - } - }).start(); - - return null; - } - }).when(readOnlyCursor).asyncReadEntries(anyInt(), anyLong(), any(), any(), any()); - - when(readOnlyCursor.hasMoreEntries()).thenAnswer(new Answer() { - @Override - public Boolean answer(InvocationOnMock invocationOnMock) throws Throwable { - return positions.get(topic) < entries; - } - }); - - when(readOnlyCursor.findNewestMatching(any(), any())).then(new Answer() { - @Override - public Position answer(InvocationOnMock invocationOnMock) throws Throwable { - Object[] args = invocationOnMock.getArguments(); - Predicate predicate - = (Predicate) args[1]; - - String schemaName = TopicName.get( - TopicName.get( - topic.replaceAll("/persistent", "")) - .getPartitionedTopicName()).getSchemaName(); - List entries = getTopicEntries(schemaName); - - Integer target = null; - for (int i=entries.size() - 1; i >= 0; i--) { - Entry entry = entries.get(i); - if (predicate.test(entry)) { - target = i; - break; - } - } - - return target == null ? null : new PositionImpl(0, target); - } - }); - - when(readOnlyCursor.getNumberOfEntries(any())).then(new Answer() { - @Override - public Long answer(InvocationOnMock invocationOnMock) throws Throwable { - Object[] args = invocationOnMock.getArguments(); - com.google.common.collect.Range range - = (com.google.common.collect.Range ) args[0]; - - return (range.upperEndpoint().getEntryId() + 1) - range.lowerEndpoint().getEntryId(); - } - }); - - when(readOnlyCursor.getCurrentLedgerInfo()).thenReturn(MLDataFormats.ManagedLedgerInfo.LedgerInfo.newBuilder().setLedgerId(0).build()); - - return readOnlyCursor; - } - }); - - PulsarConnectorCache.instance = mock(PulsarConnectorCache.class); - when(PulsarConnectorCache.instance.getManagedLedgerFactory()).thenReturn(managedLedgerFactory); - - for (Map.Entry split : splits.entrySet()) { - PulsarRecordCursor pulsarRecordCursor = new PulsarRecordCursor( - topicsToColumnHandles.get(split.getKey()), split.getValue(), - pulsarConnectorConfig, managedLedgerFactory, new ManagedLedgerConfig(), - new PulsarConnectorMetricsTracker(new NullStatsProvider()),dispatchingRowDecoderFactory); - this.pulsarRecordCursors.put(split.getKey(), pulsarRecordCursor); - } - } - - @AfterMethod(alwaysRun = true) - public void cleanup() { - completedBytes = 0L; - } - - @DataProvider(name = "rewriteNamespaceDelimiter") - public static Object[][] serviceUrls() { - return new Object[][] { - { "|" }, { null } - }; - } - - protected void updateRewriteNamespaceDelimiterIfNeeded(String delimiter) { - if (StringUtils.isNotBlank(delimiter)) { - pulsarConnectorConfig.setNamespaceDelimiterRewriteEnable(true); - pulsarConnectorConfig.setRewriteNamespaceDelimiter(delimiter); - } else { - pulsarConnectorConfig.setNamespaceDelimiterRewriteEnable(false); - } - } -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnectorConfig.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnectorConfig.java deleted file mode 100644 index 4f9dcf03979f7..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnectorConfig.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; -import org.testng.Assert; -import org.testng.annotations.Test; - -public class TestPulsarConnectorConfig { - - @Test - public void testDefaultNamespaceDelimiterRewrite() { - PulsarConnectorConfig connectorConfig = new PulsarConnectorConfig(); - Assert.assertFalse(connectorConfig.getNamespaceDelimiterRewriteEnable()); - Assert.assertEquals("/", connectorConfig.getRewriteNamespaceDelimiter()); - } - - @Test - public void testNamespaceRewriteDelimiterRestriction() { - PulsarConnectorConfig connectorConfig = new PulsarConnectorConfig(); - try { - connectorConfig.setRewriteNamespaceDelimiter("-=:.Az09_"); - } catch (Exception e) { - Assert.assertTrue(e instanceof IllegalArgumentException); - } - connectorConfig.setRewriteNamespaceDelimiter("|"); - Assert.assertEquals("|", (connectorConfig.getRewriteNamespaceDelimiter())); - connectorConfig.setRewriteNamespaceDelimiter("||"); - Assert.assertEquals("||", (connectorConfig.getRewriteNamespaceDelimiter())); - connectorConfig.setRewriteNamespaceDelimiter("$"); - Assert.assertEquals("$", (connectorConfig.getRewriteNamespaceDelimiter())); - connectorConfig.setRewriteNamespaceDelimiter("&"); - Assert.assertEquals("&", (connectorConfig.getRewriteNamespaceDelimiter())); - connectorConfig.setRewriteNamespaceDelimiter("--&"); - Assert.assertEquals("--&", (connectorConfig.getRewriteNamespaceDelimiter())); - } - - @Test - public void testDefaultBookkeeperConfig() { - int availableProcessors = Runtime.getRuntime().availableProcessors(); - PulsarConnectorConfig connectorConfig = new PulsarConnectorConfig(); - Assert.assertEquals(0, connectorConfig.getBookkeeperThrottleValue()); - Assert.assertEquals(2 * availableProcessors, connectorConfig.getBookkeeperNumIOThreads()); - Assert.assertEquals(availableProcessors, connectorConfig.getBookkeeperNumWorkerThreads()); - } - - @Test - public void testDefaultManagedLedgerConfig() { - int availableProcessors = Runtime.getRuntime().availableProcessors(); - PulsarConnectorConfig connectorConfig = new PulsarConnectorConfig(); - Assert.assertEquals(0L, connectorConfig.getManagedLedgerCacheSizeMB()); - Assert.assertEquals(availableProcessors, connectorConfig.getManagedLedgerNumSchedulerThreads()); - Assert.assertEquals(connectorConfig.getMaxSplitQueueSizeBytes(), -1); - } - - @Test - public void testGetOffloadPolices() throws Exception { - PulsarConnectorConfig connectorConfig = new PulsarConnectorConfig(); - - final String managedLedgerOffloadDriver = "s3"; - final String offloaderDirectory = "/pulsar/offloaders"; - final Integer managedLedgerOffloadMaxThreads = 5; - final String bucket = "offload-bucket"; - final String region = "us-west-2"; - final String endpoint = "http://s3.amazonaws.com"; - final String offloadProperties = "{" - + "\"s3ManagedLedgerOffloadBucket\":\"" + bucket + "\"," - + "\"s3ManagedLedgerOffloadRegion\":\"" + region + "\"," - + "\"s3ManagedLedgerOffloadServiceEndpoint\":\"" + endpoint + "\"" - + "}"; - - connectorConfig.setManagedLedgerOffloadDriver(managedLedgerOffloadDriver); - connectorConfig.setOffloadersDirectory(offloaderDirectory); - connectorConfig.setManagedLedgerOffloadMaxThreads(managedLedgerOffloadMaxThreads); - connectorConfig.setOffloaderProperties(offloadProperties); - - OffloadPoliciesImpl offloadPolicies = connectorConfig.getOffloadPolices(); - Assert.assertNotNull(offloadPolicies); - Assert.assertEquals(offloadPolicies.getManagedLedgerOffloadDriver(), managedLedgerOffloadDriver); - Assert.assertEquals(offloadPolicies.getOffloadersDirectory(), offloaderDirectory); - Assert.assertEquals((int) offloadPolicies.getManagedLedgerOffloadMaxThreads(), (int) managedLedgerOffloadMaxThreads); - Assert.assertEquals(offloadPolicies.getS3ManagedLedgerOffloadBucket(), bucket); - Assert.assertEquals(offloadPolicies.getS3ManagedLedgerOffloadRegion(), region); - Assert.assertEquals(offloadPolicies.getS3ManagedLedgerOffloadServiceEndpoint(), endpoint); - } - -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarMetadata.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarMetadata.java deleted file mode 100644 index 0db8c3231b337..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarMetadata.java +++ /dev/null @@ -1,438 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import io.airlift.log.Logger; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.*; -import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.common.schema.SchemaType; -import org.mockito.Mockito; -import org.testng.Assert; -import org.testng.annotations.Test; - -import javax.ws.rs.ClientErrorException; -import javax.ws.rs.core.Response; -import java.util.*; -import java.util.stream.Collectors; - -import static io.trino.spi.StandardErrorCode.NOT_FOUND; -import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; -import static io.trino.spi.StandardErrorCode.PERMISSION_DENIED; -import static org.mockito.Mockito.*; -import static org.testng.Assert.*; - -public class TestPulsarMetadata extends TestPulsarConnector { - - private static final Logger log = Logger.get(TestPulsarMetadata.class); - - @Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true) - public void testListSchemaNames(String delimiter) { - updateRewriteNamespaceDelimiterIfNeeded(delimiter); - List schemas = this.pulsarMetadata.listSchemaNames(mock(ConnectorSession.class)); - - - if (StringUtils.isBlank(delimiter)) { - String[] expectedSchemas = {NAMESPACE_NAME_1.toString(), NAMESPACE_NAME_2.toString(), - NAMESPACE_NAME_3.toString(), NAMESPACE_NAME_4.toString()}; - assertEquals(new HashSet<>(schemas), new HashSet<>(Arrays.asList(expectedSchemas))); - } else { - String[] expectedSchemas = { - PulsarConnectorUtils.rewriteNamespaceDelimiterIfNeeded(NAMESPACE_NAME_1.toString(), pulsarConnectorConfig), - PulsarConnectorUtils.rewriteNamespaceDelimiterIfNeeded(NAMESPACE_NAME_2.toString(), pulsarConnectorConfig), - PulsarConnectorUtils.rewriteNamespaceDelimiterIfNeeded(NAMESPACE_NAME_3.toString(), pulsarConnectorConfig), - PulsarConnectorUtils.rewriteNamespaceDelimiterIfNeeded(NAMESPACE_NAME_4.toString(), pulsarConnectorConfig)}; - assertEquals(new HashSet<>(schemas), new HashSet<>(Arrays.asList(expectedSchemas))); - } - } - - @Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true) - public void testGetTableHandle(String delimiter) { - updateRewriteNamespaceDelimiterIfNeeded(delimiter); - SchemaTableName schemaTableName = new SchemaTableName(TOPIC_1.getNamespace(), TOPIC_1.getLocalName()); - - ConnectorTableHandle connectorTableHandle - = this.pulsarMetadata.getTableHandle(mock(ConnectorSession.class), schemaTableName); - - assertTrue(connectorTableHandle instanceof PulsarTableHandle); - - PulsarTableHandle pulsarTableHandle = (PulsarTableHandle) connectorTableHandle; - - assertEquals(pulsarTableHandle.getConnectorId(), pulsarConnectorId.toString()); - assertEquals(pulsarTableHandle.getSchemaName(), TOPIC_1.getNamespace()); - assertEquals(pulsarTableHandle.getTableName(), TOPIC_1.getLocalName()); - assertEquals(pulsarTableHandle.getTopicName(), TOPIC_1.getLocalName()); - } - - @Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true) - public void testGetTableMetadata(String delimiter) { - updateRewriteNamespaceDelimiterIfNeeded(delimiter); - List allTopics = new LinkedList<>(); - allTopics.addAll(topicNames.stream().filter(topicName -> !topicName.equals(NON_SCHEMA_TOPIC)).collect(Collectors.toList())); - allTopics.addAll(partitionedTopicNames); - - for (TopicName topic : allTopics) { - PulsarTableHandle pulsarTableHandle = new PulsarTableHandle( - topic.toString(), - topic.getNamespace(), - topic.getLocalName(), - topic.getLocalName() - ); - - List fooColumnHandles = topicsToColumnHandles.get(topic); - - ConnectorTableMetadata tableMetadata = this.pulsarMetadata.getTableMetadata(mock(ConnectorSession.class), - pulsarTableHandle); - - assertEquals(tableMetadata.getTable().getSchemaName(), topic.getNamespace()); - assertEquals(tableMetadata.getTable().getTableName(), topic.getLocalName()); - assertEquals(tableMetadata.getColumns().size(), - fooColumnHandles.size()); - - List fieldNames = new LinkedList<>(fooFieldNames); - - for (PulsarInternalColumn internalField : PulsarInternalColumn.getInternalFields()) { - fieldNames.add(internalField.getName()); - } - - for (ColumnMetadata column : tableMetadata.getColumns()) { - if (PulsarInternalColumn.getInternalFieldsMap().containsKey(column.getName())) { - assertEquals(column.getComment(), - PulsarInternalColumn.getInternalFieldsMap() - .get(column.getName()).getColumnMetadata(true).getComment()); - } - - fieldNames.remove(column.getName()); - } - - assertTrue(fieldNames.isEmpty()); - } - } - - @Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true) - public void testGetTableMetadataWrongSchema(String delimiter) { - updateRewriteNamespaceDelimiterIfNeeded(delimiter); - PulsarTableHandle pulsarTableHandle = new PulsarTableHandle( - pulsarConnectorId.toString(), - "wrong-tenant/wrong-ns", - TOPIC_1.getLocalName(), - TOPIC_1.getLocalName() - ); - - try { - ConnectorTableMetadata tableMetadata = this.pulsarMetadata.getTableMetadata(mock(ConnectorSession.class), - pulsarTableHandle); - fail("Invalid schema should have generated an exception"); - } catch (TrinoException e) { - assertEquals(e.getErrorCode(), NOT_FOUND.toErrorCode()); - assertEquals(e.getMessage(), "Schema wrong-tenant/wrong-ns does not exist"); - } - } - - @Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true) - public void testGetTableMetadataWrongTable(String delimiter) { - updateRewriteNamespaceDelimiterIfNeeded(delimiter); - PulsarTableHandle pulsarTableHandle = new PulsarTableHandle( - pulsarConnectorId.toString(), - TOPIC_1.getNamespace(), - "wrong-topic", - "wrong-topic" - ); - - try { - ConnectorTableMetadata tableMetadata = this.pulsarMetadata.getTableMetadata(mock(ConnectorSession.class), - pulsarTableHandle); - fail("Invalid table should have generated an exception"); - } catch (TableNotFoundException e) { - assertEquals(e.getErrorCode(), NOT_FOUND.toErrorCode()); - assertEquals(e.getMessage(), "Table 'tenant-1/ns-1.wrong-topic' not found"); - } - } - - @Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true) - public void testGetTableMetadataTableNoSchema(String delimiter) throws PulsarAdminException { - updateRewriteNamespaceDelimiterIfNeeded(delimiter); - ClientErrorException cee = new ClientErrorException(Response.Status.NOT_FOUND); - when(this.schemas.getSchemaInfo(eq(TOPIC_1.getSchemaName()))).thenThrow( - new PulsarAdminException(cee, cee.getMessage(), cee.getResponse().getStatus())); - - PulsarTableHandle pulsarTableHandle = new PulsarTableHandle( - pulsarConnectorId.toString(), - TOPIC_1.getNamespace(), - TOPIC_1.getLocalName(), - TOPIC_1.getLocalName() - ); - - - ConnectorTableMetadata tableMetadata = this.pulsarMetadata.getTableMetadata(mock(ConnectorSession.class), - pulsarTableHandle); - assertEquals(tableMetadata.getColumns().size(), PulsarInternalColumn.getInternalFields().size() + 1); - } - - @Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true) - public void testGetTableMetadataTableBlankSchema(String delimiter) throws PulsarAdminException { - updateRewriteNamespaceDelimiterIfNeeded(delimiter); - SchemaInfo badSchemaInfo = SchemaInfo.builder() - .schema(new byte[0]) - .type(SchemaType.AVRO) - .build(); - when(this.schemas.getSchemaInfo(eq(TOPIC_1.getSchemaName()))).thenReturn(badSchemaInfo); - - PulsarTableHandle pulsarTableHandle = new PulsarTableHandle( - pulsarConnectorId.toString(), - TOPIC_1.getNamespace(), - TOPIC_1.getLocalName(), - TOPIC_1.getLocalName() - ); - - try { - ConnectorTableMetadata tableMetadata = this.pulsarMetadata.getTableMetadata(mock(ConnectorSession.class), - pulsarTableHandle); - fail("Table without schema should have generated an exception"); - } catch (TrinoException e) { - assertEquals(e.getErrorCode(), NOT_SUPPORTED.toErrorCode()); - assertEquals(e.getMessage(), - "Topic persistent://tenant-1/ns-1/topic-1 does not have a valid schema"); - } - } - - @Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true) - public void testGetTableMetadataTableInvalidSchema(String delimiter) throws PulsarAdminException { - updateRewriteNamespaceDelimiterIfNeeded(delimiter); - SchemaInfo badSchemaInfo = SchemaInfo.builder() - .schema("foo".getBytes()) - .type(SchemaType.AVRO) - .build(); - when(this.schemas.getSchemaInfo(eq(TOPIC_1.getSchemaName()))).thenReturn(badSchemaInfo); - - PulsarTableHandle pulsarTableHandle = new PulsarTableHandle( - pulsarConnectorId.toString(), - TOPIC_1.getNamespace(), - TOPIC_1.getLocalName(), - TOPIC_1.getLocalName() - ); - - try { - ConnectorTableMetadata tableMetadata = this.pulsarMetadata.getTableMetadata(mock(ConnectorSession.class), - pulsarTableHandle); - fail("Table without schema should have generated an exception"); - } catch (TrinoException e) { - assertEquals(e.getErrorCode(), NOT_SUPPORTED.toErrorCode()); - assertEquals(e.getMessage(), - "Topic persistent://tenant-1/ns-1/topic-1 does not have a valid schema"); - } - } - - @Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true) - public void testListTable(String delimiter) { - updateRewriteNamespaceDelimiterIfNeeded(delimiter); - assertTrue(this.pulsarMetadata.listTables(mock(ConnectorSession.class), Optional.empty()).isEmpty()); - assertTrue(this.pulsarMetadata.listTables(mock(ConnectorSession.class), Optional.of("wrong-tenant/wrong-ns")) - .isEmpty()); - - SchemaTableName[] expectedTopics1 = {new SchemaTableName( - TOPIC_4.getNamespace(), TOPIC_4.getLocalName()), - new SchemaTableName(PARTITIONED_TOPIC_4.getNamespace(), PARTITIONED_TOPIC_4.getLocalName()) - }; - assertEquals(this.pulsarMetadata.listTables(mock(ConnectorSession.class), - Optional.of(NAMESPACE_NAME_3.toString())), Arrays.asList(expectedTopics1)); - - SchemaTableName[] expectedTopics2 = {new SchemaTableName(TOPIC_5.getNamespace(), TOPIC_5.getLocalName()), - new SchemaTableName(TOPIC_6.getNamespace(), TOPIC_6.getLocalName()), - new SchemaTableName(PARTITIONED_TOPIC_5.getNamespace(), PARTITIONED_TOPIC_5.getLocalName()), - new SchemaTableName(PARTITIONED_TOPIC_6.getNamespace(), PARTITIONED_TOPIC_6.getLocalName()), - }; - assertEquals(new HashSet<>(this.pulsarMetadata.listTables(mock(ConnectorSession.class), - Optional.of(NAMESPACE_NAME_4.toString()))), new HashSet<>(Arrays.asList(expectedTopics2))); - } - - @Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true) - public void testGetColumnHandles(String delimiter) { - updateRewriteNamespaceDelimiterIfNeeded(delimiter); - PulsarTableHandle pulsarTableHandle = new PulsarTableHandle(pulsarConnectorId.toString(), TOPIC_1.getNamespace(), - TOPIC_1.getLocalName(), TOPIC_1.getLocalName()); - Map columnHandleMap - = new HashMap<>(this.pulsarMetadata.getColumnHandles(mock(ConnectorSession.class), pulsarTableHandle)); - - List fieldNames = new LinkedList<>(fooFieldNames); - - for (PulsarInternalColumn internalField : PulsarInternalColumn.getInternalFields()) { - fieldNames.add(internalField.getName()); - } - - for (String field : fieldNames) { - assertNotNull(columnHandleMap.get(field)); - PulsarColumnHandle pulsarColumnHandle = (PulsarColumnHandle) columnHandleMap.get(field); - PulsarInternalColumn pulsarInternalColumn = PulsarInternalColumn.getInternalFieldsMap().get(field); - if (pulsarInternalColumn != null) { - assertEquals(pulsarColumnHandle, - pulsarInternalColumn.getColumnHandle(pulsarConnectorId.toString(), false)); - } else { - assertEquals(pulsarColumnHandle.getConnectorId(), pulsarConnectorId.toString()); - assertEquals(pulsarColumnHandle.getName(), field); - assertFalse(pulsarColumnHandle.isHidden()); - } - columnHandleMap.remove(field); - } - assertTrue(columnHandleMap.isEmpty()); - } - - @Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true) - public void testListTableColumns(String delimiter) { - updateRewriteNamespaceDelimiterIfNeeded(delimiter); - Map> tableColumnsMap - = this.pulsarMetadata.listTableColumns(mock(ConnectorSession.class), - new SchemaTablePrefix(TOPIC_1.getNamespace())); - - assertEquals(tableColumnsMap.size(), 4); - List columnMetadataList - = tableColumnsMap.get(new SchemaTableName(TOPIC_1.getNamespace(), TOPIC_1.getLocalName())); - assertNotNull(columnMetadataList); - assertEquals(columnMetadataList.size(), - topicsToColumnHandles.get(TOPIC_1).size()); - - List fieldNames = new LinkedList<>(fooFieldNames); - - for (PulsarInternalColumn internalField : PulsarInternalColumn.getInternalFields()) { - fieldNames.add(internalField.getName()); - } - - for (ColumnMetadata column : columnMetadataList) { - if (PulsarInternalColumn.getInternalFieldsMap().containsKey(column.getName())) { - assertEquals(column.getComment(), - PulsarInternalColumn.getInternalFieldsMap() - .get(column.getName()).getColumnMetadata(true).getComment()); - } - - fieldNames.remove(column.getName()); - } - - assertTrue(fieldNames.isEmpty()); - - columnMetadataList = tableColumnsMap.get(new SchemaTableName(TOPIC_2.getNamespace(), TOPIC_2.getLocalName())); - assertNotNull(columnMetadataList); - assertEquals(columnMetadataList.size(), - topicsToColumnHandles.get(TOPIC_2).size()); - - fieldNames = new LinkedList<>(fooFieldNames); - - for (PulsarInternalColumn internalField : PulsarInternalColumn.getInternalFields()) { - fieldNames.add(internalField.getName()); - } - - for (ColumnMetadata column : columnMetadataList) { - if (PulsarInternalColumn.getInternalFieldsMap().containsKey(column.getName())) { - assertEquals(column.getComment(), - PulsarInternalColumn.getInternalFieldsMap() - .get(column.getName()).getColumnMetadata(true).getComment()); - } - - fieldNames.remove(column.getName()); - } - - assertTrue(fieldNames.isEmpty()); - - // test table and schema - tableColumnsMap - = this.pulsarMetadata.listTableColumns(mock(ConnectorSession.class), - new SchemaTablePrefix(TOPIC_4.getNamespace(), TOPIC_4.getLocalName())); - - assertEquals(tableColumnsMap.size(), 1); - columnMetadataList = tableColumnsMap.get(new SchemaTableName(TOPIC_4.getNamespace(), TOPIC_4.getLocalName())); - assertNotNull(columnMetadataList); - assertEquals(columnMetadataList.size(), - topicsToColumnHandles.get(TOPIC_4).size()); - - fieldNames = new LinkedList<>(fooFieldNames); - - for (PulsarInternalColumn internalField : PulsarInternalColumn.getInternalFields()) { - fieldNames.add(internalField.getName()); - } - - for (ColumnMetadata column : columnMetadataList) { - if (PulsarInternalColumn.getInternalFieldsMap().containsKey(column.getName())) { - assertEquals(column.getComment(), - PulsarInternalColumn.getInternalFieldsMap() - .get(column.getName()).getColumnMetadata(true).getComment()); - } - - fieldNames.remove(column.getName()); - } - - assertTrue(fieldNames.isEmpty()); - } - - @Test - public void testPulsarAuthCheck() { - this.pulsarConnectorConfig.setAuthorizationEnabled(true); - doNothing().when(this.pulsarAuth).checkTopicAuth(isA(ConnectorSession.class), isA(String.class)); - - // Test getTableHandle should pass the auth check - SchemaTableName schemaTableName = new SchemaTableName(TOPIC_1.getNamespace(), TOPIC_1.getLocalName()); - this.pulsarMetadata.getTableHandle(mock(ConnectorSession.class), schemaTableName); - - // Test getTableMetadata should pass the auth check - TopicName topic = TOPIC_1; - PulsarTableHandle pulsarTableHandle = new PulsarTableHandle( - topic.toString(), - topic.getNamespace(), - topic.getLocalName(), - topic.getLocalName() - ); - this.pulsarMetadata.getTableMetadata(mock(ConnectorSession.class), - pulsarTableHandle); - - // Test getColumnHandles should pass the auth check - this.pulsarMetadata.getColumnHandles(mock(ConnectorSession.class), pulsarTableHandle); - - doThrow(new TrinoException(PERMISSION_DENIED, "not authorized")).when(this.pulsarAuth) - .checkTopicAuth(isA(ConnectorSession.class), isA(String.class)); - - // Test getTableHandle should fail the auth check - try { - this.pulsarMetadata.getTableHandle(mock(ConnectorSession.class), schemaTableName); - Assert.fail("Test getTableHandle should fail the auth check"); // should fail - } catch (TrinoException e) { - Assert.assertEquals(PERMISSION_DENIED.toErrorCode(), e.getErrorCode()); - } - - // Test getTableMetadata should fail the auth check - try { - this.pulsarMetadata.getTableMetadata(mock(ConnectorSession.class), - pulsarTableHandle); - Assert.fail("Test getTableMetadata should fail the auth check"); // should fail - } catch (TrinoException e) { - Assert.assertEquals(PERMISSION_DENIED.toErrorCode(), e.getErrorCode()); - } - - // Test getColumnHandles should fail the auth check - try { - this.pulsarMetadata.getColumnHandles(mock(ConnectorSession.class), pulsarTableHandle); - Assert.fail("Test getColumnHandles should fail the auth check"); // should fail - } catch (TrinoException e) { - Assert.assertEquals(PERMISSION_DENIED.toErrorCode(), e.getErrorCode()); - } - - this.pulsarMetadata.cleanupQuery(mock(ConnectorSession.class)); - Mockito.verify(this.pulsarAuth, Mockito.times(1)).cleanSession(isA(ConnectorSession.class)); - } -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java deleted file mode 100644 index 40ced8e4f8e32..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java +++ /dev/null @@ -1,537 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import static java.util.concurrent.CompletableFuture.completedFuture; -import static org.apache.pulsar.common.protocol.Commands.serializeMetadataAndPayload; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.airlift.log.Logger; -import io.netty.buffer.ByteBuf; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.type.DecimalType; -import io.trino.spi.type.RowType; -import io.trino.spi.type.Type; -import io.trino.spi.type.VarcharType; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.math.BigDecimal; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import lombok.Data; -import org.apache.bookkeeper.mledger.AsyncCallbacks; -import org.apache.bookkeeper.mledger.Entry; -import org.apache.bookkeeper.mledger.ManagedLedgerConfig; -import org.apache.bookkeeper.mledger.ManagedLedgerFactory; -import org.apache.bookkeeper.mledger.ReadOnlyCursor; -import org.apache.bookkeeper.mledger.impl.EntryImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.bookkeeper.mledger.impl.ReadOnlyCursorImpl; -import org.apache.bookkeeper.mledger.proto.MLDataFormats; -import org.apache.bookkeeper.stats.NullStatsProvider; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.admin.Schemas; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl; -import org.apache.pulsar.common.api.proto.MessageMetadata; -import org.apache.pulsar.common.api.raw.RawMessage; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.protocol.Commands; -import org.apache.pulsar.common.protocol.schema.BytesSchemaVersion; -import org.apache.pulsar.common.schema.KeyValue; -import org.apache.pulsar.common.schema.KeyValueEncodingType; -import org.apache.pulsar.common.schema.LongSchemaVersion; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.common.schema.SchemaType; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.testng.annotations.Test; - -public class TestPulsarRecordCursor extends TestPulsarConnector { - - private static final Logger log = Logger.get(TestPulsarRecordCursor.class); - - @Test(singleThreaded = true) - public void testTopics() throws Exception { - - for (Map.Entry entry : pulsarRecordCursors.entrySet()) { - - log.info("!------ topic %s ------!", entry.getKey()); - setup(); - - List fooColumnHandles = topicsToColumnHandles.get(entry.getKey()); - PulsarRecordCursor pulsarRecordCursor = entry.getValue(); - - PulsarSqlSchemaInfoProvider pulsarSqlSchemaInfoProvider = mock(PulsarSqlSchemaInfoProvider.class); - when(pulsarSqlSchemaInfoProvider.getSchemaByVersion(any())).thenReturn(completedFuture(topicsToSchemas.get(entry.getKey().getSchemaName()))); - pulsarRecordCursor.setPulsarSqlSchemaInfoProvider(pulsarSqlSchemaInfoProvider); - - TopicName topicName = entry.getKey(); - - int count = 0; - while (pulsarRecordCursor.advanceNextPosition()) { - List columnsSeen = new LinkedList<>(); - for (int i = 0; i < fooColumnHandles.size(); i++) { - if (pulsarRecordCursor.isNull(i)) { - columnsSeen.add(fooColumnHandles.get(i).getName()); - } else { - if (fooColumnHandles.get(i).getName().equals("field1")) { - assertEquals(pulsarRecordCursor.getLong(i), ((Integer) fooFunctions.get("field1").apply(count)).longValue()); - columnsSeen.add(fooColumnHandles.get(i).getName()); - } else if (fooColumnHandles.get(i).getName().equals("field2")) { - assertEquals(pulsarRecordCursor.getSlice(i).getBytes(), ((String) fooFunctions.get("field2").apply(count)).getBytes()); - columnsSeen.add(fooColumnHandles.get(i).getName()); - } else if (fooColumnHandles.get(i).getName().equals("field3")) { - assertEquals(pulsarRecordCursor.getLong(i), Float.floatToIntBits((Float) fooFunctions.get("field3").apply(count))); - columnsSeen.add(fooColumnHandles.get(i).getName()); - } else if (fooColumnHandles.get(i).getName().equals("field4")) { - assertEquals(pulsarRecordCursor.getDouble(i), ((Double) fooFunctions.get("field4").apply(count)).doubleValue()); - columnsSeen.add(fooColumnHandles.get(i).getName()); - } else if (fooColumnHandles.get(i).getName().equals("field5")) { - assertEquals(pulsarRecordCursor.getBoolean(i), ((Boolean) fooFunctions.get("field5").apply(count)).booleanValue()); - columnsSeen.add(fooColumnHandles.get(i).getName()); - } else if (fooColumnHandles.get(i).getName().equals("field6")) { - assertEquals(pulsarRecordCursor.getLong(i), ((Long) fooFunctions.get("field6").apply(count)).longValue()); - columnsSeen.add(fooColumnHandles.get(i).getName()); - } else if (fooColumnHandles.get(i).getName().equals("timestamp")) { - pulsarRecordCursor.getLong(i); - columnsSeen.add(fooColumnHandles.get(i).getName()); - } else if (fooColumnHandles.get(i).getName().equals("time")) { - pulsarRecordCursor.getLong(i); - columnsSeen.add(fooColumnHandles.get(i).getName()); - } else if (fooColumnHandles.get(i).getName().equals("date")) { - pulsarRecordCursor.getLong(i); - columnsSeen.add(fooColumnHandles.get(i).getName()); - } else if (fooColumnHandles.get(i).getName().equals("bar")) { - assertTrue(fooColumnHandles.get(i).getType() instanceof RowType); - columnsSeen.add(fooColumnHandles.get(i).getName()); - }else if (fooColumnHandles.get(i).getName().equals("field7")) { - assertEquals(pulsarRecordCursor.getSlice(i).getBytes(), fooFunctions.get("field7").apply(count).toString().getBytes()); - columnsSeen.add(fooColumnHandles.get(i).getName()); - }else if (fooColumnHandles.get(i).getName().equals("decimal")) { - Type type = fooColumnHandles.get(i).getType(); - // In JsonDecoder, decimal trans to varcharType - if (type instanceof VarcharType) { - assertEquals(new String(pulsarRecordCursor.getSlice(i).getBytes()), - fooFunctions.get("decimal").apply(count).toString()); - } else { - DecimalType decimalType = (DecimalType) fooColumnHandles.get(i).getType(); - assertEquals(BigDecimal.valueOf(pulsarRecordCursor.getLong(i), decimalType.getScale()), fooFunctions.get("decimal").apply(count)); - } - columnsSeen.add(fooColumnHandles.get(i).getName()); - } else { - if (PulsarInternalColumn.getInternalFieldsMap().containsKey(fooColumnHandles.get(i).getName())) { - columnsSeen.add(fooColumnHandles.get(i).getName()); - } - } - } - } - assertEquals(columnsSeen.size(), fooColumnHandles.size()); - count++; - } - assertEquals(count, topicsToNumEntries.get(topicName.getSchemaName()).longValue()); - assertEquals(pulsarRecordCursor.getCompletedBytes(), completedBytes); - cleanup(); - pulsarRecordCursor.close(); - } - } - - @Test(singleThreaded = true) - public void TestKeyValueStructSchema() throws Exception { - - TopicName topicName = TopicName.get("persistent", NAMESPACE_NAME_1, "topic-4"); - Long entriesNum = 5L; - - for (KeyValueEncodingType encodingType : - Arrays.asList(KeyValueEncodingType.INLINE, KeyValueEncodingType.SEPARATED)) { - - KeyValueSchemaImpl schema = (KeyValueSchemaImpl) Schema.KeyValue(Schema.JSON(Foo.class), Schema.AVRO(Boo.class), - encodingType); - - Foo foo = new Foo(); - foo.field1 = "field1-value"; - foo.field2 = 20; - Boo boo = new Boo(); - boo.field1 = "field1-value"; - boo.field2 = true; - boo.field3 = 10.2; - - KeyValue message = new KeyValue<>(foo, boo); - List ColumnHandles = getColumnColumnHandles(topicName, schema.getSchemaInfo(), PulsarColumnHandle.HandleKeyValueType.NONE, true); - PulsarRecordCursor pulsarRecordCursor = mockKeyValueSchemaPulsarRecordCursor(entriesNum, topicName, - schema, message, ColumnHandles); - - assertNotNull(pulsarRecordCursor); - Long count = 0L; - while (pulsarRecordCursor.advanceNextPosition()) { - List columnsSeen = new LinkedList<>(); - for (int i = 0; i < ColumnHandles.size(); i++) { - if (pulsarRecordCursor.isNull(i)) { - columnsSeen.add(ColumnHandles.get(i).getName()); - } else { - if (ColumnHandles.get(i).getName().equals("field1")) { - assertEquals(pulsarRecordCursor.getSlice(i).getBytes(), boo.field1.getBytes()); - columnsSeen.add(ColumnHandles.get(i).getName()); - } else if (ColumnHandles.get(i).getName().equals("field2")) { - assertEquals(pulsarRecordCursor.getBoolean(i), boo.field2.booleanValue()); - columnsSeen.add(ColumnHandles.get(i).getName()); - } else if (ColumnHandles.get(i).getName().equals("field3")) { - assertEquals((Double) pulsarRecordCursor.getDouble(i), (Double) boo.field3); - columnsSeen.add(ColumnHandles.get(i).getName()); - } else if (ColumnHandles.get(i).getName().equals(PulsarColumnMetadata.KEY_SCHEMA_COLUMN_PREFIX + - "field1")) { - assertEquals(pulsarRecordCursor.getSlice(i).getBytes(), foo.field1.getBytes()); - columnsSeen.add(ColumnHandles.get(i).getName()); - } else if (ColumnHandles.get(i).getName().equals(PulsarColumnMetadata.KEY_SCHEMA_COLUMN_PREFIX + - "field2")) { - assertEquals(pulsarRecordCursor.getLong(i), Long.valueOf(foo.field2).longValue()); - columnsSeen.add(ColumnHandles.get(i).getName()); - } else { - if (PulsarInternalColumn.getInternalFieldsMap().containsKey(ColumnHandles.get(i).getName())) { - columnsSeen.add(ColumnHandles.get(i).getName()); - } - } - } - } - assertEquals(columnsSeen.size(), ColumnHandles.size()); - count++; - } - assertEquals(count, entriesNum); - pulsarRecordCursor.close(); - } - } - - @Test(singleThreaded = true) - public void TestKeyValuePrimitiveSchema() throws Exception { - - TopicName topicName = TopicName.get("persistent", NAMESPACE_NAME_1, "topic-4"); - Long entriesNum = 5L; - - for (KeyValueEncodingType encodingType : - Arrays.asList(KeyValueEncodingType.INLINE, KeyValueEncodingType.SEPARATED)) { - - KeyValueSchemaImpl schema = (KeyValueSchemaImpl) Schema.KeyValue(Schema.INT32, Schema.STRING, - encodingType); - - String value = "primitive_message_value"; - Integer key = 23; - KeyValue message = new KeyValue<>(key, value); - - List ColumnHandles = getColumnColumnHandles(topicName, schema.getSchemaInfo(), PulsarColumnHandle.HandleKeyValueType.NONE, true); - PulsarRecordCursor pulsarRecordCursor = mockKeyValueSchemaPulsarRecordCursor(entriesNum, topicName, - schema, message, ColumnHandles); - - assertNotNull(pulsarRecordCursor); - Long count = 0L; - while (pulsarRecordCursor.advanceNextPosition()) { - List columnsSeen = new LinkedList<>(); - for (int i = 0; i < ColumnHandles.size(); i++) { - if (pulsarRecordCursor.isNull(i)) { - columnsSeen.add(ColumnHandles.get(i).getName()); - } else { - if (ColumnHandles.get(i).getName().equals(PRIMITIVE_COLUMN_NAME)) { - assertEquals(pulsarRecordCursor.getSlice(i).getBytes(), value.getBytes()); - columnsSeen.add(ColumnHandles.get(i).getName()); - } else if (ColumnHandles.get(i).getName().equals(KEY_SCHEMA_COLUMN_PREFIX + - PRIMITIVE_COLUMN_NAME)) { - assertEquals((Long) pulsarRecordCursor.getLong(i), Long.valueOf(key)); - columnsSeen.add(ColumnHandles.get(i).getName()); - } else { - if (PulsarInternalColumn.getInternalFieldsMap().containsKey(ColumnHandles.get(i).getName())) { - columnsSeen.add(ColumnHandles.get(i).getName()); - } - } - } - } - assertEquals(columnsSeen.size(), ColumnHandles.size()); - count++; - } - assertEquals(count, entriesNum); - pulsarRecordCursor.close(); - } - } - - - /** - * mock a simple PulsarRecordCursor for KeyValueSchema test. - * @param entriesNum - * @param topicName - * @param schema - * @param message - * @param ColumnHandles - * @return - * @throws Exception - */ - private PulsarRecordCursor mockKeyValueSchemaPulsarRecordCursor(final Long entriesNum, final TopicName topicName, - final KeyValueSchemaImpl schema, KeyValue message, List ColumnHandles) throws Exception { - - ManagedLedgerFactory managedLedgerFactory = mock(ManagedLedgerFactory.class); - - when(managedLedgerFactory.openReadOnlyCursor(any(), any(), any())).then(new Answer() { - - private Map positions = new HashMap<>(); - - @Override - public ReadOnlyCursor answer(InvocationOnMock invocationOnMock) throws Throwable { - Object[] args = invocationOnMock.getArguments(); - String topic = (String) args[0]; - PositionImpl positionImpl = (PositionImpl) args[1]; - int position = positionImpl.getEntryId() == -1 ? 0 : (int) positionImpl.getEntryId(); - - positions.put(topic, position); - ReadOnlyCursorImpl readOnlyCursor = mock(ReadOnlyCursorImpl.class); - doReturn(entriesNum).when(readOnlyCursor).getNumberOfEntries(); - - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock invocation) throws Throwable { - Object[] args = invocation.getArguments(); - Integer skipEntries = (Integer) args[0]; - positions.put(topic, positions.get(topic) + skipEntries); - return null; - } - }).when(readOnlyCursor).skipEntries(anyInt()); - - when(readOnlyCursor.getReadPosition()).thenAnswer(new Answer() { - @Override - public PositionImpl answer(InvocationOnMock invocationOnMock) throws Throwable { - return PositionImpl.get(0, positions.get(topic)); - } - }); - - doAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock invocationOnMock) throws Throwable { - Object[] args = invocationOnMock.getArguments(); - Integer readEntries = (Integer) args[0]; - AsyncCallbacks.ReadEntriesCallback callback = (AsyncCallbacks.ReadEntriesCallback) args[2]; - Object ctx = args[3]; - - new Thread(new Runnable() { - @Override - public void run() { - List entries = new LinkedList<>(); - for (int i = 0; i < readEntries; i++) { - - MessageMetadata messageMetadata = - new MessageMetadata() - .setProducerName("test-producer") - .setSequenceId(positions.get(topic)) - .setPublishTime(System.currentTimeMillis()); - - if (i % 2 == 0) { - messageMetadata.setSchemaVersion(new LongSchemaVersion(1L).bytes()); - } - - if (KeyValueEncodingType.SEPARATED.equals(schema.getKeyValueEncodingType())) { - messageMetadata - .setPartitionKey(new String(schema - .getKeySchema().encode(message.getKey()), Charset.forName( - "UTF-8"))) - .setPartitionKeyB64Encoded(false); - } - - ByteBuf dataPayload = io.netty.buffer.Unpooled - .copiedBuffer(schema.encode(message)); - - ByteBuf byteBuf = serializeMetadataAndPayload( - Commands.ChecksumType.Crc32c, messageMetadata, dataPayload); - - entries.add(EntryImpl.create(0, positions.get(topic), byteBuf)); - positions.put(topic, positions.get(topic) + 1); - } - - callback.readEntriesComplete(entries, ctx); - } - }).start(); - - return null; - } - }).when(readOnlyCursor).asyncReadEntries(anyInt(), anyLong(), any(), any(), any()); - - when(readOnlyCursor.hasMoreEntries()).thenAnswer(new Answer() { - @Override - public Boolean answer(InvocationOnMock invocationOnMock) throws Throwable { - return positions.get(topic) < entriesNum; - } - }); - - when(readOnlyCursor.getNumberOfEntries(any())).then(new Answer() { - @Override - public Long answer(InvocationOnMock invocationOnMock) throws Throwable { - Object[] args = invocationOnMock.getArguments(); - com.google.common.collect.Range range - = (com.google.common.collect.Range) args[0]; - return (range.upperEndpoint().getEntryId() + 1) - range.lowerEndpoint().getEntryId(); - } - }); - - when(readOnlyCursor.getCurrentLedgerInfo()).thenReturn(MLDataFormats.ManagedLedgerInfo.LedgerInfo.newBuilder().setLedgerId(0).build()); - - return readOnlyCursor; - } - }); - - ObjectMapper objectMapper = new ObjectMapper(); - - PulsarSplit split = new PulsarSplit(0, pulsarConnectorId.toString(), - topicName.getNamespace(), topicName.getLocalName(), topicName.getLocalName(), - entriesNum, - new String(schema.getSchemaInfo().getSchema(), "ISO8859-1"), - schema.getSchemaInfo().getType(), - 0, entriesNum, - 0, 0, TupleDomain.all(), - objectMapper.writeValueAsString( - schema.getSchemaInfo().getProperties()), null); - - PulsarRecordCursor pulsarRecordCursor = spy(new PulsarRecordCursor( - ColumnHandles, split, - pulsarConnectorConfig, managedLedgerFactory, new ManagedLedgerConfig(), - new PulsarConnectorMetricsTracker(new NullStatsProvider()), dispatchingRowDecoderFactory)); - - PulsarSqlSchemaInfoProvider pulsarSqlSchemaInfoProvider = mock(PulsarSqlSchemaInfoProvider.class); - when(pulsarSqlSchemaInfoProvider.getSchemaByVersion(any())).thenReturn(completedFuture(schema.getSchemaInfo())); - pulsarRecordCursor.setPulsarSqlSchemaInfoProvider(pulsarSqlSchemaInfoProvider); - - return pulsarRecordCursor; - } - - - static final String KEY_SCHEMA_COLUMN_PREFIX = "__key."; - static final String PRIMITIVE_COLUMN_NAME = "__value__"; - - @Data - static class Foo { - private String field1; - private Integer field2; - } - - @Data - static class Boo { - private String field1; - private Boolean field2; - private Double field3; - } - - @Test - public void testGetSchemaInfo() throws Exception { - String topic = "get-schema-test"; - PulsarSplit pulsarSplit = Mockito.mock(PulsarSplit.class); - Mockito.when(pulsarSplit.getTableName()).thenReturn(TopicName.get(topic).getLocalName()); - Mockito.when(pulsarSplit.getSchemaName()).thenReturn("public/default"); - PulsarAdmin pulsarAdmin = Mockito.mock(PulsarAdmin.class); - Schemas schemas = Mockito.mock(Schemas.class); - Mockito.when(pulsarAdmin.schemas()).thenReturn(schemas); - PulsarConnectorConfig connectorConfig = spy(PulsarConnectorConfig.class); - Mockito.when(connectorConfig.getPulsarAdmin()).thenReturn(pulsarAdmin); - PulsarRecordCursor pulsarRecordCursor = spy(new PulsarRecordCursor( - new ArrayList<>(), pulsarSplit, connectorConfig, Mockito.mock(ManagedLedgerFactory.class), - new ManagedLedgerConfig(), null, null)); - - Class clazz = PulsarRecordCursor.class; - Method getSchemaInfo = clazz.getDeclaredMethod("getSchemaInfo", PulsarSplit.class); - getSchemaInfo.setAccessible(true); - Field currentMessage = clazz.getDeclaredField("currentMessage"); - currentMessage.setAccessible(true); - RawMessage rawMessage = Mockito.mock(RawMessage.class); - currentMessage.set(pulsarRecordCursor, rawMessage); - - // If the schemaType of pulsarSplit is NONE or BYTES, using bytes schema - Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.NONE); - SchemaInfo schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit); - assertEquals(SchemaType.BYTES, schemaInfo.getType()); - - Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.BYTES); - schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit); - assertEquals(SchemaType.BYTES, schemaInfo.getType()); - - Mockito.when(pulsarSplit.getSchemaName()).thenReturn(Schema.BYTEBUFFER.getSchemaInfo().getName()); - schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit); - assertEquals(SchemaType.BYTES, schemaInfo.getType()); - - // If the schemaVersion of the message is not null, try to get the schema. - Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.AVRO); - Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(0).bytes()); - Mockito.when(schemas.getSchemaInfoAsync(anyString(), eq(0L))) - .thenReturn(CompletableFuture.completedFuture(Schema.AVRO(Foo.class).getSchemaInfo())); - schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit); - assertEquals(SchemaType.AVRO, schemaInfo.getType()); - - String schemaTopic = "persistent://public/default/" + topic; - - // If the schemaVersion of the message is null and the schema of pulsarSplit is null, throw runtime exception. - Mockito.when(pulsarSplit.getSchemaInfo()).thenReturn(null); - Mockito.when(rawMessage.getSchemaVersion()).thenReturn(null); - try { - schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit); - fail("The message schema version is null and the latest schema is null, should fail."); - } catch (InvocationTargetException e) { - assertTrue(e.getCause() instanceof RuntimeException); - assertTrue(e.getCause().getMessage().contains("schema of the table " + topic + " is null")); - } - - // If the schemaVersion of the message is null, try to get the latest schema. - Mockito.when(rawMessage.getSchemaVersion()).thenReturn(null); - Mockito.when(pulsarSplit.getSchemaInfo()).thenReturn(Schema.AVRO(Foo.class).getSchemaInfo()); - schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit); - assertEquals(Schema.AVRO(Foo.class).getSchemaInfo(), schemaInfo); - - // If the specific version schema is null, throw runtime exception. - Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(1L).bytes()); - Mockito.when(schemas.getSchemaInfoAsync(schemaTopic, 1)) - .thenReturn(CompletableFuture.completedFuture(null)); - try { - schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit); - fail("The specific version " + 1 + " schema is null, should fail."); - } catch (InvocationTargetException e) { - String schemaVersion = BytesSchemaVersion.of(new LongSchemaVersion(1L).bytes()).toString(); - assertTrue(e.getCause() instanceof RuntimeException); - assertTrue(e.getCause().getMessage().contains("schema of the table " + topic + " is null")); - } - - // Get the specific version schema. - Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(2L).bytes()); - Mockito.when(schemas.getSchemaInfoAsync(schemaTopic, 2)) - .thenReturn(CompletableFuture.completedFuture(Schema.AVRO(Foo.class).getSchemaInfo())); - schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit); - assertEquals(Schema.AVRO(Foo.class).getSchemaInfo(), schemaInfo); - } - -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarSplitManager.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarSplitManager.java deleted file mode 100644 index 86b2ee56c85fe..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarSplitManager.java +++ /dev/null @@ -1,490 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.airlift.json.JsonCodec; -import io.airlift.log.Logger; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorSplitSource; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.predicate.Domain; -import io.trino.spi.predicate.Range; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.predicate.ValueSet; -import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.pulsar.client.impl.schema.JSONSchema; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; -import org.apache.pulsar.common.policies.data.OffloadedReadPriority; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.common.schema.SchemaType; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.testng.Assert; -import org.testng.annotations.Test; - -import java.io.UnsupportedEncodingException; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.TimestampType.TIMESTAMP; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.anyInt; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.fail; - -public class TestPulsarSplitManager extends TestPulsarConnector { - - private static final Logger log = Logger.get(TestPulsarSplitManager.class); - - public class ResultCaptor implements Answer { - private T result = null; - public T getResult() { - return result; - } - - @Override - public T answer(InvocationOnMock invocationOnMock) throws Throwable { - result = (T) invocationOnMock.callRealMethod(); - return result; - } - } - - @Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true) - public void testTopic(String delimiter) throws Exception { - updateRewriteNamespaceDelimiterIfNeeded(delimiter); - List topics = new LinkedList<>(); - topics.addAll(topicNames.stream().filter(topicName -> !topicName.equals(NON_SCHEMA_TOPIC)).collect(Collectors.toList())); - for (TopicName topicName : topics) { - setup(); - log.info("!----- topic: %s -----!", topicName); - PulsarTableHandle pulsarTableHandle = new PulsarTableHandle(pulsarConnectorId.toString(), - topicName.getNamespace(), - topicName.getLocalName(), - topicName.getLocalName()); - PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, TupleDomain.all()); - - final ResultCaptor> resultCaptor = new ResultCaptor<>(); - doAnswer(resultCaptor).when(this.pulsarSplitManager).getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); - - - ConnectorSplitSource connectorSplitSource = this.pulsarSplitManager.getSplits( - mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class), - pulsarTableLayoutHandle, null); - - verify(this.pulsarSplitManager, times(1)) - .getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); - - int totalSize = 0; - for (PulsarSplit pulsarSplit : resultCaptor.getResult()) { - assertEquals(pulsarSplit.getConnectorId(), pulsarConnectorId.toString()); - assertEquals(pulsarSplit.getSchemaName(), topicName.getNamespace()); - assertEquals(pulsarSplit.getTableName(), topicName.getLocalName()); - assertEquals(pulsarSplit.getSchema(), - new String(topicsToSchemas.get(topicName.getSchemaName()).getSchema())); - assertEquals(pulsarSplit.getSchemaType(), topicsToSchemas.get(topicName.getSchemaName()).getType()); - assertEquals(pulsarSplit.getStartPositionEntryId(), totalSize); - assertEquals(pulsarSplit.getStartPositionLedgerId(), 0); - assertEquals(pulsarSplit.getStartPosition(), PositionImpl.get(0, totalSize)); - assertEquals(pulsarSplit.getEndPositionLedgerId(), 0); - assertEquals(pulsarSplit.getEndPositionEntryId(), totalSize + pulsarSplit.getSplitSize()); - assertEquals(pulsarSplit.getEndPosition(), PositionImpl.get(0, totalSize + pulsarSplit.getSplitSize())); - - totalSize += pulsarSplit.getSplitSize(); - } - - assertEquals(totalSize, topicsToNumEntries.get(topicName.getSchemaName()).intValue()); - cleanup(); - } - - } - - @Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true) - public void testPartitionedTopic(String delimiter) throws Exception { - updateRewriteNamespaceDelimiterIfNeeded(delimiter); - for (TopicName topicName : partitionedTopicNames) { - setup(); - log.info("!----- topic: %s -----!", topicName); - PulsarTableHandle pulsarTableHandle = new PulsarTableHandle(pulsarConnectorId.toString(), - topicName.getNamespace(), - topicName.getLocalName(), - topicName.getLocalName()); - PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, TupleDomain.all()); - - final ResultCaptor> resultCaptor = new ResultCaptor<>(); - doAnswer(resultCaptor).when(this.pulsarSplitManager).getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); - - this.pulsarSplitManager.getSplits(mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class), - pulsarTableLayoutHandle, null); - - verify(this.pulsarSplitManager, times(1)) - .getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); - - int partitions = partitionedTopicsToPartitions.get(topicName.toString()); - - for (int i = 0; i < partitions; i++) { - List splits = getSplitsForPartition(topicName.getPartition(i), resultCaptor.getResult()); - int totalSize = 0; - for (PulsarSplit pulsarSplit : splits) { - assertEquals(pulsarSplit.getConnectorId(), pulsarConnectorId.toString()); - assertEquals(pulsarSplit.getSchemaName(), topicName.getNamespace()); - assertEquals(pulsarSplit.getTableName(), topicName.getPartition(i).getLocalName()); - assertEquals(pulsarSplit.getSchema(), - new String(topicsToSchemas.get(topicName.getSchemaName()).getSchema())); - assertEquals(pulsarSplit.getSchemaType(), topicsToSchemas.get(topicName.getSchemaName()).getType()); - assertEquals(pulsarSplit.getStartPositionEntryId(), totalSize); - assertEquals(pulsarSplit.getStartPositionLedgerId(), 0); - assertEquals(pulsarSplit.getStartPosition(), PositionImpl.get(0, totalSize)); - assertEquals(pulsarSplit.getEndPositionLedgerId(), 0); - assertEquals(pulsarSplit.getEndPositionEntryId(), totalSize + pulsarSplit.getSplitSize()); - assertEquals(pulsarSplit.getEndPosition(), PositionImpl.get(0, totalSize + pulsarSplit.getSplitSize())); - - totalSize += pulsarSplit.getSplitSize(); - } - - assertEquals(totalSize, topicsToNumEntries.get(topicName.getSchemaName()).intValue()); - } - - cleanup(); - } - } - - private List getSplitsForPartition(TopicName target, Collection splits) { - return splits.stream().filter(pulsarSplit -> { - TopicName topicName = TopicName.get(pulsarSplit.getSchemaName() + "/" + pulsarSplit.getTableName()); - return target.equals(topicName); - }).collect(Collectors.toList()); - } - - @Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true) - public void testPublishTimePredicatePushdown(String delimiter) throws Exception { - updateRewriteNamespaceDelimiterIfNeeded(delimiter); - TopicName topicName = TOPIC_1; - - setup(); - log.info("!----- topic: %s -----!", topicName); - PulsarTableHandle pulsarTableHandle = new PulsarTableHandle(pulsarConnectorId.toString(), - topicName.getNamespace(), - topicName.getLocalName(), - topicName.getLocalName()); - - - Map domainMap = new HashMap<>(); - Domain domain = Domain.create(ValueSet.ofRanges(Range.range(TIMESTAMP, currentTimeMicros + 1000L, true, - currentTimeMicros + 50000L, true)), false); - domainMap.put(PulsarInternalColumn.PUBLISH_TIME.getColumnHandle(pulsarConnectorId.toString(), false), domain); - TupleDomain tupleDomain = TupleDomain.withColumnDomains(domainMap); - - PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, tupleDomain); - - final ResultCaptor> resultCaptor = new ResultCaptor<>(); - doAnswer(resultCaptor).when(this.pulsarSplitManager) - .getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); - - ConnectorSplitSource connectorSplitSource = this.pulsarSplitManager.getSplits( - mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class), - pulsarTableLayoutHandle, null); - - - verify(this.pulsarSplitManager, times(1)) - .getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); - - int totalSize = 0; - int initalStart = 1; - for (PulsarSplit pulsarSplit : resultCaptor.getResult()) { - assertEquals(pulsarSplit.getConnectorId(), pulsarConnectorId.toString()); - assertEquals(pulsarSplit.getSchemaName(), topicName.getNamespace()); - assertEquals(pulsarSplit.getTableName(), topicName.getLocalName()); - assertEquals(pulsarSplit.getSchema(), - new String(topicsToSchemas.get(topicName.getSchemaName()).getSchema())); - assertEquals(pulsarSplit.getSchemaType(), topicsToSchemas.get(topicName.getSchemaName()).getType()); - assertEquals(pulsarSplit.getStartPositionEntryId(), initalStart); - assertEquals(pulsarSplit.getStartPositionLedgerId(), 0); - assertEquals(pulsarSplit.getStartPosition(), PositionImpl.get(0, initalStart)); - assertEquals(pulsarSplit.getEndPositionLedgerId(), 0); - assertEquals(pulsarSplit.getEndPositionEntryId(), initalStart + pulsarSplit.getSplitSize()); - assertEquals(pulsarSplit.getEndPosition(), PositionImpl.get(0, initalStart + pulsarSplit - .getSplitSize())); - - initalStart += pulsarSplit.getSplitSize(); - totalSize += pulsarSplit.getSplitSize(); - } - assertEquals(totalSize, 49); - - } - - @Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true) - public void testPublishTimePredicatePushdownPartitionedTopic(String delimiter) throws Exception { - updateRewriteNamespaceDelimiterIfNeeded(delimiter); - TopicName topicName = PARTITIONED_TOPIC_1; - - setup(); - log.info("!----- topic: %s -----!", topicName); - PulsarTableHandle pulsarTableHandle = new PulsarTableHandle(pulsarConnectorId.toString(), - topicName.getNamespace(), - topicName.getLocalName(), - topicName.getLocalName()); - - - Map domainMap = new HashMap<>(); - Domain domain = Domain.create(ValueSet.ofRanges(Range.range(TIMESTAMP, currentTimeMicros + 1000L, true, - currentTimeMicros + 50000L, true)), false); - domainMap.put(PulsarInternalColumn.PUBLISH_TIME.getColumnHandle(pulsarConnectorId.toString(), false), domain); - TupleDomain tupleDomain = TupleDomain.withColumnDomains(domainMap); - - PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, tupleDomain); - - final ResultCaptor> resultCaptor = new ResultCaptor<>(); - doAnswer(resultCaptor).when(this.pulsarSplitManager) - .getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); - - ConnectorSplitSource connectorSplitSource = this.pulsarSplitManager.getSplits( - mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class), - pulsarTableLayoutHandle, null); - - - verify(this.pulsarSplitManager, times(1)) - .getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); - - - int partitions = partitionedTopicsToPartitions.get(topicName.toString()); - for (int i = 0; i < partitions; i++) { - List splits = getSplitsForPartition(topicName.getPartition(i), resultCaptor.getResult()); - int totalSize = 0; - int initialStart = 1; - for (PulsarSplit pulsarSplit : splits) { - assertEquals(pulsarSplit.getConnectorId(), pulsarConnectorId.toString()); - assertEquals(pulsarSplit.getSchemaName(), topicName.getNamespace()); - assertEquals(pulsarSplit.getTableName(), topicName.getPartition(i).getLocalName()); - assertEquals(pulsarSplit.getSchema(), - new String(topicsToSchemas.get(topicName.getSchemaName()).getSchema())); - assertEquals(pulsarSplit.getSchemaType(), topicsToSchemas.get(topicName.getSchemaName()).getType()); - assertEquals(pulsarSplit.getStartPositionEntryId(), initialStart); - assertEquals(pulsarSplit.getStartPositionLedgerId(), 0); - assertEquals(pulsarSplit.getStartPosition(), PositionImpl.get(0, initialStart)); - assertEquals(pulsarSplit.getEndPositionLedgerId(), 0); - assertEquals(pulsarSplit.getEndPositionEntryId(), initialStart + pulsarSplit.getSplitSize()); - assertEquals(pulsarSplit.getEndPosition(), PositionImpl.get(0, initialStart + pulsarSplit.getSplitSize())); - - initialStart += pulsarSplit.getSplitSize(); - totalSize += pulsarSplit.getSplitSize(); - } - - assertEquals(totalSize, 49); - } - } - - @Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true) - public void testPartitionFilter(String delimiter) throws Exception { - updateRewriteNamespaceDelimiterIfNeeded(delimiter); - for (TopicName topicName : partitionedTopicNames) { - setup(); - log.info("!----- topic: %s -----!", topicName); - PulsarTableHandle pulsarTableHandle = mock(PulsarTableHandle.class); - when(pulsarTableHandle.getConnectorId()).thenReturn(pulsarConnectorId.toString()); - when(pulsarTableHandle.getSchemaName()).thenReturn(topicName.getNamespace()); - when(pulsarTableHandle.getTopicName()).thenReturn(topicName.getLocalName()); - when(pulsarTableHandle.getTableName()).thenReturn(topicName.getLocalName()); - - // test single domain with equal low and high of "__partition__" - Map domainMap = new HashMap<>(); - Domain domain = Domain.create(ValueSet.ofRanges(Range.range(INTEGER, 0L, true, - 0L, true)), false); - domainMap.put(PulsarInternalColumn.PARTITION.getColumnHandle(pulsarConnectorId.toString(), false), domain); - TupleDomain tupleDomain = TupleDomain.withColumnDomains(domainMap); - Collection splits = this.pulsarSplitManager.getSplitsPartitionedTopic(2, topicName, pulsarTableHandle, - schemas.getSchemaInfo(topicName.getSchemaName()), tupleDomain, null); - if (topicsToNumEntries.get(topicName.getSchemaName()) > 1) { - Assert.assertEquals(splits.size(), 2); - } - for (PulsarSplit split : splits) { - assertEquals(TopicName.getPartitionIndex(split.getTableName()), 0); - } - - // test multiple domain with equal low and high of "__partition__" - domainMap.clear(); - domain = Domain.create(ValueSet.ofRanges( - Range.range(INTEGER, 0L, true, 0L, true), - Range.range(INTEGER, 3L, true, 3L, true)), - false); - domainMap.put(PulsarInternalColumn.PARTITION.getColumnHandle(pulsarConnectorId.toString(), false), domain); - tupleDomain = TupleDomain.withColumnDomains(domainMap); - splits = this.pulsarSplitManager.getSplitsPartitionedTopic(1, topicName, pulsarTableHandle, - schemas.getSchemaInfo(topicName.getSchemaName()), tupleDomain, null); - if (topicsToNumEntries.get(topicName.getSchemaName()) > 1) { - Assert.assertEquals(splits.size(), 2); - } - for (PulsarSplit split : splits) { - assertTrue(TopicName.getPartitionIndex(split.getTableName()) == 0 || TopicName.getPartitionIndex(split.getTableName()) == 3); - } - - // test single domain with unequal low and high of "__partition__" - domainMap.clear(); - domain = Domain.create(ValueSet.ofRanges( - Range.range(INTEGER, 0L, true, 2L, true)), - false); - domainMap.put(PulsarInternalColumn.PARTITION.getColumnHandle(pulsarConnectorId.toString(), false), domain); - tupleDomain = TupleDomain.withColumnDomains(domainMap); - splits = this.pulsarSplitManager.getSplitsPartitionedTopic(2, topicName, pulsarTableHandle, - schemas.getSchemaInfo(topicName.getSchemaName()), tupleDomain, null); - if (topicsToNumEntries.get(topicName.getSchemaName()) > 1) { - Assert.assertEquals(splits.size(), 3); - } - for (PulsarSplit split : splits) { - assertTrue(TopicName.getPartitionIndex(split.getTableName()) == 0 - || TopicName.getPartitionIndex(split.getTableName()) == 1 - || TopicName.getPartitionIndex(split.getTableName()) == 2); - } - - // test multiple domain with unequal low and high of "__partition__" - domainMap.clear(); - domain = Domain.create(ValueSet.ofRanges( - Range.range(INTEGER, 0L, true, 1L, true), - Range.range(INTEGER, 3L, true, 4L, true)), - false); - domainMap.put(PulsarInternalColumn.PARTITION.getColumnHandle(pulsarConnectorId.toString(), false), domain); - tupleDomain = TupleDomain.withColumnDomains(domainMap); - splits = this.pulsarSplitManager.getSplitsPartitionedTopic(2, topicName, pulsarTableHandle, - schemas.getSchemaInfo(topicName.getSchemaName()), tupleDomain, null); - if (topicsToNumEntries.get(topicName.getSchemaName()) > 1) { - Assert.assertEquals(splits.size(), 4); - } - for (PulsarSplit split : splits) { - assertTrue(TopicName.getPartitionIndex(split.getTableName()) == 0 - || TopicName.getPartitionIndex(split.getTableName()) == 1 - || TopicName.getPartitionIndex(split.getTableName()) == 3 - || TopicName.getPartitionIndex(split.getTableName()) == 4); - } - } - - - } - - @Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true) - public void testGetSplitNonSchema(String delimiter) throws Exception { - updateRewriteNamespaceDelimiterIfNeeded(delimiter); - TopicName topicName = NON_SCHEMA_TOPIC; - setup(); - log.info("!----- topic: %s -----!", topicName); - PulsarTableHandle pulsarTableHandle = new PulsarTableHandle(pulsarConnectorId.toString(), - topicName.getNamespace(), - topicName.getLocalName(), - topicName.getLocalName()); - - Map domainMap = new HashMap<>(); - TupleDomain tupleDomain = TupleDomain.withColumnDomains(domainMap); - - PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, tupleDomain); - ConnectorSplitSource connectorSplitSource = this.pulsarSplitManager.getSplits( - mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class), - pulsarTableLayoutHandle, null); - assertNotNull(connectorSplitSource); - } - - @Test - public void pulsarSplitJsonCodecTest() throws JsonProcessingException, UnsupportedEncodingException { - OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.create( - "aws-s3", - "test-region", - "test-bucket", - "test-endpoint", - "role-", - "role-session-name", - "test-credential-id", - "test-credential-secret", - 5000, - 2000, - 1000L, - 1000L, - 5000L, - OffloadedReadPriority.BOOKKEEPER_FIRST - ); - - SchemaInfo schemaInfo = JSONSchema.of(Foo.class).getSchemaInfo(); - final String schema = new String(schemaInfo.getSchema(), "ISO8859-1"); - final String originSchemaName = schemaInfo.getName(); - final String schemaName = schemaInfo.getName(); - final String schemaInfoProperties = new ObjectMapper().writeValueAsString(schemaInfo.getProperties()); - final SchemaType schemaType = schemaInfo.getType(); - - final long splitId = 1; - final String connectorId = "connectorId"; - final String tableName = "tableName"; - final long splitSize = 5; - final long startPositionEntryId = 22; - final long endPositionEntryId = 33; - final long startPositionLedgerId = 10; - final long endPositionLedgerId = 21; - final TupleDomain tupleDomain = TupleDomain.all(); - - byte[] pulsarSplitData; - JsonCodec jsonCodec = JsonCodec.jsonCodec(PulsarSplit.class); - try { - PulsarSplit pulsarSplit = new PulsarSplit( - splitId, connectorId, schemaName, originSchemaName, tableName, splitSize, schema, - schemaType, startPositionEntryId, endPositionEntryId, startPositionLedgerId, - endPositionLedgerId, tupleDomain, schemaInfoProperties, offloadPolicies); - pulsarSplitData = jsonCodec.toJsonBytes(pulsarSplit); - } catch (Exception e) { - e.printStackTrace(); - log.error("Failed to serialize the PulsarSplit.", e); - fail("Failed to serialize the PulsarSplit."); - return; - } - - try { - PulsarSplit pulsarSplit = jsonCodec.fromJson(pulsarSplitData); - Assert.assertEquals(pulsarSplit.getSchema(), schema); - Assert.assertEquals(pulsarSplit.getOriginSchemaName(), originSchemaName); - Assert.assertEquals(pulsarSplit.getSchemaName(), schemaName); - Assert.assertEquals(pulsarSplit.getSchemaInfoProperties(), schemaInfoProperties); - Assert.assertEquals(pulsarSplit.getSchemaType(), schemaType); - Assert.assertEquals(pulsarSplit.getSplitId(), splitId); - Assert.assertEquals(pulsarSplit.getConnectorId(), connectorId); - Assert.assertEquals(pulsarSplit.getTableName(), tableName); - Assert.assertEquals(pulsarSplit.getSplitSize(), splitSize); - Assert.assertEquals(pulsarSplit.getStartPositionEntryId(), startPositionEntryId); - Assert.assertEquals(pulsarSplit.getEndPositionEntryId(), endPositionEntryId); - Assert.assertEquals(pulsarSplit.getStartPositionLedgerId(), startPositionLedgerId); - Assert.assertEquals(pulsarSplit.getEndPositionLedgerId(), endPositionLedgerId); - Assert.assertEquals(pulsarSplit.getTupleDomain(), tupleDomain); - Assert.assertEquals(pulsarSplit.getOffloadPolicies(), offloadPolicies); - } catch (Exception e) { - log.error("Failed to deserialize the PulsarSplit.", e); - fail("Failed to deserialize the PulsarSplit."); - } - - } - -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestReadChunkedMessages.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestReadChunkedMessages.java deleted file mode 100644 index 1e959e9b83059..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestReadChunkedMessages.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import com.google.common.collect.Sets; -import io.trino.spi.connector.ConnectorContext; -import io.trino.spi.predicate.TupleDomain; -import io.trino.testing.TestingConnectorContext; -import java.util.Collection; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicInteger; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.mledger.ManagedLedgerConfig; -import org.apache.bookkeeper.stats.NullStatsProvider; -import org.apache.commons.lang3.RandomUtils; -import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; -import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.impl.MessageIdImpl; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.testng.Assert; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -/** - * Test read chunked messages. - */ -@Test -@Slf4j -public class TestReadChunkedMessages extends MockedPulsarServiceBaseTest { - - private final static int MAX_MESSAGE_SIZE = 1024 * 1024; - - @EqualsAndHashCode - @Data - static class Movie { - private String name; - private Long publishTime; - private byte[] binaryData; - } - - @EqualsAndHashCode - @Data - static class MovieMessage { - private Movie movie; - private String messageId; - } - - @BeforeClass - @Override - public void setup() throws Exception { - conf.setMaxMessageSize(MAX_MESSAGE_SIZE); - conf.setManagedLedgerMaxEntriesPerLedger(5); - conf.setManagedLedgerMinLedgerRolloverTimeMinutes(0); - internalSetup(); - - admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(brokerUrl.toString()).build()); - - // so that clients can test short names - admin.tenants().createTenant("public", - new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("test"))); - admin.namespaces().createNamespace("public/default"); - admin.namespaces().setNamespaceReplicationClusters("public/default", Sets.newHashSet("test")); - } - - @AfterClass - @Override - public void cleanup() throws Exception { - internalCleanup(); - } - - @Test - public void queryTest() throws Exception { - String topic = "chunk-topic"; - TopicName topicName = TopicName.get(topic); - int messageCnt = 20; - Set messageSet = prepareChunkedData(topic, messageCnt); - SchemaInfo schemaInfo = Schema.AVRO(Movie.class).getSchemaInfo(); - - PulsarConnectorConfig connectorConfig = new PulsarConnectorConfig(); - connectorConfig.setWebServiceUrl(pulsar.getWebServiceAddress()); - PulsarSplitManager pulsarSplitManager = new PulsarSplitManager(new PulsarConnectorId("1"), connectorConfig); - Collection splits = pulsarSplitManager.getSplitsForTopic( - topicName.getPersistenceNamingEncoding(), - pulsar.getManagedLedgerFactory(), - new ManagedLedgerConfig(), - 3, - new PulsarTableHandle("1", topicName.getNamespace(), topic, topic), - schemaInfo, - topic, - TupleDomain.all(), - null); - - List columnHandleList = TestPulsarConnector.getColumnColumnHandles( - topicName, schemaInfo, PulsarColumnHandle.HandleKeyValueType.NONE, true); - ConnectorContext prestoConnectorContext = new TestingConnectorContext(); - - for (PulsarSplit split : splits) { - queryAndCheck(columnHandleList, split, connectorConfig, prestoConnectorContext, messageSet); - } - Assert.assertTrue(messageSet.isEmpty()); - } - - private Set prepareChunkedData(String topic, int messageCnt) throws PulsarClientException, InterruptedException { - pulsarClient.newConsumer(Schema.AVRO(Movie.class)) - .topic(topic) - .subscriptionName("sub") - .subscribe() - .close(); - Producer producer = pulsarClient.newProducer(Schema.AVRO(Movie.class)) - .topic(topic) - .enableBatching(false) - .enableChunking(true) - .create(); - Set messageSet = new LinkedHashSet<>(); - CountDownLatch countDownLatch = new CountDownLatch(messageCnt); - for (int i = 0; i < messageCnt; i++) { - final double dataTimes = (i % 5) * 0.5; - byte[] movieBinaryData = RandomUtils.nextBytes((int) (MAX_MESSAGE_SIZE * dataTimes)); - final int length = movieBinaryData.length; - final int index = i; - - Movie movie = new Movie(); - movie.setName("movie-" + i); - movie.setPublishTime(System.currentTimeMillis()); - movie.setBinaryData(movieBinaryData); - producer.newMessage().value(movie).sendAsync() - .whenComplete((msgId, throwable) -> { - if (throwable != null) { - log.error("Failed to produce message.", throwable); - countDownLatch.countDown(); - return; - } - MovieMessage movieMessage = new MovieMessage(); - movieMessage.setMovie(movie); - MessageIdImpl messageId = (MessageIdImpl) msgId; - movieMessage.setMessageId("(" + messageId.getLedgerId() + "," + messageId.getEntryId() + ",0)"); - messageSet.add(movieMessage); - countDownLatch.countDown(); - }); - } - countDownLatch.await(); - Assert.assertEquals(messageCnt, messageSet.size()); - producer.close(); - return messageSet; - } - - private void queryAndCheck(List columnHandleList, - PulsarSplit split, - PulsarConnectorConfig connectorConfig, - ConnectorContext prestoConnectorContext, - Set messageSet) { - PulsarRecordCursor pulsarRecordCursor = new PulsarRecordCursor( - columnHandleList, split, connectorConfig, pulsar.getManagedLedgerFactory(), - new ManagedLedgerConfig(), new PulsarConnectorMetricsTracker(new NullStatsProvider()), - new PulsarDispatchingRowDecoderFactory(prestoConnectorContext.getTypeManager())); - - AtomicInteger receiveMsgCnt = new AtomicInteger(messageSet.size()); - while (pulsarRecordCursor.advanceNextPosition()) { - Movie movie = new Movie(); - MovieMessage movieMessage = new MovieMessage(); - movieMessage.setMovie(movie); - for (int i = 0; i < columnHandleList.size(); i++) { - switch (columnHandleList.get(i).getName()) { - case "binaryData": - movie.setBinaryData(pulsarRecordCursor.getSlice(i).getBytes()); - break; - case "name": - movie.setName(new String(pulsarRecordCursor.getSlice(i).getBytes())); - break; - case "publishTime": - movie.setPublishTime(pulsarRecordCursor.getLong(i)); - break; - case "__message_id__": - movieMessage.setMessageId(new String(pulsarRecordCursor.getSlice(i).getBytes())); - default: - // do nothing - break; - } - } - - Assert.assertTrue(messageSet.contains(movieMessage)); - messageSet.remove(movieMessage); - receiveMsgCnt.decrementAndGet(); - } - } - -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/AbstractDecoderTester.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/AbstractDecoderTester.java deleted file mode 100644 index 6ed07fd1dde5f..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/AbstractDecoderTester.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.decoder; - -import io.airlift.slice.Slice; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.decoder.FieldValueProvider; -import io.trino.spi.block.Block; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorContext; -import io.trino.spi.type.Type; -import io.trino.testing.TestingConnectorContext; -import java.math.BigDecimal; -import org.apache.pulsar.common.naming.NamespaceName; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.sql.presto.PulsarAuth; -import org.apache.pulsar.sql.presto.PulsarColumnHandle; -import org.apache.pulsar.sql.presto.PulsarColumnMetadata; -import org.apache.pulsar.sql.presto.PulsarConnectorConfig; -import org.apache.pulsar.sql.presto.PulsarConnectorId; -import org.apache.pulsar.sql.presto.PulsarDispatchingRowDecoderFactory; -import org.apache.pulsar.sql.presto.PulsarMetadata; -import org.apache.pulsar.sql.presto.PulsarRowDecoder; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import static org.mockito.Mockito.spy; -import static org.testng.Assert.assertNotNull; - -/** - * Abstract superclass for TestXXDecoder (e.g. TestAvroDecoder 、TestJsonDecoder). - */ -public abstract class AbstractDecoderTester { - - protected PulsarDispatchingRowDecoderFactory decoderFactory; - protected PulsarConnectorId pulsarConnectorId = new PulsarConnectorId("test-connector"); - protected SchemaInfo schemaInfo; - protected TopicName topicName; - protected List pulsarColumnHandle; - protected PulsarRowDecoder pulsarRowDecoder; - protected DecoderTestUtil decoderTestUtil; - protected PulsarConnectorConfig pulsarConnectorConfig; - protected PulsarMetadata pulsarMetadata; - - protected void init() { - ConnectorContext prestoConnectorContext = new TestingConnectorContext(); - this.decoderFactory = new PulsarDispatchingRowDecoderFactory(prestoConnectorContext.getTypeManager()); - this.pulsarConnectorConfig = spy(PulsarConnectorConfig.class); - this.pulsarConnectorConfig.setMaxEntryReadBatchSize(1); - this.pulsarConnectorConfig.setMaxSplitEntryQueueSize(10); - this.pulsarConnectorConfig.setMaxSplitMessageQueueSize(100); - this.pulsarMetadata = new PulsarMetadata(pulsarConnectorId, this.pulsarConnectorConfig, decoderFactory, - new PulsarAuth(this.pulsarConnectorConfig)); - this.topicName = TopicName.get("persistent", NamespaceName.get("tenant-1", "ns-1"), "topic-1"); - } - - protected void checkArrayValues(Block block, Type type, Object value) { - decoderTestUtil.checkArrayValues(block, type, value); - } - - protected void checkMapValues(Block block, Type type, Object value) { - decoderTestUtil.checkMapValues(block, type, value); - } - - protected void checkRowValues(Block block, Type type, Object value) { - decoderTestUtil.checkRowValues(block, type, value); - } - - protected void checkValue(Map decodedRow, DecoderColumnHandle handle, Slice value) { - decoderTestUtil.checkValue(decodedRow, handle, value); - } - - protected void checkValue(Map decodedRow, DecoderColumnHandle handle, String value) { - decoderTestUtil.checkValue(decodedRow, handle, value); - } - - protected void checkValue(Map decodedRow, DecoderColumnHandle handle, long value) { - decoderTestUtil.checkValue(decodedRow, handle, value); - } - - protected void checkValue(Map decodedRow, DecoderColumnHandle handle, double value) { - decoderTestUtil.checkValue(decodedRow, handle, value); - } - - protected void checkValue(Map decodedRow, DecoderColumnHandle handle, boolean value) { - decoderTestUtil.checkValue(decodedRow, handle, value); - } - - protected void checkValue(Map decodedRow, DecoderColumnHandle handle, BigDecimal value) { - decoderTestUtil.checkValue(decodedRow, handle, value); - } - - protected Block getBlock(Map decodedRow, DecoderColumnHandle handle) { - FieldValueProvider provider = decodedRow.get(handle); - assertNotNull(provider); - return provider.getBlock(); - } - - protected List getColumnColumnHandles(TopicName topicName, SchemaInfo schemaInfo, - PulsarColumnHandle.HandleKeyValueType handleKeyValueType, boolean includeInternalColumn, PulsarDispatchingRowDecoderFactory dispatchingRowDecoderFactory) { - List columnHandles = new ArrayList<>(); - List columnMetadata = pulsarMetadata.getPulsarColumns(topicName, schemaInfo, - includeInternalColumn, handleKeyValueType); - - columnMetadata.forEach(column -> { - PulsarColumnMetadata pulsarColumnMetadata = (PulsarColumnMetadata) column; - columnHandles.add(new PulsarColumnHandle( - pulsarConnectorId.toString(), - pulsarColumnMetadata.getNameWithCase(), - pulsarColumnMetadata.getType(), - pulsarColumnMetadata.isHidden(), - pulsarColumnMetadata.isInternal(), - pulsarColumnMetadata.getDecoderExtraInfo().getMapping(), - pulsarColumnMetadata.getDecoderExtraInfo().getDataFormat(), pulsarColumnMetadata.getDecoderExtraInfo().getFormatHint(), - pulsarColumnMetadata.getHandleKeyValueType())); - - }); - return columnHandles; - } - -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestMessage.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestMessage.java deleted file mode 100644 index 4561282c67196..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestMessage.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.decoder; - -import java.math.BigDecimal; -import lombok.Data; - -import java.util.List; -import java.util.Map; - -public class DecoderTestMessage { - - public static enum TestEnum { - TEST_ENUM_1, - TEST_ENUM_2, - TEST_ENUM_3 - } - - public int intField; - public String stringField; - public float floatField; - public double doubleField; - public boolean booleanField; - public long longField; - @org.apache.avro.reflect.AvroSchema("{ \"type\": \"long\", \"logicalType\": \"timestamp-millis\" }") - public long timestampField; - @org.apache.avro.reflect.AvroSchema("{ \"type\": \"int\", \"logicalType\": \"time-millis\" }") - public int timeField; - @org.apache.avro.reflect.AvroSchema("{ \"type\": \"int\", \"logicalType\": \"date\" }") - public int dateField; - public TestRow rowField; - public TestEnum enumField; - @org.apache.avro.reflect.AvroSchema("{ \"type\": \"bytes\", \"logicalType\": \"decimal\", \"precision\": 4, \"scale\": 2 }") - public BigDecimal decimalField; - @org.apache.avro.reflect.AvroSchema("{ \"type\": \"bytes\", \"logicalType\": \"decimal\", \"precision\": 30, \"scale\": 2 }") - public BigDecimal longDecimalField; - - public List arrayField; - public Map mapField; - public CompositeRow compositeRow; - - public static class TestRow { - public String stringField; - public int intField; - public NestedRow nestedRow; - } - - - public static class NestedRow { - public String stringField; - public long longField; - } - - public static class CompositeRow { - public String stringField; - public List arrayField; - public Map mapField; - public NestedRow nestedRow; - public Map> structedField; - } - - /** - * POJO for cyclic detect. - */ - @Data - public static class CyclicFoo { - private String field1; - private Integer field2; - private CyclicBoo boo; - } - - @Data - public static class CyclicBoo { - private String field1; - private Boolean field2; - private CyclicFoo foo; - } - -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java deleted file mode 100644 index 60d6028f239d7..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.decoder; - -import static io.trino.testing.TestingConnectorSession.SESSION; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertTrue; -import io.airlift.slice.Slice; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.decoder.FieldValueProvider; -import io.trino.spi.block.Block; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.DecimalType; -import io.trino.spi.type.Int128; -import io.trino.spi.type.MapType; -import io.trino.spi.type.RowType; -import io.trino.spi.type.Type; -import java.math.BigDecimal; -import java.util.Map; - -/** - * Abstract util superclass for XXDecoderTestUtil (e.g. AvroDecoderTestUtil 、JsonDecoderTestUtil) - */ -public abstract class DecoderTestUtil { - - protected DecoderTestUtil() { - - } - - public abstract void checkArrayValues(Block block, Type type, Object value); - - public abstract void checkMapValues(Block block, Type type, Object value); - - public abstract void checkRowValues(Block block, Type type, Object value); - - public abstract void checkPrimitiveValue(Object actual, Object expected); - - public void checkField(Block actualBlock, Type type, int position, Object expectedValue) { - assertNotNull(type, "Type is null"); - assertNotNull(actualBlock, "actualBlock is null"); - assertTrue(!actualBlock.isNull(position)); - assertNotNull(expectedValue, "expectedValue is null"); - - if (type instanceof ArrayType) { - checkArrayValues(actualBlock.getObject(position, Block.class), type, expectedValue); - } else if (type instanceof MapType) { - checkMapValues(actualBlock.getObject(position, Block.class), type, expectedValue); - } else if (type instanceof RowType) { - checkRowValues(actualBlock.getObject(position, Block.class), type, expectedValue); - } else { - checkPrimitiveValue(getObjectValue(type, actualBlock, position), expectedValue); - } - } - - public boolean isIntegralType(Object value) { - return value instanceof Long - || value instanceof Integer - || value instanceof Short - || value instanceof Byte; - } - - public boolean isRealType(Object value) { - return value instanceof Float || value instanceof Double; - } - - public Object getObjectValue(Type type, Block block, int position) { - if (block.isNull(position)) { - return null; - } - return type.getObjectValue(SESSION, block, position); - } - - public void checkValue(Map decodedRow, DecoderColumnHandle handle, Slice value) { - FieldValueProvider provider = decodedRow.get(handle); - assertNotNull(provider); - assertEquals(provider.getSlice(), value); - } - - public void checkValue(Map decodedRow, DecoderColumnHandle handle, String value) { - FieldValueProvider provider = decodedRow.get(handle); - assertNotNull(provider); - assertEquals(provider.getSlice().toStringUtf8(), value); - } - - public void checkValue(Map decodedRow, DecoderColumnHandle handle, long value) { - FieldValueProvider provider = decodedRow.get(handle); - assertNotNull(provider); - assertEquals(provider.getLong(), value); - } - - public void checkValue(Map decodedRow, DecoderColumnHandle handle, double value) { - FieldValueProvider provider = decodedRow.get(handle); - assertNotNull(provider); - assertEquals(provider.getDouble(), value, 0.0001); - } - - public void checkValue(Map decodedRow, DecoderColumnHandle handle, boolean value) { - FieldValueProvider provider = decodedRow.get(handle); - assertNotNull(provider); - assertEquals(provider.getBoolean(), value); - } - - public void checkValue(Map decodedRow, DecoderColumnHandle handle, BigDecimal value) { - FieldValueProvider provider = decodedRow.get(handle); - DecimalType decimalType = (DecimalType) handle.getType(); - BigDecimal actualDecimal; - if (decimalType.getFixedSize() == Int128.SIZE) { - final Block block = provider.getBlock(); - final Int128 actualValue = (Int128) decimalType.getObject(block, 0); - actualDecimal = new BigDecimal(actualValue.toBigInteger(), decimalType.getScale()); - } else { - actualDecimal = BigDecimal.valueOf(provider.getLong(), decimalType.getScale()); - } - assertNotNull(provider); - assertEquals(actualDecimal, value); - } - - public void checkIsNull(Map decodedRow, DecoderColumnHandle handle) { - FieldValueProvider provider = decodedRow.get(handle); - assertNotNull(provider); - assertTrue(provider.isNull()); - } -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/AvroDecoderTestUtil.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/AvroDecoderTestUtil.java deleted file mode 100644 index a32a8d47dfc46..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/AvroDecoderTestUtil.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.decoder.avro; - -import io.trino.spi.block.Block; -import io.trino.spi.type.*; -import org.apache.avro.generic.GenericEnumSymbol; -import org.apache.avro.generic.GenericFixed; -import org.apache.avro.generic.GenericRecord; -import org.apache.pulsar.sql.presto.decoder.DecoderTestUtil; - -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Map; - -import static io.trino.spi.type.VarcharType.VARCHAR; -import static java.lang.String.format; -import static org.testng.Assert.*; - -/** - * TestUtil for AvroDecoder - */ -public class AvroDecoderTestUtil extends DecoderTestUtil { - public AvroDecoderTestUtil() { - super(); - } - - public void checkPrimitiveValue(Object actual, Object expected) { - if (actual == null || expected == null) { - assertNull(expected); - assertNull(actual); - } else if (actual instanceof CharSequence) { - assertTrue(expected instanceof CharSequence || expected instanceof GenericEnumSymbol); - assertEquals(actual.toString(), expected.toString()); - } else if (actual instanceof SqlVarbinary) { - if (expected instanceof GenericFixed) { - assertEquals(((SqlVarbinary) actual).getBytes(), ((GenericFixed) expected).bytes()); - } else if (expected instanceof ByteBuffer) { - assertEquals(((SqlVarbinary) actual).getBytes(), ((ByteBuffer) expected).array()); - } else { - fail(format("Unexpected value type %s", actual.getClass())); - } - } else if (isIntegralType(actual) && isIntegralType(expected)) { - assertEquals(((Number) actual).longValue(), ((Number) expected).longValue()); - } else if (isRealType(actual) && isRealType(expected)) { - assertEquals(((Number) actual).doubleValue(), ((Number) expected).doubleValue()); - } else { - assertEquals(actual, expected); - } - } - - - public void checkArrayValues(Block block, Type type, Object value) { - assertNotNull(type, "Type is null"); - assertTrue(type instanceof ArrayType, "Unexpected type"); - assertNotNull(block, "Block is null"); - assertNotNull(value, "Value is null"); - - List list = (List) value; - - assertEquals(block.getPositionCount(), list.size()); - Type elementType = ((ArrayType) type).getElementType(); - if (elementType instanceof ArrayType) { - for (int index = 0; index < block.getPositionCount(); index++) { - if (block.isNull(index)) { - assertNull(list.get(index)); - continue; - } - Block arrayBlock = block.getObject(index, Block.class); - checkArrayValues(arrayBlock, elementType, list.get(index)); - } - } else if (elementType instanceof MapType) { - for (int index = 0; index < block.getPositionCount(); index++) { - if (block.isNull(index)) { - assertNull(list.get(index)); - continue; - } - Block mapBlock = block.getObject(index, Block.class); - checkMapValues(mapBlock, elementType, list.get(index)); - } - } else if (elementType instanceof RowType) { - for (int index = 0; index < block.getPositionCount(); index++) { - if (block.isNull(index)) { - assertNull(list.get(index)); - continue; - } - Block rowBlock = block.getObject(index, Block.class); - checkRowValues(rowBlock, elementType, list.get(index)); - } - } else { - for (int index = 0; index < block.getPositionCount(); index++) { - checkPrimitiveValue(getObjectValue(elementType, block, index), list.get(index)); - } - } - } - - /** - * fix key as org.apache.avro.util.Utf8 - * - * @param block - * @param type - * @param value - */ - public void checkMapValues(Block block, Type type, Object value) { - assertNotNull(type, "Type is null"); - assertTrue(type instanceof MapType, "Unexpected type"); - assertTrue(((MapType) type).getKeyType() instanceof VarcharType, "Unexpected key type"); - assertNotNull(block, "Block is null"); - assertNotNull(value, "Value is null"); - - - Map expected = (Map) value; - - assertEquals(block.getPositionCount(), expected.size() * 2); - Type valueType = ((MapType) type).getValueType(); - if (valueType instanceof ArrayType) { - for (int index = 0; index < block.getPositionCount(); index += 2) { - String actualKey = VARCHAR.getSlice(block, index).toStringUtf8(); - assertTrue(expected.keySet().stream().anyMatch(e -> e.toString().equals(actualKey))); - if (block.isNull(index + 1)) { - assertNull(expected.get(actualKey)); - continue; - } - Block arrayBlock = block.getObject(index + 1, Block.class); - Object keyValue = expected.entrySet().stream().filter(e -> e.getKey().toString().equals(actualKey)).findFirst().get().getValue(); - checkArrayValues(arrayBlock, valueType, keyValue); - } - } else if (valueType instanceof MapType) { - for (int index = 0; index < block.getPositionCount(); index += 2) { - String actualKey = VARCHAR.getSlice(block, index).toStringUtf8(); - assertTrue(expected.keySet().stream().anyMatch(e -> e.toString().equals(actualKey))); - if (block.isNull(index + 1)) { - assertNull(expected.get(actualKey)); - continue; - } - Block mapBlock = block.getObject(index + 1, Block.class); - Object keyValue = expected.entrySet().stream().filter(e -> e.getKey().toString().equals(actualKey)).findFirst().get().getValue(); - checkMapValues(mapBlock, valueType, keyValue); - } - } else if (valueType instanceof RowType) { - for (int index = 0; index < block.getPositionCount(); index += 2) { - String actualKey = VARCHAR.getSlice(block, index).toStringUtf8(); - assertTrue(expected.keySet().stream().anyMatch(e -> e.toString().equals(actualKey))); - if (block.isNull(index + 1)) { - assertNull(expected.get(actualKey)); - continue; - } - Block rowBlock = block.getObject(index + 1, Block.class); - Object keyValue = expected.entrySet().stream().filter(e -> e.getKey().toString().equals(actualKey)).findFirst().get().getValue(); - checkRowValues(rowBlock, valueType, keyValue); - } - } else { - for (int index = 0; index < block.getPositionCount(); index += 2) { - String actualKey = VARCHAR.getSlice(block, index).toStringUtf8(); - assertTrue(expected.keySet().stream().anyMatch(e -> e.toString().equals(actualKey))); - Object keyValue = expected.entrySet().stream().filter(e -> e.getKey().toString().equals(actualKey)).findFirst().get().getValue(); - checkPrimitiveValue(getObjectValue(valueType, block, index + 1), keyValue); - } - } - } - - public void checkRowValues(Block block, Type type, Object value) { - assertNotNull(type, "Type is null"); - assertTrue(type instanceof RowType, "Unexpected type"); - assertNotNull(block, "Block is null"); - assertNotNull(value, "Value is null"); - - GenericRecord record = (GenericRecord) value; - RowType rowType = (RowType) type; - assertEquals(record.getSchema().getFields().size(), rowType.getFields().size(), "Avro field size mismatch"); - assertEquals(block.getPositionCount(), rowType.getFields().size(), "Presto type field size mismatch"); - for (int fieldIndex = 0; fieldIndex < rowType.getFields().size(); fieldIndex++) { - RowType.Field rowField = rowType.getFields().get(fieldIndex); - Object expectedValue = record.get(rowField.getName().get()); - if (block.isNull(fieldIndex)) { - assertNull(expectedValue); - continue; - } - checkField(block, rowField.getType(), fieldIndex, expectedValue); - } - } - -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java deleted file mode 100644 index c4e7009b9465b..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java +++ /dev/null @@ -1,329 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.decoder.avro; - -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.RealType.REAL; -import static io.trino.spi.type.TimeType.TIME_MILLIS; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static java.lang.Float.floatToIntBits; -import static org.apache.pulsar.sql.presto.TestPulsarConnector.getPulsarConnectorId; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.expectThrows; -import com.google.common.collect.ImmutableList; -import io.netty.buffer.ByteBuf; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.decoder.FieldValueProvider; -import io.trino.spi.TrinoException; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.BigintType; -import io.trino.spi.type.DecimalType; -import io.trino.spi.type.RowType; -import io.trino.spi.type.StandardTypes; -import io.trino.spi.type.Timestamps; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeSignatureParameter; -import io.trino.spi.type.VarcharType; -import java.math.BigDecimal; -import java.time.LocalDate; -import java.time.LocalTime; -import java.time.ZoneId; -import java.time.temporal.ChronoUnit; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import org.apache.pulsar.client.impl.schema.AvroSchema; -import org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord; -import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; -import org.apache.pulsar.sql.presto.PulsarColumnHandle; -import org.apache.pulsar.sql.presto.decoder.AbstractDecoderTester; -import org.apache.pulsar.sql.presto.decoder.DecoderTestMessage; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -public class TestAvroDecoder extends AbstractDecoderTester { - - private AvroSchema schema; - - @BeforeMethod - public void init() { - super.init(); - schema = AvroSchema.of(DecoderTestMessage.class); - schemaInfo = schema.getSchemaInfo(); - pulsarColumnHandle = getColumnColumnHandles(topicName, schemaInfo, PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); - pulsarRowDecoder = decoderFactory.createRowDecoder(topicName, schemaInfo, new HashSet<>(pulsarColumnHandle)); - decoderTestUtil = new AvroDecoderTestUtil(); - assertTrue(pulsarRowDecoder instanceof PulsarAvroRowDecoder); - } - - @Test - public void testPrimitiveType() { - DecoderTestMessage message = new DecoderTestMessage(); - message.stringField = "message_1"; - message.intField = 22; - message.floatField = 2.2f; - message.doubleField = 22.20D; - message.booleanField = true; - message.longField = 222L; - message.timestampField = System.currentTimeMillis(); - message.enumField = DecoderTestMessage.TestEnum.TEST_ENUM_1; - - LocalTime now = LocalTime.now(ZoneId.systemDefault()); - message.timeField = now.toSecondOfDay() * 1000; - - LocalDate localDate = LocalDate.now(); - LocalDate epoch = LocalDate.ofEpochDay(0); - message.dateField = Math.toIntExact(ChronoUnit.DAYS.between(epoch, localDate)); - - ByteBuf payload = io.netty.buffer.Unpooled - .copiedBuffer(schema.encode(message)); - Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); - - PulsarColumnHandle stringFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "stringField", VARCHAR, false, false, "stringField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, stringFieldColumnHandle, message.stringField); - - PulsarColumnHandle intFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "intField", INTEGER, false, false, "intField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, intFieldColumnHandle, message.intField); - - PulsarColumnHandle floatFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "floatField", REAL, false, false, "floatField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, floatFieldColumnHandle, floatToIntBits(message.floatField)); - - PulsarColumnHandle doubleFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "doubleField", DOUBLE, false, false, "doubleField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, doubleFieldColumnHandle, message.doubleField); - - PulsarColumnHandle booleanFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "booleanField", BOOLEAN, false, false, "booleanField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, booleanFieldColumnHandle, message.booleanField); - - PulsarColumnHandle longFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "longField", BIGINT, false, false, "longField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, longFieldColumnHandle, message.longField); - - PulsarColumnHandle enumFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "enumField", VARCHAR, false, false, "enumField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, enumFieldColumnHandle, message.enumField.toString()); - - PulsarColumnHandle timestampFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "timestampField", TIMESTAMP_MILLIS, false, false, "timestampField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, timestampFieldColumnHandle, message.timestampField * Timestamps.MICROSECONDS_PER_MILLISECOND); - - PulsarColumnHandle timeFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "timeField", TIME_MILLIS, false, false, "timeField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, timeFieldColumnHandle, (long) message.timeField * Timestamps.PICOSECONDS_PER_MILLISECOND); - } - - @Test - public void testDecimal() { - DecoderTestMessage message = new DecoderTestMessage(); - message.decimalField = BigDecimal.valueOf(2233, 2); - message.longDecimalField = new BigDecimal("1234567891234567891234567891.23"); - - ByteBuf payload = io.netty.buffer.Unpooled - .copiedBuffer(schema.encode(message)); - Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); - - PulsarColumnHandle decimalFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "decimalField", DecimalType.createDecimalType(4, 2), false, false, "decimalField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, decimalFieldColumnHandle, message.decimalField); - - PulsarColumnHandle longDecimalFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "longDecimalField", DecimalType.createDecimalType(30, 2), false, false, "longDecimalField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, longDecimalFieldColumnHandle, message.longDecimalField); - } - - @Test - public void testRow() { - DecoderTestMessage message = new DecoderTestMessage(); - message.stringField = "message_2"; - DecoderTestMessage.TestRow testRow = new DecoderTestMessage.TestRow(); - message.rowField = testRow; - testRow.intField = 22; - testRow.stringField = "message_2_testRow"; - DecoderTestMessage.NestedRow nestedRow = new DecoderTestMessage.NestedRow(); - nestedRow.longField = 222L; - nestedRow.stringField = "message_2_nestedRow"; - testRow.nestedRow = nestedRow; - - byte[] bytes = schema.encode(message); - ByteBuf payload = io.netty.buffer.Unpooled - .copiedBuffer(bytes); - GenericAvroRecord genericRecord = (GenericAvroRecord) GenericAvroSchema.of(schemaInfo).decode(bytes); - Object fieldValue = genericRecord.getAvroRecord().get("rowField"); - - Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); - RowType columnType = RowType.from(ImmutableList.builder() - .add(RowType.field("intField", INTEGER)) - .add(RowType.field("nestedRow", RowType.from(ImmutableList.builder() - .add(RowType.field("longField", BIGINT)) - .add(RowType.field("stringField", VARCHAR)) - .build()))) - .add(RowType.field("stringField", VARCHAR)) - .build()); - - PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "rowField", columnType, false, false, "rowField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - - checkRowValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue); - } - - @Test - public void testArray() { - DecoderTestMessage message = new DecoderTestMessage(); - message.arrayField = Arrays.asList("message_1", "message_2", "message_3"); - - byte[] bytes = schema.encode(message); - ByteBuf payload = io.netty.buffer.Unpooled - .copiedBuffer(bytes); - GenericAvroRecord genericRecord = (GenericAvroRecord) GenericAvroSchema.of(schemaInfo).decode(bytes); - Object fieldValue = genericRecord.getAvroRecord().get("arrayField"); - Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); - - ArrayType columnType = new ArrayType(VARCHAR); - PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "arrayField", columnType, false, false, "arrayField", - null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - - checkArrayValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue); - } - - @Test - public void testMap() { - - DecoderTestMessage message = new DecoderTestMessage(); - message.mapField = new HashMap() {{ - put("key1", 2L); - put("key2", 22L); - }}; - - byte[] bytes = schema.encode(message); - ByteBuf payload = io.netty.buffer.Unpooled - .copiedBuffer(bytes); - GenericAvroRecord genericRecord = (GenericAvroRecord) GenericAvroSchema.of(schemaInfo).decode(bytes); - Object fieldValue = genericRecord.getAvroRecord().get("mapField"); - - Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); - Type columnType = decoderFactory.getTypeManager().getParameterizedType(StandardTypes.MAP, - ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), - TypeSignatureParameter.typeParameter(BigintType.BIGINT.getTypeSignature()))); - - PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), "mapField", columnType, false, false, - "mapField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkMapValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue); - } - - @Test - public void testCompositeType() { - DecoderTestMessage message = new DecoderTestMessage(); - - DecoderTestMessage.NestedRow nestedRow = new DecoderTestMessage.NestedRow(); - nestedRow.longField = 222L; - nestedRow.stringField = "message_2_nestedRow"; - - DecoderTestMessage.CompositeRow compositeRow = new DecoderTestMessage.CompositeRow(); - DecoderTestMessage.NestedRow nestedRow1 = new DecoderTestMessage.NestedRow(); - nestedRow1.longField = 2; - nestedRow1.stringField = "nestedRow_1"; - DecoderTestMessage.NestedRow nestedRow2 = new DecoderTestMessage.NestedRow(); - nestedRow2.longField = 2; - nestedRow2.stringField = "nestedRow_2"; - compositeRow.arrayField = Arrays.asList(nestedRow1, nestedRow2); - compositeRow.stringField = "compositeRow_1"; - - compositeRow.mapField = new HashMap() {{ - put("key1", nestedRow1); - put("key2", nestedRow2); - }}; - compositeRow.nestedRow = nestedRow; - - new HashMap() {{ - put("key1_1", 2L); - put("key1_2", 22L); - }}; - compositeRow.structedField = new HashMap>() {{ - put("key2_1", Arrays.asList(2L, 3L)); - put("key2_2", Arrays.asList(2L, 3L)); - put("key2_3", Arrays.asList(2L, 3L)); - }}; - - - message.compositeRow = compositeRow; - byte[] bytes = schema.encode(message); - ByteBuf payload = io.netty.buffer.Unpooled - .copiedBuffer(bytes); - GenericAvroRecord genericRecord = (GenericAvroRecord) GenericAvroSchema.of(schemaInfo).decode(bytes); - Object fieldValue = genericRecord.getAvroRecord().get("compositeRow"); - - Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); - - RowType columnType = RowType.from(ImmutableList.builder() - .add(RowType.field("arrayField", new ArrayType( - RowType.from(ImmutableList.builder() - .add(RowType.field("longField", BIGINT)) - .add(RowType.field("stringField", VARCHAR)) - .build())))) - .add(RowType.field("mapField", decoderFactory.getTypeManager().getParameterizedType(StandardTypes.MAP, - ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), - TypeSignatureParameter.typeParameter(RowType.from(ImmutableList.builder() - .add(RowType.field("longField", BIGINT)) - .add(RowType.field("stringField", VARCHAR)) - .build()).getTypeSignature()) - )))) - .add(RowType.field("nestedRow", RowType.from(ImmutableList.builder() - .add(RowType.field("longField", BIGINT)) - .add(RowType.field("stringField", VARCHAR)) - .build()))) - .add(RowType.field("stringField", VARCHAR)) - .add(RowType.field("structedField", - decoderFactory.getTypeManager().getParameterizedType(StandardTypes.MAP, - ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), - TypeSignatureParameter.typeParameter(new ArrayType(BIGINT).getTypeSignature()))))) - .build()); - - PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "compositeRow", columnType, false, false, "compositeRow", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - - checkRowValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue); - } - - @Test(singleThreaded = true) - public void testCyclicDefinitionDetect() { - AvroSchema cyclicSchema = AvroSchema.of(DecoderTestMessage.CyclicFoo.class); - TrinoException exception = expectThrows(TrinoException.class, - () -> { - decoderFactory.extractColumnMetadata(topicName, cyclicSchema.getSchemaInfo(), - PulsarColumnHandle.HandleKeyValueType.NONE); - }); - - assertEquals("Topic " - + topicName.toString() + " schema may contains cyclic definitions.", exception.getMessage()); - - } - -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/JsonDecoderTestUtil.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/JsonDecoderTestUtil.java deleted file mode 100644 index 301a2eb882f40..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/JsonDecoderTestUtil.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.decoder.json; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.Iterators; -import io.trino.spi.block.Block; -import io.trino.spi.type.*; -import org.apache.pulsar.sql.presto.decoder.DecoderTestUtil; - -import java.io.IOException; -import java.util.Iterator; -import java.util.Map; - -import static io.trino.spi.type.VarcharType.VARCHAR; -import static java.lang.String.format; -import static org.testng.Assert.*; - -/** - * - * TestUtil for JsonDecoder - */ -public class JsonDecoderTestUtil extends DecoderTestUtil { - - public JsonDecoderTestUtil() { - super(); - } - - @Override - public void checkPrimitiveValue(Object actual, Object expected) { - assertTrue(expected instanceof JsonNode); - if (actual == null || null == expected) { - assertNull(expected); - assertNull(actual); - } else if (actual instanceof CharSequence) { - assertEquals(actual.toString(), ((JsonNode) expected).asText()); - } else if (actual instanceof SqlVarbinary) { - try { - assertEquals(((SqlVarbinary) actual).getBytes(), ((JsonNode) expected).binaryValue()); - } catch (IOException e) { - fail(format("JsonNode %s formate binary Value failed", ((JsonNode) expected).getNodeType().name())); - } - } else if (isIntegralType(actual)) { - assertEquals(((Number) actual).longValue(), ((JsonNode) expected).asLong()); - } else if (isRealType(actual)) { - assertEquals(((Number) actual).doubleValue(), ((JsonNode) expected).asDouble()); - } else { - assertEquals(actual, expected); - } - } - - @Override - public void checkMapValues(Block block, Type type, Object value) { - assertNotNull(type, "Type is null"); - assertTrue(type instanceof MapType, "Unexpected type"); - assertTrue(((MapType) type).getKeyType() instanceof VarcharType, "Unexpected key type"); - assertNotNull(block, "Block is null"); - assertNotNull(value, "Value is null"); - - assertTrue(value instanceof ObjectNode, "map node isn't ObjectNode type"); - - ObjectNode expected = (ObjectNode) value; - - Iterator> fields = expected.fields(); - - assertEquals(block.getPositionCount(), Iterators.size(expected.fields()) * 2); - Type valueType = ((MapType) type).getValueType(); - if (valueType instanceof ArrayType) { - for (int index = 0; index < block.getPositionCount(); index += 2) { - String actualKey = VARCHAR.getSlice(block, index).toStringUtf8(); - assertTrue(Iterators.any(fields, entry -> entry.getKey().equals(actualKey))); - if (block.isNull(index + 1)) { - assertNull(expected.get(actualKey)); - continue; - } - Block arrayBlock = block.getObject(index + 1, Block.class); - checkArrayValues(arrayBlock, valueType, expected.get(actualKey)); - } - } else if (valueType instanceof MapType) { - for (int index = 0; index < block.getPositionCount(); index += 2) { - String actualKey = VARCHAR.getSlice(block, index).toStringUtf8(); - assertTrue(Iterators.any(fields, entry -> entry.getKey().equals(actualKey))); - if (block.isNull(index + 1)) { - assertNull(expected.get(actualKey)); - continue; - } - Block mapBlock = block.getObject(index + 1, Block.class); - checkMapValues(mapBlock, valueType, expected.get(actualKey)); - } - } else if (valueType instanceof RowType) { - for (int index = 0; index < block.getPositionCount(); index += 2) { - String actualKey = VARCHAR.getSlice(block, index).toStringUtf8(); - assertTrue(Iterators.any(fields, entry -> entry.getKey().equals(actualKey))); - - if (block.isNull(index + 1)) { - assertNull(expected.get(actualKey)); - continue; - } - Block rowBlock = block.getObject(index + 1, Block.class); - checkRowValues(rowBlock, valueType, expected.get(actualKey)); - } - } else { - for (int index = 0; index < block.getPositionCount(); index += 2) { - String actualKey = VARCHAR.getSlice(block, index).toStringUtf8(); - Map.Entry entry = Iterators.tryFind(fields, e -> e.getKey().equals(actualKey)).get(); - assertNotNull(entry); - assertNotNull(entry.getKey()); - checkPrimitiveValue(getObjectValue(valueType, block, index + 1), entry.getValue()); - } - } - } - - @Override - public void checkRowValues(Block block, Type type, Object value) { - assertNotNull(type, "Type is null"); - assertTrue(type instanceof RowType, "Unexpected type"); - assertNotNull(block, "Block is null"); - assertNotNull(value, "Value is null"); - - ObjectNode record = (ObjectNode) value; - RowType rowType = (RowType) type; - assertEquals(Iterators.size(record.fields()), rowType.getFields().size(), "Json field size mismatch"); - assertEquals(block.getPositionCount(), rowType.getFields().size(), "Presto type field size mismatch"); - for (int fieldIndex = 0; fieldIndex < rowType.getFields().size(); fieldIndex++) { - RowType.Field rowField = rowType.getFields().get(fieldIndex); - Object expectedValue = record.get(rowField.getName().get()); - if (block.isNull(fieldIndex)) { - assertNull(expectedValue); - continue; - } - checkField(block, rowField.getType(), fieldIndex, expectedValue); - } - } - - @Override - public void checkArrayValues(Block block, Type type, Object value) { - assertNotNull(type, "Type is null"); - assertTrue(type instanceof ArrayType, "Unexpected type"); - assertNotNull(block, "Block is null"); - assertNotNull(value, "Value is null"); - - assertTrue(value instanceof ArrayNode, "Array node isn't ArrayNode type"); - ArrayNode arrayNode = (ArrayNode) value; - - assertEquals(block.getPositionCount(), arrayNode.size()); - Type elementType = ((ArrayType) type).getElementType(); - if (elementType instanceof ArrayType) { - for (int index = 0; index < block.getPositionCount(); index++) { - if (block.isNull(index)) { - assertNull(arrayNode.get(index)); - continue; - } - Block arrayBlock = block.getObject(index, Block.class); - checkArrayValues(arrayBlock, elementType, arrayNode.get(index)); - } - } else if (elementType instanceof MapType) { - for (int index = 0; index < block.getPositionCount(); index++) { - if (block.isNull(index)) { - assertNull(arrayNode.get(index)); - continue; - } - Block mapBlock = block.getObject(index, Block.class); - checkMapValues(mapBlock, elementType, arrayNode.get(index)); - } - } else if (elementType instanceof RowType) { - for (int index = 0; index < block.getPositionCount(); index++) { - if (block.isNull(index)) { - assertNull(arrayNode.get(index)); - continue; - } - Block rowBlock = block.getObject(index, Block.class); - checkRowValues(rowBlock, elementType, arrayNode.get(index)); - } - } else { - for (int index = 0; index < block.getPositionCount(); index++) { - checkPrimitiveValue(getObjectValue(elementType, block, index), arrayNode.get(index)); - } - } - } - -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/TestJsonDecoder.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/TestJsonDecoder.java deleted file mode 100644 index 4afad9b318fc5..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/TestJsonDecoder.java +++ /dev/null @@ -1,323 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.decoder.json; - -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.RealType.REAL; -import static io.trino.spi.type.TimeType.TIME_MILLIS; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static java.lang.Float.floatToIntBits; -import static org.apache.pulsar.sql.presto.TestPulsarConnector.getPulsarConnectorId; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.expectThrows; -import com.google.common.collect.ImmutableList; -import io.netty.buffer.ByteBuf; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.decoder.FieldValueProvider; -import io.trino.spi.TrinoException; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.BigintType; -import io.trino.spi.type.DecimalType; -import io.trino.spi.type.RowType; -import io.trino.spi.type.StandardTypes; -import io.trino.spi.type.Timestamps; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeSignatureParameter; -import io.trino.spi.type.VarcharType; -import java.math.BigDecimal; -import java.time.LocalDate; -import java.time.LocalTime; -import java.time.ZoneId; -import java.time.temporal.ChronoUnit; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import org.apache.pulsar.client.impl.schema.JSONSchema; -import org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord; -import org.apache.pulsar.client.impl.schema.generic.GenericJsonSchema; -import org.apache.pulsar.sql.presto.PulsarColumnHandle; -import org.apache.pulsar.sql.presto.decoder.AbstractDecoderTester; -import org.apache.pulsar.sql.presto.decoder.DecoderTestMessage; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -public class TestJsonDecoder extends AbstractDecoderTester { - - private JSONSchema schema; - - @BeforeMethod - public void init() { - super.init(); - schema = JSONSchema.of(DecoderTestMessage.class); - schemaInfo = schema.getSchemaInfo(); - pulsarColumnHandle = getColumnColumnHandles(topicName, schemaInfo, PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); - pulsarRowDecoder = decoderFactory.createRowDecoder(topicName, schemaInfo, new HashSet<>(pulsarColumnHandle)); - decoderTestUtil = new JsonDecoderTestUtil(); - assertTrue(pulsarRowDecoder instanceof PulsarJsonRowDecoder); - } - - @Test - public void testPrimitiveType() { - DecoderTestMessage message = new DecoderTestMessage(); - message.stringField = "message_1"; - message.intField = 22; - message.floatField = 2.2f; - message.doubleField = 22.20D; - message.booleanField = true; - message.longField = 222L; - message.timestampField = System.currentTimeMillis(); - message.enumField = DecoderTestMessage.TestEnum.TEST_ENUM_2; - - LocalTime now = LocalTime.now(ZoneId.systemDefault()); - message.timeField = now.toSecondOfDay() * 1000; - - LocalDate localDate = LocalDate.now(); - LocalDate epoch = LocalDate.ofEpochDay(0); - message.dateField = Math.toIntExact(ChronoUnit.DAYS.between(epoch, localDate)); - - ByteBuf payload = io.netty.buffer.Unpooled - .copiedBuffer(schema.encode(message)); - Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); - - PulsarColumnHandle stringFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "stringField", VARCHAR, false, false, "stringField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, stringFieldColumnHandle, message.stringField); - - PulsarColumnHandle intFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "intField", INTEGER, false, false, "intField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, intFieldColumnHandle, message.intField); - - PulsarColumnHandle floatFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "floatField", REAL, false, false, "floatField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, floatFieldColumnHandle, floatToIntBits(message.floatField)); - - PulsarColumnHandle doubleFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "doubleField", DOUBLE, false, false, "doubleField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, doubleFieldColumnHandle, message.doubleField); - - PulsarColumnHandle booleanFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "booleanField", BOOLEAN, false, false, "booleanField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, booleanFieldColumnHandle, message.booleanField); - - PulsarColumnHandle longFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "longField", BIGINT, false, false, "longField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, longFieldColumnHandle, message.longField); - - PulsarColumnHandle enumFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "enumField", VARCHAR, false, false, "enumField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, enumFieldColumnHandle, message.enumField.toString()); - - PulsarColumnHandle timestampFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "timestampField", TIMESTAMP_MILLIS, false, false, "timestampField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, timestampFieldColumnHandle, message.timestampField * Timestamps.MICROSECONDS_PER_MILLISECOND); - - PulsarColumnHandle timeFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "timeField", TIME_MILLIS, false, false, "timeField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, timeFieldColumnHandle, (long) message.timeField * Timestamps.PICOSECONDS_PER_MILLISECOND); - } - - @Test - public void testDecimal() { - DecoderTestMessage message = new DecoderTestMessage(); - message.decimalField = BigDecimal.valueOf(2233, 2); - message.longDecimalField = new BigDecimal("1234567891234567891234567891.23"); - - ByteBuf payload = io.netty.buffer.Unpooled - .copiedBuffer(schema.encode(message)); - Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); - - PulsarColumnHandle decimalFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "decimalField", DecimalType.createDecimalType(4, 2), false, false, "decimalField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, decimalFieldColumnHandle, message.decimalField); - - PulsarColumnHandle longDecimalFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "longDecimalField", DecimalType.createDecimalType(30, 2), false, false, "longDecimalField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, longDecimalFieldColumnHandle, message.longDecimalField); - } - - @Test - public void testArray() { - DecoderTestMessage message = new DecoderTestMessage(); - message.arrayField = Arrays.asList("message_1", "message_2", "message_3"); - - byte[] bytes = schema.encode(message); - ByteBuf payload = io.netty.buffer.Unpooled - .copiedBuffer(bytes); - - GenericJsonRecord genericRecord = (GenericJsonRecord) GenericJsonSchema.of(schemaInfo).decode(bytes); - Object fieldValue = genericRecord.getJsonNode().get("arrayField"); - Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); - - ArrayType columnType = new ArrayType(VARCHAR); - PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), "arrayField", columnType, false, false, "arrayField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - - checkArrayValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue); - } - - @Test - public void testRow() { - DecoderTestMessage message = new DecoderTestMessage(); - message.stringField = "message_2"; - DecoderTestMessage.TestRow testRow = new DecoderTestMessage.TestRow(); - message.rowField = testRow; - testRow.intField = 22; - testRow.stringField = "message_2_testRow"; - DecoderTestMessage.NestedRow nestedRow = new DecoderTestMessage.NestedRow(); - nestedRow.longField = 222L; - nestedRow.stringField = "message_2_nestedRow"; - testRow.nestedRow = nestedRow; - - byte[] bytes = schema.encode(message); - ByteBuf payload = io.netty.buffer.Unpooled - .copiedBuffer(bytes); - - GenericJsonRecord genericRecord = (GenericJsonRecord) GenericJsonSchema.of(schemaInfo).decode(bytes); - Object fieldValue = genericRecord.getJsonNode().get("rowField"); - Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); - - RowType columnType = RowType.from(ImmutableList.builder() - .add(RowType.field("intField", INTEGER)) - .add(RowType.field("nestedRow", RowType.from(ImmutableList.builder() - .add(RowType.field("longField", BIGINT)) - .add(RowType.field("stringField", VARCHAR)) - .build()))) - .add(RowType.field("stringField", VARCHAR)) - .build()); - - PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), "rowField", columnType, false, false, "rowField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - - checkRowValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue); - - } - - @Test - public void testMap() { - DecoderTestMessage message = new DecoderTestMessage(); - message.mapField = new HashMap() {{ - put("key1", 2L); - put("key2", 22L); - }}; - - byte[] bytes = schema.encode(message); - ByteBuf payload = io.netty.buffer.Unpooled - .copiedBuffer(bytes); - - GenericJsonRecord genericRecord = (GenericJsonRecord) GenericJsonSchema.of(schemaInfo).decode(bytes); - Object fieldValue = genericRecord.getJsonNode().get("mapField"); - Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); - - Type columnType = decoderFactory.getTypeManager().getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), TypeSignatureParameter.typeParameter(BigintType.BIGINT.getTypeSignature()))); - PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), "mapField", columnType, false, false, "mapField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - - checkMapValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue); - - } - - @Test - public void testCompositeType() { - DecoderTestMessage message = new DecoderTestMessage(); - - DecoderTestMessage.NestedRow nestedRow = new DecoderTestMessage.NestedRow(); - nestedRow.longField = 222L; - nestedRow.stringField = "message_2_nestedRow"; - - DecoderTestMessage.CompositeRow compositeRow = new DecoderTestMessage.CompositeRow(); - DecoderTestMessage.NestedRow nestedRow1 = new DecoderTestMessage.NestedRow(); - nestedRow1.longField = 2; - nestedRow1.stringField = "nestedRow_1"; - DecoderTestMessage.NestedRow nestedRow2 = new DecoderTestMessage.NestedRow(); - nestedRow2.longField = 2; - nestedRow2.stringField = "nestedRow_2"; - compositeRow.arrayField = Arrays.asList(nestedRow1, nestedRow2); - compositeRow.stringField = "compositeRow_1"; - - compositeRow.mapField = new HashMap() {{ - put("key1", nestedRow1); - put("key2", nestedRow2); - }}; - compositeRow.nestedRow = nestedRow; - new HashMap() {{ - put("key1_1", 2L); - put("key1_2", 22L); - }}; - compositeRow.structedField = new HashMap>() {{ - put("key2_1", Arrays.asList(2L, 3L)); - put("key2_2", Arrays.asList(2L, 3L)); - put("key2_3", Arrays.asList(2L, 3L)); - }}; - message.compositeRow = compositeRow; - - byte[] bytes = schema.encode(message); - ByteBuf payload = io.netty.buffer.Unpooled - .copiedBuffer(bytes); - GenericJsonRecord genericRecord = (GenericJsonRecord) GenericJsonSchema.of(schemaInfo).decode(bytes); - Object fieldValue = genericRecord.getJsonNode().get("compositeRow"); - Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); - - RowType columnType = RowType.from(ImmutableList.builder() - .add(RowType.field("arrayField", new ArrayType( - RowType.from(ImmutableList.builder() - .add(RowType.field("longField", BIGINT)) - .add(RowType.field("stringField", VARCHAR)) - .build())))) - .add(RowType.field("mapField", decoderFactory.getTypeManager().getParameterizedType(StandardTypes.MAP, - ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), - TypeSignatureParameter.typeParameter(RowType.from(ImmutableList.builder() - .add(RowType.field("longField", BIGINT)) - .add(RowType.field("stringField", VARCHAR)) - .build()).getTypeSignature()) - )))) - .add(RowType.field("nestedRow", RowType.from(ImmutableList.builder() - .add(RowType.field("longField", BIGINT)) - .add(RowType.field("stringField", VARCHAR)) - .build()))) - .add(RowType.field("stringField", VARCHAR)) - .add(RowType.field("structedField", - decoderFactory.getTypeManager().getParameterizedType(StandardTypes.MAP, - ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), - TypeSignatureParameter.typeParameter(new ArrayType(BIGINT).getTypeSignature()))))) - .build()); - - PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), "compositeRow", columnType, false, false, "compositeRow", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - - checkRowValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue); - } - - @Test(singleThreaded = true) - public void testCyclicDefinitionDetect() { - JSONSchema cyclicSchema = JSONSchema.of(DecoderTestMessage.CyclicFoo.class); - TrinoException exception = expectThrows(TrinoException.class, - () -> { - decoderFactory.extractColumnMetadata(topicName, cyclicSchema.getSchemaInfo(), - PulsarColumnHandle.HandleKeyValueType.NONE); - }); - - assertEquals("Topic " - + topicName.toString() + " schema may contains cyclic definitions.", exception.getMessage()); - - } - -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/primitive/PrimitiveDecoderTestUtil.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/primitive/PrimitiveDecoderTestUtil.java deleted file mode 100644 index 0824b84ec3a19..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/primitive/PrimitiveDecoderTestUtil.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.decoder.primitive; - -import io.trino.spi.block.Block; -import io.trino.spi.type.Type; -import org.apache.pulsar.sql.presto.decoder.DecoderTestUtil; - -/** - * TestUtil for PrimitiveDecoder. - * CheckXXXValues() is mock method. Because Primitive is single hierarchy, so CheckXXXValues are never actually - * invoked. - */ -public class PrimitiveDecoderTestUtil extends DecoderTestUtil { - - public PrimitiveDecoderTestUtil() { - super(); - } - - @Override - public void checkArrayValues(Block block, Type type, Object value) { - - } - - @Override - public void checkMapValues(Block block, Type type, Object value) { - - } - - @Override - public void checkRowValues(Block block, Type type, Object value) { - - } - - @Override - public void checkPrimitiveValue(Object actual, Object expected) { - - } - -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/primitive/TestPrimitiveDecoder.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/primitive/TestPrimitiveDecoder.java deleted file mode 100644 index 7c18877a9c608..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/primitive/TestPrimitiveDecoder.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.decoder.primitive; - -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.RealType.REAL; -import static io.trino.spi.type.SmallintType.SMALLINT; -import static io.trino.spi.type.TimeType.TIME_MILLIS; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.TinyintType.TINYINT; -import static io.trino.spi.type.VarbinaryType.VARBINARY; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static org.apache.pulsar.sql.presto.TestPulsarConnector.getPulsarConnectorId; -import io.airlift.slice.Slices; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.decoder.FieldValueProvider; -import io.trino.spi.type.Timestamps; -import java.sql.Time; -import java.sql.Timestamp; -import java.time.LocalTime; -import java.time.ZoneId; -import java.util.Date; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.common.schema.SchemaType; -import org.apache.pulsar.sql.presto.PulsarColumnHandle; -import org.apache.pulsar.sql.presto.PulsarRowDecoder; -import org.apache.pulsar.sql.presto.decoder.AbstractDecoderTester; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -public class TestPrimitiveDecoder extends AbstractDecoderTester { - - public static final String PRIMITIVE_COLUMN_NAME = "__value__"; - - @BeforeMethod - public void init() { - decoderTestUtil = new PrimitiveDecoderTestUtil(); - super.init(); - } - - @Test(singleThreaded = true) - public void testPrimitiveType() { - byte int8Value = 1; - SchemaInfo schemaInfoInt8 = SchemaInfo.builder().type(SchemaType.INT8).build(); - Schema schemaInt8 = Schema.getSchema(schemaInfoInt8); - List pulsarColumnHandleInt8 = getColumnColumnHandles(topicName, schemaInfoInt8, PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); - PulsarRowDecoder pulsarRowDecoderInt8 = decoderFactory.createRowDecoder(topicName, schemaInfoInt8, - new HashSet<>(pulsarColumnHandleInt8)); - Map decodedRowInt8 = - pulsarRowDecoderInt8.decodeRow(io.netty.buffer.Unpooled - .copiedBuffer(schemaInt8.encode(int8Value))).get(); - checkValue(decodedRowInt8, new PulsarColumnHandle(getPulsarConnectorId().toString(), - PRIMITIVE_COLUMN_NAME, TINYINT, false, false, PRIMITIVE_COLUMN_NAME, null, null, PulsarColumnHandle.HandleKeyValueType.NONE), int8Value); - - short int16Value = 2; - SchemaInfo schemaInfoInt16 = SchemaInfo.builder().type(SchemaType.INT16).build(); - Schema schemaInt16 = Schema.getSchema(schemaInfoInt16); - List pulsarColumnHandleInt16 = getColumnColumnHandles(topicName, schemaInfoInt16, PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); - PulsarRowDecoder pulsarRowDecoderInt16 = decoderFactory.createRowDecoder(topicName, schemaInfoInt16, - new HashSet<>(pulsarColumnHandleInt16)); - Map decodedRowInt16 = - pulsarRowDecoderInt16.decodeRow(io.netty.buffer.Unpooled - .copiedBuffer(schemaInt16.encode(int16Value))).get(); - checkValue(decodedRowInt16, new PulsarColumnHandle(getPulsarConnectorId().toString(), - PRIMITIVE_COLUMN_NAME, SMALLINT, false, false, PRIMITIVE_COLUMN_NAME, null, null, PulsarColumnHandle.HandleKeyValueType.NONE), int16Value); - - int int32Value = 2; - SchemaInfo schemaInfoInt32 = SchemaInfo.builder().type(SchemaType.INT32).build(); - Schema schemaInt32 = Schema.getSchema(schemaInfoInt32); - List pulsarColumnHandleInt32 = getColumnColumnHandles(topicName, schemaInfoInt32, - PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); - PulsarRowDecoder pulsarRowDecoderInt32 = decoderFactory.createRowDecoder(topicName, schemaInfoInt32, - new HashSet<>(pulsarColumnHandleInt32)); - Map decodedRowInt32 = - pulsarRowDecoderInt32.decodeRow(io.netty.buffer.Unpooled - .copiedBuffer(schemaInt32.encode(int32Value))).get(); - checkValue(decodedRowInt32, new PulsarColumnHandle(getPulsarConnectorId().toString(), - PRIMITIVE_COLUMN_NAME, INTEGER, false, false, PRIMITIVE_COLUMN_NAME, null, null, PulsarColumnHandle.HandleKeyValueType.NONE), int32Value); - - long int64Value = 2; - SchemaInfo schemaInfoInt64 = SchemaInfo.builder().type(SchemaType.INT64).build(); - Schema schemaInt64 = Schema.getSchema(schemaInfoInt64); - List pulsarColumnHandleInt64 = getColumnColumnHandles(topicName, schemaInfoInt64, - PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); - PulsarRowDecoder pulsarRowDecoderInt64 = decoderFactory.createRowDecoder(topicName, schemaInfoInt64, - new HashSet<>(pulsarColumnHandleInt64)); - Map decodedRowInt64 = - pulsarRowDecoderInt64.decodeRow(io.netty.buffer.Unpooled - .copiedBuffer(schemaInt64.encode(int64Value))).get(); - checkValue(decodedRowInt64, new PulsarColumnHandle(getPulsarConnectorId().toString(), - PRIMITIVE_COLUMN_NAME, BIGINT, false, false, PRIMITIVE_COLUMN_NAME, null, null, - PulsarColumnHandle.HandleKeyValueType.NONE), int64Value); - - String stringValue = "test"; - SchemaInfo schemaInfoString = SchemaInfo.builder().type(SchemaType.STRING).build(); - Schema schemaString = Schema.getSchema(schemaInfoString); - List pulsarColumnHandleString = getColumnColumnHandles(topicName, schemaInfoString, - PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); - PulsarRowDecoder pulsarRowDecoderString = decoderFactory.createRowDecoder(topicName, schemaInfoString, - new HashSet<>(pulsarColumnHandleString)); - Map decodedRowString = - pulsarRowDecoderString.decodeRow(io.netty.buffer.Unpooled - .copiedBuffer(schemaString.encode(stringValue))).get(); - checkValue(decodedRowString, new PulsarColumnHandle(getPulsarConnectorId().toString(), - PRIMITIVE_COLUMN_NAME, VARCHAR, false, false, PRIMITIVE_COLUMN_NAME, null, null, - PulsarColumnHandle.HandleKeyValueType.NONE), stringValue); - - float floatValue = 0.2f; - SchemaInfo schemaInfoFloat = SchemaInfo.builder().type(SchemaType.FLOAT).build(); - Schema schemaFloat = Schema.getSchema(schemaInfoFloat); - List pulsarColumnHandleFloat = getColumnColumnHandles(topicName, schemaInfoFloat, - PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); - PulsarRowDecoder pulsarRowDecoderFloat = decoderFactory.createRowDecoder(topicName, schemaInfoFloat, - new HashSet<>(pulsarColumnHandleFloat)); - Map decodedRowFloat = - pulsarRowDecoderFloat.decodeRow(io.netty.buffer.Unpooled - .copiedBuffer(schemaFloat.encode(floatValue))).get(); - checkValue(decodedRowFloat, new PulsarColumnHandle(getPulsarConnectorId().toString(), - PRIMITIVE_COLUMN_NAME, REAL, false, false, PRIMITIVE_COLUMN_NAME, null, null, - PulsarColumnHandle.HandleKeyValueType.NONE), Float.floatToIntBits(floatValue)); - - double doubleValue = 0.22d; - SchemaInfo schemaInfoDouble = SchemaInfo.builder().type(SchemaType.DOUBLE).build(); - Schema schemaDouble = Schema.getSchema(schemaInfoDouble); - List pulsarColumnHandleDouble = getColumnColumnHandles(topicName, schemaInfoDouble, - PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); - PulsarRowDecoder pulsarRowDecoderDouble = decoderFactory.createRowDecoder(topicName, schemaInfoDouble, - new HashSet<>(pulsarColumnHandleDouble)); - Map decodedRowDouble = - pulsarRowDecoderDouble.decodeRow(io.netty.buffer.Unpooled - .copiedBuffer(schemaDouble.encode(doubleValue))).get(); - checkValue(decodedRowDouble, new PulsarColumnHandle(getPulsarConnectorId().toString(), - PRIMITIVE_COLUMN_NAME, DOUBLE, false, false, PRIMITIVE_COLUMN_NAME, null, null, - PulsarColumnHandle.HandleKeyValueType.NONE), doubleValue); - - boolean booleanValue = true; - SchemaInfo schemaInfoBoolean = SchemaInfo.builder().type(SchemaType.BOOLEAN).build(); - Schema schemaBoolean = Schema.getSchema(schemaInfoBoolean); - List pulsarColumnHandleBoolean = getColumnColumnHandles(topicName, schemaInfoBoolean, - PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); - PulsarRowDecoder pulsarRowDecoderBoolean = decoderFactory.createRowDecoder(topicName, schemaInfoBoolean, - new HashSet<>(pulsarColumnHandleBoolean)); - Map decodedRowBoolean = - pulsarRowDecoderBoolean.decodeRow(io.netty.buffer.Unpooled - .copiedBuffer(schemaBoolean.encode(booleanValue))).get(); - checkValue(decodedRowBoolean, new PulsarColumnHandle(getPulsarConnectorId().toString(), - PRIMITIVE_COLUMN_NAME, BOOLEAN, false, false, PRIMITIVE_COLUMN_NAME, null, null, - PulsarColumnHandle.HandleKeyValueType.NONE), booleanValue); - - byte[] bytesValue = new byte[1]; - bytesValue[0] = 1; - SchemaInfo schemaInfoBytes = SchemaInfo.builder().type(SchemaType.BYTES).build(); - Schema schemaBytes = Schema.getSchema(schemaInfoBytes); - List pulsarColumnHandleBytes = getColumnColumnHandles(topicName, schemaInfoBytes, - PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); - PulsarRowDecoder pulsarRowDecoderBytes = decoderFactory.createRowDecoder(topicName, schemaInfoBytes, - new HashSet<>(pulsarColumnHandleBytes)); - Map decodedRowBytes = - pulsarRowDecoderBytes.decodeRow(io.netty.buffer.Unpooled - .copiedBuffer(schemaBytes.encode(bytesValue))).get(); - checkValue(decodedRowBytes, new PulsarColumnHandle(getPulsarConnectorId().toString(), - PRIMITIVE_COLUMN_NAME, VARBINARY, false, false, PRIMITIVE_COLUMN_NAME, null, null, - PulsarColumnHandle.HandleKeyValueType.NONE), Slices.wrappedBuffer(bytesValue)); - - Date dateValue = new Date(System.currentTimeMillis()); - SchemaInfo schemaInfoDate = SchemaInfo.builder().type(SchemaType.DATE).build(); - Schema schemaDate = Schema.getSchema(schemaInfoDate); - List pulsarColumnHandleDate = getColumnColumnHandles(topicName, schemaInfoDate, - PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); - PulsarRowDecoder pulsarRowDecoderDate = decoderFactory.createRowDecoder(topicName, schemaInfoDate, - new HashSet<>(pulsarColumnHandleDate)); - Map decodedRowDate = - pulsarRowDecoderDate.decodeRow(io.netty.buffer.Unpooled - .copiedBuffer(schemaDate.encode(dateValue))).get(); - checkValue(decodedRowDate, new PulsarColumnHandle(getPulsarConnectorId().toString(), - PRIMITIVE_COLUMN_NAME, DATE, false, false, PRIMITIVE_COLUMN_NAME, null, null, - PulsarColumnHandle.HandleKeyValueType.NONE), dateValue.getTime()); - - LocalTime now = LocalTime.now(ZoneId.systemDefault()); - Time timeValue = Time.valueOf(now); - SchemaInfo schemaInfoTime = SchemaInfo.builder().type(SchemaType.TIME).build(); - Schema schemaTime = Schema.getSchema(schemaInfoTime); - List pulsarColumnHandleTime = getColumnColumnHandles(topicName, schemaInfoTime, - PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); - PulsarRowDecoder pulsarRowDecoderTime = decoderFactory.createRowDecoder(topicName, schemaInfoTime, - new HashSet<>(pulsarColumnHandleTime)); - Map decodedRowTime = - pulsarRowDecoderTime.decodeRow(io.netty.buffer.Unpooled - .copiedBuffer(schemaTime.encode(timeValue))).get(); - checkValue(decodedRowTime, new PulsarColumnHandle(getPulsarConnectorId().toString(), - PRIMITIVE_COLUMN_NAME, TIME_MILLIS, false, false, PRIMITIVE_COLUMN_NAME, null, null, - PulsarColumnHandle.HandleKeyValueType.NONE), timeValue.getTime() * Timestamps.PICOSECONDS_PER_MILLISECOND); - - Timestamp timestampValue = new Timestamp(System.currentTimeMillis()); - SchemaInfo schemaInfoTimestamp = SchemaInfo.builder().type(SchemaType.TIMESTAMP).build(); - Schema schemaTimestamp = Schema.getSchema(schemaInfoTimestamp); - List pulsarColumnHandleTimestamp = getColumnColumnHandles(topicName, schemaInfoTimestamp, - PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); - PulsarRowDecoder pulsarRowDecoderTimestamp = decoderFactory.createRowDecoder(topicName, schemaInfoTimestamp, - new HashSet<>(pulsarColumnHandleTimestamp)); - Map decodedRowTimestamp = - pulsarRowDecoderTimestamp.decodeRow(io.netty.buffer.Unpooled - .copiedBuffer(schemaTimestamp.encode(timestampValue))).get(); - checkValue(decodedRowTimestamp, new PulsarColumnHandle(getPulsarConnectorId().toString(), - PRIMITIVE_COLUMN_NAME, TIMESTAMP_MILLIS, false, false, PRIMITIVE_COLUMN_NAME, null, null, - PulsarColumnHandle.HandleKeyValueType.NONE), timestampValue.getTime() * Timestamps.MICROSECONDS_PER_MILLISECOND); - - } - -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/ProtobufNativeDecoderTestUtil.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/ProtobufNativeDecoderTestUtil.java deleted file mode 100644 index d908d30bc4728..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/ProtobufNativeDecoderTestUtil.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.decoder.protobufnative; - -import com.google.protobuf.ByteString; -import com.google.protobuf.DynamicMessage; -import com.google.protobuf.EnumValue; -import io.trino.spi.block.Block; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.MapType; -import io.trino.spi.type.RowType; -import io.trino.spi.type.SqlVarbinary; -import io.trino.spi.type.Type; -import org.apache.pulsar.sql.presto.decoder.DecoderTestUtil; - -import java.util.List; -import java.util.Map; - -import static java.lang.String.format; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertNull; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; - -/** - * TestUtil for ProtobufNativeDecoder. - */ -public class ProtobufNativeDecoderTestUtil extends DecoderTestUtil { - - public ProtobufNativeDecoderTestUtil() { - super(); - } - - public void checkPrimitiveValue(Object actual, Object expected) { - if (actual == null || expected == null) { - assertNull(expected); - assertNull(actual); - } else if (actual instanceof CharSequence) { - assertTrue(expected instanceof CharSequence || expected instanceof EnumValue); - assertEquals(actual.toString(), expected.toString()); - if (expected instanceof EnumValue) { - assertEquals(((CharSequence) actual.toString()), ((EnumValue) expected).getName()); - } else if (expected instanceof CharSequence) { - assertEquals(actual.toString(), expected.toString()); - } - - } else if (actual instanceof SqlVarbinary) { - if (actual instanceof ByteString) { - assertEquals(((SqlVarbinary) actual).getBytes(), ((ByteString) expected).toByteArray()); - } else if (expected instanceof byte[]) { - assertEquals(((SqlVarbinary) actual).getBytes(), expected); - } else { - fail(format("Unexpected value type %s", actual.getClass())); - } - } else if (isIntegralType(actual) && isIntegralType(expected)) { - assertEquals(((Number) actual).longValue(), ((Number) expected).longValue()); - } else if (isRealType(actual) && isRealType(expected)) { - assertEquals(((Number) actual).doubleValue(), ((Number) expected).doubleValue()); - } else { - assertEquals(actual, expected); - } - } - - public void checkArrayValues(Block block, Type type, Object value) { - assertNotNull(type, "Type is null"); - assertTrue(type instanceof ArrayType, "Unexpected type"); - assertNotNull(block, "Block is null"); - assertNotNull(value, "Value is null"); - - List list = (List) value; - - assertEquals(block.getPositionCount(), list.size()); - Type elementType = ((ArrayType) type).getElementType(); - if (elementType instanceof ArrayType) { - for (int index = 0; index < block.getPositionCount(); index++) { - if (block.isNull(index)) { - assertNull(list.get(index)); - continue; - } - Block arrayBlock = block.getObject(index, Block.class); - checkArrayValues(arrayBlock, elementType, list.get(index)); - } - } else if (elementType instanceof MapType) { - for (int index = 0; index < block.getPositionCount(); index++) { - if (block.isNull(index)) { - assertNull(list.get(index)); - continue; - } - Block mapBlock = block.getObject(index, Block.class); - checkMapValues(mapBlock, elementType, list.get(index)); - } - } else if (elementType instanceof RowType) { - for (int index = 0; index < block.getPositionCount(); index++) { - if (block.isNull(index)) { - assertNull(list.get(index)); - continue; - } - Block rowBlock = block.getObject(index, Block.class); - checkRowValues(rowBlock, elementType, list.get(index)); - } - } else { - for (int index = 0; index < block.getPositionCount(); index++) { - checkPrimitiveValue(getObjectValue(elementType, block, index), list.get(index)); - } - } - } - - public void checkMapValues(Block block, Type type, Object value) { - assertNotNull(type, "Type is null"); - assertTrue(type instanceof MapType, "Unexpected type"); - assertNotNull(block, "Block is null"); - assertNotNull(value, "Value is null"); - - Map expected = PulsarProtobufNativeColumnDecoder.parseProtobufMap(value); - - assertEquals(block.getPositionCount(), expected.size() * 2); - Type valueType = ((MapType) type).getValueType(); - //protobuf3 keyType only support integral or string type - Type keyType = ((MapType) type).getKeyType(); - - //check value - if (valueType instanceof ArrayType) { - for (int index = 0; index < block.getPositionCount(); index += 2) { - Object actualKey = getObjectValue(keyType, block, index); - assertTrue(expected.keySet().stream().anyMatch(e -> e.equals(actualKey))); - if (block.isNull(index + 1)) { - assertNull(expected.get(actualKey)); - continue; - } - Block arrayBlock = block.getObject(index + 1, Block.class); - Object keyValue = expected.entrySet().stream().filter(e -> e.getKey().equals(actualKey)).findFirst().get().getValue(); - checkArrayValues(arrayBlock, valueType, keyValue); - } - } else if (valueType instanceof MapType) { - for (int index = 0; index < block.getPositionCount(); index += 2) { - Object actualKey = getObjectValue(keyType, block, index); - assertTrue(expected.keySet().stream().anyMatch(e -> e.equals(actualKey))); - if (block.isNull(index + 1)) { - assertNull(expected.get(actualKey)); - continue; - } - Block mapBlock = block.getObject(index + 1, Block.class); - Object keyValue = expected.entrySet().stream().filter(e -> e.getKey().equals(actualKey)).findFirst().get().getValue(); - checkMapValues(mapBlock, valueType, keyValue); - } - } else if (valueType instanceof RowType) { - for (int index = 0; index < block.getPositionCount(); index += 2) { - Object actualKey = getObjectValue(keyType, block, index); - assertTrue(expected.keySet().stream().anyMatch(e -> e.equals(actualKey))); - if (block.isNull(index + 1)) { - assertNull(expected.get(actualKey)); - continue; - } - Block rowBlock = block.getObject(index + 1, Block.class); - Object keyValue = expected.entrySet().stream().filter(e -> e.getKey().equals(actualKey)).findFirst().get().getValue(); - checkRowValues(rowBlock, valueType, keyValue); - } - } else { - for (int index = 0; index < block.getPositionCount(); index += 2) { - Object actualKey = getObjectValue(keyType, block, index); - assertTrue(expected.keySet().stream().anyMatch(e -> e.equals(actualKey))); - Object keyValue = expected.entrySet().stream().filter(e -> e.getKey().equals(actualKey)).findFirst().get().getValue(); - checkPrimitiveValue(getObjectValue(valueType, block, index + 1), keyValue); - } - } - } - - - public void checkRowValues(Block block, Type type, Object value) { - assertNotNull(type, "Type is null"); - assertTrue(type instanceof RowType, "Unexpected type"); - assertNotNull(block, "Block is null"); - assertNotNull(value, "Value is null"); - - DynamicMessage record = (DynamicMessage) value; - RowType rowType = (RowType) type; - assertEquals(record.getAllFields().size(), rowType.getFields().size(), "Protobuf field size mismatch"); - assertEquals(block.getPositionCount(), rowType.getFields().size(), "Presto type field size mismatch"); - for (int fieldIndex = 0; fieldIndex < rowType.getFields().size(); fieldIndex++) { - RowType.Field rowField = rowType.getFields().get(fieldIndex); - Object expectedValue = - record.getField(((DynamicMessage) value).getDescriptorForType().findFieldByName(rowField.getName().get())); - - if (block.isNull(fieldIndex)) { - assertNull(expectedValue); - continue; - } - checkField(block, rowField.getType(), fieldIndex, expectedValue); - } - } - -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/TestMsg.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/TestMsg.java deleted file mode 100644 index d1d8a5ca7e598..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/TestMsg.java +++ /dev/null @@ -1,4472 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: TestMsg.proto - -package org.apache.pulsar.sql.presto.decoder.protobufnative; - -public final class TestMsg { - private TestMsg() {} - public static void registerAllExtensions( - com.google.protobuf.ExtensionRegistryLite registry) { - } - - public static void registerAllExtensions( - com.google.protobuf.ExtensionRegistry registry) { - registerAllExtensions( - (com.google.protobuf.ExtensionRegistryLite) registry); - } - /** - * Protobuf enum {@code proto.TestEnum} - */ - public enum TestEnum - implements com.google.protobuf.ProtocolMessageEnum { - /** - * SHARED = 0; - */ - SHARED(0), - /** - * FAILOVER = 1; - */ - FAILOVER(1), - UNRECOGNIZED(-1), - ; - - /** - * SHARED = 0; - */ - public static final int SHARED_VALUE = 0; - /** - * FAILOVER = 1; - */ - public static final int FAILOVER_VALUE = 1; - - - public final int getNumber() { - if (this == UNRECOGNIZED) { - throw new java.lang.IllegalArgumentException( - "Can't get the number of an unknown enum value."); - } - return value; - } - - /** - * @param value The numeric wire value of the corresponding enum entry. - * @return The enum associated with the given numeric wire value. - * @deprecated Use {@link #forNumber(int)} instead. - */ - @java.lang.Deprecated - public static TestEnum valueOf(int value) { - return forNumber(value); - } - - /** - * @param value The numeric wire value of the corresponding enum entry. - * @return The enum associated with the given numeric wire value. - */ - public static TestEnum forNumber(int value) { - switch (value) { - case 0: return SHARED; - case 1: return FAILOVER; - default: return null; - } - } - - public static com.google.protobuf.Internal.EnumLiteMap - internalGetValueMap() { - return internalValueMap; - } - private static final com.google.protobuf.Internal.EnumLiteMap< - TestEnum> internalValueMap = - new com.google.protobuf.Internal.EnumLiteMap() { - public TestEnum findValueByNumber(int number) { - return TestEnum.forNumber(number); - } - }; - - public final com.google.protobuf.Descriptors.EnumValueDescriptor - getValueDescriptor() { - if (this == UNRECOGNIZED) { - throw new java.lang.IllegalStateException( - "Can't get the descriptor of an unrecognized enum value."); - } - return getDescriptor().getValues().get(ordinal()); - } - public final com.google.protobuf.Descriptors.EnumDescriptor - getDescriptorForType() { - return getDescriptor(); - } - public static final com.google.protobuf.Descriptors.EnumDescriptor - getDescriptor() { - return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.getDescriptor().getEnumTypes().get(0); - } - - private static final TestEnum[] VALUES = values(); - - public static TestEnum valueOf( - com.google.protobuf.Descriptors.EnumValueDescriptor desc) { - if (desc.getType() != getDescriptor()) { - throw new java.lang.IllegalArgumentException( - "EnumValueDescriptor is not for this type."); - } - if (desc.getIndex() == -1) { - return UNRECOGNIZED; - } - return VALUES[desc.getIndex()]; - } - - private final int value; - - private TestEnum(int value) { - this.value = value; - } - - // @@protoc_insertion_point(enum_scope:proto.TestEnum) - } - - public interface SubMessageOrBuilder extends - // @@protoc_insertion_point(interface_extends:proto.SubMessage) - com.google.protobuf.MessageOrBuilder { - - /** - * string foo = 1; - * @return The foo. - */ - java.lang.String getFoo(); - /** - * string foo = 1; - * @return The bytes for foo. - */ - com.google.protobuf.ByteString - getFooBytes(); - - /** - * double bar = 2; - * @return The bar. - */ - double getBar(); - - /** - * .proto.SubMessage.NestedMessage nestedMessage = 3; - * @return Whether the nestedMessage field is set. - */ - boolean hasNestedMessage(); - /** - * .proto.SubMessage.NestedMessage nestedMessage = 3; - * @return The nestedMessage. - */ - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage getNestedMessage(); - /** - * .proto.SubMessage.NestedMessage nestedMessage = 3; - */ - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessageOrBuilder getNestedMessageOrBuilder(); - } - /** - * Protobuf type {@code proto.SubMessage} - */ - public static final class SubMessage extends - com.google.protobuf.GeneratedMessageV3 implements - // @@protoc_insertion_point(message_implements:proto.SubMessage) - SubMessageOrBuilder { - private static final long serialVersionUID = 0L; - // Use SubMessage.newBuilder() to construct. - private SubMessage(com.google.protobuf.GeneratedMessageV3.Builder builder) { - super(builder); - } - private SubMessage() { - foo_ = ""; - } - - @java.lang.Override - @SuppressWarnings({"unused"}) - protected java.lang.Object newInstance( - UnusedPrivateParameter unused) { - return new SubMessage(); - } - - @java.lang.Override - public final com.google.protobuf.UnknownFieldSet - getUnknownFields() { - return this.unknownFields; - } - private SubMessage( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - this(); - if (extensionRegistry == null) { - throw new java.lang.NullPointerException(); - } - com.google.protobuf.UnknownFieldSet.Builder unknownFields = - com.google.protobuf.UnknownFieldSet.newBuilder(); - try { - boolean done = false; - while (!done) { - int tag = input.readTag(); - switch (tag) { - case 0: - done = true; - break; - case 10: { - java.lang.String s = input.readStringRequireUtf8(); - - foo_ = s; - break; - } - case 17: { - - bar_ = input.readDouble(); - break; - } - case 26: { - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.Builder subBuilder = null; - if (nestedMessage_ != null) { - subBuilder = nestedMessage_.toBuilder(); - } - nestedMessage_ = input.readMessage(org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.parser(), extensionRegistry); - if (subBuilder != null) { - subBuilder.mergeFrom(nestedMessage_); - nestedMessage_ = subBuilder.buildPartial(); - } - - break; - } - default: { - if (!parseUnknownField( - input, unknownFields, extensionRegistry, tag)) { - done = true; - } - break; - } - } - } - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw e.setUnfinishedMessage(this); - } catch (java.io.IOException e) { - throw new com.google.protobuf.InvalidProtocolBufferException( - e).setUnfinishedMessage(this); - } finally { - this.unknownFields = unknownFields.build(); - makeExtensionsImmutable(); - } - } - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_SubMessage_descriptor; - } - - @java.lang.Override - protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_SubMessage_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.class, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.Builder.class); - } - - public interface NestedMessageOrBuilder extends - // @@protoc_insertion_point(interface_extends:proto.SubMessage.NestedMessage) - com.google.protobuf.MessageOrBuilder { - - /** - * string title = 1; - * @return The title. - */ - java.lang.String getTitle(); - /** - * string title = 1; - * @return The bytes for title. - */ - com.google.protobuf.ByteString - getTitleBytes(); - - /** - * repeated string urls = 2; - * @return A list containing the urls. - */ - java.util.List - getUrlsList(); - /** - * repeated string urls = 2; - * @return The count of urls. - */ - int getUrlsCount(); - /** - * repeated string urls = 2; - * @param index The index of the element to return. - * @return The urls at the given index. - */ - java.lang.String getUrls(int index); - /** - * repeated string urls = 2; - * @param index The index of the value to return. - * @return The bytes of the urls at the given index. - */ - com.google.protobuf.ByteString - getUrlsBytes(int index); - } - /** - * Protobuf type {@code proto.SubMessage.NestedMessage} - */ - public static final class NestedMessage extends - com.google.protobuf.GeneratedMessageV3 implements - // @@protoc_insertion_point(message_implements:proto.SubMessage.NestedMessage) - NestedMessageOrBuilder { - private static final long serialVersionUID = 0L; - // Use NestedMessage.newBuilder() to construct. - private NestedMessage(com.google.protobuf.GeneratedMessageV3.Builder builder) { - super(builder); - } - private NestedMessage() { - title_ = ""; - urls_ = com.google.protobuf.LazyStringArrayList.EMPTY; - } - - @java.lang.Override - @SuppressWarnings({"unused"}) - protected java.lang.Object newInstance( - UnusedPrivateParameter unused) { - return new NestedMessage(); - } - - @java.lang.Override - public final com.google.protobuf.UnknownFieldSet - getUnknownFields() { - return this.unknownFields; - } - private NestedMessage( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - this(); - if (extensionRegistry == null) { - throw new java.lang.NullPointerException(); - } - int mutable_bitField0_ = 0; - com.google.protobuf.UnknownFieldSet.Builder unknownFields = - com.google.protobuf.UnknownFieldSet.newBuilder(); - try { - boolean done = false; - while (!done) { - int tag = input.readTag(); - switch (tag) { - case 0: - done = true; - break; - case 10: { - java.lang.String s = input.readStringRequireUtf8(); - - title_ = s; - break; - } - case 18: { - java.lang.String s = input.readStringRequireUtf8(); - if (!((mutable_bitField0_ & 0x00000001) != 0)) { - urls_ = new com.google.protobuf.LazyStringArrayList(); - mutable_bitField0_ |= 0x00000001; - } - urls_.add(s); - break; - } - default: { - if (!parseUnknownField( - input, unknownFields, extensionRegistry, tag)) { - done = true; - } - break; - } - } - } - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw e.setUnfinishedMessage(this); - } catch (java.io.IOException e) { - throw new com.google.protobuf.InvalidProtocolBufferException( - e).setUnfinishedMessage(this); - } finally { - if (((mutable_bitField0_ & 0x00000001) != 0)) { - urls_ = urls_.getUnmodifiableView(); - } - this.unknownFields = unknownFields.build(); - makeExtensionsImmutable(); - } - } - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_SubMessage_NestedMessage_descriptor; - } - - @java.lang.Override - protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_SubMessage_NestedMessage_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.class, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.Builder.class); - } - - public static final int TITLE_FIELD_NUMBER = 1; - private volatile java.lang.Object title_; - /** - * string title = 1; - * @return The title. - */ - @java.lang.Override - public java.lang.String getTitle() { - java.lang.Object ref = title_; - if (ref instanceof java.lang.String) { - return (java.lang.String) ref; - } else { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - java.lang.String s = bs.toStringUtf8(); - title_ = s; - return s; - } - } - /** - * string title = 1; - * @return The bytes for title. - */ - @java.lang.Override - public com.google.protobuf.ByteString - getTitleBytes() { - java.lang.Object ref = title_; - if (ref instanceof java.lang.String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (java.lang.String) ref); - title_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - - public static final int URLS_FIELD_NUMBER = 2; - private com.google.protobuf.LazyStringList urls_; - /** - * repeated string urls = 2; - * @return A list containing the urls. - */ - public com.google.protobuf.ProtocolStringList - getUrlsList() { - return urls_; - } - /** - * repeated string urls = 2; - * @return The count of urls. - */ - public int getUrlsCount() { - return urls_.size(); - } - /** - * repeated string urls = 2; - * @param index The index of the element to return. - * @return The urls at the given index. - */ - public java.lang.String getUrls(int index) { - return urls_.get(index); - } - /** - * repeated string urls = 2; - * @param index The index of the value to return. - * @return The bytes of the urls at the given index. - */ - public com.google.protobuf.ByteString - getUrlsBytes(int index) { - return urls_.getByteString(index); - } - - private byte memoizedIsInitialized = -1; - @java.lang.Override - public final boolean isInitialized() { - byte isInitialized = memoizedIsInitialized; - if (isInitialized == 1) return true; - if (isInitialized == 0) return false; - - memoizedIsInitialized = 1; - return true; - } - - @java.lang.Override - public void writeTo(com.google.protobuf.CodedOutputStream output) - throws java.io.IOException { - if (!getTitleBytes().isEmpty()) { - com.google.protobuf.GeneratedMessageV3.writeString(output, 1, title_); - } - for (int i = 0; i < urls_.size(); i++) { - com.google.protobuf.GeneratedMessageV3.writeString(output, 2, urls_.getRaw(i)); - } - unknownFields.writeTo(output); - } - - @java.lang.Override - public int getSerializedSize() { - int size = memoizedSize; - if (size != -1) return size; - - size = 0; - if (!getTitleBytes().isEmpty()) { - size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, title_); - } - { - int dataSize = 0; - for (int i = 0; i < urls_.size(); i++) { - dataSize += computeStringSizeNoTag(urls_.getRaw(i)); - } - size += dataSize; - size += 1 * getUrlsList().size(); - } - size += unknownFields.getSerializedSize(); - memoizedSize = size; - return size; - } - - @java.lang.Override - public boolean equals(final java.lang.Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage)) { - return super.equals(obj); - } - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage other = (org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage) obj; - - if (!getTitle() - .equals(other.getTitle())) return false; - if (!getUrlsList() - .equals(other.getUrlsList())) return false; - if (!unknownFields.equals(other.unknownFields)) return false; - return true; - } - - @java.lang.Override - public int hashCode() { - if (memoizedHashCode != 0) { - return memoizedHashCode; - } - int hash = 41; - hash = (19 * hash) + getDescriptor().hashCode(); - hash = (37 * hash) + TITLE_FIELD_NUMBER; - hash = (53 * hash) + getTitle().hashCode(); - if (getUrlsCount() > 0) { - hash = (37 * hash) + URLS_FIELD_NUMBER; - hash = (53 * hash) + getUrlsList().hashCode(); - } - hash = (29 * hash) + unknownFields.hashCode(); - memoizedHashCode = hash; - return hash; - } - - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage parseFrom( - java.nio.ByteBuffer data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage parseFrom( - java.nio.ByteBuffer data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage parseFrom( - com.google.protobuf.ByteString data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage parseFrom( - com.google.protobuf.ByteString data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage parseFrom(byte[] data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage parseFrom( - byte[] data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage parseFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage parseFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage parseDelimitedFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage parseDelimitedFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input, extensionRegistry); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage parseFrom( - com.google.protobuf.CodedInputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage parseFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - - @java.lang.Override - public Builder newBuilderForType() { return newBuilder(); } - public static Builder newBuilder() { - return DEFAULT_INSTANCE.toBuilder(); - } - public static Builder newBuilder(org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage prototype) { - return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); - } - @java.lang.Override - public Builder toBuilder() { - return this == DEFAULT_INSTANCE - ? new Builder() : new Builder().mergeFrom(this); - } - - @java.lang.Override - protected Builder newBuilderForType( - com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { - Builder builder = new Builder(parent); - return builder; - } - /** - * Protobuf type {@code proto.SubMessage.NestedMessage} - */ - public static final class Builder extends - com.google.protobuf.GeneratedMessageV3.Builder implements - // @@protoc_insertion_point(builder_implements:proto.SubMessage.NestedMessage) - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessageOrBuilder { - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_SubMessage_NestedMessage_descriptor; - } - - @java.lang.Override - protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_SubMessage_NestedMessage_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.class, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.Builder.class); - } - - // Construct using org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.newBuilder() - private Builder() { - maybeForceBuilderInitialization(); - } - - private Builder( - com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { - super(parent); - maybeForceBuilderInitialization(); - } - private void maybeForceBuilderInitialization() { - if (com.google.protobuf.GeneratedMessageV3 - .alwaysUseFieldBuilders) { - } - } - @java.lang.Override - public Builder clear() { - super.clear(); - title_ = ""; - - urls_ = com.google.protobuf.LazyStringArrayList.EMPTY; - bitField0_ = (bitField0_ & ~0x00000001); - return this; - } - - @java.lang.Override - public com.google.protobuf.Descriptors.Descriptor - getDescriptorForType() { - return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_SubMessage_NestedMessage_descriptor; - } - - @java.lang.Override - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage getDefaultInstanceForType() { - return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.getDefaultInstance(); - } - - @java.lang.Override - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage build() { - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException(result); - } - return result; - } - - @java.lang.Override - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage buildPartial() { - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage result = new org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage(this); - int from_bitField0_ = bitField0_; - result.title_ = title_; - if (((bitField0_ & 0x00000001) != 0)) { - urls_ = urls_.getUnmodifiableView(); - bitField0_ = (bitField0_ & ~0x00000001); - } - result.urls_ = urls_; - onBuilt(); - return result; - } - - @java.lang.Override - public Builder clone() { - return super.clone(); - } - @java.lang.Override - public Builder setField( - com.google.protobuf.Descriptors.FieldDescriptor field, - java.lang.Object value) { - return super.setField(field, value); - } - @java.lang.Override - public Builder clearField( - com.google.protobuf.Descriptors.FieldDescriptor field) { - return super.clearField(field); - } - @java.lang.Override - public Builder clearOneof( - com.google.protobuf.Descriptors.OneofDescriptor oneof) { - return super.clearOneof(oneof); - } - @java.lang.Override - public Builder setRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - int index, java.lang.Object value) { - return super.setRepeatedField(field, index, value); - } - @java.lang.Override - public Builder addRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - java.lang.Object value) { - return super.addRepeatedField(field, value); - } - @java.lang.Override - public Builder mergeFrom(com.google.protobuf.Message other) { - if (other instanceof org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage) { - return mergeFrom((org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage)other); - } else { - super.mergeFrom(other); - return this; - } - } - - public Builder mergeFrom(org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage other) { - if (other == org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.getDefaultInstance()) return this; - if (!other.getTitle().isEmpty()) { - title_ = other.title_; - onChanged(); - } - if (!other.urls_.isEmpty()) { - if (urls_.isEmpty()) { - urls_ = other.urls_; - bitField0_ = (bitField0_ & ~0x00000001); - } else { - ensureUrlsIsMutable(); - urls_.addAll(other.urls_); - } - onChanged(); - } - this.mergeUnknownFields(other.unknownFields); - onChanged(); - return this; - } - - @java.lang.Override - public final boolean isInitialized() { - return true; - } - - @java.lang.Override - public Builder mergeFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage parsedMessage = null; - try { - parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - parsedMessage = (org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage) e.getUnfinishedMessage(); - throw e.unwrapIOException(); - } finally { - if (parsedMessage != null) { - mergeFrom(parsedMessage); - } - } - return this; - } - private int bitField0_; - - private java.lang.Object title_ = ""; - /** - * string title = 1; - * @return The title. - */ - public java.lang.String getTitle() { - java.lang.Object ref = title_; - if (!(ref instanceof java.lang.String)) { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - java.lang.String s = bs.toStringUtf8(); - title_ = s; - return s; - } else { - return (java.lang.String) ref; - } - } - /** - * string title = 1; - * @return The bytes for title. - */ - public com.google.protobuf.ByteString - getTitleBytes() { - java.lang.Object ref = title_; - if (ref instanceof String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (java.lang.String) ref); - title_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - /** - * string title = 1; - * @param value The title to set. - * @return This builder for chaining. - */ - public Builder setTitle( - java.lang.String value) { - if (value == null) { - throw new NullPointerException(); - } - - title_ = value; - onChanged(); - return this; - } - /** - * string title = 1; - * @return This builder for chaining. - */ - public Builder clearTitle() { - - title_ = getDefaultInstance().getTitle(); - onChanged(); - return this; - } - /** - * string title = 1; - * @param value The bytes for title to set. - * @return This builder for chaining. - */ - public Builder setTitleBytes( - com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - checkByteStringIsUtf8(value); - - title_ = value; - onChanged(); - return this; - } - - private com.google.protobuf.LazyStringList urls_ = com.google.protobuf.LazyStringArrayList.EMPTY; - private void ensureUrlsIsMutable() { - if (!((bitField0_ & 0x00000001) != 0)) { - urls_ = new com.google.protobuf.LazyStringArrayList(urls_); - bitField0_ |= 0x00000001; - } - } - /** - * repeated string urls = 2; - * @return A list containing the urls. - */ - public com.google.protobuf.ProtocolStringList - getUrlsList() { - return urls_.getUnmodifiableView(); - } - /** - * repeated string urls = 2; - * @return The count of urls. - */ - public int getUrlsCount() { - return urls_.size(); - } - /** - * repeated string urls = 2; - * @param index The index of the element to return. - * @return The urls at the given index. - */ - public java.lang.String getUrls(int index) { - return urls_.get(index); - } - /** - * repeated string urls = 2; - * @param index The index of the value to return. - * @return The bytes of the urls at the given index. - */ - public com.google.protobuf.ByteString - getUrlsBytes(int index) { - return urls_.getByteString(index); - } - /** - * repeated string urls = 2; - * @param index The index to set the value at. - * @param value The urls to set. - * @return This builder for chaining. - */ - public Builder setUrls( - int index, java.lang.String value) { - if (value == null) { - throw new NullPointerException(); - } - ensureUrlsIsMutable(); - urls_.set(index, value); - onChanged(); - return this; - } - /** - * repeated string urls = 2; - * @param value The urls to add. - * @return This builder for chaining. - */ - public Builder addUrls( - java.lang.String value) { - if (value == null) { - throw new NullPointerException(); - } - ensureUrlsIsMutable(); - urls_.add(value); - onChanged(); - return this; - } - /** - * repeated string urls = 2; - * @param values The urls to add. - * @return This builder for chaining. - */ - public Builder addAllUrls( - java.lang.Iterable values) { - ensureUrlsIsMutable(); - com.google.protobuf.AbstractMessageLite.Builder.addAll( - values, urls_); - onChanged(); - return this; - } - /** - * repeated string urls = 2; - * @return This builder for chaining. - */ - public Builder clearUrls() { - urls_ = com.google.protobuf.LazyStringArrayList.EMPTY; - bitField0_ = (bitField0_ & ~0x00000001); - onChanged(); - return this; - } - /** - * repeated string urls = 2; - * @param value The bytes of the urls to add. - * @return This builder for chaining. - */ - public Builder addUrlsBytes( - com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - checkByteStringIsUtf8(value); - ensureUrlsIsMutable(); - urls_.add(value); - onChanged(); - return this; - } - @java.lang.Override - public final Builder setUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.setUnknownFields(unknownFields); - } - - @java.lang.Override - public final Builder mergeUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.mergeUnknownFields(unknownFields); - } - - - // @@protoc_insertion_point(builder_scope:proto.SubMessage.NestedMessage) - } - - // @@protoc_insertion_point(class_scope:proto.SubMessage.NestedMessage) - private static final org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage DEFAULT_INSTANCE; - static { - DEFAULT_INSTANCE = new org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage(); - } - - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage getDefaultInstance() { - return DEFAULT_INSTANCE; - } - - private static final com.google.protobuf.Parser - PARSER = new com.google.protobuf.AbstractParser() { - @java.lang.Override - public NestedMessage parsePartialFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return new NestedMessage(input, extensionRegistry); - } - }; - - public static com.google.protobuf.Parser parser() { - return PARSER; - } - - @java.lang.Override - public com.google.protobuf.Parser getParserForType() { - return PARSER; - } - - @java.lang.Override - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage getDefaultInstanceForType() { - return DEFAULT_INSTANCE; - } - - } - - public static final int FOO_FIELD_NUMBER = 1; - private volatile java.lang.Object foo_; - /** - * string foo = 1; - * @return The foo. - */ - @java.lang.Override - public java.lang.String getFoo() { - java.lang.Object ref = foo_; - if (ref instanceof java.lang.String) { - return (java.lang.String) ref; - } else { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - java.lang.String s = bs.toStringUtf8(); - foo_ = s; - return s; - } - } - /** - * string foo = 1; - * @return The bytes for foo. - */ - @java.lang.Override - public com.google.protobuf.ByteString - getFooBytes() { - java.lang.Object ref = foo_; - if (ref instanceof java.lang.String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (java.lang.String) ref); - foo_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - - public static final int BAR_FIELD_NUMBER = 2; - private double bar_; - /** - * double bar = 2; - * @return The bar. - */ - @java.lang.Override - public double getBar() { - return bar_; - } - - public static final int NESTEDMESSAGE_FIELD_NUMBER = 3; - private org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage nestedMessage_; - /** - * .proto.SubMessage.NestedMessage nestedMessage = 3; - * @return Whether the nestedMessage field is set. - */ - @java.lang.Override - public boolean hasNestedMessage() { - return nestedMessage_ != null; - } - /** - * .proto.SubMessage.NestedMessage nestedMessage = 3; - * @return The nestedMessage. - */ - @java.lang.Override - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage getNestedMessage() { - return nestedMessage_ == null ? org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.getDefaultInstance() : nestedMessage_; - } - /** - * .proto.SubMessage.NestedMessage nestedMessage = 3; - */ - @java.lang.Override - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessageOrBuilder getNestedMessageOrBuilder() { - return getNestedMessage(); - } - - private byte memoizedIsInitialized = -1; - @java.lang.Override - public final boolean isInitialized() { - byte isInitialized = memoizedIsInitialized; - if (isInitialized == 1) return true; - if (isInitialized == 0) return false; - - memoizedIsInitialized = 1; - return true; - } - - @java.lang.Override - public void writeTo(com.google.protobuf.CodedOutputStream output) - throws java.io.IOException { - if (!getFooBytes().isEmpty()) { - com.google.protobuf.GeneratedMessageV3.writeString(output, 1, foo_); - } - if (bar_ != 0D) { - output.writeDouble(2, bar_); - } - if (nestedMessage_ != null) { - output.writeMessage(3, getNestedMessage()); - } - unknownFields.writeTo(output); - } - - @java.lang.Override - public int getSerializedSize() { - int size = memoizedSize; - if (size != -1) return size; - - size = 0; - if (!getFooBytes().isEmpty()) { - size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, foo_); - } - if (bar_ != 0D) { - size += com.google.protobuf.CodedOutputStream - .computeDoubleSize(2, bar_); - } - if (nestedMessage_ != null) { - size += com.google.protobuf.CodedOutputStream - .computeMessageSize(3, getNestedMessage()); - } - size += unknownFields.getSerializedSize(); - memoizedSize = size; - return size; - } - - @java.lang.Override - public boolean equals(final java.lang.Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage)) { - return super.equals(obj); - } - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage other = (org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage) obj; - - if (!getFoo() - .equals(other.getFoo())) return false; - if (java.lang.Double.doubleToLongBits(getBar()) - != java.lang.Double.doubleToLongBits( - other.getBar())) return false; - if (hasNestedMessage() != other.hasNestedMessage()) return false; - if (hasNestedMessage()) { - if (!getNestedMessage() - .equals(other.getNestedMessage())) return false; - } - if (!unknownFields.equals(other.unknownFields)) return false; - return true; - } - - @java.lang.Override - public int hashCode() { - if (memoizedHashCode != 0) { - return memoizedHashCode; - } - int hash = 41; - hash = (19 * hash) + getDescriptor().hashCode(); - hash = (37 * hash) + FOO_FIELD_NUMBER; - hash = (53 * hash) + getFoo().hashCode(); - hash = (37 * hash) + BAR_FIELD_NUMBER; - hash = (53 * hash) + com.google.protobuf.Internal.hashLong( - java.lang.Double.doubleToLongBits(getBar())); - if (hasNestedMessage()) { - hash = (37 * hash) + NESTEDMESSAGE_FIELD_NUMBER; - hash = (53 * hash) + getNestedMessage().hashCode(); - } - hash = (29 * hash) + unknownFields.hashCode(); - memoizedHashCode = hash; - return hash; - } - - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage parseFrom( - java.nio.ByteBuffer data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage parseFrom( - java.nio.ByteBuffer data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage parseFrom( - com.google.protobuf.ByteString data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage parseFrom( - com.google.protobuf.ByteString data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage parseFrom(byte[] data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage parseFrom( - byte[] data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage parseFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage parseFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage parseDelimitedFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage parseDelimitedFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input, extensionRegistry); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage parseFrom( - com.google.protobuf.CodedInputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage parseFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - - @java.lang.Override - public Builder newBuilderForType() { return newBuilder(); } - public static Builder newBuilder() { - return DEFAULT_INSTANCE.toBuilder(); - } - public static Builder newBuilder(org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage prototype) { - return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); - } - @java.lang.Override - public Builder toBuilder() { - return this == DEFAULT_INSTANCE - ? new Builder() : new Builder().mergeFrom(this); - } - - @java.lang.Override - protected Builder newBuilderForType( - com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { - Builder builder = new Builder(parent); - return builder; - } - /** - * Protobuf type {@code proto.SubMessage} - */ - public static final class Builder extends - com.google.protobuf.GeneratedMessageV3.Builder implements - // @@protoc_insertion_point(builder_implements:proto.SubMessage) - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessageOrBuilder { - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_SubMessage_descriptor; - } - - @java.lang.Override - protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_SubMessage_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.class, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.Builder.class); - } - - // Construct using org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.newBuilder() - private Builder() { - maybeForceBuilderInitialization(); - } - - private Builder( - com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { - super(parent); - maybeForceBuilderInitialization(); - } - private void maybeForceBuilderInitialization() { - if (com.google.protobuf.GeneratedMessageV3 - .alwaysUseFieldBuilders) { - } - } - @java.lang.Override - public Builder clear() { - super.clear(); - foo_ = ""; - - bar_ = 0D; - - if (nestedMessageBuilder_ == null) { - nestedMessage_ = null; - } else { - nestedMessage_ = null; - nestedMessageBuilder_ = null; - } - return this; - } - - @java.lang.Override - public com.google.protobuf.Descriptors.Descriptor - getDescriptorForType() { - return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_SubMessage_descriptor; - } - - @java.lang.Override - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage getDefaultInstanceForType() { - return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.getDefaultInstance(); - } - - @java.lang.Override - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage build() { - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException(result); - } - return result; - } - - @java.lang.Override - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage buildPartial() { - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage result = new org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage(this); - result.foo_ = foo_; - result.bar_ = bar_; - if (nestedMessageBuilder_ == null) { - result.nestedMessage_ = nestedMessage_; - } else { - result.nestedMessage_ = nestedMessageBuilder_.build(); - } - onBuilt(); - return result; - } - - @java.lang.Override - public Builder clone() { - return super.clone(); - } - @java.lang.Override - public Builder setField( - com.google.protobuf.Descriptors.FieldDescriptor field, - java.lang.Object value) { - return super.setField(field, value); - } - @java.lang.Override - public Builder clearField( - com.google.protobuf.Descriptors.FieldDescriptor field) { - return super.clearField(field); - } - @java.lang.Override - public Builder clearOneof( - com.google.protobuf.Descriptors.OneofDescriptor oneof) { - return super.clearOneof(oneof); - } - @java.lang.Override - public Builder setRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - int index, java.lang.Object value) { - return super.setRepeatedField(field, index, value); - } - @java.lang.Override - public Builder addRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - java.lang.Object value) { - return super.addRepeatedField(field, value); - } - @java.lang.Override - public Builder mergeFrom(com.google.protobuf.Message other) { - if (other instanceof org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage) { - return mergeFrom((org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage)other); - } else { - super.mergeFrom(other); - return this; - } - } - - public Builder mergeFrom(org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage other) { - if (other == org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.getDefaultInstance()) return this; - if (!other.getFoo().isEmpty()) { - foo_ = other.foo_; - onChanged(); - } - if (other.getBar() != 0D) { - setBar(other.getBar()); - } - if (other.hasNestedMessage()) { - mergeNestedMessage(other.getNestedMessage()); - } - this.mergeUnknownFields(other.unknownFields); - onChanged(); - return this; - } - - @java.lang.Override - public final boolean isInitialized() { - return true; - } - - @java.lang.Override - public Builder mergeFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage parsedMessage = null; - try { - parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - parsedMessage = (org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage) e.getUnfinishedMessage(); - throw e.unwrapIOException(); - } finally { - if (parsedMessage != null) { - mergeFrom(parsedMessage); - } - } - return this; - } - - private java.lang.Object foo_ = ""; - /** - * string foo = 1; - * @return The foo. - */ - public java.lang.String getFoo() { - java.lang.Object ref = foo_; - if (!(ref instanceof java.lang.String)) { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - java.lang.String s = bs.toStringUtf8(); - foo_ = s; - return s; - } else { - return (java.lang.String) ref; - } - } - /** - * string foo = 1; - * @return The bytes for foo. - */ - public com.google.protobuf.ByteString - getFooBytes() { - java.lang.Object ref = foo_; - if (ref instanceof String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (java.lang.String) ref); - foo_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - /** - * string foo = 1; - * @param value The foo to set. - * @return This builder for chaining. - */ - public Builder setFoo( - java.lang.String value) { - if (value == null) { - throw new NullPointerException(); - } - - foo_ = value; - onChanged(); - return this; - } - /** - * string foo = 1; - * @return This builder for chaining. - */ - public Builder clearFoo() { - - foo_ = getDefaultInstance().getFoo(); - onChanged(); - return this; - } - /** - * string foo = 1; - * @param value The bytes for foo to set. - * @return This builder for chaining. - */ - public Builder setFooBytes( - com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - checkByteStringIsUtf8(value); - - foo_ = value; - onChanged(); - return this; - } - - private double bar_ ; - /** - * double bar = 2; - * @return The bar. - */ - @java.lang.Override - public double getBar() { - return bar_; - } - /** - * double bar = 2; - * @param value The bar to set. - * @return This builder for chaining. - */ - public Builder setBar(double value) { - - bar_ = value; - onChanged(); - return this; - } - /** - * double bar = 2; - * @return This builder for chaining. - */ - public Builder clearBar() { - - bar_ = 0D; - onChanged(); - return this; - } - - private org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage nestedMessage_; - private com.google.protobuf.SingleFieldBuilderV3< - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.Builder, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessageOrBuilder> nestedMessageBuilder_; - /** - * .proto.SubMessage.NestedMessage nestedMessage = 3; - * @return Whether the nestedMessage field is set. - */ - public boolean hasNestedMessage() { - return nestedMessageBuilder_ != null || nestedMessage_ != null; - } - /** - * .proto.SubMessage.NestedMessage nestedMessage = 3; - * @return The nestedMessage. - */ - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage getNestedMessage() { - if (nestedMessageBuilder_ == null) { - return nestedMessage_ == null ? org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.getDefaultInstance() : nestedMessage_; - } else { - return nestedMessageBuilder_.getMessage(); - } - } - /** - * .proto.SubMessage.NestedMessage nestedMessage = 3; - */ - public Builder setNestedMessage(org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage value) { - if (nestedMessageBuilder_ == null) { - if (value == null) { - throw new NullPointerException(); - } - nestedMessage_ = value; - onChanged(); - } else { - nestedMessageBuilder_.setMessage(value); - } - - return this; - } - /** - * .proto.SubMessage.NestedMessage nestedMessage = 3; - */ - public Builder setNestedMessage( - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.Builder builderForValue) { - if (nestedMessageBuilder_ == null) { - nestedMessage_ = builderForValue.build(); - onChanged(); - } else { - nestedMessageBuilder_.setMessage(builderForValue.build()); - } - - return this; - } - /** - * .proto.SubMessage.NestedMessage nestedMessage = 3; - */ - public Builder mergeNestedMessage(org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage value) { - if (nestedMessageBuilder_ == null) { - if (nestedMessage_ != null) { - nestedMessage_ = - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.newBuilder(nestedMessage_).mergeFrom(value).buildPartial(); - } else { - nestedMessage_ = value; - } - onChanged(); - } else { - nestedMessageBuilder_.mergeFrom(value); - } - - return this; - } - /** - * .proto.SubMessage.NestedMessage nestedMessage = 3; - */ - public Builder clearNestedMessage() { - if (nestedMessageBuilder_ == null) { - nestedMessage_ = null; - onChanged(); - } else { - nestedMessage_ = null; - nestedMessageBuilder_ = null; - } - - return this; - } - /** - * .proto.SubMessage.NestedMessage nestedMessage = 3; - */ - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.Builder getNestedMessageBuilder() { - - onChanged(); - return getNestedMessageFieldBuilder().getBuilder(); - } - /** - * .proto.SubMessage.NestedMessage nestedMessage = 3; - */ - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessageOrBuilder getNestedMessageOrBuilder() { - if (nestedMessageBuilder_ != null) { - return nestedMessageBuilder_.getMessageOrBuilder(); - } else { - return nestedMessage_ == null ? - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.getDefaultInstance() : nestedMessage_; - } - } - /** - * .proto.SubMessage.NestedMessage nestedMessage = 3; - */ - private com.google.protobuf.SingleFieldBuilderV3< - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.Builder, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessageOrBuilder> - getNestedMessageFieldBuilder() { - if (nestedMessageBuilder_ == null) { - nestedMessageBuilder_ = new com.google.protobuf.SingleFieldBuilderV3< - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessage.Builder, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.NestedMessageOrBuilder>( - getNestedMessage(), - getParentForChildren(), - isClean()); - nestedMessage_ = null; - } - return nestedMessageBuilder_; - } - @java.lang.Override - public final Builder setUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.setUnknownFields(unknownFields); - } - - @java.lang.Override - public final Builder mergeUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.mergeUnknownFields(unknownFields); - } - - - // @@protoc_insertion_point(builder_scope:proto.SubMessage) - } - - // @@protoc_insertion_point(class_scope:proto.SubMessage) - private static final org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage DEFAULT_INSTANCE; - static { - DEFAULT_INSTANCE = new org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage(); - } - - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage getDefaultInstance() { - return DEFAULT_INSTANCE; - } - - private static final com.google.protobuf.Parser - PARSER = new com.google.protobuf.AbstractParser() { - @java.lang.Override - public SubMessage parsePartialFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return new SubMessage(input, extensionRegistry); - } - }; - - public static com.google.protobuf.Parser parser() { - return PARSER; - } - - @java.lang.Override - public com.google.protobuf.Parser getParserForType() { - return PARSER; - } - - @java.lang.Override - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage getDefaultInstanceForType() { - return DEFAULT_INSTANCE; - } - - } - - public interface TestMessageOrBuilder extends - // @@protoc_insertion_point(interface_extends:proto.TestMessage) - com.google.protobuf.MessageOrBuilder { - - /** - * string stringField = 1; - * @return The stringField. - */ - java.lang.String getStringField(); - /** - * string stringField = 1; - * @return The bytes for stringField. - */ - com.google.protobuf.ByteString - getStringFieldBytes(); - - /** - * double doubleField = 2; - * @return The doubleField. - */ - double getDoubleField(); - - /** - * float floatField = 3; - * @return The floatField. - */ - float getFloatField(); - - /** - * int32 int32Field = 4; - * @return The int32Field. - */ - int getInt32Field(); - - /** - * int64 int64Field = 5; - * @return The int64Field. - */ - long getInt64Field(); - - /** - * uint32 uint32Field = 6; - * @return The uint32Field. - */ - int getUint32Field(); - - /** - * uint64 uint64Field = 7; - * @return The uint64Field. - */ - long getUint64Field(); - - /** - * sint32 sint32Field = 8; - * @return The sint32Field. - */ - int getSint32Field(); - - /** - * sint64 sint64Field = 9; - * @return The sint64Field. - */ - long getSint64Field(); - - /** - * fixed32 fixed32Field = 10; - * @return The fixed32Field. - */ - int getFixed32Field(); - - /** - * fixed64 fixed64Field = 11; - * @return The fixed64Field. - */ - long getFixed64Field(); - - /** - * sfixed32 sfixed32Field = 12; - * @return The sfixed32Field. - */ - int getSfixed32Field(); - - /** - * sfixed64 sfixed64Field = 13; - * @return The sfixed64Field. - */ - long getSfixed64Field(); - - /** - * bool boolField = 14; - * @return The boolField. - */ - boolean getBoolField(); - - /** - * bytes bytesField = 15; - * @return The bytesField. - */ - com.google.protobuf.ByteString getBytesField(); - - /** - * .proto.TestEnum testEnum = 16; - * @return The enum numeric value on the wire for testEnum. - */ - int getTestEnumValue(); - /** - * .proto.TestEnum testEnum = 16; - * @return The testEnum. - */ - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum getTestEnum(); - - /** - * .proto.SubMessage subMessage = 17; - * @return Whether the subMessage field is set. - */ - boolean hasSubMessage(); - /** - * .proto.SubMessage subMessage = 17; - * @return The subMessage. - */ - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage getSubMessage(); - /** - * .proto.SubMessage subMessage = 17; - */ - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessageOrBuilder getSubMessageOrBuilder(); - - /** - * repeated string repeatedField = 18; - * @return A list containing the repeatedField. - */ - java.util.List - getRepeatedFieldList(); - /** - * repeated string repeatedField = 18; - * @return The count of repeatedField. - */ - int getRepeatedFieldCount(); - /** - * repeated string repeatedField = 18; - * @param index The index of the element to return. - * @return The repeatedField at the given index. - */ - java.lang.String getRepeatedField(int index); - /** - * repeated string repeatedField = 18; - * @param index The index of the value to return. - * @return The bytes of the repeatedField at the given index. - */ - com.google.protobuf.ByteString - getRepeatedFieldBytes(int index); - - /** - * map<string, double> mapField = 19; - */ - int getMapFieldCount(); - /** - * map<string, double> mapField = 19; - */ - boolean containsMapField( - java.lang.String key); - /** - * Use {@link #getMapFieldMap()} instead. - */ - @java.lang.Deprecated - java.util.Map - getMapField(); - /** - * map<string, double> mapField = 19; - */ - java.util.Map - getMapFieldMap(); - /** - * map<string, double> mapField = 19; - */ - - double getMapFieldOrDefault( - java.lang.String key, - double defaultValue); - /** - * map<string, double> mapField = 19; - */ - - double getMapFieldOrThrow( - java.lang.String key); - - /** - * .google.protobuf.Timestamp timestampField = 20; - * @return Whether the timestampField field is set. - */ - boolean hasTimestampField(); - /** - * .google.protobuf.Timestamp timestampField = 20; - * @return The timestampField. - */ - com.google.protobuf.Timestamp getTimestampField(); - /** - * .google.protobuf.Timestamp timestampField = 20; - */ - com.google.protobuf.TimestampOrBuilder getTimestampFieldOrBuilder(); - } - /** - * Protobuf type {@code proto.TestMessage} - */ - public static final class TestMessage extends - com.google.protobuf.GeneratedMessageV3 implements - // @@protoc_insertion_point(message_implements:proto.TestMessage) - TestMessageOrBuilder { - private static final long serialVersionUID = 0L; - // Use TestMessage.newBuilder() to construct. - private TestMessage(com.google.protobuf.GeneratedMessageV3.Builder builder) { - super(builder); - } - private TestMessage() { - stringField_ = ""; - bytesField_ = com.google.protobuf.ByteString.EMPTY; - testEnum_ = 0; - repeatedField_ = com.google.protobuf.LazyStringArrayList.EMPTY; - } - - @java.lang.Override - @SuppressWarnings({"unused"}) - protected java.lang.Object newInstance( - UnusedPrivateParameter unused) { - return new TestMessage(); - } - - @java.lang.Override - public final com.google.protobuf.UnknownFieldSet - getUnknownFields() { - return this.unknownFields; - } - private TestMessage( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - this(); - if (extensionRegistry == null) { - throw new java.lang.NullPointerException(); - } - int mutable_bitField0_ = 0; - com.google.protobuf.UnknownFieldSet.Builder unknownFields = - com.google.protobuf.UnknownFieldSet.newBuilder(); - try { - boolean done = false; - while (!done) { - int tag = input.readTag(); - switch (tag) { - case 0: - done = true; - break; - case 10: { - java.lang.String s = input.readStringRequireUtf8(); - - stringField_ = s; - break; - } - case 17: { - - doubleField_ = input.readDouble(); - break; - } - case 29: { - - floatField_ = input.readFloat(); - break; - } - case 32: { - - int32Field_ = input.readInt32(); - break; - } - case 40: { - - int64Field_ = input.readInt64(); - break; - } - case 48: { - - uint32Field_ = input.readUInt32(); - break; - } - case 56: { - - uint64Field_ = input.readUInt64(); - break; - } - case 64: { - - sint32Field_ = input.readSInt32(); - break; - } - case 72: { - - sint64Field_ = input.readSInt64(); - break; - } - case 85: { - - fixed32Field_ = input.readFixed32(); - break; - } - case 89: { - - fixed64Field_ = input.readFixed64(); - break; - } - case 101: { - - sfixed32Field_ = input.readSFixed32(); - break; - } - case 105: { - - sfixed64Field_ = input.readSFixed64(); - break; - } - case 112: { - - boolField_ = input.readBool(); - break; - } - case 122: { - - bytesField_ = input.readBytes(); - break; - } - case 128: { - int rawValue = input.readEnum(); - - testEnum_ = rawValue; - break; - } - case 138: { - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.Builder subBuilder = null; - if (subMessage_ != null) { - subBuilder = subMessage_.toBuilder(); - } - subMessage_ = input.readMessage(org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.parser(), extensionRegistry); - if (subBuilder != null) { - subBuilder.mergeFrom(subMessage_); - subMessage_ = subBuilder.buildPartial(); - } - - break; - } - case 146: { - java.lang.String s = input.readStringRequireUtf8(); - if (!((mutable_bitField0_ & 0x00000001) != 0)) { - repeatedField_ = new com.google.protobuf.LazyStringArrayList(); - mutable_bitField0_ |= 0x00000001; - } - repeatedField_.add(s); - break; - } - case 154: { - if (!((mutable_bitField0_ & 0x00000002) != 0)) { - mapField_ = com.google.protobuf.MapField.newMapField( - MapFieldDefaultEntryHolder.defaultEntry); - mutable_bitField0_ |= 0x00000002; - } - com.google.protobuf.MapEntry - mapField__ = input.readMessage( - MapFieldDefaultEntryHolder.defaultEntry.getParserForType(), extensionRegistry); - mapField_.getMutableMap().put( - mapField__.getKey(), mapField__.getValue()); - break; - } - case 162: { - com.google.protobuf.Timestamp.Builder subBuilder = null; - if (timestampField_ != null) { - subBuilder = timestampField_.toBuilder(); - } - timestampField_ = input.readMessage(com.google.protobuf.Timestamp.parser(), extensionRegistry); - if (subBuilder != null) { - subBuilder.mergeFrom(timestampField_); - timestampField_ = subBuilder.buildPartial(); - } - - break; - } - default: { - if (!parseUnknownField( - input, unknownFields, extensionRegistry, tag)) { - done = true; - } - break; - } - } - } - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw e.setUnfinishedMessage(this); - } catch (java.io.IOException e) { - throw new com.google.protobuf.InvalidProtocolBufferException( - e).setUnfinishedMessage(this); - } finally { - if (((mutable_bitField0_ & 0x00000001) != 0)) { - repeatedField_ = repeatedField_.getUnmodifiableView(); - } - this.unknownFields = unknownFields.build(); - makeExtensionsImmutable(); - } - } - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_TestMessage_descriptor; - } - - @SuppressWarnings({"rawtypes"}) - @java.lang.Override - protected com.google.protobuf.MapField internalGetMapField( - int number) { - switch (number) { - case 19: - return internalGetMapField(); - default: - throw new RuntimeException( - "Invalid map field number: " + number); - } - } - @java.lang.Override - protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_TestMessage_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage.class, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage.Builder.class); - } - - public static final int STRINGFIELD_FIELD_NUMBER = 1; - private volatile java.lang.Object stringField_; - /** - * string stringField = 1; - * @return The stringField. - */ - @java.lang.Override - public java.lang.String getStringField() { - java.lang.Object ref = stringField_; - if (ref instanceof java.lang.String) { - return (java.lang.String) ref; - } else { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - java.lang.String s = bs.toStringUtf8(); - stringField_ = s; - return s; - } - } - /** - * string stringField = 1; - * @return The bytes for stringField. - */ - @java.lang.Override - public com.google.protobuf.ByteString - getStringFieldBytes() { - java.lang.Object ref = stringField_; - if (ref instanceof java.lang.String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (java.lang.String) ref); - stringField_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - - public static final int DOUBLEFIELD_FIELD_NUMBER = 2; - private double doubleField_; - /** - * double doubleField = 2; - * @return The doubleField. - */ - @java.lang.Override - public double getDoubleField() { - return doubleField_; - } - - public static final int FLOATFIELD_FIELD_NUMBER = 3; - private float floatField_; - /** - * float floatField = 3; - * @return The floatField. - */ - @java.lang.Override - public float getFloatField() { - return floatField_; - } - - public static final int INT32FIELD_FIELD_NUMBER = 4; - private int int32Field_; - /** - * int32 int32Field = 4; - * @return The int32Field. - */ - @java.lang.Override - public int getInt32Field() { - return int32Field_; - } - - public static final int INT64FIELD_FIELD_NUMBER = 5; - private long int64Field_; - /** - * int64 int64Field = 5; - * @return The int64Field. - */ - @java.lang.Override - public long getInt64Field() { - return int64Field_; - } - - public static final int UINT32FIELD_FIELD_NUMBER = 6; - private int uint32Field_; - /** - * uint32 uint32Field = 6; - * @return The uint32Field. - */ - @java.lang.Override - public int getUint32Field() { - return uint32Field_; - } - - public static final int UINT64FIELD_FIELD_NUMBER = 7; - private long uint64Field_; - /** - * uint64 uint64Field = 7; - * @return The uint64Field. - */ - @java.lang.Override - public long getUint64Field() { - return uint64Field_; - } - - public static final int SINT32FIELD_FIELD_NUMBER = 8; - private int sint32Field_; - /** - * sint32 sint32Field = 8; - * @return The sint32Field. - */ - @java.lang.Override - public int getSint32Field() { - return sint32Field_; - } - - public static final int SINT64FIELD_FIELD_NUMBER = 9; - private long sint64Field_; - /** - * sint64 sint64Field = 9; - * @return The sint64Field. - */ - @java.lang.Override - public long getSint64Field() { - return sint64Field_; - } - - public static final int FIXED32FIELD_FIELD_NUMBER = 10; - private int fixed32Field_; - /** - * fixed32 fixed32Field = 10; - * @return The fixed32Field. - */ - @java.lang.Override - public int getFixed32Field() { - return fixed32Field_; - } - - public static final int FIXED64FIELD_FIELD_NUMBER = 11; - private long fixed64Field_; - /** - * fixed64 fixed64Field = 11; - * @return The fixed64Field. - */ - @java.lang.Override - public long getFixed64Field() { - return fixed64Field_; - } - - public static final int SFIXED32FIELD_FIELD_NUMBER = 12; - private int sfixed32Field_; - /** - * sfixed32 sfixed32Field = 12; - * @return The sfixed32Field. - */ - @java.lang.Override - public int getSfixed32Field() { - return sfixed32Field_; - } - - public static final int SFIXED64FIELD_FIELD_NUMBER = 13; - private long sfixed64Field_; - /** - * sfixed64 sfixed64Field = 13; - * @return The sfixed64Field. - */ - @java.lang.Override - public long getSfixed64Field() { - return sfixed64Field_; - } - - public static final int BOOLFIELD_FIELD_NUMBER = 14; - private boolean boolField_; - /** - * bool boolField = 14; - * @return The boolField. - */ - @java.lang.Override - public boolean getBoolField() { - return boolField_; - } - - public static final int BYTESFIELD_FIELD_NUMBER = 15; - private com.google.protobuf.ByteString bytesField_; - /** - * bytes bytesField = 15; - * @return The bytesField. - */ - @java.lang.Override - public com.google.protobuf.ByteString getBytesField() { - return bytesField_; - } - - public static final int TESTENUM_FIELD_NUMBER = 16; - private int testEnum_; - /** - * .proto.TestEnum testEnum = 16; - * @return The enum numeric value on the wire for testEnum. - */ - @java.lang.Override public int getTestEnumValue() { - return testEnum_; - } - /** - * .proto.TestEnum testEnum = 16; - * @return The testEnum. - */ - @java.lang.Override public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum getTestEnum() { - @SuppressWarnings("deprecation") - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum result = org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum.valueOf(testEnum_); - return result == null ? org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum.UNRECOGNIZED : result; - } - - public static final int SUBMESSAGE_FIELD_NUMBER = 17; - private org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage subMessage_; - /** - * .proto.SubMessage subMessage = 17; - * @return Whether the subMessage field is set. - */ - @java.lang.Override - public boolean hasSubMessage() { - return subMessage_ != null; - } - /** - * .proto.SubMessage subMessage = 17; - * @return The subMessage. - */ - @java.lang.Override - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage getSubMessage() { - return subMessage_ == null ? org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.getDefaultInstance() : subMessage_; - } - /** - * .proto.SubMessage subMessage = 17; - */ - @java.lang.Override - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessageOrBuilder getSubMessageOrBuilder() { - return getSubMessage(); - } - - public static final int REPEATEDFIELD_FIELD_NUMBER = 18; - private com.google.protobuf.LazyStringList repeatedField_; - /** - * repeated string repeatedField = 18; - * @return A list containing the repeatedField. - */ - public com.google.protobuf.ProtocolStringList - getRepeatedFieldList() { - return repeatedField_; - } - /** - * repeated string repeatedField = 18; - * @return The count of repeatedField. - */ - public int getRepeatedFieldCount() { - return repeatedField_.size(); - } - /** - * repeated string repeatedField = 18; - * @param index The index of the element to return. - * @return The repeatedField at the given index. - */ - public java.lang.String getRepeatedField(int index) { - return repeatedField_.get(index); - } - /** - * repeated string repeatedField = 18; - * @param index The index of the value to return. - * @return The bytes of the repeatedField at the given index. - */ - public com.google.protobuf.ByteString - getRepeatedFieldBytes(int index) { - return repeatedField_.getByteString(index); - } - - public static final int MAPFIELD_FIELD_NUMBER = 19; - private static final class MapFieldDefaultEntryHolder { - static final com.google.protobuf.MapEntry< - java.lang.String, java.lang.Double> defaultEntry = - com.google.protobuf.MapEntry - .newDefaultInstance( - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_TestMessage_MapFieldEntry_descriptor, - com.google.protobuf.WireFormat.FieldType.STRING, - "", - com.google.protobuf.WireFormat.FieldType.DOUBLE, - 0D); - } - private com.google.protobuf.MapField< - java.lang.String, java.lang.Double> mapField_; - private com.google.protobuf.MapField - internalGetMapField() { - if (mapField_ == null) { - return com.google.protobuf.MapField.emptyMapField( - MapFieldDefaultEntryHolder.defaultEntry); - } - return mapField_; - } - - public int getMapFieldCount() { - return internalGetMapField().getMap().size(); - } - /** - * map<string, double> mapField = 19; - */ - - @java.lang.Override - public boolean containsMapField( - java.lang.String key) { - if (key == null) { throw new java.lang.NullPointerException(); } - return internalGetMapField().getMap().containsKey(key); - } - /** - * Use {@link #getMapFieldMap()} instead. - */ - @java.lang.Override - @java.lang.Deprecated - public java.util.Map getMapField() { - return getMapFieldMap(); - } - /** - * map<string, double> mapField = 19; - */ - @java.lang.Override - - public java.util.Map getMapFieldMap() { - return internalGetMapField().getMap(); - } - /** - * map<string, double> mapField = 19; - */ - @java.lang.Override - - public double getMapFieldOrDefault( - java.lang.String key, - double defaultValue) { - if (key == null) { throw new java.lang.NullPointerException(); } - java.util.Map map = - internalGetMapField().getMap(); - return map.containsKey(key) ? map.get(key) : defaultValue; - } - /** - * map<string, double> mapField = 19; - */ - @java.lang.Override - - public double getMapFieldOrThrow( - java.lang.String key) { - if (key == null) { throw new java.lang.NullPointerException(); } - java.util.Map map = - internalGetMapField().getMap(); - if (!map.containsKey(key)) { - throw new java.lang.IllegalArgumentException(); - } - return map.get(key); - } - - public static final int TIMESTAMPFIELD_FIELD_NUMBER = 20; - private com.google.protobuf.Timestamp timestampField_; - /** - * .google.protobuf.Timestamp timestampField = 20; - * @return Whether the timestampField field is set. - */ - @java.lang.Override - public boolean hasTimestampField() { - return timestampField_ != null; - } - /** - * .google.protobuf.Timestamp timestampField = 20; - * @return The timestampField. - */ - @java.lang.Override - public com.google.protobuf.Timestamp getTimestampField() { - return timestampField_ == null ? com.google.protobuf.Timestamp.getDefaultInstance() : timestampField_; - } - /** - * .google.protobuf.Timestamp timestampField = 20; - */ - @java.lang.Override - public com.google.protobuf.TimestampOrBuilder getTimestampFieldOrBuilder() { - return getTimestampField(); - } - - private byte memoizedIsInitialized = -1; - @java.lang.Override - public final boolean isInitialized() { - byte isInitialized = memoizedIsInitialized; - if (isInitialized == 1) return true; - if (isInitialized == 0) return false; - - memoizedIsInitialized = 1; - return true; - } - - @java.lang.Override - public void writeTo(com.google.protobuf.CodedOutputStream output) - throws java.io.IOException { - if (!getStringFieldBytes().isEmpty()) { - com.google.protobuf.GeneratedMessageV3.writeString(output, 1, stringField_); - } - if (doubleField_ != 0D) { - output.writeDouble(2, doubleField_); - } - if (floatField_ != 0F) { - output.writeFloat(3, floatField_); - } - if (int32Field_ != 0) { - output.writeInt32(4, int32Field_); - } - if (int64Field_ != 0L) { - output.writeInt64(5, int64Field_); - } - if (uint32Field_ != 0) { - output.writeUInt32(6, uint32Field_); - } - if (uint64Field_ != 0L) { - output.writeUInt64(7, uint64Field_); - } - if (sint32Field_ != 0) { - output.writeSInt32(8, sint32Field_); - } - if (sint64Field_ != 0L) { - output.writeSInt64(9, sint64Field_); - } - if (fixed32Field_ != 0) { - output.writeFixed32(10, fixed32Field_); - } - if (fixed64Field_ != 0L) { - output.writeFixed64(11, fixed64Field_); - } - if (sfixed32Field_ != 0) { - output.writeSFixed32(12, sfixed32Field_); - } - if (sfixed64Field_ != 0L) { - output.writeSFixed64(13, sfixed64Field_); - } - if (boolField_ != false) { - output.writeBool(14, boolField_); - } - if (!bytesField_.isEmpty()) { - output.writeBytes(15, bytesField_); - } - if (testEnum_ != org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum.SHARED.getNumber()) { - output.writeEnum(16, testEnum_); - } - if (subMessage_ != null) { - output.writeMessage(17, getSubMessage()); - } - for (int i = 0; i < repeatedField_.size(); i++) { - com.google.protobuf.GeneratedMessageV3.writeString(output, 18, repeatedField_.getRaw(i)); - } - com.google.protobuf.GeneratedMessageV3 - .serializeStringMapTo( - output, - internalGetMapField(), - MapFieldDefaultEntryHolder.defaultEntry, - 19); - if (timestampField_ != null) { - output.writeMessage(20, getTimestampField()); - } - unknownFields.writeTo(output); - } - - @java.lang.Override - public int getSerializedSize() { - int size = memoizedSize; - if (size != -1) return size; - - size = 0; - if (!getStringFieldBytes().isEmpty()) { - size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, stringField_); - } - if (doubleField_ != 0D) { - size += com.google.protobuf.CodedOutputStream - .computeDoubleSize(2, doubleField_); - } - if (floatField_ != 0F) { - size += com.google.protobuf.CodedOutputStream - .computeFloatSize(3, floatField_); - } - if (int32Field_ != 0) { - size += com.google.protobuf.CodedOutputStream - .computeInt32Size(4, int32Field_); - } - if (int64Field_ != 0L) { - size += com.google.protobuf.CodedOutputStream - .computeInt64Size(5, int64Field_); - } - if (uint32Field_ != 0) { - size += com.google.protobuf.CodedOutputStream - .computeUInt32Size(6, uint32Field_); - } - if (uint64Field_ != 0L) { - size += com.google.protobuf.CodedOutputStream - .computeUInt64Size(7, uint64Field_); - } - if (sint32Field_ != 0) { - size += com.google.protobuf.CodedOutputStream - .computeSInt32Size(8, sint32Field_); - } - if (sint64Field_ != 0L) { - size += com.google.protobuf.CodedOutputStream - .computeSInt64Size(9, sint64Field_); - } - if (fixed32Field_ != 0) { - size += com.google.protobuf.CodedOutputStream - .computeFixed32Size(10, fixed32Field_); - } - if (fixed64Field_ != 0L) { - size += com.google.protobuf.CodedOutputStream - .computeFixed64Size(11, fixed64Field_); - } - if (sfixed32Field_ != 0) { - size += com.google.protobuf.CodedOutputStream - .computeSFixed32Size(12, sfixed32Field_); - } - if (sfixed64Field_ != 0L) { - size += com.google.protobuf.CodedOutputStream - .computeSFixed64Size(13, sfixed64Field_); - } - if (boolField_ != false) { - size += com.google.protobuf.CodedOutputStream - .computeBoolSize(14, boolField_); - } - if (!bytesField_.isEmpty()) { - size += com.google.protobuf.CodedOutputStream - .computeBytesSize(15, bytesField_); - } - if (testEnum_ != org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum.SHARED.getNumber()) { - size += com.google.protobuf.CodedOutputStream - .computeEnumSize(16, testEnum_); - } - if (subMessage_ != null) { - size += com.google.protobuf.CodedOutputStream - .computeMessageSize(17, getSubMessage()); - } - { - int dataSize = 0; - for (int i = 0; i < repeatedField_.size(); i++) { - dataSize += computeStringSizeNoTag(repeatedField_.getRaw(i)); - } - size += dataSize; - size += 2 * getRepeatedFieldList().size(); - } - for (java.util.Map.Entry entry - : internalGetMapField().getMap().entrySet()) { - com.google.protobuf.MapEntry - mapField__ = MapFieldDefaultEntryHolder.defaultEntry.newBuilderForType() - .setKey(entry.getKey()) - .setValue(entry.getValue()) - .build(); - size += com.google.protobuf.CodedOutputStream - .computeMessageSize(19, mapField__); - } - if (timestampField_ != null) { - size += com.google.protobuf.CodedOutputStream - .computeMessageSize(20, getTimestampField()); - } - size += unknownFields.getSerializedSize(); - memoizedSize = size; - return size; - } - - @java.lang.Override - public boolean equals(final java.lang.Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage)) { - return super.equals(obj); - } - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage other = (org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage) obj; - - if (!getStringField() - .equals(other.getStringField())) return false; - if (java.lang.Double.doubleToLongBits(getDoubleField()) - != java.lang.Double.doubleToLongBits( - other.getDoubleField())) return false; - if (java.lang.Float.floatToIntBits(getFloatField()) - != java.lang.Float.floatToIntBits( - other.getFloatField())) return false; - if (getInt32Field() - != other.getInt32Field()) return false; - if (getInt64Field() - != other.getInt64Field()) return false; - if (getUint32Field() - != other.getUint32Field()) return false; - if (getUint64Field() - != other.getUint64Field()) return false; - if (getSint32Field() - != other.getSint32Field()) return false; - if (getSint64Field() - != other.getSint64Field()) return false; - if (getFixed32Field() - != other.getFixed32Field()) return false; - if (getFixed64Field() - != other.getFixed64Field()) return false; - if (getSfixed32Field() - != other.getSfixed32Field()) return false; - if (getSfixed64Field() - != other.getSfixed64Field()) return false; - if (getBoolField() - != other.getBoolField()) return false; - if (!getBytesField() - .equals(other.getBytesField())) return false; - if (testEnum_ != other.testEnum_) return false; - if (hasSubMessage() != other.hasSubMessage()) return false; - if (hasSubMessage()) { - if (!getSubMessage() - .equals(other.getSubMessage())) return false; - } - if (!getRepeatedFieldList() - .equals(other.getRepeatedFieldList())) return false; - if (!internalGetMapField().equals( - other.internalGetMapField())) return false; - if (hasTimestampField() != other.hasTimestampField()) return false; - if (hasTimestampField()) { - if (!getTimestampField() - .equals(other.getTimestampField())) return false; - } - if (!unknownFields.equals(other.unknownFields)) return false; - return true; - } - - @java.lang.Override - public int hashCode() { - if (memoizedHashCode != 0) { - return memoizedHashCode; - } - int hash = 41; - hash = (19 * hash) + getDescriptor().hashCode(); - hash = (37 * hash) + STRINGFIELD_FIELD_NUMBER; - hash = (53 * hash) + getStringField().hashCode(); - hash = (37 * hash) + DOUBLEFIELD_FIELD_NUMBER; - hash = (53 * hash) + com.google.protobuf.Internal.hashLong( - java.lang.Double.doubleToLongBits(getDoubleField())); - hash = (37 * hash) + FLOATFIELD_FIELD_NUMBER; - hash = (53 * hash) + java.lang.Float.floatToIntBits( - getFloatField()); - hash = (37 * hash) + INT32FIELD_FIELD_NUMBER; - hash = (53 * hash) + getInt32Field(); - hash = (37 * hash) + INT64FIELD_FIELD_NUMBER; - hash = (53 * hash) + com.google.protobuf.Internal.hashLong( - getInt64Field()); - hash = (37 * hash) + UINT32FIELD_FIELD_NUMBER; - hash = (53 * hash) + getUint32Field(); - hash = (37 * hash) + UINT64FIELD_FIELD_NUMBER; - hash = (53 * hash) + com.google.protobuf.Internal.hashLong( - getUint64Field()); - hash = (37 * hash) + SINT32FIELD_FIELD_NUMBER; - hash = (53 * hash) + getSint32Field(); - hash = (37 * hash) + SINT64FIELD_FIELD_NUMBER; - hash = (53 * hash) + com.google.protobuf.Internal.hashLong( - getSint64Field()); - hash = (37 * hash) + FIXED32FIELD_FIELD_NUMBER; - hash = (53 * hash) + getFixed32Field(); - hash = (37 * hash) + FIXED64FIELD_FIELD_NUMBER; - hash = (53 * hash) + com.google.protobuf.Internal.hashLong( - getFixed64Field()); - hash = (37 * hash) + SFIXED32FIELD_FIELD_NUMBER; - hash = (53 * hash) + getSfixed32Field(); - hash = (37 * hash) + SFIXED64FIELD_FIELD_NUMBER; - hash = (53 * hash) + com.google.protobuf.Internal.hashLong( - getSfixed64Field()); - hash = (37 * hash) + BOOLFIELD_FIELD_NUMBER; - hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean( - getBoolField()); - hash = (37 * hash) + BYTESFIELD_FIELD_NUMBER; - hash = (53 * hash) + getBytesField().hashCode(); - hash = (37 * hash) + TESTENUM_FIELD_NUMBER; - hash = (53 * hash) + testEnum_; - if (hasSubMessage()) { - hash = (37 * hash) + SUBMESSAGE_FIELD_NUMBER; - hash = (53 * hash) + getSubMessage().hashCode(); - } - if (getRepeatedFieldCount() > 0) { - hash = (37 * hash) + REPEATEDFIELD_FIELD_NUMBER; - hash = (53 * hash) + getRepeatedFieldList().hashCode(); - } - if (!internalGetMapField().getMap().isEmpty()) { - hash = (37 * hash) + MAPFIELD_FIELD_NUMBER; - hash = (53 * hash) + internalGetMapField().hashCode(); - } - if (hasTimestampField()) { - hash = (37 * hash) + TIMESTAMPFIELD_FIELD_NUMBER; - hash = (53 * hash) + getTimestampField().hashCode(); - } - hash = (29 * hash) + unknownFields.hashCode(); - memoizedHashCode = hash; - return hash; - } - - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage parseFrom( - java.nio.ByteBuffer data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage parseFrom( - java.nio.ByteBuffer data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage parseFrom( - com.google.protobuf.ByteString data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage parseFrom( - com.google.protobuf.ByteString data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage parseFrom(byte[] data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage parseFrom( - byte[] data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage parseFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage parseFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage parseDelimitedFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage parseDelimitedFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input, extensionRegistry); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage parseFrom( - com.google.protobuf.CodedInputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage parseFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - - @java.lang.Override - public Builder newBuilderForType() { return newBuilder(); } - public static Builder newBuilder() { - return DEFAULT_INSTANCE.toBuilder(); - } - public static Builder newBuilder(org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage prototype) { - return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); - } - @java.lang.Override - public Builder toBuilder() { - return this == DEFAULT_INSTANCE - ? new Builder() : new Builder().mergeFrom(this); - } - - @java.lang.Override - protected Builder newBuilderForType( - com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { - Builder builder = new Builder(parent); - return builder; - } - /** - * Protobuf type {@code proto.TestMessage} - */ - public static final class Builder extends - com.google.protobuf.GeneratedMessageV3.Builder implements - // @@protoc_insertion_point(builder_implements:proto.TestMessage) - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessageOrBuilder { - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_TestMessage_descriptor; - } - - @SuppressWarnings({"rawtypes"}) - protected com.google.protobuf.MapField internalGetMapField( - int number) { - switch (number) { - case 19: - return internalGetMapField(); - default: - throw new RuntimeException( - "Invalid map field number: " + number); - } - } - @SuppressWarnings({"rawtypes"}) - protected com.google.protobuf.MapField internalGetMutableMapField( - int number) { - switch (number) { - case 19: - return internalGetMutableMapField(); - default: - throw new RuntimeException( - "Invalid map field number: " + number); - } - } - @java.lang.Override - protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_TestMessage_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage.class, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage.Builder.class); - } - - // Construct using org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage.newBuilder() - private Builder() { - maybeForceBuilderInitialization(); - } - - private Builder( - com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { - super(parent); - maybeForceBuilderInitialization(); - } - private void maybeForceBuilderInitialization() { - if (com.google.protobuf.GeneratedMessageV3 - .alwaysUseFieldBuilders) { - } - } - @java.lang.Override - public Builder clear() { - super.clear(); - stringField_ = ""; - - doubleField_ = 0D; - - floatField_ = 0F; - - int32Field_ = 0; - - int64Field_ = 0L; - - uint32Field_ = 0; - - uint64Field_ = 0L; - - sint32Field_ = 0; - - sint64Field_ = 0L; - - fixed32Field_ = 0; - - fixed64Field_ = 0L; - - sfixed32Field_ = 0; - - sfixed64Field_ = 0L; - - boolField_ = false; - - bytesField_ = com.google.protobuf.ByteString.EMPTY; - - testEnum_ = 0; - - if (subMessageBuilder_ == null) { - subMessage_ = null; - } else { - subMessage_ = null; - subMessageBuilder_ = null; - } - repeatedField_ = com.google.protobuf.LazyStringArrayList.EMPTY; - bitField0_ = (bitField0_ & ~0x00000001); - internalGetMutableMapField().clear(); - if (timestampFieldBuilder_ == null) { - timestampField_ = null; - } else { - timestampField_ = null; - timestampFieldBuilder_ = null; - } - return this; - } - - @java.lang.Override - public com.google.protobuf.Descriptors.Descriptor - getDescriptorForType() { - return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.internal_static_proto_TestMessage_descriptor; - } - - @java.lang.Override - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage getDefaultInstanceForType() { - return org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage.getDefaultInstance(); - } - - @java.lang.Override - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage build() { - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException(result); - } - return result; - } - - @java.lang.Override - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage buildPartial() { - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage result = new org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage(this); - int from_bitField0_ = bitField0_; - result.stringField_ = stringField_; - result.doubleField_ = doubleField_; - result.floatField_ = floatField_; - result.int32Field_ = int32Field_; - result.int64Field_ = int64Field_; - result.uint32Field_ = uint32Field_; - result.uint64Field_ = uint64Field_; - result.sint32Field_ = sint32Field_; - result.sint64Field_ = sint64Field_; - result.fixed32Field_ = fixed32Field_; - result.fixed64Field_ = fixed64Field_; - result.sfixed32Field_ = sfixed32Field_; - result.sfixed64Field_ = sfixed64Field_; - result.boolField_ = boolField_; - result.bytesField_ = bytesField_; - result.testEnum_ = testEnum_; - if (subMessageBuilder_ == null) { - result.subMessage_ = subMessage_; - } else { - result.subMessage_ = subMessageBuilder_.build(); - } - if (((bitField0_ & 0x00000001) != 0)) { - repeatedField_ = repeatedField_.getUnmodifiableView(); - bitField0_ = (bitField0_ & ~0x00000001); - } - result.repeatedField_ = repeatedField_; - result.mapField_ = internalGetMapField(); - result.mapField_.makeImmutable(); - if (timestampFieldBuilder_ == null) { - result.timestampField_ = timestampField_; - } else { - result.timestampField_ = timestampFieldBuilder_.build(); - } - onBuilt(); - return result; - } - - @java.lang.Override - public Builder clone() { - return super.clone(); - } - @java.lang.Override - public Builder setField( - com.google.protobuf.Descriptors.FieldDescriptor field, - java.lang.Object value) { - return super.setField(field, value); - } - @java.lang.Override - public Builder clearField( - com.google.protobuf.Descriptors.FieldDescriptor field) { - return super.clearField(field); - } - @java.lang.Override - public Builder clearOneof( - com.google.protobuf.Descriptors.OneofDescriptor oneof) { - return super.clearOneof(oneof); - } - @java.lang.Override - public Builder setRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - int index, java.lang.Object value) { - return super.setRepeatedField(field, index, value); - } - @java.lang.Override - public Builder addRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - java.lang.Object value) { - return super.addRepeatedField(field, value); - } - @java.lang.Override - public Builder mergeFrom(com.google.protobuf.Message other) { - if (other instanceof org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage) { - return mergeFrom((org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage)other); - } else { - super.mergeFrom(other); - return this; - } - } - - public Builder mergeFrom(org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage other) { - if (other == org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage.getDefaultInstance()) return this; - if (!other.getStringField().isEmpty()) { - stringField_ = other.stringField_; - onChanged(); - } - if (other.getDoubleField() != 0D) { - setDoubleField(other.getDoubleField()); - } - if (other.getFloatField() != 0F) { - setFloatField(other.getFloatField()); - } - if (other.getInt32Field() != 0) { - setInt32Field(other.getInt32Field()); - } - if (other.getInt64Field() != 0L) { - setInt64Field(other.getInt64Field()); - } - if (other.getUint32Field() != 0) { - setUint32Field(other.getUint32Field()); - } - if (other.getUint64Field() != 0L) { - setUint64Field(other.getUint64Field()); - } - if (other.getSint32Field() != 0) { - setSint32Field(other.getSint32Field()); - } - if (other.getSint64Field() != 0L) { - setSint64Field(other.getSint64Field()); - } - if (other.getFixed32Field() != 0) { - setFixed32Field(other.getFixed32Field()); - } - if (other.getFixed64Field() != 0L) { - setFixed64Field(other.getFixed64Field()); - } - if (other.getSfixed32Field() != 0) { - setSfixed32Field(other.getSfixed32Field()); - } - if (other.getSfixed64Field() != 0L) { - setSfixed64Field(other.getSfixed64Field()); - } - if (other.getBoolField() != false) { - setBoolField(other.getBoolField()); - } - if (other.getBytesField() != com.google.protobuf.ByteString.EMPTY) { - setBytesField(other.getBytesField()); - } - if (other.testEnum_ != 0) { - setTestEnumValue(other.getTestEnumValue()); - } - if (other.hasSubMessage()) { - mergeSubMessage(other.getSubMessage()); - } - if (!other.repeatedField_.isEmpty()) { - if (repeatedField_.isEmpty()) { - repeatedField_ = other.repeatedField_; - bitField0_ = (bitField0_ & ~0x00000001); - } else { - ensureRepeatedFieldIsMutable(); - repeatedField_.addAll(other.repeatedField_); - } - onChanged(); - } - internalGetMutableMapField().mergeFrom( - other.internalGetMapField()); - if (other.hasTimestampField()) { - mergeTimestampField(other.getTimestampField()); - } - this.mergeUnknownFields(other.unknownFields); - onChanged(); - return this; - } - - @java.lang.Override - public final boolean isInitialized() { - return true; - } - - @java.lang.Override - public Builder mergeFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage parsedMessage = null; - try { - parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - parsedMessage = (org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage) e.getUnfinishedMessage(); - throw e.unwrapIOException(); - } finally { - if (parsedMessage != null) { - mergeFrom(parsedMessage); - } - } - return this; - } - private int bitField0_; - - private java.lang.Object stringField_ = ""; - /** - * string stringField = 1; - * @return The stringField. - */ - public java.lang.String getStringField() { - java.lang.Object ref = stringField_; - if (!(ref instanceof java.lang.String)) { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - java.lang.String s = bs.toStringUtf8(); - stringField_ = s; - return s; - } else { - return (java.lang.String) ref; - } - } - /** - * string stringField = 1; - * @return The bytes for stringField. - */ - public com.google.protobuf.ByteString - getStringFieldBytes() { - java.lang.Object ref = stringField_; - if (ref instanceof String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (java.lang.String) ref); - stringField_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - /** - * string stringField = 1; - * @param value The stringField to set. - * @return This builder for chaining. - */ - public Builder setStringField( - java.lang.String value) { - if (value == null) { - throw new NullPointerException(); - } - - stringField_ = value; - onChanged(); - return this; - } - /** - * string stringField = 1; - * @return This builder for chaining. - */ - public Builder clearStringField() { - - stringField_ = getDefaultInstance().getStringField(); - onChanged(); - return this; - } - /** - * string stringField = 1; - * @param value The bytes for stringField to set. - * @return This builder for chaining. - */ - public Builder setStringFieldBytes( - com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - checkByteStringIsUtf8(value); - - stringField_ = value; - onChanged(); - return this; - } - - private double doubleField_ ; - /** - * double doubleField = 2; - * @return The doubleField. - */ - @java.lang.Override - public double getDoubleField() { - return doubleField_; - } - /** - * double doubleField = 2; - * @param value The doubleField to set. - * @return This builder for chaining. - */ - public Builder setDoubleField(double value) { - - doubleField_ = value; - onChanged(); - return this; - } - /** - * double doubleField = 2; - * @return This builder for chaining. - */ - public Builder clearDoubleField() { - - doubleField_ = 0D; - onChanged(); - return this; - } - - private float floatField_ ; - /** - * float floatField = 3; - * @return The floatField. - */ - @java.lang.Override - public float getFloatField() { - return floatField_; - } - /** - * float floatField = 3; - * @param value The floatField to set. - * @return This builder for chaining. - */ - public Builder setFloatField(float value) { - - floatField_ = value; - onChanged(); - return this; - } - /** - * float floatField = 3; - * @return This builder for chaining. - */ - public Builder clearFloatField() { - - floatField_ = 0F; - onChanged(); - return this; - } - - private int int32Field_ ; - /** - * int32 int32Field = 4; - * @return The int32Field. - */ - @java.lang.Override - public int getInt32Field() { - return int32Field_; - } - /** - * int32 int32Field = 4; - * @param value The int32Field to set. - * @return This builder for chaining. - */ - public Builder setInt32Field(int value) { - - int32Field_ = value; - onChanged(); - return this; - } - /** - * int32 int32Field = 4; - * @return This builder for chaining. - */ - public Builder clearInt32Field() { - - int32Field_ = 0; - onChanged(); - return this; - } - - private long int64Field_ ; - /** - * int64 int64Field = 5; - * @return The int64Field. - */ - @java.lang.Override - public long getInt64Field() { - return int64Field_; - } - /** - * int64 int64Field = 5; - * @param value The int64Field to set. - * @return This builder for chaining. - */ - public Builder setInt64Field(long value) { - - int64Field_ = value; - onChanged(); - return this; - } - /** - * int64 int64Field = 5; - * @return This builder for chaining. - */ - public Builder clearInt64Field() { - - int64Field_ = 0L; - onChanged(); - return this; - } - - private int uint32Field_ ; - /** - * uint32 uint32Field = 6; - * @return The uint32Field. - */ - @java.lang.Override - public int getUint32Field() { - return uint32Field_; - } - /** - * uint32 uint32Field = 6; - * @param value The uint32Field to set. - * @return This builder for chaining. - */ - public Builder setUint32Field(int value) { - - uint32Field_ = value; - onChanged(); - return this; - } - /** - * uint32 uint32Field = 6; - * @return This builder for chaining. - */ - public Builder clearUint32Field() { - - uint32Field_ = 0; - onChanged(); - return this; - } - - private long uint64Field_ ; - /** - * uint64 uint64Field = 7; - * @return The uint64Field. - */ - @java.lang.Override - public long getUint64Field() { - return uint64Field_; - } - /** - * uint64 uint64Field = 7; - * @param value The uint64Field to set. - * @return This builder for chaining. - */ - public Builder setUint64Field(long value) { - - uint64Field_ = value; - onChanged(); - return this; - } - /** - * uint64 uint64Field = 7; - * @return This builder for chaining. - */ - public Builder clearUint64Field() { - - uint64Field_ = 0L; - onChanged(); - return this; - } - - private int sint32Field_ ; - /** - * sint32 sint32Field = 8; - * @return The sint32Field. - */ - @java.lang.Override - public int getSint32Field() { - return sint32Field_; - } - /** - * sint32 sint32Field = 8; - * @param value The sint32Field to set. - * @return This builder for chaining. - */ - public Builder setSint32Field(int value) { - - sint32Field_ = value; - onChanged(); - return this; - } - /** - * sint32 sint32Field = 8; - * @return This builder for chaining. - */ - public Builder clearSint32Field() { - - sint32Field_ = 0; - onChanged(); - return this; - } - - private long sint64Field_ ; - /** - * sint64 sint64Field = 9; - * @return The sint64Field. - */ - @java.lang.Override - public long getSint64Field() { - return sint64Field_; - } - /** - * sint64 sint64Field = 9; - * @param value The sint64Field to set. - * @return This builder for chaining. - */ - public Builder setSint64Field(long value) { - - sint64Field_ = value; - onChanged(); - return this; - } - /** - * sint64 sint64Field = 9; - * @return This builder for chaining. - */ - public Builder clearSint64Field() { - - sint64Field_ = 0L; - onChanged(); - return this; - } - - private int fixed32Field_ ; - /** - * fixed32 fixed32Field = 10; - * @return The fixed32Field. - */ - @java.lang.Override - public int getFixed32Field() { - return fixed32Field_; - } - /** - * fixed32 fixed32Field = 10; - * @param value The fixed32Field to set. - * @return This builder for chaining. - */ - public Builder setFixed32Field(int value) { - - fixed32Field_ = value; - onChanged(); - return this; - } - /** - * fixed32 fixed32Field = 10; - * @return This builder for chaining. - */ - public Builder clearFixed32Field() { - - fixed32Field_ = 0; - onChanged(); - return this; - } - - private long fixed64Field_ ; - /** - * fixed64 fixed64Field = 11; - * @return The fixed64Field. - */ - @java.lang.Override - public long getFixed64Field() { - return fixed64Field_; - } - /** - * fixed64 fixed64Field = 11; - * @param value The fixed64Field to set. - * @return This builder for chaining. - */ - public Builder setFixed64Field(long value) { - - fixed64Field_ = value; - onChanged(); - return this; - } - /** - * fixed64 fixed64Field = 11; - * @return This builder for chaining. - */ - public Builder clearFixed64Field() { - - fixed64Field_ = 0L; - onChanged(); - return this; - } - - private int sfixed32Field_ ; - /** - * sfixed32 sfixed32Field = 12; - * @return The sfixed32Field. - */ - @java.lang.Override - public int getSfixed32Field() { - return sfixed32Field_; - } - /** - * sfixed32 sfixed32Field = 12; - * @param value The sfixed32Field to set. - * @return This builder for chaining. - */ - public Builder setSfixed32Field(int value) { - - sfixed32Field_ = value; - onChanged(); - return this; - } - /** - * sfixed32 sfixed32Field = 12; - * @return This builder for chaining. - */ - public Builder clearSfixed32Field() { - - sfixed32Field_ = 0; - onChanged(); - return this; - } - - private long sfixed64Field_ ; - /** - * sfixed64 sfixed64Field = 13; - * @return The sfixed64Field. - */ - @java.lang.Override - public long getSfixed64Field() { - return sfixed64Field_; - } - /** - * sfixed64 sfixed64Field = 13; - * @param value The sfixed64Field to set. - * @return This builder for chaining. - */ - public Builder setSfixed64Field(long value) { - - sfixed64Field_ = value; - onChanged(); - return this; - } - /** - * sfixed64 sfixed64Field = 13; - * @return This builder for chaining. - */ - public Builder clearSfixed64Field() { - - sfixed64Field_ = 0L; - onChanged(); - return this; - } - - private boolean boolField_ ; - /** - * bool boolField = 14; - * @return The boolField. - */ - @java.lang.Override - public boolean getBoolField() { - return boolField_; - } - /** - * bool boolField = 14; - * @param value The boolField to set. - * @return This builder for chaining. - */ - public Builder setBoolField(boolean value) { - - boolField_ = value; - onChanged(); - return this; - } - /** - * bool boolField = 14; - * @return This builder for chaining. - */ - public Builder clearBoolField() { - - boolField_ = false; - onChanged(); - return this; - } - - private com.google.protobuf.ByteString bytesField_ = com.google.protobuf.ByteString.EMPTY; - /** - * bytes bytesField = 15; - * @return The bytesField. - */ - @java.lang.Override - public com.google.protobuf.ByteString getBytesField() { - return bytesField_; - } - /** - * bytes bytesField = 15; - * @param value The bytesField to set. - * @return This builder for chaining. - */ - public Builder setBytesField(com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - - bytesField_ = value; - onChanged(); - return this; - } - /** - * bytes bytesField = 15; - * @return This builder for chaining. - */ - public Builder clearBytesField() { - - bytesField_ = getDefaultInstance().getBytesField(); - onChanged(); - return this; - } - - private int testEnum_ = 0; - /** - * .proto.TestEnum testEnum = 16; - * @return The enum numeric value on the wire for testEnum. - */ - @java.lang.Override public int getTestEnumValue() { - return testEnum_; - } - /** - * .proto.TestEnum testEnum = 16; - * @param value The enum numeric value on the wire for testEnum to set. - * @return This builder for chaining. - */ - public Builder setTestEnumValue(int value) { - - testEnum_ = value; - onChanged(); - return this; - } - /** - * .proto.TestEnum testEnum = 16; - * @return The testEnum. - */ - @java.lang.Override - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum getTestEnum() { - @SuppressWarnings("deprecation") - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum result = org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum.valueOf(testEnum_); - return result == null ? org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum.UNRECOGNIZED : result; - } - /** - * .proto.TestEnum testEnum = 16; - * @param value The testEnum to set. - * @return This builder for chaining. - */ - public Builder setTestEnum(org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestEnum value) { - if (value == null) { - throw new NullPointerException(); - } - - testEnum_ = value.getNumber(); - onChanged(); - return this; - } - /** - * .proto.TestEnum testEnum = 16; - * @return This builder for chaining. - */ - public Builder clearTestEnum() { - - testEnum_ = 0; - onChanged(); - return this; - } - - private org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage subMessage_; - private com.google.protobuf.SingleFieldBuilderV3< - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.Builder, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessageOrBuilder> subMessageBuilder_; - /** - * .proto.SubMessage subMessage = 17; - * @return Whether the subMessage field is set. - */ - public boolean hasSubMessage() { - return subMessageBuilder_ != null || subMessage_ != null; - } - /** - * .proto.SubMessage subMessage = 17; - * @return The subMessage. - */ - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage getSubMessage() { - if (subMessageBuilder_ == null) { - return subMessage_ == null ? org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.getDefaultInstance() : subMessage_; - } else { - return subMessageBuilder_.getMessage(); - } - } - /** - * .proto.SubMessage subMessage = 17; - */ - public Builder setSubMessage(org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage value) { - if (subMessageBuilder_ == null) { - if (value == null) { - throw new NullPointerException(); - } - subMessage_ = value; - onChanged(); - } else { - subMessageBuilder_.setMessage(value); - } - - return this; - } - /** - * .proto.SubMessage subMessage = 17; - */ - public Builder setSubMessage( - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.Builder builderForValue) { - if (subMessageBuilder_ == null) { - subMessage_ = builderForValue.build(); - onChanged(); - } else { - subMessageBuilder_.setMessage(builderForValue.build()); - } - - return this; - } - /** - * .proto.SubMessage subMessage = 17; - */ - public Builder mergeSubMessage(org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage value) { - if (subMessageBuilder_ == null) { - if (subMessage_ != null) { - subMessage_ = - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.newBuilder(subMessage_).mergeFrom(value).buildPartial(); - } else { - subMessage_ = value; - } - onChanged(); - } else { - subMessageBuilder_.mergeFrom(value); - } - - return this; - } - /** - * .proto.SubMessage subMessage = 17; - */ - public Builder clearSubMessage() { - if (subMessageBuilder_ == null) { - subMessage_ = null; - onChanged(); - } else { - subMessage_ = null; - subMessageBuilder_ = null; - } - - return this; - } - /** - * .proto.SubMessage subMessage = 17; - */ - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.Builder getSubMessageBuilder() { - - onChanged(); - return getSubMessageFieldBuilder().getBuilder(); - } - /** - * .proto.SubMessage subMessage = 17; - */ - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessageOrBuilder getSubMessageOrBuilder() { - if (subMessageBuilder_ != null) { - return subMessageBuilder_.getMessageOrBuilder(); - } else { - return subMessage_ == null ? - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.getDefaultInstance() : subMessage_; - } - } - /** - * .proto.SubMessage subMessage = 17; - */ - private com.google.protobuf.SingleFieldBuilderV3< - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.Builder, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessageOrBuilder> - getSubMessageFieldBuilder() { - if (subMessageBuilder_ == null) { - subMessageBuilder_ = new com.google.protobuf.SingleFieldBuilderV3< - org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessage.Builder, org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.SubMessageOrBuilder>( - getSubMessage(), - getParentForChildren(), - isClean()); - subMessage_ = null; - } - return subMessageBuilder_; - } - - private com.google.protobuf.LazyStringList repeatedField_ = com.google.protobuf.LazyStringArrayList.EMPTY; - private void ensureRepeatedFieldIsMutable() { - if (!((bitField0_ & 0x00000001) != 0)) { - repeatedField_ = new com.google.protobuf.LazyStringArrayList(repeatedField_); - bitField0_ |= 0x00000001; - } - } - /** - * repeated string repeatedField = 18; - * @return A list containing the repeatedField. - */ - public com.google.protobuf.ProtocolStringList - getRepeatedFieldList() { - return repeatedField_.getUnmodifiableView(); - } - /** - * repeated string repeatedField = 18; - * @return The count of repeatedField. - */ - public int getRepeatedFieldCount() { - return repeatedField_.size(); - } - /** - * repeated string repeatedField = 18; - * @param index The index of the element to return. - * @return The repeatedField at the given index. - */ - public java.lang.String getRepeatedField(int index) { - return repeatedField_.get(index); - } - /** - * repeated string repeatedField = 18; - * @param index The index of the value to return. - * @return The bytes of the repeatedField at the given index. - */ - public com.google.protobuf.ByteString - getRepeatedFieldBytes(int index) { - return repeatedField_.getByteString(index); - } - /** - * repeated string repeatedField = 18; - * @param index The index to set the value at. - * @param value The repeatedField to set. - * @return This builder for chaining. - */ - public Builder setRepeatedField( - int index, java.lang.String value) { - if (value == null) { - throw new NullPointerException(); - } - ensureRepeatedFieldIsMutable(); - repeatedField_.set(index, value); - onChanged(); - return this; - } - /** - * repeated string repeatedField = 18; - * @param value The repeatedField to add. - * @return This builder for chaining. - */ - public Builder addRepeatedField( - java.lang.String value) { - if (value == null) { - throw new NullPointerException(); - } - ensureRepeatedFieldIsMutable(); - repeatedField_.add(value); - onChanged(); - return this; - } - /** - * repeated string repeatedField = 18; - * @param values The repeatedField to add. - * @return This builder for chaining. - */ - public Builder addAllRepeatedField( - java.lang.Iterable values) { - ensureRepeatedFieldIsMutable(); - com.google.protobuf.AbstractMessageLite.Builder.addAll( - values, repeatedField_); - onChanged(); - return this; - } - /** - * repeated string repeatedField = 18; - * @return This builder for chaining. - */ - public Builder clearRepeatedField() { - repeatedField_ = com.google.protobuf.LazyStringArrayList.EMPTY; - bitField0_ = (bitField0_ & ~0x00000001); - onChanged(); - return this; - } - /** - * repeated string repeatedField = 18; - * @param value The bytes of the repeatedField to add. - * @return This builder for chaining. - */ - public Builder addRepeatedFieldBytes( - com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - checkByteStringIsUtf8(value); - ensureRepeatedFieldIsMutable(); - repeatedField_.add(value); - onChanged(); - return this; - } - - private com.google.protobuf.MapField< - java.lang.String, java.lang.Double> mapField_; - private com.google.protobuf.MapField - internalGetMapField() { - if (mapField_ == null) { - return com.google.protobuf.MapField.emptyMapField( - MapFieldDefaultEntryHolder.defaultEntry); - } - return mapField_; - } - private com.google.protobuf.MapField - internalGetMutableMapField() { - onChanged();; - if (mapField_ == null) { - mapField_ = com.google.protobuf.MapField.newMapField( - MapFieldDefaultEntryHolder.defaultEntry); - } - if (!mapField_.isMutable()) { - mapField_ = mapField_.copy(); - } - return mapField_; - } - - public int getMapFieldCount() { - return internalGetMapField().getMap().size(); - } - /** - * map<string, double> mapField = 19; - */ - - @java.lang.Override - public boolean containsMapField( - java.lang.String key) { - if (key == null) { throw new java.lang.NullPointerException(); } - return internalGetMapField().getMap().containsKey(key); - } - /** - * Use {@link #getMapFieldMap()} instead. - */ - @java.lang.Override - @java.lang.Deprecated - public java.util.Map getMapField() { - return getMapFieldMap(); - } - /** - * map<string, double> mapField = 19; - */ - @java.lang.Override - - public java.util.Map getMapFieldMap() { - return internalGetMapField().getMap(); - } - /** - * map<string, double> mapField = 19; - */ - @java.lang.Override - - public double getMapFieldOrDefault( - java.lang.String key, - double defaultValue) { - if (key == null) { throw new java.lang.NullPointerException(); } - java.util.Map map = - internalGetMapField().getMap(); - return map.containsKey(key) ? map.get(key) : defaultValue; - } - /** - * map<string, double> mapField = 19; - */ - @java.lang.Override - - public double getMapFieldOrThrow( - java.lang.String key) { - if (key == null) { throw new java.lang.NullPointerException(); } - java.util.Map map = - internalGetMapField().getMap(); - if (!map.containsKey(key)) { - throw new java.lang.IllegalArgumentException(); - } - return map.get(key); - } - - public Builder clearMapField() { - internalGetMutableMapField().getMutableMap() - .clear(); - return this; - } - /** - * map<string, double> mapField = 19; - */ - - public Builder removeMapField( - java.lang.String key) { - if (key == null) { throw new java.lang.NullPointerException(); } - internalGetMutableMapField().getMutableMap() - .remove(key); - return this; - } - /** - * Use alternate mutation accessors instead. - */ - @java.lang.Deprecated - public java.util.Map - getMutableMapField() { - return internalGetMutableMapField().getMutableMap(); - } - /** - * map<string, double> mapField = 19; - */ - public Builder putMapField( - java.lang.String key, - double value) { - if (key == null) { throw new java.lang.NullPointerException(); } - - internalGetMutableMapField().getMutableMap() - .put(key, value); - return this; - } - /** - * map<string, double> mapField = 19; - */ - - public Builder putAllMapField( - java.util.Map values) { - internalGetMutableMapField().getMutableMap() - .putAll(values); - return this; - } - - private com.google.protobuf.Timestamp timestampField_; - private com.google.protobuf.SingleFieldBuilderV3< - com.google.protobuf.Timestamp, com.google.protobuf.Timestamp.Builder, com.google.protobuf.TimestampOrBuilder> timestampFieldBuilder_; - /** - * .google.protobuf.Timestamp timestampField = 20; - * @return Whether the timestampField field is set. - */ - public boolean hasTimestampField() { - return timestampFieldBuilder_ != null || timestampField_ != null; - } - /** - * .google.protobuf.Timestamp timestampField = 20; - * @return The timestampField. - */ - public com.google.protobuf.Timestamp getTimestampField() { - if (timestampFieldBuilder_ == null) { - return timestampField_ == null ? com.google.protobuf.Timestamp.getDefaultInstance() : timestampField_; - } else { - return timestampFieldBuilder_.getMessage(); - } - } - /** - * .google.protobuf.Timestamp timestampField = 20; - */ - public Builder setTimestampField(com.google.protobuf.Timestamp value) { - if (timestampFieldBuilder_ == null) { - if (value == null) { - throw new NullPointerException(); - } - timestampField_ = value; - onChanged(); - } else { - timestampFieldBuilder_.setMessage(value); - } - - return this; - } - /** - * .google.protobuf.Timestamp timestampField = 20; - */ - public Builder setTimestampField( - com.google.protobuf.Timestamp.Builder builderForValue) { - if (timestampFieldBuilder_ == null) { - timestampField_ = builderForValue.build(); - onChanged(); - } else { - timestampFieldBuilder_.setMessage(builderForValue.build()); - } - - return this; - } - /** - * .google.protobuf.Timestamp timestampField = 20; - */ - public Builder mergeTimestampField(com.google.protobuf.Timestamp value) { - if (timestampFieldBuilder_ == null) { - if (timestampField_ != null) { - timestampField_ = - com.google.protobuf.Timestamp.newBuilder(timestampField_).mergeFrom(value).buildPartial(); - } else { - timestampField_ = value; - } - onChanged(); - } else { - timestampFieldBuilder_.mergeFrom(value); - } - - return this; - } - /** - * .google.protobuf.Timestamp timestampField = 20; - */ - public Builder clearTimestampField() { - if (timestampFieldBuilder_ == null) { - timestampField_ = null; - onChanged(); - } else { - timestampField_ = null; - timestampFieldBuilder_ = null; - } - - return this; - } - /** - * .google.protobuf.Timestamp timestampField = 20; - */ - public com.google.protobuf.Timestamp.Builder getTimestampFieldBuilder() { - - onChanged(); - return getTimestampFieldFieldBuilder().getBuilder(); - } - /** - * .google.protobuf.Timestamp timestampField = 20; - */ - public com.google.protobuf.TimestampOrBuilder getTimestampFieldOrBuilder() { - if (timestampFieldBuilder_ != null) { - return timestampFieldBuilder_.getMessageOrBuilder(); - } else { - return timestampField_ == null ? - com.google.protobuf.Timestamp.getDefaultInstance() : timestampField_; - } - } - /** - * .google.protobuf.Timestamp timestampField = 20; - */ - private com.google.protobuf.SingleFieldBuilderV3< - com.google.protobuf.Timestamp, com.google.protobuf.Timestamp.Builder, com.google.protobuf.TimestampOrBuilder> - getTimestampFieldFieldBuilder() { - if (timestampFieldBuilder_ == null) { - timestampFieldBuilder_ = new com.google.protobuf.SingleFieldBuilderV3< - com.google.protobuf.Timestamp, com.google.protobuf.Timestamp.Builder, com.google.protobuf.TimestampOrBuilder>( - getTimestampField(), - getParentForChildren(), - isClean()); - timestampField_ = null; - } - return timestampFieldBuilder_; - } - @java.lang.Override - public final Builder setUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.setUnknownFields(unknownFields); - } - - @java.lang.Override - public final Builder mergeUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.mergeUnknownFields(unknownFields); - } - - - // @@protoc_insertion_point(builder_scope:proto.TestMessage) - } - - // @@protoc_insertion_point(class_scope:proto.TestMessage) - private static final org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage DEFAULT_INSTANCE; - static { - DEFAULT_INSTANCE = new org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage(); - } - - public static org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage getDefaultInstance() { - return DEFAULT_INSTANCE; - } - - private static final com.google.protobuf.Parser - PARSER = new com.google.protobuf.AbstractParser() { - @java.lang.Override - public TestMessage parsePartialFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return new TestMessage(input, extensionRegistry); - } - }; - - public static com.google.protobuf.Parser parser() { - return PARSER; - } - - @java.lang.Override - public com.google.protobuf.Parser getParserForType() { - return PARSER; - } - - @java.lang.Override - public org.apache.pulsar.sql.presto.decoder.protobufnative.TestMsg.TestMessage getDefaultInstanceForType() { - return DEFAULT_INSTANCE; - } - - } - - private static final com.google.protobuf.Descriptors.Descriptor - internal_static_proto_SubMessage_descriptor; - private static final - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internal_static_proto_SubMessage_fieldAccessorTable; - private static final com.google.protobuf.Descriptors.Descriptor - internal_static_proto_SubMessage_NestedMessage_descriptor; - private static final - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internal_static_proto_SubMessage_NestedMessage_fieldAccessorTable; - private static final com.google.protobuf.Descriptors.Descriptor - internal_static_proto_TestMessage_descriptor; - private static final - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internal_static_proto_TestMessage_fieldAccessorTable; - private static final com.google.protobuf.Descriptors.Descriptor - internal_static_proto_TestMessage_MapFieldEntry_descriptor; - private static final - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internal_static_proto_TestMessage_MapFieldEntry_fieldAccessorTable; - - public static com.google.protobuf.Descriptors.FileDescriptor - getDescriptor() { - return descriptor; - } - private static com.google.protobuf.Descriptors.FileDescriptor - descriptor; - static { - java.lang.String[] descriptorData = { - "\n\rTestMsg.proto\022\005proto\032\037google/protobuf/" + - "timestamp.proto\"\214\001\n\nSubMessage\022\013\n\003foo\030\001 " + - "\001(\t\022\013\n\003bar\030\002 \001(\001\0226\n\rnestedMessage\030\003 \001(\0132" + - "\037.proto.SubMessage.NestedMessage\032,\n\rNest" + - "edMessage\022\r\n\005title\030\001 \001(\t\022\014\n\004urls\030\002 \003(\t\"\302" + - "\004\n\013TestMessage\022\023\n\013stringField\030\001 \001(\t\022\023\n\013d" + - "oubleField\030\002 \001(\001\022\022\n\nfloatField\030\003 \001(\002\022\022\n\n" + - "int32Field\030\004 \001(\005\022\022\n\nint64Field\030\005 \001(\003\022\023\n\013" + - "uint32Field\030\006 \001(\r\022\023\n\013uint64Field\030\007 \001(\004\022\023" + - "\n\013sint32Field\030\010 \001(\021\022\023\n\013sint64Field\030\t \001(\022" + - "\022\024\n\014fixed32Field\030\n \001(\007\022\024\n\014fixed64Field\030\013" + - " \001(\006\022\025\n\rsfixed32Field\030\014 \001(\017\022\025\n\rsfixed64F" + - "ield\030\r \001(\020\022\021\n\tboolField\030\016 \001(\010\022\022\n\nbytesFi" + - "eld\030\017 \001(\014\022!\n\010testEnum\030\020 \001(\0162\017.proto.Test" + - "Enum\022%\n\nsubMessage\030\021 \001(\0132\021.proto.SubMess" + - "age\022\025\n\rrepeatedField\030\022 \003(\t\0222\n\010mapField\030\023" + - " \003(\0132 .proto.TestMessage.MapFieldEntry\0222" + - "\n\016timestampField\030\024 \001(\0132\032.google.protobuf" + - ".Timestamp\032/\n\rMapFieldEntry\022\013\n\003key\030\001 \001(\t" + - "\022\r\n\005value\030\002 \001(\001:\0028\001*$\n\010TestEnum\022\n\n\006SHARE" + - "D\020\000\022\014\n\010FAILOVER\020\001B>\n3org.apache.pulsar.s" + - "ql.presto.decoder.protobufnativeB\007TestMs" + - "gP\000b\006proto3" - }; - descriptor = com.google.protobuf.Descriptors.FileDescriptor - .internalBuildGeneratedFileFrom(descriptorData, - new com.google.protobuf.Descriptors.FileDescriptor[] { - com.google.protobuf.TimestampProto.getDescriptor(), - }); - internal_static_proto_SubMessage_descriptor = - getDescriptor().getMessageTypes().get(0); - internal_static_proto_SubMessage_fieldAccessorTable = new - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( - internal_static_proto_SubMessage_descriptor, - new java.lang.String[] { "Foo", "Bar", "NestedMessage", }); - internal_static_proto_SubMessage_NestedMessage_descriptor = - internal_static_proto_SubMessage_descriptor.getNestedTypes().get(0); - internal_static_proto_SubMessage_NestedMessage_fieldAccessorTable = new - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( - internal_static_proto_SubMessage_NestedMessage_descriptor, - new java.lang.String[] { "Title", "Urls", }); - internal_static_proto_TestMessage_descriptor = - getDescriptor().getMessageTypes().get(1); - internal_static_proto_TestMessage_fieldAccessorTable = new - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( - internal_static_proto_TestMessage_descriptor, - new java.lang.String[] { "StringField", "DoubleField", "FloatField", "Int32Field", "Int64Field", "Uint32Field", "Uint64Field", "Sint32Field", "Sint64Field", "Fixed32Field", "Fixed64Field", "Sfixed32Field", "Sfixed64Field", "BoolField", "BytesField", "TestEnum", "SubMessage", "RepeatedField", "MapField", "TimestampField", }); - internal_static_proto_TestMessage_MapFieldEntry_descriptor = - internal_static_proto_TestMessage_descriptor.getNestedTypes().get(0); - internal_static_proto_TestMessage_MapFieldEntry_fieldAccessorTable = new - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( - internal_static_proto_TestMessage_MapFieldEntry_descriptor, - new java.lang.String[] { "Key", "Value", }); - com.google.protobuf.TimestampProto.getDescriptor(); - } - - // @@protoc_insertion_point(outer_class_scope) -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/TestMsg.proto b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/TestMsg.proto deleted file mode 100644 index fd522bdd62652..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/TestMsg.proto +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -syntax = "proto3"; -package proto; - -import public "google/protobuf/timestamp.proto"; - -option java_package = "org.apache.pulsar.sql.presto.decoder.protobufnative"; -option java_outer_classname = "TestMsg"; - -enum TestEnum { - SHARED = 0; - FAILOVER = 1; -} - -message SubMessage { - string foo = 1; - double bar = 2; - NestedMessage nestedMessage = 3; - message NestedMessage { - string title = 1; - repeated string urls = 2; - } -} - -message TestMessage { - string stringField = 1; - double doubleField = 2; - float floatField = 3; - int32 int32Field = 4; - int64 int64Field = 5; - uint32 uint32Field = 6; - uint64 uint64Field = 7; - sint32 sint32Field = 8; - sint64 sint64Field = 9; - fixed32 fixed32Field = 10; - fixed64 fixed64Field = 11; - sfixed32 sfixed32Field = 12; - sfixed64 sfixed64Field = 13; - bool boolField = 14; - bytes bytesField = 15; - TestEnum testEnum = 16; - SubMessage subMessage = 17; - repeated string repeatedField = 18; - map mapField = 19; - google.protobuf.Timestamp timestampField = 20; -} \ No newline at end of file diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/TestProtobufNativeDecoder.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/TestProtobufNativeDecoder.java deleted file mode 100644 index 9b2cb32de0aa3..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/protobufnative/TestProtobufNativeDecoder.java +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto.decoder.protobufnative; - -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.VarbinaryType.VARBINARY; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static org.apache.pulsar.sql.presto.TestPulsarConnector.getPulsarConnectorId; -import static org.testng.Assert.assertTrue; -import com.google.common.collect.ImmutableList; -import com.google.protobuf.ByteString; -import com.google.protobuf.Timestamp; -import io.netty.buffer.ByteBuf; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.decoder.FieldValueProvider; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.RowType; -import io.trino.spi.type.StandardTypes; -import io.trino.spi.type.Timestamps; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeSignatureParameter; -import java.util.HashSet; -import java.util.Map; -import org.apache.pulsar.client.impl.schema.ProtobufNativeSchema; -import org.apache.pulsar.client.impl.schema.generic.GenericProtobufNativeRecord; -import org.apache.pulsar.client.impl.schema.generic.GenericProtobufNativeSchema; -import org.apache.pulsar.sql.presto.PulsarColumnHandle; -import org.apache.pulsar.sql.presto.decoder.AbstractDecoderTester; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -public class TestProtobufNativeDecoder extends AbstractDecoderTester { - - private ProtobufNativeSchema schema; - - @BeforeMethod - public void init() { - super.init(); - schema = ProtobufNativeSchema.of(TestMsg.TestMessage.class); - schemaInfo = schema.getSchemaInfo(); - pulsarColumnHandle = getColumnColumnHandles(topicName, schemaInfo, PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); - pulsarRowDecoder = decoderFactory.createRowDecoder(topicName, schemaInfo, new HashSet<>(pulsarColumnHandle)); - decoderTestUtil = new ProtobufNativeDecoderTestUtil(); - assertTrue(pulsarRowDecoder instanceof PulsarProtobufNativeRowDecoder); - } - - @Test - public void testPrimitiveType() { - //Time: 2921-1-1 - long mills = 30010669261001L; - Timestamp timestamp = Timestamp.newBuilder() - .setSeconds(mills / 1000) - .setNanos((int) (mills % 1000) * 1000000) - .build(); - - TestMsg.TestMessage testMessage = TestMsg.TestMessage.newBuilder() - .setStringField("aaa") - .setDoubleField(3.3D) - .setFloatField(1.1f) - .setInt32Field(33) - .setInt64Field(44L) - .setUint32Field(33) - .setUint64Field(33L) - .setSint32Field(12) - .setSint64Field(13L) - .setFixed32Field(22) - .setFixed64Field(23L) - .setSfixed32Field(31) - .setSfixed64Field(32L) - .setBoolField(true) - .setBytesField(ByteString.copyFrom("abc".getBytes())) - .setTestEnum(TestMsg.TestEnum.FAILOVER) - .setTimestampField(timestamp) - .build(); - - ByteBuf payload = io.netty.buffer.Unpooled - .copiedBuffer(schema.encode(testMessage)); - Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); - - PulsarColumnHandle stringFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "stringField", VARCHAR, false, false, "stringField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, stringFieldColumnHandle, testMessage.getStringField()); - - PulsarColumnHandle doubleFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "doubleField", DOUBLE, false, false, "doubleField", null, null, - PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, doubleFieldColumnHandle, testMessage.getDoubleField()); - - PulsarColumnHandle int32FieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "int32Field", INTEGER, false, false, "int32Field", null, null, - PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, int32FieldColumnHandle, testMessage.getInt32Field()); - - PulsarColumnHandle int64FieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "int64Field", BIGINT, false, false, "int64Field", null, null, - PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, int64FieldColumnHandle, testMessage.getInt64Field()); - - PulsarColumnHandle uint32FieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "uint32Field", INTEGER, false, false, "uint32Field", null, null, - PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, uint32FieldColumnHandle, testMessage.getUint32Field()); - - PulsarColumnHandle uint64FieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "uint64Field", BIGINT, false, false, "uint64Field", null, null, - PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, uint64FieldColumnHandle, testMessage.getUint64Field()); - - PulsarColumnHandle sint32FieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "sint32Field", INTEGER, false, false, "sint32Field", null, null, - PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, sint32FieldColumnHandle, testMessage.getSint32Field()); - - PulsarColumnHandle sint64FieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "sint64Field", BIGINT, false, false, "sint64Field", null, null, - PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, sint64FieldColumnHandle, testMessage.getSint64Field()); - - PulsarColumnHandle fixed32FieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "fixed32Field", INTEGER, false, false, "fixed32Field", null, null, - PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, fixed32FieldColumnHandle, testMessage.getFixed32Field()); - - PulsarColumnHandle fixed64FieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "fixed64Field", BIGINT, false, false, "fixed64Field", null, null, - PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, fixed64FieldColumnHandle, testMessage.getFixed64Field()); - - PulsarColumnHandle sfixed32FieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "sfixed32Field", INTEGER, false, false, "sfixed32Field", null, null, - PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, sfixed32FieldColumnHandle, testMessage.getSfixed32Field()); - - PulsarColumnHandle sfixed64FieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "sfixed64Field", BIGINT, false, false, "sfixed64Field", null, null, - PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, sfixed64FieldColumnHandle, testMessage.getSfixed64Field()); - - PulsarColumnHandle boolFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "boolField", BOOLEAN, false, false, "boolField", null, null, - PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, boolFieldColumnHandle, testMessage.getBoolField()); - - PulsarColumnHandle bytesFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "bytesField", VARBINARY, false, false, "bytesField", null, null, - PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, bytesFieldColumnHandle, testMessage.getBytesField().toStringUtf8()); - - PulsarColumnHandle enumFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "testEnum", VARCHAR, false, false, "testEnum", null, null, - PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, enumFieldColumnHandle, testMessage.getTestEnum().name()); - - PulsarColumnHandle timestampFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "timestampField", TIMESTAMP_MILLIS,false,false,"timestampField",null,null, - PulsarColumnHandle.HandleKeyValueType.NONE); - checkValue(decodedRow, timestampFieldColumnHandle, mills * Timestamps.MICROSECONDS_PER_MILLISECOND); - - } - - @Test - public void testRow() { - - TestMsg.SubMessage.NestedMessage nestedMessage = TestMsg.SubMessage.NestedMessage.newBuilder() - .setTitle("nestedMessage_title") - .addUrls("aa") - .addUrls("bb") - .build(); - TestMsg.SubMessage subMessage = TestMsg.SubMessage.newBuilder() - .setBar(0.2) - .setFoo("fooValue") - .setBar(3.9d) - .setNestedMessage(nestedMessage) - .build(); - - TestMsg.TestMessage testMessage = TestMsg.TestMessage.newBuilder().setSubMessage(subMessage).build(); - - byte[] bytes = schema.encode(testMessage); - ByteBuf payload = io.netty.buffer.Unpooled - .copiedBuffer(bytes); - - GenericProtobufNativeRecord genericRecord = - (GenericProtobufNativeRecord) GenericProtobufNativeSchema.of(schemaInfo).decode(bytes); - Object fieldValue = - genericRecord.getProtobufRecord().getField(genericRecord.getProtobufRecord().getDescriptorForType().findFieldByName("subMessage")); - - Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); - RowType columnType = RowType.from(ImmutableList.builder() - .add(RowType.field("foo", VARCHAR)) - .add(RowType.field("bar", DOUBLE)) - .add(RowType.field("nestedMessage", RowType.from(ImmutableList.builder() - .add(RowType.field("title", VARCHAR)) - .add(RowType.field("urls", new ArrayType(VARCHAR))) - .build()))) - .build()); - - PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "subMessage", columnType, false, false, "subMessage", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - - checkRowValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue); - - } - - @Test - public void testArray() { - - TestMsg.TestMessage testMessage = TestMsg.TestMessage.newBuilder() - .addRepeatedField("first").addRepeatedField("second") - .build(); - - byte[] bytes = schema.encode(testMessage); - ByteBuf payload = io.netty.buffer.Unpooled - .copiedBuffer(bytes); - - GenericProtobufNativeRecord genericRecord = - (GenericProtobufNativeRecord) GenericProtobufNativeSchema.of(schemaInfo).decode(bytes); - Object fieldValue = - genericRecord.getProtobufRecord().getField(genericRecord.getProtobufRecord().getDescriptorForType().findFieldByName("repeatedField")); - - Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); - - ArrayType columnType = new ArrayType(VARCHAR); - PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), - "repeatedField", columnType, false, false, "repeatedField", - null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - - checkArrayValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue); - } - - - @Test - public void testMap() { - - TestMsg.TestMessage testMessage = TestMsg.TestMessage.newBuilder() - .putMapField("key_a", 1.1d) - .putMapField("key_b", 2.2d) - .build(); - - byte[] bytes = schema.encode(testMessage); - ByteBuf payload = io.netty.buffer.Unpooled - .copiedBuffer(bytes); - - GenericProtobufNativeRecord genericRecord = - (GenericProtobufNativeRecord) GenericProtobufNativeSchema.of(schemaInfo).decode(bytes); - Object fieldValue = - genericRecord.getProtobufRecord().getField(genericRecord.getProtobufRecord().getDescriptorForType().findFieldByName("mapField")); - - Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); - - Type columnType = decoderFactory.getTypeManager().getParameterizedType(StandardTypes.MAP, - ImmutableList.of(TypeSignatureParameter.typeParameter(VARCHAR.getTypeSignature()), - TypeSignatureParameter.typeParameter(DOUBLE.getTypeSignature()))); - - PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), "mapField", columnType, false, false, - "mapField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); - checkMapValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue); - - } - -} diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml index 3848586605301..27d9a8cb47ce4 100644 --- a/pulsar-testclient/pom.xml +++ b/pulsar-testclient/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-testclient @@ -85,14 +84,20 @@ ${project.version} + + ${project.groupId} + pulsar-cli-utils + ${project.version} + + commons-configuration commons-configuration - com.beust - jcommander + info.picocli + picocli compile @@ -106,12 +111,36 @@ jackson-databind + + io.opentelemetry + opentelemetry-exporter-prometheus + + + io.opentelemetry + opentelemetry-exporter-otlp + + + io.opentelemetry + opentelemetry-sdk + + + io.opentelemetry + opentelemetry-sdk-extension-autoconfigure + + org.awaitility awaitility test + + com.github.tomakehurst + wiremock-jre8 + ${wiremock.version} + test + + diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/proxy/socket/client/PerformanceClient.java b/pulsar-testclient/src/main/java/org/apache/pulsar/proxy/socket/client/PerformanceClient.java index 596eb8d2c2807..4d73fd9f9b4e3 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/proxy/socket/client/PerformanceClient.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/proxy/socket/client/PerformanceClient.java @@ -20,10 +20,6 @@ import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; import com.google.common.util.concurrent.RateLimiter; import io.netty.util.concurrent.DefaultThreadFactory; import java.io.FileInputStream; @@ -53,179 +49,171 @@ import org.apache.pulsar.client.api.AuthenticationDataProvider; import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.testclient.CmdBase; import org.apache.pulsar.testclient.IMessageFormatter; import org.apache.pulsar.testclient.PerfClientUtils; -import org.apache.pulsar.testclient.PositiveNumberParameterValidator; +import org.apache.pulsar.testclient.PositiveNumberParameterConvert; import org.apache.pulsar.testclient.utils.PaddingDecimalFormat; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.websocket.client.ClientUpgradeRequest; import org.eclipse.jetty.websocket.client.WebSocketClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Model.CommandSpec; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; +import picocli.CommandLine.Spec; -public class PerformanceClient { +@Command(name = "websocket-producer", description = "Test pulsar websocket producer performance.") +public class PerformanceClient extends CmdBase { private static final LongAdder messagesSent = new LongAdder(); private static final LongAdder bytesSent = new LongAdder(); private static final LongAdder totalMessagesSent = new LongAdder(); private static final LongAdder totalBytesSent = new LongAdder(); private static IMessageFormatter messageFormatter = null; - private JCommander jc; - @Parameters(commandDescription = "Test pulsar websocket producer performance.") - static class Arguments { + @Option(names = { "-cf", "--conf-file" }, description = "Configuration file") + public String confFile; - @Parameter(names = { "-h", "--help" }, description = "Help message", help = true) - boolean help; + @Option(names = { "-u", "--proxy-url" }, description = "Pulsar Proxy URL, e.g., \"ws://localhost:8080/\"") + public String proxyURL; - @Parameter(names = { "-cf", "--conf-file" }, description = "Configuration file") - public String confFile; + @Parameters(description = "persistent://tenant/ns/my-topic", arity = "1") + public List topics; - @Parameter(names = { "-u", "--proxy-url" }, description = "Pulsar Proxy URL, e.g., \"ws://localhost:8080/\"") - public String proxyURL; + @Option(names = { "-r", "--rate" }, description = "Publish rate msg/s across topics") + public int msgRate = 100; - @Parameter(description = "persistent://tenant/ns/my-topic", required = true) - public List topics; + @Option(names = { "-s", "--size" }, description = "Message size in byte") + public int msgSize = 1024; - @Parameter(names = { "-r", "--rate" }, description = "Publish rate msg/s across topics") - public int msgRate = 100; + @Option(names = { "-t", "--num-topic" }, description = "Number of topics", + converter = PositiveNumberParameterConvert.class + ) + public int numTopics = 1; - @Parameter(names = { "-s", "--size" }, description = "Message size in byte") - public int msgSize = 1024; + @Option(names = { "--auth_plugin" }, description = "Authentication plugin class name", hidden = true) + public String deprecatedAuthPluginClassName; - @Parameter(names = { "-t", "--num-topic" }, description = "Number of topics", - validateWith = PositiveNumberParameterValidator.class) - public int numTopics = 1; + @Option(names = { "--auth-plugin" }, description = "Authentication plugin class name") + public String authPluginClassName; - @Parameter(names = { "--auth_plugin" }, description = "Authentication plugin class name", hidden = true) - public String deprecatedAuthPluginClassName; + @Option( + names = { "--auth-params" }, + description = "Authentication parameters, whose format is determined by the implementation " + + "of method `configure` in authentication plugin class, for example \"key1:val1,key2:val2\" " + + "or \"{\"key1\":\"val1\",\"key2\":\"val2\"}\".") + public String authParams; - @Parameter(names = { "--auth-plugin" }, description = "Authentication plugin class name") - public String authPluginClassName; + @Option(names = { "-m", + "--num-messages" }, description = "Number of messages to publish in total. If <= 0, it will keep" + + " publishing") + public long numMessages = 0; - @Parameter( - names = { "--auth-params" }, - description = "Authentication parameters, whose format is determined by the implementation " - + "of method `configure` in authentication plugin class, for example \"key1:val1,key2:val2\" " - + "or \"{\"key1\":\"val1\",\"key2\":\"val2\"}\".") - public String authParams; + @Option(names = { "-f", "--payload-file" }, description = "Use payload from a file instead of empty buffer") + public String payloadFilename = null; - @Parameter(names = { "-m", - "--num-messages" }, description = "Number of messages to publish in total. If <= 0, it will keep" - + " publishing") - public long numMessages = 0; + @Option(names = { "-e", "--payload-delimiter" }, + description = "The delimiter used to split lines when using payload from a file") + // here escaping \n since default value will be printed with the help text + public String payloadDelimiter = "\\n"; - @Parameter(names = { "-f", "--payload-file" }, description = "Use payload from a file instead of empty buffer") - public String payloadFilename = null; + @Option(names = { "-fp", "--format-payload" }, + description = "Format %%i as a message index in the stream from producer and/or %%t as the timestamp" + + " nanoseconds") + public boolean formatPayload = false; - @Parameter(names = { "-e", "--payload-delimiter" }, - description = "The delimiter used to split lines when using payload from a file") - // here escaping \n since default value will be printed with the help text - public String payloadDelimiter = "\\n"; + @Option(names = {"-fc", "--format-class"}, description = "Custom Formatter class name") + public String formatterClass = "org.apache.pulsar.testclient.DefaultMessageFormatter"; - @Parameter(names = { "-fp", "--format-payload" }, - description = "Format %i as a message index in the stream from producer and/or %t as the timestamp" - + " nanoseconds") - public boolean formatPayload = false; + @Option(names = { "-time", + "--test-duration" }, description = "Test duration in secs. If <= 0, it will keep publishing") + public long testTime = 0; - @Parameter(names = {"-fc", "--format-class"}, description = "Custom Formatter class name") - public String formatterClass = "org.apache.pulsar.testclient.DefaultMessageFormatter"; - - @Parameter(names = { "-time", - "--test-duration" }, description = "Test duration in secs. If <= 0, it will keep publishing") - public long testTime = 0; + public PerformanceClient() { + super("websocket-producer"); } - public Arguments loadArguments(String[] args) { - Arguments arguments = new Arguments(); - jc = new JCommander(arguments); - jc.setProgramName("pulsar-perf websocket-producer"); - try { - jc.parse(args); - } catch (ParameterException e) { - System.out.println(e.getMessage()); - jc.usage(); - PerfClientUtils.exit(1); - } + @Spec + CommandSpec spec; - if (arguments.help) { - jc.usage(); - PerfClientUtils.exit(1); - } + public void loadArguments() { + CommandLine commander = spec.commandLine(); - if (isBlank(arguments.authPluginClassName) && !isBlank(arguments.deprecatedAuthPluginClassName)) { - arguments.authPluginClassName = arguments.deprecatedAuthPluginClassName; + if (isBlank(this.authPluginClassName) && !isBlank(this.deprecatedAuthPluginClassName)) { + this.authPluginClassName = this.deprecatedAuthPluginClassName; } - if (arguments.topics.size() != 1) { + if (this.topics.size() != 1) { System.err.println("Only one topic name is allowed"); - jc.usage(); + commander.usage(commander.getOut()); PerfClientUtils.exit(1); } - if (arguments.confFile != null) { + if (this.confFile != null) { Properties prop = new Properties(System.getProperties()); try { - prop.load(new FileInputStream(arguments.confFile)); + prop.load(new FileInputStream(this.confFile)); } catch (IOException e) { log.error("Error in loading config file"); - jc.usage(); + commander.usage(commander.getOut()); PerfClientUtils.exit(1); } - if (isBlank(arguments.proxyURL)) { + if (isBlank(this.proxyURL)) { String webSocketServiceUrl = prop.getProperty("webSocketServiceUrl"); if (isNotBlank(webSocketServiceUrl)) { - arguments.proxyURL = webSocketServiceUrl; + this.proxyURL = webSocketServiceUrl; } else { String webServiceUrl = isNotBlank(prop.getProperty("webServiceUrl")) ? prop.getProperty("webServiceUrl") : prop.getProperty("serviceUrl"); if (isNotBlank(webServiceUrl)) { if (webServiceUrl.startsWith("ws://") || webServiceUrl.startsWith("wss://")) { - arguments.proxyURL = webServiceUrl; + this.proxyURL = webServiceUrl; } else if (webServiceUrl.startsWith("http://") || webServiceUrl.startsWith("https://")) { - arguments.proxyURL = webServiceUrl.replaceFirst("^http", "ws"); + this.proxyURL = webServiceUrl.replaceFirst("^http", "ws"); } } } } - if (arguments.authPluginClassName == null) { - arguments.authPluginClassName = prop.getProperty("authPlugin", null); + if (this.authPluginClassName == null) { + this.authPluginClassName = prop.getProperty("authPlugin", null); } - if (arguments.authParams == null) { - arguments.authParams = prop.getProperty("authParams", null); + if (this.authParams == null) { + this.authParams = prop.getProperty("authParams", null); } } - if (isBlank(arguments.proxyURL)) { - arguments.proxyURL = "ws://localhost:8080/"; + if (isBlank(this.proxyURL)) { + this.proxyURL = "ws://localhost:8080/"; } - if (!arguments.proxyURL.endsWith("/")) { - arguments.proxyURL += "/"; + if (!this.proxyURL.endsWith("/")) { + this.proxyURL += "/"; } - return arguments; - } - public void runPerformanceTest(Arguments arguments) throws InterruptedException, IOException { + public void runPerformanceTest() throws InterruptedException, IOException { // Read payload data from file if needed - final byte[] payloadBytes = new byte[arguments.msgSize]; + final byte[] payloadBytes = new byte[this.msgSize]; Random random = new Random(0); List payloadByteList = new ArrayList<>(); - if (arguments.payloadFilename != null) { - Path payloadFilePath = Paths.get(arguments.payloadFilename); + if (this.payloadFilename != null) { + Path payloadFilePath = Paths.get(this.payloadFilename); if (Files.notExists(payloadFilePath) || Files.size(payloadFilePath) == 0) { throw new IllegalArgumentException("Payload file doesn't exist or it is empty."); } // here escaping the default payload delimiter to correct value - String delimiter = arguments.payloadDelimiter.equals("\\n") ? "\n" : arguments.payloadDelimiter; + String delimiter = this.payloadDelimiter.equals("\\n") ? "\n" : this.payloadDelimiter; String[] payloadList = new String(Files.readAllBytes(payloadFilePath), StandardCharsets.UTF_8) .split(delimiter); log.info("Reading payloads from {} and {} records read", payloadFilePath.toAbsolutePath(), @@ -234,8 +222,8 @@ public void runPerformanceTest(Arguments arguments) throws InterruptedException, payloadByteList.add(payload.getBytes(StandardCharsets.UTF_8)); } - if (arguments.formatPayload) { - messageFormatter = getMessageFormatter(arguments.formatterClass); + if (this.formatPayload) { + messageFormatter = getMessageFormatter(this.formatterClass); } } else { for (int i = 0; i < payloadBytes.length; ++i) { @@ -247,21 +235,21 @@ public void runPerformanceTest(Arguments arguments) throws InterruptedException, ExecutorService executor = Executors.newCachedThreadPool( new DefaultThreadFactory("pulsar-perf-producer-exec")); HashMap producersMap = new HashMap<>(); - String topicName = arguments.topics.get(0); + String topicName = this.topics.get(0); String restPath = TopicName.get(topicName).getRestPath(); String produceBaseEndPoint = TopicName.get(topicName).isV2() - ? arguments.proxyURL + "ws/v2/producer/" + restPath : arguments.proxyURL + "ws/producer/" + restPath; - for (int i = 0; i < arguments.numTopics; i++) { - String topic = arguments.numTopics > 1 ? produceBaseEndPoint + i : produceBaseEndPoint; + ? this.proxyURL + "ws/v2/producer/" + restPath : this.proxyURL + "ws/producer/" + restPath; + for (int i = 0; i < this.numTopics; i++) { + String topic = this.numTopics > 1 ? produceBaseEndPoint + i : produceBaseEndPoint; URI produceUri = URI.create(topic); WebSocketClient produceClient = new WebSocketClient(new SslContextFactory(true)); ClientUpgradeRequest produceRequest = new ClientUpgradeRequest(); - if (StringUtils.isNotBlank(arguments.authPluginClassName) && StringUtils.isNotBlank(arguments.authParams)) { + if (StringUtils.isNotBlank(this.authPluginClassName) && StringUtils.isNotBlank(this.authParams)) { try { - Authentication auth = AuthenticationFactory.create(arguments.authPluginClassName, - arguments.authParams); + Authentication auth = AuthenticationFactory.create(this.authPluginClassName, + this.authParams); auth.start(); AuthenticationDataProvider authData = auth.getAuthData(); if (authData.hasDataForHttp()) { @@ -295,23 +283,23 @@ public void runPerformanceTest(Arguments arguments) throws InterruptedException, executor.submit(() -> { try { - RateLimiter rateLimiter = RateLimiter.create(arguments.msgRate); + RateLimiter rateLimiter = RateLimiter.create(this.msgRate); long startTime = System.nanoTime(); - long testEndTime = startTime + (long) (arguments.testTime * 1e9); + long testEndTime = startTime + (long) (this.testTime * 1e9); // Send messages on all topics/producers long totalSent = 0; while (true) { for (String topic : producersMap.keySet()) { - if (arguments.testTime > 0 && System.nanoTime() > testEndTime) { + if (this.testTime > 0 && System.nanoTime() > testEndTime) { log.info("------------- DONE (reached the maximum duration: [{} seconds] of production) " - + "--------------", arguments.testTime); + + "--------------", this.testTime); PerfClientUtils.exit(0); } - if (arguments.numMessages > 0) { - if (totalSent >= arguments.numMessages) { + if (this.numMessages > 0) { + if (totalSent >= this.numMessages) { log.trace("------------- DONE (reached the maximum number: [{}] of production) " - + "--------------", arguments.numMessages); + + "--------------", this.numMessages); Thread.sleep(10000); PerfClientUtils.exit(0); } @@ -325,7 +313,7 @@ public void runPerformanceTest(Arguments arguments) throws InterruptedException, } byte[] payloadData; - if (arguments.payloadFilename != null) { + if (this.payloadFilename != null) { if (messageFormatter != null) { payloadData = messageFormatter.formatMessage("", totalSent, payloadByteList.get(random.nextInt(payloadByteList.size()))); @@ -415,16 +403,16 @@ static IMessageFormatter getMessageFormatter(String formatterClass) { } } - public static void main(String[] args) throws Exception { - PerformanceClient test = new PerformanceClient(); - Arguments arguments = test.loadArguments(args); + @Override + public void run() throws Exception { + loadArguments(); PerfClientUtils.printJVMInformation(log); long start = System.nanoTime(); Runtime.getRuntime().addShutdownHook(new Thread(() -> { printAggregatedThroughput(start); printAggregatedStats(); })); - test.runPerformanceTest(arguments); + runPerformanceTest(); } private class Tuple { diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/BrokerMonitor.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/BrokerMonitor.java index 3f8969860163d..6af4925a7c664 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/BrokerMonitor.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/BrokerMonitor.java @@ -19,11 +19,8 @@ package org.apache.pulsar.testclient; import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.BROKER_LOAD_DATA_STORE_TOPIC; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; -import com.google.gson.Gson; +import static org.apache.pulsar.broker.resources.LoadBalanceResources.BROKER_TIME_AVERAGE_BASE_PATH; +import com.fasterxml.jackson.databind.ObjectReader; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -34,11 +31,12 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; -import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SizeUnit; import org.apache.pulsar.client.api.TableView; +import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.apache.pulsar.policies.data.loadbalancer.LoadManagerReport; import org.apache.pulsar.policies.data.loadbalancer.LoadReport; import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; @@ -50,19 +48,28 @@ import org.apache.zookeeper.ZooKeeper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; /** * Monitors brokers and prints to the console information about their system resource usages, their topic and bundle * counts, their message rates, and other metrics. */ -public class BrokerMonitor { +@Command(name = "monitor-brokers", + description = "Monitors brokers and prints to the console information about their system " + + "resource usages, \ntheir topic and bundle counts, their message rates, and other metrics.") +public class BrokerMonitor extends CmdBase { private static final Logger log = LoggerFactory.getLogger(BrokerMonitor.class); private static final String BROKER_ROOT = "/loadbalance/brokers"; private static final int ZOOKEEPER_TIMEOUT_MILLIS = 30000; private static final int GLOBAL_STATS_PRINT_PERIOD_MILLIS = 60000; private ZooKeeper zkClient; - private static final Gson gson = new Gson(); + private static final ObjectReader LOAD_REPORT_READER = ObjectMapperFactory.getMapper().reader() + .forType(LoadManagerReport.class); + + private static final ObjectReader TIME_AVERAGE_READER = ObjectMapperFactory.getMapper().reader() + .forType(TimeAverageBrokerData.class); // Fields common for message rows. private static final List MESSAGE_FIELDS = Arrays.asList("MSG/S IN", "MSG/S OUT", "TOTAL", "KB/S IN", @@ -84,9 +91,10 @@ public class BrokerMonitor { private static final Object[] ALLOC_MESSAGE_ROW = makeMessageRow("ALLOC MSG"); private static final Object[] GLOBAL_HEADER = { "BROKER", "BUNDLE", "MSG/S", "LONG/S", "KB/S", "MAX %" }; - private Map loadData; + private Map loadData; private static final FixedColumnLengthTableMaker localTableMaker = new FixedColumnLengthTableMaker(); + static { // Makes the table length about 120. localTableMaker.elementLength = 14; @@ -94,6 +102,7 @@ public class BrokerMonitor { } private static final FixedColumnLengthTableMaker globalTableMaker = new FixedColumnLengthTableMaker(); + static { globalTableMaker.decimalFormatter = "%.2f"; globalTableMaker.topBorder = '*'; @@ -125,7 +134,7 @@ private static void initRow(final Object[] row, final Object... elements) { // Helper method to initialize rows which hold message data. private static void initMessageRow(final Object[] row, final double messageRateIn, final double messageRateOut, - final double messageThroughputIn, final double messageThroughputOut) { + final double messageThroughputIn, final double messageThroughputOut) { initRow(row, messageRateIn, messageRateOut, messageRateIn + messageRateOut, messageThroughputIn / 1024, messageThroughputOut / 1024, (messageThroughputIn + messageThroughputOut) / 1024); @@ -143,9 +152,9 @@ private void printGlobalData() { double totalLongTermMessageRate = 0; double maxMaxUsage = 0; int i = 1; - for (final Map.Entry entry : loadData.entrySet()) { + for (final Map.Entry entry : loadData.entrySet()) { final String broker = entry.getKey(); - final Object data = entry.getValue(); + final LoadManagerReport data = entry.getValue(); rows[i] = new Object[GLOBAL_HEADER.length]; rows[i][0] = broker; int numBundles; @@ -172,11 +181,10 @@ private void printGlobalData() { final LocalBrokerData localData = (LocalBrokerData) data; numBundles = localData.getNumBundles(); messageRate = localData.getMsgRateIn() + localData.getMsgRateOut(); - final String timeAveragePath = ModularLoadManagerImpl.TIME_AVERAGE_BROKER_ZPATH + "/" + broker; + final String timeAveragePath = BROKER_TIME_AVERAGE_BASE_PATH + "/" + broker; try { - final TimeAverageBrokerData timeAverageData = gson.fromJson( - new String(zkClient.getData(timeAveragePath, false, null)), - TimeAverageBrokerData.class); + final TimeAverageBrokerData timeAverageData = TIME_AVERAGE_READER.readValue( + new String(zkClient.getData(timeAveragePath, false, null))); longTermMessageRate = timeAverageData.getLongTermMsgRateIn() + timeAverageData.getLongTermMsgRateOut(); } catch (Exception x) { @@ -304,20 +312,21 @@ private double percentUsage(final double usage, final double limit) { private synchronized void printData(final String path) { final String broker = brokerNameFromPath(path); String jsonString; + LoadManagerReport loadManagerReport; try { jsonString = new String(zkClient.getData(path, this, null)); + loadManagerReport = LOAD_REPORT_READER.readValue(jsonString); } catch (Exception ex) { throw new RuntimeException(ex); } - // Use presence of the String "allocated" to determine if this is using SimpleLoadManagerImpl. - if (jsonString.contains("allocated")) { - printLoadReport(broker, gson.fromJson(jsonString, LoadReport.class)); - } else { - final LocalBrokerData localBrokerData = gson.fromJson(jsonString, LocalBrokerData.class); - final String timeAveragePath = ModularLoadManagerImpl.TIME_AVERAGE_BROKER_ZPATH + "/" + broker; + if (loadManagerReport instanceof LoadReport) { + printLoadReport(broker, (LoadReport) loadManagerReport); + } else { + final LocalBrokerData localBrokerData = (LocalBrokerData) loadManagerReport; + final String timeAveragePath = BROKER_TIME_AVERAGE_BASE_PATH + "/" + broker; try { - final TimeAverageBrokerData timeAverageData = gson.fromJson( - new String(zkClient.getData(timeAveragePath, false, null)), TimeAverageBrokerData.class); + final TimeAverageBrokerData timeAverageData = TIME_AVERAGE_READER.readValue( + new String(zkClient.getData(timeAveragePath, false, null))); printBrokerData(broker, localBrokerData, timeAverageData); } catch (Exception e) { throw new RuntimeException(e); @@ -390,7 +399,7 @@ private synchronized void printLoadReport(final String broker, final LoadReport // Print the broker data in a tabular form for a broker using ModularLoadManagerImpl. private synchronized void printBrokerData(final String broker, final LocalBrokerData localBrokerData, - final TimeAverageBrokerData timeAverageData) { + final TimeAverageBrokerData timeAverageData) { loadData.put(broker, localBrokerData); // Initialize the constant rows. @@ -434,18 +443,15 @@ private synchronized void printBrokerData(final String broker, final LocalBroker } } - // JCommander arguments class. - @Parameters(commandDescription = "Monitors brokers and prints to the console information about their system " - + "resource usages, \ntheir topic and bundle counts, their message rates, and other metrics.") - private static class Arguments { - @Parameter(names = { "-h", "--help" }, description = "Help message", help = true) - boolean help; + @Option(names = {"--connect-string"}, description = "Zookeeper or broker connect string", required = true) + public String connectString = null; + + @Option(names = {"--extensions"}, description = "true to monitor Load Balance Extensions.") + boolean extensions = false; - @Parameter(names = { "--connect-string" }, description = "Zookeeper or broker connect string", required = true) - public String connectString = null; - @Parameter(names = { "--extensions" }, description = "true to monitor Load Balance Extensions.") - boolean extensions = false; + public BrokerMonitor() { + super("monitor-brokers"); } /** @@ -454,6 +460,7 @@ private static class Arguments { * @param zkClient Client to create this from. */ public BrokerMonitor(final ZooKeeper zkClient) { + super("monitor-brokers"); loadData = new ConcurrentHashMap<>(); this.zkClient = zkClient; } @@ -477,6 +484,7 @@ public void start() { private TableView brokerLoadDataTableView; private BrokerMonitor(String brokerServiceUrl) { + super("monitor-brokers"); try { PulsarClient client = PulsarClient.builder() .memoryLimit(0, SizeUnit.BYTES) @@ -539,32 +547,16 @@ private void startBrokerLoadDataStoreMonitor() { } } - /** - * Run a monitor from command line arguments. - * - * @param args Arguments for the monitor. - */ - public static void main(String[] args) throws Exception { - final Arguments arguments = new Arguments(); - final JCommander jc = new JCommander(arguments); - jc.setProgramName("pulsar-perf monitor-brokers"); - - try { - jc.parse(args); - } catch (ParameterException e) { - System.out.println(e.getMessage()); - jc.usage(); - PerfClientUtils.exit(1); - } - - - if (arguments.extensions) { - final BrokerMonitor monitor = new BrokerMonitor(arguments.connectString); + @Override + public void run() throws Exception { + if (this.extensions) { + final BrokerMonitor monitor = new BrokerMonitor(this.connectString); monitor.startBrokerLoadDataStoreMonitor(); } else { - final ZooKeeper zkClient = new ZooKeeper(arguments.connectString, ZOOKEEPER_TIMEOUT_MILLIS, null); + final ZooKeeper zkClient = new ZooKeeper(this.connectString, ZOOKEEPER_TIMEOUT_MILLIS, null); final BrokerMonitor monitor = new BrokerMonitor(zkClient); monitor.start(); } } + } diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/CmdBase.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/CmdBase.java new file mode 100644 index 0000000000000..6d5796ad5dda7 --- /dev/null +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/CmdBase.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.testclient; + +import java.util.concurrent.Callable; +import picocli.CommandLine; + +public abstract class CmdBase implements Callable { + private final CommandLine commander; + + public CmdBase(String cmdName) { + commander = new CommandLine(this); + commander.setCommandName(cmdName); + } + + public boolean run(String[] args) { + return commander.execute(args) == 0; + } + + public void parse(String[] args) { + commander.parseArgs(args); + } + + /** + * Validate the CLI arguments. Default implementation provides validation for the common arguments. + * Each subclass should call super.validate() and provide validation code specific to the sub-command. + * @throws Exception + */ + public void validate() throws Exception { + } + + // Picocli entrypoint. + @Override + public Integer call() throws Exception { + validate(); + run(); + return 0; + } + + public abstract void run() throws Exception; + + + protected CommandLine getCommander() { + return commander; + } + + protected void addCommand(String name, Object cmd) { + commander.addSubcommand(name, cmd); + } + + protected void addCommand(String name, Object cmd, String... aliases) { + commander.addSubcommand(name, cmd, aliases); + } + + protected class ParameterException extends CommandLine.ParameterException { + public ParameterException(String msg) { + super(commander, msg); + } + + public ParameterException(String msg, Throwable e) { + super(commander, msg, e); + } + } +} diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/CmdGenerateDocumentation.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/CmdGenerateDocumentation.java index e3aca98865527..d2e08e2cc8664 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/CmdGenerateDocumentation.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/CmdGenerateDocumentation.java @@ -18,100 +18,101 @@ */ package org.apache.pulsar.testclient; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterDescription; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; import java.lang.reflect.Constructor; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.proxy.socket.client.PerformanceClient; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Model.CommandSpec; +import picocli.CommandLine.Option; +import picocli.CommandLine.Spec; @Slf4j -public class CmdGenerateDocumentation { +@Command(name = "gen-doc", description = "Generate documentation automatically.") +public class CmdGenerateDocumentation extends CmdBase{ - @Parameters(commandDescription = "Generate documentation automatically.") - static class Arguments { - - @Parameter(names = {"-h", "--help"}, description = "Help message", help = true) - boolean help; - - @Parameter(names = {"-n", "--command-names"}, description = "List of command names") - private List commandNames = new ArrayList<>(); + @Option(names = {"-n", "--command-names"}, description = "List of command names") + private List commandNames = new ArrayList<>(); + public CmdGenerateDocumentation() { + super("gen-doc"); } - public static void main(String[] args) throws Exception { - final Arguments arguments = new Arguments(); - final JCommander jc = new JCommander(arguments); - jc.setProgramName("pulsar-perf gen-doc"); - try { - jc.parse(args); - } catch (ParameterException e) { - System.out.println(e.getMessage()); - jc.usage(); - PerfClientUtils.exit(1); - } + @Spec + CommandSpec spec; - if (arguments.help) { - jc.usage(); - PerfClientUtils.exit(1); - } + @Override + public void run() throws Exception { + CommandLine commander = spec.commandLine(); Map> cmdClassMap = new LinkedHashMap<>(); - cmdClassMap.put("produce", Class.forName("org.apache.pulsar.testclient.PerformanceProducer$Arguments")); - cmdClassMap.put("consume", Class.forName("org.apache.pulsar.testclient.PerformanceConsumer$Arguments")); - cmdClassMap.put("transaction", Class.forName("org.apache.pulsar.testclient.PerformanceTransaction$Arguments")); - cmdClassMap.put("read", Class.forName("org.apache.pulsar.testclient.PerformanceReader$Arguments")); - cmdClassMap.put("monitor-brokers", Class.forName("org.apache.pulsar.testclient.BrokerMonitor$Arguments")); - cmdClassMap.put("simulation-client", - Class.forName("org.apache.pulsar.testclient.LoadSimulationClient$MainArguments")); - cmdClassMap.put("simulation-controller", - Class.forName("org.apache.pulsar.testclient.LoadSimulationController$MainArguments")); - cmdClassMap.put("websocket-producer", - Class.forName("org.apache.pulsar.proxy.socket.client.PerformanceClient$Arguments")); - cmdClassMap.put("managed-ledger", Class.forName("org.apache.pulsar.testclient.ManagedLedgerWriter$Arguments")); + cmdClassMap.put("produce", PerformanceProducer.class); + cmdClassMap.put("consume", PerformanceConsumer.class); + cmdClassMap.put("transaction", PerformanceTransaction.class); + cmdClassMap.put("read", PerformanceReader.class); + cmdClassMap.put("monitor-brokers", BrokerMonitor.class); + cmdClassMap.put("simulation-client", LoadSimulationClient.class); + cmdClassMap.put("simulation-controller", LoadSimulationController.class); + cmdClassMap.put("websocket-producer", PerformanceClient.class); + cmdClassMap.put("managed-ledger", ManagedLedgerWriter.class); for (Map.Entry> entry : cmdClassMap.entrySet()) { String cmd = entry.getKey(); Class clazz = entry.getValue(); Constructor constructor = clazz.getDeclaredConstructor(); constructor.setAccessible(true); - jc.addCommand(cmd, constructor.newInstance()); + commander.addSubcommand(cmd, constructor.newInstance()); } - if (arguments.commandNames.size() == 0) { - for (Map.Entry cmd : jc.getCommands().entrySet()) { - generateDocument(cmd.getKey(), jc); + if (this.commandNames.size() == 0) { + for (Map.Entry cmd : commander.getSubcommands().entrySet()) { + generateDocument(cmd.getKey(), commander); } } else { - for (String commandName : arguments.commandNames) { - generateDocument(commandName, jc); + for (String commandName : this.commandNames) { + generateDocument(commandName, commander); } } } - private static String generateDocument(String module, JCommander parentCmd) { + private static String generateDocument(String module, CommandLine parentCmd) { StringBuilder sb = new StringBuilder(); - JCommander cmd = parentCmd.getCommands().get(module); + CommandLine cmd = parentCmd.getSubcommands().get(module); sb.append("## ").append(module).append("\n\n"); - sb.append(parentCmd.getUsageFormatter().getCommandDescription(module)).append("\n"); + sb.append(getCommandDescription(cmd)).append("\n"); sb.append("\n\n```shell\n") .append("$ pulsar-perf ").append(module).append(" [options]") .append("\n```"); sb.append("\n\n"); sb.append("|Flag|Description|Default|\n"); sb.append("|---|---|---|\n"); - List options = cmd.getParameters(); - options.stream().filter(ele -> !ele.getParameterAnnotation().hidden()).forEach((option) -> - sb.append("| `").append(option.getNames()) - .append("` | ").append(option.getDescription().replace("\n", " ")) - .append("|").append(option.getDefault()).append("|\n") + List options = cmd.getCommandSpec().options(); + options.stream().filter(ele -> !ele.hidden()).forEach((option) -> + sb.append("| `").append(String.join(", ", option.names())) + .append("` | ").append(getOptionDescription(option).replace("\n", " ")) + .append("|").append(option.defaultValueString()).append("|\n") ); System.out.println(sb.toString()); return sb.toString(); } + + public static String getCommandDescription(CommandLine commandLine) { + String[] description = commandLine.getCommandSpec().usageMessage().description(); + if (description != null && description.length != 0) { + return description[0]; + } + return ""; + } + + public static String getOptionDescription(CommandLine.Model.OptionSpec optionSpec) { + String[] description = optionSpec.description(); + if (description != null && description.length != 0) { + return description[0]; + } + return ""; + } } diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/LoadSimulationClient.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/LoadSimulationClient.java index 64330ae2eeea1..115733d5ecd41 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/LoadSimulationClient.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/LoadSimulationClient.java @@ -18,11 +18,8 @@ */ package org.apache.pulsar.testclient; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; import com.google.common.util.concurrent.RateLimiter; +import com.google.re2j.Pattern; import io.netty.util.concurrent.DefaultThreadFactory; import java.io.DataInputStream; import java.io.DataOutputStream; @@ -37,6 +34,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; +import org.apache.pulsar.cli.converters.picocli.ByteUnitToLongConverter; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; @@ -44,15 +42,20 @@ import org.apache.pulsar.client.api.MessageListener; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SizeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; /** * LoadSimulationClient is used to simulate client load by maintaining producers and consumers for topics. Instances of * this class are controlled across a network via LoadSimulationController. */ -public class LoadSimulationClient { +@Command(name = "simulation-client", + description = "Simulate client load by maintaining producers and consumers for topics.") +public class LoadSimulationClient extends CmdBase{ private static final Logger log = LoggerFactory.getLogger(LoadSimulationClient.class); // Values for command encodings. @@ -63,7 +66,7 @@ public class LoadSimulationClient { public static final byte STOP_GROUP_COMMAND = 4; public static final byte FIND_COMMAND = 5; - private final ExecutorService executor; + private ExecutorService executor; // Map from a message size to a cached byte[] of that size. private final Map payloadCache; @@ -71,12 +74,10 @@ public class LoadSimulationClient { private final Map topicsToTradeUnits; // Pulsar admin to create namespaces with. - private final PulsarAdmin admin; + private PulsarAdmin admin; // Pulsar client to create producers and consumers with. - private final PulsarClient client; - - private final int port; + private PulsarClient client; // A TradeUnit is a Consumer and Producer pair. The rate of message // consumption as well as size may be changed at @@ -169,18 +170,18 @@ public void start() throws Exception { } } - // JCommander arguments for starting a LoadSimulationClient. - @Parameters(commandDescription = "Simulate client load by maintaining producers and consumers for topics.") - private static class MainArguments { - @Parameter(names = { "-h", "--help" }, description = "Help message", help = true) - boolean help; + // picocli arguments for starting a LoadSimulationClient. - @Parameter(names = { "--port" }, description = "Port to listen on for controller", required = true) - public int port; + @Option(names = { "--port" }, description = "Port to listen on for controller", required = true) + public int port; + + @Option(names = { "--service-url" }, description = "Pulsar Service URL", required = true) + public String serviceURL; + + @Option(names = { "-ml", "--memory-limit", }, description = "Configure the Pulsar client memory limit " + + "(eg: 32M, 64M)", converter = ByteUnitToLongConverter.class) + public long memoryLimit = 0L; - @Parameter(names = { "--service-url" }, description = "Pulsar Service URL", required = true) - public String serviceURL; - } // Configuration class for initializing or modifying TradeUnits. private static class TradeConfiguration { @@ -269,11 +270,14 @@ private void handle(final byte command, final DataInputStream inputStream, final tradeConf.size = inputStream.readInt(); tradeConf.rate = inputStream.readDouble(); // See if a topic belongs to this tenant and group using this regex. - final String groupRegex = ".*://" + tradeConf.tenant + "/.*/" + tradeConf.group + "-.*/.*"; + final Pattern groupRegex = + Pattern.compile(".*://" + tradeConf.tenant + "/.*/" + tradeConf.group + "-.*/.*"); + for (Map.Entry entry : topicsToTradeUnits.entrySet()) { final String topic = entry.getKey(); final TradeUnit unit = entry.getValue(); - if (topic.matches(groupRegex)) { + + if (groupRegex.matcher(topic).matches()) { unit.change(tradeConf); } } @@ -282,11 +286,11 @@ private void handle(final byte command, final DataInputStream inputStream, final // Stop all topics belonging to a group. decodeGroupOptions(tradeConf, inputStream); // See if a topic belongs to this tenant and group using this regex. - final String regex = ".*://" + tradeConf.tenant + "/.*/" + tradeConf.group + "-.*/.*"; + final Pattern regex = Pattern.compile(".*://" + tradeConf.tenant + "/.*/" + tradeConf.group + "-.*/.*"); for (Map.Entry entry : topicsToTradeUnits.entrySet()) { final String topic = entry.getKey(); final TradeUnit unit = entry.getValue(); - if (topic.matches(regex)) { + if (regex.matcher(topic).matches()) { unit.stop.set(true); } } @@ -305,54 +309,40 @@ private void handle(final byte command, final DataInputStream inputStream, final private static final MessageListener ackListener = Consumer::acknowledgeAsync; /** - * Create a LoadSimulationClient with the given JCommander arguments. + * Create a LoadSimulationClient with the given picocli this. * - * @param arguments - * Arguments to configure this from. */ - public LoadSimulationClient(final MainArguments arguments) throws Exception { + public LoadSimulationClient() throws PulsarClientException { + super("simulation-client"); payloadCache = new ConcurrentHashMap<>(); topicsToTradeUnits = new ConcurrentHashMap<>(); - - admin = PulsarAdmin.builder() - .serviceHttpUrl(arguments.serviceURL) - .build(); - client = PulsarClient.builder() - .memoryLimit(0, SizeUnit.BYTES) - .serviceUrl(arguments.serviceURL) - .connectionsPerBroker(4) - .ioThreads(Runtime.getRuntime().availableProcessors()) - .statsInterval(0, TimeUnit.SECONDS) - .build(); - port = arguments.port; - executor = Executors.newCachedThreadPool(new DefaultThreadFactory("test-client")); } /** - * Start a client with command line arguments. + * Start a client with command line this. * - * @param args - * Command line arguments to pass in. */ - public static void main(String[] args) throws Exception { - final MainArguments mainArguments = new MainArguments(); - final JCommander jc = new JCommander(mainArguments); - jc.setProgramName("pulsar-perf simulation-client"); - try { - jc.parse(args); - } catch (ParameterException e) { - System.out.println(e.getMessage()); - jc.usage(); - PerfClientUtils.exit(1); - } + @Override + public void run() throws Exception { + admin = PulsarAdmin.builder() + .serviceHttpUrl(this.serviceURL) + .build(); + client = PulsarClient.builder() + .memoryLimit(this.memoryLimit, SizeUnit.BYTES) + .serviceUrl(this.serviceURL) + .connectionsPerBroker(4) + .ioThreads(Runtime.getRuntime().availableProcessors()) + .statsInterval(0, TimeUnit.SECONDS) + .build(); + executor = Executors.newCachedThreadPool(new DefaultThreadFactory("test-client")); PerfClientUtils.printJVMInformation(log); - (new LoadSimulationClient(mainArguments)).run(); + this.start(); } /** * Start listening for controller commands to create producers and consumers. */ - public void run() throws Exception { + public void start() throws Exception { final ServerSocket serverSocket = new ServerSocket(port); while (true) { diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/LoadSimulationController.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/LoadSimulationController.java index bbe535df5e289..99f443f26d7d2 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/LoadSimulationController.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/LoadSimulationController.java @@ -18,10 +18,8 @@ */ package org.apache.pulsar.testclient; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; +import static org.apache.pulsar.broker.resources.LoadBalanceResources.BUNDLE_DATA_BASE_PATH; +import static org.apache.pulsar.broker.resources.LoadBalanceResources.RESOURCE_QUOTA_BASE_PATH; import java.io.BufferedReader; import java.io.DataInputStream; import java.io.DataOutputStream; @@ -54,78 +52,72 @@ import org.apache.zookeeper.ZooKeeper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; /** * This class provides a shell for the user to dictate how simulation clients should incur load. */ -public class LoadSimulationController { +@Command(name = "simulation-controller", + description = "Provides a shell for the user to dictate how simulation clients should " + + "incur load.") +public class LoadSimulationController extends CmdBase{ private static final Logger log = LoggerFactory.getLogger(LoadSimulationController.class); - private static final String QUOTA_ROOT = "/loadbalance/resource-quota/namespace"; - private static final String BUNDLE_DATA_ROOT = "/loadbalance/bundle-data"; // Input streams for each client to send commands through. - private final DataInputStream[] inputStreams; + private DataInputStream[] inputStreams; // Output streams for each client to receive information from. - private final DataOutputStream[] outputStreams; + private DataOutputStream[] outputStreams; // client host names. - private final String[] clients; + private String[] clients; - // Port clients are listening on. - private final int clientPort; - - // The ZooKeeper cluster to run on. - private final String cluster; - - private final Random random; + private Random random; private static final ExecutorService threadPool = Executors.newCachedThreadPool(); - // JCommander arguments for starting a controller via main. - @Parameters(commandDescription = "Provides a shell for the user to dictate how simulation clients should " - + "incur load.") - private static class MainArguments { - @Parameter(names = { "-h", "--help" }, description = "Help message", help = true) - boolean help; + // picocli arguments for starting a controller via main. - @Parameter(names = { "--cluster" }, description = "Cluster to test on", required = true) - String cluster; + @Option(names = { "--cluster" }, description = "Cluster to test on", required = true) + String cluster; - @Parameter(names = { "--clients" }, description = "Comma separated list of client hostnames", required = true) - String clientHostNames; + @Option(names = { "--clients" }, description = "Comma separated list of client hostnames", required = true) + String clientHostNames; + + @Option(names = { "--client-port" }, description = "Port that the clients are listening on", required = true) + int clientPort; - @Parameter(names = { "--client-port" }, description = "Port that the clients are listening on", required = true) - int clientPort; - } - // JCommander arguments for accepting user input. + // picocli arguments for accepting user input. private static class ShellArguments { - @Parameter(description = "Command arguments:\n" + "trade tenant namespace topic\n" + @Parameters(description = "Command arguments:\n" + "trade tenant namespace topic\n" + "change tenant namespace topic\n" + "stop tenant namespace topic\n" + "trade_group tenant group_name num_namespaces\n" + "change_group tenant group_name\n" + "stop_group tenant group_name\n" + "script script_name\n" + "copy tenant_name source_zk target_zk\n" - + "stream source_zk\n" + "simulate zk\n", required = true) + + "stream source_zk\n" + "simulate zk\n", arity = "1") List commandArguments; - @Parameter(names = { "--rand-rate" }, description = "Choose message rate uniformly randomly from the next two " + @Option(names = { "--rand-rate" }, description = "Choose message rate uniformly randomly from the next two " + "comma separated values (overrides --rate)") String rangeString = ""; - @Parameter(names = { "--rate" }, description = "Messages per second") + @Option(names = { "--rate" }, description = "Messages per second") double rate = 1; - @Parameter(names = { "--rate-multiplier" }, description = "Multiplier to use for copying or streaming rates") + @Option(names = { "--rate-multiplier" }, description = "Multiplier to use for copying or streaming rates") double rateMultiplier = 1; - @Parameter(names = { "--separation" }, description = "Separation time in ms for trade_group actions " + @Option(names = { "--separation" }, description = "Separation time in ms for trade_group actions " + "(0 for no separation)") int separation = 0; - @Parameter(names = { "--size" }, description = "Message size in bytes") + @Option(names = { "--size" }, description = "Message size in bytes") int size = 1024; - @Parameter(names = { "--topics-per-namespace" }, description = "Number of topics to create per namespace in " + @Option(names = { "--topics-per-namespace" }, description = "Number of topics to create per namespace in " + "trade_group (total number of topics is num_namespaces X num_topics)") int topicsPerNamespace = 1; } @@ -212,26 +204,11 @@ public synchronized void process(final WatchedEvent event) { } /** - * Create a LoadSimulationController with the given JCommander arguments. + * Create a LoadSimulationController with the given picocli arguments. * - * @param arguments - * Arguments to create from. */ - public LoadSimulationController(final MainArguments arguments) throws Exception { - random = new Random(); - clientPort = arguments.clientPort; - cluster = arguments.cluster; - clients = arguments.clientHostNames.split(","); - final Socket[] sockets = new Socket[clients.length]; - inputStreams = new DataInputStream[clients.length]; - outputStreams = new DataOutputStream[clients.length]; - log.info("Found {} clients:", clients.length); - for (int i = 0; i < clients.length; ++i) { - sockets[i] = new Socket(clients[i], clientPort); - inputStreams[i] = new DataInputStream(sockets[i].getInputStream()); - outputStreams[i] = new DataOutputStream(sockets[i].getOutputStream()); - log.info("Connected to {}", clients[i]); - } + public LoadSimulationController() throws Exception { + super("simulation-controller"); } // Check that the expected number of application arguments matches the @@ -318,7 +295,7 @@ private void writeProducerOptions(final DataOutputStream outputStream, final She outputStream.writeDouble(arguments.rate); } - // Change producer settings for a given topic and JCommander arguments. + // Change producer settings for a given topic and picocli arguments. private void change(final ShellArguments arguments, final String topic, final int client) throws Exception { outputStreams[client].write(LoadSimulationClient.CHANGE_COMMAND); writeProducerOptions(outputStreams[client], arguments, topic); @@ -360,7 +337,7 @@ private int find(final String topic) throws Exception { return clientWithTopic; } - // Trade using the arguments parsed via JCommander and the topic name. + // Trade using the arguments parsed via picocli and the topic name. private synchronized void trade(final ShellArguments arguments, final String topic, final int client) throws Exception { // Decide which client to send to randomly to preserve statelessness of @@ -398,7 +375,7 @@ private void handleCopy(final ShellArguments arguments) throws Exception { for (int i = 0; i < clients.length; ++i) { threadLocalMaps[i] = new HashMap<>(); } - getResourceQuotas(QUOTA_ROOT, sourceZKClient, threadLocalMaps); + getResourceQuotas(RESOURCE_QUOTA_BASE_PATH, sourceZKClient, threadLocalMaps); final List futures = new ArrayList<>(clients.length); int i = 0; log.info("Copying..."); @@ -411,7 +388,7 @@ private void handleCopy(final ShellArguments arguments) throws Exception { // Simulation will send messages in and out at about the same rate, so just make the rate the // average of in and out. - final int tenantStart = QUOTA_ROOT.length() + 1; + final int tenantStart = RESOURCE_QUOTA_BASE_PATH.length() + 1; final int clusterStart = bundle.indexOf('/', tenantStart) + 1; final String sourceTenant = bundle.substring(tenantStart, clusterStart - 1); final int namespaceStart = bundle.indexOf('/', clusterStart) + 1; @@ -424,10 +401,10 @@ private void handleCopy(final ShellArguments arguments) throws Exception { final String mangledNamespace = String.format("%s-%s", manglePrefix, namespace); final BundleData bundleData = initializeBundleData(quota, arguments); final String oldAPITargetPath = String.format( - "/loadbalance/resource-quota/namespace/%s/%s/%s/0x00000000_0xffffffff", tenantName, + "%s/namespace/%s/%s/%s/0x00000000_0xffffffff", BUNDLE_DATA_BASE_PATH, tenantName, cluster, mangledNamespace); final String newAPITargetPath = String.format( - "/loadbalance/bundle-data/%s/%s/%s/0x00000000_0xffffffff", tenantName, cluster, + "%s/%s/%s/%s/0x00000000_0xffffffff", BUNDLE_DATA_BASE_PATH, tenantName, cluster, mangledNamespace); try { ZkUtils.createFullPathOptimistic(targetZKClient, oldAPITargetPath, @@ -475,7 +452,7 @@ private void handleSimulate(final ShellArguments arguments) throws Exception { for (int i = 0; i < clients.length; ++i) { threadLocalMaps[i] = new HashMap<>(); } - getResourceQuotas(QUOTA_ROOT, zkClient, threadLocalMaps); + getResourceQuotas(RESOURCE_QUOTA_BASE_PATH, zkClient, threadLocalMaps); final List futures = new ArrayList<>(clients.length); int i = 0; log.info("Simulating..."); @@ -484,9 +461,9 @@ private void handleSimulate(final ShellArguments arguments) throws Exception { futures.add(threadPool.submit(() -> { for (final Map.Entry entry : bundleToQuota.entrySet()) { final String bundle = entry.getKey(); - final String newAPIPath = bundle.replace(QUOTA_ROOT, BUNDLE_DATA_ROOT); + final String newAPIPath = bundle.replace(RESOURCE_QUOTA_BASE_PATH, BUNDLE_DATA_BASE_PATH); final ResourceQuota quota = entry.getValue(); - final int tenantStart = QUOTA_ROOT.length() + 1; + final int tenantStart = RESOURCE_QUOTA_BASE_PATH.length() + 1; final String topic = String.format("persistent://%s/t", bundle.substring(tenantStart)); final BundleData bundleData = initializeBundleData(quota, arguments); // Put the bundle data in the new ZooKeeper. @@ -632,9 +609,9 @@ private void read(final String[] args) { // Don't attempt to process blank input. if (args.length > 0 && !(args.length == 1 && args[0].isEmpty())) { final ShellArguments arguments = new ShellArguments(); - final JCommander jc = new JCommander(arguments); + final CommandLine commander = new CommandLine(arguments); try { - jc.parse(args); + commander.parseArgs(args); final String command = arguments.commandArguments.get(0); switch (command) { case "trade": @@ -687,8 +664,8 @@ private void read(final String[] args) { log.info("ERROR: Unknown command \"{}\"", command); } } catch (ParameterException ex) { - ex.printStackTrace(); - jc.usage(); + System.out.println(ex.getMessage()); + commander.usage(commander.getOut()); } catch (Exception ex) { ex.printStackTrace(); } @@ -698,7 +675,7 @@ private void read(final String[] args) { /** * Create a shell for the user to send commands to clients. */ - public void run() throws Exception { + public void start() throws Exception { BufferedReader inReader = new BufferedReader(new InputStreamReader(System.in)); while (true) { // Print the very simple prompt. @@ -711,20 +688,21 @@ public void run() throws Exception { /** * Start a controller with command line arguments. * - * @param args - * Arguments to pass in. */ - public static void main(String[] args) throws Exception { - final MainArguments arguments = new MainArguments(); - final JCommander jc = new JCommander(arguments); - jc.setProgramName("pulsar-perf simulation-controller"); - try { - jc.parse(args); - } catch (Exception ex) { - System.out.println(ex.getMessage()); - jc.usage(); - PerfClientUtils.exit(1); + @Override + public void run() throws Exception { + random = new Random(); + clients = this.clientHostNames.split(","); + final Socket[] sockets = new Socket[clients.length]; + inputStreams = new DataInputStream[clients.length]; + outputStreams = new DataOutputStream[clients.length]; + log.info("Found {} clients:", clients.length); + for (int i = 0; i < clients.length; ++i) { + sockets[i] = new Socket(clients[i], clientPort); + inputStreams[i] = new DataInputStream(sockets[i].getInputStream()); + outputStreams[i] = new DataOutputStream(sockets[i].getOutputStream()); + log.info("Connected to {}", clients[i]); } - (new LoadSimulationController(arguments)).run(); + start(); } } diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/ManagedLedgerWriter.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/ManagedLedgerWriter.java index 336461e7a68d7..8913d17474279 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/ManagedLedgerWriter.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/ManagedLedgerWriter.java @@ -19,10 +19,6 @@ package org.apache.pulsar.testclient; import static java.util.concurrent.TimeUnit.NANOSECONDS; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.Parameters; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.util.concurrent.RateLimiter; @@ -64,8 +60,14 @@ import org.apache.pulsar.testclient.utils.PaddingDecimalFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Model.CommandSpec; +import picocli.CommandLine.Option; +import picocli.CommandLine.Spec; -public class ManagedLedgerWriter { +@Command(name = "managed-ledger", description = "Write directly on managed-ledgers") +public class ManagedLedgerWriter extends CmdBase{ private static final ExecutorService executor = Executors .newCachedThreadPool(new DefaultThreadFactory("pulsar-perf-managed-ledger-exec")); @@ -78,88 +80,74 @@ public class ManagedLedgerWriter { private static Recorder recorder = new Recorder(TimeUnit.SECONDS.toMillis(120000), 5); private static Recorder cumulativeRecorder = new Recorder(TimeUnit.SECONDS.toMillis(120000), 5); - @Parameters(commandDescription = "Write directly on managed-ledgers") - static class Arguments { - @Parameter(names = { "-h", "--help" }, description = "Help message", help = true) - boolean help; + @Option(names = { "-r", "--rate" }, description = "Write rate msg/s across managed ledgers") + public int msgRate = 100; - @Parameter(names = { "-r", "--rate" }, description = "Write rate msg/s across managed ledgers") - public int msgRate = 100; + @Option(names = { "-s", "--size" }, description = "Message size") + public int msgSize = 1024; - @Parameter(names = { "-s", "--size" }, description = "Message size") - public int msgSize = 1024; + @Option(names = { "-t", "--num-topic" }, + description = "Number of managed ledgers", converter = PositiveNumberParameterConvert.class) + public int numManagedLedgers = 1; - @Parameter(names = { "-t", "--num-topic" }, - description = "Number of managed ledgers", validateWith = PositiveNumberParameterValidator.class) - public int numManagedLedgers = 1; + @Option(names = { "--threads" }, + description = "Number of threads writing", converter = PositiveNumberParameterConvert.class) + public int numThreads = 1; - @Parameter(names = { "--threads" }, - description = "Number of threads writing", validateWith = PositiveNumberParameterValidator.class) - public int numThreads = 1; + @Deprecated + @Option(names = {"-zk", "--zookeeperServers"}, + description = "ZooKeeper connection string", + hidden = true) + public String zookeeperServers; - @Deprecated - @Parameter(names = {"-zk", "--zookeeperServers"}, - description = "ZooKeeper connection string", - hidden = true) - public String zookeeperServers; + @Option(names = {"-md", + "--metadata-store"}, description = "Metadata store service URL. For example: zk:my-zk:2181") + private String metadataStoreUrl; - @Parameter(names = {"-md", - "--metadata-store"}, description = "Metadata store service URL. For example: zk:my-zk:2181") - private String metadataStoreUrl; + @Option(names = { "-o", "--max-outstanding" }, description = "Max number of outstanding requests") + public int maxOutstanding = 1000; - @Parameter(names = { "-o", "--max-outstanding" }, description = "Max number of outstanding requests") - public int maxOutstanding = 1000; + @Option(names = { "-c", + "--max-connections" }, description = "Max number of TCP connections to a single bookie") + public int maxConnections = 1; - @Parameter(names = { "-c", - "--max-connections" }, description = "Max number of TCP connections to a single bookie") - public int maxConnections = 1; + @Option(names = { "-m", + "--num-messages" }, + description = "Number of messages to publish in total. If <= 0, it will keep publishing") + public long numMessages = 0; - @Parameter(names = { "-m", - "--num-messages" }, - description = "Number of messages to publish in total. If <= 0, it will keep publishing") - public long numMessages = 0; + @Option(names = { "-e", "--ensemble-size" }, description = "Ledger ensemble size") + public int ensembleSize = 1; - @Parameter(names = { "-e", "--ensemble-size" }, description = "Ledger ensemble size") - public int ensembleSize = 1; + @Option(names = { "-w", "--write-quorum" }, description = "Ledger write quorum") + public int writeQuorum = 1; - @Parameter(names = { "-w", "--write-quorum" }, description = "Ledger write quorum") - public int writeQuorum = 1; + @Option(names = { "-a", "--ack-quorum" }, description = "Ledger ack quorum") + public int ackQuorum = 1; - @Parameter(names = { "-a", "--ack-quorum" }, description = "Ledger ack quorum") - public int ackQuorum = 1; + @Option(names = { "-dt", "--digest-type" }, description = "BookKeeper digest type") + public DigestType digestType = DigestType.CRC32C; - @Parameter(names = { "-dt", "--digest-type" }, description = "BookKeeper digest type") - public DigestType digestType = DigestType.CRC32C; - - @Parameter(names = { "-time", - "--test-duration" }, description = "Test duration in secs. If <= 0, it will keep publishing") - public long testTime = 0; + @Option(names = { "-time", + "--test-duration" }, description = "Test duration in secs. If <= 0, it will keep publishing") + public long testTime = 0; + public ManagedLedgerWriter() { + super("managed-ledger"); } - public static void main(String[] args) throws Exception { - - final Arguments arguments = new Arguments(); - JCommander jc = new JCommander(arguments); - jc.setProgramName("pulsar-perf managed-ledger"); - try { - jc.parse(args); - } catch (ParameterException e) { - System.out.println(e.getMessage()); - jc.usage(); - PerfClientUtils.exit(1); - } + @Spec + CommandSpec spec; - if (arguments.help) { - jc.usage(); - PerfClientUtils.exit(1); - } + @Override + public void run() throws Exception { + CommandLine commander = spec.commandLine(); - if (arguments.metadataStoreUrl == null && arguments.zookeeperServers == null) { + if (this.metadataStoreUrl == null && this.zookeeperServers == null) { System.err.println("Metadata store address argument is required (--metadata-store)"); - jc.usage(); + commander.usage(commander.getOut()); PerfClientUtils.exit(1); } @@ -167,17 +155,17 @@ public static void main(String[] args) throws Exception { PerfClientUtils.printJVMInformation(log); ObjectMapper m = new ObjectMapper(); ObjectWriter w = m.writerWithDefaultPrettyPrinter(); - log.info("Starting Pulsar managed-ledger perf writer with config: {}", w.writeValueAsString(arguments)); + log.info("Starting Pulsar managed-ledger perf writer with config: {}", w.writeValueAsString(this)); - byte[] payloadData = new byte[arguments.msgSize]; - ByteBuf payloadBuffer = PulsarByteBufAllocator.DEFAULT.directBuffer(arguments.msgSize); - payloadBuffer.writerIndex(arguments.msgSize); + byte[] payloadData = new byte[this.msgSize]; + ByteBuf payloadBuffer = PulsarByteBufAllocator.DEFAULT.directBuffer(this.msgSize); + payloadBuffer.writerIndex(this.msgSize); // Now processing command line arguments String managedLedgerPrefix = "test-" + DigestUtils.sha1Hex(UUID.randomUUID().toString()).substring(0, 5); - if (arguments.metadataStoreUrl == null) { - arguments.metadataStoreUrl = arguments.zookeeperServers; + if (this.metadataStoreUrl == null) { + this.metadataStoreUrl = this.zookeeperServers; } ClientConfiguration bkConf = new ClientConfiguration(); @@ -185,31 +173,31 @@ public static void main(String[] args) throws Exception { bkConf.setAddEntryTimeout(30); bkConf.setReadEntryTimeout(30); bkConf.setThrottleValue(0); - bkConf.setNumChannelsPerBookie(arguments.maxConnections); - bkConf.setMetadataServiceUri(arguments.metadataStoreUrl); + bkConf.setNumChannelsPerBookie(this.maxConnections); + bkConf.setMetadataServiceUri(this.metadataStoreUrl); ManagedLedgerFactoryConfig mlFactoryConf = new ManagedLedgerFactoryConfig(); mlFactoryConf.setMaxCacheSize(0); @Cleanup - MetadataStoreExtended metadataStore = MetadataStoreExtended.create(arguments.metadataStoreUrl, + MetadataStoreExtended metadataStore = MetadataStoreExtended.create(this.metadataStoreUrl, MetadataStoreConfig.builder().metadataStoreName(MetadataStoreConfig.METADATA_STORE).build()); ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkConf, mlFactoryConf); ManagedLedgerConfig mlConf = new ManagedLedgerConfig(); - mlConf.setEnsembleSize(arguments.ensembleSize); - mlConf.setWriteQuorumSize(arguments.writeQuorum); - mlConf.setAckQuorumSize(arguments.ackQuorum); + mlConf.setEnsembleSize(this.ensembleSize); + mlConf.setWriteQuorumSize(this.writeQuorum); + mlConf.setAckQuorumSize(this.ackQuorum); mlConf.setMinimumRolloverTime(10, TimeUnit.MINUTES); - mlConf.setMetadataEnsembleSize(arguments.ensembleSize); - mlConf.setMetadataWriteQuorumSize(arguments.writeQuorum); - mlConf.setMetadataAckQuorumSize(arguments.ackQuorum); - mlConf.setDigestType(arguments.digestType); + mlConf.setMetadataEnsembleSize(this.ensembleSize); + mlConf.setMetadataWriteQuorumSize(this.writeQuorum); + mlConf.setMetadataAckQuorumSize(this.ackQuorum); + mlConf.setDigestType(this.digestType); mlConf.setMaxSizePerLedgerMb(2048); List> futures = new ArrayList<>(); - for (int i = 0; i < arguments.numManagedLedgers; i++) { + for (int i = 0; i < this.numManagedLedgers; i++) { String name = String.format("%s-%03d", managedLedgerPrefix, i); CompletableFuture future = new CompletableFuture<>(); futures.add(future); @@ -241,23 +229,23 @@ public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { AtomicBoolean isDone = new AtomicBoolean(); Map> managedLedgersPerThread = allocateToThreads(managedLedgers, - arguments.numThreads); + this.numThreads); - for (int i = 0; i < arguments.numThreads; i++) { + for (int i = 0; i < this.numThreads; i++) { List managedLedgersForThisThread = managedLedgersPerThread.get(i); int nunManagedLedgersForThisThread = managedLedgersForThisThread.size(); - long numMessagesForThisThread = arguments.numMessages / arguments.numThreads; - int maxOutstandingForThisThread = arguments.maxOutstanding; + long numMessagesForThisThread = this.numMessages / this.numThreads; + int maxOutstandingForThisThread = this.maxOutstanding; executor.submit(() -> { try { - final double msgRate = arguments.msgRate / (double) arguments.numThreads; + final double msgRate = this.msgRate / (double) this.numThreads; final RateLimiter rateLimiter = RateLimiter.create(msgRate); // Acquire 1 sec worth of messages to have a slower ramp-up rateLimiter.acquire((int) msgRate); final long startTime = System.nanoTime(); - final long testEndTime = startTime + (long) (arguments.testTime * 1e9); + final long testEndTime = startTime + (long) (this.testTime * 1e9); final Semaphore semaphore = new Semaphore(maxOutstandingForThisThread); @@ -288,10 +276,10 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { long totalSent = 0; while (true) { for (int j = 0; j < nunManagedLedgersForThisThread; j++) { - if (arguments.testTime > 0) { + if (this.testTime > 0) { if (System.nanoTime() > testEndTime) { log.info("------------- DONE (reached the maximum duration: [{} seconds] of " - + "production) --------------", arguments.testTime); + + "production) --------------", this.testTime); isDone.set(true); Thread.sleep(5000); PerfClientUtils.exit(0); diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerfClientUtils.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerfClientUtils.java index f9e5d5ee7e6e1..6bf73e705d16c 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerfClientUtils.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerfClientUtils.java @@ -19,6 +19,7 @@ package org.apache.pulsar.testclient; import static org.apache.commons.lang3.StringUtils.isNotBlank; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk; import java.lang.management.ManagementFactory; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -67,7 +68,7 @@ public static ClientBuilder createClientBuilderFromArguments(PerformanceBaseArgu throws PulsarClientException.UnsupportedAuthenticationException { ClientBuilder clientBuilder = PulsarClient.builder() - .memoryLimit(0, SizeUnit.BYTES) + .memoryLimit(arguments.memoryLimit, SizeUnit.BYTES) .serviceUrl(arguments.serviceURL) .connectionsPerBroker(arguments.maxConnections) .ioThreads(arguments.ioThreads) @@ -76,12 +77,19 @@ public static ClientBuilder createClientBuilderFromArguments(PerformanceBaseArgu .listenerThreads(arguments.listenerThreads) .tlsTrustCertsFilePath(arguments.tlsTrustCertsFilePath) .maxLookupRequests(arguments.maxLookupRequest) - .proxyServiceUrl(arguments.proxyServiceURL, arguments.proxyProtocol); + .proxyServiceUrl(arguments.proxyServiceURL, arguments.proxyProtocol) + .openTelemetry(AutoConfiguredOpenTelemetrySdk.builder() + .build().getOpenTelemetrySdk()); if (isNotBlank(arguments.authPluginClassName)) { clientBuilder.authentication(arguments.authPluginClassName, arguments.authParams); } + if (isNotBlank(arguments.sslfactoryPlugin)) { + clientBuilder.sslFactoryPlugin(arguments.sslfactoryPlugin) + .sslFactoryPluginParams(arguments.sslFactoryPluginParams); + } + if (arguments.tlsAllowInsecureConnection != null) { clientBuilder.allowTlsInsecureConnection(arguments.tlsAllowInsecureConnection); } @@ -108,6 +116,11 @@ public static PulsarAdminBuilder createAdminBuilderFromArguments(PerformanceBase pulsarAdminBuilder.authentication(arguments.authPluginClassName, arguments.authParams); } + if (isNotBlank(arguments.sslfactoryPlugin)) { + pulsarAdminBuilder.sslFactoryPlugin(arguments.sslfactoryPlugin) + .sslFactoryPluginParams(arguments.sslFactoryPluginParams); + } + if (arguments.tlsAllowInsecureConnection != null) { pulsarAdminBuilder.allowTlsInsecureConnection(arguments.tlsAllowInsecureConnection); } diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceBaseArguments.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceBaseArguments.java index 5ae79fb0bf9a4..ee79066c32f90 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceBaseArguments.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceBaseArguments.java @@ -19,213 +19,113 @@ package org.apache.pulsar.testclient; import static org.apache.commons.lang3.StringUtils.isBlank; -import static org.apache.pulsar.testclient.PerfClientUtils.exit; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import java.io.File; -import java.io.FileInputStream; -import java.util.Properties; -import lombok.SneakyThrows; -import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.cli.converters.picocli.ByteUnitToLongConverter; import org.apache.pulsar.client.api.ProxyProtocol; +import picocli.CommandLine.Option; /** * PerformanceBaseArguments contains common CLI arguments and parsing logic available to all sub-commands. * Sub-commands should create Argument subclasses and override the `validate` method as necessary. */ -public abstract class PerformanceBaseArguments { +public abstract class PerformanceBaseArguments extends CmdBase{ - @Parameter(names = { "-h", "--help" }, description = "Print help message", help = true) - boolean help; - @Parameter(names = { "-cf", "--conf-file" }, description = "Pulsar configuration file") - public String confFile; - - @Parameter(names = { "-u", "--service-url" }, description = "Pulsar Service URL") + @Option(names = { "-u", "--service-url" }, description = "Pulsar Service URL", descriptionKey = "brokerServiceUrl") public String serviceURL; - @Parameter(names = { "--auth-plugin" }, description = "Authentication plugin class name") + @Option(names = { "--auth-plugin" }, description = "Authentication plugin class name", + descriptionKey = "authPlugin") public String authPluginClassName; - @Parameter( + @Option( names = { "--auth-params" }, description = "Authentication parameters, whose format is determined by the implementation " + "of method `configure` in authentication plugin class, for example \"key1:val1,key2:val2\" " - + "or \"{\"key1\":\"val1\",\"key2\":\"val2\"}\".") + + "or \"{\"key1\":\"val1\",\"key2\":\"val2\"}\".", descriptionKey = "authParams") public String authParams; - @Parameter(names = { - "--trust-cert-file" }, description = "Path for the trusted TLS certificate file") + @Option(names = { "--ssl-factory-plugin" }, description = "Pulsar SSL Factory plugin class name", + descriptionKey = "sslFactoryPlugin") + public String sslfactoryPlugin; + + @Option(names = { "--ssl-factory-plugin-params" }, + description = "Pulsar SSL Factory Plugin parameters in the format: " + + "\"{\"key1\":\"val1\",\"key2\":\"val2\"}\".", descriptionKey = "sslFactoryPluginParams") + public String sslFactoryPluginParams; + + @Option(names = { + "--trust-cert-file" }, description = "Path for the trusted TLS certificate file", + descriptionKey = "tlsTrustCertsFilePath") public String tlsTrustCertsFilePath = ""; - @Parameter(names = { - "--tls-allow-insecure" }, description = "Allow insecure TLS connection") + @Option(names = { + "--tls-allow-insecure" }, description = "Allow insecure TLS connection", + descriptionKey = "tlsAllowInsecureConnection") public Boolean tlsAllowInsecureConnection = null; - @Parameter(names = { - "--tls-enable-hostname-verification" }, description = "Enable TLS hostname verification") + @Option(names = { + "--tls-enable-hostname-verification" }, description = "Enable TLS hostname verification", + descriptionKey = "tlsEnableHostnameVerification") public Boolean tlsHostnameVerificationEnable = null; - @Parameter(names = { "-c", + @Option(names = { "-c", "--max-connections" }, description = "Max number of TCP connections to a single broker") public int maxConnections = 1; - @Parameter(names = { "-i", + @Option(names = { "-i", "--stats-interval-seconds" }, description = "Statistics Interval Seconds. If 0, statistics will be disabled") public long statsIntervalSeconds = 0; - @Parameter(names = {"-ioThreads", "--num-io-threads"}, description = "Set the number of threads to be " + @Option(names = {"-ioThreads", "--num-io-threads"}, description = "Set the number of threads to be " + "used for handling connections to brokers. The default value is 1.") public int ioThreads = 1; - @Parameter(names = {"-bw", "--busy-wait"}, description = "Enable Busy-Wait on the Pulsar client") + @Option(names = {"-bw", "--busy-wait"}, description = "Enable Busy-Wait on the Pulsar client") public boolean enableBusyWait = false; - @Parameter(names = { "--listener-name" }, description = "Listener name for the broker.") + @Option(names = { "--listener-name" }, description = "Listener name for the broker.") public String listenerName = null; - @Parameter(names = {"-lt", "--num-listener-threads"}, description = "Set the number of threads" + @Option(names = {"-lt", "--num-listener-threads"}, description = "Set the number of threads" + " to be used for message listeners") public int listenerThreads = 1; - @Parameter(names = {"-mlr", "--max-lookup-request"}, description = "Maximum number of lookup requests allowed " + @Option(names = {"-mlr", "--max-lookup-request"}, description = "Maximum number of lookup requests allowed " + "on each broker connection to prevent overloading a broker") public int maxLookupRequest = 50000; - @Parameter(names = { "--proxy-url" }, description = "Proxy-server URL to which to connect.") + @Option(names = { "--proxy-url" }, description = "Proxy-server URL to which to connect.", + descriptionKey = "proxyServiceUrl") String proxyServiceURL = null; - @Parameter(names = { "--proxy-protocol" }, description = "Proxy protocol to select type of routing at proxy.") + @Option(names = { "--proxy-protocol" }, description = "Proxy protocol to select type of routing at proxy.", + descriptionKey = "proxyProtocol", converter = ProxyProtocolConverter.class) ProxyProtocol proxyProtocol = null; - @Parameter(names = { "--auth_plugin" }, description = "Authentication plugin class name", hidden = true) + @Option(names = { "--auth_plugin" }, description = "Authentication plugin class name", hidden = true) public String deprecatedAuthPluginClassName; - public abstract void fillArgumentsFromProperties(Properties prop); - - @SneakyThrows - public void fillArgumentsFromProperties() { - if (confFile == null) { - return; - } - - Properties prop = new Properties(System.getProperties()); - try (FileInputStream fis = new FileInputStream(confFile)) { - prop.load(fis); - } - - if (serviceURL == null) { - serviceURL = prop.getProperty("brokerServiceUrl"); - } - - if (serviceURL == null) { - serviceURL = prop.getProperty("webServiceUrl"); - } - - // fallback to previous-version serviceUrl property to maintain backward-compatibility - if (serviceURL == null) { - serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/"); - } - - if (authPluginClassName == null) { - authPluginClassName = prop.getProperty("authPlugin", null); - } - - if (authParams == null) { - authParams = prop.getProperty("authParams", null); - } - - if (isBlank(tlsTrustCertsFilePath)) { - tlsTrustCertsFilePath = prop.getProperty("tlsTrustCertsFilePath", ""); - } - - if (tlsAllowInsecureConnection == null) { - tlsAllowInsecureConnection = Boolean.parseBoolean(prop - .getProperty("tlsAllowInsecureConnection", "")); - } - - if (tlsHostnameVerificationEnable == null) { - tlsHostnameVerificationEnable = Boolean.parseBoolean(prop - .getProperty("tlsEnableHostnameVerification", "")); - - } - - if (proxyServiceURL == null) { - proxyServiceURL = StringUtils.trimToNull(prop.getProperty("proxyServiceUrl")); - } - - if (proxyProtocol == null) { - String proxyProtocolString = null; - try { - proxyProtocolString = StringUtils.trimToNull(prop.getProperty("proxyProtocol")); - if (proxyProtocolString != null) { - proxyProtocol = ProxyProtocol.valueOf(proxyProtocolString.toUpperCase()); - } - } catch (IllegalArgumentException e) { - System.out.println("Incorrect proxyProtocol name '" + proxyProtocolString + "'"); - e.printStackTrace(); - exit(1); - } - - } - - fillArgumentsFromProperties(prop); + @Option(names = { "-ml", "--memory-limit", }, description = "Configure the Pulsar client memory limit " + + "(eg: 32M, 64M)", converter = ByteUnitToLongConverter.class) + public long memoryLimit; + public PerformanceBaseArguments(String cmdName) { + super(cmdName); } - /** - * Validate the CLI arguments. Default implementation provides validation for the common arguments. - * Each subclass should call super.validate() and provide validation code specific to the sub-command. - * @throws Exception - */ + @Override public void validate() throws Exception { - if (confFile != null && !confFile.isBlank()) { - File configFile = new File(confFile); - if (!configFile.exists()) { - throw new Exception("config file '" + confFile + "', does not exist"); - } - if (configFile.isDirectory()) { - throw new Exception("config file '" + confFile + "', is a directory"); - } - } + parseCLI(); } /** * Parse the command line args. - * @param cmdName used for the help message - * @param args String[] of CLI args * @throws ParameterException If there is a problem parsing the arguments */ - public void parseCLI(String cmdName, String[] args) { - JCommander jc = new JCommander(this); - jc.setProgramName(cmdName); - - try { - jc.parse(args); - } catch (ParameterException e) { - System.out.println("error: " + e.getMessage()); - jc.usage(); - PerfClientUtils.exit(1); - } - - if (help) { - jc.usage(); - PerfClientUtils.exit(0); - } - - fillArgumentsFromProperties(); - + public void parseCLI() { if (isBlank(authPluginClassName) && !isBlank(deprecatedAuthPluginClassName)) { authPluginClassName = deprecatedAuthPluginClassName; } - - try { - validate(); - } catch (Exception e) { - System.out.println("error: " + e.getMessage()); - PerfClientUtils.exit(1); - } } } diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java index 59dabc9302622..5126eefd9ca1e 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java @@ -19,8 +19,6 @@ package org.apache.pulsar.testclient; import static org.apache.commons.lang3.StringUtils.isNotBlank; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.util.concurrent.RateLimiter; @@ -31,7 +29,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Properties; import java.util.concurrent.Future; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; @@ -57,8 +54,11 @@ import org.apache.pulsar.testclient.utils.PaddingDecimalFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; -public class PerformanceConsumer { +@Command(name = "consume", description = "Test pulsar consumer performance.") +public class PerformanceConsumer extends PerformanceTopicListArguments{ private static final LongAdder messagesReceived = new LongAdder(); private static final LongAdder bytesReceived = new LongAdder(); private static final DecimalFormat intFormat = new PaddingDecimalFormat("0", 7); @@ -82,323 +82,319 @@ public class PerformanceConsumer { private static final Recorder recorder = new Recorder(MAX_LATENCY, 5); private static final Recorder cumulativeRecorder = new Recorder(MAX_LATENCY, 5); - @Parameters(commandDescription = "Test pulsar consumer performance.") - static class Arguments extends PerformanceTopicListArguments { + @Option(names = { "-n", "--num-consumers" }, description = "Number of consumers (per subscription), only " + + "one consumer is allowed when subscriptionType is Exclusive", + converter = PositiveNumberParameterConvert.class + ) + public int numConsumers = 1; - @Parameter(names = { "-n", "--num-consumers" }, description = "Number of consumers (per subscription), only " - + "one consumer is allowed when subscriptionType is Exclusive", - validateWith = PositiveNumberParameterValidator.class) - public int numConsumers = 1; + @Option(names = { "-ns", "--num-subscriptions" }, description = "Number of subscriptions (per topic)", + converter = PositiveNumberParameterConvert.class + ) + public int numSubscriptions = 1; - @Parameter(names = { "-ns", "--num-subscriptions" }, description = "Number of subscriptions (per topic)", - validateWith = PositiveNumberParameterValidator.class) - public int numSubscriptions = 1; + @Option(names = { "-s", "--subscriber-name" }, description = "Subscriber name prefix", hidden = true) + public String subscriberName; - @Parameter(names = { "-s", "--subscriber-name" }, description = "Subscriber name prefix", hidden = true) - public String subscriberName; + @Option(names = { "-ss", "--subscriptions" }, + description = "A list of subscriptions to consume (for example, sub1,sub2)") + public List subscriptions = Collections.singletonList("sub"); - @Parameter(names = { "-ss", "--subscriptions" }, - description = "A list of subscriptions to consume (for example, sub1,sub2)") - public List subscriptions = Collections.singletonList("sub"); + @Option(names = { "-st", "--subscription-type" }, description = "Subscription type") + public SubscriptionType subscriptionType = SubscriptionType.Exclusive; - @Parameter(names = { "-st", "--subscription-type" }, description = "Subscription type") - public SubscriptionType subscriptionType = SubscriptionType.Exclusive; + @Option(names = { "-sp", "--subscription-position" }, description = "Subscription position") + private SubscriptionInitialPosition subscriptionInitialPosition = SubscriptionInitialPosition.Latest; - @Parameter(names = { "-sp", "--subscription-position" }, description = "Subscription position") - private SubscriptionInitialPosition subscriptionInitialPosition = SubscriptionInitialPosition.Latest; + @Option(names = { "-r", "--rate" }, description = "Simulate a slow message consumer (rate in msg/s)") + public double rate = 0; - @Parameter(names = { "-r", "--rate" }, description = "Simulate a slow message consumer (rate in msg/s)") - public double rate = 0; + @Option(names = { "-q", "--receiver-queue-size" }, description = "Size of the receiver queue") + public int receiverQueueSize = 1000; - @Parameter(names = { "-q", "--receiver-queue-size" }, description = "Size of the receiver queue") - public int receiverQueueSize = 1000; + @Option(names = { "-p", "--receiver-queue-size-across-partitions" }, + description = "Max total size of the receiver queue across partitions") + public int maxTotalReceiverQueueSizeAcrossPartitions = 50000; - @Parameter(names = { "-p", "--receiver-queue-size-across-partitions" }, - description = "Max total size of the receiver queue across partitions") - public int maxTotalReceiverQueueSizeAcrossPartitions = 50000; + @Option(names = {"-aq", "--auto-scaled-receiver-queue-size"}, + description = "Enable autoScaledReceiverQueueSize") + public boolean autoScaledReceiverQueueSize = false; - @Parameter(names = {"-aq", "--auto-scaled-receiver-queue-size"}, - description = "Enable autoScaledReceiverQueueSize") - public boolean autoScaledReceiverQueueSize = false; + @Option(names = {"-rs", "--replicated" }, + description = "Whether the subscription status should be replicated") + public boolean replicatedSubscription = false; - @Parameter(names = {"-rs", "--replicated" }, - description = "Whether the subscription status should be replicated") - public boolean replicatedSubscription = false; + @Option(names = { "--acks-delay-millis" }, description = "Acknowledgements grouping delay in millis") + public int acknowledgmentsGroupingDelayMillis = 100; - @Parameter(names = { "--acks-delay-millis" }, description = "Acknowledgements grouping delay in millis") - public int acknowledgmentsGroupingDelayMillis = 100; + @Option(names = {"-m", + "--num-messages"}, + description = "Number of messages to consume in total. If <= 0, it will keep consuming") + public long numMessages = 0; - @Parameter(names = {"-m", - "--num-messages"}, - description = "Number of messages to consume in total. If <= 0, it will keep consuming") - public long numMessages = 0; + @Option(names = { "-mc", "--max_chunked_msg" }, description = "Max pending chunk messages") + private int maxPendingChunkedMessage = 0; - @Parameter(names = { "-mc", "--max_chunked_msg" }, description = "Max pending chunk messages") - private int maxPendingChunkedMessage = 0; + @Option(names = { "-ac", + "--auto_ack_chunk_q_full" }, description = "Auto ack for oldest message on queue is full") + private boolean autoAckOldestChunkedMessageOnQueueFull = false; - @Parameter(names = { "-ac", - "--auto_ack_chunk_q_full" }, description = "Auto ack for oldest message on queue is full") - private boolean autoAckOldestChunkedMessageOnQueueFull = false; + @Option(names = { "-e", + "--expire_time_incomplete_chunked_messages" }, + description = "Expire time in ms for incomplete chunk messages") + private long expireTimeOfIncompleteChunkedMessageMs = 0; - @Parameter(names = { "-e", - "--expire_time_incomplete_chunked_messages" }, - description = "Expire time in ms for incomplete chunk messages") - private long expireTimeOfIncompleteChunkedMessageMs = 0; + @Option(names = { "-v", + "--encryption-key-value-file" }, + description = "The file which contains the private key to decrypt payload") + public String encKeyFile = null; - @Parameter(names = { "-v", - "--encryption-key-value-file" }, - description = "The file which contains the private key to decrypt payload") - public String encKeyFile = null; + @Option(names = { "-time", + "--test-duration" }, description = "Test duration in secs. If <= 0, it will keep consuming") + public long testTime = 0; - @Parameter(names = { "-time", - "--test-duration" }, description = "Test duration in secs. If <= 0, it will keep consuming") - public long testTime = 0; + @Option(names = {"--batch-index-ack" }, description = "Enable or disable the batch index acknowledgment") + public boolean batchIndexAck = false; - @Parameter(names = {"--batch-index-ack" }, description = "Enable or disable the batch index acknowledgment") - public boolean batchIndexAck = false; + @Option(names = { "-pm", "--pool-messages" }, description = "Use the pooled message", arity = "1") + private boolean poolMessages = true; - @Parameter(names = { "-pm", "--pool-messages" }, description = "Use the pooled message", arity = 1) - private boolean poolMessages = true; + @Option(names = {"-tto", "--txn-timeout"}, description = "Set the time value of transaction timeout," + + " and the time unit is second. (After --txn-enable setting to true, --txn-timeout takes effect)") + public long transactionTimeout = 10; - @Parameter(names = {"-tto", "--txn-timeout"}, description = "Set the time value of transaction timeout," - + " and the time unit is second. (After --txn-enable setting to true, --txn-timeout takes effect)") - public long transactionTimeout = 10; + @Option(names = {"-nmt", "--numMessage-perTransaction"}, + description = "The number of messages acknowledged by a transaction. " + + "(After --txn-enable setting to true, -numMessage-perTransaction takes effect") + public int numMessagesPerTransaction = 50; - @Parameter(names = {"-nmt", "--numMessage-perTransaction"}, - description = "The number of messages acknowledged by a transaction. " - + "(After --txn-enable setting to true, -numMessage-perTransaction takes effect") - public int numMessagesPerTransaction = 50; + @Option(names = {"-txn", "--txn-enable"}, description = "Enable or disable the transaction") + public boolean isEnableTransaction = false; - @Parameter(names = {"-txn", "--txn-enable"}, description = "Enable or disable the transaction") - public boolean isEnableTransaction = false; + @Option(names = {"-ntxn"}, description = "The number of opened transactions, 0 means keeping open." + + "(After --txn-enable setting to true, -ntxn takes effect.)") + public long totalNumTxn = 0; - @Parameter(names = {"-ntxn"}, description = "The number of opened transactions, 0 means keeping open." - + "(After --txn-enable setting to true, -ntxn takes effect.)") - public long totalNumTxn = 0; + @Option(names = {"-abort"}, description = "Abort the transaction. (After --txn-enable " + + "setting to true, -abort takes effect)") + public boolean isAbortTransaction = false; - @Parameter(names = {"-abort"}, description = "Abort the transaction. (After --txn-enable " - + "setting to true, -abort takes effect)") - public boolean isAbortTransaction = false; + @Option(names = { "--histogram-file" }, description = "HdrHistogram output file") + public String histogramFile = null; - @Parameter(names = { "--histogram-file" }, description = "HdrHistogram output file") - public String histogramFile = null; + public PerformanceConsumer() { + super("consume"); + } - @Override - public void fillArgumentsFromProperties(Properties prop) { - } - @Override - public void validate() throws Exception { - super.validate(); - if (subscriptionType == SubscriptionType.Exclusive && numConsumers > 1) { - throw new Exception("Only one consumer is allowed when subscriptionType is Exclusive"); - } + @Override + public void validate() throws Exception { + super.validate(); + if (subscriptionType == SubscriptionType.Exclusive && numConsumers > 1) { + throw new Exception("Only one consumer is allowed when subscriptionType is Exclusive"); + } - if (subscriptions != null && subscriptions.size() != numSubscriptions) { - // keep compatibility with the previous version - if (subscriptions.size() == 1) { - if (subscriberName == null) { - subscriberName = subscriptions.get(0); - } - List defaultSubscriptions = new ArrayList<>(); - for (int i = 0; i < numSubscriptions; i++) { - defaultSubscriptions.add(String.format("%s-%d", subscriberName, i)); - } - subscriptions = defaultSubscriptions; - } else { - throw new Exception("The size of subscriptions list should be equal to --num-subscriptions"); + if (subscriptions != null && subscriptions.size() != numSubscriptions) { + // keep compatibility with the previous version + if (subscriptions.size() == 1) { + if (subscriberName == null) { + subscriberName = subscriptions.get(0); } + List defaultSubscriptions = new ArrayList<>(); + for (int i = 0; i < numSubscriptions; i++) { + defaultSubscriptions.add(String.format("%s-%d", subscriberName, i)); + } + subscriptions = defaultSubscriptions; + } else { + throw new Exception("The size of subscriptions list should be equal to --num-subscriptions"); } } } - - public static void main(String[] args) throws Exception { - final Arguments arguments = new Arguments(); - arguments.parseCLI("pulsar-perf consume", args); - + @Override + public void run() throws Exception { // Dump config variables PerfClientUtils.printJVMInformation(log); ObjectMapper m = new ObjectMapper(); ObjectWriter w = m.writerWithDefaultPrettyPrinter(); - log.info("Starting Pulsar performance consumer with config: {}", w.writeValueAsString(arguments)); + log.info("Starting Pulsar performance consumer with config: {}", w.writeValueAsString(this)); - final Recorder qRecorder = arguments.autoScaledReceiverQueueSize - ? new Recorder(arguments.receiverQueueSize, 5) : null; - final RateLimiter limiter = arguments.rate > 0 ? RateLimiter.create(arguments.rate) : null; + final Recorder qRecorder = this.autoScaledReceiverQueueSize + ? new Recorder(this.receiverQueueSize, 5) : null; + final RateLimiter limiter = this.rate > 0 ? RateLimiter.create(this.rate) : null; long startTime = System.nanoTime(); - long testEndTime = startTime + (long) (arguments.testTime * 1e9); + long testEndTime = startTime + (long) (this.testTime * 1e9); - ClientBuilder clientBuilder = PerfClientUtils.createClientBuilderFromArguments(arguments) - .enableTransaction(arguments.isEnableTransaction); + ClientBuilder clientBuilder = PerfClientUtils.createClientBuilderFromArguments(this) + .enableTransaction(this.isEnableTransaction); PulsarClient pulsarClient = clientBuilder.build(); AtomicReference atomicReference; - if (arguments.isEnableTransaction) { + if (this.isEnableTransaction) { atomicReference = new AtomicReference<>(pulsarClient.newTransaction() - .withTransactionTimeout(arguments.transactionTimeout, TimeUnit.SECONDS).build().get()); + .withTransactionTimeout(this.transactionTimeout, TimeUnit.SECONDS).build().get()); } else { atomicReference = new AtomicReference<>(null); } AtomicLong messageAckedCount = new AtomicLong(); - Semaphore messageReceiveLimiter = new Semaphore(arguments.numMessagesPerTransaction); + Semaphore messageReceiveLimiter = new Semaphore(this.numMessagesPerTransaction); Thread thread = Thread.currentThread(); MessageListener listener = (consumer, msg) -> { - if (arguments.testTime > 0) { - if (System.nanoTime() > testEndTime) { - log.info("------------------- DONE -----------------------"); - PerfClientUtils.exit(0); - thread.interrupt(); - } - } - if (arguments.totalNumTxn > 0) { - if (totalEndTxnOpFailNum.sum() + totalEndTxnOpSuccessNum.sum() >= arguments.totalNumTxn) { - log.info("------------------- DONE -----------------------"); - PerfClientUtils.exit(0); - thread.interrupt(); - } - } - if (qRecorder != null) { - qRecorder.recordValue(((ConsumerBase) consumer).getTotalIncomingMessages()); + if (this.testTime > 0) { + if (System.nanoTime() > testEndTime) { + log.info("------------------- DONE -----------------------"); + PerfClientUtils.exit(0); + thread.interrupt(); } - messagesReceived.increment(); - bytesReceived.add(msg.size()); - - totalMessagesReceived.increment(); - totalBytesReceived.add(msg.size()); - - if (arguments.numMessages > 0 && totalMessagesReceived.sum() >= arguments.numMessages) { + } + if (this.totalNumTxn > 0) { + if (totalEndTxnOpFailNum.sum() + totalEndTxnOpSuccessNum.sum() >= this.totalNumTxn) { log.info("------------------- DONE -----------------------"); PerfClientUtils.exit(0); thread.interrupt(); } + } + if (qRecorder != null) { + qRecorder.recordValue(((ConsumerBase) consumer).getTotalIncomingMessages()); + } + messagesReceived.increment(); + bytesReceived.add(msg.size()); - if (limiter != null) { - limiter.acquire(); - } + totalMessagesReceived.increment(); + totalBytesReceived.add(msg.size()); - long latencyMillis = System.currentTimeMillis() - msg.getPublishTime(); - if (latencyMillis >= 0) { - if (latencyMillis >= MAX_LATENCY) { - latencyMillis = MAX_LATENCY; - } - recorder.recordValue(latencyMillis); - cumulativeRecorder.recordValue(latencyMillis); - } - if (arguments.isEnableTransaction) { - try { - messageReceiveLimiter.acquire(); - } catch (InterruptedException e){ - log.error("Got error: ", e); - } - consumer.acknowledgeAsync(msg.getMessageId(), atomicReference.get()).thenRun(() -> { - totalMessageAck.increment(); - messageAck.increment(); - }).exceptionally(throwable ->{ - log.error("Ack message {} failed with exception", msg, throwable); - totalMessageAckFailed.increment(); - return null; - }); - } else { - consumer.acknowledgeAsync(msg).thenRun(()->{ - totalMessageAck.increment(); - messageAck.increment(); - } - ).exceptionally(throwable ->{ - log.error("Ack message {} failed with exception", msg, throwable); - totalMessageAckFailed.increment(); - return null; - } - ); + if (this.numMessages > 0 && totalMessagesReceived.sum() >= this.numMessages) { + log.info("------------------- DONE -----------------------"); + PerfClientUtils.exit(0); + thread.interrupt(); + } + + if (limiter != null) { + limiter.acquire(); + } + + long latencyMillis = System.currentTimeMillis() - msg.getPublishTime(); + if (latencyMillis >= 0) { + if (latencyMillis >= MAX_LATENCY) { + latencyMillis = MAX_LATENCY; } - if (arguments.poolMessages) { - msg.release(); + recorder.recordValue(latencyMillis); + cumulativeRecorder.recordValue(latencyMillis); + } + if (this.isEnableTransaction) { + try { + messageReceiveLimiter.acquire(); + } catch (InterruptedException e){ + log.error("Got error: ", e); } - if (arguments.isEnableTransaction - && messageAckedCount.incrementAndGet() == arguments.numMessagesPerTransaction) { - Transaction transaction = atomicReference.get(); - if (!arguments.isAbortTransaction) { - transaction.commit() - .thenRun(() -> { - if (log.isDebugEnabled()) { - log.debug("Commit transaction {}", transaction.getTxnID()); - } - totalEndTxnOpSuccessNum.increment(); - numTxnOpSuccess.increment(); - }) - .exceptionally(exception -> { - log.error("Commit transaction failed with exception : ", exception); - totalEndTxnOpFailNum.increment(); - return null; - }); - } else { - transaction.abort().thenRun(() -> { - if (log.isDebugEnabled()) { - log.debug("Abort transaction {}", transaction.getTxnID()); - } - totalEndTxnOpSuccessNum.increment(); - numTxnOpSuccess.increment(); - }).exceptionally(exception -> { - log.error("Abort transaction {} failed with exception", - transaction.getTxnID().toString(), - exception); - totalEndTxnOpFailNum.increment(); + consumer.acknowledgeAsync(msg.getMessageId(), atomicReference.get()).thenRun(() -> { + totalMessageAck.increment(); + messageAck.increment(); + }).exceptionally(throwable ->{ + log.error("Ack message {} failed with exception", msg, throwable); + totalMessageAckFailed.increment(); + return null; + }); + } else { + consumer.acknowledgeAsync(msg).thenRun(()->{ + totalMessageAck.increment(); + messageAck.increment(); + } + ).exceptionally(throwable ->{ + log.error("Ack message {} failed with exception", msg, throwable); + totalMessageAckFailed.increment(); return null; - }); - } - while (true) { - try { - Transaction newTransaction = pulsarClient.newTransaction() - .withTransactionTimeout(arguments.transactionTimeout, TimeUnit.SECONDS) - .build().get(); - atomicReference.compareAndSet(transaction, newTransaction); - totalNumTxnOpenSuccess.increment(); - messageAckedCount.set(0); - messageReceiveLimiter.release(arguments.numMessagesPerTransaction); - break; - } catch (Exception e) { - log.error("Failed to new transaction with exception:", e); - totalNumTxnOpenFail.increment(); } + ); + } + if (this.poolMessages) { + msg.release(); + } + if (this.isEnableTransaction + && messageAckedCount.incrementAndGet() == this.numMessagesPerTransaction) { + Transaction transaction = atomicReference.get(); + if (!this.isAbortTransaction) { + transaction.commit() + .thenRun(() -> { + if (log.isDebugEnabled()) { + log.debug("Commit transaction {}", transaction.getTxnID()); + } + totalEndTxnOpSuccessNum.increment(); + numTxnOpSuccess.increment(); + }) + .exceptionally(exception -> { + log.error("Commit transaction failed with exception : ", exception); + totalEndTxnOpFailNum.increment(); + return null; + }); + } else { + transaction.abort().thenRun(() -> { + if (log.isDebugEnabled()) { + log.debug("Abort transaction {}", transaction.getTxnID()); + } + totalEndTxnOpSuccessNum.increment(); + numTxnOpSuccess.increment(); + }).exceptionally(exception -> { + log.error("Abort transaction {} failed with exception", + transaction.getTxnID().toString(), + exception); + totalEndTxnOpFailNum.increment(); + return null; + }); + } + while (true) { + try { + Transaction newTransaction = pulsarClient.newTransaction() + .withTransactionTimeout(this.transactionTimeout, TimeUnit.SECONDS) + .build().get(); + atomicReference.compareAndSet(transaction, newTransaction); + totalNumTxnOpenSuccess.increment(); + messageAckedCount.set(0); + messageReceiveLimiter.release(this.numMessagesPerTransaction); + break; + } catch (Exception e) { + log.error("Failed to new transaction with exception:", e); + totalNumTxnOpenFail.increment(); } } + } }; List>> futures = new ArrayList<>(); ConsumerBuilder consumerBuilder = pulsarClient.newConsumer(Schema.BYTEBUFFER) // .messageListener(listener) // - .receiverQueueSize(arguments.receiverQueueSize) // - .maxTotalReceiverQueueSizeAcrossPartitions(arguments.maxTotalReceiverQueueSizeAcrossPartitions) - .acknowledgmentGroupTime(arguments.acknowledgmentsGroupingDelayMillis, TimeUnit.MILLISECONDS) // - .subscriptionType(arguments.subscriptionType) - .subscriptionInitialPosition(arguments.subscriptionInitialPosition) - .autoAckOldestChunkedMessageOnQueueFull(arguments.autoAckOldestChunkedMessageOnQueueFull) - .enableBatchIndexAcknowledgment(arguments.batchIndexAck) - .poolMessages(arguments.poolMessages) - .replicateSubscriptionState(arguments.replicatedSubscription) - .autoScaledReceiverQueueSizeEnabled(arguments.autoScaledReceiverQueueSize); - if (arguments.maxPendingChunkedMessage > 0) { - consumerBuilder.maxPendingChunkedMessage(arguments.maxPendingChunkedMessage); + .receiverQueueSize(this.receiverQueueSize) // + .maxTotalReceiverQueueSizeAcrossPartitions(this.maxTotalReceiverQueueSizeAcrossPartitions) + .acknowledgmentGroupTime(this.acknowledgmentsGroupingDelayMillis, TimeUnit.MILLISECONDS) // + .subscriptionType(this.subscriptionType) + .subscriptionInitialPosition(this.subscriptionInitialPosition) + .autoAckOldestChunkedMessageOnQueueFull(this.autoAckOldestChunkedMessageOnQueueFull) + .enableBatchIndexAcknowledgment(this.batchIndexAck) + .poolMessages(this.poolMessages) + .replicateSubscriptionState(this.replicatedSubscription) + .autoScaledReceiverQueueSizeEnabled(this.autoScaledReceiverQueueSize); + if (this.maxPendingChunkedMessage > 0) { + consumerBuilder.maxPendingChunkedMessage(this.maxPendingChunkedMessage); } - if (arguments.expireTimeOfIncompleteChunkedMessageMs > 0) { - consumerBuilder.expireTimeOfIncompleteChunkedMessage(arguments.expireTimeOfIncompleteChunkedMessageMs, + if (this.expireTimeOfIncompleteChunkedMessageMs > 0) { + consumerBuilder.expireTimeOfIncompleteChunkedMessage(this.expireTimeOfIncompleteChunkedMessageMs, TimeUnit.MILLISECONDS); } - if (isNotBlank(arguments.encKeyFile)) { - consumerBuilder.defaultCryptoKeyReader(arguments.encKeyFile); + if (isNotBlank(this.encKeyFile)) { + consumerBuilder.defaultCryptoKeyReader(this.encKeyFile); } - for (int i = 0; i < arguments.numTopics; i++) { - final TopicName topicName = TopicName.get(arguments.topics.get(i)); + for (int i = 0; i < this.numTopics; i++) { + final TopicName topicName = TopicName.get(this.topics.get(i)); - log.info("Adding {} consumers per subscription on topic {}", arguments.numConsumers, topicName); + log.info("Adding {} consumers per subscription on topic {}", this.numConsumers, topicName); - for (int j = 0; j < arguments.numSubscriptions; j++) { - String subscriberName = arguments.subscriptions.get(j); - for (int k = 0; k < arguments.numConsumers; k++) { + for (int j = 0; j < this.numSubscriptions; j++) { + String subscriberName = this.subscriptions.get(j); + for (int k = 0; k < this.numConsumers; k++) { futures.add(consumerBuilder.clone().topic(topicName.toString()).subscriptionName(subscriberName) .subscribeAsync()); } @@ -407,13 +403,13 @@ public static void main(String[] args) throws Exception { for (Future> future : futures) { future.get(); } - log.info("Start receiving from {} consumers per subscription on {} topics", arguments.numConsumers, - arguments.numTopics); + log.info("Start receiving from {} consumers per subscription on {} topics", this.numConsumers, + this.numTopics); long start = System.nanoTime(); Runtime.getRuntime().addShutdownHook(new Thread(() -> { - printAggregatedThroughput(start, arguments); + printAggregatedThroughput(start); printAggregatedStats(); })); @@ -424,8 +420,8 @@ public static void main(String[] args) throws Exception { Histogram qHistogram = null; HistogramLogWriter histogramLogWriter = null; - if (arguments.histogramFile != null) { - String statsFileName = arguments.histogramFile; + if (this.histogramFile != null) { + String statsFileName = this.histogramFile; log.info("Dumping latency stats to {}", statsFileName); PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false); @@ -454,7 +450,7 @@ public static void main(String[] args) throws Exception { double rateOpenTxn = 0; reportHistogram = recorder.getIntervalHistogram(reportHistogram); - if (arguments.isEnableTransaction) { + if (this.isEnableTransaction) { totalTxnOpSuccessNum = totalEndTxnOpSuccessNum.sum(); totalTxnOpFailNum = totalEndTxnOpFailNum.sum(); rateOpenTxn = numTxnOpSuccess.sumThenReset() / elapsed; @@ -475,7 +471,7 @@ public static void main(String[] args) throws Exception { reportHistogram.getValueAtPercentile(99), reportHistogram.getValueAtPercentile(99.9), reportHistogram.getValueAtPercentile(99.99), reportHistogram.getMaxValue()); - if (arguments.autoScaledReceiverQueueSize && log.isDebugEnabled() && qRecorder != null) { + if (this.autoScaledReceiverQueueSize && log.isDebugEnabled() && qRecorder != null) { qHistogram = qRecorder.getIntervalHistogram(qHistogram); log.debug("ReceiverQueueUsage: cnt={},mean={}, min={},max={},25pct={},50pct={},75pct={}", qHistogram.getTotalCount(), dec.format(qHistogram.getMean()), @@ -503,12 +499,20 @@ public static void main(String[] args) throws Exception { reportHistogram.reset(); oldTime = now; + + if (this.testTime > 0) { + if (now > testEndTime) { + log.info("------------------- DONE -----------------------"); + PerfClientUtils.exit(0); + thread.interrupt(); + } + } } pulsarClient.close(); } - private static void printAggregatedThroughput(long start, Arguments arguments) { + private void printAggregatedThroughput(long start) { double elapsed = (System.nanoTime() - start) / 1e9; double rate = totalMessagesReceived.sum() / elapsed; double throughput = totalBytesReceived.sum() / elapsed * 8 / 1024 / 1024; @@ -519,7 +523,7 @@ private static void printAggregatedThroughput(long start, Arguments arguments) { long totalnumMessageAckFailed = 0; double rateAck = totalMessageAck.sum() / elapsed; double rateOpenTxn = 0; - if (arguments.isEnableTransaction) { + if (this.isEnableTransaction) { totalEndTxnSuccess = totalEndTxnOpSuccessNum.sum(); totalEndTxnFail = totalEndTxnOpFailNum.sum(); rateOpenTxn = (totalEndTxnSuccess + totalEndTxnFail) / elapsed; diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java index 6513f0684b243..ba5be3a3c4566 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java @@ -18,16 +18,15 @@ */ package org.apache.pulsar.testclient; +import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.NANOSECONDS; -import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_BATCHING_MAX_MESSAGES; import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_MAX_PENDING_MESSAGES; import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; +import com.google.common.collect.Range; import com.google.common.util.concurrent.RateLimiter; import io.netty.util.concurrent.DefaultThreadFactory; import java.io.FileOutputStream; @@ -40,7 +39,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Properties; import java.util.Random; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -72,11 +70,16 @@ import org.apache.pulsar.testclient.utils.PaddingDecimalFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import picocli.CommandLine.Command; +import picocli.CommandLine.ITypeConverter; +import picocli.CommandLine.Option; +import picocli.CommandLine.TypeConversionException; /** * A client program to test pulsar producer performance. */ -public class PerformanceProducer { +@Command(name = "produce", description = "Test pulsar producer performance.") +public class PerformanceProducer extends PerformanceTopicListArguments{ private static final ExecutorService executor = Executors .newCachedThreadPool(new DefaultThreadFactory("pulsar-perf-producer-exec")); @@ -100,185 +103,171 @@ public class PerformanceProducer { private static IMessageFormatter messageFormatter = null; - @Parameters(commandDescription = "Test pulsar producer performance.") - static class Arguments extends PerformanceTopicListArguments { + @Option(names = { "-threads", "--num-test-threads" }, description = "Number of test threads", + converter = PositiveNumberParameterConvert.class + ) + public int numTestThreads = 1; - @Parameter(names = { "-threads", "--num-test-threads" }, description = "Number of test threads", - validateWith = PositiveNumberParameterValidator.class) - public int numTestThreads = 1; + @Option(names = { "-r", "--rate" }, description = "Publish rate msg/s across topics") + public int msgRate = 100; - @Parameter(names = { "-r", "--rate" }, description = "Publish rate msg/s across topics") - public int msgRate = 100; + @Option(names = { "-s", "--size" }, description = "Message size (bytes)") + public int msgSize = 1024; - @Parameter(names = { "-s", "--size" }, description = "Message size (bytes)") - public int msgSize = 1024; + @Option(names = { "-n", "--num-producers" }, description = "Number of producers (per topic)", + converter = PositiveNumberParameterConvert.class + ) + public int numProducers = 1; - @Parameter(names = { "-n", "--num-producers" }, description = "Number of producers (per topic)", - validateWith = PositiveNumberParameterValidator.class) - public int numProducers = 1; + @Option(names = {"--separator"}, description = "Separator between the topic and topic number") + public String separator = "-"; - @Parameter(names = {"--separator"}, description = "Separator between the topic and topic number") - public String separator = "-"; + @Option(names = {"--send-timeout"}, description = "Set the sendTimeout value default 0 to keep " + + "compatibility with previous version of pulsar-perf") + public int sendTimeout = 0; - @Parameter(names = {"--send-timeout"}, description = "Set the sendTimeout value default 0 to keep " - + "compatibility with previous version of pulsar-perf") - public int sendTimeout = 0; + @Option(names = { "-pn", "--producer-name" }, description = "Producer Name") + public String producerName = null; - @Parameter(names = { "-pn", "--producer-name" }, description = "Producer Name") - public String producerName = null; + @Option(names = { "-au", "--admin-url" }, description = "Pulsar Admin URL", descriptionKey = "webServiceUrl") + public String adminURL; - @Parameter(names = { "-au", "--admin-url" }, description = "Pulsar Admin URL") - public String adminURL; + @Option(names = { "-ch", + "--chunking" }, description = "Should split the message and publish in chunks if message size is " + + "larger than allowed max size") + private boolean chunkingAllowed = false; - @Parameter(names = { "-ch", - "--chunking" }, description = "Should split the message and publish in chunks if message size is " - + "larger than allowed max size") - private boolean chunkingAllowed = false; + @Option(names = { "-o", "--max-outstanding" }, description = "Max number of outstanding messages") + public int maxOutstanding = DEFAULT_MAX_PENDING_MESSAGES; - @Parameter(names = { "-o", "--max-outstanding" }, description = "Max number of outstanding messages") - public int maxOutstanding = DEFAULT_MAX_PENDING_MESSAGES; + @Option(names = { "-p", "--max-outstanding-across-partitions" }, description = "Max number of outstanding " + + "messages across partitions") + public int maxPendingMessagesAcrossPartitions = DEFAULT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS; - @Parameter(names = { "-p", "--max-outstanding-across-partitions" }, description = "Max number of outstanding " - + "messages across partitions") - public int maxPendingMessagesAcrossPartitions = DEFAULT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS; + @Option(names = { "-np", "--partitions" }, description = "Create partitioned topics with the given number " + + "of partitions, set 0 to not try to create the topic") + public Integer partitions = null; - @Parameter(names = { "-np", "--partitions" }, description = "Create partitioned topics with the given number " - + "of partitions, set 0 to not try to create the topic") - public Integer partitions = null; + @Option(names = { "-m", + "--num-messages" }, description = "Number of messages to publish in total. If <= 0, it will keep " + + "publishing") + public long numMessages = 0; - @Parameter(names = { "-m", - "--num-messages" }, description = "Number of messages to publish in total. If <= 0, it will keep " - + "publishing") - public long numMessages = 0; + @Option(names = { "-z", "--compression" }, description = "Compress messages payload") + public CompressionType compression = CompressionType.NONE; - @Parameter(names = { "-z", "--compression" }, description = "Compress messages payload") - public CompressionType compression = CompressionType.NONE; + @Option(names = { "-f", "--payload-file" }, description = "Use payload from an UTF-8 encoded text file and " + + "a payload will be randomly selected when publishing messages") + public String payloadFilename = null; - @Parameter(names = { "-f", "--payload-file" }, description = "Use payload from an UTF-8 encoded text file and " - + "a payload will be randomly selected when publishing messages") - public String payloadFilename = null; + @Option(names = { "-e", "--payload-delimiter" }, description = "The delimiter used to split lines when " + + "using payload from a file") + // here escaping \n since default value will be printed with the help text + public String payloadDelimiter = "\\n"; - @Parameter(names = { "-e", "--payload-delimiter" }, description = "The delimiter used to split lines when " - + "using payload from a file") - // here escaping \n since default value will be printed with the help text - public String payloadDelimiter = "\\n"; + @Option(names = { "-b", + "--batch-time-window" }, description = "Batch messages in 'x' ms window (Default: 1ms)") + public double batchTimeMillis = 1.0; - @Parameter(names = { "-b", - "--batch-time-window" }, description = "Batch messages in 'x' ms window (Default: 1ms)") - public double batchTimeMillis = 1.0; + @Option(names = { "-db", + "--disable-batching" }, description = "Disable batching if true") + public boolean disableBatching; - @Parameter(names = { "-db", - "--disable-batching" }, description = "Disable batching if true") - public boolean disableBatching; - - @Parameter(names = { + @Option(names = { "-bm", "--batch-max-messages" - }, description = "Maximum number of messages per batch") - public int batchMaxMessages = DEFAULT_BATCHING_MAX_MESSAGES; + }, description = "Maximum number of messages per batch") + public int batchMaxMessages = DEFAULT_BATCHING_MAX_MESSAGES; - @Parameter(names = { + @Option(names = { "-bb", "--batch-max-bytes" - }, description = "Maximum number of bytes per batch") - public int batchMaxBytes = 4 * 1024 * 1024; + }, description = "Maximum number of bytes per batch") + public int batchMaxBytes = 4 * 1024 * 1024; - @Parameter(names = { "-time", - "--test-duration" }, description = "Test duration in secs. If <= 0, it will keep publishing") - public long testTime = 0; + @Option(names = { "-time", + "--test-duration" }, description = "Test duration in secs. If <= 0, it will keep publishing") + public long testTime = 0; - @Parameter(names = "--warmup-time", description = "Warm-up time in seconds (Default: 1 sec)") - public double warmupTimeSeconds = 1.0; + @Option(names = "--warmup-time", description = "Warm-up time in seconds (Default: 1 sec)") + public double warmupTimeSeconds = 1.0; - @Parameter(names = { "-k", "--encryption-key-name" }, description = "The public key name to encrypt payload") - public String encKeyName = null; + @Option(names = { "-k", "--encryption-key-name" }, description = "The public key name to encrypt payload") + public String encKeyName = null; - @Parameter(names = { "-v", - "--encryption-key-value-file" }, - description = "The file which contains the public key to encrypt payload") - public String encKeyFile = null; + @Option(names = { "-v", + "--encryption-key-value-file" }, + description = "The file which contains the public key to encrypt payload") + public String encKeyFile = null; - @Parameter(names = { "-d", - "--delay" }, description = "Mark messages with a given delay in seconds") - public long delay = 0; + @Option(names = { "-d", + "--delay" }, description = "Mark messages with a given delay in seconds") + public long delay = 0; - @Parameter(names = { "-set", - "--set-event-time" }, description = "Set the eventTime on messages") - public boolean setEventTime = false; + @Option(names = { "-dr", "--delay-range"}, description = "Mark messages with a given delay by a random" + + " number of seconds. this value between the specified origin (inclusive) and the specified bound" + + " (exclusive). e.g. 1,300", converter = RangeConvert.class) + public Range delayRange = null; - @Parameter(names = { "-ef", - "--exit-on-failure" }, description = "Exit from the process on publish failure (default: disable)") - public boolean exitOnFailure = false; + @Option(names = { "-set", + "--set-event-time" }, description = "Set the eventTime on messages") + public boolean setEventTime = false; - @Parameter(names = {"-mk", "--message-key-generation-mode"}, description = "The generation mode of message key" - + ", valid options are: [autoIncrement, random]") - public String messageKeyGenerationMode = null; + @Option(names = { "-ef", + "--exit-on-failure" }, description = "Exit from the process on publish failure (default: disable)") + public boolean exitOnFailure = false; - @Parameter(names = { "-am", "--access-mode" }, description = "Producer access mode") - public ProducerAccessMode producerAccessMode = ProducerAccessMode.Shared; + @Option(names = {"-mk", "--message-key-generation-mode"}, description = "The generation mode of message key" + + ", valid options are: [autoIncrement, random]", descriptionKey = "messageKeyGenerationMode") + public String messageKeyGenerationMode = null; - @Parameter(names = { "-fp", "--format-payload" }, - description = "Format %i as a message index in the stream from producer and/or %t as the timestamp" - + " nanoseconds.") - public boolean formatPayload = false; + @Option(names = { "-am", "--access-mode" }, description = "Producer access mode") + public ProducerAccessMode producerAccessMode = ProducerAccessMode.Shared; - @Parameter(names = {"-fc", "--format-class"}, description = "Custom Formatter class name") - public String formatterClass = "org.apache.pulsar.testclient.DefaultMessageFormatter"; + @Option(names = { "-fp", "--format-payload" }, + description = "Format %%i as a message index in the stream from producer and/or %%t as the timestamp" + + " nanoseconds.") + public boolean formatPayload = false; - @Parameter(names = {"-tto", "--txn-timeout"}, description = "Set the time value of transaction timeout," - + " and the time unit is second. (After --txn-enable setting to true, --txn-timeout takes effect)") - public long transactionTimeout = 10; + @Option(names = {"-fc", "--format-class"}, description = "Custom Formatter class name") + public String formatterClass = "org.apache.pulsar.testclient.DefaultMessageFormatter"; - @Parameter(names = {"-nmt", "--numMessage-perTransaction"}, - description = "The number of messages sent by a transaction. " - + "(After --txn-enable setting to true, -nmt takes effect)") - public int numMessagesPerTransaction = 50; + @Option(names = {"-tto", "--txn-timeout"}, description = "Set the time value of transaction timeout," + + " and the time unit is second. (After --txn-enable setting to true, --txn-timeout takes effect)") + public long transactionTimeout = 10; - @Parameter(names = {"-txn", "--txn-enable"}, description = "Enable or disable the transaction") - public boolean isEnableTransaction = false; + @Option(names = {"-nmt", "--numMessage-perTransaction"}, + description = "The number of messages sent by a transaction. " + + "(After --txn-enable setting to true, -nmt takes effect)") + public int numMessagesPerTransaction = 50; - @Parameter(names = {"-abort"}, description = "Abort the transaction. (After --txn-enable " - + "setting to true, -abort takes effect)") - public boolean isAbortTransaction = false; + @Option(names = {"-txn", "--txn-enable"}, description = "Enable or disable the transaction") + public boolean isEnableTransaction = false; - @Parameter(names = { "--histogram-file" }, description = "HdrHistogram output file") - public String histogramFile = null; + @Option(names = {"-abort"}, description = "Abort the transaction. (After --txn-enable " + + "setting to true, -abort takes effect)") + public boolean isAbortTransaction = false; - @Override - public void fillArgumentsFromProperties(Properties prop) { - if (adminURL == null) { - adminURL = prop.getProperty("webServiceUrl"); - } - if (adminURL == null) { - adminURL = prop.getProperty("adminURL", "http://localhost:8080/"); - } - - if (isBlank(messageKeyGenerationMode)) { - messageKeyGenerationMode = prop.getProperty("messageKeyGenerationMode", null); - } - } - } - - public static void main(String[] args) throws Exception { - - final Arguments arguments = new Arguments(); - arguments.parseCLI("pulsar-perf produce", args); + @Option(names = { "--histogram-file" }, description = "HdrHistogram output file") + public String histogramFile = null; + @Override + public void run() throws Exception { // Dump config variables PerfClientUtils.printJVMInformation(log); ObjectMapper m = new ObjectMapper(); ObjectWriter w = m.writerWithDefaultPrettyPrinter(); - log.info("Starting Pulsar perf producer with config: {}", w.writeValueAsString(arguments)); + log.info("Starting Pulsar perf producer with config: {}", w.writeValueAsString(this)); // Read payload data from file if needed - final byte[] payloadBytes = new byte[arguments.msgSize]; + final byte[] payloadBytes = new byte[msgSize]; Random random = new Random(0); List payloadByteList = new ArrayList<>(); - if (arguments.payloadFilename != null) { - Path payloadFilePath = Paths.get(arguments.payloadFilename); + if (this.payloadFilename != null) { + Path payloadFilePath = Paths.get(this.payloadFilename); if (Files.notExists(payloadFilePath) || Files.size(payloadFilePath) == 0) { throw new IllegalArgumentException("Payload file doesn't exist or it is empty."); } // here escaping the default payload delimiter to correct value - String delimiter = arguments.payloadDelimiter.equals("\\n") ? "\n" : arguments.payloadDelimiter; + String delimiter = this.payloadDelimiter.equals("\\n") ? "\n" : this.payloadDelimiter; String[] payloadList = new String(Files.readAllBytes(payloadFilePath), StandardCharsets.UTF_8).split(delimiter); log.info("Reading payloads from {} and {} records read", payloadFilePath.toAbsolutePath(), @@ -287,8 +276,8 @@ public static void main(String[] args) throws Exception { payloadByteList.add(payload.getBytes(StandardCharsets.UTF_8)); } - if (arguments.formatPayload) { - messageFormatter = getMessageFormatter(arguments.formatterClass); + if (this.formatPayload) { + messageFormatter = getMessageFormatter(this.formatterClass); } } else { for (int i = 0; i < payloadBytes.length; ++i) { @@ -300,29 +289,29 @@ public static void main(String[] args) throws Exception { Runtime.getRuntime().addShutdownHook(new Thread(() -> { executorShutdownNow(); - printAggregatedThroughput(start, arguments); + printAggregatedThroughput(start); printAggregatedStats(); })); - if (arguments.partitions != null) { + if (this.partitions != null) { final PulsarAdminBuilder adminBuilder = PerfClientUtils - .createAdminBuilderFromArguments(arguments, arguments.adminURL); + .createAdminBuilderFromArguments(this, this.adminURL); try (PulsarAdmin adminClient = adminBuilder.build()) { - for (String topic : arguments.topics) { - log.info("Creating partitioned topic {} with {} partitions", topic, arguments.partitions); + for (String topic : this.topics) { + log.info("Creating partitioned topic {} with {} partitions", topic, this.partitions); try { - adminClient.topics().createPartitionedTopic(topic, arguments.partitions); + adminClient.topics().createPartitionedTopic(topic, this.partitions); } catch (PulsarAdminException.ConflictException alreadyExists) { if (log.isDebugEnabled()) { log.debug("Topic {} already exists: {}", topic, alreadyExists); } PartitionedTopicMetadata partitionedTopicMetadata = adminClient.topics() .getPartitionedTopicMetadata(topic); - if (partitionedTopicMetadata.partitions != arguments.partitions) { + if (partitionedTopicMetadata.partitions != this.partitions) { log.error("Topic {} already exists but it has a wrong number of partitions: {}, " + "expecting {}", - topic, partitionedTopicMetadata.partitions, arguments.partitions); + topic, partitionedTopicMetadata.partitions, this.partitions); PerfClientUtils.exit(1); } } @@ -330,23 +319,23 @@ public static void main(String[] args) throws Exception { } } - CountDownLatch doneLatch = new CountDownLatch(arguments.numTestThreads); + CountDownLatch doneLatch = new CountDownLatch(this.numTestThreads); - final long numMessagesPerThread = arguments.numMessages / arguments.numTestThreads; - final int msgRatePerThread = arguments.msgRate / arguments.numTestThreads; + final long numMessagesPerThread = this.numMessages / this.numTestThreads; + final int msgRatePerThread = this.msgRate / this.numTestThreads; - for (int i = 0; i < arguments.numTestThreads; i++) { + for (int i = 0; i < this.numTestThreads; i++) { final int threadIdx = i; executor.submit(() -> { log.info("Started performance test thread {}", threadIdx); runProducer( - threadIdx, - arguments, - numMessagesPerThread, - msgRatePerThread, - payloadByteList, - payloadBytes, - doneLatch + threadIdx, + this, + numMessagesPerThread, + msgRatePerThread, + payloadByteList, + payloadBytes, + doneLatch ); }); } @@ -357,8 +346,8 @@ public static void main(String[] args) throws Exception { Histogram reportHistogram = null; HistogramLogWriter histogramLogWriter = null; - if (arguments.histogramFile != null) { - String statsFileName = arguments.histogramFile; + if (this.histogramFile != null) { + String statsFileName = this.histogramFile; log.info("Dumping latency stats to {}", statsFileName); PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false); @@ -392,7 +381,7 @@ public static void main(String[] args) throws Exception { reportHistogram = recorder.getIntervalHistogram(reportHistogram); - if (arguments.isEnableTransaction) { + if (this.isEnableTransaction) { totalTxnOpSuccess = totalEndTxnOpSuccessNum.sum(); totalTxnOpFail = totalEndTxnOpFailNum.sum(); rateOpenTxn = numTxnOpSuccess.sumThenReset() / elapsed; @@ -424,6 +413,9 @@ public static void main(String[] args) throws Exception { oldTime = now; } } + public PerformanceProducer() { + super("produce"); + } private static void executorShutdownNow() { executor.shutdownNow(); @@ -447,86 +439,87 @@ static IMessageFormatter getMessageFormatter(String formatterClass) { } } - static ProducerBuilder createProducerBuilder(PulsarClient client, Arguments arguments, int producerId) { + ProducerBuilder createProducerBuilder(PulsarClient client, int producerId) { ProducerBuilder producerBuilder = client.newProducer() // - .sendTimeout(arguments.sendTimeout, TimeUnit.SECONDS) // - .compressionType(arguments.compression) // - .maxPendingMessages(arguments.maxOutstanding) // - .accessMode(arguments.producerAccessMode) + .sendTimeout(this.sendTimeout, TimeUnit.SECONDS) // + .compressionType(this.compression) // + .maxPendingMessages(this.maxOutstanding) // + .accessMode(this.producerAccessMode) // enable round robin message routing if it is a partitioned topic .messageRoutingMode(MessageRoutingMode.RoundRobinPartition); - if (arguments.maxPendingMessagesAcrossPartitions > 0) { - producerBuilder.maxPendingMessagesAcrossPartitions(arguments.maxPendingMessagesAcrossPartitions); + if (this.maxPendingMessagesAcrossPartitions > 0) { + producerBuilder.maxPendingMessagesAcrossPartitions(this.maxPendingMessagesAcrossPartitions); } - if (arguments.producerName != null) { - String producerName = String.format("%s%s%d", arguments.producerName, arguments.separator, producerId); + if (this.producerName != null) { + String producerName = String.format("%s%s%d", this.producerName, this.separator, producerId); producerBuilder.producerName(producerName); } - if (arguments.disableBatching || (arguments.batchTimeMillis <= 0.0 && arguments.batchMaxMessages <= 0)) { + if (this.disableBatching || (this.batchTimeMillis <= 0.0 && this.batchMaxMessages <= 0)) { producerBuilder.enableBatching(false); } else { - long batchTimeUsec = (long) (arguments.batchTimeMillis * 1000); + long batchTimeUsec = (long) (this.batchTimeMillis * 1000); producerBuilder.batchingMaxPublishDelay(batchTimeUsec, TimeUnit.MICROSECONDS).enableBatching(true); } - if (arguments.batchMaxMessages > 0) { - producerBuilder.batchingMaxMessages(arguments.batchMaxMessages); + if (this.batchMaxMessages > 0) { + producerBuilder.batchingMaxMessages(this.batchMaxMessages); } - if (arguments.batchMaxBytes > 0) { - producerBuilder.batchingMaxBytes(arguments.batchMaxBytes); + if (this.batchMaxBytes > 0) { + producerBuilder.batchingMaxBytes(this.batchMaxBytes); } // Block if queue is full else we will start seeing errors in sendAsync producerBuilder.blockIfQueueFull(true); - if (isNotBlank(arguments.encKeyName) && isNotBlank(arguments.encKeyFile)) { - producerBuilder.addEncryptionKey(arguments.encKeyName); - producerBuilder.defaultCryptoKeyReader(arguments.encKeyFile); + if (isNotBlank(this.encKeyName) && isNotBlank(this.encKeyFile)) { + producerBuilder.addEncryptionKey(this.encKeyName); + producerBuilder.defaultCryptoKeyReader(this.encKeyFile); } return producerBuilder; } - private static void runProducer(int producerId, - Arguments arguments, + private void runProducer(int producerId, + PerformanceProducer arguments, long numMessages, int msgRate, List payloadByteList, byte[] payloadBytes, CountDownLatch doneLatch) { PulsarClient client = null; + boolean produceEnough = false; try { // Now processing command line arguments List>> futures = new ArrayList<>(); ClientBuilder clientBuilder = PerfClientUtils.createClientBuilderFromArguments(arguments) - .enableTransaction(arguments.isEnableTransaction); + .enableTransaction(this.isEnableTransaction); client = clientBuilder.build(); - ProducerBuilder producerBuilder = createProducerBuilder(client, arguments, producerId); + ProducerBuilder producerBuilder = createProducerBuilder(client, producerId); AtomicReference transactionAtomicReference; - if (arguments.isEnableTransaction) { + if (this.isEnableTransaction) { producerBuilder.sendTimeout(0, TimeUnit.SECONDS); transactionAtomicReference = new AtomicReference<>(client.newTransaction() - .withTransactionTimeout(arguments.transactionTimeout, TimeUnit.SECONDS) + .withTransactionTimeout(this.transactionTimeout, TimeUnit.SECONDS) .build() .get()); } else { transactionAtomicReference = new AtomicReference<>(null); } - for (int i = 0; i < arguments.numTopics; i++) { + for (int i = 0; i < this.numTopics; i++) { - String topic = arguments.topics.get(i); - log.info("Adding {} publishers on topic {}", arguments.numProducers, topic); + String topic = this.topics.get(i); + log.info("Adding {} publishers on topic {}", this.numProducers, topic); - for (int j = 0; j < arguments.numProducers; j++) { + for (int j = 0; j < this.numProducers; j++) { ProducerBuilder prodBuilder = producerBuilder.clone().topic(topic); - if (arguments.chunkingAllowed) { + if (this.chunkingAllowed) { prodBuilder.enableChunking(true); prodBuilder.enableBatching(false); } @@ -545,39 +538,42 @@ private static void runProducer(int producerId, RateLimiter rateLimiter = RateLimiter.create(msgRate); long startTime = System.nanoTime(); - long warmupEndTime = startTime + (long) (arguments.warmupTimeSeconds * 1e9); - long testEndTime = startTime + (long) (arguments.testTime * 1e9); + long warmupEndTime = startTime + (long) (this.warmupTimeSeconds * 1e9); + long testEndTime = startTime + (long) (this.testTime * 1e9); MessageKeyGenerationMode msgKeyMode = null; - if (isNotBlank(arguments.messageKeyGenerationMode)) { + if (isNotBlank(this.messageKeyGenerationMode)) { try { - msgKeyMode = MessageKeyGenerationMode.valueOf(arguments.messageKeyGenerationMode); + msgKeyMode = MessageKeyGenerationMode.valueOf(this.messageKeyGenerationMode); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("messageKeyGenerationMode only support [autoIncrement, random]"); } } // Send messages on all topics/producers - long totalSent = 0; + AtomicLong totalSent = new AtomicLong(0); AtomicLong numMessageSend = new AtomicLong(0); - Semaphore numMsgPerTxnLimit = new Semaphore(arguments.numMessagesPerTransaction); + Semaphore numMsgPerTxnLimit = new Semaphore(this.numMessagesPerTransaction); while (true) { + if (produceEnough) { + break; + } for (Producer producer : producers) { - if (arguments.testTime > 0) { + if (this.testTime > 0) { if (System.nanoTime() > testEndTime) { log.info("------------- DONE (reached the maximum duration: [{} seconds] of production) " - + "--------------", arguments.testTime); + + "--------------", this.testTime); doneLatch.countDown(); - Thread.sleep(5000); - PerfClientUtils.exit(0); + produceEnough = true; + break; } } if (numMessages > 0) { - if (totalSent++ >= numMessages) { + if (totalSent.get() >= numMessages) { log.info("------------- DONE (reached the maximum number: {} of production) --------------" , numMessages); doneLatch.countDown(); - Thread.sleep(5000); - PerfClientUtils.exit(0); + produceEnough = true; + break; } } rateLimiter.acquire(); @@ -587,9 +583,9 @@ private static void runProducer(int producerId, byte[] payloadData; - if (arguments.payloadFilename != null) { + if (this.payloadFilename != null) { if (messageFormatter != null) { - payloadData = messageFormatter.formatMessage(arguments.producerName, totalSent, + payloadData = messageFormatter.formatMessage(this.producerName, totalSent.get(), payloadByteList.get(ThreadLocalRandom.current().nextInt(payloadByteList.size()))); } else { payloadData = payloadByteList.get( @@ -599,8 +595,8 @@ private static void runProducer(int producerId, payloadData = payloadBytes; } TypedMessageBuilder messageBuilder; - if (arguments.isEnableTransaction) { - if (arguments.numMessagesPerTransaction > 0) { + if (this.isEnableTransaction) { + if (this.numMessagesPerTransaction > 0) { try { numMsgPerTxnLimit.acquire(); } catch (InterruptedException exception){ @@ -613,23 +609,27 @@ private static void runProducer(int producerId, messageBuilder = producer.newMessage() .value(payloadData); } - if (arguments.delay > 0) { - messageBuilder.deliverAfter(arguments.delay, TimeUnit.SECONDS); + if (this.delay > 0) { + messageBuilder.deliverAfter(this.delay, TimeUnit.SECONDS); + } else if (this.delayRange != null) { + final long deliverAfter = ThreadLocalRandom.current() + .nextLong(this.delayRange.lowerEndpoint(), this.delayRange.upperEndpoint()); + messageBuilder.deliverAfter(deliverAfter, TimeUnit.SECONDS); } - if (arguments.setEventTime) { + if (this.setEventTime) { messageBuilder.eventTime(System.currentTimeMillis()); } //generate msg key if (msgKeyMode == MessageKeyGenerationMode.random) { messageBuilder.key(String.valueOf(ThreadLocalRandom.current().nextInt())); } else if (msgKeyMode == MessageKeyGenerationMode.autoIncrement) { - messageBuilder.key(String.valueOf(totalSent)); + messageBuilder.key(String.valueOf(totalSent.get())); } PulsarClient pulsarClient = client; messageBuilder.sendAsync().thenRun(() -> { bytesSent.add(payloadData.length); messagesSent.increment(); - + totalSent.incrementAndGet(); totalMessagesSent.increment(); totalBytesSent.add(payloadData.length); @@ -647,14 +647,14 @@ private static void runProducer(int producerId, } log.warn("Write message error with exception", ex); messagesFailed.increment(); - if (arguments.exitOnFailure) { + if (this.exitOnFailure) { PerfClientUtils.exit(1); } return null; }); - if (arguments.isEnableTransaction - && numMessageSend.incrementAndGet() == arguments.numMessagesPerTransaction) { - if (!arguments.isAbortTransaction) { + if (this.isEnableTransaction + && numMessageSend.incrementAndGet() == this.numMessagesPerTransaction) { + if (!this.isAbortTransaction) { transaction.commit() .thenRun(() -> { if (log.isDebugEnabled()) { @@ -688,11 +688,11 @@ private static void runProducer(int producerId, while (true) { try { Transaction newTransaction = pulsarClient.newTransaction() - .withTransactionTimeout(arguments.transactionTimeout, + .withTransactionTimeout(this.transactionTimeout, TimeUnit.SECONDS).build().get(); transactionAtomicReference.compareAndSet(transaction, newTransaction); numMessageSend.set(0); - numMsgPerTxnLimit.release(arguments.numMessagesPerTransaction); + numMsgPerTxnLimit.release(this.numMessagesPerTransaction); totalNumTxnOpenTxnSuccess.increment(); break; } catch (Exception e){ @@ -706,10 +706,12 @@ private static void runProducer(int producerId, } catch (Throwable t) { log.error("Got error", t); } finally { + if (!produceEnough) { + doneLatch.countDown(); + } if (null != client) { try { client.close(); - PerfClientUtils.exit(1); } catch (PulsarClientException e) { log.error("Failed to close test client", e); } @@ -717,7 +719,7 @@ private static void runProducer(int producerId, } } - private static void printAggregatedThroughput(long start, Arguments arguments) { + private void printAggregatedThroughput(long start) { double elapsed = (System.nanoTime() - start) / 1e9; double rate = totalMessagesSent.sum() / elapsed; double throughput = totalBytesSent.sum() / elapsed / 1024 / 1024 * 8; @@ -727,7 +729,7 @@ private static void printAggregatedThroughput(long start, Arguments arguments) { long numTransactionOpenFailed = 0; long numTransactionOpenSuccess = 0; - if (arguments.isEnableTransaction) { + if (this.isEnableTransaction) { totalTxnSuccess = totalEndTxnOpSuccessNum.sum(); totalTxnFail = totalEndTxnOpFailNum.sum(); rateOpenTxn = elapsed / (totalTxnFail + totalTxnSuccess); @@ -774,4 +776,21 @@ private static void printAggregatedStats() { public enum MessageKeyGenerationMode { autoIncrement, random } + + static class RangeConvert implements ITypeConverter> { + @Override + public Range convert(String rangeStr) { + try { + requireNonNull(rangeStr); + final String[] facts = rangeStr.split(","); + final long min = Long.parseLong(facts[0].trim()); + final long max = Long.parseLong(facts[1].trim()); + return Range.closedOpen(min, max); + } catch (Throwable ex) { + throw new TypeConversionException("Unknown delay range interval," + + " the format should be \",\". error message: " + rangeStr); + } + } + } + } diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java index ed5cc37644a31..3c6940b262f44 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java @@ -18,15 +18,12 @@ */ package org.apache.pulsar.testclient; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.util.concurrent.RateLimiter; import java.text.DecimalFormat; import java.util.ArrayList; import java.util.List; -import java.util.Properties; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.CompletableFuture; @@ -46,8 +43,11 @@ import org.apache.pulsar.testclient.utils.PaddingDecimalFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; -public class PerformanceReader { +@Command(name = "read", description = "Test pulsar reader performance.") +public class PerformanceReader extends PerformanceTopicListArguments { private static final LongAdder messagesReceived = new LongAdder(); private static final LongAdder bytesReceived = new LongAdder(); private static final DecimalFormat intFormat = new PaddingDecimalFormat("0", 7); @@ -59,62 +59,53 @@ public class PerformanceReader { private static Recorder recorder = new Recorder(TimeUnit.DAYS.toMillis(10), 5); private static Recorder cumulativeRecorder = new Recorder(TimeUnit.DAYS.toMillis(10), 5); - @Parameters(commandDescription = "Test pulsar reader performance.") - static class Arguments extends PerformanceTopicListArguments { + @Option(names = {"-r", "--rate"}, description = "Simulate a slow message reader (rate in msg/s)") + public double rate = 0; - @Parameter(names = { "-r", "--rate" }, description = "Simulate a slow message reader (rate in msg/s)") - public double rate = 0; + @Option(names = {"-m", + "--start-message-id"}, description = "Start message id. This can be either 'earliest', " + + "'latest' or a specific message id by using 'lid:eid'") + public String startMessageId = "earliest"; - @Parameter(names = { "-m", - "--start-message-id" }, description = "Start message id. This can be either 'earliest', " - + "'latest' or a specific message id by using 'lid:eid'") - public String startMessageId = "earliest"; + @Option(names = {"-q", "--receiver-queue-size"}, description = "Size of the receiver queue") + public int receiverQueueSize = 1000; - @Parameter(names = { "-q", "--receiver-queue-size" }, description = "Size of the receiver queue") - public int receiverQueueSize = 1000; + @Option(names = {"-n", + "--num-messages"}, description = "Number of messages to consume in total. If <= 0, " + + "it will keep consuming") + public long numMessages = 0; - @Parameter(names = {"-n", - "--num-messages"}, description = "Number of messages to consume in total. If <= 0, " - + "it will keep consuming") - public long numMessages = 0; + @Option(names = { + "--use-tls"}, description = "Use TLS encryption on the connection", descriptionKey = "useTls") + public boolean useTls; - @Parameter(names = { - "--use-tls" }, description = "Use TLS encryption on the connection") - public boolean useTls; - - @Parameter(names = { "-time", - "--test-duration" }, description = "Test duration in secs. If <= 0, it will keep consuming") - public long testTime = 0; + @Option(names = {"-time", + "--test-duration"}, description = "Test duration in secs. If <= 0, it will keep consuming") + public long testTime = 0; + public PerformanceReader() { + super("read"); + } - @Override - public void fillArgumentsFromProperties(Properties prop) { - if (!useTls) { - useTls = Boolean.parseBoolean(prop.getProperty("useTls")); - } - } - @Override - public void validate() throws Exception { - super.validate(); - if (startMessageId != "earliest" && startMessageId != "latest" - && (startMessageId.split(":")).length != 2) { - String errMsg = String.format("invalid start message ID '%s', must be either either 'earliest', " - + "'latest' or a specific message id by using 'lid:eid'", startMessageId); - throw new Exception(errMsg); - } + @Override + public void validate() throws Exception { + super.validate(); + if (startMessageId != "earliest" && startMessageId != "latest" + && (startMessageId.split(":")).length != 2) { + String errMsg = String.format("invalid start message ID '%s', must be either either 'earliest', " + + "'latest' or a specific message id by using 'lid:eid'", startMessageId); + throw new Exception(errMsg); } } - public static void main(String[] args) throws Exception { - final Arguments arguments = new Arguments(); - arguments.parseCLI("pulsar-perf read", args); - + @Override + public void run() throws Exception { // Dump config variables PerfClientUtils.printJVMInformation(log); ObjectMapper m = new ObjectMapper(); ObjectWriter w = m.writerWithDefaultPrettyPrinter(); - log.info("Starting Pulsar performance reader with config: {}", w.writeValueAsString(arguments)); + log.info("Starting Pulsar performance reader with config: {}", w.writeValueAsString(this)); - final RateLimiter limiter = arguments.rate > 0 ? RateLimiter.create(arguments.rate) : null; + final RateLimiter limiter = this.rate > 0 ? RateLimiter.create(this.rate) : null; ReaderListener listener = (reader, msg) -> { messagesReceived.increment(); bytesReceived.add(msg.getData().length); @@ -122,9 +113,9 @@ public static void main(String[] args) throws Exception { totalMessagesReceived.increment(); totalBytesReceived.add(msg.getData().length); - if (arguments.numMessages > 0 && totalMessagesReceived.sum() >= arguments.numMessages) { + if (this.numMessages > 0 && totalMessagesReceived.sum() >= this.numMessages) { log.info("------------- DONE (reached the maximum number: [{}] of consumption) --------------", - arguments.numMessages); + this.numMessages); PerfClientUtils.exit(0); } @@ -139,37 +130,37 @@ public static void main(String[] args) throws Exception { } }; - ClientBuilder clientBuilder = PerfClientUtils.createClientBuilderFromArguments(arguments) - .enableTls(arguments.useTls); + ClientBuilder clientBuilder = PerfClientUtils.createClientBuilderFromArguments(this) + .enableTls(this.useTls); PulsarClient pulsarClient = clientBuilder.build(); List>> futures = new ArrayList<>(); MessageId startMessageId; - if ("earliest".equals(arguments.startMessageId)) { + if ("earliest".equals(this.startMessageId)) { startMessageId = MessageId.earliest; - } else if ("latest".equals(arguments.startMessageId)) { + } else if ("latest".equals(this.startMessageId)) { startMessageId = MessageId.latest; } else { - String[] parts = arguments.startMessageId.split(":"); + String[] parts = this.startMessageId.split(":"); startMessageId = new MessageIdImpl(Long.parseLong(parts[0]), Long.parseLong(parts[1]), -1); } ReaderBuilder readerBuilder = pulsarClient.newReader() // .readerListener(listener) // - .receiverQueueSize(arguments.receiverQueueSize) // + .receiverQueueSize(this.receiverQueueSize) // .startMessageId(startMessageId); - for (int i = 0; i < arguments.numTopics; i++) { - final TopicName topicName = TopicName.get(arguments.topics.get(i)); + for (int i = 0; i < this.numTopics; i++) { + final TopicName topicName = TopicName.get(this.topics.get(i)); futures.add(readerBuilder.clone().topic(topicName.toString()).createAsync()); } FutureUtil.waitForAll(futures).get(); - log.info("Start reading from {} topics", arguments.numTopics); + log.info("Start reading from {} topics", this.numTopics); final long start = System.nanoTime(); Runtime.getRuntime().addShutdownHook(new Thread(() -> { @@ -177,17 +168,17 @@ public static void main(String[] args) throws Exception { printAggregatedStats(); })); - if (arguments.testTime > 0) { + if (this.testTime > 0) { TimerTask timoutTask = new TimerTask() { @Override public void run() { log.info("------------- DONE (reached the maximum duration: [{} seconds] of consumption) " - + "--------------", arguments.testTime); + + "--------------", testTime); PerfClientUtils.exit(0); } }; Timer timer = new Timer(); - timer.schedule(timoutTask, arguments.testTime * 1000); + timer.schedule(timoutTask, this.testTime * 1000); } long oldTime = System.nanoTime(); @@ -222,7 +213,6 @@ public void run() { pulsarClient.close(); } - private static void printAggregatedThroughput(long start) { double elapsed = (System.nanoTime() - start) / 1e9; double rate = totalMessagesReceived.sum() / elapsed; diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTopicListArguments.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTopicListArguments.java index a2f8b6af08282..e4771c3652fb1 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTopicListArguments.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTopicListArguments.java @@ -18,10 +18,11 @@ */ package org.apache.pulsar.testclient; -import com.beust.jcommander.Parameter; import java.util.ArrayList; import java.util.List; import org.apache.pulsar.common.naming.TopicName; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; /** * PerformanceTopicListArguments provides common topic list arguments which are used @@ -29,14 +30,19 @@ */ public abstract class PerformanceTopicListArguments extends PerformanceBaseArguments { - @Parameter(description = "persistent://prop/ns/my-topic", required = true) + @Parameters(description = "persistent://prop/ns/my-topic", arity = "1") public List topics; - @Parameter(names = { "-t", "--num-topics", "--num-topic" }, description = "Number of topics. Must match" + @Option(names = { "-t", "--num-topics", "--num-topic" }, description = "Number of topics. Must match" + "the given number of topic arguments.", - validateWith = PositiveNumberParameterValidator.class) + converter = PositiveNumberParameterConvert.class + ) public int numTopics = 1; + public PerformanceTopicListArguments(String cmdName) { + super(cmdName); + } + @Override public void validate() throws Exception { super.validate(); diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java index 469e6ab1f3fd6..943cfaf451032 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceTransaction.java @@ -19,20 +19,16 @@ package org.apache.pulsar.testclient; import static java.util.concurrent.TimeUnit.NANOSECONDS; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.util.concurrent.RateLimiter; import java.io.FileOutputStream; -import java.io.IOException; import java.io.PrintStream; import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.List; -import java.util.Properties; import java.util.Random; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -65,8 +61,11 @@ import org.apache.pulsar.testclient.utils.PaddingDecimalFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; -public class PerformanceTransaction { +@Command(name = "transaction", description = "Test pulsar transaction performance.") +public class PerformanceTransaction extends PerformanceBaseArguments{ private static final LongAdder totalNumEndTxnOpFailed = new LongAdder(); private static final LongAdder totalNumEndTxnOpSuccess = new LongAdder(); @@ -89,132 +88,120 @@ public class PerformanceTransaction { private static final Recorder messageSendRCumulativeRecorder = new Recorder(TimeUnit.SECONDS.toMicros(120000), 5); - @Parameters(commandDescription = "Test pulsar transaction performance.") - static class Arguments extends PerformanceBaseArguments { + @Option(names = "--topics-c", description = "All topics that need ack for a transaction", required = + true) + public List consumerTopic = Collections.singletonList("test-consume"); - @Parameter(names = "--topics-c", description = "All topics that need ack for a transaction", required = - true) - public List consumerTopic = Collections.singletonList("test-consume"); + @Option(names = "--topics-p", description = "All topics that need produce for a transaction", + required = true) + public List producerTopic = Collections.singletonList("test-produce"); - @Parameter(names = "--topics-p", description = "All topics that need produce for a transaction", - required = true) - public List producerTopic = Collections.singletonList("test-produce"); + @Option(names = {"-threads", "--num-test-threads"}, description = "Number of test threads." + + "This thread is for a new transaction to ack messages from consumer topics and produce message to " + + "producer topics, and then commit or abort this transaction. " + + "Increasing the number of threads increases the parallelism of the performance test, " + + "thereby increasing the intensity of the stress test.") + public int numTestThreads = 1; - @Parameter(names = {"-threads", "--num-test-threads"}, description = "Number of test threads." - + "This thread is for a new transaction to ack messages from consumer topics and produce message to " - + "producer topics, and then commit or abort this transaction. " - + "Increasing the number of threads increases the parallelism of the performance test, " - + "thereby increasing the intensity of the stress test.") - public int numTestThreads = 1; + @Option(names = {"-au", "--admin-url"}, description = "Pulsar Admin URL", descriptionKey = "webServiceUrl") + public String adminURL; - @Parameter(names = {"-au", "--admin-url"}, description = "Pulsar Admin URL") - public String adminURL; + @Option(names = {"-np", + "--partitions"}, description = "Create partitioned topics with a given number of partitions, 0 means" + + "not trying to create a topic") + public Integer partitions = null; - @Parameter(names = {"-np", - "--partitions"}, description = "Create partitioned topics with a given number of partitions, 0 means" - + "not trying to create a topic") - public Integer partitions = null; + @Option(names = {"-time", + "--test-duration"}, description = "Test duration (in second). 0 means keeping publishing") + public long testTime = 0; - @Parameter(names = {"-time", - "--test-duration"}, description = "Test duration (in second). 0 means keeping publishing") - public long testTime = 0; + @Option(names = {"-ss", + "--subscriptions"}, description = "A list of subscriptions to consume (for example, sub1,sub2)") + public List subscriptions = Collections.singletonList("sub"); - @Parameter(names = {"-ss", - "--subscriptions"}, description = "A list of subscriptions to consume (for example, sub1,sub2)") - public List subscriptions = Collections.singletonList("sub"); + @Option(names = {"-ns", "--num-subscriptions"}, description = "Number of subscriptions (per topic)") + public int numSubscriptions = 1; - @Parameter(names = {"-ns", "--num-subscriptions"}, description = "Number of subscriptions (per topic)") - public int numSubscriptions = 1; + @Option(names = {"-sp", "--subscription-position"}, description = "Subscription position") + private SubscriptionInitialPosition subscriptionInitialPosition = SubscriptionInitialPosition.Earliest; - @Parameter(names = {"-sp", "--subscription-position"}, description = "Subscription position") - private SubscriptionInitialPosition subscriptionInitialPosition = SubscriptionInitialPosition.Earliest; + @Option(names = {"-st", "--subscription-type"}, description = "Subscription type") + public SubscriptionType subscriptionType = SubscriptionType.Shared; - @Parameter(names = {"-st", "--subscription-type"}, description = "Subscription type") - public SubscriptionType subscriptionType = SubscriptionType.Shared; + @Option(names = {"-rs", "--replicated" }, + description = "Whether the subscription status should be replicated") + private boolean replicatedSubscription = false; - @Parameter(names = {"-rs", "--replicated" }, - description = "Whether the subscription status should be replicated") - private boolean replicatedSubscription = false; + @Option(names = {"-q", "--receiver-queue-size"}, description = "Size of the receiver queue") + public int receiverQueueSize = 1000; - @Parameter(names = {"-q", "--receiver-queue-size"}, description = "Size of the receiver queue") - public int receiverQueueSize = 1000; + @Option(names = {"-tto", "--txn-timeout"}, description = "Set the time value of transaction timeout," + + " and the time unit is second. (After --txn-enable setting to true, --txn-timeout takes effect)") + public long transactionTimeout = 5; - @Parameter(names = {"-tto", "--txn-timeout"}, description = "Set the time value of transaction timeout," - + " and the time unit is second. (After --txn-enable setting to true, --txn-timeout takes effect)") - public long transactionTimeout = 5; + @Option(names = {"-ntxn", + "--number-txn"}, description = "Set the number of transaction. 0 means keeping open." + + "If transaction disabled, it means the number of tasks. The task or transaction produces or " + + "consumes a specified number of messages.") + public long numTransactions = 0; - @Parameter(names = {"-ntxn", - "--number-txn"}, description = "Set the number of transaction. 0 means keeping open." - + "If transaction disabled, it means the number of tasks. The task or transaction produces or " - + "consumes a specified number of messages.") - public long numTransactions = 0; + @Option(names = {"-nmp", "--numMessage-perTransaction-produce"}, + description = "Set the number of messages produced in a transaction." + + "If transaction disabled, it means the number of messages produced in a task.") + public int numMessagesProducedPerTransaction = 1; - @Parameter(names = {"-nmp", "--numMessage-perTransaction-produce"}, - description = "Set the number of messages produced in a transaction." - + "If transaction disabled, it means the number of messages produced in a task.") - public int numMessagesProducedPerTransaction = 1; + @Option(names = {"-nmc", "--numMessage-perTransaction-consume"}, + description = "Set the number of messages consumed in a transaction." + + "If transaction disabled, it means the number of messages consumed in a task.") + public int numMessagesReceivedPerTransaction = 1; - @Parameter(names = {"-nmc", "--numMessage-perTransaction-consume"}, - description = "Set the number of messages consumed in a transaction." - + "If transaction disabled, it means the number of messages consumed in a task.") - public int numMessagesReceivedPerTransaction = 1; + @Option(names = {"--txn-disable"}, description = "Disable transaction") + public boolean isDisableTransaction = false; - @Parameter(names = {"--txn-disable"}, description = "Disable transaction") - public boolean isDisableTransaction = false; + @Option(names = {"-abort"}, description = "Abort the transaction. (After --txn-disEnable " + + "setting to false, -abort takes effect)") + public boolean isAbortTransaction = false; - @Parameter(names = {"-abort"}, description = "Abort the transaction. (After --txn-disEnable " - + "setting to false, -abort takes effect)") - public boolean isAbortTransaction = false; - - @Parameter(names = "-txnRate", description = "Set the rate of opened transaction or task. 0 means no limit") - public int openTxnRate = 0; - - @Override - public void fillArgumentsFromProperties(Properties prop) { - if (adminURL == null) { - adminURL = prop.getProperty("webServiceUrl"); - } - if (adminURL == null) { - adminURL = prop.getProperty("adminURL", "http://localhost:8080/"); - } - } + @Option(names = "-txnRate", description = "Set the rate of opened transaction or task. 0 means no limit") + public int openTxnRate = 0; + public PerformanceTransaction() { + super("transaction"); } - public static void main(String[] args) - throws IOException, PulsarAdminException, ExecutionException, InterruptedException { - final Arguments arguments = new Arguments(); - arguments.parseCLI("pulsar-perf transaction", args); + @Override + public void run() throws Exception { + super.parseCLI(); // Dump config variables PerfClientUtils.printJVMInformation(log); ObjectMapper m = new ObjectMapper(); ObjectWriter w = m.writerWithDefaultPrettyPrinter(); - log.info("Starting Pulsar perf transaction with config: {}", w.writeValueAsString(arguments)); + log.info("Starting Pulsar perf transaction with config: {}", w.writeValueAsString(this)); final byte[] payloadBytes = new byte[1024]; Random random = new Random(0); for (int i = 0; i < payloadBytes.length; ++i) { payloadBytes[i] = (byte) (random.nextInt(26) + 65); } - if (arguments.partitions != null) { + if (this.partitions != null) { final PulsarAdminBuilder adminBuilder = PerfClientUtils - .createAdminBuilderFromArguments(arguments, arguments.adminURL); + .createAdminBuilderFromArguments(this, this.adminURL); try (PulsarAdmin adminClient = adminBuilder.build()) { - for (String topic : arguments.producerTopic) { - log.info("Creating produce partitioned topic {} with {} partitions", topic, arguments.partitions); + for (String topic : this.producerTopic) { + log.info("Creating produce partitioned topic {} with {} partitions", topic, this.partitions); try { - adminClient.topics().createPartitionedTopic(topic, arguments.partitions); + adminClient.topics().createPartitionedTopic(topic, this.partitions); } catch (PulsarAdminException.ConflictException alreadyExists) { if (log.isDebugEnabled()) { log.debug("Topic {} already exists: {}", topic, alreadyExists); } PartitionedTopicMetadata partitionedTopicMetadata = adminClient.topics().getPartitionedTopicMetadata(topic); - if (partitionedTopicMetadata.partitions != arguments.partitions) { + if (partitionedTopicMetadata.partitions != this.partitions) { log.error( "Topic {} already exists but it has a wrong number of partitions: {}, expecting {}", - topic, partitionedTopicMetadata.partitions, arguments.partitions); + topic, partitionedTopicMetadata.partitions, this.partitions); PerfClientUtils.exit(1); } } @@ -222,35 +209,35 @@ public static void main(String[] args) } } - ClientBuilder clientBuilder = PerfClientUtils.createClientBuilderFromArguments(arguments) - .enableTransaction(!arguments.isDisableTransaction); + ClientBuilder clientBuilder = PerfClientUtils.createClientBuilderFromArguments(this) + .enableTransaction(!this.isDisableTransaction); - PulsarClient client = clientBuilder.build(); + try (PulsarClient client = clientBuilder.build()) { - ExecutorService executorService = new ThreadPoolExecutor(arguments.numTestThreads, - arguments.numTestThreads, - 0L, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>()); + ExecutorService executorService = new ThreadPoolExecutor(this.numTestThreads, + this.numTestThreads, + 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>()); - long startTime = System.nanoTime(); - long testEndTime = startTime + (long) (arguments.testTime * 1e9); - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - if (!arguments.isDisableTransaction) { - printTxnAggregatedThroughput(startTime); - } else { - printAggregatedThroughput(startTime); - } - printAggregatedStats(); - })); + long startTime = System.nanoTime(); + long testEndTime = startTime + (long) (this.testTime * 1e9); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + if (!this.isDisableTransaction) { + printTxnAggregatedThroughput(startTime); + } else { + printAggregatedThroughput(startTime); + } + printAggregatedStats(); + })); - // start perf test - AtomicBoolean executing = new AtomicBoolean(true); + // start perf test + AtomicBoolean executing = new AtomicBoolean(true); - RateLimiter rateLimiter = arguments.openTxnRate > 0 - ? RateLimiter.create(arguments.openTxnRate) + RateLimiter rateLimiter = this.openTxnRate > 0 + ? RateLimiter.create(this.openTxnRate) : null; - for (int i = 0; i < arguments.numTestThreads; i++) { + for (int i = 0; i < this.numTestThreads; i++) { executorService.submit(() -> { //The producer and consumer clients are built in advance, and then this thread is //responsible for the production and consumption tasks of the transaction through the loop. @@ -259,11 +246,11 @@ public static void main(String[] args) List>> consumers = null; AtomicReference atomicReference = null; try { - producers = buildProducers(client, arguments); - consumers = buildConsumer(client, arguments); - if (!arguments.isDisableTransaction) { + producers = buildProducers(client); + consumers = buildConsumer(client); + if (!this.isDisableTransaction) { atomicReference = new AtomicReference<>(client.newTransaction() - .withTransactionTimeout(arguments.transactionTimeout, TimeUnit.SECONDS) + .withTransactionTimeout(this.transactionTimeout, TimeUnit.SECONDS) .build() .get()); } else { @@ -277,11 +264,11 @@ public static void main(String[] args) //The while loop has no break, and finally ends the execution through the shutdownNow of //the executorService while (true) { - if (arguments.numTransactions > 0) { + if (this.numTransactions > 0) { if (totalNumTxnOpenTxnFail.sum() - + totalNumTxnOpenTxnSuccess.sum() >= arguments.numTransactions) { + + totalNumTxnOpenTxnSuccess.sum() >= this.numTransactions) { if (totalNumEndTxnOpFailed.sum() - + totalNumEndTxnOpSuccess.sum() < arguments.numTransactions) { + + totalNumEndTxnOpSuccess.sum() < this.numTransactions) { continue; } log.info("------------------- DONE -----------------------"); @@ -291,7 +278,7 @@ public static void main(String[] args) break; } } - if (arguments.testTime > 0) { + if (this.testTime > 0) { if (System.nanoTime() > testEndTime) { log.info("------------------- DONE -----------------------"); executing.compareAndSet(true, false); @@ -302,101 +289,102 @@ public static void main(String[] args) } Transaction transaction = atomicReference.get(); for (List> subscriptions : consumers) { - for (Consumer consumer : subscriptions) { - for (int j = 0; j < arguments.numMessagesReceivedPerTransaction; j++) { - Message message = null; - try { - message = consumer.receive(); - } catch (PulsarClientException e) { - log.error("Receive message failed", e); - executorService.shutdownNow(); - PerfClientUtils.exit(1); - } - long receiveTime = System.nanoTime(); - if (!arguments.isDisableTransaction) { - consumer.acknowledgeAsync(message.getMessageId(), transaction) - .thenRun(() -> { - long latencyMicros = NANOSECONDS.toMicros( - System.nanoTime() - receiveTime); - messageAckRecorder.recordValue(latencyMicros); - messageAckCumulativeRecorder.recordValue(latencyMicros); - numMessagesAckSuccess.increment(); - }).exceptionally(exception -> { - if (exception instanceof InterruptedException && !executing.get()) { - return null; - } - log.error( - "Ack message failed with transaction {} throw exception", - transaction, exception); - numMessagesAckFailed.increment(); - return null; - }); - } else { - consumer.acknowledgeAsync(message).thenRun(() -> { - long latencyMicros = NANOSECONDS.toMicros( - System.nanoTime() - receiveTime); - messageAckRecorder.recordValue(latencyMicros); - messageAckCumulativeRecorder.recordValue(latencyMicros); - numMessagesAckSuccess.increment(); - }).exceptionally(exception -> { - if (exception instanceof InterruptedException && !executing.get()) { + for (Consumer consumer : subscriptions) { + for (int j = 0; j < this.numMessagesReceivedPerTransaction; j++) { + Message message = null; + try { + message = consumer.receive(); + } catch (PulsarClientException e) { + log.error("Receive message failed", e); + executorService.shutdownNow(); + PerfClientUtils.exit(1); + } + long receiveTime = System.nanoTime(); + if (!this.isDisableTransaction) { + consumer.acknowledgeAsync(message.getMessageId(), transaction) + .thenRun(() -> { + long latencyMicros = NANOSECONDS.toMicros( + System.nanoTime() - receiveTime); + messageAckRecorder.recordValue(latencyMicros); + messageAckCumulativeRecorder.recordValue(latencyMicros); + numMessagesAckSuccess.increment(); + }).exceptionally(exception -> { + if (exception instanceof InterruptedException && !executing.get()) { + return null; + } + log.error( + "Ack message failed with transaction {} throw exception", + transaction, exception); + numMessagesAckFailed.increment(); return null; - } - log.error( - "Ack message failed with transaction {} throw exception", - transaction, exception); - numMessagesAckFailed.increment(); + }); + } else { + consumer.acknowledgeAsync(message).thenRun(() -> { + long latencyMicros = NANOSECONDS.toMicros( + System.nanoTime() - receiveTime); + messageAckRecorder.recordValue(latencyMicros); + messageAckCumulativeRecorder.recordValue(latencyMicros); + numMessagesAckSuccess.increment(); + }).exceptionally(exception -> { + if (exception instanceof InterruptedException && !executing.get()) { return null; - }); - } + } + log.error( + "Ack message failed with transaction {} throw exception", + transaction, exception); + numMessagesAckFailed.increment(); + return null; + }); + } } } } - for (Producer producer : producers){ - for (int j = 0; j < arguments.numMessagesProducedPerTransaction; j++) { + for (Producer producer : producers) { + for (int j = 0; j < this.numMessagesProducedPerTransaction; j++) { long sendTime = System.nanoTime(); - if (!arguments.isDisableTransaction) { + if (!this.isDisableTransaction) { producer.newMessage(transaction).value(payloadBytes) .sendAsync().thenRun(() -> { - long latencyMicros = NANOSECONDS.toMicros( - System.nanoTime() - sendTime); - messageSendRecorder.recordValue(latencyMicros); - messageSendRCumulativeRecorder.recordValue(latencyMicros); - numMessagesSendSuccess.increment(); - }).exceptionally(exception -> { - if (exception instanceof InterruptedException && !executing.get()) { - return null; - } - log.error("Send transaction message failed with exception : ", exception); - numMessagesSendFailed.increment(); - return null; - }); + long latencyMicros = NANOSECONDS.toMicros( + System.nanoTime() - sendTime); + messageSendRecorder.recordValue(latencyMicros); + messageSendRCumulativeRecorder.recordValue(latencyMicros); + numMessagesSendSuccess.increment(); + }).exceptionally(exception -> { + if (exception instanceof InterruptedException && !executing.get()) { + return null; + } + log.error("Send transaction message failed with exception : ", + exception); + numMessagesSendFailed.increment(); + return null; + }); } else { producer.newMessage().value(payloadBytes) .sendAsync().thenRun(() -> { - long latencyMicros = NANOSECONDS.toMicros( - System.nanoTime() - sendTime); - messageSendRecorder.recordValue(latencyMicros); - messageSendRCumulativeRecorder.recordValue(latencyMicros); - numMessagesSendSuccess.increment(); - }).exceptionally(exception -> { - if (exception instanceof InterruptedException && !executing.get()) { - return null; - } - log.error("Send message failed with exception : ", exception); - numMessagesSendFailed.increment(); - return null; - }); + long latencyMicros = NANOSECONDS.toMicros( + System.nanoTime() - sendTime); + messageSendRecorder.recordValue(latencyMicros); + messageSendRCumulativeRecorder.recordValue(latencyMicros); + numMessagesSendSuccess.increment(); + }).exceptionally(exception -> { + if (exception instanceof InterruptedException && !executing.get()) { + return null; + } + log.error("Send message failed with exception : ", exception); + numMessagesSendFailed.increment(); + return null; + }); } } } - if (rateLimiter != null){ + if (rateLimiter != null) { rateLimiter.tryAcquire(); } - if (!arguments.isDisableTransaction) { - if (!arguments.isAbortTransaction) { + if (!this.isDisableTransaction) { + if (!this.isAbortTransaction) { transaction.commit() .thenRun(() -> { numTxnOpSuccess.increment(); @@ -429,19 +417,19 @@ public static void main(String[] args) while (true) { try { Transaction newTransaction = client.newTransaction() - .withTransactionTimeout(arguments.transactionTimeout, TimeUnit.SECONDS) + .withTransactionTimeout(this.transactionTimeout, TimeUnit.SECONDS) .build() .get(); atomicReference.compareAndSet(transaction, newTransaction); totalNumTxnOpenTxnSuccess.increment(); break; - } catch (Exception throwable){ - if (throwable instanceof InterruptedException && !executing.get()) { - break; - } - log.error("Failed to new transaction with exception: ", throwable); - totalNumTxnOpenTxnFail.increment(); + } catch (Exception throwable) { + if (throwable instanceof InterruptedException && !executing.get()) { + break; } + log.error("Failed to new transaction with exception: ", throwable); + totalNumTxnOpenTxnFail.increment(); + } } } else { totalNumTxnOpenTxnSuccess.increment(); @@ -453,68 +441,68 @@ public static void main(String[] args) } + // Print report stats + long oldTime = System.nanoTime(); - // Print report stats - long oldTime = System.nanoTime(); - - Histogram reportSendHistogram = null; - Histogram reportAckHistogram = null; + Histogram reportSendHistogram = null; + Histogram reportAckHistogram = null; - String statsFileName = "perf-transaction-" + System.currentTimeMillis() + ".hgrm"; - log.info("Dumping latency stats to {}", statsFileName); + String statsFileName = "perf-transaction-" + System.currentTimeMillis() + ".hgrm"; + log.info("Dumping latency stats to {}", statsFileName); - PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false); - HistogramLogWriter histogramLogWriter = new HistogramLogWriter(histogramLog); + PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false); + HistogramLogWriter histogramLogWriter = new HistogramLogWriter(histogramLog); - // Some log header bits - histogramLogWriter.outputLogFormatVersion(); - histogramLogWriter.outputLegend(); + // Some log header bits + histogramLogWriter.outputLogFormatVersion(); + histogramLogWriter.outputLegend(); - while (executing.get()) { - try { - Thread.sleep(10000); - } catch (InterruptedException e) { - break; + while (executing.get()) { + try { + Thread.sleep(10000); + } catch (InterruptedException e) { + break; + } + long now = System.nanoTime(); + double elapsed = (now - oldTime) / 1e9; + long total = totalNumEndTxnOpFailed.sum() + totalNumTxnOpenTxnSuccess.sum(); + double rate = numTxnOpSuccess.sumThenReset() / elapsed; + reportSendHistogram = messageSendRecorder.getIntervalHistogram(reportSendHistogram); + reportAckHistogram = messageAckRecorder.getIntervalHistogram(reportAckHistogram); + String txnOrTaskLog = !this.isDisableTransaction + ? "Throughput transaction: {} transaction executes --- {} transaction/s" + : "Throughput task: {} task executes --- {} task/s"; + log.info( + txnOrTaskLog + " --- send Latency: mean: {} ms - med: {} " + + "- 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}" + + " --- ack Latency: " + + "mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: " + + "{}", + INTFORMAT.format(total), + DEC.format(rate), + DEC.format(reportSendHistogram.getMean() / 1000.0), + DEC.format(reportSendHistogram.getValueAtPercentile(50) / 1000.0), + DEC.format(reportSendHistogram.getValueAtPercentile(95) / 1000.0), + DEC.format(reportSendHistogram.getValueAtPercentile(99) / 1000.0), + DEC.format(reportSendHistogram.getValueAtPercentile(99.9) / 1000.0), + DEC.format(reportSendHistogram.getValueAtPercentile(99.99) / 1000.0), + DEC.format(reportSendHistogram.getMaxValue() / 1000.0), + DEC.format(reportAckHistogram.getMean() / 1000.0), + DEC.format(reportAckHistogram.getValueAtPercentile(50) / 1000.0), + DEC.format(reportAckHistogram.getValueAtPercentile(95) / 1000.0), + DEC.format(reportAckHistogram.getValueAtPercentile(99) / 1000.0), + DEC.format(reportAckHistogram.getValueAtPercentile(99.9) / 1000.0), + DEC.format(reportAckHistogram.getValueAtPercentile(99.99) / 1000.0), + DEC.format(reportAckHistogram.getMaxValue() / 1000.0)); + + histogramLogWriter.outputIntervalHistogram(reportSendHistogram); + histogramLogWriter.outputIntervalHistogram(reportAckHistogram); + reportSendHistogram.reset(); + reportAckHistogram.reset(); + + oldTime = now; } - long now = System.nanoTime(); - double elapsed = (now - oldTime) / 1e9; - long total = totalNumEndTxnOpFailed.sum() + totalNumTxnOpenTxnSuccess.sum(); - double rate = numTxnOpSuccess.sumThenReset() / elapsed; - reportSendHistogram = messageSendRecorder.getIntervalHistogram(reportSendHistogram); - reportAckHistogram = messageAckRecorder.getIntervalHistogram(reportAckHistogram); - String txnOrTaskLog = !arguments.isDisableTransaction - ? "Throughput transaction: {} transaction executes --- {} transaction/s" - : "Throughput task: {} task executes --- {} task/s"; - log.info( - txnOrTaskLog + " --- send Latency: mean: {} ms - med: {} " - + "- 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}" + " --- ack Latency: " - + "mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", - INTFORMAT.format(total), - DEC.format(rate), - DEC.format(reportSendHistogram.getMean() / 1000.0), - DEC.format(reportSendHistogram.getValueAtPercentile(50) / 1000.0), - DEC.format(reportSendHistogram.getValueAtPercentile(95) / 1000.0), - DEC.format(reportSendHistogram.getValueAtPercentile(99) / 1000.0), - DEC.format(reportSendHistogram.getValueAtPercentile(99.9) / 1000.0), - DEC.format(reportSendHistogram.getValueAtPercentile(99.99) / 1000.0), - DEC.format(reportSendHistogram.getMaxValue() / 1000.0), - DEC.format(reportAckHistogram.getMean() / 1000.0), - DEC.format(reportAckHistogram.getValueAtPercentile(50) / 1000.0), - DEC.format(reportAckHistogram.getValueAtPercentile(95) / 1000.0), - DEC.format(reportAckHistogram.getValueAtPercentile(99) / 1000.0), - DEC.format(reportAckHistogram.getValueAtPercentile(99.9) / 1000.0), - DEC.format(reportAckHistogram.getValueAtPercentile(99.99) / 1000.0), - DEC.format(reportAckHistogram.getMaxValue() / 1000.0)); - - histogramLogWriter.outputIntervalHistogram(reportSendHistogram); - histogramLogWriter.outputIntervalHistogram(reportAckHistogram); - reportSendHistogram.reset(); - reportAckHistogram.reset(); - - oldTime = now; } - - } @@ -607,24 +595,24 @@ private static void printAggregatedStats() { private static final Logger log = LoggerFactory.getLogger(PerformanceTransaction.class); - private static List>> buildConsumer(PulsarClient client, Arguments arguments) + private List>> buildConsumer(PulsarClient client) throws ExecutionException, InterruptedException { ConsumerBuilder consumerBuilder = client.newConsumer(Schema.BYTES) - .subscriptionType(arguments.subscriptionType) - .receiverQueueSize(arguments.receiverQueueSize) - .subscriptionInitialPosition(arguments.subscriptionInitialPosition) - .replicateSubscriptionState(arguments.replicatedSubscription); + .subscriptionType(this.subscriptionType) + .receiverQueueSize(this.receiverQueueSize) + .subscriptionInitialPosition(this.subscriptionInitialPosition) + .replicateSubscriptionState(this.replicatedSubscription); - Iterator consumerTopicsIterator = arguments.consumerTopic.iterator(); - List>> consumers = new ArrayList<>(arguments.consumerTopic.size()); + Iterator consumerTopicsIterator = this.consumerTopic.iterator(); + List>> consumers = new ArrayList<>(this.consumerTopic.size()); while (consumerTopicsIterator.hasNext()){ String topic = consumerTopicsIterator.next(); - final List> subscriptions = new ArrayList<>(arguments.numSubscriptions); + final List> subscriptions = new ArrayList<>(this.numSubscriptions); final List>> subscriptionFutures = - new ArrayList<>(arguments.numSubscriptions); + new ArrayList<>(this.numSubscriptions); log.info("Create subscriptions for topic {}", topic); - for (int j = 0; j < arguments.numSubscriptions; j++) { - String subscriberName = arguments.subscriptions.get(j); + for (int j = 0; j < this.numSubscriptions; j++) { + String subscriberName = this.subscriptions.get(j); subscriptionFutures .add(consumerBuilder.clone().topic(topic).subscriptionName(subscriberName) .subscribeAsync()); @@ -637,14 +625,14 @@ private static List>> buildConsumer(PulsarClient client, return consumers; } - private static List> buildProducers(PulsarClient client, Arguments arguments) + private List> buildProducers(PulsarClient client) throws ExecutionException, InterruptedException { ProducerBuilder producerBuilder = client.newProducer(Schema.BYTES) .sendTimeout(0, TimeUnit.SECONDS); final List>> producerFutures = new ArrayList<>(); - for (String topic : arguments.producerTopic) { + for (String topic : this.producerTopic) { log.info("Create producer for topic {}", topic); producerFutures.add(producerBuilder.clone().topic(topic).createAsync()); } diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PositiveNumberParameterValidator.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PositiveNumberParameterConvert.java similarity index 68% rename from pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PositiveNumberParameterValidator.java rename to pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PositiveNumberParameterConvert.java index 7e8fe2181cd6f..fc045eb8aaf29 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PositiveNumberParameterValidator.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PositiveNumberParameterConvert.java @@ -18,15 +18,16 @@ */ package org.apache.pulsar.testclient; -import com.beust.jcommander.IParameterValidator; -import com.beust.jcommander.ParameterException; - -public class PositiveNumberParameterValidator implements IParameterValidator { +import picocli.CommandLine.ITypeConverter; +import picocli.CommandLine.TypeConversionException; +public class PositiveNumberParameterConvert implements ITypeConverter { @Override - public void validate(String name, String value) throws ParameterException { - if (Integer.parseInt(value) <= 0) { - throw new ParameterException("Parameter " + name + " should be > 0 (found " + value + ")"); + public Integer convert(String value) { + int result = Integer.parseInt(value); + if (result <= 0) { + throw new TypeConversionException("Parameter should be > 0 (found " + value + ")"); } + return result; } } diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/ProxyProtocolConverter.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/ProxyProtocolConverter.java new file mode 100644 index 0000000000000..6cccc8ce480ae --- /dev/null +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/ProxyProtocolConverter.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.testclient; + +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.client.api.ProxyProtocol; +import picocli.CommandLine.ITypeConverter; + +public class ProxyProtocolConverter implements ITypeConverter { + + @Override + public ProxyProtocol convert(String value) throws Exception { + String proxyProtocolString = StringUtils.trimToNull(value); + if (proxyProtocolString != null) { + return ProxyProtocol.valueOf(proxyProtocolString.toUpperCase()); + } + return null; + } +} diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PulsarPerfTestTool.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PulsarPerfTestTool.java new file mode 100644 index 0000000000000..826060dc6b799 --- /dev/null +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PulsarPerfTestTool.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.testclient; + +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import java.io.FileInputStream; +import java.lang.reflect.Constructor; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import org.apache.pulsar.proxy.socket.client.PerformanceClient; +import picocli.CommandLine; + +@CommandLine.Command(name = "pulsar-perf", + scope = CommandLine.ScopeType.INHERIT, + mixinStandardHelpOptions = true, + showDefaultValues = true +) +public class PulsarPerfTestTool { + + protected Map> commandMap; + protected final CommandLine commander; + + public PulsarPerfTestTool() { + this.commander = new CommandLine(this); + commandMap = new HashMap<>(); + } + + private String[] initCommander(String[] args) throws Exception { + commandMap.put("produce", PerformanceProducer.class); + commandMap.put("consume", PerformanceConsumer.class); + commandMap.put("transaction", PerformanceTransaction.class); + commandMap.put("read", PerformanceReader.class); + commandMap.put("monitor-brokers", BrokerMonitor.class); + commandMap.put("simulation-client", LoadSimulationClient.class); + commandMap.put("simulation-controller", LoadSimulationController.class); + commandMap.put("websocket-producer", PerformanceClient.class); + commandMap.put("managed-ledger", ManagedLedgerWriter.class); + commandMap.put("gen-doc", CmdGenerateDocumentation.class); + if (args.length == 0) { + System.out.println("Usage: pulsar-perf CONF_FILE_PATH [options] [command] [command options]"); + PerfClientUtils.exit(0); + } + String configFile = args[0]; + Properties prop = new Properties(System.getProperties()); + if (configFile != null) { + try (FileInputStream fis = new FileInputStream(configFile)) { + prop.load(fis); + } + } + commander.setDefaultValueProvider(PulsarPerfTestPropertiesProvider.create(prop)); + + for (Map.Entry> c : commandMap.entrySet()) { + Constructor constructor = c.getValue().getDeclaredConstructor(); + constructor.setAccessible(true); + addCommand(c.getKey(), constructor.newInstance()); + } + + // Remove the first argument, it's the config file path + return Arrays.copyOfRange(args, 1, args.length); + } + + private void addCommand(String name, Object o) { + if (o instanceof CmdBase) { + commander.addSubcommand(name, ((CmdBase) o).getCommander()); + } else { + commander.addSubcommand(o); + } + } + + public static void main(String[] args) throws Exception { + PulsarPerfTestTool tool = new PulsarPerfTestTool(); + args = tool.initCommander(args); + + if (tool.run(args)) { + PerfClientUtils.exit(0); + } else { + PerfClientUtils.exit(1); + } + } + + protected boolean run(String[] args) { + return commander.execute(args) == 0; + } + +} + +class PulsarPerfTestPropertiesProvider extends CommandLine.PropertiesDefaultProvider{ + private static final String brokerServiceUrlKey = "brokerServiceUrl"; + private static final String webServiceUrlKey = "webServiceUrl"; + private final Properties properties; + + public PulsarPerfTestPropertiesProvider(Properties properties) { + super(properties); + this.properties = properties; + } + + static PulsarPerfTestPropertiesProvider create(Properties properties) { + if (isBlank(properties.getProperty(brokerServiceUrlKey))) { + String webServiceUrl = properties.getProperty("webServiceUrl"); + if (isNotBlank(webServiceUrl)) { + properties.put(brokerServiceUrlKey, webServiceUrl); + } else if (isNotBlank(properties.getProperty("serviceUrl"))) { + properties.put(brokerServiceUrlKey, properties.getProperty("serviceUrl", "http://localhost:8080/")); + } + } + + // Used for produce and transaction to fill parameters. + if (isBlank(properties.getProperty(webServiceUrlKey))) { + properties.put(webServiceUrlKey, properties.getProperty("adminURL", "http://localhost:8080/")); + } + + return new PulsarPerfTestPropertiesProvider(properties); + } +} diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/proxy/socket/client/PerformanceClientTest.java b/pulsar-testclient/src/test/java/org/apache/pulsar/proxy/socket/client/PerformanceClientTest.java index f623662e0e946..d45c3e8f3a4e7 100644 --- a/pulsar-testclient/src/test/java/org/apache/pulsar/proxy/socket/client/PerformanceClientTest.java +++ b/pulsar-testclient/src/test/java/org/apache/pulsar/proxy/socket/client/PerformanceClientTest.java @@ -31,29 +31,34 @@ public void testLoadArguments() throws Exception { PerformanceClient client = new PerformanceClient(); // "--proxy-url" has the highest priority - PerformanceClient.Arguments arguments = client.loadArguments( - getArgs("ws://broker0.pulsar.apache.org:8080/", "./src/test/resources/websocket_client1.conf")); - assertEquals(arguments.proxyURL, "ws://broker0.pulsar.apache.org:8080/"); + client.parse(getArgs("ws://broker0.pulsar.apache.org:8080/", "./src/test/resources/websocket_client1.conf")); + client.loadArguments(); + assertEquals(client.proxyURL, "ws://broker0.pulsar.apache.org:8080/"); // "webSocketServiceUrl" written in the conf file has the second priority - arguments = client.loadArguments(getArgs(null, "./src/test/resources/websocket_client1.conf")); - assertEquals(arguments.proxyURL, "ws://broker1.pulsar.apache.org:8080/"); + client.parse(getArgs(null, "./src/test/resources/websocket_client1.conf")); + client.loadArguments(); + assertEquals(client.proxyURL, "ws://broker1.pulsar.apache.org:8080/"); // "webServiceUrl" written in the conf file has the third priority - arguments = client.loadArguments(getArgs(null, "./src/test/resources/websocket_client2.conf")); - assertEquals(arguments.proxyURL, "ws://broker2.pulsar.apache.org:8080/"); + client.parse(getArgs(null, "./src/test/resources/websocket_client2.conf")); + client.loadArguments(); + assertEquals(client.proxyURL, "ws://broker2.pulsar.apache.org:8080/"); // "serviceUrl" written in the conf file has the fourth priority - arguments = client.loadArguments(getArgs(null, "./src/test/resources/websocket_client3.conf")); - assertEquals(arguments.proxyURL, "wss://broker3.pulsar.apache.org:8443/"); + client.parse(getArgs(null, "./src/test/resources/websocket_client3.conf")); + client.loadArguments(); + assertEquals(client.proxyURL, "wss://broker3.pulsar.apache.org:8443/"); // The default value is "ws://localhost:8080/" - arguments = client.loadArguments(getArgs(null, null)); - assertEquals(arguments.proxyURL, "ws://localhost:8080/"); + client.parse(getArgs(null, null)); + client.loadArguments(); + assertEquals(client.proxyURL, "ws://localhost:8080/"); // If the URL does not end with "/", it will be added - arguments = client.loadArguments(getArgs("ws://broker0.pulsar.apache.org:8080", null)); - assertEquals(arguments.proxyURL, "ws://broker0.pulsar.apache.org:8080/"); + client.parse(getArgs("ws://broker0.pulsar.apache.org:8080", null)); + client.loadArguments(); + assertEquals(client.proxyURL, "ws://broker0.pulsar.apache.org:8080/"); } private String[] getArgs(String proxyUrl, String confFile) { diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/GenerateDocumentionTest.java b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/GenerateDocumentionTest.java index 936275bcd41cd..e76e0cca0cb76 100644 --- a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/GenerateDocumentionTest.java +++ b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/GenerateDocumentionTest.java @@ -18,18 +18,55 @@ */ package org.apache.pulsar.testclient; +import org.testng.Assert; import org.testng.annotations.Test; +import picocli.CommandLine; public class GenerateDocumentionTest { @Test public void testGenerateDocumention() throws Exception { - CmdGenerateDocumentation.main(new String[]{}); + new CmdGenerateDocumentation().run(new String[]{}); } @Test public void testSpecifyModuleName() throws Exception { String[] args = new String[]{"-n", "produce", "-n", "consume"}; - CmdGenerateDocumentation.main(args); + new CmdGenerateDocumentation().run(args); + } + + private static final String DESC = "desc"; + @Test + public void testGetCommandOptionDescription(){ + Arguments arguments = new Arguments(); + CommandLine commander = new CommandLine(arguments); + String desc = CmdGenerateDocumentation.getCommandDescription(commander); + Assert.assertEquals(desc, DESC); + + commander.getCommandSpec().options().forEach(option -> { + String desc1 = CmdGenerateDocumentation.getOptionDescription(option); + Assert.assertEquals(desc1, DESC); + }); + + ArgumentsWithoutDesc argumentsWithoutDesc = new ArgumentsWithoutDesc(); + commander = new CommandLine(argumentsWithoutDesc); + desc = CmdGenerateDocumentation.getCommandDescription(commander); + Assert.assertEquals(desc, ""); + + commander.getCommandSpec().options().forEach(option -> { + String desc1 = CmdGenerateDocumentation.getOptionDescription(option); + Assert.assertEquals(desc1, ""); + }); + } + + @CommandLine.Command(description = DESC) + static class Arguments { + @CommandLine.Option(names = {"-h", "--help"}, description = DESC, help = true) + boolean help; + } + @CommandLine.Command() + static class ArgumentsWithoutDesc { + @CommandLine.Option(names = {"-h", "--help"}, help = true) + boolean help; } } \ No newline at end of file diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/Oauth2PerformanceTransactionTest.java b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/Oauth2PerformanceTransactionTest.java index 05c9b069aca87..e8eeb3bf51993 100644 --- a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/Oauth2PerformanceTransactionTest.java +++ b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/Oauth2PerformanceTransactionTest.java @@ -32,6 +32,7 @@ import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import org.apache.pulsar.broker.auth.MockOIDCIdentityProvider; import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.Consumer; @@ -42,7 +43,6 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; -import org.apache.pulsar.client.api.TokenOauth2AuthenticatedProducerConsumerTest; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; @@ -61,32 +61,25 @@ public class Oauth2PerformanceTransactionTest extends ProducerConsumerBase { private final String testNamespace = "perf"; private final String myNamespace = testTenant + "/" + testNamespace; private final String testTopic = "persistent://" + myNamespace + "/test-"; - private static final Logger log = LoggerFactory.getLogger(TokenOauth2AuthenticatedProducerConsumerTest.class); - - // public key in oauth2 server to verify the client passed in token. get from https://jwt.io/ - private final String TOKEN_TEST_PUBLIC_KEY = "data:;base64,MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2tZd/" - + "4gJda3U2Pc3tpgRAN7JPGWx/Gn17v/0IiZlNNRbP/Mmf0Vc6G1qsnaRaWNWOR+t6/a6ekFHJMikQ1N2X6yfz4UjMc8/G2FDPRm" - + "WjA+GURzARjVhxc/BBEYGoD0Kwvbq/u9CZm2QjlKrYaLfg3AeB09j0btNrDJ8rBsNzU6AuzChRvXj9IdcE/A/4N/UQ+S9cJ4UXP6" - + "NJbToLwajQ5km+CnxdGE6nfB7LWHvOFHjn9C2Rb9e37CFlmeKmIVFkagFM0gbmGOb6bnGI8Bp/VNGV0APef4YaBvBTqwoZ1Z4aDH" - + "y5eRxXfAMdtBkBupmBXqL6bpd15XRYUbu/7ck9QIDAQAB"; - - private final String ADMIN_ROLE = "Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x@clients"; + private static final Logger log = LoggerFactory.getLogger(Oauth2PerformanceTransactionTest.class); // Credentials File, which contains "client_id" and "client_secret" private final String CREDENTIALS_FILE = "./src/test/resources/authentication/token/credentials_file.json"; private final String authenticationPlugin = "org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2"; + private MockOIDCIdentityProvider server; private String authenticationParameters; @BeforeMethod(alwaysRun = true) @Override protected void setup() throws Exception { + server = new MockOIDCIdentityProvider("a-client-secret", "my-test-audience", 30000); Path path = Paths.get(CREDENTIALS_FILE).toAbsolutePath(); HashMap params = new HashMap<>(); - params.put("issuerUrl", new URL("https://dev-kt-aa9ne.us.auth0.com")); + params.put("issuerUrl", server.getIssuer()); params.put("privateKey", path.toUri().toURL()); - params.put("audience", "https://dev-kt-aa9ne.us.auth0.com/api/v2/"); + params.put("audience", "my-test-audience"); ObjectMapper jsonMapper = ObjectMapperFactory.create(); authenticationParameters = jsonMapper.writeValueAsString(params); @@ -96,7 +89,7 @@ protected void setup() throws Exception { conf.setAuthenticationRefreshCheckSeconds(5); Set superUserRoles = new HashSet<>(); - superUserRoles.add(ADMIN_ROLE); + superUserRoles.add("superuser"); conf.setSuperUserRoles(superUserRoles); Set providers = new HashSet<>(); @@ -107,7 +100,7 @@ protected void setup() throws Exception { // Set provider domain name Properties properties = new Properties(); - properties.setProperty("tokenPublicKey", TOKEN_TEST_PUBLIC_KEY); + properties.setProperty("tokenPublicKey", server.getBase64EncodedPublicKey()); conf.setProperties(properties); @@ -127,13 +120,14 @@ protected void setup() throws Exception { @Override protected void cleanup() throws Exception { super.internalCleanup(); + server.stop(); } // setup both admin and pulsar client protected final void clientSetup() throws Exception { Path path = Paths.get(CREDENTIALS_FILE).toAbsolutePath(); log.info("Credentials File path: {}", path); - + closeAdmin(); admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrl.toString()) .authentication(authenticationPlugin, authenticationParameters) .build()); @@ -192,7 +186,7 @@ public void testTransactionPerf() throws Exception { Thread thread = new Thread(() -> { try { - PerformanceTransaction.main(args.split(" ")); + new PerformanceTransaction().run(args.split(" ")); } catch (Exception e) { e.printStackTrace(); } diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerfClientUtilsTest.java b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerfClientUtilsTest.java index 3d734b1f910ea..ed0d055ce1188 100644 --- a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerfClientUtilsTest.java +++ b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerfClientUtilsTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.testclient; +import java.io.FileInputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -55,7 +56,7 @@ public void close() throws IOException { @Test public void testClientCreation() throws Exception { - final PerformanceBaseArguments args = new PerformanceArgumentsTestDefault(); + final PerformanceBaseArguments args = new PerformanceArgumentsTestDefault(""); args.tlsHostnameVerificationEnable = true; args.authPluginClassName = MyAuth.class.getName(); @@ -70,6 +71,7 @@ public void testClientCreation() throws Exception { args.tlsTrustCertsFilePath = "path"; args.tlsAllowInsecureConnection = true; args.maxLookupRequest = 100000; + args.memoryLimit = 10240; final ClientBuilderImpl builder = (ClientBuilderImpl)PerfClientUtils.createClientBuilderFromArguments(args); final ClientConfigurationData conf = builder.getClientConfigurationData(); @@ -89,13 +91,14 @@ public void testClientCreation() throws Exception { Assert.assertEquals(conf.getMaxLookupRequest(), 100000); Assert.assertNull(conf.getProxyServiceUrl()); Assert.assertNull(conf.getProxyProtocol()); + Assert.assertEquals(conf.getMemoryLimitBytes(), 10240L); } @Test public void testClientCreationWithProxy() throws Exception { - final PerformanceBaseArguments args = new PerformanceArgumentsTestDefault(); + final PerformanceBaseArguments args = new PerformanceArgumentsTestDefault(""); args.serviceURL = "pulsar+ssl://my-pulsar:6651"; args.proxyServiceURL = "pulsar+ssl://my-proxy-pulsar:4443"; @@ -118,11 +121,13 @@ public void testClientCreationWithProxyDefinedInConfFile() throws Exception { + "proxyServiceUrl=pulsar+ssl://my-proxy-pulsar:4443\n" + "proxyProtocol=SNI"); - final PerformanceBaseArguments args = new PerformanceArgumentsTestDefault(); - - args.confFile = testConf.toString(); - args.fillArgumentsFromProperties(); - + final PerformanceBaseArguments args = new PerformanceArgumentsTestDefault(""); + Properties prop = new Properties(System.getProperties()); + try (FileInputStream fis = new FileInputStream(testConf.toString())) { + prop.load(fis); + } + args.getCommander().setDefaultValueProvider(PulsarPerfTestPropertiesProvider.create(prop)); + args.parse(new String[]{}); final ClientBuilderImpl builder = (ClientBuilderImpl) PerfClientUtils.createClientBuilderFromArguments(args); final ClientConfigurationData conf = builder.getClientConfigurationData(); @@ -143,16 +148,19 @@ public void testClientCreationWithEmptyProxyPropertyInConfFile() throws Exceptio + "proxyServiceUrl=\n" + "proxyProtocol="); - final PerformanceBaseArguments args = new PerformanceArgumentsTestDefault(); - - args.confFile = testConf.toString(); - args.fillArgumentsFromProperties(); + final PerformanceBaseArguments args = new PerformanceArgumentsTestDefault(""); + Properties prop = new Properties(System.getProperties()); + try (FileInputStream fis = new FileInputStream(testConf.toString())) { + prop.load(fis); + } + args.getCommander().setDefaultValueProvider(PulsarPerfTestPropertiesProvider.create(prop)); + args.parse(new String[]{}); final ClientBuilderImpl builder = (ClientBuilderImpl) PerfClientUtils.createClientBuilderFromArguments(args); final ClientConfigurationData conf = builder.getClientConfigurationData(); - Assert.assertNull(conf.getProxyServiceUrl()); + Assert.assertEquals(conf.getProxyServiceUrl(),""); Assert.assertNull(conf.getProxyProtocol()); } finally { Files.deleteIfExists(testConf); @@ -161,7 +169,13 @@ public void testClientCreationWithEmptyProxyPropertyInConfFile() throws Exceptio } class PerformanceArgumentsTestDefault extends PerformanceBaseArguments { + public PerformanceArgumentsTestDefault(String cmdName) { + super(cmdName); + } + + @Override - public void fillArgumentsFromProperties(Properties prop) { + public void run() throws Exception { + } } diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceBaseArgumentsTest.java b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceBaseArgumentsTest.java index 42c93be343074..9b54fa510cee2 100644 --- a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceBaseArgumentsTest.java +++ b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceBaseArgumentsTest.java @@ -18,36 +18,42 @@ */ package org.apache.pulsar.testclient; +import static org.apache.pulsar.client.api.ProxyProtocol.SNI; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.fail; import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.atomic.AtomicBoolean; - import org.testng.Assert; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; - -import static org.apache.pulsar.client.api.ProxyProtocol.SNI; -import static org.testng.Assert.fail; - +import picocli.CommandLine; public class PerformanceBaseArgumentsTest { @Test - public void testReadFromConfigFile() { - - AtomicBoolean called = new AtomicBoolean(); - - final PerformanceBaseArguments args = new PerformanceBaseArguments() { + public void testReadFromConfigFile() throws Exception { + final PerformanceBaseArguments args = new PerformanceBaseArguments("") { @Override - public void fillArgumentsFromProperties(Properties prop) { - called.set(true); + public void run() throws Exception { + } }; - args.confFile = "./src/test/resources/perf_client1.conf"; - args.fillArgumentsFromProperties(); - Assert.assertTrue(called.get()); + + String confFile = "./src/test/resources/perf_client1.conf"; + Properties prop = new Properties(System.getProperties()); + try (FileInputStream fis = new FileInputStream(confFile)) { + prop.load(fis); + } + args.getCommander().setDefaultValueProvider(PulsarPerfTestPropertiesProvider.create(prop)); + args.parse(new String[]{}); + + Assert.assertEquals(args.serviceURL, "https://my-pulsar:8443/"); Assert.assertEquals(args.authPluginClassName, "org.apache.pulsar.testclient.PerfClientUtilsTest.MyAuth"); @@ -62,37 +68,41 @@ public void fillArgumentsFromProperties(Properties prop) { @Test public void testReadFromConfigFileWithoutProxyUrl() { - AtomicBoolean called = new AtomicBoolean(); - final PerformanceBaseArguments args = new PerformanceBaseArguments() { + final PerformanceBaseArguments args = new PerformanceBaseArguments("") { @Override - public void fillArgumentsFromProperties(Properties prop) { - called.set(true); + public void run() throws Exception { + } + }; + String confFile = "./src/test/resources/performance_client2.conf"; - File tempConfigFile = new File("./src/test/resources/performance_client2.conf"); + File tempConfigFile = new File(confFile); if (tempConfigFile.exists()) { tempConfigFile.delete(); } try { Properties props = new Properties(); - - Map configs = Map.of("brokerServiceUrl","https://my-pulsar:8443/", - "authPlugin","org.apache.pulsar.testclient.PerfClientUtilsTest.MyAuth", - "authParams", "myparams", - "tlsTrustCertsFilePath", "./path", - "tlsAllowInsecureConnection","true", - "tlsEnableHostnameVerification", "true" + + Map configs = Map.of("brokerServiceUrl", "https://my-pulsar:8443/", + "authPlugin", "org.apache.pulsar.testclient.PerfClientUtilsTest.MyAuth", + "authParams", "myparams", + "tlsTrustCertsFilePath", "./path", + "tlsAllowInsecureConnection", "true", + "tlsEnableHostnameVerification", "true" ); props.putAll(configs); FileOutputStream out = new FileOutputStream(tempConfigFile); props.store(out, "properties file"); out.close(); - args.confFile = "./src/test/resources/performance_client2.conf"; + Properties prop = new Properties(System.getProperties()); + try (FileInputStream fis = new FileInputStream(confFile)) { + prop.load(fis); + } + args.getCommander().setDefaultValueProvider(PulsarPerfTestPropertiesProvider.create(prop)); + args.parse(new String[]{}); - args.fillArgumentsFromProperties(); - Assert.assertTrue(called.get()); Assert.assertEquals(args.serviceURL, "https://my-pulsar:8443/"); Assert.assertEquals(args.authPluginClassName, "org.apache.pulsar.testclient.PerfClientUtilsTest.MyAuth"); @@ -100,7 +110,7 @@ public void fillArgumentsFromProperties(Properties prop) { Assert.assertEquals(args.tlsTrustCertsFilePath, "./path"); Assert.assertTrue(args.tlsAllowInsecureConnection); Assert.assertTrue(args.tlsHostnameVerificationEnable); - + } catch (IOException e) { e.printStackTrace(); fail("Error while updating/reading config file"); @@ -112,27 +122,27 @@ public void fillArgumentsFromProperties(Properties prop) { @Test public void testReadFromConfigFileProxyProtocolException() { - AtomicBoolean calledVar1 = new AtomicBoolean(); AtomicBoolean calledVar2 = new AtomicBoolean(); - final PerformanceBaseArguments args = new PerformanceBaseArguments() { + final PerformanceBaseArguments args = new PerformanceBaseArguments("") { @Override - public void fillArgumentsFromProperties(Properties prop) { - calledVar1.set(true); + public void run() throws Exception { + } }; - File tempConfigFile = new File("./src/test/resources/performance_client3.conf"); + String confFile = "./src/test/resources/performance_client3.conf"; + File tempConfigFile = new File(confFile); if (tempConfigFile.exists()) { tempConfigFile.delete(); } try { Properties props = new Properties(); - Map configs = Map.of("brokerServiceUrl","https://my-pulsar:8443/", - "authPlugin","org.apache.pulsar.testclient.PerfClientUtilsTest.MyAuth", + Map configs = Map.of("brokerServiceUrl", "https://my-pulsar:8443/", + "authPlugin", "org.apache.pulsar.testclient.PerfClientUtilsTest.MyAuth", "authParams", "myparams", "tlsTrustCertsFilePath", "./path", - "tlsAllowInsecureConnection","true", + "tlsAllowInsecureConnection", "true", "tlsEnableHostnameVerification", "true", "proxyServiceURL", "https://my-proxy-pulsar:4443/", "proxyProtocol", "TEST" @@ -141,15 +151,17 @@ public void fillArgumentsFromProperties(Properties prop) { FileOutputStream out = new FileOutputStream(tempConfigFile); props.store(out, "properties file"); out.close(); - args.confFile = "./src/test/resources/performance_client3.conf"; - PerfClientUtils.setExitProcedure(code -> { - calledVar2.set(true); - Assert.assertEquals(code, 1, "Incorrect exit code"); - }); - args.confFile = "./src/test/resources/performance_client3.conf"; - args.fillArgumentsFromProperties(); - Assert.assertTrue(calledVar1.get()); + Properties prop = new Properties(System.getProperties()); + try (FileInputStream fis = new FileInputStream(confFile)) { + prop.load(fis); + } + args.getCommander().setDefaultValueProvider(PulsarPerfTestPropertiesProvider.create(prop)); + try { + args.parse(new String[]{}); + }catch (CommandLine.ParameterException e){ + calledVar2.set(true); + } Assert.assertTrue(calledVar2.get()); } catch (IOException e) { e.printStackTrace(); @@ -158,4 +170,87 @@ public void fillArgumentsFromProperties(Properties prop) { tempConfigFile.delete(); } } + + @DataProvider(name = "memoryLimitCliArgumentProvider") + public Object[][] memoryLimitCliArgumentProvider() { + return new Object[][]{ + {new String[]{"-ml", "1"}, 1L}, + {new String[]{"-ml", "1K"}, 1024L}, + {new String[]{"--memory-limit", "1G"}, 1024 * 1024 * 1024} + }; + } + + @Test(dataProvider = "memoryLimitCliArgumentProvider") + public void testMemoryLimitCliArgument(String[] cliArgs, long expectedMemoryLimit) throws Exception { + for (String cmd : List.of( + "pulsar-perf read", + "pulsar-perf produce", + "pulsar-perf consume", + "pulsar-perf transaction" + )) { + // Arrange + final PerformanceBaseArguments baseArgument = new PerformanceBaseArguments("") { + @Override + public void run() throws Exception { + + } + + }; + String confFile = "./src/test/resources/perf_client1.conf"; + Properties prop = new Properties(System.getProperties()); + try (FileInputStream fis = new FileInputStream(confFile)) { + prop.load(fis); + } + baseArgument.getCommander().setDefaultValueProvider(PulsarPerfTestPropertiesProvider.create(prop)); + baseArgument.parse(new String[]{}); + + // Act + baseArgument.parseCLI(); + baseArgument.getCommander().execute(cliArgs); + + // Assert + assertEquals(baseArgument.memoryLimit, expectedMemoryLimit); + } + } + + @DataProvider(name = "invalidMemoryLimitCliArgumentProvider") + public Object[][] invalidMemoryLimitCliArgumentProvider() { + return new Object[][]{ + {new String[]{"-ml", "-1"}}, + {new String[]{"-ml", "1C"}}, + {new String[]{"--memory-limit", "1Q"}} + }; + } + + @Test + public void testMemoryLimitCliArgumentDefault() throws Exception { + for (String cmd : List.of( + "pulsar-perf read", + "pulsar-perf produce", + "pulsar-perf consume", + "pulsar-perf transaction" + )) { + // Arrange + final PerformanceBaseArguments baseArgument = new PerformanceBaseArguments("") { + @Override + public void run() throws Exception { + + } + + }; + String confFile = "./src/test/resources/perf_client1.conf"; + Properties prop = new Properties(System.getProperties()); + try (FileInputStream fis = new FileInputStream(confFile)) { + prop.load(fis); + } + baseArgument.getCommander().setDefaultValueProvider(PulsarPerfTestPropertiesProvider.create(prop)); + baseArgument.parse(new String[]{}); + + // Act + baseArgument.parseCLI(); + + // Assert + assertEquals(baseArgument.memoryLimit, 0L); + } + } } diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java index e343b1d8cbc64..519bed6cdb5ae 100644 --- a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java +++ b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java @@ -18,7 +18,15 @@ */ package org.apache.pulsar.testclient; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.fail; +import com.google.common.collect.Range; import com.google.common.collect.Sets; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.client.api.ClientBuilder; @@ -35,13 +43,6 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.fail; - @Slf4j public class PerformanceProducerTest extends MockedPulsarServiceBaseTest { private final String testTenant = "prop-xyz"; @@ -84,7 +85,8 @@ public void testMsgKey() throws Exception { String args = String.format(argString, topic, pulsar.getBrokerServiceUrl()); Thread thread = new Thread(() -> { try { - PerformanceProducer.main(args.split(" ")); + PerformanceProducer producer = new PerformanceProducer(); + producer.run(args.split(" ")); } catch (Exception e) { e.printStackTrace(); } @@ -96,26 +98,20 @@ public void testMsgKey() throws Exception { thread.start(); - int count1 = 0; - int count2 = 0; - for (int i = 0; i < 10; i++) { - Message message = consumer1.receive(1, TimeUnit.SECONDS); - if (message == null) { - break; - } - count1++; - consumer1.acknowledge(message); - } - for (int i = 0; i < 10; i++) { - Message message = consumer2.receive(1, TimeUnit.SECONDS); - if (message == null) { - break; - } - count2++; - consumer2.acknowledge(message); - } - //in key_share mode, only one consumer can get msg - Assert.assertTrue(count1 == 0 || count2 == 0); + // in key_shared mode if no message key is set, both consumers should receive messages + Awaitility.await() + .untilAsserted(() -> { + Message message = consumer1.receive(1, TimeUnit.SECONDS); + assertNotNull(message); + consumer1.acknowledge(message); + }); + + Awaitility.await() + .untilAsserted(() -> { + Message message = consumer2.receive(1, TimeUnit.SECONDS); + assertNotNull(message); + consumer2.acknowledge(message); + }); consumer1.close(); consumer2.close(); @@ -130,7 +126,8 @@ public void testMsgKey() throws Exception { String newArgs = String.format(newArgString, topic2, pulsar.getBrokerServiceUrl()); Thread thread2 = new Thread(() -> { try { - PerformanceProducer.main(newArgs.split(" ")); + PerformanceProducer producer = new PerformanceProducer(); + producer.run(newArgs.split(" ")); } catch (Exception e) { e.printStackTrace(); } @@ -146,19 +143,15 @@ public void testMsgKey() throws Exception { Awaitility.await() .untilAsserted(() -> { Message message = newConsumer1.receive(1, TimeUnit.SECONDS); - if (message != null) { - newConsumer1.acknowledge(message); - } assertNotNull(message); + newConsumer1.acknowledge(message); }); Awaitility.await() .untilAsserted(() -> { Message message = newConsumer2.receive(1, TimeUnit.SECONDS); - if (message != null) { - newConsumer2.acknowledge(message); - } assertNotNull(message); + newConsumer2.acknowledge(message); }); thread2.interrupt(); @@ -168,22 +161,23 @@ public void testMsgKey() throws Exception { @Test(timeOut = 20000) public void testBatchingDisabled() throws Exception { - PerformanceProducer.Arguments arguments = new PerformanceProducer.Arguments(); - + PerformanceProducer producer = new PerformanceProducer(); + int producerId = 0; - + String topic = testTopic + UUID.randomUUID(); - arguments.topics = List.of(topic); - arguments.msgRate = 10; - arguments.serviceURL = pulsar.getBrokerServiceUrl(); - arguments.numMessages = 500; - arguments.disableBatching = true; - - ClientBuilder clientBuilder = PerfClientUtils.createClientBuilderFromArguments(arguments) - .enableTransaction(arguments.isEnableTransaction); + producer.topics = List.of(topic); + producer.msgRate = 10; + producer.serviceURL = pulsar.getBrokerServiceUrl(); + producer.numMessages = 500; + producer.disableBatching = true; + + ClientBuilder clientBuilder = PerfClientUtils.createClientBuilderFromArguments(producer) + .enableTransaction(producer.isEnableTransaction); + @Cleanup PulsarClient client = clientBuilder.build(); - - ProducerBuilderImpl builder = (ProducerBuilderImpl) PerformanceProducer.createProducerBuilder(client, arguments, producerId); + ProducerBuilderImpl builder = (ProducerBuilderImpl) producer.createProducerBuilder(client, + producerId); Assert.assertFalse(builder.getConf().isBatchingEnabled()); } @@ -194,7 +188,8 @@ public void testCreatePartitions() throws Exception { String args = String.format(argString, topic, pulsar.getBrokerServiceUrl(), pulsar.getWebServiceAddress()); Thread thread = new Thread(() -> { try { - PerformanceProducer.main(args.split(" ")); + PerformanceProducer producer = new PerformanceProducer(); + producer.run(args.split(" ")); } catch (Exception e) { e.printStackTrace(); } @@ -225,7 +220,8 @@ public void testMaxOutstanding() throws Exception { .subscriptionType(SubscriptionType.Key_Shared).subscribe(); new Thread(() -> { try { - PerformanceProducer.main(args.split(" ")); + PerformanceProducer producer = new PerformanceProducer(); + producer.run(args.split(" ")); } catch (Exception e) { log.error("Failed to start perf producer"); } @@ -237,4 +233,12 @@ public void testMaxOutstanding() throws Exception { }); consumer.close(); } + + @Test + public void testRangeConvert() { + PerformanceProducer.RangeConvert rangeConvert = new PerformanceProducer.RangeConvert(); + Range range = rangeConvert.convert("100,200"); + Assert.assertEquals(range.lowerEndpoint(), 100); + Assert.assertEquals(range.upperEndpoint(), 200); + } } diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceTransactionTest.java b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceTransactionTest.java index 12f457587f685..c8d71d98e701b 100644 --- a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceTransactionTest.java +++ b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceTransactionTest.java @@ -133,7 +133,7 @@ public void testTxnPerf() throws Exception { Thread thread = new Thread(() -> { try { - PerformanceTransaction.main(args.split(" ")); + new PerformanceTransaction().run(args.split(" ")); } catch (Exception e) { e.printStackTrace(); } @@ -184,7 +184,7 @@ public void testProduceTxnMessage() throws InterruptedException, PulsarClientExc .subscribe(); Thread thread = new Thread(() -> { try { - PerformanceProducer.main(args.split(" ")); + new PerformanceProducer().run(args.split(" ")); } catch (Exception e) { e.printStackTrace(); } @@ -237,7 +237,7 @@ public void testConsumeTxnMessage() throws Exception { Thread thread = new Thread(() -> { try { log.info(""); - PerformanceConsumer.main(args.split(" ")); + new PerformanceConsumer().run(args.split(" ")); } catch (Exception e) { e.printStackTrace(); } diff --git a/pulsar-testclient/src/test/resources/authentication/token/credentials_file.json b/pulsar-testclient/src/test/resources/authentication/token/credentials_file.json index 698ad9d93e3da..92ab1c3b7cd87 100644 --- a/pulsar-testclient/src/test/resources/authentication/token/credentials_file.json +++ b/pulsar-testclient/src/test/resources/authentication/token/credentials_file.json @@ -1,5 +1,5 @@ { "type": "client_credentials", - "client_id":"Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x", - "client_secret":"rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N_poGAb" + "client_id":"superuser", + "client_secret":"a-client-secret" } diff --git a/pulsar-transaction/common/pom.xml b/pulsar-transaction/common/pom.xml index b2594bf26ade5..c6c67ea8309e3 100644 --- a/pulsar-transaction/common/pom.xml +++ b/pulsar-transaction/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-transaction-common diff --git a/pulsar-transaction/coordinator/pom.xml b/pulsar-transaction/coordinator/pom.xml index d6b79609c2f76..9b17487e0ad4d 100644 --- a/pulsar-transaction/coordinator/pom.xml +++ b/pulsar-transaction/coordinator/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-transaction-coordinator @@ -41,6 +41,12 @@ ${project.version} + + ${project.groupId} + pulsar-opentelemetry + ${project.version} + + ${project.groupId} managed-ledger @@ -81,30 +87,6 @@ - - org.codehaus.mojo - properties-maven-plugin - - - initialize - - set-system-properties - - - - - proto_path - ${project.parent.parent.basedir} - - - proto_search_strategy - 2 - - - - - - com.github.splunk.lightproto lightproto-maven-plugin diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TransactionMetadataStore.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TransactionMetadataStore.java index ff5adb4d409c7..850fcfb4d19ec 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TransactionMetadataStore.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TransactionMetadataStore.java @@ -133,6 +133,15 @@ default long getLowWaterMark() { */ TransactionMetadataStoreStats getMetadataStoreStats(); + /** + * Get the transaction metadata store OpenTelemetry attributes. + * + * @return TransactionMetadataStoreAttributes {@link TransactionMetadataStoreAttributes} + */ + default TransactionMetadataStoreAttributes getAttributes() { + return new TransactionMetadataStoreAttributes(this); + } + /** * Get the transactions witch timeout is bigger than given timeout. * diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TransactionMetadataStoreAttributes.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TransactionMetadataStoreAttributes.java new file mode 100644 index 0000000000000..e8ae0f6d0391f --- /dev/null +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TransactionMetadataStoreAttributes.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.transaction.coordinator; + +import io.opentelemetry.api.common.Attributes; +import lombok.Getter; +import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes; + +@Getter +public class TransactionMetadataStoreAttributes { + + private final Attributes commonAttributes; + private final Attributes txnAbortedAttributes; + private final Attributes txnActiveAttributes; + private final Attributes txnCommittedAttributes; + private final Attributes txnCreatedAttributes; + private final Attributes txnTimeoutAttributes; + + public TransactionMetadataStoreAttributes(TransactionMetadataStore store) { + this.commonAttributes = Attributes.of( + OpenTelemetryAttributes.PULSAR_TRANSACTION_COORDINATOR_ID, store.getTransactionCoordinatorID().getId()); + this.txnAbortedAttributes = Attributes.builder() + .putAll(commonAttributes) + .putAll(OpenTelemetryAttributes.TransactionStatus.ABORTED.attributes) + .build(); + this.txnActiveAttributes = Attributes.builder() + .putAll(commonAttributes) + .putAll(OpenTelemetryAttributes.TransactionStatus.ACTIVE.attributes) + .build(); + this.txnCommittedAttributes = Attributes.builder() + .putAll(commonAttributes) + .putAll(OpenTelemetryAttributes.TransactionStatus.COMMITTED.attributes) + .build(); + this.txnCreatedAttributes = Attributes.builder() + .putAll(commonAttributes) + .putAll(OpenTelemetryAttributes.TransactionStatus.CREATED.attributes) + .build(); + this.txnTimeoutAttributes = Attributes.builder() + .putAll(commonAttributes) + .putAll(OpenTelemetryAttributes.TransactionStatus.TIMEOUT.attributes) + .build(); + } +} diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TransactionTimeoutTracker.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TransactionTimeoutTracker.java index 5e84b002f3316..a681c4bad3e58 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TransactionTimeoutTracker.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TransactionTimeoutTracker.java @@ -19,7 +19,6 @@ package org.apache.pulsar.transaction.coordinator; import com.google.common.annotations.Beta; -import java.util.concurrent.CompletableFuture; /** * Represent the tracker for the timeout of the transaction. @@ -34,10 +33,8 @@ public interface TransactionTimeoutTracker extends AutoCloseable { * the sequenceId * @param timeout * the absolute timestamp for transaction timeout - * - * @return true if the transaction was added to the tracker or false if had timed out */ - CompletableFuture addTransaction(long sequenceId, long timeout); + void addTransaction(long sequenceId, long timeout); /** * When replay the log, add the txnMeta to timer task queue. diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/InMemTransactionMetadataStore.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/InMemTransactionMetadataStore.java index 0f3c5e42d7a69..7817d48487568 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/InMemTransactionMetadataStore.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/InMemTransactionMetadataStore.java @@ -23,12 +23,14 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.concurrent.atomic.LongAdder; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.policies.data.TransactionCoordinatorStats; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; import org.apache.pulsar.transaction.coordinator.TransactionMetadataStore; +import org.apache.pulsar.transaction.coordinator.TransactionMetadataStoreAttributes; import org.apache.pulsar.transaction.coordinator.TransactionSubscription; import org.apache.pulsar.transaction.coordinator.TxnMeta; import org.apache.pulsar.transaction.coordinator.exceptions.CoordinatorException.InvalidTxnStatusException; @@ -49,6 +51,11 @@ class InMemTransactionMetadataStore implements TransactionMetadataStore { private final LongAdder abortTransactionCount; private final LongAdder transactionTimeoutCount; + private volatile TransactionMetadataStoreAttributes attributes = null; + private static final AtomicReferenceFieldUpdater + ATTRIBUTES_FIELD_UPDATER = AtomicReferenceFieldUpdater.newUpdater( + InMemTransactionMetadataStore.class, TransactionMetadataStoreAttributes.class, "attributes"); + InMemTransactionMetadataStore(TransactionCoordinatorID tcID) { this.tcID = tcID; this.localID = new AtomicLong(0L); @@ -165,4 +172,13 @@ public TransactionMetadataStoreStats getMetadataStoreStats() { public List getSlowTransactions(long timeout) { return null; } + + @Override + public TransactionMetadataStoreAttributes getAttributes() { + if (attributes != null) { + return attributes; + } + return ATTRIBUTES_FIELD_UPDATER.updateAndGet(this, + old -> old != null ? old : new TransactionMetadataStoreAttributes(this)); + } } diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java index f2e1f60663d28..5de9b7f8fc7b4 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImpl.java @@ -38,8 +38,8 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerAlreadyClosedException; import org.apache.bookkeeper.mledger.ManagedLedgerFactory; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.common.naming.NamespaceName; @@ -156,7 +156,7 @@ public void replayAsync(TransactionLogReplayCallback transactionLogReplayCallbac private void readAsync(int numberOfEntriesToRead, AsyncCallbacks.ReadEntriesCallback readEntriesCallback) { - cursor.asyncReadEntries(numberOfEntriesToRead, readEntriesCallback, System.nanoTime(), PositionImpl.LATEST); + cursor.asyncReadEntries(numberOfEntriesToRead, readEntriesCallback, System.nanoTime(), PositionFactory.LATEST); } @Override @@ -264,7 +264,7 @@ public void start() { * 3. Build batched position and handle valid data. */ long[] ackSetAlreadyAck = cursor.getDeletedBatchIndexesAsLongArray( - PositionImpl.get(entry.getLedgerId(), entry.getEntryId())); + PositionFactory.create(entry.getLedgerId(), entry.getEntryId())); BitSetRecyclable bitSetAlreadyAck = null; if (ackSetAlreadyAck != null){ bitSetAlreadyAck = BitSetRecyclable.valueOf(ackSetAlreadyAck); diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java index b6eaad2e3e38f..6bd7a947e3827 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java @@ -30,6 +30,7 @@ import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.concurrent.atomic.LongAdder; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.Position; @@ -45,6 +46,7 @@ import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; import org.apache.pulsar.transaction.coordinator.TransactionLogReplayCallback; import org.apache.pulsar.transaction.coordinator.TransactionMetadataStore; +import org.apache.pulsar.transaction.coordinator.TransactionMetadataStoreAttributes; import org.apache.pulsar.transaction.coordinator.TransactionMetadataStoreState; import org.apache.pulsar.transaction.coordinator.TransactionRecoverTracker; import org.apache.pulsar.transaction.coordinator.TransactionSubscription; @@ -83,6 +85,11 @@ public class MLTransactionMetadataStore public final RecoverTimeRecord recoverTime = new RecoverTimeRecord(); private final long maxActiveTransactionsPerCoordinator; + private volatile TransactionMetadataStoreAttributes attributes = null; + private static final AtomicReferenceFieldUpdater + ATTRIBUTES_FIELD_UPDATER = AtomicReferenceFieldUpdater.newUpdater( + MLTransactionMetadataStore.class, TransactionMetadataStoreAttributes.class, "attributes"); + public MLTransactionMetadataStore(TransactionCoordinatorID tcID, MLTransactionLogImpl mlTransactionLog, TransactionTimeoutTracker timeoutTracker, @@ -549,4 +556,13 @@ public static List subscriptionToTxnSubscription( public ManagedLedger getManagedLedger() { return this.transactionLog.getManagedLedger(); } + + @Override + public TransactionMetadataStoreAttributes getAttributes() { + if (attributes != null) { + return attributes; + } + return ATTRIBUTES_FIELD_UPDATER.updateAndGet(this, + old -> old != null ? old : new TransactionMetadataStoreAttributes(this)); + } } diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionSequenceIdGenerator.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionSequenceIdGenerator.java index 204555a1cfc67..a6605046eeff6 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionSequenceIdGenerator.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionSequenceIdGenerator.java @@ -22,9 +22,7 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicLong; -import org.apache.bookkeeper.client.LedgerHandle; -import org.apache.bookkeeper.client.api.LedgerEntry; -import org.apache.bookkeeper.mledger.impl.OpAddEntry; +import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.intercept.ManagedLedgerInterceptor; import org.apache.commons.collections4.CollectionUtils; import org.apache.pulsar.transaction.coordinator.proto.TransactionMetadataEntry; @@ -42,8 +40,8 @@ public class MLTransactionSequenceIdGenerator implements ManagedLedgerIntercepto private final AtomicLong sequenceId = new AtomicLong(TC_ID_NOT_USED); @Override - public OpAddEntry beforeAddEntry(OpAddEntry op, int numberOfMessages) { - return op; + public void beforeAddEntry(AddEntryOperation op, int numberOfMessages) { + // do nothing } // When all of ledger have been deleted, we will generate sequenceId from managedLedger properties @@ -60,43 +58,23 @@ public void onManagedLedgerPropertiesInitialize(Map propertiesMa // When we don't roll over ledger, we can init sequenceId from the getLastAddConfirmed transaction metadata entry @Override - public CompletableFuture onManagedLedgerLastLedgerInitialize(String name, LedgerHandle lh) { - CompletableFuture promise = new CompletableFuture<>(); - if (lh.getLastAddConfirmed() >= 0) { - lh.readAsync(lh.getLastAddConfirmed(), lh.getLastAddConfirmed()).whenComplete((entries, ex) -> { - if (ex != null) { - log.error("[{}] Read last entry error.", name, ex); - promise.completeExceptionally(ex); - } else { - if (entries != null) { - try { - LedgerEntry ledgerEntry = entries.getEntry(lh.getLastAddConfirmed()); - if (ledgerEntry != null) { - List transactionLogs = - MLTransactionLogImpl.deserializeEntry(ledgerEntry.getEntryBuffer()); - if (!CollectionUtils.isEmpty(transactionLogs)){ - TransactionMetadataEntry lastConfirmEntry = - transactionLogs.get(transactionLogs.size() - 1); - this.sequenceId.set(lastConfirmEntry.getMaxLocalTxnId()); - } - } - entries.close(); - promise.complete(null); - } catch (Exception e) { - entries.close(); - log.error("[{}] Failed to recover the tc sequenceId from the last add confirmed entry.", - name, e); - promise.completeExceptionally(e); - } - } else { - promise.complete(null); + public CompletableFuture onManagedLedgerLastLedgerInitialize(String name, LastEntryHandle lh) { + return lh.readLastEntryAsync().thenAccept(lastEntryOptional -> { + if (lastEntryOptional.isPresent()) { + Entry lastEntry = lastEntryOptional.get(); + try { + List transactionLogs = + MLTransactionLogImpl.deserializeEntry(lastEntry.getDataBuffer()); + if (!CollectionUtils.isEmpty(transactionLogs)) { + TransactionMetadataEntry lastConfirmEntry = + transactionLogs.get(transactionLogs.size() - 1); + this.sequenceId.set(lastConfirmEntry.getMaxLocalTxnId()); } + } finally { + lastEntry.release(); } - }); - } else { - promise.complete(null); - } - return promise; + } + }); } // roll over ledger will update sequenceId to managedLedger properties diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/TxnBatchedPositionImpl.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/TxnBatchedPositionImpl.java index 2e897167aaf0a..158fb42cb7356 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/TxnBatchedPositionImpl.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/TxnBatchedPositionImpl.java @@ -20,14 +20,14 @@ import lombok.Getter; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.impl.AckSetPositionImpl; import org.apache.pulsar.common.util.collections.BitSetRecyclable; /*** - * The difference with {@link PositionImpl} is that there are two more parameters: + * The difference with {@link AckSetPositionImpl} is that there are two more parameters: * {@link #batchSize}, {@link #batchIndex}. */ -public class TxnBatchedPositionImpl extends PositionImpl { +public class TxnBatchedPositionImpl extends AckSetPositionImpl { /** The data length of current batch. **/ @Getter @@ -38,7 +38,7 @@ public class TxnBatchedPositionImpl extends PositionImpl { private final int batchIndex; public TxnBatchedPositionImpl(long ledgerId, long entryId, int batchSize, int batchIndex){ - super(ledgerId, entryId); + super(ledgerId, entryId, null); this.batchIndex = batchIndex; this.batchSize = batchSize; } @@ -48,9 +48,9 @@ public TxnBatchedPositionImpl(Position position, int batchSize, int batchIndex){ } /** - * It's exactly the same as {@link PositionImpl}，make sure that when {@link TxnBatchedPositionImpl} used as the key - * of map same as {@link PositionImpl}. {@link #batchSize} and {@link #batchIndex} should not be involved in - * calculate, just like {@link PositionImpl#ackSet} is not involved in calculate. + * It's exactly the same as {@link Position}，make sure that when {@link TxnBatchedPositionImpl} used as the key + * of map same as {@link Position}. {@link #batchSize} and {@link #batchIndex} should not be involved in + * calculate, just like the included {@link #ackSet} is not involved in calculate. * Note: In {@link java.util.concurrent.ConcurrentSkipListMap}, it use the {@link Comparable#compareTo(Object)} to * determine whether the keys are the same. In {@link java.util.HashMap}, it use the * {@link Object#hashCode()} & {@link Object#equals(Object)} to determine whether the keys are the same. @@ -58,13 +58,12 @@ public TxnBatchedPositionImpl(Position position, int batchSize, int batchIndex){ @Override public boolean equals(Object o) { return super.equals(o); - } /** - * It's exactly the same as {@link PositionImpl}，make sure that when {@link TxnBatchedPositionImpl} used as the key - * of map same as {@link PositionImpl}. {@link #batchSize} and {@link #batchIndex} should not be involved in - * calculate, just like {@link PositionImpl#ackSet} is not involved in calculate. + * It's exactly the same as {@link Position}，make sure that when {@link TxnBatchedPositionImpl} used as the key + * of map same as {@link Position}. {@link #batchSize} and {@link #batchIndex} should not be involved in + * calculate, just like the included {@link #ackSet} is not involved in calculate. * Note: In {@link java.util.concurrent.ConcurrentSkipListMap}, it use the {@link Comparable#compareTo(Object)} to * determine whether the keys are the same. In {@link java.util.HashMap}, it use the * {@link Object#hashCode()} & {@link Object#equals(Object)} to determine whether the keys are the same. @@ -75,14 +74,14 @@ public int hashCode() { } /** - * It's exactly the same as {@link PositionImpl}，to make sure that when compare to the "markDeletePosition", it - * looks like {@link PositionImpl}. {@link #batchSize} and {@link #batchIndex} should not be involved in calculate, - * just like {@link PositionImpl#ackSet} is not involved in calculate. + * It's exactly the same as {@link Position}，to make sure that when compare to the "markDeletePosition", it + * looks like {@link Position}. {@link #batchSize} and {@link #batchIndex} should not be involved in calculate, + * just like the included {@link #ackSet} is not involved in calculate. * Note: In {@link java.util.concurrent.ConcurrentSkipListMap}, it use the {@link Comparable#compareTo(Object)} to * determine whether the keys are the same. In {@link java.util.HashMap}, it use the * {@link Object#hashCode()} & {@link Object#equals(Object)} to determine whether the keys are the same. */ - public int compareTo(PositionImpl that) { + public int compareTo(Position that) { return super.compareTo(that); } diff --git a/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/MLTransactionMetadataStoreTest.java b/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/MLTransactionMetadataStoreTest.java index 3b831ad38ba1c..6ee7b3bb12883 100644 --- a/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/MLTransactionMetadataStoreTest.java +++ b/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/MLTransactionMetadataStoreTest.java @@ -18,9 +18,20 @@ */ package org.apache.pulsar.transaction.coordinator; +import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.State.WriteFailed; +import static org.apache.pulsar.transaction.coordinator.impl.DisabledTxnLogBufferedWriterMetricsStats.DISABLED_BUFFERED_WRITER_METRICS; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import io.netty.util.HashedWheelTimer; import io.netty.util.concurrent.DefaultThreadFactory; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import lombok.Cleanup; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; @@ -35,41 +46,34 @@ import org.apache.pulsar.transaction.coordinator.exceptions.CoordinatorException; import org.apache.pulsar.transaction.coordinator.exceptions.CoordinatorException.TransactionNotFoundException; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl; -import org.apache.pulsar.transaction.coordinator.impl.MLTransactionSequenceIdGenerator; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStore; +import org.apache.pulsar.transaction.coordinator.impl.MLTransactionSequenceIdGenerator; import org.apache.pulsar.transaction.coordinator.impl.TxnLogBufferedWriterConfig; import org.apache.pulsar.transaction.coordinator.proto.TxnStatus; import org.apache.pulsar.transaction.coordinator.test.MockedBookKeeperTestCase; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; - -import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.State.WriteFailed; -import static org.apache.pulsar.transaction.coordinator.impl.DisabledTxnLogBufferedWriterMetricsStats.DISABLED_BUFFERED_WRITER_METRICS; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; - public class MLTransactionMetadataStoreTest extends MockedBookKeeperTestCase { - private HashedWheelTimer transactionTimer = new HashedWheelTimer(new DefaultThreadFactory("transaction-timer"), - 1, TimeUnit.MILLISECONDS); + private HashedWheelTimer transactionTimer; public MLTransactionMetadataStoreTest() { super(3); } + @BeforeClass + public void initTimer() { + transactionTimer = new HashedWheelTimer(new DefaultThreadFactory("transaction-timer"), + 1, TimeUnit.MILLISECONDS); + } + @AfterClass - public void cleanup(){ + public void cleanupTimer(){ transactionTimer.stop(); } @@ -84,9 +88,11 @@ public void testTransactionOperation(TxnLogBufferedWriterConfig txnLogBufferedWr ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); MLTransactionSequenceIdGenerator mlTransactionSequenceIdGenerator = new MLTransactionSequenceIdGenerator(); managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionSequenceIdGenerator); + @Cleanup("closeAsync") MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, managedLedgerConfig, txnLogBufferedWriterConfig, transactionTimer, DISABLED_BUFFERED_WRITER_METRICS); mlTransactionLog.initialize().get(2, TimeUnit.SECONDS); + @Cleanup("closeAsync") MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, new TransactionTimeoutTrackerImpl(), @@ -172,9 +178,11 @@ public void testRecoverSequenceId(boolean isUseManagedLedgerProperties) throws E MLTransactionSequenceIdGenerator mlTransactionSequenceIdGenerator = new MLTransactionSequenceIdGenerator(); managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionSequenceIdGenerator); managedLedgerConfig.setMaxEntriesPerLedger(3); + @Cleanup("closeAsync") MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, managedLedgerConfig, disabledBufferedWriter, transactionTimer, DISABLED_BUFFERED_WRITER_METRICS); mlTransactionLog.initialize().get(2, TimeUnit.SECONDS); + @Cleanup("closeAsync") MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, new TransactionTimeoutTrackerImpl(), mlTransactionSequenceIdGenerator, 0L); @@ -203,6 +211,7 @@ public void testRecoverSequenceId(boolean isUseManagedLedgerProperties) throws E mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, managedLedgerConfig, disabledBufferedWriter, transactionTimer, DISABLED_BUFFERED_WRITER_METRICS); mlTransactionLog.initialize().get(2, TimeUnit.SECONDS); + transactionMetadataStore.closeAsync(); transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, new TransactionTimeoutTrackerImpl(), mlTransactionSequenceIdGenerator, 0L); @@ -229,10 +238,12 @@ public void testInitTransactionReader(TxnLogBufferedWriterConfig txnLogBufferedW managedLedgerConfig.setMaxEntriesPerLedger(2); MLTransactionSequenceIdGenerator mlTransactionSequenceIdGenerator = new MLTransactionSequenceIdGenerator(); managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionSequenceIdGenerator); + @Cleanup("closeAsync") MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, managedLedgerConfig, txnLogBufferedWriterConfig, transactionTimer, DISABLED_BUFFERED_WRITER_METRICS); mlTransactionLog.initialize().get(2, TimeUnit.SECONDS); + @Cleanup("closeAsync") MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, new TransactionTimeoutTrackerImpl(), mlTransactionSequenceIdGenerator, 0L); @@ -284,6 +295,7 @@ public void testInitTransactionReader(TxnLogBufferedWriterConfig txnLogBufferedW DISABLED_BUFFERED_WRITER_METRICS); txnLog2.initialize().get(2, TimeUnit.SECONDS); + @Cleanup("closeAsync") MLTransactionMetadataStore transactionMetadataStoreTest = new MLTransactionMetadataStore(transactionCoordinatorID, txnLog2, new TransactionTimeoutTrackerImpl(), mlTransactionSequenceIdGenerator, 0L); @@ -357,9 +369,11 @@ public void testDeleteLog(TxnLogBufferedWriterConfig txnLogBufferedWriterConfig) ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); MLTransactionSequenceIdGenerator mlTransactionSequenceIdGenerator = new MLTransactionSequenceIdGenerator(); managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionSequenceIdGenerator); + @Cleanup("closeAsync") MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, managedLedgerConfig, txnLogBufferedWriterConfig, transactionTimer, DISABLED_BUFFERED_WRITER_METRICS); mlTransactionLog.initialize().get(2, TimeUnit.SECONDS); + @Cleanup("closeAsync") MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, new TransactionTimeoutTrackerImpl(), mlTransactionSequenceIdGenerator, 0L); @@ -435,9 +449,11 @@ public void testRecoverWhenDeleteFromCursor(TxnLogBufferedWriterConfig txnLogBuf ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); MLTransactionSequenceIdGenerator mlTransactionSequenceIdGenerator = new MLTransactionSequenceIdGenerator(); managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionSequenceIdGenerator); + @Cleanup("closeAsync") MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, managedLedgerConfig, txnLogBufferedWriterConfig, transactionTimer, DISABLED_BUFFERED_WRITER_METRICS); mlTransactionLog.initialize().get(2, TimeUnit.SECONDS); + @Cleanup("closeAsync") MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, new TransactionTimeoutTrackerImpl(), mlTransactionSequenceIdGenerator, 0L); @@ -456,6 +472,7 @@ public void testRecoverWhenDeleteFromCursor(TxnLogBufferedWriterConfig txnLogBuf mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, managedLedgerConfig, txnLogBufferedWriterConfig, transactionTimer, DISABLED_BUFFERED_WRITER_METRICS); mlTransactionLog.initialize().get(2, TimeUnit.SECONDS); + transactionMetadataStore.closeAsync(); transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, new TransactionTimeoutTrackerImpl(), mlTransactionSequenceIdGenerator, 0L); @@ -475,9 +492,11 @@ public void testManageLedgerWriteFailState(TxnLogBufferedWriterConfig txnLogBuff ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); MLTransactionSequenceIdGenerator mlTransactionSequenceIdGenerator = new MLTransactionSequenceIdGenerator(); managedLedgerConfig.setManagedLedgerInterceptor(mlTransactionSequenceIdGenerator); + @Cleanup("closeAsync") MLTransactionLogImpl mlTransactionLog = new MLTransactionLogImpl(transactionCoordinatorID, factory, managedLedgerConfig, txnLogBufferedWriterConfig, transactionTimer, DISABLED_BUFFERED_WRITER_METRICS); mlTransactionLog.initialize().get(2, TimeUnit.SECONDS); + @Cleanup("closeAsync") MLTransactionMetadataStore transactionMetadataStore = new MLTransactionMetadataStore(transactionCoordinatorID, mlTransactionLog, new TransactionTimeoutTrackerImpl(), mlTransactionSequenceIdGenerator, 0L); @@ -505,8 +524,7 @@ public void testManageLedgerWriteFailState(TxnLogBufferedWriterConfig txnLogBuff public class TransactionTimeoutTrackerImpl implements TransactionTimeoutTracker { @Override - public CompletableFuture addTransaction(long sequenceId, long timeout) { - return null; + public void addTransaction(long sequenceId, long timeout) { } @Override diff --git a/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImplTest.java b/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImplTest.java index 4790b063e70f4..1d4e5dd2d0413 100644 --- a/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImplTest.java +++ b/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionLogImplTest.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.transaction.coordinator.impl; +import static org.apache.pulsar.transaction.coordinator.impl.DisabledTxnLogBufferedWriterMetricsStats.DISABLED_BUFFERED_WRITER_METRICS; +import static org.mockito.Mockito.mock; import com.google.common.collect.ComparisonChain; import io.netty.util.HashedWheelTimer; import io.netty.util.concurrent.DefaultThreadFactory; @@ -34,8 +36,8 @@ import java.util.stream.Collectors; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.common.api.proto.Subscription; import org.apache.pulsar.common.util.FutureUtil; @@ -46,8 +48,6 @@ import org.apache.pulsar.transaction.coordinator.proto.TransactionMetadataEntry; import org.apache.pulsar.transaction.coordinator.proto.TxnStatus; import org.apache.pulsar.transaction.coordinator.test.MockedBookKeeperTestCase; -import static org.apache.pulsar.transaction.coordinator.impl.DisabledTxnLogBufferedWriterMetricsStats.DISABLED_BUFFERED_WRITER_METRICS; -import static org.mockito.Mockito.*; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.DataProvider; @@ -269,10 +269,10 @@ factory, new ManagedLedgerConfig(), bufferedWriterConfigForRecover, transactionT .result(); } }).collect(Collectors.toList()); - PositionImpl markDeletedPosition = null; - LinkedHashMap batchIndexes = null; + Position markDeletedPosition = null; + LinkedHashMap batchIndexes = null; if (expectedDeletedPositions.get(0) instanceof TxnBatchedPositionImpl){ - Pair> pair = + Pair> pair = calculateBatchIndexes( expectedDeletedPositions.stream() .map(p -> (TxnBatchedPositionImpl)p) @@ -283,7 +283,7 @@ factory, new ManagedLedgerConfig(), bufferedWriterConfigForRecover, transactionT } else { markDeletedPosition = calculateMarkDeletedPosition(expectedDeletedPositions); } - final PositionImpl markDeletedPosition_final = markDeletedPosition; + final Position markDeletedPosition_final = markDeletedPosition; // Assert mark deleted position correct. Awaitility.await().atMost(2, TimeUnit.SECONDS).until(() -> { Position actualMarkDeletedPosition = managedCursor.getMarkDeletedPosition(); @@ -293,7 +293,7 @@ factory, new ManagedLedgerConfig(), bufferedWriterConfigForRecover, transactionT // Assert batchIndexes correct. if (batchIndexes != null){ // calculate last deleted position. - Map.Entry + Map.Entry lastOne = batchIndexes.entrySet().stream().reduce((a, b) -> b).get(); // Wait last one has been deleted from cursor. Awaitility.await().atMost(2, TimeUnit.SECONDS).until(() -> { @@ -301,8 +301,8 @@ factory, new ManagedLedgerConfig(), bufferedWriterConfigForRecover, transactionT return Arrays.equals(lastOne.getValue().toLongArray(), ls); }); // Verify batch indexes. - for (Map.Entry entry : batchIndexes.entrySet()){ - PositionImpl p = entry.getKey(); + for (Map.Entry entry : batchIndexes.entrySet()){ + Position p = entry.getKey(); long[] actualAckSet = managedCursor.getBatchPositionAckSet(p); Assert.assertEquals(actualAckSet, entry.getValue().toLongArray()); entry.getValue().recycle(); @@ -320,7 +320,7 @@ factory, new ManagedLedgerConfig(), bufferedWriterConfigForRecover, transactionT /*** * Calculate markDeletedPosition by {@param sortedDeletedPositions}. */ - private PositionImpl calculateMarkDeletedPosition(Collection sortedDeletedPositions){ + private Position calculateMarkDeletedPosition(Collection sortedDeletedPositions){ Position markDeletedPosition = null; for (Position position : sortedDeletedPositions){ if (markDeletedPosition == null){ @@ -338,19 +338,19 @@ private PositionImpl calculateMarkDeletedPosition(Collection sortedDel if (markDeletedPosition == null) { return null; } - return PositionImpl.get(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId()); + return PositionFactory.create(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId()); } /*** * Calculate markDeletedPosition and batchIndexes by {@param sortedDeletedPositions}. */ - private Pair> calculateBatchIndexes( + private Pair> calculateBatchIndexes( List sortedDeletedPositions){ // build batchIndexes. - LinkedHashMap batchIndexes = new LinkedHashMap<>(); + LinkedHashMap batchIndexes = new LinkedHashMap<>(); for (TxnBatchedPositionImpl batchedPosition : sortedDeletedPositions){ batchedPosition.setAckSetByIndex(); - PositionImpl k = PositionImpl.get(batchedPosition.getLedgerId(), batchedPosition.getEntryId()); + Position k = PositionFactory.create(batchedPosition.getLedgerId(), batchedPosition.getEntryId()); BitSetRecyclable bitSetRecyclable = batchIndexes.get(k); if (bitSetRecyclable == null){ bitSetRecyclable = BitSetRecyclable.valueOf(batchedPosition.getAckSet()); @@ -360,8 +360,8 @@ private Pair> calcul } // calculate markDeletedPosition. Position markDeletedPosition = null; - for (Map.Entry entry : batchIndexes.entrySet()){ - PositionImpl position = entry.getKey(); + for (Map.Entry entry : batchIndexes.entrySet()){ + Position position = entry.getKey(); BitSetRecyclable bitSetRecyclable = entry.getValue(); if (!bitSetRecyclable.isEmpty()){ break; @@ -380,7 +380,7 @@ private Pair> calcul } // remove empty bitSet. List shouldRemoveFromMap = new ArrayList<>(); - for (Map.Entry entry : batchIndexes.entrySet()) { + for (Map.Entry entry : batchIndexes.entrySet()) { BitSetRecyclable bitSetRecyclable = entry.getValue(); if (bitSetRecyclable.isEmpty()) { shouldRemoveFromMap.add(entry.getKey()); @@ -390,7 +390,7 @@ private Pair> calcul BitSetRecyclable bitSetRecyclable = batchIndexes.remove(position); bitSetRecyclable.recycle(); } - return Pair.of(PositionImpl.get(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId()), + return Pair.of(PositionFactory.create(markDeletedPosition.getLedgerId(), markDeletedPosition.getEntryId()), batchIndexes); } } \ No newline at end of file diff --git a/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/impl/TxnBatchedPositionImplTest.java b/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/impl/TxnBatchedPositionImplTest.java index 0905fdad72d40..a1263ae71d299 100644 --- a/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/impl/TxnBatchedPositionImplTest.java +++ b/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/impl/TxnBatchedPositionImplTest.java @@ -24,7 +24,8 @@ import java.util.Random; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentSkipListMap; -import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.pulsar.common.util.collections.BitSetRecyclable; import org.testng.Assert; import org.testng.annotations.DataProvider; @@ -74,7 +75,7 @@ public Object[][] testHashcodeAndEqualsData(){ /** * Why is this test needed ？ - * {@link org.apache.bookkeeper.mledger.impl.ManagedCursorImpl} maintains batchIndexes, use {@link PositionImpl} or + * {@link org.apache.bookkeeper.mledger.impl.ManagedCursorImpl} maintains batchIndexes, use {@link Position} or * {@link TxnBatchedPositionImpl} as the key. However, different maps may use "param-key.equals(key-in-map)" to * determine the contains, or use "key-in-map.equals(param-key)" or use "param-key.compareTo(key-in-map)" or use * "key-in-map.compareTo(param-key)" to determine the {@link Map#containsKey(Object)}, the these approaches may @@ -88,48 +89,48 @@ public void testKeyInMap(long ledgerId, long entryId, int batchSize, int batchIn // build data. Random random = new Random(); int v = random.nextInt(); - PositionImpl position = new PositionImpl(ledgerId, entryId); + Position position = PositionFactory.create(ledgerId, entryId); TxnBatchedPositionImpl txnBatchedPosition = new TxnBatchedPositionImpl(position, batchSize, batchIndex); // ConcurrentSkipListMap. - ConcurrentSkipListMap map1 = new ConcurrentSkipListMap<>(); + ConcurrentSkipListMap map1 = new ConcurrentSkipListMap<>(); map1.put(position, v); Assert.assertTrue(map1.containsKey(txnBatchedPosition)); - ConcurrentSkipListMap map2 = new ConcurrentSkipListMap<>(); + ConcurrentSkipListMap map2 = new ConcurrentSkipListMap<>(); map2.put(txnBatchedPosition, v); Assert.assertTrue(map2.containsKey(position)); // HashMap. - HashMap map3 = new HashMap<>(); + HashMap map3 = new HashMap<>(); map3.put(position, v); Assert.assertTrue(map3.containsKey(txnBatchedPosition)); - HashMap map4 = new HashMap<>(); + HashMap map4 = new HashMap<>(); map4.put(txnBatchedPosition, v); Assert.assertTrue(map4.containsKey(position)); // ConcurrentHashMap. - ConcurrentHashMap map5 = new ConcurrentHashMap<>(); + ConcurrentHashMap map5 = new ConcurrentHashMap<>(); map5.put(position, v); Assert.assertTrue(map5.containsKey(txnBatchedPosition)); - ConcurrentHashMap map6 = new ConcurrentHashMap<>(); + ConcurrentHashMap map6 = new ConcurrentHashMap<>(); map6.put(txnBatchedPosition, v); Assert.assertTrue(map6.containsKey(position)); // LinkedHashMap. - LinkedHashMap map7 = new LinkedHashMap<>(); + LinkedHashMap map7 = new LinkedHashMap<>(); map7.put(position, v); Assert.assertTrue(map7.containsKey(txnBatchedPosition)); - LinkedHashMap map8 = new LinkedHashMap<>(); + LinkedHashMap map8 = new LinkedHashMap<>(); map8.put(txnBatchedPosition, v); Assert.assertTrue(map8.containsKey(position)); } /** * Why is this test needed ？ - * Make sure that when compare to the "markDeletePosition", it looks like {@link PositionImpl} + * Make sure that when compare to the "markDeletePosition", it looks like {@link Position} * Note: In {@link java.util.concurrent.ConcurrentSkipListMap}, it use the {@link Comparable#compareTo(Object)} to * determine whether the keys are the same. In {@link java.util.HashMap}, it use the * {@link Object#hashCode()} & {@link Object#equals(Object)} to determine whether the keys are the same. */ @Test public void testCompareTo(){ - PositionImpl position = new PositionImpl(1, 1); + Position position = PositionFactory.create(1, 1); TxnBatchedPositionImpl txnBatchedPosition = new TxnBatchedPositionImpl(position, 2, 0); Assert.assertEquals(position.compareTo(txnBatchedPosition), 0); Assert.assertEquals(txnBatchedPosition.compareTo(position), 0); diff --git a/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/impl/TxnLogBufferedWriterTest.java b/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/impl/TxnLogBufferedWriterTest.java index 3c6b2e382e311..3147279477843 100644 --- a/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/impl/TxnLogBufferedWriterTest.java +++ b/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/impl/TxnLogBufferedWriterTest.java @@ -18,6 +18,9 @@ */ package org.apache.pulsar.transaction.coordinator.impl; +import static org.apache.pulsar.transaction.coordinator.impl.DisabledTxnLogBufferedWriterMetricsStats.DISABLED_BUFFERED_WRITER_METRICS; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; import com.fasterxml.jackson.databind.ObjectMapper; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; @@ -53,8 +56,8 @@ import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.collections4.CollectionUtils; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.transaction.coordinator.test.MockedBookKeeperTestCase; @@ -62,8 +65,6 @@ import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import static org.apache.pulsar.transaction.coordinator.impl.DisabledTxnLogBufferedWriterMetricsStats.DISABLED_BUFFERED_WRITER_METRICS; -import static org.testng.Assert.*; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -194,7 +195,7 @@ public void testMainProcess(int batchedWriteMaxRecords, int batchedWriteMaxSize, // Store the param-context, param-position, param-exception of callback function and complete-count for verify. List contextArrayOfCallback = Collections.synchronizedList(new ArrayList<>()); Map exceptionArrayOfCallback = new ConcurrentHashMap<>(); - Map> positionsOfCallback = Collections.synchronizedMap(new LinkedHashMap<>()); + Map> positionsOfCallback = Collections.synchronizedMap(new LinkedHashMap<>()); AtomicBoolean anyFlushCompleted = new AtomicBoolean(); TxnLogBufferedWriter.AddDataCallback callback = new TxnLogBufferedWriter.AddDataCallback(){ @Override @@ -204,7 +205,7 @@ public void addComplete(Position position, Object ctx) { return; } contextArrayOfCallback.add((int)ctx); - PositionImpl lightPosition = PositionImpl.get(position.getLedgerId(), position.getEntryId()); + Position lightPosition = PositionFactory.create(position.getLedgerId(), position.getEntryId()); positionsOfCallback.computeIfAbsent(lightPosition, p -> Collections.synchronizedList(new ArrayList<>())); positionsOfCallback.get(lightPosition).add(position); @@ -299,7 +300,7 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { * Note2: Verify that all entry was written in strict order. */ if (BookieErrorType.NO_ERROR == bookieErrorType) { - Iterator callbackPositionIterator = positionsOfCallback.keySet().iterator(); + Iterator callbackPositionIterator = positionsOfCallback.keySet().iterator(); List dataArrayWrite = dataSerializer.getGeneratedJsonArray(); int entryCounter = 0; while (managedCursor.hasMoreEntries()) { @@ -311,7 +312,7 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { // Get data read. Entry entry = entries.get(m); // Assert the position of the read matches the position of the callback. - PositionImpl callbackPosition = callbackPositionIterator.next(); + Position callbackPosition = callbackPositionIterator.next(); assertEquals(entry.getLedgerId(), callbackPosition.getLedgerId()); assertEquals(entry.getEntryId(), callbackPosition.getEntryId()); if (exactlyBatched) { @@ -394,7 +395,7 @@ public Object answer(InvocationOnMock invocation) throws Throwable { dataArrayFlushedToBookie.add(byteBuf.readInt()); AsyncCallbacks.AddEntryCallback callback = (AsyncCallbacks.AddEntryCallback) invocation.getArguments()[1]; - callback.addComplete(PositionImpl.get(1,1), byteBuf, + callback.addComplete(PositionFactory.create(1,1), byteBuf, invocation.getArguments()[2]); return null; } @@ -927,6 +928,7 @@ private void releaseTxnLogBufferedWriterContext(TxnLogBufferedWriterContext cont context.txnLogBufferedWriter.close().get(); context.metrics.close(); context.timer.stop(); + context.orderedExecutor.shutdownNow(); CollectorRegistry.defaultRegistry.clear(); } @@ -936,6 +938,7 @@ private static class TxnLogBufferedWriterContext{ MockedManagedLedger mockedManagedLedger; Timer timer; TxnLogBufferedWriterMetricsStats metrics; + OrderedExecutor orderedExecutor; } @AllArgsConstructor @@ -970,7 +973,7 @@ private TxnLogBufferedWriterContext createTxnBufferedWriterContextWithMetrics( dataSerializer, batchedWriteMaxRecords, batchedWriteMaxSize, batchedWriteMaxDelayInMillis, true, metricsStats); return new TxnLogBufferedWriterContext(txnLogBufferedWriter, mockedManagedLedger, transactionTimer, - metricsStats); + metricsStats, orderedExecutor); } private void verifyTheCounterMetrics(int triggeredByRecordCount, int triggeredByMaxSize, int triggeredByMaxDelay, @@ -1020,7 +1023,7 @@ public Object answer(InvocationOnMock invocation) throws Throwable { writeCounter.incrementAndGet(); AsyncCallbacks.AddEntryCallback callback = (AsyncCallbacks.AddEntryCallback) invocation.getArguments()[1]; - callback.addComplete(PositionImpl.get(1,1), (ByteBuf)invocation.getArguments()[0], + callback.addComplete(PositionFactory.create(1,1), (ByteBuf)invocation.getArguments()[0], invocation.getArguments()[2]); return null; } diff --git a/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/test/MockedBookKeeperTestCase.java b/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/test/MockedBookKeeperTestCase.java index e0b10ca0280d2..ac5aa3bd8927e 100644 --- a/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/test/MockedBookKeeperTestCase.java +++ b/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/test/MockedBookKeeperTestCase.java @@ -71,7 +71,9 @@ public MockedBookKeeperTestCase(int numBookies) { public void setUp(Method method) throws Exception { LOG.info(">>>>>> starting {}", method); metadataStore = new FaultInjectionMetadataStore(MetadataStoreExtended.create("memory:local", - MetadataStoreConfig.builder().build())); + MetadataStoreConfig.builder() + .metadataStoreName("metastore-" + method.getName()) + .build())); try { // start bookkeeper service startBookKeeper(); diff --git a/pulsar-transaction/pom.xml b/pulsar-transaction/pom.xml index 5a43cf14665f7..8407ad070b743 100644 --- a/pulsar-transaction/pom.xml +++ b/pulsar-transaction/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-transaction-parent diff --git a/pulsar-websocket/pom.xml b/pulsar-websocket/pom.xml index f742c286f0b68..8772df9c358c6 100644 --- a/pulsar-websocket/pom.xml +++ b/pulsar-websocket/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT pulsar-websocket @@ -52,6 +51,18 @@ test + + ${project.groupId} + pulsar-docs-tools + ${project.version} + + + io.swagger + * + + + + org.apache.commons commons-lang3 @@ -80,6 +91,7 @@ io.swagger swagger-core + provided diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/AbstractWebSocketHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/AbstractWebSocketHandler.java index 3eb0a0dfcf8ca..b6ed27c87b6ba 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/AbstractWebSocketHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/AbstractWebSocketHandler.java @@ -67,7 +67,7 @@ public abstract class AbstractWebSocketHandler extends WebSocketAdapter implemen protected final WebSocketService service; protected final HttpServletRequest request; - protected final TopicName topic; + protected TopicName topic; protected final Map queryParams; private static final String PULSAR_AUTH_METHOD_NAME = "X-Pulsar-Auth-Method-Name"; protected final ObjectReader consumerCommandReader = @@ -80,12 +80,12 @@ public AbstractWebSocketHandler(WebSocketService service, ServletUpgradeResponse response) { this.service = service; this.request = new WebSocketHttpServletRequestWrapper(request); - this.topic = extractTopicName(request); this.queryParams = new TreeMap<>(); request.getParameterMap().forEach((key, values) -> { queryParams.put(key, values[0]); }); + extractTopicName(request); } protected boolean checkAuth(ServletUpgradeResponse response) { @@ -244,7 +244,7 @@ protected String checkAuthentication() { return null; } - private TopicName extractTopicName(HttpServletRequest request) { + protected void extractTopicName(HttpServletRequest request) { String uri = request.getRequestURI(); List parts = Splitter.on("/").splitToList(uri); @@ -287,7 +287,7 @@ private TopicName extractTopicName(HttpServletRequest request) { } final String name = Codec.decode(topicName.toString()); - return TopicName.get(domain, namespace, name); + topic = TopicName.get(domain, namespace, name); } @VisibleForTesting diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java index 2ab62b10ee9ee..b93c4b215108e 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java @@ -41,15 +41,13 @@ import org.apache.pulsar.client.api.ConsumerCryptoFailureAction; import org.apache.pulsar.client.api.DeadLetterPolicy; import org.apache.pulsar.client.api.MessageId; -import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.PulsarClientException.AlreadyClosedException; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionMode; import org.apache.pulsar.client.api.SubscriptionType; -import org.apache.pulsar.client.api.TopicMessageId; import org.apache.pulsar.client.impl.ConsumerBuilderImpl; -import org.apache.pulsar.client.impl.TopicMessageIdImpl; import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.DateFormatter; @@ -75,7 +73,7 @@ */ public class ConsumerHandler extends AbstractWebSocketHandler { - private String subscription = null; + protected String subscription = null; private SubscriptionType subscriptionType; private SubscriptionMode subscriptionMode; private Consumer consumer; @@ -88,6 +86,10 @@ public class ConsumerHandler extends AbstractWebSocketHandler { private final LongAdder numBytesDelivered; private final LongAdder numMsgsAcked; private volatile long msgDeliveredCounter = 0; + + protected String topicsPattern; + + protected String topics; private static final AtomicLongFieldUpdater MSG_DELIVERED_COUNTER_UPDATER = AtomicLongFieldUpdater.newUpdater(ConsumerHandler.class, "msgDeliveredCounter"); @@ -123,7 +125,14 @@ public ConsumerHandler(WebSocketService service, HttpServletRequest request, Ser return; } - this.consumer = builder.topic(topic.toString()).subscriptionName(subscription).subscribe(); + if (topicsPattern != null) { + this.consumer = builder.topicsPattern(topicsPattern).subscriptionName(subscription).subscribe(); + } else if (topics != null) { + this.consumer = builder.topics(Splitter.on(",").splitToList(topics)) + .subscriptionName(subscription).subscribe(); + } else { + this.consumer = builder.topic(topic.toString()).subscriptionName(subscription).subscribe(); + } if (!this.service.addConsumer(this)) { log.warn("[{}:{}] Failed to add consumer handler for topic {}", request.getRemoteAddr(), request.getRemotePort(), topic); @@ -299,8 +308,7 @@ private void checkResumeReceive() { private void handleAck(ConsumerCommand command) throws IOException { // We should have received an ack - TopicMessageId msgId = new TopicMessageIdImpl(topic.toString(), - (MessageIdAdv) MessageId.fromByteArray(Base64.getDecoder().decode(command.messageId))); + MessageId msgId = MessageId.fromByteArray(Base64.getDecoder().decode(command.messageId)); if (log.isDebugEnabled()) { log.debug("[{}/{}] Received ack request of message {} from {} ", consumer.getTopic(), subscription, msgId, getRemote().getInetSocketAddress().toString()); @@ -424,6 +432,14 @@ protected ConsumerBuilder getConsumerConfiguration(PulsarClient client) builder.subscriptionMode(SubscriptionMode.valueOf(queryParams.get("subscriptionMode"))); } + if (queryParams.containsKey("subscriptionInitialPosition")) { + final String subscriptionInitialPosition = queryParams.get("subscriptionInitialPosition"); + checkArgument( + Enums.getIfPresent(SubscriptionInitialPosition.class, subscriptionInitialPosition).isPresent(), + "Invalid subscriptionInitialPosition %s", subscriptionInitialPosition); + builder.subscriptionInitialPosition(SubscriptionInitialPosition.valueOf(subscriptionInitialPosition)); + } + if (queryParams.containsKey("receiverQueueSize")) { builder.receiverQueueSize(Math.min(Integer.parseInt(queryParams.get("receiverQueueSize")), 1000)); } @@ -465,6 +481,8 @@ protected ConsumerBuilder getConsumerConfiguration(PulsarClient client) if (service.getCryptoKeyReader().isPresent()) { builder.cryptoKeyReader(service.getCryptoKeyReader().get()); + } else { + // If users want to decrypt messages themselves, they should set "cryptoFailureAction" to "CONSUME". } return builder; } @@ -488,7 +506,7 @@ protected Boolean isAuthorized(String authRole, AuthenticationDataSource authent } } - public static String extractSubscription(HttpServletRequest request) { + public String extractSubscription(HttpServletRequest request) { String uri = request.getRequestURI(); List parts = Splitter.on("/").splitToList(uri); diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/MultiTopicConsumerHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/MultiTopicConsumerHandler.java new file mode 100644 index 0000000000000..7fbe257d2e249 --- /dev/null +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/MultiTopicConsumerHandler.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.websocket; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.concurrent.TimeUnit.SECONDS; +import com.google.common.base.Splitter; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeoutException; +import javax.servlet.http.HttpServletRequest; +import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TopicOperation; +import org.apache.pulsar.common.util.Codec; +import org.apache.pulsar.common.util.FutureUtil; +import org.eclipse.jetty.websocket.servlet.ServletUpgradeResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Subscribing for multi-topic. + */ +public class MultiTopicConsumerHandler extends ConsumerHandler { + + public MultiTopicConsumerHandler(WebSocketService service, HttpServletRequest request, + ServletUpgradeResponse response) { + super(service, request, response); + } + + @Override + protected Boolean isAuthorized(String authRole, AuthenticationDataSource authenticationData) throws Exception { + try { + AuthenticationDataSubscription subscription = new AuthenticationDataSubscription(authenticationData, + this.subscription); + if (topics != null) { + List topicNames = Splitter.on(",").splitToList(topics); + List> futures = new ArrayList<>(); + for (String topicName : topicNames) { + futures.add(service.getAuthorizationService() + .allowTopicOperationAsync(TopicName.get(topicName), + TopicOperation.CONSUME, authRole, subscription)); + } + FutureUtil.waitForAll(futures) + .get(service.getConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS); + return futures.stream().allMatch(f -> f.join()); + } else { + return service.getAuthorizationService() + .allowTopicOperationAsync(topic, TopicOperation.CONSUME, authRole, subscription) + .get(service.getConfig().getMetadataStoreOperationTimeoutSeconds(), SECONDS); + } + } catch (TimeoutException e) { + log.warn("Time-out {} sec while checking authorization on {} ", + service.getConfig().getMetadataStoreOperationTimeoutSeconds(), topic); + throw e; + } catch (Exception e) { + log.warn("Consumer-client with Role - {} failed to get permissions for topic - {}. {}", authRole, topic, + e.getMessage()); + throw e; + } + } + + @Override + protected void extractTopicName(HttpServletRequest request) { + String uri = request.getRequestURI(); + List parts = Splitter.on("/").splitToList(uri); + + // V3 Format must be like : + // /ws/v3/consumer/my-subscription?topicsPattern="a.*" //ws/v3/consumer/my-subscription?topics="a,b,c" + checkArgument(parts.size() >= 4, "Invalid topic name format"); + checkArgument(parts.get(2).equals("v3")); + checkArgument(queryParams.containsKey("topicsPattern") || queryParams.containsKey("topics"), + "Should set topics or topicsPattern"); + checkArgument(!(queryParams.containsKey("topicsPattern") && queryParams.containsKey("topics")), + "Topics must be null when use topicsPattern"); + topicsPattern = queryParams.get("topicsPattern"); + topics = queryParams.get("topics"); + if (topicsPattern != null) { + topic = TopicName.get(topicsPattern); + } else { + // Multi topics only use the first topic name， + topic = TopicName.get(Splitter.on(",").splitToList(topics).get(0)); + } + } + + @Override + public String extractSubscription(HttpServletRequest request) { + String uri = request.getRequestURI(); + List parts = Splitter.on("/").splitToList(uri); + // v3 Format must be like : + // /ws/v3/consumer/my-subscription?topicsPattern="a.*" //ws/v3/consumer/my-subscription?topics="a,b,c" + checkArgument(parts.size() >= 5 , "Invalid topic name format"); + checkArgument(parts.get(1).equals("ws")); + checkArgument(parts.get(2).equals("v3")); + checkArgument(parts.get(4).length() > 0, "Empty subscription name"); + + return Codec.decode(parts.get(4)); + } + + private static final Logger log = LoggerFactory.getLogger(MultiTopicConsumerHandler.class); +} diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/PingPongHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/PingPongHandler.java deleted file mode 100644 index 870630abc8889..0000000000000 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/PingPongHandler.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.websocket; - -import java.io.IOException; -import java.nio.ByteBuffer; -import org.eclipse.jetty.util.BufferUtil; -import org.eclipse.jetty.websocket.api.WebSocketAdapter; -import org.eclipse.jetty.websocket.api.WebSocketPingPongListener; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class PingPongHandler extends WebSocketAdapter implements WebSocketPingPongListener { - - private static final Logger log = LoggerFactory.getLogger(PingPongHandler.class); - - @Override - public void onWebSocketPing(ByteBuffer payload) { - try { - if (log.isDebugEnabled()) { - log.debug("PING: {}", BufferUtil.toDetailString(payload)); - } - getRemote().sendPong(payload); - } catch (IOException e) { - log.warn("Failed to send pong: {}", e.getMessage()); - } - } - - @Override - public void onWebSocketPong(ByteBuffer payload) { - - } - -} \ No newline at end of file diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java index 5ad1283fe84c4..3c0f42935e6bb 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java @@ -21,10 +21,12 @@ import static com.google.common.base.Preconditions.checkArgument; import static java.lang.String.format; import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.pulsar.common.api.EncryptionContext.EncryptionKey; import static org.apache.pulsar.websocket.WebSocketError.FailedToDeserializeFromJSON; import static org.apache.pulsar.websocket.WebSocketError.PayloadEncodingError; import static org.apache.pulsar.websocket.WebSocketError.UnknownError; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectReader; import com.google.common.base.Enums; import java.io.IOException; @@ -33,25 +35,32 @@ import java.util.Base64; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.atomic.LongAdder; +import java.util.stream.Collectors; import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.client.api.CompressionType; +import org.apache.pulsar.client.api.DummyCryptoKeyReaderImpl; import org.apache.pulsar.client.api.HashingScheme; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.SchemaSerializationException; -import org.apache.pulsar.client.api.TypedMessageBuilder; +import org.apache.pulsar.client.impl.TypedMessageBuilderImpl; +import org.apache.pulsar.common.api.proto.KeyValue; import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.websocket.data.ProducerAck; import org.apache.pulsar.websocket.data.ProducerMessage; +import org.apache.pulsar.websocket.service.WSSDummyMessageCryptoImpl; import org.apache.pulsar.websocket.stats.StatsBuckets; import org.eclipse.jetty.websocket.api.WriteCallback; import org.eclipse.jetty.websocket.servlet.ServletUpgradeResponse; @@ -76,6 +85,7 @@ public class ProducerHandler extends AbstractWebSocketHandler { private final LongAdder numBytesSent; private final StatsBuckets publishLatencyStatsUSec; private volatile long msgPublishedCounter = 0; + private boolean clientSideEncrypt; private static final AtomicLongFieldUpdater MSG_PUBLISHED_COUNTER_UPDATER = AtomicLongFieldUpdater.newUpdater(ProducerHandler.class, "msgPublishedCounter"); @@ -98,16 +108,29 @@ public ProducerHandler(WebSocketService service, HttpServletRequest request, Ser try { this.producer = getProducerBuilder(service.getPulsarClient()).topic(topic.toString()).create(); + if (clientSideEncrypt) { + log.info("[{}] [{}] The producer session is created with param encryptionKeyValues, which means that" + + " message encryption will be done on the client side, then the server will skip " + + "batch message processing, message compression processing, and message encryption" + + " processing", producer.getTopic(), producer.getProducerName()); + } if (!this.service.addProducer(this)) { log.warn("[{}:{}] Failed to add producer handler for topic {}", request.getRemoteAddr(), request.getRemotePort(), topic); } } catch (Exception e) { - log.warn("[{}:{}] Failed in creating producer on topic {}: {}", request.getRemoteAddr(), - request.getRemotePort(), topic, e.getMessage()); + int errorCode = getErrorCode(e); + boolean isKnownError = errorCode != HttpServletResponse.SC_INTERNAL_SERVER_ERROR; + if (isKnownError) { + log.warn("[{}:{}] Failed in creating producer on topic {}: {}", request.getRemoteAddr(), + request.getRemotePort(), topic, e.getMessage()); + } else { + log.error("[{}:{}] Failed in creating producer on topic {}: {}", request.getRemoteAddr(), + request.getRemotePort(), topic, e.getMessage(), e); + } try { - response.sendError(getErrorCode(e), getErrorMessage(e)); + response.sendError(errorCode, getErrorMessage(e)); } catch (IOException e1) { log.warn("[{}:{}] Failed to send error: {}", request.getRemoteAddr(), request.getRemotePort(), e1.getMessage(), e1); @@ -159,7 +182,7 @@ public void onWebSocketText(String message) { } final long msgSize = rawPayload.length; - TypedMessageBuilder builder = producer.newMessage(); + TypedMessageBuilderImpl builder = (TypedMessageBuilderImpl) producer.newMessage(); try { builder.value(rawPayload); @@ -192,6 +215,37 @@ public void onWebSocketText(String message) { builder.deliverAfter(sendRequest.deliverAfterMs, TimeUnit.MILLISECONDS); } + // If client-side encryption is enabled, the attributes "encryptParam", "uncompressedMessageSize", + // "uncompressedMessageSize" and "batchSize" of message metadata must be set according to the parameters + // when the client sends messages. + if (clientSideEncrypt) { + try { + if (!StringUtils.isBlank(sendRequest.encryptionParam)) { + builder.getMetadataBuilder().setEncryptionParam(Base64.getDecoder() + .decode(sendRequest.encryptionParam)); + } + } catch (Exception e){ + String msg = format("Invalid Base64 encryptionParam error=%s", e.getMessage()); + sendAckResponse(new ProducerAck(PayloadEncodingError, msg, null, requestContext)); + return; + } + if (sendRequest.compressionType != null && sendRequest.uncompressedMessageSize != null) { + // Set compression information. + builder.getMetadataBuilder().setCompression(sendRequest.compressionType); + builder.getMetadataBuilder().setUncompressedSize(sendRequest.uncompressedMessageSize); + } else if ((org.apache.pulsar.common.api.proto.CompressionType.NONE.equals(sendRequest.compressionType) + || sendRequest.compressionType == null) + && sendRequest.uncompressedMessageSize == null) { + // Nothing to do, the method send async will set these two attributes. + } else { + // Only one param is set. + sendAckResponse(new ProducerAck(PayloadEncodingError, "the params compressionType and" + + " uncompressedMessageSize should both empty or both non-empty", + null, requestContext)); + return; + } + } + final long now = System.nanoTime(); builder.sendAsync().thenAccept(msgId -> { @@ -205,8 +259,8 @@ public void onWebSocketText(String message) { sendAckResponse(new ProducerAck(messageId, sendRequest.context)); } }).exceptionally(exception -> { - log.warn("[{}] Error occurred while producer handler was sending msg from {}: {}", producer.getTopic(), - getRemote().getInetSocketAddress().toString(), exception.getMessage()); + log.warn("[{}] Error occurred while producer handler was sending msg from {}", producer.getTopic(), + getRemote().getInetSocketAddress().toString(), exception); numMsgsFailed.increment(); sendAckResponse( new ProducerAck(UnknownError, exception.getMessage(), null, sendRequest.context)); @@ -315,30 +369,127 @@ protected ProducerBuilder getProducerBuilder(PulsarClient client) { builder.sendTimeout(Integer.parseInt(queryParams.get("sendTimeoutMillis")), TimeUnit.MILLISECONDS); } - if (queryParams.containsKey("batchingEnabled")) { - builder.enableBatching(Boolean.parseBoolean(queryParams.get("batchingEnabled"))); + if (queryParams.containsKey("messageRoutingMode")) { + checkArgument( + Enums.getIfPresent(MessageRoutingMode.class, queryParams.get("messageRoutingMode")).isPresent(), + "Invalid messageRoutingMode %s", queryParams.get("messageRoutingMode")); + MessageRoutingMode routingMode = MessageRoutingMode.valueOf(queryParams.get("messageRoutingMode")); + if (!MessageRoutingMode.CustomPartition.equals(routingMode)) { + builder.messageRoutingMode(routingMode); + } } - if (queryParams.containsKey("batchingMaxMessages")) { - builder.batchingMaxMessages(Integer.parseInt(queryParams.get("batchingMaxMessages"))); + Map encryptionKeyMap = tryToExtractJsonEncryptionKeys(); + if (encryptionKeyMap != null) { + popularProducerBuilderForClientSideEncrypt(builder, encryptionKeyMap); + } else { + popularProducerBuilderForServerSideEncrypt(builder); } + return builder; + } - if (queryParams.containsKey("maxPendingMessages")) { - builder.maxPendingMessages(Integer.parseInt(queryParams.get("maxPendingMessages"))); + private Map tryToExtractJsonEncryptionKeys() { + if (!queryParams.containsKey("encryptionKeys")) { + return null; + } + // Base64 decode. + byte[] param = null; + try { + param = Base64.getDecoder().decode(StringUtils.trim(queryParams.get("encryptionKeys"))); + } catch (Exception base64DecodeEx) { + return null; } + try { + Map keys = ObjectMapperFactory.getMapper().getObjectMapper() + .readValue(param, new TypeReference>() {}); + if (keys.isEmpty()) { + return null; + } + if (keys.values().iterator().next().getKeyValue() == null) { + return null; + } + return keys; + } catch (IOException ex) { + return null; + } + } - if (queryParams.containsKey("batchingMaxPublishDelay")) { - builder.batchingMaxPublishDelay(Integer.parseInt(queryParams.get("batchingMaxPublishDelay")), - TimeUnit.MILLISECONDS); + private void popularProducerBuilderForClientSideEncrypt(ProducerBuilder builder, + Map encryptionKeyMap) { + this.clientSideEncrypt = true; + int keysLen = encryptionKeyMap.size(); + final String[] keyNameArray = new String[keysLen]; + final byte[][] keyValueArray = new byte[keysLen][]; + final List[] keyMetadataArray = new List[keysLen]; + // Format keys. + int index = 0; + for (Map.Entry entry : encryptionKeyMap.entrySet()) { + checkArgument(StringUtils.isNotBlank(entry.getKey()), "Empty param encryptionKeys.key"); + checkArgument(entry.getValue() != null, "Empty param encryptionKeys.value"); + checkArgument(entry.getValue().getKeyValue() != null, "Empty param encryptionKeys.value.keyValue"); + keyNameArray[index] = StringUtils.trim(entry.getKey()); + keyValueArray[index] = entry.getValue().getKeyValue(); + if (entry.getValue().getMetadata() == null) { + keyMetadataArray[index] = Collections.emptyList(); + } else { + keyMetadataArray[index] = entry.getValue().getMetadata().entrySet().stream() + .map(e -> new KeyValue().setKey(e.getKey()).setValue(e.getValue())) + .collect(Collectors.toList()); + } + builder.addEncryptionKey(keyNameArray[index]); } + // Background: The order of message payload process during message sending: + // 1. The Producer will composite several message payloads into a batched message payload if the producer is + // enabled batch; + // 2. The Producer will compress the batched message payload to a compressed payload if enabled compression; + // 3. After the previous two steps, the Producer encrypts the compressed payload to an encrypted payload. + // + // Since the order of producer operation for message payloads is "compression --> encryption", users need to + // handle Compression themselves if needed. We just disable server-side batch process, server-side compression, + // and server-side encryption, and only set the message metadata that. + builder.enableBatching(false); + // Disable server-side compression, and just set compression attributes into the message metadata when sending + // messages(see the method "onWebSocketText"). + builder.compressionType(CompressionType.NONE); + // Disable server-side encryption, and just set encryption attributes into the message metadata when sending + // messages(see the method "onWebSocketText"). + builder.cryptoKeyReader(DummyCryptoKeyReaderImpl.INSTANCE); + // Set the param `enableChunking` to `false`(the default value is `false`) to prevent unexpected problems if + // the default setting is changed in the future. + builder.enableChunking(false); + // Inject encryption metadata decorator. + builder.messageCrypto(new WSSDummyMessageCryptoImpl(msgMetadata -> { + for (int i = 0; i < keyNameArray.length; i++) { + msgMetadata.addEncryptionKey().setKey(keyNameArray[i]).setValue(keyValueArray[i]) + .addAllMetadatas(keyMetadataArray[i]); + } + })); + // Do warning param check and print warning log. + printLogIfSettingDiscardedBatchedParams(); + printLogIfSettingDiscardedCompressionParams(); + } - if (queryParams.containsKey("messageRoutingMode")) { - checkArgument( - Enums.getIfPresent(MessageRoutingMode.class, queryParams.get("messageRoutingMode")).isPresent(), - "Invalid messageRoutingMode %s", queryParams.get("messageRoutingMode")); - MessageRoutingMode routingMode = MessageRoutingMode.valueOf(queryParams.get("messageRoutingMode")); - if (!MessageRoutingMode.CustomPartition.equals(routingMode)) { - builder.messageRoutingMode(routingMode); + private void popularProducerBuilderForServerSideEncrypt(ProducerBuilder builder) { + this.clientSideEncrypt = false; + if (queryParams.containsKey("batchingEnabled")) { + boolean batchingEnabled = Boolean.parseBoolean(queryParams.get("batchingEnabled")); + if (batchingEnabled) { + builder.enableBatching(true); + if (queryParams.containsKey("batchingMaxMessages")) { + builder.batchingMaxMessages(Integer.parseInt(queryParams.get("batchingMaxMessages"))); + } + + if (queryParams.containsKey("maxPendingMessages")) { + builder.maxPendingMessages(Integer.parseInt(queryParams.get("maxPendingMessages"))); + } + + if (queryParams.containsKey("batchingMaxPublishDelay")) { + builder.batchingMaxPublishDelay(Integer.parseInt(queryParams.get("batchingMaxPublishDelay")), + TimeUnit.MILLISECONDS); + } + } else { + builder.enableBatching(false); + printLogIfSettingDiscardedBatchedParams(); } } @@ -356,7 +507,27 @@ protected ProducerBuilder getProducerBuilder(PulsarClient client) { builder.addEncryptionKey(key); } } - return builder; + } + + private void printLogIfSettingDiscardedBatchedParams() { + if (clientSideEncrypt && queryParams.containsKey("batchingEnabled")) { + log.info("Since clientSideEncrypt is true, the param batchingEnabled of producer will be ignored"); + } + if (queryParams.containsKey("batchingMaxMessages")) { + log.info("Since batchingEnabled is false, the param batchingMaxMessages of producer will be ignored"); + } + if (queryParams.containsKey("maxPendingMessages")) { + log.info("Since batchingEnabled is false, the param maxPendingMessages of producer will be ignored"); + } + if (queryParams.containsKey("batchingMaxPublishDelay")) { + log.info("Since batchingEnabled is false, the param batchingMaxPublishDelay of producer will be ignored"); + } + } + + private void printLogIfSettingDiscardedCompressionParams() { + if (clientSideEncrypt && queryParams.containsKey("compressionType")) { + log.info("Since clientSideEncrypt is true, the param compressionType of producer will be ignored"); + } } private static final Logger log = LoggerFactory.getLogger(ProducerHandler.class); diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketPingPongServlet.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketMultiTopicConsumerServlet.java similarity index 79% rename from pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketPingPongServlet.java rename to pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketMultiTopicConsumerServlet.java index cc2d79ee541ba..4653cea98c15d 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketPingPongServlet.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketMultiTopicConsumerServlet.java @@ -21,15 +21,15 @@ import org.eclipse.jetty.websocket.servlet.WebSocketServlet; import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; -public class WebSocketPingPongServlet extends WebSocketServlet { +public class WebSocketMultiTopicConsumerServlet extends WebSocketServlet { private static final long serialVersionUID = 1L; - public static final String SERVLET_PATH = "/ws/pingpong"; - public static final String SERVLET_PATH_V2 = "/ws/v2/pingpong"; + public static final String SERVLET_PATH = "/ws/v3/consumer"; private final transient WebSocketService service; - public WebSocketPingPongServlet(WebSocketService service) { + public WebSocketMultiTopicConsumerServlet(WebSocketService service) { + super(); this.service = service; } @@ -39,6 +39,7 @@ public void configure(WebSocketServletFactory factory) { if (service.getConfig().getWebSocketSessionIdleTimeoutMillis() > 0) { factory.getPolicy().setIdleTimeout(service.getConfig().getWebSocketSessionIdleTimeoutMillis()); } - factory.setCreator((request, response) -> new PingPongHandler()); + factory.setCreator((request, response) -> + new MultiTopicConsumerHandler(service, request.getHttpServletRequest(), response)); } -} \ No newline at end of file +} diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketService.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketService.java index 66b2a0075ec2d..7bb4df7baa533 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketService.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketService.java @@ -23,7 +23,10 @@ import java.io.Closeable; import java.io.IOException; import java.net.MalformedURLException; +import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -44,8 +47,6 @@ import org.apache.pulsar.client.internal.PropertiesUtils; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.MetadataStoreException.NotFoundException; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; @@ -73,9 +74,9 @@ public class WebSocketService implements Closeable { private Optional cryptoKeyReader = Optional.empty(); private ClusterData localCluster; - private final ConcurrentOpenHashMap> topicProducerMap; - private final ConcurrentOpenHashMap> topicConsumerMap; - private final ConcurrentOpenHashMap> topicReaderMap; + private final Map> topicProducerMap = new ConcurrentHashMap<>(); + private final Map> topicConsumerMap = new ConcurrentHashMap<>(); + private final Map> topicReaderMap = new ConcurrentHashMap<>(); private final ProxyStats proxyStats; public WebSocketService(WebSocketProxyConfiguration config) { @@ -88,17 +89,6 @@ public WebSocketService(ClusterData localCluster, ServiceConfiguration config) { .newScheduledThreadPool(config.getWebSocketNumServiceThreads(), new DefaultThreadFactory("pulsar-websocket")); this.localCluster = localCluster; - this.topicProducerMap = - ConcurrentOpenHashMap.>newBuilder() - .build(); - this.topicConsumerMap = - ConcurrentOpenHashMap.>newBuilder() - .build(); - this.topicReaderMap = - ConcurrentOpenHashMap.>newBuilder() - .build(); this.proxyStats = new ProxyStats(this); } @@ -195,7 +185,8 @@ public synchronized void setLocalCluster(ClusterData clusterData) { private PulsarClient createClientInstance(ClusterData clusterData) throws IOException { ClientBuilder clientBuilder = PulsarClient.builder() // - .memoryLimit(0, SizeUnit.BYTES) + .memoryLimit(SizeUnit.MEGA_BYTES.toBytes(config.getWebSocketPulsarClientMemoryLimitInMB()), + SizeUnit.BYTES) .statsInterval(0, TimeUnit.SECONDS) // .enableTls(config.isTlsEnabled()) // .allowTlsInsecureConnection(config.isTlsAllowInsecureConnection()) // @@ -287,11 +278,11 @@ public boolean isAuthorizationEnabled() { public boolean addProducer(ProducerHandler producer) { return topicProducerMap .computeIfAbsent(producer.getProducer().getTopic(), - topic -> ConcurrentOpenHashSet.newBuilder().build()) + topic -> ConcurrentHashMap.newKeySet()) .add(producer); } - public ConcurrentOpenHashMap> getProducers() { + public Map> getProducers() { return topicProducerMap; } @@ -305,12 +296,11 @@ public boolean removeProducer(ProducerHandler producer) { public boolean addConsumer(ConsumerHandler consumer) { return topicConsumerMap - .computeIfAbsent(consumer.getConsumer().getTopic(), topic -> - ConcurrentOpenHashSet.newBuilder().build()) + .computeIfAbsent(consumer.getConsumer().getTopic(), topic -> ConcurrentHashMap.newKeySet()) .add(consumer); } - public ConcurrentOpenHashMap> getConsumers() { + public Map> getConsumers() { return topicConsumerMap; } @@ -323,12 +313,11 @@ public boolean removeConsumer(ConsumerHandler consumer) { } public boolean addReader(ReaderHandler reader) { - return topicReaderMap.computeIfAbsent(reader.getConsumer().getTopic(), topic -> - ConcurrentOpenHashSet.newBuilder().build()) + return topicReaderMap.computeIfAbsent(reader.getConsumer().getTopic(), topic -> ConcurrentHashMap.newKeySet()) .add(reader); } - public ConcurrentOpenHashMap> getReaders() { + public Map> getReaders() { return topicReaderMap; } diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/ProducerMessage.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/ProducerMessage.java index 4831b905514f3..12cb3b20c1994 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/ProducerMessage.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/ProducerMessage.java @@ -23,6 +23,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import org.apache.pulsar.common.api.proto.CompressionType; /** * Class represent single message to be published. @@ -69,4 +70,13 @@ public class ProducerMessage { // Base64 encoded serialized schema for payload public String valueSchema; + + // Base64 encoded serialized initialization vector used when the client encrypts. + public String encryptionParam; + + // Compression type. Do not set it if compression is not performed. + public CompressionType compressionType; + + // The size of the payload before compression. Do not set it if compression is not performed. + public Integer uncompressedMessageSize; } diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/ProxyServer.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/ProxyServer.java index 7aed43d056c67..e7523252bd960 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/ProxyServer.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/ProxyServer.java @@ -24,6 +24,9 @@ import java.util.EnumSet; import java.util.List; import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import javax.servlet.DispatcherType; import javax.servlet.Servlet; @@ -34,11 +37,22 @@ import org.apache.pulsar.broker.web.JsonMapperProvider; import org.apache.pulsar.broker.web.WebExecutorThreadPool; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.util.ExecutorProvider; +import org.apache.pulsar.common.util.DefaultPulsarSslFactory; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.apache.pulsar.jetty.tls.JettySslContextFactory; +import org.eclipse.jetty.server.ConnectionFactory; import org.eclipse.jetty.server.ConnectionLimit; +import org.eclipse.jetty.server.ForwardedRequestCustomizer; import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.ProxyConnectionFactory; +import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.server.handler.ContextHandlerCollection; import org.eclipse.jetty.server.handler.DefaultHandler; import org.eclipse.jetty.server.handler.HandlerCollection; @@ -63,6 +77,8 @@ public class ProxyServer { private ServerConnector connector; private ServerConnector connectorTls; + private PulsarSslFactory sslFactory; + private ScheduledExecutorService scheduledExecutorService; public ProxyServer(WebSocketProxyConfiguration config) throws PulsarClientException, MalformedURLException, PulsarServerException { @@ -73,45 +89,57 @@ public ProxyServer(WebSocketProxyConfiguration config) if (config.getMaxHttpServerConnections() > 0) { server.addBean(new ConnectionLimit(config.getMaxHttpServerConnections(), server)); } + + HttpConfiguration httpConfig = new HttpConfiguration(); + if (config.isWebServiceTrustXForwardedFor()) { + httpConfig.addCustomizer(new ForwardedRequestCustomizer()); + } + HttpConnectionFactory httpConnectionFactory = new HttpConnectionFactory(httpConfig); + List connectors = new ArrayList<>(); if (config.getWebServicePort().isPresent()) { - connector = new ServerConnector(server); + List connectionFactories = new ArrayList<>(); + if (config.isWebServiceHaProxyProtocolEnabled()) { + connectionFactories.add(new ProxyConnectionFactory()); + } + connectionFactories.add(httpConnectionFactory); + connector = new ServerConnector(server, connectionFactories.toArray(new ConnectionFactory[0])); connector.setPort(config.getWebServicePort().get()); connectors.add(connector); } // TLS enabled connector if (config.getWebServicePortTls().isPresent()) { try { - SslContextFactory sslCtxFactory; - if (config.isTlsEnabledWithKeyStore()) { - sslCtxFactory = JettySslContextFactory.createServerSslContextWithKeystore( - config.getTlsProvider(), - config.getTlsKeyStoreType(), - config.getTlsKeyStore(), - config.getTlsKeyStorePassword(), - config.isTlsAllowInsecureConnection(), - config.getTlsTrustStoreType(), - config.getTlsTrustStore(), - config.getTlsTrustStorePassword(), - config.isTlsRequireTrustedClientCertOnConnect(), - config.getWebServiceTlsCiphers(), - config.getWebServiceTlsProtocols(), - config.getTlsCertRefreshCheckDurationSec() - ); - } else { - sslCtxFactory = JettySslContextFactory.createServerSslContext( - config.getTlsProvider(), - config.isTlsAllowInsecureConnection(), - config.getTlsTrustCertsFilePath(), - config.getTlsCertificateFilePath(), - config.getTlsKeyFilePath(), - config.isTlsRequireTrustedClientCertOnConnect(), - config.getWebServiceTlsCiphers(), - config.getWebServiceTlsProtocols(), - config.getTlsCertRefreshCheckDurationSec()); + PulsarSslConfiguration sslConfiguration = buildSslConfiguration(config); + this.sslFactory = new DefaultPulsarSslFactory(); + this.sslFactory.initialize(sslConfiguration); + this.sslFactory.createInternalSslContext(); + this.scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ExecutorProvider + .ExtendedThreadFactory("proxy-websocket-ssl-refresh")); + if (config.getTlsCertRefreshCheckDurationSec() > 0) { + this.scheduledExecutorService.scheduleWithFixedDelay(this::refreshSslContext, + config.getTlsCertRefreshCheckDurationSec(), + config.getTlsCertRefreshCheckDurationSec(), + TimeUnit.SECONDS); } - connectorTls = new ServerConnector(server, sslCtxFactory); + SslContextFactory sslCtxFactory = + JettySslContextFactory.createSslContextFactory(config.getTlsProvider(), + sslFactory, config.isTlsRequireTrustedClientCertOnConnect(), + config.getWebServiceTlsCiphers(), config.getWebServiceTlsProtocols()); + List connectionFactories = new ArrayList<>(); + if (config.isWebServiceHaProxyProtocolEnabled()) { + connectionFactories.add(new ProxyConnectionFactory()); + } + connectionFactories.add(new SslConnectionFactory(sslCtxFactory, httpConnectionFactory.getProtocol())); + connectionFactories.add(httpConnectionFactory); + // org.eclipse.jetty.server.AbstractConnectionFactory.getFactories contains similar logic + // this is needed for TLS authentication + if (httpConfig.getCustomizer(SecureRequestCustomizer.class) == null) { + httpConfig.addCustomizer(new SecureRequestCustomizer()); + } + connectorTls = new ServerConnector(server, connectionFactories.toArray(new ConnectionFactory[0])); connectorTls.setPort(config.getWebServicePortTls().get()); connectors.add(connectorTls); } catch (Exception e) { @@ -169,7 +197,10 @@ public void start() throws PulsarServerException { .map(ServerConnector.class::cast).map(ServerConnector::getPort).map(Object::toString) .collect(Collectors.joining(","))); RequestLogHandler requestLogHandler = new RequestLogHandler(); - requestLogHandler.setRequestLog(JettyRequestLogFactory.createRequestLogger()); + boolean showDetailedAddresses = conf.getWebServiceLogDetailedAddresses() != null + ? conf.getWebServiceLogDetailedAddresses() : + (conf.isWebServiceHaProxyProtocolEnabled() || conf.isWebServiceTrustXForwardedFor()); + requestLogHandler.setRequestLog(JettyRequestLogFactory.createRequestLogger(showDetailedAddresses, server)); handlers.add(0, new ContextHandlerCollection()); handlers.add(requestLogHandler); @@ -190,6 +221,9 @@ public void start() throws PulsarServerException { public void stop() throws Exception { server.stop(); executorService.stop(); + if (scheduledExecutorService != null) { + scheduledExecutorService.shutdownNow(); + } } public Optional getListenPortHTTP() { @@ -208,5 +242,34 @@ public Optional getListenPortHTTPS() { } } + protected PulsarSslConfiguration buildSslConfiguration(WebSocketProxyConfiguration config) { + return PulsarSslConfiguration.builder() + .tlsKeyStoreType(config.getTlsKeyStoreType()) + .tlsKeyStorePath(config.getTlsKeyStore()) + .tlsKeyStorePassword(config.getTlsKeyStorePassword()) + .tlsTrustStoreType(config.getTlsTrustStoreType()) + .tlsTrustStorePath(config.getTlsTrustStore()) + .tlsTrustStorePassword(config.getTlsTrustStorePassword()) + .tlsCiphers(config.getWebServiceTlsCiphers()) + .tlsProtocols(config.getWebServiceTlsProtocols()) + .tlsTrustCertsFilePath(config.getTlsTrustCertsFilePath()) + .tlsCertificateFilePath(config.getTlsCertificateFilePath()) + .tlsKeyFilePath(config.getTlsKeyFilePath()) + .allowInsecureConnection(config.isTlsAllowInsecureConnection()) + .requireTrustedClientCertOnConnect(config.isTlsRequireTrustedClientCertOnConnect()) + .tlsEnabledWithKeystore(config.isTlsEnabledWithKeyStore()) + .serverMode(true) + .isHttps(true) + .build(); + } + + protected void refreshSslContext() { + try { + this.sslFactory.update(); + } catch (Exception e) { + log.error("Failed to refresh SSL context", e); + } + } + private static final Logger log = LoggerFactory.getLogger(ProxyServer.class); } diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WSSDummyMessageCryptoImpl.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WSSDummyMessageCryptoImpl.java new file mode 100644 index 0000000000000..43f7a368bbb56 --- /dev/null +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WSSDummyMessageCryptoImpl.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.websocket.service; + +import java.nio.ByteBuffer; +import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Supplier; +import org.apache.pulsar.client.api.CryptoKeyReader; +import org.apache.pulsar.client.api.MessageCrypto; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.common.api.proto.MessageMetadata; + +/*** + * This class is used in scenarios where the payload of the message has been encrypted and the producer does not need + * to encrypt it again. + * It discards payload encryption and only relies {@link #metadataModifierForSend} to set the encryption info into the + * message metadata. + */ +public class WSSDummyMessageCryptoImpl implements MessageCrypto { + + public static final WSSDummyMessageCryptoImpl INSTANCE_FOR_CONSUMER = + new WSSDummyMessageCryptoImpl(msgMetadata -> {}); + + private final Consumer metadataModifierForSend; + + public WSSDummyMessageCryptoImpl(Consumer metadataModifierForSend) { + this.metadataModifierForSend = metadataModifierForSend; + } + + @Override + public void addPublicKeyCipher(Set keyNames, CryptoKeyReader keyReader) + throws PulsarClientException.CryptoException {} + + @Override + public boolean removeKeyCipher(String keyName) { + return true; + } + + @Override + public int getMaxOutputSize(int inputLen) { + return inputLen; + } + + @Override + public boolean decrypt(Supplier messageMetadataSupplier, ByteBuffer payload, ByteBuffer outBuffer, + CryptoKeyReader keyReader) { + outBuffer.put(payload); + outBuffer.flip(); + return true; + } + + @Override + public synchronized void encrypt(Set encKeys, CryptoKeyReader keyReader, + Supplier messageMetadataSupplier, + ByteBuffer payload, ByteBuffer outBuffer) throws PulsarClientException { + outBuffer.put(payload); + outBuffer.flip(); + metadataModifierForSend.accept(messageMetadataSupplier.get()); + } +} diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketProxyConfiguration.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketProxyConfiguration.java index 7acfd4a64ad35..31a1adc291553 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketProxyConfiguration.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketProxyConfiguration.java @@ -96,6 +96,20 @@ public class WebSocketProxyConfiguration implements PulsarConfiguration { @FieldContext(doc = "Hostname or IP address the service binds on, default is 0.0.0.0.") private String bindAddress = "0.0.0.0"; + @FieldContext(doc = "Enable or disable the use of HA proxy protocol for resolving the client IP for http/https " + + "requests. Default is false.") + private boolean webServiceHaProxyProtocolEnabled = false; + + @FieldContext(doc = "Trust X-Forwarded-For header for resolving the client IP for http/https requests.\n" + + "Default is false.") + private boolean webServiceTrustXForwardedFor = false; + + @FieldContext(doc = + "Add detailed client/remote and server/local addresses and ports to http/https request logging.\n" + + "Defaults to true when either webServiceHaProxyProtocolEnabled or webServiceTrustXForwardedFor " + + "is enabled.") + private Boolean webServiceLogDetailedAddresses; + @FieldContext(doc = "Maximum size of a text message during parsing in WebSocket proxy") private int webSocketMaxTextFrameSize = 1024 * 1024; // --- Authentication --- @@ -162,6 +176,9 @@ public class WebSocketProxyConfiguration implements PulsarConfiguration { @FieldContext(doc = "Number of connections per broker in Pulsar client used in WebSocket proxy") private int webSocketConnectionsPerBroker = Runtime.getRuntime().availableProcessors(); + @FieldContext(doc = "Memory limit in MBs for direct memory in Pulsar Client used in WebSocket proxy") + private int webSocketPulsarClientMemoryLimitInMB = 0; + @FieldContext(doc = "Timeout of idling WebSocket session (in milliseconds)") private int webSocketSessionIdleTimeoutMillis = 300000; diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketServiceStarter.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketServiceStarter.java index fbcecc0642e34..0a445aebe3a00 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketServiceStarter.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketServiceStarter.java @@ -22,14 +22,12 @@ import static org.apache.pulsar.websocket.admin.WebSocketWebResource.ADMIN_PATH_V1; import static org.apache.pulsar.websocket.admin.WebSocketWebResource.ADMIN_PATH_V2; import static org.apache.pulsar.websocket.admin.WebSocketWebResource.ATTRIBUTE_PROXY_SERVICE_NAME; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.common.configuration.VipStatus; -import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.ShutdownUtil; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.apache.pulsar.websocket.WebSocketConsumerServlet; -import org.apache.pulsar.websocket.WebSocketPingPongServlet; +import org.apache.pulsar.websocket.WebSocketMultiTopicConsumerServlet; import org.apache.pulsar.websocket.WebSocketProducerServlet; import org.apache.pulsar.websocket.WebSocketReaderServlet; import org.apache.pulsar.websocket.WebSocketService; @@ -37,37 +35,42 @@ import org.apache.pulsar.websocket.admin.v2.WebSocketProxyStatsV2; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; +import picocli.CommandLine.ScopeType; public class WebSocketServiceStarter { + @Command(name = "websocket", showDefaultValues = true, scope = ScopeType.INHERIT) private static class Arguments { - @Parameter(description = "config file") + @Parameters(description = "config file", arity = "0..1") private String configFile = ""; - @Parameter(names = {"-h", "--help"}, description = "Show this help message") + @Option(names = {"-h", "--help"}, usageHelp = true, description = "Show this help message") private boolean help = false; - @Parameter(names = {"-g", "--generate-docs"}, description = "Generate docs") + @Option(names = {"-g", "--generate-docs"}, description = "Generate docs") private boolean generateDocs = false; } public static void main(String[] args) throws Exception { Arguments arguments = new Arguments(); - JCommander jcommander = new JCommander(); + CommandLine commander = new CommandLine(arguments); try { - jcommander.addObject(arguments); - jcommander.parse(args); + commander.parseArgs(args); if (arguments.help) { - jcommander.usage(); + commander.usage(commander.getOut()); return; } if (arguments.generateDocs && arguments.configFile != null) { CmdGenerateDocs cmd = new CmdGenerateDocs("pulsar"); - cmd.addCommand("websocket", arguments); + cmd.addCommand("websocket", commander); cmd.run(null); return; } } catch (Exception e) { - jcommander.usage(); + commander.getErr().println(e); return; } @@ -75,9 +78,7 @@ public static void main(String[] args) throws Exception { try { // load config file and start proxy service String configFile = args[0]; - log.info("Loading configuration from {}", configFile); - WebSocketProxyConfiguration config = PulsarConfigurationLoader.create(configFile, - WebSocketProxyConfiguration.class); + WebSocketProxyConfiguration config = loadConfig(configFile); ProxyServer proxyServer = new ProxyServer(config); WebSocketService service = new WebSocketService(config); start(proxyServer, service); @@ -91,16 +92,15 @@ public static void start(ProxyServer proxyServer, WebSocketService service) thro proxyServer.addWebSocketServlet(WebSocketProducerServlet.SERVLET_PATH, new WebSocketProducerServlet(service)); proxyServer.addWebSocketServlet(WebSocketConsumerServlet.SERVLET_PATH, new WebSocketConsumerServlet(service)); proxyServer.addWebSocketServlet(WebSocketReaderServlet.SERVLET_PATH, new WebSocketReaderServlet(service)); - proxyServer.addWebSocketServlet(WebSocketPingPongServlet.SERVLET_PATH, new WebSocketPingPongServlet(service)); proxyServer.addWebSocketServlet(WebSocketProducerServlet.SERVLET_PATH_V2, new WebSocketProducerServlet(service)); proxyServer.addWebSocketServlet(WebSocketConsumerServlet.SERVLET_PATH_V2, new WebSocketConsumerServlet(service)); + proxyServer.addWebSocketServlet(WebSocketMultiTopicConsumerServlet.SERVLET_PATH, + new WebSocketMultiTopicConsumerServlet(service)); proxyServer.addWebSocketServlet(WebSocketReaderServlet.SERVLET_PATH_V2, new WebSocketReaderServlet(service)); - proxyServer.addWebSocketServlet(WebSocketPingPongServlet.SERVLET_PATH_V2, - new WebSocketPingPongServlet(service)); proxyServer.addRestResource(ADMIN_PATH_V1, ATTRIBUTE_PROXY_SERVICE_NAME, service, WebSocketProxyStatsV1.class); proxyServer.addRestResource(ADMIN_PATH_V2, ATTRIBUTE_PROXY_SERVICE_NAME, service, WebSocketProxyStatsV2.class); @@ -110,6 +110,14 @@ public static void start(ProxyServer proxyServer, WebSocketService service) thro service.start(); } + private static WebSocketProxyConfiguration loadConfig(String configFile) throws Exception { + log.info("Loading configuration from {}", configFile); + WebSocketProxyConfiguration config = PulsarConfigurationLoader.create(configFile, + WebSocketProxyConfiguration.class); + PulsarConfigurationLoader.isComplete(config); + return config; + } + private static final Logger log = LoggerFactory.getLogger(WebSocketServiceStarter.class); } diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/stats/ProxyStats.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/stats/ProxyStats.java index eb1566ef7d412..4660340e9cc54 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/stats/ProxyStats.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/stats/ProxyStats.java @@ -24,11 +24,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.stats.JvmMetrics; import org.apache.pulsar.common.stats.Metrics; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.websocket.WebSocketService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,7 +41,7 @@ public class ProxyStats { private final WebSocketService service; private final JvmMetrics jvmMetrics; - private ConcurrentOpenHashMap topicStats; + private final Map topicStats = new ConcurrentHashMap<>(); private List metricsCollection; private List tempMetricsCollection; @@ -50,9 +50,6 @@ public ProxyStats(WebSocketService service) { this.service = service; this.jvmMetrics = JvmMetrics.create( service.getExecutor(), "prx", service.getConfig().getJvmGCMetricsLoggerClassName()); - this.topicStats = - ConcurrentOpenHashMap.newBuilder() - .build(); this.metricsCollection = new ArrayList<>(); this.tempMetricsCollection = new ArrayList<>(); // schedule stat generation task every 1 minute diff --git a/pulsar-websocket/src/main/resources/findbugsExclude.xml b/pulsar-websocket/src/main/resources/findbugsExclude.xml index b7f6b0bf31d51..c2b0d7dac0d3b 100644 --- a/pulsar-websocket/src/main/resources/findbugsExclude.xml +++ b/pulsar-websocket/src/main/resources/findbugsExclude.xml @@ -159,6 +159,11 @@ + + + + + @@ -199,4 +204,9 @@ + + + + + \ No newline at end of file diff --git a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/AbstractWebSocketHandlerTest.java b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/AbstractWebSocketHandlerTest.java index eec2c3d1baa6b..d21e1176f571d 100644 --- a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/AbstractWebSocketHandlerTest.java +++ b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/AbstractWebSocketHandlerTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.websocket; +import static com.google.common.base.Preconditions.checkArgument; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; @@ -37,6 +38,8 @@ import java.util.stream.Collectors; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import lombok.Cleanup; +import com.google.common.base.Splitter; import lombok.Getter; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.client.api.CompressionType; @@ -51,6 +54,7 @@ import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.websocket.service.WebSocketProxyConfiguration; import org.eclipse.jetty.http.HttpStatus; import org.eclipse.jetty.websocket.api.RemoteEndpoint; @@ -129,7 +133,7 @@ public void topicNameUrlEncodingTest() throws Exception { webSocketHandler = new WebSocketHandlerImpl(null, httpServletRequest, null); topicName = webSocketHandler.getTopic(); assertEquals(topicName.toString(), "persistent://my-property/my-ns/" + consumerV2Topic); - String sub = ConsumerHandler.extractSubscription(httpServletRequest); + String sub = extractSubscription(httpServletRequest); assertEquals(sub, consumerV2Sub); when(httpServletRequest.getRequestURI()).thenReturn(readerV2 @@ -139,6 +143,27 @@ public void topicNameUrlEncodingTest() throws Exception { assertEquals(topicName.toString(), "persistent://my-property/my-ns/" + readerV2Topic); } + public String extractSubscription(HttpServletRequest request) { + String uri = request.getRequestURI(); + List parts = Splitter.on("/").splitToList(uri); + + // v1 Format must be like : + // /ws/consumer/persistent/my-property/my-cluster/my-ns/my-topic/my-subscription + + // v2 Format must be like : + // /ws/v2/consumer/persistent/my-property/my-ns/my-topic/my-subscription + checkArgument(parts.size() == 9, "Invalid topic name format"); + checkArgument(parts.get(1).equals("ws")); + + final boolean isV2Format = parts.get(2).equals("v2"); + final int domainIndex = isV2Format ? 4 : 3; + checkArgument(parts.get(domainIndex).equals("persistent") + || parts.get(domainIndex).equals("non-persistent")); + checkArgument(parts.get(8).length() > 0, "Empty subscription name"); + + return Codec.decode(parts.get(8)); + } + @Test public void parseTopicNameTest() { String producerV1 = "/ws/producer/persistent/my-property/my-cluster/my-ns/my-topic"; @@ -388,10 +413,11 @@ public void consumerBuilderTest() throws IOException { } @Test - public void testPingFuture() { + public void testPingFuture() throws IOException { WebSocketProxyConfiguration webSocketProxyConfiguration = new WebSocketProxyConfiguration(); webSocketProxyConfiguration.setWebSocketPingDurationSeconds(5); + @Cleanup WebSocketService webSocketService = new WebSocketService(webSocketProxyConfiguration); HttpServletRequest httpServletRequest = mock(HttpServletRequest.class); diff --git a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/PingPongHandlerTest.java b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/PingPongSupportTest.java similarity index 63% rename from pulsar-websocket/src/test/java/org/apache/pulsar/websocket/PingPongHandlerTest.java rename to pulsar-websocket/src/test/java/org/apache/pulsar/websocket/PingPongSupportTest.java index 662009f1aab1a..1ce858ec4a19e 100644 --- a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/PingPongHandlerTest.java +++ b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/PingPongSupportTest.java @@ -18,13 +18,17 @@ */ package org.apache.pulsar.websocket; +import java.io.IOException; import java.net.URI; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.Future; +import javax.servlet.http.HttpServletRequest; +import lombok.Cleanup; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.web.WebExecutorThreadPool; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.server.Server; @@ -40,18 +44,26 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.Assert.assertTrue; +import org.eclipse.jetty.websocket.servlet.ServletUpgradeResponse; +import org.eclipse.jetty.websocket.servlet.WebSocketServlet; +import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -public class PingPongHandlerTest { +/** + * Test to ensure {@link AbstractWebSocketHandler} has ping/pong support + */ +public class PingPongSupportTest { - private static Server server; + private Server server; - private static final WebExecutorThreadPool executor = new WebExecutorThreadPool(6, "pulsar-websocket-web-test"); + private WebExecutorThreadPool executor; @BeforeClass - public static void setup() throws Exception { + public void setup() throws Exception { + executor = new WebExecutorThreadPool(6, "pulsar-websocket-web-test"); server = new Server(executor); List connectors = new ArrayList<>(); ServerConnector connector = new ServerConnector(server); @@ -67,9 +79,9 @@ public static void setup() throws Exception { when(config.getWebSocketMaxTextFrameSize()).thenReturn(1048576); when(config.getWebSocketSessionIdleTimeoutMillis()).thenReturn(300000); - ServletHolder servletHolder = new ServletHolder("ws-events", new WebSocketPingPongServlet(service)); + ServletHolder servletHolder = new ServletHolder("ws-events", new GenericWebSocketServlet(service)); ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS); - context.setContextPath(WebSocketPingPongServlet.SERVLET_PATH); + context.setContextPath("/ws"); context.addServlet(servletHolder, "/*"); server.setHandler(context); try { @@ -80,25 +92,68 @@ public static void setup() throws Exception { } @AfterClass(alwaysRun = true) - public static void tearDown() throws Exception { + public void tearDown() throws Exception { if (server != null) { server.stop(); } executor.stop(); } - @Test - public void testPingPong() throws Exception { + /** + * We test these different endpoints because they are parsed in the AbstractWebSocketHandler. Technically, we are + * not testing these implementations, but the ping/pong support is guaranteed as part of the framework. + */ + @DataProvider(name = "endpoint") + public static Object[][] cacheEnable() { + return new Object[][] { { "producer" }, { "consumer" }, { "reader" } }; + } + + @Test(dataProvider = "endpoint") + public void testPingPong(String endpoint) throws Exception { + @Cleanup("stop") HttpClient httpClient = new HttpClient(); WebSocketClient webSocketClient = new WebSocketClient(httpClient); webSocketClient.start(); MyWebSocket myWebSocket = new MyWebSocket(); - String webSocketUri = "ws://localhost:8080/ws/pingpong"; + String webSocketUri = "ws://localhost:8080/ws/v2/" + endpoint + "/persistent/my-property/my-ns/my-topic"; Future sessionFuture = webSocketClient.connect(myWebSocket, URI.create(webSocketUri)); sessionFuture.get().getRemote().sendPing(ByteBuffer.wrap("test".getBytes())); assertTrue(myWebSocket.getResponse().contains("test")); } + public static class GenericWebSocketHandler extends AbstractWebSocketHandler { + + public GenericWebSocketHandler(WebSocketService service, HttpServletRequest request, ServletUpgradeResponse response) { + super(service, request, response); + } + + @Override + protected Boolean isAuthorized(String authRole, AuthenticationDataSource authenticationData) throws Exception { + return true; + } + + @Override + public void close() throws IOException { + + } + } + + public static class GenericWebSocketServlet extends WebSocketServlet { + + private static final long serialVersionUID = 1L; + private final WebSocketService service; + + public GenericWebSocketServlet(WebSocketService service) { + this.service = service; + } + + @Override + public void configure(WebSocketServletFactory factory) { + factory.setCreator((request, response) -> + new GenericWebSocketHandler(service, request.getHttpServletRequest(), response)); + } + } + @WebSocket public static class MyWebSocket extends WebSocketAdapter implements WebSocketPingPongListener { diff --git a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ProducerHandlerTest.java b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ProducerHandlerTest.java index 5f773a8e2e1fb..d09b5941f2429 100644 --- a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ProducerHandlerTest.java +++ b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ProducerHandlerTest.java @@ -23,6 +23,7 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.TypedMessageBuilder; import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.client.impl.TypedMessageBuilderImpl; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.websocket.data.ProducerMessage; import org.eclipse.jetty.websocket.servlet.ServletUpgradeResponse; @@ -53,7 +54,7 @@ public void testProduceMessageAttributes() throws IOException { PulsarClient pulsarClient = mock(PulsarClient.class); ProducerBuilder producerBuilder = mock(ProducerBuilder.class); Producer producer = mock(Producer.class); - TypedMessageBuilder messageBuilder = mock(TypedMessageBuilder.class); + TypedMessageBuilder messageBuilder = mock(TypedMessageBuilderImpl.class); ProducerMessage produceRequest = new ProducerMessage(); produceRequest.setDeliverAfterMs(11111); diff --git a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ReaderHandlerTest.java b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ReaderHandlerTest.java index 002f15609cdb7..9ad9368d27793 100644 --- a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ReaderHandlerTest.java +++ b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ReaderHandlerTest.java @@ -18,12 +18,14 @@ */ package org.apache.pulsar.websocket; +import java.util.List; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.ReaderBuilder; +import org.apache.pulsar.client.api.TopicMessageId; import org.apache.pulsar.client.impl.ConsumerImpl; import org.apache.pulsar.client.impl.MultiTopicsConsumerImpl; import org.apache.pulsar.client.impl.MultiTopicsReaderImpl; @@ -214,5 +216,15 @@ public CompletableFuture seekAsync(long timestamp) { public void close() throws IOException { } + + @Override + public List getLastMessageIds() throws PulsarClientException { + return null; + } + + @Override + public CompletableFuture> getLastMessageIdsAsync() { + return null; + } } } diff --git a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/WebSocketHttpServletRequestWrapperTest.java b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/WebSocketHttpServletRequestWrapperTest.java index b11529bf2f176..48a822272b8bd 100644 --- a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/WebSocketHttpServletRequestWrapperTest.java +++ b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/WebSocketHttpServletRequestWrapperTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.websocket; +import lombok.Cleanup; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.websocket.service.WebSocketProxyConfiguration; import org.eclipse.jetty.websocket.servlet.UpgradeHttpServletRequest; @@ -70,6 +71,7 @@ public void mockRequestTest() throws Exception { WebSocketProxyConfiguration.class); String publicKeyPath = "file://" + this.getClass().getClassLoader().getResource("my-public.key").getFile(); config.getProperties().setProperty("tokenPublicKey", publicKeyPath); + @Cleanup WebSocketService service = new WebSocketService(config); service.start(); diff --git a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/WebSocketProxyConfigurationTest.java b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/WebSocketProxyConfigurationTest.java index 92b4238cddd7d..a5ec63045bad3 100644 --- a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/WebSocketProxyConfigurationTest.java +++ b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/WebSocketProxyConfigurationTest.java @@ -34,6 +34,7 @@ import java.io.PrintWriter; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; public class WebSocketProxyConfigurationTest { @@ -64,6 +65,7 @@ public void testBackwardCompatibility() throws IOException { printWriter.println("metadataStoreCacheExpirySeconds=500"); printWriter.println("zooKeeperSessionTimeoutMillis=-1"); printWriter.println("zooKeeperCacheExpirySeconds=-1"); + printWriter.println("cryptoKeyReaderFactoryClassName="); } testConfigFile.deleteOnExit(); stream = new FileInputStream(testConfigFile); @@ -71,6 +73,7 @@ public void testBackwardCompatibility() throws IOException { stream.close(); assertEquals(serviceConfig.getMetadataStoreSessionTimeoutMillis(), 60); assertEquals(serviceConfig.getMetadataStoreCacheExpirySeconds(), 500); + assertNull(serviceConfig.getCryptoKeyReaderFactoryClassName()); testConfigFile = new File("tmp." + System.currentTimeMillis() + ".properties"); if (testConfigFile.exists()) { @@ -81,6 +84,7 @@ public void testBackwardCompatibility() throws IOException { printWriter.println("metadataStoreCacheExpirySeconds=30"); printWriter.println("zooKeeperSessionTimeoutMillis=100"); printWriter.println("zooKeeperCacheExpirySeconds=300"); + printWriter.println("cryptoKeyReaderFactoryClassName=A.class"); } testConfigFile.deleteOnExit(); stream = new FileInputStream(testConfigFile); @@ -88,6 +92,7 @@ public void testBackwardCompatibility() throws IOException { stream.close(); assertEquals(serviceConfig.getMetadataStoreSessionTimeoutMillis(), 100); assertEquals(serviceConfig.getMetadataStoreCacheExpirySeconds(), 300); + assertEquals(serviceConfig.getCryptoKeyReaderFactoryClassName(), "A.class"); } @Test diff --git a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/service/WebSocketServiceStarterTest.java b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/service/WebSocketServiceStarterTest.java index c898a07e22800..f9a190cf8662d 100644 --- a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/service/WebSocketServiceStarterTest.java +++ b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/service/WebSocketServiceStarterTest.java @@ -19,12 +19,12 @@ package org.apache.pulsar.websocket.service; import static org.testng.Assert.assertTrue; -import com.beust.jcommander.Parameter; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.lang.reflect.Field; import java.util.Arrays; import org.testng.annotations.Test; +import picocli.CommandLine.Option; public class WebSocketServiceStarterTest { @Test @@ -43,9 +43,9 @@ public void testMainGenerateDocs() throws Exception { Field[] fields = argumentsClass.getDeclaredFields(); for (Field field : fields) { - boolean fieldHasAnno = field.isAnnotationPresent(Parameter.class); + boolean fieldHasAnno = field.isAnnotationPresent(Option.class); if (fieldHasAnno) { - Parameter fieldAnno = field.getAnnotation(Parameter.class); + Option fieldAnno = field.getAnnotation(Option.class); String[] names = fieldAnno.names(); if (names.length == 0) { continue; diff --git a/src/assembly-source-package.xml b/src/assembly-source-package.xml deleted file mode 100644 index 00f00bfe3a5c7..0000000000000 --- a/src/assembly-source-package.xml +++ /dev/null @@ -1,110 +0,0 @@ - - - - - source-release - - tar.gz - - - - - . - - true - - - src/*.sh - src/*.py - docker/pulsar/scripts/*.sh - docker/pulsar/scripts/*.py - - - data/** - logs/** - - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/).*${project.build.directory}.*] - - - - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?maven-eclipse\.xml] - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.project] - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.classpath] - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?[^/]*\.iws] - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.idea(/.*)?] - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?out(/.*)?] - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?[^/]*\.ipr] - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?[^/]*\.iml] - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.settings(/.*)?] - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.externalToolBuilders(/.*)?] - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.deployables(/.*)?] - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.wtpmodules(/.*)?] - - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?cobertura\.ser] - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?pom\.xml\.versionsBackup] - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?dependency-reduced-pom\.xml] - - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?pom\.xml\.releaseBackup] - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?release\.properties] - - - - - ${project.build.directory}/maven-shared-archive-resources/META-INF - - - - src - /src - - *.sh - *.py - - 0755 - - - docker/pulsar/scripts - /docker/pulsar/scripts - - *.sh - *.py - - 0755 - - - diff --git a/src/check-binary-license.sh b/src/check-binary-license.sh index 3a6d266345f30..6aec8b7cf1bd9 100755 --- a/src/check-binary-license.sh +++ b/src/check-binary-license.sh @@ -27,21 +27,13 @@ # all error fatal set -e -# skip checks for Presto licenses if 1. argument is "--no-presto"/"no-pulsar-sql" -# this is to allow building the server distribution without Pulsar SQL -NO_PRESTO=0 -if [[ "$1" == "--no-presto" || "$1" == "--no-pulsar-sql" ]]; then - NO_PRESTO=1 - shift -fi - TARBALL="$1" if [ -z $TARBALL ]; then echo "Usage: $0 " exit 1 fi -JARS=$(tar -tf $TARBALL | grep '\.jar' | grep -v 'trino/' | grep -v '/examples/' | grep -v '/instances/' | grep -v pulsar-client | grep -v pulsar-common | grep -v pulsar-package | grep -v pulsar-websocket | grep -v bouncy-castle-bc | sed 's!.*/!!' | sort) +JARS=$(tar -tf $TARBALL | grep '\.jar' | grep -v '/examples/' | grep -v '/instances/' | grep -v pulsar-client | grep -v pulsar-cli-utils | grep -v pulsar-common | grep -v pulsar-package | grep -v pulsar-websocket | grep -v bouncy-castle-bc | sed 's!.*/!!' | sort) LICENSEPATH=$(tar -tf $TARBALL | awk '/^[^\/]*\/LICENSE/') LICENSE=$(tar -O -xf $TARBALL "$LICENSEPATH") @@ -94,39 +86,6 @@ for J in $NOTICEJARS; do fi done -if [ "$NO_PRESTO" -ne 1 ]; then - # check pulsar sql jars - JARS=$(tar -tf $TARBALL | grep '\.jar' | grep 'trino/' | grep -v pulsar-client | grep -v bouncy-castle-bc | grep -v pulsar-metadata | grep -v 'managed-ledger' | grep -v 'pulsar-client-admin' | grep -v 'pulsar-client-api' | grep -v 'pulsar-functions-api' | grep -v 'pulsar-presto-connector-original' | grep -v 'pulsar-presto-distribution' | grep -v 'pulsar-common' | grep -v 'pulsar-functions-proto' | grep -v 'pulsar-functions-utils' | grep -v 'pulsar-io-core' | grep -v 'pulsar-transaction-common' | grep -v 'pulsar-package-core' | sed 's!.*/!!' | sort) - if [ -n "$JARS" ]; then - LICENSEPATH=$(tar -tf $TARBALL | awk '/^[^\/]*\/trino\/LICENSE/') - LICENSE=$(tar -O -xf $TARBALL "$LICENSEPATH") - LICENSEJARS=$(echo "$LICENSE" | sed -nE 's!.* (.*\.jar).*!\1!gp') - - - for J in $JARS; do - echo $J | grep -q "org.apache.pulsar" - if [ $? == 0 ]; then - continue - fi - - echo "$LICENSE" | grep -q $J - if [ $? != 0 ]; then - echo $J unaccounted for in trino/LICENSE - EXIT=1 - fi - done - - # Check all jars mentioned in LICENSE are bundled - for J in $LICENSEJARS; do - echo "$JARS" | grep -q $J - if [ $? != 0 ]; then - echo $J mentioned in trino/LICENSE, but not bundled - EXIT=2 - fi - done - fi -fi - if [ $EXIT != 0 ]; then echo echo It looks like there are issues with the LICENSE/NOTICE. diff --git a/src/gen-pulsar-bom.sh b/src/gen-pulsar-bom.sh new file mode 100755 index 0000000000000..e9fb455d5ad3d --- /dev/null +++ b/src/gen-pulsar-bom.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Script to generate a BOM using the following approach: +# +# - do a local publish to get list of modules for BOM +# - for each published module add dependency entry +# - replace the current section with +# entries gathered in previous step + +set -e + +# Determine top level project directory +SRC_DIR=$(dirname "$0") +ROOT_DIR=`cd ${SRC_DIR}/..; pwd` +LOCAL_DEPLOY_DIR=${ROOT_DIR}/target/staging-deploy +PULSAR_BOM_DIR=${ROOT_DIR}/pulsar-bom + +pushd ${ROOT_DIR} > /dev/null +echo "Performing local publish to determine modules for BOM." +rm -rf ${LOCAL_DEPLOY_DIR} +./mvnw deploy -DaltDeploymentRepository=local::default::file:${LOCAL_DEPLOY_DIR} -DskipTests +./mvnw deploy -DaltDeploymentRepository=local::default::file:${LOCAL_DEPLOY_DIR} -DskipTests -f tests/pom.xml -pl org.apache.pulsar.tests:tests-parent,org.apache.pulsar.tests:integration +echo "$(ls ${LOCAL_DEPLOY_DIR}/org/apache/pulsar | wc -l) modules locally published to ${LOCAL_DEPLOY_DIR}." +popd > /dev/null + +DEPENDENCY_MGMT_PRE=$(cat <<-END + + +END +) +DEPENDENCY_BLOCK=$(cat <<-END + + org.apache.pulsar + @ARTIFACT_ID@ + \${project.version} + +END +) +DEPENDENCY_MGMT_POST=$(cat <<-END + + + +END +) +ALL_DEPS="" +NEWLINE=$'\n' + +pushd ${LOCAL_DEPLOY_DIR}/org/apache/pulsar/ > /dev/null +echo "Traversing locally published modules." +for f in */ +do + ARTIFACT_ID="${f%/}" + DEPENDENCY=$(echo "${DEPENDENCY_BLOCK/@ARTIFACT_ID@/$ARTIFACT_ID}") + if [ "${ARTIFACT_ID}" = "pulsar-bom" ]; then + continue + elif [ -z "$ALL_DEPS" ]; then + ALL_DEPS="$DEPENDENCY" + else + ALL_DEPS="$ALL_DEPS$NEWLINE$DEPENDENCY" + fi +done +popd > /dev/null + +POM_XML=$(<${PULSAR_BOM_DIR}/pom.xml) +POM_XML=$(echo "${POM_XML%%*}") +echo "$POM_XML$DEPENDENCY_MGMT_PRE$NEWLINE$ALL_DEPS$NEWLINE$DEPENDENCY_MGMT_POST" > ${PULSAR_BOM_DIR}/pom.xml + +echo "Created BOM ${PULSAR_BOM_DIR}/pom.xml." +echo "You must manually inspect changes and submit a PR with the changes." diff --git a/src/owasp-dependency-check-false-positives.xml b/src/owasp-dependency-check-false-positives.xml index 345be8f4d2c06..5abcae4efd532 100644 --- a/src/owasp-dependency-check-false-positives.xml +++ b/src/owasp-dependency-check-false-positives.xml @@ -201,4 +201,12 @@ flat_project is not used at all. cpe:/a:flat_project:flat + + + + ^pkg:maven/org\.eclipse\.jetty/jetty\-servlets@.*$ + CVE-2023-36479 + \ No newline at end of file diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml index 5a595af2c0a79..1ce7392a4898d 100644 --- a/src/owasp-dependency-check-suppressions.xml +++ b/src/owasp-dependency-check-suppressions.xml @@ -26,11 +26,6 @@ .*grpc-netty-shaded.* cpe:/a:netty:netty - - Suppress all pulsar-presto-distribution vulnerabilities - .*pulsar-presto-distribution-.* - .* - Suppress libthrift-0.12.0.jar vulnerabilities org.apache.thrift:libthrift:0.12.0 @@ -109,7 +104,15 @@ b87878db57d5cfc2ca7d3972cc8f7486bf02fbca CVE-2020-8908 - + + + b87878db57d5cfc2ca7d3972cc8f7486bf02fbca + CVE-2023-2976 + fa9a1ccda7d78edb51a3a33d3493566092786a30 CVE-2021-25263 + + + d3b929509399a698915b24ff47db781d0c526760 + CVE-2023-2976 + CVE-2021-42550 - - - 861af62ae22a71d30f401a80049397fe7ff44423 - CVE-2020-15113 + Ignore etdc CVEs in jetcd + .*jetcd.* + cpe:/a:etcd:etcd - - - 663f2ccc0ec7797954c333fa75feeb7d559948b0 - CVE-2020-15113 + Ignore etdc CVEs in jetcd + .*jetcd.* + cpe:/a:redhat:etcd - - - abd0ffcd4e66046057c3bfb34affc0de870a038b - CVE-2020-15113 - - - - - a2e99802fec5586daca0c4daae975fe601a057a8 - CVE-2017-8359 - - - - a2e99802fec5586daca0c4daae975fe601a057a8 - CVE-2020-15113 - - - - a2e99802fec5586daca0c4daae975fe601a057a8 - CVE-2020-7768 - - - - a2e99802fec5586daca0c4daae975fe601a057a8 - CVE-2017-7861 - - - - a2e99802fec5586daca0c4daae975fe601a057a8 - CVE-2017-9431 - - - - a2e99802fec5586daca0c4daae975fe601a057a8 - CVE-2017-7860 + Ignore grpc CVEs in jetcd + .*jetcd-grpc.* + cpe:/a:grpc:grpc @@ -485,5 +445,23 @@ ]]> CVE-2020-8908 - + + + CVE-2023-35116 + + + + CVE-2023-37475 + + + + CVE-2023-4586 + diff --git a/src/rename-netty-native-libs.cmd b/src/rename-netty-native-libs.cmd deleted file mode 100644 index 9003f6d0ef499..0000000000000 --- a/src/rename-netty-native-libs.cmd +++ /dev/null @@ -1,82 +0,0 @@ -@REM -@REM Licensed to the Apache Software Foundation (ASF) under one -@REM or more contributor license agreements. See the NOTICE file -@REM distributed with this work for additional information -@REM regarding copyright ownership. The ASF licenses this file -@REM to you under the Apache License, Version 2.0 (the -@REM "License"); you may not use this file except in compliance -@REM with the License. You may obtain a copy of the License at -@REM -@REM http://www.apache.org/licenses/LICENSE-2.0 -@REM -@REM Unless required by applicable law or agreed to in writing, -@REM software distributed under the License is distributed on an -@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@REM KIND, either express or implied. See the License for the -@REM specific language governing permissions and limitations -@REM under the License. -@REM - -@echo off - -set ARTIFACT_ID=%1 -set JAR_PATH=%cd%/target/%ARTIFACT_ID%.jar -set FILE_PREFIX=META-INF/native - -:: echo %JAR_PATH% -:: echo %FILE_PREFIX% - -ECHO. -echo ----- Renaming epoll lib in %JAR_PATH% ------ -set TMP_DIR=%temp%\tmp_pulsar - -rd %TMP_DIR% /s /q -mkdir %TMP_DIR% - -set UNZIP_CMD=unzip -q %JAR_PATH% -d %TMP_DIR% -call %UNZIP_CMD% - -:: echo %UNZIP_CMD% -:: echo %TMP_DIR% - -cd /d %TMP_DIR%/%FILE_PREFIX% - -:: Loop through the number of groups -SET Obj_Length=2 -SET Obj[0].FROM=libnetty_transport_native_epoll_x86_64.so -SET Obj[0].TO=liborg_apache_pulsar_shade_netty_transport_native_epoll_x86_64.so -SET Obj[1].FROM=libnetty_tcnative_linux_x86_64.so -SET Obj[1].TO=liborg_apache_pulsar_shade_netty_tcnative_linux_x86_64.so -SET Obj_Index=0 - -:LoopStart -IF %Obj_Index% EQU %Obj_Length% GOTO END - -SET Obj_Current.FROM=0 -SET Obj_Current.TO=0 - -FOR /F "usebackq delims==. tokens=1-3" %%I IN (`SET Obj[%Obj_Index%]`) DO ( - SET Obj_Current.%%J=%%K.so -) - -echo "Renaming %Obj_Current.FROM% -> %Obj_Current.TO%" -call ren %Obj_Current.FROM% %Obj_Current.TO% - -SET /A Obj_Index=%Obj_Index% + 1 - -GOTO LoopStart -:: Loop end - -:END -cd /d %TMP_DIR% - -:: Overwrite the original ZIP archive -rd %JAR_PATH% /s /q -set ZIP_CMD=zip -q -r %JAR_PATH% . -:: echo %ZIP_CMD% -call %ZIP_CMD% -:: echo %TMP_DIR% -rd %TMP_DIR% /s /q - -exit /b 0 -:: echo.&pause&goto:eof \ No newline at end of file diff --git a/src/rename-netty-native-libs.sh b/src/rename-netty-native-libs.sh deleted file mode 100755 index 44b971a02c912..0000000000000 --- a/src/rename-netty-native-libs.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -e - -ARTIFACT_ID=$1 -JAR_PATH="$PWD/target/$ARTIFACT_ID.jar" - -FILE_PREFIX='META-INF/native' - -FILES_TO_RENAME=( - 'libnetty_transport_native_epoll_x86_64.so liborg_apache_pulsar_shade_netty_transport_native_epoll_x86_64.so' - 'libnetty_tcnative_linux_x86_64.so liborg_apache_pulsar_shade_netty_tcnative_linux_x86_64.so' - 'libnetty_resolver_dns_native_macos_aarch_64.jnilib liborg_apache_pulsar_shade_netty_resolver_dns_native_macos_aarch_64.jnilib' - 'libnetty_resolver_dns_native_macos_x86_64.jnilib liborg_apache_pulsar_shade_netty_resolver_dns_native_macos_x86_64.jnilib' -) - -echo "----- Renaming epoll lib in $JAR_PATH ------" -TMP_DIR=`mktemp -d` -CUR_DIR=$(pwd) -cd ${TMP_DIR} -# exclude `META-INF/LICENSE` -unzip -q $JAR_PATH -x "META-INF/LICENSE" -# include `META-INF/LICENSE` as LICENSE.netty. -# This approach is to get around the issue that MacOS is not able to recognize the difference between `META-INF/LICENSE` and `META-INF/license/`. -unzip -p $JAR_PATH META-INF/LICENSE > META-INF/LICENSE.netty -cd ${CUR_DIR} - -pushd $TMP_DIR - -for line in "${FILES_TO_RENAME[@]}"; do - read -r -a A <<< "$line" - FROM=${A[0]} - TO=${A[1]} - - if [ -f $FILE_PREFIX/$FROM ]; then - echo "Renaming $FROM -> $TO" - mv $FILE_PREFIX/$FROM $FILE_PREFIX/$TO - fi -done - -# Overwrite the original ZIP archive -rm $JAR_PATH -zip -q -r $JAR_PATH . -popd - -rm -rf $TMP_DIR diff --git a/src/set-project-version.sh b/src/set-project-version.sh index cf67e37682ff1..f3f01009b19ef 100755 --- a/src/set-project-version.sh +++ b/src/set-project-version.sh @@ -38,6 +38,7 @@ OLD_VERSION=`python3 ${ROOT_DIR}/src/get-project-version.py` mvn versions:set -DnewVersion=$NEW_VERSION mvn versions:set -DnewVersion=$NEW_VERSION -pl buildtools +mvn versions:set -DnewVersion=$NEW_VERSION -pl pulsar-bom # Set terraform ansible deployment pulsar version sed -i -e "s/${OLD_VERSION}/${NEW_VERSION}/g" ${TERRAFORM_DIR}/deploy-pulsar.yaml diff --git a/src/stage-release.sh b/src/stage-release.sh index 9f7511e7b0763..6c715b58b195e 100755 --- a/src/stage-release.sh +++ b/src/stage-release.sh @@ -26,13 +26,17 @@ if [ $# -eq 0 ]; then fi DEST_PATH=$1 +DEST_PATH="$(cd "$DEST_PATH" && pwd)" pushd $(dirname "$0") PULSAR_PATH=$(git rev-parse --show-toplevel) VERSION=`./get-project-version.py` popd -cp $PULSAR_PATH/target/apache-pulsar-$VERSION-src.tar.gz $DEST_PATH +pushd "$(dirname "$0")/.." +git archive --format=tar.gz --output="$DEST_PATH/apache-pulsar-$VERSION-src.tar.gz" --prefix="apache-pulsar-$VERSION-src/" HEAD +popd + cp $PULSAR_PATH/distribution/server/target/apache-pulsar-$VERSION-bin.tar.gz $DEST_PATH cp $PULSAR_PATH/distribution/offloaders/target/apache-pulsar-offloaders-$VERSION-bin.tar.gz $DEST_PATH cp $PULSAR_PATH/distribution/shell/target/apache-pulsar-shell-$VERSION-bin.tar.gz $DEST_PATH diff --git a/structured-event-log/pom.xml b/structured-event-log/pom.xml index c42a75397d0ca..3ce05565413b9 100644 --- a/structured-event-log/pom.xml +++ b/structured-event-log/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT structured-event-log @@ -51,7 +50,7 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl test diff --git a/testmocks/pom.xml b/testmocks/pom.xml index 30a29c7e3f9c4..b8b22200948a1 100644 --- a/testmocks/pom.xml +++ b/testmocks/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT testmocks diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/BookKeeperTestClient.java b/testmocks/src/main/java/org/apache/bookkeeper/client/BookKeeperTestClient.java index d023427e3be31..dd33c2c4532bf 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/BookKeeperTestClient.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/BookKeeperTestClient.java @@ -52,7 +52,6 @@ public BookKeeperTestClient(ClientConfiguration conf, ZooKeeper zkc) throws IOException, InterruptedException, BKException { super(conf, zkc, null, new UnpooledByteBufAllocator(false), NullStatsLogger.INSTANCE, null, null, null); - this.statsProvider = statsProvider; } public BookKeeperTestClient(ClientConfiguration conf) diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java index f0d279ef25050..4516cfea01f05 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java @@ -18,6 +18,7 @@ */ package org.apache.bookkeeper.client; +import static com.google.common.base.Preconditions.checkArgument; import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.Arrays; @@ -89,6 +90,7 @@ public static Collection getMockEnsemble() { } final Queue addEntryDelaysMillis = new ConcurrentLinkedQueue<>(); + final Queue addEntryResponseDelaysMillis = new ConcurrentLinkedQueue<>(); final List> failures = new ArrayList<>(); final List> addEntryFailures = new ArrayList<>(); @@ -367,6 +369,11 @@ public synchronized void addEntryDelay(long delay, TimeUnit unit) { addEntryDelaysMillis.add(unit.toMillis(delay)); } + public synchronized void addEntryResponseDelay(long delay, TimeUnit unit) { + checkArgument(delay >= 0, "The delay time must not be negative."); + addEntryResponseDelaysMillis.add(unit.toMillis(delay)); + } + static int getExceptionCode(Throwable t) { if (t instanceof BKException) { return ((BKException) t).getCode(); diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockLedgerHandle.java b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockLedgerHandle.java index dea33a0e67662..aa61e541d0d6b 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockLedgerHandle.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockLedgerHandle.java @@ -197,6 +197,13 @@ public void asyncAddEntry(final ByteBuf data, final AddCallback cb, final Object cb.addComplete(PulsarMockBookKeeper.getExceptionCode(exception), PulsarMockLedgerHandle.this, LedgerHandle.INVALID_ENTRY_ID, ctx); } else { + Long responseDelayMillis = bk.addEntryResponseDelaysMillis.poll(); + if (responseDelayMillis != null) { + try { + Thread.sleep(responseDelayMillis); + } catch (InterruptedException e) { + } + } cb.addComplete(BKException.Code.OK, PulsarMockLedgerHandle.this, entryId, ctx); } }, bk.executor); diff --git a/testmocks/src/main/java/org/apache/zookeeper/MockZooKeeper.java b/testmocks/src/main/java/org/apache/zookeeper/MockZooKeeper.java index 0c0f7ec9ed1d4..f32036e53f001 100644 --- a/testmocks/src/main/java/org/apache/zookeeper/MockZooKeeper.java +++ b/testmocks/src/main/java/org/apache/zookeeper/MockZooKeeper.java @@ -1114,7 +1114,7 @@ Optional programmedFailure(Op op, String path) { Optional failure = failures.stream().filter(f -> f.predicate.test(op, path)).findFirst(); if (failure.isPresent()) { failures.remove(failure.get()); - return Optional.of(failure.get().failReturnCode); + return Optional.ofNullable(failure.get().failReturnCode); } else { return Optional.empty(); } @@ -1131,6 +1131,18 @@ public void failConditional(KeeperException.Code rc, BiPredicate pre failures.add(new Failure(rc, predicate)); } + public void delay(long millis, BiPredicate predicate) { + failures.add(new Failure(null, (op, s) -> { + if (predicate.test(op, s)) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) {} + return true; + } + return false; + })); + } + public void setAlwaysFail(KeeperException.Code rc) { this.alwaysFail.set(rc); } diff --git a/tests/bc_2_0_0/pom.xml b/tests/bc_2_0_0/pom.xml index eacf4623edff5..48b45ccb56fa4 100644 --- a/tests/bc_2_0_0/pom.xml +++ b/tests/bc_2_0_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT bc_2_0_0 diff --git a/tests/bc_2_0_1/pom.xml b/tests/bc_2_0_1/pom.xml index d624c66170d56..b7a15fb9c7390 100644 --- a/tests/bc_2_0_1/pom.xml +++ b/tests/bc_2_0_1/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT bc_2_0_1 diff --git a/tests/bc_2_6_0/pom.xml b/tests/bc_2_6_0/pom.xml index 2cd8e7f786809..38c3f77917316 100644 --- a/tests/bc_2_6_0/pom.xml +++ b/tests/bc_2_6_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT 4.0.0 diff --git a/tests/certificate-authority/ec/broker_client.cert.pem b/tests/certificate-authority/ec/broker_client.cert.pem new file mode 100644 index 0000000000000..2993ed41ad9d6 --- /dev/null +++ b/tests/certificate-authority/ec/broker_client.cert.pem @@ -0,0 +1,9 @@ +-----BEGIN CERTIFICATE----- +MIIBIjCBygIUSAxJKNrIEmn3SVyw5rcYhwhKulwwCgYIKoZIzj0EAwIwETEPMA0G +A1UEAwwGQ0FSb290MB4XDTIzMTEyNDExNTE1M1oXDTMzMTEyMTExNTE1M1owGDEW +MBQGA1UEAwwNYnJva2VyX2NsaWVudDBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA +BGxRL4naRhrTZ9T2WdMBkCNmiamkrzEiDO55RVjhpHGWIoqPOvzs8i97vCVx39GV +vV/9agDp2nSuXYW8ax3UKnkwCgYIKoZIzj0EAwIDRwAwRAIge8qxnGgmv5h+Yw3Y +Ab/6xFD5QWERGMlfIl4ZCO3o6S0CICS/4jj45GfAPZS9QPfuo15rEa9Rbvvmmi+K +yY0JA0SP +-----END CERTIFICATE----- diff --git a/tests/certificate-authority/ec/broker_client.csr.pem b/tests/certificate-authority/ec/broker_client.csr.pem new file mode 100644 index 0000000000000..1f10a3c77f2b6 --- /dev/null +++ b/tests/certificate-authority/ec/broker_client.csr.pem @@ -0,0 +1,7 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIHTMHoCAQAwGDEWMBQGA1UEAwwNYnJva2VyX2NsaWVudDBZMBMGByqGSM49AgEG +CCqGSM49AwEHA0IABGxRL4naRhrTZ9T2WdMBkCNmiamkrzEiDO55RVjhpHGWIoqP +Ovzs8i97vCVx39GVvV/9agDp2nSuXYW8ax3UKnmgADAKBggqhkjOPQQDAgNJADBG +AiEA8sGFcbQuUGIUTCXTQ0z9b0eIYFIDVOcGSInQ+0unMJMCIQCmH0GlXZRGB2lx +HtfIz76HNnVu153LsHE11AEx7d/j2g== +-----END CERTIFICATE REQUEST----- diff --git a/tests/certificate-authority/ec/broker_client.key-pk8.pem b/tests/certificate-authority/ec/broker_client.key-pk8.pem new file mode 100644 index 0000000000000..124073b024564 --- /dev/null +++ b/tests/certificate-authority/ec/broker_client.key-pk8.pem @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgA92tkFXxKHYUJbeB +vvnMaGBnP2IenpF66Fikb06xbUKhRANCAARsUS+J2kYa02fU9lnTAZAjZomppK8x +IgzueUVY4aRxliKKjzr87PIve7wlcd/Rlb1f/WoA6dp0rl2FvGsd1Cp5 +-----END PRIVATE KEY----- diff --git a/tests/certificate-authority/ec/broker_client.key.pem b/tests/certificate-authority/ec/broker_client.key.pem new file mode 100644 index 0000000000000..4d4b5163b1bb4 --- /dev/null +++ b/tests/certificate-authority/ec/broker_client.key.pem @@ -0,0 +1,8 @@ +-----BEGIN EC PARAMETERS----- +BggqhkjOPQMBBw== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIAPdrZBV8Sh2FCW3gb75zGhgZz9iHp6ReuhYpG9OsW1CoAoGCCqGSM49 +AwEHoUQDQgAEbFEvidpGGtNn1PZZ0wGQI2aJqaSvMSIM7nlFWOGkcZYiio86/Ozy +L3u8JXHf0ZW9X/1qAOnadK5dhbxrHdQqeQ== +-----END EC PRIVATE KEY----- diff --git a/tests/certificate-authority/ec/ca.cert.pem b/tests/certificate-authority/ec/ca.cert.pem new file mode 100644 index 0000000000000..c10385d997e86 --- /dev/null +++ b/tests/certificate-authority/ec/ca.cert.pem @@ -0,0 +1,10 @@ +-----BEGIN CERTIFICATE----- +MIIBeDCCAR2gAwIBAgIUKRGzcPm3RVuI7tXdPDAZZ7Vhqs8wCgYIKoZIzj0EAwIw +ETEPMA0GA1UEAwwGQ0FSb290MB4XDTIzMTEyNDExNTExNVoXDTMzMTEyMTExNTEx +NVowETEPMA0GA1UEAwwGQ0FSb290MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE +kOKZaL45B7PUB+G25GLP1PPfTkio/DaHUML+KJjxpdCnSmq+mt/EAQWlqNPB1hJv +6kOJ52vSxKe02BMeuROed6NTMFEwHQYDVR0OBBYEFDkqfvrnJ7PJhxJ7FTA7o8+b +f+CRMB8GA1UdIwQYMBaAFDkqfvrnJ7PJhxJ7FTA7o8+bf+CRMA8GA1UdEwEB/wQF +MAMBAf8wCgYIKoZIzj0EAwIDSQAwRgIhAN9+TWNNbIz8rMdkf4LGoIeQzYcAEyGJ +90ORM5JciBdaAiEA8UsuQBD4wO1t6plnRydkGMTeb1dNDEnhsuXOXBps8fE= +-----END CERTIFICATE----- diff --git a/tests/certificate-authority/ec/ca.cert.srl b/tests/certificate-authority/ec/ca.cert.srl new file mode 100644 index 0000000000000..a30f44e979e72 --- /dev/null +++ b/tests/certificate-authority/ec/ca.cert.srl @@ -0,0 +1 @@ +480C4928DAC81269F7495CB0E6B71887084ABA5D diff --git a/tests/certificate-authority/ec/ca.key.pem b/tests/certificate-authority/ec/ca.key.pem new file mode 100644 index 0000000000000..1255354584869 --- /dev/null +++ b/tests/certificate-authority/ec/ca.key.pem @@ -0,0 +1,8 @@ +-----BEGIN EC PARAMETERS----- +BggqhkjOPQMBBw== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIPT1Jap2sJ7NUGWT6q0fnSRoVRNNryWe/JHPwttyQke4oAoGCCqGSM49 +AwEHoUQDQgAEkOKZaL45B7PUB+G25GLP1PPfTkio/DaHUML+KJjxpdCnSmq+mt/E +AQWlqNPB1hJv6kOJ52vSxKe02BMeuROedw== +-----END EC PRIVATE KEY----- diff --git a/tests/certificate-authority/ec/certificate_generation.txt b/tests/certificate-authority/ec/certificate_generation.txt new file mode 100644 index 0000000000000..7a6caa7b8f4be --- /dev/null +++ b/tests/certificate-authority/ec/certificate_generation.txt @@ -0,0 +1,34 @@ +# CA Private Key +openssl ecparam -name secp256r1 -genkey -out ca.key.pem +# Request certificate +openssl req -x509 -new -nodes -key ca.key.pem -subj "/CN=CARoot" -days 3650 -out ca.cert.pem + +# Server Private Key +openssl ecparam -name secp256r1 -genkey -out server.key.pem +# Convert to pkcs8 +openssl pkcs8 -topk8 -inform PEM -outform PEM -in server.key.pem -out server.key-pk8.pem -nocrypt +# Request certificate +openssl req -new -config server.conf -key server.key.pem -out server.csr.pem -sha256 +# Sign with CA +openssl x509 -req -in server.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out server.cert.pem -days 3650 -extensions v3_ext -extfile server.conf -sha256 + +# Broker internal client Private Key +openssl ecparam -name secp256r1 -genkey -out broker_client.key.pem +# Convert to pkcs8 +openssl pkcs8 -topk8 -inform PEM -outform PEM -in broker_client.key.pem -out broker_client.key-pk8.pem -nocrypt +# Request certificate +openssl req -new -subj "/CN=broker_client" -key broker_client.key.pem -out broker_client.csr.pem -sha256 +# Sign with CA +openssl x509 -req -in broker_client.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out broker_client.cert.pem -days 3650 -sha256 + + +# Client Private Key +openssl ecparam -name secp256r1 -genkey -out client.key.pem +# Convert to pkcs8 +openssl pkcs8 -topk8 -inform PEM -outform PEM -in client.key.pem -out client.key-pk8.pem -nocrypt +# Request certificate +openssl req -new -subj "/CN=client" -key client.key.pem -out client.csr.pem -sha256 +# Sign with CA +openssl x509 -req -in client.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out client.cert.pem -days 3650 -sha256 + + diff --git a/tests/certificate-authority/ec/client.cert.pem b/tests/certificate-authority/ec/client.cert.pem new file mode 100644 index 0000000000000..87701a6938d25 --- /dev/null +++ b/tests/certificate-authority/ec/client.cert.pem @@ -0,0 +1,8 @@ +-----BEGIN CERTIFICATE----- +MIIBHDCBwwIUSAxJKNrIEmn3SVyw5rcYhwhKul0wCgYIKoZIzj0EAwIwETEPMA0G +A1UEAwwGQ0FSb290MB4XDTIzMTEyNDExNTIwNVoXDTMzMTEyMTExNTIwNVowETEP +MA0GA1UEAwwGY2xpZW50MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE4QZJuqZS +mSDbjkoFGKvtYmSVaJ3IjtmgWsgQio4F5phIXpM6IZZfcLkJToY0b9W2jGhODK55 +jA+zkRxHrICkwTAKBggqhkjOPQQDAgNIADBFAiEA0iGNqg4t16SxFdZJu7o9gK8R +XVXphQ/9XAtw4XqfCUYCIGLoExE9XKdkzZ+sahFOpKD6YLZ1GgPRBPpBJFBGTYu7 +-----END CERTIFICATE----- diff --git a/tests/certificate-authority/ec/client.csr.pem b/tests/certificate-authority/ec/client.csr.pem new file mode 100644 index 0000000000000..4ec08d410f504 --- /dev/null +++ b/tests/certificate-authority/ec/client.csr.pem @@ -0,0 +1,7 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIHLMHMCAQAwETEPMA0GA1UEAwwGY2xpZW50MFkwEwYHKoZIzj0CAQYIKoZIzj0D +AQcDQgAE4QZJuqZSmSDbjkoFGKvtYmSVaJ3IjtmgWsgQio4F5phIXpM6IZZfcLkJ +ToY0b9W2jGhODK55jA+zkRxHrICkwaAAMAoGCCqGSM49BAMCA0gAMEUCIQDNZOBD +Z/YAWKEeRSVqhPvIpFYob1gmQfDcBJdG8e0K8wIgcfO0PLquIZP9P8VrDkkLQdZ9 +krOKk+F/LF9aqQBHTbU= +-----END CERTIFICATE REQUEST----- diff --git a/tests/certificate-authority/ec/client.key-pk8.pem b/tests/certificate-authority/ec/client.key-pk8.pem new file mode 100644 index 0000000000000..2b07827f21472 --- /dev/null +++ b/tests/certificate-authority/ec/client.key-pk8.pem @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgrC3O+TuZ82b1bD1M +SI9lMu6aaebqfoggcnaaAyUUstKhRANCAAThBkm6plKZINuOSgUYq+1iZJVonciO +2aBayBCKjgXmmEhekzohll9wuQlOhjRv1baMaE4MrnmMD7ORHEesgKTB +-----END PRIVATE KEY----- diff --git a/tests/certificate-authority/ec/client.key.pem b/tests/certificate-authority/ec/client.key.pem new file mode 100644 index 0000000000000..ac1207fa51c0b --- /dev/null +++ b/tests/certificate-authority/ec/client.key.pem @@ -0,0 +1,8 @@ +-----BEGIN EC PARAMETERS----- +BggqhkjOPQMBBw== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIKwtzvk7mfNm9Ww9TEiPZTLummnm6n6IIHJ2mgMlFLLSoAoGCCqGSM49 +AwEHoUQDQgAE4QZJuqZSmSDbjkoFGKvtYmSVaJ3IjtmgWsgQio4F5phIXpM6IZZf +cLkJToY0b9W2jGhODK55jA+zkRxHrICkwQ== +-----END EC PRIVATE KEY----- diff --git a/tests/certificate-authority/ec/jks/broker_client.cert.pem b/tests/certificate-authority/ec/jks/broker_client.cert.pem new file mode 100644 index 0000000000000..8a12e941d4e43 --- /dev/null +++ b/tests/certificate-authority/ec/jks/broker_client.cert.pem @@ -0,0 +1,10 @@ +-----BEGIN NEW CERTIFICATE REQUEST----- +MIIBXjCCAQQCAQAwcjEQMA4GA1UEBhMHVW5rbm93bjEQMA4GA1UECBMHVW5rbm93 +bjEQMA4GA1UEBxMHVW5rbm93bjEQMA4GA1UEChMHVW5rbm93bjEQMA4GA1UECxMH +VW5rbm93bjEWMBQGA1UEAwwNYnJva2VyX2NsaWVudDBZMBMGByqGSM49AgEGCCqG +SM49AwEHA0IABNEOf45UIs53Va887xTFRkZlmCnJUwYeu50pEll1APUwcldIHMXY +EqRqoTOcBtSRx4CpO9LMPFmyCS1E+afXnbKgMDAuBgkqhkiG9w0BCQ4xITAfMB0G +A1UdDgQWBBQFrlAl1jTZMagQVrax+OLTDJAQujAKBggqhkjOPQQDAgNIADBFAiBA +sgj2HrKwxCfoUbBIjYqRcLPRRVBsbYOGk4e2uFTZPwIhAN/AdQn786S/ebnwSUzR +yPyKEH+Qspx9nB08sQNn9N6U +-----END NEW CERTIFICATE REQUEST----- diff --git a/tests/certificate-authority/ec/jks/broker_client.keystore.jks b/tests/certificate-authority/ec/jks/broker_client.keystore.jks new file mode 100644 index 0000000000000..81ecf4497198c Binary files /dev/null and b/tests/certificate-authority/ec/jks/broker_client.keystore.jks differ diff --git a/tests/certificate-authority/ec/jks/broker_client.signed.cert.pem b/tests/certificate-authority/ec/jks/broker_client.signed.cert.pem new file mode 100644 index 0000000000000..b91c69400c5d1 --- /dev/null +++ b/tests/certificate-authority/ec/jks/broker_client.signed.cert.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBfTCCASQCFAJ6wB27laA1BCNConaAQPValPtaMAoGCCqGSM49BAMCMBExDzAN +BgNVBAMMBkNBUm9vdDAeFw0yMzExMjUwNzAzNTNaFw0zMzExMjIwNzAzNTNaMHIx +EDAOBgNVBAYTB1Vua25vd24xEDAOBgNVBAgTB1Vua25vd24xEDAOBgNVBAcTB1Vu +a25vd24xEDAOBgNVBAoTB1Vua25vd24xEDAOBgNVBAsTB1Vua25vd24xFjAUBgNV +BAMMDWJyb2tlcl9jbGllbnQwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATRDn+O +VCLOd1WvPO8UxUZGZZgpyVMGHrudKRJZdQD1MHJXSBzF2BKkaqEznAbUkceAqTvS +zDxZsgktRPmn152yMAoGCCqGSM49BAMCA0cAMEQCIArXdTOx19Nn/a6bsfTYurQW +4cepF5VKKijEjzyV69/BAiBpg60QwoZeSmz6bmil2zSb65jXrTzwhLpUZckVuHKn +og== +-----END CERTIFICATE----- diff --git a/tests/certificate-authority/ec/jks/ca.cert.pem b/tests/certificate-authority/ec/jks/ca.cert.pem new file mode 100644 index 0000000000000..a235464be7064 --- /dev/null +++ b/tests/certificate-authority/ec/jks/ca.cert.pem @@ -0,0 +1,10 @@ +-----BEGIN CERTIFICATE----- +MIIBdjCCAR2gAwIBAgIUfHm94cF84m6FrJVNywJI4qTGZAEwCgYIKoZIzj0EAwIw +ETEPMA0GA1UEAwwGQ0FSb290MB4XDTIzMTEyNTAxMzQzM1oXDTMzMTEyMjAxMzQz +M1owETEPMA0GA1UEAwwGQ0FSb290MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE +Sxvkij8HQ+g07SnOLz1in81iGKY7lOAbJ1r4ihMVnOVjS2A4ZVGXHM2wp5ZB9r3Y +jPByBiaPApm/J17JwlXynqNTMFEwHQYDVR0OBBYEFKqDJwbgz0/Q3EKJ78OVJI5k +8+RYMB8GA1UdIwQYMBaAFKqDJwbgz0/Q3EKJ78OVJI5k8+RYMA8GA1UdEwEB/wQF +MAMBAf8wCgYIKoZIzj0EAwIDRwAwRAIgEF9RiwV0oBh9x1AvLFPoK5nnUlJ+0MNE +zz8Zw284zkICIDUZOPN/E7ZmTKzfoZ0EkxRrinEZ5M538aNbYFAUYoK+ +-----END CERTIFICATE----- diff --git a/tests/certificate-authority/ec/jks/ca.cert.srl b/tests/certificate-authority/ec/jks/ca.cert.srl new file mode 100644 index 0000000000000..c7b003ddff287 --- /dev/null +++ b/tests/certificate-authority/ec/jks/ca.cert.srl @@ -0,0 +1 @@ +027AC01DBB95A035042342A2768040F55A94FB5B diff --git a/tests/certificate-authority/ec/jks/ca.key.pem b/tests/certificate-authority/ec/jks/ca.key.pem new file mode 100644 index 0000000000000..57e595f139525 --- /dev/null +++ b/tests/certificate-authority/ec/jks/ca.key.pem @@ -0,0 +1,8 @@ +-----BEGIN EC PARAMETERS----- +BggqhkjOPQMBBw== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIJ/5AX63GN8cadJUCa5Aza5592JS7go9TXNfYemS4Ku4oAoGCCqGSM49 +AwEHoUQDQgAESxvkij8HQ+g07SnOLz1in81iGKY7lOAbJ1r4ihMVnOVjS2A4ZVGX +HM2wp5ZB9r3YjPByBiaPApm/J17JwlXyng== +-----END EC PRIVATE KEY----- diff --git a/tests/certificate-authority/ec/jks/ca.truststore.jks b/tests/certificate-authority/ec/jks/ca.truststore.jks new file mode 100644 index 0000000000000..e2a667b21d6ac Binary files /dev/null and b/tests/certificate-authority/ec/jks/ca.truststore.jks differ diff --git a/tests/certificate-authority/ec/jks/client.cert.pem b/tests/certificate-authority/ec/jks/client.cert.pem new file mode 100644 index 0000000000000..022e63c57e077 --- /dev/null +++ b/tests/certificate-authority/ec/jks/client.cert.pem @@ -0,0 +1,10 @@ +-----BEGIN NEW CERTIFICATE REQUEST----- +MIIBVjCB/QIBADBrMRAwDgYDVQQGEwdVbmtub3duMRAwDgYDVQQIEwdVbmtub3du +MRAwDgYDVQQHEwdVbmtub3duMRAwDgYDVQQKEwdVbmtub3duMRAwDgYDVQQLEwdV +bmtub3duMQ8wDQYDVQQDEwZjbGllbnQwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNC +AARVggD+3riNuIAZs9O/7kA3z2jC1cKEyBvftGwhixWf5iywL2pjw8/j2fyZ7Ya1 +nl2gFGHy1uaQKYizURJX9kQcoDAwLgYJKoZIhvcNAQkOMSEwHzAdBgNVHQ4EFgQU +OQskWj9xk/eM5xQQhsQcPKZWCjcwCgYIKoZIzj0EAwIDSAAwRQIgGoPhD18yLhqK +fL9I2ailJ2+ijoxHxO8ZhMR/8rxrv/oCIQDJfWQQtbcMBuMJoIdVFomFmERdr/Ix +wttK875kwsfRQQ== +-----END NEW CERTIFICATE REQUEST----- diff --git a/tests/certificate-authority/ec/jks/client.keystore.jks b/tests/certificate-authority/ec/jks/client.keystore.jks new file mode 100644 index 0000000000000..cdca07eb2a43f Binary files /dev/null and b/tests/certificate-authority/ec/jks/client.keystore.jks differ diff --git a/tests/certificate-authority/ec/jks/client.signed.cert.pem b/tests/certificate-authority/ec/jks/client.signed.cert.pem new file mode 100644 index 0000000000000..f0d8b87dbc662 --- /dev/null +++ b/tests/certificate-authority/ec/jks/client.signed.cert.pem @@ -0,0 +1,10 @@ +-----BEGIN CERTIFICATE----- +MIIBdjCCAR0CFAJ6wB27laA1BCNConaAQPValPtbMAoGCCqGSM49BAMCMBExDzAN +BgNVBAMMBkNBUm9vdDAeFw0yMzExMjUwNzAzNTdaFw0zMzExMjIwNzAzNTdaMGsx +EDAOBgNVBAYTB1Vua25vd24xEDAOBgNVBAgTB1Vua25vd24xEDAOBgNVBAcTB1Vu +a25vd24xEDAOBgNVBAoTB1Vua25vd24xEDAOBgNVBAsTB1Vua25vd24xDzANBgNV +BAMTBmNsaWVudDBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABFWCAP7euI24gBmz +07/uQDfPaMLVwoTIG9+0bCGLFZ/mLLAvamPDz+PZ/JnthrWeXaAUYfLW5pApiLNR +Elf2RBwwCgYIKoZIzj0EAwIDRwAwRAIgEOvnGOeS9KdZ301pMOqovNDxxr1Sd4hy +rp9a3+3LBvECIBxzHQ/IZN8nt9eG9Cm8vikOlmI1AmvMIKmo1n/9NwGr +-----END CERTIFICATE----- diff --git a/tests/certificate-authority/ec/jks/key_store_generation.txt b/tests/certificate-authority/ec/jks/key_store_generation.txt new file mode 100644 index 0000000000000..62c48e9a089f1 --- /dev/null +++ b/tests/certificate-authority/ec/jks/key_store_generation.txt @@ -0,0 +1,33 @@ +# CA Private Key +openssl ecparam -name secp256r1 -genkey -out ca.key.pem +# Request certificate +openssl req -x509 -new -nodes -key ca.key.pem -subj "/CN=CARoot" -days 3650 -out ca.cert.pem +# Build Trust Cert +keytool -keystore ca.truststore.jks -alias ca -importcert -file ca.cert.pem -storepass rootpw -keypass rootpw -noprompt + + +# Create server keystore +keytool -keystore server.keystore.jks -alias server -keyalg EC -validity 3600 -genkey -storepass serverpw -keypass serverpw -dname 'CN=server,OU=Unknown,O=Unknown,L=Unknown,ST=Unknown,C=Unknown' -noprompt +# Export the certificate from the keystore: +keytool -keystore server.keystore.jks -alias server -certreq -file server.cert.pem -storepass serverpw -keypass serverpw -noprompt +# Sign it with the CA: +openssl x509 -req -in server.cert.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out server.signed.cert.pem -days 3650 -sha256 +# Import signed cert into key store +keytool -keystore server.keystore.jks -alias ca -importcert -file ca.cert.pem -storepass serverpw -keypass serverpw -noprompt +keytool -keystore server.keystore.jks -alias server -importcert -file server.signed.cert.pem -storepass serverpw -keypass serverpw -noprompt + + +# Create broker client keystore +keytool -keystore broker_client.keystore.jks -alias broker_client -keyalg EC -validity 3600 -genkey -storepass brokerclientpw -keypass brokerclientpw -dname 'CN=broker_client,OU=Unknown,O=Unknown,L=Unknown,ST=Unknown,C=Unknown' -noprompt +keytool -keystore broker_client.keystore.jks -alias broker_client -certreq -file broker_client.cert.pem -storepass brokerclientpw -keypass brokerclientpw -noprompt +openssl x509 -req -in broker_client.cert.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out broker_client.signed.cert.pem -days 3650 -sha256 +keytool -keystore broker_client.keystore.jks -alias ca -importcert -file ca.cert.pem -storepass brokerclientpw -keypass brokerclientpw -noprompt +keytool -keystore broker_client.keystore.jks -alias broker_client -importcert -file broker_client.signed.cert.pem -storepass brokerclientpw -keypass brokerclientpw -noprompt + + +# Create client keystore +keytool -keystore client.keystore.jks -alias client -keyalg EC -validity 3600 -genkey -storepass clientpw -keypass clientpw -dname 'CN=client,OU=Unknown,O=Unknown,L=Unknown,ST=Unknown,C=Unknown' -noprompt +keytool -keystore client.keystore.jks -alias client -certreq -file client.cert.pem -storepass clientpw -keypass clientpw -noprompt +openssl x509 -req -in client.cert.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out client.signed.cert.pem -days 3650 -sha256 +keytool -keystore client.keystore.jks -alias ca -importcert -file ca.cert.pem -storepass clientpw -keypass clientpw -noprompt +keytool -keystore client.keystore.jks -alias client -importcert -file client.signed.cert.pem -storepass clientpw -keypass clientpw -noprompt \ No newline at end of file diff --git a/tests/certificate-authority/ec/jks/server.cert.pem b/tests/certificate-authority/ec/jks/server.cert.pem new file mode 100644 index 0000000000000..e63f822f13a99 --- /dev/null +++ b/tests/certificate-authority/ec/jks/server.cert.pem @@ -0,0 +1,10 @@ +-----BEGIN NEW CERTIFICATE REQUEST----- +MIIBVzCB/QIBADBrMRAwDgYDVQQGEwdVbmtub3duMRAwDgYDVQQIEwdVbmtub3du +MRAwDgYDVQQHEwdVbmtub3duMRAwDgYDVQQKEwdVbmtub3duMRAwDgYDVQQLEwdV +bmtub3duMQ8wDQYDVQQDEwZzZXJ2ZXIwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNC +AAQ0HBuze29SNGa33jTMzNZ+i4ZmQtccZGrGR8SfJPJVENahKSqJnoLgfHmzjfya +XyYr/uwP4LYIqrWw9nsYZmb+oDAwLgYJKoZIhvcNAQkOMSEwHzAdBgNVHQ4EFgQU +af0LXo7UTDd+QWFpoEkvqJPs+I0wCgYIKoZIzj0EAwIDSQAwRgIhANzNFj7zWN22 +uiNcz1EUvD8HS9C7R6Fk6Ps5Z54RNTtDAiEAlcDLOkHcgehHBIi79sfC9ZFj0Acg +kcDY79IQi1k4gsc= +-----END NEW CERTIFICATE REQUEST----- diff --git a/tests/certificate-authority/ec/jks/server.keystore.jks b/tests/certificate-authority/ec/jks/server.keystore.jks new file mode 100644 index 0000000000000..25c9eb4d1b07f Binary files /dev/null and b/tests/certificate-authority/ec/jks/server.keystore.jks differ diff --git a/tests/certificate-authority/ec/jks/server.signed.cert.pem b/tests/certificate-authority/ec/jks/server.signed.cert.pem new file mode 100644 index 0000000000000..79a09731735ee --- /dev/null +++ b/tests/certificate-authority/ec/jks/server.signed.cert.pem @@ -0,0 +1,10 @@ +-----BEGIN CERTIFICATE----- +MIIBdzCCAR0CFAJ6wB27laA1BCNConaAQPValPtQMAoGCCqGSM49BAMCMBExDzAN +BgNVBAMMBkNBUm9vdDAeFw0yMzExMjUwMTQzMDRaFw0zMzExMjIwMTQzMDRaMGsx +EDAOBgNVBAYTB1Vua25vd24xEDAOBgNVBAgTB1Vua25vd24xEDAOBgNVBAcTB1Vu +a25vd24xEDAOBgNVBAoTB1Vua25vd24xEDAOBgNVBAsTB1Vua25vd24xDzANBgNV +BAMTBnNlcnZlcjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABDQcG7N7b1I0Zrfe +NMzM1n6LhmZC1xxkasZHxJ8k8lUQ1qEpKomeguB8ebON/JpfJiv+7A/gtgiqtbD2 +exhmZv4wCgYIKoZIzj0EAwIDSAAwRQIgG4IatfLHoaCGVPDxnYV3XkWzVJpAEdX6 +QIDYgmdogckCIQDpJJle7jw6PNA1o3nSZJ2o2GCOg9nmmNaKVBQfxL2E/g== +-----END CERTIFICATE----- diff --git a/tests/certificate-authority/ec/server.cert.pem b/tests/certificate-authority/ec/server.cert.pem new file mode 100644 index 0000000000000..184aa882e2828 --- /dev/null +++ b/tests/certificate-authority/ec/server.cert.pem @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB9DCCAZqgAwIBAgIUSAxJKNrIEmn3SVyw5rcYhwhKulswCgYIKoZIzj0EAwIw +ETEPMA0GA1UEAwwGQ0FSb290MB4XDTIzMTEyNDExNTE0MloXDTMzMTEyMTExNTE0 +MlowETEPMA0GA1UEAwwGc2VydmVyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE +8xsai5lXx2Y7TbmzB1sZr2RunBOtzHFelNBmryjgkatf0yIEy9/cCmH+DvJfjvG1 +hfZDvnVFBPaoDFwgmvb26KOBzzCBzDBMBgNVHSMERTBDgBQ5Kn765yezyYcSexUw +O6PPm3/gkaEVpBMwETEPMA0GA1UEAwwGQ0FSb290ghQpEbNw+bdFW4ju1d08MBln +tWGqzzAJBgNVHRMEAjAAMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEF +BQcDATAtBgNVHREEJjAkggZwdWxzYXKCDnB1bHNhci5kZWZhdWx0hwR/AAABhwTA +qAECMB0GA1UdDgQWBBQe+uKXtB+I7vfU+mRAMvuNYbWJSTAKBggqhkjOPQQDAgNI +ADBFAiEAlCUpm4I5F6+OPS/lEJKIEQJILHivB3lPYW/OgXlpq5UCIFuUVgYwQ2ca +yildeQibDy/gbxLCFVzDtYrVKf7SZSK+ +-----END CERTIFICATE----- diff --git a/pulsar-io/kinesis/src/test/resources/sinkConfig.yaml b/tests/certificate-authority/ec/server.conf similarity index 67% rename from pulsar-io/kinesis/src/test/resources/sinkConfig.yaml rename to tests/certificate-authority/ec/server.conf index 7d99db65d079a..557c2c27202db 100644 --- a/pulsar-io/kinesis/src/test/resources/sinkConfig.yaml +++ b/tests/certificate-authority/ec/server.conf @@ -17,11 +17,24 @@ # under the License. # -{ - "awsEndpoint" : "https://some.endpoint.aws", - "awsRegion": "us-east-1", - "awsKinesisStreamName": "my-stream", - "awsCredentialPluginParam": "{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}", - "messageFormat": "ONLY_RAW_PAYLOAD", - "retainOrdering": "true" -} \ No newline at end of file +[ req ] +default_bits = 2048 +prompt = no +default_md = sha256 +distinguished_name = dn + +[ v3_ext ] +authorityKeyIdentifier=keyid,issuer:always +basicConstraints=CA:FALSE +keyUsage=critical, digitalSignature, keyEncipherment +extendedKeyUsage=serverAuth +subjectAltName=@alt_names + +[ dn ] +CN = server + +[ alt_names ] +DNS.1 = pulsar +DNS.2 = pulsar.default +IP.1 = 127.0.0.1 +IP.2 = 192.168.1.2 diff --git a/tests/certificate-authority/ec/server.csr.pem b/tests/certificate-authority/ec/server.csr.pem new file mode 100644 index 0000000000000..ac75bb2d1ff64 --- /dev/null +++ b/tests/certificate-authority/ec/server.csr.pem @@ -0,0 +1,7 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIHLMHMCAQAwETEPMA0GA1UEAwwGc2VydmVyMFkwEwYHKoZIzj0CAQYIKoZIzj0D +AQcDQgAE8xsai5lXx2Y7TbmzB1sZr2RunBOtzHFelNBmryjgkatf0yIEy9/cCmH+ +DvJfjvG1hfZDvnVFBPaoDFwgmvb26KAAMAoGCCqGSM49BAMCA0gAMEUCIFUCpVkb +5u0EEY/4zcXFTHahm4xq/GAziFZsGS3mjwncAiEA2RGraZwclbHwjBiIChd56Xim +SHyZ2voxfe+xJG7uX8g= +-----END CERTIFICATE REQUEST----- diff --git a/tests/certificate-authority/ec/server.key-pk8.pem b/tests/certificate-authority/ec/server.key-pk8.pem new file mode 100644 index 0000000000000..f30bd1cc58cc7 --- /dev/null +++ b/tests/certificate-authority/ec/server.key-pk8.pem @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgnYGgcNz49WhMgVPD +LmI1fYtfI/YqWDrd2jtnXFGNawShRANCAATzGxqLmVfHZjtNubMHWxmvZG6cE63M +cV6U0GavKOCRq1/TIgTL39wKYf4O8l+O8bWF9kO+dUUE9qgMXCCa9vbo +-----END PRIVATE KEY----- diff --git a/tests/certificate-authority/ec/server.key.pem b/tests/certificate-authority/ec/server.key.pem new file mode 100644 index 0000000000000..1725f1be43fb1 --- /dev/null +++ b/tests/certificate-authority/ec/server.key.pem @@ -0,0 +1,8 @@ +-----BEGIN EC PARAMETERS----- +BggqhkjOPQMBBw== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIJ2BoHDc+PVoTIFTwy5iNX2LXyP2Klg63do7Z1xRjWsEoAoGCCqGSM49 +AwEHoUQDQgAE8xsai5lXx2Y7TbmzB1sZr2RunBOtzHFelNBmryjgkatf0yIEy9/c +CmH+DvJfjvG1hfZDvnVFBPaoDFwgmvb26A== +-----END EC PRIVATE KEY----- diff --git a/tests/certificate-authority/generate_keystore.sh b/tests/certificate-authority/generate_keystore.sh index faf808324b0d9..4f928192d12d6 100755 --- a/tests/certificate-authority/generate_keystore.sh +++ b/tests/certificate-authority/generate_keystore.sh @@ -54,6 +54,9 @@ java ../RemoveJksPassword.java broker.truststore.jks 111111 broker.truststore.no java ../RemoveJksPassword.java proxy.truststore.jks 111111 proxy.truststore.nopassword.jks java ../RemoveJksPassword.java proxy-and-client.truststore.jks 111111 proxy-and-client.truststore.nopassword.jks +# write broker truststore to pem file for use in http client as a ca cert +keytool -keystore broker.truststore.jks -exportcert -alias truststore | openssl x509 -inform der -text > broker.truststore.pem + # cleanup rm broker.cer rm client.cer diff --git a/tests/certificate-authority/jks/broker.truststore.pem b/tests/certificate-authority/jks/broker.truststore.pem new file mode 100644 index 0000000000000..e0fcbf6c2da5d --- /dev/null +++ b/tests/certificate-authority/jks/broker.truststore.pem @@ -0,0 +1,73 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 17442377827579325010 (0xf20fc6387303ea52) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=localhost + Validity + Not Before: Feb 13 05:52:32 2023 GMT + Not After : Jan 20 05:52:32 2123 GMT + Subject: C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:e8:b7:df:ee:32:88:98:0e:07:18:41:fd:d8:e2: + 34:c8:7e:54:86:9b:d8:a3:7b:47:6e:e9:a1:8c:74: + f3:0f:bb:a6:83:43:11:77:c4:9a:ab:53:74:64:46: + fe:81:8f:72:03:97:0b:37:78:6e:ff:6e:64:68:eb: + 9e:b0:6b:ae:79:6a:50:e1:1e:1d:39:df:51:0b:04: + 8e:89:4b:e4:02:6b:e4:12:d3:41:47:2c:8e:75:30: + 14:1a:ed:32:93:a6:fe:73:7d:dc:e0:d4:93:d9:8c: + 44:55:d2:53:4a:0c:2e:f9:95:5f:4f:cb:2f:e5:41: + c5:bc:33:88:eb:d8:52:0e:19:55:36:02:31:0f:81: + 0b:c2:4f:35:63:a6:06:e3:3b:93:3c:04:5c:63:32: + 3d:c3:26:50:96:b7:8c:67:8e:9e:d1:a3:7e:7b:54: + 86:35:07:fe:15:32:fb:6d:4e:e7:4c:97:95:32:e2: + d8:04:8b:e2:00:4b:85:64:91:70:ae:24:88:07:47: + 45:7d:19:c3:c1:25:02:50:f8:f4:0a:a3:7b:6a:11: + c3:6f:b9:da:06:a0:2d:3d:65:47:8e:28:d2:18:8a: + 45:bb:79:7b:ba:d5:d8:29:4d:4e:da:fd:b2:1e:eb: + b6:59:1b:f1:c9:8a:ea:ac:7d:72:dc:8d:e7:43:7d: + d3:c7 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 88:3A:59:9D:B2:72:AC:64:FB:F3:8D:79:ED:54:FE:20:C3:83:49:24 + Signature Algorithm: sha256WithRSAEncryption + 71:88:eb:c7:f2:f2:84:35:f7:ab:bc:3c:ce:be:11:f9:c9:36: + b5:1e:93:96:cf:66:06:4a:f1:b6:f5:cc:97:b9:cd:93:0f:8a: + 66:62:85:cb:fd:c1:63:7e:38:d9:02:0c:6b:04:38:7b:ec:82: + e6:25:f3:8c:99:8d:d1:20:c6:eb:5e:75:9e:b6:f0:ec:ad:9a: + 76:22:41:bd:88:e9:c3:7f:3d:8e:9a:f1:4a:b3:e2:30:ca:e0: + 68:39:8d:7c:e6:db:dc:fd:44:75:66:55:d3:a7:8f:eb:fb:28: + 86:bd:ae:3e:93:9d:f6:76:32:db:05:4d:6e:34:92:16:21:85: + d3:0e:2f:73:d3:89:f7:69:6e:0c:74:35:95:8c:e3:ff:c6:f7: + 5c:b4:55:c0:49:7d:00:ae:6c:88:5b:46:31:b3:62:64:62:b0: + 0d:9c:7c:d0:84:68:d9:e3:52:ed:35:f8:5a:6f:9e:57:62:4b: + 36:8f:f7:3e:31:15:fc:bb:df:51:c4:fd:92:96:59:d5:55:3e: + 7b:fb:2f:2e:5a:15:be:ec:46:38:e0:c9:b4:46:cb:5c:71:4a: + fe:c7:19:2c:84:1c:d9:15:12:fc:40:cc:7c:fc:f4:38:22:f6: + ba:db:f2:05:97:c1:26:f5:cb:c0:ed:ba:16:e6:33:0c:37:3a: + 07:b3:a3:ca +-----BEGIN CERTIFICATE----- +MIIDgjCCAmqgAwIBAgIJAPIPxjhzA+pSMA0GCSqGSIb3DQEBCwUAMG4xEDAOBgNV +BAYTB1Vua25vd24xEDAOBgNVBAgTB1Vua25vd24xEDAOBgNVBAcTB1Vua25vd24x +EDAOBgNVBAoTB1Vua25vd24xEDAOBgNVBAsTB1Vua25vd24xEjAQBgNVBAMTCWxv +Y2FsaG9zdDAgFw0yMzAyMTMwNTUyMzJaGA8yMTIzMDEyMDA1NTIzMlowbjEQMA4G +A1UEBhMHVW5rbm93bjEQMA4GA1UECBMHVW5rbm93bjEQMA4GA1UEBxMHVW5rbm93 +bjEQMA4GA1UEChMHVW5rbm93bjEQMA4GA1UECxMHVW5rbm93bjESMBAGA1UEAxMJ +bG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6Lff7jKI +mA4HGEH92OI0yH5UhpvYo3tHbumhjHTzD7umg0MRd8Saq1N0ZEb+gY9yA5cLN3hu +/25kaOuesGuueWpQ4R4dOd9RCwSOiUvkAmvkEtNBRyyOdTAUGu0yk6b+c33c4NST +2YxEVdJTSgwu+ZVfT8sv5UHFvDOI69hSDhlVNgIxD4ELwk81Y6YG4zuTPARcYzI9 +wyZQlreMZ46e0aN+e1SGNQf+FTL7bU7nTJeVMuLYBIviAEuFZJFwriSIB0dFfRnD +wSUCUPj0CqN7ahHDb7naBqAtPWVHjijSGIpFu3l7utXYKU1O2v2yHuu2WRvxyYrq +rH1y3I3nQ33TxwIDAQABoyEwHzAdBgNVHQ4EFgQUiDpZnbJyrGT784157VT+IMOD +SSQwDQYJKoZIhvcNAQELBQADggEBAHGI68fy8oQ196u8PM6+EfnJNrUek5bPZgZK +8bb1zJe5zZMPimZihcv9wWN+ONkCDGsEOHvsguYl84yZjdEgxutedZ628OytmnYi +Qb2I6cN/PY6a8Uqz4jDK4Gg5jXzm29z9RHVmVdOnj+v7KIa9rj6TnfZ2MtsFTW40 +khYhhdMOL3PTifdpbgx0NZWM4//G91y0VcBJfQCubIhbRjGzYmRisA2cfNCEaNnj +Uu01+FpvnldiSzaP9z4xFfy731HE/ZKWWdVVPnv7Ly5aFb7sRjjgybRGy1xxSv7H +GSyEHNkVEvxAzHz89Dgi9rrb8gWXwSb1y8DtuhbmMww3Ogezo8o= +-----END CERTIFICATE----- diff --git a/tests/docker-images/java-test-functions/pom.xml b/tests/docker-images/java-test-functions/pom.xml index 0afb21c6a03a2..0d066854230f4 100644 --- a/tests/docker-images/java-test-functions/pom.xml +++ b/tests/docker-images/java-test-functions/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT 4.0.0 java-test-functions diff --git a/tests/docker-images/java-test-image/Dockerfile b/tests/docker-images/java-test-image/Dockerfile index 9e852050edf2e..805f20a0570db 100644 --- a/tests/docker-images/java-test-image/Dockerfile +++ b/tests/docker-images/java-test-image/Dockerfile @@ -17,49 +17,16 @@ # under the License. # -FROM ubuntu:20.04 +ARG PULSAR_IMAGE +FROM $PULSAR_IMAGE -RUN groupadd -g 10001 pulsar -RUN adduser -u 10000 --gid 10001 --disabled-login --disabled-password --gecos '' pulsar - -ARG PULSAR_TARBALL=target/pulsar-server-distribution-bin.tar.gz -ADD ${PULSAR_TARBALL} / -RUN mv /apache-pulsar-* /pulsar -RUN chown -R root:root /pulsar +# Base pulsar image is designed not be modified, though we need to add more scripts +USER root COPY target/scripts /pulsar/bin RUN chmod a+rx /pulsar/bin/* -WORKDIR /pulsar - -ARG DEBIAN_FRONTEND=noninteractive -ARG UBUNTU_MIRROR=mirror://mirrors.ubuntu.com/mirrors.txt -ARG UBUNTU_SECURITY_MIRROR=http://security.ubuntu.com/ubuntu/ - -RUN sed -i -e "s|http://archive\.ubuntu\.com/ubuntu/|${UBUNTU_MIRROR:-mirror://mirrors.ubuntu.com/mirrors.txt}|g" \ - -e "s|http://security\.ubuntu\.com/ubuntu/|${UBUNTU_SECURITY_MIRROR:-http://security.ubuntu.com/ubuntu/}|g" /etc/apt/sources.list \ - && apt-get update \ - && apt-get -y dist-upgrade \ - && apt-get -y install wget apt-transport-https - -# Install Eclipse Temurin Package -RUN mkdir -p /etc/apt/keyrings \ - && wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | tee /etc/apt/keyrings/adoptium.asc \ - && echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list \ - && apt-get update \ - && apt-get -y dist-upgrade \ - && apt-get -y install temurin-17-jdk \ - && export ARCH=$(uname -m | sed -r 's/aarch64/arm64/g' | awk '!/arm64/{$0="amd64"}1') \ - && echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/temurin-17-jdk-$ARCH/conf/security/java.security - -# /pulsar/bin/watch-znode.py requires python3-kazoo -# /pulsar/bin/pulsar-managed-ledger-admin requires python3-protobuf -# gen-yml-from-env.py requires python3-yaml -# make python3 the default -RUN apt-get install -y python3-kazoo python3-protobuf python3-yaml \ - && update-alternatives --install /usr/bin/python python /usr/bin/python3 10 - -RUN apt-get install -y supervisor procps curl less netcat dnsutils iputils-ping +RUN apk add --no-cache supervisor RUN mkdir -p /var/log/pulsar \ && mkdir -p /var/run/supervisor/ \ @@ -72,13 +39,3 @@ RUN mv /etc/supervisord/conf.d/supervisord.conf /etc/supervisord.conf COPY target/certificate-authority /pulsar/certificate-authority/ COPY target/java-test-functions.jar /pulsar/examples/ - -ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE - -RUN chown -R pulsar:0 /pulsar && chmod -R g=u /pulsar - -# cleanup -RUN apt-get -y --purge autoremove \ - && apt-get autoclean \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* diff --git a/tests/docker-images/java-test-image/pom.xml b/tests/docker-images/java-test-image/pom.xml index 84b02ad137cfe..ec842c9c26996 100644 --- a/tests/docker-images/java-test-image/pom.xml +++ b/tests/docker-images/java-test-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT 4.0.0 java-test-image @@ -33,11 +33,6 @@ docker - - target/pulsar-server-distribution-bin.tar.gz - ${env.UBUNTU_MIRROR} - ${env.UBUNTU_SECURITY_MIRROR} - integrationTests @@ -148,15 +143,19 @@ package build + tag ${docker.organization}/java-test-image + + ${docker.organization}/${docker.image}:${project.version}-${git.commit.id.abbrev} + ${project.basedir} - latest + ${docker.tag} ${project.version} true diff --git a/tests/docker-images/java-test-plugins/pom.xml b/tests/docker-images/java-test-plugins/pom.xml index 16b0ec1911793..63997ff5e4a49 100644 --- a/tests/docker-images/java-test-plugins/pom.xml +++ b/tests/docker-images/java-test-plugins/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT 4.0.0 java-test-plugins diff --git a/tests/docker-images/latest-version-image/Dockerfile b/tests/docker-images/latest-version-image/Dockerfile index f0093fa1eafc5..0645dd2e78aab 100644 --- a/tests/docker-images/latest-version-image/Dockerfile +++ b/tests/docker-images/latest-version-image/Dockerfile @@ -18,26 +18,10 @@ # # build go lang examples first in a separate layer +ARG PULSAR_ALL_IMAGE +ARG PULSAR_IMAGE -FROM apachepulsar/pulsar:latest as pulsar-function-go - -# Use root for builder -USER root - -RUN rm -rf /var/lib/apt/lists/* && apt-get update -RUN apt-get install -y procps curl git build-essential - -ENV GOLANG_VERSION 1.15.8 - -RUN export ARCH=$(uname -m | sed -r 's/aarch64/arm64/g' | awk '!/arm64/{$0="amd64"}1') \ - && curl -sSL https://golang.org/dl/go$GOLANG_VERSION.linux-$ARCH.tar.gz | tar -C /usr/local -xz -ENV PATH /usr/local/go/bin:$PATH - -RUN mkdir -p /go/src /go/bin && chmod -R 777 /go -ENV GOROOT /usr/local/go -ENV GOPATH /go -ENV PATH /go/bin:$PATH -ENV GO111MODULE=on +FROM golang:1.21-alpine as pulsar-function-go COPY target/pulsar-function-go/ /go/src/github.com/apache/pulsar/pulsar-function-go RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go && go install ./... @@ -45,12 +29,12 @@ RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go/pf && go install RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go/examples && go install ./... # Reference pulsar-all to copy connectors from there -FROM apachepulsar/pulsar-all:latest as pulsar-all +FROM $PULSAR_ALL_IMAGE as pulsar-all ######################################## ###### Main image build ######################################## -FROM apachepulsar/pulsar:latest +FROM $PULSAR_IMAGE # Switch to run as the root user to simplify building container and then running # supervisord. Each of the pulsar components are spawned by supervisord and their @@ -58,27 +42,19 @@ FROM apachepulsar/pulsar:latest # However, any processes exec'ing into the containers will run as root, by default. USER root -# We need to define the user in order for supervisord to work correctly -# We don't need a user defined in the public docker image, though. -RUN adduser -u 10000 --gid 0 --disabled-login --disabled-password --gecos '' pulsar - -RUN rm -rf /var/lib/apt/lists/* && apt update - -RUN apt-get clean && apt-get update && apt-get install -y supervisor vim procps curl +RUN apk add --no-cache supervisor procps curl RUN mkdir -p /var/log/pulsar && mkdir -p /var/run/supervisor/ COPY conf/supervisord.conf /etc/supervisord.conf COPY conf/global-zk.conf conf/local-zk.conf conf/bookie.conf conf/broker.conf conf/functions_worker.conf \ - conf/proxy.conf conf/presto_worker.conf conf/websocket.conf /etc/supervisord/conf.d/ + conf/proxy.conf conf/websocket.conf /etc/supervisord/conf.d/ -COPY scripts/init-cluster.sh scripts/run-global-zk.sh scripts/run-local-zk.sh \ - scripts/run-bookie.sh scripts/run-broker.sh scripts/run-functions-worker.sh scripts/run-proxy.sh scripts/run-presto-worker.sh \ +COPY scripts/run-global-zk.sh scripts/run-local-zk.sh \ + scripts/run-bookie.sh scripts/run-broker.sh scripts/run-functions-worker.sh scripts/run-proxy.sh \ scripts/run-standalone.sh scripts/run-websocket.sh \ /pulsar/bin/ -COPY conf/presto/jvm.config /pulsar/trino/conf - # copy python test examples RUN mkdir -p /pulsar/instances/deps COPY python-examples/exclamation_lib.py /pulsar/instances/deps/ @@ -87,6 +63,7 @@ COPY python-examples/exclamation.zip /pulsar/examples/python-examples/ COPY python-examples/producer_schema.py /pulsar/examples/python-examples/ COPY python-examples/consumer_schema.py /pulsar/examples/python-examples/ COPY python-examples/exception_function.py /pulsar/examples/python-examples/ +RUN chmod g+rx /pulsar/examples/python-examples/ # copy java test examples COPY target/java-test-functions.jar /pulsar/examples/ @@ -117,17 +94,20 @@ COPY --from=pulsar-all /pulsar/connectors/pulsar-io-kinesis-*.nar /pulsar/connec # download Oracle JDBC driver for Oracle Debezium Connector tests RUN mkdir -p META-INF/bundled-dependencies -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/ojdbc8/19.3.0.0/ojdbc8-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/ucp/19.3.0.0/ucp-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/oraclepki/19.3.0.0/oraclepki-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/osdt_cert/19.3.0.0/osdt_cert-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/osdt_core/19.3.0.0/osdt_core-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/simplefan/19.3.0.0/simplefan-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/orai18n/19.3.0.0/orai18n-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/xdb/19.3.0.0/xdb-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/xmlparserv2/19.3.0.0/xmlparserv2-19.3.0.0.jar +RUN cd META-INF/bundled-dependencies && curl -sSL https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/ojdbc8/19.3.0.0/ojdbc8-19.3.0.0.jar -o ojdbc8-19.3.0.0.jar +RUN cd META-INF/bundled-dependencies && curl -sSL https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/ucp/19.3.0.0/ucp-19.3.0.0.jar -o ucp-19.3.0.0.jar +RUN cd META-INF/bundled-dependencies && curl -sSL https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/oraclepki/19.3.0.0/oraclepki-19.3.0.0.jar -o oraclepki-19.3.0.0.jar +RUN cd META-INF/bundled-dependencies && curl -sSL https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/osdt_cert/19.3.0.0/osdt_cert-19.3.0.0.jar -o osdt_cert-19.3.0.0.jar +RUN cd META-INF/bundled-dependencies && curl -sSL https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/osdt_core/19.3.0.0/osdt_core-19.3.0.0.jar -o osdt_core-19.3.0.0.jar +RUN cd META-INF/bundled-dependencies && curl -sSL https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/simplefan/19.3.0.0/simplefan-19.3.0.0.jar -o simplefan-19.3.0.0.jar +RUN cd META-INF/bundled-dependencies && curl -sSL https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/orai18n/19.3.0.0/orai18n-19.3.0.0.jar -o orai18n-19.3.0.0.jar +RUN cd META-INF/bundled-dependencies && curl -sSL https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/xdb/19.3.0.0/xdb-19.3.0.0.jar -o xdb-19.3.0.0.jar +RUN cd META-INF/bundled-dependencies && curl -sSL https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/xmlparserv2/19.3.0.0/xmlparserv2-19.3.0.0.jar -o xmlparserv2-19.3.0.0.jar RUN jar uf connectors/pulsar-io-debezium-oracle-*.nar META-INF/bundled-dependencies/ojdbc8-19.3.0.0.jar META-INF/bundled-dependencies/ucp-19.3.0.0.jar META-INF/bundled-dependencies/oraclepki-19.3.0.0.jar META-INF/bundled-dependencies/osdt_cert-19.3.0.0.jar META-INF/bundled-dependencies/osdt_core-19.3.0.0.jar META-INF/bundled-dependencies/simplefan-19.3.0.0.jar META-INF/bundled-dependencies/orai18n-19.3.0.0.jar META-INF/bundled-dependencies/xdb-19.3.0.0.jar META-INF/bundled-dependencies/xmlparserv2-19.3.0.0.jar +# Fix permissions for filesystem offloader +RUN mkdir -p pulsar +RUN chmod g+rwx pulsar CMD bash diff --git a/tests/docker-images/latest-version-image/conf/bookie.conf b/tests/docker-images/latest-version-image/conf/bookie.conf index 07547bcaef6d3..df7501057a58f 100644 --- a/tests/docker-images/latest-version-image/conf/bookie.conf +++ b/tests/docker-images/latest-version-image/conf/bookie.conf @@ -22,7 +22,7 @@ autostart=false redirect_stderr=true stdout_logfile=/var/log/pulsar/bookie.log directory=/pulsar -environment=PULSAR_MEM="-Xmx128M -XX:MaxDirectMemorySize=512M",PULSAR_GC="-XX:+UseZGC" +environment=PULSAR_MEM="-Xmx128M -XX:MaxDirectMemorySize=512M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/pulsar -XX:+ExitOnOutOfMemoryError",PULSAR_GC="-XX:+UseZGC" command=/pulsar/bin/pulsar bookie user=pulsar stopwaitsecs=15 diff --git a/tests/docker-images/latest-version-image/conf/broker.conf b/tests/docker-images/latest-version-image/conf/broker.conf index 63be36437741b..790dace8d6d85 100644 --- a/tests/docker-images/latest-version-image/conf/broker.conf +++ b/tests/docker-images/latest-version-image/conf/broker.conf @@ -22,7 +22,7 @@ autostart=false redirect_stderr=true stdout_logfile=/var/log/pulsar/broker.log directory=/pulsar -environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC" +environment=PULSAR_MEM="-Xmx150M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/pulsar -XX:+ExitOnOutOfMemoryError",PULSAR_GC="-XX:+UseZGC" command=/pulsar/bin/pulsar broker user=pulsar stopwaitsecs=15 diff --git a/tests/docker-images/latest-version-image/conf/functions_worker.conf b/tests/docker-images/latest-version-image/conf/functions_worker.conf index 8072639a0d4a2..b5d151ce3f9be 100644 --- a/tests/docker-images/latest-version-image/conf/functions_worker.conf +++ b/tests/docker-images/latest-version-image/conf/functions_worker.conf @@ -22,7 +22,7 @@ autostart=false redirect_stderr=true stdout_logfile=/var/log/pulsar/functions_worker.log directory=/pulsar -environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC" +environment=PULSAR_MEM="-Xmx150M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/pulsar -XX:+ExitOnOutOfMemoryError",PULSAR_GC="-XX:+UseZGC" command=/pulsar/bin/pulsar functions-worker user=pulsar stopwaitsecs=15 \ No newline at end of file diff --git a/tests/docker-images/latest-version-image/conf/global-zk.conf b/tests/docker-images/latest-version-image/conf/global-zk.conf index e5ffd2eb9e769..ef521506846c8 100644 --- a/tests/docker-images/latest-version-image/conf/global-zk.conf +++ b/tests/docker-images/latest-version-image/conf/global-zk.conf @@ -22,7 +22,7 @@ autostart=false redirect_stderr=true stdout_logfile=/var/log/pulsar/global-zk.log directory=/pulsar -environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC" +environment=PULSAR_MEM="-Xmx128M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/pulsar -XX:+ExitOnOutOfMemoryError",PULSAR_GC="-XX:+UseZGC" command=/pulsar/bin/pulsar configuration-store user=pulsar stopwaitsecs=15 \ No newline at end of file diff --git a/tests/docker-images/latest-version-image/conf/local-zk.conf b/tests/docker-images/latest-version-image/conf/local-zk.conf index c96543db8a865..d6bfdcb621b43 100644 --- a/tests/docker-images/latest-version-image/conf/local-zk.conf +++ b/tests/docker-images/latest-version-image/conf/local-zk.conf @@ -22,7 +22,7 @@ autostart=false redirect_stderr=true stdout_logfile=/var/log/pulsar/local-zk.log directory=/pulsar -environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC" +environment=PULSAR_MEM="-Xmx128M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/pulsar -XX:+ExitOnOutOfMemoryError",PULSAR_GC="-XX:+UseZGC" command=/pulsar/bin/pulsar zookeeper user=pulsar stopwaitsecs=15 \ No newline at end of file diff --git a/tests/docker-images/latest-version-image/conf/presto/jvm.config b/tests/docker-images/latest-version-image/conf/presto/jvm.config deleted file mode 100644 index 406283fefe64d..0000000000000 --- a/tests/docker-images/latest-version-image/conf/presto/jvm.config +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - --server --Xms128M --Xmx1500M --XX:+UseZGC --XX:+UseGCOverheadLimit --XX:+ExplicitGCInvokesConcurrent --XX:+HeapDumpOnOutOfMemoryError --XX:+ExitOnOutOfMemoryError --Dpresto-temporarily-allow-java8=true --Djdk.attach.allowAttachSelf=true diff --git a/tests/docker-images/latest-version-image/conf/presto_worker.conf b/tests/docker-images/latest-version-image/conf/presto_worker.conf deleted file mode 100644 index 5a60ea550031c..0000000000000 --- a/tests/docker-images/latest-version-image/conf/presto_worker.conf +++ /dev/null @@ -1,28 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -[program:presto-worker] -autostart=false -redirect_stderr=true -stdout_logfile=/var/log/pulsar/presto_worker.log -directory=/pulsar -environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC" -command=/pulsar/bin/pulsar sql-worker start -user=pulsar -stopwaitsecs=15 \ No newline at end of file diff --git a/tests/docker-images/latest-version-image/conf/proxy.conf b/tests/docker-images/latest-version-image/conf/proxy.conf index 343a0f9614e30..17a0a658b4226 100644 --- a/tests/docker-images/latest-version-image/conf/proxy.conf +++ b/tests/docker-images/latest-version-image/conf/proxy.conf @@ -22,7 +22,7 @@ autostart=false redirect_stderr=true stdout_logfile=/var/log/pulsar/proxy.log directory=/pulsar -environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC" +environment=PULSAR_MEM="-Xmx150M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/pulsar -XX:+ExitOnOutOfMemoryError",PULSAR_GC="-XX:+UseZGC" command=/pulsar/bin/pulsar proxy user=pulsar stopwaitsecs=15 \ No newline at end of file diff --git a/tests/docker-images/latest-version-image/conf/websocket.conf b/tests/docker-images/latest-version-image/conf/websocket.conf index 0418c4cbc26a3..7625dba3e030d 100644 --- a/tests/docker-images/latest-version-image/conf/websocket.conf +++ b/tests/docker-images/latest-version-image/conf/websocket.conf @@ -22,7 +22,7 @@ autostart=false redirect_stderr=true stdout_logfile=/var/log/pulsar/pulsar-websocket.log directory=/pulsar -environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC" +environment=PULSAR_MEM="-Xmx150M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/pulsar -XX:+ExitOnOutOfMemoryError",PULSAR_GC="-XX:+UseZGC" command=/pulsar/bin/pulsar websocket user=pulsar stopwaitsecs=15 \ No newline at end of file diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml index 2aac3b14be2ea..6e1d2219cff7c 100644 --- a/tests/docker-images/latest-version-image/pom.xml +++ b/tests/docker-images/latest-version-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT 4.0.0 latest-version-image @@ -152,11 +152,15 @@ - ${docker.organization}/pulsar-test-latest-version + ${docker.organization}/${docker.image}-test-latest-version ${project.basedir} + + ${docker.organization}/${docker.image}:${project.version}-${git.commit.id.abbrev} + ${docker.organization}/${docker.image}-all:${project.version}-${git.commit.id.abbrev} + - latest + ${docker.tag} ${project.version} true diff --git a/tests/docker-images/latest-version-image/scripts/run-bookie.sh b/tests/docker-images/latest-version-image/scripts/run-bookie.sh index 64466eb2d9a54..e454e6676455b 100755 --- a/tests/docker-images/latest-version-image/scripts/run-bookie.sh +++ b/tests/docker-images/latest-version-image/scripts/run-bookie.sh @@ -29,6 +29,5 @@ if [ -z "$NO_AUTOSTART" ]; then sed -i 's/autostart=.*/autostart=true/' /etc/supervisord/conf.d/bookie.conf fi -bin/watch-znode.py -z $zkServers -p /initialized-$clusterName -w exec /usr/bin/supervisord -c /etc/supervisord.conf diff --git a/tests/docker-images/latest-version-image/scripts/run-broker.sh b/tests/docker-images/latest-version-image/scripts/run-broker.sh index 6ed5d60c39e6e..4f89f145f2b41 100755 --- a/tests/docker-images/latest-version-image/scripts/run-broker.sh +++ b/tests/docker-images/latest-version-image/scripts/run-broker.sh @@ -25,6 +25,5 @@ if [ -z "$NO_AUTOSTART" ]; then sed -i 's/autostart=.*/autostart=true/' /etc/supervisord/conf.d/broker.conf fi -bin/watch-znode.py -z $zookeeperServers -p /initialized-$clusterName -w exec /usr/bin/supervisord -c /etc/supervisord.conf diff --git a/tests/docker-images/latest-version-image/scripts/run-functions-worker.sh b/tests/docker-images/latest-version-image/scripts/run-functions-worker.sh index 3fadf960ee351..cd9d7593dbf25 100755 --- a/tests/docker-images/latest-version-image/scripts/run-functions-worker.sh +++ b/tests/docker-images/latest-version-image/scripts/run-functions-worker.sh @@ -26,6 +26,5 @@ if [ -z "$NO_AUTOSTART" ]; then sed -i 's/autostart=.*/autostart=true/' /etc/supervisord/conf.d/functions_worker.conf fi -bin/watch-znode.py -z $zookeeperServers -p /initialized-$clusterName -w exec /usr/bin/supervisord -c /etc/supervisord.conf diff --git a/tests/docker-images/latest-version-image/scripts/run-presto-worker.sh b/tests/docker-images/latest-version-image/scripts/run-presto-worker.sh deleted file mode 100755 index 8c934cbf173e5..0000000000000 --- a/tests/docker-images/latest-version-image/scripts/run-presto-worker.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -bin/apply-config-from-env-with-prefix.py SQL_PREFIX_ trino/conf/catalog/pulsar.properties && \ - bin/apply-config-from-env.py conf/pulsar_env.sh - -if [ -z "$NO_AUTOSTART" ]; then - sed -i 's/autostart=.*/autostart=true/' /etc/supervisord/conf.d/presto_worker.conf -fi - -bin/watch-znode.py -z $zookeeperServers -p /initialized-$clusterName -w -exec /usr/bin/supervisord -c /etc/supervisord.conf diff --git a/tests/docker-images/latest-version-image/scripts/run-proxy.sh b/tests/docker-images/latest-version-image/scripts/run-proxy.sh index 4836a890bda46..f44ed0bb6584b 100755 --- a/tests/docker-images/latest-version-image/scripts/run-proxy.sh +++ b/tests/docker-images/latest-version-image/scripts/run-proxy.sh @@ -25,5 +25,4 @@ if [ -z "$NO_AUTOSTART" ]; then sed -i 's/autostart=.*/autostart=true/' /etc/supervisord/conf.d/proxy.conf fi -bin/watch-znode.py -z $zookeeperServers -p /initialized-$clusterName -w exec /usr/bin/supervisord -c /etc/supervisord.conf diff --git a/tests/docker-images/latest-version-image/scripts/run-websocket.sh b/tests/docker-images/latest-version-image/scripts/run-websocket.sh index a49ee11176868..34e4b9016afd3 100755 --- a/tests/docker-images/latest-version-image/scripts/run-websocket.sh +++ b/tests/docker-images/latest-version-image/scripts/run-websocket.sh @@ -25,5 +25,4 @@ if [ -z "$NO_AUTOSTART" ]; then sed -i 's/autostart=.*/autostart=true/' /etc/supervisord/conf.d/websocket.conf fi -bin/watch-znode.py -z $zookeeperServers -p /initialized-$clusterName -w exec /usr/bin/supervisord -c /etc/supervisord.conf diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml index ce45569c2f1a9..d9c9ab366e821 100644 --- a/tests/docker-images/pom.xml +++ b/tests/docker-images/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT docker-images Apache Pulsar :: Tests :: Docker Images @@ -52,6 +52,20 @@ latest-version-image java-test-image + + + + pl.project13.maven + git-commit-id-plugin + + false + true + true + false + + + + diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index 453b67522b917..b865cfe9dd2ed 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT integration @@ -55,6 +55,13 @@ ${project.version} test + + org.apache.pulsar + pulsar-broker-common + ${project.version} + test-jar + test + org.apache.pulsar pulsar-common @@ -73,6 +80,12 @@ ${project.version} test + + org.apache.pulsar + pulsar-proxy + ${project.version} + test + org.apache.pulsar managed-ledger @@ -102,6 +115,12 @@ test + + dev.failsafe + failsafe + test + + org.testcontainers mysql @@ -126,6 +145,11 @@ docker-java-core test + + org.bouncycastle + bcpkix-jdk18on + test + org.apache.pulsar @@ -164,7 +188,6 @@ test - com.rabbitmq amqp-client @@ -179,15 +202,14 @@ - io.trino - trino-jdbc - ${trino.version} + org.awaitility + awaitility test - org.awaitility - awaitility + io.rest-assured + rest-assured test diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/auth/token/PulsarTokenAuthenticationBaseSuite.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/auth/token/PulsarTokenAuthenticationBaseSuite.java index e986c7e5c7b76..d5421267ba22f 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/auth/token/PulsarTokenAuthenticationBaseSuite.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/auth/token/PulsarTokenAuthenticationBaseSuite.java @@ -68,7 +68,7 @@ public abstract class PulsarTokenAuthenticationBaseSuite extends PulsarClusterTe protected static final String PROXY_ROLE = "proxy"; protected static final String REGULAR_USER_ROLE = "client"; - protected ZKContainer cmdContainer; + protected ZKContainer cmdContainer; @BeforeClass(alwaysRun = true) @Override @@ -76,7 +76,7 @@ public final void setupCluster() throws Exception { incrementSetupNumber(); // Before starting the cluster, generate the secret key and the token // Use Zk container to have 1 container available before starting the cluster - this.cmdContainer = new ZKContainer<>("cli-setup"); + this.cmdContainer = new ZKContainer("cli-setup"); cmdContainer .withNetwork(Network.newNetwork()) .withNetworkAliases(ZKContainer.NAME) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BookkeeperInstallWithHttpServerEnabledTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BookkeeperInstallWithHttpServerEnabledTest.java new file mode 100644 index 0000000000000..03d7f974ab39b --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BookkeeperInstallWithHttpServerEnabledTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.bookkeeper; + +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.tests.integration.docker.ContainerExecResult; +import org.apache.pulsar.tests.integration.topologies.PulsarCluster; +import org.apache.pulsar.tests.integration.topologies.PulsarClusterSpec; +import org.apache.pulsar.tests.integration.topologies.PulsarClusterTestBase; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.stream.Stream; + +import static java.util.stream.Collectors.joining; +import static org.testng.Assert.assertEquals; + +/** + * Test bookkeeper setup with http server enabled. + */ +@Slf4j +public class BookkeeperInstallWithHttpServerEnabledTest extends PulsarClusterTestBase { + + @BeforeClass(alwaysRun = true) + @Override + public final void setupCluster() throws Exception { + incrementSetupNumber(); + + final String clusterName = Stream.of(this.getClass().getSimpleName(), randomName(5)) + .filter(s -> !s.isEmpty()) + .collect(joining("-")); + bookkeeperEnvs.put("httpServerEnabled", "true"); + bookieAdditionalPorts.add(8000); + PulsarClusterSpec spec = PulsarClusterSpec.builder() + .numBookies(2) + .numBrokers(1) + .bookkeeperEnvs(bookkeeperEnvs) + .bookieAdditionalPorts(bookieAdditionalPorts) + .clusterName(clusterName) + .build(); + + log.info("Setting up cluster {} with {} bookies, {} brokers", + spec.clusterName(), spec.numBookies(), spec.numBrokers()); + + pulsarCluster = PulsarCluster.forSpec(spec); + pulsarCluster.start(); + + log.info("Cluster {} is setup", spec.clusterName()); + } + + @AfterClass(alwaysRun = true) + @Override + public final void tearDownCluster() throws Exception { + super.tearDownCluster(); + } + + @Test + public void testBookieHttpServerIsRunning() throws Exception { + ContainerExecResult result = pulsarCluster.getAnyBookie().execCmd( + PulsarCluster.CURL, + "-X", + "GET", + "http://localhost:8000/heartbeat"); + assertEquals(result.getExitCode(), 0); + assertEquals(result.getStdout(), "OK\n"); + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BrokerInstallWithEntryMetadataInterceptorsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BrokerInstallWithEntryMetadataInterceptorsTest.java new file mode 100644 index 0000000000000..dd7a7ccff78f6 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BrokerInstallWithEntryMetadataInterceptorsTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.bookkeeper; + +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.tests.integration.docker.ContainerExecResult; +import org.apache.pulsar.tests.integration.topologies.PulsarCluster; +import org.apache.pulsar.tests.integration.topologies.PulsarClusterSpec; +import org.apache.pulsar.tests.integration.topologies.PulsarClusterTestBase; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.stream.Stream; + +import static java.util.stream.Collectors.joining; +import static org.testng.AssertJUnit.assertEquals; + +/*** + * Test that verifies that regression in BookKeeper 4.16.0 is fixed. + * + * Anti-regression test for issue https://github.com/apache/pulsar/issues/20091. + */ +@Slf4j +public class BrokerInstallWithEntryMetadataInterceptorsTest extends PulsarClusterTestBase { + private static final String PREFIX = "PULSAR_PREFIX_"; + + @BeforeClass(alwaysRun = true) + @Override + public final void setupCluster() throws Exception { + incrementSetupNumber(); + + final String clusterName = Stream.of(this.getClass().getSimpleName(), randomName(5)) + .filter(s -> !s.isEmpty()) + .collect(joining("-")); + brokerEnvs.put(PREFIX + "exposingBrokerEntryMetadataToClientEnabled", "true"); + brokerEnvs.put(PREFIX + "brokerEntryMetadataInterceptors", "org.apache.pulsar.common.intercept.AppendBrokerTimestampMetadataInterceptor"); + PulsarClusterSpec spec = PulsarClusterSpec.builder() + .numBookies(2) + .numBrokers(1) + .brokerEnvs(brokerEnvs) + .clusterName(clusterName) + .build(); + + log.info("Setting up cluster {} with {} bookies, {} brokers", + spec.clusterName(), spec.numBookies(), spec.numBrokers()); + + pulsarCluster = PulsarCluster.forSpec(spec); + pulsarCluster.start(); + + log.info("Cluster {} is setup", spec.clusterName()); + } + + @AfterClass(alwaysRun = true) + @Override + public final void tearDownCluster() throws Exception { + super.tearDownCluster(); + } + + @Test + public void testBrokerHttpServerIsRunning() throws Exception { + ContainerExecResult result = pulsarCluster.getAnyBroker().execCmd( + PulsarCluster.CURL, + "-X", + "GET", + "http://localhost:8080/admin/v2/brokers/health"); + assertEquals(result.getExitCode(), 0); + assertEquals("ok", result.getStdout()); + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/AdminMultiHostTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/AdminMultiHostTest.java index a6c996a2c75b9..c9c32e689b61b 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/AdminMultiHostTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/AdminMultiHostTest.java @@ -23,6 +23,7 @@ import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import lombok.Cleanup; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.common.naming.TopicVersion; import org.apache.pulsar.tests.TestRetrySupport; @@ -62,6 +63,7 @@ public void cleanup() { @Test public void testAdminMultiHost() throws Exception { String hosts = pulsarCluster.getAllBrokersHttpServiceUrl(); + @Cleanup PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(hosts).build(); // all brokers alive Assert.assertEquals(admin.brokers().getActiveBrokers(clusterName).size(), 3); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java index 7e8f55429244b..3be15c7aee7f1 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java @@ -64,7 +64,7 @@ public void testDeprecatedCommands() throws Exception { "--allowed-clusters", pulsarCluster.getClusterName(), "--admin-roles", "admin" ); - assertTrue(result.getStderr().contains("deprecated")); + assertEquals(result.getExitCode(), 0L); result = pulsarCluster.runAdminCommandOnAnyBroker( "properties", "list"); @@ -367,7 +367,8 @@ public void testPropertiesCLI() throws Exception { "-r", ""); fail("Command should have exited with non-zero"); } catch (ContainerExecException e) { - assertEquals(e.getResult().getStderr(), "rack name is invalid, it should not be null, empty or '/'\n\n"); + assertTrue( + e.getResult().getStderr().startsWith("rack name is invalid, it should not be null, empty or '/'")); } try { @@ -377,7 +378,7 @@ public void testPropertiesCLI() throws Exception { "set-schema-autoupdate-strategy", namespace); } catch (ContainerExecException e) { - assertEquals(e.getResult().getStderr(), "Either --compatibility or --disabled must be specified\n\n"); + assertTrue(e.getResult().getStderr().startsWith("Either --compatibility or --disabled must be specified")); } } @@ -402,7 +403,7 @@ public void testSchemaCLI() throws Exception { "upload", topicName, "-f", - "/pulsar/conf/schema_example.conf" + "/pulsar/conf/schema_example.json" ); result.assertNoOutput(); @@ -445,7 +446,7 @@ public void testSchemaCLI() throws Exception { topicName); fail("Command should have exited with non-zero"); } catch (ContainerExecException e) { - assertEquals(e.getResult().getStderr(), "Invalid schema type xml. Valid options are: avro, json\n\n"); + assertTrue(e.getResult().getStderr().startsWith("Invalid schema type xml. Valid options are: avro, json")); } } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/ClientToolTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/ClientToolTest.java index c61411ad150a1..0d6b6f1abe4cf 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/ClientToolTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/ClientToolTest.java @@ -59,7 +59,7 @@ public void testProduceConsumeThroughProxy() throws Exception { private void testProduceConsume(String serviceUrl, String topicName) throws Exception { List data = randomStrings(); // Using the ZK container as it is separate from brokers, so its environment resembles real world usage more - ZKContainer clientToolContainer = pulsarCluster.getZooKeeper(); + ZKContainer clientToolContainer = pulsarCluster.getZooKeeper(); produce(clientToolContainer, serviceUrl, topicName, data); List consumed = consume(clientToolContainer, serviceUrl, topicName); assertEquals(consumed, data); @@ -86,13 +86,14 @@ private List consume(ChaosContainer container, String url, String top + "\nError output:\n" + result.getStderr()); } String output = result.getStdout(); - Pattern message = Pattern.compile("----- got message -----\nkey:\\[null\\], properties:\\[\\], content:(.*)"); + Pattern message = Pattern.compile( + "----- got message -----\npublishTime:\\[(.*)\\], eventTime:\\[(.*)\\], key:\\[null\\], " + + "properties:\\[\\], content:(.*)"); Matcher matcher = message.matcher(output); List received = new ArrayList<>(MESSAGE_COUNT); while (matcher.find()) { - received.add(matcher.group(1)); + received.add(matcher.group(3)); } return received; } - } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/ClusterMetadataTearDownTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/ClusterMetadataTearDownTest.java index fcbc27d4dedd8..491506112a364 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/ClusterMetadataTearDownTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/ClusterMetadataTearDownTest.java @@ -69,7 +69,6 @@ public class ClusterMetadataTearDownTest extends TestRetrySupport { .clusterName("ClusterMetadataTearDownTest-" + UUID.randomUUID().toString().substring(0, 8)) .numProxies(0) .numFunctionWorkers(0) - .enablePrestoWorker(false) .build(); private PulsarCluster pulsarCluster; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/HealthCheckTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/HealthCheckTest.java index 9a3559f142480..6de44574610ab 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/HealthCheckTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/HealthCheckTest.java @@ -53,7 +53,7 @@ public class HealthCheckTest extends TestRetrySupport { .clusterName("HealthCheckTest-" + UUID.randomUUID().toString().substring(0, 8)) .numProxies(0) .numFunctionWorkers(0) - .enablePrestoWorker(false).build(); + .build(); private PulsarCluster pulsarCluster = null; @@ -95,7 +95,7 @@ private void assertHealthcheckFailure() throws Exception { @Test public void testZooKeeperDown() throws Exception { - pulsarCluster.getZooKeeper().execCmd("pkill", "-STOP", "-f", "QuorumPeerMain"); + pulsarCluster.getZooKeeper().execCmd("pkill", "-STOP", "java"); assertHealthcheckFailure(); } @@ -103,7 +103,7 @@ public void testZooKeeperDown() throws Exception { // @Test // public void testBrokerDown() throws Exception { // for (BrokerContainer b : pulsarCluster.getBrokers()) { - // b.execCmd("pkill", "-STOP", "-f", "PulsarBrokerStarter"); + // b.execCmd("pkill", "-STOP", "java"); // } // assertHealthcheckFailure(); // } @@ -111,7 +111,7 @@ public void testZooKeeperDown() throws Exception { @Test public void testBookKeeperDown() throws Exception { for (BKContainer b : pulsarCluster.getBookies()) { - b.execCmd("pkill", "-STOP", "-f", "Main"); + b.execCmd("pkill", "-STOP", "java"); } assertHealthcheckFailure(); } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/PerfToolTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/PerfToolTest.java index 9a0865a0085fd..8c4f3a137aa31 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/PerfToolTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/PerfToolTest.java @@ -38,7 +38,7 @@ public void testProduce() throws Exception { String serviceUrl = "pulsar://" + pulsarCluster.getProxy().getContainerName() + ":" + PulsarContainer.BROKER_PORT; final String topicName = getNonPartitionedTopic("testProduce", true); // Using the ZK container as it is separate from brokers, so its environment resembles real world usage more - ZKContainer clientToolContainer = pulsarCluster.getZooKeeper(); + ZKContainer clientToolContainer = pulsarCluster.getZooKeeper(); ContainerExecResult produceResult = produceWithPerfTool(clientToolContainer, serviceUrl, topicName, MESSAGE_COUNT); checkOutputForLogs(produceResult,"PerformanceProducer - Aggregated throughput stats", "PerformanceProducer - Aggregated latency stats"); @@ -49,7 +49,7 @@ public void testConsume() throws Exception { String serviceUrl = "pulsar://" + pulsarCluster.getProxy().getContainerName() + ":" + PulsarContainer.BROKER_PORT; final String topicName = getNonPartitionedTopic("testConsume", true); // Using the ZK container as it is separate from brokers, so its environment resembles real world usage more - ZKContainer clientToolContainer = pulsarCluster.getZooKeeper(); + ZKContainer clientToolContainer = pulsarCluster.getZooKeeper(); ContainerExecResult consumeResult = consumeWithPerfTool(clientToolContainer, serviceUrl, topicName); checkOutputForLogs(consumeResult,"PerformanceConsumer - Aggregated throughput stats", "PerformanceConsumer - Aggregated latency stats"); @@ -60,7 +60,7 @@ public void testRead() throws Exception { String serviceUrl = "pulsar://" + pulsarCluster.getProxy().getContainerName() + ":" + PulsarContainer.BROKER_PORT; final String topicName = getNonPartitionedTopic("testRead", true); // Using the ZK container as it is separate from brokers, so its environment resembles real world usage more - ZKContainer clientToolContainer = pulsarCluster.getZooKeeper(); + ZKContainer clientToolContainer = pulsarCluster.getZooKeeper(); ContainerExecResult readResult = readWithPerfTool(clientToolContainer, serviceUrl, topicName); checkOutputForLogs(readResult,"PerformanceReader - Aggregated throughput stats ", "PerformanceReader - Aggregated latency stats"); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/BrokerContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/BrokerContainer.java index e0e85af024bc2..a51397050b97f 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/BrokerContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/BrokerContainer.java @@ -28,11 +28,20 @@ public class BrokerContainer extends PulsarContainer { public static final String NAME = "pulsar-broker"; public BrokerContainer(String clusterName, String hostName) { - super(clusterName, hostName, hostName, "bin/run-broker.sh", BROKER_PORT, BROKER_PORT_TLS, - BROKER_HTTP_PORT, BROKER_HTTPS_PORT, DEFAULT_HTTP_PATH, DEFAULT_IMAGE_NAME); + this(clusterName, hostName, false); + } + + public BrokerContainer(String clusterName, String hostName, boolean enableTls) { + super(clusterName, hostName, hostName, "bin/run-broker.sh", BROKER_PORT, + enableTls ? BROKER_PORT_TLS : 0, BROKER_HTTP_PORT, + enableTls ? BROKER_HTTPS_PORT : 0, DEFAULT_HTTP_PATH, DEFAULT_IMAGE_NAME); tailContainerLog(); } + public String getHostName() { + return super.hostname; + } + @Override protected void afterStart() { DockerUtils.runCommandAsyncWithLogging(this.dockerClient, this.getContainerId(), diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/CSContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/CSContainer.java index 3642b6f1d84e7..de67d5672330e 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/CSContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/CSContainer.java @@ -21,7 +21,7 @@ /** * A pulsar container that runs configuration store. */ -public class CSContainer extends ZKContainer { +public class CSContainer extends PulsarContainer { public static final String NAME = "configuration-store"; @@ -34,4 +34,9 @@ public CSContainer(String clusterName) { CS_PORT, INVALID_PORT); } + + @Override + protected boolean isCodeCoverageEnabled() { + return false; + } } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumMsSqlContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumMsSqlContainer.java index 357fd8724d738..61acdae37696b 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumMsSqlContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/DebeziumMsSqlContainer.java @@ -37,7 +37,7 @@ public class DebeziumMsSqlContainer extends ChaosContainer { + + private static final String IMAGE_NAME = "otel/opentelemetry-collector-contrib:latest"; + private static final String NAME = "otel-collector"; + + public static final int PROMETHEUS_EXPORTER_PORT = 8889; + private static final int OTLP_RECEIVER_PORT = 4317; + private static final int ZPAGES_PORT = 55679; + + public OpenTelemetryCollectorContainer(String clusterName) { + super(clusterName, IMAGE_NAME); + } + + @Override + protected void configure() { + super.configure(); + + this.withCopyFileToContainer( + MountableFile.forClasspathResource("containers/otel-collector-config.yaml", 0644), + "/etc/otel-collector-config.yaml") + .withCommand("--config=/etc/otel-collector-config.yaml") + .withExposedPorts(OTLP_RECEIVER_PORT, PROMETHEUS_EXPORTER_PORT, ZPAGES_PORT) + .waitingFor(new HttpWaitStrategy() + .forPath("/debug/servicez") + .forPort(ZPAGES_PORT) + .forStatusCode(HttpStatus.SC_OK) + .withStartupTimeout(Duration.ofSeconds(300))); + } + + @Override + public String getContainerName() { + return clusterName + "-" + NAME; + } + + public String getOtlpEndpoint() { + return String.format("http://%s:%d", NAME, OTLP_RECEIVER_PORT); + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PrestoWorkerContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PrestoWorkerContainer.java deleted file mode 100644 index 717dbab6e2be6..0000000000000 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PrestoWorkerContainer.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.tests.integration.containers; - -import org.apache.pulsar.tests.integration.utils.DockerUtils; - -/** - * A pulsar container that runs the presto worker - */ -public class PrestoWorkerContainer extends PulsarContainer { - - public static final String NAME = "presto-worker"; - public static final int PRESTO_HTTP_PORT = 8081; - - public PrestoWorkerContainer(String clusterName, String hostname) { - super( - clusterName, - hostname, - hostname, - "bin/run-presto-worker.sh", - -1, - PRESTO_HTTP_PORT, - "/v1/info/state"); - tailContainerLog(); - } - - @Override - protected void afterStart() { - DockerUtils.runCommandAsyncWithLogging(this.dockerClient, this.getContainerId(), - "tail", "-f", "/pulsar/trino/var/log/launcher.log"); - DockerUtils.runCommandAsyncWithLogging(this.dockerClient, this.getContainerId(), - "tail", "-f", "/var/log/pulsar/presto_worker.log"); - DockerUtils.runCommandAsyncWithLogging(this.dockerClient, this.getContainerId(), - "tail", "-f", "/pulsar/trino/var/log/server.log"); - } - - @Override - protected void beforeStop() { - super.beforeStop(); - if (null != getContainerId()) { - DockerUtils.dumpContainerDirToTargetCompressed( - getDockerClient(), - getContainerId(), - "/pulsar/trino/var/log" - ); - } - } - - public String getUrl() { - return String.format("%s:%s", getHost(), getMappedPort(PrestoWorkerContainer.PRESTO_HTTP_PORT)); - } -} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ProxyContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ProxyContainer.java index 53283447378f5..f3926878f37c5 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ProxyContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ProxyContainer.java @@ -28,8 +28,13 @@ public class ProxyContainer extends PulsarContainer { public static final String NAME = "pulsar-proxy"; public ProxyContainer(String clusterName, String hostName) { - super(clusterName, hostName, hostName, "bin/run-proxy.sh", BROKER_PORT, BROKER_PORT_TLS, BROKER_HTTP_PORT, - BROKER_HTTPS_PORT, DEFAULT_HTTP_PATH, DEFAULT_IMAGE_NAME); + this(clusterName, hostName, false); + } + + public ProxyContainer(String clusterName, String hostName, boolean enableTls) { + super(clusterName, hostName, hostName, "bin/run-proxy.sh", BROKER_PORT, + enableTls ? BROKER_PORT_TLS : 0, BROKER_HTTP_PORT, + enableTls ? BROKER_HTTPS_PORT : 0, DEFAULT_HTTP_PATH, DEFAULT_IMAGE_NAME); } @Override diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PulsarContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PulsarContainer.java index dc11acd00c3f2..77cdc1bfd28a9 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PulsarContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PulsarContainer.java @@ -26,6 +26,7 @@ import java.time.Duration; import java.util.Objects; import java.util.UUID; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.apache.pulsar.tests.integration.docker.ContainerExecResult; @@ -70,7 +71,8 @@ public abstract class PulsarContainer> exte public static final boolean PULSAR_CONTAINERS_LEAVE_RUNNING = Boolean.parseBoolean(System.getenv("PULSAR_CONTAINERS_LEAVE_RUNNING")); - private final String hostname; + @Getter + protected final String hostname; private final String serviceName; private final String serviceEntryPoint; private final int servicePort; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PulsarInitMetadataContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PulsarInitMetadataContainer.java new file mode 100644 index 0000000000000..4251ed3bd57ac --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PulsarInitMetadataContainer.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.tests.integration.containers; + +import java.io.IOException; +import lombok.extern.slf4j.Slf4j; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; + +/** + * Initialize the Pulsar metadata + */ +@Slf4j +public class PulsarInitMetadataContainer extends GenericContainer { + + public static final String NAME = "init-metadata"; + + private final String clusterName; + private final String metadataStoreUrl; + private final String configurationMetadataStoreUrl; + private final String brokerHostname; + + public PulsarInitMetadataContainer(Network network, + String clusterName, + String metadataStoreUrl, + String configurationMetadataStoreUrl, + String brokerHostname) { + this.clusterName = clusterName; + this.metadataStoreUrl = metadataStoreUrl; + this.configurationMetadataStoreUrl = configurationMetadataStoreUrl; + this.brokerHostname = brokerHostname; + setDockerImageName(PulsarContainer.DEFAULT_IMAGE_NAME); + withNetwork(network); + + setCommand("sleep 1000000"); + } + + + public void initialize() throws Exception { + start(); + ExecResult res = this.execInContainer( + "/pulsar/bin/pulsar", "initialize-cluster-metadata", + "--cluster", clusterName, + "--metadata-store", metadataStoreUrl, + "--configuration-metadata-store", configurationMetadataStoreUrl, + "--web-service-url", "http://" + brokerHostname + ":8080/", + "--broker-service-url", "pulsar://" + brokerHostname + ":6650/" + ); + + if (res.getExitCode() == 0) { + log.info("Successfully initialized cluster"); + } else { + log.warn("Failed to initialize Pulsar cluster. exit code: " + res.getExitCode()); + log.warn("STDOUT: " + res.getStdout()); + log.warn("STDERR: " + res.getStderr()); + throw new IOException("Failed to initialized Pulsar Cluster"); + } + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ZKContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ZKContainer.java index da3fb05a518f4..c55eb3242b46a 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ZKContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ZKContainer.java @@ -18,17 +18,13 @@ */ package org.apache.pulsar.tests.integration.containers; -import org.apache.pulsar.tests.integration.utils.DockerUtils; - /** * A pulsar container that runs zookeeper. */ -public class ZKContainer> extends PulsarContainer { +public class ZKContainer extends PulsarContainer { public static final String NAME = "zookeeper"; - private volatile boolean dumpZkDataBeforeStop = false; - public ZKContainer(String clusterName) { super( clusterName, @@ -39,37 +35,6 @@ public ZKContainer(String clusterName) { INVALID_PORT); } - public ZKContainer(String clusterName, - String hostname, - String serviceName, - String serviceEntryPoint, - int servicePort, - int httpPort) { - super( - clusterName, - hostname, - serviceName, - serviceEntryPoint, - servicePort, - httpPort); - } - - public void enableDumpZkDataBeforeStop(boolean enabled) { - this.dumpZkDataBeforeStop = enabled; - } - - @Override - protected void beforeStop() { - super.beforeStop(); - if (null != getContainerId() && dumpZkDataBeforeStop) { - DockerUtils.dumpContainerDirToTargetCompressed( - getDockerClient(), - getContainerId(), - "/pulsar/data/zookeeper" - ); - } - } - @Override protected boolean isCodeCoverageEnabled() { return false; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index 1e54764ad5d2b..b78a832f60933 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -27,7 +27,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.swagger.util.Json; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -63,10 +62,12 @@ import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord; import org.apache.pulsar.common.functions.ConsumerConfig; +import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.policies.data.FunctionStatsImpl; import org.apache.pulsar.common.policies.data.FunctionStatus; import org.apache.pulsar.common.policies.data.FunctionStatusUtil; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; +import org.apache.pulsar.common.policies.data.SubscriptionStats; import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.KeyValueEncodingType; @@ -380,10 +381,10 @@ protected void testFunctionNegAck(Runtime runtime) throws Exception { if (runtime == Runtime.PYTHON) { submitFunction( runtime, inputTopicName, outputTopicName, functionName, EXCEPTION_FUNCTION_PYTHON_FILE, - EXCEPTION_PYTHON_CLASS, schema); + EXCEPTION_PYTHON_CLASS, schema, null); } else { submitFunction( - runtime, inputTopicName, outputTopicName, functionName, null, EXCEPTION_JAVA_CLASS, schema); + runtime, inputTopicName, outputTopicName, functionName, null, EXCEPTION_JAVA_CLASS, schema, null); } // get function info @@ -564,7 +565,7 @@ protected void testPublishFunction(Runtime runtime) throws Exception { PUBLISH_JAVA_CLASS, schema, Collections.singletonMap("publish-topic", outputTopicName), - null, null, null, null, null); + null, null, null, null, null, null); break; case PYTHON: ConsumerConfig consumerConfig = new ConsumerConfig(); @@ -581,7 +582,7 @@ protected void testPublishFunction(Runtime runtime) throws Exception { PUBLISH_PYTHON_CLASS, schema, Collections.singletonMap("publish-topic", outputTopicName), - objectMapper.writeValueAsString(inputSpecs), "string", null, null, null); + objectMapper.writeValueAsString(inputSpecs), "string", null, null, null, null); break; case GO: submitFunction( @@ -593,7 +594,7 @@ protected void testPublishFunction(Runtime runtime) throws Exception { null, schema, Collections.singletonMap("publish-topic", outputTopicName), - null, null, null, null, null); + null, null, null, null, null, null); } // get function info @@ -668,6 +669,15 @@ protected void testExclamationFunction(Runtime runtime, boolean pyZip, boolean multipleInput, boolean withExtraDeps) throws Exception { + testExclamationFunction(runtime, isTopicPattern, pyZip, multipleInput, withExtraDeps, null); + } + + protected void testExclamationFunction(Runtime runtime, + boolean isTopicPattern, + boolean pyZip, + boolean multipleInput, + boolean withExtraDeps, + java.util.function.Consumer commandGeneratorConsumer) throws Exception { if (functionRuntimeType == FunctionRuntimeType.THREAD && (runtime == Runtime.PYTHON || runtime == Runtime.GO)) { // python&go can only run on process mode return; @@ -697,10 +707,10 @@ protected void testExclamationFunction(Runtime runtime, // submit the exclamation function submitExclamationFunction( - runtime, inputTopicName, outputTopicName, functionName, pyZip, withExtraDeps, schema); + runtime, inputTopicName, outputTopicName, functionName, pyZip, withExtraDeps, schema, commandGeneratorConsumer); // get function info - getFunctionInfoSuccess(functionName); + final String info = getFunctionInfoSuccess(functionName); // get function stats getFunctionStatsEmpty(functionName); @@ -729,6 +739,22 @@ protected void testExclamationFunction(Runtime runtime, //get function status getFunctionStatus(functionName, 0, true, 2); + // update code file + switch (runtime) { + case JAVA: + updateFunctionCodeFile(functionName, Runtime.JAVA, "test"); + break; + case PYTHON: + updateFunctionCodeFile(functionName, Runtime.PYTHON, EXCLAMATION_PYTHON_FILE); + break; + case GO: + updateFunctionCodeFile(functionName, Runtime.GO, EXCLAMATION_GO_FILE); + break; + } + + checkSubscriptionType(inputTopicName, + ObjectMapperFactory.getMapper().getObjectMapper().readValue(info, FunctionConfig.class)); + // delete function deleteFunction(functionName); @@ -740,6 +766,41 @@ protected void testExclamationFunction(Runtime runtime, } + private void checkSubscriptionType(String topic, FunctionConfig config) { + List topics = new ArrayList<>(); + if (topic.endsWith(".*")) { + topics.add(topic.substring(0, topic.length() - 2) + "1"); + topics.add(topic.substring(0, topic.length() - 2) + "2"); + } else if (topic.contains(",")) { + topics.addAll(Arrays.asList(topic.split(","))); + } else { + topics.add(topic); + } + topics.stream().forEach(t -> { + try { + ContainerExecResult result = pulsarCluster.getAnyBroker().execCmd( + PulsarCluster.ADMIN_SCRIPT, + "topics", + "stats", + t); + TopicStats topicStats = ObjectMapperFactory.getMapper().reader() + .readValue(result.getStdout(), TopicStats.class); + assertEquals(topicStats.getSubscriptions().size(), 1); + final SubscriptionStats sub = topicStats.getSubscriptions().values().iterator() + .next(); + if (config.getRetainOrdering()) { + assertEquals(sub.getType(), "Failover"); + } else if (config.getRetainKeyOrdering()) { + assertEquals(sub.getType(), "Key_Shared"); + } else { + assertEquals(sub.getType(), "Shared"); + } + } catch (Exception e) { + fail("Command should have exited with non-zero"); + } + }); + } + private void submitExclamationFunction(Runtime runtime, String inputTopicName, String outputTopicName, @@ -747,6 +808,18 @@ private void submitExclamationFunction(Runtime runtime, boolean pyZip, boolean withExtraDeps, Schema schema) throws Exception { + submitExclamationFunction(runtime, inputTopicName, outputTopicName, functionName, pyZip, + withExtraDeps, schema, null); + } + + private void submitExclamationFunction(Runtime runtime, + String inputTopicName, + String outputTopicName, + String functionName, + boolean pyZip, + boolean withExtraDeps, + Schema schema, + java.util.function.Consumer commandGeneratorConsumer) throws Exception { submitFunction( runtime, inputTopicName, @@ -756,7 +829,8 @@ private void submitExclamationFunction(Runtime runtime, withExtraDeps, false, getExclamationClass(runtime, pyZip, withExtraDeps), - schema); + schema, + commandGeneratorConsumer); } private void submitFunction(Runtime runtime, @@ -767,7 +841,8 @@ private void submitFunction(Runtime runtime, boolean withExtraDeps, boolean isPublishFunction, String functionClass, - Schema inputTopicSchema) throws Exception { + Schema inputTopicSchema, + java.util.function.Consumer commandGeneratorConsumer) throws Exception { String file = null; if (Runtime.JAVA == runtime) { @@ -790,7 +865,8 @@ private void submitFunction(Runtime runtime, } } - submitFunction(runtime, inputTopicName, outputTopicName, functionName, file, functionClass, inputTopicSchema); + submitFunction(runtime, inputTopicName, outputTopicName, functionName, file, functionClass, inputTopicSchema, + commandGeneratorConsumer); } private void submitFunction(Runtime runtime, @@ -799,9 +875,11 @@ private void submitFunction(Runtime runtime, String functionName, String functionFile, String functionClass, - Schema inputTopicSchema) throws Exception { + Schema inputTopicSchema, + java.util.function.Consumer commandGeneratorConsumer) throws Exception { submitFunction(runtime, inputTopicName, outputTopicName, functionName, functionFile, functionClass, - inputTopicSchema, null, null, null, null, null, null); + inputTopicSchema, null, null, null, null, null, null, + commandGeneratorConsumer); } private void submitFunction(Runtime runtime, @@ -816,7 +894,8 @@ private void submitFunction(Runtime runtime, String outputSchemaType, SubscriptionInitialPosition subscriptionInitialPosition, String inputTypeClassName, - String outputTypeClassName) throws Exception { + String outputTypeClassName, + java.util.function.Consumer commandGeneratorConsumer) throws Exception { if (StringUtils.isNotEmpty(inputTopicName)) { ensureSubscriptionCreated( @@ -852,6 +931,9 @@ private void submitFunction(Runtime runtime, if (outputTypeClassName != null) { generator.setOutputTypeClassName(outputTypeClassName); } + if (commandGeneratorConsumer != null) { + commandGeneratorConsumer.accept(generator); + } String command = ""; switch (runtime) { @@ -894,6 +976,22 @@ private void updateFunctionParallelism(String functionName, int parallelism) thr assertTrue(result.getStdout().contains("Updated successfully")); } + private void updateFunctionCodeFile(String functionName, Runtime runtime, String codeFile) throws Exception { + + CommandGenerator generator = new CommandGenerator(); + generator.setFunctionName(functionName); + generator.setRuntime(runtime); + String command = generator.generateUpdateFunctionCommand(codeFile); + + log.info("---------- Function command: {}", command); + String[] commands = { + "sh", "-c", command + }; + ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd( + commands); + assertTrue(result.getStdout().contains("Updated successfully")); + } + protected void submitFunction(Runtime runtime, String inputTopicName, String outputTopicName, @@ -966,7 +1064,7 @@ private void ensureSubscriptionCreated(String inputTopicName, } } - protected void getFunctionInfoSuccess(String functionName) throws Exception { + protected String getFunctionInfoSuccess(String functionName) throws Exception { ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd( PulsarCluster.ADMIN_SCRIPT, "functions", @@ -978,8 +1076,10 @@ protected void getFunctionInfoSuccess(String functionName) throws Exception { log.info("FUNCTION STATE: {}", result.getStdout()); assertTrue(result.getStdout().contains("\"name\": \"" + functionName + "\"")); + return result.getStdout(); } + protected void getFunctionStatsEmpty(String functionName) throws Exception { ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd( PulsarCluster.ADMIN_SCRIPT, @@ -1229,7 +1329,8 @@ private void publishAndConsumeMessages(String inputTopic, } for (int i = 0; i < numMessages; i++) { - Message msg = consumer.receive(30, TimeUnit.SECONDS); + log.info("Trying to receive message.. {}/{}", i, numMessages); + Message msg = consumer.receive(30, TimeUnit.MINUTES); log.info("Received: {}", msg.getValue()); assertTrue(expectedMessages.contains(msg.getValue())); expectedMessages.remove(msg.getValue()); @@ -1336,7 +1437,8 @@ protected void testAutoSchemaFunction() throws Exception { false, false, AutoSchemaFunction.class.getName(), - Schema.AVRO(CustomObject.class)); + Schema.AVRO(CustomObject.class), + null); // get function info getFunctionInfoSuccess(functionName); @@ -1446,7 +1548,8 @@ protected void testAvroSchemaFunction(Runtime runtime) throws Exception { functionName, null, AvroSchemaTestFunction.class.getName(), - Schema.AVRO(AvroTestObject.class)); + Schema.AVRO(AvroTestObject.class), + null); } else if (runtime == Runtime.PYTHON) { ConsumerConfig consumerConfig = new ConsumerConfig(); consumerConfig.setSchemaType("avro"); @@ -1462,7 +1565,8 @@ protected void testAvroSchemaFunction(Runtime runtime) throws Exception { AVRO_SCHEMA_PYTHON_CLASS, Schema.AVRO(AvroTestObject.class), null, objectMapper.writeValueAsString(inputSpecs), "avro", null, - "avro_schema_test_function.AvroTestObject", "avro_schema_test_function.AvroTestObject"); + "avro_schema_test_function.AvroTestObject", "avro_schema_test_function.AvroTestObject", + null); } log.info("pulsar submitFunction"); @@ -1539,7 +1643,7 @@ protected void testInitFunction(Runtime runtime) throws Exception { // submit the exclamation function submitFunction(runtime, inputTopicName, outputTopicName, functionName, null, InitializableFunction.class.getName(), schema, - Collections.singletonMap("publish-topic", outputTopicName), null, null, null, null, null); + Collections.singletonMap("publish-topic", outputTopicName), null, null, null, null, null, null); // publish and consume result publishAndConsumeMessages(inputTopicName, outputTopicName, numMessages); @@ -1672,13 +1776,17 @@ private void publishAndConsumeMessages(String inputTopic, Message msg = consumer.receive(30, TimeUnit.SECONDS); if (msg == null) { log.info("Input topic stats: {}", - Json.pretty(pulsarAdmin.topics().getStats(inputTopic, true))); + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString( + pulsarAdmin.topics().getStats(inputTopic, true))); log.info("Output topic stats: {}", - Json.pretty(pulsarAdmin.topics().getStats(outputTopic, true))); + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString( + pulsarAdmin.topics().getStats(outputTopic, true))); log.info("Input topic internal-stats: {}", - Json.pretty(pulsarAdmin.topics().getInternalStats(inputTopic, true))); + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString( + pulsarAdmin.topics().getInternalStats(inputTopic, true))); log.info("Output topic internal-stats: {}", - Json.pretty(pulsarAdmin.topics().getInternalStats(outputTopic, true))); + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString( + pulsarAdmin.topics().getInternalStats(outputTopic, true))); } else { String logMsg = new String(msg.getValue(), UTF_8); log.info("Received message: '{}'", logMsg); @@ -1728,7 +1836,7 @@ protected void testGenericObjectFunction(String function, boolean removeAgeField null, null, SchemaType.NONE.name(), - SubscriptionInitialPosition.Earliest, null, null); + SubscriptionInitialPosition.Earliest, null, null, null); try { if (keyValue) { @Cleanup @@ -1844,7 +1952,8 @@ protected void testRecordFunction() throws Exception { functionName, null, RecordFunction.class.getName(), - Schema.AUTO_CONSUME()); + Schema.AUTO_CONSUME(), + null); try { @Cleanup Producer producer = pulsarClient @@ -1914,7 +2023,7 @@ protected void testMergeFunction() throws Exception { null, inputSpecNode.toString(), SchemaType.AUTO_PUBLISH.name().toUpperCase(), - SubscriptionInitialPosition.Earliest, null, null); + SubscriptionInitialPosition.Earliest, null, null, null); getFunctionInfoSuccess(functionName); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarStateTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarStateTest.java index 45f4ab5343557..856e4edfea023 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarStateTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarStateTest.java @@ -25,13 +25,16 @@ import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; import static org.testng.Assert.fail; import com.google.common.base.Utf8; +import com.google.gson.Gson; import java.util.Base64; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Producer; @@ -68,15 +71,22 @@ public class PulsarStateTest extends PulsarStandaloneTestSuite { @Test(groups = {"python_state", "state", "function", "python_function"}) public void testPythonWordCountFunction() throws Exception { + String functionName = "test-wordcount-py-fn-" + randomName(8); + doTestPythonWordCountFunction(functionName); + + // after a function is deleted, its state should be clean + // we just recreate and test the word count function again, and it should have same result + doTestPythonWordCountFunction(functionName); + } + + private void doTestPythonWordCountFunction(String functionName) throws Exception { String inputTopicName = "test-wordcount-py-input-" + randomName(8); String outputTopicName = "test-wordcount-py-output-" + randomName(8); - String functionName = "test-wordcount-py-fn-" + randomName(8); final int numMessages = 10; - // submit the exclamation function submitExclamationFunction( - Runtime.PYTHON, inputTopicName, outputTopicName, functionName); + Runtime.PYTHON, inputTopicName, outputTopicName, functionName); // get function info getFunctionInfoSuccess(functionName); @@ -88,12 +98,35 @@ public void testPythonWordCountFunction() throws Exception { getFunctionStatus(functionName, numMessages); // get state - queryState(functionName, "hello", numMessages); - queryState(functionName, "test", numMessages); + queryState(functionName, "hello", numMessages, numMessages - 1); + queryState(functionName, "test", numMessages, numMessages - 1); for (int i = 0; i < numMessages; i++) { - queryState(functionName, "message-" + i, 1); + queryState(functionName, "message-" + i, 1, 0); } + // test put state + String state = "{\"key\":\"test-string\",\"stringValue\":\"test value\"}"; + String expect = "\"stringValue\": \"test value\""; + putAndQueryState(functionName, "test-string", state, expect); + + String numberState = "{\"key\":\"test-number\",\"numberValue\":20}"; + String expectNumber = "\"numberValue\": 20"; + putAndQueryState(functionName, "test-number", numberState, expectNumber); + + byte[] valueBytes = Base64.getDecoder().decode(VALUE_BASE64); + String bytesString = Base64.getEncoder().encodeToString(valueBytes); + String byteState = "{\"key\":\"test-bytes\",\"byteValue\":\"" + bytesString + "\"}"; + putAndQueryStateByte(functionName, "test-bytes", byteState, valueBytes); + + String valueStr = "hello pulsar"; + byte[] valueStrBytes = valueStr.getBytes(UTF_8); + String bytesStrString = Base64.getEncoder().encodeToString(valueStrBytes); + String byteStrState = "{\"key\":\"test-str-bytes\",\"byteValue\":\"" + bytesStrString + "\"}"; + putAndQueryState(functionName, "test-str-bytes", byteStrState, valueStr); + + String byteStrStateWithEmptyValues = "{\"key\":\"test-str-bytes\",\"byteValue\":\"" + bytesStrString + "\",\"stringValue\":\"\",\"numberValue\":0}"; + putAndQueryState(functionName, "test-str-bytes", byteStrStateWithEmptyValues, valueStr); + // delete function deleteFunction(functionName); @@ -128,6 +161,22 @@ public void testSourceState() throws Exception { assertEquals(functionState.getStringValue(), "val1"); } + // query a non-exist key should get a 404 error + { + PulsarAdminException e = expectThrows(PulsarAdminException.class, () -> { + admin.functions().getFunctionState("public", "default", sourceName, "non-exist"); + }); + assertEquals(e.getStatusCode(), 404); + } + + // query a non-exist instance should get a 404 error + { + PulsarAdminException e = expectThrows(PulsarAdminException.class, () -> { + admin.functions().getFunctionState("public", "default", "non-exist", "non-exist"); + }); + assertEquals(e.getStatusCode(), 404); + } + Awaitility.await().ignoreExceptions().untilAsserted(() -> { FunctionState functionState = admin.functions().getFunctionState("public", "default", sourceName, "now"); assertTrue(functionState.getStringValue().matches("val1-.*")); @@ -170,6 +219,22 @@ public void testSinkState() throws Exception { assertEquals(functionState.getStringValue(), "val1"); } + // query a non-exist key should get a 404 error + { + PulsarAdminException e = expectThrows(PulsarAdminException.class, () -> { + admin.functions().getFunctionState("public", "default", sinkName, "non-exist"); + }); + assertEquals(e.getStatusCode(), 404); + } + + // query a non-exist instance should get a 404 error + { + PulsarAdminException e = expectThrows(PulsarAdminException.class, () -> { + admin.functions().getFunctionState("public", "default", "non-exist", "non-exist"); + }); + assertEquals(e.getStatusCode(), 404); + } + for (int i = 0; i < numMessages; i++) { producer.send("foo"); } @@ -192,6 +257,20 @@ public void testSinkState() throws Exception { getSinkInfoNotFound(sinkName); } + @Test(groups = {"python_state", "state", "function", "python_function"}) + public void testNonExistFunction() throws Exception { + String functionName = "non-exist-function-" + randomName(8); + try (PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(container.getHttpServiceUrl()).build()) { + // query a non-exist instance should get a 404 error + { + PulsarAdminException e = expectThrows(PulsarAdminException.class, () -> { + admin.functions().getFunctionState("public", "default", functionName, "non-exist"); + }); + assertEquals(e.getStatusCode(), 404); + } + } + } + @Test(groups = {"java_state", "state", "function", "java_function"}) public void testBytes2StringNotUTF8() { byte[] valueBytes = Base64.getDecoder().decode(VALUE_BASE64); @@ -434,7 +513,7 @@ private void getFunctionStatus(String functionName, int numMessages) throws Exce assertTrue(result.getStdout().contains("\"numSuccessfullyProcessed\" : " + numMessages)); } - private void queryState(String functionName, String key, int amount) + private void queryState(String functionName, String key, int amount, long version) throws Exception { ContainerExecResult result = container.execCmd( PulsarCluster.ADMIN_SCRIPT, @@ -446,6 +525,60 @@ private void queryState(String functionName, String key, int amount) "--key", key ); assertTrue(result.getStdout().contains("\"numberValue\": " + amount)); + assertTrue(result.getStdout().contains("\"version\": " + version)); + assertFalse(result.getStdout().contains("stringValue")); + assertFalse(result.getStdout().contains("byteValue")); + } + + private void putAndQueryState(String functionName, String key, String state, String expect) + throws Exception { + container.execCmd( + PulsarCluster.ADMIN_SCRIPT, + "functions", + "putstate", + "--tenant", "public", + "--namespace", "default", + "--name", functionName, + "--state", state + ); + + ContainerExecResult result = container.execCmd( + PulsarCluster.ADMIN_SCRIPT, + "functions", + "querystate", + "--tenant", "public", + "--namespace", "default", + "--name", functionName, + "--key", key + ); + assertTrue(result.getStdout().contains(expect)); + } + + private void putAndQueryStateByte(String functionName, String key, String state, byte[] expect) + throws Exception { + container.execCmd( + PulsarCluster.ADMIN_SCRIPT, + "functions", + "putstate", + "--tenant", "public", + "--namespace", "default", + "--name", functionName, + "--state", state + ); + + ContainerExecResult result = container.execCmd( + PulsarCluster.ADMIN_SCRIPT, + "functions", + "querystate", + "--tenant", "public", + "--namespace", "default", + "--name", functionName, + "--key", key + ); + + FunctionState byteState = new Gson().fromJson(result.getStdout(), FunctionState.class); + assertNull(byteState.getStringValue()); + assertEquals(byteState.getByteValue(), expect); } private void publishAndConsumeMessages(String inputTopic, diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java index 87a52d27e8927..9ba210b998877 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java @@ -69,4 +69,21 @@ public void testAvroSchemaFunctionTest() throws Exception { testAvroSchemaFunction(Runtime.PYTHON); } + @Test(groups = {"python_function", "function"}) + public void testRetainOrderingTest() throws Exception { + testExclamationFunction(Runtime.PYTHON, false, false, false, + false, generator -> { + generator.setRetainOrdering(true); + }); + } + + @Test(groups = {"python_function", "function"}) + public void testRetainKeyOrderingTest() throws Exception { + testExclamationFunction(Runtime.PYTHON, false, false, false, + false, generator -> { + System.out.println("calling generator.setRetainKeyOrdering(true);"); + generator.setRetainKeyOrdering(true); + }); + } + } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java index adc791fab4dc8..e0fbd6040076d 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java @@ -64,6 +64,8 @@ public enum Runtime { private String outputTypeClassName; private String schemaType; private SubscriptionInitialPosition subscriptionInitialPosition; + private Boolean retainOrdering; + private Boolean retainKeyOrdering; private Map userConfig = new HashMap<>(); public static final String JAVAJAR = "/pulsar/examples/java-test-functions.jar"; @@ -227,6 +229,12 @@ public String generateCreateFunctionCommand(String codeFile) { if (subscriptionInitialPosition != null) { commandBuilder.append(" --subs-position " + subscriptionInitialPosition.name()); } + if (retainOrdering != null) { + commandBuilder.append(" --retain-ordering "); + } + if (retainKeyOrdering != null) { + commandBuilder.append(" --retain-key-ordering "); + } switch (runtime){ case JAVA: diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarGenericObjectSinkTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarGenericObjectSinkTest.java index 194da6cfa98f5..a53f858cc310f 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarGenericObjectSinkTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarGenericObjectSinkTest.java @@ -35,7 +35,6 @@ import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.tests.integration.docker.ContainerExecException; import org.apache.pulsar.tests.integration.docker.ContainerExecResult; -import org.apache.pulsar.tests.integration.presto.StockProtoMessage; import org.apache.pulsar.tests.integration.suites.PulsarStandaloneTestSuite; import org.apache.pulsar.tests.integration.topologies.PulsarCluster; import org.testng.annotations.Test; @@ -110,9 +109,7 @@ public void testGenericObjectSink() throws Exception { new SinkSpec<>("test-kv-sink-input-kv-avro-json-inl-" + randomName(8), Schema.KeyValue(Schema.AVRO(PojoKey.class), Schema.JSON(Pojo.class), KeyValueEncodingType.INLINE), new KeyValue<>(PojoKey.builder().field1("a").build(), Pojo.builder().field1("a").field2(2).build())), new SinkSpec("test-kv-sink-input-kv-avro-json-sep-" + randomName(8), - Schema.KeyValue(Schema.AVRO(PojoKey.class), Schema.JSON(Pojo.class), KeyValueEncodingType.SEPARATED), new KeyValue<>(PojoKey.builder().field1("a").build(), Pojo.builder().field1("a").field2(2).build())), - new SinkSpec("test-kv-sink-input-protobuf-native-" + randomName(8), - Schema.PROTOBUF_NATIVE(StockProtoMessage.Stock.class), StockProtoMessage.Stock.newBuilder().setEntryId(1).setSymbol("s").setSharePrice(0.0).build()) + Schema.KeyValue(Schema.AVRO(PojoKey.class), Schema.JSON(Pojo.class), KeyValueEncodingType.SEPARATED), new KeyValue<>(PojoKey.builder().field1("a").build(), Pojo.builder().field1("a").field2(2).build())) ); final int numRecordsPerTopic = 2; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarIOTestRunner.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarIOTestRunner.java index 4492f6a407520..7c47a0dcff89b 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarIOTestRunner.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarIOTestRunner.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.tests.integration.io; +import dev.failsafe.RetryPolicy; import java.time.Duration; import java.util.LinkedHashMap; import java.util.Map; @@ -33,7 +34,6 @@ import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; -import net.jodah.failsafe.RetryPolicy; @Slf4j public abstract class PulsarIOTestRunner { @@ -42,10 +42,11 @@ public abstract class PulsarIOTestRunner { final Duration ONE_MINUTE = Duration.ofMinutes(1); final Duration TEN_SECONDS = Duration.ofSeconds(10); - protected final RetryPolicy statusRetryPolicy = new RetryPolicy() + protected final RetryPolicy statusRetryPolicy = RetryPolicy.builder() .withMaxDuration(ONE_MINUTE) .withDelay(TEN_SECONDS) - .onRetry(e -> log.error("Retry ... ")); + .onRetry(e -> log.error("Retry ... ")) + .build(); protected PulsarCluster pulsarCluster; protected String functionRuntimeType; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearch7SinkTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearch7SinkTester.java index 65b38c677bfc5..d99fcad252706 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearch7SinkTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearch7SinkTester.java @@ -19,7 +19,6 @@ package org.apache.pulsar.tests.integration.io.sinks; import java.util.Optional; -import org.apache.pulsar.tests.integration.topologies.PulsarCluster; import org.testcontainers.elasticsearch.ElasticsearchContainer; public class ElasticSearch7SinkTester extends ElasticSearchSinkTester { @@ -32,8 +31,9 @@ public ElasticSearch7SinkTester(boolean schemaEnable) { super(schemaEnable); } + @Override - protected ElasticsearchContainer createSinkService(PulsarCluster cluster) { + protected ElasticsearchContainer createElasticContainer() { return new ElasticsearchContainer(ELASTICSEARCH_7) .withEnv("ES_JAVA_OPTS", "-Xms128m -Xmx256m"); } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearch8SinkTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearch8SinkTester.java index bb52c4ff03fea..8e7617a82a5b9 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearch8SinkTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearch8SinkTester.java @@ -19,13 +19,12 @@ package org.apache.pulsar.tests.integration.io.sinks; import java.util.Optional; -import org.apache.pulsar.tests.integration.topologies.PulsarCluster; import org.testcontainers.elasticsearch.ElasticsearchContainer; public class ElasticSearch8SinkTester extends ElasticSearchSinkTester { public static final String ELASTICSEARCH_8 = Optional.ofNullable(System.getenv("ELASTICSEARCH_IMAGE_V8")) - .orElse("docker.elastic.co/elasticsearch/elasticsearch:8.5.1"); + .orElse("docker.elastic.co/elasticsearch/elasticsearch:8.5.3"); public ElasticSearch8SinkTester(boolean schemaEnable) { @@ -33,7 +32,7 @@ public ElasticSearch8SinkTester(boolean schemaEnable) { } @Override - protected ElasticsearchContainer createSinkService(PulsarCluster cluster) { + protected ElasticsearchContainer createElasticContainer() { return new ElasticsearchContainer(ELASTICSEARCH_8) .withEnv("ES_JAVA_OPTS", "-Xms128m -Xmx256m") .withEnv("xpack.security.enabled", "false") diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearchSinkTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearchSinkTester.java index 546dd1b9113ab..0784055d29003 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearchSinkTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearchSinkTester.java @@ -19,15 +19,6 @@ package org.apache.pulsar.tests.integration.io.sinks; import static org.testng.Assert.assertTrue; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch.core.SearchRequest; import co.elastic.clients.elasticsearch.core.SearchResponse; @@ -35,6 +26,13 @@ import co.elastic.clients.transport.ElasticsearchTransport; import co.elastic.clients.transport.rest_client.RestClientTransport; import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import lombok.AllArgsConstructor; import lombok.Cleanup; import lombok.Data; @@ -46,6 +44,7 @@ import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.apache.pulsar.tests.integration.topologies.PulsarCluster; import org.awaitility.Awaitility; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; @@ -100,6 +99,29 @@ public ElasticSearchSinkTester(boolean schemaEnable) { } } + @Override + protected final ElasticsearchContainer createSinkService(PulsarCluster cluster) { + ElasticsearchContainer elasticContainer = createElasticContainer(); + configureElasticContainer(elasticContainer); + return elasticContainer; + } + + protected void configureElasticContainer(ElasticsearchContainer elasticContainer) { + if (!isOpenSearch()) { + elasticContainer.withEnv("ingest.geoip.downloader.enabled", "false"); + } + + // allow disk to fill up beyond default 90% threshold + elasticContainer.withEnv("cluster.routing.allocation.disk.threshold_enabled", "false"); + + elasticContainer.withLogConsumer(o -> log.info("elastic> {}", o.getUtf8String())); + } + + protected boolean isOpenSearch() { + return false; + } + + protected abstract ElasticsearchContainer createElasticContainer(); @Override public void prepareSink() throws Exception { diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/OpenSearchSinkTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/OpenSearchSinkTester.java index 1e10cc4189c1a..8daed8d5c04d5 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/OpenSearchSinkTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/OpenSearchSinkTester.java @@ -18,9 +18,10 @@ */ package org.apache.pulsar.tests.integration.io.sinks; +import static org.testng.Assert.assertTrue; +import java.util.Map; import java.util.Optional; import org.apache.http.HttpHost; -import org.apache.pulsar.tests.integration.topologies.PulsarCluster; import org.awaitility.Awaitility; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; @@ -31,14 +32,10 @@ import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testcontainers.utility.DockerImageName; -import java.util.Map; - -import static org.testng.Assert.assertTrue; - public class OpenSearchSinkTester extends ElasticSearchSinkTester { public static final String OPENSEARCH = Optional.ofNullable(System.getenv("OPENSEARCH_IMAGE")) - .orElse("opensearchproject/opensearch:1.2.4"); + .orElse("opensearchproject/opensearch:2.16.0"); private RestHighLevelClient elasticClient; @@ -48,15 +45,20 @@ public OpenSearchSinkTester(boolean schemaEnable) { } @Override - protected ElasticsearchContainer createSinkService(PulsarCluster cluster) { + protected ElasticsearchContainer createElasticContainer() { DockerImageName dockerImageName = DockerImageName.parse(OPENSEARCH) .asCompatibleSubstituteFor("docker.elastic.co/elasticsearch/elasticsearch"); return new ElasticsearchContainer(dockerImageName) + .withEnv("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "0pEn7earch!") .withEnv("OPENSEARCH_JAVA_OPTS", "-Xms128m -Xmx256m") .withEnv("bootstrap.memory_lock", "true") .withEnv("plugins.security.disabled", "true"); } + protected boolean isOpenSearch() { + return true; + } + @Override public void prepareSink() throws Exception { RestClientBuilder builder = RestClient.builder( diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarIOSinkRunner.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarIOSinkRunner.java index e5b524ebbef8b..3736bd0155343 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarIOSinkRunner.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarIOSinkRunner.java @@ -22,6 +22,7 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import dev.failsafe.Failsafe; import java.util.LinkedHashMap; import java.util.Map; @@ -46,7 +47,6 @@ import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; -import net.jodah.failsafe.Failsafe; @Slf4j public class PulsarIOSinkRunner extends PulsarIOTestRunner { diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarIOSourceRunner.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarIOSourceRunner.java index b843e146e2985..daf645020ce5a 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarIOSourceRunner.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarIOSourceRunner.java @@ -22,6 +22,7 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import dev.failsafe.Failsafe; import java.util.Map; import org.apache.commons.lang3.StringUtils; @@ -45,7 +46,6 @@ import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; -import net.jodah.failsafe.Failsafe; @Slf4j public class PulsarIOSourceRunner extends PulsarIOTestRunner { diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumMsSqlSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumMsSqlSourceTester.java index a745cae60409d..0b9a63340bd75 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumMsSqlSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumMsSqlSourceTester.java @@ -100,7 +100,7 @@ private ContainerExecResult runSqlCmd(String cmd, boolean useTestDb) throws Exce log.info("Executing \"{}\"", cmd); ContainerExecResult response = this.debeziumMsSqlContainer .execCmd("/bin/bash", "-c", - "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P \"" + "/opt/mssql-tools18/bin/sqlcmd -C -S localhost -U sa -P \"" + DebeziumMsSqlContainer.SA_PASSWORD + "\" -Q \"" + (useTestDb ? "USE TestDB; " : "") + cmd diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarIODebeziumSourceRunner.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarIODebeziumSourceRunner.java index 762dd34e17c91..8f45f0604e378 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarIODebeziumSourceRunner.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarIODebeziumSourceRunner.java @@ -19,9 +19,9 @@ package org.apache.pulsar.tests.integration.io.sources.debezium; import com.google.common.base.Preconditions; +import dev.failsafe.Failsafe; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; -import net.jodah.failsafe.Failsafe; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.SubscriptionInitialPosition; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java new file mode 100644 index 0000000000000..759e689b24d0f --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java @@ -0,0 +1,477 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.loadbalance; + +import static org.apache.pulsar.tests.integration.containers.PulsarContainer.BROKER_HTTP_PORT; +import static org.assertj.core.api.Assertions.assertThat; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import com.google.common.collect.Sets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.common.policies.data.AutoFailoverPolicyData; +import org.apache.pulsar.common.policies.data.AutoFailoverPolicyType; +import org.apache.pulsar.common.policies.data.BundlesData; +import org.apache.pulsar.common.policies.data.FailureDomain; +import org.apache.pulsar.common.policies.data.NamespaceIsolationData; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.tests.TestRetrySupport; +import org.apache.pulsar.tests.integration.containers.BrokerContainer; +import org.apache.pulsar.tests.integration.topologies.PulsarCluster; +import org.apache.pulsar.tests.integration.topologies.PulsarClusterSpec; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Factory; +import org.testng.annotations.Test; + + +/** + * Integration tests for Pulsar ExtensibleLoadManagerImpl. + */ +@Slf4j +public class ExtensibleLoadManagerTest extends TestRetrySupport { + + private static final int NUM_BROKERS = 3; + private static final String DEFAULT_TENANT = "my-tenant"; + private static final String DEFAULT_NAMESPACE = DEFAULT_TENANT + "/my-namespace"; + private static final String nsSuffix = "-anti-affinity-enabled"; + + private final String clusterName = "MultiLoadManagerTest-" + UUID.randomUUID(); + private final PulsarClusterSpec spec = PulsarClusterSpec.builder() + .clusterName(clusterName) + .numBrokers(NUM_BROKERS).build(); + private PulsarCluster pulsarCluster = null; + private String hosts; + private PulsarAdmin admin; + protected String serviceUnitStateTableViewClassName; + + @Factory(dataProvider = "serviceUnitStateTableViewClassName") + public ExtensibleLoadManagerTest(String serviceUnitStateTableViewClassName) { + this.serviceUnitStateTableViewClassName = serviceUnitStateTableViewClassName; + } + + @DataProvider(name = "serviceUnitStateTableViewClassName") + public static Object[][] serviceUnitStateTableViewClassName() { + return new Object[][]{ + {"org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateMetadataStoreTableViewImpl"}, + {"org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateTableViewImpl"} + }; + } + + @BeforeClass(alwaysRun = true) + public void setup() throws Exception { + incrementSetupNumber(); + Map brokerEnvs = new HashMap<>(); + brokerEnvs.put("loadManagerClassName", + "org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl"); + brokerEnvs.put("loadBalancerLoadSheddingStrategy", + "org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder"); + brokerEnvs.put("loadManagerServiceUnitStateTableViewClassName", + serviceUnitStateTableViewClassName); + brokerEnvs.put("forceDeleteNamespaceAllowed", "true"); + brokerEnvs.put("loadBalancerDebugModeEnabled", "true"); + brokerEnvs.put("PULSAR_MEM", "-Xmx512M"); + spec.brokerEnvs(brokerEnvs); + pulsarCluster = PulsarCluster.forSpec(spec); + pulsarCluster.start(); + + admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); + // all brokers alive + assertEquals(admin.brokers().getActiveBrokers(clusterName).size(), NUM_BROKERS); + + admin.tenants().createTenant(DEFAULT_TENANT, + new TenantInfoImpl(new HashSet<>(), Set.of(pulsarCluster.getClusterName()))); + admin.namespaces().createNamespace(DEFAULT_NAMESPACE, 100); + } + + @AfterClass(alwaysRun = true) + public void cleanup() { + markCurrentSetupNumberCleaned(); + if (pulsarCluster != null) { + pulsarCluster.stop(); + pulsarCluster = null; + } + if (admin != null) { + admin.close(); + admin = null; + } + } + + @BeforeMethod(alwaysRun = true) + public void startBroker() { + if (pulsarCluster != null) { + pulsarCluster.getBrokers().forEach(brokerContainer -> { + if (!brokerContainer.isRunning()) { + brokerContainer.start(); + } + }); + String topicName = "persistent://" + DEFAULT_NAMESPACE + "/startBrokerCheck"; + Awaitility.await().atMost(120, TimeUnit.SECONDS).ignoreExceptions().until( + () -> { + for (BrokerContainer brokerContainer : pulsarCluster.getBrokers()) { + try (PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl( + brokerContainer.getHttpServiceUrl()).build()) { + if (admin.brokers().getActiveBrokers(clusterName).size() != NUM_BROKERS) { + return false; + } + try { + admin.topics().createPartitionedTopic(topicName, 10); + } catch (PulsarAdminException.ConflictException e) { + // expected + } + admin.lookups().lookupPartitionedTopic(topicName); + } + } + return true; + } + ); + } + } + + @Test(timeOut = 40 * 1000) + public void testConcurrentLookups() throws Exception { + String topicName = "persistent://" + DEFAULT_NAMESPACE + "/testConcurrentLookups"; + List admins = new ArrayList<>(); + int numAdminForBroker = 10; + for (int i = 0; i < numAdminForBroker; i++) { + admins.add(PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build()); + } + + admin.topics().createPartitionedTopic(topicName, 100); + + var executor = Executors.newFixedThreadPool(admins.size()); + + CountDownLatch latch = new CountDownLatch(admins.size()); + List> result = new CopyOnWriteArrayList<>(); + for(var admin : admins) { + executor.execute(() -> { + try { + result.add(admin.lookups().lookupPartitionedTopic(topicName)); + } catch (PulsarAdminException e) { + log.error("Lookup partitioned topic failed.", e); + } + latch.countDown(); + }); + } + latch.await(); + + assertEquals(result.size(), admins.size()); + + for (int i = 1; i < admins.size(); i++) { + assertEquals(result.get(i - 1), result.get(i)); + } + admins.forEach(a -> a.close()); + executor.shutdown(); + } + + @Test(timeOut = 30 * 1000) + public void testTransferAdminApi() throws Exception { + String topicName = "persistent://" + DEFAULT_NAMESPACE + "/testUnloadAdminApi"; + createNonPartitionedTopicAndRetry(topicName); + String broker = admin.lookups().lookupTopic(topicName); + + int index = extractBrokerIndex(broker); + + String bundleRange = admin.lookups().getBundleRange(topicName); + + // Test transfer to current broker. + try { + admin.namespaces().unloadNamespaceBundle(DEFAULT_NAMESPACE, bundleRange, getBrokerUrl(index)); + fail(); + } catch (PulsarAdminException ex) { + assertTrue(ex.getMessage().contains("cannot be transfer to same broker")); + } + + int transferToIndex = generateRandomExcludingX(NUM_BROKERS, index); + assertNotEquals(transferToIndex, index); + String transferTo = getBrokerUrl(transferToIndex); + admin.namespaces().unloadNamespaceBundle(DEFAULT_NAMESPACE, bundleRange, transferTo); + + broker = admin.lookups().lookupTopic(topicName); + + index = extractBrokerIndex(broker); + assertEquals(index, transferToIndex); + } + + @Test(timeOut = 30 * 1000) + public void testSplitBundleAdminApi() throws Exception { + String topicName = "persistent://" + DEFAULT_NAMESPACE + "/testSplitBundleAdminApi"; + createNonPartitionedTopicAndRetry(topicName); + String broker = admin.lookups().lookupTopic(topicName); + log.info("The topic: {} owned by {}", topicName, broker); + BundlesData bundles = admin.namespaces().getBundles(DEFAULT_NAMESPACE); + int numBundles = bundles.getNumBundles(); + var bundleRanges = bundles.getBoundaries().stream().map(Long::decode).sorted().toList(); + String firstBundle = bundleRanges.get(0) + "_" + bundleRanges.get(1); + admin.namespaces().splitNamespaceBundle(DEFAULT_NAMESPACE, firstBundle, true, null); + long mid = bundleRanges.get(0) + (bundleRanges.get(1) - bundleRanges.get(0)) / 2; + Awaitility.waitAtMost(10, TimeUnit.SECONDS).pollDelay(100, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> { + BundlesData bundlesData = admin.namespaces().getBundles(DEFAULT_NAMESPACE); + assertEquals(bundlesData.getNumBundles(), numBundles + 1); + String lowBundle = String.format("0x%08x", bundleRanges.get(0)); + String midBundle = String.format("0x%08x", mid); + String highBundle = String.format("0x%08x", bundleRanges.get(1)); + assertTrue(bundlesData.getBoundaries().contains(lowBundle)); + assertTrue(bundlesData.getBoundaries().contains(midBundle)); + assertTrue(bundlesData.getBoundaries().contains(highBundle)); + } + ); + + + // Test split bundle with invalid bundle range. + try { + admin.namespaces().splitNamespaceBundle(DEFAULT_NAMESPACE, "invalid", true, null); + fail(); + } catch (PulsarAdminException ex) { + assertTrue(ex.getMessage().contains("Invalid bundle range")); + } + } + + @Test(timeOut = 30 * 1000) + public void testDeleteNamespace() throws Exception { + String namespace = DEFAULT_TENANT + "/test-delete-namespace"; + String topicName = "persistent://" + namespace + "/test-delete-namespace-topic"; + admin.namespaces().createNamespace(namespace); + admin.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet(clusterName)); + assertTrue(admin.namespaces().getNamespaces(DEFAULT_TENANT).contains(namespace)); + admin.topics().createPartitionedTopic(topicName, 2); + String broker = admin.lookups().lookupTopic(topicName); + log.info("The topic: {} owned by: {}", topicName, broker); + admin.namespaces().deleteNamespace(namespace, true); + assertFalse(admin.namespaces().getNamespaces(DEFAULT_TENANT).contains(namespace)); + } + + @Test(timeOut = 120 * 1000) + public void testStopBroker() throws Exception { + String topicName = "persistent://" + DEFAULT_NAMESPACE + "/test-stop-broker-topic"; + + createNonPartitionedTopicAndRetry(topicName); + String broker = admin.lookups().lookupTopic(topicName); + log.info("The topic: {} owned by: {}", topicName, broker); + + int idx = extractBrokerIndex(broker); + for (BrokerContainer container : pulsarCluster.getBrokers()) { + String name = container.getHostName(); + if (name.contains(String.valueOf(idx))) { + container.stop(); + } + } + + Awaitility.waitAtMost(60, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> { + String broker1 = admin.lookups().lookupTopic(topicName); + assertNotEquals(broker1, broker); + }); + + } + + @Test(timeOut = 80 * 1000) + public void testAntiaffinityPolicy() throws PulsarAdminException { + final String namespaceAntiAffinityGroup = "my-anti-affinity-filter"; + final String antiAffinityEnabledNameSpace = DEFAULT_TENANT + "/my-ns-filter" + nsSuffix; + final int numPartition = 20; + + List activeBrokers = admin.brokers().getActiveBrokers(); + + assertEquals(activeBrokers.size(), NUM_BROKERS); + + for (int i = 0; i < activeBrokers.size(); i++) { + String namespace = antiAffinityEnabledNameSpace + "-" + i; + admin.namespaces().createNamespace(namespace, 10); + admin.namespaces().setNamespaceAntiAffinityGroup(namespace, namespaceAntiAffinityGroup); + admin.clusters().createFailureDomain(clusterName, namespaceAntiAffinityGroup, FailureDomain.builder() + .brokers(Set.of(activeBrokers.get(i))).build()); + } + + Set result = new HashSet<>(); + for (int i = 0; i < activeBrokers.size(); i++) { + final String topic = "persistent://" + antiAffinityEnabledNameSpace + "-" + i +"/topic"; + admin.topics().createPartitionedTopic(topic, numPartition); + + Map topicToBroker = admin.lookups().lookupPartitionedTopic(topic); + + assertEquals(topicToBroker.size(), numPartition); + + HashSet brokers = new HashSet<>(topicToBroker.values()); + + assertEquals(brokers.size(), 1); + result.add(brokers.iterator().next()); + log.info("Topic: {}, lookup result: {}", topic, brokers.iterator().next()); + } + + assertEquals(result.size(), NUM_BROKERS); + } + + @Test(timeOut = 300 * 1000) + public void testIsolationPolicy() throws Exception { + final String namespaceIsolationPolicyName = "my-isolation-policy"; + final String isolationEnabledNameSpace = DEFAULT_TENANT + "/my-isolation-policy" + nsSuffix; + Map parameters1 = new HashMap<>(); + parameters1.put("min_limit", "1"); + parameters1.put("usage_threshold", "100"); + + Awaitility.await().atMost(10, TimeUnit.SECONDS).ignoreExceptions().untilAsserted( + () -> { + List activeBrokers = admin.brokers().getActiveBrokersAsync() + .get(5, TimeUnit.SECONDS); + assertEquals(activeBrokers.size(), NUM_BROKERS); + } + ); + try { + admin.namespaces().createNamespace(isolationEnabledNameSpace); + } catch (PulsarAdminException.ConflictException e) { + //expected when retried + } + + try { + admin.clusters() + .createNamespaceIsolationPolicy(clusterName, namespaceIsolationPolicyName, NamespaceIsolationData + .builder() + .namespaces(List.of(isolationEnabledNameSpace)) + .autoFailoverPolicy(AutoFailoverPolicyData.builder() + .policyType(AutoFailoverPolicyType.min_available) + .parameters(parameters1) + .build()) + .primary(List.of(getHostName(0))) + .secondary(List.of(getHostName(1))) + .build()); + } catch (PulsarAdminException.ConflictException e) { + //expected when retried + } + + final String topic = "persistent://" + isolationEnabledNameSpace + "/topic"; + createNonPartitionedTopicAndRetry(topic); + + String broker = admin.lookups().lookupTopic(topic); + assertEquals(extractBrokerIndex(broker), 0); + + for (BrokerContainer container : pulsarCluster.getBrokers()) { + String name = container.getHostName(); + if (name.contains("0")) { + container.stop(); + } + } + + Awaitility.await().atMost(60, TimeUnit.SECONDS).ignoreExceptions().untilAsserted( + () -> { + List activeBrokers = admin.brokers().getActiveBrokersAsync() + .get(5, TimeUnit.SECONDS); + assertEquals(activeBrokers.size(), 2); + } + ); + + Awaitility.await().atMost(60, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> { + String ownerBroker = admin.lookups().lookupTopicAsync(topic).get(5, TimeUnit.SECONDS); + assertEquals(extractBrokerIndex(ownerBroker), 1); + }); + + for (BrokerContainer container : pulsarCluster.getBrokers()) { + String name = container.getHostName(); + if (name.contains("1")) { + container.stop(); + } + } + + Awaitility.await().atMost(60, TimeUnit.SECONDS).ignoreExceptions().untilAsserted( + () -> { + List activeBrokers = admin.brokers().getActiveBrokersAsync().get(5, TimeUnit.SECONDS); + assertEquals(activeBrokers.size(), 1); + } + ); + + Awaitility.await().atMost(60, TimeUnit.SECONDS).ignoreExceptions().untilAsserted( + () -> { + try { + admin.lookups().lookupTopicAsync(topic).get(5, TimeUnit.SECONDS); + fail(); + } catch (Exception ex) { + log.error("Failed to lookup topic: ", ex); + assertThat(ex.getMessage()).contains("Service Unavailable"); + } + } + ); + + } + + private void createNonPartitionedTopicAndRetry(String topicName) throws Exception { + Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() -> { + try { + admin.topics().createNonPartitionedTopic(topicName); + return true; + } catch (PulsarAdminException.ConflictException e) { + return true; + //expected when retried + } catch (Exception e) { + log.error("Failed to create topic: ", e); + return false; + } + }); + } + + private String getBrokerUrl(int index) { + return String.format("pulsar-broker-%d:%d", index, BROKER_HTTP_PORT); + } + + private String getHostName(int index) { + return String.format("pulsar-broker-%d", index); + } + + private int extractBrokerIndex(String broker) { + String pattern = "pulsar://.*-(\\d+):\\d+"; + Pattern compiledPattern = Pattern.compile(pattern); + Matcher matcher = compiledPattern.matcher(broker); + if (!matcher.find()){ + throw new IllegalArgumentException("Failed to extract broker index"); + } + return Integer.parseInt(matcher.group(1)); + } + + private int generateRandomExcludingX(int n, int x) { + Random random = new Random(); + int randomNumber; + + do { + randomNumber = random.nextInt(n); + } while (randomNumber == x); + + return randomNumber; + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/messaging/MessagingBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/messaging/MessagingBase.java index 0e7106ef65ea1..ddedacc531a7c 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/messaging/MessagingBase.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/messaging/MessagingBase.java @@ -36,6 +36,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -150,11 +151,11 @@ protected String getPartitionedTopic(String topicPrefix, boolean isPersistent, i } } } - // Make sure key will not be distributed to multiple consumers + // Make sure key will not be distributed to multiple consumers (except null key) Set allKeys = Sets.newHashSet(); - consumerKeys.forEach((k, v) -> v.forEach(key -> { + consumerKeys.forEach((k, v) -> v.stream().filter(Objects::nonNull).forEach(key -> { assertTrue(allKeys.add(key), - "Key "+ key + "is distributed to multiple consumers" ); + "Key " + key + " is distributed to multiple consumers" ); })); assertEquals(messagesReceived.size(), messagesToReceive); } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/messaging/MessagingSmokeTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/messaging/MessagingSmokeTest.java new file mode 100644 index 0000000000000..618053ac000e2 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/messaging/MessagingSmokeTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.messaging; + +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder; +import org.apache.pulsar.common.naming.TopicDomain; +import org.testng.ITest; +import org.testng.annotations.Factory; +import org.testng.annotations.Test; + +public class MessagingSmokeTest extends TopicMessagingBase implements ITest { + + @Factory + public static Object[] messagingTests() { + List tests = List.of( + new MessagingSmokeTest("Extensible Load Manager", + Map.of("loadManagerClassName", ExtensibleLoadManagerImpl.class.getName(), + "loadBalancerLoadSheddingStrategy", TransferShedder.class.getName())), + new MessagingSmokeTest("Extensible Load Manager with TX Coordinator", + Map.of("loadManagerClassName", ExtensibleLoadManagerImpl.class.getName(), + "loadBalancerLoadSheddingStrategy", TransferShedder.class.getName(), + "transactionCoordinatorEnabled", "true")) + ); + return tests.toArray(); + } + + private final String name; + + public MessagingSmokeTest(String name, Map brokerEnvs) { + super(); + this.brokerEnvs.putAll(brokerEnvs); + this.name = name; + } + + @Override + public String getTestName() { + return name; + } + + @Test(dataProvider = "serviceUrlAndTopicDomain") + public void testNonPartitionedTopicMessagingWithExclusive(Supplier serviceUrl, TopicDomain topicDomain) + throws Exception { + nonPartitionedTopicSendAndReceiveWithExclusive(serviceUrl.get(), TopicDomain.persistent.equals(topicDomain)); + } + + @Test(dataProvider = "serviceUrlAndTopicDomain") + public void testPartitionedTopicMessagingWithExclusive(Supplier serviceUrl, TopicDomain topicDomain) + throws Exception { + partitionedTopicSendAndReceiveWithExclusive(serviceUrl.get(), TopicDomain.persistent.equals(topicDomain)); + } + + @Test(dataProvider = "serviceUrlAndTopicDomain") + public void testNonPartitionedTopicMessagingWithFailover(Supplier serviceUrl, TopicDomain topicDomain) + throws Exception { + nonPartitionedTopicSendAndReceiveWithFailover(serviceUrl.get(), TopicDomain.persistent.equals(topicDomain)); + } + + @Test(dataProvider = "serviceUrlAndTopicDomain") + public void testPartitionedTopicMessagingWithFailover(Supplier serviceUrl, TopicDomain topicDomain) + throws Exception { + partitionedTopicSendAndReceiveWithFailover(serviceUrl.get(), TopicDomain.persistent.equals(topicDomain)); + } + + @Test(dataProvider = "serviceUrlAndTopicDomain") + public void testNonPartitionedTopicMessagingWithShared(Supplier serviceUrl, TopicDomain topicDomain) + throws Exception { + nonPartitionedTopicSendAndReceiveWithShared(serviceUrl.get(), TopicDomain.persistent.equals(topicDomain)); + } + + @Test(dataProvider = "serviceUrlAndTopicDomain") + public void testPartitionedTopicMessagingWithShared(Supplier serviceUrl, TopicDomain topicDomain) + throws Exception { + partitionedTopicSendAndReceiveWithShared(serviceUrl.get(), TopicDomain.persistent.equals(topicDomain)); + } + + @Test(dataProvider = "serviceUrlAndTopicDomain") + public void testNonPartitionedTopicMessagingWithKeyShared(Supplier serviceUrl, TopicDomain topicDomain) + throws Exception { + nonPartitionedTopicSendAndReceiveWithKeyShared(serviceUrl.get(), TopicDomain.persistent.equals(topicDomain)); + } + + @Test(dataProvider = "serviceUrlAndTopicDomain") + public void testPartitionedTopicMessagingWithKeyShared(Supplier serviceUrl, TopicDomain topicDomain) + throws Exception { + partitionedTopicSendAndReceiveWithKeyShared(serviceUrl.get(), TopicDomain.persistent.equals(topicDomain)); + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/metrics/OpenTelemetrySanityTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/metrics/OpenTelemetrySanityTest.java new file mode 100644 index 0000000000000..31e600f3aa812 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/metrics/OpenTelemetrySanityTest.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.metrics; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.awaitility.Awaitility.waitAtMost; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.PulsarVersion; +import org.apache.pulsar.broker.stats.PulsarBrokerOpenTelemetry; +import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient; +import org.apache.pulsar.functions.worker.PulsarWorkerOpenTelemetry; +import org.apache.pulsar.proxy.stats.PulsarProxyOpenTelemetry; +import org.apache.pulsar.tests.integration.containers.ChaosContainer; +import org.apache.pulsar.tests.integration.containers.OpenTelemetryCollectorContainer; +import org.apache.pulsar.tests.integration.topologies.FunctionRuntimeType; +import org.apache.pulsar.tests.integration.topologies.PulsarCluster; +import org.apache.pulsar.tests.integration.topologies.PulsarClusterSpec; +import org.apache.pulsar.tests.integration.topologies.PulsarTestBase; +import org.testng.annotations.Test; + +public class OpenTelemetrySanityTest { + + // Validate that the OpenTelemetry metrics can be exported to a remote OpenTelemetry collector. + @Test(timeOut = 360_000) + public void testOpenTelemetryMetricsOtlpExport() throws Exception { + var clusterName = "testOpenTelemetryMetrics-" + UUID.randomUUID(); + var openTelemetryCollectorContainer = new OpenTelemetryCollectorContainer(clusterName); + + var exporter = "otlp"; + var otlpEndpointProp = + Pair.of("OTEL_EXPORTER_OTLP_ENDPOINT", openTelemetryCollectorContainer.getOtlpEndpoint()); + + var brokerCollectorProps = getOpenTelemetryProps(exporter, otlpEndpointProp); + var proxyCollectorProps = getOpenTelemetryProps(exporter, otlpEndpointProp); + var functionWorkerCollectorProps = getOpenTelemetryProps(exporter, otlpEndpointProp); + + var spec = PulsarClusterSpec.builder() + .clusterName(clusterName) + .brokerEnvs(brokerCollectorProps) + .proxyEnvs(proxyCollectorProps) + .externalService("otel-collector", openTelemetryCollectorContainer) + .functionWorkerEnvs(functionWorkerCollectorProps) + .build(); + @Cleanup("stop") + var pulsarCluster = PulsarCluster.forSpec(spec); + pulsarCluster.start(); + pulsarCluster.setupFunctionWorkers(PulsarTestBase.randomName(), FunctionRuntimeType.PROCESS, 1); + + // TODO: Validate cluster name and service version are present once + // https://github.com/open-telemetry/opentelemetry-java/issues/6108 is solved. + var metricName = "queueSize_ratio"; // Sent automatically by the OpenTelemetry SDK. + waitAtMost(90, TimeUnit.SECONDS).ignoreExceptions().pollInterval(1, TimeUnit.SECONDS).until(() -> { + var metrics = getMetricsFromPrometheus( + openTelemetryCollectorContainer, OpenTelemetryCollectorContainer.PROMETHEUS_EXPORTER_PORT); + return !metrics.findByNameAndLabels(metricName, "job", PulsarBrokerOpenTelemetry.SERVICE_NAME).isEmpty(); + }); + waitAtMost(90, TimeUnit.SECONDS).ignoreExceptions().pollInterval(1, TimeUnit.SECONDS).until(() -> { + var metrics = getMetricsFromPrometheus( + openTelemetryCollectorContainer, OpenTelemetryCollectorContainer.PROMETHEUS_EXPORTER_PORT); + return !metrics.findByNameAndLabels(metricName, "job", PulsarProxyOpenTelemetry.SERVICE_NAME).isEmpty(); + }); + waitAtMost(90, TimeUnit.SECONDS).ignoreExceptions().pollInterval(1, TimeUnit.SECONDS).until(() -> { + var metrics = getMetricsFromPrometheus( + openTelemetryCollectorContainer, OpenTelemetryCollectorContainer.PROMETHEUS_EXPORTER_PORT); + return !metrics.findByNameAndLabels(metricName, "job", PulsarWorkerOpenTelemetry.SERVICE_NAME).isEmpty(); + }); + } + + /* + * Validate that the OpenTelemetry metrics can be exported to a local Prometheus endpoint running in the same + * process space as the broker/proxy/function-worker. + * https://github.com/open-telemetry/opentelemetry-java/blob/main/sdk-extensions/autoconfigure/README.md#prometheus-exporter + */ + @Test(timeOut = 360_000) + public void testOpenTelemetryMetricsPrometheusExport() throws Exception { + var prometheusExporterPort = 9464; + var clusterName = "testOpenTelemetryMetrics-" + UUID.randomUUID(); + + var exporter = "prometheus"; + var prometheusExporterPortProp = + Pair.of("OTEL_EXPORTER_PROMETHEUS_PORT", Integer.toString(prometheusExporterPort)); + + var brokerCollectorProps = getOpenTelemetryProps(exporter, prometheusExporterPortProp); + var proxyCollectorProps = getOpenTelemetryProps(exporter, prometheusExporterPortProp); + var functionWorkerCollectorProps = getOpenTelemetryProps(exporter, prometheusExporterPortProp); + + var spec = PulsarClusterSpec.builder() + .clusterName(clusterName) + .brokerEnvs(brokerCollectorProps) + .brokerAdditionalPorts(List.of(prometheusExporterPort)) + .proxyEnvs(proxyCollectorProps) + .proxyAdditionalPorts(List.of(prometheusExporterPort)) + .functionWorkerEnvs(functionWorkerCollectorProps) + .functionWorkerAdditionalPorts(List.of(prometheusExporterPort)) + .build(); + @Cleanup("stop") + var pulsarCluster = PulsarCluster.forSpec(spec); + pulsarCluster.start(); + pulsarCluster.setupFunctionWorkers(PulsarTestBase.randomName(), FunctionRuntimeType.PROCESS, 1); + + var targetInfoMetricName = "target_info"; // Sent automatically by the OpenTelemetry SDK. + var cpuCountMetricName = "jvm_cpu_count"; // Configured by the OpenTelemetryService. + waitAtMost(90, TimeUnit.SECONDS).ignoreExceptions().pollInterval(1, TimeUnit.SECONDS).untilAsserted(() -> { + var expectedMetrics = new String[] {targetInfoMetricName, cpuCountMetricName, "pulsar_broker_topic_producer_count"}; + var actualMetrics = getMetricsFromPrometheus(pulsarCluster.getBroker(0), prometheusExporterPort); + assertThat(expectedMetrics).allMatch(expectedMetric -> !actualMetrics.findByNameAndLabels(expectedMetric, + Pair.of("pulsar_cluster", clusterName), + Pair.of("service_name", PulsarBrokerOpenTelemetry.SERVICE_NAME), + Pair.of("service_version", PulsarVersion.getVersion()), + Pair.of("host_name", pulsarCluster.getBroker(0).getHostname())).isEmpty()); + }); + waitAtMost(90, TimeUnit.SECONDS).ignoreExceptions().pollInterval(1, TimeUnit.SECONDS).untilAsserted(() -> { + var expectedMetrics = new String[] {targetInfoMetricName, cpuCountMetricName}; + var actualMetrics = getMetricsFromPrometheus(pulsarCluster.getProxy(), prometheusExporterPort); + assertThat(expectedMetrics).allMatch(expectedMetric -> !actualMetrics.findByNameAndLabels(expectedMetric, + Pair.of("pulsar_cluster", clusterName), + Pair.of("service_name", PulsarProxyOpenTelemetry.SERVICE_NAME), + Pair.of("service_version", PulsarVersion.getVersion()), + Pair.of("host_name", pulsarCluster.getProxy().getHostname())).isEmpty()); + }); + waitAtMost(90, TimeUnit.SECONDS).ignoreExceptions().pollInterval(1, TimeUnit.SECONDS).untilAsserted(() -> { + var expectedMetrics = new String[] {targetInfoMetricName, cpuCountMetricName}; + var actualMetrics = getMetricsFromPrometheus(pulsarCluster.getAnyWorker(), prometheusExporterPort); + assertThat(expectedMetrics).allMatch(expectedMetric -> !actualMetrics.findByNameAndLabels(expectedMetric, + Pair.of("pulsar_cluster", clusterName), + Pair.of("service_name", PulsarWorkerOpenTelemetry.SERVICE_NAME), + Pair.of("service_version", PulsarVersion.getVersion()), + Pair.of("host_name", pulsarCluster.getAnyWorker().getHostname())).isEmpty()); + }); + } + + private static PrometheusMetricsClient.Metrics getMetricsFromPrometheus(ChaosContainer container, int port) { + var client = new PrometheusMetricsClient(container.getHost(), container.getMappedPort(port)); + return client.getMetrics(); + } + + private static Map getOpenTelemetryProps(String exporter, Pair ... extraProps) { + var defaultProps = Map.of( + "OTEL_SDK_DISABLED", "false", + "OTEL_METRIC_EXPORT_INTERVAL", "1000", + "OTEL_METRICS_EXPORTER", exporter + ); + var props = new HashMap<>(defaultProps); + Arrays.stream(extraProps).forEach(p -> props.put(p.getKey(), p.getValue())); + return props; + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/offload/TestFileSystemOffload.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/offload/TestFileSystemOffload.java index 4504b58ca920b..a58ec92bafe3f 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/offload/TestFileSystemOffload.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/offload/TestFileSystemOffload.java @@ -49,7 +49,7 @@ protected Map getEnv() { result.put("managedLedgerMaxEntriesPerLedger", String.valueOf(getNumEntriesPerLedger())); result.put("managedLedgerMinLedgerRolloverTimeMinutes", "0"); result.put("managedLedgerOffloadDriver", "filesystem"); - result.put("fileSystemURI", "file:///"); + result.put("fileSystemURI", "file:///tmp"); return result; } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/oxia/OxiaContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/oxia/OxiaContainer.java new file mode 100644 index 0000000000000..18d2dd77b7d46 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/oxia/OxiaContainer.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.tests.integration.oxia; + +import java.time.Duration; +import org.apache.pulsar.tests.integration.containers.ChaosContainer; +import org.apache.pulsar.tests.integration.containers.PulsarContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +public class OxiaContainer extends ChaosContainer { + + public static final String NAME = "oxia"; + + public static final int OXIA_PORT = 6648; + public static final int METRICS_PORT = 8080; + private static final int DEFAULT_SHARDS = 1; + + private static final String DEFAULT_IMAGE_NAME = "streamnative/oxia:main"; + + public OxiaContainer(String clusterName) { + this(clusterName, DEFAULT_IMAGE_NAME, DEFAULT_SHARDS); + } + + @SuppressWarnings("resource") + OxiaContainer(String clusterName, String imageName, int shards) { + super(clusterName, imageName); + if (shards <= 0) { + throw new IllegalArgumentException("shards must be greater than zero"); + } + addExposedPorts(OXIA_PORT, METRICS_PORT); + this.withCreateContainerCmdModifier(createContainerCmd -> { + createContainerCmd.withHostName("oxia"); + createContainerCmd.withName(getContainerName()); + }); + setCommand("oxia", "standalone", + "--shards=" + shards, + "--wal-sync-data=false"); + waitingFor( + Wait.forHttp("/metrics") + .forPort(METRICS_PORT) + .forStatusCode(200) + .withStartupTimeout(Duration.ofSeconds(30))); + + PulsarContainer.configureLeaveContainerRunning(this); + } + + public String getServiceAddress() { + return OxiaContainer.NAME + ":" + OXIA_PORT; + } + + @Override + public String getContainerName() { + return clusterName + "-oxia"; + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/oxia/OxiaSmokeTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/oxia/OxiaSmokeTest.java new file mode 100644 index 0000000000000..d55c437b4f89a --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/oxia/OxiaSmokeTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.oxia; + +import java.util.function.Supplier; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.tests.integration.suites.PulsarTestSuite; +import org.apache.pulsar.tests.integration.topologies.PulsarClusterSpec; +import org.testng.annotations.Test; + +/** + * Test pulsar produce/consume semantics + */ +@Slf4j +public class OxiaSmokeTest extends PulsarTestSuite { + + protected PulsarClusterSpec.PulsarClusterSpecBuilder beforeSetupCluster( + String clusterName, PulsarClusterSpec.PulsarClusterSpecBuilder specBuilder) { + specBuilder.enableOxia(true); + return specBuilder; + } + + // + // Test Basic Publish & Consume Operations + // + + @Test(dataProvider = "ServiceUrlAndTopics") + public void testPublishAndConsume(Supplier serviceUrl, boolean isPersistent) throws Exception { + super.testPublishAndConsume(serviceUrl.get(), isPersistent); + } + +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/Stock.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/Stock.java deleted file mode 100644 index 93b2d91838b0b..0000000000000 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/Stock.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.tests.integration.presto; - -import java.util.Objects; - -public class Stock { - - private int entryId; - private String symbol; - private double sharePrice; - - public Stock(int entryId, String symbol, double sharePrice) { - this.entryId = entryId; - this.symbol = symbol; - this.sharePrice = sharePrice; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Stock stock = (Stock) o; - return entryId == stock.entryId && - Double.compare(stock.sharePrice, sharePrice) == 0 && - Objects.equals(symbol, stock.symbol); - } - - @Override - public int hashCode() { - return Objects.hash(symbol, sharePrice); - } - - @Override - public String toString() { - return "Stock{" + - "entryId=" + entryId + - ", symbol='" + symbol + '\'' + - ", sharePrice=" + sharePrice + - '}'; - } - - public int getEntryId() { - return entryId; - } - - public void setEntryId(int entryId) { - this.entryId = entryId; - } - - public String getSymbol() { - return symbol; - } - - public void setSymbol(String symbol) { - this.symbol = symbol; - } - - public double getSharePrice() { - return sharePrice; - } - - public void setSharePrice(double sharePrice) { - this.sharePrice = sharePrice; - } -} \ No newline at end of file diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/StockMsg.proto b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/StockMsg.proto deleted file mode 100644 index 8e50c5843218f..0000000000000 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/StockMsg.proto +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -syntax = "proto3"; -package proto; - -option java_package = "org.apache.pulsar.tests.integration.presto"; -option java_outer_classname = "StockProtoMessage"; - -message Stock { - int32 entryId = 1; - string symbol = 2; - double sharePrice = 3; -} \ No newline at end of file diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/StockProtoMessage.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/StockProtoMessage.java deleted file mode 100644 index 67941a923ca67..0000000000000 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/StockProtoMessage.java +++ /dev/null @@ -1,731 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: StockMsg.proto - -package org.apache.pulsar.tests.integration.presto; - -@SuppressWarnings("deprecation") -public final class StockProtoMessage { - private StockProtoMessage() {} - public static void registerAllExtensions( - com.google.protobuf.ExtensionRegistryLite registry) { - } - - public static void registerAllExtensions( - com.google.protobuf.ExtensionRegistry registry) { - registerAllExtensions( - (com.google.protobuf.ExtensionRegistryLite) registry); - } - public interface StockOrBuilder extends - // @@protoc_insertion_point(interface_extends:proto.Stock) - com.google.protobuf.MessageOrBuilder { - - /** - * int32 entryId = 1; - */ - int getEntryId(); - - /** - * string symbol = 2; - */ - java.lang.String getSymbol(); - /** - * string symbol = 2; - */ - com.google.protobuf.ByteString - getSymbolBytes(); - - /** - * double sharePrice = 3; - */ - double getSharePrice(); - } - /** - * Protobuf type {@code proto.Stock} - */ - public static final class Stock extends - com.google.protobuf.GeneratedMessageV3 implements - // @@protoc_insertion_point(message_implements:proto.Stock) - StockOrBuilder { - private static final long serialVersionUID = 0L; - // Use Stock.newBuilder() to construct. - private Stock(com.google.protobuf.GeneratedMessageV3.Builder builder) { - super(builder); - } - private Stock() { - entryId_ = 0; - symbol_ = ""; - sharePrice_ = 0D; - } - - @java.lang.Override - public final com.google.protobuf.UnknownFieldSet - getUnknownFields() { - return this.unknownFields; - } - private Stock( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - this(); - if (extensionRegistry == null) { - throw new java.lang.IllegalArgumentException(); - } - int mutable_bitField0_ = 0; - com.google.protobuf.UnknownFieldSet.Builder unknownFields = - com.google.protobuf.UnknownFieldSet.newBuilder(); - try { - boolean done = false; - while (!done) { - int tag = input.readTag(); - switch (tag) { - case 0: - done = true; - break; - default: { - if (!parseUnknownFieldProto3( - input, unknownFields, extensionRegistry, tag)) { - done = true; - } - break; - } - case 8: { - - entryId_ = input.readInt32(); - break; - } - case 18: { - java.lang.String s = input.readStringRequireUtf8(); - - symbol_ = s; - break; - } - case 25: { - - sharePrice_ = input.readDouble(); - break; - } - } - } - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw e.setUnfinishedMessage(this); - } catch (java.io.IOException e) { - throw new com.google.protobuf.InvalidProtocolBufferException( - e).setUnfinishedMessage(this); - } finally { - this.unknownFields = unknownFields.build(); - makeExtensionsImmutable(); - } - } - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.apache.pulsar.tests.integration.presto.StockProtoMessage.internal_static_proto_Stock_descriptor; - } - - protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.apache.pulsar.tests.integration.presto.StockProtoMessage.internal_static_proto_Stock_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock.class, org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock.Builder.class); - } - - public static final int ENTRYID_FIELD_NUMBER = 1; - private int entryId_; - /** - * int32 entryId = 1; - */ - public int getEntryId() { - return entryId_; - } - - public static final int SYMBOL_FIELD_NUMBER = 2; - private volatile java.lang.Object symbol_; - /** - * string symbol = 2; - */ - public java.lang.String getSymbol() { - java.lang.Object ref = symbol_; - if (ref instanceof java.lang.String) { - return (java.lang.String) ref; - } else { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - java.lang.String s = bs.toStringUtf8(); - symbol_ = s; - return s; - } - } - /** - * string symbol = 2; - */ - public com.google.protobuf.ByteString - getSymbolBytes() { - java.lang.Object ref = symbol_; - if (ref instanceof java.lang.String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (java.lang.String) ref); - symbol_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - - public static final int SHAREPRICE_FIELD_NUMBER = 3; - private double sharePrice_; - /** - * double sharePrice = 3; - */ - public double getSharePrice() { - return sharePrice_; - } - - private byte memoizedIsInitialized = -1; - public final boolean isInitialized() { - byte isInitialized = memoizedIsInitialized; - if (isInitialized == 1) return true; - if (isInitialized == 0) return false; - - memoizedIsInitialized = 1; - return true; - } - - public void writeTo(com.google.protobuf.CodedOutputStream output) - throws java.io.IOException { - if (entryId_ != 0) { - output.writeInt32(1, entryId_); - } - if (!getSymbolBytes().isEmpty()) { - com.google.protobuf.GeneratedMessageV3.writeString(output, 2, symbol_); - } - if (sharePrice_ != 0D) { - output.writeDouble(3, sharePrice_); - } - unknownFields.writeTo(output); - } - - public int getSerializedSize() { - int size = memoizedSize; - if (size != -1) return size; - - size = 0; - if (entryId_ != 0) { - size += com.google.protobuf.CodedOutputStream - .computeInt32Size(1, entryId_); - } - if (!getSymbolBytes().isEmpty()) { - size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, symbol_); - } - if (sharePrice_ != 0D) { - size += com.google.protobuf.CodedOutputStream - .computeDoubleSize(3, sharePrice_); - } - size += unknownFields.getSerializedSize(); - memoizedSize = size; - return size; - } - - @java.lang.Override - public boolean equals(final java.lang.Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock)) { - return super.equals(obj); - } - org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock other = (org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock) obj; - - boolean result = true; - result = result && (getEntryId() - == other.getEntryId()); - result = result && getSymbol() - .equals(other.getSymbol()); - result = result && ( - java.lang.Double.doubleToLongBits(getSharePrice()) - == java.lang.Double.doubleToLongBits( - other.getSharePrice())); - result = result && unknownFields.equals(other.unknownFields); - return result; - } - - @java.lang.Override - public int hashCode() { - if (memoizedHashCode != 0) { - return memoizedHashCode; - } - int hash = 41; - hash = (19 * hash) + getDescriptor().hashCode(); - hash = (37 * hash) + ENTRYID_FIELD_NUMBER; - hash = (53 * hash) + getEntryId(); - hash = (37 * hash) + SYMBOL_FIELD_NUMBER; - hash = (53 * hash) + getSymbol().hashCode(); - hash = (37 * hash) + SHAREPRICE_FIELD_NUMBER; - hash = (53 * hash) + com.google.protobuf.Internal.hashLong( - java.lang.Double.doubleToLongBits(getSharePrice())); - hash = (29 * hash) + unknownFields.hashCode(); - memoizedHashCode = hash; - return hash; - } - - public static org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock parseFrom( - java.nio.ByteBuffer data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock parseFrom( - java.nio.ByteBuffer data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock parseFrom( - com.google.protobuf.ByteString data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock parseFrom( - com.google.protobuf.ByteString data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock parseFrom(byte[] data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock parseFrom( - byte[] data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock parseFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - public static org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock parseFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - public static org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock parseDelimitedFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input); - } - public static org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock parseDelimitedFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input, extensionRegistry); - } - public static org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock parseFrom( - com.google.protobuf.CodedInputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - public static org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock parseFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - - public Builder newBuilderForType() { return newBuilder(); } - public static Builder newBuilder() { - return DEFAULT_INSTANCE.toBuilder(); - } - public static Builder newBuilder(org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock prototype) { - return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); - } - public Builder toBuilder() { - return this == DEFAULT_INSTANCE - ? new Builder() : new Builder().mergeFrom(this); - } - - @java.lang.Override - protected Builder newBuilderForType( - com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { - Builder builder = new Builder(parent); - return builder; - } - /** - * Protobuf type {@code proto.Stock} - */ - @SuppressWarnings("cast") - public static final class Builder extends - com.google.protobuf.GeneratedMessageV3.Builder implements - // @@protoc_insertion_point(builder_implements:proto.Stock) - org.apache.pulsar.tests.integration.presto.StockProtoMessage.StockOrBuilder { - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.apache.pulsar.tests.integration.presto.StockProtoMessage.internal_static_proto_Stock_descriptor; - } - - protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.apache.pulsar.tests.integration.presto.StockProtoMessage.internal_static_proto_Stock_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock.class, org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock.Builder.class); - } - - // Construct using org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock.newBuilder() - private Builder() { - maybeForceBuilderInitialization(); - } - - private Builder( - com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { - super(parent); - maybeForceBuilderInitialization(); - } - private void maybeForceBuilderInitialization() { - if (com.google.protobuf.GeneratedMessageV3 - .alwaysUseFieldBuilders) { - } - } - public Builder clear() { - super.clear(); - entryId_ = 0; - - symbol_ = ""; - - sharePrice_ = 0D; - - return this; - } - - public com.google.protobuf.Descriptors.Descriptor - getDescriptorForType() { - return org.apache.pulsar.tests.integration.presto.StockProtoMessage.internal_static_proto_Stock_descriptor; - } - - public org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock getDefaultInstanceForType() { - return org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock.getDefaultInstance(); - } - - public org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock build() { - org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException(result); - } - return result; - } - - public org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock buildPartial() { - org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock result = new org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock(this); - result.entryId_ = entryId_; - result.symbol_ = symbol_; - result.sharePrice_ = sharePrice_; - onBuilt(); - return result; - } - - public Builder clone() { - return (Builder) super.clone(); - } - public Builder setField( - com.google.protobuf.Descriptors.FieldDescriptor field, - java.lang.Object value) { - return (Builder) super.setField(field, value); - } - public Builder clearField( - com.google.protobuf.Descriptors.FieldDescriptor field) { - return (Builder) super.clearField(field); - } - public Builder clearOneof( - com.google.protobuf.Descriptors.OneofDescriptor oneof) { - return (Builder) super.clearOneof(oneof); - } - public Builder setRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - int index, java.lang.Object value) { - return (Builder) super.setRepeatedField(field, index, value); - } - public Builder addRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - java.lang.Object value) { - return (Builder) super.addRepeatedField(field, value); - } - public Builder mergeFrom(com.google.protobuf.Message other) { - if (other instanceof org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock) { - return mergeFrom((org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock)other); - } else { - super.mergeFrom(other); - return this; - } - } - - public Builder mergeFrom(org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock other) { - if (other == org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock.getDefaultInstance()) return this; - if (other.getEntryId() != 0) { - setEntryId(other.getEntryId()); - } - if (!other.getSymbol().isEmpty()) { - symbol_ = other.symbol_; - onChanged(); - } - if (other.getSharePrice() != 0D) { - setSharePrice(other.getSharePrice()); - } - this.mergeUnknownFields(other.unknownFields); - onChanged(); - return this; - } - - public final boolean isInitialized() { - return true; - } - - public Builder mergeFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock parsedMessage = null; - try { - parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - parsedMessage = (org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock) e.getUnfinishedMessage(); - throw e.unwrapIOException(); - } finally { - if (parsedMessage != null) { - mergeFrom(parsedMessage); - } - } - return this; - } - - private int entryId_ ; - /** - * int32 entryId = 1; - */ - public int getEntryId() { - return entryId_; - } - /** - * int32 entryId = 1; - */ - public Builder setEntryId(int value) { - - entryId_ = value; - onChanged(); - return this; - } - /** - * int32 entryId = 1; - */ - public Builder clearEntryId() { - - entryId_ = 0; - onChanged(); - return this; - } - - private java.lang.Object symbol_ = ""; - /** - * string symbol = 2; - */ - public java.lang.String getSymbol() { - java.lang.Object ref = symbol_; - if (!(ref instanceof java.lang.String)) { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - java.lang.String s = bs.toStringUtf8(); - symbol_ = s; - return s; - } else { - return (java.lang.String) ref; - } - } - /** - * string symbol = 2; - */ - public com.google.protobuf.ByteString - getSymbolBytes() { - java.lang.Object ref = symbol_; - if (ref instanceof String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (java.lang.String) ref); - symbol_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - /** - * string symbol = 2; - */ - public Builder setSymbol( - java.lang.String value) { - if (value == null) { - throw new IllegalArgumentException(); - } - - symbol_ = value; - onChanged(); - return this; - } - /** - * string symbol = 2; - */ - public Builder clearSymbol() { - - symbol_ = getDefaultInstance().getSymbol(); - onChanged(); - return this; - } - /** - * string symbol = 2; - */ - public Builder setSymbolBytes( - com.google.protobuf.ByteString value) { - if (value == null) { - throw new IllegalArgumentException(); - } - checkByteStringIsUtf8(value); - - symbol_ = value; - onChanged(); - return this; - } - - private double sharePrice_ ; - /** - * double sharePrice = 3; - */ - public double getSharePrice() { - return sharePrice_; - } - /** - * double sharePrice = 3; - */ - public Builder setSharePrice(double value) { - - sharePrice_ = value; - onChanged(); - return this; - } - /** - * double sharePrice = 3; - */ - public Builder clearSharePrice() { - - sharePrice_ = 0D; - onChanged(); - return this; - } - public final Builder setUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.setUnknownFieldsProto3(unknownFields); - } - - public final Builder mergeUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.mergeUnknownFields(unknownFields); - } - - - // @@protoc_insertion_point(builder_scope:proto.Stock) - } - - // @@protoc_insertion_point(class_scope:proto.Stock) - private static final org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock DEFAULT_INSTANCE; - static { - DEFAULT_INSTANCE = new org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock(); - } - - public static org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock getDefaultInstance() { - return DEFAULT_INSTANCE; - } - - private static final com.google.protobuf.Parser - PARSER = new com.google.protobuf.AbstractParser() { - public Stock parsePartialFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return new Stock(input, extensionRegistry); - } - }; - - public static com.google.protobuf.Parser parser() { - return PARSER; - } - - @java.lang.Override - public com.google.protobuf.Parser getParserForType() { - return PARSER; - } - - public org.apache.pulsar.tests.integration.presto.StockProtoMessage.Stock getDefaultInstanceForType() { - return DEFAULT_INSTANCE; - } - - } - - private static final com.google.protobuf.Descriptors.Descriptor - internal_static_proto_Stock_descriptor; - private static final - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internal_static_proto_Stock_fieldAccessorTable; - - public static com.google.protobuf.Descriptors.FileDescriptor - getDescriptor() { - return descriptor; - } - private static com.google.protobuf.Descriptors.FileDescriptor - descriptor; - static { - java.lang.String[] descriptorData = { - "\n\016StockMsg.proto\022\005proto\"<\n\005Stock\022\017\n\007entr" + - "yId\030\001 \001(\005\022\016\n\006symbol\030\002 \001(\t\022\022\n\nsharePrice\030" + - "\003 \001(\001B?\n*org.apache.pulsar.tests.integra" + - "tion.prestoB\021StockProtoMessageb\006proto3" - }; - com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = - new com.google.protobuf.Descriptors.FileDescriptor. InternalDescriptorAssigner() { - public com.google.protobuf.ExtensionRegistry assignDescriptors( - com.google.protobuf.Descriptors.FileDescriptor root) { - descriptor = root; - return null; - } - }; - com.google.protobuf.Descriptors.FileDescriptor - .internalBuildGeneratedFileFrom(descriptorData, - new com.google.protobuf.Descriptors.FileDescriptor[] { - }, assigner); - internal_static_proto_Stock_descriptor = - getDescriptor().getMessageTypes().get(0); - internal_static_proto_Stock_fieldAccessorTable = new - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( - internal_static_proto_Stock_descriptor, - new java.lang.String[] { "EntryId", "Symbol", "SharePrice", }); - } - - // @@protoc_insertion_point(outer_class_scope) -} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java deleted file mode 100644 index 7658883441f5b..0000000000000 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java +++ /dev/null @@ -1,366 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.tests.integration.presto; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertTrue; - -import java.nio.ByteBuffer; -import lombok.Cleanup; -import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.client.api.CompressionType; -import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.impl.schema.AvroSchema; -import org.apache.pulsar.client.impl.schema.JSONSchema; -import org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl; -import org.apache.pulsar.client.impl.schema.ProtobufNativeSchema; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.protocol.Commands; -import org.apache.pulsar.common.schema.KeyValue; -import org.apache.pulsar.common.schema.KeyValueEncodingType; -import org.apache.pulsar.common.schema.SchemaType; -import org.apache.pulsar.tests.integration.docker.ContainerExecException; -import org.apache.pulsar.tests.integration.docker.ContainerExecResult; -import org.testng.Assert; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - - -/** - * Test basic Pulsar SQL query, the Pulsar SQL is standalone mode. - */ -@Slf4j -public class TestBasicPresto extends TestPulsarSQLBase { - - private static final int NUM_OF_STOCKS = 10; - - private void setupPresto() throws Exception { - log.info("[TestBasicPresto] setupPresto..."); - pulsarCluster.startPrestoWorker(); - initJdbcConnection(); - } - - private void teardownPresto() { - log.info("[TestBasicPresto] tearing down..."); - pulsarCluster.stopPrestoWorker(); - } - - @Override - public void setupCluster() throws Exception { - super.setupCluster(); - setupPresto(); - } - - @Override - public void tearDownCluster() throws Exception { - teardownPresto(); - super.tearDownCluster(); - } - - @DataProvider(name = "schemaProvider") - public Object[][] schemaProvider() { - return new Object[][] { - { Schema.BYTES}, - { Schema.BYTEBUFFER}, - { Schema.STRING}, - { AvroSchema.of(Stock.class)}, - { JSONSchema.of(Stock.class)}, - { ProtobufNativeSchema.of(StockProtoMessage.Stock.class)}, - { Schema.KeyValue(Schema.AVRO(Stock.class), Schema.AVRO(Stock.class), KeyValueEncodingType.INLINE) }, - { Schema.KeyValue(Schema.AVRO(Stock.class), Schema.AVRO(Stock.class), KeyValueEncodingType.SEPARATED) } - }; - } - - @Test(dataProvider = "batchingAndCompression") - public void testSimpleSQLQuery(boolean batchEnabled, CompressionType compressionType) throws Exception { - TopicName topicName = TopicName.get("public/default/stocks_batched_" + randomName(5)); - pulsarSQLBasicTest(topicName, batchEnabled, false, JSONSchema.of(Stock.class), compressionType); - } - - @Test(dataProvider = "schemaProvider") - public void testForSchema(Schema schema) throws Exception { - String schemaFlag; - if (schema.getSchemaInfo().getType().isStruct()) { - schemaFlag = schema.getSchemaInfo().getType().name(); - } else if(schema.getSchemaInfo().getType().equals(SchemaType.KEY_VALUE)) { - schemaFlag = schema.getSchemaInfo().getType().name() + "_" - + ((KeyValueSchemaImpl) schema).getKeyValueEncodingType(); - } else { - // Because some schema types are same(such as BYTES and BYTEBUFFER), so use the schema name as flag. - schemaFlag = schema.getSchemaInfo().getName(); - } - String topic = String.format("public/default/schema_%s_test_%s", schemaFlag, randomName(5)).toLowerCase(); - pulsarSQLBasicTest(TopicName.get(topic), false, false, schema, CompressionType.NONE); - } - - @Test - public void testForUppercaseTopic() throws Exception { - TopicName topicName = TopicName.get("public/default/case_UPPER_topic_" + randomName(5)); - pulsarSQLBasicTest(topicName, false, false, JSONSchema.of(Stock.class), CompressionType.NONE); - } - - @Test - public void testForDifferentCaseTopic() throws Exception { - String tableName = "diff_case_topic_" + randomName(5); - - String topic1 = "public/default/" + tableName.toUpperCase(); - TopicName topicName1 = TopicName.get(topic1); - prepareData(topicName1, false, false, JSONSchema.of(Stock.class), CompressionType.NONE); - - String topic2 = "public/default/" + tableName; - TopicName topicName2 = TopicName.get(topic2); - prepareData(topicName2, false, false, JSONSchema.of(Stock.class), CompressionType.NONE); - - try { - String query = "select * from pulsar.\"public/default\".\"" + tableName + "\""; - execQuery(query); - Assert.fail("The testForDifferentCaseTopic query [" + query + "] should be failed."); - } catch (ContainerExecException e) { - log.warn("Expected exception. result stderr: {}", e.getResult().getStderr(), e); - assertTrue(e.getResult().getStderr().contains("There are multiple topics")); - assertTrue(e.getResult().getStderr().contains(topic1)); - assertTrue(e.getResult().getStderr().contains(topic2)); - assertTrue(e.getResult().getStderr().contains("matched the table name public/default/" + tableName)); - } - } - - @Test - public void testListTopicShouldNotShowNonPersistentTopics() throws Exception { - String tableName = "non_persistent" + randomName(5); - - String topic1 = "non-persistent://public/default/" + tableName.toUpperCase(); - TopicName topicName1 = TopicName.get(topic1); - prepareData(topicName1, false, false, JSONSchema.of(Stock.class), CompressionType.NONE); - - String query = "show tables from pulsar.\"public/default\""; - ContainerExecResult result = execQuery(query); - assertFalse(result.getStdout().contains("non_persistent")); - } - - @SuppressWarnings("unchecked") - @Override - protected int prepareData(TopicName topicName, - boolean isBatch, - boolean useNsOffloadPolices, - Schema schema, - CompressionType compressionType) throws Exception { - - if (schema.getSchemaInfo().getName().equals(Schema.BYTES.getSchemaInfo().getName())) { - prepareDataForBytesSchema(topicName, isBatch, compressionType); - } else if (schema.getSchemaInfo().getName().equals(Schema.BYTEBUFFER.getSchemaInfo().getName())) { - prepareDataForByteBufferSchema(topicName, isBatch, compressionType); - } else if (schema.getSchemaInfo().getType().equals(SchemaType.STRING)) { - prepareDataForStringSchema(topicName, isBatch, compressionType); - } else if (schema.getSchemaInfo().getType().equals(SchemaType.JSON) - || schema.getSchemaInfo().getType().equals(SchemaType.AVRO)) { - prepareDataForStructSchema(topicName, isBatch, (Schema) schema, compressionType); - } else if (schema.getSchemaInfo().getType().equals(SchemaType.PROTOBUF_NATIVE)) { - prepareDataForProtobufNativeSchema(topicName, isBatch, (Schema) schema, compressionType); - } else if (schema.getSchemaInfo().getType().equals(SchemaType.KEY_VALUE)) { - prepareDataForKeyValueSchema(topicName, (Schema>) schema, compressionType); - } - - return NUM_OF_STOCKS; - } - - private void prepareDataForBytesSchema(TopicName topicName, - boolean isBatch, - CompressionType compressionType) throws PulsarClientException { - @Cleanup - Producer producer = pulsarClient.newProducer(Schema.BYTES) - .topic(topicName.toString()) - .enableBatching(isBatch) - .compressionType(compressionType) - .create(); - - for (int i = 0 ; i < NUM_OF_STOCKS; ++i) { - producer.send(("bytes schema test" + i).getBytes()); - } - producer.flush(); - } - - private void prepareDataForByteBufferSchema(TopicName topicName, - boolean isBatch, - CompressionType compressionType) throws PulsarClientException { - @Cleanup - Producer producer = pulsarClient.newProducer(Schema.BYTEBUFFER) - .topic(topicName.toString()) - .enableBatching(isBatch) - .compressionType(compressionType) - .create(); - - for (int i = 0 ; i < NUM_OF_STOCKS; ++i) { - producer.send(ByteBuffer.wrap(("bytes schema test" + i).getBytes())); - } - producer.flush(); - } - - private void prepareDataForStringSchema(TopicName topicName, - boolean isBatch, - CompressionType compressionType) throws PulsarClientException { - @Cleanup - Producer producer = pulsarClient.newProducer(Schema.STRING) - .topic(topicName.toString()) - .enableBatching(isBatch) - .compressionType(compressionType) - .create(); - - for (int i = 0 ; i < NUM_OF_STOCKS; ++i) { - producer.send("string" + i); - } - producer.flush(); - } - - private void prepareDataForStructSchema(TopicName topicName, - boolean isBatch, - Schema schema, - CompressionType compressionType) throws Exception { - @Cleanup - Producer producer = pulsarClient.newProducer(schema) - .topic(topicName.toString()) - .enableBatching(isBatch) - .compressionType(compressionType) - .create(); - - for (int i = 0 ; i < NUM_OF_STOCKS; ++i) { - final Stock stock = new Stock(i, "STOCK_" + i, 100.0 + i * 10); - producer.send(stock); - } - producer.flush(); - } - - private void prepareDataForProtobufNativeSchema(TopicName topicName, - boolean isBatch, - Schema schema, - CompressionType compressionType) throws Exception { - @Cleanup - Producer producer = pulsarClient.newProducer(schema) - .topic(topicName.toString()) - .enableBatching(isBatch) - .compressionType(compressionType) - .create(); - - for (int i = 0 ; i < NUM_OF_STOCKS; ++i) { - final StockProtoMessage.Stock stock = StockProtoMessage.Stock.newBuilder(). - setEntryId(i).setSymbol("STOCK_" + i).setSharePrice(100.0 + i * 10).build(); - producer.send(stock); - } - producer.flush(); - } - - private void prepareDataForKeyValueSchema(TopicName topicName, - Schema> schema, - CompressionType compressionType) throws Exception { - @Cleanup - Producer> producer = pulsarClient.newProducer(schema) - .topic(topicName.toString()) - .compressionType(compressionType) - .create(); - - for (int i = 0 ; i < NUM_OF_STOCKS; ++i) { - int j = 100 * i; - final Stock stock1 = new Stock(j, "STOCK_" + j, 100.0 + j * 10); - final Stock stock2 = new Stock(i, "STOCK_" + i, 100.0 + i * 10); - producer.send(new KeyValue<>(stock1, stock2)); - } - } - - @Override - protected void validateContent(int messageNum, String[] contentArr, Schema schema) { - switch (schema.getSchemaInfo().getType()) { - case BYTES: - log.info("Skip validate content for BYTES schema type."); - break; - case STRING: - validateContentForStringSchema(messageNum, contentArr); - log.info("finish validate content for STRING schema type."); - break; - case JSON: - case AVRO: - case PROTOBUF_NATIVE: - validateContentForStructSchema(messageNum, contentArr); - log.info("finish validate content for {} schema type.", schema.getSchemaInfo().getType()); - break; - case KEY_VALUE: - validateContentForKeyValueSchema(messageNum, contentArr); - log.info("finish validate content for KEY_VALUE {} schema type.", - ((KeyValueSchemaImpl) schema).getKeyValueEncodingType()); - } - } - - private void validateContentForStringSchema(int messageNum, String[] contentArr) { - for (int i = 0; i < messageNum; i++) { - assertThat(contentArr).contains("\"string" + i + "\""); - } - } - - private void validateContentForStructSchema(int messageNum, String[] contentArr) { - for (int i = 0; i < messageNum; ++i) { - assertThat(contentArr).contains("\"" + i + "\""); - assertThat(contentArr).contains("\"" + "STOCK_" + i + "\""); - assertThat(contentArr).contains("\"" + (100.0 + i * 10) + "\""); - } - } - - private void validateContentForKeyValueSchema(int messageNum, String[] contentArr) { - for (int i = 0; i < messageNum; ++i) { - int j = 100 * i; - assertThat(contentArr).contains("\"" + i + "\""); - assertThat(contentArr).contains("\"" + "STOCK_" + i + "\""); - assertThat(contentArr).contains("\"" + (100.0 + i * 10) + "\""); - - assertThat(contentArr).contains("\"" + j + "\""); - assertThat(contentArr).contains("\"" + "STOCK_" + j + "\""); - assertThat(contentArr).contains("\"" + (100.0 + j * 10) + "\""); - } - } - - @Test(timeOut = 1000 * 30) - public void testQueueBigEntry() throws Exception { - String tableName = "big_data_" + randomName(5); - String topic = "persistent://public/default/" + tableName; - - @Cleanup - Producer producer = pulsarClient.newProducer(Schema.BYTES) - .topic(topic) - .enableBatching(false) - .create(); - - // Make sure that the data length bigger than the default maxMessageSize - int dataLength = Commands.DEFAULT_MAX_MESSAGE_SIZE + 2 * 1024 * 1024; - Assert.assertTrue(dataLength < pulsarCluster.getSpec().maxMessageSize()); - byte[] data = new byte[dataLength]; - for (int i = 0; i < dataLength; i++) { - data[i] = 'a'; - } - - int messageCnt = 5; - log.info("start produce big entry data, data length: {}", dataLength); - for (int i = 0 ; i < messageCnt; ++i) { - producer.newMessage().value(data).send(); - } - - int count = selectCount("public/default", tableName); - Assert.assertEquals(count, messageCnt); - } - -} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPrestoQueryTieredStorage.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPrestoQueryTieredStorage.java deleted file mode 100644 index 5fba3b5eba60c..0000000000000 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPrestoQueryTieredStorage.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.tests.integration.presto; - -import static com.google.common.base.Preconditions.checkNotNull; -import static org.assertj.core.api.Assertions.assertThat; - -import lombok.Cleanup; -import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.client.BookKeeper; -import org.apache.bookkeeper.conf.ClientConfiguration; -import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.api.CompressionType; -import org.apache.pulsar.client.api.Consumer; -import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.api.SubscriptionInitialPosition; -import org.apache.pulsar.client.impl.MessageIdImpl; -import org.apache.pulsar.client.impl.schema.JSONSchema; -import org.apache.pulsar.common.naming.NamespaceName; -import org.apache.pulsar.common.naming.TopicDomain; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.tests.integration.containers.S3Container; -import org.testng.Assert; -import org.testng.annotations.Test; - -/** - * Test presto query from tiered storage, the Pulsar SQL is cluster mode. - */ -@Slf4j -public class TestPrestoQueryTieredStorage extends TestPulsarSQLBase { - - private final String TENANT = "presto"; - private final String NAMESPACE = "ts"; - - private S3Container s3Container; - - @Override - public void setupCluster() throws Exception { - super.setupCluster(); - setupExtraContainers(); - } - - @Override - public void tearDownCluster() throws Exception { - teardownPresto(); - super.tearDownCluster(); - } - - private void setupExtraContainers() throws Exception { - log.info("[TestPrestoQueryTieredStorage] setupExtraContainers..."); - pulsarCluster.runAdminCommandOnAnyBroker( "tenants", - "create", "--allowed-clusters", pulsarCluster.getClusterName(), - "--admin-roles", "offload-admin", TENANT); - - pulsarCluster.runAdminCommandOnAnyBroker( - "namespaces", - "create", "--clusters", pulsarCluster.getClusterName(), - NamespaceName.get(TENANT, NAMESPACE).toString()); - - s3Container = new S3Container( - pulsarCluster.getClusterName(), - S3Container.NAME) - .withNetwork(pulsarCluster.getNetwork()) - .withNetworkAliases(S3Container.NAME); - s3Container.start(); - - String offloadProperties = getOffloadProperties(BUCKET, null, ENDPOINT); - pulsarCluster.startPrestoWorker(OFFLOAD_DRIVER, offloadProperties); - pulsarCluster.startPrestoFollowWorkers(1, OFFLOAD_DRIVER, offloadProperties); - initJdbcConnection(); - } - - private String getOffloadProperties(String bucket, String region, String endpoint) { - checkNotNull(bucket); - StringBuilder sb = new StringBuilder(); - sb.append("{"); - sb.append("\"s3ManagedLedgerOffloadBucket\":").append("\"").append(bucket).append("\","); - if (StringUtils.isNotEmpty(region)) { - sb.append("\"s3ManagedLedgerOffloadRegion\":").append("\"").append(region).append("\","); - } - if (StringUtils.isNotEmpty(endpoint)) { - sb.append("\"s3ManagedLedgerOffloadServiceEndpoint\":").append("\"").append(endpoint).append("\""); - } - sb.append("}"); - return sb.toString(); - } - - public void teardownPresto() { - log.info("[TestPrestoQueryTieredStorage] tearing down..."); - if (null != s3Container) { - s3Container.stop(); - } - - pulsarCluster.stopPrestoWorker(); - } - - @Test - public void testQueryTieredStorage1() throws Exception { - TopicName topicName = TopicName.get( - TopicDomain.persistent.value(), TENANT, NAMESPACE, "stocks_ts_nons_" + randomName(5)); - pulsarSQLBasicTest(topicName, false, false, JSONSchema.of(Stock.class), CompressionType.NONE); - } - - @Test - public void testQueryTieredStorage2() throws Exception { - TopicName topicName = TopicName.get( - TopicDomain.persistent.value(), TENANT, NAMESPACE, "stocks_ts_ns_" + randomName(5)); - pulsarSQLBasicTest(topicName, false, true, JSONSchema.of(Stock.class), CompressionType.NONE); - } - - @Override - protected int prepareData(TopicName topicName, - boolean isBatch, - boolean useNsOffloadPolices, - Schema schema, - CompressionType compressionType) throws Exception { - @Cleanup - Consumer consumer = pulsarClient.newConsumer(JSONSchema.of(Stock.class)) - .topic(topicName.toString()) - .subscriptionName("test") - .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) - .subscribe(); - - @Cleanup - Producer producer = pulsarClient.newProducer(JSONSchema.of(Stock.class)) - .topic(topicName.toString()) - .compressionType(compressionType) - .create(); - - long firstLedgerId = -1; - int sendMessageCnt = 0; - while (true) { - Stock stock = new Stock( - sendMessageCnt,"STOCK_" + sendMessageCnt, 100.0 + sendMessageCnt * 10); - MessageIdImpl messageId = (MessageIdImpl) producer.send(stock); - sendMessageCnt ++; - if (firstLedgerId == -1) { - firstLedgerId = messageId.getLedgerId(); - } - if (messageId.getLedgerId() > firstLedgerId) { - log.info("ledger rollover firstLedgerId: {}, currentLedgerId: {}", - firstLedgerId, messageId.getLedgerId()); - break; - } - Thread.sleep(100); - } - - offloadAndDeleteFromBK(useNsOffloadPolices, topicName); - return sendMessageCnt; - } - - private void offloadAndDeleteFromBK(boolean useNsOffloadPolices, TopicName topicName) { - String adminUrl = pulsarCluster.getHttpServiceUrl(); - try (PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(adminUrl).build()) { - // read managed ledger info, check ledgers exist - long firstLedger = admin.topics().getInternalStats(topicName.toString()).ledgers.get(0).ledgerId; - - String output = ""; - - if (useNsOffloadPolices) { - pulsarCluster.runAdminCommandOnAnyBroker( - "namespaces", "set-offload-policies", - "--bucket", "pulsar-integtest", - "--driver", OFFLOAD_DRIVER, - "--endpoint", "http://" + S3Container.NAME + ":9090", - "--offloadAfterElapsed", "1000", - topicName.getNamespace()); - - output = pulsarCluster.runAdminCommandOnAnyBroker( - "namespaces", "get-offload-policies", topicName.getNamespace()).getStdout(); - Assert.assertTrue(output.contains("pulsar-integtest")); - Assert.assertTrue(output.contains(OFFLOAD_DRIVER)); - } - - // offload with a low threshold - output = pulsarCluster.runAdminCommandOnAnyBroker("topics", - "offload", "--size-threshold", "0", topicName.toString()).getStdout(); - Assert.assertTrue(output.contains("Offload triggered")); - - output = pulsarCluster.runAdminCommandOnAnyBroker("topics", - "offload-status", "-w", topicName.toString()).getStdout(); - Assert.assertTrue(output.contains("Offload was a success")); - - // delete the first ledger, so that we cannot possibly read from it - ClientConfiguration bkConf = new ClientConfiguration(); - bkConf.setZkServers(pulsarCluster.getZKConnString()); - try (BookKeeper bk = new BookKeeper(bkConf)) { - bk.deleteLedger(firstLedger); - } catch (Exception e) { - log.error("Failed to delete from BookKeeper.", e); - Assert.fail("Failed to delete from BookKeeper."); - } - - // Unload topic to clear all caches, open handles, etc - admin.topics().unload(topicName.toString()); - } catch (Exception e) { - Assert.fail("Failed to deleteOffloadedDataFromBK."); - } - } - - @Override - protected void validateContent(int messageNum, String[] contentArr, Schema schema) { - for (int i = 0; i < messageNum; ++i) { - assertThat(contentArr).contains("\"" + i + "\""); - assertThat(contentArr).contains("\"" + "STOCK_" + i + "\""); - assertThat(contentArr).contains("\"" + (100.0 + i * 10) + "\""); - } - } - -} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPulsarSQLAuth.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPulsarSQLAuth.java deleted file mode 100644 index 0a9bb5e19592a..0000000000000 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPulsarSQLAuth.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.tests.integration.presto; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; -import io.jsonwebtoken.SignatureAlgorithm; -import java.time.Duration; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import javax.crypto.SecretKey; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.common.policies.data.AuthAction; -import org.apache.pulsar.tests.integration.containers.BrokerContainer; -import org.apache.pulsar.tests.integration.containers.PrestoWorkerContainer; -import org.apache.pulsar.tests.integration.docker.ContainerExecException; -import org.apache.pulsar.tests.integration.docker.ContainerExecResult; -import org.apache.pulsar.tests.integration.topologies.PulsarClusterSpec; -import org.awaitility.Awaitility; -import org.testng.annotations.Test; - -@Slf4j -public class TestPulsarSQLAuth extends TestPulsarSQLBase { - private SecretKey secretKey; - private String adminToken; - private PulsarAdmin admin; - - @Override - protected PulsarClusterSpec.PulsarClusterSpecBuilder beforeSetupCluster(String clusterName, - PulsarClusterSpec.PulsarClusterSpecBuilder specBuilder) { - specBuilder = super.beforeSetupCluster(clusterName, specBuilder); - specBuilder.enablePrestoWorker(true); - return specBuilder; - } - - @Override - protected void beforeStartCluster() { - secretKey = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); - adminToken = AuthTokenUtils.createToken(secretKey, "admin", Optional.empty()); - - Map envMap = new HashMap<>(); - envMap.put("authenticationEnabled", "true"); - envMap.put("authenticationProviders", "org.apache.pulsar.broker.authentication.AuthenticationProviderToken"); - envMap.put("authorizationEnabled", "true"); - envMap.put("tokenSecretKey", AuthTokenUtils.encodeKeyBase64(secretKey)); - envMap.put("superUserRoles", "admin"); - envMap.put("brokerDeleteInactiveTopicsEnabled", "false"); - - for (BrokerContainer brokerContainer : pulsarCluster.getBrokers()) { - brokerContainer.withEnv(envMap); - } - - PrestoWorkerContainer prestoWorkerContainer = pulsarCluster.getPrestoWorkerContainer(); - - prestoWorkerContainer - .withEnv("SQL_PREFIX_pulsar.auth-plugin", "org.apache.pulsar.client.impl.auth.AuthenticationToken") - .withEnv("SQL_PREFIX_pulsar.auth-params", adminToken) - .withEnv("pulsar.broker-binary-service-url", "pulsar://pulsar-broker-0:6650") - .withEnv("pulsar.authorization-enabled", "true"); - - } - - @Override - public void setupCluster() throws Exception { - super.setupCluster(); - initJdbcConnection(); - admin = PulsarAdmin.builder() - .serviceHttpUrl(pulsarCluster.getHttpServiceUrl()) - .authentication("org.apache.pulsar.client.impl.auth.AuthenticationToken", adminToken) - .build(); - } - - @Override - public void tearDownCluster() throws Exception { - super.tearDownCluster(); - } - - @Test - public void testPulsarSQLAuthCheck() throws PulsarAdminException { - String passRole = RandomStringUtils.randomAlphabetic(4) + "-pass"; - String deniedRole = RandomStringUtils.randomAlphabetic(4) + "-denied"; - String passToken = AuthTokenUtils.createToken(secretKey, passRole, Optional.empty()); - String deniedToken = AuthTokenUtils.createToken(secretKey, deniedRole, Optional.empty()); - String topic = "testPulsarSQLAuthCheck"; - - admin.topics().grantPermission(topic, passRole, EnumSet.of(AuthAction.consume)); - - admin.topics().createNonPartitionedTopic(topic); - - String queryAllDataSql = String.format("select * from pulsar.\"%s\".\"%s\";", "public/default", topic); - - assertSQLExecution( - () -> { - try { - ContainerExecResult containerExecResult = - execQuery(queryAllDataSql, new HashMap<>() {{ - put("auth-plugin", - "org.apache.pulsar.client.impl.auth.AuthenticationToken"); - put("auth-params", passToken); - }}); - assertEquals(containerExecResult.getExitCode(), 0); - } catch (ContainerExecException e) { - fail(String.format("assertSQLExecution fail: %s", e.getLocalizedMessage())); - } - } - ); - - assertSQLExecution( - () -> { - try { - execQuery(queryAllDataSql, new HashMap<>() {{ - put("auth-plugin", - "org.apache.pulsar.client.impl.auth.AuthenticationToken"); - put("auth-params", "invalid-token"); - }}); - fail("Should not pass"); - } catch (ContainerExecException e) { - // Authorization error - assertEquals(e.getResult().getExitCode(), 1); - log.info(e.getResult().getStderr()); - assertTrue(e.getResult().getStderr().contains("Failed to authenticate")); - } - } - ); - - assertSQLExecution( - () -> { - try { - execQuery(queryAllDataSql, new HashMap<>() {{ - put("auth-plugin", - "org.apache.pulsar.client.impl.auth.AuthenticationToken"); - put("auth-params", deniedToken); - }}); - fail("Should not pass"); - } catch (ContainerExecException e) { - // Authorization error - assertEquals(e.getResult().getExitCode(), 1); - log.info(e.getResult().getStderr()); - assertTrue(e.getResult().getStderr().contains("not authorized")); - } - } - ); - } - - @Test - public void testCheckAuthForMultipleTopics() throws PulsarAdminException { - String testRole = RandomStringUtils.randomAlphabetic(4) + "-test"; - String testToken = AuthTokenUtils.createToken(secretKey, testRole, Optional.empty()); - String topic1 = "testCheckAuthForMultipleTopics1"; - String topic2 = "testCheckAuthForMultipleTopics2"; - - admin.topics().grantPermission(topic1, testRole, EnumSet.of(AuthAction.consume)); - - admin.topics().createNonPartitionedTopic(topic1); - - admin.topics().createPartitionedTopic(topic2, 2); // Test for partitioned topic - - String queryAllDataSql = - String.format("select * from pulsar.\"public/default\".\"%s\", pulsar.\"public/default\".\"%s\";", - topic1, topic2); - - assertSQLExecution( - () -> { - try { - execQuery(queryAllDataSql, new HashMap<>() {{ - put("auth-plugin", - "org.apache.pulsar.client.impl.auth.AuthenticationToken"); - put("auth-params", testToken); - }}); - fail("Should not pass"); - } catch (ContainerExecException e) { - // Authorization error - assertEquals(e.getResult().getExitCode(), 1); - log.info(e.getResult().getStderr()); - } - } - ); - - admin.topics().grantPermission(topic2, testRole, EnumSet.of(AuthAction.consume)); - - assertSQLExecution( - () -> { - try { - ContainerExecResult containerExecResult = - execQuery(queryAllDataSql, new HashMap<>() {{ - put("auth-plugin", - "org.apache.pulsar.client.impl.auth.AuthenticationToken"); - put("auth-params", testToken); - }}); - - assertEquals(containerExecResult.getExitCode(), 0); - } catch (ContainerExecException e) { - fail(String.format("assertSQLExecution fail: %s", e.getLocalizedMessage())); - } - } - ); - } - - private void assertSQLExecution(org.awaitility.core.ThrowingRunnable assertion) { - Awaitility.await() - .pollDelay(Duration.ofMillis(0)) - .pollInterval(Duration.ofSeconds(3)) - .atMost(Duration.ofSeconds(15)) - .untilAsserted(assertion); - } -} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPulsarSQLBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPulsarSQLBase.java deleted file mode 100644 index 2833be67bc8e3..0000000000000 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPulsarSQLBase.java +++ /dev/null @@ -1,325 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.tests.integration.presto; - -import static org.assertj.core.api.Assertions.assertThat; - -import com.google.common.base.Stopwatch; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Timestamp; -import java.time.Duration; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import lombok.extern.slf4j.Slf4j; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.Response; -import org.apache.pulsar.client.api.CompressionType; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.schema.SchemaType; -import org.apache.pulsar.tests.integration.docker.ContainerExecException; -import org.apache.pulsar.tests.integration.docker.ContainerExecResult; -import org.apache.pulsar.tests.integration.suites.PulsarSQLTestSuite; -import org.apache.pulsar.tests.integration.topologies.PulsarCluster; -import org.awaitility.Awaitility; -import org.testng.Assert; -import org.testng.annotations.DataProvider; - - -/** - * Pulsar SQL test base. - */ -@Slf4j -public class TestPulsarSQLBase extends PulsarSQLTestSuite { - - protected void pulsarSQLBasicTest(TopicName topic, - boolean isBatch, - boolean useNsOffloadPolices, - Schema schema, - CompressionType compressionType) throws Exception { - log.info("Pulsar SQL basic test. topic: {}", topic); - - waitPulsarSQLReady(); - - log.info("start prepare data for query. topic: {}", topic); - int messageCnt = prepareData(topic, isBatch, useNsOffloadPolices, schema, compressionType); - log.info("finish prepare data for query. topic: {}, messageCnt: {}", topic, messageCnt); - - validateMetadata(topic); - - validateData(topic, messageCnt, schema); - - log.info("Finish Pulsar SQL basic test. topic: {}", topic); - } - - @DataProvider(name = "batchingAndCompression") - public static Object[][] batchingAndCompression() { - return new Object[][] { - { true, CompressionType.ZLIB }, - { true, CompressionType.ZSTD }, - { true, CompressionType.SNAPPY }, - { true, CompressionType.LZ4 }, - { true, CompressionType.NONE }, - { false, CompressionType.ZLIB }, - { false, CompressionType.ZSTD }, - { false, CompressionType.SNAPPY }, - { false, CompressionType.LZ4 }, - { false, CompressionType.NONE }, - }; - } - - public void waitPulsarSQLReady() throws Exception { - // wait until presto worker started - ContainerExecResult result; - do { - try { - result = execQuery("show catalogs;"); - assertThat(result.getExitCode()).isEqualTo(0); - assertThat(result.getStdout()).contains("pulsar", "system"); - break; - } catch (ContainerExecException cee) { - if (cee.getResult().getStderr().contains("Presto server is still initializing")) { - Thread.sleep(10000); - } else { - throw cee; - } - } - } while (true); - - // check presto follow workers start finish. - if (pulsarCluster.getSqlFollowWorkerContainers() != null - && pulsarCluster.getSqlFollowWorkerContainers().size() > 0) { - OkHttpClient okHttpClient = new OkHttpClient(); - Request request = new Request.Builder() - .header("X-Trino-User", "test-user") - .url("http://" + pulsarCluster.getPrestoWorkerContainer().getUrl() + "/v1/node") - .build(); - do { - try (Response response = okHttpClient.newCall(request).execute()) { - Assert.assertNotNull(response.body()); - String nodeJsonStr = response.body().string(); - Assert.assertTrue(nodeJsonStr.length() > 0); - log.info("presto node info: {}", nodeJsonStr); - if (nodeJsonStr.contains("uri")) { - log.info("presto node exist."); - break; - } - Thread.sleep(1000); - } - } while (true); - } - } - - protected int prepareData(TopicName topicName, - boolean isBatch, - boolean useNsOffloadPolices, - Schema schema, - CompressionType compressionType) throws Exception { - throw new Exception("Unsupported operation prepareData."); - } - - public void validateMetadata(TopicName topicName) throws Exception { - ContainerExecResult result = execQuery("show schemas in pulsar;"); - assertThat(result.getExitCode()).isEqualTo(0); - assertThat(result.getStdout()).contains(topicName.getNamespace()); - - pulsarCluster.getBroker(0) - .execCmd( - "/bin/bash", - "-c", "bin/pulsar-admin namespaces unload " + topicName.getNamespace()); - - Awaitility.await().untilAsserted( - () -> { - ContainerExecResult r = execQuery( - String.format("show tables in pulsar.\"%s\";", topicName.getNamespace())); - assertThat(r.getExitCode()).isEqualTo(0); - // the show tables query return lowercase table names, so ignore case - assertThat(r.getStdout()).containsIgnoringCase(topicName.getLocalName()); - } - ); - } - - protected void validateContent(int messageNum, String[] contentArr, Schema schema) throws Exception { - throw new Exception("Unsupported operation validateContent."); - } - - private void validateData(TopicName topicName, int messageNum, Schema schema) throws Exception { - String namespace = topicName.getNamespace(); - String topic = topicName.getLocalName(); - - final String queryAllDataSql; - if (schema.getSchemaInfo().getType().isStruct() - || schema.getSchemaInfo().getType().equals(SchemaType.KEY_VALUE)) { - queryAllDataSql = String.format("select * from pulsar.\"%s\".\"%s\" order by entryid;", namespace, topic); - } else { - queryAllDataSql = String.format("select * from pulsar.\"%s\".\"%s\";", namespace, topic); - } - - Awaitility.await() - // first poll immediately - .pollDelay(Duration.ofMillis(0)) - // use relatively long poll interval so that polling doesn't consume too much resources - .pollInterval(Duration.ofSeconds(3)) - // retry up to 15 seconds from first attempt - .atMost(Duration.ofSeconds(15)) - .untilAsserted( - () -> { - ContainerExecResult containerExecResult = execQuery(queryAllDataSql); - assertThat(containerExecResult.getExitCode()).isEqualTo(0); - log.info("select sql query output \n{}", containerExecResult.getStdout()); - String[] split = containerExecResult.getStdout().split("\n"); - assertThat(split.length).isEqualTo(messageNum); - String[] contentArr = containerExecResult.getStdout().split("\n|,"); - validateContent(messageNum, contentArr, schema); - } - ); - - // test predicate pushdown - String query = String.format("select * from pulsar" + - ".\"%s\".\"%s\" order by __publish_time__", namespace, topic); - log.info("Executing query: {}", query); - ResultSet res = connection.createStatement().executeQuery(query); - - List timestamps = new LinkedList<>(); - while (res.next()) { - printCurrent(res); - timestamps.add(res.getTimestamp("__publish_time__")); - } - log.info("Executing query: result for topic {} timestamps size {}", topic, timestamps.size()); - - assertThat(timestamps.size()).isGreaterThan(messageNum - 2); - - query = String.format("select * from pulsar" + - ".\"%s\".\"%s\" where __publish_time__ > timestamp '%s' order by __publish_time__", - namespace, topic, timestamps.get(timestamps.size() / 2)); - log.info("Executing query: {}", query); - res = connection.createStatement().executeQuery(query); - - List returnedTimestamps = new LinkedList<>(); - while (res.next()) { - printCurrent(res); - returnedTimestamps.add(res.getTimestamp("__publish_time__")); - } - - log.info("Executing query: result for topic {} returnedTimestamps size: {}", topic, returnedTimestamps.size()); - if (timestamps.size() % 2 == 0) { - // for example: total size 10, the right receive number is 4, so 4 + 1 == 10 / 2 - assertThat(returnedTimestamps.size() + 1).isEqualTo(timestamps.size() / 2); - } else { - // for example: total size 101, the right receive number is 50, so 50 == (101 - 1) / 2 - assertThat(returnedTimestamps.size()).isEqualTo((timestamps.size() - 1) / 2); - } - - // Try with a predicate that has a earlier time than any entry - // Should return all rows - query = String.format("select * from pulsar.\"%s\".\"%s\" where " - + "__publish_time__ > from_unixtime(%s) order by __publish_time__", namespace, topic, 0); - log.info("Executing query: {}", query); - res = connection.createStatement().executeQuery(query); - - returnedTimestamps = new LinkedList<>(); - while (res.next()) { - printCurrent(res); - returnedTimestamps.add(res.getTimestamp("__publish_time__")); - } - - log.info("Executing query: result for topic {} returnedTimestamps size: {}", topic, returnedTimestamps.size()); - assertThat(returnedTimestamps.size()).isEqualTo(timestamps.size()); - - // Try with a predicate that has a latter time than any entry - // Should return no rows - - query = String.format("select * from pulsar.\"%s\".\"%s\" where " - + "__publish_time__ > from_unixtime(%s) order by __publish_time__", namespace, topic, 99999999999L); - log.info("Executing query: {}", query); - res = connection.createStatement().executeQuery(query); - - returnedTimestamps = new LinkedList<>(); - while (res.next()) { - printCurrent(res); - returnedTimestamps.add(res.getTimestamp("__publish_time__")); - } - - log.info("Executing query: result for topic {} returnedTimestamps size: {}", topic, returnedTimestamps.size()); - assertThat(returnedTimestamps.size()).isEqualTo(0); - - int count = selectCount(namespace, topic); - assertThat(count).isGreaterThan(messageNum - 2); - } - - public ContainerExecResult execQuery(final String query) throws Exception { - return execQuery(query, null); - } - - public ContainerExecResult execQuery(final String query, Map extraCredentials) throws Exception { - ContainerExecResult containerExecResult; - - StringBuilder extraCredentialsString = new StringBuilder(" "); - - if (extraCredentials != null) { - for (Map.Entry entry : extraCredentials.entrySet()) { - extraCredentialsString.append( - String.format("--extra-credential %s=%s ", entry.getKey(), entry.getValue())); - } - } - - containerExecResult = pulsarCluster.getPrestoWorkerContainer() - .execCmd("/bin/bash", "-c", - PulsarCluster.PULSAR_COMMAND_SCRIPT + " sql" + extraCredentialsString + "--execute " + "'" - + query + "'"); - - Stopwatch sw = Stopwatch.createStarted(); - while (containerExecResult.getExitCode() != 0 && sw.elapsed(TimeUnit.SECONDS) < 120) { - TimeUnit.MILLISECONDS.sleep(500); - containerExecResult = pulsarCluster.getPrestoWorkerContainer() - .execCmd("/bin/bash", "-c", - PulsarCluster.PULSAR_COMMAND_SCRIPT + " sql" + extraCredentialsString + "--execute " + "'" - + query + "'"); - } - - return containerExecResult; - - } - - private static void printCurrent(ResultSet rs) throws SQLException { - ResultSetMetaData rsmd = rs.getMetaData(); - int columnsNumber = rsmd.getColumnCount(); - for (int i = 1; i <= columnsNumber; i++) { - if (i > 1) System.out.print(", "); - String columnValue = rs.getString(i); - System.out.print(columnValue + " " + rsmd.getColumnName(i)); - } - System.out.println(""); - - } - - protected int selectCount(String namespace, String tableName) throws SQLException { - String query = String.format("select count(*) from pulsar.\"%s\".\"%s\"", namespace, tableName); - log.info("Executing count query: {}", query); - ResultSet res = connection.createStatement().executeQuery(query); - res.next(); - return res.getInt("_col0"); - } - -} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaTest.java index 8bb6de74c661d..d0421063b2d90 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaTest.java @@ -31,6 +31,8 @@ import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.tests.integration.schema.Schemas.Person; import org.apache.pulsar.tests.integration.schema.Schemas.PersonConsumeSchema; import org.apache.pulsar.tests.integration.schema.Schemas.Student; @@ -316,5 +318,14 @@ public void testPrimitiveSchemaTypeCompatibilityCheck() { } + @Test + public void testDeletePartitionedTopicWhenTopicReferenceIsNotReady() throws Exception { + final String topic = "persistent://public/default/tp-ref"; + admin.topics().createPartitionedTopic(topic, 20); + admin.schemas().createSchema(topic, + SchemaInfo.builder().type(SchemaType.STRING).schema(new byte[0]).build()); + admin.topics().deletePartitionedTopic(topic, false); + } + } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/suites/PulsarSQLTestSuite.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/suites/PulsarSQLTestSuite.java deleted file mode 100644 index 029f408ef6661..0000000000000 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/suites/PulsarSQLTestSuite.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.tests.integration.suites; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.HashMap; -import java.util.Map; - -import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.common.protocol.Commands; -import org.apache.pulsar.tests.integration.containers.BrokerContainer; -import org.apache.pulsar.tests.integration.containers.S3Container; -import org.apache.pulsar.tests.integration.topologies.PulsarClusterSpec; - -/** - * Pulsar SQL test suite. - */ -@Slf4j -public abstract class PulsarSQLTestSuite extends PulsarTestSuite { - - public static final int ENTRIES_PER_LEDGER = 100; - public static final String OFFLOAD_DRIVER = "aws-s3"; - public static final String BUCKET = "pulsar-integtest"; - public static final String ENDPOINT = "http://" + S3Container.NAME + ":9090"; - - protected Connection connection = null; - protected PulsarClient pulsarClient = null; - - @Override - protected PulsarClusterSpec.PulsarClusterSpecBuilder beforeSetupCluster(String clusterName, PulsarClusterSpec.PulsarClusterSpecBuilder specBuilder) { - specBuilder.queryLastMessage(true); - specBuilder.clusterName("pulsar-sql-test"); - specBuilder.numBrokers(1); - specBuilder.maxMessageSize(2 * Commands.DEFAULT_MAX_MESSAGE_SIZE); - return super.beforeSetupCluster(clusterName, specBuilder); - } - - @Override - protected void beforeStartCluster() throws Exception { - Map envMap = new HashMap<>(); - envMap.put("managedLedgerMaxEntriesPerLedger", String.valueOf(ENTRIES_PER_LEDGER)); - envMap.put("managedLedgerMinLedgerRolloverTimeMinutes", "0"); - envMap.put("managedLedgerOffloadDriver", OFFLOAD_DRIVER); - envMap.put("s3ManagedLedgerOffloadBucket", BUCKET); - envMap.put("s3ManagedLedgerOffloadServiceEndpoint", ENDPOINT); - - for (BrokerContainer brokerContainer : pulsarCluster.getBrokers()) { - brokerContainer.withEnv(envMap); - } - } - - @Override - public void setupCluster() throws Exception { - super.setupCluster(); - pulsarClient = PulsarClient.builder() - .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) - .build(); - } - - protected void initJdbcConnection() throws SQLException { - if (pulsarCluster.getPrestoWorkerContainer() == null) { - log.error("The presto work container isn't exist."); - return; - } - String url = String.format("jdbc:trino://%s", pulsarCluster.getPrestoWorkerContainer().getUrl()); - connection = DriverManager.getConnection(url, "test", null); - } - - @Override - public void tearDownCluster() throws Exception { - close(); - super.tearDownCluster(); - } - - protected void close() { - if (connection != null) { - try { - connection.close(); - } catch (SQLException e) { - log.error("Failed to close sql connection.", e); - } - } - if (pulsarClient != null) { - try { - pulsarClient.close(); - } catch (PulsarClientException e) { - log.error("Failed to close pulsar client.", e); - } - } - } -} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/tls/ClientTlsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/tls/ClientTlsTest.java index 59ff978cafa06..080912cd49262 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/tls/ClientTlsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/tls/ClientTlsTest.java @@ -29,6 +29,7 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.tests.integration.suites.PulsarTestSuite; +import org.apache.pulsar.tests.integration.topologies.PulsarClusterSpec; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -41,6 +42,14 @@ private static String loadCertificateAuthorityFile(String name) { return Resources.getResource("certificate-authority/" + name).getPath(); } + @Override + protected PulsarClusterSpec.PulsarClusterSpecBuilder beforeSetupCluster( + String clusterName, + PulsarClusterSpec.PulsarClusterSpecBuilder specBuilder) { + specBuilder.enableTls(true); + return specBuilder; + } + @DataProvider(name = "adminUrls") public Object[][] adminUrls() { return new Object[][]{ diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java index fcc0feec6d44f..90f08a9639471 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java @@ -35,18 +35,21 @@ import java.util.Map; import java.util.Objects; import java.util.function.Function; +import lombok.Cleanup; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; +import org.apache.pulsar.client.impl.auth.AuthenticationTls; import org.apache.pulsar.tests.integration.containers.BKContainer; import org.apache.pulsar.tests.integration.containers.BrokerContainer; import org.apache.pulsar.tests.integration.containers.CSContainer; -import org.apache.pulsar.tests.integration.containers.PrestoWorkerContainer; import org.apache.pulsar.tests.integration.containers.ProxyContainer; import org.apache.pulsar.tests.integration.containers.PulsarContainer; +import org.apache.pulsar.tests.integration.containers.PulsarInitMetadataContainer; import org.apache.pulsar.tests.integration.containers.WorkerContainer; import org.apache.pulsar.tests.integration.containers.ZKContainer; import org.apache.pulsar.tests.integration.docker.ContainerExecResult; +import org.apache.pulsar.tests.integration.oxia.OxiaContainer; import org.testcontainers.containers.BindMode; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.Network; @@ -69,9 +72,12 @@ public class PulsarCluster { * @return the built pulsar cluster */ public static PulsarCluster forSpec(PulsarClusterSpec spec) { - CSContainer csContainer = new CSContainer(spec.clusterName) - .withNetwork(Network.newNetwork()) - .withNetworkAliases(CSContainer.NAME); + CSContainer csContainer = null; + if (!spec.enableOxia) { + csContainer = new CSContainer(spec.clusterName) + .withNetwork(Network.newNetwork()) + .withNetworkAliases(CSContainer.NAME); + } return new PulsarCluster(spec, csContainer, false); } @@ -85,46 +91,58 @@ public static PulsarCluster forSpec(PulsarClusterSpec spec, CSContainer csContai @Getter private final String clusterName; private final Network network; - private final ZKContainer zkContainer; + private final ZKContainer zkContainer; + + private final OxiaContainer oxiaContainer; private final CSContainer csContainer; private final boolean sharedCsContainer; private final Map bookieContainers; private final Map brokerContainers; private final Map workerContainers; private final ProxyContainer proxyContainer; - private PrestoWorkerContainer prestoWorkerContainer; - @Getter - private Map sqlFollowWorkerContainers; private Map> externalServices = Collections.emptyMap(); private Map> externalServiceEnvs; - private final boolean enablePrestoWorker; + private final Map functionWorkerEnvs; + private final List functionWorkerAdditionalPorts; + + private final String metadataStoreUrl; + private final String configurationMetadataStoreUrl; private PulsarCluster(PulsarClusterSpec spec, CSContainer csContainer, boolean sharedCsContainer) { this.spec = spec; this.sharedCsContainer = sharedCsContainer; this.clusterName = spec.clusterName(); - this.network = csContainer.getNetwork(); - this.enablePrestoWorker = spec.enablePrestoWorker(); - - this.sqlFollowWorkerContainers = Maps.newTreeMap(); - if (enablePrestoWorker) { - prestoWorkerContainer = buildPrestoWorkerContainer( - PrestoWorkerContainer.NAME, true, null, null); + if (csContainer != null ) { + this.network = csContainer.getNetwork(); } else { - prestoWorkerContainer = null; + this.network = Network.newNetwork(); } - this.zkContainer = new ZKContainer(clusterName); - this.zkContainer - .withNetwork(network) - .withNetworkAliases(appendClusterName(ZKContainer.NAME)) - .withEnv("clusterName", clusterName) - .withEnv("zkServers", appendClusterName(ZKContainer.NAME)) - .withEnv("configurationStore", CSContainer.NAME + ":" + CS_PORT) - .withEnv("forceSync", "no") - .withEnv("pulsarNode", appendClusterName("pulsar-broker-0")); + + if (spec.enableOxia) { + this.zkContainer = null; + this.oxiaContainer = new OxiaContainer(clusterName); + this.oxiaContainer + .withNetwork(network) + .withNetworkAliases(appendClusterName(OxiaContainer.NAME)); + metadataStoreUrl = "oxia://" + oxiaContainer.getServiceAddress(); + configurationMetadataStoreUrl = metadataStoreUrl; + } else { + this.oxiaContainer = null; + this.zkContainer = new ZKContainer(clusterName); + this.zkContainer + .withNetwork(network) + .withNetworkAliases(appendClusterName(ZKContainer.NAME)) + .withEnv("clusterName", clusterName) + .withEnv("zkServers", appendClusterName(ZKContainer.NAME)) + .withEnv("configurationStore", CSContainer.NAME + ":" + CS_PORT) + .withEnv("forceSync", "no") + .withEnv("pulsarNode", appendClusterName("pulsar-broker-0")); + metadataStoreUrl = appendClusterName(ZKContainer.NAME); + configurationMetadataStoreUrl = CSContainer.NAME + ":" + CS_PORT; + } this.csContainer = csContainer; @@ -132,88 +150,122 @@ private PulsarCluster(PulsarClusterSpec spec, CSContainer csContainer, boolean s this.brokerContainers = Maps.newTreeMap(); this.workerContainers = Maps.newTreeMap(); - this.proxyContainer = new ProxyContainer(appendClusterName("pulsar-proxy"), ProxyContainer.NAME) + this.proxyContainer = new ProxyContainer(clusterName, appendClusterName(ProxyContainer.NAME), spec.enableTls) .withNetwork(network) .withNetworkAliases(appendClusterName("pulsar-proxy")) - .withEnv("zkServers", appendClusterName(ZKContainer.NAME)) - .withEnv("zookeeperServers", appendClusterName(ZKContainer.NAME)) - .withEnv("configurationStoreServers", CSContainer.NAME + ":" + CS_PORT) - .withEnv("clusterName", clusterName) - // enable mTLS - .withEnv("webServicePortTls", String.valueOf(BROKER_HTTPS_PORT)) - .withEnv("servicePortTls", String.valueOf(BROKER_PORT_TLS)) - .withEnv("forwardAuthorizationCredentials", "true") - .withEnv("tlsRequireTrustedClientCertOnConnect", "true") - .withEnv("tlsAllowInsecureConnection", "false") - .withEnv("tlsCertificateFilePath", "/pulsar/certificate-authority/server-keys/proxy.cert.pem") - .withEnv("tlsKeyFilePath", "/pulsar/certificate-authority/server-keys/proxy.key-pk8.pem") - .withEnv("tlsTrustCertsFilePath", "/pulsar/certificate-authority/certs/ca.cert.pem"); + .withEnv("metadataStoreUrl", metadataStoreUrl) + .withEnv("configurationMetadataStoreUrl", configurationMetadataStoreUrl) + .withEnv("clusterName", clusterName); + // enable mTLS + if (spec.enableTls) { + proxyContainer + .withEnv("webServicePortTls", String.valueOf(BROKER_HTTPS_PORT)) + .withEnv("servicePortTls", String.valueOf(BROKER_PORT_TLS)) + .withEnv("forwardAuthorizationCredentials", "true") + .withEnv("tlsRequireTrustedClientCertOnConnect", "true") + .withEnv("tlsAllowInsecureConnection", "false") + .withEnv("tlsCertificateFilePath", "/pulsar/certificate-authority/server-keys/proxy.cert.pem") + .withEnv("tlsKeyFilePath", "/pulsar/certificate-authority/server-keys/proxy.key-pk8.pem") + .withEnv("tlsTrustCertsFilePath", "/pulsar/certificate-authority/certs/ca.cert.pem") + .withEnv("brokerClientAuthenticationPlugin", AuthenticationTls.class.getName()) + .withEnv("brokerClientAuthenticationParameters", String.format("tlsCertFile:%s,tlsKeyFile:%s", + "/pulsar/certificate-authority/client-keys/admin.cert.pem", + "/pulsar/certificate-authority/client-keys/admin.key-pk8.pem")) + .withEnv("tlsEnabledWithBroker", "true") + .withEnv("brokerClientTrustCertsFilePath", "/pulsar/certificate-authority/certs/ca.cert.pem") + .withEnv("brokerClientCertificateFilePath", + "/pulsar/certificate-authority/server-keys/proxy.cert.pem") + .withEnv("brokerClientKeyFilePath", "/pulsar/certificate-authority/server-keys/proxy.key-pk8.pem"); + + } if (spec.proxyEnvs != null) { spec.proxyEnvs.forEach(this.proxyContainer::withEnv); } if (spec.proxyMountFiles != null) { spec.proxyMountFiles.forEach(this.proxyContainer::withFileSystemBind); } + if (spec.proxyAdditionalPorts != null) { + spec.proxyAdditionalPorts.forEach(this.proxyContainer::addExposedPort); + } // create bookies bookieContainers.putAll( - runNumContainers("bookie", spec.numBookies(), (name) -> new BKContainer(clusterName, name) - .withNetwork(network) - .withNetworkAliases(appendClusterName(name)) - .withEnv("zkServers", appendClusterName(ZKContainer.NAME)) - .withEnv("useHostNameAsBookieID", "true") - // Disable fsyncs for tests since they're slow within the containers - .withEnv("journalSyncData", "false") - .withEnv("journalMaxGroupWaitMSec", "0") - .withEnv("clusterName", clusterName) - .withEnv("diskUsageThreshold", "0.99") - .withEnv("nettyMaxFrameSizeBytes", "" + spec.maxMessageSize) - ) + runNumContainers("bookie", spec.numBookies(), (name) -> { + BKContainer bookieContainer = new BKContainer(clusterName, name) + .withNetwork(network) + .withNetworkAliases(appendClusterName(name)) + .withEnv("metadataServiceUri", "metadata-store:" + metadataStoreUrl) + .withEnv("useHostNameAsBookieID", "true") + // Disable fsyncs for tests since they're slow within the containers + .withEnv("journalSyncData", "false") + .withEnv("journalMaxGroupWaitMSec", "0") + .withEnv("clusterName", clusterName) + .withEnv("PULSAR_PREFIX_diskUsageWarnThreshold", "0.95") + .withEnv("diskUsageThreshold", "0.99") + .withEnv("PULSAR_PREFIX_diskUsageLwmThreshold", "0.97") + .withEnv("nettyMaxFrameSizeBytes", String.valueOf(spec.maxMessageSize)); + if (spec.bookkeeperEnvs != null) { + bookieContainer.withEnv(spec.bookkeeperEnvs); + } + if (spec.bookieAdditionalPorts != null) { + spec.bookieAdditionalPorts.forEach(bookieContainer::addExposedPort); + } + return bookieContainer; + }) ); // create brokers brokerContainers.putAll( - runNumContainers("broker", spec.numBrokers(), (name) -> { - BrokerContainer brokerContainer = new BrokerContainer(clusterName, appendClusterName(name)) - .withNetwork(network) - .withNetworkAliases(appendClusterName(name)) - .withEnv("zkServers", appendClusterName(ZKContainer.NAME)) - .withEnv("zookeeperServers", appendClusterName(ZKContainer.NAME)) - .withEnv("configurationStoreServers", CSContainer.NAME + ":" + CS_PORT) - .withEnv("clusterName", clusterName) - .withEnv("brokerServiceCompactionMonitorIntervalInSeconds", "1") - .withEnv("loadBalancerOverrideBrokerNicSpeedGbps", "1") - // used in s3 tests - .withEnv("AWS_ACCESS_KEY_ID", "accesskey").withEnv("AWS_SECRET_KEY", "secretkey") - .withEnv("maxMessageSize", "" + spec.maxMessageSize) - // enable mTLS - .withEnv("webServicePortTls", String.valueOf(BROKER_HTTPS_PORT)) - .withEnv("brokerServicePortTls", String.valueOf(BROKER_PORT_TLS)) - .withEnv("authenticateOriginalAuthData", "true") - .withEnv("tlsRequireTrustedClientCertOnConnect", "true") - .withEnv("tlsAllowInsecureConnection", "false") - .withEnv("tlsCertificateFilePath", "/pulsar/certificate-authority/server-keys/broker.cert.pem") - .withEnv("tlsKeyFilePath", "/pulsar/certificate-authority/server-keys/broker.key-pk8.pem") - .withEnv("tlsTrustCertsFilePath", "/pulsar/certificate-authority/certs/ca.cert.pem"); - if (spec.queryLastMessage) { - brokerContainer.withEnv("bookkeeperExplicitLacIntervalInMills", "10"); - brokerContainer.withEnv("bookkeeperUseV2WireProtocol", "false"); - } - if (spec.brokerEnvs != null) { - brokerContainer.withEnv(spec.brokerEnvs); - } - if (spec.brokerMountFiles != null) { - spec.brokerMountFiles.forEach(brokerContainer::withFileSystemBind); - } - if (spec.brokerAdditionalPorts() != null) { - spec.brokerAdditionalPorts().forEach(brokerContainer::addExposedPort); - } - return brokerContainer; - } - )); + runNumContainers("broker", spec.numBrokers(), (name) -> { + BrokerContainer brokerContainer = + new BrokerContainer(clusterName, appendClusterName(name), spec.enableTls) + .withNetwork(network) + .withNetworkAliases(appendClusterName(name)) + .withEnv("metadataStoreUrl", metadataStoreUrl) + .withEnv("configurationMetadataStoreUrl", configurationMetadataStoreUrl) + .withEnv("clusterName", clusterName) + .withEnv("brokerServiceCompactionMonitorIntervalInSeconds", "1") + .withEnv("loadBalancerOverrideBrokerNicSpeedGbps", "1") + // used in s3 tests + .withEnv("AWS_ACCESS_KEY_ID", "accesskey").withEnv("AWS_SECRET_KEY", + "secretkey") + .withEnv("maxMessageSize", "" + spec.maxMessageSize); + if (spec.enableTls) { + // enable mTLS + brokerContainer + .withEnv("webServicePortTls", String.valueOf(BROKER_HTTPS_PORT)) + .withEnv("brokerServicePortTls", String.valueOf(BROKER_PORT_TLS)) + .withEnv("authenticateOriginalAuthData", "true") + .withEnv("tlsAllowInsecureConnection", "false") + .withEnv("tlsRequireTrustedClientCertOnConnect", "true") + .withEnv("tlsTrustCertsFilePath", "/pulsar/certificate-authority/certs/ca" + + ".cert.pem") + .withEnv("tlsCertificateFilePath", + "/pulsar/certificate-authority/server-keys/broker.cert.pem") + .withEnv("tlsKeyFilePath", + "/pulsar/certificate-authority/server-keys/broker.key-pk8.pem"); + } + if (spec.queryLastMessage) { + brokerContainer.withEnv("bookkeeperExplicitLacIntervalInMills", "10"); + brokerContainer.withEnv("bookkeeperUseV2WireProtocol", "false"); + } + if (spec.brokerEnvs != null) { + brokerContainer.withEnv(spec.brokerEnvs); + } + if (spec.brokerMountFiles != null) { + spec.brokerMountFiles.forEach(brokerContainer::withFileSystemBind); + } + if (spec.brokerAdditionalPorts() != null) { + spec.brokerAdditionalPorts().forEach(brokerContainer::addExposedPort); + } + return brokerContainer; + } + )); spec.classPathVolumeMounts.forEach((key, value) -> { - zkContainer.withClasspathResourceMapping(key, value, BindMode.READ_WRITE); + if (zkContainer != null) { + zkContainer.withClasspathResourceMapping(key, value, BindMode.READ_WRITE); + } proxyContainer.withClasspathResourceMapping(key, value, BindMode.READ_WRITE); bookieContainers.values().forEach(c -> c.withClasspathResourceMapping(key, value, BindMode.READ_WRITE)); @@ -221,6 +273,8 @@ private PulsarCluster(PulsarClusterSpec spec, CSContainer csContainer, boolean s workerContainers.values().forEach(c -> c.withClasspathResourceMapping(key, value, BindMode.READ_WRITE)); }); + functionWorkerEnvs = spec.functionWorkerEnvs; + functionWorkerAdditionalPorts = spec.functionWorkerAdditionalPorts; } public String getPlainTextServiceUrl() { @@ -269,20 +323,33 @@ public Map> getExternalServices() { } public void start() throws Exception { - // start the local zookeeper - zkContainer.start(); - log.info("Successfully started local zookeeper container."); - // start the configuration store - if (!sharedCsContainer) { - csContainer.start(); - log.info("Successfully started configuration store container."); + if (!spec.enableOxia) { + // start the local zookeeper + zkContainer.start(); + log.info("Successfully started local zookeeper container."); + + // start the configuration store + if (!sharedCsContainer) { + csContainer.start(); + log.info("Successfully started configuration store container."); + } + } else { + oxiaContainer.start(); } - // init the cluster - zkContainer.execCmd( - "bin/init-cluster.sh"); - log.info("Successfully initialized the cluster."); + { + // Run cluster metadata initialization + @Cleanup + PulsarInitMetadataContainer init = new PulsarInitMetadataContainer( + network, + clusterName, + metadataStoreUrl, + configurationMetadataStoreUrl, + appendClusterName("pulsar-broker-0") + ); + init.initialize(); + } // start bookies bookieContainers.values().forEach(BKContainer::start); @@ -300,11 +367,6 @@ public void start() throws Exception { log.info("\tBinary Service Url : {}", getPlainTextServiceUrl()); log.info("\tHttp Service Url : {}", getHttpServiceUrl()); - if (enablePrestoWorker) { - log.info("Starting Presto Worker"); - prestoWorkerContainer.start(); - } - // start external services this.externalServices = spec.externalServices; this.externalServiceEnvs = spec.externalServiceEnvs; @@ -314,7 +376,8 @@ public void start() throws Exception { serviceContainer.withNetwork(network); serviceContainer.withNetworkAliases(service.getKey()); if (null != externalServiceEnvs && null != externalServiceEnvs.get(service.getKey())) { - Map env = externalServiceEnvs.getOrDefault(service.getKey(), Collections.emptyMap()); + Map env = + externalServiceEnvs.getOrDefault(service.getKey(), Collections.emptyMap()); serviceContainer.withEnv(env); } PulsarContainer.configureLeaveContainerRunning(serviceContainer); @@ -358,10 +421,6 @@ private static Map runNumContainers(Strin return containers; } - public PrestoWorkerContainer getPrestoWorkerContainer() { - return prestoWorkerContainer; - } - public synchronized void stop() { if (PULSAR_CONTAINERS_LEAVE_RUNNING) { logIgnoringStopDueToLeaveRunning(); @@ -374,8 +433,6 @@ public synchronized void stop() { stopInParallel(externalServices.values()); } - stopPrestoWorker(); - if (null != proxyContainer) { proxyContainer.stop(); } @@ -392,6 +449,10 @@ public synchronized void stop() { zkContainer.stop(); } + if (oxiaContainer != null) { + oxiaContainer.stop(); + } + try { network.close(); } catch (Exception e) { @@ -405,91 +466,8 @@ private static void stopInParallel(Collection> con .forEach(GenericContainer::stop); } - public void startPrestoWorker() { - startPrestoWorker(null, null); - } - - public void startPrestoWorker(String offloadDriver, String offloadProperties) { - log.info("[startPrestoWorker] offloadDriver: {}, offloadProperties: {}", offloadDriver, offloadProperties); - if (null == prestoWorkerContainer) { - prestoWorkerContainer = buildPrestoWorkerContainer( - PrestoWorkerContainer.NAME, true, offloadDriver, offloadProperties); - } - prestoWorkerContainer.start(); - log.info("[{}] Presto coordinator start finished.", prestoWorkerContainer.getContainerName()); - } - - public void stopPrestoWorker() { - if (PULSAR_CONTAINERS_LEAVE_RUNNING) { - logIgnoringStopDueToLeaveRunning(); - return; - } - if (sqlFollowWorkerContainers != null && sqlFollowWorkerContainers.size() > 0) { - for (PrestoWorkerContainer followWorker : sqlFollowWorkerContainers.values()) { - followWorker.stop(); - log.info("Stopped presto follow worker {}.", followWorker.getContainerName()); - } - sqlFollowWorkerContainers.clear(); - log.info("Stopped all presto follow workers."); - } - if (null != prestoWorkerContainer) { - prestoWorkerContainer.stop(); - log.info("Stopped presto coordinator."); - prestoWorkerContainer = null; - } - } - - public void startPrestoFollowWorkers(int numSqlFollowWorkers, String offloadDriver, String offloadProperties) { - log.info("start presto follow worker containers."); - sqlFollowWorkerContainers.putAll(runNumContainers( - "sql-follow-worker", - numSqlFollowWorkers, - (name) -> { - log.info("build presto follow worker with name {}", name); - return buildPrestoWorkerContainer(name, false, offloadDriver, offloadProperties); - } - )); - // Start workers that have been initialized - sqlFollowWorkerContainers.values().parallelStream().forEach(PrestoWorkerContainer::start); - log.info("Successfully started {} presto follow worker containers.", sqlFollowWorkerContainers.size()); - } - - private PrestoWorkerContainer buildPrestoWorkerContainer(String hostName, boolean isCoordinator, - String offloadDriver, String offloadProperties) { - String resourcePath = isCoordinator ? "presto-coordinator-config.properties" - : "presto-follow-worker-config.properties"; - PrestoWorkerContainer container = new PrestoWorkerContainer( - clusterName, hostName) - .withNetwork(network) - .withNetworkAliases(hostName) - .withEnv("clusterName", clusterName) - .withEnv("zkServers", ZKContainer.NAME) - .withEnv("zookeeperServers", ZKContainer.NAME + ":" + ZKContainer.ZK_PORT) - .withEnv("pulsar.metadata-url", "zk:" + ZKContainer.NAME + ":" + ZKContainer.ZK_PORT) - .withEnv("pulsar.web-service-url", "http://pulsar-broker-0:8080") - .withEnv("SQL_PREFIX_pulsar.max-message-size", "" + spec.maxMessageSize) - .withClasspathResourceMapping( - resourcePath, "/pulsar/trino/conf/config.properties", BindMode.READ_WRITE); - if (spec.queryLastMessage) { - container.withEnv("pulsar.bookkeeper-use-v2-protocol", "false") - .withEnv("pulsar.bookkeeper-explicit-interval", "10"); - } - if (offloadDriver != null && offloadProperties != null) { - log.info("[startPrestoWorker] set offload env offloadDriver: {}, offloadProperties: {}", - offloadDriver, offloadProperties); - // used to query from tiered storage - container.withEnv("SQL_PREFIX_pulsar.managed-ledger-offload-driver", offloadDriver); - container.withEnv("SQL_PREFIX_pulsar.offloader-properties", offloadProperties); - container.withEnv("SQL_PREFIX_pulsar.offloaders-directory", "/pulsar/offloaders"); - container.withEnv("AWS_ACCESS_KEY_ID", "accesskey"); - container.withEnv("AWS_SECRET_KEY", "secretkey"); - } - log.info("[{}] build presto worker container. isCoordinator: {}, resourcePath: {}", - container.getContainerName(), isCoordinator, resourcePath); - return container; - } - - public synchronized void setupFunctionWorkers(String suffix, FunctionRuntimeType runtimeType, int numFunctionWorkers) { + public synchronized void setupFunctionWorkers(String suffix, FunctionRuntimeType runtimeType, + int numFunctionWorkers) { switch (runtimeType) { case THREAD: startFunctionWorkersWithThreadContainerFactory(suffix, numFunctionWorkers); @@ -501,37 +479,18 @@ public synchronized void setupFunctionWorkers(String suffix, FunctionRuntimeType } private void startFunctionWorkersWithProcessContainerFactory(String suffix, int numFunctionWorkers) { - String serviceUrl = "pulsar://pulsar-broker-0:" + PulsarContainer.BROKER_PORT; - String httpServiceUrl = "http://pulsar-broker-0:" + PulsarContainer.BROKER_HTTP_PORT; workerContainers.putAll(runNumContainers( "functions-worker-process-" + suffix, numFunctionWorkers, - (name) -> new WorkerContainer(clusterName, name) - .withNetwork(network) - .withNetworkAliases(name) - // worker settings - .withEnv("PF_workerId", name) - .withEnv("PF_workerHostname", name) - .withEnv("PF_workerPort", "" + PulsarContainer.BROKER_HTTP_PORT) - .withEnv("PF_pulsarFunctionsCluster", clusterName) - .withEnv("PF_pulsarServiceUrl", serviceUrl) - .withEnv("PF_pulsarWebServiceUrl", httpServiceUrl) - // script - .withEnv("clusterName", clusterName) - .withEnv("zookeeperServers", ZKContainer.NAME) - // bookkeeper tools - .withEnv("zkServers", ZKContainer.NAME) + (name) -> createWorkerContainer(name) )); this.startWorkers(); } - private void startFunctionWorkersWithThreadContainerFactory(String suffix, int numFunctionWorkers) { + private WorkerContainer createWorkerContainer(String name) { String serviceUrl = "pulsar://pulsar-broker-0:" + PulsarContainer.BROKER_PORT; String httpServiceUrl = "http://pulsar-broker-0:" + PulsarContainer.BROKER_HTTP_PORT; - workerContainers.putAll(runNumContainers( - "functions-worker-thread-" + suffix, - numFunctionWorkers, - (name) -> new WorkerContainer(clusterName, name) + return new WorkerContainer(clusterName, name) .withNetwork(network) .withNetworkAliases(name) // worker settings @@ -541,13 +500,23 @@ private void startFunctionWorkersWithThreadContainerFactory(String suffix, int n .withEnv("PF_pulsarFunctionsCluster", clusterName) .withEnv("PF_pulsarServiceUrl", serviceUrl) .withEnv("PF_pulsarWebServiceUrl", httpServiceUrl) - .withEnv("PF_functionRuntimeFactoryClassName", "org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory") - .withEnv("PF_functionRuntimeFactoryConfigs_threadGroupName", "pf-container-group") // script .withEnv("clusterName", clusterName) .withEnv("zookeeperServers", ZKContainer.NAME) // bookkeeper tools .withEnv("zkServers", ZKContainer.NAME) + .withEnv(functionWorkerEnvs) + .withExposedPorts(functionWorkerAdditionalPorts.toArray(new Integer[0])); + } + + private void startFunctionWorkersWithThreadContainerFactory(String suffix, int numFunctionWorkers) { + workerContainers.putAll(runNumContainers( + "functions-worker-thread-" + suffix, + numFunctionWorkers, + (name) -> createWorkerContainer(name) + .withEnv("PF_functionRuntimeFactoryClassName", + "org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory") + .withEnv("PF_functionRuntimeFactoryConfigs_threadGroupName", "pf-container-group") )); this.startWorkers(); } @@ -588,9 +557,9 @@ public void startContainers(Map> containers) { containers.forEach((name, container) -> { PulsarContainer.configureLeaveContainerRunning(container); container - .withNetwork(network) - .withNetworkAliases(name) - .start(); + .withNetwork(network) + .withNetworkAliases(name) + .start(); log.info("Successfully start container {}.", name); }); } @@ -662,15 +631,16 @@ public ZKContainer getZooKeeper() { return zkContainer; } - public ContainerExecResult runAdminCommandOnAnyBroker(String...commands) throws Exception { + public ContainerExecResult runAdminCommandOnAnyBroker(String... commands) throws Exception { return runCommandOnAnyBrokerWithScript(ADMIN_SCRIPT, commands); } - public ContainerExecResult runPulsarBaseCommandOnAnyBroker(String...commands) throws Exception { + public ContainerExecResult runPulsarBaseCommandOnAnyBroker(String... commands) throws Exception { return runCommandOnAnyBrokerWithScript(PULSAR_COMMAND_SCRIPT, commands); } - private ContainerExecResult runCommandOnAnyBrokerWithScript(String scriptType, String...commands) throws Exception { + private ContainerExecResult runCommandOnAnyBrokerWithScript(String scriptType, String... commands) + throws Exception { BrokerContainer container = getAnyBroker(); String[] cmds = new String[commands.length + 1]; cmds[0] = scriptType; @@ -704,8 +674,8 @@ public void startZooKeeper() { public ContainerExecResult createNamespace(String nsName) throws Exception { return runAdminCommandOnAnyBroker( - "namespaces", "create", "public/" + nsName, - "--clusters", clusterName); + "namespaces", "create", "public/" + nsName, + "--clusters", clusterName); } public ContainerExecResult createPartitionedTopic(String topicName, int partitions) throws Exception { @@ -716,8 +686,8 @@ public ContainerExecResult createPartitionedTopic(String topicName, int partitio public ContainerExecResult enableDeduplication(String nsName, boolean enabled) throws Exception { return runAdminCommandOnAnyBroker( - "namespaces", "set-deduplication", "public/" + nsName, - enabled ? "--enable" : "--disable"); + "namespaces", "set-deduplication", "public/" + nsName, + enabled ? "--enable" : "--disable"); } public void dumpFunctionLogs(String name) { @@ -730,7 +700,8 @@ public void dumpFunctionLogs(String name) { }); log.info("Function {} logs {}", name, logs); } catch (com.github.dockerjava.api.exception.NotFoundException notFound) { - log.info("Cannot download {} logs from {} not found exception {}", name, container.getContainerName(), notFound.toString()); + log.info("Cannot download {} logs from {} not found exception {}", name, container.getContainerName(), + notFound.toString()); } catch (Throwable err) { log.info("Cannot download {} logs from {}", name, container.getContainerName(), err); } @@ -740,4 +711,8 @@ public void dumpFunctionLogs(String name) { private String appendClusterName(String name) { return sharedCsContainer ? clusterName + "-" + name : name; } + + public BKContainer getAnyBookie() { + return getAnyContainer(bookieContainers, "bookie"); + } } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java index 385af99a6644b..8a991be49fad0 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java @@ -81,14 +81,6 @@ public class PulsarClusterSpec { @Default int numFunctionWorkers = 0; - /** - * Enable a Presto Worker Node - * - * @return the flag whether presto worker is enabled - */ - @Default - boolean enablePrestoWorker = false; - /** * Allow to query the last message */ @@ -150,6 +142,17 @@ public class PulsarClusterSpec { */ Map brokerEnvs; + /** + * Specify envs for bookkeeper. + */ + Map bookkeeperEnvs; + + /** + * Specify envs for function workers. + */ + @Singular + Map functionWorkerEnvs; + /** * Specify mount files. */ @@ -167,4 +170,29 @@ public class PulsarClusterSpec { * Additional ports to expose on broker containers. */ List brokerAdditionalPorts; + + /** + * Additional ports to expose on bookie containers. + */ + List bookieAdditionalPorts; + + /** + * Additional ports to expose on proxy containers. + */ + List proxyAdditionalPorts; + + /** + * Additional ports to expose on function workers. + */ + @Singular + List functionWorkerAdditionalPorts; + + /** + * Enable TLS for connection. + */ + @Default + boolean enableTls = false; + + @Default + boolean enableOxia = false; } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterTestBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterTestBase.java index d7a1906ec582e..93e2221ab2493 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterTestBase.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterTestBase.java @@ -25,6 +25,7 @@ import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.common.naming.TopicDomain; import org.testng.annotations.DataProvider; import java.util.stream.Stream; @@ -34,8 +35,10 @@ @Slf4j public abstract class PulsarClusterTestBase extends PulsarTestBase { protected final Map brokerEnvs = new HashMap<>(); + protected final Map bookkeeperEnvs = new HashMap<>(); protected final Map proxyEnvs = new HashMap<>(); protected final List brokerAdditionalPorts = new LinkedList<>(); + protected final List bookieAdditionalPorts = new LinkedList<>(); @Override protected final void setup() throws Exception { @@ -84,6 +87,20 @@ public Object[][] serviceAndAdminUrls() { }; } + @DataProvider + public Object[][] serviceUrlAndTopicDomain() { + return new Object[][] { + { + stringSupplier(() -> getPulsarCluster().getPlainTextServiceUrl()), + TopicDomain.persistent + }, + { + stringSupplier(() -> getPulsarCluster().getPlainTextServiceUrl()), + TopicDomain.non_persistent + }, + }; + } + protected PulsarAdmin pulsarAdmin; protected PulsarCluster pulsarCluster; diff --git a/docker/pulsar/scripts/install-pulsar-client.sh b/tests/integration/src/test/resources/containers/otel-collector-config.yaml old mode 100755 new mode 100644 similarity index 68% rename from docker/pulsar/scripts/install-pulsar-client.sh rename to tests/integration/src/test/resources/containers/otel-collector-config.yaml index 0951b2aec1b60..2ba532f3c6cba --- a/docker/pulsar/scripts/install-pulsar-client.sh +++ b/tests/integration/src/test/resources/containers/otel-collector-config.yaml @@ -1,4 +1,3 @@ -#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -18,13 +17,28 @@ # under the License. # -set -x +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 -# TODO: remove these lines once grpcio doesn't need to compile from source on ARM64 platform -ARCH=$(uname -m | sed -r 's/aarch64/arm64/g' | awk '!/arm64/{$0="amd64"}1') -if [ "${ARCH}" == "arm64" ]; then - apt update - apt -y install build-essential python3-dev -fi +exporters: + prometheus: + endpoint: "0.0.0.0:8889" -pip3 install pulsar-client[all]==${PULSAR_CLIENT_PYTHON_VERSION} +processors: + batch: + +extensions: + health_check: + zpages: + endpoint: :55679 + +service: + extensions: [zpages, health_check] + pipelines: + metrics: + receivers: [otlp] + processors: [batch] + exporters: [prometheus] \ No newline at end of file diff --git a/tests/integration/src/test/resources/presto-coordinator-config.properties b/tests/integration/src/test/resources/presto-coordinator-config.properties deleted file mode 100644 index 8e554370b3731..0000000000000 --- a/tests/integration/src/test/resources/presto-coordinator-config.properties +++ /dev/null @@ -1,42 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -coordinator=true - -node.id=ffffffff-ffff-ffff-ffff-ffffffffffff -node.environment=test -http-server.http.port=8081 - -discovery-server.enabled=true -discovery.uri=http://presto-worker:8081 - -exchange.http-client.max-connections=1000 -exchange.http-client.max-connections-per-server=1000 -exchange.http-client.connect-timeout=1m -exchange.http-client.idle-timeout=1m - -scheduler.http-client.max-connections=1000 -scheduler.http-client.max-connections-per-server=1000 -scheduler.http-client.connect-timeout=1m -scheduler.http-client.idle-timeout=1m - -query.client.timeout=5m -query.min-expire-age=30m - -node-scheduler.include-coordinator=true diff --git a/tests/integration/src/test/resources/pulsar-sql.xml b/tests/integration/src/test/resources/pulsar-loadbalance.xml similarity index 68% rename from tests/integration/src/test/resources/pulsar-sql.xml rename to tests/integration/src/test/resources/pulsar-loadbalance.xml index 1ab4d479ad041..dfc4536e25592 100644 --- a/tests/integration/src/test/resources/pulsar-sql.xml +++ b/tests/integration/src/test/resources/pulsar-loadbalance.xml @@ -19,12 +19,10 @@ --> - - + + - - - + - + \ No newline at end of file diff --git a/tests/integration/src/test/resources/pulsar-messaging.xml b/tests/integration/src/test/resources/pulsar-messaging.xml index cfbdb22587034..a34670267dc2a 100644 --- a/tests/integration/src/test/resources/pulsar-messaging.xml +++ b/tests/integration/src/test/resources/pulsar-messaging.xml @@ -28,7 +28,10 @@ + + + \ No newline at end of file diff --git a/tests/integration/src/test/resources/pulsar-metrics.xml b/tests/integration/src/test/resources/pulsar-metrics.xml new file mode 100644 index 0000000000000..1c87f2bdf0d06 --- /dev/null +++ b/tests/integration/src/test/resources/pulsar-metrics.xml @@ -0,0 +1,28 @@ + + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar.xml b/tests/integration/src/test/resources/pulsar.xml index 7fbe9fa24d8d9..aa9a59a6cda64 100644 --- a/tests/integration/src/test/resources/pulsar.xml +++ b/tests/integration/src/test/resources/pulsar.xml @@ -24,12 +24,12 @@ - + @@ -37,5 +37,6 @@ + diff --git a/tests/pom.xml b/tests/pom.xml index ecdff86a8b1b2..9bc3c2402886e 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT org.apache.pulsar.tests tests-parent diff --git a/tests/pulsar-client-admin-shade-test/pom.xml b/tests/pulsar-client-admin-shade-test/pom.xml index b0019d01b04ee..7007d7330d20b 100644 --- a/tests/pulsar-client-admin-shade-test/pom.xml +++ b/tests/pulsar-client-admin-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-client-admin-shade-test diff --git a/tests/pulsar-client-admin-shade-test/src/test/java/org/apache/pulsar/tests/integration/SmokeTest.java b/tests/pulsar-client-admin-shade-test/src/test/java/org/apache/pulsar/tests/integration/SmokeTest.java index 990a0a8211c71..b5c615b743cf7 100644 --- a/tests/pulsar-client-admin-shade-test/src/test/java/org/apache/pulsar/tests/integration/SmokeTest.java +++ b/tests/pulsar-client-admin-shade-test/src/test/java/org/apache/pulsar/tests/integration/SmokeTest.java @@ -81,6 +81,7 @@ public void checkClient() throws PulsarClientException { @Test public void checkAdmin() throws PulsarClientException, PulsarAdminException { + @Cleanup PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarContainer.getPulsarAdminUrl()).build(); List expectedNamespacesList = new ArrayList<>(); expectedNamespacesList.add("public/default"); diff --git a/tests/pulsar-client-all-shade-test/pom.xml b/tests/pulsar-client-all-shade-test/pom.xml index db11fec205a93..cd1446b333935 100644 --- a/tests/pulsar-client-all-shade-test/pom.xml +++ b/tests/pulsar-client-all-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-client-all-shade-test diff --git a/tests/pulsar-client-all-shade-test/src/test/java/org/apache/pulsar/tests/integration/SmokeTest.java b/tests/pulsar-client-all-shade-test/src/test/java/org/apache/pulsar/tests/integration/SmokeTest.java index e5eef415a513d..c5075118f08e3 100644 --- a/tests/pulsar-client-all-shade-test/src/test/java/org/apache/pulsar/tests/integration/SmokeTest.java +++ b/tests/pulsar-client-all-shade-test/src/test/java/org/apache/pulsar/tests/integration/SmokeTest.java @@ -81,6 +81,7 @@ public void checkClient() throws PulsarClientException { @Test public void checkAdmin() throws PulsarClientException, PulsarAdminException { + @Cleanup PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarContainer.getPulsarAdminUrl()).build(); List expectedNamespacesList = new ArrayList<>(); expectedNamespacesList.add("public/default"); diff --git a/tests/pulsar-client-shade-test/pom.xml b/tests/pulsar-client-shade-test/pom.xml index cf5aff0291fb7..417925851298c 100644 --- a/tests/pulsar-client-shade-test/pom.xml +++ b/tests/pulsar-client-shade-test/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 3.1.0-SNAPSHOT + 4.0.0-SNAPSHOT pulsar-client-shade-test diff --git a/tests/pulsar-client-shade-test/src/test/java/org/apache/pulsar/tests/integration/SimpleProducerConsumerTest.java b/tests/pulsar-client-shade-test/src/test/java/org/apache/pulsar/tests/integration/SimpleProducerConsumerTest.java index 23cdac6d0934a..a0a54e4604ae8 100644 --- a/tests/pulsar-client-shade-test/src/test/java/org/apache/pulsar/tests/integration/SimpleProducerConsumerTest.java +++ b/tests/pulsar-client-shade-test/src/test/java/org/apache/pulsar/tests/integration/SimpleProducerConsumerTest.java @@ -87,13 +87,13 @@ public void setup() throws Exception { .build(); lookupUrl = new URI(pulsarContainer.getPlainTextPulsarBrokerUrl()); + @Cleanup PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarContainer.getPulsarAdminUrl()).build(); admin.tenants().createTenant("my-property", TenantInfo.builder().adminRoles(new HashSet<>(Arrays.asList("appid1", "appid2"))) .allowedClusters(Collections.singleton("standalone")).build()); admin.namespaces().createNamespace("my-property/my-ns"); admin.namespaces().setNamespaceReplicationClusters("my-property/my-ns", Collections.singleton("standalone")); - admin.close(); } @Override diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index 113673cba0f58..1ec1c816fcc48 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar tiered-storage-parent - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT tiered-storage-file-system @@ -48,7 +47,35 @@ org.slf4j - slf4j-log4j12 + * + + + + + + org.apache.hadoop + hadoop-hdfs-client + ${hdfs-offload-version3} + + + org.apache.avro + avro + + + org.mortbay.jetty + jetty + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + javax.servlet + servlet-api @@ -82,13 +109,27 @@ ${hdfs-offload-version3} test - - io.netty - netty-all - + + io.netty + netty-all + + + org.bouncycastle + * + + + org.slf4j + * + + + org.bouncycastle + bcpkix-jdk18on + test + + io.netty netty-codec-http @@ -167,7 +208,6 @@ org.owasp dependency-check-maven - ${dependency-check-maven.version} diff --git a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java index bdeb88e9ac93c..91e7e902eab8a 100644 --- a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java +++ b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java @@ -20,7 +20,6 @@ import static org.apache.bookkeeper.mledger.offload.OffloadUtils.parseLedgerMetadata; import io.netty.buffer.ByteBuf; -import io.netty.buffer.PooledByteBufAllocator; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -28,6 +27,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.api.LastConfirmedAndEntry; import org.apache.bookkeeper.client.api.LedgerEntries; @@ -37,9 +37,12 @@ import org.apache.bookkeeper.client.impl.LedgerEntriesImpl; import org.apache.bookkeeper.client.impl.LedgerEntryImpl; import org.apache.bookkeeper.mledger.LedgerOffloaderStats; +import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.MapFile; +import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; +import org.apache.pulsar.common.naming.TopicName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,6 +54,13 @@ public class FileStoreBackedReadHandleImpl implements ReadHandle { private final LedgerMetadata ledgerMetadata; private final LedgerOffloaderStats offloaderStats; private final String managedLedgerName; + private final String topicName; + enum State { + Opened, + Closed + } + private volatile State state; + private final AtomicReference> closeFuture = new AtomicReference<>(); private FileStoreBackedReadHandleImpl(ExecutorService executor, MapFile.Reader reader, long ledgerId, LedgerOffloaderStats offloaderStats, @@ -60,15 +70,17 @@ private FileStoreBackedReadHandleImpl(ExecutorService executor, MapFile.Reader r this.reader = reader; this.offloaderStats = offloaderStats; this.managedLedgerName = managedLedgerName; + this.topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName); LongWritable key = new LongWritable(); BytesWritable value = new BytesWritable(); try { key.set(FileSystemManagedLedgerOffloader.METADATA_KEY_INDEX); long startReadIndexTime = System.nanoTime(); reader.get(key, value); - offloaderStats.recordReadOffloadIndexLatency(managedLedgerName, + offloaderStats.recordReadOffloadIndexLatency(topicName, System.nanoTime() - startReadIndexTime, TimeUnit.NANOSECONDS); this.ledgerMetadata = parseLedgerMetadata(ledgerId, value.copyBytes()); + state = State.Opened; } catch (IOException e) { log.error("Fail to read LedgerMetadata for ledgerId {}", ledgerId); @@ -89,15 +101,20 @@ public LedgerMetadata getLedgerMetadata() { @Override public CompletableFuture closeAsync() { - CompletableFuture promise = new CompletableFuture<>(); + if (closeFuture.get() != null || !closeFuture.compareAndSet(null, new CompletableFuture<>())) { + return closeFuture.get(); + } + + CompletableFuture promise = closeFuture.get(); executor.execute(() -> { - try { - reader.close(); - promise.complete(null); - } catch (IOException t) { - promise.completeExceptionally(t); - } - }); + try { + reader.close(); + state = State.Closed; + promise.complete(null); + } catch (IOException t) { + promise.completeExceptionally(t); + } + }); return promise; } @@ -108,6 +125,12 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr } CompletableFuture promise = new CompletableFuture<>(); executor.execute(() -> { + if (state == State.Closed) { + log.warn("Reading a closed read handler. Ledger ID: {}, Read range: {}-{}", + ledgerId, firstEntry, lastEntry); + promise.completeExceptionally(new ManagedLedgerException.OffloadReadHandleClosedException()); + return; + } if (firstEntry > lastEntry || firstEntry < 0 || lastEntry > getLastAddConfirmed()) { @@ -125,17 +148,17 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr while (entriesToRead > 0) { long startReadTime = System.nanoTime(); reader.next(key, value); - this.offloaderStats.recordReadOffloadDataLatency(managedLedgerName, + this.offloaderStats.recordReadOffloadDataLatency(topicName, System.nanoTime() - startReadTime, TimeUnit.NANOSECONDS); int length = value.getLength(); long entryId = key.get(); if (entryId == nextExpectedId) { - ByteBuf buf = PooledByteBufAllocator.DEFAULT.buffer(length, length); + ByteBuf buf = PulsarByteBufAllocator.DEFAULT.buffer(length, length); entries.add(LedgerEntryImpl.create(ledgerId, entryId, length, buf)); buf.writeBytes(value.copyBytes()); entriesToRead--; nextExpectedId++; - this.offloaderStats.recordReadOffloadBytes(managedLedgerName, length); + this.offloaderStats.recordReadOffloadBytes(topicName, length); } else if (entryId > lastEntry) { log.info("Expected to read {}, but read {}, which is greater than last entry {}", nextExpectedId, entryId, lastEntry); @@ -144,7 +167,7 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr } promise.complete(LedgerEntriesImpl.create(entries)); } catch (Throwable t) { - this.offloaderStats.recordReadOffloadError(managedLedgerName); + this.offloaderStats.recordReadOffloadError(topicName); promise.completeExceptionally(t); entries.forEach(LedgerEntry::close); } diff --git a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java index d5e09ba725421..56612adc1ef80 100644 --- a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java +++ b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java @@ -21,6 +21,7 @@ import static org.apache.bookkeeper.mledger.offload.OffloadUtils.buildLedgerMetadataFormat; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.MoreExecutors; import io.netty.util.Recycler; import java.io.IOException; import java.util.Iterator; @@ -45,6 +46,8 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.MapFile; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,7 +66,7 @@ public class FileSystemManagedLedgerOffloader implements LedgerOffloader { private OrderedScheduler scheduler; private static final long ENTRIES_PER_READ = 100; private OrderedScheduler assignmentScheduler; - private OffloadPoliciesImpl offloadPolicies; + private OffloadPolicies offloadPolicies; private final LedgerOffloaderStats offloaderStats; public static boolean driverSupported(String driver) { @@ -197,9 +200,10 @@ public void run() { return; } long ledgerId = readHandle.getId(); - final String topicName = extraMetadata.get(MANAGED_LEDGER_NAME); - String storagePath = getStoragePath(storageBasePath, topicName); + final String managedLedgerName = extraMetadata.get(MANAGED_LEDGER_NAME); + String storagePath = getStoragePath(storageBasePath, managedLedgerName); String dataFilePath = getDataFilePath(storagePath, ledgerId, uuid); + final String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName); LongWritable key = new LongWritable(); BytesWritable value = new BytesWritable(); try { @@ -241,7 +245,7 @@ public void run() { promise.complete(null); } catch (Exception e) { log.error("Exception when get CompletableFuture : ManagerLedgerName: {}, " - + "LedgerId: {}, UUID: {} ", topicName, ledgerId, uuid, e); + + "LedgerId: {}, UUID: {} ", managedLedgerName, ledgerId, uuid, e); if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } @@ -306,22 +310,27 @@ public static FileSystemWriter create(LedgerEntries ledgerEntriesOnce, @Override public void run() { String managedLedgerName = ledgerReader.extraMetadata.get(MANAGED_LEDGER_NAME); + String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName); if (ledgerReader.fileSystemWriteException == null) { Iterator iterator = ledgerEntriesOnce.iterator(); while (iterator.hasNext()) { LedgerEntry entry = iterator.next(); long entryId = entry.getEntryId(); key.set(entryId); + byte[] currentEntryBytes; + int currentEntrySize; try { - value.set(entry.getEntryBytes(), 0, entry.getEntryBytes().length); + currentEntryBytes = entry.getEntryBytes(); + currentEntrySize = currentEntryBytes.length; + value.set(currentEntryBytes, 0, currentEntrySize); dataWriter.append(key, value); } catch (IOException e) { ledgerReader.fileSystemWriteException = e; - ledgerReader.offloaderStats.recordWriteToStorageError(managedLedgerName); + ledgerReader.offloaderStats.recordWriteToStorageError(topicName); break; } haveOffloadEntryNumber.incrementAndGet(); - ledgerReader.offloaderStats.recordOffloadBytes(managedLedgerName, entry.getLength()); + ledgerReader.offloaderStats.recordOffloadBytes(topicName, currentEntrySize); } } countDownLatch.countDown(); @@ -367,6 +376,7 @@ public CompletableFuture deleteOffloaded(long ledgerId, UUID uid, Map promise = new CompletableFuture<>(); try { fileSystem.delete(new Path(dataFilePath), true); @@ -376,11 +386,11 @@ public CompletableFuture deleteOffloaded(long ledgerId, UUID uid, Map - this.offloaderStats.recordDeleteOffloadOps(ledgerName, t == null)); + this.offloaderStats.recordDeleteOffloadOps(topicName, t == null)); } @Override - public OffloadPoliciesImpl getOffloadPolicies() { + public OffloadPolicies getOffloadPolicies() { return offloadPolicies; } @@ -393,5 +403,8 @@ public void close() { log.error("FileSystemManagedLedgerOffloader close failed!", e); } } + if (assignmentScheduler != null) { + MoreExecutors.shutdownAndAwaitTermination(assignmentScheduler, 5, TimeUnit.SECONDS); + } } } diff --git a/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/FileStoreTestBase.java b/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/FileStoreTestBase.java index 3e6cd8745dc01..9609362e9b770 100644 --- a/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/FileStoreTestBase.java +++ b/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/FileStoreTestBase.java @@ -18,40 +18,66 @@ */ package org.apache.bookkeeper.mledger.offload.filesystem; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Properties; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.mledger.LedgerOffloaderStats; import org.apache.bookkeeper.mledger.offload.filesystem.impl.FileSystemManagedLedgerOffloader; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.MiniDFSCluster; - import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; +import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; -import java.io.File; -import java.nio.file.Files; -import java.util.Properties; -import java.util.concurrent.Executors; - public abstract class FileStoreTestBase { protected FileSystemManagedLedgerOffloader fileSystemManagedLedgerOffloader; - protected OrderedScheduler scheduler = OrderedScheduler.newSchedulerBuilder().numThreads(1).name("offloader").build(); + protected OrderedScheduler scheduler; protected final String basePath = "pulsar"; private MiniDFSCluster hdfsCluster; private String hdfsURI; protected LedgerOffloaderStats offloaderStats; + private ScheduledExecutorService scheduledExecutorService; + + @BeforeClass(alwaysRun = true) + public final void beforeClass() throws Exception { + init(); + } + + public void init() throws Exception { + scheduler = OrderedScheduler.newSchedulerBuilder().numThreads(1).name("offloader").build(); + } + + @AfterClass(alwaysRun = true) + public final void afterClass() throws IOException { + cleanup(); + } + + public void cleanup() throws IOException { + if (scheduler != null) { + scheduler.shutdownNow(); + scheduler = null; + } + } @BeforeMethod(alwaysRun = true) public void start() throws Exception { File baseDir = Files.createTempDirectory(basePath).toFile().getAbsoluteFile(); Configuration conf = new Configuration(); conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath()); + conf.set("dfs.namenode.gc.time.monitor.enable", "false"); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf); hdfsCluster = builder.build(); hdfsURI = "hdfs://localhost:"+ hdfsCluster.getNameNodePort() + "/"; Properties properties = new Properties(); - this.offloaderStats = LedgerOffloaderStats.create(true, true, Executors.newScheduledThreadPool(1), 60); + scheduledExecutorService = Executors.newScheduledThreadPool(1); + this.offloaderStats = LedgerOffloaderStats.create(true, true, scheduledExecutorService, 60); fileSystemManagedLedgerOffloader = new FileSystemManagedLedgerOffloader( OffloadPoliciesImpl.create(properties), scheduler, hdfsURI, basePath, offloaderStats); @@ -59,8 +85,18 @@ public void start() throws Exception { @AfterMethod(alwaysRun = true) public void tearDown() { - hdfsCluster.shutdown(true, true); - hdfsCluster.close(); + if (fileSystemManagedLedgerOffloader != null) { + fileSystemManagedLedgerOffloader.close(); + fileSystemManagedLedgerOffloader = null; + } + if (hdfsCluster != null) { + hdfsCluster.shutdown(true, true); + hdfsCluster = null; + } + if (scheduledExecutorService != null) { + scheduledExecutorService.shutdownNow(); + scheduledExecutorService = null; + } } public String getURI() { diff --git a/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloaderTest.java b/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloaderTest.java index 1aebab571c971..71fe5ec72193a 100644 --- a/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloaderTest.java +++ b/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloaderTest.java @@ -19,6 +19,15 @@ package org.apache.bookkeeper.mledger.offload.filesystem.impl; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import java.io.IOException; +import java.net.URI; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.UUID; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.client.PulsarMockBookKeeper; @@ -32,31 +41,35 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.pulsar.common.naming.TopicName; +import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import java.net.URI; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.UUID; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertTrue; public class FileSystemManagedLedgerOffloaderTest extends FileStoreTestBase { - private final PulsarMockBookKeeper bk; - private String topic = "public/default/testOffload"; - private String storagePath = createStoragePath(topic); + private PulsarMockBookKeeper bk; + private String managedLedgerName = "public/default/persistent/testOffload"; + private String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName); + private String storagePath = createStoragePath(managedLedgerName); private LedgerHandle lh; private ReadHandle toWrite; private final int numberOfEntries = 601; private Map map = new HashMap<>(); - public FileSystemManagedLedgerOffloaderTest() throws Exception { + @Override + public void init() throws Exception { + super.init(); this.bk = new PulsarMockBookKeeper(scheduler); this.toWrite = buildReadHandle(); - map.put("ManagedLedgerName", topic); + map.put("ManagedLedgerName", managedLedgerName); + } + + @Override + public void cleanup() throws IOException { + if (bk != null) { + bk.shutdown(); + } + super.cleanup(); } private ReadHandle buildReadHandle() throws Exception { @@ -83,6 +96,12 @@ public void start() throws Exception { super.start(); } + @AfterMethod(alwaysRun = true) + @Override + public void tearDown() { + super.tearDown(); + } + @Test public void testOffloadAndRead() throws Exception { LedgerOffloader offloader = fileSystemManagedLedgerOffloader; @@ -125,10 +144,10 @@ public void testOffloadAndReadMetrics() throws Exception { offloader.offload(toWrite, uuid, map).get(); LedgerOffloaderStatsImpl offloaderStats = (LedgerOffloaderStatsImpl) this.offloaderStats; - assertTrue(offloaderStats.getOffloadError(topic) == 0); - assertTrue(offloaderStats.getOffloadBytes(topic) > 0); - assertTrue(offloaderStats.getReadLedgerLatency(topic).count > 0); - assertTrue(offloaderStats.getWriteStorageError(topic) == 0); + assertTrue(offloaderStats.getOffloadError(topicName) == 0); + assertTrue(offloaderStats.getOffloadBytes(topicName) > 0); + assertTrue(offloaderStats.getReadLedgerLatency(topicName).count > 0); + assertTrue(offloaderStats.getWriteStorageError(topicName) == 0); ReadHandle toTest = offloader.readOffloaded(toWrite.getId(), uuid, map).get(); LedgerEntries toTestEntries = toTest.read(0, numberOfEntries - 1); @@ -137,10 +156,10 @@ public void testOffloadAndReadMetrics() throws Exception { LedgerEntry toTestEntry = toTestIter.next(); } - assertTrue(offloaderStats.getReadOffloadError(topic) == 0); - assertTrue(offloaderStats.getReadOffloadBytes(topic) > 0); - assertTrue(offloaderStats.getReadOffloadDataLatency(topic).count > 0); - assertTrue(offloaderStats.getReadOffloadIndexLatency(topic).count > 0); + assertTrue(offloaderStats.getReadOffloadError(topicName) == 0); + assertTrue(offloaderStats.getReadOffloadBytes(topicName) > 0); + assertTrue(offloaderStats.getReadOffloadDataLatency(topicName).count > 0); + assertTrue(offloaderStats.getReadOffloadIndexLatency(topicName).count > 0); } @Test diff --git a/tiered-storage/jcloud/pom.xml b/tiered-storage/jcloud/pom.xml index d7ad763ea97b4..fbf366f2ffced 100644 --- a/tiered-storage/jcloud/pom.xml +++ b/tiered-storage/jcloud/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar tiered-storage-parent - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT tiered-storage-jcloud diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java index 2c9165674444d..60363cf8406db 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java @@ -25,6 +25,7 @@ import org.apache.bookkeeper.mledger.LedgerOffloaderStats; import org.apache.bookkeeper.mledger.LedgerOffloaderStatsDisable; import org.apache.bookkeeper.mledger.offload.jcloud.impl.BlobStoreManagedLedgerOffloader; +import org.apache.bookkeeper.mledger.offload.jcloud.impl.OffsetsCache; import org.apache.bookkeeper.mledger.offload.jcloud.provider.JCloudBlobStoreProvider; import org.apache.bookkeeper.mledger.offload.jcloud.provider.TieredStorageConfiguration; import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; @@ -33,12 +34,7 @@ * A jcloud based offloader factory. */ public class JCloudLedgerOffloaderFactory implements LedgerOffloaderFactory { - - public static JCloudLedgerOffloaderFactory of() { - return INSTANCE; - } - - private static final JCloudLedgerOffloaderFactory INSTANCE = new JCloudLedgerOffloaderFactory(); + private final OffsetsCache entryOffsetsCache = new OffsetsCache(); @Override public boolean isDriverSupported(String driverName) { @@ -58,6 +54,12 @@ public BlobStoreManagedLedgerOffloader create(OffloadPoliciesImpl offloadPolicie TieredStorageConfiguration config = TieredStorageConfiguration.create(offloadPolicies.toProperties()); - return BlobStoreManagedLedgerOffloader.create(config, userMetadata, scheduler, offloaderStats); + return BlobStoreManagedLedgerOffloader.create(config, userMetadata, scheduler, offloaderStats, + entryOffsetsCache); + } + + @Override + public void close() throws Exception { + entryOffsetsCache.close(); } } diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamImpl.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamImpl.java index aa27df46c5e65..6ebbe5bce582a 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamImpl.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamImpl.java @@ -26,7 +26,9 @@ import org.apache.bookkeeper.mledger.offload.jcloud.BackedInputStream; import org.apache.bookkeeper.mledger.offload.jcloud.impl.DataBlockUtils.VersionCheck; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; +import org.apache.pulsar.common.naming.TopicName; import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.KeyNotFoundException; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.options.GetOptions; import org.slf4j.Logger; @@ -44,6 +46,7 @@ public class BlobStoreBackedInputStreamImpl extends BackedInputStream { private final int bufferSize; private LedgerOffloaderStats offloaderStats; private String managedLedgerName; + private String topicName; private long cursor; private long bufferOffsetStart; @@ -71,6 +74,7 @@ public BlobStoreBackedInputStreamImpl(BlobStore blobStore, String bucket, String this(blobStore, bucket, key, versionCheck, objectLen, bufferSize); this.offloaderStats = offloaderStats; this.managedLedgerName = managedLedgerName; + this.topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName); } /** @@ -92,6 +96,9 @@ private boolean refillBufferIfNeeded() throws IOException { try { long startReadTime = System.nanoTime(); Blob blob = blobStore.getBlob(bucket, key, new GetOptions().range(startRange, endRange)); + if (blob == null) { + throw new KeyNotFoundException(bucket, key, ""); + } versionCheck.check(key, blob); try (InputStream stream = blob.getPayload().openStream()) { @@ -100,9 +107,7 @@ private boolean refillBufferIfNeeded() throws IOException { bufferOffsetEnd = endRange; long bytesRead = endRange - startRange + 1; int bytesToCopy = (int) bytesRead; - while (bytesToCopy > 0) { - bytesToCopy -= buffer.writeBytes(stream, bytesToCopy); - } + fillBuffer(stream, bytesToCopy); cursor += buffer.readableBytes(); } @@ -110,13 +115,17 @@ private boolean refillBufferIfNeeded() throws IOException { // because JClouds streams the content // and actually the HTTP call finishes when the stream is fully read if (this.offloaderStats != null) { - this.offloaderStats.recordReadOffloadDataLatency(managedLedgerName, + this.offloaderStats.recordReadOffloadDataLatency(topicName, System.nanoTime() - startReadTime, TimeUnit.NANOSECONDS); - this.offloaderStats.recordReadOffloadBytes(managedLedgerName, endRange - startRange + 1); + this.offloaderStats.recordReadOffloadBytes(topicName, endRange - startRange + 1); } } catch (Throwable e) { if (null != this.offloaderStats) { - this.offloaderStats.recordReadOffloadError(this.managedLedgerName); + this.offloaderStats.recordReadOffloadError(this.topicName); + } + // If the blob is not found, the original exception is thrown and handled by the caller. + if (e instanceof KeyNotFoundException) { + throw e; } throw new IOException("Error reading from BlobStore", e); } @@ -124,6 +133,20 @@ private boolean refillBufferIfNeeded() throws IOException { return true; } + void fillBuffer(InputStream is, int bytesToCopy) throws IOException { + while (bytesToCopy > 0) { + int writeBytes = buffer.writeBytes(is, bytesToCopy); + if (writeBytes < 0) { + break; + } + bytesToCopy -= writeBytes; + } + } + + ByteBuf getBuffer() { + return buffer; + } + @Override public int read() throws IOException { if (refillBufferIfNeeded()) { diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java index cdabe5ece0ba2..e050d74a332bc 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java @@ -18,8 +18,7 @@ */ package org.apache.bookkeeper.mledger.offload.jcloud.impl; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; +import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.ByteBuf; import java.io.DataInputStream; import java.io.IOException; @@ -30,6 +29,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.api.LastConfirmedAndEntry; import org.apache.bookkeeper.client.api.LedgerEntries; @@ -45,41 +45,40 @@ import org.apache.bookkeeper.mledger.offload.jcloud.OffloadIndexBlockBuilder; import org.apache.bookkeeper.mledger.offload.jcloud.impl.DataBlockUtils.VersionCheck; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; +import org.apache.pulsar.common.naming.TopicName; import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.KeyNotFoundException; import org.jclouds.blobstore.domain.Blob; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class BlobStoreBackedReadHandleImpl implements ReadHandle { private static final Logger log = LoggerFactory.getLogger(BlobStoreBackedReadHandleImpl.class); - private static final int CACHE_TTL_SECONDS = - Integer.getInteger("pulsar.jclouds.readhandleimpl.offsetsscache.ttl.seconds", 30 * 60); private final long ledgerId; private final OffloadIndexBlock index; private final BackedInputStream inputStream; private final DataInputStream dataStream; private final ExecutorService executor; - // this Cache is accessed only by one thread - private final Cache entryOffsets = CacheBuilder - .newBuilder() - .expireAfterAccess(CACHE_TTL_SECONDS, TimeUnit.SECONDS) - .build(); + private final OffsetsCache entryOffsetsCache; + private final AtomicReference> closeFuture = new AtomicReference<>(); enum State { Opened, Closed } - private State state = null; + private volatile State state = null; private BlobStoreBackedReadHandleImpl(long ledgerId, OffloadIndexBlock index, - BackedInputStream inputStream, ExecutorService executor) { + BackedInputStream inputStream, ExecutorService executor, + OffsetsCache entryOffsetsCache) { this.ledgerId = ledgerId; this.index = index; this.inputStream = inputStream; this.dataStream = new DataInputStream(inputStream); this.executor = executor; + this.entryOffsetsCache = entryOffsetsCache; state = State.Opened; } @@ -95,18 +94,21 @@ public LedgerMetadata getLedgerMetadata() { @Override public CompletableFuture closeAsync() { - CompletableFuture promise = new CompletableFuture<>(); + if (closeFuture.get() != null || !closeFuture.compareAndSet(null, new CompletableFuture<>())) { + return closeFuture.get(); + } + + CompletableFuture promise = closeFuture.get(); executor.execute(() -> { - try { - index.close(); - inputStream.close(); - entryOffsets.invalidateAll(); - state = State.Closed; - promise.complete(null); - } catch (IOException t) { - promise.completeExceptionally(t); - } - }); + try { + index.close(); + inputStream.close(); + state = State.Closed; + promise.complete(null); + } catch (IOException t) { + promise.completeExceptionally(t); + } + }); return promise; } @@ -155,7 +157,7 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr long entryId = dataStream.readLong(); if (entryId == nextExpectedId) { - entryOffsets.put(entryId, currentPosition); + entryOffsetsCache.put(ledgerId, entryId, currentPosition); ByteBuf buf = PulsarByteBufAllocator.DEFAULT.buffer(length, length); entries.add(LedgerEntryImpl.create(ledgerId, entryId, length, buf)); int toWrite = length; @@ -194,7 +196,11 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr } catch (Throwable t) { log.error("Failed to read entries {} - {} from the offloader in ledger {}", firstEntry, lastEntry, ledgerId, t); - promise.completeExceptionally(t); + if (t instanceof KeyNotFoundException) { + promise.completeExceptionally(new BKException.BKNoSuchLedgerExistsException()); + } else { + promise.completeExceptionally(t); + } entries.forEach(LedgerEntry::close); } }); @@ -202,7 +208,7 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr } private void seekToEntry(long nextExpectedId) throws IOException { - Long knownOffset = entryOffsets.getIfPresent(nextExpectedId); + Long knownOffset = entryOffsetsCache.getIfPresent(ledgerId, nextExpectedId); if (knownOffset != null) { inputStream.seek(knownOffset); } else { @@ -256,11 +262,13 @@ public static ReadHandle open(ScheduledExecutorService executor, BlobStore blobStore, String bucket, String key, String indexKey, VersionCheck versionCheck, long ledgerId, int readBufferSize, - LedgerOffloaderStats offloaderStats, String managedLedgerName) - throws IOException { + LedgerOffloaderStats offloaderStats, String managedLedgerName, + OffsetsCache entryOffsetsCache) + throws IOException, BKException.BKNoSuchLedgerExistsException { int retryCount = 3; OffloadIndexBlock index = null; IOException lastException = null; + String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName); // The following retry is used to avoid to some network issue cause read index file failure. // If it can not recovery in the retry, we will throw the exception and the dispatcher will schedule to // next read. @@ -269,7 +277,11 @@ public static ReadHandle open(ScheduledExecutorService executor, while (retryCount-- > 0) { long readIndexStartTime = System.nanoTime(); Blob blob = blobStore.getBlob(bucket, indexKey); - offloaderStats.recordReadOffloadIndexLatency(managedLedgerName, + if (blob == null) { + log.error("{} not found in container {}", indexKey, bucket); + throw new BKException.BKNoSuchLedgerExistsException(); + } + offloaderStats.recordReadOffloadIndexLatency(topicName, System.nanoTime() - readIndexStartTime, TimeUnit.NANOSECONDS); versionCheck.check(indexKey, blob); OffloadIndexBlockBuilder indexBuilder = OffloadIndexBlockBuilder.create(); @@ -292,10 +304,11 @@ public static ReadHandle open(ScheduledExecutorService executor, BackedInputStream inputStream = new BlobStoreBackedInputStreamImpl(blobStore, bucket, key, versionCheck, index.getDataObjectLength(), readBufferSize, offloaderStats, managedLedgerName); - return new BlobStoreBackedReadHandleImpl(ledgerId, index, inputStream, executor); + return new BlobStoreBackedReadHandleImpl(ledgerId, index, inputStream, executor, entryOffsetsCache); } // for testing + @VisibleForTesting State getState() { return this.state; } diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImplV2.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImplV2.java index 2e3d0b08970ca..502f475174cee 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImplV2.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImplV2.java @@ -30,6 +30,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import lombok.val; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.api.LastConfirmedAndEntry; @@ -46,7 +47,9 @@ import org.apache.bookkeeper.mledger.offload.jcloud.OffloadIndexBlockV2Builder; import org.apache.bookkeeper.mledger.offload.jcloud.impl.DataBlockUtils.VersionCheck; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; +import org.apache.pulsar.common.naming.TopicName; import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.KeyNotFoundException; import org.jclouds.blobstore.domain.Blob; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,7 +62,8 @@ public class BlobStoreBackedReadHandleImplV2 implements ReadHandle { private final List inputStreams; private final List dataStreams; private final ExecutorService executor; - private State state = null; + private volatile State state = null; + private final AtomicReference> closeFuture = new AtomicReference<>(); enum State { Opened, @@ -122,7 +126,11 @@ public LedgerMetadata getLedgerMetadata() { @Override public CompletableFuture closeAsync() { - CompletableFuture promise = new CompletableFuture<>(); + if (closeFuture.get() != null || !closeFuture.compareAndSet(null, new CompletableFuture<>())) { + return closeFuture.get(); + } + + CompletableFuture promise = closeFuture.get(); executor.execute(() -> { try { for (OffloadIndexBlockV2 indexBlock : indices) { @@ -142,7 +150,9 @@ public CompletableFuture closeAsync() { @Override public CompletableFuture readAsync(long firstEntry, long lastEntry) { - log.debug("Ledger {}: reading {} - {}", getId(), firstEntry, lastEntry); + if (log.isDebugEnabled()) { + log.debug("Ledger {}: reading {} - {}", getId(), firstEntry, lastEntry); + } CompletableFuture promise = new CompletableFuture<>(); executor.execute(() -> { if (state == State.Closed) { @@ -215,7 +225,11 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr } } } catch (Throwable t) { - promise.completeExceptionally(t); + if (t instanceof KeyNotFoundException) { + promise.completeExceptionally(new BKException.BKNoSuchLedgerExistsException()); + } else { + promise.completeExceptionally(t); + } entries.forEach(LedgerEntry::close); } @@ -294,16 +308,21 @@ public static ReadHandle open(ScheduledExecutorService executor, VersionCheck versionCheck, long ledgerId, int readBufferSize, LedgerOffloaderStats offloaderStats, String managedLedgerName) - throws IOException { + throws IOException, BKException.BKNoSuchLedgerExistsException { List inputStreams = new LinkedList<>(); List indice = new LinkedList<>(); + String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName); for (int i = 0; i < indexKeys.size(); i++) { String indexKey = indexKeys.get(i); String key = keys.get(i); log.debug("open bucket: {} index key: {}", bucket, indexKey); long startTime = System.nanoTime(); Blob blob = blobStore.getBlob(bucket, indexKey); - offloaderStats.recordReadOffloadIndexLatency(managedLedgerName, + if (blob == null) { + log.error("{} not found in container {}", indexKey, bucket); + throw new BKException.BKNoSuchLedgerExistsException(); + } + offloaderStats.recordReadOffloadIndexLatency(topicName, System.nanoTime() - startTime, TimeUnit.NANOSECONDS); log.debug("indexKey blob: {} {}", indexKey, blob); versionCheck.check(indexKey, blob); diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java index 6d69b5edbc3fb..b4ed940c9cdca 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java @@ -51,9 +51,9 @@ import org.apache.bookkeeper.mledger.OffloadedLedgerMetadata; import org.apache.bookkeeper.mledger.OffloadedLedgerMetadataConsumer; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.EntryImpl; import org.apache.bookkeeper.mledger.impl.OffloadSegmentInfoImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.bookkeeper.mledger.offload.jcloud.BlockAwareSegmentInputStream; import org.apache.bookkeeper.mledger.offload.jcloud.OffloadIndexBlock; import org.apache.bookkeeper.mledger.offload.jcloud.OffloadIndexBlock.IndexInputStream; @@ -63,6 +63,8 @@ import org.apache.bookkeeper.mledger.offload.jcloud.provider.BlobStoreLocation; import org.apache.bookkeeper.mledger.offload.jcloud.provider.TieredStorageConfiguration; import org.apache.bookkeeper.mledger.proto.MLDataFormats; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.domain.Blob; @@ -106,9 +108,10 @@ public class BlobStoreManagedLedgerOffloader implements LedgerOffloader { private AtomicLong bufferLength = new AtomicLong(0); private AtomicLong segmentLength = new AtomicLong(0); private final long maxBufferLength; + private final OffsetsCache entryOffsetsCache; private final ConcurrentLinkedQueue offloadBuffer = new ConcurrentLinkedQueue<>(); private CompletableFuture offloadResult; - private volatile PositionImpl lastOfferedPosition = PositionImpl.LATEST; + private volatile Position lastOfferedPosition = PositionFactory.LATEST; private final Duration maxSegmentCloseTime; private final long minSegmentCloseTimeMillis; private final long segmentBeginTimeMillis; @@ -121,13 +124,16 @@ public class BlobStoreManagedLedgerOffloader implements LedgerOffloader { public static BlobStoreManagedLedgerOffloader create(TieredStorageConfiguration config, Map userMetadata, OrderedScheduler scheduler, - LedgerOffloaderStats offloaderStats) throws IOException { + LedgerOffloaderStats offloaderStats, + OffsetsCache entryOffsetsCache) + throws IOException { - return new BlobStoreManagedLedgerOffloader(config, scheduler, userMetadata, offloaderStats); + return new BlobStoreManagedLedgerOffloader(config, scheduler, userMetadata, offloaderStats, entryOffsetsCache); } BlobStoreManagedLedgerOffloader(TieredStorageConfiguration config, OrderedScheduler scheduler, - Map userMetadata, LedgerOffloaderStats offloaderStats) { + Map userMetadata, LedgerOffloaderStats offloaderStats, + OffsetsCache entryOffsetsCache) { this.scheduler = scheduler; this.userMetadata = userMetadata; @@ -138,6 +144,7 @@ public static BlobStoreManagedLedgerOffloader create(TieredStorageConfiguration this.minSegmentCloseTimeMillis = Duration.ofSeconds(config.getMinSegmentTimeInSecond()).toMillis(); //ensure buffer can have enough content to fill a block this.maxBufferLength = Math.max(config.getWriteBufferSizeInBytes(), config.getMinBlockSizeInBytes()); + this.entryOffsetsCache = entryOffsetsCache; this.segmentBeginTimeMillis = System.currentTimeMillis(); if (!Strings.isNullOrEmpty(config.getRegion())) { this.writeLocation = new LocationBuilder() @@ -153,11 +160,17 @@ public static BlobStoreManagedLedgerOffloader create(TieredStorageConfiguration config.getProvider().getDriver(), config.getServiceEndpoint(), config.getBucket(), config.getRegion()); - blobStores.putIfAbsent(config.getBlobStoreLocation(), config.getBlobStore()); this.offloaderStats = offloaderStats; log.info("The ledger offloader was created."); } + private BlobStore getBlobStore(BlobStoreLocation blobStoreLocation) { + return blobStores.computeIfAbsent(blobStoreLocation, location -> { + log.info("Creating blob store for location {}", location); + return config.getBlobStore(); + }); + } + @Override public String getOffloadDriverName() { return config.getDriver(); @@ -176,12 +189,13 @@ public Map getOffloadDriverMetadata() { public CompletableFuture offload(ReadHandle readHandle, UUID uuid, Map extraMetadata) { - final String topicName = extraMetadata.get(MANAGED_LEDGER_NAME); - final BlobStore writeBlobStore = blobStores.get(config.getBlobStoreLocation()); - log.info("offload {} uuid {} extraMetadata {} to {} {}", readHandle.getId(), uuid, extraMetadata, - config.getBlobStoreLocation(), writeBlobStore); + final String managedLedgerName = extraMetadata.get(MANAGED_LEDGER_NAME); + final String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName); CompletableFuture promise = new CompletableFuture<>(); scheduler.chooseThread(readHandle.getId()).execute(() -> { + final BlobStore writeBlobStore = getBlobStore(config.getBlobStoreLocation()); + log.info("offload {} uuid {} extraMetadata {} to {} {}", readHandle.getId(), uuid, extraMetadata, + config.getBlobStoreLocation(), writeBlobStore); if (readHandle.getLength() == 0 || !readHandle.isClosed() || readHandle.getLastAddConfirmed() < 0) { promise.completeExceptionally( new IllegalArgumentException("An empty or open ledger should never be offloaded")); @@ -226,7 +240,7 @@ public CompletableFuture offload(ReadHandle readHandle, .calculateBlockSize(config.getMaxBlockSizeInBytes(), readHandle, startEntry, entryBytesWritten); try (BlockAwareSegmentInputStream blockStream = new BlockAwareSegmentInputStreamImpl( - readHandle, startEntry, blockSize, this.offloaderStats, topicName)) { + readHandle, startEntry, blockSize, this.offloaderStats, managedLedgerName)) { Payload partPayload = Payloads.newInputStreamPayload(blockStream); partPayload.getContentMetadata().setContentLength((long) blockSize); @@ -328,7 +342,7 @@ public CompletableFuture streamingOffload(@NonNull ManagedLedger driverMetadata); log.debug("begin offload with {}:{}", beginLedger, beginEntry); this.offloadResult = new CompletableFuture<>(); - blobStore = blobStores.get(config.getBlobStoreLocation()); + blobStore = getBlobStore(config.getBlobStoreLocation()); streamingIndexBuilder = OffloadIndexBlockV2Builder.create(); streamingDataBlockKey = segmentInfo.uuid.toString(); streamingDataIndexKey = String.format("%s-index", segmentInfo.uuid); @@ -511,7 +525,7 @@ private synchronized boolean closeSegment() { return result; } - private PositionImpl lastOffered() { + private Position lastOffered() { return lastOfferedPosition; } @@ -534,19 +548,20 @@ public CompletableFuture readOffloaded(long ledgerId, UUID uid, BlobStoreLocation bsKey = getBlobStoreLocation(offloadDriverMetadata); String readBucket = bsKey.getBucket(); - BlobStore readBlobstore = blobStores.get(config.getBlobStoreLocation()); CompletableFuture promise = new CompletableFuture<>(); String key = DataBlockUtils.dataBlockOffloadKey(ledgerId, uid); String indexKey = DataBlockUtils.indexBlockOffloadKey(ledgerId, uid); scheduler.chooseThread(ledgerId).execute(() -> { try { + BlobStore readBlobstore = getBlobStore(config.getBlobStoreLocation()); promise.complete(BlobStoreBackedReadHandleImpl.open(scheduler.chooseThread(ledgerId), readBlobstore, readBucket, key, indexKey, DataBlockUtils.VERSION_CHECK, ledgerId, config.getReadBufferSizeInBytes(), - this.offloaderStats, offloadDriverMetadata.get(MANAGED_LEDGER_NAME))); + this.offloaderStats, offloadDriverMetadata.get(MANAGED_LEDGER_NAME), + this.entryOffsetsCache)); } catch (Throwable t) { log.error("Failed readOffloaded: ", t); promise.completeExceptionally(t); @@ -560,7 +575,6 @@ public CompletableFuture readOffloaded(long ledgerId, MLDataFormats. Map offloadDriverMetadata) { BlobStoreLocation bsKey = getBlobStoreLocation(offloadDriverMetadata); String readBucket = bsKey.getBucket(); - BlobStore readBlobstore = blobStores.get(config.getBlobStoreLocation()); CompletableFuture promise = new CompletableFuture<>(); final List offloadSegmentList = ledgerContext.getOffloadSegmentList(); List keys = Lists.newLinkedList(); @@ -575,6 +589,7 @@ public CompletableFuture readOffloaded(long ledgerId, MLDataFormats. scheduler.chooseThread(ledgerId).execute(() -> { try { + BlobStore readBlobstore = getBlobStore(config.getBlobStoreLocation()); promise.complete(BlobStoreBackedReadHandleImplV2.open(scheduler.chooseThread(ledgerId), readBlobstore, readBucket, keys, indexKeys, @@ -594,11 +609,11 @@ public CompletableFuture deleteOffloaded(long ledgerId, UUID uid, Map offloadDriverMetadata) { BlobStoreLocation bsKey = getBlobStoreLocation(offloadDriverMetadata); String readBucket = bsKey.getBucket(offloadDriverMetadata); - BlobStore readBlobstore = blobStores.get(config.getBlobStoreLocation()); CompletableFuture promise = new CompletableFuture<>(); scheduler.chooseThread(ledgerId).execute(() -> { try { + BlobStore readBlobstore = getBlobStore(config.getBlobStoreLocation()); readBlobstore.removeBlobs(readBucket, ImmutableList.of(DataBlockUtils.dataBlockOffloadKey(ledgerId, uid), DataBlockUtils.indexBlockOffloadKey(ledgerId, uid))); @@ -611,7 +626,8 @@ public CompletableFuture deleteOffloaded(long ledgerId, UUID uid, return promise.whenComplete((__, t) -> { if (null != this.ml) { - this.offloaderStats.recordDeleteOffloadOps(this.ml.getName(), t == null); + this.offloaderStats.recordDeleteOffloadOps( + TopicName.fromPersistenceNamingEncoding(this.ml.getName()), t == null); } }); } @@ -620,11 +636,11 @@ public CompletableFuture deleteOffloaded(long ledgerId, UUID uid, public CompletableFuture deleteOffloaded(UUID uid, Map offloadDriverMetadata) { BlobStoreLocation bsKey = getBlobStoreLocation(offloadDriverMetadata); String readBucket = bsKey.getBucket(offloadDriverMetadata); - BlobStore readBlobstore = blobStores.get(config.getBlobStoreLocation()); CompletableFuture promise = new CompletableFuture<>(); scheduler.execute(() -> { try { + BlobStore readBlobstore = getBlobStore(config.getBlobStoreLocation()); readBlobstore.removeBlobs(readBucket, ImmutableList.of(uid.toString(), DataBlockUtils.indexBlockOffloadKey(uid))); @@ -636,11 +652,12 @@ public CompletableFuture deleteOffloaded(UUID uid, Map off }); return promise.whenComplete((__, t) -> - this.offloaderStats.recordDeleteOffloadOps(this.ml.getName(), t == null)); + this.offloaderStats.recordDeleteOffloadOps( + TopicName.fromPersistenceNamingEncoding(this.ml.getName()), t == null)); } @Override - public OffloadPoliciesImpl getOffloadPolicies() { + public OffloadPolicies getOffloadPolicies() { Properties properties = new Properties(); properties.putAll(config.getConfigProperties()); return OffloadPoliciesImpl.create(properties); @@ -663,7 +680,7 @@ public void scanLedgers(OffloadedLedgerMetadataConsumer consumer, Map entriesByteBuf = null; private LedgerOffloaderStats offloaderStats; + private String managedLedgerName; private String topicName; private int currentOffset = 0; private final AtomicBoolean close = new AtomicBoolean(false); @@ -91,7 +93,8 @@ public BlockAwareSegmentInputStreamImpl(ReadHandle ledger, long startEntryId, in LedgerOffloaderStats offloaderStats, String ledgerName) { this(ledger, startEntryId, blockSize); this.offloaderStats = offloaderStats; - this.topicName = ledgerName; + this.managedLedgerName = ledgerName; + this.topicName = TopicName.fromPersistenceNamingEncoding(ledgerName); } private ByteBuf readEntries(int len) throws IOException { @@ -183,7 +186,7 @@ private List readNextEntriesFromLedger(long start, long maxNumberEntrie log.debug("read ledger entries. start: {}, end: {} cost {}", start, end, TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - startTime)); } - if (offloaderStats != null && topicName != null) { + if (offloaderStats != null && managedLedgerName != null) { offloaderStats.recordReadLedgerLatency(topicName, System.nanoTime() - startTime, TimeUnit.NANOSECONDS); } diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffsetsCache.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffsetsCache.java new file mode 100644 index 0000000000000..6651b199e4e60 --- /dev/null +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffsetsCache.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.offload.jcloud.impl; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +public class OffsetsCache implements AutoCloseable { + private static final int CACHE_TTL_SECONDS = + Integer.getInteger("pulsar.jclouds.readhandleimpl.offsetsscache.ttl.seconds", 5 * 60); + // limit the cache size to avoid OOM + // 1 million entries consumes about 60MB of heap space + private static final int CACHE_MAX_SIZE = + Integer.getInteger("pulsar.jclouds.readhandleimpl.offsetsscache.max.size", 1_000_000); + private final ScheduledExecutorService cacheEvictionExecutor; + + record Key(long ledgerId, long entryId) { + + } + + private final Cache entryOffsetsCache; + + public OffsetsCache() { + if (CACHE_MAX_SIZE > 0) { + entryOffsetsCache = CacheBuilder + .newBuilder() + .expireAfterAccess(CACHE_TTL_SECONDS, TimeUnit.SECONDS) + .maximumSize(CACHE_MAX_SIZE) + .build(); + cacheEvictionExecutor = + Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat("jcloud-offsets-cache-eviction").build()); + int period = Math.max(CACHE_TTL_SECONDS / 2, 1); + cacheEvictionExecutor.scheduleAtFixedRate(() -> { + entryOffsetsCache.cleanUp(); + }, period, period, TimeUnit.SECONDS); + } else { + cacheEvictionExecutor = null; + entryOffsetsCache = null; + } + } + + public void put(long ledgerId, long entryId, long currentPosition) { + if (entryOffsetsCache != null) { + entryOffsetsCache.put(new Key(ledgerId, entryId), currentPosition); + } + } + + public Long getIfPresent(long ledgerId, long entryId) { + return entryOffsetsCache != null ? entryOffsetsCache.getIfPresent(new Key(ledgerId, entryId)) : null; + } + + public void clear() { + if (entryOffsetsCache != null) { + entryOffsetsCache.invalidateAll(); + } + } + + @Override + public void close() { + if (cacheEvictionExecutor != null) { + cacheEvictionExecutor.shutdownNow(); + } + } +} diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/BlobStoreBackedInputStreamTest.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/BlobStoreBackedInputStreamTest.java index 775310925a1a3..3e5c4b609dfec 100644 --- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/BlobStoreBackedInputStreamTest.java +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/BlobStoreBackedInputStreamTest.java @@ -32,6 +32,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.offload.jcloud.impl.BlobStoreBackedInputStreamImpl; import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.KeyNotFoundException; import org.jclouds.blobstore.domain.Blob; import org.jclouds.io.Payload; import org.jclouds.io.Payloads; @@ -142,8 +143,8 @@ public void testReadingFullObjectByBytes() throws Exception { assertStreamsMatchByBytes(toTest, toCompare); } - @Test(expectedExceptions = IOException.class) - public void testErrorOnRead() throws Exception { + @Test(expectedExceptions = KeyNotFoundException.class) + public void testNotFoundOnRead() throws Exception { BackedInputStream toTest = new BlobStoreBackedInputStreamImpl(blobStore, BUCKET, "doesn't exist", (key, md) -> {}, 1234, 1000); diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamTest.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamTest.java new file mode 100644 index 0000000000000..951180e4e18c8 --- /dev/null +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.offload.jcloud.impl; + +import static org.testng.Assert.assertEquals; + +import java.io.IOException; +import java.io.InputStream; +import org.apache.bookkeeper.mledger.offload.jcloud.BlobStoreTestBase; +import org.testng.annotations.Test; + +public class BlobStoreBackedInputStreamTest extends BlobStoreTestBase { + + @Test + public void testFillBuffer() throws Exception { + BlobStoreBackedInputStreamImpl bis = new BlobStoreBackedInputStreamImpl( + blobStore, BUCKET, "testFillBuffer", (k, md) -> { + }, 2048, 512); + + InputStream is = new InputStream() { + int count = 10; + + @Override + public int read() throws IOException { + if (count-- > 0) { + return 1; + } else { + return -1; + } + } + }; + bis.fillBuffer(is, 20); + assertEquals(bis.getBuffer().readableBytes(), 10); + } +} diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderBase.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderBase.java index 89d9021d36d7d..75faf098b409b 100644 --- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderBase.java +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderBase.java @@ -33,6 +33,7 @@ import org.jclouds.blobstore.BlobStore; import org.jclouds.domain.Credentials; import org.testng.Assert; +import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; public abstract class BlobStoreManagedLedgerOffloaderBase { @@ -46,6 +47,7 @@ public abstract class BlobStoreManagedLedgerOffloaderBase { protected final JCloudBlobStoreProvider provider; protected TieredStorageConfiguration config; protected BlobStore blobStore = null; + protected final OffsetsCache entryOffsetsCache = new OffsetsCache(); protected BlobStoreManagedLedgerOffloaderBase() throws Exception { scheduler = OrderedScheduler.newSchedulerBuilder().numThreads(5).name("offloader").build(); @@ -56,6 +58,13 @@ protected BlobStoreManagedLedgerOffloaderBase() throws Exception { @AfterMethod(alwaysRun = true) public void cleanupMockBookKeeper() { bk.getLedgerMap().clear(); + entryOffsetsCache.clear(); + } + + @AfterClass(alwaysRun = true) + public void cleanup() throws Exception { + entryOffsetsCache.close(); + scheduler.shutdownNow(); } protected static MockManagedLedger createMockManagedLedger() { diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderStreamingTest.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderStreamingTest.java index 9056281a308f2..e706e4254cb11 100644 --- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderStreamingTest.java +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderStreamingTest.java @@ -18,16 +18,19 @@ */ package org.apache.bookkeeper.mledger.offload.jcloud.impl; +import static org.apache.bookkeeper.client.api.BKException.Code.NoSuchLedgerExistsException; import static org.mockito.AdditionalAnswers.delegatesTo; import static org.mockito.Mockito.mock; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.fail; import java.io.IOException; import java.util.HashMap; import java.util.LinkedList; import java.util.Map; import java.util.Random; import java.util.UUID; +import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.api.LedgerEntries; import org.apache.bookkeeper.client.api.LedgerEntry; import org.apache.bookkeeper.client.api.ReadHandle; @@ -79,7 +82,7 @@ private BlobStoreManagedLedgerOffloader getOffloader(String bucket, Map(), scheduler, this.offloaderStats); + .create(mockedConfig, new HashMap(), scheduler, this.offloaderStats, entryOffsetsCache); return offloader; } @@ -88,7 +91,7 @@ private BlobStoreManagedLedgerOffloader getOffloader(String bucket, BlobStore mo mockedConfig = mock(TieredStorageConfiguration.class, delegatesTo(getConfiguration(bucket, additionalConfig))); Mockito.doReturn(mockedBlobStore).when(mockedConfig).getBlobStore(); BlobStoreManagedLedgerOffloader offloader = BlobStoreManagedLedgerOffloader - .create(mockedConfig, new HashMap(), scheduler, this.offloaderStats); + .create(mockedConfig, new HashMap(), scheduler, this.offloaderStats, entryOffsetsCache); return offloader; } @@ -445,4 +448,55 @@ public void testInvalidEntryIds() throws Exception { } catch (Exception e) { } } + + @Test + public void testReadNotExistLedger() throws Exception { + LedgerOffloader offloader = getOffloader(new HashMap() {{ + put(TieredStorageConfiguration.MAX_OFFLOAD_SEGMENT_SIZE_IN_BYTES, "1000"); + put(config.getKeys(TieredStorageConfiguration.METADATA_FIELD_MAX_BLOCK_SIZE).get(0), "5242880"); + put(TieredStorageConfiguration.MAX_OFFLOAD_SEGMENT_ROLLOVER_TIME_SEC, "600"); + }}); + ManagedLedger ml = createMockManagedLedger(); + UUID uuid = UUID.randomUUID(); + long beginLedger = 0; + long beginEntry = 0; + + Map driverMeta = new HashMap() {{ + put(TieredStorageConfiguration.METADATA_FIELD_BUCKET, BUCKET); + }}; + OffloadHandle offloadHandle = offloader + .streamingOffload(ml, uuid, beginLedger, beginEntry, driverMeta).get(); + + // Segment should closed because size in bytes full + final LinkedList entries = new LinkedList<>(); + for (int i = 0; i < 10; i++) { + final byte[] data = new byte[100]; + random.nextBytes(data); + final EntryImpl entry = EntryImpl.create(0, i, data); + offloadHandle.offerEntry(entry); + entries.add(entry); + } + + final LedgerOffloader.OffloadResult offloadResult = offloadHandle.getOffloadResultAsync().get(); + assertEquals(offloadResult.endLedger, 0); + assertEquals(offloadResult.endEntry, 9); + final OffloadContext.Builder contextBuilder = OffloadContext.newBuilder(); + contextBuilder.addOffloadSegment( + MLDataFormats.OffloadSegment.newBuilder() + .setUidLsb(uuid.getLeastSignificantBits()) + .setUidMsb(uuid.getMostSignificantBits()) + .setComplete(true).setEndEntryId(9).build()); + + final ReadHandle readHandle = offloader.readOffloaded(0, contextBuilder.build(), driverMeta).get(); + + // delete blob(ledger) + blobStore.removeBlob(BUCKET, uuid.toString()); + + try { + readHandle.read(0, 9); + fail("Should be read fail"); + } catch (BKException e) { + assertEquals(e.getCode(), NoSuchLedgerExistsException); + } + } } diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java index ef0bea29e35c3..bf6ede896ab28 100644 --- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java @@ -18,14 +18,16 @@ */ package org.apache.bookkeeper.mledger.offload.jcloud.impl; +import static org.apache.bookkeeper.client.api.BKException.Code.NoSuchLedgerExistsException; import static org.mockito.AdditionalAnswers.delegatesTo; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; -import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -38,6 +40,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.api.LedgerEntries; import org.apache.bookkeeper.client.api.LedgerEntry; @@ -45,22 +48,25 @@ import org.apache.bookkeeper.mledger.LedgerOffloader; import org.apache.bookkeeper.mledger.LedgerOffloaderStats; import org.apache.bookkeeper.mledger.ManagedLedgerException; -import org.apache.bookkeeper.mledger.impl.LedgerOffloaderStatsImpl; import org.apache.bookkeeper.mledger.OffloadedLedgerMetadata; +import org.apache.bookkeeper.mledger.impl.LedgerOffloaderStatsImpl; import org.apache.bookkeeper.mledger.offload.jcloud.provider.JCloudBlobStoreProvider; import org.apache.bookkeeper.mledger.offload.jcloud.provider.TieredStorageConfiguration; +import org.apache.pulsar.common.naming.TopicName; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.options.CopyOptions; import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; +import org.testng.annotations.AfterClass; import org.testng.annotations.Test; import org.testng.collections.Maps; public class BlobStoreManagedLedgerOffloaderTest extends BlobStoreManagedLedgerOffloaderBase { private static final Logger log = LoggerFactory.getLogger(BlobStoreManagedLedgerOffloaderTest.class); + private final ScheduledExecutorService scheduledExecutorService; private TieredStorageConfiguration mockedConfig; private final LedgerOffloaderStats offloaderStats; @@ -71,13 +77,20 @@ public class BlobStoreManagedLedgerOffloaderTest extends BlobStoreManagedLedgerO assertNotNull(provider); provider.validate(config); blobStore = provider.getBlobStore(config); - this.offloaderStats = LedgerOffloaderStats.create(true, true, Executors.newScheduledThreadPool(1), 60); + scheduledExecutorService = Executors.newScheduledThreadPool(1); + this.offloaderStats = LedgerOffloaderStats.create(true, true, scheduledExecutorService, 60); + } + + @AfterClass(alwaysRun = true) + protected void cleanupInstance() throws Exception { + offloaderStats.close(); + scheduledExecutorService.shutdownNow(); } private BlobStoreManagedLedgerOffloader getOffloader() throws IOException { return getOffloader(BUCKET); } - + private BlobStoreManagedLedgerOffloader getOffloader(BlobStore mockedBlobStore) throws IOException { return getOffloader(BUCKET, mockedBlobStore); } @@ -85,14 +98,16 @@ private BlobStoreManagedLedgerOffloader getOffloader(BlobStore mockedBlobStore) private BlobStoreManagedLedgerOffloader getOffloader(String bucket) throws IOException { mockedConfig = mock(TieredStorageConfiguration.class, delegatesTo(getConfiguration(bucket))); Mockito.doReturn(blobStore).when(mockedConfig).getBlobStore(); // Use the REAL blobStore - BlobStoreManagedLedgerOffloader offloader = BlobStoreManagedLedgerOffloader.create(mockedConfig, new HashMap(), scheduler, this.offloaderStats); + BlobStoreManagedLedgerOffloader offloader = BlobStoreManagedLedgerOffloader.create(mockedConfig, new HashMap(), scheduler, this.offloaderStats, + entryOffsetsCache); return offloader; } - + private BlobStoreManagedLedgerOffloader getOffloader(String bucket, BlobStore mockedBlobStore) throws IOException { mockedConfig = mock(TieredStorageConfiguration.class, delegatesTo(getConfiguration(bucket))); - Mockito.doReturn(mockedBlobStore).when(mockedConfig).getBlobStore(); - BlobStoreManagedLedgerOffloader offloader = BlobStoreManagedLedgerOffloader.create(mockedConfig, new HashMap(), scheduler, this.offloaderStats); + Mockito.doReturn(mockedBlobStore).when(mockedConfig).getBlobStore(); + BlobStoreManagedLedgerOffloader offloader = BlobStoreManagedLedgerOffloader.create(mockedConfig, new HashMap(), scheduler, this.offloaderStats, + entryOffsetsCache); return offloader; } @@ -172,10 +187,10 @@ public void testOffloadAndReadMetrics() throws Exception { LedgerOffloader offloader = getOffloader(); UUID uuid = UUID.randomUUID(); - - String topic = "test"; + String managedLegerName = "public/default/persistent/testOffload"; + String topic = TopicName.fromPersistenceNamingEncoding(managedLegerName); Map extraMap = new HashMap<>(); - extraMap.put("ManagedLedgerName", topic); + extraMap.put("ManagedLedgerName", managedLegerName); offloader.offload(toWrite, uuid, extraMap).get(); LedgerOffloaderStatsImpl offloaderStats = (LedgerOffloaderStatsImpl) this.offloaderStats; @@ -187,7 +202,7 @@ public void testOffloadAndReadMetrics() throws Exception { Map map = new HashMap<>(); map.putAll(offloader.getOffloadDriverMetadata()); - map.put("ManagedLedgerName", topic); + map.put("ManagedLedgerName", managedLegerName); ReadHandle toTest = offloader.readOffloaded(toWrite.getId(), uuid, map).get(); LedgerEntries toTestEntries = toTest.read(0, toTest.getLastAddConfirmed()); Iterator toTestIter = toTestEntries.iterator(); @@ -208,9 +223,9 @@ public void testOffloadFailInitDataBlockUpload() throws Exception { String failureString = "fail InitDataBlockUpload"; // mock throw exception when initiateMultipartUpload - try { + try { BlobStore spiedBlobStore = mock(BlobStore.class, delegatesTo(blobStore)); - + Mockito .doThrow(new RuntimeException(failureString)) .when(spiedBlobStore).initiateMultipartUpload(any(), any(), any()); @@ -234,7 +249,7 @@ public void testOffloadFailDataBlockPartUpload() throws Exception { // mock throw exception when uploadPart try { - + BlobStore spiedBlobStore = mock(BlobStore.class, delegatesTo(blobStore)); Mockito .doThrow(new RuntimeException(failureString)) @@ -268,7 +283,7 @@ public void testOffloadFailDataBlockUploadComplete() throws Exception { .when(spiedBlobStore).abortMultipartUpload(any()); BlobStoreManagedLedgerOffloader offloader = getOffloader(spiedBlobStore); - offloader.offload(readHandle, uuid, new HashMap<>()).get(); + offloader.offload(readHandle, uuid, new HashMap<>()).get(); Assert.fail("Should throw exception for when completeMultipartUpload"); } catch (Exception e) { @@ -287,7 +302,7 @@ public void testOffloadFailPutIndexBlock() throws Exception { String failureString = "fail putObject"; // mock throw exception when putObject - try { + try { BlobStore spiedBlobStore = mock(BlobStore.class, delegatesTo(blobStore)); Mockito .doThrow(new RuntimeException(failureString)) @@ -379,7 +394,7 @@ public void testOffloadReadInvalidEntryIds() throws Exception { public void testDeleteOffloaded() throws Exception { ReadHandle readHandle = buildReadHandle(DEFAULT_BLOCK_SIZE, 1); UUID uuid = UUID.randomUUID(); - + BlobStoreManagedLedgerOffloader offloader = getOffloader(); // verify object exist after offload @@ -398,13 +413,13 @@ public void testDeleteOffloadedFail() throws Exception { String failureString = "fail deleteOffloaded"; ReadHandle readHandle = buildReadHandle(DEFAULT_BLOCK_SIZE, 1); UUID uuid = UUID.randomUUID(); - + BlobStore spiedBlobStore = mock(BlobStore.class, delegatesTo(blobStore)); Mockito .doThrow(new RuntimeException(failureString)) .when(spiedBlobStore).removeBlobs(any(), any()); - + BlobStoreManagedLedgerOffloader offloader = getOffloader(spiedBlobStore); try { @@ -590,4 +605,25 @@ public void testReadWithAClosedLedgerHandler() throws Exception { throw e; } } + + @Test + public void testReadNotExistLedger() throws Exception { + ReadHandle toWrite = buildReadHandle(DEFAULT_BLOCK_SIZE, 3); + LedgerOffloader offloader = getOffloader(); + + UUID uuid = UUID.randomUUID(); + offloader.offload(toWrite, uuid, new HashMap<>()).get(); + ReadHandle offloadRead = offloader.readOffloaded(toWrite.getId(), uuid, Collections.emptyMap()).get(); + assertEquals(offloadRead.getLastAddConfirmed(), toWrite.getLastAddConfirmed()); + + // delete blob(ledger) + blobStore.removeBlob(BUCKET, DataBlockUtils.dataBlockOffloadKey(toWrite.getId(), uuid)); + + try { + offloadRead.read(0, offloadRead.getLastAddConfirmed()); + fail("Should be read fail"); + } catch (BKException e) { + assertEquals(e.getCode(), NoSuchLedgerExistsException); + } + } } diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/MockManagedLedger.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/MockManagedLedger.java index 774b0143f956e..60dcbb8b3acd8 100644 --- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/MockManagedLedger.java +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/MockManagedLedger.java @@ -18,8 +18,10 @@ */ package org.apache.bookkeeper.mledger.offload.jcloud.impl; +import com.google.common.collect.Range; import io.netty.buffer.ByteBuf; import java.util.Map; +import java.util.NavigableMap; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.function.Predicate; @@ -32,6 +34,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerMXBean; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionBound; import org.apache.bookkeeper.mledger.intercept.ManagedLedgerInterceptor; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo; import org.apache.pulsar.common.api.proto.CommandSubscribe; @@ -177,6 +180,11 @@ public long getNumberOfEntries() { return 0; } + @Override + public long getNumberOfEntries(Range range) { + return 0; + } + @Override public long getNumberOfActiveEntries() { return 0; @@ -274,7 +282,7 @@ public boolean isTerminated() { @Override public ManagedLedgerConfig getConfig() { - return null; + return new ManagedLedgerConfig(); } @Override @@ -372,8 +380,8 @@ public CompletableFuture getManagedLedgerInternalSta } @Override - public void checkInactiveLedgerAndRollOver() { - + public boolean checkInactiveLedgerAndRollOver() { + return false; } @Override @@ -381,6 +389,51 @@ public void checkCursorsToCacheEntries() { // no-op } + @Override + public void asyncReadEntry(Position position, AsyncCallbacks.ReadEntryCallback callback, Object ctx) { + + } + + @Override + public NavigableMap getLedgersInfo() { + return null; + } + + @Override + public Position getNextValidPosition(Position position) { + return null; + } + + @Override + public Position getPreviousPosition(Position position) { + return null; + } + + @Override + public long getEstimatedBacklogSize(Position position) { + return 0; + } + + @Override + public Position getPositionAfterN(Position startPosition, long n, PositionBound startRange) { + return null; + } + + @Override + public int getPendingAddEntriesCount() { + return 0; + } + + @Override + public long getCacheSize() { + return 0; + } + + @Override + public Position getFirstPosition() { + return null; + } + @Override public CompletableFuture asyncMigrate() { // no-op diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffsetsCacheTest.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffsetsCacheTest.java new file mode 100644 index 0000000000000..86a72c7b5547e --- /dev/null +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffsetsCacheTest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.offload.jcloud.impl; + +import lombok.extern.slf4j.Slf4j; +import org.testng.annotations.Test; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; + +@Slf4j +public class OffsetsCacheTest { + + @Test + public void testCache() throws Exception { + System.setProperty("pulsar.jclouds.readhandleimpl.offsetsscache.ttl.seconds", "1"); + OffsetsCache offsetsCache = new OffsetsCache(); + assertNull(offsetsCache.getIfPresent(1, 2)); + offsetsCache.put(1, 1, 1); + assertEquals(offsetsCache.getIfPresent(1, 1), 1); + offsetsCache.clear(); + assertNull(offsetsCache.getIfPresent(1, 1)); + // test ttl + offsetsCache.put(1, 2, 2); + assertEquals(offsetsCache.getIfPresent(1, 2), 2); + Thread.sleep(1500); + assertNull(offsetsCache.getIfPresent(1, 2)); + offsetsCache.close(); + } +} diff --git a/tiered-storage/jcloud/src/test/resources/log4j2-test.yml b/tiered-storage/jcloud/src/test/resources/log4j2-test.yml index cab9dd0dd5625..f5ee5c9a53dd7 100644 --- a/tiered-storage/jcloud/src/test/resources/log4j2-test.yml +++ b/tiered-storage/jcloud/src/test/resources/log4j2-test.yml @@ -33,12 +33,12 @@ Configuration: name: STDOUT target: SYSTEM_OUT PatternLayout: - Pattern: "%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t:%C@%L] %-5level %logger{36} - %msg%n" + Pattern: "%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n" File: name: File fileName: ${filename} PatternLayout: - Pattern: "%d %p %C{1.} [%t] %m%n" + Pattern: "%d %p %c{1.} [%t] %m%n" Filters: ThresholdFilter: level: error diff --git a/tiered-storage/pom.xml b/tiered-storage/pom.xml index d6d2a28e31324..cb64e70542708 100644 --- a/tiered-storage/pom.xml +++ b/tiered-storage/pom.xml @@ -25,8 +25,7 @@ org.apache.pulsar pulsar - 3.1.0-SNAPSHOT - .. + 4.0.0-SNAPSHOT tiered-storage-parent diff --git a/wiki/proposals/PIP-template.md b/wiki/proposals/PIP-template.md deleted file mode 100644 index 752878a56b519..0000000000000 --- a/wiki/proposals/PIP-template.md +++ /dev/null @@ -1,91 +0,0 @@ -# Pulsar Improvement Proposal Template - -* **Status**: "one of ['Under Discussion', 'Accepted', 'Adopted', 'Rejected']" -* **Author**: (Names) -* **Pull Request**: (Link to the main pull request to resolve this PIP) -* **Mailing List discussion**: (Link to the mailing list discussion) -* **Release**: (Which release include this PIP) - -### Motivation - -_Describe the problems you are trying to solve_ - -### Public Interfaces - -_Briefly list any new interfaces that will be introduced as part of this proposal or any existing interfaces that will be removed or changed. The purpose of this section is to concisely call out the public contract that will come along with this feature._ - -A public interface is any change to the following: - -- Data format, Metadata format -- The wire protocol and API behavior -- Any class in the public packages -- Monitoring -- Command-line tools and arguments -- Configuration settings -- Anything else that will likely break existing users in some way when they upgrade - -### Proposed Changes - -_Describe the new thing you want to do in appropriate detail. This may be fairly extensive and have large subsections of its own. Or it may be a few sentences. Use judgment based on the scope of the change._ - -### Compatibility, Deprecation, and Migration Plan - -- What impact (if any) will there be on existing users? -- If we are changing behavior how will we phase out the older behavior? -- If we need special migration tools, describe them here. -- When will we remove the existing behavior? - -### Test Plan - -_Describe in few sentences how the BP will be tested. We are mostly interested in system tests (since unit-tests are specific to implementation details). How will we know that the implementation works as expected? How will we know nothing broke?_ - -### Rejected Alternatives - -_If there are alternative ways of accomplishing the same thing, what were they? The purpose of this section is to motivate why the design is the way it is and not some other way._ - ---- - -``` -* **Status**: "one of ['Under Discussion', 'Accepted', 'Adopted', 'Rejected']" -* **Author**: (Names) -* **Pull Request**: (Link to the main pull request to resolve this PIP) -* **Mailing List discussion**: (Link to the mailing list discussion) -* **Release**: (Which release include this PIP) - -### Motivation - -_Describe the problems you are trying to solve_ - -### Public Interfaces - -_Briefly list any new interfaces that will be introduced as part of this proposal or any existing interfaces that will be removed or changed. The purpose of this section is to concisely call out the public contract that will come along with this feature._ - -A public interface is any change to the following: - -- Data format, Metadata format -- The wire protocol and API behavior -- Any class in the public packages -- Monitoring -- Command-line tools and arguments -- Configuration settings -- Anything else that will likely break existing users in some way when they upgrade - -### Proposed Changes - -_Describe the new thing you want to do in appropriate detail. This may be fairly extensive and have large subsections of its own. Or it may be a few sentences. Use judgment based on the scope of the change._ - -### Compatibility, Deprecation, and Migration Plan - -- What impact (if any) will there be on existing users? -- If we are changing behavior how will we phase out the older behavior? -- If we need special migration tools, describe them here. -- When will we remove the existing behavior? - -### Test Plan - -_Describe in few sentences how the BP will be tested. We are mostly interested in system tests (since unit-tests are specific to implementation details). How will we know that the implementation works as expected? How will we know nothing broke?_ - -### Rejected Alternatives - -_If there are alternative ways of accomplishing the same thing, what were they? The purpose of this section is to motivate why the design is the way it is and not some other way._ -``` \ No newline at end of file diff --git a/wiki/proposals/PIP.md b/wiki/proposals/PIP.md deleted file mode 100644 index fc68905726eb8..0000000000000 --- a/wiki/proposals/PIP.md +++ /dev/null @@ -1,120 +0,0 @@ -# Pulsar Improvement Proposal (PIP) - -## What is a PIP? - -The PIP is a "Pulsar Improvement Proposal" and it's the mechanism used to -propose changes to the Apache Pulsar codebases. - -The changes might be in terms of new features, large code refactoring, changes -to APIs. - -In practical terms, the PIP defines a process in which developers can submit -a design doc, receive feedback and get the "go ahead" to execute. - -## What is the goal of a PIP? - -There are several goals for the PIP process: - -1. Ensure community technical discussion of major changes to the Apache Pulsar - codebase - -2. Provide clear and thorough design documentation of the proposed changes. - Make sure every Pulsar developer will have enough context to effectively - perform a code review of the Pull Requests. - -3. Use the PIP document to serve as the starting base on which to create the - documentation for the new feature. - -4. Have greater scrutiny to changes are affecting the public APIs to reduce - chances of introducing breaking changes or APIs that are not expressing - an ideal semantic. - - -It is not a goal for PIP to add undue process or slow-down the development. - -## When is a PIP required? - -* Any new feature for Pulsar brokers or client -* Any change to the public APIs (Client APIs, REST APIs, Plugin APIs) -* Any change to the wire protocol APIs -* Any change to the API of Pulsar CLI tools (eg: new options) -* Any change to the semantic of existing functionality, even when current - behavior is incorrect. -* Any large code change that will touch multiple components -* Any changes to the metrics (metrics endpoint, topic stats, topics internal stats, broker stats, etc.) - -## When is a PIP *not* required? - -* Bug-fixes -* Simple enhancements that won't affect the APIs or the semantic -* Documentation changes -* Website changes -* Build scripts changes (except: a complete rewrite) - -## Who can create a PIP? - -Any person willing to contribute to the Apache Pulsar project is welcome to -create a PIP. - -## How does the PIP process work? - -A PIP proposal can be in these states: -1. **DRAFT**: (Optional) This might be used for contributors to collaborate and - to seek feedback on an incomplete version of the proposal. - -2. **DISCUSSION**: The proposal has been submitted to the community for - feedback and approval. - -3. **ACCEPTED**: The proposal has been accepted by the Pulsar project. - -4. **REJECTED**: The proposal has not been accepted by the Pulsar project. - -5. **IMPLEMENTED**: The implementation of the proposed changes have been - completed and everything has been merged. - -5. **RELEASED**: The proposed changes have been included in an official - Apache Pulsar release. - -The process works in the following way: - -1. The author(s) of the proposal will create a GitHub issue ticket choosing the - template for PIP proposals. The issue title should be "PIP-xxx: title", where - the "xxx" number should be chosen to be the next number from the existing PIP - issues, listed [here](https://github.com/apache/pulsar/issues?q=is%3Aissue+label%3APIP+) -2. The author(s) will send a note to the dev@pulsar.apache.org mailing list - to start the discussion, using subject prefix `[DISCUSS] PIP-xxx: {PIP TITLE}`. The discussion - need to happen in the mailing list. Please avoid discussing it using - GitHub comments in the PIP GitHub issue, as it creates two tracks - of feedback. -3. Based on the discussion and feedback, some changes might be applied by - authors to the text of the proposal. -4. Once some consensus is reached, there will be a vote to formally approve - the proposal. - The vote will be held on the dev@pulsar.apache.org mailing list, by - sending a message using subject `[VOTE] PIP-xxx: {PIP TITLE}". - Everyone is welcome to vote on the proposal, though only the the vote of the PMC - members will be considered binding. - It is required to have a lazy majority of at least 3 binding +1s votes. - The vote should stay open for at least 48 hours. -5. When the vote is closed, if the outcome is positive, the state of the - proposal is updated, and the Pull Requests associated with this proposal can - start to get merged into the master branch. - -All the Pull Requests that are created, should always reference the -PIP-XXX in the -commit log message and the PR title. - -## Labels of a PIP - -In addition to its current state, the GitHub issue for the PIP will also be -tagged with other labels. - -Some examples: -* Execution status: In progress, Completed, Need Help, ... -* Targeted Pulsar release: 2.9, 2.10, ... - - -## Template for a PIP design doc - -Read [the template file](/.github/ISSUE_TEMPLATE/pip.md). -